diff options
Diffstat (limited to 'contrib')
107 files changed, 16722 insertions, 0 deletions
diff --git a/contrib/README.md b/contrib/README.md new file mode 100644 index 0000000000..ae1372e95d --- /dev/null +++ b/contrib/README.md @@ -0,0 +1,42 @@ +Repository Tools +--------------------- + +### [Developer tools](/contrib/devtools) ### +Specific tools for developers working on this repository. +Additional tools, including the `github-merge.py` script, are available in the [maintainer-tools](https://github.com/bitcoin-core/bitcoin-maintainer-tools) repository. + +### [Verify-Commits](/contrib/verify-commits) ### +Tool to verify that every merge commit was signed by a developer using the `github-merge.py` script. + +### [Linearize](/contrib/linearize) ### +Construct a linear, no-fork, best version of the blockchain. + +### [Qos](/contrib/qos) ### + +A Linux bash script that will set up traffic control (tc) to limit the outgoing bandwidth for connections to the Bitcoin network. This means one can have an always-on bitcoind instance running, and another local bitcoind/bitcoin-qt instance which connects to this node and receives blocks from it. + +### [Seeds](/contrib/seeds) ### +Utility to generate the pnSeed[] array that is compiled into the client. + +Build Tools and Keys +--------------------- + +### Packaging ### +The [Debian](/contrib/debian) subfolder contains the copyright file. + +All other packaging related files can be found in the [bitcoin-core/packaging](https://github.com/bitcoin-core/packaging) repository. + +### [Builder keys](/contrib/builder-keys) +PGP keys used for signing Bitcoin Core [release](/doc/release-process.md) results. + +### [MacDeploy](/contrib/macdeploy) ### +Scripts and notes for Mac builds. + +Test and Verify Tools +--------------------- + +### [TestGen](/contrib/testgen) ### +Utilities to generate test vectors for the data-driven Bitcoin tests. + +### [Verify Binaries](/contrib/verifybinaries) ### +This script attempts to download and verify the signature file SHA256SUMS.asc from bitcoin.org. diff --git a/contrib/bitcoin-cli.bash-completion b/contrib/bitcoin-cli.bash-completion new file mode 100644 index 0000000000..ddea58a05c --- /dev/null +++ b/contrib/bitcoin-cli.bash-completion @@ -0,0 +1,141 @@ +# bash programmable completion for bitcoin-cli(1) +# Copyright (c) 2012-2019 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +# call $bitcoin-cli for RPC +_bitcoin_rpc() { + # determine already specified args necessary for RPC + local rpcargs=() + for i in ${COMP_LINE}; do + case "$i" in + -conf=*|-datadir=*|-regtest|-rpc*|-testnet) + rpcargs=( "${rpcargs[@]}" "$i" ) + ;; + esac + done + $bitcoin_cli "${rpcargs[@]}" "$@" +} + +_bitcoin_cli() { + local cur prev words=() cword + local bitcoin_cli + + # save and use original argument to invoke bitcoin-cli for -help, help and RPC + # as bitcoin-cli might not be in $PATH + bitcoin_cli="$1" + + COMPREPLY=() + _get_comp_words_by_ref -n = cur prev words cword + + if ((cword > 5)); then + case ${words[cword-5]} in + sendtoaddress) + COMPREPLY=( $( compgen -W "true false" -- "$cur" ) ) + return 0 + ;; + esac + fi + + if ((cword > 4)); then + case ${words[cword-4]} in + importaddress|listtransactions|setban) + COMPREPLY=( $( compgen -W "true false" -- "$cur" ) ) + return 0 + ;; + signrawtransactionwithkey|signrawtransactionwithwallet) + COMPREPLY=( $( compgen -W "ALL NONE SINGLE ALL|ANYONECANPAY NONE|ANYONECANPAY SINGLE|ANYONECANPAY" -- "$cur" ) ) + return 0 + ;; + esac + fi + + if ((cword > 3)); then + case ${words[cword-3]} in + addmultisigaddress) + return 0 + ;; + getbalance|gettxout|importaddress|importpubkey|importprivkey|listreceivedbyaddress|listsinceblock) + COMPREPLY=( $( compgen -W "true false" -- "$cur" ) ) + return 0 + ;; + esac + fi + + if ((cword > 2)); then + case ${words[cword-2]} in + addnode) + COMPREPLY=( $( compgen -W "add remove onetry" -- "$cur" ) ) + return 0 + ;; + setban) + COMPREPLY=( $( compgen -W "add remove" -- "$cur" ) ) + return 0 + ;; + fundrawtransaction|getblock|getblockheader|getmempoolancestors|getmempooldescendants|getrawtransaction|gettransaction|listreceivedbyaddress|sendrawtransaction) + COMPREPLY=( $( compgen -W "true false" -- "$cur" ) ) + return 0 + ;; + esac + fi + + case "$prev" in + backupwallet|dumpwallet|importwallet) + _filedir + return 0 + ;; + getaddednodeinfo|getrawmempool|lockunspent) + COMPREPLY=( $( compgen -W "true false" -- "$cur" ) ) + return 0 + ;; + getbalance|getnewaddress|listtransactions|sendmany) + return 0 + ;; + esac + + case "$cur" in + -conf=*) + cur="${cur#*=}" + _filedir + return 0 + ;; + -datadir=*) + cur="${cur#*=}" + _filedir -d + return 0 + ;; + -*=*) # prevent nonsense completions + return 0 + ;; + *) + local helpopts commands + + # only parse -help if senseful + if [[ -z "$cur" || "$cur" =~ ^- ]]; then + helpopts=$($bitcoin_cli -help 2>&1 | awk '$1 ~ /^-/ { sub(/=.*/, "="); print $1 }' ) + fi + + # only parse help if senseful + if [[ -z "$cur" || "$cur" =~ ^[a-z] ]]; then + commands=$(_bitcoin_rpc help 2>/dev/null | awk '$1 ~ /^[a-z]/ { print $1; }') + fi + + COMPREPLY=( $( compgen -W "$helpopts $commands" -- "$cur" ) ) + + # Prevent space if an argument is desired + if [[ $COMPREPLY == *= ]]; then + compopt -o nospace + fi + return 0 + ;; + esac +} && +complete -F _bitcoin_cli bitcoin-cli + +# Local variables: +# mode: shell-script +# sh-basic-offset: 4 +# sh-indent-comment: t +# indent-tabs-mode: nil +# End: +# ex: ts=4 sw=4 et filetype=sh diff --git a/contrib/bitcoin-tx.bash-completion b/contrib/bitcoin-tx.bash-completion new file mode 100644 index 0000000000..a83d2979ed --- /dev/null +++ b/contrib/bitcoin-tx.bash-completion @@ -0,0 +1,57 @@ +# bash programmable completion for bitcoin-tx(1) +# Copyright (c) 2016 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +_bitcoin_tx() { + local cur prev words=() cword + local bitcoin_tx + + # save and use original argument to invoke bitcoin-tx for -help + # it might not be in $PATH + bitcoin_tx="$1" + + COMPREPLY=() + _get_comp_words_by_ref -n =: cur prev words cword + + case "$cur" in + load=*:*) + cur="${cur#load=*:}" + _filedir + return 0 + ;; + *=*) # prevent attempts to complete other arguments + return 0 + ;; + esac + + if [[ "$cword" == 1 || ( "$prev" != "-create" && "$prev" == -* ) ]]; then + # only options (or an uncompletable hex-string) allowed + # parse bitcoin-tx -help for options + local helpopts + helpopts=$($bitcoin_tx -help | sed -e '/^ -/ p' -e d ) + COMPREPLY=( $( compgen -W "$helpopts" -- "$cur" ) ) + else + # only commands are allowed + # parse -help for commands + local helpcmds + helpcmds=$($bitcoin_tx -help | sed -e '1,/Commands:/d' -e 's/=.*/=/' -e '/^ [a-z]/ p' -e d ) + COMPREPLY=( $( compgen -W "$helpcmds" -- "$cur" ) ) + fi + + # Prevent space if an argument is desired + if [[ $COMPREPLY == *= ]]; then + compopt -o nospace + fi + + return 0 +} && +complete -F _bitcoin_tx bitcoin-tx + +# Local variables: +# mode: shell-script +# sh-basic-offset: 4 +# sh-indent-comment: t +# indent-tabs-mode: nil +# End: +# ex: ts=4 sw=4 et filetype=sh diff --git a/contrib/bitcoind.bash-completion b/contrib/bitcoind.bash-completion new file mode 100644 index 0000000000..ec1d9512d4 --- /dev/null +++ b/contrib/bitcoind.bash-completion @@ -0,0 +1,56 @@ +# bash programmable completion for bitcoind(1) and bitcoin-qt(1) +# Copyright (c) 2012-2019 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +_bitcoind() { + local cur prev words=() cword + local bitcoind + + # save and use original argument to invoke bitcoind for -help + # it might not be in $PATH + bitcoind="$1" + + COMPREPLY=() + _get_comp_words_by_ref -n = cur prev words cword + + case "$cur" in + -conf=*|-pid=*|-loadblock=*|-rpccookiefile=*|-wallet=*) + cur="${cur#*=}" + _filedir + return 0 + ;; + -datadir=*) + cur="${cur#*=}" + _filedir -d + return 0 + ;; + -*=*) # prevent nonsense completions + return 0 + ;; + *) + + # only parse -help if sensible + if [[ -z "$cur" || "$cur" =~ ^- ]]; then + local helpopts + helpopts=$($bitcoind -help 2>&1 | awk '$1 ~ /^-/ { sub(/=.*/, "="); print $1 }' ) + COMPREPLY=( $( compgen -W "$helpopts" -- "$cur" ) ) + fi + + # Prevent space if an argument is desired + if [[ $COMPREPLY == *= ]]; then + compopt -o nospace + fi + return 0 + ;; + esac +} && +complete -F _bitcoind bitcoind bitcoin-qt + +# Local variables: +# mode: shell-script +# sh-basic-offset: 4 +# sh-indent-comment: t +# indent-tabs-mode: nil +# End: +# ex: ts=4 sw=4 et filetype=sh diff --git a/contrib/builder-keys/README.md b/contrib/builder-keys/README.md new file mode 100644 index 0000000000..a6179d6012 --- /dev/null +++ b/contrib/builder-keys/README.md @@ -0,0 +1,33 @@ +## PGP keys of builders and Developers + +The file `keys.txt` contains fingerprints of the public keys of builders and +active developers. + +The associated keys are mainly used to sign git commits or the build results +of Guix builds. + +The most recent version of each pgp key can be found on most pgp key servers. + +Fetch the latest version from the key server to see if any key was revoked in +the meantime. +To fetch the latest version of all pgp keys in your gpg homedir, + +```sh +gpg --refresh-keys +``` + +To fetch keys of builders and active developers, feed the list of fingerprints +of the primary keys into gpg: + +On \*NIX: +```sh +while read fingerprint keyholder_name; do gpg --keyserver hkps://keys.openpgp.org --recv-keys ${fingerprint}; done < ./keys.txt +``` + +On Windows (requires Gpg4win >= 4.0.0): +``` +FOR /F "tokens=1" %i IN (keys.txt) DO gpg --keyserver hkps://keys.openpgp.org --recv-keys %i +``` + +Add your key to the list if you provided Guix attestations for two major or +minor releases of Bitcoin Core. diff --git a/contrib/builder-keys/keys.txt b/contrib/builder-keys/keys.txt new file mode 100644 index 0000000000..c70069b440 --- /dev/null +++ b/contrib/builder-keys/keys.txt @@ -0,0 +1,56 @@ +9D3CC86A72F8494342EA5FD10A41BDC3F4FAFF1C Aaron Clauson (sipsorcery) +617C90010B3BD370B0AC7D424BB42E31C79111B8 Akira Takizawa (akx20000) +E944AE667CF960B1004BC32FCA662BE18B877A60 Andreas Schildbach (aschildbach) +152812300785C96444D3334D17565732E08E5E41 Andrew Chow (achow101) +590B7292695AFFA5B672CBB2E13FC145CD3F4304 Antoine Poinsot (darosior) +0AD83877C1F0CD1EE9BD660AD7CC770B81FD22A8 Ben Carman (benthecarman) +912FD3228387123DC97E0E57D5566241A0295FA9 BtcDrak (btcdrak) +04017A2A6D9A0CCDC81D8EC296AB007F1A7ED999 Carl Dong (dongcarl) +C519EBCF3B926298946783EFF6430754120EC2F4 Christian Decker (cdecker) +18AE2F798E0D239755DA4FD24B79F986CBDF8736 Chun Kuan Le (ken2812221) +101598DC823C1B5F9A6624ABA5E0907A0380E6C3 CoinForensics (CoinForensics) +F20F56EF6A067F70E8A5C99FFF95FAA971697405 centaur (centaur) +C060A6635913D98A3587D7DB1C2491FFEB0EF770 Cory Fields (cfields) +BF6273FAEF7CC0BA1F562E50989F6B3048A116B5 Dev Random (devrandom) +6D3170C1DC2C6FD0AEEBCA6743811D1A26623924 Douglas Roark (droark) +948444FCE03B05BA5AB0591EC37B1C1D44C786EE Duncan Dean (dunxen) +1C6621605EC50319C463D56C7F81D87985D61612 Emanuele Cisbani (cisba) +9A1689B60D1B3CCE9262307A2F40A9BF167FBA47 Erik Mossberg (erkmos) +D35176BE9264832E4ACA8986BF0792FBE95DC863 fivepiece (fivepiece) +6F993B250557E7B016ADE5713BDCDA2D87A881D9 Fuzzbawls (Fuzzbawls) +01CDF4627A3B88AAE4A571C87588242FBE38D3A8 Gavin Andresen (gavinandresen) +D1DBF2C4B96F2DEBF4C16654410108112E7EA81F Hennadii Stepanov (hebasto) +A2FD494D0021AA9B4FA58F759102B7AE654A4A5A Ilyas Ridhuan (IlyasRidhuan) +2688F5A9A4BE0F295E921E8A25F27A38A47AD566 James O'Beirne (jamesob) +D3F22A3A4C366C2DCB66D3722DA9C5A7FA81EA35 Jarol Rodriguez (jarolrod) +7480909378D544EA6B6DCEB7535B12980BB8A4D3 Jeffri H Frontz (jhfrontz) +D3CC177286005BB8FF673294C5242A1AB3936517 jl2012 (jl2012) +82921A4B88FD454B7EB8CE3C796C4109063D4EAF Jon Atack (jonatack) +32EE5C4C3FA15CCADB46ABE529D4BCB6416F53EC Jonas Schnelli (jonasschnelli) +4B4E840451149DD7FB0D633477DFAB5C3108B9A8 Jorge Timon (jtimon) +C42AFF7C61B3E44A1454CD3557AF762DB3353322 Karl-Johan Alm (kallewoof) +70A1D47DD44F59DF8B22244333E472FE870C7E5D Kristaps Kaupe (kristapsk) +30DE693AE0DE9E37B3E7EB6BBFF0F67810C1EED1 Lisa Neigut (niftynei) +E463A93F5F3117EEDE6C7316BD02942421F4889F Luke Dashjr (luke-jr) +B8B3F1C0E58C15DB6A81D30C3648A882F4316B9B Marco Falke (marco) +07DF3E57A548CCFB7530709189BBB8663E2E65CE Matt Corallo (BlueMatt) +CA03882CB1FC067B5D3ACFE4D300116E1C875A3D MeshCollider (meshcollider) +E777299FC265DD04793070EB944D35F9AC3DB76A Michael Ford (fanquake) +AD5764F4ADCE1B99BDFD179E12335A271D4D62EC Michael Tidwell (miketwenty1) +9692B91BBF0E8D34DFD33B1882C5C009628ECF0C Michagogo (michagogo) +C57E4B42223FDE851D4F69DD28DF2724F241D8EE midnightmagic (midnightmagic) +F4FC70F07310028424EFC20A8E4256593F177720 Oliver Gugger (guggero, Oliver Gugger) +D62A803E27E7F43486035ADBBCD04D8E9CCCAC2A Paul Rabahy (prab) +37EC7D7B0A217CDB4B4E007E7FAB114267E4FA04 Peter Todd (petertodd) +D762373D24904A3E42F33B08B9A408E71DAAC974 Pieter Wuille [Location: Leuven, Belgium] (sipa) +133EAC179436F14A5CF1B794860FEB804E669320 Pieter Wuille (sipa) +6A8F9C266528E25AEB1D7731C2371D91CB716EA7 Sebastian Falbesoner (theStack) +A8FC55F3B04BA3146F3492E79303B33A305224CB Sebastian Kung (TheCharlatan) +ED9BDF7AD6A55E232E84524257FF9BDBCC301009 Sjors Provoost (sjors) +867345026B6763E8B07EE73AB6737117397F5C4F Stephan Oeste (Emzy) +9EDAFF80E080659604F4A76B2EBB056FD847F8A7 Stephan Oeste (Emzy) +6DEEF79B050C4072509B743F8C275BC595448867 Tomas Kanocz (KanoczTomas) +AEC1884398647C47413C1C3FB1179EB7347DC10D Warren Togami (wtogami) +74E2DEF5D77260B98BC19438099BAD163C70FBFA Will Clark (will8clark) +79D00BAC68B56D422F945A8F8E3A8F3247DBCBBF Willy Ko (willyko) +71A3B16735405025D447E8F274810B012346C9A6 Wladimir J. van der Laan (laanwj) diff --git a/contrib/debian/copyright b/contrib/debian/copyright new file mode 100644 index 0000000000..95a281ce05 --- /dev/null +++ b/contrib/debian/copyright @@ -0,0 +1,161 @@ +Format: http://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ +Upstream-Name: Bitcoin +Upstream-Contact: Satoshi Nakamoto <satoshin@gmx.com> + irc://#bitcoin-core-dev@libera.chat +Source: https://github.com/bitcoin/bitcoin + +Files: * +Copyright: 2009-2022, Bitcoin Core Developers +License: Expat +Comment: The Bitcoin Core Developers encompasses the current developers listed on bitcoin.org, + as well as the numerous contributors to the project. + +Files: debian/* +Copyright: 2010-2011, Jonas Smedegaard <dr@jones.dk> + 2011, Matt Corallo <matt@bluematt.me> +License: GPL-2+ + +Files: src/secp256k1/build-aux/m4/ax_jni_include_dir.m4 +Copyright: 2008 Don Anderson <dda@sleepycat.com> +License: GNU-All-permissive-License + +Files: src/secp256k1/build-aux/m4/ax_prog_cc_for_build.m4 +Copyright: 2008 Paolo Bonzini <bonzini@gnu.org> +License: GNU-All-permissive-License + +Files: src/qt/res/icons/add.png + src/qt/res/icons/address-book.png + src/qt/res/icons/chevron.png + src/qt/res/icons/edit.png + src/qt/res/icons/editcopy.png + src/qt/res/icons/editpaste.png + src/qt/res/icons/export.png + src/qt/res/icons/eye.png + src/qt/res/icons/history.png + src/qt/res/icons/lock_*.png + src/qt/res/icons/overview.png + src/qt/res/icons/receive.png + src/qt/res/icons/remove.png + src/qt/res/icons/send.png + src/qt/res/icons/synced.png + src/qt/res/icons/transaction*.png + src/qt/res/icons/tx_output.png + src/qt/res/icons/warning.png +Copyright: Stephen Hutchings (and more) + http://typicons.com +License: Expat +Comment: Site: https://github.com/stephenhutchings/typicons.font + +Files: src/qt/res/icons/connect*.png + src/qt/res/src/connect-*.svg + src/qt/res/icons/network_disabled.png + src/qt/res/src/network_disabled.svg +Copyright: Marco Falke + Luke Dashjr +License: Expat +Comment: Inspired by Stephen Hutchings' Typicons + +Files: src/qt/res/icons/tx_mined.png + src/qt/res/src/mine.svg + src/qt/res/icons/fontbigger.png + src/qt/res/icons/fontsmaller.png + src/qt/res/icons/hd_disabled.png + src/qt/res/src/hd_disabled.svg + src/qt/res/icons/hd_enabled.png + src/qt/res/src/hd_enabled.svg +Copyright: Jonas Schnelli +License: Expat + +Files: src/qt/res/icons/clock*.png + src/qt/res/icons/eye_*.png + src/qt/res/icons/tx_in*.png + src/qt/res/src/clock_*.svg + src/qt/res/src/tx_*.svg +Copyright: Stephen Hutchings, Jonas Schnelli +License: Expat +Comment: Modifications of Stephen Hutchings' Typicons + +Files: src/qt/res/icons/bitcoin.* + share/pixmaps/bitcoin* + src/qt/res/src/bitcoin.svg +Copyright: Bitboy, Jonas Schnelli +License: public-domain +Comment: Site: https://bitcointalk.org/?topic=1756.0 + +Files: src/qt/res/icons/proxy.png + src/qt/res/src/proxy.svg +Copyright: Cristian Mircea Messel +License: public-domain + +Files: src/qt/res/fonts/RobotoMono-Bold.ttf +License: Apache-2.0 +Comment: Site: https://fonts.google.com/specimen/Roboto+Mono + + +License: Expat + Permission is hereby granted, free of charge, to any person obtaining a + copy of this software and associated documentation files (the + "Software"), to deal in the Software without restriction, including + without limitation the rights to use, copy, modify, merge, publish, + distribute, sublicense, and/or sell copies of the Software, and to + permit persons to whom the Software is furnished to do so, subject to + the following conditions: + . + The above copyright notice and this permission notice shall be included + in all copies or substantial portions of the Software. + . + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS + OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY + CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, + TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +License: GNU-All-permissive-License + Copying and distribution of this file, with or without modification, are + permitted in any medium without royalty provided the copyright notice + and this notice are preserved. This file is offered as-is, without any + warranty. + +License: GPL-2+ + This program is free software; you can redistribute it and/or modify it + under the terms of the GNU General Public License as published by the + Free Software Foundation; either version 2, or (at your option) any + later version. + . + This program is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. +Comment: + On Debian systems the GNU General Public License (GPL) version 2 is + located in '/usr/share/common-licenses/GPL-2'. + . + You should have received a copy of the GNU General Public License along + with this program. If not, see <http://www.gnu.org/licenses/>. + +License: GPL-3+ + Permission is granted to copy, distribute and/or modify this document + under the terms of the GNU General Public License, Version 3 or any + later version published by the Free Software Foundation. +Comment: + On Debian systems the GNU General Public License (GPL) version 3 is + located in '/usr/share/common-licenses/GPL-3'. + . + You should have received a copy of the GNU General Public License along + with this program. If not, see <http://www.gnu.org/licenses/>. + +License: public-domain + This work is in the public domain. + +License: Apache-2.0 + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/contrib/devtools/README.md b/contrib/devtools/README.md new file mode 100644 index 0000000000..54b1a85588 --- /dev/null +++ b/contrib/devtools/README.md @@ -0,0 +1,146 @@ +Contents +======== +This directory contains tools for developers working on this repository. + +clang-format-diff.py +=================== + +A script to format unified git diffs according to [.clang-format](../../src/.clang-format). + +Requires `clang-format`, installed e.g. via `brew install clang-format` on macOS, +or `sudo apt install clang-format` on Debian/Ubuntu. + +For instance, to format the last commit with 0 lines of context, +the script should be called from the git root folder as follows. + +``` +git diff -U0 HEAD~1.. | ./contrib/devtools/clang-format-diff.py -p1 -i -v +``` + +copyright\_header.py +==================== + +Provides utilities for managing copyright headers of `The Bitcoin Core +developers` in repository source files. It has three subcommands: + +``` +$ ./copyright_header.py report <base_directory> [verbose] +$ ./copyright_header.py update <base_directory> +$ ./copyright_header.py insert <file> +``` +Running these subcommands without arguments displays a usage string. + +copyright\_header.py report \<base\_directory\> [verbose] +--------------------------------------------------------- + +Produces a report of all copyright header notices found inside the source files +of a repository. Useful to quickly visualize the state of the headers. +Specifying `verbose` will list the full filenames of files of each category. + +copyright\_header.py update \<base\_directory\> [verbose] +--------------------------------------------------------- +Updates all the copyright headers of `The Bitcoin Core developers` which were +changed in a year more recent than is listed. For example: +``` +// Copyright (c) <firstYear>-<lastYear> The Bitcoin Core developers +``` +will be updated to: +``` +// Copyright (c) <firstYear>-<lastModifiedYear> The Bitcoin Core developers +``` +where `<lastModifiedYear>` is obtained from the `git log` history. + +This subcommand also handles copyright headers that have only a single year. In +those cases: +``` +// Copyright (c) <year> The Bitcoin Core developers +``` +will be updated to: +``` +// Copyright (c) <year>-<lastModifiedYear> The Bitcoin Core developers +``` +where the update is appropriate. + +copyright\_header.py insert \<file\> +------------------------------------ +Inserts a copyright header for `The Bitcoin Core developers` at the top of the +file in either Python or C++ style as determined by the file extension. If the +file is a Python file and it has `#!` starting the first line, the header is +inserted in the line below it. + +The copyright dates will be set to be `<year_introduced>-<current_year>` where +`<year_introduced>` is according to the `git log` history. If +`<year_introduced>` is equal to `<current_year>`, it will be set as a single +year rather than two hyphenated years. + +If the file already has a copyright for `The Bitcoin Core developers`, the +script will exit. + +gen-manpages.py +=============== + +A small script to automatically create manpages in ../../doc/man by running the release binaries with the -help option. +This requires help2man which can be found at: https://www.gnu.org/software/help2man/ + +With in-tree builds this tool can be run from any directory within the +repostitory. To use this tool with out-of-tree builds set `BUILDDIR`. For +example: + +```bash +BUILDDIR=$PWD/build contrib/devtools/gen-manpages.py +``` + +gen-bitcoin-conf.sh +=================== + +Generates a bitcoin.conf file in `share/examples/` by parsing the output from `bitcoind --help`. This script is run during the +release process to include a bitcoin.conf with the release binaries and can also be run by users to generate a file locally. +When generating a file as part of the release process, make sure to commit the changes after running the script. + +With in-tree builds this tool can be run from any directory within the +repository. To use this tool with out-of-tree builds set `BUILDDIR`. For +example: + +```bash +BUILDDIR=$PWD/build contrib/devtools/gen-bitcoin-conf.sh +``` + +security-check.py and test-security-check.py +============================================ + +Perform basic security checks on a series of executables. + +symbol-check.py +=============== + +A script to check that release executables only contain +certain symbols and are only linked against allowed libraries. + +For Linux this means checking for allowed gcc, glibc and libstdc++ version symbols. +This makes sure they are still compatible with the minimum supported distribution versions. + +For macOS and Windows we check that the executables are only linked against libraries we allow. + +Example usage: + + find ../path/to/executables -type f -executable | xargs python3 contrib/devtools/symbol-check.py + +If no errors occur the return value will be 0 and the output will be empty. + +If there are any errors the return value will be 1 and output like this will be printed: + + .../64/test_bitcoin: symbol memcpy from unsupported version GLIBC_2.14 + .../64/test_bitcoin: symbol __fdelt_chk from unsupported version GLIBC_2.15 + .../64/test_bitcoin: symbol std::out_of_range::~out_of_range() from unsupported version GLIBCXX_3.4.15 + .../64/test_bitcoin: symbol _ZNSt8__detail15_List_nod from unsupported version GLIBCXX_3.4.15 + +circular-dependencies.py +======================== + +Run this script from the root of the source tree (`src/`) to find circular dependencies in the source code. +This looks only at which files include other files, treating the `.cpp` and `.h` file as one unit. + +Example usage: + + cd .../src + ../contrib/devtools/circular-dependencies.py {*,*/*,*/*/*}.{h,cpp} diff --git a/contrib/devtools/circular-dependencies.py b/contrib/devtools/circular-dependencies.py new file mode 100755 index 0000000000..b1d9f2b7db --- /dev/null +++ b/contrib/devtools/circular-dependencies.py @@ -0,0 +1,92 @@ +#!/usr/bin/env python3 +# Copyright (c) 2018-2020 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +import sys +import re +from typing import Dict, List, Set + +MAPPING = { + 'core_read.cpp': 'core_io.cpp', + 'core_write.cpp': 'core_io.cpp', +} + +# Directories with header-based modules, where the assumption that .cpp files +# define functions and variables declared in corresponding .h files is +# incorrect. +HEADER_MODULE_PATHS = [ + 'interfaces/' +] + +def module_name(path): + if path in MAPPING: + path = MAPPING[path] + if any(path.startswith(dirpath) for dirpath in HEADER_MODULE_PATHS): + return path + if path.endswith(".h"): + return path[:-2] + if path.endswith(".c"): + return path[:-2] + if path.endswith(".cpp"): + return path[:-4] + return None + +files = dict() +deps: Dict[str, Set[str]] = dict() + +RE = re.compile("^#include <(.*)>") + +# Iterate over files, and create list of modules +for arg in sys.argv[1:]: + module = module_name(arg) + if module is None: + print("Ignoring file %s (does not constitute module)\n" % arg) + else: + files[arg] = module + deps[module] = set() + +# Iterate again, and build list of direct dependencies for each module +# TODO: implement support for multiple include directories +for arg in sorted(files.keys()): + module = files[arg] + with open(arg, 'r', encoding="utf8") as f: + for line in f: + match = RE.match(line) + if match: + include = match.group(1) + included_module = module_name(include) + if included_module is not None and included_module in deps and included_module != module: + deps[module].add(included_module) + +# Loop to find the shortest (remaining) circular dependency +have_cycle: bool = False +while True: + shortest_cycle = None + for module in sorted(deps.keys()): + # Build the transitive closure of dependencies of module + closure: Dict[str, List[str]] = dict() + for dep in deps[module]: + closure[dep] = [] + while True: + old_size = len(closure) + old_closure_keys = sorted(closure.keys()) + for src in old_closure_keys: + for dep in deps[src]: + if dep not in closure: + closure[dep] = closure[src] + [src] + if len(closure) == old_size: + break + # If module is in its own transitive closure, it's a circular dependency; check if it is the shortest + if module in closure and (shortest_cycle is None or len(closure[module]) + 1 < len(shortest_cycle)): + shortest_cycle = [module] + closure[module] + if shortest_cycle is None: + break + # We have the shortest circular dependency; report it + module = shortest_cycle[0] + print("Circular dependency: %s" % (" -> ".join(shortest_cycle + [module]))) + # And then break the dependency to avoid repeating in other cycles + deps[shortest_cycle[-1]] = deps[shortest_cycle[-1]] - set([module]) + have_cycle = True + +sys.exit(1 if have_cycle else 0) diff --git a/contrib/devtools/clang-format-diff.py b/contrib/devtools/clang-format-diff.py new file mode 100755 index 0000000000..98eee67f43 --- /dev/null +++ b/contrib/devtools/clang-format-diff.py @@ -0,0 +1,166 @@ +#!/usr/bin/env python3 +# +#===- clang-format-diff.py - ClangFormat Diff Reformatter ----*- python -*--===# +# +# The LLVM Compiler Infrastructure +# +# This file is distributed under the University of Illinois Open Source +# License. +# +# ============================================================ +# +# University of Illinois/NCSA +# Open Source License +# +# Copyright (c) 2007-2015 University of Illinois at Urbana-Champaign. +# All rights reserved. +# +# Developed by: +# +# LLVM Team +# +# University of Illinois at Urbana-Champaign +# +# http://llvm.org +# +# Permission is hereby granted, free of charge, to any person obtaining a copy of +# this software and associated documentation files (the "Software"), to deal with +# the Software without restriction, including without limitation the rights to +# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +# of the Software, and to permit persons to whom the Software is furnished to do +# so, subject to the following conditions: +# +# * Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimers. +# +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimers in the +# documentation and/or other materials provided with the distribution. +# +# * Neither the names of the LLVM Team, University of Illinois at +# Urbana-Champaign, nor the names of its contributors may be used to +# endorse or promote products derived from this Software without specific +# prior written permission. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH THE +# SOFTWARE. +# +# ============================================================ +# +#===------------------------------------------------------------------------===# + +r""" +ClangFormat Diff Reformatter +============================ + +This script reads input from a unified diff and reformats all the changed +lines. This is useful to reformat all the lines touched by a specific patch. +Example usage for git/svn users: + + git diff -U0 HEAD^ | clang-format-diff.py -p1 -i + svn diff --diff-cmd=diff -x-U0 | clang-format-diff.py -i + +""" + +import argparse +import difflib +import io +import re +import subprocess +import sys + + +# Change this to the full path if clang-format is not on the path. +binary = 'clang-format' + + +def main(): + parser = argparse.ArgumentParser(description= + 'Reformat changed lines in diff. Without -i ' + 'option just output the diff that would be ' + 'introduced.') + parser.add_argument('-i', action='store_true', default=False, + help='apply edits to files instead of displaying a diff') + parser.add_argument('-p', metavar='NUM', default=0, + help='strip the smallest prefix containing P slashes') + parser.add_argument('-regex', metavar='PATTERN', default=None, + help='custom pattern selecting file paths to reformat ' + '(case sensitive, overrides -iregex)') + parser.add_argument('-iregex', metavar='PATTERN', default= + r'.*\.(cpp|cc|c\+\+|cxx|c|cl|h|hpp|m|mm|inc|js|ts|proto' + r'|protodevel|java)', + help='custom pattern selecting file paths to reformat ' + '(case insensitive, overridden by -regex)') + parser.add_argument('-sort-includes', action='store_true', default=False, + help='let clang-format sort include blocks') + parser.add_argument('-v', '--verbose', action='store_true', + help='be more verbose, ineffective without -i') + args = parser.parse_args() + + # Extract changed lines for each file. + filename = None + lines_by_file = {} + for line in sys.stdin: + match = re.search(r'^\+\+\+\ (.*?/){%s}(\S*)' % args.p, line) + if match: + filename = match.group(2) + if filename is None: + continue + + if args.regex is not None: + if not re.match('^%s$' % args.regex, filename): + continue + else: + if not re.match('^%s$' % args.iregex, filename, re.IGNORECASE): + continue + + match = re.search(r'^@@.*\+(\d+)(,(\d+))?', line) + if match: + start_line = int(match.group(1)) + line_count = 1 + if match.group(3): + line_count = int(match.group(3)) + if line_count == 0: + continue + end_line = start_line + line_count - 1 + lines_by_file.setdefault(filename, []).extend( + ['-lines', str(start_line) + ':' + str(end_line)]) + + # Reformat files containing changes in place. + for filename, lines in lines_by_file.items(): + if args.i and args.verbose: + print('Formatting {}'.format(filename)) + command = [binary, filename] + if args.i: + command.append('-i') + if args.sort_includes: + command.append('-sort-includes') + command.extend(lines) + command.extend(['-style=file', '-fallback-style=none']) + p = subprocess.Popen(command, + stdout=subprocess.PIPE, + stderr=None, + stdin=subprocess.PIPE, + universal_newlines=True) + stdout, stderr = p.communicate() + if p.returncode != 0: + sys.exit(p.returncode) + + if not args.i: + with open(filename, encoding="utf8") as f: + code = f.readlines() + formatted_code = io.StringIO(stdout).readlines() + diff = difflib.unified_diff(code, formatted_code, + filename, filename, + '(before formatting)', '(after formatting)') + diff_string = ''.join(diff) + if len(diff_string) > 0: + sys.stdout.write(diff_string) + +if __name__ == '__main__': + main() diff --git a/contrib/devtools/copyright_header.py b/contrib/devtools/copyright_header.py new file mode 100755 index 0000000000..680de1f1b3 --- /dev/null +++ b/contrib/devtools/copyright_header.py @@ -0,0 +1,604 @@ +#!/usr/bin/env python3 +# Copyright (c) 2016-2021 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +import re +import fnmatch +import sys +import subprocess +import datetime +import os + +################################################################################ +# file filtering +################################################################################ + +EXCLUDE = [ + # auto generated: + 'src/qt/bitcoinstrings.cpp', + 'src/chainparamsseeds.h', + # other external copyrights: + 'src/reverse_iterator.h', + 'src/test/fuzz/FuzzedDataProvider.h', + 'src/tinyformat.h', + 'src/bench/nanobench.h', + 'test/functional/test_framework/bignum.py', + # python init: + '*__init__.py', +] +EXCLUDE_COMPILED = re.compile('|'.join([fnmatch.translate(m) for m in EXCLUDE])) + +EXCLUDE_DIRS = [ + # git subtrees + "src/crypto/ctaes/", + "src/leveldb/", + "src/minisketch", + "src/secp256k1/", + "src/crc32c/", +] + +INCLUDE = ['*.h', '*.cpp', '*.cc', '*.c', '*.mm', '*.py', '*.sh', '*.bash-completion'] +INCLUDE_COMPILED = re.compile('|'.join([fnmatch.translate(m) for m in INCLUDE])) + +def applies_to_file(filename): + for excluded_dir in EXCLUDE_DIRS: + if filename.startswith(excluded_dir): + return False + return ((EXCLUDE_COMPILED.match(filename) is None) and + (INCLUDE_COMPILED.match(filename) is not None)) + +################################################################################ +# obtain list of files in repo according to INCLUDE and EXCLUDE +################################################################################ + +GIT_LS_CMD = 'git ls-files --full-name'.split(' ') +GIT_TOPLEVEL_CMD = 'git rev-parse --show-toplevel'.split(' ') + +def call_git_ls(base_directory): + out = subprocess.check_output([*GIT_LS_CMD, base_directory]) + return [f for f in out.decode("utf-8").split('\n') if f != ''] + +def call_git_toplevel(): + "Returns the absolute path to the project root" + return subprocess.check_output(GIT_TOPLEVEL_CMD).strip().decode("utf-8") + +def get_filenames_to_examine(base_directory): + "Returns an array of absolute paths to any project files in the base_directory that pass the include/exclude filters" + root = call_git_toplevel() + filenames = call_git_ls(base_directory) + return sorted([os.path.join(root, filename) for filename in filenames if + applies_to_file(filename)]) + +################################################################################ +# define and compile regexes for the patterns we are looking for +################################################################################ + + +COPYRIGHT_WITH_C = r'Copyright \(c\)' +COPYRIGHT_WITHOUT_C = 'Copyright' +ANY_COPYRIGHT_STYLE = '(%s|%s)' % (COPYRIGHT_WITH_C, COPYRIGHT_WITHOUT_C) + +YEAR = "20[0-9][0-9]" +YEAR_RANGE = '(%s)(-%s)?' % (YEAR, YEAR) +YEAR_LIST = '(%s)(, %s)+' % (YEAR, YEAR) +ANY_YEAR_STYLE = '(%s|%s)' % (YEAR_RANGE, YEAR_LIST) +ANY_COPYRIGHT_STYLE_OR_YEAR_STYLE = ("%s %s" % (ANY_COPYRIGHT_STYLE, + ANY_YEAR_STYLE)) + +ANY_COPYRIGHT_COMPILED = re.compile(ANY_COPYRIGHT_STYLE_OR_YEAR_STYLE) + +def compile_copyright_regex(copyright_style, year_style, name): + return re.compile(r'%s %s,? %s( +\*)?\n' % (copyright_style, year_style, name)) + +EXPECTED_HOLDER_NAMES = [ + r"Satoshi Nakamoto", + r"The Bitcoin Core developers", + r"BitPay Inc\.", + r"University of Illinois at Urbana-Champaign\.", + r"Pieter Wuille", + r"Wladimir J\. van der Laan", + r"Jeff Garzik", + r"Jan-Klaas Kollhof", + r"ArtForz -- public domain half-a-node", + r"Intel Corporation ?", + r"The Zcash developers", + r"Jeremy Rubin", +] + +DOMINANT_STYLE_COMPILED = {} +YEAR_LIST_STYLE_COMPILED = {} +WITHOUT_C_STYLE_COMPILED = {} + +for holder_name in EXPECTED_HOLDER_NAMES: + DOMINANT_STYLE_COMPILED[holder_name] = ( + compile_copyright_regex(COPYRIGHT_WITH_C, YEAR_RANGE, holder_name)) + YEAR_LIST_STYLE_COMPILED[holder_name] = ( + compile_copyright_regex(COPYRIGHT_WITH_C, YEAR_LIST, holder_name)) + WITHOUT_C_STYLE_COMPILED[holder_name] = ( + compile_copyright_regex(COPYRIGHT_WITHOUT_C, ANY_YEAR_STYLE, + holder_name)) + +################################################################################ +# search file contents for copyright message of particular category +################################################################################ + +def get_count_of_copyrights_of_any_style_any_holder(contents): + return len(ANY_COPYRIGHT_COMPILED.findall(contents)) + +def file_has_dominant_style_copyright_for_holder(contents, holder_name): + match = DOMINANT_STYLE_COMPILED[holder_name].search(contents) + return match is not None + +def file_has_year_list_style_copyright_for_holder(contents, holder_name): + match = YEAR_LIST_STYLE_COMPILED[holder_name].search(contents) + return match is not None + +def file_has_without_c_style_copyright_for_holder(contents, holder_name): + match = WITHOUT_C_STYLE_COMPILED[holder_name].search(contents) + return match is not None + +################################################################################ +# get file info +################################################################################ + +def read_file(filename): + return open(filename, 'r', encoding="utf8").read() + +def gather_file_info(filename): + info = {} + info['filename'] = filename + c = read_file(filename) + info['contents'] = c + + info['all_copyrights'] = get_count_of_copyrights_of_any_style_any_holder(c) + + info['classified_copyrights'] = 0 + info['dominant_style'] = {} + info['year_list_style'] = {} + info['without_c_style'] = {} + for holder_name in EXPECTED_HOLDER_NAMES: + has_dominant_style = ( + file_has_dominant_style_copyright_for_holder(c, holder_name)) + has_year_list_style = ( + file_has_year_list_style_copyright_for_holder(c, holder_name)) + has_without_c_style = ( + file_has_without_c_style_copyright_for_holder(c, holder_name)) + info['dominant_style'][holder_name] = has_dominant_style + info['year_list_style'][holder_name] = has_year_list_style + info['without_c_style'][holder_name] = has_without_c_style + if has_dominant_style or has_year_list_style or has_without_c_style: + info['classified_copyrights'] = info['classified_copyrights'] + 1 + return info + +################################################################################ +# report execution +################################################################################ + +SEPARATOR = '-'.join(['' for _ in range(80)]) + +def print_filenames(filenames, verbose): + if not verbose: + return + for filename in filenames: + print("\t%s" % filename) + +def print_report(file_infos, verbose): + print(SEPARATOR) + examined = [i['filename'] for i in file_infos] + print("%d files examined according to INCLUDE and EXCLUDE fnmatch rules" % + len(examined)) + print_filenames(examined, verbose) + + print(SEPARATOR) + print('') + zero_copyrights = [i['filename'] for i in file_infos if + i['all_copyrights'] == 0] + print("%4d with zero copyrights" % len(zero_copyrights)) + print_filenames(zero_copyrights, verbose) + one_copyright = [i['filename'] for i in file_infos if + i['all_copyrights'] == 1] + print("%4d with one copyright" % len(one_copyright)) + print_filenames(one_copyright, verbose) + two_copyrights = [i['filename'] for i in file_infos if + i['all_copyrights'] == 2] + print("%4d with two copyrights" % len(two_copyrights)) + print_filenames(two_copyrights, verbose) + three_copyrights = [i['filename'] for i in file_infos if + i['all_copyrights'] == 3] + print("%4d with three copyrights" % len(three_copyrights)) + print_filenames(three_copyrights, verbose) + four_or_more_copyrights = [i['filename'] for i in file_infos if + i['all_copyrights'] >= 4] + print("%4d with four or more copyrights" % len(four_or_more_copyrights)) + print_filenames(four_or_more_copyrights, verbose) + print('') + print(SEPARATOR) + print('Copyrights with dominant style:\ne.g. "Copyright (c)" and ' + '"<year>" or "<startYear>-<endYear>":\n') + for holder_name in EXPECTED_HOLDER_NAMES: + dominant_style = [i['filename'] for i in file_infos if + i['dominant_style'][holder_name]] + if len(dominant_style) > 0: + print("%4d with '%s'" % (len(dominant_style), + holder_name.replace('\n', '\\n'))) + print_filenames(dominant_style, verbose) + print('') + print(SEPARATOR) + print('Copyrights with year list style:\ne.g. "Copyright (c)" and ' + '"<year1>, <year2>, ...":\n') + for holder_name in EXPECTED_HOLDER_NAMES: + year_list_style = [i['filename'] for i in file_infos if + i['year_list_style'][holder_name]] + if len(year_list_style) > 0: + print("%4d with '%s'" % (len(year_list_style), + holder_name.replace('\n', '\\n'))) + print_filenames(year_list_style, verbose) + print('') + print(SEPARATOR) + print('Copyrights with no "(c)" style:\ne.g. "Copyright" and "<year>" or ' + '"<startYear>-<endYear>":\n') + for holder_name in EXPECTED_HOLDER_NAMES: + without_c_style = [i['filename'] for i in file_infos if + i['without_c_style'][holder_name]] + if len(without_c_style) > 0: + print("%4d with '%s'" % (len(without_c_style), + holder_name.replace('\n', '\\n'))) + print_filenames(without_c_style, verbose) + + print('') + print(SEPARATOR) + + unclassified_copyrights = [i['filename'] for i in file_infos if + i['classified_copyrights'] < i['all_copyrights']] + print("%d with unexpected copyright holder names" % + len(unclassified_copyrights)) + print_filenames(unclassified_copyrights, verbose) + print(SEPARATOR) + +def exec_report(base_directory, verbose): + filenames = get_filenames_to_examine(base_directory) + file_infos = [gather_file_info(f) for f in filenames] + print_report(file_infos, verbose) + +################################################################################ +# report cmd +################################################################################ + +REPORT_USAGE = """ +Produces a report of all copyright header notices found inside the source files +of a repository. + +Usage: + $ ./copyright_header.py report <base_directory> [verbose] + +Arguments: + <base_directory> - The base directory of a bitcoin source code repository. + [verbose] - Includes a list of every file of each subcategory in the report. +""" + +def report_cmd(argv): + if len(argv) == 2: + sys.exit(REPORT_USAGE) + + base_directory = argv[2] + if not os.path.exists(base_directory): + sys.exit("*** bad <base_directory>: %s" % base_directory) + + if len(argv) == 3: + verbose = False + elif argv[3] == 'verbose': + verbose = True + else: + sys.exit("*** unknown argument: %s" % argv[2]) + + exec_report(base_directory, verbose) + +################################################################################ +# query git for year of last change +################################################################################ + +GIT_LOG_CMD = "git log --pretty=format:%%ai %s" + +def call_git_log(filename): + out = subprocess.check_output((GIT_LOG_CMD % filename).split(' ')) + return out.decode("utf-8").split('\n') + +def get_git_change_years(filename): + git_log_lines = call_git_log(filename) + if len(git_log_lines) == 0: + return [datetime.date.today().year] + # timestamp is in ISO 8601 format. e.g. "2016-09-05 14:25:32 -0600" + return [line.split(' ')[0].split('-')[0] for line in git_log_lines] + +def get_most_recent_git_change_year(filename): + return max(get_git_change_years(filename)) + +################################################################################ +# read and write to file +################################################################################ + +def read_file_lines(filename): + with open(filename, 'r', encoding="utf8") as f: + file_lines = f.readlines() + return file_lines + +def write_file_lines(filename, file_lines): + with open(filename, 'w', encoding="utf8") as f: + f.write(''.join(file_lines)) + +################################################################################ +# update header years execution +################################################################################ + +COPYRIGHT = r'Copyright \(c\)' +YEAR = "20[0-9][0-9]" +YEAR_RANGE = '(%s)(-%s)?' % (YEAR, YEAR) +HOLDER = 'The Bitcoin Core developers' +UPDATEABLE_LINE_COMPILED = re.compile(' '.join([COPYRIGHT, YEAR_RANGE, HOLDER])) + +def get_updatable_copyright_line(file_lines): + index = 0 + for line in file_lines: + if UPDATEABLE_LINE_COMPILED.search(line) is not None: + return index, line + index = index + 1 + return None, None + +def parse_year_range(year_range): + year_split = year_range.split('-') + start_year = year_split[0] + if len(year_split) == 1: + return start_year, start_year + return start_year, year_split[1] + +def year_range_to_str(start_year, end_year): + if start_year == end_year: + return start_year + return "%s-%s" % (start_year, end_year) + +def create_updated_copyright_line(line, last_git_change_year): + copyright_splitter = 'Copyright (c) ' + copyright_split = line.split(copyright_splitter) + # Preserve characters on line that are ahead of the start of the copyright + # notice - they are part of the comment block and vary from file-to-file. + before_copyright = copyright_split[0] + after_copyright = copyright_split[1] + + space_split = after_copyright.split(' ') + year_range = space_split[0] + start_year, end_year = parse_year_range(year_range) + if end_year >= last_git_change_year: + return line + return (before_copyright + copyright_splitter + + year_range_to_str(start_year, last_git_change_year) + ' ' + + ' '.join(space_split[1:])) + +def update_updatable_copyright(filename): + file_lines = read_file_lines(filename) + index, line = get_updatable_copyright_line(file_lines) + if not line: + print_file_action_message(filename, "No updatable copyright.") + return + last_git_change_year = get_most_recent_git_change_year(filename) + new_line = create_updated_copyright_line(line, last_git_change_year) + if line == new_line: + print_file_action_message(filename, "Copyright up-to-date.") + return + file_lines[index] = new_line + write_file_lines(filename, file_lines) + print_file_action_message(filename, + "Copyright updated! -> %s" % last_git_change_year) + +def exec_update_header_year(base_directory): + for filename in get_filenames_to_examine(base_directory): + update_updatable_copyright(filename) + +################################################################################ +# update cmd +################################################################################ + +UPDATE_USAGE = """ +Updates all the copyright headers of "The Bitcoin Core developers" which were +changed in a year more recent than is listed. For example: + +// Copyright (c) <firstYear>-<lastYear> The Bitcoin Core developers + +will be updated to: + +// Copyright (c) <firstYear>-<lastModifiedYear> The Bitcoin Core developers + +where <lastModifiedYear> is obtained from the 'git log' history. + +This subcommand also handles copyright headers that have only a single year. In those cases: + +// Copyright (c) <year> The Bitcoin Core developers + +will be updated to: + +// Copyright (c) <year>-<lastModifiedYear> The Bitcoin Core developers + +where the update is appropriate. + +Usage: + $ ./copyright_header.py update <base_directory> + +Arguments: + <base_directory> - The base directory of a bitcoin source code repository. +""" + +def print_file_action_message(filename, action): + print("%-52s %s" % (filename, action)) + +def update_cmd(argv): + if len(argv) != 3: + sys.exit(UPDATE_USAGE) + + base_directory = argv[2] + if not os.path.exists(base_directory): + sys.exit("*** bad base_directory: %s" % base_directory) + exec_update_header_year(base_directory) + +################################################################################ +# inserted copyright header format +################################################################################ + +def get_header_lines(header, start_year, end_year): + lines = header.split('\n')[1:-1] + lines[0] = lines[0] % year_range_to_str(start_year, end_year) + return [line + '\n' for line in lines] + +CPP_HEADER = ''' +// Copyright (c) %s The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. +''' + +def get_cpp_header_lines_to_insert(start_year, end_year): + return reversed(get_header_lines(CPP_HEADER, start_year, end_year)) + +SCRIPT_HEADER = ''' +# Copyright (c) %s The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +''' + +def get_script_header_lines_to_insert(start_year, end_year): + return reversed(get_header_lines(SCRIPT_HEADER, start_year, end_year)) + +################################################################################ +# query git for year of last change +################################################################################ + +def get_git_change_year_range(filename): + years = get_git_change_years(filename) + return min(years), max(years) + +################################################################################ +# check for existing core copyright +################################################################################ + +def file_already_has_core_copyright(file_lines): + index, _ = get_updatable_copyright_line(file_lines) + return index is not None + +################################################################################ +# insert header execution +################################################################################ + +def file_has_hashbang(file_lines): + if len(file_lines) < 1: + return False + if len(file_lines[0]) <= 2: + return False + return file_lines[0][:2] == '#!' + +def insert_script_header(filename, file_lines, start_year, end_year): + if file_has_hashbang(file_lines): + insert_idx = 1 + else: + insert_idx = 0 + header_lines = get_script_header_lines_to_insert(start_year, end_year) + for line in header_lines: + file_lines.insert(insert_idx, line) + write_file_lines(filename, file_lines) + +def insert_cpp_header(filename, file_lines, start_year, end_year): + file_lines.insert(0, '\n') + header_lines = get_cpp_header_lines_to_insert(start_year, end_year) + for line in header_lines: + file_lines.insert(0, line) + write_file_lines(filename, file_lines) + +def exec_insert_header(filename, style): + file_lines = read_file_lines(filename) + if file_already_has_core_copyright(file_lines): + sys.exit('*** %s already has a copyright by The Bitcoin Core developers' + % (filename)) + start_year, end_year = get_git_change_year_range(filename) + if style in ['python', 'shell']: + insert_script_header(filename, file_lines, start_year, end_year) + else: + insert_cpp_header(filename, file_lines, start_year, end_year) + +################################################################################ +# insert cmd +################################################################################ + +INSERT_USAGE = """ +Inserts a copyright header for "The Bitcoin Core developers" at the top of the +file in either Python or C++ style as determined by the file extension. If the +file is a Python file and it has a '#!' starting the first line, the header is +inserted in the line below it. + +The copyright dates will be set to be: + +"<year_introduced>-<current_year>" + +where <year_introduced> is according to the 'git log' history. If +<year_introduced> is equal to <current_year>, the date will be set to be: + +"<current_year>" + +If the file already has a copyright for "The Bitcoin Core developers", the +script will exit. + +Usage: + $ ./copyright_header.py insert <file> + +Arguments: + <file> - A source file in the bitcoin repository. +""" + +def insert_cmd(argv): + if len(argv) != 3: + sys.exit(INSERT_USAGE) + + filename = argv[2] + if not os.path.isfile(filename): + sys.exit("*** bad filename: %s" % filename) + _, extension = os.path.splitext(filename) + if extension not in ['.h', '.cpp', '.cc', '.c', '.py', '.sh']: + sys.exit("*** cannot insert for file extension %s" % extension) + + if extension == '.py': + style = 'python' + elif extension == '.sh': + style = 'shell' + else: + style = 'cpp' + exec_insert_header(filename, style) + +################################################################################ +# UI +################################################################################ + +USAGE = """ +copyright_header.py - utilities for managing copyright headers of 'The Bitcoin +Core developers' in repository source files. + +Usage: + $ ./copyright_header <subcommand> + +Subcommands: + report + update + insert + +To see subcommand usage, run them without arguments. +""" + +SUBCOMMANDS = ['report', 'update', 'insert'] + +if __name__ == "__main__": + if len(sys.argv) == 1: + sys.exit(USAGE) + subcommand = sys.argv[1] + if subcommand not in SUBCOMMANDS: + sys.exit(USAGE) + if subcommand == 'report': + report_cmd(sys.argv) + elif subcommand == 'update': + update_cmd(sys.argv) + elif subcommand == 'insert': + insert_cmd(sys.argv) diff --git a/contrib/devtools/gen-bitcoin-conf.sh b/contrib/devtools/gen-bitcoin-conf.sh new file mode 100755 index 0000000000..2ebbd42022 --- /dev/null +++ b/contrib/devtools/gen-bitcoin-conf.sh @@ -0,0 +1,83 @@ +#!/usr/bin/env bash +# Copyright (c) 2021 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +export LC_ALL=C +TOPDIR=${TOPDIR:-$(git rev-parse --show-toplevel)} +BUILDDIR=${BUILDDIR:-$TOPDIR} +BINDIR=${BINDIR:-$BUILDDIR/src} +BITCOIND=${BITCOIND:-$BINDIR/bitcoind} +SHARE_EXAMPLES_DIR=${SHARE_EXAMPLES_DIR:-$TOPDIR/share/examples} +EXAMPLE_CONF_FILE=${EXAMPLE_CONF_FILE:-$SHARE_EXAMPLES_DIR/bitcoin.conf} + +[ ! -x "$BITCOIND" ] && echo "$BITCOIND not found or not executable." && exit 1 + +DIRTY="" +VERSION_OUTPUT=$($BITCOIND --version) +if [[ $VERSION_OUTPUT == *"dirty"* ]]; then + DIRTY="${DIRTY}${BITCOIND}\n" +fi + +if [ -n "$DIRTY" ] +then + echo -e "WARNING: $BITCOIND was built from a dirty tree.\n" + echo -e "To safely generate a bitcoin.conf file, please commit your changes to $BITCOIND, rebuild, then run this script again.\n" +fi + +echo 'Generating example bitcoin.conf file in share/examples/' + +# create the directory, if it doesn't exist +mkdir -p "${SHARE_EXAMPLES_DIR}" + +# create the header text +cat > "${EXAMPLE_CONF_FILE}" << 'EOF' +## +## bitcoin.conf configuration file. +## Generated by contrib/devtools/gen-bitcoin-conf.sh. +## +## Lines beginning with # are comments. +## All possible configuration options are provided. To use, copy this file +## to your data directory (default or specified by -datadir), uncomment +## options you would like to change, and save the file. +## + + +### Options +EOF + +# parse the output from bitcoind --help +# adding newlines is a bit funky to ensure portability for BSD +# see here for more details: https://stackoverflow.com/a/24575385 +${BITCOIND} --help \ + | sed '1,/Print this help message and exit/d' \ + | sed -E 's/^[[:space:]]{2}\-/#/' \ + | sed -E 's/^[[:space:]]{7}/# /' \ + | sed -E '/[=[:space:]]/!s/#.*$/&=1/' \ + | awk '/^#[a-z]/{x=$0;next}{if (NF==0) print x"\n",x="";else print}' \ + | sed 's,\(^[[:upper:]].*\)\:$,\ +### \1,' \ + | sed 's/[[:space:]]*$//' >> "${EXAMPLE_CONF_FILE}" + +# create the footer text +cat >> "${EXAMPLE_CONF_FILE}" << 'EOF' + +# [Sections] +# Most options will apply to all networks. To confine an option to a specific +# network, add it under the relevant section below. +# +# Note: If not specified under a network section, the options addnode, connect, +# port, bind, rpcport, rpcbind, and wallet will only apply to mainnet. + +# Options for mainnet +[main] + +# Options for testnet +[test] + +# Options for signet +[signet] + +# Options for regtest +[regtest] +EOF diff --git a/contrib/devtools/gen-manpages.py b/contrib/devtools/gen-manpages.py new file mode 100755 index 0000000000..26612cc444 --- /dev/null +++ b/contrib/devtools/gen-manpages.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python3 +# Copyright (c) 2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +import os +import subprocess +import sys +import tempfile + +BINARIES = [ +'src/bitcoind', +'src/bitcoin-cli', +'src/bitcoin-tx', +'src/bitcoin-wallet', +'src/bitcoin-util', +'src/qt/bitcoin-qt', +] + +# Paths to external utilities. +git = os.getenv('GIT', 'git') +help2man = os.getenv('HELP2MAN', 'help2man') + +# If not otherwise specified, get top directory from git. +topdir = os.getenv('TOPDIR') +if not topdir: + r = subprocess.run([git, 'rev-parse', '--show-toplevel'], stdout=subprocess.PIPE, check=True, universal_newlines=True) + topdir = r.stdout.rstrip() + +# Get input and output directories. +builddir = os.getenv('BUILDDIR', topdir) +mandir = os.getenv('MANDIR', os.path.join(topdir, 'doc/man')) + +# Verify that all the required binaries are usable, and extract copyright +# message in a first pass. +versions = [] +for relpath in BINARIES: + abspath = os.path.join(builddir, relpath) + try: + r = subprocess.run([abspath, '--version'], stdout=subprocess.PIPE, universal_newlines=True) + except IOError: + print(f'{abspath} not found or not an executable', file=sys.stderr) + sys.exit(1) + # take first line (which must contain version) + verstr = r.stdout.splitlines()[0] + # last word of line is the actual version e.g. v22.99.0-5c6b3d5b3508 + verstr = verstr.split()[-1] + assert verstr.startswith('v') + # remaining lines are copyright + copyright = r.stdout.split('\n')[1:] + assert copyright[0].startswith('Copyright (C)') + + versions.append((abspath, verstr, copyright)) + +if any(verstr.endswith('-dirty') for (_, verstr, _) in versions): + print("WARNING: Binaries were built from a dirty tree.") + print('man pages generated from dirty binaries should NOT be committed.') + print('To properly generate man pages, please commit your changes (or discard them), rebuild, then run this script again.') + print() + +with tempfile.NamedTemporaryFile('w', suffix='.h2m') as footer: + # Create copyright footer, and write it to a temporary include file. + # Copyright is the same for all binaries, so just use the first. + footer.write('[COPYRIGHT]\n') + footer.write('\n'.join(versions[0][2]).strip()) + footer.flush() + + # Call the binaries through help2man to produce a manual page for each of them. + for (abspath, verstr, _) in versions: + outname = os.path.join(mandir, os.path.basename(abspath) + '.1') + print(f'Generating {outname}…') + subprocess.run([help2man, '-N', '--version-string=' + verstr, '--include=' + footer.name, '-o', outname, abspath], check=True) diff --git a/contrib/devtools/iwyu/bitcoin.core.imp b/contrib/devtools/iwyu/bitcoin.core.imp new file mode 100644 index 0000000000..ce7786f58c --- /dev/null +++ b/contrib/devtools/iwyu/bitcoin.core.imp @@ -0,0 +1,6 @@ +# Fixups / upstreamed changes +[ + { include: [ "<bits/termios-c_lflag.h>", private, "<termios.h>", public ] }, + { include: [ "<bits/termios-struct.h>", private, "<termios.h>", public ] }, + { include: [ "<bits/termios-tcflow.h>", private, "<termios.h>", public ] }, +] diff --git a/contrib/devtools/security-check.py b/contrib/devtools/security-check.py new file mode 100755 index 0000000000..05c0af029e --- /dev/null +++ b/contrib/devtools/security-check.py @@ -0,0 +1,263 @@ +#!/usr/bin/env python3 +# Copyright (c) 2015-2021 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +''' +Perform basic security checks on a series of executables. +Exit status will be 0 if successful, and the program will be silent. +Otherwise the exit status will be 1 and it will log which executables failed which checks. +''' +import sys +from typing import List + +import lief #type:ignore + +def check_ELF_RELRO(binary) -> bool: + ''' + Check for read-only relocations. + GNU_RELRO program header must exist + Dynamic section must have BIND_NOW flag + ''' + have_gnu_relro = False + for segment in binary.segments: + # Note: not checking p_flags == PF_R: here as linkers set the permission differently + # This does not affect security: the permission flags of the GNU_RELRO program + # header are ignored, the PT_LOAD header determines the effective permissions. + # However, the dynamic linker need to write to this area so these are RW. + # Glibc itself takes care of mprotecting this area R after relocations are finished. + # See also https://marc.info/?l=binutils&m=1498883354122353 + if segment.type == lief.ELF.SEGMENT_TYPES.GNU_RELRO: + have_gnu_relro = True + + have_bindnow = False + try: + flags = binary.get(lief.ELF.DYNAMIC_TAGS.FLAGS) + if flags.value & lief.ELF.DYNAMIC_FLAGS.BIND_NOW: + have_bindnow = True + except: + have_bindnow = False + + return have_gnu_relro and have_bindnow + +def check_ELF_Canary(binary) -> bool: + ''' + Check for use of stack canary + ''' + return binary.has_symbol('__stack_chk_fail') + +def check_ELF_separate_code(binary): + ''' + Check that sections are appropriately separated in virtual memory, + based on their permissions. This checks for missing -Wl,-z,separate-code + and potentially other problems. + ''' + R = lief.ELF.SEGMENT_FLAGS.R + W = lief.ELF.SEGMENT_FLAGS.W + E = lief.ELF.SEGMENT_FLAGS.X + EXPECTED_FLAGS = { + # Read + execute + '.init': R | E, + '.plt': R | E, + '.plt.got': R | E, + '.plt.sec': R | E, + '.text': R | E, + '.fini': R | E, + # Read-only data + '.interp': R, + '.note.gnu.property': R, + '.note.gnu.build-id': R, + '.note.ABI-tag': R, + '.gnu.hash': R, + '.dynsym': R, + '.dynstr': R, + '.gnu.version': R, + '.gnu.version_r': R, + '.rela.dyn': R, + '.rela.plt': R, + '.rodata': R, + '.eh_frame_hdr': R, + '.eh_frame': R, + '.qtmetadata': R, + '.gcc_except_table': R, + '.stapsdt.base': R, + # Writable data + '.init_array': R | W, + '.fini_array': R | W, + '.dynamic': R | W, + '.got': R | W, + '.data': R | W, + '.bss': R | W, + } + if binary.header.machine_type == lief.ELF.ARCH.PPC64: + # .plt is RW on ppc64 even with separate-code + EXPECTED_FLAGS['.plt'] = R | W + # For all LOAD program headers get mapping to the list of sections, + # and for each section, remember the flags of the associated program header. + flags_per_section = {} + for segment in binary.segments: + if segment.type == lief.ELF.SEGMENT_TYPES.LOAD: + for section in segment.sections: + flags_per_section[section.name] = segment.flags + # Spot-check ELF LOAD program header flags per section + # If these sections exist, check them against the expected R/W/E flags + for (section, flags) in flags_per_section.items(): + if section in EXPECTED_FLAGS: + if int(EXPECTED_FLAGS[section]) != int(flags): + return False + return True + +def check_ELF_control_flow(binary) -> bool: + ''' + Check for control flow instrumentation + ''' + main = binary.get_function_address('main') + content = binary.get_content_from_virtual_address(main, 4, lief.Binary.VA_TYPES.AUTO) + + if content == [243, 15, 30, 250]: # endbr64 + return True + return False + +def check_PE_DYNAMIC_BASE(binary) -> bool: + '''PIE: DllCharacteristics bit 0x40 signifies dynamicbase (ASLR)''' + return lief.PE.DLL_CHARACTERISTICS.DYNAMIC_BASE in binary.optional_header.dll_characteristics_lists + +# Must support high-entropy 64-bit address space layout randomization +# in addition to DYNAMIC_BASE to have secure ASLR. +def check_PE_HIGH_ENTROPY_VA(binary) -> bool: + '''PIE: DllCharacteristics bit 0x20 signifies high-entropy ASLR''' + return lief.PE.DLL_CHARACTERISTICS.HIGH_ENTROPY_VA in binary.optional_header.dll_characteristics_lists + +def check_PE_RELOC_SECTION(binary) -> bool: + '''Check for a reloc section. This is required for functional ASLR.''' + return binary.has_relocations + +def check_PE_control_flow(binary) -> bool: + ''' + Check for control flow instrumentation + ''' + main = binary.get_symbol('main').value + + section_addr = binary.section_from_rva(main).virtual_address + virtual_address = binary.optional_header.imagebase + section_addr + main + + content = binary.get_content_from_virtual_address(virtual_address, 4, lief.Binary.VA_TYPES.VA) + + if content == [243, 15, 30, 250]: # endbr64 + return True + return False + +def check_MACHO_NOUNDEFS(binary) -> bool: + ''' + Check for no undefined references. + ''' + return binary.header.has(lief.MachO.HEADER_FLAGS.NOUNDEFS) + +def check_MACHO_LAZY_BINDINGS(binary) -> bool: + ''' + Check for no lazy bindings. + We don't use or check for MH_BINDATLOAD. See #18295. + ''' + return binary.dyld_info.lazy_bind == (0,0) + +def check_MACHO_Canary(binary) -> bool: + ''' + Check for use of stack canary + ''' + return binary.has_symbol('___stack_chk_fail') + +def check_PIE(binary) -> bool: + ''' + Check for position independent executable (PIE), + allowing for address space randomization. + ''' + return binary.is_pie + +def check_NX(binary) -> bool: + ''' + Check for no stack execution + ''' + return binary.has_nx + +def check_MACHO_control_flow(binary) -> bool: + ''' + Check for control flow instrumentation + ''' + content = binary.get_content_from_virtual_address(binary.entrypoint, 4, lief.Binary.VA_TYPES.AUTO) + + if content == [243, 15, 30, 250]: # endbr64 + return True + return False + +BASE_ELF = [ + ('PIE', check_PIE), + ('NX', check_NX), + ('RELRO', check_ELF_RELRO), + ('Canary', check_ELF_Canary), + ('separate_code', check_ELF_separate_code), +] + +BASE_PE = [ + ('PIE', check_PIE), + ('DYNAMIC_BASE', check_PE_DYNAMIC_BASE), + ('HIGH_ENTROPY_VA', check_PE_HIGH_ENTROPY_VA), + ('NX', check_NX), + ('RELOC_SECTION', check_PE_RELOC_SECTION), + ('CONTROL_FLOW', check_PE_control_flow), +] + +BASE_MACHO = [ + ('NOUNDEFS', check_MACHO_NOUNDEFS), + ('LAZY_BINDINGS', check_MACHO_LAZY_BINDINGS), + ('Canary', check_MACHO_Canary), +] + +CHECKS = { + lief.EXE_FORMATS.ELF: { + lief.ARCHITECTURES.X86: BASE_ELF + [('CONTROL_FLOW', check_ELF_control_flow)], + lief.ARCHITECTURES.ARM: BASE_ELF, + lief.ARCHITECTURES.ARM64: BASE_ELF, + lief.ARCHITECTURES.PPC: BASE_ELF, + lief.ARCHITECTURES.RISCV: BASE_ELF, + }, + lief.EXE_FORMATS.PE: { + lief.ARCHITECTURES.X86: BASE_PE, + }, + lief.EXE_FORMATS.MACHO: { + lief.ARCHITECTURES.X86: BASE_MACHO + [('PIE', check_PIE), + ('NX', check_NX), + ('CONTROL_FLOW', check_MACHO_control_flow)], + lief.ARCHITECTURES.ARM64: BASE_MACHO, + } +} + +if __name__ == '__main__': + retval: int = 0 + for filename in sys.argv[1:]: + try: + binary = lief.parse(filename) + etype = binary.format + arch = binary.abstract.header.architecture + binary.concrete + + if etype == lief.EXE_FORMATS.UNKNOWN: + print(f'{filename}: unknown executable format') + retval = 1 + continue + + if arch == lief.ARCHITECTURES.NONE: + print(f'{filename}: unknown architecture') + retval = 1 + continue + + failed: List[str] = [] + for (name, func) in CHECKS[etype][arch]: + if not func(binary): + failed.append(name) + if failed: + print(f'{filename}: failed {" ".join(failed)}') + retval = 1 + except IOError: + print(f'{filename}: cannot open') + retval = 1 + sys.exit(retval) + diff --git a/contrib/devtools/split-debug.sh.in b/contrib/devtools/split-debug.sh.in new file mode 100644 index 0000000000..92b72b1446 --- /dev/null +++ b/contrib/devtools/split-debug.sh.in @@ -0,0 +1,10 @@ +#!/bin/sh +set -e +if [ $# -ne 3 ]; + then echo "usage: $0 <input> <stripped-binary> <debug-binary>" +fi + +@OBJCOPY@ --enable-deterministic-archives -p --only-keep-debug $1 $3 +@OBJCOPY@ --enable-deterministic-archives -p --strip-debug $1 $2 +@STRIP@ --enable-deterministic-archives -p -s $2 +@OBJCOPY@ --enable-deterministic-archives -p --add-gnu-debuglink=$3 $2 diff --git a/contrib/devtools/symbol-check.py b/contrib/devtools/symbol-check.py new file mode 100755 index 0000000000..23d29af3f1 --- /dev/null +++ b/contrib/devtools/symbol-check.py @@ -0,0 +1,290 @@ +#!/usr/bin/env python3 +# Copyright (c) 2014 Wladimir J. van der Laan +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +''' +A script to check that release executables only contain certain symbols +and are only linked against allowed libraries. + +Example usage: + + find ../path/to/binaries -type f -executable | xargs python3 contrib/devtools/symbol-check.py +''' +import sys +from typing import List, Dict + +import lief #type:ignore + +# Debian 9 (Stretch) EOL: 2022. https://wiki.debian.org/DebianReleases#Production_Releases +# +# - g++ version 6.3.0 (https://packages.debian.org/search?suite=stretch&arch=any&searchon=names&keywords=g%2B%2B) +# - libc version 2.24 (https://packages.debian.org/search?suite=stretch&arch=any&searchon=names&keywords=libc6) +# +# Ubuntu 16.04 (Xenial) EOL: 2026. https://wiki.ubuntu.com/Releases +# +# - g++ version 5.3.1 +# - libc version 2.23 +# +# CentOS Stream 8 EOL: 2024. https://wiki.centos.org/About/Product +# +# - g++ version 8.5.0 (http://mirror.centos.org/centos/8-stream/AppStream/x86_64/os/Packages/) +# - libc version 2.28 (http://mirror.centos.org/centos/8-stream/AppStream/x86_64/os/Packages/) +# +# See https://gcc.gnu.org/onlinedocs/libstdc++/manual/abi.html for more info. + +MAX_VERSIONS = { +'GCC': (4,8,0), +'GLIBC': { + lief.ELF.ARCH.i386: (2,18), + lief.ELF.ARCH.x86_64: (2,18), + lief.ELF.ARCH.ARM: (2,18), + lief.ELF.ARCH.AARCH64:(2,18), + lief.ELF.ARCH.PPC64: (2,18), + lief.ELF.ARCH.RISCV: (2,27), +}, +'LIBATOMIC': (1,0), +'V': (0,5,0), # xkb (bitcoin-qt only) +} +# See here for a description of _IO_stdin_used: +# https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=634261#109 + +# Ignore symbols that are exported as part of every executable +IGNORE_EXPORTS = { +'environ', '_environ', '__environ', '_fini', '_init', 'stdin', +'stdout', 'stderr', +} + +# Expected linker-loader names can be found here: +# https://sourceware.org/glibc/wiki/ABIList?action=recall&rev=16 +ELF_INTERPRETER_NAMES: Dict[lief.ELF.ARCH, Dict[lief.ENDIANNESS, str]] = { + lief.ELF.ARCH.i386: { + lief.ENDIANNESS.LITTLE: "/lib/ld-linux.so.2", + }, + lief.ELF.ARCH.x86_64: { + lief.ENDIANNESS.LITTLE: "/lib64/ld-linux-x86-64.so.2", + }, + lief.ELF.ARCH.ARM: { + lief.ENDIANNESS.LITTLE: "/lib/ld-linux-armhf.so.3", + }, + lief.ELF.ARCH.AARCH64: { + lief.ENDIANNESS.LITTLE: "/lib/ld-linux-aarch64.so.1", + }, + lief.ELF.ARCH.PPC64: { + lief.ENDIANNESS.BIG: "/lib64/ld64.so.1", + lief.ENDIANNESS.LITTLE: "/lib64/ld64.so.2", + }, + lief.ELF.ARCH.RISCV: { + lief.ENDIANNESS.LITTLE: "/lib/ld-linux-riscv64-lp64d.so.1", + }, +} + +# Allowed NEEDED libraries +ELF_ALLOWED_LIBRARIES = { +# bitcoind and bitcoin-qt +'libgcc_s.so.1', # GCC base support +'libc.so.6', # C library +'libpthread.so.0', # threading +'libm.so.6', # math library +'librt.so.1', # real-time (clock) +'libatomic.so.1', +'ld-linux-x86-64.so.2', # 64-bit dynamic linker +'ld-linux.so.2', # 32-bit dynamic linker +'ld-linux-aarch64.so.1', # 64-bit ARM dynamic linker +'ld-linux-armhf.so.3', # 32-bit ARM dynamic linker +'ld64.so.1', # POWER64 ABIv1 dynamic linker +'ld64.so.2', # POWER64 ABIv2 dynamic linker +'ld-linux-riscv64-lp64d.so.1', # 64-bit RISC-V dynamic linker +# bitcoin-qt only +'libxcb.so.1', # part of X11 +'libxkbcommon.so.0', # keyboard keymapping +'libxkbcommon-x11.so.0', # keyboard keymapping +'libfontconfig.so.1', # font support +'libfreetype.so.6', # font parsing +'libdl.so.2', # programming interface to dynamic linker +'libxcb-icccm.so.4', +'libxcb-image.so.0', +'libxcb-shm.so.0', +'libxcb-keysyms.so.1', +'libxcb-randr.so.0', +'libxcb-render-util.so.0', +'libxcb-render.so.0', +'libxcb-shape.so.0', +'libxcb-sync.so.1', +'libxcb-xfixes.so.0', +'libxcb-xinerama.so.0', +'libxcb-xkb.so.1', +} + +MACHO_ALLOWED_LIBRARIES = { +# bitcoind and bitcoin-qt +'libc++.1.dylib', # C++ Standard Library +'libSystem.B.dylib', # libc, libm, libpthread, libinfo +# bitcoin-qt only +'AppKit', # user interface +'ApplicationServices', # common application tasks. +'Carbon', # deprecated c back-compat API +'ColorSync', +'CoreFoundation', # low level func, data types +'CoreGraphics', # 2D rendering +'CoreServices', # operating system services +'CoreText', # interface for laying out text and handling fonts. +'CoreVideo', # video processing +'Foundation', # base layer functionality for apps/frameworks +'ImageIO', # read and write image file formats. +'IOKit', # user-space access to hardware devices and drivers. +'IOSurface', # cross process image/drawing buffers +'libobjc.A.dylib', # Objective-C runtime library +'Metal', # 3D graphics +'Security', # access control and authentication +'QuartzCore', # animation +} + +PE_ALLOWED_LIBRARIES = { +'ADVAPI32.dll', # security & registry +'IPHLPAPI.DLL', # IP helper API +'KERNEL32.dll', # win32 base APIs +'msvcrt.dll', # C standard library for MSVC +'SHELL32.dll', # shell API +'USER32.dll', # user interface +'WS2_32.dll', # sockets +# bitcoin-qt only +'dwmapi.dll', # desktop window manager +'GDI32.dll', # graphics device interface +'IMM32.dll', # input method editor +'NETAPI32.dll', +'ole32.dll', # component object model +'OLEAUT32.dll', # OLE Automation API +'SHLWAPI.dll', # light weight shell API +'USERENV.dll', +'UxTheme.dll', +'VERSION.dll', # version checking +'WINMM.dll', # WinMM audio API +'WTSAPI32.dll', +} + +def check_version(max_versions, version, arch) -> bool: + (lib, _, ver) = version.rpartition('_') + ver = tuple([int(x) for x in ver.split('.')]) + if not lib in max_versions: + return False + if isinstance(max_versions[lib], tuple): + return ver <= max_versions[lib] + else: + return ver <= max_versions[lib][arch] + +def check_imported_symbols(binary) -> bool: + ok: bool = True + + for symbol in binary.imported_symbols: + if not symbol.imported: + continue + + version = symbol.symbol_version if symbol.has_version else None + + if version: + aux_version = version.symbol_version_auxiliary.name if version.has_auxiliary_version else None + if aux_version and not check_version(MAX_VERSIONS, aux_version, binary.header.machine_type): + print(f'{filename}: symbol {symbol.name} from unsupported version {version}') + ok = False + return ok + +def check_exported_symbols(binary) -> bool: + ok: bool = True + + for symbol in binary.dynamic_symbols: + if not symbol.exported: + continue + name = symbol.name + if binary.header.machine_type == lief.ELF.ARCH.RISCV or name in IGNORE_EXPORTS: + continue + print(f'{binary.name}: export of symbol {name} not allowed!') + ok = False + return ok + +def check_ELF_libraries(binary) -> bool: + ok: bool = True + for library in binary.libraries: + if library not in ELF_ALLOWED_LIBRARIES: + print(f'{filename}: {library} is not in ALLOWED_LIBRARIES!') + ok = False + return ok + +def check_MACHO_libraries(binary) -> bool: + ok: bool = True + for dylib in binary.libraries: + split = dylib.name.split('/') + if split[-1] not in MACHO_ALLOWED_LIBRARIES: + print(f'{split[-1]} is not in ALLOWED_LIBRARIES!') + ok = False + return ok + +def check_MACHO_min_os(binary) -> bool: + if binary.build_version.minos == [10,15,0]: + return True + return False + +def check_MACHO_sdk(binary) -> bool: + if binary.build_version.sdk == [11, 0, 0]: + return True + return False + +def check_PE_libraries(binary) -> bool: + ok: bool = True + for dylib in binary.libraries: + if dylib not in PE_ALLOWED_LIBRARIES: + print(f'{dylib} is not in ALLOWED_LIBRARIES!') + ok = False + return ok + +def check_PE_subsystem_version(binary) -> bool: + major: int = binary.optional_header.major_subsystem_version + minor: int = binary.optional_header.minor_subsystem_version + if major == 6 and minor == 1: + return True + return False + +def check_ELF_interpreter(binary) -> bool: + expected_interpreter = ELF_INTERPRETER_NAMES[binary.header.machine_type][binary.abstract.header.endianness] + + return binary.concrete.interpreter == expected_interpreter + +CHECKS = { +lief.EXE_FORMATS.ELF: [ + ('IMPORTED_SYMBOLS', check_imported_symbols), + ('EXPORTED_SYMBOLS', check_exported_symbols), + ('LIBRARY_DEPENDENCIES', check_ELF_libraries), + ('INTERPRETER_NAME', check_ELF_interpreter), +], +lief.EXE_FORMATS.MACHO: [ + ('DYNAMIC_LIBRARIES', check_MACHO_libraries), + ('MIN_OS', check_MACHO_min_os), + ('SDK', check_MACHO_sdk), +], +lief.EXE_FORMATS.PE: [ + ('DYNAMIC_LIBRARIES', check_PE_libraries), + ('SUBSYSTEM_VERSION', check_PE_subsystem_version), +] +} + +if __name__ == '__main__': + retval: int = 0 + for filename in sys.argv[1:]: + try: + binary = lief.parse(filename) + etype = binary.format + if etype == lief.EXE_FORMATS.UNKNOWN: + print(f'{filename}: unknown executable format') + retval = 1 + continue + + failed: List[str] = [] + for (name, func) in CHECKS[etype]: + if not func(binary): + failed.append(name) + if failed: + print(f'{filename}: failed {" ".join(failed)}') + retval = 1 + except IOError: + print(f'{filename}: cannot open') + retval = 1 + sys.exit(retval) diff --git a/contrib/devtools/test-security-check.py b/contrib/devtools/test-security-check.py new file mode 100755 index 0000000000..d3d225f3ab --- /dev/null +++ b/contrib/devtools/test-security-check.py @@ -0,0 +1,151 @@ +#!/usr/bin/env python3 +# Copyright (c) 2015-2021 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +''' +Test script for security-check.py +''' +import lief #type:ignore +import os +import subprocess +from typing import List +import unittest + +from utils import determine_wellknown_cmd + +def write_testcode(filename): + with open(filename, 'w', encoding="utf8") as f: + f.write(''' + #include <stdio.h> + int main() + { + printf("the quick brown fox jumps over the lazy god\\n"); + return 0; + } + ''') + +def clean_files(source, executable): + os.remove(source) + os.remove(executable) + +def call_security_check(cc, source, executable, options): + # This should behave the same as AC_TRY_LINK, so arrange well-known flags + # in the same order as autoconf would. + # + # See the definitions for ac_link in autoconf's lib/autoconf/c.m4 file for + # reference. + env_flags: List[str] = [] + for var in ['CFLAGS', 'CPPFLAGS', 'LDFLAGS']: + env_flags += filter(None, os.environ.get(var, '').split(' ')) + + subprocess.run([*cc,source,'-o',executable] + env_flags + options, check=True) + p = subprocess.run(['./contrib/devtools/security-check.py',executable], stdout=subprocess.PIPE, universal_newlines=True) + return (p.returncode, p.stdout.rstrip()) + +def get_arch(cc, source, executable): + subprocess.run([*cc, source, '-o', executable], check=True) + binary = lief.parse(executable) + arch = binary.abstract.header.architecture + os.remove(executable) + return arch + +class TestSecurityChecks(unittest.TestCase): + def test_ELF(self): + source = 'test1.c' + executable = 'test1' + cc = determine_wellknown_cmd('CC', 'gcc') + write_testcode(source) + arch = get_arch(cc, source, executable) + + if arch == lief.ARCHITECTURES.X86: + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-zexecstack','-fno-stack-protector','-Wl,-znorelro','-no-pie','-fno-PIE', '-Wl,-z,separate-code']), + (1, executable+': failed PIE NX RELRO Canary CONTROL_FLOW')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-fno-stack-protector','-Wl,-znorelro','-no-pie','-fno-PIE', '-Wl,-z,separate-code']), + (1, executable+': failed PIE RELRO Canary CONTROL_FLOW')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-fstack-protector-all','-Wl,-znorelro','-no-pie','-fno-PIE', '-Wl,-z,separate-code']), + (1, executable+': failed PIE RELRO CONTROL_FLOW')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-fstack-protector-all','-Wl,-znorelro','-pie','-fPIE', '-Wl,-z,separate-code']), + (1, executable+': failed RELRO CONTROL_FLOW')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-fstack-protector-all','-Wl,-zrelro','-Wl,-z,now','-pie','-fPIE', '-Wl,-z,noseparate-code']), + (1, executable+': failed separate_code CONTROL_FLOW')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-fstack-protector-all','-Wl,-zrelro','-Wl,-z,now','-pie','-fPIE', '-Wl,-z,separate-code']), + (1, executable+': failed CONTROL_FLOW')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-fstack-protector-all','-Wl,-zrelro','-Wl,-z,now','-pie','-fPIE', '-Wl,-z,separate-code', '-fcf-protection=full']), + (0, '')) + else: + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-zexecstack','-fno-stack-protector','-Wl,-znorelro','-no-pie','-fno-PIE', '-Wl,-z,separate-code']), + (1, executable+': failed PIE NX RELRO Canary')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-fno-stack-protector','-Wl,-znorelro','-no-pie','-fno-PIE', '-Wl,-z,separate-code']), + (1, executable+': failed PIE RELRO Canary')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-fstack-protector-all','-Wl,-znorelro','-no-pie','-fno-PIE', '-Wl,-z,separate-code']), + (1, executable+': failed PIE RELRO')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-fstack-protector-all','-Wl,-znorelro','-pie','-fPIE', '-Wl,-z,separate-code']), + (1, executable+': failed RELRO')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-fstack-protector-all','-Wl,-zrelro','-Wl,-z,now','-pie','-fPIE', '-Wl,-z,noseparate-code']), + (1, executable+': failed separate_code')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-fstack-protector-all','-Wl,-zrelro','-Wl,-z,now','-pie','-fPIE', '-Wl,-z,separate-code']), + (0, '')) + + clean_files(source, executable) + + def test_PE(self): + source = 'test1.c' + executable = 'test1.exe' + cc = determine_wellknown_cmd('CC', 'x86_64-w64-mingw32-gcc') + write_testcode(source) + + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--disable-nxcompat','-Wl,--disable-reloc-section','-Wl,--disable-dynamicbase','-Wl,--disable-high-entropy-va','-no-pie','-fno-PIE']), + (1, executable+': failed PIE DYNAMIC_BASE HIGH_ENTROPY_VA NX RELOC_SECTION CONTROL_FLOW')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--disable-reloc-section','-Wl,--disable-dynamicbase','-Wl,--disable-high-entropy-va','-no-pie','-fno-PIE']), + (1, executable+': failed PIE DYNAMIC_BASE HIGH_ENTROPY_VA RELOC_SECTION CONTROL_FLOW')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--enable-reloc-section','-Wl,--disable-dynamicbase','-Wl,--disable-high-entropy-va','-no-pie','-fno-PIE']), + (1, executable+': failed PIE DYNAMIC_BASE HIGH_ENTROPY_VA CONTROL_FLOW')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--enable-reloc-section','-Wl,--disable-dynamicbase','-Wl,--disable-high-entropy-va','-pie','-fPIE']), + (1, executable+': failed PIE DYNAMIC_BASE HIGH_ENTROPY_VA CONTROL_FLOW')) # -pie -fPIE does nothing unless --dynamicbase is also supplied + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--enable-reloc-section','-Wl,--dynamicbase','-Wl,--disable-high-entropy-va','-pie','-fPIE']), + (1, executable+': failed HIGH_ENTROPY_VA CONTROL_FLOW')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--enable-reloc-section','-Wl,--dynamicbase','-Wl,--high-entropy-va','-pie','-fPIE']), + (1, executable+': failed CONTROL_FLOW')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--enable-reloc-section','-Wl,--dynamicbase','-Wl,--high-entropy-va','-pie','-fPIE', '-fcf-protection=full']), + (0, '')) + + clean_files(source, executable) + + def test_MACHO(self): + source = 'test1.c' + executable = 'test1' + cc = determine_wellknown_cmd('CC', 'clang') + write_testcode(source) + arch = get_arch(cc, source, executable) + + if arch == lief.ARCHITECTURES.X86: + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-Wl,-flat_namespace','-Wl,-allow_stack_execute','-fno-stack-protector']), + (1, executable+': failed NOUNDEFS LAZY_BINDINGS Canary PIE NX CONTROL_FLOW')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-Wl,-flat_namespace','-Wl,-allow_stack_execute','-fstack-protector-all']), + (1, executable+': failed NOUNDEFS LAZY_BINDINGS PIE NX CONTROL_FLOW')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-Wl,-flat_namespace','-fstack-protector-all']), + (1, executable+': failed NOUNDEFS LAZY_BINDINGS PIE CONTROL_FLOW')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-fstack-protector-all']), + (1, executable+': failed LAZY_BINDINGS PIE CONTROL_FLOW')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-Wl,-bind_at_load','-fstack-protector-all']), + (1, executable+': failed PIE CONTROL_FLOW')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-Wl,-bind_at_load','-fstack-protector-all', '-fcf-protection=full']), + (1, executable+': failed PIE')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-pie','-Wl,-bind_at_load','-fstack-protector-all', '-fcf-protection=full']), + (0, '')) + else: + # arm64 darwin doesn't support non-PIE binaries, control flow or executable stacks + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-flat_namespace','-fno-stack-protector']), + (1, executable+': failed NOUNDEFS LAZY_BINDINGS Canary')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-flat_namespace','-fstack-protector-all']), + (1, executable+': failed NOUNDEFS LAZY_BINDINGS')) + self.assertEqual(call_security_check(cc, source, executable, ['-fstack-protector-all']), + (1, executable+': failed LAZY_BINDINGS')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-bind_at_load','-fstack-protector-all']), + (0, '')) + + + clean_files(source, executable) + +if __name__ == '__main__': + unittest.main() diff --git a/contrib/devtools/test-symbol-check.py b/contrib/devtools/test-symbol-check.py new file mode 100755 index 0000000000..2881e3efac --- /dev/null +++ b/contrib/devtools/test-symbol-check.py @@ -0,0 +1,204 @@ +#!/usr/bin/env python3 +# Copyright (c) 2020-2021 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +''' +Test script for symbol-check.py +''' +import os +import subprocess +from typing import List +import unittest + +from utils import determine_wellknown_cmd + +def call_symbol_check(cc: List[str], source, executable, options): + # This should behave the same as AC_TRY_LINK, so arrange well-known flags + # in the same order as autoconf would. + # + # See the definitions for ac_link in autoconf's lib/autoconf/c.m4 file for + # reference. + env_flags: List[str] = [] + for var in ['CFLAGS', 'CPPFLAGS', 'LDFLAGS']: + env_flags += filter(None, os.environ.get(var, '').split(' ')) + + subprocess.run([*cc,source,'-o',executable] + env_flags + options, check=True) + p = subprocess.run(['./contrib/devtools/symbol-check.py',executable], stdout=subprocess.PIPE, universal_newlines=True) + os.remove(source) + os.remove(executable) + return (p.returncode, p.stdout.rstrip()) + +def get_machine(cc: List[str]): + p = subprocess.run([*cc,'-dumpmachine'], stdout=subprocess.PIPE, universal_newlines=True) + return p.stdout.rstrip() + +class TestSymbolChecks(unittest.TestCase): + def test_ELF(self): + source = 'test1.c' + executable = 'test1' + cc = determine_wellknown_cmd('CC', 'gcc') + + # there's no way to do this test for RISC-V at the moment; we build for + # RISC-V in a glibc 2.27 environment and we allow all symbols from 2.27. + if 'riscv' in get_machine(cc): + self.skipTest("test not available for RISC-V") + + # nextup was introduced in GLIBC 2.24, so is newer than our supported + # glibc (2.18), and available in our release build environment (2.24). + with open(source, 'w', encoding="utf8") as f: + f.write(''' + #define _GNU_SOURCE + #include <math.h> + + double nextup(double x); + + int main() + { + nextup(3.14); + return 0; + } + ''') + + self.assertEqual(call_symbol_check(cc, source, executable, ['-lm']), + (1, executable + ': symbol nextup from unsupported version GLIBC_2.24(3)\n' + + executable + ': failed IMPORTED_SYMBOLS')) + + # -lutil is part of the libc6 package so a safe bet that it's installed + # it's also out of context enough that it's unlikely to ever become a real dependency + source = 'test2.c' + executable = 'test2' + with open(source, 'w', encoding="utf8") as f: + f.write(''' + #include <utmp.h> + + int main() + { + login(0); + return 0; + } + ''') + + self.assertEqual(call_symbol_check(cc, source, executable, ['-lutil']), + (1, executable + ': libutil.so.1 is not in ALLOWED_LIBRARIES!\n' + + executable + ': failed LIBRARY_DEPENDENCIES')) + + # finally, check a simple conforming binary + source = 'test3.c' + executable = 'test3' + with open(source, 'w', encoding="utf8") as f: + f.write(''' + #include <stdio.h> + + int main() + { + printf("42"); + return 0; + } + ''') + + self.assertEqual(call_symbol_check(cc, source, executable, []), + (0, '')) + + def test_MACHO(self): + source = 'test1.c' + executable = 'test1' + cc = determine_wellknown_cmd('CC', 'clang') + + with open(source, 'w', encoding="utf8") as f: + f.write(''' + #include <expat.h> + + int main() + { + XML_ExpatVersion(); + return 0; + } + + ''') + + self.assertEqual(call_symbol_check(cc, source, executable, ['-lexpat', '-Wl,-platform_version','-Wl,macos', '-Wl,11.4', '-Wl,11.4']), + (1, 'libexpat.1.dylib is not in ALLOWED_LIBRARIES!\n' + + f'{executable}: failed DYNAMIC_LIBRARIES MIN_OS SDK')) + + source = 'test2.c' + executable = 'test2' + with open(source, 'w', encoding="utf8") as f: + f.write(''' + #include <CoreGraphics/CoreGraphics.h> + + int main() + { + CGMainDisplayID(); + return 0; + } + ''') + + self.assertEqual(call_symbol_check(cc, source, executable, ['-framework', 'CoreGraphics', '-Wl,-platform_version','-Wl,macos', '-Wl,11.4', '-Wl,11.4']), + (1, f'{executable}: failed MIN_OS SDK')) + + source = 'test3.c' + executable = 'test3' + with open(source, 'w', encoding="utf8") as f: + f.write(''' + int main() + { + return 0; + } + ''') + + self.assertEqual(call_symbol_check(cc, source, executable, ['-Wl,-platform_version','-Wl,macos', '-Wl,10.15', '-Wl,11.4']), + (1, f'{executable}: failed SDK')) + + def test_PE(self): + source = 'test1.c' + executable = 'test1.exe' + cc = determine_wellknown_cmd('CC', 'x86_64-w64-mingw32-gcc') + + with open(source, 'w', encoding="utf8") as f: + f.write(''' + #include <pdh.h> + + int main() + { + PdhConnectMachineA(NULL); + return 0; + } + ''') + + self.assertEqual(call_symbol_check(cc, source, executable, ['-lpdh', '-Wl,--major-subsystem-version', '-Wl,6', '-Wl,--minor-subsystem-version', '-Wl,1']), + (1, 'pdh.dll is not in ALLOWED_LIBRARIES!\n' + + executable + ': failed DYNAMIC_LIBRARIES')) + + source = 'test2.c' + executable = 'test2.exe' + + with open(source, 'w', encoding="utf8") as f: + f.write(''' + int main() + { + return 0; + } + ''') + + self.assertEqual(call_symbol_check(cc, source, executable, ['-Wl,--major-subsystem-version', '-Wl,9', '-Wl,--minor-subsystem-version', '-Wl,9']), + (1, executable + ': failed SUBSYSTEM_VERSION')) + + source = 'test3.c' + executable = 'test3.exe' + with open(source, 'w', encoding="utf8") as f: + f.write(''' + #include <combaseapi.h> + + int main() + { + CoFreeUnusedLibrariesEx(0,0); + return 0; + } + ''') + + self.assertEqual(call_symbol_check(cc, source, executable, ['-lole32', '-Wl,--major-subsystem-version', '-Wl,6', '-Wl,--minor-subsystem-version', '-Wl,1']), + (0, '')) + + +if __name__ == '__main__': + unittest.main() diff --git a/contrib/devtools/test_deterministic_coverage.sh b/contrib/devtools/test_deterministic_coverage.sh new file mode 100755 index 0000000000..8501c72f04 --- /dev/null +++ b/contrib/devtools/test_deterministic_coverage.sh @@ -0,0 +1,151 @@ +#!/usr/bin/env bash +# +# Copyright (c) 2019-2020 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +# +# Test for deterministic coverage across unit test runs. + +export LC_ALL=C + +# Use GCOV_EXECUTABLE="gcov" if compiling with gcc. +# Use GCOV_EXECUTABLE="llvm-cov gcov" if compiling with clang. +GCOV_EXECUTABLE="gcov" + +# Disable tests known to cause non-deterministic behaviour and document the source or point of non-determinism. +NON_DETERMINISTIC_TESTS=( + "blockfilter_index_tests/blockfilter_index_initial_sync" # src/checkqueue.h: In CCheckQueue::Loop(): while (queue.empty()) { ... } + "coinselector_tests/knapsack_solver_test" # coinselector_tests.cpp: if (equal_sets(setCoinsRet, setCoinsRet2)) + "fs_tests/fsbridge_fstream" # deterministic test failure? + "miner_tests/CreateNewBlock_validity" # validation.cpp: if (GetMainSignals().CallbacksPending() > 10) + "scheduler_tests/manythreads" # scheduler.cpp: CScheduler::serviceQueue() + "scheduler_tests/singlethreadedscheduler_ordered" # scheduler.cpp: CScheduler::serviceQueue() + "txvalidationcache_tests/checkinputs_test" # validation.cpp: if (GetMainSignals().CallbacksPending() > 10) + "txvalidationcache_tests/tx_mempool_block_doublespend" # validation.cpp: if (GetMainSignals().CallbacksPending() > 10) + "txindex_tests/txindex_initial_sync" # validation.cpp: if (GetMainSignals().CallbacksPending() > 10) + "txvalidation_tests/tx_mempool_reject_coinbase" # validation.cpp: if (GetMainSignals().CallbacksPending() > 10) + "validation_block_tests/processnewblock_signals_ordering" # validation.cpp: if (GetMainSignals().CallbacksPending() > 10) + "wallet_tests/coin_mark_dirty_immature_credit" # validation.cpp: if (GetMainSignals().CallbacksPending() > 10) + "wallet_tests/dummy_input_size_test" # validation.cpp: if (GetMainSignals().CallbacksPending() > 10) + "wallet_tests/importmulti_rescan" # validation.cpp: if (GetMainSignals().CallbacksPending() > 10) + "wallet_tests/importwallet_rescan" # validation.cpp: if (GetMainSignals().CallbacksPending() > 10) + "wallet_tests/ListCoins" # validation.cpp: if (GetMainSignals().CallbacksPending() > 10) + "wallet_tests/scan_for_wallet_transactions" # validation.cpp: if (GetMainSignals().CallbacksPending() > 10) + "wallet_tests/wallet_disableprivkeys" # validation.cpp: if (GetMainSignals().CallbacksPending() > 10) +) + +TEST_BITCOIN_BINARY="src/test/test_bitcoin" + +print_usage() { + echo "Usage: $0 [custom test filter (default: all but known non-deterministic tests)] [number of test runs (default: 2)]" +} + +N_TEST_RUNS=2 +BOOST_TEST_RUN_FILTERS="" +if [[ $# != 0 ]]; then + if [[ $1 == "--help" ]]; then + print_usage + exit + fi + PARSED_ARGUMENTS=0 + if [[ $1 =~ [a-z] ]]; then + BOOST_TEST_RUN_FILTERS=$1 + PARSED_ARGUMENTS=$((PARSED_ARGUMENTS + 1)) + shift + fi + if [[ $1 =~ ^[0-9]+$ ]]; then + N_TEST_RUNS=$1 + PARSED_ARGUMENTS=$((PARSED_ARGUMENTS + 1)) + shift + fi + if [[ ${PARSED_ARGUMENTS} == 0 || $# -gt 2 || ${N_TEST_RUNS} -lt 2 ]]; then + print_usage + exit + fi +fi +if [[ ${BOOST_TEST_RUN_FILTERS} == "" ]]; then + BOOST_TEST_RUN_FILTERS="$(IFS=":"; echo "!${NON_DETERMINISTIC_TESTS[*]}" | sed 's/:/:!/g')" +else + echo "Using Boost test filter: ${BOOST_TEST_RUN_FILTERS}" + echo +fi + +if ! command -v gcov > /dev/null; then + echo "Error: gcov not installed. Exiting." + exit 1 +fi + +if ! command -v gcovr > /dev/null; then + echo "Error: gcovr not installed. Exiting." + exit 1 +fi + +if [[ ! -e ${TEST_BITCOIN_BINARY} ]]; then + echo "Error: Executable ${TEST_BITCOIN_BINARY} not found. Run \"./configure --enable-lcov\" and compile." + exit 1 +fi + +get_file_suffix_count() { + find src/ -type f -name "*.$1" | wc -l +} + +if [[ $(get_file_suffix_count gcno) == 0 ]]; then + echo "Error: Could not find any *.gcno files. The *.gcno files are generated by the compiler. Run \"./configure --enable-lcov\" and re-compile." + exit 1 +fi + +get_covr_filename() { + echo "gcovr.run-$1.txt" +} + +TEST_RUN_ID=0 +while [[ ${TEST_RUN_ID} -lt ${N_TEST_RUNS} ]]; do + TEST_RUN_ID=$((TEST_RUN_ID + 1)) + echo "[$(date +"%Y-%m-%d %H:%M:%S")] Measuring coverage, run #${TEST_RUN_ID} of ${N_TEST_RUNS}" + find src/ -type f -name "*.gcda" -exec rm {} \; + if [[ $(get_file_suffix_count gcda) != 0 ]]; then + echo "Error: Stale *.gcda files found. Exiting." + exit 1 + fi + TEST_OUTPUT_TEMPFILE=$(mktemp) + if ! BOOST_TEST_RUN_FILTERS="${BOOST_TEST_RUN_FILTERS}" ${TEST_BITCOIN_BINARY} > "${TEST_OUTPUT_TEMPFILE}" 2>&1; then + cat "${TEST_OUTPUT_TEMPFILE}" + rm "${TEST_OUTPUT_TEMPFILE}" + exit 1 + fi + rm "${TEST_OUTPUT_TEMPFILE}" + if [[ $(get_file_suffix_count gcda) == 0 ]]; then + echo "Error: Running the test suite did not create any *.gcda files. The gcda files are generated when the instrumented test programs are executed. Run \"./configure --enable-lcov\" and re-compile." + exit 1 + fi + GCOVR_TEMPFILE=$(mktemp) + if ! gcovr --gcov-executable "${GCOV_EXECUTABLE}" -r src/ > "${GCOVR_TEMPFILE}"; then + echo "Error: gcovr failed. Output written to ${GCOVR_TEMPFILE}. Exiting." + exit 1 + fi + GCOVR_FILENAME=$(get_covr_filename ${TEST_RUN_ID}) + mv "${GCOVR_TEMPFILE}" "${GCOVR_FILENAME}" + if grep -E "^TOTAL *0 *0 " "${GCOVR_FILENAME}"; then + echo "Error: Spurious gcovr output. Make sure the correct GCOV_EXECUTABLE variable is set in $0 (\"gcov\" for gcc, \"llvm-cov gcov\" for clang)." + exit 1 + fi + if [[ ${TEST_RUN_ID} != 1 ]]; then + COVERAGE_DIFF=$(diff -u "$(get_covr_filename 1)" "${GCOVR_FILENAME}") + if [[ ${COVERAGE_DIFF} != "" ]]; then + echo + echo "The line coverage is non-deterministic between runs. Exiting." + echo + echo "The test suite must be deterministic in the sense that the set of lines executed at least" + echo "once must be identical between runs. This is a necessary condition for meaningful" + echo "coverage measuring." + echo + echo "${COVERAGE_DIFF}" + exit 1 + fi + rm "${GCOVR_FILENAME}" + fi +done + +echo +echo "Coverage test passed: Deterministic coverage across ${N_TEST_RUNS} runs." +exit diff --git a/contrib/devtools/utils.py b/contrib/devtools/utils.py new file mode 100755 index 0000000000..68ad1c3aba --- /dev/null +++ b/contrib/devtools/utils.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python3 +# Copyright (c) 2021 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +''' +Common utility functions +''' +import shutil +import sys +import os +from typing import List + + +def determine_wellknown_cmd(envvar, progname) -> List[str]: + maybe_env = os.getenv(envvar) + maybe_which = shutil.which(progname) + if maybe_env: + return maybe_env.split(' ') # Well-known vars are often meant to be word-split + elif maybe_which: + return [ maybe_which ] + else: + sys.exit(f"{progname} not found") diff --git a/contrib/devtools/utxo_snapshot.sh b/contrib/devtools/utxo_snapshot.sh new file mode 100755 index 0000000000..dee25ff67b --- /dev/null +++ b/contrib/devtools/utxo_snapshot.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash +# +# Copyright (c) 2019 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +# +export LC_ALL=C + +set -ueo pipefail + +if (( $# < 3 )); then + echo 'Usage: utxo_snapshot.sh <generate-at-height> <snapshot-out-path> <bitcoin-cli-call ...>' + echo + echo " if <snapshot-out-path> is '-', don't produce a snapshot file but instead print the " + echo " expected assumeutxo hash" + echo + echo 'Examples:' + echo + echo " ./contrib/devtools/utxo_snapshot.sh 570000 utxo.dat ./src/bitcoin-cli -datadir=\$(pwd)/testdata" + echo ' ./contrib/devtools/utxo_snapshot.sh 570000 - ./src/bitcoin-cli' + exit 1 +fi + +GENERATE_AT_HEIGHT="${1}"; shift; +OUTPUT_PATH="${1}"; shift; +# Most of the calls we make take a while to run, so pad with a lengthy timeout. +BITCOIN_CLI_CALL="${*} -rpcclienttimeout=9999999" + +# Block we'll invalidate/reconsider to rewind/fast-forward the chain. +PIVOT_BLOCKHASH=$($BITCOIN_CLI_CALL getblockhash $(( GENERATE_AT_HEIGHT + 1 )) ) + +(>&2 echo "Rewinding chain back to height ${GENERATE_AT_HEIGHT} (by invalidating ${PIVOT_BLOCKHASH}); this may take a while") +${BITCOIN_CLI_CALL} invalidateblock "${PIVOT_BLOCKHASH}" + +if [[ "${OUTPUT_PATH}" = "-" ]]; then + (>&2 echo "Generating txoutset info...") + ${BITCOIN_CLI_CALL} gettxoutsetinfo | grep hash_serialized_2 | sed 's/^.*: "\(.\+\)\+",/\1/g' +else + (>&2 echo "Generating UTXO snapshot...") + ${BITCOIN_CLI_CALL} dumptxoutset "${OUTPUT_PATH}" +fi + +(>&2 echo "Restoring chain to original height; this may take a while") +${BITCOIN_CLI_CALL} reconsiderblock "${PIVOT_BLOCKHASH}" diff --git a/contrib/filter-lcov.py b/contrib/filter-lcov.py new file mode 100755 index 0000000000..db780ad53b --- /dev/null +++ b/contrib/filter-lcov.py @@ -0,0 +1,28 @@ +#!/usr/bin/env python3 +# Copyright (c) 2017-2020 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +import argparse + +parser = argparse.ArgumentParser(description='Remove the coverage data from a tracefile for all files matching the pattern.') +parser.add_argument('--pattern', '-p', action='append', help='the pattern of files to remove', required=True) +parser.add_argument('tracefile', help='the tracefile to remove the coverage data from') +parser.add_argument('outfile', help='filename for the output to be written to') + +args = parser.parse_args() +tracefile = args.tracefile +pattern = args.pattern +outfile = args.outfile + +in_remove = False +with open(tracefile, 'r', encoding="utf8") as f: + with open(outfile, 'w', encoding="utf8") as wf: + for line in f: + for p in pattern: + if line.startswith("SF:") and p in line: + in_remove = True + if not in_remove: + wf.write(line) + if line == 'end_of_record\n': + in_remove = False diff --git a/contrib/guix/INSTALL.md b/contrib/guix/INSTALL.md new file mode 100644 index 0000000000..68aae18731 --- /dev/null +++ b/contrib/guix/INSTALL.md @@ -0,0 +1,801 @@ +# Guix Installation and Setup + +This only needs to be done once per machine. If you have already completed the +installation and setup, please proceed to [perform a build](./README.md). + +Otherwise, you may choose from one of the following options to install Guix: + +1. Using the official **shell installer script** [⤓ skip to section][install-script] + - Maintained by Guix developers + - Easiest (automatically performs *most* setup) + - Works on nearly all Linux distributions + - Only installs latest release + - Binary installation only, requires high level of trust + - Note: The script needs to be run as root, so it should be inspected before it's run +2. Using the official **binary tarball** [⤓ skip to section][install-bin-tarball] + - Maintained by Guix developers + - Normal difficulty (full manual setup required) + - Works on nearly all Linux distributions + - Installs any release + - Binary installation only, requires high level of trust +3. Using fanquake's **Docker image** [↗︎ external instructions][install-fanquake-docker] + - Maintained by fanquake + - Easy (automatically performs *some* setup) + - Works wherever Docker images work + - Installs any release + - Binary installation only, requires high level of trust +4. Using a **distribution-maintained package** [⤓ skip to section][install-distro-pkg] + - Maintained by distribution's Guix package maintainer + - Normal difficulty (manual setup required) + - Works only on distributions with Guix packaged, see: https://repology.org/project/guix/versions + - Installs a release decided on by package maintainer + - Source or binary installation depending on the distribution +5. Building **from source** [⤓ skip to section][install-source] + - Maintained by you + - Hard, but rewarding + - Can be made to work on most Linux distributions + - Installs any commit (more granular) + - Source installation, requires lower level of trust + +## Options 1 and 2: Using the official shell installer script or binary tarball + +The installation instructions for both the official shell installer script and +the binary tarballs can be found in the GNU Guix Manual's [Binary Installation +section](https://guix.gnu.org/manual/en/html_node/Binary-Installation.html). + +Note that running through the binary tarball installation steps is largely +equivalent to manually performing what the shell installer script does. + +Note that at the time of writing (July 5th, 2021), the shell installer script +automatically creates an `/etc/profile.d` entry which the binary tarball +installation instructions do not ask you to create. However, you will likely +need this entry for better desktop integration. Please see [this +section](#add-an-etcprofiled-entry) for instructions on how to add a +`/etc/profile.d/guix.sh` entry. + +Regardless of which installation option you chose, the changes to +`/etc/profile.d` will not take effect until the next shell or desktop session, +so you should log out and log back in. + +## Option 3: Using fanquake's Docker image + +Please refer to fanquake's instructions +[here](https://github.com/fanquake/core-review/tree/master/guix). + +Note that the `Dockerfile` is largely equivalent to running through the binary +tarball installation steps. + +## Option 4: Using a distribution-maintained package + +Note that this section is based on the distro packaging situation at the time of +writing (July 2021). Guix is expected to be more widely packaged over time. For +an up-to-date view on Guix's package status/version across distros, please see: +https://repology.org/project/guix/versions + +### Debian 11 (Bullseye)/Ubuntu 21.04 (Hirsute Hippo) + +Guix v1.2.0 is available as a distribution package starting in [Debian +11](https://packages.debian.org/bullseye/guix) and [Ubuntu +21.04](https://packages.ubuntu.com/hirsute/guix). + +Note that if you intend on using Guix without using any substitutes (more +details [here][security-model]), v1.2.0 has a known problem when building GnuTLS +from source. Solutions and workarounds are documented +[here](#gnutls-test-suite-fail-status-request-revoked). + + +To install: +```sh +sudo apt install guix +``` + +For up-to-date information on Debian and Ubuntu's release history: +- [Debian release history](https://www.debian.org/releases/) +- [Ubuntu release history](https://ubuntu.com/about/release-cycle) + +### Arch Linux + +Guix is available in the AUR as +[`guix`](https://aur.archlinux.org/packages/guix/), please follow the +installation instructions in the Arch Linux Wiki ([live +link](https://wiki.archlinux.org/index.php/Guix#AUR_Package_Installation), +[2021/03/30 +permalink](https://wiki.archlinux.org/index.php?title=Guix&oldid=637559#AUR_Package_Installation)) +to install Guix. + +At the time of writing (2021/03/30), the `check` phase will fail if the path to +guix's build directory is longer than 36 characters due to an anachronistic +character limit on the shebang line. Since the `check` phase happens after the +`build` phase, which may take quite a long time, it is recommended that users +either: + +1. Skip the `check` phase + - For `makepkg`: `makepkg --nocheck ...` + - For `yay`: `yay --mflags="--nocheck" ...` + - For `paru`: `paru --nocheck ...` +2. Or, check their build directory's length beforehand + - For those building with `makepkg`: `pwd | wc -c` + +## Option 5: Building from source + +Building Guix from source is a rather involved process but a rewarding one for +those looking to minimize trust and maximize customizability (e.g. building a +particular commit of Guix). Previous experience with using autotools-style build +systems to build packages from source will be helpful. *hic sunt dracones.* + +I strongly urge you to at least skim through the entire section once before you +start issuing commands, as it will save you a lot of unnecessary pain and +anguish. + +### Installing common build tools + +There are a few basic build tools that are required for most things we'll build, +so let's install them now: + +Text transformation/i18n: +- `autopoint` (sometimes packaged in `gettext`) +- `help2man` +- `po4a` +- `texinfo` + +Build system tools: +- `g++` w/ C++11 support +- `libtool` +- `autoconf` +- `automake` +- `pkg-config` (sometimes packaged as `pkgconf`) +- `make` +- `cmake` + +Miscellaneous: +- `git` +- `gnupg` +- `python3` + +### Building and Installing Guix's dependencies + +In order to build Guix itself from source, we need to first make sure that the +necessary dependencies are installed and discoverable. The most up-to-date list +of Guix's dependencies is kept in the ["Requirements" +section](https://guix.gnu.org/manual/en/html_node/Requirements.html) of the Guix +Reference Manual. + +Depending on your distribution, most or all of these dependencies may already be +packaged and installable without manually building and installing. + +For reference, the graphic below outlines Guix v1.3.0's dependency graph: + + + +#### Guile + +##### Choosing a Guile version and sticking to it + +One of the first things you need to decide is which Guile version you want to +use: Guile v2.2 or Guile v3.0. Unlike the python2 to python3 transition, Guile +v2.2 and Guile v3.0 are largely compatible, as evidenced by the fact that most +Guile packages and even [Guix +itself](https://guix.gnu.org/en/blog/2020/guile-3-and-guix/) support running on +both. + +What is important here is that you **choose one**, and you **remain consistent** +with your choice throughout **all Guile-related packages**, no matter if they +are installed via the distribution's package manager or installed from source. +This is because the files for Guile packages are installed to directories which +are separated based on the Guile version. + +###### Example: Checking that Ubuntu's `guile-git` is compatible with your chosen Guile version + +On Ubuntu Focal: + +```sh +$ apt show guile-git +Package: guile-git +... +Depends: guile-2.2, guile-bytestructures, libgit2-dev +... +``` + +As you can see, the package `guile-git` depends on `guile-2.2`, meaning that it +was likely built for Guile v2.2. This means that if you decided to use Guile +v3.0 on Ubuntu Focal, you would need to build guile-git from source instead of +using the distribution package. + +On Ubuntu Hirsute: + +```sh +$ apt show guile-git +Package: guile-git +... +Depends: guile-3.0 | guile-2.2, guile-bytestructures (>= 1.0.7-3~), libgit2-dev (>= 1.0) +... +``` + +In this case, `guile-git` depends on either `guile-3.0` or `guile-2.2`, meaning +that it would work no matter what Guile version you decided to use. + +###### Corner case: Multiple versions of Guile on one system + +It is recommended to only install one version of Guile, so that build systems do +not get confused about which Guile to use. + +However, if you insist on having both Guile v2.2 and Guile v3.0 installed on +your system, then you need to **consistently** specify one of +`GUILE_EFFECTIVE_VERSION=3.0` or `GUILE_EFFECTIVE_VERSION=2.2` to all +`./configure` invocations for Guix and its dependencies. + +##### Installing Guile + +Guile is most likely already packaged for your distribution, so after you have +[chosen a Guile version](#choosing-a-guile-version-and-sticking-to-it), install +it via your distribution's package manager. + +If your distribution splits packages into `-dev`-suffixed and +non-`-dev`-suffixed sub-packages (as is the case for Debian-derived +distributions), please make sure to install both. For example, to install Guile +v2.2 on Debian/Ubuntu: + +```sh +apt install guile-2.2 guile-2.2-dev +``` + +#### Mixing distribution packages and source-built packages + +At the time of writing, most distributions have _some_ of Guix's dependencies +packaged, but not all. This means that you may want to install the distribution +package for some dependencies, and manually build-from-source for others. + +Distribution packages usually install to `/usr`, which is different from the +default `./configure` prefix of source-built packages: `/usr/local`. + +This means that if you mix-and-match distribution packages and source-built +packages and do not specify exactly `--prefix=/usr` to `./configure` for +source-built packages, you will need to augment the `GUILE_LOAD_PATH` and +`GUILE_LOAD_COMPILED_PATH` environment variables so that Guile will look +under the right prefix and find your source-built packages. + +For example, if you are using Guile v2.2, and have Guile packages in the +`/usr/local` prefix, either add the following lines to your `.profile` or +`.bash_profile` so that the environment variable is properly set for all future +shell logins, or paste the lines into a POSIX-style shell to temporarily modify +the environment variables of your current shell session. + +```sh +# Help Guile v2.2.x find packages in /usr/local +export GUILE_LOAD_PATH="/usr/local/share/guile/site/2.2${GUILE_LOAD_PATH:+:}$GUILE_LOAD_PATH" +export GUILE_LOAD_COMPILED_PATH="/usr/local/lib/guile/2.2/site-ccache${GUILE_LOAD_COMPILED_PATH:+:}$GUILE_COMPILED_LOAD_PATH" +``` + +Note that these environment variables are used to check for packages during +`./configure`, so they should be set as soon as possible should you want to use +a prefix other than `/usr`. + +#### Building and installing source-built packages + +***IMPORTANT**: A few dependencies have non-obvious quirks/errata which are +documented in the sub-sections immediately below. Please read these sections +before proceeding to build and install these packages.* + +Although you should always refer to the README or INSTALL files for the most +accurate information, most of these dependencies use autoconf-style build +systems (check if there's a `configure.ac` file), and will likely do the right +thing with the following: + +Clone the repository and check out the latest release: +```sh +git clone <git-repo-of-dependency>/<dependency>.git +cd <dependency> +git tag -l # check for the latest release +git checkout <latest-release> +``` + +For autoconf-based build systems (if `./autogen.sh` or `configure.ac` exists at +the root of the repository): + +```sh +./autogen.sh || autoreconf -vfi +./configure --prefix=<prefix> +make +sudo make install +``` + +For CMake-based build systems (if `CMakeLists.txt` exists at the root of the +repository): + +```sh +mkdir build && cd build +cmake .. -DCMAKE_INSTALL_PREFIX=<prefix> +sudo cmake --build . --target install +``` + +If you choose not to specify exactly `--prefix=/usr` to `./configure`, please +make sure you've carefully read the [previous section] on mixing distribution +packages and source-built packages. + +##### Binding packages require `-dev`-suffixed packages + +Relevant for: +- Everyone + +When building bindings, the `-dev`-suffixed version of the original package +needs to be installed. For example, building `Guile-zlib` on Debian-derived +distributions requires that `zlib1g-dev` is installed. + +When using bindings, the `-dev`-suffixed version of the original package still +needs to be installed. This is particularly problematic when distribution +packages are mispackaged like `guile-sqlite3` is in Ubuntu Focal such that +installing `guile-sqlite3` does not automatically install `libsqlite3-dev` as a +dependency. + +Below is a list of relevant Guile bindings and their corresponding `-dev` +packages in Debian at the time of writing. + +| Guile binding package | -dev Debian package | +|-----------------------|---------------------| +| guile-gcrypt | libgcrypt-dev | +| guile-git | libgit2-dev | +| guile-lzlib | liblz-dev | +| guile-ssh | libssh-dev | +| guile-sqlite3 | libsqlite3-dev | +| guile-zlib | zlib1g-dev | + +##### `guile-git` actually depends on `libgit2 >= 1.1` + +Relevant for: +- Those building `guile-git` from source against `libgit2 < 1.1` +- Those installing `guile-git` from their distribution where `guile-git` is + built against `libgit2 < 1.1` + +As of v0.4.0, `guile-git` claims to only require `libgit2 >= 0.28.0`, however, +it actually requires `libgit2 >= 1.1`, otherwise, it will be confused by a +reference of `origin/keyring`: instead of interpreting the reference as "the +'keyring' branch of the 'origin' remote", the reference is interpreted as "the +branch literally named 'origin/keyring'" + +This is especially notable because Ubuntu Focal packages `libgit2 v0.28.4`, and +`guile-git` is built against it. + +Should you be in this situation, you need to build both `libgit2 v1.1.x` and +`guile-git` from source. + +Source: https://logs.guix.gnu.org/guix/2020-11-12.log#232527 + +##### `{scheme,guile}-bytestructures` v1.0.8 and v1.0.9 are broken for Guile v2.2 + +Relevant for: +- Those building `{scheme,guile}-bytestructures` from source against Guile v2.2 + +Commit +[707eea3](https://github.com/TaylanUB/scheme-bytestructures/commit/707eea3a85e1e375e86702229ebf73d496377669) +introduced a regression for Guile v2.2 and was first included in v1.0.8, this +was later corrected in commit +[ec9a721](https://github.com/TaylanUB/scheme-bytestructures/commit/ec9a721957c17bcda13148f8faa5f06934431ff7) +and included in v1.1.0. + +TL;DR If you decided to use Guile v2.2, do not use `{scheme,guile}-bytestructures` v1.0.8 or v1.0.9. + +### Building and Installing Guix itself + +Start by cloning Guix: + +``` +git clone https://git.savannah.gnu.org/git/guix.git +cd guix +``` + +You will likely want to build the latest release, however, if the latest release +when you're reading this is still 1.2.0 then you may want to use 95aca29 instead +to avoid a problem in the GnuTLS test suite. + +``` +git branch -a -l 'origin/version-*' # check for the latest release +git checkout <latest-release> +``` + +Bootstrap the build system: +``` +./bootstrap +``` + +Configure with the recommended `--localstatedir` flag: +``` +./configure --localstatedir=/var +``` + +Note: If you intend to hack on Guix in the future, you will need to supply the +same `--localstatedir=` flag for all future Guix `./configure` invocations. See +the last paragraph of this +[section](https://guix.gnu.org/manual/en/html_node/Requirements.html) for more +details. + +Build Guix (this will take a while): +``` +make -j$(nproc) +``` + +Install Guix: + +``` +sudo make install +``` + +### Post-"build from source" Setup + +#### Creating and starting a `guix-daemon-original` service with a fixed `argv[0]` + +At this point, guix will be installed to `${bindir}`, which is likely +`/usr/local/bin` if you did not override directory variables at +`./configure`-time. More information on standard Automake directory variables +can be found +[here](https://www.gnu.org/software/automake/manual/html_node/Standard-Directory-Variables.html). + +However, the Guix init scripts and service configurations for Upstart, systemd, +SysV, and OpenRC are installed (in `${libdir}`) to launch +`${localstatedir}/guix/profiles/per-user/root/current-guix/bin/guix-daemon`, +which does not yet exist, and will only exist after [`root` performs their first +`guix pull`](#guix-pull-as-root). + +We need to create a `-original` version of these init scripts that's pointed to +the binaries we just built and `make install`'ed in `${bindir}` (normally, +`/usr/local/bin`). + +Example for `systemd`, run as `root`: + +```sh +# Create guix-daemon-original.service by modifying guix-daemon.service +libdir=# set according to your PREFIX (default is /usr/local/lib) +bindir="$(dirname $(command -v guix-daemon))" +sed -E -e "s|/\S*/guix/profiles/per-user/root/current-guix/bin/guix-daemon|${bindir}/guix-daemon|" "${libdir}"/systemd/system/guix-daemon.service > /etc/systemd/system/guix-daemon-original.service +chmod 664 /etc/systemd/system/guix-daemon-original.service + +# Make systemd recognize the new service +systemctl daemon-reload + +# Make sure that the non-working guix-daemon.service is stopped and disabled +systemctl stop guix-daemon +systemctl disable guix-daemon + +# Make sure that the working guix-daemon-original.service is started and enabled +systemctl enable guix-daemon-original +systemctl start guix-daemon-original +``` + +#### Creating `guix-daemon` users / groups + +Please see the [relevant +section](https://guix.gnu.org/manual/en/html_node/Build-Environment-Setup.html) +in the Guix Reference Manual for more details. + +## Optional setup + +At this point, you are set up to [use Guix to build Bitcoin +Core](./README.md#usage). However, if you want to polish your setup a bit and +make it "what Guix intended", then read the next few subsections. + +### Add an `/etc/profile.d` entry + +This section definitely does not apply to you if you installed Guix using: +1. The shell installer script +2. fanquake's Docker image +3. Debian's `guix` package + +#### Background + +Although Guix knows how to update itself and its packages, it does so in a +non-invasive way (it does not modify `/usr/local/bin/guix`). + +Instead, it does the following: + +- After a `guix pull`, it updates + `/var/guix/profiles/per-user/$USER/current-guix`, and creates a symlink + targeting this directory at `$HOME/.config/guix/current` + +- After a `guix install`, it updates + `/var/guix/profiles/per-user/$USER/guix-profile`, and creates a symlink + targeting this directory at `$HOME/.guix-profile` + +Therefore, in order for these operations to affect your shell/desktop sessions +(and for the principle of least astonishment to hold), their corresponding +directories have to be added to well-known environment variables like `$PATH`, +`$INFOPATH`, `$XDG_DATA_DIRS`, etc. + +In other words, if `$HOME/.config/guix/current/bin` does not exist in your +`$PATH`, a `guix pull` will have no effect on what `guix` you are using. Same +goes for `$HOME/.guix-profile/bin`, `guix install`, and installed packages. + +Helpfully, after a `guix pull` or `guix install`, a message will be printed like +so: + +``` +hint: Consider setting the necessary environment variables by running: + + GUIX_PROFILE="$HOME/.guix-profile" + . "$GUIX_PROFILE/etc/profile" + +Alternately, see `guix package --search-paths -p "$HOME/.guix-profile"'. +``` + +However, this is somewhat tedious to do for both `guix pull` and `guix install` +for each user on the system that wants to properly use `guix`. I recommend that +you instead add an entry to `/etc/profile.d` instead. This is done by default +when installing the Debian package later than 1.2.0-4 and when using the shell +script installer. + +#### Instructions + +Create `/etc/profile.d/guix.sh` with the following content: +```sh +# _GUIX_PROFILE: `guix pull` profile +_GUIX_PROFILE="$HOME/.config/guix/current" +if [ -L $_GUIX_PROFILE ]; then + export PATH="$_GUIX_PROFILE/bin${PATH:+:}$PATH" + # Export INFOPATH so that the updated info pages can be found + # and read by both /usr/bin/info and/or $GUIX_PROFILE/bin/info + # When INFOPATH is unset, add a trailing colon so that Emacs + # searches 'Info-default-directory-list'. + export INFOPATH="$_GUIX_PROFILE/share/info:$INFOPATH" +fi + +# GUIX_PROFILE: User's default profile +GUIX_PROFILE="$HOME/.guix-profile" +[ -L $GUIX_PROFILE ] || return +GUIX_LOCPATH="$GUIX_PROFILE/lib/locale" +export GUIX_PROFILE GUIX_LOCPATH + +[ -f "$GUIX_PROFILE/etc/profile" ] && . "$GUIX_PROFILE/etc/profile" + +# set XDG_DATA_DIRS to include Guix installations +export XDG_DATA_DIRS="$GUIX_PROFILE/share:${XDG_DATA_DIRS:-/usr/local/share/:/usr/share/}" +``` + +Please note that this will not take effect until the next shell or desktop +session (log out and log back in). + +### `guix pull` as root + +Before you do this, you need to read the section on [choosing your security +model][security-model] and adjust `guix` and `guix-daemon` flags according to +your choice, as invoking `guix pull` may pull substitutes from substitute +servers (which you may not want). + +As mentioned in a previous section, Guix expects +`${localstatedir}/guix/profiles/per-user/root/current-guix` to be populated with +`root`'s Guix profile, `guix pull`-ed and built by some former version of Guix. +However, this is not the case when we build from source. Therefore, we need to +perform a `guix pull` as `root`: + +```sh +sudo --login guix pull --branch=version-<latest-release-version> +# or +sudo --login guix pull --commit=<particular-commit> +``` + +`guix pull` is quite a long process (especially if you're using +`--no-substitute`). If you encounter build problems, please refer to the +[troubleshooting section](#troubleshooting). + +Note that running a bare `guix pull` with no commit or branch specified will +pull the latest commit on Guix's master branch, which is likely fine, but not +recommended. + +If you installed Guix from source, you may get an error like the following: +```sh +error: while creating symlink '/root/.config/guix/current' No such file or directory +``` +To resolve this, simply: +``` +sudo mkdir -p /root/.config/guix +``` +Then try the `guix pull` command again. + +After the `guix pull` finishes successfully, +`${localstatedir}/guix/profiles/per-user/root/current-guix` should be populated. + +#### Using the newly-pulled `guix` by restarting the daemon + +Depending on how you installed Guix, you should now make sure that your init +scripts and service configurations point to the newly-pulled `guix-daemon`. + +##### If you built Guix from source + +If you followed the instructions for [fixing argv\[0\]][fix-argv0], you can now +do the following: + +```sh +systemctl stop guix-daemon-original +systemctl disable guix-daemon-original + +systemctl enable guix-daemon +systemctl start guix-daemon +``` + +##### If you installed Guix via the Debian/Ubuntu distribution packages + +You will need to create a `guix-daemon-latest` service which points to the new +`guix` rather than a pinned one. + +```sh +# Create guix-daemon-latest.service by modifying guix-daemon.service +sed -E -e "s|/usr/bin/guix-daemon|/var/guix/profiles/per-user/root/current-guix/bin/guix-daemon|" /etc/systemd/system/guix-daemon.service > /lib/systemd/system/guix-daemon-latest.service +chmod 664 /lib/systemd/system/guix-daemon-latest.service + +# Make systemd recognize the new service +systemctl daemon-reload + +# Make sure that the old guix-daemon.service is stopped and disabled +systemctl stop guix-daemon +systemctl disable guix-daemon + +# Make sure that the new guix-daemon-latest.service is started and enabled +systemctl enable guix-daemon-latest +systemctl start guix-daemon-latest +``` + +##### If you installed Guix via lantw44's Arch Linux AUR package + +At the time of writing (July 5th, 2021) the systemd unit for "updated Guix" is +`guix-daemon-latest.service`, therefore, you should do the following: + +```sh +systemctl stop guix-daemon +systemctl disable guix-daemon + +systemctl enable guix-daemon-latest +systemctl start guix-daemon-latest +``` + +##### Otherwise... + +Simply do: + +```sh +systemctl restart guix-daemon +``` + +### Checking everything + +If you followed all the steps above to make your Guix setup "prim and proper," +you can check that you did everything properly by running through this +checklist. + +1. `/etc/profile.d/guix.sh` should exist and be sourced at each shell login + +2. `guix describe` should not print `guix describe: error: failed to determine + origin`, but rather something like: + + ``` + Generation 38 Feb 22 2021 16:39:31 (current) + guix f350df4 + repository URL: https://git.savannah.gnu.org/git/guix.git + branch: version-1.2.0 + commit: f350df405fbcd5b9e27e6b6aa500da7f101f41e7 + ``` + +3. `guix-daemon` should be running from `${localstatedir}/guix/profiles/per-user/root/current-guix` + +# Troubleshooting + +## Derivation failed to build + +When you see a build failure like below: + +``` +building /gnu/store/...-foo-3.6.12.drv... +/ 'check' phasenote: keeping build directory `/tmp/guix-build-foo-3.6.12.drv-0' +builder for `/gnu/store/...-foo-3.6.12.drv' failed with exit code 1 +build of /gnu/store/...-foo-3.6.12.drv failed +View build log at '/var/log/guix/drvs/../...-foo-3.6.12.drv.bz2'. +cannot build derivation `/gnu/store/...-qux-7.69.1.drv': 1 dependencies couldn't be built +cannot build derivation `/gnu/store/...-bar-3.16.5.drv': 1 dependencies couldn't be built +cannot build derivation `/gnu/store/...-baz-2.0.5.drv': 1 dependencies couldn't be built +guix time-machine: error: build of `/gnu/store/...-baz-2.0.5.drv' failed +``` + +It means that `guix` failed to build a package named `foo`, which was a +dependency of `qux`, `bar`, and `baz`. Importantly, note that the last "failed" +line is not necessarily the root cause, the first "failed" line is. + +Most of the time, the build failure is due to a spurious test failure or the +package's build system/test suite breaking when running multi-threaded. To +rebuild _just_ this derivation in a single-threaded fashion (please don't forget +to add other `guix` flags like `--no-substitutes` as appropriate): + +```sh +$ guix build --cores=1 /gnu/store/...-foo-3.6.12.drv +``` + +If the single-threaded rebuild did not succeed, you may need to dig deeper. +You may view `foo`'s build logs in `less` like so (please replace paths with the +path you see in the build failure output): + +```sh +$ bzcat /var/log/guix/drvs/../...-foo-3.6.12.drv.bz2 | less +``` + +`foo`'s build directory is also preserved and available at +`/tmp/guix-build-foo-3.6.12.drv-0`. However, if you fail to build `foo` multiple +times, it may be `/tmp/...drv-1` or `/tmp/...drv-2`. Always consult the build +failure output for the most accurate, up-to-date information. + +### python(-minimal): [Errno 84] Invalid or incomplete multibyte or wide character + +This error occurs when your `$TMPDIR` (default: /tmp) exists on a filesystem +which rejects characters not present in the UTF-8 character code set. An example +is ZFS with the utf8only=on option set. + +More information: https://bugs.python.org/issue37584 + +### GnuTLS: test-suite FAIL: status-request-revoked + +*The derivation is likely identified by: `/gnu/store/vhphki5sg9xkdhh2pbc8gi6vhpfzryf0-gnutls-3.6.12.drv`* + +This unfortunate error is most common for non-substitute builders who installed +Guix v1.2.0. The problem stems from the fact that one of GnuTLS's tests uses a +hardcoded certificate which expired on 2020-10-24. + +What's more unfortunate is that this GnuTLS derivation is somewhat special in +Guix's dependency graph and is not affected by the package transformation flags +like `--without-tests=`. + +The easiest solution for those encountering this problem is to install a newer +version of Guix. However, there are ways to work around this issue: + +#### Workaround 1: Using substitutes for this single derivation + +If you've authorized the official Guix build farm's key (more info +[here](./README.md#step-1-authorize-the-signing-keys)), then you can use +substitutes just for this single derivation by invoking the following: + +```sh +guix build --substitute-urls="https://ci.guix.gnu.org" /gnu/store/vhphki5sg9xkdhh2pbc8gi6vhpfzryf0-gnutls-3.6.12.drv +``` + +See [this section](./README.md#removing-authorized-keys) for instructions on how +to remove authorized keys if you don't want to keep the build farm's key +authorized. + +#### Workaround 2: Temporarily setting the system clock back + +This workaround was described [here](https://issues.guix.gnu.org/44559#5). + +Basically: +1. Turn off networking +2. Turn off NTP +3. Set system time to 2020-10-01 +4. guix build --no-substitutes /gnu/store/vhphki5sg9xkdhh2pbc8gi6vhpfzryf0-gnutls-3.6.12.drv +5. Set system time back to accurate current time +6. Turn NTP back on +7. Turn networking back on + +### coreutils: FAIL: tests/tail-2/inotify-dir-recreate + +The inotify-dir-create test fails on "remote" filesystems such as overlayfs +(Docker's default filesystem) due to the filesystem being mistakenly recognized +as non-remote. + +A relatively easy workaround to this is to make sure that a somewhat traditional +filesystem is mounted at `/tmp` (where `guix-daemon` performs its builds). For +Docker users, this might mean [using a volume][docker/volumes], [binding +mounting][docker/bind-mnt] from host, or (for those with enough RAM and swap) +[mounting a tmpfs][docker/tmpfs] using the `--tmpfs` flag. + +Please see the following links for more details: + +- An upstream coreutils bug has been filed: [debbugs#47940](https://debbugs.gnu.org/cgi/bugreport.cgi?bug=47940) +- A Guix bug detailing the underlying problem has been filed: [guix-issues#47935](https://issues.guix.gnu.org/47935) +- A commit to skip this test in Guix has been merged into the core-updates branch: +[savannah/guix@6ba1058](https://git.savannah.gnu.org/cgit/guix.git/commit/?id=6ba1058df0c4ce5611c2367531ae5c3cdc729ab4) + + +[install-script]: #options-1-and-2-using-the-official-shell-installer-script-or-binary-tarball +[install-bin-tarball]: #options-1-and-2-using-the-official-shell-installer-script-or-binary-tarball +[install-fanquake-docker]: #option-3-using-fanquakes-docker-image +[install-distro-pkg]: #option-4-using-a-distribution-maintained-package +[install-source]: #option-5-building-from-source + +[fix-argv0]: #creating-and-starting-a-guix-daemon-original-service-with-a-fixed-argv0 +[security-model]: ./README.md#choosing-your-security-model + +[docker/volumes]: https://docs.docker.com/storage/volumes/ +[docker/bind-mnt]: https://docs.docker.com/storage/bind-mounts/ +[docker/tmpfs]: https://docs.docker.com/storage/tmpfs/ diff --git a/contrib/guix/README.md b/contrib/guix/README.md new file mode 100644 index 0000000000..af5607c710 --- /dev/null +++ b/contrib/guix/README.md @@ -0,0 +1,484 @@ +# Bootstrappable Bitcoin Core Builds + +This directory contains the files necessary to perform bootstrappable Bitcoin +Core builds. + +[Bootstrappability][b17e] furthers our binary security guarantees by allowing us +to _audit and reproduce_ our toolchain instead of blindly _trusting_ binary +downloads. + +We achieve bootstrappability by using Guix as a functional package manager. + +# Requirements + +Conservatively, you will need an x86_64 machine with: + +- 16GB of free disk space on the partition that /gnu/store will reside in +- 8GB of free disk space **per platform triple** you're planning on building + (see the `HOSTS` [environment variable description][env-vars-list]) + +# Installation and Setup + +If you don't have Guix installed and set up, please follow the instructions in +[INSTALL.md](./INSTALL.md) + +# Usage + +If you haven't considered your security model yet, please read [the relevant +section](#choosing-your-security-model) before proceeding to perform a build. + +## Making the Xcode SDK available for macOS cross-compilation + +In order to perform a build for macOS (which is included in the default set of +platform triples to build), you'll need to extract the macOS SDK tarball using +tools found in the [`macdeploy` directory](../macdeploy/README.md). + +You can then either point to the SDK using the `SDK_PATH` environment variable: + +```sh +# Extract the SDK tarball to /path/to/parent/dir/of/extracted/SDK/Xcode-<foo>-<bar>-extracted-SDK-with-libcxx-headers +tar -C /path/to/parent/dir/of/extracted/SDK -xaf /path/to/Xcode-<foo>-<bar>-extracted-SDK-with-libcxx-headers.tar.gz + +# Indicate where to locate the SDK tarball +export SDK_PATH=/path/to/parent/dir/of/extracted/SDK +``` + +or extract it into `depends/SDKs`: + +```sh +mkdir -p depends/SDKs +tar -C depends/SDKs -xaf /path/to/SDK/tarball +``` + +## Building + +*The author highly recommends at least reading over the [common usage patterns +and examples](#common-guix-build-invocation-patterns-and-examples) section below +before starting a build. For a full list of customization options, see the +[recognized environment variables][env-vars-list] section.* + +To build Bitcoin Core reproducibly with all default options, invoke the +following from the top of a clean repository: + +```sh +./contrib/guix/guix-build +``` + +## Codesigning build outputs + +The `guix-codesign` command attaches codesignatures (produced by codesigners) to +existing non-codesigned outputs. Please see the [release process +documentation](/doc/release-process.md) for more context. + +It respects many of the same environment variable flags as `guix-build`, with 2 +crucial differences: + +1. Since only Windows and macOS build outputs require codesigning, the `HOSTS` + environment variable will have a sane default value of `x86_64-w64-mingw32 + x86_64-apple-darwin arm64-apple-darwin` instead of all the platforms. +2. The `guix-codesign` command ***requires*** a `DETACHED_SIGS_REPO` flag. + * _**DETACHED_SIGS_REPO**_ + + Set the directory where detached codesignatures can be found for the current + Bitcoin Core version being built. + + _REQUIRED environment variable_ + +An invocation with all default options would look like: + +``` +env DETACHED_SIGS_REPO=<path/to/bitcoin-detached-sigs> ./contrib/guix/guix-codesign +``` + +## Cleaning intermediate work directories + +By default, `guix-build` leaves all intermediate files or "work directories" +(e.g. `depends/work`, `guix-build-*/distsrc-*`) intact at the end of a build so +that they are available to the user (to aid in debugging, etc.). However, these +directories usually take up a large amount of disk space. Therefore, a +`guix-clean` convenience script is provided which cleans the current `git` +worktree to save disk space: + +``` +./contrib/guix/guix-clean +``` + + +## Attesting to build outputs + +Much like how Gitian build outputs are attested to in a `gitian.sigs` +repository, Guix build outputs are attested to in the [`guix.sigs` +repository](https://github.com/bitcoin-core/guix.sigs). + +After you've cloned the `guix.sigs` repository, to attest to the current +worktree's commit/tag: + +``` +env GUIX_SIGS_REPO=<path/to/guix.sigs> SIGNER=<gpg-key-name> ./contrib/guix/guix-attest +``` + +See `./contrib/guix/guix-attest --help` for more information on the various ways +`guix-attest` can be invoked. + +## Verifying build output attestations + +After at least one other signer has uploaded their signatures to the `guix.sigs` +repository: + +``` +git -C <path/to/guix.sigs> pull +env GUIX_SIGS_REPO=<path/to/guix.sigs> ./contrib/guix/guix-verify +``` + + +## Common `guix-build` invocation patterns and examples + +### Keeping caches and SDKs outside of the worktree + +If you perform a lot of builds and have a bunch of worktrees, you may find it +more efficient to keep the depends tree's download cache, build cache, and SDKs +outside of the worktrees to avoid duplicate downloads and unnecessary builds. To +help with this situation, the `guix-build` script honours the `SOURCES_PATH`, +`BASE_CACHE`, and `SDK_PATH` environment variables and will pass them on to the +depends tree so that you can do something like: + +```sh +env SOURCES_PATH="$HOME/depends-SOURCES_PATH" BASE_CACHE="$HOME/depends-BASE_CACHE" SDK_PATH="$HOME/macOS-SDKs" ./contrib/guix/guix-build +``` + +Note that the paths that these environment variables point to **must be +directories**, and **NOT symlinks to directories**. + +See the [recognized environment variables][env-vars-list] section for more +details. + +### Building a subset of platform triples + +Sometimes you only want to build a subset of the supported platform triples, in +which case you can override the default list by setting the space-separated +`HOSTS` environment variable: + +```sh +env HOSTS='x86_64-w64-mingw32 x86_64-apple-darwin' ./contrib/guix/guix-build +``` + +See the [recognized environment variables][env-vars-list] section for more +details. + +### Controlling the number of threads used by `guix` build commands + +Depending on your system's RAM capacity, you may want to decrease the number of +threads used to decrease RAM usage or vice versa. + +By default, the scripts under `./contrib/guix` will invoke all `guix` build +commands with `--cores="$JOBS"`. Note that `$JOBS` defaults to `$(nproc)` if not +specified. However, astute manual readers will also notice that `guix` build +commands also accept a `--max-jobs=` flag (which defaults to 1 if unspecified). + +Here is the difference between `--cores=` and `--max-jobs=`: + +> Note: When I say "derivation," think "package" + +`--cores=` + + - controls the number of CPU cores to build each derivation. This is the value + passed to `make`'s `--jobs=` flag. + +`--max-jobs=` + + - controls how many derivations can be built in parallel + - defaults to 1 + +Therefore, the default is for `guix` build commands to build one derivation at a +time, utilizing `$JOBS` threads. + +Specifying the `$JOBS` environment variable will only modify `--cores=`, but you +can also modify the value for `--max-jobs=` by specifying +`$ADDITIONAL_GUIX_COMMON_FLAGS`. For example, if you have a LOT of memory, you +may want to set: + +```sh +export ADDITIONAL_GUIX_COMMON_FLAGS='--max-jobs=8' +``` + +Which allows for a maximum of 8 derivations to be built at the same time, each +utilizing `$JOBS` threads. + +Or, if you'd like to avoid spurious build failures caused by issues with +parallelism within a single package, but would still like to build multiple +packages when the dependency graph allows for it, you may want to try: + +```sh +export JOBS=1 ADDITIONAL_GUIX_COMMON_FLAGS='--max-jobs=8' +``` + +See the [recognized environment variables][env-vars-list] section for more +details. + +## Recognized environment variables + +* _**HOSTS**_ + + Override the space-separated list of platform triples for which to perform a + bootstrappable build. + + _(defaults to "x86\_64-linux-gnu arm-linux-gnueabihf aarch64-linux-gnu + riscv64-linux-gnu powerpc64-linux-gnu powerpc64le-linux-gnu + x86\_64-w64-mingw32 x86\_64-apple-darwin arm64-apple-darwin")_ + +* _**SOURCES_PATH**_ + + Set the depends tree download cache for sources. This is passed through to the + depends tree. Setting this to the same directory across multiple builds of the + depends tree can eliminate unnecessary redownloading of package sources. + + The path that this environment variable points to **must be a directory**, and + **NOT a symlink to a directory**. + +* _**BASE_CACHE**_ + + Set the depends tree cache for built packages. This is passed through to the + depends tree. Setting this to the same directory across multiple builds of the + depends tree can eliminate unnecessary building of packages. + + The path that this environment variable points to **must be a directory**, and + **NOT a symlink to a directory**. + +* _**SDK_PATH**_ + + Set the path where _extracted_ SDKs can be found. This is passed through to + the depends tree. Note that this is should be set to the _parent_ directory of + the actual SDK (e.g. `SDK_PATH=$HOME/Downloads/macOS-SDKs` instead of + `$HOME/Downloads/macOS-SDKs/Xcode-12.2-12B45b-extracted-SDK-with-libcxx-headers`). + + The path that this environment variable points to **must be a directory**, and + **NOT a symlink to a directory**. + +* _**JOBS**_ + + Override the number of jobs to run simultaneously, you might want to do so on + a memory-limited machine. This may be passed to: + + - `guix` build commands as in `guix environment --cores="$JOBS"` + - `make` as in `make --jobs="$JOBS"` + - `xargs` as in `xargs -P"$JOBS"` + + See [here](#controlling-the-number-of-threads-used-by-guix-build-commands) for + more details. + + _(defaults to the value of `nproc` outside the container)_ + +* _**SOURCE_DATE_EPOCH**_ + + Override the reference UNIX timestamp used for bit-for-bit reproducibility, + the variable name conforms to [standard][r12e/source-date-epoch]. + + _(defaults to the output of `$(git log --format=%at -1)`)_ + +* _**V**_ + + If non-empty, will pass `V=1` to all `make` invocations, making `make` output + verbose. + + Note that any given value is ignored. The variable is only checked for + emptiness. More concretely, this means that `V=` (setting `V` to the empty + string) is interpreted the same way as not setting `V` at all, and that `V=0` + has the same effect as `V=1`. + +* _**SUBSTITUTE_URLS**_ + + A whitespace-delimited list of URLs from which to download pre-built packages. + A URL is only used if its signing key is authorized (refer to the [substitute + servers section](#option-1-building-with-substitutes) for more details). + +* _**ADDITIONAL_GUIX_COMMON_FLAGS**_ + + Additional flags to be passed to all `guix` commands. + +* _**ADDITIONAL_GUIX_TIMEMACHINE_FLAGS**_ + + Additional flags to be passed to `guix time-machine`. + +* _**ADDITIONAL_GUIX_ENVIRONMENT_FLAGS**_ + + Additional flags to be passed to the invocation of `guix environment` inside + `guix time-machine`. + +# Choosing your security model + +No matter how you installed Guix, you need to decide on your security model for +building packages with Guix. + +Guix allows us to achieve better binary security by using our CPU time to build +everything from scratch. However, it doesn't sacrifice user choice in pursuit of +this: users can decide whether or not to use **substitutes** (pre-built +packages). + +## Option 1: Building with substitutes + +### Step 1: Authorize the signing keys + +Depending on the installation procedure you followed, you may have already +authorized the Guix build farm key. In particular, the official shell installer +script asks you if you want the key installed, and the debian distribution +package authorized the key during installation. + +You can check the current list of authorized keys at `/etc/guix/acl`. + +At the time of writing, a `/etc/guix/acl` with just the Guix build farm key +authorized looks something like: + +```lisp +(acl + (entry + (public-key + (ecc + (curve Ed25519) + (q #8D156F295D24B0D9A86FA5741A840FF2D24F60F7B6C4134814AD55625971B394#) + ) + ) + (tag + (guix import) + ) + ) + ) +``` + +If you've determined that the official Guix build farm key hasn't been +authorized, and you would like to authorize it, run the following as root: + +``` +guix archive --authorize < /var/guix/profiles/per-user/root/current-guix/share/guix/ci.guix.gnu.org.pub +``` + +If +`/var/guix/profiles/per-user/root/current-guix/share/guix/ci.guix.gnu.org.pub` +doesn't exist, try: + +```sh +guix archive --authorize < <PREFIX>/share/guix/ci.guix.gnu.org.pub +``` + +Where `<PREFIX>` is likely: +- `/usr` if you installed from a distribution package +- `/usr/local` if you installed Guix from source and didn't supply any + prefix-modifying flags to Guix's `./configure` + +For dongcarl's substitute server at https://guix.carldong.io, run as root: + +```sh +wget -qO- 'https://guix.carldong.io/signing-key.pub' | guix archive --authorize +``` + +#### Removing authorized keys + +To remove previously authorized keys, simply edit `/etc/guix/acl` and remove the +`(entry (public-key ...))` entry. + +### Step 2: Specify the substitute servers + +Once its key is authorized, the official Guix build farm at +https://ci.guix.gnu.org is automatically used unless the `--no-substitutes` flag +is supplied. This default list of substitute servers is overridable both on a +`guix-daemon` level and when you invoke `guix` commands. See examples below for +the various ways of adding dongcarl's substitute server after having [authorized +his signing key](#authorize-the-signing-keys). + +Change the **default list** of substitute servers by starting `guix-daemon` with +the `--substitute-urls` option (you will likely need to edit your init script): + +```sh +guix-daemon <cmd> --substitute-urls='https://guix.carldong.io https://ci.guix.gnu.org' +``` + +Override the default list of substitute servers by passing the +`--substitute-urls` option for invocations of `guix` commands: + +```sh +guix <cmd> --substitute-urls='https://guix.carldong.io https://ci.guix.gnu.org' +``` + +For scripts under `./contrib/guix`, set the `SUBSTITUTE_URLS` environment +variable: + +```sh +export SUBSTITUTE_URLS='https://guix.carldong.io https://ci.guix.gnu.org' +``` + +## Option 2: Disabling substitutes on an ad-hoc basis + +If you prefer not to use any substitutes, make sure to supply `--no-substitutes` +like in the following snippet. The first build will take a while, but the +resulting packages will be cached for future builds. + +For direct invocations of `guix`: +```sh +guix <cmd> --no-substitutes +``` + +For the scripts under `./contrib/guix/`: +```sh +export ADDITIONAL_GUIX_COMMON_FLAGS='--no-substitutes' +``` + +## Option 3: Disabling substitutes by default + +`guix-daemon` accepts a `--no-substitutes` flag, which will make sure that, +unless otherwise overridden by a command line invocation, no substitutes will be +used. + +If you start `guix-daemon` using an init script, you can edit said script to +supply this flag. + + +# Purging/Uninstalling Guix + +In the extraordinarily rare case where you messed up your Guix installation in +an irreversible way, you may want to completely purge Guix from your system and +start over. + +1. Uninstall Guix itself according to the way you installed it (e.g. `sudo apt + purge guix` for Ubuntu packaging, `sudo make uninstall` for a build from source). +2. Remove all build users and groups + + You may check for relevant users and groups using: + + ``` + getent passwd | grep guix + getent group | grep guix + ``` + + Then, you may remove users and groups using: + + ``` + sudo userdel <user> + sudo groupdel <group> + ``` + +3. Remove all possible Guix-related directories + - `/var/guix/` + - `/var/log/guix/` + - `/gnu/` + - `/etc/guix/` + - `/home/*/.config/guix/` + - `/home/*/.cache/guix/` + - `/home/*/.guix-profile/` + - `/root/.config/guix/` + - `/root/.cache/guix/` + - `/root/.guix-profile/` + +[b17e]: https://bootstrappable.org/ +[r12e/source-date-epoch]: https://reproducible-builds.org/docs/source-date-epoch/ + +[guix/install.sh]: https://git.savannah.gnu.org/cgit/guix.git/plain/etc/guix-install.sh +[guix/bin-install]: https://www.gnu.org/software/guix/manual/en/html_node/Binary-Installation.html +[guix/env-setup]: https://www.gnu.org/software/guix/manual/en/html_node/Build-Environment-Setup.html +[guix/substitutes]: https://www.gnu.org/software/guix/manual/en/html_node/Substitutes.html +[guix/substitute-server-auth]: https://www.gnu.org/software/guix/manual/en/html_node/Substitute-Server-Authorization.html +[guix/time-machine]: https://guix.gnu.org/manual/en/html_node/Invoking-guix-time_002dmachine.html + +[debian/guix-bullseye]: https://packages.debian.org/bullseye/guix +[ubuntu/guix-hirsute]: https://packages.ubuntu.com/hirsute/guix +[fanquake/guix-docker]: https://github.com/fanquake/core-review/tree/master/guix + +[env-vars-list]: #recognized-environment-variables diff --git a/contrib/guix/guix-attest b/contrib/guix/guix-attest new file mode 100755 index 0000000000..b0ef28dc3f --- /dev/null +++ b/contrib/guix/guix-attest @@ -0,0 +1,263 @@ +#!/usr/bin/env bash +export LC_ALL=C +set -e -o pipefail + +# Source the common prelude, which: +# 1. Checks if we're at the top directory of the Bitcoin Core repository +# 2. Defines a few common functions and variables +# +# shellcheck source=libexec/prelude.bash +source "$(dirname "${BASH_SOURCE[0]}")/libexec/prelude.bash" + + +################### +## Sanity Checks ## +################### + +################ +# Required non-builtin commands should be invokable +################ + +check_tools cat env basename mkdir diff sort + +if [ -z "$NO_SIGN" ]; then + # make it possible to override the gpg binary + GPG=${GPG:-gpg} + + # $GPG can contain extra arguments passed to the binary + # so let's check only the existence of arg[0] + # shellcheck disable=SC2206 + GPG_ARRAY=($GPG) + check_tools "${GPG_ARRAY[0]}" +fi + +################ +# Required env vars should be non-empty +################ + +cmd_usage() { +cat <<EOF +Synopsis: + + env GUIX_SIGS_REPO=<path/to/guix.sigs> \\ + SIGNER=GPG_KEY_NAME[=SIGNER_NAME] \\ + [ NO_SIGN=1 ] + ./contrib/guix/guix-attest + +Example w/o overriding signing name: + + env GUIX_SIGS_REPO=/home/achow101/guix.sigs \\ + SIGNER=achow101 \\ + ./contrib/guix/guix-attest + +Example overriding signing name: + + env GUIX_SIGS_REPO=/home/dongcarl/guix.sigs \\ + SIGNER=0x96AB007F1A7ED999=dongcarl \\ + ./contrib/guix/guix-attest + +Example w/o signing, just creating SHA256SUMS: + + env GUIX_SIGS_REPO=/home/achow101/guix.sigs \\ + SIGNER=achow101 \\ + NO_SIGN=1 \\ + ./contrib/guix/guix-attest + +EOF +} + +if [ -z "$GUIX_SIGS_REPO" ] || [ -z "$SIGNER" ]; then + cmd_usage + exit 1 +fi + +################ +# GUIX_SIGS_REPO should exist as a directory +################ + +if [ ! -d "$GUIX_SIGS_REPO" ]; then +cat << EOF +ERR: The specified GUIX_SIGS_REPO is not an existent directory: + + '$GUIX_SIGS_REPO' + +Hint: Please clone the guix.sigs repository and point to it with the + GUIX_SIGS_REPO environment variable. + +EOF +cmd_usage +exit 1 +fi + +################ +# The key specified in SIGNER should be usable +################ + +IFS='=' read -r gpg_key_name signer_name <<< "$SIGNER" +if [ -z "${signer_name}" ]; then + signer_name="$gpg_key_name" +fi + +if [ -z "$NO_SIGN" ] && ! ${GPG} --dry-run --list-secret-keys "${gpg_key_name}" >/dev/null 2>&1; then + echo "ERR: GPG can't seem to find any key named '${gpg_key_name}'" + exit 1 +fi + +################ +# We should be able to find at least one output +################ + +echo "Looking for build output SHA256SUMS fragments in ${OUTDIR_BASE}" + +shopt -s nullglob +sha256sum_fragments=( "$OUTDIR_BASE"/*/SHA256SUMS.part ) # This expands to an array of directories... +shopt -u nullglob + +noncodesigned_fragments=() +codesigned_fragments=() + +if (( ${#sha256sum_fragments[@]} )); then + echo "Found build output SHA256SUMS fragments:" + for outdir in "${sha256sum_fragments[@]}"; do + echo " '$outdir'" + case "$outdir" in + "$OUTDIR_BASE"/*-codesigned/SHA256SUMS.part) + codesigned_fragments+=("$outdir") + ;; + *) + noncodesigned_fragments+=("$outdir") + ;; + esac + done + echo +else + echo "ERR: Could not find any build output SHA256SUMS fragments in ${OUTDIR_BASE}" + exit 1 +fi + +############## +## Attest ## +############## + +# Usage: out_name $outdir +# +# HOST: The output directory being attested +# +out_name() { + basename "$(dirname "$1")" +} + +shasum_already_exists() { +cat <<EOF +-- + +ERR: An ${1} file already exists for '${VERSION}' and attests + differently. You likely previously attested to a partial build (e.g. one + where you specified the HOST environment variable). + + See the diff above for more context. + +Hint: You may wish to remove the existing attestations and their signatures by + invoking: + + rm '${PWD}/${1}'{,.asc} + + Then try running this script again. + +EOF +} + +echo "Attesting to build outputs for version: '${VERSION}'" +echo "" + +# Given a SHA256SUMS file as stdin that has lines like: +# 0ba536819b221a91d3d42e978be016aac918f40984754d74058aa0c921cd3ea6 a/b/d/c/d/s/bitcoin-22.0rc2-riscv64-linux-gnu.tar.gz +# ... +# +# Replace each line's file name with its basename: +# 0ba536819b221a91d3d42e978be016aac918f40984754d74058aa0c921cd3ea6 bitcoin-22.0rc2-riscv64-linux-gnu.tar.gz +# ... +# +basenameify_SHA256SUMS() { + sed -E 's@(^[[:xdigit:]]{64}[[:space:]]+).+/([^/]+$)@\1\2@' +} + +outsigdir="$GUIX_SIGS_REPO/$VERSION/$signer_name" +mkdir -p "$outsigdir" +( + cd "$outsigdir" + + temp_noncodesigned="$(mktemp)" + trap 'rm -rf -- "$temp_noncodesigned"' EXIT + + if (( ${#noncodesigned_fragments[@]} )); then + cat "${noncodesigned_fragments[@]}" \ + | sort -u \ + | sort -k2 \ + | basenameify_SHA256SUMS \ + > "$temp_noncodesigned" + if [ -e noncodesigned.SHA256SUMS ]; then + # The SHA256SUMS already exists, make sure it's exactly what we + # expect, error out if not + if diff -u noncodesigned.SHA256SUMS "$temp_noncodesigned"; then + echo "A noncodesigned.SHA256SUMS file already exists for '${VERSION}' and is up-to-date." + else + shasum_already_exists noncodesigned.SHA256SUMS + exit 1 + fi + else + mv "$temp_noncodesigned" noncodesigned.SHA256SUMS + fi + else + echo "ERR: No noncodesigned outputs found for '${VERSION}', exiting..." + exit 1 + fi + + temp_all="$(mktemp)" + trap 'rm -rf -- "$temp_all"' EXIT + + if (( ${#codesigned_fragments[@]} )); then + # Note: all.SHA256SUMS attests to all of $sha256sum_fragments, but is + # not needed if there are no $codesigned_fragments + cat "${sha256sum_fragments[@]}" \ + | sort -u \ + | sort -k2 \ + | basenameify_SHA256SUMS \ + > "$temp_all" + if [ -e all.SHA256SUMS ]; then + # The SHA256SUMS already exists, make sure it's exactly what we + # expect, error out if not + if diff -u all.SHA256SUMS "$temp_all"; then + echo "An all.SHA256SUMS file already exists for '${VERSION}' and is up-to-date." + else + shasum_already_exists all.SHA256SUMS + exit 1 + fi + else + mv "$temp_all" all.SHA256SUMS + fi + else + # It is fine to have the codesigned outputs be missing (perhaps the + # detached codesigs have not been published yet), just print a log + # message instead of erroring out + echo "INFO: No codesigned outputs found for '${VERSION}', skipping..." + fi + + if [ -z "$NO_SIGN" ]; then + echo "Signing SHA256SUMS to produce SHA256SUMS.asc" + for i in *.SHA256SUMS; do + if [ ! -e "$i".asc ]; then + ${GPG} --detach-sign \ + --digest-algo sha256 \ + --local-user "$gpg_key_name" \ + --armor \ + --output "$i".asc "$i" + else + echo "Signature already there" + fi + done + else + echo "Not signing SHA256SUMS as \$NO_SIGN is not empty" + fi + echo "" +) diff --git a/contrib/guix/guix-build b/contrib/guix/guix-build new file mode 100755 index 0000000000..74b24b9612 --- /dev/null +++ b/contrib/guix/guix-build @@ -0,0 +1,463 @@ +#!/usr/bin/env bash +export LC_ALL=C +set -e -o pipefail + +# Source the common prelude, which: +# 1. Checks if we're at the top directory of the Bitcoin Core repository +# 2. Defines a few common functions and variables +# +# shellcheck source=libexec/prelude.bash +source "$(dirname "${BASH_SOURCE[0]}")/libexec/prelude.bash" + + +################### +## SANITY CHECKS ## +################### + +################ +# Required non-builtin commands should be invocable +################ + +check_tools cat mkdir make getent curl git guix + +################ +# GUIX_BUILD_OPTIONS should be empty +################ +# +# GUIX_BUILD_OPTIONS is an environment variable recognized by guix commands that +# can perform builds. This seems like what we want instead of +# ADDITIONAL_GUIX_COMMON_FLAGS, but the value of GUIX_BUILD_OPTIONS is actually +# _appended_ to normal command-line options. Meaning that they will take +# precedence over the command-specific ADDITIONAL_GUIX_<CMD>_FLAGS. +# +# This seems like a poor user experience. Thus we check for GUIX_BUILD_OPTIONS's +# existence here and direct users of this script to use our (more flexible) +# custom environment variables. +if [ -n "$GUIX_BUILD_OPTIONS" ]; then +cat << EOF +Error: Environment variable GUIX_BUILD_OPTIONS is not empty: + '$GUIX_BUILD_OPTIONS' + +Unfortunately this script is incompatible with GUIX_BUILD_OPTIONS, please unset +GUIX_BUILD_OPTIONS and use ADDITIONAL_GUIX_COMMON_FLAGS to set build options +across guix commands or ADDITIONAL_GUIX_<CMD>_FLAGS to set build options for a +specific guix command. + +See contrib/guix/README.md for more details. +EOF +exit 1 +fi + +################ +# The git worktree should not be dirty +################ + +if ! git diff-index --quiet HEAD -- && [ -z "$FORCE_DIRTY_WORKTREE" ]; then +cat << EOF +ERR: The current git worktree is dirty, which may lead to broken builds. + + Aborting... + +Hint: To make your git worktree clean, You may want to: + 1. Commit your changes, + 2. Stash your changes, or + 3. Set the 'FORCE_DIRTY_WORKTREE' environment variable if you insist on + using a dirty worktree +EOF +exit 1 +fi + +mkdir -p "$VERSION_BASE" + +################ +# Build directories should not exist +################ + +# Default to building for all supported HOSTs (overridable by environment) +export HOSTS="${HOSTS:-x86_64-linux-gnu arm-linux-gnueabihf aarch64-linux-gnu riscv64-linux-gnu powerpc64-linux-gnu powerpc64le-linux-gnu + x86_64-w64-mingw32 + x86_64-apple-darwin arm64-apple-darwin}" + +# Usage: distsrc_for_host HOST +# +# HOST: The current platform triple we're building for +# +distsrc_for_host() { + echo "${DISTSRC_BASE}/distsrc-${VERSION}-${1}" +} + +# Accumulate a list of build directories that already exist... +hosts_distsrc_exists="" +for host in $HOSTS; do + if [ -e "$(distsrc_for_host "$host")" ]; then + hosts_distsrc_exists+=" ${host}" + fi +done + +if [ -n "$hosts_distsrc_exists" ]; then +# ...so that we can print them out nicely in an error message +cat << EOF +ERR: Build directories for this commit already exist for the following platform + triples you're attempting to build, probably because of previous builds. + Please remove, or otherwise deal with them prior to starting another build. + + Aborting... + +Hint: To blow everything away, you may want to use: + + $ ./contrib/guix/guix-clean + +Specifically, this will remove all files without an entry in the index, +excluding the SDK directory, the depends download cache, the depends built +packages cache, the garbage collector roots for Guix environments, and the +output directory. +EOF +for host in $hosts_distsrc_exists; do + echo " ${host} '$(distsrc_for_host "$host")'" +done +exit 1 +else + mkdir -p "$DISTSRC_BASE" +fi + +################ +# When building for darwin, the macOS SDK should exist +################ + +for host in $HOSTS; do + case "$host" in + *darwin*) + OSX_SDK="$(make -C "${PWD}/depends" --no-print-directory HOST="$host" print-OSX_SDK | sed 's@^[^=]\+=@@g')" + if [ -e "$OSX_SDK" ]; then + echo "Found macOS SDK at '${OSX_SDK}', using..." + break + else + echo "macOS SDK does not exist at '${OSX_SDK}', please place the extracted, untarred SDK there to perform darwin builds, or define SDK_PATH environment variable. Exiting..." + exit 1 + fi + ;; + esac +done + +################ +# VERSION_BASE should have enough space +################ + +avail_KiB="$(df -Pk "$VERSION_BASE" | sed 1d | tr -s ' ' | cut -d' ' -f4)" +total_required_KiB=0 +for host in $HOSTS; do + case "$host" in + *darwin*) required_KiB=440000 ;; + *mingw*) required_KiB=7600000 ;; + *) required_KiB=6400000 ;; + esac + total_required_KiB=$((total_required_KiB+required_KiB)) +done + +if (( total_required_KiB > avail_KiB )); then + total_required_GiB=$((total_required_KiB / 1048576)) + avail_GiB=$((avail_KiB / 1048576)) + echo "Performing a Bitcoin Core Guix build for the selected HOSTS requires ${total_required_GiB} GiB, however, only ${avail_GiB} GiB is available. Please free up some disk space before performing the build." + exit 1 +fi + +################ +# Check that we can connect to the guix-daemon +################ + +cat << EOF +Checking that we can connect to the guix-daemon... + +Hint: If this hangs, you may want to try turning your guix-daemon off and on + again. + +EOF +if ! guix gc --list-failures > /dev/null; then +cat << EOF + +ERR: Failed to connect to the guix-daemon, please ensure that one is running and + reachable. +EOF +exit 1 +fi + +# Developer note: we could use `guix repl` for this check and run: +# +# (import (guix store)) (close-connection (open-connection)) +# +# However, the internal API is likely to change more than the CLI invocation + +################ +# Services database must have basic entries +################ + +if ! getent services http https ftp > /dev/null 2>&1; then +cat << EOF +ERR: Your system's C library cannot find service database entries for at least + one of the following services: http, https, ftp. + +Hint: Most likely, /etc/services does not exist yet (common for docker images + and minimal distros), or you don't have permissions to access it. + + If /etc/services does not exist yet, you may want to install the + appropriate package for your distro which provides it. + + On Debian/Ubuntu: netbase + On Arch Linux: iana-etc + + For more information, see: getent(1), services(5) + +EOF + +fi + +######### +# SETUP # +######### + +# Determine the maximum number of jobs to run simultaneously (overridable by +# environment) +JOBS="${JOBS:-$(nproc)}" + +# Usage: host_to_commonname HOST +# +# HOST: The current platform triple we're building for +# +host_to_commonname() { + case "$1" in + *darwin*) echo osx ;; + *mingw*) echo win ;; + *linux*) echo linux ;; + *) exit 1 ;; + esac +} + +# Determine the reference time used for determinism (overridable by environment) +SOURCE_DATE_EPOCH="${SOURCE_DATE_EPOCH:-$(git -c log.showSignature=false log --format=%at -1)}" + +# Precious directories are those which should not be cleaned between successive +# guix builds +depends_precious_dir_names='SOURCES_PATH BASE_CACHE SDK_PATH' +precious_dir_names="${depends_precious_dir_names} OUTDIR_BASE PROFILES_BASE" + +# Usage: contains IFS-SEPARATED-LIST ITEM +contains() { + for i in ${1}; do + if [ "$i" = "${2}" ]; then + return 0 # Found! + fi + done + return 1 +} + +# If the user explicitly specified a precious directory, create it so we +# can map it into the container +for precious_dir_name in $precious_dir_names; do + precious_dir_path="${!precious_dir_name}" + if [ -n "$precious_dir_path" ]; then + if [ ! -e "$precious_dir_path" ]; then + mkdir -p "$precious_dir_path" + elif [ -L "$precious_dir_path" ]; then + echo "ERR: ${precious_dir_name} cannot be a symbolic link" + exit 1 + elif [ ! -d "$precious_dir_path" ]; then + echo "ERR: ${precious_dir_name} must be a directory" + exit 1 + fi + fi +done + +mkdir -p "$VAR_BASE" + +# Record the _effective_ values of precious directories such that guix-clean can +# avoid clobbering them if appropriate. +# +# shellcheck disable=SC2046,SC2086 +{ + # Get depends precious dir definitions from depends + make -C "${PWD}/depends" \ + --no-print-directory \ + -- $(printf "print-%s\n" $depends_precious_dir_names) + + # Get remaining precious dir definitions from the environment + for precious_dir_name in $precious_dir_names; do + precious_dir_path="${!precious_dir_name}" + if ! contains "$depends_precious_dir_names" "$precious_dir_name"; then + echo "${precious_dir_name}=${precious_dir_path}" + fi + done +} > "${VAR_BASE}/precious_dirs" + +# Make sure an output directory exists for our builds +OUTDIR_BASE="${OUTDIR_BASE:-${VERSION_BASE}/output}" +mkdir -p "$OUTDIR_BASE" + +# Download the depends sources now as we won't have internet access in the build +# container +for host in $HOSTS; do + make -C "${PWD}/depends" -j"$JOBS" download-"$(host_to_commonname "$host")" ${V:+V=1} ${SOURCES_PATH:+SOURCES_PATH="$SOURCES_PATH"} +done + +# Usage: outdir_for_host HOST SUFFIX +# +# HOST: The current platform triple we're building for +# +outdir_for_host() { + echo "${OUTDIR_BASE}/${1}${2:+-${2}}" +} + +# Usage: profiledir_for_host HOST SUFFIX +# +# HOST: The current platform triple we're building for +# +profiledir_for_host() { + echo "${PROFILES_BASE}/${1}${2:+-${2}}" +} + + +######### +# BUILD # +######### + +# Function to be called when building for host ${1} and the user interrupts the +# build +int_trap() { +cat << EOF +** INT received while building ${1}, you may want to clean up the relevant + work directories (e.g. distsrc-*) before rebuilding + +Hint: To blow everything away, you may want to use: + + $ ./contrib/guix/guix-clean + +Specifically, this will remove all files without an entry in the index, +excluding the SDK directory, the depends download cache, the depends built +packages cache, the garbage collector roots for Guix environments, and the +output directory. +EOF +} + +# Deterministically build Bitcoin Core +# shellcheck disable=SC2153 +for host in $HOSTS; do + + # Display proper warning when the user interrupts the build + trap 'int_trap ${host}' INT + + ( + # Required for 'contrib/guix/manifest.scm' to output the right manifest + # for the particular $HOST we're building for + export HOST="$host" + + # shellcheck disable=SC2030 +cat << EOF +INFO: Building ${VERSION:?not set} for platform triple ${HOST:?not set}: + ...using reference timestamp: ${SOURCE_DATE_EPOCH:?not set} + ...running at most ${JOBS:?not set} jobs + ...from worktree directory: '${PWD}' + ...bind-mounted in container to: '/bitcoin' + ...in build directory: '$(distsrc_for_host "$HOST")' + ...bind-mounted in container to: '$(DISTSRC_BASE=/distsrc-base && distsrc_for_host "$HOST")' + ...outputting in: '$(outdir_for_host "$HOST")' + ...bind-mounted in container to: '$(OUTDIR_BASE=/outdir-base && outdir_for_host "$HOST")' +EOF + + # Run the build script 'contrib/guix/libexec/build.sh' in the build + # container specified by 'contrib/guix/manifest.scm'. + # + # Explanation of `guix environment` flags: + # + # --container run command within an isolated container + # + # Running in an isolated container minimizes build-time differences + # between machines and improves reproducibility + # + # --pure unset existing environment variables + # + # Same rationale as --container + # + # --no-cwd do not share current working directory with an + # isolated container + # + # When --container is specified, the default behavior is to share + # the current working directory with the isolated container at the + # same exact path (e.g. mapping '/home/satoshi/bitcoin/' to + # '/home/satoshi/bitcoin/'). This means that the $PWD inside the + # container becomes a source of irreproducibility. --no-cwd disables + # this behaviour. + # + # --share=SPEC for containers, share writable host file system + # according to SPEC + # + # --share="$PWD"=/bitcoin + # + # maps our current working directory to /bitcoin + # inside the isolated container, which we later cd + # into. + # + # While we don't want to map our current working directory to the + # same exact path (as this introduces irreproducibility), we do want + # it to be at a _fixed_ path _somewhere_ inside the isolated + # container so that we have something to build. '/bitcoin' was + # chosen arbitrarily. + # + # ${SOURCES_PATH:+--share="$SOURCES_PATH"} + # + # make the downloaded depends sources path available + # inside the isolated container + # + # The isolated container has no network access as it's in a + # different network namespace from the main machine, so we have to + # make the downloaded depends sources available to it. The sources + # should have been downloaded prior to this invocation. + # + # --keep-failed keep build tree of failed builds + # + # When builds of the Guix environment itself (not Bitcoin Core) + # fail, it is useful for the build tree to be kept for debugging + # purposes. + # + # ${SUBSTITUTE_URLS:+--substitute-urls="$SUBSTITUTE_URLS"} + # + # fetch substitute from SUBSTITUTE_URLS if they are + # authorized + # + # Depending on the user's security model, it may be desirable to use + # substitutes (pre-built packages) from servers that the user trusts. + # Please read the README.md in the same directory as this file for + # more information. + # + # shellcheck disable=SC2086,SC2031 + time-machine environment --manifest="${PWD}/contrib/guix/manifest.scm" \ + --container \ + --pure \ + --no-cwd \ + --share="$PWD"=/bitcoin \ + --share="$DISTSRC_BASE"=/distsrc-base \ + --share="$OUTDIR_BASE"=/outdir-base \ + --expose="$(git rev-parse --git-common-dir)" \ + ${SOURCES_PATH:+--share="$SOURCES_PATH"} \ + ${BASE_CACHE:+--share="$BASE_CACHE"} \ + ${SDK_PATH:+--share="$SDK_PATH"} \ + --cores="$JOBS" \ + --keep-failed \ + --fallback \ + --link-profile \ + --root="$(profiledir_for_host "${HOST}")" \ + ${SUBSTITUTE_URLS:+--substitute-urls="$SUBSTITUTE_URLS"} \ + ${ADDITIONAL_GUIX_COMMON_FLAGS} ${ADDITIONAL_GUIX_ENVIRONMENT_FLAGS} \ + -- env HOST="$host" \ + DISTNAME="$DISTNAME" \ + JOBS="$JOBS" \ + SOURCE_DATE_EPOCH="${SOURCE_DATE_EPOCH:?unable to determine value}" \ + ${V:+V=1} \ + ${SOURCES_PATH:+SOURCES_PATH="$SOURCES_PATH"} \ + ${BASE_CACHE:+BASE_CACHE="$BASE_CACHE"} \ + ${SDK_PATH:+SDK_PATH="$SDK_PATH"} \ + DISTSRC="$(DISTSRC_BASE=/distsrc-base && distsrc_for_host "$HOST")" \ + OUTDIR="$(OUTDIR_BASE=/outdir-base && outdir_for_host "$HOST")" \ + DIST_ARCHIVE_BASE=/outdir-base/dist-archive \ + bash -c "cd /bitcoin && bash contrib/guix/libexec/build.sh" + ) + +done diff --git a/contrib/guix/guix-clean b/contrib/guix/guix-clean new file mode 100755 index 0000000000..9af0a793cf --- /dev/null +++ b/contrib/guix/guix-clean @@ -0,0 +1,83 @@ +#!/usr/bin/env bash +export LC_ALL=C +set -e -o pipefail + +# Source the common prelude, which: +# 1. Checks if we're at the top directory of the Bitcoin Core repository +# 2. Defines a few common functions and variables +# +# shellcheck source=libexec/prelude.bash +source "$(dirname "${BASH_SOURCE[0]}")/libexec/prelude.bash" + + +################### +## Sanity Checks ## +################### + +################ +# Required non-builtin commands should be invokable +################ + +check_tools cat mkdir make git guix + + +############# +## Clean ## +############# + +# Usage: under_dir MAYBE_PARENT MAYBE_CHILD +# +# If MAYBE_CHILD is a subdirectory of MAYBE_PARENT, print the relative path +# from MAYBE_PARENT to MAYBE_CHILD. Otherwise, return 1 as the error code. +# +# NOTE: This does not perform any symlink-resolving or path canonicalization. +# +under_dir() { + local path_residue + path_residue="${2##"${1}"}" + if [ -z "$path_residue" ] || [ "$path_residue" = "$2" ]; then + return 1 + else + echo "$path_residue" + fi +} + +# Usage: dir_under_git_root MAYBE_CHILD +# +# If MAYBE_CHILD is under the current git repository and exists, print the +# relative path from the git repository's top-level directory to MAYBE_CHILD, +# otherwise, exit with an error code. +# +dir_under_git_root() { + local rv + rv="$(under_dir "$(git_root)" "$1")" + [ -n "$rv" ] && echo "$rv" +} + +shopt -s nullglob +found_precious_dirs_files=( "${version_base_prefix}"*/"${var_base_basename}/precious_dirs" ) # This expands to an array of directories... +shopt -u nullglob + +exclude_flags=() + +for precious_dirs_file in "${found_precious_dirs_files[@]}"; do + # Make sure the precious directories (e.g. SOURCES_PATH, BASE_CACHE, SDK_PATH) + # are excluded from git-clean + echo "Found precious_dirs file: '${precious_dirs_file}'" + + # Exclude the precious_dirs file itself + if dirs_file_exclude_fragment=$(dir_under_git_root "$(dirname "$precious_dirs_file")"); then + exclude_flags+=( --exclude="${dirs_file_exclude_fragment}/precious_dirs" ) + fi + + # Read each 'name=dir' pair from the precious_dirs file + while IFS='=' read -r name dir; do + # Add an exclusion flag if the precious directory is under the git root. + if under=$(dir_under_git_root "$dir"); then + echo "Avoiding ${name}: ${under}" + exclude_flags+=( --exclude="$under" ) + fi + done < "$precious_dirs_file" +done + +git clean -xdff "${exclude_flags[@]}" diff --git a/contrib/guix/guix-codesign b/contrib/guix/guix-codesign new file mode 100755 index 0000000000..3279d431aa --- /dev/null +++ b/contrib/guix/guix-codesign @@ -0,0 +1,378 @@ +#!/usr/bin/env bash +export LC_ALL=C +set -e -o pipefail + +# Source the common prelude, which: +# 1. Checks if we're at the top directory of the Bitcoin Core repository +# 2. Defines a few common functions and variables +# +# shellcheck source=libexec/prelude.bash +source "$(dirname "${BASH_SOURCE[0]}")/libexec/prelude.bash" + + +################### +## SANITY CHECKS ## +################### + +################ +# Required non-builtin commands should be invocable +################ + +check_tools cat mkdir git guix + +################ +# Required env vars should be non-empty +################ + +cmd_usage() { + cat <<EOF +Synopsis: + + env DETACHED_SIGS_REPO=<path/to/bitcoin-detached-sigs> \\ + ./contrib/guix/guix-codesign + +EOF +} + +if [ -z "$DETACHED_SIGS_REPO" ]; then + cmd_usage + exit 1 +fi + +################ +# GUIX_BUILD_OPTIONS should be empty +################ +# +# GUIX_BUILD_OPTIONS is an environment variable recognized by guix commands that +# can perform builds. This seems like what we want instead of +# ADDITIONAL_GUIX_COMMON_FLAGS, but the value of GUIX_BUILD_OPTIONS is actually +# _appended_ to normal command-line options. Meaning that they will take +# precedence over the command-specific ADDITIONAL_GUIX_<CMD>_FLAGS. +# +# This seems like a poor user experience. Thus we check for GUIX_BUILD_OPTIONS's +# existence here and direct users of this script to use our (more flexible) +# custom environment variables. +if [ -n "$GUIX_BUILD_OPTIONS" ]; then +cat << EOF +Error: Environment variable GUIX_BUILD_OPTIONS is not empty: + '$GUIX_BUILD_OPTIONS' + +Unfortunately this script is incompatible with GUIX_BUILD_OPTIONS, please unset +GUIX_BUILD_OPTIONS and use ADDITIONAL_GUIX_COMMON_FLAGS to set build options +across guix commands or ADDITIONAL_GUIX_<CMD>_FLAGS to set build options for a +specific guix command. + +See contrib/guix/README.md for more details. +EOF +exit 1 +fi + +################ +# The codesignature git worktree should not be dirty +################ + +if ! git -C "$DETACHED_SIGS_REPO" diff-index --quiet HEAD -- && [ -z "$FORCE_DIRTY_WORKTREE" ]; then + cat << EOF +ERR: The DETACHED CODESIGNATURE git worktree is dirty, which may lead to broken builds. + + Aborting... + +Hint: To make your git worktree clean, You may want to: + 1. Commit your changes, + 2. Stash your changes, or + 3. Set the 'FORCE_DIRTY_WORKTREE' environment variable if you insist on + using a dirty worktree +EOF + exit 1 +fi + +################ +# Build directories should not exist +################ + +# Default to building for all supported HOSTs (overridable by environment) +export HOSTS="${HOSTS:-x86_64-w64-mingw32 x86_64-apple-darwin arm64-apple-darwin}" + +# Usage: distsrc_for_host HOST +# +# HOST: The current platform triple we're building for +# +distsrc_for_host() { + echo "${DISTSRC_BASE}/distsrc-${VERSION}-${1}-codesigned" +} + +# Accumulate a list of build directories that already exist... +hosts_distsrc_exists="" +for host in $HOSTS; do + if [ -e "$(distsrc_for_host "$host")" ]; then + hosts_distsrc_exists+=" ${host}" + fi +done + +if [ -n "$hosts_distsrc_exists" ]; then +# ...so that we can print them out nicely in an error message +cat << EOF +ERR: Build directories for this commit already exist for the following platform + triples you're attempting to build, probably because of previous builds. + Please remove, or otherwise deal with them prior to starting another build. + + Aborting... + +Hint: To blow everything away, you may want to use: + + $ ./contrib/guix/guix-clean + +Specifically, this will remove all files without an entry in the index, +excluding the SDK directory, the depends download cache, the depends built +packages cache, the garbage collector roots for Guix environments, and the +output directory. +EOF +for host in $hosts_distsrc_exists; do + echo " ${host} '$(distsrc_for_host "$host")'" +done +exit 1 +else + mkdir -p "$DISTSRC_BASE" +fi + + +################ +# Unsigned tarballs SHOULD exist +################ + +# Usage: outdir_for_host HOST SUFFIX +# +# HOST: The current platform triple we're building for +# +outdir_for_host() { + echo "${OUTDIR_BASE}/${1}${2:+-${2}}" +} + + +unsigned_tarball_for_host() { + case "$1" in + *mingw*) + echo "$(outdir_for_host "$1")/${DISTNAME}-win64-unsigned.tar.gz" + ;; + *darwin*) + echo "$(outdir_for_host "$1")/${DISTNAME}-${1}-unsigned.tar.gz" + ;; + *) + exit 1 + ;; + esac +} + +# Accumulate a list of build directories that already exist... +hosts_unsigned_tarball_missing="" +for host in $HOSTS; do + if [ ! -e "$(unsigned_tarball_for_host "$host")" ]; then + hosts_unsigned_tarball_missing+=" ${host}" + fi +done + +if [ -n "$hosts_unsigned_tarball_missing" ]; then + # ...so that we can print them out nicely in an error message + cat << EOF +ERR: Unsigned tarballs do not exist +... + +EOF +for host in $hosts_unsigned_tarball_missing; do + echo " ${host} '$(unsigned_tarball_for_host "$host")'" +done +exit 1 +fi + +################ +# Check that we can connect to the guix-daemon +################ + +cat << EOF +Checking that we can connect to the guix-daemon... + +Hint: If this hangs, you may want to try turning your guix-daemon off and on + again. + +EOF +if ! guix gc --list-failures > /dev/null; then + cat << EOF + +ERR: Failed to connect to the guix-daemon, please ensure that one is running and + reachable. +EOF + exit 1 +fi + +# Developer note: we could use `guix repl` for this check and run: +# +# (import (guix store)) (close-connection (open-connection)) +# +# However, the internal API is likely to change more than the CLI invocation + + +######### +# SETUP # +######### + +# Determine the maximum number of jobs to run simultaneously (overridable by +# environment) +JOBS="${JOBS:-$(nproc)}" + +# Determine the reference time used for determinism (overridable by environment) +SOURCE_DATE_EPOCH="${SOURCE_DATE_EPOCH:-$(git -c log.showSignature=false log --format=%at -1)}" + +# Make sure an output directory exists for our builds +OUTDIR_BASE="${OUTDIR_BASE:-${VERSION_BASE}/output}" +mkdir -p "$OUTDIR_BASE" + +# Usage: profiledir_for_host HOST SUFFIX +# +# HOST: The current platform triple we're building for +# +profiledir_for_host() { + echo "${PROFILES_BASE}/${1}${2:+-${2}}" +} + +######### +# BUILD # +######### + +# Function to be called when codesigning for host ${1} and the user interrupts +# the codesign +int_trap() { +cat << EOF +** INT received while codesigning ${1}, you may want to clean up the relevant + work directories (e.g. distsrc-*) before recodesigning + +Hint: To blow everything away, you may want to use: + + $ ./contrib/guix/guix-clean + +Specifically, this will remove all files without an entry in the index, +excluding the SDK directory, the depends download cache, the depends built +packages cache, the garbage collector roots for Guix environments, and the +output directory. +EOF +} + +# Deterministically build Bitcoin Core +# shellcheck disable=SC2153 +for host in $HOSTS; do + + # Display proper warning when the user interrupts the build + trap 'int_trap ${host}' INT + + ( + # Required for 'contrib/guix/manifest.scm' to output the right manifest + # for the particular $HOST we're building for + export HOST="$host" + + # shellcheck disable=SC2030 +cat << EOF +INFO: Codesigning ${VERSION:?not set} for platform triple ${HOST:?not set}: + ...using reference timestamp: ${SOURCE_DATE_EPOCH:?not set} + ...from worktree directory: '${PWD}' + ...bind-mounted in container to: '/bitcoin' + ...in build directory: '$(distsrc_for_host "$HOST")' + ...bind-mounted in container to: '$(DISTSRC_BASE=/distsrc-base && distsrc_for_host "$HOST")' + ...outputting in: '$(outdir_for_host "$HOST" codesigned)' + ...bind-mounted in container to: '$(OUTDIR_BASE=/outdir-base && outdir_for_host "$HOST" codesigned)' + ...using detached signatures in: '${DETACHED_SIGS_REPO:?not set}' + ...bind-mounted in container to: '/detached-sigs' +EOF + + + # Run the build script 'contrib/guix/libexec/build.sh' in the build + # container specified by 'contrib/guix/manifest.scm'. + # + # Explanation of `guix environment` flags: + # + # --container run command within an isolated container + # + # Running in an isolated container minimizes build-time differences + # between machines and improves reproducibility + # + # --pure unset existing environment variables + # + # Same rationale as --container + # + # --no-cwd do not share current working directory with an + # isolated container + # + # When --container is specified, the default behavior is to share + # the current working directory with the isolated container at the + # same exact path (e.g. mapping '/home/satoshi/bitcoin/' to + # '/home/satoshi/bitcoin/'). This means that the $PWD inside the + # container becomes a source of irreproducibility. --no-cwd disables + # this behaviour. + # + # --share=SPEC for containers, share writable host file system + # according to SPEC + # + # --share="$PWD"=/bitcoin + # + # maps our current working directory to /bitcoin + # inside the isolated container, which we later cd + # into. + # + # While we don't want to map our current working directory to the + # same exact path (as this introduces irreproducibility), we do want + # it to be at a _fixed_ path _somewhere_ inside the isolated + # container so that we have something to build. '/bitcoin' was + # chosen arbitrarily. + # + # ${SOURCES_PATH:+--share="$SOURCES_PATH"} + # + # make the downloaded depends sources path available + # inside the isolated container + # + # The isolated container has no network access as it's in a + # different network namespace from the main machine, so we have to + # make the downloaded depends sources available to it. The sources + # should have been downloaded prior to this invocation. + # + # ${SUBSTITUTE_URLS:+--substitute-urls="$SUBSTITUTE_URLS"} + # + # fetch substitute from SUBSTITUTE_URLS if they are + # authorized + # + # Depending on the user's security model, it may be desirable to use + # substitutes (pre-built packages) from servers that the user trusts. + # Please read the README.md in the same directory as this file for + # more information. + # + # shellcheck disable=SC2086,SC2031 + time-machine environment --manifest="${PWD}/contrib/guix/manifest.scm" \ + --container \ + --pure \ + --no-cwd \ + --share="$PWD"=/bitcoin \ + --share="$DISTSRC_BASE"=/distsrc-base \ + --share="$OUTDIR_BASE"=/outdir-base \ + --share="$DETACHED_SIGS_REPO"=/detached-sigs \ + --expose="$(git rev-parse --git-common-dir)" \ + --expose="$(git -C "$DETACHED_SIGS_REPO" rev-parse --git-common-dir)" \ + ${SOURCES_PATH:+--share="$SOURCES_PATH"} \ + --cores="$JOBS" \ + --keep-failed \ + --fallback \ + --link-profile \ + --root="$(profiledir_for_host "${HOST}" codesigned)" \ + ${SUBSTITUTE_URLS:+--substitute-urls="$SUBSTITUTE_URLS"} \ + ${ADDITIONAL_GUIX_COMMON_FLAGS} ${ADDITIONAL_GUIX_ENVIRONMENT_FLAGS} \ + -- env HOST="$host" \ + DISTNAME="$DISTNAME" \ + JOBS="$JOBS" \ + SOURCE_DATE_EPOCH="${SOURCE_DATE_EPOCH:?unable to determine value}" \ + ${V:+V=1} \ + ${SOURCES_PATH:+SOURCES_PATH="$SOURCES_PATH"} \ + DISTSRC="$(DISTSRC_BASE=/distsrc-base && distsrc_for_host "$HOST")" \ + OUTDIR="$(OUTDIR_BASE=/outdir-base && outdir_for_host "$HOST" codesigned)" \ + DIST_ARCHIVE_BASE=/outdir-base/dist-archive \ + DETACHED_SIGS_REPO=/detached-sigs \ + UNSIGNED_TARBALL="$(OUTDIR_BASE=/outdir-base && unsigned_tarball_for_host "$HOST")" \ + bash -c "cd /bitcoin && bash contrib/guix/libexec/codesign.sh" + ) + +done diff --git a/contrib/guix/guix-verify b/contrib/guix/guix-verify new file mode 100755 index 0000000000..02ae022741 --- /dev/null +++ b/contrib/guix/guix-verify @@ -0,0 +1,174 @@ +#!/usr/bin/env bash +export LC_ALL=C +set -e -o pipefail + +# Source the common prelude, which: +# 1. Checks if we're at the top directory of the Bitcoin Core repository +# 2. Defines a few common functions and variables +# +# shellcheck source=libexec/prelude.bash +source "$(dirname "${BASH_SOURCE[0]}")/libexec/prelude.bash" + + +################### +## Sanity Checks ## +################### + +################ +# Required non-builtin commands should be invokable +################ + +check_tools cat diff gpg + +################ +# Required env vars should be non-empty +################ + +cmd_usage() { +cat <<EOF +Synopsis: + + env GUIX_SIGS_REPO=<path/to/guix.sigs> [ SIGNER=<signer> ] ./contrib/guix/guix-verify + +Example overriding signer's manifest to use as base + + env GUIX_SIGS_REPO=/home/dongcarl/guix.sigs SIGNER=achow101 ./contrib/guix/guix-verify + +EOF +} + +if [ -z "$GUIX_SIGS_REPO" ]; then + cmd_usage + exit 1 +fi + +################ +# GUIX_SIGS_REPO should exist as a directory +################ + +if [ ! -d "$GUIX_SIGS_REPO" ]; then +cat << EOF +ERR: The specified GUIX_SIGS_REPO is not an existent directory: + + '$GUIX_SIGS_REPO' + +Hint: Please clone the guix.sigs repository and point to it with the + GUIX_SIGS_REPO environment variable. + +EOF +cmd_usage +exit 1 +fi + +############## +## Verify ## +############## + +OUTSIGDIR_BASE="${GUIX_SIGS_REPO}/${VERSION}" +echo "Looking for signature directories in '${OUTSIGDIR_BASE}'" +echo "" + +# Usage: verify compare_manifest current_manifest +verify() { + local compare_manifest="$1" + local current_manifest="$2" + if ! gpg --quiet --batch --verify "$current_manifest".asc "$current_manifest" 1>&2; then + echo "ERR: Failed to verify GPG signature in '${current_manifest}'" + echo "" + echo "Hint: Either the signature is invalid or the public key is missing" + echo "" + failure=1 + elif ! diff --report-identical "$compare_manifest" "$current_manifest" 1>&2; then + echo "ERR: The SHA256SUMS attestation in these two directories differ:" + echo " '${compare_manifest}'" + echo " '${current_manifest}'" + echo "" + failure=1 + else + echo "Verified: '${current_manifest}'" + echo "" + fi +} + +shopt -s nullglob +all_noncodesigned=( "$OUTSIGDIR_BASE"/*/noncodesigned.SHA256SUMS ) +shopt -u nullglob + +echo "--------------------" +echo "" +if (( ${#all_noncodesigned[@]} )); then + compare_noncodesigned="${all_noncodesigned[0]}" + if [[ -n "$SIGNER" ]]; then + signer_noncodesigned="$OUTSIGDIR_BASE/$SIGNER/noncodesigned.SHA256SUMS" + if [[ -f "$signer_noncodesigned" ]]; then + echo "Using $SIGNER's manifest as the base to compare against" + compare_noncodesigned="$signer_noncodesigned" + else + echo "Unable to find $SIGNER's manifest, using the first one found" + fi + else + echo "No SIGNER provided, using the first manifest found" + fi + + for current_manifest in "${all_noncodesigned[@]}"; do + verify "$compare_noncodesigned" "$current_manifest" + done + + echo "DONE: Checking output signatures for noncodesigned.SHA256SUMS" + echo "" +else + echo "WARN: No signature directories with noncodesigned.SHA256SUMS found" + echo "" +fi + +shopt -s nullglob +all_all=( "$OUTSIGDIR_BASE"/*/all.SHA256SUMS ) +shopt -u nullglob + +echo "--------------------" +echo "" +if (( ${#all_all[@]} )); then + compare_all="${all_all[0]}" + if [[ -n "$SIGNER" ]]; then + signer_all="$OUTSIGDIR_BASE/$SIGNER/all.SHA256SUMS" + if [[ -f "$signer_all" ]]; then + echo "Using $SIGNER's manifest as the base to compare against" + compare_all="$signer_all" + else + echo "Unable to find $SIGNER's manifest, using the first one found" + fi + else + echo "No SIGNER provided, using the first manifest found" + fi + + for current_manifest in "${all_all[@]}"; do + verify "$compare_all" "$current_manifest" + done + + # Sanity check: there should be no entries that exist in + # noncodesigned.SHA256SUMS that doesn't exist in all.SHA256SUMS + if [[ "$(comm -23 <(sort "$compare_noncodesigned") <(sort "$compare_all") | wc -c)" -ne 0 ]]; then + echo "ERR: There are unique lines in noncodesigned.SHA256SUMS which" + echo " do not exist in all.SHA256SUMS, something went very wrong." + exit 1 + fi + + echo "DONE: Checking output signatures for all.SHA256SUMS" + echo "" +else + echo "WARN: No signature directories with all.SHA256SUMS found" + echo "" +fi + +echo "====================" +echo "" +if (( ${#all_noncodesigned[@]} + ${#all_all[@]} == 0 )); then + echo "ERR: Unable to perform any verifications as no signature directories" + echo " were found" + echo "" + exit 1 +fi + +if [ -n "$failure" ]; then + exit 1 +fi diff --git a/contrib/guix/libexec/build.sh b/contrib/guix/libexec/build.sh new file mode 100755 index 0000000000..ad3129184c --- /dev/null +++ b/contrib/guix/libexec/build.sh @@ -0,0 +1,440 @@ +#!/usr/bin/env bash +# Copyright (c) 2019-2021 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +export LC_ALL=C +set -e -o pipefail +export TZ=UTC + +# Although Guix _does_ set umask when building its own packages (in our case, +# this is all packages in manifest.scm), it does not set it for `guix +# environment`. It does make sense for at least `guix environment --container` +# to set umask, so if that change gets merged upstream and we bump the +# time-machine to a commit which includes the aforementioned change, we can +# remove this line. +# +# This line should be placed before any commands which creates files. +umask 0022 + +if [ -n "$V" ]; then + # Print both unexpanded (-v) and expanded (-x) forms of commands as they are + # read from this file. + set -vx + # Set VERBOSE for CMake-based builds + export VERBOSE="$V" +fi + +# Check that required environment variables are set +cat << EOF +Required environment variables as seen inside the container: + DIST_ARCHIVE_BASE: ${DIST_ARCHIVE_BASE:?not set} + DISTNAME: ${DISTNAME:?not set} + HOST: ${HOST:?not set} + SOURCE_DATE_EPOCH: ${SOURCE_DATE_EPOCH:?not set} + JOBS: ${JOBS:?not set} + DISTSRC: ${DISTSRC:?not set} + OUTDIR: ${OUTDIR:?not set} +EOF + +ACTUAL_OUTDIR="${OUTDIR}" +OUTDIR="${DISTSRC}/output" + +##################### +# Environment Setup # +##################### + +# The depends folder also serves as a base-prefix for depends packages for +# $HOSTs after successfully building. +BASEPREFIX="${PWD}/depends" + +# Given a package name and an output name, return the path of that output in our +# current guix environment +store_path() { + grep --extended-regexp "/[^-]{32}-${1}-[^-]+${2:+-${2}}" "${GUIX_ENVIRONMENT}/manifest" \ + | head --lines=1 \ + | sed --expression='s|^[[:space:]]*"||' \ + --expression='s|"[[:space:]]*$||' +} + + +# Set environment variables to point the NATIVE toolchain to the right +# includes/libs +NATIVE_GCC="$(store_path gcc-toolchain)" +NATIVE_GCC_STATIC="$(store_path gcc-toolchain static)" + +unset LIBRARY_PATH +unset CPATH +unset C_INCLUDE_PATH +unset CPLUS_INCLUDE_PATH +unset OBJC_INCLUDE_PATH +unset OBJCPLUS_INCLUDE_PATH + +export LIBRARY_PATH="${NATIVE_GCC}/lib:${NATIVE_GCC}/lib64:${NATIVE_GCC_STATIC}/lib:${NATIVE_GCC_STATIC}/lib64" +export C_INCLUDE_PATH="${NATIVE_GCC}/include" +export CPLUS_INCLUDE_PATH="${NATIVE_GCC}/include/c++:${NATIVE_GCC}/include" +export OBJC_INCLUDE_PATH="${NATIVE_GCC}/include" +export OBJCPLUS_INCLUDE_PATH="${NATIVE_GCC}/include/c++:${NATIVE_GCC}/include" + +prepend_to_search_env_var() { + export "${1}=${2}${!1:+:}${!1}" +} + +# Set environment variables to point the CROSS toolchain to the right +# includes/libs for $HOST +case "$HOST" in + *mingw*) + # Determine output paths to use in CROSS_* environment variables + CROSS_GLIBC="$(store_path "mingw-w64-x86_64-winpthreads")" + CROSS_GCC="$(store_path "gcc-cross-${HOST}")" + CROSS_GCC_LIB_STORE="$(store_path "gcc-cross-${HOST}" lib)" + CROSS_GCC_LIBS=( "${CROSS_GCC_LIB_STORE}/lib/gcc/${HOST}"/* ) # This expands to an array of directories... + CROSS_GCC_LIB="${CROSS_GCC_LIBS[0]}" # ...we just want the first one (there should only be one) + + # The search path ordering is generally: + # 1. gcc-related search paths + # 2. libc-related search paths + # 2. kernel-header-related search paths (not applicable to mingw-w64 hosts) + export CROSS_C_INCLUDE_PATH="${CROSS_GCC_LIB}/include:${CROSS_GCC_LIB}/include-fixed:${CROSS_GLIBC}/include" + export CROSS_CPLUS_INCLUDE_PATH="${CROSS_GCC}/include/c++:${CROSS_GCC}/include/c++/${HOST}:${CROSS_GCC}/include/c++/backward:${CROSS_C_INCLUDE_PATH}" + export CROSS_LIBRARY_PATH="${CROSS_GCC_LIB_STORE}/lib:${CROSS_GCC_LIB}:${CROSS_GLIBC}/lib" + ;; + *darwin*) + # The CROSS toolchain for darwin uses the SDK and ignores environment variables. + # See depends/hosts/darwin.mk for more details. + ;; + *linux*) + CROSS_GLIBC="$(store_path "glibc-cross-${HOST}")" + CROSS_GLIBC_STATIC="$(store_path "glibc-cross-${HOST}" static)" + CROSS_KERNEL="$(store_path "linux-libre-headers-cross-${HOST}")" + CROSS_GCC="$(store_path "gcc-cross-${HOST}")" + CROSS_GCC_LIB_STORE="$(store_path "gcc-cross-${HOST}" lib)" + CROSS_GCC_LIBS=( "${CROSS_GCC_LIB_STORE}/lib/gcc/${HOST}"/* ) # This expands to an array of directories... + CROSS_GCC_LIB="${CROSS_GCC_LIBS[0]}" # ...we just want the first one (there should only be one) + + export CROSS_C_INCLUDE_PATH="${CROSS_GCC_LIB}/include:${CROSS_GCC_LIB}/include-fixed:${CROSS_GLIBC}/include:${CROSS_KERNEL}/include" + export CROSS_CPLUS_INCLUDE_PATH="${CROSS_GCC}/include/c++:${CROSS_GCC}/include/c++/${HOST}:${CROSS_GCC}/include/c++/backward:${CROSS_C_INCLUDE_PATH}" + export CROSS_LIBRARY_PATH="${CROSS_GCC_LIB_STORE}/lib:${CROSS_GCC_LIB}:${CROSS_GLIBC}/lib:${CROSS_GLIBC_STATIC}/lib" + ;; + *) + exit 1 ;; +esac + +# Sanity check CROSS_*_PATH directories +IFS=':' read -ra PATHS <<< "${CROSS_C_INCLUDE_PATH}:${CROSS_CPLUS_INCLUDE_PATH}:${CROSS_LIBRARY_PATH}" +for p in "${PATHS[@]}"; do + if [ -n "$p" ] && [ ! -d "$p" ]; then + echo "'$p' doesn't exist or isn't a directory... Aborting..." + exit 1 + fi +done + +# Disable Guix ld auto-rpath behavior +case "$HOST" in + *darwin*) + # The auto-rpath behavior is necessary for darwin builds as some native + # tools built by depends refer to and depend on Guix-built native + # libraries + # + # After the native packages in depends are built, the ld wrapper should + # no longer affect our build, as clang would instead reach for + # x86_64-apple-darwin-ld from cctools + ;; + *) export GUIX_LD_WRAPPER_DISABLE_RPATH=yes ;; +esac + +# Make /usr/bin if it doesn't exist +[ -e /usr/bin ] || mkdir -p /usr/bin + +# Symlink file and env to a conventional path +[ -e /usr/bin/file ] || ln -s --no-dereference "$(command -v file)" /usr/bin/file +[ -e /usr/bin/env ] || ln -s --no-dereference "$(command -v env)" /usr/bin/env + +# Determine the correct value for -Wl,--dynamic-linker for the current $HOST +case "$HOST" in + *linux*) + glibc_dynamic_linker=$( + case "$HOST" in + x86_64-linux-gnu) echo /lib64/ld-linux-x86-64.so.2 ;; + arm-linux-gnueabihf) echo /lib/ld-linux-armhf.so.3 ;; + aarch64-linux-gnu) echo /lib/ld-linux-aarch64.so.1 ;; + riscv64-linux-gnu) echo /lib/ld-linux-riscv64-lp64d.so.1 ;; + powerpc64-linux-gnu) echo /lib64/ld64.so.1;; + powerpc64le-linux-gnu) echo /lib64/ld64.so.2;; + *) exit 1 ;; + esac + ) + ;; +esac + +# Environment variables for determinism +export TAR_OPTIONS="--owner=0 --group=0 --numeric-owner --mtime='@${SOURCE_DATE_EPOCH}' --sort=name" +export TZ="UTC" +case "$HOST" in + *darwin*) + # cctools AR, unlike GNU binutils AR, does not have a deterministic mode + # or a configure flag to enable determinism by default, it only + # understands if this env-var is set or not. See: + # + # https://github.com/tpoechtrager/cctools-port/blob/55562e4073dea0fbfd0b20e0bf69ffe6390c7f97/cctools/ar/archive.c#L334 + export ZERO_AR_DATE=yes + ;; +esac + +#################### +# Depends Building # +#################### + +# Build the depends tree, overriding variables that assume multilib gcc +make -C depends --jobs="$JOBS" HOST="$HOST" \ + ${V:+V=1} \ + ${SOURCES_PATH+SOURCES_PATH="$SOURCES_PATH"} \ + ${BASE_CACHE+BASE_CACHE="$BASE_CACHE"} \ + ${SDK_PATH+SDK_PATH="$SDK_PATH"} \ + x86_64_linux_CC=x86_64-linux-gnu-gcc \ + x86_64_linux_CXX=x86_64-linux-gnu-g++ \ + x86_64_linux_AR=x86_64-linux-gnu-ar \ + x86_64_linux_RANLIB=x86_64-linux-gnu-ranlib \ + x86_64_linux_NM=x86_64-linux-gnu-nm \ + x86_64_linux_STRIP=x86_64-linux-gnu-strip \ + qt_config_opts_x86_64_linux='-platform linux-g++ -xplatform bitcoin-linux-g++' \ + FORCE_USE_SYSTEM_CLANG=1 + + +########################### +# Source Tarball Building # +########################### + +GIT_ARCHIVE="${DIST_ARCHIVE_BASE}/${DISTNAME}.tar.gz" + +# Create the source tarball if not already there +if [ ! -e "$GIT_ARCHIVE" ]; then + mkdir -p "$(dirname "$GIT_ARCHIVE")" + git archive --prefix="${DISTNAME}/" --output="$GIT_ARCHIVE" HEAD +fi + +mkdir -p "$OUTDIR" + +########################### +# Binary Tarball Building # +########################### + +# CONFIGFLAGS +CONFIGFLAGS="--enable-reduce-exports --disable-bench --disable-gui-tests --disable-fuzz-binary" + +# CFLAGS +HOST_CFLAGS="-O2 -g" +case "$HOST" in + *linux*) HOST_CFLAGS+=" -ffile-prefix-map=${PWD}=." ;; + *mingw*) HOST_CFLAGS+=" -fno-ident" ;; + *darwin*) unset HOST_CFLAGS ;; +esac + +# CXXFLAGS +HOST_CXXFLAGS="$HOST_CFLAGS" + +case "$HOST" in + arm-linux-gnueabihf) HOST_CXXFLAGS="${HOST_CXXFLAGS} -Wno-psabi" ;; +esac + +# LDFLAGS +case "$HOST" in + *linux*) HOST_LDFLAGS="-Wl,--as-needed -Wl,--dynamic-linker=$glibc_dynamic_linker -static-libstdc++ -Wl,-O2" ;; + *mingw*) HOST_LDFLAGS="-Wl,--no-insert-timestamp" ;; +esac + +# Using --no-tls-get-addr-optimize retains compatibility with glibc 2.18, by +# avoiding a PowerPC64 optimisation available in glibc 2.22 and later. +# https://sourceware.org/binutils/docs-2.35/ld/PowerPC64-ELF64.html +case "$HOST" in + *powerpc64*) HOST_LDFLAGS="${HOST_LDFLAGS} -Wl,--no-tls-get-addr-optimize" ;; +esac + +# Make $HOST-specific native binaries from depends available in $PATH +export PATH="${BASEPREFIX}/${HOST}/native/bin:${PATH}" +mkdir -p "$DISTSRC" +( + cd "$DISTSRC" + + # Extract the source tarball + tar --strip-components=1 -xf "${GIT_ARCHIVE}" + + ./autogen.sh + + # Configure this DISTSRC for $HOST + # shellcheck disable=SC2086 + env CONFIG_SITE="${BASEPREFIX}/${HOST}/share/config.site" \ + ./configure --prefix=/ \ + --disable-ccache \ + --disable-maintainer-mode \ + --disable-dependency-tracking \ + ${CONFIGFLAGS} \ + ${HOST_CFLAGS:+CFLAGS="${HOST_CFLAGS}"} \ + ${HOST_CXXFLAGS:+CXXFLAGS="${HOST_CXXFLAGS}"} \ + ${HOST_LDFLAGS:+LDFLAGS="${HOST_LDFLAGS}"} + + sed -i.old 's/-lstdc++ //g' config.status libtool + + # Build Bitcoin Core + make --jobs="$JOBS" ${V:+V=1} + + # Check that symbol/security checks tools are sane. + make test-security-check ${V:+V=1} + # Perform basic security checks on a series of executables. + make -C src --jobs=1 check-security ${V:+V=1} + # Check that executables only contain allowed version symbols. + make -C src --jobs=1 check-symbols ${V:+V=1} + + mkdir -p "$OUTDIR" + + # Make the os-specific installers + case "$HOST" in + *mingw*) + make deploy ${V:+V=1} BITCOIN_WIN_INSTALLER="${OUTDIR}/${DISTNAME}-win64-setup-unsigned.exe" + ;; + esac + + # Setup the directory where our Bitcoin Core build for HOST will be + # installed. This directory will also later serve as the input for our + # binary tarballs. + INSTALLPATH="${PWD}/installed/${DISTNAME}" + mkdir -p "${INSTALLPATH}" + # Install built Bitcoin Core to $INSTALLPATH + case "$HOST" in + *darwin*) + make install-strip DESTDIR="${INSTALLPATH}" ${V:+V=1} + ;; + *) + make install DESTDIR="${INSTALLPATH}" ${V:+V=1} + ;; + esac + + case "$HOST" in + *darwin*) + make osx_volname ${V:+V=1} + make deploydir ${V:+V=1} + mkdir -p "unsigned-app-${HOST}" + cp --target-directory="unsigned-app-${HOST}" \ + osx_volname \ + contrib/macdeploy/detached-sig-create.sh + mv --target-directory="unsigned-app-${HOST}" dist + ( + cd "unsigned-app-${HOST}" + find . -print0 \ + | sort --zero-terminated \ + | tar --create --no-recursion --mode='u+rw,go+r-w,a+X' --null --files-from=- \ + | gzip -9n > "${OUTDIR}/${DISTNAME}-${HOST}-unsigned.tar.gz" \ + || ( rm -f "${OUTDIR}/${DISTNAME}-${HOST}-unsigned.tar.gz" && exit 1 ) + ) + make deploy ${V:+V=1} OSX_DMG="${OUTDIR}/${DISTNAME}-${HOST}-unsigned.dmg" + ;; + esac + ( + cd installed + + case "$HOST" in + *mingw*) + mv --target-directory="$DISTNAME"/lib/ "$DISTNAME"/bin/*.dll + ;; + esac + + # Prune libtool and object archives + find . -name "lib*.la" -delete + find . -name "lib*.a" -delete + + # Prune pkg-config files + rm -rf "${DISTNAME}/lib/pkgconfig" + + case "$HOST" in + *darwin*) ;; + *) + # Split binaries and libraries from their debug symbols + { + find "${DISTNAME}/bin" -type f -executable -print0 + find "${DISTNAME}/lib" -type f -print0 + } | xargs -0 -P"$JOBS" -I{} "${DISTSRC}/contrib/devtools/split-debug.sh" {} {} {}.dbg + ;; + esac + + case "$HOST" in + *mingw*) + cp "${DISTSRC}/doc/README_windows.txt" "${DISTNAME}/readme.txt" + ;; + *linux*) + cp "${DISTSRC}/README.md" "${DISTNAME}/" + ;; + esac + + # copy over the example bitcoin.conf file. if contrib/devtools/gen-bitcoin-conf.sh + # has not been run before buildling, this file will be a stub + cp "${DISTSRC}/share/examples/bitcoin.conf" "${DISTNAME}/" + + # Finally, deterministically produce {non-,}debug binary tarballs ready + # for release + case "$HOST" in + *mingw*) + find "${DISTNAME}" -not -name "*.dbg" -print0 \ + | xargs -0r touch --no-dereference --date="@${SOURCE_DATE_EPOCH}" + find "${DISTNAME}" -not -name "*.dbg" \ + | sort \ + | zip -X@ "${OUTDIR}/${DISTNAME}-${HOST//x86_64-w64-mingw32/win64}.zip" \ + || ( rm -f "${OUTDIR}/${DISTNAME}-${HOST//x86_64-w64-mingw32/win64}.zip" && exit 1 ) + find "${DISTNAME}" -name "*.dbg" -print0 \ + | xargs -0r touch --no-dereference --date="@${SOURCE_DATE_EPOCH}" + find "${DISTNAME}" -name "*.dbg" \ + | sort \ + | zip -X@ "${OUTDIR}/${DISTNAME}-${HOST//x86_64-w64-mingw32/win64}-debug.zip" \ + || ( rm -f "${OUTDIR}/${DISTNAME}-${HOST//x86_64-w64-mingw32/win64}-debug.zip" && exit 1 ) + ;; + *linux*) + find "${DISTNAME}" -not -name "*.dbg" -print0 \ + | sort --zero-terminated \ + | tar --create --no-recursion --mode='u+rw,go+r-w,a+X' --null --files-from=- \ + | gzip -9n > "${OUTDIR}/${DISTNAME}-${HOST}.tar.gz" \ + || ( rm -f "${OUTDIR}/${DISTNAME}-${HOST}.tar.gz" && exit 1 ) + find "${DISTNAME}" -name "*.dbg" -print0 \ + | sort --zero-terminated \ + | tar --create --no-recursion --mode='u+rw,go+r-w,a+X' --null --files-from=- \ + | gzip -9n > "${OUTDIR}/${DISTNAME}-${HOST}-debug.tar.gz" \ + || ( rm -f "${OUTDIR}/${DISTNAME}-${HOST}-debug.tar.gz" && exit 1 ) + ;; + *darwin*) + find "${DISTNAME}" -print0 \ + | sort --zero-terminated \ + | tar --create --no-recursion --mode='u+rw,go+r-w,a+X' --null --files-from=- \ + | gzip -9n > "${OUTDIR}/${DISTNAME}-${HOST}.tar.gz" \ + || ( rm -f "${OUTDIR}/${DISTNAME}-${HOST}.tar.gz" && exit 1 ) + ;; + esac + ) # $DISTSRC/installed + + case "$HOST" in + *mingw*) + cp -rf --target-directory=. contrib/windeploy + ( + cd ./windeploy + mkdir -p unsigned + cp --target-directory=unsigned/ "${OUTDIR}/${DISTNAME}-win64-setup-unsigned.exe" + find . -print0 \ + | sort --zero-terminated \ + | tar --create --no-recursion --mode='u+rw,go+r-w,a+X' --null --files-from=- \ + | gzip -9n > "${OUTDIR}/${DISTNAME}-win64-unsigned.tar.gz" \ + || ( rm -f "${OUTDIR}/${DISTNAME}-win64-unsigned.tar.gz" && exit 1 ) + ) + ;; + esac +) # $DISTSRC + +rm -rf "$ACTUAL_OUTDIR" +mv --no-target-directory "$OUTDIR" "$ACTUAL_OUTDIR" \ + || ( rm -rf "$ACTUAL_OUTDIR" && exit 1 ) + +( + cd /outdir-base + { + echo "$GIT_ARCHIVE" + find "$ACTUAL_OUTDIR" -type f + } | xargs realpath --relative-base="$PWD" \ + | xargs sha256sum \ + | sort -k2 \ + | sponge "$ACTUAL_OUTDIR"/SHA256SUMS.part +) diff --git a/contrib/guix/libexec/codesign.sh b/contrib/guix/libexec/codesign.sh new file mode 100755 index 0000000000..9a5d3a1ce5 --- /dev/null +++ b/contrib/guix/libexec/codesign.sh @@ -0,0 +1,113 @@ +#!/usr/bin/env bash +# Copyright (c) 2021 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +export LC_ALL=C +set -e -o pipefail +export TZ=UTC + +# Although Guix _does_ set umask when building its own packages (in our case, +# this is all packages in manifest.scm), it does not set it for `guix +# environment`. It does make sense for at least `guix environment --container` +# to set umask, so if that change gets merged upstream and we bump the +# time-machine to a commit which includes the aforementioned change, we can +# remove this line. +# +# This line should be placed before any commands which creates files. +umask 0022 + +if [ -n "$V" ]; then + # Print both unexpanded (-v) and expanded (-x) forms of commands as they are + # read from this file. + set -vx + # Set VERBOSE for CMake-based builds + export VERBOSE="$V" +fi + +# Check that required environment variables are set +cat << EOF +Required environment variables as seen inside the container: + UNSIGNED_TARBALL: ${UNSIGNED_TARBALL:?not set} + DETACHED_SIGS_REPO: ${DETACHED_SIGS_REPO:?not set} + DIST_ARCHIVE_BASE: ${DIST_ARCHIVE_BASE:?not set} + DISTNAME: ${DISTNAME:?not set} + HOST: ${HOST:?not set} + SOURCE_DATE_EPOCH: ${SOURCE_DATE_EPOCH:?not set} + DISTSRC: ${DISTSRC:?not set} + OUTDIR: ${OUTDIR:?not set} +EOF + +ACTUAL_OUTDIR="${OUTDIR}" +OUTDIR="${DISTSRC}/output" + +git_head_version() { + local recent_tag + if recent_tag="$(git -C "$1" describe --exact-match HEAD 2> /dev/null)"; then + echo "${recent_tag#v}" + else + git -C "$1" rev-parse --short=12 HEAD + fi +} + +CODESIGNATURE_GIT_ARCHIVE="${DIST_ARCHIVE_BASE}/${DISTNAME}-codesignatures-$(git_head_version "$DETACHED_SIGS_REPO").tar.gz" + +# Create the codesignature tarball if not already there +if [ ! -e "$CODESIGNATURE_GIT_ARCHIVE" ]; then + mkdir -p "$(dirname "$CODESIGNATURE_GIT_ARCHIVE")" + git -C "$DETACHED_SIGS_REPO" archive --output="$CODESIGNATURE_GIT_ARCHIVE" HEAD +fi + +mkdir -p "$OUTDIR" + +mkdir -p "$DISTSRC" +( + cd "$DISTSRC" + + tar -xf "$UNSIGNED_TARBALL" + + mkdir -p codesignatures + tar -C codesignatures -xf "$CODESIGNATURE_GIT_ARCHIVE" + + case "$HOST" in + *mingw*) + find "$PWD" -name "*-unsigned.exe" | while read -r infile; do + infile_base="$(basename "$infile")" + + # Codesigned *-unsigned.exe and output to OUTDIR + osslsigncode attach-signature \ + -in "$infile" \ + -out "${OUTDIR}/${infile_base/-unsigned}" \ + -sigin codesignatures/win/"$infile_base".pem + done + ;; + *darwin*) + # Apply detached codesignatures to dist/ (in-place) + signapple apply dist/Bitcoin-Qt.app codesignatures/osx/dist + + # Make a DMG from dist/ + xorrisofs -D -l -V "$(< osx_volname)" -no-pad -r -dir-mode 0755 \ + -o "${OUTDIR}/${DISTNAME}-${HOST}.dmg" \ + dist \ + -- -volume_date all_file_dates ="$SOURCE_DATE_EPOCH" + ;; + *) + exit 1 + ;; + esac +) # $DISTSRC + +rm -rf "$ACTUAL_OUTDIR" +mv --no-target-directory "$OUTDIR" "$ACTUAL_OUTDIR" \ + || ( rm -rf "$ACTUAL_OUTDIR" && exit 1 ) + +( + cd /outdir-base + { + echo "$UNSIGNED_TARBALL" + echo "$CODESIGNATURE_GIT_ARCHIVE" + find "$ACTUAL_OUTDIR" -type f + } | xargs realpath --relative-base="$PWD" \ + | xargs sha256sum \ + | sort -k2 \ + | sponge "$ACTUAL_OUTDIR"/SHA256SUMS.part +) diff --git a/contrib/guix/libexec/prelude.bash b/contrib/guix/libexec/prelude.bash new file mode 100644 index 0000000000..3eb8fc02da --- /dev/null +++ b/contrib/guix/libexec/prelude.bash @@ -0,0 +1,82 @@ +#!/usr/bin/env bash +export LC_ALL=C +set -e -o pipefail + +# shellcheck source=contrib/shell/realpath.bash +source contrib/shell/realpath.bash + +# shellcheck source=contrib/shell/git-utils.bash +source contrib/shell/git-utils.bash + +################ +# Required non-builtin commands should be invocable +################ + +check_tools() { + for cmd in "$@"; do + if ! command -v "$cmd" > /dev/null 2>&1; then + echo "ERR: This script requires that '$cmd' is installed and available in your \$PATH" + exit 1 + fi + done +} + +check_tools cat env readlink dirname basename git + +################ +# We should be at the top directory of the repository +################ + +same_dir() { + local resolved1 resolved2 + resolved1="$(bash_realpath "${1}")" + resolved2="$(bash_realpath "${2}")" + [ "$resolved1" = "$resolved2" ] +} + +if ! same_dir "${PWD}" "$(git_root)"; then +cat << EOF +ERR: This script must be invoked from the top level of the git repository + +Hint: This may look something like: + env FOO=BAR ./contrib/guix/guix-<blah> + +EOF +exit 1 +fi + +################ +# Execute "$@" in a pinned, possibly older version of Guix, for reproducibility +# across time. +time-machine() { + # shellcheck disable=SC2086 + guix time-machine --url=https://git.savannah.gnu.org/git/guix.git \ + --commit=998eda3067c7d21e0d9bb3310d2f5a14b8f1c681 \ + --cores="$JOBS" \ + --keep-failed \ + --fallback \ + ${SUBSTITUTE_URLS:+--substitute-urls="$SUBSTITUTE_URLS"} \ + ${ADDITIONAL_GUIX_COMMON_FLAGS} ${ADDITIONAL_GUIX_TIMEMACHINE_FLAGS} \ + -- "$@" +} + + +################ +# Set common variables +################ + +VERSION="${FORCE_VERSION:-$(git_head_version)}" +DISTNAME="${DISTNAME:-bitcoin-${VERSION}}" + +version_base_prefix="${PWD}/guix-build-" +VERSION_BASE="${version_base_prefix}${VERSION}" # TOP + +DISTSRC_BASE="${DISTSRC_BASE:-${VERSION_BASE}}" + +OUTDIR_BASE="${OUTDIR_BASE:-${VERSION_BASE}/output}" + +var_base_basename="var" +VAR_BASE="${VAR_BASE:-${VERSION_BASE}/${var_base_basename}}" + +profiles_base_basename="profiles" +PROFILES_BASE="${PROFILES_BASE:-${VAR_BASE}/${profiles_base_basename}}" diff --git a/contrib/guix/manifest.scm b/contrib/guix/manifest.scm new file mode 100644 index 0000000000..df51d659e8 --- /dev/null +++ b/contrib/guix/manifest.scm @@ -0,0 +1,612 @@ +(use-modules (gnu) + (gnu packages) + (gnu packages autotools) + (gnu packages base) + (gnu packages bash) + (gnu packages bison) + (gnu packages certs) + (gnu packages cdrom) + (gnu packages check) + (gnu packages cmake) + (gnu packages commencement) + (gnu packages compression) + (gnu packages cross-base) + (gnu packages curl) + (gnu packages file) + (gnu packages gawk) + (gnu packages gcc) + (gnu packages gnome) + (gnu packages installers) + (gnu packages linux) + (gnu packages llvm) + (gnu packages mingw) + (gnu packages moreutils) + (gnu packages perl) + (gnu packages pkg-config) + (gnu packages python) + (gnu packages python-crypto) + (gnu packages python-web) + (gnu packages shells) + (gnu packages tls) + (gnu packages version-control) + (guix build-system gnu) + (guix build-system python) + (guix build-system trivial) + (guix download) + (guix gexp) + (guix git-download) + ((guix licenses) #:prefix license:) + (guix packages) + (guix profiles) + (guix utils)) + +(define-syntax-rule (search-our-patches file-name ...) + "Return the list of absolute file names corresponding to each +FILE-NAME found in ./patches relative to the current file." + (parameterize + ((%patch-path (list (string-append (dirname (current-filename)) "/patches")))) + (list (search-patch file-name) ...))) + +(define (make-ssp-fixed-gcc xgcc) + "Given a XGCC package, return a modified package that uses the SSP function +from glibc instead of from libssp.so. Our `symbol-check' script will complain if +we link against libssp.so, and thus will ensure that this works properly. + +Taken from: +http://www.linuxfromscratch.org/hlfs/view/development/chapter05/gcc-pass1.html" + (package + (inherit xgcc) + (arguments + (substitute-keyword-arguments (package-arguments xgcc) + ((#:make-flags flags) + `(cons "gcc_cv_libc_provides_ssp=yes" ,flags)))))) + +(define (make-gcc-rpath-link xgcc) + "Given a XGCC package, return a modified package that replace each instance of +-rpath in the default system spec that's inserted by Guix with -rpath-link" + (package + (inherit xgcc) + (arguments + (substitute-keyword-arguments (package-arguments xgcc) + ((#:phases phases) + `(modify-phases ,phases + (add-after 'pre-configure 'replace-rpath-with-rpath-link + (lambda _ + (substitute* (cons "gcc/config/rs6000/sysv4.h" + (find-files "gcc/config" + "^gnu-user.*\\.h$")) + (("-rpath=") "-rpath-link=")) + #t)))))))) + +(define (make-cross-toolchain target + base-gcc-for-libc + base-kernel-headers + base-libc + base-gcc) + "Create a cross-compilation toolchain package for TARGET" + (let* ((xbinutils (cross-binutils target)) + ;; 1. Build a cross-compiling gcc without targeting any libc, derived + ;; from BASE-GCC-FOR-LIBC + (xgcc-sans-libc (cross-gcc target + #:xgcc base-gcc-for-libc + #:xbinutils xbinutils)) + ;; 2. Build cross-compiled kernel headers with XGCC-SANS-LIBC, derived + ;; from BASE-KERNEL-HEADERS + (xkernel (cross-kernel-headers target + base-kernel-headers + xgcc-sans-libc + xbinutils)) + ;; 3. Build a cross-compiled libc with XGCC-SANS-LIBC and XKERNEL, + ;; derived from BASE-LIBC + (xlibc (cross-libc target + base-libc + xgcc-sans-libc + xbinutils + xkernel)) + ;; 4. Build a cross-compiling gcc targeting XLIBC, derived from + ;; BASE-GCC + (xgcc (cross-gcc target + #:xgcc base-gcc + #:xbinutils xbinutils + #:libc xlibc))) + ;; Define a meta-package that propagates the resulting XBINUTILS, XLIBC, and + ;; XGCC + (package + (name (string-append target "-toolchain")) + (version (package-version xgcc)) + (source #f) + (build-system trivial-build-system) + (arguments '(#:builder (begin (mkdir %output) #t))) + (propagated-inputs + `(("binutils" ,xbinutils) + ("libc" ,xlibc) + ("libc:static" ,xlibc "static") + ("gcc" ,xgcc) + ("gcc-lib" ,xgcc "lib"))) + (synopsis (string-append "Complete GCC tool chain for " target)) + (description (string-append "This package provides a complete GCC tool +chain for " target " development.")) + (home-page (package-home-page xgcc)) + (license (package-license xgcc))))) + +(define base-gcc gcc-10) +(define base-linux-kernel-headers linux-libre-headers-5.15) + +(define* (make-bitcoin-cross-toolchain target + #:key + (base-gcc-for-libc base-gcc) + (base-kernel-headers base-linux-kernel-headers) + (base-libc (make-glibc-without-werror glibc-2.24)) + (base-gcc (make-gcc-rpath-link base-gcc))) + "Convenience wrapper around MAKE-CROSS-TOOLCHAIN with default values +desirable for building Bitcoin Core release binaries." + (make-cross-toolchain target + base-gcc-for-libc + base-kernel-headers + base-libc + base-gcc)) + +(define (make-gcc-with-pthreads gcc) + (package-with-extra-configure-variable gcc "--enable-threads" "posix")) + +(define (make-mingw-w64-cross-gcc cross-gcc) + (package-with-extra-patches cross-gcc + (search-our-patches "vmov-alignment.patch" + "gcc-broken-longjmp.patch"))) + +(define (make-mingw-pthreads-cross-toolchain target) + "Create a cross-compilation toolchain package for TARGET" + (let* ((xbinutils (cross-binutils target)) + (pthreads-xlibc mingw-w64-x86_64-winpthreads) + (pthreads-xgcc (make-gcc-with-pthreads + (cross-gcc target + #:xgcc (make-ssp-fixed-gcc (make-mingw-w64-cross-gcc base-gcc)) + #:xbinutils xbinutils + #:libc pthreads-xlibc)))) + ;; Define a meta-package that propagates the resulting XBINUTILS, XLIBC, and + ;; XGCC + (package + (name (string-append target "-posix-toolchain")) + (version (package-version pthreads-xgcc)) + (source #f) + (build-system trivial-build-system) + (arguments '(#:builder (begin (mkdir %output) #t))) + (propagated-inputs + `(("binutils" ,xbinutils) + ("libc" ,pthreads-xlibc) + ("gcc" ,pthreads-xgcc) + ("gcc-lib" ,pthreads-xgcc "lib"))) + (synopsis (string-append "Complete GCC tool chain for " target)) + (description (string-append "This package provides a complete GCC tool +chain for " target " development.")) + (home-page (package-home-page pthreads-xgcc)) + (license (package-license pthreads-xgcc))))) + +(define (make-nsis-for-gcc-10 base-nsis) + (package-with-extra-patches base-nsis + (search-our-patches "nsis-gcc-10-memmove.patch"))) + +(define (fix-ppc64-nx-default lief) + (package-with-extra-patches lief + (search-our-patches "lief-fix-ppc64-nx-default.patch"))) + +(define-public lief + (package + (name "python-lief") + (version "0.12.1") + (source + (origin + (method git-fetch) + (uri (git-reference + (url "https://github.com/lief-project/LIEF.git") + (commit version))) + (file-name (git-file-name name version)) + (sha256 + (base32 + "1xzbh3bxy4rw1yamnx68da1v5s56ay4g081cyamv67256g0qy2i1")))) + (build-system python-build-system) + (arguments + `(#:phases + (modify-phases %standard-phases + (add-after 'unpack 'parallel-jobs + ;; build with multiple cores + (lambda _ + (substitute* "setup.py" (("self.parallel if self.parallel else 1") (number->string (parallel-job-count))))))))) + (native-inputs + `(("cmake" ,cmake))) + (home-page "https://github.com/lief-project/LIEF") + (synopsis "Library to Instrument Executable Formats") + (description "Python library to to provide a cross platform library which can +parse, modify and abstract ELF, PE and MachO formats.") + (license license:asl2.0))) + +(define osslsigncode + (package + (name "osslsigncode") + (version "2.0") + (source (origin + (method url-fetch) + (uri (string-append "https://github.com/mtrojnar/" + name "/archive/" version ".tar.gz")) + (sha256 + (base32 + "0byri6xny770wwb2nciq44j5071122l14bvv65axdd70nfjf0q2s")))) + (build-system gnu-build-system) + (native-inputs + `(("pkg-config" ,pkg-config) + ("autoconf" ,autoconf) + ("automake" ,automake) + ("libtool" ,libtool))) + (inputs + `(("openssl" ,openssl))) + (arguments + `(#:configure-flags + `("--without-gsf" + "--without-curl" + "--disable-dependency-tracking"))) + (home-page "https://github.com/mtrojnar/osslsigncode") + (synopsis "Authenticode signing and timestamping tool") + (description "osslsigncode is a small tool that implements part of the +functionality of the Microsoft tool signtool.exe - more exactly the Authenticode +signing and timestamping. But osslsigncode is based on OpenSSL and cURL, and +thus should be able to compile on most platforms where these exist.") + (license license:gpl3+))) ; license is with openssl exception + +(define-public python-elfesteem + (let ((commit "87bbd79ab7e361004c98cc8601d4e5f029fd8bd5")) + (package + (name "python-elfesteem") + (version (git-version "0.1" "1" commit)) + (source + (origin + (method git-fetch) + (uri (git-reference + (url "https://github.com/LRGH/elfesteem") + (commit commit))) + (file-name (git-file-name name commit)) + (sha256 + (base32 + "1nyvjisvyxyxnd0023xjf5846xd03lwawp5pfzr8vrky7wwm5maz")) + (patches (search-our-patches "elfsteem-value-error-python-39.patch")))) + (build-system python-build-system) + ;; There are no tests, but attempting to run python setup.py test leads to + ;; PYTHONPATH problems, just disable the test + (arguments '(#:tests? #f)) + (home-page "https://github.com/LRGH/elfesteem") + (synopsis "ELF/PE/Mach-O parsing library") + (description "elfesteem parses ELF, PE and Mach-O files.") + (license license:lgpl2.1)))) + +(define-public python-oscrypto + (package + (name "python-oscrypto") + (version "1.2.1") + (source + (origin + (method git-fetch) + (uri (git-reference + (url "https://github.com/wbond/oscrypto") + (commit version))) + (file-name (git-file-name name version)) + (sha256 + (base32 + "1d4d8s4z340qhvb3g5m5v3436y3a71yc26wk4749q64m09kxqc3l")) + (patches (search-our-patches "oscrypto-hard-code-openssl.patch")))) + (build-system python-build-system) + (native-search-paths + (list (search-path-specification + (variable "SSL_CERT_FILE") + (file-type 'regular) + (separator #f) ;single entry + (files '("etc/ssl/certs/ca-certificates.crt"))))) + + (propagated-inputs + `(("python-asn1crypto" ,python-asn1crypto) + ("openssl" ,openssl))) + (arguments + `(#:phases + (modify-phases %standard-phases + (add-after 'unpack 'hard-code-path-to-libscrypt + (lambda* (#:key inputs #:allow-other-keys) + (let ((openssl (assoc-ref inputs "openssl"))) + (substitute* "oscrypto/__init__.py" + (("@GUIX_OSCRYPTO_USE_OPENSSL@") + (string-append openssl "/lib/libcrypto.so" "," openssl "/lib/libssl.so"))) + #t))) + (add-after 'unpack 'disable-broken-tests + (lambda _ + ;; This test is broken as there is no keyboard interrupt. + (substitute* "tests/test_trust_list.py" + (("^(.*)class TrustListTests" line indent) + (string-append indent + "@unittest.skip(\"Disabled by Guix\")\n" + line))) + (substitute* "tests/test_tls.py" + (("^(.*)class TLSTests" line indent) + (string-append indent + "@unittest.skip(\"Disabled by Guix\")\n" + line))) + #t)) + (replace 'check + (lambda _ + (invoke "python" "run.py" "tests") + #t))))) + (home-page "https://github.com/wbond/oscrypto") + (synopsis "Compiler-free Python crypto library backed by the OS") + (description "oscrypto is a compilation-free, always up-to-date encryption library for Python.") + (license license:expat))) + +(define-public python-oscryptotests + (package (inherit python-oscrypto) + (name "python-oscryptotests") + (propagated-inputs + `(("python-oscrypto" ,python-oscrypto))) + (arguments + `(#:tests? #f + #:phases + (modify-phases %standard-phases + (add-after 'unpack 'hard-code-path-to-libscrypt + (lambda* (#:key inputs #:allow-other-keys) + (chdir "tests") + #t))))))) + +(define-public python-certvalidator + (let ((commit "a145bf25eb75a9f014b3e7678826132efbba6213")) + (package + (name "python-certvalidator") + (version (git-version "0.1" "1" commit)) + (source + (origin + (method git-fetch) + (uri (git-reference + (url "https://github.com/achow101/certvalidator") + (commit commit))) + (file-name (git-file-name name commit)) + (sha256 + (base32 + "1qw2k7xis53179lpqdqyylbcmp76lj7sagp883wmxg5i7chhc96k")))) + (build-system python-build-system) + (propagated-inputs + `(("python-asn1crypto" ,python-asn1crypto) + ("python-oscrypto" ,python-oscrypto) + ("python-oscryptotests", python-oscryptotests))) ;; certvalidator tests import oscryptotests + (arguments + `(#:phases + (modify-phases %standard-phases + (add-after 'unpack 'disable-broken-tests + (lambda _ + (substitute* "tests/test_certificate_validator.py" + (("^(.*)class CertificateValidatorTests" line indent) + (string-append indent + "@unittest.skip(\"Disabled by Guix\")\n" + line))) + (substitute* "tests/test_crl_client.py" + (("^(.*)def test_fetch_crl" line indent) + (string-append indent + "@unittest.skip(\"Disabled by Guix\")\n" + line))) + (substitute* "tests/test_ocsp_client.py" + (("^(.*)def test_fetch_ocsp" line indent) + (string-append indent + "@unittest.skip(\"Disabled by Guix\")\n" + line))) + (substitute* "tests/test_registry.py" + (("^(.*)def test_build_paths" line indent) + (string-append indent + "@unittest.skip(\"Disabled by Guix\")\n" + line))) + (substitute* "tests/test_validate.py" + (("^(.*)def test_revocation_mode_hard" line indent) + (string-append indent + "@unittest.skip(\"Disabled by Guix\")\n" + line))) + #t)) + (replace 'check + (lambda _ + (invoke "python" "run.py" "tests") + #t))))) + (home-page "https://github.com/wbond/certvalidator") + (synopsis "Python library for validating X.509 certificates and paths") + (description "certvalidator is a Python library for validating X.509 +certificates or paths. Supports various options, including: validation at a +specific moment in time, whitelisting and revocation checks.") + (license license:expat)))) + +(define-public python-altgraph + (package + (name "python-altgraph") + (version "0.17") + (source + (origin + (method git-fetch) + (uri (git-reference + (url "https://github.com/ronaldoussoren/altgraph") + (commit (string-append "v" version)))) + (file-name (git-file-name name version)) + (sha256 + (base32 + "09sm4srvvkw458pn48ga9q7ykr4xlz7q8gh1h9w7nxpf001qgpwb")))) + (build-system python-build-system) + (home-page "https://github.com/ronaldoussoren/altgraph") + (synopsis "Python graph (network) package") + (description "altgraph is a fork of graphlib: a graph (network) package for +constructing graphs, BFS and DFS traversals, topological sort, shortest paths, +etc. with graphviz output.") + (license license:expat))) + + +(define-public python-macholib + (package + (name "python-macholib") + (version "1.14") + (source + (origin + (method git-fetch) + (uri (git-reference + (url "https://github.com/ronaldoussoren/macholib") + (commit (string-append "v" version)))) + (file-name (git-file-name name version)) + (sha256 + (base32 + "0aislnnfsza9wl4f0vp45ivzlc0pzhp9d4r08700slrypn5flg42")))) + (build-system python-build-system) + (propagated-inputs + `(("python-altgraph" ,python-altgraph))) + (arguments + '(#:phases + (modify-phases %standard-phases + (add-after 'unpack 'disable-broken-tests + (lambda _ + ;; This test is broken as there is no keyboard interrupt. + (substitute* "macholib_tests/test_command_line.py" + (("^(.*)class TestCmdLine" line indent) + (string-append indent + "@unittest.skip(\"Disabled by Guix\")\n" + line))) + (substitute* "macholib_tests/test_dyld.py" + (("^(.*)def test_\\S+_find" line indent) + (string-append indent + "@unittest.skip(\"Disabled by Guix\")\n" + line)) + (("^(.*)def testBasic" line indent) + (string-append indent + "@unittest.skip(\"Disabled by Guix\")\n" + line)) + ) + #t))))) + (home-page "https://github.com/ronaldoussoren/macholib") + (synopsis "Python library for analyzing and editing Mach-O headers") + (description "macholib is a Macho-O header analyzer and editor. It's +typically used as a dependency analysis tool, and also to rewrite dylib +references in Mach-O headers to be @executable_path relative. Though this tool +targets a platform specific file format, it is pure python code that is platform +and endian independent.") + (license license:expat))) + +(define-public python-signapple + (let ((commit "8a945a2e7583be2665cf3a6a89d665b70ecd1ab6")) + (package + (name "python-signapple") + (version (git-version "0.1" "1" commit)) + (source + (origin + (method git-fetch) + (uri (git-reference + (url "https://github.com/achow101/signapple") + (commit commit))) + (file-name (git-file-name name commit)) + (sha256 + (base32 + "0fr1hangvfyiwflca6jg5g8zvg3jc9qr7vd2c12ff89pznf38dlg")))) + (build-system python-build-system) + (propagated-inputs + `(("python-asn1crypto" ,python-asn1crypto) + ("python-oscrypto" ,python-oscrypto) + ("python-certvalidator" ,python-certvalidator) + ("python-elfesteem" ,python-elfesteem) + ("python-requests" ,python-requests) + ("python-macholib" ,python-macholib))) + ;; There are no tests, but attempting to run python setup.py test leads to + ;; problems, just disable the test + (arguments '(#:tests? #f)) + (home-page "https://github.com/achow101/signapple") + (synopsis "Mach-O binary signature tool") + (description "signapple is a Python tool for creating, verifying, and +inspecting signatures in Mach-O binaries.") + (license license:expat)))) + +(define (make-glibc-without-werror glibc) + (package-with-extra-configure-variable glibc "enable_werror" "no")) + +(define-public glibc-2.24 + (package + (inherit glibc-2.31) + (version "2.24") + (source (origin + (method git-fetch) + (uri (git-reference + (url "https://sourceware.org/git/glibc.git") + (commit "0d7f1ed30969886c8dde62fbf7d2c79967d4bace"))) + (file-name (git-file-name "glibc" "0d7f1ed30969886c8dde62fbf7d2c79967d4bace")) + (sha256 + (base32 + "0g5hryia5v1k0qx97qffgwzrz4lr4jw3s5kj04yllhswsxyjbic3")) + (patches (search-our-patches "glibc-ldd-x86_64.patch" + "glibc-versioned-locpath.patch" + "glibc-2.24-elfm-loadaddr-dynamic-rewrite.patch" + "glibc-2.24-no-build-time-cxx-header-run.patch" + "glibc-2.24-fcommon.patch")))))) + +(define-public glibc-2.27/bitcoin-patched + (package + (inherit glibc-2.31) + (version "2.27") + (source (origin + (method git-fetch) + (uri (git-reference + (url "https://sourceware.org/git/glibc.git") + (commit "23158b08a0908f381459f273a984c6fd328363cb"))) + (file-name (git-file-name "glibc" "23158b08a0908f381459f273a984c6fd328363cb")) + (sha256 + (base32 + "1b2n1gxv9f4fd5yy68qjbnarhf8mf4vmlxk10i3328c1w5pmp0ca")) + (patches (search-our-patches "glibc-ldd-x86_64.patch" + "glibc-2.27-riscv64-Use-__has_include-to-include-asm-syscalls.h.patch" + "glibc-2.27-dont-redefine-nss-database.patch")))))) + +(packages->manifest + (append + (list ;; The Basics + bash + which + coreutils + util-linux + ;; File(system) inspection + file + grep + diffutils + findutils + ;; File transformation + patch + gawk + sed + moreutils + ;; Compression and archiving + tar + bzip2 + gzip + xz + ;; Build tools + gnu-make + libtool-2.4.7 + autoconf-2.71 + automake + pkg-config + bison + ;; Native GCC 10 toolchain + gcc-toolchain-10 + (list gcc-toolchain-10 "static") + ;; Scripting + perl + python-3 + ;; Git + git + ;; Tests + (fix-ppc64-nx-default lief)) + (let ((target (getenv "HOST"))) + (cond ((string-suffix? "-mingw32" target) + ;; Windows + (list zip + (make-mingw-pthreads-cross-toolchain "x86_64-w64-mingw32") + (make-nsis-for-gcc-10 nsis-x86_64) + osslsigncode)) + ((string-contains target "-linux-") + (list (cond ((string-contains target "riscv64-") + (make-bitcoin-cross-toolchain target + #:base-libc (make-glibc-without-werror glibc-2.27/bitcoin-patched) + #:base-kernel-headers base-linux-kernel-headers)) + (else + (make-bitcoin-cross-toolchain target))))) + ((string-contains target "darwin") + (list clang-toolchain-10 binutils cmake xorriso python-signapple)) + (else '()))))) diff --git a/contrib/guix/patches/elfsteem-value-error-python-39.patch b/contrib/guix/patches/elfsteem-value-error-python-39.patch new file mode 100644 index 0000000000..21e1228afd --- /dev/null +++ b/contrib/guix/patches/elfsteem-value-error-python-39.patch @@ -0,0 +1,13 @@ +diff --git a/examples/otool.py b/examples/otool.py +index 2b8efc0..d797b2e 100755 +--- a/examples/otool.py ++++ b/examples/otool.py +@@ -342,7 +342,7 @@ if __name__ == '__main__': + try: + e = macho_init.MACHO(raw, + parseSymbols = False) +- except ValueError, err: ++ except ValueError as err: + print("%s:" %file) + print(" %s" % err) + continue diff --git a/contrib/guix/patches/gcc-broken-longjmp.patch b/contrib/guix/patches/gcc-broken-longjmp.patch new file mode 100644 index 0000000000..1cfc0918b0 --- /dev/null +++ b/contrib/guix/patches/gcc-broken-longjmp.patch @@ -0,0 +1,68 @@ +commit eb5698897c52702498938592d7f76e67d126451f +Author: Eric Botcazou <ebotcazou@adacore.com> +Date: Wed May 5 22:48:51 2021 +0200 + + Fix PR target/100402 + + This is a regression for 64-bit Windows present from mainline down to the 9 + branch and introduced by the fix for PR target/99234. Again SEH, but with + a twist related to the way MinGW implements setjmp/longjmp, which turns out + to be piggybacked on SEH with recent versions of MinGW, i.e. the longjmp + performs a bona-fide unwinding of the stack, because it calls RtlUnwindEx + with the second argument initially passed to setjmp, which is the result of + __builtin_frame_address (0) in the MinGW header file: + + define setjmp(BUF) _setjmp((BUF), __builtin_frame_address (0)) + + This means that we directly expose the frame pointer to the SEH machinery + here (unlike with regular exception handling where we use an intermediate + CFA) and thus that we cannot do whatever we want with it. The old code + would leave it unaligned, i.e. not multiple of 16, whereas the new code + aligns it, but this breaks for some reason; at least it appears that a + .seh_setframe directive with 0 as second argument always works, so the + fix aligns it this way. + + gcc/ + PR target/100402 + * config/i386/i386.c (ix86_compute_frame_layout): For a SEH target, + always return the establisher frame for __builtin_frame_address (0). + gcc/testsuite/ + * gcc.c-torture/execute/20210505-1.c: New test. + +diff --git a/gcc/config/i386/i386.c b/gcc/config/i386/i386.c +index 2f838840e96..06ad1b2274e 100644 +--- a/gcc/config/i386/i386.c ++++ b/gcc/config/i386/i386.c +@@ -6356,12 +6356,29 @@ ix86_compute_frame_layout (void) + area, see the SEH code in config/i386/winnt.c for the rationale. */ + frame->hard_frame_pointer_offset = frame->sse_reg_save_offset; + +- /* If we can leave the frame pointer where it is, do so. Also, return ++ /* If we can leave the frame pointer where it is, do so; however return + the establisher frame for __builtin_frame_address (0) or else if the +- frame overflows the SEH maximum frame size. */ ++ frame overflows the SEH maximum frame size. ++ ++ Note that the value returned by __builtin_frame_address (0) is quite ++ constrained, because setjmp is piggybacked on the SEH machinery with ++ recent versions of MinGW: ++ ++ # elif defined(__SEH__) ++ # if defined(__aarch64__) || defined(_ARM64_) ++ # define setjmp(BUF) _setjmp((BUF), __builtin_sponentry()) ++ # elif (__MINGW_GCC_VERSION < 40702) ++ # define setjmp(BUF) _setjmp((BUF), mingw_getsp()) ++ # else ++ # define setjmp(BUF) _setjmp((BUF), __builtin_frame_address (0)) ++ # endif ++ ++ and the second argument passed to _setjmp, if not null, is forwarded ++ to the TargetFrame parameter of RtlUnwindEx by longjmp (after it has ++ built an ExceptionRecord on the fly describing the setjmp buffer). */ + const HOST_WIDE_INT diff + = frame->stack_pointer_offset - frame->hard_frame_pointer_offset; +- if (diff <= 255) ++ if (diff <= 255 && !crtl->accesses_prior_frames) + { + /* The resulting diff will be a multiple of 16 lower than 255, + i.e. at most 240 as required by the unwind data structure. */ diff --git a/contrib/guix/patches/glibc-2.24-elfm-loadaddr-dynamic-rewrite.patch b/contrib/guix/patches/glibc-2.24-elfm-loadaddr-dynamic-rewrite.patch new file mode 100644 index 0000000000..5c4d0c6ebe --- /dev/null +++ b/contrib/guix/patches/glibc-2.24-elfm-loadaddr-dynamic-rewrite.patch @@ -0,0 +1,62 @@ +https://sourceware.org/git/?p=glibc.git;a=commit;h=a68ba2f3cd3cbe32c1f31e13c20ed13487727b32 + +commit 6b02af31e9a721bb15a11380cd22d53b621711f8 +Author: Szabolcs Nagy <szabolcs.nagy@arm.com> +Date: Wed Oct 18 17:26:23 2017 +0100 + + [AARCH64] Rewrite elf_machine_load_address using _DYNAMIC symbol + + This patch rewrites aarch64 elf_machine_load_address to use special _DYNAMIC + symbol instead of _dl_start. + + The static address of _DYNAMIC symbol is stored in the first GOT entry. + Here is the change which makes this solution work (part of binutils 2.24): + https://sourceware.org/ml/binutils/2013-06/msg00248.html + + i386, x86_64 targets use the same method to do this as well. + + The original implementation relies on a trick that R_AARCH64_ABS32 relocation + being resolved at link time and the static address fits in the 32bits. + However, in LP64, normally, the address is defined to be 64 bit. + + Here is the C version one which should be portable in all cases. + + * sysdeps/aarch64/dl-machine.h (elf_machine_load_address): Use + _DYNAMIC symbol to calculate load address. + +diff --git a/sysdeps/aarch64/dl-machine.h b/sysdeps/aarch64/dl-machine.h +index e86d8b5b63..5a5b8a5de5 100644 +--- a/sysdeps/aarch64/dl-machine.h ++++ b/sysdeps/aarch64/dl-machine.h +@@ -49,26 +49,11 @@ elf_machine_load_address (void) + /* To figure out the load address we use the definition that for any symbol: + dynamic_addr(symbol) = static_addr(symbol) + load_addr + +- The choice of symbol is arbitrary. The static address we obtain +- by constructing a non GOT reference to the symbol, the dynamic +- address of the symbol we compute using adrp/add to compute the +- symbol's address relative to the PC. +- This depends on 32bit relocations being resolved at link time +- and that the static address fits in the 32bits. */ +- +- ElfW(Addr) static_addr; +- ElfW(Addr) dynamic_addr; +- +- asm (" \n" +-" adrp %1, _dl_start; \n" +-" add %1, %1, #:lo12:_dl_start \n" +-" ldr %w0, 1f \n" +-" b 2f \n" +-"1: \n" +-" .word _dl_start \n" +-"2: \n" +- : "=r" (static_addr), "=r" (dynamic_addr)); +- return dynamic_addr - static_addr; ++ _DYNAMIC sysmbol is used here as its link-time address stored in ++ the special unrelocated first GOT entry. */ ++ ++ extern ElfW(Dyn) _DYNAMIC[] attribute_hidden; ++ return (ElfW(Addr)) &_DYNAMIC - elf_machine_dynamic (); + } + + /* Set up the loaded object described by L so its unrelocated PLT diff --git a/contrib/guix/patches/glibc-2.24-fcommon.patch b/contrib/guix/patches/glibc-2.24-fcommon.patch new file mode 100644 index 0000000000..2bc32ede90 --- /dev/null +++ b/contrib/guix/patches/glibc-2.24-fcommon.patch @@ -0,0 +1,32 @@ +commit 264a4a0dbe1f4369db315080034b500bed66016c +Author: fanquake <fanquake@gmail.com> +Date: Fri May 6 11:03:04 2022 +0100 + + build: use -fcommon to retain legacy behaviour with GCC 10 + + GCC 10 started using -fno-common by default, which causes issues with + the powerpc builds using gibc 2.24. A patch was commited to glibc to fix + the issue, 18363b4f010da9ba459b13310b113ac0647c2fcc but is non-trvial + to backport, and was broken in at least one way, see the followup in + commit 7650321ce037302bfc2f026aa19e0213b8d02fe6. + + For now, retain the legacy GCC behaviour by passing -fcommon when + building glibc. + + https://gcc.gnu.org/onlinedocs/gcc/Code-Gen-Options.html. + https://sourceware.org/git/?p=glibc.git;a=commit;h=18363b4f010da9ba459b13310b113ac0647c2fcc + https://sourceware.org/git/?p=glibc.git;a=commit;h=7650321ce037302bfc2f026aa19e0213b8d02fe6 + +diff --git a/Makeconfig b/Makeconfig +index ee379f5852..63c4a2f234 100644 +--- a/Makeconfig ++++ b/Makeconfig +@@ -824,7 +824,7 @@ ifeq "$(strip $(+cflags))" "" + +cflags := $(default_cflags) + endif # $(+cflags) == "" + +-+cflags += $(cflags-cpu) $(+gccwarn) $(+merge-constants) $(+math-flags) +++cflags += $(cflags-cpu) $(+gccwarn) $(+merge-constants) $(+math-flags) -fcommon + +gcc-nowarn := -w + + # Don't duplicate options if we inherited variables from the parent. diff --git a/contrib/guix/patches/glibc-2.24-no-build-time-cxx-header-run.patch b/contrib/guix/patches/glibc-2.24-no-build-time-cxx-header-run.patch new file mode 100644 index 0000000000..11fe7fdc99 --- /dev/null +++ b/contrib/guix/patches/glibc-2.24-no-build-time-cxx-header-run.patch @@ -0,0 +1,100 @@ +https://sourceware.org/git/?p=glibc.git;a=commit;h=fc3e1337be1c6935ab58bd13520f97a535cf70cc + +commit dc23a45db566095e83ff0b7a57afc87fb5ca89a1 +Author: Florian Weimer <fweimer@redhat.com> +Date: Wed Sep 21 10:45:32 2016 +0200 + + Avoid running $(CXX) during build to obtain header file paths + + This reduces the build time somewhat and is particularly noticeable + during rebuilds with few code changes. + +diff --git a/Makerules b/Makerules +index 7e4077ee50..c338850de5 100644 +--- a/Makerules ++++ b/Makerules +@@ -121,14 +121,10 @@ ifneq (,$(CXX)) + # will be used instead of /usr/include/stdlib.h and /usr/include/math.h. + before-compile := $(common-objpfx)cstdlib $(common-objpfx)cmath \ + $(before-compile) +-cstdlib=$(shell echo "\#include <cstdlib>" | $(CXX) -M -MP -x c++ - \ +- | sed -n "/cstdlib:/{s/:$$//;p}") +-$(common-objpfx)cstdlib: $(cstdlib) ++$(common-objpfx)cstdlib: $(c++-cstdlib-header) + $(INSTALL_DATA) $< $@T + $(move-if-change) $@T $@ +-cmath=$(shell echo "\#include <cmath>" | $(CXX) -M -MP -x c++ - \ +- | sed -n "/cmath:/{s/:$$//;p}") +-$(common-objpfx)cmath: $(cmath) ++$(common-objpfx)cmath: $(c++-cmath-header) + $(INSTALL_DATA) $< $@T + $(move-if-change) $@T $@ + endif +diff --git a/config.make.in b/config.make.in +index 95c6f36876..04a8b3ed7f 100644 +--- a/config.make.in ++++ b/config.make.in +@@ -45,6 +45,8 @@ defines = @DEFINES@ + sysheaders = @sysheaders@ + sysincludes = @SYSINCLUDES@ + c++-sysincludes = @CXX_SYSINCLUDES@ ++c++-cstdlib-header = @CXX_CSTDLIB_HEADER@ ++c++-cmath-header = @CXX_CMATH_HEADER@ + all-warnings = @all_warnings@ + enable-werror = @enable_werror@ + +diff --git a/configure b/configure +index 17625e1041..6ff252744b 100755 +--- a/configure ++++ b/configure +@@ -635,6 +635,8 @@ BISON + INSTALL_INFO + PERL + BASH_SHELL ++CXX_CMATH_HEADER ++CXX_CSTDLIB_HEADER + CXX_SYSINCLUDES + SYSINCLUDES + AUTOCONF +@@ -5054,6 +5056,18 @@ fi + + + ++# Obtain some C++ header file paths. This is used to make a local ++# copy of those headers in Makerules. ++if test -n "$CXX"; then ++ find_cxx_header () { ++ echo "#include <$1>" | $CXX -M -MP -x c++ - | sed -n "/$1:/{s/:\$//;p}" ++ } ++ CXX_CSTDLIB_HEADER="$(find_cxx_header cstdlib)" ++ CXX_CMATH_HEADER="$(find_cxx_header cmath)" ++fi ++ ++ ++ + # Test if LD_LIBRARY_PATH contains the notation for the current directory + # since this would lead to problems installing/building glibc. + # LD_LIBRARY_PATH contains the current directory if one of the following +diff --git a/configure.ac b/configure.ac +index 33bcd62180..9938ab0dc2 100644 +--- a/configure.ac ++++ b/configure.ac +@@ -1039,6 +1039,18 @@ fi + AC_SUBST(SYSINCLUDES) + AC_SUBST(CXX_SYSINCLUDES) + ++# Obtain some C++ header file paths. This is used to make a local ++# copy of those headers in Makerules. ++if test -n "$CXX"; then ++ find_cxx_header () { ++ echo "#include <$1>" | $CXX -M -MP -x c++ - | sed -n "/$1:/{s/:\$//;p}" ++ } ++ CXX_CSTDLIB_HEADER="$(find_cxx_header cstdlib)" ++ CXX_CMATH_HEADER="$(find_cxx_header cmath)" ++fi ++AC_SUBST(CXX_CSTDLIB_HEADER) ++AC_SUBST(CXX_CMATH_HEADER) ++ + # Test if LD_LIBRARY_PATH contains the notation for the current directory + # since this would lead to problems installing/building glibc. + # LD_LIBRARY_PATH contains the current directory if one of the following diff --git a/contrib/guix/patches/glibc-2.27-dont-redefine-nss-database.patch b/contrib/guix/patches/glibc-2.27-dont-redefine-nss-database.patch new file mode 100644 index 0000000000..16a595d613 --- /dev/null +++ b/contrib/guix/patches/glibc-2.27-dont-redefine-nss-database.patch @@ -0,0 +1,87 @@ +commit 78a90c2f74a2012dd3eff302189e47ff6779a757 +Author: Andreas Schwab <schwab@linux-m68k.org> +Date: Fri Mar 2 23:07:14 2018 +0100 + + Fix multiple definitions of __nss_*_database (bug 22918) + + (cherry picked from commit eaf6753f8aac33a36deb98c1031d1bad7b593d2d) + +diff --git a/nscd/gai.c b/nscd/gai.c +index d081747797..576fd0045b 100644 +--- a/nscd/gai.c ++++ b/nscd/gai.c +@@ -45,3 +45,6 @@ + #ifdef HAVE_LIBIDN + # include <libidn/idn-stub.c> + #endif ++ ++/* Some variables normally defined in libc. */ ++service_user *__nss_hosts_database attribute_hidden; +diff --git a/nss/nsswitch.c b/nss/nsswitch.c +index d5e655974f..b0f0c11a3e 100644 +--- a/nss/nsswitch.c ++++ b/nss/nsswitch.c +@@ -62,7 +62,7 @@ static service_library *nss_new_service (name_database *database, + + /* Declare external database variables. */ + #define DEFINE_DATABASE(name) \ +- extern service_user *__nss_##name##_database attribute_hidden; \ ++ service_user *__nss_##name##_database attribute_hidden; \ + weak_extern (__nss_##name##_database) + #include "databases.def" + #undef DEFINE_DATABASE +diff --git a/nss/nsswitch.h b/nss/nsswitch.h +index eccb535ef5..63573b9ebc 100644 +--- a/nss/nsswitch.h ++++ b/nss/nsswitch.h +@@ -226,10 +226,10 @@ libc_hidden_proto (__nss_hostname_digits_dots) + #define MAX_NR_ADDRS 48 + + /* Prototypes for __nss_*_lookup2 functions. */ +-#define DEFINE_DATABASE(arg) \ +- service_user *__nss_##arg##_database attribute_hidden; \ +- int __nss_##arg##_lookup2 (service_user **, const char *, \ +- const char *, void **); \ ++#define DEFINE_DATABASE(arg) \ ++ extern service_user *__nss_##arg##_database attribute_hidden; \ ++ int __nss_##arg##_lookup2 (service_user **, const char *, \ ++ const char *, void **); \ + libc_hidden_proto (__nss_##arg##_lookup2) + #include "databases.def" + #undef DEFINE_DATABASE +diff --git a/posix/tst-rfc3484-2.c b/posix/tst-rfc3484-2.c +index f509534ca9..8c64ac59ff 100644 +--- a/posix/tst-rfc3484-2.c ++++ b/posix/tst-rfc3484-2.c +@@ -58,6 +58,7 @@ _res_hconf_init (void) + #undef USE_NSCD + #include "../sysdeps/posix/getaddrinfo.c" + ++service_user *__nss_hosts_database attribute_hidden; + + /* This is the beginning of the real test code. The above defines + (among other things) the function rfc3484_sort. */ +diff --git a/posix/tst-rfc3484-3.c b/posix/tst-rfc3484-3.c +index ae44087a10..1c61aaf844 100644 +--- a/posix/tst-rfc3484-3.c ++++ b/posix/tst-rfc3484-3.c +@@ -58,6 +58,7 @@ _res_hconf_init (void) + #undef USE_NSCD + #include "../sysdeps/posix/getaddrinfo.c" + ++service_user *__nss_hosts_database attribute_hidden; + + /* This is the beginning of the real test code. The above defines + (among other things) the function rfc3484_sort. */ +diff --git a/posix/tst-rfc3484.c b/posix/tst-rfc3484.c +index 7f191abbbc..8f45848e44 100644 +--- a/posix/tst-rfc3484.c ++++ b/posix/tst-rfc3484.c +@@ -58,6 +58,7 @@ _res_hconf_init (void) + #undef USE_NSCD + #include "../sysdeps/posix/getaddrinfo.c" + ++service_user *__nss_hosts_database attribute_hidden; + + /* This is the beginning of the real test code. The above defines + (among other things) the function rfc3484_sort. */ diff --git a/contrib/guix/patches/glibc-2.27-riscv64-Use-__has_include-to-include-asm-syscalls.h.patch b/contrib/guix/patches/glibc-2.27-riscv64-Use-__has_include-to-include-asm-syscalls.h.patch new file mode 100644 index 0000000000..c0f8495c41 --- /dev/null +++ b/contrib/guix/patches/glibc-2.27-riscv64-Use-__has_include-to-include-asm-syscalls.h.patch @@ -0,0 +1,76 @@ +Note that this has been modified from the original commit, to use __has_include +instead of __has_include__, as the later was causing build failures with GCC 10. +See also: http://lists.busybox.net/pipermail/buildroot/2020-July/590376.html. + +https://sourceware.org/git/?p=glibc.git;a=commit;h=0b9c84906f653978fb8768c7ebd0ee14a47e662e + +From 562c52cc81a4e456a62e6455feb32732049e9070 Mon Sep 17 00:00:00 2001 +From: "H.J. Lu" <hjl.tools@gmail.com> +Date: Mon, 31 Dec 2018 09:26:42 -0800 +Subject: [PATCH] riscv: Use __has_include__ to include <asm/syscalls.h> [BZ + #24022] + +<asm/syscalls.h> has been removed by + +commit 27f8899d6002e11a6e2d995e29b8deab5aa9cc25 +Author: David Abdurachmanov <david.abdurachmanov@gmail.com> +Date: Thu Nov 8 20:02:39 2018 +0100 + + riscv: add asm/unistd.h UAPI header + + Marcin Juszkiewicz reported issues while generating syscall table for riscv + using 4.20-rc1. The patch refactors our unistd.h files to match some other + architectures. + + - Add asm/unistd.h UAPI header, which has __ARCH_WANT_NEW_STAT only for 64-bit + - Remove asm/syscalls.h UAPI header and merge to asm/unistd.h + - Adjust kernel asm/unistd.h + + So now asm/unistd.h UAPI header should show all syscalls for riscv. + +<asm/syscalls.h> may be restored by + +Subject: [PATCH] riscv: restore asm/syscalls.h UAPI header +Date: Tue, 11 Dec 2018 09:09:35 +0100 + +UAPI header asm/syscalls.h was merged into UAPI asm/unistd.h header, +which did resolve issue with missing syscalls macros resulting in +glibc (2.28) build failure. It also broke glibc in a different way: +asm/syscalls.h is being used by glibc. I noticed this while doing +Fedora 30/Rawhide mass rebuild. + +The patch returns asm/syscalls.h header and incl. it into asm/unistd.h. +I plan to send a patch to glibc to use asm/unistd.h instead of +asm/syscalls.h + +In the meantime, we use __has_include__, which was added to GCC 5, to +check if <asm/syscalls.h> exists before including it. Tested with +build-many-glibcs.py for riscv against kernel 4.19.12 and 4.20-rc7. + + [BZ #24022] + * sysdeps/unix/sysv/linux/riscv/flush-icache.c: Check if + <asm/syscalls.h> exists with __has_include__ before including it. +--- + sysdeps/unix/sysv/linux/riscv/flush-icache.c | 6 +++++- + 1 file changed, 5 insertions(+), 1 deletion(-) + +diff --git a/sysdeps/unix/sysv/linux/riscv/flush-icache.c b/sysdeps/unix/sysv/linux/riscv/flush-icache.c +index d612ef4c6c..0b2042620b 100644 +--- a/sysdeps/unix/sysv/linux/riscv/flush-icache.c ++++ b/sysdeps/unix/sysv/linux/riscv/flush-icache.c +@@ -21,7 +21,11 @@ + #include <stdlib.h> + #include <atomic.h> + #include <sys/cachectl.h> +-#include <asm/syscalls.h> ++#if __has_include (<asm/syscalls.h>) ++# include <asm/syscalls.h> ++#else ++# include <asm/unistd.h> ++#endif + + typedef int (*func_type) (void *, void *, unsigned long int); + +-- +2.31.1 + diff --git a/contrib/guix/patches/glibc-ldd-x86_64.patch b/contrib/guix/patches/glibc-ldd-x86_64.patch new file mode 100644 index 0000000000..b1b6d5a548 --- /dev/null +++ b/contrib/guix/patches/glibc-ldd-x86_64.patch @@ -0,0 +1,10 @@ +By default, 'RTDLLIST' in 'ldd' refers to 'lib64/ld-linux-x86-64.so', whereas +it's in 'lib/' for us. This patch fixes that. + +--- glibc-2.17/sysdeps/unix/sysv/linux/x86_64/ldd-rewrite.sed 2012-12-25 04:02:13.000000000 +0100 ++++ glibc-2.17/sysdeps/unix/sysv/linux/x86_64/ldd-rewrite.sed 2013-09-15 23:08:03.000000000 +0200 +@@ -1,3 +1,3 @@ + /LD_TRACE_LOADED_OBJECTS=1/a\ + add_env="$add_env LD_LIBRARY_VERSION=\\$verify_out" +-s_^\(RTLDLIST=\)\(.*lib\)\(\|64\|x32\)\(/[^/]*\)\(-x86-64\|-x32\)\(\.so\.[0-9.]*\)[ ]*$_\1"\2\4\6 \264\4-x86-64\6 \2x32\4-x32\6"_ ++s_^\(RTLDLIST=\)\(.*lib\)\(\|64\|x32\)\(/[^/]*\)\(-x86-64\|-x32\)\(\.so\.[0-9.]*\)[ ]*$_\1"\2\4\6 \2\4-x86-64\6 \2x32\4-x32\6"_ diff --git a/contrib/guix/patches/glibc-versioned-locpath.patch b/contrib/guix/patches/glibc-versioned-locpath.patch new file mode 100644 index 0000000000..bc7652127f --- /dev/null +++ b/contrib/guix/patches/glibc-versioned-locpath.patch @@ -0,0 +1,240 @@ +The format of locale data can be incompatible between libc versions, and +loading incompatible data can lead to 'setlocale' returning EINVAL at best +or triggering an assertion failure at worst. See +https://lists.gnu.org/archive/html/guix-devel/2015-09/msg00717.html +for background information. + +To address that, this patch changes libc to honor a new 'GUIX_LOCPATH' +variable, and to look for locale data in version-specific sub-directories of +that variable. So, if GUIX_LOCPATH=/foo:/bar, locale data is searched for in +/foo/X.Y and /bar/X.Y, where X.Y is the libc version number. + +That way, a single 'GUIX_LOCPATH' setting can work even if different libc +versions coexist on the system. + +--- a/locale/newlocale.c ++++ b/locale/newlocale.c +@@ -30,6 +30,7 @@ + /* Lock for protecting global data. */ + __libc_rwlock_define (extern , __libc_setlocale_lock attribute_hidden) + ++extern error_t compute_locale_search_path (char **, size_t *); + + /* Use this when we come along an error. */ + #define ERROR_RETURN \ +@@ -48,7 +49,6 @@ __newlocale (int category_mask, const char *locale, __locale_t base) + __locale_t result_ptr; + char *locale_path; + size_t locale_path_len; +- const char *locpath_var; + int cnt; + size_t names_len; + +@@ -102,17 +102,8 @@ __newlocale (int category_mask, const char *locale, __locale_t base) + locale_path = NULL; + locale_path_len = 0; + +- locpath_var = getenv ("LOCPATH"); +- if (locpath_var != NULL && locpath_var[0] != '\0') +- { +- if (__argz_create_sep (locpath_var, ':', +- &locale_path, &locale_path_len) != 0) +- return NULL; +- +- if (__argz_add_sep (&locale_path, &locale_path_len, +- _nl_default_locale_path, ':') != 0) +- return NULL; +- } ++ if (compute_locale_search_path (&locale_path, &locale_path_len) != 0) ++ return NULL; + + /* Get the names for the locales we are interested in. We either + allow a composite name or a single name. */ +diff --git a/locale/setlocale.c b/locale/setlocale.c +index ead030d..0c0e314 100644 +--- a/locale/setlocale.c ++++ b/locale/setlocale.c +@@ -215,12 +215,65 @@ setdata (int category, struct __locale_data *data) + } + } + ++/* Return in *LOCALE_PATH and *LOCALE_PATH_LEN the locale data search path as ++ a colon-separated list. Return ENOMEN on error, zero otherwise. */ ++error_t ++compute_locale_search_path (char **locale_path, size_t *locale_path_len) ++{ ++ char* guix_locpath_var = getenv ("GUIX_LOCPATH"); ++ char *locpath_var = getenv ("LOCPATH"); ++ ++ if (guix_locpath_var != NULL && guix_locpath_var[0] != '\0') ++ { ++ /* Entries in 'GUIX_LOCPATH' take precedence over 'LOCPATH'. These ++ entries are systematically prefixed with "/X.Y" where "X.Y" is the ++ libc version. */ ++ if (__argz_create_sep (guix_locpath_var, ':', ++ locale_path, locale_path_len) != 0 ++ || __argz_suffix_entries (locale_path, locale_path_len, ++ "/" VERSION) != 0) ++ goto bail_out; ++ } ++ ++ if (locpath_var != NULL && locpath_var[0] != '\0') ++ { ++ char *reg_locale_path = NULL; ++ size_t reg_locale_path_len = 0; ++ ++ if (__argz_create_sep (locpath_var, ':', ++ ®_locale_path, ®_locale_path_len) != 0) ++ goto bail_out; ++ ++ if (__argz_append (locale_path, locale_path_len, ++ reg_locale_path, reg_locale_path_len) != 0) ++ goto bail_out; ++ ++ free (reg_locale_path); ++ } ++ ++ if (*locale_path != NULL) ++ { ++ /* Append the system default locale directory. */ ++ if (__argz_add_sep (locale_path, locale_path_len, ++ _nl_default_locale_path, ':') != 0) ++ goto bail_out; ++ } ++ ++ return 0; ++ ++ bail_out: ++ free (*locale_path); ++ *locale_path = NULL; ++ *locale_path_len = 0; ++ ++ return ENOMEM; ++} ++ + char * + setlocale (int category, const char *locale) + { + char *locale_path; + size_t locale_path_len; +- const char *locpath_var; + char *composite; + + /* Sanity check for CATEGORY argument. */ +@@ -251,17 +304,10 @@ setlocale (int category, const char *locale) + locale_path = NULL; + locale_path_len = 0; + +- locpath_var = getenv ("LOCPATH"); +- if (locpath_var != NULL && locpath_var[0] != '\0') ++ if (compute_locale_search_path (&locale_path, &locale_path_len) != 0) + { +- if (__argz_create_sep (locpath_var, ':', +- &locale_path, &locale_path_len) != 0 +- || __argz_add_sep (&locale_path, &locale_path_len, +- _nl_default_locale_path, ':') != 0) +- { +- __libc_rwlock_unlock (__libc_setlocale_lock); +- return NULL; +- } ++ __libc_rwlock_unlock (__libc_setlocale_lock); ++ return NULL; + } + + if (category == LC_ALL) +diff --git a/string/Makefile b/string/Makefile +index 8424a61..f925503 100644 +--- a/string/Makefile ++++ b/string/Makefile +@@ -38,7 +38,7 @@ routines := strcat strchr strcmp strcoll strcpy strcspn \ + swab strfry memfrob memmem rawmemchr strchrnul \ + $(addprefix argz-,append count create ctsep next \ + delete extract insert stringify \ +- addsep replace) \ ++ addsep replace suffix) \ + envz basename \ + strcoll_l strxfrm_l string-inlines memrchr \ + xpg-strerror strerror_l +diff --git a/string/argz-suffix.c b/string/argz-suffix.c +new file mode 100644 +index 0000000..505b0f2 +--- /dev/null ++++ b/string/argz-suffix.c +@@ -0,0 +1,56 @@ ++/* Copyright (C) 2015 Free Software Foundation, Inc. ++ This file is part of the GNU C Library. ++ Contributed by Ludovic Courtès <ludo@gnu.org>. ++ ++ The GNU C Library is free software; you can redistribute it and/or ++ modify it under the terms of the GNU Lesser General Public ++ License as published by the Free Software Foundation; either ++ version 2.1 of the License, or (at your option) any later version. ++ ++ The GNU C Library is distributed in the hope that it will be useful, ++ but WITHOUT ANY WARRANTY; without even the implied warranty of ++ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ++ Lesser General Public License for more details. ++ ++ You should have received a copy of the GNU Lesser General Public ++ License along with the GNU C Library; if not, see ++ <http://www.gnu.org/licenses/>. */ ++ ++#include <argz.h> ++#include <errno.h> ++#include <stdlib.h> ++#include <string.h> ++ ++ ++error_t ++__argz_suffix_entries (char **argz, size_t *argz_len, const char *suffix) ++ ++{ ++ size_t suffix_len = strlen (suffix); ++ size_t count = __argz_count (*argz, *argz_len); ++ size_t new_argz_len = *argz_len + count * suffix_len; ++ char *new_argz = malloc (new_argz_len); ++ ++ if (new_argz) ++ { ++ char *p = new_argz, *entry; ++ ++ for (entry = *argz; ++ entry != NULL; ++ entry = argz_next (*argz, *argz_len, entry)) ++ { ++ p = stpcpy (p, entry); ++ p = stpcpy (p, suffix); ++ p++; ++ } ++ ++ free (*argz); ++ *argz = new_argz; ++ *argz_len = new_argz_len; ++ ++ return 0; ++ } ++ else ++ return ENOMEM; ++} ++weak_alias (__argz_suffix_entries, argz_suffix_entries) +diff --git a/string/argz.h b/string/argz.h +index bb62a31..d276a35 100644 +--- a/string/argz.h ++++ b/string/argz.h +@@ -134,6 +134,16 @@ extern error_t argz_replace (char **__restrict __argz, + const char *__restrict __str, + const char *__restrict __with, + unsigned int *__restrict __replace_count); ++ ++/* Suffix each entry of ARGZ & ARGZ_LEN with SUFFIX. Return 0 on success, ++ and ENOMEN if memory cannot be allocated. */ ++extern error_t __argz_suffix_entries (char **__restrict __argz, ++ size_t *__restrict __argz_len, ++ const char *__restrict __suffix); ++extern error_t argz_suffix_entries (char **__restrict __argz, ++ size_t *__restrict __argz_len, ++ const char *__restrict __suffix); ++ + + /* Returns the next entry in ARGZ & ARGZ_LEN after ENTRY, or NULL if there + are no more. If entry is NULL, then the first entry is returned. This diff --git a/contrib/guix/patches/lief-fix-ppc64-nx-default.patch b/contrib/guix/patches/lief-fix-ppc64-nx-default.patch new file mode 100644 index 0000000000..101bc1ddc0 --- /dev/null +++ b/contrib/guix/patches/lief-fix-ppc64-nx-default.patch @@ -0,0 +1,29 @@ +Correct default for Binary::has_nx on ppc64 + +From the Linux kernel source: + + * This is the default if a program doesn't have a PT_GNU_STACK + * program header entry. The PPC64 ELF ABI has a non executable stack + * stack by default, so in the absence of a PT_GNU_STACK program header + * we turn execute permission off. + +This patch can be dropped the next time we update LIEF. + +diff --git a/src/ELF/Binary.cpp b/src/ELF/Binary.cpp +index a90be1ab..fd2d9764 100644 +--- a/src/ELF/Binary.cpp ++++ b/src/ELF/Binary.cpp +@@ -1084,7 +1084,12 @@ bool Binary::has_nx() const { + return segment->type() == SEGMENT_TYPES::PT_GNU_STACK; + }); + if (it_stack == std::end(segments_)) { +- return false; ++ if (header().machine_type() == ARCH::EM_PPC64) { ++ // The PPC64 ELF ABI has a non-executable stack by default. ++ return true; ++ } else { ++ return false; ++ } + } + + return !(*it_stack)->has(ELF_SEGMENT_FLAGS::PF_X); diff --git a/contrib/guix/patches/nsis-gcc-10-memmove.patch b/contrib/guix/patches/nsis-gcc-10-memmove.patch new file mode 100644 index 0000000000..a1aadfd4f3 --- /dev/null +++ b/contrib/guix/patches/nsis-gcc-10-memmove.patch @@ -0,0 +1,23 @@ +commit f6df41524e703dc471e283e566a48e05a735b7f2 +Author: Anders <anders_k@users.sourceforge.net> +Date: Sat Jun 27 23:18:45 2020 +0000 + + Don't let GCC 10 generate memmove calls (bug #1248) + + git-svn-id: https://svn.code.sf.net/p/nsis/code/NSIS/trunk@7189 212acab6-be3b-0410-9dea-997c60f758d6 + +diff --git a/SCons/Config/gnu b/SCons/Config/gnu +index bfcb362d..21fa446b 100644 +--- a/SCons/Config/gnu ++++ b/SCons/Config/gnu +@@ -103,6 +103,10 @@ stub_env.Append(LINKFLAGS = ['$NODEFLIBS_FLAG']) # no standard libraries + stub_env.Append(LINKFLAGS = ['$ALIGN_FLAG']) # 512 bytes align + stub_env.Append(LINKFLAGS = ['$MAP_FLAG']) # generate map file + ++conf = FlagsConfigure(stub_env) ++conf.CheckCompileFlag('-fno-tree-loop-distribute-patterns') # GCC 10: Don't generate msvcrt!memmove calls (bug #1248) ++conf.Finish() ++ + stub_uenv = stub_env.Clone() + stub_uenv.Append(CPPDEFINES = ['_UNICODE', 'UNICODE']) + diff --git a/contrib/guix/patches/oscrypto-hard-code-openssl.patch b/contrib/guix/patches/oscrypto-hard-code-openssl.patch new file mode 100644 index 0000000000..32027f2d09 --- /dev/null +++ b/contrib/guix/patches/oscrypto-hard-code-openssl.patch @@ -0,0 +1,13 @@ +diff --git a/oscrypto/__init__.py b/oscrypto/__init__.py +index eb27313..371ab24 100644 +--- a/oscrypto/__init__.py ++++ b/oscrypto/__init__.py +@@ -302,3 +302,8 @@ def load_order(): + 'oscrypto._win.tls', + 'oscrypto.tls', + ] ++ ++ ++paths = '@GUIX_OSCRYPTO_USE_OPENSSL@'.split(',') ++assert len(paths) == 2, 'Value for OSCRYPTO_USE_OPENSSL env var must be two paths separated by a comma' ++use_openssl(*paths) diff --git a/contrib/guix/patches/vmov-alignment.patch b/contrib/guix/patches/vmov-alignment.patch new file mode 100644 index 0000000000..072f76eafd --- /dev/null +++ b/contrib/guix/patches/vmov-alignment.patch @@ -0,0 +1,267 @@ +Description: Use unaligned VMOV instructions +Author: Stephen Kitt <skitt@debian.org> +Bug-Debian: https://bugs.debian.org/939559 + +Based on a patch originally by Claude Heiland-Allen <claude@mathr.co.uk> + +--- a/gcc/config/i386/sse.md ++++ b/gcc/config/i386/sse.md +@@ -1058,17 +1058,11 @@ + { + if (FLOAT_MODE_P (GET_MODE_INNER (<MODE>mode))) + { +- if (misaligned_operand (operands[1], <MODE>mode)) +- return "vmovu<ssemodesuffix>\t{%1, %0%{%3%}%N2|%0%{%3%}%N2, %1}"; +- else +- return "vmova<ssemodesuffix>\t{%1, %0%{%3%}%N2|%0%{%3%}%N2, %1}"; ++ return "vmovu<ssemodesuffix>\t{%1, %0%{%3%}%N2|%0%{%3%}%N2, %1}"; + } + else + { +- if (misaligned_operand (operands[1], <MODE>mode)) +- return "vmovdqu<ssescalarsize>\t{%1, %0%{%3%}%N2|%0%{%3%}%N2, %1}"; +- else +- return "vmovdqa<ssescalarsize>\t{%1, %0%{%3%}%N2|%0%{%3%}%N2, %1}"; ++ return "vmovdqu<ssescalarsize>\t{%1, %0%{%3%}%N2|%0%{%3%}%N2, %1}"; + } + } + [(set_attr "type" "ssemov") +@@ -1184,17 +1178,11 @@ + { + if (FLOAT_MODE_P (GET_MODE_INNER (<MODE>mode))) + { +- if (misaligned_operand (operands[0], <MODE>mode)) +- return "vmovu<ssemodesuffix>\t{%1, %0%{%2%}|%0%{%2%}, %1}"; +- else +- return "vmova<ssemodesuffix>\t{%1, %0%{%2%}|%0%{%2%}, %1}"; ++ return "vmovu<ssemodesuffix>\t{%1, %0%{%2%}|%0%{%2%}, %1}"; + } + else + { +- if (misaligned_operand (operands[0], <MODE>mode)) +- return "vmovdqu<ssescalarsize>\t{%1, %0%{%2%}|%0%{%2%}, %1}"; +- else +- return "vmovdqa<ssescalarsize>\t{%1, %0%{%2%}|%0%{%2%}, %1}"; ++ return "vmovdqu<ssescalarsize>\t{%1, %0%{%2%}|%0%{%2%}, %1}"; + } + } + [(set_attr "type" "ssemov") +@@ -7806,7 +7794,7 @@ + "TARGET_SSE && !(MEM_P (operands[0]) && MEM_P (operands[1]))" + "@ + %vmovlps\t{%1, %0|%q0, %1} +- %vmovaps\t{%1, %0|%0, %1} ++ %vmovups\t{%1, %0|%0, %1} + %vmovlps\t{%1, %d0|%d0, %q1}" + [(set_attr "type" "ssemov") + (set_attr "prefix" "maybe_vex") +@@ -13997,29 +13985,15 @@ + switch (<MODE>mode) + { + case E_V8DFmode: +- if (misaligned_operand (operands[2], <ssequartermode>mode)) +- return "vmovupd\t{%2, %x0|%x0, %2}"; +- else +- return "vmovapd\t{%2, %x0|%x0, %2}"; ++ return "vmovupd\t{%2, %x0|%x0, %2}"; + case E_V16SFmode: +- if (misaligned_operand (operands[2], <ssequartermode>mode)) +- return "vmovups\t{%2, %x0|%x0, %2}"; +- else +- return "vmovaps\t{%2, %x0|%x0, %2}"; ++ return "vmovups\t{%2, %x0|%x0, %2}"; + case E_V8DImode: +- if (misaligned_operand (operands[2], <ssequartermode>mode)) +- return which_alternative == 2 ? "vmovdqu64\t{%2, %x0|%x0, %2}" ++ return which_alternative == 2 ? "vmovdqu64\t{%2, %x0|%x0, %2}" + : "vmovdqu\t{%2, %x0|%x0, %2}"; +- else +- return which_alternative == 2 ? "vmovdqa64\t{%2, %x0|%x0, %2}" +- : "vmovdqa\t{%2, %x0|%x0, %2}"; + case E_V16SImode: +- if (misaligned_operand (operands[2], <ssequartermode>mode)) +- return which_alternative == 2 ? "vmovdqu32\t{%2, %x0|%x0, %2}" ++ return which_alternative == 2 ? "vmovdqu32\t{%2, %x0|%x0, %2}" + : "vmovdqu\t{%2, %x0|%x0, %2}"; +- else +- return which_alternative == 2 ? "vmovdqa32\t{%2, %x0|%x0, %2}" +- : "vmovdqa\t{%2, %x0|%x0, %2}"; + default: + gcc_unreachable (); + } +@@ -21225,63 +21199,27 @@ + switch (get_attr_mode (insn)) + { + case MODE_V16SF: +- if (misaligned_operand (operands[1], <ssehalfvecmode>mode)) +- return "vmovups\t{%1, %t0|%t0, %1}"; +- else +- return "vmovaps\t{%1, %t0|%t0, %1}"; ++ return "vmovups\t{%1, %t0|%t0, %1}"; + case MODE_V8DF: +- if (misaligned_operand (operands[1], <ssehalfvecmode>mode)) +- return "vmovupd\t{%1, %t0|%t0, %1}"; +- else +- return "vmovapd\t{%1, %t0|%t0, %1}"; ++ return "vmovupd\t{%1, %t0|%t0, %1}"; + case MODE_V8SF: +- if (misaligned_operand (operands[1], <ssehalfvecmode>mode)) +- return "vmovups\t{%1, %x0|%x0, %1}"; +- else +- return "vmovaps\t{%1, %x0|%x0, %1}"; ++ return "vmovups\t{%1, %x0|%x0, %1}"; + case MODE_V4DF: +- if (misaligned_operand (operands[1], <ssehalfvecmode>mode)) +- return "vmovupd\t{%1, %x0|%x0, %1}"; +- else +- return "vmovapd\t{%1, %x0|%x0, %1}"; ++ return "vmovupd\t{%1, %x0|%x0, %1}"; + case MODE_XI: +- if (misaligned_operand (operands[1], <ssehalfvecmode>mode)) +- { +- if (which_alternative == 2) +- return "vmovdqu\t{%1, %t0|%t0, %1}"; +- else if (GET_MODE_SIZE (<ssescalarmode>mode) == 8) +- return "vmovdqu64\t{%1, %t0|%t0, %1}"; +- else +- return "vmovdqu32\t{%1, %t0|%t0, %1}"; +- } ++ if (which_alternative == 2) ++ return "vmovdqu\t{%1, %t0|%t0, %1}"; ++ else if (GET_MODE_SIZE (<ssescalarmode>mode) == 8) ++ return "vmovdqu64\t{%1, %t0|%t0, %1}"; + else +- { +- if (which_alternative == 2) +- return "vmovdqa\t{%1, %t0|%t0, %1}"; +- else if (GET_MODE_SIZE (<ssescalarmode>mode) == 8) +- return "vmovdqa64\t{%1, %t0|%t0, %1}"; +- else +- return "vmovdqa32\t{%1, %t0|%t0, %1}"; +- } ++ return "vmovdqu32\t{%1, %t0|%t0, %1}"; + case MODE_OI: +- if (misaligned_operand (operands[1], <ssehalfvecmode>mode)) +- { +- if (which_alternative == 2) +- return "vmovdqu\t{%1, %x0|%x0, %1}"; +- else if (GET_MODE_SIZE (<ssescalarmode>mode) == 8) +- return "vmovdqu64\t{%1, %x0|%x0, %1}"; +- else +- return "vmovdqu32\t{%1, %x0|%x0, %1}"; +- } ++ if (which_alternative == 2) ++ return "vmovdqu\t{%1, %x0|%x0, %1}"; ++ else if (GET_MODE_SIZE (<ssescalarmode>mode) == 8) ++ return "vmovdqu64\t{%1, %x0|%x0, %1}"; + else +- { +- if (which_alternative == 2) +- return "vmovdqa\t{%1, %x0|%x0, %1}"; +- else if (GET_MODE_SIZE (<ssescalarmode>mode) == 8) +- return "vmovdqa64\t{%1, %x0|%x0, %1}"; +- else +- return "vmovdqa32\t{%1, %x0|%x0, %1}"; +- } ++ return "vmovdqu32\t{%1, %x0|%x0, %1}"; + default: + gcc_unreachable (); + } +--- a/gcc/config/i386/i386.c ++++ b/gcc/config/i386/i386.c +@@ -4981,13 +4981,13 @@ + switch (type) + { + case opcode_int: +- opcode = misaligned_p ? "vmovdqu32" : "vmovdqa32"; ++ opcode = "vmovdqu32"; + break; + case opcode_float: +- opcode = misaligned_p ? "vmovups" : "vmovaps"; ++ opcode = "vmovups"; + break; + case opcode_double: +- opcode = misaligned_p ? "vmovupd" : "vmovapd"; ++ opcode = "vmovupd"; + break; + } + } +@@ -4996,16 +4996,16 @@ + switch (scalar_mode) + { + case E_SFmode: +- opcode = misaligned_p ? "%vmovups" : "%vmovaps"; ++ opcode = "%vmovups"; + break; + case E_DFmode: +- opcode = misaligned_p ? "%vmovupd" : "%vmovapd"; ++ opcode = "%vmovupd"; + break; + case E_TFmode: + if (evex_reg_p) +- opcode = misaligned_p ? "vmovdqu64" : "vmovdqa64"; ++ opcode = "vmovdqu64"; + else +- opcode = misaligned_p ? "%vmovdqu" : "%vmovdqa"; ++ opcode = "%vmovdqu"; + break; + default: + gcc_unreachable (); +@@ -5017,48 +5017,32 @@ + { + case E_QImode: + if (evex_reg_p) +- opcode = (misaligned_p +- ? (TARGET_AVX512BW +- ? "vmovdqu8" +- : "vmovdqu64") +- : "vmovdqa64"); ++ opcode = TARGET_AVX512BW ? "vmovdqu8" : "vmovdqu64"; + else +- opcode = (misaligned_p +- ? (TARGET_AVX512BW +- ? "vmovdqu8" +- : "%vmovdqu") +- : "%vmovdqa"); ++ opcode = TARGET_AVX512BW ? "vmovdqu8" : "%vmovdqu"; + break; + case E_HImode: + if (evex_reg_p) +- opcode = (misaligned_p +- ? (TARGET_AVX512BW +- ? "vmovdqu16" +- : "vmovdqu64") +- : "vmovdqa64"); ++ opcode = TARGET_AVX512BW ? "vmovdqu16" : "vmovdqu64"; + else +- opcode = (misaligned_p +- ? (TARGET_AVX512BW +- ? "vmovdqu16" +- : "%vmovdqu") +- : "%vmovdqa"); ++ opcode = TARGET_AVX512BW ? "vmovdqu16" : "%vmovdqu"; + break; + case E_SImode: + if (evex_reg_p) +- opcode = misaligned_p ? "vmovdqu32" : "vmovdqa32"; ++ opcode = "vmovdqu32"; + else +- opcode = misaligned_p ? "%vmovdqu" : "%vmovdqa"; ++ opcode = "%vmovdqu"; + break; + case E_DImode: + case E_TImode: + case E_OImode: + if (evex_reg_p) +- opcode = misaligned_p ? "vmovdqu64" : "vmovdqa64"; ++ opcode = "vmovdqu64"; + else +- opcode = misaligned_p ? "%vmovdqu" : "%vmovdqa"; ++ opcode = "%vmovdqu"; + break; + case E_XImode: +- opcode = misaligned_p ? "vmovdqu64" : "vmovdqa64"; ++ opcode = "vmovdqu64"; + break; + default: + gcc_unreachable (); diff --git a/contrib/init/README.md b/contrib/init/README.md new file mode 100644 index 0000000000..affc7c2e75 --- /dev/null +++ b/contrib/init/README.md @@ -0,0 +1,12 @@ +Sample configuration files for: +``` +systemd: bitcoind.service +Upstart: bitcoind.conf +OpenRC: bitcoind.openrc + bitcoind.openrcconf +CentOS: bitcoind.init +macOS: org.bitcoin.bitcoind.plist +``` +have been made available to assist packagers in creating node packages here. + +See [doc/init.md](../../doc/init.md) for more information. diff --git a/contrib/init/bitcoind.conf b/contrib/init/bitcoind.conf new file mode 100644 index 0000000000..dde1bd0c4d --- /dev/null +++ b/contrib/init/bitcoind.conf @@ -0,0 +1,65 @@ +description "Bitcoin Core Daemon" + +start on runlevel [2345] +stop on starting rc RUNLEVEL=[016] + +env BITCOIND_BIN="/usr/bin/bitcoind" +env BITCOIND_USER="bitcoin" +env BITCOIND_GROUP="bitcoin" +env BITCOIND_PIDDIR="/var/run/bitcoind" +# upstart can't handle variables constructed with other variables +env BITCOIND_PIDFILE="/var/run/bitcoind/bitcoind.pid" +env BITCOIND_CONFIGFILE="/etc/bitcoin/bitcoin.conf" +env BITCOIND_DATADIR="/var/lib/bitcoind" + +expect fork + +respawn +respawn limit 5 120 +kill timeout 600 + +pre-start script + # this will catch non-existent config files + # bitcoind will check and exit with this very warning, but it can do so + # long after forking, leaving upstart to think everything started fine. + # since this is a commonly encountered case on install, just check and + # warn here. + if ! grep -qs '^rpcpassword=' "$BITCOIND_CONFIGFILE" ; then + echo "ERROR: You must set a secure rpcpassword to run bitcoind." + echo "The setting must appear in $BITCOIND_CONFIGFILE" + echo + echo "This password is security critical to securing wallets " + echo "and must not be the same as the rpcuser setting." + echo "You can generate a suitable random password using the following " + echo "command from the shell:" + echo + echo "bash -c 'tr -dc a-zA-Z0-9 < /dev/urandom | head -c32 && echo'" + echo + echo "It is recommended that you also set alertnotify so you are " + echo "notified of problems:" + echo + echo "ie: alertnotify=echo %%s | mail -s \"Bitcoin Alert\"" \ + "admin@foo.com" + echo + exit 1 + fi + + mkdir -p "$BITCOIND_PIDDIR" + chmod 0755 "$BITCOIND_PIDDIR" + chown $BITCOIND_USER:$BITCOIND_GROUP "$BITCOIND_PIDDIR" + chown $BITCOIND_USER:$BITCOIND_GROUP "$BITCOIND_CONFIGFILE" + chmod 0660 "$BITCOIND_CONFIGFILE" +end script + +exec start-stop-daemon \ + --start \ + --pidfile "$BITCOIND_PIDFILE" \ + --chuid $BITCOIND_USER:$BITCOIND_GROUP \ + --exec "$BITCOIND_BIN" \ + -- \ + -pid="$BITCOIND_PIDFILE" \ + -conf="$BITCOIND_CONFIGFILE" \ + -datadir="$BITCOIND_DATADIR" \ + -disablewallet \ + -daemon + diff --git a/contrib/init/bitcoind.init b/contrib/init/bitcoind.init new file mode 100644 index 0000000000..19e1f76d09 --- /dev/null +++ b/contrib/init/bitcoind.init @@ -0,0 +1,67 @@ +#!/usr/bin/env bash +# +# bitcoind The bitcoin core server. +# +# +# chkconfig: 345 80 20 +# description: bitcoind +# processname: bitcoind +# + +# Source function library. +. /etc/init.d/functions + +# you can override defaults in /etc/sysconfig/bitcoind, see below +if [ -f /etc/sysconfig/bitcoind ]; then + . /etc/sysconfig/bitcoind +fi + +RETVAL=0 + +prog=bitcoind +# you can override the lockfile via BITCOIND_LOCKFILE in /etc/sysconfig/bitcoind +lockfile=${BITCOIND_LOCKFILE-/var/lock/subsys/bitcoind} + +# bitcoind defaults to /usr/bin/bitcoind, override with BITCOIND_BIN +bitcoind=${BITCOIND_BIN-/usr/bin/bitcoind} + +# bitcoind opts default to -disablewallet, override with BITCOIND_OPTS +bitcoind_opts=${BITCOIND_OPTS--disablewallet} + +start() { + echo -n $"Starting $prog: " + daemon $DAEMONOPTS $bitcoind $bitcoind_opts + RETVAL=$? + echo + [ $RETVAL -eq 0 ] && touch $lockfile + return $RETVAL +} + +stop() { + echo -n $"Stopping $prog: " + killproc $prog -t600 + RETVAL=$? + echo + [ $RETVAL -eq 0 ] && rm -f $lockfile + return $RETVAL +} + +case "$1" in + start) + start + ;; + stop) + stop + ;; + status) + status $prog + ;; + restart) + stop + start + ;; + *) + echo "Usage: service $prog {start|stop|status|restart}" + exit 1 + ;; +esac diff --git a/contrib/init/bitcoind.openrc b/contrib/init/bitcoind.openrc new file mode 100644 index 0000000000..013a1a6070 --- /dev/null +++ b/contrib/init/bitcoind.openrc @@ -0,0 +1,93 @@ +#!/sbin/openrc-run + +# backward compatibility for existing gentoo layout +# +if [ -d "/var/lib/bitcoin/.bitcoin" ]; then + BITCOIND_DEFAULT_DATADIR="/var/lib/bitcoin/.bitcoin" +else + BITCOIND_DEFAULT_DATADIR="/var/lib/bitcoind" +fi + +BITCOIND_CONFIGFILE=${BITCOIND_CONFIGFILE:-/etc/bitcoin/bitcoin.conf} +BITCOIND_PIDDIR=${BITCOIND_PIDDIR:-/var/run/bitcoind} +BITCOIND_PIDFILE=${BITCOIND_PIDFILE:-${BITCOIND_PIDDIR}/bitcoind.pid} +BITCOIND_DATADIR=${BITCOIND_DATADIR:-${BITCOIND_DEFAULT_DATADIR}} +BITCOIND_USER=${BITCOIND_USER:-${BITCOIN_USER:-bitcoin}} +BITCOIND_GROUP=${BITCOIND_GROUP:-bitcoin} +BITCOIND_BIN=${BITCOIND_BIN:-/usr/bin/bitcoind} +BITCOIND_NICE=${BITCOIND_NICE:-${NICELEVEL:-0}} +BITCOIND_OPTS="${BITCOIND_OPTS:-${BITCOIN_OPTS}}" + +name="Bitcoin Core Daemon" +description="Bitcoin cryptocurrency P2P network daemon" + +command="/usr/bin/bitcoind" +command_args="-pid=\"${BITCOIND_PIDFILE}\" \ + -conf=\"${BITCOIND_CONFIGFILE}\" \ + -datadir=\"${BITCOIND_DATADIR}\" \ + -daemon \ + ${BITCOIND_OPTS}" + +required_files="${BITCOIND_CONFIGFILE}" +start_stop_daemon_args="-u ${BITCOIND_USER} \ + -N ${BITCOIND_NICE} -w 2000" +pidfile="${BITCOIND_PIDFILE}" + +# The retry schedule to use when stopping the daemon. Could be either +# a timeout in seconds or multiple signal/timeout pairs (like +# "SIGKILL/180 SIGTERM/300") +retry="${BITCOIND_SIGTERM_TIMEOUT}" + +depend() { + need localmount net +} + +# verify +# 1) that the datadir exists and is writable (or create it) +# 2) that a directory for the pid exists and is writable +# 3) ownership and permissions on the config file +start_pre() { + checkpath \ + -d \ + --mode 0750 \ + --owner "${BITCOIND_USER}:${BITCOIND_GROUP}" \ + "${BITCOIND_DATADIR}" + + checkpath \ + -d \ + --mode 0755 \ + --owner "${BITCOIND_USER}:${BITCOIND_GROUP}" \ + "${BITCOIND_PIDDIR}" + + checkpath -f \ + -o "${BITCOIND_USER}:${BITCOIND_GROUP}" \ + -m 0660 \ + "${BITCOIND_CONFIGFILE}" + + checkconfig || return 1 +} + +checkconfig() +{ + if grep -qs '^rpcuser=' "${BITCOIND_CONFIGFILE}" && \ + ! grep -qs '^rpcpassword=' "${BITCOIND_CONFIGFILE}" ; then + eerror "" + eerror "ERROR: You must set a secure rpcpassword to run bitcoind." + eerror "The setting must appear in ${BITCOIND_CONFIGFILE}" + eerror "" + eerror "This password is security critical to securing wallets " + eerror "and must not be the same as the rpcuser setting." + eerror "You can generate a suitable random password using the following " + eerror "command from the shell:" + eerror "" + eerror "bash -c 'tr -dc a-zA-Z0-9 < /dev/urandom | head -c32 && echo'" + eerror "" + eerror "It is recommended that you also set alertnotify so you are " + eerror "notified of problems:" + eerror "" + eerror "ie: alertnotify=echo %%s | mail -s \"Bitcoin Alert\"" \ + "admin@foo.com" + eerror "" + return 1 + fi +} diff --git a/contrib/init/bitcoind.openrcconf b/contrib/init/bitcoind.openrcconf new file mode 100644 index 0000000000..c8a22a08d9 --- /dev/null +++ b/contrib/init/bitcoind.openrcconf @@ -0,0 +1,33 @@ +# /etc/conf.d/bitcoind: config file for /etc/init.d/bitcoind + +# Config file location +#BITCOIND_CONFIGFILE="/etc/bitcoin/bitcoin.conf" + +# What directory to write pidfile to? (created and owned by $BITCOIND_USER) +#BITCOIND_PIDDIR="/var/run/bitcoind" + +# What filename to give the pidfile +#BITCOIND_PIDFILE="${BITCOIND_PIDDIR}/bitcoind.pid" + +# Where to write bitcoind data (be mindful that the blockchain is large) +#BITCOIND_DATADIR="/var/lib/bitcoind" + +# User and group to own bitcoind process +#BITCOIND_USER="bitcoin" +#BITCOIND_GROUP="bitcoin" + +# Path to bitcoind executable +#BITCOIND_BIN="/usr/bin/bitcoind" + +# Nice value to run bitcoind under +#BITCOIND_NICE=0 + +# Additional options (avoid -conf and -datadir, use flags above) +#BITCOIND_OPTS="" + +# The timeout in seconds OpenRC will wait for bitcoind to terminate +# after a SIGTERM has been raised. +# Note that this will be mapped as argument to start-stop-daemon's +# '--retry' option, which means you can specify a retry schedule +# here. For more information see man 8 start-stop-daemon. +BITCOIND_SIGTERM_TIMEOUT=600 diff --git a/contrib/init/bitcoind.service b/contrib/init/bitcoind.service new file mode 100644 index 0000000000..93de353bb4 --- /dev/null +++ b/contrib/init/bitcoind.service @@ -0,0 +1,82 @@ +# It is not recommended to modify this file in-place, because it will +# be overwritten during package upgrades. If you want to add further +# options or overwrite existing ones then use +# $ systemctl edit bitcoind.service +# See "man systemd.service" for details. + +# Note that almost all daemon options could be specified in +# /etc/bitcoin/bitcoin.conf, but keep in mind those explicitly +# specified as arguments in ExecStart= will override those in the +# config file. + +[Unit] +Description=Bitcoin daemon +Documentation=https://github.com/bitcoin/bitcoin/blob/master/doc/init.md + +# https://www.freedesktop.org/wiki/Software/systemd/NetworkTarget/ +After=network-online.target +Wants=network-online.target + +[Service] +ExecStart=/usr/bin/bitcoind -daemonwait \ + -pid=/run/bitcoind/bitcoind.pid \ + -conf=/etc/bitcoin/bitcoin.conf \ + -datadir=/var/lib/bitcoind + +# Make sure the config directory is readable by the service user +PermissionsStartOnly=true +ExecStartPre=/bin/chgrp bitcoin /etc/bitcoin + +# Process management +#################### + +Type=forking +PIDFile=/run/bitcoind/bitcoind.pid +Restart=on-failure +TimeoutStartSec=infinity +TimeoutStopSec=600 + +# Directory creation and permissions +#################################### + +# Run as bitcoin:bitcoin +User=bitcoin +Group=bitcoin + +# /run/bitcoind +RuntimeDirectory=bitcoind +RuntimeDirectoryMode=0710 + +# /etc/bitcoin +ConfigurationDirectory=bitcoin +ConfigurationDirectoryMode=0710 + +# /var/lib/bitcoind +StateDirectory=bitcoind +StateDirectoryMode=0710 + +# Hardening measures +#################### + +# Provide a private /tmp and /var/tmp. +PrivateTmp=true + +# Mount /usr, /boot/ and /etc read-only for the process. +ProtectSystem=full + +# Deny access to /home, /root and /run/user +ProtectHome=true + +# Disallow the process and all of its children to gain +# new privileges through execve(). +NoNewPrivileges=true + +# Use a new /dev namespace only populated with API pseudo devices +# such as /dev/null, /dev/zero and /dev/random. +PrivateDevices=true + +# Deny the creation of writable and executable memory mappings. +MemoryDenyWriteExecute=true + +[Install] +WantedBy=multi-user.target diff --git a/contrib/init/org.bitcoin.bitcoind.plist b/contrib/init/org.bitcoin.bitcoind.plist new file mode 100644 index 0000000000..95b5342f1e --- /dev/null +++ b/contrib/init/org.bitcoin.bitcoind.plist @@ -0,0 +1,14 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd"> +<plist version="1.0"> +<dict> + <key>Label</key> + <string>org.bitcoin.bitcoind</string> + <key>ProgramArguments</key> + <array> + <string>/usr/local/bin/bitcoind</string> + </array> + <key>RunAtLoad</key> + <true/> +</dict> +</plist> diff --git a/contrib/install_db4.sh b/contrib/install_db4.sh new file mode 100755 index 0000000000..2850c4b993 --- /dev/null +++ b/contrib/install_db4.sh @@ -0,0 +1,260 @@ +#!/bin/sh +# Copyright (c) 2017-2021 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +# Install libdb4.8 (Berkeley DB). + +export LC_ALL=C +set -e + +if [ -z "${1}" ]; then + echo "Usage: $0 <base-dir> [<extra-bdb-configure-flag> ...]" + echo + echo "Must specify a single argument: the directory in which db4 will be built." + echo "This is probably \`pwd\` if you're at the root of the bitcoin repository." + exit 1 +fi + +expand_path() { + cd "${1}" && pwd -P +} + +BDB_PREFIX="$(expand_path "${1}")/db4"; shift; +BDB_VERSION='db-4.8.30.NC' +BDB_HASH='12edc0df75bf9abd7f82f821795bcee50f42cb2e5f76a6a281b85732798364ef' +BDB_URL="https://download.oracle.com/berkeley-db/${BDB_VERSION}.tar.gz" + +check_exists() { + command -v "$1" >/dev/null +} + +sha256_check() { + # Args: <sha256_hash> <filename> + # + if check_exists sha256sum; then + echo "${1} ${2}" | sha256sum -c + elif check_exists sha256; then + if [ "$(uname)" = "FreeBSD" ]; then + sha256 -c "${1}" "${2}" + else + echo "${1} ${2}" | sha256 -c + fi + else + echo "${1} ${2}" | shasum -a 256 -c + fi +} + +http_get() { + # Args: <url> <filename> <sha256_hash> + # + # It's acceptable that we don't require SSL here because we manually verify + # content hashes below. + # + if [ -f "${2}" ]; then + echo "File ${2} already exists; not downloading again" + elif check_exists curl; then + curl --insecure --retry 5 "${1}" -o "${2}" + elif check_exists wget; then + wget --no-check-certificate "${1}" -O "${2}" + else + echo "Simple transfer utilities 'curl' and 'wget' not found. Please install one of them and try again." + exit 1 + fi + + sha256_check "${3}" "${2}" +} + +# Ensure the commands we use exist on the system +if ! check_exists patch; then + echo "Command-line tool 'patch' not found. Install patch and try again." + exit 1 +fi + +mkdir -p "${BDB_PREFIX}" +http_get "${BDB_URL}" "${BDB_VERSION}.tar.gz" "${BDB_HASH}" +tar -xzvf ${BDB_VERSION}.tar.gz -C "$BDB_PREFIX" +cd "${BDB_PREFIX}/${BDB_VERSION}/" + +# Apply a patch necessary when building with clang and c++11 (see https://community.oracle.com/thread/3952592) +patch --ignore-whitespace -p1 << 'EOF' +commit 3311d68f11d1697565401eee6efc85c34f022ea7 +Author: fanquake <fanquake@gmail.com> +Date: Mon Aug 17 20:03:56 2020 +0800 + + Fix C++11 compatibility + +diff --git a/dbinc/atomic.h b/dbinc/atomic.h +index 0034dcc..7c11d4a 100644 +--- a/dbinc/atomic.h ++++ b/dbinc/atomic.h +@@ -70,7 +70,7 @@ typedef struct { + * These have no memory barriers; the caller must include them when necessary. + */ + #define atomic_read(p) ((p)->value) +-#define atomic_init(p, val) ((p)->value = (val)) ++#define atomic_init_db(p, val) ((p)->value = (val)) + + #ifdef HAVE_ATOMIC_SUPPORT + +@@ -144,7 +144,7 @@ typedef LONG volatile *interlocked_val; + #define atomic_inc(env, p) __atomic_inc(p) + #define atomic_dec(env, p) __atomic_dec(p) + #define atomic_compare_exchange(env, p, o, n) \ +- __atomic_compare_exchange((p), (o), (n)) ++ __atomic_compare_exchange_db((p), (o), (n)) + static inline int __atomic_inc(db_atomic_t *p) + { + int temp; +@@ -176,7 +176,7 @@ static inline int __atomic_dec(db_atomic_t *p) + * http://gcc.gnu.org/onlinedocs/gcc-4.1.0/gcc/Atomic-Builtins.html + * which configure could be changed to use. + */ +-static inline int __atomic_compare_exchange( ++static inline int __atomic_compare_exchange_db( + db_atomic_t *p, atomic_value_t oldval, atomic_value_t newval) + { + atomic_value_t was; +@@ -206,7 +206,7 @@ static inline int __atomic_compare_exchange( + #define atomic_dec(env, p) (--(p)->value) + #define atomic_compare_exchange(env, p, oldval, newval) \ + (DB_ASSERT(env, atomic_read(p) == (oldval)), \ +- atomic_init(p, (newval)), 1) ++ atomic_init_db(p, (newval)), 1) + #else + #define atomic_inc(env, p) __atomic_inc(env, p) + #define atomic_dec(env, p) __atomic_dec(env, p) +diff --git a/mp/mp_fget.c b/mp/mp_fget.c +index 5fdee5a..0b75f57 100644 +--- a/mp/mp_fget.c ++++ b/mp/mp_fget.c +@@ -617,7 +617,7 @@ alloc: /* Allocate a new buffer header and data space. */ + + /* Initialize enough so we can call __memp_bhfree. */ + alloc_bhp->flags = 0; +- atomic_init(&alloc_bhp->ref, 1); ++ atomic_init_db(&alloc_bhp->ref, 1); + #ifdef DIAGNOSTIC + if ((uintptr_t)alloc_bhp->buf & (sizeof(size_t) - 1)) { + __db_errx(env, +@@ -911,7 +911,7 @@ alloc: /* Allocate a new buffer header and data space. */ + MVCC_MPROTECT(bhp->buf, mfp->stat.st_pagesize, + PROT_READ); + +- atomic_init(&alloc_bhp->ref, 1); ++ atomic_init_db(&alloc_bhp->ref, 1); + MUTEX_LOCK(env, alloc_bhp->mtx_buf); + alloc_bhp->priority = bhp->priority; + alloc_bhp->pgno = bhp->pgno; +diff --git a/mp/mp_mvcc.c b/mp/mp_mvcc.c +index 34467d2..f05aa0c 100644 +--- a/mp/mp_mvcc.c ++++ b/mp/mp_mvcc.c +@@ -276,7 +276,7 @@ __memp_bh_freeze(dbmp, infop, hp, bhp, need_frozenp) + #else + memcpy(frozen_bhp, bhp, SSZA(BH, buf)); + #endif +- atomic_init(&frozen_bhp->ref, 0); ++ atomic_init_db(&frozen_bhp->ref, 0); + if (mutex != MUTEX_INVALID) + frozen_bhp->mtx_buf = mutex; + else if ((ret = __mutex_alloc(env, MTX_MPOOL_BH, +@@ -428,7 +428,7 @@ __memp_bh_thaw(dbmp, infop, hp, frozen_bhp, alloc_bhp) + #endif + alloc_bhp->mtx_buf = mutex; + MUTEX_LOCK(env, alloc_bhp->mtx_buf); +- atomic_init(&alloc_bhp->ref, 1); ++ atomic_init_db(&alloc_bhp->ref, 1); + F_CLR(alloc_bhp, BH_FROZEN); + } + +diff --git a/mp/mp_region.c b/mp/mp_region.c +index e6cece9..ddbe906 100644 +--- a/mp/mp_region.c ++++ b/mp/mp_region.c +@@ -224,7 +224,7 @@ __memp_init(env, dbmp, reginfo_off, htab_buckets, max_nreg) + MTX_MPOOL_FILE_BUCKET, 0, &htab[i].mtx_hash)) != 0) + return (ret); + SH_TAILQ_INIT(&htab[i].hash_bucket); +- atomic_init(&htab[i].hash_page_dirty, 0); ++ atomic_init_db(&htab[i].hash_page_dirty, 0); + } + + /* +@@ -269,7 +269,7 @@ __memp_init(env, dbmp, reginfo_off, htab_buckets, max_nreg) + hp->mtx_hash = (mtx_base == MUTEX_INVALID) ? MUTEX_INVALID : + mtx_base + i; + SH_TAILQ_INIT(&hp->hash_bucket); +- atomic_init(&hp->hash_page_dirty, 0); ++ atomic_init_db(&hp->hash_page_dirty, 0); + #ifdef HAVE_STATISTICS + hp->hash_io_wait = 0; + hp->hash_frozen = hp->hash_thawed = hp->hash_frozen_freed = 0; +diff --git a/mutex/mut_method.c b/mutex/mut_method.c +index 2588763..5c6d516 100644 +--- a/mutex/mut_method.c ++++ b/mutex/mut_method.c +@@ -426,7 +426,7 @@ atomic_compare_exchange(env, v, oldval, newval) + MUTEX_LOCK(env, mtx); + ret = atomic_read(v) == oldval; + if (ret) +- atomic_init(v, newval); ++ atomic_init_db(v, newval); + MUTEX_UNLOCK(env, mtx); + + return (ret); +diff --git a/mutex/mut_tas.c b/mutex/mut_tas.c +index f3922e0..e40fcdf 100644 +--- a/mutex/mut_tas.c ++++ b/mutex/mut_tas.c +@@ -46,7 +46,7 @@ __db_tas_mutex_init(env, mutex, flags) + + #ifdef HAVE_SHARED_LATCHES + if (F_ISSET(mutexp, DB_MUTEX_SHARED)) +- atomic_init(&mutexp->sharecount, 0); ++ atomic_init_db(&mutexp->sharecount, 0); + else + #endif + if (MUTEX_INIT(&mutexp->tas)) { +@@ -486,7 +486,7 @@ __db_tas_mutex_unlock(env, mutex) + F_CLR(mutexp, DB_MUTEX_LOCKED); + /* Flush flag update before zeroing count */ + MEMBAR_EXIT(); +- atomic_init(&mutexp->sharecount, 0); ++ atomic_init_db(&mutexp->sharecount, 0); + } else { + DB_ASSERT(env, sharecount > 0); + MEMBAR_EXIT(); +EOF + +# The packaged config.guess and config.sub are ancient (2009) and can cause build issues. +# Replace them with modern versions. +# See https://github.com/bitcoin/bitcoin/issues/16064 +CONFIG_GUESS_URL='https://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.guess;hb=4550d2f15b3a7ce2451c1f29500b9339430c877f' +CONFIG_GUESS_HASH='c8f530e01840719871748a8071113435bdfdf75b74c57e78e47898edea8754ae' +CONFIG_SUB_URL='https://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.sub;hb=4550d2f15b3a7ce2451c1f29500b9339430c877f' +CONFIG_SUB_HASH='3969f7d5f6967ccc6f792401b8ef3916a1d1b1d0f0de5a4e354c95addb8b800e' + +rm -f "dist/config.guess" +rm -f "dist/config.sub" + +http_get "${CONFIG_GUESS_URL}" dist/config.guess "${CONFIG_GUESS_HASH}" +http_get "${CONFIG_SUB_URL}" dist/config.sub "${CONFIG_SUB_HASH}" + +cd build_unix/ + +"${BDB_PREFIX}/${BDB_VERSION}/dist/configure" \ + --enable-cxx --disable-shared --disable-replication --with-pic --prefix="${BDB_PREFIX}" \ + "${@}" + +make install + +echo +echo "db4 build complete." +echo +# shellcheck disable=SC2016 +echo 'When compiling bitcoind, run `./configure` in the following way:' +echo +echo " export BDB_PREFIX='${BDB_PREFIX}'" +# shellcheck disable=SC2016 +echo ' ./configure BDB_LIBS="-L${BDB_PREFIX}/lib -ldb_cxx-4.8" BDB_CFLAGS="-I${BDB_PREFIX}/include" ...' diff --git a/contrib/linearize/README.md b/contrib/linearize/README.md new file mode 100644 index 0000000000..25a1c7351a --- /dev/null +++ b/contrib/linearize/README.md @@ -0,0 +1,54 @@ +# Linearize +Construct a linear, no-fork, best version of the Bitcoin blockchain. + +## Step 1: Download hash list + + $ ./linearize-hashes.py linearize.cfg > hashlist.txt + +Required configuration file settings for linearize-hashes: +* RPC: `datadir` (Required if `rpcuser` and `rpcpassword` are not specified) +* RPC: `rpcuser`, `rpcpassword` (Required if `datadir` is not specified) + +Optional config file setting for linearize-hashes: +* RPC: `host` (Default: `127.0.0.1`) +* RPC: `port` (Default: `8332`) +* Blockchain: `min_height`, `max_height` +* `rev_hash_bytes`: If true, the written block hash list will be +byte-reversed. (In other words, the hash returned by getblockhash will have its +bytes reversed.) False by default. Intended for generation of +standalone hash lists but safe to use with linearize-data.py, which will output +the same data no matter which byte format is chosen. + +The `linearize-hashes` script requires a connection, local or remote, to a +JSON-RPC server. Running `bitcoind` or `bitcoin-qt -server` will be sufficient. + +## Step 2: Copy local block data + + $ ./linearize-data.py linearize.cfg + +Required configuration file settings: +* `output_file`: The file that will contain the final blockchain. + or +* `output`: Output directory for linearized `blocks/blkNNNNN.dat` output. + +Optional config file setting for linearize-data: +* `debug_output`: Some printouts may not always be desired. If true, such output +will be printed. +* `file_timestamp`: Set each file's last-accessed and last-modified times, +respectively, to the current time and to the timestamp of the most recent block +written to the script's blockchain. +* `genesis`: The hash of the genesis block in the blockchain. +* `input`: bitcoind blocks/ directory containing blkNNNNN.dat +* `hashlist`: text file containing list of block hashes created by +linearize-hashes.py. +* `max_out_sz`: Maximum size for files created by the `output_file` option. +(Default: `1000*1000*1000 bytes`) +* `netmagic`: Network magic number. +* `out_of_order_cache_sz`: If out-of-order blocks are being read, the block can +be written to a cache so that the blockchain doesn't have to be sought again. +This option specifies the cache size. (Default: `100*1000*1000 bytes`) +* `rev_hash_bytes`: If true, the block hash list written by linearize-hashes.py +will be byte-reversed when read by linearize-data.py. See the linearize-hashes +entry for more information. +* `split_timestamp`: Split blockchain files when a new month is first seen, in +addition to reaching a maximum file size (`max_out_sz`). diff --git a/contrib/linearize/example-linearize.cfg b/contrib/linearize/example-linearize.cfg new file mode 100644 index 0000000000..5f566261ca --- /dev/null +++ b/contrib/linearize/example-linearize.cfg @@ -0,0 +1,63 @@ +# bitcoind RPC settings (linearize-hashes) +rpcuser=someuser +rpcpassword=somepassword +#datadir=~/.bitcoin +host=127.0.0.1 + +#mainnet default +port=8332 + +#testnet default +#port=18332 + +#regtest default +#port=18443 + +#signet default +#port=38332 + +# bootstrap.dat hashlist settings (linearize-hashes) +max_height=313000 + +# bootstrap.dat input/output settings (linearize-data) + +# mainnet +netmagic=f9beb4d9 +genesis=000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f +input=/home/example/.bitcoin/blocks + +# testnet +#netmagic=0b110907 +#genesis=000000000933ea01ad0ee984209779baaec3ced90fa3f408719526f8d77f4943 +#input=/home/example/.bitcoin/testnet3/blocks + +# regtest +#netmagic=fabfb5da +#genesis=0f9188f13cb7b2c71f2a335e3a4fc328bf5beb436012afca590b1a11466e2206 +#input=/home/example/.bitcoin/regtest/blocks + +# signet +#netmagic=0a03cf40 +#genesis=00000008819873e925422c1ff0f99f7cc9bbb232af63a077a480a3633bee1ef6 +#input=/home/example/.bitcoin/signet/blocks + +# "output" option causes blockchain files to be written to the given location, +# with "output_file" ignored. If not used, "output_file" is used instead. +# output=/home/example/blockchain_directory +output_file=/home/example/Downloads/bootstrap.dat +hashlist=hashlist.txt + +# Maximum size in bytes of out-of-order blocks cache in memory +out_of_order_cache_sz = 100000000 + +# Do we want the reverse the hash bytes coming from getblockhash? +rev_hash_bytes = False + +# On a new month, do we want to set the access and modify times of the new +# blockchain file? +file_timestamp = 0 +# Do we want to split the blockchain files given a new month or specific height? +split_timestamp = 0 + +# Do we want debug printouts? +debug_output = False diff --git a/contrib/linearize/linearize-data.py b/contrib/linearize/linearize-data.py new file mode 100755 index 0000000000..b72c7b0d08 --- /dev/null +++ b/contrib/linearize/linearize-data.py @@ -0,0 +1,308 @@ +#!/usr/bin/env python3 +# +# linearize-data.py: Construct a linear, no-fork version of the chain. +# +# Copyright (c) 2013-2021 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +# + +import struct +import re +import os +import os.path +import sys +import hashlib +import datetime +import time +import glob +from collections import namedtuple + +settings = {} + +def calc_hash_str(blk_hdr): + blk_hdr_hash = hashlib.sha256(hashlib.sha256(blk_hdr).digest()).digest() + return blk_hdr_hash[::-1].hex() + +def get_blk_dt(blk_hdr): + members = struct.unpack("<I", blk_hdr[68:68+4]) + nTime = members[0] + dt = datetime.datetime.fromtimestamp(nTime) + dt_ym = datetime.datetime(dt.year, dt.month, 1) + return (dt_ym, nTime) + +# When getting the list of block hashes, undo any byte reversals. +def get_block_hashes(settings): + blkindex = [] + with open(settings['hashlist'], "r", encoding="utf8") as f: + for line in f: + line = line.rstrip() + if settings['rev_hash_bytes'] == 'true': + line = bytes.fromhex(line)[::-1].hex() + blkindex.append(line) + + print("Read " + str(len(blkindex)) + " hashes") + + return blkindex + +# The block map shouldn't give or receive byte-reversed hashes. +def mkblockmap(blkindex): + blkmap = {} + for height,hash in enumerate(blkindex): + blkmap[hash] = height + return blkmap + +# This gets the first block file ID that exists from the input block +# file directory. +def getFirstBlockFileId(block_dir_path): + # First, this sets up a pattern to search for block files, for + # example 'blkNNNNN.dat'. + blkFilePattern = os.path.join(block_dir_path, "blk[0-9][0-9][0-9][0-9][0-9].dat") + + # This search is done with glob + blkFnList = glob.glob(blkFilePattern) + + if len(blkFnList) == 0: + print("blocks not pruned - starting at 0") + return 0 + # We then get the lexicographic minimum, which should be the first + # block file name. + firstBlkFilePath = min(blkFnList) + firstBlkFn = os.path.basename(firstBlkFilePath) + + # now, the string should be ['b','l','k','N','N','N','N','N','.','d','a','t'] + # So get the ID by choosing: 3 4 5 6 7 + # The ID is not necessarily 0 if this is a pruned node. + blkId = int(firstBlkFn[3:8]) + return blkId + +# Block header and extent on disk +BlockExtent = namedtuple('BlockExtent', ['fn', 'offset', 'inhdr', 'blkhdr', 'size']) + +class BlockDataCopier: + def __init__(self, settings, blkindex, blkmap): + self.settings = settings + self.blkindex = blkindex + self.blkmap = blkmap + + # Get first occurring block file id - for pruned nodes this + # will not necessarily be 0 + self.inFn = getFirstBlockFileId(self.settings['input']) + self.inF = None + self.outFn = 0 + self.outsz = 0 + self.outF = None + self.outFname = None + self.blkCountIn = 0 + self.blkCountOut = 0 + + self.lastDate = datetime.datetime(2000, 1, 1) + self.highTS = 1408893517 - 315360000 + self.timestampSplit = False + self.fileOutput = True + self.setFileTime = False + self.maxOutSz = settings['max_out_sz'] + if 'output' in settings: + self.fileOutput = False + if settings['file_timestamp'] != 0: + self.setFileTime = True + if settings['split_timestamp'] != 0: + self.timestampSplit = True + # Extents and cache for out-of-order blocks + self.blockExtents = {} + self.outOfOrderData = {} + self.outOfOrderSize = 0 # running total size for items in outOfOrderData + + def writeBlock(self, inhdr, blk_hdr, rawblock): + blockSizeOnDisk = len(inhdr) + len(blk_hdr) + len(rawblock) + if not self.fileOutput and ((self.outsz + blockSizeOnDisk) > self.maxOutSz): + self.outF.close() + if self.setFileTime: + os.utime(self.outFname, (int(time.time()), self.highTS)) + self.outF = None + self.outFname = None + self.outFn = self.outFn + 1 + self.outsz = 0 + + (blkDate, blkTS) = get_blk_dt(blk_hdr) + if self.timestampSplit and (blkDate > self.lastDate): + print("New month " + blkDate.strftime("%Y-%m") + " @ " + self.hash_str) + self.lastDate = blkDate + if self.outF: + self.outF.close() + if self.setFileTime: + os.utime(self.outFname, (int(time.time()), self.highTS)) + self.outF = None + self.outFname = None + self.outFn = self.outFn + 1 + self.outsz = 0 + + if not self.outF: + if self.fileOutput: + self.outFname = self.settings['output_file'] + else: + self.outFname = os.path.join(self.settings['output'], "blk%05d.dat" % self.outFn) + print("Output file " + self.outFname) + self.outF = open(self.outFname, "wb") + + self.outF.write(inhdr) + self.outF.write(blk_hdr) + self.outF.write(rawblock) + self.outsz = self.outsz + len(inhdr) + len(blk_hdr) + len(rawblock) + + self.blkCountOut = self.blkCountOut + 1 + if blkTS > self.highTS: + self.highTS = blkTS + + if (self.blkCountOut % 1000) == 0: + print('%i blocks scanned, %i blocks written (of %i, %.1f%% complete)' % + (self.blkCountIn, self.blkCountOut, len(self.blkindex), 100.0 * self.blkCountOut / len(self.blkindex))) + + def inFileName(self, fn): + return os.path.join(self.settings['input'], "blk%05d.dat" % fn) + + def fetchBlock(self, extent): + '''Fetch block contents from disk given extents''' + with open(self.inFileName(extent.fn), "rb") as f: + f.seek(extent.offset) + return f.read(extent.size) + + def copyOneBlock(self): + '''Find the next block to be written in the input, and copy it to the output.''' + extent = self.blockExtents.pop(self.blkCountOut) + if self.blkCountOut in self.outOfOrderData: + # If the data is cached, use it from memory and remove from the cache + rawblock = self.outOfOrderData.pop(self.blkCountOut) + self.outOfOrderSize -= len(rawblock) + else: # Otherwise look up data on disk + rawblock = self.fetchBlock(extent) + + self.writeBlock(extent.inhdr, extent.blkhdr, rawblock) + + def run(self): + while self.blkCountOut < len(self.blkindex): + if not self.inF: + fname = self.inFileName(self.inFn) + print("Input file " + fname) + try: + self.inF = open(fname, "rb") + except IOError: + print("Premature end of block data") + return + + inhdr = self.inF.read(8) + if (not inhdr or (inhdr[0] == "\0")): + self.inF.close() + self.inF = None + self.inFn = self.inFn + 1 + continue + + inMagic = inhdr[:4] + if (inMagic != self.settings['netmagic']): + # Seek backwards 7 bytes (skipping the first byte in the previous search) + # and continue searching from the new position if the magic bytes are not + # found. + self.inF.seek(-7, os.SEEK_CUR) + continue + inLenLE = inhdr[4:] + su = struct.unpack("<I", inLenLE) + inLen = su[0] - 80 # length without header + blk_hdr = self.inF.read(80) + inExtent = BlockExtent(self.inFn, self.inF.tell(), inhdr, blk_hdr, inLen) + + self.hash_str = calc_hash_str(blk_hdr) + if not self.hash_str in blkmap: + # Because blocks can be written to files out-of-order as of 0.10, the script + # may encounter blocks it doesn't know about. Treat as debug output. + if settings['debug_output'] == 'true': + print("Skipping unknown block " + self.hash_str) + self.inF.seek(inLen, os.SEEK_CUR) + continue + + blkHeight = self.blkmap[self.hash_str] + self.blkCountIn += 1 + + if self.blkCountOut == blkHeight: + # If in-order block, just copy + rawblock = self.inF.read(inLen) + self.writeBlock(inhdr, blk_hdr, rawblock) + + # See if we can catch up to prior out-of-order blocks + while self.blkCountOut in self.blockExtents: + self.copyOneBlock() + + else: # If out-of-order, skip over block data for now + self.blockExtents[blkHeight] = inExtent + if self.outOfOrderSize < self.settings['out_of_order_cache_sz']: + # If there is space in the cache, read the data + # Reading the data in file sequence instead of seeking and fetching it later is preferred, + # but we don't want to fill up memory + self.outOfOrderData[blkHeight] = self.inF.read(inLen) + self.outOfOrderSize += inLen + else: # If no space in cache, seek forward + self.inF.seek(inLen, os.SEEK_CUR) + + print("Done (%i blocks written)" % (self.blkCountOut)) + +if __name__ == '__main__': + if len(sys.argv) != 2: + print("Usage: linearize-data.py CONFIG-FILE") + sys.exit(1) + + with open(sys.argv[1], encoding="utf8") as f: + for line in f: + # skip comment lines + m = re.search(r'^\s*#', line) + if m: + continue + + # parse key=value lines + m = re.search(r'^(\w+)\s*=\s*(\S.*)$', line) + if m is None: + continue + settings[m.group(1)] = m.group(2) + + # Force hash byte format setting to be lowercase to make comparisons easier. + # Also place upfront in case any settings need to know about it. + if 'rev_hash_bytes' not in settings: + settings['rev_hash_bytes'] = 'false' + settings['rev_hash_bytes'] = settings['rev_hash_bytes'].lower() + + if 'netmagic' not in settings: + settings['netmagic'] = 'f9beb4d9' + if 'genesis' not in settings: + settings['genesis'] = '000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f' + if 'input' not in settings: + settings['input'] = 'input' + if 'hashlist' not in settings: + settings['hashlist'] = 'hashlist.txt' + if 'file_timestamp' not in settings: + settings['file_timestamp'] = 0 + if 'split_timestamp' not in settings: + settings['split_timestamp'] = 0 + if 'max_out_sz' not in settings: + settings['max_out_sz'] = 1000 * 1000 * 1000 + if 'out_of_order_cache_sz' not in settings: + settings['out_of_order_cache_sz'] = 100 * 1000 * 1000 + if 'debug_output' not in settings: + settings['debug_output'] = 'false' + + settings['max_out_sz'] = int(settings['max_out_sz']) + settings['split_timestamp'] = int(settings['split_timestamp']) + settings['file_timestamp'] = int(settings['file_timestamp']) + settings['netmagic'] = bytes.fromhex(settings['netmagic']) + settings['out_of_order_cache_sz'] = int(settings['out_of_order_cache_sz']) + settings['debug_output'] = settings['debug_output'].lower() + + if 'output_file' not in settings and 'output' not in settings: + print("Missing output file / directory") + sys.exit(1) + + blkindex = get_block_hashes(settings) + blkmap = mkblockmap(blkindex) + + # Block hash map won't be byte-reversed. Neither should the genesis hash. + if not settings['genesis'] in blkmap: + print("Genesis block not found in hashlist") + else: + BlockDataCopier(settings, blkindex, blkmap).run() diff --git a/contrib/linearize/linearize-hashes.py b/contrib/linearize/linearize-hashes.py new file mode 100755 index 0000000000..5959300e74 --- /dev/null +++ b/contrib/linearize/linearize-hashes.py @@ -0,0 +1,146 @@ +#!/usr/bin/env python3 +# +# linearize-hashes.py: List blocks in a linear, no-fork version of the chain. +# +# Copyright (c) 2013-2019 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +# + +from http.client import HTTPConnection +import json +import re +import base64 +import sys +import os +import os.path + +settings = {} + +class BitcoinRPC: + def __init__(self, host, port, username, password): + authpair = "%s:%s" % (username, password) + authpair = authpair.encode('utf-8') + self.authhdr = b"Basic " + base64.b64encode(authpair) + self.conn = HTTPConnection(host, port=port, timeout=30) + + def execute(self, obj): + try: + self.conn.request('POST', '/', json.dumps(obj), + { 'Authorization' : self.authhdr, + 'Content-type' : 'application/json' }) + except ConnectionRefusedError: + print('RPC connection refused. Check RPC settings and the server status.', + file=sys.stderr) + return None + + resp = self.conn.getresponse() + if resp is None: + print("JSON-RPC: no response", file=sys.stderr) + return None + + body = resp.read().decode('utf-8') + resp_obj = json.loads(body) + return resp_obj + + @staticmethod + def build_request(idx, method, params): + obj = { 'version' : '1.1', + 'method' : method, + 'id' : idx } + if params is None: + obj['params'] = [] + else: + obj['params'] = params + return obj + + @staticmethod + def response_is_error(resp_obj): + return 'error' in resp_obj and resp_obj['error'] is not None + +def get_block_hashes(settings, max_blocks_per_call=10000): + rpc = BitcoinRPC(settings['host'], settings['port'], + settings['rpcuser'], settings['rpcpassword']) + + height = settings['min_height'] + while height < settings['max_height']+1: + num_blocks = min(settings['max_height']+1-height, max_blocks_per_call) + batch = [] + for x in range(num_blocks): + batch.append(rpc.build_request(x, 'getblockhash', [height + x])) + + reply = rpc.execute(batch) + if reply is None: + print('Cannot continue. Program will halt.') + return None + + for x,resp_obj in enumerate(reply): + if rpc.response_is_error(resp_obj): + print('JSON-RPC: error at height', height+x, ': ', resp_obj['error'], file=sys.stderr) + sys.exit(1) + assert(resp_obj['id'] == x) # assume replies are in-sequence + if settings['rev_hash_bytes'] == 'true': + resp_obj['result'] = bytes.fromhex(resp_obj['result'])[::-1].hex() + print(resp_obj['result']) + + height += num_blocks + +def get_rpc_cookie(): + # Open the cookie file + with open(os.path.join(os.path.expanduser(settings['datadir']), '.cookie'), 'r', encoding="ascii") as f: + combined = f.readline() + combined_split = combined.split(":") + settings['rpcuser'] = combined_split[0] + settings['rpcpassword'] = combined_split[1] + +if __name__ == '__main__': + if len(sys.argv) != 2: + print("Usage: linearize-hashes.py CONFIG-FILE") + sys.exit(1) + + with open(sys.argv[1], encoding="utf8") as f: + for line in f: + # skip comment lines + m = re.search(r'^\s*#', line) + if m: + continue + + # parse key=value lines + m = re.search(r'^(\w+)\s*=\s*(\S.*)$', line) + if m is None: + continue + settings[m.group(1)] = m.group(2) + + if 'host' not in settings: + settings['host'] = '127.0.0.1' + if 'port' not in settings: + settings['port'] = 8332 + if 'min_height' not in settings: + settings['min_height'] = 0 + if 'max_height' not in settings: + settings['max_height'] = 313000 + if 'rev_hash_bytes' not in settings: + settings['rev_hash_bytes'] = 'false' + + use_userpass = True + use_datadir = False + if 'rpcuser' not in settings or 'rpcpassword' not in settings: + use_userpass = False + if 'datadir' in settings and not use_userpass: + use_datadir = True + if not use_userpass and not use_datadir: + print("Missing datadir or username and/or password in cfg file", file=sys.stderr) + sys.exit(1) + + settings['port'] = int(settings['port']) + settings['min_height'] = int(settings['min_height']) + settings['max_height'] = int(settings['max_height']) + + # Force hash byte format setting to be lowercase to make comparisons easier. + settings['rev_hash_bytes'] = settings['rev_hash_bytes'].lower() + + # Get the rpc user and pass from the cookie if the datadir is set + if use_datadir: + get_rpc_cookie() + + get_block_hashes(settings) diff --git a/contrib/macdeploy/LICENSE b/contrib/macdeploy/LICENSE new file mode 100644 index 0000000000..94a9ed024d --- /dev/null +++ b/contrib/macdeploy/LICENSE @@ -0,0 +1,674 @@ + GNU GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. <http://fsf.org/> + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU General Public License is a free, copyleft license for +software and other kinds of works. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +the GNU General Public License is intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. We, the Free Software Foundation, use the +GNU General Public License for most of our software; it applies also to +any other work released this way by its authors. You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + To protect your rights, we need to prevent others from denying you +these rights or asking you to surrender the rights. Therefore, you have +certain responsibilities if you distribute copies of the software, or if +you modify it: responsibilities to respect the freedom of others. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must pass on to the recipients the same +freedoms that you received. You must make sure that they, too, receive +or can get the source code. And you must show them these terms so they +know their rights. + + Developers that use the GNU GPL protect your rights with two steps: +(1) assert copyright on the software, and (2) offer you this License +giving you legal permission to copy, distribute and/or modify it. + + For the developers' and authors' protection, the GPL clearly explains +that there is no warranty for this free software. For both users' and +authors' sake, the GPL requires that modified versions be marked as +changed, so that their problems will not be attributed erroneously to +authors of previous versions. + + Some devices are designed to deny users access to install or run +modified versions of the software inside them, although the manufacturer +can do so. This is fundamentally incompatible with the aim of +protecting users' freedom to change the software. The systematic +pattern of such abuse occurs in the area of products for individuals to +use, which is precisely where it is most unacceptable. Therefore, we +have designed this version of the GPL to prohibit the practice for those +products. If such problems arise substantially in other domains, we +stand ready to extend this provision to those domains in future versions +of the GPL, as needed to protect the freedom of users. + + Finally, every program is threatened constantly by software patents. +States should not allow patents to restrict development and use of +software on general-purpose computers, but in those that do, we wish to +avoid the special danger that patents applied to a free program could +make it effectively proprietary. To prevent this, the GPL assures that +patents cannot be used to render the program non-free. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Use with the GNU Affero General Public License. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU Affero General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the special requirements of the GNU Affero General Public License, +section 13, concerning interaction through a network will apply to the +combination as such. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + <one line to give the program's name and a brief idea of what it does.> + Copyright (C) <year> <name of author> + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. + +Also add information on how to contact you by electronic and paper mail. + + If the program does terminal interaction, make it output a short +notice like this when it starts in an interactive mode: + + <program> Copyright (C) <year> <name of author> + This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, your program's commands +might be different; for a GUI interface, you would use an "about box". + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU GPL, see +<http://www.gnu.org/licenses/>. + + The GNU General Public License does not permit incorporating your program +into proprietary programs. If your program is a subroutine library, you +may consider it more useful to permit linking proprietary applications with +the library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. But first, please read +<http://www.gnu.org/philosophy/why-not-lgpl.html>. diff --git a/contrib/macdeploy/README.md b/contrib/macdeploy/README.md new file mode 100644 index 0000000000..599a0bfa6c --- /dev/null +++ b/contrib/macdeploy/README.md @@ -0,0 +1,112 @@ +# MacOS Deployment + +The `macdeployqtplus` script should not be run manually. Instead, after building as usual: + +```bash +make deploy +``` + +When complete, it will have produced `Bitcoin-Core.dmg`. + +## SDK Extraction + +### Step 1: Obtaining `Xcode.app` + +A free Apple Developer Account is required to proceed. + +Our current macOS SDK +(`Xcode-12.2-12B45b-extracted-SDK-with-libcxx-headers.tar.gz`) +can be extracted from +[Xcode_12.2.xip](https://download.developer.apple.com/Developer_Tools/Xcode_12.2/Xcode_12.2.xip). + +Alternatively, after logging in to your account go to 'Downloads', then 'More' +and search for [`Xcode 12.2`](https://developer.apple.com/download/all/?q=Xcode%2012.2). + +An Apple ID and cookies enabled for the hostname are needed to download this. + +The `sha256sum` of the downloaded XIP archive should be `28d352f8c14a43d9b8a082ac6338dc173cb153f964c6e8fb6ba389e5be528bd0`. + +After Xcode version 7.x, Apple started shipping the `Xcode.app` in a `.xip` +archive. This makes the SDK less-trivial to extract on non-macOS machines. One +approach (tested on Debian Buster) is outlined below: + +```bash +# Install/clone tools needed for extracting Xcode.app +apt install cpio +git clone https://github.com/bitcoin-core/apple-sdk-tools.git + +# Unpack Xcode_12.2.xip and place the resulting Xcode.app in your current +# working directory +python3 apple-sdk-tools/extract_xcode.py -f Xcode_12.2.xip | cpio -d -i +``` + +On macOS the process is more straightforward: + +```bash +xip -x Xcode_12.2.xip +``` + +### Step 2: Generating `Xcode-12.2-12B45b-extracted-SDK-with-libcxx-headers.tar.gz` from `Xcode.app` + +To generate `Xcode-12.2-12B45b-extracted-SDK-with-libcxx-headers.tar.gz`, run +the script [`gen-sdk`](./gen-sdk) with the path to `Xcode.app` (extracted in the +previous stage) as the first argument. + +```bash +# Generate a Xcode-12.2-12B45b-extracted-SDK-with-libcxx-headers.tar.gz from +# the supplied Xcode.app +./contrib/macdeploy/gen-sdk '/path/to/Xcode.app' +``` + +The `sha256sum` of the generated TAR.GZ archive should be `df75d30ecafc429e905134333aeae56ac65fac67cb4182622398fd717df77619`. + +## Deterministic macOS DMG Notes + +Working macOS DMGs are created in Linux by combining a recent `clang`, the Apple +`binutils` (`ld`, `ar`, etc) and DMG authoring tools. + +Apple uses `clang` extensively for development and has upstreamed the necessary +functionality so that a vanilla clang can take advantage. It supports the use of `-F`, +`-target`, `-mmacosx-version-min`, and `-isysroot`, which are all necessary when +building for macOS. + +Apple's version of `binutils` (called `cctools`) contains lots of functionality missing in the +FSF's `binutils`. In addition to extra linker options for frameworks and sysroots, several +other tools are needed as well such as `install_name_tool`, `lipo`, and `nmedit`. These +do not build under Linux, so they have been patched to do so. The work here was used as +a starting point: [mingwandroid/toolchain4](https://github.com/mingwandroid/toolchain4). + +In order to build a working toolchain, the following source packages are needed from +Apple: `cctools`, `dyld`, and `ld64`. + +These tools inject timestamps by default, which produce non-deterministic binaries. The +`ZERO_AR_DATE` environment variable is used to disable that. + +This version of `cctools` has been patched to use the current version of `clang`'s headers +and its `libLTO.so` rather than those from `llvmgcc`, as it was originally done in `toolchain4`. + +To complicate things further, all builds must target an Apple SDK. These SDKs are free to +download, but not redistributable. See the SDK Extraction notes above for how to obtain it. + +The Guix process builds 2 sets of files: Linux tools, then Apple binaries which are +created using these tools. The build process has been designed to avoid including the +SDK's files in Guix's outputs. All interim tarballs are fully deterministic and may be freely +redistributed. + +[`xorrisofs`](https://www.gnu.org/software/xorriso/) is used to create the DMG. + +A background image is added to DMG files by inserting a `.DS_Store` during creation. + +As of OS X 10.9 Mavericks, using an Apple-blessed key to sign binaries is a requirement in +order to satisfy the new Gatekeeper requirements. Because this private key cannot be +shared, we'll have to be a bit creative in order for the build process to remain somewhat +deterministic. Here's how it works: + +- Builders use Guix to create an unsigned release. This outputs an unsigned DMG which + users may choose to bless and run. It also outputs an unsigned app structure in the form + of a tarball, which also contains all of the tools that have been previously (deterministically) + built in order to create a final DMG. +- The Apple keyholder uses this unsigned app to create a detached signature, using the + script that is also included there. Detached signatures are available from this [repository](https://github.com/bitcoin-core/bitcoin-detached-sigs). +- Builders feed the unsigned app + detached signature back into Guix. It uses the + pre-built tools to recombine the pieces into a deterministic DMG. diff --git a/contrib/macdeploy/background.tiff b/contrib/macdeploy/background.tiff Binary files differnew file mode 100644 index 0000000000..1fb088c837 --- /dev/null +++ b/contrib/macdeploy/background.tiff diff --git a/contrib/macdeploy/detached-sig-create.sh b/contrib/macdeploy/detached-sig-create.sh new file mode 100755 index 0000000000..f393331084 --- /dev/null +++ b/contrib/macdeploy/detached-sig-create.sh @@ -0,0 +1,31 @@ +#!/bin/sh +# Copyright (c) 2014-2021 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +export LC_ALL=C +set -e + +ROOTDIR=dist +BUNDLE="${ROOTDIR}/Bitcoin-Qt.app" +BINARY="${BUNDLE}/Contents/MacOS/Bitcoin-Qt" +SIGNAPPLE=signapple +TEMPDIR=sign.temp +ARCH=$(${SIGNAPPLE} info ${BINARY} | head -n 1 | cut -d " " -f 1) +OUT="signature-osx-${ARCH}.tar.gz" +OUTROOT=osx/dist + +if [ -z "$1" ]; then + echo "usage: $0 <signapple args>" + echo "example: $0 <path to key>" + exit 1 +fi + +rm -rf ${TEMPDIR} +mkdir -p ${TEMPDIR} + +${SIGNAPPLE} sign -f --detach "${TEMPDIR}/${OUTROOT}" "$@" "${BUNDLE}" + +tar -C "${TEMPDIR}" -czf "${OUT}" . +rm -rf "${TEMPDIR}" +echo "Created ${OUT}" diff --git a/contrib/macdeploy/gen-sdk b/contrib/macdeploy/gen-sdk new file mode 100755 index 0000000000..6efaaccb8e --- /dev/null +++ b/contrib/macdeploy/gen-sdk @@ -0,0 +1,116 @@ +#!/usr/bin/env python3 +import argparse +import plistlib +import pathlib +import sys +import tarfile +import gzip +import os +import contextlib + +# monkey-patch Python 3.8 and older to fix wrong TAR header handling +# see https://github.com/bitcoin/bitcoin/pull/24534 +# and https://github.com/python/cpython/pull/18080 for more info +if sys.version_info < (3, 9): + _old_create_header = tarfile.TarInfo._create_header + def _create_header(info, format, encoding, errors): + buf = _old_create_header(info, format, encoding, errors) + # replace devmajor/devminor with binary zeroes + buf = buf[:329] + bytes(16) + buf[345:] + # recompute checksum + chksum = tarfile.calc_chksums(buf)[0] + buf = buf[:-364] + bytes("%06o\0" % chksum, "ascii") + buf[-357:] + return buf + tarfile.TarInfo._create_header = staticmethod(_create_header) + +@contextlib.contextmanager +def cd(path): + """Context manager that restores PWD even if an exception was raised.""" + old_pwd = os.getcwd() + os.chdir(str(path)) + try: + yield + finally: + os.chdir(old_pwd) + +def run(): + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawTextHelpFormatter) + + parser.add_argument('xcode_app', metavar='XCODEAPP', nargs=1) + parser.add_argument("-o", metavar='OUTSDKTGZ', nargs=1, dest='out_sdktgz', required=False) + + args = parser.parse_args() + + xcode_app = pathlib.Path(args.xcode_app[0]).resolve() + assert xcode_app.is_dir(), "The supplied Xcode.app path '{}' either does not exist or is not a directory".format(xcode_app) + + xcode_app_plist = xcode_app.joinpath("Contents/version.plist") + with xcode_app_plist.open('rb') as fp: + pl = plistlib.load(fp) + xcode_version = pl['CFBundleShortVersionString'] + xcode_build_id = pl['ProductBuildVersion'] + print("Found Xcode (version: {xcode_version}, build id: {xcode_build_id})".format(xcode_version=xcode_version, xcode_build_id=xcode_build_id)) + + sdk_dir = xcode_app.joinpath("Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk") + sdk_plist = sdk_dir.joinpath("System/Library/CoreServices/SystemVersion.plist") + with sdk_plist.open('rb') as fp: + pl = plistlib.load(fp) + sdk_version = pl['ProductVersion'] + sdk_build_id = pl['ProductBuildVersion'] + print("Found MacOSX SDK (version: {sdk_version}, build id: {sdk_build_id})".format(sdk_version=sdk_version, sdk_build_id=sdk_build_id)) + + out_name = "Xcode-{xcode_version}-{xcode_build_id}-extracted-SDK-with-libcxx-headers".format(xcode_version=xcode_version, xcode_build_id=xcode_build_id) + + xcode_libcxx_dir = xcode_app.joinpath("Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/include/c++/v1") + assert xcode_libcxx_dir.is_dir() + + if args.out_sdktgz: + out_sdktgz_path = pathlib.Path(args.out_sdktgz_path) + else: + # Construct our own out_sdktgz if not specified on the command line + out_sdktgz_path = pathlib.Path("./{}.tar.gz".format(out_name)) + + def tarfp_add_with_base_change(tarfp, dir_to_add, alt_base_dir): + """Add all files in dir_to_add to tarfp, but prepent MEMBERPREFIX to the files' + names + + e.g. if the only file under /root/bazdir is /root/bazdir/qux, invoking: + + tarfp_add_with_base_change(tarfp, "foo/bar", "/root/bazdir") + + would result in the following members being added to tarfp: + + foo/bar/ -> corresponding to /root/bazdir + foo/bar/qux -> corresponding to /root/bazdir/qux + + """ + def change_tarinfo_base(tarinfo): + if tarinfo.name and tarinfo.name.startswith("./"): + tarinfo.name = str(pathlib.Path(alt_base_dir, tarinfo.name)) + if tarinfo.linkname and tarinfo.linkname.startswith("./"): + tarinfo.linkname = str(pathlib.Path(alt_base_dir, tarinfo.linkname)) + # make metadata deterministic + tarinfo.mtime = 0 + tarinfo.uid, tarinfo.uname = 0, '' + tarinfo.gid, tarinfo.gname = 0, '' + # don't use isdir() as there are also executable files present + tarinfo.mode = 0o0755 if tarinfo.mode & 0o0100 else 0o0644 + return tarinfo + with cd(dir_to_add): + # recursion already adds entries in sorted order + tarfp.add(".", recursive=True, filter=change_tarinfo_base) + + print("Creating output .tar.gz file...") + with out_sdktgz_path.open("wb") as fp: + with gzip.GzipFile(fileobj=fp, mode='wb', compresslevel=9, mtime=0) as gzf: + with tarfile.open(mode="w", fileobj=gzf, format=tarfile.GNU_FORMAT) as tarfp: + print("Adding MacOSX SDK {} files...".format(sdk_version)) + tarfp_add_with_base_change(tarfp, sdk_dir, out_name) + print("Adding libc++ headers...") + tarfp_add_with_base_change(tarfp, xcode_libcxx_dir, "{}/usr/include/c++/v1".format(out_name)) + print("Done! Find the resulting gzipped tarball at:") + print(out_sdktgz_path.resolve()) + +if __name__ == '__main__': + run() diff --git a/contrib/macdeploy/macdeployqtplus b/contrib/macdeploy/macdeployqtplus new file mode 100755 index 0000000000..2420539b7c --- /dev/null +++ b/contrib/macdeploy/macdeployqtplus @@ -0,0 +1,598 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2011 Patrick "p2k" Schneider <me@p2k-network.org> +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +# + +import sys, re, os, platform, shutil, stat, subprocess, os.path +from argparse import ArgumentParser +from ds_store import DSStore +from mac_alias import Alias +from pathlib import Path +from subprocess import PIPE, run +from typing import List, Optional + +# This is ported from the original macdeployqt with modifications + +class FrameworkInfo(object): + def __init__(self): + self.frameworkDirectory = "" + self.frameworkName = "" + self.frameworkPath = "" + self.binaryDirectory = "" + self.binaryName = "" + self.binaryPath = "" + self.version = "" + self.installName = "" + self.deployedInstallName = "" + self.sourceFilePath = "" + self.destinationDirectory = "" + self.sourceResourcesDirectory = "" + self.sourceVersionContentsDirectory = "" + self.sourceContentsDirectory = "" + self.destinationResourcesDirectory = "" + self.destinationVersionContentsDirectory = "" + + def __eq__(self, other): + if self.__class__ == other.__class__: + return self.__dict__ == other.__dict__ + else: + return False + + def __str__(self): + return f""" Framework name: {self.frameworkName} + Framework directory: {self.frameworkDirectory} + Framework path: {self.frameworkPath} + Binary name: {self.binaryName} + Binary directory: {self.binaryDirectory} + Binary path: {self.binaryPath} + Version: {self.version} + Install name: {self.installName} + Deployed install name: {self.deployedInstallName} + Source file Path: {self.sourceFilePath} + Deployed Directory (relative to bundle): {self.destinationDirectory} +""" + + def isDylib(self): + return self.frameworkName.endswith(".dylib") + + def isQtFramework(self): + if self.isDylib(): + return self.frameworkName.startswith("libQt") + else: + return self.frameworkName.startswith("Qt") + + reOLine = re.compile(r'^(.+) \(compatibility version [0-9.]+, current version [0-9.]+\)$') + bundleFrameworkDirectory = "Contents/Frameworks" + bundleBinaryDirectory = "Contents/MacOS" + + @classmethod + def fromOtoolLibraryLine(cls, line: str) -> Optional['FrameworkInfo']: + # Note: line must be trimmed + if line == "": + return None + + # Don't deploy system libraries + if line.startswith("/System/Library/") or line.startswith("@executable_path") or line.startswith("/usr/lib/"): + return None + + m = cls.reOLine.match(line) + if m is None: + raise RuntimeError(f"otool line could not be parsed: {line}") + + path = m.group(1) + + info = cls() + info.sourceFilePath = path + info.installName = path + + if path.endswith(".dylib"): + dirname, filename = os.path.split(path) + info.frameworkName = filename + info.frameworkDirectory = dirname + info.frameworkPath = path + + info.binaryDirectory = dirname + info.binaryName = filename + info.binaryPath = path + info.version = "-" + + info.installName = path + info.deployedInstallName = f"@executable_path/../Frameworks/{info.binaryName}" + info.sourceFilePath = path + info.destinationDirectory = cls.bundleFrameworkDirectory + else: + parts = path.split("/") + i = 0 + # Search for the .framework directory + for part in parts: + if part.endswith(".framework"): + break + i += 1 + if i == len(parts): + raise RuntimeError(f"Could not find .framework or .dylib in otool line: {line}") + + info.frameworkName = parts[i] + info.frameworkDirectory = "/".join(parts[:i]) + info.frameworkPath = os.path.join(info.frameworkDirectory, info.frameworkName) + + info.binaryName = parts[i+3] + info.binaryDirectory = "/".join(parts[i+1:i+3]) + info.binaryPath = os.path.join(info.binaryDirectory, info.binaryName) + info.version = parts[i+2] + + info.deployedInstallName = f"@executable_path/../Frameworks/{os.path.join(info.frameworkName, info.binaryPath)}" + info.destinationDirectory = os.path.join(cls.bundleFrameworkDirectory, info.frameworkName, info.binaryDirectory) + + info.sourceResourcesDirectory = os.path.join(info.frameworkPath, "Resources") + info.sourceContentsDirectory = os.path.join(info.frameworkPath, "Contents") + info.sourceVersionContentsDirectory = os.path.join(info.frameworkPath, "Versions", info.version, "Contents") + info.destinationResourcesDirectory = os.path.join(cls.bundleFrameworkDirectory, info.frameworkName, "Resources") + info.destinationVersionContentsDirectory = os.path.join(cls.bundleFrameworkDirectory, info.frameworkName, "Versions", info.version, "Contents") + + return info + +class ApplicationBundleInfo(object): + def __init__(self, path: str): + self.path = path + # for backwards compatibility reasons, this must remain as Bitcoin-Qt + self.binaryPath = os.path.join(path, "Contents", "MacOS", "Bitcoin-Qt") + if not os.path.exists(self.binaryPath): + raise RuntimeError(f"Could not find bundle binary for {path}") + self.resourcesPath = os.path.join(path, "Contents", "Resources") + self.pluginPath = os.path.join(path, "Contents", "PlugIns") + +class DeploymentInfo(object): + def __init__(self): + self.qtPath = None + self.pluginPath = None + self.deployedFrameworks = [] + + def detectQtPath(self, frameworkDirectory: str): + parentDir = os.path.dirname(frameworkDirectory) + if os.path.exists(os.path.join(parentDir, "translations")): + # Classic layout, e.g. "/usr/local/Trolltech/Qt-4.x.x" + self.qtPath = parentDir + else: + self.qtPath = os.getenv("QTDIR", None) + + if self.qtPath is not None: + pluginPath = os.path.join(self.qtPath, "plugins") + if os.path.exists(pluginPath): + self.pluginPath = pluginPath + + def usesFramework(self, name: str) -> bool: + for framework in self.deployedFrameworks: + if framework.endswith(".framework"): + if framework.startswith(f"{name}."): + return True + elif framework.endswith(".dylib"): + if framework.startswith(f"lib{name}."): + return True + return False + +def getFrameworks(binaryPath: str, verbose: int) -> List[FrameworkInfo]: + if verbose: + print(f"Inspecting with otool: {binaryPath}") + otoolbin=os.getenv("OTOOL", "otool") + otool = run([otoolbin, "-L", binaryPath], stdout=PIPE, stderr=PIPE, universal_newlines=True) + if otool.returncode != 0: + sys.stderr.write(otool.stderr) + sys.stderr.flush() + raise RuntimeError(f"otool failed with return code {otool.returncode}") + + otoolLines = otool.stdout.split("\n") + otoolLines.pop(0) # First line is the inspected binary + if ".framework" in binaryPath or binaryPath.endswith(".dylib"): + otoolLines.pop(0) # Frameworks and dylibs list themselves as a dependency. + + libraries = [] + for line in otoolLines: + line = line.replace("@loader_path", os.path.dirname(binaryPath)) + info = FrameworkInfo.fromOtoolLibraryLine(line.strip()) + if info is not None: + if verbose: + print("Found framework:") + print(info) + libraries.append(info) + + return libraries + +def runInstallNameTool(action: str, *args): + installnametoolbin=os.getenv("INSTALL_NAME_TOOL", "install_name_tool") + run([installnametoolbin, "-"+action] + list(args), check=True) + +def changeInstallName(oldName: str, newName: str, binaryPath: str, verbose: int): + if verbose: + print("Using install_name_tool:") + print(" in", binaryPath) + print(" change reference", oldName) + print(" to", newName) + runInstallNameTool("change", oldName, newName, binaryPath) + +def changeIdentification(id: str, binaryPath: str, verbose: int): + if verbose: + print("Using install_name_tool:") + print(" change identification in", binaryPath) + print(" to", id) + runInstallNameTool("id", id, binaryPath) + +def runStrip(binaryPath: str, verbose: int): + stripbin=os.getenv("STRIP", "strip") + if verbose: + print("Using strip:") + print(" stripped", binaryPath) + run([stripbin, "-x", binaryPath], check=True) + +def copyFramework(framework: FrameworkInfo, path: str, verbose: int) -> Optional[str]: + if framework.sourceFilePath.startswith("Qt"): + #standard place for Nokia Qt installer's frameworks + fromPath = f"/Library/Frameworks/{framework.sourceFilePath}" + else: + fromPath = framework.sourceFilePath + toDir = os.path.join(path, framework.destinationDirectory) + toPath = os.path.join(toDir, framework.binaryName) + + if framework.isDylib(): + if not os.path.exists(fromPath): + raise RuntimeError(f"No file at {fromPath}") + + if os.path.exists(toPath): + return None # Already there + + if not os.path.exists(toDir): + os.makedirs(toDir) + + shutil.copy2(fromPath, toPath) + if verbose: + print("Copied:", fromPath) + print(" to:", toPath) + else: + to_dir = os.path.join(path, "Contents", "Frameworks", framework.frameworkName) + if os.path.exists(to_dir): + return None # Already there + + from_dir = framework.frameworkPath + if not os.path.exists(from_dir): + raise RuntimeError(f"No directory at {from_dir}") + + shutil.copytree(from_dir, to_dir, symlinks=True) + if verbose: + print("Copied:", from_dir) + print(" to:", to_dir) + + headers_link = os.path.join(to_dir, "Headers") + if os.path.exists(headers_link): + os.unlink(headers_link) + + headers_dir = os.path.join(to_dir, framework.binaryDirectory, "Headers") + if os.path.exists(headers_dir): + shutil.rmtree(headers_dir) + + permissions = os.stat(toPath) + if not permissions.st_mode & stat.S_IWRITE: + os.chmod(toPath, permissions.st_mode | stat.S_IWRITE) + + return toPath + +def deployFrameworks(frameworks: List[FrameworkInfo], bundlePath: str, binaryPath: str, strip: bool, verbose: int, deploymentInfo: Optional[DeploymentInfo] = None) -> DeploymentInfo: + if deploymentInfo is None: + deploymentInfo = DeploymentInfo() + + while len(frameworks) > 0: + framework = frameworks.pop(0) + deploymentInfo.deployedFrameworks.append(framework.frameworkName) + + print("Processing", framework.frameworkName, "...") + + # Get the Qt path from one of the Qt frameworks + if deploymentInfo.qtPath is None and framework.isQtFramework(): + deploymentInfo.detectQtPath(framework.frameworkDirectory) + + if framework.installName.startswith("@executable_path") or framework.installName.startswith(bundlePath): + print(framework.frameworkName, "already deployed, skipping.") + continue + + # install_name_tool the new id into the binary + changeInstallName(framework.installName, framework.deployedInstallName, binaryPath, verbose) + + # Copy framework to app bundle. + deployedBinaryPath = copyFramework(framework, bundlePath, verbose) + # Skip the rest if already was deployed. + if deployedBinaryPath is None: + continue + + if strip: + runStrip(deployedBinaryPath, verbose) + + # install_name_tool it a new id. + changeIdentification(framework.deployedInstallName, deployedBinaryPath, verbose) + # Check for framework dependencies + dependencies = getFrameworks(deployedBinaryPath, verbose) + + for dependency in dependencies: + changeInstallName(dependency.installName, dependency.deployedInstallName, deployedBinaryPath, verbose) + + # Deploy framework if necessary. + if dependency.frameworkName not in deploymentInfo.deployedFrameworks and dependency not in frameworks: + frameworks.append(dependency) + + return deploymentInfo + +def deployFrameworksForAppBundle(applicationBundle: ApplicationBundleInfo, strip: bool, verbose: int) -> DeploymentInfo: + frameworks = getFrameworks(applicationBundle.binaryPath, verbose) + if len(frameworks) == 0: + print(f"Warning: Could not find any external frameworks to deploy in {applicationBundle.path}.") + return DeploymentInfo() + else: + return deployFrameworks(frameworks, applicationBundle.path, applicationBundle.binaryPath, strip, verbose) + +def deployPlugins(appBundleInfo: ApplicationBundleInfo, deploymentInfo: DeploymentInfo, strip: bool, verbose: int): + plugins = [] + if deploymentInfo.pluginPath is None: + return + for dirpath, dirnames, filenames in os.walk(deploymentInfo.pluginPath): + pluginDirectory = os.path.relpath(dirpath, deploymentInfo.pluginPath) + + if pluginDirectory not in ['styles', 'platforms']: + continue + + for pluginName in filenames: + pluginPath = os.path.join(pluginDirectory, pluginName) + + if pluginName.split('.')[0] not in ['libqminimal', 'libqcocoa', 'libqmacstyle']: + continue + + plugins.append((pluginDirectory, pluginName)) + + for pluginDirectory, pluginName in plugins: + print("Processing plugin", os.path.join(pluginDirectory, pluginName), "...") + + sourcePath = os.path.join(deploymentInfo.pluginPath, pluginDirectory, pluginName) + destinationDirectory = os.path.join(appBundleInfo.pluginPath, pluginDirectory) + if not os.path.exists(destinationDirectory): + os.makedirs(destinationDirectory) + + destinationPath = os.path.join(destinationDirectory, pluginName) + shutil.copy2(sourcePath, destinationPath) + if verbose: + print("Copied:", sourcePath) + print(" to:", destinationPath) + + if strip: + runStrip(destinationPath, verbose) + + dependencies = getFrameworks(destinationPath, verbose) + + for dependency in dependencies: + changeInstallName(dependency.installName, dependency.deployedInstallName, destinationPath, verbose) + + # Deploy framework if necessary. + if dependency.frameworkName not in deploymentInfo.deployedFrameworks: + deployFrameworks([dependency], appBundleInfo.path, destinationPath, strip, verbose, deploymentInfo) + +ap = ArgumentParser(description="""Improved version of macdeployqt. + +Outputs a ready-to-deploy app in a folder "dist" and optionally wraps it in a .dmg file. +Note, that the "dist" folder will be deleted before deploying on each run. + +Optionally, Qt translation files (.qm) can be added to the bundle.""") + +ap.add_argument("app_bundle", nargs=1, metavar="app-bundle", help="application bundle to be deployed") +ap.add_argument("appname", nargs=1, metavar="appname", help="name of the app being deployed") +ap.add_argument("-verbose", nargs="?", const=True, help="Output additional debugging information") +ap.add_argument("-no-plugins", dest="plugins", action="store_false", default=True, help="skip plugin deployment") +ap.add_argument("-no-strip", dest="strip", action="store_false", default=True, help="don't run 'strip' on the binaries") +ap.add_argument("-dmg", nargs="?", const="", metavar="basename", help="create a .dmg disk image") +ap.add_argument("-translations-dir", nargs=1, metavar="path", default=None, help="Path to Qt's translations. Base translations will automatically be added to the bundle's resources.") + +config = ap.parse_args() + +verbose = config.verbose + +# ------------------------------------------------ + +app_bundle = config.app_bundle[0] +appname = config.appname[0] + +if not os.path.exists(app_bundle): + sys.stderr.write(f"Error: Could not find app bundle \"{app_bundle}\"\n") + sys.exit(1) + +# ------------------------------------------------ + +if os.path.exists("dist"): + print("+ Removing existing dist folder +") + shutil.rmtree("dist") + +if os.path.exists(appname + ".dmg"): + print("+ Removing existing DMG +") + os.unlink(appname + ".dmg") + +if os.path.exists(appname + ".temp.dmg"): + os.unlink(appname + ".temp.dmg") + +# ------------------------------------------------ + +target = os.path.join("dist", "Bitcoin-Qt.app") + +print("+ Copying source bundle +") +if verbose: + print(app_bundle, "->", target) + +os.mkdir("dist") +shutil.copytree(app_bundle, target, symlinks=True) + +applicationBundle = ApplicationBundleInfo(target) + +# ------------------------------------------------ + +print("+ Deploying frameworks +") + +try: + deploymentInfo = deployFrameworksForAppBundle(applicationBundle, config.strip, verbose) + if deploymentInfo.qtPath is None: + deploymentInfo.qtPath = os.getenv("QTDIR", None) + if deploymentInfo.qtPath is None: + sys.stderr.write("Warning: Could not detect Qt's path, skipping plugin deployment!\n") + config.plugins = False +except RuntimeError as e: + sys.stderr.write(f"Error: {str(e)}\n") + sys.exit(1) + +# ------------------------------------------------ + +if config.plugins: + print("+ Deploying plugins +") + + try: + deployPlugins(applicationBundle, deploymentInfo, config.strip, verbose) + except RuntimeError as e: + sys.stderr.write(f"Error: {str(e)}\n") + sys.exit(1) + +# ------------------------------------------------ + +if config.translations_dir: + if not Path(config.translations_dir[0]).exists(): + sys.stderr.write(f"Error: Could not find translation dir \"{config.translations_dir[0]}\"\n") + sys.exit(1) + +print("+ Adding Qt translations +") + +translations = Path(config.translations_dir[0]) + +regex = re.compile('qt_[a-z]*(.qm|_[A-Z]*.qm)') + +lang_files = [x for x in translations.iterdir() if regex.match(x.name)] + +for file in lang_files: + if verbose: + print(file.as_posix(), "->", os.path.join(applicationBundle.resourcesPath, file.name)) + shutil.copy2(file.as_posix(), os.path.join(applicationBundle.resourcesPath, file.name)) + +# ------------------------------------------------ + +print("+ Installing qt.conf +") + +qt_conf="""[Paths] +Translations=Resources +Plugins=PlugIns +""" + +with open(os.path.join(applicationBundle.resourcesPath, "qt.conf"), "wb") as f: + f.write(qt_conf.encode()) + +# ------------------------------------------------ + +print("+ Generating .DS_Store +") + +output_file = os.path.join("dist", ".DS_Store") + +ds = DSStore.open(output_file, 'w+') + +ds['.']['bwsp'] = { + 'WindowBounds': '{{300, 280}, {500, 343}}', + 'PreviewPaneVisibility': False, +} + +icvp = { + 'gridOffsetX': 0.0, + 'textSize': 12.0, + 'viewOptionsVersion': 1, + 'backgroundImageAlias': b'\x00\x00\x00\x00\x02\x1e\x00\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xd1\x94\\\xb0H+\x00\x05\x00\x00\x00\x98\x0fbackground.tiff\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x99\xd19\xb0\xf8\x00\x00\x00\x00\x00\x00\x00\x00\xff\xff\xff\xff\x00\x00\r\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x0b.background\x00\x00\x10\x00\x08\x00\x00\xd1\x94\\\xb0\x00\x00\x00\x11\x00\x08\x00\x00\xd19\xb0\xf8\x00\x00\x00\x01\x00\x04\x00\x00\x00\x98\x00\x0e\x00 \x00\x0f\x00b\x00a\x00c\x00k\x00g\x00r\x00o\x00u\x00n\x00d\x00.\x00t\x00i\x00f\x00f\x00\x0f\x00\x02\x00\x00\x00\x12\x00\x1c/.background/background.tiff\x00\x14\x01\x06\x00\x00\x00\x00\x01\x06\x00\x02\x00\x00\x0cMacintosh HD\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xce\x97\xab\xc3H+\x00\x00\x01\x88[\x88\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02u\xab\x8d\xd1\x94\\\xb0devrddsk\xff\xff\xff\xff\x00\x00\t \x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x07bitcoin\x00\x00\x10\x00\x08\x00\x00\xce\x97\xab\xc3\x00\x00\x00\x11\x00\x08\x00\x00\xd1\x94\\\xb0\x00\x00\x00\x01\x00\x14\x01\x88[\x88\x00\x16\xa9\t\x00\x08\xfaR\x00\x08\xfaQ\x00\x02d\x8e\x00\x0e\x00\x02\x00\x00\x00\x0f\x00\x1a\x00\x0c\x00M\x00a\x00c\x00i\x00n\x00t\x00o\x00s\x00h\x00 \x00H\x00D\x00\x13\x00\x01/\x00\x00\x15\x00\x02\x00\x14\xff\xff\x00\x00\xff\xff\x00\x00', + 'backgroundColorBlue': 1.0, + 'iconSize': 96.0, + 'backgroundColorGreen': 1.0, + 'arrangeBy': 'none', + 'showIconPreview': True, + 'gridSpacing': 100.0, + 'gridOffsetY': 0.0, + 'showItemInfo': False, + 'labelOnBottom': True, + 'backgroundType': 2, + 'backgroundColorRed': 1.0 +} +alias = Alias().from_bytes(icvp['backgroundImageAlias']) +alias.volume.name = appname +alias.volume.posix_path = '/Volumes/' + appname +icvp['backgroundImageAlias'] = alias.to_bytes() +ds['.']['icvp'] = icvp + +ds['.']['vSrn'] = ('long', 1) + +ds['Applications']['Iloc'] = (370, 156) +ds['Bitcoin-Qt.app']['Iloc'] = (128, 156) + +ds.flush() +ds.close() + +# ------------------------------------------------ + +if platform.system() == "Darwin": + subprocess.check_call(f"codesign --deep --force --sign - {target}", shell=True) + +print("+ Installing background.tiff +") + +bg_path = os.path.join('dist', '.background', 'background.tiff') +os.mkdir(os.path.dirname(bg_path)) + +tiff_path = os.path.join('contrib', 'macdeploy', 'background.tiff') +shutil.copy2(tiff_path, bg_path) + +# ------------------------------------------------ + +print("+ Generating symlink for /Applications +") + +os.symlink("/Applications", os.path.join('dist', "Applications")) + +# ------------------------------------------------ + +if config.dmg is not None: + + print("+ Preparing .dmg disk image +") + + if verbose: + print("Determining size of \"dist\"...") + size = 0 + for path, dirs, files in os.walk("dist"): + for file in files: + size += os.path.getsize(os.path.join(path, file)) + size += int(size * 0.15) + + if verbose: + print("Creating temp image for modification...") + + tempname: str = appname + ".temp.dmg" + + run(["hdiutil", "create", tempname, "-srcfolder", "dist", "-format", "UDRW", "-size", str(size), "-volname", appname], check=True, universal_newlines=True) + + if verbose: + print("Attaching temp image...") + output = run(["hdiutil", "attach", tempname, "-readwrite"], check=True, universal_newlines=True, stdout=PIPE).stdout + + print("+ Finalizing .dmg disk image +") + + run(["hdiutil", "detach", f"/Volumes/{appname}"], universal_newlines=True) + + run(["hdiutil", "convert", tempname, "-format", "UDZO", "-o", appname, "-imagekey", "zlib-level=9"], check=True, universal_newlines=True) + + os.unlink(tempname) + +# ------------------------------------------------ + +print("+ Done +") + +sys.exit(0) diff --git a/contrib/message-capture/message-capture-docs.md b/contrib/message-capture/message-capture-docs.md new file mode 100644 index 0000000000..7301968461 --- /dev/null +++ b/contrib/message-capture/message-capture-docs.md @@ -0,0 +1,25 @@ +# Per-Peer Message Capture + +## Purpose + +This feature allows for message capture on a per-peer basis. It answers the simple question: "Can I see what messages my node is sending and receiving?" + +## Usage and Functionality + +* Run `bitcoind` with the `-capturemessages` option. +* Look in the `message_capture` folder in your datadir. + * Typically this will be `~/.bitcoin/message_capture`. + * See that there are many folders inside, one for each peer names with its IP address and port. + * Inside each peer's folder there are two `.dat` files: one is for received messages (`msgs_recv.dat`) and the other is for sent messages (`msgs_sent.dat`). +* Run `contrib/message-capture/message-capture-parser.py` with the proper arguments. + * See the `-h` option for help. + * To see all messages, both sent and received, for all peers use: + ``` + ./contrib/message-capture/message-capture-parser.py -o out.json \ + ~/.bitcoin/message_capture/**/*.dat + ``` + * Note: The messages in the given `.dat` files will be interleaved in chronological order. So, giving both received and sent `.dat` files (as above with `*.dat`) will result in all messages being interleaved in chronological order. + * If an output file is not provided (i.e. the `-o` option is not used), then the output prints to `stdout`. +* View the resulting output. + * The output file is `JSON` formatted. + * Suggestion: use `jq` to view the output, with `jq . out.json` diff --git a/contrib/message-capture/message-capture-parser.py b/contrib/message-capture/message-capture-parser.py new file mode 100755 index 0000000000..eefd22a60e --- /dev/null +++ b/contrib/message-capture/message-capture-parser.py @@ -0,0 +1,215 @@ +#!/usr/bin/env python3 +# Copyright (c) 2020 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Parse message capture binary files. To be used in conjunction with -capturemessages.""" + +import argparse +import os +import shutil +import sys +from io import BytesIO +import json +from pathlib import Path +from typing import Any, List, Optional + +sys.path.append(os.path.join(os.path.dirname(__file__), '../../test/functional')) + +from test_framework.messages import ser_uint256 # noqa: E402 +from test_framework.p2p import MESSAGEMAP # noqa: E402 + +TIME_SIZE = 8 +LENGTH_SIZE = 4 +MSGTYPE_SIZE = 12 + +# The test framework classes stores hashes as large ints in many cases. +# These are variables of type uint256 in core. +# There isn't a way to distinguish between a large int and a large int that is actually a blob of bytes. +# As such, they are itemized here. +# Any variables with these names that are of type int are actually uint256 variables. +# (These can be easily found by looking for calls to deser_uint256, deser_uint256_vector, and uint256_from_str in messages.py) +HASH_INTS = [ + "blockhash", + "block_hash", + "hash", + "hashMerkleRoot", + "hashPrevBlock", + "hashstop", + "prev_header", + "sha256", + "stop_hash", +] + +HASH_INT_VECTORS = [ + "hashes", + "headers", + "vHave", + "vHash", +] + + +class ProgressBar: + def __init__(self, total: float): + self.total = total + self.running = 0 + + def set_progress(self, progress: float): + cols = shutil.get_terminal_size()[0] + if cols <= 12: + return + max_blocks = cols - 9 + num_blocks = int(max_blocks * progress) + print('\r[ {}{} ] {:3.0f}%' + .format('#' * num_blocks, + ' ' * (max_blocks - num_blocks), + progress * 100), + end ='') + + def update(self, more: float): + self.running += more + self.set_progress(self.running / self.total) + + +def to_jsonable(obj: Any) -> Any: + if hasattr(obj, "__dict__"): + return obj.__dict__ + elif hasattr(obj, "__slots__"): + ret = {} # type: Any + for slot in obj.__slots__: + val = getattr(obj, slot, None) + if slot in HASH_INTS and isinstance(val, int): + ret[slot] = ser_uint256(val).hex() + elif slot in HASH_INT_VECTORS: + assert all(isinstance(a, int) for a in val) + ret[slot] = [ser_uint256(a).hex() for a in val] + else: + ret[slot] = to_jsonable(val) + return ret + elif isinstance(obj, list): + return [to_jsonable(a) for a in obj] + elif isinstance(obj, bytes): + return obj.hex() + else: + return obj + + +def process_file(path: str, messages: List[Any], recv: bool, progress_bar: Optional[ProgressBar]) -> None: + with open(path, 'rb') as f_in: + if progress_bar: + bytes_read = 0 + + while True: + if progress_bar: + # Update progress bar + diff = f_in.tell() - bytes_read - 1 + progress_bar.update(diff) + bytes_read = f_in.tell() - 1 + + # Read the Header + tmp_header_raw = f_in.read(TIME_SIZE + LENGTH_SIZE + MSGTYPE_SIZE) + if not tmp_header_raw: + break + tmp_header = BytesIO(tmp_header_raw) + time = int.from_bytes(tmp_header.read(TIME_SIZE), "little") # type: int + msgtype = tmp_header.read(MSGTYPE_SIZE).split(b'\x00', 1)[0] # type: bytes + length = int.from_bytes(tmp_header.read(LENGTH_SIZE), "little") # type: int + + # Start converting the message to a dictionary + msg_dict = {} + msg_dict["direction"] = "recv" if recv else "sent" + msg_dict["time"] = time + msg_dict["size"] = length # "size" is less readable here, but more readable in the output + + msg_ser = BytesIO(f_in.read(length)) + + # Determine message type + if msgtype not in MESSAGEMAP: + # Unrecognized message type + try: + msgtype_tmp = msgtype.decode() + if not msgtype_tmp.isprintable(): + raise UnicodeDecodeError + msg_dict["msgtype"] = msgtype_tmp + except UnicodeDecodeError: + msg_dict["msgtype"] = "UNREADABLE" + msg_dict["body"] = msg_ser.read().hex() + msg_dict["error"] = "Unrecognized message type." + messages.append(msg_dict) + print(f"WARNING - Unrecognized message type {msgtype} in {path}", file=sys.stderr) + continue + + # Deserialize the message + msg = MESSAGEMAP[msgtype]() + msg_dict["msgtype"] = msgtype.decode() + + try: + msg.deserialize(msg_ser) + except KeyboardInterrupt: + raise + except Exception: + # Unable to deserialize message body + msg_ser.seek(0, os.SEEK_SET) + msg_dict["body"] = msg_ser.read().hex() + msg_dict["error"] = "Unable to deserialize message." + messages.append(msg_dict) + print(f"WARNING - Unable to deserialize message in {path}", file=sys.stderr) + continue + + # Convert body of message into a jsonable object + if length: + msg_dict["body"] = to_jsonable(msg) + messages.append(msg_dict) + + if progress_bar: + # Update the progress bar to the end of the current file + # in case we exited the loop early + f_in.seek(0, os.SEEK_END) # Go to end of file + diff = f_in.tell() - bytes_read - 1 + progress_bar.update(diff) + + +def main(): + parser = argparse.ArgumentParser( + description=__doc__, + epilog="EXAMPLE \n\t{0} -o out.json <data-dir>/message_capture/**/*.dat".format(sys.argv[0]), + formatter_class=argparse.RawTextHelpFormatter) + parser.add_argument( + "capturepaths", + nargs='+', + help="binary message capture files to parse.") + parser.add_argument( + "-o", "--output", + help="output file. If unset print to stdout") + parser.add_argument( + "-n", "--no-progress-bar", + action='store_true', + help="disable the progress bar. Automatically set if the output is not a terminal") + args = parser.parse_args() + capturepaths = [Path.cwd() / Path(capturepath) for capturepath in args.capturepaths] + output = Path.cwd() / Path(args.output) if args.output else False + use_progress_bar = (not args.no_progress_bar) and sys.stdout.isatty() + + messages = [] # type: List[Any] + if use_progress_bar: + total_size = sum(capture.stat().st_size for capture in capturepaths) + progress_bar = ProgressBar(total_size) + else: + progress_bar = None + + for capture in capturepaths: + process_file(str(capture), messages, "recv" in capture.stem, progress_bar) + + messages.sort(key=lambda msg: msg['time']) + + if use_progress_bar: + progress_bar.set_progress(1) + + jsonrep = json.dumps(messages) + if output: + with open(str(output), 'w+', encoding="utf8") as f_out: + f_out.write(jsonrep) + else: + print(jsonrep) + +if __name__ == "__main__": + main() diff --git a/contrib/qos/README.md b/contrib/qos/README.md new file mode 100644 index 0000000000..0ded87c58f --- /dev/null +++ b/contrib/qos/README.md @@ -0,0 +1,5 @@ +### QoS (Quality of service) ### + +This is a Linux bash script that will set up tc to limit the outgoing bandwidth for connections to the Bitcoin network. It limits outbound TCP traffic with a source or destination port of 8333, but not if the destination IP is within a LAN. + +This means one can have an always-on bitcoind instance running, and another local bitcoind/bitcoin-qt instance which connects to this node and receives blocks from it. diff --git a/contrib/qos/tc.sh b/contrib/qos/tc.sh new file mode 100755 index 0000000000..7ebcbf2251 --- /dev/null +++ b/contrib/qos/tc.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash +# +# Copyright (c) 2017-2021 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +export LC_ALL=C +#network interface on which to limit traffic +IF="eth0" +#limit of the network interface in question +LINKCEIL="1gbit" +#limit outbound Bitcoin protocol traffic to this rate +LIMIT="160kbit" +#defines the IPv4 address space for which you wish to disable rate limiting +LOCALNET_V4="192.168.0.0/16" +#defines the IPv6 address space for which you wish to disable rate limiting +LOCALNET_V6="fe80::/10" + +#delete existing rules ('Error: Cannot delete qdisc with handle of zero.' means there weren't any.) +tc qdisc del dev ${IF} root + +#add root class +tc qdisc add dev ${IF} root handle 1: htb default 10 + +#add parent class +tc class add dev ${IF} parent 1: classid 1:1 htb rate ${LINKCEIL} ceil ${LINKCEIL} + +#add our two classes. one unlimited, another limited +tc class add dev ${IF} parent 1:1 classid 1:10 htb rate ${LINKCEIL} ceil ${LINKCEIL} prio 0 +tc class add dev ${IF} parent 1:1 classid 1:11 htb rate ${LIMIT} ceil ${LIMIT} prio 1 + +#add handles to our classes so packets marked with <x> go into the class with "... handle <x> fw ..." +tc filter add dev ${IF} parent 1: protocol ip prio 1 handle 1 fw classid 1:10 +tc filter add dev ${IF} parent 1: protocol ip prio 2 handle 2 fw classid 1:11 + +if [ -n "${LOCALNET_V6}" ] ; then + # v6 cannot have the same priority value as v4 + tc filter add dev ${IF} parent 1: protocol ipv6 prio 3 handle 1 fw classid 1:10 + tc filter add dev ${IF} parent 1: protocol ipv6 prio 4 handle 2 fw classid 1:11 +fi + +#delete any existing rules +#disable for now +#ret=0 +#while [ $ret -eq 0 ]; do +# iptables -t mangle -D OUTPUT 1 +# ret=$? +#done + +#limit outgoing traffic to and from port 8333. but not when dealing with a host on the local network +# (defined by $LOCALNET_V4 and $LOCALNET_V6) +# --set-mark marks packages matching these criteria with the number "2" (v4) +# --set-mark marks packages matching these criteria with the number "4" (v6) +# these packets are filtered by the tc filter with "handle 2" +# this filter sends the packages into the 1:11 class, and this class is limited to ${LIMIT} +iptables -t mangle -A OUTPUT -p tcp -m tcp --dport 8333 ! -d ${LOCALNET_V4} -j MARK --set-mark 0x2 +iptables -t mangle -A OUTPUT -p tcp -m tcp --sport 8333 ! -d ${LOCALNET_V4} -j MARK --set-mark 0x2 + +if [ -n "${LOCALNET_V6}" ] ; then + ip6tables -t mangle -A OUTPUT -p tcp -m tcp --dport 8333 ! -d ${LOCALNET_V6} -j MARK --set-mark 0x4 + ip6tables -t mangle -A OUTPUT -p tcp -m tcp --sport 8333 ! -d ${LOCALNET_V6} -j MARK --set-mark 0x4 +fi diff --git a/contrib/seeds/.gitignore b/contrib/seeds/.gitignore new file mode 100644 index 0000000000..d9a2451f70 --- /dev/null +++ b/contrib/seeds/.gitignore @@ -0,0 +1,2 @@ +seeds_main.txt +asmap-filled.dat diff --git a/contrib/seeds/README.md b/contrib/seeds/README.md new file mode 100644 index 0000000000..b2ea7522ac --- /dev/null +++ b/contrib/seeds/README.md @@ -0,0 +1,18 @@ +# Seeds + +Utility to generate the seeds.txt list that is compiled into the client +(see [src/chainparamsseeds.h](/src/chainparamsseeds.h) and other utilities in [contrib/seeds](/contrib/seeds)). + +Be sure to update `PATTERN_AGENT` in `makeseeds.py` to include the current version, +and remove old versions as necessary (at a minimum when GetDesirableServiceFlags +changes its default return value, as those are the services which seeds are added +to addrman with). + +The seeds compiled into the release are created from sipa's DNS seed and AS map +data. Run the following commands from the `/contrib/seeds` directory: + + curl https://bitcoin.sipa.be/seeds.txt.gz | gzip -dc > seeds_main.txt + curl https://bitcoin.sipa.be/asmap-filled.dat > asmap-filled.dat + python3 makeseeds.py -a asmap-filled.dat < seeds_main.txt > nodes_main.txt + cat nodes_main_manual.txt >> nodes_main.txt + python3 generate-seeds.py . > ../../src/chainparamsseeds.h diff --git a/contrib/seeds/asmap.py b/contrib/seeds/asmap.py new file mode 100644 index 0000000000..e28e5cf532 --- /dev/null +++ b/contrib/seeds/asmap.py @@ -0,0 +1,815 @@ +# Copyright (c) 2022 Pieter Wuille +# Distributed under the MIT software license, see the accompanying +# file LICENSE or http://www.opensource.org/licenses/mit-license.php. + +""" +This module provides the ASNEntry and ASMap classes. +""" + +import copy +import ipaddress +import random +import unittest +from enum import Enum +from functools import total_ordering +from typing import Callable, Dict, Iterable, List, Optional, Tuple, Union, overload + +def net_to_prefix(net: Union[ipaddress.IPv4Network,ipaddress.IPv6Network]) -> List[bool]: + """ + Convert an IPv4 or IPv6 network to a prefix represented as a list of bits. + + IPv4 ranges are remapped to their IPv4-mapped IPv6 range (::ffff:0:0/96). + """ + num_bits = net.prefixlen + netrange = int.from_bytes(net.network_address.packed, 'big') + + # Map an IPv4 prefix into IPv6 space. + if isinstance(net, ipaddress.IPv4Network): + num_bits += 96 + netrange += 0xffff00000000 + + # Strip unused bottom bits. + assert (netrange & ((1 << (128 - num_bits)) - 1)) == 0 + return [((netrange >> (127 - i)) & 1) != 0 for i in range(num_bits)] + +def prefix_to_net(prefix: List[bool]) -> Union[ipaddress.IPv4Network,ipaddress.IPv6Network]: + """The reverse operation of net_to_prefix.""" + # Convert to number + netrange = sum(b << (127 - i) for i, b in enumerate(prefix)) + num_bits = len(prefix) + assert num_bits <= 128 + + # Return IPv4 range if in ::ffff:0:0/96 + if num_bits >= 96 and (netrange >> 32) == 0xffff: + return ipaddress.IPv4Network((netrange & 0xffffffff, num_bits - 96), True) + + # Return IPv6 range otherwise. + return ipaddress.IPv6Network((netrange, num_bits), True) + +# Shortcut for (prefix, ASN) entries. +ASNEntry = Tuple[List[bool], int] + +# Shortcut for (prefix, old ASN, new ASN) entries. +ASNDiff = Tuple[List[bool], int, int] + +class _VarLenCoder: + """ + A class representing a custom variable-length binary encoder/decoder for + integers. Each object represents a different coder, with different parameters + minval and clsbits. + + The encoding is easiest to describe using an example. Let's say minval=100 and + clsbits=[4,2,2,3]. In that case: + - x in [100..115]: encoded as [0] + [4-bit BE encoding of (x-100)]. + - x in [116..119]: encoded as [1,0] + [2-bit BE encoding of (x-116)]. + - x in [120..123]: encoded as [1,1,0] + [2-bit BE encoding of (x-120)]. + - x in [124..131]: encoded as [1,1,1] + [3-bit BE encoding of (x-124)]. + + In general, every number is encoded as: + - First, k "1"-bits, where k is the class the number falls in (there is one class + per element of clsbits). + - Then, a "0"-bit, unless k is the highest class, in which case there is nothing. + - Lastly, clsbits[k] bits encoding in big endian the position in its class that + number falls into. + - Every class k consists of 2^clsbits[k] consecutive integers. k=0 starts at minval, + other classes start one past the last element of the class before it. + """ + + def __init__(self, minval: int, clsbits: List[int]): + """Construct a new _VarLenCoder.""" + self._minval = minval + self._clsbits = clsbits + self._maxval = minval + sum(1 << b for b in clsbits) - 1 + + def can_encode(self, val: int) -> bool: + """Check whether value val is in the range this coder supports.""" + return self._minval <= val <= self._maxval + + def encode(self, val: int, ret: List[int]) -> None: + """Append encoding of val onto integer list ret.""" + + assert self._minval <= val <= self._maxval + val -= self._minval + bits = 0 + for k, bits in enumerate(self._clsbits): + if val >> bits: + # If the value will not fit in class k, subtract its range from v, + # emit a "1" bit and continue with the next class. + val -= 1 << bits + ret.append(1) + else: + if k + 1 < len(self._clsbits): + # Unless we're in the last class, emit a "0" bit. + ret.append(0) + break + # And then encode v (now the position within the class) in big endian. + ret.extend((val >> (bits - 1 - b)) & 1 for b in range(bits)) + + def encode_size(self, val: int) -> int: + """Compute how many bits are needed to encode val.""" + assert self._minval <= val <= self._maxval + val -= self._minval + ret = 0 + bits = 0 + for k, bits in enumerate(self._clsbits): + if val >> bits: + val -= 1 << bits + ret += 1 + else: + ret += k + 1 < len(self._clsbits) + break + return ret + bits + + def decode(self, stream, bitpos) -> Tuple[int,int]: + """Decode a number starting at bitpos in stream, returning value and new bitpos.""" + val = self._minval + bits = 0 + for k, bits in enumerate(self._clsbits): + bit = 0 + if k + 1 < len(self._clsbits): + bit = stream[bitpos] + bitpos += 1 + if not bit: + break + val += 1 << bits + for i in range(bits): + bit = stream[bitpos] + bitpos += 1 + val += bit << (bits - 1 - i) + return val, bitpos + +# Variable-length encoders used in the binary asmap format. +_CODER_INS = _VarLenCoder(0, [0, 0, 1]) +_CODER_ASN = _VarLenCoder(1, list(range(15, 25))) +_CODER_MATCH = _VarLenCoder(2, list(range(1, 9))) +_CODER_JUMP = _VarLenCoder(17, list(range(5, 31))) + +class _Instruction(Enum): + """One instruction in the binary asmap format.""" + # A return instruction, encoded as [0], returns a constant ASN. It is followed by + # an integer using the ASN encoding. + RETURN = 0 + # A jump instruction, encoded as [1,0] inspects the next unused bit in the input + # and either continues execution (if 0), or skips a specified number of bits (if 1). + # It is followed by an integer, and then two subprograms. The integer uses jump encoding + # and corresponds to the length of the first subprogram (so it can be skipped). + JUMP = 1 + # A match instruction, encoded as [1,1,0] inspects 1 or more of the next unused bits + # in the input with its argument. If they all match, execution continues. If they do + # not, failure is returned. If a default instruction has been executed before, instead + # of failure the default instruction's argument is returned. It is followed by an + # integer in match encoding, and a subprogram. That value is at least 2 bits and at + # most 9 bits. An n-bit value signifies matching (n-1) bits in the input with the lower + # (n-1) bits in the match value. + MATCH = 2 + # A default instruction, encoded as [1,1,1] sets the default variable to its argument, + # and continues execution. It is followed by an integer in ASN encoding, and a subprogram. + DEFAULT = 3 + # Not an actual instruction, but a way to encode the empty program that fails. In the + # encoder, it is used more generally to represent the failure case inside MATCH instructions, + # which may (if used inside the context of a DEFAULT instruction) actually correspond to + # a successful return. In this usage, they're always converted to an actual MATCH or RETURN + # before the top level is reached (see make_default below). + END = 4 + +class _BinNode: + """A class representing a (node of) the parsed binary asmap format.""" + + @overload + def __init__(self, ins: _Instruction): ... + @overload + def __init__(self, ins: _Instruction, arg1: int): ... + @overload + def __init__(self, ins: _Instruction, arg1: "_BinNode", arg2: "_BinNode"): ... + @overload + def __init__(self, ins: _Instruction, arg1: int, arg2: "_BinNode"): ... + + def __init__(self, ins: _Instruction, arg1=None, arg2=None): + """ + Construct a new asmap node. Possibilities are: + - _BinNode(_Instruction.RETURN, asn) + - _BinNode(_Instruction.JUMP, node_0, node_1) + - _BinNode(_Instruction.MATCH, val, node) + - _BinNode(_Instruction.DEFAULT, asn, node) + - _BinNode(_Instruction.END) + """ + self.ins = ins + self.arg1 = arg1 + self.arg2 = arg2 + if ins == _Instruction.RETURN: + assert isinstance(arg1, int) + assert arg2 is None + self.size = _CODER_INS.encode_size(ins.value) + _CODER_ASN.encode_size(arg1) + elif ins == _Instruction.JUMP: + assert isinstance(arg1, _BinNode) + assert isinstance(arg2, _BinNode) + self.size = (_CODER_INS.encode_size(ins.value) + _CODER_JUMP.encode_size(arg1.size) + + arg1.size + arg2.size) + elif ins == _Instruction.DEFAULT: + assert isinstance(arg1, int) + assert isinstance(arg2, _BinNode) + self.size = _CODER_INS.encode_size(ins.value) + _CODER_ASN.encode_size(arg1) + arg2.size + elif ins == _Instruction.MATCH: + assert isinstance(arg1, int) + assert isinstance(arg2, _BinNode) + self.size = (_CODER_INS.encode_size(ins.value) + _CODER_MATCH.encode_size(arg1) + + arg2.size) + elif ins == _Instruction.END: + assert arg1 is None + assert arg2 is None + self.size = 0 + else: + assert False + + @staticmethod + def make_end() -> "_BinNode": + """Constructor for a _BinNode with just an END instruction.""" + return _BinNode(_Instruction.END) + + @staticmethod + def make_leaf(val: int) -> "_BinNode": + """Constructor for a _BinNode of just a RETURN instruction.""" + assert val is not None and val > 0 + return _BinNode(_Instruction.RETURN, val) + + @staticmethod + def make_branch(node0: "_BinNode", node1: "_BinNode") -> "_BinNode": + """ + Construct a _BinNode corresponding to running either the node0 or node1 subprogram, + based on the next input bit. It exploits shortcuts that are possible in the encoding, + and uses either a JUMP, MATCH, or END instruction. + """ + if node0.ins == _Instruction.END and node1.ins == _Instruction.END: + return node0 + if node0.ins == _Instruction.END: + if node1.ins == _Instruction.MATCH and node1.arg1 <= 0xFF: + return _BinNode(node1.ins, node1.arg1 + (1 << node1.arg1.bit_length()), node1.arg2) + return _BinNode(_Instruction.MATCH, 3, node1) + if node1.ins == _Instruction.END: + if node0.ins == _Instruction.MATCH and node0.arg1 <= 0xFF: + return _BinNode(node0.ins, node0.arg1 + (1 << (node0.arg1.bit_length() - 1)), + node0.arg2) + return _BinNode(_Instruction.MATCH, 2, node0) + return _BinNode(_Instruction.JUMP, node0, node1) + + @staticmethod + def make_default(val: int, sub: "_BinNode") -> "_BinNode": + """ + Construct a _BinNode that corresponds to the specified subprogram, with the specified + default value. It exploits shortcuts that are possible in the encoding, and will use + either a DEFAULT or a RETURN instruction.""" + assert val is not None and val > 0 + if sub.ins == _Instruction.END: + return _BinNode(_Instruction.RETURN, val) + if sub.ins in (_Instruction.RETURN, _Instruction.DEFAULT): + return sub + return _BinNode(_Instruction.DEFAULT, val, sub) + +@total_ordering +class ASMap: + """ + A class whose objects represent a mapping from subnets to ASNs. + + Internally the mapping is stored as a binary trie, but can be converted + from/to a list of ASNEntry objects, and from/to the binary asmap file format. + + In the trie representation, nodes are represented as bare lists for efficiency + and ease of manipulation: + - [0] means an unassigned subnet (no ASN mapping for it is present) + - [int] means a subnet mapped entirely to the specified ASN. + - [node,node] means a subnet whose lower half and upper half have different + - mappings, represented by new trie nodes. + """ + + def update(self, prefix: List[bool], asn: int) -> None: + """Update this ASMap object to map prefix to the specified asn.""" + assert asn == 0 or _CODER_ASN.can_encode(asn) + + def recurse(node: List, offset: int) -> None: + if offset == len(prefix): + # Reached the end of prefix; overwrite this node. + node.clear() + node.append(asn) + return + if len(node) == 1: + # Need to descend into a leaf node; split it up. + oldasn = node[0] + node.clear() + node.append([oldasn]) + node.append([oldasn]) + # Descend into the node. + recurse(node[prefix[offset]], offset + 1) + # If the result is two identical leaf children, merge them. + if len(node[0]) == 1 and len(node[1]) == 1 and node[0] == node[1]: + oldasn = node[0][0] + node.clear() + node.append(oldasn) + recurse(self._trie, 0) + + def update_multi(self, entries: List[Tuple[List[bool], int]]) -> None: + """Apply multiple update operations, where longer prefixes take precedence.""" + entries.sort(key=lambda entry: len(entry[0])) + for prefix, asn in entries: + self.update(prefix, asn) + + def _set_trie(self, trie) -> None: + """Set trie directly. Internal use only.""" + def recurse(node: List) -> None: + if len(node) < 2: + return + recurse(node[0]) + recurse(node[1]) + if len(node[0]) == 2: + return + if node[0] == node[1]: + if len(node[0]) == 0: + node.clear() + else: + asn = node[0][0] + node.clear() + node.append(asn) + recurse(trie) + self._trie = trie + + def __init__(self, entries: Optional[Iterable[ASNEntry]] = None) -> None: + """Construct an ASMap object from an optional list of entries.""" + self._trie = [0] + if entries is not None: + def entry_key(entry): + """Sort function that places shorter prefixes first.""" + prefix, asn = entry + return len(prefix), prefix, asn + for prefix, asn in sorted(entries, key=entry_key): + self.update(prefix, asn) + + def lookup(self, prefix: List[bool]) -> Optional[int]: + """Look up a prefix. Returns ASN, or 0 if unassigned, or None if indeterminate.""" + node = self._trie + for bit in prefix: + if len(node) == 1: + break + node = node[bit] + if len(node) == 1: + return node[0] + return None + + def _to_entries_flat(self, fill: bool = False) -> List[ASNEntry]: + """Convert an ASMap object to a list of non-overlapping (prefix, asn) objects.""" + prefix : List[bool] = [] + + def recurse(node: List) -> List[ASNEntry]: + ret = [] + if len(node) == 1: + if node[0] > 0: + ret = [(list(prefix), node[0])] + elif len(node) == 2: + prefix.append(False) + ret = recurse(node[0]) + prefix[-1] = True + ret += recurse(node[1]) + prefix.pop() + if fill and len(ret) > 1: + asns = set(x[1] for x in ret) + if len(asns) == 1: + ret = [(list(prefix), list(asns)[0])] + return ret + return recurse(self._trie) + + def _to_entries_minimal(self, fill: bool = False) -> List[ASNEntry]: + """Convert a trie to a minimal list of ASNEntry objects, exploiting overlap.""" + prefix : List[bool] = [] + + def recurse(node: List) -> (Tuple[Dict[Optional[int], List[ASNEntry]], bool]): + if len(node) == 1 and node[0] == 0: + return {None if fill else 0: []}, True + if len(node) == 1: + return {node[0]: [], None: [(list(prefix), node[0])]}, False + ret: Dict[Optional[int], List[ASNEntry]] = {} + prefix.append(False) + left, lhole = recurse(node[0]) + prefix[-1] = True + right, rhole = recurse(node[1]) + prefix.pop() + hole = not fill and (lhole or rhole) + def candidate(ctx: Optional[int], res0: Optional[List[ASNEntry]], + res1: Optional[List[ASNEntry]]): + if res0 is not None and res1 is not None: + if ctx not in ret or len(res0) + len(res1) < len(ret[ctx]): + ret[ctx] = res0 + res1 + for ctx in set(left) | set(right): + candidate(ctx, left.get(ctx), right.get(ctx)) + candidate(ctx, left.get(None), right.get(ctx)) + candidate(ctx, left.get(ctx), right.get(None)) + if not hole: + for ctx in list(ret): + if ctx is not None: + candidate(None, [(list(prefix), ctx)], ret[ctx]) + if None in ret: + ret = {ctx:entries for ctx, entries in ret.items() + if ctx is None or len(entries) < len(ret[None])} + if hole: + ret = {ctx:entries for ctx, entries in ret.items() if ctx is None or ctx == 0} + return ret, hole + res, _ = recurse(self._trie) + return res[0] if 0 in res else res[None] + + def __str__(self) -> str: + """Convert this ASMap object to a string containing Python code constructing it.""" + return f"ASMap({self._trie})" + + def to_entries(self, overlapping: bool = True, fill: bool = False) -> List[ASNEntry]: + """ + Convert the mappings in this ASMap object to a list of ASNEntry objects. + + Arguments: + overlapping: Permit the subnets in the resulting ASNEntry to overlap. + Setting this can result in a shorter list. + fill: Permit the resulting ASNEntry objects to cover subnets that + are unassigned in this ASMap object. Setting this can + result in a shorter list. + """ + if overlapping: + return self._to_entries_minimal(fill) + return self._to_entries_flat(fill) + + @staticmethod + def from_random(num_leaves: int = 10, max_asn: int = 6, + unassigned_prob: float = 0.5) -> "ASMap": + """ + Construct a random ASMap object, with specified: + - Number of leaves in its trie (at least 1) + - Maximum ASN value (at least 1) + - Probability for leaf nodes to be unassigned + + The number of leaves in the resulting object may be less than what is + requested. This method is mostly intended for testing. + """ + assert num_leaves >= 1 + assert max_asn >= 1 or unassigned_prob == 1 + assert _CODER_ASN.can_encode(max_asn) + assert 0.0 <= unassigned_prob <= 1.0 + trie: List = [] + leaves = [trie] + ret = ASMap() + for i in range(1, num_leaves): + idx = random.randrange(i) + leaf = leaves[idx] + lastleaf = leaves.pop() + if idx + 1 < i: + leaves[idx] = lastleaf + leaf.append([]) + leaf.append([]) + leaves.append(leaf[0]) + leaves.append(leaf[1]) + for leaf in leaves: + if random.random() >= unassigned_prob: + leaf.append(random.randrange(1, max_asn + 1)) + else: + leaf.append(0) + #pylint: disable=protected-access + ret._set_trie(trie) + return ret + + def _to_binnode(self, fill: bool = False) -> _BinNode: + """Convert a trie to a _BinNode object.""" + def recurse(node: List) -> Tuple[Dict[Optional[int], _BinNode], bool]: + if len(node) == 1 and node[0] == 0: + return {(None if fill else 0): _BinNode.make_end()}, True + if len(node) == 1: + return {None: _BinNode.make_leaf(node[0]), node[0]: _BinNode.make_end()}, False + ret: Dict[Optional[int], _BinNode] = {} + left, lhole = recurse(node[0]) + right, rhole = recurse(node[1]) + hole = (lhole or rhole) and not fill + + def candidate(ctx: Optional[int], arg1, arg2, func: Callable): + if arg1 is not None and arg2 is not None: + cand = func(arg1, arg2) + if ctx not in ret or cand.size < ret[ctx].size: + ret[ctx] = cand + + for ctx in set(left) | set(right): + candidate(ctx, left.get(ctx), right.get(ctx), _BinNode.make_branch) + candidate(ctx, left.get(None), right.get(ctx), _BinNode.make_branch) + candidate(ctx, left.get(ctx), right.get(None), _BinNode.make_branch) + if not hole: + for ctx in set(ret) - set([None]): + candidate(None, ctx, ret[ctx], _BinNode.make_default) + if None in ret: + ret = {ctx:enc for ctx, enc in ret.items() + if ctx is None or enc.size < ret[None].size} + if hole: + ret = {ctx:enc for ctx, enc in ret.items() if ctx is None or ctx == 0} + return ret, hole + res, _ = recurse(self._trie) + return res[0] if 0 in res else res[None] + + @staticmethod + def _from_binnode(binnode: _BinNode) -> "ASMap": + """Construct an ASMap object from a _BinNode. Internal use only.""" + def recurse(node: _BinNode, default: int) -> List: + if node.ins == _Instruction.RETURN: + return [node.arg1] + if node.ins == _Instruction.JUMP: + return [recurse(node.arg1, default), recurse(node.arg2, default)] + if node.ins == _Instruction.MATCH: + val = node.arg1 + sub = recurse(node.arg2, default) + while val >= 2: + bit = val & 1 + val >>= 1 + if bit: + sub = [[default], sub] + else: + sub = [sub, [default]] + return sub + assert node.ins == _Instruction.DEFAULT + return recurse(node.arg2, node.arg1) + ret = ASMap() + if binnode.ins != _Instruction.END: + #pylint: disable=protected-access + ret._set_trie(recurse(binnode, 0)) + return ret + + def to_binary(self, fill: bool = False) -> bytes: + """ + Convert this ASMap object to binary. + + Argument: + fill: permit the resulting binary encoder to contain mappers for + unassigned subnets in this ASMap object. Doing so may + reduce the size of the encoding. + Returns: + A bytes object with the encoding of this ASMap object. + """ + bits: List[int] = [] + + def recurse(node: _BinNode) -> None: + _CODER_INS.encode(node.ins.value, bits) + if node.ins == _Instruction.RETURN: + _CODER_ASN.encode(node.arg1, bits) + elif node.ins == _Instruction.JUMP: + _CODER_JUMP.encode(node.arg1.size, bits) + recurse(node.arg1) + recurse(node.arg2) + elif node.ins == _Instruction.DEFAULT: + _CODER_ASN.encode(node.arg1, bits) + recurse(node.arg2) + else: + assert node.ins == _Instruction.MATCH + _CODER_MATCH.encode(node.arg1, bits) + recurse(node.arg2) + + binnode = self._to_binnode(fill) + if binnode.ins != _Instruction.END: + recurse(binnode) + + val = 0 + nbits = 0 + ret = [] + for bit in bits: + val += (bit << nbits) + nbits += 1 + if nbits == 8: + ret.append(val) + val = 0 + nbits = 0 + if nbits: + ret.append(val) + return bytes(ret) + + @staticmethod + def from_binary(bindata: bytes) -> Optional["ASMap"]: + """Decode an ASMap object from the provided binary encoding.""" + + bits: List[int] = [] + for byte in bindata: + bits.extend((byte >> i) & 1 for i in range(8)) + + def recurse(bitpos: int) -> Tuple[_BinNode, int]: + insval, bitpos = _CODER_INS.decode(bits, bitpos) + ins = _Instruction(insval) + if ins == _Instruction.RETURN: + asn, bitpos = _CODER_ASN.decode(bits, bitpos) + return _BinNode(ins, asn), bitpos + if ins == _Instruction.JUMP: + jump, bitpos = _CODER_JUMP.decode(bits, bitpos) + left, bitpos1 = recurse(bitpos) + if bitpos1 != bitpos + jump: + raise ValueError("Inconsistent jump") + right, bitpos = recurse(bitpos1) + return _BinNode(ins, left, right), bitpos + if ins == _Instruction.MATCH: + match, bitpos = _CODER_MATCH.decode(bits, bitpos) + sub, bitpos = recurse(bitpos) + return _BinNode(ins, match, sub), bitpos + assert ins == _Instruction.DEFAULT + asn, bitpos = _CODER_ASN.decode(bits, bitpos) + sub, bitpos = recurse(bitpos) + return _BinNode(ins, asn, sub), bitpos + + if len(bits) == 0: + binnode = _BinNode(_Instruction.END) + else: + try: + binnode, bitpos = recurse(0) + except (ValueError, IndexError): + return None + if bitpos < len(bits) - 7: + return None + if not all(bit == 0 for bit in bits[bitpos:]): + return None + + return ASMap._from_binnode(binnode) + + def __lt__(self, other: "ASMap") -> bool: + return self._trie < other._trie + + def __eq__(self, other: object) -> bool: + if isinstance(other, ASMap): + return self._trie == other._trie + return False + + def extends(self, req: "ASMap") -> bool: + """Determine whether this matches req for all subranges where req is assigned.""" + def recurse(actual: List, require: List) -> bool: + if len(require) == 1 and require[0] == 0: + return True + if len(require) == 1: + if len(actual) == 1: + return bool(require[0] == actual[0]) + return recurse(actual[0], require) and recurse(actual[1], require) + if len(actual) == 2: + return recurse(actual[0], require[0]) and recurse(actual[1], require[1]) + return recurse(actual, require[0]) and recurse(actual, require[1]) + assert isinstance(req, ASMap) + #pylint: disable=protected-access + return recurse(self._trie, req._trie) + + def diff(self, other: "ASMap") -> List[ASNDiff]: + """Compute the diff from self to other.""" + prefix: List[bool] = [] + ret: List[ASNDiff] = [] + + def recurse(old_node: List, new_node: List): + if len(old_node) == 1 and len(new_node) == 1: + if old_node[0] != new_node[0]: + ret.append((list(prefix), old_node[0], new_node[0])) + else: + old_left: List = old_node if len(old_node) == 1 else old_node[0] + old_right: List = old_node if len(old_node) == 1 else old_node[1] + new_left: List = new_node if len(new_node) == 1 else new_node[0] + new_right: List = new_node if len(new_node) == 1 else new_node[1] + prefix.append(False) + recurse(old_left, new_left) + prefix[-1] = True + recurse(old_right, new_right) + prefix.pop() + assert isinstance(other, ASMap) + #pylint: disable=protected-access + recurse(self._trie, other._trie) + return ret + + def __copy__(self) -> "ASMap": + """Construct a copy of this ASMap object. Its state will not be shared.""" + ret = ASMap() + #pylint: disable=protected-access + ret._set_trie(copy.deepcopy(self._trie)) + return ret + + def __deepcopy__(self, _) -> "ASMap": + # ASMap objects do not allow sharing of the _trie member, so we don't need the memoization. + return self.__copy__() + + +class TestASMap(unittest.TestCase): + """Unit tests for this module.""" + + def test_ipv6_prefix_roundtrips(self) -> None: + """Test that random IPv6 network ranges roundtrip through prefix encoding.""" + for _ in range(20): + net_bits = random.getrandbits(128) + for prefix_len in range(0, 129): + masked_bits = (net_bits >> (128 - prefix_len)) << (128 - prefix_len) + net = ipaddress.IPv6Network((masked_bits.to_bytes(16, 'big'), prefix_len)) + prefix = net_to_prefix(net) + self.assertTrue(len(prefix) <= 128) + net2 = prefix_to_net(prefix) + self.assertEqual(net, net2) + + def test_ipv4_prefix_roundtrips(self) -> None: + """Test that random IPv4 network ranges roundtrip through prefix encoding.""" + for _ in range(100): + net_bits = random.getrandbits(32) + for prefix_len in range(0, 33): + masked_bits = (net_bits >> (32 - prefix_len)) << (32 - prefix_len) + net = ipaddress.IPv4Network((masked_bits.to_bytes(4, 'big'), prefix_len)) + prefix = net_to_prefix(net) + self.assertTrue(32 <= len(prefix) <= 128) + net2 = prefix_to_net(prefix) + self.assertEqual(net, net2) + + def test_asmap_roundtrips(self) -> None: + """Test case that verifies random ASMap objects roundtrip to/from entries/binary.""" + # Iterate over the number of leaves the random test ASMap objects have. + for leaves in range(1, 20): + # Iterate over the number of bits in the AS numbers used. + for asnbits in range(0, 24): + # Iterate over the probability that leaves are unassigned. + for pct in range(101): + # Construct a random ASMap object according to the above parameters. + asmap = ASMap.from_random(num_leaves=leaves, max_asn=1 + (1 << asnbits), + unassigned_prob=0.01 * pct) + # Run tests for to_entries and construction from those entries, both + # for overlapping and non-overlapping ones. + for overlapping in [False, True]: + entries = asmap.to_entries(overlapping=overlapping, fill=False) + random.shuffle(entries) + asmap2 = ASMap(entries) + assert asmap2 is not None + self.assertEqual(asmap2, asmap) + entries = asmap.to_entries(overlapping=overlapping, fill=True) + random.shuffle(entries) + asmap2 = ASMap(entries) + assert asmap2 is not None + self.assertTrue(asmap2.extends(asmap)) + + # Run tests for to_binary and construction from binary. + enc = asmap.to_binary(fill=False) + asmap3 = ASMap.from_binary(enc) + assert asmap3 is not None + self.assertEqual(asmap3, asmap) + enc = asmap.to_binary(fill=True) + asmap3 = ASMap.from_binary(enc) + assert asmap3 is not None + self.assertTrue(asmap3.extends(asmap)) + + def test_patching(self) -> None: + """Test behavior of update, lookup, extends, and diff.""" + #pylint: disable=too-many-locals,too-many-nested-blocks + # Iterate over the number of leaves the random test ASMap objects have. + for leaves in range(1, 20): + # Iterate over the number of bits in the AS numbers used. + for asnbits in range(0, 10): + # Iterate over the probability that leaves are unassigned. + for pct in range(0, 101): + # Construct a random ASMap object according to the above parameters. + asmap = ASMap.from_random(num_leaves=leaves, max_asn=1 + (1 << asnbits), + unassigned_prob=0.01 * pct) + # Make a copy of that asmap object to which patches will be applied. + # It starts off being equal to asmap. + patched = copy.copy(asmap) + # Keep a list of patches performed. + patches: List[ASNEntry] = [] + # Initially there cannot be any difference. + self.assertEqual(asmap.diff(patched), []) + # Make 5 patches, each building on top of the previous ones. + for _ in range(0, 5): + # Construct a random path and new ASN to assign it to, apply it to patched, + # and remember it in patches. + pathlen = random.randrange(5) + path = [random.getrandbits(1) != 0 for _ in range(pathlen)] + newasn = random.randrange(1 + (1 << asnbits)) + patched.update(path, newasn) + patches = [(path, newasn)] + patches + + # Compute the diff, and whether asmap extends patched, and the other way + # around. + diff = asmap.diff(patched) + self.assertEqual(asmap == patched, len(diff) == 0) + extends = asmap.extends(patched) + back_extends = patched.extends(asmap) + # Determine whether those extends results are consistent with the diff + # result. + self.assertEqual(extends, all(d[2] == 0 for d in diff)) + self.assertEqual(back_extends, all(d[1] == 0 for d in diff)) + # For every diff found: + for path, old_asn, new_asn in diff: + # Verify asmap and patched actually differ there. + self.assertTrue(old_asn != new_asn) + self.assertEqual(asmap.lookup(path), old_asn) + self.assertEqual(patched.lookup(path), new_asn) + for _ in range(2): + # Extend the path far enough that it's smaller than any mapped + # range, and check the lookup holds there too. + spec_path = list(path) + while len(spec_path) < 32: + spec_path.append(random.getrandbits(1) != 0) + self.assertEqual(asmap.lookup(spec_path), old_asn) + self.assertEqual(patched.lookup(spec_path), new_asn) + # Search through the list of performed patches to find the last one + # applying to the extended path (note that patches is in reverse + # order, so the first match should work). + found = False + for patch_path, patch_asn in patches: + if spec_path[:len(patch_path)] == patch_path: + # When found, it must match whatever the result was patched + # to. + self.assertEqual(new_asn, patch_asn) + found = True + break + # And such a patch must exist. + self.assertTrue(found) + +if __name__ == '__main__': + unittest.main() diff --git a/contrib/seeds/generate-seeds.py b/contrib/seeds/generate-seeds.py new file mode 100755 index 0000000000..44345e3987 --- /dev/null +++ b/contrib/seeds/generate-seeds.py @@ -0,0 +1,178 @@ +#!/usr/bin/env python3 +# Copyright (c) 2014-2021 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +''' +Script to generate list of seed nodes for chainparams.cpp. + +This script expects two text files in the directory that is passed as an +argument: + + nodes_main.txt + nodes_test.txt + +These files must consist of lines in the format + + <ip>:<port> + [<ipv6>]:<port> + <onion>.onion:<port> + <i2p>.b32.i2p:<port> + +The output will be two data structures with the peers in binary format: + + static const uint8_t chainparams_seed_{main,test}[]={ + ... + } + +These should be pasted into `src/chainparamsseeds.h`. +''' + +from base64 import b32decode +from enum import Enum +import struct +import sys +import os +import re + +class BIP155Network(Enum): + IPV4 = 1 + IPV6 = 2 + TORV2 = 3 # no longer supported + TORV3 = 4 + I2P = 5 + CJDNS = 6 + +def name_to_bip155(addr): + '''Convert address string to BIP155 (networkID, addr) tuple.''' + if addr.endswith('.onion'): + vchAddr = b32decode(addr[0:-6], True) + if len(vchAddr) == 35: + assert vchAddr[34] == 3 + return (BIP155Network.TORV3, vchAddr[:32]) + elif len(vchAddr) == 10: + return (BIP155Network.TORV2, vchAddr) + else: + raise ValueError('Invalid onion %s' % vchAddr) + elif addr.endswith('.b32.i2p'): + vchAddr = b32decode(addr[0:-8] + '====', True) + if len(vchAddr) == 32: + return (BIP155Network.I2P, vchAddr) + else: + raise ValueError(f'Invalid I2P {vchAddr}') + elif '.' in addr: # IPv4 + return (BIP155Network.IPV4, bytes((int(x) for x in addr.split('.')))) + elif ':' in addr: # IPv6 or CJDNS + sub = [[], []] # prefix, suffix + x = 0 + addr = addr.split(':') + for i,comp in enumerate(addr): + if comp == '': + if i == 0 or i == (len(addr)-1): # skip empty component at beginning or end + continue + x += 1 # :: skips to suffix + assert(x < 2) + else: # two bytes per component + val = int(comp, 16) + sub[x].append(val >> 8) + sub[x].append(val & 0xff) + nullbytes = 16 - len(sub[0]) - len(sub[1]) + assert((x == 0 and nullbytes == 0) or (x == 1 and nullbytes > 0)) + addr_bytes = bytes(sub[0] + ([0] * nullbytes) + sub[1]) + if addr_bytes[0] == 0xfc: + # Assume that seeds with fc00::/8 addresses belong to CJDNS, + # not to the publicly unroutable "Unique Local Unicast" network, see + # RFC4193: https://datatracker.ietf.org/doc/html/rfc4193#section-8 + return (BIP155Network.CJDNS, addr_bytes) + else: + return (BIP155Network.IPV6, addr_bytes) + else: + raise ValueError('Could not parse address %s' % addr) + +def parse_spec(s): + '''Convert endpoint string to BIP155 (networkID, addr, port) tuple.''' + match = re.match(r'\[([0-9a-fA-F:]+)\](?::([0-9]+))?$', s) + if match: # ipv6 + host = match.group(1) + port = match.group(2) + elif s.count(':') > 1: # ipv6, no port + host = s + port = '' + else: + (host,_,port) = s.partition(':') + + if not port: + port = 0 + else: + port = int(port) + + host = name_to_bip155(host) + + if host[0] == BIP155Network.TORV2: + return None # TORV2 is no longer supported, so we ignore it + else: + return host + (port, ) + +def ser_compact_size(l): + r = b"" + if l < 253: + r = struct.pack("B", l) + elif l < 0x10000: + r = struct.pack("<BH", 253, l) + elif l < 0x100000000: + r = struct.pack("<BI", 254, l) + else: + r = struct.pack("<BQ", 255, l) + return r + +def bip155_serialize(spec): + ''' + Serialize (networkID, addr, port) tuple to BIP155 binary format. + ''' + r = b"" + r += struct.pack('B', spec[0].value) + r += ser_compact_size(len(spec[1])) + r += spec[1] + r += struct.pack('>H', spec[2]) + return r + +def process_nodes(g, f, structname): + g.write('static const uint8_t %s[] = {\n' % structname) + for line in f: + comment = line.find('#') + if comment != -1: + line = line[0:comment] + line = line.strip() + if not line: + continue + + spec = parse_spec(line) + if spec is None: # ignore this entry (e.g. no longer supported addresses like TORV2) + continue + blob = bip155_serialize(spec) + hoststr = ','.join(('0x%02x' % b) for b in blob) + g.write(f' {hoststr},\n') + g.write('};\n') + +def main(): + if len(sys.argv)<2: + print(('Usage: %s <path_to_nodes_txt>' % sys.argv[0]), file=sys.stderr) + sys.exit(1) + g = sys.stdout + indir = sys.argv[1] + g.write('#ifndef BITCOIN_CHAINPARAMSSEEDS_H\n') + g.write('#define BITCOIN_CHAINPARAMSSEEDS_H\n') + g.write('/**\n') + g.write(' * List of fixed seed nodes for the bitcoin network\n') + g.write(' * AUTOGENERATED by contrib/seeds/generate-seeds.py\n') + g.write(' *\n') + g.write(' * Each line contains a BIP155 serialized (networkID, addr, port) tuple.\n') + g.write(' */\n') + with open(os.path.join(indir,'nodes_main.txt'), 'r', encoding="utf8") as f: + process_nodes(g, f, 'chainparams_seed_main') + g.write('\n') + with open(os.path.join(indir,'nodes_test.txt'), 'r', encoding="utf8") as f: + process_nodes(g, f, 'chainparams_seed_test') + g.write('#endif // BITCOIN_CHAINPARAMSSEEDS_H\n') + +if __name__ == '__main__': + main() diff --git a/contrib/seeds/makeseeds.py b/contrib/seeds/makeseeds.py new file mode 100755 index 0000000000..37c6f5fd7c --- /dev/null +++ b/contrib/seeds/makeseeds.py @@ -0,0 +1,235 @@ +#!/usr/bin/env python3 +# Copyright (c) 2013-2020 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +# +# Generate seeds.txt from Pieter's DNS seeder +# + +import argparse +import ipaddress +import re +import sys +import collections +from typing import List, Dict, Union + +from asmap import ASMap, net_to_prefix + +NSEEDS=512 + +MAX_SEEDS_PER_ASN = { + 'ipv4': 2, + 'ipv6': 10, +} + +MIN_BLOCKS = 730000 + +PATTERN_IPV4 = re.compile(r"^((\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})):(\d+)$") +PATTERN_IPV6 = re.compile(r"^\[([0-9a-z:]+)\]:(\d+)$") +PATTERN_ONION = re.compile(r"^([a-z2-7]{56}\.onion):(\d+)$") +PATTERN_AGENT = re.compile( + r"^/Satoshi:(" + r"0.14.(0|1|2|3|99)|" + r"0.15.(0|1|2|99)|" + r"0.16.(0|1|2|3|99)|" + r"0.17.(0|0.1|1|2|99)|" + r"0.18.(0|1|99)|" + r"0.19.(0|1|2|99)|" + r"0.20.(0|1|2|99)|" + r"0.21.(0|1|2|99)|" + r"22.(0|99)|" + r"23.99" + r")") + +def parseline(line: str) -> Union[dict, None]: + """ Parses a line from `seeds_main.txt` into a dictionary of details for that line. + or `None`, if the line could not be parsed. + """ + sline = line.split() + if len(sline) < 11: + # line too short to be valid, skip it. + return None + m = PATTERN_IPV4.match(sline[0]) + sortkey = None + ip = None + if m is None: + m = PATTERN_IPV6.match(sline[0]) + if m is None: + m = PATTERN_ONION.match(sline[0]) + if m is None: + return None + else: + net = 'onion' + ipstr = sortkey = m.group(1) + port = int(m.group(2)) + else: + net = 'ipv6' + if m.group(1) in ['::']: # Not interested in localhost + return None + ipstr = m.group(1) + sortkey = ipstr # XXX parse IPv6 into number, could use name_to_ipv6 from generate-seeds + port = int(m.group(2)) + else: + # Do IPv4 sanity check + ip = 0 + for i in range(0,4): + if int(m.group(i+2)) < 0 or int(m.group(i+2)) > 255: + return None + ip = ip + (int(m.group(i+2)) << (8*(3-i))) + if ip == 0: + return None + net = 'ipv4' + sortkey = ip + ipstr = m.group(1) + port = int(m.group(6)) + # Skip bad results. + if sline[1] == 0: + return None + # Extract uptime %. + uptime30 = float(sline[7][:-1]) + # Extract Unix timestamp of last success. + lastsuccess = int(sline[2]) + # Extract protocol version. + version = int(sline[10]) + # Extract user agent. + agent = sline[11][1:-1] + # Extract service flags. + service = int(sline[9], 16) + # Extract blocks. + blocks = int(sline[8]) + # Construct result. + return { + 'net': net, + 'ip': ipstr, + 'port': port, + 'ipnum': ip, + 'uptime': uptime30, + 'lastsuccess': lastsuccess, + 'version': version, + 'agent': agent, + 'service': service, + 'blocks': blocks, + 'sortkey': sortkey, + } + +def dedup(ips: List[Dict]) -> List[Dict]: + """ Remove duplicates from `ips` where multiple ips share address and port. """ + d = {} + for ip in ips: + d[ip['ip'],ip['port']] = ip + return list(d.values()) + +def filtermultiport(ips: List[Dict]) -> List[Dict]: + """ Filter out hosts with more nodes per IP""" + hist = collections.defaultdict(list) + for ip in ips: + hist[ip['sortkey']].append(ip) + return [value[0] for (key,value) in list(hist.items()) if len(value)==1] + +# Based on Greg Maxwell's seed_filter.py +def filterbyasn(asmap: ASMap, ips: List[Dict], max_per_asn: Dict, max_per_net: int) -> List[Dict]: + """ Prunes `ips` by + (a) trimming ips to have at most `max_per_net` ips from each net (e.g. ipv4, ipv6); and + (b) trimming ips to have at most `max_per_asn` ips from each asn in each net. + """ + # Sift out ips by type + ips_ipv46 = [ip for ip in ips if ip['net'] in ['ipv4', 'ipv6']] + ips_onion = [ip for ip in ips if ip['net'] == 'onion'] + + # Filter IPv46 by ASN, and limit to max_per_net per network + result = [] + net_count: Dict[str, int] = collections.defaultdict(int) + asn_count: Dict[int, int] = collections.defaultdict(int) + + for i, ip in enumerate(ips_ipv46): + if net_count[ip['net']] == max_per_net: + # do not add this ip as we already too many + # ips from this network + continue + asn = asmap.lookup(net_to_prefix(ipaddress.ip_network(ip['ip']))) + if not asn or asn_count[ip['net'], asn] == max_per_asn[ip['net']]: + # do not add this ip as we already have too many + # ips from this ASN on this network + continue + asn_count[ip['net'], asn] += 1 + net_count[ip['net']] += 1 + ip['asn'] = asn + result.append(ip) + + # Add back Onions (up to max_per_net) + result.extend(ips_onion[0:max_per_net]) + return result + +def ip_stats(ips: List[Dict]) -> str: + """ Format and return pretty string from `ips`. """ + hist: Dict[str, int] = collections.defaultdict(int) + for ip in ips: + if ip is not None: + hist[ip['net']] += 1 + + return f"{hist['ipv4']:6d} {hist['ipv6']:6d} {hist['onion']:6d}" + +def parse_args(): + argparser = argparse.ArgumentParser(description='Generate a list of bitcoin node seed ip addresses.') + argparser.add_argument("-a","--asmap", help='the location of the asmap asn database file (required)', required=True) + return argparser.parse_args() + +def main(): + args = parse_args() + + print(f'Loading asmap database "{args.asmap}"…', end='', file=sys.stderr, flush=True) + with open(args.asmap, 'rb') as f: + asmap = ASMap.from_binary(f.read()) + print('Done.', file=sys.stderr) + + print('Loading and parsing DNS seeds…', end='', file=sys.stderr, flush=True) + lines = sys.stdin.readlines() + ips = [parseline(line) for line in lines] + print('Done.', file=sys.stderr) + + print('\x1b[7m IPv4 IPv6 Onion Pass \x1b[0m', file=sys.stderr) + print(f'{ip_stats(ips):s} Initial', file=sys.stderr) + # Skip entries with invalid address. + ips = [ip for ip in ips if ip is not None] + print(f'{ip_stats(ips):s} Skip entries with invalid address', file=sys.stderr) + # Skip duplicates (in case multiple seeds files were concatenated) + ips = dedup(ips) + print(f'{ip_stats(ips):s} After removing duplicates', file=sys.stderr) + # Enforce minimal number of blocks. + ips = [ip for ip in ips if ip['blocks'] >= MIN_BLOCKS] + print(f'{ip_stats(ips):s} Enforce minimal number of blocks', file=sys.stderr) + # Require service bit 1. + ips = [ip for ip in ips if (ip['service'] & 1) == 1] + print(f'{ip_stats(ips):s} Require service bit 1', file=sys.stderr) + # Require at least 50% 30-day uptime for clearnet, 10% for onion. + req_uptime = { + 'ipv4': 50, + 'ipv6': 50, + 'onion': 10, + } + ips = [ip for ip in ips if ip['uptime'] > req_uptime[ip['net']]] + print(f'{ip_stats(ips):s} Require minimum uptime', file=sys.stderr) + # Require a known and recent user agent. + ips = [ip for ip in ips if PATTERN_AGENT.match(ip['agent'])] + print(f'{ip_stats(ips):s} Require a known and recent user agent', file=sys.stderr) + # Sort by availability (and use last success as tie breaker) + ips.sort(key=lambda x: (x['uptime'], x['lastsuccess'], x['ip']), reverse=True) + # Filter out hosts with multiple bitcoin ports, these are likely abusive + ips = filtermultiport(ips) + print(f'{ip_stats(ips):s} Filter out hosts with multiple bitcoin ports', file=sys.stderr) + # Look up ASNs and limit results, both per ASN and globally. + ips = filterbyasn(asmap, ips, MAX_SEEDS_PER_ASN, NSEEDS) + print(f'{ip_stats(ips):s} Look up ASNs and limit results per ASN and per net', file=sys.stderr) + # Sort the results by IP address (for deterministic output). + ips.sort(key=lambda x: (x['net'], x['sortkey'])) + for ip in ips: + if ip['net'] == 'ipv6': + print(f"[{ip['ip']}]:{ip['port']}", end="") + else: + print(f"{ip['ip']}:{ip['port']}", end="") + if 'asn' in ip: + print(f" # AS{ip['asn']}", end="") + print() + +if __name__ == '__main__': + main() diff --git a/contrib/seeds/nodes_main.txt b/contrib/seeds/nodes_main.txt new file mode 100644 index 0000000000..d8e34bdb60 --- /dev/null +++ b/contrib/seeds/nodes_main.txt @@ -0,0 +1,689 @@ +2.37.30.144:8777 +2.138.174.158:8333 +2.152.78.124:8333 +5.8.18.154:8333 +5.45.74.50:8333 +5.79.123.3:8333 +5.102.168.217:22220 +5.103.137.146:9333 +5.128.87.126:8333 +5.172.132.200:8333 +5.188.62.18:8333 +5.254.101.226:8334 +8.210.18.56:8333 +8.210.92.32:8333 +14.13.34.225:16181 +14.39.151.167:8333 +18.196.79.108:8333 +18.218.139.58:48333 +20.184.15.116:8433 +23.175.0.220:8333 +23.233.107.21:8333 +24.35.68.229:8333 +24.37.3.26:8333 +24.102.91.203:8333 +24.116.153.115:8333 +24.134.6.165:8333 +24.155.218.13:8333 +24.160.137.173:8333 +24.177.106.85:8333 +24.184.0.146:8333 +24.194.222.116:8333 +24.205.215.192:8333 +27.124.108.19:8333 +31.14.40.64:8333 +31.47.202.112:8333 +31.165.115.7:8333 +34.65.45.157:8333 +34.78.48.104:8333 +34.80.134.68:8333 +34.101.132.198:8333 +34.227.68.216:8333 +35.137.212.22:8333 +35.231.190.134:8333 +37.1.217.35:8333 +37.15.62.32:8333 +37.143.118.174:8333 +37.200.59.67:8333 +37.205.9.165:8333 +38.23.180.228:8333 +38.65.119.26:8333 +38.141.134.140:8333 +39.109.122.127:8444 +41.79.70.146:8333 +41.193.122.191:8333 +43.225.62.107:8333 +45.35.73.152:8333 +45.43.97.103:8333 +45.63.10.52:20008 +45.84.153.40:8333 +45.95.64.225:8333 +45.129.180.214:8333 +45.154.255.162:8333 +45.226.80.102:8333 +46.6.10.230:8333 +46.23.87.218:8333 +46.32.50.98:8333 +46.47.84.85:8333 +46.48.126.58:8333 +46.146.248.89:8333 +46.165.221.209:9333 +46.166.142.2:8333 +46.166.162.45:20001 +46.173.50.58:8333 +46.175.178.3:8333 +46.188.30.118:8333 +46.219.120.59:3673 +46.229.238.187:8333 +47.93.230.171:8333 +47.100.162.210:18332 +47.144.106.249:8333 +47.188.70.205:8333 +47.227.226.242:8333 +50.2.13.164:8333 +50.5.46.195:8333 +50.45.128.28:8333 +51.148.153.60:8333 +51.154.62.103:8333 +51.154.131.18:8333 +51.158.150.155:8333 +51.159.2.218:8333 +54.198.19.34:8333 +58.105.168.41:8333 +58.158.0.86:8333 +60.251.129.61:8336 +61.239.91.250:8333 +62.28.190.194:8333 +62.152.58.16:9421 +62.171.129.32:8333 +62.251.54.163:8333 +63.247.147.166:8333 +64.33.68.176:8333 +64.156.192.61:8333 +64.187.175.226:8333 +64.233.245.39:8333 +64.237.82.149:8333 +65.101.247.26:8333 +66.29.129.218:8333 +66.49.204.11:8333 +66.58.243.215:8333 +66.85.234.129:8333 +66.130.120.52:8333 +67.10.121.145:8333 +67.210.228.203:8333 +67.213.87.21:8333 +68.181.4.12:8333 +69.7.124.146:8333 +69.8.175.201:8333 +69.59.18.22:8333 +69.119.193.9:8333 +69.130.201.27:8333 +69.131.101.176:8333 +70.15.194.32:8333 +70.64.27.12:8333 +72.29.170.151:8333 +72.74.34.99:8333 +72.133.177.119:8333 +73.166.84.222:8333 +74.67.240.204:8333 +74.91.115.229:8333 +74.118.137.119:8333 +74.213.251.203:8333 +74.220.255.190:8333 +76.11.60.155:8333 +76.66.144.127:8333 +77.70.16.245:8333 +77.85.204.149:8333 +77.105.87.97:8333 +77.120.113.69:8433 +77.120.113.71:8433 +77.120.122.116:8433 +77.120.122.118:8433 +77.162.190.90:8333 +77.167.245.239:55544 +77.232.41.189:8333 +78.20.227.249:8333 +78.21.167.8:8333 +78.27.139.13:8333 +78.43.208.25:8333 +78.63.28.146:8333 +78.72.228.239:8333 +78.108.102.8:8333 +78.129.0.39:8333 +78.129.169.69:8333 +79.77.182.180:8333 +79.77.182.183:8333 +79.107.178.59:8333 +80.55.225.158:8333 +80.64.211.102:8333 +80.64.211.103:8333 +80.71.57.50:8333 +80.81.3.27:8333 +80.82.55.43:8333 +80.88.172.227:64264 +80.89.203.172:8001 +80.93.213.246:8333 +80.147.82.165:8333 +80.229.28.60:8333 +80.247.233.40:8333 +80.255.8.93:8333 +81.7.17.202:8333 +81.10.241.165:8333 +81.21.86.157:8333 +81.171.22.143:8333 +81.237.206.224:8343 +82.69.23.195:8333 +82.96.96.40:8333 +82.116.50.101:8333 +82.136.99.122:8333 +82.149.97.25:17567 +82.154.24.209:8333 +82.165.241.50:8333 +82.197.218.253:8333 +82.202.68.231:8333 +83.137.41.10:8333 +83.208.6.211:8333 +83.217.8.31:44420 +83.220.110.48:8333 +83.222.138.85:8333 +83.243.191.199:8333 +84.22.139.57:8333 +84.27.155.17:8333 +84.75.28.247:8333 +84.112.60.16:8333 +84.211.7.56:8333 +84.237.7.249:8333 +85.23.51.177:8333 +85.24.145.198:8333 +85.184.138.108:8333 +85.194.238.134:8333 +85.195.54.110:8333 +85.208.71.36:8333 +85.208.71.39:8333 +85.214.136.45:8333 +85.214.161.252:8333 +85.227.245.128:8333 +86.18.34.243:8333 +86.20.50.170:8333 +86.49.105.90:8333 +86.76.7.132:8333 +86.100.26.188:8333 +86.106.143.143:55373 +86.120.58.66:8333 +86.133.251.239:8901 +86.149.8.23:8901 +87.78.197.234:8333 +87.120.8.5:20008 +87.121.37.156:8333 +88.82.181.44:8333 +88.87.93.52:1691 +88.98.235.134:8333 +88.136.187.214:8333 +88.147.244.250:8333 +88.148.153.148:8333 +88.212.45.166:8333 +88.212.55.138:8333 +89.38.96.153:9273 +89.47.161.135:8333 +89.88.62.190:8333 +89.158.32.44:8333 +89.163.145.240:8333 +89.163.249.234:3673 +89.176.196.80:8333 +89.216.21.96:8333 +90.84.227.255:8333 +90.146.130.214:8333 +90.250.9.1:8333 +91.93.194.154:8333 +91.106.188.229:8333 +91.126.40.109:8333 +91.137.127.123:8333 +91.147.232.98:8333 +91.152.123.18:8333 +91.178.17.120:8333 +91.204.99.178:8333 +91.223.175.14:8333 +92.42.110.242:8333 +92.53.90.84:8333 +92.221.155.228:8333 +93.57.81.162:8333 +93.95.88.13:8333 +93.103.13.1:8333 +93.123.180.164:8333 +93.190.117.26:8333 +94.105.125.240:8333 +94.110.23.215:8333 +94.154.159.99:8333 +94.189.161.119:8333 +94.203.255.70:8333 +94.232.173.93:8333 +95.79.122.99:8333 +95.80.1.110:8333 +95.83.73.31:8333 +95.110.133.223:8333 +95.110.234.93:8333 +95.164.65.194:8333 +95.165.8.182:8333 +95.174.219.101:8333 +95.191.130.100:8333 +95.214.53.154:8333 +95.215.205.180:8333 +96.43.130.234:8333 +98.25.201.31:8333 +98.128.247.182:8333 +98.171.21.129:8333 +99.147.135.161:8333 +101.100.163.118:8327 +102.132.245.16:8333 +102.182.204.96:8333 +102.182.235.245:8333 +103.14.245.250:8333 +103.47.192.15:8333 +103.84.84.250:8335 +103.99.168.130:8333 +103.99.168.140:8333 +103.198.192.14:20008 +103.232.104.227:8333 +104.143.2.195:8333 +104.172.235.227:8333 +104.238.220.199:8333 +107.11.115.68:8333 +107.173.166.43:8333 +108.4.212.83:8333 +109.136.73.97:8333 +109.173.98.23:8333 +109.190.68.116:8333 +109.235.246.60:8333 +109.248.206.13:8333 +110.12.64.96:8333 +111.90.140.46:8333 +111.90.159.184:50001 +113.107.201.131:8333 +115.47.141.250:8885 +116.58.171.67:8333 +116.87.57.218:8333 +116.202.161.56:8333 +117.51.159.130:8333 +118.103.126.140:28333 +121.45.190.210:8333 +121.99.193.25:8333 +122.112.148.153:8339 +122.148.135.234:8333 +128.0.190.26:8333 +128.65.194.136:8333 +129.126.172.115:8333 +129.226.125.10:8333 +131.188.40.191:8333 +134.195.185.52:8333 +135.180.44.61:8333 +136.52.114.123:8333 +136.56.170.96:8333 +137.116.213.143:8333 +137.226.34.46:8333 +138.43.233.57:8333 +139.130.41.82:8333 +140.190.12.129:8333 +142.4.105.77:8333 +142.54.181.218:8333 +143.177.231.247:8333 +143.178.64.10:8333 +144.34.161.65:18333 +146.4.124.134:8333 +146.83.56.69:8333 +146.90.193.68:8333 +146.196.55.156:28833 +148.66.50.50:8335 +148.251.1.20:8343 +151.48.95.212:8333 +151.252.193.245:8333 +152.44.137.83:8333 +152.115.191.196:8333 +154.221.31.86:8333 +156.17.103.2:8088 +157.138.20.22:8333 +158.58.188.37:8333 +158.140.209.79:8333 +159.89.230.128:8333 +159.246.25.52:8333 +160.20.59.250:8433 +162.0.234.190:8333 +162.62.26.218:8333 +162.250.188.194:8333 +162.251.70.82:8333 +163.158.206.255:8333 +164.68.105.105:8333 +165.228.174.117:8333 +166.62.82.103:32771 +166.70.49.26:8333 +166.78.241.9:8333 +166.78.241.25:8333 +167.71.73.244:8333 +167.179.147.155:8333 +168.91.238.8:8333 +172.105.21.216:8333 +172.117.105.95:8333 +173.23.103.30:8000 +173.205.92.151:54805 +173.205.92.154:54805 +173.205.92.157:54805 +173.208.152.218:8333 +173.241.227.243:8333 +174.3.4.232:8333 +174.17.11.22:8333 +174.88.241.167:8333 +174.114.102.41:8333 +174.114.250.86:8333 +174.142.191.136:8333 +175.39.72.87:8333 +176.12.16.135:8333 +176.37.23.30:8333 +176.62.179.221:8333 +176.74.136.237:8333 +176.99.6.226:8333 +176.212.185.153:8333 +177.81.236.117:8333 +178.19.106.26:8333 +178.21.118.178:8333 +178.33.232.69:8333 +178.79.84.139:8333 +178.124.162.209:8333 +178.132.2.246:8333 +178.150.96.46:8333 +178.162.212.44:8333 +178.193.226.120:8333 +178.236.137.63:8333 +180.150.46.187:8333 +181.164.210.228:8530 +183.110.220.210:30301 +184.95.58.166:8336 +184.164.147.82:41333 +184.171.208.109:8333 +185.17.143.220:8333 +185.21.217.49:8333 +185.25.48.184:8333 +185.28.96.16:8333 +185.31.136.246:8333 +185.64.116.15:8333 +185.68.249.91:8333 +185.108.247.190:8333 +185.141.60.36:8333 +185.148.3.227:8333 +185.148.145.74:8333 +185.159.20.143:8333 +185.167.113.59:8333 +185.185.26.141:8111 +185.189.132.178:57780 +185.204.197.112:8333 +185.209.70.17:8333 +185.220.156.193:8333 +185.238.129.113:8333 +185.239.221.5:8333 +185.244.217.39:8333 +185.254.97.164:8333 +186.33.167.11:8333 +188.32.14.31:8334 +188.42.40.234:18333 +188.134.8.36:8333 +188.138.88.14:8333 +188.156.110.239:8333 +188.165.244.143:8333 +188.213.68.38:8333 +188.214.129.65:20012 +188.242.15.74:8333 +188.244.4.78:8333 +189.39.6.82:8333 +189.207.46.32:8333 +189.212.121.74:8333 +192.3.11.20:8333 +192.65.170.15:8333 +192.146.137.44:8333 +192.182.157.119:8333 +192.187.109.141:8333 +192.227.80.83:8333 +193.10.203.23:8334 +193.32.127.160:58477 +193.32.127.162:58477 +193.58.196.212:8333 +193.106.29.106:8333 +193.138.154.43:8333 +193.178.170.232:8333 +193.196.37.62:8333 +193.222.130.14:8333 +193.234.50.227:8333 +194.14.246.205:8333 +194.135.135.69:8333 +194.147.113.201:8333 +194.165.30.20:8333 +194.219.62.23:8333 +195.56.63.4:8333 +195.134.183.188:8333 +195.208.103.30:8444 +195.208.103.31:8444 +198.1.231.6:8333 +198.12.14.136:8333 +198.84.237.70:8333 +198.178.120.5:8112 +199.48.92.184:8333 +199.68.199.19:8333 +199.182.184.204:8333 +199.189.242.141:8333 +199.247.7.208:8333 +200.122.181.37:8333 +201.191.6.103:8333 +202.107.219.130:8333 +202.108.211.135:8333 +203.94.33.112:8333 +203.130.48.117:8885 +203.132.94.196:8333 +203.162.13.181:8332 +204.191.201.43:8333 +204.229.10.90:8333 +205.178.41.124:8333 +206.55.178.157:8333 +206.126.203.8:8333 +206.174.115.96:8333 +206.223.153.52:8333 +207.188.159.25:8333 +207.229.46.80:8333 +209.58.145.157:8333 +209.126.81.147:8333 +209.145.63.150:8333 +209.209.10.30:8333 +209.237.127.227:8333 +212.99.226.36:9020 +212.185.86.84:8333 +212.227.211.87:8333 +213.5.36.58:8333 +213.89.236.219:8333 +213.93.145.183:8333 +213.214.66.182:8333 +216.41.249.178:8333 +216.146.251.8:8333 +216.249.70.22:8333 +217.11.240.4:8333 +217.15.178.7:8333 +217.24.233.116:8333 +217.64.148.98:51401 +217.113.121.169:8333 +217.170.124.170:8333 +220.132.135.54:8333 +220.221.58.25:8333 +220.233.178.199:8333 +221.219.97.105:2001 +[2001:1608:1b:f9::1]:26491 +[2001:1620:510::2]:8333 +[2001:1bc0:c1::2000]:8333 +[2001:470:1f0a:89a::2]:8333 +[2001:470:de5a::ec]:9333 +[2001:4b98:dc0:45:216:3eff:fea2:95cd]:8333 +[2001:4dd0:3564:0:fd76:c1d3:1854:5bd9]:8333 +[2001:4de8:b1b2:1:0:dead:beef:7]:8333 +[2001:638:a000:4140::ffff:191]:8333 +[2001:648:2800:131:4b1f:f6fc:20f7:f99f]:8333 +[2001:678:cc8::1:10:88]:20008 +[2001:67c:26b4:ff00::44]:8333 +[2001:67c:2db8:13::92]:8333 +[2001:7c0:2310:0:f816:3eff:fe6c:4f58]:8333 +[2001:818:ea1b:7600:f053:aade:f47b:b701]:8333 +[2001:8f1:1404:3700:8e49:715a:2e09:b634]:9444 +[2001:985:55a0:1::2]:8333 +[2001:999:270:2c2c:c8b:3a20:3f2f:318f]:8333 +[2001:b07:ac9:442b:79d6:bbbe:b37c:a783]:8333 +[2002:2f5b:a5f9::2f5b:a5f9]:8885 +[2002:b6ff:3dca::b6ff:3dca]:28364 +[2400:2410:cea2:d00:41bc:c9ea:861b:51ee]:8333 +[2400:3b00:20:c:bacb:29ff:feab:8886]:8333 +[2401:d002:3902:700:d72c:5e22:4e95:389d]:8333 +[2403:6200:88a0:fb17:f5f2:d8b5:b7ba:f4d3]:8333 +[2405:9800:b910:5f8e:1830:f630:2cc6:88fb]:8333 +[2405:9800:b970:c64c:109f:74e7:ae5f:87c7]:8333 +[2405:aa00:2::40]:8333 +[2407:8800:bc61:2202:d63d:7eff:fe6c:dc36]:8333 +[2408:8248:7004:f831::83c]:8333 +[2409:10:ca20:1df0:224:e8ff:fe1f:60d9]:8333 +[240b:11:43a1:bd00:e589:f8a7:49b:3b86]:8333 +[240d:1a:791:3400:d65d:64ff:fe28:927e]:8333 +[240d:1a:791:3400:d681:d7ff:fef6:a21e]:10050 +[2600:1700:5b2b:5f::8040]:8333 +[2600:2104:1003:c5ab:dc5e:90ff:fe18:1d08]:8333 +[2600:3c00:e002:2e32::1:14]:8333 +[2600:8805:2400:14e:12dd:b1ff:fef2:3013]:8333 +[2602:ffb8::208:72:57:200]:8333 +[2603:301f:1ebf:e000:e23f:49ff:fee7:7431]:8333 +[2603:6081:1800:6600:16dd:a9ff:feee:b2f3]:8333 +[2604:1380:1000:7400::1]:8333 +[2604:4500::2e06]:8112 +[2604:5500:c134:4000:7285:c2ff:fe4a:e143]:32797 +[2604:5500:c134:4000::3fc]:32797 +[2604:7c00:120:4b::eb24]:8333 +[2605:6400:30:f220::]:8333 +[2605:6f80:0:7:fc1b:ccff:fe8a:d822]:8333 +[2605:ae00:203::203]:8333 +[2605:c000:2a0a:1::102]:8333 +[2605:f700:c0:827:225:90ff:fee3:34a6]:8333 +[2607:9280:b:73b:250:56ff:fe14:25b5]:8333 +[2607:f2f8:ad40:bc1::1]:8333 +[2607:fa18:3a01::20]:8333 +[2620:11c:5001:1118:d267:e5ff:fee9:e673]:8333 +[2620:11c:5001:2199:d267:e5ff:fee9:e673]:8333 +[2620:6:2003:105:2d8:61ff:fe0f:853]:8333 +[2620:6e:a000:1:42:42:42:42]:8333 +[2803:cf00:af8:f200:b89e:cf34:92c7:2d26]:8333 +[2804:14c:65d1:402c:bc53:bf5d:68a:2136]:8333 +[2804:7f1:e783:d401:661c:67ff:feba:5547]:8333 +[2804:d57:5537:4800:21e:67ff:fea8:d798]:8333 +[2804:d57:5537:4800:3615:9eff:fe23:d610]:8333 +[2806:2f0:2080:62a:86f:1a01:c44f:1794]:8333 +[2a00:1028:8382:bf22:5f7f:b78f:2737:7739]:8333 +[2a00:12e0:101:99:20c:29ff:fe29:d03f]:8333 +[2a00:1328:e101:c00::163]:8333 +[2a00:1630:10:1003:0:b19:b00b:babe]:8333 +[2a00:1768:2001:27::ef6a]:8333 +[2a00:1828:a004:2::666]:8333 +[2a00:1838:2a:1400:92e2:baff:fe4a:c416]:8333 +[2a00:1c10:2:709::217]:22220 +[2a00:1f40:5001:108:5d17:7703:b0f5:4133]:8333 +[2a00:6020:15dd:ee00:c8c2:2c77:1749:35db]:8333 +[2a00:6020:b482:9200:491a:358c:d8f7:1da]:8333 +[2a00:7145:c1:1:ae29:727:2b87:f64]:5141 +[2a00:8a60:e012:a00::21]:8333 +[2a00:a040:100:f3:45a5:ac0:fea3:71e1]:8333 +[2a01:488:2000:9801::d]:8333 +[2a01:490:16:301::2]:8333 +[2a01:5200:6c:6162:7a61:746b:6f2e:736b]:8333 +[2a01:6380:fffe:73:4e3:b3cc:a871:36d1]:8333 +[2a01:7a0:2:137c::3]:8333 +[2a01:7c8:aac9:c9:5054:ff:fedf:ff95]:8333 +[2a01:7c8:d001:1c1:5054:ff:feee:3e1a]:8333 +[2a01:8740:1:ffc5::8c6a]:8333 +[2a01:cb00:d3d:7700:227:eff:fe28:c565]:8333 +[2a01:d0:0:1c::253]:8333 +[2a01:d0:bef2::12]:8333 +[2a01:e0a:9fb:b0e0:54f8:1901:6e83:62c1]:8333 +[2a01:e0a:aa7:c8c0:9679:affa:b6e5:efc7]:8333 +[2a02:13b8:f000:101::a]:8333 +[2a02:168:6328:0:2a8:2cff:fe68:e32c]:8333 +[2a02:2780:9000:70::7]:8333 +[2a02:2e02:3900:5400:a099:e1ff:feb6:d0e]:8333 +[2a02:390:9000:0:aaa1:59ff:fe43:b57b]:8333 +[2a02:58:97:7d20::60]:8333 +[2a02:6d40:305e:601:dea6:32ff:fe44:4b25]:8333 +[2a02:7a01::91:228:45:130]:8333 +[2a02:7aa0:1619::adc:8de0]:8333 +[2a02:7b40:3e4d:998d::1]:8333 +[2a02:7b40:592f:a187::1]:8333 +[2a02:8388:e5c6:d380:201:2eff:fe82:b3cc]:8333 +[2a02:9a0:102::110]:8333 +[2a02:a311:8143:8c00::4]:8353 +[2a02:af8:fab0:808:85:234:145:132]:8333 +[2a02:e00:fff0:506::1]:8444 +[2a02:e00:fff0:506::a]:8444 +[2a02:e98:20:1504::1]:8333 +[2a03:4000:47:f1::1]:8333 +[2a03:6000:870:0:46:23:87:218]:8333 +[2a03:7380:3015:524:afc5:d3bc:7c66:8f94]:8333 +[2a03:ec0:0:928:8c00:93ff:fe84:a007]:8333 +[2a03:ec0:0:928::701]:8333 +[2a04:2180:0:2::aa]:8333 +[2a04:52c0:101:29e::]:8333 +[2a04:52c0:103:c455::1]:8333 +[2a04:bc40:1dc3:8d::2:1001]:8333 +[2a05:1500:702:0:1c00:40ff:fe00:c]:8333 +[2a06:dd00:10:3:225:90ff:fe32:64cc]:8333 +[2a06:dd00:1:22:225:90ff:fe0e:bd48]:8333 +[2a07:6b47:100:464::9357:ffda]:8333 +[2a07:a880:4601:1062:b4b4:bd2a:39d4:7acf]:51401 +[2a07:abc4::1:946]:8333 +[2a07:abc4::89:234:180:194]:8333 +[2a09:2681:102::210]:8333 +[2a0a:c801:1:7::183]:8333 +[2a0b:f300:2:6::2]:8333 +[2a0c:59c0:18::a20e]:57658 +[2a0d:5600:24:a8e::a91e]:55373 +[2a0d:eb00:8005:1::13]:8333 +[2a10:4740:45:1:a013:d1ff:fe85:36e3]:8333 +[2a10:8b40:1::103]:8335 +[2c0f:f8f0:da51:0:70c3:eea9:9717:9579]:8333 + +# manually added 2021-03 for minimal torv3 bootstrap support +2g5qfdkn2vvcbqhzcyvyiitg4ceukybxklraxjnu7atlhd22gdwywaid.onion:8333 +2jmtxvyup3ijr7u6uvu7ijtnojx4g5wodvaedivbv74w4vzntxbrhvad.onion:8333 +37m62wn7dz3uqpathpc4qfmgrbupachj52nt3jbtbjugpbu54kbud7yd.onion:8333 +5g72ppm3krkorsfopcm2bi7wlv4ohhs4u4mlseymasn7g7zhdcyjpfid.onion:8333 +7cgwjuwi5ehvcay4tazy7ya6463bndjk6xzrttw5t3xbpq4p22q6fyid.onion:8333 +7pyrpvqdhmayxggpcyqn5l3m5vqkw3qubnmgwlpya2mdo6x7pih7r7id.onion:8333 +b64xcbleqmwgq2u46bh4hegnlrzzvxntyzbmucn3zt7cssm7y4ubv3id.onion:8333 +ejxefzf5fpst4mg2rib7grksvscl7p6fvjp6agzgfc2yglxnjtxc3aid.onion:8333 +fjdyxicpm4o42xmedlwl3uvk5gmqdfs5j37wir52327vncjzvtpfv7yd.onion:8333 +fpz6r5ppsakkwypjcglz6gcnwt7ytfhxskkfhzu62tnylcknh3eq6pad.onion:8333 +fzhn4uoxfbfss7h7d6ffbn266ca432ekbbzvqtsdd55ylgxn4jucm5qd.onion:8333 +gxo5anvfnffnftfy5frkgvplq3rpga2ie3tcblo2vl754fvnhgorn5yd.onion:8333 +ifdu5qvbofrt4ekui2iyb3kbcyzcsglazhx2hn4wfskkrx2v24qxriid.onion:8333 +itz3oxsihs62muvknc237xabl5f6w6rfznfhbpayrslv2j2ubels47yd.onion:8333 +lrjh6fywjqttmlifuemq3puhvmshxzzyhoqx7uoufali57eypuenzzid.onion:8333 +m7cbpjolo662uel7rpaid46as2otcj44vvwg3gccodnvaeuwbm3anbyd.onion:8333 +opnyfyeiibe5qo5a3wbxzbb4xdiagc32bbce46owmertdknta5mi7uyd.onion:8333 +owjsdxmzla6d7lrwkbmetywqym5cyswpihciesfl5qdv2vrmwsgy4uqd.onion:8333 +q7kgmd7n7h27ds4fg7wocgniuqb3oe2zxp4nfe4skd5da6wyipibqzqd.onion:8333 +rp7k2go3s5lyj3fnj6zn62ktarlrsft2ohlsxkyd7v3e3idqyptvread.onion:8333 +sys54sv4xv3hn3sdiv3oadmzqpgyhd4u4xphv4xqk64ckvaxzm57a7yd.onion:8333 +tddeij4qigtjr6jfnrmq6btnirmq5msgwcsdpcdjr7atftm7cxlqztid.onion:8333 +vi5bnbxkleeqi6hfccjochnn65lcxlfqs4uwgmhudph554zibiusqnad.onion:8333 +xqt25cobm5zqucac3634zfght72he6u3eagfyej5ellbhcdgos7t2had.onion:8333 + +# manually added 2021-08 for minimal i2p bootstrap support +a5qsnv3maw77mlmmzlcglu6twje6ttctd3fhpbfwcbpmewx6fczq.b32.i2p:0 +bitcornrd36coazsbzsz4pdebyzvaplmsalq4kpoljmn6cg6x5zq.b32.i2p:0 +c4gfnttsuwqomiygupdqqqyy5y5emnk5c73hrfvatri67prd7vyq.b32.i2p:0 +dhtq2p76tyhi442aidb3vd2bv7yxxjuddpb2jydnnrl2ons5bhha.b32.i2p:0 +h3r6bkn46qxftwja53pxiykntegfyfjqtnzbm6iv6r5mungmqgmq.b32.i2p:0 +hnbbyjpxx54623l555sta7pocy3se4sdgmuebi5k6reesz5rjp6q.b32.i2p:0 +jz3s4eurm5vzjresf4mwo7oni4bk36daolwxh4iqtewakylgkxmq.b32.i2p:0 +kokkmpquqlkptu5hkmzqlttsmtwxicldr4so7wqsufk6bwf32nma.b32.i2p:0 +sedndhv5vpcgdmykyi5st4yqhdxl3hpdtglta4do435wupahhx6q.b32.i2p:0 +wwbw7nqr3ahkqv62cuqfwgtneekvvpnuc4i4f6yo7tpoqjswvcwa.b32.i2p:0 +zsxwyo6qcn3chqzwxnseusqgsnuw3maqnztkiypyfxtya4snkoka.b32.i2p:0 + +# manually added 2022-01 for minimal cjdns bootstrap support +[fc32:17ea:e415:c3bf:9808:149d:b5a2:c9aa]:8333 +[fcc7:be49:ccd1:dc91:3125:f0da:457d:8ce]:8333 diff --git a/contrib/seeds/nodes_main_manual.txt b/contrib/seeds/nodes_main_manual.txt new file mode 100644 index 0000000000..a6e0b8763a --- /dev/null +++ b/contrib/seeds/nodes_main_manual.txt @@ -0,0 +1,43 @@ + +# manually added 2021-03 for minimal torv3 bootstrap support +2g5qfdkn2vvcbqhzcyvyiitg4ceukybxklraxjnu7atlhd22gdwywaid.onion:8333 +2jmtxvyup3ijr7u6uvu7ijtnojx4g5wodvaedivbv74w4vzntxbrhvad.onion:8333 +37m62wn7dz3uqpathpc4qfmgrbupachj52nt3jbtbjugpbu54kbud7yd.onion:8333 +5g72ppm3krkorsfopcm2bi7wlv4ohhs4u4mlseymasn7g7zhdcyjpfid.onion:8333 +7cgwjuwi5ehvcay4tazy7ya6463bndjk6xzrttw5t3xbpq4p22q6fyid.onion:8333 +7pyrpvqdhmayxggpcyqn5l3m5vqkw3qubnmgwlpya2mdo6x7pih7r7id.onion:8333 +b64xcbleqmwgq2u46bh4hegnlrzzvxntyzbmucn3zt7cssm7y4ubv3id.onion:8333 +ejxefzf5fpst4mg2rib7grksvscl7p6fvjp6agzgfc2yglxnjtxc3aid.onion:8333 +fjdyxicpm4o42xmedlwl3uvk5gmqdfs5j37wir52327vncjzvtpfv7yd.onion:8333 +fpz6r5ppsakkwypjcglz6gcnwt7ytfhxskkfhzu62tnylcknh3eq6pad.onion:8333 +fzhn4uoxfbfss7h7d6ffbn266ca432ekbbzvqtsdd55ylgxn4jucm5qd.onion:8333 +gxo5anvfnffnftfy5frkgvplq3rpga2ie3tcblo2vl754fvnhgorn5yd.onion:8333 +ifdu5qvbofrt4ekui2iyb3kbcyzcsglazhx2hn4wfskkrx2v24qxriid.onion:8333 +itz3oxsihs62muvknc237xabl5f6w6rfznfhbpayrslv2j2ubels47yd.onion:8333 +lrjh6fywjqttmlifuemq3puhvmshxzzyhoqx7uoufali57eypuenzzid.onion:8333 +m7cbpjolo662uel7rpaid46as2otcj44vvwg3gccodnvaeuwbm3anbyd.onion:8333 +opnyfyeiibe5qo5a3wbxzbb4xdiagc32bbce46owmertdknta5mi7uyd.onion:8333 +owjsdxmzla6d7lrwkbmetywqym5cyswpihciesfl5qdv2vrmwsgy4uqd.onion:8333 +q7kgmd7n7h27ds4fg7wocgniuqb3oe2zxp4nfe4skd5da6wyipibqzqd.onion:8333 +rp7k2go3s5lyj3fnj6zn62ktarlrsft2ohlsxkyd7v3e3idqyptvread.onion:8333 +sys54sv4xv3hn3sdiv3oadmzqpgyhd4u4xphv4xqk64ckvaxzm57a7yd.onion:8333 +tddeij4qigtjr6jfnrmq6btnirmq5msgwcsdpcdjr7atftm7cxlqztid.onion:8333 +vi5bnbxkleeqi6hfccjochnn65lcxlfqs4uwgmhudph554zibiusqnad.onion:8333 +xqt25cobm5zqucac3634zfght72he6u3eagfyej5ellbhcdgos7t2had.onion:8333 + +# manually added 2021-08 for minimal i2p bootstrap support +a5qsnv3maw77mlmmzlcglu6twje6ttctd3fhpbfwcbpmewx6fczq.b32.i2p:0 +bitcornrd36coazsbzsz4pdebyzvaplmsalq4kpoljmn6cg6x5zq.b32.i2p:0 +c4gfnttsuwqomiygupdqqqyy5y5emnk5c73hrfvatri67prd7vyq.b32.i2p:0 +dhtq2p76tyhi442aidb3vd2bv7yxxjuddpb2jydnnrl2ons5bhha.b32.i2p:0 +h3r6bkn46qxftwja53pxiykntegfyfjqtnzbm6iv6r5mungmqgmq.b32.i2p:0 +hnbbyjpxx54623l555sta7pocy3se4sdgmuebi5k6reesz5rjp6q.b32.i2p:0 +jz3s4eurm5vzjresf4mwo7oni4bk36daolwxh4iqtewakylgkxmq.b32.i2p:0 +kokkmpquqlkptu5hkmzqlttsmtwxicldr4so7wqsufk6bwf32nma.b32.i2p:0 +sedndhv5vpcgdmykyi5st4yqhdxl3hpdtglta4do435wupahhx6q.b32.i2p:0 +wwbw7nqr3ahkqv62cuqfwgtneekvvpnuc4i4f6yo7tpoqjswvcwa.b32.i2p:0 +zsxwyo6qcn3chqzwxnseusqgsnuw3maqnztkiypyfxtya4snkoka.b32.i2p:0 + +# manually added 2022-01 for minimal cjdns bootstrap support +[fc32:17ea:e415:c3bf:9808:149d:b5a2:c9aa]:8333 +[fcc7:be49:ccd1:dc91:3125:f0da:457d:8ce]:8333 diff --git a/contrib/seeds/nodes_test.txt b/contrib/seeds/nodes_test.txt new file mode 100644 index 0000000000..118bec280e --- /dev/null +++ b/contrib/seeds/nodes_test.txt @@ -0,0 +1,16 @@ +# List of fixed seed nodes for testnet + +# Onion nodes +35k2va6vyw4oo5ly2quvcszgdqr56kcnfgcqpnpcffut4jn3mhhwgbid.onion:18333 +blo2esfvk2rr7sr4jspmu3vt2vpgr5rigflsj645fnku7v4qmljurtid.onion:18333 +fuckcswupr5rmlvx2kqqrrosxvjyong4hatmuvxsvtcwe4dsh5rus7qd.onion:18333 +gblylyacjlitd2ywdmo2qqylwtdky7kgeqfvlhiw4zdag4x62tx54hyd.onion:18333 +gzwpduv33l7yze3bcdzj3inebiyjwddjnwvnjhh5wvnv4me76mjt2kad.onion:18333 +h3rphzofxzq52tb63mg5f6kc4my3fkcrgh3m5qryeatts43iljbawiid.onion:18333 +kf4qlhek34b3kgyxyodlmvgm4bxfrjsbjtgayyaiuyhr2eoyfgtm3bad.onion:18333 +mc7k47ndjvvhcgs54wmjzxvate4rtuybbjoryikdssjhcxlx27psbyqd.onion:18333 +mrhiniicugfo7mgrwv3wtolk3tptlcw2uq7ih6sq43fa4k4zbilut3yd.onion:18333 +uiudyws3qizgmepfoh7wwjmsoxoxut4qrmotjjhrn247xnjopr7sfcid.onion:18333 +zc2wvoqcezcrf64trji6jmhtss34a5ds5ntzdhqegzvex3ynrd7nxcad.onion:18333 +zd5m3dgdn46naj36pxvvcalfw2paecle6sdxq64ptwxtxjomkywpklqd.onion:18333 + diff --git a/contrib/shell/git-utils.bash b/contrib/shell/git-utils.bash new file mode 100644 index 0000000000..37bac1f38d --- /dev/null +++ b/contrib/shell/git-utils.bash @@ -0,0 +1,14 @@ +#!/usr/bin/env bash + +git_root() { + git rev-parse --show-toplevel 2> /dev/null +} + +git_head_version() { + local recent_tag + if recent_tag="$(git describe --exact-match HEAD 2> /dev/null)"; then + echo "${recent_tag#v}" + else + git rev-parse --short=12 HEAD + fi +} diff --git a/contrib/shell/realpath.bash b/contrib/shell/realpath.bash new file mode 100644 index 0000000000..389b77b562 --- /dev/null +++ b/contrib/shell/realpath.bash @@ -0,0 +1,71 @@ +#!/usr/bin/env bash + +# Based on realpath.sh written by Michael Kropat +# Found at: https://github.com/mkropat/sh-realpath/blob/65512368b8155b176b67122aa395ac580d9acc5b/realpath.sh + +bash_realpath() { + canonicalize_path "$(resolve_symlinks "$1")" +} + +resolve_symlinks() { + _resolve_symlinks "$1" +} + +_resolve_symlinks() { + _assert_no_path_cycles "$@" || return + + local dir_context path + if path=$(readlink -- "$1"); then + dir_context=$(dirname -- "$1") + _resolve_symlinks "$(_prepend_dir_context_if_necessary "$dir_context" "$path")" "$@" + else + printf '%s\n' "$1" + fi +} + +_prepend_dir_context_if_necessary() { + if [ "$1" = . ]; then + printf '%s\n' "$2" + else + _prepend_path_if_relative "$1" "$2" + fi +} + +_prepend_path_if_relative() { + case "$2" in + /* ) printf '%s\n' "$2" ;; + * ) printf '%s\n' "$1/$2" ;; + esac +} + +_assert_no_path_cycles() { + local target path + + target=$1 + shift + + for path in "$@"; do + if [ "$path" = "$target" ]; then + return 1 + fi + done +} + +canonicalize_path() { + if [ -d "$1" ]; then + _canonicalize_dir_path "$1" + else + _canonicalize_file_path "$1" + fi +} + +_canonicalize_dir_path() { + (cd "$1" 2>/dev/null && pwd -P) +} + +_canonicalize_file_path() { + local dir file + dir=$(dirname -- "$1") + file=$(basename -- "$1") + (cd "$dir" 2>/dev/null && printf '%s/%s\n' "$(pwd -P)" "$file") +} diff --git a/contrib/signet/README.md b/contrib/signet/README.md new file mode 100644 index 0000000000..706b296c54 --- /dev/null +++ b/contrib/signet/README.md @@ -0,0 +1,83 @@ +Contents +======== +This directory contains tools related to Signet, both for running a Signet yourself and for using one. + +getcoins.py +=========== + +A script to call a faucet to get Signet coins. + +Syntax: `getcoins.py [-h|--help] [-c|--cmd=<bitcoin-cli path>] [-f|--faucet=<faucet URL>] [-a|--addr=<signet bech32 address>] [-p|--password=<faucet password>] [--] [<bitcoin-cli args>]` + +* `--cmd` lets you customize the bitcoin-cli path. By default it will look for it in the PATH +* `--faucet` lets you specify which faucet to use; the faucet is assumed to be compatible with https://github.com/kallewoof/bitcoin-faucet +* `--addr` lets you specify a Signet address; by default, the address must be a bech32 address. This and `--cmd` above complement each other (i.e. you do not need `bitcoin-cli` if you use `--addr`) +* `--password` lets you specify a faucet password; this is handy if you are in a classroom and set up your own faucet for your students; (above faucet does not limit by IP when password is enabled) + +If using the default network, invoking the script with no arguments should be sufficient under normal +circumstances, but if multiple people are behind the same IP address, the faucet will by default only +accept one claim per day. See `--password` above. + +miner +===== + +You will first need to pick a difficulty target. Since signet chains are primarily protected by a signature rather than proof of work, there is no need to spend as much energy as possible mining, however you may wish to choose to spend more time than the absolute minimum. The calibrate subcommand can be used to pick a target appropriate for your hardware, eg: + + cd src/ + MINER="../contrib/signet/miner" + GRIND="./bitcoin-util grind" + $MINER calibrate --grind-cmd="$GRIND" + nbits=1e00f403 for 25s average mining time + +It defaults to estimating an nbits value resulting in 25s average time to find a block, but the --seconds parameter can be used to pick a different target, or the --nbits parameter can be used to estimate how long it will take for a given difficulty. + +To mine the first block in your custom chain, you can run: + + CLI="./bitcoin-cli -conf=mysignet.conf" + ADDR=$($CLI -signet getnewaddress) + NBITS=1e00f403 + $MINER --cli="$CLI" generate --grind-cmd="$GRIND" --address="$ADDR" --nbits=$NBITS + +This will mine a single block with a backdated timestamp designed to allow 100 blocks to be mined as quickly as possible, so that it is possible to do transactions. + +Adding the --ongoing parameter will then cause the signet miner to create blocks indefinitely. It will pick the time between blocks so that difficulty is adjusted to match the provided --nbits value. + + $MINER --cli="$CLI" generate --grind-cmd="$GRIND" --address="$ADDR" --nbits=$NBITS --ongoing + +Other options +------------- + +The --debug and --quiet options are available to control how noisy the signet miner's output is. Note that the --debug, --quiet and --cli parameters must all appear before the subcommand (generate, calibrate, etc) if used. + +Instead of specifying --ongoing, you can specify --max-blocks=N to mine N blocks and stop. + +The --set-block-time option is available to manually move timestamps forward or backward (subject to the rules that blocktime must be greater than mediantime, and dates can't be more than two hours in the future). It can only be used when mining a single block (ie, not when using --ongoing or --max-blocks greater than 1). + +Instead of using a single address, a ranged descriptor may be provided via the --descriptor parameter, with the reward for the block at height H being sent to the H'th address generated from the descriptor. + +Instead of calculating a specific nbits value, --min-nbits can be specified instead, in which case the minimum signet difficulty will be targeted. Signet's minimum difficulty corresponds to --nbits=1e0377ae. + +By default, the signet miner mines blocks at fixed intervals with minimal variation. If you want blocks to appear more randomly, as they do in mainnet, specify the --poisson option. + +Using the --multiminer parameter allows mining to be distributed amongst multiple miners. For example, if you have 3 miners and want to share blocks between them, specify --multiminer=1/3 on one, --multiminer=2/3 on another, and --multiminer=3/3 on the last one. If you want one to do 10% of blocks and two others to do 45% each, --multiminer=1-10/100 on the first, and --multiminer=11-55 and --multiminer=56-100 on the others. Note that which miner mines which block is determined by the previous block hash, so occasional runs of one miner doing many blocks in a row is to be expected. + +When --multiminer is used, if a miner is down and does not mine a block within five minutes of when it is due, the other miners will automatically act as redundant backups ensuring the chain does not halt. The --backup-delay parameter can be used to change how long a given miner waits, allowing one to be the primary backup (after five minutes) and another to be the secondary backup (after six minutes, eg). + +The --standby-delay parameter can be used to make a backup miner that only mines if a block doesn't arrive on time. This can be combined with --multiminer if desired. Setting --standby-delay also prevents the first block from being mined immediately. + +Advanced usage +-------------- + +The process generate follows internally is to get a block template, convert that into a PSBT, sign the PSBT, move the signature from the signed PSBT into the block template's coinbase, grind proof of work for the block, and then submit the block to the network. + +These steps can instead be done explicitly: + + $CLI -signet getblocktemplate '{"rules": ["signet","segwit"]}' | + $MINER --cli="$CLI" genpsbt --address="$ADDR" | + $CLI -signet -stdin walletprocesspsbt | + jq -r .psbt | + $MINER --cli="$CLI" solvepsbt --grind-cmd="$GRIND" | + $CLI -signet -stdin submitblock + +This is intended to allow you to replace part of the pipeline for further experimentation (eg, to sign the block with a hardware wallet). + diff --git a/contrib/signet/getcoins.py b/contrib/signet/getcoins.py new file mode 100755 index 0000000000..147d12600d --- /dev/null +++ b/contrib/signet/getcoins.py @@ -0,0 +1,158 @@ +#!/usr/bin/env python3 +# Copyright (c) 2020-2021 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +import argparse +import io +import requests +import subprocess +import sys +import xml.etree.ElementTree + +DEFAULT_GLOBAL_FAUCET = 'https://signetfaucet.com/claim' +DEFAULT_GLOBAL_CAPTCHA = 'https://signetfaucet.com/captcha' +GLOBAL_FIRST_BLOCK_HASH = '00000086d6b2636cb2a392d45edc4ec544a10024d30141c9adf4bfd9de533b53' + +# braille unicode block +BASE = 0x2800 +BIT_PER_PIXEL = [ + [0x01, 0x08], + [0x02, 0x10], + [0x04, 0x20], + [0x40, 0x80], +] +BW = 2 +BH = 4 + +# imagemagick or compatible fork (used for converting SVG) +CONVERT = 'convert' + +class PPMImage: + ''' + Load a PPM image (Pillow-ish API). + ''' + def __init__(self, f): + if f.readline() != b'P6\n': + raise ValueError('Invalid ppm format: header') + line = f.readline() + (width, height) = (int(x) for x in line.rstrip().split(b' ')) + if f.readline() != b'255\n': + raise ValueError('Invalid ppm format: color depth') + data = f.read(width * height * 3) + stride = width * 3 + self.size = (width, height) + self._grid = [[tuple(data[stride * y + 3 * x:stride * y + 3 * (x + 1)]) for x in range(width)] for y in range(height)] + + def getpixel(self, pos): + return self._grid[pos[1]][pos[0]] + +def print_image(img, threshold=128): + '''Print black-and-white image to terminal in braille unicode characters.''' + x_blocks = (img.size[0] + BW - 1) // BW + y_blocks = (img.size[1] + BH - 1) // BH + + for yb in range(y_blocks): + line = [] + for xb in range(x_blocks): + ch = BASE + for y in range(BH): + for x in range(BW): + try: + val = img.getpixel((xb * BW + x, yb * BH + y)) + except IndexError: + pass + else: + if val[0] < threshold: + ch |= BIT_PER_PIXEL[y][x] + line.append(chr(ch)) + print(''.join(line)) + +parser = argparse.ArgumentParser(description='Script to get coins from a faucet.', epilog='You may need to start with double-dash (--) when providing bitcoin-cli arguments.') +parser.add_argument('-c', '--cmd', dest='cmd', default='bitcoin-cli', help='bitcoin-cli command to use') +parser.add_argument('-f', '--faucet', dest='faucet', default=DEFAULT_GLOBAL_FAUCET, help='URL of the faucet') +parser.add_argument('-g', '--captcha', dest='captcha', default=DEFAULT_GLOBAL_CAPTCHA, help='URL of the faucet captcha, or empty if no captcha is needed') +parser.add_argument('-a', '--addr', dest='addr', default='', help='Bitcoin address to which the faucet should send') +parser.add_argument('-p', '--password', dest='password', default='', help='Faucet password, if any') +parser.add_argument('-n', '--amount', dest='amount', default='0.001', help='Amount to request (0.001-0.1, default is 0.001)') +parser.add_argument('-i', '--imagemagick', dest='imagemagick', default=CONVERT, help='Path to imagemagick convert utility') +parser.add_argument('bitcoin_cli_args', nargs='*', help='Arguments to pass on to bitcoin-cli (default: -signet)') + +args = parser.parse_args() + +if args.bitcoin_cli_args == []: + args.bitcoin_cli_args = ['-signet'] + + +def bitcoin_cli(rpc_command_and_params): + argv = [args.cmd] + args.bitcoin_cli_args + rpc_command_and_params + try: + return subprocess.check_output(argv).strip().decode() + except FileNotFoundError: + raise SystemExit(f"The binary {args.cmd} could not be found") + except subprocess.CalledProcessError: + cmdline = ' '.join(argv) + raise SystemExit(f"-----\nError while calling {cmdline} (see output above).") + + +if args.faucet.lower() == DEFAULT_GLOBAL_FAUCET: + # Get the hash of the block at height 1 of the currently active signet chain + curr_signet_hash = bitcoin_cli(['getblockhash', '1']) + if curr_signet_hash != GLOBAL_FIRST_BLOCK_HASH: + raise SystemExit('The global faucet cannot be used with a custom Signet network. Please use the global signet or setup your custom faucet to use this functionality.\n') +else: + # For custom faucets, don't request captcha by default. + if args.captcha == DEFAULT_GLOBAL_CAPTCHA: + args.captcha = '' + +if args.addr == '': + # get address for receiving coins + args.addr = bitcoin_cli(['getnewaddress', 'faucet', 'bech32']) + +data = {'address': args.addr, 'password': args.password, 'amount': args.amount} + +# Store cookies +# for debugging: print(session.cookies.get_dict()) +session = requests.Session() + +if args.captcha != '': # Retrieve a captcha + try: + res = session.get(args.captcha) + res.raise_for_status() + except requests.exceptions.RequestException as e: + raise SystemExit(f"Unexpected error when contacting faucet: {e}") + + # Size limitation + svg = xml.etree.ElementTree.fromstring(res.content) + if svg.attrib.get('width') != '150' or svg.attrib.get('height') != '50': + raise SystemExit("Captcha size doesn't match expected dimensions 150x50") + + # Convert SVG image to PPM, and load it + try: + rv = subprocess.run([args.imagemagick, 'svg:-', '-depth', '8', 'ppm:-'], input=res.content, check=True, capture_output=True) + except FileNotFoundError: + raise SystemExit(f"The binary {args.imagemagick} could not be found. Please make sure ImageMagick (or a compatible fork) is installed and that the correct path is specified.") + + img = PPMImage(io.BytesIO(rv.stdout)) + + # Terminal interaction + print_image(img) + print(f"Captcha from URL {args.captcha}") + data['captcha'] = input('Enter captcha: ') + +try: + res = session.post(args.faucet, data=data) +except: + raise SystemExit(f"Unexpected error when contacting faucet: {sys.exc_info()[0]}") + +# Display the output as per the returned status code +if res: + # When the return code is in between 200 and 400 i.e. successful + print(res.text) +elif res.status_code == 404: + print('The specified faucet URL does not exist. Please check for any server issues/typo.') +elif res.status_code == 429: + print('The script does not allow for repeated transactions as the global faucet is rate-limitied to 1 request/IP/day. You can access the faucet website to get more coins manually') +else: + print(f'Returned Error Code {res.status_code}\n{res.text}\n') + print('Please check the provided arguments for their validity and/or any possible typo.') diff --git a/contrib/signet/miner b/contrib/signet/miner new file mode 100755 index 0000000000..b366b98e2d --- /dev/null +++ b/contrib/signet/miner @@ -0,0 +1,631 @@ +#!/usr/bin/env python3 +# Copyright (c) 2020 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +import argparse +import base64 +import json +import logging +import math +import os +import re +import struct +import sys +import time +import subprocess + +from io import BytesIO + +PATH_BASE_CONTRIB_SIGNET = os.path.abspath(os.path.dirname(os.path.realpath(__file__))) +PATH_BASE_TEST_FUNCTIONAL = os.path.abspath(os.path.join(PATH_BASE_CONTRIB_SIGNET, "..", "..", "test", "functional")) +sys.path.insert(0, PATH_BASE_TEST_FUNCTIONAL) + +from test_framework.blocktools import WITNESS_COMMITMENT_HEADER, script_BIP34_coinbase_height # noqa: E402 +from test_framework.messages import CBlock, CBlockHeader, COutPoint, CTransaction, CTxIn, CTxInWitness, CTxOut, from_hex, deser_string, hash256, ser_compact_size, ser_string, ser_uint256, tx_from_hex, uint256_from_str # noqa: E402 +from test_framework.script import CScriptOp # noqa: E402 + +logging.basicConfig( + format='%(asctime)s %(levelname)s %(message)s', + level=logging.INFO, + datefmt='%Y-%m-%d %H:%M:%S') + +SIGNET_HEADER = b"\xec\xc7\xda\xa2" +PSBT_SIGNET_BLOCK = b"\xfc\x06signetb" # proprietary PSBT global field holding the block being signed +RE_MULTIMINER = re.compile("^(\d+)(-(\d+))?/(\d+)$") + +# #### some helpers that could go into test_framework + +# like from_hex, but without the hex part +def FromBinary(cls, stream): + """deserialize a binary stream (or bytes object) into an object""" + # handle bytes object by turning it into a stream + was_bytes = isinstance(stream, bytes) + if was_bytes: + stream = BytesIO(stream) + obj = cls() + obj.deserialize(stream) + if was_bytes: + assert len(stream.read()) == 0 + return obj + +class PSBTMap: + """Class for serializing and deserializing PSBT maps""" + + def __init__(self, map=None): + self.map = map if map is not None else {} + + def deserialize(self, f): + m = {} + while True: + k = deser_string(f) + if len(k) == 0: + break + v = deser_string(f) + if len(k) == 1: + k = k[0] + assert k not in m + m[k] = v + self.map = m + + def serialize(self): + m = b"" + for k,v in self.map.items(): + if isinstance(k, int) and 0 <= k and k <= 255: + k = bytes([k]) + m += ser_compact_size(len(k)) + k + m += ser_compact_size(len(v)) + v + m += b"\x00" + return m + +class PSBT: + """Class for serializing and deserializing PSBTs""" + + def __init__(self): + self.g = PSBTMap() + self.i = [] + self.o = [] + self.tx = None + + def deserialize(self, f): + assert f.read(5) == b"psbt\xff" + self.g = FromBinary(PSBTMap, f) + assert 0 in self.g.map + self.tx = FromBinary(CTransaction, self.g.map[0]) + self.i = [FromBinary(PSBTMap, f) for _ in self.tx.vin] + self.o = [FromBinary(PSBTMap, f) for _ in self.tx.vout] + return self + + def serialize(self): + assert isinstance(self.g, PSBTMap) + assert isinstance(self.i, list) and all(isinstance(x, PSBTMap) for x in self.i) + assert isinstance(self.o, list) and all(isinstance(x, PSBTMap) for x in self.o) + assert 0 in self.g.map + tx = FromBinary(CTransaction, self.g.map[0]) + assert len(tx.vin) == len(self.i) + assert len(tx.vout) == len(self.o) + + psbt = [x.serialize() for x in [self.g] + self.i + self.o] + return b"psbt\xff" + b"".join(psbt) + + def to_base64(self): + return base64.b64encode(self.serialize()).decode("utf8") + + @classmethod + def from_base64(cls, b64psbt): + return FromBinary(cls, base64.b64decode(b64psbt)) + +# ##### + +def create_coinbase(height, value, spk): + cb = CTransaction() + cb.vin = [CTxIn(COutPoint(0, 0xffffffff), script_BIP34_coinbase_height(height), 0xffffffff)] + cb.vout = [CTxOut(value, spk)] + return cb + +def get_witness_script(witness_root, witness_nonce): + commitment = uint256_from_str(hash256(ser_uint256(witness_root) + ser_uint256(witness_nonce))) + return b"\x6a" + CScriptOp.encode_op_pushdata(WITNESS_COMMITMENT_HEADER + ser_uint256(commitment)) + +def signet_txs(block, challenge): + # assumes signet solution has not been added yet so does not need + # to be removed + + txs = block.vtx[:] + txs[0] = CTransaction(txs[0]) + txs[0].vout[-1].scriptPubKey += CScriptOp.encode_op_pushdata(SIGNET_HEADER) + hashes = [] + for tx in txs: + tx.rehash() + hashes.append(ser_uint256(tx.sha256)) + mroot = block.get_merkle_root(hashes) + + sd = b"" + sd += struct.pack("<i", block.nVersion) + sd += ser_uint256(block.hashPrevBlock) + sd += ser_uint256(mroot) + sd += struct.pack("<I", block.nTime) + + to_spend = CTransaction() + to_spend.nVersion = 0 + to_spend.nLockTime = 0 + to_spend.vin = [CTxIn(COutPoint(0, 0xFFFFFFFF), b"\x00" + CScriptOp.encode_op_pushdata(sd), 0)] + to_spend.vout = [CTxOut(0, challenge)] + to_spend.rehash() + + spend = CTransaction() + spend.nVersion = 0 + spend.nLockTime = 0 + spend.vin = [CTxIn(COutPoint(to_spend.sha256, 0), b"", 0)] + spend.vout = [CTxOut(0, b"\x6a")] + + return spend, to_spend + +def do_createpsbt(block, signme, spendme): + psbt = PSBT() + psbt.g = PSBTMap( {0: signme.serialize(), + PSBT_SIGNET_BLOCK: block.serialize() + } ) + psbt.i = [ PSBTMap( {0: spendme.serialize(), + 3: bytes([1,0,0,0])}) + ] + psbt.o = [ PSBTMap() ] + return psbt.to_base64() + +def do_decode_psbt(b64psbt): + psbt = PSBT.from_base64(b64psbt) + + assert len(psbt.tx.vin) == 1 + assert len(psbt.tx.vout) == 1 + assert PSBT_SIGNET_BLOCK in psbt.g.map + + scriptSig = psbt.i[0].map.get(7, b"") + scriptWitness = psbt.i[0].map.get(8, b"\x00") + + return FromBinary(CBlock, psbt.g.map[PSBT_SIGNET_BLOCK]), ser_string(scriptSig) + scriptWitness + +def finish_block(block, signet_solution, grind_cmd): + block.vtx[0].vout[-1].scriptPubKey += CScriptOp.encode_op_pushdata(SIGNET_HEADER + signet_solution) + block.vtx[0].rehash() + block.hashMerkleRoot = block.calc_merkle_root() + if grind_cmd is None: + block.solve() + else: + headhex = CBlockHeader.serialize(block).hex() + cmd = grind_cmd.split(" ") + [headhex] + newheadhex = subprocess.run(cmd, stdout=subprocess.PIPE, input=b"", check=True).stdout.strip() + newhead = from_hex(CBlockHeader(), newheadhex.decode('utf8')) + block.nNonce = newhead.nNonce + block.rehash() + return block + +def generate_psbt(tmpl, reward_spk, *, blocktime=None): + signet_spk = tmpl["signet_challenge"] + signet_spk_bin = bytes.fromhex(signet_spk) + + cbtx = create_coinbase(height=tmpl["height"], value=tmpl["coinbasevalue"], spk=reward_spk) + cbtx.vin[0].nSequence = 2**32-2 + cbtx.rehash() + + block = CBlock() + block.nVersion = tmpl["version"] + block.hashPrevBlock = int(tmpl["previousblockhash"], 16) + block.nTime = tmpl["curtime"] if blocktime is None else blocktime + if block.nTime < tmpl["mintime"]: + block.nTime = tmpl["mintime"] + block.nBits = int(tmpl["bits"], 16) + block.nNonce = 0 + block.vtx = [cbtx] + [tx_from_hex(t["data"]) for t in tmpl["transactions"]] + + witnonce = 0 + witroot = block.calc_witness_merkle_root() + cbwit = CTxInWitness() + cbwit.scriptWitness.stack = [ser_uint256(witnonce)] + block.vtx[0].wit.vtxinwit = [cbwit] + block.vtx[0].vout.append(CTxOut(0, get_witness_script(witroot, witnonce))) + + signme, spendme = signet_txs(block, signet_spk_bin) + + return do_createpsbt(block, signme, spendme) + +def get_reward_address(args, height): + if args.address is not None: + return args.address + + if '*' not in args.descriptor: + addr = json.loads(args.bcli("deriveaddresses", args.descriptor))[0] + args.address = addr + return addr + + remove = [k for k in args.derived_addresses.keys() if k+20 <= height] + for k in remove: + del args.derived_addresses[k] + + addr = args.derived_addresses.get(height, None) + if addr is None: + addrs = json.loads(args.bcli("deriveaddresses", args.descriptor, "[%d,%d]" % (height, height+20))) + addr = addrs[0] + for k, a in enumerate(addrs): + args.derived_addresses[height+k] = a + + return addr + +def get_reward_addr_spk(args, height): + assert args.address is not None or args.descriptor is not None + + if hasattr(args, "reward_spk"): + return args.address, args.reward_spk + + reward_addr = get_reward_address(args, height) + reward_spk = bytes.fromhex(json.loads(args.bcli("getaddressinfo", reward_addr))["scriptPubKey"]) + if args.address is not None: + # will always be the same, so cache + args.reward_spk = reward_spk + + return reward_addr, reward_spk + +def do_genpsbt(args): + tmpl = json.load(sys.stdin) + _, reward_spk = get_reward_addr_spk(args, tmpl["height"]) + psbt = generate_psbt(tmpl, reward_spk) + print(psbt) + +def do_solvepsbt(args): + block, signet_solution = do_decode_psbt(sys.stdin.read()) + block = finish_block(block, signet_solution, args.grind_cmd) + print(block.serialize().hex()) + +def nbits_to_target(nbits): + shift = (nbits >> 24) & 0xff + return (nbits & 0x00ffffff) * 2**(8*(shift - 3)) + +def target_to_nbits(target): + tstr = "{0:x}".format(target) + if len(tstr) < 6: + tstr = ("000000"+tstr)[-6:] + if len(tstr) % 2 != 0: + tstr = "0" + tstr + if int(tstr[0],16) >= 0x8: + # avoid "negative" + tstr = "00" + tstr + fix = int(tstr[:6], 16) + sz = len(tstr)//2 + if tstr[6:] != "0"*(sz*2-6): + fix += 1 + + return int("%02x%06x" % (sz,fix), 16) + +def seconds_to_hms(s): + if s == 0: + return "0s" + neg = (s < 0) + if neg: + s = -s + out = "" + if s % 60 > 0: + out = "%ds" % (s % 60) + s //= 60 + if s % 60 > 0: + out = "%dm%s" % (s % 60, out) + s //= 60 + if s > 0: + out = "%dh%s" % (s, out) + if neg: + out = "-" + out + return out + +def next_block_delta(last_nbits, last_hash, ultimate_target, do_poisson): + # strategy: + # 1) work out how far off our desired target we are + # 2) cap it to a factor of 4 since that's the best we can do in a single retarget period + # 3) use that to work out the desired average interval in this retarget period + # 4) if doing poisson, use the last hash to pick a uniformly random number in [0,1), and work out a random multiplier to vary the average by + # 5) cap the resulting interval between 1 second and 1 hour to avoid extremes + + INTERVAL = 600.0*2016/2015 # 10 minutes, adjusted for the off-by-one bug + + current_target = nbits_to_target(last_nbits) + retarget_factor = ultimate_target / current_target + retarget_factor = max(0.25, min(retarget_factor, 4.0)) + + avg_interval = INTERVAL * retarget_factor + + if do_poisson: + det_rand = int(last_hash[-8:], 16) * 2**-32 + this_interval_variance = -math.log1p(-det_rand) + else: + this_interval_variance = 1 + + this_interval = avg_interval * this_interval_variance + this_interval = max(1, min(this_interval, 3600)) + + return this_interval + +def next_block_is_mine(last_hash, my_blocks): + det_rand = int(last_hash[-16:-8], 16) + return my_blocks[0] <= (det_rand % my_blocks[2]) < my_blocks[1] + +def do_generate(args): + if args.max_blocks is not None: + if args.ongoing: + logging.error("Cannot specify both --ongoing and --max-blocks") + return 1 + if args.max_blocks < 1: + logging.error("N must be a positive integer") + return 1 + max_blocks = args.max_blocks + elif args.ongoing: + max_blocks = None + else: + max_blocks = 1 + + if args.set_block_time is not None and max_blocks != 1: + logging.error("Cannot specify --ongoing or --max-blocks > 1 when using --set-block-time") + return 1 + if args.set_block_time is not None and args.set_block_time < 0: + args.set_block_time = time.time() + logging.info("Treating negative block time as current time (%d)" % (args.set_block_time)) + + if args.min_nbits: + if args.nbits is not None: + logging.error("Cannot specify --nbits and --min-nbits") + return 1 + args.nbits = "1e0377ae" + logging.info("Using nbits=%s" % (args.nbits)) + + if args.set_block_time is None: + if args.nbits is None or len(args.nbits) != 8: + logging.error("Must specify --nbits (use calibrate command to determine value)") + return 1 + + if args.multiminer is None: + my_blocks = (0,1,1) + else: + if not args.ongoing: + logging.error("Cannot specify --multiminer without --ongoing") + return 1 + m = RE_MULTIMINER.match(args.multiminer) + if m is None: + logging.error("--multiminer argument must be k/m or j-k/m") + return 1 + start,_,stop,total = m.groups() + if stop is None: + stop = start + start, stop, total = map(int, (start, stop, total)) + if stop < start or start <= 0 or total < stop or total == 0: + logging.error("Inconsistent values for --multiminer") + return 1 + my_blocks = (start-1, stop, total) + + ultimate_target = nbits_to_target(int(args.nbits,16)) + + mined_blocks = 0 + bestheader = {"hash": None} + lastheader = None + while max_blocks is None or mined_blocks < max_blocks: + + # current status? + bci = json.loads(args.bcli("getblockchaininfo")) + + if bestheader["hash"] != bci["bestblockhash"]: + bestheader = json.loads(args.bcli("getblockheader", bci["bestblockhash"])) + + if lastheader is None: + lastheader = bestheader["hash"] + elif bestheader["hash"] != lastheader: + next_delta = next_block_delta(int(bestheader["bits"], 16), bestheader["hash"], ultimate_target, args.poisson) + next_delta += bestheader["time"] - time.time() + next_is_mine = next_block_is_mine(bestheader["hash"], my_blocks) + logging.info("Received new block at height %d; next in %s (%s)", bestheader["height"], seconds_to_hms(next_delta), ("mine" if next_is_mine else "backup")) + lastheader = bestheader["hash"] + + # when is the next block due to be mined? + now = time.time() + if args.set_block_time is not None: + logging.debug("Setting start time to %d", args.set_block_time) + mine_time = args.set_block_time + action_time = now + is_mine = True + elif bestheader["height"] == 0: + time_delta = next_block_delta(int(bestheader["bits"], 16), bci["bestblockhash"], ultimate_target, args.poisson) + time_delta *= 100 # 100 blocks + logging.info("Backdating time for first block to %d minutes ago" % (time_delta/60)) + mine_time = now - time_delta + action_time = now + is_mine = True + else: + time_delta = next_block_delta(int(bestheader["bits"], 16), bci["bestblockhash"], ultimate_target, args.poisson) + mine_time = bestheader["time"] + time_delta + + is_mine = next_block_is_mine(bci["bestblockhash"], my_blocks) + + action_time = mine_time + if not is_mine: + action_time += args.backup_delay + + if args.standby_delay > 0: + action_time += args.standby_delay + elif mined_blocks == 0: + # for non-standby, always mine immediately on startup, + # even if the next block shouldn't be ours + action_time = now + + # don't want fractional times so round down + mine_time = int(mine_time) + action_time = int(action_time) + + # can't mine a block 2h in the future; 1h55m for some safety + action_time = max(action_time, mine_time - 6900) + + # ready to go? otherwise sleep and check for new block + if now < action_time: + sleep_for = min(action_time - now, 60) + if mine_time < now: + # someone else might have mined the block, + # so check frequently, so we don't end up late + # mining the next block if it's ours + sleep_for = min(20, sleep_for) + minestr = "mine" if is_mine else "backup" + logging.debug("Sleeping for %s, next block due in %s (%s)" % (seconds_to_hms(sleep_for), seconds_to_hms(mine_time - now), minestr)) + time.sleep(sleep_for) + continue + + # gbt + tmpl = json.loads(args.bcli("getblocktemplate", '{"rules":["signet","segwit"]}')) + if tmpl["previousblockhash"] != bci["bestblockhash"]: + logging.warning("GBT based off unexpected block (%s not %s), retrying", tmpl["previousblockhash"], bci["bestblockhash"]) + time.sleep(1) + continue + + logging.debug("GBT template: %s", tmpl) + + if tmpl["mintime"] > mine_time: + logging.info("Updating block time from %d to %d", mine_time, tmpl["mintime"]) + mine_time = tmpl["mintime"] + if mine_time > now: + logging.error("GBT mintime is in the future: %d is %d seconds later than %d", mine_time, (mine_time-now), now) + return 1 + + # address for reward + reward_addr, reward_spk = get_reward_addr_spk(args, tmpl["height"]) + + # mine block + logging.debug("Mining block delta=%s start=%s mine=%s", seconds_to_hms(mine_time-bestheader["time"]), mine_time, is_mine) + mined_blocks += 1 + psbt = generate_psbt(tmpl, reward_spk, blocktime=mine_time) + input_stream = os.linesep.join([psbt, "true", "ALL"]).encode('utf8') + psbt_signed = json.loads(args.bcli("-stdin", "walletprocesspsbt", input=input_stream)) + if not psbt_signed.get("complete",False): + logging.debug("Generated PSBT: %s" % (psbt,)) + sys.stderr.write("PSBT signing failed\n") + return 1 + block, signet_solution = do_decode_psbt(psbt_signed["psbt"]) + block = finish_block(block, signet_solution, args.grind_cmd) + + # submit block + r = args.bcli("-stdin", "submitblock", input=block.serialize().hex().encode('utf8')) + + # report + bstr = "block" if is_mine else "backup block" + + next_delta = next_block_delta(block.nBits, block.hash, ultimate_target, args.poisson) + next_delta += block.nTime - time.time() + next_is_mine = next_block_is_mine(block.hash, my_blocks) + + logging.debug("Block hash %s payout to %s", block.hash, reward_addr) + logging.info("Mined %s at height %d; next in %s (%s)", bstr, tmpl["height"], seconds_to_hms(next_delta), ("mine" if next_is_mine else "backup")) + if r != "": + logging.warning("submitblock returned %s for height %d hash %s", r, tmpl["height"], block.hash) + lastheader = block.hash + +def do_calibrate(args): + if args.nbits is not None and args.seconds is not None: + sys.stderr.write("Can only specify one of --nbits or --seconds\n") + return 1 + if args.nbits is not None and len(args.nbits) != 8: + sys.stderr.write("Must specify 8 hex digits for --nbits\n") + return 1 + + TRIALS = 600 # gets variance down pretty low + TRIAL_BITS = 0x1e3ea75f # takes about 5m to do 600 trials + + header = CBlockHeader() + header.nBits = TRIAL_BITS + targ = nbits_to_target(header.nBits) + + start = time.time() + count = 0 + for i in range(TRIALS): + header.nTime = i + header.nNonce = 0 + headhex = header.serialize().hex() + cmd = args.grind_cmd.split(" ") + [headhex] + newheadhex = subprocess.run(cmd, stdout=subprocess.PIPE, input=b"", check=True).stdout.strip() + + avg = (time.time() - start) * 1.0 / TRIALS + + if args.nbits is not None: + want_targ = nbits_to_target(int(args.nbits,16)) + want_time = avg*targ/want_targ + else: + want_time = args.seconds if args.seconds is not None else 25 + want_targ = int(targ*(avg/want_time)) + + print("nbits=%08x for %ds average mining time" % (target_to_nbits(want_targ), want_time)) + return 0 + +def bitcoin_cli(basecmd, args, **kwargs): + cmd = basecmd + ["-signet"] + args + logging.debug("Calling bitcoin-cli: %r", cmd) + out = subprocess.run(cmd, stdout=subprocess.PIPE, **kwargs, check=True).stdout + if isinstance(out, bytes): + out = out.decode('utf8') + return out.strip() + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--cli", default="bitcoin-cli", type=str, help="bitcoin-cli command") + parser.add_argument("--debug", action="store_true", help="Print debugging info") + parser.add_argument("--quiet", action="store_true", help="Only print warnings/errors") + + cmds = parser.add_subparsers(help="sub-commands") + genpsbt = cmds.add_parser("genpsbt", help="Generate a block PSBT for signing") + genpsbt.set_defaults(fn=do_genpsbt) + + solvepsbt = cmds.add_parser("solvepsbt", help="Solve a signed block PSBT") + solvepsbt.set_defaults(fn=do_solvepsbt) + + generate = cmds.add_parser("generate", help="Mine blocks") + generate.set_defaults(fn=do_generate) + generate.add_argument("--ongoing", action="store_true", help="Keep mining blocks") + generate.add_argument("--max-blocks", default=None, type=int, help="Max blocks to mine (default=1)") + generate.add_argument("--set-block-time", default=None, type=int, help="Set block time (unix timestamp)") + generate.add_argument("--nbits", default=None, type=str, help="Target nBits (specify difficulty)") + generate.add_argument("--min-nbits", action="store_true", help="Target minimum nBits (use min difficulty)") + generate.add_argument("--poisson", action="store_true", help="Simulate randomised block times") + generate.add_argument("--multiminer", default=None, type=str, help="Specify which set of blocks to mine (eg: 1-40/100 for the first 40%%, 2/3 for the second 3rd)") + generate.add_argument("--backup-delay", default=300, type=int, help="Seconds to delay before mining blocks reserved for other miners (default=300)") + generate.add_argument("--standby-delay", default=0, type=int, help="Seconds to delay before mining blocks (default=0)") + + calibrate = cmds.add_parser("calibrate", help="Calibrate difficulty") + calibrate.set_defaults(fn=do_calibrate) + calibrate.add_argument("--nbits", type=str, default=None) + calibrate.add_argument("--seconds", type=int, default=None) + + for sp in [genpsbt, generate]: + sp.add_argument("--address", default=None, type=str, help="Address for block reward payment") + sp.add_argument("--descriptor", default=None, type=str, help="Descriptor for block reward payment") + + for sp in [solvepsbt, generate, calibrate]: + sp.add_argument("--grind-cmd", default=None, type=str, required=(sp==calibrate), help="Command to grind a block header for proof-of-work") + + args = parser.parse_args(sys.argv[1:]) + + args.bcli = lambda *a, input=b"", **kwargs: bitcoin_cli(args.cli.split(" "), list(a), input=input, **kwargs) + + if hasattr(args, "address") and hasattr(args, "descriptor"): + if args.address is None and args.descriptor is None: + sys.stderr.write("Must specify --address or --descriptor\n") + return 1 + elif args.address is not None and args.descriptor is not None: + sys.stderr.write("Only specify one of --address or --descriptor\n") + return 1 + args.derived_addresses = {} + + if args.debug: + logging.getLogger().setLevel(logging.DEBUG) + elif args.quiet: + logging.getLogger().setLevel(logging.WARNING) + else: + logging.getLogger().setLevel(logging.INFO) + + if hasattr(args, "fn"): + return args.fn(args) + else: + logging.error("Must specify command") + return 1 + +if __name__ == "__main__": + main() + + diff --git a/contrib/testgen/README.md b/contrib/testgen/README.md new file mode 100644 index 0000000000..2f0288df16 --- /dev/null +++ b/contrib/testgen/README.md @@ -0,0 +1,8 @@ +### TestGen ### + +Utilities to generate test vectors for the data-driven Bitcoin tests. + +To use inside a scripted-diff (or just execute directly): + + ./gen_key_io_test_vectors.py valid 70 > ../../src/test/data/key_io_valid.json + ./gen_key_io_test_vectors.py invalid 70 > ../../src/test/data/key_io_invalid.json diff --git a/contrib/testgen/gen_key_io_test_vectors.py b/contrib/testgen/gen_key_io_test_vectors.py new file mode 100755 index 0000000000..7bfb1d76a8 --- /dev/null +++ b/contrib/testgen/gen_key_io_test_vectors.py @@ -0,0 +1,265 @@ +#!/usr/bin/env python3 +# Copyright (c) 2012-2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +''' +Generate valid and invalid base58/bech32(m) address and private key test vectors. +''' + +from itertools import islice +import os +import random +import sys + +sys.path.append(os.path.join(os.path.dirname(__file__), '../../test/functional')) + +from test_framework.address import base58_to_byte, byte_to_base58, b58chars # noqa: E402 +from test_framework.script import OP_0, OP_1, OP_2, OP_3, OP_16, OP_DUP, OP_EQUAL, OP_EQUALVERIFY, OP_HASH160, OP_CHECKSIG # noqa: E402 +from test_framework.segwit_addr import bech32_encode, decode_segwit_address, convertbits, CHARSET, Encoding # noqa: E402 + +# key types +PUBKEY_ADDRESS = 0 +SCRIPT_ADDRESS = 5 +PUBKEY_ADDRESS_TEST = 111 +SCRIPT_ADDRESS_TEST = 196 +PUBKEY_ADDRESS_REGTEST = 111 +SCRIPT_ADDRESS_REGTEST = 196 +PRIVKEY = 128 +PRIVKEY_TEST = 239 +PRIVKEY_REGTEST = 239 + +# script +pubkey_prefix = (OP_DUP, OP_HASH160, 20) +pubkey_suffix = (OP_EQUALVERIFY, OP_CHECKSIG) +script_prefix = (OP_HASH160, 20) +script_suffix = (OP_EQUAL,) +p2wpkh_prefix = (OP_0, 20) +p2wsh_prefix = (OP_0, 32) +p2tr_prefix = (OP_1, 32) + +metadata_keys = ['isPrivkey', 'chain', 'isCompressed', 'tryCaseFlip'] +# templates for valid sequences +templates = [ + # prefix, payload_size, suffix, metadata, output_prefix, output_suffix + # None = N/A + ((PUBKEY_ADDRESS,), 20, (), (False, 'main', None, None), pubkey_prefix, pubkey_suffix), + ((SCRIPT_ADDRESS,), 20, (), (False, 'main', None, None), script_prefix, script_suffix), + ((PUBKEY_ADDRESS_TEST,), 20, (), (False, 'test', None, None), pubkey_prefix, pubkey_suffix), + ((SCRIPT_ADDRESS_TEST,), 20, (), (False, 'test', None, None), script_prefix, script_suffix), + ((PUBKEY_ADDRESS_TEST,), 20, (), (False, 'signet', None, None), pubkey_prefix, pubkey_suffix), + ((SCRIPT_ADDRESS_TEST,), 20, (), (False, 'signet', None, None), script_prefix, script_suffix), + ((PUBKEY_ADDRESS_REGTEST,), 20, (), (False, 'regtest', None, None), pubkey_prefix, pubkey_suffix), + ((SCRIPT_ADDRESS_REGTEST,), 20, (), (False, 'regtest', None, None), script_prefix, script_suffix), + ((PRIVKEY,), 32, (), (True, 'main', False, None), (), ()), + ((PRIVKEY,), 32, (1,), (True, 'main', True, None), (), ()), + ((PRIVKEY_TEST,), 32, (), (True, 'test', False, None), (), ()), + ((PRIVKEY_TEST,), 32, (1,), (True, 'test', True, None), (), ()), + ((PRIVKEY_TEST,), 32, (), (True, 'signet', False, None), (), ()), + ((PRIVKEY_TEST,), 32, (1,), (True, 'signet', True, None), (), ()), + ((PRIVKEY_REGTEST,), 32, (), (True, 'regtest', False, None), (), ()), + ((PRIVKEY_REGTEST,), 32, (1,), (True, 'regtest', True, None), (), ()) +] +# templates for valid bech32 sequences +bech32_templates = [ + # hrp, version, witprog_size, metadata, encoding, output_prefix + ('bc', 0, 20, (False, 'main', None, True), Encoding.BECH32, p2wpkh_prefix), + ('bc', 0, 32, (False, 'main', None, True), Encoding.BECH32, p2wsh_prefix), + ('bc', 1, 32, (False, 'main', None, True), Encoding.BECH32M, p2tr_prefix), + ('bc', 2, 2, (False, 'main', None, True), Encoding.BECH32M, (OP_2, 2)), + ('tb', 0, 20, (False, 'test', None, True), Encoding.BECH32, p2wpkh_prefix), + ('tb', 0, 32, (False, 'test', None, True), Encoding.BECH32, p2wsh_prefix), + ('tb', 1, 32, (False, 'test', None, True), Encoding.BECH32M, p2tr_prefix), + ('tb', 3, 16, (False, 'test', None, True), Encoding.BECH32M, (OP_3, 16)), + ('tb', 0, 20, (False, 'signet', None, True), Encoding.BECH32, p2wpkh_prefix), + ('tb', 0, 32, (False, 'signet', None, True), Encoding.BECH32, p2wsh_prefix), + ('tb', 1, 32, (False, 'signet', None, True), Encoding.BECH32M, p2tr_prefix), + ('tb', 3, 32, (False, 'signet', None, True), Encoding.BECH32M, (OP_3, 32)), + ('bcrt', 0, 20, (False, 'regtest', None, True), Encoding.BECH32, p2wpkh_prefix), + ('bcrt', 0, 32, (False, 'regtest', None, True), Encoding.BECH32, p2wsh_prefix), + ('bcrt', 1, 32, (False, 'regtest', None, True), Encoding.BECH32M, p2tr_prefix), + ('bcrt', 16, 40, (False, 'regtest', None, True), Encoding.BECH32M, (OP_16, 40)) +] +# templates for invalid bech32 sequences +bech32_ng_templates = [ + # hrp, version, witprog_size, encoding, invalid_bech32, invalid_checksum, invalid_char + ('tc', 0, 20, Encoding.BECH32, False, False, False), + ('bt', 1, 32, Encoding.BECH32M, False, False, False), + ('tb', 17, 32, Encoding.BECH32M, False, False, False), + ('bcrt', 3, 1, Encoding.BECH32M, False, False, False), + ('bc', 15, 41, Encoding.BECH32M, False, False, False), + ('tb', 0, 16, Encoding.BECH32, False, False, False), + ('bcrt', 0, 32, Encoding.BECH32, True, False, False), + ('bc', 0, 16, Encoding.BECH32, True, False, False), + ('tb', 0, 32, Encoding.BECH32, False, True, False), + ('bcrt', 0, 20, Encoding.BECH32, False, False, True), + ('bc', 0, 20, Encoding.BECH32M, False, False, False), + ('tb', 0, 32, Encoding.BECH32M, False, False, False), + ('bcrt', 0, 20, Encoding.BECH32M, False, False, False), + ('bc', 1, 32, Encoding.BECH32, False, False, False), + ('tb', 2, 16, Encoding.BECH32, False, False, False), + ('bcrt', 16, 20, Encoding.BECH32, False, False, False), +] + +def is_valid(v): + '''Check vector v for validity''' + if len(set(v) - set(b58chars)) > 0: + return is_valid_bech32(v) + try: + payload, version = base58_to_byte(v) + result = bytes([version]) + payload + except ValueError: # thrown if checksum doesn't match + return is_valid_bech32(v) + for template in templates: + prefix = bytearray(template[0]) + suffix = bytearray(template[2]) + if result.startswith(prefix) and result.endswith(suffix): + if (len(result) - len(prefix) - len(suffix)) == template[1]: + return True + return is_valid_bech32(v) + +def is_valid_bech32(v): + '''Check vector v for bech32 validity''' + for hrp in ['bc', 'tb', 'bcrt']: + if decode_segwit_address(hrp, v) != (None, None): + return True + return False + +def gen_valid_base58_vector(template): + '''Generate valid base58 vector''' + prefix = bytearray(template[0]) + payload = rand_bytes(size=template[1]) + suffix = bytearray(template[2]) + dst_prefix = bytearray(template[4]) + dst_suffix = bytearray(template[5]) + assert len(prefix) == 1 + rv = byte_to_base58(payload + suffix, prefix[0]) + return rv, dst_prefix + payload + dst_suffix + +def gen_valid_bech32_vector(template): + '''Generate valid bech32 vector''' + hrp = template[0] + witver = template[1] + witprog = rand_bytes(size=template[2]) + encoding = template[4] + dst_prefix = bytearray(template[5]) + rv = bech32_encode(encoding, hrp, [witver] + convertbits(witprog, 8, 5)) + return rv, dst_prefix + witprog + +def gen_valid_vectors(): + '''Generate valid test vectors''' + glist = [gen_valid_base58_vector, gen_valid_bech32_vector] + tlist = [templates, bech32_templates] + while True: + for template, valid_vector_generator in [(t, g) for g, l in zip(glist, tlist) for t in l]: + rv, payload = valid_vector_generator(template) + assert is_valid(rv) + metadata = {x: y for x, y in zip(metadata_keys,template[3]) if y is not None} + hexrepr = payload.hex() + yield (rv, hexrepr, metadata) + +def gen_invalid_base58_vector(template): + '''Generate possibly invalid vector''' + # kinds of invalid vectors: + # invalid prefix + # invalid payload length + # invalid (randomized) suffix (add random data) + # corrupt checksum + corrupt_prefix = randbool(0.2) + randomize_payload_size = randbool(0.2) + corrupt_suffix = randbool(0.2) + + if corrupt_prefix: + prefix = rand_bytes(size=1) + else: + prefix = bytearray(template[0]) + + if randomize_payload_size: + payload = rand_bytes(size=max(int(random.expovariate(0.5)), 50)) + else: + payload = rand_bytes(size=template[1]) + + if corrupt_suffix: + suffix = rand_bytes(size=len(template[2])) + else: + suffix = bytearray(template[2]) + + assert len(prefix) == 1 + val = byte_to_base58(payload + suffix, prefix[0]) + if random.randint(0,10)<1: # line corruption + if randbool(): # add random character to end + val += random.choice(b58chars) + else: # replace random character in the middle + n = random.randint(0, len(val)) + val = val[0:n] + random.choice(b58chars) + val[n+1:] + + return val + +def gen_invalid_bech32_vector(template): + '''Generate possibly invalid bech32 vector''' + no_data = randbool(0.1) + to_upper = randbool(0.1) + hrp = template[0] + witver = template[1] + witprog = rand_bytes(size=template[2]) + encoding = template[3] + + if no_data: + rv = bech32_encode(encoding, hrp, []) + else: + data = [witver] + convertbits(witprog, 8, 5) + if template[4] and not no_data: + if template[2] % 5 in {2, 4}: + data[-1] |= 1 + else: + data.append(0) + rv = bech32_encode(encoding, hrp, data) + + if template[5]: + i = len(rv) - random.randrange(1, 7) + rv = rv[:i] + random.choice(CHARSET.replace(rv[i], '')) + rv[i + 1:] + if template[6]: + i = len(hrp) + 1 + random.randrange(0, len(rv) - len(hrp) - 4) + rv = rv[:i] + rv[i:i + 4].upper() + rv[i + 4:] + + if to_upper: + rv = rv.swapcase() + + return rv + +def randbool(p = 0.5): + '''Return True with P(p)''' + return random.random() < p + +def rand_bytes(*, size): + return bytearray(random.getrandbits(8) for _ in range(size)) + +def gen_invalid_vectors(): + '''Generate invalid test vectors''' + # start with some manual edge-cases + yield "", + yield "x", + glist = [gen_invalid_base58_vector, gen_invalid_bech32_vector] + tlist = [templates, bech32_ng_templates] + while True: + for template, invalid_vector_generator in [(t, g) for g, l in zip(glist, tlist) for t in l]: + val = invalid_vector_generator(template) + if not is_valid(val): + yield val, + +if __name__ == '__main__': + import json + iters = {'valid':gen_valid_vectors, 'invalid':gen_invalid_vectors} + random.seed(42) + try: + uiter = iters[sys.argv[1]] + except IndexError: + uiter = gen_valid_vectors + try: + count = int(sys.argv[2]) + except IndexError: + count = 0 + + data = list(islice(uiter(), count)) + json.dump(data, sys.stdout, sort_keys=True, indent=4) + sys.stdout.write('\n') + diff --git a/contrib/tracing/README.md b/contrib/tracing/README.md new file mode 100644 index 0000000000..a409a23ef8 --- /dev/null +++ b/contrib/tracing/README.md @@ -0,0 +1,288 @@ +Example scripts for User-space, Statically Defined Tracing (USDT) +================================================================= + +This directory contains scripts showcasing User-space, Statically Defined +Tracing (USDT) support for Bitcoin Core on Linux using. For more information on +USDT support in Bitcoin Core see the [USDT documentation]. + +[USDT documentation]: ../../doc/tracing.md + + +Examples for the two main eBPF front-ends, [bpftrace] and +[BPF Compiler Collection (BCC)], with support for USDT, are listed. BCC is used +for complex tools and daemons and `bpftrace` is preferred for one-liners and +shorter scripts. + +[bpftrace]: https://github.com/iovisor/bpftrace +[BPF Compiler Collection (BCC)]: https://github.com/iovisor/bcc + + +To develop and run bpftrace and BCC scripts you need to install the +corresponding packages. See [installing bpftrace] and [installing BCC] for more +information. For development there exist a [bpftrace Reference Guide], a +[BCC Reference Guide], and a [bcc Python Developer Tutorial]. + +[installing bpftrace]: https://github.com/iovisor/bpftrace/blob/master/INSTALL.md +[installing BCC]: https://github.com/iovisor/bcc/blob/master/INSTALL.md +[bpftrace Reference Guide]: https://github.com/iovisor/bpftrace/blob/master/docs/reference_guide.md +[BCC Reference Guide]: https://github.com/iovisor/bcc/blob/master/docs/reference_guide.md +[bcc Python Developer Tutorial]: https://github.com/iovisor/bcc/blob/master/docs/tutorial_bcc_python_developer.md + +## Examples + +The bpftrace examples contain a relative path to the `bitcoind` binary. By +default, the scripts should be run from the repository-root and assume a +self-compiled `bitcoind` binary. The paths in the examples can be changed, for +example, to point to release builds if needed. See the +[Bitcoin Core USDT documentation] on how to list available tracepoints in your +`bitcoind` binary. + +[Bitcoin Core USDT documentation]: ../../doc/tracing.md#listing-available-tracepoints + +**WARNING: eBPF programs require root privileges to be loaded into a Linux +kernel VM. This means the bpftrace and BCC examples must be executed with root +privileges. Make sure to carefully review any scripts that you run with root +privileges first!** + +### log_p2p_traffic.bt + +A bpftrace script logging information about inbound and outbound P2P network +messages. Based on the `net:inbound_message` and `net:outbound_message` +tracepoints. + +By default, `bpftrace` limits strings to 64 bytes due to the limited stack size +in the eBPF VM. For example, Tor v3 addresses exceed the string size limit which +results in the port being cut off during logging. The string size limit can be +increased with the `BPFTRACE_STRLEN` environment variable (`BPFTRACE_STRLEN=70` +works fine). + +``` +$ bpftrace contrib/tracing/log_p2p_traffic.bt +``` + +Output +``` +outbound 'ping' msg to peer 11 (outbound-full-relay, [2a02:b10c:f747:1:ef:fake:ipv6:addr]:8333) with 8 bytes +inbound 'pong' msg from peer 11 (outbound-full-relay, [2a02:b10c:f747:1:ef:fake:ipv6:addr]:8333) with 8 bytes +inbound 'inv' msg from peer 16 (outbound-full-relay, XX.XX.XXX.121:8333) with 37 bytes +outbound 'getdata' msg to peer 16 (outbound-full-relay, XX.XX.XXX.121:8333) with 37 bytes +inbound 'tx' msg from peer 16 (outbound-full-relay, XX.XX.XXX.121:8333) with 222 bytes +outbound 'inv' msg to peer 9 (outbound-full-relay, faketorv3addressa2ufa6odvoi3s77j4uegey0xb10csyfyve2t33curbyd.onion:8333) with 37 bytes +outbound 'inv' msg to peer 7 (outbound-full-relay, XX.XX.XXX.242:8333) with 37 bytes +… +``` + +### p2p_monitor.py + +A BCC Python script using curses for an interactive P2P message monitor. Based +on the `net:inbound_message` and `net:outbound_message` tracepoints. + +Inbound and outbound traffic is listed for each peer together with information +about the connection. Peers can be selected individually to view recent P2P +messages. + +``` +$ python3 contrib/tracing/p2p_monitor.py ./src/bitcoind +``` + +Lists selectable peers and traffic and connection information. +``` + P2P Message Monitor + Navigate with UP/DOWN or J/K and select a peer with ENTER or SPACE to see individual P2P messages + + PEER OUTBOUND INBOUND TYPE ADDR + 0 46 398 byte 61 1407590 byte block-relay-only XX.XX.XXX.196:8333 + 11 1156 253570 byte 3431 2394924 byte outbound-full-relay XXX.X.XX.179:8333 + 13 3425 1809620 byte 1236 305458 byte inbound XXX.X.X.X:60380 + 16 1046 241633 byte 1589 1199220 byte outbound-full-relay 4faketorv2pbfu7x.onion:8333 + 19 577 181679 byte 390 148951 byte outbound-full-relay kfake4vctorjv2o2.onion:8333 + 20 11 1248 byte 13 1283 byte block-relay-only [2600:fake:64d9:b10c:4436:aaaa:fe:bb]:8333 + 21 11 1248 byte 13 1299 byte block-relay-only XX.XXX.X.155:8333 + 22 5 103 byte 1 102 byte feeler XX.XX.XXX.173:8333 + 23 11 1248 byte 12 1255 byte block-relay-only XX.XXX.XXX.220:8333 + 24 3 103 byte 1 102 byte feeler XXX.XXX.XXX.64:8333 +… +``` + +Showing recent P2P messages between our node and a selected peer. + +``` + ---------------------------------------------------------------------- + | PEER 16 (4faketorv2pbfu7x.onion:8333) | + | OUR NODE outbound-full-relay PEER | + | <--- sendcmpct (9 bytes) | + | inv (37 byte) ---> | + | <--- ping (8 bytes) | + | pong (8 byte) ---> | + | inv (37 byte) ---> | + | <--- addr (31 bytes) | + | inv (37 byte) ---> | + | <--- getheaders (1029 bytes) | + | headers (1 byte) ---> | + | <--- feefilter (8 bytes) | + | <--- pong (8 bytes) | + | <--- headers (82 bytes) | + | <--- addr (30003 bytes) | + | inv (1261 byte) ---> | + | … | + +``` + +### log_raw_p2p_msgs.py + +A BCC Python script showcasing eBPF and USDT limitations when passing data +larger than about 32kb. Based on the `net:inbound_message` and +`net:outbound_message` tracepoints. + +Bitcoin P2P messages can be larger than 32kb (e.g. `tx`, `block`, ...). The +eBPF VM's stack is limited to 512 bytes, and we can't allocate more than about +32kb for a P2P message in the eBPF VM. The **message data is cut off** when the +message is larger than MAX_MSG_DATA_LENGTH (see script). This can be detected +in user-space by comparing the data length to the message length variable. The +message is cut off when the data length is smaller than the message length. +A warning is included with the printed message data. + +Data is submitted to user-space (i.e. to this script) via a ring buffer. The +throughput of the ring buffer is limited. Each p2p_message is about 32kb in +size. In- or outbound messages submitted to the ring buffer in rapid +succession fill the ring buffer faster than it can be read. Some messages are +lost. BCC prints: `Possibly lost 2 samples` on lost messages. + + +``` +$ python3 contrib/tracing/log_raw_p2p_msgs.py ./src/bitcoind +``` + +``` +Logging raw P2P messages. +Messages larger that about 32kb will be cut off! +Some messages might be lost! + outbound msg 'inv' from peer 4 (outbound-full-relay, XX.XXX.XX.4:8333) with 253 bytes: 0705000000be2245c8f844c9f763748e1a7… +… +Warning: incomplete message (only 32568 out of 53552 bytes)! inbound msg 'tx' from peer 32 (outbound-full-relay, XX.XXX.XXX.43:8333) with 53552 bytes: 020000000001fd3c01939c85ad6756ed9fc… +… +Possibly lost 2 samples +``` + +### connectblock_benchmark.bt + +A `bpftrace` script to benchmark the `ConnectBlock()` function during, for +example, a blockchain re-index. Based on the `validation:block_connected` USDT +tracepoint. + +The script takes three positional arguments. The first two arguments, the start, +and end height indicate between which blocks the benchmark should be run. The +third acts as a duration threshold in milliseconds. When the `ConnectBlock()` +function takes longer than the threshold, information about the block, is +printed. For more details, see the header comment in the script. + +The following command can be used to benchmark, for example, `ConnectBlock()` +between height 20000 and 38000 on SigNet while logging all blocks that take +longer than 25ms to connect. + +``` +$ bpftrace contrib/tracing/connectblock_benchmark.bt 20000 38000 25 +``` + +In a different terminal, starting Bitcoin Core in SigNet mode and with +re-indexing enabled. + +``` +$ ./src/bitcoind -signet -reindex +``` + +This produces the following output. +``` +Attaching 5 probes... +ConnectBlock Benchmark between height 20000 and 38000 inclusive +Logging blocks taking longer than 25 ms to connect. +Starting Connect Block Benchmark between height 20000 and 38000. +BENCH 39 blk/s 59 tx/s 59 inputs/s 20 sigops/s (height 20038) +Block 20492 (000000f555653bb05e2f3c6e79925e01a20dd57033f4dc7c354b46e34735d32b) 20 tx 2319 ins 2318 sigops took 38 ms +BENCH 1840 blk/s 2117 tx/s 4478 inputs/s 2471 sigops/s (height 21879) +BENCH 1816 blk/s 4972 tx/s 4982 inputs/s 125 sigops/s (height 23695) +BENCH 2095 blk/s 2890 tx/s 2910 inputs/s 152 sigops/s (height 25790) +BENCH 1684 blk/s 3979 tx/s 4053 inputs/s 288 sigops/s (height 27474) +BENCH 1155 blk/s 3216 tx/s 3252 inputs/s 115 sigops/s (height 28629) +BENCH 1797 blk/s 2488 tx/s 2503 inputs/s 111 sigops/s (height 30426) +BENCH 1849 blk/s 6318 tx/s 6569 inputs/s 12189 sigops/s (height 32275) +BENCH 946 blk/s 20209 tx/s 20775 inputs/s 83809 sigops/s (height 33221) +Block 33406 (0000002adfe4a15cfcd53bd890a89bbae836e5bb7f38bac566f61ad4548c87f6) 25 tx 2045 ins 2090 sigops took 29 ms +Block 33687 (00000073231307a9828e5607ceb8156b402efe56747271a4442e75eb5b77cd36) 52 tx 1797 ins 1826 sigops took 26 ms +BENCH 582 blk/s 21581 tx/s 27673 inputs/s 60345 sigops/s (height 33803) +BENCH 1035 blk/s 19735 tx/s 19776 inputs/s 51355 sigops/s (height 34838) +Block 35625 (0000006b00b347390c4768ea9df2655e9ff4b120f29d78594a2a702f8a02c997) 20 tx 3374 ins 3371 sigops took 49 ms +BENCH 887 blk/s 17857 tx/s 22191 inputs/s 24404 sigops/s (height 35725) +Block 35937 (000000d816d13d6e39b471cd4368db60463a764ba1f29168606b04a22b81ea57) 75 tx 3943 ins 3940 sigops took 61 ms +BENCH 823 blk/s 16298 tx/s 21031 inputs/s 18440 sigops/s (height 36548) +Block 36583 (000000c3e260556dbf42968aae3f904dba8b8c1ff96a6f6e3aa5365d2e3ad317) 24 tx 2198 ins 2194 sigops took 34 ms +Block 36700 (000000b3b173de9e65a3cfa738d976af6347aaf83fa17ab3f2a4d2ede3ddfac4) 73 tx 1615 ins 1611 sigops took 31 ms +Block 36832 (0000007859578c02c1ac37dabd1b9ec19b98f350b56935f5dd3a41e9f79f836e) 34 tx 1440 ins 1436 sigops took 26 ms +BENCH 613 blk/s 16718 tx/s 25074 inputs/s 23022 sigops/s (height 37161) +Block 37870 (000000f5c1086291ba2d943fb0c3bc82e71c5ee341ee117681d1456fbf6c6c38) 25 tx 1517 ins 1514 sigops took 29 ms +BENCH 811 blk/s 16031 tx/s 20921 inputs/s 18696 sigops/s (height 37972) + +Took 14055 ms to connect the blocks between height 20000 and 38000. + +Histogram of block connection times in milliseconds (ms). +@durations: +[0] 16838 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@| +[1] 882 |@@ | +[2, 4) 236 | | +[4, 8) 23 | | +[8, 16) 9 | | +[16, 32) 9 | | +[32, 64) 4 | | +``` + +### log_utxocache_flush.py + +A BCC Python script to log the UTXO cache flushes. Based on the +`utxocache:flush` tracepoint. + +```bash +$ python3 contrib/tracing/log_utxocache_flush.py ./src/bitcoind +``` + +``` +Logging utxocache flushes. Ctrl-C to end... +Duration (µs) Mode Coins Count Memory Usage Prune +730451 IF_NEEDED 22990 3323.54 kB True +637657 ALWAYS 122320 17124.80 kB False +81349 ALWAYS 0 1383.49 kB False +``` + +### log_utxos.bt + +A `bpftrace` script to log information about the coins that are added, spent, or +uncached from the UTXO set. Based on the `utxocache:add`, `utxocache:spend` and +`utxocache:uncache` tracepoints. + +```bash +$ bpftrace contrib/tracing/log_utxos.bt +``` + +This should produce an output similar to the following. If you see bpftrace +warnings like `Lost 24 events`, the eBPF perf ring-buffer is filled faster +than it is being read. You can increase the ring-buffer size by setting the +ENV variable `BPFTRACE_PERF_RB_PAGES` (default 64) at a cost of higher +memory usage. See the [bpftrace reference guide] for more information. + +[bpftrace reference guide]: https://github.com/iovisor/bpftrace/blob/master/docs/reference_guide.md#98-bpftrace_perf_rb_pages + +```bash +Attaching 4 probes... +OP Outpoint Value Height Coinbase +Added 6ba9ad857e1ef2eb2a2c94f06813c414c7ab273e3d6bd7ad64e000315a887e7c:1 10000 2094512 No +Spent fa7dc4db56637a151f6649d8f26732956d1c5424c82aae400a83d02b2cc2c87b:0 182264897 2094512 No +Added eeb2f099b1af6a2a12e6ddd2eeb16fc5968582241d7f08ba202d28b60ac264c7:0 10000 2094512 No +Added eeb2f099b1af6a2a12e6ddd2eeb16fc5968582241d7f08ba202d28b60ac264c7:1 182254756 2094512 No +Added a0c7f4ec9cccef2d89672a624a4e6c8237a17572efdd4679eea9e9ee70d2db04:0 10072679 2094513 Yes +Spent 25e0df5cc1aeb1b78e6056bf403e5e8b7e41f138060ca0a50a50134df0549a5e:2 540 2094508 No +Spent 42f383c04e09c26a2378272ec33aa0c1bf4883ca5ab739e8b7e06be5a5787d61:1 3848399 2007724 No +Added f85e3b4b89270863a389395cc9a4123e417ab19384cef96533c6649abd6b0561:0 3788399 2094513 No +Added f85e3b4b89270863a389395cc9a4123e417ab19384cef96533c6649abd6b0561:2 540 2094513 No +Spent a05880b8c77971ed0b9f73062c7c4cdb0ff3856ab14cbf8bc481ed571cd34b83:1 5591281046 2094511 No +Added eb689865f7d957938978d6207918748f74e6aa074f47874724327089445b0960:0 5589696005 2094513 No +Added eb689865f7d957938978d6207918748f74e6aa074f47874724327089445b0960:1 1565556 2094513 No +``` diff --git a/contrib/tracing/connectblock_benchmark.bt b/contrib/tracing/connectblock_benchmark.bt new file mode 100755 index 0000000000..6e7a98ef07 --- /dev/null +++ b/contrib/tracing/connectblock_benchmark.bt @@ -0,0 +1,156 @@ +#!/usr/bin/env bpftrace + +/* + + USAGE: + + bpftrace contrib/tracing/connectblock_benchmark.bt <start height> <end height> <logging threshold in ms> + + - <start height> sets the height at which the benchmark should start. Setting + the start height to 0 starts the benchmark immediately, even before the + first block is connected. + - <end height> sets the height after which the benchmark should end. Setting + the end height to 0 disables the benchmark. The script only logs blocks + over <logging threshold in ms>. + - Threshold <logging threshold in ms> + + This script requires a 'bitcoind' binary compiled with eBPF support and the + 'validation:block_connected' USDT. By default, it's assumed that 'bitcoind' is + located in './src/bitcoind'. This can be modified in the script below. + + EXAMPLES: + + bpftrace contrib/tracing/connectblock_benchmark.bt 300000 680000 1000 + + When run together 'bitcoind -reindex', this benchmarks the time it takes to + connect the blocks between height 300.000 and 680.000 (inclusive) and prints + details about all blocks that take longer than 1000ms to connect. Prints a + histogram with block connection times when the benchmark is finished. + + + bpftrace contrib/tracing/connectblock_benchmark.bt 0 0 500 + + When running together 'bitcoind', all newly connected blocks that + take longer than 500ms to connect are logged. A histogram with block + connection times is shown when the script is terminated. + +*/ + +BEGIN +{ + $start_height = $1; + $end_height = $2; + $logging_threshold_ms = $3; + + if ($end_height < $start_height) { + printf("Error: start height (%d) larger than end height (%d)!\n", $start_height, $end_height); + exit(); + } + + if ($end_height > 0) { + printf("ConnectBlock benchmark between height %d and %d inclusive\n", $start_height, $end_height); + } else { + printf("ConnectBlock logging starting at height %d\n", $start_height); + } + + if ($logging_threshold_ms > 0) { + printf("Logging blocks taking longer than %d ms to connect.\n", $3); + } + + if ($start_height == 0) { + @start = nsecs; + } +} + +/* + Attaches to the 'validation:block_connected' USDT and collects stats when the + connected block is between the start and end height (or the end height is + unset). +*/ +usdt:./src/bitcoind:validation:block_connected /arg1 >= $1 && (arg1 <= $2 || $2 == 0 )/ +{ + $height = arg1; + $transactions = arg2; + $inputs = arg3; + $sigops = arg4; + $duration = (uint64) arg5; + + @height = $height; + + @blocks = @blocks + 1; + @transactions = @transactions + $transactions; + @inputs = @inputs + $inputs; + @sigops = @sigops + $sigops; + + @durations = hist($duration / 1000); + + if ($height == $1 && $height != 0) { + @start = nsecs; + printf("Starting Connect Block Benchmark between height %d and %d.\n", $1, $2); + } + + if ($2 > 0 && $height >= $2) { + @end = nsecs; + $duration = @end - @start; + printf("\nTook %d ms to connect the blocks between height %d and %d.\n", $duration / 1000000, $1, $2); + exit(); + } +} + +/* + Attaches to the 'validation:block_connected' USDT and logs information about + blocks where the time it took to connect the block is above the + <logging threshold in ms>. +*/ +usdt:./src/bitcoind:validation:block_connected / (uint64) arg5 / 1000> $3 / +{ + $hash = arg0; + $height = (int32) arg1; + $transactions = (uint64) arg2; + $inputs = (int32) arg3; + $sigops = (int64) arg4; + $duration = (int64) arg5; + + + printf("Block %d (", $height); + /* Prints each byte of the block hash as hex in big-endian (the block-explorer format) */ + $p = $hash + 31; + unroll(32) { + $b = *(uint8*)$p; + printf("%02x", $b); + $p -= 1; + } + printf(") %4d tx %5d ins %5d sigops took %4d ms\n", $transactions, $inputs, $sigops, (uint64) $duration / 1000); +} + + +/* + Prints stats about the blocks, transactions, inputs, and sigops processed in + the last second (if any). +*/ +interval:s:1 { + if (@blocks > 0) { + printf("BENCH %4d blk/s %6d tx/s %7d inputs/s %8d sigops/s (height %d)\n", @blocks, @transactions, @inputs, @sigops, @height); + + zero(@blocks); + zero(@transactions); + zero(@inputs); + zero(@sigops); + } +} + +END +{ + printf("\nHistogram of block connection times in milliseconds (ms).\n"); + print(@durations); + + clear(@durations); + clear(@blocks); + clear(@transactions); + clear(@inputs); + clear(@sigops); + clear(@height); + clear(@start); + clear(@end); +} + diff --git a/contrib/tracing/log_p2p_traffic.bt b/contrib/tracing/log_p2p_traffic.bt new file mode 100755 index 0000000000..f62956aa5e --- /dev/null +++ b/contrib/tracing/log_p2p_traffic.bt @@ -0,0 +1,28 @@ +#!/usr/bin/env bpftrace + +BEGIN +{ + printf("Logging P2P traffic\n") +} + +usdt:./src/bitcoind:net:inbound_message +{ + $peer_id = (int64) arg0; + $peer_addr = str(arg1); + $peer_type = str(arg2); + $msg_type = str(arg3); + $msg_len = arg4; + printf("inbound '%s' msg from peer %d (%s, %s) with %d bytes\n", $msg_type, $peer_id, $peer_type, $peer_addr, $msg_len); +} + +usdt:./src/bitcoind:net:outbound_message +{ + $peer_id = (int64) arg0; + $peer_addr = str(arg1); + $peer_type = str(arg2); + $msg_type = str(arg3); + $msg_len = arg4; + + printf("outbound '%s' msg to peer %d (%s, %s) with %d bytes\n", $msg_type, $peer_id, $peer_type, $peer_addr, $msg_len); +} + diff --git a/contrib/tracing/log_raw_p2p_msgs.py b/contrib/tracing/log_raw_p2p_msgs.py new file mode 100755 index 0000000000..c0ab704106 --- /dev/null +++ b/contrib/tracing/log_raw_p2p_msgs.py @@ -0,0 +1,183 @@ +#!/usr/bin/env python3 +# Copyright (c) 2021 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +""" Demonstration of eBPF limitations and the effect on USDT with the + net:inbound_message and net:outbound_message tracepoints. """ + +# This script shows a limitation of eBPF when data larger than 32kb is passed to +# user-space. It uses BCC (https://github.com/iovisor/bcc) to load a sandboxed +# eBPF program into the Linux kernel (root privileges are required). The eBPF +# program attaches to two statically defined tracepoints. The tracepoint +# 'net:inbound_message' is called when a new P2P message is received, and +# 'net:outbound_message' is called on outbound P2P messages. The eBPF program +# submits the P2P messages to this script via a BPF ring buffer. The submitted +# messages are printed. + +# eBPF Limitations: +# +# Bitcoin P2P messages can be larger than 32kb (e.g. tx, block, ...). The eBPF +# VM's stack is limited to 512 bytes, and we can't allocate more than about 32kb +# for a P2P message in the eBPF VM. The message data is cut off when the message +# is larger than MAX_MSG_DATA_LENGTH (see definition below). This can be detected +# in user-space by comparing the data length to the message length variable. The +# message is cut off when the data length is smaller than the message length. +# A warning is included with the printed message data. +# +# Data is submitted to user-space (i.e. to this script) via a ring buffer. The +# throughput of the ring buffer is limited. Each p2p_message is about 32kb in +# size. In- or outbound messages submitted to the ring buffer in rapid +# succession fill the ring buffer faster than it can be read. Some messages are +# lost. +# +# BCC prints: "Possibly lost 2 samples" on lost messages. + +import sys +from bcc import BPF, USDT + +# BCC: The C program to be compiled to an eBPF program (by BCC) and loaded into +# a sandboxed Linux kernel VM. +program = """ +#include <uapi/linux/ptrace.h> + +#define MIN(a,b) ({ __typeof__ (a) _a = (a); __typeof__ (b) _b = (b); _a < _b ? _a : _b; }) + +// Maximum possible allocation size +// from include/linux/percpu.h in the Linux kernel +#define PCPU_MIN_UNIT_SIZE (32 << 10) + +// Tor v3 addresses are 62 chars + 6 chars for the port (':12345'). +#define MAX_PEER_ADDR_LENGTH 62 + 6 +#define MAX_PEER_CONN_TYPE_LENGTH 20 +#define MAX_MSG_TYPE_LENGTH 20 +#define MAX_MSG_DATA_LENGTH PCPU_MIN_UNIT_SIZE - 200 + +struct p2p_message +{ + u64 peer_id; + char peer_addr[MAX_PEER_ADDR_LENGTH]; + char peer_conn_type[MAX_PEER_CONN_TYPE_LENGTH]; + char msg_type[MAX_MSG_TYPE_LENGTH]; + u64 msg_size; + u8 msg[MAX_MSG_DATA_LENGTH]; +}; + +// We can't store the p2p_message struct on the eBPF stack as it is limited to +// 512 bytes and P2P message can be bigger than 512 bytes. However, we can use +// an BPF-array with a length of 1 to allocate up to 32768 bytes (this is +// defined by PCPU_MIN_UNIT_SIZE in include/linux/percpu.h in the Linux kernel). +// Also see https://github.com/iovisor/bcc/issues/2306 +BPF_ARRAY(msg_arr, struct p2p_message, 1); + +// Two BPF perf buffers for pushing data (here P2P messages) to user-space. +BPF_PERF_OUTPUT(inbound_messages); +BPF_PERF_OUTPUT(outbound_messages); + +int trace_inbound_message(struct pt_regs *ctx) { + int idx = 0; + struct p2p_message *msg = msg_arr.lookup(&idx); + + // lookup() does not return a NULL pointer. However, the BPF verifier + // requires an explicit check that that the `msg` pointer isn't a NULL + // pointer. See https://github.com/iovisor/bcc/issues/2595 + if (msg == NULL) return 1; + + bpf_usdt_readarg(1, ctx, &msg->peer_id); + bpf_usdt_readarg_p(2, ctx, &msg->peer_addr, MAX_PEER_ADDR_LENGTH); + bpf_usdt_readarg_p(3, ctx, &msg->peer_conn_type, MAX_PEER_CONN_TYPE_LENGTH); + bpf_usdt_readarg_p(4, ctx, &msg->msg_type, MAX_MSG_TYPE_LENGTH); + bpf_usdt_readarg(5, ctx, &msg->msg_size); + bpf_usdt_readarg_p(6, ctx, &msg->msg, MIN(msg->msg_size, MAX_MSG_DATA_LENGTH)); + + inbound_messages.perf_submit(ctx, msg, sizeof(*msg)); + return 0; +}; + +int trace_outbound_message(struct pt_regs *ctx) { + int idx = 0; + struct p2p_message *msg = msg_arr.lookup(&idx); + + // lookup() does not return a NULL pointer. However, the BPF verifier + // requires an explicit check that that the `msg` pointer isn't a NULL + // pointer. See https://github.com/iovisor/bcc/issues/2595 + if (msg == NULL) return 1; + + bpf_usdt_readarg(1, ctx, &msg->peer_id); + bpf_usdt_readarg_p(2, ctx, &msg->peer_addr, MAX_PEER_ADDR_LENGTH); + bpf_usdt_readarg_p(3, ctx, &msg->peer_conn_type, MAX_PEER_CONN_TYPE_LENGTH); + bpf_usdt_readarg_p(4, ctx, &msg->msg_type, MAX_MSG_TYPE_LENGTH); + bpf_usdt_readarg(5, ctx, &msg->msg_size); + bpf_usdt_readarg_p(6, ctx, &msg->msg, MIN(msg->msg_size, MAX_MSG_DATA_LENGTH)); + + outbound_messages.perf_submit(ctx, msg, sizeof(*msg)); + return 0; +}; +""" + + +def print_message(event, inbound): + print(f"%s %s msg '%s' from peer %d (%s, %s) with %d bytes: %s" % + ( + f"Warning: incomplete message (only %d out of %d bytes)!" % ( + len(event.msg), event.msg_size) if len(event.msg) < event.msg_size else "", + "inbound" if inbound else "outbound", + event.msg_type.decode("utf-8"), + event.peer_id, + event.peer_conn_type.decode("utf-8"), + event.peer_addr.decode("utf-8"), + event.msg_size, + bytes(event.msg[:event.msg_size]).hex(), + ) + ) + + +def main(bitcoind_path): + bitcoind_with_usdts = USDT(path=str(bitcoind_path)) + + # attaching the trace functions defined in the BPF program to the tracepoints + bitcoind_with_usdts.enable_probe( + probe="inbound_message", fn_name="trace_inbound_message") + bitcoind_with_usdts.enable_probe( + probe="outbound_message", fn_name="trace_outbound_message") + bpf = BPF(text=program, usdt_contexts=[bitcoind_with_usdts]) + + # BCC: perf buffer handle function for inbound_messages + def handle_inbound(_, data, size): + """ Inbound message handler. + + Called each time a message is submitted to the inbound_messages BPF table.""" + + event = bpf["inbound_messages"].event(data) + print_message(event, True) + + # BCC: perf buffer handle function for outbound_messages + + def handle_outbound(_, data, size): + """ Outbound message handler. + + Called each time a message is submitted to the outbound_messages BPF table.""" + + event = bpf["outbound_messages"].event(data) + print_message(event, False) + + # BCC: add handlers to the inbound and outbound perf buffers + bpf["inbound_messages"].open_perf_buffer(handle_inbound) + bpf["outbound_messages"].open_perf_buffer(handle_outbound) + + print("Logging raw P2P messages.") + print("Messages larger that about 32kb will be cut off!") + print("Some messages might be lost!") + while True: + try: + bpf.perf_buffer_poll() + except KeyboardInterrupt: + exit() + + +if __name__ == "__main__": + if len(sys.argv) < 2: + print("USAGE:", sys.argv[0], "path/to/bitcoind") + exit() + path = sys.argv[1] + main(path) diff --git a/contrib/tracing/log_utxocache_flush.py b/contrib/tracing/log_utxocache_flush.py new file mode 100755 index 0000000000..8c073bea0d --- /dev/null +++ b/contrib/tracing/log_utxocache_flush.py @@ -0,0 +1,107 @@ +#!/usr/bin/env python3 +# Copyright (c) 2021 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +import sys +import ctypes +from bcc import BPF, USDT + +"""Example logging Bitcoin Core utxo set cache flushes utilizing + the utxocache:flush tracepoint.""" + +# USAGE: ./contrib/tracing/log_utxocache_flush.py path/to/bitcoind + +# BCC: The C program to be compiled to an eBPF program (by BCC) and loaded into +# a sandboxed Linux kernel VM. +program = """ +# include <uapi/linux/ptrace.h> + +struct data_t +{ + u64 duration; + u32 mode; + u64 coins_count; + u64 coins_mem_usage; + bool is_flush_for_prune; +}; + +// BPF perf buffer to push the data to user space. +BPF_PERF_OUTPUT(flush); + +int trace_flush(struct pt_regs *ctx) { + struct data_t data = {}; + bpf_usdt_readarg(1, ctx, &data.duration); + bpf_usdt_readarg(2, ctx, &data.mode); + bpf_usdt_readarg(3, ctx, &data.coins_count); + bpf_usdt_readarg(4, ctx, &data.coins_mem_usage); + bpf_usdt_readarg(5, ctx, &data.is_flush_for_prune); + flush.perf_submit(ctx, &data, sizeof(data)); + return 0; +} +""" + +FLUSH_MODES = [ + 'NONE', + 'IF_NEEDED', + 'PERIODIC', + 'ALWAYS' +] + + +class Data(ctypes.Structure): + # define output data structure corresponding to struct data_t + _fields_ = [ + ("duration", ctypes.c_uint64), + ("mode", ctypes.c_uint32), + ("coins_count", ctypes.c_uint64), + ("coins_mem_usage", ctypes.c_uint64), + ("is_flush_for_prune", ctypes.c_bool) + ] + + +def print_event(event): + print("%-15d %-10s %-15d %-15s %-8s" % ( + event.duration, + FLUSH_MODES[event.mode], + event.coins_count, + "%.2f kB" % (event.coins_mem_usage/1000), + event.is_flush_for_prune + )) + + +def main(bitcoind_path): + bitcoind_with_usdts = USDT(path=str(bitcoind_path)) + + # attaching the trace functions defined in the BPF program + # to the tracepoints + bitcoind_with_usdts.enable_probe( + probe="flush", fn_name="trace_flush") + b = BPF(text=program, usdt_contexts=[bitcoind_with_usdts]) + + def handle_flush(_, data, size): + """ Coins Flush handler. + Called each time coin caches and indexes are flushed.""" + event = ctypes.cast(data, ctypes.POINTER(Data)).contents + print_event(event) + + b["flush"].open_perf_buffer(handle_flush) + print("Logging utxocache flushes. Ctrl-C to end...") + print("%-15s %-10s %-15s %-15s %-8s" % ("Duration (µs)", "Mode", + "Coins Count", "Memory Usage", + "Flush for Prune")) + + while True: + try: + b.perf_buffer_poll() + except KeyboardInterrupt: + exit(0) + + +if __name__ == "__main__": + if len(sys.argv) < 2: + print("USAGE: ", sys.argv[0], "path/to/bitcoind") + exit(1) + + path = sys.argv[1] + main(path) diff --git a/contrib/tracing/log_utxos.bt b/contrib/tracing/log_utxos.bt new file mode 100755 index 0000000000..54d5010f82 --- /dev/null +++ b/contrib/tracing/log_utxos.bt @@ -0,0 +1,86 @@ +#!/usr/bin/env bpftrace + +/* + + USAGE: + + bpftrace contrib/tracing/log_utxos.bt + + This script requires a 'bitcoind' binary compiled with eBPF support and the + 'utxocache' tracepoints. By default, it's assumed that 'bitcoind' is + located in './src/bitcoind'. This can be modified in the script below. + + NOTE: requires bpftrace v0.12.0 or above. +*/ + +BEGIN +{ + printf("%-7s %-71s %16s %7s %8s\n", + "OP", "Outpoint", "Value", "Height", "Coinbase"); +} + +/* + Attaches to the 'utxocache:add' tracepoint and prints additions to the UTXO set cache. +*/ +usdt:./src/bitcoind:utxocache:add +{ + $txid = arg0; + $index = (uint32)arg1; + $height = (uint32)arg2; + $value = (int64)arg3; + $isCoinbase = arg4; + + printf("Added "); + $p = $txid + 31; + unroll(32) { + $b = *(uint8*)$p; + printf("%02x", $b); + $p-=1; + } + + printf(":%-6d %16ld %7d %s\n", $index, $value, $height, ($isCoinbase ? "Yes" : "No" )); +} + +/* + Attaches to the 'utxocache:spent' tracepoint and prints spents from the UTXO set cache. +*/ +usdt:./src/bitcoind:utxocache:spent +{ + $txid = arg0; + $index = (uint32)arg1; + $height = (uint32)arg2; + $value = (int64)arg3; + $isCoinbase = arg4; + + printf("Spent "); + $p = $txid + 31; + unroll(32) { + $b = *(uint8*)$p; + printf("%02x", $b); + $p-=1; + } + + printf(":%-6d %16ld %7d %s\n", $index, $value, $height, ($isCoinbase ? "Yes" : "No" )); +} + +/* + Attaches to the 'utxocache:uncache' tracepoint and uncache UTXOs from the UTXO set cache. +*/ +usdt:./src/bitcoind:utxocache:uncache +{ + $txid = arg0; + $index = (uint32)arg1; + $height = (uint32)arg2; + $value = (int64)arg3; + $isCoinbase = arg4; + + printf("Uncache "); + $p = $txid + 31; + unroll(32) { + $b = *(uint8*)$p; + printf("%02x", $b); + $p-=1; + } + + printf(":%-6d %16ld %7d %s\n", $index, $value, $height, ($isCoinbase ? "Yes" : "No" )); +} diff --git a/contrib/tracing/p2p_monitor.py b/contrib/tracing/p2p_monitor.py new file mode 100755 index 0000000000..4ff701cac3 --- /dev/null +++ b/contrib/tracing/p2p_monitor.py @@ -0,0 +1,253 @@ +#!/usr/bin/env python3 +# Copyright (c) 2021 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +""" Interactive bitcoind P2P network traffic monitor utilizing USDT and the + net:inbound_message and net:outbound_message tracepoints. """ + +# This script demonstrates what USDT for Bitcoin Core can enable. It uses BCC +# (https://github.com/iovisor/bcc) to load a sandboxed eBPF program into the +# Linux kernel (root privileges are required). The eBPF program attaches to two +# statically defined tracepoints. The tracepoint 'net:inbound_message' is called +# when a new P2P message is received, and 'net:outbound_message' is called on +# outbound P2P messages. The eBPF program submits the P2P messages to +# this script via a BPF ring buffer. + +import sys +import curses +from curses import wrapper, panel +from bcc import BPF, USDT + +# BCC: The C program to be compiled to an eBPF program (by BCC) and loaded into +# a sandboxed Linux kernel VM. +program = """ +#include <uapi/linux/ptrace.h> + +// Tor v3 addresses are 62 chars + 6 chars for the port (':12345'). +// I2P addresses are 60 chars + 6 chars for the port (':12345'). +#define MAX_PEER_ADDR_LENGTH 62 + 6 +#define MAX_PEER_CONN_TYPE_LENGTH 20 +#define MAX_MSG_TYPE_LENGTH 20 + +struct p2p_message +{ + u64 peer_id; + char peer_addr[MAX_PEER_ADDR_LENGTH]; + char peer_conn_type[MAX_PEER_CONN_TYPE_LENGTH]; + char msg_type[MAX_MSG_TYPE_LENGTH]; + u64 msg_size; +}; + + +// Two BPF perf buffers for pushing data (here P2P messages) to user space. +BPF_PERF_OUTPUT(inbound_messages); +BPF_PERF_OUTPUT(outbound_messages); + +int trace_inbound_message(struct pt_regs *ctx) { + struct p2p_message msg = {}; + + bpf_usdt_readarg(1, ctx, &msg.peer_id); + bpf_usdt_readarg_p(2, ctx, &msg.peer_addr, MAX_PEER_ADDR_LENGTH); + bpf_usdt_readarg_p(3, ctx, &msg.peer_conn_type, MAX_PEER_CONN_TYPE_LENGTH); + bpf_usdt_readarg_p(4, ctx, &msg.msg_type, MAX_MSG_TYPE_LENGTH); + bpf_usdt_readarg(5, ctx, &msg.msg_size); + + inbound_messages.perf_submit(ctx, &msg, sizeof(msg)); + return 0; +}; + +int trace_outbound_message(struct pt_regs *ctx) { + struct p2p_message msg = {}; + + bpf_usdt_readarg(1, ctx, &msg.peer_id); + bpf_usdt_readarg_p(2, ctx, &msg.peer_addr, MAX_PEER_ADDR_LENGTH); + bpf_usdt_readarg_p(3, ctx, &msg.peer_conn_type, MAX_PEER_CONN_TYPE_LENGTH); + bpf_usdt_readarg_p(4, ctx, &msg.msg_type, MAX_MSG_TYPE_LENGTH); + bpf_usdt_readarg(5, ctx, &msg.msg_size); + + outbound_messages.perf_submit(ctx, &msg, sizeof(msg)); + return 0; +}; +""" + + +class Message: + """ A P2P network message. """ + msg_type = "" + size = 0 + data = bytes() + inbound = False + + def __init__(self, msg_type, size, inbound): + self.msg_type = msg_type + self.size = size + self.inbound = inbound + + +class Peer: + """ A P2P network peer. """ + id = 0 + address = "" + connection_type = "" + last_messages = list() + + total_inbound_msgs = 0 + total_inbound_bytes = 0 + total_outbound_msgs = 0 + total_outbound_bytes = 0 + + def __init__(self, id, address, connection_type): + self.id = id + self.address = address + self.connection_type = connection_type + self.last_messages = list() + + def add_message(self, message): + self.last_messages.append(message) + if len(self.last_messages) > 25: + self.last_messages.pop(0) + if message.inbound: + self.total_inbound_bytes += message.size + self.total_inbound_msgs += 1 + else: + self.total_outbound_bytes += message.size + self.total_outbound_msgs += 1 + + +def main(bitcoind_path): + peers = dict() + + bitcoind_with_usdts = USDT(path=str(bitcoind_path)) + + # attaching the trace functions defined in the BPF program to the tracepoints + bitcoind_with_usdts.enable_probe( + probe="inbound_message", fn_name="trace_inbound_message") + bitcoind_with_usdts.enable_probe( + probe="outbound_message", fn_name="trace_outbound_message") + bpf = BPF(text=program, usdt_contexts=[bitcoind_with_usdts]) + + # BCC: perf buffer handle function for inbound_messages + def handle_inbound(_, data, size): + """ Inbound message handler. + + Called each time a message is submitted to the inbound_messages BPF table.""" + event = bpf["inbound_messages"].event(data) + if event.peer_id not in peers: + peer = Peer(event.peer_id, event.peer_addr.decode( + "utf-8"), event.peer_conn_type.decode("utf-8")) + peers[peer.id] = peer + peers[event.peer_id].add_message( + Message(event.msg_type.decode("utf-8"), event.msg_size, True)) + + # BCC: perf buffer handle function for outbound_messages + def handle_outbound(_, data, size): + """ Outbound message handler. + + Called each time a message is submitted to the outbound_messages BPF table.""" + event = bpf["outbound_messages"].event(data) + if event.peer_id not in peers: + peer = Peer(event.peer_id, event.peer_addr.decode( + "utf-8"), event.peer_conn_type.decode("utf-8")) + peers[peer.id] = peer + peers[event.peer_id].add_message( + Message(event.msg_type.decode("utf-8"), event.msg_size, False)) + + # BCC: add handlers to the inbound and outbound perf buffers + bpf["inbound_messages"].open_perf_buffer(handle_inbound) + bpf["outbound_messages"].open_perf_buffer(handle_outbound) + + wrapper(loop, bpf, peers) + + +def loop(screen, bpf, peers): + screen.nodelay(1) + cur_list_pos = 0 + win = curses.newwin(30, 70, 2, 7) + win.erase() + win.border(ord("|"), ord("|"), ord("-"), ord("-"), + ord("-"), ord("-"), ord("-"), ord("-")) + info_panel = panel.new_panel(win) + info_panel.hide() + + ROWS_AVALIABLE_FOR_LIST = curses.LINES - 5 + scroll = 0 + + while True: + try: + # BCC: poll the perf buffers for new events or timeout after 50ms + bpf.perf_buffer_poll(timeout=50) + + ch = screen.getch() + if (ch == curses.KEY_DOWN or ch == ord("j")) and cur_list_pos < len( + peers.keys()) -1 and info_panel.hidden(): + cur_list_pos += 1 + if cur_list_pos >= ROWS_AVALIABLE_FOR_LIST: + scroll += 1 + if (ch == curses.KEY_UP or ch == ord("k")) and cur_list_pos > 0 and info_panel.hidden(): + cur_list_pos -= 1 + if scroll > 0: + scroll -= 1 + if ch == ord('\n') or ch == ord(' '): + if info_panel.hidden(): + info_panel.show() + else: + info_panel.hide() + screen.erase() + render(screen, peers, cur_list_pos, scroll, ROWS_AVALIABLE_FOR_LIST, info_panel) + curses.panel.update_panels() + screen.refresh() + except KeyboardInterrupt: + exit() + + +def render(screen, peers, cur_list_pos, scroll, ROWS_AVALIABLE_FOR_LIST, info_panel): + """ renders the list of peers and details panel + + This code is unrelated to USDT, BCC and BPF. + """ + header_format = "%6s %-20s %-20s %-22s %-67s" + row_format = "%6s %-5d %9d byte %-5d %9d byte %-22s %-67s" + + screen.addstr(0, 1, (" P2P Message Monitor "), curses.A_REVERSE) + screen.addstr( + 1, 0, (" Navigate with UP/DOWN or J/K and select a peer with ENTER or SPACE to see individual P2P messages"), curses.A_NORMAL) + screen.addstr(3, 0, + header_format % ("PEER", "OUTBOUND", "INBOUND", "TYPE", "ADDR"), curses.A_BOLD | curses.A_UNDERLINE) + peer_list = sorted(peers.keys())[scroll:ROWS_AVALIABLE_FOR_LIST+scroll] + for i, peer_id in enumerate(peer_list): + peer = peers[peer_id] + screen.addstr(i + 4, 0, + row_format % (peer.id, peer.total_outbound_msgs, peer.total_outbound_bytes, + peer.total_inbound_msgs, peer.total_inbound_bytes, + peer.connection_type, peer.address), + curses.A_REVERSE if i + scroll == cur_list_pos else curses.A_NORMAL) + if i + scroll == cur_list_pos: + info_window = info_panel.window() + info_window.erase() + info_window.border( + ord("|"), ord("|"), ord("-"), ord("-"), + ord("-"), ord("-"), ord("-"), ord("-")) + + info_window.addstr( + 1, 1, f"PEER {peer.id} ({peer.address})".center(68), curses.A_REVERSE | curses.A_BOLD) + info_window.addstr( + 2, 1, f" OUR NODE{peer.connection_type:^54}PEER ", + curses.A_BOLD) + for i, msg in enumerate(peer.last_messages): + if msg.inbound: + info_window.addstr( + i + 3, 1, "%68s" % + (f"<--- {msg.msg_type} ({msg.size} bytes) "), curses.A_NORMAL) + else: + info_window.addstr( + i + 3, 1, " %s (%d byte) --->" % + (msg.msg_type, msg.size), curses.A_NORMAL) + + +if __name__ == "__main__": + if len(sys.argv) < 2: + print("USAGE:", sys.argv[0], "path/to/bitcoind") + exit() + path = sys.argv[1] + main(path) diff --git a/contrib/valgrind.supp b/contrib/valgrind.supp new file mode 100644 index 0000000000..6efe49254b --- /dev/null +++ b/contrib/valgrind.supp @@ -0,0 +1,137 @@ +# This valgrind suppressions file includes known Valgrind warnings in our +# dependencies that cannot be fixed in-tree. +# +# Example use: +# $ valgrind --suppressions=contrib/valgrind.supp src/test/test_bitcoin +# $ valgrind --suppressions=contrib/valgrind.supp --leak-check=full \ +# --show-leak-kinds=all src/test/test_bitcoin +# +# To create suppressions for found issues, use the --gen-suppressions=all option: +# $ valgrind --suppressions=contrib/valgrind.supp --leak-check=full \ +# --show-leak-kinds=all --gen-suppressions=all --show-reachable=yes \ +# --error-limit=no src/test/test_bitcoin +# +# Note that suppressions may depend on OS and/or library versions. +# Tested on: +# * aarch64 (Ubuntu 22.04 system libs, clang, without gui) +# * x86_64 (Ubuntu 22.04 system libs, clang, without gui) +{ + Suppress libstdc++ warning - https://gcc.gnu.org/bugzilla/show_bug.cgi?id=65434 + Memcheck:Leak + match-leak-kinds: reachable + fun:malloc + obj:*/libstdc++.* + fun:call_init.part.0 + fun:call_init + fun:_dl_init + obj:*/ld-*.so +} +{ + Suppress libdb warning - https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=662917 + Memcheck:Cond + obj:*/libdb_cxx-*.so + fun:__log_put +} +{ + Suppress libdb warning + Memcheck:Param + pwrite64(buf) + fun:pwrite + fun:__os_io +} +{ + Suppress libdb warning + Memcheck:Cond + fun:__log_putr.isra.1 +} +{ + Suppress libdb warning + Memcheck:Param + pwrite64(buf) + ... + obj:*/libdb_cxx-*.so +} +{ + Suppress uninitialized bytes warning in compat code + Memcheck:Param + ioctl(TCSET{S,SW,SF}) + fun:tcsetattr +} +{ + Suppress libdb warning + Memcheck:Leak + fun:malloc + ... + obj:*/libdb_cxx-*.so +} +{ + Suppress leaks on init + Memcheck:Leak + ... + fun:_Z11AppInitMainR11NodeContext +} +{ + Suppress leaks on shutdown + Memcheck:Leak + ... + fun:_Z8ShutdownR11NodeContext +} +{ + Ignore GUI warning + Memcheck:Leak + ... + obj:/usr/lib64/libgdk-3.so.0.2404.7 +} +{ + Suppress leveldb warning (leveldb::InitModule()) - https://github.com/google/leveldb/issues/113 + Memcheck:Leak + match-leak-kinds: reachable + fun:_Znwm + fun:_ZN7leveldbL10InitModuleEv +} +{ + Suppress leveldb warning (leveldb::Env::Default()) - https://github.com/google/leveldb/issues/113 + Memcheck:Leak + match-leak-kinds: reachable + fun:_Znwm + ... + fun:_ZN7leveldbL14InitDefaultEnvEv +} +{ + Suppress leveldb leak + Memcheck:Leak + match-leak-kinds: reachable + fun:_Znwm + ... + fun:_ZN7leveldb6DBImpl14BackgroundCallEv +} +{ + Suppress leveldb leak + Memcheck:Leak + fun:_Znwm + ... + fun:GetCoin +} +{ + Suppress LogInstance still reachable memory warning + Memcheck:Leak + match-leak-kinds: reachable + fun:_Znwm + fun:_Z11LogInstancev +} +{ + Suppress secp256k1_context_create still reachable memory warning + Memcheck:Leak + match-leak-kinds: reachable + fun:malloc + ... + fun:secp256k1_context_create +} +{ + Suppress BCLog::Logger::StartLogging() still reachable memory warning + Memcheck:Leak + match-leak-kinds: reachable + fun:malloc + ... + fun:_ZN5BCLog6Logger12StartLoggingEv +} diff --git a/contrib/verify-commits/README.md b/contrib/verify-commits/README.md new file mode 100644 index 0000000000..b8b15280ba --- /dev/null +++ b/contrib/verify-commits/README.md @@ -0,0 +1,57 @@ +Tooling for verification of PGP signed commits +---------------------------------------------- + +This is an incomplete work in progress, but currently includes a pre-push hook +script (`pre-push-hook.sh`) for maintainers to ensure that their own commits +are PGP signed (nearly always merge commits), as well as a Python 3 script to verify +commits against a trusted keys list. + + +Using verify-commits.py safely +------------------------------ + +Remember that you can't use an untrusted script to verify itself. This means +that checking out code, then running `verify-commits.py` against `HEAD` is +_not_ safe, because the version of `verify-commits.py` that you just ran could +be backdoored. Instead, you need to use a trusted version of verify-commits +prior to checkout to make sure you're checking out only code signed by trusted +keys: + + ```sh + git fetch origin && \ + ./contrib/verify-commits/verify-commits.py origin/master && \ + git checkout origin/master + ``` + +Note that the above isn't a good UI/UX yet, and needs significant improvements +to make it more convenient and reduce the chance of errors; pull-reqs +improving this process would be much appreciated. + +Configuration files +------------------- + +* `trusted-git-root`: This file should contain a single git commit hash which is the first unsigned git commit (hence it is the "root of trust"). +* `trusted-sha512-root-commit`: This file should contain a single git commit hash which is the first commit without a SHA512 root commitment. +* `trusted-keys`: This file should contain a \n-delimited list of all PGP fingerprints of authorized commit signers (primary, not subkeys). +* `allow-revsig-commits`: This file should contain a \n-delimited list of git commit hashes. See next section for more info. + +Import trusted keys +------------------- +In order to check the commit signatures, you must add the trusted PGP keys to your machine. [GnuPG](https://gnupg.org/) may be used to import the trusted keys by running the following command: + +```sh +gpg --keyserver hkps://keys.openpgp.org --recv-keys $(<contrib/verify-commits/trusted-keys) +``` + +Key expiry/revocation +--------------------- + +When a key (or subkey) which has signed old commits expires or is revoked, +verify-commits will start failing to verify all commits which were signed by +said key. In order to avoid bumping the root-of-trust `trusted-git-root` +file, individual commits which were signed by such a key can be added to the +`allow-revsig-commits` file. That way, the PGP signatures are still verified +but no new commits can be signed by any expired/revoked key. To easily build a +list of commits which need to be added, verify-commits.py can be edited to test +each commit with BITCOIN_VERIFY_COMMITS_ALLOW_REVSIG set to both 1 and 0, and +those which need it set to 1 printed. diff --git a/contrib/verify-commits/allow-incorrect-sha512-commits b/contrib/verify-commits/allow-incorrect-sha512-commits new file mode 100644 index 0000000000..c572806f26 --- /dev/null +++ b/contrib/verify-commits/allow-incorrect-sha512-commits @@ -0,0 +1,2 @@ +f8feaa4636260b599294c7285bcf1c8b7737f74e +8040ae6fc576e9504186f2ae3ff2c8125de1095c diff --git a/contrib/verify-commits/allow-revsig-commits b/contrib/verify-commits/allow-revsig-commits new file mode 100644 index 0000000000..0bb299b8fa --- /dev/null +++ b/contrib/verify-commits/allow-revsig-commits @@ -0,0 +1,645 @@ +a06ede9a138d0fb86b0de17c42b936d9fe6e2158 +923dc447eaa8e017985b2afbbb12dd1283fbea0e +71148b8947fe8b4d756822420a7f31c380159425 +6696b4635ceb9b47aaa63244bff9032fa7b08354 +812714fd80e96e28cd288c553c83838cecbfc2d9 +8a445c5651edb9a1f51497055b7ddf4402be9188 +e126d0c12ca66278d9e7b12187c5ff4fc02a7e6c +3908fc4728059719bed0e1c7b1c8b388c2d4a8da +8b66bf74e2a349e71eaa183af81fa63eaee76ad2 +05950427d310654774031764a7141a1a4fd9c6e4 +07fd147b9f12e9205afd66a624edce357977d615 +12e31127948fa4bb01c3bddc1b8c85b432f7465b +8c87f175d335e9d9e93f987d871ae9f05f6a10a7 +46b249e578e8a3dfbe85bc7253a12e82ef4b658b +a55716abe5662ec74c2f8af93023f1e7cca901fc +f646275b90b1de93bc62b4c4d045d75ac0b96eee +c252685aa5867631e9a5ef07ccae7c7c25cae8ff +a7d55c93385359952d85decd5037843ac70ba3d4 +7dac1e5e9e887f5f6ff146e812a05bd3bf281eae +2a524b8e8fe69ef487fd8ea1b4f7a03f473ed201 +ce5c1f4acae43477989cdf9a82ed33703919cda2 +2db4cbcc437f51f5dac82cc4de46f383b92e6f11 +7aa700424cbda387536373d8dfec88aee43f950e +b99a093afed880f23fb279c443cc6ae5e379cc43 +b83264d9c7a8ddb79f64bd9540caddc8632ef31f +57e337d40e94ba33d8cd265c134d6ef857b32b59 +a1dcf2e1087beaf3981739fd2bb74f35ecad630a +d38b0d7a6b6056cba26999b702815775e2437d87 +815640ec6af9a38d6a2da4a4400056e2f4105080 +09c4fd157c5b88df2d97fad4826c79b094db90c9 +2efcfa5acfacb958973d9e8125e1d81f102e2dfd +dc6dee41f7cf2ba93fcd0fea7c157e4b2775d439 +ad826b3df9f763b49f1e3e3d50c4efdd438c7547 +c1a52276848d8caa9a9789dff176408c1aa6b1ed +3bf06e9bac57b5b5a746677b75e297a7b154bdbd +72ae6f8cf0224370e8121d6769b21e612ca15d6f +a143b88dbd4971ecfdd1d39a494489c8f2db0344 +76fec09d878d6dbf214bdb6228d480bd9195db4c +93566e0c37c5ae104095474fea89f00dcb40f551 +407d9232ef5cb1ebf6cff21f3d13e07ea4158eeb +9346f8429957e356d21c665bab59fe45bcf1f74e +6eeac6e30d65f9a972067c1ea8c49978c8e631ac +dc6b9406bdfab2af8c86cb080cb3e6cf8f2385d8 +9f554e03ebe5701c1b75ff03b3d6152095c0cad3 +05009935f9ac070197113954d680bc2c9150b9b3 +508404de98a8a5435f52916cef8f328e82651961 +ed0cc50afed146c27f6d8129c683c225fb940093 +6429cfa8a70308241c576aeb92ffe3db5203b2ef +6898213409811b140843c3d89af43328c3b22fad +5b2ea29cf4fd298346437bb16a54407f8c1f9dca +e2a1a1ee895149c544d4ae295466611f0cec3094 +e82fb872ff5cc8fd22d43327c1ee3e755f61c562 +19b0f33de0efd9da788e8e4f3fdc2a9e159abdb1 +89de1538ce1f8c00f80e8d11f43e1b77e24d7dea +de07fdcf77e97b8613091285e4d0a734f5de7492 +01680195f8aa586c55c44767397380def3a23b54 +05e1c85fb687c82ae477c72d4a7e2d6b0c692167 +c072b8fd95cd4fa84f08189a0cd8b173ea2dbb8e +9a0ed08b40b15ae2b791aa8549b53e69934b4ea7 +53f8f226bd1d627c4a6dec5862a1d4ea5a933e45 +9d0f43b7ca7241d8a018fd35dd3bc01555235ec6 +f12d2b5a8ac397e4bcaefcc19898f8ff5705dea5 +8250de13587ed05ca45df3e12c5dc9bcb1500e2c +d727f77e390426e9e463336bda08d50c451c7086 +484312bda2d43e3ea60047be076332299463adf8 +c7e05b35ab0a791c7a8e2d863e716fdec6f3f671 +b9c1cd81848da9de1baf9c2f29c19c50e549de13 +8ea7d31e384975019733b5778feabbd9955c79d8 +f798b891bcecea9548eedacae70eeb9906c1ddbf +ebefe7a00b46579cdd1e033a8c7fd8ce9aa578e4 +ad087638ee4864d6244ec9381ff764bfa6ee5086 +66db2d62d59817320c9182fc18e75a93b76828ea +7ce9ac5c83b1844a518ef2e12e87aae3cacdfe58 +4286f43025149cf44207c3ad98e4a1f068520ada +cd0c5135ab2291aaa5410ac919bad3fc87249a4a +66ed450d771a8fc01c159a8402648ebd1c35eb4c +a82f03393a32842d49236e8666ee57805ca701f8 +f972b04d63eb8af79ff3cec1dc561ed13dfa6053 +ec45cc5e27668171b55271b0c735194c70e7da41 +715e9fd7454f7a48d7adba7d42f662c20a3e3367 +2e0a99037dcc35bc63ba0d54371bc678af737c8e +7fa8d758598407f3bf0beb0118dc122ea5340736 +6a22373771edbc3c7513cacb9355f880c73c2cbf +b89ef131147f71a96152a7b5c4374266cdf539b2 +01d8359983e2f77b5118fede3ffa947072c666c8 +58f0c929a3d70a4bff79cc200f1c186f71ef1675 +950be19727a581970591d8f8138dfe4725750382 +425278d17bd0edf8a3a7cc81e55016f7fd8e7726 +c028c7b7557da2baff7af8840108e8be4db8e0c6 +47a7cfb0aa2498f6801026d258a59f9de48f60b0 +f6b7df3155ddb4cedfbcf5d3eb3383d4614b3a85 +d72098038f3b55a714ed8adb34fab547b15eb0d5 +c49c825bd9f4764536b45df5a684d97173673fc7 +33799afe83eec4200ff140e9bf5eae83701a4d7f +5c3f8ddcaa1164079105c452429fccf8127b01b6 +1f01443567b03ac75a91c810f1733f5c21b5699d +b3e42b6d02e8d19658a9135e427ebceab5367779 +69b3a6dd9d9a0adf5506c8b9fde42187356bd4a8 +bafd075c5e6a1088ef0f1aa0b0b224e026a3d3e0 +7daa3adb242d9c8728fdb15c6af6596aaad5502f +514993554c370f4cf30a109ac28d5d64893dbf0a +c8d2473e6cb042e7275a10c49d3f6a4a91bf0166 +386f4385ab04b0b2c3d47bddc0dc0f2de7354964 +9f33dba05c01ecc5c56eb1284ab7d64d42f55171 +7466a26cab5d66665991433947964a638f5b957e +b43aba89e356ff95b706e80d4802f60fc46a569a +02b7e8319aef2a870264ad4fa2e3bb18664dcc36 +f686002a8eba820a40ac2f34a6e8f57b2b5cc54c +2b1c50b9352ab1dc40b0f877db23c1fa4048fae3 +2405ce1df043f778b8efb9205009500cbc17313a +4ad3b3c72c73d61e0a0cab541dca20acf651320d +4ba3d4f4393d81148422d24d222fe7ed00130194 +8ee5c7b747171e335793c74cd9d2f7491da58164 +872c921c0a208b04bd0713758e52fcab5b7c1684 +00d1680498c5550e7db1f359202d3433a092fafd +585db41e9ab7a6fb262c8bad7f427cdbdc497188 +18462960c0f13bd07d8f52b61e7d7bc17e991eea +0630974647dacaf25e7fcb7f9cbb785bb078ede6 +0f58d7f3d62f012f2584f5e781fc73de4763dd9e +3d16f581538b0974853e820508e8b3093269d2fd +66e91420ab233cf1dac64504e0dc129019bf8c0d +d8d9162f5bad39b2720dd2b2da237c6159e4755f +29fad97c320c892ab6a480c81e2078ec22ab354b +791c3ea61b4e49fd46a1a71b84ca99ddf69d2ff7 +a312e201ba56742499a5480b5f2115f01505c217 +ce56fdd2e8cdf94fd0ab76d71adbfa755e23ce7d +480f42630cbd598c04fa59ee0e406f56904ecffb +6012f1caf744ac9b53383d7d10a8f1b70ca2c0e1 +ded6a2afa549f693dcabb430ce0862f8631360c8 +07090c5339436f856e79a8036d1c85deeb453803 +0e265916d1c6a63e4a3821dab9db597b5ec64b46 +e4ffcacc2187d3419c8ea12b82fb06d82d8751d2 +e117cfe45eee9169409e74a44ef4a866be25bc35 +dcfe218626b05204e9fbc95ba5d95ca0eb72ec9b +23481fa50301201ef5a60675ef899aa6ce94ca03 +27c59dc502f29cf1d76290556c21e366145e3b2e +4a62ddd01873d18dbca96c81d756be1020249b45 +a233fb4f1d037e68ff70eef3a9f5b7bf1d631918 +b2089c51cc4af2f7e1c0ec75be9449ee222b1d69 +c997f8808256521397f1c003bb1e9896fee6eaa0 +5dc00f68c49c46a380a98d06233f90528b8e2557 +fe53d5f3636aed064823bc220d828c7ff08d1d52 +935eb8de039dec65669a96a1c3b86f4b03a1b86c +0277173b1defb63216d40a8d8805ae6d5d563c26 +2a30e67d20f76bbcd9a7d445f616f005316e0a1a +d32528e733f2711b34dbc41fbb2bb0f153bf7e9a +4cad91663df381d0dff8526f3b4aa74569dfb626 +1b06ed136f17b526360617a70026aed5ded5746c +895fbd768f0c89cea3f78acac58b233d4e3a145e +f0295becbf3ef1fb78095306408789253fe0c114 +8d573198638e52e2dbd9abc609861430f9d2bcc3 +9d9c4185fadaf243bb97c226e2fef16b65299699 +eebe4580bc8d6484d79ecb24dd87412221cf2ea7 +9cf6393a4f82b9c81d3b4b468a17a89db10531a2 +598a9c4e4dcd03c6d80fba005de729a6a3aeba7e +6970b30c6f1d2be7947295fe18f2390649b17a4b +f359afcc410432ed5d30001acda0c66741ee8935 +126000ba9e7ff16271be2f4eef3df99ade8d624f +b5e4b9b5100ec15217d43edb5f4149439f4b20a5 +b987ca4ee495a7fff82f0ac14ef0753bfb7586e2 +b03013396cb2f4bf25746388b3982a2c3616e16b +9a97f39afaa890caa7987c6bc001b9a66e3e74e8 +cad504bf4c302f7a72e0a0e191f3fdbafda7340f +45cf8a03cb57b8639a8d47323bde46ba22d9eeaf +b7450cdbd89a1c862f4d4d8bf093f8a0b5448f9c +0910cbe4ef31eb95fd76c7c2f820419fe64a3150 +92a810d04b906722c9efe60e3997243c71ff3d4c +45173fa6fca9537abb0a0554f731d14b9f89c456 +fd4ca17360e6fc0c9bb76bf6b5b07c9102c12728 +ddff3447f29b62d79a33f728791f42fa9436216e +36a5a4404836da323c755523fbd27563a8e84f94 +c991b304dee368f506cfee27ddaa333f1f82c518 +d38d1a3e75aa97ffa8755ddd431754a6d0942964 +a332a7d5a15214015f9553fdb2bcf80a1a4b8dc0 +604e08c83cf58ca7e7cda2ab284c1ace7bb12977 +18a1bbad98bd4321f15e7921d9aec91661499d90 +8049241e226c16bd07b029c0cb4b62ac40f0c923 +797441ee995aac59f55d59a93ecb55e8ecbe7dbc +62fdf9b07087b80d2142799bdd2324f61483359d +f60b4ad57912b78a96af08046a503f7905610a8c +13e31dd6548d64a5992f439e74bb424bf88aca04 +fbce66a982679b5409a295be5c99a2eef429cabf +9f2c2dba21855b8cb9b193b1819be73fa4a23a99 +a89221873a3ee2451c73b41bbe2d99d36f439d31 +3d6ad407770e13958e157bf026cae0bfb9254899 +901ba3e3819405306414628306746552b0aa1d28 +7a43fbb959c38e025e558e472ad57de357539894 +0d89fa0877930c6c8a539a656c1009ad8ab6755b +54aedc013744c86b11157423fa3cffc9a51eef02 +f0c1f8abb0182da557d07372b938f3a0a4bb906f +4ed818060ecf4a38a02c8cb48f6cbc78d2ee7708 +3bdf242fc68a8d767932c6214455d4d413effbc9 +5e468994fbb349e8eefc996954a31a67a34aaa15 +41aa9c4a801a01eca1fad22a7095372d23dace60 +2adbddb03840ad71e843c6c4a207a13e871cd1d4 +13e352dc53dec0127c5f94a60055d0ca829420dc +95e14dc81dd30ee0d396ad08dca9a6980d16eee1 +61fb80660f73e5aa5b69302ecc7ac33da206ba5a +05a761932edd05cf94ffe938908baf058f38632a +ee92243e66f2df03b3a759a8ffb75dc06f0cea0d +22cdf93c062eeaa0f8f9d6220f01b67240073dfb +76b33491596736ca804e3a29bd8398d7a1516ab7 +6e4e98ee8ce2da3cca2e2fd210e9e8dbc9b1c936 +c838283ecdfb9490425bb071b7c22e542de46c7c +5e3f5e4f25b65b583d3bfefac9e1148035781089 +f7388e93d3dd91a90239aedac4ec58404f103a2e +0a2f46b0158b6fc7244a585913b0925c0acf707f +dd561667cb7ccbbfed3134b05a565971ef6f5873 +6f01dcf63873a5e42798635ab4026c9a5f9fa213 +70fec9e36bcd1a3d93df019be084aaf89cecd7d7 +f9b74ef3fc74fd7d2aa94560820341f03cda8e12 +998c3046fab2b52bc9f141cfb588a18c05506a86 +89cc4f905e30b913ca20e4192d538cc5cbe2c38d +87d90efd69b64f769116956a5db89e536e9e3714 +5aeaa9ccd1568a77e075dbe2bd2435bd60c87c91 +bfb270acfa30713dc8c968bb9ee40cf5a2360359 +1b8c88451b0554502435d3883c528ad0aad1b09b +57ee73990f1ce29916adfd99f93eae1ccea1a43b +808c84f89d0edcef9ddaab0b849a382719f6ec9e +14b860bf64020451ced823b859da8cb912278ab9 +c63364610f4a041df1c1bd81d01b1f6856160749 +92eadc395071876d77f3babddc056b4325bdbabc +e93fff1463ae906fc986bf98c3b118c82f171546 +9ccafb1d7bdd172a9b963444072a844da379c4f7 +b4a509a3f817121c3df98ddfd96b2769e18a3e5a +dbc4ae03963014ab4b7957d62ba59dbd8f938c33 +8ddf60db7ad636b6a31b590251c671ded635fa1d +f199b8a33d9443a258a1f49a1a29674cd9ee9a20 +e542728cde676f218c552d841d0af29b92f9800b +763231051596b8e3455b839911ad6a3a1f1c3c74 +ff4cd6075b12fb32b9a906deea3ed033e3f9560a +9c3c9cdae3e20b5bdea91a0631edac5116bbc89f +93d20a734d2ee873832bed8ca5c05cf8e539c53c +ef8340d25f7c5dd5682bdecea97ce84cfce1493c +69c7ecef405d168f658a9cc7996da84c17f61e66 +4ce2f3d0d33346e9f0e96851689ee6550b2a72e3 +44e1fd926cfb0df0fbd8c41de8cd65ed8d5d6e18 +d6d2c8503c4039b682196d83a67dc28359c10c5c +ae233c4ec3d14a97c6195059f52873cdba2b4755 +0f399a9ff227896265cafab9b2e9fab6cdb9b5b9 +f4ed44ab4a8f9a87ba678d5fd1449fbf636103dc +7fcd61b2613c211bb042a82a889655178be6a212 +42973f834445d7735738bdba8847812ba3c34d95 +8df48b36ed3201d938b9974ecbee455d7dc2fb84 +96ac26e56627f0c24213fcd3a1cce9fc95f1f661 +cce94c518a46b7b0006f984bbe4d69e8749182d2 +801dd40666d1e6009920ad3ff755c7bb993b2a62 +ce829855cfca103dde55661fa1524e66b139d063 +b148803b181e30213e8a7f3bd89c8239e9dcb866 +c377feaad87f8109f85da6caf62602b30c20effc +b37cab65c63e051ebc5b491da9bd687581df94df +16e41844e7d6c5876d2caaeef6010656950c6ec5 +ee50c9e48786dea0d9df2e45805c25565c100fe3 +11dacc6154c42bc6fe3ba94c1823f8a46e4fe81a +791a0e6ddade27d1b69f4861a6640de60b9553cf +638e6c59da4fad987c437592174b188510193b2e +52f8877525d5238f3440e73710507be889d14127 +2a56baf395bf11835d784c4f8634f4525deed6a1 +bc561b4b7d6a3f71649d37d5eb9047c29efa2b13 +31809d6f8514c4a8d5677e947e3f1ebb0db210b9 +a31e9ad4f027955d43c04a05517244647e250161 +777519bd96f68c18150a0f5942f8f97a91937f5e +4eb1f39d421024d9666cec61deaf96715ffae4c6 +50fae68d416b4b8ec4ca192923dfd5ae9ea42773 +ce665863b137ac4a7470cf006a92aa7694faca71 +81f8c0378b2ab5ea0d7b65635cb529bd3c69127c +108222b9c323a05cc9339368f10ddd0859f62b43 +28f788e47e58f2b462351d6989348a4e1a241b2b +d81dccf191a48a6b59c3747d7b4ccbe3535dde40 +a90e6d2bffc422ddcdb771c53aac0bceb970a2c4 +91e49c51f1aecc9e1d75457f4920d52a4b0a133c +60dd9cc470584960431de425e2a9ffbed0e8034a +ede386c2193fc31351e193b3a8cf30030d6be62c +a084767b40c0d3ba8fa8f8d60f1e8d99a9dc3457 +3f726c99f819f97f2ab21b94d34c6b3129cd883a +77fc469fc78cdd87c29f398d46ac58dbb9ef62c0 +4ae6d0fbef60ccbecf8f23bb482e201b3678f7a3 +8858b6ddd3bce9daa08da6e05de3ca863a399c15 +22e301a3d56dc9e6878380ee92c7d19ca43119d2 +c484ec6c9b85ca4e331e395c564ae232fd0681dd +a46a671e253528e450bd57645c400bf761da07ab +655970d9c60ae6850daf452457e14e21047c0e1b +b6a48914c50631914192aa11b19205436a9c664d +7db65c363a0cc6ca7cdb04de9a973ab70013baad +6366941275344dac7e2130b0c972e90117d37ed0 +4fb2586661471a1572c2df2a5a091011d45eb7c4 +d7be7b39fa1021ec4518186afe145ee948e12a94 +85aec87b11ec41295558175c63f1f5a849460fdf +aeb31756276034dd506fdf97c8aaade0e7e584f5 +ac016e17d20253129a0287cee7e1d06b7ef15966 +bf74d377fb8e20140da6eac1407414928384bcea +2c811e08db651a4aed6ea0f7c1972d60de6de8ab +e5d26e47c7a482c072a7fe47bb84c56854734184 +96a63a3e0cefe920819bd42add0041837b1214a1 +e526ca6284b9e13be1b912b80dd73a34e739b539 +ecd21357f16106e541e9c2854ead2a906659b938 +4b5a7ce0c301ad971f383eb60f61bf9b4026efda +929fd7276c0f0c30b9416f61a6f5f35d763d81e4 +fa8a0639f7b0ce04030b72b4d5be4f0aa36fc5cb +f1f1605c22a6283bbfd757055fcf2b584a857709 +0c173a15ca1bf20999f74987988985508c9de463 +df0793f324e33066cc746c0cb1d053d35733d626 +2b0179d8a9b75397937126b36114df0dddeab40c +bf0a08be281dc42241e7f264c2a20515eb4781bb +3895e25a77363ae8b49358fb793f50fa8b271e2d +1fc783fc08bc078239537535f174ab8a489772c0 +1d4805ce04645f3203b0cfd3d66ea710e7433eb4 +d3b58704d1d325875fc605580c1c02b825c1bbcc +ed88e3194c4bc43aeafef929da7b419d03dea1ad +dd07f47b79628668e29cc0143b21e790100ee445 +65cc7aacfbfc7b747926375280a1d839e88d576b +080ec5209172ac9605f1434559dbb3c1e012b10a +416af3edf5b5ab265acf95568f2bc9eabd3d96de +e0a7801223fd573863939e76cb633f1dcc2d22c4 +4bc853b50fd9127687eb9e4f3b679dd261a4fa96 +c68a9a69278aa194fed96bd9733d32af3690a11e +c38f540298f0e188df5ed68fd56c623b9ac8331b +643fa0b22d70e459d7f7ec3d728ae4811dc5158f +e053e05c130549f43953f1d70e724dc9ce3e1b85 +75e898c094eea533d1dfaf141c6afccc3072c49f +2805d606bc46bf5589093a1b92d3542c13ce50c2 +32751807c9c06011eb689cba56b401a6302699c0 +30853e16d332816752dafcfca92147c7ffef5b54 +bea5b00cfe95cd37832305c0f93c339a22a7d79d +c871f323b418fac27bf834843ca26985010df53f +329fc1dce7a1c372c8b10c2f2f8732b2c60daff0 +1aefc94dd78d6e0c9209cb09fc16f53dedf42108 +8e5725666b519b61fcdc3141da5c6a57c1959909 +a4ca0b042365061020627a8c045cddacea3312ec +8bd16ee12fc8ef6723e0572c29b979c15b92b4f4 +87abe20fc118721cc5efdbd94a8462468cd1da2b +4b766fcdd4ca16399075d1e081a321b3b05ce516 +f6241b3e420e19f3f0507cbbc872fe9218916a02 +7ee523604851af62c0a47c07ee023a8710ef32f7 +776ba233e939fe41a74c6b2632b93a0679a32c71 +6a796b2b53fe542e0f340f250f4f20d69efed8d0 +23d78c4dd01bc74ba35db3e3df95280f6f1b2e22 +f4b15e2de97c4f8cdbb40bef4c9d0ab2807974d9 +fff72de5bf8ac7b70208e655f237b80e70e18851 +170bc2c381f86a523de2fc8b71d62ade66303c0d +314ebdfcb38d4b4c977579f787d5e1a20d068c94 +e9274839bf316b1972d80d28e45759f898edbf86 +75171f099e82e3527d7c3469b15891bd92227ec2 +3c5e6c94caf40395e031fbde44a0cca46fdd76ec +dc8fc0c73bebbc1c48ac5540026030c9cc00ec23 +492d22f92919d8d9d59568318c26c1e2ac4890cc +80c3a734298e824f9321c4efdd446086a3baad89 +47535d7c3ec79c5978cdcc03a5351ddbbb22538d +1b25b6df0f08f7474228c5b6ed13b58682e1e440 +c530c15180631cea95e9c292cf7fabde9dca9db3 +2723bcdce3248417e98e6c43207bef74d34076c1 +ed22eb4a62bd8d5369aaec87d4cbdc03c9f16368 +9111df9673beb6d6616d491a5478f09b5f14d040 +d86bb075bf6d1e78c1e4f3dd38b0ea828ef5ecfe +50a1cc0f0aef1514b917a5a3f4476967170b429d +6ce733747e160ca699711f2c47e686284ca9aa07 +b44adf92342ad4f9c343ba29c081a91687932936 +88799ea1b1c08f4bc1a487c9e3c2effd5e1650ae +080d7c700fc3291560d79fc590e05b8e2bad984f +12af74b289f8cdc6caf850dc6c802f9936b1e8b3 +8e4f7e72410df3ba430082c7cf385f26fd75b033 +8ac80412867118172dc4172494304e19969e9489 +f2734c2828f69d9cfd535e5eab0592a7674b2b61 +0b9fb682890b8fe10cec54072b809a5efe57d33d +5b029aaedb5fcf7cadd249607dd28eb3f233ab8c +79af9fbd8c3c0e54702a9c92b171f134bd4466c8 +c412fd805ddf3282dc2e1f28e30f51ffcb1f1da2 +111849345bb5140f86b48e730ceab4bff45fa2e9 +a0b1e57b20a17177ed5a9a54e4a8aab597a546b4 +ca209230c8e73745cf8cfc79f500c9c46e103306 +a230b0588788dbe1ac84622aea169c577b381241 +dfef6b6af08097f0676a2323085558fbbd3c48c6 +3192e5278abca7c1f3b4a2a7f77a0ce941c73985 +7c7ddd9ead99a8b5033a1a5d4698032c9e2b3a92 +10b930dde8f14e9cb661810e97a33bbf144fc55c +9225de2cf652fe2bf6e50636824cdb641546f57d +598ef9c44b3ea2cc142c175f077b493f39f5ba22 +c49355c7170a64bdd7864cc3ba9a64916b67fe7c +857d1e171e051b254a617f27b39f6a551054cee2 +21833f9456f6ad5bc06321ad6d9590f42ce0195c +8910b4717e5bb946ee6988f7fe9fd461f53a5935 +5703dff0939f05c7457cebd6fc61d88ab13afe41 +8bfa13b15b84cb372950fb7b25a1080173060b6a +ac23a7c1f19b3d8c326ffe75c8e13edf285f90fe +19be26afe3d04783a92d032b55bf3fb1e2ae63cc +f7ec7cfd38b543ba81ac7bed5b77f9a19739460b +36afd4db4442c45d4078b1a7ad16a1872b5bee0d +88c2ae3ed2bb5d367dd408c9255cd8f1e7a36c7d +a13a417cdcfdfd1f1b3bf997bb6ffe6e69b096b9 +d6064a89ac97dc0d2ce9da3982e1a4e25afaeda8 +7146d96de3e15a80cafbab2af48ff6f65d8e41bb +5628c70f2a44567695e5331fe2293c5b7f35b629 +7ff4a538a8682cdf02a4bcd6f15499c841001b73 +aa5fa642b0e7ce2ea55e2298886f212f11a8894e +8efd1c820b9a782d8608d54d924658536178295c +50a226563cd8d7c0a5e8448e87fede0eb72a8354 +b860915f8b0dae98e57a254d11575ea41f5c5a79 +d304fef3746039183f51b3ac8f4774dcf3a64f59 +53ab12d9318d5d195ccc77028b0e3ae66dc6e1fd +668de70be039a4f1ffcf20aeae2a22ee71fc55a8 +0fea960ca917b73aff853fe88476174c8a313863 +f89502306dcf6393a2c7b0efbb0fa728fc582137 +ff58b1c3bdff5e5f687f10f9e40ce495ca49674e +0b96abc35f1a9d46a27eeddd7df418d107c29c57 +b0b57a17306a7e963a4fe463f84e2b150a00a859 +4105cb6fd964ad13099ca83b1fdf3d35f3961f74 +23281a4dc3afc42a001346caec4dbb8193f0bb53 +8daf103fa138f9a184448ebf1c2e03b9dbd96f21 +02e5308c1b9f3771bbe49bc5036215fa2bd66aa9 +a65ced1a66575c652baf5084644b8647f531be8c +2456a835f0bc7796d9ff71f64837fa6790e2b7cc +9ec1330b455c1ab2eb6b89f8a2ab885677d4ae8a +0b738075bd43fbd4410e30a51e0498cbfd2b7513 +98c80e374b84e5a9c2d5c36889a0b1ebed5b814b +25720fc394e27a951bcad26095fb5a711bfacb8f +4cfd57d2e38207d78722ce8c9274ba8dd700d1cc +0fc1c31a878e93d938c67db3f958e82e3c39659f +df1ab5b4d67b46b5e9e840b1fbe0ff02520831f9 +5bc3b6cede8dabdf3f4f27ddb03723cbb7cde51a +c2ea1e6561caba3abffce361abc800822b9e0efe +caa2f106d704ec3ade63498031dd58d34510bc76 +dce853ef76ef90c46d84294225088d595467d08c +dbc8a8c86ae50059fddb2d6834fa5f0c9bbf9b71 +0f921e6a0492c4e9f037a9ed91f474885032d68c +041331e1da23e4136fd046ed870cdcc177464176 +e6ba5068f107ac234576e77cedbd748b665369c2 +76fcd9d5034143a5b041766552670d19f926097d +72bf1b3d0962304850a3ef5fe375db4bff1d0a39 +919db037f1f5cc73cdcaef92dd9cb0e7f5c8dec3 +c36229b0b2e9d4554053f5c9fc451ac29a493b1f +9e4bb312e6958d2baa309ba670e5eed1523c6f47 +d7ba4a233bd5a6f8fadee681c68a995e23fe36d7 +98514988a3d3e8b7dbf0463884a5c38f5ed5562d +5412c08c3cf13577566064edd04da021c37b7cbe +31bcc667863f368157efa1143a78623a5db8f0d1 +7bd1aa566fb4a4fe194f209085649f2c722b0cff +c4522e71c7e1d8ecfd70112e9375b9d00d6733a8 +e22f409f18881b63a8e747036584a71217f40e6e +97ec6e5c9098a1240655cfcab05b6cd5eedb6cd1 +bc121b0eb19713ec72002b5be03ba5ac35903a17 +c98f6b3d93a2cc1b49a6db425ea2b661089d0f9e +0de7fd36de57a68e543b4c1f184fba192c398c73 +e662d281b837c25b2b70525aa8fe8af894339823 +44adf683ad232db8ce0cb89b3e236a1f5944cfb0 +cb2ed300a89ebf9f0654da869ced665ed8b2abe7 +0a6d48d9ed60b0b02177059ab116f8f46d2cbed3 +b42291334651fff46dbfe5947a726f65cb9d7dfe +e5364991daecb73aca3bb5ac37f2619d7a89211b +4a2b170c075ce703cbdc82519a48016a9ee3f99c +924de0bd75a7f75df65d7d15f9d1587a2e794abf +1253f8692fc3a11be9430685cd405236a68df6c3 +2b799ae9e1e0a540f9a5971ddf27d83254668279 +c9bdf9a75f9fde8cd011e4aa94be4ed4347078a3 +3d69ecb4edeb80003a1a41442e320898a30dbd9c +f08222e882b18c1f279308636e03beceece2dbf1 +23e03f8d26d7bd03273a5dcbdcfe3905dfb49ffb +03dd707dc027fbf6f24120213f8eb66571600374 +d0754799698de2c032abcb8198ee5d5401063213 +072116fceb2294b97d1c40f79305f2e3ff71812b +e66cc1d58e16bf1650dd6479fed64ecaca8c6098 +f137753a2dcd8229f89d1d1ac28039364e5850b4 +61d191fbf953700ba8aeadc9c8cf4c195efbd10c +76f3c02fb01a6df98fbd8c16ac21d159d4649d37 +6013c73b3312e11b447ed387426749014716f820 +6faffb8a83db3f209a303a4464dbdd597faad5a4 +cc9e8aca5f950c78dcfeff63c441ba993c1fe12f +8ca69a2a88a77eb06149fa049ab1a7e6de38b321 +2f71490d21796594ca6f55e375558944de9db5a0 +08cc5fd666456cb476467473ed1880c90c92dedb +e31a43c725ebe641d7c219c3886eee18eebf0bb8 +52b5a8785de760a204b2b0aab19dfaf79c2c3ff0 +483e8e4f4875a1a621ec9e9df2880d3037d95ed7 +1e5799c52535a3fc20e885916f1e7ed33ecc7f46 +a82e5d8220bbc8b5d786bed99b0876f530b9b7cc +7fe6c5c993706e8395cdaf7977bee793c06f48f3 +2a0836f6d5e7c1d7e97bedb0e0ea33dcaf981f77 +ddc308068d69c6c9aa629ee3c4ce75e1d1cf08b5 +ec139a5621a9c9f03e1988391a3c7c6c5d849776 +c01a6c48b982d625fd9f4f69005878781d3d56fa +95a983d56dbda457e3bf8766d59bac74c7aa5699 +760741a00833876976389ed7a6b73f36ee5b4c13 +6e5e5abba6f8bbbe61c22795df440dfafcfdc378 +cf2cecb18779ce83de9adebf382dff1c19b12840 +af9b7a9f2f73b1a2f9728106774dd13e8d1cdd8d +115735d547fdeade822f547eb3e8c8f9961a9b07 +c2c69edf37b5c02aafa01d0407dadbf5ef8751b5 +a072d1a83787e786d074a4b5871b0b961781f7c6 +ed2cd59e258f756b2eaed7909a60956ade6ef7ee +ae5575ba41c8a782805afb1c08730343cfc22397 +6ff2c8d29f6b5a5c2ce63f0a16f3bb0dbd049451 +a80de15113166354cdf208e3d8b6e25f4511a591 +06bd4f637f15e769f088d9051a5af94bbb0217a3 +6700cc993cc07fb0f5b8b577ff8c4afcf0b18274 +37f9a1f627c0995d89b62923e75cd092600894f9 +8844ef15ded02d5ed86fb95aaf251235fcef2396 +1b87e5b5b184a0a6c683eda23b36393822b57f03 +e2bf830bb6c1bfa038c943dd6f5d92a406bd723f +423ca302a3ee87000530da3c105f269b8fabece7 +4e14afe42fdd468d5de11df8cc13defdcb8e83f8 +3e90fe6534206412ea22beaa445cf20d28fbe718 +88b77c7da0a672c89e24df37ea6e9085b4e2a05c +0ad104190465d8d65c2344bbe10dcf3df025d86c +5c7df7022bcd360e6af00b9458b1a3fd54e1cc9a +59ad56851a342d2c62f6b38bf15002b23ab439e1 +d8cd7b137fb075616f31d2b43b85fa2e27ea7477 +655937ebcbf681ededf86b1f0f60aac45c73393d +abdfd2d0e3ebec7dbead89317ee9192189a35809 +e439aeb30c0439001a781c5979aec41e1fc2aa50 +b9b26d9c3615d15669ae0a049c1dede39a9e59a9 +fdf146f3293c487afdc4d6d9f6b64099aa8bd28a +16e3b175781caacee403a2dc40cd6c70448e12ef +b30c62d4b954df05bf404cfbeb5b728282201496 +b3f377daaa86cd7755a552fa3adfeb195835f58e +0a8f519a0626d7cd385114ce610d23215c051b3f +544f3234384b2f6c290e987ad18576e1b50d7db9 +91482e5bf22d283d32c9f83c8057f10971848107 +e754c6e33194e9ed69ba5350c5139b0423b645fe +dc1e54206d76e5fa378d28a18ae1fb2bcf714485 +b2863c0685a5c12f829095cbabaf26ccc49e46ec +b14db5abab405a708f0166293f1ea12222a6bf03 +8010ded6da56842c09b14665343cd189d7e08401 +d387507aeca652a5569825af65243536f2ce26ea +27bf14f6f3e0fb1f348f13c1b54fc6b67b3bef6e +f8d470e24606297dab95e30b1d39ff664fbda31d +b25a4c2284babdf1e8cf0ec3b1402200dd25f33f +1329ef1f00e4fad83937ddd8721d0292ccfe7808 +9a1ad2c5cbdfa3114d05df57103c34f72e087f26 +1e90862f5d0b5f7dcc18fb018b2bbaa323dcca1d +ad552a54c56a420be84b47154882c3e4c76f9bdd +90b1c7e5c50551f39d4983008d1d5ab3b085803e +d6b2235ca45e072961e25a35e6a159e97c9e556b +2643fa50869f22672cbc72ac497d9c30234075b8 +01f909828d126d5443bc28758f51781eccdf5848 +f54f3738c8ce839c413d7b6b719be2ff341536ca +418ae49ee1eac2c9d6cd4ba83c036a41f1afe922 +5a666428b0f11d62af2002bd54a45ff2f79f30cd +a07e8caa5d5000286604458e6887f57fec7fdcbb +8b262eb2d80bfa27ae8501078ce47bc1407e9c55 +5df84de583c900e00fef63bedaef32786f205a33 +4ba6da55743a55189164e29e45ac9e73a074d808 +88430cbab4dca36b6a867364cab319cde6a9ebca +e0f7515f5500968c86e5a9f4912d83d4abc5b2b9 +9b8b1079ddab64ac955766536c38d23dc57bc499 +af20f9b1d485582b8c8aa8294bac4f2c540246d2 +7be9a9a570c1140048f8781ced1111e1d930e517 +2bac3e484114c30548e286972525dd799dbd0a5b +df529dcc65e8037c5a3a3ad74545be294a770f07 +6acd8700bc0ee1d10207a362c1e07372ba274041 +ffc6e48b2983189dc0ce7de0a038e5329bc07b1b +252ae7111cbff09a4cbc5caee9e02b6ed3580476 +b3ecb7bab6074377d87c700bf0c5d351e5d3174f +d9fdac130a5ed1d96fcac6bb87c10bec9d596b17 +a07e8caa5d5000286604458e6887f57fec7fdcbb +8b262eb2d80bfa27ae8501078ce47bc1407e9c55 +5df84de583c900e00fef63bedaef32786f205a33 +4ba6da55743a55189164e29e45ac9e73a074d808 +5bea05bc1d17aa43cbdf3a3413241f8132790d93 +c17f11f7b43ad3bd9e242c67db1f3679558a0581 +5ea932a51083837cdd27715e10a3a0d5d553af24 +033c78671b91b12d589ebff6c5ede8d94d7500f8 +ef8a634358848847e006c43ce621bc17a612fd1f +ba216b5fa63e7e6cae847d1e3621f5c54840f898 +26fee4f6bd9aec62c6caa60683ad66574cf16aa6 +6ab0e4cf49549640b903bf5fce0e6035b8116397 +326a5652e0d25fdb60c337ef4f1c98a63e0748f0 +424be03305143cbe5da5d5adb54d73d3dc3747b6 +38c201f47c0bc388a05cdb35d6137150fa90193e +12ed800ab870e0fc527a84d6e4584b10c8d239f5 +aeed345c9bade5d52a3fbf0a943203f6c82e6344 +c6223b3daab0328ca742b1cc3c15e89e698630bb +877678710800a4d78afc12519424f232f1a583d3 +6c4fecfaf7beefad0d1c3f8520bf50bb515a0716 +98212745c8acb5cc4e688bbb3979bfd46b25f98a +b9bceaf1c081a84d9fcc680372614e797b168a9e +1afc22a7667a7a5c66b4b5d7f50832356dd5ec12 +3255d6347b1f9eccbec3d6d93d4a424087a3b35b +ec20f01ba0945a3113797ac98a6b3500e24603d4 +75b5643c47c3b382ed97a9f5e2bdc883a0f98709 +fee0d803fb55c8d85b5cd1ff69d799c5ad522e18 +565494619d809655fa94d274bb2202d25553e485 +ad6fce67b9bb6eee864c8431ad3291aebaa2e5d2 +99c7db8731cc77f143b52f544b3fdd93033ed20d +b4d03be3cac04da8b5d5fa17e29c5220b75d970b +ef37f2033c4ae104585cd980141262f95d33166e +5cfdda2503c995cdd563b1a2a29162ac298d173d +c5904e871479514b2e2e18b4fdbbe468c4e5ec8e +10b22e3141a603ec891d2cfc7100c29c7409aabe +afd2fca911c4a5e3a4d1f0993a226d40f250aff4 +505955052e60e0681865f3064e005ca0d3aa90bf +8fdd23a224ba236874ef662c4ca311b002dbcab3 +1c011ff430106b5f727f2eaa0f7f4883cd2122a3 +ec8a50b8d786a8cd1192e692ab19b46979add582 +f90603ac6d24f5263649675d51233f1fce8b2ecd +b7d6623c76e1468f2a93db5a3120580e2784d74a +66270a416edb1610f276124483feceef9cba93ff +e4fcbf797ed3b472d352ac3794ec82f581209c50 +479afa0f8486146a35f1fb96be1826061ecbcf23 +2a09a3891fde052a585dc019eea9fba26d42445d +90a002ea647dcea57a2ed4294eab77897168ba1d +30c21306c17165c3925fea4ac9d1a4763c6d2a99 +b3eb0d6485510f2bdf36d256ab60ce29b8213744 +efbcf2b1d5ff4ee7132eae9c9e203d2b875c545b +b33ca14f594e2cf2a16ef27778169deb7cc9f4dc +d636f3943d39ec893dab2d2546f77f3f2607769d +cafe24f039e117d53288387c2720f44f27deecd0 +de8db47b7ff351f3287c5efb85102ba8836058d6 +d76e84a21416ef77e78138e326d4d249454e79dc +7a74f88a26cf251ba36b26f604f1ac9940fd9c92 +1ad3d4e1261f4a444d982a1470c257c78233bda3 +8d9f45ea6a5e4220e44d34139438eea75a07530b +c98ebf1bfb29a8203b5090412afeb333384213cd +f18bb49547095020a30e81b648075bc7e707515c +76f268b9bd1b69eb7784c5324abbb67f3e395b97 +e801084decf4542d57cf5ddb95820643766a172a +be3e042c20e2f3449b7b55d1cab0a80b0c6f00af +400fdd08cc95f1e85afafd07ddd9c0bed11483ea +098b01dc58ff555c473ae58c92c34b03a77eda5f +7cc2c670e3d7cf26454ac8547a94ec2c8ca90b34 +1088b02f0ccd7358d2b7076bb9e122d59d502d02 +f94b7d5bfa911ea7125920589723ee63a3eec9f0 +b4b057a3e0712dd16b50cbcfe7d613e4413ffa1c +b40ceed98a112f4f0d07351ce07270d9ff2bf796 +4cb8757aae1ae31e5519d81e854f44ed062d9836 +f2f7e97e8cc24cf7a2b7954cb74ecfd0f91a95ad +ae786098bc58b1ca92f596a698b23aeade9cd2cd +c33652576ce21694b33a94832378f737dd6959fb +e317c0d19201ff75fa7afedf93a9d1cd2c560af2 +bee35299716cc72cb7d5bd4daa9fddca05c58378 +318ea50a1c2f612e750a93e36620dd0c4531e9cf +b6ee855b411ee9bc39f935d0da3298a773a2ed37 +daf3e7def7b9e5db7a32f5a20b5c4e09e3f0dd18 +bc64b5aa0fc543fe8fd3dbaec275f89df44dc409 +3f57c55dba6ef1fda2bdf6fd9abd8ca7eb6828e4 +431a548faaf51c7a5fc89b6e479187a1c0e29805 +e4bbd3d230f22401ba0a0a72c8ed41ee1bd098a0 +c45da32047cac54afc99cf9b8a539389c577ded1 +ab1f1d32469180b3d011e9625d67c86a22b55903 +a550f6e415fd8aec8c45d4704712a408c37ecd18 +c73af5416b66f09cec0eb106f5a10f9bb6ef9cb1 +a077a90da88f12d9f10c8b85840bdb847a98b0a0 +c5e9e428a9198c8c4076f239b5eaa8dc95e7985b +b7365f0545b1a6862e3277b2b2139ee0d5aee1cf +4bd0e9b90a39c5c6a016b83882ae44cb4d28f1f8 +7438ceac716fdfe6621728c05e718eaa89dd89aa +4e3efd47e0d50c6cd1dc81ccc9669a5b2658f495 diff --git a/contrib/verify-commits/allow-unclean-merge-commits b/contrib/verify-commits/allow-unclean-merge-commits new file mode 100644 index 0000000000..7aab274b9a --- /dev/null +++ b/contrib/verify-commits/allow-unclean-merge-commits @@ -0,0 +1,4 @@ +6052d509105790a26b3ad5df43dd61e7f1b24a12 +3798e5de334c3deb5f71302b782f6b8fbd5087f1 +326ffed09bfcc209a2efd6a2ebc69edf6bd200b5 +97d83739db0631be5d4ba86af3616014652c00ec diff --git a/contrib/verify-commits/gpg.sh b/contrib/verify-commits/gpg.sh new file mode 100755 index 0000000000..db5bfce208 --- /dev/null +++ b/contrib/verify-commits/gpg.sh @@ -0,0 +1,65 @@ +#!/bin/sh +# Copyright (c) 2014-2019 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +export LC_ALL=C +INPUT=$(cat /dev/stdin) +VALID=false +REVSIG=false +IFS=' +' +if [ "$BITCOIN_VERIFY_COMMITS_ALLOW_SHA1" = 1 ]; then + GPG_RES="$(printf '%s\n' "$INPUT" | gpg --trust-model always "$@" 2>/dev/null)" +else + # Note how we've disabled SHA1 with the --weak-digest option, disabling + # signatures - including selfsigs - that use SHA1. While you might think that + # collision attacks shouldn't be an issue as they'd be an attack on yourself, + # in fact because what's being signed is a commit object that's + # semi-deterministically generated by untrusted input (the pull-req) in theory + # an attacker could construct a pull-req that results in a commit object that + # they've created a collision for. Not the most likely attack, but preventing + # it is pretty easy so we do so as a "belt-and-suspenders" measure. + GPG_RES="" + for LINE in $(gpg --version); do + case "$LINE" in + "gpg (GnuPG) 1.4.1"*|"gpg (GnuPG) 2.0."*) + echo "Please upgrade to at least gpg 2.1.10 to check for weak signatures" > /dev/stderr + GPG_RES="$(printf '%s\n' "$INPUT" | gpg --trust-model always "$@" 2>/dev/null)" + ;; + # We assume if you're running 2.1+, you're probably running 2.1.10+ + # gpg will fail otherwise + # We assume if you're running 1.X, it is either 1.4.1X or 1.4.20+ + # gpg will fail otherwise + esac + done + [ "$GPG_RES" = "" ] && GPG_RES="$(printf '%s\n' "$INPUT" | gpg --trust-model always --weak-digest sha1 "$@" 2>/dev/null)" +fi +for LINE in $GPG_RES; do + case "$LINE" in + "[GNUPG:] VALIDSIG "*) + while read KEY; do + [ "${LINE#?GNUPG:? VALIDSIG * * * * * * * * * }" = "$KEY" ] && VALID=true + done < ./contrib/verify-commits/trusted-keys + ;; + "[GNUPG:] REVKEYSIG "*) + [ "$BITCOIN_VERIFY_COMMITS_ALLOW_REVSIG" != 1 ] && exit 1 + REVSIG=true + GOODREVSIG="[GNUPG:] GOODSIG ${LINE#* * *}" + ;; + "[GNUPG:] EXPKEYSIG "*) + [ "$BITCOIN_VERIFY_COMMITS_ALLOW_REVSIG" != 1 ] && exit 1 + REVSIG=true + GOODREVSIG="[GNUPG:] GOODSIG ${LINE#* * *}" + ;; + esac +done +if ! $VALID; then + exit 1 +fi +if $VALID && $REVSIG; then + printf '%s\n' "$INPUT" | gpg --trust-model always "$@" 2>/dev/null | grep "^\[GNUPG:\] \(NEWSIG\|SIG_ID\|VALIDSIG\)" + echo "$GOODREVSIG" +else + printf '%s\n' "$INPUT" | gpg --trust-model always "$@" 2>/dev/null +fi diff --git a/contrib/verify-commits/pre-push-hook.sh b/contrib/verify-commits/pre-push-hook.sh new file mode 100755 index 0000000000..9953248378 --- /dev/null +++ b/contrib/verify-commits/pre-push-hook.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +# Copyright (c) 2014-2021 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +export LC_ALL=C +if ! [[ "$2" =~ ^(git@)?(www.)?github.com(:|/)bitcoin/bitcoin(.git)?$ ]]; then + exit 0 +fi + +while read LINE; do + set -- A "$LINE" + if [ "$4" != "refs/heads/master" ]; then + continue + fi + if ! ./contrib/verify-commits/verify-commits.py "$3" > /dev/null 2>&1; then + echo "ERROR: A commit is not signed, can't push" + ./contrib/verify-commits/verify-commits.py + exit 1 + fi +done < /dev/stdin diff --git a/contrib/verify-commits/trusted-git-root b/contrib/verify-commits/trusted-git-root new file mode 100644 index 0000000000..1c42195961 --- /dev/null +++ b/contrib/verify-commits/trusted-git-root @@ -0,0 +1 @@ +577bd51a4b8de066466a445192c1c653872657e2 diff --git a/contrib/verify-commits/trusted-keys b/contrib/verify-commits/trusted-keys new file mode 100644 index 0000000000..2f8a21009a --- /dev/null +++ b/contrib/verify-commits/trusted-keys @@ -0,0 +1,6 @@ +71A3B16735405025D447E8F274810B012346C9A6 +133EAC179436F14A5CF1B794860FEB804E669320 +B8B3F1C0E58C15DB6A81D30C3648A882F4316B9B +E777299FC265DD04793070EB944D35F9AC3DB76A +D1DBF2C4B96F2DEBF4C16654410108112E7EA81F +152812300785C96444D3334D17565732E08E5E41 diff --git a/contrib/verify-commits/trusted-sha512-root-commit b/contrib/verify-commits/trusted-sha512-root-commit new file mode 100644 index 0000000000..7d41f90ad7 --- /dev/null +++ b/contrib/verify-commits/trusted-sha512-root-commit @@ -0,0 +1 @@ +309bf16257b2395ce502017be627186b749ee749 diff --git a/contrib/verify-commits/verify-commits.py b/contrib/verify-commits/verify-commits.py new file mode 100755 index 0000000000..2ff14c1f86 --- /dev/null +++ b/contrib/verify-commits/verify-commits.py @@ -0,0 +1,170 @@ +#!/usr/bin/env python3 +# Copyright (c) 2018-2019 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Verify commits against a trusted keys list.""" +import argparse +import hashlib +import logging +import os +import subprocess +import sys +import time + +GIT = os.getenv('GIT', 'git') + +def tree_sha512sum(commit='HEAD'): + """Calculate the Tree-sha512 for the commit. + + This is copied from github-merge.py. See https://github.com/bitcoin-core/bitcoin-maintainer-tools.""" + + # request metadata for entire tree, recursively + files = [] + blob_by_name = {} + for line in subprocess.check_output([GIT, 'ls-tree', '--full-tree', '-r', commit]).splitlines(): + name_sep = line.index(b'\t') + metadata = line[:name_sep].split() # perms, 'blob', blobid + assert metadata[1] == b'blob' + name = line[name_sep + 1:] + files.append(name) + blob_by_name[name] = metadata[2] + + files.sort() + # open connection to git-cat-file in batch mode to request data for all blobs + # this is much faster than launching it per file + p = subprocess.Popen([GIT, 'cat-file', '--batch'], stdout=subprocess.PIPE, stdin=subprocess.PIPE) + overall = hashlib.sha512() + for f in files: + blob = blob_by_name[f] + # request blob + p.stdin.write(blob + b'\n') + p.stdin.flush() + # read header: blob, "blob", size + reply = p.stdout.readline().split() + assert reply[0] == blob and reply[1] == b'blob' + size = int(reply[2]) + # hash the blob data + intern = hashlib.sha512() + ptr = 0 + while ptr < size: + bs = min(65536, size - ptr) + piece = p.stdout.read(bs) + if len(piece) == bs: + intern.update(piece) + else: + raise IOError('Premature EOF reading git cat-file output') + ptr += bs + dig = intern.hexdigest() + assert p.stdout.read(1) == b'\n' # ignore LF that follows blob data + # update overall hash with file hash + overall.update(dig.encode("utf-8")) + overall.update(" ".encode("utf-8")) + overall.update(f) + overall.update("\n".encode("utf-8")) + p.stdin.close() + if p.wait(): + raise IOError('Non-zero return value executing git cat-file') + return overall.hexdigest() + +def main(): + + # Enable debug logging if running in CI + if 'CI' in os.environ and os.environ['CI'].lower() == "true": + logging.getLogger().setLevel(logging.DEBUG) + + # Parse arguments + parser = argparse.ArgumentParser(usage='%(prog)s [options] [commit id]') + parser.add_argument('--disable-tree-check', action='store_false', dest='verify_tree', help='disable SHA-512 tree check') + parser.add_argument('--clean-merge', type=float, dest='clean_merge', default=float('inf'), help='Only check clean merge after <NUMBER> days ago (default: %(default)s)', metavar='NUMBER') + parser.add_argument('commit', nargs='?', default='HEAD', help='Check clean merge up to commit <commit>') + args = parser.parse_args() + + # get directory of this program and read data files + dirname = os.path.dirname(os.path.abspath(__file__)) + print("Using verify-commits data from " + dirname) + with open(dirname + "/trusted-git-root", "r", encoding="utf8") as f: + verified_root = f.read().splitlines()[0] + with open(dirname + "/trusted-sha512-root-commit", "r", encoding="utf8") as f: + verified_sha512_root = f.read().splitlines()[0] + with open(dirname + "/allow-revsig-commits", "r", encoding="utf8") as f: + revsig_allowed = f.read().splitlines() + with open(dirname + "/allow-unclean-merge-commits", "r", encoding="utf8") as f: + unclean_merge_allowed = f.read().splitlines() + with open(dirname + "/allow-incorrect-sha512-commits", "r", encoding="utf8") as f: + incorrect_sha512_allowed = f.read().splitlines() + + # Set commit and branch and set variables + current_commit = args.commit + if ' ' in current_commit: + print("Commit must not contain spaces", file=sys.stderr) + sys.exit(1) + verify_tree = args.verify_tree + no_sha1 = True + prev_commit = "" + initial_commit = current_commit + branch = subprocess.check_output([GIT, 'show', '-s', '--format=%H', initial_commit]).decode('utf8').splitlines()[0] + + # Iterate through commits + while True: + + # Log a message to prevent Travis from timing out + logging.debug("verify-commits: [in-progress] processing commit {}".format(current_commit[:8])) + + if current_commit == verified_root: + print('There is a valid path from "{}" to {} where all commits are signed!'.format(initial_commit, verified_root)) + sys.exit(0) + if current_commit == verified_sha512_root: + if verify_tree: + print("All Tree-SHA512s matched up to {}".format(verified_sha512_root), file=sys.stderr) + verify_tree = False + no_sha1 = False + + os.environ['BITCOIN_VERIFY_COMMITS_ALLOW_SHA1'] = "0" if no_sha1 else "1" + os.environ['BITCOIN_VERIFY_COMMITS_ALLOW_REVSIG'] = "1" if current_commit in revsig_allowed else "0" + + # Check that the commit (and parents) was signed with a trusted key + if subprocess.call([GIT, '-c', 'gpg.program={}/gpg.sh'.format(dirname), 'verify-commit', current_commit], stdout=subprocess.DEVNULL): + if prev_commit != "": + print("No parent of {} was signed with a trusted key!".format(prev_commit), file=sys.stderr) + print("Parents are:", file=sys.stderr) + parents = subprocess.check_output([GIT, 'show', '-s', '--format=format:%P', prev_commit]).decode('utf8').splitlines()[0].split(' ') + for parent in parents: + subprocess.call([GIT, 'show', '-s', parent], stdout=sys.stderr) + else: + print("{} was not signed with a trusted key!".format(current_commit), file=sys.stderr) + sys.exit(1) + + # Check the Tree-SHA512 + if (verify_tree or prev_commit == "") and current_commit not in incorrect_sha512_allowed: + tree_hash = tree_sha512sum(current_commit) + if ("Tree-SHA512: {}".format(tree_hash)) not in subprocess.check_output([GIT, 'show', '-s', '--format=format:%B', current_commit]).decode('utf8').splitlines(): + print("Tree-SHA512 did not match for commit " + current_commit, file=sys.stderr) + sys.exit(1) + + # Merge commits should only have two parents + parents = subprocess.check_output([GIT, 'show', '-s', '--format=format:%P', current_commit]).decode('utf8').splitlines()[0].split(' ') + if len(parents) > 2: + print("Commit {} is an octopus merge".format(current_commit), file=sys.stderr) + sys.exit(1) + + # Check that the merge commit is clean + commit_time = int(subprocess.check_output([GIT, 'show', '-s', '--format=format:%ct', current_commit]).decode('utf8').splitlines()[0]) + check_merge = commit_time > time.time() - args.clean_merge * 24 * 60 * 60 # Only check commits in clean_merge days + allow_unclean = current_commit in unclean_merge_allowed + if len(parents) == 2 and check_merge and not allow_unclean: + current_tree = subprocess.check_output([GIT, 'show', '--format=%T', current_commit]).decode('utf8').splitlines()[0] + subprocess.call([GIT, 'checkout', '--force', '--quiet', parents[0]]) + subprocess.call([GIT, 'merge', '--no-ff', '--quiet', '--no-gpg-sign', parents[1]], stdout=subprocess.DEVNULL) + recreated_tree = subprocess.check_output([GIT, 'show', '--format=format:%T', 'HEAD']).decode('utf8').splitlines()[0] + if current_tree != recreated_tree: + print("Merge commit {} is not clean".format(current_commit), file=sys.stderr) + subprocess.call([GIT, 'diff', current_commit]) + subprocess.call([GIT, 'checkout', '--force', '--quiet', branch]) + sys.exit(1) + subprocess.call([GIT, 'checkout', '--force', '--quiet', branch]) + + prev_commit = current_commit + current_commit = parents[0] + +if __name__ == '__main__': + main() diff --git a/contrib/verifybinaries/README.md b/contrib/verifybinaries/README.md new file mode 100644 index 0000000000..c50d4bef71 --- /dev/null +++ b/contrib/verifybinaries/README.md @@ -0,0 +1,41 @@ +### Verify Binaries + +#### Preparation: + +Make sure you obtain the proper release signing key and verify the fingerprint with several independent sources. + +```sh +$ gpg --fingerprint "Bitcoin Core binary release signing key" +pub 4096R/36C2E964 2015-06-24 [expires: YYYY-MM-DD] + Key fingerprint = 01EA 5486 DE18 A882 D4C2 6845 90C8 019E 36C2 E964 +uid Wladimir J. van der Laan (Bitcoin Core binary release signing key) <laanwj@gmail.com> +``` + +#### Usage: + +This script attempts to download the signature file `SHA256SUMS.asc` from https://bitcoin.org. + +It first checks if the signature passes, and then downloads the files specified in the file, and checks if the hashes of these files match those that are specified in the signature file. + +The script returns 0 if everything passes the checks. It returns 1 if either the signature check or the hash check doesn't pass. If an error occurs the return value is 2. + + +```sh +./verify.py bitcoin-core-0.11.2 +./verify.py bitcoin-core-0.12.0 +./verify.py bitcoin-core-0.13.0-rc3 +``` + +If you only want to download the binaries of certain platform, add the corresponding suffix, e.g.: + +```sh +./verify.py bitcoin-core-0.11.2-osx +./verify.py 0.12.0-linux +./verify.py bitcoin-core-0.13.0-rc3-win64 +``` + +If you do not want to keep the downloaded binaries, specify anything as the second parameter. + +```sh +./verify.py bitcoin-core-0.13.0 delete +``` diff --git a/contrib/verifybinaries/verify.py b/contrib/verifybinaries/verify.py new file mode 100755 index 0000000000..b5e4f1318b --- /dev/null +++ b/contrib/verifybinaries/verify.py @@ -0,0 +1,183 @@ +#!/usr/bin/env python3 +# Copyright (c) 2020-2021 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Script for verifying Bitcoin Core release binaries + +This script attempts to download the signature file SHA256SUMS.asc from +bitcoincore.org and bitcoin.org and compares them. +It first checks if the signature passes, and then downloads the files +specified in the file, and checks if the hashes of these files match those +that are specified in the signature file. +The script returns 0 if everything passes the checks. It returns 1 if either +the signature check or the hash check doesn't pass. If an error occurs the +return value is >= 2. +""" +from hashlib import sha256 +import os +import subprocess +import sys +from textwrap import indent + +WORKINGDIR = "/tmp/bitcoin_verify_binaries" +HASHFILE = "hashes.tmp" +HOST1 = "https://bitcoincore.org" +HOST2 = "https://bitcoin.org" +VERSIONPREFIX = "bitcoin-core-" +SIGNATUREFILENAME = "SHA256SUMS.asc" + + +def parse_version_string(version_str): + if version_str.startswith(VERSIONPREFIX): # remove version prefix + version_str = version_str[len(VERSIONPREFIX):] + + parts = version_str.split('-') + version_base = parts[0] + version_rc = "" + version_os = "" + if len(parts) == 2: # "<version>-rcN" or "version-platform" + if "rc" in parts[1]: + version_rc = parts[1] + else: + version_os = parts[1] + elif len(parts) == 3: # "<version>-rcN-platform" + version_rc = parts[1] + version_os = parts[2] + + return version_base, version_rc, version_os + + +def download_with_wget(remote_file, local_file=None): + if local_file: + wget_args = ['wget', '-O', local_file, remote_file] + else: + # use timestamping mechanism if local filename is not explicitly set + wget_args = ['wget', '-N', remote_file] + + result = subprocess.run(wget_args, + stderr=subprocess.STDOUT, stdout=subprocess.PIPE) + return result.returncode == 0, result.stdout.decode().rstrip() + + +def files_are_equal(filename1, filename2): + with open(filename1, 'rb') as file1: + contents1 = file1.read() + with open(filename2, 'rb') as file2: + contents2 = file2.read() + return contents1 == contents2 + + +def verify_with_gpg(signature_filename, output_filename): + result = subprocess.run(['gpg', '--yes', '--decrypt', '--output', + output_filename, signature_filename], + stderr=subprocess.STDOUT, stdout=subprocess.PIPE) + return result.returncode, result.stdout.decode().rstrip() + + +def remove_files(filenames): + for filename in filenames: + os.remove(filename) + + +def main(args): + # sanity check + if len(args) < 1: + print("Error: need to specify a version on the command line") + return 3 + + # determine remote dir dependent on provided version string + version_base, version_rc, os_filter = parse_version_string(args[0]) + remote_dir = f"/bin/{VERSIONPREFIX}{version_base}/" + if version_rc: + remote_dir += f"test.{version_rc}/" + remote_sigfile = remote_dir + SIGNATUREFILENAME + + # create working directory + os.makedirs(WORKINGDIR, exist_ok=True) + os.chdir(WORKINGDIR) + + # fetch first signature file + sigfile1 = SIGNATUREFILENAME + success, output = download_with_wget(HOST1 + remote_sigfile, sigfile1) + if not success: + print("Error: couldn't fetch signature file. " + "Have you specified the version number in the following format?") + print(f"[{VERSIONPREFIX}]<version>[-rc[0-9]][-platform] " + f"(example: {VERSIONPREFIX}0.21.0-rc3-osx)") + print("wget output:") + print(indent(output, '\t')) + return 4 + + # fetch second signature file + sigfile2 = SIGNATUREFILENAME + ".2" + success, output = download_with_wget(HOST2 + remote_sigfile, sigfile2) + if not success: + print("bitcoin.org failed to provide signature file, " + "but bitcoincore.org did?") + print("wget output:") + print(indent(output, '\t')) + remove_files([sigfile1]) + return 5 + + # ensure that both signature files are equal + if not files_are_equal(sigfile1, sigfile2): + print("bitcoin.org and bitcoincore.org signature files were not equal?") + print(f"See files {WORKINGDIR}/{sigfile1} and {WORKINGDIR}/{sigfile2}") + return 6 + + # check signature and extract data into file + retval, output = verify_with_gpg(sigfile1, HASHFILE) + if retval != 0: + if retval == 1: + print("Bad signature.") + elif retval == 2: + print("gpg error. Do you have the Bitcoin Core binary release " + "signing key installed?") + print("gpg output:") + print(indent(output, '\t')) + remove_files([sigfile1, sigfile2, HASHFILE]) + return 1 + + # extract hashes/filenames of binaries to verify from hash file; + # each line has the following format: "<hash> <binary_filename>" + with open(HASHFILE, 'r', encoding='utf8') as hash_file: + hashes_to_verify = [ + line.split()[:2] for line in hash_file if os_filter in line] + remove_files([HASHFILE]) + if not hashes_to_verify: + print("error: no files matched the platform specified") + return 7 + + # download binaries + for _, binary_filename in hashes_to_verify: + print(f"Downloading {binary_filename}") + download_with_wget(HOST1 + remote_dir + binary_filename) + + # verify hashes + offending_files = [] + for hash_expected, binary_filename in hashes_to_verify: + with open(binary_filename, 'rb') as binary_file: + hash_calculated = sha256(binary_file.read()).hexdigest() + if hash_calculated != hash_expected: + offending_files.append(binary_filename) + if offending_files: + print("Hashes don't match.") + print("Offending files:") + print('\n'.join(offending_files)) + return 1 + verified_binaries = [entry[1] for entry in hashes_to_verify] + + # clean up files if desired + if len(args) >= 2: + print("Clean up the binaries") + remove_files([sigfile1, sigfile2] + verified_binaries) + else: + print(f"Keep the binaries in {WORKINGDIR}") + + print("Verified hashes of") + print('\n'.join(verified_binaries)) + return 0 + + +if __name__ == '__main__': + sys.exit(main(sys.argv[1:])) diff --git a/contrib/windeploy/detached-sig-create.sh b/contrib/windeploy/detached-sig-create.sh new file mode 100755 index 0000000000..82fcf2d406 --- /dev/null +++ b/contrib/windeploy/detached-sig-create.sh @@ -0,0 +1,36 @@ +#!/bin/sh +# Copyright (c) 2014-2021 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +export LC_ALL=C +if [ -z "$OSSLSIGNCODE" ]; then + OSSLSIGNCODE=osslsigncode +fi + +if [ -z "$1" ]; then + echo "usage: $0 <osslcodesign args>" + echo "example: $0 -key codesign.key" + exit 1 +fi + +OUT=signature-win.tar.gz +SRCDIR=unsigned +WORKDIR=./.tmp +OUTDIR="${WORKDIR}/out" +OUTSUBDIR="${OUTDIR}/win" +TIMESERVER=http://timestamp.comodoca.com +CERTFILE="win-codesign.cert" + +mkdir -p "${OUTSUBDIR}" +# shellcheck disable=SC2046 +basename -a $(ls -1 "${SRCDIR}"/*-unsigned.exe) | while read UNSIGNED; do + echo Signing "${UNSIGNED}" + "${OSSLSIGNCODE}" sign -certs "${CERTFILE}" -t "${TIMESERVER}" -h sha256 -in "${SRCDIR}/${UNSIGNED}" -out "${WORKDIR}/${UNSIGNED}" "$@" + "${OSSLSIGNCODE}" extract-signature -pem -in "${WORKDIR}/${UNSIGNED}" -out "${OUTSUBDIR}/${UNSIGNED}.pem" && rm "${WORKDIR}/${UNSIGNED}" +done + +rm -f "${OUT}" +tar -C "${OUTDIR}" -czf "${OUT}" . +rm -rf "${WORKDIR}" +echo "Created ${OUT}" diff --git a/contrib/windeploy/win-codesign.cert b/contrib/windeploy/win-codesign.cert new file mode 100644 index 0000000000..22f17296b6 --- /dev/null +++ b/contrib/windeploy/win-codesign.cert @@ -0,0 +1,112 @@ +-----BEGIN CERTIFICATE-----
+MIIHfDCCBWSgAwIBAgIQCmVvdQal72U2QxbUTT3SRTANBgkqhkiG9w0BAQsFADBp
+MQswCQYDVQQGEwJVUzEXMBUGA1UEChMORGlnaUNlcnQsIEluYy4xQTA/BgNVBAMT
+OERpZ2lDZXJ0IFRydXN0ZWQgRzQgQ29kZSBTaWduaW5nIFJTQTQwOTYgU0hBMzg0
+IDIwMjEgQ0ExMB4XDTIyMDUyNDAwMDAwMFoXDTI0MDUyOTIzNTk1OVowgYAxCzAJ
+BgNVBAYTAlVTMREwDwYDVQQIEwhEZWxhd2FyZTEOMAwGA1UEBxMFTGV3ZXMxJjAk
+BgNVBAoTHUJpdGNvaW4gQ29yZSBDb2RlIFNpZ25pbmcgTExDMSYwJAYDVQQDEx1C
+aXRjb2luIENvcmUgQ29kZSBTaWduaW5nIExMQzCCAiIwDQYJKoZIhvcNAQEBBQAD
+ggIPADCCAgoCggIBALewxfjztuRTDNAGf7zkqqWNEt28CZmVJHoYltVRxtE1BP45
+BfmptH5eM1JC/XosTPytHRFeOkO4YVAtiELxK9S/82OZlKA7Mx7PW6vv1184u8+m
+P3WpTN/KAZTaW9fB0ELTSCuqsvXq2crM2T7NudJnSyWh2VBjLfPPCAcYwzyGKQbl
+jQWjFEJDJWFK83t9mK/v0WQgA3jGJeaz+V6CYXMS7UgpdG8dUhg9o63gYJZAW5pY
+RIsNRcJCM5LHhwEMW5329UsTmYCfP7/53emepbQ0n8ijVZjgaJ+LZ8NspBLSeCiF
+9UPCKX82uWiQAUTbYHCfSi3I0f3wQidXL9ZY+PXmalM7BMuQ+c2xEcl97CnhrDzx
+EBwZvvOC9wGoG+8+epV4TjUZWf+7QN1ZYeg1rai7c7c8u9ILogE8su2xVoz333TH
+CDvScIgnQXmk+cbKMBtg9kM0F+aLWsN2xVf0uAj3U7sdXLrfJeW0DZIktWtTBQzX
+O/OE4Ka+1WFnDg0HJIih0cTjl9YYvfe53L4pCGy+qGt/XGBRqCMfXp3g+H9FGR5r
+pensVVcsrv3GbTfYdlpdmp9OHH5G57GTAZueobCZg7r7RKK0zPU9EiTLJxzyXuai
+v/Ksd8eIhHRjewMaQuAtQM1tO+oKAbLF0v2M7v7/aVT76X32JllYAizm3zjvAgMB
+AAGjggIGMIICAjAfBgNVHSMEGDAWgBRoN+Drtjv4XxGG+/5hewiIZfROQjAdBgNV
+HQ4EFgQUvCpU58PIuofv0kHJ3Ty0YDKEy3cwDgYDVR0PAQH/BAQDAgeAMBMGA1Ud
+JQQMMAoGCCsGAQUFBwMDMIG1BgNVHR8Ega0wgaowU6BRoE+GTWh0dHA6Ly9jcmwz
+LmRpZ2ljZXJ0LmNvbS9EaWdpQ2VydFRydXN0ZWRHNENvZGVTaWduaW5nUlNBNDA5
+NlNIQTM4NDIwMjFDQTEuY3JsMFOgUaBPhk1odHRwOi8vY3JsNC5kaWdpY2VydC5j
+b20vRGlnaUNlcnRUcnVzdGVkRzRDb2RlU2lnbmluZ1JTQTQwOTZTSEEzODQyMDIx
+Q0ExLmNybDA+BgNVHSAENzA1MDMGBmeBDAEEATApMCcGCCsGAQUFBwIBFhtodHRw
+Oi8vd3d3LmRpZ2ljZXJ0LmNvbS9DUFMwgZQGCCsGAQUFBwEBBIGHMIGEMCQGCCsG
+AQUFBzABhhhodHRwOi8vb2NzcC5kaWdpY2VydC5jb20wXAYIKwYBBQUHMAKGUGh0
+dHA6Ly9jYWNlcnRzLmRpZ2ljZXJ0LmNvbS9EaWdpQ2VydFRydXN0ZWRHNENvZGVT
+aWduaW5nUlNBNDA5NlNIQTM4NDIwMjFDQTEuY3J0MAwGA1UdEwEB/wQCMAAwDQYJ
+KoZIhvcNAQELBQADggIBABhpTZufRws1vrtI0xB1/UWrSEJxdPHivfpXE708dzum
+Jh3TFzpsEUCQX5BJJet1l7x92sKNeAL7votA+8O8YvMD64Kim7VKA2BB8AOHKQbp
+r1c2iZBwwofInviRYvsrvQta6KBy2KOe1L/l0KnpUazL9Tv4VKvuWAw/Qc0/eTQr
+NZRsmADORxnZ1qW+SpF+/WbazIYjod/Oqb1U3on+PzyiGD3SjzNhsdFRptqzrIaY
+UVV+2XHG4fN6A8wkyQL5NIVXGiK7rqS5VrRAv58Lf1ZZTghdAL+5SySE0OsR9t0K
+W73ZB9pxbuZZ6Zfxjotjw+IilCEm3ADbc7Eb2ijI4x8mix0XWMUrhL34s7/jRyDi
+P+30aSgjWp611tp/EYRW5kpIaFR8AesDdM0DSSCCRXOMwQG2Tq2+CnqItB5oLNPp
+2XySwlIWvmjbzsREfIpE3yh3bxmHY+vFIc2R0nNkbWNIT6AGtaEQ7oWkgpK8YMkA
+QCf4EUC4Qa7qHiH6YSmYJhjApBLC7UDwevgwxuDrwimWAj+tDkzdnENMcBp4SAy6
+LwUuDi2IU6HRSXWdh2YEkDbc3FdwknnnEWaB4dlRL85YjHyLXN0KiE7SKTj1LfR4
+dGeDqVUlDj9D5+X4a7F89wLP/um40/52HUQv5t5WcNr/47r9aVkx9DHs1b8oUnLg
+-----END CERTIFICATE-----
+-----BEGIN CERTIFICATE----- +MIIGsDCCBJigAwIBAgIQCK1AsmDSnEyfXs2pvZOu2TANBgkqhkiG9w0BAQwFADBi +MQswCQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3 +d3cuZGlnaWNlcnQuY29tMSEwHwYDVQQDExhEaWdpQ2VydCBUcnVzdGVkIFJvb3Qg +RzQwHhcNMjEwNDI5MDAwMDAwWhcNMzYwNDI4MjM1OTU5WjBpMQswCQYDVQQGEwJV +UzEXMBUGA1UEChMORGlnaUNlcnQsIEluYy4xQTA/BgNVBAMTOERpZ2lDZXJ0IFRy +dXN0ZWQgRzQgQ29kZSBTaWduaW5nIFJTQTQwOTYgU0hBMzg0IDIwMjEgQ0ExMIIC +IjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEA1bQvQtAorXi3XdU5WRuxiEL1 +M4zrPYGXcMW7xIUmMJ+kjmjYXPXrNCQH4UtP03hD9BfXHtr50tVnGlJPDqFX/IiZ +wZHMgQM+TXAkZLON4gh9NH1MgFcSa0OamfLFOx/y78tHWhOmTLMBICXzENOLsvsI +8IrgnQnAZaf6mIBJNYc9URnokCF4RS6hnyzhGMIazMXuk0lwQjKP+8bqHPNlaJGi +TUyCEUhSaN4QvRRXXegYE2XFf7JPhSxIpFaENdb5LpyqABXRN/4aBpTCfMjqGzLm +ysL0p6MDDnSlrzm2q2AS4+jWufcx4dyt5Big2MEjR0ezoQ9uo6ttmAaDG7dqZy3S +vUQakhCBj7A7CdfHmzJawv9qYFSLScGT7eG0XOBv6yb5jNWy+TgQ5urOkfW+0/tv +k2E0XLyTRSiDNipmKF+wc86LJiUGsoPUXPYVGUztYuBeM/Lo6OwKp7ADK5GyNnm+ +960IHnWmZcy740hQ83eRGv7bUKJGyGFYmPV8AhY8gyitOYbs1LcNU9D4R+Z1MI3s +MJN2FKZbS110YU0/EpF23r9Yy3IQKUHw1cVtJnZoEUETWJrcJisB9IlNWdt4z4FK +PkBHX8mBUHOFECMhWWCKZFTBzCEa6DgZfGYczXg4RTCZT/9jT0y7qg0IU0F8WD1H +s/q27IwyCQLMbDwMVhECAwEAAaOCAVkwggFVMBIGA1UdEwEB/wQIMAYBAf8CAQAw +HQYDVR0OBBYEFGg34Ou2O/hfEYb7/mF7CIhl9E5CMB8GA1UdIwQYMBaAFOzX44LS +cV1kTN8uZz/nupiuHA9PMA4GA1UdDwEB/wQEAwIBhjATBgNVHSUEDDAKBggrBgEF +BQcDAzB3BggrBgEFBQcBAQRrMGkwJAYIKwYBBQUHMAGGGGh0dHA6Ly9vY3NwLmRp +Z2ljZXJ0LmNvbTBBBggrBgEFBQcwAoY1aHR0cDovL2NhY2VydHMuZGlnaWNlcnQu +Y29tL0RpZ2lDZXJ0VHJ1c3RlZFJvb3RHNC5jcnQwQwYDVR0fBDwwOjA4oDagNIYy +aHR0cDovL2NybDMuZGlnaWNlcnQuY29tL0RpZ2lDZXJ0VHJ1c3RlZFJvb3RHNC5j +cmwwHAYDVR0gBBUwEzAHBgVngQwBAzAIBgZngQwBBAEwDQYJKoZIhvcNAQEMBQAD +ggIBADojRD2NCHbuj7w6mdNW4AIapfhINPMstuZ0ZveUcrEAyq9sMCcTEp6QRJ9L +/Z6jfCbVN7w6XUhtldU/SfQnuxaBRVD9nL22heB2fjdxyyL3WqqQz/WTauPrINHV +UHmImoqKwba9oUgYftzYgBoRGRjNYZmBVvbJ43bnxOQbX0P4PpT/djk9ntSZz0rd +KOtfJqGVWEjVGv7XJz/9kNF2ht0csGBc8w2o7uCJob054ThO2m67Np375SFTWsPK +6Wrxoj7bQ7gzyE84FJKZ9d3OVG3ZXQIUH0AzfAPilbLCIXVzUstG2MQ0HKKlS43N +b3Y3LIU/Gs4m6Ri+kAewQ3+ViCCCcPDMyu/9KTVcH4k4Vfc3iosJocsL6TEa/y4Z +XDlx4b6cpwoG1iZnt5LmTl/eeqxJzy6kdJKt2zyknIYf48FWGysj/4+16oh7cGvm +oLr9Oj9FpsToFpFSi0HASIRLlk2rREDjjfAVKM7t8RhWByovEMQMCGQ8M4+uKIw8 +y4+ICw2/O/TOHnuO77Xry7fwdxPm5yg/rBKupS8ibEH5glwVZsxsDsrFhsP2JjMM +B0ug0wcCampAMEhLNKhRILutG4UI4lkNbcoFUCvqShyepf2gpx8GdOfy1lKQ/a+F +SCH5Vzu0nAPthkX0tGFuv2jiJmCG6sivqf6UHedjGzqGVnhO +-----END CERTIFICATE----- +-----BEGIN CERTIFICATE----- +MIIFkDCCA3igAwIBAgIQBZsbV56OITLiOQe9p3d1XDANBgkqhkiG9w0BAQwFADBi +MQswCQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3 +d3cuZGlnaWNlcnQuY29tMSEwHwYDVQQDExhEaWdpQ2VydCBUcnVzdGVkIFJvb3Qg +RzQwHhcNMTMwODAxMTIwMDAwWhcNMzgwMTE1MTIwMDAwWjBiMQswCQYDVQQGEwJV +UzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3d3cuZGlnaWNlcnQu +Y29tMSEwHwYDVQQDExhEaWdpQ2VydCBUcnVzdGVkIFJvb3QgRzQwggIiMA0GCSqG +SIb3DQEBAQUAA4ICDwAwggIKAoICAQC/5pBzaN675F1KPDAiMGkz7MKnJS7JIT3y +ithZwuEppz1Yq3aaza57G4QNxDAf8xukOBbrVsaXbR2rsnnyyhHS5F/WBTxSD1If +xp4VpX6+n6lXFllVcq9ok3DCsrp1mWpzMpTREEQQLt+C8weE5nQ7bXHiLQwb7iDV +ySAdYyktzuxeTsiT+CFhmzTrBcZe7FsavOvJz82sNEBfsXpm7nfISKhmV1efVFiO +DCu3T6cw2Vbuyntd463JT17lNecxy9qTXtyOj4DatpGYQJB5w3jHtrHEtWoYOAMQ +jdjUN6QuBX2I9YI+EJFwq1WCQTLX2wRzKm6RAXwhTNS8rhsDdV14Ztk6MUSaM0C/ +CNdaSaTC5qmgZ92kJ7yhTzm1EVgX9yRcRo9k98FpiHaYdj1ZXUJ2h4mXaXpI8OCi +EhtmmnTK3kse5w5jrubU75KSOp493ADkRSWJtppEGSt+wJS00mFt6zPZxd9LBADM +fRyVw4/3IbKyEbe7f/LVjHAsQWCqsWMYRJUadmJ+9oCw++hkpjPRiQfhvbfmQ6QY +uKZ3AeEPlAwhHbJUKSWJbOUOUlFHdL4mrLZBdd56rF+NP8m800ERElvlEFDrMcXK +chYiCd98THU/Y+whX8QgUWtvsauGi0/C1kVfnSD8oR7FwI+isX4KJpn15GkvmB0t +9dmpsh3lGwIDAQABo0IwQDAPBgNVHRMBAf8EBTADAQH/MA4GA1UdDwEB/wQEAwIB +hjAdBgNVHQ4EFgQU7NfjgtJxXWRM3y5nP+e6mK4cD08wDQYJKoZIhvcNAQEMBQAD +ggIBALth2X2pbL4XxJEbw6GiAI3jZGgPVs93rnD5/ZpKmbnJeFwMDF/k5hQpVgs2 +SV1EY+CtnJYYZhsjDT156W1r1lT40jzBQ0CuHVD1UvyQO7uYmWlrx8GnqGikJ9yd ++SeuMIW59mdNOj6PWTkiU0TryF0Dyu1Qen1iIQqAyHNm0aAFYF/opbSnr6j3bTWc +fFqK1qI4mfN4i/RN0iAL3gTujJtHgXINwBQy7zBZLq7gcfJW5GqXb5JQbZaNaHqa +sjYUegbyJLkJEVDXCLG4iXqEI2FCKeWjzaIgQdfRnGTZ6iahixTXTBmyUEFxPT9N +cCOGDErcgdLMMpSEDQgJlxxPwO5rIHQw0uA5NBCFIRUBCOhVMt5xSdkoF1BN5r5N +0XWs0Mr7QbhDparTwwVETyw2m+L64kW4I1NsBm9nVX9GtUw/bihaeSbSpKhil9Ie +4u1Ki7wb/UdKDd9nZn6yW0HQO+T0O/QEY+nvwlQAUaCKKsnOeMzV6ocEGLPOr0mI +r/OSmbaz5mEP0oUA51Aa5BuVnRmhuZyxm7EAHu/QD09CbMkKvO5D+jpxpchNJqU1 +/YldvIViHTLSoCtU7ZpXwdv6EM8Zt4tKG48BtieVU+i2iW1bvGjUI+iLUaJW+fCm +gKDWHrO8Dw9TdSmq6hN35N6MgSGtBxBHEa2HPQfRdbzP82Z+ +-----END CERTIFICATE----- diff --git a/contrib/zmq/zmq_sub.py b/contrib/zmq/zmq_sub.py new file mode 100755 index 0000000000..d8087a4db3 --- /dev/null +++ b/contrib/zmq/zmq_sub.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python3 +# Copyright (c) 2014-2021 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +""" + ZMQ example using python3's asyncio + + Bitcoin should be started with the command line arguments: + bitcoind -testnet -daemon \ + -zmqpubrawtx=tcp://127.0.0.1:28332 \ + -zmqpubrawblock=tcp://127.0.0.1:28332 \ + -zmqpubhashtx=tcp://127.0.0.1:28332 \ + -zmqpubhashblock=tcp://127.0.0.1:28332 \ + -zmqpubsequence=tcp://127.0.0.1:28332 + + We use the asyncio library here. `self.handle()` installs itself as a + future at the end of the function. Since it never returns with the event + loop having an empty stack of futures, this creates an infinite loop. An + alternative is to wrap the contents of `handle` inside `while True`. + + A blocking example using python 2.7 can be obtained from the git history: + https://github.com/bitcoin/bitcoin/blob/37a7fe9e440b83e2364d5498931253937abe9294/contrib/zmq/zmq_sub.py +""" + +import asyncio +import zmq +import zmq.asyncio +import signal +import struct +import sys + +if (sys.version_info.major, sys.version_info.minor) < (3, 5): + print("This example only works with Python 3.5 and greater") + sys.exit(1) + +port = 28332 + +class ZMQHandler(): + def __init__(self): + self.loop = asyncio.get_event_loop() + self.zmqContext = zmq.asyncio.Context() + + self.zmqSubSocket = self.zmqContext.socket(zmq.SUB) + self.zmqSubSocket.setsockopt(zmq.RCVHWM, 0) + self.zmqSubSocket.setsockopt_string(zmq.SUBSCRIBE, "hashblock") + self.zmqSubSocket.setsockopt_string(zmq.SUBSCRIBE, "hashtx") + self.zmqSubSocket.setsockopt_string(zmq.SUBSCRIBE, "rawblock") + self.zmqSubSocket.setsockopt_string(zmq.SUBSCRIBE, "rawtx") + self.zmqSubSocket.setsockopt_string(zmq.SUBSCRIBE, "sequence") + self.zmqSubSocket.connect("tcp://127.0.0.1:%i" % port) + + async def handle(self) : + topic, body, seq = await self.zmqSubSocket.recv_multipart() + sequence = "Unknown" + if len(seq) == 4: + sequence = str(struct.unpack('<I', seq)[-1]) + if topic == b"hashblock": + print('- HASH BLOCK ('+sequence+') -') + print(body.hex()) + elif topic == b"hashtx": + print('- HASH TX ('+sequence+') -') + print(body.hex()) + elif topic == b"rawblock": + print('- RAW BLOCK HEADER ('+sequence+') -') + print(body[:80].hex()) + elif topic == b"rawtx": + print('- RAW TX ('+sequence+') -') + print(body.hex()) + elif topic == b"sequence": + hash = body[:32].hex() + label = chr(body[32]) + mempool_sequence = None if len(body) != 32+1+8 else struct.unpack("<Q", body[32+1:])[0] + print('- SEQUENCE ('+sequence+') -') + print(hash, label, mempool_sequence) + # schedule ourselves to receive the next message + asyncio.ensure_future(self.handle()) + + def start(self): + self.loop.add_signal_handler(signal.SIGINT, self.stop) + self.loop.create_task(self.handle()) + self.loop.run_forever() + + def stop(self): + self.loop.stop() + self.zmqContext.destroy() + +daemon = ZMQHandler() +daemon.start() |