diff options
48 files changed, 2091 insertions, 1670 deletions
diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md index 811a246df1..55cebc008f 100644 --- a/.github/ISSUE_TEMPLATE.md +++ b/.github/ISSUE_TEMPLATE.md @@ -1,23 +1,21 @@ -<!--- Remove sections that do not apply --> +<!-- This issue tracker is only for technical issues related to Bitcoin Core. -This issue tracker is only for technical issues related to Bitcoin Core. +General bitcoin questions and/or support requests are best directed to the Bitcoin StackExchange at https://bitcoin.stackexchange.com. -General bitcoin questions and/or support requests are best directed to the [Bitcoin StackExchange](https://bitcoin.stackexchange.com). +For reporting security issues, please read instructions at https://bitcoincore.org/en/contact/. -For reporting security issues, please read instructions at [https://bitcoincore.org/en/contact/](https://bitcoincore.org/en/contact/). +If the node is "stuck" during sync or giving "block checksum mismatch" errors, please ensure your hardware is stable by running memtest and observe CPU temperature with a load-test tool such as linpack before creating an issue! --> -If the node is "stuck" during sync or giving "block checksum mismatch" errors, please ensure your hardware is stable by running memtest and observe CPU temperature with a load-test tool such as linpack before creating an issue! +<!-- Describe the issue --> +<!--- What behavior did you expect? --> -### Describe the issue -#### What behavior did you expect? +<!--- What was the actual behavior (provide screenshots if the issue is GUI-related)? --> -#### What was the actual behavior (provide screenshots if the issue is GUI-related)? +<!--- How reliably can you reproduce the issue, what are the steps to do so? --> -#### How reliably can you reproduce the issue, what are the steps to do so? +<!-- What version of Bitcoin Core are you using, where did you get it (website, self-compiled, etc)? --> -### What version of Bitcoin Core are you using, where did you get it (website, self-compiled, etc)? +<!-- What type of machine are you observing the error on (OS/CPU and disk type)? --> -### What type of machine are you observing the error on (OS/CPU and disk type)? - -### Any extra information that might be useful in the debugging process. -This is normally the contents of a `debug.log` or `config.log` file. Raw text or a link to a pastebin type site are preferred. +<!-- Any extra information that might be useful in the debugging process. --> +<!--- This is normally the contents of a `debug.log` or `config.log` file. Raw text or a link to a pastebin type site are preferred. --> diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 9f95d3f818..8381bd2448 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -186,7 +186,7 @@ In general, all pull requests must: the project (for example refactoring for modularisation); - Be well peer reviewed; - Have unit tests and functional tests where appropriate; - - Follow code style guidelines; + - Follow code style guidelines ([C++](doc/developer-notes.md), [functional tests](test/functional/README.md)); - Not break the existing test suite; - Where bugs are fixed, where possible, there should be unit tests demonstrating the bug and also proving the fix. This helps prevent regression. diff --git a/Makefile.am b/Makefile.am index a7092bb334..9b791cc0e8 100644 --- a/Makefile.am +++ b/Makefile.am @@ -42,6 +42,7 @@ DIST_CONTRIB = $(top_srcdir)/contrib/bitcoin-cli.bash-completion \ $(top_srcdir)/contrib/bitcoin-tx.bash-completion \ $(top_srcdir)/contrib/bitcoind.bash-completion \ $(top_srcdir)/contrib/init \ + $(top_srcdir)/contrib/install_db4.sh \ $(top_srcdir)/contrib/rpm DIST_SHARE = \ $(top_srcdir)/share/genbuild.sh \ diff --git a/contrib/gitian-keys/willyko-key.pgp b/contrib/gitian-keys/willyko-key.pgp new file mode 100644 index 0000000000..a3b58ad994 --- /dev/null +++ b/contrib/gitian-keys/willyko-key.pgp @@ -0,0 +1,85 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- +Version: GnuPG v1 + +mQINBFgs/RoBEADFxycJTUvwqzBZZ0aBZXbmr8Ppd3EPrgBRd47k7uwanf7UFmvY +Xt4gMEI+EdV0GuoQ0SeoAmQqc5Fxu3AQe2XFbiF+ZNNYT3+V/5GAzWsAH22ncQr0 +AuK95pPi+PZ+M2h669cq/RzFUXZDew0NobR2oBS5h6g3rgmmejVLRqnUpWkkSrqi +aNgD2GSn8g820wM6LpdxcjTqmMpHHT5owAbv0UP3IcdtpBaS5McoUXK+OAdKK/Zw +JQ0J1kx4vIyNwuPD3klziGQw8Izb/gFpWg8XaJmMhD5BxNuXJC58Bj9/sFTc0GDQ +VKMFpYpNi8a6hLPFb4hMjYF77awoz57HtyOOsS03KO/57QE1htx+2NeDm4XkZSBk ++wrU3zgbtmOBcfzEHS/HrROksYDi+Qw3HZL98nfDEWNfsDzfhMZ9wHdM3NsR2xk6 +oNtX0CprS1n2Xr2AY9X1oNgiZCJaSftU67j3lr+9gHOH61ktxt3cUCDodUFjkpKn +r1CQ2LB63AoUbwGMAeozdXZWzbXJAJbcH9G77zEi9rW0WA2yMSxTXHlpE9MS0UcE +BVkIMv2b9iQzlhiS8jh8AiKFO1PuT26Cw52N/lSPhA81zw79pZfSYwKFICGHYfvw +ozZeN9Q+PPl5tqi/3SExxlZKe8EmaveTrUfKMBS4lQO2gWe0bCFgLOIzIwARAQAB +tB1XaWxseSBLbyA8d2lsbHlrQHN5c2NvaW4ub3JnPokCOAQTAQIAIgUCWCz9GgIb +AwYLCQgHAwIGFQgCCQoLBBYCAwECHgECF4AACgkQjjqPMkfby7+0wA//cX7Tc3Nz +19ApwSbGfC8pJA/nSybcVivroJRftpzeOmYrVM084T9REvYwugl89djvxn6m96iQ +kqoUGWhBVBtDReVCL7z53G42lHjemaFcxBhIazKxO0qvcc/UXUVOs2OdUbzObDFL +dHO5xBVqEnW3sq+r4blsXR8U79B9IIri4+2hy4OoEjYv9DzBaaoaqU+F3mudXbmo +R+hsWc+mklV++TX/kuw6EWT8tusFjXrfqqKcKPRPhbn48OSGWsEPc7yELf7pYFR8 +uDU40faJqkvQ83h5WMTDAhLxd/918ZitqBhjSP+7Humf2YhSto7YmtEWlbeAW+qy +TcBYkK6SJh8Do3xZd/prFBKEu395n5VQKuLjXaOjqMc1oDHQyPJJjXSN5thLHvan +z7nNLt2QZO/kxXITDdbWlktVe/WSoive7TuY4dGuX4Si2z9wyhFYxtZDsqE0qmqN +jIDAZ7u8Qq/LGqpdjOmYr2fEwHe1yVIS+BtVGvtShkX+J+QPb8qBl1d7Ii5i5Afl +GJoLLIUFkPcIRTYPZpppGSuqfyWdNnaasbLH44lxJisSMMw+fxZabt2bykYN/ZXa +RP/ItDj81vklg+n6r4f/nZTF1r0UUy4LbSbBY15B4Xm0Tdvh1PMfj/w2q10l7bZB +XLi9Z/QPaW7TyzaBuLkVckbVFn2nYnXfzHG5Ag0EWCz9GgEQALCgTibFnw+Q3PEL +G5/peQcQqHxrPAB37HV39B1DedGhVUa6aGSUaLoNMyUjUX1HWN3mWFKTYVB4CH5Y +xjaXUwxdwCZgBNe4TDglKFPuc+frlSTZxDVE9/fjArmrUP6TPU447ujspyngGLa5 +et5Uig/LxIX/+Mm0ZiYJxb1rMJwK998U1Ev1aHxgNjwTI2ehcKu8CAGOyflzh6a2 +iTBUmLfnQMv5248P2d4P8WDiPq61CWTYTMCFqHqkYKy7h9BYIuMajw3KsgOUNfL2 +1e9Ue8yv5UchZ+GDlBjidIkoK1nd2uJ0kPJkafLGWbcliJfvXxKliZnSbz1Cd4A0 +HDKKCwiuwSLy2aYbs7IRtAOyDER4+fjBcqtf0QTIvoAdNZ9gL64DKVaB58vuSixj +K1i83XbTOt3q821HxxBrX9u6HP2E5kFdxT2KHDbisAWNP0rFnHVpjugehKFfZb6q +jbDt3nQL5uCQ8gTNCd4fsoSK6KhCDjamDXlKmaGlxqwOV4W8ZwihoeGt690h7NIH +h4eiSmMOej3or32lcDETEwrjA2PxvcFsikFc56hRkTaSyyBEH2xhkRrjXMqiQfH0 +j7iOY2PWpFEuu2HVzqe5dBXzn9sMIwxeNCxR/P+xHMqPUlgD1SXEYCNLvvzD6p0+ +kqSe7PiJoEIv351T3hnBhQ6rK0ChABEBAAGJAh8EGAECAAkFAlgs/RoCGwwACgkQ +jjqPMkfby7/mQA//YsAOdDBl0GscB1PBNXi8VMlI7yG9cqiGrYnZX7h4wUoGEbPI +jap/PixIsxBCf1BqBRDJdFyvzH9amLlcaVNdCyh6Yt1Pi8kassmz/kbIYgpbFkIL +ES9N24N7BZ94P77OQy5wic+B4WqJnVrtKr9JBalgBSOMqtccYCma5Ew00mqp+FXM +suDyBk2HXyl+u6/rRmqZ+BoU8iRpus9F80LFKGEsAgjLjKv68KmApzjunzsBotKk +g9AsBk4ygbp+nECAtsxpbLMo4hPr4qWm7G4mU5g4xOK2chpAPeqyf0857RWgsXaO +kjrUu/M5Hme2eIlXwBF14ac4QPnY1rlAIaulvXzmQnMYQFZiw9vaTOdqBFHjkh7T +XYRAr589Woo25PfMJCbC+Rop6ku6sCFMorbBwojyRhFJnk9xsy5kP5D9IhkPAKu/ +/ABlei0xPOl/gCUUJP7aIikZgS5lAk1TSe/R+yV6ExNwudtLw1G0K2/sY3B4Xo3X +Q1lTAQPlnAIeK/vlbttLZNIBWquw4cPAkPpIyjmE1dd6jGQdUyZE22uPBx+gpq1w +AacmVLwvPMe1De0ilJOzj6KpXWBCwt0DWXWztovpBVcAC+qbTrZF9H5dllpqyzKt +OvxzGssjrX4rDkOx7MyVa2tnXmeCuSN/RvlOUwPvf5zYM8Wh9g7fc6jcDQu5Ag0E +WfkOfAEQAMNkzAQqSenpXtHsnuCqM1oMMF2kRzny/Jqh3q3BxZ8MHLDhoRRaTENu +lA4APRXMNM/wlZJUSLX8wWBhufnsPtMf6OOVMZ4AVbXHjUgyJ7lO1zHdj0u6PpYP +9gmHthIz7FF+cxHj4ziC4CmtRctrn+/U4MwYtNPhxkTnS26oOZes/HXMYSvQBMgT +AP27GNOBiJRthjIEITvSvS0YZOxgLtWgGiks/pGUw5wm1rguuQVyZ1/LfXBooYJo +u/v21AEjpuTg7JlwbqXr2k5LojAGq7AxDyWy21IW0E45Gog38zg/hwNll+hjRbSu +pipf74WXR6xMMlW6A+XWUvElkicfDx8e9LJUnqWbZ+FL7X4SB54ZHNCvfo/8Ug1V +2tiY9WbUZL9n5ZQHNlk3J+UK/KDvwey1VzKPFjpQNlfahhnppDGiCey+mERjI+75 +gPbk0ctOAEYXgLJjoonGX+iByAfY0YyJF281CtaK/sXQU+TzLLT15WET+gYGsJdY +xh1PdPscNdSgYudvbKZoFnqUwEGEfD8dT5bjOphfY5+LvGUR2GuLNZpMidcduTYf +SWAY/vQHQIJArXu29BKscm3tg6tzXu3l9p/bGIQUQB7obN91y3xD3BLICIPRGhKE +924wxxCuH1vLKmxWDdAAxKo+rEdLJ/rbZnjWQENEFiJ114fBk2NVABEBAAGJBEQE +GAEIAA8FAln5DnwCGwIFCQPCZwACKQkQjjqPMkfby7/BXSAEGQEIAAYFAln5DnwA +CgkQYFGSo/6YSmANoQ//SbcKxkop2zA2HrWS4THcEJQwSJ0KGAN/VB83JQhoWThX +CWxsFNJjBy7+rsoXd3wQG1/aN42nTuj+eh+R6WJJaqqnMqd52l4Kc1kJA6z4DGsy +3azCDvyzibM0AkJyMJyYi6HRKjzA4M+xKR1HoT/NdQUP5CBUVfvMblSaOWiw4rja +IhWcbgbQ+Zam/VaV5l1O90eaD9tL3twSfPLYZ/wkeO63jJKHBpI8fpMql/bLg9WD +Au3h/lU63NWe5lZO1z/jIdfiTSvg8nu162vcOgmUCWo9spkybjJd0Mx6ZId79rVo +58lwZ4QoaMgPGoVP67LyLOxJTIXeyG5xr1LxhMPMGbnBhlnMQrboLV9kPEL3raHE +EEKDTtZimVK3ZxmfyBd6MDmwcL/K73xu/R8be9TgdwD8/BZJSOTkO87qZ82G9T7E +oY5IHU+qd41/Yjbut7AVtAlCr5Lor31EYvZh3gI/H8uZFddOu37Ij7e9Fw2ywv3A +wPks89tfOvahkfCOJ29znB+uQYpJ461jjhdkB7EHG4ae07M5rRtkNbIc3dqbnMhz +VA3JpRJN77xPXV7uITHo1s+b50RvWmfYW91zvipaSZxbMLuGBMhn/1QaM1djLOYN +JordDBwEr2bi5a063yUbZrk6ddECuyxndDHWDNr+Tqx6o7lmAT48UJ199zA4scbf +2g/9EiRPGcRovsn1tUdjzfmWDxhrRV6F3rYJB1+i6Mqeg2iHHYxxiNDXcuWYXHQ/ +WPWLk5+lgh0rQbrE7InzEejoM0FIHzLTm0lSQpau50/PT2FiH6sOEEDyT2IhBtXX +eOnKAi1IfGNMzEaPEY8PXH78dEGv0iXIgy4l8Bc57q09Z9R/OUi+Yb1p+S5F/aOi +7Jd53GGE1bfBIlsMos092XoiMdvKmAczyCUIempKHUBPoqfJge77qk7zJKkyM3Dk +VX0lXLdhj0PfslFrNf2uRF4uZkmfUV7peeD023c0/SVp3ILUAVds52yawi6Exv4a +bbvhIw72fc31frCRBqc9HVsBraoozzE9bksG1MdNI3GgKxecOu9lldedlIqi4lO4 +7kTVDLEmcsQO+sSxkXQz2sMSD01CQndpPuhFNlqvVnfK+Kv8pSG37VzSSQz1nt5K +w/fJBo4T/ztR7D9RzbSDxBP8Jjaa+UYabjab5HcE0JI4CpgmzIOB7qPVbYCn+LNX +c8Xw5/9iTw+ayawl7PCGRfd14/OPRzI8vS0I9bF8AG84XM46yxAtYieH/9RI3b6/ +GiQYDkBNi6Kb1LfSzx8oKAkbMgiy4y3vWxLQnE34bAoXjGiYdAMliOsyGcvmnObD +GmSTIlIqunq60CyhaUSIkl2VRhjzz0igfS9751XEvnjeXDc= +=PVBi +-----END PGP PUBLIC KEY BLOCK----- diff --git a/contrib/install_db4.sh b/contrib/install_db4.sh new file mode 100755 index 0000000000..1d33e8d3ba --- /dev/null +++ b/contrib/install_db4.sh @@ -0,0 +1,87 @@ +#!/bin/sh + +# Install libdb4.8 (Berkeley DB). + +set -e + +if [ -z "${1}" ]; then + echo "Usage: ./install_db4.sh <base-dir> [<extra-bdb-configure-flag> ...]" + echo + echo "Must specify a single argument: the directory in which db5 will be built." + echo "This is probably \`pwd\` if you're at the root of the bitcoin repository." + exit 1 +fi + +expand_path() { + echo "$(cd "${1}" && pwd -P)" +} + +BDB_PREFIX="$(expand_path ${1})/db4"; shift; +BDB_EXTRA_CONFIGURE_FLAGS="${@}" +BDB_VERSION='db-4.8.30.NC' +BDB_HASH='12edc0df75bf9abd7f82f821795bcee50f42cb2e5f76a6a281b85732798364ef' +BDB_URL="https://download.oracle.com/berkeley-db/${BDB_VERSION}.tar.gz" + +check_exists() { + which "$1" >/dev/null 2>&1 +} + +sha256_check() { + # Args: <sha256_hash> <filename> + # + if check_exists sha256sum; then + echo "${1} ${2}" | sha256sum -c + elif check_exists sha256; then + echo "${1} ${2}" | sha256 -c + else + echo "${1} ${2}" | shasum -a 256 -c + fi +} + +http_get() { + # Args: <url> <filename> <sha256_hash> + # + # It's acceptable that we don't require SSL here because we manually verify + # content hashes below. + # + if [ -f "${2}" ]; then + echo "File ${2} already exists; not downloading again" + elif check_exists curl; then + curl --insecure "${1}" -o "${2}" + else + wget --no-check-certificate "${1}" -O "${2}" + fi + + sha256_check "${3}" "${2}" +} + +mkdir -p "${BDB_PREFIX}" +http_get "${BDB_URL}" "${BDB_VERSION}.tar.gz" "${BDB_HASH}" +tar -xzvf ${BDB_VERSION}.tar.gz -C "$BDB_PREFIX" +cd "${BDB_PREFIX}/${BDB_VERSION}/" + +# Apply a patch when building on OS X to make the build work with Xcode. +# +if [ "$(uname)" = "Darwin" ]; then + BDB_OSX_ATOMIC_PATCH_URL='https://raw.githubusercontent.com/narkoleptik/os-x-berkeleydb-patch/0007e2846ae3fc9757849f5277018f4179ad17ef/atomic.patch' + BDB_OSX_ATOMIC_PATCH_HASH='ba0e2b4f53e9cb0ec58f60a979b53b8567b4565f0384886196f1fc1ef111d151' + + http_get "${BDB_OSX_ATOMIC_PATCH_URL}" atomic.patch "${BDB_OSX_ATOMIC_PATCH_HASH}" + patch -p1 < atomic.patch +fi + +cd build_unix/ + +"${BDB_PREFIX}/${BDB_VERSION}/dist/configure" \ + --enable-cxx --disable-shared --with-pic --prefix="${BDB_PREFIX}" \ + "${BDB_EXTRA_CONFIGURE_FLAGS}" + +make install + +echo +echo "db4 build complete." +echo +echo 'When compiling bitcoind, run `./configure` in the following way:' +echo +echo " export BDB_PREFIX='${BDB_PREFIX}'" +echo ' ./configure LDFLAGS="-L${BDB_PREFIX}/lib/" CPPFLAGS="-I${BDB_PREFIX}/include/" ...' diff --git a/doc/build-openbsd.md b/doc/build-openbsd.md index 760bb69b15..cd1d217b47 100644 --- a/doc/build-openbsd.md +++ b/doc/build-openbsd.md @@ -38,28 +38,17 @@ The default C++ compiler that comes with OpenBSD 6.2 is g++ 4.2.1. This version BerkeleyDB is only necessary for the wallet functionality. To skip this, pass `--disable-wallet` to `./configure`. -See "Berkeley DB" in [build-unix.md](build-unix.md#berkeley-db) for instructions on how to build BerkeleyDB 4.8. -You cannot use the BerkeleyDB library from ports, for the same reason as boost above (g++/libstd++ incompatibility). +It is recommended to use Berkeley DB 4.8. You cannot use the BerkeleyDB library +from ports, for the same reason as boost above (g++/libstd++ incompatibility). +If you have to build it yourself, you can use [the installation script included +in contrib/](contrib/install_db4.sh) like so -```bash -# Pick some path to install BDB to, here we create a directory within the bitcoin directory -BITCOIN_ROOT=$(pwd) -BDB_PREFIX="${BITCOIN_ROOT}/db4" -mkdir -p $BDB_PREFIX - -# Fetch the source and verify that it is not tampered with -curl -o db-4.8.30.NC.tar.gz 'http://download.oracle.com/berkeley-db/db-4.8.30.NC.tar.gz' -echo '12edc0df75bf9abd7f82f821795bcee50f42cb2e5f76a6a281b85732798364ef db-4.8.30.NC.tar.gz' | sha256 -c -# MUST output: (SHA256) db-4.8.30.NC.tar.gz: OK -tar -xzf db-4.8.30.NC.tar.gz - -# Build the library and install to specified prefix -cd db-4.8.30.NC/build_unix/ -# Note: Do a static build so that it can be embedded into the executable, instead of having to find a .so at runtime -../dist/configure --enable-cxx --disable-shared --with-pic --prefix=$BDB_PREFIX CC=egcc CXX=eg++ CPP=ecpp -make install # do NOT use -jX, this is broken +```shell +./contrib/install_db4.sh `pwd` CC=egcc CXX=eg++ CPP=ecpp ``` +from the root of the repository. + ### Resource limits The standard ulimit restrictions in OpenBSD are very strict: diff --git a/doc/build-osx.md b/doc/build-osx.md index 836c856a64..6f08562ff8 100644 --- a/doc/build-osx.md +++ b/doc/build-osx.md @@ -26,6 +26,20 @@ If you want to build the disk image with `make deploy` (.dmg / optional), you ne NOTE: Building with Qt4 is still supported, however, could result in a broken UI. Building with Qt5 is recommended. +Berkeley DB +----------- +It is recommended to use Berkeley DB 4.8. If you have to build it yourself, +you can use [the installation script included in contrib/](contrib/install_db4.sh) +like so + +```shell +./contrib/install_db4.sh . +``` + +from the root of the repository. + +**Note**: You only need Berkeley DB if the wallet is enabled (see the section *Disable-Wallet mode* below). + Build Bitcoin Core ------------------------ diff --git a/doc/build-unix.md b/doc/build-unix.md index 8a102abaea..5d3329e2cf 100644 --- a/doc/build-unix.md +++ b/doc/build-unix.md @@ -165,33 +165,16 @@ turned off by default. See the configure options for upnp behavior desired: Berkeley DB ----------- -It is recommended to use Berkeley DB 4.8. If you have to build it yourself: +It is recommended to use Berkeley DB 4.8. If you have to build it yourself, +you can use [the installation script included in contrib/](contrib/install_db4.sh) +like so -```bash -BITCOIN_ROOT=$(pwd) - -# Pick some path to install BDB to, here we create a directory within the bitcoin directory -BDB_PREFIX="${BITCOIN_ROOT}/db4" -mkdir -p $BDB_PREFIX - -# Fetch the source and verify that it is not tampered with -wget 'http://download.oracle.com/berkeley-db/db-4.8.30.NC.tar.gz' -echo '12edc0df75bf9abd7f82f821795bcee50f42cb2e5f76a6a281b85732798364ef db-4.8.30.NC.tar.gz' | sha256sum -c -# -> db-4.8.30.NC.tar.gz: OK -tar -xzvf db-4.8.30.NC.tar.gz - -# Build the library and install to our prefix -cd db-4.8.30.NC/build_unix/ -# Note: Do a static build so that it can be embedded into the executable, instead of having to find a .so at runtime -../dist/configure --enable-cxx --disable-shared --with-pic --prefix=$BDB_PREFIX -make install - -# Configure Bitcoin Core to use our own-built instance of BDB -cd $BITCOIN_ROOT -./autogen.sh -./configure LDFLAGS="-L${BDB_PREFIX}/lib/" CPPFLAGS="-I${BDB_PREFIX}/include/" # (other args...) +```shell +./contrib/install_db4.sh `pwd` ``` +from the root of the repository. + **Note**: You only need Berkeley DB if the wallet is enabled (see the section *Disable-Wallet mode* below). Boost diff --git a/doc/build-windows.md b/doc/build-windows.md index 9e3e38d188..9e0e66e522 100644 --- a/doc/build-windows.md +++ b/doc/build-windows.md @@ -3,29 +3,28 @@ WINDOWS BUILD NOTES Below are some notes on how to build Bitcoin Core for Windows. -Most developers use cross-compilation from Ubuntu to build executables for -Windows. This is also used to build the release binaries. - -Building on Ubuntu Trusty 14.04 is recommended. -At the time of writing the Windows Subsystem for Linux installs Ubuntu Xenial 16.04. The default cross -compiler package for Ubuntu Xenial does not produce working executables for some of the bitcoin applications. -It is possible to build on Ubuntu Xenial by installing the cross compiler packages from -Ubuntu Zesty, see the steps below. -Building on Ubuntu Zesty 17.04 up to 17.10 has been verified to work. +The options known to work for building Bitcoin Core on Windows are: + +* On Linux using the [Mingw-w64](https://mingw-w64.org/doku.php) cross compiler tool chain. Ubuntu Trusty 14.04 is recommended +and is the platform used to build the Bitcoin Core Windows release binaries. +* On Windows using [Windows +Subsystem for Linux (WSL)](https://msdn.microsoft.com/commandline/wsl/about) and the Mingw-w64 cross compiler tool chain. + +Other options which may work but which have not been extensively tested are (please contribute instructions): -While there are potentially a number of ways to build on Windows (for example using msys / mingw-w64), -using the Windows Subsystem For Linux is the most straightforward. If you are building with -another method, please contribute the instructions here for others who are running versions -of Windows that are not compatible with the Windows Subsystem for Linux. +* On Windows using a POSIX compatibility layer application such as [cygwin](http://www.cygwin.com/) or [msys2](http://www.msys2.org/). +* On Windows using a native compiler tool chain such as [Visual Studio](https://www.visualstudio.com). -Compiling with Windows Subsystem For Linux -------------------------------------------- +Installing Windows Subsystem for Linux +--------------------------------------- With Windows 10, Microsoft has released a new feature named the [Windows Subsystem for Linux (WSL)](https://msdn.microsoft.com/commandline/wsl/about). This feature allows you to run a bash shell directly on Windows in an Ubuntu-based environment. Within this environment you can cross compile for Windows without -the need for a separate Linux VM or server. +the need for a separate Linux VM or server. Note that while WSL can be installed with +other Linux variants, such as OpenSUSE, the following instructions have only been +tested with Ubuntu. This feature is not supported in versions of Windows prior to Windows 10 or on Windows Server SKUs. In addition, it is available [only for 64-bit versions of @@ -50,10 +49,15 @@ After the bash shell is active, you can follow the instructions below, starting with the "Cross-compilation" section. Compiling the 64-bit version is recommended but it is possible to compile the 32-bit version. -Cross-compilation -------------------- +Cross-compilation for Ubuntu and Windows Subsystem for Linux +------------------------------------------------------------ -These steps can be performed on, for example, an Ubuntu VM. The depends system +At the time of writing the Windows Subsystem for Linux installs Ubuntu Xenial 16.04. The Mingw-w64 package +for Ubuntu Xenial does not produce working executables for some of the Bitcoin Core applications. +It is possible to build on Ubuntu Xenial by installing the cross compiler packages from Ubuntu Zesty, see the steps below. +Building on Ubuntu Zesty 17.04 up to 17.10 has been verified to work. + +The steps below can be performed on Ubuntu (including in a VM) or WSL. The depends system will also work on other Linux distributions, however the commands for installing the toolchain will be different. @@ -94,6 +98,18 @@ Ubuntu Zesty 17.04 <sup>[2](#footnote2)</sup>: Once the tool chain is installed the build steps are common: +Note that for WSL the Bitcoin Core source path MUST be somewhere in the default mount file system, for +example /usr/src/bitcoin, AND not under /mnt/d/. If this is not the case the dependency autoconf scripts will fail. +This means you cannot use a directory that located directly on the host Windows file system to perform the build. + +The next three steps are an example of how to acquire the source in an appropriate way. + + cd /usr/src + sudo git clone https://github.com/bitcoin/bitcoin.git + sudo chmod -R a+rw bitcoin + +Once the source code is ready the build steps are below. + PATH=$(echo "$PATH" | sed -e 's/:\/mnt.*//g') # strip out problematic Windows %PATH% imported var cd depends make HOST=x86_64-w64-mingw32 @@ -112,6 +128,16 @@ For Ubuntu Xenial 16.04, Ubuntu Zesty 17.04 and Windows Subsystem for Linux <sup sudo update-alternatives --config i686-w64-mingw32-g++ # Set the default mingw32 g++ compiler option to posix. +Note that for WSL the Bitcoin Core source path MUST be somewhere in the default mount file system, for +example /usr/src/bitcoin, AND not under /mnt/d/. If this is not the case the dependency autoconf scripts will fail. +This means you cannot use a directory that located directly on the host Windows file system to perform the build. + +The next three steps are an example of how to acquire the source in an appropriate way. + + cd /usr/src + sudo git clone https://github.com/bitcoin/bitcoin.git + sudo chmod -R a+rw bitcoin + Then build using: PATH=$(echo "$PATH" | sed -e 's/:\/mnt.*//g') # strip out problematic Windows %PATH% imported var @@ -139,16 +165,16 @@ way. This will install to `c:\workspace\bitcoin`, for example: Footnotes --------- -<a name="footnote1">1</a>: There is currently a bug in the 64 bit mingw-w64 cross compiler packaged for WSL/Ubuntu Xenial 16.04 that +<a name="footnote1">1</a>: There is currently a bug in the 64 bit Mingw-w64 cross compiler packaged for WSL/Ubuntu Xenial 16.04 that causes two of the bitcoin executables to crash shortly after start up. The bug is related to the -fstack-protector-all g++ compiler flag which is used to mitigate buffer overflows. -Installing the mingw-w64 packages from the Ubuntu 17 distribution solves the issue, however, this is not +Installing the Mingw-w64 packages from the Ubuntu 17 distribution solves the issue, however, this is not an officially supported approach and it's only recommended if you are prepared to reinstall WSL/Ubuntu should something break. -<a name="footnote2">2</a>: Starting from Ubuntu Xenial 16.04 both the 32 and 64 bit mingw-w64 packages install two different +<a name="footnote2">2</a>: Starting from Ubuntu Xenial 16.04 both the 32 and 64 bit Mingw-w64 packages install two different compiler options to allow a choice between either posix or win32 threads. The default option is win32 threads which is the more efficient since it will result in binary code that links directly with the Windows kernel32.lib. Unfortunately, the headers required to support win32 threads conflict with some of the classes in the C++11 standard library in particular std::mutex. -It's not possible to build the bitcoin code using the win32 version of the mingw-w64 cross compilers (at least not without +It's not possible to build the bitcoin code using the win32 version of the Mingw-w64 cross compilers (at least not without modifying headers in the bitcoin source code). diff --git a/doc/dependencies.md b/doc/dependencies.md index 964c03ba88..6e1d1ee992 100644 --- a/doc/dependencies.md +++ b/doc/dependencies.md @@ -13,7 +13,7 @@ These are the dependencies currently used by Bitcoin Core. You can find instruct | Expat | [2.2.1](https://libexpat.github.io/) | | No | Yes | | | fontconfig | [2.12.1](https://www.freedesktop.org/software/fontconfig/release/) | | No | Yes | | | FreeType | [2.7.1](http://download.savannah.gnu.org/releases/freetype) | | No | | | -| GCC | | [4.7+](https://gcc.gnu.org/) | | | | +| GCC | | [4.8+](https://gcc.gnu.org/) | | | | | HarfBuzz-NG | | | | | | | libevent | [2.1.8-stable](https://github.com/libevent/libevent/releases) | 2.0.22 | No | | | | libjpeg | | | | | [Yes](https://github.com/bitcoin/bitcoin/blob/master/depends/packages/qt.mk#L75) | diff --git a/doc/release-notes.md b/doc/release-notes.md index 6e4fed0a2d..2c63b1f88e 100644 --- a/doc/release-notes.md +++ b/doc/release-notes.md @@ -20,7 +20,7 @@ How to Upgrade ============== If you are running an older version, shut it down. Wait until it has completely -shut down (which might take a few minutes for older versions), then run the +shut down (which might take a few minutes for older versions), then run the installer (on Windows) or just copy over `/Applications/Bitcoin-Qt` (on Mac) or `bitcoind`/`bitcoin-qt` (on Linux). @@ -56,12 +56,31 @@ frequently tested on them. Notable changes =============== +GCC 4.8.x +-------------- +The minimum version of GCC required to compile Bitcoin Core is now 4.8. No effort will be +made to support older versions of GCC. See discussion in issue #11732 for more information. + HD-wallets by default --------------------- Due to a backward-incompatible change in the wallet database, wallets created with version 0.16.0 will be rejected by previous versions. Also, version 0.16.0 will only create hierarchical deterministic (HD) wallets. +Custom wallet directories +--------------------- +The ability to specify a directory other than the default data directory in which to store +wallets has been added. An existing directory can be specified using the `-walletdir=<dir>` +argument. Wallets loaded via `-wallet` arguments must be in this wallet directory. Care should be taken +when choosing a wallet directory location, as if it becomes unavailable during operation, +funds may be lost. + +Default wallet directory change +-------------------------- +On new installations (if the data directory doesn't exist), wallets will now be stored in a +new `wallets/` subdirectory inside the data directory. If this `wallets/` subdirectory +doesn't exist (i.e. on existing nodes), the current datadir root is used instead, as it was. + Low-level RPC changes ---------------------- - The deprecated RPC `getinfo` was removed. It is recommended that the more specific RPCs are used: diff --git a/src/Makefile.am b/src/Makefile.am index 29e4ae56d0..4b65774fc6 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -168,6 +168,7 @@ BITCOIN_CORE_H = \ wallet/rpcwallet.h \ wallet/wallet.h \ wallet/walletdb.h \ + wallet/walletutil.h \ warnings.h \ zmq/zmqabstractnotifier.h \ zmq/zmqconfig.h\ @@ -249,6 +250,7 @@ libbitcoin_wallet_a_SOURCES = \ wallet/rpcwallet.cpp \ wallet/wallet.cpp \ wallet/walletdb.cpp \ + wallet/walletutil.cpp \ $(BITCOIN_CORE_H) # crypto primitives library diff --git a/src/Makefile.qt.include b/src/Makefile.qt.include index cbf3960cc7..0bdde06772 100644 --- a/src/Makefile.qt.include +++ b/src/Makefile.qt.include @@ -438,7 +438,7 @@ $(QT_QRC_CPP): $(QT_QRC) $(QT_FORMS_H) $(RES_ICONS) $(RES_IMAGES) $(RES_MOVIES) $(AM_V_GEN) QT_SELECT=$(QT_SELECT) $(RCC) -name bitcoin $< | \ $(SED) -e '/^\*\*.*Created:/d' -e '/^\*\*.*by:/d' > $@ -CLEAN_QT = $(nodist_qt_libbitcoinqt_a_SOURCES) $(QT_QM) $(QT_FORMS_H) qt/*.gcda qt/*.gcno +CLEAN_QT = $(nodist_qt_libbitcoinqt_a_SOURCES) $(QT_QM) $(QT_FORMS_H) qt/*.gcda qt/*.gcno qt/temp_bitcoin_locale.qrc CLEANFILES += $(CLEAN_QT) diff --git a/src/Makefile.test.include b/src/Makefile.test.include index f037705aaf..06175be3fc 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -80,6 +80,7 @@ BITCOIN_TESTS =\ test/timedata_tests.cpp \ test/torcontrol_tests.cpp \ test/transaction_tests.cpp \ + test/txvalidation_tests.cpp \ test/txvalidationcache_tests.cpp \ test/versionbits_tests.cpp \ test/uint256_tests.cpp \ diff --git a/src/bitcoin-cli.cpp b/src/bitcoin-cli.cpp index 5e3b76a295..136981b709 100644 --- a/src/bitcoin-cli.cpp +++ b/src/bitcoin-cli.cpp @@ -213,6 +213,9 @@ public: /** Create a simulated `getinfo` request. */ UniValue PrepareRequest(const std::string& method, const std::vector<std::string>& args) override { + if (!args.empty()) { + throw std::runtime_error("-getinfo takes no arguments"); + } UniValue result(UniValue::VARR); result.push_back(JSONRPCRequestObj("getnetworkinfo", NullUniValue, ID_NETWORKINFO)); result.push_back(JSONRPCRequestObj("getblockchaininfo", NullUniValue, ID_BLOCKCHAININFO)); diff --git a/src/compat/byteswap.h b/src/compat/byteswap.h index 8930534721..a6df6ded7a 100644 --- a/src/compat/byteswap.h +++ b/src/compat/byteswap.h @@ -37,7 +37,7 @@ inline uint16_t bswap_16(uint16_t x) { return (x >> 8) | (x << 8); } -#endif // HAVE_DECL_BSWAP16 +#endif // HAVE_DECL_BSWAP16 == 0 #if HAVE_DECL_BSWAP_32 == 0 inline uint32_t bswap_32(uint32_t x) @@ -45,7 +45,7 @@ inline uint32_t bswap_32(uint32_t x) return (((x & 0xff000000U) >> 24) | ((x & 0x00ff0000U) >> 8) | ((x & 0x0000ff00U) << 8) | ((x & 0x000000ffU) << 24)); } -#endif // HAVE_DECL_BSWAP32 +#endif // HAVE_DECL_BSWAP32 == 0 #if HAVE_DECL_BSWAP_64 == 0 inline uint64_t bswap_64(uint64_t x) @@ -59,7 +59,7 @@ inline uint64_t bswap_64(uint64_t x) | ((x & 0x000000000000ff00ull) << 40) | ((x & 0x00000000000000ffull) << 56)); } -#endif // HAVE_DECL_BSWAP64 +#endif // HAVE_DECL_BSWAP64 == 0 #endif // defined(__APPLE__) diff --git a/src/init.cpp b/src/init.cpp index 439eaacfcc..3cc797a17c 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -1030,11 +1030,6 @@ bool AppInitParameterInteraction() fPruneMode = true; } - RegisterAllCoreRPCCommands(tableRPC); -#ifdef ENABLE_WALLET - RegisterWalletRPC(tableRPC); -#endif - nConnectTimeout = gArgs.GetArg("-timeout", DEFAULT_CONNECT_TIMEOUT); if (nConnectTimeout <= 0) nConnectTimeout = DEFAULT_CONNECT_TIMEOUT; @@ -1239,6 +1234,14 @@ bool AppInitMain(boost::thread_group& threadGroup, CScheduler& scheduler) GetMainSignals().RegisterBackgroundSignalScheduler(scheduler); GetMainSignals().RegisterWithMempoolSignals(mempool); + /* Register RPC commands regardless of -server setting so they will be + * available in the GUI RPC console even if external calls are disabled. + */ + RegisterAllCoreRPCCommands(tableRPC); +#ifdef ENABLE_WALLET + RegisterWalletRPC(tableRPC); +#endif + /* Start the RPC server already. It will be started in "warmup" mode * and not really process calls already (but it will signify connections * that the server is there and will be ready later). Warmup mode will diff --git a/src/qt/intro.cpp b/src/qt/intro.cpp index 9e4c765101..7f8a8394e6 100644 --- a/src/qt/intro.cpp +++ b/src/qt/intro.cpp @@ -214,7 +214,10 @@ bool Intro::pickDataDirectory() } dataDir = intro.getDataDirectory(); try { - TryCreateDirectories(GUIUtil::qstringToBoostPath(dataDir)); + if (TryCreateDirectories(GUIUtil::qstringToBoostPath(dataDir))) { + // If a new data directory has been created, make wallets subdirectory too + TryCreateDirectories(GUIUtil::qstringToBoostPath(dataDir) / "wallets"); + } break; } catch (const fs::filesystem_error&) { QMessageBox::critical(0, tr(PACKAGE_NAME), diff --git a/src/qt/rpcconsole.cpp b/src/qt/rpcconsole.cpp index 6687d62baa..54a6e837c1 100644 --- a/src/qt/rpcconsole.cpp +++ b/src/qt/rpcconsole.cpp @@ -392,11 +392,37 @@ void RPCExecutor::request(const QString &command) { std::string result; std::string executableCommand = command.toStdString() + "\n"; + + // Catch the console-only-help command before RPC call is executed and reply with help text as-if a RPC reply. + if(executableCommand == "help-console\n") + { + Q_EMIT reply(RPCConsole::CMD_REPLY, QString(("\n" + "This console accepts RPC commands using the standard syntax.\n" + " example: getblockhash 0\n\n" + + "This console can also accept RPC commands using parenthesized syntax.\n" + " example: getblockhash(0)\n\n" + + "Commands may be nested when specified with the parenthesized syntax.\n" + " example: getblock(getblockhash(0) 1)\n\n" + + "A space or a comma can be used to delimit arguments for either syntax.\n" + " example: getblockhash 0\n" + " getblockhash,0\n\n" + + "Named results can be queried with a non-quoted key string in brackets.\n" + " example: getblock(getblockhash(0) true)[tx]\n\n" + + "Results without keys can be queried using an integer in brackets.\n" + " example: getblock(getblockhash(0),true)[tx][0]\n\n"))); + return; + } if(!RPCConsole::RPCExecuteCommandLine(result, executableCommand)) { Q_EMIT reply(RPCConsole::CMD_ERROR, QString("Parse error: unbalanced ' or \"")); return; } + Q_EMIT reply(RPCConsole::CMD_REPLY, QString::fromStdString(result)); } catch (UniValue& objError) @@ -645,6 +671,7 @@ void RPCConsole::setClientModel(ClientModel *model) wordList << ("help " + commandList[i]).c_str(); } + wordList << "help-console"; wordList.sort(); autoCompleter = new QCompleter(wordList, this); autoCompleter->setModelSorting(QCompleter::CaseSensitivelySortedModel); @@ -750,10 +777,11 @@ void RPCConsole::clear(bool clearHistory) message(CMD_REPLY, (tr("Welcome to the %1 RPC console.").arg(tr(PACKAGE_NAME)) + "<br>" + tr("Use up and down arrows to navigate history, and %1 to clear screen.").arg("<b>"+clsKey+"</b>") + "<br>" + - tr("Type <b>help</b> for an overview of available commands.")) + - "<br><span class=\"secwarning\">" + + tr("Type %1 for an overview of available commands.").arg("<b>help</b>") + "<br>" + + tr("For more information on using this console type %1.").arg("<b>help-console</b>") + + "<br><span class=\"secwarning\"><br>" + tr("WARNING: Scammers have been active, telling users to type commands here, stealing their wallet contents. Do not use this console without fully understanding the ramifications of a command.") + - "</span>", + "</span>"), true); } diff --git a/src/qt/walletmodeltransaction.cpp b/src/qt/walletmodeltransaction.cpp index 3bf3e295ff..2188b39b92 100644 --- a/src/qt/walletmodeltransaction.cpp +++ b/src/qt/walletmodeltransaction.cpp @@ -10,7 +10,6 @@ WalletModelTransaction::WalletModelTransaction(const QList<SendCoinsRecipient> &_recipients) : recipients(_recipients), walletTransaction(0), - keyChange(0), fee(0) { walletTransaction = new CWalletTx(); @@ -18,7 +17,6 @@ WalletModelTransaction::WalletModelTransaction(const QList<SendCoinsRecipient> & WalletModelTransaction::~WalletModelTransaction() { - delete keyChange; delete walletTransaction; } @@ -91,10 +89,10 @@ CAmount WalletModelTransaction::getTotalTransactionAmount() const void WalletModelTransaction::newPossibleKeyChange(CWallet *wallet) { - keyChange = new CReserveKey(wallet); + keyChange.reset(new CReserveKey(wallet)); } CReserveKey *WalletModelTransaction::getPossibleKeyChange() { - return keyChange; + return keyChange.get(); } diff --git a/src/qt/walletmodeltransaction.h b/src/qt/walletmodeltransaction.h index d75875eefd..519cf9df99 100644 --- a/src/qt/walletmodeltransaction.h +++ b/src/qt/walletmodeltransaction.h @@ -40,7 +40,7 @@ public: private: QList<SendCoinsRecipient> recipients; CWalletTx *walletTransaction; - CReserveKey *keyChange; + std::unique_ptr<CReserveKey> keyChange; CAmount fee; }; diff --git a/src/rpc/rawtransaction.cpp b/src/rpc/rawtransaction.cpp index e65959bf0e..3aff1e9fbf 100644 --- a/src/rpc/rawtransaction.cpp +++ b/src/rpc/rawtransaction.cpp @@ -970,6 +970,10 @@ UniValue sendrawtransaction(const JSONRPCRequest& request) } } else if (fHaveChain) { throw JSONRPCError(RPC_TRANSACTION_ALREADY_IN_CHAIN, "transaction already in block chain"); + } else { + // Make sure we don't block forever if re-sending + // a transaction already in mempool. + promise.set_value(); } } // cs_main diff --git a/src/test/txvalidation_tests.cpp b/src/test/txvalidation_tests.cpp new file mode 100644 index 0000000000..2d1eb7b772 --- /dev/null +++ b/src/test/txvalidation_tests.cpp @@ -0,0 +1,61 @@ +// Copyright (c) 2017 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <validation.h> +#include <txmempool.h> +#include <amount.h> +#include <consensus/validation.h> +#include <primitives/transaction.h> +#include <script/script.h> +#include <test/test_bitcoin.h> + +#include <boost/test/unit_test.hpp> + + +BOOST_AUTO_TEST_SUITE(txvalidation_tests) + +/** + * Ensure that the mempool won't accept coinbase transactions. + */ +BOOST_FIXTURE_TEST_CASE(tx_mempool_reject_coinbase, TestChain100Setup) +{ + CScript scriptPubKey = CScript() << ToByteVector(coinbaseKey.GetPubKey()) << OP_CHECKSIG; + CMutableTransaction coinbaseTx; + + coinbaseTx.nVersion = 1; + coinbaseTx.vin.resize(1); + coinbaseTx.vout.resize(1); + coinbaseTx.vin[0].scriptSig = CScript() << OP_11 << OP_EQUAL; + coinbaseTx.vout[0].nValue = 1 * CENT; + coinbaseTx.vout[0].scriptPubKey = scriptPubKey; + + assert(CTransaction(coinbaseTx).IsCoinBase()); + + CValidationState state; + + LOCK(cs_main); + + unsigned int initialPoolSize = mempool.size(); + + BOOST_CHECK_EQUAL( + false, + AcceptToMemoryPool(mempool, state, MakeTransactionRef(coinbaseTx), + nullptr /* pfMissingInputs */, + nullptr /* plTxnReplaced */, + true /* bypass_limits */, + 0 /* nAbsurdFee */)); + + // Check that the transaction hasn't been added to mempool. + BOOST_CHECK_EQUAL(mempool.size(), initialPoolSize); + + // Check that the validation state reflects the unsuccessful attempt. + BOOST_CHECK(state.IsInvalid()); + BOOST_CHECK_EQUAL(state.GetRejectReason(), "coinbase"); + + int nDoS; + BOOST_CHECK_EQUAL(state.IsInvalid(nDoS), true); + BOOST_CHECK_EQUAL(nDoS, 100); +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/src/util.cpp b/src/util.cpp index b87dd091be..d58f39e969 100644 --- a/src/util.cpp +++ b/src/util.cpp @@ -574,7 +574,10 @@ const fs::path &GetDataDir(bool fNetSpecific) if (fNetSpecific) path /= BaseParams().DataDir(); - fs::create_directories(path); + if (fs::create_directories(path)) { + // This is the first run, create wallets subdirectory too + fs::create_directories(path / "wallets"); + } return path; } diff --git a/src/wallet/db.cpp b/src/wallet/db.cpp index c6fd95f250..79ff27279c 100644 --- a/src/wallet/db.cpp +++ b/src/wallet/db.cpp @@ -11,6 +11,7 @@ #include <protocol.h> #include <util.h> #include <utilstrencodings.h> +#include <wallet/walletutil.h> #include <stdint.h> @@ -257,7 +258,7 @@ bool CDB::Recover(const std::string& filename, void *callbackDataIn, bool (*reco return fSuccess; } -bool CDB::VerifyEnvironment(const std::string& walletFile, const fs::path& dataDir, std::string& errorStr) +bool CDB::VerifyEnvironment(const std::string& walletFile, const fs::path& walletDir, std::string& errorStr) { LogPrintf("Using BerkeleyDB version %s\n", DbEnv::version(0, 0, 0)); LogPrintf("Using wallet %s\n", walletFile); @@ -265,15 +266,15 @@ bool CDB::VerifyEnvironment(const std::string& walletFile, const fs::path& dataD // Wallet file must be a plain filename without a directory if (walletFile != fs::basename(walletFile) + fs::extension(walletFile)) { - errorStr = strprintf(_("Wallet %s resides outside data directory %s"), walletFile, dataDir.string()); + errorStr = strprintf(_("Wallet %s resides outside wallet directory %s"), walletFile, walletDir.string()); return false; } - if (!bitdb.Open(dataDir)) + if (!bitdb.Open(walletDir)) { // try moving the database env out of the way - fs::path pathDatabase = dataDir / "database"; - fs::path pathDatabaseBak = dataDir / strprintf("database.%d.bak", GetTime()); + fs::path pathDatabase = walletDir / "database"; + fs::path pathDatabaseBak = walletDir / strprintf("database.%d.bak", GetTime()); try { fs::rename(pathDatabase, pathDatabaseBak); LogPrintf("Moved old %s to %s. Retrying.\n", pathDatabase.string(), pathDatabaseBak.string()); @@ -282,18 +283,18 @@ bool CDB::VerifyEnvironment(const std::string& walletFile, const fs::path& dataD } // try again - if (!bitdb.Open(dataDir)) { + if (!bitdb.Open(walletDir)) { // if it still fails, it probably means we can't even create the database env - errorStr = strprintf(_("Error initializing wallet database environment %s!"), GetDataDir()); + errorStr = strprintf(_("Error initializing wallet database environment %s!"), walletDir); return false; } } return true; } -bool CDB::VerifyDatabaseFile(const std::string& walletFile, const fs::path& dataDir, std::string& warningStr, std::string& errorStr, CDBEnv::recoverFunc_type recoverFunc) +bool CDB::VerifyDatabaseFile(const std::string& walletFile, const fs::path& walletDir, std::string& warningStr, std::string& errorStr, CDBEnv::recoverFunc_type recoverFunc) { - if (fs::exists(dataDir / walletFile)) + if (fs::exists(walletDir / walletFile)) { std::string backup_filename; CDBEnv::VerifyResult r = bitdb.Verify(walletFile, recoverFunc, backup_filename); @@ -303,7 +304,7 @@ bool CDB::VerifyDatabaseFile(const std::string& walletFile, const fs::path& data " Original %s saved as %s in %s; if" " your balance or transactions are incorrect you should" " restore from a backup."), - walletFile, backup_filename, dataDir); + walletFile, backup_filename, walletDir); } if (r == CDBEnv::RECOVER_FAIL) { @@ -407,7 +408,7 @@ CDB::CDB(CWalletDBWrapper& dbw, const char* pszMode, bool fFlushOnCloseIn) : pdb { LOCK(env->cs_db); - if (!env->Open(GetDataDir())) + if (!env->Open(GetWalletDir())) throw std::runtime_error("CDB: Failed to open database environment."); pdb = env->mapDb[strFilename]; @@ -695,7 +696,7 @@ bool CWalletDBWrapper::Backup(const std::string& strDest) env->mapFileUseCount.erase(strFile); // Copy wallet file - fs::path pathSrc = GetDataDir() / strFile; + fs::path pathSrc = GetWalletDir() / strFile; fs::path pathDest(strDest); if (fs::is_directory(pathDest)) pathDest /= strFile; diff --git a/src/wallet/db.h b/src/wallet/db.h index 5c5948aa29..ed2ee65cac 100644 --- a/src/wallet/db.h +++ b/src/wallet/db.h @@ -167,9 +167,9 @@ public: ideal to be called periodically */ static bool PeriodicFlush(CWalletDBWrapper& dbw); /* verifies the database environment */ - static bool VerifyEnvironment(const std::string& walletFile, const fs::path& dataDir, std::string& errorStr); + static bool VerifyEnvironment(const std::string& walletFile, const fs::path& walletDir, std::string& errorStr); /* verifies the database file */ - static bool VerifyDatabaseFile(const std::string& walletFile, const fs::path& dataDir, std::string& warningStr, std::string& errorStr, CDBEnv::recoverFunc_type recoverFunc); + static bool VerifyDatabaseFile(const std::string& walletFile, const fs::path& walletDir, std::string& warningStr, std::string& errorStr, CDBEnv::recoverFunc_type recoverFunc); public: template <typename K, typename T> diff --git a/src/wallet/init.cpp b/src/wallet/init.cpp index ca6dcccf78..67c46df87d 100644 --- a/src/wallet/init.cpp +++ b/src/wallet/init.cpp @@ -9,8 +9,9 @@ #include <util.h> #include <utilmoneystr.h> #include <validation.h> -#include <wallet/wallet.h> #include <wallet/rpcwallet.h> +#include <wallet/wallet.h> +#include <wallet/walletutil.h> std::string GetWalletHelpString(bool showDebug) { @@ -34,6 +35,7 @@ std::string GetWalletHelpString(bool showDebug) strUsage += HelpMessageOpt("-upgradewallet", _("Upgrade wallet to latest format on startup")); strUsage += HelpMessageOpt("-wallet=<file>", _("Specify wallet file (within data directory)") + " " + strprintf(_("(default: %s)"), DEFAULT_WALLET_DAT)); strUsage += HelpMessageOpt("-walletbroadcast", _("Make the wallet broadcast transactions") + " " + strprintf(_("(default: %u)"), DEFAULT_WALLETBROADCAST)); + strUsage += HelpMessageOpt("-walletdir=<dir>", _("Specify directory to hold wallets (default: <datadir>/wallets if it exists, otherwise <datadir>)")); strUsage += HelpMessageOpt("-walletnotify=<cmd>", _("Execute command when a wallet transaction changes (%s in cmd is replaced by TxID)")); strUsage += HelpMessageOpt("-zapwallettxes=<mode>", _("Delete all wallet transactions and only recover those parts of the blockchain through -rescan on startup") + " " + _("(1 = keep tx meta data e.g. account owner and payment request information, 2 = drop tx meta data)")); @@ -191,6 +193,12 @@ bool VerifyWallets() return true; } + if (gArgs.IsArgSet("-walletdir") && !fs::is_directory(GetWalletDir())) { + return InitError(strprintf(_("Error: Specified wallet directory \"%s\" does not exist."), gArgs.GetArg("-walletdir", "").c_str())); + } + + LogPrintf("Using wallet directory %s\n", GetWalletDir().string()); + uiInterface.InitMessage(_("Verifying wallet(s)...")); // Keep track of each wallet absolute path to detect duplicates. @@ -205,7 +213,7 @@ bool VerifyWallets() return InitError(strprintf(_("Error loading wallet %s. Invalid characters in -wallet filename."), walletFile)); } - fs::path wallet_path = fs::absolute(walletFile, GetDataDir()); + fs::path wallet_path = fs::absolute(walletFile, GetWalletDir()); if (fs::exists(wallet_path) && (!fs::is_regular_file(wallet_path) || fs::is_symlink(wallet_path))) { return InitError(strprintf(_("Error loading wallet %s. -wallet filename must be a regular file."), walletFile)); @@ -216,7 +224,7 @@ bool VerifyWallets() } std::string strError; - if (!CWalletDB::VerifyEnvironment(walletFile, GetDataDir().string(), strError)) { + if (!CWalletDB::VerifyEnvironment(walletFile, GetWalletDir().string(), strError)) { return InitError(strError); } @@ -230,7 +238,7 @@ bool VerifyWallets() } std::string strWarning; - bool dbV = CWalletDB::VerifyDatabaseFile(walletFile, GetDataDir().string(), strWarning, strError); + bool dbV = CWalletDB::VerifyDatabaseFile(walletFile, GetWalletDir().string(), strWarning, strError); if (!strWarning.empty()) { InitWarning(strWarning); } diff --git a/src/wallet/rpcwallet.cpp b/src/wallet/rpcwallet.cpp index 23559eb350..9a7861f978 100644 --- a/src/wallet/rpcwallet.cpp +++ b/src/wallet/rpcwallet.cpp @@ -26,6 +26,7 @@ #include <wallet/feebumper.h> #include <wallet/wallet.h> #include <wallet/walletdb.h> +#include <wallet/walletutil.h> #include <init.h> // For StartShutdown diff --git a/src/wallet/test/accounting_tests.cpp b/src/wallet/test/accounting_tests.cpp index a9cdc89e6f..b95bb14335 100644 --- a/src/wallet/test/accounting_tests.cpp +++ b/src/wallet/test/accounting_tests.cpp @@ -10,18 +10,16 @@ #include <boost/test/unit_test.hpp> -extern CWallet* pwalletMain; - BOOST_FIXTURE_TEST_SUITE(accounting_tests, WalletTestingSetup) static void -GetResults(std::map<CAmount, CAccountingEntry>& results) +GetResults(CWallet *wallet, std::map<CAmount, CAccountingEntry>& results) { std::list<CAccountingEntry> aes; results.clear(); - BOOST_CHECK(pwalletMain->ReorderTransactions() == DB_LOAD_OK); - pwalletMain->ListAccountCreditDebit("", aes); + BOOST_CHECK(wallet->ReorderTransactions() == DB_LOAD_OK); + wallet->ListAccountCreditDebit("", aes); for (CAccountingEntry& ae : aes) { results[ae.nOrderPos] = ae; @@ -54,7 +52,7 @@ BOOST_AUTO_TEST_CASE(acc_orderupgrade) ae.strOtherAccount = "c"; pwalletMain->AddAccountingEntry(ae); - GetResults(results); + GetResults(pwalletMain.get(), results); BOOST_CHECK(pwalletMain->nOrderPosNext == 3); BOOST_CHECK(2 == results.size()); @@ -70,7 +68,7 @@ BOOST_AUTO_TEST_CASE(acc_orderupgrade) ae.nOrderPos = pwalletMain->IncOrderPosNext(); pwalletMain->AddAccountingEntry(ae); - GetResults(results); + GetResults(pwalletMain.get(), results); BOOST_CHECK(results.size() == 3); BOOST_CHECK(pwalletMain->nOrderPosNext == 4); @@ -102,7 +100,7 @@ BOOST_AUTO_TEST_CASE(acc_orderupgrade) vpwtx[2]->nTimeReceived = (unsigned int)1333333329; vpwtx[2]->nOrderPos = -1; - GetResults(results); + GetResults(pwalletMain.get(), results); BOOST_CHECK(results.size() == 3); BOOST_CHECK(pwalletMain->nOrderPosNext == 6); @@ -120,7 +118,7 @@ BOOST_AUTO_TEST_CASE(acc_orderupgrade) ae.nOrderPos = -1; pwalletMain->AddAccountingEntry(ae); - GetResults(results); + GetResults(pwalletMain.get(), results); BOOST_CHECK(results.size() == 4); BOOST_CHECK(pwalletMain->nOrderPosNext == 7); diff --git a/src/wallet/test/wallet_test_fixture.cpp b/src/wallet/test/wallet_test_fixture.cpp index 995ec640b1..3ee83d2d7c 100644 --- a/src/wallet/test/wallet_test_fixture.cpp +++ b/src/wallet/test/wallet_test_fixture.cpp @@ -6,9 +6,6 @@ #include <rpc/server.h> #include <wallet/db.h> -#include <wallet/wallet.h> - -std::unique_ptr<CWallet> pwalletMain; WalletTestingSetup::WalletTestingSetup(const std::string& chainName): TestingSetup(chainName) @@ -27,7 +24,6 @@ WalletTestingSetup::WalletTestingSetup(const std::string& chainName): WalletTestingSetup::~WalletTestingSetup() { UnregisterValidationInterface(pwalletMain.get()); - pwalletMain.reset(); bitdb.Flush(true); bitdb.Reset(); diff --git a/src/wallet/test/wallet_test_fixture.h b/src/wallet/test/wallet_test_fixture.h index a341bc698a..292d654438 100644 --- a/src/wallet/test/wallet_test_fixture.h +++ b/src/wallet/test/wallet_test_fixture.h @@ -7,11 +7,15 @@ #include <test/test_bitcoin.h> +#include <wallet/wallet.h> + /** Testing setup and teardown for wallet. */ struct WalletTestingSetup: public TestingSetup { explicit WalletTestingSetup(const std::string& chainName = CBaseChainParams::MAIN); ~WalletTestingSetup(); + + std::unique_ptr<CWallet> pwalletMain; }; #endif diff --git a/src/wallet/test/wallet_tests.cpp b/src/wallet/test/wallet_tests.cpp index 7ae1e74dbe..80e31a1ce0 100644 --- a/src/wallet/test/wallet_tests.cpp +++ b/src/wallet/test/wallet_tests.cpp @@ -19,8 +19,6 @@ #include <boost/test/unit_test.hpp> #include <univalue.h> -extern CWallet* pwalletMain; - extern UniValue importmulti(const JSONRPCRequest& request); extern UniValue dumpwallet(const JSONRPCRequest& request); extern UniValue importwallet(const JSONRPCRequest& request); diff --git a/src/wallet/walletdb.cpp b/src/wallet/walletdb.cpp index 66b8dea4ad..5116d6419e 100644 --- a/src/wallet/walletdb.cpp +++ b/src/wallet/walletdb.cpp @@ -814,14 +814,14 @@ bool CWalletDB::RecoverKeysOnlyFilter(void *callbackData, CDataStream ssKey, CDa return true; } -bool CWalletDB::VerifyEnvironment(const std::string& walletFile, const fs::path& dataDir, std::string& errorStr) +bool CWalletDB::VerifyEnvironment(const std::string& walletFile, const fs::path& walletDir, std::string& errorStr) { - return CDB::VerifyEnvironment(walletFile, dataDir, errorStr); + return CDB::VerifyEnvironment(walletFile, walletDir, errorStr); } -bool CWalletDB::VerifyDatabaseFile(const std::string& walletFile, const fs::path& dataDir, std::string& warningStr, std::string& errorStr) +bool CWalletDB::VerifyDatabaseFile(const std::string& walletFile, const fs::path& walletDir, std::string& warningStr, std::string& errorStr) { - return CDB::VerifyDatabaseFile(walletFile, dataDir, warningStr, errorStr, CWalletDB::Recover); + return CDB::VerifyDatabaseFile(walletFile, walletDir, warningStr, errorStr, CWalletDB::Recover); } bool CWalletDB::WriteDestData(const std::string &address, const std::string &key, const std::string &value) diff --git a/src/wallet/walletdb.h b/src/wallet/walletdb.h index 0a85fc3bc0..e815bcfeda 100644 --- a/src/wallet/walletdb.h +++ b/src/wallet/walletdb.h @@ -226,9 +226,9 @@ public: /* Function to determine if a certain KV/key-type is a key (cryptographical key) type */ static bool IsKeyType(const std::string& strType); /* verifies the database environment */ - static bool VerifyEnvironment(const std::string& walletFile, const fs::path& dataDir, std::string& errorStr); + static bool VerifyEnvironment(const std::string& walletFile, const fs::path& walletDir, std::string& errorStr); /* verifies the database file */ - static bool VerifyDatabaseFile(const std::string& walletFile, const fs::path& dataDir, std::string& warningStr, std::string& errorStr); + static bool VerifyDatabaseFile(const std::string& walletFile, const fs::path& walletDir, std::string& warningStr, std::string& errorStr); //! write the hdchain model (external chain child index counter) bool WriteHDChain(const CHDChain& chain); diff --git a/src/wallet/walletutil.cpp b/src/wallet/walletutil.cpp new file mode 100644 index 0000000000..fbb5215a51 --- /dev/null +++ b/src/wallet/walletutil.cpp @@ -0,0 +1,27 @@ +// Copyright (c) 2017 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include "wallet/walletutil.h" + +fs::path GetWalletDir() +{ + fs::path path; + + if (gArgs.IsArgSet("-walletdir")) { + path = fs::system_complete(gArgs.GetArg("-walletdir", "")); + if (!fs::is_directory(path)) { + // If the path specified doesn't exist, we return the deliberately + // invalid empty string. + path = ""; + } + } else { + path = GetDataDir(); + // If a wallets directory exists, use that, otherwise default to GetDataDir + if (fs::is_directory(path / "wallets")) { + path /= "wallets"; + } + } + + return path; +} diff --git a/src/wallet/walletutil.h b/src/wallet/walletutil.h new file mode 100644 index 0000000000..a94f286a44 --- /dev/null +++ b/src/wallet/walletutil.h @@ -0,0 +1,13 @@ +// Copyright (c) 2017 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_WALLET_UTIL_H +#define BITCOIN_WALLET_UTIL_H + +#include "util.h" + +//! Get the path of the wallet directory. +fs::path GetWalletDir(); + +#endif // BITCOIN_WALLET_UTIL_H diff --git a/test/functional/bitcoin_cli.py b/test/functional/bitcoin_cli.py index 996cbb8a12..d1cd3b3620 100755 --- a/test/functional/bitcoin_cli.py +++ b/test/functional/bitcoin_cli.py @@ -35,8 +35,11 @@ class TestBitcoinCli(BitcoinTestFramework): assert_equal(["foo", "bar"], self.nodes[0].cli('-rpcuser=%s' % user, '-stdin', '-stdinrpcpass', input=password + "\nfoo\nbar").echo()) assert_raises_process_error(1, "incorrect rpcuser or rpcpassword", self.nodes[0].cli('-rpcuser=%s' % user, '-stdin', '-stdinrpcpass', input="foo").echo) + self.log.info("Make sure that -getinfo with arguments fails") + assert_raises_process_error(1, "-getinfo takes no arguments", self.nodes[0].cli('-getinfo').help) + self.log.info("Compare responses from `bitcoin-cli -getinfo` and the RPCs data is retrieved from.") - cli_get_info = self.nodes[0].cli('-getinfo').help() + cli_get_info = self.nodes[0].cli().send_cli('-getinfo') wallet_info = self.nodes[0].getwalletinfo() network_info = self.nodes[0].getnetworkinfo() blockchain_info = self.nodes[0].getblockchaininfo() diff --git a/test/functional/keypool-topup.py b/test/functional/keypool-topup.py index 160a0f7ae5..e7af3c3987 100755 --- a/test/functional/keypool-topup.py +++ b/test/functional/keypool-topup.py @@ -33,7 +33,7 @@ class KeypoolRestoreTest(BitcoinTestFramework): self.stop_node(1) - shutil.copyfile(self.tmpdir + "/node1/regtest/wallet.dat", self.tmpdir + "/wallet.bak") + shutil.copyfile(self.tmpdir + "/node1/regtest/wallets/wallet.dat", self.tmpdir + "/wallet.bak") self.start_node(1, self.extra_args[1]) connect_nodes_bi(self.nodes, 0, 1) @@ -56,7 +56,7 @@ class KeypoolRestoreTest(BitcoinTestFramework): self.stop_node(1) - shutil.copyfile(self.tmpdir + "/wallet.bak", self.tmpdir + "/node1/regtest/wallet.dat") + shutil.copyfile(self.tmpdir + "/wallet.bak", self.tmpdir + "/node1/regtest/wallets/wallet.dat") self.log.info("Verify keypool is restored and balance is correct") diff --git a/test/functional/minchainwork.py b/test/functional/minchainwork.py index 35cd7ad141..90a3de0e0d 100755 --- a/test/functional/minchainwork.py +++ b/test/functional/minchainwork.py @@ -18,7 +18,7 @@ only succeeds past a given node once its nMinimumChainWork has been exceeded. import time from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import sync_blocks, connect_nodes, assert_equal +from test_framework.util import connect_nodes, assert_equal # 2 hashes required per regtest block (with no difficulty adjustment) REGTEST_WORK_PER_BLOCK = 2 diff --git a/test/functional/multiwallet.py b/test/functional/multiwallet.py index 7a0fbce477..06409b6f31 100755 --- a/test/functional/multiwallet.py +++ b/test/functional/multiwallet.py @@ -16,10 +16,10 @@ class MultiWalletTest(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = True self.num_nodes = 1 - self.extra_args = [['-wallet=w1', '-wallet=w2', '-wallet=w3']] + self.extra_args = [['-wallet=w1', '-wallet=w2', '-wallet=w3', '-wallet=w']] def run_test(self): - assert_equal(set(self.nodes[0].listwallets()), {"w1", "w2", "w3"}) + assert_equal(set(self.nodes[0].listwallets()), {"w1", "w2", "w3", "w"}) self.stop_node(0) @@ -27,23 +27,46 @@ class MultiWalletTest(BitcoinTestFramework): self.assert_start_raises_init_error(0, ['-wallet=w1', '-wallet=w1'], 'Error loading wallet w1. Duplicate -wallet filename specified.') # should not initialize if wallet file is a directory - os.mkdir(os.path.join(self.options.tmpdir, 'node0', 'regtest', 'w11')) + wallet_dir = os.path.join(self.options.tmpdir, 'node0', 'regtest', 'wallets') + os.mkdir(os.path.join(wallet_dir, 'w11')) self.assert_start_raises_init_error(0, ['-wallet=w11'], 'Error loading wallet w11. -wallet filename must be a regular file.') # should not initialize if one wallet is a copy of another - shutil.copyfile(os.path.join(self.options.tmpdir, 'node0', 'regtest', 'w2'), - os.path.join(self.options.tmpdir, 'node0', 'regtest', 'w22')) + shutil.copyfile(os.path.join(wallet_dir, 'w2'), os.path.join(wallet_dir, 'w22')) self.assert_start_raises_init_error(0, ['-wallet=w2', '-wallet=w22'], 'duplicates fileid') # should not initialize if wallet file is a symlink - os.symlink(os.path.join(self.options.tmpdir, 'node0', 'regtest', 'w1'), os.path.join(self.options.tmpdir, 'node0', 'regtest', 'w12')) + os.symlink(os.path.join(wallet_dir, 'w1'), os.path.join(wallet_dir, 'w12')) self.assert_start_raises_init_error(0, ['-wallet=w12'], 'Error loading wallet w12. -wallet filename must be a regular file.') + # should not initialize if the specified walletdir does not exist + self.assert_start_raises_init_error(0, ['-walletdir=bad'], 'Error: Specified wallet directory "bad" does not exist.') + + # if wallets/ doesn't exist, datadir should be the default wallet dir + wallet_dir2 = os.path.join(self.options.tmpdir, 'node0', 'regtest', 'walletdir') + os.rename(wallet_dir, wallet_dir2) + self.start_node(0, ['-wallet=w4', '-wallet=w5']) + assert_equal(set(self.nodes[0].listwallets()), {"w4", "w5"}) + w5 = self.nodes[0].get_wallet_rpc("w5") + w5.generate(1) + self.stop_node(0) + + # now if wallets/ exists again, but the rootdir is specified as the walletdir, w4 and w5 should still be loaded + os.rename(wallet_dir2, wallet_dir) + self.start_node(0, ['-wallet=w4', '-wallet=w5', '-walletdir=' + os.path.join(self.options.tmpdir, 'node0', 'regtest')]) + assert_equal(set(self.nodes[0].listwallets()), {"w4", "w5"}) + w5 = self.nodes[0].get_wallet_rpc("w5") + w5_info = w5.getwalletinfo() + assert_equal(w5_info['immature_balance'], 50) + + self.stop_node(0) + self.start_node(0, self.extra_args[0]) w1 = self.nodes[0].get_wallet_rpc("w1") w2 = self.nodes[0].get_wallet_rpc("w2") w3 = self.nodes[0].get_wallet_rpc("w3") + w4 = self.nodes[0].get_wallet_rpc("w") wallet_bad = self.nodes[0].get_wallet_rpc("bad") w1.generate(1) @@ -69,18 +92,22 @@ class MultiWalletTest(BitcoinTestFramework): w3_name = w3.getwalletinfo()['walletname'] assert_equal(w3_name, "w3") - assert_equal({"w1", "w2", "w3"}, {w1_name, w2_name, w3_name}) + w4_name = w4.getwalletinfo()['walletname'] + assert_equal(w4_name, "w") w1.generate(101) assert_equal(w1.getbalance(), 100) assert_equal(w2.getbalance(), 0) assert_equal(w3.getbalance(), 0) + assert_equal(w4.getbalance(), 0) w1.sendtoaddress(w2.getnewaddress(), 1) w1.sendtoaddress(w3.getnewaddress(), 2) + w1.sendtoaddress(w4.getnewaddress(), 3) w1.generate(1) assert_equal(w2.getbalance(), 1) assert_equal(w3.getbalance(), 2) + assert_equal(w4.getbalance(), 3) batch = w1.batch([w1.getblockchaininfo.get_request(), w1.getwalletinfo.get_request()]) assert_equal(batch[0]["result"]["chain"], "regtest") diff --git a/test/functional/p2p-fingerprint.py b/test/functional/p2p-fingerprint.py index 4b6446fc5b..a8ce68374c 100755 --- a/test/functional/p2p-fingerprint.py +++ b/test/functional/p2p-fingerprint.py @@ -24,7 +24,6 @@ from test_framework.mininode import ( from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, - p2p_port, ) class P2PFingerprintTest(BitcoinTestFramework): diff --git a/test/functional/sendheaders.py b/test/functional/sendheaders.py index 55bb80ea00..68c0d95b4f 100755 --- a/test/functional/sendheaders.py +++ b/test/functional/sendheaders.py @@ -4,11 +4,13 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test behavior of headers messages to announce blocks. -Setup: +Setup: -- Two nodes, two p2p connections to node0. One p2p connection should only ever - receive inv's (omitted from testing description below, this is our control). - Second node is used for creating reorgs. +- Two nodes: + - node0 is the node-under-test. We create two p2p connections to it. The + first p2p connection is a control and should only ever receive inv's. The + second p2p connection tests the headers sending logic. + - node1 is used to create reorgs. test_null_locators ================== @@ -83,35 +85,45 @@ d. Announce 49 headers that don't connect. e. Announce one more that doesn't connect. Expect: disconnect. """ - -from test_framework.mininode import * -from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import * from test_framework.blocktools import create_block, create_coinbase +from test_framework.mininode import ( + CBlockHeader, + CInv, + NODE_WITNESS, + NetworkThread, + NodeConnCB, + mininode_lock, + msg_block, + msg_getblocks, + msg_getdata, + msg_getheaders, + msg_headers, + msg_inv, + msg_sendheaders, +) +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import ( + assert_equal, + sync_blocks, + wait_until, +) +DIRECT_FETCH_RESPONSE_TIME = 0.05 -direct_fetch_response_time = 0.05 - -class TestNode(NodeConnCB): +class BaseNode(NodeConnCB): def __init__(self): super().__init__() self.block_announced = False self.last_blockhash_announced = None - def clear_last_announcement(self): - with mininode_lock: - self.block_announced = False - self.last_message.pop("inv", None) - self.last_message.pop("headers", None) - - # Request data for a list of block hashes - def get_data(self, block_hashes): + def send_get_data(self, block_hashes): + """Request data for a list of block hashes.""" msg = msg_getdata() for x in block_hashes: msg.inv.append(CInv(2, x)) self.connection.send_message(msg) - def get_headers(self, locator, hashstop): + def send_get_headers(self, locator, hashstop): msg = msg_getheaders() msg.locator.vHave = locator msg.hashstop = hashstop @@ -122,6 +134,27 @@ class TestNode(NodeConnCB): msg.inv = [CInv(2, blockhash)] self.connection.send_message(msg) + def send_header_for_blocks(self, new_blocks): + headers_message = msg_headers() + headers_message.headers = [CBlockHeader(b) for b in new_blocks] + self.send_message(headers_message) + + def send_getblocks(self, locator): + getblocks_message = msg_getblocks() + getblocks_message.locator.vHave = locator + self.send_message(getblocks_message) + + def wait_for_getdata(self, hash_list, timeout=60): + if hash_list == []: + return + + test_function = lambda: "getdata" in self.last_message and [x.hash for x in self.last_message["getdata"].inv] == hash_list + wait_until(test_function, timeout=timeout, lock=mininode_lock) + + def wait_for_block_announcement(self, block_hash, timeout=60): + test_function = lambda: self.last_blockhash_announced == block_hash + wait_until(test_function, timeout=timeout, lock=mininode_lock) + def on_inv(self, conn, message): self.block_announced = True self.last_blockhash_announced = message.inv[-1].hash @@ -132,97 +165,79 @@ class TestNode(NodeConnCB): message.headers[-1].calc_sha256() self.last_blockhash_announced = message.headers[-1].sha256 - # Test whether the last announcement we received had the - # right header or the right inv - # inv and headers should be lists of block hashes + def clear_last_announcement(self): + with mininode_lock: + self.block_announced = False + self.last_message.pop("inv", None) + self.last_message.pop("headers", None) + def check_last_announcement(self, headers=None, inv=None): - expect_headers = headers if headers != None else [] - expect_inv = inv if inv != None else [] + """Test whether the last announcement received had the right header or the right inv. + + inv and headers should be lists of block hashes.""" + test_function = lambda: self.block_announced wait_until(test_function, timeout=60, lock=mininode_lock) + with mininode_lock: self.block_announced = False - success = True compare_inv = [] if "inv" in self.last_message: compare_inv = [x.hash for x in self.last_message["inv"].inv] - if compare_inv != expect_inv: - success = False + if inv is not None: + assert_equal(compare_inv, inv) - hash_headers = [] + compare_headers = [] if "headers" in self.last_message: - # treat headers as a list of block hashes - hash_headers = [ x.sha256 for x in self.last_message["headers"].headers ] - if hash_headers != expect_headers: - success = False + compare_headers = [x.sha256 for x in self.last_message["headers"].headers] + if headers is not None: + assert_equal(compare_headers, headers) self.last_message.pop("inv", None) self.last_message.pop("headers", None) - return success - - def wait_for_getdata(self, hash_list, timeout=60): - if hash_list == []: - return - - test_function = lambda: "getdata" in self.last_message and [x.hash for x in self.last_message["getdata"].inv] == hash_list - wait_until(test_function, timeout=timeout, lock=mininode_lock) - return - - def wait_for_block_announcement(self, block_hash, timeout=60): - test_function = lambda: self.last_blockhash_announced == block_hash - wait_until(test_function, timeout=timeout, lock=mininode_lock) - return - - def send_header_for_blocks(self, new_blocks): - headers_message = msg_headers() - headers_message.headers = [ CBlockHeader(b) for b in new_blocks ] - self.send_message(headers_message) - - def send_getblocks(self, locator): - getblocks_message = msg_getblocks() - getblocks_message.locator.vHave = locator - self.send_message(getblocks_message) class SendHeadersTest(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = True self.num_nodes = 2 - # mine count blocks and return the new tip def mine_blocks(self, count): + """Mine count blocks and return the new tip.""" + # Clear out last block announcement from each p2p listener [x.clear_last_announcement() for x in self.nodes[0].p2ps] self.nodes[0].generate(count) return int(self.nodes[0].getbestblockhash(), 16) - # mine a reorg that invalidates length blocks (replacing them with - # length+1 blocks). - # Note: we clear the state of our p2p connections after the - # to-be-reorged-out blocks are mined, so that we don't break later tests. - # return the list of block hashes newly mined def mine_reorg(self, length): - self.nodes[0].generate(length) # make sure all invalidated blocks are node0's + """Mine a reorg that invalidates length blocks (replacing them with # length+1 blocks). + + Note: we clear the state of our p2p connections after the + to-be-reorged-out blocks are mined, so that we don't break later tests. + return the list of block hashes newly mined.""" + + self.nodes[0].generate(length) # make sure all invalidated blocks are node0's sync_blocks(self.nodes, wait=0.1) for x in self.nodes[0].p2ps: x.wait_for_block_announcement(int(self.nodes[0].getbestblockhash(), 16)) x.clear_last_announcement() tip_height = self.nodes[1].getblockcount() - hash_to_invalidate = self.nodes[1].getblockhash(tip_height-(length-1)) + hash_to_invalidate = self.nodes[1].getblockhash(tip_height - (length - 1)) self.nodes[1].invalidateblock(hash_to_invalidate) - all_hashes = self.nodes[1].generate(length+1) # Must be longer than the orig chain + all_hashes = self.nodes[1].generate(length + 1) # Must be longer than the orig chain sync_blocks(self.nodes, wait=0.1) return [int(x, 16) for x in all_hashes] def run_test(self): # Setup the p2p connections and start up the network thread. - inv_node = self.nodes[0].add_p2p_connection(TestNode()) - # Set nServices to 0 for test_node, so no block download will occur outside of - # direct fetching - test_node = self.nodes[0].add_p2p_connection(TestNode(), services=NODE_WITNESS) + inv_node = self.nodes[0].add_p2p_connection(BaseNode()) + # Make sure NODE_NETWORK is not set for test_node, so no block download + # will occur outside of direct fetching + test_node = self.nodes[0].add_p2p_connection(BaseNode(), services=NODE_WITNESS) - NetworkThread().start() # Start up network handling in another thread + NetworkThread().start() # Start up network handling in another thread # Test logic begins here inv_node.wait_for_verack() @@ -232,27 +247,32 @@ class SendHeadersTest(BitcoinTestFramework): inv_node.sync_with_ping() test_node.sync_with_ping() - self.test_null_locators(test_node) + self.test_null_locators(test_node, inv_node) self.test_nonnull_locators(test_node, inv_node) - def test_null_locators(self, test_node): + def test_null_locators(self, test_node, inv_node): tip = self.nodes[0].getblockheader(self.nodes[0].generate(1)[0]) tip_hash = int(tip["hash"], 16) + inv_node.check_last_announcement(inv=[tip_hash], headers=[]) + test_node.check_last_announcement(inv=[tip_hash], headers=[]) + self.log.info("Verify getheaders with null locator and valid hashstop returns headers.") test_node.clear_last_announcement() - test_node.get_headers(locator=[], hashstop=tip_hash) - assert_equal(test_node.check_last_announcement(headers=[tip_hash]), True) + test_node.send_get_headers(locator=[], hashstop=tip_hash) + test_node.check_last_announcement(headers=[tip_hash]) self.log.info("Verify getheaders with null locator and invalid hashstop does not return headers.") block = create_block(int(tip["hash"], 16), create_coinbase(tip["height"] + 1), tip["mediantime"] + 1) block.solve() test_node.send_header_for_blocks([block]) test_node.clear_last_announcement() - test_node.get_headers(locator=[], hashstop=int(block.hash, 16)) + test_node.send_get_headers(locator=[], hashstop=int(block.hash, 16)) test_node.sync_with_ping() assert_equal(test_node.block_announced, False) + inv_node.clear_last_announcement() test_node.send_message(msg_block(block)) + inv_node.check_last_announcement(inv=[int(block.hash, 16)], headers=[]) def test_nonnull_locators(self, test_node, inv_node): tip = int(self.nodes[0].getbestblockhash(), 16) @@ -263,30 +283,30 @@ class SendHeadersTest(BitcoinTestFramework): for i in range(4): old_tip = tip tip = self.mine_blocks(1) - assert_equal(inv_node.check_last_announcement(inv=[tip]), True) - assert_equal(test_node.check_last_announcement(inv=[tip]), True) + inv_node.check_last_announcement(inv=[tip], headers=[]) + test_node.check_last_announcement(inv=[tip], headers=[]) # Try a few different responses; none should affect next announcement if i == 0: # first request the block - test_node.get_data([tip]) + test_node.send_get_data([tip]) test_node.wait_for_block(tip) elif i == 1: # next try requesting header and block - test_node.get_headers(locator=[old_tip], hashstop=tip) - test_node.get_data([tip]) + test_node.send_get_headers(locator=[old_tip], hashstop=tip) + test_node.send_get_data([tip]) test_node.wait_for_block(tip) - test_node.clear_last_announcement() # since we requested headers... + test_node.clear_last_announcement() # since we requested headers... elif i == 2: # this time announce own block via headers height = self.nodes[0].getblockcount() last_time = self.nodes[0].getblock(self.nodes[0].getbestblockhash())['time'] block_time = last_time + 1 - new_block = create_block(tip, create_coinbase(height+1), block_time) + new_block = create_block(tip, create_coinbase(height + 1), block_time) new_block.solve() test_node.send_header_for_blocks([new_block]) test_node.wait_for_getdata([new_block.sha256]) test_node.send_message(msg_block(new_block)) - test_node.sync_with_ping() # make sure this block is processed + test_node.sync_with_ping() # make sure this block is processed inv_node.clear_last_announcement() test_node.clear_last_announcement() @@ -297,15 +317,15 @@ class SendHeadersTest(BitcoinTestFramework): # commence and keep working. test_node.send_message(msg_sendheaders()) prev_tip = int(self.nodes[0].getbestblockhash(), 16) - test_node.get_headers(locator=[prev_tip], hashstop=0) + test_node.send_get_headers(locator=[prev_tip], hashstop=0) test_node.sync_with_ping() # Now that we've synced headers, headers announcements should work tip = self.mine_blocks(1) - assert_equal(inv_node.check_last_announcement(inv=[tip]), True) - assert_equal(test_node.check_last_announcement(headers=[tip]), True) + inv_node.check_last_announcement(inv=[tip], headers=[]) + test_node.check_last_announcement(headers=[tip]) - height = self.nodes[0].getblockcount()+1 + height = self.nodes[0].getblockcount() + 1 block_time += 10 # Advance far enough ahead for i in range(10): # Mine i blocks, and alternate announcing either via @@ -314,7 +334,7 @@ class SendHeadersTest(BitcoinTestFramework): # with block header, even though the blocks are never requested for j in range(2): blocks = [] - for b in range(i+1): + for b in range(i + 1): blocks.append(create_block(tip, create_coinbase(height), block_time)) blocks[-1].solve() tip = blocks[-1].sha256 @@ -328,7 +348,7 @@ class SendHeadersTest(BitcoinTestFramework): test_node.send_header_for_blocks(blocks) # Test that duplicate inv's won't result in duplicate # getdata requests, or duplicate headers announcements - [ inv_node.send_block_inv(x.sha256) for x in blocks ] + [inv_node.send_block_inv(x.sha256) for x in blocks] test_node.wait_for_getdata([x.sha256 for x in blocks]) inv_node.sync_with_ping() else: @@ -339,7 +359,7 @@ class SendHeadersTest(BitcoinTestFramework): # getdata requests (the check is further down) inv_node.send_header_for_blocks(blocks) inv_node.sync_with_ping() - [ test_node.send_message(msg_block(x)) for x in blocks ] + [test_node.send_message(msg_block(x)) for x in blocks] test_node.sync_with_ping() inv_node.sync_with_ping() # This block should not be announced to the inv node (since it also @@ -347,8 +367,8 @@ class SendHeadersTest(BitcoinTestFramework): assert "inv" not in inv_node.last_message assert "headers" not in inv_node.last_message tip = self.mine_blocks(1) - assert_equal(inv_node.check_last_announcement(inv=[tip]), True) - assert_equal(test_node.check_last_announcement(headers=[tip]), True) + inv_node.check_last_announcement(inv=[tip], headers=[]) + test_node.check_last_announcement(headers=[tip]) height += 1 block_time += 1 @@ -362,16 +382,16 @@ class SendHeadersTest(BitcoinTestFramework): # First try mining a reorg that can propagate with header announcement new_block_hashes = self.mine_reorg(length=7) tip = new_block_hashes[-1] - assert_equal(inv_node.check_last_announcement(inv=[tip]), True) - assert_equal(test_node.check_last_announcement(headers=new_block_hashes), True) + inv_node.check_last_announcement(inv=[tip], headers=[]) + test_node.check_last_announcement(headers=new_block_hashes) - block_time += 8 + block_time += 8 # Mine a too-large reorg, which should be announced with a single inv new_block_hashes = self.mine_reorg(length=8) tip = new_block_hashes[-1] - assert_equal(inv_node.check_last_announcement(inv=[tip]), True) - assert_equal(test_node.check_last_announcement(inv=[tip]), True) + inv_node.check_last_announcement(inv=[tip], headers=[]) + test_node.check_last_announcement(inv=[tip], headers=[]) block_time += 9 @@ -379,42 +399,42 @@ class SendHeadersTest(BitcoinTestFramework): fork_point = int(fork_point, 16) # Use getblocks/getdata - test_node.send_getblocks(locator = [fork_point]) - assert_equal(test_node.check_last_announcement(inv=new_block_hashes), True) - test_node.get_data(new_block_hashes) + test_node.send_getblocks(locator=[fork_point]) + test_node.check_last_announcement(inv=new_block_hashes, headers=[]) + test_node.send_get_data(new_block_hashes) test_node.wait_for_block(new_block_hashes[-1]) for i in range(3): # Mine another block, still should get only an inv tip = self.mine_blocks(1) - assert_equal(inv_node.check_last_announcement(inv=[tip]), True) - assert_equal(test_node.check_last_announcement(inv=[tip]), True) + inv_node.check_last_announcement(inv=[tip], headers=[]) + test_node.check_last_announcement(inv=[tip], headers=[]) if i == 0: # Just get the data -- shouldn't cause headers announcements to resume - test_node.get_data([tip]) + test_node.send_get_data([tip]) test_node.wait_for_block(tip) elif i == 1: # Send a getheaders message that shouldn't trigger headers announcements # to resume (best header sent will be too old) - test_node.get_headers(locator=[fork_point], hashstop=new_block_hashes[1]) - test_node.get_data([tip]) + test_node.send_get_headers(locator=[fork_point], hashstop=new_block_hashes[1]) + test_node.send_get_data([tip]) test_node.wait_for_block(tip) elif i == 2: - test_node.get_data([tip]) + test_node.send_get_data([tip]) test_node.wait_for_block(tip) # This time, try sending either a getheaders to trigger resumption - # of headers announcements, or mine a new block and inv it, also + # of headers announcements, or mine a new block and inv it, also # triggering resumption of headers announcements. if j == 0: - test_node.get_headers(locator=[tip], hashstop=0) + test_node.send_get_headers(locator=[tip], hashstop=0) test_node.sync_with_ping() else: test_node.send_block_inv(tip) test_node.sync_with_ping() # New blocks should now be announced with header tip = self.mine_blocks(1) - assert_equal(inv_node.check_last_announcement(inv=[tip]), True) - assert_equal(test_node.check_last_announcement(headers=[tip]), True) + inv_node.check_last_announcement(inv=[tip], headers=[]) + test_node.check_last_announcement(headers=[tip]) self.log.info("Part 3: success!") @@ -434,7 +454,7 @@ class SendHeadersTest(BitcoinTestFramework): height += 1 inv_node.send_message(msg_block(blocks[-1])) - inv_node.sync_with_ping() # Make sure blocks are processed + inv_node.sync_with_ping() # Make sure blocks are processed test_node.last_message.pop("getdata", None) test_node.send_header_for_blocks(blocks) test_node.sync_with_ping() @@ -453,9 +473,9 @@ class SendHeadersTest(BitcoinTestFramework): test_node.send_header_for_blocks(blocks) test_node.sync_with_ping() - test_node.wait_for_getdata([x.sha256 for x in blocks], timeout=direct_fetch_response_time) + test_node.wait_for_getdata([x.sha256 for x in blocks], timeout=DIRECT_FETCH_RESPONSE_TIME) - [ test_node.send_message(msg_block(x)) for x in blocks ] + [test_node.send_message(msg_block(x)) for x in blocks] test_node.sync_with_ping() @@ -484,13 +504,13 @@ class SendHeadersTest(BitcoinTestFramework): # both blocks (same work as tip) test_node.send_header_for_blocks(blocks[1:2]) test_node.sync_with_ping() - test_node.wait_for_getdata([x.sha256 for x in blocks[0:2]], timeout=direct_fetch_response_time) + test_node.wait_for_getdata([x.sha256 for x in blocks[0:2]], timeout=DIRECT_FETCH_RESPONSE_TIME) # Announcing 16 more headers should trigger direct fetch for 14 more # blocks test_node.send_header_for_blocks(blocks[2:18]) test_node.sync_with_ping() - test_node.wait_for_getdata([x.sha256 for x in blocks[2:16]], timeout=direct_fetch_response_time) + test_node.wait_for_getdata([x.sha256 for x in blocks[2:16]], timeout=DIRECT_FETCH_RESPONSE_TIME) # Announcing 1 more header should not trigger any response test_node.last_message.pop("getdata", None) @@ -502,7 +522,7 @@ class SendHeadersTest(BitcoinTestFramework): self.log.info("Part 4: success!") # Now deliver all those blocks we announced. - [ test_node.send_message(msg_block(x)) for x in blocks ] + [test_node.send_message(msg_block(x)) for x in blocks] self.log.info("Part 5: Testing handling of unconnecting headers") # First we test that receipt of an unconnecting header doesn't prevent @@ -524,7 +544,7 @@ class SendHeadersTest(BitcoinTestFramework): test_node.wait_for_getheaders() test_node.send_header_for_blocks(blocks) test_node.wait_for_getdata([x.sha256 for x in blocks]) - [ test_node.send_message(msg_block(x)) for x in blocks ] + [test_node.send_message(msg_block(x)) for x in blocks] test_node.sync_with_ping() assert_equal(int(self.nodes[0].getbestblockhash(), 16), blocks[1].sha256) @@ -532,7 +552,7 @@ class SendHeadersTest(BitcoinTestFramework): # Now we test that if we repeatedly don't send connecting headers, we # don't go into an infinite loop trying to get them to connect. MAX_UNCONNECTING_HEADERS = 10 - for j in range(MAX_UNCONNECTING_HEADERS+1): + for j in range(MAX_UNCONNECTING_HEADERS + 1): blocks.append(create_block(tip, create_coinbase(height), block_time)) blocks[-1].solve() tip = blocks[-1].sha256 @@ -554,11 +574,11 @@ class SendHeadersTest(BitcoinTestFramework): # Now try to see how many unconnecting headers we can send # before we get disconnected. Should be 5*MAX_UNCONNECTING_HEADERS - for i in range(5*MAX_UNCONNECTING_HEADERS - 1): + for i in range(5 * MAX_UNCONNECTING_HEADERS - 1): # Send a header that doesn't connect, check that we get a getheaders. with mininode_lock: test_node.last_message.pop("getheaders", None) - test_node.send_header_for_blocks([blocks[i%len(blocks)]]) + test_node.send_header_for_blocks([blocks[i % len(blocks)]]) test_node.wait_for_getheaders() # Eventually this stops working. diff --git a/test/functional/test_framework/messages.py b/test/functional/test_framework/messages.py new file mode 100644 index 0000000000..eee24910cb --- /dev/null +++ b/test/functional/test_framework/messages.py @@ -0,0 +1,1304 @@ +#!/usr/bin/env python3 +# Copyright (c) 2010 ArtForz -- public domain half-a-node +# Copyright (c) 2012 Jeff Garzik +# Copyright (c) 2010-2017 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Bitcoin test framework primitive and message strcutures + +CBlock, CTransaction, CBlockHeader, CTxIn, CTxOut, etc....: + data structures that should map to corresponding structures in + bitcoin/primitives + +msg_block, msg_tx, msg_headers, etc.: + data structures that represent network messages + +ser_*, deser_*: functions that handle serialization/deserialization.""" +from codecs import encode +import copy +import hashlib +from io import BytesIO +import random +import socket +import struct +import time + +from test_framework.siphash import siphash256 +from test_framework.util import hex_str_to_bytes, bytes_to_hex_str, wait_until + +MIN_VERSION_SUPPORTED = 60001 +MY_VERSION = 70014 # past bip-31 for ping/pong +MY_SUBVERSION = b"/python-mininode-tester:0.0.3/" +MY_RELAY = 1 # from version 70001 onwards, fRelay should be appended to version messages (BIP37) + +MAX_INV_SZ = 50000 +MAX_BLOCK_BASE_SIZE = 1000000 + +COIN = 100000000 # 1 btc in satoshis + +NODE_NETWORK = (1 << 0) +# NODE_GETUTXO = (1 << 1) +# NODE_BLOOM = (1 << 2) +NODE_WITNESS = (1 << 3) +NODE_UNSUPPORTED_SERVICE_BIT_5 = (1 << 5) +NODE_UNSUPPORTED_SERVICE_BIT_7 = (1 << 7) + +# Serialization/deserialization tools +def sha256(s): + return hashlib.new('sha256', s).digest() + +def ripemd160(s): + return hashlib.new('ripemd160', s).digest() + +def hash256(s): + return sha256(sha256(s)) + +def ser_compact_size(l): + r = b"" + if l < 253: + r = struct.pack("B", l) + elif l < 0x10000: + r = struct.pack("<BH", 253, l) + elif l < 0x100000000: + r = struct.pack("<BI", 254, l) + else: + r = struct.pack("<BQ", 255, l) + return r + +def deser_compact_size(f): + nit = struct.unpack("<B", f.read(1))[0] + if nit == 253: + nit = struct.unpack("<H", f.read(2))[0] + elif nit == 254: + nit = struct.unpack("<I", f.read(4))[0] + elif nit == 255: + nit = struct.unpack("<Q", f.read(8))[0] + return nit + +def deser_string(f): + nit = deser_compact_size(f) + return f.read(nit) + +def ser_string(s): + return ser_compact_size(len(s)) + s + +def deser_uint256(f): + r = 0 + for i in range(8): + t = struct.unpack("<I", f.read(4))[0] + r += t << (i * 32) + return r + + +def ser_uint256(u): + rs = b"" + for i in range(8): + rs += struct.pack("<I", u & 0xFFFFFFFF) + u >>= 32 + return rs + + +def uint256_from_str(s): + r = 0 + t = struct.unpack("<IIIIIIII", s[:32]) + for i in range(8): + r += t[i] << (i * 32) + return r + + +def uint256_from_compact(c): + nbytes = (c >> 24) & 0xFF + v = (c & 0xFFFFFF) << (8 * (nbytes - 3)) + return v + + +def deser_vector(f, c): + nit = deser_compact_size(f) + r = [] + for i in range(nit): + t = c() + t.deserialize(f) + r.append(t) + return r + + +# ser_function_name: Allow for an alternate serialization function on the +# entries in the vector (we use this for serializing the vector of transactions +# for a witness block). +def ser_vector(l, ser_function_name=None): + r = ser_compact_size(len(l)) + for i in l: + if ser_function_name: + r += getattr(i, ser_function_name)() + else: + r += i.serialize() + return r + + +def deser_uint256_vector(f): + nit = deser_compact_size(f) + r = [] + for i in range(nit): + t = deser_uint256(f) + r.append(t) + return r + + +def ser_uint256_vector(l): + r = ser_compact_size(len(l)) + for i in l: + r += ser_uint256(i) + return r + + +def deser_string_vector(f): + nit = deser_compact_size(f) + r = [] + for i in range(nit): + t = deser_string(f) + r.append(t) + return r + + +def ser_string_vector(l): + r = ser_compact_size(len(l)) + for sv in l: + r += ser_string(sv) + return r + + +# Deserialize from a hex string representation (eg from RPC) +def FromHex(obj, hex_string): + obj.deserialize(BytesIO(hex_str_to_bytes(hex_string))) + return obj + +# Convert a binary-serializable object to hex (eg for submission via RPC) +def ToHex(obj): + return bytes_to_hex_str(obj.serialize()) + +# Objects that map to bitcoind objects, which can be serialized/deserialized + +class CAddress(): + def __init__(self): + self.nServices = 1 + self.pchReserved = b"\x00" * 10 + b"\xff" * 2 + self.ip = "0.0.0.0" + self.port = 0 + + def deserialize(self, f): + self.nServices = struct.unpack("<Q", f.read(8))[0] + self.pchReserved = f.read(12) + self.ip = socket.inet_ntoa(f.read(4)) + self.port = struct.unpack(">H", f.read(2))[0] + + def serialize(self): + r = b"" + r += struct.pack("<Q", self.nServices) + r += self.pchReserved + r += socket.inet_aton(self.ip) + r += struct.pack(">H", self.port) + return r + + def __repr__(self): + return "CAddress(nServices=%i ip=%s port=%i)" % (self.nServices, + self.ip, self.port) + +MSG_WITNESS_FLAG = 1<<30 + +class CInv(): + typemap = { + 0: "Error", + 1: "TX", + 2: "Block", + 1|MSG_WITNESS_FLAG: "WitnessTx", + 2|MSG_WITNESS_FLAG : "WitnessBlock", + 4: "CompactBlock" + } + + def __init__(self, t=0, h=0): + self.type = t + self.hash = h + + def deserialize(self, f): + self.type = struct.unpack("<i", f.read(4))[0] + self.hash = deser_uint256(f) + + def serialize(self): + r = b"" + r += struct.pack("<i", self.type) + r += ser_uint256(self.hash) + return r + + def __repr__(self): + return "CInv(type=%s hash=%064x)" \ + % (self.typemap[self.type], self.hash) + + +class CBlockLocator(): + def __init__(self): + self.nVersion = MY_VERSION + self.vHave = [] + + def deserialize(self, f): + self.nVersion = struct.unpack("<i", f.read(4))[0] + self.vHave = deser_uint256_vector(f) + + def serialize(self): + r = b"" + r += struct.pack("<i", self.nVersion) + r += ser_uint256_vector(self.vHave) + return r + + def __repr__(self): + return "CBlockLocator(nVersion=%i vHave=%s)" \ + % (self.nVersion, repr(self.vHave)) + + +class COutPoint(): + def __init__(self, hash=0, n=0): + self.hash = hash + self.n = n + + def deserialize(self, f): + self.hash = deser_uint256(f) + self.n = struct.unpack("<I", f.read(4))[0] + + def serialize(self): + r = b"" + r += ser_uint256(self.hash) + r += struct.pack("<I", self.n) + return r + + def __repr__(self): + return "COutPoint(hash=%064x n=%i)" % (self.hash, self.n) + + +class CTxIn(): + def __init__(self, outpoint=None, scriptSig=b"", nSequence=0): + if outpoint is None: + self.prevout = COutPoint() + else: + self.prevout = outpoint + self.scriptSig = scriptSig + self.nSequence = nSequence + + def deserialize(self, f): + self.prevout = COutPoint() + self.prevout.deserialize(f) + self.scriptSig = deser_string(f) + self.nSequence = struct.unpack("<I", f.read(4))[0] + + def serialize(self): + r = b"" + r += self.prevout.serialize() + r += ser_string(self.scriptSig) + r += struct.pack("<I", self.nSequence) + return r + + def __repr__(self): + return "CTxIn(prevout=%s scriptSig=%s nSequence=%i)" \ + % (repr(self.prevout), bytes_to_hex_str(self.scriptSig), + self.nSequence) + + +class CTxOut(): + def __init__(self, nValue=0, scriptPubKey=b""): + self.nValue = nValue + self.scriptPubKey = scriptPubKey + + def deserialize(self, f): + self.nValue = struct.unpack("<q", f.read(8))[0] + self.scriptPubKey = deser_string(f) + + def serialize(self): + r = b"" + r += struct.pack("<q", self.nValue) + r += ser_string(self.scriptPubKey) + return r + + def __repr__(self): + return "CTxOut(nValue=%i.%08i scriptPubKey=%s)" \ + % (self.nValue // COIN, self.nValue % COIN, + bytes_to_hex_str(self.scriptPubKey)) + + +class CScriptWitness(): + def __init__(self): + # stack is a vector of strings + self.stack = [] + + def __repr__(self): + return "CScriptWitness(%s)" % \ + (",".join([bytes_to_hex_str(x) for x in self.stack])) + + def is_null(self): + if self.stack: + return False + return True + + +class CTxInWitness(): + def __init__(self): + self.scriptWitness = CScriptWitness() + + def deserialize(self, f): + self.scriptWitness.stack = deser_string_vector(f) + + def serialize(self): + return ser_string_vector(self.scriptWitness.stack) + + def __repr__(self): + return repr(self.scriptWitness) + + def is_null(self): + return self.scriptWitness.is_null() + + +class CTxWitness(): + def __init__(self): + self.vtxinwit = [] + + def deserialize(self, f): + for i in range(len(self.vtxinwit)): + self.vtxinwit[i].deserialize(f) + + def serialize(self): + r = b"" + # This is different than the usual vector serialization -- + # we omit the length of the vector, which is required to be + # the same length as the transaction's vin vector. + for x in self.vtxinwit: + r += x.serialize() + return r + + def __repr__(self): + return "CTxWitness(%s)" % \ + (';'.join([repr(x) for x in self.vtxinwit])) + + def is_null(self): + for x in self.vtxinwit: + if not x.is_null(): + return False + return True + + +class CTransaction(): + def __init__(self, tx=None): + if tx is None: + self.nVersion = 1 + self.vin = [] + self.vout = [] + self.wit = CTxWitness() + self.nLockTime = 0 + self.sha256 = None + self.hash = None + else: + self.nVersion = tx.nVersion + self.vin = copy.deepcopy(tx.vin) + self.vout = copy.deepcopy(tx.vout) + self.nLockTime = tx.nLockTime + self.sha256 = tx.sha256 + self.hash = tx.hash + self.wit = copy.deepcopy(tx.wit) + + def deserialize(self, f): + self.nVersion = struct.unpack("<i", f.read(4))[0] + self.vin = deser_vector(f, CTxIn) + flags = 0 + if len(self.vin) == 0: + flags = struct.unpack("<B", f.read(1))[0] + # Not sure why flags can't be zero, but this + # matches the implementation in bitcoind + if (flags != 0): + self.vin = deser_vector(f, CTxIn) + self.vout = deser_vector(f, CTxOut) + else: + self.vout = deser_vector(f, CTxOut) + if flags != 0: + self.wit.vtxinwit = [CTxInWitness() for i in range(len(self.vin))] + self.wit.deserialize(f) + self.nLockTime = struct.unpack("<I", f.read(4))[0] + self.sha256 = None + self.hash = None + + def serialize_without_witness(self): + r = b"" + r += struct.pack("<i", self.nVersion) + r += ser_vector(self.vin) + r += ser_vector(self.vout) + r += struct.pack("<I", self.nLockTime) + return r + + # Only serialize with witness when explicitly called for + def serialize_with_witness(self): + flags = 0 + if not self.wit.is_null(): + flags |= 1 + r = b"" + r += struct.pack("<i", self.nVersion) + if flags: + dummy = [] + r += ser_vector(dummy) + r += struct.pack("<B", flags) + r += ser_vector(self.vin) + r += ser_vector(self.vout) + if flags & 1: + if (len(self.wit.vtxinwit) != len(self.vin)): + # vtxinwit must have the same length as vin + self.wit.vtxinwit = self.wit.vtxinwit[:len(self.vin)] + for i in range(len(self.wit.vtxinwit), len(self.vin)): + self.wit.vtxinwit.append(CTxInWitness()) + r += self.wit.serialize() + r += struct.pack("<I", self.nLockTime) + return r + + # Regular serialization is without witness -- must explicitly + # call serialize_with_witness to include witness data. + def serialize(self): + return self.serialize_without_witness() + + # Recalculate the txid (transaction hash without witness) + def rehash(self): + self.sha256 = None + self.calc_sha256() + + # We will only cache the serialization without witness in + # self.sha256 and self.hash -- those are expected to be the txid. + def calc_sha256(self, with_witness=False): + if with_witness: + # Don't cache the result, just return it + return uint256_from_str(hash256(self.serialize_with_witness())) + + if self.sha256 is None: + self.sha256 = uint256_from_str(hash256(self.serialize_without_witness())) + self.hash = encode(hash256(self.serialize())[::-1], 'hex_codec').decode('ascii') + + def is_valid(self): + self.calc_sha256() + for tout in self.vout: + if tout.nValue < 0 or tout.nValue > 21000000 * COIN: + return False + return True + + def __repr__(self): + return "CTransaction(nVersion=%i vin=%s vout=%s wit=%s nLockTime=%i)" \ + % (self.nVersion, repr(self.vin), repr(self.vout), repr(self.wit), self.nLockTime) + + +class CBlockHeader(): + def __init__(self, header=None): + if header is None: + self.set_null() + else: + self.nVersion = header.nVersion + self.hashPrevBlock = header.hashPrevBlock + self.hashMerkleRoot = header.hashMerkleRoot + self.nTime = header.nTime + self.nBits = header.nBits + self.nNonce = header.nNonce + self.sha256 = header.sha256 + self.hash = header.hash + self.calc_sha256() + + def set_null(self): + self.nVersion = 1 + self.hashPrevBlock = 0 + self.hashMerkleRoot = 0 + self.nTime = 0 + self.nBits = 0 + self.nNonce = 0 + self.sha256 = None + self.hash = None + + def deserialize(self, f): + self.nVersion = struct.unpack("<i", f.read(4))[0] + self.hashPrevBlock = deser_uint256(f) + self.hashMerkleRoot = deser_uint256(f) + self.nTime = struct.unpack("<I", f.read(4))[0] + self.nBits = struct.unpack("<I", f.read(4))[0] + self.nNonce = struct.unpack("<I", f.read(4))[0] + self.sha256 = None + self.hash = None + + def serialize(self): + r = b"" + r += struct.pack("<i", self.nVersion) + r += ser_uint256(self.hashPrevBlock) + r += ser_uint256(self.hashMerkleRoot) + r += struct.pack("<I", self.nTime) + r += struct.pack("<I", self.nBits) + r += struct.pack("<I", self.nNonce) + return r + + def calc_sha256(self): + if self.sha256 is None: + r = b"" + r += struct.pack("<i", self.nVersion) + r += ser_uint256(self.hashPrevBlock) + r += ser_uint256(self.hashMerkleRoot) + r += struct.pack("<I", self.nTime) + r += struct.pack("<I", self.nBits) + r += struct.pack("<I", self.nNonce) + self.sha256 = uint256_from_str(hash256(r)) + self.hash = encode(hash256(r)[::-1], 'hex_codec').decode('ascii') + + def rehash(self): + self.sha256 = None + self.calc_sha256() + return self.sha256 + + def __repr__(self): + return "CBlockHeader(nVersion=%i hashPrevBlock=%064x hashMerkleRoot=%064x nTime=%s nBits=%08x nNonce=%08x)" \ + % (self.nVersion, self.hashPrevBlock, self.hashMerkleRoot, + time.ctime(self.nTime), self.nBits, self.nNonce) + + +class CBlock(CBlockHeader): + def __init__(self, header=None): + super(CBlock, self).__init__(header) + self.vtx = [] + + def deserialize(self, f): + super(CBlock, self).deserialize(f) + self.vtx = deser_vector(f, CTransaction) + + def serialize(self, with_witness=False): + r = b"" + r += super(CBlock, self).serialize() + if with_witness: + r += ser_vector(self.vtx, "serialize_with_witness") + else: + r += ser_vector(self.vtx) + return r + + # Calculate the merkle root given a vector of transaction hashes + @classmethod + def get_merkle_root(cls, hashes): + while len(hashes) > 1: + newhashes = [] + for i in range(0, len(hashes), 2): + i2 = min(i+1, len(hashes)-1) + newhashes.append(hash256(hashes[i] + hashes[i2])) + hashes = newhashes + return uint256_from_str(hashes[0]) + + def calc_merkle_root(self): + hashes = [] + for tx in self.vtx: + tx.calc_sha256() + hashes.append(ser_uint256(tx.sha256)) + return self.get_merkle_root(hashes) + + def calc_witness_merkle_root(self): + # For witness root purposes, the hash of the + # coinbase, with witness, is defined to be 0...0 + hashes = [ser_uint256(0)] + + for tx in self.vtx[1:]: + # Calculate the hashes with witness data + hashes.append(ser_uint256(tx.calc_sha256(True))) + + return self.get_merkle_root(hashes) + + def is_valid(self): + self.calc_sha256() + target = uint256_from_compact(self.nBits) + if self.sha256 > target: + return False + for tx in self.vtx: + if not tx.is_valid(): + return False + if self.calc_merkle_root() != self.hashMerkleRoot: + return False + return True + + def solve(self): + self.rehash() + target = uint256_from_compact(self.nBits) + while self.sha256 > target: + self.nNonce += 1 + self.rehash() + + def __repr__(self): + return "CBlock(nVersion=%i hashPrevBlock=%064x hashMerkleRoot=%064x nTime=%s nBits=%08x nNonce=%08x vtx=%s)" \ + % (self.nVersion, self.hashPrevBlock, self.hashMerkleRoot, + time.ctime(self.nTime), self.nBits, self.nNonce, repr(self.vtx)) + + +class PrefilledTransaction(): + def __init__(self, index=0, tx = None): + self.index = index + self.tx = tx + + def deserialize(self, f): + self.index = deser_compact_size(f) + self.tx = CTransaction() + self.tx.deserialize(f) + + def serialize(self, with_witness=False): + r = b"" + r += ser_compact_size(self.index) + if with_witness: + r += self.tx.serialize_with_witness() + else: + r += self.tx.serialize_without_witness() + return r + + def serialize_with_witness(self): + return self.serialize(with_witness=True) + + def __repr__(self): + return "PrefilledTransaction(index=%d, tx=%s)" % (self.index, repr(self.tx)) + +# This is what we send on the wire, in a cmpctblock message. +class P2PHeaderAndShortIDs(): + def __init__(self): + self.header = CBlockHeader() + self.nonce = 0 + self.shortids_length = 0 + self.shortids = [] + self.prefilled_txn_length = 0 + self.prefilled_txn = [] + + def deserialize(self, f): + self.header.deserialize(f) + self.nonce = struct.unpack("<Q", f.read(8))[0] + self.shortids_length = deser_compact_size(f) + for i in range(self.shortids_length): + # shortids are defined to be 6 bytes in the spec, so append + # two zero bytes and read it in as an 8-byte number + self.shortids.append(struct.unpack("<Q", f.read(6) + b'\x00\x00')[0]) + self.prefilled_txn = deser_vector(f, PrefilledTransaction) + self.prefilled_txn_length = len(self.prefilled_txn) + + # When using version 2 compact blocks, we must serialize with_witness. + def serialize(self, with_witness=False): + r = b"" + r += self.header.serialize() + r += struct.pack("<Q", self.nonce) + r += ser_compact_size(self.shortids_length) + for x in self.shortids: + # We only want the first 6 bytes + r += struct.pack("<Q", x)[0:6] + if with_witness: + r += ser_vector(self.prefilled_txn, "serialize_with_witness") + else: + r += ser_vector(self.prefilled_txn) + return r + + def __repr__(self): + return "P2PHeaderAndShortIDs(header=%s, nonce=%d, shortids_length=%d, shortids=%s, prefilled_txn_length=%d, prefilledtxn=%s" % (repr(self.header), self.nonce, self.shortids_length, repr(self.shortids), self.prefilled_txn_length, repr(self.prefilled_txn)) + +# P2P version of the above that will use witness serialization (for compact +# block version 2) +class P2PHeaderAndShortWitnessIDs(P2PHeaderAndShortIDs): + def serialize(self): + return super(P2PHeaderAndShortWitnessIDs, self).serialize(with_witness=True) + +# Calculate the BIP 152-compact blocks shortid for a given transaction hash +def calculate_shortid(k0, k1, tx_hash): + expected_shortid = siphash256(k0, k1, tx_hash) + expected_shortid &= 0x0000ffffffffffff + return expected_shortid + +# This version gets rid of the array lengths, and reinterprets the differential +# encoding into indices that can be used for lookup. +class HeaderAndShortIDs(): + def __init__(self, p2pheaders_and_shortids = None): + self.header = CBlockHeader() + self.nonce = 0 + self.shortids = [] + self.prefilled_txn = [] + self.use_witness = False + + if p2pheaders_and_shortids != None: + self.header = p2pheaders_and_shortids.header + self.nonce = p2pheaders_and_shortids.nonce + self.shortids = p2pheaders_and_shortids.shortids + last_index = -1 + for x in p2pheaders_and_shortids.prefilled_txn: + self.prefilled_txn.append(PrefilledTransaction(x.index + last_index + 1, x.tx)) + last_index = self.prefilled_txn[-1].index + + def to_p2p(self): + if self.use_witness: + ret = P2PHeaderAndShortWitnessIDs() + else: + ret = P2PHeaderAndShortIDs() + ret.header = self.header + ret.nonce = self.nonce + ret.shortids_length = len(self.shortids) + ret.shortids = self.shortids + ret.prefilled_txn_length = len(self.prefilled_txn) + ret.prefilled_txn = [] + last_index = -1 + for x in self.prefilled_txn: + ret.prefilled_txn.append(PrefilledTransaction(x.index - last_index - 1, x.tx)) + last_index = x.index + return ret + + def get_siphash_keys(self): + header_nonce = self.header.serialize() + header_nonce += struct.pack("<Q", self.nonce) + hash_header_nonce_as_str = sha256(header_nonce) + key0 = struct.unpack("<Q", hash_header_nonce_as_str[0:8])[0] + key1 = struct.unpack("<Q", hash_header_nonce_as_str[8:16])[0] + return [ key0, key1 ] + + # Version 2 compact blocks use wtxid in shortids (rather than txid) + def initialize_from_block(self, block, nonce=0, prefill_list = [0], use_witness = False): + self.header = CBlockHeader(block) + self.nonce = nonce + self.prefilled_txn = [ PrefilledTransaction(i, block.vtx[i]) for i in prefill_list ] + self.shortids = [] + self.use_witness = use_witness + [k0, k1] = self.get_siphash_keys() + for i in range(len(block.vtx)): + if i not in prefill_list: + tx_hash = block.vtx[i].sha256 + if use_witness: + tx_hash = block.vtx[i].calc_sha256(with_witness=True) + self.shortids.append(calculate_shortid(k0, k1, tx_hash)) + + def __repr__(self): + return "HeaderAndShortIDs(header=%s, nonce=%d, shortids=%s, prefilledtxn=%s" % (repr(self.header), self.nonce, repr(self.shortids), repr(self.prefilled_txn)) + + +class BlockTransactionsRequest(): + + def __init__(self, blockhash=0, indexes = None): + self.blockhash = blockhash + self.indexes = indexes if indexes != None else [] + + def deserialize(self, f): + self.blockhash = deser_uint256(f) + indexes_length = deser_compact_size(f) + for i in range(indexes_length): + self.indexes.append(deser_compact_size(f)) + + def serialize(self): + r = b"" + r += ser_uint256(self.blockhash) + r += ser_compact_size(len(self.indexes)) + for x in self.indexes: + r += ser_compact_size(x) + return r + + # helper to set the differentially encoded indexes from absolute ones + def from_absolute(self, absolute_indexes): + self.indexes = [] + last_index = -1 + for x in absolute_indexes: + self.indexes.append(x-last_index-1) + last_index = x + + def to_absolute(self): + absolute_indexes = [] + last_index = -1 + for x in self.indexes: + absolute_indexes.append(x+last_index+1) + last_index = absolute_indexes[-1] + return absolute_indexes + + def __repr__(self): + return "BlockTransactionsRequest(hash=%064x indexes=%s)" % (self.blockhash, repr(self.indexes)) + + +class BlockTransactions(): + + def __init__(self, blockhash=0, transactions = None): + self.blockhash = blockhash + self.transactions = transactions if transactions != None else [] + + def deserialize(self, f): + self.blockhash = deser_uint256(f) + self.transactions = deser_vector(f, CTransaction) + + def serialize(self, with_witness=False): + r = b"" + r += ser_uint256(self.blockhash) + if with_witness: + r += ser_vector(self.transactions, "serialize_with_witness") + else: + r += ser_vector(self.transactions) + return r + + def __repr__(self): + return "BlockTransactions(hash=%064x transactions=%s)" % (self.blockhash, repr(self.transactions)) + + +# Objects that correspond to messages on the wire +class msg_version(): + command = b"version" + + def __init__(self): + self.nVersion = MY_VERSION + self.nServices = NODE_NETWORK | NODE_WITNESS + self.nTime = int(time.time()) + self.addrTo = CAddress() + self.addrFrom = CAddress() + self.nNonce = random.getrandbits(64) + self.strSubVer = MY_SUBVERSION + self.nStartingHeight = -1 + self.nRelay = MY_RELAY + + def deserialize(self, f): + self.nVersion = struct.unpack("<i", f.read(4))[0] + if self.nVersion == 10300: + self.nVersion = 300 + self.nServices = struct.unpack("<Q", f.read(8))[0] + self.nTime = struct.unpack("<q", f.read(8))[0] + self.addrTo = CAddress() + self.addrTo.deserialize(f) + + if self.nVersion >= 106: + self.addrFrom = CAddress() + self.addrFrom.deserialize(f) + self.nNonce = struct.unpack("<Q", f.read(8))[0] + self.strSubVer = deser_string(f) + else: + self.addrFrom = None + self.nNonce = None + self.strSubVer = None + self.nStartingHeight = None + + if self.nVersion >= 209: + self.nStartingHeight = struct.unpack("<i", f.read(4))[0] + else: + self.nStartingHeight = None + + if self.nVersion >= 70001: + # Relay field is optional for version 70001 onwards + try: + self.nRelay = struct.unpack("<b", f.read(1))[0] + except: + self.nRelay = 0 + else: + self.nRelay = 0 + + def serialize(self): + r = b"" + r += struct.pack("<i", self.nVersion) + r += struct.pack("<Q", self.nServices) + r += struct.pack("<q", self.nTime) + r += self.addrTo.serialize() + r += self.addrFrom.serialize() + r += struct.pack("<Q", self.nNonce) + r += ser_string(self.strSubVer) + r += struct.pack("<i", self.nStartingHeight) + r += struct.pack("<b", self.nRelay) + return r + + def __repr__(self): + return 'msg_version(nVersion=%i nServices=%i nTime=%s addrTo=%s addrFrom=%s nNonce=0x%016X strSubVer=%s nStartingHeight=%i nRelay=%i)' \ + % (self.nVersion, self.nServices, time.ctime(self.nTime), + repr(self.addrTo), repr(self.addrFrom), self.nNonce, + self.strSubVer, self.nStartingHeight, self.nRelay) + + +class msg_verack(): + command = b"verack" + + def __init__(self): + pass + + def deserialize(self, f): + pass + + def serialize(self): + return b"" + + def __repr__(self): + return "msg_verack()" + + +class msg_addr(): + command = b"addr" + + def __init__(self): + self.addrs = [] + + def deserialize(self, f): + self.addrs = deser_vector(f, CAddress) + + def serialize(self): + return ser_vector(self.addrs) + + def __repr__(self): + return "msg_addr(addrs=%s)" % (repr(self.addrs)) + + +class msg_inv(): + command = b"inv" + + def __init__(self, inv=None): + if inv is None: + self.inv = [] + else: + self.inv = inv + + def deserialize(self, f): + self.inv = deser_vector(f, CInv) + + def serialize(self): + return ser_vector(self.inv) + + def __repr__(self): + return "msg_inv(inv=%s)" % (repr(self.inv)) + + +class msg_getdata(): + command = b"getdata" + + def __init__(self, inv=None): + self.inv = inv if inv != None else [] + + def deserialize(self, f): + self.inv = deser_vector(f, CInv) + + def serialize(self): + return ser_vector(self.inv) + + def __repr__(self): + return "msg_getdata(inv=%s)" % (repr(self.inv)) + + +class msg_getblocks(): + command = b"getblocks" + + def __init__(self): + self.locator = CBlockLocator() + self.hashstop = 0 + + def deserialize(self, f): + self.locator = CBlockLocator() + self.locator.deserialize(f) + self.hashstop = deser_uint256(f) + + def serialize(self): + r = b"" + r += self.locator.serialize() + r += ser_uint256(self.hashstop) + return r + + def __repr__(self): + return "msg_getblocks(locator=%s hashstop=%064x)" \ + % (repr(self.locator), self.hashstop) + + +class msg_tx(): + command = b"tx" + + def __init__(self, tx=CTransaction()): + self.tx = tx + + def deserialize(self, f): + self.tx.deserialize(f) + + def serialize(self): + return self.tx.serialize_without_witness() + + def __repr__(self): + return "msg_tx(tx=%s)" % (repr(self.tx)) + +class msg_witness_tx(msg_tx): + + def serialize(self): + return self.tx.serialize_with_witness() + + +class msg_block(): + command = b"block" + + def __init__(self, block=None): + if block is None: + self.block = CBlock() + else: + self.block = block + + def deserialize(self, f): + self.block.deserialize(f) + + def serialize(self): + return self.block.serialize() + + def __repr__(self): + return "msg_block(block=%s)" % (repr(self.block)) + +# for cases where a user needs tighter control over what is sent over the wire +# note that the user must supply the name of the command, and the data +class msg_generic(): + def __init__(self, command, data=None): + self.command = command + self.data = data + + def serialize(self): + return self.data + + def __repr__(self): + return "msg_generic()" + +class msg_witness_block(msg_block): + + def serialize(self): + r = self.block.serialize(with_witness=True) + return r + +class msg_getaddr(): + command = b"getaddr" + + def __init__(self): + pass + + def deserialize(self, f): + pass + + def serialize(self): + return b"" + + def __repr__(self): + return "msg_getaddr()" + + +class msg_ping(): + command = b"ping" + + def __init__(self, nonce=0): + self.nonce = nonce + + def deserialize(self, f): + self.nonce = struct.unpack("<Q", f.read(8))[0] + + def serialize(self): + r = b"" + r += struct.pack("<Q", self.nonce) + return r + + def __repr__(self): + return "msg_ping(nonce=%08x)" % self.nonce + + +class msg_pong(): + command = b"pong" + + def __init__(self, nonce=0): + self.nonce = nonce + + def deserialize(self, f): + self.nonce = struct.unpack("<Q", f.read(8))[0] + + def serialize(self): + r = b"" + r += struct.pack("<Q", self.nonce) + return r + + def __repr__(self): + return "msg_pong(nonce=%08x)" % self.nonce + + +class msg_mempool(): + command = b"mempool" + + def __init__(self): + pass + + def deserialize(self, f): + pass + + def serialize(self): + return b"" + + def __repr__(self): + return "msg_mempool()" + +class msg_sendheaders(): + command = b"sendheaders" + + def __init__(self): + pass + + def deserialize(self, f): + pass + + def serialize(self): + return b"" + + def __repr__(self): + return "msg_sendheaders()" + + +# getheaders message has +# number of entries +# vector of hashes +# hash_stop (hash of last desired block header, 0 to get as many as possible) +class msg_getheaders(): + command = b"getheaders" + + def __init__(self): + self.locator = CBlockLocator() + self.hashstop = 0 + + def deserialize(self, f): + self.locator = CBlockLocator() + self.locator.deserialize(f) + self.hashstop = deser_uint256(f) + + def serialize(self): + r = b"" + r += self.locator.serialize() + r += ser_uint256(self.hashstop) + return r + + def __repr__(self): + return "msg_getheaders(locator=%s, stop=%064x)" \ + % (repr(self.locator), self.hashstop) + + +# headers message has +# <count> <vector of block headers> +class msg_headers(): + command = b"headers" + + def __init__(self, headers=None): + self.headers = headers if headers is not None else [] + + def deserialize(self, f): + # comment in bitcoind indicates these should be deserialized as blocks + blocks = deser_vector(f, CBlock) + for x in blocks: + self.headers.append(CBlockHeader(x)) + + def serialize(self): + blocks = [CBlock(x) for x in self.headers] + return ser_vector(blocks) + + def __repr__(self): + return "msg_headers(headers=%s)" % repr(self.headers) + + +class msg_reject(): + command = b"reject" + REJECT_MALFORMED = 1 + + def __init__(self): + self.message = b"" + self.code = 0 + self.reason = b"" + self.data = 0 + + def deserialize(self, f): + self.message = deser_string(f) + self.code = struct.unpack("<B", f.read(1))[0] + self.reason = deser_string(f) + if (self.code != self.REJECT_MALFORMED and + (self.message == b"block" or self.message == b"tx")): + self.data = deser_uint256(f) + + def serialize(self): + r = ser_string(self.message) + r += struct.pack("<B", self.code) + r += ser_string(self.reason) + if (self.code != self.REJECT_MALFORMED and + (self.message == b"block" or self.message == b"tx")): + r += ser_uint256(self.data) + return r + + def __repr__(self): + return "msg_reject: %s %d %s [%064x]" \ + % (self.message, self.code, self.reason, self.data) + +class msg_feefilter(): + command = b"feefilter" + + def __init__(self, feerate=0): + self.feerate = feerate + + def deserialize(self, f): + self.feerate = struct.unpack("<Q", f.read(8))[0] + + def serialize(self): + r = b"" + r += struct.pack("<Q", self.feerate) + return r + + def __repr__(self): + return "msg_feefilter(feerate=%08x)" % self.feerate + +class msg_sendcmpct(): + command = b"sendcmpct" + + def __init__(self): + self.announce = False + self.version = 1 + + def deserialize(self, f): + self.announce = struct.unpack("<?", f.read(1))[0] + self.version = struct.unpack("<Q", f.read(8))[0] + + def serialize(self): + r = b"" + r += struct.pack("<?", self.announce) + r += struct.pack("<Q", self.version) + return r + + def __repr__(self): + return "msg_sendcmpct(announce=%s, version=%lu)" % (self.announce, self.version) + +class msg_cmpctblock(): + command = b"cmpctblock" + + def __init__(self, header_and_shortids = None): + self.header_and_shortids = header_and_shortids + + def deserialize(self, f): + self.header_and_shortids = P2PHeaderAndShortIDs() + self.header_and_shortids.deserialize(f) + + def serialize(self): + r = b"" + r += self.header_and_shortids.serialize() + return r + + def __repr__(self): + return "msg_cmpctblock(HeaderAndShortIDs=%s)" % repr(self.header_and_shortids) + +class msg_getblocktxn(): + command = b"getblocktxn" + + def __init__(self): + self.block_txn_request = None + + def deserialize(self, f): + self.block_txn_request = BlockTransactionsRequest() + self.block_txn_request.deserialize(f) + + def serialize(self): + r = b"" + r += self.block_txn_request.serialize() + return r + + def __repr__(self): + return "msg_getblocktxn(block_txn_request=%s)" % (repr(self.block_txn_request)) + +class msg_blocktxn(): + command = b"blocktxn" + + def __init__(self): + self.block_transactions = BlockTransactions() + + def deserialize(self, f): + self.block_transactions.deserialize(f) + + def serialize(self): + r = b"" + r += self.block_transactions.serialize() + return r + + def __repr__(self): + return "msg_blocktxn(block_transactions=%s)" % (repr(self.block_transactions)) + +class msg_witness_blocktxn(msg_blocktxn): + def serialize(self): + r = b"" + r += self.block_transactions.serialize(with_witness=True) + return r diff --git a/test/functional/test_framework/mininode.py b/test/functional/test_framework/mininode.py index 24ee09b81c..24c96b5681 100755 --- a/test/functional/test_framework/mininode.py +++ b/test/functional/test_framework/mininode.py @@ -12,1337 +12,50 @@ found in the mini-node branch of http://github.com/jgarzik/pynode. NodeConn: an object which manages p2p connectivity to a bitcoin node NodeConnCB: a base class that describes the interface for receiving callbacks with network messages from a NodeConn -CBlock, CTransaction, CBlockHeader, CTxIn, CTxOut, etc....: - data structures that should map to corresponding structures in - bitcoin/primitives -msg_block, msg_tx, msg_headers, etc.: - data structures that represent network messages -ser_*, deser_*: functions that handle serialization/deserialization """ - import asyncore -from codecs import encode from collections import defaultdict -import copy -import hashlib from io import BytesIO import logging -import random import socket import struct import sys import time from threading import RLock, Thread -from test_framework.siphash import siphash256 -from test_framework.util import hex_str_to_bytes, bytes_to_hex_str, wait_until - -MIN_VERSION_SUPPORTED = 60001 -MY_VERSION = 70014 # past bip-31 for ping/pong -MY_SUBVERSION = b"/python-mininode-tester:0.0.3/" -MY_RELAY = 1 # from version 70001 onwards, fRelay should be appended to version messages (BIP37) - -MAX_INV_SZ = 50000 -MAX_BLOCK_BASE_SIZE = 1000000 - -COIN = 100000000 # 1 btc in satoshis - -NODE_NETWORK = (1 << 0) -# NODE_GETUTXO = (1 << 1) -# NODE_BLOOM = (1 << 2) -NODE_WITNESS = (1 << 3) -NODE_UNSUPPORTED_SERVICE_BIT_5 = (1 << 5) -NODE_UNSUPPORTED_SERVICE_BIT_7 = (1 << 7) +from test_framework.messages import * logger = logging.getLogger("TestFramework.mininode") -# Keep our own socket map for asyncore, so that we can track disconnects -# ourselves (to workaround an issue with closing an asyncore socket when -# using select) -mininode_socket_map = dict() - -# One lock for synchronizing all data access between the networking thread (see -# NetworkThread below) and the thread running the test logic. For simplicity, -# NodeConn acquires this lock whenever delivering a message to a NodeConnCB, -# and whenever adding anything to the send buffer (in send_message()). This -# lock should be acquired in the thread running the test logic to synchronize -# access to any data shared with the NodeConnCB or NodeConn. -mininode_lock = RLock() - -# Serialization/deserialization tools -def sha256(s): - return hashlib.new('sha256', s).digest() - -def ripemd160(s): - return hashlib.new('ripemd160', s).digest() - -def hash256(s): - return sha256(sha256(s)) - -def ser_compact_size(l): - r = b"" - if l < 253: - r = struct.pack("B", l) - elif l < 0x10000: - r = struct.pack("<BH", 253, l) - elif l < 0x100000000: - r = struct.pack("<BI", 254, l) - else: - r = struct.pack("<BQ", 255, l) - return r - -def deser_compact_size(f): - nit = struct.unpack("<B", f.read(1))[0] - if nit == 253: - nit = struct.unpack("<H", f.read(2))[0] - elif nit == 254: - nit = struct.unpack("<I", f.read(4))[0] - elif nit == 255: - nit = struct.unpack("<Q", f.read(8))[0] - return nit - -def deser_string(f): - nit = deser_compact_size(f) - return f.read(nit) - -def ser_string(s): - return ser_compact_size(len(s)) + s - -def deser_uint256(f): - r = 0 - for i in range(8): - t = struct.unpack("<I", f.read(4))[0] - r += t << (i * 32) - return r - - -def ser_uint256(u): - rs = b"" - for i in range(8): - rs += struct.pack("<I", u & 0xFFFFFFFF) - u >>= 32 - return rs - - -def uint256_from_str(s): - r = 0 - t = struct.unpack("<IIIIIIII", s[:32]) - for i in range(8): - r += t[i] << (i * 32) - return r - - -def uint256_from_compact(c): - nbytes = (c >> 24) & 0xFF - v = (c & 0xFFFFFF) << (8 * (nbytes - 3)) - return v - - -def deser_vector(f, c): - nit = deser_compact_size(f) - r = [] - for i in range(nit): - t = c() - t.deserialize(f) - r.append(t) - return r - - -# ser_function_name: Allow for an alternate serialization function on the -# entries in the vector (we use this for serializing the vector of transactions -# for a witness block). -def ser_vector(l, ser_function_name=None): - r = ser_compact_size(len(l)) - for i in l: - if ser_function_name: - r += getattr(i, ser_function_name)() - else: - r += i.serialize() - return r - - -def deser_uint256_vector(f): - nit = deser_compact_size(f) - r = [] - for i in range(nit): - t = deser_uint256(f) - r.append(t) - return r - - -def ser_uint256_vector(l): - r = ser_compact_size(len(l)) - for i in l: - r += ser_uint256(i) - return r - - -def deser_string_vector(f): - nit = deser_compact_size(f) - r = [] - for i in range(nit): - t = deser_string(f) - r.append(t) - return r - - -def ser_string_vector(l): - r = ser_compact_size(len(l)) - for sv in l: - r += ser_string(sv) - return r - - -def deser_int_vector(f): - nit = deser_compact_size(f) - r = [] - for i in range(nit): - t = struct.unpack("<i", f.read(4))[0] - r.append(t) - return r - - -def ser_int_vector(l): - r = ser_compact_size(len(l)) - for i in l: - r += struct.pack("<i", i) - return r - -# Deserialize from a hex string representation (eg from RPC) -def FromHex(obj, hex_string): - obj.deserialize(BytesIO(hex_str_to_bytes(hex_string))) - return obj - -# Convert a binary-serializable object to hex (eg for submission via RPC) -def ToHex(obj): - return bytes_to_hex_str(obj.serialize()) - -# Objects that map to bitcoind objects, which can be serialized/deserialized - -class CAddress(): - def __init__(self): - self.nServices = 1 - self.pchReserved = b"\x00" * 10 + b"\xff" * 2 - self.ip = "0.0.0.0" - self.port = 0 - - def deserialize(self, f): - self.nServices = struct.unpack("<Q", f.read(8))[0] - self.pchReserved = f.read(12) - self.ip = socket.inet_ntoa(f.read(4)) - self.port = struct.unpack(">H", f.read(2))[0] - - def serialize(self): - r = b"" - r += struct.pack("<Q", self.nServices) - r += self.pchReserved - r += socket.inet_aton(self.ip) - r += struct.pack(">H", self.port) - return r - - def __repr__(self): - return "CAddress(nServices=%i ip=%s port=%i)" % (self.nServices, - self.ip, self.port) - -MSG_WITNESS_FLAG = 1<<30 - -class CInv(): - typemap = { - 0: "Error", - 1: "TX", - 2: "Block", - 1|MSG_WITNESS_FLAG: "WitnessTx", - 2|MSG_WITNESS_FLAG : "WitnessBlock", - 4: "CompactBlock" - } - - def __init__(self, t=0, h=0): - self.type = t - self.hash = h - - def deserialize(self, f): - self.type = struct.unpack("<i", f.read(4))[0] - self.hash = deser_uint256(f) - - def serialize(self): - r = b"" - r += struct.pack("<i", self.type) - r += ser_uint256(self.hash) - return r - - def __repr__(self): - return "CInv(type=%s hash=%064x)" \ - % (self.typemap[self.type], self.hash) - - -class CBlockLocator(): - def __init__(self): - self.nVersion = MY_VERSION - self.vHave = [] - - def deserialize(self, f): - self.nVersion = struct.unpack("<i", f.read(4))[0] - self.vHave = deser_uint256_vector(f) - - def serialize(self): - r = b"" - r += struct.pack("<i", self.nVersion) - r += ser_uint256_vector(self.vHave) - return r - - def __repr__(self): - return "CBlockLocator(nVersion=%i vHave=%s)" \ - % (self.nVersion, repr(self.vHave)) - - -class COutPoint(): - def __init__(self, hash=0, n=0): - self.hash = hash - self.n = n - - def deserialize(self, f): - self.hash = deser_uint256(f) - self.n = struct.unpack("<I", f.read(4))[0] - - def serialize(self): - r = b"" - r += ser_uint256(self.hash) - r += struct.pack("<I", self.n) - return r - - def __repr__(self): - return "COutPoint(hash=%064x n=%i)" % (self.hash, self.n) - - -class CTxIn(): - def __init__(self, outpoint=None, scriptSig=b"", nSequence=0): - if outpoint is None: - self.prevout = COutPoint() - else: - self.prevout = outpoint - self.scriptSig = scriptSig - self.nSequence = nSequence - - def deserialize(self, f): - self.prevout = COutPoint() - self.prevout.deserialize(f) - self.scriptSig = deser_string(f) - self.nSequence = struct.unpack("<I", f.read(4))[0] - - def serialize(self): - r = b"" - r += self.prevout.serialize() - r += ser_string(self.scriptSig) - r += struct.pack("<I", self.nSequence) - return r - - def __repr__(self): - return "CTxIn(prevout=%s scriptSig=%s nSequence=%i)" \ - % (repr(self.prevout), bytes_to_hex_str(self.scriptSig), - self.nSequence) - - -class CTxOut(): - def __init__(self, nValue=0, scriptPubKey=b""): - self.nValue = nValue - self.scriptPubKey = scriptPubKey - - def deserialize(self, f): - self.nValue = struct.unpack("<q", f.read(8))[0] - self.scriptPubKey = deser_string(f) - - def serialize(self): - r = b"" - r += struct.pack("<q", self.nValue) - r += ser_string(self.scriptPubKey) - return r - - def __repr__(self): - return "CTxOut(nValue=%i.%08i scriptPubKey=%s)" \ - % (self.nValue // COIN, self.nValue % COIN, - bytes_to_hex_str(self.scriptPubKey)) - - -class CScriptWitness(): - def __init__(self): - # stack is a vector of strings - self.stack = [] - - def __repr__(self): - return "CScriptWitness(%s)" % \ - (",".join([bytes_to_hex_str(x) for x in self.stack])) - - def is_null(self): - if self.stack: - return False - return True - - -class CTxInWitness(): - def __init__(self): - self.scriptWitness = CScriptWitness() - - def deserialize(self, f): - self.scriptWitness.stack = deser_string_vector(f) - - def serialize(self): - return ser_string_vector(self.scriptWitness.stack) - - def __repr__(self): - return repr(self.scriptWitness) - - def is_null(self): - return self.scriptWitness.is_null() - - -class CTxWitness(): - def __init__(self): - self.vtxinwit = [] - - def deserialize(self, f): - for i in range(len(self.vtxinwit)): - self.vtxinwit[i].deserialize(f) - - def serialize(self): - r = b"" - # This is different than the usual vector serialization -- - # we omit the length of the vector, which is required to be - # the same length as the transaction's vin vector. - for x in self.vtxinwit: - r += x.serialize() - return r - - def __repr__(self): - return "CTxWitness(%s)" % \ - (';'.join([repr(x) for x in self.vtxinwit])) - - def is_null(self): - for x in self.vtxinwit: - if not x.is_null(): - return False - return True - - -class CTransaction(): - def __init__(self, tx=None): - if tx is None: - self.nVersion = 1 - self.vin = [] - self.vout = [] - self.wit = CTxWitness() - self.nLockTime = 0 - self.sha256 = None - self.hash = None - else: - self.nVersion = tx.nVersion - self.vin = copy.deepcopy(tx.vin) - self.vout = copy.deepcopy(tx.vout) - self.nLockTime = tx.nLockTime - self.sha256 = tx.sha256 - self.hash = tx.hash - self.wit = copy.deepcopy(tx.wit) - - def deserialize(self, f): - self.nVersion = struct.unpack("<i", f.read(4))[0] - self.vin = deser_vector(f, CTxIn) - flags = 0 - if len(self.vin) == 0: - flags = struct.unpack("<B", f.read(1))[0] - # Not sure why flags can't be zero, but this - # matches the implementation in bitcoind - if (flags != 0): - self.vin = deser_vector(f, CTxIn) - self.vout = deser_vector(f, CTxOut) - else: - self.vout = deser_vector(f, CTxOut) - if flags != 0: - self.wit.vtxinwit = [CTxInWitness() for i in range(len(self.vin))] - self.wit.deserialize(f) - self.nLockTime = struct.unpack("<I", f.read(4))[0] - self.sha256 = None - self.hash = None - - def serialize_without_witness(self): - r = b"" - r += struct.pack("<i", self.nVersion) - r += ser_vector(self.vin) - r += ser_vector(self.vout) - r += struct.pack("<I", self.nLockTime) - return r - - # Only serialize with witness when explicitly called for - def serialize_with_witness(self): - flags = 0 - if not self.wit.is_null(): - flags |= 1 - r = b"" - r += struct.pack("<i", self.nVersion) - if flags: - dummy = [] - r += ser_vector(dummy) - r += struct.pack("<B", flags) - r += ser_vector(self.vin) - r += ser_vector(self.vout) - if flags & 1: - if (len(self.wit.vtxinwit) != len(self.vin)): - # vtxinwit must have the same length as vin - self.wit.vtxinwit = self.wit.vtxinwit[:len(self.vin)] - for i in range(len(self.wit.vtxinwit), len(self.vin)): - self.wit.vtxinwit.append(CTxInWitness()) - r += self.wit.serialize() - r += struct.pack("<I", self.nLockTime) - return r - - # Regular serialization is without witness -- must explicitly - # call serialize_with_witness to include witness data. - def serialize(self): - return self.serialize_without_witness() - - # Recalculate the txid (transaction hash without witness) - def rehash(self): - self.sha256 = None - self.calc_sha256() - - # We will only cache the serialization without witness in - # self.sha256 and self.hash -- those are expected to be the txid. - def calc_sha256(self, with_witness=False): - if with_witness: - # Don't cache the result, just return it - return uint256_from_str(hash256(self.serialize_with_witness())) - - if self.sha256 is None: - self.sha256 = uint256_from_str(hash256(self.serialize_without_witness())) - self.hash = encode(hash256(self.serialize())[::-1], 'hex_codec').decode('ascii') - - def is_valid(self): - self.calc_sha256() - for tout in self.vout: - if tout.nValue < 0 or tout.nValue > 21000000 * COIN: - return False - return True - - def __repr__(self): - return "CTransaction(nVersion=%i vin=%s vout=%s wit=%s nLockTime=%i)" \ - % (self.nVersion, repr(self.vin), repr(self.vout), repr(self.wit), self.nLockTime) - - -class CBlockHeader(): - def __init__(self, header=None): - if header is None: - self.set_null() - else: - self.nVersion = header.nVersion - self.hashPrevBlock = header.hashPrevBlock - self.hashMerkleRoot = header.hashMerkleRoot - self.nTime = header.nTime - self.nBits = header.nBits - self.nNonce = header.nNonce - self.sha256 = header.sha256 - self.hash = header.hash - self.calc_sha256() - - def set_null(self): - self.nVersion = 1 - self.hashPrevBlock = 0 - self.hashMerkleRoot = 0 - self.nTime = 0 - self.nBits = 0 - self.nNonce = 0 - self.sha256 = None - self.hash = None - - def deserialize(self, f): - self.nVersion = struct.unpack("<i", f.read(4))[0] - self.hashPrevBlock = deser_uint256(f) - self.hashMerkleRoot = deser_uint256(f) - self.nTime = struct.unpack("<I", f.read(4))[0] - self.nBits = struct.unpack("<I", f.read(4))[0] - self.nNonce = struct.unpack("<I", f.read(4))[0] - self.sha256 = None - self.hash = None - - def serialize(self): - r = b"" - r += struct.pack("<i", self.nVersion) - r += ser_uint256(self.hashPrevBlock) - r += ser_uint256(self.hashMerkleRoot) - r += struct.pack("<I", self.nTime) - r += struct.pack("<I", self.nBits) - r += struct.pack("<I", self.nNonce) - return r - - def calc_sha256(self): - if self.sha256 is None: - r = b"" - r += struct.pack("<i", self.nVersion) - r += ser_uint256(self.hashPrevBlock) - r += ser_uint256(self.hashMerkleRoot) - r += struct.pack("<I", self.nTime) - r += struct.pack("<I", self.nBits) - r += struct.pack("<I", self.nNonce) - self.sha256 = uint256_from_str(hash256(r)) - self.hash = encode(hash256(r)[::-1], 'hex_codec').decode('ascii') - - def rehash(self): - self.sha256 = None - self.calc_sha256() - return self.sha256 - - def __repr__(self): - return "CBlockHeader(nVersion=%i hashPrevBlock=%064x hashMerkleRoot=%064x nTime=%s nBits=%08x nNonce=%08x)" \ - % (self.nVersion, self.hashPrevBlock, self.hashMerkleRoot, - time.ctime(self.nTime), self.nBits, self.nNonce) - - -class CBlock(CBlockHeader): - def __init__(self, header=None): - super(CBlock, self).__init__(header) - self.vtx = [] - - def deserialize(self, f): - super(CBlock, self).deserialize(f) - self.vtx = deser_vector(f, CTransaction) - - def serialize(self, with_witness=False): - r = b"" - r += super(CBlock, self).serialize() - if with_witness: - r += ser_vector(self.vtx, "serialize_with_witness") - else: - r += ser_vector(self.vtx) - return r - - # Calculate the merkle root given a vector of transaction hashes - @classmethod - def get_merkle_root(cls, hashes): - while len(hashes) > 1: - newhashes = [] - for i in range(0, len(hashes), 2): - i2 = min(i+1, len(hashes)-1) - newhashes.append(hash256(hashes[i] + hashes[i2])) - hashes = newhashes - return uint256_from_str(hashes[0]) - - def calc_merkle_root(self): - hashes = [] - for tx in self.vtx: - tx.calc_sha256() - hashes.append(ser_uint256(tx.sha256)) - return self.get_merkle_root(hashes) - - def calc_witness_merkle_root(self): - # For witness root purposes, the hash of the - # coinbase, with witness, is defined to be 0...0 - hashes = [ser_uint256(0)] - - for tx in self.vtx[1:]: - # Calculate the hashes with witness data - hashes.append(ser_uint256(tx.calc_sha256(True))) - - return self.get_merkle_root(hashes) - - def is_valid(self): - self.calc_sha256() - target = uint256_from_compact(self.nBits) - if self.sha256 > target: - return False - for tx in self.vtx: - if not tx.is_valid(): - return False - if self.calc_merkle_root() != self.hashMerkleRoot: - return False - return True - - def solve(self): - self.rehash() - target = uint256_from_compact(self.nBits) - while self.sha256 > target: - self.nNonce += 1 - self.rehash() - - def __repr__(self): - return "CBlock(nVersion=%i hashPrevBlock=%064x hashMerkleRoot=%064x nTime=%s nBits=%08x nNonce=%08x vtx=%s)" \ - % (self.nVersion, self.hashPrevBlock, self.hashMerkleRoot, - time.ctime(self.nTime), self.nBits, self.nNonce, repr(self.vtx)) - - -class PrefilledTransaction(): - def __init__(self, index=0, tx = None): - self.index = index - self.tx = tx - - def deserialize(self, f): - self.index = deser_compact_size(f) - self.tx = CTransaction() - self.tx.deserialize(f) - - def serialize(self, with_witness=False): - r = b"" - r += ser_compact_size(self.index) - if with_witness: - r += self.tx.serialize_with_witness() - else: - r += self.tx.serialize_without_witness() - return r - - def serialize_with_witness(self): - return self.serialize(with_witness=True) - - def __repr__(self): - return "PrefilledTransaction(index=%d, tx=%s)" % (self.index, repr(self.tx)) - -# This is what we send on the wire, in a cmpctblock message. -class P2PHeaderAndShortIDs(): - def __init__(self): - self.header = CBlockHeader() - self.nonce = 0 - self.shortids_length = 0 - self.shortids = [] - self.prefilled_txn_length = 0 - self.prefilled_txn = [] - - def deserialize(self, f): - self.header.deserialize(f) - self.nonce = struct.unpack("<Q", f.read(8))[0] - self.shortids_length = deser_compact_size(f) - for i in range(self.shortids_length): - # shortids are defined to be 6 bytes in the spec, so append - # two zero bytes and read it in as an 8-byte number - self.shortids.append(struct.unpack("<Q", f.read(6) + b'\x00\x00')[0]) - self.prefilled_txn = deser_vector(f, PrefilledTransaction) - self.prefilled_txn_length = len(self.prefilled_txn) - - # When using version 2 compact blocks, we must serialize with_witness. - def serialize(self, with_witness=False): - r = b"" - r += self.header.serialize() - r += struct.pack("<Q", self.nonce) - r += ser_compact_size(self.shortids_length) - for x in self.shortids: - # We only want the first 6 bytes - r += struct.pack("<Q", x)[0:6] - if with_witness: - r += ser_vector(self.prefilled_txn, "serialize_with_witness") - else: - r += ser_vector(self.prefilled_txn) - return r - - def __repr__(self): - return "P2PHeaderAndShortIDs(header=%s, nonce=%d, shortids_length=%d, shortids=%s, prefilled_txn_length=%d, prefilledtxn=%s" % (repr(self.header), self.nonce, self.shortids_length, repr(self.shortids), self.prefilled_txn_length, repr(self.prefilled_txn)) - -# P2P version of the above that will use witness serialization (for compact -# block version 2) -class P2PHeaderAndShortWitnessIDs(P2PHeaderAndShortIDs): - def serialize(self): - return super(P2PHeaderAndShortWitnessIDs, self).serialize(with_witness=True) - -# Calculate the BIP 152-compact blocks shortid for a given transaction hash -def calculate_shortid(k0, k1, tx_hash): - expected_shortid = siphash256(k0, k1, tx_hash) - expected_shortid &= 0x0000ffffffffffff - return expected_shortid - -# This version gets rid of the array lengths, and reinterprets the differential -# encoding into indices that can be used for lookup. -class HeaderAndShortIDs(): - def __init__(self, p2pheaders_and_shortids = None): - self.header = CBlockHeader() - self.nonce = 0 - self.shortids = [] - self.prefilled_txn = [] - self.use_witness = False - - if p2pheaders_and_shortids != None: - self.header = p2pheaders_and_shortids.header - self.nonce = p2pheaders_and_shortids.nonce - self.shortids = p2pheaders_and_shortids.shortids - last_index = -1 - for x in p2pheaders_and_shortids.prefilled_txn: - self.prefilled_txn.append(PrefilledTransaction(x.index + last_index + 1, x.tx)) - last_index = self.prefilled_txn[-1].index - - def to_p2p(self): - if self.use_witness: - ret = P2PHeaderAndShortWitnessIDs() - else: - ret = P2PHeaderAndShortIDs() - ret.header = self.header - ret.nonce = self.nonce - ret.shortids_length = len(self.shortids) - ret.shortids = self.shortids - ret.prefilled_txn_length = len(self.prefilled_txn) - ret.prefilled_txn = [] - last_index = -1 - for x in self.prefilled_txn: - ret.prefilled_txn.append(PrefilledTransaction(x.index - last_index - 1, x.tx)) - last_index = x.index - return ret - - def get_siphash_keys(self): - header_nonce = self.header.serialize() - header_nonce += struct.pack("<Q", self.nonce) - hash_header_nonce_as_str = sha256(header_nonce) - key0 = struct.unpack("<Q", hash_header_nonce_as_str[0:8])[0] - key1 = struct.unpack("<Q", hash_header_nonce_as_str[8:16])[0] - return [ key0, key1 ] - - # Version 2 compact blocks use wtxid in shortids (rather than txid) - def initialize_from_block(self, block, nonce=0, prefill_list = [0], use_witness = False): - self.header = CBlockHeader(block) - self.nonce = nonce - self.prefilled_txn = [ PrefilledTransaction(i, block.vtx[i]) for i in prefill_list ] - self.shortids = [] - self.use_witness = use_witness - [k0, k1] = self.get_siphash_keys() - for i in range(len(block.vtx)): - if i not in prefill_list: - tx_hash = block.vtx[i].sha256 - if use_witness: - tx_hash = block.vtx[i].calc_sha256(with_witness=True) - self.shortids.append(calculate_shortid(k0, k1, tx_hash)) - - def __repr__(self): - return "HeaderAndShortIDs(header=%s, nonce=%d, shortids=%s, prefilledtxn=%s" % (repr(self.header), self.nonce, repr(self.shortids), repr(self.prefilled_txn)) - - -class BlockTransactionsRequest(): - - def __init__(self, blockhash=0, indexes = None): - self.blockhash = blockhash - self.indexes = indexes if indexes != None else [] - - def deserialize(self, f): - self.blockhash = deser_uint256(f) - indexes_length = deser_compact_size(f) - for i in range(indexes_length): - self.indexes.append(deser_compact_size(f)) - - def serialize(self): - r = b"" - r += ser_uint256(self.blockhash) - r += ser_compact_size(len(self.indexes)) - for x in self.indexes: - r += ser_compact_size(x) - return r - - # helper to set the differentially encoded indexes from absolute ones - def from_absolute(self, absolute_indexes): - self.indexes = [] - last_index = -1 - for x in absolute_indexes: - self.indexes.append(x-last_index-1) - last_index = x - - def to_absolute(self): - absolute_indexes = [] - last_index = -1 - for x in self.indexes: - absolute_indexes.append(x+last_index+1) - last_index = absolute_indexes[-1] - return absolute_indexes - - def __repr__(self): - return "BlockTransactionsRequest(hash=%064x indexes=%s)" % (self.blockhash, repr(self.indexes)) - - -class BlockTransactions(): - - def __init__(self, blockhash=0, transactions = None): - self.blockhash = blockhash - self.transactions = transactions if transactions != None else [] - - def deserialize(self, f): - self.blockhash = deser_uint256(f) - self.transactions = deser_vector(f, CTransaction) - - def serialize(self, with_witness=False): - r = b"" - r += ser_uint256(self.blockhash) - if with_witness: - r += ser_vector(self.transactions, "serialize_with_witness") - else: - r += ser_vector(self.transactions) - return r - - def __repr__(self): - return "BlockTransactions(hash=%064x transactions=%s)" % (self.blockhash, repr(self.transactions)) - - -# Objects that correspond to messages on the wire -class msg_version(): - command = b"version" - - def __init__(self): - self.nVersion = MY_VERSION - self.nServices = NODE_NETWORK | NODE_WITNESS - self.nTime = int(time.time()) - self.addrTo = CAddress() - self.addrFrom = CAddress() - self.nNonce = random.getrandbits(64) - self.strSubVer = MY_SUBVERSION - self.nStartingHeight = -1 - self.nRelay = MY_RELAY - - def deserialize(self, f): - self.nVersion = struct.unpack("<i", f.read(4))[0] - if self.nVersion == 10300: - self.nVersion = 300 - self.nServices = struct.unpack("<Q", f.read(8))[0] - self.nTime = struct.unpack("<q", f.read(8))[0] - self.addrTo = CAddress() - self.addrTo.deserialize(f) - - if self.nVersion >= 106: - self.addrFrom = CAddress() - self.addrFrom.deserialize(f) - self.nNonce = struct.unpack("<Q", f.read(8))[0] - self.strSubVer = deser_string(f) - else: - self.addrFrom = None - self.nNonce = None - self.strSubVer = None - self.nStartingHeight = None - - if self.nVersion >= 209: - self.nStartingHeight = struct.unpack("<i", f.read(4))[0] - else: - self.nStartingHeight = None - - if self.nVersion >= 70001: - # Relay field is optional for version 70001 onwards - try: - self.nRelay = struct.unpack("<b", f.read(1))[0] - except: - self.nRelay = 0 - else: - self.nRelay = 0 - - def serialize(self): - r = b"" - r += struct.pack("<i", self.nVersion) - r += struct.pack("<Q", self.nServices) - r += struct.pack("<q", self.nTime) - r += self.addrTo.serialize() - r += self.addrFrom.serialize() - r += struct.pack("<Q", self.nNonce) - r += ser_string(self.strSubVer) - r += struct.pack("<i", self.nStartingHeight) - r += struct.pack("<b", self.nRelay) - return r - - def __repr__(self): - return 'msg_version(nVersion=%i nServices=%i nTime=%s addrTo=%s addrFrom=%s nNonce=0x%016X strSubVer=%s nStartingHeight=%i nRelay=%i)' \ - % (self.nVersion, self.nServices, time.ctime(self.nTime), - repr(self.addrTo), repr(self.addrFrom), self.nNonce, - self.strSubVer, self.nStartingHeight, self.nRelay) - - -class msg_verack(): - command = b"verack" - - def __init__(self): - pass - - def deserialize(self, f): - pass - - def serialize(self): - return b"" - - def __repr__(self): - return "msg_verack()" - - -class msg_addr(): - command = b"addr" - - def __init__(self): - self.addrs = [] - - def deserialize(self, f): - self.addrs = deser_vector(f, CAddress) - - def serialize(self): - return ser_vector(self.addrs) - - def __repr__(self): - return "msg_addr(addrs=%s)" % (repr(self.addrs)) - - -class msg_inv(): - command = b"inv" - - def __init__(self, inv=None): - if inv is None: - self.inv = [] - else: - self.inv = inv - - def deserialize(self, f): - self.inv = deser_vector(f, CInv) - - def serialize(self): - return ser_vector(self.inv) - - def __repr__(self): - return "msg_inv(inv=%s)" % (repr(self.inv)) - - -class msg_getdata(): - command = b"getdata" - - def __init__(self, inv=None): - self.inv = inv if inv != None else [] - - def deserialize(self, f): - self.inv = deser_vector(f, CInv) - - def serialize(self): - return ser_vector(self.inv) - - def __repr__(self): - return "msg_getdata(inv=%s)" % (repr(self.inv)) - - -class msg_getblocks(): - command = b"getblocks" - - def __init__(self): - self.locator = CBlockLocator() - self.hashstop = 0 - - def deserialize(self, f): - self.locator = CBlockLocator() - self.locator.deserialize(f) - self.hashstop = deser_uint256(f) - - def serialize(self): - r = b"" - r += self.locator.serialize() - r += ser_uint256(self.hashstop) - return r - - def __repr__(self): - return "msg_getblocks(locator=%s hashstop=%064x)" \ - % (repr(self.locator), self.hashstop) - - -class msg_tx(): - command = b"tx" - - def __init__(self, tx=CTransaction()): - self.tx = tx - - def deserialize(self, f): - self.tx.deserialize(f) - - def serialize(self): - return self.tx.serialize_without_witness() - - def __repr__(self): - return "msg_tx(tx=%s)" % (repr(self.tx)) - -class msg_witness_tx(msg_tx): - - def serialize(self): - return self.tx.serialize_with_witness() - - -class msg_block(): - command = b"block" - - def __init__(self, block=None): - if block is None: - self.block = CBlock() - else: - self.block = block - - def deserialize(self, f): - self.block.deserialize(f) - - def serialize(self): - return self.block.serialize() - - def __repr__(self): - return "msg_block(block=%s)" % (repr(self.block)) - -# for cases where a user needs tighter control over what is sent over the wire -# note that the user must supply the name of the command, and the data -class msg_generic(): - def __init__(self, command, data=None): - self.command = command - self.data = data - - def serialize(self): - return self.data - - def __repr__(self): - return "msg_generic()" - -class msg_witness_block(msg_block): - - def serialize(self): - r = self.block.serialize(with_witness=True) - return r - -class msg_getaddr(): - command = b"getaddr" - - def __init__(self): - pass - - def deserialize(self, f): - pass - - def serialize(self): - return b"" - - def __repr__(self): - return "msg_getaddr()" - - -class msg_ping(): - command = b"ping" - - def __init__(self, nonce=0): - self.nonce = nonce - - def deserialize(self, f): - self.nonce = struct.unpack("<Q", f.read(8))[0] - - def serialize(self): - r = b"" - r += struct.pack("<Q", self.nonce) - return r - - def __repr__(self): - return "msg_ping(nonce=%08x)" % self.nonce - - -class msg_pong(): - command = b"pong" - - def __init__(self, nonce=0): - self.nonce = nonce - - def deserialize(self, f): - self.nonce = struct.unpack("<Q", f.read(8))[0] - - def serialize(self): - r = b"" - r += struct.pack("<Q", self.nonce) - return r - - def __repr__(self): - return "msg_pong(nonce=%08x)" % self.nonce - - -class msg_mempool(): - command = b"mempool" - - def __init__(self): - pass - - def deserialize(self, f): - pass - - def serialize(self): - return b"" - - def __repr__(self): - return "msg_mempool()" - -class msg_sendheaders(): - command = b"sendheaders" - - def __init__(self): - pass - - def deserialize(self, f): - pass - - def serialize(self): - return b"" - - def __repr__(self): - return "msg_sendheaders()" - - -# getheaders message has -# number of entries -# vector of hashes -# hash_stop (hash of last desired block header, 0 to get as many as possible) -class msg_getheaders(): - command = b"getheaders" - - def __init__(self): - self.locator = CBlockLocator() - self.hashstop = 0 - - def deserialize(self, f): - self.locator = CBlockLocator() - self.locator.deserialize(f) - self.hashstop = deser_uint256(f) - - def serialize(self): - r = b"" - r += self.locator.serialize() - r += ser_uint256(self.hashstop) - return r - - def __repr__(self): - return "msg_getheaders(locator=%s, stop=%064x)" \ - % (repr(self.locator), self.hashstop) - - -# headers message has -# <count> <vector of block headers> -class msg_headers(): - command = b"headers" - - def __init__(self, headers=None): - self.headers = headers if headers is not None else [] - - def deserialize(self, f): - # comment in bitcoind indicates these should be deserialized as blocks - blocks = deser_vector(f, CBlock) - for x in blocks: - self.headers.append(CBlockHeader(x)) - - def serialize(self): - blocks = [CBlock(x) for x in self.headers] - return ser_vector(blocks) - - def __repr__(self): - return "msg_headers(headers=%s)" % repr(self.headers) - - -class msg_reject(): - command = b"reject" - REJECT_MALFORMED = 1 - - def __init__(self): - self.message = b"" - self.code = 0 - self.reason = b"" - self.data = 0 - - def deserialize(self, f): - self.message = deser_string(f) - self.code = struct.unpack("<B", f.read(1))[0] - self.reason = deser_string(f) - if (self.code != self.REJECT_MALFORMED and - (self.message == b"block" or self.message == b"tx")): - self.data = deser_uint256(f) - - def serialize(self): - r = ser_string(self.message) - r += struct.pack("<B", self.code) - r += ser_string(self.reason) - if (self.code != self.REJECT_MALFORMED and - (self.message == b"block" or self.message == b"tx")): - r += ser_uint256(self.data) - return r - - def __repr__(self): - return "msg_reject: %s %d %s [%064x]" \ - % (self.message, self.code, self.reason, self.data) - -class msg_feefilter(): - command = b"feefilter" - - def __init__(self, feerate=0): - self.feerate = feerate - - def deserialize(self, f): - self.feerate = struct.unpack("<Q", f.read(8))[0] - - def serialize(self): - r = b"" - r += struct.pack("<Q", self.feerate) - return r - - def __repr__(self): - return "msg_feefilter(feerate=%08x)" % self.feerate - -class msg_sendcmpct(): - command = b"sendcmpct" - - def __init__(self): - self.announce = False - self.version = 1 - - def deserialize(self, f): - self.announce = struct.unpack("<?", f.read(1))[0] - self.version = struct.unpack("<Q", f.read(8))[0] - - def serialize(self): - r = b"" - r += struct.pack("<?", self.announce) - r += struct.pack("<Q", self.version) - return r - - def __repr__(self): - return "msg_sendcmpct(announce=%s, version=%lu)" % (self.announce, self.version) - -class msg_cmpctblock(): - command = b"cmpctblock" - - def __init__(self, header_and_shortids = None): - self.header_and_shortids = header_and_shortids - - def deserialize(self, f): - self.header_and_shortids = P2PHeaderAndShortIDs() - self.header_and_shortids.deserialize(f) - - def serialize(self): - r = b"" - r += self.header_and_shortids.serialize() - return r - - def __repr__(self): - return "msg_cmpctblock(HeaderAndShortIDs=%s)" % repr(self.header_and_shortids) - -class msg_getblocktxn(): - command = b"getblocktxn" - - def __init__(self): - self.block_txn_request = None - - def deserialize(self, f): - self.block_txn_request = BlockTransactionsRequest() - self.block_txn_request.deserialize(f) - - def serialize(self): - r = b"" - r += self.block_txn_request.serialize() - return r - - def __repr__(self): - return "msg_getblocktxn(block_txn_request=%s)" % (repr(self.block_txn_request)) - -class msg_blocktxn(): - command = b"blocktxn" - - def __init__(self): - self.block_transactions = BlockTransactions() - - def deserialize(self, f): - self.block_transactions.deserialize(f) - - def serialize(self): - r = b"" - r += self.block_transactions.serialize() - return r - - def __repr__(self): - return "msg_blocktxn(block_transactions=%s)" % (repr(self.block_transactions)) - -class msg_witness_blocktxn(msg_blocktxn): - def serialize(self): - r = b"" - r += self.block_transactions.serialize(with_witness=True) - return r +MESSAGEMAP = { + b"addr": msg_addr, + b"block": msg_block, + b"blocktxn": msg_blocktxn, + b"cmpctblock": msg_cmpctblock, + b"feefilter": msg_feefilter, + b"getaddr": msg_getaddr, + b"getblocks": msg_getblocks, + b"getblocktxn": msg_getblocktxn, + b"getdata": msg_getdata, + b"getheaders": msg_getheaders, + b"headers": msg_headers, + b"inv": msg_inv, + b"mempool": msg_mempool, + b"ping": msg_ping, + b"pong": msg_pong, + b"reject": msg_reject, + b"sendcmpct": msg_sendcmpct, + b"sendheaders": msg_sendheaders, + b"tx": msg_tx, + b"verack": msg_verack, + b"version": msg_version, +} + +MAGIC_BYTES = { + "mainnet": b"\xf9\xbe\xb4\xd9", # mainnet + "testnet3": b"\x0b\x11\x09\x07", # testnet3 + "regtest": b"\xfa\xbf\xb5\xda", # regtest +} class NodeConnCB(): """Callback and helper functions for P2P connection to a bitcoind node. @@ -1486,34 +199,6 @@ class NodeConn(asyncore.dispatcher): """The actual NodeConn class This class provides an interface for a p2p connection to a specified node.""" - messagemap = { - b"version": msg_version, - b"verack": msg_verack, - b"addr": msg_addr, - b"inv": msg_inv, - b"getdata": msg_getdata, - b"getblocks": msg_getblocks, - b"tx": msg_tx, - b"block": msg_block, - b"getaddr": msg_getaddr, - b"ping": msg_ping, - b"pong": msg_pong, - b"headers": msg_headers, - b"getheaders": msg_getheaders, - b"reject": msg_reject, - b"mempool": msg_mempool, - b"feefilter": msg_feefilter, - b"sendheaders": msg_sendheaders, - b"sendcmpct": msg_sendcmpct, - b"cmpctblock": msg_cmpctblock, - b"getblocktxn": msg_getblocktxn, - b"blocktxn": msg_blocktxn - } - MAGIC_BYTES = { - "mainnet": b"\xf9\xbe\xb4\xd9", # mainnet - "testnet3": b"\x0b\x11\x09\x07", # testnet3 - "regtest": b"\xfa\xbf\xb5\xda", # regtest - } def __init__(self, dstaddr, dstport, callback, net="regtest", services=NODE_NETWORK|NODE_WITNESS, send_version=True): asyncore.dispatcher.__init__(self, map=mininode_socket_map) @@ -1547,6 +232,8 @@ class NodeConn(asyncore.dispatcher): except: self.handle_close() + # Connection and disconnection methods + def handle_connect(self): if self.state != "connected": logger.debug("Connected & Listening: %s:%d" % (self.dstaddr, self.dstport)) @@ -1564,44 +251,30 @@ class NodeConn(asyncore.dispatcher): pass self.cb.on_close(self) + def disconnect_node(self): + """ Disconnect the p2p connection. + + Called by the test logic thread. Causes the p2p connection + to be disconnected on the next iteration of the asyncore loop.""" + self.disconnect = True + + # Socket read methods + + def readable(self): + return True + def handle_read(self): t = self.recv(8192) if len(t) > 0: self.recvbuf += t self.got_data() - def readable(self): - return True - - def writable(self): - with mininode_lock: - pre_connection = self.state == "connecting" - length = len(self.sendbuf) - return (length > 0 or pre_connection) - - def handle_write(self): - with mininode_lock: - # asyncore does not expose socket connection, only the first read/write - # event, thus we must check connection manually here to know when we - # actually connect - if self.state == "connecting": - self.handle_connect() - if not self.writable(): - return - - try: - sent = self.send(self.sendbuf) - except: - self.handle_close() - return - self.sendbuf = self.sendbuf[sent:] - def got_data(self): try: while True: if len(self.recvbuf) < 4: return - if self.recvbuf[:4] != self.MAGIC_BYTES[self.network]: + if self.recvbuf[:4] != MAGIC_BYTES[self.network]: raise ValueError("got garbage %s" % repr(self.recvbuf)) if len(self.recvbuf) < 4 + 12 + 4 + 4: return @@ -1616,23 +289,54 @@ class NodeConn(asyncore.dispatcher): if checksum != h[:4]: raise ValueError("got bad checksum " + repr(self.recvbuf)) self.recvbuf = self.recvbuf[4+12+4+4+msglen:] - if command not in self.messagemap: + if command not in MESSAGEMAP: raise ValueError("Received unknown command from %s:%d: '%s' %s" % (self.dstaddr, self.dstport, command, repr(msg))) f = BytesIO(msg) - t = self.messagemap[command]() + t = MESSAGEMAP[command]() t.deserialize(f) self.got_message(t) except Exception as e: logger.exception('Error reading message:', repr(e)) raise + def got_message(self, message): + if self.last_sent + 30 * 60 < time.time(): + self.send_message(MESSAGEMAP[b'ping']()) + self._log_message("receive", message) + self.cb.deliver(self, message) + + # Socket write methods + + def writable(self): + with mininode_lock: + pre_connection = self.state == "connecting" + length = len(self.sendbuf) + return (length > 0 or pre_connection) + + def handle_write(self): + with mininode_lock: + # asyncore does not expose socket connection, only the first read/write + # event, thus we must check connection manually here to know when we + # actually connect + if self.state == "connecting": + self.handle_connect() + if not self.writable(): + return + + try: + sent = self.send(self.sendbuf) + except: + self.handle_close() + return + self.sendbuf = self.sendbuf[sent:] + def send_message(self, message, pushbuf=False): if self.state != "connected" and not pushbuf: raise IOError('Not connected, no pushbuf') self._log_message("send", message) command = message.command data = message.serialize() - tmsg = self.MAGIC_BYTES[self.network] + tmsg = MAGIC_BYTES[self.network] tmsg += command tmsg += b"\x00" * (12 - len(command)) tmsg += struct.pack("<I", len(data)) @@ -1651,11 +355,7 @@ class NodeConn(asyncore.dispatcher): self.sendbuf += tmsg self.last_sent = time.time() - def got_message(self, message): - if self.last_sent + 30 * 60 < time.time(): - self.send_message(self.messagemap[b'ping']()) - self._log_message("receive", message) - self.cb.deliver(self, message) + # Class utility methods def _log_message(self, direction, msg): if direction == "send": @@ -1667,9 +367,19 @@ class NodeConn(asyncore.dispatcher): log_message += "... (msg truncated)" logger.debug(log_message) - def disconnect_node(self): - self.disconnect = True +# Keep our own socket map for asyncore, so that we can track disconnects +# ourselves (to workaround an issue with closing an asyncore socket when +# using select) +mininode_socket_map = dict() + +# One lock for synchronizing all data access between the networking thread (see +# NetworkThread below) and the thread running the test logic. For simplicity, +# NodeConn acquires this lock whenever delivering a message to a NodeConnCB, +# and whenever adding anything to the send buffer (in send_message()). This +# lock should be acquired in the thread running the test logic to synchronize +# access to any data shared with the NodeConnCB or NodeConn. +mininode_lock = RLock() class NetworkThread(Thread): def run(self): @@ -1681,6 +391,6 @@ class NetworkThread(Thread): for fd, obj in mininode_socket_map.items(): if obj.disconnect: disconnected.append(obj) - [ obj.handle_close() for obj in disconnected ] + [obj.handle_close() for obj in disconnected] asyncore.loop(0.1, use_poll=True, map=mininode_socket_map, count=1) logger.debug("Network thread closing") diff --git a/test/functional/test_framework/test_framework.py b/test/functional/test_framework/test_framework.py index 8df50474f3..4590b4c650 100755 --- a/test/functional/test_framework/test_framework.py +++ b/test/functional/test_framework/test_framework.py @@ -432,7 +432,7 @@ class BitcoinTestFramework(): self.disable_mocktime() for i in range(MAX_NODES): os.remove(log_filename(self.options.cachedir, i, "debug.log")) - os.remove(log_filename(self.options.cachedir, i, "db.log")) + os.remove(log_filename(self.options.cachedir, i, "wallets/db.log")) os.remove(log_filename(self.options.cachedir, i, "peers.dat")) os.remove(log_filename(self.options.cachedir, i, "fee_estimates.dat")) diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py index ca36426a0a..d953e1585c 100755 --- a/test/functional/test_runner.py +++ b/test/functional/test_runner.py @@ -300,7 +300,11 @@ def run_tests(test_list, src_dir, build_dir, exeext, tmpdir, jobs=1, enable_cove if len(test_list) > 1 and jobs > 1: # Populate cache - subprocess.check_output([tests_dir + 'create_cache.py'] + flags + ["--tmpdir=%s/cache" % tmpdir]) + try: + subprocess.check_output([tests_dir + 'create_cache.py'] + flags + ["--tmpdir=%s/cache" % tmpdir]) + except Exception as e: + print(e.output) + raise e #Run Tests job_queue = TestHandler(jobs, tests_dir, tmpdir, test_list, flags) diff --git a/test/functional/wallet-hd.py b/test/functional/wallet-hd.py index 9b6ce68609..d21656a971 100755 --- a/test/functional/wallet-hd.py +++ b/test/functional/wallet-hd.py @@ -73,7 +73,7 @@ class WalletHDTest(BitcoinTestFramework): # otherwise node1 would auto-recover all funds in flag the keypool keys as used shutil.rmtree(os.path.join(tmpdir, "node1/regtest/blocks")) shutil.rmtree(os.path.join(tmpdir, "node1/regtest/chainstate")) - shutil.copyfile(os.path.join(tmpdir, "hd.bak"), os.path.join(tmpdir, "node1/regtest/wallet.dat")) + shutil.copyfile(os.path.join(tmpdir, "hd.bak"), os.path.join(tmpdir, "node1/regtest/wallets/wallet.dat")) self.start_node(1) # Assert that derivation is deterministic diff --git a/test/functional/walletbackup.py b/test/functional/walletbackup.py index 85a149793e..8ef5620cd8 100755 --- a/test/functional/walletbackup.py +++ b/test/functional/walletbackup.py @@ -90,9 +90,9 @@ class WalletBackupTest(BitcoinTestFramework): self.stop_node(2) def erase_three(self): - os.remove(self.options.tmpdir + "/node0/regtest/wallet.dat") - os.remove(self.options.tmpdir + "/node1/regtest/wallet.dat") - os.remove(self.options.tmpdir + "/node2/regtest/wallet.dat") + os.remove(self.options.tmpdir + "/node0/regtest/wallets/wallet.dat") + os.remove(self.options.tmpdir + "/node1/regtest/wallets/wallet.dat") + os.remove(self.options.tmpdir + "/node2/regtest/wallets/wallet.dat") def run_test(self): self.log.info("Generating initial blockchain") @@ -154,9 +154,9 @@ class WalletBackupTest(BitcoinTestFramework): shutil.rmtree(self.options.tmpdir + "/node2/regtest/chainstate") # Restore wallets from backup - shutil.copyfile(tmpdir + "/node0/wallet.bak", tmpdir + "/node0/regtest/wallet.dat") - shutil.copyfile(tmpdir + "/node1/wallet.bak", tmpdir + "/node1/regtest/wallet.dat") - shutil.copyfile(tmpdir + "/node2/wallet.bak", tmpdir + "/node2/regtest/wallet.dat") + shutil.copyfile(tmpdir + "/node0/wallet.bak", tmpdir + "/node0/regtest/wallets/wallet.dat") + shutil.copyfile(tmpdir + "/node1/wallet.bak", tmpdir + "/node1/regtest/wallets/wallet.dat") + shutil.copyfile(tmpdir + "/node2/wallet.bak", tmpdir + "/node2/regtest/wallets/wallet.dat") self.log.info("Re-starting nodes") self.start_three() @@ -192,10 +192,10 @@ class WalletBackupTest(BitcoinTestFramework): # Backup to source wallet file must fail sourcePaths = [ - tmpdir + "/node0/regtest/wallet.dat", - tmpdir + "/node0/./regtest/wallet.dat", - tmpdir + "/node0/regtest/", - tmpdir + "/node0/regtest"] + tmpdir + "/node0/regtest/wallets/wallet.dat", + tmpdir + "/node0/./regtest/wallets/wallet.dat", + tmpdir + "/node0/regtest/wallets/", + tmpdir + "/node0/regtest/wallets"] for sourcePath in sourcePaths: assert_raises_rpc_error(-4, "backup failed", self.nodes[0].backupwallet, sourcePath) |