diff options
74 files changed, 1542 insertions, 1273 deletions
diff --git a/.style.yapf b/.style.yapf index fe6227baf6..69d8c6aee4 100644 --- a/.style.yapf +++ b/.style.yapf @@ -55,7 +55,7 @@ blank_line_before_nested_class_or_def=False coalesce_brackets=False # The column limit. -column_limit=79 +column_limit=160 # The style for continuation alignment. Possible values are: # diff --git a/.travis.yml b/.travis.yml index ef8da16936..21ba5461dd 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,3 +1,30 @@ +# The test build matrix (stage: test) is constructed to test a wide range of +# configurations, rather than a single pass/fail. This helps to catch build +# failures and logic errors that present on platforms other than the ones the +# author has tested. +# +# Some builders use the dependency-generator in `./depends`, rather than using +# apt-get to install build dependencies. This guarantees that the tester is +# using the same versions as Gitian, so the build results are nearly identical +# to what would be found in a final release. +# +# In order to avoid rebuilding all dependencies for each build, the binaries +# are cached and re-used when possible. Changes in the dependency-generator +# will trigger cache-invalidation and rebuilds as necessary. +# +# These caches can be manually removed if necessary. This is one of the very +# few manual operations that is possible with Travis, and it can be done by a +# Bitcoin Core GitHub member via the Travis web interface [0]. +# +# Travis CI uploads the cache after the script phase of the build [1]. +# However, the build is terminated without saving the chache if it takes over +# 50 minutes [2]. Thus, if we spent too much time in early build stages, fail +# with an error and save the cache. +# +# [0] https://travis-ci.org/bitcoin/bitcoin/caches +# [1] https://docs.travis-ci.com/user/caching/#build-phases +# [2] https://docs.travis-ci.com/user/customizing-the-build#build-timeouts + dist: xenial os: linux language: minimal @@ -26,6 +53,7 @@ env: - SDK_URL=https://bitcoincore.org/depends-sources/sdks - WINEDEBUG=fixme-all - DOCKER_PACKAGES="build-essential libtool autotools-dev automake pkg-config bsdmainutils curl git ca-certificates ccache" + - CACHE_ERR_MSG="Error! Initial build successful, but not enough time remains to run later build stages and tests. Please manually re-run this job by using the travis restart button or asking a bitcoin maintainer to restart. The next run should not time out because the build cache has been saved." before_install: - set -o errexit; source .travis/test_03_before_install.sh install: @@ -33,8 +61,11 @@ install: before_script: - set -o errexit; source .travis/test_05_before_script.sh script: - - if [ $SECONDS -gt 1200 ]; then set +o errexit; echo "Travis early exit to cache current state"; false; else set -o errexit; source .travis/test_06_script_a.sh; fi - - if [ $SECONDS -gt 1800 ]; then set +o errexit; echo "Travis early exit to cache current state"; false; else set -o errexit; source .travis/test_06_script_b.sh; fi + - export CONTINUE=1 + - if [ $SECONDS -gt 1200 ]; then export CONTINUE=0; fi # Likely the depends build took very long + - if [ $CONTINUE = "1" ]; then set -o errexit; source .travis/test_06_script_a.sh; else set +o errexit; echo "$CACHE_ERR_MSG"; false; fi + - if [ $SECONDS -gt 1800 ]; then export CONTINUE=0; fi # Likely the build took very long + - if [ $CONTINUE = "1" ]; then set -o errexit; source .travis/test_06_script_b.sh; else set +o errexit; echo "$CACHE_ERR_MSG"; false; fi after_script: - echo $TRAVIS_COMMIT_RANGE - echo $TRAVIS_COMMIT_LOG diff --git a/.travis/lint_06_script.sh b/.travis/lint_06_script.sh index 701e6d8005..eeebc80ec0 100755 --- a/.travis/lint_06_script.sh +++ b/.travis/lint_06_script.sh @@ -20,6 +20,6 @@ test/lint/lint-all.sh if [ "$TRAVIS_REPO_SLUG" = "bitcoin/bitcoin" -a "$TRAVIS_EVENT_TYPE" = "cron" ]; then git log --merges --before="2 days ago" -1 --format='%H' > ./contrib/verify-commits/trusted-sha512-root-commit - while read -r LINE; do travis_retry gpg --keyserver hkp://subset.pool.sks-keyservers.net --recv-keys $LINE; done < contrib/verify-commits/trusted-keys && - travis_wait 50 contrib/verify-commits/verify-commits.py --clean-merge=2; + travis_retry gpg --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys $(<contrib/verify-commits/trusted-keys) && + ./contrib/verify-commits/verify-commits.py --clean-merge=2; fi diff --git a/contrib/devtools/github-merge.py b/contrib/devtools/github-merge.py index ab834b7623..41a640b464 100755 --- a/contrib/devtools/github-merge.py +++ b/contrib/devtools/github-merge.py @@ -47,13 +47,12 @@ def git_config_get(option, default=None): except subprocess.CalledProcessError: return default -def retrieve_pr_info(repo,pull,ghtoken): +def retrieve_json(req, ghtoken): ''' - Retrieve pull request information from github. - Return None if no title can be found, or an error happens. + Retrieve json from github. + Return None if an error happens. ''' try: - req = Request("https://api.github.com/repos/"+repo+"/pulls/"+pull) if ghtoken is not None: req.add_header('Authorization', 'token ' + ghtoken) result = urlopen(req) @@ -69,6 +68,18 @@ def retrieve_pr_info(repo,pull,ghtoken): print('Warning: unable to retrieve pull information from github: %s' % e) return None +def retrieve_pr_info(repo,pull,ghtoken): + req = Request("https://api.github.com/repos/"+repo+"/pulls/"+pull) + return retrieve_json(req,ghtoken) + +def retrieve_pr_comments(repo,pull,ghtoken): + req = Request("https://api.github.com/repos/"+repo+"/issues/"+pull+"/comments") + return retrieve_json(req,ghtoken) + +def retrieve_pr_reviews(repo,pull,ghtoken): + req = Request("https://api.github.com/repos/"+repo+"/pulls/"+pull+"/reviews") + return retrieve_json(req,ghtoken) + def ask_prompt(text): print(text,end=" ",file=stderr) stderr.flush() @@ -133,6 +144,16 @@ def tree_sha512sum(commit='HEAD'): raise IOError('Non-zero return value executing git cat-file') return overall.hexdigest() +def get_acks_from_comments(head_commit, comments): + assert len(head_commit) == 6 + ack_str ='\n\nACKs for commit {}:\n'.format(head_commit) + for c in comments: + review = [l for l in c['body'].split('\r\n') if 'ACK' in l and head_commit in l] + if review: + ack_str += ' {}:\n'.format(c['user']['login']) + ack_str += ' {}\n'.format(review[0]) + return ack_str + def print_merge_details(pull, title, branch, base_branch, head_branch): print('%s#%s%s %s %sinto %s%s' % (ATTR_RESET+ATTR_PR,pull,ATTR_RESET,title,ATTR_RESET+ATTR_PR,branch,ATTR_RESET)) subprocess.check_call([GIT,'log','--graph','--topo-order','--pretty=format:'+COMMIT_FORMAT,base_branch+'..'+head_branch]) @@ -185,6 +206,9 @@ def main(): info = retrieve_pr_info(repo,pull,ghtoken) if info is None: sys.exit(1) + comments = retrieve_pr_comments(repo,pull,ghtoken) + retrieve_pr_reviews(repo,pull,ghtoken) + if comments is None: + sys.exit(1) title = info['title'].strip() body = info['body'].strip() # precedence order for destination branch argument: @@ -238,6 +262,7 @@ def main(): message = firstline + '\n\n' message += subprocess.check_output([GIT,'log','--no-merges','--topo-order','--pretty=format:%h %s (%an)',base_branch+'..'+head_branch]).decode('utf-8') message += '\n\nPull request description:\n\n ' + body.replace('\n', '\n ') + '\n' + message += get_acks_from_comments(head_commit=subprocess.check_output([GIT,'log','-1','--pretty=format:%H',head_branch]).decode('utf-8')[:6], comments=comments) try: subprocess.check_call([GIT,'merge','-q','--commit','--no-edit','--no-ff','-m',message.encode('utf-8'),head_branch]) except subprocess.CalledProcessError: diff --git a/contrib/verify-commits/verify-commits.py b/contrib/verify-commits/verify-commits.py index 6bbed01073..255ce75092 100755 --- a/contrib/verify-commits/verify-commits.py +++ b/contrib/verify-commits/verify-commits.py @@ -5,6 +5,7 @@ """Verify commits against a trusted keys list.""" import argparse import hashlib +import logging import os import subprocess import sys @@ -66,6 +67,11 @@ def tree_sha512sum(commit='HEAD'): return overall.hexdigest() def main(): + + # Enable debug logging if running in CI + if 'CI' in os.environ and os.environ['CI'].lower() == "true": + logging.getLogger().setLevel(logging.DEBUG) + # Parse arguments parser = argparse.ArgumentParser(usage='%(prog)s [options] [commit id]') parser.add_argument('--disable-tree-check', action='store_false', dest='verify_tree', help='disable SHA-512 tree check') @@ -95,6 +101,10 @@ def main(): # Iterate through commits while True: + + # Log a message to prevent Travis from timing out + logging.debug("verify-commits: [in-progress] processing commit {}".format(current_commit[:8])) + if current_commit == verified_root: print('There is a valid path from "{}" to {} where all commits are signed!'.format(initial_commit, verified_root)) sys.exit(0) diff --git a/contrib/windeploy/win-codesign.cert b/contrib/windeploy/win-codesign.cert index 200b30a3f0..5bc5dc5809 100644 --- a/contrib/windeploy/win-codesign.cert +++ b/contrib/windeploy/win-codesign.cert @@ -1,99 +1,100 @@ -----BEGIN CERTIFICATE----- -MIIFTTCCBDWgAwIBAgIRALlW05RLwG2hMQMX5d/o5J8wDQYJKoZIhvcNAQELBQAw -fTELMAkGA1UEBhMCR0IxGzAZBgNVBAgTEkdyZWF0ZXIgTWFuY2hlc3RlcjEQMA4G -A1UEBxMHU2FsZm9yZDEaMBgGA1UEChMRQ09NT0RPIENBIExpbWl0ZWQxIzAhBgNV -BAMTGkNPTU9ETyBSU0EgQ29kZSBTaWduaW5nIENBMB4XDTE2MDIwMzAwMDAwMFoX -DTE5MDMwNTIzNTk1OVowgbUxCzAJBgNVBAYTAlVTMQ4wDAYDVQQRDAU5ODEwNDEL -MAkGA1UECAwCV0ExEDAOBgNVBAcMB1NlYXR0bGUxEDAOBgNVBAkMB1N0ZSAzMDAx -FzAVBgNVBAkMDjcxIENvbHVtYmlhIFN0MSUwIwYDVQQKDBxUaGUgQml0Y29pbiBG -b3VuZGF0aW9uLCBJbmMuMSUwIwYDVQQDDBxUaGUgQml0Y29pbiBGb3VuZGF0aW9u -LCBJbmMuMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAw37Vrv9Gbku0 -+kuV0t89TuyxtAcmT7QE4GcwESKKjmkxfzD9a0qlhqk8GfQ+fw4DHNN+nLKNv7xB -bk6aS7J2v2DcXkOjrP99P9jqgTkp7MC04VtG3OqVRGB+gum0pptRovYZUQXIdkY7 -GJOok/NDagwKiiUe2V2meZ7UctsZNvYeilQdTgKIIhrMB9NowCOhT8ocVL4Ki55/ -l7hukJn3fueCM3fHTwY2/1gaGsOHoCkFRsD7vokjAVpiY+8rUgvHjb0gxgojiVGd -6a6/F5XJwKJacvUyN4Hfc2K5lRMQjTTmo4aWNWIa0iJ3TK9BHpdSLJBqerMPvmnM -kkapS+ZTNQIDAQABo4IBjTCCAYkwHwYDVR0jBBgwFoAUKZFg/4pN+uv5pmq4z/nm -S71JzhIwHQYDVR0OBBYEFONpQ+cV82URVe+V8G57377KxxexMA4GA1UdDwEB/wQE -AwIHgDAMBgNVHRMBAf8EAjAAMBMGA1UdJQQMMAoGCCsGAQUFBwMDMBEGCWCGSAGG -+EIBAQQEAwIEEDBGBgNVHSAEPzA9MDsGDCsGAQQBsjEBAgEDAjArMCkGCCsGAQUF -BwIBFh1odHRwczovL3NlY3VyZS5jb21vZG8ubmV0L0NQUzBDBgNVHR8EPDA6MDig -NqA0hjJodHRwOi8vY3JsLmNvbW9kb2NhLmNvbS9DT01PRE9SU0FDb2RlU2lnbmlu -Z0NBLmNybDB0BggrBgEFBQcBAQRoMGYwPgYIKwYBBQUHMAKGMmh0dHA6Ly9jcnQu -Y29tb2RvY2EuY29tL0NPTU9ET1JTQUNvZGVTaWduaW5nQ0EuY3J0MCQGCCsGAQUF -BzABhhhodHRwOi8vb2NzcC5jb21vZG9jYS5jb20wDQYJKoZIhvcNAQELBQADggEB -AGnBSi9K/9rgTAyKFKrfGWSfNOwAghmsnsvpZSQ7QyoGWBFKSgCs/70kErl18oHA -g7Y8loQB1yukZmJaCa3OvGud7smn45TCh0TMf4EpP20Wxf4rMQTxwAatasHL3+vi -I+Nl5bsRZ09kWjvayqLII5upjS/yq0JfpmyGl5k2C/fIpztq0iOLvqWlXcL4+51r -cMUAfX6E6EaZQm//ikp+w2+7MEXTKguOuV3gwsrTy0DsvkZl4YDgx/FA4ImzXopv -d+3KJPLvO+OSBqUD3JPwXHnuJqGAbLBFyyCa/feGUjLlR8cxcNWLWdp4qxtoIUPG -3wTsC9YgrglS0F7FKMXlNRY= +MIIFcTCCBFmgAwIBAgIRALWcUnSOxv9FQW3xdaMDO6swDQYJKoZIhvcNAQELBQAw +fDELMAkGA1UEBhMCR0IxGzAZBgNVBAgTEkdyZWF0ZXIgTWFuY2hlc3RlcjEQMA4G +A1UEBxMHU2FsZm9yZDEYMBYGA1UEChMPU2VjdGlnbyBMaW1pdGVkMSQwIgYDVQQD +ExtTZWN0aWdvIFJTQSBDb2RlIFNpZ25pbmcgQ0EwHhcNMTkwMzI3MDAwMDAwWhcN +MjAwMzI2MjM1OTU5WjCBtDELMAkGA1UEBhMCQ0gxDTALBgNVBBEMBDgwMDUxCzAJ +BgNVBAgMAlpIMRAwDgYDVQQHDAdaw7xyaWNoMRcwFQYDVQQJDA5NYXR0ZW5nYXNz +ZSAyNzEuMCwGA1UECgwlQml0Y29pbiBDb3JlIENvZGUgU2lnbmluZyBBc3NvY2lh +dGlvbjEuMCwGA1UEAwwlQml0Y29pbiBDb3JlIENvZGUgU2lnbmluZyBBc3NvY2lh +dGlvbjCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAK04VDwiY1wxcW3E +WTTGmnbciCwETwC96DG4qcoH2PPNsVy3dfwGh0C02Qj2vL64IfwIGUFSgREvyjZk +CNhEuJO2e0nO0rKNNH5v/JO+P7/VYPZkF5a3uUz9ulmihULXioieHB/q0l6BmiJL ++cYaMVfidL9Y+IJwgiTqjnpRhv1Ik083SPsu6GcfQT9MJfY/+xse2EP0l4GfdFE6 +DRcWjiC8UHpfpGYcImzSFZZpbFbqoAyhueCl28QU4f8QAbS6BqNfaAK9MMACWDcK +eTz3C5JK6CiUxOnGIxilXhljuybFUjR4jGl5eTRpuPWk95NTTYS36q+bx/1nYelx +0n4nnDMCAwEAAaOCAbMwggGvMB8GA1UdIwQYMBaAFA7hOqhTOjHVir7Bu61nGgOF +rTQOMB0GA1UdDgQWBBRbN7ECrPCdVvh58enwy3Dix46h2jAOBgNVHQ8BAf8EBAMC +B4AwDAYDVR0TAQH/BAIwADATBgNVHSUEDDAKBggrBgEFBQcDAzARBglghkgBhvhC +AQEEBAMCBBAwQAYDVR0gBDkwNzA1BgwrBgEEAbIxAQIBAwIwJTAjBggrBgEFBQcC +ARYXaHR0cHM6Ly9zZWN0aWdvLmNvbS9DUFMwQwYDVR0fBDwwOjA4oDagNIYyaHR0 +cDovL2NybC5zZWN0aWdvLmNvbS9TZWN0aWdvUlNBQ29kZVNpZ25pbmdDQS5jcmww +cwYIKwYBBQUHAQEEZzBlMD4GCCsGAQUFBzAChjJodHRwOi8vY3J0LnNlY3RpZ28u +Y29tL1NlY3RpZ29SU0FDb2RlU2lnbmluZ0NBLmNydDAjBggrBgEFBQcwAYYXaHR0 +cDovL29jc3Auc2VjdGlnby5jb20wKwYDVR0RBCQwIoEgam9uYXNAYml0Y29pbmNv +cmVjb2Rlc2lnbmluZy5vcmcwDQYJKoZIhvcNAQELBQADggEBAF/AIXcFBWCC2Red +SHN4Cvko5mdSkDNgzjVFc+OwAJ5RdOgbERde4PnHm3Qmrnx+uMetVnmrC8Fv1Iwb +kkR0bdbWBj6lF6zMsClIN6WJEfY+qfj1qi7wyucu+3OElYRC9bm5Lf0mEHQr8lJ1 +lGvAjPh+/hmxoVNbHFMZ1Ea+BrbjVwiSznt0gzdMh0CispBZKLWCIwRwi+hFjQrw +Z7RLH8HeCJ5Ojl/OTDQqh6AylQ7l9w9KHsUt4Jqy/AnCCyAj2/6xjdwnuo3tCZwb +g/9CydiAacD/83odphEeC2iBa+0wsj9bWmyYKY7S9n0u+wm3wBfZbSVMDDPk/la1 +3qCUDLk= -----END CERTIFICATE----- -----BEGIN CERTIFICATE----- -MIIF4DCCA8igAwIBAgIQLnyHzA6TSlL+lP0ct800rzANBgkqhkiG9w0BAQwFADCB -hTELMAkGA1UEBhMCR0IxGzAZBgNVBAgTEkdyZWF0ZXIgTWFuY2hlc3RlcjEQMA4G -A1UEBxMHU2FsZm9yZDEaMBgGA1UEChMRQ09NT0RPIENBIExpbWl0ZWQxKzApBgNV -BAMTIkNPTU9ETyBSU0EgQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkwHhcNMTMwNTA5 -MDAwMDAwWhcNMjgwNTA4MjM1OTU5WjB9MQswCQYDVQQGEwJHQjEbMBkGA1UECBMS -R3JlYXRlciBNYW5jaGVzdGVyMRAwDgYDVQQHEwdTYWxmb3JkMRowGAYDVQQKExFD -T01PRE8gQ0EgTGltaXRlZDEjMCEGA1UEAxMaQ09NT0RPIFJTQSBDb2RlIFNpZ25p -bmcgQ0EwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCmmJBjd5E0f4rR -3elnMRHrzB79MR2zuWJXP5O8W+OfHiQyESdrvFGRp8+eniWzX4GoGA8dHiAwDvth -e4YJs+P9omidHCydv3Lj5HWg5TUjjsmK7hoMZMfYQqF7tVIDSzqwjiNLS2PgIpQ3 -e9V5kAoUGFEs5v7BEvAcP2FhCoyi3PbDMKrNKBh1SMF5WgjNu4xVjPfUdpA6M0ZQ -c5hc9IVKaw+A3V7Wvf2pL8Al9fl4141fEMJEVTyQPDFGy3CuB6kK46/BAW+QGiPi -XzjbxghdR7ODQfAuADcUuRKqeZJSzYcPe9hiKaR+ML0btYxytEjy4+gh+V5MYnmL -Agaff9ULAgMBAAGjggFRMIIBTTAfBgNVHSMEGDAWgBS7r34CPfqm8TyEjq3uOJjs -2TIy1DAdBgNVHQ4EFgQUKZFg/4pN+uv5pmq4z/nmS71JzhIwDgYDVR0PAQH/BAQD -AgGGMBIGA1UdEwEB/wQIMAYBAf8CAQAwEwYDVR0lBAwwCgYIKwYBBQUHAwMwEQYD -VR0gBAowCDAGBgRVHSAAMEwGA1UdHwRFMEMwQaA/oD2GO2h0dHA6Ly9jcmwuY29t -b2RvY2EuY29tL0NPTU9ET1JTQUNlcnRpZmljYXRpb25BdXRob3JpdHkuY3JsMHEG -CCsGAQUFBwEBBGUwYzA7BggrBgEFBQcwAoYvaHR0cDovL2NydC5jb21vZG9jYS5j -b20vQ09NT0RPUlNBQWRkVHJ1c3RDQS5jcnQwJAYIKwYBBQUHMAGGGGh0dHA6Ly9v -Y3NwLmNvbW9kb2NhLmNvbTANBgkqhkiG9w0BAQwFAAOCAgEAAj8COcPu+Mo7id4M -bU2x8U6ST6/COCwEzMVjEasJY6+rotcCP8xvGcM91hoIlP8l2KmIpysQGuCbsQci -GlEcOtTh6Qm/5iR0rx57FjFuI+9UUS1SAuJ1CAVM8bdR4VEAxof2bO4QRHZXavHf -WGshqknUfDdOvf+2dVRAGDZXZxHNTwLk/vPa/HUX2+y392UJI0kfQ1eD6n4gd2HI -TfK7ZU2o94VFB696aSdlkClAi997OlE5jKgfcHmtbUIgos8MbAOMTM1zB5TnWo46 -BLqioXwfy2M6FafUFRunUkcyqfS/ZEfRqh9TTjIwc8Jvt3iCnVz/RrtrIh2IC/gb -qjSm/Iz13X9ljIwxVzHQNuxHoc/Li6jvHBhYxQZ3ykubUa9MCEp6j+KjUuKOjswm -5LLY5TjCqO3GgZw1a6lYYUoKl7RLQrZVnb6Z53BtWfhtKgx/GWBfDJqIbDCsUgmQ -Fhv/K53b0CDKieoofjKOGd97SDMe12X4rsn4gxSTdn1k0I7OvjV9/3IxTZ+evR5s -L6iPDAZQ+4wns3bJ9ObXwzTijIchhmH+v1V04SF3AwpobLvkyanmz1kl63zsRQ55 -ZmjoIs2475iFTZYRPAmK0H+8KCgT+2rKVI2SXM3CZZgGns5IW9S1N5NGQXwH3c/6 -Q++6Z2H/fUnguzB9XIDj5hY5S6c= +MIIF3jCCA8agAwIBAgIQAf1tMPyjylGoG7xkDjUDLTANBgkqhkiG9w0BAQwFADCB +iDELMAkGA1UEBhMCVVMxEzARBgNVBAgTCk5ldyBKZXJzZXkxFDASBgNVBAcTC0pl +cnNleSBDaXR5MR4wHAYDVQQKExVUaGUgVVNFUlRSVVNUIE5ldHdvcmsxLjAsBgNV +BAMTJVVTRVJUcnVzdCBSU0EgQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkwHhcNMTAw +MjAxMDAwMDAwWhcNMzgwMTE4MjM1OTU5WjCBiDELMAkGA1UEBhMCVVMxEzARBgNV +BAgTCk5ldyBKZXJzZXkxFDASBgNVBAcTC0plcnNleSBDaXR5MR4wHAYDVQQKExVU +aGUgVVNFUlRSVVNUIE5ldHdvcmsxLjAsBgNVBAMTJVVTRVJUcnVzdCBSU0EgQ2Vy +dGlmaWNhdGlvbiBBdXRob3JpdHkwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIK +AoICAQCAEmUXNg7D2wiz0KxXDXbtzSfTTK1Qg2HiqiBNCS1kCdzOiZ/MPans9s/B +3PHTsdZ7NygRK0faOca8Ohm0X6a9fZ2jY0K2dvKpOyuR+OJv0OwWIJAJPuLodMkY +tJHUYmTbf6MG8YgYapAiPLz+E/CHFHv25B+O1ORRxhFnRghRy4YUVD+8M/5+bJz/ +Fp0YvVGONaanZshyZ9shZrHUm3gDwFA66Mzw3LyeTP6vBZY1H1dat//O+T23LLb2 +VN3I5xI6Ta5MirdcmrS3ID3KfyI0rn47aGYBROcBTkZTmzNg95S+UzeQc0PzMsNT +79uq/nROacdrjGCT3sTHDN/hMq7MkztReJVni+49Vv4M0GkPGw/zJSZrM233bkf6 +c0Plfg6lZrEpfDKEY1WJxA3Bk1QwGROs0303p+tdOmw1XNtB1xLaqUkL39iAigmT +Yo61Zs8liM2EuLE/pDkP2QKe6xJMlXzzawWpXhaDzLhn4ugTncxbgtNMs+1b/97l +c6wjOy0AvzVVdAlJ2ElYGn+SNuZRkg7zJn0cTRe8yexDJtC/QV9AqURE9JnnV4ee +UB9XVKg+/XRjL7FQZQnmWEIuQxpMtPAlR1n6BB6T1CZGSlCBst6+eLf8ZxXhyVeE +Hg9j1uliutZfVS7qXMYoCAQlObgOK6nyTJccBz8NUvXt7y+CDwIDAQABo0IwQDAd +BgNVHQ4EFgQUU3m/WqorSs9UgOHYm8Cd8rIDZsswDgYDVR0PAQH/BAQDAgEGMA8G +A1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQEMBQADggIBAFzUfA3P9wF9QZllDHPF +Up/L+M+ZBn8b2kMVn54CVVeWFPFSPCeHlCjtHzoBN6J2/FNQwISbxmtOuowhT6KO +VWKR82kV2LyI48SqC/3vqOlLVSoGIG1VeCkZ7l8wXEskEVX/JJpuXior7gtNn3/3 +ATiUFJVDBwn7YKnuHKsSjKCaXqeYalltiz8I+8jRRa8YFWSQEg9zKC7F4iRO/Fjs +8PRF/iKz6y+O0tlFYQXBl2+odnKPi4w2r78NBc5xjeambx9spnFixdjQg3IM8WcR +iQycE0xyNN+81XHfqnHd4blsjDwSXWXavVcStkNr/+XeTWYRUc+ZruwXtuhxkYze +Sf7dNXGiFSeUHM9h4ya7b6NnJSFd5t0dCy5oGzuCr+yDZ4XUmFF0sbmZgIn/f3gZ +XHlKYC6SQK5MNyosycdiyA5d9zZbyuAlJQG03RoHnHcAP9Dc1ew91Pq7P8yF1m9/ +qS3fuQL39ZeatTXaw2ewh0qpKJ4jjv9cJ2vhsE/zB+4ALtRZh8tSQZXq9EfX7mRB +VXyNWQKV3WKdwrnuWih0hKWbt5DHDAff9Yk2dDLWKMGwsAvgnEzDHNb842m1R0aB +L6KCq9NjRHDEjf8tM7qtj3u1cIiuPhnPQCjY/MiQu12ZIvVS5ljFH4gxQ+6IHdfG +jjxDah2nGN59PRbxYvnKkKj9 -----END CERTIFICATE----- -----BEGIN CERTIFICATE----- -MIIF2DCCA8CgAwIBAgIQTKr5yttjb+Af907YWwOGnTANBgkqhkiG9w0BAQwFADCB -hTELMAkGA1UEBhMCR0IxGzAZBgNVBAgTEkdyZWF0ZXIgTWFuY2hlc3RlcjEQMA4G -A1UEBxMHU2FsZm9yZDEaMBgGA1UEChMRQ09NT0RPIENBIExpbWl0ZWQxKzApBgNV -BAMTIkNPTU9ETyBSU0EgQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkwHhcNMTAwMTE5 -MDAwMDAwWhcNMzgwMTE4MjM1OTU5WjCBhTELMAkGA1UEBhMCR0IxGzAZBgNVBAgT -EkdyZWF0ZXIgTWFuY2hlc3RlcjEQMA4GA1UEBxMHU2FsZm9yZDEaMBgGA1UEChMR -Q09NT0RPIENBIExpbWl0ZWQxKzApBgNVBAMTIkNPTU9ETyBSU0EgQ2VydGlmaWNh -dGlvbiBBdXRob3JpdHkwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQCR -6FSS0gpWsawNJN3Fz0RndJkrN6N9I3AAcbxT38T6KhKPS38QVr2fcHK3YX/JSw8X -pz3jsARh7v8Rl8f0hj4K+j5c+ZPmNHrZFGvnnLOFoIJ6dq9xkNfs/Q36nGz637CC -9BR++b7Epi9Pf5l/tfxnQ3K9DADWietrLNPtj5gcFKt+5eNu/Nio5JIk2kNrYrhV -/erBvGy2i/MOjZrkm2xpmfh4SDBF1a3hDTxFYPwyllEnvGfDyi62a+pGx8cgoLEf -Zd5ICLqkTqnyg0Y3hOvozIFIQ2dOciqbXL1MGyiKXCJ7tKuY2e7gUYPDCUZObT6Z -+pUX2nwzV0E8jVHtC7ZcryxjGt9XyD+86V3Em69FmeKjWiS0uqlWPc9vqv9JWL7w -qP/0uK3pN/u6uPQLOvnoQ0IeidiEyxPx2bvhiWC4jChWrBQdnArncevPDt09qZah -SL0896+1DSJMwBGB7FY79tOi4lu3sgQiUpWAk2nojkxl8ZEDLXB0AuqLZxUpaVIC -u9ffUGpVRr+goyhhf3DQw6KqLCGqR84onAZFdr+CGCe01a60y1Dma/RMhnEw6abf -Fobg2P9A3fvQQoh/ozM6LlweQRGBY84YcWsr7KaKtzFcOmpH4MN5WdYgGq/yapiq -crxXStJLnbsQ/LBMQeXtHT1eKJ2czL+zUdqnR+WEUwIDAQABo0IwQDAdBgNVHQ4E -FgQUu69+Aj36pvE8hI6t7jiY7NkyMtQwDgYDVR0PAQH/BAQDAgEGMA8GA1UdEwEB -/wQFMAMBAf8wDQYJKoZIhvcNAQEMBQADggIBAArx1UaEt65Ru2yyTUEUAJNMnMvl -wFTPoCWOAvn9sKIN9SCYPBMtrFaisNZ+EZLpLrqeLppysb0ZRGxhNaKatBYSaVqM -4dc+pBroLwP0rmEdEBsqpIt6xf4FpuHA1sj+nq6PK7o9mfjYcwlYRm6mnPTXJ9OV -2jeDchzTc+CiR5kDOF3VSXkAKRzH7JsgHAckaVd4sjn8OoSgtZx8jb8uk2Intzna -FxiuvTwJaP+EmzzV1gsD41eeFPfR60/IvYcjt7ZJQ3mFXLrrkguhxuhoqEwWsRqZ -CuhTLJK7oQkYdQxlqHvLI7cawiiFwxv/0Cti76R7CZGYZ4wUAc1oBmpjIXUDgIiK -boHGhfKppC3n9KUkEEeDys30jXlYsQab5xoq2Z0B15R97QNKyvDb6KkBPvVWmcke -jkk9u+UJueBPSZI9FoJAzMxZxuY67RIuaTxslbH9qh17f4a+Hg4yRvv7E491f0yL -S0Zj/gA0QHDBw7mh3aZw4gSzQbzpgJHqZJx64SIDqZxubw5lT2yHh17zbqD5daWb -QOhTsiedSrnAdyGN/4fy3ryM7xfft0kL0fJuMAsaDk527RH89elWsn2/x20Kk4yl -0MC2Hb46TpSi125sC8KKfPog88Tk5c0NqMuRkrF8hey1FGlmDoLnzc7ILaZRfyHB -NVOFBkpdn627G190 +MIIF9TCCA92gAwIBAgIQHaJIMG+bJhjQguCWfTPTajANBgkqhkiG9w0BAQwFADCB +iDELMAkGA1UEBhMCVVMxEzARBgNVBAgTCk5ldyBKZXJzZXkxFDASBgNVBAcTC0pl +cnNleSBDaXR5MR4wHAYDVQQKExVUaGUgVVNFUlRSVVNUIE5ldHdvcmsxLjAsBgNV +BAMTJVVTRVJUcnVzdCBSU0EgQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkwHhcNMTgx +MTAyMDAwMDAwWhcNMzAxMjMxMjM1OTU5WjB8MQswCQYDVQQGEwJHQjEbMBkGA1UE +CBMSR3JlYXRlciBNYW5jaGVzdGVyMRAwDgYDVQQHEwdTYWxmb3JkMRgwFgYDVQQK +Ew9TZWN0aWdvIExpbWl0ZWQxJDAiBgNVBAMTG1NlY3RpZ28gUlNBIENvZGUgU2ln +bmluZyBDQTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAIYijTKFehif +SfCWL2MIHi3cfJ8Uz+MmtiVmKUCGVEZ0MWLFEO2yhyemmcuVMMBW9aR1xqkOUGKl +UZEQauBLYq798PgYrKf/7i4zIPoMGYmobHutAMNhodxpZW0fbieW15dRhqb0J+V8 +aouVHltg1X7XFpKcAC9o95ftanK+ODtj3o+/bkxBXRIgCFnoOc2P0tbPBrRXBbZO +oT5Xax+YvMRi1hsLjcdmG0qfnYHEckC14l/vC0X/o84Xpi1VsLewvFRqnbyNVlPG +8Lp5UEks9wO5/i9lNfIi6iwHr0bZ+UYc3Ix8cSjz/qfGFN1VkW6KEQ3fBiSVfQ+n +oXw62oY1YdMCAwEAAaOCAWQwggFgMB8GA1UdIwQYMBaAFFN5v1qqK0rPVIDh2JvA +nfKyA2bLMB0GA1UdDgQWBBQO4TqoUzox1Yq+wbutZxoDha00DjAOBgNVHQ8BAf8E +BAMCAYYwEgYDVR0TAQH/BAgwBgEB/wIBADAdBgNVHSUEFjAUBggrBgEFBQcDAwYI +KwYBBQUHAwgwEQYDVR0gBAowCDAGBgRVHSAAMFAGA1UdHwRJMEcwRaBDoEGGP2h0 +dHA6Ly9jcmwudXNlcnRydXN0LmNvbS9VU0VSVHJ1c3RSU0FDZXJ0aWZpY2F0aW9u +QXV0aG9yaXR5LmNybDB2BggrBgEFBQcBAQRqMGgwPwYIKwYBBQUHMAKGM2h0dHA6 +Ly9jcnQudXNlcnRydXN0LmNvbS9VU0VSVHJ1c3RSU0FBZGRUcnVzdENBLmNydDAl +BggrBgEFBQcwAYYZaHR0cDovL29jc3AudXNlcnRydXN0LmNvbTANBgkqhkiG9w0B +AQwFAAOCAgEATWNQ7Uc0SmGk295qKoyb8QAAHh1iezrXMsL2s+Bjs/thAIiaG20Q +BwRPvrjqiXgi6w9G7PNGXkBGiRL0C3danCpBOvzW9Ovn9xWVM8Ohgyi33i/klPeF +M4MtSkBIv5rCT0qxjyT0s4E307dksKYjalloUkJf/wTr4XRleQj1qZPea3FAmZa6 +ePG5yOLDCBaxq2NayBWAbXReSnV+pbjDbLXP30p5h1zHQE1jNfYw08+1Cg4LBH+g +S667o6XQhACTPlNdNKUANWlsvp8gJRANGftQkGG+OY96jk32nw4e/gdREmaDJhlI +lc5KycF/8zoFm/lv34h/wCOe0h5DekUxwZxNqfBZslkZ6GqNKQQCd3xLS81wvjqy +VVp4Pry7bwMQJXcVNIr5NsxDkuS6T/FikyglVyn7URnHoSVAaoRXxrKdsbwcCtp8 +Z359LukoTBh+xHsxQXGaSynsCz1XUNLK3f2eBVHlRHjdAd6xdZgNVCT98E7j4viD +vXK6yz067vBeF5Jobchh+abxKgoLpbn0nu6YMgWFnuv5gynTxix9vTp3Los3QqBq +gu07SqqUEKThDfgXxbZaeTMYkuO1dfih6Y4KJR7kHvGfWocj/5+kUZ77OYARzdu1 +xKeogG/lU9Tg46LC0lsa+jImLWpXcBw8pFguo/NbSwfcMlnzh6cabVg= -----END CERTIFICATE----- diff --git a/depends/Makefile b/depends/Makefile index 50cc77ddeb..dc2a1e626c 100644 --- a/depends/Makefile +++ b/depends/Makefile @@ -192,4 +192,6 @@ download-win: @$(MAKE) -s HOST=x86_64-w64-mingw32 download-one download: download-osx download-linux download-win +$(foreach package,$(all_packages),$(eval $(call ext_add_stages,$(package)))) + .PHONY: install cached clean clean-all download-one download-osx download-linux download-win download check-packages check-sources diff --git a/depends/README.md b/depends/README.md index b69584193a..68a83a2aea 100644 --- a/depends/README.md +++ b/depends/README.md @@ -72,7 +72,7 @@ The following can be set when running make: make FOO=bar NO_WALLET: Don't download/build/cache libs needed to enable the wallet NO_UPNP: Don't download/build/cache packages needed for enabling upnp DEBUG: disable some optimizations and enable more runtime checking - RAPIDCHECK: build rapidcheck (experimental) + RAPIDCHECK: build rapidcheck (experimental, requires cmake) HOST_ID_SALT: Optional salt to use when generating host package ids BUILD_ID_SALT: Optional salt to use when generating build package ids diff --git a/depends/funcs.mk b/depends/funcs.mk index 9ff68f6f3c..8f03c5f37a 100644 --- a/depends/funcs.mk +++ b/depends/funcs.mk @@ -171,13 +171,13 @@ $($(1)_extracted): | $($(1)_fetched) $(AT)mkdir -p $$(@D) $(AT)cd $$(@D); $(call $(1)_extract_cmds,$(1)) $(AT)touch $$@ -$($(1)_preprocessed): | $($(1)_dependencies) $($(1)_extracted) +$($(1)_preprocessed): | $($(1)_extracted) $(AT)echo Preprocessing $(1)... $(AT)mkdir -p $$(@D) $($(1)_patch_dir) $(AT)$(foreach patch,$($(1)_patches),cd $(PATCHES_PATH)/$(1); cp $(patch) $($(1)_patch_dir) ;) $(AT)cd $$(@D); $(call $(1)_preprocess_cmds, $(1)) $(AT)touch $$@ -$($(1)_configured): | $($(1)_preprocessed) +$($(1)_configured): | $($(1)_dependencies) $($(1)_preprocessed) $(AT)echo Configuring $(1)... $(AT)rm -rf $(host_prefix); mkdir -p $(host_prefix)/lib; cd $(host_prefix); $(foreach package,$($(1)_all_dependencies), tar --no-same-owner -xf $($(package)_cached); ) $(AT)mkdir -p $$(@D) @@ -214,6 +214,14 @@ $(1): | $($(1)_cached_checksum) endef +stages = fetched extracted preprocessed configured built staged postprocessed cached cached_checksum + +define ext_add_stages +$(foreach stage,$(stages), + $(1)_$(stage): $($(1)_$(stage)) + .PHONY: $(1)_$(stage)) +endef + # These functions create the build targets for each package. They must be # broken down into small steps so that each part is done for all packages # before moving on to the next step. Otherwise, a package's info diff --git a/depends/packages/rapidcheck.mk b/depends/packages/rapidcheck.mk index 19cf1cae2e..a35e091c80 100644 --- a/depends/packages/rapidcheck.mk +++ b/depends/packages/rapidcheck.mk @@ -1,18 +1,17 @@ package=rapidcheck -$(package)_version=10fc0cb -$(package)_download_path=https://github.com/MarcoFalke/rapidcheck/archive -$(package)_file_name=$(package)-$($(package)_version).tar.gz -$(package)_sha256_hash=9640926223c00af45bce4c7df8b756b5458a89b2ba74cfe3e404467f13ce26df +$(package)_version=3eb9b4ff69f4ff2d9932e8f852c2b2a61d7c20d3 +$(package)_download_path=https://github.com/emil-e/rapidcheck/archive +$(package)_file_name=$($(package)_version).tar.gz +$(package)_sha256_hash=5fbf82755c9a647127e62563be079448ff8b1db9ca80a52a673dd9a88fdb714b define $(package)_config_cmds - cmake -DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=true . + cmake -DCMAKE_INSTALL_PREFIX=$($(package)_staging_dir)$(host_prefix) -DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=true -DRC_INSTALL_ALL_EXTRAS=ON endef define $(package)_build_cmds - $(MAKE) && \ - mkdir -p $($(package)_staging_dir)$(host_prefix)/include && \ - cp -a include/* $($(package)_staging_dir)$(host_prefix)/include/ && \ - cp -a extras/boost_test/include/rapidcheck/* $($(package)_staging_dir)$(host_prefix)/include/rapidcheck/ && \ - mkdir -p $($(package)_staging_dir)$(host_prefix)/lib && \ - cp -a librapidcheck.a $($(package)_staging_dir)$(host_prefix)/lib/ + $(MAKE) rapidcheck +endef + +define $(package)_stage_cmds + $(MAKE) rapidcheck install endef diff --git a/doc/README.md b/doc/README.md index 8876ffdd72..b4fa933c8e 100644 --- a/doc/README.md +++ b/doc/README.md @@ -57,7 +57,6 @@ The Bitcoin repo's [root README](/README.md) contains relevant information on th - [Source Code Documentation (External Link)](https://dev.visucore.com/bitcoin/doxygen/) - [Translation Process](translation_process.md) - [Translation Strings Policy](translation_strings_policy.md) -- [Travis CI](travis-ci.md) - [JSON-RPC Interface](JSON-RPC-interface.md) - [Unauthenticated REST Interface](REST-interface.md) - [Shared Libraries](shared-libraries.md) diff --git a/doc/bips.md b/doc/bips.md index 76edc94c29..eb24ce6f66 100644 --- a/doc/bips.md +++ b/doc/bips.md @@ -1,4 +1,4 @@ -BIPs that are implemented by Bitcoin Core (up-to-date up to **v0.17.0**): +BIPs that are implemented by Bitcoin Core (up-to-date up to **v0.18.0**): * [`BIP 9`](https://github.com/bitcoin/bips/blob/master/bip-0009.mediawiki): The changes allowing multiple soft-forks to be deployed in parallel have been implemented since **v0.12.1** ([PR #7575](https://github.com/bitcoin/bitcoin/pull/7575)) * [`BIP 11`](https://github.com/bitcoin/bips/blob/master/bip-0011.mediawiki): Multisig outputs are standard since **v0.6.0** ([PR #669](https://github.com/bitcoin/bitcoin/pull/669)). @@ -15,11 +15,11 @@ BIPs that are implemented by Bitcoin Core (up-to-date up to **v0.17.0**): * [`BIP 35`](https://github.com/bitcoin/bips/blob/master/bip-0035.mediawiki): The 'mempool' protocol message (and the protocol version bump to 60002) has been implemented since **v0.7.0** ([PR #1641](https://github.com/bitcoin/bitcoin/pull/1641)). * [`BIP 37`](https://github.com/bitcoin/bips/blob/master/bip-0037.mediawiki): The bloom filtering for transaction relaying, partial Merkle trees for blocks, and the protocol version bump to 70001 (enabling low-bandwidth SPV clients) has been implemented since **v0.8.0** ([PR #1795](https://github.com/bitcoin/bitcoin/pull/1795)). * [`BIP 42`](https://github.com/bitcoin/bips/blob/master/bip-0042.mediawiki): The bug that would have caused the subsidy schedule to resume after block 13440000 was fixed in **v0.9.2** ([PR #3842](https://github.com/bitcoin/bitcoin/pull/3842)). -* [`BIP 61`](https://github.com/bitcoin/bips/blob/master/bip-0061.mediawiki): The 'reject' protocol message (and the protocol version bump to 70002) was added in **v0.9.0** ([PR #3185](https://github.com/bitcoin/bitcoin/pull/3185)). Starting *v0.17.0*, whether to send reject messages can be configured with the `-enablebip61` option. +* [`BIP 61`](https://github.com/bitcoin/bips/blob/master/bip-0061.mediawiki): The 'reject' protocol message (and the protocol version bump to 70002) was added in **v0.9.0** ([PR #3185](https://github.com/bitcoin/bitcoin/pull/3185)). Starting **v0.17.0**, whether to send reject messages can be configured with the `-enablebip61` option, and support is deprecated as of **v0.18.0**. * [`BIP 65`](https://github.com/bitcoin/bips/blob/master/bip-0065.mediawiki): The CHECKLOCKTIMEVERIFY softfork was merged in **v0.12.0** ([PR #6351](https://github.com/bitcoin/bitcoin/pull/6351)), and backported to **v0.11.2** and **v0.10.4**. Mempool-only CLTV was added in [PR #6124](https://github.com/bitcoin/bitcoin/pull/6124). * [`BIP 66`](https://github.com/bitcoin/bips/blob/master/bip-0066.mediawiki): The strict DER rules and associated version 3 blocks have been implemented since **v0.10.0** ([PR #5713](https://github.com/bitcoin/bitcoin/pull/5713)). * [`BIP 68`](https://github.com/bitcoin/bips/blob/master/bip-0068.mediawiki): Sequence locks have been implemented as of **v0.12.1** ([PR #7184](https://github.com/bitcoin/bitcoin/pull/7184)), and have been activated since *block 419328*. -* [`BIP 70`](https://github.com/bitcoin/bips/blob/master/bip-0070.mediawiki) [`71`](https://github.com/bitcoin/bips/blob/master/bip-0071.mediawiki) [`72`](https://github.com/bitcoin/bips/blob/master/bip-0072.mediawiki): Payment Protocol support has been available in Bitcoin Core GUI since **v0.9.0** ([PR #5216](https://github.com/bitcoin/bitcoin/pull/5216)). +* [`BIP 70`](https://github.com/bitcoin/bips/blob/master/bip-0070.mediawiki) [`71`](https://github.com/bitcoin/bips/blob/master/bip-0071.mediawiki) [`72`](https://github.com/bitcoin/bips/blob/master/bip-0072.mediawiki): Payment Protocol support has been available in Bitcoin Core GUI since **v0.9.0** ([PR #5216](https://github.com/bitcoin/bitcoin/pull/5216)). Support can be optionally disabled at build time since **v0.18.0** ([PR 14451](https://github.com/bitcoin/bitcoin/pull/14451)). * [`BIP 90`](https://github.com/bitcoin/bips/blob/master/bip-0090.mediawiki): Trigger mechanism for activation of BIPs 34, 65, and 66 has been simplified to block height checks since **v0.14.0** ([PR #8391](https://github.com/bitcoin/bitcoin/pull/8391)). * [`BIP 111`](https://github.com/bitcoin/bips/blob/master/bip-0111.mediawiki): `NODE_BLOOM` service bit added, and enforced for all peer versions as of **v0.13.0** ([PR #6579](https://github.com/bitcoin/bitcoin/pull/6579) and [PR #6641](https://github.com/bitcoin/bitcoin/pull/6641)). * [`BIP 112`](https://github.com/bitcoin/bips/blob/master/bip-0112.mediawiki): The CHECKSEQUENCEVERIFY opcode has been implemented since **v0.12.1** ([PR #7524](https://github.com/bitcoin/bitcoin/pull/7524)) and has been activated since *block 419328*. @@ -33,7 +33,7 @@ BIPs that are implemented by Bitcoin Core (up-to-date up to **v0.17.0**): * [`BIP 145`](https://github.com/bitcoin/bips/blob/master/bip-0145.mediawiki): getblocktemplate updates for Segregated Witness as of **v0.13.0** ([PR 8149](https://github.com/bitcoin/bitcoin/pull/8149)). * [`BIP 147`](https://github.com/bitcoin/bips/blob/master/bip-0147.mediawiki): NULLDUMMY softfork as of **v0.13.1** ([PR 8636](https://github.com/bitcoin/bitcoin/pull/8636) and [PR 8937](https://github.com/bitcoin/bitcoin/pull/8937)). * [`BIP 152`](https://github.com/bitcoin/bips/blob/master/bip-0152.mediawiki): Compact block transfer and related optimizations are used as of **v0.13.0** ([PR 8068](https://github.com/bitcoin/bitcoin/pull/8068)). -* [`BIP 159`](https://github.com/bitcoin/bips/blob/master/bip-0159.mediawiki): NODE_NETWORK_LIMITED service bit [signaling only] is supported as of **v0.16.0** ([PR 11740](https://github.com/bitcoin/bitcoin/pull/11740)). +* [`BIP 159`](https://github.com/bitcoin/bips/blob/master/bip-0159.mediawiki): The NODE_NETWORK_LIMITED service bit is signalled as of **v0.16.0** ([PR 11740](https://github.com/bitcoin/bitcoin/pull/11740)), and such nodes are connected to as of **v0.17.0** ([PR 10387](https://github.com/bitcoin/bitcoin/pull/10387)). * [`BIP 173`](https://github.com/bitcoin/bips/blob/master/bip-0173.mediawiki): Bech32 addresses for native Segregated Witness outputs are supported as of **v0.16.0** ([PR 11167](https://github.com/bitcoin/bitcoin/pull/11167)). * [`BIP 174`](https://github.com/bitcoin/bips/blob/master/bip-0174.mediawiki): RPCs to operate on Partially Signed Bitcoin Transactions (PSBT) are present as of **v0.17.0** ([PR 13557](https://github.com/bitcoin/bitcoin/pull/13557)). * [`BIP 176`](https://github.com/bitcoin/bips/blob/master/bip-0176.mediawiki): Bits Denomination [QT only] is supported as of **v0.16.0** ([PR 12035](https://github.com/bitcoin/bitcoin/pull/12035)). diff --git a/doc/dependencies.md b/doc/dependencies.md index 0fb36184c2..c445e2e23f 100644 --- a/doc/dependencies.md +++ b/doc/dependencies.md @@ -17,7 +17,7 @@ These are the dependencies currently used by Bitcoin Core. You can find instruct | libevent | [2.1.8-stable](https://github.com/libevent/libevent/releases) | 2.0.22 | No | | | | libjpeg | | | | | [Yes](https://github.com/bitcoin/bitcoin/blob/master/depends/packages/qt.mk#L65) | | libpng | | | | | [Yes](https://github.com/bitcoin/bitcoin/blob/master/depends/packages/qt.mk#L64) | -| libsrvg | | | | | | +| librsvg | | | | | | | MiniUPnPc | [2.0.20180203](http://miniupnp.free.fr/files) | | No | | | | OpenSSL | [1.0.1k](https://www.openssl.org/source) | | Yes | | | | PCRE | | | | | [Yes](https://github.com/bitcoin/bitcoin/blob/master/depends/packages/qt.mk#L66) | @@ -43,4 +43,4 @@ Some dependencies are not needed in all configurations. The following are some f * ZeroMQ is needed only with the `--with-zmq` option. #### Other -* librsvg is only needed if you need to run `make deploy` on (cross-compliation to) macOS.
\ No newline at end of file +* librsvg is only needed if you need to run `make deploy` on (cross-compliation to) macOS. diff --git a/doc/descriptors.md b/doc/descriptors.md index 5dbcd95e1d..dbdac2c5b6 100644 --- a/doc/descriptors.md +++ b/doc/descriptors.md @@ -1,11 +1,18 @@ # Support for Output Descriptors in Bitcoin Core -Since Bitcoin Core v0.17, there is support for Output Descriptors in the -`scantxoutset` RPC call. This is a simple language which can be used to -describe collections of output scripts. - -This document describes the language. For the specifics on usage for scanning -the UTXO set, see the `scantxoutset` RPC help. +Since Bitcoin Core v0.17, there is support for Output Descriptors. This is a +simple language which can be used to describe collections of output scripts. +Supporting RPCs are: +- `scantxoutset` takes as input descriptors to scan for, and also reports + specialized descriptors for the matching UTXOs. +- `getdescriptorinfo` analyzes a descriptor, and reports a canonicalized version + with checksum added. +- `deriveaddresses` takes as input a descriptor and computes the corresponding + addresses. +- `listunspent` outputs a specialized descriptor for the reported unspent outputs. + +This document describes the language. For the specifics on usage, see the RPC +documentation for the functions mentioned above. ## Features diff --git a/doc/productivity.md b/doc/productivity.md index e0df558944..a93228ebdb 100644 --- a/doc/productivity.md +++ b/doc/productivity.md @@ -8,6 +8,7 @@ Table of Contents * [Cache compilations with `ccache`](#cache-compilations-with-ccache) * [Disable features with `./configure`](#disable-features-with-configure) * [Make use of your threads with `make -j`](#make-use-of-your-threads-with-make--j) + * [Only build what you need](#only-build-what-you-need) * [Multiple working directories with `git worktrees`](#multiple-working-directories-with-git-worktrees) * [Writing code](#writing-code) * [Format C/C++/Protobuf diffs with `clang-format-diff.py`](#format-ccprotobuf-diffs-with-clang-format-diffpy) @@ -32,6 +33,17 @@ Install `ccache` through your distribution's package manager, and run `./configu To use ccache for all your C/C++ projects, follow the symlinks method [here](https://ccache.samba.org/manual/latest.html#_run_modes) to set it up. +To get the most out of ccache, put something like this in `~/.ccache/ccache.conf`: + +``` +max_size = 50.0G # or whatever cache size you prefer; default is 5G; 0 means unlimited +base_dir = /home/yourname # or wherever you keep your source files +``` + +Note: base_dir is required for ccache to share cached compiles of the same file across different repositories / paths; it will only do this for paths under base_dir. So this option is required for effective use of ccache with git worktrees (described below). + +You _must not_ set base_dir to "/", or anywhere that contains system headers (according to the ccache docs). + ### Disable features with `./configure` After running `./autogen.sh`, which generates the `./configure` file, use `./configure --help` to identify features that you can disable to save on compilation time. A few common flags: @@ -43,6 +55,8 @@ After running `./autogen.sh`, which generates the `./configure` file, use `./con --without-gui ``` +If you do need the wallet enabled, it is common for devs to add `--with-incompatible-bdb`. This uses your system bdb version for the wallet, so you don't have to find a copy of bdb 4.8. Wallets from such a build will be incompatible with any release binary (and vice versa), so use with caution on mainnet. + ### Make use of your threads with `make -j` If you have multiple threads on your machine, you can tell `make` to utilize all of them with: @@ -51,6 +65,20 @@ If you have multiple threads on your machine, you can tell `make` to utilize all make -j"$(($(nproc)+1))" ``` +### Only build what you need + +When rebuilding during development, note that running `make`, without giving a target, will do a lot of work you probably don't need. It will build the GUI (unless you've disabled it) and all the tests (which take much longer to build than the app does). + +Obviously, it is important to build and run the tests at appropriate times -- but when you just want a quick compile to check your work, consider picking one or a set of build targets relevant to what you're working on, e.g.: + +```sh +make src/bitcoind src/bitcoin-cli +make src/qt/bitcoin-qt +make -C src bitcoin_bench +``` + +(You can and should combine this with `-j`, as above, for a parallel build.) + ### Multiple working directories with `git worktrees` If you work with multiple branches or multiple copies of the repository, you should try `git worktrees`. diff --git a/doc/psbt.md b/doc/psbt.md index 560b45ef31..9d85af0348 100644 --- a/doc/psbt.md +++ b/doc/psbt.md @@ -67,6 +67,9 @@ hardware implementations will typically implement multiple roles simultaneously. input a PSBT, adds UTXO, key, and script data to inputs and outputs that miss it, and optionally signs inputs. Where possible it also finalizes the partial signatures. +- **`utxoupdatepsbt` (Updater)** is a node RPC that takes a PSBT and updates it + to include information available from the UTXO set (works only for SegWit + inputs). - **`finalizepsbt` (Finalizer, Extractor)** is a utility RPC that finalizes any partial signatures, and if all inputs are finalized, converts the result to a fully signed transaction which can be broadcast with `sendrawtransaction`. @@ -74,8 +77,15 @@ hardware implementations will typically implement multiple roles simultaneously. can be used at any point in the workflow to merge information added to different versions of the same PSBT. In particular it is useful to combine the output of multiple Updaters or Signers. +- **`joinpsbts`** (Creator) is a utility RPC that joins multiple PSBTs together, + concatenating the inputs and outputs. This can be used to construct CoinJoin + transactions. - **`decodepsbt`** is a diagnostic utility RPC which will show all information in a PSBT in human-readable form, as well as compute its eventual fee if known. +- **`analyzepsbt`** is a utility RPC that examines an RPC and reports the + next steps in the workflow if known, computes the fee of the resulting + transaction, and estimates the weight and feerate if possible. + ### Workflows diff --git a/doc/release-notes-15620.md b/doc/release-notes-15620.md new file mode 100644 index 0000000000..bf89a70a4e --- /dev/null +++ b/doc/release-notes-15620.md @@ -0,0 +1,13 @@ +Updated RPCs +------------ + +* The -maxtxfee setting no longer has any effect on non-wallet RPCs. + + The `sendrawtransaction` and `testmempoolaccept` RPC methods previously + accepted an `allowhighfees` parameter to fail the mempool acceptance in case + the transaction's fee would exceed the value of the command line argument + `-maxtxfee`. To uncouple the RPCs from the global option, they now have a + hardcoded default for the maximum transaction fee, that can be changed for + both RPCs on a per-call basis with the `maxfeerate` parameter. The + `allowhighfees` boolean option has been removed and replaced by the + `maxfeerate` numeric option. diff --git a/doc/release-notes-15637.md b/doc/release-notes-15637.md new file mode 100644 index 0000000000..048d5e7218 --- /dev/null +++ b/doc/release-notes-15637.md @@ -0,0 +1,3 @@ +RPC changes +----------- +In getmempoolancestors, getmempooldescendants, getmempoolentry and getrawmempool RPCs, to be consistent with the returned value and other RPCs such as getrawtransaction, vsize has been added and size is now deprecated. size will only be returned if bitcoind is started with `-deprecatedrpc=size`.
\ No newline at end of file diff --git a/doc/release-notes.md b/doc/release-notes.md index ebcdcda306..d4acbb9c88 100644 --- a/doc/release-notes.md +++ b/doc/release-notes.md @@ -66,16 +66,33 @@ platform. Notable changes =============== -Example item +Updated RPCs ------------ +Note: some low-level RPC changes mainly useful for testing are described in the +Low-level Changes section below. + +* The `sendmany` RPC had an argument `minconf` that was not well specified and + would lead to RPC errors even when the wallet's coin selection would succeed. + The `sendtoaddress` RPC never had this check, so to normalize the behavior, + `minconf` is now ignored in `sendmany`. If the coin selection does not + succeed due to missing coins, it will still throw an RPC error. Be reminded + that coin selection is influenced by the `-spendzeroconfchange`, + `-limitancestorcount`, `-limitdescendantcount` and `-walletrejectlongchains` + command line arguments. + Low-level changes ================= -Example item +Configuration ------------ +* An error is issued where previously a warning was issued when a setting in + the config file was specified in the default section, but not overridden for + the selected network. This change takes only effect if the selected network + is not mainnet. + Credits ======= diff --git a/doc/shared-libraries.md b/doc/shared-libraries.md index dc363582cc..e960863a80 100644 --- a/doc/shared-libraries.md +++ b/doc/shared-libraries.md @@ -7,11 +7,11 @@ The purpose of this library is to make the verification functionality that is cr ### API -The interface is defined in the C header `bitcoinconsensus.h` located in `src/script/bitcoinconsensus.h`. +The interface is defined in the C header `bitcoinconsensus.h` located in `src/script/bitcoinconsensus.h`. #### Version -`bitcoinconsensus_version` returns an `unsigned int` with the API version *(currently at an experimental `0`)*. +`bitcoinconsensus_version` returns an `unsigned int` with the API version *(currently `1`)*. #### Script Validation diff --git a/doc/travis-ci.md b/doc/travis-ci.md deleted file mode 100644 index 38085cec35..0000000000 --- a/doc/travis-ci.md +++ /dev/null @@ -1,42 +0,0 @@ -Travis CI -========= - -Support for using travis-ci has been added in order to automate pull-testing. -See [travis-ci.org](https://travis-ci.org/) for more info - -This procedure is different than the pull-tester that came before it in a few -ways. - -There is nothing to administer. This is a major feature as it means -that builds have no local state. Because there is no ability to login to the -builders to install packages (tools, dependencies, etc), the entire build -procedure must instead be controlled by a declarative script `.travis.yml`. -This script declares each build configuration, creates virtual machines as -necessary, builds, then discards the virtual machines. - -A build matrix is constructed to test a wide range of configurations, rather -than a single pass/fail. This helps to catch build failures and logic errors -that present on platforms other than the ones the author has tested. This -matrix is defined in the build script and can be changed at any time. - -All builders use the dependency-generator in the [depends dir](/depends), rather than -using apt-get to install build dependencies. This guarantees that the tester -is using the same versions as Gitian, so the build results are nearly identical -to what would be found in a final release. However, this also means that builds -will fail if new dependencies are introduced without being added to the -dependency generator. - -In order to avoid rebuilding all dependencies for each build, the binaries are -cached and re-used when possible. Changes in the dependency-generator will -trigger cache-invalidation and rebuilds as necessary. - -These caches can be manually removed if necessary. This is one of the very few -manual operations that is possible with Travis, and it can be done by the -Bitcoin Core committer via the Travis web interface. - -In some cases, secure strings may be needed for hiding sensitive info such as -private keys or URLs. The travis client may be used to create these strings: -http://docs.travis-ci.com/user/encryption-keys/ - -For the details of the build descriptor, see the official docs: -http://docs.travis-ci.com/user/build-configuration/ diff --git a/src/Makefile.am b/src/Makefile.am index aadd402e3c..0385c825ea 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -346,6 +346,8 @@ crypto_libbitcoin_crypto_base_a_SOURCES = \ crypto/hmac_sha256.h \ crypto/hmac_sha512.cpp \ crypto/hmac_sha512.h \ + crypto/poly1305.h \ + crypto/poly1305.cpp \ crypto/ripemd160.cpp \ crypto/ripemd160.h \ crypto/sha1.cpp \ @@ -432,8 +434,8 @@ libbitcoin_common_a_SOURCES = \ netaddress.cpp \ netbase.cpp \ policy/feerate.cpp \ - psbt.cpp \ protocol.cpp \ + psbt.cpp \ scheduler.cpp \ script/descriptor.cpp \ script/ismine.cpp \ @@ -495,6 +497,8 @@ if TARGET_WINDOWS bitcoind_SOURCES += bitcoind-res.rc endif +# Libraries below may be listed more than once to resolve circular dependencies (see +# https://eli.thegreenplace.net/2013/07/09/library-order-in-static-linking#circular-dependency) bitcoind_LDADD = \ $(LIBBITCOIN_SERVER) \ $(LIBBITCOIN_WALLET) \ diff --git a/src/Makefile.bench.include b/src/Makefile.bench.include index 3042f6df19..b84360e84b 100644 --- a/src/Makefile.bench.include +++ b/src/Makefile.bench.include @@ -31,6 +31,7 @@ bench_bench_bitcoin_SOURCES = \ bench/base58.cpp \ bench/bech32.cpp \ bench/lockedpool.cpp \ + bench/poly1305.cpp \ bench/prevector.cpp nodist_bench_bench_bitcoin_SOURCES = $(GENERATED_BENCH_FILES) diff --git a/src/bench/poly1305.cpp b/src/bench/poly1305.cpp new file mode 100644 index 0000000000..16342d0fbe --- /dev/null +++ b/src/bench/poly1305.cpp @@ -0,0 +1,41 @@ +// Copyright (c) 2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <iostream> + +#include <bench/bench.h> +#include <crypto/poly1305.h> + +/* Number of bytes to process per iteration */ +static constexpr uint64_t BUFFER_SIZE_TINY = 64; +static constexpr uint64_t BUFFER_SIZE_SMALL = 256; +static constexpr uint64_t BUFFER_SIZE_LARGE = 1024*1024; + +static void POLY1305(benchmark::State& state, size_t buffersize) +{ + std::vector<unsigned char> tag(POLY1305_TAGLEN, 0); + std::vector<unsigned char> key(POLY1305_KEYLEN, 0); + std::vector<unsigned char> in(buffersize, 0); + while (state.KeepRunning()) + poly1305_auth(tag.data(), in.data(), in.size(), key.data()); +} + +static void POLY1305_64BYTES(benchmark::State& state) +{ + POLY1305(state, BUFFER_SIZE_TINY); +} + +static void POLY1305_256BYTES(benchmark::State& state) +{ + POLY1305(state, BUFFER_SIZE_SMALL); +} + +static void POLY1305_1MB(benchmark::State& state) +{ + POLY1305(state, BUFFER_SIZE_LARGE); +} + +BENCHMARK(POLY1305_64BYTES, 500000); +BENCHMARK(POLY1305_256BYTES, 250000); +BENCHMARK(POLY1305_1MB, 340); diff --git a/src/core_io.h b/src/core_io.h index ae377eb6e8..19fb7b29f6 100644 --- a/src/core_io.h +++ b/src/core_io.h @@ -16,7 +16,6 @@ class CBlockHeader; class CScript; class CTransaction; struct CMutableTransaction; -struct PartiallySignedTransaction; class uint256; class UniValue; @@ -37,11 +36,6 @@ bool DecodeHexBlockHeader(CBlockHeader&, const std::string& hex_header); */ bool ParseHashStr(const std::string& strHex, uint256& result); std::vector<unsigned char> ParseHexUV(const UniValue& v, const std::string& strName); - -//! Decode a base64ed PSBT into a PartiallySignedTransaction -NODISCARD bool DecodeBase64PSBT(PartiallySignedTransaction& decoded_psbt, const std::string& base64_psbt, std::string& error); -//! Decode a raw (binary blob) PSBT into a PartiallySignedTransaction -NODISCARD bool DecodeRawPSBT(PartiallySignedTransaction& decoded_psbt, const std::string& raw_psbt, std::string& error); int ParseSighashString(const UniValue& sighash); // core_write.cpp diff --git a/src/core_read.cpp b/src/core_read.cpp index 536a7f4f17..a879a375ce 100644 --- a/src/core_read.cpp +++ b/src/core_read.cpp @@ -4,7 +4,6 @@ #include <core_io.h> -#include <psbt.h> #include <primitives/block.h> #include <primitives/transaction.h> #include <script/script.h> @@ -177,33 +176,6 @@ bool DecodeHexBlk(CBlock& block, const std::string& strHexBlk) return true; } -bool DecodeBase64PSBT(PartiallySignedTransaction& psbt, const std::string& base64_tx, std::string& error) -{ - bool invalid; - std::string tx_data = DecodeBase64(base64_tx, &invalid); - if (invalid) { - error = "invalid base64"; - return false; - } - return DecodeRawPSBT(psbt, tx_data, error); -} - -bool DecodeRawPSBT(PartiallySignedTransaction& psbt, const std::string& tx_data, std::string& error) -{ - CDataStream ss_data(tx_data.data(), tx_data.data() + tx_data.size(), SER_NETWORK, PROTOCOL_VERSION); - try { - ss_data >> psbt; - if (!ss_data.empty()) { - error = "extra data after PSBT"; - return false; - } - } catch (const std::exception& e) { - error = e.what(); - return false; - } - return true; -} - bool ParseHashStr(const std::string& strHex, uint256& result) { if ((strHex.size() != 64) || !IsHex(strHex)) diff --git a/src/crypto/aes.cpp b/src/crypto/aes.cpp index 919ea593b7..2dc2133434 100644 --- a/src/crypto/aes.cpp +++ b/src/crypto/aes.cpp @@ -12,36 +12,6 @@ extern "C" { #include <crypto/ctaes/ctaes.c> } -AES128Encrypt::AES128Encrypt(const unsigned char key[16]) -{ - AES128_init(&ctx, key); -} - -AES128Encrypt::~AES128Encrypt() -{ - memset(&ctx, 0, sizeof(ctx)); -} - -void AES128Encrypt::Encrypt(unsigned char ciphertext[16], const unsigned char plaintext[16]) const -{ - AES128_encrypt(&ctx, 1, ciphertext, plaintext); -} - -AES128Decrypt::AES128Decrypt(const unsigned char key[16]) -{ - AES128_init(&ctx, key); -} - -AES128Decrypt::~AES128Decrypt() -{ - memset(&ctx, 0, sizeof(ctx)); -} - -void AES128Decrypt::Decrypt(unsigned char plaintext[16], const unsigned char ciphertext[16]) const -{ - AES128_decrypt(&ctx, 1, plaintext, ciphertext); -} - AES256Encrypt::AES256Encrypt(const unsigned char key[32]) { AES256_init(&ctx, key); @@ -182,35 +152,3 @@ AES256CBCDecrypt::~AES256CBCDecrypt() { memset(iv, 0, sizeof(iv)); } - -AES128CBCEncrypt::AES128CBCEncrypt(const unsigned char key[AES128_KEYSIZE], const unsigned char ivIn[AES_BLOCKSIZE], bool padIn) - : enc(key), pad(padIn) -{ - memcpy(iv, ivIn, AES_BLOCKSIZE); -} - -AES128CBCEncrypt::~AES128CBCEncrypt() -{ - memset(iv, 0, AES_BLOCKSIZE); -} - -int AES128CBCEncrypt::Encrypt(const unsigned char* data, int size, unsigned char* out) const -{ - return CBCEncrypt(enc, iv, data, size, pad, out); -} - -AES128CBCDecrypt::AES128CBCDecrypt(const unsigned char key[AES128_KEYSIZE], const unsigned char ivIn[AES_BLOCKSIZE], bool padIn) - : dec(key), pad(padIn) -{ - memcpy(iv, ivIn, AES_BLOCKSIZE); -} - -AES128CBCDecrypt::~AES128CBCDecrypt() -{ - memset(iv, 0, AES_BLOCKSIZE); -} - -int AES128CBCDecrypt::Decrypt(const unsigned char* data, int size, unsigned char* out) const -{ - return CBCDecrypt(dec, iv, data, size, pad, out); -} diff --git a/src/crypto/aes.h b/src/crypto/aes.h index fdad70c593..e06c8de272 100644 --- a/src/crypto/aes.h +++ b/src/crypto/aes.h @@ -12,33 +12,8 @@ extern "C" { } static const int AES_BLOCKSIZE = 16; -static const int AES128_KEYSIZE = 16; static const int AES256_KEYSIZE = 32; -/** An encryption class for AES-128. */ -class AES128Encrypt -{ -private: - AES128_ctx ctx; - -public: - explicit AES128Encrypt(const unsigned char key[16]); - ~AES128Encrypt(); - void Encrypt(unsigned char ciphertext[16], const unsigned char plaintext[16]) const; -}; - -/** A decryption class for AES-128. */ -class AES128Decrypt -{ -private: - AES128_ctx ctx; - -public: - explicit AES128Decrypt(const unsigned char key[16]); - ~AES128Decrypt(); - void Decrypt(unsigned char plaintext[16], const unsigned char ciphertext[16]) const; -}; - /** An encryption class for AES-256. */ class AES256Encrypt { @@ -89,30 +64,4 @@ private: unsigned char iv[AES_BLOCKSIZE]; }; -class AES128CBCEncrypt -{ -public: - AES128CBCEncrypt(const unsigned char key[AES128_KEYSIZE], const unsigned char ivIn[AES_BLOCKSIZE], bool padIn); - ~AES128CBCEncrypt(); - int Encrypt(const unsigned char* data, int size, unsigned char* out) const; - -private: - const AES128Encrypt enc; - const bool pad; - unsigned char iv[AES_BLOCKSIZE]; -}; - -class AES128CBCDecrypt -{ -public: - AES128CBCDecrypt(const unsigned char key[AES128_KEYSIZE], const unsigned char ivIn[AES_BLOCKSIZE], bool padIn); - ~AES128CBCDecrypt(); - int Decrypt(const unsigned char* data, int size, unsigned char* out) const; - -private: - const AES128Decrypt dec; - const bool pad; - unsigned char iv[AES_BLOCKSIZE]; -}; - #endif // BITCOIN_CRYPTO_AES_H diff --git a/src/crypto/poly1305.cpp b/src/crypto/poly1305.cpp new file mode 100644 index 0000000000..8a86c9601c --- /dev/null +++ b/src/crypto/poly1305.cpp @@ -0,0 +1,141 @@ +// Copyright (c) 2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +// Based on the public domain implementation by Andrew Moon +// poly1305-donna-unrolled.c from https://github.com/floodyberry/poly1305-donna + +#include <crypto/common.h> +#include <crypto/poly1305.h> + +#include <string.h> + +#define mul32x32_64(a,b) ((uint64_t)(a) * (b)) + +void poly1305_auth(unsigned char out[POLY1305_TAGLEN], const unsigned char *m, size_t inlen, const unsigned char key[POLY1305_KEYLEN]) { + uint32_t t0,t1,t2,t3; + uint32_t h0,h1,h2,h3,h4; + uint32_t r0,r1,r2,r3,r4; + uint32_t s1,s2,s3,s4; + uint32_t b, nb; + size_t j; + uint64_t t[5]; + uint64_t f0,f1,f2,f3; + uint64_t g0,g1,g2,g3,g4; + uint64_t c; + unsigned char mp[16]; + + /* clamp key */ + t0 = ReadLE32(key+0); + t1 = ReadLE32(key+4); + t2 = ReadLE32(key+8); + t3 = ReadLE32(key+12); + + /* precompute multipliers */ + r0 = t0 & 0x3ffffff; t0 >>= 26; t0 |= t1 << 6; + r1 = t0 & 0x3ffff03; t1 >>= 20; t1 |= t2 << 12; + r2 = t1 & 0x3ffc0ff; t2 >>= 14; t2 |= t3 << 18; + r3 = t2 & 0x3f03fff; t3 >>= 8; + r4 = t3 & 0x00fffff; + + s1 = r1 * 5; + s2 = r2 * 5; + s3 = r3 * 5; + s4 = r4 * 5; + + /* init state */ + h0 = 0; + h1 = 0; + h2 = 0; + h3 = 0; + h4 = 0; + + /* full blocks */ + if (inlen < 16) goto poly1305_donna_atmost15bytes; +poly1305_donna_16bytes: + m += 16; + inlen -= 16; + + t0 = ReadLE32(m-16); + t1 = ReadLE32(m-12); + t2 = ReadLE32(m-8); + t3 = ReadLE32(m-4); + + h0 += t0 & 0x3ffffff; + h1 += ((((uint64_t)t1 << 32) | t0) >> 26) & 0x3ffffff; + h2 += ((((uint64_t)t2 << 32) | t1) >> 20) & 0x3ffffff; + h3 += ((((uint64_t)t3 << 32) | t2) >> 14) & 0x3ffffff; + h4 += (t3 >> 8) | (1 << 24); + + +poly1305_donna_mul: + t[0] = mul32x32_64(h0,r0) + mul32x32_64(h1,s4) + mul32x32_64(h2,s3) + mul32x32_64(h3,s2) + mul32x32_64(h4,s1); + t[1] = mul32x32_64(h0,r1) + mul32x32_64(h1,r0) + mul32x32_64(h2,s4) + mul32x32_64(h3,s3) + mul32x32_64(h4,s2); + t[2] = mul32x32_64(h0,r2) + mul32x32_64(h1,r1) + mul32x32_64(h2,r0) + mul32x32_64(h3,s4) + mul32x32_64(h4,s3); + t[3] = mul32x32_64(h0,r3) + mul32x32_64(h1,r2) + mul32x32_64(h2,r1) + mul32x32_64(h3,r0) + mul32x32_64(h4,s4); + t[4] = mul32x32_64(h0,r4) + mul32x32_64(h1,r3) + mul32x32_64(h2,r2) + mul32x32_64(h3,r1) + mul32x32_64(h4,r0); + + h0 = (uint32_t)t[0] & 0x3ffffff; c = (t[0] >> 26); + t[1] += c; h1 = (uint32_t)t[1] & 0x3ffffff; b = (uint32_t)(t[1] >> 26); + t[2] += b; h2 = (uint32_t)t[2] & 0x3ffffff; b = (uint32_t)(t[2] >> 26); + t[3] += b; h3 = (uint32_t)t[3] & 0x3ffffff; b = (uint32_t)(t[3] >> 26); + t[4] += b; h4 = (uint32_t)t[4] & 0x3ffffff; b = (uint32_t)(t[4] >> 26); + h0 += b * 5; + + if (inlen >= 16) goto poly1305_donna_16bytes; + + /* final bytes */ +poly1305_donna_atmost15bytes: + if (!inlen) goto poly1305_donna_finish; + + for (j = 0; j < inlen; j++) mp[j] = m[j]; + mp[j++] = 1; + for (; j < 16; j++) mp[j] = 0; + inlen = 0; + + t0 = ReadLE32(mp+0); + t1 = ReadLE32(mp+4); + t2 = ReadLE32(mp+8); + t3 = ReadLE32(mp+12); + + h0 += t0 & 0x3ffffff; + h1 += ((((uint64_t)t1 << 32) | t0) >> 26) & 0x3ffffff; + h2 += ((((uint64_t)t2 << 32) | t1) >> 20) & 0x3ffffff; + h3 += ((((uint64_t)t3 << 32) | t2) >> 14) & 0x3ffffff; + h4 += (t3 >> 8); + + goto poly1305_donna_mul; + +poly1305_donna_finish: + b = h0 >> 26; h0 = h0 & 0x3ffffff; + h1 += b; b = h1 >> 26; h1 = h1 & 0x3ffffff; + h2 += b; b = h2 >> 26; h2 = h2 & 0x3ffffff; + h3 += b; b = h3 >> 26; h3 = h3 & 0x3ffffff; + h4 += b; b = h4 >> 26; h4 = h4 & 0x3ffffff; + h0 += b * 5; b = h0 >> 26; h0 = h0 & 0x3ffffff; + h1 += b; + + g0 = h0 + 5; b = g0 >> 26; g0 &= 0x3ffffff; + g1 = h1 + b; b = g1 >> 26; g1 &= 0x3ffffff; + g2 = h2 + b; b = g2 >> 26; g2 &= 0x3ffffff; + g3 = h3 + b; b = g3 >> 26; g3 &= 0x3ffffff; + g4 = h4 + b - (1 << 26); + + b = (g4 >> 31) - 1; + nb = ~b; + h0 = (h0 & nb) | (g0 & b); + h1 = (h1 & nb) | (g1 & b); + h2 = (h2 & nb) | (g2 & b); + h3 = (h3 & nb) | (g3 & b); + h4 = (h4 & nb) | (g4 & b); + + f0 = ((h0 ) | (h1 << 26)) + (uint64_t)ReadLE32(&key[16]); + f1 = ((h1 >> 6) | (h2 << 20)) + (uint64_t)ReadLE32(&key[20]); + f2 = ((h2 >> 12) | (h3 << 14)) + (uint64_t)ReadLE32(&key[24]); + f3 = ((h3 >> 18) | (h4 << 8)) + (uint64_t)ReadLE32(&key[28]); + + WriteLE32(&out[ 0], f0); f1 += (f0 >> 32); + WriteLE32(&out[ 4], f1); f2 += (f1 >> 32); + WriteLE32(&out[ 8], f2); f3 += (f2 >> 32); + WriteLE32(&out[12], f3); +} diff --git a/src/crypto/poly1305.h b/src/crypto/poly1305.h new file mode 100644 index 0000000000..1598b013b9 --- /dev/null +++ b/src/crypto/poly1305.h @@ -0,0 +1,17 @@ +// Copyright (c) 2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_CRYPTO_POLY1305_H +#define BITCOIN_CRYPTO_POLY1305_H + +#include <stdint.h> +#include <stdlib.h> + +#define POLY1305_KEYLEN 32 +#define POLY1305_TAGLEN 16 + +void poly1305_auth(unsigned char out[POLY1305_TAGLEN], const unsigned char *m, size_t inlen, + const unsigned char key[POLY1305_KEYLEN]); + +#endif // BITCOIN_CRYPTO_POLY1305_H diff --git a/src/init.cpp b/src/init.cpp index 0898d0ff25..ec8800ef07 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -500,7 +500,7 @@ void SetupServerArgs() gArgs.AddArg("-mocktime=<n>", "Replace actual time with <n> seconds since epoch (default: 0)", true, OptionsCategory::DEBUG_TEST); gArgs.AddArg("-maxsigcachesize=<n>", strprintf("Limit sum of signature cache and script execution cache sizes to <n> MiB (default: %u)", DEFAULT_MAX_SIG_CACHE_SIZE), true, OptionsCategory::DEBUG_TEST); gArgs.AddArg("-maxtipage=<n>", strprintf("Maximum tip age in seconds to consider node in initial block download (default: %u)", DEFAULT_MAX_TIP_AGE), true, OptionsCategory::DEBUG_TEST); - gArgs.AddArg("-maxtxfee=<amt>", strprintf("Maximum total fees (in %s) to use in a single wallet transaction or raw transaction; setting this too low may abort large transactions (default: %s)", + gArgs.AddArg("-maxtxfee=<amt>", strprintf("Maximum total fees (in %s) to use in a single wallet transaction; setting this too low may abort large transactions (default: %s)", // TODO move setting to wallet CURRENCY_UNIT, FormatMoney(DEFAULT_TRANSACTION_MAXFEE)), false, OptionsCategory::DEBUG_TEST); gArgs.AddArg("-printpriority", strprintf("Log transaction fee per kB when mining blocks (default: %u)", DEFAULT_PRINTPRIORITY), true, OptionsCategory::DEBUG_TEST); gArgs.AddArg("-printtoconsole", "Send trace/debug info to console (default: 1 when no -daemon. To disable logging to file, set -nodebuglogfile)", false, OptionsCategory::DEBUG_TEST); @@ -828,19 +828,6 @@ void InitParameterInteraction() if (gArgs.SoftSetBoolArg("-whitelistrelay", true)) LogPrintf("%s: parameter interaction: -whitelistforcerelay=1 -> setting -whitelistrelay=1\n", __func__); } - - // Warn if network-specific options (-addnode, -connect, etc) are - // specified in default section of config file, but not overridden - // on the command line or in this network's section of the config file. - std::string network = gArgs.GetChainName(); - for (const auto& arg : gArgs.GetUnsuitableSectionOnlyArgs()) { - InitWarning(strprintf(_("Config setting for %s only applied on %s network when in [%s] section."), arg, network, network)); - } - - // Warn if unrecognized section name are present in the config file. - for (const auto& section : gArgs.GetUnrecognizedSections()) { - InitWarning(strprintf("%s:%i " + _("Section [%s] is not recognized."), section.m_file, section.m_line, section.m_name)); - } } static std::string ResolveErrMsg(const char * const optname, const std::string& strBind) @@ -950,6 +937,19 @@ bool AppInitParameterInteraction() // also see: InitParameterInteraction() + // Warn if network-specific options (-addnode, -connect, etc) are + // specified in default section of config file, but not overridden + // on the command line or in this network's section of the config file. + std::string network = gArgs.GetChainName(); + for (const auto& arg : gArgs.GetUnsuitableSectionOnlyArgs()) { + return InitError(strprintf(_("Config setting for %s only applied on %s network when in [%s] section."), arg, network, network)); + } + + // Warn if unrecognized section name are present in the config file. + for (const auto& section : gArgs.GetUnrecognizedSections()) { + InitWarning(strprintf("%s:%i " + _("Section [%s] is not recognized."), section.m_file, section.m_line, section.m_name)); + } + if (!fs::is_directory(GetBlocksDir())) { return InitError(strprintf(_("Specified blocks directory \"%s\" does not exist."), gArgs.GetArg("-blocksdir", "").c_str())); } diff --git a/src/interfaces/chain.cpp b/src/interfaces/chain.cpp index 0c765f2092..f278e5de95 100644 --- a/src/interfaces/chain.cpp +++ b/src/interfaces/chain.cpp @@ -202,17 +202,11 @@ public: { m_notifications->BlockDisconnected(*block); } - void ChainStateFlushed(const CBlockLocator& locator) override { m_notifications->ChainStateFlushed(locator); } - void ResendWalletTransactions(int64_t best_block_time, CConnman*) override + void UpdatedBlockTip(const CBlockIndex* index, const CBlockIndex* fork_index, bool is_ibd) override { - // `cs_main` is always held when this method is called, so it is safe to - // call `assumeLocked`. This is awkward, and the `assumeLocked` method - // should be able to be removed entirely if `ResendWalletTransactions` - // is replaced by a wallet timer as suggested in - // https://github.com/bitcoin/bitcoin/issues/15619 - auto locked_chain = m_chain.assumeLocked(); - m_notifications->ResendWalletTransactions(*locked_chain, best_block_time); + m_notifications->UpdatedBlockTip(); } + void ChainStateFlushed(const CBlockLocator& locator) override { m_notifications->ChainStateFlushed(locator); } Chain& m_chain; Chain::Notifications* m_notifications; }; @@ -347,6 +341,7 @@ public: CAmount maxTxFee() override { return ::maxTxFee; } bool getPruneMode() override { return ::fPruneMode; } bool p2pEnabled() override { return g_connman != nullptr; } + bool isReadyToBroadcast() override { return !::fImporting && !::fReindex && !IsInitialBlockDownload(); } bool isInitialBlockDownload() override { return IsInitialBlockDownload(); } bool shutdownRequested() override { return ShutdownRequested(); } int64_t getAdjustedTime() override { return GetAdjustedTime(); } @@ -367,6 +362,13 @@ public: { return MakeUnique<RpcHandlerImpl>(command); } + void requestMempoolTransactions(Notifications& notifications) override + { + LOCK2(::cs_main, ::mempool.cs); + for (const CTxMemPoolEntry& entry : ::mempool.mapTx) { + notifications.TransactionAddedToMempool(entry.GetSharedTx()); + } + } }; } // namespace diff --git a/src/interfaces/chain.h b/src/interfaces/chain.h index ae546c4150..3ace5a0ece 100644 --- a/src/interfaces/chain.h +++ b/src/interfaces/chain.h @@ -224,6 +224,9 @@ public: //! Check if p2p enabled. virtual bool p2pEnabled() = 0; + //! Check if the node is ready to broadcast transactions. + virtual bool isReadyToBroadcast() = 0; + //! Check if in IBD. virtual bool isInitialBlockDownload() = 0; @@ -257,8 +260,8 @@ public: virtual void TransactionRemovedFromMempool(const CTransactionRef& ptx) {} virtual void BlockConnected(const CBlock& block, const std::vector<CTransactionRef>& tx_conflicted) {} virtual void BlockDisconnected(const CBlock& block) {} + virtual void UpdatedBlockTip() {} virtual void ChainStateFlushed(const CBlockLocator& locator) {} - virtual void ResendWalletTransactions(Lock& locked_chain, int64_t best_block_time) {} }; //! Register handler for notifications. @@ -270,6 +273,16 @@ public: //! Register handler for RPC. Command is not copied, so reference //! needs to remain valid until Handler is disconnected. virtual std::unique_ptr<Handler> handleRpc(const CRPCCommand& command) = 0; + + //! Synchronously send TransactionAddedToMempool notifications about all + //! current mempool transactions to the specified handler and return after + //! the last one is sent. These notifications aren't coordinated with async + //! notifications sent by handleNotifications, so out of date async + //! notifications from handleNotifications can arrive during and after + //! synchronous notifications from requestMempoolTransactions. Clients need + //! to be prepared to handle this by ignoring notifications about unknown + //! removed transactions and already added new transactions. + virtual void requestMempoolTransactions(Notifications& notifications) = 0; }; //! Interface to let node manage chain clients (wallets, or maybe tools for diff --git a/src/interfaces/wallet.cpp b/src/interfaces/wallet.cpp index 60173b29ac..0bb9739c9c 100644 --- a/src/interfaces/wallet.cpp +++ b/src/interfaces/wallet.cpp @@ -358,15 +358,16 @@ public: } WalletBalances getBalances() override { + const auto bal = m_wallet->GetBalance(); WalletBalances result; - result.balance = m_wallet->GetBalance(); - result.unconfirmed_balance = m_wallet->GetUnconfirmedBalance(); - result.immature_balance = m_wallet->GetImmatureBalance(); + result.balance = bal.m_mine_trusted; + result.unconfirmed_balance = bal.m_mine_untrusted_pending; + result.immature_balance = bal.m_mine_immature; result.have_watch_only = m_wallet->HaveWatchOnly(); if (result.have_watch_only) { - result.watch_only_balance = m_wallet->GetBalance(ISMINE_WATCH_ONLY); - result.unconfirmed_watch_only_balance = m_wallet->GetUnconfirmedWatchOnlyBalance(); - result.immature_watch_only_balance = m_wallet->GetImmatureWatchOnlyBalance(); + result.watch_only_balance = bal.m_watchonly_trusted; + result.unconfirmed_watch_only_balance = bal.m_watchonly_untrusted_pending; + result.immature_watch_only_balance = bal.m_watchonly_immature; } return result; } @@ -382,7 +383,7 @@ public: num_blocks = locked_chain->getHeight().get_value_or(-1); return true; } - CAmount getBalance() override { return m_wallet->GetBalance(); } + CAmount getBalance() override { return m_wallet->GetBalance().m_mine_trusted; } CAmount getAvailableBalance(const CCoinControl& coin_control) override { return m_wallet->GetAvailableBalance(&coin_control); diff --git a/src/net.cpp b/src/net.cpp index ccab4a1718..1335804b06 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -2627,7 +2627,6 @@ CNode::CNode(NodeId idIn, ServiceFlags nLocalServicesIn, int nMyStartingHeightIn { hSocket = hSocketIn; addrName = addrNameIn == "" ? addr.ToStringIPPort() : addrNameIn; - strSubVer = ""; hashContinue = uint256(); filterInventoryKnown.reset(); pfilter = MakeUnique<CBloomFilter>(); @@ -53,7 +53,7 @@ static const unsigned int MAX_LOCATOR_SZ = 101; static const unsigned int MAX_ADDR_TO_SEND = 1000; /** Maximum length of incoming protocol messages (no message over 4 MB is currently acceptable). */ static const unsigned int MAX_PROTOCOL_MESSAGE_LENGTH = 4 * 1000 * 1000; -/** Maximum length of strSubVer in `version` message */ +/** Maximum length of the user agent string in `version` message */ static const unsigned int MAX_SUBVERSION_LENGTH = 256; /** Maximum number of automatic outgoing nodes */ static const int MAX_OUTBOUND_CONNECTIONS = 8; @@ -650,12 +650,12 @@ public: // Bind address of our side of the connection const CAddress addrBind; std::atomic<int> nVersion{0}; - // strSubVer is whatever byte array we read from the wire. However, this field is intended - // to be printed out, displayed to humans in various forms and so on. So we sanitize it and - // store the sanitized version in cleanSubVer. The original should be used when dealing with - // the network or wire types and the cleaned string used when displayed or logged. - std::string strSubVer GUARDED_BY(cs_SubVer), cleanSubVer GUARDED_BY(cs_SubVer); - CCriticalSection cs_SubVer; // used for both cleanSubVer and strSubVer + RecursiveMutex cs_SubVer; + /** + * cleanSubVer is a sanitized string of the user agent byte array we read + * from the wire. This cleaned string can safely be logged or displayed. + */ + std::string cleanSubVer GUARDED_BY(cs_SubVer){}; bool m_prefer_evict{false}; // This peer is preferred for eviction. bool fWhitelisted{false}; // This peer can bypass DoS banning. bool fFeeler{false}; // If true this node is being used as a short lived feeler. @@ -739,6 +739,8 @@ public: CAmount lastSentFeeFilter{0}; int64_t nextSendTimeFeeFilter{0}; + std::set<uint256> orphan_work_set; + CNode(NodeId id, ServiceFlags nLocalServicesIn, int nMyStartingHeightIn, SOCKET hSocketIn, const CAddress &addrIn, uint64_t nKeyedNetGroupIn, uint64_t nLocalHostNonceIn, const CAddress &addrBindIn, const std::string &addrNameIn = "", bool fInboundIn = false); ~CNode(); CNode(const CNode&) = delete; diff --git a/src/net_processing.cpp b/src/net_processing.cpp index 0247b9cc7e..044fcc90cd 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -175,8 +175,6 @@ namespace { /** Expiration-time ordered list of (expire time, relay map entry) pairs. */ std::deque<std::pair<int64_t, MapRelay::iterator>> vRelayExpiration GUARDED_BY(cs_main); - std::atomic<int64_t> nTimeBestReceived(0); // Used only to inform the wallet of when we last received a block - struct IteratorComparator { template<typename I> @@ -1121,8 +1119,6 @@ void PeerLogicValidation::UpdatedBlockTip(const CBlockIndex *pindexNew, const CB }); connman->WakeMessageHandler(); } - - nTimeBestReceived = GetTime(); } /** @@ -1713,6 +1709,67 @@ bool static ProcessHeadersMessage(CNode *pfrom, CConnman *connman, const std::ve return true; } +void static ProcessOrphanTx(CConnman* connman, std::set<uint256>& orphan_work_set, std::list<CTransactionRef>& removed_txn) EXCLUSIVE_LOCKS_REQUIRED(cs_main, g_cs_orphans) +{ + AssertLockHeld(cs_main); + AssertLockHeld(g_cs_orphans); + std::set<NodeId> setMisbehaving; + bool done = false; + while (!done && !orphan_work_set.empty()) { + const uint256 orphanHash = *orphan_work_set.begin(); + orphan_work_set.erase(orphan_work_set.begin()); + + auto orphan_it = mapOrphanTransactions.find(orphanHash); + if (orphan_it == mapOrphanTransactions.end()) continue; + + const CTransactionRef porphanTx = orphan_it->second.tx; + const CTransaction& orphanTx = *porphanTx; + NodeId fromPeer = orphan_it->second.fromPeer; + bool fMissingInputs2 = false; + // Use a dummy CValidationState so someone can't setup nodes to counter-DoS based on orphan + // resolution (that is, feeding people an invalid transaction based on LegitTxX in order to get + // anyone relaying LegitTxX banned) + CValidationState stateDummy; + + if (setMisbehaving.count(fromPeer)) continue; + if (AcceptToMemoryPool(mempool, stateDummy, porphanTx, &fMissingInputs2, &removed_txn, false /* bypass_limits */, 0 /* nAbsurdFee */)) { + LogPrint(BCLog::MEMPOOL, " accepted orphan tx %s\n", orphanHash.ToString()); + RelayTransaction(orphanTx, connman); + for (unsigned int i = 0; i < orphanTx.vout.size(); i++) { + auto it_by_prev = mapOrphanTransactionsByPrev.find(COutPoint(orphanHash, i)); + if (it_by_prev != mapOrphanTransactionsByPrev.end()) { + for (const auto& elem : it_by_prev->second) { + orphan_work_set.insert(elem->first); + } + } + } + EraseOrphanTx(orphanHash); + done = true; + } else if (!fMissingInputs2) { + int nDos = 0; + if (stateDummy.IsInvalid(nDos) && nDos > 0) { + // Punish peer that gave us an invalid orphan tx + Misbehaving(fromPeer, nDos); + setMisbehaving.insert(fromPeer); + LogPrint(BCLog::MEMPOOL, " invalid orphan tx %s\n", orphanHash.ToString()); + } + // Has inputs but not accepted to mempool + // Probably non-standard or insufficient fee + LogPrint(BCLog::MEMPOOL, " removed orphan tx %s\n", orphanHash.ToString()); + if (!orphanTx.HasWitness() && !stateDummy.CorruptionPossible()) { + // Do not use rejection cache for witness transactions or + // witness-stripped transactions, as they can have been malleated. + // See https://github.com/bitcoin/bitcoin/issues/8279 for details. + assert(recentRejects); + recentRejects->insert(orphanHash); + } + EraseOrphanTx(orphanHash); + done = true; + } + mempool.check(pcoinsTip.get()); + } +} + bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStream& vRecv, int64_t nTimeReceived, const CChainParams& chainparams, CConnman* connman, const std::atomic<bool>& interruptMsgProc, bool enable_bip61) { LogPrint(BCLog::NET, "received: %s (%u bytes) peer=%d\n", SanitizeString(strCommand), vRecv.size(), pfrom->GetId()); @@ -1782,7 +1839,6 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr ServiceFlags nServices; int nVersion; int nSendVersion; - std::string strSubVer; std::string cleanSubVer; int nStartingHeight = -1; bool fRelay = true; @@ -1819,6 +1875,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr if (!vRecv.empty()) vRecv >> addrFrom >> nNonce; if (!vRecv.empty()) { + std::string strSubVer; vRecv >> LIMITED_STRING(strSubVer, MAX_SUBVERSION_LENGTH); cleanSubVer = SanitizeString(strSubVer); } @@ -1850,7 +1907,6 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr pfrom->SetAddrLocal(addrMe); { LOCK(pfrom->cs_SubVer); - pfrom->strSubVer = strSubVer; pfrom->cleanSubVer = cleanSubVer; } pfrom->nStartingHeight = nStartingHeight; @@ -2342,8 +2398,6 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr return true; } - std::deque<COutPoint> vWorkQueue; - std::vector<uint256> vEraseQueue; CTransactionRef ptx; vRecv >> ptx; const CTransaction& tx = *ptx; @@ -2368,7 +2422,12 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr mempool.check(pcoinsTip.get()); RelayTransaction(tx, connman); for (unsigned int i = 0; i < tx.vout.size(); i++) { - vWorkQueue.emplace_back(inv.hash, i); + auto it_by_prev = mapOrphanTransactionsByPrev.find(COutPoint(inv.hash, i)); + if (it_by_prev != mapOrphanTransactionsByPrev.end()) { + for (const auto& elem : it_by_prev->second) { + pfrom->orphan_work_set.insert(elem->first); + } + } } pfrom->nLastTXTime = GetTime(); @@ -2379,65 +2438,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr mempool.size(), mempool.DynamicMemoryUsage() / 1000); // Recursively process any orphan transactions that depended on this one - std::set<NodeId> setMisbehaving; - while (!vWorkQueue.empty()) { - auto itByPrev = mapOrphanTransactionsByPrev.find(vWorkQueue.front()); - vWorkQueue.pop_front(); - if (itByPrev == mapOrphanTransactionsByPrev.end()) - continue; - for (auto mi = itByPrev->second.begin(); - mi != itByPrev->second.end(); - ++mi) - { - const CTransactionRef& porphanTx = (*mi)->second.tx; - const CTransaction& orphanTx = *porphanTx; - const uint256& orphanHash = orphanTx.GetHash(); - NodeId fromPeer = (*mi)->second.fromPeer; - bool fMissingInputs2 = false; - // Use a dummy CValidationState so someone can't setup nodes to counter-DoS based on orphan - // resolution (that is, feeding people an invalid transaction based on LegitTxX in order to get - // anyone relaying LegitTxX banned) - CValidationState stateDummy; - - - if (setMisbehaving.count(fromPeer)) - continue; - if (AcceptToMemoryPool(mempool, stateDummy, porphanTx, &fMissingInputs2, &lRemovedTxn, false /* bypass_limits */, 0 /* nAbsurdFee */)) { - LogPrint(BCLog::MEMPOOL, " accepted orphan tx %s\n", orphanHash.ToString()); - RelayTransaction(orphanTx, connman); - for (unsigned int i = 0; i < orphanTx.vout.size(); i++) { - vWorkQueue.emplace_back(orphanHash, i); - } - vEraseQueue.push_back(orphanHash); - } - else if (!fMissingInputs2) - { - int nDos = 0; - if (stateDummy.IsInvalid(nDos) && nDos > 0) - { - // Punish peer that gave us an invalid orphan tx - Misbehaving(fromPeer, nDos); - setMisbehaving.insert(fromPeer); - LogPrint(BCLog::MEMPOOL, " invalid orphan tx %s\n", orphanHash.ToString()); - } - // Has inputs but not accepted to mempool - // Probably non-standard or insufficient fee - LogPrint(BCLog::MEMPOOL, " removed orphan tx %s\n", orphanHash.ToString()); - vEraseQueue.push_back(orphanHash); - if (!orphanTx.HasWitness() && !stateDummy.CorruptionPossible()) { - // Do not use rejection cache for witness transactions or - // witness-stripped transactions, as they can have been malleated. - // See https://github.com/bitcoin/bitcoin/issues/8279 for details. - assert(recentRejects); - recentRejects->insert(orphanHash); - } - } - mempool.check(pcoinsTip.get()); - } - } - - for (const uint256& hash : vEraseQueue) - EraseOrphanTx(hash); + ProcessOrphanTx(connman, pfrom->orphan_work_set, lRemovedTxn); } else if (fMissingInputs) { @@ -3169,11 +3170,21 @@ bool PeerLogicValidation::ProcessMessages(CNode* pfrom, std::atomic<bool>& inter if (!pfrom->vRecvGetData.empty()) ProcessGetData(pfrom, chainparams, connman, interruptMsgProc); + if (!pfrom->orphan_work_set.empty()) { + std::list<CTransactionRef> removed_txn; + LOCK2(cs_main, g_cs_orphans); + ProcessOrphanTx(connman, pfrom->orphan_work_set, removed_txn); + for (const CTransactionRef& removedTx : removed_txn) { + AddToCompactExtraTransactions(removedTx); + } + } + if (pfrom->fDisconnect) return false; // this maintains the order of responses if (!pfrom->vRecvGetData.empty()) return true; + if (!pfrom->orphan_work_set.empty()) return true; // Don't bother if send buffer is too full to respond anyway if (pfrom->fPauseSend) @@ -3535,14 +3546,6 @@ bool PeerLogicValidation::SendMessages(CNode* pto) } } - // Resend wallet transactions that haven't gotten in a block yet - // Except during reindex, importing and IBD, when old wallet - // transactions become unconfirmed and spams other nodes. - if (!fReindex && !fImporting && !IsInitialBlockDownload()) - { - GetMainSignals().Broadcast(nTimeBestReceived, connman); - } - // // Try sending block announcements via headers // diff --git a/src/psbt.cpp b/src/psbt.cpp index 0fb7d49d7d..184129e330 100644 --- a/src/psbt.cpp +++ b/src/psbt.cpp @@ -2,9 +2,14 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <coins.h> +#include <consensus/tx_verify.h> +#include <policy/policy.h> #include <psbt.h> #include <util/strencodings.h> +#include <numeric> + PartiallySignedTransaction::PartiallySignedTransaction(const CMutableTransaction& tx) : tx(tx) { inputs.resize(tx.vin.size()); @@ -205,7 +210,7 @@ void PSBTOutput::Merge(const PSBTOutput& output) if (redeem_script.empty() && !output.redeem_script.empty()) redeem_script = output.redeem_script; if (witness_script.empty() && !output.witness_script.empty()) witness_script = output.witness_script; } -bool PSBTInputSigned(PSBTInput& input) +bool PSBTInputSigned(const PSBTInput& input) { return !input.final_script_sig.empty() || !input.final_script_witness.IsNull(); } @@ -325,3 +330,162 @@ TransactionError CombinePSBTs(PartiallySignedTransaction& out, const std::vector return TransactionError::OK; } + +std::string PSBTRoleName(PSBTRole role) { + switch (role) { + case PSBTRole::UPDATER: return "updater"; + case PSBTRole::SIGNER: return "signer"; + case PSBTRole::FINALIZER: return "finalizer"; + case PSBTRole::EXTRACTOR: return "extractor"; + } +} + +PSBTAnalysis AnalyzePSBT(PartiallySignedTransaction psbtx) +{ + // Go through each input and build status + PSBTAnalysis result; + + bool calc_fee = true; + bool all_final = true; + bool only_missing_sigs = true; + bool only_missing_final = false; + CAmount in_amt = 0; + + result.inputs.resize(psbtx.tx->vin.size()); + + for (unsigned int i = 0; i < psbtx.tx->vin.size(); ++i) { + PSBTInput& input = psbtx.inputs[i]; + PSBTInputAnalysis& input_analysis = result.inputs[i]; + + // Check for a UTXO + CTxOut utxo; + if (psbtx.GetInputUTXO(utxo, i)) { + in_amt += utxo.nValue; + input_analysis.has_utxo = true; + } else { + input_analysis.has_utxo = false; + input_analysis.is_final = false; + input_analysis.next = PSBTRole::UPDATER; + calc_fee = false; + } + + // Check if it is final + if (!utxo.IsNull() && !PSBTInputSigned(input)) { + input_analysis.is_final = false; + all_final = false; + + // Figure out what is missing + SignatureData outdata; + bool complete = SignPSBTInput(DUMMY_SIGNING_PROVIDER, psbtx, i, 1, &outdata); + + // Things are missing + if (!complete) { + input_analysis.missing_pubkeys = outdata.missing_pubkeys; + input_analysis.missing_redeem_script = outdata.missing_redeem_script; + input_analysis.missing_witness_script = outdata.missing_witness_script; + input_analysis.missing_sigs = outdata.missing_sigs; + + // If we are only missing signatures and nothing else, then next is signer + if (outdata.missing_pubkeys.empty() && outdata.missing_redeem_script.IsNull() && outdata.missing_witness_script.IsNull() && !outdata.missing_sigs.empty()) { + input_analysis.next = PSBTRole::SIGNER; + } else { + only_missing_sigs = false; + input_analysis.next = PSBTRole::UPDATER; + } + } else { + only_missing_final = true; + input_analysis.next = PSBTRole::FINALIZER; + } + } else if (!utxo.IsNull()){ + input_analysis.is_final = true; + } + } + + if (all_final) { + only_missing_sigs = false; + result.next = PSBTRole::EXTRACTOR; + } + if (calc_fee) { + // Get the output amount + CAmount out_amt = std::accumulate(psbtx.tx->vout.begin(), psbtx.tx->vout.end(), CAmount(0), + [](CAmount a, const CTxOut& b) { + return a += b.nValue; + } + ); + + // Get the fee + CAmount fee = in_amt - out_amt; + result.fee = fee; + + // Estimate the size + CMutableTransaction mtx(*psbtx.tx); + CCoinsView view_dummy; + CCoinsViewCache view(&view_dummy); + bool success = true; + + for (unsigned int i = 0; i < psbtx.tx->vin.size(); ++i) { + PSBTInput& input = psbtx.inputs[i]; + Coin newcoin; + + if (!SignPSBTInput(DUMMY_SIGNING_PROVIDER, psbtx, i, 1, nullptr, true) || !psbtx.GetInputUTXO(newcoin.out, i)) { + success = false; + break; + } else { + mtx.vin[i].scriptSig = input.final_script_sig; + mtx.vin[i].scriptWitness = input.final_script_witness; + newcoin.nHeight = 1; + view.AddCoin(psbtx.tx->vin[i].prevout, std::move(newcoin), true); + } + } + + if (success) { + CTransaction ctx = CTransaction(mtx); + size_t size = GetVirtualTransactionSize(ctx, GetTransactionSigOpCost(ctx, view, STANDARD_SCRIPT_VERIFY_FLAGS)); + result.estimated_vsize = size; + // Estimate fee rate + CFeeRate feerate(fee, size); + result.estimated_feerate = feerate; + } + + if (only_missing_sigs) { + result.next = PSBTRole::SIGNER; + } else if (only_missing_final) { + result.next = PSBTRole::FINALIZER; + } else if (all_final) { + result.next = PSBTRole::EXTRACTOR; + } else { + result.next = PSBTRole::UPDATER; + } + } else { + result.next = PSBTRole::UPDATER; + } + + return result; +} + +bool DecodeBase64PSBT(PartiallySignedTransaction& psbt, const std::string& base64_tx, std::string& error) +{ + bool invalid; + std::string tx_data = DecodeBase64(base64_tx, &invalid); + if (invalid) { + error = "invalid base64"; + return false; + } + return DecodeRawPSBT(psbt, tx_data, error); +} + +bool DecodeRawPSBT(PartiallySignedTransaction& psbt, const std::string& tx_data, std::string& error) +{ + CDataStream ss_data(tx_data.data(), tx_data.data() + tx_data.size(), SER_NETWORK, PROTOCOL_VERSION); + try { + ss_data >> psbt; + if (!ss_data.empty()) { + error = "extra data after PSBT"; + return false; + } + } catch (const std::exception& e) { + error = e.what(); + return false; + } + return true; +} diff --git a/src/psbt.h b/src/psbt.h index c889dad361..fcb3337a53 100644 --- a/src/psbt.h +++ b/src/psbt.h @@ -7,6 +7,8 @@ #include <attributes.h> #include <node/transaction.h> +#include <optional.h> +#include <policy/feerate.h> #include <primitives/transaction.h> #include <pubkey.h> #include <script/sign.h> @@ -548,8 +550,42 @@ struct PartiallySignedTransaction } }; +enum class PSBTRole { + UPDATER, + SIGNER, + FINALIZER, + EXTRACTOR +}; + +/** + * Holds an analysis of one input from a PSBT + */ +struct PSBTInputAnalysis { + bool has_utxo; //!< Whether we have UTXO information for this input + bool is_final; //!< Whether the input has all required information including signatures + PSBTRole next; //!< Which of the BIP 174 roles needs to handle this input next + + std::vector<CKeyID> missing_pubkeys; //!< Pubkeys whose BIP32 derivation path is missing + std::vector<CKeyID> missing_sigs; //!< Pubkeys whose signatures are missing + uint160 missing_redeem_script; //!< Hash160 of redeem script, if missing + uint256 missing_witness_script; //!< SHA256 of witness script, if missing +}; + +/** + * Holds the results of AnalyzePSBT (miscellaneous information about a PSBT) + */ +struct PSBTAnalysis { + Optional<size_t> estimated_vsize; //!< Estimated weight of the transaction + Optional<CFeeRate> estimated_feerate; //!< Estimated feerate (fee / weight) of the transaction + Optional<CAmount> fee; //!< Amount of fee being paid by the transaction + std::vector<PSBTInputAnalysis> inputs; //!< More information about the individual inputs of the transaction + PSBTRole next; //!< Which of the BIP 174 roles needs to handle the transaction next +}; + +std::string PSBTRoleName(PSBTRole role); + /** Checks whether a PSBTInput is already signed. */ -bool PSBTInputSigned(PSBTInput& input); +bool PSBTInputSigned(const PSBTInput& input); /** Signs a PSBTInput, verifying that all provided data matches what is being signed. */ bool SignPSBTInput(const SigningProvider& provider, PartiallySignedTransaction& psbt, int index, int sighash = SIGHASH_ALL, SignatureData* out_sigdata = nullptr, bool use_dummy = false); @@ -580,4 +616,17 @@ bool FinalizeAndExtractPSBT(PartiallySignedTransaction& psbtx, CMutableTransacti */ NODISCARD TransactionError CombinePSBTs(PartiallySignedTransaction& out, const std::vector<PartiallySignedTransaction>& psbtxs); +/** + * Provides helpful miscellaneous information about where a PSBT is in the signing workflow. + * + * @param[in] psbtx the PSBT to analyze + * @return A PSBTAnalysis with information about the provided PSBT. + */ +PSBTAnalysis AnalyzePSBT(PartiallySignedTransaction psbtx); + +//! Decode a base64ed PSBT into a PartiallySignedTransaction +NODISCARD bool DecodeBase64PSBT(PartiallySignedTransaction& decoded_psbt, const std::string& base64_psbt, std::string& error); +//! Decode a raw (binary blob) PSBT into a PartiallySignedTransaction +NODISCARD bool DecodeRawPSBT(PartiallySignedTransaction& decoded_psbt, const std::string& raw_psbt, std::string& error); + #endif // BITCOIN_PSBT_H diff --git a/src/qt/forms/receivecoinsdialog.ui b/src/qt/forms/receivecoinsdialog.ui index 2f916d0b44..8876ea1337 100644 --- a/src/qt/forms/receivecoinsdialog.ui +++ b/src/qt/forms/receivecoinsdialog.ui @@ -189,7 +189,7 @@ </widget> </item> <item> - <widget class="QCheckBox" name="useBech32"> + <widget class="QCheckBox" name="useLegacyAddress"> <property name="sizePolicy"> <sizepolicy hsizetype="Fixed" vsizetype="Fixed"> <horstretch>0</horstretch> @@ -206,10 +206,10 @@ <enum>Qt::StrongFocus</enum> </property> <property name="toolTip"> - <string>Native segwit addresses (aka Bech32 or BIP-173) reduce your transaction fees later on and offer better protection against typos, but old wallets don't support them. When unchecked, an address compatible with older wallets will be created instead.</string> + <string>Native segwit addresses (aka Bech32 or BIP-173) reduce your transaction fees later on and offer better protection against typos, but old wallets don't support them. When checked, an address compatible with older wallets will be created instead.</string> </property> <property name="text"> - <string>Generate native segwit (Bech32) address</string> + <string>Generate legacy address</string> </property> </widget> </item> @@ -360,7 +360,7 @@ <tabstops> <tabstop>reqLabel</tabstop> <tabstop>reqAmount</tabstop> - <tabstop>useBech32</tabstop> + <tabstop>useLegacyAddress</tabstop> <tabstop>reqMessage</tabstop> <tabstop>receiveButton</tabstop> <tabstop>clearButton</tabstop> diff --git a/src/qt/receivecoinsdialog.cpp b/src/qt/receivecoinsdialog.cpp index 22a79a12bb..fc58090dcd 100644 --- a/src/qt/receivecoinsdialog.cpp +++ b/src/qt/receivecoinsdialog.cpp @@ -95,9 +95,9 @@ void ReceiveCoinsDialog::setModel(WalletModel *_model) columnResizingFixer = new GUIUtil::TableViewLastColumnResizingFixer(tableView, AMOUNT_MINIMUM_COLUMN_WIDTH, DATE_COLUMN_WIDTH, this); if (model->wallet().getDefaultAddressType() == OutputType::BECH32) { - ui->useBech32->setCheckState(Qt::Checked); + ui->useLegacyAddress->setCheckState(Qt::Unchecked); } else { - ui->useBech32->setCheckState(Qt::Unchecked); + ui->useLegacyAddress->setCheckState(Qt::Checked); } // Set the button to be enabled or disabled based on whether the wallet can give out new addresses. @@ -150,7 +150,7 @@ void ReceiveCoinsDialog::on_receiveButton_clicked() QString label = ui->reqLabel->text(); /* Generate new receiving address */ OutputType address_type; - if (ui->useBech32->isChecked()) { + if (!ui->useLegacyAddress->isChecked()) { address_type = OutputType::BECH32; } else { address_type = model->wallet().getDefaultAddressType(); diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index 1827aec637..d35f458b2e 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -378,7 +378,9 @@ static UniValue getdifficulty(const JSONRPCRequest& request) static std::string EntryDescriptionString() { - return " \"size\" : n, (numeric) virtual transaction size as defined in BIP 141. This is different from actual serialized size for witness transactions as witness data is discounted.\n" + return " \"vsize\" : n, (numeric) virtual transaction size as defined in BIP 141. This is different from actual serialized size for witness transactions as witness data is discounted.\n" + " \"size\" : n, (numeric) (DEPRECATED) same as vsize. Only returned if bitcoind is started with -deprecatedrpc=size\n" + " size will be completely removed in v0.20.\n" " \"fee\" : n, (numeric) transaction fee in " + CURRENCY_UNIT + " (DEPRECATED)\n" " \"modifiedfee\" : n, (numeric) transaction fee with fee deltas used for mining priority (DEPRECATED)\n" " \"time\" : n, (numeric) local time transaction entered pool in seconds since 1 Jan 1970 GMT\n" @@ -416,7 +418,8 @@ static void entryToJSON(const CTxMemPool& pool, UniValue& info, const CTxMemPool fees.pushKV("descendant", ValueFromAmount(e.GetModFeesWithDescendants())); info.pushKV("fees", fees); - info.pushKV("size", (int)e.GetTxSize()); + info.pushKV("vsize", (int)e.GetTxSize()); + if (IsDeprecatedRPCEnabled("size")) info.pushKV("size", (int)e.GetTxSize()); info.pushKV("fee", ValueFromAmount(e.GetFee())); info.pushKV("modifiedfee", ValueFromAmount(e.GetModifiedFee())); info.pushKV("time", e.GetTime()); diff --git a/src/rpc/mining.cpp b/src/rpc/mining.cpp index f2acb8fbf5..480d610235 100644 --- a/src/rpc/mining.cpp +++ b/src/rpc/mining.cpp @@ -444,10 +444,10 @@ static UniValue getblocktemplate(const JSONRPCRequest& request) throw JSONRPCError(RPC_CLIENT_P2P_DISABLED, "Error: Peer-to-peer functionality missing or disabled"); if (g_connman->GetNodeCount(CConnman::CONNECTIONS_ALL) == 0) - throw JSONRPCError(RPC_CLIENT_NOT_CONNECTED, "Bitcoin is not connected!"); + throw JSONRPCError(RPC_CLIENT_NOT_CONNECTED, PACKAGE_NAME " is not connected!"); if (IsInitialBlockDownload()) - throw JSONRPCError(RPC_CLIENT_IN_INITIAL_DOWNLOAD, "Bitcoin is downloading blocks..."); + throw JSONRPCError(RPC_CLIENT_IN_INITIAL_DOWNLOAD, PACKAGE_NAME " is in initial sync and waiting for blocks..."); static unsigned int nTransactionsUpdatedLast; diff --git a/src/rpc/rawtransaction.cpp b/src/rpc/rawtransaction.cpp index e3d8fb4e81..6ecde84cdb 100644 --- a/src/rpc/rawtransaction.cpp +++ b/src/rpc/rawtransaction.cpp @@ -40,6 +40,11 @@ #include <univalue.h> +/** High fee for sendrawtransaction and testmempoolaccept. + * By default, transaction with a fee higher than this will be rejected by the + * RPCs. This can be overriden with the maxfeerate argument. + */ +constexpr static CAmount DEFAULT_MAX_RAW_TX_FEE{COIN / 10}; static void TxToJSON(const CTransaction& tx, const uint256 hashBlock, UniValue& entry) { @@ -610,33 +615,54 @@ static UniValue decoderawtransaction(const JSONRPCRequest& request) return result; } +static std::string GetAllOutputTypes() +{ + std::string ret; + for (int i = TX_NONSTANDARD; i <= TX_WITNESS_UNKNOWN; ++i) { + if (i != TX_NONSTANDARD) ret += ", "; + ret += GetTxnOutputType(static_cast<txnouttype>(i)); + } + return ret; +} + static UniValue decodescript(const JSONRPCRequest& request) { - if (request.fHelp || request.params.size() != 1) - throw std::runtime_error( - RPCHelpMan{"decodescript", + const RPCHelpMan help{"decodescript", "\nDecode a hex-encoded script.\n", { {"hexstring", RPCArg::Type::STR_HEX, RPCArg::Optional::NO, "the hex-encoded script"}, }, RPCResult{ "{\n" - " \"asm\":\"asm\", (string) Script public key\n" - " \"hex\":\"hex\", (string) hex-encoded public key\n" - " \"type\":\"type\", (string) The output type\n" - " \"reqSigs\": n, (numeric) The required signatures\n" - " \"addresses\": [ (json array of string)\n" - " \"address\" (string) bitcoin address\n" + " \"asm\":\"asm\", (string) Script public key\n" + " \"type\":\"type\", (string) The output type (e.g. "+GetAllOutputTypes()+")\n" + " \"reqSigs\": n, (numeric) The required signatures\n" + " \"addresses\": [ (json array of string)\n" + " \"address\" (string) bitcoin address\n" " ,...\n" " ],\n" - " \"p2sh\",\"address\" (string) address of P2SH script wrapping this redeem script (not returned if the script is already a P2SH).\n" + " \"p2sh\":\"str\" (string) address of P2SH script wrapping this redeem script (not returned if the script is already a P2SH).\n" + " \"segwit\": { (json object) Result of a witness script public key wrapping this redeem script (not returned if the script is a P2SH or witness).\n" + " \"asm\":\"str\", (string) String representation of the script public key\n" + " \"hex\":\"hexstr\", (string) Hex string of the script public key\n" + " \"type\":\"str\", (string) The type of the script public key (e.g. witness_v0_keyhash or witness_v0_scripthash)\n" + " \"reqSigs\": n, (numeric) The required signatures (always 1)\n" + " \"addresses\": [ (json array of string) (always length 1)\n" + " \"address\" (string) segwit address\n" + " ,...\n" + " ],\n" + " \"p2sh-segwit\":\"str\" (string) address of the P2SH script wrapping this witness redeem script.\n" "}\n" }, RPCExamples{ HelpExampleCli("decodescript", "\"hexstring\"") + HelpExampleRpc("decodescript", "\"hexstring\"") }, - }.ToString()); + }; + + if (request.fHelp || !help.IsValidNumArgs(request.params.size())) { + throw std::runtime_error(help.ToString()); + } RPCTypeCheck(request.params, {UniValue::VSTR}); @@ -648,7 +674,7 @@ static UniValue decodescript(const JSONRPCRequest& request) } else { // Empty scripts are valid } - ScriptPubKeyToUniv(script, r, false); + ScriptPubKeyToUniv(script, r, /* fIncludeHex */ false); UniValue type; type = find_value(r, "type"); @@ -682,7 +708,7 @@ static UniValue decodescript(const JSONRPCRequest& request) // Newer segwit program versions should be considered when then become available. segwitScr = GetScriptForDestination(WitnessV0ScriptHash(script)); } - ScriptPubKeyToUniv(segwitScr, sr, true); + ScriptPubKeyToUniv(segwitScr, sr, /* fIncludeHex */ true); sr.pushKV("p2sh-segwit", EncodeDestination(CScriptID(segwitScr))); r.pushKV("segwit", sr); } @@ -964,7 +990,7 @@ static UniValue signrawtransactionwithkey(const JSONRPCRequest& request) {"scriptPubKey", RPCArg::Type::STR_HEX, RPCArg::Optional::NO, "script key"}, {"redeemScript", RPCArg::Type::STR_HEX, RPCArg::Optional::OMITTED, "(required for P2SH) redeem script"}, {"witnessScript", RPCArg::Type::STR_HEX, RPCArg::Optional::OMITTED, "(required for P2WSH or P2SH-P2WSH) witness script"}, - {"amount", RPCArg::Type::AMOUNT, RPCArg::Optional::NO, "The amount spent"}, + {"amount", RPCArg::Type::AMOUNT, RPCArg::Optional::OMITTED, "(required for Segwit inputs) the amount spent"}, }, }, }, @@ -1023,14 +1049,14 @@ static UniValue signrawtransactionwithkey(const JSONRPCRequest& request) static UniValue sendrawtransaction(const JSONRPCRequest& request) { - if (request.fHelp || request.params.size() < 1 || request.params.size() > 2) - throw std::runtime_error( - RPCHelpMan{"sendrawtransaction", + const RPCHelpMan help{"sendrawtransaction", "\nSubmits raw transaction (serialized, hex-encoded) to local node and network.\n" "\nAlso see createrawtransaction and signrawtransactionwithkey calls.\n", { {"hexstring", RPCArg::Type::STR_HEX, RPCArg::Optional::NO, "The hex string of the raw transaction"}, - {"maxfeerate", RPCArg::Type::AMOUNT, /* default */ FormatMoney(maxTxFee), "Reject transactions whose fee rate is higher than the specified value, expressed in " + CURRENCY_UNIT + "/kB\n"}, + {"maxfeerate", RPCArg::Type::AMOUNT, /* default */ FormatMoney(DEFAULT_MAX_RAW_TX_FEE), + "Reject transactions whose fee rate is higher than the specified value, expressed in " + CURRENCY_UNIT + + "/kB.\nSet to 0 to accept any fee rate.\n"}, }, RPCResult{ "\"hex\" (string) The transaction hash in hex\n" @@ -1045,7 +1071,11 @@ static UniValue sendrawtransaction(const JSONRPCRequest& request) "\nAs a JSON-RPC call\n" + HelpExampleRpc("sendrawtransaction", "\"signedhex\"") }, - }.ToString()); + }; + + if (request.fHelp || !help.IsValidNumArgs(request.params.size())) { + throw std::runtime_error(help.ToString()); + } RPCTypeCheck(request.params, { UniValue::VSTR, @@ -1058,7 +1088,7 @@ static UniValue sendrawtransaction(const JSONRPCRequest& request) throw JSONRPCError(RPC_DESERIALIZATION_ERROR, "TX decode failed"); CTransactionRef tx(MakeTransactionRef(std::move(mtx))); - CAmount max_raw_tx_fee = maxTxFee; + CAmount max_raw_tx_fee = DEFAULT_MAX_RAW_TX_FEE; // TODO: temporary migration code for old clients. Remove in v0.20 if (request.params[1].isBool()) { throw JSONRPCError(RPC_INVALID_PARAMETER, "Second argument must be numeric (maxfeerate) and no longer supports a boolean. To allow a transaction with high fees, set maxfeerate to 0."); @@ -1085,9 +1115,7 @@ static UniValue sendrawtransaction(const JSONRPCRequest& request) static UniValue testmempoolaccept(const JSONRPCRequest& request) { - if (request.fHelp || request.params.size() < 1 || request.params.size() > 2) { - throw std::runtime_error( - RPCHelpMan{"testmempoolaccept", + const RPCHelpMan help{"testmempoolaccept", "\nReturns result of mempool acceptance tests indicating if raw transaction (serialized, hex-encoded) would be accepted by mempool.\n" "\nThis checks if the transaction violates the consensus or policy rules.\n" "\nSee sendrawtransaction call.\n", @@ -1098,7 +1126,7 @@ static UniValue testmempoolaccept(const JSONRPCRequest& request) {"rawtx", RPCArg::Type::STR_HEX, RPCArg::Optional::OMITTED, ""}, }, }, - {"maxfeerate", RPCArg::Type::AMOUNT, /* default */ FormatMoney(maxTxFee), "Reject transactions whose fee rate is higher than the specified value, expressed in " + CURRENCY_UNIT + "/kB\n"}, + {"maxfeerate", RPCArg::Type::AMOUNT, /* default */ FormatMoney(DEFAULT_MAX_RAW_TX_FEE), "Reject transactions whose fee rate is higher than the specified value, expressed in " + CURRENCY_UNIT + "/kB\n"}, }, RPCResult{ "[ (array) The result of the mempool acceptance test for each raw transaction in the input array.\n" @@ -1120,7 +1148,10 @@ static UniValue testmempoolaccept(const JSONRPCRequest& request) "\nAs a JSON-RPC call\n" + HelpExampleRpc("testmempoolaccept", "[\"signedhex\"]") }, - }.ToString()); + }; + + if (request.fHelp || !help.IsValidNumArgs(request.params.size())) { + throw std::runtime_error(help.ToString()); } RPCTypeCheck(request.params, { @@ -1139,7 +1170,7 @@ static UniValue testmempoolaccept(const JSONRPCRequest& request) CTransactionRef tx(MakeTransactionRef(std::move(mtx))); const uint256& tx_hash = tx->GetHash(); - CAmount max_raw_tx_fee = maxTxFee; + CAmount max_raw_tx_fee = DEFAULT_MAX_RAW_TX_FEE; // TODO: temporary migration code for old clients. Remove in v0.20 if (request.params[1].isBool()) { throw JSONRPCError(RPC_INVALID_PARAMETER, "Second argument must be numeric (maxfeerate) and no longer supports a boolean. To allow a transaction with high fees, set maxfeerate to 0."); @@ -1520,7 +1551,6 @@ UniValue combinepsbt(const JSONRPCRequest& request) throw JSONRPCTransactionError(error); } - UniValue result(UniValue::VOBJ); CDataStream ssTx(SER_NETWORK, PROTOCOL_VERSION); ssTx << merged_psbt; return EncodeBase64((unsigned char*)ssTx.data(), ssTx.size()); @@ -1915,148 +1945,56 @@ UniValue analyzepsbt(const JSONRPCRequest& request) throw JSONRPCError(RPC_DESERIALIZATION_ERROR, strprintf("TX decode failed %s", error)); } - // Go through each input and build status + PSBTAnalysis psbta = AnalyzePSBT(psbtx); + UniValue result(UniValue::VOBJ); UniValue inputs_result(UniValue::VARR); - bool calc_fee = true; - bool all_final = true; - bool only_missing_sigs = true; - bool only_missing_final = false; - CAmount in_amt = 0; - for (unsigned int i = 0; i < psbtx.tx->vin.size(); ++i) { - PSBTInput& input = psbtx.inputs[i]; + for (const auto& input : psbta.inputs) { UniValue input_univ(UniValue::VOBJ); UniValue missing(UniValue::VOBJ); - // Check for a UTXO - CTxOut utxo; - if (psbtx.GetInputUTXO(utxo, i)) { - in_amt += utxo.nValue; - input_univ.pushKV("has_utxo", true); - } else { - input_univ.pushKV("has_utxo", false); - input_univ.pushKV("is_final", false); - input_univ.pushKV("next", "updater"); - calc_fee = false; - } + input_univ.pushKV("has_utxo", input.has_utxo); + input_univ.pushKV("is_final", input.is_final); + input_univ.pushKV("next", PSBTRoleName(input.next)); - // Check if it is final - if (!utxo.IsNull() && !PSBTInputSigned(input)) { - input_univ.pushKV("is_final", false); - all_final = false; - - // Figure out what is missing - SignatureData outdata; - bool complete = SignPSBTInput(DUMMY_SIGNING_PROVIDER, psbtx, i, 1, &outdata); - - // Things are missing - if (!complete) { - if (!outdata.missing_pubkeys.empty()) { - // Missing pubkeys - UniValue missing_pubkeys_univ(UniValue::VARR); - for (const CKeyID& pubkey : outdata.missing_pubkeys) { - missing_pubkeys_univ.push_back(HexStr(pubkey)); - } - missing.pushKV("pubkeys", missing_pubkeys_univ); - } - if (!outdata.missing_redeem_script.IsNull()) { - // Missing redeemScript - missing.pushKV("redeemscript", HexStr(outdata.missing_redeem_script)); - } - if (!outdata.missing_witness_script.IsNull()) { - // Missing witnessScript - missing.pushKV("witnessscript", HexStr(outdata.missing_witness_script)); - } - if (!outdata.missing_sigs.empty()) { - // Missing sigs - UniValue missing_sigs_univ(UniValue::VARR); - for (const CKeyID& pubkey : outdata.missing_sigs) { - missing_sigs_univ.push_back(HexStr(pubkey)); - } - missing.pushKV("signatures", missing_sigs_univ); - } - input_univ.pushKV("missing", missing); - - // If we are only missing signatures and nothing else, then next is signer - if (outdata.missing_pubkeys.empty() && outdata.missing_redeem_script.IsNull() && outdata.missing_witness_script.IsNull() && !outdata.missing_sigs.empty()) { - input_univ.pushKV("next", "signer"); - } else { - only_missing_sigs = false; - input_univ.pushKV("next", "updater"); - } - } else { - only_missing_final = true; - input_univ.pushKV("next", "finalizer"); + if (!input.missing_pubkeys.empty()) { + UniValue missing_pubkeys_univ(UniValue::VARR); + for (const CKeyID& pubkey : input.missing_pubkeys) { + missing_pubkeys_univ.push_back(HexStr(pubkey)); + } + missing.pushKV("pubkeys", missing_pubkeys_univ); + } + if (!input.missing_redeem_script.IsNull()) { + missing.pushKV("redeemscript", HexStr(input.missing_redeem_script)); + } + if (!input.missing_witness_script.IsNull()) { + missing.pushKV("witnessscript", HexStr(input.missing_witness_script)); + } + if (!input.missing_sigs.empty()) { + UniValue missing_sigs_univ(UniValue::VARR); + for (const CKeyID& pubkey : input.missing_sigs) { + missing_sigs_univ.push_back(HexStr(pubkey)); } - } else if (!utxo.IsNull()){ - input_univ.pushKV("is_final", true); + missing.pushKV("signatures", missing_sigs_univ); + } + if (!missing.getKeys().empty()) { + input_univ.pushKV("missing", missing); } inputs_result.push_back(input_univ); } result.pushKV("inputs", inputs_result); - if (all_final) { - only_missing_sigs = false; - result.pushKV("next", "extractor"); + if (psbta.estimated_vsize != nullopt) { + result.pushKV("estimated_vsize", (int)*psbta.estimated_vsize); } - if (calc_fee) { - // Get the output amount - CAmount out_amt = std::accumulate(psbtx.tx->vout.begin(), psbtx.tx->vout.end(), CAmount(0), - [](CAmount a, const CTxOut& b) { - return a += b.nValue; - } - ); - - // Get the fee - CAmount fee = in_amt - out_amt; - - // Estimate the size - CMutableTransaction mtx(*psbtx.tx); - CCoinsView view_dummy; - CCoinsViewCache view(&view_dummy); - bool success = true; - - for (unsigned int i = 0; i < psbtx.tx->vin.size(); ++i) { - PSBTInput& input = psbtx.inputs[i]; - if (SignPSBTInput(DUMMY_SIGNING_PROVIDER, psbtx, i, 1, nullptr, true)) { - mtx.vin[i].scriptSig = input.final_script_sig; - mtx.vin[i].scriptWitness = input.final_script_witness; - - Coin newcoin; - if (!psbtx.GetInputUTXO(newcoin.out, i)) { - success = false; - break; - } - newcoin.nHeight = 1; - view.AddCoin(psbtx.tx->vin[i].prevout, std::move(newcoin), true); - } else { - success = false; - break; - } - } - - if (success) { - CTransaction ctx = CTransaction(mtx); - size_t size = GetVirtualTransactionSize(ctx, GetTransactionSigOpCost(ctx, view, STANDARD_SCRIPT_VERIFY_FLAGS)); - result.pushKV("estimated_vsize", (int)size); - // Estimate fee rate - CFeeRate feerate(fee, size); - result.pushKV("estimated_feerate", ValueFromAmount(feerate.GetFeePerK())); - } - result.pushKV("fee", ValueFromAmount(fee)); - - if (only_missing_sigs) { - result.pushKV("next", "signer"); - } else if (only_missing_final) { - result.pushKV("next", "finalizer"); - } else if (all_final) { - result.pushKV("next", "extractor"); - } else { - result.pushKV("next", "updater"); - } - } else { - result.pushKV("next", "updater"); + if (psbta.estimated_feerate != nullopt) { + result.pushKV("estimated_feerate", ValueFromAmount(psbta.estimated_feerate->GetFeePerK())); } + if (psbta.fee != nullopt) { + result.pushKV("fee", ValueFromAmount(*psbta.fee)); + } + result.pushKV("next", PSBTRoleName(psbta.next)); + return result; } diff --git a/src/rpc/server.cpp b/src/rpc/server.cpp index e803fabcc6..276db453cb 100644 --- a/src/rpc/server.cpp +++ b/src/rpc/server.cpp @@ -297,8 +297,20 @@ static UniValue getrpcinfo(const JSONRPCRequest& request) RPCHelpMan{"getrpcinfo", "\nReturns details of the RPC server.\n", {}, - RPCResults{}, - RPCExamples{""}, + RPCResult{ + "{\n" + " \"active_commands\" (array) All active commands\n" + " [\n" + " { (object) Information about an active command\n" + " \"method\" (string) The name of the RPC command \n" + " \"duration\" (numeric) The running time in microseconds\n" + " },...\n" + " ]\n" + "}\n" + }, + RPCExamples{ + HelpExampleCli("getrpcinfo", "") + + HelpExampleRpc("getrpcinfo", "")}, }.ToString() ); } diff --git a/src/rpc/util.cpp b/src/rpc/util.cpp index 10979b43b0..4cd63fada6 100644 --- a/src/rpc/util.cpp +++ b/src/rpc/util.cpp @@ -165,6 +165,10 @@ UniValue JSONRPCTransactionError(TransactionError terr, const std::string& err_s } } +/** + * A pair of strings that can be aligned (through padding) with other Sections + * later on + */ struct Section { Section(const std::string& left, const std::string& right) : m_left{left}, m_right{right} {} @@ -172,6 +176,10 @@ struct Section { const std::string m_right; }; +/** + * Keeps track of RPCArgs by transforming them into sections for the purpose + * of serializing everything to a single string + */ struct Sections { std::vector<Section> m_sections; size_t m_max_pad{0}; @@ -182,16 +190,26 @@ struct Sections { m_sections.push_back(s); } + /** + * Serializing RPCArgs depends on the outer type. Only arrays and + * dictionaries can be nested in json. The top-level outer type is "named + * arguments", a mix between a dictionary and arrays. + */ enum class OuterType { ARR, OBJ, NAMED_ARG, // Only set on first recursion }; + /** + * Recursive helper to translate an RPCArg into sections + */ void Push(const RPCArg& arg, const size_t current_indent = 5, const OuterType outer_type = OuterType::NAMED_ARG) { const auto indent = std::string(current_indent, ' '); const auto indent_next = std::string(current_indent + 2, ' '); + const bool push_name{outer_type == OuterType::OBJ}; // Dictionary keys must have a name + switch (arg.m_type) { case RPCArg::Type::STR_HEX: case RPCArg::Type::STR: @@ -201,10 +219,10 @@ struct Sections { case RPCArg::Type::BOOL: { if (outer_type == OuterType::NAMED_ARG) return; // Nothing more to do for non-recursive types on first recursion auto left = indent; - if (arg.m_type_str.size() != 0 && outer_type == OuterType::OBJ) { + if (arg.m_type_str.size() != 0 && push_name) { left += "\"" + arg.m_name + "\": " + arg.m_type_str.at(0); } else { - left += outer_type == OuterType::OBJ ? arg.ToStringObj(/* oneline */ false) : arg.ToString(/* oneline */ false); + left += push_name ? arg.ToStringObj(/* oneline */ false) : arg.ToString(/* oneline */ false); } left += ","; PushSection({left, arg.ToDescriptionString()}); @@ -213,7 +231,7 @@ struct Sections { case RPCArg::Type::OBJ: case RPCArg::Type::OBJ_USER_KEYS: { const auto right = outer_type == OuterType::NAMED_ARG ? "" : arg.ToDescriptionString(); - PushSection({indent + "{", right}); + PushSection({indent + (push_name ? "\"" + arg.m_name + "\": " : "") + "{", right}); for (const auto& arg_inner : arg.m_inner) { Push(arg_inner, current_indent + 2, OuterType::OBJ); } @@ -225,7 +243,7 @@ struct Sections { } case RPCArg::Type::ARR: { auto left = indent; - left += outer_type == OuterType::OBJ ? "\"" + arg.m_name + "\": " : ""; + left += push_name ? "\"" + arg.m_name + "\": " : ""; left += "["; const auto right = outer_type == OuterType::NAMED_ARG ? "" : arg.ToDescriptionString(); PushSection({left, right}); @@ -241,6 +259,9 @@ struct Sections { } } + /** + * Concatenate all sections with proper padding + */ std::string ToString() const { std::string ret; diff --git a/src/script/descriptor.cpp b/src/script/descriptor.cpp index 43448d7222..a333d4d4ac 100644 --- a/src/script/descriptor.cpp +++ b/src/script/descriptor.cpp @@ -436,7 +436,7 @@ public: pubkeys.reserve(entries.size()); for (auto& entry : entries) { pubkeys.push_back(entry.first); - out.origins.emplace(entry.first.GetID(), std::move(entry.second)); + out.origins.emplace(entry.first.GetID(), std::make_pair<CPubKey, KeyOriginInfo>(CPubKey(entry.first), std::move(entry.second))); } if (m_script_arg) { for (const auto& subscript : subscripts) { diff --git a/src/script/sign.cpp b/src/script/sign.cpp index 320956d0c4..36dd68a3d8 100644 --- a/src/script/sign.cpp +++ b/src/script/sign.cpp @@ -483,7 +483,13 @@ bool HidingSigningProvider::GetKeyOrigin(const CKeyID& keyid, KeyOriginInfo& inf bool FlatSigningProvider::GetCScript(const CScriptID& scriptid, CScript& script) const { return LookupHelper(scripts, scriptid, script); } bool FlatSigningProvider::GetPubKey(const CKeyID& keyid, CPubKey& pubkey) const { return LookupHelper(pubkeys, keyid, pubkey); } -bool FlatSigningProvider::GetKeyOrigin(const CKeyID& keyid, KeyOriginInfo& info) const { return LookupHelper(origins, keyid, info); } +bool FlatSigningProvider::GetKeyOrigin(const CKeyID& keyid, KeyOriginInfo& info) const +{ + std::pair<CPubKey, KeyOriginInfo> out; + bool ret = LookupHelper(origins, keyid, out); + if (ret) info = std::move(out.second); + return ret; +} bool FlatSigningProvider::GetKey(const CKeyID& keyid, CKey& key) const { return LookupHelper(keys, keyid, key); } FlatSigningProvider Merge(const FlatSigningProvider& a, const FlatSigningProvider& b) diff --git a/src/script/sign.h b/src/script/sign.h index 491fb54c45..f746325b90 100644 --- a/src/script/sign.h +++ b/src/script/sign.h @@ -77,7 +77,7 @@ struct FlatSigningProvider final : public SigningProvider { std::map<CScriptID, CScript> scripts; std::map<CKeyID, CPubKey> pubkeys; - std::map<CKeyID, KeyOriginInfo> origins; + std::map<CKeyID, std::pair<CPubKey, KeyOriginInfo>> origins; std::map<CKeyID, CKey> keys; bool GetCScript(const CScriptID& scriptid, CScript& script) const override; diff --git a/src/test/crypto_tests.cpp b/src/test/crypto_tests.cpp index 86cb00a78f..2cc38a0679 100644 --- a/src/test/crypto_tests.cpp +++ b/src/test/crypto_tests.cpp @@ -4,6 +4,7 @@ #include <crypto/aes.h> #include <crypto/chacha20.h> +#include <crypto/poly1305.h> #include <crypto/ripemd160.h> #include <crypto/sha1.h> #include <crypto/sha256.h> @@ -66,26 +67,6 @@ static void TestHMACSHA512(const std::string &hexkey, const std::string &hexin, TestVector(CHMAC_SHA512(key.data(), key.size()), ParseHex(hexin), ParseHex(hexout)); } -static void TestAES128(const std::string &hexkey, const std::string &hexin, const std::string &hexout) -{ - std::vector<unsigned char> key = ParseHex(hexkey); - std::vector<unsigned char> in = ParseHex(hexin); - std::vector<unsigned char> correctout = ParseHex(hexout); - std::vector<unsigned char> buf, buf2; - - assert(key.size() == 16); - assert(in.size() == 16); - assert(correctout.size() == 16); - AES128Encrypt enc(key.data()); - buf.resize(correctout.size()); - buf2.resize(correctout.size()); - enc.Encrypt(buf.data(), in.data()); - BOOST_CHECK_EQUAL(HexStr(buf), HexStr(correctout)); - AES128Decrypt dec(key.data()); - dec.Decrypt(buf2.data(), buf.data()); - BOOST_CHECK_EQUAL(HexStr(buf2), HexStr(in)); -} - static void TestAES256(const std::string &hexkey, const std::string &hexin, const std::string &hexout) { std::vector<unsigned char> key = ParseHex(hexkey); @@ -105,47 +86,6 @@ static void TestAES256(const std::string &hexkey, const std::string &hexin, cons BOOST_CHECK(buf == in); } -static void TestAES128CBC(const std::string &hexkey, const std::string &hexiv, bool pad, const std::string &hexin, const std::string &hexout) -{ - std::vector<unsigned char> key = ParseHex(hexkey); - std::vector<unsigned char> iv = ParseHex(hexiv); - std::vector<unsigned char> in = ParseHex(hexin); - std::vector<unsigned char> correctout = ParseHex(hexout); - std::vector<unsigned char> realout(in.size() + AES_BLOCKSIZE); - - // Encrypt the plaintext and verify that it equals the cipher - AES128CBCEncrypt enc(key.data(), iv.data(), pad); - int size = enc.Encrypt(in.data(), in.size(), realout.data()); - realout.resize(size); - BOOST_CHECK(realout.size() == correctout.size()); - BOOST_CHECK_MESSAGE(realout == correctout, HexStr(realout) + std::string(" != ") + hexout); - - // Decrypt the cipher and verify that it equals the plaintext - std::vector<unsigned char> decrypted(correctout.size()); - AES128CBCDecrypt dec(key.data(), iv.data(), pad); - size = dec.Decrypt(correctout.data(), correctout.size(), decrypted.data()); - decrypted.resize(size); - BOOST_CHECK(decrypted.size() == in.size()); - BOOST_CHECK_MESSAGE(decrypted == in, HexStr(decrypted) + std::string(" != ") + hexin); - - // Encrypt and re-decrypt substrings of the plaintext and verify that they equal each-other - for(std::vector<unsigned char>::iterator i(in.begin()); i != in.end(); ++i) - { - std::vector<unsigned char> sub(i, in.end()); - std::vector<unsigned char> subout(sub.size() + AES_BLOCKSIZE); - int _size = enc.Encrypt(sub.data(), sub.size(), subout.data()); - if (_size != 0) - { - subout.resize(_size); - std::vector<unsigned char> subdecrypted(subout.size()); - _size = dec.Decrypt(subout.data(), subout.size(), subdecrypted.data()); - subdecrypted.resize(_size); - BOOST_CHECK(decrypted.size() == in.size()); - BOOST_CHECK_MESSAGE(subdecrypted == sub, HexStr(subdecrypted) + std::string(" != ") + HexStr(sub)); - } - } -} - static void TestAES256CBC(const std::string &hexkey, const std::string &hexiv, bool pad, const std::string &hexin, const std::string &hexout) { std::vector<unsigned char> key = ParseHex(hexkey); @@ -200,6 +140,17 @@ static void TestChaCha20(const std::string &hexkey, uint64_t nonce, uint64_t see BOOST_CHECK(out == outres); } +static void TestPoly1305(const std::string &hexmessage, const std::string &hexkey, const std::string& hextag) +{ + std::vector<unsigned char> key = ParseHex(hexkey); + std::vector<unsigned char> m = ParseHex(hexmessage); + std::vector<unsigned char> tag = ParseHex(hextag); + std::vector<unsigned char> tagres; + tagres.resize(POLY1305_TAGLEN); + poly1305_auth(tagres.data(), m.data(), m.size(), key.data()); + BOOST_CHECK(tag == tagres); +} + static std::string LongTestString() { std::string ret; for (int i=0; i<200000; i++) { @@ -428,14 +379,9 @@ BOOST_AUTO_TEST_CASE(hmac_sha512_testvectors) { BOOST_AUTO_TEST_CASE(aes_testvectors) { // AES test vectors from FIPS 197. - TestAES128("000102030405060708090a0b0c0d0e0f", "00112233445566778899aabbccddeeff", "69c4e0d86a7b0430d8cdb78070b4c55a"); TestAES256("000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f", "00112233445566778899aabbccddeeff", "8ea2b7ca516745bfeafc49904b496089"); // AES-ECB test vectors from NIST sp800-38a. - TestAES128("2b7e151628aed2a6abf7158809cf4f3c", "6bc1bee22e409f96e93d7e117393172a", "3ad77bb40d7a3660a89ecaf32466ef97"); - TestAES128("2b7e151628aed2a6abf7158809cf4f3c", "ae2d8a571e03ac9c9eb76fac45af8e51", "f5d3d58503b9699de785895a96fdbaaf"); - TestAES128("2b7e151628aed2a6abf7158809cf4f3c", "30c81c46a35ce411e5fbc1191a0a52ef", "43b1cd7f598ece23881b00e3ed030688"); - TestAES128("2b7e151628aed2a6abf7158809cf4f3c", "f69f2445df4f9b17ad2b417be66c3710", "7b0c785e27e8ad3f8223207104725dd4"); TestAES256("603deb1015ca71be2b73aef0857d77811f352c073b6108d72d9810a30914dff4", "6bc1bee22e409f96e93d7e117393172a", "f3eed1bdb5d2a03c064b5a7e3db181f8"); TestAES256("603deb1015ca71be2b73aef0857d77811f352c073b6108d72d9810a30914dff4", "ae2d8a571e03ac9c9eb76fac45af8e51", "591ccb10d410ed26dc5ba74a31362870"); TestAES256("603deb1015ca71be2b73aef0857d77811f352c073b6108d72d9810a30914dff4", "30c81c46a35ce411e5fbc1191a0a52ef", "b6ed21b99ca6f4f9f153e7b1beafed1d"); @@ -443,27 +389,6 @@ BOOST_AUTO_TEST_CASE(aes_testvectors) { } BOOST_AUTO_TEST_CASE(aes_cbc_testvectors) { - - // NIST AES CBC 128-bit encryption test-vectors - TestAES128CBC("2b7e151628aed2a6abf7158809cf4f3c", "000102030405060708090A0B0C0D0E0F", false, \ - "6bc1bee22e409f96e93d7e117393172a", "7649abac8119b246cee98e9b12e9197d"); - TestAES128CBC("2b7e151628aed2a6abf7158809cf4f3c", "7649ABAC8119B246CEE98E9B12E9197D", false, \ - "ae2d8a571e03ac9c9eb76fac45af8e51", "5086cb9b507219ee95db113a917678b2"); - TestAES128CBC("2b7e151628aed2a6abf7158809cf4f3c", "5086cb9b507219ee95db113a917678b2", false, \ - "30c81c46a35ce411e5fbc1191a0a52ef", "73bed6b8e3c1743b7116e69e22229516"); - TestAES128CBC("2b7e151628aed2a6abf7158809cf4f3c", "73bed6b8e3c1743b7116e69e22229516", false, \ - "f69f2445df4f9b17ad2b417be66c3710", "3ff1caa1681fac09120eca307586e1a7"); - - // The same vectors with padding enabled - TestAES128CBC("2b7e151628aed2a6abf7158809cf4f3c", "000102030405060708090A0B0C0D0E0F", true, \ - "6bc1bee22e409f96e93d7e117393172a", "7649abac8119b246cee98e9b12e9197d8964e0b149c10b7b682e6e39aaeb731c"); - TestAES128CBC("2b7e151628aed2a6abf7158809cf4f3c", "7649ABAC8119B246CEE98E9B12E9197D", true, \ - "ae2d8a571e03ac9c9eb76fac45af8e51", "5086cb9b507219ee95db113a917678b255e21d7100b988ffec32feeafaf23538"); - TestAES128CBC("2b7e151628aed2a6abf7158809cf4f3c", "5086cb9b507219ee95db113a917678b2", true, \ - "30c81c46a35ce411e5fbc1191a0a52ef", "73bed6b8e3c1743b7116e69e22229516f6eccda327bf8e5ec43718b0039adceb"); - TestAES128CBC("2b7e151628aed2a6abf7158809cf4f3c", "73bed6b8e3c1743b7116e69e22229516", true, \ - "f69f2445df4f9b17ad2b417be66c3710", "3ff1caa1681fac09120eca307586e1a78cb82807230e1321d3fae00d18cc2012"); - // NIST AES CBC 256-bit encryption test-vectors TestAES256CBC("603deb1015ca71be2b73aef0857d77811f352c073b6108d72d9810a30914dff4", \ "000102030405060708090A0B0C0D0E0F", false, "6bc1bee22e409f96e93d7e117393172a", \ @@ -524,6 +449,76 @@ BOOST_AUTO_TEST_CASE(chacha20_testvector) "fab78c9"); } +BOOST_AUTO_TEST_CASE(poly1305_testvector) +{ + // RFC 7539, section 2.5.2. + TestPoly1305("43727970746f6772617068696320466f72756d2052657365617263682047726f7570", + "85d6be7857556d337f4452fe42d506a80103808afb0db2fd4abff6af4149f51b", + "a8061dc1305136c6c22b8baf0c0127a9"); + + // RFC 7539, section A.3. + TestPoly1305("00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" + "000000000000000000000000000", + "0000000000000000000000000000000000000000000000000000000000000000", + "00000000000000000000000000000000"); + + TestPoly1305("416e79207375626d697373696f6e20746f20746865204945544620696e74656e6465642062792074686520436f6e747269627" + "5746f7220666f72207075626c69636174696f6e20617320616c6c206f722070617274206f6620616e204945544620496e7465" + "726e65742d4472616674206f722052464320616e6420616e792073746174656d656e74206d6164652077697468696e2074686" + "520636f6e74657874206f6620616e204945544620616374697669747920697320636f6e7369646572656420616e2022494554" + "4620436f6e747269627574696f6e222e20537563682073746174656d656e747320696e636c756465206f72616c20737461746" + "56d656e747320696e20494554462073657373696f6e732c2061732077656c6c206173207772697474656e20616e6420656c65" + "6374726f6e696320636f6d6d756e69636174696f6e73206d61646520617420616e792074696d65206f7220706c6163652c207" + "768696368206172652061646472657373656420746f", + "0000000000000000000000000000000036e5f6b5c5e06070f0efca96227a863e", + "36e5f6b5c5e06070f0efca96227a863e"); + + TestPoly1305("416e79207375626d697373696f6e20746f20746865204945544620696e74656e6465642062792074686520436f6e747269627" + "5746f7220666f72207075626c69636174696f6e20617320616c6c206f722070617274206f6620616e204945544620496e7465" + "726e65742d4472616674206f722052464320616e6420616e792073746174656d656e74206d6164652077697468696e2074686" + "520636f6e74657874206f6620616e204945544620616374697669747920697320636f6e7369646572656420616e2022494554" + "4620436f6e747269627574696f6e222e20537563682073746174656d656e747320696e636c756465206f72616c20737461746" + "56d656e747320696e20494554462073657373696f6e732c2061732077656c6c206173207772697474656e20616e6420656c65" + "6374726f6e696320636f6d6d756e69636174696f6e73206d61646520617420616e792074696d65206f7220706c6163652c207" + "768696368206172652061646472657373656420746f", + "36e5f6b5c5e06070f0efca96227a863e00000000000000000000000000000000", + "f3477e7cd95417af89a6b8794c310cf0"); + + TestPoly1305("2754776173206272696c6c69672c20616e642074686520736c6974687920746f7665730a446964206779726520616e6420676" + "96d626c6520696e2074686520776162653a0a416c6c206d696d737920776572652074686520626f726f676f7665732c0a416e" + "6420746865206d6f6d65207261746873206f757467726162652e", + "1c9240a5eb55d38af333888604f6b5f0473917c1402b80099dca5cbc207075c0", + "4541669a7eaaee61e708dc7cbcc5eb62"); + + TestPoly1305("ffffffffffffffffffffffffffffffff", + "0200000000000000000000000000000000000000000000000000000000000000", + "03000000000000000000000000000000"); + + TestPoly1305("02000000000000000000000000000000", + "02000000000000000000000000000000ffffffffffffffffffffffffffffffff", + "03000000000000000000000000000000"); + + TestPoly1305("fffffffffffffffffffffffffffffffff0ffffffffffffffffffffffffffffff11000000000000000000000000000000", + "0100000000000000000000000000000000000000000000000000000000000000", + "05000000000000000000000000000000"); + + TestPoly1305("fffffffffffffffffffffffffffffffffbfefefefefefefefefefefefefefefe01010101010101010101010101010101", + "0100000000000000000000000000000000000000000000000000000000000000", + "00000000000000000000000000000000"); + + TestPoly1305("fdffffffffffffffffffffffffffffff", + "0200000000000000000000000000000000000000000000000000000000000000", + "faffffffffffffffffffffffffffffff"); + + TestPoly1305("e33594d7505e43b900000000000000003394d7505e4379cd01000000000000000000000000000000000000000000000001000000000000000000000000000000", + "0100000000000000040000000000000000000000000000000000000000000000", + "14000000000000005500000000000000"); + + TestPoly1305("e33594d7505e43b900000000000000003394d7505e4379cd010000000000000000000000000000000000000000000000", + "0100000000000000040000000000000000000000000000000000000000000000", + "13000000000000000000000000000000"); +} + BOOST_AUTO_TEST_CASE(countbits_tests) { FastRandomContext ctx; diff --git a/src/test/descriptor_tests.cpp b/src/test/descriptor_tests.cpp index ff2b8d4fc9..826615cb51 100644 --- a/src/test/descriptor_tests.cpp +++ b/src/test/descriptor_tests.cpp @@ -154,8 +154,8 @@ void Check(const std::string& prv, const std::string& pub, int flags, const std: // Test whether the observed key path is present in the 'paths' variable (which contains expected, unobserved paths), // and then remove it from that set. for (const auto& origin : script_provider.origins) { - BOOST_CHECK_MESSAGE(paths.count(origin.second.path), "Unexpected key path: " + prv); - left_paths.erase(origin.second.path); + BOOST_CHECK_MESSAGE(paths.count(origin.second.second.path), "Unexpected key path: " + prv); + left_paths.erase(origin.second.second.path); } } } diff --git a/src/validation.cpp b/src/validation.cpp index ae3985485e..eee050dc89 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -2799,7 +2799,7 @@ bool CChainState::InvalidateBlock(CValidationState& state, const CChainParams& c bool ret = DisconnectTip(state, chainparams, &disconnectpool); // DisconnectTip will add transactions to disconnectpool. // Adjust the mempool to be consistent with the new tip, adding - // transactions back to the mempool if disconnecting was succesful, + // transactions back to the mempool if disconnecting was successful, // and we're not doing a very deep invalidation (in which case // keeping the mempool up to date is probably futile anyway). UpdateMempoolForReorg(disconnectpool, /* fAddToMempool = */ (++disconnected <= 10) && ret); diff --git a/src/validationinterface.cpp b/src/validationinterface.cpp index 70c274d20e..5d0ee1d1fc 100644 --- a/src/validationinterface.cpp +++ b/src/validationinterface.cpp @@ -25,7 +25,6 @@ struct ValidationInterfaceConnections { boost::signals2::scoped_connection BlockDisconnected; boost::signals2::scoped_connection TransactionRemovedFromMempool; boost::signals2::scoped_connection ChainStateFlushed; - boost::signals2::scoped_connection Broadcast; boost::signals2::scoped_connection BlockChecked; boost::signals2::scoped_connection NewPoWValidBlock; }; @@ -37,7 +36,6 @@ struct MainSignalsInstance { boost::signals2::signal<void (const std::shared_ptr<const CBlock> &)> BlockDisconnected; boost::signals2::signal<void (const CTransactionRef &)> TransactionRemovedFromMempool; boost::signals2::signal<void (const CBlockLocator &)> ChainStateFlushed; - boost::signals2::signal<void (int64_t nBestBlockTime, CConnman* connman)> Broadcast; boost::signals2::signal<void (const CBlock&, const CValidationState&)> BlockChecked; boost::signals2::signal<void (const CBlockIndex *, const std::shared_ptr<const CBlock>&)> NewPoWValidBlock; @@ -101,7 +99,6 @@ void RegisterValidationInterface(CValidationInterface* pwalletIn) { conns.BlockDisconnected = g_signals.m_internals->BlockDisconnected.connect(std::bind(&CValidationInterface::BlockDisconnected, pwalletIn, std::placeholders::_1)); conns.TransactionRemovedFromMempool = g_signals.m_internals->TransactionRemovedFromMempool.connect(std::bind(&CValidationInterface::TransactionRemovedFromMempool, pwalletIn, std::placeholders::_1)); conns.ChainStateFlushed = g_signals.m_internals->ChainStateFlushed.connect(std::bind(&CValidationInterface::ChainStateFlushed, pwalletIn, std::placeholders::_1)); - conns.Broadcast = g_signals.m_internals->Broadcast.connect(std::bind(&CValidationInterface::ResendWalletTransactions, pwalletIn, std::placeholders::_1, std::placeholders::_2)); conns.BlockChecked = g_signals.m_internals->BlockChecked.connect(std::bind(&CValidationInterface::BlockChecked, pwalletIn, std::placeholders::_1, std::placeholders::_2)); conns.NewPoWValidBlock = g_signals.m_internals->NewPoWValidBlock.connect(std::bind(&CValidationInterface::NewPoWValidBlock, pwalletIn, std::placeholders::_1, std::placeholders::_2)); } @@ -175,10 +172,6 @@ void CMainSignals::ChainStateFlushed(const CBlockLocator &locator) { }); } -void CMainSignals::Broadcast(int64_t nBestBlockTime, CConnman* connman) { - m_internals->Broadcast(nBestBlockTime, connman); -} - void CMainSignals::BlockChecked(const CBlock& block, const CValidationState& state) { m_internals->BlockChecked(block, state); } diff --git a/src/validationinterface.h b/src/validationinterface.h index f0374e8e78..78f3026a80 100644 --- a/src/validationinterface.h +++ b/src/validationinterface.h @@ -134,8 +134,6 @@ protected: * Called on a background thread. */ virtual void ChainStateFlushed(const CBlockLocator &locator) {} - /** Tells listeners to broadcast their data. */ - virtual void ResendWalletTransactions(int64_t nBestBlockTime, CConnman* connman) {} /** * Notifies listeners of a block validation result. * If the provided CValidationState IsValid, the provided block @@ -184,7 +182,6 @@ public: void BlockConnected(const std::shared_ptr<const CBlock> &, const CBlockIndex *pindex, const std::shared_ptr<const std::vector<CTransactionRef>> &); void BlockDisconnected(const std::shared_ptr<const CBlock> &); void ChainStateFlushed(const CBlockLocator &); - void Broadcast(int64_t nBestBlockTime, CConnman* connman); void BlockChecked(const CBlock&, const CValidationState&); void NewPoWValidBlock(const CBlockIndex *, const std::shared_ptr<const CBlock>&); }; diff --git a/src/wallet/init.cpp b/src/wallet/init.cpp index 76a7eaa681..a8096dc671 100644 --- a/src/wallet/init.cpp +++ b/src/wallet/init.cpp @@ -211,8 +211,9 @@ void StartWallets(CScheduler& scheduler) pwallet->postInitProcess(); } - // Run a thread to flush wallet periodically + // Schedule periodic wallet flushes and tx rebroadcasts scheduler.scheduleEvery(MaybeCompactWalletDB, 500); + scheduler.scheduleEvery(MaybeResendWalletTxs, 1000); } void FlushWallets() diff --git a/src/wallet/rpcdump.cpp b/src/wallet/rpcdump.cpp index 6c3b5a49dc..c339e111ba 100644 --- a/src/wallet/rpcdump.cpp +++ b/src/wallet/rpcdump.cpp @@ -348,7 +348,11 @@ UniValue importaddress(const JSONRPCRequest& request) if (fRescan) { RescanWallet(*pwallet, reserver); - pwallet->ReacceptWalletTransactions(); + { + auto locked_chain = pwallet->chain().lock(); + LOCK(pwallet->cs_wallet); + pwallet->ReacceptWalletTransactions(*locked_chain); + } } return NullUniValue; @@ -532,7 +536,11 @@ UniValue importpubkey(const JSONRPCRequest& request) if (fRescan) { RescanWallet(*pwallet, reserver); - pwallet->ReacceptWalletTransactions(); + { + auto locked_chain = pwallet->chain().lock(); + LOCK(pwallet->cs_wallet); + pwallet->ReacceptWalletTransactions(*locked_chain); + } } return NullUniValue; @@ -892,7 +900,7 @@ struct ImportData // Output data std::set<CScript> import_scripts; std::map<CKeyID, bool> used_keys; //!< Import these private keys if available (the value indicates whether if the key is required for solvability) - std::map<CKeyID, KeyOriginInfo> key_origins; + std::map<CKeyID, std::pair<CPubKey, KeyOriginInfo>> key_origins; }; enum class ScriptContext @@ -1189,6 +1197,9 @@ static UniValue ProcessImportDescriptor(ImportData& import_data, std::map<CKeyID bool spendable = std::all_of(pubkey_map.begin(), pubkey_map.end(), [&](const std::pair<CKeyID, CPubKey>& used_key) { return privkey_map.count(used_key.first) > 0; + }) && std::all_of(import_data.key_origins.begin(), import_data.key_origins.end(), + [&](const std::pair<CKeyID, std::pair<CPubKey, KeyOriginInfo>>& entry) { + return privkey_map.count(entry.first) > 0; }); if (!watch_only && !spendable) { warnings.push_back("Some private keys are missing, outputs will be considered watchonly. If this is intentional, specify the watchonly flag."); @@ -1266,7 +1277,10 @@ static UniValue ProcessImport(CWallet * const pwallet, const UniValue& data, con throw JSONRPCError(RPC_WALLET_ERROR, "Error adding key to wallet"); } pwallet->UpdateTimeFirstKey(timestamp); - } + } + for (const auto& entry : import_data.key_origins) { + pwallet->AddKeyOrigin(entry.second.first, entry.second.second); + } for (const CKeyID& id : ordered_pubkeys) { auto entry = pubkey_map.find(id); if (entry == pubkey_map.end()) { @@ -1277,10 +1291,6 @@ static UniValue ProcessImport(CWallet * const pwallet, const UniValue& data, con if (!pwallet->GetPubKey(id, temp) && !pwallet->AddWatchOnly(GetScriptForRawPubKey(pubkey), timestamp)) { throw JSONRPCError(RPC_WALLET_ERROR, "Error adding address to wallet"); } - const auto& key_orig_it = import_data.key_origins.find(id); - if (key_orig_it != import_data.key_origins.end()) { - pwallet->AddKeyOrigin(pubkey, key_orig_it->second); - } pwallet->mapKeyMetadata[id].nCreateTime = timestamp; // Add to keypool only works with pubkeys @@ -1468,7 +1478,11 @@ UniValue importmulti(const JSONRPCRequest& mainRequest) } if (fRescan && fRunScan && requests.size()) { int64_t scannedTime = pwallet->RescanFromTime(nLowestTimestamp, reserver, true /* update */); - pwallet->ReacceptWalletTransactions(); + { + auto locked_chain = pwallet->chain().lock(); + LOCK(pwallet->cs_wallet); + pwallet->ReacceptWalletTransactions(*locked_chain); + } if (pwallet->IsAbortingRescan()) { throw JSONRPCError(RPC_MISC_ERROR, "Rescan aborted by user."); diff --git a/src/wallet/rpcwallet.cpp b/src/wallet/rpcwallet.cpp index d32613fc77..d9ae0b9bd6 100644 --- a/src/wallet/rpcwallet.cpp +++ b/src/wallet/rpcwallet.cpp @@ -308,7 +308,7 @@ static UniValue setlabel(const JSONRPCRequest& request) static CTransactionRef SendMoney(interfaces::Chain::Lock& locked_chain, CWallet * const pwallet, const CTxDestination &address, CAmount nValue, bool fSubtractFeeFromAmount, const CCoinControl& coin_control, mapValue_t mapValue) { - CAmount curBalance = pwallet->GetBalance(); + CAmount curBalance = pwallet->GetBalance().m_mine_trusted; // Check amount if (nValue <= 0) @@ -761,12 +761,14 @@ static UniValue getbalance(const JSONRPCRequest& request) min_depth = request.params[1].get_int(); } - isminefilter filter = ISMINE_SPENDABLE; + bool include_watchonly = false; if (!request.params[2].isNull() && request.params[2].get_bool()) { - filter = filter | ISMINE_WATCH_ONLY; + include_watchonly = true; } - return ValueFromAmount(pwallet->GetBalance(filter, min_depth)); + const auto bal = pwallet->GetBalance(min_depth); + + return ValueFromAmount(bal.m_mine_trusted + (include_watchonly ? bal.m_watchonly_trusted : 0)); } static UniValue getunconfirmedbalance(const JSONRPCRequest &request) @@ -794,7 +796,7 @@ static UniValue getunconfirmedbalance(const JSONRPCRequest &request) auto locked_chain = pwallet->chain().lock(); LOCK(pwallet->cs_wallet); - return ValueFromAmount(pwallet->GetUnconfirmedBalance()); + return ValueFromAmount(pwallet->GetBalance().m_mine_untrusted_pending); } @@ -807,9 +809,7 @@ static UniValue sendmany(const JSONRPCRequest& request) return NullUniValue; } - if (request.fHelp || request.params.size() < 2 || request.params.size() > 8) - throw std::runtime_error( - RPCHelpMan{"sendmany", + const RPCHelpMan help{"sendmany", "\nSend multiple times. Amounts are double-precision floating point numbers." + HelpRequiringPassphrase(pwallet) + "\n", { @@ -819,7 +819,7 @@ static UniValue sendmany(const JSONRPCRequest& request) {"address", RPCArg::Type::AMOUNT, RPCArg::Optional::NO, "The bitcoin address is the key, the numeric amount (can be string) in " + CURRENCY_UNIT + " is the value"}, }, }, - {"minconf", RPCArg::Type::NUM, /* default */ "1", "Only use the balance confirmed at least this many times."}, + {"minconf", RPCArg::Type::NUM, RPCArg::Optional::OMITTED_NAMED_ARG, "Ignored dummy value"}, {"comment", RPCArg::Type::STR, RPCArg::Optional::OMITTED_NAMED_ARG, "A comment"}, {"subtractfeefrom", RPCArg::Type::ARR, RPCArg::Optional::OMITTED_NAMED_ARG, "A json array with addresses.\n" " The fee will be equally deducted from the amount of each selected address.\n" @@ -850,7 +850,11 @@ static UniValue sendmany(const JSONRPCRequest& request) "\nAs a JSON-RPC call\n" + HelpExampleRpc("sendmany", "\"\", {\"1D1ZrZNe3JUo7ZycKEYQQiQAWd9y54F4XX\":0.01,\"1353tsE8YMTA4EuV7dgUXGjNFf9KpVvKHz\":0.02}, 6, \"testing\"") }, - }.ToString()); + }; + + if (request.fHelp || !help.IsValidNumArgs(request.params.size())) { + throw std::runtime_error(help.ToString()); + } // Make sure the results are valid at least up to the most recent block // the user could have gotten from another RPC command prior to now @@ -867,9 +871,6 @@ static UniValue sendmany(const JSONRPCRequest& request) throw JSONRPCError(RPC_INVALID_PARAMETER, "Dummy value must be set to \"\""); } UniValue sendTo = request.params[1].get_obj(); - int nMinDepth = 1; - if (!request.params[2].isNull()) - nMinDepth = request.params[2].get_int(); mapValue_t mapValue; if (!request.params[3].isNull() && !request.params[3].get_str().empty()) @@ -897,7 +898,6 @@ static UniValue sendmany(const JSONRPCRequest& request) std::set<CTxDestination> destinations; std::vector<CRecipient> vecSend; - CAmount totalAmount = 0; std::vector<std::string> keys = sendTo.getKeys(); for (const std::string& name_ : keys) { CTxDestination dest = DecodeDestination(name_); @@ -914,7 +914,6 @@ static UniValue sendmany(const JSONRPCRequest& request) CAmount nAmount = AmountFromValue(sendTo[name_]); if (nAmount <= 0) throw JSONRPCError(RPC_TYPE_ERROR, "Invalid amount for send"); - totalAmount += nAmount; bool fSubtractFeeFromAmount = false; for (unsigned int idx = 0; idx < subtractFeeFromAmount.size(); idx++) { @@ -929,11 +928,6 @@ static UniValue sendmany(const JSONRPCRequest& request) EnsureWalletIsUnlocked(pwallet); - // Check funds - if (totalAmount > pwallet->GetLegacyBalance(ISMINE_SPENDABLE, nMinDepth)) { - throw JSONRPCError(RPC_WALLET_INSUFFICIENT_FUNDS, "Wallet has insufficient funds"); - } - // Shuffle recipient list std::shuffle(vecSend.begin(), vecSend.end(), FastRandomContext()); @@ -2424,11 +2418,12 @@ static UniValue getwalletinfo(const JSONRPCRequest& request) UniValue obj(UniValue::VOBJ); size_t kpExternalSize = pwallet->KeypoolCountExternalKeys(); + const auto bal = pwallet->GetBalance(); obj.pushKV("walletname", pwallet->GetName()); obj.pushKV("walletversion", pwallet->GetVersion()); - obj.pushKV("balance", ValueFromAmount(pwallet->GetBalance())); - obj.pushKV("unconfirmed_balance", ValueFromAmount(pwallet->GetUnconfirmedBalance())); - obj.pushKV("immature_balance", ValueFromAmount(pwallet->GetImmatureBalance())); + obj.pushKV("balance", ValueFromAmount(bal.m_mine_trusted)); + obj.pushKV("unconfirmed_balance", ValueFromAmount(bal.m_mine_untrusted_pending)); + obj.pushKV("immature_balance", ValueFromAmount(bal.m_mine_immature)); obj.pushKV("txcount", (int)pwallet->mapWallet.size()); obj.pushKV("keypoololdest", pwallet->GetOldestKeyPoolTime()); obj.pushKV("keypoolsize", (int64_t)kpExternalSize); @@ -2668,50 +2663,6 @@ static UniValue unloadwallet(const JSONRPCRequest& request) return NullUniValue; } -static UniValue resendwallettransactions(const JSONRPCRequest& request) -{ - std::shared_ptr<CWallet> const wallet = GetWalletForJSONRPCRequest(request); - CWallet* const pwallet = wallet.get(); - - if (!EnsureWalletIsAvailable(pwallet, request.fHelp)) { - return NullUniValue; - } - - if (request.fHelp || request.params.size() != 0) - throw std::runtime_error( - RPCHelpMan{"resendwallettransactions", - "Immediately re-broadcast unconfirmed wallet transactions to all peers.\n" - "Intended only for testing; the wallet code periodically re-broadcasts\n" - "automatically.\n", - {}, - RPCResult{ - "Returns an RPC error if -walletbroadcast is set to false.\n" - "Returns array of transaction ids that were re-broadcast.\n" - }, - RPCExamples{""}, - }.ToString() - ); - - if (!pwallet->chain().p2pEnabled()) { - throw JSONRPCError(RPC_CLIENT_P2P_DISABLED, "Error: Peer-to-peer functionality missing or disabled"); - } - - auto locked_chain = pwallet->chain().lock(); - LOCK(pwallet->cs_wallet); - - if (!pwallet->GetBroadcastTransactions()) { - throw JSONRPCError(RPC_WALLET_ERROR, "Error: Wallet transaction broadcasting is disabled with -walletbroadcast"); - } - - std::vector<uint256> txids = pwallet->ResendWalletTransactionsBefore(*locked_chain, GetTime()); - UniValue result(UniValue::VARR); - for (const uint256& txid : txids) - { - result.push_back(txid.ToString()); - } - return result; -} - static UniValue listunspent(const JSONRPCRequest& request) { std::shared_ptr<CWallet> const wallet = GetWalletForJSONRPCRequest(request); @@ -3150,7 +3101,7 @@ UniValue signrawtransactionwithwallet(const JSONRPCRequest& request) {"scriptPubKey", RPCArg::Type::STR_HEX, RPCArg::Optional::NO, "script key"}, {"redeemScript", RPCArg::Type::STR_HEX, RPCArg::Optional::OMITTED, "(required for P2SH) redeem script"}, {"witnessScript", RPCArg::Type::STR_HEX, RPCArg::Optional::OMITTED, "(required for P2WSH or P2SH-P2WSH) witness script"}, - {"amount", RPCArg::Type::AMOUNT, RPCArg::Optional::NO, "The amount spent"}, + {"amount", RPCArg::Type::AMOUNT, RPCArg::Optional::OMITTED, "(required for Segwit inputs) the amount spent"}, }, }, }, @@ -4085,7 +4036,6 @@ UniValue importmulti(const JSONRPCRequest& request); static const CRPCCommand commands[] = { // category name actor (function) argNames // --------------------- ------------------------ ----------------------- ---------- - { "hidden", "resendwallettransactions", &resendwallettransactions, {} }, { "rawtransactions", "fundrawtransaction", &fundrawtransaction, {"hexstring","options","iswitness"} }, { "wallet", "abandontransaction", &abandontransaction, {"txid"} }, { "wallet", "abortrescan", &abortrescan, {} }, diff --git a/src/wallet/test/wallet_tests.cpp b/src/wallet/test/wallet_tests.cpp index af57dbf5f6..7c8085dc20 100644 --- a/src/wallet/test/wallet_tests.cpp +++ b/src/wallet/test/wallet_tests.cpp @@ -58,7 +58,7 @@ BOOST_FIXTURE_TEST_CASE(scan_for_wallet_transactions, TestChain100Setup) BOOST_CHECK(result.last_failed_block.IsNull()); BOOST_CHECK(result.last_scanned_block.IsNull()); BOOST_CHECK(!result.last_scanned_height); - BOOST_CHECK_EQUAL(wallet.GetImmatureBalance(), 0); + BOOST_CHECK_EQUAL(wallet.GetBalance().m_mine_immature, 0); } // Verify ScanForWalletTransactions picks up transactions in both the old @@ -73,7 +73,7 @@ BOOST_FIXTURE_TEST_CASE(scan_for_wallet_transactions, TestChain100Setup) BOOST_CHECK(result.last_failed_block.IsNull()); BOOST_CHECK_EQUAL(result.last_scanned_block, newTip->GetBlockHash()); BOOST_CHECK_EQUAL(*result.last_scanned_height, newTip->nHeight); - BOOST_CHECK_EQUAL(wallet.GetImmatureBalance(), 100 * COIN); + BOOST_CHECK_EQUAL(wallet.GetBalance().m_mine_immature, 100 * COIN); } // Prune the older block file. @@ -92,7 +92,7 @@ BOOST_FIXTURE_TEST_CASE(scan_for_wallet_transactions, TestChain100Setup) BOOST_CHECK_EQUAL(result.last_failed_block, oldTip->GetBlockHash()); BOOST_CHECK_EQUAL(result.last_scanned_block, newTip->GetBlockHash()); BOOST_CHECK_EQUAL(*result.last_scanned_height, newTip->nHeight); - BOOST_CHECK_EQUAL(wallet.GetImmatureBalance(), 50 * COIN); + BOOST_CHECK_EQUAL(wallet.GetBalance().m_mine_immature, 50 * COIN); } // Prune the remaining block file. @@ -110,7 +110,7 @@ BOOST_FIXTURE_TEST_CASE(scan_for_wallet_transactions, TestChain100Setup) BOOST_CHECK_EQUAL(result.last_failed_block, newTip->GetBlockHash()); BOOST_CHECK(result.last_scanned_block.IsNull()); BOOST_CHECK(!result.last_scanned_height); - BOOST_CHECK_EQUAL(wallet.GetImmatureBalance(), 0); + BOOST_CHECK_EQUAL(wallet.GetBalance().m_mine_immature, 0); } } diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index 95756f988f..9f89cbefa0 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -1276,6 +1276,10 @@ void CWallet::BlockDisconnected(const CBlock& block) { } } +void CWallet::UpdatedBlockTip() +{ + m_best_block_time = GetTime(); +} void CWallet::BlockUntilSyncedToCurrentChain() { @@ -1861,13 +1865,11 @@ CWallet::ScanResult CWallet::ScanForWalletTransactions(const uint256& start_bloc return result; } -void CWallet::ReacceptWalletTransactions() +void CWallet::ReacceptWalletTransactions(interfaces::Chain::Lock& locked_chain) { // If transactions aren't being broadcasted, don't let them into local mempool either if (!fBroadcastTransactions) return; - auto locked_chain = chain().lock(); - LOCK(cs_wallet); std::map<int64_t, CWalletTx*> mapSorted; // Sort pending wallet transactions based on their initial wallet insertion order @@ -1877,7 +1879,7 @@ void CWallet::ReacceptWalletTransactions() CWalletTx& wtx = item.second; assert(wtx.GetHash() == wtxid); - int nDepth = wtx.GetDepthInMainChain(*locked_chain); + int nDepth = wtx.GetDepthInMainChain(locked_chain); if (!wtx.IsCoinBase() && (nDepth == 0 && !wtx.isAbandoned())) { mapSorted.insert(std::make_pair(wtx.nOrderPos, &wtx)); @@ -1888,7 +1890,7 @@ void CWallet::ReacceptWalletTransactions() for (const std::pair<const int64_t, CWalletTx*>& item : mapSorted) { CWalletTx& wtx = *(item.second); CValidationState state; - wtx.AcceptToMemoryPool(*locked_chain, state); + wtx.AcceptToMemoryPool(locked_chain, state); } } @@ -2112,184 +2114,95 @@ bool CWalletTx::IsEquivalentTo(const CWalletTx& _tx) const return CTransaction(tx1) == CTransaction(tx2); } -std::vector<uint256> CWallet::ResendWalletTransactionsBefore(interfaces::Chain::Lock& locked_chain, int64_t nTime) +// Rebroadcast transactions from the wallet. We do this on a random timer +// to slightly obfuscate which transactions come from our wallet. +// +// Ideally, we'd only resend transactions that we think should have been +// mined in the most recent block. Any transaction that wasn't in the top +// blockweight of transactions in the mempool shouldn't have been mined, +// and so is probably just sitting in the mempool waiting to be confirmed. +// Rebroadcasting does nothing to speed up confirmation and only damages +// privacy. +void CWallet::ResendWalletTransactions() { - std::vector<uint256> result; - - LOCK(cs_wallet); - - // Sort them in chronological order - std::multimap<unsigned int, CWalletTx*> mapSorted; - for (std::pair<const uint256, CWalletTx>& item : mapWallet) - { - CWalletTx& wtx = item.second; - // Don't rebroadcast if newer than nTime: - if (wtx.nTimeReceived > nTime) - continue; - mapSorted.insert(std::make_pair(wtx.nTimeReceived, &wtx)); - } - for (const std::pair<const unsigned int, CWalletTx*>& item : mapSorted) - { - CWalletTx& wtx = *item.second; - if (wtx.RelayWalletTransaction(locked_chain)) { - result.push_back(wtx.GetHash()); - } - } - return result; -} + // During reindex, importing and IBD, old wallet transactions become + // unconfirmed. Don't resend them as that would spam other nodes. + if (!chain().isReadyToBroadcast()) return; -void CWallet::ResendWalletTransactions(interfaces::Chain::Lock& locked_chain, int64_t nBestBlockTime) -{ // Do this infrequently and randomly to avoid giving away // that these are our transactions. - if (GetTime() < nNextResend || !fBroadcastTransactions) - return; + if (GetTime() < nNextResend || !fBroadcastTransactions) return; bool fFirst = (nNextResend == 0); nNextResend = GetTime() + GetRand(30 * 60); - if (fFirst) - return; + if (fFirst) return; // Only do it if there's been a new block since last time - if (nBestBlockTime < nLastResend) - return; + if (m_best_block_time < nLastResend) return; nLastResend = GetTime(); - // Rebroadcast unconfirmed txes older than 5 minutes before the last - // block was found: - std::vector<uint256> relayed = ResendWalletTransactionsBefore(locked_chain, nBestBlockTime-5*60); - if (!relayed.empty()) - WalletLogPrintf("%s: rebroadcast %u unconfirmed transactions\n", __func__, relayed.size()); -} - -/** @} */ // end of mapWallet - - + int relayed_tx_count = 0; - -/** @defgroup Actions - * - * @{ - */ - - -CAmount CWallet::GetBalance(const isminefilter& filter, const int min_depth) const -{ - CAmount nTotal = 0; - { + { // locked_chain and cs_wallet scope auto locked_chain = chain().lock(); LOCK(cs_wallet); - for (const auto& entry : mapWallet) - { - const CWalletTx* pcoin = &entry.second; - if (pcoin->IsTrusted(*locked_chain) && pcoin->GetDepthInMainChain(*locked_chain) >= min_depth) { - nTotal += pcoin->GetAvailableCredit(*locked_chain, true, filter); - } - } - } - - return nTotal; -} -CAmount CWallet::GetUnconfirmedBalance() const -{ - CAmount nTotal = 0; - { - auto locked_chain = chain().lock(); - LOCK(cs_wallet); - for (const auto& entry : mapWallet) - { - const CWalletTx* pcoin = &entry.second; - if (!pcoin->IsTrusted(*locked_chain) && pcoin->GetDepthInMainChain(*locked_chain) == 0 && pcoin->InMempool()) - nTotal += pcoin->GetAvailableCredit(*locked_chain); + // Relay transactions + for (std::pair<const uint256, CWalletTx>& item : mapWallet) { + CWalletTx& wtx = item.second; + // only rebroadcast unconfirmed txes older than 5 minutes before the + // last block was found + if (wtx.nTimeReceived > m_best_block_time - 5 * 60) continue; + if (wtx.RelayWalletTransaction(*locked_chain)) ++relayed_tx_count; } - } - return nTotal; -} + } // locked_chain and cs_wallet -CAmount CWallet::GetImmatureBalance() const -{ - CAmount nTotal = 0; - { - auto locked_chain = chain().lock(); - LOCK(cs_wallet); - for (const auto& entry : mapWallet) - { - const CWalletTx* pcoin = &entry.second; - nTotal += pcoin->GetImmatureCredit(*locked_chain); - } + if (relayed_tx_count > 0) { + WalletLogPrintf("%s: rebroadcast %u unconfirmed transactions\n", __func__, relayed_tx_count); } - return nTotal; } -CAmount CWallet::GetUnconfirmedWatchOnlyBalance() const +/** @} */ // end of mapWallet + +void MaybeResendWalletTxs() { - CAmount nTotal = 0; - { - auto locked_chain = chain().lock(); - LOCK(cs_wallet); - for (const auto& entry : mapWallet) - { - const CWalletTx* pcoin = &entry.second; - if (!pcoin->IsTrusted(*locked_chain) && pcoin->GetDepthInMainChain(*locked_chain) == 0 && pcoin->InMempool()) - nTotal += pcoin->GetAvailableCredit(*locked_chain, true, ISMINE_WATCH_ONLY); - } + for (const std::shared_ptr<CWallet>& pwallet : GetWallets()) { + pwallet->ResendWalletTransactions(); } - return nTotal; } -CAmount CWallet::GetImmatureWatchOnlyBalance() const + +/** @defgroup Actions + * + * @{ + */ + + +CWallet::Balance CWallet::GetBalance(const int min_depth) const { - CAmount nTotal = 0; + Balance ret; { auto locked_chain = chain().lock(); LOCK(cs_wallet); for (const auto& entry : mapWallet) { - const CWalletTx* pcoin = &entry.second; - nTotal += pcoin->GetImmatureWatchOnlyCredit(*locked_chain); - } - } - return nTotal; -} - -// Calculate total balance in a different way from GetBalance. The biggest -// difference is that GetBalance sums up all unspent TxOuts paying to the -// wallet, while this sums up both spent and unspent TxOuts paying to the -// wallet, and then subtracts the values of TxIns spending from the wallet. This -// also has fewer restrictions on which unconfirmed transactions are considered -// trusted. -CAmount CWallet::GetLegacyBalance(const isminefilter& filter, int minDepth) const -{ - auto locked_chain = chain().lock(); - LOCK(cs_wallet); - - CAmount balance = 0; - for (const auto& entry : mapWallet) { - const CWalletTx& wtx = entry.second; - const int depth = wtx.GetDepthInMainChain(*locked_chain); - if (depth < 0 || !locked_chain->checkFinalTx(*wtx.tx) || wtx.IsImmatureCoinBase(*locked_chain)) { - continue; - } - - // Loop through tx outputs and add incoming payments. For outgoing txs, - // treat change outputs specially, as part of the amount debited. - CAmount debit = wtx.GetDebit(filter); - const bool outgoing = debit > 0; - for (const CTxOut& out : wtx.tx->vout) { - if (outgoing && IsChange(out)) { - debit -= out.nValue; - } else if (IsMine(out) & filter && depth >= minDepth) { - balance += out.nValue; + const CWalletTx& wtx = entry.second; + const bool is_trusted{wtx.IsTrusted(*locked_chain)}; + const int tx_depth{wtx.GetDepthInMainChain(*locked_chain)}; + const CAmount tx_credit_mine{wtx.GetAvailableCredit(*locked_chain, /* fUseCache */ true, ISMINE_SPENDABLE)}; + const CAmount tx_credit_watchonly{wtx.GetAvailableCredit(*locked_chain, /* fUseCache */ true, ISMINE_WATCH_ONLY)}; + if (is_trusted && tx_depth >= min_depth) { + ret.m_mine_trusted += tx_credit_mine; + ret.m_watchonly_trusted += tx_credit_watchonly; } - } - - // For outgoing txs, subtract amount debited. - if (outgoing) { - balance -= debit; + if (!is_trusted && tx_depth == 0 && wtx.InMempool()) { + ret.m_mine_untrusted_pending += tx_credit_mine; + ret.m_watchonly_untrusted_pending += tx_credit_watchonly; + } + ret.m_mine_immature += wtx.GetImmatureCredit(*locked_chain); + ret.m_watchonly_immature += wtx.GetImmatureWatchOnlyCredit(*locked_chain); } } - - return balance; + return ret; } CAmount CWallet::GetAvailableBalance(const CCoinControl* coinControl) const @@ -2318,25 +2231,25 @@ void CWallet::AvailableCoins(interfaces::Chain::Lock& locked_chain, std::vector< for (const auto& entry : mapWallet) { const uint256& wtxid = entry.first; - const CWalletTx* pcoin = &entry.second; + const CWalletTx& wtx = entry.second; - if (!locked_chain.checkFinalTx(*pcoin->tx)) { + if (!locked_chain.checkFinalTx(*wtx.tx)) { continue; } - if (pcoin->IsImmatureCoinBase(locked_chain)) + if (wtx.IsImmatureCoinBase(locked_chain)) continue; - int nDepth = pcoin->GetDepthInMainChain(locked_chain); + int nDepth = wtx.GetDepthInMainChain(locked_chain); if (nDepth < 0) continue; // We should not consider coins which aren't at least in our mempool // It's possible for these to be conflicted via ancestors which we may never be able to detect - if (nDepth == 0 && !pcoin->InMempool()) + if (nDepth == 0 && !wtx.InMempool()) continue; - bool safeTx = pcoin->IsTrusted(locked_chain); + bool safeTx = wtx.IsTrusted(locked_chain); // We should not consider coins from transactions that are replacing // other transactions. @@ -2353,7 +2266,7 @@ void CWallet::AvailableCoins(interfaces::Chain::Lock& locked_chain, std::vector< // be a 1-block reorg away from the chain where transactions A and C // were accepted to another chain where B, B', and C were all // accepted. - if (nDepth == 0 && pcoin->mapValue.count("replaces_txid")) { + if (nDepth == 0 && wtx.mapValue.count("replaces_txid")) { safeTx = false; } @@ -2365,7 +2278,7 @@ void CWallet::AvailableCoins(interfaces::Chain::Lock& locked_chain, std::vector< // intending to replace A', but potentially resulting in a scenario // where A, A', and D could all be accepted (instead of just B and // D, or just A and A' like the user would want). - if (nDepth == 0 && pcoin->mapValue.count("replaced_by_txid")) { + if (nDepth == 0 && wtx.mapValue.count("replaced_by_txid")) { safeTx = false; } @@ -2376,8 +2289,8 @@ void CWallet::AvailableCoins(interfaces::Chain::Lock& locked_chain, std::vector< if (nDepth < nMinDepth || nDepth > nMaxDepth) continue; - for (unsigned int i = 0; i < pcoin->tx->vout.size(); i++) { - if (pcoin->tx->vout[i].nValue < nMinimumAmount || pcoin->tx->vout[i].nValue > nMaximumAmount) + for (unsigned int i = 0; i < wtx.tx->vout.size(); i++) { + if (wtx.tx->vout[i].nValue < nMinimumAmount || wtx.tx->vout[i].nValue > nMaximumAmount) continue; if (coinControl && coinControl->HasSelected() && !coinControl->fAllowOtherInputs && !coinControl->IsSelected(COutPoint(entry.first, i))) @@ -2389,20 +2302,20 @@ void CWallet::AvailableCoins(interfaces::Chain::Lock& locked_chain, std::vector< if (IsSpent(locked_chain, wtxid, i)) continue; - isminetype mine = IsMine(pcoin->tx->vout[i]); + isminetype mine = IsMine(wtx.tx->vout[i]); if (mine == ISMINE_NO) { continue; } - bool solvable = IsSolvable(*this, pcoin->tx->vout[i].scriptPubKey); + bool solvable = IsSolvable(*this, wtx.tx->vout[i].scriptPubKey); bool spendable = ((mine & ISMINE_SPENDABLE) != ISMINE_NO) || (((mine & ISMINE_WATCH_ONLY) != ISMINE_NO) && (coinControl && coinControl->fAllowWatchOnly && solvable)); - vCoins.push_back(COutput(pcoin, i, nDepth, spendable, solvable, safeTx, (coinControl && coinControl->fAllowWatchOnly))); + vCoins.push_back(COutput(&wtx, i, nDepth, spendable, solvable, safeTx, (coinControl && coinControl->fAllowWatchOnly))); // Checks the sum amount of all UTXO's. if (nMinimumSumAmount != MAX_MONEY) { - nTotal += pcoin->tx->vout[i].nValue; + nTotal += wtx.tx->vout[i].nValue; if (nTotal >= nMinimumSumAmount) { return; @@ -2560,13 +2473,13 @@ bool CWallet::SelectCoins(const std::vector<COutput>& vAvailableCoins, const CAm std::map<uint256, CWalletTx>::const_iterator it = mapWallet.find(outpoint.hash); if (it != mapWallet.end()) { - const CWalletTx* pcoin = &it->second; + const CWalletTx& wtx = it->second; // Clearly invalid input, fail - if (pcoin->tx->vout.size() <= outpoint.n) + if (wtx.tx->vout.size() <= outpoint.n) return false; // Just to calculate the marginal byte size - nValueFromPresetInputs += pcoin->tx->vout[outpoint.n].nValue; - setPresetCoins.insert(CInputCoin(pcoin->tx, outpoint.n)); + nValueFromPresetInputs += wtx.tx->vout[outpoint.n].nValue; + setPresetCoins.insert(CInputCoin(wtx.tx, outpoint.n)); } else return false; // TODO: Allow non-wallet inputs } @@ -3604,27 +3517,27 @@ std::map<CTxDestination, CAmount> CWallet::GetAddressBalances(interfaces::Chain: LOCK(cs_wallet); for (const auto& walletEntry : mapWallet) { - const CWalletTx *pcoin = &walletEntry.second; + const CWalletTx& wtx = walletEntry.second; - if (!pcoin->IsTrusted(locked_chain)) + if (!wtx.IsTrusted(locked_chain)) continue; - if (pcoin->IsImmatureCoinBase(locked_chain)) + if (wtx.IsImmatureCoinBase(locked_chain)) continue; - int nDepth = pcoin->GetDepthInMainChain(locked_chain); - if (nDepth < (pcoin->IsFromMe(ISMINE_ALL) ? 0 : 1)) + int nDepth = wtx.GetDepthInMainChain(locked_chain); + if (nDepth < (wtx.IsFromMe(ISMINE_ALL) ? 0 : 1)) continue; - for (unsigned int i = 0; i < pcoin->tx->vout.size(); i++) + for (unsigned int i = 0; i < wtx.tx->vout.size(); i++) { CTxDestination addr; - if (!IsMine(pcoin->tx->vout[i])) + if (!IsMine(wtx.tx->vout[i])) continue; - if(!ExtractDestination(pcoin->tx->vout[i].scriptPubKey, addr)) + if(!ExtractDestination(wtx.tx->vout[i].scriptPubKey, addr)) continue; - CAmount n = IsSpent(locked_chain, walletEntry.first, i) ? 0 : pcoin->tx->vout[i].nValue; + CAmount n = IsSpent(locked_chain, walletEntry.first, i) ? 0 : wtx.tx->vout[i].nValue; if (!balances.count(addr)) balances[addr] = 0; @@ -3644,13 +3557,13 @@ std::set< std::set<CTxDestination> > CWallet::GetAddressGroupings() for (const auto& walletEntry : mapWallet) { - const CWalletTx *pcoin = &walletEntry.second; + const CWalletTx& wtx = walletEntry.second; - if (pcoin->tx->vin.size() > 0) + if (wtx.tx->vin.size() > 0) { bool any_mine = false; // group all input addresses with each other - for (const CTxIn& txin : pcoin->tx->vin) + for (const CTxIn& txin : wtx.tx->vin) { CTxDestination address; if(!IsMine(txin)) /* If this input isn't mine, ignore it */ @@ -3664,7 +3577,7 @@ std::set< std::set<CTxDestination> > CWallet::GetAddressGroupings() // group change with input addresses if (any_mine) { - for (const CTxOut& txout : pcoin->tx->vout) + for (const CTxOut& txout : wtx.tx->vout) if (IsChange(txout)) { CTxDestination txoutAddr; @@ -3681,7 +3594,7 @@ std::set< std::set<CTxDestination> > CWallet::GetAddressGroupings() } // group lone addrs by themselves - for (const auto& txout : pcoin->tx->vout) + for (const auto& txout : wtx.tx->vout) if (IsMine(txout)) { CTxDestination address; @@ -4398,9 +4311,15 @@ std::shared_ptr<CWallet> CWallet::CreateWalletFromFile(interfaces::Chain& chain, void CWallet::postInitProcess() { + auto locked_chain = chain().lock(); + LOCK(cs_wallet); + // Add wallet transactions that aren't already in a block to mempool // Do this here as mempool requires genesis block to be loaded - ReacceptWalletTransactions(); + ReacceptWalletTransactions(*locked_chain); + + // Update wallet transactions with current mempool transactions. + chain().requestMempoolTransactions(*this); } bool CWallet::BackupWallet(const std::string& strDest) diff --git a/src/wallet/wallet.h b/src/wallet/wallet.h index 86448efcaf..587b2814c0 100644 --- a/src/wallet/wallet.h +++ b/src/wallet/wallet.h @@ -657,6 +657,8 @@ private: int64_t nNextResend = 0; int64_t nLastResend = 0; bool fBroadcastTransactions = false; + // Local time that the tip block was received. Used to schedule wallet rebroadcasts. + std::atomic<int64_t> m_best_block_time {0}; /** * Used to keep track of spent outpoints, and @@ -926,6 +928,7 @@ public: void TransactionAddedToMempool(const CTransactionRef& tx) override; void BlockConnected(const CBlock& block, const std::vector<CTransactionRef>& vtxConflicted) override; void BlockDisconnected(const CBlock& block) override; + void UpdatedBlockTip() override; int64_t RescanFromTime(int64_t startTime, const WalletRescanReserver& reserver, bool update); struct ScanResult { @@ -945,16 +948,17 @@ public: }; ScanResult ScanForWalletTransactions(const uint256& first_block, const uint256& last_block, const WalletRescanReserver& reserver, bool fUpdate); void TransactionRemovedFromMempool(const CTransactionRef &ptx) override; - void ReacceptWalletTransactions(); - void ResendWalletTransactions(interfaces::Chain::Lock& locked_chain, int64_t nBestBlockTime) override; - // ResendWalletTransactionsBefore may only be called if fBroadcastTransactions! - std::vector<uint256> ResendWalletTransactionsBefore(interfaces::Chain::Lock& locked_chain, int64_t nTime); - CAmount GetBalance(const isminefilter& filter=ISMINE_SPENDABLE, const int min_depth=0) const; - CAmount GetUnconfirmedBalance() const; - CAmount GetImmatureBalance() const; - CAmount GetUnconfirmedWatchOnlyBalance() const; - CAmount GetImmatureWatchOnlyBalance() const; - CAmount GetLegacyBalance(const isminefilter& filter, int minDepth) const; + void ReacceptWalletTransactions(interfaces::Chain::Lock& locked_chain) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + void ResendWalletTransactions(); + struct Balance { + CAmount m_mine_trusted{0}; //!< Trusted, at depth=GetBalance.min_depth or more + CAmount m_mine_untrusted_pending{0}; //!< Untrusted, but in mempool (pending) + CAmount m_mine_immature{0}; //!< Immature coinbases in the main chain + CAmount m_watchonly_trusted{0}; + CAmount m_watchonly_untrusted_pending{0}; + CAmount m_watchonly_immature{0}; + }; + Balance GetBalance(int min_depth = 0) const; CAmount GetAvailableBalance(const CCoinControl* coinControl = nullptr) const; OutputType TransactionChangeType(OutputType change_type, const std::vector<CRecipient>& vecSend); @@ -1228,6 +1232,12 @@ public: friend struct WalletTestingSetup; }; +/** + * Called periodically by the schedule thread. Prompts individual wallets to resend + * their transactions. Actual rebroadcast schedule is managed by the wallets themselves. + */ +void MaybeResendWalletTxs(); + /** A key allocated from the key pool. */ class CReserveKey final : public CReserveScript { diff --git a/test/functional/feature_config_args.py b/test/functional/feature_config_args.py index 13481b0478..2b93e3c24d 100755 --- a/test/functional/feature_config_args.py +++ b/test/functional/feature_config_args.py @@ -25,6 +25,10 @@ class ConfArgsTest(BitcoinTestFramework): conf.write('-dash=1\n') self.nodes[0].assert_start_raises_init_error(expected_msg='Error reading configuration file: parse error on line 1: -dash=1, options in configuration file must be specified without leading -') + with open(inc_conf_file_path, 'w', encoding='utf8') as conf: + conf.write("wallet=foo\n") + self.nodes[0].assert_start_raises_init_error(expected_msg='Error: Config setting for -wallet only applied on regtest network when in [regtest] section.') + with open(inc_conf_file_path, 'w', encoding='utf-8') as conf: conf.write('nono\n') self.nodes[0].assert_start_raises_init_error(expected_msg='Error reading configuration file: parse error on line 1: nono, if you intended to specify a negated option, use nono=1 instead') diff --git a/test/functional/feature_pruning.py b/test/functional/feature_pruning.py index 12da2655ee..49951e24f3 100755 --- a/test/functional/feature_pruning.py +++ b/test/functional/feature_pruning.py @@ -8,11 +8,13 @@ WARNING: This test uses 4GB of disk space. This test takes 30 mins or more (up to 2 hours) """ +import os +from test_framework.blocktools import create_coinbase +from test_framework.messages import CBlock, ToHex +from test_framework.script import CScript, OP_RETURN, OP_NOP from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import assert_equal, assert_greater_than, assert_raises_rpc_error, connect_nodes, mine_large_block, sync_blocks, wait_until - -import os +from test_framework.util import assert_equal, assert_greater_than, assert_raises_rpc_error, connect_nodes, disconnect_nodes, sync_blocks, wait_until MIN_BLOCKS_TO_KEEP = 288 @@ -21,19 +23,59 @@ MIN_BLOCKS_TO_KEEP = 288 # compatible with pruning based on key creation time. TIMESTAMP_WINDOW = 2 * 60 * 60 +def mine_large_blocks(node, n): + # Make a large scriptPubKey for the coinbase transaction. This is OP_RETURN + # followed by 950k of OP_NOP. This would be non-standard in a non-coinbase + # transaction but is consensus valid. + + # Get the block parameters for the first block + big_script = CScript([OP_RETURN] + [OP_NOP] * 950000) + best_block = node.getblock(node.getbestblockhash()) + height = int(best_block["height"]) + 1 + try: + # Static variable ensures that time is monotonicly increasing and is therefore + # different for each block created => blockhash is unique. + mine_large_blocks.nTime = min(mine_large_blocks.nTime, int(best_block["time"])) + 1 + except AttributeError: + mine_large_blocks.nTime = int(best_block["time"]) + 1 + previousblockhash = int(best_block["hash"], 16) + + for _ in range(n): + # Build the coinbase transaction (with large scriptPubKey) + coinbase_tx = create_coinbase(height) + coinbase_tx.vin[0].nSequence = 2 ** 32 - 1 + coinbase_tx.vout[0].scriptPubKey = big_script + coinbase_tx.rehash() + + # Build the block + block = CBlock() + block.nVersion = best_block["version"] + block.hashPrevBlock = previousblockhash + block.nTime = mine_large_blocks.nTime + block.nBits = int('207fffff', 16) + block.nNonce = 0 + block.vtx = [coinbase_tx] + block.hashMerkleRoot = block.calc_merkle_root() + block.solve() + + # Submit to the node + node.submitblock(ToHex(block)) + + previousblockhash = block.sha256 + height += 1 + mine_large_blocks.nTime += 1 def calc_usage(blockdir): - return sum(os.path.getsize(blockdir+f) for f in os.listdir(blockdir) if os.path.isfile(os.path.join(blockdir, f))) / (1024. * 1024.) + return sum(os.path.getsize(blockdir + f) for f in os.listdir(blockdir) if os.path.isfile(os.path.join(blockdir, f))) / (1024. * 1024.) class PruneTest(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = True self.num_nodes = 6 - self.rpc_timeout = 900 # Create nodes 0 and 1 to mine. # Create node 2 to test pruning. - self.full_node_default_args = ["-maxreceivebuffer=20000", "-checkblocks=5", "-limitdescendantcount=100", "-limitdescendantsize=5000", "-limitancestorcount=100", "-limitancestorsize=5000"] + self.full_node_default_args = ["-maxreceivebuffer=20000", "-checkblocks=5"] # Create nodes 3 and 4 to test manual pruning (they will be re-started with manual pruning later) # Create nodes 5 to test wallet in prune mode, but do not connect self.extra_args = [ @@ -55,7 +97,7 @@ class PruneTest(BitcoinTestFramework): connect_nodes(self.nodes[0], 1) connect_nodes(self.nodes[1], 2) - connect_nodes(self.nodes[2], 0) + connect_nodes(self.nodes[0], 2) connect_nodes(self.nodes[0], 3) connect_nodes(self.nodes[0], 4) sync_blocks(self.nodes[0:5]) @@ -71,21 +113,19 @@ class PruneTest(BitcoinTestFramework): self.nodes[1].generate(200) sync_blocks(self.nodes[0:2]) self.nodes[0].generate(150) + # Then mine enough full blocks to create more than 550MiB of data - for i in range(645): - mine_large_block(self.nodes[0], self.utxo_cache_0) + mine_large_blocks(self.nodes[0], 645) sync_blocks(self.nodes[0:5]) def test_height_min(self): - if not os.path.isfile(os.path.join(self.prunedir, "blk00000.dat")): - raise AssertionError("blk00000.dat is missing, pruning too early") + assert os.path.isfile(os.path.join(self.prunedir, "blk00000.dat")), "blk00000.dat is missing, pruning too early" self.log.info("Success") self.log.info("Though we're already using more than 550MiB, current usage: %d" % calc_usage(self.prunedir)) self.log.info("Mining 25 more blocks should cause the first block file to be pruned") # Pruning doesn't run until we're allocating another chunk, 20 full blocks past the height cutoff will ensure this - for i in range(25): - mine_large_block(self.nodes[0], self.utxo_cache_0) + mine_large_blocks(self.nodes[0], 25) # Wait for blk00000.dat to be pruned wait_until(lambda: not os.path.isfile(os.path.join(self.prunedir, "blk00000.dat")), timeout=30) @@ -93,8 +133,7 @@ class PruneTest(BitcoinTestFramework): self.log.info("Success") usage = calc_usage(self.prunedir) self.log.info("Usage should be below target: %d" % usage) - if (usage > 550): - raise AssertionError("Pruning target not being met") + assert_greater_than(550, usage) def create_chain_with_staleblocks(self): # Create stale blocks in manageable sized chunks @@ -103,26 +142,17 @@ class PruneTest(BitcoinTestFramework): for j in range(12): # Disconnect node 0 so it can mine a longer reorg chain without knowing about node 1's soon-to-be-stale chain # Node 2 stays connected, so it hears about the stale blocks and then reorg's when node0 reconnects - # Stopping node 0 also clears its mempool, so it doesn't have node1's transactions to accidentally mine - self.stop_node(0) - self.start_node(0, extra_args=self.full_node_default_args) + disconnect_nodes(self.nodes[0], 1) + disconnect_nodes(self.nodes[0], 2) # Mine 24 blocks in node 1 - for i in range(24): - if j == 0: - mine_large_block(self.nodes[1], self.utxo_cache_1) - else: - # Add node1's wallet transactions back to the mempool, to - # avoid the mined blocks from being too small. - self.nodes[1].resendwallettransactions() - self.nodes[1].generate(1) #tx's already in mempool from previous disconnects + mine_large_blocks(self.nodes[1], 24) # Reorg back with 25 block chain from node 0 - for i in range(25): - mine_large_block(self.nodes[0], self.utxo_cache_0) + mine_large_blocks(self.nodes[0], 25) # Create connections in the order so both nodes can see the reorg at the same time - connect_nodes(self.nodes[1], 0) - connect_nodes(self.nodes[2], 0) + connect_nodes(self.nodes[0], 1) + connect_nodes(self.nodes[0], 2) sync_blocks(self.nodes[0:3]) self.log.info("Usage can be over target because of high stale rate: %d" % calc_usage(self.prunedir)) @@ -130,63 +160,48 @@ class PruneTest(BitcoinTestFramework): def reorg_test(self): # Node 1 will mine a 300 block chain starting 287 blocks back from Node 0 and Node 2's tip # This will cause Node 2 to do a reorg requiring 288 blocks of undo data to the reorg_test chain - # Reboot node 1 to clear its mempool (hopefully make the invalidate faster) - # Lower the block max size so we don't keep mining all our big mempool transactions (from disconnected blocks) - self.stop_node(1) - self.start_node(1, extra_args=["-maxreceivebuffer=20000","-blockmaxweight=20000", "-checkblocks=5"]) height = self.nodes[1].getblockcount() self.log.info("Current block height: %d" % height) - invalidheight = height-287 - badhash = self.nodes[1].getblockhash(invalidheight) - self.log.info("Invalidating block %s at height %d" % (badhash,invalidheight)) - self.nodes[1].invalidateblock(badhash) + self.forkheight = height - 287 + self.forkhash = self.nodes[1].getblockhash(self.forkheight) + self.log.info("Invalidating block %s at height %d" % (self.forkhash, self.forkheight)) + self.nodes[1].invalidateblock(self.forkhash) # We've now switched to our previously mined-24 block fork on node 1, but that's not what we want # So invalidate that fork as well, until we're on the same chain as node 0/2 (but at an ancestor 288 blocks ago) - mainchainhash = self.nodes[0].getblockhash(invalidheight - 1) - curhash = self.nodes[1].getblockhash(invalidheight - 1) + mainchainhash = self.nodes[0].getblockhash(self.forkheight - 1) + curhash = self.nodes[1].getblockhash(self.forkheight - 1) while curhash != mainchainhash: self.nodes[1].invalidateblock(curhash) - curhash = self.nodes[1].getblockhash(invalidheight - 1) + curhash = self.nodes[1].getblockhash(self.forkheight - 1) - assert self.nodes[1].getblockcount() == invalidheight - 1 + assert self.nodes[1].getblockcount() == self.forkheight - 1 self.log.info("New best height: %d" % self.nodes[1].getblockcount()) - # Reboot node1 to clear those giant tx's from mempool - self.stop_node(1) - self.start_node(1, extra_args=["-maxreceivebuffer=20000","-blockmaxweight=20000", "-checkblocks=5"]) + # Disconnect node1 and generate the new chain + disconnect_nodes(self.nodes[0], 1) + disconnect_nodes(self.nodes[1], 2) self.log.info("Generating new longer chain of 300 more blocks") self.nodes[1].generate(300) self.log.info("Reconnect nodes") connect_nodes(self.nodes[0], 1) - connect_nodes(self.nodes[2], 1) + connect_nodes(self.nodes[1], 2) sync_blocks(self.nodes[0:3], timeout=120) self.log.info("Verify height on node 2: %d" % self.nodes[2].getblockcount()) - self.log.info("Usage possibly still high bc of stale blocks in block files: %d" % calc_usage(self.prunedir)) - - self.log.info("Mine 220 more blocks so we have requisite history (some blocks will be big and cause pruning of previous chain)") + self.log.info("Usage possibly still high because of stale blocks in block files: %d" % calc_usage(self.prunedir)) - # Get node0's wallet transactions back in its mempool, to avoid the - # mined blocks from being too small. - self.nodes[0].resendwallettransactions() + self.log.info("Mine 220 more large blocks so we have requisite history") - for i in range(22): - # This can be slow, so do this in multiple RPC calls to avoid - # RPC timeouts. - self.nodes[0].generate(10) #node 0 has many large tx's in its mempool from the disconnects - sync_blocks(self.nodes[0:3], timeout=300) + mine_large_blocks(self.nodes[0], 220) usage = calc_usage(self.prunedir) self.log.info("Usage should be below target: %d" % usage) - if (usage > 550): - raise AssertionError("Pruning target not being met") - - return invalidheight,badhash + assert_greater_than(550, usage) def reorg_back(self): # Verify that a block on the old main chain fork has been pruned away @@ -219,17 +234,17 @@ class PruneTest(BitcoinTestFramework): blocks_to_mine = first_reorg_height + 1 - self.mainchainheight self.log.info("Rewind node 0 to prev main chain to mine longer chain to trigger redownload. Blocks needed: %d" % blocks_to_mine) self.nodes[0].invalidateblock(curchainhash) - assert self.nodes[0].getblockcount() == self.mainchainheight - assert self.nodes[0].getbestblockhash() == self.mainchainhash2 + assert_equal(self.nodes[0].getblockcount(), self.mainchainheight) + assert_equal(self.nodes[0].getbestblockhash(), self.mainchainhash2) goalbesthash = self.nodes[0].generate(blocks_to_mine)[-1] goalbestheight = first_reorg_height + 1 self.log.info("Verify node 2 reorged back to the main chain, some blocks of which it had to redownload") # Wait for Node 2 to reorg to proper height wait_until(lambda: self.nodes[2].getblockcount() >= goalbestheight, timeout=900) - assert self.nodes[2].getbestblockhash() == goalbesthash + assert_equal(self.nodes[2].getbestblockhash(), goalbesthash) # Verify we can now have the data for a block previously pruned - assert self.nodes[2].getblock(self.forkhash)["height"] == self.forkheight + assert_equal(self.nodes[2].getblock(self.forkhash)["height"], self.forkheight) def manual_test(self, node_number, use_timestamp): # at this point, node has 995 blocks and has not yet run in prune mode @@ -287,38 +302,30 @@ class PruneTest(BitcoinTestFramework): # height=100 too low to prune first block file so this is a no-op prune(100) - if not has_block(0): - raise AssertionError("blk00000.dat is missing when should still be there") + assert has_block(0), "blk00000.dat is missing when should still be there" # Does nothing node.pruneblockchain(height(0)) - if not has_block(0): - raise AssertionError("blk00000.dat is missing when should still be there") + assert has_block(0), "blk00000.dat is missing when should still be there" # height=500 should prune first file prune(500) - if has_block(0): - raise AssertionError("blk00000.dat is still there, should be pruned by now") - if not has_block(1): - raise AssertionError("blk00001.dat is missing when should still be there") + assert not has_block(0), "blk00000.dat is still there, should be pruned by now" + assert has_block(1), "blk00001.dat is missing when should still be there" # height=650 should prune second file prune(650) - if has_block(1): - raise AssertionError("blk00001.dat is still there, should be pruned by now") + assert not has_block(1), "blk00001.dat is still there, should be pruned by now" # height=1000 should not prune anything more, because tip-288 is in blk00002.dat. prune(1000, 1001 - MIN_BLOCKS_TO_KEEP) - if not has_block(2): - raise AssertionError("blk00002.dat is still there, should be pruned by now") + assert has_block(2), "blk00002.dat is still there, should be pruned by now" # advance the tip so blk00002.dat and blk00003.dat can be pruned (the last 288 blocks should now be in blk00004.dat) node.generate(288) prune(1000) - if has_block(2): - raise AssertionError("blk00002.dat is still there, should be pruned by now") - if has_block(3): - raise AssertionError("blk00003.dat is still there, should be pruned by now") + assert not has_block(2), "blk00002.dat is still there, should be pruned by now" + assert not has_block(3), "blk00003.dat is still there, should be pruned by now" # stop node, start back up with auto-prune at 550 MiB, make sure still runs self.stop_node(node_number) @@ -339,21 +346,14 @@ class PruneTest(BitcoinTestFramework): connect_nodes(self.nodes[0], 5) nds = [self.nodes[0], self.nodes[5]] sync_blocks(nds, wait=5, timeout=300) - self.stop_node(5) #stop and start to trigger rescan + self.stop_node(5) # stop and start to trigger rescan self.start_node(5, extra_args=["-prune=550"]) self.log.info("Success") def run_test(self): - self.log.info("Warning! This test requires 4GB of disk space and takes over 30 mins (up to 2 hours)") - self.log.info("Mining a big blockchain of 995 blocks") - - # Determine default relay fee - self.relayfee = self.nodes[0].getnetworkinfo()["relayfee"] - - # Cache for utxos, as the listunspent may take a long time later in the test - self.utxo_cache_0 = [] - self.utxo_cache_1 = [] + self.log.info("Warning! This test requires 4GB of disk space") + self.log.info("Mining a big blockchain of 995 blocks") self.create_big_chain() # Chain diagram key: # * blocks on main chain @@ -394,11 +394,11 @@ class PruneTest(BitcoinTestFramework): # +...+(1044) &.. $...$(1319) # Save some current chain state for later use - self.mainchainheight = self.nodes[2].getblockcount() #1320 + self.mainchainheight = self.nodes[2].getblockcount() # 1320 self.mainchainhash2 = self.nodes[2].getblockhash(self.mainchainheight) self.log.info("Check that we can survive a 288 block reorg still") - (self.forkheight,self.forkhash) = self.reorg_test() #(1033, ) + self.reorg_test() # (1033, ) # Now create a 288 block reorg by mining a longer chain on N1 # First disconnect N1 # Then invalidate 1033 on main chain and 1032 on fork so height is 1032 on main chain diff --git a/test/functional/interface_rpc.py b/test/functional/interface_rpc.py index 67ab1f0130..49ae0fb1a9 100755 --- a/test/functional/interface_rpc.py +++ b/test/functional/interface_rpc.py @@ -4,9 +4,19 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Tests some generic aspects of the RPC interface.""" +from test_framework.authproxy import JSONRPCException from test_framework.test_framework import BitcoinTestFramework from test_framework.util import assert_equal, assert_greater_than_or_equal +def expect_http_status(expected_http_status, expected_rpc_code, + fcn, *args): + try: + fcn(*args) + raise AssertionError("Expected RPC error %d, got none" % expected_rpc_code) + except JSONRPCException as exc: + assert_equal(exc.error["code"], expected_rpc_code) + assert_equal(exc.http_status, expected_http_status) + class RPCInterfaceTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 1 @@ -48,9 +58,16 @@ class RPCInterfaceTest(BitcoinTestFramework): assert_equal(result_by_id[3]['error'], None) assert result_by_id[3]['result'] is not None + def test_http_status_codes(self): + self.log.info("Testing HTTP status codes for JSON-RPC requests...") + + expect_http_status(404, -32601, self.nodes[0].invalidmethod) + expect_http_status(500, -8, self.nodes[0].getblockhash, 42) + def run_test(self): self.test_getrpcinfo() self.test_batch_request() + self.test_http_status_codes() if __name__ == '__main__': diff --git a/test/functional/mempool_packages.py b/test/functional/mempool_packages.py index 7cee215d54..507cdfa9f5 100755 --- a/test/functional/mempool_packages.py +++ b/test/functional/mempool_packages.py @@ -58,9 +58,9 @@ class MempoolPackagesTest(BitcoinTestFramework): assert_equal(len(mempool), MAX_ANCESTORS) descendant_count = 1 descendant_fees = 0 - descendant_size = 0 + descendant_vsize = 0 - ancestor_size = sum([mempool[tx]['size'] for tx in mempool]) + ancestor_vsize = sum([mempool[tx]['vsize'] for tx in mempool]) ancestor_count = MAX_ANCESTORS ancestor_fees = sum([mempool[tx]['fee'] for tx in mempool]) @@ -79,15 +79,15 @@ class MempoolPackagesTest(BitcoinTestFramework): assert_equal(mempool[x]['fees']['modified'], mempool[x]['modifiedfee']) assert_equal(mempool[x]['descendantfees'], descendant_fees * COIN) assert_equal(mempool[x]['fees']['descendant'], descendant_fees) - descendant_size += mempool[x]['size'] - assert_equal(mempool[x]['descendantsize'], descendant_size) + descendant_vsize += mempool[x]['vsize'] + assert_equal(mempool[x]['descendantsize'], descendant_vsize) descendant_count += 1 # Check that ancestor calculations are correct assert_equal(mempool[x]['ancestorcount'], ancestor_count) assert_equal(mempool[x]['ancestorfees'], ancestor_fees * COIN) - assert_equal(mempool[x]['ancestorsize'], ancestor_size) - ancestor_size -= mempool[x]['size'] + assert_equal(mempool[x]['ancestorsize'], ancestor_vsize) + ancestor_vsize -= mempool[x]['vsize'] ancestor_fees -= mempool[x]['fee'] ancestor_count -= 1 diff --git a/test/functional/mining_prioritisetransaction.py b/test/functional/mining_prioritisetransaction.py index 687a5b527b..b0a069be81 100755 --- a/test/functional/mining_prioritisetransaction.py +++ b/test/functional/mining_prioritisetransaction.py @@ -64,7 +64,7 @@ class PrioritiseTransactionTest(BitcoinTestFramework): for i in range(3): for j in txids[i]: assert j in mempool - sizes[i] += mempool[j]['size'] + sizes[i] += mempool[j]['vsize'] assert sizes[i] > MAX_BLOCK_BASE_SIZE # Fail => raise utxo_count # add a fee delta to something in the cheapest bucket and make sure it gets mined diff --git a/test/functional/p2p_compactblocks.py b/test/functional/p2p_compactblocks.py index 82a14af9da..3ca6bec254 100755 --- a/test/functional/p2p_compactblocks.py +++ b/test/functional/p2p_compactblocks.py @@ -7,19 +7,18 @@ Version 1 compact blocks are pre-segwit (txids) Version 2 compact blocks are post-segwit (wtxids) """ -from decimal import Decimal import random from test_framework.blocktools import create_block, create_coinbase, add_witness_commitment -from test_framework.messages import BlockTransactions, BlockTransactionsRequest, calculate_shortid, CBlock, CBlockHeader, CInv, COutPoint, CTransaction, CTxIn, CTxInWitness, CTxOut, FromHex, HeaderAndShortIDs, msg_block, msg_blocktxn, msg_cmpctblock, msg_getblocktxn, msg_getdata, msg_getheaders, msg_headers, msg_inv, msg_sendcmpct, msg_sendheaders, msg_tx, msg_witness_block, msg_witness_blocktxn, MSG_WITNESS_FLAG, NODE_NETWORK, NODE_WITNESS, P2PHeaderAndShortIDs, PrefilledTransaction, ser_uint256, ToHex +from test_framework.messages import BlockTransactions, BlockTransactionsRequest, calculate_shortid, CBlock, CBlockHeader, CInv, COutPoint, CTransaction, CTxIn, CTxInWitness, CTxOut, FromHex, HeaderAndShortIDs, msg_block, msg_blocktxn, msg_cmpctblock, msg_getblocktxn, msg_getdata, msg_getheaders, msg_headers, msg_inv, msg_sendcmpct, msg_sendheaders, msg_tx, msg_witness_block, msg_witness_blocktxn, MSG_WITNESS_FLAG, NODE_NETWORK, P2PHeaderAndShortIDs, PrefilledTransaction, ser_uint256, ToHex from test_framework.mininode import mininode_lock, P2PInterface from test_framework.script import CScript, OP_TRUE, OP_DROP from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import assert_equal, get_bip9_status, satoshi_round, sync_blocks, wait_until +from test_framework.util import assert_equal, get_bip9_status, wait_until # TestP2PConn: A peer we use to send messages to bitcoind, and store responses. class TestP2PConn(P2PInterface): - def __init__(self): + def __init__(self, cmpct_version): super().__init__() self.last_sendcmpct = [] self.block_announced = False @@ -27,6 +26,7 @@ class TestP2PConn(P2PInterface): # This is for synchronizing the p2p message traffic, # so we can eg wait until a particular block is announced. self.announced_blockhashes = set() + self.cmpct_version = cmpct_version def on_sendcmpct(self, message): self.last_sendcmpct.append(message) @@ -94,11 +94,7 @@ class TestP2PConn(P2PInterface): class CompactBlocksTest(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = True - # Node0 = pre-segwit, node1 = segwit-aware - self.num_nodes = 2 - # This test was written assuming SegWit is activated using BIP9 at height 432 (3x confirmation window). - # TODO: Rewrite this test to support SegWit being always active. - self.extra_args = [["-vbparams=segwit:0:0"], ["-vbparams=segwit:0:999999999999", "-txindex"]] + self.num_nodes = 1 self.utxos = [] def skip_test_if_missing_module(self): @@ -117,11 +113,10 @@ class CompactBlocksTest(BitcoinTestFramework): # Create 10 more anyone-can-spend utxo's for testing. def make_utxos(self): - # Doesn't matter which node we use, just use node0. block = self.build_block_on_tip(self.nodes[0]) - self.test_node.send_and_ping(msg_block(block)) + self.segwit_node.send_and_ping(msg_block(block)) assert int(self.nodes[0].getbestblockhash(), 16) == block.sha256 - self.nodes[0].generate(100) + self.nodes[0].generatetoaddress(100, self.nodes[0].getnewaddress(address_type="bech32")) total_value = block.vtx[0].vout[0].nValue out_value = total_value // 10 @@ -135,10 +130,10 @@ class CompactBlocksTest(BitcoinTestFramework): block2.vtx.append(tx) block2.hashMerkleRoot = block2.calc_merkle_root() block2.solve() - self.test_node.send_and_ping(msg_block(block2)) + self.segwit_node.send_and_ping(msg_block(block2)) assert_equal(int(self.nodes[0].getbestblockhash(), 16), block2.sha256) self.utxos.extend([[tx.sha256, i, out_value] for i in range(10)]) - return + # Test "sendcmpct" (between peers preferring the same version): # - No compact block announcements unless sendcmpct is sent. @@ -149,7 +144,10 @@ class CompactBlocksTest(BitcoinTestFramework): # are made with compact blocks. # If old_node is passed in, request compact blocks with version=preferred-1 # and verify that it receives block announcements via compact block. - def test_sendcmpct(self, node, test_node, preferred_version, old_node=None): + def test_sendcmpct(self, test_node, old_node=None): + preferred_version = test_node.cmpct_version + node = self.nodes[0] + # Make sure we get a SENDCMPCT message from our peer def received_sendcmpct(): return (len(test_node.last_sendcmpct) > 0) @@ -251,23 +249,18 @@ class CompactBlocksTest(BitcoinTestFramework): # This index will be too high prefilled_txn = PrefilledTransaction(1, block.vtx[0]) cmpct_block.prefilled_txn = [prefilled_txn] - self.test_node.send_await_disconnect(msg_cmpctblock(cmpct_block)) + self.segwit_node.send_await_disconnect(msg_cmpctblock(cmpct_block)) assert_equal(int(self.nodes[0].getbestblockhash(), 16), block.hashPrevBlock) # Compare the generated shortids to what we expect based on BIP 152, given # bitcoind's choice of nonce. - def test_compactblock_construction(self, node, test_node, version, use_witness_address): + def test_compactblock_construction(self, test_node, use_witness_address=True): + version = test_node.cmpct_version + node = self.nodes[0] # Generate a bunch of transactions. node.generate(101) num_transactions = 25 address = node.getnewaddress() - if use_witness_address: - # Want at least one segwit spend, so move all funds to - # a witness address. - address = node.getnewaddress(address_type='bech32') - value_to_send = node.getbalance() - node.sendtoaddress(address, satoshi_round(value_to_send - Decimal(0.1))) - node.generate(1) segwit_tx_generated = False for i in range(num_transactions): @@ -285,7 +278,7 @@ class CompactBlocksTest(BitcoinTestFramework): test_node.wait_for_block_announcement(tip) # Make sure we will receive a fast-announce compact block - self.request_cb_announcements(test_node, node, version) + self.request_cb_announcements(test_node) # Now mine a block, and look at the resulting compact block. test_node.clear_block_announcement() @@ -375,7 +368,9 @@ class CompactBlocksTest(BitcoinTestFramework): # Post-segwit: upgraded nodes would only make this request of cb-version-2, # NODE_WITNESS peers. Unupgraded nodes would still make this request of # any cb-version-1-supporting peer. - def test_compactblock_requests(self, node, test_node, version, segwit): + def test_compactblock_requests(self, test_node, segwit=True): + version = test_node.cmpct_version + node = self.nodes[0] # Try announcing a block with an inv or header, expect a compactblock # request for announce in ["inv", "header"]: @@ -440,7 +435,9 @@ class CompactBlocksTest(BitcoinTestFramework): # Test that we only receive getblocktxn requests for transactions that the # node needs, and that responding to them causes the block to be # reconstructed. - def test_getblocktxn_requests(self, node, test_node, version): + def test_getblocktxn_requests(self, test_node): + version = test_node.cmpct_version + node = self.nodes[0] with_witness = (version == 2) def test_getblocktxn_response(compact_block, peer, expected_result): @@ -523,9 +520,9 @@ class CompactBlocksTest(BitcoinTestFramework): # Incorrectly responding to a getblocktxn shouldn't cause the block to be # permanently failed. - def test_incorrect_blocktxn_response(self, node, test_node, version): - if (len(self.utxos) == 0): - self.make_utxos() + def test_incorrect_blocktxn_response(self, test_node): + version = test_node.cmpct_version + node = self.nodes[0] utxo = self.utxos.pop(0) block = self.build_block_with_transactions(node, utxo, 10) @@ -579,7 +576,9 @@ class CompactBlocksTest(BitcoinTestFramework): test_node.send_and_ping(msg_block(block)) assert_equal(int(node.getbestblockhash(), 16), block.sha256) - def test_getblocktxn_handler(self, node, test_node, version): + def test_getblocktxn_handler(self, test_node): + version = test_node.cmpct_version + node = self.nodes[0] # bitcoind will not send blocktxn responses for blocks whose height is # more than 10 blocks deep. MAX_GETBLOCKTXN_DEPTH = 10 @@ -626,7 +625,8 @@ class CompactBlocksTest(BitcoinTestFramework): assert_equal(test_node.last_message["block"].block.sha256, int(block_hash, 16)) assert "blocktxn" not in test_node.last_message - def test_compactblocks_not_at_tip(self, node, test_node): + def test_compactblocks_not_at_tip(self, test_node): + node = self.nodes[0] # Test that requesting old compactblocks doesn't work. MAX_CMPCTBLOCK_DEPTH = 5 new_blocks = [] @@ -681,11 +681,8 @@ class CompactBlocksTest(BitcoinTestFramework): with mininode_lock: assert "blocktxn" not in test_node.last_message - def activate_segwit(self, node): - node.generate(144 * 3) - assert_equal(get_bip9_status(node, "segwit")["status"], 'active') - - def test_end_to_end_block_relay(self, node, listeners): + def test_end_to_end_block_relay(self, listeners): + node = self.nodes[0] utxo = self.utxos.pop(0) block = self.build_block_with_transactions(node, utxo, 10) @@ -706,7 +703,8 @@ class CompactBlocksTest(BitcoinTestFramework): # Test that we don't get disconnected if we relay a compact block with valid header, # but invalid transactions. - def test_invalid_tx_in_compactblock(self, node, test_node, use_segwit): + def test_invalid_tx_in_compactblock(self, test_node, use_segwit=True): + node = self.nodes[0] assert len(self.utxos) utxo = self.utxos[0] @@ -733,16 +731,18 @@ class CompactBlocksTest(BitcoinTestFramework): # Helper for enabling cb announcements # Send the sendcmpct request and sync headers - def request_cb_announcements(self, peer, node, version): + def request_cb_announcements(self, peer): + node = self.nodes[0] tip = node.getbestblockhash() peer.get_headers(locator=[int(tip, 16)], hashstop=0) msg = msg_sendcmpct() - msg.version = version + msg.version = peer.cmpct_version msg.announce = True peer.send_and_ping(msg) - def test_compactblock_reconstruction_multiple_peers(self, node, stalling_peer, delivery_peer): + def test_compactblock_reconstruction_multiple_peers(self, stalling_peer, delivery_peer): + node = self.nodes[0] assert len(self.utxos) def announce_cmpct_block(node, peer): @@ -793,126 +793,55 @@ class CompactBlocksTest(BitcoinTestFramework): def run_test(self): # Setup the p2p connections - self.test_node = self.nodes[0].add_p2p_connection(TestP2PConn()) - self.segwit_node = self.nodes[1].add_p2p_connection(TestP2PConn(), services=NODE_NETWORK | NODE_WITNESS) - self.old_node = self.nodes[1].add_p2p_connection(TestP2PConn(), services=NODE_NETWORK) + self.segwit_node = self.nodes[0].add_p2p_connection(TestP2PConn(cmpct_version=2)) + self.old_node = self.nodes[0].add_p2p_connection(TestP2PConn(cmpct_version=1), services=NODE_NETWORK) + self.additional_segwit_node = self.nodes[0].add_p2p_connection(TestP2PConn(cmpct_version=2)) # We will need UTXOs to construct transactions in later tests. self.make_utxos() - self.log.info("Running tests, pre-segwit activation:") + assert_equal(get_bip9_status(self.nodes[0], "segwit")["status"], 'active') self.log.info("Testing SENDCMPCT p2p message... ") - self.test_sendcmpct(self.nodes[0], self.test_node, 1) - sync_blocks(self.nodes) - self.test_sendcmpct(self.nodes[1], self.segwit_node, 2, old_node=self.old_node) - sync_blocks(self.nodes) - - self.log.info("Testing compactblock construction...") - self.test_compactblock_construction(self.nodes[0], self.test_node, 1, False) - sync_blocks(self.nodes) - self.test_compactblock_construction(self.nodes[1], self.segwit_node, 2, False) - sync_blocks(self.nodes) - - self.log.info("Testing compactblock requests... ") - self.test_compactblock_requests(self.nodes[0], self.test_node, 1, False) - sync_blocks(self.nodes) - self.test_compactblock_requests(self.nodes[1], self.segwit_node, 2, False) - sync_blocks(self.nodes) - - self.log.info("Testing getblocktxn requests...") - self.test_getblocktxn_requests(self.nodes[0], self.test_node, 1) - sync_blocks(self.nodes) - self.test_getblocktxn_requests(self.nodes[1], self.segwit_node, 2) - sync_blocks(self.nodes) - - self.log.info("Testing getblocktxn handler...") - self.test_getblocktxn_handler(self.nodes[0], self.test_node, 1) - sync_blocks(self.nodes) - self.test_getblocktxn_handler(self.nodes[1], self.segwit_node, 2) - self.test_getblocktxn_handler(self.nodes[1], self.old_node, 1) - sync_blocks(self.nodes) - - self.log.info("Testing compactblock requests/announcements not at chain tip...") - self.test_compactblocks_not_at_tip(self.nodes[0], self.test_node) - sync_blocks(self.nodes) - self.test_compactblocks_not_at_tip(self.nodes[1], self.segwit_node) - self.test_compactblocks_not_at_tip(self.nodes[1], self.old_node) - sync_blocks(self.nodes) - - self.log.info("Testing handling of incorrect blocktxn responses...") - self.test_incorrect_blocktxn_response(self.nodes[0], self.test_node, 1) - sync_blocks(self.nodes) - self.test_incorrect_blocktxn_response(self.nodes[1], self.segwit_node, 2) - sync_blocks(self.nodes) - - # End-to-end block relay tests - self.log.info("Testing end-to-end block relay...") - self.request_cb_announcements(self.test_node, self.nodes[0], 1) - self.request_cb_announcements(self.old_node, self.nodes[1], 1) - self.request_cb_announcements(self.segwit_node, self.nodes[1], 2) - self.test_end_to_end_block_relay(self.nodes[0], [self.segwit_node, self.test_node, self.old_node]) - self.test_end_to_end_block_relay(self.nodes[1], [self.segwit_node, self.test_node, self.old_node]) - - self.log.info("Testing handling of invalid compact blocks...") - self.test_invalid_tx_in_compactblock(self.nodes[0], self.test_node, False) - self.test_invalid_tx_in_compactblock(self.nodes[1], self.segwit_node, False) - self.test_invalid_tx_in_compactblock(self.nodes[1], self.old_node, False) - - self.log.info("Testing reconstructing compact blocks from all peers...") - self.test_compactblock_reconstruction_multiple_peers(self.nodes[1], self.segwit_node, self.old_node) - sync_blocks(self.nodes) - - # Advance to segwit activation - self.log.info("Advancing to segwit activation") - self.activate_segwit(self.nodes[1]) - self.log.info("Running tests, post-segwit activation...") + self.test_sendcmpct(self.segwit_node, old_node=self.old_node) + self.test_sendcmpct(self.additional_segwit_node) self.log.info("Testing compactblock construction...") - self.test_compactblock_construction(self.nodes[1], self.old_node, 1, True) - self.test_compactblock_construction(self.nodes[1], self.segwit_node, 2, True) - sync_blocks(self.nodes) - - self.log.info("Testing compactblock requests (unupgraded node)... ") - self.test_compactblock_requests(self.nodes[0], self.test_node, 1, True) - - self.log.info("Testing getblocktxn requests (unupgraded node)...") - self.test_getblocktxn_requests(self.nodes[0], self.test_node, 1) - - # Need to manually sync node0 and node1, because post-segwit activation, - # node1 will not download blocks from node0. - self.log.info("Syncing nodes...") - assert self.nodes[0].getbestblockhash() != self.nodes[1].getbestblockhash() - while (self.nodes[0].getblockcount() > self.nodes[1].getblockcount()): - block_hash = self.nodes[0].getblockhash(self.nodes[1].getblockcount() + 1) - self.nodes[1].submitblock(self.nodes[0].getblock(block_hash, False)) - assert_equal(self.nodes[0].getbestblockhash(), self.nodes[1].getbestblockhash()) + self.test_compactblock_construction(self.old_node) + self.test_compactblock_construction(self.segwit_node) self.log.info("Testing compactblock requests (segwit node)... ") - self.test_compactblock_requests(self.nodes[1], self.segwit_node, 2, True) + self.test_compactblock_requests(self.segwit_node) self.log.info("Testing getblocktxn requests (segwit node)...") - self.test_getblocktxn_requests(self.nodes[1], self.segwit_node, 2) - sync_blocks(self.nodes) + self.test_getblocktxn_requests(self.segwit_node) self.log.info("Testing getblocktxn handler (segwit node should return witnesses)...") - self.test_getblocktxn_handler(self.nodes[1], self.segwit_node, 2) - self.test_getblocktxn_handler(self.nodes[1], self.old_node, 1) + self.test_getblocktxn_handler(self.segwit_node) + self.test_getblocktxn_handler(self.old_node) + + self.log.info("Testing compactblock requests/announcements not at chain tip...") + self.test_compactblocks_not_at_tip(self.segwit_node) + self.test_compactblocks_not_at_tip(self.old_node) + + self.log.info("Testing handling of incorrect blocktxn responses...") + self.test_incorrect_blocktxn_response(self.segwit_node) + + self.log.info("Testing reconstructing compact blocks from all peers...") + self.test_compactblock_reconstruction_multiple_peers(self.segwit_node, self.additional_segwit_node) # Test that if we submitblock to node1, we'll get a compact block # announcement to all peers. # (Post-segwit activation, blocks won't propagate from node0 to node1 # automatically, so don't bother testing a block announced to node0.) self.log.info("Testing end-to-end block relay...") - self.request_cb_announcements(self.test_node, self.nodes[0], 1) - self.request_cb_announcements(self.old_node, self.nodes[1], 1) - self.request_cb_announcements(self.segwit_node, self.nodes[1], 2) - self.test_end_to_end_block_relay(self.nodes[1], [self.segwit_node, self.test_node, self.old_node]) + self.request_cb_announcements(self.old_node) + self.request_cb_announcements(self.segwit_node) + self.test_end_to_end_block_relay([self.segwit_node, self.old_node]) self.log.info("Testing handling of invalid compact blocks...") - self.test_invalid_tx_in_compactblock(self.nodes[0], self.test_node, False) - self.test_invalid_tx_in_compactblock(self.nodes[1], self.segwit_node, True) - self.test_invalid_tx_in_compactblock(self.nodes[1], self.old_node, True) + self.test_invalid_tx_in_compactblock(self.segwit_node) + self.test_invalid_tx_in_compactblock(self.old_node) self.log.info("Testing invalid index in cmpctblock message...") self.test_invalid_cmpctblock_message() diff --git a/test/functional/test_framework/authproxy.py b/test/functional/test_framework/authproxy.py index 1140fe9b3e..4ba6ac1db2 100644 --- a/test/functional/test_framework/authproxy.py +++ b/test/functional/test_framework/authproxy.py @@ -35,6 +35,7 @@ ServiceProxy class: import base64 import decimal +from http import HTTPStatus import http.client import json import logging @@ -49,13 +50,14 @@ USER_AGENT = "AuthServiceProxy/0.1" log = logging.getLogger("BitcoinRPC") class JSONRPCException(Exception): - def __init__(self, rpc_error): + def __init__(self, rpc_error, http_status=None): try: errmsg = '%(message)s (%(code)i)' % rpc_error except (KeyError, TypeError): errmsg = '' super().__init__(errmsg) self.error = rpc_error + self.http_status = http_status def EncodeDecimal(o): @@ -120,8 +122,11 @@ class AuthServiceProxy(): def get_request(self, *args, **argsn): AuthServiceProxy.__id_count += 1 - log.debug("-%s-> %s %s" % (AuthServiceProxy.__id_count, self._service_name, - json.dumps(args, default=EncodeDecimal, ensure_ascii=self.ensure_ascii))) + log.debug("-{}-> {} {}".format( + AuthServiceProxy.__id_count, + self._service_name, + json.dumps(args or argsn, default=EncodeDecimal, ensure_ascii=self.ensure_ascii), + )) if args and argsn: raise ValueError('Cannot handle both named and positional arguments') return {'version': '1.1', @@ -131,19 +136,26 @@ class AuthServiceProxy(): def __call__(self, *args, **argsn): postdata = json.dumps(self.get_request(*args, **argsn), default=EncodeDecimal, ensure_ascii=self.ensure_ascii) - response = self._request('POST', self.__url.path, postdata.encode('utf-8')) + response, status = self._request('POST', self.__url.path, postdata.encode('utf-8')) if response['error'] is not None: - raise JSONRPCException(response['error']) + raise JSONRPCException(response['error'], status) elif 'result' not in response: raise JSONRPCException({ - 'code': -343, 'message': 'missing JSON-RPC result'}) + 'code': -343, 'message': 'missing JSON-RPC result'}, status) + elif status != HTTPStatus.OK: + raise JSONRPCException({ + 'code': -342, 'message': 'non-200 HTTP status code but no JSON-RPC error'}, status) else: return response['result'] def batch(self, rpc_call_list): postdata = json.dumps(list(rpc_call_list), default=EncodeDecimal, ensure_ascii=self.ensure_ascii) log.debug("--> " + postdata) - return self._request('POST', self.__url.path, postdata.encode('utf-8')) + response, status = self._request('POST', self.__url.path, postdata.encode('utf-8')) + if status != HTTPStatus.OK: + raise JSONRPCException({ + 'code': -342, 'message': 'non-200 HTTP status code but no JSON-RPC error'}, status) + return response def _get_response(self): req_start_time = time.time() @@ -162,8 +174,9 @@ class AuthServiceProxy(): content_type = http_response.getheader('Content-Type') if content_type != 'application/json': - raise JSONRPCException({ - 'code': -342, 'message': 'non-JSON HTTP response with \'%i %s\' from server' % (http_response.status, http_response.reason)}) + raise JSONRPCException( + {'code': -342, 'message': 'non-JSON HTTP response with \'%i %s\' from server' % (http_response.status, http_response.reason)}, + http_response.status) responsedata = http_response.read().decode('utf8') response = json.loads(responsedata, parse_float=decimal.Decimal) @@ -172,7 +185,7 @@ class AuthServiceProxy(): log.debug("<-%s- [%.6f] %s" % (response["id"], elapsed, json.dumps(response["result"], default=EncodeDecimal, ensure_ascii=self.ensure_ascii))) else: log.debug("<-- [%.6f] %s" % (elapsed, responsedata)) - return response + return response, http_response.status def __truediv__(self, relative_uri): return AuthServiceProxy("{}/{}".format(self.__service_url, relative_uri), self._service_name, connection=self.__conn) diff --git a/test/functional/test_framework/test_node.py b/test/functional/test_framework/test_node.py index ec5d3b267e..8b2006a05c 100755 --- a/test/functional/test_framework/test_node.py +++ b/test/functional/test_framework/test_node.py @@ -369,7 +369,7 @@ class TestNode(): stderr=subprocess.DEVNULL, stdout=subprocess.DEVNULL) == 0 if not sys.platform.startswith('linux'): - self.log.warning("Can't profile with perf; only availabe on Linux platforms") + self.log.warning("Can't profile with perf; only available on Linux platforms") return None if not test_success('which perf'): diff --git a/test/functional/test_framework/util.py b/test/functional/test_framework/util.py index f771a37b99..190301b215 100644 --- a/test/functional/test_framework/util.py +++ b/test/functional/test_framework/util.py @@ -323,12 +323,14 @@ def get_auth_cookie(datadir): if line.startswith("rpcpassword="): assert password is None # Ensure that there is only one rpcpassword line password = line.split("=")[1].strip("\n") - if os.path.isfile(os.path.join(datadir, "regtest", ".cookie")) and os.access(os.path.join(datadir, "regtest", ".cookie"), os.R_OK): + try: with open(os.path.join(datadir, "regtest", ".cookie"), 'r', encoding="ascii") as f: userpass = f.read() split_userpass = userpass.split(':') user = split_userpass[0] password = split_userpass[1] + except OSError: + pass if user is None or password is None: raise ValueError("No RPC credentials") return user, password diff --git a/test/functional/wallet_balance.py b/test/functional/wallet_balance.py index 05c97e0340..3a65b684c8 100755 --- a/test/functional/wallet_balance.py +++ b/test/functional/wallet_balance.py @@ -129,5 +129,17 @@ class WalletTest(BitcoinTestFramework): # getbalance with minconf=2 will show the new balance. assert_equal(self.nodes[1].getbalance(minconf=2), Decimal('0')) + # check mempool transactions count for wallet unconfirmed balance after + # dynamically loading the wallet. + before = self.nodes[1].getunconfirmedbalance() + dst = self.nodes[1].getnewaddress() + self.nodes[1].unloadwallet('') + self.nodes[0].sendtoaddress(dst, 0.1) + self.sync_all() + self.nodes[1].loadwallet('') + after = self.nodes[1].getunconfirmedbalance() + assert_equal(before + Decimal('0.1'), after) + + if __name__ == '__main__': WalletTest().main() diff --git a/test/functional/wallet_basic.py b/test/functional/wallet_basic.py index b7be605f62..c89f42fc8a 100755 --- a/test/functional/wallet_basic.py +++ b/test/functional/wallet_basic.py @@ -14,7 +14,6 @@ from test_framework.util import ( assert_raises_rpc_error, connect_nodes_bi, sync_blocks, - sync_mempools, wait_until, ) @@ -216,23 +215,9 @@ class WalletTest(BitcoinTestFramework): assert_equal(self.nodes[2].getbalance(), node_2_bal) node_0_bal = self.check_fee_amount(self.nodes[0].getbalance(), node_0_bal + Decimal('10'), fee_per_byte, self.get_vsize(self.nodes[2].gettransaction(txid)['hex'])) - # Test ResendWalletTransactions: - # Create a couple of transactions, then start up a fourth - # node (nodes[3]) and ask nodes[0] to rebroadcast. - # EXPECT: nodes[3] should have those transactions in its mempool. - txid1 = self.nodes[0].sendtoaddress(self.nodes[1].getnewaddress(), 1) - txid2 = self.nodes[1].sendtoaddress(self.nodes[0].getnewaddress(), 1) - sync_mempools(self.nodes[0:2]) - self.start_node(3) connect_nodes_bi(self.nodes, 0, 3) - sync_blocks(self.nodes) - - relayed = self.nodes[0].resendwallettransactions() - assert_equal(set(relayed), {txid1, txid2}) - sync_mempools(self.nodes) - - assert txid1 in self.nodes[3].getrawmempool() + self.sync_all() # check if we can list zero value tx as available coins # 1. create raw_tx diff --git a/test/functional/wallet_importmulti.py b/test/functional/wallet_importmulti.py index 5bfbaa2f0b..81c650f4c1 100755 --- a/test/functional/wallet_importmulti.py +++ b/test/functional/wallet_importmulti.py @@ -629,7 +629,8 @@ class ImportMultiTest(BitcoinTestFramework): self.log.info("Should import a 1-of-2 bare multisig from descriptor") self.test_importmulti({"desc": descsum_create("multi(1," + key1.pubkey + "," + key2.pubkey + ")"), "timestamp": "now"}, - success=True) + success=True, + warnings=["Some private keys are missing, outputs will be considered watchonly. If this is intentional, specify the watchonly flag."]) self.log.info("Should not treat individual keys from the imported bare multisig as watchonly") test_address(self.nodes[1], key1.p2pkh_addr, @@ -759,7 +760,7 @@ class ImportMultiTest(BitcoinTestFramework): assert_equal(addr2, newaddr2) # Import a multisig and make sure the keys don't go into the keypool - self.log.info('Imported scripts with pubkeys shoud not have their pubkeys go into the keypool') + self.log.info('Imported scripts with pubkeys should not have their pubkeys go into the keypool') addr1 = self.nodes[0].getnewaddress() addr2 = self.nodes[0].getnewaddress() pub1 = self.nodes[0].getaddressinfo(addr1)['pubkey'] diff --git a/test/functional/wallet_resendwallettransactions.py b/test/functional/wallet_resendwallettransactions.py index 00bf58d709..ec5e230e5a 100755 --- a/test/functional/wallet_resendwallettransactions.py +++ b/test/functional/wallet_resendwallettransactions.py @@ -2,31 +2,76 @@ # Copyright (c) 2017-2018 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. -"""Test resendwallettransactions RPC.""" +"""Test that the wallet resends transactions periodically.""" +from collections import defaultdict +import time +from test_framework.blocktools import create_block, create_coinbase +from test_framework.messages import ToHex +from test_framework.mininode import P2PInterface, mininode_lock from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import assert_equal, assert_raises_rpc_error +from test_framework.util import assert_equal, wait_until + +class P2PStoreTxInvs(P2PInterface): + def __init__(self): + super().__init__() + self.tx_invs_received = defaultdict(int) + + def on_inv(self, message): + # Store how many times invs have been received for each tx. + for i in message.inv: + if i.type == 1: + # save txid + self.tx_invs_received[i.hash] += 1 class ResendWalletTransactionsTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 1 - self.extra_args = [['--walletbroadcast=false']] def skip_test_if_missing_module(self): self.skip_if_no_wallet() def run_test(self): - # Should raise RPC_WALLET_ERROR (-4) if walletbroadcast is disabled. - assert_raises_rpc_error(-4, "Error: Wallet transaction broadcasting is disabled with -walletbroadcast", self.nodes[0].resendwallettransactions) + node = self.nodes[0] # alias + + node.add_p2p_connection(P2PStoreTxInvs()) + + self.log.info("Create a new transaction and wait until it's broadcast") + txid = int(node.sendtoaddress(node.getnewaddress(), 1), 16) + + # Wallet rebroadcast is first scheduled 1 sec after startup (see + # nNextResend in ResendWalletTransactions()). Sleep for just over a + # second to be certain that it has been called before the first + # setmocktime call below. + time.sleep(1.1) + + # Can take a few seconds due to transaction trickling + wait_until(lambda: node.p2p.tx_invs_received[txid] >= 1, lock=mininode_lock) + + # Add a second peer since txs aren't rebroadcast to the same peer (see filterInventoryKnown) + node.add_p2p_connection(P2PStoreTxInvs()) + + self.log.info("Create a block") + # Create and submit a block without the transaction. + # Transactions are only rebroadcast if there has been a block at least five minutes + # after the last time we tried to broadcast. Use mocktime and give an extra minute to be sure. + block_time = int(time.time()) + 6 * 60 + node.setmocktime(block_time) + block = create_block(int(node.getbestblockhash(), 16), create_coinbase(node.getblockchaininfo()['blocks']), block_time) + block.nVersion = 3 + block.rehash() + block.solve() + node.submitblock(ToHex(block)) - # Should return an empty array if there aren't unconfirmed wallet transactions. - self.stop_node(0) - self.start_node(0, extra_args=[]) - assert_equal(self.nodes[0].resendwallettransactions(), []) + # Transaction should not be rebroadcast + node.p2ps[1].sync_with_ping() + assert_equal(node.p2ps[1].tx_invs_received[txid], 0) - # Should return an array with the unconfirmed wallet transaction. - txid = self.nodes[0].sendtoaddress(self.nodes[0].getnewaddress(), 1) - assert_equal(self.nodes[0].resendwallettransactions(), [txid]) + self.log.info("Transaction should be rebroadcast after 30 minutes") + # Use mocktime and give an extra 5 minutes to be sure. + rebroadcast_time = int(time.time()) + 41 * 60 + node.setmocktime(rebroadcast_time) + wait_until(lambda: node.p2ps[1].tx_invs_received[txid] >= 1, lock=mininode_lock) if __name__ == '__main__': ResendWalletTransactionsTest().main() diff --git a/test/sanitizer_suppressions/ubsan b/test/sanitizer_suppressions/ubsan index d55119b266..643272de52 100644 --- a/test/sanitizer_suppressions/ubsan +++ b/test/sanitizer_suppressions/ubsan @@ -15,6 +15,7 @@ unsigned-integer-overflow:coded_stream.h unsigned-integer-overflow:core_write.cpp unsigned-integer-overflow:crypto/chacha20.cpp unsigned-integer-overflow:crypto/ctaes/ctaes.c +unsigned-integer-overflow:crypto/poly1305.cpp unsigned-integer-overflow:crypto/ripemd160.cpp unsigned-integer-overflow:crypto/sha1.cpp unsigned-integer-overflow:crypto/sha256.cpp |