diff options
46 files changed, 410 insertions, 282 deletions
diff --git a/.travis.yml b/.travis.yml index 31b3d6d96e..95ef36bf02 100644 --- a/.travis.yml +++ b/.travis.yml @@ -64,6 +64,7 @@ before_script: - if [ -n "$OSX_SDK" -a -f depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz ]; then tar -C depends/SDKs -xf depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz; fi - make $MAKEJOBS -C depends HOST=$HOST $DEP_OPTS script: + - export TRAVIS_COMMIT_LOG=`git log --format=fuller -1` - if [ -n "$USE_SHELL" ]; then export CONFIG_SHELL="$USE_SHELL"; fi - OUTDIR=$BASE_OUTDIR/$TRAVIS_PULL_REQUEST/$TRAVIS_JOB_NUMBER-$HOST - BITCOIN_CONFIG_ALL="--disable-dependency-tracking --prefix=$TRAVIS_BUILD_DIR/depends/$HOST --bindir=$OUTDIR/bin --libdir=$OUTDIR/lib" @@ -79,4 +80,6 @@ script: - if [ "$RUN_TESTS" = "true" ]; then make check; fi - if [ "$RUN_TESTS" = "true" ]; then qa/pull-tester/rpc-tests.py --coverage; fi after_script: + - echo $TRAVIS_COMMIT_RANGE + - echo $TRAVIS_COMMIT_LOG - if [ "$TRAVIS_PULL_REQUEST" != "false" ]; then (echo "Upload goes here. Something like: scp -r $BASE_OUTDIR server" || echo "upload failed"); fi diff --git a/depends/Makefile b/depends/Makefile index ef5a20e6c3..3ddfc85a45 100644 --- a/depends/Makefile +++ b/depends/Makefile @@ -15,6 +15,8 @@ BASEDIR = $(CURDIR) HASH_LENGTH:=11 DOWNLOAD_CONNECT_TIMEOUT:=10 DOWNLOAD_RETRIES:=3 +HOST_ID_SALT ?= salt +BUILD_ID_SALT ?= salt host:=$(BUILD) ifneq ($(HOST),) @@ -73,6 +75,20 @@ include builders/$(build_os).mk include builders/default.mk include packages/packages.mk +build_id_string:=$(BUILD_ID_SALT) +build_id_string+=$(shell $(build_CC) --version 2>/dev/null) +build_id_string+=$(shell $(build_AR) --version 2>/dev/null) +build_id_string+=$(shell $(build_CXX) --version 2>/dev/null) +build_id_string+=$(shell $(build_RANLIB) --version 2>/dev/null) +build_id_string+=$(shell $(build_STRIP) --version 2>/dev/null) + +$(host_arch)_$(host_os)_id_string:=$(HOST_ID_SALT) +$(host_arch)_$(host_os)_id_string+=$(shell $(host_CC) --version 2>/dev/null) +$(host_arch)_$(host_os)_id_string+=$(shell $(host_AR) --version 2>/dev/null) +$(host_arch)_$(host_os)_id_string+=$(shell $(host_CXX) --version 2>/dev/null) +$(host_arch)_$(host_os)_id_string+=$(shell $(host_RANLIB) --version 2>/dev/null) +$(host_arch)_$(host_os)_id_string+=$(shell $(host_STRIP) --version 2>/dev/null) + qt_packages_$(NO_QT) = $(qt_packages) $(qt_$(host_os)_packages) qt_native_packages_$(NO_QT) = $(qt_native_packages) wallet_packages_$(NO_WALLET) = $(wallet_packages) @@ -90,7 +106,7 @@ include funcs.mk toolchain_path=$($($(host_arch)_$(host_os)_native_toolchain)_prefixbin) final_build_id_long+=$(shell $(build_SHA256SUM) config.site.in) -final_build_id+=$(shell echo -n $(final_build_id_long) | $(build_SHA256SUM) | cut -c-$(HASH_LENGTH)) +final_build_id+=$(shell echo -n "$(final_build_id_long)" | $(build_SHA256SUM) | cut -c-$(HASH_LENGTH)) $(host_prefix)/.stamp_$(final_build_id): $(native_packages) $(packages) $(AT)rm -rf $(@D) $(AT)mkdir -p $(@D) @@ -131,9 +147,9 @@ endef define check_or_remove_sources mkdir -p $($(package)_source_dir); cd $($(package)_source_dir); \ - $(build_SHA256SUM) -c $($(package)_fetched) >/dev/null 2>/dev/null || \ - ( if test -f $($(package)_all_sources); then echo "Checksum missing or mismatched for $(package) source. Forcing re-download."; fi; \ - rm -f $($(package)_all_sources) $($(1)_fetched)) + test -f $($(package)_fetched) && ( $(build_SHA256SUM) -c $($(package)_fetched) >/dev/null 2>/dev/null || \ + ( echo "Checksum missing or mismatched for $(package) source. Forcing re-download."; \ + rm -f $($(package)_all_sources) $($(1)_fetched))) || true endef check-packages: diff --git a/depends/README.md b/depends/README.md index d85e652c60..271bbd80bf 100644 --- a/depends/README.md +++ b/depends/README.md @@ -38,6 +38,8 @@ The following can be set when running make: make FOO=bar NO_WALLET: Don't download/build/cache libs needed to enable the wallet NO_UPNP: Don't download/build/cache packages needed for enabling upnp DEBUG: disable some optimizations and enable more runtime checking + HOST_ID_SALT: Optional salt to use when generating host package ids + BUILD_ID_SALT: Optional salt to use when generating build package ids If some packages are not built, for example `make NO_WALLET=1`, the appropriate options will be passed to bitcoin's configure. In this case, `--disable-wallet`. diff --git a/depends/funcs.mk b/depends/funcs.mk index 050a9b1321..15e404e42d 100644 --- a/depends/funcs.mk +++ b/depends/funcs.mk @@ -19,15 +19,19 @@ define int_get_all_dependencies $(sort $(foreach dep,$(2),$(2) $(call int_get_all_dependencies,$(1),$($(dep)_dependencies)))) endef -define fetch_file -(test -f $$($(1)_source_dir)/$(4) || \ - ( mkdir -p $$($(1)_download_dir) && echo Fetching $(1)... && \ - ( $(build_DOWNLOAD) "$$($(1)_download_dir)/$(4).temp" "$(2)/$(3)" || \ - $(build_DOWNLOAD) "$$($(1)_download_dir)/$(4).temp" "$(FALLBACK_DOWNLOAD_PATH)/$(3)" ) && \ +define fetch_file_inner + ( mkdir -p $$($(1)_download_dir) && echo Fetching $(3) from $(2) && \ + $(build_DOWNLOAD) "$$($(1)_download_dir)/$(4).temp" "$(2)/$(3)" && \ echo "$(5) $$($(1)_download_dir)/$(4).temp" > $$($(1)_download_dir)/.$(4).hash && \ $(build_SHA256SUM) -c $$($(1)_download_dir)/.$(4).hash && \ mv $$($(1)_download_dir)/$(4).temp $$($(1)_source_dir)/$(4) && \ - rm -rf $$($(1)_download_dir) )) + rm -rf $$($(1)_download_dir) ) +endef + +define fetch_file + ( test -f $$($(1)_source_dir)/$(4) || \ + ( $(call fetch_file_inner,$(1),$(2),$(3),$(4),$(5)) || \ + $(call fetch_file_inner,$(1),$(FALLBACK_DOWNLOAD_PATH),$(3),$(4),$(5)))) endef define int_get_build_recipe_hash @@ -39,7 +43,7 @@ define int_get_build_id $(eval $(1)_dependencies += $($(1)_$(host_arch)_$(host_os)_dependencies) $($(1)_$(host_os)_dependencies)) $(eval $(1)_all_dependencies:=$(call int_get_all_dependencies,$(1),$($($(1)_type)_native_toolchain) $($(1)_dependencies))) $(foreach dep,$($(1)_all_dependencies),$(eval $(1)_build_id_deps+=$(dep)-$($(dep)_version)-$($(dep)_recipe_hash))) -$(eval $(1)_build_id_long:=$(1)-$($(1)_version)-$($(1)_recipe_hash)-$(release_type) $($(1)_build_id_deps)) +$(eval $(1)_build_id_long:=$(1)-$($(1)_version)-$($(1)_recipe_hash)-$(release_type) $($(1)_build_id_deps) $($($(1)_type)_id_string)) $(eval $(1)_build_id:=$(shell echo -n "$($(1)_build_id_long)" | $(build_SHA256SUM) | cut -c-$(HASH_LENGTH))) final_build_id_long+=$($(package)_build_id_long) diff --git a/depends/packages/native_cctools.mk b/depends/packages/native_cctools.mk index 1c1bcf199a..b5603a8d48 100644 --- a/depends/packages/native_cctools.mk +++ b/depends/packages/native_cctools.mk @@ -17,6 +17,10 @@ $(call fetch_file,$(package),$($(package)_clang_download_path),$($(package)_clan endef define $(package)_extract_cmds + mkdir -p $($(package)_extract_dir) && \ + echo "$($(package)_sha256_hash) $($(package)_source)" > $($(package)_extract_dir)/.$($(package)_file_name).hash && \ + echo "$($(package)_clang_sha256_hash) $($(package)_source_dir)/$($(package)_clang_file_name)" >> $($(package)_extract_dir)/.$($(package)_file_name).hash && \ + $(build_SHA256SUM) -c $($(package)_extract_dir)/.$($(package)_file_name).hash && \ mkdir -p toolchain/bin toolchain/lib/clang/3.5/include && \ tar --strip-components=1 -C toolchain -xf $($(package)_source_dir)/$($(package)_clang_file_name) && \ echo "#!/bin/sh" > toolchain/bin/$(host)-dsymutil && \ diff --git a/depends/packages/qt.mk b/depends/packages/qt.mk index 901b761fde..77df77b73f 100644 --- a/depends/packages/qt.mk +++ b/depends/packages/qt.mk @@ -108,8 +108,8 @@ endef define $(package)_extract_cmds mkdir -p $($(package)_extract_dir) && \ echo "$($(package)_sha256_hash) $($(package)_source)" > $($(package)_extract_dir)/.$($(package)_file_name).hash && \ - echo "$($(package)_qttranslations_sha256_hash) $($(package)_source_dir)/$($(package)_qttranslations_file_name)" > $($(package)_extract_dir)/.$($(package)_file_name).hash && \ - echo "$($(package)_qttools_sha256_hash) $($(package)_source_dir)/$($(package)_qttools_file_name)" > $($(package)_extract_dir)/.$($(package)_file_name).hash && \ + echo "$($(package)_qttranslations_sha256_hash) $($(package)_source_dir)/$($(package)_qttranslations_file_name)" >> $($(package)_extract_dir)/.$($(package)_file_name).hash && \ + echo "$($(package)_qttools_sha256_hash) $($(package)_source_dir)/$($(package)_qttools_file_name)" >> $($(package)_extract_dir)/.$($(package)_file_name).hash && \ $(build_SHA256SUM) -c $($(package)_extract_dir)/.$($(package)_file_name).hash && \ mkdir qtbase && \ tar --strip-components=1 -xf $($(package)_source) -C qtbase && \ diff --git a/qa/rpc-tests/bip65-cltv-p2p.py b/qa/rpc-tests/bip65-cltv-p2p.py index 54559c3541..99d74344ae 100755 --- a/qa/rpc-tests/bip65-cltv-p2p.py +++ b/qa/rpc-tests/bip65-cltv-p2p.py @@ -10,7 +10,6 @@ from test_framework.mininode import CTransaction, NetworkThread from test_framework.blocktools import create_coinbase, create_block from test_framework.comptool import TestInstance, TestManager from test_framework.script import CScript, OP_1NEGATE, OP_CHECKLOCKTIMEVERIFY, OP_DROP -from binascii import unhexlify from io import BytesIO import time @@ -60,7 +59,7 @@ class BIP65Test(ComparisonTestFramework): rawtx = node.createrawtransaction(inputs, outputs) signresult = node.signrawtransaction(rawtx) tx = CTransaction() - f = BytesIO(unhexlify(signresult['hex'])) + f = BytesIO(hex_str_to_bytes(signresult['hex'])) tx.deserialize(f) return tx diff --git a/qa/rpc-tests/bip68-112-113-p2p.py b/qa/rpc-tests/bip68-112-113-p2p.py index 35c831cb8b..3bcfdabe26 100755 --- a/qa/rpc-tests/bip68-112-113-p2p.py +++ b/qa/rpc-tests/bip68-112-113-p2p.py @@ -10,7 +10,6 @@ from test_framework.mininode import ToHex, CTransaction, NetworkThread from test_framework.blocktools import create_coinbase, create_block from test_framework.comptool import TestInstance, TestManager from test_framework.script import * -from binascii import unhexlify from io import BytesIO import time @@ -119,7 +118,7 @@ class BIP68_112_113Test(ComparisonTestFramework): outputs = { to_address : amount } rawtx = node.createrawtransaction(inputs, outputs) tx = CTransaction() - f = BytesIO(unhexlify(rawtx)) + f = BytesIO(hex_str_to_bytes(rawtx)) tx.deserialize(f) return tx @@ -127,7 +126,7 @@ class BIP68_112_113Test(ComparisonTestFramework): rawtx = ToHex(unsignedtx) signresult = node.signrawtransaction(rawtx) tx = CTransaction() - f = BytesIO(unhexlify(signresult['hex'])) + f = BytesIO(hex_str_to_bytes(signresult['hex'])) tx.deserialize(f) return tx diff --git a/qa/rpc-tests/bip9-softforks.py b/qa/rpc-tests/bip9-softforks.py index e63343d352..ddca3c2e33 100755 --- a/qa/rpc-tests/bip9-softforks.py +++ b/qa/rpc-tests/bip9-softforks.py @@ -10,7 +10,6 @@ from test_framework.mininode import CTransaction, NetworkThread from test_framework.blocktools import create_coinbase, create_block from test_framework.comptool import TestInstance, TestManager from test_framework.script import CScript, OP_1NEGATE, OP_NOP3, OP_DROP -from binascii import hexlify, unhexlify from io import BytesIO import time import itertools @@ -30,7 +29,6 @@ test that enforcement has triggered ''' - class BIP9SoftForksTest(ComparisonTestFramework): def __init__(self): @@ -53,15 +51,15 @@ class BIP9SoftForksTest(ComparisonTestFramework): outputs = { to_address : amount } rawtx = node.createrawtransaction(inputs, outputs) tx = CTransaction() - f = BytesIO(unhexlify(rawtx)) + f = BytesIO(hex_str_to_bytes(rawtx)) tx.deserialize(f) tx.nVersion = 2 return tx def sign_transaction(self, node, tx): - signresult = node.signrawtransaction(hexlify(tx.serialize())) + signresult = node.signrawtransaction(bytes_to_hex_str(tx.serialize())) tx = CTransaction() - f = BytesIO(unhexlify(signresult['hex'])) + f = BytesIO(hex_str_to_bytes(signresult['hex'])) tx.deserialize(f) return tx @@ -180,7 +178,6 @@ class BIP9SoftForksTest(ComparisonTestFramework): NetworkThread().start() # Start up network handling in another thread - def get_tests(self): for test in itertools.chain( self.test_BIP('csv', 536870913, self.sequence_lock_invalidate, self.donothing), diff --git a/qa/rpc-tests/bipdersig-p2p.py b/qa/rpc-tests/bipdersig-p2p.py index 95be385d93..bba86a50c4 100755 --- a/qa/rpc-tests/bipdersig-p2p.py +++ b/qa/rpc-tests/bipdersig-p2p.py @@ -10,7 +10,6 @@ from test_framework.mininode import CTransaction, NetworkThread from test_framework.blocktools import create_coinbase, create_block from test_framework.comptool import TestInstance, TestManager from test_framework.script import CScript -from binascii import unhexlify from io import BytesIO import time @@ -25,7 +24,7 @@ def unDERify(tx): newscript = [] for i in scriptSig: if (len(newscript) == 0): - newscript.append(i[0:-1] + '\0' + i[-1]) + newscript.append(i[0:-1] + b'\0' + i[-1:]) else: newscript.append(i) tx.vin[0].scriptSig = CScript(newscript) @@ -68,7 +67,7 @@ class BIP66Test(ComparisonTestFramework): rawtx = node.createrawtransaction(inputs, outputs) signresult = node.signrawtransaction(rawtx) tx = CTransaction() - f = BytesIO(unhexlify(signresult['hex'])) + f = BytesIO(hex_str_to_bytes(signresult['hex'])) tx.deserialize(f) return tx diff --git a/qa/rpc-tests/decodescript.py b/qa/rpc-tests/decodescript.py index 2dfafac2fc..578844f2c5 100755 --- a/qa/rpc-tests/decodescript.py +++ b/qa/rpc-tests/decodescript.py @@ -6,7 +6,6 @@ from test_framework.test_framework import BitcoinTestFramework from test_framework.util import * from test_framework.mininode import * -from binascii import hexlify, unhexlify from io import BytesIO class DecodeScriptTest(BitcoinTestFramework): @@ -131,7 +130,7 @@ class DecodeScriptTest(BitcoinTestFramework): assert_equal('OP_DUP OP_HASH160 dc863734a218bfe83ef770ee9d41a27f824a6e56 OP_EQUALVERIFY OP_CHECKSIG', rpc_result['vout'][0]['scriptPubKey']['asm']) assert_equal('OP_HASH160 2a5edea39971049a540474c6a99edf0aa4074c58 OP_EQUAL', rpc_result['vout'][1]['scriptPubKey']['asm']) txSave = CTransaction() - txSave.deserialize(BytesIO(unhexlify(tx))) + txSave.deserialize(BytesIO(hex_str_to_bytes(tx))) # make sure that a specifically crafted op_return value will not pass all the IsDERSignature checks and then get decoded as a sighash type tx = '01000000015ded05872fdbda629c7d3d02b194763ce3b9b1535ea884e3c8e765d42e316724020000006b48304502204c10d4064885c42638cbff3585915b322de33762598321145ba033fc796971e2022100bb153ad3baa8b757e30a2175bd32852d2e1cb9080f84d7e32fcdfd667934ef1b012103163c0ff73511ea1743fb5b98384a2ff09dd06949488028fd819f4d83f56264efffffffff0200000000000000000b6a0930060201000201000180380100000000001976a9141cabd296e753837c086da7a45a6c2fe0d49d7b7b88ac00000000' @@ -147,7 +146,7 @@ class DecodeScriptTest(BitcoinTestFramework): # some more full transaction tests of varying specific scriptSigs. used instead of # tests in decodescript_script_sig because the decodescript RPC is specifically # for working on scriptPubKeys (argh!). - push_signature = hexlify(txSave.vin[0].scriptSig)[2:(0x48*2+4)] + push_signature = bytes_to_hex_str(txSave.vin[0].scriptSig)[2:(0x48*2+4)] signature = push_signature[2:] der_signature = signature[:-2] signature_sighash_decoded = der_signature + '[ALL]' @@ -156,25 +155,24 @@ class DecodeScriptTest(BitcoinTestFramework): signature_2_sighash_decoded = der_signature + '[NONE|ANYONECANPAY]' # 1) P2PK scriptSig - txSave.vin[0].scriptSig = unhexlify(push_signature) - rpc_result = self.nodes[0].decoderawtransaction(hexlify(txSave.serialize())) + txSave.vin[0].scriptSig = hex_str_to_bytes(push_signature) + rpc_result = self.nodes[0].decoderawtransaction(bytes_to_hex_str(txSave.serialize())) assert_equal(signature_sighash_decoded, rpc_result['vin'][0]['scriptSig']['asm']) # make sure that the sighash decodes come out correctly for a more complex / lesser used case. - txSave.vin[0].scriptSig = unhexlify(push_signature_2) - rpc_result = self.nodes[0].decoderawtransaction(hexlify(txSave.serialize())) + txSave.vin[0].scriptSig = hex_str_to_bytes(push_signature_2) + rpc_result = self.nodes[0].decoderawtransaction(bytes_to_hex_str(txSave.serialize())) assert_equal(signature_2_sighash_decoded, rpc_result['vin'][0]['scriptSig']['asm']) # 2) multisig scriptSig - txSave.vin[0].scriptSig = unhexlify('00' + push_signature + push_signature_2) - rpc_result = self.nodes[0].decoderawtransaction(hexlify(txSave.serialize())) + txSave.vin[0].scriptSig = hex_str_to_bytes('00' + push_signature + push_signature_2) + rpc_result = self.nodes[0].decoderawtransaction(bytes_to_hex_str(txSave.serialize())) assert_equal('0 ' + signature_sighash_decoded + ' ' + signature_2_sighash_decoded, rpc_result['vin'][0]['scriptSig']['asm']) # 3) test a scriptSig that contains more than push operations. # in fact, it contains an OP_RETURN with data specially crafted to cause improper decode if the code does not catch it. - txSave.vin[0].scriptSig = unhexlify('6a143011020701010101010101020601010101010101') - rpc_result = self.nodes[0].decoderawtransaction(hexlify(txSave.serialize())) - print(hexlify('636174')) + txSave.vin[0].scriptSig = hex_str_to_bytes('6a143011020701010101010101020601010101010101') + rpc_result = self.nodes[0].decoderawtransaction(bytes_to_hex_str(txSave.serialize())) assert_equal('OP_RETURN 3011020701010101010101020601010101010101', rpc_result['vin'][0]['scriptSig']['asm']) def run_test(self): diff --git a/qa/rpc-tests/fundrawtransaction.py b/qa/rpc-tests/fundrawtransaction.py index 82c9e48a49..4492ea398f 100755 --- a/qa/rpc-tests/fundrawtransaction.py +++ b/qa/rpc-tests/fundrawtransaction.py @@ -148,7 +148,6 @@ class RawTransactionsTest(BitcoinTestFramework): assert_equal(fee + totalOut, utx['amount']) #compare vin total and totalout+fee - ##################################################################### # test a fundrawtransaction with which will not get a change output # ##################################################################### @@ -178,7 +177,6 @@ class RawTransactionsTest(BitcoinTestFramework): assert_equal(fee + totalOut, utx['amount']) #compare vin total and totalout+fee - ######################################################################### # test a fundrawtransaction with a VIN smaller than the required amount # ######################################################################### @@ -484,7 +482,6 @@ class RawTransactionsTest(BitcoinTestFramework): assert_equal(oldBalance+Decimal('51.10000000'), self.nodes[0].getbalance()) - ############################################### # multiple (~19) inputs tx test | Compare fee # ############################################### diff --git a/qa/rpc-tests/getblocktemplate_proposals.py b/qa/rpc-tests/getblocktemplate_proposals.py index d2cb4ab8d6..07bfe69c6d 100755 --- a/qa/rpc-tests/getblocktemplate_proposals.py +++ b/qa/rpc-tests/getblocktemplate_proposals.py @@ -130,7 +130,7 @@ class GetBlockTemplateProposalTest(BitcoinTestFramework): # Test 5: Add an invalid tx to the end (non-duplicate) txlist.append(bytearray(txlist[0])) - txlist[-1][4+1] = b'\xff' + txlist[-1][4+1] = 0xff assert_template(node, tmpl, txlist, 'bad-txns-inputs-missingorspent') txlist.pop() diff --git a/qa/rpc-tests/httpbasics.py b/qa/rpc-tests/httpbasics.py index eff4c6e808..cf37976a4e 100755 --- a/qa/rpc-tests/httpbasics.py +++ b/qa/rpc-tests/httpbasics.py @@ -9,7 +9,6 @@ from test_framework.test_framework import BitcoinTestFramework from test_framework.util import * -import base64 try: import http.client as httplib @@ -31,7 +30,7 @@ class HTTPBasicsTest (BitcoinTestFramework): ################################################# url = urlparse.urlparse(self.nodes[0].url) authpair = url.username + ':' + url.password - headers = {"Authorization": "Basic " + base64.b64encode(authpair)} + headers = {"Authorization": "Basic " + str_to_b64str(authpair)} conn = httplib.HTTPConnection(url.hostname, url.port) conn.connect() @@ -48,7 +47,7 @@ class HTTPBasicsTest (BitcoinTestFramework): conn.close() #same should be if we add keep-alive because this should be the std. behaviour - headers = {"Authorization": "Basic " + base64.b64encode(authpair), "Connection": "keep-alive"} + headers = {"Authorization": "Basic " + str_to_b64str(authpair), "Connection": "keep-alive"} conn = httplib.HTTPConnection(url.hostname, url.port) conn.connect() @@ -65,7 +64,7 @@ class HTTPBasicsTest (BitcoinTestFramework): conn.close() #now do the same with "Connection: close" - headers = {"Authorization": "Basic " + base64.b64encode(authpair), "Connection":"close"} + headers = {"Authorization": "Basic " + str_to_b64str(authpair), "Connection":"close"} conn = httplib.HTTPConnection(url.hostname, url.port) conn.connect() @@ -77,7 +76,7 @@ class HTTPBasicsTest (BitcoinTestFramework): #node1 (2nd node) is running with disabled keep-alive option urlNode1 = urlparse.urlparse(self.nodes[1].url) authpair = urlNode1.username + ':' + urlNode1.password - headers = {"Authorization": "Basic " + base64.b64encode(authpair)} + headers = {"Authorization": "Basic " + str_to_b64str(authpair)} conn = httplib.HTTPConnection(urlNode1.hostname, urlNode1.port) conn.connect() @@ -88,7 +87,7 @@ class HTTPBasicsTest (BitcoinTestFramework): #node2 (third node) is running with standard keep-alive parameters which means keep-alive is on urlNode2 = urlparse.urlparse(self.nodes[2].url) authpair = urlNode2.username + ':' + urlNode2.password - headers = {"Authorization": "Basic " + base64.b64encode(authpair)} + headers = {"Authorization": "Basic " + str_to_b64str(authpair)} conn = httplib.HTTPConnection(urlNode2.hostname, urlNode2.port) conn.connect() diff --git a/qa/rpc-tests/invalidblockrequest.py b/qa/rpc-tests/invalidblockrequest.py index daad312d36..de6be8d5b5 100755 --- a/qa/rpc-tests/invalidblockrequest.py +++ b/qa/rpc-tests/invalidblockrequest.py @@ -77,9 +77,9 @@ class InvalidBlockRequestTest(ComparisonTestFramework): block2 = create_block(self.tip, create_coinbase(height), self.block_time) self.block_time += 1 - # chr(81) is OP_TRUE - tx1 = create_transaction(self.block1.vtx[0], 0, chr(81), 50 * COIN) - tx2 = create_transaction(tx1, 0, chr(81), 50 * COIN) + # b'0x51' is OP_TRUE + tx1 = create_transaction(self.block1.vtx[0], 0, b'\x51', 50 * COIN) + tx2 = create_transaction(tx1, 0, b'\x51', 50 * COIN) block2.vtx.extend([tx1, tx2]) block2.hashMerkleRoot = block2.calc_merkle_root() @@ -95,7 +95,7 @@ class InvalidBlockRequestTest(ComparisonTestFramework): assert(block2_orig.vtx != block2.vtx) self.tip = block2.sha256 - yield TestInstance([[block2, RejectResult(16,'bad-txns-duplicate')], [block2_orig, True]]) + yield TestInstance([[block2, RejectResult(16, b'bad-txns-duplicate')], [block2_orig, True]]) height += 1 ''' @@ -110,7 +110,7 @@ class InvalidBlockRequestTest(ComparisonTestFramework): block3.rehash() block3.solve() - yield TestInstance([[block3, RejectResult(16,'bad-cb-amount')]]) + yield TestInstance([[block3, RejectResult(16, b'bad-cb-amount')]]) if __name__ == '__main__': diff --git a/qa/rpc-tests/invalidtxrequest.py b/qa/rpc-tests/invalidtxrequest.py index 8fe471ccd3..7b8199bab0 100755 --- a/qa/rpc-tests/invalidtxrequest.py +++ b/qa/rpc-tests/invalidtxrequest.py @@ -61,10 +61,10 @@ class InvalidTxRequestTest(ComparisonTestFramework): height += 1 yield test - # chr(100) is OP_NOTIF + # b'\x64' is OP_NOTIF # Transaction will be rejected with code 16 (REJECT_INVALID) - tx1 = create_transaction(self.block1.vtx[0], 0, chr(100), 50 * COIN - 12000) - yield TestInstance([[tx1, RejectResult(16, 'mandatory-script-verify-flag-failed')]]) + tx1 = create_transaction(self.block1.vtx[0], 0, b'\x64', 50 * COIN - 12000) + yield TestInstance([[tx1, RejectResult(16, b'mandatory-script-verify-flag-failed')]]) # TODO: test further transactions... diff --git a/qa/rpc-tests/listtransactions.py b/qa/rpc-tests/listtransactions.py index 8fe72d95d1..0783a1f3d3 100755 --- a/qa/rpc-tests/listtransactions.py +++ b/qa/rpc-tests/listtransactions.py @@ -9,11 +9,10 @@ from test_framework.test_framework import BitcoinTestFramework from test_framework.util import * from test_framework.mininode import CTransaction, COIN from io import BytesIO -import binascii def txFromHex(hexstring): tx = CTransaction() - f = BytesIO(binascii.unhexlify(hexstring)) + f = BytesIO(hex_str_to_bytes(hexstring)) tx.deserialize(f) return tx @@ -167,7 +166,7 @@ class ListTransactionsTest(BitcoinTestFramework): tx3 = self.nodes[0].createrawtransaction(inputs, outputs) tx3_modified = txFromHex(tx3) tx3_modified.vin[0].nSequence = 0 - tx3 = binascii.hexlify(tx3_modified.serialize()).decode('utf-8') + tx3 = bytes_to_hex_str(tx3_modified.serialize()) tx3_signed = self.nodes[0].signrawtransaction(tx3)['hex'] txid_3 = self.nodes[0].sendrawtransaction(tx3_signed) @@ -193,7 +192,7 @@ class ListTransactionsTest(BitcoinTestFramework): # Replace tx3, and check that tx4 becomes unknown tx3_b = tx3_modified tx3_b.vout[0].nValue -= int(Decimal("0.004") * COIN) # bump the fee - tx3_b = binascii.hexlify(tx3_b.serialize()).decode('utf-8') + tx3_b = bytes_to_hex_str(tx3_b.serialize()) tx3_b_signed = self.nodes[0].signrawtransaction(tx3_b)['hex'] txid_3b = self.nodes[0].sendrawtransaction(tx3_b_signed, True) assert(is_opt_in(self.nodes[0], txid_3b)) diff --git a/qa/rpc-tests/multi_rpc.py b/qa/rpc-tests/multi_rpc.py index 2452b77319..afb18cf3da 100755 --- a/qa/rpc-tests/multi_rpc.py +++ b/qa/rpc-tests/multi_rpc.py @@ -53,7 +53,7 @@ class HTTPBasicsTest (BitcoinTestFramework): password2 = "8/F3uMDw4KSEbw96U3CA1C4X05dkHDN2BPFjTgZW4KI=" authpairnew = "rt:"+password - headers = {"Authorization": "Basic " + base64.b64encode(authpair)} + headers = {"Authorization": "Basic " + str_to_b64str(authpair)} conn = httplib.HTTPConnection(url.hostname, url.port) conn.connect() @@ -63,7 +63,7 @@ class HTTPBasicsTest (BitcoinTestFramework): conn.close() #Use new authpair to confirm both work - headers = {"Authorization": "Basic " + base64.b64encode(authpairnew)} + headers = {"Authorization": "Basic " + str_to_b64str(authpairnew)} conn = httplib.HTTPConnection(url.hostname, url.port) conn.connect() @@ -74,7 +74,7 @@ class HTTPBasicsTest (BitcoinTestFramework): #Wrong login name with rt's password authpairnew = "rtwrong:"+password - headers = {"Authorization": "Basic " + base64.b64encode(authpairnew)} + headers = {"Authorization": "Basic " + str_to_b64str(authpairnew)} conn = httplib.HTTPConnection(url.hostname, url.port) conn.connect() @@ -85,7 +85,7 @@ class HTTPBasicsTest (BitcoinTestFramework): #Wrong password for rt authpairnew = "rt:"+password+"wrong" - headers = {"Authorization": "Basic " + base64.b64encode(authpairnew)} + headers = {"Authorization": "Basic " + str_to_b64str(authpairnew)} conn = httplib.HTTPConnection(url.hostname, url.port) conn.connect() @@ -96,7 +96,7 @@ class HTTPBasicsTest (BitcoinTestFramework): #Correct for rt2 authpairnew = "rt2:"+password2 - headers = {"Authorization": "Basic " + base64.b64encode(authpairnew)} + headers = {"Authorization": "Basic " + str_to_b64str(authpairnew)} conn = httplib.HTTPConnection(url.hostname, url.port) conn.connect() @@ -107,7 +107,7 @@ class HTTPBasicsTest (BitcoinTestFramework): #Wrong password for rt2 authpairnew = "rt2:"+password2+"wrong" - headers = {"Authorization": "Basic " + base64.b64encode(authpairnew)} + headers = {"Authorization": "Basic " + str_to_b64str(authpairnew)} conn = httplib.HTTPConnection(url.hostname, url.port) conn.connect() @@ -117,6 +117,5 @@ class HTTPBasicsTest (BitcoinTestFramework): conn.close() - if __name__ == '__main__': HTTPBasicsTest ().main () diff --git a/qa/rpc-tests/p2p-feefilter.py b/qa/rpc-tests/p2p-feefilter.py index f85c18dcd5..281b6ca37a 100755 --- a/qa/rpc-tests/p2p-feefilter.py +++ b/qa/rpc-tests/p2p-feefilter.py @@ -14,7 +14,7 @@ FeeFilterTest -- test processing of feefilter messages ''' def hashToHex(hash): - return format(hash, '064x').decode('utf-8') + return format(hash, '064x') # Wait up to 60 secs to see if the testnode has received all the expected invs def allInvsMatch(invsExpected, testnode): diff --git a/qa/rpc-tests/p2p-fullblocktest.py b/qa/rpc-tests/p2p-fullblocktest.py index 131350c98d..ae82d9dca9 100755 --- a/qa/rpc-tests/p2p-fullblocktest.py +++ b/qa/rpc-tests/p2p-fullblocktest.py @@ -33,7 +33,7 @@ class FullBlockTest(ComparisonTestFramework): self.num_nodes = 1 self.block_heights = {} self.coinbase_key = CECKey() - self.coinbase_key.set_secretbytes(bytes("horsebattery")) + self.coinbase_key.set_secretbytes(b"horsebattery") self.coinbase_pubkey = self.coinbase_key.get_pubkey() self.block_time = int(time.time())+1 self.tip = None @@ -70,7 +70,7 @@ class FullBlockTest(ComparisonTestFramework): block = create_block(base_block_hash, coinbase, self.block_time) if (spend != None): tx = CTransaction() - tx.vin.append(CTxIn(COutPoint(spend.tx.sha256, spend.n), "", 0xffffffff)) # no signature yet + tx.vin.append(CTxIn(COutPoint(spend.tx.sha256, spend.n), b"", 0xffffffff)) # no signature yet # This copies the java comparison tool testing behavior: the first # txout has a garbage scriptPubKey, "to make sure we're not # pre-verifying too much" (?) @@ -80,7 +80,7 @@ class FullBlockTest(ComparisonTestFramework): else: tx.vout.append(CTxOut(1, script)) # Now sign it if necessary - scriptSig = "" + scriptSig = b"" scriptPubKey = bytearray(spend.tx.vout[spend.n].scriptPubKey) if (scriptPubKey[0] == OP_TRUE): # looks like an anyone-can-spend scriptSig = CScript([OP_TRUE]) @@ -225,7 +225,7 @@ class FullBlockTest(ComparisonTestFramework): # \-> b3 (1) -> b4 (2) tip(6) block(9, spend=out4, additional_coinbase_value=1) - yield rejected(RejectResult(16, 'bad-cb-amount')) + yield rejected(RejectResult(16, b'bad-cb-amount')) # Create a fork that ends in a block with too much fee (the one that causes the reorg) @@ -237,7 +237,7 @@ class FullBlockTest(ComparisonTestFramework): yield rejected() block(11, spend=out4, additional_coinbase_value=1) - yield rejected(RejectResult(16, 'bad-cb-amount')) + yield rejected(RejectResult(16, b'bad-cb-amount')) # Try again, but with a valid fork first @@ -279,7 +279,7 @@ class FullBlockTest(ComparisonTestFramework): out6 = get_spendable_output() too_many_checksigs = CScript([OP_CHECKSIG] * (1000000 // 50)) block(16, spend=out6, script=too_many_checksigs) - yield rejected(RejectResult(16, 'bad-blk-sigops')) + yield rejected(RejectResult(16, b'bad-blk-sigops')) # Attempt to spend a transaction created on a different fork @@ -288,7 +288,7 @@ class FullBlockTest(ComparisonTestFramework): # \-> b3 (1) -> b4 (2) tip(15) block(17, spend=txout_b3) - yield rejected(RejectResult(16, 'bad-txns-inputs-missingorspent')) + yield rejected(RejectResult(16, b'bad-txns-inputs-missingorspent')) # Attempt to spend a transaction created on a different fork (on a fork this time) # genesis -> b1 (0) -> b2 (1) -> b5 (2) -> b6 (3) @@ -309,7 +309,7 @@ class FullBlockTest(ComparisonTestFramework): tip(15) out7 = get_spendable_output() block(20, spend=out7) - yield rejected(RejectResult(16, 'bad-txns-premature-spend-of-coinbase')) + yield rejected(RejectResult(16, b'bad-txns-premature-spend-of-coinbase')) # Attempt to spend a coinbase at depth too low (on a fork this time) # genesis -> b1 (0) -> b2 (1) -> b5 (2) -> b6 (3) @@ -333,7 +333,7 @@ class FullBlockTest(ComparisonTestFramework): old_hash = b23.sha256 tx = CTransaction() script_length = MAX_BLOCK_SIZE - len(b23.serialize()) - 69 - script_output = CScript([chr(0)*script_length]) + script_output = CScript([b'\x00' * script_length]) tx.vout.append(CTxOut(0, script_output)) tx.vin.append(CTxIn(COutPoint(b23.vtx[1].sha256, 1))) b23 = update_block(23, [tx]) @@ -345,11 +345,11 @@ class FullBlockTest(ComparisonTestFramework): tip(15) b24 = block(24, spend=out6) script_length = MAX_BLOCK_SIZE - len(b24.serialize()) - 69 - script_output = CScript([chr(0)*(script_length+1)]) + script_output = CScript([b'\x00' * (script_length+1)]) tx.vout = [CTxOut(0, script_output)] b24 = update_block(24, [tx]) assert_equal(len(b24.serialize()), MAX_BLOCK_SIZE+1) - yield rejected(RejectResult(16, 'bad-blk-length')) + yield rejected(RejectResult(16, b'bad-blk-length')) b25 = block(25, spend=out7) yield rejected() @@ -361,12 +361,12 @@ class FullBlockTest(ComparisonTestFramework): # \-> b3 (1) -> b4 (2) tip(15) b26 = block(26, spend=out6) - b26.vtx[0].vin[0].scriptSig = chr(0) + b26.vtx[0].vin[0].scriptSig = b'\x00' b26.vtx[0].rehash() # update_block causes the merkle root to get updated, even with no new # transactions, and updates the required state. b26 = update_block(26, []) - yield rejected(RejectResult(16, 'bad-cb-length')) + yield rejected(RejectResult(16, b'bad-cb-length')) # Extend the b26 chain to make sure bitcoind isn't accepting b26 b27 = block(27, spend=out7) @@ -375,10 +375,10 @@ class FullBlockTest(ComparisonTestFramework): # Now try a too-large-coinbase script tip(15) b28 = block(28, spend=out6) - b28.vtx[0].vin[0].scriptSig = chr(0)*101 + b28.vtx[0].vin[0].scriptSig = b'\x00' * 101 b28.vtx[0].rehash() b28 = update_block(28, []) - yield rejected(RejectResult(16, 'bad-cb-length')) + yield rejected(RejectResult(16, b'bad-cb-length')) # Extend the b28 chain to make sure bitcoind isn't accepted b28 b29 = block(29, spend=out7) @@ -390,7 +390,7 @@ class FullBlockTest(ComparisonTestFramework): # b30 has a max-sized coinbase scriptSig. tip(23) b30 = block(30) - b30.vtx[0].vin[0].scriptSig = chr(0)*100 + b30.vtx[0].vin[0].scriptSig = b'\x00' * 100 b30.vtx[0].rehash() b30 = update_block(30, []) yield accepted() diff --git a/qa/rpc-tests/proxy_test.py b/qa/rpc-tests/proxy_test.py index b3c65573ea..91c871ddc9 100755 --- a/qa/rpc-tests/proxy_test.py +++ b/qa/rpc-tests/proxy_test.py @@ -86,7 +86,7 @@ class ProxyTest(BitcoinTestFramework): assert(isinstance(cmd, Socks5Command)) # Note: bitcoind's SOCKS5 implementation only sends atyp DOMAINNAME, even if connecting directly to IPv4/IPv6 assert_equal(cmd.atyp, AddressType.DOMAINNAME) - assert_equal(cmd.addr, "15.61.23.23") + assert_equal(cmd.addr, b"15.61.23.23") assert_equal(cmd.port, 1234) if not auth: assert_equal(cmd.username, None) @@ -100,7 +100,7 @@ class ProxyTest(BitcoinTestFramework): assert(isinstance(cmd, Socks5Command)) # Note: bitcoind's SOCKS5 implementation only sends atyp DOMAINNAME, even if connecting directly to IPv4/IPv6 assert_equal(cmd.atyp, AddressType.DOMAINNAME) - assert_equal(cmd.addr, "1233:3432:2434:2343:3234:2345:6546:4534") + assert_equal(cmd.addr, b"1233:3432:2434:2343:3234:2345:6546:4534") assert_equal(cmd.port, 5443) if not auth: assert_equal(cmd.username, None) @@ -113,7 +113,7 @@ class ProxyTest(BitcoinTestFramework): cmd = proxies[2].queue.get() assert(isinstance(cmd, Socks5Command)) assert_equal(cmd.atyp, AddressType.DOMAINNAME) - assert_equal(cmd.addr, "bitcoinostk4e4re.onion") + assert_equal(cmd.addr, b"bitcoinostk4e4re.onion") assert_equal(cmd.port, 8333) if not auth: assert_equal(cmd.username, None) @@ -125,7 +125,7 @@ class ProxyTest(BitcoinTestFramework): cmd = proxies[3].queue.get() assert(isinstance(cmd, Socks5Command)) assert_equal(cmd.atyp, AddressType.DOMAINNAME) - assert_equal(cmd.addr, "node.noumenon") + assert_equal(cmd.addr, b"node.noumenon") assert_equal(cmd.port, 8333) if not auth: assert_equal(cmd.username, None) diff --git a/qa/rpc-tests/pruning.py b/qa/rpc-tests/pruning.py index dd2adea950..eccd157e5c 100755 --- a/qa/rpc-tests/pruning.py +++ b/qa/rpc-tests/pruning.py @@ -311,7 +311,7 @@ class PruneTest(BitcoinTestFramework): # \ \ # ++...++(1044) .. # - # N0 ********************(1032) @@...@@@(1552) + # N0 ********************(1032) @@...@@@(1552) # \ # *...**(1320) diff --git a/qa/rpc-tests/rawtransactions.py b/qa/rpc-tests/rawtransactions.py index 762a6d6a30..e38ef6c8b1 100755 --- a/qa/rpc-tests/rawtransactions.py +++ b/qa/rpc-tests/rawtransactions.py @@ -88,8 +88,6 @@ class RawTransactionsTest(BitcoinTestFramework): assert_equal(self.nodes[2].getbalance(), bal+Decimal('1.20000000')) #node2 has both keys of the 2of2 ms addr., tx should affect the balance - - # 2of3 test from different nodes bal = self.nodes[2].getbalance() addr1 = self.nodes[1].getnewaddress() diff --git a/qa/rpc-tests/replace-by-fee.py b/qa/rpc-tests/replace-by-fee.py index b951900c4d..4c8ef6de26 100755 --- a/qa/rpc-tests/replace-by-fee.py +++ b/qa/rpc-tests/replace-by-fee.py @@ -11,15 +11,11 @@ from test_framework.test_framework import BitcoinTestFramework from test_framework.util import * from test_framework.script import * from test_framework.mininode import * -import binascii MAX_REPLACEMENT_LIMIT = 100 -def satoshi_round(amount): - return Decimal(amount).quantize(Decimal('0.00000001'), rounding=ROUND_DOWN) - def txToHex(tx): - return binascii.hexlify(tx.serialize()).decode('utf-8') + return bytes_to_hex_str(tx.serialize()) def make_utxo(node, amount, confirmed=True, scriptPubKey=CScript([1])): """Create a txout with a given amount and scriptPubKey @@ -53,9 +49,7 @@ def make_utxo(node, amount, confirmed=True, scriptPubKey=CScript([1])): tx2.vout = [CTxOut(amount, scriptPubKey)] tx2.rehash() - binascii.hexlify(tx2.serialize()).decode('utf-8') - - signed_tx = node.signrawtransaction(binascii.hexlify(tx2.serialize()).decode('utf-8')) + signed_tx = node.signrawtransaction(txToHex(tx2)) txid = node.sendrawtransaction(signed_tx['hex'], True) diff --git a/qa/rpc-tests/rest.py b/qa/rpc-tests/rest.py index 3c8a405bdd..359f9239fb 100755 --- a/qa/rpc-tests/rest.py +++ b/qa/rpc-tests/rest.py @@ -39,7 +39,7 @@ def http_get_call(host, port, path, response_object = 0): if response_object: return conn.getresponse() - return conn.getresponse().read() + return conn.getresponse().read().decode('utf-8') #allows simple http post calls with a request body def http_post_call(host, port, path, requestdata = '', response_object = 0): @@ -141,9 +141,9 @@ class RESTTest (BitcoinTestFramework): bb_hash = self.nodes[0].getbestblockhash() binaryRequest = b'\x01\x02' - binaryRequest += binascii.unhexlify(txid) + binaryRequest += hex_str_to_bytes(txid) binaryRequest += pack("i", n) - binaryRequest += binascii.unhexlify(vintx) + binaryRequest += hex_str_to_bytes(vintx) binaryRequest += pack("i", 0) bin_response = http_post_call(url.hostname, url.port, '/rest/getutxos'+self.FORMAT_SEPARATOR+'bin', binaryRequest) @@ -234,7 +234,7 @@ class RESTTest (BitcoinTestFramework): assert_equal(response_hex.status, 200) assert_greater_than(int(response_hex.getheader('content-length')), 160) response_hex_str = response_hex.read() - assert_equal(encode(response_str, "hex")[0:160], response_hex_str[0:160]) + assert_equal(encode(response_str, "hex_codec")[0:160], response_hex_str[0:160]) # compare with hex block header response_header_hex = http_get_call(url.hostname, url.port, '/rest/headers/1/'+bb_hash+self.FORMAT_SEPARATOR+"hex", True) @@ -242,7 +242,7 @@ class RESTTest (BitcoinTestFramework): assert_greater_than(int(response_header_hex.getheader('content-length')), 160) response_header_hex_str = response_header_hex.read() assert_equal(response_hex_str[0:160], response_header_hex_str[0:160]) - assert_equal(encode(response_header_str, "hex")[0:160], response_header_hex_str[0:160]) + assert_equal(encode(response_header_str, "hex_codec")[0:160], response_header_hex_str[0:160]) # check json format block_json_string = http_get_call(url.hostname, url.port, '/rest/block/'+bb_hash+self.FORMAT_SEPARATOR+'json') @@ -252,7 +252,7 @@ class RESTTest (BitcoinTestFramework): # compare with json block header response_header_json = http_get_call(url.hostname, url.port, '/rest/headers/1/'+bb_hash+self.FORMAT_SEPARATOR+"json", True) assert_equal(response_header_json.status, 200) - response_header_json_str = response_header_json.read() + response_header_json_str = response_header_json.read().decode('utf-8') json_obj = json.loads(response_header_json_str, parse_float=Decimal) assert_equal(len(json_obj), 1) #ensure that there is one header in the json response assert_equal(json_obj[0]['hash'], bb_hash) #request/response hash should be the same @@ -276,7 +276,7 @@ class RESTTest (BitcoinTestFramework): self.sync_all() response_header_json = http_get_call(url.hostname, url.port, '/rest/headers/5/'+bb_hash+self.FORMAT_SEPARATOR+"json", True) assert_equal(response_header_json.status, 200) - response_header_json_str = response_header_json.read() + response_header_json_str = response_header_json.read().decode('utf-8') json_obj = json.loads(response_header_json_str) assert_equal(len(json_obj), 5) #now we should have 5 header objects @@ -292,7 +292,6 @@ class RESTTest (BitcoinTestFramework): assert_greater_than(int(response.getheader('content-length')), 10) - # check block tx details # let's make 3 tx and mine them on node 1 txs = [] diff --git a/qa/rpc-tests/test_framework/blocktools.py b/qa/rpc-tests/test_framework/blocktools.py index afa0f5f9ba..384f40e62d 100644 --- a/qa/rpc-tests/test_framework/blocktools.py +++ b/qa/rpc-tests/test_framework/blocktools.py @@ -62,6 +62,6 @@ def create_transaction(prevtx, n, sig, value): tx = CTransaction() assert(n < len(prevtx.vout)) tx.vin.append(CTxIn(COutPoint(prevtx.sha256, n), sig, 0xffffffff)) - tx.vout.append(CTxOut(value, "")) + tx.vout.append(CTxOut(value, b"")) tx.calc_sha256() return tx diff --git a/qa/rpc-tests/test_framework/comptool.py b/qa/rpc-tests/test_framework/comptool.py index 6279070fbc..17626cf8da 100755 --- a/qa/rpc-tests/test_framework/comptool.py +++ b/qa/rpc-tests/test_framework/comptool.py @@ -31,7 +31,7 @@ class RejectResult(object): ''' Outcome that expects rejection of a transaction or block. ''' - def __init__(self, code, reason=''): + def __init__(self, code, reason=b''): self.code = code self.reason = reason def match(self, other): @@ -97,9 +97,9 @@ class TestNode(NodeConnCB): raise AssertionError("Got pong for unknown ping [%s]" % repr(message)) def on_reject(self, conn, message): - if message.message == 'tx': + if message.message == b'tx': self.tx_reject_map[message.data] = RejectResult(message.code, message.reason) - if message.message == 'block': + if message.message == b'block': self.block_reject_map[message.data] = RejectResult(message.code, message.reason) def send_inv(self, obj): diff --git a/qa/rpc-tests/test_framework/mininode.py b/qa/rpc-tests/test_framework/mininode.py index 53f5e8805a..5ee5b13275 100755 --- a/qa/rpc-tests/test_framework/mininode.py +++ b/qa/rpc-tests/test_framework/mininode.py @@ -20,10 +20,10 @@ import struct import socket import asyncore -import binascii import time import sys import random +from binascii import hexlify, unhexlify from io import BytesIO from codecs import encode import hashlib @@ -34,7 +34,7 @@ import copy BIP0031_VERSION = 60000 MY_VERSION = 60001 # past bip-31 for ping/pong -MY_SUBVERSION = "/python-mininode-tester:0.0.1/" +MY_SUBVERSION = b"/python-mininode-tester:0.0.2/" MAX_INV_SZ = 50000 MAX_BLOCK_SIZE = 1000000 @@ -131,7 +131,7 @@ def deser_vector(f, c): def ser_vector(l): - r = "" + r = b"" if len(l) < 253: r = struct.pack("B", len(l)) elif len(l) < 0x10000: @@ -161,7 +161,7 @@ def deser_uint256_vector(f): def ser_uint256_vector(l): - r = "" + r = b"" if len(l) < 253: r = struct.pack("B", len(l)) elif len(l) < 0x10000: @@ -221,7 +221,7 @@ def deser_int_vector(f): def ser_int_vector(l): - r = "" + r = b"" if len(l) < 253: r = struct.pack("B", len(l)) elif len(l) < 0x10000: @@ -236,19 +236,19 @@ def ser_int_vector(l): # Deserialize from a hex string representation (eg from RPC) def FromHex(obj, hex_string): - obj.deserialize(BytesIO(binascii.unhexlify(hex_string))) + obj.deserialize(BytesIO(unhexlify(hex_string.encode('ascii')))) return obj # Convert a binary-serializable object to hex (eg for submission via RPC) def ToHex(obj): - return binascii.hexlify(obj.serialize()).decode('utf-8') + return hexlify(obj.serialize()).decode('ascii') # Objects that map to bitcoind objects, which can be serialized/deserialized class CAddress(object): def __init__(self): self.nServices = 1 - self.pchReserved = "\x00" * 10 + "\xff" * 2 + self.pchReserved = b"\x00" * 10 + b"\xff" * 2 self.ip = "0.0.0.0" self.port = 0 @@ -259,7 +259,7 @@ class CAddress(object): self.port = struct.unpack(">H", f.read(2))[0] def serialize(self): - r = "" + r = b"" r += struct.pack("<Q", self.nServices) r += self.pchReserved r += socket.inet_aton(self.ip) @@ -286,7 +286,7 @@ class CInv(object): self.hash = deser_uint256(f) def serialize(self): - r = "" + r = b"" r += struct.pack("<i", self.type) r += ser_uint256(self.hash) return r @@ -306,7 +306,7 @@ class CBlockLocator(object): self.vHave = deser_uint256_vector(f) def serialize(self): - r = "" + r = b"" r += struct.pack("<i", self.nVersion) r += ser_uint256_vector(self.vHave) return r @@ -326,7 +326,7 @@ class COutPoint(object): self.n = struct.unpack("<I", f.read(4))[0] def serialize(self): - r = "" + r = b"" r += ser_uint256(self.hash) r += struct.pack("<I", self.n) return r @@ -336,7 +336,7 @@ class COutPoint(object): class CTxIn(object): - def __init__(self, outpoint=None, scriptSig="", nSequence=0): + def __init__(self, outpoint=None, scriptSig=b"", nSequence=0): if outpoint is None: self.prevout = COutPoint() else: @@ -351,7 +351,7 @@ class CTxIn(object): self.nSequence = struct.unpack("<I", f.read(4))[0] def serialize(self): - r = "" + r = b"" r += self.prevout.serialize() r += ser_string(self.scriptSig) r += struct.pack("<I", self.nSequence) @@ -359,12 +359,12 @@ class CTxIn(object): def __repr__(self): return "CTxIn(prevout=%s scriptSig=%s nSequence=%i)" \ - % (repr(self.prevout), binascii.hexlify(self.scriptSig), + % (repr(self.prevout), hexlify(self.scriptSig), self.nSequence) class CTxOut(object): - def __init__(self, nValue=0, scriptPubKey=""): + def __init__(self, nValue=0, scriptPubKey=b""): self.nValue = nValue self.scriptPubKey = scriptPubKey @@ -373,7 +373,7 @@ class CTxOut(object): self.scriptPubKey = deser_string(f) def serialize(self): - r = "" + r = b"" r += struct.pack("<q", self.nValue) r += ser_string(self.scriptPubKey) return r @@ -381,7 +381,7 @@ class CTxOut(object): def __repr__(self): return "CTxOut(nValue=%i.%08i scriptPubKey=%s)" \ % (self.nValue // COIN, self.nValue % COIN, - binascii.hexlify(self.scriptPubKey)) + hexlify(self.scriptPubKey)) class CTransaction(object): @@ -410,7 +410,7 @@ class CTransaction(object): self.hash = None def serialize(self): - r = "" + r = b"" r += struct.pack("<i", self.nVersion) r += ser_vector(self.vin) r += ser_vector(self.vout) @@ -424,7 +424,7 @@ class CTransaction(object): def calc_sha256(self): if self.sha256 is None: self.sha256 = uint256_from_str(hash256(self.serialize())) - self.hash = encode(hash256(self.serialize())[::-1], 'hex') + self.hash = encode(hash256(self.serialize())[::-1], 'hex_codec').decode('ascii') def is_valid(self): self.calc_sha256() @@ -474,7 +474,7 @@ class CBlockHeader(object): self.hash = None def serialize(self): - r = "" + r = b"" r += struct.pack("<i", self.nVersion) r += ser_uint256(self.hashPrevBlock) r += ser_uint256(self.hashMerkleRoot) @@ -485,7 +485,7 @@ class CBlockHeader(object): def calc_sha256(self): if self.sha256 is None: - r = "" + r = b"" r += struct.pack("<i", self.nVersion) r += ser_uint256(self.hashPrevBlock) r += ser_uint256(self.hashMerkleRoot) @@ -493,7 +493,7 @@ class CBlockHeader(object): r += struct.pack("<I", self.nBits) r += struct.pack("<I", self.nNonce) self.sha256 = uint256_from_str(hash256(r)) - self.hash = encode(hash256(r)[::-1], 'hex') + self.hash = encode(hash256(r)[::-1], 'hex_codec').decode('ascii') def rehash(self): self.sha256 = None @@ -516,7 +516,7 @@ class CBlock(CBlockHeader): self.vtx = deser_vector(f, CTransaction) def serialize(self): - r = "" + r = b"" r += super(CBlock, self).serialize() r += ser_vector(self.vtx) return r @@ -571,9 +571,9 @@ class CUnsignedAlert(object): self.nMaxVer = 0 self.setSubVer = [] self.nPriority = 0 - self.strComment = "" - self.strStatusBar = "" - self.strReserved = "" + self.strComment = b"" + self.strStatusBar = b"" + self.strReserved = b"" def deserialize(self, f): self.nVersion = struct.unpack("<i", f.read(4))[0] @@ -591,7 +591,7 @@ class CUnsignedAlert(object): self.strReserved = deser_string(f) def serialize(self): - r = "" + r = b"" r += struct.pack("<i", self.nVersion) r += struct.pack("<q", self.nRelayUntil) r += struct.pack("<q", self.nExpiration) @@ -616,8 +616,8 @@ class CUnsignedAlert(object): class CAlert(object): def __init__(self): - self.vchMsg = "" - self.vchSig = "" + self.vchMsg = b"" + self.vchSig = b"" def deserialize(self, f): self.vchMsg = deser_string(f) @@ -636,7 +636,7 @@ class CAlert(object): # Objects that correspond to messages on the wire class msg_version(object): - command = "version" + command = b"version" def __init__(self): self.nVersion = MY_VERSION @@ -672,7 +672,7 @@ class msg_version(object): self.nStartingHeight = None def serialize(self): - r = "" + r = b"" r += struct.pack("<i", self.nVersion) r += struct.pack("<Q", self.nServices) r += struct.pack("<q", self.nTime) @@ -691,7 +691,7 @@ class msg_version(object): class msg_verack(object): - command = "verack" + command = b"verack" def __init__(self): pass @@ -700,14 +700,14 @@ class msg_verack(object): pass def serialize(self): - return "" + return b"" def __repr__(self): return "msg_verack()" class msg_addr(object): - command = "addr" + command = b"addr" def __init__(self): self.addrs = [] @@ -723,7 +723,7 @@ class msg_addr(object): class msg_alert(object): - command = "alert" + command = b"alert" def __init__(self): self.alert = CAlert() @@ -733,7 +733,7 @@ class msg_alert(object): self.alert.deserialize(f) def serialize(self): - r = "" + r = b"" r += self.alert.serialize() return r @@ -742,7 +742,7 @@ class msg_alert(object): class msg_inv(object): - command = "inv" + command = b"inv" def __init__(self, inv=None): if inv is None: @@ -761,7 +761,7 @@ class msg_inv(object): class msg_getdata(object): - command = "getdata" + command = b"getdata" def __init__(self, inv=None): self.inv = inv if inv != None else [] @@ -777,7 +777,7 @@ class msg_getdata(object): class msg_getblocks(object): - command = "getblocks" + command = b"getblocks" def __init__(self): self.locator = CBlockLocator() @@ -789,7 +789,7 @@ class msg_getblocks(object): self.hashstop = deser_uint256(f) def serialize(self): - r = "" + r = b"" r += self.locator.serialize() r += ser_uint256(self.hashstop) return r @@ -800,7 +800,7 @@ class msg_getblocks(object): class msg_tx(object): - command = "tx" + command = b"tx" def __init__(self, tx=CTransaction()): self.tx = tx @@ -816,7 +816,7 @@ class msg_tx(object): class msg_block(object): - command = "block" + command = b"block" def __init__(self, block=None): if block is None: @@ -835,7 +835,7 @@ class msg_block(object): class msg_getaddr(object): - command = "getaddr" + command = b"getaddr" def __init__(self): pass @@ -844,14 +844,14 @@ class msg_getaddr(object): pass def serialize(self): - return "" + return b"" def __repr__(self): return "msg_getaddr()" class msg_ping_prebip31(object): - command = "ping" + command = b"ping" def __init__(self): pass @@ -860,14 +860,14 @@ class msg_ping_prebip31(object): pass def serialize(self): - return "" + return b"" def __repr__(self): return "msg_ping() (pre-bip31)" class msg_ping(object): - command = "ping" + command = b"ping" def __init__(self, nonce=0L): self.nonce = nonce @@ -876,7 +876,7 @@ class msg_ping(object): self.nonce = struct.unpack("<Q", f.read(8))[0] def serialize(self): - r = "" + r = b"" r += struct.pack("<Q", self.nonce) return r @@ -885,16 +885,16 @@ class msg_ping(object): class msg_pong(object): - command = "pong" + command = b"pong" - def __init__(self, nonce=0L): + def __init__(self, nonce=0): self.nonce = nonce def deserialize(self, f): self.nonce = struct.unpack("<Q", f.read(8))[0] def serialize(self): - r = "" + r = b"" r += struct.pack("<Q", self.nonce) return r @@ -903,7 +903,7 @@ class msg_pong(object): class msg_mempool(object): - command = "mempool" + command = b"mempool" def __init__(self): pass @@ -912,13 +912,13 @@ class msg_mempool(object): pass def serialize(self): - return "" + return b"" def __repr__(self): return "msg_mempool()" class msg_sendheaders(object): - command = "sendheaders" + command = b"sendheaders" def __init__(self): pass @@ -927,7 +927,7 @@ class msg_sendheaders(object): pass def serialize(self): - return "" + return b"" def __repr__(self): return "msg_sendheaders()" @@ -937,7 +937,7 @@ class msg_sendheaders(object): # vector of hashes # hash_stop (hash of last desired block header, 0 to get as many as possible) class msg_getheaders(object): - command = "getheaders" + command = b"getheaders" def __init__(self): self.locator = CBlockLocator() @@ -949,7 +949,7 @@ class msg_getheaders(object): self.hashstop = deser_uint256(f) def serialize(self): - r = "" + r = b"" r += self.locator.serialize() r += ser_uint256(self.hashstop) return r @@ -962,7 +962,7 @@ class msg_getheaders(object): # headers message has # <count> <vector of block headers> class msg_headers(object): - command = "headers" + command = b"headers" def __init__(self): self.headers = [] @@ -982,10 +982,10 @@ class msg_headers(object): class msg_reject(object): - command = "reject" + command = b"reject" def __init__(self): - self.message = "" + self.message = b"" self.code = 0 self.reason = "" self.data = 0L @@ -1025,7 +1025,7 @@ def wait_until(predicate, attempts=float('inf'), timeout=float('inf')): return False class msg_feefilter(object): - command = "feefilter" + command = b"feefilter" def __init__(self, feerate=0L): self.feerate = feerate @@ -1034,7 +1034,7 @@ class msg_feefilter(object): self.feerate = struct.unpack("<Q", f.read(8))[0] def serialize(self): - r = "" + r = b"" r += struct.pack("<Q", self.feerate) return r @@ -1149,28 +1149,28 @@ class SingleNodeConnCB(NodeConnCB): # This class provides an interface for a p2p connection to a specified node class NodeConn(asyncore.dispatcher): messagemap = { - "version": msg_version, - "verack": msg_verack, - "addr": msg_addr, - "alert": msg_alert, - "inv": msg_inv, - "getdata": msg_getdata, - "getblocks": msg_getblocks, - "tx": msg_tx, - "block": msg_block, - "getaddr": msg_getaddr, - "ping": msg_ping, - "pong": msg_pong, - "headers": msg_headers, - "getheaders": msg_getheaders, - "reject": msg_reject, - "mempool": msg_mempool, - "feefilter": msg_feefilter + b"version": msg_version, + b"verack": msg_verack, + b"addr": msg_addr, + b"alert": msg_alert, + b"inv": msg_inv, + b"getdata": msg_getdata, + b"getblocks": msg_getblocks, + b"tx": msg_tx, + b"block": msg_block, + b"getaddr": msg_getaddr, + b"ping": msg_ping, + b"pong": msg_pong, + b"headers": msg_headers, + b"getheaders": msg_getheaders, + b"reject": msg_reject, + b"mempool": msg_mempool, + b"feefilter": msg_feefilter } MAGIC_BYTES = { - "mainnet": "\xf9\xbe\xb4\xd9", # mainnet - "testnet3": "\x0b\x11\x09\x07", # testnet3 - "regtest": "\xfa\xbf\xb5\xda" # regtest + "mainnet": b"\xf9\xbe\xb4\xd9", # mainnet + "testnet3": b"\x0b\x11\x09\x07", # testnet3 + "regtest": b"\xfa\xbf\xb5\xda" # regtest } def __init__(self, dstaddr, dstport, rpc, callback, net="regtest", services=1): @@ -1179,8 +1179,8 @@ class NodeConn(asyncore.dispatcher): self.dstaddr = dstaddr self.dstport = dstport self.create_socket(socket.AF_INET, socket.SOCK_STREAM) - self.sendbuf = "" - self.recvbuf = "" + self.sendbuf = b"" + self.recvbuf = b"" self.ver_send = 209 self.ver_recv = 209 self.last_sent = 0 @@ -1217,8 +1217,8 @@ class NodeConn(asyncore.dispatcher): self.show_debug_msg("MiniNode: Closing Connection to %s:%d... " % (self.dstaddr, self.dstport)) self.state = "closed" - self.recvbuf = "" - self.sendbuf = "" + self.recvbuf = b"" + self.sendbuf = b"" try: self.close() except: @@ -1261,7 +1261,7 @@ class NodeConn(asyncore.dispatcher): if self.ver_recv < 209: if len(self.recvbuf) < 4 + 12 + 4: return - command = self.recvbuf[4:4+12].split("\x00", 1)[0] + command = self.recvbuf[4:4+12].split(b"\x00", 1)[0] msglen = struct.unpack("<i", self.recvbuf[4+12:4+12+4])[0] checksum = None if len(self.recvbuf) < 4 + 12 + 4 + msglen: @@ -1271,7 +1271,7 @@ class NodeConn(asyncore.dispatcher): else: if len(self.recvbuf) < 4 + 12 + 4 + 4: return - command = self.recvbuf[4:4+12].split("\x00", 1)[0] + command = self.recvbuf[4:4+12].split(b"\x00", 1)[0] msglen = struct.unpack("<i", self.recvbuf[4+12:4+12+4])[0] checksum = self.recvbuf[4+12+4:4+12+4+4] if len(self.recvbuf) < 4 + 12 + 4 + 4 + msglen: @@ -1301,7 +1301,7 @@ class NodeConn(asyncore.dispatcher): data = message.serialize() tmsg = self.MAGIC_BYTES[self.network] tmsg += command - tmsg += "\x00" * (12 - len(command)) + tmsg += b"\x00" * (12 - len(command)) tmsg += struct.pack("<I", len(data)) if self.ver_send >= 209: th = sha256(data) @@ -1313,11 +1313,11 @@ class NodeConn(asyncore.dispatcher): self.last_sent = time.time() def got_message(self, message): - if message.command == "version": + if message.command == b"version": if message.nVersion <= BIP0031_VERSION: - self.messagemap['ping'] = msg_ping_prebip31 + self.messagemap[b'ping'] = msg_ping_prebip31 if self.last_sent + 30 * 60 < time.time(): - self.send_message(self.messagemap['ping']()) + self.send_message(self.messagemap[b'ping']()) self.show_debug_msg("Recv %s" % repr(message)) self.cb.deliver(self, message) diff --git a/qa/rpc-tests/test_framework/netutil.py b/qa/rpc-tests/test_framework/netutil.py index bbc58a14ec..52a7ab7489 100644 --- a/qa/rpc-tests/test_framework/netutil.py +++ b/qa/rpc-tests/test_framework/netutil.py @@ -4,13 +4,14 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. # Linux network utilities + import sys import socket import fcntl import struct import array import os -import binascii +from binascii import unhexlify, hexlify # Roughly based on http://voorloopnul.com/blog/a-python-netstat-in-less-than-100-lines-of-code/ by Ricardo Pascal STATE_ESTABLISHED = '01' @@ -43,7 +44,7 @@ def _remove_empty(array): def _convert_ip_port(array): host,port = array.split(':') # convert host from mangled-per-four-bytes form as used by kernel - host = binascii.unhexlify(host) + host = unhexlify(host) host_out = '' for x in range(0, len(host) // 4): (val,) = struct.unpack('=I', host[x*4:(x+1)*4]) @@ -94,7 +95,7 @@ def all_interfaces(): max_possible = 8 # initial value while True: bytes = max_possible * struct_size - names = array.array('B', '\0' * bytes) + names = array.array('B', b'\0' * bytes) outbytes = struct.unpack('iL', fcntl.ioctl( s.fileno(), 0x8912, # SIOCGIFCONF @@ -105,7 +106,7 @@ def all_interfaces(): else: break namestr = names.tostring() - return [(namestr[i:i+16].split('\0', 1)[0], + return [(namestr[i:i+16].split(b'\0', 1)[0], socket.inet_ntoa(namestr[i+20:i+24])) for i in range(0, outbytes, struct_size)] @@ -136,7 +137,7 @@ def addr_to_hex(addr): addr = sub[0] + ([0] * nullbytes) + sub[1] else: raise ValueError('Could not parse address %s' % addr) - return binascii.hexlify(bytearray(addr)) + return hexlify(bytearray(addr)).decode('ascii') def test_ipv6_local(): ''' diff --git a/qa/rpc-tests/test_framework/script.py b/qa/rpc-tests/test_framework/script.py index bf5e25fb27..5fb5758f81 100644 --- a/qa/rpc-tests/test_framework/script.py +++ b/qa/rpc-tests/test_framework/script.py @@ -629,7 +629,7 @@ class CScriptNum(object): neg = obj.value < 0 absvalue = -obj.value if neg else obj.value while (absvalue): - r.append(chr(absvalue & 0xff)) + r.append(absvalue & 0xff) absvalue >>= 8 if r[-1] & 0x80: r.append(0x80 if neg else 0) @@ -777,7 +777,7 @@ class CScript(bytes): # need to change def _repr(o): if isinstance(o, bytes): - return "x('%s')" % hexlify(o).decode('utf8') + return b"x('%s')" % hexlify(o).decode('ascii') else: return repr(o) diff --git a/qa/rpc-tests/test_framework/socks5.py b/qa/rpc-tests/test_framework/socks5.py index 12327a6c5f..f725d97701 100644 --- a/qa/rpc-tests/test_framework/socks5.py +++ b/qa/rpc-tests/test_framework/socks5.py @@ -102,7 +102,7 @@ class Socks5Connection(object): addr = recvall(self.conn, 4) elif atyp == AddressType.DOMAINNAME: n = recvall(self.conn, 1)[0] - addr = str(recvall(self.conn, n)) + addr = recvall(self.conn, n) elif atyp == AddressType.IPV6: addr = recvall(self.conn, 16) else: diff --git a/qa/rpc-tests/test_framework/util.py b/qa/rpc-tests/test_framework/util.py index acb7f8a6b4..8b720b54a1 100644 --- a/qa/rpc-tests/test_framework/util.py +++ b/qa/rpc-tests/test_framework/util.py @@ -1,6 +1,8 @@ # Copyright (c) 2014-2015 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. + + # # Helpful routines for regression testing # @@ -9,6 +11,8 @@ import os import sys +from binascii import hexlify, unhexlify +from base64 import b64encode from decimal import Decimal, ROUND_DOWN import json import random @@ -91,6 +95,15 @@ def check_json_precision(): def count_bytes(hex_string): return len(bytearray.fromhex(hex_string)) +def bytes_to_hex_str(byte_str): + return hexlify(byte_str).decode('ascii') + +def hex_str_to_bytes(hex_str): + return unhexlify(hex_str.encode('ascii')) + +def str_to_b64str(string): + return b64encode(string.encode('utf-8')).decode('ascii') + def sync_blocks(rpc_connections, wait=1): """ Wait until everybody has the same block count @@ -466,7 +479,7 @@ def assert_is_hash_string(string, length=64): "String %r contains invalid characters for a hash." % string) def satoshi_round(amount): - return Decimal(amount).quantize(Decimal('0.00000001'), rounding=ROUND_DOWN) + return Decimal(amount).quantize(Decimal('0.00000001'), rounding=ROUND_DOWN) # Helper to create at least "count" utxos # Pass in a fee that is sufficient for relay and mining new transactions. diff --git a/qa/rpc-tests/zmq_test.py b/qa/rpc-tests/zmq_test.py index 88532541ab..3a8d62ef2e 100755 --- a/qa/rpc-tests/zmq_test.py +++ b/qa/rpc-tests/zmq_test.py @@ -28,8 +28,8 @@ class ZMQTest (BitcoinTestFramework): def setup_nodes(self): self.zmqContext = zmq.Context() self.zmqSubSocket = self.zmqContext.socket(zmq.SUB) - self.zmqSubSocket.setsockopt(zmq.SUBSCRIBE, "hashblock") - self.zmqSubSocket.setsockopt(zmq.SUBSCRIBE, "hashtx") + self.zmqSubSocket.setsockopt(zmq.SUBSCRIBE, b"hashblock") + self.zmqSubSocket.setsockopt(zmq.SUBSCRIBE, b"hashtx") self.zmqSubSocket.connect("tcp://127.0.0.1:%i" % self.port) return start_nodes(4, self.options.tmpdir, extra_args=[ ['-zmqpubhashtx=tcp://127.0.0.1:'+str(self.port), '-zmqpubhashblock=tcp://127.0.0.1:'+str(self.port)], @@ -46,13 +46,13 @@ class ZMQTest (BitcoinTestFramework): print "listen..." msg = self.zmqSubSocket.recv_multipart() - topic = str(msg[0]) + topic = msg[0] body = msg[1] msg = self.zmqSubSocket.recv_multipart() - topic = str(msg[0]) + topic = msg[0] body = msg[1] - blkhash = binascii.hexlify(body) + blkhash = bytes_to_hex_str(body) assert_equal(genhashes[0], blkhash) #blockhash from generate must be equal to the hash received over zmq @@ -63,10 +63,10 @@ class ZMQTest (BitcoinTestFramework): zmqHashes = [] for x in range(0,n*2): msg = self.zmqSubSocket.recv_multipart() - topic = str(msg[0]) + topic = msg[0] body = msg[1] - if topic == "hashblock": - zmqHashes.append(binascii.hexlify(body)) + if topic == b"hashblock": + zmqHashes.append(bytes_to_hex_str(body)) for x in range(0,n): assert_equal(genhashes[x], zmqHashes[x]) #blockhash from generate must be equal to the hash received over zmq @@ -77,11 +77,11 @@ class ZMQTest (BitcoinTestFramework): # now we should receive a zmq msg because the tx was broadcast msg = self.zmqSubSocket.recv_multipart() - topic = str(msg[0]) + topic = msg[0] body = msg[1] hashZMQ = "" - if topic == "hashtx": - hashZMQ = binascii.hexlify(body) + if topic == b"hashtx": + hashZMQ = bytes_to_hex_str(body) assert_equal(hashRPC, hashZMQ) #blockhash from generate must be equal to the hash received over zmq diff --git a/src/base58.h b/src/base58.h index a3980118aa..cccebc9e0e 100644 --- a/src/base58.h +++ b/src/base58.h @@ -164,7 +164,7 @@ public: CBitcoinExtKeyBase() {} }; -typedef CBitcoinExtKeyBase<CExtKey, 74, CChainParams::EXT_SECRET_KEY> CBitcoinExtKey; -typedef CBitcoinExtKeyBase<CExtPubKey, 74, CChainParams::EXT_PUBLIC_KEY> CBitcoinExtPubKey; +typedef CBitcoinExtKeyBase<CExtKey, BIP32_EXTKEY_SIZE, CChainParams::EXT_SECRET_KEY> CBitcoinExtKey; +typedef CBitcoinExtKeyBase<CExtPubKey, BIP32_EXTKEY_SIZE, CChainParams::EXT_PUBLIC_KEY> CBitcoinExtPubKey; #endif // BITCOIN_BASE58_H diff --git a/src/key.cpp b/src/key.cpp index 28ba5144e4..6a3d9aa140 100644 --- a/src/key.cpp +++ b/src/key.cpp @@ -275,7 +275,7 @@ CExtPubKey CExtKey::Neuter() const { return ret; } -void CExtKey::Encode(unsigned char code[74]) const { +void CExtKey::Encode(unsigned char code[BIP32_EXTKEY_SIZE]) const { code[0] = nDepth; memcpy(code+1, vchFingerprint, 4); code[5] = (nChild >> 24) & 0xFF; code[6] = (nChild >> 16) & 0xFF; @@ -286,12 +286,12 @@ void CExtKey::Encode(unsigned char code[74]) const { memcpy(code+42, key.begin(), 32); } -void CExtKey::Decode(const unsigned char code[74]) { +void CExtKey::Decode(const unsigned char code[BIP32_EXTKEY_SIZE]) { nDepth = code[0]; memcpy(vchFingerprint, code+1, 4); nChild = (code[5] << 24) | (code[6] << 16) | (code[7] << 8) | code[8]; memcpy(chaincode.begin(), code+9, 32); - key.Set(code+42, code+74, true); + key.Set(code+42, code+BIP32_EXTKEY_SIZE, true); } bool ECC_InitSanityCheck() { @@ -164,11 +164,28 @@ struct CExtKey { a.chaincode == b.chaincode && a.key == b.key; } - void Encode(unsigned char code[74]) const; - void Decode(const unsigned char code[74]); + void Encode(unsigned char code[BIP32_EXTKEY_SIZE]) const; + void Decode(const unsigned char code[BIP32_EXTKEY_SIZE]); bool Derive(CExtKey& out, unsigned int nChild) const; CExtPubKey Neuter() const; void SetMaster(const unsigned char* seed, unsigned int nSeedLen); + template <typename Stream> + void Serialize(Stream& s, int nType, int nVersion) const + { + unsigned int len = BIP32_EXTKEY_SIZE; + ::WriteCompactSize(s, len); + unsigned char code[BIP32_EXTKEY_SIZE]; + Encode(code); + s.write((const char *)&code[0], len); + } + template <typename Stream> + void Unserialize(Stream& s, int nType, int nVersion) + { + unsigned int len = ::ReadCompactSize(s); + unsigned char code[BIP32_EXTKEY_SIZE]; + s.read((char *)&code[0], len); + Decode(code); + } }; /** Initialize the elliptic curve support. May not be called twice without calling ECC_Stop first. */ diff --git a/src/main.cpp b/src/main.cpp index f84fb8dd56..ed9ee16545 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -4344,10 +4344,12 @@ bool static AlreadyHave(const CInv& inv) EXCLUSIVE_LOCKS_REQUIRED(cs_main) recentRejects->reset(); } + // Use pcoinsTip->HaveCoinsInCache as a quick approximation to exclude + // requesting or processing some txs which have already been included in a block return recentRejects->contains(inv.hash) || mempool.exists(inv.hash) || mapOrphanTransactions.count(inv.hash) || - pcoinsTip->HaveCoins(inv.hash); + pcoinsTip->HaveCoinsInCache(inv.hash); } case MSG_BLOCK: return mapBlockIndex.count(inv.hash); diff --git a/src/policy/rbf.cpp b/src/policy/rbf.cpp index 98b1a1ba4c..133cff611d 100644 --- a/src/policy/rbf.cpp +++ b/src/policy/rbf.cpp @@ -14,33 +14,34 @@ bool SignalsOptInRBF(const CTransaction &tx) return false; } -bool IsRBFOptIn(const CTxMemPoolEntry &entry, CTxMemPool &pool) +RBFTransactionState IsRBFOptIn(const CTransaction &tx, CTxMemPool &pool) { AssertLockHeld(pool.cs); CTxMemPool::setEntries setAncestors; // First check the transaction itself. - if (SignalsOptInRBF(entry.GetTx())) { - return true; + if (SignalsOptInRBF(tx)) { + return RBF_TRANSACTIONSTATE_REPLACEABLE_BIP125; } // If this transaction is not in our mempool, then we can't be sure // we will know about all its inputs. - if (!pool.exists(entry.GetTx().GetHash())) { - throw std::runtime_error("Cannot determine RBF opt-in signal for non-mempool transaction\n"); + if (!pool.exists(tx.GetHash())) { + return RBF_TRANSACTIONSTATE_UNKNOWN; } // If all the inputs have nSequence >= maxint-1, it still might be // signaled for RBF if any unconfirmed parents have signaled. uint64_t noLimit = std::numeric_limits<uint64_t>::max(); std::string dummy; + CTxMemPoolEntry entry = *pool.mapTx.find(tx.GetHash()); pool.CalculateMemPoolAncestors(entry, setAncestors, noLimit, noLimit, noLimit, noLimit, dummy, false); BOOST_FOREACH(CTxMemPool::txiter it, setAncestors) { if (SignalsOptInRBF(it->GetTx())) { - return true; + return RBF_TRANSACTIONSTATE_REPLACEABLE_BIP125; } } - return false; + return RBF_TRANSACTIONSTATE_FINAL; } diff --git a/src/policy/rbf.h b/src/policy/rbf.h index 925ce0d9bd..5a711dba07 100644 --- a/src/policy/rbf.h +++ b/src/policy/rbf.h @@ -7,6 +7,12 @@ #include "txmempool.h" +enum RBFTransactionState { + RBF_TRANSACTIONSTATE_UNKNOWN, + RBF_TRANSACTIONSTATE_REPLACEABLE_BIP125, + RBF_TRANSACTIONSTATE_FINAL +}; + // Check whether the sequence numbers on this transaction are signaling // opt-in to replace-by-fee, according to BIP 125 bool SignalsOptInRBF(const CTransaction &tx); @@ -15,6 +21,6 @@ bool SignalsOptInRBF(const CTransaction &tx); // according to BIP 125 // This involves checking sequence numbers of the transaction, as well // as the sequence numbers of all in-mempool ancestors. -bool IsRBFOptIn(const CTxMemPoolEntry &entry, CTxMemPool &pool); +RBFTransactionState IsRBFOptIn(const CTransaction &tx, CTxMemPool &pool); #endif // BITCOIN_POLICY_RBF_H diff --git a/src/pubkey.cpp b/src/pubkey.cpp index db06a89285..be4ee27cd4 100644 --- a/src/pubkey.cpp +++ b/src/pubkey.cpp @@ -246,7 +246,7 @@ bool CPubKey::Derive(CPubKey& pubkeyChild, ChainCode &ccChild, unsigned int nChi return true; } -void CExtPubKey::Encode(unsigned char code[74]) const { +void CExtPubKey::Encode(unsigned char code[BIP32_EXTKEY_SIZE]) const { code[0] = nDepth; memcpy(code+1, vchFingerprint, 4); code[5] = (nChild >> 24) & 0xFF; code[6] = (nChild >> 16) & 0xFF; @@ -256,12 +256,12 @@ void CExtPubKey::Encode(unsigned char code[74]) const { memcpy(code+41, pubkey.begin(), 33); } -void CExtPubKey::Decode(const unsigned char code[74]) { +void CExtPubKey::Decode(const unsigned char code[BIP32_EXTKEY_SIZE]) { nDepth = code[0]; memcpy(vchFingerprint, code+1, 4); nChild = (code[5] << 24) | (code[6] << 16) | (code[7] << 8) | code[8]; memcpy(chaincode.begin(), code+9, 32); - pubkey.Set(code+41, code+74); + pubkey.Set(code+41, code+BIP32_EXTKEY_SIZE); } bool CExtPubKey::Derive(CExtPubKey &out, unsigned int nChild) const { diff --git a/src/pubkey.h b/src/pubkey.h index e1a17b6582..db5444ea9d 100644 --- a/src/pubkey.h +++ b/src/pubkey.h @@ -23,6 +23,8 @@ * script supports up to 75 for single byte push */ +const unsigned int BIP32_EXTKEY_SIZE = 74; + /** A reference to a CKey: the Hash160 of its serialized public key */ class CKeyID : public uint160 { @@ -205,9 +207,33 @@ struct CExtPubKey { a.chaincode == b.chaincode && a.pubkey == b.pubkey; } - void Encode(unsigned char code[74]) const; - void Decode(const unsigned char code[74]); + void Encode(unsigned char code[BIP32_EXTKEY_SIZE]) const; + void Decode(const unsigned char code[BIP32_EXTKEY_SIZE]); bool Derive(CExtPubKey& out, unsigned int nChild) const; + + unsigned int GetSerializeSize(int nType, int nVersion) const + { + return BIP32_EXTKEY_SIZE+1; //add one byte for the size (compact int) + } + template <typename Stream> + void Serialize(Stream& s, int nType, int nVersion) const + { + unsigned int len = BIP32_EXTKEY_SIZE; + ::WriteCompactSize(s, len); + unsigned char code[BIP32_EXTKEY_SIZE]; + Encode(code); + s.write((const char *)&code[0], len); + } + template <typename Stream> + void Unserialize(Stream& s, int nType, int nVersion) + { + unsigned int len = ::ReadCompactSize(s); + unsigned char code[BIP32_EXTKEY_SIZE]; + if (len != BIP32_EXTKEY_SIZE) + throw std::runtime_error("Invalid extended key size\n"); + s.read((char *)&code[0], len); + Decode(code); + } }; /** Users of this module must hold an ECCVerifyHandle. The constructor and diff --git a/src/rpc/net.cpp b/src/rpc/net.cpp index 017cd6ca32..ce14d034ce 100644 --- a/src/rpc/net.cpp +++ b/src/rpc/net.cpp @@ -97,9 +97,9 @@ UniValue getpeerinfo(const UniValue& params, bool fHelp) " \"bytesrecv\": n, (numeric) The total bytes received\n" " \"conntime\": ttt, (numeric) The connection time in seconds since epoch (Jan 1 1970 GMT)\n" " \"timeoffset\": ttt, (numeric) The time offset in seconds\n" - " \"pingtime\": n, (numeric) ping time\n" - " \"minping\": n, (numeric) minimum observed ping time\n" - " \"pingwait\": n, (numeric) ping wait\n" + " \"pingtime\": n, (numeric) ping time (if available)\n" + " \"minping\": n, (numeric) minimum observed ping time (if any at all)\n" + " \"pingwait\": n, (numeric) ping wait (if non-zero)\n" " \"version\": v, (numeric) The peer version, such as 7001\n" " \"subver\": \"/Satoshi:0.8.5/\", (string) The string version\n" " \"inbound\": true|false, (boolean) Inbound (true) or Outbound (false)\n" @@ -150,8 +150,10 @@ UniValue getpeerinfo(const UniValue& params, bool fHelp) obj.push_back(Pair("bytesrecv", stats.nRecvBytes)); obj.push_back(Pair("conntime", stats.nTimeConnected)); obj.push_back(Pair("timeoffset", stats.nTimeOffset)); - obj.push_back(Pair("pingtime", stats.dPingTime)); - obj.push_back(Pair("minping", stats.dPingMin)); + if (stats.dPingTime > 0.0) + obj.push_back(Pair("pingtime", stats.dPingTime)); + if (stats.dPingMin < std::numeric_limits<int64_t>::max()/1e6) + obj.push_back(Pair("minping", stats.dPingMin)); if (stats.dPingWait > 0.0) obj.push_back(Pair("pingwait", stats.dPingWait)); obj.push_back(Pair("version", stats.nVersion)); diff --git a/src/sync.cpp b/src/sync.cpp index 8df8ae43f4..641ed2c8ca 100644 --- a/src/sync.cpp +++ b/src/sync.cpp @@ -56,11 +56,24 @@ private: }; typedef std::vector<std::pair<void*, CLockLocation> > LockStack; +typedef std::map<std::pair<void*, void*>, LockStack> LockOrders; +typedef std::set<std::pair<void*, void*> > InvLockOrders; -static boost::mutex dd_mutex; -static std::map<std::pair<void*, void*>, LockStack> lockorders; -static boost::thread_specific_ptr<LockStack> lockstack; +struct LockData { + // Very ugly hack: as the global constructs and destructors run single + // threaded, we use this boolean to know whether LockData still exists, + // as DeleteLock can get called by global CCriticalSection destructors + // after LockData disappears. + bool available; + LockData() : available(true) {} + ~LockData() { available = false; } + LockOrders lockorders; + InvLockOrders invlockorders; + boost::mutex dd_mutex; +} static lockdata; + +boost::thread_specific_ptr<LockStack> lockstack; static void potential_deadlock_detected(const std::pair<void*, void*>& mismatch, const LockStack& s1, const LockStack& s2) { @@ -117,7 +130,7 @@ static void push_lock(void* c, const CLockLocation& locklocation, bool fTry) if (lockstack.get() == NULL) lockstack.reset(new LockStack); - dd_mutex.lock(); + boost::unique_lock<boost::mutex> lock(lockdata.dd_mutex); (*lockstack).push_back(std::make_pair(c, locklocation)); @@ -127,23 +140,21 @@ static void push_lock(void* c, const CLockLocation& locklocation, bool fTry) break; std::pair<void*, void*> p1 = std::make_pair(i.first, c); - if (lockorders.count(p1)) + if (lockdata.lockorders.count(p1)) continue; - lockorders[p1] = (*lockstack); + lockdata.lockorders[p1] = (*lockstack); std::pair<void*, void*> p2 = std::make_pair(c, i.first); - if (lockorders.count(p2)) - potential_deadlock_detected(p1, lockorders[p2], lockorders[p1]); + lockdata.invlockorders.insert(p2); + if (lockdata.lockorders.count(p2)) + potential_deadlock_detected(p1, lockdata.lockorders[p2], lockdata.lockorders[p1]); } } - dd_mutex.unlock(); } static void pop_lock() { - dd_mutex.lock(); (*lockstack).pop_back(); - dd_mutex.unlock(); } void EnterCritical(const char* pszName, const char* pszFile, int nLine, void* cs, bool fTry) @@ -173,4 +184,26 @@ void AssertLockHeldInternal(const char* pszName, const char* pszFile, int nLine, abort(); } +void DeleteLock(void* cs) +{ + if (!lockdata.available) { + // We're already shutting down. + return; + } + boost::unique_lock<boost::mutex> lock(lockdata.dd_mutex); + std::pair<void*, void*> item = std::make_pair(cs, (void*)0); + LockOrders::iterator it = lockdata.lockorders.lower_bound(item); + while (it != lockdata.lockorders.end() && it->first.first == cs) { + std::pair<void*, void*> invitem = std::make_pair(it->first.second, it->first.first); + lockdata.invlockorders.erase(invitem); + lockdata.lockorders.erase(it++); + } + InvLockOrders::iterator invit = lockdata.invlockorders.lower_bound(item); + while (invit != lockdata.invlockorders.end() && invit->first == cs) { + std::pair<void*, void*> invinvitem = std::make_pair(invit->second, invit->first); + lockdata.lockorders.erase(invinvitem); + lockdata.invlockorders.erase(invit++); + } +} + #endif /* DEBUG_LOCKORDER */ diff --git a/src/sync.h b/src/sync.h index 34dd8c228e..0c58fb6b4e 100644 --- a/src/sync.h +++ b/src/sync.h @@ -71,30 +71,39 @@ public: } }; -/** - * Wrapped boost mutex: supports recursive locking, but no waiting - * TODO: We should move away from using the recursive lock by default. - */ -typedef AnnotatedMixin<boost::recursive_mutex> CCriticalSection; - -/** Wrapped boost mutex: supports waiting but not recursive locking */ -typedef AnnotatedMixin<boost::mutex> CWaitableCriticalSection; - -/** Just a typedef for boost::condition_variable, can be wrapped later if desired */ -typedef boost::condition_variable CConditionVariable; - #ifdef DEBUG_LOCKORDER void EnterCritical(const char* pszName, const char* pszFile, int nLine, void* cs, bool fTry = false); void LeaveCritical(); std::string LocksHeld(); void AssertLockHeldInternal(const char* pszName, const char* pszFile, int nLine, void* cs); +void DeleteLock(void* cs); #else void static inline EnterCritical(const char* pszName, const char* pszFile, int nLine, void* cs, bool fTry = false) {} void static inline LeaveCritical() {} void static inline AssertLockHeldInternal(const char* pszName, const char* pszFile, int nLine, void* cs) {} +void static inline DeleteLock(void* cs) {} #endif #define AssertLockHeld(cs) AssertLockHeldInternal(#cs, __FILE__, __LINE__, &cs) +/** + * Wrapped boost mutex: supports recursive locking, but no waiting + * TODO: We should move away from using the recursive lock by default. + */ +class CCriticalSection : public AnnotatedMixin<boost::recursive_mutex> +{ +public: + ~CCriticalSection() { + DeleteLock((void*)this); + } +}; + +typedef CCriticalSection CDynamicCriticalSection; +/** Wrapped boost mutex: supports waiting but not recursive locking */ +typedef AnnotatedMixin<boost::mutex> CWaitableCriticalSection; + +/** Just a typedef for boost::condition_variable, can be wrapped later if desired */ +typedef boost::condition_variable CConditionVariable; + #ifdef DEBUG_LOCKCONTENTION void PrintLockContention(const char* pszName, const char* pszFile, int nLine); #endif diff --git a/src/test/bip32_tests.cpp b/src/test/bip32_tests.cpp index ce29e692db..7f1c2a32dd 100644 --- a/src/test/bip32_tests.cpp +++ b/src/test/bip32_tests.cpp @@ -117,6 +117,22 @@ void RunTest(const TestVector &test) { } key = keyNew; pubkey = pubkeyNew; + + CDataStream ssPub(SER_DISK, CLIENT_VERSION); + ssPub << pubkeyNew; + BOOST_CHECK(ssPub.size() == 75); + + CDataStream ssPriv(SER_DISK, CLIENT_VERSION); + ssPriv << keyNew; + BOOST_CHECK(ssPriv.size() == 75); + + CExtPubKey pubCheck; + CExtKey privCheck; + ssPub >> pubCheck; + ssPriv >> privCheck; + + BOOST_CHECK(pubCheck == pubkeyNew); + BOOST_CHECK(privCheck == keyNew); } } diff --git a/src/wallet/rpcwallet.cpp b/src/wallet/rpcwallet.cpp index 61c9846e11..5511e9d3ab 100644 --- a/src/wallet/rpcwallet.cpp +++ b/src/wallet/rpcwallet.cpp @@ -82,15 +82,11 @@ void WalletTxToJSON(const CWalletTx& wtx, UniValue& entry) std::string rbfStatus = "no"; if (confirms <= 0) { LOCK(mempool.cs); - if (!mempool.exists(hash)) { - if (SignalsOptInRBF(wtx)) { - rbfStatus = "yes"; - } else { - rbfStatus = "unknown"; - } - } else if (IsRBFOptIn(*mempool.mapTx.find(hash), mempool)) { + RBFTransactionState rbfState = IsRBFOptIn(wtx, mempool); + if (rbfState == RBF_TRANSACTIONSTATE_UNKNOWN) + rbfStatus = "unknown"; + else if (rbfState == RBF_TRANSACTIONSTATE_REPLACEABLE_BIP125) rbfStatus = "yes"; - } } entry.push_back(Pair("bip125-replaceable", rbfStatus)); |