diff options
44 files changed, 201 insertions, 146 deletions
diff --git a/configure.ac b/configure.ac index 97f44f269b..9ce10b7454 100644 --- a/configure.ac +++ b/configure.ac @@ -509,6 +509,9 @@ AC_SEARCH_LIBS([inet_pton], [nsl resolv], [AC_DEFINE(HAVE_INET_PTON, 1, [Define AC_CHECK_DECLS([strnlen]) +# Check for daemon(3), unrelated to --with-daemon (although used by it) +AC_CHECK_DECLS([daemon]) + AC_CHECK_DECLS([le16toh, le32toh, le64toh, htole16, htole32, htole64, be16toh, be32toh, be64toh, htobe16, htobe32, htobe64],,, [#if HAVE_ENDIAN_H #include <endian.h> diff --git a/doc/gitian-building.md b/doc/gitian-building.md index 4914d90df4..442d2d605f 100644 --- a/doc/gitian-building.md +++ b/doc/gitian-building.md @@ -336,6 +336,8 @@ There will be a lot of warnings printed during the build of the image. These can Getting and building the inputs -------------------------------- +At this point you have two options, you can either use the automated script (found in [contrib/gitian-build.sh](/contrib/gitian-build.sh)) or you could manually do everything by following this guide. If you're using the automated script, then run it with the "--setup" command. Afterwards, run it with the "--build" command (example: "contrib/gitian-building.sh -b signer 0.13.0"). Otherwise ignore this. + Follow the instructions in [doc/release-process.md](release-process.md#fetch-and-create-inputs-first-time-or-when-dependency-versions-change) in the bitcoin repository under 'Fetch and create inputs' to install sources which require manual intervention. Also optionally follow the next step: 'Seed the Gitian sources cache diff --git a/doc/release-process.md b/doc/release-process.md index 04d9473c20..d07f9a89ad 100644 --- a/doc/release-process.md +++ b/doc/release-process.md @@ -19,6 +19,8 @@ Before every major release: ### First time / New builders +If you're using the automated script (found in [contrib/gitian-build.sh](/contrib/gitian-build.sh)), then at this point you should run it with the "--setup" command. Otherwise ignore this. + Check out the source code in the following directory hierarchy. cd /path/to/your/toplevel/build @@ -62,6 +64,8 @@ Tag version (or release candidate) in git ### Setup and perform Gitian builds +If you're using the automated script (found in [contrib/gitian-build.sh](/contrib/gitian-build.sh)), then at this point you should run it with the "--build" command. Otherwise ignore this. + Setup Gitian descriptors: pushd ./bitcoin diff --git a/qa/rpc-tests/bip9-softforks.py b/qa/rpc-tests/bip9-softforks.py index 979d1410c2..be6ddde112 100755 --- a/qa/rpc-tests/bip9-softforks.py +++ b/qa/rpc-tests/bip9-softforks.py @@ -195,7 +195,6 @@ class BIP9SoftForksTest(ComparisonTestFramework): # Restart all self.test.block_store.close() stop_nodes(self.nodes) - wait_bitcoinds() shutil.rmtree(self.options.tmpdir) self.setup_chain() self.setup_network() diff --git a/qa/rpc-tests/forknotify.py b/qa/rpc-tests/forknotify.py index 5a3f75c808..a1901aedab 100755 --- a/qa/rpc-tests/forknotify.py +++ b/qa/rpc-tests/forknotify.py @@ -22,7 +22,7 @@ class ForkNotifyTest(BitcoinTestFramework): def setup_network(self): self.nodes = [] self.alert_filename = os.path.join(self.options.tmpdir, "alert.txt") - with open(self.alert_filename, 'w') as f: + with open(self.alert_filename, 'w', encoding='utf8') as f: pass # Just open then close to create zero-length file self.nodes.append(start_node(0, self.options.tmpdir, ["-blockversion=2", "-alertnotify=echo %s >> \"" + self.alert_filename + "\""])) @@ -44,7 +44,7 @@ class ForkNotifyTest(BitcoinTestFramework): self.nodes[1].generate(1) self.sync_all() - with open(self.alert_filename, 'r') as f: + with open(self.alert_filename, 'r', encoding='utf8') as f: alert_text = f.read() if len(alert_text) == 0: @@ -56,7 +56,7 @@ class ForkNotifyTest(BitcoinTestFramework): self.nodes[1].generate(1) self.sync_all() - with open(self.alert_filename, 'r') as f: + with open(self.alert_filename, 'r', encoding='utf8') as f: alert_text2 = f.read() if alert_text != alert_text2: diff --git a/qa/rpc-tests/fundrawtransaction.py b/qa/rpc-tests/fundrawtransaction.py index eeb8476634..8c45578fcf 100755 --- a/qa/rpc-tests/fundrawtransaction.py +++ b/qa/rpc-tests/fundrawtransaction.py @@ -470,7 +470,6 @@ class RawTransactionsTest(BitcoinTestFramework): self.nodes[1].encryptwallet("test") self.nodes.pop(1) stop_nodes(self.nodes) - wait_bitcoinds() self.nodes = start_nodes(self.num_nodes, self.options.tmpdir) # This test is not meant to test fee estimation and we'd like diff --git a/qa/rpc-tests/maxuploadtarget.py b/qa/rpc-tests/maxuploadtarget.py index 125d4eb275..d0e9fe9a3f 100755 --- a/qa/rpc-tests/maxuploadtarget.py +++ b/qa/rpc-tests/maxuploadtarget.py @@ -75,7 +75,7 @@ class TestNode(NodeConnCB): def received_pong(): return (self.last_pong.nonce == self.ping_counter) self.connection.send_message(msg_ping(nonce=self.ping_counter)) - success = wait_until(received_pong, timeout) + success = wait_until(received_pong, timeout=timeout) self.ping_counter += 1 return success diff --git a/qa/rpc-tests/multi_rpc.py b/qa/rpc-tests/multi_rpc.py index 24373b257d..cdeb94caa0 100755 --- a/qa/rpc-tests/multi_rpc.py +++ b/qa/rpc-tests/multi_rpc.py @@ -26,7 +26,7 @@ class HTTPBasicsTest (BitcoinTestFramework): #Append rpcauth to bitcoin.conf before initialization rpcauth = "rpcauth=rt:93648e835a54c573682c2eb19f882535$7681e9c5b74bdd85e78166031d2058e1069b3ed7ed967c93fc63abba06f31144" rpcauth2 = "rpcauth=rt2:f8607b1a88861fac29dfccf9b52ff9f$ff36a0c23c8c62b4846112e50fa888416e94c17bfd4c42f88fd8f55ec6a3137e" - with open(os.path.join(self.options.tmpdir+"/node0", "bitcoin.conf"), 'a') as f: + with open(os.path.join(self.options.tmpdir+"/node0", "bitcoin.conf"), 'a', encoding='utf8') as f: f.write(rpcauth+"\n") f.write(rpcauth2+"\n") diff --git a/qa/rpc-tests/nulldummy.py b/qa/rpc-tests/nulldummy.py index 6488a92364..54b7eac376 100755 --- a/qa/rpc-tests/nulldummy.py +++ b/qa/rpc-tests/nulldummy.py @@ -119,6 +119,8 @@ class NULLDUMMYTest(BitcoinTestFramework): node.sendrawtransaction(bytes_to_hex_str(tx.serialize_with_witness()), True) except JSONRPCException as exp: assert_equal(exp.error["message"], msg) + else: + assert_equal('', msg) return tx.hash diff --git a/qa/rpc-tests/p2p-compactblocks.py b/qa/rpc-tests/p2p-compactblocks.py index bf4fb43add..ac4655a841 100755 --- a/qa/rpc-tests/p2p-compactblocks.py +++ b/qa/rpc-tests/p2p-compactblocks.py @@ -237,6 +237,8 @@ class CompactBlocksTest(BitcoinTestFramework): for i in range(num_transactions): self.nodes[0].sendtoaddress(address, 0.1) + self.test_node.sync_with_ping() + # Now mine a block, and look at the resulting compact block. self.test_node.clear_block_announcement() block_hash = int(self.nodes[0].generate(1)[0], 16) diff --git a/qa/rpc-tests/p2p-mempool.py b/qa/rpc-tests/p2p-mempool.py index 5c5d778f42..382d7f1e82 100755 --- a/qa/rpc-tests/p2p-mempool.py +++ b/qa/rpc-tests/p2p-mempool.py @@ -63,7 +63,7 @@ class TestNode(NodeConnCB): def received_pong(): return (self.last_pong.nonce == self.ping_counter) self.connection.send_message(msg_ping(nonce=self.ping_counter)) - success = wait_until(received_pong, timeout) + success = wait_until(received_pong, timeout=timeout) self.ping_counter += 1 return success diff --git a/qa/rpc-tests/p2p-versionbits-warning.py b/qa/rpc-tests/p2p-versionbits-warning.py index 00dbbc02cf..fc3eddddee 100755 --- a/qa/rpc-tests/p2p-versionbits-warning.py +++ b/qa/rpc-tests/p2p-versionbits-warning.py @@ -72,7 +72,7 @@ class VersionBitsWarningTest(BitcoinTestFramework): def setup_network(self): self.alert_filename = os.path.join(self.options.tmpdir, "alert.txt") # Open and close to create zero-length file - with open(self.alert_filename, 'w') as _: + with open(self.alert_filename, 'w', encoding='utf8') as _: pass self.extra_args = [["-debug", "-logtimemicros=1", "-alertnotify=echo %s >> \"" + self.alert_filename + "\""]] self.nodes = start_nodes(self.num_nodes, self.options.tmpdir, self.extra_args) @@ -95,7 +95,7 @@ class VersionBitsWarningTest(BitcoinTestFramework): peer.sync_with_ping() def test_versionbits_in_alert_file(self): - with open(self.alert_filename, 'r') as f: + with open(self.alert_filename, 'r', encoding='utf8') as f: alert_text = f.read() assert(VB_PATTERN.match(alert_text)) @@ -144,10 +144,9 @@ class VersionBitsWarningTest(BitcoinTestFramework): # is cleared, and restart the node. This should move the versionbit state # to ACTIVE. self.nodes[0].generate(VB_PERIOD) - stop_node(self.nodes[0], 0) - wait_bitcoinds() + stop_nodes(self.nodes) # Empty out the alert file - with open(self.alert_filename, 'w') as _: + with open(self.alert_filename, 'w', encoding='utf8') as _: pass self.nodes = start_nodes(self.num_nodes, self.options.tmpdir, self.extra_args) @@ -156,8 +155,7 @@ class VersionBitsWarningTest(BitcoinTestFramework): assert(WARN_UNKNOWN_RULES_ACTIVE in self.nodes[0].getinfo()["errors"]) assert(WARN_UNKNOWN_RULES_ACTIVE in self.nodes[0].getmininginfo()["errors"]) assert(WARN_UNKNOWN_RULES_ACTIVE in self.nodes[0].getnetworkinfo()["warnings"]) - stop_node(self.nodes[0], 0) - wait_bitcoinds() + stop_nodes(self.nodes) self.test_versionbits_in_alert_file() # Test framework expects the node to still be running... diff --git a/qa/rpc-tests/reindex.py b/qa/rpc-tests/reindex.py index abbbb10336..25cf4c1679 100755 --- a/qa/rpc-tests/reindex.py +++ b/qa/rpc-tests/reindex.py @@ -7,7 +7,11 @@ # Test -reindex and -reindex-chainstate with CheckBlockIndex # from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import * +from test_framework.util import ( + start_nodes, + stop_nodes, + assert_equal, +) import time class ReindexTest(BitcoinTestFramework): @@ -18,16 +22,14 @@ class ReindexTest(BitcoinTestFramework): self.num_nodes = 1 def setup_network(self): - self.nodes = [] - self.is_network_split = False - self.nodes.append(start_node(0, self.options.tmpdir)) + self.nodes = start_nodes(self.num_nodes, self.options.tmpdir) def reindex(self, justchainstate=False): self.nodes[0].generate(3) blockcount = self.nodes[0].getblockcount() - stop_node(self.nodes[0], 0) - wait_bitcoinds() - self.nodes[0]=start_node(0, self.options.tmpdir, ["-debug", "-reindex-chainstate" if justchainstate else "-reindex", "-checkblockindex=1"]) + stop_nodes(self.nodes) + extra_args = [["-debug", "-reindex-chainstate" if justchainstate else "-reindex", "-checkblockindex=1"]] + self.nodes = start_nodes(self.num_nodes, self.options.tmpdir, extra_args) while self.nodes[0].getblockcount() < blockcount: time.sleep(0.1) assert_equal(self.nodes[0].getblockcount(), blockcount) diff --git a/qa/rpc-tests/rpcbind_test.py b/qa/rpc-tests/rpcbind_test.py index 085024e268..d78d0b884e 100755 --- a/qa/rpc-tests/rpcbind_test.py +++ b/qa/rpc-tests/rpcbind_test.py @@ -40,7 +40,6 @@ class RPCBindTest(BitcoinTestFramework): assert_equal(set(get_bind_addrs(pid)), set(expected)) finally: stop_nodes(self.nodes) - wait_bitcoinds() def run_allowip_test(self, allow_ips, rpchost, rpcport): ''' @@ -56,7 +55,6 @@ class RPCBindTest(BitcoinTestFramework): finally: node = None # make sure connection will be garbage collected and closed stop_nodes(self.nodes) - wait_bitcoinds() def run_test(self): # due to OS-specific network stats queries, this test works only on Linux diff --git a/qa/rpc-tests/test_framework/authproxy.py b/qa/rpc-tests/test_framework/authproxy.py index f5e0be20d1..fd7f32b5c6 100644 --- a/qa/rpc-tests/test_framework/authproxy.py +++ b/qa/rpc-tests/test_framework/authproxy.py @@ -130,8 +130,9 @@ class AuthServiceProxy(object): return self._get_response() else: raise - except BrokenPipeError: - # Python 3.5+ raises this instead of BadStatusLine when the connection was reset + except (BrokenPipeError,ConnectionResetError): + # Python 3.5+ raises BrokenPipeError instead of BadStatusLine when the connection was reset + # ConnectionResetError happens on FreeBSD with Python 3.4 self.__conn.close() self.__conn.request(method, path, postdata, headers) return self._get_response() diff --git a/qa/rpc-tests/test_framework/coverage.py b/qa/rpc-tests/test_framework/coverage.py index 23fce61014..13b33869f5 100644 --- a/qa/rpc-tests/test_framework/coverage.py +++ b/qa/rpc-tests/test_framework/coverage.py @@ -50,7 +50,7 @@ class AuthServiceProxyWrapper(object): rpc_method = self.auth_service_proxy_instance._service_name if self.coverage_logfile: - with open(self.coverage_logfile, 'a+') as f: + with open(self.coverage_logfile, 'a+', encoding='utf8') as f: f.write("%s\n" % rpc_method) return return_val @@ -100,7 +100,7 @@ def write_all_rpc_commands(dirname, node): if line and not line.startswith('='): commands.add("%s\n" % line.split()[0]) - with open(filename, 'w') as f: + with open(filename, 'w', encoding='utf8') as f: f.writelines(list(commands)) return True diff --git a/qa/rpc-tests/test_framework/mininode.py b/qa/rpc-tests/test_framework/mininode.py index caffab3535..88a3b7e0f7 100755 --- a/qa/rpc-tests/test_framework/mininode.py +++ b/qa/rpc-tests/test_framework/mininode.py @@ -1320,7 +1320,7 @@ class msg_reject(object): % (self.message, self.code, self.reason, self.data) # Helper function -def wait_until(predicate, attempts=float('inf'), timeout=float('inf')): +def wait_until(predicate, *, attempts=float('inf'), timeout=float('inf')): attempt = 0 elapsed = 0 @@ -1536,7 +1536,7 @@ class SingleNodeConnCB(NodeConnCB): def received_pong(): return (self.last_pong.nonce == self.ping_counter) self.send_message(msg_ping(nonce=self.ping_counter)) - success = wait_until(received_pong, timeout) + success = wait_until(received_pong, timeout=timeout) self.ping_counter += 1 return success diff --git a/qa/rpc-tests/test_framework/netutil.py b/qa/rpc-tests/test_framework/netutil.py index 573b06772d..b92a9f6e1c 100644 --- a/qa/rpc-tests/test_framework/netutil.py +++ b/qa/rpc-tests/test_framework/netutil.py @@ -58,7 +58,7 @@ def netstat(typ='tcp'): To get pid of all network process running on system, you must run this script as superuser ''' - with open('/proc/net/'+typ,'r') as f: + with open('/proc/net/'+typ,'r',encoding='utf8') as f: content = f.readlines() content.pop(0) result = [] diff --git a/qa/rpc-tests/test_framework/test_framework.py b/qa/rpc-tests/test_framework/test_framework.py index 186cf866cf..e6fc5fd8a2 100755 --- a/qa/rpc-tests/test_framework/test_framework.py +++ b/qa/rpc-tests/test_framework/test_framework.py @@ -21,7 +21,6 @@ from .util import ( sync_mempools, stop_nodes, stop_node, - wait_bitcoinds, enable_coverage, check_json_precision, initialize_chain_clean, @@ -81,7 +80,6 @@ class BitcoinTestFramework(object): """ assert not self.is_network_split stop_nodes(self.nodes) - wait_bitcoinds() self.setup_network(True) def sync_all(self): @@ -100,7 +98,6 @@ class BitcoinTestFramework(object): """ assert self.is_network_split stop_nodes(self.nodes) - wait_bitcoinds() self.setup_network(False) def main(self): @@ -170,7 +167,6 @@ class BitcoinTestFramework(object): if not self.options.noshutdown: print("Stopping nodes") stop_nodes(self.nodes) - wait_bitcoinds() else: print("Note: bitcoinds were not stopped and may still be running") diff --git a/qa/rpc-tests/test_framework/util.py b/qa/rpc-tests/test_framework/util.py index eee77f1a10..c6b0367b41 100644 --- a/qa/rpc-tests/test_framework/util.py +++ b/qa/rpc-tests/test_framework/util.py @@ -161,7 +161,7 @@ def initialize_datadir(dirname, n): if not os.path.isdir(datadir): os.makedirs(datadir) rpc_u, rpc_p = rpc_auth_pair(n) - with open(os.path.join(datadir, "bitcoin.conf"), 'w') as f: + with open(os.path.join(datadir, "bitcoin.conf"), 'w', encoding='utf8') as f: f.write("regtest=1\n") f.write("rpcuser=" + rpc_u + "\n") f.write("rpcpassword=" + rpc_p + "\n") @@ -266,7 +266,6 @@ def initialize_chain(test_dir, num_nodes, cachedir): # Shut them down, and clean up cache directories: stop_nodes(rpcs) - wait_bitcoinds() disable_mocktime() for i in range(MAX_NODES): os.remove(log_filename(cachedir, i, "debug.log")) @@ -365,6 +364,7 @@ def stop_nodes(nodes): except http.client.CannotSendRequest as e: print("WARN: Unable to stop node: " + repr(e)) del nodes[:] # Emptying array closes connections as a side effect + wait_bitcoinds() def set_node_times(nodes, t): for node in nodes: diff --git a/qa/rpc-tests/wallet-dump.py b/qa/rpc-tests/wallet-dump.py index 6028d2c20b..a37096a40c 100755 --- a/qa/rpc-tests/wallet-dump.py +++ b/qa/rpc-tests/wallet-dump.py @@ -12,7 +12,7 @@ def read_dump(file_name, addrs, hd_master_addr_old): Read the given dump, count the addrs that match, count change and reserve. Also check that the old hd_master is inactive """ - with open(file_name) as inputfile: + with open(file_name, encoding='utf8') as inputfile: found_addr = 0 found_addr_chg = 0 found_addr_rsv = 0 diff --git a/qa/rpc-tests/wallet.py b/qa/rpc-tests/wallet.py index 3420be1a2e..e43f6ea5d2 100755 --- a/qa/rpc-tests/wallet.py +++ b/qa/rpc-tests/wallet.py @@ -199,7 +199,6 @@ class WalletTest (BitcoinTestFramework): #do some -walletbroadcast tests stop_nodes(self.nodes) - wait_bitcoinds() self.nodes = start_nodes(3, self.options.tmpdir, [["-walletbroadcast=0"],["-walletbroadcast=0"],["-walletbroadcast=0"]]) connect_nodes_bi(self.nodes,0,1) connect_nodes_bi(self.nodes,1,2) @@ -225,7 +224,6 @@ class WalletTest (BitcoinTestFramework): #restart the nodes with -walletbroadcast=1 stop_nodes(self.nodes) - wait_bitcoinds() self.nodes = start_nodes(3, self.options.tmpdir) connect_nodes_bi(self.nodes,0,1) connect_nodes_bi(self.nodes,1,2) @@ -335,7 +333,6 @@ class WalletTest (BitcoinTestFramework): for m in maintenance: print("check " + m) stop_nodes(self.nodes) - wait_bitcoinds() self.nodes = start_nodes(3, self.options.tmpdir, [[m]] * 3) while m == '-reindex' and [block_count] * 3 != [self.nodes[i].getblockcount() for i in range(3)]: # reindex will leave rpc warm up "early"; Wait for it to finish diff --git a/src/bitcoin-cli.cpp b/src/bitcoin-cli.cpp index 68f5d90f51..9d4c4e53bd 100644 --- a/src/bitcoin-cli.cpp +++ b/src/bitcoin-cli.cpp @@ -92,7 +92,7 @@ static bool AppInitRPC(int argc, char* argv[]) return false; } try { - ReadConfigFile(mapArgs, mapMultiArgs); + ReadConfigFile(GetArg("-conf", BITCOIN_CONF_FILENAME), mapArgs, mapMultiArgs); } catch (const std::exception& e) { fprintf(stderr,"Error reading configuration file: %s\n", e.what()); return false; @@ -209,7 +209,7 @@ UniValue CallRPC(const string& strMethod, const UniValue& params) if (!GetAuthCookie(&strRPCUserColonPass)) { throw runtime_error(strprintf( _("Could not locate RPC credentials. No authentication cookie could be found, and no rpcpassword is set in the configuration file (%s)"), - GetConfigFile().string().c_str())); + GetConfigFile(GetArg("-conf", BITCOIN_CONF_FILENAME)).string().c_str())); } } else { diff --git a/src/bitcoind.cpp b/src/bitcoind.cpp index 322298d1b3..351463c256 100644 --- a/src/bitcoind.cpp +++ b/src/bitcoind.cpp @@ -9,6 +9,7 @@ #include "chainparams.h" #include "clientversion.h" +#include "compat.h" #include "rpc/server.h" #include "init.h" #include "noui.h" @@ -103,7 +104,7 @@ bool AppInit(int argc, char* argv[]) } try { - ReadConfigFile(mapArgs, mapMultiArgs); + ReadConfigFile(GetArg("-conf", BITCOIN_CONF_FILENAME), mapArgs, mapMultiArgs); } catch (const std::exception& e) { fprintf(stderr,"Error reading configuration file: %s\n", e.what()); return false; @@ -127,29 +128,21 @@ bool AppInit(int argc, char* argv[]) fprintf(stderr, "Error: There is no RPC client functionality in bitcoind anymore. Use the bitcoin-cli utility instead.\n"); exit(1); } -#ifndef WIN32 if (GetBoolArg("-daemon", false)) { +#if HAVE_DECL_DAEMON fprintf(stdout, "Bitcoin server starting\n"); // Daemonize - pid_t pid = fork(); - if (pid < 0) - { - fprintf(stderr, "Error: fork() returned %d errno %d\n", pid, errno); + if (daemon(1, 0)) { // don't chdir (1), do close FDs (0) + fprintf(stderr, "Error: daemon() failed: %s\n", strerror(errno)); return false; } - if (pid > 0) // Parent process, pid is child process id - { - return true; - } - // Child process falls through to rest of initialization - - pid_t sid = setsid(); - if (sid < 0) - fprintf(stderr, "Error: setsid() returned %d errno %d\n", sid, errno); +#else + fprintf(stderr, "Error: -daemon is not supported on this operating system\n"); + return false; +#endif // HAVE_DECL_DAEMON } -#endif SoftSetBoolArg("-server", true); // Set this early so that parameter interactions go to console diff --git a/src/init.cpp b/src/init.cpp index cb8da06d64..eefef7ba0b 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -324,7 +324,7 @@ std::string HelpMessage(HelpMessageMode mode) strUsage += HelpMessageOpt("-conf=<file>", strprintf(_("Specify configuration file (default: %s)"), BITCOIN_CONF_FILENAME)); if (mode == HMM_BITCOIND) { -#ifndef WIN32 +#if HAVE_DECL_DAEMON strUsage += HelpMessageOpt("-daemon", _("Run in the background as a daemon and accept commands")); #endif } @@ -1064,7 +1064,7 @@ bool AppInit2(boost::thread_group& threadGroup, CScheduler& scheduler) LogPrintf("Startup time: %s\n", DateTimeStrFormat("%Y-%m-%d %H:%M:%S", GetTime())); LogPrintf("Default data directory %s\n", GetDefaultDataDir().string()); LogPrintf("Using data directory %s\n", strDataDir); - LogPrintf("Using config file %s\n", GetConfigFile().string()); + LogPrintf("Using config file %s\n", GetConfigFile(GetArg("-conf", BITCOIN_CONF_FILENAME)).string()); LogPrintf("Using at most %i connections (%i file descriptors available)\n", nMaxConnections, nFD); LogPrintf("Using %u threads for script verification\n", nScriptCheckThreads); diff --git a/src/main.cpp b/src/main.cpp index c33b41ac4e..437e972382 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -6253,11 +6253,12 @@ bool ProcessMessages(CNode* pfrom, CConnman& connman) // Checksum CDataStream& vRecv = msg.vRecv; uint256 hash = Hash(vRecv.begin(), vRecv.begin() + nMessageSize); - unsigned int nChecksum = ReadLE32((unsigned char*)&hash); - if (nChecksum != hdr.nChecksum) + if (memcmp(hash.begin(), hdr.pchChecksum, CMessageHeader::CHECKSUM_SIZE) != 0) { - LogPrintf("%s(%s, %u bytes): CHECKSUM ERROR nChecksum=%08x hdr.nChecksum=%08x\n", __func__, - SanitizeString(strCommand), nMessageSize, nChecksum, hdr.nChecksum); + LogPrintf("%s(%s, %u bytes): CHECKSUM ERROR expected %s was %s\n", __func__, + SanitizeString(strCommand), nMessageSize, + HexStr(hash.begin(), hash.begin()+CMessageHeader::CHECKSUM_SIZE), + HexStr(hdr.pchChecksum, hdr.pchChecksum+CMessageHeader::CHECKSUM_SIZE)); continue; } @@ -6350,7 +6351,7 @@ bool SendMessages(CNode* pto, CConnman& connman) // Ping automatically sent as a latency probe & keepalive. pingSend = true; } - if (pingSend) { + if (pingSend && !pto->fDisconnect) { uint64_t nonce = 0; while (nonce == 0) { GetRandBytes((unsigned char*)&nonce, sizeof(nonce)); @@ -6431,7 +6432,7 @@ bool SendMessages(CNode* pto, CConnman& connman) if (pindexBestHeader == NULL) pindexBestHeader = chainActive.Tip(); bool fFetch = state.fPreferredDownload || (nPreferredDownload == 0 && !pto->fClient && !pto->fOneShot); // Download if this is a nice peer, or we have no nice peers and this one might do. - if (!state.fSyncStarted && !pto->fClient && !fImporting && !fReindex) { + if (!state.fSyncStarted && !pto->fClient && !pto->fDisconnect && !fImporting && !fReindex) { // Only actively request headers from a single peer, unless we're close to today. if ((nSyncStarted == 0 && fFetch) || pindexBestHeader->GetBlockTime() > GetAdjustedTime() - 24 * 60 * 60) { state.fSyncStarted = true; diff --git a/src/net.cpp b/src/net.cpp index cce06f2d64..770e2d2959 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -2661,10 +2661,8 @@ void CNode::EndMessage(const char* pszCommand) UNLOCK_FUNCTION(cs_vSend) // Set the checksum uint256 hash = Hash(ssSend.begin() + CMessageHeader::HEADER_SIZE, ssSend.end()); - unsigned int nChecksum = 0; - memcpy(&nChecksum, &hash, sizeof(nChecksum)); - assert(ssSend.size () >= CMessageHeader::CHECKSUM_OFFSET + sizeof(nChecksum)); - memcpy((char*)&ssSend[CMessageHeader::CHECKSUM_OFFSET], &nChecksum, sizeof(nChecksum)); + assert(ssSend.size () >= CMessageHeader::CHECKSUM_OFFSET + CMessageHeader::CHECKSUM_SIZE); + memcpy((char*)&ssSend[CMessageHeader::CHECKSUM_OFFSET], hash.begin(), CMessageHeader::CHECKSUM_SIZE); LogPrint("net", "(%d bytes) peer=%d\n", nSize, id); diff --git a/src/prevector.h b/src/prevector.h index a0e1e140b4..25bce522dc 100644 --- a/src/prevector.h +++ b/src/prevector.h @@ -475,6 +475,14 @@ public: return ((size_t)(sizeof(T))) * _union.capacity; } } + + value_type* data() { + return item_ptr(0); + } + + const value_type* data() const { + return item_ptr(0); + } }; #pragma pack(pop) diff --git a/src/protocol.cpp b/src/protocol.cpp index 247c6c2120..54ad62b1a2 100644 --- a/src/protocol.cpp +++ b/src/protocol.cpp @@ -79,7 +79,7 @@ CMessageHeader::CMessageHeader(const MessageStartChars& pchMessageStartIn) memcpy(pchMessageStart, pchMessageStartIn, MESSAGE_START_SIZE); memset(pchCommand, 0, sizeof(pchCommand)); nMessageSize = -1; - nChecksum = 0; + memset(pchChecksum, 0, CHECKSUM_SIZE); } CMessageHeader::CMessageHeader(const MessageStartChars& pchMessageStartIn, const char* pszCommand, unsigned int nMessageSizeIn) @@ -88,7 +88,7 @@ CMessageHeader::CMessageHeader(const MessageStartChars& pchMessageStartIn, const memset(pchCommand, 0, sizeof(pchCommand)); strncpy(pchCommand, pszCommand, COMMAND_SIZE); nMessageSize = nMessageSizeIn; - nChecksum = 0; + memset(pchChecksum, 0, CHECKSUM_SIZE); } std::string CMessageHeader::GetCommand() const diff --git a/src/protocol.h b/src/protocol.h index 9b474ec79c..1bc1c25b37 100644 --- a/src/protocol.h +++ b/src/protocol.h @@ -45,15 +45,15 @@ public: READWRITE(FLATDATA(pchMessageStart)); READWRITE(FLATDATA(pchCommand)); READWRITE(nMessageSize); - READWRITE(nChecksum); + READWRITE(FLATDATA(pchChecksum)); } // TODO: make private (improves encapsulation) public: enum { COMMAND_SIZE = 12, - MESSAGE_SIZE_SIZE = sizeof(int), - CHECKSUM_SIZE = sizeof(int), + MESSAGE_SIZE_SIZE = 4, + CHECKSUM_SIZE = 4, MESSAGE_SIZE_OFFSET = MESSAGE_START_SIZE + COMMAND_SIZE, CHECKSUM_OFFSET = MESSAGE_SIZE_OFFSET + MESSAGE_SIZE_SIZE, @@ -61,8 +61,8 @@ public: }; char pchMessageStart[MESSAGE_START_SIZE]; char pchCommand[COMMAND_SIZE]; - unsigned int nMessageSize; - unsigned int nChecksum; + uint32_t nMessageSize; + uint8_t pchChecksum[CHECKSUM_SIZE]; }; /** diff --git a/src/qt/bitcoin.cpp b/src/qt/bitcoin.cpp index 430e6dd0e8..9986af4957 100644 --- a/src/qt/bitcoin.cpp +++ b/src/qt/bitcoin.cpp @@ -590,7 +590,7 @@ int main(int argc, char *argv[]) return 1; } try { - ReadConfigFile(mapArgs, mapMultiArgs); + ReadConfigFile(GetArg("-conf", BITCOIN_CONF_FILENAME), mapArgs, mapMultiArgs); } catch (const std::exception& e) { QMessageBox::critical(0, QObject::tr(PACKAGE_NAME), QObject::tr("Error: Cannot parse configuration file: %1. Only use key=value syntax.").arg(e.what())); diff --git a/src/rpc/protocol.cpp b/src/rpc/protocol.cpp index f5275062a2..bb885bb5a6 100644 --- a/src/rpc/protocol.cpp +++ b/src/rpc/protocol.cpp @@ -77,9 +77,10 @@ boost::filesystem::path GetAuthCookieFile() bool GenerateAuthCookie(std::string *cookie_out) { - unsigned char rand_pwd[32]; - GetRandBytes(rand_pwd, 32); - std::string cookie = COOKIEAUTH_USER + ":" + EncodeBase64(&rand_pwd[0],32); + const size_t COOKIE_SIZE = 32; + unsigned char rand_pwd[COOKIE_SIZE]; + GetRandBytes(rand_pwd, COOKIE_SIZE); + std::string cookie = COOKIEAUTH_USER + ":" + HexStr(rand_pwd, rand_pwd+COOKIE_SIZE); /** the umask determines what permissions are used to create this file - * these are set to 077 in init.cpp unless overridden with -sysperms. diff --git a/src/serialize.h b/src/serialize.h index 04ab9aa2e7..1f51da82ff 100644 --- a/src/serialize.h +++ b/src/serialize.h @@ -44,33 +44,32 @@ inline T* NCONST_PTR(const T* val) return const_cast<T*>(val); } -/** - * Get begin pointer of vector (non-const version). - * @note These functions avoid the undefined case of indexing into an empty - * vector, as well as that of indexing after the end of the vector. +/** + * Important: Do not use the following functions in new code, but use v.data() + * and v.data() + v.size() respectively directly. They were once introduced to + * have a compatible, safe way to get the begin and end pointer of a vector. + * However with C++11 the language has built-in functionality for this and it's + * more readable to just use that. */ template <typename V> inline typename V::value_type* begin_ptr(V& v) { - return v.empty() ? NULL : &v[0]; + return v.data(); } -/** Get begin pointer of vector (const version) */ template <typename V> inline const typename V::value_type* begin_ptr(const V& v) { - return v.empty() ? NULL : &v[0]; + return v.data(); } -/** Get end pointer of vector (non-const version) */ template <typename V> inline typename V::value_type* end_ptr(V& v) { - return v.empty() ? NULL : (&v[0] + v.size()); + return v.data() + v.size(); } -/** Get end pointer of vector (const version) */ template <typename V> inline const typename V::value_type* end_ptr(const V& v) { - return v.empty() ? NULL : (&v[0] + v.size()); + return v.data() + v.size(); } /* diff --git a/src/test/README.md b/src/test/README.md index 61462642bf..3afdefe5fc 100644 --- a/src/test/README.md +++ b/src/test/README.md @@ -30,3 +30,11 @@ example, to run just the getarg_tests verbosely: Run `test_bitcoin --help` for the full list. +### bitcoin-util-test.py + +The test directory also contains the bitcoin-util-test.py tool, which tests bitcoin utils (currently just bitcoin-tx). This test gets run automatically during the `make check` build process. It is also possible to run the test manually from the src directory: + +``` +test/bitcoin-util-test.py --srcdir=[current directory] + +```
\ No newline at end of file diff --git a/src/test/bctest.py b/src/test/bctest.py index 8105b87ffa..39a27fcd0c 100644 --- a/src/test/bctest.py +++ b/src/test/bctest.py @@ -24,6 +24,9 @@ def bctest(testDir, testObj, exeext): if "output_cmp" in testObj: outputFn = testObj['output_cmp'] outputData = open(testDir + "/" + outputFn).read() + if not outputData: + print("Output data missing for " + outputFn) + sys.exit(1) proc = subprocess.Popen(execrun, stdin=stdinCfg, stdout=subprocess.PIPE, stderr=subprocess.PIPE,universal_newlines=True) try: outs = proc.communicate(input=inputData) diff --git a/src/test/bitcoin-util-test.py b/src/test/bitcoin-util-test.py index 882b5c67b8..225b3324e9 100755 --- a/src/test/bitcoin-util-test.py +++ b/src/test/bitcoin-util-test.py @@ -6,8 +6,24 @@ from __future__ import division,print_function,unicode_literals import os import bctest import buildenv +import argparse -if __name__ == '__main__': - bctest.bctester(os.environ["srcdir"] + "/test/data", - "bitcoin-util-test.json",buildenv) +help_text="""Test framework for bitcoin utils. + +Runs automatically during `make check`. + +Can also be run manually from the src directory by specifiying the source directory: +test/bitcoin-util-test.py --src=[srcdir] +""" + + +if __name__ == '__main__': + try: + srcdir = os.environ["srcdir"] + except: + parser = argparse.ArgumentParser(description=help_text) + parser.add_argument('-s', '--srcdir') + args = parser.parse_args() + srcdir = args.srcdir + bctest.bctester(srcdir + "/test/data", "bitcoin-util-test.json", buildenv) diff --git a/src/test/data/txcreate2.json b/src/test/data/txcreate2.json index e69de29bb2..c56293eaf2 100644 --- a/src/test/data/txcreate2.json +++ b/src/test/data/txcreate2.json @@ -0,0 +1,19 @@ +{ + "txid": "cf90229625e9eb10f6be8156bf6aa5ec2eca19a42b1e05c11f3029b560a32e13", + "version": 1, + "locktime": 0, + "vin": [ + ], + "vout": [ + { + "value": 0.00, + "n": 0, + "scriptPubKey": { + "asm": "", + "hex": "", + "type": "nonstandard" + } + } + ], + "hex": "01000000000100000000000000000000000000" +} diff --git a/src/univalue/.travis.yml b/src/univalue/.travis.yml index d318d9cc8f..132743d349 100644 --- a/src/univalue/.travis.yml +++ b/src/univalue/.travis.yml @@ -1,4 +1,3 @@ - language: cpp compiler: @@ -26,6 +25,7 @@ addons: - pkg-config before_script: + - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew uninstall libtool; brew install libtool; fi - if [ -n "$USE_SHELL" ]; then export CONFIG_SHELL="$USE_SHELL"; fi - test -n "$USE_SHELL" && eval '"$USE_SHELL" -c "./autogen.sh"' || ./autogen.sh diff --git a/src/univalue/include/univalue.h b/src/univalue/include/univalue.h index e48b905bfb..e8ce283519 100644 --- a/src/univalue/include/univalue.h +++ b/src/univalue/include/univalue.h @@ -142,10 +142,10 @@ private: public: // Strict type-specific getters, these throw std::runtime_error if the // value is of unexpected type - std::vector<std::string> getKeys() const; - std::vector<UniValue> getValues() const; + const std::vector<std::string>& getKeys() const; + const std::vector<UniValue>& getValues() const; bool get_bool() const; - std::string get_str() const; + const std::string& get_str() const; int get_int() const; int64_t get_int64() const; double get_real() const; diff --git a/src/univalue/lib/univalue.cpp b/src/univalue/lib/univalue.cpp index 1f8cee6d29..5a2860c13f 100644 --- a/src/univalue/lib/univalue.cpp +++ b/src/univalue/lib/univalue.cpp @@ -283,14 +283,14 @@ const UniValue& find_value(const UniValue& obj, const std::string& name) return NullUniValue; } -std::vector<std::string> UniValue::getKeys() const +const std::vector<std::string>& UniValue::getKeys() const { if (typ != VOBJ) throw std::runtime_error("JSON value is not an object as expected"); return keys; } -std::vector<UniValue> UniValue::getValues() const +const std::vector<UniValue>& UniValue::getValues() const { if (typ != VOBJ && typ != VARR) throw std::runtime_error("JSON value is not an object or array as expected"); @@ -304,7 +304,7 @@ bool UniValue::get_bool() const return getBool(); } -std::string UniValue::get_str() const +const std::string& UniValue::get_str() const { if (typ != VSTR) throw std::runtime_error("JSON value is not a string as expected"); diff --git a/src/util.cpp b/src/util.cpp index 93cc0412b5..c20ede6221 100644 --- a/src/util.cpp +++ b/src/util.cpp @@ -518,19 +518,20 @@ void ClearDatadirCache() pathCachedNetSpecific = boost::filesystem::path(); } -boost::filesystem::path GetConfigFile() +boost::filesystem::path GetConfigFile(const std::string& confPath) { - boost::filesystem::path pathConfigFile(GetArg("-conf", BITCOIN_CONF_FILENAME)); + boost::filesystem::path pathConfigFile(confPath); if (!pathConfigFile.is_complete()) pathConfigFile = GetDataDir(false) / pathConfigFile; return pathConfigFile; } -void ReadConfigFile(map<string, string>& mapSettingsRet, +void ReadConfigFile(const std::string& confPath, + map<string, string>& mapSettingsRet, map<string, vector<string> >& mapMultiSettingsRet) { - boost::filesystem::ifstream streamConfig(GetConfigFile()); + boost::filesystem::ifstream streamConfig(GetConfigFile(confPath)); if (!streamConfig.good()) return; // No bitcoin.conf file is OK diff --git a/src/util.h b/src/util.h index 45b3658557..bbb9b5db82 100644 --- a/src/util.h +++ b/src/util.h @@ -102,12 +102,12 @@ bool TryCreateDirectory(const boost::filesystem::path& p); boost::filesystem::path GetDefaultDataDir(); const boost::filesystem::path &GetDataDir(bool fNetSpecific = true); void ClearDatadirCache(); -boost::filesystem::path GetConfigFile(); +boost::filesystem::path GetConfigFile(const std::string& confPath); #ifndef WIN32 boost::filesystem::path GetPidFile(); void CreatePidFile(const boost::filesystem::path &path, pid_t pid); #endif -void ReadConfigFile(std::map<std::string, std::string>& mapSettingsRet, std::map<std::string, std::vector<std::string> >& mapMultiSettingsRet); +void ReadConfigFile(const std::string& confPath, std::map<std::string, std::string>& mapSettingsRet, std::map<std::string, std::vector<std::string> >& mapMultiSettingsRet); #ifdef WIN32 boost::filesystem::path GetSpecialFolderPath(int nFolder, bool fCreate = true); #endif diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index 43e22383f4..a1f69dd94d 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -100,43 +100,7 @@ CPubKey CWallet::GenerateNewKey() // use HD key derivation if HD was enabled during wallet creation if (IsHDEnabled()) { - // for now we use a fixed keypath scheme of m/0'/0'/k - CKey key; //master key seed (256bit) - CExtKey masterKey; //hd master key - CExtKey accountKey; //key at m/0' - CExtKey externalChainChildKey; //key at m/0'/0' - CExtKey childKey; //key at m/0'/0'/<n>' - - // try to get the master key - if (!GetKey(hdChain.masterKeyID, key)) - throw std::runtime_error(std::string(__func__) + ": Master key not found"); - - masterKey.SetMaster(key.begin(), key.size()); - - // derive m/0' - // use hardened derivation (child keys >= 0x80000000 are hardened after bip32) - masterKey.Derive(accountKey, BIP32_HARDENED_KEY_LIMIT); - - // derive m/0'/0' - accountKey.Derive(externalChainChildKey, BIP32_HARDENED_KEY_LIMIT); - - // derive child key at next index, skip keys already known to the wallet - do - { - // always derive hardened keys - // childIndex | BIP32_HARDENED_KEY_LIMIT = derive childIndex in hardened child-index-range - // example: 1 | BIP32_HARDENED_KEY_LIMIT == 0x80000001 == 2147483649 - externalChainChildKey.Derive(childKey, hdChain.nExternalChainCounter | BIP32_HARDENED_KEY_LIMIT); - metadata.hdKeypath = "m/0'/0'/"+std::to_string(hdChain.nExternalChainCounter)+"'"; - metadata.hdMasterKeyID = hdChain.masterKeyID; - // increment childkey index - hdChain.nExternalChainCounter++; - } while(HaveKey(childKey.key.GetPubKey().GetID())); - secret = childKey.key; - - // update the chain model in the database - if (!CWalletDB(strWalletFile).WriteHDChain(hdChain)) - throw std::runtime_error(std::string(__func__) + ": Writing HD chain model failed"); + DeriveNewChildKey(metadata, secret); } else { secret.MakeNewKey(fCompressed); } @@ -157,6 +121,46 @@ CPubKey CWallet::GenerateNewKey() return pubkey; } +void CWallet::DeriveNewChildKey(CKeyMetadata& metadata, CKey& secret) +{ + // for now we use a fixed keypath scheme of m/0'/0'/k + CKey key; //master key seed (256bit) + CExtKey masterKey; //hd master key + CExtKey accountKey; //key at m/0' + CExtKey externalChainChildKey; //key at m/0'/0' + CExtKey childKey; //key at m/0'/0'/<n>' + + // try to get the master key + if (!GetKey(hdChain.masterKeyID, key)) + throw std::runtime_error(std::string(__func__) + ": Master key not found"); + + masterKey.SetMaster(key.begin(), key.size()); + + // derive m/0' + // use hardened derivation (child keys >= 0x80000000 are hardened after bip32) + masterKey.Derive(accountKey, BIP32_HARDENED_KEY_LIMIT); + + // derive m/0'/0' + accountKey.Derive(externalChainChildKey, BIP32_HARDENED_KEY_LIMIT); + + // derive child key at next index, skip keys already known to the wallet + do { + // always derive hardened keys + // childIndex | BIP32_HARDENED_KEY_LIMIT = derive childIndex in hardened child-index-range + // example: 1 | BIP32_HARDENED_KEY_LIMIT == 0x80000001 == 2147483649 + externalChainChildKey.Derive(childKey, hdChain.nExternalChainCounter | BIP32_HARDENED_KEY_LIMIT); + metadata.hdKeypath = "m/0'/0'/" + std::to_string(hdChain.nExternalChainCounter) + "'"; + metadata.hdMasterKeyID = hdChain.masterKeyID; + // increment childkey index + hdChain.nExternalChainCounter++; + } while (HaveKey(childKey.key.GetPubKey().GetID())); + secret = childKey.key; + + // update the chain model in the database + if (!CWalletDB(strWalletFile).WriteHDChain(hdChain)) + throw std::runtime_error(std::string(__func__) + ": Writing HD chain model failed"); +} + bool CWallet::AddKeyPubKey(const CKey& secret, const CPubKey &pubkey) { AssertLockHeld(cs_wallet); // mapKeyMetadata diff --git a/src/wallet/wallet.h b/src/wallet/wallet.h index c700e0ca7c..077edcab2d 100644 --- a/src/wallet/wallet.h +++ b/src/wallet/wallet.h @@ -698,6 +698,7 @@ public: * Generate a new key */ CPubKey GenerateNewKey(); + void DeriveNewChildKey(CKeyMetadata& metadata, CKey& secret); //! Adds a key to the store, and saves it to disk. bool AddKeyPubKey(const CKey& key, const CPubKey &pubkey); //! Adds a key to the store, without saving it to disk (used by LoadWallet) |