From 54729f3f4e6765dfded590af5fb28c88331685f8 Mon Sep 17 00:00:00 2001 From: Andrew Chow Date: Tue, 26 May 2020 20:52:59 -0400 Subject: Add libsqlite3 --- configure.ac | 4 ++++ src/Makefile.am | 2 +- src/Makefile.bench.include | 2 +- src/Makefile.qt.include | 2 +- src/Makefile.qttest.include | 2 +- src/Makefile.test.include | 2 +- 6 files changed, 9 insertions(+), 5 deletions(-) diff --git a/configure.ac b/configure.ac index 76c6487f08..6db9dab404 100644 --- a/configure.ac +++ b/configure.ac @@ -1222,6 +1222,9 @@ if test x$enable_wallet != xno; then if test x$suppress_external_warnings != xno ; then BDB_CPPFLAGS=SUPPRESS_WARNINGS($BDB_CPPFLAGS) fi + + dnl Check for sqlite3 + PKG_CHECK_MODULES([SQLITE], [sqlite3 >= 3.7.17], , [AC_MSG_ERROR([sqlite3 not found.])]) fi dnl Check for libminiupnpc (optional) @@ -1643,6 +1646,7 @@ AC_SUBST(LIBTOOL_APP_LDFLAGS) AC_SUBST(USE_UPNP) AC_SUBST(USE_QRCODE) AC_SUBST(BOOST_LIBS) +AC_SUBST(SQLITE_LIBS) AC_SUBST(TESTDEFS) AC_SUBST(MINIUPNPC_CPPFLAGS) AC_SUBST(MINIUPNPC_LIBS) diff --git a/src/Makefile.am b/src/Makefile.am index aa63b5f516..398d904fe6 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -589,7 +589,7 @@ bitcoin_bin_ldadd = \ $(LIBMEMENV) \ $(LIBSECP256K1) -bitcoin_bin_ldadd += $(BOOST_LIBS) $(BDB_LIBS) $(MINIUPNPC_LIBS) $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) $(ZMQ_LIBS) +bitcoin_bin_ldadd += $(BOOST_LIBS) $(BDB_LIBS) $(MINIUPNPC_LIBS) $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) $(ZMQ_LIBS) $(SQLITE_LIBS) bitcoind_SOURCES = $(bitcoin_daemon_sources) bitcoind_CPPFLAGS = $(bitcoin_bin_cppflags) diff --git a/src/Makefile.bench.include b/src/Makefile.bench.include index bd9143a381..beb3f8dfd2 100644 --- a/src/Makefile.bench.include +++ b/src/Makefile.bench.include @@ -74,7 +74,7 @@ bench_bench_bitcoin_SOURCES += bench/coin_selection.cpp bench_bench_bitcoin_SOURCES += bench/wallet_balance.cpp endif -bench_bench_bitcoin_LDADD += $(BOOST_LIBS) $(BDB_LIBS) $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) $(MINIUPNPC_LIBS) +bench_bench_bitcoin_LDADD += $(BOOST_LIBS) $(BDB_LIBS) $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) $(MINIUPNPC_LIBS) $(SQLITE_LIBS) bench_bench_bitcoin_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) $(PTHREAD_FLAGS) CLEAN_BITCOIN_BENCH = bench/*.gcda bench/*.gcno $(GENERATED_BENCH_FILES) diff --git a/src/Makefile.qt.include b/src/Makefile.qt.include index a6236ef19b..f46310a603 100644 --- a/src/Makefile.qt.include +++ b/src/Makefile.qt.include @@ -321,7 +321,7 @@ bitcoin_qt_ldadd += $(LIBBITCOIN_ZMQ) $(ZMQ_LIBS) endif bitcoin_qt_ldadd += $(LIBBITCOIN_CLI) $(LIBBITCOIN_COMMON) $(LIBBITCOIN_UTIL) $(LIBBITCOIN_CONSENSUS) $(LIBBITCOIN_CRYPTO) $(LIBUNIVALUE) $(LIBLEVELDB) $(LIBLEVELDB_SSE42) $(LIBMEMENV) \ $(BOOST_LIBS) $(QT_LIBS) $(QT_DBUS_LIBS) $(QR_LIBS) $(BDB_LIBS) $(MINIUPNPC_LIBS) $(LIBSECP256K1) \ - $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) + $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) $(SQLITE_LIBS) bitcoin_qt_ldflags = $(RELDFLAGS) $(AM_LDFLAGS) $(QT_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) $(PTHREAD_FLAGS) bitcoin_qt_libtoolflags = $(AM_LIBTOOLFLAGS) --tag CXX diff --git a/src/Makefile.qttest.include b/src/Makefile.qttest.include index d300398b25..c05dd38737 100644 --- a/src/Makefile.qttest.include +++ b/src/Makefile.qttest.include @@ -56,7 +56,7 @@ endif qt_test_test_bitcoin_qt_LDADD += $(LIBBITCOIN_CLI) $(LIBBITCOIN_COMMON) $(LIBBITCOIN_UTIL) $(LIBBITCOIN_CONSENSUS) $(LIBBITCOIN_CRYPTO) $(LIBUNIVALUE) $(LIBLEVELDB) \ $(LIBLEVELDB_SSE42) $(LIBMEMENV) $(BOOST_LIBS) $(QT_DBUS_LIBS) $(QT_TEST_LIBS) $(QT_LIBS) \ $(QR_LIBS) $(BDB_LIBS) $(MINIUPNPC_LIBS) $(LIBSECP256K1) \ - $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) + $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) $(SQLITE_LIBS) qt_test_test_bitcoin_qt_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(QT_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) $(PTHREAD_FLAGS) qt_test_test_bitcoin_qt_CXXFLAGS = $(AM_CXXFLAGS) $(QT_PIE_FLAGS) diff --git a/src/Makefile.test.include b/src/Makefile.test.include index 06dde87ddd..c6a4a4edc4 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -315,7 +315,7 @@ test_test_bitcoin_LDADD += $(LIBBITCOIN_SERVER) $(LIBBITCOIN_CLI) $(LIBBITCOIN_C $(LIBLEVELDB) $(LIBLEVELDB_SSE42) $(LIBMEMENV) $(BOOST_LIBS) $(BOOST_UNIT_TEST_FRAMEWORK_LIB) $(LIBSECP256K1) $(EVENT_LIBS) $(EVENT_PTHREADS_LIBS) test_test_bitcoin_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_test_bitcoin_LDADD += $(BDB_LIBS) $(MINIUPNPC_LIBS) +test_test_bitcoin_LDADD += $(BDB_LIBS) $(MINIUPNPC_LIBS) $(SQLITE_LIBS) test_test_bitcoin_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) $(PTHREAD_FLAGS) -static if ENABLE_ZMQ -- cgit v1.2.3 From e87df8258090138d5c22ac46b8602b618620e8a1 Mon Sep 17 00:00:00 2001 From: Andrew Chow Date: Tue, 26 May 2020 23:18:08 -0400 Subject: Add sqlite to travis and depends --- .travis.yml | 1 + build_msvc/vcpkg.json | 3 ++- ci/test/00_setup_env_native_asan.sh | 2 +- ci/test/00_setup_env_native_msan.sh | 2 +- ci/test/00_setup_env_native_valgrind.sh | 2 +- contrib/gitian-descriptors/gitian-linux.yml | 2 +- contrib/gitian-descriptors/gitian-win.yml | 2 +- depends/Makefile | 5 ++++- depends/README.md | 4 ++++ depends/packages/packages.mk | 3 ++- depends/packages/sqlite.mk | 26 ++++++++++++++++++++++++++ 11 files changed, 44 insertions(+), 8 deletions(-) create mode 100644 depends/packages/sqlite.mk diff --git a/.travis.yml b/.travis.yml index b1bbfb091d..42fa653d8c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -132,6 +132,7 @@ jobs: - berkeley-db4 - miniupnpc - qrencode + - sqlite - ccache - zeromq env: >- diff --git a/build_msvc/vcpkg.json b/build_msvc/vcpkg.json index 5d5f94227f..dfd3929c44 100644 --- a/build_msvc/vcpkg.json +++ b/build_msvc/vcpkg.json @@ -9,6 +9,7 @@ "boost-signals2", "boost-test", "boost-thread", + "sqlite3", "double-conversion", { "name": "libevent", @@ -16,4 +17,4 @@ }, "zeromq" ] -} \ No newline at end of file +} diff --git a/ci/test/00_setup_env_native_asan.sh b/ci/test/00_setup_env_native_asan.sh index 251ece7984..191b8049b0 100644 --- a/ci/test/00_setup_env_native_asan.sh +++ b/ci/test/00_setup_env_native_asan.sh @@ -7,7 +7,7 @@ export LC_ALL=C.UTF-8 export CONTAINER_NAME=ci_native_asan -export PACKAGES="clang llvm python3-zmq qtbase5-dev qttools5-dev-tools libevent-dev bsdmainutils libboost-system-dev libboost-filesystem-dev libboost-test-dev libboost-thread-dev libdb5.3++-dev libminiupnpc-dev libzmq3-dev libqrencode-dev" +export PACKAGES="clang llvm python3-zmq qtbase5-dev qttools5-dev-tools libevent-dev bsdmainutils libboost-system-dev libboost-filesystem-dev libboost-test-dev libboost-thread-dev libdb5.3++-dev libminiupnpc-dev libzmq3-dev libqrencode-dev libsqlite3-dev" export DOCKER_NAME_TAG=ubuntu:20.04 export NO_DEPENDS=1 export GOAL="install" diff --git a/ci/test/00_setup_env_native_msan.sh b/ci/test/00_setup_env_native_msan.sh index 6a4979990b..b88ee2b50f 100644 --- a/ci/test/00_setup_env_native_msan.sh +++ b/ci/test/00_setup_env_native_msan.sh @@ -15,7 +15,7 @@ export BDB_PREFIX="${BASE_ROOT_DIR}/db4" export CONTAINER_NAME="ci_native_msan" export PACKAGES="clang-9 llvm-9 cmake" -export DEP_OPTS="NO_WALLET=1 NO_QT=1 CC='clang' CXX='clang++' CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}' boost_cxxflags='-std=c++11 -fvisibility=hidden -fPIC ${MSAN_AND_LIBCXX_FLAGS}' zeromq_cxxflags='-std=c++11 ${MSAN_AND_LIBCXX_FLAGS}'" +export DEP_OPTS="NO_BDB=1 NO_QT=1 CC='clang' CXX='clang++' CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}' boost_cxxflags='-std=c++11 -fvisibility=hidden -fPIC ${MSAN_AND_LIBCXX_FLAGS}' zeromq_cxxflags='-std=c++11 ${MSAN_AND_LIBCXX_FLAGS}'" export GOAL="install" export BITCOIN_CONFIG="--enable-wallet --with-sanitizers=memory --with-asm=no --prefix=${BASE_ROOT_DIR}/depends/x86_64-pc-linux-gnu/ CC=clang CXX=clang++ CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}' BDB_LIBS='-L${BDB_PREFIX}/lib -ldb_cxx-4.8' BDB_CFLAGS='-I${BDB_PREFIX}/include'" export USE_MEMORY_SANITIZER="true" diff --git a/ci/test/00_setup_env_native_valgrind.sh b/ci/test/00_setup_env_native_valgrind.sh index 710d9e1011..bfaea13a25 100644 --- a/ci/test/00_setup_env_native_valgrind.sh +++ b/ci/test/00_setup_env_native_valgrind.sh @@ -7,7 +7,7 @@ export LC_ALL=C.UTF-8 export CONTAINER_NAME=ci_native_valgrind -export PACKAGES="valgrind clang llvm python3-zmq libevent-dev bsdmainutils libboost-system-dev libboost-filesystem-dev libboost-test-dev libboost-thread-dev libdb5.3++-dev libminiupnpc-dev libzmq3-dev" +export PACKAGES="valgrind clang llvm python3-zmq libevent-dev bsdmainutils libboost-system-dev libboost-filesystem-dev libboost-test-dev libboost-thread-dev libdb5.3++-dev libminiupnpc-dev libzmq3-dev libsqlite3-dev" export USE_VALGRIND=1 export NO_DEPENDS=1 export TEST_RUNNER_EXTRA="--exclude rpc_bind" # Excluded for now, see https://github.com/bitcoin/bitcoin/issues/17765#issuecomment-602068547 diff --git a/contrib/gitian-descriptors/gitian-linux.yml b/contrib/gitian-descriptors/gitian-linux.yml index e86ff83798..65f9a2e5c9 100644 --- a/contrib/gitian-descriptors/gitian-linux.yml +++ b/contrib/gitian-descriptors/gitian-linux.yml @@ -78,7 +78,7 @@ script: | echo "REAL=\`which -a ${i}-${prog}-8 | grep -v ${WRAP_DIR}/${i}-${prog} | head -1\`" >> ${WRAP_DIR}/${i}-${prog} echo "export LD_PRELOAD='/usr/\$LIB/faketime/libfaketime.so.1'" >> ${WRAP_DIR}/${i}-${prog} echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${i}-${prog} - echo "\$REAL \$@" >> $WRAP_DIR/${i}-${prog} + echo "\$REAL \"\$@\"" >> $WRAP_DIR/${i}-${prog} chmod +x ${WRAP_DIR}/${i}-${prog} fi done diff --git a/contrib/gitian-descriptors/gitian-win.yml b/contrib/gitian-descriptors/gitian-win.yml index d05b6d426d..5f671b95ce 100644 --- a/contrib/gitian-descriptors/gitian-win.yml +++ b/contrib/gitian-descriptors/gitian-win.yml @@ -81,7 +81,7 @@ script: | echo "REAL=\`which -a ${i}-${prog}-posix | grep -v ${WRAP_DIR}/${i}-${prog} | head -1\`" >> ${WRAP_DIR}/${i}-${prog} echo "export LD_PRELOAD='/usr/\$LIB/faketime/libfaketime.so.1'" >> ${WRAP_DIR}/${i}-${prog} echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${i}-${prog} - echo "\$REAL \$@" >> $WRAP_DIR/${i}-${prog} + echo "\$REAL \"\$@\"" >> $WRAP_DIR/${i}-${prog} chmod +x ${WRAP_DIR}/${i}-${prog} done done diff --git a/depends/Makefile b/depends/Makefile index 2bc5df974a..1ad21f6821 100644 --- a/depends/Makefile +++ b/depends/Makefile @@ -134,7 +134,10 @@ qrencode_packages_$(NO_QR) = $(qrencode_packages) qt_packages_$(NO_QT) = $(qt_packages) $(qt_$(host_os)_packages) $(qt_$(host_arch)_$(host_os)_packages) $(qrencode_packages_) -wallet_packages_$(NO_WALLET) = $(wallet_packages) +bdb_packages_$(NO_BDB) = $(bdb_packages) +sqlite_packages_$(NO_SQLITE) = $(sqlite_packages) +wallet_packages_$(NO_WALLET) = $(bdb_packages_) $(sqlite_packages_) + upnp_packages_$(NO_UPNP) = $(upnp_packages) zmq_packages_$(NO_ZMQ) = $(zmq_packages) multiprocess_packages_$(MULTIPROCESS) = $(multiprocess_packages) diff --git a/depends/README.md b/depends/README.md index 2356e8be59..869ebe12fd 100644 --- a/depends/README.md +++ b/depends/README.md @@ -99,6 +99,10 @@ The following can be set when running make: `make FOO=bar`
Don't download/build/cache packages needed for enabling zeromq
NO_WALLET
Don't download/build/cache libs needed to enable the wallet
+
NO_BDB
+
Don't download/build/cache BerkeleyDB
+
NO_SQLITE
+
Don't download/build/cache SQLite
NO_UPNP
Don't download/build/cache packages needed for enabling upnp
MULTIPROCESS
diff --git a/depends/packages/packages.mk b/depends/packages/packages.mk index 8fe2c771c9..4627acb521 100644 --- a/depends/packages/packages.mk +++ b/depends/packages/packages.mk @@ -10,7 +10,8 @@ qt_android_packages=qt qt_darwin_packages=qt qt_mingw32_packages=qt -wallet_packages=bdb +bdb_packages=bdb +sqlite_packages=sqlite zmq_packages=zeromq diff --git a/depends/packages/sqlite.mk b/depends/packages/sqlite.mk new file mode 100644 index 0000000000..5b3a61b239 --- /dev/null +++ b/depends/packages/sqlite.mk @@ -0,0 +1,26 @@ +package=sqlite +$(package)_version=3320100 +$(package)_download_path=https://sqlite.org/2020/ +$(package)_file_name=sqlite-autoconf-$($(package)_version).tar.gz +$(package)_sha256_hash=486748abfb16abd8af664e3a5f03b228e5f124682b0c942e157644bf6fff7d10 + +define $(package)_set_vars +$(package)_config_opts=--disable-shared --disable-readline --disable-dynamic-extensions --enable-option-checking +$(package)_config_opts_linux=--with-pic +endef + +define $(package)_config_cmds + $($(package)_autoconf) +endef + +define $(package)_build_cmds + $(MAKE) libsqlite3.la +endef + +define $(package)_stage_cmds + $(MAKE) DESTDIR=$($(package)_staging_dir) install-libLTLIBRARIES install-includeHEADERS install-pkgconfigDATA +endef + +define $(package)_postprocess_cmds + rm lib/*.la +endef -- cgit v1.2.3 From 7577b6e1c88a1a7b45ecf5c7f1735bae6f5a82bf Mon Sep 17 00:00:00 2001 From: Andrew Chow Date: Tue, 26 May 2020 20:53:01 -0400 Subject: Add SQLiteDatabase and SQLiteBatch dummy classes --- src/Makefile.am | 2 + src/wallet/sqlite.cpp | 109 ++++++++++++++++++++++++++++++++++++++++++++++++++ src/wallet/sqlite.h | 99 +++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 210 insertions(+) create mode 100644 src/wallet/sqlite.cpp create mode 100644 src/wallet/sqlite.h diff --git a/src/Makefile.am b/src/Makefile.am index 398d904fe6..e359ae2cb2 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -257,6 +257,7 @@ BITCOIN_CORE_H = \ wallet/rpcwallet.h \ wallet/salvage.h \ wallet/scriptpubkeyman.h \ + wallet/sqlite.h \ wallet/wallet.h \ wallet/walletdb.h \ wallet/wallettool.h \ @@ -370,6 +371,7 @@ libbitcoin_wallet_a_SOURCES = \ wallet/rpcwallet.cpp \ wallet/salvage.cpp \ wallet/scriptpubkeyman.cpp \ + wallet/sqlite.cpp \ wallet/wallet.cpp \ wallet/walletdb.cpp \ wallet/walletutil.cpp \ diff --git a/src/wallet/sqlite.cpp b/src/wallet/sqlite.cpp new file mode 100644 index 0000000000..0ea1317795 --- /dev/null +++ b/src/wallet/sqlite.cpp @@ -0,0 +1,109 @@ +// Copyright (c) 2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include + +#include +#include +#include +#include + +#include + +static const char* const DATABASE_FILENAME = "wallet.dat"; + +SQLiteDatabase::SQLiteDatabase(const fs::path& dir_path, const fs::path& file_path, bool mock) + : WalletDatabase(), m_mock(mock), m_dir_path(dir_path.string()), m_file_path(file_path.string()) +{ +} + +SQLiteDatabase::~SQLiteDatabase() +{ +} + +void SQLiteDatabase::Open() +{ +} + +bool SQLiteDatabase::Rewrite(const char* skip) +{ + return false; +} + +bool SQLiteDatabase::Backup(const std::string& dest) const +{ + return false; +} + +void SQLiteDatabase::Close() +{ +} + +std::unique_ptr SQLiteDatabase::MakeBatch(bool flush_on_close) +{ + return nullptr; +} + +void SQLiteBatch::Close() +{ +} + +bool SQLiteBatch::ReadKey(CDataStream&& key, CDataStream& value) +{ + return false; +} + +bool SQLiteBatch::WriteKey(CDataStream&& key, CDataStream&& value, bool overwrite) +{ + return false; +} + +bool SQLiteBatch::EraseKey(CDataStream&& key) +{ + return false; +} + +bool SQLiteBatch::HasKey(CDataStream&& key) +{ + return false; +} + +bool SQLiteBatch::StartCursor() +{ + return false; +} + +bool SQLiteBatch::ReadAtCursor(CDataStream& key, CDataStream& value, bool& complete) +{ + return false; +} + +void SQLiteBatch::CloseCursor() +{ +} + +bool SQLiteBatch::TxnBegin() +{ + return false; +} + +bool SQLiteBatch::TxnCommit() +{ + return false; +} + +bool SQLiteBatch::TxnAbort() +{ + return false; +} + +bool ExistsSQLiteDatabase(const fs::path& path) +{ + return false; +} + +std::unique_ptr MakeSQLiteDatabase(const fs::path& path, const DatabaseOptions& options, DatabaseStatus& status, bilingual_str& error) +{ + return MakeUnique(path, path / DATABASE_FILENAME); +} diff --git a/src/wallet/sqlite.h b/src/wallet/sqlite.h new file mode 100644 index 0000000000..0c9ab32fc8 --- /dev/null +++ b/src/wallet/sqlite.h @@ -0,0 +1,99 @@ +// Copyright (c) 2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_WALLET_SQLITE_H +#define BITCOIN_WALLET_SQLITE_H + +#include + +struct bilingual_str; +class SQLiteDatabase; + +/** RAII class that provides access to a WalletDatabase */ +class SQLiteBatch : public DatabaseBatch +{ +private: + SQLiteDatabase& m_database; + + bool ReadKey(CDataStream&& key, CDataStream& value) override; + bool WriteKey(CDataStream&& key, CDataStream&& value, bool overwrite = true) override; + bool EraseKey(CDataStream&& key) override; + bool HasKey(CDataStream&& key) override; + +public: + explicit SQLiteBatch(SQLiteDatabase& database); + + /* No-op. See commeng on SQLiteDatabase::Flush */ + void Flush() override {} + + void Close() override; + + bool StartCursor() override; + bool ReadAtCursor(CDataStream& key, CDataStream& value, bool& complete) override; + void CloseCursor() override; + bool TxnBegin() override; + bool TxnCommit() override; + bool TxnAbort() override; +}; + +/** An instance of this class represents one SQLite3 database. + **/ +class SQLiteDatabase : public WalletDatabase +{ +private: + const bool m_mock{false}; + + const std::string m_dir_path; + + const std::string m_file_path; + +public: + SQLiteDatabase() = delete; + + /** Create DB handle to real database */ + SQLiteDatabase(const fs::path& dir_path, const fs::path& file_path, bool mock = false); + + ~SQLiteDatabase(); + + /** Open the database if it is not already opened */ + void Open() override; + + /** Close the database */ + void Close() override; + + /* These functions are unused */ + void AddRef() override { assert(false); } + void RemoveRef() override { assert(false); } + + /** Rewrite the entire database on disk */ + bool Rewrite(const char* skip = nullptr) override; + + /** Back up the entire database to a file. + */ + bool Backup(const std::string& dest) const override; + + /** No-ops + * + * SQLite always flushes everything to the database file after each transaction + * (each Read/Write/Erase that we do is its own transaction unless we called + * TxnBegin) so there is no need to have Flush or Periodic Flush. + * + * There is no DB env to reload, so ReloadDbEnv has nothing to do + */ + void Flush() override {} + bool PeriodicFlush() override { return false; } + void ReloadDbEnv() override {} + + void IncrementUpdateCounter() override { ++nUpdateCounter; } + + std::string Filename() override { return m_file_path; } + + /** Make a SQLiteBatch connected to this database */ + std::unique_ptr MakeBatch(bool flush_on_close = true) override; +}; + +bool ExistsSQLiteDatabase(const fs::path& path); +std::unique_ptr MakeSQLiteDatabase(const fs::path& path, const DatabaseOptions& options, DatabaseStatus& status, bilingual_str& error); + +#endif // BITCOIN_WALLET_SQLITE_H -- cgit v1.2.3 From ca8b7e04ab89f99075b093fa248919fd10acbdf7 Mon Sep 17 00:00:00 2001 From: Andrew Chow Date: Tue, 26 May 2020 20:53:05 -0400 Subject: Implement SQLiteDatabaseVersion --- src/wallet/sqlite.cpp | 6 ++++++ src/wallet/sqlite.h | 2 ++ 2 files changed, 8 insertions(+) diff --git a/src/wallet/sqlite.cpp b/src/wallet/sqlite.cpp index 0ea1317795..c406269cab 100644 --- a/src/wallet/sqlite.cpp +++ b/src/wallet/sqlite.cpp @@ -9,6 +9,7 @@ #include #include +#include #include static const char* const DATABASE_FILENAME = "wallet.dat"; @@ -107,3 +108,8 @@ std::unique_ptr MakeSQLiteDatabase(const fs::path& path, const D { return MakeUnique(path, path / DATABASE_FILENAME); } + +std::string SQLiteDatabaseVersion() +{ + return std::string(sqlite3_libversion()); +} diff --git a/src/wallet/sqlite.h b/src/wallet/sqlite.h index 0c9ab32fc8..b529301771 100644 --- a/src/wallet/sqlite.h +++ b/src/wallet/sqlite.h @@ -96,4 +96,6 @@ public: bool ExistsSQLiteDatabase(const fs::path& path); std::unique_ptr MakeSQLiteDatabase(const fs::path& path, const DatabaseOptions& options, DatabaseStatus& status, bilingual_str& error); +std::string SQLiteDatabaseVersion(); + #endif // BITCOIN_WALLET_SQLITE_H -- cgit v1.2.3 From 5a488b3d77326a0d957c1233493061da1b6ec207 Mon Sep 17 00:00:00 2001 From: Andrew Chow Date: Tue, 26 May 2020 20:53:06 -0400 Subject: Constructors, destructors, and relevant private fields for SQLiteDatabase/Batch --- src/wallet/sqlite.cpp | 11 +++++++++++ src/wallet/sqlite.h | 5 +++++ 2 files changed, 16 insertions(+) diff --git a/src/wallet/sqlite.cpp b/src/wallet/sqlite.cpp index c406269cab..9a9904e17d 100644 --- a/src/wallet/sqlite.cpp +++ b/src/wallet/sqlite.cpp @@ -4,6 +4,7 @@ #include +#include #include #include #include @@ -17,10 +18,15 @@ static const char* const DATABASE_FILENAME = "wallet.dat"; SQLiteDatabase::SQLiteDatabase(const fs::path& dir_path, const fs::path& file_path, bool mock) : WalletDatabase(), m_mock(mock), m_dir_path(dir_path.string()), m_file_path(file_path.string()) { + LogPrintf("Using SQLite Version %s\n", SQLiteDatabaseVersion()); + LogPrintf("Using wallet %s\n", m_dir_path); + + Open(); } SQLiteDatabase::~SQLiteDatabase() { + Close(); } void SQLiteDatabase::Open() @@ -46,6 +52,11 @@ std::unique_ptr SQLiteDatabase::MakeBatch(bool flush_on_close) return nullptr; } +SQLiteBatch::SQLiteBatch(SQLiteDatabase& database) + : m_database(database) +{ +} + void SQLiteBatch::Close() { } diff --git a/src/wallet/sqlite.h b/src/wallet/sqlite.h index b529301771..e56533b7b6 100644 --- a/src/wallet/sqlite.h +++ b/src/wallet/sqlite.h @@ -7,6 +7,8 @@ #include +#include + struct bilingual_str; class SQLiteDatabase; @@ -23,6 +25,7 @@ private: public: explicit SQLiteBatch(SQLiteDatabase& database); + ~SQLiteBatch() override { Close(); } /* No-op. See commeng on SQLiteDatabase::Flush */ void Flush() override {} @@ -91,6 +94,8 @@ public: /** Make a SQLiteBatch connected to this database */ std::unique_ptr MakeBatch(bool flush_on_close = true) override; + + sqlite3* m_db{nullptr}; }; bool ExistsSQLiteDatabase(const fs::path& path); -- cgit v1.2.3 From 3bfa0fe1259280f8c32b41a798c9453b73f89b02 Mon Sep 17 00:00:00 2001 From: Andrew Chow Date: Tue, 26 May 2020 20:53:24 -0400 Subject: Initialize and Shutdown sqlite3 globals sqlite3 recommends that sqlite3_initialize be called when the application starts, and sqlite3_shutdown when it stops. Since we don't always use sqlite3, we initialize it when a SQLiteDatabse is constructed (calling sqlite3_initialize after initialized is a no-op). We call sqlite3_shutdown when we see that there are no databases opened. The number of open databases is tracked by an atomic g_dbs_open. --- src/wallet/sqlite.cpp | 57 +++++++++++++++++++++++++++++++++++++++++++++++---- src/wallet/sqlite.h | 2 ++ 2 files changed, 55 insertions(+), 4 deletions(-) diff --git a/src/wallet/sqlite.cpp b/src/wallet/sqlite.cpp index 9a9904e17d..ed331b6470 100644 --- a/src/wallet/sqlite.cpp +++ b/src/wallet/sqlite.cpp @@ -5,6 +5,7 @@ #include #include +#include #include #include #include @@ -15,18 +16,66 @@ static const char* const DATABASE_FILENAME = "wallet.dat"; +static Mutex g_sqlite_mutex; +static int g_sqlite_count GUARDED_BY(g_sqlite_mutex) = 0; + +static void ErrorLogCallback(void* arg, int code, const char* msg) +{ + // From sqlite3_config() documentation for the SQLITE_CONFIG_LOG option: + // "The void pointer that is the second argument to SQLITE_CONFIG_LOG is passed through as + // the first parameter to the application-defined logger function whenever that function is + // invoked." + // Assert that this is the case: + assert(arg == nullptr); + LogPrintf("SQLite Error. Code: %d. Message: %s\n", code, msg); +} + SQLiteDatabase::SQLiteDatabase(const fs::path& dir_path, const fs::path& file_path, bool mock) : WalletDatabase(), m_mock(mock), m_dir_path(dir_path.string()), m_file_path(file_path.string()) { - LogPrintf("Using SQLite Version %s\n", SQLiteDatabaseVersion()); - LogPrintf("Using wallet %s\n", m_dir_path); - - Open(); + { + LOCK(g_sqlite_mutex); + LogPrintf("Using SQLite Version %s\n", SQLiteDatabaseVersion()); + LogPrintf("Using wallet %s\n", m_dir_path); + + if (++g_sqlite_count == 1) { + // Setup logging + int ret = sqlite3_config(SQLITE_CONFIG_LOG, ErrorLogCallback, nullptr); + if (ret != SQLITE_OK) { + throw std::runtime_error(strprintf("SQLiteDatabase: Failed to setup error log: %s\n", sqlite3_errstr(ret))); + } + } + int ret = sqlite3_initialize(); // This is a no-op if sqlite3 is already initialized + if (ret != SQLITE_OK) { + throw std::runtime_error(strprintf("SQLiteDatabase: Failed to initialize SQLite: %s\n", sqlite3_errstr(ret))); + } + } + + try { + Open(); + } catch (const std::runtime_error&) { + // If open fails, cleanup this object and rethrow the exception + Cleanup(); + throw; + } } SQLiteDatabase::~SQLiteDatabase() +{ + Cleanup(); +} + +void SQLiteDatabase::Cleanup() noexcept { Close(); + + LOCK(g_sqlite_mutex); + if (--g_sqlite_count == 0) { + int ret = sqlite3_shutdown(); + if (ret != SQLITE_OK) { + LogPrintf("SQLiteDatabase: Failed to shutdown SQLite: %s\n", sqlite3_errstr(ret)); + } + } } void SQLiteDatabase::Open() diff --git a/src/wallet/sqlite.h b/src/wallet/sqlite.h index e56533b7b6..8171cf27d6 100644 --- a/src/wallet/sqlite.h +++ b/src/wallet/sqlite.h @@ -51,6 +51,8 @@ private: const std::string m_file_path; + void Cleanup() noexcept; + public: SQLiteDatabase() = delete; -- cgit v1.2.3 From a0de83372be83f59015cd3d61af2303b74fb64b5 Mon Sep 17 00:00:00 2001 From: Andrew Chow Date: Tue, 26 May 2020 20:53:30 -0400 Subject: Implement SQLiteDatabase::Open --- src/wallet/sqlite.cpp | 69 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) diff --git a/src/wallet/sqlite.cpp b/src/wallet/sqlite.cpp index ed331b6470..fb0bef14ee 100644 --- a/src/wallet/sqlite.cpp +++ b/src/wallet/sqlite.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include #include @@ -80,6 +81,72 @@ void SQLiteDatabase::Cleanup() noexcept void SQLiteDatabase::Open() { + int flags = SQLITE_OPEN_FULLMUTEX | SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE; + if (m_mock) { + flags |= SQLITE_OPEN_MEMORY; // In memory database for mock db + } + + if (m_db == nullptr) { + TryCreateDirectories(m_dir_path); + int ret = sqlite3_open_v2(m_file_path.c_str(), &m_db, flags, nullptr); + if (ret != SQLITE_OK) { + throw std::runtime_error(strprintf("SQLiteDatabase: Failed to open database: %s\n", sqlite3_errstr(ret))); + } + } + + if (sqlite3_db_readonly(m_db, "main") != 0) { + throw std::runtime_error("SQLiteDatabase: Database opened in readonly mode but read-write permissions are needed"); + } + + // Acquire an exclusive lock on the database + // First change the locking mode to exclusive + int ret = sqlite3_exec(m_db, "PRAGMA locking_mode = exclusive", nullptr, nullptr, nullptr); + if (ret != SQLITE_OK) { + throw std::runtime_error(strprintf("SQLiteDatabase: Unable to change database locking mode to exclusive: %s\n", sqlite3_errstr(ret))); + } + // Now begin a transaction to acquire the exclusive lock. This lock won't be released until we close because of the exclusive locking mode. + ret = sqlite3_exec(m_db, "BEGIN EXCLUSIVE TRANSACTION", nullptr, nullptr, nullptr); + if (ret != SQLITE_OK) { + throw std::runtime_error("SQLiteDatabase: Unable to obtain an exclusive lock on the database, is it being used by another bitcoind?\n"); + } + ret = sqlite3_exec(m_db, "COMMIT", nullptr, nullptr, nullptr); + if (ret != SQLITE_OK) { + throw std::runtime_error(strprintf("SQLiteDatabase: Unable to end exclusive lock transaction: %s\n", sqlite3_errstr(ret))); + } + + // Enable fullfsync for the platforms that use it + ret = sqlite3_exec(m_db, "PRAGMA fullfsync = true", nullptr, nullptr, nullptr); + if (ret != SQLITE_OK) { + throw std::runtime_error(strprintf("SQLiteDatabase: Failed to enable fullfsync: %s\n", sqlite3_errstr(ret))); + } + + // Make the table for our key-value pairs + // First check that the main table exists + sqlite3_stmt* check_main_stmt{nullptr}; + ret = sqlite3_prepare_v2(m_db, "SELECT name FROM sqlite_master WHERE type='table' AND name='main'", -1, &check_main_stmt, nullptr); + if (ret != SQLITE_OK) { + throw std::runtime_error(strprintf("SQLiteDatabase: Failed to prepare statement to check table existence: %s\n", sqlite3_errstr(ret))); + } + ret = sqlite3_step(check_main_stmt); + if (sqlite3_finalize(check_main_stmt) != SQLITE_OK) { + throw std::runtime_error(strprintf("SQLiteDatabase: Failed to finalize statement checking table existence: %s\n", sqlite3_errstr(ret))); + } + bool table_exists; + if (ret == SQLITE_DONE) { + table_exists = false; + } else if (ret == SQLITE_ROW) { + table_exists = true; + } else { + throw std::runtime_error(strprintf("SQLiteDatabase: Failed to execute statement to check table existence: %s\n", sqlite3_errstr(ret))); + } + + // Do the db setup things because the table doesn't exist only when we are creating a new wallet + if (!table_exists) { + ret = sqlite3_exec(m_db, "CREATE TABLE main(key BLOB PRIMARY KEY NOT NULL, value BLOB NOT NULL)", nullptr, nullptr, nullptr); + if (ret != SQLITE_OK) { + throw std::runtime_error(strprintf("SQLiteDatabase: Failed to create new database: %s\n", sqlite3_errstr(ret))); + } + } } bool SQLiteDatabase::Rewrite(const char* skip) @@ -104,6 +171,8 @@ std::unique_ptr SQLiteDatabase::MakeBatch(bool flush_on_close) SQLiteBatch::SQLiteBatch(SQLiteDatabase& database) : m_database(database) { + // Make sure we have a db handle + assert(m_database.m_db); } void SQLiteBatch::Close() -- cgit v1.2.3 From 93825352a36456283bf87e39b5888363ee242f21 Mon Sep 17 00:00:00 2001 From: Andrew Chow Date: Tue, 26 May 2020 20:53:32 -0400 Subject: Implement SQLiteDatabase::Close --- src/wallet/sqlite.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/wallet/sqlite.cpp b/src/wallet/sqlite.cpp index fb0bef14ee..7d96575a64 100644 --- a/src/wallet/sqlite.cpp +++ b/src/wallet/sqlite.cpp @@ -161,6 +161,11 @@ bool SQLiteDatabase::Backup(const std::string& dest) const void SQLiteDatabase::Close() { + int res = sqlite3_close(m_db); + if (res != SQLITE_OK) { + throw std::runtime_error(strprintf("SQLiteDatabase: Failed to close database: %s\n", sqlite3_errstr(res))); + } + m_db = nullptr; } std::unique_ptr SQLiteDatabase::MakeBatch(bool flush_on_close) -- cgit v1.2.3 From 6636a2608a4e5906ee8092d5731595542261e0ad Mon Sep 17 00:00:00 2001 From: Andrew Chow Date: Tue, 16 Jun 2020 14:57:30 -0400 Subject: Implement SQLiteBatch::Close --- src/wallet/sqlite.cpp | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/wallet/sqlite.cpp b/src/wallet/sqlite.cpp index 7d96575a64..d89508c9df 100644 --- a/src/wallet/sqlite.cpp +++ b/src/wallet/sqlite.cpp @@ -182,6 +182,14 @@ SQLiteBatch::SQLiteBatch(SQLiteDatabase& database) void SQLiteBatch::Close() { + // If m_db is in a transaction (i.e. not in autocommit mode), then abort the transaction in progress + if (m_database.m_db && sqlite3_get_autocommit(m_database.m_db) == 0) { + if (TxnAbort()) { + LogPrintf("SQLiteBatch: Batch closed unexpectedly without the transaction being explicitly committed or aborted\n"); + } else { + LogPrintf("SQLiteBatch: Batch closed and failed to abort transaction\n"); + } + } } bool SQLiteBatch::ReadKey(CDataStream&& key, CDataStream& value) -- cgit v1.2.3 From 7aa45620e2f2178145a2eca58ccbab3cecff08fb Mon Sep 17 00:00:00 2001 From: Andrew Chow Date: Tue, 26 May 2020 20:53:41 -0400 Subject: Add SetupSQLStatements --- src/wallet/sqlite.cpp | 59 +++++++++++++++++++++++++++++++++++++++++++++++++++ src/wallet/sqlite.h | 8 +++++++ 2 files changed, 67 insertions(+) diff --git a/src/wallet/sqlite.cpp b/src/wallet/sqlite.cpp index d89508c9df..cacb5554bb 100644 --- a/src/wallet/sqlite.cpp +++ b/src/wallet/sqlite.cpp @@ -61,6 +61,36 @@ SQLiteDatabase::SQLiteDatabase(const fs::path& dir_path, const fs::path& file_pa } } +void SQLiteBatch::SetupSQLStatements() +{ + int res; + if (!m_read_stmt) { + if ((res = sqlite3_prepare_v2(m_database.m_db, "SELECT value FROM main WHERE key = ?", -1, &m_read_stmt, nullptr)) != SQLITE_OK) { + throw std::runtime_error(strprintf("SQLiteDatabase: Failed to setup SQL statements: %s\n", sqlite3_errstr(res))); + } + } + if (!m_insert_stmt) { + if ((res = sqlite3_prepare_v2(m_database.m_db, "INSERT INTO main VALUES(?, ?)", -1, &m_insert_stmt, nullptr)) != SQLITE_OK) { + throw std::runtime_error(strprintf("SQLiteDatabase: Failed to setup SQL statements: %s\n", sqlite3_errstr(res))); + } + } + if (!m_overwrite_stmt) { + if ((res = sqlite3_prepare_v2(m_database.m_db, "INSERT or REPLACE into main values(?, ?)", -1, &m_overwrite_stmt, nullptr)) != SQLITE_OK) { + throw std::runtime_error(strprintf("SQLiteDatabase: Failed to setup SQL statements: %s\n", sqlite3_errstr(res))); + } + } + if (!m_delete_stmt) { + if ((res = sqlite3_prepare_v2(m_database.m_db, "DELETE FROM main WHERE key = ?", -1, &m_delete_stmt, nullptr)) != SQLITE_OK) { + throw std::runtime_error(strprintf("SQLiteDatabase: Failed to setup SQL statements: %s\n", sqlite3_errstr(res))); + } + } + if (!m_cursor_stmt) { + if ((res = sqlite3_prepare_v2(m_database.m_db, "SELECT key, value FROM main", -1, &m_cursor_stmt, nullptr)) != SQLITE_OK) { + throw std::runtime_error(strprintf("SQLiteDatabase: Failed to setup SQL statements : %s\n", sqlite3_errstr(res))); + } + } +} + SQLiteDatabase::~SQLiteDatabase() { Cleanup(); @@ -178,6 +208,8 @@ SQLiteBatch::SQLiteBatch(SQLiteDatabase& database) { // Make sure we have a db handle assert(m_database.m_db); + + SetupSQLStatements(); } void SQLiteBatch::Close() @@ -190,6 +222,33 @@ void SQLiteBatch::Close() LogPrintf("SQLiteBatch: Batch closed and failed to abort transaction\n"); } } + + // Free all of the prepared statements + int ret = sqlite3_finalize(m_read_stmt); + if (ret != SQLITE_OK) { + LogPrintf("SQLiteBatch: Batch closed but could not finalize read statement: %s\n", sqlite3_errstr(ret)); + } + ret = sqlite3_finalize(m_insert_stmt); + if (ret != SQLITE_OK) { + LogPrintf("SQLiteBatch: Batch closed but could not finalize insert statement: %s\n", sqlite3_errstr(ret)); + } + ret = sqlite3_finalize(m_overwrite_stmt); + if (ret != SQLITE_OK) { + LogPrintf("SQLiteBatch: Batch closed but could not finalize overwrite statement: %s\n", sqlite3_errstr(ret)); + } + ret = sqlite3_finalize(m_delete_stmt); + if (ret != SQLITE_OK) { + LogPrintf("SQLiteBatch: Batch closed but could not finalize delete statement: %s\n", sqlite3_errstr(ret)); + } + ret = sqlite3_finalize(m_cursor_stmt); + if (ret != SQLITE_OK) { + LogPrintf("SQLiteBatch: Batch closed but could not finalize cursor statement: %s\n", sqlite3_errstr(ret)); + } + m_read_stmt = nullptr; + m_insert_stmt = nullptr; + m_overwrite_stmt = nullptr; + m_delete_stmt = nullptr; + m_cursor_stmt = nullptr; } bool SQLiteBatch::ReadKey(CDataStream&& key, CDataStream& value) diff --git a/src/wallet/sqlite.h b/src/wallet/sqlite.h index 8171cf27d6..4d4dca1d20 100644 --- a/src/wallet/sqlite.h +++ b/src/wallet/sqlite.h @@ -18,6 +18,14 @@ class SQLiteBatch : public DatabaseBatch private: SQLiteDatabase& m_database; + sqlite3_stmt* m_read_stmt{nullptr}; + sqlite3_stmt* m_insert_stmt{nullptr}; + sqlite3_stmt* m_overwrite_stmt{nullptr}; + sqlite3_stmt* m_delete_stmt{nullptr}; + sqlite3_stmt* m_cursor_stmt{nullptr}; + + void SetupSQLStatements(); + bool ReadKey(CDataStream&& key, CDataStream& value) override; bool WriteKey(CDataStream&& key, CDataStream&& value, bool overwrite = true) override; bool EraseKey(CDataStream&& key) override; -- cgit v1.2.3 From bf90e033f4fe86cfb90492c7e0962278ea3a146d Mon Sep 17 00:00:00 2001 From: Andrew Chow Date: Tue, 26 May 2020 20:53:44 -0400 Subject: Implement SQLiteBatch::ReadKey, WriteKey, EraseKey, and HasKey --- src/wallet/sqlite.cpp | 104 ++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 100 insertions(+), 4 deletions(-) diff --git a/src/wallet/sqlite.cpp b/src/wallet/sqlite.cpp index cacb5554bb..aae32e404e 100644 --- a/src/wallet/sqlite.cpp +++ b/src/wallet/sqlite.cpp @@ -253,22 +253,118 @@ void SQLiteBatch::Close() bool SQLiteBatch::ReadKey(CDataStream&& key, CDataStream& value) { - return false; + if (!m_database.m_db) return false; + assert(m_read_stmt); + + // Bind: leftmost parameter in statement is index 1 + int res = sqlite3_bind_blob(m_read_stmt, 1, key.data(), key.size(), SQLITE_STATIC); + if (res != SQLITE_OK) { + LogPrintf("%s: Unable to bind statement: %s\n", __func__, sqlite3_errstr(res)); + sqlite3_clear_bindings(m_read_stmt); + sqlite3_reset(m_read_stmt); + return false; + } + res = sqlite3_step(m_read_stmt); + if (res != SQLITE_ROW) { + if (res != SQLITE_DONE) { + // SQLITE_DONE means "not found", don't log an error in that case. + LogPrintf("%s: Unable to execute statement: %s\n", __func__, sqlite3_errstr(res)); + } + sqlite3_clear_bindings(m_read_stmt); + sqlite3_reset(m_read_stmt); + return false; + } + // Leftmost column in result is index 0 + const char* data = reinterpret_cast(sqlite3_column_blob(m_read_stmt, 0)); + int data_size = sqlite3_column_bytes(m_read_stmt, 0); + value.write(data, data_size); + + sqlite3_clear_bindings(m_read_stmt); + sqlite3_reset(m_read_stmt); + return true; } bool SQLiteBatch::WriteKey(CDataStream&& key, CDataStream&& value, bool overwrite) { - return false; + if (!m_database.m_db) return false; + assert(m_insert_stmt && m_overwrite_stmt); + + sqlite3_stmt* stmt; + if (overwrite) { + stmt = m_overwrite_stmt; + } else { + stmt = m_insert_stmt; + } + + // Bind: leftmost parameter in statement is index 1 + // Insert index 1 is key, 2 is value + int res = sqlite3_bind_blob(stmt, 1, key.data(), key.size(), SQLITE_STATIC); + if (res != SQLITE_OK) { + LogPrintf("%s: Unable to bind key to statement: %s\n", __func__, sqlite3_errstr(res)); + sqlite3_clear_bindings(stmt); + sqlite3_reset(stmt); + return false; + } + res = sqlite3_bind_blob(stmt, 2, value.data(), value.size(), SQLITE_STATIC); + if (res != SQLITE_OK) { + LogPrintf("%s: Unable to bind value to statement: %s\n", __func__, sqlite3_errstr(res)); + sqlite3_clear_bindings(stmt); + sqlite3_reset(stmt); + return false; + } + + // Execute + res = sqlite3_step(stmt); + sqlite3_clear_bindings(stmt); + sqlite3_reset(stmt); + if (res != SQLITE_DONE) { + LogPrintf("%s: Unable to execute statement: %s\n", __func__, sqlite3_errstr(res)); + } + return res == SQLITE_DONE; } bool SQLiteBatch::EraseKey(CDataStream&& key) { - return false; + if (!m_database.m_db) return false; + assert(m_delete_stmt); + + // Bind: leftmost parameter in statement is index 1 + int res = sqlite3_bind_blob(m_delete_stmt, 1, key.data(), key.size(), SQLITE_STATIC); + if (res != SQLITE_OK) { + LogPrintf("%s: Unable to bind statement: %s\n", __func__, sqlite3_errstr(res)); + sqlite3_clear_bindings(m_delete_stmt); + sqlite3_reset(m_delete_stmt); + return false; + } + + // Execute + res = sqlite3_step(m_delete_stmt); + sqlite3_clear_bindings(m_delete_stmt); + sqlite3_reset(m_delete_stmt); + if (res != SQLITE_DONE) { + LogPrintf("%s: Unable to execute statement: %s\n", __func__, sqlite3_errstr(res)); + } + return res == SQLITE_DONE; } bool SQLiteBatch::HasKey(CDataStream&& key) { - return false; + if (!m_database.m_db) return false; + assert(m_read_stmt); + + // Bind: leftmost parameter in statement is index 1 + bool ret = false; + int res = sqlite3_bind_blob(m_read_stmt, 1, key.data(), key.size(), SQLITE_STATIC); + if (res == SQLITE_OK) { + res = sqlite3_step(m_read_stmt); + if (res == SQLITE_ROW) { + ret = true; + } + } + + sqlite3_clear_bindings(m_read_stmt); + sqlite3_reset(m_read_stmt); + return ret; } bool SQLiteBatch::StartCursor() -- cgit v1.2.3 From f6f9cd6a64842ef23777312f2465e826ca04b886 Mon Sep 17 00:00:00 2001 From: Andrew Chow Date: Tue, 26 May 2020 20:53:46 -0400 Subject: Implement SQLiteBatch::StartCursor, ReadAtCursor, and CloseCursor --- src/wallet/sqlite.cpp | 30 ++++++++++++++++++++++++++++-- src/wallet/sqlite.h | 2 ++ 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/src/wallet/sqlite.cpp b/src/wallet/sqlite.cpp index aae32e404e..d684eef87e 100644 --- a/src/wallet/sqlite.cpp +++ b/src/wallet/sqlite.cpp @@ -369,16 +369,42 @@ bool SQLiteBatch::HasKey(CDataStream&& key) bool SQLiteBatch::StartCursor() { - return false; + assert(!m_cursor_init); + if (!m_database.m_db) return false; + m_cursor_init = true; + return true; } bool SQLiteBatch::ReadAtCursor(CDataStream& key, CDataStream& value, bool& complete) { - return false; + complete = false; + + if (!m_cursor_init) return false; + + int res = sqlite3_step(m_cursor_stmt); + if (res == SQLITE_DONE) { + complete = true; + return true; + } + if (res != SQLITE_ROW) { + LogPrintf("SQLiteBatch::ReadAtCursor: Unable to execute cursor step: %s\n", sqlite3_errstr(res)); + return false; + } + + // Leftmost column in result is index 0 + const char* key_data = reinterpret_cast(sqlite3_column_blob(m_cursor_stmt, 0)); + int key_data_size = sqlite3_column_bytes(m_cursor_stmt, 0); + key.write(key_data, key_data_size); + const char* value_data = reinterpret_cast(sqlite3_column_blob(m_cursor_stmt, 1)); + int value_data_size = sqlite3_column_bytes(m_cursor_stmt, 1); + value.write(value_data, value_data_size); + return true; } void SQLiteBatch::CloseCursor() { + sqlite3_reset(m_cursor_stmt); + m_cursor_init = false; } bool SQLiteBatch::TxnBegin() diff --git a/src/wallet/sqlite.h b/src/wallet/sqlite.h index 4d4dca1d20..dca6560abb 100644 --- a/src/wallet/sqlite.h +++ b/src/wallet/sqlite.h @@ -18,6 +18,8 @@ class SQLiteBatch : public DatabaseBatch private: SQLiteDatabase& m_database; + bool m_cursor_init = false; + sqlite3_stmt* m_read_stmt{nullptr}; sqlite3_stmt* m_insert_stmt{nullptr}; sqlite3_stmt* m_overwrite_stmt{nullptr}; -- cgit v1.2.3 From ac5c1617e7f4273daf24c24da1f6bc5ef5ab2d2b Mon Sep 17 00:00:00 2001 From: Andrew Chow Date: Tue, 26 May 2020 20:53:48 -0400 Subject: Implement SQLiteDatabase::Backup --- src/wallet/sqlite.cpp | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/src/wallet/sqlite.cpp b/src/wallet/sqlite.cpp index d684eef87e..8ad82743ab 100644 --- a/src/wallet/sqlite.cpp +++ b/src/wallet/sqlite.cpp @@ -186,7 +186,29 @@ bool SQLiteDatabase::Rewrite(const char* skip) bool SQLiteDatabase::Backup(const std::string& dest) const { - return false; + sqlite3* db_copy; + int res = sqlite3_open(dest.c_str(), &db_copy); + if (res != SQLITE_OK) { + sqlite3_close(db_copy); + return false; + } + sqlite3_backup* backup = sqlite3_backup_init(db_copy, "main", m_db, "main"); + if (!backup) { + LogPrintf("%s: Unable to begin backup: %s\n", __func__, sqlite3_errmsg(m_db)); + sqlite3_close(db_copy); + return false; + } + // Specifying -1 will copy all of the pages + res = sqlite3_backup_step(backup, -1); + if (res != SQLITE_DONE) { + LogPrintf("%s: Unable to backup: %s\n", __func__, sqlite3_errstr(res)); + sqlite3_backup_finish(backup); + sqlite3_close(db_copy); + return false; + } + res = sqlite3_backup_finish(backup); + sqlite3_close(db_copy); + return res == SQLITE_OK; } void SQLiteDatabase::Close() -- cgit v1.2.3 From 010e3659069e6f97dd7b24483f50ed71042b84b0 Mon Sep 17 00:00:00 2001 From: Andrew Chow Date: Tue, 26 May 2020 20:53:50 -0400 Subject: Implement SQLiteDatabase::TxnBegin, TxnCommit, and TxnAbort --- src/wallet/sqlite.cpp | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/src/wallet/sqlite.cpp b/src/wallet/sqlite.cpp index 8ad82743ab..1889313dd5 100644 --- a/src/wallet/sqlite.cpp +++ b/src/wallet/sqlite.cpp @@ -431,17 +431,32 @@ void SQLiteBatch::CloseCursor() bool SQLiteBatch::TxnBegin() { - return false; + if (!m_database.m_db || sqlite3_get_autocommit(m_database.m_db) == 0) return false; + int res = sqlite3_exec(m_database.m_db, "BEGIN TRANSACTION", nullptr, nullptr, nullptr); + if (res != SQLITE_OK) { + LogPrintf("SQLiteBatch: Failed to begin the transaction\n"); + } + return res == SQLITE_OK; } bool SQLiteBatch::TxnCommit() { - return false; + if (!m_database.m_db || sqlite3_get_autocommit(m_database.m_db) != 0) return false; + int res = sqlite3_exec(m_database.m_db, "COMMIT TRANSACTION", nullptr, nullptr, nullptr); + if (res != SQLITE_OK) { + LogPrintf("SQLiteBatch: Failed to commit the transaction\n"); + } + return res == SQLITE_OK; } bool SQLiteBatch::TxnAbort() { - return false; + if (!m_database.m_db || sqlite3_get_autocommit(m_database.m_db) != 0) return false; + int res = sqlite3_exec(m_database.m_db, "ROLLBACK TRANSACTION", nullptr, nullptr, nullptr); + if (res != SQLITE_OK) { + LogPrintf("SQLiteBatch: Failed to abort the transaction\n"); + } + return res == SQLITE_OK; } bool ExistsSQLiteDatabase(const fs::path& path) -- cgit v1.2.3 From b4df8fdb19fcded7e6d491ecf0b705cac0ec76a1 Mon Sep 17 00:00:00 2001 From: Andrew Chow Date: Tue, 26 May 2020 20:53:57 -0400 Subject: Implement SQLiteDatabase::Rewrite Rewrite uses the VACUUM command which does exactly what we want. A specific advertised use case is to compact a database and ensure that any deleted data is actually deleted. --- src/wallet/sqlite.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/wallet/sqlite.cpp b/src/wallet/sqlite.cpp index 1889313dd5..f9c4dfb2a5 100644 --- a/src/wallet/sqlite.cpp +++ b/src/wallet/sqlite.cpp @@ -181,7 +181,9 @@ void SQLiteDatabase::Open() bool SQLiteDatabase::Rewrite(const char* skip) { - return false; + // Rewrite the database using the VACUUM command: https://sqlite.org/lang_vacuum.html + int ret = sqlite3_exec(m_db, "VACUUM", nullptr, nullptr, nullptr); + return ret == SQLITE_OK; } bool SQLiteDatabase::Backup(const std::string& dest) const -- cgit v1.2.3 From 727e6b2a4ee5abb7f2dcbc9f7778291908dc28ad Mon Sep 17 00:00:00 2001 From: Andrew Chow Date: Tue, 26 May 2020 20:54:00 -0400 Subject: Implement SQLiteDatabase::Verify --- src/wallet/sqlite.cpp | 52 ++++++++++++++++++++++++++++++++++++++++++++++++++- src/wallet/sqlite.h | 2 ++ 2 files changed, 53 insertions(+), 1 deletion(-) diff --git a/src/wallet/sqlite.cpp b/src/wallet/sqlite.cpp index f9c4dfb2a5..bfdd859162 100644 --- a/src/wallet/sqlite.cpp +++ b/src/wallet/sqlite.cpp @@ -109,6 +109,44 @@ void SQLiteDatabase::Cleanup() noexcept } } +bool SQLiteDatabase::Verify(bilingual_str& error) +{ + assert(m_db); + + sqlite3_stmt* stmt{nullptr}; + ret = sqlite3_prepare_v2(m_db, "PRAGMA integrity_check", -1, &stmt, nullptr); + if (ret != SQLITE_OK) { + sqlite3_finalize(stmt); + error = strprintf(_("SQLiteDatabase: Failed to prepare statement to verify database: %s"), sqlite3_errstr(ret)); + return false; + } + while (true) { + ret = sqlite3_step(stmt); + if (ret == SQLITE_DONE) { + break; + } + if (ret != SQLITE_ROW) { + error = strprintf(_("SQLiteDatabase: Failed to execute statement to verify database: %s"), sqlite3_errstr(ret)); + break; + } + const char* msg = (const char*)sqlite3_column_text(stmt, 0); + if (!msg) { + error = strprintf(_("SQLiteDatabase: Failed to read database verification error: %s"), sqlite3_errstr(ret)); + break; + } + std::string str_msg(msg); + if (str_msg == "ok") { + continue; + } + if (error.empty()) { + error = _("Failed to verify database") + Untranslated("\n"); + } + error += Untranslated(strprintf("%s\n", str_msg)); + } + sqlite3_finalize(stmt); + return error.empty(); +} + void SQLiteDatabase::Open() { int flags = SQLITE_OPEN_FULLMUTEX | SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE; @@ -468,7 +506,19 @@ bool ExistsSQLiteDatabase(const fs::path& path) std::unique_ptr MakeSQLiteDatabase(const fs::path& path, const DatabaseOptions& options, DatabaseStatus& status, bilingual_str& error) { - return MakeUnique(path, path / DATABASE_FILENAME); + const fs::path file = path / DATABASE_FILENAME; + try { + auto db = MakeUnique(path, file); + if (options.verify && !db->Verify(error)) { + status = DatabaseStatus::FAILED_VERIFY; + return nullptr; + } + return db; + } catch (const std::runtime_error& e) { + status = DatabaseStatus::FAILED_LOAD; + error.original = e.what(); + return nullptr; + } } std::string SQLiteDatabaseVersion() diff --git a/src/wallet/sqlite.h b/src/wallet/sqlite.h index dca6560abb..c6a3f7f503 100644 --- a/src/wallet/sqlite.h +++ b/src/wallet/sqlite.h @@ -71,6 +71,8 @@ public: ~SQLiteDatabase(); + bool Verify(bilingual_str& error); + /** Open the database if it is not already opened */ void Open() override; -- cgit v1.2.3 From 6045f77003f167bee9a85e2d53f8fc6ff2e297d8 Mon Sep 17 00:00:00 2001 From: Andrew Chow Date: Tue, 16 Jun 2020 15:38:12 -0400 Subject: Implement SQLiteDatabase::MakeBatch --- src/wallet/sqlite.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/wallet/sqlite.cpp b/src/wallet/sqlite.cpp index bfdd859162..5c30d72e84 100644 --- a/src/wallet/sqlite.cpp +++ b/src/wallet/sqlite.cpp @@ -262,7 +262,8 @@ void SQLiteDatabase::Close() std::unique_ptr SQLiteDatabase::MakeBatch(bool flush_on_close) { - return nullptr; + // We ignore flush_on_close because we don't do manual flushing for SQLite + return MakeUnique(*this); } SQLiteBatch::SQLiteBatch(SQLiteDatabase& database) -- cgit v1.2.3 From ac38a87225be0f1103ff9629d63980550d2f372b Mon Sep 17 00:00:00 2001 From: Andrew Chow Date: Tue, 26 May 2020 20:54:05 -0400 Subject: Determine wallet file type based on file magic --- src/wallet/bdb.cpp | 27 ++++++++++++++++++++++++++- src/wallet/bdb.h | 2 +- src/wallet/db.h | 3 +++ src/wallet/sqlite.cpp | 26 +++++++++++++++++++++++++- src/wallet/sqlite.h | 1 + src/wallet/walletdb.cpp | 23 +++++++++++++++++++++++ src/wallet/walletutil.cpp | 31 ++++--------------------------- 7 files changed, 83 insertions(+), 30 deletions(-) diff --git a/src/wallet/bdb.cpp b/src/wallet/bdb.cpp index ae3c7ae7bb..85aae0170d 100644 --- a/src/wallet/bdb.cpp +++ b/src/wallet/bdb.cpp @@ -813,7 +813,7 @@ bool ExistsBerkeleyDatabase(const fs::path& path) fs::path env_directory; std::string data_filename; SplitWalletPath(path, env_directory, data_filename); - return IsBerkeleyBtree(env_directory / data_filename); + return IsBDBFile(env_directory / data_filename); } std::unique_ptr MakeBerkeleyDatabase(const fs::path& path, const DatabaseOptions& options, DatabaseStatus& status, bilingual_str& error) @@ -839,3 +839,28 @@ std::unique_ptr MakeBerkeleyDatabase(const fs::path& path, con status = DatabaseStatus::SUCCESS; return db; } + +bool IsBDBFile(const fs::path& path) +{ + if (!fs::exists(path)) return false; + + // A Berkeley DB Btree file has at least 4K. + // This check also prevents opening lock files. + boost::system::error_code ec; + auto size = fs::file_size(path, ec); + if (ec) LogPrintf("%s: %s %s\n", __func__, ec.message(), path.string()); + if (size < 4096) return false; + + fsbridge::ifstream file(path, std::ios::binary); + if (!file.is_open()) return false; + + file.seekg(12, std::ios::beg); // Magic bytes start at offset 12 + uint32_t data = 0; + file.read((char*) &data, sizeof(data)); // Read 4 bytes of file to compare against magic + + // Berkeley DB Btree magic bytes, from: + // https://github.com/file/file/blob/5824af38469ec1ca9ac3ffd251e7afe9dc11e227/magic/Magdir/database#L74-L75 + // - big endian systems - 00 05 31 62 + // - little endian systems - 62 31 05 00 + return data == 0x00053162 || data == 0x62310500; +} diff --git a/src/wallet/bdb.h b/src/wallet/bdb.h index 070590872b..5403e95ee4 100644 --- a/src/wallet/bdb.h +++ b/src/wallet/bdb.h @@ -87,7 +87,7 @@ public: std::shared_ptr GetWalletEnv(const fs::path& wallet_path, std::string& database_filename); /** Check format of database file */ -bool IsBerkeleyBtree(const fs::path& path); +bool IsBDBFile(const fs::path& path); class BerkeleyBatch; diff --git a/src/wallet/db.h b/src/wallet/db.h index 0004fc1afa..3ecccd4e00 100644 --- a/src/wallet/db.h +++ b/src/wallet/db.h @@ -8,6 +8,7 @@ #include #include +#include #include #include #include @@ -194,11 +195,13 @@ public: enum class DatabaseFormat { BERKELEY, + SQLITE, }; struct DatabaseOptions { bool require_existing = false; bool require_create = false; + Optional require_format; uint64_t create_flags = 0; SecureString create_passphrase; bool verify = true; diff --git a/src/wallet/sqlite.cpp b/src/wallet/sqlite.cpp index 5c30d72e84..ce390440d9 100644 --- a/src/wallet/sqlite.cpp +++ b/src/wallet/sqlite.cpp @@ -502,7 +502,8 @@ bool SQLiteBatch::TxnAbort() bool ExistsSQLiteDatabase(const fs::path& path) { - return false; + const fs::path file = path / DATABASE_FILENAME; + return fs::symlink_status(file).type() == fs::regular_file && IsSQLiteFile(file); } std::unique_ptr MakeSQLiteDatabase(const fs::path& path, const DatabaseOptions& options, DatabaseStatus& status, bilingual_str& error) @@ -526,3 +527,26 @@ std::string SQLiteDatabaseVersion() { return std::string(sqlite3_libversion()); } + +bool IsSQLiteFile(const fs::path& path) +{ + if (!fs::exists(path)) return false; + + // A SQLite Database file is at least 512 bytes. + boost::system::error_code ec; + auto size = fs::file_size(path, ec); + if (ec) LogPrintf("%s: %s %s\n", __func__, ec.message(), path.string()); + if (size < 512) return false; + + fsbridge::ifstream file(path, std::ios::binary); + if (!file.is_open()) return false; + + // Magic is at beginning and is 16 bytes long + char magic[16]; + file.read(magic, 16); + file.close(); + + // Check the magic, see https://sqlite.org/fileformat2.html + std::string magic_str(magic); + return magic_str == std::string("SQLite format 3"); +} diff --git a/src/wallet/sqlite.h b/src/wallet/sqlite.h index c6a3f7f503..5e5e93903b 100644 --- a/src/wallet/sqlite.h +++ b/src/wallet/sqlite.h @@ -116,5 +116,6 @@ bool ExistsSQLiteDatabase(const fs::path& path); std::unique_ptr MakeSQLiteDatabase(const fs::path& path, const DatabaseOptions& options, DatabaseStatus& status, bilingual_str& error); std::string SQLiteDatabaseVersion(); +bool IsSQLiteFile(const fs::path& path); #endif // BITCOIN_WALLET_SQLITE_H diff --git a/src/wallet/walletdb.cpp b/src/wallet/walletdb.cpp index 5bf21eb91f..0092a29cb4 100644 --- a/src/wallet/walletdb.cpp +++ b/src/wallet/walletdb.cpp @@ -15,6 +15,7 @@ #include #include #include +#include #include #include @@ -1011,6 +1012,14 @@ std::unique_ptr MakeDatabase(const fs::path& path, const Databas if (ExistsBerkeleyDatabase(path)) { format = DatabaseFormat::BERKELEY; } + if (ExistsSQLiteDatabase(path)) { + if (format) { + error = Untranslated(strprintf("Failed to load database path '%s'. Data is in ambiguous format.", path.string())); + status = DatabaseStatus::FAILED_BAD_FORMAT; + return nullptr; + } + format = DatabaseFormat::SQLITE; + } } else if (options.require_existing) { error = Untranslated(strprintf("Failed to load database path '%s'. Path does not exist.", path.string())); status = DatabaseStatus::FAILED_NOT_FOUND; @@ -1029,6 +1038,20 @@ std::unique_ptr MakeDatabase(const fs::path& path, const Databas return nullptr; } + // A db already exists so format is set, but options also specifies the format, so make sure they agree + if (format && options.require_format && format != options.require_format) { + error = Untranslated(strprintf("Failed to load database path '%s'. Data is not in required format.", path.string())); + status = DatabaseStatus::FAILED_BAD_FORMAT; + return nullptr; + } + + // Format is not set when a db doesn't already exist, so use the format specified by the options if it is set. + if (!format && options.require_format) format = options.require_format; + + if (format && format == DatabaseFormat::SQLITE) { + return MakeSQLiteDatabase(path, options, status, error); + } + return MakeBerkeleyDatabase(path, options, status, error); } diff --git a/src/wallet/walletutil.cpp b/src/wallet/walletutil.cpp index e4c72aed98..23cdb8f64c 100644 --- a/src/wallet/walletutil.cpp +++ b/src/wallet/walletutil.cpp @@ -7,6 +7,8 @@ #include #include +bool ExistsBerkeleyDatabase(const fs::path& path); + fs::path GetWalletDir() { fs::path path; @@ -29,31 +31,6 @@ fs::path GetWalletDir() return path; } -bool IsBerkeleyBtree(const fs::path& path) -{ - if (!fs::exists(path)) return false; - - // A Berkeley DB Btree file has at least 4K. - // This check also prevents opening lock files. - boost::system::error_code ec; - auto size = fs::file_size(path, ec); - if (ec) LogPrintf("%s: %s %s\n", __func__, ec.message(), path.string()); - if (size < 4096) return false; - - fsbridge::ifstream file(path, std::ios::binary); - if (!file.is_open()) return false; - - file.seekg(12, std::ios::beg); // Magic bytes start at offset 12 - uint32_t data = 0; - file.read((char*) &data, sizeof(data)); // Read 4 bytes of file to compare against magic - - // Berkeley DB Btree magic bytes, from: - // https://github.com/file/file/blob/5824af38469ec1ca9ac3ffd251e7afe9dc11e227/magic/Magdir/database#L74-L75 - // - big endian systems - 00 05 31 62 - // - little endian systems - 62 31 05 00 - return data == 0x00053162 || data == 0x62310500; -} - std::vector ListWalletDir() { const fs::path wallet_dir = GetWalletDir(); @@ -71,10 +48,10 @@ std::vector ListWalletDir() // This can be replaced by boost::filesystem::lexically_relative once boost is bumped to 1.60. const fs::path path = it->path().string().substr(offset); - if (it->status().type() == fs::directory_file && IsBerkeleyBtree(it->path() / "wallet.dat")) { + if (it->status().type() == fs::directory_file && ExistsBerkeleyDatabase(it->path())) { // Found a directory which contains wallet.dat btree file, add it as a wallet. paths.emplace_back(path); - } else if (it.level() == 0 && it->symlink_status().type() == fs::regular_file && IsBerkeleyBtree(it->path())) { + } else if (it.level() == 0 && it->symlink_status().type() == fs::regular_file && ExistsBerkeleyDatabase(it->path())) { if (it->path().filename() == "wallet.dat") { // Found top-level wallet.dat btree file, add top level directory "" // as a wallet. -- cgit v1.2.3 From 9b78f3ce8ed1867c37f6b9fff98f74582d44b789 Mon Sep 17 00:00:00 2001 From: Andrew Chow Date: Thu, 11 Jun 2020 16:24:17 -0400 Subject: walletutil: Wallets can also be sqlite --- src/wallet/walletutil.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/wallet/walletutil.cpp b/src/wallet/walletutil.cpp index 23cdb8f64c..a2a55f9751 100644 --- a/src/wallet/walletutil.cpp +++ b/src/wallet/walletutil.cpp @@ -8,6 +8,7 @@ #include bool ExistsBerkeleyDatabase(const fs::path& path); +bool ExistsSQLiteDatabase(const fs::path& path); fs::path GetWalletDir() { @@ -48,7 +49,8 @@ std::vector ListWalletDir() // This can be replaced by boost::filesystem::lexically_relative once boost is bumped to 1.60. const fs::path path = it->path().string().substr(offset); - if (it->status().type() == fs::directory_file && ExistsBerkeleyDatabase(it->path())) { + if (it->status().type() == fs::directory_file && + (ExistsBerkeleyDatabase(it->path()) || ExistsSQLiteDatabase(it->path()))) { // Found a directory which contains wallet.dat btree file, add it as a wallet. paths.emplace_back(path); } else if (it.level() == 0 && it->symlink_status().type() == fs::regular_file && ExistsBerkeleyDatabase(it->path())) { -- cgit v1.2.3 From 9af5de3798c49f86f27bb79396e075fb8c1b2381 Mon Sep 17 00:00:00 2001 From: Andrew Chow Date: Tue, 26 May 2020 20:54:13 -0400 Subject: Use SQLite for descriptor wallets MakeWalletDatabase no longer has a default DatabaseFormat. Instead callers, like CWallet::Create, need to specify the database type to create if the file does not exist. If it exists and NONE is given, then CreateWalletDatabase will try to autodetect the type. --- src/wallet/wallet.cpp | 4 +++- src/wallet/wallet.h | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index dfcfaf489a..6b7d05fdf3 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -243,11 +243,13 @@ std::shared_ptr LoadWallet(interfaces::Chain& chain, const std::string& return wallet; } -std::shared_ptr CreateWallet(interfaces::Chain& chain, const std::string& name, Optional load_on_start, const DatabaseOptions& options, DatabaseStatus& status, bilingual_str& error, std::vector& warnings) +std::shared_ptr CreateWallet(interfaces::Chain& chain, const std::string& name, Optional load_on_start, DatabaseOptions& options, DatabaseStatus& status, bilingual_str& error, std::vector& warnings) { uint64_t wallet_creation_flags = options.create_flags; const SecureString& passphrase = options.create_passphrase; + if (wallet_creation_flags & WALLET_FLAG_DESCRIPTORS) options.require_format = DatabaseFormat::SQLITE; + // Indicate that the wallet is actually supposed to be blank and not just blank to make it encrypted bool create_blank = (wallet_creation_flags & WALLET_FLAG_BLANK_WALLET); diff --git a/src/wallet/wallet.h b/src/wallet/wallet.h index fb08cb4085..245144a1c9 100644 --- a/src/wallet/wallet.h +++ b/src/wallet/wallet.h @@ -55,7 +55,7 @@ bool RemoveWallet(const std::shared_ptr& wallet, Optional load_on std::vector> GetWallets(); std::shared_ptr GetWallet(const std::string& name); std::shared_ptr LoadWallet(interfaces::Chain& chain, const std::string& name, Optional load_on_start, const DatabaseOptions& options, DatabaseStatus& status, bilingual_str& error, std::vector& warnings); -std::shared_ptr CreateWallet(interfaces::Chain& chain, const std::string& name, Optional load_on_start, const DatabaseOptions& options, DatabaseStatus& status, bilingual_str& error, std::vector& warnings); +std::shared_ptr CreateWallet(interfaces::Chain& chain, const std::string& name, Optional load_on_start, DatabaseOptions& options, DatabaseStatus& status, bilingual_str& error, std::vector& warnings); std::unique_ptr HandleLoadWallet(LoadWalletFn load_wallet); std::unique_ptr MakeWalletDatabase(const std::string& name, const DatabaseOptions& options, DatabaseStatus& status, bilingual_str& error); -- cgit v1.2.3 From 9d3d2d263c331e3c77b8f0d01ecc9fea0407dd17 Mon Sep 17 00:00:00 2001 From: Andrew Chow Date: Wed, 9 Sep 2020 19:30:20 -0400 Subject: Use network magic as sqlite wallet application ID --- src/wallet/sqlite.cpp | 45 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 44 insertions(+), 1 deletion(-) diff --git a/src/wallet/sqlite.cpp b/src/wallet/sqlite.cpp index ce390440d9..473413084b 100644 --- a/src/wallet/sqlite.cpp +++ b/src/wallet/sqlite.cpp @@ -4,6 +4,8 @@ #include +#include +#include #include #include #include @@ -113,6 +115,28 @@ bool SQLiteDatabase::Verify(bilingual_str& error) { assert(m_db); + // Check the application ID matches our network magic + sqlite3_stmt* app_id_stmt{nullptr}; + int ret = sqlite3_prepare_v2(m_db, "PRAGMA application_id", -1, &app_id_stmt, nullptr); + if (ret != SQLITE_OK) { + sqlite3_finalize(app_id_stmt); + error = strprintf(_("SQLiteDatabase: Failed to prepare the statement to fetch the application id: %s"), sqlite3_errstr(ret)); + return false; + } + ret = sqlite3_step(app_id_stmt); + if (ret != SQLITE_ROW) { + sqlite3_finalize(app_id_stmt); + error = strprintf(_("SQLiteDatabase: Failed to fetch the application id: %s"), sqlite3_errstr(ret)); + return false; + } + uint32_t app_id = static_cast(sqlite3_column_int(app_id_stmt, 0)); + sqlite3_finalize(app_id_stmt); + uint32_t net_magic = ReadBE32(Params().MessageStart()); + if (app_id != net_magic) { + error = strprintf(_("SQLiteDatabase: Unexpected application id. Expected %u, got %u"), net_magic, app_id); + return false; + } + sqlite3_stmt* stmt{nullptr}; ret = sqlite3_prepare_v2(m_db, "PRAGMA integrity_check", -1, &stmt, nullptr); if (ret != SQLITE_OK) { @@ -214,6 +238,14 @@ void SQLiteDatabase::Open() if (ret != SQLITE_OK) { throw std::runtime_error(strprintf("SQLiteDatabase: Failed to create new database: %s\n", sqlite3_errstr(ret))); } + + // Set the application id + uint32_t app_id = ReadBE32(Params().MessageStart()); + std::string set_app_id = strprintf("PRAGMA application_id = %d", static_cast(app_id)); + ret = sqlite3_exec(m_db, set_app_id.c_str(), nullptr, nullptr, nullptr); + if (ret != SQLITE_OK) { + throw std::runtime_error(strprintf("SQLiteDatabase: Failed to set the application id: %s\n", sqlite3_errstr(ret))); + } } } @@ -544,9 +576,20 @@ bool IsSQLiteFile(const fs::path& path) // Magic is at beginning and is 16 bytes long char magic[16]; file.read(magic, 16); + + // Application id is at offset 68 and 4 bytes long + file.seekg(68, std::ios::beg); + char app_id[4]; + file.read(app_id, 4); + file.close(); // Check the magic, see https://sqlite.org/fileformat2.html std::string magic_str(magic); - return magic_str == std::string("SQLite format 3"); + if (magic_str != std::string("SQLite format 3")) { + return false; + } + + // Check the application id matches our network magic + return memcmp(Params().MessageStart(), app_id, 4) == 0; } -- cgit v1.2.3 From 6173269866306058fcb1cc825b9eb681838678ca Mon Sep 17 00:00:00 2001 From: Andrew Chow Date: Wed, 9 Sep 2020 20:22:32 -0400 Subject: Set and check the sqlite user version --- src/wallet/sqlite.cpp | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/src/wallet/sqlite.cpp b/src/wallet/sqlite.cpp index 473413084b..8a6b547feb 100644 --- a/src/wallet/sqlite.cpp +++ b/src/wallet/sqlite.cpp @@ -18,6 +18,7 @@ #include static const char* const DATABASE_FILENAME = "wallet.dat"; +static constexpr int32_t WALLET_SCHEMA_VERSION = 0; static Mutex g_sqlite_mutex; static int g_sqlite_count GUARDED_BY(g_sqlite_mutex) = 0; @@ -137,6 +138,27 @@ bool SQLiteDatabase::Verify(bilingual_str& error) return false; } + // Check our schema version + sqlite3_stmt* user_ver_stmt{nullptr}; + ret = sqlite3_prepare_v2(m_db, "PRAGMA user_version", -1, &user_ver_stmt, nullptr); + if (ret != SQLITE_OK) { + sqlite3_finalize(user_ver_stmt); + error = strprintf(_("SQLiteDatabase: Failed to prepare the statement to fetch sqlite wallet schema version: %s"), sqlite3_errstr(ret)); + return false; + } + ret = sqlite3_step(user_ver_stmt); + if (ret != SQLITE_ROW) { + sqlite3_finalize(user_ver_stmt); + error = strprintf(_("SQLiteDatabase: Failed to fetch sqlite wallet schema version: %s"), sqlite3_errstr(ret)); + return false; + } + int32_t user_ver = sqlite3_column_int(user_ver_stmt, 0); + sqlite3_finalize(user_ver_stmt); + if (user_ver != WALLET_SCHEMA_VERSION) { + error = strprintf(_("SQLiteDatabase: Unknown sqlite wallet schema version %d. Only version %d is supported"), user_ver, WALLET_SCHEMA_VERSION); + return false; + } + sqlite3_stmt* stmt{nullptr}; ret = sqlite3_prepare_v2(m_db, "PRAGMA integrity_check", -1, &stmt, nullptr); if (ret != SQLITE_OK) { @@ -246,6 +268,13 @@ void SQLiteDatabase::Open() if (ret != SQLITE_OK) { throw std::runtime_error(strprintf("SQLiteDatabase: Failed to set the application id: %s\n", sqlite3_errstr(ret))); } + + // Set the user version + std::string set_user_ver = strprintf("PRAGMA user_version = %d", WALLET_SCHEMA_VERSION); + ret = sqlite3_exec(m_db, set_user_ver.c_str(), nullptr, nullptr, nullptr); + if (ret != SQLITE_OK) { + throw std::runtime_error(strprintf("SQLiteDatabase: Failed to set the wallet schema version: %s\n", sqlite3_errstr(ret))); + } } } -- cgit v1.2.3 From f023b7cac0eb16d3c1bf40f1f7898b290de4cc73 Mon Sep 17 00:00:00 2001 From: Andrew Chow Date: Wed, 23 Sep 2020 12:16:40 -0400 Subject: wallet: Enforce sqlite serialized threading mode --- src/wallet/sqlite.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/wallet/sqlite.cpp b/src/wallet/sqlite.cpp index 8a6b547feb..02a161ecbd 100644 --- a/src/wallet/sqlite.cpp +++ b/src/wallet/sqlite.cpp @@ -48,6 +48,11 @@ SQLiteDatabase::SQLiteDatabase(const fs::path& dir_path, const fs::path& file_pa if (ret != SQLITE_OK) { throw std::runtime_error(strprintf("SQLiteDatabase: Failed to setup error log: %s\n", sqlite3_errstr(ret))); } + // Force serialized threading mode + ret = sqlite3_config(SQLITE_CONFIG_SERIALIZED); + if (ret != SQLITE_OK) { + throw std::runtime_error(strprintf("SQLiteDatabase: Failed to configure serialized threading mode: %s\n", sqlite3_errstr(ret))); + } } int ret = sqlite3_initialize(); // This is a no-op if sqlite3 is already initialized if (ret != SQLITE_OK) { -- cgit v1.2.3 From 6c6639ac9f6e1677da066cf809f9e3fa4d2e7c32 Mon Sep 17 00:00:00 2001 From: Andrew Chow Date: Wed, 30 Sep 2020 12:24:12 -0400 Subject: Include sqlite3 in documentation --- doc/build-osx.md | 4 ++-- doc/build-unix.md | 11 ++++++++++- doc/dependencies.md | 2 ++ doc/files.md | 3 ++- 4 files changed, 16 insertions(+), 4 deletions(-) diff --git a/doc/build-osx.md b/doc/build-osx.md index 7b76117c8b..2a7d71eea6 100644 --- a/doc/build-osx.md +++ b/doc/build-osx.md @@ -19,7 +19,7 @@ Then install [Homebrew](https://brew.sh). ## Dependencies ```shell -brew install automake berkeley-db4 libtool boost miniupnpc pkg-config python qt libevent qrencode +brew install automake berkeley-db4 libtool boost miniupnpc pkg-config python qt libevent qrencode sqlite ``` If you run into issues, check [Homebrew's troubleshooting page](https://docs.brew.sh/Troubleshooting). @@ -79,7 +79,7 @@ compiled in `disable-wallet` mode with: ./configure --disable-wallet ``` -In this case there is no dependency on Berkeley DB 4.8. +In this case there is no dependency on Berkeley DB 4.8 and SQLite. Mining is also possible in disable-wallet mode using the `getblocktemplate` RPC call. diff --git a/doc/build-unix.md b/doc/build-unix.md index 6b51db5f55..c076fb6fff 100644 --- a/doc/build-unix.md +++ b/doc/build-unix.md @@ -46,6 +46,7 @@ Optional dependencies: libqrencode | QR codes in GUI | Optional for generating QR codes (only needed when GUI enabled) univalue | Utility | JSON parsing and encoding (bundled version will be used unless --with-system-univalue passed to configure) libzmq3 | ZMQ notification | Optional, allows generating ZMQ notifications (requires ZMQ version >= 4.0.0) + sqlite3 | SQLite DB | Wallet storage (only needed when wallet enabled) For the versions used, see [dependencies.md](dependencies.md) @@ -91,6 +92,10 @@ pass `--with-incompatible-bdb` to configure. Otherwise, you can build from self-compiled `depends` (see above). +SQLite is required for the wallet: + + sudo apt install libsqlite3-dev + To build Bitcoin Core without wallet, see [*Disable-wallet mode*](/doc/build-unix.md#disable-wallet-mode) @@ -144,6 +149,10 @@ libqrencode (optional) can be installed with: sudo dnf install qrencode-devel +SQLite can be installed with: + + sudo dnf install sqlite-devel + Notes ----- The release is built with GCC and then "strip bitcoind" to strip the debug @@ -238,7 +247,7 @@ disable-wallet mode with: ./configure --disable-wallet -In this case there is no dependency on Berkeley DB 4.8. +In this case there is no dependency on Berkeley DB 4.8 and SQLite. Mining is also possible in disable-wallet mode using the `getblocktemplate` RPC call. diff --git a/doc/dependencies.md b/doc/dependencies.md index 92dea65309..ddd50ef296 100644 --- a/doc/dependencies.md +++ b/doc/dependencies.md @@ -21,6 +21,7 @@ These are the dependencies currently used by Bitcoin Core. You can find instruct | Python (tests) | | [3.5](https://www.python.org/downloads) | | | | | qrencode | [3.4.4](https://fukuchi.org/works/qrencode) | | No | | | | Qt | [5.9.8](https://download.qt.io/official_releases/qt/) | [5.5.1](https://github.com/bitcoin/bitcoin/issues/13478) | No | | | +| SQLite | [3.32.1](https://sqlite.org/download.html) | [3.7.17](https://github.com/bitcoin/bitcoin/pull/19077) | | | | | XCB | | | | | [Yes](https://github.com/bitcoin/bitcoin/blob/master/depends/packages/qt.mk) (Linux only) | | xkbcommon | | | | | [Yes](https://github.com/bitcoin/bitcoin/blob/master/depends/packages/qt.mk) (Linux only) | | ZeroMQ | [4.3.1](https://github.com/zeromq/libzmq/releases) | 4.0.0 | No | | | @@ -33,6 +34,7 @@ Some dependencies are not needed in all configurations. The following are some f #### Options passed to `./configure` * MiniUPnPc is not needed with `--with-miniupnpc=no`. * Berkeley DB is not needed with `--disable-wallet`. +* SQLite is not needed with `--disable-wallet`. * Qt is not needed with `--without-gui`. * If the qrencode dependency is absent, QR support won't be added. To force an error when that happens, pass `--with-qrencode`. * ZeroMQ is needed only with the `--with-zmq` option. diff --git a/doc/files.md b/doc/files.md index 64cff43d3f..e3f195de43 100644 --- a/doc/files.md +++ b/doc/files.md @@ -72,8 +72,9 @@ Subdirectory | File(s) | Description -------------|-------------------|------------ `database/` | BDB logging files | Part of BDB environment; created at start and deleted on shutdown; a user *must keep it as safe* as personal wallet `wallet.dat` `./` | `db.log` | BDB error file -`./` | `wallet.dat` | Personal wallet (BDB) with keys and transactions +`./` | `wallet.dat` | Personal wallet with keys and transactions. May be either a Berkeley DB or SQLite database file. `./` | `.walletlock` | Wallet lock file +`./` | `wallet.dat-journal` | SQLite Rollback Journal file for `wallet.dat`. Usually created at start and deleted on shutdown. A user *must keep it as safe* as the `wallet.dat` file. 1. Each user-defined wallet named "wallet_name" resides in `wallets/wallet_name/` subdirectory. -- cgit v1.2.3 From 310b0fde04639b7446efd5c1d2701caa4b991b86 Mon Sep 17 00:00:00 2001 From: Andrew Chow Date: Fri, 3 Apr 2020 18:49:36 -0400 Subject: Run dumpwallet for legacy wallets only in wallet_backup.py Descriptor wallets don't support dumpwallet, so make the tests that do dumpwallet legacy wallet only. --- test/functional/test_runner.py | 1 + test/functional/wallet_backup.py | 43 +++++++++++++++++++++------------------- 2 files changed, 24 insertions(+), 20 deletions(-) diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py index 2e757d7090..6bd4f20df9 100755 --- a/test/functional/test_runner.py +++ b/test/functional/test_runner.py @@ -88,6 +88,7 @@ BASE_SCRIPTS = [ 'wallet_hd.py', 'wallet_hd.py --descriptors', 'wallet_backup.py', + 'wallet_backup.py --descriptors', # vv Tests less than 5m vv 'mining_getblocktemplate_longpoll.py', 'feature_maxuploadtarget.py', diff --git a/test/functional/wallet_backup.py b/test/functional/wallet_backup.py index 36049dcb45..0cf8e6d926 100755 --- a/test/functional/wallet_backup.py +++ b/test/functional/wallet_backup.py @@ -135,11 +135,13 @@ class WalletBackupTest(BitcoinTestFramework): self.log.info("Backing up") self.nodes[0].backupwallet(os.path.join(self.nodes[0].datadir, 'wallet.bak')) - self.nodes[0].dumpwallet(os.path.join(self.nodes[0].datadir, 'wallet.dump')) self.nodes[1].backupwallet(os.path.join(self.nodes[1].datadir, 'wallet.bak')) - self.nodes[1].dumpwallet(os.path.join(self.nodes[1].datadir, 'wallet.dump')) self.nodes[2].backupwallet(os.path.join(self.nodes[2].datadir, 'wallet.bak')) - self.nodes[2].dumpwallet(os.path.join(self.nodes[2].datadir, 'wallet.dump')) + + if not self.options.descriptors: + self.nodes[0].dumpwallet(os.path.join(self.nodes[0].datadir, 'wallet.dump')) + self.nodes[1].dumpwallet(os.path.join(self.nodes[1].datadir, 'wallet.dump')) + self.nodes[2].dumpwallet(os.path.join(self.nodes[2].datadir, 'wallet.dump')) self.log.info("More transactions") for _ in range(5): @@ -183,29 +185,30 @@ class WalletBackupTest(BitcoinTestFramework): assert_equal(self.nodes[1].getbalance(), balance1) assert_equal(self.nodes[2].getbalance(), balance2) - self.log.info("Restoring using dumped wallet") - self.stop_three() - self.erase_three() + if not self.options.descriptors: + self.log.info("Restoring using dumped wallet") + self.stop_three() + self.erase_three() - #start node2 with no chain - shutil.rmtree(os.path.join(self.nodes[2].datadir, self.chain, 'blocks')) - shutil.rmtree(os.path.join(self.nodes[2].datadir, self.chain, 'chainstate')) + #start node2 with no chain + shutil.rmtree(os.path.join(self.nodes[2].datadir, self.chain, 'blocks')) + shutil.rmtree(os.path.join(self.nodes[2].datadir, self.chain, 'chainstate')) - self.start_three() + self.start_three() - assert_equal(self.nodes[0].getbalance(), 0) - assert_equal(self.nodes[1].getbalance(), 0) - assert_equal(self.nodes[2].getbalance(), 0) + assert_equal(self.nodes[0].getbalance(), 0) + assert_equal(self.nodes[1].getbalance(), 0) + assert_equal(self.nodes[2].getbalance(), 0) - self.nodes[0].importwallet(os.path.join(self.nodes[0].datadir, 'wallet.dump')) - self.nodes[1].importwallet(os.path.join(self.nodes[1].datadir, 'wallet.dump')) - self.nodes[2].importwallet(os.path.join(self.nodes[2].datadir, 'wallet.dump')) + self.nodes[0].importwallet(os.path.join(self.nodes[0].datadir, 'wallet.dump')) + self.nodes[1].importwallet(os.path.join(self.nodes[1].datadir, 'wallet.dump')) + self.nodes[2].importwallet(os.path.join(self.nodes[2].datadir, 'wallet.dump')) - self.sync_blocks() + self.sync_blocks() - assert_equal(self.nodes[0].getbalance(), balance0) - assert_equal(self.nodes[1].getbalance(), balance1) - assert_equal(self.nodes[2].getbalance(), balance2) + assert_equal(self.nodes[0].getbalance(), balance0) + assert_equal(self.nodes[1].getbalance(), balance1) + assert_equal(self.nodes[2].getbalance(), balance2) # Backup to source wallet file must fail sourcePaths = [ -- cgit v1.2.3 From c4a29d0a90b821c443c10891d9326c534d15cf97 Mon Sep 17 00:00:00 2001 From: Russell Yanofsky Date: Wed, 7 Oct 2020 14:50:05 -0400 Subject: Update wallet_multiwallet.py for descriptor and sqlite wallets --- test/functional/test_framework/test_framework.py | 2 +- test/functional/test_runner.py | 1 + test/functional/wallet_multiwallet.py | 35 ++++++++++++++++-------- 3 files changed, 26 insertions(+), 12 deletions(-) diff --git a/test/functional/test_framework/test_framework.py b/test/functional/test_framework/test_framework.py index 2824d80434..115ffa3e8d 100755 --- a/test/functional/test_framework/test_framework.py +++ b/test/functional/test_framework/test_framework.py @@ -103,7 +103,7 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): self.supports_cli = True self.bind_to_localhost_only = True self.parse_args() - self.default_wallet_name = "" + self.default_wallet_name = "default_wallet" if self.options.descriptors else "" self.wallet_data_filename = "wallet.dat" # Optional list of wallet names that can be set in set_test_params to # create and import keys to. If unset, default is len(nodes) * diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py index 6bd4f20df9..6b746b1fee 100755 --- a/test/functional/test_runner.py +++ b/test/functional/test_runner.py @@ -141,6 +141,7 @@ BASE_SCRIPTS = [ 'mempool_reorg.py', 'mempool_persist.py', 'wallet_multiwallet.py', + 'wallet_multiwallet.py --descriptors', 'wallet_multiwallet.py --usecli', 'wallet_createwallet.py', 'wallet_createwallet.py --usecli', diff --git a/test/functional/wallet_multiwallet.py b/test/functional/wallet_multiwallet.py index a0787dd289..61791a756c 100755 --- a/test/functional/wallet_multiwallet.py +++ b/test/functional/wallet_multiwallet.py @@ -60,8 +60,10 @@ class MultiWalletTest(BitcoinTestFramework): wallet = lambda name: node.get_wallet_rpc(name) def wallet_file(name): + if name == self.default_wallet_name: + return wallet_dir(self.default_wallet_name, self.wallet_data_filename) if os.path.isdir(wallet_dir(name)): - return wallet_dir(name, self.wallet_data_filename) + return wallet_dir(name, "wallet.dat") return wallet_dir(name) assert_equal(self.nodes[0].listwalletdir(), { 'wallets': [{ 'name': self.default_wallet_name }] }) @@ -77,13 +79,18 @@ class MultiWalletTest(BitcoinTestFramework): # rename wallet.dat to make sure plain wallet file paths (as opposed to # directory paths) can be loaded - os.rename(wallet_dir(self.default_wallet_name, self.wallet_data_filename), wallet_dir("w8")) - # create another dummy wallet for use in testing backups later - self.start_node(0, ["-nowallet", "-wallet=" + self.default_wallet_name]) + self.start_node(0, ["-nowallet", "-wallet=empty", "-wallet=plain"]) + node.createwallet("created") self.stop_nodes() empty_wallet = os.path.join(self.options.tmpdir, 'empty.dat') - os.rename(wallet_dir(self.default_wallet_name, self.wallet_data_filename), empty_wallet) + os.rename(wallet_file("empty"), empty_wallet) + shutil.rmtree(wallet_dir("empty")) + empty_created_wallet = os.path.join(self.options.tmpdir, 'empty.created.dat') + os.rename(wallet_dir("created", self.wallet_data_filename), empty_created_wallet) + shutil.rmtree(wallet_dir("created")) + os.rename(wallet_file("plain"), wallet_dir("w8")) + shutil.rmtree(wallet_dir("plain")) # restart node with a mix of wallet names: # w1, w2, w3 - to verify new wallets created when non-existing paths specified @@ -151,7 +158,7 @@ class MultiWalletTest(BitcoinTestFramework): competing_wallet_dir = os.path.join(self.options.tmpdir, 'competing_walletdir') os.mkdir(competing_wallet_dir) self.restart_node(0, ['-walletdir=' + competing_wallet_dir]) - exp_stderr = r"Error: Error initializing wallet database environment \"\S+competing_walletdir\"!" + exp_stderr = r"Error: Error initializing wallet database environment \"\S+competing_walletdir\S*\"!" self.nodes[1].assert_start_raises_init_error(['-walletdir=' + competing_wallet_dir], exp_stderr, match=ErrorMatch.PARTIAL_REGEX) self.restart_node(0, extra_args) @@ -246,12 +253,13 @@ class MultiWalletTest(BitcoinTestFramework): assert_raises_rpc_error(-18, "Wallet file verification failed. Failed to load database path '{}'. Path does not exist.".format(path), self.nodes[0].loadwallet, 'wallets') # Fail to load duplicate wallets - path = os.path.join(self.options.tmpdir, "node0", "regtest", "wallets", "w1", self.wallet_data_filename) + path = os.path.join(self.options.tmpdir, "node0", "regtest", "wallets", "w1", "wallet.dat") assert_raises_rpc_error(-4, "Wallet file verification failed. Refusing to load database. Data file '{}' is already loaded.".format(path), self.nodes[0].loadwallet, wallet_names[0]) # Fail to load duplicate wallets by different ways (directory and filepath) - path = os.path.join(self.options.tmpdir, "node0", "regtest", "wallets", self.wallet_data_filename) - assert_raises_rpc_error(-4, "Wallet file verification failed. Refusing to load database. Data file '{}' is already loaded.".format(path), self.nodes[0].loadwallet, self.wallet_data_filename) + if not self.options.descriptors: + path = os.path.join(self.options.tmpdir, "node0", "regtest", "wallets", "wallet.dat") + assert_raises_rpc_error(-4, "Wallet file verification failed. Refusing to load database. Data file '{}' is already loaded.".format(path), self.nodes[0].loadwallet, 'wallet.dat') # Fail to load if one wallet is a copy of another assert_raises_rpc_error(-4, "BerkeleyDatabase: Can't open database w8_copy (duplicates fileid", self.nodes[0].loadwallet, 'w8_copy') @@ -334,9 +342,11 @@ class MultiWalletTest(BitcoinTestFramework): rpc = self.nodes[0].get_wallet_rpc(wallet_name) addr = rpc.getnewaddress() backup = os.path.join(self.options.tmpdir, 'backup.dat') + if os.path.exists(backup): + os.unlink(backup) rpc.backupwallet(backup) self.nodes[0].unloadwallet(wallet_name) - shutil.copyfile(empty_wallet, wallet_file(wallet_name)) + shutil.copyfile(empty_created_wallet if wallet_name == self.default_wallet_name else empty_wallet, wallet_file(wallet_name)) self.nodes[0].loadwallet(wallet_name) assert_equal(rpc.getaddressinfo(addr)['ismine'], False) self.nodes[0].unloadwallet(wallet_name) @@ -348,7 +358,10 @@ class MultiWalletTest(BitcoinTestFramework): self.start_node(1) wallet = os.path.join(self.options.tmpdir, 'my_wallet') self.nodes[0].createwallet(wallet) - assert_raises_rpc_error(-4, "Error initializing wallet database environment", self.nodes[1].loadwallet, wallet) + if self.options.descriptors: + assert_raises_rpc_error(-4, "Unable to obtain an exclusive lock", self.nodes[1].loadwallet, wallet) + else: + assert_raises_rpc_error(-4, "Error initializing wallet database environment", self.nodes[1].loadwallet, wallet) self.nodes[0].unloadwallet(wallet) self.nodes[1].loadwallet(wallet) -- cgit v1.2.3