aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorSamuel Dobson <dobsonsa68@gmail.com>2020-10-15 19:20:18 +1300
committerSamuel Dobson <dobsonsa68@gmail.com>2020-10-15 20:12:29 +1300
commit8ed37f6c8497c8ec9e919dab35f63201e5f75f11 (patch)
tree97da615b4c20737dfb09a06172b7416ab2470607
parentf2e6d14430137a271d153348d207df6ab8086bc6 (diff)
parentc4a29d0a90b821c443c10891d9326c534d15cf97 (diff)
downloadbitcoin-8ed37f6c8497c8ec9e919dab35f63201e5f75f11.tar.xz
Merge #19077: wallet: Add sqlite as an alternative wallet database and use it for new descriptor wallets
c4a29d0a90b821c443c10891d9326c534d15cf97 Update wallet_multiwallet.py for descriptor and sqlite wallets (Russell Yanofsky) 310b0fde04639b7446efd5c1d2701caa4b991b86 Run dumpwallet for legacy wallets only in wallet_backup.py (Andrew Chow) 6c6639ac9f6e1677da066cf809f9e3fa4d2e7c32 Include sqlite3 in documentation (Andrew Chow) f023b7cac0eb16d3c1bf40f1f7898b290de4cc73 wallet: Enforce sqlite serialized threading mode (Andrew Chow) 6173269866306058fcb1cc825b9eb681838678ca Set and check the sqlite user version (Andrew Chow) 9d3d2d263c331e3c77b8f0d01ecc9fea0407dd17 Use network magic as sqlite wallet application ID (Andrew Chow) 9af5de3798c49f86f27bb79396e075fb8c1b2381 Use SQLite for descriptor wallets (Andrew Chow) 9b78f3ce8ed1867c37f6b9fff98f74582d44b789 walletutil: Wallets can also be sqlite (Andrew Chow) ac38a87225be0f1103ff9629d63980550d2f372b Determine wallet file type based on file magic (Andrew Chow) 6045f77003f167bee9a85e2d53f8fc6ff2e297d8 Implement SQLiteDatabase::MakeBatch (Andrew Chow) 727e6b2a4ee5abb7f2dcbc9f7778291908dc28ad Implement SQLiteDatabase::Verify (Andrew Chow) b4df8fdb19fcded7e6d491ecf0b705cac0ec76a1 Implement SQLiteDatabase::Rewrite (Andrew Chow) 010e3659069e6f97dd7b24483f50ed71042b84b0 Implement SQLiteDatabase::TxnBegin, TxnCommit, and TxnAbort (Andrew Chow) ac5c1617e7f4273daf24c24da1f6bc5ef5ab2d2b Implement SQLiteDatabase::Backup (Andrew Chow) f6f9cd6a64842ef23777312f2465e826ca04b886 Implement SQLiteBatch::StartCursor, ReadAtCursor, and CloseCursor (Andrew Chow) bf90e033f4fe86cfb90492c7e0962278ea3a146d Implement SQLiteBatch::ReadKey, WriteKey, EraseKey, and HasKey (Andrew Chow) 7aa45620e2f2178145a2eca58ccbab3cecff08fb Add SetupSQLStatements (Andrew Chow) 6636a2608a4e5906ee8092d5731595542261e0ad Implement SQLiteBatch::Close (Andrew Chow) 93825352a36456283bf87e39b5888363ee242f21 Implement SQLiteDatabase::Close (Andrew Chow) a0de83372be83f59015cd3d61af2303b74fb64b5 Implement SQLiteDatabase::Open (Andrew Chow) 3bfa0fe1259280f8c32b41a798c9453b73f89b02 Initialize and Shutdown sqlite3 globals (Andrew Chow) 5a488b3d77326a0d957c1233493061da1b6ec207 Constructors, destructors, and relevant private fields for SQLiteDatabase/Batch (Andrew Chow) ca8b7e04ab89f99075b093fa248919fd10acbdf7 Implement SQLiteDatabaseVersion (Andrew Chow) 7577b6e1c88a1a7b45ecf5c7f1735bae6f5a82bf Add SQLiteDatabase and SQLiteBatch dummy classes (Andrew Chow) e87df8258090138d5c22ac46b8602b618620e8a1 Add sqlite to travis and depends (Andrew Chow) 54729f3f4e6765dfded590af5fb28c88331685f8 Add libsqlite3 (Andrew Chow) Pull request description: This PR adds a new class `SQLiteDatabase` which is a subclass of `WalletDatabase`. This provides access to a SQLite database that is used to store the wallet records. To keep compatibility with BDB and to complexity of the change down, we don't make use of many SQLite's features. We use it strictly as a key-value store. We create a table `main` which has two columns, `key` and `value` both with the type `blob`. For new descriptor wallets, we will create a `SQLiteDatabase` instead of a `BerkeleyDatabase`. There is no requirement that all SQLite wallets are descriptor wallets, nor is there a requirement that all descriptor wallets be SQLite wallets. This allows for existing descriptor wallets to work as well as keeping open the option to migrate existing wallets to SQLite. We keep the name `wallet.dat` for SQLite wallets. We are able to determine which database type to use by searching for specific magic bytes in the `wallet.dat` file. SQLite begins it's files with a null terminated string `SQLite format 3`. BDB has `0x00053162` at byte 12 (note that the byte order of this integer depends on the system endianness). So when we see that there is a `wallet.dat` file that we want to open, we check for the magic bytes to determine which database system to use. I decided to keep the `wallet.dat` naming to keep things like backup script to continue to function as they won't need to be modified to look for a different file name. It also simplifies a couple of things in the implementation and the tests as `wallet.dat` is something that is specifically being looked for. If we don't want this behavior, then I do have another branch which creates `wallet.sqlite` files instead, but I find that this direction is easier. ACKs for top commit: Sjors: re-utACK c4a29d0a90b821c443c10891d9326c534d15cf97 promag: Tested ACK c4a29d0a90b821c443c10891d9326c534d15cf97. fjahr: reACK c4a29d0a90b821c443c10891d9326c534d15cf97 S3RK: Re-review ACK c4a29d0a90b821c443c10891d9326c534d15cf97 meshcollider: re-utACK c4a29d0a90b821c443c10891d9326c534d15cf97 hebasto: re-ACK c4a29d0a90b821c443c10891d9326c534d15cf97, only rebased since my [previous](https://github.com/bitcoin/bitcoin/pull/19077#pullrequestreview-507743699) review, verified with `git range-diff master d18892dcc c4a29d0a9`. ryanofsky: Code review ACK c4a29d0a90b821c443c10891d9326c534d15cf97. I am honestly confused about reasons for locking into `wallet.dat` again when it's so easy now to use a clean format. I assume I'm just very dense, or there's some unstated reason, because the only thing that's been brought up are unrealistic compatibility scenarios (all require actively creating a wallet with non-default descriptor+sqlite option, then trying to using the descriptor+sqlite wallets with old software or scripts and ignoring the results) that we didn't pay attention to with previous PRs like #11687, which did not require any active interfaction. jonatack: ACK c4a29d0a90b821c443c10891d9326c534d15cf97, debug builds and test runs after rebase to latest master @ c2c4dbaebd9, some manual testing creating, using, unloading and reloading a few different new sqlite descriptor wallets over several node restarts/shutdowns. Tree-SHA512: 19145732e5001484947352d3175a660b5102bc6e833f227a55bd41b9b2f4d92737bbed7cead64b75b509decf9e1408cd81c185ab1fb4b90561aee427c4f9751c
-rw-r--r--.travis.yml1
-rw-r--r--build_msvc/vcpkg.json3
-rw-r--r--ci/test/00_setup_env_native_asan.sh2
-rw-r--r--ci/test/00_setup_env_native_msan.sh2
-rw-r--r--ci/test/00_setup_env_native_valgrind.sh2
-rw-r--r--configure.ac4
-rw-r--r--contrib/gitian-descriptors/gitian-linux.yml2
-rw-r--r--contrib/gitian-descriptors/gitian-win.yml2
-rw-r--r--depends/Makefile5
-rw-r--r--depends/README.md4
-rw-r--r--depends/packages/packages.mk3
-rw-r--r--depends/packages/sqlite.mk26
-rw-r--r--doc/build-osx.md4
-rw-r--r--doc/build-unix.md11
-rw-r--r--doc/dependencies.md2
-rw-r--r--doc/files.md3
-rw-r--r--src/Makefile.am4
-rw-r--r--src/Makefile.bench.include2
-rw-r--r--src/Makefile.qt.include2
-rw-r--r--src/Makefile.qttest.include2
-rw-r--r--src/Makefile.test.include2
-rw-r--r--src/wallet/bdb.cpp27
-rw-r--r--src/wallet/bdb.h2
-rw-r--r--src/wallet/db.h3
-rw-r--r--src/wallet/sqlite.cpp629
-rw-r--r--src/wallet/sqlite.h121
-rw-r--r--src/wallet/wallet.cpp4
-rw-r--r--src/wallet/wallet.h2
-rw-r--r--src/wallet/walletdb.cpp23
-rw-r--r--src/wallet/walletutil.cpp33
-rwxr-xr-xtest/functional/test_framework/test_framework.py2
-rwxr-xr-xtest/functional/test_runner.py2
-rwxr-xr-xtest/functional/wallet_backup.py43
-rwxr-xr-xtest/functional/wallet_multiwallet.py35
34 files changed, 934 insertions, 80 deletions
diff --git a/.travis.yml b/.travis.yml
index b1bbfb091d..42fa653d8c 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -132,6 +132,7 @@ jobs:
- berkeley-db4
- miniupnpc
- qrencode
+ - sqlite
- ccache
- zeromq
env: >-
diff --git a/build_msvc/vcpkg.json b/build_msvc/vcpkg.json
index 5d5f94227f..dfd3929c44 100644
--- a/build_msvc/vcpkg.json
+++ b/build_msvc/vcpkg.json
@@ -9,6 +9,7 @@
"boost-signals2",
"boost-test",
"boost-thread",
+ "sqlite3",
"double-conversion",
{
"name": "libevent",
@@ -16,4 +17,4 @@
},
"zeromq"
]
-} \ No newline at end of file
+}
diff --git a/ci/test/00_setup_env_native_asan.sh b/ci/test/00_setup_env_native_asan.sh
index 251ece7984..191b8049b0 100644
--- a/ci/test/00_setup_env_native_asan.sh
+++ b/ci/test/00_setup_env_native_asan.sh
@@ -7,7 +7,7 @@
export LC_ALL=C.UTF-8
export CONTAINER_NAME=ci_native_asan
-export PACKAGES="clang llvm python3-zmq qtbase5-dev qttools5-dev-tools libevent-dev bsdmainutils libboost-system-dev libboost-filesystem-dev libboost-test-dev libboost-thread-dev libdb5.3++-dev libminiupnpc-dev libzmq3-dev libqrencode-dev"
+export PACKAGES="clang llvm python3-zmq qtbase5-dev qttools5-dev-tools libevent-dev bsdmainutils libboost-system-dev libboost-filesystem-dev libboost-test-dev libboost-thread-dev libdb5.3++-dev libminiupnpc-dev libzmq3-dev libqrencode-dev libsqlite3-dev"
export DOCKER_NAME_TAG=ubuntu:20.04
export NO_DEPENDS=1
export GOAL="install"
diff --git a/ci/test/00_setup_env_native_msan.sh b/ci/test/00_setup_env_native_msan.sh
index 6a4979990b..b88ee2b50f 100644
--- a/ci/test/00_setup_env_native_msan.sh
+++ b/ci/test/00_setup_env_native_msan.sh
@@ -15,7 +15,7 @@ export BDB_PREFIX="${BASE_ROOT_DIR}/db4"
export CONTAINER_NAME="ci_native_msan"
export PACKAGES="clang-9 llvm-9 cmake"
-export DEP_OPTS="NO_WALLET=1 NO_QT=1 CC='clang' CXX='clang++' CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}' boost_cxxflags='-std=c++11 -fvisibility=hidden -fPIC ${MSAN_AND_LIBCXX_FLAGS}' zeromq_cxxflags='-std=c++11 ${MSAN_AND_LIBCXX_FLAGS}'"
+export DEP_OPTS="NO_BDB=1 NO_QT=1 CC='clang' CXX='clang++' CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}' boost_cxxflags='-std=c++11 -fvisibility=hidden -fPIC ${MSAN_AND_LIBCXX_FLAGS}' zeromq_cxxflags='-std=c++11 ${MSAN_AND_LIBCXX_FLAGS}'"
export GOAL="install"
export BITCOIN_CONFIG="--enable-wallet --with-sanitizers=memory --with-asm=no --prefix=${BASE_ROOT_DIR}/depends/x86_64-pc-linux-gnu/ CC=clang CXX=clang++ CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}' BDB_LIBS='-L${BDB_PREFIX}/lib -ldb_cxx-4.8' BDB_CFLAGS='-I${BDB_PREFIX}/include'"
export USE_MEMORY_SANITIZER="true"
diff --git a/ci/test/00_setup_env_native_valgrind.sh b/ci/test/00_setup_env_native_valgrind.sh
index 710d9e1011..bfaea13a25 100644
--- a/ci/test/00_setup_env_native_valgrind.sh
+++ b/ci/test/00_setup_env_native_valgrind.sh
@@ -7,7 +7,7 @@
export LC_ALL=C.UTF-8
export CONTAINER_NAME=ci_native_valgrind
-export PACKAGES="valgrind clang llvm python3-zmq libevent-dev bsdmainutils libboost-system-dev libboost-filesystem-dev libboost-test-dev libboost-thread-dev libdb5.3++-dev libminiupnpc-dev libzmq3-dev"
+export PACKAGES="valgrind clang llvm python3-zmq libevent-dev bsdmainutils libboost-system-dev libboost-filesystem-dev libboost-test-dev libboost-thread-dev libdb5.3++-dev libminiupnpc-dev libzmq3-dev libsqlite3-dev"
export USE_VALGRIND=1
export NO_DEPENDS=1
export TEST_RUNNER_EXTRA="--exclude rpc_bind" # Excluded for now, see https://github.com/bitcoin/bitcoin/issues/17765#issuecomment-602068547
diff --git a/configure.ac b/configure.ac
index 76c6487f08..6db9dab404 100644
--- a/configure.ac
+++ b/configure.ac
@@ -1222,6 +1222,9 @@ if test x$enable_wallet != xno; then
if test x$suppress_external_warnings != xno ; then
BDB_CPPFLAGS=SUPPRESS_WARNINGS($BDB_CPPFLAGS)
fi
+
+ dnl Check for sqlite3
+ PKG_CHECK_MODULES([SQLITE], [sqlite3 >= 3.7.17], , [AC_MSG_ERROR([sqlite3 not found.])])
fi
dnl Check for libminiupnpc (optional)
@@ -1643,6 +1646,7 @@ AC_SUBST(LIBTOOL_APP_LDFLAGS)
AC_SUBST(USE_UPNP)
AC_SUBST(USE_QRCODE)
AC_SUBST(BOOST_LIBS)
+AC_SUBST(SQLITE_LIBS)
AC_SUBST(TESTDEFS)
AC_SUBST(MINIUPNPC_CPPFLAGS)
AC_SUBST(MINIUPNPC_LIBS)
diff --git a/contrib/gitian-descriptors/gitian-linux.yml b/contrib/gitian-descriptors/gitian-linux.yml
index e86ff83798..65f9a2e5c9 100644
--- a/contrib/gitian-descriptors/gitian-linux.yml
+++ b/contrib/gitian-descriptors/gitian-linux.yml
@@ -78,7 +78,7 @@ script: |
echo "REAL=\`which -a ${i}-${prog}-8 | grep -v ${WRAP_DIR}/${i}-${prog} | head -1\`" >> ${WRAP_DIR}/${i}-${prog}
echo "export LD_PRELOAD='/usr/\$LIB/faketime/libfaketime.so.1'" >> ${WRAP_DIR}/${i}-${prog}
echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${i}-${prog}
- echo "\$REAL \$@" >> $WRAP_DIR/${i}-${prog}
+ echo "\$REAL \"\$@\"" >> $WRAP_DIR/${i}-${prog}
chmod +x ${WRAP_DIR}/${i}-${prog}
fi
done
diff --git a/contrib/gitian-descriptors/gitian-win.yml b/contrib/gitian-descriptors/gitian-win.yml
index d05b6d426d..5f671b95ce 100644
--- a/contrib/gitian-descriptors/gitian-win.yml
+++ b/contrib/gitian-descriptors/gitian-win.yml
@@ -81,7 +81,7 @@ script: |
echo "REAL=\`which -a ${i}-${prog}-posix | grep -v ${WRAP_DIR}/${i}-${prog} | head -1\`" >> ${WRAP_DIR}/${i}-${prog}
echo "export LD_PRELOAD='/usr/\$LIB/faketime/libfaketime.so.1'" >> ${WRAP_DIR}/${i}-${prog}
echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${i}-${prog}
- echo "\$REAL \$@" >> $WRAP_DIR/${i}-${prog}
+ echo "\$REAL \"\$@\"" >> $WRAP_DIR/${i}-${prog}
chmod +x ${WRAP_DIR}/${i}-${prog}
done
done
diff --git a/depends/Makefile b/depends/Makefile
index 2bc5df974a..1ad21f6821 100644
--- a/depends/Makefile
+++ b/depends/Makefile
@@ -134,7 +134,10 @@ qrencode_packages_$(NO_QR) = $(qrencode_packages)
qt_packages_$(NO_QT) = $(qt_packages) $(qt_$(host_os)_packages) $(qt_$(host_arch)_$(host_os)_packages) $(qrencode_packages_)
-wallet_packages_$(NO_WALLET) = $(wallet_packages)
+bdb_packages_$(NO_BDB) = $(bdb_packages)
+sqlite_packages_$(NO_SQLITE) = $(sqlite_packages)
+wallet_packages_$(NO_WALLET) = $(bdb_packages_) $(sqlite_packages_)
+
upnp_packages_$(NO_UPNP) = $(upnp_packages)
zmq_packages_$(NO_ZMQ) = $(zmq_packages)
multiprocess_packages_$(MULTIPROCESS) = $(multiprocess_packages)
diff --git a/depends/README.md b/depends/README.md
index 2356e8be59..869ebe12fd 100644
--- a/depends/README.md
+++ b/depends/README.md
@@ -99,6 +99,10 @@ The following can be set when running make: `make FOO=bar`
<dd>Don't download/build/cache packages needed for enabling zeromq</dd>
<dt>NO_WALLET</dt>
<dd>Don't download/build/cache libs needed to enable the wallet</dd>
+<dt>NO_BDB</dt>
+<dd>Don't download/build/cache BerkeleyDB</dd>
+<dt>NO_SQLITE</dt>
+<dd>Don't download/build/cache SQLite</dd>
<dt>NO_UPNP</dt>
<dd>Don't download/build/cache packages needed for enabling upnp</dd>
<dt>MULTIPROCESS</dt>
diff --git a/depends/packages/packages.mk b/depends/packages/packages.mk
index 8fe2c771c9..4627acb521 100644
--- a/depends/packages/packages.mk
+++ b/depends/packages/packages.mk
@@ -10,7 +10,8 @@ qt_android_packages=qt
qt_darwin_packages=qt
qt_mingw32_packages=qt
-wallet_packages=bdb
+bdb_packages=bdb
+sqlite_packages=sqlite
zmq_packages=zeromq
diff --git a/depends/packages/sqlite.mk b/depends/packages/sqlite.mk
new file mode 100644
index 0000000000..5b3a61b239
--- /dev/null
+++ b/depends/packages/sqlite.mk
@@ -0,0 +1,26 @@
+package=sqlite
+$(package)_version=3320100
+$(package)_download_path=https://sqlite.org/2020/
+$(package)_file_name=sqlite-autoconf-$($(package)_version).tar.gz
+$(package)_sha256_hash=486748abfb16abd8af664e3a5f03b228e5f124682b0c942e157644bf6fff7d10
+
+define $(package)_set_vars
+$(package)_config_opts=--disable-shared --disable-readline --disable-dynamic-extensions --enable-option-checking
+$(package)_config_opts_linux=--with-pic
+endef
+
+define $(package)_config_cmds
+ $($(package)_autoconf)
+endef
+
+define $(package)_build_cmds
+ $(MAKE) libsqlite3.la
+endef
+
+define $(package)_stage_cmds
+ $(MAKE) DESTDIR=$($(package)_staging_dir) install-libLTLIBRARIES install-includeHEADERS install-pkgconfigDATA
+endef
+
+define $(package)_postprocess_cmds
+ rm lib/*.la
+endef
diff --git a/doc/build-osx.md b/doc/build-osx.md
index 7b76117c8b..2a7d71eea6 100644
--- a/doc/build-osx.md
+++ b/doc/build-osx.md
@@ -19,7 +19,7 @@ Then install [Homebrew](https://brew.sh).
## Dependencies
```shell
-brew install automake berkeley-db4 libtool boost miniupnpc pkg-config python qt libevent qrencode
+brew install automake berkeley-db4 libtool boost miniupnpc pkg-config python qt libevent qrencode sqlite
```
If you run into issues, check [Homebrew's troubleshooting page](https://docs.brew.sh/Troubleshooting).
@@ -79,7 +79,7 @@ compiled in `disable-wallet` mode with:
./configure --disable-wallet
```
-In this case there is no dependency on Berkeley DB 4.8.
+In this case there is no dependency on Berkeley DB 4.8 and SQLite.
Mining is also possible in disable-wallet mode using the `getblocktemplate` RPC call.
diff --git a/doc/build-unix.md b/doc/build-unix.md
index 6b51db5f55..c076fb6fff 100644
--- a/doc/build-unix.md
+++ b/doc/build-unix.md
@@ -46,6 +46,7 @@ Optional dependencies:
libqrencode | QR codes in GUI | Optional for generating QR codes (only needed when GUI enabled)
univalue | Utility | JSON parsing and encoding (bundled version will be used unless --with-system-univalue passed to configure)
libzmq3 | ZMQ notification | Optional, allows generating ZMQ notifications (requires ZMQ version >= 4.0.0)
+ sqlite3 | SQLite DB | Wallet storage (only needed when wallet enabled)
For the versions used, see [dependencies.md](dependencies.md)
@@ -91,6 +92,10 @@ pass `--with-incompatible-bdb` to configure.
Otherwise, you can build from self-compiled `depends` (see above).
+SQLite is required for the wallet:
+
+ sudo apt install libsqlite3-dev
+
To build Bitcoin Core without wallet, see [*Disable-wallet mode*](/doc/build-unix.md#disable-wallet-mode)
@@ -144,6 +149,10 @@ libqrencode (optional) can be installed with:
sudo dnf install qrencode-devel
+SQLite can be installed with:
+
+ sudo dnf install sqlite-devel
+
Notes
-----
The release is built with GCC and then "strip bitcoind" to strip the debug
@@ -238,7 +247,7 @@ disable-wallet mode with:
./configure --disable-wallet
-In this case there is no dependency on Berkeley DB 4.8.
+In this case there is no dependency on Berkeley DB 4.8 and SQLite.
Mining is also possible in disable-wallet mode using the `getblocktemplate` RPC call.
diff --git a/doc/dependencies.md b/doc/dependencies.md
index 92dea65309..ddd50ef296 100644
--- a/doc/dependencies.md
+++ b/doc/dependencies.md
@@ -21,6 +21,7 @@ These are the dependencies currently used by Bitcoin Core. You can find instruct
| Python (tests) | | [3.5](https://www.python.org/downloads) | | | |
| qrencode | [3.4.4](https://fukuchi.org/works/qrencode) | | No | | |
| Qt | [5.9.8](https://download.qt.io/official_releases/qt/) | [5.5.1](https://github.com/bitcoin/bitcoin/issues/13478) | No | | |
+| SQLite | [3.32.1](https://sqlite.org/download.html) | [3.7.17](https://github.com/bitcoin/bitcoin/pull/19077) | | | |
| XCB | | | | | [Yes](https://github.com/bitcoin/bitcoin/blob/master/depends/packages/qt.mk) (Linux only) |
| xkbcommon | | | | | [Yes](https://github.com/bitcoin/bitcoin/blob/master/depends/packages/qt.mk) (Linux only) |
| ZeroMQ | [4.3.1](https://github.com/zeromq/libzmq/releases) | 4.0.0 | No | | |
@@ -33,6 +34,7 @@ Some dependencies are not needed in all configurations. The following are some f
#### Options passed to `./configure`
* MiniUPnPc is not needed with `--with-miniupnpc=no`.
* Berkeley DB is not needed with `--disable-wallet`.
+* SQLite is not needed with `--disable-wallet`.
* Qt is not needed with `--without-gui`.
* If the qrencode dependency is absent, QR support won't be added. To force an error when that happens, pass `--with-qrencode`.
* ZeroMQ is needed only with the `--with-zmq` option.
diff --git a/doc/files.md b/doc/files.md
index 64cff43d3f..e3f195de43 100644
--- a/doc/files.md
+++ b/doc/files.md
@@ -72,8 +72,9 @@ Subdirectory | File(s) | Description
-------------|-------------------|------------
`database/` | BDB logging files | Part of BDB environment; created at start and deleted on shutdown; a user *must keep it as safe* as personal wallet `wallet.dat`
`./` | `db.log` | BDB error file
-`./` | `wallet.dat` | Personal wallet (BDB) with keys and transactions
+`./` | `wallet.dat` | Personal wallet with keys and transactions. May be either a Berkeley DB or SQLite database file.
`./` | `.walletlock` | Wallet lock file
+`./` | `wallet.dat-journal` | SQLite Rollback Journal file for `wallet.dat`. Usually created at start and deleted on shutdown. A user *must keep it as safe* as the `wallet.dat` file.
1. Each user-defined wallet named "wallet_name" resides in `wallets/wallet_name/` subdirectory.
diff --git a/src/Makefile.am b/src/Makefile.am
index 1dd821f968..b0d36717ce 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -257,6 +257,7 @@ BITCOIN_CORE_H = \
wallet/rpcwallet.h \
wallet/salvage.h \
wallet/scriptpubkeyman.h \
+ wallet/sqlite.h \
wallet/wallet.h \
wallet/walletdb.h \
wallet/wallettool.h \
@@ -371,6 +372,7 @@ libbitcoin_wallet_a_SOURCES = \
wallet/rpcwallet.cpp \
wallet/salvage.cpp \
wallet/scriptpubkeyman.cpp \
+ wallet/sqlite.cpp \
wallet/wallet.cpp \
wallet/walletdb.cpp \
wallet/walletutil.cpp \
@@ -590,7 +592,7 @@ bitcoin_bin_ldadd = \
$(LIBMEMENV) \
$(LIBSECP256K1)
-bitcoin_bin_ldadd += $(BOOST_LIBS) $(BDB_LIBS) $(MINIUPNPC_LIBS) $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) $(ZMQ_LIBS)
+bitcoin_bin_ldadd += $(BOOST_LIBS) $(BDB_LIBS) $(MINIUPNPC_LIBS) $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) $(ZMQ_LIBS) $(SQLITE_LIBS)
bitcoind_SOURCES = $(bitcoin_daemon_sources)
bitcoind_CPPFLAGS = $(bitcoin_bin_cppflags)
diff --git a/src/Makefile.bench.include b/src/Makefile.bench.include
index bd9143a381..beb3f8dfd2 100644
--- a/src/Makefile.bench.include
+++ b/src/Makefile.bench.include
@@ -74,7 +74,7 @@ bench_bench_bitcoin_SOURCES += bench/coin_selection.cpp
bench_bench_bitcoin_SOURCES += bench/wallet_balance.cpp
endif
-bench_bench_bitcoin_LDADD += $(BOOST_LIBS) $(BDB_LIBS) $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) $(MINIUPNPC_LIBS)
+bench_bench_bitcoin_LDADD += $(BOOST_LIBS) $(BDB_LIBS) $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) $(MINIUPNPC_LIBS) $(SQLITE_LIBS)
bench_bench_bitcoin_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) $(PTHREAD_FLAGS)
CLEAN_BITCOIN_BENCH = bench/*.gcda bench/*.gcno $(GENERATED_BENCH_FILES)
diff --git a/src/Makefile.qt.include b/src/Makefile.qt.include
index a6236ef19b..f46310a603 100644
--- a/src/Makefile.qt.include
+++ b/src/Makefile.qt.include
@@ -321,7 +321,7 @@ bitcoin_qt_ldadd += $(LIBBITCOIN_ZMQ) $(ZMQ_LIBS)
endif
bitcoin_qt_ldadd += $(LIBBITCOIN_CLI) $(LIBBITCOIN_COMMON) $(LIBBITCOIN_UTIL) $(LIBBITCOIN_CONSENSUS) $(LIBBITCOIN_CRYPTO) $(LIBUNIVALUE) $(LIBLEVELDB) $(LIBLEVELDB_SSE42) $(LIBMEMENV) \
$(BOOST_LIBS) $(QT_LIBS) $(QT_DBUS_LIBS) $(QR_LIBS) $(BDB_LIBS) $(MINIUPNPC_LIBS) $(LIBSECP256K1) \
- $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS)
+ $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) $(SQLITE_LIBS)
bitcoin_qt_ldflags = $(RELDFLAGS) $(AM_LDFLAGS) $(QT_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) $(PTHREAD_FLAGS)
bitcoin_qt_libtoolflags = $(AM_LIBTOOLFLAGS) --tag CXX
diff --git a/src/Makefile.qttest.include b/src/Makefile.qttest.include
index d300398b25..c05dd38737 100644
--- a/src/Makefile.qttest.include
+++ b/src/Makefile.qttest.include
@@ -56,7 +56,7 @@ endif
qt_test_test_bitcoin_qt_LDADD += $(LIBBITCOIN_CLI) $(LIBBITCOIN_COMMON) $(LIBBITCOIN_UTIL) $(LIBBITCOIN_CONSENSUS) $(LIBBITCOIN_CRYPTO) $(LIBUNIVALUE) $(LIBLEVELDB) \
$(LIBLEVELDB_SSE42) $(LIBMEMENV) $(BOOST_LIBS) $(QT_DBUS_LIBS) $(QT_TEST_LIBS) $(QT_LIBS) \
$(QR_LIBS) $(BDB_LIBS) $(MINIUPNPC_LIBS) $(LIBSECP256K1) \
- $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS)
+ $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) $(SQLITE_LIBS)
qt_test_test_bitcoin_qt_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(QT_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) $(PTHREAD_FLAGS)
qt_test_test_bitcoin_qt_CXXFLAGS = $(AM_CXXFLAGS) $(QT_PIE_FLAGS)
diff --git a/src/Makefile.test.include b/src/Makefile.test.include
index 3af30db4db..835b12ce3e 100644
--- a/src/Makefile.test.include
+++ b/src/Makefile.test.include
@@ -316,7 +316,7 @@ test_test_bitcoin_LDADD += $(LIBBITCOIN_SERVER) $(LIBBITCOIN_CLI) $(LIBBITCOIN_C
$(LIBLEVELDB) $(LIBLEVELDB_SSE42) $(LIBMEMENV) $(BOOST_LIBS) $(BOOST_UNIT_TEST_FRAMEWORK_LIB) $(LIBSECP256K1) $(EVENT_LIBS) $(EVENT_PTHREADS_LIBS)
test_test_bitcoin_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS)
-test_test_bitcoin_LDADD += $(BDB_LIBS) $(MINIUPNPC_LIBS)
+test_test_bitcoin_LDADD += $(BDB_LIBS) $(MINIUPNPC_LIBS) $(SQLITE_LIBS)
test_test_bitcoin_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) $(PTHREAD_FLAGS) -static
if ENABLE_ZMQ
diff --git a/src/wallet/bdb.cpp b/src/wallet/bdb.cpp
index ae3c7ae7bb..85aae0170d 100644
--- a/src/wallet/bdb.cpp
+++ b/src/wallet/bdb.cpp
@@ -813,7 +813,7 @@ bool ExistsBerkeleyDatabase(const fs::path& path)
fs::path env_directory;
std::string data_filename;
SplitWalletPath(path, env_directory, data_filename);
- return IsBerkeleyBtree(env_directory / data_filename);
+ return IsBDBFile(env_directory / data_filename);
}
std::unique_ptr<BerkeleyDatabase> MakeBerkeleyDatabase(const fs::path& path, const DatabaseOptions& options, DatabaseStatus& status, bilingual_str& error)
@@ -839,3 +839,28 @@ std::unique_ptr<BerkeleyDatabase> MakeBerkeleyDatabase(const fs::path& path, con
status = DatabaseStatus::SUCCESS;
return db;
}
+
+bool IsBDBFile(const fs::path& path)
+{
+ if (!fs::exists(path)) return false;
+
+ // A Berkeley DB Btree file has at least 4K.
+ // This check also prevents opening lock files.
+ boost::system::error_code ec;
+ auto size = fs::file_size(path, ec);
+ if (ec) LogPrintf("%s: %s %s\n", __func__, ec.message(), path.string());
+ if (size < 4096) return false;
+
+ fsbridge::ifstream file(path, std::ios::binary);
+ if (!file.is_open()) return false;
+
+ file.seekg(12, std::ios::beg); // Magic bytes start at offset 12
+ uint32_t data = 0;
+ file.read((char*) &data, sizeof(data)); // Read 4 bytes of file to compare against magic
+
+ // Berkeley DB Btree magic bytes, from:
+ // https://github.com/file/file/blob/5824af38469ec1ca9ac3ffd251e7afe9dc11e227/magic/Magdir/database#L74-L75
+ // - big endian systems - 00 05 31 62
+ // - little endian systems - 62 31 05 00
+ return data == 0x00053162 || data == 0x62310500;
+}
diff --git a/src/wallet/bdb.h b/src/wallet/bdb.h
index 070590872b..5403e95ee4 100644
--- a/src/wallet/bdb.h
+++ b/src/wallet/bdb.h
@@ -87,7 +87,7 @@ public:
std::shared_ptr<BerkeleyEnvironment> GetWalletEnv(const fs::path& wallet_path, std::string& database_filename);
/** Check format of database file */
-bool IsBerkeleyBtree(const fs::path& path);
+bool IsBDBFile(const fs::path& path);
class BerkeleyBatch;
diff --git a/src/wallet/db.h b/src/wallet/db.h
index 0004fc1afa..3ecccd4e00 100644
--- a/src/wallet/db.h
+++ b/src/wallet/db.h
@@ -8,6 +8,7 @@
#include <clientversion.h>
#include <fs.h>
+#include <optional.h>
#include <streams.h>
#include <support/allocators/secure.h>
#include <util/memory.h>
@@ -194,11 +195,13 @@ public:
enum class DatabaseFormat {
BERKELEY,
+ SQLITE,
};
struct DatabaseOptions {
bool require_existing = false;
bool require_create = false;
+ Optional<DatabaseFormat> require_format;
uint64_t create_flags = 0;
SecureString create_passphrase;
bool verify = true;
diff --git a/src/wallet/sqlite.cpp b/src/wallet/sqlite.cpp
new file mode 100644
index 0000000000..02a161ecbd
--- /dev/null
+++ b/src/wallet/sqlite.cpp
@@ -0,0 +1,629 @@
+// Copyright (c) 2020 The Bitcoin Core developers
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+#include <wallet/sqlite.h>
+
+#include <chainparams.h>
+#include <crypto/common.h>
+#include <logging.h>
+#include <sync.h>
+#include <util/memory.h>
+#include <util/strencodings.h>
+#include <util/system.h>
+#include <util/translation.h>
+#include <wallet/db.h>
+
+#include <sqlite3.h>
+#include <stdint.h>
+
+static const char* const DATABASE_FILENAME = "wallet.dat";
+static constexpr int32_t WALLET_SCHEMA_VERSION = 0;
+
+static Mutex g_sqlite_mutex;
+static int g_sqlite_count GUARDED_BY(g_sqlite_mutex) = 0;
+
+static void ErrorLogCallback(void* arg, int code, const char* msg)
+{
+ // From sqlite3_config() documentation for the SQLITE_CONFIG_LOG option:
+ // "The void pointer that is the second argument to SQLITE_CONFIG_LOG is passed through as
+ // the first parameter to the application-defined logger function whenever that function is
+ // invoked."
+ // Assert that this is the case:
+ assert(arg == nullptr);
+ LogPrintf("SQLite Error. Code: %d. Message: %s\n", code, msg);
+}
+
+SQLiteDatabase::SQLiteDatabase(const fs::path& dir_path, const fs::path& file_path, bool mock)
+ : WalletDatabase(), m_mock(mock), m_dir_path(dir_path.string()), m_file_path(file_path.string())
+{
+ {
+ LOCK(g_sqlite_mutex);
+ LogPrintf("Using SQLite Version %s\n", SQLiteDatabaseVersion());
+ LogPrintf("Using wallet %s\n", m_dir_path);
+
+ if (++g_sqlite_count == 1) {
+ // Setup logging
+ int ret = sqlite3_config(SQLITE_CONFIG_LOG, ErrorLogCallback, nullptr);
+ if (ret != SQLITE_OK) {
+ throw std::runtime_error(strprintf("SQLiteDatabase: Failed to setup error log: %s\n", sqlite3_errstr(ret)));
+ }
+ // Force serialized threading mode
+ ret = sqlite3_config(SQLITE_CONFIG_SERIALIZED);
+ if (ret != SQLITE_OK) {
+ throw std::runtime_error(strprintf("SQLiteDatabase: Failed to configure serialized threading mode: %s\n", sqlite3_errstr(ret)));
+ }
+ }
+ int ret = sqlite3_initialize(); // This is a no-op if sqlite3 is already initialized
+ if (ret != SQLITE_OK) {
+ throw std::runtime_error(strprintf("SQLiteDatabase: Failed to initialize SQLite: %s\n", sqlite3_errstr(ret)));
+ }
+ }
+
+ try {
+ Open();
+ } catch (const std::runtime_error&) {
+ // If open fails, cleanup this object and rethrow the exception
+ Cleanup();
+ throw;
+ }
+}
+
+void SQLiteBatch::SetupSQLStatements()
+{
+ int res;
+ if (!m_read_stmt) {
+ if ((res = sqlite3_prepare_v2(m_database.m_db, "SELECT value FROM main WHERE key = ?", -1, &m_read_stmt, nullptr)) != SQLITE_OK) {
+ throw std::runtime_error(strprintf("SQLiteDatabase: Failed to setup SQL statements: %s\n", sqlite3_errstr(res)));
+ }
+ }
+ if (!m_insert_stmt) {
+ if ((res = sqlite3_prepare_v2(m_database.m_db, "INSERT INTO main VALUES(?, ?)", -1, &m_insert_stmt, nullptr)) != SQLITE_OK) {
+ throw std::runtime_error(strprintf("SQLiteDatabase: Failed to setup SQL statements: %s\n", sqlite3_errstr(res)));
+ }
+ }
+ if (!m_overwrite_stmt) {
+ if ((res = sqlite3_prepare_v2(m_database.m_db, "INSERT or REPLACE into main values(?, ?)", -1, &m_overwrite_stmt, nullptr)) != SQLITE_OK) {
+ throw std::runtime_error(strprintf("SQLiteDatabase: Failed to setup SQL statements: %s\n", sqlite3_errstr(res)));
+ }
+ }
+ if (!m_delete_stmt) {
+ if ((res = sqlite3_prepare_v2(m_database.m_db, "DELETE FROM main WHERE key = ?", -1, &m_delete_stmt, nullptr)) != SQLITE_OK) {
+ throw std::runtime_error(strprintf("SQLiteDatabase: Failed to setup SQL statements: %s\n", sqlite3_errstr(res)));
+ }
+ }
+ if (!m_cursor_stmt) {
+ if ((res = sqlite3_prepare_v2(m_database.m_db, "SELECT key, value FROM main", -1, &m_cursor_stmt, nullptr)) != SQLITE_OK) {
+ throw std::runtime_error(strprintf("SQLiteDatabase: Failed to setup SQL statements : %s\n", sqlite3_errstr(res)));
+ }
+ }
+}
+
+SQLiteDatabase::~SQLiteDatabase()
+{
+ Cleanup();
+}
+
+void SQLiteDatabase::Cleanup() noexcept
+{
+ Close();
+
+ LOCK(g_sqlite_mutex);
+ if (--g_sqlite_count == 0) {
+ int ret = sqlite3_shutdown();
+ if (ret != SQLITE_OK) {
+ LogPrintf("SQLiteDatabase: Failed to shutdown SQLite: %s\n", sqlite3_errstr(ret));
+ }
+ }
+}
+
+bool SQLiteDatabase::Verify(bilingual_str& error)
+{
+ assert(m_db);
+
+ // Check the application ID matches our network magic
+ sqlite3_stmt* app_id_stmt{nullptr};
+ int ret = sqlite3_prepare_v2(m_db, "PRAGMA application_id", -1, &app_id_stmt, nullptr);
+ if (ret != SQLITE_OK) {
+ sqlite3_finalize(app_id_stmt);
+ error = strprintf(_("SQLiteDatabase: Failed to prepare the statement to fetch the application id: %s"), sqlite3_errstr(ret));
+ return false;
+ }
+ ret = sqlite3_step(app_id_stmt);
+ if (ret != SQLITE_ROW) {
+ sqlite3_finalize(app_id_stmt);
+ error = strprintf(_("SQLiteDatabase: Failed to fetch the application id: %s"), sqlite3_errstr(ret));
+ return false;
+ }
+ uint32_t app_id = static_cast<uint32_t>(sqlite3_column_int(app_id_stmt, 0));
+ sqlite3_finalize(app_id_stmt);
+ uint32_t net_magic = ReadBE32(Params().MessageStart());
+ if (app_id != net_magic) {
+ error = strprintf(_("SQLiteDatabase: Unexpected application id. Expected %u, got %u"), net_magic, app_id);
+ return false;
+ }
+
+ // Check our schema version
+ sqlite3_stmt* user_ver_stmt{nullptr};
+ ret = sqlite3_prepare_v2(m_db, "PRAGMA user_version", -1, &user_ver_stmt, nullptr);
+ if (ret != SQLITE_OK) {
+ sqlite3_finalize(user_ver_stmt);
+ error = strprintf(_("SQLiteDatabase: Failed to prepare the statement to fetch sqlite wallet schema version: %s"), sqlite3_errstr(ret));
+ return false;
+ }
+ ret = sqlite3_step(user_ver_stmt);
+ if (ret != SQLITE_ROW) {
+ sqlite3_finalize(user_ver_stmt);
+ error = strprintf(_("SQLiteDatabase: Failed to fetch sqlite wallet schema version: %s"), sqlite3_errstr(ret));
+ return false;
+ }
+ int32_t user_ver = sqlite3_column_int(user_ver_stmt, 0);
+ sqlite3_finalize(user_ver_stmt);
+ if (user_ver != WALLET_SCHEMA_VERSION) {
+ error = strprintf(_("SQLiteDatabase: Unknown sqlite wallet schema version %d. Only version %d is supported"), user_ver, WALLET_SCHEMA_VERSION);
+ return false;
+ }
+
+ sqlite3_stmt* stmt{nullptr};
+ ret = sqlite3_prepare_v2(m_db, "PRAGMA integrity_check", -1, &stmt, nullptr);
+ if (ret != SQLITE_OK) {
+ sqlite3_finalize(stmt);
+ error = strprintf(_("SQLiteDatabase: Failed to prepare statement to verify database: %s"), sqlite3_errstr(ret));
+ return false;
+ }
+ while (true) {
+ ret = sqlite3_step(stmt);
+ if (ret == SQLITE_DONE) {
+ break;
+ }
+ if (ret != SQLITE_ROW) {
+ error = strprintf(_("SQLiteDatabase: Failed to execute statement to verify database: %s"), sqlite3_errstr(ret));
+ break;
+ }
+ const char* msg = (const char*)sqlite3_column_text(stmt, 0);
+ if (!msg) {
+ error = strprintf(_("SQLiteDatabase: Failed to read database verification error: %s"), sqlite3_errstr(ret));
+ break;
+ }
+ std::string str_msg(msg);
+ if (str_msg == "ok") {
+ continue;
+ }
+ if (error.empty()) {
+ error = _("Failed to verify database") + Untranslated("\n");
+ }
+ error += Untranslated(strprintf("%s\n", str_msg));
+ }
+ sqlite3_finalize(stmt);
+ return error.empty();
+}
+
+void SQLiteDatabase::Open()
+{
+ int flags = SQLITE_OPEN_FULLMUTEX | SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE;
+ if (m_mock) {
+ flags |= SQLITE_OPEN_MEMORY; // In memory database for mock db
+ }
+
+ if (m_db == nullptr) {
+ TryCreateDirectories(m_dir_path);
+ int ret = sqlite3_open_v2(m_file_path.c_str(), &m_db, flags, nullptr);
+ if (ret != SQLITE_OK) {
+ throw std::runtime_error(strprintf("SQLiteDatabase: Failed to open database: %s\n", sqlite3_errstr(ret)));
+ }
+ }
+
+ if (sqlite3_db_readonly(m_db, "main") != 0) {
+ throw std::runtime_error("SQLiteDatabase: Database opened in readonly mode but read-write permissions are needed");
+ }
+
+ // Acquire an exclusive lock on the database
+ // First change the locking mode to exclusive
+ int ret = sqlite3_exec(m_db, "PRAGMA locking_mode = exclusive", nullptr, nullptr, nullptr);
+ if (ret != SQLITE_OK) {
+ throw std::runtime_error(strprintf("SQLiteDatabase: Unable to change database locking mode to exclusive: %s\n", sqlite3_errstr(ret)));
+ }
+ // Now begin a transaction to acquire the exclusive lock. This lock won't be released until we close because of the exclusive locking mode.
+ ret = sqlite3_exec(m_db, "BEGIN EXCLUSIVE TRANSACTION", nullptr, nullptr, nullptr);
+ if (ret != SQLITE_OK) {
+ throw std::runtime_error("SQLiteDatabase: Unable to obtain an exclusive lock on the database, is it being used by another bitcoind?\n");
+ }
+ ret = sqlite3_exec(m_db, "COMMIT", nullptr, nullptr, nullptr);
+ if (ret != SQLITE_OK) {
+ throw std::runtime_error(strprintf("SQLiteDatabase: Unable to end exclusive lock transaction: %s\n", sqlite3_errstr(ret)));
+ }
+
+ // Enable fullfsync for the platforms that use it
+ ret = sqlite3_exec(m_db, "PRAGMA fullfsync = true", nullptr, nullptr, nullptr);
+ if (ret != SQLITE_OK) {
+ throw std::runtime_error(strprintf("SQLiteDatabase: Failed to enable fullfsync: %s\n", sqlite3_errstr(ret)));
+ }
+
+ // Make the table for our key-value pairs
+ // First check that the main table exists
+ sqlite3_stmt* check_main_stmt{nullptr};
+ ret = sqlite3_prepare_v2(m_db, "SELECT name FROM sqlite_master WHERE type='table' AND name='main'", -1, &check_main_stmt, nullptr);
+ if (ret != SQLITE_OK) {
+ throw std::runtime_error(strprintf("SQLiteDatabase: Failed to prepare statement to check table existence: %s\n", sqlite3_errstr(ret)));
+ }
+ ret = sqlite3_step(check_main_stmt);
+ if (sqlite3_finalize(check_main_stmt) != SQLITE_OK) {
+ throw std::runtime_error(strprintf("SQLiteDatabase: Failed to finalize statement checking table existence: %s\n", sqlite3_errstr(ret)));
+ }
+ bool table_exists;
+ if (ret == SQLITE_DONE) {
+ table_exists = false;
+ } else if (ret == SQLITE_ROW) {
+ table_exists = true;
+ } else {
+ throw std::runtime_error(strprintf("SQLiteDatabase: Failed to execute statement to check table existence: %s\n", sqlite3_errstr(ret)));
+ }
+
+ // Do the db setup things because the table doesn't exist only when we are creating a new wallet
+ if (!table_exists) {
+ ret = sqlite3_exec(m_db, "CREATE TABLE main(key BLOB PRIMARY KEY NOT NULL, value BLOB NOT NULL)", nullptr, nullptr, nullptr);
+ if (ret != SQLITE_OK) {
+ throw std::runtime_error(strprintf("SQLiteDatabase: Failed to create new database: %s\n", sqlite3_errstr(ret)));
+ }
+
+ // Set the application id
+ uint32_t app_id = ReadBE32(Params().MessageStart());
+ std::string set_app_id = strprintf("PRAGMA application_id = %d", static_cast<int32_t>(app_id));
+ ret = sqlite3_exec(m_db, set_app_id.c_str(), nullptr, nullptr, nullptr);
+ if (ret != SQLITE_OK) {
+ throw std::runtime_error(strprintf("SQLiteDatabase: Failed to set the application id: %s\n", sqlite3_errstr(ret)));
+ }
+
+ // Set the user version
+ std::string set_user_ver = strprintf("PRAGMA user_version = %d", WALLET_SCHEMA_VERSION);
+ ret = sqlite3_exec(m_db, set_user_ver.c_str(), nullptr, nullptr, nullptr);
+ if (ret != SQLITE_OK) {
+ throw std::runtime_error(strprintf("SQLiteDatabase: Failed to set the wallet schema version: %s\n", sqlite3_errstr(ret)));
+ }
+ }
+}
+
+bool SQLiteDatabase::Rewrite(const char* skip)
+{
+ // Rewrite the database using the VACUUM command: https://sqlite.org/lang_vacuum.html
+ int ret = sqlite3_exec(m_db, "VACUUM", nullptr, nullptr, nullptr);
+ return ret == SQLITE_OK;
+}
+
+bool SQLiteDatabase::Backup(const std::string& dest) const
+{
+ sqlite3* db_copy;
+ int res = sqlite3_open(dest.c_str(), &db_copy);
+ if (res != SQLITE_OK) {
+ sqlite3_close(db_copy);
+ return false;
+ }
+ sqlite3_backup* backup = sqlite3_backup_init(db_copy, "main", m_db, "main");
+ if (!backup) {
+ LogPrintf("%s: Unable to begin backup: %s\n", __func__, sqlite3_errmsg(m_db));
+ sqlite3_close(db_copy);
+ return false;
+ }
+ // Specifying -1 will copy all of the pages
+ res = sqlite3_backup_step(backup, -1);
+ if (res != SQLITE_DONE) {
+ LogPrintf("%s: Unable to backup: %s\n", __func__, sqlite3_errstr(res));
+ sqlite3_backup_finish(backup);
+ sqlite3_close(db_copy);
+ return false;
+ }
+ res = sqlite3_backup_finish(backup);
+ sqlite3_close(db_copy);
+ return res == SQLITE_OK;
+}
+
+void SQLiteDatabase::Close()
+{
+ int res = sqlite3_close(m_db);
+ if (res != SQLITE_OK) {
+ throw std::runtime_error(strprintf("SQLiteDatabase: Failed to close database: %s\n", sqlite3_errstr(res)));
+ }
+ m_db = nullptr;
+}
+
+std::unique_ptr<DatabaseBatch> SQLiteDatabase::MakeBatch(bool flush_on_close)
+{
+ // We ignore flush_on_close because we don't do manual flushing for SQLite
+ return MakeUnique<SQLiteBatch>(*this);
+}
+
+SQLiteBatch::SQLiteBatch(SQLiteDatabase& database)
+ : m_database(database)
+{
+ // Make sure we have a db handle
+ assert(m_database.m_db);
+
+ SetupSQLStatements();
+}
+
+void SQLiteBatch::Close()
+{
+ // If m_db is in a transaction (i.e. not in autocommit mode), then abort the transaction in progress
+ if (m_database.m_db && sqlite3_get_autocommit(m_database.m_db) == 0) {
+ if (TxnAbort()) {
+ LogPrintf("SQLiteBatch: Batch closed unexpectedly without the transaction being explicitly committed or aborted\n");
+ } else {
+ LogPrintf("SQLiteBatch: Batch closed and failed to abort transaction\n");
+ }
+ }
+
+ // Free all of the prepared statements
+ int ret = sqlite3_finalize(m_read_stmt);
+ if (ret != SQLITE_OK) {
+ LogPrintf("SQLiteBatch: Batch closed but could not finalize read statement: %s\n", sqlite3_errstr(ret));
+ }
+ ret = sqlite3_finalize(m_insert_stmt);
+ if (ret != SQLITE_OK) {
+ LogPrintf("SQLiteBatch: Batch closed but could not finalize insert statement: %s\n", sqlite3_errstr(ret));
+ }
+ ret = sqlite3_finalize(m_overwrite_stmt);
+ if (ret != SQLITE_OK) {
+ LogPrintf("SQLiteBatch: Batch closed but could not finalize overwrite statement: %s\n", sqlite3_errstr(ret));
+ }
+ ret = sqlite3_finalize(m_delete_stmt);
+ if (ret != SQLITE_OK) {
+ LogPrintf("SQLiteBatch: Batch closed but could not finalize delete statement: %s\n", sqlite3_errstr(ret));
+ }
+ ret = sqlite3_finalize(m_cursor_stmt);
+ if (ret != SQLITE_OK) {
+ LogPrintf("SQLiteBatch: Batch closed but could not finalize cursor statement: %s\n", sqlite3_errstr(ret));
+ }
+ m_read_stmt = nullptr;
+ m_insert_stmt = nullptr;
+ m_overwrite_stmt = nullptr;
+ m_delete_stmt = nullptr;
+ m_cursor_stmt = nullptr;
+}
+
+bool SQLiteBatch::ReadKey(CDataStream&& key, CDataStream& value)
+{
+ if (!m_database.m_db) return false;
+ assert(m_read_stmt);
+
+ // Bind: leftmost parameter in statement is index 1
+ int res = sqlite3_bind_blob(m_read_stmt, 1, key.data(), key.size(), SQLITE_STATIC);
+ if (res != SQLITE_OK) {
+ LogPrintf("%s: Unable to bind statement: %s\n", __func__, sqlite3_errstr(res));
+ sqlite3_clear_bindings(m_read_stmt);
+ sqlite3_reset(m_read_stmt);
+ return false;
+ }
+ res = sqlite3_step(m_read_stmt);
+ if (res != SQLITE_ROW) {
+ if (res != SQLITE_DONE) {
+ // SQLITE_DONE means "not found", don't log an error in that case.
+ LogPrintf("%s: Unable to execute statement: %s\n", __func__, sqlite3_errstr(res));
+ }
+ sqlite3_clear_bindings(m_read_stmt);
+ sqlite3_reset(m_read_stmt);
+ return false;
+ }
+ // Leftmost column in result is index 0
+ const char* data = reinterpret_cast<const char*>(sqlite3_column_blob(m_read_stmt, 0));
+ int data_size = sqlite3_column_bytes(m_read_stmt, 0);
+ value.write(data, data_size);
+
+ sqlite3_clear_bindings(m_read_stmt);
+ sqlite3_reset(m_read_stmt);
+ return true;
+}
+
+bool SQLiteBatch::WriteKey(CDataStream&& key, CDataStream&& value, bool overwrite)
+{
+ if (!m_database.m_db) return false;
+ assert(m_insert_stmt && m_overwrite_stmt);
+
+ sqlite3_stmt* stmt;
+ if (overwrite) {
+ stmt = m_overwrite_stmt;
+ } else {
+ stmt = m_insert_stmt;
+ }
+
+ // Bind: leftmost parameter in statement is index 1
+ // Insert index 1 is key, 2 is value
+ int res = sqlite3_bind_blob(stmt, 1, key.data(), key.size(), SQLITE_STATIC);
+ if (res != SQLITE_OK) {
+ LogPrintf("%s: Unable to bind key to statement: %s\n", __func__, sqlite3_errstr(res));
+ sqlite3_clear_bindings(stmt);
+ sqlite3_reset(stmt);
+ return false;
+ }
+ res = sqlite3_bind_blob(stmt, 2, value.data(), value.size(), SQLITE_STATIC);
+ if (res != SQLITE_OK) {
+ LogPrintf("%s: Unable to bind value to statement: %s\n", __func__, sqlite3_errstr(res));
+ sqlite3_clear_bindings(stmt);
+ sqlite3_reset(stmt);
+ return false;
+ }
+
+ // Execute
+ res = sqlite3_step(stmt);
+ sqlite3_clear_bindings(stmt);
+ sqlite3_reset(stmt);
+ if (res != SQLITE_DONE) {
+ LogPrintf("%s: Unable to execute statement: %s\n", __func__, sqlite3_errstr(res));
+ }
+ return res == SQLITE_DONE;
+}
+
+bool SQLiteBatch::EraseKey(CDataStream&& key)
+{
+ if (!m_database.m_db) return false;
+ assert(m_delete_stmt);
+
+ // Bind: leftmost parameter in statement is index 1
+ int res = sqlite3_bind_blob(m_delete_stmt, 1, key.data(), key.size(), SQLITE_STATIC);
+ if (res != SQLITE_OK) {
+ LogPrintf("%s: Unable to bind statement: %s\n", __func__, sqlite3_errstr(res));
+ sqlite3_clear_bindings(m_delete_stmt);
+ sqlite3_reset(m_delete_stmt);
+ return false;
+ }
+
+ // Execute
+ res = sqlite3_step(m_delete_stmt);
+ sqlite3_clear_bindings(m_delete_stmt);
+ sqlite3_reset(m_delete_stmt);
+ if (res != SQLITE_DONE) {
+ LogPrintf("%s: Unable to execute statement: %s\n", __func__, sqlite3_errstr(res));
+ }
+ return res == SQLITE_DONE;
+}
+
+bool SQLiteBatch::HasKey(CDataStream&& key)
+{
+ if (!m_database.m_db) return false;
+ assert(m_read_stmt);
+
+ // Bind: leftmost parameter in statement is index 1
+ bool ret = false;
+ int res = sqlite3_bind_blob(m_read_stmt, 1, key.data(), key.size(), SQLITE_STATIC);
+ if (res == SQLITE_OK) {
+ res = sqlite3_step(m_read_stmt);
+ if (res == SQLITE_ROW) {
+ ret = true;
+ }
+ }
+
+ sqlite3_clear_bindings(m_read_stmt);
+ sqlite3_reset(m_read_stmt);
+ return ret;
+}
+
+bool SQLiteBatch::StartCursor()
+{
+ assert(!m_cursor_init);
+ if (!m_database.m_db) return false;
+ m_cursor_init = true;
+ return true;
+}
+
+bool SQLiteBatch::ReadAtCursor(CDataStream& key, CDataStream& value, bool& complete)
+{
+ complete = false;
+
+ if (!m_cursor_init) return false;
+
+ int res = sqlite3_step(m_cursor_stmt);
+ if (res == SQLITE_DONE) {
+ complete = true;
+ return true;
+ }
+ if (res != SQLITE_ROW) {
+ LogPrintf("SQLiteBatch::ReadAtCursor: Unable to execute cursor step: %s\n", sqlite3_errstr(res));
+ return false;
+ }
+
+ // Leftmost column in result is index 0
+ const char* key_data = reinterpret_cast<const char*>(sqlite3_column_blob(m_cursor_stmt, 0));
+ int key_data_size = sqlite3_column_bytes(m_cursor_stmt, 0);
+ key.write(key_data, key_data_size);
+ const char* value_data = reinterpret_cast<const char*>(sqlite3_column_blob(m_cursor_stmt, 1));
+ int value_data_size = sqlite3_column_bytes(m_cursor_stmt, 1);
+ value.write(value_data, value_data_size);
+ return true;
+}
+
+void SQLiteBatch::CloseCursor()
+{
+ sqlite3_reset(m_cursor_stmt);
+ m_cursor_init = false;
+}
+
+bool SQLiteBatch::TxnBegin()
+{
+ if (!m_database.m_db || sqlite3_get_autocommit(m_database.m_db) == 0) return false;
+ int res = sqlite3_exec(m_database.m_db, "BEGIN TRANSACTION", nullptr, nullptr, nullptr);
+ if (res != SQLITE_OK) {
+ LogPrintf("SQLiteBatch: Failed to begin the transaction\n");
+ }
+ return res == SQLITE_OK;
+}
+
+bool SQLiteBatch::TxnCommit()
+{
+ if (!m_database.m_db || sqlite3_get_autocommit(m_database.m_db) != 0) return false;
+ int res = sqlite3_exec(m_database.m_db, "COMMIT TRANSACTION", nullptr, nullptr, nullptr);
+ if (res != SQLITE_OK) {
+ LogPrintf("SQLiteBatch: Failed to commit the transaction\n");
+ }
+ return res == SQLITE_OK;
+}
+
+bool SQLiteBatch::TxnAbort()
+{
+ if (!m_database.m_db || sqlite3_get_autocommit(m_database.m_db) != 0) return false;
+ int res = sqlite3_exec(m_database.m_db, "ROLLBACK TRANSACTION", nullptr, nullptr, nullptr);
+ if (res != SQLITE_OK) {
+ LogPrintf("SQLiteBatch: Failed to abort the transaction\n");
+ }
+ return res == SQLITE_OK;
+}
+
+bool ExistsSQLiteDatabase(const fs::path& path)
+{
+ const fs::path file = path / DATABASE_FILENAME;
+ return fs::symlink_status(file).type() == fs::regular_file && IsSQLiteFile(file);
+}
+
+std::unique_ptr<SQLiteDatabase> MakeSQLiteDatabase(const fs::path& path, const DatabaseOptions& options, DatabaseStatus& status, bilingual_str& error)
+{
+ const fs::path file = path / DATABASE_FILENAME;
+ try {
+ auto db = MakeUnique<SQLiteDatabase>(path, file);
+ if (options.verify && !db->Verify(error)) {
+ status = DatabaseStatus::FAILED_VERIFY;
+ return nullptr;
+ }
+ return db;
+ } catch (const std::runtime_error& e) {
+ status = DatabaseStatus::FAILED_LOAD;
+ error.original = e.what();
+ return nullptr;
+ }
+}
+
+std::string SQLiteDatabaseVersion()
+{
+ return std::string(sqlite3_libversion());
+}
+
+bool IsSQLiteFile(const fs::path& path)
+{
+ if (!fs::exists(path)) return false;
+
+ // A SQLite Database file is at least 512 bytes.
+ boost::system::error_code ec;
+ auto size = fs::file_size(path, ec);
+ if (ec) LogPrintf("%s: %s %s\n", __func__, ec.message(), path.string());
+ if (size < 512) return false;
+
+ fsbridge::ifstream file(path, std::ios::binary);
+ if (!file.is_open()) return false;
+
+ // Magic is at beginning and is 16 bytes long
+ char magic[16];
+ file.read(magic, 16);
+
+ // Application id is at offset 68 and 4 bytes long
+ file.seekg(68, std::ios::beg);
+ char app_id[4];
+ file.read(app_id, 4);
+
+ file.close();
+
+ // Check the magic, see https://sqlite.org/fileformat2.html
+ std::string magic_str(magic);
+ if (magic_str != std::string("SQLite format 3")) {
+ return false;
+ }
+
+ // Check the application id matches our network magic
+ return memcmp(Params().MessageStart(), app_id, 4) == 0;
+}
diff --git a/src/wallet/sqlite.h b/src/wallet/sqlite.h
new file mode 100644
index 0000000000..5e5e93903b
--- /dev/null
+++ b/src/wallet/sqlite.h
@@ -0,0 +1,121 @@
+// Copyright (c) 2020 The Bitcoin Core developers
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+#ifndef BITCOIN_WALLET_SQLITE_H
+#define BITCOIN_WALLET_SQLITE_H
+
+#include <wallet/db.h>
+
+#include <sqlite3.h>
+
+struct bilingual_str;
+class SQLiteDatabase;
+
+/** RAII class that provides access to a WalletDatabase */
+class SQLiteBatch : public DatabaseBatch
+{
+private:
+ SQLiteDatabase& m_database;
+
+ bool m_cursor_init = false;
+
+ sqlite3_stmt* m_read_stmt{nullptr};
+ sqlite3_stmt* m_insert_stmt{nullptr};
+ sqlite3_stmt* m_overwrite_stmt{nullptr};
+ sqlite3_stmt* m_delete_stmt{nullptr};
+ sqlite3_stmt* m_cursor_stmt{nullptr};
+
+ void SetupSQLStatements();
+
+ bool ReadKey(CDataStream&& key, CDataStream& value) override;
+ bool WriteKey(CDataStream&& key, CDataStream&& value, bool overwrite = true) override;
+ bool EraseKey(CDataStream&& key) override;
+ bool HasKey(CDataStream&& key) override;
+
+public:
+ explicit SQLiteBatch(SQLiteDatabase& database);
+ ~SQLiteBatch() override { Close(); }
+
+ /* No-op. See commeng on SQLiteDatabase::Flush */
+ void Flush() override {}
+
+ void Close() override;
+
+ bool StartCursor() override;
+ bool ReadAtCursor(CDataStream& key, CDataStream& value, bool& complete) override;
+ void CloseCursor() override;
+ bool TxnBegin() override;
+ bool TxnCommit() override;
+ bool TxnAbort() override;
+};
+
+/** An instance of this class represents one SQLite3 database.
+ **/
+class SQLiteDatabase : public WalletDatabase
+{
+private:
+ const bool m_mock{false};
+
+ const std::string m_dir_path;
+
+ const std::string m_file_path;
+
+ void Cleanup() noexcept;
+
+public:
+ SQLiteDatabase() = delete;
+
+ /** Create DB handle to real database */
+ SQLiteDatabase(const fs::path& dir_path, const fs::path& file_path, bool mock = false);
+
+ ~SQLiteDatabase();
+
+ bool Verify(bilingual_str& error);
+
+ /** Open the database if it is not already opened */
+ void Open() override;
+
+ /** Close the database */
+ void Close() override;
+
+ /* These functions are unused */
+ void AddRef() override { assert(false); }
+ void RemoveRef() override { assert(false); }
+
+ /** Rewrite the entire database on disk */
+ bool Rewrite(const char* skip = nullptr) override;
+
+ /** Back up the entire database to a file.
+ */
+ bool Backup(const std::string& dest) const override;
+
+ /** No-ops
+ *
+ * SQLite always flushes everything to the database file after each transaction
+ * (each Read/Write/Erase that we do is its own transaction unless we called
+ * TxnBegin) so there is no need to have Flush or Periodic Flush.
+ *
+ * There is no DB env to reload, so ReloadDbEnv has nothing to do
+ */
+ void Flush() override {}
+ bool PeriodicFlush() override { return false; }
+ void ReloadDbEnv() override {}
+
+ void IncrementUpdateCounter() override { ++nUpdateCounter; }
+
+ std::string Filename() override { return m_file_path; }
+
+ /** Make a SQLiteBatch connected to this database */
+ std::unique_ptr<DatabaseBatch> MakeBatch(bool flush_on_close = true) override;
+
+ sqlite3* m_db{nullptr};
+};
+
+bool ExistsSQLiteDatabase(const fs::path& path);
+std::unique_ptr<SQLiteDatabase> MakeSQLiteDatabase(const fs::path& path, const DatabaseOptions& options, DatabaseStatus& status, bilingual_str& error);
+
+std::string SQLiteDatabaseVersion();
+bool IsSQLiteFile(const fs::path& path);
+
+#endif // BITCOIN_WALLET_SQLITE_H
diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp
index dfcfaf489a..6b7d05fdf3 100644
--- a/src/wallet/wallet.cpp
+++ b/src/wallet/wallet.cpp
@@ -243,11 +243,13 @@ std::shared_ptr<CWallet> LoadWallet(interfaces::Chain& chain, const std::string&
return wallet;
}
-std::shared_ptr<CWallet> CreateWallet(interfaces::Chain& chain, const std::string& name, Optional<bool> load_on_start, const DatabaseOptions& options, DatabaseStatus& status, bilingual_str& error, std::vector<bilingual_str>& warnings)
+std::shared_ptr<CWallet> CreateWallet(interfaces::Chain& chain, const std::string& name, Optional<bool> load_on_start, DatabaseOptions& options, DatabaseStatus& status, bilingual_str& error, std::vector<bilingual_str>& warnings)
{
uint64_t wallet_creation_flags = options.create_flags;
const SecureString& passphrase = options.create_passphrase;
+ if (wallet_creation_flags & WALLET_FLAG_DESCRIPTORS) options.require_format = DatabaseFormat::SQLITE;
+
// Indicate that the wallet is actually supposed to be blank and not just blank to make it encrypted
bool create_blank = (wallet_creation_flags & WALLET_FLAG_BLANK_WALLET);
diff --git a/src/wallet/wallet.h b/src/wallet/wallet.h
index fb08cb4085..245144a1c9 100644
--- a/src/wallet/wallet.h
+++ b/src/wallet/wallet.h
@@ -55,7 +55,7 @@ bool RemoveWallet(const std::shared_ptr<CWallet>& wallet, Optional<bool> load_on
std::vector<std::shared_ptr<CWallet>> GetWallets();
std::shared_ptr<CWallet> GetWallet(const std::string& name);
std::shared_ptr<CWallet> LoadWallet(interfaces::Chain& chain, const std::string& name, Optional<bool> load_on_start, const DatabaseOptions& options, DatabaseStatus& status, bilingual_str& error, std::vector<bilingual_str>& warnings);
-std::shared_ptr<CWallet> CreateWallet(interfaces::Chain& chain, const std::string& name, Optional<bool> load_on_start, const DatabaseOptions& options, DatabaseStatus& status, bilingual_str& error, std::vector<bilingual_str>& warnings);
+std::shared_ptr<CWallet> CreateWallet(interfaces::Chain& chain, const std::string& name, Optional<bool> load_on_start, DatabaseOptions& options, DatabaseStatus& status, bilingual_str& error, std::vector<bilingual_str>& warnings);
std::unique_ptr<interfaces::Handler> HandleLoadWallet(LoadWalletFn load_wallet);
std::unique_ptr<WalletDatabase> MakeWalletDatabase(const std::string& name, const DatabaseOptions& options, DatabaseStatus& status, bilingual_str& error);
diff --git a/src/wallet/walletdb.cpp b/src/wallet/walletdb.cpp
index 5bf21eb91f..0092a29cb4 100644
--- a/src/wallet/walletdb.cpp
+++ b/src/wallet/walletdb.cpp
@@ -15,6 +15,7 @@
#include <util/time.h>
#include <util/translation.h>
#include <wallet/bdb.h>
+#include <wallet/sqlite.h>
#include <wallet/wallet.h>
#include <atomic>
@@ -1011,6 +1012,14 @@ std::unique_ptr<WalletDatabase> MakeDatabase(const fs::path& path, const Databas
if (ExistsBerkeleyDatabase(path)) {
format = DatabaseFormat::BERKELEY;
}
+ if (ExistsSQLiteDatabase(path)) {
+ if (format) {
+ error = Untranslated(strprintf("Failed to load database path '%s'. Data is in ambiguous format.", path.string()));
+ status = DatabaseStatus::FAILED_BAD_FORMAT;
+ return nullptr;
+ }
+ format = DatabaseFormat::SQLITE;
+ }
} else if (options.require_existing) {
error = Untranslated(strprintf("Failed to load database path '%s'. Path does not exist.", path.string()));
status = DatabaseStatus::FAILED_NOT_FOUND;
@@ -1029,6 +1038,20 @@ std::unique_ptr<WalletDatabase> MakeDatabase(const fs::path& path, const Databas
return nullptr;
}
+ // A db already exists so format is set, but options also specifies the format, so make sure they agree
+ if (format && options.require_format && format != options.require_format) {
+ error = Untranslated(strprintf("Failed to load database path '%s'. Data is not in required format.", path.string()));
+ status = DatabaseStatus::FAILED_BAD_FORMAT;
+ return nullptr;
+ }
+
+ // Format is not set when a db doesn't already exist, so use the format specified by the options if it is set.
+ if (!format && options.require_format) format = options.require_format;
+
+ if (format && format == DatabaseFormat::SQLITE) {
+ return MakeSQLiteDatabase(path, options, status, error);
+ }
+
return MakeBerkeleyDatabase(path, options, status, error);
}
diff --git a/src/wallet/walletutil.cpp b/src/wallet/walletutil.cpp
index e4c72aed98..a2a55f9751 100644
--- a/src/wallet/walletutil.cpp
+++ b/src/wallet/walletutil.cpp
@@ -7,6 +7,9 @@
#include <logging.h>
#include <util/system.h>
+bool ExistsBerkeleyDatabase(const fs::path& path);
+bool ExistsSQLiteDatabase(const fs::path& path);
+
fs::path GetWalletDir()
{
fs::path path;
@@ -29,31 +32,6 @@ fs::path GetWalletDir()
return path;
}
-bool IsBerkeleyBtree(const fs::path& path)
-{
- if (!fs::exists(path)) return false;
-
- // A Berkeley DB Btree file has at least 4K.
- // This check also prevents opening lock files.
- boost::system::error_code ec;
- auto size = fs::file_size(path, ec);
- if (ec) LogPrintf("%s: %s %s\n", __func__, ec.message(), path.string());
- if (size < 4096) return false;
-
- fsbridge::ifstream file(path, std::ios::binary);
- if (!file.is_open()) return false;
-
- file.seekg(12, std::ios::beg); // Magic bytes start at offset 12
- uint32_t data = 0;
- file.read((char*) &data, sizeof(data)); // Read 4 bytes of file to compare against magic
-
- // Berkeley DB Btree magic bytes, from:
- // https://github.com/file/file/blob/5824af38469ec1ca9ac3ffd251e7afe9dc11e227/magic/Magdir/database#L74-L75
- // - big endian systems - 00 05 31 62
- // - little endian systems - 62 31 05 00
- return data == 0x00053162 || data == 0x62310500;
-}
-
std::vector<fs::path> ListWalletDir()
{
const fs::path wallet_dir = GetWalletDir();
@@ -71,10 +49,11 @@ std::vector<fs::path> ListWalletDir()
// This can be replaced by boost::filesystem::lexically_relative once boost is bumped to 1.60.
const fs::path path = it->path().string().substr(offset);
- if (it->status().type() == fs::directory_file && IsBerkeleyBtree(it->path() / "wallet.dat")) {
+ if (it->status().type() == fs::directory_file &&
+ (ExistsBerkeleyDatabase(it->path()) || ExistsSQLiteDatabase(it->path()))) {
// Found a directory which contains wallet.dat btree file, add it as a wallet.
paths.emplace_back(path);
- } else if (it.level() == 0 && it->symlink_status().type() == fs::regular_file && IsBerkeleyBtree(it->path())) {
+ } else if (it.level() == 0 && it->symlink_status().type() == fs::regular_file && ExistsBerkeleyDatabase(it->path())) {
if (it->path().filename() == "wallet.dat") {
// Found top-level wallet.dat btree file, add top level directory ""
// as a wallet.
diff --git a/test/functional/test_framework/test_framework.py b/test/functional/test_framework/test_framework.py
index 2824d80434..115ffa3e8d 100755
--- a/test/functional/test_framework/test_framework.py
+++ b/test/functional/test_framework/test_framework.py
@@ -103,7 +103,7 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass):
self.supports_cli = True
self.bind_to_localhost_only = True
self.parse_args()
- self.default_wallet_name = ""
+ self.default_wallet_name = "default_wallet" if self.options.descriptors else ""
self.wallet_data_filename = "wallet.dat"
# Optional list of wallet names that can be set in set_test_params to
# create and import keys to. If unset, default is len(nodes) *
diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py
index 2e757d7090..6b746b1fee 100755
--- a/test/functional/test_runner.py
+++ b/test/functional/test_runner.py
@@ -88,6 +88,7 @@ BASE_SCRIPTS = [
'wallet_hd.py',
'wallet_hd.py --descriptors',
'wallet_backup.py',
+ 'wallet_backup.py --descriptors',
# vv Tests less than 5m vv
'mining_getblocktemplate_longpoll.py',
'feature_maxuploadtarget.py',
@@ -140,6 +141,7 @@ BASE_SCRIPTS = [
'mempool_reorg.py',
'mempool_persist.py',
'wallet_multiwallet.py',
+ 'wallet_multiwallet.py --descriptors',
'wallet_multiwallet.py --usecli',
'wallet_createwallet.py',
'wallet_createwallet.py --usecli',
diff --git a/test/functional/wallet_backup.py b/test/functional/wallet_backup.py
index 36049dcb45..0cf8e6d926 100755
--- a/test/functional/wallet_backup.py
+++ b/test/functional/wallet_backup.py
@@ -135,11 +135,13 @@ class WalletBackupTest(BitcoinTestFramework):
self.log.info("Backing up")
self.nodes[0].backupwallet(os.path.join(self.nodes[0].datadir, 'wallet.bak'))
- self.nodes[0].dumpwallet(os.path.join(self.nodes[0].datadir, 'wallet.dump'))
self.nodes[1].backupwallet(os.path.join(self.nodes[1].datadir, 'wallet.bak'))
- self.nodes[1].dumpwallet(os.path.join(self.nodes[1].datadir, 'wallet.dump'))
self.nodes[2].backupwallet(os.path.join(self.nodes[2].datadir, 'wallet.bak'))
- self.nodes[2].dumpwallet(os.path.join(self.nodes[2].datadir, 'wallet.dump'))
+
+ if not self.options.descriptors:
+ self.nodes[0].dumpwallet(os.path.join(self.nodes[0].datadir, 'wallet.dump'))
+ self.nodes[1].dumpwallet(os.path.join(self.nodes[1].datadir, 'wallet.dump'))
+ self.nodes[2].dumpwallet(os.path.join(self.nodes[2].datadir, 'wallet.dump'))
self.log.info("More transactions")
for _ in range(5):
@@ -183,29 +185,30 @@ class WalletBackupTest(BitcoinTestFramework):
assert_equal(self.nodes[1].getbalance(), balance1)
assert_equal(self.nodes[2].getbalance(), balance2)
- self.log.info("Restoring using dumped wallet")
- self.stop_three()
- self.erase_three()
+ if not self.options.descriptors:
+ self.log.info("Restoring using dumped wallet")
+ self.stop_three()
+ self.erase_three()
- #start node2 with no chain
- shutil.rmtree(os.path.join(self.nodes[2].datadir, self.chain, 'blocks'))
- shutil.rmtree(os.path.join(self.nodes[2].datadir, self.chain, 'chainstate'))
+ #start node2 with no chain
+ shutil.rmtree(os.path.join(self.nodes[2].datadir, self.chain, 'blocks'))
+ shutil.rmtree(os.path.join(self.nodes[2].datadir, self.chain, 'chainstate'))
- self.start_three()
+ self.start_three()
- assert_equal(self.nodes[0].getbalance(), 0)
- assert_equal(self.nodes[1].getbalance(), 0)
- assert_equal(self.nodes[2].getbalance(), 0)
+ assert_equal(self.nodes[0].getbalance(), 0)
+ assert_equal(self.nodes[1].getbalance(), 0)
+ assert_equal(self.nodes[2].getbalance(), 0)
- self.nodes[0].importwallet(os.path.join(self.nodes[0].datadir, 'wallet.dump'))
- self.nodes[1].importwallet(os.path.join(self.nodes[1].datadir, 'wallet.dump'))
- self.nodes[2].importwallet(os.path.join(self.nodes[2].datadir, 'wallet.dump'))
+ self.nodes[0].importwallet(os.path.join(self.nodes[0].datadir, 'wallet.dump'))
+ self.nodes[1].importwallet(os.path.join(self.nodes[1].datadir, 'wallet.dump'))
+ self.nodes[2].importwallet(os.path.join(self.nodes[2].datadir, 'wallet.dump'))
- self.sync_blocks()
+ self.sync_blocks()
- assert_equal(self.nodes[0].getbalance(), balance0)
- assert_equal(self.nodes[1].getbalance(), balance1)
- assert_equal(self.nodes[2].getbalance(), balance2)
+ assert_equal(self.nodes[0].getbalance(), balance0)
+ assert_equal(self.nodes[1].getbalance(), balance1)
+ assert_equal(self.nodes[2].getbalance(), balance2)
# Backup to source wallet file must fail
sourcePaths = [
diff --git a/test/functional/wallet_multiwallet.py b/test/functional/wallet_multiwallet.py
index a0787dd289..61791a756c 100755
--- a/test/functional/wallet_multiwallet.py
+++ b/test/functional/wallet_multiwallet.py
@@ -60,8 +60,10 @@ class MultiWalletTest(BitcoinTestFramework):
wallet = lambda name: node.get_wallet_rpc(name)
def wallet_file(name):
+ if name == self.default_wallet_name:
+ return wallet_dir(self.default_wallet_name, self.wallet_data_filename)
if os.path.isdir(wallet_dir(name)):
- return wallet_dir(name, self.wallet_data_filename)
+ return wallet_dir(name, "wallet.dat")
return wallet_dir(name)
assert_equal(self.nodes[0].listwalletdir(), { 'wallets': [{ 'name': self.default_wallet_name }] })
@@ -77,13 +79,18 @@ class MultiWalletTest(BitcoinTestFramework):
# rename wallet.dat to make sure plain wallet file paths (as opposed to
# directory paths) can be loaded
- os.rename(wallet_dir(self.default_wallet_name, self.wallet_data_filename), wallet_dir("w8"))
-
# create another dummy wallet for use in testing backups later
- self.start_node(0, ["-nowallet", "-wallet=" + self.default_wallet_name])
+ self.start_node(0, ["-nowallet", "-wallet=empty", "-wallet=plain"])
+ node.createwallet("created")
self.stop_nodes()
empty_wallet = os.path.join(self.options.tmpdir, 'empty.dat')
- os.rename(wallet_dir(self.default_wallet_name, self.wallet_data_filename), empty_wallet)
+ os.rename(wallet_file("empty"), empty_wallet)
+ shutil.rmtree(wallet_dir("empty"))
+ empty_created_wallet = os.path.join(self.options.tmpdir, 'empty.created.dat')
+ os.rename(wallet_dir("created", self.wallet_data_filename), empty_created_wallet)
+ shutil.rmtree(wallet_dir("created"))
+ os.rename(wallet_file("plain"), wallet_dir("w8"))
+ shutil.rmtree(wallet_dir("plain"))
# restart node with a mix of wallet names:
# w1, w2, w3 - to verify new wallets created when non-existing paths specified
@@ -151,7 +158,7 @@ class MultiWalletTest(BitcoinTestFramework):
competing_wallet_dir = os.path.join(self.options.tmpdir, 'competing_walletdir')
os.mkdir(competing_wallet_dir)
self.restart_node(0, ['-walletdir=' + competing_wallet_dir])
- exp_stderr = r"Error: Error initializing wallet database environment \"\S+competing_walletdir\"!"
+ exp_stderr = r"Error: Error initializing wallet database environment \"\S+competing_walletdir\S*\"!"
self.nodes[1].assert_start_raises_init_error(['-walletdir=' + competing_wallet_dir], exp_stderr, match=ErrorMatch.PARTIAL_REGEX)
self.restart_node(0, extra_args)
@@ -246,12 +253,13 @@ class MultiWalletTest(BitcoinTestFramework):
assert_raises_rpc_error(-18, "Wallet file verification failed. Failed to load database path '{}'. Path does not exist.".format(path), self.nodes[0].loadwallet, 'wallets')
# Fail to load duplicate wallets
- path = os.path.join(self.options.tmpdir, "node0", "regtest", "wallets", "w1", self.wallet_data_filename)
+ path = os.path.join(self.options.tmpdir, "node0", "regtest", "wallets", "w1", "wallet.dat")
assert_raises_rpc_error(-4, "Wallet file verification failed. Refusing to load database. Data file '{}' is already loaded.".format(path), self.nodes[0].loadwallet, wallet_names[0])
# Fail to load duplicate wallets by different ways (directory and filepath)
- path = os.path.join(self.options.tmpdir, "node0", "regtest", "wallets", self.wallet_data_filename)
- assert_raises_rpc_error(-4, "Wallet file verification failed. Refusing to load database. Data file '{}' is already loaded.".format(path), self.nodes[0].loadwallet, self.wallet_data_filename)
+ if not self.options.descriptors:
+ path = os.path.join(self.options.tmpdir, "node0", "regtest", "wallets", "wallet.dat")
+ assert_raises_rpc_error(-4, "Wallet file verification failed. Refusing to load database. Data file '{}' is already loaded.".format(path), self.nodes[0].loadwallet, 'wallet.dat')
# Fail to load if one wallet is a copy of another
assert_raises_rpc_error(-4, "BerkeleyDatabase: Can't open database w8_copy (duplicates fileid", self.nodes[0].loadwallet, 'w8_copy')
@@ -334,9 +342,11 @@ class MultiWalletTest(BitcoinTestFramework):
rpc = self.nodes[0].get_wallet_rpc(wallet_name)
addr = rpc.getnewaddress()
backup = os.path.join(self.options.tmpdir, 'backup.dat')
+ if os.path.exists(backup):
+ os.unlink(backup)
rpc.backupwallet(backup)
self.nodes[0].unloadwallet(wallet_name)
- shutil.copyfile(empty_wallet, wallet_file(wallet_name))
+ shutil.copyfile(empty_created_wallet if wallet_name == self.default_wallet_name else empty_wallet, wallet_file(wallet_name))
self.nodes[0].loadwallet(wallet_name)
assert_equal(rpc.getaddressinfo(addr)['ismine'], False)
self.nodes[0].unloadwallet(wallet_name)
@@ -348,7 +358,10 @@ class MultiWalletTest(BitcoinTestFramework):
self.start_node(1)
wallet = os.path.join(self.options.tmpdir, 'my_wallet')
self.nodes[0].createwallet(wallet)
- assert_raises_rpc_error(-4, "Error initializing wallet database environment", self.nodes[1].loadwallet, wallet)
+ if self.options.descriptors:
+ assert_raises_rpc_error(-4, "Unable to obtain an exclusive lock", self.nodes[1].loadwallet, wallet)
+ else:
+ assert_raises_rpc_error(-4, "Error initializing wallet database environment", self.nodes[1].loadwallet, wallet)
self.nodes[0].unloadwallet(wallet)
self.nodes[1].loadwallet(wallet)