Add <datadir>/settings.json persistent settings storage.

Persistent settings are used in followup PRs #15936 to unify gui settings
between bitcoin-qt and bitcoind, and #15937 to add a load_on_startup flag to
the loadwallet RPC and maintain a dynamic list of wallets that should be loaded
on startup that also can be shared between bitcoind and bitcoin-qt.

13 files changed, 255 insertions, 6 deletions

diff --git a/doc/files.md b/doc/files.md
index cd23d547bb..5475826329 100644
--- a/doc/files.md
+++ b/doc/files.md
@@ -50,7 +50,7 @@ Subdirectory       | File(s)               | Description
 `indexes/blockfilter/basic/`    | `fltrNNNNN.dat`<sup>[\[2\]](#note2)</sup> | Blockfilter index filters for the basic filtertype; *optional*, used if `-blockfilterindex=basic`
 `wallets/`         |                       | [Contains wallets](#multi-wallet-environment); can be specified by `-walletdir` option; if `wallets/` subdirectory does not exist, a wallet resides in the data directory
 `./`               | `banlist.dat`         | Stores the IPs/subnets of banned nodes
-`./`               | `bitcoin.conf`        | Contains [configuration settings](bitcoin-conf.md) for `bitcoind` or `bitcoin-qt`; can be specified by `-conf` option
+`./`               | `bitcoin.conf`        | User-defined [configuration settings](bitcoin-conf.md) for `bitcoind` or `bitcoin-qt`. File is not written to by the software and must be created manually. Path can be specified by `-conf` option
 `./`               | `bitcoind.pid`        | Stores the process ID (PID) of `bitcoind` or `bitcoin-qt` while running; created at start and deleted on shutdown; can be specified by `-pid` option
 `./`               | `debug.log`           | Contains debug information and general logging generated by `bitcoind` or `bitcoin-qt`; can be specified by `-debuglogfile` option
 `./`               | `fee_estimates.dat`   | Stores statistics used to estimate minimum transaction fees and priorities required for confirmation
@@ -58,6 +58,7 @@ Subdirectory       | File(s)               | Description
 `./`               | `mempool.dat`         | Dump of the mempool's transactions
 `./`               | `onion_private_key`   | Cached Tor hidden service private key for `-listenonion` option
 `./`               | `peers.dat`           | Peer IP address database (custom format)
+`./`               | `settings.json`       | Read-write settings set through GUI or RPC interfaces, augmenting manual settings from [bitcoin.conf](bitcoin-conf.md). File is created automatically if read-write settings storage is not disabled with `-nosettings` option. Path can be specified with `-settings` option
 `./`               | `.cookie`             | Session RPC authentication cookie; if used, created at start and deleted on shutdown; can be specified by `-rpccookiefile` option
 `./`               | `.lock`               | Data directory lock file
 
diff --git a/src/bitcoind.cpp b/src/bitcoind.cpp
index 3dcce92ab5..b04cc12059 100644
--- a/src/bitcoind.cpp
+++ b/src/bitcoind.cpp
@@ -101,6 +101,11 @@ static bool AppInit(int argc, char* argv[])
             }
         }
 
+        if (!gArgs.InitSettings(error)) {
+            InitError(Untranslated(error));
+            return false;
+        }
+
         // -server defaults to true for bitcoind but not for the GUI so do this here
         gArgs.SoftSetBoolArg("-server", true);
         // Set this early so that parameter interactions go to console
diff --git a/src/init.cpp b/src/init.cpp
index 16f765574f..9308070dd8 100644
--- a/src/init.cpp
+++ b/src/init.cpp
@@ -397,7 +397,7 @@ void SetupServerArgs(NodeContext& node)
 #endif
     gArgs.AddArg("-blockreconstructionextratxn=<n>", strprintf("Extra transactions to keep in memory for compact block reconstructions (default: %u)", DEFAULT_BLOCK_RECONSTRUCTION_EXTRA_TXN), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
     gArgs.AddArg("-blocksonly", strprintf("Whether to reject transactions from network peers. Automatic broadcast and rebroadcast of any transactions from inbound peers is disabled, unless the peer has the 'forcerelay' permission. RPC transactions are not affected. (default: %u)", DEFAULT_BLOCKSONLY), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
-    gArgs.AddArg("-conf=<file>", strprintf("Specify configuration file. Relative paths will be prefixed by datadir location. (default: %s)", BITCOIN_CONF_FILENAME), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
+    gArgs.AddArg("-conf=<file>", strprintf("Specify path to read-only configuration file. Relative paths will be prefixed by datadir location. (default: %s)", BITCOIN_CONF_FILENAME), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
     gArgs.AddArg("-datadir=<dir>", "Specify data directory", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
     gArgs.AddArg("-dbbatchsize", strprintf("Maximum database write batch size in bytes (default: %u)", nDefaultDbBatchSize), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::OPTIONS);
     gArgs.AddArg("-dbcache=<n>", strprintf("Maximum database cache size <n> MiB (%d to %d, default: %d). In addition, unused mempool memory is shared for this cache (see -maxmempool).", nMinDbCache, nMaxDbCache, nDefaultDbCache), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
@@ -418,6 +418,7 @@ void SetupServerArgs(NodeContext& node)
             "(default: 0 = disable pruning blocks, 1 = allow manual pruning via RPC, >=%u = automatically prune block files to stay under the specified target size in MiB)", MIN_DISK_SPACE_FOR_BLOCK_FILES / 1024 / 1024), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
     gArgs.AddArg("-reindex", "Rebuild chain state and block index from the blk*.dat files on disk", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
     gArgs.AddArg("-reindex-chainstate", "Rebuild chain state from the currently indexed blocks. When in pruning mode or if blocks on disk might be corrupted, use full -reindex instead.", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
+    gArgs.AddArg("-settings=<file>", strprintf("Specify path to dynamic settings data file. Can be disabled with -nosettings. File is written at runtime and not meant to be edited by users (use %s instead for custom settings). Relative paths will be prefixed by datadir location. (default: %s)", BITCOIN_CONF_FILENAME, BITCOIN_SETTINGS_FILENAME), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
 #ifndef WIN32
     gArgs.AddArg("-sysperms", "Create new files with system default permissions, instead of umask 077 (only effective with disabled wallet functionality)", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
 #else
diff --git a/src/interfaces/node.cpp b/src/interfaces/node.cpp
index 834a16ecf5..33f0dac263 100644
--- a/src/interfaces/node.cpp
+++ b/src/interfaces/node.cpp
@@ -66,6 +66,7 @@ public:
     bool softSetArg(const std::string& arg, const std::string& value) override { return gArgs.SoftSetArg(arg, value); }
     bool softSetBoolArg(const std::string& arg, bool value) override { return gArgs.SoftSetBoolArg(arg, value); }
     void selectParams(const std::string& network) override { SelectParams(network); }
+    bool initSettings(std::string& error) override { return gArgs.InitSettings(error); }
     uint64_t getAssumedBlockchainSize() override { return Params().AssumedBlockchainSize(); }
     uint64_t getAssumedChainStateSize() override { return Params().AssumedChainStateSize(); }
     std::string getNetwork() override { return Params().NetworkIDString(); }
diff --git a/src/interfaces/node.h b/src/interfaces/node.h
index b88b5bc14e..a9680c42b5 100644
--- a/src/interfaces/node.h
+++ b/src/interfaces/node.h
@@ -66,6 +66,11 @@ public:
     //! Choose network parameters.
     virtual void selectParams(const std::string& network) = 0;
 
+    //! Read and update <datadir>/settings.json file with saved settings. This
+    //! needs to be called after selectParams() because the settings file
+    //! location is network-specific.
+    virtual bool initSettings(std::string& error) = 0;
+
     //! Get the (assumed) blockchain size.
     virtual uint64_t getAssumedBlockchainSize() = 0;
 
diff --git a/src/qt/bitcoin.cpp b/src/qt/bitcoin.cpp
index c7dd16d2ed..fe6b01ff72 100644
--- a/src/qt/bitcoin.cpp
+++ b/src/qt/bitcoin.cpp
@@ -528,6 +528,11 @@ int GuiMain(int argc, char* argv[])
     // Parse URIs on command line -- this can affect Params()
     PaymentServer::ipcParseCommandLine(*node, argc, argv);
 #endif
+    if (!node->initSettings(error)) {
+        node->initError(Untranslated(error));
+        QMessageBox::critical(nullptr, PACKAGE_NAME, QObject::tr("Error initializing settings: %1").arg(QString::fromStdString(error)));
+        return EXIT_FAILURE;
+    }
 
     QScopedPointer<const NetworkStyle> networkStyle(NetworkStyle::instantiate(Params().NetworkIDString()));
     assert(!networkStyle.isNull());
diff --git a/src/test/util_tests.cpp b/src/test/util_tests.cpp
index 257328974b..cc2a5fcdbd 100644
--- a/src/test/util_tests.cpp
+++ b/src/test/util_tests.cpp
@@ -9,6 +9,7 @@
 #include <key.h>  // For CKey
 #include <optional.h>
 #include <sync.h>
+#include <test/util/logging.h>
 #include <test/util/setup_common.h>
 #include <test/util/str.h>
 #include <uint256.h>
@@ -1128,6 +1129,28 @@ BOOST_FIXTURE_TEST_CASE(util_ChainMerge, ChainMergeTestingSetup)
     BOOST_CHECK_EQUAL(out_sha_hex, "f0b3a3c29869edc765d579c928f7f1690a71fbb673b49ccf39cbc4de18156a0d");
 }
 
+BOOST_AUTO_TEST_CASE(util_ReadWriteSettings)
+{
+    // Test writing setting.
+    TestArgsManager args1;
+    args1.LockSettings([&](util::Settings& settings) { settings.rw_settings["name"] = "value"; });
+    args1.WriteSettingsFile();
+
+    // Test reading setting.
+    TestArgsManager args2;
+    args2.ReadSettingsFile();
+    args2.LockSettings([&](util::Settings& settings) { BOOST_CHECK_EQUAL(settings.rw_settings["name"].get_str(), "value"); });
+
+    // Test error logging, and remove previously written setting.
+    {
+        ASSERT_DEBUG_LOG("Failed renaming settings file");
+        fs::remove(GetDataDir() / "settings.json");
+        fs::create_directory(GetDataDir() / "settings.json");
+        args2.WriteSettingsFile();
+        fs::remove(GetDataDir() / "settings.json");
+    }
+}
+
 BOOST_AUTO_TEST_CASE(util_FormatMoney)
 {
     BOOST_CHECK_EQUAL(FormatMoney(0), "0.00");
diff --git a/src/util/settings.cpp b/src/util/settings.cpp
index 34894e994e..b92b1d30c3 100644
--- a/src/util/settings.cpp
+++ b/src/util/settings.cpp
@@ -13,12 +13,13 @@ namespace {
 enum class Source {
    FORCED,
    COMMAND_LINE,
+   RW_SETTINGS,
    CONFIG_FILE_NETWORK_SECTION,
    CONFIG_FILE_DEFAULT_SECTION
 };
 
 //! Merge settings from multiple sources in precedence order:
-//! Forced config > command line > config file network-specific section > config file default section
+//! Forced config > command line > read-write settings file > config file network-specific section > config file default section
 //!
 //! This function is provided with a callback function fn that contains
 //! specific logic for how to merge the sources.
@@ -33,6 +34,10 @@ static void MergeSettings(const Settings& settings, const std::string& section,
     if (auto* values = FindKey(settings.command_line_options, name)) {
         fn(SettingsSpan(*values), Source::COMMAND_LINE);
     }
+    // Merge in the read-write settings
+    if (const SettingsValue* value = FindKey(settings.rw_settings, name)) {
+        fn(SettingsSpan(*value), Source::RW_SETTINGS);
+    }
     // Merge in the network-specific section of the config file
     if (!section.empty()) {
         if (auto* map = FindKey(settings.ro_config, section)) {
diff --git a/src/util/settings.h b/src/util/settings.h
index ee160e1ac8..ed36349232 100644
--- a/src/util/settings.h
+++ b/src/util/settings.h
@@ -26,13 +26,15 @@ namespace util {
 //!       https://github.com/bitcoin/bitcoin/pull/15934/files#r337691812)
 using SettingsValue = UniValue;
 
-//! Stored bitcoin settings. This struct combines settings from the command line
-//! and a read-only configuration file.
+//! Stored settings. This struct combines settings from the command line, a
+//! read-only configuration file, and a read-write runtime settings file.
 struct Settings {
     //! Map of setting name to forced setting value.
     std::map<std::string, SettingsValue> forced_settings;
     //! Map of setting name to list of command line values.
     std::map<std::string, std::vector<SettingsValue>> command_line_options;
+    //! Map of setting name to read-write file setting value.
+    std::map<std::string, SettingsValue> rw_settings;
     //! Map of config section name and setting name to list of config file values.
     std::map<std::string, std::map<std::string, std::vector<SettingsValue>>> ro_config;
 };
@@ -48,7 +50,7 @@ bool WriteSettings(const fs::path& path,
     std::vector<std::string>& errors);
 
 //! Get settings value from combined sources: forced settings, command line
-//! arguments and the read-only config file.
+//! arguments, runtime read-write settings, and the read-only config file.
 //!
 //! @param ignore_default_section_config - ignore values in the default section
 //!                                        of the config file (part before any
diff --git a/src/util/system.cpp b/src/util/system.cpp
index 7e7ba840cd..8164e884b1 100644
--- a/src/util/system.cpp
+++ b/src/util/system.cpp
@@ -73,6 +73,7 @@
 const int64_t nStartupTime = GetTime();
 
 const char * const BITCOIN_CONF_FILENAME = "bitcoin.conf";
+const char * const BITCOIN_SETTINGS_FILENAME = "settings.json";
 
 ArgsManager gArgs;
 
@@ -372,6 +373,84 @@ bool ArgsManager::IsArgSet(const std::string& strArg) const
     return !GetSetting(strArg).isNull();
 }
 
+bool ArgsManager::InitSettings(std::string& error)
+{
+    if (!GetSettingsPath()) {
+        return true; // Do nothing if settings file disabled.
+    }
+
+    std::vector<std::string> errors;
+    if (!ReadSettingsFile(&errors)) {
+        error = strprintf("Failed loading settings file:\n- %s\n", Join(errors, "\n- "));
+        return false;
+    }
+    if (!WriteSettingsFile(&errors)) {
+        error = strprintf("Failed saving settings file:\n- %s\n", Join(errors, "\n- "));
+        return false;
+    }
+    return true;
+}
+
+bool ArgsManager::GetSettingsPath(fs::path* filepath, bool temp) const
+{
+    if (IsArgNegated("-settings")) {
+        return false;
+    }
+    if (filepath) {
+        std::string settings = GetArg("-settings", BITCOIN_SETTINGS_FILENAME);
+        *filepath = fs::absolute(temp ? settings + ".tmp" : settings, GetDataDir(/* net_specific= */ true));
+    }
+    return true;
+}
+
+static void SaveErrors(const std::vector<std::string> errors, std::vector<std::string>* error_out)
+{
+    for (const auto& error : errors) {
+        if (error_out) {
+            error_out->emplace_back(error);
+        } else {
+            LogPrintf("%s\n", error);
+        }
+    }
+}
+
+bool ArgsManager::ReadSettingsFile(std::vector<std::string>* errors)
+{
+    fs::path path;
+    if (!GetSettingsPath(&path, /* temp= */ false)) {
+        return true; // Do nothing if settings file disabled.
+    }
+
+    LOCK(cs_args);
+    m_settings.rw_settings.clear();
+    std::vector<std::string> read_errors;
+    if (!util::ReadSettings(path, m_settings.rw_settings, read_errors)) {
+        SaveErrors(read_errors, errors);
+        return false;
+    }
+    return true;
+}
+
+bool ArgsManager::WriteSettingsFile(std::vector<std::string>* errors) const
+{
+    fs::path path, path_tmp;
+    if (!GetSettingsPath(&path, /* temp= */ false) || !GetSettingsPath(&path_tmp, /* temp= */ true)) {
+        throw std::logic_error("Attempt to write settings file when dynamic settings are disabled.");
+    }
+
+    LOCK(cs_args);
+    std::vector<std::string> write_errors;
+    if (!util::WriteSettings(path_tmp, m_settings.rw_settings, write_errors)) {
+        SaveErrors(write_errors, errors);
+        return false;
+    }
+    if (!RenameOver(path_tmp, path)) {
+        SaveErrors({strprintf("Failed renaming settings file %s to %s\n", path_tmp.string(), path.string())}, errors);
+        return false;
+    }
+    return true;
+}
+
 bool ArgsManager::IsArgNegated(const std::string& strArg) const
 {
     return GetSetting(strArg).isFalse();
@@ -893,6 +972,9 @@ void ArgsManager::LogArgs() const
     for (const auto& section : m_settings.ro_config) {
         logArgsPrefix("Config file arg:", section.first, section.second);
     }
+    for (const auto& setting : m_settings.rw_settings) {
+        LogPrintf("Setting file arg: %s = %s\n", setting.first, setting.second.write());
+    }
     logArgsPrefix("Command-line arg:", "", m_settings.command_line_options);
 }
 
diff --git a/src/util/system.h b/src/util/system.h
index a5eea5dfab..0bd14cc9ea 100644
--- a/src/util/system.h
+++ b/src/util/system.h
@@ -41,6 +41,7 @@
 int64_t GetStartupTime();
 
 extern const char * const BITCOIN_CONF_FILENAME;
+extern const char * const BITCOIN_SETTINGS_FILENAME;
 
 void SetupEnvironment();
 bool SetupNetworking();
@@ -334,6 +335,39 @@ public:
     Optional<unsigned int> GetArgFlags(const std::string& name) const;
 
     /**
+     * Read and update settings file with saved settings. This needs to be
+     * called after SelectParams() because the settings file location is
+     * network-specific.
+     */
+    bool InitSettings(std::string& error);
+
+    /**
+     * Get settings file path, or return false if read-write settings were
+     * disabled with -nosettings.
+     */
+    bool GetSettingsPath(fs::path* filepath = nullptr, bool temp = false) const;
+
+    /**
+     * Read settings file. Push errors to vector, or log them if null.
+     */
+    bool ReadSettingsFile(std::vector<std::string>* errors = nullptr);
+
+    /**
+     * Write settings file. Push errors to vector, or log them if null.
+     */
+    bool WriteSettingsFile(std::vector<std::string>* errors = nullptr) const;
+
+    /**
+     * Access settings with lock held.
+     */
+    template <typename Fn>
+    void LockSettings(Fn&& fn)
+    {
+        LOCK(cs_args);
+        fn(m_settings);
+    }
+
+    /**
      * Log the config file options and the command line arguments,
      * useful for troubleshooting.
      */
diff --git a/test/functional/feature_settings.py b/test/functional/feature_settings.py
new file mode 100755
index 0000000000..c565854bb0
--- /dev/null
+++ b/test/functional/feature_settings.py
@@ -0,0 +1,84 @@
+#!/usr/bin/env python3
+# Copyright (c) 2017-2020 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+"""Test various command line arguments and configuration file parameters."""
+
+import json
+
+from pathlib import Path
+
+from test_framework.test_framework import BitcoinTestFramework
+from test_framework.test_node import ErrorMatch
+from test_framework.util import assert_equal
+
+
+class SettingsTest(BitcoinTestFramework):
+    def set_test_params(self):
+        self.setup_clean_chain = True
+        self.num_nodes = 1
+
+    def run_test(self):
+        node, = self.nodes
+        settings = Path(node.datadir, self.chain, "settings.json")
+        conf = Path(node.datadir, "bitcoin.conf")
+
+        # Assert empty settings file was created
+        self.stop_node(0)
+        with settings.open() as fp:
+            assert_equal(json.load(fp), {})
+
+        # Assert settings are parsed and logged
+        with settings.open("w") as fp:
+            json.dump({"string": "string", "num": 5, "bool": True, "null": None, "list": [6,7]}, fp)
+        with node.assert_debug_log(expected_msgs=[
+                'Setting file arg: string = "string"',
+                'Setting file arg: num = 5',
+                'Setting file arg: bool = true',
+                'Setting file arg: null = null',
+                'Setting file arg: list = [6,7]']):
+            self.start_node(0)
+            self.stop_node(0)
+
+        # Assert settings are unchanged after shutdown
+        with settings.open() as fp:
+            assert_equal(json.load(fp), {"string": "string", "num": 5, "bool": True, "null": None, "list": [6,7]})
+
+        # Test invalid json
+        with settings.open("w") as fp:
+            fp.write("invalid json")
+        node.assert_start_raises_init_error(expected_msg='Unable to parse settings file', match=ErrorMatch.PARTIAL_REGEX)
+
+        # Test invalid json object
+        with settings.open("w") as fp:
+            fp.write('"string"')
+        node.assert_start_raises_init_error(expected_msg='Found non-object value "string" in settings file', match=ErrorMatch.PARTIAL_REGEX)
+
+        # Test invalid settings file containing duplicate keys
+        with settings.open("w") as fp:
+            fp.write('{"key": 1, "key": 2}')
+        node.assert_start_raises_init_error(expected_msg='Found duplicate key key in settings file', match=ErrorMatch.PARTIAL_REGEX)
+
+        # Test invalid settings file is ignored with command line -nosettings
+        with node.assert_debug_log(expected_msgs=['Command-line arg: settings=false']):
+            self.start_node(0, extra_args=["-nosettings"])
+            self.stop_node(0)
+
+        # Test invalid settings file is ignored with config file -nosettings
+        with conf.open('a') as conf:
+            conf.write('nosettings=1\n')
+        with node.assert_debug_log(expected_msgs=['Config file arg: [regtest] settings=false']):
+            self.start_node(0)
+            self.stop_node(0)
+
+        # Test alternate settings path
+        altsettings = Path(node.datadir, "altsettings.json")
+        with altsettings.open("w") as fp:
+            fp.write('{"key": "value"}')
+        with node.assert_debug_log(expected_msgs=['Setting file arg: key = "value"']):
+            self.start_node(0, extra_args=["-settings={}".format(altsettings)])
+            self.stop_node(0)
+
+
+if __name__ == '__main__':
+    SettingsTest().main()
diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py
index 867a9909a8..d2d177b290 100755
--- a/test/functional/test_runner.py
+++ b/test/functional/test_runner.py
@@ -243,6 +243,7 @@ BASE_SCRIPTS = [
     'p2p_permissions.py',
     'feature_blocksdir.py',
     'feature_config_args.py',
+    'feature_settings.py',
     'rpc_getdescriptorinfo.py',
     'rpc_getpeerinfo_banscore_deprecation.py',
     'rpc_help.py',