Merge #15382: util: add RunCommandParseJSON

31cf68a3ad1f0a5537c8419e2912b55fbfb88fa0 [util] add RunCommandParseJSON (Sjors Provoost)
c17f54ee535faaedf9033717403e1f775b5f1530 [ci] use boost::process (Sjors Provoost)
32128ba682033560d6eb2e4848a9f77a842016d2 [doc] include Doxygen comments for HAVE_BOOST_PROCESS (Sjors Provoost)
3c84d85f7d218fa27e9343c5cd1a55e519218980 [build] msvc: add boost::process (Sjors Provoost)
c47e4bbf0b44f2de1278f9538124ec98ee0815bb [build] make boost-process opt-in (Sjors Provoost)
929cda5470f98d1ef85c05b1cad4e2fb9227e3b0 configure: add ax_boost_process (Sjors Provoost)
8314c23d7b39fc36dde8b40b03b6efbe96f85698 [depends] boost: patch unused variable in boost_process (Sjors Provoost)

Pull request description:

  Prerequisite for external signer support in #16546. Big picture overview in [this gist](https://gist.github.com/Sjors/29d06728c685e6182828c1ce9b74483d).

  This adds a new dependency [boost process](https://github.com/boostorg/process/tree/boost-1.64.0). This is part of Boost since 1.64 which is part of `depends`. Because the minimum Boost version is 1.47, this functionality is skipped for older versions of Boost.

  Use `./configure --with-boost-process` to opt in, which checks for the presence of Boost::Process.

  We add `UniValue runCommandParseJSON(const std::string& strCommand)` to `system.{h,cpp}` which calls an arbitrary command and processes the JSON returned by it. This is currently only called by the test suite.

  ~For testing purposes this adds a new regtest-only RPC method `runcommand`, as well as `test/mocks/command.py` used by functional tests.~ (this is no longer the case)

  TODO:
  - [ ] review boost process in #15440

ACKs for top commit:
  achow101:
    ACK 31cf68a3ad1f0a5537c8419e2912b55fbfb88fa0
  hebasto:
    re-ACK 31cf68a3ad1f0a5537c8419e2912b55fbfb88fa0, only rebased (verified with `git range-diff`) and removed an unintentional tab character since the [previous](https://github.com/bitcoin/bitcoin/pull/15382#pullrequestreview-458371035) review.
  meshcollider:
    Very light utACK 31cf68a3ad1f0a5537c8419e2912b55fbfb88fa0, although I am not very confident with build stuff.
  promag:
    Code review ACK 31cf68a3ad1f0a5537c8419e2912b55fbfb88fa0, don't mind the nit.
  ryanofsky:
    Code review ACK 31cf68a3ad1f0a5537c8419e2912b55fbfb88fa0. I left some comments below that could be ignored or followed up later. The current change is clean and comprehensive.

Tree-SHA512: c506e747014b263606e1f538ed4624a8ad7bcf4e025cb700c12cc5739964e254dc04a2bbb848996b170e2ccec3fbfa4fe9e2b3976b191222cfb82fc3e6ab182d

4 files changed, 149 insertions, 0 deletions

diff --git a/src/Makefile.test.include b/src/Makefile.test.include
index b961f8d5b9..085ebdcb8e 100644
--- a/src/Makefile.test.include
+++ b/src/Makefile.test.include
@@ -265,6 +265,7 @@ BITCOIN_TESTS =\
   test/skiplist_tests.cpp \
   test/streams_tests.cpp \
   test/sync_tests.cpp \
+  test/system_tests.cpp \
   test/util_threadnames_tests.cpp \
   test/timedata_tests.cpp \
   test/torcontrol_tests.cpp \
diff --git a/src/test/system_tests.cpp b/src/test/system_tests.cpp
new file mode 100644
index 0000000000..a55145c738
--- /dev/null
+++ b/src/test/system_tests.cpp
@@ -0,0 +1,95 @@
+// Copyright (c) 2019 The Bitcoin Core developers
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or http://www.opensource.org/licenses/mit-license.php.
+//
+#include <test/util/setup_common.h>
+#include <util/system.h>
+#include <univalue.h>
+
+#ifdef HAVE_BOOST_PROCESS
+#include <boost/process.hpp>
+#endif // HAVE_BOOST_PROCESS
+
+#include <boost/test/unit_test.hpp>
+
+BOOST_FIXTURE_TEST_SUITE(system_tests, BasicTestingSetup)
+
+// At least one test is required (in case HAVE_BOOST_PROCESS is not defined).
+// Workaround for https://github.com/bitcoin/bitcoin/issues/19128
+BOOST_AUTO_TEST_CASE(dummy)
+{
+    BOOST_CHECK(true);
+}
+
+#ifdef HAVE_BOOST_PROCESS
+
+bool checkMessage(const std::runtime_error& ex)
+{
+    // On Linux & Mac: "No such file or directory"
+    // On Windows: "The system cannot find the file specified."
+    const std::string what(ex.what());
+    BOOST_CHECK(what.find("file") != std::string::npos);
+    return true;
+}
+
+bool checkMessageFalse(const std::runtime_error& ex)
+{
+    BOOST_CHECK_EQUAL(ex.what(), std::string("RunCommandParseJSON error: process(false) returned 1: \n"));
+    return true;
+}
+
+bool checkMessageStdErr(const std::runtime_error& ex)
+{
+    const std::string what(ex.what());
+    BOOST_CHECK(what.find("RunCommandParseJSON error:") != std::string::npos);
+    return checkMessage(ex);
+}
+
+BOOST_AUTO_TEST_CASE(run_command)
+{
+    {
+        const UniValue result = RunCommandParseJSON("");
+        BOOST_CHECK(result.isNull());
+    }
+    {
+#ifdef WIN32
+        // Windows requires single quotes to prevent escaping double quotes from the JSON...
+        const UniValue result = RunCommandParseJSON("echo '{\"success\": true}'");
+#else
+        // ... but Linux and macOS echo a single quote if it's used
+        const UniValue result = RunCommandParseJSON("echo \"{\"success\": true}\"");
+#endif
+        BOOST_CHECK(result.isObject());
+        const UniValue& success = find_value(result, "success");
+        BOOST_CHECK(!success.isNull());
+        BOOST_CHECK_EQUAL(success.getBool(), true);
+    }
+    {
+        // An invalid command is handled by Boost
+        BOOST_CHECK_EXCEPTION(RunCommandParseJSON("invalid_command"), boost::process::process_error, checkMessage); // Command failed
+    }
+    {
+        // Return non-zero exit code, no output to stderr
+        BOOST_CHECK_EXCEPTION(RunCommandParseJSON("false"), std::runtime_error, checkMessageFalse);
+    }
+    {
+        // Return non-zero exit code, with error message for stderr
+        BOOST_CHECK_EXCEPTION(RunCommandParseJSON("ls nosuchfile"), std::runtime_error, checkMessageStdErr);
+    }
+    {
+        BOOST_REQUIRE_THROW(RunCommandParseJSON("echo \"{\""), std::runtime_error); // Unable to parse JSON
+    }
+    // Test std::in, except for Windows
+#ifndef WIN32
+    {
+        const UniValue result = RunCommandParseJSON("cat", "{\"success\": true}");
+        BOOST_CHECK(result.isObject());
+        const UniValue& success = find_value(result, "success");
+        BOOST_CHECK(!success.isNull());
+        BOOST_CHECK_EQUAL(success.getBool(), true);
+    }
+#endif
+}
+#endif // HAVE_BOOST_PROCESS
+
+BOOST_AUTO_TEST_SUITE_END()
diff --git a/src/util/system.cpp b/src/util/system.cpp
index 8164e884b1..066cfe8892 100644
--- a/src/util/system.cpp
+++ b/src/util/system.cpp
@@ -6,6 +6,10 @@
 #include <sync.h>
 #include <util/system.h>
 
+#ifdef HAVE_BOOST_PROCESS
+#include <boost/process.hpp>
+#endif // HAVE_BOOST_PROCESS
+
 #include <chainparamsbase.h>
 #include <util/strencodings.h>
 #include <util/string.h>
@@ -1161,6 +1165,43 @@ void runCommand(const std::string& strCommand)
 }
 #endif
 
+#ifdef HAVE_BOOST_PROCESS
+UniValue RunCommandParseJSON(const std::string& str_command, const std::string& str_std_in)
+{
+    namespace bp = boost::process;
+
+    UniValue result_json;
+    bp::opstream stdin_stream;
+    bp::ipstream stdout_stream;
+    bp::ipstream stderr_stream;
+
+    if (str_command.empty()) return UniValue::VNULL;
+
+    bp::child c(
+        str_command,
+        bp::std_out > stdout_stream,
+        bp::std_err > stderr_stream,
+        bp::std_in < stdin_stream
+    );
+    if (!str_std_in.empty()) {
+        stdin_stream << str_std_in << std::endl;
+    }
+    stdin_stream.pipe().close();
+
+    std::string result;
+    std::string error;
+    std::getline(stdout_stream, result);
+    std::getline(stderr_stream, error);
+
+    c.wait();
+    const int n_error = c.exit_code();
+    if (n_error) throw std::runtime_error(strprintf("RunCommandParseJSON error: process(%s) returned %d: %s\n", str_command, n_error, error));
+    if (!result_json.read(result)) throw std::runtime_error("Unable to parse JSON: " + result);
+
+    return result_json;
+}
+#endif // HAVE_BOOST_PROCESS
+
 void SetupEnvironment()
 {
 #ifdef HAVE_MALLOPT_ARENA_MAX
diff --git a/src/util/system.h b/src/util/system.h
index 0bd14cc9ea..1df194ca84 100644
--- a/src/util/system.h
+++ b/src/util/system.h
@@ -37,6 +37,8 @@
 
 #include <boost/thread/condition_variable.hpp> // for boost::thread_interrupted
 
+class UniValue;
+
 // Application startup time (used for uptime calculation)
 int64_t GetStartupTime();
 
@@ -96,6 +98,16 @@ std::string ShellEscape(const std::string& arg);
 #if HAVE_SYSTEM
 void runCommand(const std::string& strCommand);
 #endif
+#ifdef HAVE_BOOST_PROCESS
+/**
+ * Execute a command which returns JSON, and parse the result.
+ *
+ * @param str_command The command to execute, including any arguments
+ * @param str_std_in string to pass to stdin
+ * @return parsed JSON
+ */
+UniValue RunCommandParseJSON(const std::string& str_command, const std::string& str_std_in="");
+#endif // HAVE_BOOST_PROCESS
 
 /**
  * Most paths passed as configuration arguments are treated as relative to