Bump univalue and fix json formatting in tests

This merge commit bumps the univalue subtree and also updates the whitespace
for some failing tests.

5 files changed, 249 insertions, 207 deletions

diff --git a/src/univalue/lib/univalue.cpp b/src/univalue/lib/univalue.cpp
index 5a2860c13f..d8ad7c4b90 100644
--- a/src/univalue/lib/univalue.cpp
+++ b/src/univalue/lib/univalue.cpp
@@ -4,75 +4,12 @@
 // file COPYING or http://www.opensource.org/licenses/mit-license.php.
 
 #include <stdint.h>
-#include <errno.h>
 #include <iomanip>
-#include <limits>
 #include <sstream>
-#include <stdexcept>
 #include <stdlib.h>
-#include <string.h>
 
 #include "univalue.h"
 
-namespace 
-{
-static bool ParsePrechecks(const std::string& str)
-{
-    if (str.empty()) // No empty string allowed
-        return false;
-    if (str.size() >= 1 && (json_isspace(str[0]) || json_isspace(str[str.size()-1]))) // No padding allowed
-        return false;
-    if (str.size() != strlen(str.c_str())) // No embedded NUL characters allowed
-        return false;
-    return true;
-}
-
-bool ParseInt32(const std::string& str, int32_t *out)
-{
-    if (!ParsePrechecks(str))
-        return false;
-    char *endp = NULL;
-    errno = 0; // strtol will not set errno if valid
-    long int n = strtol(str.c_str(), &endp, 10);
-    if(out) *out = (int32_t)n;
-    // Note that strtol returns a *long int*, so even if strtol doesn't report a over/underflow
-    // we still have to check that the returned value is within the range of an *int32_t*. On 64-bit
-    // platforms the size of these types may be different.
-    return endp && *endp == 0 && !errno &&
-        n >= std::numeric_limits<int32_t>::min() &&
-        n <= std::numeric_limits<int32_t>::max();
-}
-
-bool ParseInt64(const std::string& str, int64_t *out)
-{
-    if (!ParsePrechecks(str))
-        return false;
-    char *endp = NULL;
-    errno = 0; // strtoll will not set errno if valid
-    long long int n = strtoll(str.c_str(), &endp, 10);
-    if(out) *out = (int64_t)n;
-    // Note that strtoll returns a *long long int*, so even if strtol doesn't report a over/underflow
-    // we still have to check that the returned value is within the range of an *int64_t*.
-    return endp && *endp == 0 && !errno &&
-        n >= std::numeric_limits<int64_t>::min() &&
-        n <= std::numeric_limits<int64_t>::max();
-}
-
-bool ParseDouble(const std::string& str, double *out)
-{
-    if (!ParsePrechecks(str))
-        return false;
-    if (str.size() >= 2 && str[0] == '0' && str[1] == 'x') // No hexadecimal floats allowed
-        return false;
-    std::istringstream text(str);
-    text.imbue(std::locale::classic());
-    double result;
-    text >> result;
-    if(out) *out = result;
-    return text.eof() && !text.fail();
-}
-}
-
 using namespace std;
 
 const UniValue NullUniValue;
@@ -104,7 +41,7 @@ static bool validNumStr(const string& s)
 {
     string tokenVal;
     unsigned int consumed;
-    enum jtokentype tt = getJsonToken(tokenVal, consumed, s.c_str());
+    enum jtokentype tt = getJsonToken(tokenVal, consumed, s.data(), s.data() + s.size());
     return (tt == JTOK_NUMBER);
 }
 
@@ -189,13 +126,22 @@ bool UniValue::push_backV(const std::vector<UniValue>& vec)
     return true;
 }
 
+void UniValue::__pushKV(const std::string& key, const UniValue& val_)
+{
+    keys.push_back(key);
+    values.push_back(val_);
+}
+
 bool UniValue::pushKV(const std::string& key, const UniValue& val_)
 {
     if (typ != VOBJ)
         return false;
 
-    keys.push_back(key);
-    values.push_back(val_);
+    size_t idx;
+    if (findKey(key, idx))
+        values[idx] = val_;
+    else
+        __pushKV(key, val_);
     return true;
 }
 
@@ -204,30 +150,43 @@ bool UniValue::pushKVs(const UniValue& obj)
     if (typ != VOBJ || obj.typ != VOBJ)
         return false;
 
-    for (unsigned int i = 0; i < obj.keys.size(); i++) {
-        keys.push_back(obj.keys[i]);
-        values.push_back(obj.values.at(i));
-    }
+    for (size_t i = 0; i < obj.keys.size(); i++)
+        __pushKV(obj.keys[i], obj.values.at(i));
 
     return true;
 }
 
-int UniValue::findKey(const std::string& key) const
+void UniValue::getObjMap(std::map<std::string,UniValue>& kv) const
+{
+    if (typ != VOBJ)
+        return;
+
+    kv.clear();
+    for (size_t i = 0; i < keys.size(); i++)
+        kv[keys[i]] = values[i];
+}
+
+bool UniValue::findKey(const std::string& key, size_t& retIdx) const
 {
-    for (unsigned int i = 0; i < keys.size(); i++) {
-        if (keys[i] == key)
-            return (int) i;
+    for (size_t i = 0; i < keys.size(); i++) {
+        if (keys[i] == key) {
+            retIdx = i;
+            return true;
+        }
     }
 
-    return -1;
+    return false;
 }
 
-bool UniValue::checkObject(const std::map<std::string,UniValue::VType>& t)
+bool UniValue::checkObject(const std::map<std::string,UniValue::VType>& t) const
 {
+    if (typ != VOBJ)
+        return false;
+
     for (std::map<std::string,UniValue::VType>::const_iterator it = t.begin();
          it != t.end(); ++it) {
-        int idx = findKey(it->first);
-        if (idx < 0)
+        size_t idx = 0;
+        if (!findKey(it->first, idx))
             return false;
 
         if (values.at(idx).getType() != it->second)
@@ -242,14 +201,14 @@ const UniValue& UniValue::operator[](const std::string& key) const
     if (typ != VOBJ)
         return NullUniValue;
 
-    int index = findKey(key);
-    if (index < 0)
+    size_t index = 0;
+    if (!findKey(key, index))
         return NullUniValue;
 
     return values.at(index);
 }
 
-const UniValue& UniValue::operator[](unsigned int index) const
+const UniValue& UniValue::operator[](size_t index) const
 {
     if (typ != VOBJ && typ != VARR)
         return NullUniValue;
@@ -283,75 +242,3 @@ const UniValue& find_value(const UniValue& obj, const std::string& name)
     return NullUniValue;
 }
 
-const std::vector<std::string>& UniValue::getKeys() const
-{
-    if (typ != VOBJ)
-        throw std::runtime_error("JSON value is not an object as expected");
-    return keys;
-}
-
-const std::vector<UniValue>& UniValue::getValues() const
-{
-    if (typ != VOBJ && typ != VARR)
-        throw std::runtime_error("JSON value is not an object or array as expected");
-    return values;
-}
-
-bool UniValue::get_bool() const
-{
-    if (typ != VBOOL)
-        throw std::runtime_error("JSON value is not a boolean as expected");
-    return getBool();
-}
-
-const std::string& UniValue::get_str() const
-{
-    if (typ != VSTR)
-        throw std::runtime_error("JSON value is not a string as expected");
-    return getValStr();
-}
-
-int UniValue::get_int() const
-{
-    if (typ != VNUM)
-        throw std::runtime_error("JSON value is not an integer as expected");
-    int32_t retval;
-    if (!ParseInt32(getValStr(), &retval))
-        throw std::runtime_error("JSON integer out of range");
-    return retval;
-}
-
-int64_t UniValue::get_int64() const
-{
-    if (typ != VNUM)
-        throw std::runtime_error("JSON value is not an integer as expected");
-    int64_t retval;
-    if (!ParseInt64(getValStr(), &retval))
-        throw std::runtime_error("JSON integer out of range");
-    return retval;
-}
-
-double UniValue::get_real() const
-{
-    if (typ != VNUM)
-        throw std::runtime_error("JSON value is not a number as expected");
-    double retval;
-    if (!ParseDouble(getValStr(), &retval))
-        throw std::runtime_error("JSON double out of range");
-    return retval;
-}
-
-const UniValue& UniValue::get_obj() const
-{
-    if (typ != VOBJ)
-        throw std::runtime_error("JSON value is not an object as expected");
-    return *this;
-}
-
-const UniValue& UniValue::get_array() const
-{
-    if (typ != VARR)
-        throw std::runtime_error("JSON value is not an array as expected");
-    return *this;
-}
-
diff --git a/src/univalue/lib/univalue_get.cpp b/src/univalue/lib/univalue_get.cpp
new file mode 100644
index 0000000000..eabcf2dad1
--- /dev/null
+++ b/src/univalue/lib/univalue_get.cpp
@@ -0,0 +1,147 @@
+// Copyright 2014 BitPay Inc.
+// Copyright 2015 Bitcoin Core Developers
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+#include <stdint.h>
+#include <errno.h>
+#include <string.h>
+#include <stdlib.h>
+#include <stdexcept>
+#include <vector>
+#include <limits>
+#include <string>
+
+#include "univalue.h"
+
+namespace
+{
+static bool ParsePrechecks(const std::string& str)
+{
+    if (str.empty()) // No empty string allowed
+        return false;
+    if (str.size() >= 1 && (json_isspace(str[0]) || json_isspace(str[str.size()-1]))) // No padding allowed
+        return false;
+    if (str.size() != strlen(str.c_str())) // No embedded NUL characters allowed
+        return false;
+    return true;
+}
+
+bool ParseInt32(const std::string& str, int32_t *out)
+{
+    if (!ParsePrechecks(str))
+        return false;
+    char *endp = NULL;
+    errno = 0; // strtol will not set errno if valid
+    long int n = strtol(str.c_str(), &endp, 10);
+    if(out) *out = (int32_t)n;
+    // Note that strtol returns a *long int*, so even if strtol doesn't report a over/underflow
+    // we still have to check that the returned value is within the range of an *int32_t*. On 64-bit
+    // platforms the size of these types may be different.
+    return endp && *endp == 0 && !errno &&
+        n >= std::numeric_limits<int32_t>::min() &&
+        n <= std::numeric_limits<int32_t>::max();
+}
+
+bool ParseInt64(const std::string& str, int64_t *out)
+{
+    if (!ParsePrechecks(str))
+        return false;
+    char *endp = NULL;
+    errno = 0; // strtoll will not set errno if valid
+    long long int n = strtoll(str.c_str(), &endp, 10);
+    if(out) *out = (int64_t)n;
+    // Note that strtoll returns a *long long int*, so even if strtol doesn't report a over/underflow
+    // we still have to check that the returned value is within the range of an *int64_t*.
+    return endp && *endp == 0 && !errno &&
+        n >= std::numeric_limits<int64_t>::min() &&
+        n <= std::numeric_limits<int64_t>::max();
+}
+
+bool ParseDouble(const std::string& str, double *out)
+{
+    if (!ParsePrechecks(str))
+        return false;
+    if (str.size() >= 2 && str[0] == '0' && str[1] == 'x') // No hexadecimal floats allowed
+        return false;
+    std::istringstream text(str);
+    text.imbue(std::locale::classic());
+    double result;
+    text >> result;
+    if(out) *out = result;
+    return text.eof() && !text.fail();
+}
+}
+
+const std::vector<std::string>& UniValue::getKeys() const
+{
+    if (typ != VOBJ)
+        throw std::runtime_error("JSON value is not an object as expected");
+    return keys;
+}
+
+const std::vector<UniValue>& UniValue::getValues() const
+{
+    if (typ != VOBJ && typ != VARR)
+        throw std::runtime_error("JSON value is not an object or array as expected");
+    return values;
+}
+
+bool UniValue::get_bool() const
+{
+    if (typ != VBOOL)
+        throw std::runtime_error("JSON value is not a boolean as expected");
+    return getBool();
+}
+
+const std::string& UniValue::get_str() const
+{
+    if (typ != VSTR)
+        throw std::runtime_error("JSON value is not a string as expected");
+    return getValStr();
+}
+
+int UniValue::get_int() const
+{
+    if (typ != VNUM)
+        throw std::runtime_error("JSON value is not an integer as expected");
+    int32_t retval;
+    if (!ParseInt32(getValStr(), &retval))
+        throw std::runtime_error("JSON integer out of range");
+    return retval;
+}
+
+int64_t UniValue::get_int64() const
+{
+    if (typ != VNUM)
+        throw std::runtime_error("JSON value is not an integer as expected");
+    int64_t retval;
+    if (!ParseInt64(getValStr(), &retval))
+        throw std::runtime_error("JSON integer out of range");
+    return retval;
+}
+
+double UniValue::get_real() const
+{
+    if (typ != VNUM)
+        throw std::runtime_error("JSON value is not a number as expected");
+    double retval;
+    if (!ParseDouble(getValStr(), &retval))
+        throw std::runtime_error("JSON double out of range");
+    return retval;
+}
+
+const UniValue& UniValue::get_obj() const
+{
+    if (typ != VOBJ)
+        throw std::runtime_error("JSON value is not an object as expected");
+    return *this;
+}
+
+const UniValue& UniValue::get_array() const
+{
+    if (typ != VARR)
+        throw std::runtime_error("JSON value is not an array as expected");
+    return *this;
+}
+
diff --git a/src/univalue/lib/univalue_read.cpp b/src/univalue/lib/univalue_read.cpp
index 95bac6958d..ae75cb462a 100644
--- a/src/univalue/lib/univalue_read.cpp
+++ b/src/univalue/lib/univalue_read.cpp
@@ -43,21 +43,21 @@ static const char *hatoui(const char *first, const char *last,
 }
 
 enum jtokentype getJsonToken(string& tokenVal, unsigned int& consumed,
-                            const char *raw)
+                            const char *raw, const char *end)
 {
     tokenVal.clear();
     consumed = 0;
 
     const char *rawStart = raw;
 
-    while ((*raw) && (json_isspace(*raw)))             // skip whitespace
+    while (raw < end && (json_isspace(*raw)))          // skip whitespace
         raw++;
 
-    switch (*raw) {
-
-    case 0:
+    if (raw >= end)
         return JTOK_NONE;
 
+    switch (*raw) {
+
     case '{':
         raw++;
         consumed = (raw - rawStart);
@@ -127,40 +127,40 @@ enum jtokentype getJsonToken(string& tokenVal, unsigned int& consumed,
         numStr += *raw;                       // copy first char
         raw++;
 
-        if ((*first == '-') && (!json_isdigit(*raw)))
+        if ((*first == '-') && (raw < end) && (!json_isdigit(*raw)))
             return JTOK_ERR;
 
-        while ((*raw) && json_isdigit(*raw)) {     // copy digits
+        while (raw < end && json_isdigit(*raw)) {  // copy digits
             numStr += *raw;
             raw++;
         }
 
         // part 2: frac
-        if (*raw == '.') {
+        if (raw < end && *raw == '.') {
             numStr += *raw;                   // copy .
             raw++;
 
-            if (!json_isdigit(*raw))
+            if (raw >= end || !json_isdigit(*raw))
                 return JTOK_ERR;
-            while ((*raw) && json_isdigit(*raw)) { // copy digits
+            while (raw < end && json_isdigit(*raw)) { // copy digits
                 numStr += *raw;
                 raw++;
             }
         }
 
         // part 3: exp
-        if (*raw == 'e' || *raw == 'E') {
+        if (raw < end && (*raw == 'e' || *raw == 'E')) {
             numStr += *raw;                   // copy E
             raw++;
 
-            if (*raw == '-' || *raw == '+') { // copy +/-
+            if (raw < end && (*raw == '-' || *raw == '+')) { // copy +/-
                 numStr += *raw;
                 raw++;
             }
 
-            if (!json_isdigit(*raw))
+            if (raw >= end || !json_isdigit(*raw))
                 return JTOK_ERR;
-            while ((*raw) && json_isdigit(*raw)) { // copy digits
+            while (raw < end && json_isdigit(*raw)) { // copy digits
                 numStr += *raw;
                 raw++;
             }
@@ -177,13 +177,16 @@ enum jtokentype getJsonToken(string& tokenVal, unsigned int& consumed,
         string valStr;
         JSONUTF8StringFilter writer(valStr);
 
-        while (*raw) {
-            if ((unsigned char)*raw < 0x20)
+        while (true) {
+            if (raw >= end || (unsigned char)*raw < 0x20)
                 return JTOK_ERR;
 
             else if (*raw == '\\') {
                 raw++;                        // skip backslash
 
+                if (raw >= end)
+                    return JTOK_ERR;
+
                 switch (*raw) {
                 case '"':  writer.push_back('\"'); break;
                 case '\\': writer.push_back('\\'); break;
@@ -196,7 +199,8 @@ enum jtokentype getJsonToken(string& tokenVal, unsigned int& consumed,
 
                 case 'u': {
                     unsigned int codepoint;
-                    if (hatoui(raw + 1, raw + 1 + 4, codepoint) !=
+                    if (raw + 1 + 4 >= end ||
+                        hatoui(raw + 1, raw + 1 + 4, codepoint) !=
                                raw + 1 + 4)
                         return JTOK_ERR;
                     writer.push_back_u(codepoint);
@@ -246,7 +250,7 @@ enum expect_bits {
 #define setExpect(bit) (expectMask |= EXP_##bit)
 #define clearExpect(bit) (expectMask &= ~EXP_##bit)
 
-bool UniValue::read(const char *raw)
+bool UniValue::read(const char *raw, size_t size)
 {
     clear();
 
@@ -257,10 +261,11 @@ bool UniValue::read(const char *raw)
     unsigned int consumed;
     enum jtokentype tok = JTOK_NONE;
     enum jtokentype last_tok = JTOK_NONE;
+    const char* end = raw + size;
     do {
         last_tok = tok;
 
-        tok = getJsonToken(tokenVal, consumed, raw);
+        tok = getJsonToken(tokenVal, consumed, raw, end);
         if (tok == JTOK_NONE || tok == JTOK_ERR)
             return false;
         raw += consumed;
@@ -371,9 +376,6 @@ bool UniValue::read(const char *raw)
         case JTOK_KW_NULL:
         case JTOK_KW_TRUE:
         case JTOK_KW_FALSE: {
-            if (!stack.size())
-                return false;
-
             UniValue tmpVal;
             switch (tok) {
             case JTOK_KW_NULL:
@@ -388,6 +390,11 @@ bool UniValue::read(const char *raw)
             default: /* impossible */ break;
             }
 
+            if (!stack.size()) {
+                *this = tmpVal;
+                break;
+            }
+
             UniValue *top = stack.back();
             top->values.push_back(tmpVal);
 
@@ -396,10 +403,12 @@ bool UniValue::read(const char *raw)
             }
 
         case JTOK_NUMBER: {
-            if (!stack.size())
-                return false;
-
             UniValue tmpVal(VNUM, tokenVal);
+            if (!stack.size()) {
+                *this = tmpVal;
+                break;
+            }
+
             UniValue *top = stack.back();
             top->values.push_back(tmpVal);
 
@@ -408,17 +417,18 @@ bool UniValue::read(const char *raw)
             }
 
         case JTOK_STRING: {
-            if (!stack.size())
-                return false;
-
-            UniValue *top = stack.back();
-
             if (expect(OBJ_NAME)) {
+                UniValue *top = stack.back();
                 top->keys.push_back(tokenVal);
                 clearExpect(OBJ_NAME);
                 setExpect(COLON);
             } else {
                 UniValue tmpVal(VSTR, tokenVal);
+                if (!stack.size()) {
+                    *this = tmpVal;
+                    break;
+                }
+                UniValue *top = stack.back();
                 top->values.push_back(tmpVal);
             }
 
@@ -432,7 +442,7 @@ bool UniValue::read(const char *raw)
     } while (!stack.empty ());
 
     /* Check that nothing follows the initial construct (parsed above).  */
-    tok = getJsonToken(tokenVal, consumed, raw);
+    tok = getJsonToken(tokenVal, consumed, raw, end);
     if (tok != JTOK_NONE)
         return false;
 
diff --git a/src/univalue/lib/univalue_utffilter.h b/src/univalue/lib/univalue_utffilter.h
index 2fb6a492d1..20d4043009 100644
--- a/src/univalue/lib/univalue_utffilter.h
+++ b/src/univalue/lib/univalue_utffilter.h
@@ -46,19 +46,19 @@ public:
         }
     }
     // Write codepoint directly, possibly collating surrogate pairs
-    void push_back_u(unsigned int codepoint)
+    void push_back_u(unsigned int codepoint_)
     {
         if (state) // Only accept full codepoints in open state
             is_valid = false;
-        if (codepoint >= 0xD800 && codepoint < 0xDC00) { // First half of surrogate pair
+        if (codepoint_ >= 0xD800 && codepoint_ < 0xDC00) { // First half of surrogate pair
             if (surpair) // Two subsequent surrogate pair openers - fail
                 is_valid = false;
             else
-                surpair = codepoint;
-        } else if (codepoint >= 0xDC00 && codepoint < 0xE000) { // Second half of surrogate pair
+                surpair = codepoint_;
+        } else if (codepoint_ >= 0xDC00 && codepoint_ < 0xE000) { // Second half of surrogate pair
             if (surpair) { // Open surrogate pair, expect second half
                 // Compute code point from UTF-16 surrogate pair
-                append_codepoint(0x10000 | ((surpair - 0xD800)<<10) | (codepoint - 0xDC00));
+                append_codepoint(0x10000 | ((surpair - 0xD800)<<10) | (codepoint_ - 0xDC00));
                 surpair = 0;
             } else // Second half doesn't follow a first half - fail
                 is_valid = false;
@@ -66,7 +66,7 @@ public:
             if (surpair) // First half of surrogate pair not followed by second - fail
                 is_valid = false;
             else
-                append_codepoint(codepoint);
+                append_codepoint(codepoint_);
         }
     }
     // Check that we're in a state where the string can be ended
@@ -96,22 +96,22 @@ private:
     //  Two subsequent \u.... may have to be replaced with one actual codepoint.
     unsigned int surpair; // First half of open UTF-16 surrogate pair, or 0
 
-    void append_codepoint(unsigned int codepoint)
+    void append_codepoint(unsigned int codepoint_)
     {
-        if (codepoint <= 0x7f)
-            str.push_back((char)codepoint);
-        else if (codepoint <= 0x7FF) {
-            str.push_back((char)(0xC0 | (codepoint >> 6)));
-            str.push_back((char)(0x80 | (codepoint & 0x3F)));
-        } else if (codepoint <= 0xFFFF) {
-            str.push_back((char)(0xE0 | (codepoint >> 12)));
-            str.push_back((char)(0x80 | ((codepoint >> 6) & 0x3F)));
-            str.push_back((char)(0x80 | (codepoint & 0x3F)));
-        } else if (codepoint <= 0x1FFFFF) {
-            str.push_back((char)(0xF0 | (codepoint >> 18)));
-            str.push_back((char)(0x80 | ((codepoint >> 12) & 0x3F)));
-            str.push_back((char)(0x80 | ((codepoint >> 6) & 0x3F)));
-            str.push_back((char)(0x80 | (codepoint & 0x3F)));
+        if (codepoint_ <= 0x7f)
+            str.push_back((char)codepoint_);
+        else if (codepoint_ <= 0x7FF) {
+            str.push_back((char)(0xC0 | (codepoint_ >> 6)));
+            str.push_back((char)(0x80 | (codepoint_ & 0x3F)));
+        } else if (codepoint_ <= 0xFFFF) {
+            str.push_back((char)(0xE0 | (codepoint_ >> 12)));
+            str.push_back((char)(0x80 | ((codepoint_ >> 6) & 0x3F)));
+            str.push_back((char)(0x80 | (codepoint_ & 0x3F)));
+        } else if (codepoint_ <= 0x1FFFFF) {
+            str.push_back((char)(0xF0 | (codepoint_ >> 18)));
+            str.push_back((char)(0x80 | ((codepoint_ >> 12) & 0x3F)));
+            str.push_back((char)(0x80 | ((codepoint_ >> 6) & 0x3F)));
+            str.push_back((char)(0x80 | (codepoint_ & 0x3F)));
         }
     }
 };
diff --git a/src/univalue/lib/univalue_write.cpp b/src/univalue/lib/univalue_write.cpp
index cfbdad3284..cf27835991 100644
--- a/src/univalue/lib/univalue_write.cpp
+++ b/src/univalue/lib/univalue_write.cpp
@@ -79,8 +79,6 @@ void UniValue::writeArray(unsigned int prettyIndent, unsigned int indentLevel, s
         s += values[i].write(prettyIndent, indentLevel + 1);
         if (i != (values.size() - 1)) {
             s += ",";
-            if (prettyIndent)
-                s += " ";
         }
         if (prettyIndent)
             s += "\n";