diff options
author | Wladimir J. van der Laan <laanwj@gmail.com> | 2016-06-10 15:19:51 +0200 |
---|---|---|
committer | Wladimir J. van der Laan <laanwj@gmail.com> | 2016-06-10 15:19:51 +0200 |
commit | 60ab9b200654ef0914459711cf2b22be16be3dc2 (patch) | |
tree | b884bbb13dd9c2e911ed27d33043be1084f97989 /lib | |
parent | 982709199f1b4e9e35211c419a81938f9f1dd4ed (diff) |
Squashed 'src/univalue/' changes from 2740c4f..f32df99
f32df99 Merge branch '2016_04_unicode' into bitcoin
280b191 Merge remote-tracking branch 'jgarzik/master' into bitcoin
c9a716c Handle UTF-8
bed8dd9 Version 1.0.2.
5e7985a Merge pull request #14 from laanwj/2015_11_escape_plan
git-subtree-dir: src/univalue
git-subtree-split: f32df99e96d99ab49e5eeda16cac93747d388245
Diffstat (limited to 'lib')
-rw-r--r-- | lib/univalue_read.cpp | 37 | ||||
-rw-r--r-- | lib/univalue_utffilter.h | 119 | ||||
-rw-r--r-- | lib/univalue_write.cpp | 11 |
3 files changed, 135 insertions, 32 deletions
diff --git a/lib/univalue_read.cpp b/lib/univalue_read.cpp index c7516b9628..95bac6958d 100644 --- a/lib/univalue_read.cpp +++ b/lib/univalue_read.cpp @@ -6,6 +6,7 @@ #include <vector> #include <stdio.h> #include "univalue.h" +#include "univalue_utffilter.h" using namespace std; @@ -174,41 +175,31 @@ enum jtokentype getJsonToken(string& tokenVal, unsigned int& consumed, raw++; // skip " string valStr; + JSONUTF8StringFilter writer(valStr); while (*raw) { - if (*raw < 0x20) + if ((unsigned char)*raw < 0x20) return JTOK_ERR; else if (*raw == '\\') { raw++; // skip backslash switch (*raw) { - case '"': valStr += "\""; break; - case '\\': valStr += "\\"; break; - case '/': valStr += "/"; break; - case 'b': valStr += "\b"; break; - case 'f': valStr += "\f"; break; - case 'n': valStr += "\n"; break; - case 'r': valStr += "\r"; break; - case 't': valStr += "\t"; break; + case '"': writer.push_back('\"'); break; + case '\\': writer.push_back('\\'); break; + case '/': writer.push_back('/'); break; + case 'b': writer.push_back('\b'); break; + case 'f': writer.push_back('\f'); break; + case 'n': writer.push_back('\n'); break; + case 'r': writer.push_back('\r'); break; + case 't': writer.push_back('\t'); break; case 'u': { unsigned int codepoint; if (hatoui(raw + 1, raw + 1 + 4, codepoint) != raw + 1 + 4) return JTOK_ERR; - - if (codepoint <= 0x7f) - valStr.push_back((char)codepoint); - else if (codepoint <= 0x7FF) { - valStr.push_back((char)(0xC0 | (codepoint >> 6))); - valStr.push_back((char)(0x80 | (codepoint & 0x3F))); - } else if (codepoint <= 0xFFFF) { - valStr.push_back((char)(0xE0 | (codepoint >> 12))); - valStr.push_back((char)(0x80 | ((codepoint >> 6) & 0x3F))); - valStr.push_back((char)(0x80 | (codepoint & 0x3F))); - } - + writer.push_back_u(codepoint); raw += 4; break; } @@ -226,11 +217,13 @@ enum jtokentype getJsonToken(string& tokenVal, unsigned int& consumed, } else { - valStr += *raw; + writer.push_back(*raw); raw++; } } + if (!writer.finalize()) + return JTOK_ERR; tokenVal = valStr; consumed = (raw - rawStart); return JTOK_STRING; diff --git a/lib/univalue_utffilter.h b/lib/univalue_utffilter.h new file mode 100644 index 0000000000..0e330dce9c --- /dev/null +++ b/lib/univalue_utffilter.h @@ -0,0 +1,119 @@ +// Copyright 2016 Wladimir J. van der Laan +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. +#ifndef UNIVALUE_UTFFILTER_H +#define UNIVALUE_UTFFILTER_H + +#include <string> + +/** + * Filter that generates and validates UTF-8, as well as collates UTF-16 + * surrogate pairs as specified in RFC4627. + */ +class JSONUTF8StringFilter +{ +public: + JSONUTF8StringFilter(std::string &s): + str(s), is_valid(true), codepoint(0), state(0), surpair(0) + { + } + // Write single 8-bit char (may be part of UTF-8 sequence) + void push_back(unsigned char ch) + { + if (state == 0) { + if (ch < 0x80) // 7-bit ASCII, fast direct pass-through + str.push_back(ch); + else if (ch < 0xc0) // Mid-sequence character, invalid in this state + is_valid = false; + else if (ch < 0xe0) { // Start of 2-byte sequence + codepoint = (ch & 0x1f) << 6; + state = 6; + } else if (ch < 0xf0) { // Start of 3-byte sequence + codepoint = (ch & 0x0f) << 12; + state = 12; + } else if (ch < 0xf8) { // Start of 4-byte sequence + codepoint = (ch & 0x07) << 18; + state = 18; + } else // Reserved, invalid + is_valid = false; + } else { + if ((ch & 0xc0) != 0x80) // Not a continuation, invalid + is_valid = false; + state -= 6; + codepoint |= (ch & 0x3f) << state; + if (state == 0) + push_back_u(codepoint); + } + } + // Write codepoint directly, possibly collating surrogate pairs + void push_back_u(unsigned int codepoint) + { + if (state) // Only accept full codepoints in open state + is_valid = false; + if (codepoint >= 0xD800 && codepoint < 0xDC00) { // First half of surrogate pair + if (surpair) // Two subsequent surrogate pair openers - fail + is_valid = false; + else + surpair = codepoint; + } else if (codepoint >= 0xDC00 && codepoint < 0xE000) { // Second half of surrogate pair + if (surpair) { // Open surrogate pair, expect second half + // Compute code point from UTF-16 surrogate pair + append_codepoint(0x10000 | ((surpair - 0xD800)<<10) | (codepoint - 0xDC00)); + surpair = 0; + } else // Second half doesn't follow a first half - fail + is_valid = false; + } else { + if (surpair) // First half of surrogate pair not followed by second - fail + is_valid = false; + else + append_codepoint(codepoint); + } + } + // Check that we're in a state where the string can be ended + // No open sequences, no open surrogate pairs, etc + bool finalize() + { + if (state || surpair) + is_valid = false; + return is_valid; + } +private: + std::string &str; + bool is_valid; + // Current UTF-8 decoding state + unsigned int codepoint; + int state; // Top bit to be filled in for next UTF-8 byte, or 0 + + // Keep track of the following state to handle the following section of + // RFC4627: + // + // To escape an extended character that is not in the Basic Multilingual + // Plane, the character is represented as a twelve-character sequence, + // encoding the UTF-16 surrogate pair. So, for example, a string + // containing only the G clef character (U+1D11E) may be represented as + // "\uD834\uDD1E". + // + // Two subsequent \u.... may have to be replaced with one actual codepoint. + unsigned int surpair; // First half of open UTF-16 surrogate pair, or 0 + + void append_codepoint(unsigned int codepoint) + { + if (codepoint <= 0x7f) + str.push_back((char)codepoint); + else if (codepoint <= 0x7FF) { + str.push_back((char)(0xC0 | (codepoint >> 6))); + str.push_back((char)(0x80 | (codepoint & 0x3F))); + } else if (codepoint <= 0xFFFF) { + str.push_back((char)(0xE0 | (codepoint >> 12))); + str.push_back((char)(0x80 | ((codepoint >> 6) & 0x3F))); + str.push_back((char)(0x80 | (codepoint & 0x3F))); + } else if (codepoint <= 0x1FFFFF) { + str.push_back((char)(0xF0 | (codepoint >> 18))); + str.push_back((char)(0x80 | ((codepoint >> 12) & 0x3F))); + str.push_back((char)(0x80 | ((codepoint >> 6) & 0x3F))); + str.push_back((char)(0x80 | (codepoint & 0x3F))); + } + } +}; + +#endif diff --git a/lib/univalue_write.cpp b/lib/univalue_write.cpp index ceb4cc9166..cfbdad3284 100644 --- a/lib/univalue_write.cpp +++ b/lib/univalue_write.cpp @@ -8,8 +8,6 @@ #include "univalue.h" #include "univalue_escapes.h" -// TODO: Using UTF8 - using namespace std; static string json_escape(const string& inS) @@ -23,15 +21,8 @@ static string json_escape(const string& inS) if (escStr) outS += escStr; - - else if (ch < 0x80) + else outS += ch; - - else { // TODO handle UTF-8 properly - char tmpesc[16]; - sprintf(tmpesc, "\\u%04x", ch); - outS += tmpesc; - } } return outS; |