diff --git a/src/lib_json/json_writer.cpp b/src/lib_json/json_writer.cpp index 72799445d..6acc177ee 100644 --- a/src/lib_json/json_writer.cpp +++ b/src/lib_json/json_writer.cpp @@ -131,9 +131,26 @@ static unsigned int utf8ToCodepoint(const char*& s, const char* e) { if (firstByte < 0x80) return firstByte; + // Checks that the `count` bytes following the lead byte are continuation + // bytes (10xxxxxx). On failure `s` is left on the last valid continuation + // byte, so the whole malformed prefix is replaced by a single U+FFFD and the + // offending byte is decoded on its own. + const auto hasTrailingBytes = [&s](int count) { + for (int i = 1; i <= count; ++i) { + if ((static_cast(s[i]) & 0xC0) != 0x80) { + s += i - 1; + return false; + } + } + return true; + }; + if (firstByte < 0xE0) { if (e - s < 2) return REPLACEMENT_CHARACTER; + // a malformed continuation byte does not belong to this sequence + if (!hasTrailingBytes(1)) + return REPLACEMENT_CHARACTER; unsigned int calculated = ((firstByte & 0x1F) << 6) | (static_cast(s[1]) & 0x3F); @@ -145,6 +162,8 @@ static unsigned int utf8ToCodepoint(const char*& s, const char* e) { if (firstByte < 0xF0) { if (e - s < 3) return REPLACEMENT_CHARACTER; + if (!hasTrailingBytes(2)) + return REPLACEMENT_CHARACTER; unsigned int calculated = ((firstByte & 0x0F) << 12) | ((static_cast(s[1]) & 0x3F) << 6) | @@ -161,12 +180,17 @@ static unsigned int utf8ToCodepoint(const char*& s, const char* e) { if (firstByte < 0xF8) { if (e - s < 4) return REPLACEMENT_CHARACTER; + if (!hasTrailingBytes(3)) + return REPLACEMENT_CHARACTER; unsigned int calculated = ((firstByte & 0x07) << 18) | ((static_cast(s[1]) & 0x3F) << 12) | ((static_cast(s[2]) & 0x3F) << 6) | (static_cast(s[3]) & 0x3F); s += 3; + // codepoints beyond U+10FFFF are invalid + if (calculated > 0x10FFFF) + return REPLACEMENT_CHARACTER; // oversized encoded characters are invalid return calculated < 0x10000 ? REPLACEMENT_CHARACTER : calculated; } diff --git a/src/test_lib_json/main.cpp b/src/test_lib_json/main.cpp index e09b87b84..c917224e7 100644 --- a/src/test_lib_json/main.cpp +++ b/src/test_lib_json/main.cpp @@ -2890,6 +2890,40 @@ JSONTEST_FIXTURE_LOCAL(StreamWriterTest, unicode) { "\"\\t\\n\\ud806\\udca1=\\u0133\\ud82c\\udd1b\\uff67\"\n}"); } +// Malformed UTF-8 must not swallow the bytes that follow a broken sequence. +JSONTEST_FIXTURE_LOCAL(StreamWriterTest, invalidUtf8) { + Json::StreamWriterBuilder b; + b.settings_["indentation"] = ""; + + // 0xE0 announces a 3-byte sequence, but 'A'/'B' are not continuation bytes: + // only the lead byte is replaced, the ASCII must be preserved. + Json::Value bad3(std::string("\xE0" + "AB")); + JSONTEST_ASSERT_STRING_EQUAL("\"\\ufffdAB\"", Json::writeString(b, bad3)); + + // 0xF0 announces a 4-byte sequence with no valid continuation bytes. + Json::Value bad4(std::string("\xF0" + "XYZ")); + JSONTEST_ASSERT_STRING_EQUAL("\"\\ufffdXYZ\"", Json::writeString(b, bad4)); + + // A valid continuation prefix cut short by a non-continuation byte is one + // broken sequence: a single replacement, and the ASCII is preserved. + Json::Value prefix3(std::string("\xE2\x82" + "A")); + JSONTEST_ASSERT_STRING_EQUAL("\"\\ufffdA\"", Json::writeString(b, prefix3)); + Json::Value prefix4(std::string("\xF0\x9F\x98" + "A")); + JSONTEST_ASSERT_STRING_EQUAL("\"\\ufffdA\"", Json::writeString(b, prefix4)); + + // A 4-byte sequence that decodes past U+10FFFF is not a valid codepoint. + Json::Value over(std::string("\xF7\xBF\xBF\xBF")); + JSONTEST_ASSERT_STRING_EQUAL("\"\\ufffd\"", Json::writeString(b, over)); + + // A valid multibyte sequence still round-trips unchanged. + Json::Value euro(std::string("\xE2\x82\xAC")); + JSONTEST_ASSERT_STRING_EQUAL("\"\\u20ac\"", Json::writeString(b, euro)); +} + // Control chars should be escaped regardless of UTF-8 input encoding. JSONTEST_FIXTURE_LOCAL(StreamWriterTest, escapeControlCharacters) { auto uEscape = [](unsigned ch) {