validate utf-8 continuation bytes in utf8ToCodepoint

src/lib_json/json_writer.cpp

@@ -131,9 +131,26 @@ static unsigned int utf8ToCodepoint(const char*& s, const char* e) {
   if (firstByte < 0x80)
     return firstByte;
 
+  // Checks that the `count` bytes following the lead byte are continuation
+  // bytes (10xxxxxx). On failure `s` is left on the last valid continuation
+  // byte, so the whole malformed prefix is replaced by a single U+FFFD and the
+  // offending byte is decoded on its own.
+  const auto hasTrailingBytes = [&s](int count) {
+    for (int i = 1; i <= count; ++i) {
+      if ((static_cast<unsigned char>(s[i]) & 0xC0) != 0x80) {
+        s += i - 1;
+        return false;
+      }
+    }
+    return true;
+  };
+
   if (firstByte < 0xE0) {
     if (e - s < 2)
       return REPLACEMENT_CHARACTER;
+    // a malformed continuation byte does not belong to this sequence
+    if (!hasTrailingBytes(1))
+      return REPLACEMENT_CHARACTER;
 
     unsigned int calculated =
         ((firstByte & 0x1F) << 6) | (static_cast<unsigned int>(s[1]) & 0x3F);
@@ -145,6 +162,8 @@ static unsigned int utf8ToCodepoint(const char*& s, const char* e) {
   if (firstByte < 0xF0) {
     if (e - s < 3)
       return REPLACEMENT_CHARACTER;
+    if (!hasTrailingBytes(2))
+      return REPLACEMENT_CHARACTER;
 
     unsigned int calculated = ((firstByte & 0x0F) << 12) |
                               ((static_cast<unsigned int>(s[1]) & 0x3F) << 6) |
@@ -161,12 +180,17 @@ static unsigned int utf8ToCodepoint(const char*& s, const char* e) {
   if (firstByte < 0xF8) {
     if (e - s < 4)
       return REPLACEMENT_CHARACTER;
+    if (!hasTrailingBytes(3))
+      return REPLACEMENT_CHARACTER;
 
     unsigned int calculated = ((firstByte & 0x07) << 18) |
                               ((static_cast<unsigned int>(s[1]) & 0x3F) << 12) |
                               ((static_cast<unsigned int>(s[2]) & 0x3F) << 6) |
                               (static_cast<unsigned int>(s[3]) & 0x3F);
     s += 3;
+    // codepoints beyond U+10FFFF are invalid
+    if (calculated > 0x10FFFF)
+      return REPLACEMENT_CHARACTER;
     // oversized encoded characters are invalid
     return calculated < 0x10000 ? REPLACEMENT_CHARACTER : calculated;
   }

src/test_lib_json/main.cpp

@@ -2890,6 +2890,40 @@ JSONTEST_FIXTURE_LOCAL(StreamWriterTest, unicode) {
                   "\"\\t\\n\\ud806\\udca1=\\u0133\\ud82c\\udd1b\\uff67\"\n}");
 }
 
+// Malformed UTF-8 must not swallow the bytes that follow a broken sequence.
+JSONTEST_FIXTURE_LOCAL(StreamWriterTest, invalidUtf8) {
+  Json::StreamWriterBuilder b;
+  b.settings_["indentation"] = "";
+
+  // 0xE0 announces a 3-byte sequence, but 'A'/'B' are not continuation bytes:
+  // only the lead byte is replaced, the ASCII must be preserved.
+  Json::Value bad3(std::string("\xE0"
+                               "AB"));
+  JSONTEST_ASSERT_STRING_EQUAL("\"\\ufffdAB\"", Json::writeString(b, bad3));
+
+  // 0xF0 announces a 4-byte sequence with no valid continuation bytes.
+  Json::Value bad4(std::string("\xF0"
+                               "XYZ"));
+  JSONTEST_ASSERT_STRING_EQUAL("\"\\ufffdXYZ\"", Json::writeString(b, bad4));
+
+  // A valid continuation prefix cut short by a non-continuation byte is one
+  // broken sequence: a single replacement, and the ASCII is preserved.
+  Json::Value prefix3(std::string("\xE2\x82"
+                                  "A"));
+  JSONTEST_ASSERT_STRING_EQUAL("\"\\ufffdA\"", Json::writeString(b, prefix3));
+  Json::Value prefix4(std::string("\xF0\x9F\x98"
+                                  "A"));
+  JSONTEST_ASSERT_STRING_EQUAL("\"\\ufffdA\"", Json::writeString(b, prefix4));
+
+  // A 4-byte sequence that decodes past U+10FFFF is not a valid codepoint.
+  Json::Value over(std::string("\xF7\xBF\xBF\xBF"));
+  JSONTEST_ASSERT_STRING_EQUAL("\"\\ufffd\"", Json::writeString(b, over));
+
+  // A valid multibyte sequence still round-trips unchanged.
+  Json::Value euro(std::string("\xE2\x82\xAC"));
+  JSONTEST_ASSERT_STRING_EQUAL("\"\\u20ac\"", Json::writeString(b, euro));
+}
+
 // Control chars should be escaped regardless of UTF-8 input encoding.
 JSONTEST_FIXTURE_LOCAL(StreamWriterTest, escapeControlCharacters) {
   auto uEscape = [](unsigned ch) {