MDEV-39143 Replicate MySQL binary JSON to a MariaDB slave

mysql-test/suite/rpl/r/rpl_from_mysql80.result

@@ -1,6 +1,7 @@
 include/master-slave.inc
 [connection master]
 connection slave;
+INSTALL SONAME 'type_mysql_json';
 include/stop_slave.inc
 connection master;
 include/rpl_stop_server.inc [server_number=1]
@@ -25,7 +26,6 @@ CHANGE MASTER TO Master_log_file='master-bin.000001', Master_log_pos=2297;
 START SLAVE IO_THREAD;
 START SLAVE SQL_THREAD;
 include/wait_for_slave_io_to_start.inc
-include/wait_for_slave_sql_error.inc [errno=1594]
 SELECT * FROM t1 ORDER BY a;
 a	b	c
 1	0	
@@ -35,16 +35,16 @@ a	b	c
 5	0	after compressed
 SELECT * FROM t2 ORDER BY a;
 a	b
-1	{"a": "hulu", "b": "[zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy]", "c": "bulu"}
-STOP SLAVE IO_THREAD;
-include/wait_for_slave_io_to_stop.inc
+1	{"a": "hulu", "b": "oOooOooOooOooOooOooOooOooOooOooOooOooOooOooOooOooOooOooOooOooOooOooOooOooOooOooOooOooOooOooOooOooOooOooOooOooOooOooOooOooOooOooOooOooOooOooOooOooOooOo", "c": "bulu"}
+STOP SLAVE;
+include/wait_for_slave_to_stop.inc
 CHANGE MASTER TO Master_log_file='master-bin.000002', Master_log_pos=4;
 START SLAVE IO_THREAD;
 START SLAVE SQL_THREAD;
 include/wait_for_slave_io_to_start.inc
 include/wait_for_slave_sql_to_start.inc
 DROP TABLE t1, t2;
+UNINSTALL SONAME 'type_mysql_json';
 CALL mtr.add_suppression('TRANSACTION_PAYLOAD_EVENT event. You can avoid this event by specifying');
-CALL mtr.add_suppression('PARTIAL_UPDATE_ROWS_EVENT event. You can avoid this event by specifying');
 connection master;
 include/rpl_end.inc

mysql-test/suite/rpl/r/rpl_json_partial_insert.result

@@ -0,0 +1,36 @@
+include/master-slave.inc
+[connection master]
+connection slave;
+INSTALL SONAME 'type_mysql_json';
+include/stop_slave.inc
+connection master;
+include/rpl_stop_server.inc [server_number=1]
+include/rpl_start_server.inc [server_number=1]
+connection slave;
+CHANGE MASTER TO Master_log_file='master-bin.000001', Master_log_pos=4, Master_use_gtid=No;
+call mtr.add_suppression("MDEV-39143: cannot apply a partial JSON update");
+call mtr.add_suppression("Slave SQL.*Error executing row event");
+START SLAVE;
+include/wait_for_slave_sql_error.inc [errno=1105]
+# Slave stopped, Last_SQL_Error matches the expected refusal:
+# Could not execute MySQL Update_rows_partial event on table jsontest.t1; MDEV-39143: cannot apply a partial JSON update that inserts at a from-the-end array index: path '$.arr[last-9]' on jsontest.t1.j. MariaDB's JSON_ARRAY_INSERT resolves '[last-N]' differently from MySQL, so the result would silently differ from the source., Error_code: 1105; handler error HA_ERR_GENERIC; the event's master log master-bin.000001, end_log_pos 2453
+#
+# Rows applied before the halt: id=1 got its object-member INSERT and
+# id=2 its array append.  id=3 and id=4 keep the inserted values.
+SELECT id, JSON_QUERY(j,'$.arr') AS arr, JSON_VALUE(j,'$.newkey') AS newkey
+FROM jsontest.t1 ORDER BY id;
+id	arr	newkey
+1	[1, 2, 3]	added
+2	[1, 2, 3, 99]	NULL
+3	[1, 2, 3]	NULL
+4	[1, 2, 3]	NULL
+connection slave;
+STOP SLAVE IO_THREAD;
+include/wait_for_slave_io_to_stop.inc
+DROP DATABASE jsontest;
+CHANGE MASTER TO Master_log_file='master-bin.000002', Master_log_pos=4, Master_use_gtid=No;
+START SLAVE;
+connection master;
+connection slave;
+UNINSTALL SONAME 'type_mysql_json';
+include/rpl_end.inc

mysql-test/suite/rpl/r/rpl_json_partial_update.result

@@ -0,0 +1,27 @@
+include/master-slave.inc
+[connection master]
+connection slave;
+INSTALL SONAME 'type_mysql_json';
+include/stop_slave.inc
+connection master;
+include/rpl_stop_server.inc [server_number=1]
+include/rpl_start_server.inc [server_number=1]
+connection slave;
+CHANGE MASTER TO Master_log_file='master-bin.000001', Master_log_pos=4, Master_use_gtid=No;
+START SLAVE;
+# id=1: inserted as a full document, then modified by four partial-JSON
+# events (REPLACE of members a and n twice, REMOVE of arr element 2).
+SELECT id, JSON_VALUE(j,'$.a') AS a, JSON_VALUE(j,'$.n') AS n,
+JSON_QUERY(j,'$.arr') AS arr FROM testdb.tj ORDER BY id;
+id	a	n	arr
+1	zzzz	44	[1, 2]
+connection slave;
+STOP SLAVE;
+include/wait_for_slave_to_stop.inc
+DROP DATABASE testdb;
+CHANGE MASTER TO Master_log_file='master-bin.000002', Master_log_pos=4, Master_use_gtid=No;
+START SLAVE;
+connection master;
+connection slave;
+UNINSTALL SONAME 'type_mysql_json';
+include/rpl_end.inc

mysql-test/suite/rpl/t/rpl_from_mysql80.test

@@ -59,6 +59,7 @@
 
 
 --connection slave
+INSTALL SONAME 'type_mysql_json';
 --source include/stop_slave.inc
 
 --connection master
@@ -99,14 +100,26 @@ CHANGE MASTER TO Master_log_file='master-bin.000001', Master_log_pos=2297;
 START SLAVE IO_THREAD;
 START SLAVE SQL_THREAD;
 --source include/wait_for_slave_io_to_start.inc
---let $slave_sql_errno= 1594
---source include/wait_for_slave_sql_error.inc
+
+# MDEV-39143: PARTIAL_UPDATE_ROWS_EVENT is now applied instead of rejected.
+# Wait for the JSON_REPLACE partial update (the last event in the injected
+# file) to reach t2, rather than for a fixed position, since the exact byte
+# offset is fragile against any future re-recording of the fixture.
+# LIKE avoids '$' which mysqltest would interpolate inside --let; the
+# repeating unit is 3 chars ('oOo'), so 'oOooOo' is the substring it produces.
+--let $wait_condition= SELECT COUNT(*)=1 FROM test.t2 WHERE a=1 AND b LIKE '%oOooOo%'
+--source include/wait_condition.inc
 
 SELECT * FROM t1 ORDER BY a;
 SELECT * FROM t2 ORDER BY a;
 
-STOP SLAVE IO_THREAD;
---source include/wait_for_slave_io_to_stop.inc
+# The SQL thread is genuinely caught up and running now that the partial
+# JSON update applies instead of erroring, so both threads must be stopped
+# before CHANGE MASTER (STOP SLAVE IO_THREAD alone left it running here
+# before MDEV-39143, when the SQL thread had already halted on the event 39
+# rejection).
+STOP SLAVE;
+--source include/wait_for_slave_to_stop.inc
 # Restart replication after the MySQL 8.0 file.
 CHANGE MASTER TO Master_log_file='master-bin.000002', Master_log_pos=4;
 START SLAVE IO_THREAD;
@@ -116,8 +129,8 @@ START SLAVE SQL_THREAD;
 --sync_with_master
 
 DROP TABLE t1, t2;
+UNINSTALL SONAME 'type_mysql_json';
 CALL mtr.add_suppression('TRANSACTION_PAYLOAD_EVENT event. You can avoid this event by specifying');
-CALL mtr.add_suppression('PARTIAL_UPDATE_ROWS_EVENT event. You can avoid this event by specifying');
 
 --connection master
 --source include/rpl_end.inc

mysql-test/suite/rpl/t/rpl_json_partial_insert.test

@@ -0,0 +1,95 @@
+# MDEV-39143: INSERT diffs in a MySQL partial JSON update.
+#
+# Companion to rpl_json_partial_update.test and driven the same way - see the
+# header there for how the recorded MySQL binlog is injected.  The fixture,
+# std_data/mdev39143_partial_json_insert.000001, was recorded on MySQL 9.2
+# with binlog_row_value_options=PARTIAL_JSON from:
+#
+#   UPDATE t1 SET j=JSON_SET(j,'$.newkey','added')  WHERE id=1;
+#   UPDATE t1 SET j=JSON_SET(j,'$.arr[7]',99)       WHERE id=2;
+#   UPDATE t1 SET j=JSON_SET(j,'$.arr[last-9]',77)  WHERE id=3;
+#   UPDATE t1 SET j=JSON_SET(j,'$.arr[1]',55)       WHERE id=4;
+#   UPDATE t1 SET j=JSON_REMOVE(j,'$.arr[2]')       WHERE id=4;
+#
+# MySQL logs the first as an INSERT diff on an object-member path and the
+# next two as INSERT diffs on array-cell paths; its own mysqlbinlog renders
+# them as JSON_INSERT, JSON_ARRAY_INSERT and JSON_ARRAY_INSERT respectively.
+# The slave must make the same distinction: id=3 is the discriminating case,
+# where the path resolves before the start of the array so the value is
+# prepended and the existing elements shift right.  JSON_INSERT would leave
+# the document untouched there, silently.
+#
+# MySQL's own contents after those updates, read from the source table:
+#   id=1 arr [1, 2, 3]     newkey "added"
+#   id=2 arr [1, 2, 3, 99]
+#   id=3 arr [77, 1, 2, 3]
+#   id=4 arr [1, 55]
+#
+# id=3 cannot be applied correctly yet and the slave deliberately stops on
+# it.  MySQL emits an array-cell INSERT only for a cell that does not exist,
+# which for '[last-9]' means it clamped the position to 0 and prepended.
+# MariaDB's JSON_ARRAY_INSERT resolves a from-the-end index differently - it
+# appends once the index goes negative, and is off by one even in range -
+# while JSON_EXTRACT, JSON_REPLACE and JSON_REMOVE all agree with MySQL on
+# the same paths.  That looks like a pre-existing JSON_ARRAY_INSERT bug
+# reachable from plain SQL, so rather than store a document that silently
+# differs from the source, the apply path refuses the diff and halts.
+#
+# This test therefore asserts the halt, and that the diffs before it applied
+# correctly: id=1 (object member) and id=2 (array append, where both
+# functions agree).  id=4's REPLACE and REMOVE come after the failing event
+# and are not reached.  When JSON_ARRAY_INSERT is fixed, drop the guard in
+# apply_partial_json_column() and this test should expect MySQL's values
+# above for all four rows.
+
+--source include/have_innodb.inc
+--source include/have_binlog_format_row.inc
+--source include/master-slave.inc
+
+--connection slave
+INSTALL SONAME 'type_mysql_json';
+--source include/stop_slave.inc
+
+--connection master
+--let $datadir= `SELECT @@datadir`
+--let $rpl_server_number= 1
+--source include/rpl_stop_server.inc
+--remove_file $datadir/master-bin.000001
+--copy_file $MYSQL_TEST_DIR/std_data/mdev39143_partial_json_insert.000001 $datadir/master-bin.000001
+--source include/rpl_start_server.inc
+
+--connection slave
+CHANGE MASTER TO Master_log_file='master-bin.000001', Master_log_pos=4, Master_use_gtid=No;
+call mtr.add_suppression("MDEV-39143: cannot apply a partial JSON update");
+call mtr.add_suppression("Slave SQL.*Error executing row event");
+START SLAVE;
+
+# The slave must stop on the id=3 diff rather than apply it.
+--let $slave_sql_errno= 1105
+--source include/wait_for_slave_sql_error.inc
+--let $err= query_get_value(SHOW SLAVE STATUS, Last_SQL_Error, 1)
+--echo # Slave stopped, Last_SQL_Error matches the expected refusal:
+--echo # $err
+
+--echo #
+--echo # Rows applied before the halt: id=1 got its object-member INSERT and
+--echo # id=2 its array append.  id=3 and id=4 keep the inserted values.
+SELECT id, JSON_QUERY(j,'$.arr') AS arr, JSON_VALUE(j,'$.newkey') AS newkey
+  FROM jsontest.t1 ORDER BY id;
+
+# Cleanup.
+# The SQL thread is already stopped by the error above, so only
+# the IO thread needs stopping before CHANGE MASTER; wait_for_slave_to_stop
+# would itself fail on the pending Last_SQL_Error.
+--connection slave
+STOP SLAVE IO_THREAD;
+--source include/wait_for_slave_io_to_stop.inc
+DROP DATABASE jsontest;
+CHANGE MASTER TO Master_log_file='master-bin.000002', Master_log_pos=4, Master_use_gtid=No;
+START SLAVE;
+--connection master
+--save_master_pos
+--connection slave
+--sync_with_master
+UNINSTALL SONAME 'type_mysql_json';
+--source include/rpl_end.inc

mysql-test/suite/rpl/t/rpl_json_partial_update.test

@@ -0,0 +1,75 @@
+# MDEV-39143: replicate MySQL binary JSON (full documents and partial diffs)
+# from a MySQL master to a MariaDB slave.
+#
+# No MySQL server runs here.  Both servers started by master-slave.inc below
+# are MariaDB; the MySQL side is supplied by a binlog recorded once on a real
+# MySQL 9.2 server and checked in as
+# std_data/mdev39143_partial_json.000001.  The test stops the MariaDB master,
+# replaces the master's own master-bin.000001 with that file, and restarts it.
+# The master is then only a file server for those bytes, and the slave pulls
+# them over the ordinary replication protocol.  What is under test is the
+# slave apply path: PARTIAL_UPDATE_ROWS_EVENT (39) parsing, Json_diff
+# decoding, and diff application - see unpack_row() in sql/rpl_record.cc.
+#
+# Important points:
+#  - The recorded events are fixed.  The SQL the MySQL master ran is not
+#    visible in this file, only its effect (described at the SELECT below);
+#    extending coverage needs a MySQL server, a re-recording, and a new file.
+#  - The Json_diff encoding exercised is MySQL 9.2's specifically.
+#  - have_binlog_format_row.inc constrains the MariaDB master's own binlog
+#    format; the injected events are row events regardless of it.
+--source include/have_innodb.inc
+--source include/have_binlog_format_row.inc
+--source include/master-slave.inc
+
+# The slave decodes MySQL binary JSON with the read-only mysql_json plugin,
+# which is not installed by default.  Uninstalled again in the teardown.
+--connection slave
+INSTALL SONAME 'type_mysql_json';
+--source include/stop_slave.inc
+
+# Substitute the recorded MySQL binlog for the master's own.  The server must
+# be down while the file is swapped, and is restarted so it reopens the new
+# master-bin.000001 and starts a fresh master-bin.000002 of its own.
+--connection master
+--let $datadir= `SELECT @@datadir`
+--let $rpl_server_number= 1
+--source include/rpl_stop_server.inc
+--remove_file $datadir/master-bin.000001
+--copy_file $MYSQL_TEST_DIR/std_data/mdev39143_partial_json.000001 $datadir/master-bin.000001
+--source include/rpl_start_server.inc
+
+--connection slave
+CHANGE MASTER TO Master_log_file='master-bin.000001', Master_log_pos=4, Master_use_gtid=No;
+START SLAVE;
+
+# Wait until the last partial JSON update has been applied to id=1.
+# (LIKE avoids '$' which mysqltest would interpolate inside --let.)
+--let $wait_condition= SELECT COUNT(*)=1 FROM testdb.tj WHERE id=1 AND j LIKE '%"a": "zzzz"%' AND j LIKE '%"n": 44%'
+--source include/wait_condition.inc
+
+--echo # id=1: inserted as a full document, then modified by four partial-JSON
+--echo # events (REPLACE of members a and n twice, REMOVE of arr element 2).
+SELECT id, JSON_VALUE(j,'$.a') AS a, JSON_VALUE(j,'$.n') AS n,
+       JSON_QUERY(j,'$.arr') AS arr FROM testdb.tj ORDER BY id;
+
+# Cleanup.
+# The slave is sitting at the end of the injected binlog, which the
+# master did not write and knows nothing about, so save_master_pos/
+# sync_with_master cannot line the two up.  Repoint the slave at
+# master-bin.000002 - the empty binlog the master opened on restart - so that
+# the two agree on a position and rpl_end.inc can shut down cleanly.
+--connection slave
+STOP SLAVE;
+--source include/wait_for_slave_to_stop.inc
+DROP DATABASE testdb;
+CHANGE MASTER TO Master_log_file='master-bin.000002', Master_log_pos=4, Master_use_gtid=No;
+START SLAVE;
+--connection master
+--save_master_pos
+--connection slave
+--sync_with_master
+# Remove the plugin again: it is installed into mysql.plugin, which MTR's
+# check-testcase compares against the pre-test server state.
+UNINSTALL SONAME 'type_mysql_json';
+--source include/rpl_end.inc

sql/log_event.cc

@@ -1234,6 +1234,15 @@ Log_event *Log_event::read_log_event_no_checksum(
     case UPDATE_ROWS_EVENT:
       ev= new Update_rows_log_event(buf, event_len, fdle);
       break;
+    // MySQL 8.0: partial JSON update
+    case PARTIAL_UPDATE_ROWS_EVENT:
+      /*
+        MDEV-39143: same event structure as UPDATE_ROWS_EVENT; the after-image
+        may carry JSON diffs, handled in unpack_row() via the
+        is_partial_json_after_image flag.
+      */
+      ev= new Update_rows_log_event(buf, event_len, fdle);
+      break;
     case DELETE_ROWS_EVENT_V1:
     case DELETE_ROWS_EVENT:
       ev= new Delete_rows_log_event(buf, event_len, fdle);
@@ -1302,14 +1311,6 @@ Log_event *Log_event::read_log_event_no_checksum(
         "'binlog_transaction_compression=0' in the MySQL server";
       ev= NULL;
       break;
-    case PARTIAL_UPDATE_ROWS_EVENT:             // MySQL 8.0
-      *error=
-        "Found incompatible MySQL 8.0 PARTIAL_UPDATE_ROWS_EVENT event. "
-        "You can avoid this event by specifying "
-        "'binlog-row-value-options=\"\"' in the MySQL server";
-      ev= NULL;
-      break;
-
     case PRE_GA_WRITE_ROWS_EVENT:
     case PRE_GA_UPDATE_ROWS_EVENT:
     case PRE_GA_DELETE_ROWS_EVENT:

sql/log_event.h

@@ -809,7 +809,9 @@ static inline bool LOG_EVENT_IS_UPDATE_ROW(enum Log_event_type type)
 {
   return type == UPDATE_ROWS_EVENT || type == UPDATE_ROWS_EVENT_V1 ||
     type == UPDATE_ROWS_COMPRESSED_EVENT ||
-    type == UPDATE_ROWS_COMPRESSED_EVENT_V1;
+    type == UPDATE_ROWS_COMPRESSED_EVENT_V1 ||
+    // MySQL 8.0+ partial JSON update (MDEV-39143)
+    type == PARTIAL_UPDATE_ROWS_EVENT;
 }
 
 
@@ -5089,6 +5091,14 @@ class Rows_log_event : public Log_event
   int find_row(rpl_group_info *);
   int update_sequence();
 
+  /*
+    True for a MySQL PARTIAL_UPDATE_ROWS_EVENT (binlog_row_value_options=
+    PARTIAL_JSON on the master).  Its update after-image may carry JSON diffs
+    instead of full JSON values for some columns.
+  */
+  bool is_partial_json_update() const
+  { return m_type == PARTIAL_UPDATE_ROWS_EVENT; }
+
   // Unpack the current row into m_table->record[0], but with
   // a different columns bitmap.
   int unpack_current_row(rpl_group_info *rgi, MY_BITMAP const *cols)
@@ -5097,7 +5107,8 @@ class Rows_log_event : public Log_event
 
     ASSERT_OR_RETURN_ERROR(m_curr_row <= m_rows_end, HA_ERR_CORRUPT_EVENT);
     return ::unpack_row(rgi, m_table, m_width, m_curr_row, cols,
-                        &m_curr_row_end, m_rows_end);
+                        &m_curr_row_end, m_rows_end,
+                        is_partial_json_update() && cols == &m_cols_ai);
   }
 
   // Unpack the current row into m_table->record[0]