diff --git a/CHANGELOG.md b/CHANGELOG.md index 52c92a57a..205f9f423 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -44,6 +44,13 @@ ### Bug Fixes +- **[jdbc-v2]** Fixed JDBC escape processing rewriting text inside string literals and quoted identifiers. Because + `PreparedStatement` inlines bound parameters into the statement text, a bound value containing `{fn ` (or `{d '...'}` + / `{ts '...'}`) was re-read as SQL syntax: the `{fn ` was removed together with the next `}` found anywhere in the + statement — usually the closing brace of an unrelated `Map`/`Tuple` literal in another value or row — corrupting the + inserted data or failing with a server-side `SYNTAX_ERROR`. Escape sequences are now recognized only outside of quoted + text, and a `{fn ...}` escape is unwrapped at its matching closing brace, so nested braces (e.g. a `{name:Type}` query + parameter or a nested escape) stay balanced. (https://github.com/ClickHouse/clickhouse-java/issues/2995) - **[client-v2]** Fixed LZ4 input streams not closing their underlying HTTP response stream. Closing an LZ4 stream returned by `QueryResponse.getInputStream()` now releases the wrapped transport stream, including after a partial read. (https://github.com/ClickHouse/clickhouse-java/issues/2985) diff --git a/docs/features.md b/docs/features.md index 915e8ac9e..b92a6e358 100644 --- a/docs/features.md +++ b/docs/features.md @@ -77,7 +77,7 @@ Compatibility-sensitive traits: - Batch execution: Supports batched statements and prepared-statement batches, including multi-row rewrite for eligible `INSERT ... VALUES` statements. - Prepared statements: Supports `?` parameters through client-side SQL rendering and validates that all parameters are bound before execution. - SQL parsing and classification: Classifies SQL to distinguish queries, updates, inserts, `USE`, and role-changing statements, with selectable parser backends. -- JDBC escape processing: Translates supported JDBC escape syntax for dates, timestamps, and functions before execution. +- JDBC escape processing: Translates supported JDBC escape syntax for dates, timestamps, and functions before execution. Escape sequences are only recognized outside of quoted text, so string literals and quoted identifiers — including inlined parameter values that contain `{fn `, `{d '...'}`, or `{ts '...'}` — are passed through unchanged. - Result set streaming: Streams result sets from ClickHouse binary formats and `FORMAT JSONEachRow`, enforces max-row limits, and manages result-set lifecycle correctly. - Binary string reads: `ResultSet#getBytes(int|String)` and `ResultSet#getBinaryStream(int|String)` return the raw bytes of a `String`/`FixedString` column. Combined with the `binary_string_support` connection property, non-UTF-8/binary content stored in `String` columns round-trips byte-for-byte; `NULL` values report `null` with `wasNull()` set. `ResultSet#getObject(...)` never exposes the internal `StringValue` holder for these columns: `getObject(column, byte[].class)` returns the raw bytes, while `getObject(column, Object.class)` and the no-type `getObject(column)` overloads return a decoded `String`. - Result-set metadata: Exposes JDBC `ResultSetMetaData` backed by ClickHouse column schema. diff --git a/jdbc-v2/src/main/java/com/clickhouse/jdbc/StatementImpl.java b/jdbc-v2/src/main/java/com/clickhouse/jdbc/StatementImpl.java index aa32d202f..9dd164a13 100644 --- a/jdbc-v2/src/main/java/com/clickhouse/jdbc/StatementImpl.java +++ b/jdbc-v2/src/main/java/com/clickhouse/jdbc/StatementImpl.java @@ -19,16 +19,26 @@ import java.sql.SQLException; import java.sql.SQLWarning; import java.sql.Statement; +import java.util.ArrayDeque; import java.util.ArrayList; +import java.util.Deque; import java.util.List; import java.util.UUID; import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.TimeUnit; import java.util.function.Supplier; +import java.util.regex.Matcher; +import java.util.regex.Pattern; public class StatementImpl implements Statement, JdbcV2Wrapper { private static final Logger LOG = LoggerFactory.getLogger(StatementImpl.class); + // Escape sequences are only recognized outside of quoted text, so their patterns are matched at a given position + private static final Pattern DATE_ESCAPE = Pattern.compile("\\{d '([^']*)'\\}"); + private static final Pattern TIMESTAMP_ESCAPE = Pattern.compile("\\{ts '([^']*)'\\}"); + private static final String FUNCTION_ESCAPE_PREFIX = "{fn "; + private static final int PLAIN_BRACE = -1; + // Attributes ConnectionImpl connection; protected int queryTimeout; @@ -88,14 +98,61 @@ public static String escapedSQLToNative(String sql) { if (sql == null) { throw new IllegalArgumentException("SQL may not be null"); } - // Replace {d 'YYYY-MM-DD'} with corresponding SQL date format - sql = sql.replaceAll("\\{d '([^']*)'\\}", "toDate('$1')"); - // Replace {ts 'YYYY-MM-DD HH:mm:ss'} with corresponding SQL timestamp format - sql = sql.replaceAll("\\{ts '([^']*)'\\}", "timestamp('$1')"); + final int len = sql.length(); + final StringBuilder sb = new StringBuilder(len); + // Position in `sb` where a dropped `{fn ` prefix started, or PLAIN_BRACE for any other open brace + final Deque openBraces = new ArrayDeque<>(); + final Matcher dateEscape = DATE_ESCAPE.matcher(sql); + final Matcher timestampEscape = TIMESTAMP_ESCAPE.matcher(sql); + + int i = 0; + while (i < len) { + char ch = sql.charAt(i); + if (isQuote(ch)) { + // Quoted text (string literal or quoted identifier) is data, not syntax: copy it verbatim + i = appendQuotedText(sb, sql, i); + } else if (isCommentStart(sql, i)) { + // Comments are not syntax either and may contain unbalanced quotes and braces: copy them verbatim + i = appendComment(sb, sql, i); + } else if (ch == '{') { + if (lookingAt(dateEscape, i, len)) { + // Replace {d 'YYYY-MM-DD'} with corresponding SQL date format + sb.append("toDate('").append(dateEscape.group(1)).append("')"); + i = dateEscape.end(); + } else if (lookingAt(timestampEscape, i, len)) { + // Replace {ts 'YYYY-MM-DD HH:mm:ss'} with corresponding SQL timestamp format + sb.append("timestamp('").append(timestampEscape.group(1)).append("')"); + i = timestampEscape.end(); + } else if (sql.startsWith(FUNCTION_ESCAPE_PREFIX, i)) { + // Unwrap function escape syntax {fn } (e.g., {fn UCASE(name)}): the prefix is dropped + // here and the matching closing brace is dropped when it is reached + openBraces.push(sb.length()); + i += FUNCTION_ESCAPE_PREFIX.length(); + } else { + // Not an escape sequence (e.g. a map literal or a {name:Type} query parameter) + openBraces.push(PLAIN_BRACE); + sb.append(ch); + i++; + } + } else if (ch == '}') { + if (openBraces.isEmpty() || openBraces.pop() == PLAIN_BRACE) { + sb.append(ch); + } + i++; + } else { + sb.append(ch); + i++; + } + } - // Replace function escape syntax {fn } (e.g., {fn UCASE(name)}) - sql = sql.replaceAll("\\{fn ([^\\}]*)\\}", "$1"); + // Restore the prefix of any function escape that was never closed - such text is left as it was written. + // Positions are in descending order, so earlier insertions do not shift later ones. + for (Integer position : openBraces) { + if (position != PLAIN_BRACE) { + sb.insert(position, FUNCTION_ESCAPE_PREFIX); + } + } // Handle outer escape syntax //sql = sql.replaceAll("\\{escape '([^']*)'\\}", "'$1'"); @@ -103,7 +160,81 @@ public static String escapedSQLToNative(String sql) { // Note: do not remove new lines because they may be used to delimit comments // Add more replacements as needed for other JDBC escape sequences - return sql; + return sb.toString(); + } + + private static boolean isQuote(char ch) { + return ch == '\'' || ch == '"' || ch == '`'; + } + + private static boolean isCommentStart(String sql, int index) { + char ch = sql.charAt(index); + return ch == '#' || (ch == '-' && sql.startsWith("--", index)) || (ch == '/' && sql.startsWith("/*", index)); + } + + private static boolean lookingAt(Matcher matcher, int index, int end) { + matcher.region(index, end); + return matcher.lookingAt(); + } + + /** + * Appends the comment starting at {@code start} to {@code sb} without any modification. + * + * @param sb target buffer + * @param sql statement text + * @param start index of the first character of the comment + * @return index right after the comment, or the end of the statement when a block comment is not closed + */ + private static int appendComment(StringBuilder sb, String sql, int start) { + final int len = sql.length(); + int end; + if (sql.charAt(start) == '/') { // block comment + end = sql.indexOf("*/", start + 2); + end = end < 0 ? len : end + 2; + } else { // line comment - the new line itself is not part of it + end = sql.indexOf('\n', start); + end = end < 0 ? len : end; + } + + sb.append(sql, start, end); + return end; + } + + /** + * Appends the quoted text starting at {@code start} to {@code sb} without any modification. + * + * @param sb target buffer + * @param sql statement text + * @param start index of the opening quote + * @return index right after the closing quote, or the end of the statement when the quote is not closed + */ + private static int appendQuotedText(StringBuilder sb, String sql, int start) { + final int len = sql.length(); + final char quote = sql.charAt(start); + sb.append(quote); + + int i = start + 1; + while (i < len) { + char ch = sql.charAt(i); + if (ch == '\\' && i + 1 < len) { // a backslash escapes the next character + sb.append(ch).append(sql.charAt(i + 1)); + i += 2; + } else if (ch == quote) { + sb.append(ch); + i++; + if (i < len && sql.charAt(i) == quote) { // a doubled quote is an escaped quote + sb.append(quote); + i++; + } else { + break; + } + } else { + sb.append(ch); + i++; + } + } + + return i; } protected String getLastStatementSql() { diff --git a/jdbc-v2/src/test/java/com/clickhouse/jdbc/PreparedStatementTest.java b/jdbc-v2/src/test/java/com/clickhouse/jdbc/PreparedStatementTest.java index 14c19f7a9..87facc42f 100644 --- a/jdbc-v2/src/test/java/com/clickhouse/jdbc/PreparedStatementTest.java +++ b/jdbc-v2/src/test/java/com/clickhouse/jdbc/PreparedStatementTest.java @@ -1349,6 +1349,63 @@ public void testJdbcEscapeSyntax() throws Exception { } } + @Test(groups = {"integration"}) + public void testBoundValuesContainingJdbcEscapeSyntax() throws Exception { + try (Connection conn = getJdbcConnection(Map.of(ASYNC_INSERT_SETTING_KEY, ServerSettings.OFF))) { + final String table = "test_bound_values_with_escape_syntax"; + try (Statement stmt = conn.createStatement()) { + stmt.execute("DROP TABLE IF EXISTS " + table); + stmt.execute("CREATE TABLE " + table + + "(v1 Int32, v2 Map(String, String), v3 String, v4 Int32) Engine MergeTree ORDER BY (v1)"); + } + + Map map1 = new LinkedHashMap<>(); + map1.put("user", "Z!F3{fn "); + map1.put("region", "{d '2024-01-02'}"); + Map map2 = Collections.singletonMap("user", "}"); + + try (PreparedStatement stmt = conn.prepareStatement("INSERT INTO " + table + " VALUES (?, ?, ?, ?)")) { + stmt.setInt(1, 1); + stmt.setObject(2, map1); + stmt.setString(3, "{fn UCASE('a')}"); + stmt.setInt(4, 40); + stmt.addBatch(); + stmt.setInt(1, 2); + stmt.setObject(2, map2); + stmt.setString(3, "{ts '2024-01-02 02:01:01'}"); + stmt.setInt(4, 41); + stmt.addBatch(); + assertEquals(stmt.executeBatch(), new int[]{1, 1}); + } + + try (PreparedStatement stmt = conn.prepareStatement("SELECT ? AS v1, ? AS v2")) { + stmt.setString(1, "Z!F3{fn "); + stmt.setString(2, "}"); + try (ResultSet rs = stmt.executeQuery()) { + assertTrue(rs.next()); + assertEquals(rs.getString(1), "Z!F3{fn "); + assertEquals(rs.getString(2), "}"); + assertFalse(rs.next()); + } + } + + try (Statement stmt = conn.createStatement(); + ResultSet rs = stmt.executeQuery("SELECT * FROM " + table + " ORDER BY v1")) { + assertTrue(rs.next()); + assertEquals(rs.getInt(1), 1); + assertEquals(rs.getObject(2), map1); + assertEquals(rs.getString(3), "{fn UCASE('a')}"); + assertEquals(rs.getInt(4), 40); + assertTrue(rs.next()); + assertEquals(rs.getInt(1), 2); + assertEquals(rs.getObject(2), map2); + assertEquals(rs.getString(3), "{ts '2024-01-02 02:01:01'}"); + assertEquals(rs.getInt(4), 41); + assertFalse(rs.next()); + } + } + } + @Test(groups = {"integration "}) public void testStatementsWithDatabaseInTableIdentifier() throws Exception { try (Connection conn = getJdbcConnection(Map.of(ASYNC_INSERT_SETTING_KEY, ServerSettings.OFF))) { diff --git a/jdbc-v2/src/test/java/com/clickhouse/jdbc/StatementTest.java b/jdbc-v2/src/test/java/com/clickhouse/jdbc/StatementTest.java index db84ee366..e4a4e5c08 100644 --- a/jdbc-v2/src/test/java/com/clickhouse/jdbc/StatementTest.java +++ b/jdbc-v2/src/test/java/com/clickhouse/jdbc/StatementTest.java @@ -1655,6 +1655,46 @@ public static Object[][] testUnknownStatementTest_DP() { }; } + @DataProvider + public static Object[][] escapedSQLToNativeDP() { + return new Object[][] { + // JDBC escape sequences outside of quoted text are still translated + {"SELECT {fn UCASE(name)} FROM t", "SELECT UCASE(name) FROM t"}, + {"SELECT {fn CONCAT('Hello', 'World')}", "SELECT CONCAT('Hello', 'World')"}, + {"SELECT {fn ABS({fn MOD(10, 3)})}", "SELECT ABS(MOD(10, 3))"}, + {"SELECT {d '2024-01-02'}, {ts '2024-01-02 02:01:01'}", "SELECT toDate('2024-01-02'), timestamp('2024-01-02 02:01:01')"}, + {"SELECT {d ?}", "SELECT {d ?}"}, + {"SELECT {p1:String}", "SELECT {p1:String}"}, + {"SELECT {fn toString({p1:Int32})}", "SELECT toString({p1:Int32})"}, + {"SELECT {fn CONCAT({d '2024-01-02'}, 'x')}", "SELECT CONCAT(toDate('2024-01-02'), 'x')"}, + {"SELECT 1} AS a", "SELECT 1} AS a"}, + {"", ""}, + // an escape sequence that is never closed is left as it was written + {"SELECT {fn UCASE(x", "SELECT {fn UCASE(x"}, + {"SELECT {fn a(, {fn b(", "SELECT {fn a(, {fn b("}, + // quoted text is data, never syntax: it must be passed through verbatim + {"INSERT INTO t VALUES ({'k':'Z!F3{fn '})", "INSERT INTO t VALUES ({'k':'Z!F3{fn '})"}, + {"SELECT 'a{fn b' AS x, 'c}d' AS y", "SELECT 'a{fn b' AS x, 'c}d' AS y"}, + {"SELECT 'a\\'b{fn c' AS x, '}' AS y", "SELECT 'a\\'b{fn c' AS x, '}' AS y"}, + {"SELECT '{d ''2024-01-02''}' AS x", "SELECT '{d ''2024-01-02''}' AS x"}, + {"SELECT 1 AS \"a{fn \", 2 AS \"b}\"", "SELECT 1 AS \"a{fn \", 2 AS \"b}\""}, + {"SELECT 1 AS `a{fn `, 2 AS `b}`", "SELECT 1 AS `a{fn `, 2 AS `b}`"}, + {"SELECT '{fn UCASE(name)}' AS x", "SELECT '{fn UCASE(name)}' AS x"}, + {"SELECT 'unterminated {fn ", "SELECT 'unterminated {fn "}, + {"SELECT 'a\\\\' AS x, {fn UCASE(y)}", "SELECT 'a\\\\' AS x, UCASE(y)"}, + {"SELECT 1 AS \"a\"\"}b\", {fn UCASE(y)}", "SELECT 1 AS \"a\"\"}b\", UCASE(y)"}, + // comments are not syntax either + {"SELECT 1 -- it's {fn \n, {fn UCASE(y)}", "SELECT 1 -- it's {fn \n, UCASE(y)"}, + {"SELECT /* it's {fn */ {fn UCASE(y)}", "SELECT /* it's {fn */ UCASE(y)"}, + {"SELECT 1 # don't {fn ", "SELECT 1 # don't {fn "}, + }; + } + + @Test(dataProvider = "escapedSQLToNativeDP") + public void testEscapedSQLToNative(String sql, String expected) { + assertEquals(StatementImpl.escapedSQLToNative(sql), expected); + } + private static String getDBName(Statement stmt) throws SQLException { try (ResultSet rs = stmt.executeQuery("SELECT database()")) { rs.next();