Fix jdbc-v2: NPE when the JavaCC parser cannot parse an INSERT VALUES list

CHANGELOG.md

@@ -44,6 +44,13 @@
 
 ### Bug Fixes 
 
+- **[jdbc-v2]** Fixed `Connection#prepareStatement` throwing a `NullPointerException` for an
+  `INSERT ... VALUES (...)` statement whose values list the default JavaCC parser cannot parse — most commonly one
+  containing a heredoc string (`$$...$$`), which the grammar has no token for, but also any other unparsable token
+  inside the list. The parser's error recovery left the values list's start position recorded without its matching end
+  position, which was then unboxed unguarded. Both positions are now dropped together, so the driver falls back to its
+  generic parameter-substitution path and such statements are prepared and executed successfully. The `ANTLR4`
+  parser backends were not affected. (https://github.com/ClickHouse/clickhouse-java/issues/3013)
 - **[client-v2]** Fixed LZ4 input streams not closing their underlying HTTP response stream. Closing an LZ4 stream
   returned by `QueryResponse.getInputStream()` now releases the wrapped transport stream, including after a partial
   read. (https://github.com/ClickHouse/clickhouse-java/issues/2985)

jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/SqlParserFacade.java

@@ -100,8 +100,9 @@ public ParsedPreparedStatement parsePreparedStatement(String sql) {
             stmt.setAssignValuesGroups(parsedStmt.getValueGroups());
 
             Integer startIndex = parsedStmt.getPositions().get(ClickHouseSqlStatement.KEYWORD_VALUES_START);
-            if (startIndex != null) {
-                int endIndex = parsedStmt.getPositions().get(ClickHouseSqlStatement.KEYWORD_VALUES_END);
+            Integer endIndexValue = parsedStmt.getPositions().get(ClickHouseSqlStatement.KEYWORD_VALUES_END);
+            if (startIndex != null && endIndexValue != null) {
+                int endIndex = endIndexValue;
                 stmt.setAssignValuesListStartPosition(startIndex);
                 stmt.setAssignValuesListStopPosition(endIndex);
                 String query = parsedStmt.getSQL();

jdbc-v2/src/main/javacc/ClickHouseSqlParser.jj

@@ -622,6 +622,10 @@ void dataClause(): {} {
             <FORMAT> <IDENTIFIER> { token_source.format = token.image; } )? (anyExprList())?
     } catch (ParseException e) {
         // FIXME introduce a lexical state in next release with consideration of delimiter from the context
+        // The values list was abandoned mid-way, so its start/end positions can only be recorded partially.
+        // Drop both so consumers either get a complete pair or none at all.
+        token_source.removePosition(ClickHouseSqlStatement.KEYWORD_VALUES_START);
+        token_source.removePosition(ClickHouseSqlStatement.KEYWORD_VALUES_END);
         Token nextToken;
         do {
             nextToken = getNextToken();

jdbc-v2/src/test/java/com/clickhouse/jdbc/PreparedStatementTest.java

@@ -878,6 +878,29 @@ void testMetabaseBug01() throws Exception {
         }
     }
 
+    @Test(groups = { "integration" })
+    void testInsertWithHeredocValue() throws Exception {
+        final String table = "test_insert_heredoc";
+        try (Connection conn = getJdbcConnection()) {
+            try (Statement stmt = conn.createStatement()) {
+                stmt.execute("DROP TABLE IF EXISTS " + table);
+                stmt.execute("CREATE TABLE " + table + " (s String, n Int32) Engine MergeTree ORDER BY ()");
+            }
+            try (PreparedStatement stmt = conn.prepareStatement(
+                    "INSERT INTO " + table + " (s, n) VALUES ($$a@b$$, ?)")) {
+                stmt.setInt(1, 42);
+                assertEquals(stmt.executeUpdate(), 1);
+            }
+            try (Statement stmt = conn.createStatement();
+                 ResultSet rs = stmt.executeQuery("SELECT s, n FROM " + table)) {
+                assertTrue(rs.next());
+                assertEquals(rs.getString(1), "a@b");
+                assertEquals(rs.getInt(2), 42);
+                assertFalse(rs.next());
+            }
+        }
+    }
+
     @Test(groups = { "integration" })
     void testStatementSplit() throws Exception {
         try (Connection conn = getJdbcConnection()) {

jdbc-v2/src/test/java/com/clickhouse/jdbc/internal/BaseSqlParserFacadeTest.java

@@ -151,6 +151,49 @@ public static Object[][] testPreparedStatementInsertSQLDP() {
         };
     }
 
+    @Test(dataProvider = "testInsertWithUnsupportedValuesListDP")
+    public void testInsertWithUnsupportedValuesList(String sql) {
+        ParsedPreparedStatement parsed = parser.parsePreparedStatement(sql);
+        assertTrue(parsed.isInsert(), "Should be of insert type");
+
+        int start = parsed.getAssignValuesListStartPosition();
+        int stop = parsed.getAssignValuesListStopPosition();
+        assertEquals(start > -1, stop > -1, "Values list start and stop positions should be both set or both unset");
+        if (start > -1) {
+            assertTrue(stop > start, "Values list should stop after it starts");
+            assertEquals(sql.charAt(start), '(', "Values list should start with an opening parenthesis");
+            assertEquals(sql.charAt(stop), ')', "Values list should end with a closing parenthesis");
+        }
+    }
+
+    @DataProvider
+    public static Object[][] testInsertWithUnsupportedValuesListDP() {
+        return new Object[][] {
+                { "INSERT INTO t VALUES ($$?$$, ?)" },
+                { "INSERT INTO t VALUES ($$a@b$$, ?)" },
+                { "INSERT INTO t VALUES ($$a@b$$, ?);" },
+                { "INSERT INTO t VALUES (?, )" },
+                { "INSERT INTO t VALUES (@@, ?)" },
+                { "INSERT INTO t VALUES (1, ?), (@@, ?)" },
+        };
+    }
+
+    @Test(dataProvider = "testInsertValuesListPositionsDP")
+    public void testInsertValuesListPositions(String sql, int start, int stop) {
+        ParsedPreparedStatement parsed = parser.parsePreparedStatement(sql);
+        assertEquals(parsed.getAssignValuesListStartPosition(), start, "Values list start position does not match");
+        assertEquals(parsed.getAssignValuesListStopPosition(), stop, "Values list stop position does not match");
+    }
+
+    @DataProvider
+    public static Object[][] testInsertValuesListPositionsDP() {
+        return new Object[][] {
+                { "INSERT INTO t VALUES (?, ?)", 21, 26 },
+                { "INSERT INTO t (a, b) VALUES (1, ?)", 28, 33 },
+                { "INSERT INTO t VALUES ($$x$$, ?)", 21, 30 },
+        };
+    }
+
     @Test
     public void testStmtWithCasts() {
         String sql = "SELECT ?::integer, ?, '?:: integer' FROM table WHERE v = ?::integer"; // CAST(?, INTEGER)