From 47492abfd4ede8e18a79f523bd093ae246b1e20f Mon Sep 17 00:00:00 2001 From: Theodore Li Date: Sat, 27 Jun 2026 13:44:17 -0700 Subject: [PATCH 01/26] =?UTF-8?q?feat(table):=20v2=20table=20block=20?= =?UTF-8?q?=E2=80=94=20predicate=20grammar,=20unified=20predicate=20engine?= =?UTF-8?q?,=20cursor=20pagination?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Unify row-matching on one `fieldPredicate` leaf (filter compiler, upsert conflict probe, unique checks) — fixes the upsert wedge on case-mismatched unique values; equality/in is case-sensitive everywhere, text matches stay ILIKE - v2 nestable all/any predicate grammar: types, `buildPredicateClause`, structured contract schema, query-builder converters (+ `predicateToFilter` bridge) - Cursor pagination: opaque codec + `QueryResult.nextCursor` (offset gone on v2 surface) - `table_v2` block + `table_query_rows_v2` tool + POST /api/table/[tableId]/query route; v1 `table` hidden from toolbar; bulk update/delete author predicates too - Agent grammar: regenerate copilot tool-catalog TS; `user_table` server tool parses predicates (query → predicate, bulk → predicateToFilter) - Fix `replaceTableRowsWithTx` row[col.name] → row[getColumnId(col)] keying bug Co-Authored-By: Claude Opus 4.8 (1M context) --- .../app/api/table/[tableId]/query/route.ts | 107 +++ apps/sim/app/api/table/row-wire.ts | 19 +- apps/sim/blocks/blocks/table.ts | 3 + apps/sim/blocks/blocks/table_v2.ts | 687 ++++++++++++++++++ apps/sim/blocks/registry.ts | 2 + .../api/contracts/tables-predicate.test.ts | 74 ++ apps/sim/lib/api/contracts/tables.ts | 119 +++ .../lib/copilot/generated/tool-catalog-v1.ts | 2 +- .../lib/copilot/generated/tool-schemas-v1.ts | 192 ++--- .../tools/server/table/user-table.test.ts | 71 +- .../copilot/tools/server/table/user-table.ts | 13 +- apps/sim/lib/table/__tests__/sql.test.ts | 138 +++- .../lib/table/__tests__/validation.test.ts | 6 +- apps/sim/lib/table/column-keys.ts | 28 + .../__tests__/converters.test.ts | 124 +++- .../sim/lib/table/query-builder/converters.ts | 102 +++ .../lib/table/rows/__tests__/cursor.test.ts | 48 ++ apps/sim/lib/table/rows/cursor.ts | 59 ++ apps/sim/lib/table/rows/service.ts | 88 ++- apps/sim/lib/table/sql.ts | 220 +++--- apps/sim/lib/table/types.ts | 56 ++ apps/sim/lib/table/validation.ts | 75 +- apps/sim/tools/registry.ts | 2 + apps/sim/tools/table/index.ts | 1 + apps/sim/tools/table/query_rows_v2.ts | 102 +++ apps/sim/tools/table/types.ts | 22 + scripts/check-api-validation-contracts.ts | 4 +- 27 files changed, 2088 insertions(+), 276 deletions(-) create mode 100644 apps/sim/app/api/table/[tableId]/query/route.ts create mode 100644 apps/sim/blocks/blocks/table_v2.ts create mode 100644 apps/sim/lib/api/contracts/tables-predicate.test.ts create mode 100644 apps/sim/lib/table/rows/__tests__/cursor.test.ts create mode 100644 apps/sim/lib/table/rows/cursor.ts create mode 100644 apps/sim/tools/table/query_rows_v2.ts diff --git a/apps/sim/app/api/table/[tableId]/query/route.ts b/apps/sim/app/api/table/[tableId]/query/route.ts new file mode 100644 index 00000000000..a94f64fa767 --- /dev/null +++ b/apps/sim/app/api/table/[tableId]/query/route.ts @@ -0,0 +1,107 @@ +import { createLogger } from '@sim/logger' +import { type NextRequest, NextResponse } from 'next/server' +import { queryTableRowsV2Contract } from '@/lib/api/contracts/tables' +import { parseRequest } from '@/lib/api/server' +import { isZodError, validationErrorResponse } from '@/lib/api/server/validation' +import { checkSessionOrInternalAuth } from '@/lib/auth/hybrid' +import { generateRequestId } from '@/lib/core/utils/request' +import { withRouteHandler } from '@/lib/core/utils/with-route-handler' +import type { Sort, TableSchema } from '@/lib/table' +import { decodeCursor } from '@/lib/table/rows/cursor' +import { queryRows } from '@/lib/table/rows/service' +import { TableQueryValidationError } from '@/lib/table/sql' +import { rowWireTranslators } from '@/app/api/table/row-wire' +import { accessError, checkAccess } from '@/app/api/table/utils' + +const logger = createLogger('TableQueryV2API') + +interface TableQueryV2RouteParams { + params: Promise<{ tableId: string }> +} + +/** + * POST /api/table/[tableId]/query — v2 row query. Structured `all`/`any` + * predicate grammar + opaque cursor pagination (no offset on the wire). Shares + * the same engine as the legacy GET /rows route via `queryRows`. + */ +export const POST = withRouteHandler( + async (request: NextRequest, context: TableQueryV2RouteParams) => { + const requestId = generateRequestId() + + try { + const authResult = await checkSessionOrInternalAuth(request, { requireWorkflowId: false }) + if (!authResult.success || !authResult.userId) { + return NextResponse.json({ error: 'Authentication required' }, { status: 401 }) + } + + const parsed = await parseRequest(queryTableRowsV2Contract, request, context) + if (!parsed.success) return parsed.response + const { params, body } = parsed.data + const { tableId } = params + + const accessResult = await checkAccess(tableId, authResult.userId, 'read') + if (!accessResult.ok) return accessError(accessResult, requestId, tableId) + const { table } = accessResult + + if (body.workspaceId !== table.workspaceId) { + logger.warn( + `[${requestId}] Workspace ID mismatch for table ${tableId}. Provided: ${body.workspaceId}, Actual: ${table.workspaceId}` + ) + return NextResponse.json({ error: 'Invalid workspace ID' }, { status: 400 }) + } + + const wire = rowWireTranslators(authResult.authType, table.schema as TableSchema) + const cursor = body.cursor ? decodeCursor(body.cursor) : undefined + + const sortSpec = body.sort ? wire.sortSpecIn(body.sort) : undefined + const sort: Sort | undefined = sortSpec?.length + ? Object.fromEntries(sortSpec.map((s) => [s.field, s.direction])) + : undefined + + const result = await queryRows( + table, + { + predicate: body.predicate ? wire.predicateIn(body.predicate) : undefined, + sort, + limit: body.limit, + after: cursor?.after, + offset: cursor?.offset, + // Only the first page (no inbound cursor) pays for the total count. + includeTotal: !body.cursor, + }, + requestId + ) + + return NextResponse.json({ + success: true, + data: { + rows: result.rows.map((r) => ({ + id: r.id, + data: wire.dataOut(r.data), + executions: r.executions, + position: r.position, + orderKey: r.orderKey ?? undefined, + createdAt: + r.createdAt instanceof Date ? r.createdAt.toISOString() : String(r.createdAt), + updatedAt: + r.updatedAt instanceof Date ? r.updatedAt.toISOString() : String(r.updatedAt), + })), + rowCount: result.rowCount, + totalCount: result.totalCount, + limit: result.limit, + nextCursor: result.nextCursor, + }, + }) + } catch (error) { + if (isZodError(error)) return validationErrorResponse(error) + if (error instanceof TableQueryValidationError) { + return NextResponse.json({ error: error.message }, { status: 400 }) + } + if (error instanceof Error && error.message === 'Invalid cursor') { + return NextResponse.json({ error: 'Invalid cursor' }, { status: 400 }) + } + logger.error(`[${requestId}] Error querying rows (v2):`, error) + return NextResponse.json({ error: 'Failed to query rows' }, { status: 500 }) + } + } +) diff --git a/apps/sim/app/api/table/row-wire.ts b/apps/sim/app/api/table/row-wire.ts index 89c4cb93af7..e2218f4db0f 100644 --- a/apps/sim/app/api/table/row-wire.ts +++ b/apps/sim/app/api/table/row-wire.ts @@ -1,12 +1,14 @@ import { AuthType, type AuthTypeValue } from '@/lib/auth/hybrid' -import type { Filter, RowData, Sort, TableSchema } from '@/lib/table' +import type { Filter, RowData, Sort, SortSpec, TablePredicate, TableSchema } from '@/lib/table' import { buildIdByName, buildNameById, filterNamesToIds, + predicateNamesToIds, rowDataIdToName, rowDataNameToId, sortNamesToIds, + sortSpecNamesToIds, } from '@/lib/table' export interface RowWireTranslators { @@ -18,6 +20,10 @@ export interface RowWireTranslators { filterIn: (filter: Filter) => Filter /** Inbound sort: wire field refs → storage column ids. */ sortIn: (sort: Sort) => Sort + /** Inbound v2 predicate: wire field refs → storage column ids. */ + predicateIn: (predicate: TablePredicate) => TablePredicate + /** Inbound v2 sort spec: wire field refs → storage column ids. */ + sortSpecIn: (sort: SortSpec) => SortSpec } /** @@ -33,7 +39,14 @@ export function rowWireTranslators( ): RowWireTranslators { if (authType !== AuthType.INTERNAL_JWT) { const identity = (value: T): T => value - return { dataIn: identity, dataOut: identity, filterIn: identity, sortIn: identity } + return { + dataIn: identity, + dataOut: identity, + filterIn: identity, + sortIn: identity, + predicateIn: identity, + sortSpecIn: identity, + } } const idByName = buildIdByName(schema) const nameById = buildNameById(schema) @@ -42,5 +55,7 @@ export function rowWireTranslators( dataOut: (data) => rowDataIdToName(data, nameById), filterIn: (filter) => filterNamesToIds(filter, idByName), sortIn: (sort) => sortNamesToIds(sort, idByName), + predicateIn: (predicate) => predicateNamesToIds(predicate, idByName), + sortSpecIn: (sort) => sortSpecNamesToIds(sort, idByName), } } diff --git a/apps/sim/blocks/blocks/table.ts b/apps/sim/blocks/blocks/table.ts index 281bcd077e7..3d8bb5a3ba8 100644 --- a/apps/sim/blocks/blocks/table.ts +++ b/apps/sim/blocks/blocks/table.ts @@ -181,6 +181,9 @@ export const TableBlock: BlockConfig = { type: 'table', name: 'Table', description: 'User-defined data tables', + // Superseded by the v2 Table block (`table_v2`). Existing v1 blocks keep + // working; new workflows pick it up from the toolbar instead. + hideFromToolbar: true, longDescription: 'Create and manage custom data tables. Store, query, and manipulate structured data within workflows.', docsLink: 'https://docs.simstudio.ai/tools/table', diff --git a/apps/sim/blocks/blocks/table_v2.ts b/apps/sim/blocks/blocks/table_v2.ts new file mode 100644 index 00000000000..9bf284ac99a --- /dev/null +++ b/apps/sim/blocks/blocks/table_v2.ts @@ -0,0 +1,687 @@ +import { toError } from '@sim/utils/errors' +import { TableIcon } from '@/components/icons' +import { TABLE_LIMITS } from '@/lib/table/constants' +import { + filterRulesToPredicate, + predicateToFilter, + sortRulesToSortSpec, +} from '@/lib/table/query-builder/converters' +import type { TablePredicate } from '@/lib/table/types' +import type { BlockConfig } from '@/blocks/types' +import type { TableQueryV2Response } from '@/tools/table/types' +import { getTrigger } from '@/triggers' + +/** Resolve a bulk-op filter from the predicate grammar (builder or JSON editor). */ +function resolveBulkPredicate( + mode: string | undefined, + builder: unknown, + editor: string | unknown +): TablePredicate | null { + if (mode === 'builder' && builder) { + return filterRulesToPredicate(builder as Parameters[0]) + } + if (editor) return parseJSON(editor, 'Predicate') as TablePredicate + return null +} + +/** + * Table v2 — same operations as the v1 Table block, but `query_rows` speaks the + * legible `all`/`any` predicate grammar and paginates with an opaque cursor + * (no offset). The filter compiler, upsert conflict probe, and unique checks all + * share one case-sensitive containment leaf, so upserts can't wedge on a + * case-mismatched unique value the way they could under v1. + */ + +function parseJSON(value: string | unknown, fieldName: string): unknown { + if (typeof value !== 'string') return value + try { + return JSON.parse(value) + } catch (error) { + const errorMsg = toError(error).message + const unquotedValueMatch = value.match( + /:\s*([a-zA-Z][a-zA-Z0-9_\s]*[a-zA-Z0-9]|[a-zA-Z])\s*[,}]/ + ) + let hint = + 'Make sure all property names are in double quotes (e.g., {"name": "value"} not {name: "value"}).' + if (unquotedValueMatch) { + hint = + 'It looks like a string value is not quoted. When using block references in JSON, wrap them in double quotes: {"field": ""} not {"field": }.' + } + throw new Error(`Invalid JSON in ${fieldName}: ${errorMsg}. ${hint}`) + } +} + +interface TableBlockParams { + operation: string + tableId?: string + rowId?: string + data?: string | unknown + rows?: string | unknown + bulkPredicate?: string | unknown + predicate?: string | unknown + sort?: string | unknown + limit?: string + builderMode?: string + filterBuilder?: unknown + sortBuilder?: unknown + bulkFilterMode?: string + bulkFilterBuilder?: unknown + conflictColumn?: string +} + +interface ParsedParams { + tableId?: string + rowId?: string + data?: unknown + rows?: unknown + filter?: unknown + predicate?: unknown + sort?: unknown + limit?: number + conflictTarget?: string +} + +const paramTransformers: Record ParsedParams> = { + insert_row: (params) => ({ + tableId: params.tableId, + data: parseJSON(params.data, 'Row Data'), + }), + + upsert_row: (params) => ({ + tableId: params.tableId, + data: parseJSON(params.data, 'Row Data'), + conflictTarget: params.conflictColumn || undefined, + }), + + batch_insert_rows: (params) => ({ + tableId: params.tableId, + rows: parseJSON(params.rows, 'Rows Data'), + }), + + update_row: (params) => ({ + tableId: params.tableId, + rowId: params.rowId, + data: parseJSON(params.data, 'Row Data'), + }), + + // Bulk write-by-filter authors in the v2 predicate grammar, then converts to a + // legacy `Filter` for the existing bulk engine (sync + async-job paths). The + // conversion is lossless — both compile through the same `fieldPredicate` leaf. + update_rows_by_filter: (params) => { + const predicate = resolveBulkPredicate( + params.bulkFilterMode, + params.bulkFilterBuilder, + params.bulkPredicate + ) + return { + tableId: params.tableId, + filter: predicate ? predicateToFilter(predicate) : undefined, + data: parseJSON(params.data, 'Row Data'), + limit: params.limit ? Number.parseInt(params.limit) : undefined, + } + }, + + delete_row: (params) => ({ + tableId: params.tableId, + rowId: params.rowId, + }), + + delete_rows_by_filter: (params) => { + const predicate = resolveBulkPredicate( + params.bulkFilterMode, + params.bulkFilterBuilder, + params.bulkPredicate + ) + return { + tableId: params.tableId, + filter: predicate ? predicateToFilter(predicate) : undefined, + limit: params.limit ? Number.parseInt(params.limit) : undefined, + } + }, + + get_row: (params) => ({ + tableId: params.tableId, + rowId: params.rowId, + }), + + get_schema: (params) => ({ + tableId: params.tableId, + }), + + query_rows: (params) => { + let predicate: unknown + if (params.builderMode === 'builder' && params.filterBuilder) { + predicate = + filterRulesToPredicate( + params.filterBuilder as Parameters[0] + ) || undefined + } else if (params.predicate) { + predicate = parseJSON(params.predicate, 'Predicate') + } + + let sort: unknown + if (params.builderMode === 'builder' && params.sortBuilder) { + sort = + sortRulesToSortSpec(params.sortBuilder as Parameters[0]) || + undefined + } else if (params.sort) { + sort = parseJSON(params.sort, 'Sort') + } + + return { + tableId: params.tableId, + predicate, + sort, + limit: params.limit ? Number.parseInt(params.limit) : 100, + } + }, +} + +export const TableV2Block: BlockConfig = { + type: 'table_v2', + name: 'Table', + description: 'User-defined data tables', + longDescription: + 'Create and manage custom data tables. Store, query, and manipulate structured data within workflows. ' + + 'Query Rows filters with a predicate tree — { all: [...] } is AND, { any: [...] } is OR, and each leaf is ' + + '{ field, op, value }. Operators: eq, ne, gt, gte, lt, lte, in, nin, contains, ncontains, startsWith, ' + + 'endsWith, isEmpty, isNotEmpty (equality and in are case-sensitive; the text matches are case-insensitive). ' + + 'Pagination is cursor-based: pass the nextCursor from a prior result to fetch the next page.', + bestPractices: ` +- To fetch specific rows, use Query Rows with a predicate (e.g. { all: [{ field: "slack_user_id", op: "in", value: ["U1","U2"] }] }) — do NOT read every row and filter downstream with a Condition block. +- Use "Get Row by ID" only when you have the row's id; there is no fetch-by-value besides the predicate. +- Combine conditions with all (AND) / any (OR); groups nest. +- Example: players who won ≥10 and are active → { all: [{ field: "wins", op: "gte", value: 10 }, { field: "status", op: "eq", value: "active" }] }. +- To page through results, pass the previous response's nextCursor back as the cursor; omit it for the first page.`, + docsLink: 'https://docs.simstudio.ai/tools/table', + category: 'blocks', + bgColor: '#10B981', + icon: TableIcon, + subBlocks: [ + { + id: 'operation', + title: 'Operation', + type: 'dropdown', + options: [ + { label: 'Query Rows', id: 'query_rows' }, + { label: 'Insert Row', id: 'insert_row' }, + { label: 'Upsert Row', id: 'upsert_row' }, + { label: 'Batch Insert Rows', id: 'batch_insert_rows' }, + { label: 'Update Rows by Filter', id: 'update_rows_by_filter' }, + { label: 'Delete Rows by Filter', id: 'delete_rows_by_filter' }, + { label: 'Update Row by ID', id: 'update_row' }, + { label: 'Delete Row by ID', id: 'delete_row' }, + { label: 'Get Row by ID', id: 'get_row' }, + { label: 'Get Schema', id: 'get_schema' }, + ], + value: () => 'query_rows', + }, + + { + id: 'tableSelector', + title: 'Table', + type: 'table-selector', + canonicalParamId: 'tableId', + mode: 'basic', + placeholder: 'Select a table', + required: true, + }, + { + id: 'manualTableId', + title: 'Table ID', + type: 'short-input', + canonicalParamId: 'tableId', + mode: 'advanced', + placeholder: 'Enter table ID', + required: true, + }, + + { + id: 'rowId', + title: 'Row ID', + type: 'short-input', + placeholder: 'row_xxxxx', + dependsOn: ['tableId'], + condition: { field: 'operation', value: ['get_row', 'update_row', 'delete_row'] }, + required: true, + }, + + { + id: 'data', + title: 'Row Data (JSON)', + type: 'code', + placeholder: '{"column_name": "value"}', + condition: { + field: 'operation', + value: ['insert_row', 'upsert_row', 'update_row', 'update_rows_by_filter'], + }, + required: true, + wandConfig: { + enabled: true, + maintainHistory: true, + prompt: `Generate row data as a JSON object matching the table's column schema. + +### CONTEXT +{context} + +### INSTRUCTION +Return ONLY a valid JSON object with field values based on the table's columns. No explanations or markdown. + +IMPORTANT: Reference the table schema visible in the table selector to know which columns exist and their types. + +### EXAMPLES + +Table with columns: email (string), name (string), age (number) +"user with email john@example.com and age 25" +→ {"email": "john@example.com", "name": "John", "age": 25} + +Return ONLY the data JSON:`, + generationType: 'table-schema', + }, + }, + + { + id: 'conflictColumnSelector', + title: 'Conflict Column', + type: 'column-selector', + canonicalParamId: 'conflictColumn', + mode: 'basic', + selectorKey: 'table.columns', + placeholder: 'Select a unique column', + dependsOn: ['tableSelector'], + condition: { field: 'operation', value: 'upsert_row' }, + }, + { + id: 'manualConflictColumn', + title: 'Conflict Column', + type: 'short-input', + canonicalParamId: 'conflictColumn', + mode: 'advanced', + placeholder: 'Enter the column id', + dependsOn: ['tableId'], + condition: { field: 'operation', value: 'upsert_row' }, + }, + + { + id: 'rows', + title: 'Rows Data (Array of JSON)', + type: 'code', + placeholder: '[{"col1": "val1"}, {"col1": "val2"}]', + condition: { field: 'operation', value: 'batch_insert_rows' }, + required: true, + wandConfig: { + enabled: true, + maintainHistory: true, + prompt: `Generate an array of row data objects matching the table's column schema. + +### CONTEXT +{context} + +### INSTRUCTION +Return ONLY a valid JSON array of objects. Each object represents one row. No explanations or markdown. +Maximum ${TABLE_LIMITS.MAX_BATCH_INSERT_SIZE} rows per batch. + +Return ONLY the rows array:`, + generationType: 'table-schema', + }, + }, + + // Bulk write filter — v2 predicate grammar, converted to a Filter for the bulk engine + { + id: 'bulkFilterMode', + title: 'Filter Mode', + type: 'dropdown', + options: [ + { label: 'Builder', id: 'builder' }, + { label: 'Editor', id: 'json' }, + ], + value: () => 'builder', + condition: { + field: 'operation', + value: ['update_rows_by_filter', 'delete_rows_by_filter'], + }, + }, + { + id: 'bulkFilterBuilder', + title: 'Filter Conditions', + type: 'filter-builder', + required: { + field: 'operation', + value: ['update_rows_by_filter', 'delete_rows_by_filter'], + }, + condition: { + field: 'operation', + value: ['update_rows_by_filter', 'delete_rows_by_filter'], + and: { field: 'bulkFilterMode', value: 'builder' }, + }, + }, + { + id: 'bulkPredicate', + title: 'Predicate', + type: 'code', + placeholder: '{"all": [{"field": "status", "op": "eq", "value": "active"}]}', + condition: { + field: 'operation', + value: ['update_rows_by_filter', 'delete_rows_by_filter'], + and: { field: 'bulkFilterMode', value: 'json' }, + }, + required: true, + wandConfig: { + enabled: true, + maintainHistory: true, + prompt: `Generate a predicate for selecting rows to modify. + +### CONTEXT +{context} + +### INSTRUCTION +Return ONLY a valid JSON predicate. No explanations or markdown. + +A predicate is a nestable tree: { "all": [...] } means AND, { "any": [...] } means OR, and each leaf is { "field", "op", "value" }. + +### OPERATORS +eq, ne (case-sensitive equality); gt, gte, lt, lte (comparison); in, nin (array membership); contains, ncontains, startsWith, endsWith (case-insensitive text); isEmpty, isNotEmpty (no value). + +### EXAMPLES +"status is archived" → {"all": [{"field": "status", "op": "eq", "value": "archived"}]} +"age under 18 or status banned" → {"any": [{"field": "age", "op": "lt", "value": 18}, {"field": "status", "op": "eq", "value": "banned"}]} + +Return ONLY the predicate JSON:`, + generationType: 'table-schema', + }, + }, + + // Query rows — v2 predicate grammar + { + id: 'builderMode', + title: 'Input Mode', + type: 'dropdown', + options: [ + { label: 'Builder', id: 'builder' }, + { label: 'Editor', id: 'json' }, + ], + value: () => 'builder', + condition: { field: 'operation', value: 'query_rows' }, + }, + { + id: 'filterBuilder', + title: 'Filter Conditions', + type: 'filter-builder', + condition: { + field: 'operation', + value: 'query_rows', + and: { field: 'builderMode', value: 'builder' }, + }, + }, + { + id: 'sortBuilder', + title: 'Sort Order', + type: 'sort-builder', + condition: { + field: 'operation', + value: 'query_rows', + and: { field: 'builderMode', value: 'builder' }, + }, + }, + { + id: 'predicate', + title: 'Predicate', + type: 'code', + placeholder: '{"all": [{"field": "status", "op": "eq", "value": "active"}]}', + condition: { + field: 'operation', + value: 'query_rows', + and: { field: 'builderMode', value: 'builder', not: true }, + }, + wandConfig: { + enabled: true, + maintainHistory: true, + prompt: `Generate a predicate for selecting rows in a table. + +### CONTEXT +{context} + +### INSTRUCTION +Return ONLY a valid JSON predicate. No explanations or markdown. + +A predicate is a nestable tree: { "all": [...] } means AND, { "any": [...] } means OR, and each leaf is { "field", "op", "value" }. + +### OPERATORS +- eq / ne: equals / not-equals (case-sensitive) +- gt / gte / lt / lte: numeric or date comparison +- in / nin: value in / not in an array +- contains / ncontains / startsWith / endsWith: case-insensitive text match +- isEmpty / isNotEmpty: cell is null/empty / present (no value) + +### EXAMPLES + +"rows where status is active" +→ {"all": [{"field": "status", "op": "eq", "value": "active"}]} + +"age over 18 and status pending" +→ {"all": [{"field": "age", "op": "gte", "value": 18}, {"field": "status", "op": "eq", "value": "pending"}]} + +"status active or pending" +→ {"any": [{"field": "status", "op": "eq", "value": "active"}, {"field": "status", "op": "eq", "value": "pending"}]} + +Return ONLY the predicate JSON:`, + generationType: 'table-schema', + }, + }, + { + id: 'sort', + title: 'Sort', + type: 'code', + placeholder: '[{"field": "createdAt", "direction": "desc"}]', + condition: { + field: 'operation', + value: 'query_rows', + and: { field: 'builderMode', value: 'builder', not: true }, + }, + wandConfig: { + enabled: true, + maintainHistory: true, + prompt: `Generate sort order for table query results as a JSON array. + +### CONTEXT +{context} + +### INSTRUCTION +Return ONLY a valid JSON array of { "field", "direction": "asc" | "desc" }. No explanations or markdown. + +### EXAMPLES + +"newest first" → [{"field": "createdAt", "direction": "desc"}] +"age descending, then name ascending" → [{"field": "age", "direction": "desc"}, {"field": "name", "direction": "asc"}] + +Return ONLY the sort JSON:`, + generationType: 'table-schema', + }, + }, + { + id: 'limit', + title: 'Limit', + type: 'short-input', + placeholder: '100', + condition: { + field: 'operation', + value: ['query_rows', 'update_rows_by_filter', 'delete_rows_by_filter'], + }, + }, + { + id: 'cursor', + title: 'Cursor', + type: 'short-input', + placeholder: 'Paste a nextCursor to fetch the next page', + condition: { field: 'operation', value: 'query_rows' }, + }, + ...getTrigger('table_new_row').subBlocks, + ], + + tools: { + access: [ + 'table_insert_row', + 'table_batch_insert_rows', + 'table_upsert_row', + 'table_update_row', + 'table_update_rows_by_filter', + 'table_delete_row', + 'table_delete_rows_by_filter', + 'table_query_rows_v2', + 'table_get_row', + 'table_get_schema', + ], + config: { + tool: (params) => { + const toolMap: Record = { + insert_row: 'table_insert_row', + batch_insert_rows: 'table_batch_insert_rows', + upsert_row: 'table_upsert_row', + update_row: 'table_update_row', + update_rows_by_filter: 'table_update_rows_by_filter', + delete_row: 'table_delete_row', + delete_rows_by_filter: 'table_delete_rows_by_filter', + query_rows: 'table_query_rows_v2', + get_row: 'table_get_row', + get_schema: 'table_get_schema', + } + return toolMap[params.operation] || 'table_query_rows_v2' + }, + params: (params) => { + const { operation, ...rest } = params + const transformer = paramTransformers[operation] + if (transformer) { + return transformer(rest as TableBlockParams) + } + return rest + }, + }, + }, + + inputs: { + operation: { type: 'string', description: 'Table operation to perform' }, + tableId: { type: 'string', description: 'Table identifier' }, + data: { type: 'json', description: 'Row data for insert/update' }, + rows: { type: 'array', description: 'Array of row data for batch insert' }, + rowId: { type: 'string', description: 'Row identifier for ID-based operations' }, + bulkFilterMode: { + type: 'string', + description: 'Filter input mode for bulk operations (builder or json)', + }, + bulkFilterBuilder: { + type: 'json', + description: 'Visual filter builder conditions for bulk operations', + }, + bulkPredicate: { + type: 'json', + description: + 'Predicate selecting rows for bulk update/delete: nestable { all | any: [...] } of { field, op, value } leaves.', + }, + predicate: { + type: 'json', + description: + 'Query predicate: nestable { all | any: [...] } of { field, op, value } leaves. Operators: eq, ne, gt, gte, lt, lte, in, nin, contains, ncontains, startsWith, endsWith, isEmpty, isNotEmpty.', + }, + limit: { type: 'number', description: 'Query or bulk operation limit' }, + cursor: { type: 'string', description: 'Opaque pagination cursor from a prior query response' }, + builderMode: { + type: 'string', + description: 'Input mode for filter and sort (builder or json)', + }, + filterBuilder: { type: 'json', description: 'Visual filter builder conditions' }, + sortBuilder: { type: 'json', description: 'Visual sort builder conditions' }, + sort: { type: 'json', description: 'Sort order as [{ field, direction }]' }, + conflictColumn: { + type: 'string', + description: + 'Unique column to match on for upsert (required if the table has multiple unique columns)', + }, + }, + + outputs: { + success: { type: 'boolean', description: 'Operation success status' }, + row: { + type: 'json', + description: 'Single row data', + condition: { + field: 'operation', + value: ['get_row', 'insert_row', 'upsert_row', 'update_row'], + }, + }, + operation: { + type: 'string', + description: 'Operation performed (insert or update)', + condition: { field: 'operation', value: 'upsert_row' }, + }, + rows: { + type: 'array', + description: 'Array of rows', + condition: { field: 'operation', value: ['query_rows', 'batch_insert_rows'] }, + }, + rowCount: { + type: 'number', + description: 'Rows returned (query) or total rows in the table (get schema)', + condition: { field: 'operation', value: ['query_rows', 'get_schema'] }, + }, + totalCount: { + type: 'number', + description: 'Total rows matching the predicate (first page only)', + condition: { field: 'operation', value: 'query_rows' }, + }, + nextCursor: { + type: 'string', + description: 'Cursor to fetch the next page, or null on the last page', + condition: { field: 'operation', value: 'query_rows' }, + }, + insertedCount: { + type: 'number', + description: 'Number of rows inserted', + condition: { field: 'operation', value: 'batch_insert_rows' }, + }, + updatedCount: { + type: 'number', + description: 'Number of rows updated', + condition: { field: 'operation', value: 'update_rows_by_filter' }, + }, + updatedRowIds: { + type: 'array', + description: 'IDs of updated rows', + condition: { field: 'operation', value: 'update_rows_by_filter' }, + }, + deletedCount: { + type: 'number', + description: 'Number of rows deleted', + condition: { field: 'operation', value: ['delete_row', 'delete_rows_by_filter'] }, + }, + deletedRowIds: { + type: 'array', + description: 'IDs of deleted rows', + condition: { field: 'operation', value: 'delete_rows_by_filter' }, + }, + name: { + type: 'string', + description: 'Table name', + condition: { field: 'operation', value: 'get_schema' }, + }, + columns: { + type: 'array', + description: 'Column definitions (each includes its stable id)', + condition: { field: 'operation', value: 'get_schema' }, + }, + columnCount: { + type: 'number', + description: 'Number of columns', + condition: { field: 'operation', value: 'get_schema' }, + }, + maxRows: { + type: 'number', + description: "Max rows per table for the workspace's plan", + condition: { field: 'operation', value: 'get_schema' }, + }, + message: { type: 'string', description: 'Operation status message' }, + }, + triggers: { + enabled: true, + available: ['table_new_row'], + }, +} diff --git a/apps/sim/blocks/registry.ts b/apps/sim/blocks/registry.ts index 8ee9b75e7d1..f896578a6c5 100644 --- a/apps/sim/blocks/registry.ts +++ b/apps/sim/blocks/registry.ts @@ -285,6 +285,7 @@ import { STSBlock, STSBlockMeta } from '@/blocks/blocks/sts' import { SttBlock, SttV2Block } from '@/blocks/blocks/stt' import { SupabaseBlock, SupabaseBlockMeta } from '@/blocks/blocks/supabase' import { TableBlock } from '@/blocks/blocks/table' +import { TableV2Block } from '@/blocks/blocks/table_v2' import { TailscaleBlock, TailscaleBlockMeta } from '@/blocks/blocks/tailscale' import { TavilyBlock, TavilyBlockMeta } from '@/blocks/blocks/tavily' import { TelegramBlock, TelegramBlockMeta } from '@/blocks/blocks/telegram' @@ -595,6 +596,7 @@ const BLOCK_REGISTRY: Record = { stt_v2: SttV2Block, supabase: SupabaseBlock, table: TableBlock, + table_v2: TableV2Block, tailscale: TailscaleBlock, tavily: TavilyBlock, telegram: TelegramBlock, diff --git a/apps/sim/lib/api/contracts/tables-predicate.test.ts b/apps/sim/lib/api/contracts/tables-predicate.test.ts new file mode 100644 index 00000000000..2b48a4c5023 --- /dev/null +++ b/apps/sim/lib/api/contracts/tables-predicate.test.ts @@ -0,0 +1,74 @@ +/** + * @vitest-environment node + */ +import { describe, expect, it } from 'vitest' +import { + queryTableRowsV2BodySchema, + sortSpecSchema, + tablePredicateSchema, +} from '@/lib/api/contracts/tables' + +describe('tablePredicateSchema', () => { + it('accepts a nested all/any tree', () => { + const parsed = tablePredicateSchema.safeParse({ + all: [ + { field: 'slack_user_id', op: 'in', value: ['U1', 'U2'] }, + { + any: [ + { field: 's', op: 'eq', value: 'a' }, + { field: 's', op: 'eq', value: 'b' }, + ], + }, + ], + }) + expect(parsed.success).toBe(true) + }) + + it('accepts a valueless op without a value', () => { + expect(tablePredicateSchema.safeParse({ all: [{ field: 'n', op: 'isEmpty' }] }).success).toBe( + true + ) + }) + + it('rejects an unknown operator', () => { + expect( + tablePredicateSchema.safeParse({ all: [{ field: 'n', op: 'regex', value: 'x' }] }).success + ).toBe(false) + }) + + it('rejects an empty group', () => { + expect(tablePredicateSchema.safeParse({ all: [] }).success).toBe(false) + }) + + it('rejects a node that is neither a leaf nor a group', () => { + expect(tablePredicateSchema.safeParse({ all: [{ foo: 'bar' }] }).success).toBe(false) + }) +}) + +describe('queryTableRowsV2BodySchema', () => { + it('defaults limit and accepts predicate + cursor (no offset)', () => { + const parsed = queryTableRowsV2BodySchema.parse({ + workspaceId: 'ws-1', + predicate: { all: [{ field: 'wins', op: 'gte', value: 10 }] }, + cursor: 'abc', + }) + expect(parsed.limit).toBeGreaterThan(0) + expect('offset' in parsed).toBe(false) + }) + + it('rejects limit over the max', () => { + expect( + queryTableRowsV2BodySchema.safeParse({ workspaceId: 'ws-1', limit: 100000 }).success + ).toBe(false) + }) +}) + +describe('sortSpecSchema', () => { + it('accepts an ordered field/direction list', () => { + expect(sortSpecSchema.safeParse([{ field: 'wins', direction: 'desc' }]).success).toBe(true) + }) + + it('rejects a bad direction', () => { + expect(sortSpecSchema.safeParse([{ field: 'wins', direction: 'sideways' }]).success).toBe(false) + }) +}) diff --git a/apps/sim/lib/api/contracts/tables.ts b/apps/sim/lib/api/contracts/tables.ts index b9d0512b0c0..7bffe667b12 100644 --- a/apps/sim/lib/api/contracts/tables.ts +++ b/apps/sim/lib/api/contracts/tables.ts @@ -5,10 +5,12 @@ import type { CsvHeaderMapping, EnrichmentRunDetail, Filter, + PredicateNode, RowData, Sort, TableDefinition, TableMetadata, + TablePredicate, TableRow, TableRowsCursor, } from '@/lib/table' @@ -260,6 +262,77 @@ const nonEmptyFilterSchema = domainObjectSchema().refine( const filterSchema = domainObjectSchema() +/* --------------------------- v2 predicate grammar --------------------------- */ + +const MAX_PREDICATE_MEMBERS = 100 +const MAX_SORT_FIELDS = 32 +const MAX_FIELD_LENGTH = 128 + +/** v2 bare-operator set — the legible grammar mothership and clients author. */ +export const filterOpSchema = z.enum([ + 'eq', + 'ne', + 'gt', + 'gte', + 'lt', + 'lte', + 'in', + 'nin', + 'contains', + 'ncontains', + 'startsWith', + 'endsWith', + 'isEmpty', + 'isNotEmpty', +]) + +const predicateValueSchema: z.ZodType = z.lazy(() => + z.union([z.string(), z.number(), z.boolean(), z.null(), z.array(predicateValueSchema)]) +) + +const predicateLeafSchema = z.object({ + field: z.string().min(1, 'predicate field is required').max(MAX_FIELD_LENGTH), + op: filterOpSchema, + value: predicateValueSchema.optional(), +}) + +/** + * Recursive `all`/`any` filter tree. A group is `{ all: [...] }` (AND) or + * `{ any: [...] }` (OR); members are leaf predicates or nested groups. A single + * cast pins the output to the canonical `TablePredicate` (zod's structural output + * is assignable but invariant under `z.ZodType`). + */ +export const tablePredicateSchema = z.lazy(() => + z.union([ + z.object({ + all: z + .array(predicateNodeSchema) + .min(1, 'all group cannot be empty') + .max(MAX_PREDICATE_MEMBERS), + }), + z.object({ + any: z + .array(predicateNodeSchema) + .min(1, 'any group cannot be empty') + .max(MAX_PREDICATE_MEMBERS), + }), + ]) +) as z.ZodType + +const predicateNodeSchema = z.lazy(() => + z.union([predicateLeafSchema, tablePredicateSchema]) +) as z.ZodType + +/** v2 sort: an ordered list of `{ field, direction }`. */ +export const sortSpecSchema = z + .array( + z.object({ + field: z.string().min(1, 'sort field is required').max(MAX_FIELD_LENGTH), + direction: z.enum(['asc', 'desc']), + }) + ) + .max(MAX_SORT_FIELDS) + const optionalPositiveLimit = (max: number, label: string) => z.preprocess( (value) => (value === null || value === undefined || value === '' ? undefined : Number(value)), @@ -530,6 +603,52 @@ export const listTableRowsContract = defineRouteContract({ }, }) +/** + * v2 query surface: structured predicate grammar + opaque cursor pagination. + * No `offset` (the cursor encodes paging state) and no after/sort refine (the + * cursor carries the sort context). POST because the predicate is a JSON tree. + */ +export const queryTableRowsV2BodySchema = z.object({ + workspaceId: z.string().min(1, 'Workspace ID is required'), + predicate: tablePredicateSchema.optional(), + sort: sortSpecSchema.optional(), + limit: z + .preprocess( + (value) => + value === null || value === undefined || value === '' ? undefined : Number(value), + z + .number({ error: 'Limit must be a number' }) + .int('Limit must be an integer') + .min(1, 'Limit must be at least 1') + .max(TABLE_LIMITS.MAX_QUERY_LIMIT, `Limit cannot exceed ${TABLE_LIMITS.MAX_QUERY_LIMIT}`) + .optional() + ) + .default(TABLE_LIMITS.DEFAULT_QUERY_LIMIT), + cursor: z.string().min(1, 'cursor must be a non-empty token').optional(), +}) + +export type QueryTableRowsV2Body = z.input + +export const queryTableRowsV2Contract = defineRouteContract({ + method: 'POST', + path: '/api/table/[tableId]/query', + params: tableIdParamsSchema, + body: queryTableRowsV2BodySchema, + response: { + mode: 'json', + schema: successResponseSchema( + z.object({ + rows: z.array(tableRowSchema), + rowCount: z.number(), + totalCount: z.number().nullable(), + limit: z.number(), + nextCursor: z.string().nullable(), + }) + ), + }, +}) +export type QueryTableRowsV2Response = ContractJsonResponse + export const findTableRowsQuerySchema = z.object({ workspaceId: z.string().min(1, 'Workspace ID is required'), q: z.string().min(1, 'Search query is required'), diff --git a/apps/sim/lib/copilot/generated/tool-catalog-v1.ts b/apps/sim/lib/copilot/generated/tool-catalog-v1.ts index 4a4c24a1594..55e0399c7d2 100644 --- a/apps/sim/lib/copilot/generated/tool-catalog-v1.ts +++ b/apps/sim/lib/copilot/generated/tool-catalog-v1.ts @@ -3865,7 +3865,7 @@ export const UserTable: ToolCatalogEntry = { filter: { type: 'object', description: - 'MongoDB-style filter for query_rows, update_rows_by_filter, delete_rows_by_filter', + 'Predicate-tree filter for query_rows, update_rows_by_filter, delete_rows_by_filter. A filter is { all: [...] } (AND) or { any: [...] } (OR) wrapping leaf nodes { field, op, value }; groups nest. Ops (bare, no $): eq, ne, gt, gte, lt, lte, in, nin, contains, ncontains, startsWith, endsWith, isEmpty, isNotEmpty. eq/ne/in/nin are case-sensitive; contains/ncontains/startsWith/endsWith are case-insensitive; isEmpty/isNotEmpty take no value; in/nin take an array. Example: { all: [{ field: "age", op: "gte", value: 18 }, { field: "status", op: "eq", value: "pending" }] }.', }, groupId: { type: 'string', diff --git a/apps/sim/lib/copilot/generated/tool-schemas-v1.ts b/apps/sim/lib/copilot/generated/tool-schemas-v1.ts index dcaea0db6ea..f42c995c2b8 100644 --- a/apps/sim/lib/copilot/generated/tool-schemas-v1.ts +++ b/apps/sim/lib/copilot/generated/tool-schemas-v1.ts @@ -10,7 +10,7 @@ export interface ToolRuntimeSchemaEntry { } export const TOOL_RUNTIME_SCHEMAS: Record = { - agent: { + ['agent']: { parameters: { properties: { request: { @@ -23,7 +23,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - auth: { + ['auth']: { parameters: { properties: { request: { @@ -36,7 +36,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - check_deployment_status: { + ['check_deployment_status']: { parameters: { type: 'object', properties: { @@ -48,7 +48,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - complete_scheduled_task: { + ['complete_scheduled_task']: { parameters: { type: 'object', properties: { @@ -61,7 +61,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - crawl_website: { + ['crawl_website']: { parameters: { type: 'object', properties: { @@ -96,7 +96,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - create_file: { + ['create_file']: { parameters: { type: 'object', properties: { @@ -162,7 +162,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { required: ['success', 'message'], }, }, - create_file_folder: { + ['create_file_folder']: { parameters: { type: 'object', properties: { @@ -180,7 +180,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - create_workflow: { + ['create_workflow']: { parameters: { type: 'object', properties: { @@ -205,7 +205,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - create_workspace_mcp_server: { + ['create_workspace_mcp_server']: { parameters: { type: 'object', properties: { @@ -238,7 +238,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - delete_file: { + ['delete_file']: { parameters: { type: 'object', properties: { @@ -268,7 +268,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { required: ['success', 'message'], }, }, - delete_file_folder: { + ['delete_file_folder']: { parameters: { type: 'object', properties: { @@ -284,7 +284,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - delete_workflow: { + ['delete_workflow']: { parameters: { type: 'object', properties: { @@ -300,7 +300,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - delete_workspace_mcp_server: { + ['delete_workspace_mcp_server']: { parameters: { type: 'object', properties: { @@ -313,7 +313,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - deploy: { + ['deploy']: { parameters: { properties: { request: { @@ -327,7 +327,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - deploy_api: { + ['deploy_api']: { parameters: { type: 'object', properties: { @@ -411,7 +411,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { ], }, }, - deploy_chat: { + ['deploy_chat']: { parameters: { type: 'object', properties: { @@ -570,7 +570,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { ], }, }, - deploy_mcp: { + ['deploy_mcp']: { parameters: { type: 'object', properties: { @@ -686,7 +686,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { required: ['deploymentType', 'deploymentStatus'], }, }, - diff_workflows: { + ['diff_workflows']: { parameters: { type: 'object', properties: { @@ -710,7 +710,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - download_to_workspace_file: { + ['download_to_workspace_file']: { parameters: { type: 'object', properties: { @@ -759,7 +759,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - edit_content: { + ['edit_content']: { parameters: { type: 'object', properties: { @@ -791,7 +791,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { required: ['success', 'message'], }, }, - edit_workflow: { + ['edit_workflow']: { parameters: { type: 'object', properties: { @@ -830,7 +830,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - enrichment_run: { + ['enrichment_run']: { parameters: { type: 'object', properties: { @@ -874,7 +874,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { required: ['matched', 'result'], }, }, - ffmpeg: { + ['ffmpeg']: { parameters: { type: 'object', properties: { @@ -1055,7 +1055,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - file: { + ['file']: { parameters: { properties: { prompt: { @@ -1068,7 +1068,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - function_execute: { + ['function_execute']: { parameters: { type: 'object', properties: { @@ -1206,7 +1206,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - generate_api_key: { + ['generate_api_key']: { parameters: { type: 'object', properties: { @@ -1224,7 +1224,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - generate_audio: { + ['generate_audio']: { parameters: { type: 'object', properties: { @@ -1376,7 +1376,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - generate_image: { + ['generate_image']: { parameters: { type: 'object', properties: { @@ -1504,7 +1504,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - generate_video: { + ['generate_video']: { parameters: { type: 'object', properties: { @@ -1671,7 +1671,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - get_block_outputs: { + ['get_block_outputs']: { parameters: { type: 'object', properties: { @@ -1692,7 +1692,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - get_block_upstream_references: { + ['get_block_upstream_references']: { parameters: { type: 'object', properties: { @@ -1714,7 +1714,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - get_deployed_workflow_state: { + ['get_deployed_workflow_state']: { parameters: { type: 'object', properties: { @@ -1727,7 +1727,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - get_deployment_log: { + ['get_deployment_log']: { parameters: { type: 'object', properties: { @@ -1740,7 +1740,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - get_page_contents: { + ['get_page_contents']: { parameters: { type: 'object', properties: { @@ -1768,14 +1768,14 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - get_platform_actions: { + ['get_platform_actions']: { parameters: { type: 'object', properties: {}, }, resultSchema: undefined, }, - get_scheduled_task_logs: { + ['get_scheduled_task_logs']: { parameters: { type: 'object', properties: { @@ -1800,7 +1800,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - get_workflow_data: { + ['get_workflow_data']: { parameters: { type: 'object', properties: { @@ -1819,7 +1819,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - get_workflow_run_options: { + ['get_workflow_run_options']: { parameters: { type: 'object', properties: { @@ -1832,7 +1832,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - glob: { + ['glob']: { parameters: { type: 'object', properties: { @@ -1851,7 +1851,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - grep: { + ['grep']: { parameters: { type: 'object', properties: { @@ -1899,7 +1899,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - knowledge: { + ['knowledge']: { parameters: { properties: { request: { @@ -1912,7 +1912,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - knowledge_base: { + ['knowledge_base']: { parameters: { type: 'object', properties: { @@ -2105,7 +2105,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { required: ['success', 'message'], }, }, - list_file_folders: { + ['list_file_folders']: { parameters: { type: 'object', properties: { @@ -2117,7 +2117,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - list_integration_tools: { + ['list_integration_tools']: { parameters: { properties: { integration: { @@ -2131,14 +2131,14 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - list_user_workspaces: { + ['list_user_workspaces']: { parameters: { type: 'object', properties: {}, }, resultSchema: undefined, }, - list_workspace_mcp_servers: { + ['list_workspace_mcp_servers']: { parameters: { type: 'object', properties: { @@ -2151,7 +2151,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - load_deployment: { + ['load_deployment']: { parameters: { type: 'object', properties: { @@ -2170,7 +2170,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - load_integration_tool: { + ['load_integration_tool']: { parameters: { properties: { tool_ids: { @@ -2187,7 +2187,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - manage_credential: { + ['manage_credential']: { parameters: { type: 'object', properties: { @@ -2216,7 +2216,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - manage_custom_tool: { + ['manage_custom_tool']: { parameters: { type: 'object', properties: { @@ -2296,7 +2296,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - manage_folder: { + ['manage_folder']: { parameters: { type: 'object', properties: { @@ -2335,7 +2335,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - manage_mcp_tool: { + ['manage_mcp_tool']: { parameters: { type: 'object', properties: { @@ -2387,7 +2387,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - manage_scheduled_task: { + ['manage_scheduled_task']: { parameters: { type: 'object', properties: { @@ -2462,7 +2462,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - manage_skill: { + ['manage_skill']: { parameters: { type: 'object', properties: { @@ -2495,7 +2495,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - materialize_file: { + ['materialize_file']: { parameters: { type: 'object', properties: { @@ -2519,7 +2519,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - media: { + ['media']: { parameters: { properties: { prompt: { @@ -2532,7 +2532,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - move_file: { + ['move_file']: { parameters: { type: 'object', properties: { @@ -2553,7 +2553,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - move_file_folder: { + ['move_file_folder']: { parameters: { type: 'object', properties: { @@ -2571,7 +2571,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - move_workflow: { + ['move_workflow']: { parameters: { type: 'object', properties: { @@ -2591,7 +2591,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - oauth_get_auth_link: { + ['oauth_get_auth_link']: { parameters: { type: 'object', properties: { @@ -2605,7 +2605,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - oauth_request_access: { + ['oauth_request_access']: { parameters: { type: 'object', properties: { @@ -2619,7 +2619,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - open_resource: { + ['open_resource']: { parameters: { type: 'object', properties: { @@ -2653,7 +2653,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - promote_to_live: { + ['promote_to_live']: { parameters: { type: 'object', properties: { @@ -2672,7 +2672,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - query_logs: { + ['query_logs']: { parameters: { type: 'object', properties: { @@ -2783,7 +2783,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - read: { + ['read']: { parameters: { type: 'object', properties: { @@ -2810,7 +2810,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - redeploy: { + ['redeploy']: { parameters: { type: 'object', properties: { @@ -2889,7 +2889,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { ], }, }, - rename_file: { + ['rename_file']: { parameters: { type: 'object', properties: { @@ -2925,7 +2925,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { required: ['success', 'message'], }, }, - rename_file_folder: { + ['rename_file_folder']: { parameters: { type: 'object', properties: { @@ -2942,7 +2942,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - rename_workflow: { + ['rename_workflow']: { parameters: { type: 'object', properties: { @@ -2959,7 +2959,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - research: { + ['research']: { parameters: { properties: { topic: { @@ -2972,7 +2972,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - respond: { + ['respond']: { parameters: { additionalProperties: true, properties: { @@ -2995,7 +2995,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - restore_resource: { + ['restore_resource']: { parameters: { type: 'object', properties: { @@ -3013,7 +3013,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - run: { + ['run']: { parameters: { properties: { context: { @@ -3030,7 +3030,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - run_block: { + ['run_block']: { parameters: { type: 'object', properties: { @@ -3062,7 +3062,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - run_from_block: { + ['run_from_block']: { parameters: { type: 'object', properties: { @@ -3094,7 +3094,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - run_workflow: { + ['run_workflow']: { parameters: { type: 'object', properties: { @@ -3132,7 +3132,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - run_workflow_until_block: { + ['run_workflow_until_block']: { parameters: { type: 'object', properties: { @@ -3175,7 +3175,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - scheduled_task: { + ['scheduled_task']: { parameters: { properties: { request: { @@ -3188,7 +3188,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - scrape_page: { + ['scrape_page']: { parameters: { type: 'object', properties: { @@ -3209,7 +3209,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - search_documentation: { + ['search_documentation']: { parameters: { type: 'object', properties: { @@ -3226,7 +3226,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - search_library_docs: { + ['search_library_docs']: { parameters: { type: 'object', properties: { @@ -3247,7 +3247,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - search_online: { + ['search_online']: { parameters: { type: 'object', properties: { @@ -3287,7 +3287,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - search_patterns: { + ['search_patterns']: { parameters: { type: 'object', properties: { @@ -3309,7 +3309,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - set_block_enabled: { + ['set_block_enabled']: { parameters: { type: 'object', properties: { @@ -3331,7 +3331,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - set_environment_variables: { + ['set_environment_variables']: { parameters: { type: 'object', properties: { @@ -3365,7 +3365,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - set_global_workflow_variables: { + ['set_global_workflow_variables']: { parameters: { type: 'object', properties: { @@ -3406,7 +3406,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - superagent: { + ['superagent']: { parameters: { properties: { task: { @@ -3420,7 +3420,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - table: { + ['table']: { parameters: { properties: { request: { @@ -3433,7 +3433,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - update_deployment_version: { + ['update_deployment_version']: { parameters: { type: 'object', properties: { @@ -3462,7 +3462,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - update_scheduled_task_history: { + ['update_scheduled_task_history']: { parameters: { type: 'object', properties: { @@ -3480,7 +3480,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - update_workspace_mcp_server: { + ['update_workspace_mcp_server']: { parameters: { type: 'object', properties: { @@ -3505,7 +3505,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - user_memory: { + ['user_memory']: { parameters: { type: 'object', properties: { @@ -3554,7 +3554,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - user_table: { + ['user_table']: { parameters: { type: 'object', properties: { @@ -3622,7 +3622,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { filter: { type: 'object', description: - 'MongoDB-style filter for query_rows, update_rows_by_filter, delete_rows_by_filter', + 'Predicate-tree filter for query_rows, update_rows_by_filter, delete_rows_by_filter. A filter is { all: [...] } (AND) or { any: [...] } (OR) wrapping leaf nodes { field, op, value }; groups nest. Ops (bare, no $): eq, ne, gt, gte, lt, lte, in, nin, contains, ncontains, startsWith, endsWith, isEmpty, isNotEmpty. eq/ne/in/nin are case-sensitive; contains/ncontains/startsWith/endsWith are case-insensitive; isEmpty/isNotEmpty take no value; in/nin take an array. Example: { all: [{ field: "age", op: "gte", value: 18 }, { field: "status", op: "eq", value: "pending" }] }.', }, groupId: { type: 'string', @@ -3917,7 +3917,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { required: ['success', 'message'], }, }, - workflow: { + ['workflow']: { parameters: { properties: { prompt: { @@ -3930,7 +3930,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - workspace_file: { + ['workspace_file']: { parameters: { type: 'object', properties: { diff --git a/apps/sim/lib/copilot/tools/server/table/user-table.test.ts b/apps/sim/lib/copilot/tools/server/table/user-table.test.ts index b7320df3254..0d6925cbc79 100644 --- a/apps/sim/lib/copilot/tools/server/table/user-table.test.ts +++ b/apps/sim/lib/copilot/tools/server/table/user-table.test.ts @@ -742,7 +742,11 @@ describe('userTableServerTool.delete_rows_by_filter', () => { const result = await userTableServerTool.execute( { operation: 'delete_rows_by_filter', - args: { tableId: 'tbl_1', filter: { name: 'x' }, limit: 5000 }, + args: { + tableId: 'tbl_1', + filter: { all: [{ field: 'name', op: 'eq', value: 'x' }] }, + limit: 5000, + }, }, { userId: 'user-1', workspaceId: 'workspace-1' } ) @@ -763,7 +767,10 @@ describe('userTableServerTool.delete_rows_by_filter', () => { it('deletes inline when the unbounded match count is within the cap', async () => { const result = await userTableServerTool.execute( - { operation: 'delete_rows_by_filter', args: { tableId: 'tbl_1', filter: { name: 'x' } } }, + { + operation: 'delete_rows_by_filter', + args: { tableId: 'tbl_1', filter: { all: [{ field: 'name', op: 'eq', value: 'x' }] } }, + }, { userId: 'user-1', workspaceId: 'workspace-1' } ) @@ -781,7 +788,11 @@ describe('userTableServerTool.delete_rows_by_filter', () => { const result = await userTableServerTool.execute( { operation: 'delete_rows_by_filter', - args: { tableId: 'tbl_1', filter: { name: 'x' }, limit: 100 }, + args: { + tableId: 'tbl_1', + filter: { all: [{ field: 'name', op: 'eq', value: 'x' }] }, + limit: 100, + }, }, { userId: 'user-1', workspaceId: 'workspace-1' } ) @@ -801,7 +812,10 @@ describe('userTableServerTool.delete_rows_by_filter', () => { }) const result = await userTableServerTool.execute( - { operation: 'delete_rows_by_filter', args: { tableId: 'tbl_1', filter: { name: 'x' } } }, + { + operation: 'delete_rows_by_filter', + args: { tableId: 'tbl_1', filter: { all: [{ field: 'name', op: 'eq', value: 'x' }] } }, + }, { userId: 'user-1', workspaceId: 'workspace-1' } ) await flushDetached() @@ -836,7 +850,10 @@ describe('userTableServerTool.delete_rows_by_filter', () => { mockMarkTableJobRunning.mockResolvedValueOnce(false) const result = await userTableServerTool.execute( - { operation: 'delete_rows_by_filter', args: { tableId: 'tbl_1', filter: { name: 'x' } } }, + { + operation: 'delete_rows_by_filter', + args: { tableId: 'tbl_1', filter: { all: [{ field: 'name', op: 'eq', value: 'x' }] } }, + }, { userId: 'user-1', workspaceId: 'workspace-1' } ) @@ -850,7 +867,11 @@ describe('userTableServerTool.delete_rows_by_filter', () => { const result = await userTableServerTool.execute( { operation: 'delete_rows_by_filter', - args: { tableId: 'tbl_1', filter: { name: 'x' }, limit: 100 }, + args: { + tableId: 'tbl_1', + filter: { all: [{ field: 'name', op: 'eq', value: 'x' }] }, + limit: 100, + }, }, { userId: 'user-1', workspaceId: 'workspace-1' } ) @@ -881,7 +902,12 @@ describe('userTableServerTool.update_rows_by_filter', () => { const result = await userTableServerTool.execute( { operation: 'update_rows_by_filter', - args: { tableId: 'tbl_1', filter: { name: 'x' }, data: { age: 1 }, limit: 5000 }, + args: { + tableId: 'tbl_1', + filter: { all: [{ field: 'name', op: 'eq', value: 'x' }] }, + data: { age: 1 }, + limit: 5000, + }, }, { userId: 'user-1', workspaceId: 'workspace-1' } ) @@ -901,7 +927,11 @@ describe('userTableServerTool.update_rows_by_filter', () => { const result = await userTableServerTool.execute( { operation: 'update_rows_by_filter', - args: { tableId: 'tbl_1', filter: { name: 'x' }, data: { age: 1 } }, + args: { + tableId: 'tbl_1', + filter: { all: [{ field: 'name', op: 'eq', value: 'x' }] }, + data: { age: 1 }, + }, }, { userId: 'user-1', workspaceId: 'workspace-1' } ) @@ -922,7 +952,11 @@ describe('userTableServerTool.update_rows_by_filter', () => { const result = await userTableServerTool.execute( { operation: 'update_rows_by_filter', - args: { tableId: 'tbl_1', filter: { name: 'x' }, data: { age: 1 } }, + args: { + tableId: 'tbl_1', + filter: { all: [{ field: 'name', op: 'eq', value: 'x' }] }, + data: { age: 1 }, + }, }, { userId: 'user-1', workspaceId: 'workspace-1' } ) @@ -958,7 +992,11 @@ describe('userTableServerTool.update_rows_by_filter', () => { const result = await userTableServerTool.execute( { operation: 'update_rows_by_filter', - args: { tableId: 'tbl_1', filter: { email: 'x' }, data: { email: 'y' } }, + args: { + tableId: 'tbl_1', + filter: { all: [{ field: 'email', op: 'eq', value: 'x' }] }, + data: { email: 'y' }, + }, }, { userId: 'user-1', workspaceId: 'workspace-1' } ) @@ -980,7 +1018,11 @@ describe('userTableServerTool.update_rows_by_filter', () => { const result = await userTableServerTool.execute( { operation: 'update_rows_by_filter', - args: { tableId: 'tbl_1', filter: { name: 'x' }, data: { age: 1 } }, + args: { + tableId: 'tbl_1', + filter: { all: [{ field: 'name', op: 'eq', value: 'x' }] }, + data: { age: 1 }, + }, }, { userId: 'user-1', workspaceId: 'workspace-1' } ) @@ -994,7 +1036,12 @@ describe('userTableServerTool.update_rows_by_filter', () => { const result = await userTableServerTool.execute( { operation: 'update_rows_by_filter', - args: { tableId: 'tbl_1', filter: { name: 'x' }, data: { age: 1 }, limit: 100 }, + args: { + tableId: 'tbl_1', + filter: { all: [{ field: 'name', op: 'eq', value: 'x' }] }, + data: { age: 1 }, + limit: 100, + }, }, { userId: 'user-1', workspaceId: 'workspace-1' } ) diff --git a/apps/sim/lib/copilot/tools/server/table/user-table.ts b/apps/sim/lib/copilot/tools/server/table/user-table.ts index ef1a8fed1b2..435511fbd45 100644 --- a/apps/sim/lib/copilot/tools/server/table/user-table.ts +++ b/apps/sim/lib/copilot/tools/server/table/user-table.ts @@ -27,7 +27,7 @@ import { import { buildIdByName, buildNameById, - filterNamesToIds, + predicateNamesToIds, rowDataIdToName, rowDataNameToId, sortNamesToIds, @@ -44,6 +44,7 @@ import { import { markTableDeleteFailed, runTableDelete } from '@/lib/table/delete-runner' import { runTableImport, type TableImportPayload } from '@/lib/table/import-runner' import { markTableJobRunning, releaseJobClaim } from '@/lib/table/jobs/service' +import { predicateToFilter } from '@/lib/table/query-builder/converters' import { batchInsertRows, batchUpdateRows, @@ -608,7 +609,7 @@ export const userTableServerTool: BaseServerTool const result = await queryRows( table, { - filter: args.filter ? filterNamesToIds(args.filter, idByName) : undefined, + predicate: args.filter ? predicateNamesToIds(args.filter, idByName) : undefined, sort: args.sort ? sortNamesToIds(args.sort, idByName) : undefined, limit: args.limit !== undefined @@ -728,7 +729,9 @@ export const userTableServerTool: BaseServerTool const requestId = generateId().slice(0, 8) const idByName = buildIdByName(table.schema) - const idFilter = filterNamesToIds(args.filter, idByName) + // Agent authors the v2 predicate grammar; convert to a Filter for the + // bulk engine (same fieldPredicate leaf → identical SQL). + const idFilter = predicateToFilter(predicateNamesToIds(args.filter, idByName)) const idData = rowDataNameToId(args.data, idByName) // Inline handles up to MAX_BULK_OPERATION_SIZE rows in one request; a larger operation @@ -823,7 +826,9 @@ export const userTableServerTool: BaseServerTool const requestId = generateId().slice(0, 8) const idByName = buildIdByName(table.schema) - const idFilter = filterNamesToIds(args.filter, idByName) + // Agent authors the v2 predicate grammar; convert to a Filter for the + // bulk engine (same fieldPredicate leaf → identical SQL). + const idFilter = predicateToFilter(predicateNamesToIds(args.filter, idByName)) // Inline handles up to MAX_BULK_OPERATION_SIZE rows; a larger delete (an explicit limit // above the cap, or unbounded "delete everything matching") hands off to the background diff --git a/apps/sim/lib/table/__tests__/sql.test.ts b/apps/sim/lib/table/__tests__/sql.test.ts index ca293848779..eb5ba91b19e 100644 --- a/apps/sim/lib/table/__tests__/sql.test.ts +++ b/apps/sim/lib/table/__tests__/sql.test.ts @@ -13,8 +13,13 @@ * substrings like `::timestamptz` against the generated SQL. */ import { describe, expect, it } from 'vitest' -import { buildFilterClause, buildSortClause } from '@/lib/table/sql' -import type { ColumnDefinition, Filter, Sort } from '@/lib/table/types' +import { + buildFilterClause, + buildPredicateClause, + buildSortClause, + fieldPredicate, +} from '@/lib/table/sql' +import type { ColumnDefinition, Filter, Sort, TablePredicate } from '@/lib/table/types' type SqlNode = | { strings: ArrayLike; values: unknown[] } @@ -460,3 +465,132 @@ describe('SQL Builder', () => { }) }) }) + +describe('fieldPredicate (shared leaf)', () => { + const r = ( + op: Parameters[2], + value: unknown, + colType?: ColumnDefinition['type'] + ) => render(fieldPredicate(TABLE, 'wins', op, value as never, colType)) + + it('eq emits case-sensitive JSONB containment (no lower())', () => { + const out = render(fieldPredicate(TABLE, 'slack_user_id', 'eq', 'U333', undefined)) + expect(out).toContain('user_table_rows.data @>') + expect(out).toContain('"slack_user_id":"U333"') + expect(out).not.toContain('lower(') + // Case is preserved verbatim — U333 and u333 are distinct values. + expect(out).not.toContain('u333') + }) + + it('ne negates the containment clause', () => { + expect(r('ne', 'x')).toContain('NOT (') + expect(r('ne', 'x')).toContain('data @>') + }) + + it('in with one value is a single containment; many values OR together', () => { + expect(render(fieldPredicate(TABLE, 'slack_user_id', 'in', ['U1'], undefined))).toContain( + '"slack_user_id":"U1"' + ) + const many = render(fieldPredicate(TABLE, 'slack_user_id', 'in', ['U1', 'U2'], undefined)) + expect(many).toContain('"slack_user_id":"U1"') + expect(many).toContain('"slack_user_id":"U2"') + expect(many).toContain(' OR ') + }) + + it('nin ANDs negated containments', () => { + const out = render(fieldPredicate(TABLE, 'slack_user_id', 'nin', ['U1', 'U2'], undefined)) + expect(out).toContain('NOT (') + expect(out).toContain(' AND ') + }) + + it('empty in/nin arrays are a no-op (undefined)', () => { + expect(fieldPredicate(TABLE, 'wins', 'in', [], undefined)).toBeUndefined() + expect(fieldPredicate(TABLE, 'wins', 'nin', [], undefined)).toBeUndefined() + }) + + it('range ops cast by column type', () => { + expect(r('gte', 10, 'number')).toContain('::numeric') + expect(r('gt', '2024-01-01', 'date')).toContain('::timestamptz') + }) + + it('text ops use case-insensitive ILIKE', () => { + expect(render(fieldPredicate(TABLE, 'name', 'contains', 'jo', undefined))).toContain('ILIKE') + expect(render(fieldPredicate(TABLE, 'name', 'startsWith', 'jo', undefined))).toContain('ILIKE') + }) + + it('isEmpty / isNotEmpty emit emptiness checks', () => { + expect(render(fieldPredicate(TABLE, 'name', 'isEmpty', undefined, undefined))).toContain( + 'IS NULL' + ) + expect(render(fieldPredicate(TABLE, 'name', 'isNotEmpty', undefined, undefined))).toContain( + 'IS NOT NULL' + ) + }) + + it('validates the field name', () => { + expect(() => fieldPredicate(TABLE, "x'; DROP", 'eq', 1, undefined)).toThrow( + 'Invalid field name' + ) + }) + + it('rejects an unknown operator', () => { + expect(() => fieldPredicate(TABLE, 'wins', 'bogus' as never, 1, undefined)).toThrow( + 'Invalid operator' + ) + }) +}) + +describe('buildPredicateClause (v2 grammar)', () => { + it('all joins members with AND', () => { + const p: TablePredicate = { + all: [ + { field: 'slack_user_id', op: 'in', value: ['U1', 'U2'] }, + { field: 'wins', op: 'gte', value: 10 }, + ], + } + const out = render(buildPredicateClause(p, TABLE, [{ name: 'wins', type: 'number' }])) + expect(out).toContain(' AND ') + expect(out).toContain('"slack_user_id":"U1"') + expect(out).toContain('::numeric') + }) + + it('any joins members with OR', () => { + const p: TablePredicate = { + any: [ + { field: 'status', op: 'eq', value: 'active' }, + { field: 'status', op: 'eq', value: 'pending' }, + ], + } + const out = render(buildPredicateClause(p, TABLE, [])) + expect(out).toContain(' OR ') + expect(out).toContain('"status":"active"') + expect(out).toContain('"status":"pending"') + }) + + it('nests groups', () => { + const p: TablePredicate = { + all: [ + { field: 'wins', op: 'gte', value: 1 }, + { + any: [ + { field: 's', op: 'eq', value: 'a' }, + { field: 's', op: 'eq', value: 'b' }, + ], + }, + ], + } + const out = render(buildPredicateClause(p, TABLE, [])) + expect(out).toContain(' AND ') + expect(out).toContain(' OR ') + }) + + it('an empty group is a no-op (undefined)', () => { + expect(buildPredicateClause({ all: [] }, TABLE, [])).toBeUndefined() + expect(buildPredicateClause({ any: [] }, TABLE, [])).toBeUndefined() + }) + + it('validates leaf field names', () => { + const p: TablePredicate = { all: [{ field: 'bad name', op: 'eq', value: 1 }] } + expect(() => buildPredicateClause(p, TABLE, [])).toThrow('Invalid field name') + }) +}) diff --git a/apps/sim/lib/table/__tests__/validation.test.ts b/apps/sim/lib/table/__tests__/validation.test.ts index 4ebfe9a6ffa..44bb0e1370d 100644 --- a/apps/sim/lib/table/__tests__/validation.test.ts +++ b/apps/sim/lib/table/__tests__/validation.test.ts @@ -455,10 +455,12 @@ describe('Validation', () => { expect(result.errors[0]).toContain('abc123') }) - it('should be case-insensitive for string comparisons', () => { + it('should be case-sensitive for string comparisons', () => { + // U333 vs u333: differing case is a DISTINCT value (matches the DB + // containment leaf). This is the v2 contract that fixes the upsert wedge. const data = { id: 'ABC123', email: 'new@example.com', name: 'New User' } const result = validateUniqueConstraints(data, schema, existingRows) - expect(result.valid).toBe(false) + expect(result.valid).toBe(true) }) it('should exclude specified row from checks (for updates)', () => { diff --git a/apps/sim/lib/table/column-keys.ts b/apps/sim/lib/table/column-keys.ts index ec770b047ea..b89ac3cf352 100644 --- a/apps/sim/lib/table/column-keys.ts +++ b/apps/sim/lib/table/column-keys.ts @@ -12,8 +12,11 @@ import { generateId } from '@sim/utils/id' import type { ColumnDefinition, Filter, + PredicateNode, RowData, Sort, + SortSpec, + TablePredicate, TableSchema, WorkflowGroup, } from '@/lib/table/types' @@ -171,6 +174,31 @@ export function sortNamesToIds(sort: Sort, idByName: ReadonlyMap return out } +/** + * Translates a v2 predicate's leaf `field` names → column ids (recursing through + * `all`/`any` groups). Fields with no matching column pass through unchanged. + * The v2 analogue of {@link filterNamesToIds}. + */ +export function predicateNamesToIds( + predicate: TablePredicate, + idByName: ReadonlyMap +): TablePredicate { + const remap = (node: PredicateNode): PredicateNode => { + if ('field' in node) return { ...node, field: idByName.get(node.field) ?? node.field } + if ('all' in node) return { all: node.all.map(remap) } + return { any: node.any.map(remap) } + } + return remap(predicate) as TablePredicate +} + +/** Translates a v2 sort spec's field names → column ids. Unknown fields pass through. */ +export function sortSpecNamesToIds( + sort: SortSpec, + idByName: ReadonlyMap +): SortSpec { + return sort.map((s) => ({ field: idByName.get(s.field) ?? s.field, direction: s.direction })) +} + /** * Remaps a stored row keyed by column **id** back to **name** keying for the * wire. Used at the name-translating boundaries on the way out. Ids with no diff --git a/apps/sim/lib/table/query-builder/__tests__/converters.test.ts b/apps/sim/lib/table/query-builder/__tests__/converters.test.ts index 9af9bcb96ec..d148db3ed8a 100644 --- a/apps/sim/lib/table/query-builder/__tests__/converters.test.ts +++ b/apps/sim/lib/table/query-builder/__tests__/converters.test.ts @@ -6,8 +6,15 @@ * valueless `$empty` operator that maps to two distinct UI operators. */ import { describe, expect, it } from 'vitest' -import { filterRulesToFilter, filterToRules } from '@/lib/table/query-builder/converters' -import type { FilterRule } from '@/lib/table/types' +import { + filterRulesToFilter, + filterRulesToPredicate, + filterToRules, + predicateToFilter, + predicateToFilterRules, + sortRulesToSortSpec, +} from '@/lib/table/query-builder/converters' +import type { FilterRule, SortRule, TablePredicate } from '@/lib/table/types' function rule(overrides: Partial): FilterRule { return { @@ -117,3 +124,116 @@ describe('filterToRules', () => { expect(filterRulesToFilter(rules)).toEqual(original) }) }) + +describe('filterRulesToPredicate (v2)', () => { + it('an AND group becomes a single all-group', () => { + const p = filterRulesToPredicate([ + rule({ column: 'slack_user_id', operator: 'in', value: 'U1, U2' }), + rule({ column: 'wins', operator: 'gte', value: '10' }), + ]) + expect(p).toEqual({ + all: [ + { field: 'slack_user_id', op: 'in', value: ['U1', 'U2'] }, + { field: 'wins', op: 'gte', value: 10 }, + ], + }) + }) + + it('an or boundary splits into an any-of-all groups', () => { + const p = filterRulesToPredicate([ + rule({ column: 'status', operator: 'eq', value: 'active' }), + rule({ column: 'status', operator: 'eq', value: 'pending', logicalOperator: 'or' }), + ]) + expect(p).toEqual({ + any: [ + { all: [{ field: 'status', op: 'eq', value: 'active' }] }, + { all: [{ field: 'status', op: 'eq', value: 'pending' }] }, + ], + }) + }) + + it('valueless ops omit the value', () => { + const p = filterRulesToPredicate([rule({ column: 'note', operator: 'isEmpty' })]) + expect(p).toEqual({ all: [{ field: 'note', op: 'isEmpty' }] }) + }) + + it('returns null for no rules', () => { + expect(filterRulesToPredicate([])).toBeNull() + }) + + it('round-trips rules → predicate → rules (operator + value preserved)', () => { + const rules: FilterRule[] = [ + rule({ column: 'wins', operator: 'gte', value: '10' }), + rule({ column: 'name', operator: 'contains', value: 'jo', logicalOperator: 'or' }), + ] + const back = predicateToFilterRules(filterRulesToPredicate(rules)) + expect(back.map((r) => [r.column, r.operator, r.value, r.logicalOperator])).toEqual([ + ['wins', 'gte', '10', 'and'], + ['name', 'contains', 'jo', 'or'], + ]) + }) +}) + +describe('predicateToFilterRules (v2)', () => { + it('flattens an all-group to and-joined rules', () => { + const p: TablePredicate = { + all: [ + { field: 'a', op: 'eq', value: 1 }, + { field: 'b', op: 'isNotEmpty' }, + ], + } + const rules = predicateToFilterRules(p) + expect(rules.map((r) => [r.column, r.operator, r.value])).toEqual([ + ['a', 'eq', '1'], + ['b', 'isNotEmpty', ''], + ]) + }) + + it('returns [] for null', () => { + expect(predicateToFilterRules(null)).toEqual([]) + }) +}) + +describe('predicateToFilter (v2 → legacy)', () => { + it('maps an all-group to $and with bare-op leaves', () => { + const p: TablePredicate = { + all: [ + { field: 'slack_user_id', op: 'in', value: ['U1', 'U2'] }, + { field: 'wins', op: 'gte', value: 10 }, + { field: 'name', op: 'eq', value: 'x' }, + ], + } + expect(predicateToFilter(p)).toEqual({ + $and: [{ slack_user_id: { $in: ['U1', 'U2'] } }, { wins: { $gte: 10 } }, { name: 'x' }], + }) + }) + + it('maps an any-group to $or and valueless ops to $empty', () => { + const p: TablePredicate = { + any: [ + { field: 'a', op: 'isEmpty' }, + { field: 'b', op: 'isNotEmpty' }, + ], + } + expect(predicateToFilter(p)).toEqual({ + $or: [{ a: { $empty: true } }, { b: { $empty: false } }], + }) + }) +}) + +describe('sortRulesToSortSpec (v2)', () => { + it('maps rules to an ordered field/direction list', () => { + const rules: SortRule[] = [ + { id: '1', column: 'wins', direction: 'desc' }, + { id: '2', column: 'name', direction: 'asc' }, + ] + expect(sortRulesToSortSpec(rules)).toEqual([ + { field: 'wins', direction: 'desc' }, + { field: 'name', direction: 'asc' }, + ]) + }) + + it('skips column-less rules and returns null when empty', () => { + expect(sortRulesToSortSpec([{ id: '1', column: '', direction: 'asc' }])).toBeNull() + }) +}) diff --git a/apps/sim/lib/table/query-builder/converters.ts b/apps/sim/lib/table/query-builder/converters.ts index 4d6b5f8753e..a789fa0b29e 100644 --- a/apps/sim/lib/table/query-builder/converters.ts +++ b/apps/sim/lib/table/query-builder/converters.ts @@ -6,11 +6,15 @@ import { generateShortId } from '@sim/utils/id' import { isRecordLike } from '@sim/utils/object' import type { Filter, + FilterOp, FilterRule, JsonValue, + Predicate, Sort, SortDirection, SortRule, + SortSpec, + TablePredicate, } from '@/lib/table/types' /** Converts UI filter rules to a Filter object for API queries. */ @@ -227,3 +231,101 @@ function formatValueForBuilder(value: JsonValue): string { function normalizeSortDirection(direction: string): SortDirection { return direction === 'desc' ? 'desc' : 'asc' } + +/* ----------------------------- v2 grammar ----------------------------- */ + +const VALUELESS_OPS = new Set(['isEmpty', 'isNotEmpty']) + +function ruleToPredicate(rule: FilterRule): Predicate { + const op = rule.operator as FilterOp + if (VALUELESS_OPS.has(op)) return { field: rule.column, op } + return { field: rule.column, op, value: parseValue(rule.value, rule.operator) } +} + +/** + * Converts UI builder rules to a v2 `TablePredicate`. Rules within an `or` + * boundary form an `all` group; multiple groups are combined under `any`. + * Mirrors {@link filterRulesToFilter} but emits the bare-operator grammar. + */ +export function filterRulesToPredicate(rules: FilterRule[]): TablePredicate | null { + if (rules.length === 0) return null + + const groups: Predicate[][] = [] + let current: Predicate[] = [] + + for (const rule of rules) { + if (rule.logicalOperator === 'or' && current.length > 0) { + groups.push(current) + current = [] + } + if (!rule.column) continue + current.push(ruleToPredicate(rule)) + } + if (current.length > 0) groups.push(current) + + if (groups.length === 0) return null + if (groups.length === 1) return { all: groups[0] } + return { any: groups.map((group) => ({ all: group })) } +} + +function predicateLeafToRule(p: Predicate): FilterRule { + return { + id: generateShortId(), + logicalOperator: 'and', + column: p.field, + operator: p.op, + value: VALUELESS_OPS.has(p.op) ? '' : formatValueForBuilder(p.value as JsonValue), + } +} + +/** Flattens a predicate node into builder rules (best-effort for deep nesting). */ +function predicateGroupToRules(node: TablePredicate | Predicate): FilterRule[] { + if ('field' in node) return [predicateLeafToRule(node)] + const members = 'all' in node ? node.all : node.any + return members.flatMap((member) => + 'field' in member ? [predicateLeafToRule(member)] : predicateGroupToRules(member) + ) +} + +/** Converts a v2 `TablePredicate` back to UI builder rules. */ +export function predicateToFilterRules(predicate: TablePredicate | null): FilterRule[] { + if (!predicate) return [] + if ('any' in predicate) { + const groups = predicate.any + .map((node) => predicateGroupToRules(node)) + .filter((g) => g.length > 0) + return applyLogicalOperators(groups) + } + return predicateGroupToRules(predicate) +} + +/** Converts UI sort rules to a v2 `SortSpec` (ordered `{ field, direction }`). */ +export function sortRulesToSortSpec(rules: SortRule[]): SortSpec | null { + const spec: SortSpec = [] + for (const rule of rules) { + if (rule.column) spec.push({ field: rule.column, direction: rule.direction }) + } + return spec.length > 0 ? spec : null +} + +function predicateLeafToFilterValue(p: Predicate): Filter[string] { + if (p.op === 'isEmpty') return { $empty: true } + if (p.op === 'isNotEmpty') return { $empty: false } + if (p.op === 'eq') return p.value as Filter[string] + return { [`$${p.op}`]: p.value } as Filter[string] +} + +/** + * Converts a v2 `TablePredicate` to a legacy `$`-grammar `Filter`. Lossless — + * both compile through the same `fieldPredicate` leaf, so the resulting SQL is + * identical. Lets v2 surfaces author in the predicate grammar while the bulk + * update/delete engine (sync + async-job paths) keeps consuming `Filter`. + */ +export function predicateToFilter(predicate: TablePredicate): Filter { + const nodeToFilter = (node: Predicate | TablePredicate): Filter => { + if ('field' in node) return { [node.field]: predicateLeafToFilterValue(node) } + if ('all' in node) return { $and: node.all.map(nodeToFilter) } + return { $or: node.any.map(nodeToFilter) } + } + return nodeToFilter(predicate) +} diff --git a/apps/sim/lib/table/rows/__tests__/cursor.test.ts b/apps/sim/lib/table/rows/__tests__/cursor.test.ts new file mode 100644 index 00000000000..606af572932 --- /dev/null +++ b/apps/sim/lib/table/rows/__tests__/cursor.test.ts @@ -0,0 +1,48 @@ +/** + * @vitest-environment node + */ +import { describe, expect, it } from 'vitest' +import { decodeCursor, encodeCursor } from '@/lib/table/rows/cursor' + +describe('cursor codec', () => { + it('encodes a keyset cursor for the default order and round-trips it', () => { + const token = encodeCursor({ + lastRow: { id: 'row-9', orderKey: 'a0' }, + nextOffset: 100, + }) + expect(typeof token).toBe('string') + expect(decodeCursor(token)).toEqual({ after: { orderKey: 'a0', id: 'row-9' } }) + }) + + it('falls back to an offset cursor when a custom sort is active', () => { + const token = encodeCursor({ + lastRow: { id: 'row-9', orderKey: 'a0' }, + sort: { wins: 'desc' }, + nextOffset: 100, + }) + expect(decodeCursor(token)).toEqual({ offset: 100 }) + }) + + it('falls back to an offset cursor when the row has no order key (legacy)', () => { + const token = encodeCursor({ + lastRow: { id: 'row-9', orderKey: undefined }, + nextOffset: 50, + }) + expect(decodeCursor(token)).toEqual({ offset: 50 }) + }) + + it('produces opaque base64url with no raw orderKey/offset leaking', () => { + const token = encodeCursor({ lastRow: { id: 'r', orderKey: 'zz' }, nextOffset: 0 }) + expect(token).not.toContain('orderKey') + expect(token).not.toContain('{') + expect(token).toMatch(/^[A-Za-z0-9_-]+$/) + }) + + it('throws on a malformed cursor', () => { + expect(() => decodeCursor('not-base64-$$$')).toThrow('Invalid cursor') + // Valid base64url but wrong shape. + expect(() => decodeCursor(Buffer.from('{"x":1}').toString('base64url'))).toThrow( + 'Invalid cursor' + ) + }) +}) diff --git a/apps/sim/lib/table/rows/cursor.ts b/apps/sim/lib/table/rows/cursor.ts new file mode 100644 index 00000000000..895c2c72cb2 --- /dev/null +++ b/apps/sim/lib/table/rows/cursor.ts @@ -0,0 +1,59 @@ +/** + * Opaque pagination cursor for the v2 table-query surface. + * + * The token is a base64url-encoded JSON payload that hides whether paging is + * keyset- or offset-based, so callers (the v2 tools, the agent) only ever echo + * an opaque `cursor` and never juggle `orderKey`/`id`/`offset` themselves. + * + * - Default order → keyset on `(order_key, id)` (`{ k, i }`), an index seek. + * - Sorted views / rows not yet order-key-backfilled → offset fallback (`{ o }`), + * because `(order_key, id)` keyset can't seek a data-column ordering. + */ + +import type { Sort, TableRow, TableRowsCursor } from '@/lib/table/types' + +type CursorPayload = { k: string; i: string } | { o: number } + +function toBase64Url(json: string): string { + return Buffer.from(json, 'utf8').toString('base64url') +} + +function fromBase64Url(token: string): string { + return Buffer.from(token, 'base64url').toString('utf8') +} + +/** + * Builds the cursor for the page *after* `lastRow`. Uses keyset for the default + * order when the row carries an `orderKey`; otherwise falls back to `nextOffset` + * (the offset to resume at = current offset + rows returned). + */ +export function encodeCursor(args: { + lastRow: Pick + sort?: Sort + nextOffset: number +}): string { + const isDefaultOrder = !args.sort || Object.keys(args.sort).length === 0 + const payload: CursorPayload = + isDefaultOrder && args.lastRow.orderKey + ? { k: args.lastRow.orderKey, i: args.lastRow.id } + : { o: args.nextOffset } + return toBase64Url(JSON.stringify(payload)) +} + +/** Decodes an opaque cursor into the `queryRows` paging inputs it stands for. */ +export function decodeCursor(token: string): { after?: TableRowsCursor; offset?: number } { + let payload: CursorPayload + try { + payload = JSON.parse(fromBase64Url(token)) + } catch { + throw new Error('Invalid cursor') + } + + if ('k' in payload && 'i' in payload) { + return { after: { orderKey: String(payload.k), id: String(payload.i) } } + } + if ('o' in payload && typeof payload.o === 'number' && Number.isInteger(payload.o)) { + return { offset: payload.o } + } + throw new Error('Invalid cursor') +} diff --git a/apps/sim/lib/table/rows/service.ts b/apps/sim/lib/table/rows/service.ts index 988695e87b5..d51955225d1 100644 --- a/apps/sim/lib/table/rows/service.ts +++ b/apps/sim/lib/table/rows/service.ts @@ -28,6 +28,7 @@ import { getColumnId } from '@/lib/table/column-keys' import { TABLE_LIMITS, USER_TABLE_ROWS_SQL_NAME } from '@/lib/table/constants' import { nKeysBetween } from '@/lib/table/order-key' import { type DbExecutor, type DbTransaction, withSeqscanOff } from '@/lib/table/planner' +import { encodeCursor } from '@/lib/table/rows/cursor' import { applyExecutionsPatch, deriveExecClearsForDataPatch, @@ -46,7 +47,13 @@ import { resolveBatchInsertOrderKeys, resolveInsertOrderKey, } from '@/lib/table/rows/ordering' -import { buildFilterClause, buildSortClause, escapeLikePattern } from '@/lib/table/sql' +import { + buildFilterClause, + buildPredicateClause, + buildSortClause, + escapeLikePattern, + fieldPredicate, +} from '@/lib/table/sql' import { fireTableTrigger } from '@/lib/table/trigger' import { scaledStatementTimeoutMs, setTableTxTimeouts } from '@/lib/table/tx' import type { @@ -415,15 +422,19 @@ export async function replaceTableRowsWithTx( if (uniqueColumns.length > 0 && data.rows.length > 0) { const seen = new Map>() for (const col of uniqueColumns) { - seen.set(col.name, new Map()) + seen.set(getColumnId(col), new Map()) } for (let i = 0; i < data.rows.length; i++) { const row = data.rows[i] for (const col of uniqueColumns) { - const value = row[col.name] + // Coerced rows are keyed by column id, not display name — reading + // `row[col.name]` silently misses renamed columns and lets dupes through. + const colId = getColumnId(col) + const value = row[colId] if (value === null || value === undefined) continue - const normalized = typeof value === 'string' ? value.toLowerCase() : JSON.stringify(value) - const map = seen.get(col.name)! + // Case-sensitive, consistent with the unique-constraint check leaf. + const normalized = typeof value === 'string' ? value : JSON.stringify(value) + const map = seen.get(colId)! if (map.has(normalized)) { throw new Error( `Row ${i + 1}: Column "${col.name}" must be unique. Value "${String(value)}" duplicates row ${map.get(normalized)! + 1} in batch` @@ -562,12 +573,21 @@ export async function upsertRow( throw new Error(`Upsert requires a value for the conflict target column "${targetColumnName}"`) } - // `data->` and `data->>` accept the JSON key as a parameterized text value; - // no need for `sql.raw` interpolation. - const matchFilter = - typeof targetValue === 'string' - ? sql`${userTableRows.data}->>${targetColumnKey}::text = ${String(targetValue)}` - : sql`(${userTableRows.data}->${targetColumnKey}::text)::jsonb = ${JSON.stringify(targetValue)}::jsonb` + // Build the conflict probe through the SAME leaf as the unique-constraint check + // (`fieldPredicate` → case-sensitive JSONB containment). This is what makes + // "find the row to update" and "is this value unique" agree: a value differing + // only in case can no longer slip past the probe and then trip the guard. + // `eq` always yields a clause for a non-null value (guaranteed above). + const matchFilter = fieldPredicate( + USER_TABLE_ROWS_SQL_NAME, + targetColumnKey, + 'eq', + targetValue, + table.schema.columns.find((c) => getColumnId(c) === targetColumnKey)?.type + ) + if (!matchFilter) { + throw new Error('Failed to build upsert conflict predicate') + } // Resolve the plan limit BEFORE the tx (the lookup is a separate pool read; doing // it inside the tx would hold a connection + the row-order lock during it). The @@ -955,6 +975,7 @@ export async function queryRows( ): Promise { const { filter, + predicate, sort, limit = TABLE_LIMITS.DEFAULT_QUERY_LIMIT, offset = 0, @@ -979,12 +1000,17 @@ export async function queryRows( deleteMask ) + // v2 predicate takes precedence over the legacy `$`-filter; both compile to a + // WHERE through the same `fieldPredicate` leaf. + const userClause = predicate + ? buildPredicateClause(predicate, tableName, columns) + : filter && Object.keys(filter).length > 0 + ? buildFilterClause(filter, tableName, columns) + : undefined + let whereClause = baseConditions - if (filter && Object.keys(filter).length > 0) { - const filterClause = buildFilterClause(filter, tableName, columns) - if (filterClause) { - whereClause = and(baseConditions, filterClause) - } + if (userClause) { + whereClause = and(baseConditions, userClause) } // Keyset page: seek past the cursor on the default `(order_key, id)` order instead of paying @@ -1016,7 +1042,7 @@ export async function queryRows( // the planner seq-scans and sorts the whole shared relation on every page // (9.7s measured on a 1M-row table; 0.76s tenant-bounded). Default-order // pages already stream the `(table_id, order_key, id)` index. - const hasFilter = Boolean(filter && Object.keys(filter).length > 0) + const hasFilter = Boolean(userClause) const rowsPromise = sort ? withSeqscanOff(async (trx) => buildPageQuery(trx)) : buildPageQuery(db) const countPromise = includeTotal ? hasFilter @@ -1041,20 +1067,30 @@ export async function queryRows( `[${requestId}] Queried ${rows.length} rows from table ${table.id} (total: ${totalCount})` ) + const mappedRows = rows.map((r) => ({ + id: r.id, + data: r.data as RowData, + executions: executionsByRow?.get(r.id) ?? {}, + position: r.position, + orderKey: r.orderKey ?? undefined, + createdAt: r.createdAt, + updatedAt: r.updatedAt, + })) + + // Opaque next-page cursor: null once we've returned a short (final) page. + const lastRow = mappedRows[mappedRows.length - 1] + const nextCursor = + mappedRows.length < limit || !lastRow + ? null + : encodeCursor({ lastRow, sort, nextOffset: offset + mappedRows.length }) + return { - rows: rows.map((r) => ({ - id: r.id, - data: r.data as RowData, - executions: executionsByRow?.get(r.id) ?? {}, - position: r.position, - orderKey: r.orderKey ?? undefined, - createdAt: r.createdAt, - updatedAt: r.updatedAt, - })), + rows: mappedRows, rowCount: rows.length, totalCount, limit, offset, + nextCursor, } } diff --git a/apps/sim/lib/table/sql.ts b/apps/sim/lib/table/sql.ts index 1e4f4a35ea0..b1cc6360335 100644 --- a/apps/sim/lib/table/sql.ts +++ b/apps/sim/lib/table/sql.ts @@ -13,8 +13,12 @@ import type { ColumnDefinition, ConditionOperators, Filter, + FilterOp, JsonValue, + Predicate, + PredicateNode, Sort, + TablePredicate, } from '@/lib/table/types' /** @@ -178,6 +182,48 @@ function buildFilterClauseInternal( return sql.join(conditions, sql.raw(' AND ')) } +/** + * Builds a WHERE clause from a v2 `TablePredicate` (nestable `all`/`any` groups + * of `{ field, op, value }` leaves). Sibling of `buildFilterClause`: same engine, + * same `fieldPredicate` leaf — only the grammar differs. Returns `undefined` when + * the tree contributes no conditions (empty groups, all-no-op leaves). + * + * @throws {TableQueryValidationError} if a field name is invalid or an operator is not allowed + */ +export function buildPredicateClause( + predicate: TablePredicate, + tableName: string, + columns: ColumnDefinition[] +): SQL | undefined { + return buildPredicateNode(predicate, tableName, buildColumnTypeMap(columns)) +} + +function isPredicateGroup(node: PredicateNode): node is TablePredicate { + return 'all' in node || 'any' in node +} + +function buildPredicateNode( + node: PredicateNode, + tableName: string, + columnTypeMap: ColumnTypeMap +): SQL | undefined { + if (isPredicateGroup(node)) { + const isAll = 'all' in node + const members = isAll ? node.all : node.any + const clauses: SQL[] = [] + for (const member of members) { + const clause = buildPredicateNode(member, tableName, columnTypeMap) + if (clause) clauses.push(clause) + } + if (clauses.length === 0) return undefined + if (clauses.length === 1) return clauses[0] + return sql`(${sql.join(clauses, sql.raw(isAll ? ' AND ' : ' OR '))})` + } + + const leaf = node as Predicate + return fieldPredicate(tableName, leaf.field, leaf.op, leaf.value, columnTypeMap.get(leaf.field)) +} + /** * Builds an ORDER BY clause from a sort object. * @@ -313,104 +359,106 @@ function buildFieldCondition( if (typeof condition === 'object' && condition !== null && !Array.isArray(condition)) { for (const [op, value] of Object.entries(condition)) { - // Validate operator to ensure only allowed operators are used + // Validate against the legacy `$`-whitelist, then normalize onto the shared + // `FilterOp` so v1 and v2 emit byte-identical leaf SQL. validateOperator(op) - switch (op) { - case '$eq': - conditions.push(buildContainmentClause(tableName, field, value as JsonValue)) - break - - case '$ne': - conditions.push( - sql`NOT (${buildContainmentClause(tableName, field, value as JsonValue)})` - ) - break - - case '$gt': - conditions.push( - buildComparisonClause(tableName, field, '>', value as number | string, columnType) - ) - break - - case '$gte': - conditions.push( - buildComparisonClause(tableName, field, '>=', value as number | string, columnType) - ) - break - - case '$lt': - conditions.push( - buildComparisonClause(tableName, field, '<', value as number | string, columnType) - ) - break - - case '$lte': - conditions.push( - buildComparisonClause(tableName, field, '<=', value as number | string, columnType) - ) - break - - case '$in': - if (Array.isArray(value) && value.length > 0) { - if (value.length === 1) { - // Single value then use containment clause - conditions.push(buildContainmentClause(tableName, field, value[0])) - } else { - // Multiple values then use OR clause - const inConditions = value.map((v) => buildContainmentClause(tableName, field, v)) - conditions.push(sql`(${sql.join(inConditions, sql.raw(' OR '))})`) - } - } - break - - case '$nin': - if (Array.isArray(value) && value.length > 0) { - const ninConditions = value.map( - (v) => sql`NOT (${buildContainmentClause(tableName, field, v)})` - ) - conditions.push(sql`(${sql.join(ninConditions, sql.raw(' AND '))})`) - } - break - - case '$contains': - conditions.push(buildLikeClause(tableName, field, value as string, 'contains')) - break - - case '$ncontains': - conditions.push( - buildLikeClause(tableName, field, value as string, 'contains', { negate: true }) - ) - break - - case '$startsWith': - conditions.push(buildLikeClause(tableName, field, value as string, 'startsWith')) - break - - case '$endsWith': - conditions.push(buildLikeClause(tableName, field, value as string, 'endsWith')) - break - - case '$empty': - conditions.push(buildEmptyClause(tableName, field, coerceEmptyFlag(field, value))) - break - - default: - // This should never happen due to validateOperator, but added for completeness. - // Throw a plain Error (→ 500) since reaching this default means the switch - // and ALLOWED_OPERATORS have drifted — that's a programmer error, not a caller error. - throw new Error(`Unsupported operator: ${op}`) + if (op === '$empty') { + // `$empty: true/false` maps onto the valueless v2 ops. + const filterOp: FilterOp = coerceEmptyFlag(field, value) ? 'isEmpty' : 'isNotEmpty' + const clause = fieldPredicate(tableName, field, filterOp, undefined, columnType) + if (clause) conditions.push(clause) + continue } + + // Every other `$op` is `op` minus the leading `$` (e.g. `$gte` → `gte`). + const clause = fieldPredicate(tableName, field, op.slice(1) as FilterOp, value, columnType) + if (clause) conditions.push(clause) } } else { // Simple value (primitive or null) - shorthand for equality. // Example: { name: 'John' } is equivalent to { name: { $eq: 'John' } } - conditions.push(buildContainmentClause(tableName, field, condition)) + const clause = fieldPredicate(tableName, field, 'eq', condition, columnType) + if (clause) conditions.push(clause) } return conditions } +/** + * The single leaf primitive: compiles one `field op value` into SQL. Every + * matcher routes through here — both filter compilers (`buildFilterClause` for + * the legacy `$`-grammar, `buildPredicateClause` for the v2 grammar), the upsert + * conflict probe, and the unique-constraint checks. Centralizing the leaf means + * equality/case/null/cast semantics are defined exactly once, so "find the row" + * and "is this value unique" can never disagree. + * + * Returns `undefined` when the predicate is a no-op (empty `in`/`nin` array), + * matching the legacy behavior of emitting no clause. + * + * Equality (`eq`/`ne`/`in`/`nin`) uses case-sensitive JSONB containment (GIN + * indexed). Text matches (`contains`/`ncontains`/`startsWith`/`endsWith`) are + * ILIKE (case-insensitive). Ranges cast per column type. + */ +export function fieldPredicate( + tableName: string, + field: string, + op: FilterOp, + value: JsonValue | undefined, + columnType: ColumnType | undefined +): SQL | undefined { + validateFieldName(field) + + switch (op) { + case 'eq': + return buildContainmentClause(tableName, field, value as JsonValue) + + case 'ne': + return sql`NOT (${buildContainmentClause(tableName, field, value as JsonValue)})` + + case 'gt': + return buildComparisonClause(tableName, field, '>', value as number | string, columnType) + case 'gte': + return buildComparisonClause(tableName, field, '>=', value as number | string, columnType) + case 'lt': + return buildComparisonClause(tableName, field, '<', value as number | string, columnType) + case 'lte': + return buildComparisonClause(tableName, field, '<=', value as number | string, columnType) + + case 'in': { + if (!Array.isArray(value) || value.length === 0) return undefined + if (value.length === 1) return buildContainmentClause(tableName, field, value[0]) + const inConditions = value.map((v) => buildContainmentClause(tableName, field, v)) + return sql`(${sql.join(inConditions, sql.raw(' OR '))})` + } + + case 'nin': { + if (!Array.isArray(value) || value.length === 0) return undefined + const ninConditions = value.map( + (v) => sql`NOT (${buildContainmentClause(tableName, field, v)})` + ) + return sql`(${sql.join(ninConditions, sql.raw(' AND '))})` + } + + case 'contains': + return buildLikeClause(tableName, field, value as string, 'contains') + case 'ncontains': + return buildLikeClause(tableName, field, value as string, 'contains', { negate: true }) + case 'startsWith': + return buildLikeClause(tableName, field, value as string, 'startsWith') + case 'endsWith': + return buildLikeClause(tableName, field, value as string, 'endsWith') + + case 'isEmpty': + return buildEmptyClause(tableName, field, true) + case 'isNotEmpty': + return buildEmptyClause(tableName, field, false) + + default: + throw new TableQueryValidationError(`Invalid operator "${op}"`) + } +} + /** * Builds SQL clauses from nested filters and joins them with the specified operator. * diff --git a/apps/sim/lib/table/types.ts b/apps/sim/lib/table/types.ts index ba08af9c83e..6d5fd8922b4 100644 --- a/apps/sim/lib/table/types.ts +++ b/apps/sim/lib/table/types.ts @@ -423,6 +423,51 @@ export interface Filter { [key: string]: ColumnValue | ConditionOperators | Filter[] | undefined } +/** + * v2 filter operators (bare, no `$`). Equality and `in`/`nin` are case-sensitive + * (JSONB containment, GIN-indexed); the text ops `contains`/`ncontains`/ + * `startsWith`/`endsWith` are ILIKE (case-insensitive). `isEmpty`/`isNotEmpty` + * are valueless. This is the canonical operator set the shared `fieldPredicate` + * leaf understands; the legacy `$`-operators normalize onto it. + */ +export type FilterOp = + | 'eq' + | 'ne' + | 'gt' + | 'gte' + | 'lt' + | 'lte' + | 'in' + | 'nin' + | 'contains' + | 'ncontains' + | 'startsWith' + | 'endsWith' + | 'isEmpty' + | 'isNotEmpty' + +/** A single v2 leaf predicate: `field op value`. `value` is omitted for `isEmpty`/`isNotEmpty`. */ +export interface Predicate { + field: string + op: FilterOp + value?: JsonValue +} + +/** + * v2 nestable filter tree. A group is either `{ all: [...] }` (AND) or + * `{ any: [...] }` (OR); members are leaves or nested groups. Replaces the + * MongoDB-style `Filter` on the v2 surface — same engine, legible grammar. + * + * @example + * { all: [{ field: 'slack_user_id', op: 'in', value: ['U1','U2'] }, { field: 'wins', op: 'gte', value: 10 }] } + * { any: [{ field: 'status', op: 'eq', value: 'active' }, { field: 'status', op: 'eq', value: 'pending' }] } + */ +export type PredicateNode = Predicate | TablePredicate +export type TablePredicate = { all: PredicateNode[] } | { any: PredicateNode[] } + +/** v2 sort specification: an ordered list of `{ field, direction }`. */ +export type SortSpec = Array<{ field: string; direction: SortDirection }> + export interface ValidationResult { valid: boolean errors: string[] @@ -454,6 +499,11 @@ export interface SortRule { export interface QueryOptions { filter?: Filter + /** + * v2 nestable predicate. When set it takes precedence over `filter` — the two + * compile through the same `fieldPredicate` leaf, so callers pick one grammar. + */ + predicate?: TablePredicate sort?: Sort limit?: number offset?: number @@ -480,6 +530,12 @@ export interface QueryResult { totalCount: number | null limit: number offset: number + /** + * Opaque cursor for the next page, or `null` on the last page (fewer rows than + * `limit`). The v2 surface exposes only this — callers echo it back as `cursor` + * and never construct keyset/offset state themselves. See `rows/cursor.ts`. + */ + nextCursor: string | null } export interface BulkOperationResult { diff --git a/apps/sim/lib/table/validation.ts b/apps/sim/lib/table/validation.ts index b42e2a62b8d..5aba0083bb8 100644 --- a/apps/sim/lib/table/validation.ts +++ b/apps/sim/lib/table/validation.ts @@ -7,8 +7,14 @@ import { userTableRows } from '@sim/db/schema' import { and, eq, or, type SQL, sql } from 'drizzle-orm' import { NextResponse } from 'next/server' import { getColumnId } from '@/lib/table/column-keys' -import { COLUMN_TYPES, NAME_PATTERN, TABLE_LIMITS } from '@/lib/table/constants' +import { + COLUMN_TYPES, + NAME_PATTERN, + TABLE_LIMITS, + USER_TABLE_ROWS_SQL_NAME, +} from '@/lib/table/constants' import { withSeqscanOff } from '@/lib/table/planner' +import { fieldPredicate } from '@/lib/table/sql' import type { ColumnDefinition, JsonValue, @@ -388,12 +394,8 @@ export function validateUniqueConstraints( const duplicate = existingRows.find((row) => { if (excludeRowId && row.id === excludeRowId) return false - - const existingValue = row.data[key] - if (typeof value === 'string' && typeof existingValue === 'string') { - return value.toLowerCase() === existingValue.toLowerCase() - } - return value === existingValue + // Case-sensitive, matching the DB unique-check leaf (`fieldPredicate` eq). + return value === row.data[key] }) if (duplicate) { @@ -436,27 +438,13 @@ export async function checkUniqueConstraintsDb( for (const column of uniqueColumns) { const key = getColumnId(column) - if (!NAME_PATTERN.test(key)) { - throw new Error(`Invalid column id: ${key}`) - } - const value = data[key] if (value === null || value === undefined) continue - if (typeof value === 'string') { - conditions.push({ - column, - value, - sql: sql`lower(${userTableRows.data}->>${sql.raw(`'${key}'`)}) = ${value.toLowerCase()}`, - }) - } else { - // For other types, use direct JSONB comparison - conditions.push({ - column, - value, - sql: sql`(${userTableRows.data}->${sql.raw(`'${key}'`)})::jsonb = ${JSON.stringify(value)}::jsonb`, - }) - } + // Same leaf as the upsert conflict probe → case-sensitive JSONB containment + // (GIN-indexed). `eq` always yields a clause for a non-null value. + const clause = fieldPredicate(USER_TABLE_ROWS_SQL_NAME, key, 'eq', value, column.type) + if (clause) conditions.push({ column, value, sql: clause }) } if (conditions.length === 0) { @@ -464,9 +452,9 @@ export async function checkUniqueConstraintsDb( } // Query for each unique column separately to provide specific error messages. - // Tenant-bounded: `lower(data->>'col') = ...` is unestimatable, so the planner - // otherwise seq-scans the whole shared relation per check — 3.5s on every - // insert/edit when the value is unique (no early exit). With an external + // The predicate is now case-sensitive JSONB containment (`data @> {...}`), + // which can use the GIN index. We still pin `enable_seqscan = off` (tenant- + // bounded) defensively for the small-table / cold-stats case. With an external // transaction the flag is set on it directly — opening our own transaction // inside the caller's would be the nested pool checkout the migration- // hardening work eliminated (self-deadlock under pool exhaustion). @@ -554,8 +542,7 @@ export async function checkBatchUniqueConstraintsDb( const value = rowData[key] if (value === null || value === undefined) continue - const normalizedValue = - typeof value === 'string' ? value.toLowerCase() : JSON.stringify(value) + const normalizedValue = typeof value === 'string' ? value : JSON.stringify(value) // Check for duplicate within batch const columnValueMap = batchValueMap.get(key)! @@ -591,14 +578,22 @@ export async function checkBatchUniqueConstraintsDb( const valueArray = Array.from(values) const valueConditions = valueArray.map((normalizedValue) => { - // Check if the original values are strings (normalized values for strings are lowercase) - // We need to determine the type from the column definition or the first row that has this value - const isStringColumn = column.type === 'string' - - if (isStringColumn) { - return sql`lower(${userTableRows.data}->>${sql.raw(`'${columnId}'`)}) = ${normalizedValue}` + // Reconstruct the original typed value from its normalized key: string + // columns store the raw string; others store JSON.stringify(value). + const originalValue: JsonValue = + column.type === 'string' ? normalizedValue : JSON.parse(normalizedValue) + // Same case-sensitive containment leaf as every other matcher. + const clause = fieldPredicate( + USER_TABLE_ROWS_SQL_NAME, + columnId, + 'eq', + originalValue, + column.type + ) + if (!clause) { + throw new Error(`Failed to build unique-constraint predicate for column "${column.name}"`) } - return sql`(${userTableRows.data}->${sql.raw(`'${columnId}'`)})::jsonb = ${normalizedValue}::jsonb` + return clause }) const conflictingRows = await ex @@ -616,9 +611,7 @@ export async function checkBatchUniqueConstraintsDb( const conflictData = conflict.data as RowData const conflictValue = conflictData[columnId] const normalizedConflictValue = - typeof conflictValue === 'string' - ? conflictValue.toLowerCase() - : JSON.stringify(conflictValue) + typeof conflictValue === 'string' ? conflictValue : JSON.stringify(conflictValue) // Find which batch rows have this conflicting value for (let i = 0; i < rows.length; i++) { @@ -626,7 +619,7 @@ export async function checkBatchUniqueConstraintsDb( if (rowValue === null || rowValue === undefined) continue const normalizedRowValue = - typeof rowValue === 'string' ? rowValue.toLowerCase() : JSON.stringify(rowValue) + typeof rowValue === 'string' ? rowValue : JSON.stringify(rowValue) if (normalizedRowValue === normalizedConflictValue) { // Check if this row already has errors for this column diff --git a/apps/sim/tools/registry.ts b/apps/sim/tools/registry.ts index 7f8f537c98d..bbf8823e411 100644 --- a/apps/sim/tools/registry.ts +++ b/apps/sim/tools/registry.ts @@ -3582,6 +3582,7 @@ import { tableInsertRowTool, tableListTool, tableQueryRowsTool, + tableQueryRowsV2Tool, tableUpdateRowsByFilterTool, tableUpdateRowTool, tableUpsertRowTool, @@ -7424,6 +7425,7 @@ export const tools: Record = { table_delete_row: tableDeleteRowTool, table_delete_rows_by_filter: tableDeleteRowsByFilterTool, table_query_rows: tableQueryRowsTool, + table_query_rows_v2: tableQueryRowsV2Tool, table_get_row: tableGetRowTool, table_get_schema: tableGetSchemaTool, mailchimp_get_audiences: mailchimpGetAudiencesTool, diff --git a/apps/sim/tools/table/index.ts b/apps/sim/tools/table/index.ts index 1d4b84596c0..7b99fecedce 100644 --- a/apps/sim/tools/table/index.ts +++ b/apps/sim/tools/table/index.ts @@ -7,6 +7,7 @@ export * from './get_schema' export * from './insert_row' export * from './list' export * from './query_rows' +export * from './query_rows_v2' export * from './types' export * from './update_row' export * from './update_rows_by_filter' diff --git a/apps/sim/tools/table/query_rows_v2.ts b/apps/sim/tools/table/query_rows_v2.ts new file mode 100644 index 00000000000..709a9546d22 --- /dev/null +++ b/apps/sim/tools/table/query_rows_v2.ts @@ -0,0 +1,102 @@ +import { TABLE_LIMITS } from '@/lib/table/constants' +import type { TableQueryV2Response, TableRowQueryV2Params } from '@/tools/table/types' +import type { ToolConfig } from '@/tools/types' + +/** + * v2 row query: a structured `all`/`any` predicate grammar and opaque cursor + * pagination (no offset). Hits POST `/api/table/[tableId]/query`. + */ +export const tableQueryRowsV2Tool: ToolConfig = { + id: 'table_query_rows_v2', + name: 'Query Rows', + description: + 'Query rows with a structured predicate and cursor pagination. The predicate is a nestable tree: ' + + '{ "all": [ { "field": "status", "op": "eq", "value": "active" } ] } (all = AND, any = OR). ' + + 'Operators: eq, ne, gt, gte, lt, lte, in, nin, contains, ncontains, startsWith, endsWith, isEmpty, isNotEmpty. ' + + 'To page, pass the nextCursor from the previous response back as cursor; omit it for the first page.', + version: '1.0.0', + + params: { + tableId: { + type: 'string', + required: true, + description: 'Table ID', + visibility: 'user-only', + }, + predicate: { + type: 'json', + required: false, + description: + 'Nestable predicate tree of { all | any: [...] } groups and { field, op, value } leaves. Omit to match all rows.', + visibility: 'user-or-llm', + }, + sort: { + type: 'json', + required: false, + description: 'Ordered list of { field, direction: "asc" | "desc" }.', + visibility: 'user-or-llm', + }, + limit: { + type: 'number', + required: false, + description: `Maximum rows to return (default: ${TABLE_LIMITS.DEFAULT_QUERY_LIMIT}, max: ${TABLE_LIMITS.MAX_QUERY_LIMIT})`, + visibility: 'user-or-llm', + }, + cursor: { + type: 'string', + required: false, + description: 'Opaque pagination cursor returned by a prior query. Omit for the first page.', + visibility: 'user-or-llm', + }, + }, + + request: { + url: (params: TableRowQueryV2Params) => `/api/table/${params.tableId}/query`, + method: 'POST', + headers: () => ({ 'Content-Type': 'application/json' }), + body: (params: TableRowQueryV2Params) => { + const workspaceId = params._context?.workspaceId + if (!workspaceId) { + throw new Error('Workspace ID is required in execution context') + } + return { + workspaceId, + ...(params.predicate ? { predicate: params.predicate } : {}), + ...(params.sort ? { sort: params.sort } : {}), + ...(params.limit !== undefined ? { limit: params.limit } : {}), + ...(params.cursor ? { cursor: params.cursor } : {}), + } + }, + }, + + transformResponse: async (response): Promise => { + const result = await response.json() + const data = result.data || result + + return { + success: true, + output: { + rows: data.rows, + rowCount: data.rowCount, + totalCount: data.totalCount, + limit: data.limit, + nextCursor: data.nextCursor, + }, + } + }, + + outputs: { + success: { type: 'boolean', description: 'Whether the query succeeded' }, + rows: { type: 'array', description: 'Query result rows' }, + rowCount: { type: 'number', description: 'Number of rows returned' }, + totalCount: { + type: 'number', + description: 'Total rows matching the predicate (computed on the first page only)', + }, + limit: { type: 'number', description: 'Limit used in the query' }, + nextCursor: { + type: 'string', + description: 'Cursor to fetch the next page, or null on the last page', + }, + }, +} diff --git a/apps/sim/tools/table/types.ts b/apps/sim/tools/table/types.ts index ee3e347a590..51a9b37a8d4 100644 --- a/apps/sim/tools/table/types.ts +++ b/apps/sim/tools/table/types.ts @@ -3,7 +3,9 @@ import type { Filter, RowData, Sort, + SortSpec, TableDefinition, + TablePredicate, TableRow, TableSchema, } from '@/lib/table/types' @@ -56,6 +58,26 @@ export interface TableRowGetParams { _context?: WorkflowToolExecutionContext } +/** v2 query params: nestable predicate + opaque cursor (no offset). */ +export interface TableRowQueryV2Params { + tableId: string + predicate?: TablePredicate + sort?: SortSpec + limit?: number + cursor?: string + _context?: WorkflowToolExecutionContext +} + +export interface TableQueryV2Response extends ToolResponse { + output: { + rows: TableRow[] + rowCount: number + totalCount: number | null + limit: number + nextCursor: string | null + } +} + export interface TableCreateResponse extends ToolResponse { output: { table: TableDefinition diff --git a/scripts/check-api-validation-contracts.ts b/scripts/check-api-validation-contracts.ts index a45c2d84914..f3f42af7240 100644 --- a/scripts/check-api-validation-contracts.ts +++ b/scripts/check-api-validation-contracts.ts @@ -9,8 +9,8 @@ const QUERY_HOOKS_DIR = path.join(ROOT, 'apps/sim/hooks/queries') const SELECTOR_HOOKS_DIR = path.join(ROOT, 'apps/sim/hooks/selectors') const BASELINE = { - totalRoutes: 862, - zodRoutes: 862, + totalRoutes: 863, + zodRoutes: 863, nonZodRoutes: 0, } as const From d74c0c0b003bc979411740b3af0c2d4bb05f3939 Mon Sep 17 00:00:00 2001 From: Theodore Li Date: Fri, 3 Jul 2026 12:11:33 -0700 Subject: [PATCH 02/26] feat(table): switch v2 filter grammar to PostgREST strings + unbounded query with byte guard - New parser lib/table/query-builder/postgrest.ts: PostgREST querystring (wins=gte.10&status=in.(active,pending), or=()/and=() groups, not. prefix) -> TablePredicate IR; serializers for the visual builder path - Contract/tool/route/copilot user_table: filter/order are PostgREST strings, parsed + validated server-side - table_v2 block: Builder/Editor filter mode dropdown (visual builders serialize to PostgREST), builder-only fields hidden from the LLM - queryRows: no default row limit; 10MB result byte guard; engine like/ilike ops; bulk update/delete deterministic (order_key, id) ordering under limit Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01Bd25zCzh21omjFhRz32ov6 --- .../app/api/table/[tableId]/query/route.ts | 19 +- .../sim/app/api/table/[tableId]/rows/route.ts | 34 +- apps/sim/blocks/blocks/table_v2.ts | 371 +++++++---------- .../api/contracts/tables-predicate.test.ts | 97 ++--- apps/sim/lib/api/contracts/tables.ts | 118 ++---- .../lib/copilot/generated/tool-schemas-v1.ts | 190 ++++----- .../tools/handlers/function-execute.ts | 9 +- .../tools/server/table/user-table.test.ts | 39 +- .../copilot/tools/server/table/user-table.ts | 45 ++- .../service-filter-threading.test.ts | 62 +++ apps/sim/lib/table/__tests__/sql.test.ts | 38 +- apps/sim/lib/table/constants.ts | 7 + .../__tests__/converters.test.ts | 55 +++ .../query-builder/__tests__/postgrest.test.ts | 197 +++++++++ .../sim/lib/table/query-builder/converters.ts | 35 +- apps/sim/lib/table/query-builder/postgrest.ts | 374 ++++++++++++++++++ apps/sim/lib/table/rows/service.ts | 54 ++- apps/sim/lib/table/sql.ts | 74 ++++ apps/sim/lib/table/types.ts | 14 +- apps/sim/tools/registry.minimal.ts | 23 ++ apps/sim/tools/table/query_rows_v2.ts | 34 +- apps/sim/tools/table/types.ts | 8 +- 22 files changed, 1358 insertions(+), 539 deletions(-) create mode 100644 apps/sim/lib/table/query-builder/__tests__/postgrest.test.ts create mode 100644 apps/sim/lib/table/query-builder/postgrest.ts diff --git a/apps/sim/app/api/table/[tableId]/query/route.ts b/apps/sim/app/api/table/[tableId]/query/route.ts index a94f64fa767..9c25c4b942d 100644 --- a/apps/sim/app/api/table/[tableId]/query/route.ts +++ b/apps/sim/app/api/table/[tableId]/query/route.ts @@ -7,6 +7,7 @@ import { checkSessionOrInternalAuth } from '@/lib/auth/hybrid' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import type { Sort, TableSchema } from '@/lib/table' +import { parsePostgrestFilter, parsePostgrestOrder } from '@/lib/table/query-builder/postgrest' import { decodeCursor } from '@/lib/table/rows/cursor' import { queryRows } from '@/lib/table/rows/service' import { TableQueryValidationError } from '@/lib/table/sql' @@ -20,8 +21,8 @@ interface TableQueryV2RouteParams { } /** - * POST /api/table/[tableId]/query — v2 row query. Structured `all`/`any` - * predicate grammar + opaque cursor pagination (no offset on the wire). Shares + * POST /api/table/[tableId]/query — v2 row query. PostgREST filter/order strings + * (parsed server-side) + opaque cursor pagination (no offset on the wire). Shares * the same engine as the legacy GET /rows route via `queryRows`. */ export const POST = withRouteHandler( @@ -50,10 +51,18 @@ export const POST = withRouteHandler( return NextResponse.json({ error: 'Invalid workspace ID' }, { status: 400 }) } - const wire = rowWireTranslators(authResult.authType, table.schema as TableSchema) + const schema = table.schema as TableSchema + const wire = rowWireTranslators(authResult.authType, schema) const cursor = body.cursor ? decodeCursor(body.cursor) : undefined - const sortSpec = body.sort ? wire.sortSpecIn(body.sort) : undefined + // PostgREST filter/order arrive as strings (column NAMES); parse with the + // schema for type coercion, then translate names → storage ids. + const predicate = body.filter + ? wire.predicateIn(parsePostgrestFilter(body.filter, schema.columns)) + : undefined + const sortSpec = body.order + ? wire.sortSpecIn(parsePostgrestOrder(body.order, schema.columns)) + : undefined const sort: Sort | undefined = sortSpec?.length ? Object.fromEntries(sortSpec.map((s) => [s.field, s.direction])) : undefined @@ -61,7 +70,7 @@ export const POST = withRouteHandler( const result = await queryRows( table, { - predicate: body.predicate ? wire.predicateIn(body.predicate) : undefined, + predicate, sort, limit: body.limit, after: cursor?.after, diff --git a/apps/sim/app/api/table/[tableId]/rows/route.ts b/apps/sim/app/api/table/[tableId]/rows/route.ts index ca239534696..190a6e700ab 100644 --- a/apps/sim/app/api/table/[tableId]/rows/route.ts +++ b/apps/sim/app/api/table/[tableId]/rows/route.ts @@ -13,7 +13,14 @@ import { isZodError, validationErrorResponse } from '@/lib/api/server/validation import { type AuthTypeValue, checkSessionOrInternalAuth } from '@/lib/auth/hybrid' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import type { Filter, RowData, Sort, TableRowsCursor, TableSchema } from '@/lib/table' +import type { + ColumnDefinition, + Filter, + RowData, + Sort, + TableRowsCursor, + TableSchema, +} from '@/lib/table' import { batchInsertRows, batchUpdateRows, @@ -25,6 +32,8 @@ import { validateRowData, validateRowSize, } from '@/lib/table' +import { predicateToFilter } from '@/lib/table/query-builder/converters' +import { parsePostgrestFilter } from '@/lib/table/query-builder/postgrest' import { queryRows } from '@/lib/table/rows/service' import { TableQueryValidationError } from '@/lib/table/sql' import { rowWireTranslators } from '@/app/api/table/row-wire' @@ -32,6 +41,15 @@ import { accessError, checkAccess, rowWriteErrorResponse } from '@/app/api/table const logger = createLogger('TableRowsAPI') +/** + * Resolves a bulk-op filter to a name-keyed legacy `Filter`. v2 callers send a + * PostgREST string (parsed → predicate → legacy Filter via the shared leaf); + * v1 callers send the legacy object as-is. Caller then runs `wire.filterIn`. + */ +function resolveBulkFilter(raw: string | Filter, columns: ColumnDefinition[]): Filter { + return typeof raw === 'string' ? predicateToFilter(parsePostgrestFilter(raw, columns)) : raw +} + interface TableRowsRouteParams { params: Promise<{ tableId: string }> } @@ -352,7 +370,12 @@ export const PUT = withRouteHandler( const result = await updateRowsByFilter( table, { - filter: wire.filterIn(validated.filter as Filter), + filter: wire.filterIn( + resolveBulkFilter( + validated.filter as string | Filter, + (table.schema as TableSchema).columns + ) + ), data: patchData, limit: validated.limit, actorUserId: authResult.userId, @@ -457,7 +480,12 @@ export const DELETE = withRouteHandler( const result = await deleteRowsByFilter( table, { - filter: wire.filterIn(validated.filter as Filter), + filter: wire.filterIn( + resolveBulkFilter( + validated.filter as string | Filter, + (table.schema as TableSchema).columns + ) + ), limit: validated.limit, }, requestId diff --git a/apps/sim/blocks/blocks/table_v2.ts b/apps/sim/blocks/blocks/table_v2.ts index 9bf284ac99a..e6972c1a2dd 100644 --- a/apps/sim/blocks/blocks/table_v2.ts +++ b/apps/sim/blocks/blocks/table_v2.ts @@ -2,34 +2,21 @@ import { toError } from '@sim/utils/errors' import { TableIcon } from '@/components/icons' import { TABLE_LIMITS } from '@/lib/table/constants' import { - filterRulesToPredicate, - predicateToFilter, - sortRulesToSortSpec, + filterRulesToPostgrest, + sortRulesToPostgrestOrder, } from '@/lib/table/query-builder/converters' -import type { TablePredicate } from '@/lib/table/types' +import type { FilterRule, SortRule } from '@/lib/table/types' import type { BlockConfig } from '@/blocks/types' import type { TableQueryV2Response } from '@/tools/table/types' import { getTrigger } from '@/triggers' -/** Resolve a bulk-op filter from the predicate grammar (builder or JSON editor). */ -function resolveBulkPredicate( - mode: string | undefined, - builder: unknown, - editor: string | unknown -): TablePredicate | null { - if (mode === 'builder' && builder) { - return filterRulesToPredicate(builder as Parameters[0]) - } - if (editor) return parseJSON(editor, 'Predicate') as TablePredicate - return null -} - /** - * Table v2 — same operations as the v1 Table block, but `query_rows` speaks the - * legible `all`/`any` predicate grammar and paginates with an opaque cursor - * (no offset). The filter compiler, upsert conflict probe, and unique checks all - * share one case-sensitive containment leaf, so upserts can't wedge on a - * case-mismatched unique value the way they could under v1. + * Table v2 — same operations as the v1 Table block, but the filter grammar is + * PostgREST (`wins=gte.10&status=in.(active,pending)`), parsed + validated + * server-side. Pagination is an opaque cursor (no offset). The filter compiler, + * upsert conflict probe, and unique checks share one case-sensitive containment + * leaf, so upserts can't wedge on a case-mismatched unique value the way they + * could under v1. */ function parseJSON(value: string | unknown, fieldName: string): unknown { @@ -57,27 +44,58 @@ interface TableBlockParams { rowId?: string data?: string | unknown rows?: string | unknown - bulkPredicate?: string | unknown - predicate?: string | unknown - sort?: string | unknown - limit?: string - builderMode?: string + filterMode?: string filterBuilder?: unknown sortBuilder?: unknown - bulkFilterMode?: string - bulkFilterBuilder?: unknown + filter?: string + order?: string + limit?: string + cursor?: string conflictColumn?: string } +/** + * Resolves the effective PostgREST filter/order from either the visual builders + * (when Filter Mode is "builder") or the raw querystring fields. Both modes + * serialize to the same PostgREST wire format the route parses. Mode defaults to + * "builder" to match the dropdown default. + */ +function resolveFilter(params: TableBlockParams): string | undefined { + // Serialize the visual builder only when it holds real rules. The builder + // shape is a UI affordance — the agent authors the PostgREST `filter` string + // directly and never populates it, so anything that isn't a non-empty + // FilterRule[] falls back to the string (never iterate a non-array, which + // threw "rules is not iterable" at runtime). + if ( + params.filterMode !== 'editor' && + Array.isArray(params.filterBuilder) && + params.filterBuilder.length > 0 + ) { + return filterRulesToPostgrest(params.filterBuilder as FilterRule[]) ?? undefined + } + return params.filter || undefined +} + +function resolveOrder(params: TableBlockParams): string | undefined { + if ( + params.filterMode !== 'editor' && + Array.isArray(params.sortBuilder) && + params.sortBuilder.length > 0 + ) { + return sortRulesToPostgrestOrder(params.sortBuilder as SortRule[]) ?? undefined + } + return params.order || undefined +} + interface ParsedParams { tableId?: string rowId?: string data?: unknown rows?: unknown - filter?: unknown - predicate?: unknown - sort?: unknown + filter?: string + order?: string limit?: number + cursor?: string conflictTarget?: string } @@ -104,40 +122,24 @@ const paramTransformers: Record ParsedPara data: parseJSON(params.data, 'Row Data'), }), - // Bulk write-by-filter authors in the v2 predicate grammar, then converts to a - // legacy `Filter` for the existing bulk engine (sync + async-job paths). The - // conversion is lossless — both compile through the same `fieldPredicate` leaf. - update_rows_by_filter: (params) => { - const predicate = resolveBulkPredicate( - params.bulkFilterMode, - params.bulkFilterBuilder, - params.bulkPredicate - ) - return { - tableId: params.tableId, - filter: predicate ? predicateToFilter(predicate) : undefined, - data: parseJSON(params.data, 'Row Data'), - limit: params.limit ? Number.parseInt(params.limit) : undefined, - } - }, + // Bulk write-by-filter takes a PostgREST string, parsed server-side. + update_rows_by_filter: (params) => ({ + tableId: params.tableId, + filter: resolveFilter(params), + data: parseJSON(params.data, 'Row Data'), + limit: params.limit ? Number.parseInt(params.limit) : undefined, + }), delete_row: (params) => ({ tableId: params.tableId, rowId: params.rowId, }), - delete_rows_by_filter: (params) => { - const predicate = resolveBulkPredicate( - params.bulkFilterMode, - params.bulkFilterBuilder, - params.bulkPredicate - ) - return { - tableId: params.tableId, - filter: predicate ? predicateToFilter(predicate) : undefined, - limit: params.limit ? Number.parseInt(params.limit) : undefined, - } - }, + delete_rows_by_filter: (params) => ({ + tableId: params.tableId, + filter: resolveFilter(params), + limit: params.limit ? Number.parseInt(params.limit) : undefined, + }), get_row: (params) => ({ tableId: params.tableId, @@ -148,33 +150,14 @@ const paramTransformers: Record ParsedPara tableId: params.tableId, }), - query_rows: (params) => { - let predicate: unknown - if (params.builderMode === 'builder' && params.filterBuilder) { - predicate = - filterRulesToPredicate( - params.filterBuilder as Parameters[0] - ) || undefined - } else if (params.predicate) { - predicate = parseJSON(params.predicate, 'Predicate') - } - - let sort: unknown - if (params.builderMode === 'builder' && params.sortBuilder) { - sort = - sortRulesToSortSpec(params.sortBuilder as Parameters[0]) || - undefined - } else if (params.sort) { - sort = parseJSON(params.sort, 'Sort') - } - - return { - tableId: params.tableId, - predicate, - sort, - limit: params.limit ? Number.parseInt(params.limit) : 100, - } - }, + query_rows: (params) => ({ + tableId: params.tableId, + filter: resolveFilter(params), + order: resolveOrder(params), + // No limit → return all matching rows (server enforces a 10MB byte guard). + limit: params.limit ? Number.parseInt(params.limit) : undefined, + cursor: params.cursor || undefined, + }), } export const TableV2Block: BlockConfig = { @@ -183,16 +166,20 @@ export const TableV2Block: BlockConfig = { description: 'User-defined data tables', longDescription: 'Create and manage custom data tables. Store, query, and manipulate structured data within workflows. ' + - 'Query Rows filters with a predicate tree — { all: [...] } is AND, { any: [...] } is OR, and each leaf is ' + - '{ field, op, value }. Operators: eq, ne, gt, gte, lt, lte, in, nin, contains, ncontains, startsWith, ' + - 'endsWith, isEmpty, isNotEmpty (equality and in are case-sensitive; the text matches are case-insensitive). ' + - 'Pagination is cursor-based: pass the nextCursor from a prior result to fetch the next page.', + 'Query Rows filters with a PostgREST querystring — `wins=gte.10&status=in.(active,pending)` (top-level ' + + 'params AND; `or=(a.eq.1,b.eq.2)` / `and=(...)` for groups). Operators: eq, neq, gt, gte, lt, lte, in, ' + + 'like, ilike, match, imatch, is.null (negate with not., e.g. not.in, not.is.null). Order is `wins.desc,name.asc`. ' + + 'Query Rows returns every matching row by default (no row cap; the server fails fast if the result exceeds 10MB). ' + + 'Set a limit to page — pass the nextCursor from a prior result to fetch the next page.', bestPractices: ` -- To fetch specific rows, use Query Rows with a predicate (e.g. { all: [{ field: "slack_user_id", op: "in", value: ["U1","U2"] }] }) — do NOT read every row and filter downstream with a Condition block. -- Use "Get Row by ID" only when you have the row's id; there is no fetch-by-value besides the predicate. -- Combine conditions with all (AND) / any (OR); groups nest. -- Example: players who won ≥10 and are active → { all: [{ field: "wins", op: "gte", value: 10 }, { field: "status", op: "eq", value: "active" }] }. -- To page through results, pass the previous response's nextCursor back as the cursor; omit it for the first page.`, +- To fetch specific rows, use Query Rows with a PostgREST filter (e.g. slack_user_id=in.(U1,U2)) — do NOT read every row and filter downstream with a Condition block. +- Use "Get Row by ID" only when you have the row's id; otherwise filter with the querystring. +- Top-level params AND together; use or=(...) / and=(...) for explicit or nested logic. +- Example: players who won ≥10 and are active → wins=gte.10&status=eq.active. +- like/ilike use * as the wildcard (e.g. name=ilike.*jo*); match/imatch are regex. +- Query Rows returns ALL matching rows by default — leave Limit empty unless you want a bounded page. +- To page through a bounded result, set a Limit and pass the previous response's nextCursor back as the cursor; omit it for the first page. +- Columns are scalar (string/number/boolean/date) or opaque json — there are no array columns, so cs/cd/ov (containment/overlap) are unsupported; for substring use ilike.*x*.`, docsLink: 'https://docs.simstudio.ai/tools/table', category: 'blocks', bgColor: '#10B981', @@ -326,183 +313,116 @@ Return ONLY the rows array:`, }, }, - // Bulk write filter — v2 predicate grammar, converted to a Filter for the bulk engine + // Filter mode — visual builders vs raw PostgREST querystring (query + bulk). + // Builder UI is human-only; the agent authors the PostgREST `filter`/`order` + // strings, so the mode + builders are hidden from the tool-input context. { - id: 'bulkFilterMode', + id: 'filterMode', title: 'Filter Mode', type: 'dropdown', + paramVisibility: 'user-only', options: [ { label: 'Builder', id: 'builder' }, - { label: 'Editor', id: 'json' }, + { label: 'Editor', id: 'editor' }, ], value: () => 'builder', condition: { field: 'operation', - value: ['update_rows_by_filter', 'delete_rows_by_filter'], + value: ['query_rows', 'update_rows_by_filter', 'delete_rows_by_filter'], }, }, + + // Visual filter builder (builder mode) — query + bulk update/delete { - id: 'bulkFilterBuilder', + id: 'filterBuilder', title: 'Filter Conditions', type: 'filter-builder', + paramVisibility: 'user-only', required: { field: 'operation', value: ['update_rows_by_filter', 'delete_rows_by_filter'], + and: { field: 'filterMode', value: 'builder' }, }, condition: { field: 'operation', - value: ['update_rows_by_filter', 'delete_rows_by_filter'], - and: { field: 'bulkFilterMode', value: 'builder' }, - }, - }, - { - id: 'bulkPredicate', - title: 'Predicate', - type: 'code', - placeholder: '{"all": [{"field": "status", "op": "eq", "value": "active"}]}', - condition: { - field: 'operation', - value: ['update_rows_by_filter', 'delete_rows_by_filter'], - and: { field: 'bulkFilterMode', value: 'json' }, - }, - required: true, - wandConfig: { - enabled: true, - maintainHistory: true, - prompt: `Generate a predicate for selecting rows to modify. - -### CONTEXT -{context} - -### INSTRUCTION -Return ONLY a valid JSON predicate. No explanations or markdown. - -A predicate is a nestable tree: { "all": [...] } means AND, { "any": [...] } means OR, and each leaf is { "field", "op", "value" }. - -### OPERATORS -eq, ne (case-sensitive equality); gt, gte, lt, lte (comparison); in, nin (array membership); contains, ncontains, startsWith, endsWith (case-insensitive text); isEmpty, isNotEmpty (no value). - -### EXAMPLES -"status is archived" → {"all": [{"field": "status", "op": "eq", "value": "archived"}]} -"age under 18 or status banned" → {"any": [{"field": "age", "op": "lt", "value": 18}, {"field": "status", "op": "eq", "value": "banned"}]} - -Return ONLY the predicate JSON:`, - generationType: 'table-schema', + value: ['query_rows', 'update_rows_by_filter', 'delete_rows_by_filter'], + and: { field: 'filterMode', value: 'builder' }, }, }, - // Query rows — v2 predicate grammar - { - id: 'builderMode', - title: 'Input Mode', - type: 'dropdown', - options: [ - { label: 'Builder', id: 'builder' }, - { label: 'Editor', id: 'json' }, - ], - value: () => 'builder', - condition: { field: 'operation', value: 'query_rows' }, - }, - { - id: 'filterBuilder', - title: 'Filter Conditions', - type: 'filter-builder', - condition: { - field: 'operation', - value: 'query_rows', - and: { field: 'builderMode', value: 'builder' }, - }, - }, + // Visual sort builder (builder mode) — query only { id: 'sortBuilder', title: 'Sort Order', type: 'sort-builder', + paramVisibility: 'user-only', condition: { field: 'operation', value: 'query_rows', - and: { field: 'builderMode', value: 'builder' }, + and: { field: 'filterMode', value: 'builder' }, }, }, + + // Filter — PostgREST querystring (editor mode), query + bulk update/delete { - id: 'predicate', - title: 'Predicate', - type: 'code', - placeholder: '{"all": [{"field": "status", "op": "eq", "value": "active"}]}', + id: 'filter', + title: 'Filter', + type: 'long-input', + rows: 2, + placeholder: 'wins=gte.10&status=in.(active,pending)', + required: { + field: 'operation', + value: ['update_rows_by_filter', 'delete_rows_by_filter'], + and: { field: 'filterMode', value: 'editor' }, + }, condition: { field: 'operation', - value: 'query_rows', - and: { field: 'builderMode', value: 'builder', not: true }, + value: ['query_rows', 'update_rows_by_filter', 'delete_rows_by_filter'], + and: { field: 'filterMode', value: 'editor' }, }, wandConfig: { enabled: true, maintainHistory: true, - prompt: `Generate a predicate for selecting rows in a table. + prompt: `Generate a PostgREST filter querystring for selecting rows. ### CONTEXT {context} ### INSTRUCTION -Return ONLY a valid JSON predicate. No explanations or markdown. +Return ONLY the querystring. No explanations, surrounding quotes, or markdown. -A predicate is a nestable tree: { "all": [...] } means AND, { "any": [...] } means OR, and each leaf is { "field", "op", "value" }. +Top-level params join with & (AND). Use or=(a.op.v,b.op.v) / and=(...) for OR or nested logic. ### OPERATORS -- eq / ne: equals / not-equals (case-sensitive) -- gt / gte / lt / lte: numeric or date comparison -- in / nin: value in / not in an array -- contains / ncontains / startsWith / endsWith: case-insensitive text match -- isEmpty / isNotEmpty: cell is null/empty / present (no value) +eq, neq, gt, gte, lt, lte; in.(a,b); like / ilike (use * as the wildcard); match / imatch (regex); is.null. Negate with not. (e.g. not.in.(...), not.is.null). ### EXAMPLES +"status is active" → status=eq.active +"wins at least 10 and active" → wins=gte.10&active=is.true +"status active or pending" → or=(status.eq.active,status.eq.pending) +"name contains jo (any case)" → name=ilike.*jo* -"rows where status is active" -→ {"all": [{"field": "status", "op": "eq", "value": "active"}]} - -"age over 18 and status pending" -→ {"all": [{"field": "age", "op": "gte", "value": 18}, {"field": "status", "op": "eq", "value": "pending"}]} - -"status active or pending" -→ {"any": [{"field": "status", "op": "eq", "value": "active"}, {"field": "status", "op": "eq", "value": "pending"}]} - -Return ONLY the predicate JSON:`, +Return ONLY the querystring:`, generationType: 'table-schema', }, }, + // Order — PostgREST order (editor mode), query only { - id: 'sort', - title: 'Sort', - type: 'code', - placeholder: '[{"field": "createdAt", "direction": "desc"}]', + id: 'order', + title: 'Order', + type: 'short-input', + placeholder: 'wins.desc,name.asc', condition: { field: 'operation', value: 'query_rows', - and: { field: 'builderMode', value: 'builder', not: true }, - }, - wandConfig: { - enabled: true, - maintainHistory: true, - prompt: `Generate sort order for table query results as a JSON array. - -### CONTEXT -{context} - -### INSTRUCTION -Return ONLY a valid JSON array of { "field", "direction": "asc" | "desc" }. No explanations or markdown. - -### EXAMPLES - -"newest first" → [{"field": "createdAt", "direction": "desc"}] -"age descending, then name ascending" → [{"field": "age", "direction": "desc"}, {"field": "name", "direction": "asc"}] - -Return ONLY the sort JSON:`, - generationType: 'table-schema', + and: { field: 'filterMode', value: 'editor' }, }, }, { id: 'limit', title: 'Limit', type: 'short-input', - placeholder: '100', + placeholder: 'Leave empty for all rows', condition: { field: 'operation', value: ['query_rows', 'update_rows_by_filter', 'delete_rows_by_filter'], @@ -564,33 +484,20 @@ Return ONLY the sort JSON:`, data: { type: 'json', description: 'Row data for insert/update' }, rows: { type: 'array', description: 'Array of row data for batch insert' }, rowId: { type: 'string', description: 'Row identifier for ID-based operations' }, - bulkFilterMode: { + filterMode: { type: 'string', description: 'Filter input mode: builder or editor' }, + filterBuilder: { type: 'json', description: 'Visual filter builder conditions' }, + sortBuilder: { type: 'json', description: 'Visual sort builder conditions' }, + filter: { type: 'string', - description: 'Filter input mode for bulk operations (builder or json)', - }, - bulkFilterBuilder: { - type: 'json', - description: 'Visual filter builder conditions for bulk operations', - }, - bulkPredicate: { - type: 'json', description: - 'Predicate selecting rows for bulk update/delete: nestable { all | any: [...] } of { field, op, value } leaves.', + 'PostgREST filter querystring, e.g. wins=gte.10&status=in.(active,pending). Used by query and bulk update/delete.', }, - predicate: { - type: 'json', - description: - 'Query predicate: nestable { all | any: [...] } of { field, op, value } leaves. Operators: eq, ne, gt, gte, lt, lte, in, nin, contains, ncontains, startsWith, endsWith, isEmpty, isNotEmpty.', + order: { type: 'string', description: 'PostgREST order, e.g. wins.desc,name.asc' }, + limit: { + type: 'number', + description: 'Row limit; leave empty to return all matching rows (10MB byte cap)', }, - limit: { type: 'number', description: 'Query or bulk operation limit' }, cursor: { type: 'string', description: 'Opaque pagination cursor from a prior query response' }, - builderMode: { - type: 'string', - description: 'Input mode for filter and sort (builder or json)', - }, - filterBuilder: { type: 'json', description: 'Visual filter builder conditions' }, - sortBuilder: { type: 'json', description: 'Visual sort builder conditions' }, - sort: { type: 'json', description: 'Sort order as [{ field, direction }]' }, conflictColumn: { type: 'string', description: diff --git a/apps/sim/lib/api/contracts/tables-predicate.test.ts b/apps/sim/lib/api/contracts/tables-predicate.test.ts index 2b48a4c5023..1d56d50d00d 100644 --- a/apps/sim/lib/api/contracts/tables-predicate.test.ts +++ b/apps/sim/lib/api/contracts/tables-predicate.test.ts @@ -1,74 +1,77 @@ /** * @vitest-environment node + * + * The v2 query/bulk filter wire format is now a PostgREST string (parsed + + * validated server-side by `parsePostgrestFilter`). The contract only bounds the + * string; grammar validation lives in the parser tests (`postgrest.test.ts`). */ import { describe, expect, it } from 'vitest' import { + deleteTableRowsBodySchema, queryTableRowsV2BodySchema, - sortSpecSchema, - tablePredicateSchema, + updateRowsByFilterBodySchema, } from '@/lib/api/contracts/tables' -describe('tablePredicateSchema', () => { - it('accepts a nested all/any tree', () => { - const parsed = tablePredicateSchema.safeParse({ - all: [ - { field: 'slack_user_id', op: 'in', value: ['U1', 'U2'] }, - { - any: [ - { field: 's', op: 'eq', value: 'a' }, - { field: 's', op: 'eq', value: 'b' }, - ], - }, - ], +describe('queryTableRowsV2BodySchema', () => { + it('accepts a PostgREST filter/order string, leaves limit unbounded, has no offset', () => { + const parsed = queryTableRowsV2BodySchema.parse({ + workspaceId: 'ws-1', + filter: 'wins=gte.10&status=in.(active,pending)', + order: 'wins.desc', + cursor: 'abc', }) - expect(parsed.success).toBe(true) + expect(parsed.filter).toBe('wins=gte.10&status=in.(active,pending)') + // Omitted limit stays undefined — the query returns all matching rows. + expect(parsed.limit).toBeUndefined() + expect('offset' in parsed).toBe(false) }) - it('accepts a valueless op without a value', () => { - expect(tablePredicateSchema.safeParse({ all: [{ field: 'n', op: 'isEmpty' }] }).success).toBe( - true - ) + it('allows omitting the filter (match all)', () => { + expect(queryTableRowsV2BodySchema.safeParse({ workspaceId: 'ws-1' }).success).toBe(true) }) - it('rejects an unknown operator', () => { + it('rejects an empty filter string and an over-long one', () => { + expect(queryTableRowsV2BodySchema.safeParse({ workspaceId: 'ws-1', filter: '' }).success).toBe( + false + ) expect( - tablePredicateSchema.safeParse({ all: [{ field: 'n', op: 'regex', value: 'x' }] }).success + queryTableRowsV2BodySchema.safeParse({ workspaceId: 'ws-1', filter: 'x'.repeat(5000) }) + .success ).toBe(false) }) - it('rejects an empty group', () => { - expect(tablePredicateSchema.safeParse({ all: [] }).success).toBe(false) - }) - - it('rejects a node that is neither a leaf nor a group', () => { - expect(tablePredicateSchema.safeParse({ all: [{ foo: 'bar' }] }).success).toBe(false) + it('accepts a large explicit limit (no row cap) but rejects limit < 1', () => { + expect( + queryTableRowsV2BodySchema.safeParse({ workspaceId: 'ws-1', limit: 100000 }).success + ).toBe(true) + expect(queryTableRowsV2BodySchema.safeParse({ workspaceId: 'ws-1', limit: 0 }).success).toBe( + false + ) }) }) -describe('queryTableRowsV2BodySchema', () => { - it('defaults limit and accepts predicate + cursor (no offset)', () => { - const parsed = queryTableRowsV2BodySchema.parse({ - workspaceId: 'ws-1', - predicate: { all: [{ field: 'wins', op: 'gte', value: 10 }] }, - cursor: 'abc', - }) - expect(parsed.limit).toBeGreaterThan(0) - expect('offset' in parsed).toBe(false) - }) - - it('rejects limit over the max', () => { +describe('bulk schemas accept either a PostgREST string or the legacy filter object', () => { + it('delete accepts a PostgREST string filter', () => { expect( - queryTableRowsV2BodySchema.safeParse({ workspaceId: 'ws-1', limit: 100000 }).success - ).toBe(false) + deleteTableRowsBodySchema.safeParse({ workspaceId: 'ws-1', filter: 'status=eq.archived' }) + .success + ).toBe(true) }) -}) -describe('sortSpecSchema', () => { - it('accepts an ordered field/direction list', () => { - expect(sortSpecSchema.safeParse([{ field: 'wins', direction: 'desc' }]).success).toBe(true) + it('delete still accepts the legacy object filter (v1 callers)', () => { + expect( + deleteTableRowsBodySchema.safeParse({ workspaceId: 'ws-1', filter: { status: 'archived' } }) + .success + ).toBe(true) }) - it('rejects a bad direction', () => { - expect(sortSpecSchema.safeParse([{ field: 'wins', direction: 'sideways' }]).success).toBe(false) + it('update accepts a PostgREST string filter', () => { + expect( + updateRowsByFilterBodySchema.safeParse({ + workspaceId: 'ws-1', + filter: 'wins=gte.10', + data: { active: false }, + }).success + ).toBe(true) }) }) diff --git a/apps/sim/lib/api/contracts/tables.ts b/apps/sim/lib/api/contracts/tables.ts index 7bffe667b12..65a1d218617 100644 --- a/apps/sim/lib/api/contracts/tables.ts +++ b/apps/sim/lib/api/contracts/tables.ts @@ -5,12 +5,10 @@ import type { CsvHeaderMapping, EnrichmentRunDetail, Filter, - PredicateNode, RowData, Sort, TableDefinition, TableMetadata, - TablePredicate, TableRow, TableRowsCursor, } from '@/lib/table' @@ -262,76 +260,29 @@ const nonEmptyFilterSchema = domainObjectSchema().refine( const filterSchema = domainObjectSchema() -/* --------------------------- v2 predicate grammar --------------------------- */ - -const MAX_PREDICATE_MEMBERS = 100 -const MAX_SORT_FIELDS = 32 -const MAX_FIELD_LENGTH = 128 - -/** v2 bare-operator set — the legible grammar mothership and clients author. */ -export const filterOpSchema = z.enum([ - 'eq', - 'ne', - 'gt', - 'gte', - 'lt', - 'lte', - 'in', - 'nin', - 'contains', - 'ncontains', - 'startsWith', - 'endsWith', - 'isEmpty', - 'isNotEmpty', -]) +/* --------------------------- v2 PostgREST grammar --------------------------- */ -const predicateValueSchema: z.ZodType = z.lazy(() => - z.union([z.string(), z.number(), z.boolean(), z.null(), z.array(predicateValueSchema)]) -) +const MAX_FILTER_LENGTH = 4096 -const predicateLeafSchema = z.object({ - field: z.string().min(1, 'predicate field is required').max(MAX_FIELD_LENGTH), - op: filterOpSchema, - value: predicateValueSchema.optional(), -}) +/** + * v2 filter wire format: a PostgREST querystring fragment (e.g. + * `wins=gte.10&status=in.(active,pending)`), parsed + validated server-side by + * `parsePostgrestFilter`. The boundary only bounds length; the parser is the + * semantic validator. + */ +const postgrestFilterSchema = z + .string() + .min(1, 'filter must be a non-empty PostgREST querystring') + .max(MAX_FILTER_LENGTH, `filter cannot exceed ${MAX_FILTER_LENGTH} characters`) + +/** v2 order wire format: PostgREST `order` (e.g. `wins.desc,name.asc`). */ +const postgrestOrderSchema = z.string().min(1).max(512) /** - * Recursive `all`/`any` filter tree. A group is `{ all: [...] }` (AND) or - * `{ any: [...] }` (OR); members are leaf predicates or nested groups. A single - * cast pins the output to the canonical `TablePredicate` (zod's structural output - * is assignable but invariant under `z.ZodType`). + * Bulk update/delete filter accepts either the v2 PostgREST string (v2 block / + * agent) or the legacy `$`-operator object (v1 callers / stored workflows). */ -export const tablePredicateSchema = z.lazy(() => - z.union([ - z.object({ - all: z - .array(predicateNodeSchema) - .min(1, 'all group cannot be empty') - .max(MAX_PREDICATE_MEMBERS), - }), - z.object({ - any: z - .array(predicateNodeSchema) - .min(1, 'any group cannot be empty') - .max(MAX_PREDICATE_MEMBERS), - }), - ]) -) as z.ZodType - -const predicateNodeSchema = z.lazy(() => - z.union([predicateLeafSchema, tablePredicateSchema]) -) as z.ZodType - -/** v2 sort: an ordered list of `{ field, direction }`. */ -export const sortSpecSchema = z - .array( - z.object({ - field: z.string().min(1, 'sort field is required').max(MAX_FIELD_LENGTH), - direction: z.enum(['asc', 'desc']), - }) - ) - .max(MAX_SORT_FIELDS) +const bulkFilterSchema = z.union([postgrestFilterSchema, nonEmptyFilterSchema]) const optionalPositiveLimit = (max: number, label: string) => z.preprocess( @@ -351,7 +302,7 @@ export const deleteTableRowBodySchema = z.object({ export const deleteTableRowsBodySchema = z .object({ workspaceId: z.string().min(1, 'Workspace ID is required'), - filter: nonEmptyFilterSchema.optional(), + filter: bulkFilterSchema.optional(), limit: optionalPositiveLimit(TABLE_LIMITS.MAX_BULK_OPERATION_SIZE, 'Limit').optional(), rowIds: z .array(z.string().min(1)) @@ -416,7 +367,7 @@ export const tableRowsQuerySchema = tableRowsQueryBaseSchema.refine( export const updateRowsByFilterBodySchema = z.object({ workspaceId: z.string().min(1, 'Workspace ID is required'), - filter: nonEmptyFilterSchema, + filter: bulkFilterSchema, data: rowDataSchema, limit: optionalPositiveLimit(TABLE_LIMITS.MAX_BULK_OPERATION_SIZE, 'Limit').optional(), }) @@ -606,24 +557,23 @@ export const listTableRowsContract = defineRouteContract({ /** * v2 query surface: structured predicate grammar + opaque cursor pagination. * No `offset` (the cursor encodes paging state) and no after/sort refine (the - * cursor carries the sort context). POST because the predicate is a JSON tree. + * cursor carries the sort context). `filter`/`order` are PostgREST strings, + * parsed server-side. POST so the (potentially long) filter rides the body. */ export const queryTableRowsV2BodySchema = z.object({ workspaceId: z.string().min(1, 'Workspace ID is required'), - predicate: tablePredicateSchema.optional(), - sort: sortSpecSchema.optional(), - limit: z - .preprocess( - (value) => - value === null || value === undefined || value === '' ? undefined : Number(value), - z - .number({ error: 'Limit must be a number' }) - .int('Limit must be an integer') - .min(1, 'Limit must be at least 1') - .max(TABLE_LIMITS.MAX_QUERY_LIMIT, `Limit cannot exceed ${TABLE_LIMITS.MAX_QUERY_LIMIT}`) - .optional() - ) - .default(TABLE_LIMITS.DEFAULT_QUERY_LIMIT), + filter: postgrestFilterSchema.optional(), + order: postgrestOrderSchema.optional(), + // Omitted limit returns every matching row (bounded by the server's byte + // guard, not a row cap). An explicit limit is honored as-is for paging. + limit: z.preprocess( + (value) => (value === null || value === undefined || value === '' ? undefined : Number(value)), + z + .number({ error: 'Limit must be a number' }) + .int('Limit must be an integer') + .min(1, 'Limit must be at least 1') + .optional() + ), cursor: z.string().min(1, 'cursor must be a non-empty token').optional(), }) diff --git a/apps/sim/lib/copilot/generated/tool-schemas-v1.ts b/apps/sim/lib/copilot/generated/tool-schemas-v1.ts index f42c995c2b8..987646a5c2a 100644 --- a/apps/sim/lib/copilot/generated/tool-schemas-v1.ts +++ b/apps/sim/lib/copilot/generated/tool-schemas-v1.ts @@ -10,7 +10,7 @@ export interface ToolRuntimeSchemaEntry { } export const TOOL_RUNTIME_SCHEMAS: Record = { - ['agent']: { + agent: { parameters: { properties: { request: { @@ -23,7 +23,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - ['auth']: { + auth: { parameters: { properties: { request: { @@ -36,7 +36,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - ['check_deployment_status']: { + check_deployment_status: { parameters: { type: 'object', properties: { @@ -48,7 +48,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - ['complete_scheduled_task']: { + complete_scheduled_task: { parameters: { type: 'object', properties: { @@ -61,7 +61,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - ['crawl_website']: { + crawl_website: { parameters: { type: 'object', properties: { @@ -96,7 +96,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - ['create_file']: { + create_file: { parameters: { type: 'object', properties: { @@ -162,7 +162,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { required: ['success', 'message'], }, }, - ['create_file_folder']: { + create_file_folder: { parameters: { type: 'object', properties: { @@ -180,7 +180,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - ['create_workflow']: { + create_workflow: { parameters: { type: 'object', properties: { @@ -205,7 +205,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - ['create_workspace_mcp_server']: { + create_workspace_mcp_server: { parameters: { type: 'object', properties: { @@ -238,7 +238,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - ['delete_file']: { + delete_file: { parameters: { type: 'object', properties: { @@ -268,7 +268,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { required: ['success', 'message'], }, }, - ['delete_file_folder']: { + delete_file_folder: { parameters: { type: 'object', properties: { @@ -284,7 +284,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - ['delete_workflow']: { + delete_workflow: { parameters: { type: 'object', properties: { @@ -300,7 +300,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - ['delete_workspace_mcp_server']: { + delete_workspace_mcp_server: { parameters: { type: 'object', properties: { @@ -313,7 +313,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - ['deploy']: { + deploy: { parameters: { properties: { request: { @@ -327,7 +327,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - ['deploy_api']: { + deploy_api: { parameters: { type: 'object', properties: { @@ -411,7 +411,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { ], }, }, - ['deploy_chat']: { + deploy_chat: { parameters: { type: 'object', properties: { @@ -570,7 +570,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { ], }, }, - ['deploy_mcp']: { + deploy_mcp: { parameters: { type: 'object', properties: { @@ -686,7 +686,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { required: ['deploymentType', 'deploymentStatus'], }, }, - ['diff_workflows']: { + diff_workflows: { parameters: { type: 'object', properties: { @@ -710,7 +710,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - ['download_to_workspace_file']: { + download_to_workspace_file: { parameters: { type: 'object', properties: { @@ -759,7 +759,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - ['edit_content']: { + edit_content: { parameters: { type: 'object', properties: { @@ -791,7 +791,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { required: ['success', 'message'], }, }, - ['edit_workflow']: { + edit_workflow: { parameters: { type: 'object', properties: { @@ -830,7 +830,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - ['enrichment_run']: { + enrichment_run: { parameters: { type: 'object', properties: { @@ -874,7 +874,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { required: ['matched', 'result'], }, }, - ['ffmpeg']: { + ffmpeg: { parameters: { type: 'object', properties: { @@ -1055,7 +1055,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - ['file']: { + file: { parameters: { properties: { prompt: { @@ -1068,7 +1068,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - ['function_execute']: { + function_execute: { parameters: { type: 'object', properties: { @@ -1206,7 +1206,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - ['generate_api_key']: { + generate_api_key: { parameters: { type: 'object', properties: { @@ -1224,7 +1224,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - ['generate_audio']: { + generate_audio: { parameters: { type: 'object', properties: { @@ -1376,7 +1376,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - ['generate_image']: { + generate_image: { parameters: { type: 'object', properties: { @@ -1504,7 +1504,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - ['generate_video']: { + generate_video: { parameters: { type: 'object', properties: { @@ -1671,7 +1671,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - ['get_block_outputs']: { + get_block_outputs: { parameters: { type: 'object', properties: { @@ -1692,7 +1692,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - ['get_block_upstream_references']: { + get_block_upstream_references: { parameters: { type: 'object', properties: { @@ -1714,7 +1714,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - ['get_deployed_workflow_state']: { + get_deployed_workflow_state: { parameters: { type: 'object', properties: { @@ -1727,7 +1727,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - ['get_deployment_log']: { + get_deployment_log: { parameters: { type: 'object', properties: { @@ -1740,7 +1740,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - ['get_page_contents']: { + get_page_contents: { parameters: { type: 'object', properties: { @@ -1768,14 +1768,14 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - ['get_platform_actions']: { + get_platform_actions: { parameters: { type: 'object', properties: {}, }, resultSchema: undefined, }, - ['get_scheduled_task_logs']: { + get_scheduled_task_logs: { parameters: { type: 'object', properties: { @@ -1800,7 +1800,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - ['get_workflow_data']: { + get_workflow_data: { parameters: { type: 'object', properties: { @@ -1819,7 +1819,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - ['get_workflow_run_options']: { + get_workflow_run_options: { parameters: { type: 'object', properties: { @@ -1832,7 +1832,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - ['glob']: { + glob: { parameters: { type: 'object', properties: { @@ -1851,7 +1851,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - ['grep']: { + grep: { parameters: { type: 'object', properties: { @@ -1899,7 +1899,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - ['knowledge']: { + knowledge: { parameters: { properties: { request: { @@ -1912,7 +1912,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - ['knowledge_base']: { + knowledge_base: { parameters: { type: 'object', properties: { @@ -2105,7 +2105,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { required: ['success', 'message'], }, }, - ['list_file_folders']: { + list_file_folders: { parameters: { type: 'object', properties: { @@ -2117,7 +2117,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - ['list_integration_tools']: { + list_integration_tools: { parameters: { properties: { integration: { @@ -2131,14 +2131,14 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - ['list_user_workspaces']: { + list_user_workspaces: { parameters: { type: 'object', properties: {}, }, resultSchema: undefined, }, - ['list_workspace_mcp_servers']: { + list_workspace_mcp_servers: { parameters: { type: 'object', properties: { @@ -2151,7 +2151,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - ['load_deployment']: { + load_deployment: { parameters: { type: 'object', properties: { @@ -2170,7 +2170,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - ['load_integration_tool']: { + load_integration_tool: { parameters: { properties: { tool_ids: { @@ -2187,7 +2187,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - ['manage_credential']: { + manage_credential: { parameters: { type: 'object', properties: { @@ -2216,7 +2216,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - ['manage_custom_tool']: { + manage_custom_tool: { parameters: { type: 'object', properties: { @@ -2296,7 +2296,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - ['manage_folder']: { + manage_folder: { parameters: { type: 'object', properties: { @@ -2335,7 +2335,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - ['manage_mcp_tool']: { + manage_mcp_tool: { parameters: { type: 'object', properties: { @@ -2387,7 +2387,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - ['manage_scheduled_task']: { + manage_scheduled_task: { parameters: { type: 'object', properties: { @@ -2462,7 +2462,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - ['manage_skill']: { + manage_skill: { parameters: { type: 'object', properties: { @@ -2495,7 +2495,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - ['materialize_file']: { + materialize_file: { parameters: { type: 'object', properties: { @@ -2519,7 +2519,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - ['media']: { + media: { parameters: { properties: { prompt: { @@ -2532,7 +2532,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - ['move_file']: { + move_file: { parameters: { type: 'object', properties: { @@ -2553,7 +2553,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - ['move_file_folder']: { + move_file_folder: { parameters: { type: 'object', properties: { @@ -2571,7 +2571,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - ['move_workflow']: { + move_workflow: { parameters: { type: 'object', properties: { @@ -2591,7 +2591,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - ['oauth_get_auth_link']: { + oauth_get_auth_link: { parameters: { type: 'object', properties: { @@ -2605,7 +2605,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - ['oauth_request_access']: { + oauth_request_access: { parameters: { type: 'object', properties: { @@ -2619,7 +2619,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - ['open_resource']: { + open_resource: { parameters: { type: 'object', properties: { @@ -2653,7 +2653,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - ['promote_to_live']: { + promote_to_live: { parameters: { type: 'object', properties: { @@ -2672,7 +2672,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - ['query_logs']: { + query_logs: { parameters: { type: 'object', properties: { @@ -2783,7 +2783,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - ['read']: { + read: { parameters: { type: 'object', properties: { @@ -2810,7 +2810,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - ['redeploy']: { + redeploy: { parameters: { type: 'object', properties: { @@ -2889,7 +2889,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { ], }, }, - ['rename_file']: { + rename_file: { parameters: { type: 'object', properties: { @@ -2925,7 +2925,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { required: ['success', 'message'], }, }, - ['rename_file_folder']: { + rename_file_folder: { parameters: { type: 'object', properties: { @@ -2942,7 +2942,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - ['rename_workflow']: { + rename_workflow: { parameters: { type: 'object', properties: { @@ -2959,7 +2959,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - ['research']: { + research: { parameters: { properties: { topic: { @@ -2972,7 +2972,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - ['respond']: { + respond: { parameters: { additionalProperties: true, properties: { @@ -2995,7 +2995,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - ['restore_resource']: { + restore_resource: { parameters: { type: 'object', properties: { @@ -3013,7 +3013,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - ['run']: { + run: { parameters: { properties: { context: { @@ -3030,7 +3030,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - ['run_block']: { + run_block: { parameters: { type: 'object', properties: { @@ -3062,7 +3062,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - ['run_from_block']: { + run_from_block: { parameters: { type: 'object', properties: { @@ -3094,7 +3094,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - ['run_workflow']: { + run_workflow: { parameters: { type: 'object', properties: { @@ -3132,7 +3132,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - ['run_workflow_until_block']: { + run_workflow_until_block: { parameters: { type: 'object', properties: { @@ -3175,7 +3175,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - ['scheduled_task']: { + scheduled_task: { parameters: { properties: { request: { @@ -3188,7 +3188,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - ['scrape_page']: { + scrape_page: { parameters: { type: 'object', properties: { @@ -3209,7 +3209,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - ['search_documentation']: { + search_documentation: { parameters: { type: 'object', properties: { @@ -3226,7 +3226,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - ['search_library_docs']: { + search_library_docs: { parameters: { type: 'object', properties: { @@ -3247,7 +3247,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - ['search_online']: { + search_online: { parameters: { type: 'object', properties: { @@ -3287,7 +3287,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - ['search_patterns']: { + search_patterns: { parameters: { type: 'object', properties: { @@ -3309,7 +3309,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - ['set_block_enabled']: { + set_block_enabled: { parameters: { type: 'object', properties: { @@ -3331,7 +3331,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - ['set_environment_variables']: { + set_environment_variables: { parameters: { type: 'object', properties: { @@ -3365,7 +3365,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - ['set_global_workflow_variables']: { + set_global_workflow_variables: { parameters: { type: 'object', properties: { @@ -3406,7 +3406,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - ['superagent']: { + superagent: { parameters: { properties: { task: { @@ -3420,7 +3420,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - ['table']: { + table: { parameters: { properties: { request: { @@ -3433,7 +3433,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - ['update_deployment_version']: { + update_deployment_version: { parameters: { type: 'object', properties: { @@ -3462,7 +3462,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - ['update_scheduled_task_history']: { + update_scheduled_task_history: { parameters: { type: 'object', properties: { @@ -3480,7 +3480,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - ['update_workspace_mcp_server']: { + update_workspace_mcp_server: { parameters: { type: 'object', properties: { @@ -3505,7 +3505,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - ['user_memory']: { + user_memory: { parameters: { type: 'object', properties: { @@ -3554,7 +3554,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - ['user_table']: { + user_table: { parameters: { type: 'object', properties: { @@ -3917,7 +3917,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { required: ['success', 'message'], }, }, - ['workflow']: { + workflow: { parameters: { properties: { prompt: { @@ -3930,7 +3930,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - ['workspace_file']: { + workspace_file: { parameters: { type: 'object', properties: { diff --git a/apps/sim/lib/copilot/tools/handlers/function-execute.ts b/apps/sim/lib/copilot/tools/handlers/function-execute.ts index bc32699d66a..9a25a516569 100644 --- a/apps/sim/lib/copilot/tools/handlers/function-execute.ts +++ b/apps/sim/lib/copilot/tools/handlers/function-execute.ts @@ -4,6 +4,7 @@ import { resolveWorkflowAliasForWorkspace } from '@/lib/copilot/vfs/workflow-ali import { isPlanAliasPath, workflowAliasSandboxPath } from '@/lib/copilot/vfs/workflow-aliases' import { isFeatureEnabled } from '@/lib/core/config/feature-flags' import { getColumnId } from '@/lib/table/column-keys' +import { TABLE_LIMITS } from '@/lib/table/constants' import { formatCsvValue, neutralizeCsvFormula, toCsvRow } from '@/lib/table/export-format' import { queryRows } from '@/lib/table/rows/service' import { getTableById, listTables } from '@/lib/table/service' @@ -388,7 +389,13 @@ export async function resolveInputFiles( continue } - const rows = await queryRows(table, {}, 'copilot-fn-exec') + // Keep the prior bounded mount — draining the whole table here was backed + // out for OOM, so don't ride the new unbounded queryRows default. + const rows = await queryRows( + table, + { limit: TABLE_LIMITS.DEFAULT_QUERY_LIMIT }, + 'copilot-fn-exec' + ) const columns = table.schema.columns const csvLines = [toCsvRow(columns.map((column) => neutralizeCsvFormula(column.name)))] diff --git a/apps/sim/lib/copilot/tools/server/table/user-table.test.ts b/apps/sim/lib/copilot/tools/server/table/user-table.test.ts index 0d6925cbc79..2edef95820b 100644 --- a/apps/sim/lib/copilot/tools/server/table/user-table.test.ts +++ b/apps/sim/lib/copilot/tools/server/table/user-table.test.ts @@ -690,7 +690,7 @@ describe('userTableServerTool.query_rows', () => { }) }) - it('clamps an over-large query limit to MAX_QUERY_LIMIT instead of rejecting', async () => { + it('passes an explicit limit through unchanged (no row cap)', async () => { const result = await userTableServerTool.execute( { operation: 'query_rows', args: { tableId: 'tbl_1', limit: 100000 } }, { userId: 'user-1', workspaceId: 'workspace-1' } @@ -698,7 +698,18 @@ describe('userTableServerTool.query_rows', () => { expect(result.success).toBe(true) const options = mockQueryRows.mock.calls[0][1] as Record - expect(options.limit).toBe(1000) + expect(options.limit).toBe(100000) + }) + + it('omits the limit so queryRows returns every matching row', async () => { + const result = await userTableServerTool.execute( + { operation: 'query_rows', args: { tableId: 'tbl_1' } }, + { userId: 'user-1', workspaceId: 'workspace-1' } + ) + + expect(result.success).toBe(true) + const options = mockQueryRows.mock.calls[0][1] as Record + expect(options.limit).toBeUndefined() }) it('queries without execution metadata and passes limit/offset through', async () => { @@ -744,7 +755,7 @@ describe('userTableServerTool.delete_rows_by_filter', () => { operation: 'delete_rows_by_filter', args: { tableId: 'tbl_1', - filter: { all: [{ field: 'name', op: 'eq', value: 'x' }] }, + filter: 'name=eq.x', limit: 5000, }, }, @@ -769,7 +780,7 @@ describe('userTableServerTool.delete_rows_by_filter', () => { const result = await userTableServerTool.execute( { operation: 'delete_rows_by_filter', - args: { tableId: 'tbl_1', filter: { all: [{ field: 'name', op: 'eq', value: 'x' }] } }, + args: { tableId: 'tbl_1', filter: 'name=eq.x' }, }, { userId: 'user-1', workspaceId: 'workspace-1' } ) @@ -790,7 +801,7 @@ describe('userTableServerTool.delete_rows_by_filter', () => { operation: 'delete_rows_by_filter', args: { tableId: 'tbl_1', - filter: { all: [{ field: 'name', op: 'eq', value: 'x' }] }, + filter: 'name=eq.x', limit: 100, }, }, @@ -814,7 +825,7 @@ describe('userTableServerTool.delete_rows_by_filter', () => { const result = await userTableServerTool.execute( { operation: 'delete_rows_by_filter', - args: { tableId: 'tbl_1', filter: { all: [{ field: 'name', op: 'eq', value: 'x' }] } }, + args: { tableId: 'tbl_1', filter: 'name=eq.x' }, }, { userId: 'user-1', workspaceId: 'workspace-1' } ) @@ -852,7 +863,7 @@ describe('userTableServerTool.delete_rows_by_filter', () => { const result = await userTableServerTool.execute( { operation: 'delete_rows_by_filter', - args: { tableId: 'tbl_1', filter: { all: [{ field: 'name', op: 'eq', value: 'x' }] } }, + args: { tableId: 'tbl_1', filter: 'name=eq.x' }, }, { userId: 'user-1', workspaceId: 'workspace-1' } ) @@ -869,7 +880,7 @@ describe('userTableServerTool.delete_rows_by_filter', () => { operation: 'delete_rows_by_filter', args: { tableId: 'tbl_1', - filter: { all: [{ field: 'name', op: 'eq', value: 'x' }] }, + filter: 'name=eq.x', limit: 100, }, }, @@ -904,7 +915,7 @@ describe('userTableServerTool.update_rows_by_filter', () => { operation: 'update_rows_by_filter', args: { tableId: 'tbl_1', - filter: { all: [{ field: 'name', op: 'eq', value: 'x' }] }, + filter: 'name=eq.x', data: { age: 1 }, limit: 5000, }, @@ -929,7 +940,7 @@ describe('userTableServerTool.update_rows_by_filter', () => { operation: 'update_rows_by_filter', args: { tableId: 'tbl_1', - filter: { all: [{ field: 'name', op: 'eq', value: 'x' }] }, + filter: 'name=eq.x', data: { age: 1 }, }, }, @@ -954,7 +965,7 @@ describe('userTableServerTool.update_rows_by_filter', () => { operation: 'update_rows_by_filter', args: { tableId: 'tbl_1', - filter: { all: [{ field: 'name', op: 'eq', value: 'x' }] }, + filter: 'name=eq.x', data: { age: 1 }, }, }, @@ -994,7 +1005,7 @@ describe('userTableServerTool.update_rows_by_filter', () => { operation: 'update_rows_by_filter', args: { tableId: 'tbl_1', - filter: { all: [{ field: 'email', op: 'eq', value: 'x' }] }, + filter: 'email=eq.x', data: { email: 'y' }, }, }, @@ -1020,7 +1031,7 @@ describe('userTableServerTool.update_rows_by_filter', () => { operation: 'update_rows_by_filter', args: { tableId: 'tbl_1', - filter: { all: [{ field: 'name', op: 'eq', value: 'x' }] }, + filter: 'name=eq.x', data: { age: 1 }, }, }, @@ -1038,7 +1049,7 @@ describe('userTableServerTool.update_rows_by_filter', () => { operation: 'update_rows_by_filter', args: { tableId: 'tbl_1', - filter: { all: [{ field: 'name', op: 'eq', value: 'x' }] }, + filter: 'name=eq.x', data: { age: 1 }, limit: 100, }, diff --git a/apps/sim/lib/copilot/tools/server/table/user-table.ts b/apps/sim/lib/copilot/tools/server/table/user-table.ts index 435511fbd45..a85e687433a 100644 --- a/apps/sim/lib/copilot/tools/server/table/user-table.ts +++ b/apps/sim/lib/copilot/tools/server/table/user-table.ts @@ -30,7 +30,7 @@ import { predicateNamesToIds, rowDataIdToName, rowDataNameToId, - sortNamesToIds, + sortSpecNamesToIds, } from '@/lib/table/column-keys' import { columnTypeForLeaf, deriveOutputColumnName } from '@/lib/table/column-naming' import { @@ -45,6 +45,7 @@ import { markTableDeleteFailed, runTableDelete } from '@/lib/table/delete-runner import { runTableImport, type TableImportPayload } from '@/lib/table/import-runner' import { markTableJobRunning, releaseJobClaim } from '@/lib/table/jobs/service' import { predicateToFilter } from '@/lib/table/query-builder/converters' +import { parsePostgrestFilter, parsePostgrestOrder } from '@/lib/table/query-builder/postgrest' import { batchInsertRows, batchUpdateRows, @@ -603,18 +604,26 @@ export const userTableServerTool: BaseServerTool const requestId = generateId().slice(0, 8) const idByName = buildIdByName(table.schema) const nameById = buildNameById(table.schema) - // The model may request any number; we serve at most MAX_QUERY_LIMIT per page so a single - // tool result can't drain a whole table. `totalCount` in the response signals truncation, - // and the model pages with `offset`. + // PostgREST filter/order strings, parsed with the schema (column NAMES + // → coercion) then translated to storage ids. + const predicate = args.filter + ? predicateNamesToIds(parsePostgrestFilter(args.filter, table.schema.columns), idByName) + : undefined + const orderSpec = args.order + ? sortSpecNamesToIds(parsePostgrestOrder(args.order, table.schema.columns), idByName) + : undefined + const sort = orderSpec?.length + ? Object.fromEntries(orderSpec.map((s) => [s.field, s.direction])) + : undefined + // No limit returns every matching row (bounded by the server's 10MB byte + // guard, which fails fast rather than truncating). An explicit limit is + // honored as-is so the model can still page with `offset` when it wants to. const result = await queryRows( table, { - predicate: args.filter ? predicateNamesToIds(args.filter, idByName) : undefined, - sort: args.sort ? sortNamesToIds(args.sort, idByName) : undefined, - limit: - args.limit !== undefined - ? Math.min(args.limit, TABLE_LIMITS.MAX_QUERY_LIMIT) - : undefined, + predicate, + sort, + limit: args.limit, offset: args.offset, withExecutions: false, }, @@ -729,9 +738,11 @@ export const userTableServerTool: BaseServerTool const requestId = generateId().slice(0, 8) const idByName = buildIdByName(table.schema) - // Agent authors the v2 predicate grammar; convert to a Filter for the - // bulk engine (same fieldPredicate leaf → identical SQL). - const idFilter = predicateToFilter(predicateNamesToIds(args.filter, idByName)) + // Agent authors a PostgREST filter string; parse → predicate → Filter + // for the bulk engine (same fieldPredicate leaf → identical SQL). + const idFilter = predicateToFilter( + predicateNamesToIds(parsePostgrestFilter(args.filter, table.schema.columns), idByName) + ) const idData = rowDataNameToId(args.data, idByName) // Inline handles up to MAX_BULK_OPERATION_SIZE rows in one request; a larger operation @@ -826,9 +837,11 @@ export const userTableServerTool: BaseServerTool const requestId = generateId().slice(0, 8) const idByName = buildIdByName(table.schema) - // Agent authors the v2 predicate grammar; convert to a Filter for the - // bulk engine (same fieldPredicate leaf → identical SQL). - const idFilter = predicateToFilter(predicateNamesToIds(args.filter, idByName)) + // Agent authors a PostgREST filter string; parse → predicate → Filter + // for the bulk engine (same fieldPredicate leaf → identical SQL). + const idFilter = predicateToFilter( + predicateNamesToIds(parsePostgrestFilter(args.filter, table.schema.columns), idByName) + ) // Inline handles up to MAX_BULK_OPERATION_SIZE rows; a larger delete (an explicit limit // above the cap, or unbounded "delete everything matching") hands off to the background diff --git a/apps/sim/lib/table/__tests__/service-filter-threading.test.ts b/apps/sim/lib/table/__tests__/service-filter-threading.test.ts index c0ade663506..83281337dd4 100644 --- a/apps/sim/lib/table/__tests__/service-filter-threading.test.ts +++ b/apps/sim/lib/table/__tests__/service-filter-threading.test.ts @@ -10,6 +10,7 @@ import { dbChainMock, dbChainMockFns, resetDbChainMock } from '@sim/testing' import { sql } from 'drizzle-orm' import { beforeEach, describe, expect, it, vi } from 'vitest' +import { TABLE_LIMITS } from '@/lib/table/constants' import { buildFilterClause, buildSortClause } from '@/lib/table/sql' import type { ColumnDefinition, TableDefinition } from '@/lib/table/types' @@ -18,6 +19,8 @@ vi.mock('@sim/db', () => dbChainMock) vi.mock('@/lib/table/sql', () => ({ buildFilterClause: vi.fn(() => sql`true`), buildSortClause: vi.fn(() => sql`true`), + buildPredicateClause: vi.fn(() => sql`true`), + TableQueryValidationError: class TableQueryValidationError extends Error {}, })) vi.mock('@/lib/table/trigger', () => ({ @@ -123,3 +126,62 @@ describe('service filter threading', () => { ) }) }) + +describe('bulk update/delete limited-subset ordering', () => { + beforeEach(() => { + vi.clearAllMocks() + resetDbChainMock() + }) + + it('orders the match query when updateRowsByFilter has a limit', async () => { + await updateRowsByFilter( + TABLE, + { filter: { score: { $gt: 0 } }, data: { name: 'x' }, limit: 5 }, + 'req-1' + ) + expect(dbChainMockFns.orderBy).toHaveBeenCalled() + expect(dbChainMockFns.limit).toHaveBeenCalledWith(5) + }) + + it('does not order an unbounded updateRowsByFilter', async () => { + dbChainMockFns.where.mockResolvedValueOnce([]) + await updateRowsByFilter(TABLE, { filter: { score: { $gt: 0 } }, data: { name: 'x' } }, 'req-1') + expect(dbChainMockFns.orderBy).not.toHaveBeenCalled() + }) + + it('orders the match query when deleteRowsByFilter has a limit', async () => { + await deleteRowsByFilter(TABLE, { filter: { score: { $gt: 0 } }, limit: 3 }, 'req-1') + expect(dbChainMockFns.orderBy).toHaveBeenCalled() + expect(dbChainMockFns.limit).toHaveBeenCalledWith(3) + }) +}) + +describe('queryRows byte guard', () => { + beforeEach(() => { + vi.clearAllMocks() + resetDbChainMock() + }) + + it('returns an empty page without tripping the guard', async () => { + const result = await queryRows(TABLE, { includeTotal: false, withExecutions: false }, 'req-1') + expect(result.rows).toEqual([]) + }) + + it('fails fast when the result exceeds the byte budget', async () => { + const oversized = 'x'.repeat(TABLE_LIMITS.MAX_QUERY_RESULT_BYTES + 1) + dbChainMockFns.orderBy.mockResolvedValueOnce([ + { + id: 'row_1', + data: { blob: oversized }, + position: 0, + orderKey: 'a0', + createdAt: new Date('2024-01-01'), + updatedAt: new Date('2024-01-01'), + }, + ]) + + await expect( + queryRows(TABLE, { includeTotal: false, withExecutions: false }, 'req-1') + ).rejects.toThrow(/10MB/) + }) +}) diff --git a/apps/sim/lib/table/__tests__/sql.test.ts b/apps/sim/lib/table/__tests__/sql.test.ts index eb5ba91b19e..0b2062784a9 100644 --- a/apps/sim/lib/table/__tests__/sql.test.ts +++ b/apps/sim/lib/table/__tests__/sql.test.ts @@ -518,15 +518,43 @@ describe('fieldPredicate (shared leaf)', () => { expect(render(fieldPredicate(TABLE, 'name', 'startsWith', 'jo', undefined))).toContain('ILIKE') }) - it('isEmpty / isNotEmpty emit emptiness checks', () => { - expect(render(fieldPredicate(TABLE, 'name', 'isEmpty', undefined, undefined))).toContain( - 'IS NULL' - ) - expect(render(fieldPredicate(TABLE, 'name', 'isNotEmpty', undefined, undefined))).toContain( + it('isEmpty / isNotEmpty emit emptiness checks (null OR empty string)', () => { + const empty = render(fieldPredicate(TABLE, 'name', 'isEmpty', undefined, undefined)) + expect(empty).toContain('IS NULL') + expect(empty).toContain("= ''") + const notEmpty = render(fieldPredicate(TABLE, 'name', 'isNotEmpty', undefined, undefined)) + expect(notEmpty).toContain('IS NOT NULL') + }) + + it('isNull / isNotNull are strict null checks (no empty-string clause)', () => { + const isNull = render(fieldPredicate(TABLE, 'name', 'isNull', undefined, undefined)) + expect(isNull).toContain('IS NULL') + expect(isNull).not.toContain("= ''") + expect(render(fieldPredicate(TABLE, 'name', 'isNotNull', undefined, undefined))).toContain( 'IS NOT NULL' ) }) + it('like / ilike map * to % and escape literal % / _', () => { + const like = render(fieldPredicate(TABLE, 'name', 'like', 'jo*n', undefined)) + expect(like).toContain("data->>'name'") + expect(like).toContain('LIKE') + expect(like).not.toContain('ILIKE') + expect(like).toContain('jo%n') + expect(render(fieldPredicate(TABLE, 'name', 'ilike', '*foo*', undefined))).toContain('ILIKE') + // literal % is escaped to match itself, not act as a wildcard + expect(render(fieldPredicate(TABLE, 'name', 'like', '50%*', undefined))).toContain('50\\%%') + }) + + it('match / imatch emit POSIX regex (~ / ~*)', () => { + const m = render(fieldPredicate(TABLE, 'name', 'match', '^jo.*n$', undefined)) + expect(m).toContain("data->>'name'") + expect(m).toContain('~') + expect(m).not.toContain('~*') + expect(m).toContain('^jo.*n$') + expect(render(fieldPredicate(TABLE, 'name', 'imatch', 'foo', undefined))).toContain('~*') + }) + it('validates the field name', () => { expect(() => fieldPredicate(TABLE, "x'; DROP", 'eq', 1, undefined)).toThrow( 'Invalid field name' diff --git a/apps/sim/lib/table/constants.ts b/apps/sim/lib/table/constants.ts index 22a3bd9a595..549b2413abf 100644 --- a/apps/sim/lib/table/constants.ts +++ b/apps/sim/lib/table/constants.ts @@ -16,6 +16,13 @@ export const TABLE_LIMITS = { MAX_DESCRIPTION_LENGTH: 500, DEFAULT_QUERY_LIMIT: 100, MAX_QUERY_LIMIT: 1000, + /** + * Byte ceiling for a single v2 query result. The v2 query returns every + * matching row by default (no row cap); this guard fails the query fast + * instead of streaming an unbounded payload. Measured against the serialized + * row data, not the full HTTP envelope. + */ + MAX_QUERY_RESULT_BYTES: 10 * 1024 * 1024, // 10MB /** Batch size for bulk update operations */ UPDATE_BATCH_SIZE: 100, /** Batch size for bulk delete operations */ diff --git a/apps/sim/lib/table/query-builder/__tests__/converters.test.ts b/apps/sim/lib/table/query-builder/__tests__/converters.test.ts index d148db3ed8a..78b1380a109 100644 --- a/apps/sim/lib/table/query-builder/__tests__/converters.test.ts +++ b/apps/sim/lib/table/query-builder/__tests__/converters.test.ts @@ -8,10 +8,12 @@ import { describe, expect, it } from 'vitest' import { filterRulesToFilter, + filterRulesToPostgrest, filterRulesToPredicate, filterToRules, predicateToFilter, predicateToFilterRules, + sortRulesToPostgrestOrder, sortRulesToSortSpec, } from '@/lib/table/query-builder/converters' import type { FilterRule, SortRule, TablePredicate } from '@/lib/table/types' @@ -237,3 +239,56 @@ describe('sortRulesToSortSpec (v2)', () => { expect(sortRulesToSortSpec([{ id: '1', column: '', direction: 'asc' }])).toBeNull() }) }) + +describe('filterRulesToPostgrest', () => { + it('serializes builder rules to a PostgREST querystring', () => { + expect( + filterRulesToPostgrest([ + rule({ column: 'wins', operator: 'gte', value: '10' }), + rule({ column: 'status', operator: 'eq', value: 'active' }), + ]) + ).toBe('wins=gte.10&status=eq.active') + }) + + it('serializes an OR boundary to an or() group', () => { + expect( + filterRulesToPostgrest([ + rule({ column: 'status', operator: 'eq', value: 'active' }), + rule({ column: 'status', operator: 'eq', value: 'pending', logicalOperator: 'or' }), + ]) + ).toBe('or=(status.eq.active,status.eq.pending)') + }) + + it('maps builder-only ops onto PostgREST forms', () => { + expect( + filterRulesToPostgrest([rule({ column: 'name', operator: 'contains', value: 'jo' })]) + ).toBe('name=ilike.*jo*') + expect(filterRulesToPostgrest([rule({ column: 'name', operator: 'isEmpty' })])).toBe( + 'name=is.null' + ) + }) + + it('returns null for an empty builder', () => { + expect(filterRulesToPostgrest([])).toBeNull() + }) + + it('returns null (does not throw "rules is not iterable") for a non-array value', () => { + expect(filterRulesToPostgrest({} as unknown as FilterRule[])).toBeNull() + expect(filterRulesToPostgrest(undefined as unknown as FilterRule[])).toBeNull() + }) +}) + +describe('sortRulesToPostgrestOrder', () => { + it('serializes sort rules to a PostgREST order string', () => { + expect( + sortRulesToPostgrestOrder([ + { id: '1', column: 'wins', direction: 'desc' }, + { id: '2', column: 'name', direction: 'asc' }, + ]) + ).toBe('wins.desc,name.asc') + }) + + it('returns null when empty', () => { + expect(sortRulesToPostgrestOrder([])).toBeNull() + }) +}) diff --git a/apps/sim/lib/table/query-builder/__tests__/postgrest.test.ts b/apps/sim/lib/table/query-builder/__tests__/postgrest.test.ts new file mode 100644 index 00000000000..df414e91935 --- /dev/null +++ b/apps/sim/lib/table/query-builder/__tests__/postgrest.test.ts @@ -0,0 +1,197 @@ +/** + * @vitest-environment node + */ +import { describe, expect, it } from 'vitest' +import { + parsePostgrestFilter, + parsePostgrestOrder, + predicateToPostgrest, + sortSpecToPostgrestOrder, +} from '@/lib/table/query-builder/postgrest' +import type { ColumnDefinition, TablePredicate } from '@/lib/table/types' + +const COLS: ColumnDefinition[] = [ + { name: 'wins', type: 'number' }, + { name: 'status', type: 'string' }, + { name: 'active', type: 'boolean' }, + { name: 'name', type: 'string' }, + { name: 'slack_user_id', type: 'string' }, +] + +describe('parsePostgrestFilter', () => { + it('parses top-level params as an AND of leaves, coercing by column type', () => { + expect(parsePostgrestFilter('wins=gte.10&status=eq.active', COLS)).toEqual({ + all: [ + { field: 'wins', op: 'gte', value: 10 }, + { field: 'status', op: 'eq', value: 'active' }, + ], + }) + }) + + it('coerces booleans and numbers', () => { + expect(parsePostgrestFilter('active=eq.true&wins=eq.5', COLS)).toEqual({ + all: [ + { field: 'active', op: 'eq', value: true }, + { field: 'wins', op: 'eq', value: 5 }, + ], + }) + }) + + it('parses in.(...) lists with per-element coercion', () => { + expect(parsePostgrestFilter('slack_user_id=in.(U1,U2)', COLS)).toEqual({ + all: [{ field: 'slack_user_id', op: 'in', value: ['U1', 'U2'] }], + }) + expect(parsePostgrestFilter('wins=in.(1,2,3)', COLS)).toEqual({ + all: [{ field: 'wins', op: 'in', value: [1, 2, 3] }], + }) + }) + + it('maps neq→ne, is.null→isNull, not.is.null→isNotNull', () => { + expect(parsePostgrestFilter('status=neq.x', COLS)).toEqual({ + all: [{ field: 'status', op: 'ne', value: 'x' }], + }) + expect(parsePostgrestFilter('status=is.null', COLS)).toEqual({ + all: [{ field: 'status', op: 'isNull' }], + }) + expect(parsePostgrestFilter('status=not.is.null', COLS)).toEqual({ + all: [{ field: 'status', op: 'isNotNull' }], + }) + }) + + it('supports not.eq → ne and not.in → nin', () => { + expect(parsePostgrestFilter('status=not.eq.x', COLS)).toEqual({ + all: [{ field: 'status', op: 'ne', value: 'x' }], + }) + expect(parsePostgrestFilter('slack_user_id=not.in.(U1,U2)', COLS)).toEqual({ + all: [{ field: 'slack_user_id', op: 'nin', value: ['U1', 'U2'] }], + }) + }) + + it('keeps like/ilike/match values as raw text (wildcards/regex preserved)', () => { + expect(parsePostgrestFilter('name=ilike.*jo*', COLS)).toEqual({ + all: [{ field: 'name', op: 'ilike', value: '*jo*' }], + }) + expect(parsePostgrestFilter('name=match.^a.*z$', COLS)).toEqual({ + all: [{ field: 'name', op: 'match', value: '^a.*z$' }], + }) + }) + + it('parses or=(...) and nested and()/or() groups', () => { + expect(parsePostgrestFilter('or=(status.eq.active,status.eq.pending)', COLS)).toEqual({ + all: [ + { + any: [ + { field: 'status', op: 'eq', value: 'active' }, + { field: 'status', op: 'eq', value: 'pending' }, + ], + }, + ], + }) + const nested = parsePostgrestFilter('and=(wins.gte.1,or(status.eq.a,status.eq.b))', COLS) + expect(nested).toEqual({ + all: [ + { + all: [ + { field: 'wins', op: 'gte', value: 1 }, + { + any: [ + { field: 'status', op: 'eq', value: 'a' }, + { field: 'status', op: 'eq', value: 'b' }, + ], + }, + ], + }, + ], + }) + }) + + it('rejects unsupported ops with an actionable error', () => { + expect(() => parsePostgrestFilter('tags=cs.{x}', COLS)).toThrow(/not supported/) + expect(() => parsePostgrestFilter('doc=fts.hello', COLS)).toThrow(/full-text/) + }) + + it('rejects unknown ops, bad fields, and unsupported negation', () => { + expect(() => parsePostgrestFilter('wins=bogus.1', COLS)).toThrow(/Unknown filter operator/) + expect(() => parsePostgrestFilter('bad name=eq.1', COLS)).toThrow(/Invalid filter column/) + expect(() => parsePostgrestFilter('name=not.ilike.x', COLS)).toThrow(/not supported/) + }) + + it('rejects empty / malformed input', () => { + expect(() => parsePostgrestFilter('', COLS)).toThrow(/empty/) + expect(() => parsePostgrestFilter('justakey', COLS)).toThrow(/Malformed/) + }) + + it('honors quotes when splitting lists', () => { + expect(parsePostgrestFilter('status=in.("a,b",c)', COLS)).toEqual({ + all: [{ field: 'status', op: 'in', value: ['a,b', 'c'] }], + }) + }) +}) + +describe('parsePostgrestOrder', () => { + it('parses col.dir pairs and defaults to asc', () => { + expect(parsePostgrestOrder('wins.desc,name', COLS)).toEqual([ + { field: 'wins', direction: 'desc' }, + { field: 'name', direction: 'asc' }, + ]) + }) + it('allows createdAt/updatedAt and rejects unknown columns', () => { + expect(parsePostgrestOrder('createdAt.desc', COLS)).toEqual([ + { field: 'createdAt', direction: 'desc' }, + ]) + expect(() => parsePostgrestOrder('nope.asc', COLS)).toThrow(/Unknown sort column/) + }) +}) + +describe('predicateToPostgrest round-trips', () => { + it('round-trips an AND of leaves', () => { + const p: TablePredicate = { + all: [ + { field: 'wins', op: 'gte', value: 10 }, + { field: 'slack_user_id', op: 'in', value: ['U1', 'U2'] }, + ], + } + const str = predicateToPostgrest(p) + expect(str).toBe('wins=gte.10&slack_user_id=in.(U1,U2)') + expect(parsePostgrestFilter(str, COLS)).toEqual(p) + }) + + it('round-trips an OR group', () => { + const p: TablePredicate = { + all: [ + { + any: [ + { field: 'status', op: 'eq', value: 'active' }, + { field: 'status', op: 'eq', value: 'pending' }, + ], + }, + ], + } + expect(parsePostgrestFilter(predicateToPostgrest(p), COLS)).toEqual(p) + }) + + it('serializes builder-only ops onto PostgREST forms', () => { + expect(leaf('contains', 'foo')).toBe('name=ilike.*foo*') + expect(leaf('startsWith', 'foo')).toBe('name=ilike.foo*') + expect(leaf('isEmpty')).toBe('name=is.null') + }) + + it('order spec round-trips', () => { + const s = [ + { field: 'wins', direction: 'desc' as const }, + { field: 'name', direction: 'asc' as const }, + ] + expect(sortSpecToPostgrestOrder(s)).toBe('wins.desc,name.asc') + expect(parsePostgrestOrder(sortSpecToPostgrestOrder(s), COLS)).toEqual(s) + }) +}) + +function leaf(op: string, value?: string): string { + return predicateToPostgrest({ + all: [ + value === undefined + ? { field: 'name', op: op as never } + : { field: 'name', op: op as never, value }, + ], + }) +} diff --git a/apps/sim/lib/table/query-builder/converters.ts b/apps/sim/lib/table/query-builder/converters.ts index a789fa0b29e..41dfc0fca32 100644 --- a/apps/sim/lib/table/query-builder/converters.ts +++ b/apps/sim/lib/table/query-builder/converters.ts @@ -4,6 +4,7 @@ import { generateShortId } from '@sim/utils/id' import { isRecordLike } from '@sim/utils/object' +import { predicateToPostgrest, sortSpecToPostgrestOrder } from '@/lib/table/query-builder/postgrest' import type { Filter, FilterOp, @@ -145,7 +146,14 @@ function applyLogicalOperators(groups: FilterRule[][]): FilterRule[] { } const ARRAY_OPERATORS = new Set(['in', 'nin']) -const TEXT_MATCH_OPERATORS = new Set(['contains', 'ncontains', 'startsWith', 'endsWith']) +const TEXT_MATCH_OPERATORS = new Set([ + 'contains', + 'ncontains', + 'startsWith', + 'endsWith', + 'match', + 'imatch', +]) function parseValue(value: string, operator: string): JsonValue { if (ARRAY_OPERATORS.has(operator)) { @@ -234,7 +242,7 @@ function normalizeSortDirection(direction: string): SortDirection { /* ----------------------------- v2 grammar ----------------------------- */ -const VALUELESS_OPS = new Set(['isEmpty', 'isNotEmpty']) +const VALUELESS_OPS = new Set(['isEmpty', 'isNotEmpty', 'isNull', 'isNotNull']) function ruleToPredicate(rule: FilterRule): Predicate { const op = rule.operator as FilterOp @@ -248,7 +256,9 @@ function ruleToPredicate(rule: FilterRule): Predicate { * Mirrors {@link filterRulesToFilter} but emits the bare-operator grammar. */ export function filterRulesToPredicate(rules: FilterRule[]): TablePredicate | null { - if (rules.length === 0) return null + // Tolerate a non-array (the builder value can arrive malformed from an agent + // that doesn't speak the rule shape) instead of throwing "rules is not iterable". + if (!Array.isArray(rules) || rules.length === 0) return null const groups: Predicate[][] = [] let current: Predicate[] = [] @@ -308,9 +318,28 @@ export function sortRulesToSortSpec(rules: SortRule[]): SortSpec | null { return spec.length > 0 ? spec : null } +/** + * Serializes UI filter-builder rules straight to a PostgREST filter querystring, + * so the v2 block can offer the visual builder while the wire stays PostgREST. + * Returns `null` when the builder is empty. + */ +export function filterRulesToPostgrest(rules: FilterRule[]): string | null { + const predicate = filterRulesToPredicate(rules) + return predicate ? predicateToPostgrest(predicate) : null +} + +/** Serializes UI sort-builder rules to a PostgREST `order` string, or `null`. */ +export function sortRulesToPostgrestOrder(rules: SortRule[]): string | null { + const spec = sortRulesToSortSpec(rules) + return spec ? sortSpecToPostgrestOrder(spec) : null +} + function predicateLeafToFilterValue(p: Predicate): Filter[string] { if (p.op === 'isEmpty') return { $empty: true } if (p.op === 'isNotEmpty') return { $empty: false } + // Valueless null checks carry a dummy `true` so the key survives JSON transport. + if (p.op === 'isNull') return { $isNull: true } as Filter[string] + if (p.op === 'isNotNull') return { $isNotNull: true } as Filter[string] if (p.op === 'eq') return p.value as Filter[string] return { [`$${p.op}`]: p.value } as Filter[string] } diff --git a/apps/sim/lib/table/query-builder/postgrest.ts b/apps/sim/lib/table/query-builder/postgrest.ts new file mode 100644 index 00000000000..fb0cf2e9d55 --- /dev/null +++ b/apps/sim/lib/table/query-builder/postgrest.ts @@ -0,0 +1,374 @@ +/** + * PostgREST filter grammar ↔ the internal `TablePredicate` IR. + * + * This is the v2 wire/authoring grammar: a PostgREST-style querystring fragment + * that mothership and clients write, parsed here into the same `TablePredicate` + * the engine already compiles (`buildPredicateClause` → `fieldPredicate`). The + * parser is the validator now that the boundary param is an opaque string — + * unknown ops, bad fields, and malformed input throw `TableQueryValidationError` + * (mapped to HTTP 400). Values are never interpolated into SQL; they ride through + * the parameterized `fieldPredicate` leaf. + * + * Supported (PostgREST token → FilterOp): eq, neq→ne, gt, gte, lt, lte, + * in, like, ilike, match, imatch, is.null→isNull. Negation `not.` is supported + * for `eq`→ne, `in`→nin, and `is.null`→isNotNull. Logic: top-level `&` (AND), + * `and=(...)`, `or=(...)`, and nested `and(...)`/`or(...)` inside groups. + * + * Unsupported (clear error): cs, cd, ov, fts/plfts/phfts/wfts, sl/sr/nxr/nxl/adj + * (no array/range/full-text columns), and `not.` outside eq/in/is.null. + */ + +import { NAME_PATTERN } from '@/lib/table/constants' +import { TableQueryValidationError } from '@/lib/table/sql' +import type { + ColumnDefinition, + FilterOp, + JsonValue, + Predicate, + PredicateNode, + SortDirection, + SortSpec, + TablePredicate, +} from '@/lib/table/types' + +type ColumnType = ColumnDefinition['type'] + +const MAX_FILTER_LENGTH = 4096 + +/** Ops whose value is a text pattern — never coerced to number/boolean. */ +const TEXT_OPS = new Set(['like', 'ilike', 'match', 'imatch']) + +/* ------------------------------- parsing ------------------------------- */ + +/** Splits on a delimiter at depth 0, respecting `(...)` nesting and `"..."` quotes. */ +function splitTopLevel(input: string, delimiter: string): string[] { + const parts: string[] = [] + let depth = 0 + let inQuotes = false + let current = '' + for (let i = 0; i < input.length; i++) { + const ch = input[i] + if (ch === '"') { + inQuotes = !inQuotes + current += ch + } else if (inQuotes) { + current += ch + } else if (ch === '(') { + depth++ + current += ch + } else if (ch === ')') { + depth-- + if (depth < 0) throw new TableQueryValidationError('Unbalanced parentheses in filter') + current += ch + } else if (ch === delimiter && depth === 0) { + parts.push(current) + current = '' + } else { + current += ch + } + } + if (depth !== 0 || inQuotes) { + throw new TableQueryValidationError('Unbalanced parentheses or quotes in filter') + } + parts.push(current) + return parts +} + +function validateField(field: string): void { + if (!NAME_PATTERN.test(field)) { + throw new TableQueryValidationError( + `Invalid filter column "${field}". Use a column name (letters, digits, underscore).` + ) + } +} + +/** Coerces a raw PostgREST value string to the column's stored JSON type. */ +function coerceValue(raw: string, type: ColumnType | undefined): JsonValue { + // Strip one layer of PostgREST double-quoting (used to escape reserved chars). + const v = raw.length >= 2 && raw.startsWith('"') && raw.endsWith('"') ? raw.slice(1, -1) : raw + if (type === 'number') { + const n = Number(v) + if (v.trim() === '' || Number.isNaN(n)) { + throw new TableQueryValidationError(`Expected a number for column value, got "${v}"`) + } + return n + } + if (type === 'boolean') { + if (v === 'true') return true + if (v === 'false') return false + throw new TableQueryValidationError(`Expected true/false for boolean column, got "${v}"`) + } + // string / date / json: keep as text (dates compare as ISO strings). + return v +} + +/** Parses a PostgREST list literal `(a,b,"c,d")` into its elements. */ +function parseList(raw: string): string[] { + const trimmed = raw.trim() + if (!trimmed.startsWith('(') || !trimmed.endsWith(')')) { + throw new TableQueryValidationError(`Expected a list like in.(a,b), got "${raw}"`) + } + const inner = trimmed.slice(1, -1) + if (inner.trim() === '') return [] + return splitTopLevel(inner, ',').map((s) => s.trim()) +} + +const UNSUPPORTED_OPS: Record = { + cs: 'array/jsonb containment', + cd: 'array/jsonb containment', + ov: 'array/range overlap', + fts: 'full-text search', + plfts: 'full-text search', + phfts: 'full-text search', + wfts: 'full-text search', + sl: 'range', + sr: 'range', + nxr: 'range', + nxl: 'range', + adj: 'range', +} + +/** + * Parses one leaf `field` + `op.value` spec into a `Predicate`. `opSpec` is the + * part after the field (e.g. `gte.10`, `in.(a,b)`, `is.null`, `not.eq.5`). + */ +function parseLeaf(field: string, opSpec: string, typeByName: Map): Predicate { + validateField(field) + const type = typeByName.get(field) + + let negated = false + let rest = opSpec + if (rest.startsWith('not.')) { + negated = true + rest = rest.slice(4) + } + + const dot = rest.indexOf('.') + if (dot === -1) throw new TableQueryValidationError(`Malformed filter on "${field}": "${opSpec}"`) + const token = rest.slice(0, dot) + const rawValue = rest.slice(dot + 1) + + if (UNSUPPORTED_OPS[token]) { + throw new TableQueryValidationError( + `Operator "${token}" (${UNSUPPORTED_OPS[token]}) is not supported on user tables.` + ) + } + + // is.null / is.true / is.false + if (token === 'is') { + if (rawValue === 'null') { + return { field, op: negated ? 'isNotNull' : 'isNull' } + } + if (rawValue === 'true' || rawValue === 'false') { + if (negated) throw unsupportedNegation(token) + return { field, op: 'eq', value: rawValue === 'true' } + } + throw new TableQueryValidationError( + `Unsupported "is.${rawValue}" — use is.null, is.true, is.false` + ) + } + + if (token === 'in') { + const items = parseList(rawValue).map((el) => coerceValue(el, type)) + return { field, op: negated ? 'nin' : 'in', value: items } + } + + // Scalar / text ops + const opMap: Record = { + eq: 'eq', + neq: 'ne', + gt: 'gt', + gte: 'gte', + lt: 'lt', + lte: 'lte', + like: 'like', + ilike: 'ilike', + match: 'match', + imatch: 'imatch', + } + const op = opMap[token] + if (!op) { + throw new TableQueryValidationError(`Unknown filter operator "${token}" on column "${field}".`) + } + if (negated) { + // Only eq/in/is.null negate cleanly into our op set. + if (op === 'eq') return { field, op: 'ne', value: scalarValue(rawValue, op, type) } + throw unsupportedNegation(token) + } + return { field, op, value: scalarValue(rawValue, op, type) } +} + +function scalarValue(raw: string, op: FilterOp, type: ColumnType | undefined): JsonValue { + // Text/pattern ops keep the raw string (the `*` wildcard and regex are textual). + if (TEXT_OPS.has(op)) { + return raw.length >= 2 && raw.startsWith('"') && raw.endsWith('"') ? raw.slice(1, -1) : raw + } + return coerceValue(raw, type) +} + +function unsupportedNegation(token: string): TableQueryValidationError { + return new TableQueryValidationError( + `Negation "not.${token}" is not supported — only not.eq, not.in, and not.is.null.` + ) +} + +/** Parses the comma-separated items inside an `and(...)`/`or(...)` group. */ +function parseGroupItems(content: string, typeByName: Map): PredicateNode[] { + return splitTopLevel(content, ',').map((item) => parseGroupItem(item.trim(), typeByName)) +} + +/** A single item inside a group: a nested `and()/or()` group, or a `field.op.value` leaf. */ +function parseGroupItem(item: string, typeByName: Map): PredicateNode { + const groupMatch = /^(and|or)\((.*)\)$/s.exec(item) + if (groupMatch) { + const members = parseGroupItems(groupMatch[2], typeByName) + return groupMatch[1] === 'and' ? { all: members } : { any: members } + } + // Leaf in dot-form: field.op.value — field is up to the first dot. + const dot = item.indexOf('.') + if (dot === -1) throw new TableQueryValidationError(`Malformed filter condition: "${item}"`) + return parseLeaf(item.slice(0, dot), item.slice(dot + 1), typeByName) +} + +/** + * Parses a PostgREST filter querystring fragment into a `TablePredicate`. + * Top-level `&`-separated params AND together; `and=(...)`/`or=(...)` form groups. + */ +export function parsePostgrestFilter(input: string, columns: ColumnDefinition[]): TablePredicate { + if (typeof input !== 'string' || input.trim() === '') { + throw new TableQueryValidationError('Filter is empty') + } + if (input.length > MAX_FILTER_LENGTH) { + throw new TableQueryValidationError(`Filter exceeds ${MAX_FILTER_LENGTH} characters`) + } + const typeByName = new Map(columns.map((c) => [c.name, c.type])) + + const nodes: PredicateNode[] = [] + for (const segment of splitTopLevel(input, '&')) { + const trimmed = segment.trim() + if (trimmed === '') continue + const eq = trimmed.indexOf('=') + if (eq === -1) throw new TableQueryValidationError(`Malformed filter param: "${trimmed}"`) + const key = trimmed.slice(0, eq).trim() + const rhs = trimmed.slice(eq + 1).trim() + + if (key === 'and' || key === 'or') { + const members = parseList(rhs) + // re-parse as group items (parseList split top-level commas already) + const parsed = members.map((m) => parseGroupItem(m, typeByName)) + nodes.push(key === 'and' ? { all: parsed } : { any: parsed }) + continue + } + nodes.push(parseLeaf(key, rhs, typeByName)) + } + + if (nodes.length === 0) throw new TableQueryValidationError('Filter has no conditions') + return { all: nodes } +} + +/** Parses a PostgREST `order` value (`col.desc,col2.asc`) into a SortSpec. */ +export function parsePostgrestOrder(input: string, columns: ColumnDefinition[]): SortSpec { + const valid = new Set(columns.map((c) => c.name).concat(['createdAt', 'updatedAt'])) + const spec: SortSpec = [] + for (const part of input.split(',')) { + const seg = part.trim() + if (seg === '') continue + const [field, dirRaw = 'asc'] = seg.split('.') + validateField(field) + if (!valid.has(field)) { + throw new TableQueryValidationError(`Unknown sort column "${field}"`) + } + const direction: SortDirection = dirRaw === 'desc' ? 'desc' : 'asc' + spec.push({ field, direction }) + } + return spec +} + +/* ------------------------------ serializing ------------------------------ */ + +function leafToOpSpec(p: Predicate): string { + const v = p.value + switch (p.op) { + case 'eq': + return `eq.${serializeValue(v)}` + case 'ne': + return `neq.${serializeValue(v)}` + case 'gt': + case 'gte': + case 'lt': + case 'lte': + return `${p.op}.${serializeValue(v)}` + case 'in': + return `in.(${(v as JsonValue[]).map(serializeValue).join(',')})` + case 'nin': + return `not.in.(${(v as JsonValue[]).map(serializeValue).join(',')})` + case 'like': + case 'ilike': + case 'match': + case 'imatch': + return `${p.op}.${serializeValue(v)}` + // Builder-only ops map onto PostgREST text/null forms. + case 'contains': + return `ilike.*${serializeValue(v)}*` + case 'ncontains': + return `not.ilike.*${serializeValue(v)}*` + case 'startsWith': + return `ilike.${serializeValue(v)}*` + case 'endsWith': + return `ilike.*${serializeValue(v)}` + case 'isNull': + case 'isEmpty': + return 'is.null' + case 'isNotNull': + case 'isNotEmpty': + return 'not.is.null' + default: + throw new TableQueryValidationError(`Cannot serialize operator "${p.op}"`) + } +} + +/** Quotes values containing reserved chars so they survive a round-trip. */ +function serializeValue(value: JsonValue | undefined): string { + const s = value === null || value === undefined ? 'null' : String(value) + return /[,.()&="]/.test(s) ? `"${s.replace(/"/g, '\\"')}"` : s +} + +function isGroup(node: PredicateNode): node is TablePredicate { + return 'all' in node || 'any' in node +} + +/** Serializes a node as a group item (dot-form leaves, function-form groups). */ +function nodeToGroupItem(node: PredicateNode): string { + if (isGroup(node)) { + const isAll = 'all' in node + const members = isAll ? node.all : node.any + // A single-member group adds no logic — flatten it so a builder-emitted + // `{ all: [leaf] }` serializes to the leaf, not a redundant `and(leaf)`. + if (members.length === 1) return nodeToGroupItem(members[0]) + return `${isAll ? 'and' : 'or'}(${members.map(nodeToGroupItem).join(',')})` + } + return `${node.field}.${leafToOpSpec(node)}` +} + +/** + * Serializes a `TablePredicate` to a PostgREST querystring fragment (for the + * builder UI). A top-level `all` becomes `&`-joined params; `any` becomes + * `or=(...)`; nested groups use the function form. + */ +export function predicateToPostgrest(predicate: TablePredicate): string { + if ('any' in predicate) { + return `or=(${predicate.any.map(nodeToGroupItem).join(',')})` + } + return predicate.all + .map((node) => + isGroup(node) + ? `${'all' in node ? 'and' : 'or'}=(${(('all' in node ? node.all : node.any) as PredicateNode[]).map(nodeToGroupItem).join(',')})` + : `${node.field}=${leafToOpSpec(node)}` + ) + .join('&') +} + +/** Serializes a SortSpec to a PostgREST `order` value. */ +export function sortSpecToPostgrestOrder(sort: SortSpec): string { + return sort.map((s) => `${s.field}.${s.direction}`).join(',') +} diff --git a/apps/sim/lib/table/rows/service.ts b/apps/sim/lib/table/rows/service.ts index d51955225d1..b9ed66dcc20 100644 --- a/apps/sim/lib/table/rows/service.ts +++ b/apps/sim/lib/table/rows/service.ts @@ -53,6 +53,7 @@ import { buildSortClause, escapeLikePattern, fieldPredicate, + TableQueryValidationError, } from '@/lib/table/sql' import { fireTableTrigger } from '@/lib/table/trigger' import { scaledStatementTimeoutMs, setTableTxTimeouts } from '@/lib/table/tx' @@ -977,7 +978,9 @@ export async function queryRows( filter, predicate, sort, - limit = TABLE_LIMITS.DEFAULT_QUERY_LIMIT, + // No default: an undefined limit returns every matching row, bounded only by + // the MAX_QUERY_RESULT_BYTES fail-fast guard below. + limit, offset = 0, after, includeTotal = true, @@ -1031,6 +1034,7 @@ export async function queryRows( .from(userTableRows) .where(pageWhere ?? baseConditions) .orderBy(buildRowOrderBySql(sort, tableName, columns, fractionalOrdering)) + if (limit === undefined) return query return after ? query.limit(limit) : query.limit(limit).offset(offset) } @@ -1077,10 +1081,23 @@ export async function queryRows( updatedAt: r.updatedAt, })) - // Opaque next-page cursor: null once we've returned a short (final) page. + // Fail fast if the result outgrows the byte budget instead of returning a + // silently truncated page — the caller narrows it with a filter or a limit. + let resultBytes = 0 + for (const row of mappedRows) { + resultBytes += Buffer.byteLength(JSON.stringify(row.data)) + if (resultBytes > TABLE_LIMITS.MAX_QUERY_RESULT_BYTES) { + throw new TableQueryValidationError( + `Query result exceeds the ${TABLE_LIMITS.MAX_QUERY_RESULT_BYTES / (1024 * 1024)}MB limit. Add a filter or a limit to narrow the result.` + ) + } + } + + // Opaque next-page cursor: only meaningful for a bounded page that filled up. + // An unbounded query returns everything, so there is never a next page. const lastRow = mappedRows[mappedRows.length - 1] const nextCursor = - mappedRows.length < limit || !lastRow + limit === undefined || mappedRows.length < limit || !lastRow ? null : encodeCursor({ lastRow, sort, nextOffset: offset + mappedRows.length }) @@ -1088,7 +1105,7 @@ export async function queryRows( rows: mappedRows, rowCount: rows.length, totalCount, - limit, + limit: limit ?? mappedRows.length, offset, nextCursor, } @@ -1366,17 +1383,26 @@ export async function updateRowsByFilter( eq(userTableRows.workspaceId, table.workspaceId) ) + // A limit selects a SUBSET, so impose the default `(order_key, id)` order — + // without it Postgres returns planner-arbitrary rows and "update the first N" + // is nondeterministic. Sort is irrelevant (and skipped) when every match is updated. + const fractionalOrdering = data.limit + ? await isFeatureEnabled('tables-fractional-ordering') + : false + // Tenant-bounded: the jsonb filter is unestimatable and otherwise sends the planner to a // whole-shared-relation seq scan (14.4s measured on a 1M-row table). const matchingRows = await withSeqscanOff(async (trx) => { - let query = trx + const base = trx .select({ id: userTableRows.id, data: userTableRows.data }) .from(userTableRows) .where(and(baseConditions, filterClause)) if (data.limit) { - query = query.limit(data.limit) as typeof query + return base + .orderBy(buildRowOrderBySql(undefined, tableName, table.schema.columns, fractionalOrdering)) + .limit(data.limit) } - return query + return base }) if (matchingRows.length === 0) { @@ -1709,16 +1735,24 @@ export async function deleteRowsByFilter( eq(userTableRows.workspaceId, table.workspaceId) ) + // A limit deletes a SUBSET, so order deterministically by `(order_key, id)` — + // see updateRowsByFilter. Unbounded deletes affect every match, so order is moot. + const fractionalOrdering = data.limit + ? await isFeatureEnabled('tables-fractional-ordering') + : false + // Tenant-bounded for the same reason as updateRowsByFilter — see withSeqscanOff. const matchingRows = await withSeqscanOff(async (trx) => { - let query = trx + const base = trx .select({ id: userTableRows.id, position: userTableRows.position }) .from(userTableRows) .where(and(baseConditions, filterClause)) if (data.limit) { - query = query.limit(data.limit) as typeof query + return base + .orderBy(buildRowOrderBySql(undefined, tableName, table.schema.columns, fractionalOrdering)) + .limit(data.limit) } - return query + return base }) if (matchingRows.length === 0) { diff --git a/apps/sim/lib/table/sql.ts b/apps/sim/lib/table/sql.ts index b1cc6360335..fb6b2d7e389 100644 --- a/apps/sim/lib/table/sql.ts +++ b/apps/sim/lib/table/sql.ts @@ -78,7 +78,13 @@ const ALLOWED_OPERATORS = new Set([ '$ncontains', '$startsWith', '$endsWith', + '$like', + '$ilike', + '$match', + '$imatch', '$empty', + '$isNull', + '$isNotNull', ]) /** @@ -449,11 +455,26 @@ export function fieldPredicate( case 'endsWith': return buildLikeClause(tableName, field, value as string, 'endsWith') + case 'like': + return buildPatternClause(tableName, field, value as string, false) + case 'ilike': + return buildPatternClause(tableName, field, value as string, true) + case 'isEmpty': return buildEmptyClause(tableName, field, true) case 'isNotEmpty': return buildEmptyClause(tableName, field, false) + case 'match': + return buildRegexClause(tableName, field, value as string, false) + case 'imatch': + return buildRegexClause(tableName, field, value as string, true) + + case 'isNull': + return buildNullClause(tableName, field, true) + case 'isNotNull': + return buildNullClause(tableName, field, false) + default: throw new TableQueryValidationError(`Invalid operator "${op}"`) } @@ -543,6 +564,28 @@ export function escapeLikePattern(value: string): string { return value.replace(/[\\%_]/g, '\\$&') } +/** + * General LIKE/ILIKE pattern match (PostgREST `like`/`ilike`). The caller's `*` + * is the only wildcard — it maps to SQL `%`; any literal `%`/`_`/`\` in the + * value is escaped so it matches itself. Empty/exact patterns are allowed (an + * empty pattern matches only the empty string, not every row, so it's not the + * footgun the positional `buildLikeClause` guards against). Cannot use the GIN + * index; sequential scan bounded by the `table_id` btree prefix. + */ +function buildPatternClause( + tableName: string, + field: string, + value: string, + caseInsensitive: boolean +): SQL { + const escapedField = field.replace(/'/g, "''") + const pattern = String(value) + .replace(/[\\%_]/g, '\\$&') + .replace(/\*/g, '%') + const cell = sql.raw(`${tableName}.data->>'${escapedField}'`) + return caseInsensitive ? sql`${cell} ILIKE ${pattern}` : sql`${cell} LIKE ${pattern}` +} + /** * Builds a case-insensitive pattern match against a JSONB cell using ILIKE. * `position` controls wildcard placement: `contains` → `%value%`, `startsWith` @@ -612,6 +655,37 @@ function buildEmptyClause(tableName: string, field: string, isEmpty: boolean): S : sql`(${cell} IS NOT NULL AND ${cell} <> '')` } +/** + * Builds a POSIX regex match against a JSONB cell (`~` case-sensitive, `~*` + * case-insensitive). The pattern is parameterized (no interpolation); Postgres + * validates it at runtime. Negation surfaces null cells, mirroring `$ne`/ + * `$ncontains`. Cannot use the GIN index — sequential scan bounded by the + * `table_id` btree prefix. + */ +function buildRegexClause( + tableName: string, + field: string, + value: string, + caseInsensitive: boolean +): SQL { + const escapedField = field.replace(/'/g, "''") + const cell = sql.raw(`${tableName}.data->>'${escapedField}'`) + const pattern = String(value) + return caseInsensitive ? sql`${cell} ~* ${pattern}` : sql`${cell} ~ ${pattern}` +} + +/** + * Strict null check on a JSONB cell — distinct from `isEmpty`/`isNotEmpty`, + * which also treat the empty string as empty. `isNull` matches an absent key or + * JSON null (both surfaced as SQL NULL by `->>`); the negation requires the cell + * to be present (an empty string counts as not-null here). + */ +function buildNullClause(tableName: string, field: string, isNull: boolean): SQL { + const escapedField = field.replace(/'/g, "''") + const cell = sql.raw(`${tableName}.data->>'${escapedField}'`) + return isNull ? sql`${cell} IS NULL` : sql`${cell} IS NOT NULL` +} + /** * Builds a single ORDER BY clause for a field. * Timestamp fields use direct column access, others use JSONB text extraction. diff --git a/apps/sim/lib/table/types.ts b/apps/sim/lib/table/types.ts index 6d5fd8922b4..afe43699b1c 100644 --- a/apps/sim/lib/table/types.ts +++ b/apps/sim/lib/table/types.ts @@ -426,9 +426,11 @@ export interface Filter { /** * v2 filter operators (bare, no `$`). Equality and `in`/`nin` are case-sensitive * (JSONB containment, GIN-indexed); the text ops `contains`/`ncontains`/ - * `startsWith`/`endsWith` are ILIKE (case-insensitive). `isEmpty`/`isNotEmpty` - * are valueless. This is the canonical operator set the shared `fieldPredicate` - * leaf understands; the legacy `$`-operators normalize onto it. + * `startsWith`/`endsWith` are ILIKE (case-insensitive); `match`/`imatch` are + * POSIX regex (`~` / `~*`). `isEmpty`/`isNotEmpty` match null OR empty string; + * `isNull`/`isNotNull` are strict null checks. The four `is*` ops are valueless. + * This is the canonical operator set the shared `fieldPredicate` leaf + * understands; the legacy `$`-operators normalize onto it. */ export type FilterOp = | 'eq' @@ -443,8 +445,14 @@ export type FilterOp = | 'ncontains' | 'startsWith' | 'endsWith' + | 'like' + | 'ilike' + | 'match' + | 'imatch' | 'isEmpty' | 'isNotEmpty' + | 'isNull' + | 'isNotNull' /** A single v2 leaf predicate: `field op value`. `value` is omitted for `isEmpty`/`isNotEmpty`. */ export interface Predicate { diff --git a/apps/sim/tools/registry.minimal.ts b/apps/sim/tools/registry.minimal.ts index d1b34868ec8..b313c2e075f 100644 --- a/apps/sim/tools/registry.minimal.ts +++ b/apps/sim/tools/registry.minimal.ts @@ -1,5 +1,17 @@ import { functionExecuteTool } from '@/tools/function' import { httpRequestTool } from '@/tools/http' +import { + tableBatchInsertRowsTool, + tableDeleteRowsByFilterTool, + tableDeleteRowTool, + tableGetRowTool, + tableGetSchemaTool, + tableInsertRowTool, + tableQueryRowsV2Tool, + tableUpdateRowsByFilterTool, + tableUpdateRowTool, + tableUpsertRowTool, +} from '@/tools/table' import type { ToolConfig } from '@/tools/types' /** @@ -13,4 +25,15 @@ import type { ToolConfig } from '@/tools/types' export const tools: Record = { http_request: httpRequestTool, function_execute: functionExecuteTool, + // Table v2 block operations (so the v2 Table block is runnable in minimal mode). + table_insert_row: tableInsertRowTool, + table_batch_insert_rows: tableBatchInsertRowsTool, + table_upsert_row: tableUpsertRowTool, + table_update_row: tableUpdateRowTool, + table_update_rows_by_filter: tableUpdateRowsByFilterTool, + table_delete_row: tableDeleteRowTool, + table_delete_rows_by_filter: tableDeleteRowsByFilterTool, + table_query_rows_v2: tableQueryRowsV2Tool, + table_get_row: tableGetRowTool, + table_get_schema: tableGetSchemaTool, } diff --git a/apps/sim/tools/table/query_rows_v2.ts b/apps/sim/tools/table/query_rows_v2.ts index 709a9546d22..967fc156f69 100644 --- a/apps/sim/tools/table/query_rows_v2.ts +++ b/apps/sim/tools/table/query_rows_v2.ts @@ -1,19 +1,20 @@ -import { TABLE_LIMITS } from '@/lib/table/constants' import type { TableQueryV2Response, TableRowQueryV2Params } from '@/tools/table/types' import type { ToolConfig } from '@/tools/types' /** - * v2 row query: a structured `all`/`any` predicate grammar and opaque cursor - * pagination (no offset). Hits POST `/api/table/[tableId]/query`. + * v2 row query: PostgREST filter grammar + opaque cursor pagination (no offset). + * Hits POST `/api/table/[tableId]/query`. */ export const tableQueryRowsV2Tool: ToolConfig = { id: 'table_query_rows_v2', name: 'Query Rows', description: - 'Query rows with a structured predicate and cursor pagination. The predicate is a nestable tree: ' + - '{ "all": [ { "field": "status", "op": "eq", "value": "active" } ] } (all = AND, any = OR). ' + - 'Operators: eq, ne, gt, gte, lt, lte, in, nin, contains, ncontains, startsWith, endsWith, isEmpty, isNotEmpty. ' + - 'To page, pass the nextCursor from the previous response back as cursor; omit it for the first page.', + 'Query rows with a PostgREST filter string and cursor pagination. ' + + 'Filter is a querystring fragment: `wins=gte.10&status=in.(active,pending)` (top-level params AND; ' + + '`or=(a.eq.1,b.eq.2)` / `and=(...)` for groups). Operators: eq, neq, gt, gte, lt, lte, in, like, ilike, ' + + 'match, imatch, is.null (negate with not., e.g. not.in, not.is.null). ' + + 'Order is PostgREST `order` (e.g. `wins.desc,name.asc`). To page, pass the nextCursor from the previous ' + + 'response back as cursor; omit it for the first page.', version: '1.0.0', params: { @@ -23,23 +24,24 @@ export const tableQueryRowsV2Tool: ToolConfig Date: Fri, 3 Jul 2026 18:31:45 -0700 Subject: [PATCH 03/26] feat(table): byte-bounded query drain (5MB) + PostgREST round-trip and cursor hardening MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit queryRows now drains rows in adaptively-sized bounded batches instead of one unbounded SELECT. Omitted limit returns the entire result and fails fast (400) past the 5MB byte budget; bounded pages byte-cut early and signal remaining rows via nextCursor (witnessed by a peek row, never inferred from page size). Cursor codec gains a compound {k,i,o} shape for unkeyed-row resumes and a keysetValid gate closing the fractional-flag-off keyset/order mismatch. Council must-fix cluster: - Serializer round-trip: whole-pattern quoting (contains with dots), backslash quote escaping, nlike/nilike engine ops (ncontains parses again), isEmpty desugars to or=(f.is.null,f.eq."") preserving null-or-empty semantics - Unconditional name→id translation on PostgREST string paths (session-auth filters no longer silently match zero rows) - Cursor decode hardening (object guard, o>=0), 400 on keyset cursor + order - Block: cursor "null" artifact guard, NaN limit fails fast, filterBuilder required-flag removed (serializer hard-block with agent-set filter string) - Parser: reject empty or=()/and=()/in.(), Number.isFinite, strict sort dirs - TableQueryValidationError moved to client-safe lib/table/errors.ts (drops the drizzle-orm edge from client-bundled block defs) Consumers: export stops on nextCursor (byte-cut pages no longer truncate), legacy/v1 routes expose nextCursor additively, v2 route drops executions, copilot query_rows reports partial pages with a continue offset, v1 Table block tools registered in the minimal registry. Agent catalog regenerated from the copilot fork (PostgREST string filter, order param, new pagination semantics). Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01Bd25zCzh21omjFhRz32ov6 --- .../app/api/table/[tableId]/export/route.ts | 4 +- .../api/table/[tableId]/query/route.test.ts | 167 +++++++++++++ .../app/api/table/[tableId]/query/route.ts | 26 +- .../table/[tableId]/rows/find/route.test.ts | 2 +- .../api/table/[tableId]/rows/find/route.ts | 2 +- .../api/table/[tableId]/rows/route.test.ts | 69 ++++- .../sim/app/api/table/[tableId]/rows/route.ts | 54 ++-- .../app/api/v1/tables/[tableId]/rows/route.ts | 6 +- apps/sim/blocks/blocks/table_v2.test.ts | 95 +++++++ apps/sim/blocks/blocks/table_v2.ts | 55 ++-- apps/sim/lib/api/contracts/tables.ts | 10 +- .../lib/copilot/generated/tool-catalog-v1.ts | 19 +- .../lib/copilot/generated/tool-schemas-v1.ts | 19 +- .../copilot/tools/server/table/user-table.ts | 15 +- apps/sim/lib/core/config/env.ts | 1 - apps/sim/lib/table/__tests__/paging.test.ts | 33 --- .../service-filter-threading.test.ts | 193 ++++++++++++-- apps/sim/lib/table/constants.ts | 25 +- apps/sim/lib/table/errors.ts | 14 ++ apps/sim/lib/table/index.ts | 1 + .../__tests__/converters.test.ts | 3 +- .../query-builder/__tests__/postgrest.test.ts | 78 +++++- apps/sim/lib/table/query-builder/postgrest.ts | 145 ++++++++--- .../lib/table/rows/__tests__/cursor.test.ts | 41 ++- apps/sim/lib/table/rows/cursor.ts | 71 ++++-- apps/sim/lib/table/rows/paging.ts | 23 -- apps/sim/lib/table/rows/service.ts | 236 ++++++++++++++---- apps/sim/lib/table/sql.ts | 44 ++-- apps/sim/lib/table/types.ts | 16 +- apps/sim/tools/registry.minimal.ts | 4 +- apps/sim/tools/table/query_rows_v2.ts | 8 +- packages/testing/src/mocks/database.mock.ts | 15 +- 32 files changed, 1192 insertions(+), 302 deletions(-) create mode 100644 apps/sim/app/api/table/[tableId]/query/route.test.ts create mode 100644 apps/sim/blocks/blocks/table_v2.test.ts delete mode 100644 apps/sim/lib/table/__tests__/paging.test.ts create mode 100644 apps/sim/lib/table/errors.ts delete mode 100644 apps/sim/lib/table/rows/paging.ts diff --git a/apps/sim/app/api/table/[tableId]/export/route.ts b/apps/sim/app/api/table/[tableId]/export/route.ts index 75208e3d982..c4c3631bb84 100644 --- a/apps/sim/app/api/table/[tableId]/export/route.ts +++ b/apps/sim/app/api/table/[tableId]/export/route.ts @@ -87,7 +87,9 @@ export const GET = withRouteHandler(async (request: NextRequest, { params }: Rou } } - if (result.rows.length < EXPORT_BATCH_SIZE) break + // A page can be cut by the byte budget before reaching EXPORT_BATCH_SIZE, + // so a short page does NOT mean the export is done — only a null cursor does. + if (!result.nextCursor) break offset += result.rows.length } diff --git a/apps/sim/app/api/table/[tableId]/query/route.test.ts b/apps/sim/app/api/table/[tableId]/query/route.test.ts new file mode 100644 index 00000000000..98ef63f97dc --- /dev/null +++ b/apps/sim/app/api/table/[tableId]/query/route.test.ts @@ -0,0 +1,167 @@ +/** + * @vitest-environment node + * + * v2 query route: PostgREST string parsing, unconditional name→id translation + * (session auth included — the string grammar is name-keyed for every caller), + * cursor validation, and the response envelope. + */ +import { hybridAuthMockFns } from '@sim/testing' +import { NextRequest } from 'next/server' +import { beforeEach, describe, expect, it, vi } from 'vitest' +import type { TableDefinition } from '@/lib/table/types' + +const { mockCheckAccess, mockQueryRows } = vi.hoisted(() => ({ + mockCheckAccess: vi.fn(), + mockQueryRows: vi.fn(), +})) + +vi.mock('@/app/api/table/utils', async () => { + const { NextResponse } = await import('next/server') + return { + checkAccess: mockCheckAccess, + accessError: (result: { status: number }) => + NextResponse.json({ error: 'Access denied' }, { status: result.status }), + } +}) + +vi.mock('@/lib/table', async () => { + // row-wire pulls the column-keys helpers through this barrel. + const columnKeys = await import('@/lib/table/column-keys') + return { ...columnKeys } +}) + +vi.mock('@/lib/table/rows/service', () => ({ + queryRows: mockQueryRows, +})) + +import { encodeCursor } from '@/lib/table/rows/cursor' +import { POST } from '@/app/api/table/[tableId]/query/route' + +function buildTable(): TableDefinition { + return { + id: 'tbl_1', + name: 'People', + description: null, + schema: { + columns: [ + { id: 'col_aaa', name: 'name', type: 'string' }, + { id: 'col_bbb', name: 'wins', type: 'number' }, + ], + }, + metadata: null, + rowCount: 0, + maxRows: 100, + workspaceId: 'workspace-1', + createdBy: 'user-1', + archivedAt: null, + createdAt: new Date('2024-01-01'), + updatedAt: new Date('2024-01-01'), + } +} + +function authAs(authType: 'session' | 'internal_jwt') { + hybridAuthMockFns.mockCheckSessionOrInternalAuth.mockResolvedValue({ + success: true, + userId: 'user-1', + authType, + }) +} + +function callQuery(body: Record) { + const req = new NextRequest('http://localhost:3000/api/table/tbl_1/query', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(body), + }) + return POST(req, { params: Promise.resolve({ tableId: 'tbl_1' }) }) +} + +const EMPTY_RESULT = { + rows: [], + rowCount: 0, + totalCount: 0, + limit: 0, + offset: 0, + nextCursor: null, +} + +describe('POST /api/table/[tableId]/query', () => { + beforeEach(() => { + vi.clearAllMocks() + mockCheckAccess.mockResolvedValue({ ok: true, table: buildTable() }) + mockQueryRows.mockResolvedValue(EMPTY_RESULT) + }) + + it('translates filter/order column names to storage ids for SESSION auth too', async () => { + authAs('session') + const res = await callQuery({ + workspaceId: 'workspace-1', + filter: 'name=eq.John&wins=gte.10', + order: 'wins.desc', + }) + + expect(res.status).toBe(200) + const options = mockQueryRows.mock.calls[0][1] + expect(options.predicate).toEqual({ + all: [ + { field: 'col_aaa', op: 'eq', value: 'John' }, + { field: 'col_bbb', op: 'gte', value: 10 }, + ], + }) + expect(options.sort).toEqual({ col_bbb: 'desc' }) + expect(options.withExecutions).toBe(false) + }) + + it('rejects a keyset cursor combined with a custom order', async () => { + authAs('internal_jwt') + const cursor = encodeCursor({ + lastRow: { id: 'row_1', orderKey: 'a1' }, + keysetValid: true, + nextOffset: 1, + }) + const res = await callQuery({ workspaceId: 'workspace-1', order: 'wins.desc', cursor }) + + expect(res.status).toBe(400) + const body = await res.json() + expect(body.error).toMatch(/not valid for a sorted query/) + expect(mockQueryRows).not.toHaveBeenCalled() + }) + + it('returns 400 (not 500) for a cursor that decodes to a JSON primitive', async () => { + authAs('internal_jwt') + const res = await callQuery({ + workspaceId: 'workspace-1', + cursor: Buffer.from('42').toString('base64url'), + }) + + expect(res.status).toBe(400) + expect((await res.json()).error).toBe('Invalid cursor') + }) + + it('returns 400 with the parser message for an invalid filter', async () => { + authAs('internal_jwt') + const res = await callQuery({ workspaceId: 'workspace-1', filter: 'wins=bogus.1' }) + + expect(res.status).toBe(400) + expect((await res.json()).error).toMatch(/Unknown filter operator/) + }) + + it('passes nextCursor through the response envelope and skips the count on later pages', async () => { + authAs('internal_jwt') + mockQueryRows.mockResolvedValue({ ...EMPTY_RESULT, nextCursor: 'tok' }) + const cursor = encodeCursor({ + lastRow: { id: 'row_1', orderKey: 'a1' }, + keysetValid: true, + nextOffset: 1, + }) + + const res = await callQuery({ workspaceId: 'workspace-1', cursor }) + + expect(res.status).toBe(200) + const body = await res.json() + expect(body.data.nextCursor).toBe('tok') + const options = mockQueryRows.mock.calls[0][1] + expect(options.includeTotal).toBe(false) + expect(options.after).toEqual({ orderKey: 'a1', id: 'row_1' }) + }) +}) diff --git a/apps/sim/app/api/table/[tableId]/query/route.ts b/apps/sim/app/api/table/[tableId]/query/route.ts index 9c25c4b942d..a1a6d4ec9a8 100644 --- a/apps/sim/app/api/table/[tableId]/query/route.ts +++ b/apps/sim/app/api/table/[tableId]/query/route.ts @@ -7,10 +7,11 @@ import { checkSessionOrInternalAuth } from '@/lib/auth/hybrid' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import type { Sort, TableSchema } from '@/lib/table' +import { buildIdByName, predicateNamesToIds, sortSpecNamesToIds } from '@/lib/table/column-keys' +import { TableQueryValidationError } from '@/lib/table/errors' import { parsePostgrestFilter, parsePostgrestOrder } from '@/lib/table/query-builder/postgrest' import { decodeCursor } from '@/lib/table/rows/cursor' import { queryRows } from '@/lib/table/rows/service' -import { TableQueryValidationError } from '@/lib/table/sql' import { rowWireTranslators } from '@/app/api/table/row-wire' import { accessError, checkAccess } from '@/app/api/table/utils' @@ -55,13 +56,25 @@ export const POST = withRouteHandler( const wire = rowWireTranslators(authResult.authType, schema) const cursor = body.cursor ? decodeCursor(body.cursor) : undefined - // PostgREST filter/order arrive as strings (column NAMES); parse with the - // schema for type coercion, then translate names → storage ids. + // A keyset cursor encodes a position on the DEFAULT `(order_key, id)` + // order; combining it with a custom order would silently return page 1 + // of the sorted view relabeled as a next page. + if (cursor?.after && body.order) { + return NextResponse.json( + { error: 'Cursor is not valid for a sorted query. Restart paging without the cursor.' }, + { status: 400 } + ) + } + + // PostgREST filter/order strings are column-NAME-keyed by construction + // (the parser validates against names), so translate names → storage ids + // unconditionally — unlike row data, this is not authType-dependent. + const idByName = buildIdByName(schema) const predicate = body.filter - ? wire.predicateIn(parsePostgrestFilter(body.filter, schema.columns)) + ? predicateNamesToIds(parsePostgrestFilter(body.filter, schema.columns), idByName) : undefined const sortSpec = body.order - ? wire.sortSpecIn(parsePostgrestOrder(body.order, schema.columns)) + ? sortSpecNamesToIds(parsePostgrestOrder(body.order, schema.columns), idByName) : undefined const sort: Sort | undefined = sortSpec?.length ? Object.fromEntries(sortSpec.map((s) => [s.field, s.direction])) @@ -77,6 +90,9 @@ export const POST = withRouteHandler( offset: cursor?.offset, // Only the first page (no inbound cursor) pays for the total count. includeTotal: !body.cursor, + // Executions are grid UI state; the v2 surface returns row data only + // and the byte budget deliberately measures just `data`. + withExecutions: false, }, requestId ) diff --git a/apps/sim/app/api/table/[tableId]/rows/find/route.test.ts b/apps/sim/app/api/table/[tableId]/rows/find/route.test.ts index 139427066cb..111324b14de 100644 --- a/apps/sim/app/api/table/[tableId]/rows/find/route.test.ts +++ b/apps/sim/app/api/table/[tableId]/rows/find/route.test.ts @@ -121,7 +121,7 @@ describe('GET /api/table/[tableId]/rows/find', () => { }) it('maps a TableQueryValidationError to 400', async () => { - const { TableQueryValidationError } = await import('@/lib/table/sql') + const { TableQueryValidationError } = await import('@/lib/table/errors') mockFindRowMatches.mockRejectedValueOnce(new TableQueryValidationError('Invalid field name')) const res = await callGet({ workspaceId: 'workspace-1', q: 'foo' }) expect(res.status).toBe(400) diff --git a/apps/sim/app/api/table/[tableId]/rows/find/route.ts b/apps/sim/app/api/table/[tableId]/rows/find/route.ts index 77a476ee145..81e5500bd9a 100644 --- a/apps/sim/app/api/table/[tableId]/rows/find/route.ts +++ b/apps/sim/app/api/table/[tableId]/rows/find/route.ts @@ -6,8 +6,8 @@ import { checkSessionOrInternalAuth } from '@/lib/auth/hybrid' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import type { Sort } from '@/lib/table' +import { TableQueryValidationError } from '@/lib/table/errors' import { findRowMatches } from '@/lib/table/rows/service' -import { TableQueryValidationError } from '@/lib/table/sql' import { accessError, checkAccess } from '@/app/api/table/utils' const logger = createLogger('TableRowsFindAPI') diff --git a/apps/sim/app/api/table/[tableId]/rows/route.test.ts b/apps/sim/app/api/table/[tableId]/rows/route.test.ts index 8c5fbab9276..7751091551e 100644 --- a/apps/sim/app/api/table/[tableId]/rows/route.test.ts +++ b/apps/sim/app/api/table/[tableId]/rows/route.test.ts @@ -6,11 +6,20 @@ import { NextRequest } from 'next/server' import { beforeEach, describe, expect, it, vi } from 'vitest' import type { TableDefinition } from '@/lib/table' -const { mockCheckAccess, mockInsertRow, mockValidateRowData, mockQueryRows } = vi.hoisted(() => ({ +const { + mockCheckAccess, + mockInsertRow, + mockValidateRowData, + mockQueryRows, + mockUpdateRowsByFilter, + mockDeleteRowsByFilter, +} = vi.hoisted(() => ({ mockCheckAccess: vi.fn(), mockInsertRow: vi.fn(), mockValidateRowData: vi.fn(), mockQueryRows: vi.fn(), + mockUpdateRowsByFilter: vi.fn(), + mockDeleteRowsByFilter: vi.fn(), })) vi.mock('@/app/api/table/utils', async () => { @@ -31,9 +40,9 @@ vi.mock('@/lib/table', async () => { insertRow: mockInsertRow, batchInsertRows: vi.fn(), batchUpdateRows: vi.fn(), - deleteRowsByFilter: vi.fn(), + deleteRowsByFilter: mockDeleteRowsByFilter, deleteRowsByIds: vi.fn(), - updateRowsByFilter: vi.fn(), + updateRowsByFilter: mockUpdateRowsByFilter, validateBatchRows: vi.fn(), validateRowData: mockValidateRowData, validateRowSize: vi.fn(() => ({ valid: true })), @@ -48,7 +57,7 @@ vi.mock('@/lib/table/sql', () => ({ TableQueryValidationError: class TableQueryValidationError extends Error {}, })) -import { GET, POST } from '@/app/api/table/[tableId]/rows/route' +import { DELETE, GET, POST, PUT } from '@/app/api/table/[tableId]/rows/route' function buildTable(): TableDefinition { return { @@ -218,3 +227,55 @@ describe('GET /api/table/[tableId]/rows', () => { expect(body.data.rows[0].data).toEqual({ col_aaa: 'Ada', col_bbb: 36 }) }) }) + +describe('PUT/DELETE /api/table/[tableId]/rows — PostgREST string filters', () => { + beforeEach(() => { + vi.clearAllMocks() + mockCheckAccess.mockResolvedValue({ ok: true, table: buildTable() }) + mockUpdateRowsByFilter.mockResolvedValue({ affectedCount: 1, affectedRowIds: ['row_1'] }) + mockDeleteRowsByFilter.mockResolvedValue({ affectedCount: 1, affectedRowIds: ['row_1'] }) + }) + + function callPut(body: Record) { + const req = new NextRequest('http://localhost:3000/api/table/tbl_1/rows', { + method: 'PUT', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(body), + }) + return PUT(req, { params: Promise.resolve({ tableId: 'tbl_1' }) }) + } + + function callDelete(body: Record) { + const req = new NextRequest('http://localhost:3000/api/table/tbl_1/rows', { + method: 'DELETE', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(body), + }) + return DELETE(req, { params: Promise.resolve({ tableId: 'tbl_1' }) }) + } + + it('PUT translates a name-keyed PostgREST string to an id-keyed filter under SESSION auth', async () => { + authAs('session') + const res = await callPut({ + workspaceId: 'workspace-1', + filter: 'Name=eq.Ada', + data: { col_aaa: 'Grace' }, + }) + + expect(res.status).toBe(200) + const args = mockUpdateRowsByFilter.mock.calls[0][1] + expect(args.filter).toEqual({ $and: [{ col_aaa: 'Ada' }] }) + }) + + it('DELETE accepts the PostgREST string and rejects an invalid one with 400', async () => { + authAs('internal_jwt') + const ok = await callDelete({ workspaceId: 'workspace-1', filter: 'Age=gte.30' }) + expect(ok.status).toBe(200) + const args = mockDeleteRowsByFilter.mock.calls[0][1] + expect(args.filter).toEqual({ $and: [{ col_bbb: { $gte: 30 } }] }) + + const bad = await callDelete({ workspaceId: 'workspace-1', filter: 'Age=bogus.1' }) + expect(bad.status).toBe(400) + expect((await bad.json()).error).toMatch(/Unknown filter operator/) + }) +}) diff --git a/apps/sim/app/api/table/[tableId]/rows/route.ts b/apps/sim/app/api/table/[tableId]/rows/route.ts index 190a6e700ab..05c7fd316d0 100644 --- a/apps/sim/app/api/table/[tableId]/rows/route.ts +++ b/apps/sim/app/api/table/[tableId]/rows/route.ts @@ -13,14 +13,7 @@ import { isZodError, validationErrorResponse } from '@/lib/api/server/validation import { type AuthTypeValue, checkSessionOrInternalAuth } from '@/lib/auth/hybrid' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import type { - ColumnDefinition, - Filter, - RowData, - Sort, - TableRowsCursor, - TableSchema, -} from '@/lib/table' +import type { Filter, RowData, Sort, TableRowsCursor, TableSchema } from '@/lib/table' import { batchInsertRows, batchUpdateRows, @@ -32,22 +25,34 @@ import { validateRowData, validateRowSize, } from '@/lib/table' +import { buildIdByName, predicateNamesToIds } from '@/lib/table/column-keys' +import { TableQueryValidationError } from '@/lib/table/errors' import { predicateToFilter } from '@/lib/table/query-builder/converters' import { parsePostgrestFilter } from '@/lib/table/query-builder/postgrest' import { queryRows } from '@/lib/table/rows/service' -import { TableQueryValidationError } from '@/lib/table/sql' -import { rowWireTranslators } from '@/app/api/table/row-wire' +import { type RowWireTranslators, rowWireTranslators } from '@/app/api/table/row-wire' import { accessError, checkAccess, rowWriteErrorResponse } from '@/app/api/table/utils' const logger = createLogger('TableRowsAPI') /** - * Resolves a bulk-op filter to a name-keyed legacy `Filter`. v2 callers send a - * PostgREST string (parsed → predicate → legacy Filter via the shared leaf); - * v1 callers send the legacy object as-is. Caller then runs `wire.filterIn`. + * Resolves a bulk-op filter to a storage-id-keyed legacy `Filter`. PostgREST + * strings are column-NAME-keyed by construction (the parser validates against + * names), so they translate names → ids unconditionally — unlike the legacy + * object form, whose keying follows the caller's wire dialect (`wire.filterIn`: + * ids from the UI, names from workflow tools). */ -function resolveBulkFilter(raw: string | Filter, columns: ColumnDefinition[]): Filter { - return typeof raw === 'string' ? predicateToFilter(parsePostgrestFilter(raw, columns)) : raw +function resolveBulkFilter( + raw: string | Filter, + schema: TableSchema, + wire: RowWireTranslators +): Filter { + if (typeof raw === 'string') { + return predicateToFilter( + predicateNamesToIds(parsePostgrestFilter(raw, schema.columns), buildIdByName(schema)) + ) + } + return wire.filterIn(raw) } interface TableRowsRouteParams { @@ -306,6 +311,7 @@ export const GET = withRouteHandler( totalCount: result.totalCount, limit: result.limit, offset: result.offset, + nextCursor: result.nextCursor, }, }) } catch (error) { @@ -370,11 +376,10 @@ export const PUT = withRouteHandler( const result = await updateRowsByFilter( table, { - filter: wire.filterIn( - resolveBulkFilter( - validated.filter as string | Filter, - (table.schema as TableSchema).columns - ) + filter: resolveBulkFilter( + validated.filter as string | Filter, + table.schema as TableSchema, + wire ), data: patchData, limit: validated.limit, @@ -480,11 +485,10 @@ export const DELETE = withRouteHandler( const result = await deleteRowsByFilter( table, { - filter: wire.filterIn( - resolveBulkFilter( - validated.filter as string | Filter, - (table.schema as TableSchema).columns - ) + filter: resolveBulkFilter( + validated.filter as string | Filter, + table.schema as TableSchema, + wire ), limit: validated.limit, }, diff --git a/apps/sim/app/api/v1/tables/[tableId]/rows/route.ts b/apps/sim/app/api/v1/tables/[tableId]/rows/route.ts index 5f637c3a26a..9b8da7384c9 100644 --- a/apps/sim/app/api/v1/tables/[tableId]/rows/route.ts +++ b/apps/sim/app/api/v1/tables/[tableId]/rows/route.ts @@ -31,8 +31,8 @@ import { validateRowData, validateRowSize, } from '@/lib/table' +import { TableQueryValidationError } from '@/lib/table/errors' import { queryRows } from '@/lib/table/rows/service' -import { TableQueryValidationError } from '@/lib/table/sql' import { accessError, checkAccess, rowWriteErrorResponse } from '@/app/api/table/utils' import { checkRateLimit, @@ -185,6 +185,10 @@ export const GET = withRouteHandler(async (request: NextRequest, context: TableR totalCount: result.totalCount, limit: result.limit, offset: result.offset, + // Non-null when more rows exist; a page may return fewer than `limit` + // rows (byte budget) with more remaining, so page fullness is not a + // termination signal — external pagers should stop on null. + nextCursor: result.nextCursor, }, }) } catch (error) { diff --git a/apps/sim/blocks/blocks/table_v2.test.ts b/apps/sim/blocks/blocks/table_v2.test.ts new file mode 100644 index 00000000000..d1cf64970b3 --- /dev/null +++ b/apps/sim/blocks/blocks/table_v2.test.ts @@ -0,0 +1,95 @@ +/** + * @vitest-environment node + * + * Param-transformer behavior for the v2 Table block: filter/order resolution + * across builder vs editor modes, the required query limit, cursor artifact + * handling, and fail-fast limit parsing. + */ +import { describe, expect, it, vi } from 'vitest' + +vi.mock('@/triggers', () => ({ + getTrigger: vi.fn(() => ({ subBlocks: [] })), +})) + +import { TableV2Block } from '@/blocks/blocks/table_v2' + +function params(input: Record): Record { + return TableV2Block.tools.config?.params?.(input as never) as Record +} + +describe('table_v2 query_rows transformer', () => { + it('keeps limit optional (byte-budget page) but fails fast on a non-numeric one', () => { + const out = params({ operation: 'query_rows', tableId: 't' }) + expect(out.limit).toBeUndefined() + expect(() => params({ operation: 'query_rows', tableId: 't', limit: 'abc' })).toThrow( + /Invalid Limit/ + ) + }) + + it('treats interpolated "null"/"undefined" cursor artifacts as absent', () => { + const out = params({ operation: 'query_rows', tableId: 't', limit: '10', cursor: 'null' }) + expect(out.cursor).toBeUndefined() + const out2 = params({ + operation: 'query_rows', + tableId: 't', + limit: '10', + cursor: 'undefined', + }) + expect(out2.cursor).toBeUndefined() + const out3 = params({ operation: 'query_rows', tableId: 't', limit: '10', cursor: 'tok' }) + expect(out3.cursor).toBe('tok') + }) + + it('serializes builder rules to PostgREST when the builder holds rules', () => { + const out = params({ + operation: 'query_rows', + tableId: 't', + limit: '10', + filterMode: 'builder', + filterBuilder: [ + { id: '1', logicalOperator: 'and', column: 'wins', operator: 'gte', value: '10' }, + ], + sortBuilder: [{ id: '1', column: 'wins', direction: 'desc' }], + filter: 'name=eq.ignored', + }) + expect(out.filter).toBe('wins=gte.10') + expect(out.order).toBe('wins.desc') + }) + + it('falls back to the raw PostgREST string when the builder value is not a non-empty array', () => { + const out = params({ + operation: 'query_rows', + tableId: 't', + limit: '10', + filterMode: 'builder', + filterBuilder: {}, + filter: 'name=eq.test', + }) + expect(out.filter).toBe('name=eq.test') + }) +}) + +describe('table_v2 bulk transformers', () => { + it('fails fast on a non-numeric bulk limit instead of widening to every match', () => { + expect(() => + params({ + operation: 'delete_rows_by_filter', + tableId: 't', + filterMode: 'editor', + filter: 'name=eq.x', + limit: 'abc', + }) + ).toThrow(/Invalid Limit/) + }) + + it('keeps the bulk limit optional', () => { + const out = params({ + operation: 'delete_rows_by_filter', + tableId: 't', + filterMode: 'editor', + filter: 'name=eq.x', + }) + expect(out.limit).toBeUndefined() + expect(out.filter).toBe('name=eq.x') + }) +}) diff --git a/apps/sim/blocks/blocks/table_v2.ts b/apps/sim/blocks/blocks/table_v2.ts index e6972c1a2dd..5857416c1df 100644 --- a/apps/sim/blocks/blocks/table_v2.ts +++ b/apps/sim/blocks/blocks/table_v2.ts @@ -99,6 +99,28 @@ interface ParsedParams { conflictTarget?: string } +/** + * Fail-fast limit parsing for bulk ops: an unparseable limit must never + * silently widen the operation to every matching row. + */ +function parseOptionalLimit(raw: string | undefined): number | undefined { + if (!raw) return undefined + const value = Number.parseInt(raw, 10) + if (Number.isNaN(value)) { + throw new Error(`Invalid Limit "${raw}" — expected a number.`) + } + return value +} + +/** + * `` interpolates the terminating page's `null` as the + * literal string "null" — treat those artifacts as "no cursor". + */ +function resolveCursor(raw: string | undefined): string | undefined { + if (!raw || raw === 'null' || raw === 'undefined') return undefined + return raw +} + const paramTransformers: Record ParsedParams> = { insert_row: (params) => ({ tableId: params.tableId, @@ -127,7 +149,7 @@ const paramTransformers: Record ParsedPara tableId: params.tableId, filter: resolveFilter(params), data: parseJSON(params.data, 'Row Data'), - limit: params.limit ? Number.parseInt(params.limit) : undefined, + limit: parseOptionalLimit(params.limit), }), delete_row: (params) => ({ @@ -138,7 +160,7 @@ const paramTransformers: Record ParsedPara delete_rows_by_filter: (params) => ({ tableId: params.tableId, filter: resolveFilter(params), - limit: params.limit ? Number.parseInt(params.limit) : undefined, + limit: parseOptionalLimit(params.limit), }), get_row: (params) => ({ @@ -154,9 +176,10 @@ const paramTransformers: Record ParsedPara tableId: params.tableId, filter: resolveFilter(params), order: resolveOrder(params), - // No limit → return all matching rows (server enforces a 10MB byte guard). - limit: params.limit ? Number.parseInt(params.limit) : undefined, - cursor: params.cursor || undefined, + // Omitted limit returns the entire result (server fails fast over 5MB); + // with a limit, nextCursor signals when more rows remain. + limit: parseOptionalLimit(params.limit), + cursor: resolveCursor(params.cursor), }), } @@ -169,16 +192,16 @@ export const TableV2Block: BlockConfig = { 'Query Rows filters with a PostgREST querystring — `wins=gte.10&status=in.(active,pending)` (top-level ' + 'params AND; `or=(a.eq.1,b.eq.2)` / `and=(...)` for groups). Operators: eq, neq, gt, gte, lt, lte, in, ' + 'like, ilike, match, imatch, is.null (negate with not., e.g. not.in, not.is.null). Order is `wins.desc,name.asc`. ' + - 'Query Rows returns every matching row by default (no row cap; the server fails fast if the result exceeds 10MB). ' + - 'Set a limit to page — pass the nextCursor from a prior result to fetch the next page.', + 'Query Rows returns every matching row when Limit is omitted (fails if the result exceeds 5MB — add a filter ' + + 'or a Limit). With a Limit, responses page: a non-null nextCursor means more rows exist — pass it back as the cursor.', bestPractices: ` - To fetch specific rows, use Query Rows with a PostgREST filter (e.g. slack_user_id=in.(U1,U2)) — do NOT read every row and filter downstream with a Condition block. - Use "Get Row by ID" only when you have the row's id; otherwise filter with the querystring. - Top-level params AND together; use or=(...) / and=(...) for explicit or nested logic. - Example: players who won ≥10 and are active → wins=gte.10&status=eq.active. - like/ilike use * as the wildcard (e.g. name=ilike.*jo*); match/imatch are regex. -- Query Rows returns ALL matching rows by default — leave Limit empty unless you want a bounded page. -- To page through a bounded result, set a Limit and pass the previous response's nextCursor back as the cursor; omit it for the first page. +- Omit Limit to get the entire matching result in one response — the query fails with a clear error if it exceeds 5MB (narrow with a filter or set a Limit). +- With a Limit, pages can end at the Limit or the 5MB byte budget, whichever comes first — pass nextCursor back as the cursor and loop until it is null; never infer completion from page size. - Columns are scalar (string/number/boolean/date) or opaque json — there are no array columns, so cs/cd/ov (containment/overlap) are unsupported; for substring use ilike.*x*.`, docsLink: 'https://docs.simstudio.ai/tools/table', category: 'blocks', @@ -338,11 +361,10 @@ Return ONLY the rows array:`, title: 'Filter Conditions', type: 'filter-builder', paramVisibility: 'user-only', - required: { - field: 'operation', - value: ['update_rows_by_filter', 'delete_rows_by_filter'], - and: { field: 'filterMode', value: 'builder' }, - }, + // Deliberately NOT `required`: an agent can satisfy the bulk ops with the + // `filter` string while filterMode sits at its 'builder' default, and the + // serializer hard-blocks execution on missing required fields. The service + // itself fails closed when no filter resolves ("Filter is required..."). condition: { field: 'operation', value: ['query_rows', 'update_rows_by_filter', 'delete_rows_by_filter'], @@ -422,7 +444,7 @@ Return ONLY the querystring:`, id: 'limit', title: 'Limit', type: 'short-input', - placeholder: 'Leave empty for all rows', + placeholder: 'Leave empty for all rows (fails over 5MB)', condition: { field: 'operation', value: ['query_rows', 'update_rows_by_filter', 'delete_rows_by_filter'], @@ -495,7 +517,8 @@ Return ONLY the querystring:`, order: { type: 'string', description: 'PostgREST order, e.g. wins.desc,name.asc' }, limit: { type: 'number', - description: 'Row limit; leave empty to return all matching rows (10MB byte cap)', + description: + 'Page row limit (optional — omit to return the entire result, which fails if it exceeds 5MB); optional cap for bulk update/delete', }, cursor: { type: 'string', description: 'Opaque pagination cursor from a prior query response' }, conflictColumn: { diff --git a/apps/sim/lib/api/contracts/tables.ts b/apps/sim/lib/api/contracts/tables.ts index 65a1d218617..5e092a48f50 100644 --- a/apps/sim/lib/api/contracts/tables.ts +++ b/apps/sim/lib/api/contracts/tables.ts @@ -549,13 +549,16 @@ export const listTableRowsContract = defineRouteContract({ totalCount: z.number().nullable(), limit: z.number(), offset: z.number(), + /** Non-null when more rows exist — a page may be cut by the byte budget + * before reaching `limit`, so page fullness is not a termination signal. */ + nextCursor: z.string().nullable(), }) ), }, }) /** - * v2 query surface: structured predicate grammar + opaque cursor pagination. + * v2 query surface: PostgREST filter/order strings + opaque cursor pagination. * No `offset` (the cursor encodes paging state) and no after/sort refine (the * cursor carries the sort context). `filter`/`order` are PostgREST strings, * parsed server-side. POST so the (potentially long) filter rides the body. @@ -564,8 +567,9 @@ export const queryTableRowsV2BodySchema = z.object({ workspaceId: z.string().min(1, 'Workspace ID is required'), filter: postgrestFilterSchema.optional(), order: postgrestOrderSchema.optional(), - // Omitted limit returns every matching row (bounded by the server's byte - // guard, not a row cap). An explicit limit is honored as-is for paging. + // Omitted limit returns the ENTIRE matching result, failing fast (400) when + // it exceeds the response byte budget. An explicit limit caps the page row + // count; the byte budget may still end a page early with nextCursor set. limit: z.preprocess( (value) => (value === null || value === undefined || value === '' ? undefined : Number(value)), z diff --git a/apps/sim/lib/copilot/generated/tool-catalog-v1.ts b/apps/sim/lib/copilot/generated/tool-catalog-v1.ts index 55e0399c7d2..971ce14d68c 100644 --- a/apps/sim/lib/copilot/generated/tool-catalog-v1.ts +++ b/apps/sim/lib/copilot/generated/tool-catalog-v1.ts @@ -3863,9 +3863,9 @@ export const UserTable: ToolCatalogEntry = { 'Canonical workspace file VFS path for create_from_file/import_file, e.g. files/{path}/{name}.', }, filter: { - type: 'object', + type: 'string', description: - 'Predicate-tree filter for query_rows, update_rows_by_filter, delete_rows_by_filter. A filter is { all: [...] } (AND) or { any: [...] } (OR) wrapping leaf nodes { field, op, value }; groups nest. Ops (bare, no $): eq, ne, gt, gte, lt, lte, in, nin, contains, ncontains, startsWith, endsWith, isEmpty, isNotEmpty. eq/ne/in/nin are case-sensitive; contains/ncontains/startsWith/endsWith are case-insensitive; isEmpty/isNotEmpty take no value; in/nin take an array. Example: { all: [{ field: "age", op: "gte", value: 18 }, { field: "status", op: "eq", value: "pending" }] }.', + 'PostgREST querystring filter for query_rows, update_rows_by_filter, delete_rows_by_filter. A filter is a querystring fragment of field=op.value params joined by & (AND). Ops: eq, neq, gt, gte, lt, lte, in, like, ilike (use * as the wildcard), match, imatch (regex), is.null. Lists: in.(a,b,c) — never empty. Negate with a not. prefix (not.eq, not.in, not.is.null, not.like, not.ilike). Groups: or=(status.eq.active,status.eq.pending) and and=(...) — never empty; nest groups with the function form or(...)/and(...). Use ilike.*x* for case-insensitive substring match. Examples: "status=eq.active"; "wins=gte.18&status=eq.pending"; "or=(status.eq.active,status.eq.pending)"; "name=ilike.*jo*"; "slack_user_id=in.(U1,U2)".', }, groupId: { type: 'string', @@ -3900,7 +3900,7 @@ export const UserTable: ToolCatalogEntry = { limit: { type: 'number', description: - 'Maximum rows to return or affect (optional, default 100). Omit on update_rows_by_filter / delete_rows_by_filter to act on every match.', + 'Maximum rows per page for query_rows (optional). Omit to fetch the ENTIRE matching result in one response — the call fails if the result exceeds the 5MB budget (narrow with a filter or set a limit). With a limit, a page may end early at the byte budget with more remaining; a non-null nextCursor in the result means more rows exist (continue with offset). On update_rows_by_filter / delete_rows_by_filter, caps affected rows; omit to act on every match.', }, mapping: { type: 'object', @@ -3953,7 +3953,13 @@ export const UserTable: ToolCatalogEntry = { }, offset: { type: 'number', - description: 'Number of rows to skip (optional for query_rows, default 0)', + description: + 'Number of rows to skip for query_rows (optional, default 0; works with or without limit). Use the offset from the previous result\'s "more available" message to fetch the next page.', + }, + order: { + type: 'string', + description: + 'PostgREST order string for query_rows (optional). Comma-separated column.dir terms where dir is asc or desc, e.g. "wins.desc,name.asc".', }, outputColumnNames: { type: 'object', @@ -4045,11 +4051,6 @@ export const UserTable: ToolCatalogEntry = { "Cancellation scope for cancel_table_runs. 'all' cancels in-flight runs across the whole table; 'row' cancels only the row identified by rowId.", enum: ['all', 'row'], }, - sort: { - type: 'object', - description: - "Sort specification as { field: 'asc' | 'desc' } (optional for query_rows)", - }, tableId: { type: 'string', description: diff --git a/apps/sim/lib/copilot/generated/tool-schemas-v1.ts b/apps/sim/lib/copilot/generated/tool-schemas-v1.ts index 987646a5c2a..e1eed73e18a 100644 --- a/apps/sim/lib/copilot/generated/tool-schemas-v1.ts +++ b/apps/sim/lib/copilot/generated/tool-schemas-v1.ts @@ -3620,9 +3620,9 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { 'Canonical workspace file VFS path for create_from_file/import_file, e.g. files/{path}/{name}.', }, filter: { - type: 'object', + type: 'string', description: - 'Predicate-tree filter for query_rows, update_rows_by_filter, delete_rows_by_filter. A filter is { all: [...] } (AND) or { any: [...] } (OR) wrapping leaf nodes { field, op, value }; groups nest. Ops (bare, no $): eq, ne, gt, gte, lt, lte, in, nin, contains, ncontains, startsWith, endsWith, isEmpty, isNotEmpty. eq/ne/in/nin are case-sensitive; contains/ncontains/startsWith/endsWith are case-insensitive; isEmpty/isNotEmpty take no value; in/nin take an array. Example: { all: [{ field: "age", op: "gte", value: 18 }, { field: "status", op: "eq", value: "pending" }] }.', + 'PostgREST querystring filter for query_rows, update_rows_by_filter, delete_rows_by_filter. A filter is a querystring fragment of field=op.value params joined by & (AND). Ops: eq, neq, gt, gte, lt, lte, in, like, ilike (use * as the wildcard), match, imatch (regex), is.null. Lists: in.(a,b,c) — never empty. Negate with a not. prefix (not.eq, not.in, not.is.null, not.like, not.ilike). Groups: or=(status.eq.active,status.eq.pending) and and=(...) — never empty; nest groups with the function form or(...)/and(...). Use ilike.*x* for case-insensitive substring match. Examples: "status=eq.active"; "wins=gte.18&status=eq.pending"; "or=(status.eq.active,status.eq.pending)"; "name=ilike.*jo*"; "slack_user_id=in.(U1,U2)".', }, groupId: { type: 'string', @@ -3659,7 +3659,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { limit: { type: 'number', description: - 'Maximum rows to return or affect (optional, default 100). Omit on update_rows_by_filter / delete_rows_by_filter to act on every match.', + 'Maximum rows per page for query_rows (optional). Omit to fetch the ENTIRE matching result in one response — the call fails if the result exceeds the 5MB budget (narrow with a filter or set a limit). With a limit, a page may end early at the byte budget with more remaining; a non-null nextCursor in the result means more rows exist (continue with offset). On update_rows_by_filter / delete_rows_by_filter, caps affected rows; omit to act on every match.', }, mapping: { type: 'object', @@ -3718,7 +3718,13 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, offset: { type: 'number', - description: 'Number of rows to skip (optional for query_rows, default 0)', + description: + 'Number of rows to skip for query_rows (optional, default 0; works with or without limit). Use the offset from the previous result\'s "more available" message to fetch the next page.', + }, + order: { + type: 'string', + description: + 'PostgREST order string for query_rows (optional). Comma-separated column.dir terms where dir is asc or desc, e.g. "wins.desc,name.asc".', }, outputColumnNames: { type: 'object', @@ -3820,11 +3826,6 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { "Cancellation scope for cancel_table_runs. 'all' cancels in-flight runs across the whole table; 'row' cancels only the row identified by rowId.", enum: ['all', 'row'], }, - sort: { - type: 'object', - description: - "Sort specification as { field: 'asc' | 'desc' } (optional for query_rows)", - }, tableId: { type: 'string', description: diff --git a/apps/sim/lib/copilot/tools/server/table/user-table.ts b/apps/sim/lib/copilot/tools/server/table/user-table.ts index a85e687433a..1a9359cba53 100644 --- a/apps/sim/lib/copilot/tools/server/table/user-table.ts +++ b/apps/sim/lib/copilot/tools/server/table/user-table.ts @@ -615,9 +615,11 @@ export const userTableServerTool: BaseServerTool const sort = orderSpec?.length ? Object.fromEntries(orderSpec.map((s) => [s.field, s.direction])) : undefined - // No limit returns every matching row (bounded by the server's 10MB byte - // guard, which fails fast rather than truncating). An explicit limit is - // honored as-is so the model can still page with `offset` when it wants to. + // No limit returns the ENTIRE matching result, failing fast once the + // 5MB byte budget is exceeded (caught below → structured tool error + // the model can react to by adding a filter or a limit). An explicit + // limit pages; byte-cut pages set nextCursor and the message says how + // to continue with `offset`. const result = await queryRows( table, { @@ -630,9 +632,14 @@ export const userTableServerTool: BaseServerTool requestId ) + // nextCursor covers both cut kinds (explicit limit or the 5MB byte + // budget) — either way the truthful signal is "more rows exist". + const message = result.nextCursor + ? `Returned ${result.rows.length} of ${result.totalCount} rows (more available — pass offset=${result.offset + result.rows.length} to continue)` + : `Returned ${result.rows.length} of ${result.totalCount} rows` return { success: true, - message: `Returned ${result.rows.length} of ${result.totalCount} rows`, + message, data: { ...result, rows: result.rows.map((r) => ({ ...r, data: rowDataIdToName(r.data, nameById) })), diff --git a/apps/sim/lib/core/config/env.ts b/apps/sim/lib/core/config/env.ts index 1c03ce965c8..fe224161b03 100644 --- a/apps/sim/lib/core/config/env.ts +++ b/apps/sim/lib/core/config/env.ts @@ -96,7 +96,6 @@ export const env = createEnv({ ENTERPRISE_TABLES_LIMIT: z.number().optional(), // Max user tables per workspace on enterprise tier (default: 10000) ENTERPRISE_TABLE_ROWS_LIMIT: z.number().optional(), // Max rows per table on enterprise tier (default: 1000000) TABLE_MAX_ROW_SIZE_BYTES: z.number().optional(), // Max serialized size in bytes of a single user-table row (default: 409600) - TABLE_MAX_PAGE_BYTES: z.number().optional(), // Dev-preview: byte budget per row-page read; pages cut early past it (unset = disabled) // Credit-tier Stripe prices (monthly) STRIPE_PRICE_TIER_25_MO: z.string().min(1).optional(), // Pro: $25/mo (6,000 credits) diff --git a/apps/sim/lib/table/__tests__/paging.test.ts b/apps/sim/lib/table/__tests__/paging.test.ts deleted file mode 100644 index 22a4a55bcf5..00000000000 --- a/apps/sim/lib/table/__tests__/paging.test.ts +++ /dev/null @@ -1,33 +0,0 @@ -/** - * @vitest-environment node - */ -import { describe, expect, it } from 'vitest' -import { trimRowsToByteBudget } from '@/lib/table/rows/paging' - -function row(id: string, bytes: number) { - // {"b":"aaa…"} serializes to bytes + 8 chars of envelope. - return { id, data: { b: 'a'.repeat(Math.max(0, bytes - 8)) } } -} - -describe('trimRowsToByteBudget', () => { - it('returns all rows when they fit the budget', () => { - const rows = [row('r1', 100), row('r2', 100)] - expect(trimRowsToByteBudget(rows, 1000)).toBe(rows) - }) - - it('keeps the longest prefix within the budget', () => { - const rows = [row('r1', 400), row('r2', 400), row('r3', 400)] - const kept = trimRowsToByteBudget(rows, 900) - expect(kept.map((r) => r.id)).toEqual(['r1', 'r2']) - }) - - it('always keeps the first row even when it alone exceeds the budget', () => { - const rows = [row('r1', 5000), row('r2', 100)] - const kept = trimRowsToByteBudget(rows, 1000) - expect(kept.map((r) => r.id)).toEqual(['r1']) - }) - - it('returns empty input unchanged', () => { - expect(trimRowsToByteBudget([], 1000)).toEqual([]) - }) -}) diff --git a/apps/sim/lib/table/__tests__/service-filter-threading.test.ts b/apps/sim/lib/table/__tests__/service-filter-threading.test.ts index 83281337dd4..90ed3283043 100644 --- a/apps/sim/lib/table/__tests__/service-filter-threading.test.ts +++ b/apps/sim/lib/table/__tests__/service-filter-threading.test.ts @@ -11,11 +11,20 @@ import { dbChainMock, dbChainMockFns, resetDbChainMock } from '@sim/testing' import { sql } from 'drizzle-orm' import { beforeEach, describe, expect, it, vi } from 'vitest' import { TABLE_LIMITS } from '@/lib/table/constants' +import { decodeCursor } from '@/lib/table/rows/cursor' import { buildFilterClause, buildSortClause } from '@/lib/table/sql' import type { ColumnDefinition, TableDefinition } from '@/lib/table/types' +const { mockIsFeatureEnabled } = vi.hoisted(() => ({ + mockIsFeatureEnabled: vi.fn(async () => true), +})) + vi.mock('@sim/db', () => dbChainMock) +vi.mock('@/lib/core/config/feature-flags', () => ({ + isFeatureEnabled: mockIsFeatureEnabled, +})) + vi.mock('@/lib/table/sql', () => ({ buildFilterClause: vi.fn(() => sql`true`), buildSortClause: vi.fn(() => sql`true`), @@ -156,32 +165,186 @@ describe('bulk update/delete limited-subset ordering', () => { }) }) -describe('queryRows byte guard', () => { +describe('queryRows byte budget', () => { beforeEach(() => { vi.clearAllMocks() resetDbChainMock() }) - it('returns an empty page without tripping the guard', async () => { + const row = (i: number, blobBytes: number) => ({ + id: `row_${i}`, + data: { blob: 'x'.repeat(blobBytes) }, + position: i, + orderKey: `a${i}`, + createdAt: new Date('2024-01-01'), + updatedAt: new Date('2024-01-01'), + }) + + it('returns an empty page with a null cursor', async () => { const result = await queryRows(TABLE, { includeTotal: false, withExecutions: false }, 'req-1') expect(result.rows).toEqual([]) + expect(result.nextCursor).toBeNull() }) - it('fails fast when the result exceeds the byte budget', async () => { - const oversized = 'x'.repeat(TABLE_LIMITS.MAX_QUERY_RESULT_BYTES + 1) - dbChainMockFns.orderBy.mockResolvedValueOnce([ - { - id: 'row_1', - data: { blob: oversized }, - position: 0, - orderKey: 'a0', - createdAt: new Date('2024-01-01'), - updatedAt: new Date('2024-01-01'), - }, - ]) + it('fails fast when an UNBOUNDED query exceeds the byte budget (no partial page)', async () => { + const perRow = Math.floor(TABLE_LIMITS.MAX_QUERY_RESULT_BYTES * 0.6) + // First .limit() call is pendingDeleteMask's job probe; the second is the drain batch. + dbChainMockFns.limit.mockResolvedValueOnce([]) + dbChainMockFns.limit.mockResolvedValueOnce([row(1, perRow), row(2, perRow)]) await expect( queryRows(TABLE, { includeTotal: false, withExecutions: false }, 'req-1') - ).rejects.toThrow(/10MB/) + ).rejects.toThrow(/exceeds the 5MB limit/) + }) + + it('returns the entire result when an unbounded query fits the budget', async () => { + dbChainMockFns.limit.mockResolvedValueOnce([]) + dbChainMockFns.limit.mockResolvedValueOnce([row(1, 8), row(2, 8)]) + + const result = await queryRows(TABLE, { includeTotal: false, withExecutions: false }, 'req-1') + + expect(result.rows).toHaveLength(2) + expect(result.nextCursor).toBeNull() + }) + + it('byte-cuts a BOUNDED page and returns a resume cursor instead of throwing', async () => { + const perRow = Math.floor(TABLE_LIMITS.MAX_QUERY_RESULT_BYTES * 0.6) + dbChainMockFns.limit.mockResolvedValueOnce([]) + dbChainMockFns.limit.mockResolvedValueOnce([row(1, perRow), row(2, perRow)]) + + const result = await queryRows( + TABLE, + { limit: 5, includeTotal: false, withExecutions: false }, + 'req-1' + ) + + expect(result.rows).toHaveLength(1) + expect(result.rows[0].id).toBe('row_1') + // Fractional ordering is on (global flag mock) → keyset cursor resuming + // after the last included row. + expect(decodeCursor(result.nextCursor as string)).toEqual({ + after: { orderKey: 'a1', id: 'row_1' }, + }) + }) + + it('returns a single over-budget row alone on a bounded page', async () => { + dbChainMockFns.limit.mockResolvedValueOnce([]) + dbChainMockFns.limit.mockResolvedValueOnce([ + row(1, TABLE_LIMITS.MAX_QUERY_RESULT_BYTES + 1024), + row(2, 8), + ]) + + const result = await queryRows( + TABLE, + { limit: 2, includeTotal: false, withExecutions: false }, + 'req-1' + ) + + expect(result.rows).toHaveLength(1) + expect(result.rows[0].id).toBe('row_1') + expect(result.nextCursor).not.toBeNull() + }) + + // First-batch worst-case bytes stay within ~4× the budget at the max row size. + const FIRST_BATCH_CAP = Math.max( + 1, + Math.floor((4 * TABLE_LIMITS.MAX_QUERY_RESULT_BYTES) / TABLE_LIMITS.MAX_ROW_SIZE_BYTES) + ) + + it('caps the first unbounded batch and asks limit+1 when a limit is set', async () => { + await queryRows(TABLE, { includeTotal: false, withExecutions: false }, 'req-1') + // Call 1 = pendingDeleteMask probe (limit 1); call 2 = first drain batch. + expect(dbChainMockFns.limit).toHaveBeenNthCalledWith(2, FIRST_BATCH_CAP + 1) + + vi.clearAllMocks() + resetDbChainMock() + await queryRows(TABLE, { limit: 5, includeTotal: false, withExecutions: false }, 'req-1') + expect(dbChainMockFns.limit).toHaveBeenNthCalledWith(2, 6) + }) + + it('limit-cut with a witness row emits a cursor; exact-boundary page does not', async () => { + dbChainMockFns.limit.mockResolvedValueOnce([]) + dbChainMockFns.limit.mockResolvedValueOnce([row(1, 8), row(2, 8), row(3, 8)]) + const cut = await queryRows( + TABLE, + { limit: 2, includeTotal: false, withExecutions: false }, + 'req-1' + ) + expect(cut.rows.map((r) => r.id)).toEqual(['row_1', 'row_2']) + expect(cut.nextCursor).not.toBeNull() + + vi.clearAllMocks() + resetDbChainMock() + dbChainMockFns.limit.mockResolvedValueOnce([]) + dbChainMockFns.limit.mockResolvedValueOnce([row(1, 8), row(2, 8)]) + const exact = await queryRows( + TABLE, + { limit: 2, includeTotal: false, withExecutions: false }, + 'req-1' + ) + expect(exact.rows).toHaveLength(2) + // The +1 peek proved the data ended exactly at the limit — no dead cursor. + expect(exact.nextCursor).toBeNull() + }) + + it('drains across multiple batches and grows the batch size adaptively', async () => { + const firstAsk = FIRST_BATCH_CAP + 1 + const batch1 = Array.from({ length: firstAsk }, (_, i) => row(i, 8)) + dbChainMockFns.limit.mockResolvedValueOnce([]) + dbChainMockFns.limit.mockResolvedValueOnce(batch1) + dbChainMockFns.limit.mockResolvedValueOnce([row(firstAsk, 8)]) + + const result = await queryRows(TABLE, { includeTotal: false, withExecutions: false }, 'req-1') + + expect(result.rows).toHaveLength(firstAsk + 1) + expect(result.nextCursor).toBeNull() + const firstBatchAsk = (dbChainMockFns.limit.mock.calls[1] ?? [])[0] as number + const secondBatchAsk = (dbChainMockFns.limit.mock.calls[2] ?? [])[0] as number + expect(secondBatchAsk).toBeGreaterThan(firstBatchAsk) + }) + + it('sort path advances by offset and emits an offset cursor with inbound offset applied', async () => { + const perRow = Math.floor(TABLE_LIMITS.MAX_QUERY_RESULT_BYTES * 0.6) + dbChainMockFns.limit.mockResolvedValueOnce([]) + // Sorted batch with inbound offset 40 terminates at .offset(40). + dbChainMockFns.offset.mockResolvedValueOnce([row(1, perRow), row(2, perRow)]) + + const result = await queryRows( + TABLE, + { sort: { name: 'asc' }, offset: 40, limit: 5, includeTotal: false, withExecutions: false }, + 'req-1' + ) + + expect(dbChainMockFns.offset).toHaveBeenCalledWith(40) + expect(result.rows).toHaveLength(1) + expect(decodeCursor(result.nextCursor as string)).toEqual({ offset: 41 }) + }) + + it('emits a compound cursor when rows past the inbound anchor are unkeyed', async () => { + const unkeyedRow = (i: number, blobBytes: number) => ({ ...row(i, blobBytes), orderKey: null }) + const perRow = Math.floor(TABLE_LIMITS.MAX_QUERY_RESULT_BYTES * 0.4) + dbChainMockFns.limit.mockResolvedValueOnce([]) + dbChainMockFns.limit.mockResolvedValueOnce([ + unkeyedRow(6, perRow), + unkeyedRow(7, perRow), + unkeyedRow(8, perRow), + ]) + + const result = await queryRows( + TABLE, + { + after: { orderKey: 'a5', id: 'row_5' }, + limit: 10, + includeTotal: false, + withExecutions: false, + }, + 'req-1' + ) + + expect(result.rows.map((r) => r.id)).toEqual(['row_6', 'row_7']) + expect(decodeCursor(result.nextCursor as string)).toEqual({ + after: { orderKey: 'a5', id: 'row_5' }, + offset: 2, + }) }) }) diff --git a/apps/sim/lib/table/constants.ts b/apps/sim/lib/table/constants.ts index 7530f952886..01b48ec6df7 100644 --- a/apps/sim/lib/table/constants.ts +++ b/apps/sim/lib/table/constants.ts @@ -16,12 +16,16 @@ export const TABLE_LIMITS = { DEFAULT_QUERY_LIMIT: 100, MAX_QUERY_LIMIT: 1000, /** - * Byte ceiling for a single v2 query result. The v2 query returns every - * matching row by default (no row cap); this guard fails the query fast - * instead of streaming an unbounded payload. Measured against the serialized - * row data, not the full HTTP envelope. + * Byte budget for one query response. `queryRows` drains rows in bounded + * batches; an UNBOUNDED query (no limit) that outgrows the budget fails fast + * (the whole result was promised — a partial page would be silent truncation), + * while a BOUNDED page cuts early and returns the partial list with + * `nextCursor` set. At least one row is always returned on a bounded page. + * Measured against serialized row `data` only, not the full HTTP envelope. */ - MAX_QUERY_RESULT_BYTES: 10 * 1024 * 1024, // 10MB + MAX_QUERY_RESULT_BYTES: 5 * 1024 * 1024, // 5MB + /** Hard row cap per internal fetch batch inside queryRows' bounded drain loop. */ + QUERY_BATCH_MAX_ROWS: 10000, /** Batch size for bulk update operations */ UPDATE_BATCH_SIZE: 100, /** Batch size for bulk delete operations */ @@ -67,17 +71,6 @@ export const DEFAULT_TABLE_PLAN_LIMITS = { }, } as const -/** - * Byte budget for one page of row reads, or null when disabled (the default). - * Dev-preview of the byte-bounded pagination follow-up: set `TABLE_MAX_PAGE_BYTES` - * to cut pages early once their serialized row data exceeds the budget. The - * production version moves the cut into SQL — see the pagination-hardening plan. - */ -export function getMaxPageBytes(): number | null { - const value = envNumber(env.TABLE_MAX_PAGE_BYTES, 0, { min: 0, integer: true }) - return value > 0 ? value : null -} - /** * Maximum serialized size in bytes of a single row. Defaults to * `TABLE_LIMITS.MAX_ROW_SIZE_BYTES`; overridable via the diff --git a/apps/sim/lib/table/errors.ts b/apps/sim/lib/table/errors.ts new file mode 100644 index 00000000000..01745ee88af --- /dev/null +++ b/apps/sim/lib/table/errors.ts @@ -0,0 +1,14 @@ +/** + * Error thrown when caller-supplied filter or sort input is malformed. + * Routes should map this to HTTP 400 with the message preserved. + * + * Lives outside `sql.ts` so client-bundled modules (the block definitions pull + * in the PostgREST serializers) can reference it without dragging drizzle-orm + * into the browser chunk. + */ +export class TableQueryValidationError extends Error { + constructor(message: string) { + super(message) + this.name = 'TableQueryValidationError' + } +} diff --git a/apps/sim/lib/table/index.ts b/apps/sim/lib/table/index.ts index c79a13f182c..9520e998a63 100644 --- a/apps/sim/lib/table/index.ts +++ b/apps/sim/lib/table/index.ts @@ -9,6 +9,7 @@ export * from '@/lib/table/billing' export * from '@/lib/table/column-keys' export * from '@/lib/table/columns/service' export * from '@/lib/table/constants' +export * from '@/lib/table/errors' export * from '@/lib/table/import' export * from '@/lib/table/import-data' export * from '@/lib/table/jobs/service' diff --git a/apps/sim/lib/table/query-builder/__tests__/converters.test.ts b/apps/sim/lib/table/query-builder/__tests__/converters.test.ts index 78b1380a109..5efc1e5c5f7 100644 --- a/apps/sim/lib/table/query-builder/__tests__/converters.test.ts +++ b/apps/sim/lib/table/query-builder/__tests__/converters.test.ts @@ -263,8 +263,9 @@ describe('filterRulesToPostgrest', () => { expect( filterRulesToPostgrest([rule({ column: 'name', operator: 'contains', value: 'jo' })]) ).toBe('name=ilike.*jo*') + // isEmpty keeps its null-OR-empty-string semantics through the string form. expect(filterRulesToPostgrest([rule({ column: 'name', operator: 'isEmpty' })])).toBe( - 'name=is.null' + 'or=(name.is.null,name.eq."")' ) }) diff --git a/apps/sim/lib/table/query-builder/__tests__/postgrest.test.ts b/apps/sim/lib/table/query-builder/__tests__/postgrest.test.ts index df414e91935..2d536565079 100644 --- a/apps/sim/lib/table/query-builder/__tests__/postgrest.test.ts +++ b/apps/sim/lib/table/query-builder/__tests__/postgrest.test.ts @@ -113,7 +113,24 @@ describe('parsePostgrestFilter', () => { it('rejects unknown ops, bad fields, and unsupported negation', () => { expect(() => parsePostgrestFilter('wins=bogus.1', COLS)).toThrow(/Unknown filter operator/) expect(() => parsePostgrestFilter('bad name=eq.1', COLS)).toThrow(/Invalid filter column/) - expect(() => parsePostgrestFilter('name=not.ilike.x', COLS)).toThrow(/not supported/) + expect(() => parsePostgrestFilter('name=not.match.x', COLS)).toThrow(/not supported/) + }) + + it('parses not.like / not.ilike into the negated pattern ops', () => { + expect(parsePostgrestFilter('name=not.ilike.*jo*', COLS)).toEqual({ + all: [{ field: 'name', op: 'nilike', value: '*jo*' }], + }) + expect(parsePostgrestFilter('name=not.like.jo*', COLS)).toEqual({ + all: [{ field: 'name', op: 'nlike', value: 'jo*' }], + }) + }) + + it('rejects silent-widening inputs: empty groups, empty in-lists, non-finite numbers', () => { + expect(() => parsePostgrestFilter('wins=gte.1&or=()', COLS)).toThrow(/Empty or=\(\) group/) + expect(() => parsePostgrestFilter('and=()', COLS)).toThrow(/Empty and=\(\) group/) + expect(() => parsePostgrestFilter('status=in.()', COLS)).toThrow(/Empty in\.\(\) list/) + expect(() => parsePostgrestFilter('wins=eq.Infinity', COLS)).toThrow(/Expected a number/) + expect(() => parsePostgrestFilter('wins=eq.1e400', COLS)).toThrow(/Expected a number/) }) it('rejects empty / malformed input', () => { @@ -173,7 +190,64 @@ describe('predicateToPostgrest round-trips', () => { it('serializes builder-only ops onto PostgREST forms', () => { expect(leaf('contains', 'foo')).toBe('name=ilike.*foo*') expect(leaf('startsWith', 'foo')).toBe('name=ilike.foo*') - expect(leaf('isEmpty')).toBe('name=is.null') + expect(leaf('ncontains', 'foo')).toBe('name=not.ilike.*foo*') + // Emptiness desugars to groups preserving null-OR-empty-string semantics. + expect(leaf('isEmpty')).toBe('or=(name.is.null,name.eq."")') + expect(leaf('isNotEmpty')).toBe('and=(name.not.is.null,name.neq."")') + }) + + it('round-trips substring ops with reserved characters (whole-pattern quoting)', () => { + const str = leaf('contains', 'example.com') + expect(str).toBe('name=ilike."*example.com*"') + expect(parsePostgrestFilter(str, COLS)).toEqual({ + all: [{ field: 'name', op: 'ilike', value: '*example.com*' }], + }) + + const ncontains = leaf('ncontains', 'a,b(c)=d') + expect(parsePostgrestFilter(ncontains, COLS)).toEqual({ + all: [{ field: 'name', op: 'nilike', value: '*a,b(c)=d*' }], + }) + }) + + it('round-trips embedded quotes and backslashes through escaping', () => { + const p: TablePredicate = { + all: [{ field: 'name', op: 'eq', value: 'She said "hi"' }], + } + expect(parsePostgrestFilter(predicateToPostgrest(p), COLS)).toEqual(p) + + const list: TablePredicate = { + all: [{ field: 'status', op: 'in', value: ['a"b', 'c\\d'] }], + } + expect(parsePostgrestFilter(predicateToPostgrest(list), COLS)).toEqual(list) + }) + + it('round-trips emptiness ops to their semantically-equal group form', () => { + expect(parsePostgrestFilter(leaf('isEmpty'), COLS)).toEqual({ + all: [ + { + any: [ + { field: 'name', op: 'isNull' }, + { field: 'name', op: 'eq', value: '' }, + ], + }, + ], + }) + expect(parsePostgrestFilter(leaf('isNotEmpty'), COLS)).toEqual({ + all: [ + { + all: [ + { field: 'name', op: 'isNotNull' }, + { field: 'name', op: 'ne', value: '' }, + ], + }, + ], + }) + }) + + it('rejects uppercase/typo sort directions instead of silently sorting asc', () => { + expect(() => parsePostgrestOrder('wins.DESC', COLS)).toThrow(/Unknown sort direction/) + expect(() => parsePostgrestOrder('wins.dsc', COLS)).toThrow(/Unknown sort direction/) + expect(() => parsePostgrestOrder('wins.desc.extra', COLS)).toThrow(/Malformed sort/) }) it('order spec round-trips', () => { diff --git a/apps/sim/lib/table/query-builder/postgrest.ts b/apps/sim/lib/table/query-builder/postgrest.ts index fb0cf2e9d55..83ae63cbe41 100644 --- a/apps/sim/lib/table/query-builder/postgrest.ts +++ b/apps/sim/lib/table/query-builder/postgrest.ts @@ -11,15 +11,21 @@ * * Supported (PostgREST token → FilterOp): eq, neq→ne, gt, gte, lt, lte, * in, like, ilike, match, imatch, is.null→isNull. Negation `not.` is supported - * for `eq`→ne, `in`→nin, and `is.null`→isNotNull. Logic: top-level `&` (AND), - * `and=(...)`, `or=(...)`, and nested `and(...)`/`or(...)` inside groups. + * for `eq`→ne, `in`→nin, `is.null`→isNotNull, `like`→nlike, and `ilike`→nilike. + * Logic: top-level `&` (AND), `and=(...)`, `or=(...)`, and nested + * `and(...)`/`or(...)` inside groups. + * + * Values containing reserved characters are double-quoted; a literal `"` or `\` + * inside a quoted value is backslash-escaped (`\"`, `\\`). The builder-only + * emptiness ops have no single PostgREST form and serialize to desugared groups: + * isEmpty → `or=(f.is.null,f.eq."")`, isNotEmpty → `and=(f.not.is.null,f.neq."")`. * * Unsupported (clear error): cs, cd, ov, fts/plfts/phfts/wfts, sl/sr/nxr/nxl/adj - * (no array/range/full-text columns), and `not.` outside eq/in/is.null. + * (no array/range/full-text columns), and `not.` outside eq/in/is.null/like/ilike. */ import { NAME_PATTERN } from '@/lib/table/constants' -import { TableQueryValidationError } from '@/lib/table/sql' +import { TableQueryValidationError } from '@/lib/table/errors' import type { ColumnDefinition, FilterOp, @@ -36,11 +42,15 @@ type ColumnType = ColumnDefinition['type'] const MAX_FILTER_LENGTH = 4096 /** Ops whose value is a text pattern — never coerced to number/boolean. */ -const TEXT_OPS = new Set(['like', 'ilike', 'match', 'imatch']) +const TEXT_OPS = new Set(['like', 'ilike', 'nlike', 'nilike', 'match', 'imatch']) /* ------------------------------- parsing ------------------------------- */ -/** Splits on a delimiter at depth 0, respecting `(...)` nesting and `"..."` quotes. */ +/** + * Splits on a delimiter at depth 0, respecting `(...)` nesting and `"..."` + * quotes. Inside quotes, a backslash escapes the next character (`\"`, `\\`), + * matching what {@link serializeValue} emits. + */ function splitTopLevel(input: string, delimiter: string): string[] { const parts: string[] = [] let depth = 0 @@ -48,7 +58,10 @@ function splitTopLevel(input: string, delimiter: string): string[] { let current = '' for (let i = 0; i < input.length; i++) { const ch = input[i] - if (ch === '"') { + if (inQuotes && ch === '\\' && i + 1 < input.length) { + current += ch + input[i + 1] + i++ + } else if (ch === '"') { inQuotes = !inQuotes current += ch } else if (inQuotes) { @@ -74,6 +87,17 @@ function splitTopLevel(input: string, delimiter: string): string[] { return parts } +/** + * Strips one layer of PostgREST double-quoting and unescapes `\"`/`\\`, + * the inverse of {@link serializeValue}. + */ +function unquote(raw: string): string { + if (raw.length >= 2 && raw.startsWith('"') && raw.endsWith('"')) { + return raw.slice(1, -1).replace(/\\(["\\])/g, '$1') + } + return raw +} + function validateField(field: string): void { if (!NAME_PATTERN.test(field)) { throw new TableQueryValidationError( @@ -84,11 +108,10 @@ function validateField(field: string): void { /** Coerces a raw PostgREST value string to the column's stored JSON type. */ function coerceValue(raw: string, type: ColumnType | undefined): JsonValue { - // Strip one layer of PostgREST double-quoting (used to escape reserved chars). - const v = raw.length >= 2 && raw.startsWith('"') && raw.endsWith('"') ? raw.slice(1, -1) : raw + const v = unquote(raw) if (type === 'number') { const n = Number(v) - if (v.trim() === '' || Number.isNaN(n)) { + if (v.trim() === '' || !Number.isFinite(n)) { throw new TableQueryValidationError(`Expected a number for column value, got "${v}"`) } return n @@ -170,6 +193,11 @@ function parseLeaf(field: string, opSpec: string, typeByName: Map coerceValue(el, type)) + if (items.length === 0) { + // An empty list would compile to no condition at all and silently widen + // the match — reject instead. + throw new TableQueryValidationError(`Empty in.() list on column "${field}"`) + } return { field, op: negated ? 'nin' : 'in', value: items } } @@ -191,8 +219,9 @@ function parseLeaf(field: string, opSpec: string, typeByName: Map= 2 && raw.startsWith('"') && raw.endsWith('"') ? raw.slice(1, -1) : raw - } + if (TEXT_OPS.has(op)) return unquote(raw) return coerceValue(raw, type) } function unsupportedNegation(token: string): TableQueryValidationError { return new TableQueryValidationError( - `Negation "not.${token}" is not supported — only not.eq, not.in, and not.is.null.` + `Negation "not.${token}" is not supported — only not.eq, not.in, not.is.null, not.like, not.ilike.` ) } @@ -254,6 +281,10 @@ export function parsePostgrestFilter(input: string, columns: ColumnDefinition[]) if (key === 'and' || key === 'or') { const members = parseList(rhs) + if (members.length === 0) { + // An empty group compiles to no condition — a silent no-op filter. + throw new TableQueryValidationError(`Empty ${key}=() group in filter`) + } // re-parse as group items (parseList split top-level commas already) const parsed = members.map((m) => parseGroupItem(m, typeByName)) nodes.push(key === 'and' ? { all: parsed } : { any: parsed }) @@ -273,12 +304,22 @@ export function parsePostgrestOrder(input: string, columns: ColumnDefinition[]): for (const part of input.split(',')) { const seg = part.trim() if (seg === '') continue - const [field, dirRaw = 'asc'] = seg.split('.') + const segments = seg.split('.') + if (segments.length > 2) { + throw new TableQueryValidationError(`Malformed sort "${seg}" — use column or column.desc`) + } + const [field, dirRaw = 'asc'] = segments validateField(field) if (!valid.has(field)) { throw new TableQueryValidationError(`Unknown sort column "${field}"`) } - const direction: SortDirection = dirRaw === 'desc' ? 'desc' : 'asc' + // Strict: a typo'd direction must not silently sort ascending. + if (dirRaw !== 'asc' && dirRaw !== 'desc') { + throw new TableQueryValidationError( + `Unknown sort direction "${dirRaw}" on "${field}" — use asc or desc (lowercase)` + ) + } + const direction: SortDirection = dirRaw spec.push({ field, direction }) } return spec @@ -307,36 +348,81 @@ function leafToOpSpec(p: Predicate): string { case 'match': case 'imatch': return `${p.op}.${serializeValue(v)}` - // Builder-only ops map onto PostgREST text/null forms. + case 'nlike': + return `not.like.${serializeValue(v)}` + case 'nilike': + return `not.ilike.${serializeValue(v)}` + // Builder-only substring ops: build the whole wildcard pattern FIRST, then + // serialize it as ONE value, so quoting wraps the entire pattern and the + // parser's whole-value unquote recovers it (`ilike."*example.com*"`). case 'contains': - return `ilike.*${serializeValue(v)}*` + return `ilike.${serializeValue(`*${asText(v)}*`)}` case 'ncontains': - return `not.ilike.*${serializeValue(v)}*` + return `not.ilike.${serializeValue(`*${asText(v)}*`)}` case 'startsWith': - return `ilike.${serializeValue(v)}*` + return `ilike.${serializeValue(`${asText(v)}*`)}` case 'endsWith': - return `ilike.*${serializeValue(v)}` + return `ilike.${serializeValue(`*${asText(v)}`)}` case 'isNull': - case 'isEmpty': return 'is.null' case 'isNotNull': - case 'isNotEmpty': return 'not.is.null' default: + // isEmpty/isNotEmpty are desugared into groups before leaves serialize. throw new TableQueryValidationError(`Cannot serialize operator "${p.op}"`) } } -/** Quotes values containing reserved chars so they survive a round-trip. */ +function asText(value: JsonValue | undefined): string { + return value === null || value === undefined ? '' : String(value) +} + +/** + * Quotes values containing reserved chars (and the empty string) so they + * survive a round-trip; literal `"`/`\` inside a quoted value are escaped. + * Inverse of {@link unquote}. + */ function serializeValue(value: JsonValue | undefined): string { const s = value === null || value === undefined ? 'null' : String(value) - return /[,.()&="]/.test(s) ? `"${s.replace(/"/g, '\\"')}"` : s + if (s === '') return '""' + if (!/[,.()&="\\]/.test(s)) return s + return `"${s.replace(/\\/g, '\\\\').replace(/"/g, '\\"')}"` } function isGroup(node: PredicateNode): node is TablePredicate { return 'all' in node || 'any' in node } +/** + * Rewrites emptiness leaves into their PostgREST-expressible groups, preserving + * the legacy "null OR empty string" semantics through the string round-trip: + * isEmpty → `(f is null OR f = '')`, isNotEmpty → `(f not null AND f <> '')`. + */ +function desugarEmptiness(node: PredicateNode): PredicateNode { + if (isGroup(node)) { + return 'all' in node + ? { all: node.all.map(desugarEmptiness) } + : { any: node.any.map(desugarEmptiness) } + } + if (node.op === 'isEmpty') { + return { + any: [ + { field: node.field, op: 'isNull' }, + { field: node.field, op: 'eq', value: '' }, + ], + } + } + if (node.op === 'isNotEmpty') { + return { + all: [ + { field: node.field, op: 'isNotNull' }, + { field: node.field, op: 'ne', value: '' }, + ], + } + } + return node +} + /** Serializes a node as a group item (dot-form leaves, function-form groups). */ function nodeToGroupItem(node: PredicateNode): string { if (isGroup(node)) { @@ -356,10 +442,11 @@ function nodeToGroupItem(node: PredicateNode): string { * `or=(...)`; nested groups use the function form. */ export function predicateToPostgrest(predicate: TablePredicate): string { - if ('any' in predicate) { - return `or=(${predicate.any.map(nodeToGroupItem).join(',')})` + const desugared = desugarEmptiness(predicate) as TablePredicate + if ('any' in desugared) { + return `or=(${desugared.any.map(nodeToGroupItem).join(',')})` } - return predicate.all + return desugared.all .map((node) => isGroup(node) ? `${'all' in node ? 'and' : 'or'}=(${(('all' in node ? node.all : node.any) as PredicateNode[]).map(nodeToGroupItem).join(',')})` diff --git a/apps/sim/lib/table/rows/__tests__/cursor.test.ts b/apps/sim/lib/table/rows/__tests__/cursor.test.ts index 606af572932..80a5163ec45 100644 --- a/apps/sim/lib/table/rows/__tests__/cursor.test.ts +++ b/apps/sim/lib/table/rows/__tests__/cursor.test.ts @@ -5,34 +5,50 @@ import { describe, expect, it } from 'vitest' import { decodeCursor, encodeCursor } from '@/lib/table/rows/cursor' describe('cursor codec', () => { - it('encodes a keyset cursor for the default order and round-trips it', () => { + it('encodes a keyset cursor when keyset is valid and round-trips it', () => { const token = encodeCursor({ lastRow: { id: 'row-9', orderKey: 'a0' }, + keysetValid: true, nextOffset: 100, }) expect(typeof token).toBe('string') expect(decodeCursor(token)).toEqual({ after: { orderKey: 'a0', id: 'row-9' } }) }) - it('falls back to an offset cursor when a custom sort is active', () => { + it('falls back to an offset cursor when keyset is not valid (custom sort / flag off)', () => { const token = encodeCursor({ lastRow: { id: 'row-9', orderKey: 'a0' }, - sort: { wins: 'desc' }, + keysetValid: false, nextOffset: 100, }) expect(decodeCursor(token)).toEqual({ offset: 100 }) }) - it('falls back to an offset cursor when the row has no order key (legacy)', () => { + it('emits a compound cursor when the last row is unkeyed but an anchor is known', () => { const token = encodeCursor({ lastRow: { id: 'row-9', orderKey: undefined }, + keysetValid: true, + nextOffset: 50, + seekBase: { anchor: { orderKey: 'a5', id: 'row-5' }, offsetFromAnchor: 4 }, + }) + expect(decodeCursor(token)).toEqual({ after: { orderKey: 'a5', id: 'row-5' }, offset: 4 }) + }) + + it('falls back to a whole-view offset when the row has no order key and no anchor', () => { + const token = encodeCursor({ + lastRow: { id: 'row-9', orderKey: undefined }, + keysetValid: true, nextOffset: 50, }) expect(decodeCursor(token)).toEqual({ offset: 50 }) }) it('produces opaque base64url with no raw orderKey/offset leaking', () => { - const token = encodeCursor({ lastRow: { id: 'r', orderKey: 'zz' }, nextOffset: 0 }) + const token = encodeCursor({ + lastRow: { id: 'r', orderKey: 'zz' }, + keysetValid: true, + nextOffset: 0, + }) expect(token).not.toContain('orderKey') expect(token).not.toContain('{') expect(token).toMatch(/^[A-Za-z0-9_-]+$/) @@ -45,4 +61,19 @@ describe('cursor codec', () => { 'Invalid cursor' ) }) + + it('throws on valid JSON that is not an object (no raw TypeError leak)', () => { + for (const body of ['42', 'null', '"hi"', 'true', '[1,2]']) { + expect(() => decodeCursor(Buffer.from(body).toString('base64url'))).toThrow('Invalid cursor') + } + }) + + it('rejects negative and non-integer offsets', () => { + expect(() => decodeCursor(Buffer.from('{"o":-1}').toString('base64url'))).toThrow( + 'Invalid cursor' + ) + expect(() => decodeCursor(Buffer.from('{"o":1.5}').toString('base64url'))).toThrow( + 'Invalid cursor' + ) + }) }) diff --git a/apps/sim/lib/table/rows/cursor.ts b/apps/sim/lib/table/rows/cursor.ts index 895c2c72cb2..2016c9d0c84 100644 --- a/apps/sim/lib/table/rows/cursor.ts +++ b/apps/sim/lib/table/rows/cursor.ts @@ -6,13 +6,16 @@ * an opaque `cursor` and never juggle `orderKey`/`id`/`offset` themselves. * * - Default order → keyset on `(order_key, id)` (`{ k, i }`), an index seek. - * - Sorted views / rows not yet order-key-backfilled → offset fallback (`{ o }`), - * because `(order_key, id)` keyset can't seek a data-column ordering. + * - Sorted views → whole-view offset (`{ o }`), because `(order_key, id)` + * keyset can't seek a data-column ordering. + * - Keyset page whose last row lacks an `orderKey` (not yet backfilled) → + * compound (`{ k, i, o }`): seek to the last keyed anchor, then OFFSET past + * the unkeyed rows consumed after it. */ -import type { Sort, TableRow, TableRowsCursor } from '@/lib/table/types' +import type { TableRow, TableRowsCursor } from '@/lib/table/types' -type CursorPayload = { k: string; i: string } | { o: number } +type CursorPayload = { k: string; i: string } | { o: number } | { k: string; i: string; o: number } function toBase64Url(json: string): string { return Buffer.from(json, 'utf8').toString('base64url') @@ -23,37 +26,69 @@ function fromBase64Url(token: string): string { } /** - * Builds the cursor for the page *after* `lastRow`. Uses keyset for the default - * order when the row carries an `orderKey`; otherwise falls back to `nextOffset` - * (the offset to resume at = current offset + rows returned). + * Builds the cursor for the page *after* `lastRow`. + * + * Shape selection: + * 1. `keysetValid` and the row carries an `orderKey` → `{ k, i }`. + * 2. `keysetValid` with a known prior anchor (last row unkeyed) → `{ k, i, o }`. + * 3. Otherwise → `{ o: nextOffset }` (whole-view offset). + * + * `keysetValid` must only be true when the `(order_key, id)` index order is + * authoritative for the page: no custom sort AND fractional ordering enabled. + * Passing false forces the offset shape, which is correct under any ordering. */ export function encodeCursor(args: { lastRow: Pick - sort?: Sort + keysetValid: boolean nextOffset: number + seekBase?: { anchor: TableRowsCursor; offsetFromAnchor: number } }): string { - const isDefaultOrder = !args.sort || Object.keys(args.sort).length === 0 - const payload: CursorPayload = - isDefaultOrder && args.lastRow.orderKey - ? { k: args.lastRow.orderKey, i: args.lastRow.id } - : { o: args.nextOffset } + let payload: CursorPayload + if (args.keysetValid && args.lastRow.orderKey) { + payload = { k: args.lastRow.orderKey, i: args.lastRow.id } + } else if (args.seekBase) { + // An anchor is in effect (inbound seek or last keyed row) but a plain + // keyset can't stand alone — resume by seeking the anchor then offsetting + // past the rows consumed after it. Never valid under a custom sort, where + // callers must not pass a seekBase. + payload = { + k: args.seekBase.anchor.orderKey, + i: args.seekBase.anchor.id, + o: args.seekBase.offsetFromAnchor, + } + } else { + payload = { o: args.nextOffset } + } return toBase64Url(JSON.stringify(payload)) } /** Decodes an opaque cursor into the `queryRows` paging inputs it stands for. */ export function decodeCursor(token: string): { after?: TableRowsCursor; offset?: number } { - let payload: CursorPayload + let payload: unknown try { payload = JSON.parse(fromBase64Url(token)) } catch { throw new Error('Invalid cursor') } + if (typeof payload !== 'object' || payload === null || Array.isArray(payload)) { + throw new Error('Invalid cursor') + } - if ('k' in payload && 'i' in payload) { - return { after: { orderKey: String(payload.k), id: String(payload.i) } } + const record = payload as Record + const hasKeyset = typeof record.k === 'string' && typeof record.i === 'string' + const hasOffset = typeof record.o === 'number' && Number.isInteger(record.o) && record.o >= 0 + + if (hasKeyset && hasOffset) { + return { + after: { orderKey: record.k as string, id: record.i as string }, + offset: record.o as number, + } + } + if (hasKeyset) { + return { after: { orderKey: record.k as string, id: record.i as string } } } - if ('o' in payload && typeof payload.o === 'number' && Number.isInteger(payload.o)) { - return { offset: payload.o } + if (hasOffset) { + return { offset: record.o as number } } throw new Error('Invalid cursor') } diff --git a/apps/sim/lib/table/rows/paging.ts b/apps/sim/lib/table/rows/paging.ts deleted file mode 100644 index a96b98c041f..00000000000 --- a/apps/sim/lib/table/rows/paging.ts +++ /dev/null @@ -1,23 +0,0 @@ -/** - * Cuts a fetched page to a byte budget: keeps the longest prefix of rows whose - * serialized `data` fits within `maxBytes`, always keeping at least one row so - * pagination makes forward progress even when a single row exceeds the budget. - * - * The budget counts `data` only — the per-row envelope (`id`, `position`, - * `orderKey`, timestamps, executions) is not measured, so actual response - * payloads run slightly over `maxBytes`. Callers must leave headroom; the - * production SQL-side cut should account for the same overhead. - */ -export function trimRowsToByteBudget( - rows: T[], - maxBytes: number -): T[] { - let total = 0 - let kept = 0 - for (const row of rows) { - total += Buffer.byteLength(JSON.stringify(row.data)) - if (kept > 0 && total > maxBytes) break - kept++ - } - return kept === rows.length ? rows : rows.slice(0, kept) -} diff --git a/apps/sim/lib/table/rows/service.ts b/apps/sim/lib/table/rows/service.ts index b05b24afe06..783d5cc85d6 100644 --- a/apps/sim/lib/table/rows/service.ts +++ b/apps/sim/lib/table/rows/service.ts @@ -25,7 +25,8 @@ import { wouldExceedRowLimit, } from '@/lib/table/billing' import { getColumnId } from '@/lib/table/column-keys' -import { getMaxPageBytes, TABLE_LIMITS, USER_TABLE_ROWS_SQL_NAME } from '@/lib/table/constants' +import { TABLE_LIMITS, USER_TABLE_ROWS_SQL_NAME } from '@/lib/table/constants' +import { TableQueryValidationError } from '@/lib/table/errors' import { nKeysBetween } from '@/lib/table/order-key' import { type DbExecutor, type DbTransaction, withSeqscanOff } from '@/lib/table/planner' import { encodeCursor } from '@/lib/table/rows/cursor' @@ -47,14 +48,12 @@ import { resolveBatchInsertOrderKeys, resolveInsertOrderKey, } from '@/lib/table/rows/ordering' -import { trimRowsToByteBudget } from '@/lib/table/rows/paging' import { buildFilterClause, buildPredicateClause, buildSortClause, escapeLikePattern, fieldPredicate, - TableQueryValidationError, } from '@/lib/table/sql' import { fireTableTrigger } from '@/lib/table/trigger' import { scaledStatementTimeoutMs, setTableTxTimeouts } from '@/lib/table/tx' @@ -1017,38 +1016,17 @@ export async function queryRows( whereClause = and(baseConditions, userClause) } - // Keyset page: seek past the cursor on the default `(order_key, id)` order instead of paying - // OFFSET's scan-and-discard of every prior row (O(N²) across a deep scroll / full drain). Only - // valid without a custom sort — the contract rejects `after` + `sort` together. The count below - // deliberately excludes the cursor: totals cover the whole view, not the remaining pages. - const pageWhere = - after && !sort - ? and( - whereClause, - sql`(${userTableRows.orderKey}, ${userTableRows.id}) > (${after.orderKey}, ${after.id})` - ) - : whereClause - - const buildPageQuery = (executor: DbExecutor) => { - const query = executor - .select() - .from(userTableRows) - .where(pageWhere ?? baseConditions) - .orderBy(buildRowOrderBySql(sort, tableName, columns, fractionalOrdering)) - if (limit === undefined) return query - return after ? query.limit(limit) : query.limit(limit).offset(offset) - } + // Keyset seeks are only authoritative when the page order IS the + // `(order_key, id)` index order: no custom sort and fractional ordering on. + const keysetValid = !sort && fractionalOrdering - // Count and page fetch are independent reads — run them concurrently so the + // Count and page drain are independent reads — run them concurrently so the // `includeTotal` hot path doesn't pay two serial round-trips. Filtered counts // go through the tenant-bounded variant (see countRowsTenantBounded); the // unfiltered count already plans an index-only scan on the table_id prefix. - // Custom column sorts order by `data->>'col'` — unestimatable, so left alone - // the planner seq-scans and sorts the whole shared relation on every page - // (9.7s measured on a 1M-row table; 0.76s tenant-bounded). Default-order - // pages already stream the `(table_id, order_key, id)` index. + // The count uses the full-view WHERE (no cursor seek): totals cover the whole + // view, not the remaining pages. const hasFilter = Boolean(userClause) - const rowsPromise = sort ? withSeqscanOff(async (trx) => buildPageQuery(trx)) : buildPageQuery(db) const countPromise = includeTotal ? hasFilter ? countRowsTenantBounded(whereClause) @@ -1059,12 +1037,22 @@ export async function queryRows( .then((r) => Number(r[0].count)) : null - const [fetchedRows, totalCount] = await Promise.all([rowsPromise, countPromise]) + // The inbound seek is honored whenever there's no custom sort (matching the + // pre-drain behavior even when fractional ordering is off); `keysetValid` + // only gates re-anchoring and plain-keyset cursor emission. + const drainPromise = fetchRowsBounded({ + baseWhere: whereClause ?? baseConditions, + orderBy: buildRowOrderBySql(sort, tableName, columns, fractionalOrdering), + sorted: Boolean(sort), + keysetValid, + seek: !sort ? after : undefined, + startOffset: offset, + limit, + budgetBytes: TABLE_LIMITS.MAX_QUERY_RESULT_BYTES, + }) - // Dev-preview byte cut (TABLE_MAX_PAGE_BYTES, off by default): clients terminate on - // empty page / totalCount, never page fullness, so a short page is safe to return. - const maxPageBytes = getMaxPageBytes() - const rows = maxPageBytes === null ? fetchedRows : trimRowsToByteBudget(fetchedRows, maxPageBytes) + const [fetched, totalCount] = await Promise.all([drainPromise, countPromise]) + const rows = fetched.rows const executionsByRow = withExecutions ? await loadExecutionsByRow( @@ -1074,7 +1062,7 @@ export async function queryRows( : null logger.info( - `[${requestId}] Queried ${rows.length} rows from table ${table.id} (total: ${totalCount})` + `[${requestId}] Queried ${rows.length} rows from table ${table.id} (total: ${totalCount}, bytes: ${fetched.bytes}, more: ${fetched.hasMore})` ) const mappedRows = rows.map((r) => ({ @@ -1087,25 +1075,21 @@ export async function queryRows( updatedAt: r.updatedAt, })) - // Fail fast if the result outgrows the byte budget instead of returning a - // silently truncated page — the caller narrows it with a filter or a limit. - let resultBytes = 0 - for (const row of mappedRows) { - resultBytes += Buffer.byteLength(JSON.stringify(row.data)) - if (resultBytes > TABLE_LIMITS.MAX_QUERY_RESULT_BYTES) { - throw new TableQueryValidationError( - `Query result exceeds the ${TABLE_LIMITS.MAX_QUERY_RESULT_BYTES / (1024 * 1024)}MB limit. Add a filter or a limit to narrow the result.` - ) - } - } - - // Opaque next-page cursor: only meaningful for a bounded page that filled up. - // An unbounded query returns everything, so there is never a next page. + // Opaque next-page cursor: non-null whenever more matching rows exist beyond + // this page — whether the page was cut by `limit` or by the byte budget. The + // drain loop proves `hasMore` with a fetched-but-unreturned witness row. const lastRow = mappedRows[mappedRows.length - 1] const nextCursor = - limit === undefined || mappedRows.length < limit || !lastRow - ? null - : encodeCursor({ lastRow, sort, nextOffset: offset + mappedRows.length }) + fetched.hasMore && lastRow + ? encodeCursor({ + lastRow, + keysetValid, + nextOffset: offset + mappedRows.length, + seekBase: fetched.anchor + ? { anchor: fetched.anchor, offsetFromAnchor: fetched.anchorOffset } + : undefined, + }) + : null return { rows: mappedRows, @@ -1117,6 +1101,152 @@ export async function queryRows( } } +interface BoundedFetchParams { + /** Tenant + delete-mask + user filter — WITHOUT any seek predicate. */ + baseWhere: SQL | undefined + orderBy: SQL + /** Custom sort present → per-batch withSeqscanOff (JSONB order is unestimatable). */ + sorted: boolean + /** `(order_key, id)` order is authoritative → keyset re-anchoring + seeks. */ + keysetValid: boolean + /** Inbound seek anchor (decoded keyset/compound cursor), if any. */ + seek?: TableRowsCursor + /** Inbound offset — the whole-view offset, or the past-anchor offset of a compound cursor. */ + startOffset: number + limit?: number + budgetBytes: number +} + +interface BoundedFetchResult { + rows: Array + bytes: number + /** Proven by a fetched-but-unreturned witness row — never inferred from page fullness. */ + hasMore: boolean + /** Final keyset anchor, for compound-cursor emission when the last row is unkeyed. */ + anchor?: TableRowsCursor + /** Rows consumed past `anchor` (0 when the anchor is the last returned row). */ + anchorOffset: number +} + +/** Belt-and-braces bound on drain iterations; unreachable in practice. */ +const MAX_QUERY_BATCHES = 1000 + +/** + * Drains rows in adaptively-sized bounded batches until the caller's `limit` + * or the byte budget cuts the page. Never issues an unbounded SELECT: the + * first batch is capped so its worst-case bytes stay within ~4× the budget at + * the max row size, and later batches are sized from the observed average. + * + * Advance strategy: when `keysetValid`, the loop re-anchors on each consumed + * keyed row and seeks `(order_key, id) > (anchor)` — delete-tolerant and an + * index seek. Otherwise (custom sort, flag off) it advances by OFFSET from the + * inbound position; rows deleted mid-drain can skip/duplicate exactly as + * cross-request offset paging already does. + * + * Always returns at least one row when any match exists, even if that row + * alone exceeds the budget. + */ +async function fetchRowsBounded(params: BoundedFetchParams): Promise { + const { baseWhere, orderBy, sorted, keysetValid, limit, budgetBytes } = params + + const firstBatchCap = Math.max(1, Math.floor((4 * budgetBytes) / TABLE_LIMITS.MAX_ROW_SIZE_BYTES)) + + const rows: Array = [] + let bytes = 0 + let maxRowBytes = 0 + let hasMore = false + let anchor = params.seek + let anchorOffset = params.startOffset + let consumedSinceAnchor = 0 + + const nextBatchRows = (): number => { + if (rows.length === 0) return Math.min(limit ?? firstBatchCap, firstBatchCap) + const avg = Math.max(1, bytes / rows.length) + const remaining = budgetBytes - bytes + const byAverage = Math.ceil(remaining / avg) + 1 + const varianceCap = Math.ceil((8 * remaining) / Math.max(maxRowBytes, 1)) + return Math.max(1, Math.min(byAverage, TABLE_LIMITS.QUERY_BATCH_MAX_ROWS, varianceCap)) + } + + const runBatch = (batchSeek: TableRowsCursor | undefined, batchOffset: number, ask: number) => { + const buildQuery = (executor: DbExecutor) => { + const seekWhere = batchSeek + ? and( + baseWhere, + sql`(${userTableRows.orderKey}, ${userTableRows.id}) > (${batchSeek.orderKey}, ${batchSeek.id})` + ) + : baseWhere + const query = executor + .select() + .from(userTableRows) + .where(seekWhere) + .orderBy(orderBy) + .limit(ask) + return batchOffset > 0 ? query.offset(batchOffset) : query + } + // Custom column sorts order by `data->>'col'` — unestimatable, so left + // alone the planner seq-scans and sorts the whole shared relation on every + // page (9.7s measured on a 1M-row table; 0.76s tenant-bounded). One tx per + // batch: SET LOCAL dies with the tx, and holding a tx across JS accounting + // between batches would pin a pooled connection. + return sorted ? withSeqscanOff(async (trx) => buildQuery(trx)) : buildQuery(db) + } + + for (let iteration = 0; iteration < MAX_QUERY_BATCHES; iteration++) { + const limitRemaining = limit === undefined ? Number.POSITIVE_INFINITY : limit - rows.length + const target = Math.min(nextBatchRows(), limitRemaining) + const ask = target + 1 // +1 = witness row proving more data exists past a cut + const batch = await runBatch(anchor, anchorOffset + consumedSinceAnchor, ask) + if (batch.length === 0) break + + let cut = false + for (const row of batch) { + const rowBytes = Buffer.byteLength(JSON.stringify(row.data)) + if (rows.length > 0 && bytes + rowBytes > budgetBytes) { + // Unbounded queries promise the ENTIRE result — a partial page would be + // silent truncation, so fail fast instead (the drain has only fetched + // ~budget bytes at this point, never the whole table). + if (limit === undefined) { + throw new TableQueryValidationError( + `Query result exceeds the ${Math.floor(budgetBytes / (1024 * 1024))}MB limit. Add a filter or a limit to narrow the result.` + ) + } + // Bounded page: byte cut with `row` as the witness. Requires a + // non-empty page so a single over-budget row is still returned alone. + hasMore = true + cut = true + break + } + // Limit cut: `row` is the +1 peek witness. + if (rows.length === limit) { + hasMore = true + cut = true + break + } + rows.push(row) + bytes += rowBytes + consumedSinceAnchor++ + if (rowBytes > maxRowBytes) maxRowBytes = rowBytes + if (keysetValid && row.orderKey) { + anchor = { orderKey: row.orderKey, id: row.id } + anchorOffset = 0 + consumedSinceAnchor = 0 + } + } + if (cut) break + // Short batch = the source is exhausted; hasMore stays false. + if (batch.length < ask) break + } + + return { + rows, + bytes, + hasMore, + anchor, + anchorOffset: anchorOffset + consumedSinceAnchor, + } +} + /** * Gets a single row by ID. * diff --git a/apps/sim/lib/table/sql.ts b/apps/sim/lib/table/sql.ts index fb6b2d7e389..51d2bc0bc07 100644 --- a/apps/sim/lib/table/sql.ts +++ b/apps/sim/lib/table/sql.ts @@ -9,6 +9,7 @@ import type { SQL } from 'drizzle-orm' import { sql } from 'drizzle-orm' import { getColumnId } from '@/lib/table/column-keys' import { NAME_PATTERN } from '@/lib/table/constants' +import { TableQueryValidationError } from '@/lib/table/errors' import type { ColumnDefinition, ConditionOperators, @@ -21,17 +22,6 @@ import type { TablePredicate, } from '@/lib/table/types' -/** - * Error thrown when caller-supplied filter or sort input is malformed. - * Routes should map this to HTTP 400 with the message preserved. - */ -export class TableQueryValidationError extends Error { - constructor(message: string) { - super(message) - this.name = 'TableQueryValidationError' - } -} - type ColumnType = ColumnDefinition['type'] type ColumnTypeMap = ReadonlyMap @@ -80,6 +70,8 @@ const ALLOWED_OPERATORS = new Set([ '$endsWith', '$like', '$ilike', + '$nlike', + '$nilike', '$match', '$imatch', '$empty', @@ -456,9 +448,19 @@ export function fieldPredicate( return buildLikeClause(tableName, field, value as string, 'endsWith') case 'like': - return buildPatternClause(tableName, field, value as string, false) + return buildPatternClause(tableName, field, value as string, { caseInsensitive: false }) case 'ilike': - return buildPatternClause(tableName, field, value as string, true) + return buildPatternClause(tableName, field, value as string, { caseInsensitive: true }) + case 'nlike': + return buildPatternClause(tableName, field, value as string, { + caseInsensitive: false, + negate: true, + }) + case 'nilike': + return buildPatternClause(tableName, field, value as string, { + caseInsensitive: true, + negate: true, + }) case 'isEmpty': return buildEmptyClause(tableName, field, true) @@ -569,21 +571,29 @@ export function escapeLikePattern(value: string): string { * is the only wildcard — it maps to SQL `%`; any literal `%`/`_`/`\` in the * value is escaped so it matches itself. Empty/exact patterns are allowed (an * empty pattern matches only the empty string, not every row, so it's not the - * footgun the positional `buildLikeClause` guards against). Cannot use the GIN - * index; sequential scan bounded by the `table_id` btree prefix. + * footgun the positional `buildLikeClause` guards against). `negate` inverts + * the match and keeps null cells — "does not match" retains empty rows, + * mirroring `buildLikeClause`'s ncontains semantics. Cannot use the GIN index; + * sequential scan bounded by the `table_id` btree prefix. */ function buildPatternClause( tableName: string, field: string, value: string, - caseInsensitive: boolean + options: { caseInsensitive: boolean; negate?: boolean } ): SQL { const escapedField = field.replace(/'/g, "''") const pattern = String(value) .replace(/[\\%_]/g, '\\$&') .replace(/\*/g, '%') const cell = sql.raw(`${tableName}.data->>'${escapedField}'`) - return caseInsensitive ? sql`${cell} ILIKE ${pattern}` : sql`${cell} LIKE ${pattern}` + const match = options.caseInsensitive + ? sql`${cell} ILIKE ${pattern}` + : sql`${cell} LIKE ${pattern}` + if (!options.negate) return match + return options.caseInsensitive + ? sql`(${cell} IS NULL OR ${cell} NOT ILIKE ${pattern})` + : sql`(${cell} IS NULL OR ${cell} NOT LIKE ${pattern})` } /** diff --git a/apps/sim/lib/table/types.ts b/apps/sim/lib/table/types.ts index afe43699b1c..53528f958af 100644 --- a/apps/sim/lib/table/types.ts +++ b/apps/sim/lib/table/types.ts @@ -447,6 +447,8 @@ export type FilterOp = | 'endsWith' | 'like' | 'ilike' + | 'nlike' + | 'nilike' | 'match' | 'imatch' | 'isEmpty' @@ -513,10 +515,15 @@ export interface QueryOptions { */ predicate?: TablePredicate sort?: Sort + /** Page row cap. Omitted = return the ENTIRE matching result, failing fast if + * it exceeds the response byte budget (`MAX_QUERY_RESULT_BYTES`). A bounded + * page may byte-cut early with `nextCursor` set. Never an unbounded fetch — + * the drain stops at the budget either way. */ limit?: number offset?: number /** Keyset cursor for the default `(order_key, id)` order — see {@link TableRowsCursor}. - * Mutually exclusive with `sort` and `offset`; takes precedence over `offset` when set. */ + * Mutually exclusive with `sort`. May be combined with `offset` (a compound + * cursor seeks the anchor, then offsets past unkeyed rows consumed after it). */ after?: TableRowsCursor /** * When true (default), runs a `COUNT(*)` and returns `totalCount` as a number. @@ -539,9 +546,10 @@ export interface QueryResult { limit: number offset: number /** - * Opaque cursor for the next page, or `null` on the last page (fewer rows than - * `limit`). The v2 surface exposes only this — callers echo it back as `cursor` - * and never construct keyset/offset state themselves. See `rows/cursor.ts`. + * Opaque cursor for the next page — non-null whenever more matching rows + * exist beyond this page, whether the page was cut by `limit` or by the + * response byte budget. Callers echo it back as `cursor` and never construct + * keyset/offset state themselves. See `rows/cursor.ts`. */ nextCursor: string | null } diff --git a/apps/sim/tools/registry.minimal.ts b/apps/sim/tools/registry.minimal.ts index b313c2e075f..f8787f0fad1 100644 --- a/apps/sim/tools/registry.minimal.ts +++ b/apps/sim/tools/registry.minimal.ts @@ -7,6 +7,7 @@ import { tableGetRowTool, tableGetSchemaTool, tableInsertRowTool, + tableQueryRowsTool, tableQueryRowsV2Tool, tableUpdateRowsByFilterTool, tableUpdateRowTool, @@ -25,7 +26,8 @@ import type { ToolConfig } from '@/tools/types' export const tools: Record = { http_request: httpRequestTool, function_execute: functionExecuteTool, - // Table v2 block operations (so the v2 Table block is runnable in minimal mode). + // Table block operations (v1 + v2 Table blocks are both runnable in minimal mode). + table_query_rows: tableQueryRowsTool, table_insert_row: tableInsertRowTool, table_batch_insert_rows: tableBatchInsertRowsTool, table_upsert_row: tableUpsertRowTool, diff --git a/apps/sim/tools/table/query_rows_v2.ts b/apps/sim/tools/table/query_rows_v2.ts index 967fc156f69..8874e313658 100644 --- a/apps/sim/tools/table/query_rows_v2.ts +++ b/apps/sim/tools/table/query_rows_v2.ts @@ -13,8 +13,10 @@ export const tableQueryRowsV2Tool: ToolConfig Promise.resolve([] as unknown[])) +const offset = vi.fn(() => Promise.resolve([] as unknown[])) +// `.limit()` is awaitable directly AND carries `.offset` for `.limit(n).offset(m)` +// chains. A `limit.mockResolvedValueOnce(...)` override loses `.offset` — tests +// exercising offset batches override `dbChainMockFns.offset` instead. +const limitBuilder = () => { + const thenable: any = Promise.resolve([] as unknown[]) + thenable.offset = offset + return thenable +} +const limit = vi.fn(limitBuilder) const returning = vi.fn(() => Promise.resolve([] as unknown[])) const execute = vi.fn(() => Promise.resolve([] as unknown[])) @@ -169,6 +178,7 @@ export const dbChainMockFns = { from, where, limit, + offset, orderBy, returning, innerJoin, @@ -211,7 +221,8 @@ export function resetDbChainMock(): void { update.mockImplementation(() => ({ set })) set.mockImplementation(() => ({ where })) del.mockImplementation(() => ({ where })) - limit.mockImplementation(() => Promise.resolve([] as unknown[])) + limit.mockImplementation(limitBuilder) + offset.mockImplementation(() => Promise.resolve([] as unknown[])) orderBy.mockImplementation(terminalBuilder) returning.mockImplementation(() => Promise.resolve([] as unknown[])) having.mockImplementation(terminalBuilder) From ded16c43fe2e8d48bea5cda8ee34d202b993cade Mon Sep 17 00:00:00 2001 From: Theodore Li Date: Wed, 8 Jul 2026 15:22:57 -0700 Subject: [PATCH 04/26] feat(table): predicate-object filter grammar + cursor pagination Replace the PostgREST querystring wire with the typed {all|any:[{field,op,value}]} predicate object across the engine, contracts, internal query/bulk routes, the public v2 read API (GET /api/v2/tables + POST .../query), the table_v2 block (canonical Builder/JSON filter toggle), the query_rows_v2 tool, and the copilot user_table tool. Adds validatePredicate schema-aware validation. Track A engine hardening: read-path statement timeout, comparator negation (not.gt family), createdAt/updatedAt filtering, machine error codes, cursor version field. Mothership pagination is now cursor-only (offset removed from the agent surface). Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01UBRahKrk4BV23spVMckskA --- .../api/table/[tableId]/query/route.test.ts | 33 +- .../app/api/table/[tableId]/query/route.ts | 201 ++++---- .../api/table/[tableId]/rows/route.test.ts | 20 +- .../sim/app/api/table/[tableId]/rows/route.ts | 30 +- apps/sim/app/api/v1/middleware.ts | 13 +- .../v2/tables/[tableId]/query/route.test.ts | 237 +++++++++ .../api/v2/tables/[tableId]/query/route.ts | 171 +++++++ apps/sim/app/api/v2/tables/route.test.ts | 104 ++++ apps/sim/app/api/v2/tables/route.ts | 73 +++ apps/sim/blocks/blocks/table_v2.test.ts | 32 +- apps/sim/blocks/blocks/table_v2.ts | 207 ++++---- .../api/contracts/tables-predicate.test.ts | 87 ++-- apps/sim/lib/api/contracts/tables.ts | 82 +++- apps/sim/lib/api/contracts/v2/tables/index.ts | 115 +++++ .../lib/copilot/generated/tool-catalog-v1.ts | 20 +- .../lib/copilot/generated/tool-schemas-v1.ts | 210 ++++---- .../tools/server/table/user-table.test.ts | 77 ++- .../copilot/tools/server/table/user-table.ts | 74 ++- apps/sim/lib/table/__tests__/sql.test.ts | 28 +- apps/sim/lib/table/constants.ts | 31 ++ apps/sim/lib/table/errors.ts | 16 +- apps/sim/lib/table/planner.ts | 55 ++- .../__tests__/converters.test.ts | 57 +-- .../query-builder/__tests__/postgrest.test.ts | 271 ---------- .../query-builder/__tests__/validate.test.ts | 99 ++++ .../sim/lib/table/query-builder/converters.ts | 17 - apps/sim/lib/table/query-builder/postgrest.ts | 461 ------------------ apps/sim/lib/table/query-builder/validate.ts | 113 +++++ .../lib/table/rows/__tests__/cursor.test.ts | 23 +- apps/sim/lib/table/rows/cursor.ts | 31 +- apps/sim/lib/table/rows/service.ts | 37 +- apps/sim/lib/table/sql.ts | 72 ++- apps/sim/lib/table/types.ts | 26 +- apps/sim/tools/table/query_rows_v2.ts | 24 +- apps/sim/tools/table/types.ts | 8 +- scripts/check-api-validation-contracts.ts | 4 +- 36 files changed, 1791 insertions(+), 1368 deletions(-) create mode 100644 apps/sim/app/api/v2/tables/[tableId]/query/route.test.ts create mode 100644 apps/sim/app/api/v2/tables/[tableId]/query/route.ts create mode 100644 apps/sim/app/api/v2/tables/route.test.ts create mode 100644 apps/sim/app/api/v2/tables/route.ts create mode 100644 apps/sim/lib/api/contracts/v2/tables/index.ts delete mode 100644 apps/sim/lib/table/query-builder/__tests__/postgrest.test.ts create mode 100644 apps/sim/lib/table/query-builder/__tests__/validate.test.ts delete mode 100644 apps/sim/lib/table/query-builder/postgrest.ts create mode 100644 apps/sim/lib/table/query-builder/validate.ts diff --git a/apps/sim/app/api/table/[tableId]/query/route.test.ts b/apps/sim/app/api/table/[tableId]/query/route.test.ts index 98ef63f97dc..b6b6ba6e2c5 100644 --- a/apps/sim/app/api/table/[tableId]/query/route.test.ts +++ b/apps/sim/app/api/table/[tableId]/query/route.test.ts @@ -92,12 +92,17 @@ describe('POST /api/table/[tableId]/query', () => { mockQueryRows.mockResolvedValue(EMPTY_RESULT) }) - it('translates filter/order column names to storage ids for SESSION auth too', async () => { + it('translates predicate/sort column names to storage ids for SESSION auth too', async () => { authAs('session') const res = await callQuery({ workspaceId: 'workspace-1', - filter: 'name=eq.John&wins=gte.10', - order: 'wins.desc', + predicate: { + all: [ + { field: 'name', op: 'eq', value: 'John' }, + { field: 'wins', op: 'gte', value: 10 }, + ], + }, + sort: [{ field: 'wins', direction: 'desc' }], }) expect(res.status).toBe(200) @@ -112,18 +117,23 @@ describe('POST /api/table/[tableId]/query', () => { expect(options.withExecutions).toBe(false) }) - it('rejects a keyset cursor combined with a custom order', async () => { + it('rejects a keyset cursor combined with a custom sort', async () => { authAs('internal_jwt') const cursor = encodeCursor({ lastRow: { id: 'row_1', orderKey: 'a1' }, keysetValid: true, nextOffset: 1, }) - const res = await callQuery({ workspaceId: 'workspace-1', order: 'wins.desc', cursor }) + const res = await callQuery({ + workspaceId: 'workspace-1', + sort: [{ field: 'wins', direction: 'desc' }], + cursor, + }) expect(res.status).toBe(400) const body = await res.json() expect(body.error).toMatch(/not valid for a sorted query/) + expect(body.code).toBe('CURSOR_SORT_CONFLICT') expect(mockQueryRows).not.toHaveBeenCalled() }) @@ -135,15 +145,20 @@ describe('POST /api/table/[tableId]/query', () => { }) expect(res.status).toBe(400) - expect((await res.json()).error).toBe('Invalid cursor') + const body = await res.json() + expect(body.error).toBe('Invalid cursor') + expect(body.code).toBe('INVALID_CURSOR') }) - it('returns 400 with the parser message for an invalid filter', async () => { + it('returns 400 for a predicate referencing an unknown column', async () => { authAs('internal_jwt') - const res = await callQuery({ workspaceId: 'workspace-1', filter: 'wins=bogus.1' }) + const res = await callQuery({ + workspaceId: 'workspace-1', + predicate: { all: [{ field: 'nope', op: 'eq', value: 1 }] }, + }) expect(res.status).toBe(400) - expect((await res.json()).error).toMatch(/Unknown filter operator/) + expect((await res.json()).error).toMatch(/Unknown filter column/) }) it('passes nextCursor through the response envelope and skips the count on later pages', async () => { diff --git a/apps/sim/app/api/table/[tableId]/query/route.ts b/apps/sim/app/api/table/[tableId]/query/route.ts index a1a6d4ec9a8..7cfd2954497 100644 --- a/apps/sim/app/api/table/[tableId]/query/route.ts +++ b/apps/sim/app/api/table/[tableId]/query/route.ts @@ -1,6 +1,6 @@ import { createLogger } from '@sim/logger' import { type NextRequest, NextResponse } from 'next/server' -import { queryTableRowsV2Contract } from '@/lib/api/contracts/tables' +import { rowQueryContract } from '@/lib/api/contracts/tables' import { parseRequest } from '@/lib/api/server' import { isZodError, validationErrorResponse } from '@/lib/api/server/validation' import { checkSessionOrInternalAuth } from '@/lib/auth/hybrid' @@ -9,124 +9,127 @@ import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import type { Sort, TableSchema } from '@/lib/table' import { buildIdByName, predicateNamesToIds, sortSpecNamesToIds } from '@/lib/table/column-keys' import { TableQueryValidationError } from '@/lib/table/errors' -import { parsePostgrestFilter, parsePostgrestOrder } from '@/lib/table/query-builder/postgrest' +import { validatePredicate, validateSortSpec } from '@/lib/table/query-builder/validate' import { decodeCursor } from '@/lib/table/rows/cursor' import { queryRows } from '@/lib/table/rows/service' import { rowWireTranslators } from '@/app/api/table/row-wire' import { accessError, checkAccess } from '@/app/api/table/utils' -const logger = createLogger('TableQueryV2API') +const logger = createLogger('TableRowQueryAPI') -interface TableQueryV2RouteParams { +interface RowQueryRouteParams { params: Promise<{ tableId: string }> } /** - * POST /api/table/[tableId]/query — v2 row query. PostgREST filter/order strings - * (parsed server-side) + opaque cursor pagination (no offset on the wire). Shares - * the same engine as the legacy GET /rows route via `queryRows`. + * POST /api/table/[tableId]/query — v2 row query. Typed `predicate`/`sort` + * objects (validated server-side) + opaque cursor pagination (no offset on the + * wire). Shares the same engine as the legacy GET /rows route via `queryRows`. */ -export const POST = withRouteHandler( - async (request: NextRequest, context: TableQueryV2RouteParams) => { - const requestId = generateRequestId() +export const POST = withRouteHandler(async (request: NextRequest, context: RowQueryRouteParams) => { + const requestId = generateRequestId() - try { - const authResult = await checkSessionOrInternalAuth(request, { requireWorkflowId: false }) - if (!authResult.success || !authResult.userId) { - return NextResponse.json({ error: 'Authentication required' }, { status: 401 }) - } - - const parsed = await parseRequest(queryTableRowsV2Contract, request, context) - if (!parsed.success) return parsed.response - const { params, body } = parsed.data - const { tableId } = params - - const accessResult = await checkAccess(tableId, authResult.userId, 'read') - if (!accessResult.ok) return accessError(accessResult, requestId, tableId) - const { table } = accessResult + try { + const authResult = await checkSessionOrInternalAuth(request, { requireWorkflowId: false }) + if (!authResult.success || !authResult.userId) { + return NextResponse.json({ error: 'Authentication required' }, { status: 401 }) + } - if (body.workspaceId !== table.workspaceId) { - logger.warn( - `[${requestId}] Workspace ID mismatch for table ${tableId}. Provided: ${body.workspaceId}, Actual: ${table.workspaceId}` - ) - return NextResponse.json({ error: 'Invalid workspace ID' }, { status: 400 }) - } + const parsed = await parseRequest(rowQueryContract, request, context) + if (!parsed.success) return parsed.response + const { params, body } = parsed.data + const { tableId } = params - const schema = table.schema as TableSchema - const wire = rowWireTranslators(authResult.authType, schema) - const cursor = body.cursor ? decodeCursor(body.cursor) : undefined + const accessResult = await checkAccess(tableId, authResult.userId, 'read') + if (!accessResult.ok) return accessError(accessResult, requestId, tableId) + const { table } = accessResult - // A keyset cursor encodes a position on the DEFAULT `(order_key, id)` - // order; combining it with a custom order would silently return page 1 - // of the sorted view relabeled as a next page. - if (cursor?.after && body.order) { - return NextResponse.json( - { error: 'Cursor is not valid for a sorted query. Restart paging without the cursor.' }, - { status: 400 } - ) - } + if (body.workspaceId !== table.workspaceId) { + logger.warn( + `[${requestId}] Workspace ID mismatch for table ${tableId}. Provided: ${body.workspaceId}, Actual: ${table.workspaceId}` + ) + return NextResponse.json({ error: 'Invalid workspace ID' }, { status: 400 }) + } - // PostgREST filter/order strings are column-NAME-keyed by construction - // (the parser validates against names), so translate names → storage ids - // unconditionally — unlike row data, this is not authType-dependent. - const idByName = buildIdByName(schema) - const predicate = body.filter - ? predicateNamesToIds(parsePostgrestFilter(body.filter, schema.columns), idByName) - : undefined - const sortSpec = body.order - ? sortSpecNamesToIds(parsePostgrestOrder(body.order, schema.columns), idByName) - : undefined - const sort: Sort | undefined = sortSpec?.length - ? Object.fromEntries(sortSpec.map((s) => [s.field, s.direction])) - : undefined + const schema = table.schema as TableSchema + const wire = rowWireTranslators(authResult.authType, schema) + const cursor = body.cursor ? decodeCursor(body.cursor) : undefined - const result = await queryRows( - table, + // A keyset cursor encodes a position on the DEFAULT `(order_key, id)` + // order; combining it with a custom order would silently return page 1 + // of the sorted view relabeled as a next page. + if (cursor?.after && body.sort?.length) { + return NextResponse.json( { - predicate, - sort, - limit: body.limit, - after: cursor?.after, - offset: cursor?.offset, - // Only the first page (no inbound cursor) pays for the total count. - includeTotal: !body.cursor, - // Executions are grid UI state; the v2 surface returns row data only - // and the byte budget deliberately measures just `data`. - withExecutions: false, + error: 'Cursor is not valid for a sorted query. Restart paging without the cursor.', + code: 'CURSOR_SORT_CONFLICT', }, - requestId + { status: 400 } ) + } - return NextResponse.json({ - success: true, - data: { - rows: result.rows.map((r) => ({ - id: r.id, - data: wire.dataOut(r.data), - executions: r.executions, - position: r.position, - orderKey: r.orderKey ?? undefined, - createdAt: - r.createdAt instanceof Date ? r.createdAt.toISOString() : String(r.createdAt), - updatedAt: - r.updatedAt instanceof Date ? r.updatedAt.toISOString() : String(r.updatedAt), - })), - rowCount: result.rowCount, - totalCount: result.totalCount, - limit: result.limit, - nextCursor: result.nextCursor, - }, - }) - } catch (error) { - if (isZodError(error)) return validationErrorResponse(error) - if (error instanceof TableQueryValidationError) { - return NextResponse.json({ error: error.message }, { status: 400 }) - } - if (error instanceof Error && error.message === 'Invalid cursor') { - return NextResponse.json({ error: 'Invalid cursor' }, { status: 400 }) - } - logger.error(`[${requestId}] Error querying rows (v2):`, error) - return NextResponse.json({ error: 'Failed to query rows' }, { status: 500 }) + // Predicate/sort fields are column-NAME-keyed by construction (the caller + // authors names), so validate against the schema then translate names → + // storage ids unconditionally — unlike row data, this is not authType-dependent. + const idByName = buildIdByName(schema) + let predicate = body.predicate + if (predicate) { + validatePredicate(predicate, schema.columns) + predicate = predicateNamesToIds(predicate, idByName) + } + let sortSpec = body.sort + if (sortSpec?.length) { + validateSortSpec(sortSpec, schema.columns) + sortSpec = sortSpecNamesToIds(sortSpec, idByName) + } + const sort: Sort | undefined = sortSpec?.length + ? Object.fromEntries(sortSpec.map((s) => [s.field, s.direction])) + : undefined + + const result = await queryRows( + table, + { + predicate, + sort, + limit: body.limit, + after: cursor?.after, + offset: cursor?.offset, + // Only the first page (no inbound cursor) pays for the total count. + includeTotal: !body.cursor, + // Executions are grid UI state; the v2 surface returns row data only + // and the byte budget deliberately measures just `data`. + withExecutions: false, + }, + requestId + ) + + return NextResponse.json({ + success: true, + data: { + rows: result.rows.map((r) => ({ + id: r.id, + data: wire.dataOut(r.data), + executions: r.executions, + position: r.position, + orderKey: r.orderKey ?? undefined, + createdAt: r.createdAt instanceof Date ? r.createdAt.toISOString() : String(r.createdAt), + updatedAt: r.updatedAt instanceof Date ? r.updatedAt.toISOString() : String(r.updatedAt), + })), + rowCount: result.rowCount, + totalCount: result.totalCount, + limit: result.limit, + nextCursor: result.nextCursor, + }, + }) + } catch (error) { + if (isZodError(error)) return validationErrorResponse(error) + if (error instanceof TableQueryValidationError) { + return NextResponse.json( + { error: error.message, ...(error.code ? { code: error.code } : {}) }, + { status: 400 } + ) } + logger.error(`[${requestId}] Error querying rows (v2):`, error) + return NextResponse.json({ error: 'Failed to query rows' }, { status: 500 }) } -) +}) diff --git a/apps/sim/app/api/table/[tableId]/rows/route.test.ts b/apps/sim/app/api/table/[tableId]/rows/route.test.ts index 7751091551e..17596ca3735 100644 --- a/apps/sim/app/api/table/[tableId]/rows/route.test.ts +++ b/apps/sim/app/api/table/[tableId]/rows/route.test.ts @@ -228,7 +228,7 @@ describe('GET /api/table/[tableId]/rows', () => { }) }) -describe('PUT/DELETE /api/table/[tableId]/rows — PostgREST string filters', () => { +describe('PUT/DELETE /api/table/[tableId]/rows — predicate filters', () => { beforeEach(() => { vi.clearAllMocks() mockCheckAccess.mockResolvedValue({ ok: true, table: buildTable() }) @@ -254,11 +254,11 @@ describe('PUT/DELETE /api/table/[tableId]/rows — PostgREST string filters', () return DELETE(req, { params: Promise.resolve({ tableId: 'tbl_1' }) }) } - it('PUT translates a name-keyed PostgREST string to an id-keyed filter under SESSION auth', async () => { + it('PUT translates a name-keyed predicate to an id-keyed filter under SESSION auth', async () => { authAs('session') const res = await callPut({ workspaceId: 'workspace-1', - filter: 'Name=eq.Ada', + filter: { all: [{ field: 'Name', op: 'eq', value: 'Ada' }] }, data: { col_aaa: 'Grace' }, }) @@ -267,15 +267,21 @@ describe('PUT/DELETE /api/table/[tableId]/rows — PostgREST string filters', () expect(args.filter).toEqual({ $and: [{ col_aaa: 'Ada' }] }) }) - it('DELETE accepts the PostgREST string and rejects an invalid one with 400', async () => { + it('DELETE accepts the predicate and rejects an unknown column with 400', async () => { authAs('internal_jwt') - const ok = await callDelete({ workspaceId: 'workspace-1', filter: 'Age=gte.30' }) + const ok = await callDelete({ + workspaceId: 'workspace-1', + filter: { all: [{ field: 'Age', op: 'gte', value: 30 }] }, + }) expect(ok.status).toBe(200) const args = mockDeleteRowsByFilter.mock.calls[0][1] expect(args.filter).toEqual({ $and: [{ col_bbb: { $gte: 30 } }] }) - const bad = await callDelete({ workspaceId: 'workspace-1', filter: 'Age=bogus.1' }) + const bad = await callDelete({ + workspaceId: 'workspace-1', + filter: { all: [{ field: 'Nope', op: 'eq', value: 1 }] }, + }) expect(bad.status).toBe(400) - expect((await bad.json()).error).toMatch(/Unknown filter operator/) + expect((await bad.json()).error).toMatch(/Unknown filter column/) }) }) diff --git a/apps/sim/app/api/table/[tableId]/rows/route.ts b/apps/sim/app/api/table/[tableId]/rows/route.ts index 05c7fd316d0..d18362da483 100644 --- a/apps/sim/app/api/table/[tableId]/rows/route.ts +++ b/apps/sim/app/api/table/[tableId]/rows/route.ts @@ -28,29 +28,33 @@ import { import { buildIdByName, predicateNamesToIds } from '@/lib/table/column-keys' import { TableQueryValidationError } from '@/lib/table/errors' import { predicateToFilter } from '@/lib/table/query-builder/converters' -import { parsePostgrestFilter } from '@/lib/table/query-builder/postgrest' +import { validatePredicate } from '@/lib/table/query-builder/validate' import { queryRows } from '@/lib/table/rows/service' +import type { TablePredicate } from '@/lib/table/types' import { type RowWireTranslators, rowWireTranslators } from '@/app/api/table/row-wire' import { accessError, checkAccess, rowWriteErrorResponse } from '@/app/api/table/utils' const logger = createLogger('TableRowsAPI') +function isTablePredicate(raw: TablePredicate | Filter): raw is TablePredicate { + return 'all' in raw || 'any' in raw +} + /** - * Resolves a bulk-op filter to a storage-id-keyed legacy `Filter`. PostgREST - * strings are column-NAME-keyed by construction (the parser validates against - * names), so they translate names → ids unconditionally — unlike the legacy - * object form, whose keying follows the caller's wire dialect (`wire.filterIn`: - * ids from the UI, names from workflow tools). + * Resolves a bulk-op filter to a storage-id-keyed legacy `Filter`. The v2 + * predicate tree is column-NAME-keyed by construction (the caller authors + * names), so it validates then translates names → ids unconditionally — unlike + * the legacy object form, whose keying follows the caller's wire dialect + * (`wire.filterIn`: ids from the UI, names from workflow tools). */ function resolveBulkFilter( - raw: string | Filter, + raw: TablePredicate | Filter, schema: TableSchema, wire: RowWireTranslators ): Filter { - if (typeof raw === 'string') { - return predicateToFilter( - predicateNamesToIds(parsePostgrestFilter(raw, schema.columns), buildIdByName(schema)) - ) + if (isTablePredicate(raw)) { + validatePredicate(raw, schema.columns) + return predicateToFilter(predicateNamesToIds(raw, buildIdByName(schema))) } return wire.filterIn(raw) } @@ -377,7 +381,7 @@ export const PUT = withRouteHandler( table, { filter: resolveBulkFilter( - validated.filter as string | Filter, + validated.filter as TablePredicate | Filter, table.schema as TableSchema, wire ), @@ -486,7 +490,7 @@ export const DELETE = withRouteHandler( table, { filter: resolveBulkFilter( - validated.filter as string | Filter, + validated.filter as TablePredicate | Filter, table.schema as TableSchema, wire ), diff --git a/apps/sim/app/api/v1/middleware.ts b/apps/sim/app/api/v1/middleware.ts index 51d69070f32..42ab29836e0 100644 --- a/apps/sim/app/api/v1/middleware.ts +++ b/apps/sim/app/api/v1/middleware.ts @@ -12,7 +12,12 @@ import { authenticateV1Request } from '@/app/api/v1/auth' const logger = createLogger('V1Middleware') const rateLimiter = new RateLimiter() -export type V1Endpoint = +/** + * Endpoint labels for public API auth/rate-limit telemetry. Version-neutral: the + * v1 and v2 public surfaces share the same `authenticateV1Request` + `api-endpoint` + * rate bucket, so the label is only a log/metric dimension, not a policy switch. + */ +export type ApiEndpoint = | 'logs' | 'logs-detail' | 'workflows' @@ -31,6 +36,8 @@ export type V1Endpoint = | 'knowledge-detail' | 'knowledge-search' | 'copilot-chat' + | 'v2-tables' + | 'v2-table-rows' export interface RateLimitResult { allowed: boolean @@ -52,7 +59,7 @@ export interface AuthorizedRequest { export async function checkRateLimit( request: NextRequest, - endpoint: V1Endpoint = 'logs' + endpoint: ApiEndpoint = 'logs' ): Promise { try { const auth = await authenticateV1Request(request) @@ -115,7 +122,7 @@ export async function checkRateLimit( */ export async function authenticateRequest( request: NextRequest, - endpoint: V1Endpoint + endpoint: ApiEndpoint ): Promise { const requestId = generateRequestId() const rateLimit = await checkRateLimit(request, endpoint) diff --git a/apps/sim/app/api/v2/tables/[tableId]/query/route.test.ts b/apps/sim/app/api/v2/tables/[tableId]/query/route.test.ts new file mode 100644 index 00000000000..f60aac10be5 --- /dev/null +++ b/apps/sim/app/api/v2/tables/[tableId]/query/route.test.ts @@ -0,0 +1,237 @@ +/** + * @vitest-environment node + * + * Public v2 query POST: typed predicate name→id translation, bounded-default vs + * explicit-unbounded limit, regex gating, cursor validation, workspace scoping, + * and name-keyed row output. + */ +import { NextRequest } from 'next/server' +import { beforeEach, describe, expect, it, vi } from 'vitest' +import type { TableDefinition } from '@/lib/table/types' + +const { mockCheckAccess, mockQueryRows, mockCheckRateLimit, mockCheckWorkspaceScope } = vi.hoisted( + () => ({ + mockCheckAccess: vi.fn(), + mockQueryRows: vi.fn(), + mockCheckRateLimit: vi.fn(), + mockCheckWorkspaceScope: vi.fn(), + }) +) + +vi.mock('@/app/api/v1/middleware', async () => { + const { NextResponse } = await import('next/server') + return { + checkRateLimit: mockCheckRateLimit, + checkWorkspaceScope: mockCheckWorkspaceScope, + createRateLimitResponse: (r: { error?: string }) => + NextResponse.json( + { error: r.error ?? 'Rate limit exceeded' }, + { status: r.error ? 401 : 429 } + ), + } +}) + +vi.mock('@/app/api/table/utils', async () => { + const { NextResponse } = await import('next/server') + return { + checkAccess: mockCheckAccess, + accessError: (result: { status: number }) => + NextResponse.json({ error: 'Access denied' }, { status: result.status }), + } +}) + +vi.mock('@/lib/table', async () => { + const columnKeys = await import('@/lib/table/column-keys') + return { ...columnKeys } +}) + +vi.mock('@/lib/table/rows/service', () => ({ queryRows: mockQueryRows })) + +import { encodeCursor } from '@/lib/table/rows/cursor' +import { POST } from '@/app/api/v2/tables/[tableId]/query/route' + +function buildTable(): TableDefinition { + return { + id: 'tbl_1', + name: 'People', + description: null, + schema: { + columns: [ + { id: 'col_name', name: 'name', type: 'string' }, + { id: 'col_wins', name: 'wins', type: 'number' }, + { id: 'col_status', name: 'status', type: 'string' }, + ], + }, + metadata: null, + rowCount: 0, + maxRows: 100, + workspaceId: 'workspace-1', + createdBy: 'user-1', + archivedAt: null, + createdAt: new Date('2024-01-01'), + updatedAt: new Date('2024-01-01'), + } +} + +const EMPTY_RESULT = { + rows: [], + rowCount: 0, + totalCount: 0, + limit: 100, + offset: 0, + nextCursor: null, +} + +function callQuery(body: Record) { + const req = new NextRequest('http://localhost:3000/api/v2/tables/tbl_1/query', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(body), + }) + return POST(req, { params: Promise.resolve({ tableId: 'tbl_1' }) }) +} + +describe('POST /api/v2/tables/[tableId]/query', () => { + beforeEach(() => { + vi.clearAllMocks() + mockCheckRateLimit.mockResolvedValue({ + allowed: true, + userId: 'user-1', + keyType: 'workspace', + workspaceId: 'workspace-1', + }) + mockCheckWorkspaceScope.mockResolvedValue(null) + mockCheckAccess.mockResolvedValue({ ok: true, table: buildTable() }) + mockQueryRows.mockResolvedValue(EMPTY_RESULT) + }) + + it('translates a name-keyed predicate to storage ids', async () => { + const res = await callQuery({ + workspaceId: 'workspace-1', + predicate: { + all: [ + { field: 'status', op: 'eq', value: 'active' }, + { field: 'wins', op: 'gte', value: 10 }, + ], + }, + }) + expect(res.status).toBe(200) + expect(mockQueryRows.mock.calls[0][1].predicate).toEqual({ + all: [ + { field: 'col_status', op: 'eq', value: 'active' }, + { field: 'col_wins', op: 'gte', value: 10 }, + ], + }) + }) + + it('applies the bounded default limit when omitted', async () => { + await callQuery({ workspaceId: 'workspace-1' }) + expect(mockQueryRows.mock.calls[0][1].limit).toBe(100) + }) + + it('treats limit=0 as the explicit unbounded opt-in', async () => { + await callQuery({ workspaceId: 'workspace-1', limit: 0 }) + expect(mockQueryRows.mock.calls[0][1].limit).toBeUndefined() + }) + + it('rejects a limit above the max', async () => { + const res = await callQuery({ workspaceId: 'workspace-1', limit: 5000 }) + expect(res.status).toBe(400) + expect(mockQueryRows).not.toHaveBeenCalled() + }) + + it('gates regex ops (match/imatch) off the public surface', async () => { + const res = await callQuery({ + workspaceId: 'workspace-1', + predicate: { all: [{ field: 'name', op: 'match', value: '^jo' }] }, + }) + expect(res.status).toBe(400) + expect((await res.json()).error).toMatch(/Regex filters.*not supported/) + expect(mockQueryRows).not.toHaveBeenCalled() + }) + + it('rejects a predicate referencing an unknown column', async () => { + const res = await callQuery({ + workspaceId: 'workspace-1', + predicate: { all: [{ field: 'nope', op: 'eq', value: 1 }] }, + }) + expect(res.status).toBe(400) + expect((await res.json()).error).toMatch(/Unknown filter column/) + }) + + it('rejects a keyset cursor combined with a custom sort', async () => { + const cursor = encodeCursor({ + lastRow: { id: 'r1', orderKey: 'a1' }, + keysetValid: true, + nextOffset: 1, + }) + const res = await callQuery({ + workspaceId: 'workspace-1', + sort: [{ field: 'wins', direction: 'desc' }], + cursor, + }) + expect(res.status).toBe(400) + expect((await res.json()).code).toBe('CURSOR_SORT_CONFLICT') + }) + + it('returns 400 INVALID_CURSOR for a malformed cursor', async () => { + const res = await callQuery({ + workspaceId: 'workspace-1', + cursor: Buffer.from('42').toString('base64url'), + }) + expect(res.status).toBe(400) + expect((await res.json()).code).toBe('INVALID_CURSOR') + }) + + it('surfaces a workspace-scope 403 from the middleware', async () => { + const { NextResponse } = await import('next/server') + mockCheckWorkspaceScope.mockResolvedValue( + NextResponse.json({ error: 'not authorized' }, { status: 403 }) + ) + const res = await callQuery({ workspaceId: 'workspace-1' }) + expect(res.status).toBe(403) + expect(mockQueryRows).not.toHaveBeenCalled() + }) + + it('rejects a workspace-id mismatch against the table', async () => { + const res = await callQuery({ workspaceId: 'other-ws' }) + expect(res.status).toBe(400) + expect((await res.json()).error).toBe('Invalid workspace ID') + }) + + it('returns name-keyed row data with no storage internals and a private cache header', async () => { + mockQueryRows.mockResolvedValue({ + rows: [ + { + id: 'r1', + data: { col_status: 'active', col_wins: 12 }, + position: 3, + orderKey: 'a5', + executions: {}, + createdAt: new Date('2024-02-02'), + updatedAt: new Date('2024-02-03'), + }, + ], + rowCount: 1, + totalCount: 1, + limit: 100, + offset: 0, + nextCursor: null, + }) + const res = await callQuery({ workspaceId: 'workspace-1' }) + expect(res.headers.get('Cache-Control')).toBe('private, no-store') + expect((await res.json()).data.rows[0]).toEqual({ + id: 'r1', + data: { status: 'active', wins: 12 }, + createdAt: '2024-02-02T00:00:00.000Z', + updatedAt: '2024-02-03T00:00:00.000Z', + }) + }) + + it('returns the rate-limit response when the limiter denies the request', async () => { + mockCheckRateLimit.mockResolvedValue({ allowed: false }) + const res = await callQuery({ workspaceId: 'workspace-1' }) + expect(res.status).toBe(429) + expect(mockCheckAccess).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/app/api/v2/tables/[tableId]/query/route.ts b/apps/sim/app/api/v2/tables/[tableId]/query/route.ts new file mode 100644 index 00000000000..0041ff9f4ba --- /dev/null +++ b/apps/sim/app/api/v2/tables/[tableId]/query/route.ts @@ -0,0 +1,171 @@ +import { createLogger } from '@sim/logger' +import { type NextRequest, NextResponse } from 'next/server' +import { V2_DEFAULT_ROW_LIMIT, v2QueryRowsContract } from '@/lib/api/contracts/v2/tables' +import { parseRequest, validationErrorResponseFromError } from '@/lib/api/server' +import { generateRequestId } from '@/lib/core/utils/request' +import { withRouteHandler } from '@/lib/core/utils/with-route-handler' +import type { PredicateNode, Sort, TablePredicate, TableSchema } from '@/lib/table' +import { + buildIdByName, + buildNameById, + predicateNamesToIds, + rowDataIdToName, + sortSpecNamesToIds, +} from '@/lib/table' +import { TableQueryValidationError } from '@/lib/table/errors' +import { validatePredicate, validateSortSpec } from '@/lib/table/query-builder/validate' +import { decodeCursor } from '@/lib/table/rows/cursor' +import { queryRows } from '@/lib/table/rows/service' +import { accessError, checkAccess } from '@/app/api/table/utils' +import { + checkRateLimit, + checkWorkspaceScope, + createRateLimitResponse, +} from '@/app/api/v1/middleware' + +const logger = createLogger('V2TableQueryAPI') + +export const dynamic = 'force-dynamic' +export const revalidate = 0 + +/** Filters may carry user data; keep query responses out of shared caches. */ +const PRIVATE_NO_STORE = { 'Cache-Control': 'private, no-store' } as const + +interface QueryRouteParams { + params: Promise<{ tableId: string }> +} + +/** + * Regex ops (`match`/`imatch`) are gated off the public surface — a + * catastrophic-backtracking pattern can pin a shared-pool connection. Walks the + * predicate tree and throws on any regex leaf. + */ +function assertNoRegexOps(node: PredicateNode): void { + if ('all' in node || 'any' in node) { + for (const child of 'all' in node ? node.all : node.any) assertNoRegexOps(child) + return + } + if (node.op === 'match' || node.op === 'imatch') { + throw new TableQueryValidationError( + 'Regex filters (match/imatch) are not supported on the public API — use like/ilike for pattern match.', + 'INVALID_FILTER' + ) + } +} + +/** + * POST /api/v2/tables/[tableId]/query — public row query. Typed `predicate`/`sort` + * objects + opaque cursor pagination. Default page {@link V2_DEFAULT_ROW_LIMIT}; + * `limit=0` = unbounded (whole result or 400). Regex ops are gated off. + */ +export const POST = withRouteHandler(async (request: NextRequest, context: QueryRouteParams) => { + const requestId = generateRequestId() + + try { + const rateLimit = await checkRateLimit(request, 'v2-table-rows') + if (!rateLimit.allowed) return createRateLimitResponse(rateLimit) + + const userId = rateLimit.userId! + const parsed = await parseRequest(v2QueryRowsContract, request, context) + if (!parsed.success) return parsed.response + + const { tableId } = parsed.data.params + const { workspaceId, sort, cursor: cursorToken, limit } = parsed.data.body + + const scopeError = await checkWorkspaceScope(rateLimit, workspaceId) + if (scopeError) return scopeError + + const accessResult = await checkAccess(tableId, userId, 'read') + if (!accessResult.ok) return accessError(accessResult, requestId, tableId) + const { table } = accessResult + + if (workspaceId !== table.workspaceId) { + return NextResponse.json({ error: 'Invalid workspace ID' }, { status: 400 }) + } + + const schema = table.schema as TableSchema + const cursor = cursorToken ? decodeCursor(cursorToken) : undefined + + // A keyset cursor is bound to the DEFAULT `(order_key, id)` order; combining + // it with a custom sort would relabel page 1 of the sorted view as a next page. + if (cursor?.after && sort?.length) { + return NextResponse.json( + { + error: 'Cursor is not valid for a sorted query. Restart paging without the cursor.', + code: 'CURSOR_SORT_CONFLICT', + }, + { status: 400, headers: PRIVATE_NO_STORE } + ) + } + + const idByName = buildIdByName(schema) + const nameById = buildNameById(schema) + let predicate: TablePredicate | undefined = parsed.data.body.predicate + if (predicate) { + assertNoRegexOps(predicate) + validatePredicate(predicate, schema.columns) + predicate = predicateNamesToIds(predicate, idByName) + } + let sortSpec = sort + if (sortSpec?.length) { + validateSortSpec(sortSpec, schema.columns) + sortSpec = sortSpecNamesToIds(sortSpec, idByName) + } + const sortObj: Sort | undefined = sortSpec?.length + ? Object.fromEntries(sortSpec.map((s) => [s.field, s.direction])) + : undefined + + // Public default is a bounded page (unlike the internal surface's unbounded + // omit). `limit=0` is the explicit unbounded opt-in. + const effectiveLimit = + limit === undefined ? V2_DEFAULT_ROW_LIMIT : limit === 0 ? undefined : limit + + const result = await queryRows( + table, + { + predicate, + sort: sortObj, + limit: effectiveLimit, + after: cursor?.after, + offset: cursor?.offset, + includeTotal: !cursorToken, + withExecutions: false, + }, + requestId + ) + + return NextResponse.json( + { + success: true, + data: { + rows: result.rows.map((r) => ({ + id: r.id, + data: rowDataIdToName(r.data, nameById), + createdAt: + r.createdAt instanceof Date ? r.createdAt.toISOString() : String(r.createdAt), + updatedAt: + r.updatedAt instanceof Date ? r.updatedAt.toISOString() : String(r.updatedAt), + })), + rowCount: result.rowCount, + totalCount: result.totalCount, + limit: result.limit, + nextCursor: result.nextCursor, + }, + }, + { headers: PRIVATE_NO_STORE } + ) + } catch (error) { + const validationResponse = validationErrorResponseFromError(error) + if (validationResponse) return validationResponse + + if (error instanceof TableQueryValidationError) { + return NextResponse.json( + { error: error.message, ...(error.code ? { code: error.code } : {}) }, + { status: 400, headers: PRIVATE_NO_STORE } + ) + } + + logger.error(`[${requestId}] Error querying rows (v2 public):`, error) + return NextResponse.json({ error: 'Failed to query rows' }, { status: 500 }) + } +}) diff --git a/apps/sim/app/api/v2/tables/route.test.ts b/apps/sim/app/api/v2/tables/route.test.ts new file mode 100644 index 00000000000..4b2711653da --- /dev/null +++ b/apps/sim/app/api/v2/tables/route.test.ts @@ -0,0 +1,104 @@ +/** + * @vitest-environment node + * + * Public v2 tables list: auth/scope gating, typed summary output, private cache header. + */ +import { NextRequest } from 'next/server' +import { beforeEach, describe, expect, it, vi } from 'vitest' +import type { TableDefinition } from '@/lib/table/types' + +const { mockListTables, mockCheckRateLimit, mockValidateWorkspaceAccess } = vi.hoisted(() => ({ + mockListTables: vi.fn(), + mockCheckRateLimit: vi.fn(), + mockValidateWorkspaceAccess: vi.fn(), +})) + +vi.mock('@/app/api/v1/middleware', async () => { + const { NextResponse } = await import('next/server') + return { + checkRateLimit: mockCheckRateLimit, + validateWorkspaceAccess: mockValidateWorkspaceAccess, + createRateLimitResponse: (r: { error?: string }) => + NextResponse.json( + { error: r.error ?? 'Rate limit exceeded' }, + { status: r.error ? 401 : 429 } + ), + } +}) + +vi.mock('@/lib/table', async () => { + const actual = await import('@/lib/table/column-keys') + return { ...actual, listTables: mockListTables } +}) + +import { GET } from '@/app/api/v2/tables/route' + +function buildTable(): TableDefinition { + return { + id: 'tbl_1', + name: 'People', + description: 'A table', + schema: { columns: [{ id: 'col_name', name: 'name', type: 'string' }] }, + metadata: null, + rowCount: 5, + maxRows: 100, + workspaceId: 'workspace-1', + createdBy: 'user-1', + archivedAt: null, + createdAt: new Date('2024-01-01'), + updatedAt: new Date('2024-01-02'), + } +} + +function callList(query: string) { + const req = new NextRequest(`http://localhost:3000/api/v2/tables?${query}`) + return GET(req) +} + +describe('GET /api/v2/tables', () => { + beforeEach(() => { + vi.clearAllMocks() + mockCheckRateLimit.mockResolvedValue({ allowed: true, userId: 'user-1', keyType: 'workspace' }) + mockValidateWorkspaceAccess.mockResolvedValue(null) + mockListTables.mockResolvedValue([buildTable()]) + }) + + it('returns a typed table summary with a private cache header', async () => { + const res = await callList('workspaceId=workspace-1') + expect(res.status).toBe(200) + expect(res.headers.get('Cache-Control')).toBe('private, no-store') + const body = await res.json() + expect(body.data.totalCount).toBe(1) + expect(body.data.tables[0]).toMatchObject({ + id: 'tbl_1', + name: 'People', + description: 'A table', + rowCount: 5, + maxRows: 100, + createdAt: '2024-01-01T00:00:00.000Z', + }) + expect(body.data.tables[0].schema.columns[0].name).toBe('name') + }) + + it('400s when workspaceId is missing', async () => { + const res = await callList('') + expect(res.status).toBe(400) + expect(mockListTables).not.toHaveBeenCalled() + }) + + it('surfaces an access-denied response from the middleware', async () => { + const { NextResponse } = await import('next/server') + mockValidateWorkspaceAccess.mockResolvedValue( + NextResponse.json({ error: 'Access denied' }, { status: 403 }) + ) + const res = await callList('workspaceId=workspace-1') + expect(res.status).toBe(403) + expect(mockListTables).not.toHaveBeenCalled() + }) + + it('returns the rate-limit response when denied', async () => { + mockCheckRateLimit.mockResolvedValue({ allowed: false }) + const res = await callList('workspaceId=workspace-1') + expect(res.status).toBe(429) + }) +}) diff --git a/apps/sim/app/api/v2/tables/route.ts b/apps/sim/app/api/v2/tables/route.ts new file mode 100644 index 00000000000..87b4da8d014 --- /dev/null +++ b/apps/sim/app/api/v2/tables/route.ts @@ -0,0 +1,73 @@ +import { createLogger } from '@sim/logger' +import { type NextRequest, NextResponse } from 'next/server' +import { v2ListTablesContract } from '@/lib/api/contracts/v2/tables' +import { parseRequest, validationErrorResponseFromError } from '@/lib/api/server' +import { generateRequestId } from '@/lib/core/utils/request' +import { withRouteHandler } from '@/lib/core/utils/with-route-handler' +import { listTables, type TableSchema } from '@/lib/table' +import { normalizeColumn } from '@/app/api/table/utils' +import { + checkRateLimit, + createRateLimitResponse, + validateWorkspaceAccess, +} from '@/app/api/v1/middleware' + +const logger = createLogger('V2TablesAPI') + +export const dynamic = 'force-dynamic' +export const revalidate = 0 + +/** Filters/ids can appear in query strings; keep list responses out of shared caches. */ +const PRIVATE_NO_STORE = { 'Cache-Control': 'private, no-store' } as const + +/** GET /api/v2/tables — list all tables in a workspace. */ +export const GET = withRouteHandler(async (request: NextRequest) => { + const requestId = generateRequestId() + + try { + const rateLimit = await checkRateLimit(request, 'v2-tables') + if (!rateLimit.allowed) return createRateLimitResponse(rateLimit) + + const userId = rateLimit.userId! + const parsed = await parseRequest(v2ListTablesContract, request, {}) + if (!parsed.success) return parsed.response + + const { workspaceId } = parsed.data.query + + const accessError = await validateWorkspaceAccess(rateLimit, userId, workspaceId) + if (accessError) return accessError + + const tables = await listTables(workspaceId) + + return NextResponse.json( + { + success: true, + data: { + tables: tables.map((t) => { + const schemaData = t.schema as TableSchema + return { + id: t.id, + name: t.name, + description: t.description, + schema: { columns: schemaData.columns.map(normalizeColumn) }, + rowCount: t.rowCount, + maxRows: t.maxRows, + createdAt: + t.createdAt instanceof Date ? t.createdAt.toISOString() : String(t.createdAt), + updatedAt: + t.updatedAt instanceof Date ? t.updatedAt.toISOString() : String(t.updatedAt), + } + }), + totalCount: tables.length, + }, + }, + { headers: PRIVATE_NO_STORE } + ) + } catch (error) { + const validationResponse = validationErrorResponseFromError(error) + if (validationResponse) return validationResponse + + logger.error(`[${requestId}] Error listing tables:`, error) + return NextResponse.json({ error: 'Failed to list tables' }, { status: 500 }) + } +}) diff --git a/apps/sim/blocks/blocks/table_v2.test.ts b/apps/sim/blocks/blocks/table_v2.test.ts index d1cf64970b3..b333d77542c 100644 --- a/apps/sim/blocks/blocks/table_v2.test.ts +++ b/apps/sim/blocks/blocks/table_v2.test.ts @@ -40,43 +40,42 @@ describe('table_v2 query_rows transformer', () => { expect(out3.cursor).toBe('tok') }) - it('serializes builder rules to PostgREST when the builder holds rules', () => { + it('compiles builder rules (basic canonical value) to a predicate + sort spec', () => { const out = params({ operation: 'query_rows', tableId: 't', limit: '10', - filterMode: 'builder', - filterBuilder: [ + // filterInput/sortInput carry the ACTIVE canonical mode's value; the visual + // builders (basic) emit rule arrays. + filterInput: [ { id: '1', logicalOperator: 'and', column: 'wins', operator: 'gte', value: '10' }, ], - sortBuilder: [{ id: '1', column: 'wins', direction: 'desc' }], - filter: 'name=eq.ignored', + sortInput: [{ id: '1', column: 'wins', direction: 'desc' }], }) - expect(out.filter).toBe('wins=gte.10') - expect(out.order).toBe('wins.desc') + expect(out.filter).toEqual({ all: [{ field: 'wins', op: 'gte', value: 10 }] }) + expect(out.order).toEqual([{ field: 'wins', direction: 'desc' }]) }) - it('falls back to the raw PostgREST string when the builder value is not a non-empty array', () => { + it('parses the editor JSON string (advanced canonical value) into a predicate', () => { const out = params({ operation: 'query_rows', tableId: 't', limit: '10', - filterMode: 'builder', - filterBuilder: {}, - filter: 'name=eq.test', + filterInput: '{"all":[{"field":"name","op":"eq","value":"test"}]}', }) - expect(out.filter).toBe('name=eq.test') + expect(out.filter).toEqual({ all: [{ field: 'name', op: 'eq', value: 'test' }] }) }) }) describe('table_v2 bulk transformers', () => { + const editorFilter = '{"all":[{"field":"name","op":"eq","value":"x"}]}' + it('fails fast on a non-numeric bulk limit instead of widening to every match', () => { expect(() => params({ operation: 'delete_rows_by_filter', tableId: 't', - filterMode: 'editor', - filter: 'name=eq.x', + filterInput: editorFilter, limit: 'abc', }) ).toThrow(/Invalid Limit/) @@ -86,10 +85,9 @@ describe('table_v2 bulk transformers', () => { const out = params({ operation: 'delete_rows_by_filter', tableId: 't', - filterMode: 'editor', - filter: 'name=eq.x', + filterInput: editorFilter, }) expect(out.limit).toBeUndefined() - expect(out.filter).toBe('name=eq.x') + expect(out.filter).toEqual({ all: [{ field: 'name', op: 'eq', value: 'x' }] }) }) }) diff --git a/apps/sim/blocks/blocks/table_v2.ts b/apps/sim/blocks/blocks/table_v2.ts index 5857416c1df..ce9a92656cc 100644 --- a/apps/sim/blocks/blocks/table_v2.ts +++ b/apps/sim/blocks/blocks/table_v2.ts @@ -1,18 +1,15 @@ import { toError } from '@sim/utils/errors' import { TableIcon } from '@/components/icons' import { TABLE_LIMITS } from '@/lib/table/constants' -import { - filterRulesToPostgrest, - sortRulesToPostgrestOrder, -} from '@/lib/table/query-builder/converters' -import type { FilterRule, SortRule } from '@/lib/table/types' +import { filterRulesToPredicate, sortRulesToSortSpec } from '@/lib/table/query-builder/converters' +import type { FilterRule, SortRule, SortSpec, TablePredicate } from '@/lib/table/types' import type { BlockConfig } from '@/blocks/types' import type { TableQueryV2Response } from '@/tools/table/types' import { getTrigger } from '@/triggers' /** - * Table v2 — same operations as the v1 Table block, but the filter grammar is - * PostgREST (`wins=gte.10&status=in.(active,pending)`), parsed + validated + * Table v2 — same operations as the v1 Table block, but the filter grammar is a + * typed predicate tree (`{all:[{field:'wins',op:'gte',value:10}]}`), validated * server-side. Pagination is an opaque cursor (no offset). The filter compiler, * upsert conflict probe, and unique checks share one case-sensitive containment * leaf, so upserts can't wedge on a case-mismatched unique value the way they @@ -44,47 +41,36 @@ interface TableBlockParams { rowId?: string data?: string | unknown rows?: string | unknown - filterMode?: string - filterBuilder?: unknown - sortBuilder?: unknown - filter?: string - order?: string + filterInput?: unknown + sortInput?: unknown limit?: string cursor?: string conflictColumn?: string } /** - * Resolves the effective PostgREST filter/order from either the visual builders - * (when Filter Mode is "builder") or the raw querystring fields. Both modes - * serialize to the same PostgREST wire format the route parses. Mode defaults to - * "builder" to match the dropdown default. + * Resolves the effective predicate from the `filterInput` canonical param, which + * carries whichever mode's value the ⟷ toggle has active: a FilterRule[] (visual + * builder / basic) or a predicate-JSON string (editor / advanced, agent-authored). + * An array is builder rules; a string is raw JSON. Both compile to the same + * `TablePredicate` the route consumes. */ -function resolveFilter(params: TableBlockParams): string | undefined { - // Serialize the visual builder only when it holds real rules. The builder - // shape is a UI affordance — the agent authors the PostgREST `filter` string - // directly and never populates it, so anything that isn't a non-empty - // FilterRule[] falls back to the string (never iterate a non-array, which - // threw "rules is not iterable" at runtime). - if ( - params.filterMode !== 'editor' && - Array.isArray(params.filterBuilder) && - params.filterBuilder.length > 0 - ) { - return filterRulesToPostgrest(params.filterBuilder as FilterRule[]) ?? undefined +function resolveFilter(params: TableBlockParams): TablePredicate | undefined { + const raw = params.filterInput + if (Array.isArray(raw)) { + return raw.length > 0 ? (filterRulesToPredicate(raw as FilterRule[]) ?? undefined) : undefined } - return params.filter || undefined + const parsed = parseJSON(raw, 'Filter') + return (parsed as TablePredicate | undefined) || undefined } -function resolveOrder(params: TableBlockParams): string | undefined { - if ( - params.filterMode !== 'editor' && - Array.isArray(params.sortBuilder) && - params.sortBuilder.length > 0 - ) { - return sortRulesToPostgrestOrder(params.sortBuilder as SortRule[]) ?? undefined +function resolveOrder(params: TableBlockParams): SortSpec | undefined { + const raw = params.sortInput + if (Array.isArray(raw)) { + return raw.length > 0 ? (sortRulesToSortSpec(raw as SortRule[]) ?? undefined) : undefined } - return params.order || undefined + const parsed = parseJSON(raw, 'Sort') + return (parsed as SortSpec | undefined) || undefined } interface ParsedParams { @@ -92,8 +78,8 @@ interface ParsedParams { rowId?: string data?: unknown rows?: unknown - filter?: string - order?: string + filter?: TablePredicate + order?: SortSpec limit?: number cursor?: string conflictTarget?: string @@ -144,7 +130,7 @@ const paramTransformers: Record ParsedPara data: parseJSON(params.data, 'Row Data'), }), - // Bulk write-by-filter takes a PostgREST string, parsed server-side. + // Bulk write-by-filter takes a predicate object, validated server-side. update_rows_by_filter: (params) => ({ tableId: params.tableId, filter: resolveFilter(params), @@ -189,20 +175,21 @@ export const TableV2Block: BlockConfig = { description: 'User-defined data tables', longDescription: 'Create and manage custom data tables. Store, query, and manipulate structured data within workflows. ' + - 'Query Rows filters with a PostgREST querystring — `wins=gte.10&status=in.(active,pending)` (top-level ' + - 'params AND; `or=(a.eq.1,b.eq.2)` / `and=(...)` for groups). Operators: eq, neq, gt, gte, lt, lte, in, ' + - 'like, ilike, match, imatch, is.null (negate with not., e.g. not.in, not.is.null). Order is `wins.desc,name.asc`. ' + - 'Query Rows returns every matching row when Limit is omitted (fails if the result exceeds 5MB — add a filter ' + - 'or a Limit). With a Limit, responses page: a non-null nextCursor means more rows exist — pass it back as the cursor.', + 'Query Rows filters with a predicate tree — `{"all":[{"field":"wins","op":"gte","value":10}]}` ' + + '(`all` = AND, `any` = OR; groups nest). Operators: eq, ne, gt, gte, lt, lte, in, nin, like, ilike, ' + + 'nlike, nilike, contains, startsWith, endsWith, isNull, isNotNull, isEmpty, isNotEmpty. Order is a sort ' + + 'spec `[{"field":"wins","direction":"desc"}]`. Query Rows returns every matching row when Limit is omitted ' + + '(fails if the result exceeds 5MB — add a filter or a Limit). With a Limit, responses page: a non-null ' + + 'nextCursor means more rows exist — pass it back as the cursor.', bestPractices: ` -- To fetch specific rows, use Query Rows with a PostgREST filter (e.g. slack_user_id=in.(U1,U2)) — do NOT read every row and filter downstream with a Condition block. -- Use "Get Row by ID" only when you have the row's id; otherwise filter with the querystring. -- Top-level params AND together; use or=(...) / and=(...) for explicit or nested logic. -- Example: players who won ≥10 and are active → wins=gte.10&status=eq.active. -- like/ilike use * as the wildcard (e.g. name=ilike.*jo*); match/imatch are regex. +- To fetch specific rows, use Query Rows with a predicate filter (e.g. {"all":[{"field":"slack_user_id","op":"in","value":["U1","U2"]}]}) — do NOT read every row and filter downstream with a Condition block. +- Use "Get Row by ID" only when you have the row's id; otherwise filter with a predicate. +- A group is {"all":[...]} (AND) or {"any":[...]} (OR); nest groups as members for mixed logic. +- Example: players who won ≥10 and are active → {"all":[{"field":"wins","op":"gte","value":10},{"field":"status","op":"eq","value":"active"}]}. +- like/ilike use * as the wildcard (e.g. {"field":"name","op":"ilike","value":"*jo*"}). - Omit Limit to get the entire matching result in one response — the query fails with a clear error if it exceeds 5MB (narrow with a filter or set a Limit). - With a Limit, pages can end at the Limit or the 5MB byte budget, whichever comes first — pass nextCursor back as the cursor and loop until it is null; never infer completion from page size. -- Columns are scalar (string/number/boolean/date) or opaque json — there are no array columns, so cs/cd/ov (containment/overlap) are unsupported; for substring use ilike.*x*.`, +- Columns are scalar (string/number/boolean/date) or opaque json — there are no array columns; for substring use ilike with *x*.`, docsLink: 'https://docs.simstudio.ai/tools/table', category: 'blocks', bgColor: '#10B981', @@ -336,109 +323,80 @@ Return ONLY the rows array:`, }, }, - // Filter mode — visual builders vs raw PostgREST querystring (query + bulk). - // Builder UI is human-only; the agent authors the PostgREST `filter`/`order` - // strings, so the mode + builders are hidden from the tool-input context. - { - id: 'filterMode', - title: 'Filter Mode', - type: 'dropdown', - paramVisibility: 'user-only', - options: [ - { label: 'Builder', id: 'builder' }, - { label: 'Editor', id: 'editor' }, - ], - value: () => 'builder', - condition: { - field: 'operation', - value: ['query_rows', 'update_rows_by_filter', 'delete_rows_by_filter'], - }, - }, - - // Visual filter builder (builder mode) — query + bulk update/delete + // Filter — canonical Builder⟷Editor toggle (the ⟷ swap icon on the field), + // query + bulk update/delete. Basic = visual builder (human-only); Advanced + // = predicate JSON the agent authors directly. Both members are deliberately + // NOT `required` (the canonical group must share required status): the service + // fails closed when no filter resolves for a bulk op ("Filter is required..."). { id: 'filterBuilder', - title: 'Filter Conditions', + title: 'Filter', type: 'filter-builder', + canonicalParamId: 'filterInput', + mode: 'basic', paramVisibility: 'user-only', - // Deliberately NOT `required`: an agent can satisfy the bulk ops with the - // `filter` string while filterMode sits at its 'builder' default, and the - // serializer hard-blocks execution on missing required fields. The service - // itself fails closed when no filter resolves ("Filter is required..."). condition: { field: 'operation', value: ['query_rows', 'update_rows_by_filter', 'delete_rows_by_filter'], - and: { field: 'filterMode', value: 'builder' }, }, }, - - // Visual sort builder (builder mode) — query only - { - id: 'sortBuilder', - title: 'Sort Order', - type: 'sort-builder', - paramVisibility: 'user-only', - condition: { - field: 'operation', - value: 'query_rows', - and: { field: 'filterMode', value: 'builder' }, - }, - }, - - // Filter — PostgREST querystring (editor mode), query + bulk update/delete { id: 'filter', title: 'Filter', - type: 'long-input', - rows: 2, - placeholder: 'wins=gte.10&status=in.(active,pending)', - required: { - field: 'operation', - value: ['update_rows_by_filter', 'delete_rows_by_filter'], - and: { field: 'filterMode', value: 'editor' }, - }, + type: 'code', + canonicalParamId: 'filterInput', + mode: 'advanced', + placeholder: '{"all":[{"field":"wins","op":"gte","value":10}]}', condition: { field: 'operation', value: ['query_rows', 'update_rows_by_filter', 'delete_rows_by_filter'], - and: { field: 'filterMode', value: 'editor' }, }, wandConfig: { enabled: true, maintainHistory: true, - prompt: `Generate a PostgREST filter querystring for selecting rows. + prompt: `Generate a predicate filter object (JSON) for selecting rows. ### CONTEXT {context} ### INSTRUCTION -Return ONLY the querystring. No explanations, surrounding quotes, or markdown. +Return ONLY the JSON object. No explanations, surrounding quotes, or markdown. -Top-level params join with & (AND). Use or=(a.op.v,b.op.v) / and=(...) for OR or nested logic. +A predicate is a tree: {"all":[...]} (AND) or {"any":[...]} (OR); members are leaves {"field","op","value"} or nested groups. ### OPERATORS -eq, neq, gt, gte, lt, lte; in.(a,b); like / ilike (use * as the wildcard); match / imatch (regex); is.null. Negate with not. (e.g. not.in.(...), not.is.null). +eq, ne, gt, gte, lt, lte, in, nin (in/nin take an array value), like, ilike (use * as the wildcard), nlike, nilike, contains, startsWith, endsWith, isNull, isNotNull, isEmpty, isNotEmpty. ### EXAMPLES -"status is active" → status=eq.active -"wins at least 10 and active" → wins=gte.10&active=is.true -"status active or pending" → or=(status.eq.active,status.eq.pending) -"name contains jo (any case)" → name=ilike.*jo* +"status is active" → {"all":[{"field":"status","op":"eq","value":"active"}]} +"wins at least 10 and active" → {"all":[{"field":"wins","op":"gte","value":10},{"field":"active","op":"eq","value":true}]} +"status active or pending" → {"any":[{"field":"status","op":"eq","value":"active"},{"field":"status","op":"eq","value":"pending"}]} +"name contains jo (any case)" → {"all":[{"field":"name","op":"ilike","value":"*jo*"}]} -Return ONLY the querystring:`, +Return ONLY the JSON object:`, generationType: 'table-schema', }, }, - // Order — PostgREST order (editor mode), query only + + // Order — canonical Builder⟷Editor toggle, query only. Basic = visual sort + // builder (human-only); Advanced = sort-spec JSON the agent authors. + { + id: 'sortBuilder', + title: 'Order', + type: 'sort-builder', + canonicalParamId: 'sortInput', + mode: 'basic', + paramVisibility: 'user-only', + condition: { field: 'operation', value: 'query_rows' }, + }, { id: 'order', title: 'Order', type: 'short-input', - placeholder: 'wins.desc,name.asc', - condition: { - field: 'operation', - value: 'query_rows', - and: { field: 'filterMode', value: 'editor' }, - }, + canonicalParamId: 'sortInput', + mode: 'advanced', + placeholder: '[{"field":"wins","direction":"desc"}]', + condition: { field: 'operation', value: 'query_rows' }, }, { id: 'limit', @@ -506,15 +464,16 @@ Return ONLY the querystring:`, data: { type: 'json', description: 'Row data for insert/update' }, rows: { type: 'array', description: 'Array of row data for batch insert' }, rowId: { type: 'string', description: 'Row identifier for ID-based operations' }, - filterMode: { type: 'string', description: 'Filter input mode: builder or editor' }, - filterBuilder: { type: 'json', description: 'Visual filter builder conditions' }, - sortBuilder: { type: 'json', description: 'Visual sort builder conditions' }, - filter: { - type: 'string', + filterInput: { + type: 'json', + description: + 'Filter — a predicate object {"all":[{"field":"wins","op":"gte","value":10}]} (or visual builder conditions). Used by query and bulk update/delete.', + }, + sortInput: { + type: 'json', description: - 'PostgREST filter querystring, e.g. wins=gte.10&status=in.(active,pending). Used by query and bulk update/delete.', + 'Order — a sort spec [{"field":"wins","direction":"desc"}] (or visual sort conditions). Query only.', }, - order: { type: 'string', description: 'PostgREST order, e.g. wins.desc,name.asc' }, limit: { type: 'number', description: diff --git a/apps/sim/lib/api/contracts/tables-predicate.test.ts b/apps/sim/lib/api/contracts/tables-predicate.test.ts index 1d56d50d00d..f576af54cf9 100644 --- a/apps/sim/lib/api/contracts/tables-predicate.test.ts +++ b/apps/sim/lib/api/contracts/tables-predicate.test.ts @@ -1,60 +1,87 @@ /** * @vitest-environment node * - * The v2 query/bulk filter wire format is now a PostgREST string (parsed + - * validated server-side by `parsePostgrestFilter`). The contract only bounds the - * string; grammar validation lives in the parser tests (`postgrest.test.ts`). + * The v2 query/bulk filter wire format is the typed `{ all | any: [...] }` + * predicate tree. The contract validates structure; column-level validation + * (unknown field, json-op) runs server-side in `validate.ts`. */ import { describe, expect, it } from 'vitest' import { deleteTableRowsBodySchema, - queryTableRowsV2BodySchema, + rowQueryBodySchema, updateRowsByFilterBodySchema, } from '@/lib/api/contracts/tables' -describe('queryTableRowsV2BodySchema', () => { - it('accepts a PostgREST filter/order string, leaves limit unbounded, has no offset', () => { - const parsed = queryTableRowsV2BodySchema.parse({ +describe('rowQueryBodySchema', () => { + it('accepts a predicate/sort object, leaves limit unbounded, has no offset', () => { + const parsed = rowQueryBodySchema.parse({ workspaceId: 'ws-1', - filter: 'wins=gte.10&status=in.(active,pending)', - order: 'wins.desc', + predicate: { + all: [ + { field: 'wins', op: 'gte', value: 10 }, + { field: 'status', op: 'in', value: ['active', 'pending'] }, + ], + }, + sort: [{ field: 'wins', direction: 'desc' }], cursor: 'abc', }) - expect(parsed.filter).toBe('wins=gte.10&status=in.(active,pending)') + expect(parsed.predicate).toEqual({ + all: [ + { field: 'wins', op: 'gte', value: 10 }, + { field: 'status', op: 'in', value: ['active', 'pending'] }, + ], + }) // Omitted limit stays undefined — the query returns all matching rows. expect(parsed.limit).toBeUndefined() expect('offset' in parsed).toBe(false) }) - it('allows omitting the filter (match all)', () => { - expect(queryTableRowsV2BodySchema.safeParse({ workspaceId: 'ws-1' }).success).toBe(true) + it('accepts a nested any/all predicate', () => { + expect( + rowQueryBodySchema.safeParse({ + workspaceId: 'ws-1', + predicate: { + any: [ + { field: 'status', op: 'eq', value: 'active' }, + { all: [{ field: 'wins', op: 'gte', value: 5 }] }, + ], + }, + }).success + ).toBe(true) + }) + + it('allows omitting the predicate (match all)', () => { + expect(rowQueryBodySchema.safeParse({ workspaceId: 'ws-1' }).success).toBe(true) }) - it('rejects an empty filter string and an over-long one', () => { - expect(queryTableRowsV2BodySchema.safeParse({ workspaceId: 'ws-1', filter: '' }).success).toBe( - false - ) + it('rejects an unknown operator and a malformed leaf', () => { expect( - queryTableRowsV2BodySchema.safeParse({ workspaceId: 'ws-1', filter: 'x'.repeat(5000) }) - .success + rowQueryBodySchema.safeParse({ + workspaceId: 'ws-1', + predicate: { all: [{ field: 'wins', op: 'bogus', value: 1 }] }, + }).success + ).toBe(false) + expect( + rowQueryBodySchema.safeParse({ + workspaceId: 'ws-1', + predicate: { all: [{ op: 'eq', value: 1 }] }, + }).success ).toBe(false) }) it('accepts a large explicit limit (no row cap) but rejects limit < 1', () => { - expect( - queryTableRowsV2BodySchema.safeParse({ workspaceId: 'ws-1', limit: 100000 }).success - ).toBe(true) - expect(queryTableRowsV2BodySchema.safeParse({ workspaceId: 'ws-1', limit: 0 }).success).toBe( - false - ) + expect(rowQueryBodySchema.safeParse({ workspaceId: 'ws-1', limit: 100000 }).success).toBe(true) + expect(rowQueryBodySchema.safeParse({ workspaceId: 'ws-1', limit: 0 }).success).toBe(false) }) }) -describe('bulk schemas accept either a PostgREST string or the legacy filter object', () => { - it('delete accepts a PostgREST string filter', () => { +describe('bulk schemas accept either a predicate tree or the legacy filter object', () => { + it('delete accepts a predicate filter', () => { expect( - deleteTableRowsBodySchema.safeParse({ workspaceId: 'ws-1', filter: 'status=eq.archived' }) - .success + deleteTableRowsBodySchema.safeParse({ + workspaceId: 'ws-1', + filter: { all: [{ field: 'status', op: 'eq', value: 'archived' }] }, + }).success ).toBe(true) }) @@ -65,11 +92,11 @@ describe('bulk schemas accept either a PostgREST string or the legacy filter obj ).toBe(true) }) - it('update accepts a PostgREST string filter', () => { + it('update accepts a predicate filter', () => { expect( updateRowsByFilterBodySchema.safeParse({ workspaceId: 'ws-1', - filter: 'wins=gte.10', + filter: { all: [{ field: 'wins', op: 'gte', value: 10 }] }, data: { active: false }, }).success ).toBe(true) diff --git a/apps/sim/lib/api/contracts/tables.ts b/apps/sim/lib/api/contracts/tables.ts index 5e092a48f50..67eca8ef7b2 100644 --- a/apps/sim/lib/api/contracts/tables.ts +++ b/apps/sim/lib/api/contracts/tables.ts @@ -5,14 +5,23 @@ import type { CsvHeaderMapping, EnrichmentRunDetail, Filter, + PredicateNode, RowData, Sort, + SortSpec, TableDefinition, TableMetadata, + TablePredicate, TableRow, TableRowsCursor, } from '@/lib/table' -import { COLUMN_TYPES, NAME_PATTERN, TABLE_LIMITS } from '@/lib/table/constants' +import { + COLUMN_TYPES, + FILTER_OPS, + NAME_PATTERN, + SORT_DIRECTIONS, + TABLE_LIMITS, +} from '@/lib/table/constants' import { CSV_MAX_FILE_SIZE_BYTES } from '@/lib/table/import' export const domainObjectSchema = () => z.custom(isRecordLike) @@ -260,29 +269,52 @@ const nonEmptyFilterSchema = domainObjectSchema().refine( const filterSchema = domainObjectSchema() -/* --------------------------- v2 PostgREST grammar --------------------------- */ +/* --------------------------- v2 predicate grammar --------------------------- */ -const MAX_FILTER_LENGTH = 4096 +/** Max members in one `all`/`any` group — a generous bound against pathological trees. */ +const MAX_PREDICATE_GROUP_SIZE = 100 +/** Max sort keys — more than a few is already a smell. */ +const MAX_SORT_KEYS = 16 /** - * v2 filter wire format: a PostgREST querystring fragment (e.g. - * `wins=gte.10&status=in.(active,pending)`), parsed + validated server-side by - * `parsePostgrestFilter`. The boundary only bounds length; the parser is the - * semantic validator. + * v2 filter wire format: the typed `{ all | any: [...] }` predicate tree (same + * shape the engine consumes). Structure is validated here; schema-awareness + * (unknown column, json-op rejection) is enforced server-side by + * `validatePredicate` once the table's columns are known. */ -const postgrestFilterSchema = z - .string() - .min(1, 'filter must be a non-empty PostgREST querystring') - .max(MAX_FILTER_LENGTH, `filter cannot exceed ${MAX_FILTER_LENGTH} characters`) +const predicateLeafSchema = z.object({ + field: z.string().min(1, 'field is required').max(128), + op: z.enum(FILTER_OPS), + value: z.unknown().optional(), +}) + +const predicateNodeSchema: z.ZodType = z.lazy(() => + z.union([predicateGroupSchema, predicateLeafSchema]) +) -/** v2 order wire format: PostgREST `order` (e.g. `wins.desc,name.asc`). */ -const postgrestOrderSchema = z.string().min(1).max(512) +export const predicateSchema: z.ZodType = z.lazy(() => + z.union([ + z.object({ all: z.array(predicateNodeSchema).max(MAX_PREDICATE_GROUP_SIZE) }), + z.object({ any: z.array(predicateNodeSchema).max(MAX_PREDICATE_GROUP_SIZE) }), + ]) +) +const predicateGroupSchema = predicateSchema + +/** v2 sort wire format: an ordered list of `{ field, direction }`. */ +export const sortSpecSchema: z.ZodType = z + .array( + z.object({ + field: z.string().min(1, 'field is required').max(128), + direction: z.enum(SORT_DIRECTIONS), + }) + ) + .max(MAX_SORT_KEYS) /** - * Bulk update/delete filter accepts either the v2 PostgREST string (v2 block / + * Bulk update/delete filter accepts either the v2 predicate tree (v2 block / * agent) or the legacy `$`-operator object (v1 callers / stored workflows). */ -const bulkFilterSchema = z.union([postgrestFilterSchema, nonEmptyFilterSchema]) +const bulkFilterSchema = z.union([predicateSchema, nonEmptyFilterSchema]) const optionalPositiveLimit = (max: number, label: string) => z.preprocess( @@ -558,15 +590,15 @@ export const listTableRowsContract = defineRouteContract({ }) /** - * v2 query surface: PostgREST filter/order strings + opaque cursor pagination. + * v2 query surface: typed `predicate`/`sort` objects + opaque cursor pagination. * No `offset` (the cursor encodes paging state) and no after/sort refine (the - * cursor carries the sort context). `filter`/`order` are PostgREST strings, - * parsed server-side. POST so the (potentially long) filter rides the body. + * cursor carries the sort context). Structure is validated here; column-level + * validation (unknown field, json-op) runs server-side via `validatePredicate`. */ -export const queryTableRowsV2BodySchema = z.object({ +export const rowQueryBodySchema = z.object({ workspaceId: z.string().min(1, 'Workspace ID is required'), - filter: postgrestFilterSchema.optional(), - order: postgrestOrderSchema.optional(), + predicate: predicateSchema.optional(), + sort: sortSpecSchema.optional(), // Omitted limit returns the ENTIRE matching result, failing fast (400) when // it exceeds the response byte budget. An explicit limit caps the page row // count; the byte budget may still end a page early with nextCursor set. @@ -581,13 +613,13 @@ export const queryTableRowsV2BodySchema = z.object({ cursor: z.string().min(1, 'cursor must be a non-empty token').optional(), }) -export type QueryTableRowsV2Body = z.input +export type RowQueryBody = z.input -export const queryTableRowsV2Contract = defineRouteContract({ +export const rowQueryContract = defineRouteContract({ method: 'POST', path: '/api/table/[tableId]/query', params: tableIdParamsSchema, - body: queryTableRowsV2BodySchema, + body: rowQueryBodySchema, response: { mode: 'json', schema: successResponseSchema( @@ -601,7 +633,7 @@ export const queryTableRowsV2Contract = defineRouteContract({ ), }, }) -export type QueryTableRowsV2Response = ContractJsonResponse +export type RowQueryResponse = ContractJsonResponse export const findTableRowsQuerySchema = z.object({ workspaceId: z.string().min(1, 'Workspace ID is required'), diff --git a/apps/sim/lib/api/contracts/v2/tables/index.ts b/apps/sim/lib/api/contracts/v2/tables/index.ts new file mode 100644 index 00000000000..34e3902f01d --- /dev/null +++ b/apps/sim/lib/api/contracts/v2/tables/index.ts @@ -0,0 +1,115 @@ +import { z } from 'zod' +import { workspaceIdSchema } from '@/lib/api/contracts/primitives' +import { + predicateSchema, + sortSpecSchema, + tableColumnSchema, + tableIdParamsSchema, +} from '@/lib/api/contracts/tables' +import { defineRouteContract } from '@/lib/api/contracts/types' + +/** + * Public v2 tables API — typed predicate grammar, cursor paging. + * + * Filters are the `{ all | any: [...] }` predicate tree (same shape the engine + * consumes); no string querystring dialect. Response bodies are fully typed. Row + * data is name-keyed and carries no storage internals (`position`/`orderKey`/ + * `executions`) — the public wire is `{ id, data, createdAt, updatedAt }`. + */ + +/** Default page size when `limit` is omitted (bounded, unlike the internal surface). */ +export const V2_DEFAULT_ROW_LIMIT = 100 +/** Hard cap on an explicit page `limit`. Larger pulls use `limit=0` or the async export. */ +export const V2_MAX_ROW_LIMIT = 1000 + +const successResponseSchema = (dataSchema: T) => + z.object({ + success: z.literal(true), + data: dataSchema, + }) + +const v2TableSummarySchema = z.object({ + id: z.string(), + name: z.string(), + description: z.string().nullable(), + schema: z.object({ columns: z.array(tableColumnSchema) }), + rowCount: z.number(), + maxRows: z.number(), + createdAt: z.string(), + updatedAt: z.string(), +}) + +/** Public row shape: name-keyed data, no storage internals. */ +export const v2TableRowSchema = z.object({ + id: z.string(), + data: z.record(z.string(), z.unknown()), + createdAt: z.string(), + updatedAt: z.string(), +}) + +export const v2ListTablesQuerySchema = z.object({ + workspaceId: workspaceIdSchema, +}) + +/** + * Rows query body. `predicate`/`sort` are the typed predicate tree / sort spec. + * `limit`: omitted → {@link V2_DEFAULT_ROW_LIMIT}; `0` → unbounded (whole result + * or a 400 `TABLE_QUERY_RESULT_TOO_LARGE`); `1..{@link V2_MAX_ROW_LIMIT}` → page cap. + */ +export const v2QueryRowsBodySchema = z.object({ + workspaceId: workspaceIdSchema, + predicate: predicateSchema.optional(), + sort: sortSpecSchema.optional(), + limit: z + .number({ error: 'Limit must be a number' }) + .int('Limit must be an integer') + .min(0, 'Limit must be at least 0 (use 0 for an unbounded query)') + .max( + V2_MAX_ROW_LIMIT, + `Limit cannot exceed ${V2_MAX_ROW_LIMIT}; use limit=0 for a full result or the async export for large datasets` + ) + .optional(), + cursor: z.string().min(1, 'cursor must be a non-empty token').optional(), +}) + +export type V2ListTablesQuery = z.output +export type V2QueryRowsBody = z.input +export type V2TableRow = z.output + +export const v2ListTablesContract = defineRouteContract({ + method: 'GET', + path: '/api/v2/tables', + query: v2ListTablesQuerySchema, + response: { + mode: 'json', + schema: successResponseSchema( + z.object({ + tables: z.array(v2TableSummarySchema), + totalCount: z.number(), + }) + ), + }, +}) + +export const v2QueryRowsContract = defineRouteContract({ + method: 'POST', + path: '/api/v2/tables/[tableId]/query', + params: tableIdParamsSchema, + body: v2QueryRowsBodySchema, + response: { + mode: 'json', + schema: successResponseSchema( + z.object({ + rows: z.array(v2TableRowSchema), + rowCount: z.number(), + totalCount: z.number().nullable(), + limit: z.number(), + /** Opaque, short-lived token. Non-null when more rows remain; stop on null. */ + nextCursor: z.string().nullable(), + }) + ), + }, +}) + +export type V2ListTablesResponse = z.output<(typeof v2ListTablesContract)['response']['schema']> +export type V2QueryRowsResponse = z.output<(typeof v2QueryRowsContract)['response']['schema']> diff --git a/apps/sim/lib/copilot/generated/tool-catalog-v1.ts b/apps/sim/lib/copilot/generated/tool-catalog-v1.ts index 971ce14d68c..519a867c0a4 100644 --- a/apps/sim/lib/copilot/generated/tool-catalog-v1.ts +++ b/apps/sim/lib/copilot/generated/tool-catalog-v1.ts @@ -3834,6 +3834,11 @@ export const UserTable: ToolCatalogEntry = { description: 'Array of column names to delete at once (for delete_column). Preferred over columnName when deleting multiple columns.', }, + cursor: { + type: 'string', + description: + 'Opaque pagination cursor for query_rows (optional). Omit for the first page; to fetch the next page, pass back the nextCursor from the previous result\'s "more available" message verbatim. Cannot be combined with a fresh order — the cursor already encodes the paging position.', + }, data: { type: 'object', description: 'Row data as key-value pairs (required for insert_row, update_row)', @@ -3863,9 +3868,9 @@ export const UserTable: ToolCatalogEntry = { 'Canonical workspace file VFS path for create_from_file/import_file, e.g. files/{path}/{name}.', }, filter: { - type: 'string', + type: 'object', description: - 'PostgREST querystring filter for query_rows, update_rows_by_filter, delete_rows_by_filter. A filter is a querystring fragment of field=op.value params joined by & (AND). Ops: eq, neq, gt, gte, lt, lte, in, like, ilike (use * as the wildcard), match, imatch (regex), is.null. Lists: in.(a,b,c) — never empty. Negate with a not. prefix (not.eq, not.in, not.is.null, not.like, not.ilike). Groups: or=(status.eq.active,status.eq.pending) and and=(...) — never empty; nest groups with the function form or(...)/and(...). Use ilike.*x* for case-insensitive substring match. Examples: "status=eq.active"; "wins=gte.18&status=eq.pending"; "or=(status.eq.active,status.eq.pending)"; "name=ilike.*jo*"; "slack_user_id=in.(U1,U2)".', + 'Predicate filter object for query_rows, update_rows_by_filter, delete_rows_by_filter. A predicate is a tree: {"all":[...]} (AND) or {"any":[...]} (OR); members are leaves {field, op, value} or nested groups. Ops: eq, ne, gt, gte, lt, lte, in, nin, like, ilike (use * as the wildcard), nlike, nilike, contains, ncontains, startsWith, endsWith, isNull, isNotNull, isEmpty, isNotEmpty. in/nin take a non-empty array value. Examples: {"all":[{"field":"status","op":"eq","value":"active"}]}; {"all":[{"field":"wins","op":"gte","value":18},{"field":"status","op":"eq","value":"pending"}]}; {"any":[{"field":"status","op":"eq","value":"active"},{"field":"status","op":"eq","value":"pending"}]}; {"all":[{"field":"name","op":"ilike","value":"*jo*"}]}; {"all":[{"field":"slack_user_id","op":"in","value":["U1","U2"]}]}.', }, groupId: { type: 'string', @@ -3900,7 +3905,7 @@ export const UserTable: ToolCatalogEntry = { limit: { type: 'number', description: - 'Maximum rows per page for query_rows (optional). Omit to fetch the ENTIRE matching result in one response — the call fails if the result exceeds the 5MB budget (narrow with a filter or set a limit). With a limit, a page may end early at the byte budget with more remaining; a non-null nextCursor in the result means more rows exist (continue with offset). On update_rows_by_filter / delete_rows_by_filter, caps affected rows; omit to act on every match.', + 'Maximum rows per page for query_rows (optional). Omit to fetch the ENTIRE matching result in one response — the call fails if the result exceeds the 5MB budget (narrow with a filter or set a limit). With a limit, a page may end early at the byte budget with more remaining; a non-null nextCursor in the result means more rows exist (continue with cursor). On update_rows_by_filter / delete_rows_by_filter, caps affected rows; omit to act on every match.', }, mapping: { type: 'object', @@ -3951,15 +3956,10 @@ export const UserTable: ToolCatalogEntry = { description: 'New column type (optional for update_column). Types: string, number, boolean, date, json', }, - offset: { - type: 'number', - description: - 'Number of rows to skip for query_rows (optional, default 0; works with or without limit). Use the offset from the previous result\'s "more available" message to fetch the next page.', - }, order: { - type: 'string', + type: 'array', description: - 'PostgREST order string for query_rows (optional). Comma-separated column.dir terms where dir is asc or desc, e.g. "wins.desc,name.asc".', + 'Sort spec for query_rows (optional). Ordered list of {field, direction} where direction is asc or desc, e.g. [{"field":"wins","direction":"desc"},{"field":"name","direction":"asc"}].', }, outputColumnNames: { type: 'object', diff --git a/apps/sim/lib/copilot/generated/tool-schemas-v1.ts b/apps/sim/lib/copilot/generated/tool-schemas-v1.ts index e1eed73e18a..d2020a432f9 100644 --- a/apps/sim/lib/copilot/generated/tool-schemas-v1.ts +++ b/apps/sim/lib/copilot/generated/tool-schemas-v1.ts @@ -10,7 +10,7 @@ export interface ToolRuntimeSchemaEntry { } export const TOOL_RUNTIME_SCHEMAS: Record = { - agent: { + ['agent']: { parameters: { properties: { request: { @@ -23,7 +23,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - auth: { + ['auth']: { parameters: { properties: { request: { @@ -36,7 +36,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - check_deployment_status: { + ['check_deployment_status']: { parameters: { type: 'object', properties: { @@ -48,7 +48,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - complete_scheduled_task: { + ['complete_scheduled_task']: { parameters: { type: 'object', properties: { @@ -61,7 +61,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - crawl_website: { + ['crawl_website']: { parameters: { type: 'object', properties: { @@ -96,7 +96,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - create_file: { + ['create_file']: { parameters: { type: 'object', properties: { @@ -162,7 +162,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { required: ['success', 'message'], }, }, - create_file_folder: { + ['create_file_folder']: { parameters: { type: 'object', properties: { @@ -180,7 +180,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - create_workflow: { + ['create_workflow']: { parameters: { type: 'object', properties: { @@ -205,7 +205,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - create_workspace_mcp_server: { + ['create_workspace_mcp_server']: { parameters: { type: 'object', properties: { @@ -238,7 +238,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - delete_file: { + ['delete_file']: { parameters: { type: 'object', properties: { @@ -268,7 +268,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { required: ['success', 'message'], }, }, - delete_file_folder: { + ['delete_file_folder']: { parameters: { type: 'object', properties: { @@ -284,7 +284,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - delete_workflow: { + ['delete_workflow']: { parameters: { type: 'object', properties: { @@ -300,7 +300,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - delete_workspace_mcp_server: { + ['delete_workspace_mcp_server']: { parameters: { type: 'object', properties: { @@ -313,7 +313,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - deploy: { + ['deploy']: { parameters: { properties: { request: { @@ -327,7 +327,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - deploy_api: { + ['deploy_api']: { parameters: { type: 'object', properties: { @@ -411,7 +411,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { ], }, }, - deploy_chat: { + ['deploy_chat']: { parameters: { type: 'object', properties: { @@ -570,7 +570,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { ], }, }, - deploy_mcp: { + ['deploy_mcp']: { parameters: { type: 'object', properties: { @@ -686,7 +686,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { required: ['deploymentType', 'deploymentStatus'], }, }, - diff_workflows: { + ['diff_workflows']: { parameters: { type: 'object', properties: { @@ -710,7 +710,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - download_to_workspace_file: { + ['download_to_workspace_file']: { parameters: { type: 'object', properties: { @@ -759,7 +759,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - edit_content: { + ['edit_content']: { parameters: { type: 'object', properties: { @@ -791,7 +791,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { required: ['success', 'message'], }, }, - edit_workflow: { + ['edit_workflow']: { parameters: { type: 'object', properties: { @@ -830,7 +830,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - enrichment_run: { + ['enrichment_run']: { parameters: { type: 'object', properties: { @@ -874,7 +874,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { required: ['matched', 'result'], }, }, - ffmpeg: { + ['ffmpeg']: { parameters: { type: 'object', properties: { @@ -1055,7 +1055,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - file: { + ['file']: { parameters: { properties: { prompt: { @@ -1068,7 +1068,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - function_execute: { + ['function_execute']: { parameters: { type: 'object', properties: { @@ -1206,7 +1206,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - generate_api_key: { + ['generate_api_key']: { parameters: { type: 'object', properties: { @@ -1224,7 +1224,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - generate_audio: { + ['generate_audio']: { parameters: { type: 'object', properties: { @@ -1376,7 +1376,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - generate_image: { + ['generate_image']: { parameters: { type: 'object', properties: { @@ -1504,7 +1504,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - generate_video: { + ['generate_video']: { parameters: { type: 'object', properties: { @@ -1671,7 +1671,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - get_block_outputs: { + ['get_block_outputs']: { parameters: { type: 'object', properties: { @@ -1692,7 +1692,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - get_block_upstream_references: { + ['get_block_upstream_references']: { parameters: { type: 'object', properties: { @@ -1714,7 +1714,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - get_deployed_workflow_state: { + ['get_deployed_workflow_state']: { parameters: { type: 'object', properties: { @@ -1727,7 +1727,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - get_deployment_log: { + ['get_deployment_log']: { parameters: { type: 'object', properties: { @@ -1740,7 +1740,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - get_page_contents: { + ['get_page_contents']: { parameters: { type: 'object', properties: { @@ -1768,14 +1768,14 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - get_platform_actions: { + ['get_platform_actions']: { parameters: { type: 'object', properties: {}, }, resultSchema: undefined, }, - get_scheduled_task_logs: { + ['get_scheduled_task_logs']: { parameters: { type: 'object', properties: { @@ -1800,7 +1800,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - get_workflow_data: { + ['get_workflow_data']: { parameters: { type: 'object', properties: { @@ -1819,7 +1819,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - get_workflow_run_options: { + ['get_workflow_run_options']: { parameters: { type: 'object', properties: { @@ -1832,7 +1832,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - glob: { + ['glob']: { parameters: { type: 'object', properties: { @@ -1851,7 +1851,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - grep: { + ['grep']: { parameters: { type: 'object', properties: { @@ -1899,7 +1899,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - knowledge: { + ['knowledge']: { parameters: { properties: { request: { @@ -1912,7 +1912,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - knowledge_base: { + ['knowledge_base']: { parameters: { type: 'object', properties: { @@ -2105,7 +2105,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { required: ['success', 'message'], }, }, - list_file_folders: { + ['list_file_folders']: { parameters: { type: 'object', properties: { @@ -2117,7 +2117,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - list_integration_tools: { + ['list_integration_tools']: { parameters: { properties: { integration: { @@ -2131,14 +2131,14 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - list_user_workspaces: { + ['list_user_workspaces']: { parameters: { type: 'object', properties: {}, }, resultSchema: undefined, }, - list_workspace_mcp_servers: { + ['list_workspace_mcp_servers']: { parameters: { type: 'object', properties: { @@ -2151,7 +2151,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - load_deployment: { + ['load_deployment']: { parameters: { type: 'object', properties: { @@ -2170,7 +2170,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - load_integration_tool: { + ['load_integration_tool']: { parameters: { properties: { tool_ids: { @@ -2187,7 +2187,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - manage_credential: { + ['manage_credential']: { parameters: { type: 'object', properties: { @@ -2216,7 +2216,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - manage_custom_tool: { + ['manage_custom_tool']: { parameters: { type: 'object', properties: { @@ -2296,7 +2296,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - manage_folder: { + ['manage_folder']: { parameters: { type: 'object', properties: { @@ -2335,7 +2335,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - manage_mcp_tool: { + ['manage_mcp_tool']: { parameters: { type: 'object', properties: { @@ -2387,7 +2387,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - manage_scheduled_task: { + ['manage_scheduled_task']: { parameters: { type: 'object', properties: { @@ -2462,7 +2462,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - manage_skill: { + ['manage_skill']: { parameters: { type: 'object', properties: { @@ -2495,7 +2495,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - materialize_file: { + ['materialize_file']: { parameters: { type: 'object', properties: { @@ -2519,7 +2519,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - media: { + ['media']: { parameters: { properties: { prompt: { @@ -2532,7 +2532,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - move_file: { + ['move_file']: { parameters: { type: 'object', properties: { @@ -2553,7 +2553,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - move_file_folder: { + ['move_file_folder']: { parameters: { type: 'object', properties: { @@ -2571,7 +2571,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - move_workflow: { + ['move_workflow']: { parameters: { type: 'object', properties: { @@ -2591,7 +2591,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - oauth_get_auth_link: { + ['oauth_get_auth_link']: { parameters: { type: 'object', properties: { @@ -2605,7 +2605,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - oauth_request_access: { + ['oauth_request_access']: { parameters: { type: 'object', properties: { @@ -2619,7 +2619,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - open_resource: { + ['open_resource']: { parameters: { type: 'object', properties: { @@ -2653,7 +2653,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - promote_to_live: { + ['promote_to_live']: { parameters: { type: 'object', properties: { @@ -2672,7 +2672,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - query_logs: { + ['query_logs']: { parameters: { type: 'object', properties: { @@ -2783,7 +2783,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - read: { + ['read']: { parameters: { type: 'object', properties: { @@ -2810,7 +2810,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - redeploy: { + ['redeploy']: { parameters: { type: 'object', properties: { @@ -2889,7 +2889,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { ], }, }, - rename_file: { + ['rename_file']: { parameters: { type: 'object', properties: { @@ -2925,7 +2925,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { required: ['success', 'message'], }, }, - rename_file_folder: { + ['rename_file_folder']: { parameters: { type: 'object', properties: { @@ -2942,7 +2942,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - rename_workflow: { + ['rename_workflow']: { parameters: { type: 'object', properties: { @@ -2959,7 +2959,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - research: { + ['research']: { parameters: { properties: { topic: { @@ -2972,7 +2972,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - respond: { + ['respond']: { parameters: { additionalProperties: true, properties: { @@ -2995,7 +2995,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - restore_resource: { + ['restore_resource']: { parameters: { type: 'object', properties: { @@ -3013,7 +3013,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - run: { + ['run']: { parameters: { properties: { context: { @@ -3030,7 +3030,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - run_block: { + ['run_block']: { parameters: { type: 'object', properties: { @@ -3062,7 +3062,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - run_from_block: { + ['run_from_block']: { parameters: { type: 'object', properties: { @@ -3094,7 +3094,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - run_workflow: { + ['run_workflow']: { parameters: { type: 'object', properties: { @@ -3132,7 +3132,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - run_workflow_until_block: { + ['run_workflow_until_block']: { parameters: { type: 'object', properties: { @@ -3175,7 +3175,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - scheduled_task: { + ['scheduled_task']: { parameters: { properties: { request: { @@ -3188,7 +3188,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - scrape_page: { + ['scrape_page']: { parameters: { type: 'object', properties: { @@ -3209,7 +3209,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - search_documentation: { + ['search_documentation']: { parameters: { type: 'object', properties: { @@ -3226,7 +3226,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - search_library_docs: { + ['search_library_docs']: { parameters: { type: 'object', properties: { @@ -3247,7 +3247,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - search_online: { + ['search_online']: { parameters: { type: 'object', properties: { @@ -3287,7 +3287,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - search_patterns: { + ['search_patterns']: { parameters: { type: 'object', properties: { @@ -3309,7 +3309,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - set_block_enabled: { + ['set_block_enabled']: { parameters: { type: 'object', properties: { @@ -3331,7 +3331,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - set_environment_variables: { + ['set_environment_variables']: { parameters: { type: 'object', properties: { @@ -3365,7 +3365,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - set_global_workflow_variables: { + ['set_global_workflow_variables']: { parameters: { type: 'object', properties: { @@ -3406,7 +3406,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - superagent: { + ['superagent']: { parameters: { properties: { task: { @@ -3420,7 +3420,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - table: { + ['table']: { parameters: { properties: { request: { @@ -3433,7 +3433,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - update_deployment_version: { + ['update_deployment_version']: { parameters: { type: 'object', properties: { @@ -3462,7 +3462,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - update_scheduled_task_history: { + ['update_scheduled_task_history']: { parameters: { type: 'object', properties: { @@ -3480,7 +3480,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - update_workspace_mcp_server: { + ['update_workspace_mcp_server']: { parameters: { type: 'object', properties: { @@ -3505,7 +3505,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - user_memory: { + ['user_memory']: { parameters: { type: 'object', properties: { @@ -3554,7 +3554,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - user_table: { + ['user_table']: { parameters: { type: 'object', properties: { @@ -3586,6 +3586,11 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { description: 'Array of column names to delete at once (for delete_column). Preferred over columnName when deleting multiple columns.', }, + cursor: { + type: 'string', + description: + 'Opaque pagination cursor for query_rows (optional). Omit for the first page; to fetch the next page, pass back the nextCursor from the previous result\'s "more available" message verbatim. Cannot be combined with a fresh order — the cursor already encodes the paging position.', + }, data: { type: 'object', description: 'Row data as key-value pairs (required for insert_row, update_row)', @@ -3620,9 +3625,9 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { 'Canonical workspace file VFS path for create_from_file/import_file, e.g. files/{path}/{name}.', }, filter: { - type: 'string', + type: 'object', description: - 'PostgREST querystring filter for query_rows, update_rows_by_filter, delete_rows_by_filter. A filter is a querystring fragment of field=op.value params joined by & (AND). Ops: eq, neq, gt, gte, lt, lte, in, like, ilike (use * as the wildcard), match, imatch (regex), is.null. Lists: in.(a,b,c) — never empty. Negate with a not. prefix (not.eq, not.in, not.is.null, not.like, not.ilike). Groups: or=(status.eq.active,status.eq.pending) and and=(...) — never empty; nest groups with the function form or(...)/and(...). Use ilike.*x* for case-insensitive substring match. Examples: "status=eq.active"; "wins=gte.18&status=eq.pending"; "or=(status.eq.active,status.eq.pending)"; "name=ilike.*jo*"; "slack_user_id=in.(U1,U2)".', + 'Predicate filter object for query_rows, update_rows_by_filter, delete_rows_by_filter. A predicate is a tree: {"all":[...]} (AND) or {"any":[...]} (OR); members are leaves {field, op, value} or nested groups. Ops: eq, ne, gt, gte, lt, lte, in, nin, like, ilike (use * as the wildcard), nlike, nilike, contains, ncontains, startsWith, endsWith, isNull, isNotNull, isEmpty, isNotEmpty. in/nin take a non-empty array value. Examples: {"all":[{"field":"status","op":"eq","value":"active"}]}; {"all":[{"field":"wins","op":"gte","value":18},{"field":"status","op":"eq","value":"pending"}]}; {"any":[{"field":"status","op":"eq","value":"active"},{"field":"status","op":"eq","value":"pending"}]}; {"all":[{"field":"name","op":"ilike","value":"*jo*"}]}; {"all":[{"field":"slack_user_id","op":"in","value":["U1","U2"]}]}.', }, groupId: { type: 'string', @@ -3659,7 +3664,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { limit: { type: 'number', description: - 'Maximum rows per page for query_rows (optional). Omit to fetch the ENTIRE matching result in one response — the call fails if the result exceeds the 5MB budget (narrow with a filter or set a limit). With a limit, a page may end early at the byte budget with more remaining; a non-null nextCursor in the result means more rows exist (continue with offset). On update_rows_by_filter / delete_rows_by_filter, caps affected rows; omit to act on every match.', + 'Maximum rows per page for query_rows (optional). Omit to fetch the ENTIRE matching result in one response — the call fails if the result exceeds the 5MB budget (narrow with a filter or set a limit). With a limit, a page may end early at the byte budget with more remaining; a non-null nextCursor in the result means more rows exist (continue with cursor). On update_rows_by_filter / delete_rows_by_filter, caps affected rows; omit to act on every match.', }, mapping: { type: 'object', @@ -3716,15 +3721,10 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { description: 'New column type (optional for update_column). Types: string, number, boolean, date, json', }, - offset: { - type: 'number', - description: - 'Number of rows to skip for query_rows (optional, default 0; works with or without limit). Use the offset from the previous result\'s "more available" message to fetch the next page.', - }, order: { - type: 'string', + type: 'array', description: - 'PostgREST order string for query_rows (optional). Comma-separated column.dir terms where dir is asc or desc, e.g. "wins.desc,name.asc".', + 'Sort spec for query_rows (optional). Ordered list of {field, direction} where direction is asc or desc, e.g. [{"field":"wins","direction":"desc"},{"field":"name","direction":"asc"}].', }, outputColumnNames: { type: 'object', @@ -3918,7 +3918,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { required: ['success', 'message'], }, }, - workflow: { + ['workflow']: { parameters: { properties: { prompt: { @@ -3931,7 +3931,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - workspace_file: { + ['workspace_file']: { parameters: { type: 'object', properties: { diff --git a/apps/sim/lib/copilot/tools/server/table/user-table.test.ts b/apps/sim/lib/copilot/tools/server/table/user-table.test.ts index 2edef95820b..6e09dfe8519 100644 --- a/apps/sim/lib/copilot/tools/server/table/user-table.test.ts +++ b/apps/sim/lib/copilot/tools/server/table/user-table.test.ts @@ -133,6 +133,7 @@ vi.mock('@/lib/table/billing', () => ({ })) import { userTableServerTool } from '@/lib/copilot/tools/server/table/user-table' +import { encodeCursor } from '@/lib/table/rows/cursor' function buildTable(overrides: Partial = {}): TableDefinition { return { @@ -712,17 +713,61 @@ describe('userTableServerTool.query_rows', () => { expect(options.limit).toBeUndefined() }) - it('queries without execution metadata and passes limit/offset through', async () => { + it('decodes an opaque cursor into after/offset and skips the count', async () => { + const cursor = encodeCursor({ + lastRow: { id: 'row_9', orderKey: 'a9' }, + keysetValid: true, + nextOffset: 20, + }) const result = await userTableServerTool.execute( - { operation: 'query_rows', args: { tableId: 'tbl_1', limit: 2, offset: 10 } }, + { operation: 'query_rows', args: { tableId: 'tbl_1', limit: 2, cursor } }, { userId: 'user-1', workspaceId: 'workspace-1' } ) expect(result.success).toBe(true) const options = mockQueryRows.mock.calls[0][1] as Record expect(options.withExecutions).toBe(false) - expect(options.offset).toBe(10) - expect(result.data?.nextCursor).toBeUndefined() + expect(options.after).toEqual({ orderKey: 'a9', id: 'row_9' }) + // A cursor page never re-counts. + expect(options.includeTotal).toBe(false) + }) + + it('surfaces the opaque nextCursor (not an offset) in the "more available" message', async () => { + mockQueryRows.mockResolvedValueOnce({ + rows: [queryRow(1), queryRow(2)], + rowCount: 2, + totalCount: 10, + limit: 2, + offset: 0, + nextCursor: 'CURSOR_TOKEN_ABC', + }) + const result = await userTableServerTool.execute( + { operation: 'query_rows', args: { tableId: 'tbl_1', limit: 2 } }, + { userId: 'user-1', workspaceId: 'workspace-1' } + ) + + expect(result.message).toContain('more available') + expect(result.message).toContain('cursor=CURSOR_TOKEN_ABC') + expect(result.message).not.toContain('offset=') + }) + + it('rejects a keyset cursor combined with a custom sort', async () => { + const cursor = encodeCursor({ + lastRow: { id: 'row_9', orderKey: 'a9' }, + keysetValid: true, + nextOffset: 20, + }) + const result = await userTableServerTool.execute( + { + operation: 'query_rows', + args: { tableId: 'tbl_1', cursor, order: [{ field: 'name', direction: 'desc' }] }, + }, + { userId: 'user-1', workspaceId: 'workspace-1' } + ) + + expect(result.success).toBe(false) + expect(result.message).toMatch(/not valid for a sorted query/i) + expect(mockQueryRows).not.toHaveBeenCalled() }) }) @@ -755,7 +800,7 @@ describe('userTableServerTool.delete_rows_by_filter', () => { operation: 'delete_rows_by_filter', args: { tableId: 'tbl_1', - filter: 'name=eq.x', + filter: { all: [{ field: 'name', op: 'eq', value: 'x' }] }, limit: 5000, }, }, @@ -780,7 +825,7 @@ describe('userTableServerTool.delete_rows_by_filter', () => { const result = await userTableServerTool.execute( { operation: 'delete_rows_by_filter', - args: { tableId: 'tbl_1', filter: 'name=eq.x' }, + args: { tableId: 'tbl_1', filter: { all: [{ field: 'name', op: 'eq', value: 'x' }] } }, }, { userId: 'user-1', workspaceId: 'workspace-1' } ) @@ -801,7 +846,7 @@ describe('userTableServerTool.delete_rows_by_filter', () => { operation: 'delete_rows_by_filter', args: { tableId: 'tbl_1', - filter: 'name=eq.x', + filter: { all: [{ field: 'name', op: 'eq', value: 'x' }] }, limit: 100, }, }, @@ -825,7 +870,7 @@ describe('userTableServerTool.delete_rows_by_filter', () => { const result = await userTableServerTool.execute( { operation: 'delete_rows_by_filter', - args: { tableId: 'tbl_1', filter: 'name=eq.x' }, + args: { tableId: 'tbl_1', filter: { all: [{ field: 'name', op: 'eq', value: 'x' }] } }, }, { userId: 'user-1', workspaceId: 'workspace-1' } ) @@ -863,7 +908,7 @@ describe('userTableServerTool.delete_rows_by_filter', () => { const result = await userTableServerTool.execute( { operation: 'delete_rows_by_filter', - args: { tableId: 'tbl_1', filter: 'name=eq.x' }, + args: { tableId: 'tbl_1', filter: { all: [{ field: 'name', op: 'eq', value: 'x' }] } }, }, { userId: 'user-1', workspaceId: 'workspace-1' } ) @@ -880,7 +925,7 @@ describe('userTableServerTool.delete_rows_by_filter', () => { operation: 'delete_rows_by_filter', args: { tableId: 'tbl_1', - filter: 'name=eq.x', + filter: { all: [{ field: 'name', op: 'eq', value: 'x' }] }, limit: 100, }, }, @@ -915,7 +960,7 @@ describe('userTableServerTool.update_rows_by_filter', () => { operation: 'update_rows_by_filter', args: { tableId: 'tbl_1', - filter: 'name=eq.x', + filter: { all: [{ field: 'name', op: 'eq', value: 'x' }] }, data: { age: 1 }, limit: 5000, }, @@ -940,7 +985,7 @@ describe('userTableServerTool.update_rows_by_filter', () => { operation: 'update_rows_by_filter', args: { tableId: 'tbl_1', - filter: 'name=eq.x', + filter: { all: [{ field: 'name', op: 'eq', value: 'x' }] }, data: { age: 1 }, }, }, @@ -965,7 +1010,7 @@ describe('userTableServerTool.update_rows_by_filter', () => { operation: 'update_rows_by_filter', args: { tableId: 'tbl_1', - filter: 'name=eq.x', + filter: { all: [{ field: 'name', op: 'eq', value: 'x' }] }, data: { age: 1 }, }, }, @@ -1005,7 +1050,7 @@ describe('userTableServerTool.update_rows_by_filter', () => { operation: 'update_rows_by_filter', args: { tableId: 'tbl_1', - filter: 'email=eq.x', + filter: { all: [{ field: 'email', op: 'eq', value: 'x' }] }, data: { email: 'y' }, }, }, @@ -1031,7 +1076,7 @@ describe('userTableServerTool.update_rows_by_filter', () => { operation: 'update_rows_by_filter', args: { tableId: 'tbl_1', - filter: 'name=eq.x', + filter: { all: [{ field: 'name', op: 'eq', value: 'x' }] }, data: { age: 1 }, }, }, @@ -1049,7 +1094,7 @@ describe('userTableServerTool.update_rows_by_filter', () => { operation: 'update_rows_by_filter', args: { tableId: 'tbl_1', - filter: 'name=eq.x', + filter: { all: [{ field: 'name', op: 'eq', value: 'x' }] }, data: { age: 1 }, limit: 100, }, diff --git a/apps/sim/lib/copilot/tools/server/table/user-table.ts b/apps/sim/lib/copilot/tools/server/table/user-table.ts index 1a9359cba53..227192e611d 100644 --- a/apps/sim/lib/copilot/tools/server/table/user-table.ts +++ b/apps/sim/lib/copilot/tools/server/table/user-table.ts @@ -45,7 +45,8 @@ import { markTableDeleteFailed, runTableDelete } from '@/lib/table/delete-runner import { runTableImport, type TableImportPayload } from '@/lib/table/import-runner' import { markTableJobRunning, releaseJobClaim } from '@/lib/table/jobs/service' import { predicateToFilter } from '@/lib/table/query-builder/converters' -import { parsePostgrestFilter, parsePostgrestOrder } from '@/lib/table/query-builder/postgrest' +import { validatePredicate, validateSortSpec } from '@/lib/table/query-builder/validate' +import { decodeCursor } from '@/lib/table/rows/cursor' import { batchInsertRows, batchUpdateRows, @@ -64,8 +65,10 @@ import type { ColumnDefinition, Filter, RowData, + SortSpec, TableDefinition, TableDeleteJobPayload, + TablePredicate, TableUpdateJobPayload, WorkflowGroup, WorkflowGroupDependencies, @@ -604,39 +607,62 @@ export const userTableServerTool: BaseServerTool const requestId = generateId().slice(0, 8) const idByName = buildIdByName(table.schema) const nameById = buildNameById(table.schema) - // PostgREST filter/order strings, parsed with the schema (column NAMES - // → coercion) then translated to storage ids. - const predicate = args.filter - ? predicateNamesToIds(parsePostgrestFilter(args.filter, table.schema.columns), idByName) - : undefined - const orderSpec = args.order - ? sortSpecNamesToIds(parsePostgrestOrder(args.order, table.schema.columns), idByName) - : undefined + // Typed predicate/sort objects, validated against the schema (column + // NAMES) then translated to storage ids. + let predicate: TablePredicate | undefined + if (args.filter) { + validatePredicate(args.filter, table.schema.columns) + predicate = predicateNamesToIds(args.filter, idByName) + } + let orderSpec = args.order as SortSpec | undefined + if (orderSpec?.length) { + validateSortSpec(orderSpec, table.schema.columns) + orderSpec = sortSpecNamesToIds(orderSpec, idByName) + } const sort = orderSpec?.length ? Object.fromEntries(orderSpec.map((s) => [s.field, s.direction])) : undefined + + // Opaque cursor pagination (keyset seek on the default order; the token + // hides an internal offset only for custom-sorted views, which a keyset + // physically can't page). A keyset cursor is bound to the default order, + // so it can't be combined with a fresh sort. + const cursor = args.cursor ? decodeCursor(args.cursor) : undefined + if (cursor?.after && sort) { + return { + success: false, + message: + 'Cursor is not valid for a sorted query. Restart paging without the cursor (omit it on the first sorted page).', + } + } + // No limit returns the ENTIRE matching result, failing fast once the // 5MB byte budget is exceeded (caught below → structured tool error // the model can react to by adding a filter or a limit). An explicit - // limit pages; byte-cut pages set nextCursor and the message says how - // to continue with `offset`. + // limit pages; byte-cut pages set nextCursor and the message says to + // continue with the opaque cursor. const result = await queryRows( table, { predicate, sort, limit: args.limit, - offset: args.offset, + after: cursor?.after, + offset: cursor?.offset, + // Only the first page (no inbound cursor) pays for the COUNT(*). + includeTotal: !args.cursor, withExecutions: false, }, requestId ) // nextCursor covers both cut kinds (explicit limit or the 5MB byte - // budget) — either way the truthful signal is "more rows exist". + // budget) — either way the truthful signal is "more rows exist". The + // token is opaque; the agent echoes it back as `cursor` to continue. + const countSuffix = result.totalCount != null ? ` of ${result.totalCount}` : '' const message = result.nextCursor - ? `Returned ${result.rows.length} of ${result.totalCount} rows (more available — pass offset=${result.offset + result.rows.length} to continue)` - : `Returned ${result.rows.length} of ${result.totalCount} rows` + ? `Returned ${result.rows.length}${countSuffix} rows (more available — pass cursor=${result.nextCursor} to continue)` + : `Returned ${result.rows.length}${countSuffix} rows` return { success: true, message, @@ -745,11 +771,10 @@ export const userTableServerTool: BaseServerTool const requestId = generateId().slice(0, 8) const idByName = buildIdByName(table.schema) - // Agent authors a PostgREST filter string; parse → predicate → Filter - // for the bulk engine (same fieldPredicate leaf → identical SQL). - const idFilter = predicateToFilter( - predicateNamesToIds(parsePostgrestFilter(args.filter, table.schema.columns), idByName) - ) + // Agent authors a predicate object; validate → translate → Filter for + // the bulk engine (same fieldPredicate leaf → identical SQL). + validatePredicate(args.filter, table.schema.columns) + const idFilter = predicateToFilter(predicateNamesToIds(args.filter, idByName)) const idData = rowDataNameToId(args.data, idByName) // Inline handles up to MAX_BULK_OPERATION_SIZE rows in one request; a larger operation @@ -844,11 +869,10 @@ export const userTableServerTool: BaseServerTool const requestId = generateId().slice(0, 8) const idByName = buildIdByName(table.schema) - // Agent authors a PostgREST filter string; parse → predicate → Filter - // for the bulk engine (same fieldPredicate leaf → identical SQL). - const idFilter = predicateToFilter( - predicateNamesToIds(parsePostgrestFilter(args.filter, table.schema.columns), idByName) - ) + // Agent authors a predicate object; validate → translate → Filter for + // the bulk engine (same fieldPredicate leaf → identical SQL). + validatePredicate(args.filter, table.schema.columns) + const idFilter = predicateToFilter(predicateNamesToIds(args.filter, idByName)) // Inline handles up to MAX_BULK_OPERATION_SIZE rows; a larger delete (an explicit limit // above the cap, or unbounded "delete everything matching") hands off to the background diff --git a/apps/sim/lib/table/__tests__/sql.test.ts b/apps/sim/lib/table/__tests__/sql.test.ts index 0b2062784a9..0841c788b97 100644 --- a/apps/sim/lib/table/__tests__/sql.test.ts +++ b/apps/sim/lib/table/__tests__/sql.test.ts @@ -395,12 +395,12 @@ describe('SQL Builder', () => { expect(out).toBe(`(${TABLE}.data->>'birthDate')::timestamptz ASC NULLS LAST`) }) - it('sorts createdAt / updatedAt as direct column refs', () => { + it('sorts createdAt / updatedAt as direct snake_case column refs', () => { expect(render(buildSortClause({ createdAt: 'desc' }, TABLE, NO_COLUMNS))).toBe( - `${TABLE}.createdAt DESC` + `${TABLE}.created_at DESC` ) expect(render(buildSortClause({ updatedAt: 'asc' }, TABLE, NO_COLUMNS))).toBe( - `${TABLE}.updatedAt ASC` + `${TABLE}.updated_at ASC` ) }) @@ -566,6 +566,28 @@ describe('fieldPredicate (shared leaf)', () => { 'Invalid operator' ) }) + + it('filters createdAt / updatedAt as real timestamptz columns, not JSONB keys', () => { + const gte = render(fieldPredicate(TABLE, 'createdAt', 'gte', '2026-01-01', 'date')) + expect(gte).toContain(`${TABLE}.created_at`) + expect(gte).toContain('::timestamptz') + expect(gte).not.toContain("data->>'createdAt'") + + const lt = render(fieldPredicate(TABLE, 'updatedAt', 'lt', '2026-06-01', 'date')) + expect(lt).toContain(`${TABLE}.updated_at`) + expect(lt).toContain('::timestamptz') + }) + + it('supports in / isNull on system columns', () => { + expect(render(fieldPredicate(TABLE, 'createdAt', 'isNull', undefined, 'date'))).toContain( + `${TABLE}.created_at IS NULL` + ) + const inClause = render( + fieldPredicate(TABLE, 'createdAt', 'in', ['2026-01-01', '2026-02-01'], 'date') + ) + expect(inClause).toContain(`${TABLE}.created_at`) + expect(inClause).toContain('::timestamptz') + }) }) describe('buildPredicateClause (v2 grammar)', () => { diff --git a/apps/sim/lib/table/constants.ts b/apps/sim/lib/table/constants.ts index 01b48ec6df7..f768ccb5be8 100644 --- a/apps/sim/lib/table/constants.ts +++ b/apps/sim/lib/table/constants.ts @@ -135,6 +135,37 @@ export function getTablePlanLimits(): TablePlanLimitsByPlan { export const COLUMN_TYPES = ['string', 'number', 'boolean', 'date', 'json'] as const +/** + * The v2 filter operators, as a runtime tuple so both the `FilterOp` type and + * the boundary Zod enum derive from one source. Order is not significant. + */ +export const FILTER_OPS = [ + 'eq', + 'ne', + 'gt', + 'gte', + 'lt', + 'lte', + 'in', + 'nin', + 'contains', + 'ncontains', + 'startsWith', + 'endsWith', + 'like', + 'ilike', + 'nlike', + 'nilike', + 'match', + 'imatch', + 'isEmpty', + 'isNotEmpty', + 'isNull', + 'isNotNull', +] as const + +export const SORT_DIRECTIONS = ['asc', 'desc'] as const + export const NAME_PATTERN = /^[a-z_][a-z0-9_]*$/i export const USER_TABLE_ROWS_SQL_NAME = 'user_table_rows' diff --git a/apps/sim/lib/table/errors.ts b/apps/sim/lib/table/errors.ts index 01745ee88af..bb47bdf4367 100644 --- a/apps/sim/lib/table/errors.ts +++ b/apps/sim/lib/table/errors.ts @@ -1,3 +1,14 @@ +/** + * Stable, machine-readable codes for table query failures. SDKs and clients + * branch on these instead of string-matching human-facing messages. + */ +export type TableQueryErrorCode = + | 'TABLE_QUERY_RESULT_TOO_LARGE' + | 'INVALID_CURSOR' + | 'CURSOR_SORT_CONFLICT' + | 'INVALID_FILTER' + | 'INVALID_ORDER' + /** * Error thrown when caller-supplied filter or sort input is malformed. * Routes should map this to HTTP 400 with the message preserved. @@ -7,8 +18,11 @@ * into the browser chunk. */ export class TableQueryValidationError extends Error { - constructor(message: string) { + readonly code?: TableQueryErrorCode + + constructor(message: string, code?: TableQueryErrorCode) { super(message) this.name = 'TableQueryValidationError' + this.code = code } } diff --git a/apps/sim/lib/table/planner.ts b/apps/sim/lib/table/planner.ts index 848e2c2d6de..85f4969b6f6 100644 --- a/apps/sim/lib/table/planner.ts +++ b/apps/sim/lib/table/planner.ts @@ -5,19 +5,52 @@ export type DbExecutor = typeof db | DbTransaction export type DbTransaction = Parameters[0]>[0] /** - * Runs `fn` with seq scans penalized (`SET LOCAL`, so the flag dies with the - * transaction). JSONB predicates and sort keys (`->>` extraction, `@>` - * containment, lateral `jsonb_each_text`) are opaque to the planner — it - * estimates a handful of matching rows and picks a parallel seq scan over the - * entire shared `user_table_rows` relation (every tenant's rows) instead of the - * tenant's own index. Measured on a 1M-row table inside a 12M-row relation: - * filtered count 12.7s → 1.0s, sorted page 9.7s → 0.76s, filtered bulk select - * 14.4s → tenant-bounded. The flag only penalizes the plan shape: if no index - * plan exists, the seq scan still runs. + * Statement/lock timeout for user-table READ queries. Reads are bounded tighter + * than the write path because filter shape is caller-controlled: a public API + * key can drive an arbitrary predicate (a catastrophic-backtracking `match` + * regex, a wide unindexed scan), and an untimed read pins a connection from the + * small shared `web` pool — one abusive key can starve every tenant. `SET LOCAL` + * scopes both to the surrounding transaction, so they die when it commits. */ -export async function withSeqscanOff(fn: (trx: DbTransaction) => Promise): Promise { +const READ_STATEMENT_TIMEOUT_MS = 15_000 +const READ_LOCK_TIMEOUT_MS = 3_000 + +async function setReadTimeouts(trx: DbTransaction): Promise { + await trx.execute(sql.raw(`SET LOCAL statement_timeout = '${READ_STATEMENT_TIMEOUT_MS}ms'`)) + await trx.execute(sql.raw(`SET LOCAL lock_timeout = '${READ_LOCK_TIMEOUT_MS}ms'`)) +} + +/** + * Runs a user-table read inside a transaction that always caps `statement_timeout` + * / `lock_timeout` (see {@link setReadTimeouts}). Pass `seqscanOff` for queries + * with no tenant-bounded index plan — custom column sorts and filtered counts — + * where the planner otherwise seq-scans the whole shared `user_table_rows` + * relation (every tenant's rows); see {@link withSeqscanOff} for the measured + * deltas. Default-order keyset/offset pages already stream the + * `(table_id, order_key, id)` index, so they take the timeout without the flag. + */ +export async function withReadGuards( + fn: (trx: DbTransaction) => Promise, + opts?: { seqscanOff?: boolean } +): Promise { return db.transaction(async (trx) => { - await trx.execute(sql`SET LOCAL enable_seqscan = off`) + await setReadTimeouts(trx) + if (opts?.seqscanOff) await trx.execute(sql`SET LOCAL enable_seqscan = off`) return fn(trx) }) } + +/** + * Runs `fn` with seq scans penalized (`SET LOCAL`, so the flag dies with the + * transaction) AND the read timeouts above. JSONB predicates and sort keys + * (`->>` extraction, `@>` containment, lateral `jsonb_each_text`) are opaque to + * the planner — it estimates a handful of matching rows and picks a parallel + * seq scan over the entire shared `user_table_rows` relation (every tenant's + * rows) instead of the tenant's own index. Measured on a 1M-row table inside a + * 12M-row relation: filtered count 12.7s → 1.0s, sorted page 9.7s → 0.76s, + * filtered bulk select 14.4s → tenant-bounded. The flag only penalizes the plan + * shape: if no index plan exists, the seq scan still runs (and the timeout caps it). + */ +export async function withSeqscanOff(fn: (trx: DbTransaction) => Promise): Promise { + return withReadGuards(fn, { seqscanOff: true }) +} diff --git a/apps/sim/lib/table/query-builder/__tests__/converters.test.ts b/apps/sim/lib/table/query-builder/__tests__/converters.test.ts index 5efc1e5c5f7..ec3dd911995 100644 --- a/apps/sim/lib/table/query-builder/__tests__/converters.test.ts +++ b/apps/sim/lib/table/query-builder/__tests__/converters.test.ts @@ -8,12 +8,10 @@ import { describe, expect, it } from 'vitest' import { filterRulesToFilter, - filterRulesToPostgrest, filterRulesToPredicate, filterToRules, predicateToFilter, predicateToFilterRules, - sortRulesToPostgrestOrder, sortRulesToSortSpec, } from '@/lib/table/query-builder/converters' import type { FilterRule, SortRule, TablePredicate } from '@/lib/table/types' @@ -240,56 +238,7 @@ describe('sortRulesToSortSpec (v2)', () => { }) }) -describe('filterRulesToPostgrest', () => { - it('serializes builder rules to a PostgREST querystring', () => { - expect( - filterRulesToPostgrest([ - rule({ column: 'wins', operator: 'gte', value: '10' }), - rule({ column: 'status', operator: 'eq', value: 'active' }), - ]) - ).toBe('wins=gte.10&status=eq.active') - }) - - it('serializes an OR boundary to an or() group', () => { - expect( - filterRulesToPostgrest([ - rule({ column: 'status', operator: 'eq', value: 'active' }), - rule({ column: 'status', operator: 'eq', value: 'pending', logicalOperator: 'or' }), - ]) - ).toBe('or=(status.eq.active,status.eq.pending)') - }) - - it('maps builder-only ops onto PostgREST forms', () => { - expect( - filterRulesToPostgrest([rule({ column: 'name', operator: 'contains', value: 'jo' })]) - ).toBe('name=ilike.*jo*') - // isEmpty keeps its null-OR-empty-string semantics through the string form. - expect(filterRulesToPostgrest([rule({ column: 'name', operator: 'isEmpty' })])).toBe( - 'or=(name.is.null,name.eq."")' - ) - }) - - it('returns null for an empty builder', () => { - expect(filterRulesToPostgrest([])).toBeNull() - }) - - it('returns null (does not throw "rules is not iterable") for a non-array value', () => { - expect(filterRulesToPostgrest({} as unknown as FilterRule[])).toBeNull() - expect(filterRulesToPostgrest(undefined as unknown as FilterRule[])).toBeNull() - }) -}) - -describe('sortRulesToPostgrestOrder', () => { - it('serializes sort rules to a PostgREST order string', () => { - expect( - sortRulesToPostgrestOrder([ - { id: '1', column: 'wins', direction: 'desc' }, - { id: '2', column: 'name', direction: 'asc' }, - ]) - ).toBe('wins.desc,name.asc') - }) - - it('returns null when empty', () => { - expect(sortRulesToPostgrestOrder([])).toBeNull() - }) +it('does not throw "rules is not iterable" for a non-array builder value', () => { + expect(filterRulesToPredicate({} as unknown as FilterRule[])).toBeNull() + expect(filterRulesToPredicate(undefined as unknown as FilterRule[])).toBeNull() }) diff --git a/apps/sim/lib/table/query-builder/__tests__/postgrest.test.ts b/apps/sim/lib/table/query-builder/__tests__/postgrest.test.ts deleted file mode 100644 index 2d536565079..00000000000 --- a/apps/sim/lib/table/query-builder/__tests__/postgrest.test.ts +++ /dev/null @@ -1,271 +0,0 @@ -/** - * @vitest-environment node - */ -import { describe, expect, it } from 'vitest' -import { - parsePostgrestFilter, - parsePostgrestOrder, - predicateToPostgrest, - sortSpecToPostgrestOrder, -} from '@/lib/table/query-builder/postgrest' -import type { ColumnDefinition, TablePredicate } from '@/lib/table/types' - -const COLS: ColumnDefinition[] = [ - { name: 'wins', type: 'number' }, - { name: 'status', type: 'string' }, - { name: 'active', type: 'boolean' }, - { name: 'name', type: 'string' }, - { name: 'slack_user_id', type: 'string' }, -] - -describe('parsePostgrestFilter', () => { - it('parses top-level params as an AND of leaves, coercing by column type', () => { - expect(parsePostgrestFilter('wins=gte.10&status=eq.active', COLS)).toEqual({ - all: [ - { field: 'wins', op: 'gte', value: 10 }, - { field: 'status', op: 'eq', value: 'active' }, - ], - }) - }) - - it('coerces booleans and numbers', () => { - expect(parsePostgrestFilter('active=eq.true&wins=eq.5', COLS)).toEqual({ - all: [ - { field: 'active', op: 'eq', value: true }, - { field: 'wins', op: 'eq', value: 5 }, - ], - }) - }) - - it('parses in.(...) lists with per-element coercion', () => { - expect(parsePostgrestFilter('slack_user_id=in.(U1,U2)', COLS)).toEqual({ - all: [{ field: 'slack_user_id', op: 'in', value: ['U1', 'U2'] }], - }) - expect(parsePostgrestFilter('wins=in.(1,2,3)', COLS)).toEqual({ - all: [{ field: 'wins', op: 'in', value: [1, 2, 3] }], - }) - }) - - it('maps neq→ne, is.null→isNull, not.is.null→isNotNull', () => { - expect(parsePostgrestFilter('status=neq.x', COLS)).toEqual({ - all: [{ field: 'status', op: 'ne', value: 'x' }], - }) - expect(parsePostgrestFilter('status=is.null', COLS)).toEqual({ - all: [{ field: 'status', op: 'isNull' }], - }) - expect(parsePostgrestFilter('status=not.is.null', COLS)).toEqual({ - all: [{ field: 'status', op: 'isNotNull' }], - }) - }) - - it('supports not.eq → ne and not.in → nin', () => { - expect(parsePostgrestFilter('status=not.eq.x', COLS)).toEqual({ - all: [{ field: 'status', op: 'ne', value: 'x' }], - }) - expect(parsePostgrestFilter('slack_user_id=not.in.(U1,U2)', COLS)).toEqual({ - all: [{ field: 'slack_user_id', op: 'nin', value: ['U1', 'U2'] }], - }) - }) - - it('keeps like/ilike/match values as raw text (wildcards/regex preserved)', () => { - expect(parsePostgrestFilter('name=ilike.*jo*', COLS)).toEqual({ - all: [{ field: 'name', op: 'ilike', value: '*jo*' }], - }) - expect(parsePostgrestFilter('name=match.^a.*z$', COLS)).toEqual({ - all: [{ field: 'name', op: 'match', value: '^a.*z$' }], - }) - }) - - it('parses or=(...) and nested and()/or() groups', () => { - expect(parsePostgrestFilter('or=(status.eq.active,status.eq.pending)', COLS)).toEqual({ - all: [ - { - any: [ - { field: 'status', op: 'eq', value: 'active' }, - { field: 'status', op: 'eq', value: 'pending' }, - ], - }, - ], - }) - const nested = parsePostgrestFilter('and=(wins.gte.1,or(status.eq.a,status.eq.b))', COLS) - expect(nested).toEqual({ - all: [ - { - all: [ - { field: 'wins', op: 'gte', value: 1 }, - { - any: [ - { field: 'status', op: 'eq', value: 'a' }, - { field: 'status', op: 'eq', value: 'b' }, - ], - }, - ], - }, - ], - }) - }) - - it('rejects unsupported ops with an actionable error', () => { - expect(() => parsePostgrestFilter('tags=cs.{x}', COLS)).toThrow(/not supported/) - expect(() => parsePostgrestFilter('doc=fts.hello', COLS)).toThrow(/full-text/) - }) - - it('rejects unknown ops, bad fields, and unsupported negation', () => { - expect(() => parsePostgrestFilter('wins=bogus.1', COLS)).toThrow(/Unknown filter operator/) - expect(() => parsePostgrestFilter('bad name=eq.1', COLS)).toThrow(/Invalid filter column/) - expect(() => parsePostgrestFilter('name=not.match.x', COLS)).toThrow(/not supported/) - }) - - it('parses not.like / not.ilike into the negated pattern ops', () => { - expect(parsePostgrestFilter('name=not.ilike.*jo*', COLS)).toEqual({ - all: [{ field: 'name', op: 'nilike', value: '*jo*' }], - }) - expect(parsePostgrestFilter('name=not.like.jo*', COLS)).toEqual({ - all: [{ field: 'name', op: 'nlike', value: 'jo*' }], - }) - }) - - it('rejects silent-widening inputs: empty groups, empty in-lists, non-finite numbers', () => { - expect(() => parsePostgrestFilter('wins=gte.1&or=()', COLS)).toThrow(/Empty or=\(\) group/) - expect(() => parsePostgrestFilter('and=()', COLS)).toThrow(/Empty and=\(\) group/) - expect(() => parsePostgrestFilter('status=in.()', COLS)).toThrow(/Empty in\.\(\) list/) - expect(() => parsePostgrestFilter('wins=eq.Infinity', COLS)).toThrow(/Expected a number/) - expect(() => parsePostgrestFilter('wins=eq.1e400', COLS)).toThrow(/Expected a number/) - }) - - it('rejects empty / malformed input', () => { - expect(() => parsePostgrestFilter('', COLS)).toThrow(/empty/) - expect(() => parsePostgrestFilter('justakey', COLS)).toThrow(/Malformed/) - }) - - it('honors quotes when splitting lists', () => { - expect(parsePostgrestFilter('status=in.("a,b",c)', COLS)).toEqual({ - all: [{ field: 'status', op: 'in', value: ['a,b', 'c'] }], - }) - }) -}) - -describe('parsePostgrestOrder', () => { - it('parses col.dir pairs and defaults to asc', () => { - expect(parsePostgrestOrder('wins.desc,name', COLS)).toEqual([ - { field: 'wins', direction: 'desc' }, - { field: 'name', direction: 'asc' }, - ]) - }) - it('allows createdAt/updatedAt and rejects unknown columns', () => { - expect(parsePostgrestOrder('createdAt.desc', COLS)).toEqual([ - { field: 'createdAt', direction: 'desc' }, - ]) - expect(() => parsePostgrestOrder('nope.asc', COLS)).toThrow(/Unknown sort column/) - }) -}) - -describe('predicateToPostgrest round-trips', () => { - it('round-trips an AND of leaves', () => { - const p: TablePredicate = { - all: [ - { field: 'wins', op: 'gte', value: 10 }, - { field: 'slack_user_id', op: 'in', value: ['U1', 'U2'] }, - ], - } - const str = predicateToPostgrest(p) - expect(str).toBe('wins=gte.10&slack_user_id=in.(U1,U2)') - expect(parsePostgrestFilter(str, COLS)).toEqual(p) - }) - - it('round-trips an OR group', () => { - const p: TablePredicate = { - all: [ - { - any: [ - { field: 'status', op: 'eq', value: 'active' }, - { field: 'status', op: 'eq', value: 'pending' }, - ], - }, - ], - } - expect(parsePostgrestFilter(predicateToPostgrest(p), COLS)).toEqual(p) - }) - - it('serializes builder-only ops onto PostgREST forms', () => { - expect(leaf('contains', 'foo')).toBe('name=ilike.*foo*') - expect(leaf('startsWith', 'foo')).toBe('name=ilike.foo*') - expect(leaf('ncontains', 'foo')).toBe('name=not.ilike.*foo*') - // Emptiness desugars to groups preserving null-OR-empty-string semantics. - expect(leaf('isEmpty')).toBe('or=(name.is.null,name.eq."")') - expect(leaf('isNotEmpty')).toBe('and=(name.not.is.null,name.neq."")') - }) - - it('round-trips substring ops with reserved characters (whole-pattern quoting)', () => { - const str = leaf('contains', 'example.com') - expect(str).toBe('name=ilike."*example.com*"') - expect(parsePostgrestFilter(str, COLS)).toEqual({ - all: [{ field: 'name', op: 'ilike', value: '*example.com*' }], - }) - - const ncontains = leaf('ncontains', 'a,b(c)=d') - expect(parsePostgrestFilter(ncontains, COLS)).toEqual({ - all: [{ field: 'name', op: 'nilike', value: '*a,b(c)=d*' }], - }) - }) - - it('round-trips embedded quotes and backslashes through escaping', () => { - const p: TablePredicate = { - all: [{ field: 'name', op: 'eq', value: 'She said "hi"' }], - } - expect(parsePostgrestFilter(predicateToPostgrest(p), COLS)).toEqual(p) - - const list: TablePredicate = { - all: [{ field: 'status', op: 'in', value: ['a"b', 'c\\d'] }], - } - expect(parsePostgrestFilter(predicateToPostgrest(list), COLS)).toEqual(list) - }) - - it('round-trips emptiness ops to their semantically-equal group form', () => { - expect(parsePostgrestFilter(leaf('isEmpty'), COLS)).toEqual({ - all: [ - { - any: [ - { field: 'name', op: 'isNull' }, - { field: 'name', op: 'eq', value: '' }, - ], - }, - ], - }) - expect(parsePostgrestFilter(leaf('isNotEmpty'), COLS)).toEqual({ - all: [ - { - all: [ - { field: 'name', op: 'isNotNull' }, - { field: 'name', op: 'ne', value: '' }, - ], - }, - ], - }) - }) - - it('rejects uppercase/typo sort directions instead of silently sorting asc', () => { - expect(() => parsePostgrestOrder('wins.DESC', COLS)).toThrow(/Unknown sort direction/) - expect(() => parsePostgrestOrder('wins.dsc', COLS)).toThrow(/Unknown sort direction/) - expect(() => parsePostgrestOrder('wins.desc.extra', COLS)).toThrow(/Malformed sort/) - }) - - it('order spec round-trips', () => { - const s = [ - { field: 'wins', direction: 'desc' as const }, - { field: 'name', direction: 'asc' as const }, - ] - expect(sortSpecToPostgrestOrder(s)).toBe('wins.desc,name.asc') - expect(parsePostgrestOrder(sortSpecToPostgrestOrder(s), COLS)).toEqual(s) - }) -}) - -function leaf(op: string, value?: string): string { - return predicateToPostgrest({ - all: [ - value === undefined - ? { field: 'name', op: op as never } - : { field: 'name', op: op as never, value }, - ], - }) -} diff --git a/apps/sim/lib/table/query-builder/__tests__/validate.test.ts b/apps/sim/lib/table/query-builder/__tests__/validate.test.ts new file mode 100644 index 00000000000..0b721aa52b0 --- /dev/null +++ b/apps/sim/lib/table/query-builder/__tests__/validate.test.ts @@ -0,0 +1,99 @@ +/** + * @vitest-environment node + */ +import { describe, expect, it } from 'vitest' +import { TableQueryValidationError } from '@/lib/table/errors' +import { validatePredicate, validateSortSpec } from '@/lib/table/query-builder/validate' +import type { ColumnDefinition } from '@/lib/table/types' + +const COLS: ColumnDefinition[] = [ + { name: 'wins', type: 'number' }, + { name: 'status', type: 'string' }, + { name: 'metadata', type: 'json' }, +] + +describe('validatePredicate', () => { + it('accepts a valid predicate over real + system columns', () => { + expect(() => + validatePredicate( + { + all: [ + { field: 'wins', op: 'gte', value: 10 }, + { field: 'createdAt', op: 'lt', value: '2026-01-01' }, + { any: [{ field: 'status', op: 'eq', value: 'active' }] }, + ], + }, + COLS + ) + ).not.toThrow() + }) + + it('rejects an unknown column', () => { + expect(() => validatePredicate({ all: [{ field: 'nope', op: 'eq', value: 1 }] }, COLS)).toThrow( + /Unknown filter column/ + ) + }) + + it('rejects equality/containment ops on a json column', () => { + for (const op of ['eq', 'ne', 'in', 'nin'] as const) { + const value = op === 'in' || op === 'nin' ? ['x'] : 'x' + expect(() => validatePredicate({ all: [{ field: 'metadata', op, value }] }, COLS)).toThrow( + /json column/ + ) + } + }) + + it('allows text-match / null ops on a json column', () => { + expect(() => + validatePredicate({ all: [{ field: 'metadata', op: 'ilike', value: '*x*' }] }, COLS) + ).not.toThrow() + expect(() => + validatePredicate({ all: [{ field: 'metadata', op: 'isNull' }] }, COLS) + ).not.toThrow() + }) + + it('rejects an empty in/nin array', () => { + expect(() => + validatePredicate({ all: [{ field: 'wins', op: 'in', value: [] }] }, COLS) + ).toThrow(/non-empty array/) + }) + + it('rejects an invalid field name', () => { + expect(() => + validatePredicate({ all: [{ field: "x'; DROP", op: 'eq', value: 1 }] }, COLS) + ).toThrow(/Invalid filter column/) + }) + + it('carries the INVALID_FILTER code', () => { + try { + validatePredicate({ all: [{ field: 'nope', op: 'eq', value: 1 }] }, COLS) + expect.unreachable() + } catch (e) { + expect(e).toBeInstanceOf(TableQueryValidationError) + expect((e as TableQueryValidationError).code).toBe('INVALID_FILTER') + } + }) +}) + +describe('validateSortSpec', () => { + it('accepts real and system columns', () => { + expect(() => + validateSortSpec( + [ + { field: 'wins', direction: 'desc' }, + { field: 'updatedAt', direction: 'asc' }, + ], + COLS + ) + ).not.toThrow() + }) + + it('rejects an unknown sort column with INVALID_ORDER', () => { + try { + validateSortSpec([{ field: 'nope', direction: 'asc' }], COLS) + expect.unreachable() + } catch (e) { + expect((e as TableQueryValidationError).code).toBe('INVALID_ORDER') + } + }) +}) diff --git a/apps/sim/lib/table/query-builder/converters.ts b/apps/sim/lib/table/query-builder/converters.ts index 41dfc0fca32..d95d7dc7035 100644 --- a/apps/sim/lib/table/query-builder/converters.ts +++ b/apps/sim/lib/table/query-builder/converters.ts @@ -4,7 +4,6 @@ import { generateShortId } from '@sim/utils/id' import { isRecordLike } from '@sim/utils/object' -import { predicateToPostgrest, sortSpecToPostgrestOrder } from '@/lib/table/query-builder/postgrest' import type { Filter, FilterOp, @@ -318,22 +317,6 @@ export function sortRulesToSortSpec(rules: SortRule[]): SortSpec | null { return spec.length > 0 ? spec : null } -/** - * Serializes UI filter-builder rules straight to a PostgREST filter querystring, - * so the v2 block can offer the visual builder while the wire stays PostgREST. - * Returns `null` when the builder is empty. - */ -export function filterRulesToPostgrest(rules: FilterRule[]): string | null { - const predicate = filterRulesToPredicate(rules) - return predicate ? predicateToPostgrest(predicate) : null -} - -/** Serializes UI sort-builder rules to a PostgREST `order` string, or `null`. */ -export function sortRulesToPostgrestOrder(rules: SortRule[]): string | null { - const spec = sortRulesToSortSpec(rules) - return spec ? sortSpecToPostgrestOrder(spec) : null -} - function predicateLeafToFilterValue(p: Predicate): Filter[string] { if (p.op === 'isEmpty') return { $empty: true } if (p.op === 'isNotEmpty') return { $empty: false } diff --git a/apps/sim/lib/table/query-builder/postgrest.ts b/apps/sim/lib/table/query-builder/postgrest.ts deleted file mode 100644 index 83ae63cbe41..00000000000 --- a/apps/sim/lib/table/query-builder/postgrest.ts +++ /dev/null @@ -1,461 +0,0 @@ -/** - * PostgREST filter grammar ↔ the internal `TablePredicate` IR. - * - * This is the v2 wire/authoring grammar: a PostgREST-style querystring fragment - * that mothership and clients write, parsed here into the same `TablePredicate` - * the engine already compiles (`buildPredicateClause` → `fieldPredicate`). The - * parser is the validator now that the boundary param is an opaque string — - * unknown ops, bad fields, and malformed input throw `TableQueryValidationError` - * (mapped to HTTP 400). Values are never interpolated into SQL; they ride through - * the parameterized `fieldPredicate` leaf. - * - * Supported (PostgREST token → FilterOp): eq, neq→ne, gt, gte, lt, lte, - * in, like, ilike, match, imatch, is.null→isNull. Negation `not.` is supported - * for `eq`→ne, `in`→nin, `is.null`→isNotNull, `like`→nlike, and `ilike`→nilike. - * Logic: top-level `&` (AND), `and=(...)`, `or=(...)`, and nested - * `and(...)`/`or(...)` inside groups. - * - * Values containing reserved characters are double-quoted; a literal `"` or `\` - * inside a quoted value is backslash-escaped (`\"`, `\\`). The builder-only - * emptiness ops have no single PostgREST form and serialize to desugared groups: - * isEmpty → `or=(f.is.null,f.eq."")`, isNotEmpty → `and=(f.not.is.null,f.neq."")`. - * - * Unsupported (clear error): cs, cd, ov, fts/plfts/phfts/wfts, sl/sr/nxr/nxl/adj - * (no array/range/full-text columns), and `not.` outside eq/in/is.null/like/ilike. - */ - -import { NAME_PATTERN } from '@/lib/table/constants' -import { TableQueryValidationError } from '@/lib/table/errors' -import type { - ColumnDefinition, - FilterOp, - JsonValue, - Predicate, - PredicateNode, - SortDirection, - SortSpec, - TablePredicate, -} from '@/lib/table/types' - -type ColumnType = ColumnDefinition['type'] - -const MAX_FILTER_LENGTH = 4096 - -/** Ops whose value is a text pattern — never coerced to number/boolean. */ -const TEXT_OPS = new Set(['like', 'ilike', 'nlike', 'nilike', 'match', 'imatch']) - -/* ------------------------------- parsing ------------------------------- */ - -/** - * Splits on a delimiter at depth 0, respecting `(...)` nesting and `"..."` - * quotes. Inside quotes, a backslash escapes the next character (`\"`, `\\`), - * matching what {@link serializeValue} emits. - */ -function splitTopLevel(input: string, delimiter: string): string[] { - const parts: string[] = [] - let depth = 0 - let inQuotes = false - let current = '' - for (let i = 0; i < input.length; i++) { - const ch = input[i] - if (inQuotes && ch === '\\' && i + 1 < input.length) { - current += ch + input[i + 1] - i++ - } else if (ch === '"') { - inQuotes = !inQuotes - current += ch - } else if (inQuotes) { - current += ch - } else if (ch === '(') { - depth++ - current += ch - } else if (ch === ')') { - depth-- - if (depth < 0) throw new TableQueryValidationError('Unbalanced parentheses in filter') - current += ch - } else if (ch === delimiter && depth === 0) { - parts.push(current) - current = '' - } else { - current += ch - } - } - if (depth !== 0 || inQuotes) { - throw new TableQueryValidationError('Unbalanced parentheses or quotes in filter') - } - parts.push(current) - return parts -} - -/** - * Strips one layer of PostgREST double-quoting and unescapes `\"`/`\\`, - * the inverse of {@link serializeValue}. - */ -function unquote(raw: string): string { - if (raw.length >= 2 && raw.startsWith('"') && raw.endsWith('"')) { - return raw.slice(1, -1).replace(/\\(["\\])/g, '$1') - } - return raw -} - -function validateField(field: string): void { - if (!NAME_PATTERN.test(field)) { - throw new TableQueryValidationError( - `Invalid filter column "${field}". Use a column name (letters, digits, underscore).` - ) - } -} - -/** Coerces a raw PostgREST value string to the column's stored JSON type. */ -function coerceValue(raw: string, type: ColumnType | undefined): JsonValue { - const v = unquote(raw) - if (type === 'number') { - const n = Number(v) - if (v.trim() === '' || !Number.isFinite(n)) { - throw new TableQueryValidationError(`Expected a number for column value, got "${v}"`) - } - return n - } - if (type === 'boolean') { - if (v === 'true') return true - if (v === 'false') return false - throw new TableQueryValidationError(`Expected true/false for boolean column, got "${v}"`) - } - // string / date / json: keep as text (dates compare as ISO strings). - return v -} - -/** Parses a PostgREST list literal `(a,b,"c,d")` into its elements. */ -function parseList(raw: string): string[] { - const trimmed = raw.trim() - if (!trimmed.startsWith('(') || !trimmed.endsWith(')')) { - throw new TableQueryValidationError(`Expected a list like in.(a,b), got "${raw}"`) - } - const inner = trimmed.slice(1, -1) - if (inner.trim() === '') return [] - return splitTopLevel(inner, ',').map((s) => s.trim()) -} - -const UNSUPPORTED_OPS: Record = { - cs: 'array/jsonb containment', - cd: 'array/jsonb containment', - ov: 'array/range overlap', - fts: 'full-text search', - plfts: 'full-text search', - phfts: 'full-text search', - wfts: 'full-text search', - sl: 'range', - sr: 'range', - nxr: 'range', - nxl: 'range', - adj: 'range', -} - -/** - * Parses one leaf `field` + `op.value` spec into a `Predicate`. `opSpec` is the - * part after the field (e.g. `gte.10`, `in.(a,b)`, `is.null`, `not.eq.5`). - */ -function parseLeaf(field: string, opSpec: string, typeByName: Map): Predicate { - validateField(field) - const type = typeByName.get(field) - - let negated = false - let rest = opSpec - if (rest.startsWith('not.')) { - negated = true - rest = rest.slice(4) - } - - const dot = rest.indexOf('.') - if (dot === -1) throw new TableQueryValidationError(`Malformed filter on "${field}": "${opSpec}"`) - const token = rest.slice(0, dot) - const rawValue = rest.slice(dot + 1) - - if (UNSUPPORTED_OPS[token]) { - throw new TableQueryValidationError( - `Operator "${token}" (${UNSUPPORTED_OPS[token]}) is not supported on user tables.` - ) - } - - // is.null / is.true / is.false - if (token === 'is') { - if (rawValue === 'null') { - return { field, op: negated ? 'isNotNull' : 'isNull' } - } - if (rawValue === 'true' || rawValue === 'false') { - if (negated) throw unsupportedNegation(token) - return { field, op: 'eq', value: rawValue === 'true' } - } - throw new TableQueryValidationError( - `Unsupported "is.${rawValue}" — use is.null, is.true, is.false` - ) - } - - if (token === 'in') { - const items = parseList(rawValue).map((el) => coerceValue(el, type)) - if (items.length === 0) { - // An empty list would compile to no condition at all and silently widen - // the match — reject instead. - throw new TableQueryValidationError(`Empty in.() list on column "${field}"`) - } - return { field, op: negated ? 'nin' : 'in', value: items } - } - - // Scalar / text ops - const opMap: Record = { - eq: 'eq', - neq: 'ne', - gt: 'gt', - gte: 'gte', - lt: 'lt', - lte: 'lte', - like: 'like', - ilike: 'ilike', - match: 'match', - imatch: 'imatch', - } - const op = opMap[token] - if (!op) { - throw new TableQueryValidationError(`Unknown filter operator "${token}" on column "${field}".`) - } - if (negated) { - if (op === 'eq') return { field, op: 'ne', value: scalarValue(rawValue, op, type) } - if (op === 'like') return { field, op: 'nlike', value: scalarValue(rawValue, 'like', type) } - if (op === 'ilike') return { field, op: 'nilike', value: scalarValue(rawValue, 'ilike', type) } - throw unsupportedNegation(token) - } - return { field, op, value: scalarValue(rawValue, op, type) } -} - -function scalarValue(raw: string, op: FilterOp, type: ColumnType | undefined): JsonValue { - // Text/pattern ops keep the raw string (the `*` wildcard and regex are textual). - if (TEXT_OPS.has(op)) return unquote(raw) - return coerceValue(raw, type) -} - -function unsupportedNegation(token: string): TableQueryValidationError { - return new TableQueryValidationError( - `Negation "not.${token}" is not supported — only not.eq, not.in, not.is.null, not.like, not.ilike.` - ) -} - -/** Parses the comma-separated items inside an `and(...)`/`or(...)` group. */ -function parseGroupItems(content: string, typeByName: Map): PredicateNode[] { - return splitTopLevel(content, ',').map((item) => parseGroupItem(item.trim(), typeByName)) -} - -/** A single item inside a group: a nested `and()/or()` group, or a `field.op.value` leaf. */ -function parseGroupItem(item: string, typeByName: Map): PredicateNode { - const groupMatch = /^(and|or)\((.*)\)$/s.exec(item) - if (groupMatch) { - const members = parseGroupItems(groupMatch[2], typeByName) - return groupMatch[1] === 'and' ? { all: members } : { any: members } - } - // Leaf in dot-form: field.op.value — field is up to the first dot. - const dot = item.indexOf('.') - if (dot === -1) throw new TableQueryValidationError(`Malformed filter condition: "${item}"`) - return parseLeaf(item.slice(0, dot), item.slice(dot + 1), typeByName) -} - -/** - * Parses a PostgREST filter querystring fragment into a `TablePredicate`. - * Top-level `&`-separated params AND together; `and=(...)`/`or=(...)` form groups. - */ -export function parsePostgrestFilter(input: string, columns: ColumnDefinition[]): TablePredicate { - if (typeof input !== 'string' || input.trim() === '') { - throw new TableQueryValidationError('Filter is empty') - } - if (input.length > MAX_FILTER_LENGTH) { - throw new TableQueryValidationError(`Filter exceeds ${MAX_FILTER_LENGTH} characters`) - } - const typeByName = new Map(columns.map((c) => [c.name, c.type])) - - const nodes: PredicateNode[] = [] - for (const segment of splitTopLevel(input, '&')) { - const trimmed = segment.trim() - if (trimmed === '') continue - const eq = trimmed.indexOf('=') - if (eq === -1) throw new TableQueryValidationError(`Malformed filter param: "${trimmed}"`) - const key = trimmed.slice(0, eq).trim() - const rhs = trimmed.slice(eq + 1).trim() - - if (key === 'and' || key === 'or') { - const members = parseList(rhs) - if (members.length === 0) { - // An empty group compiles to no condition — a silent no-op filter. - throw new TableQueryValidationError(`Empty ${key}=() group in filter`) - } - // re-parse as group items (parseList split top-level commas already) - const parsed = members.map((m) => parseGroupItem(m, typeByName)) - nodes.push(key === 'and' ? { all: parsed } : { any: parsed }) - continue - } - nodes.push(parseLeaf(key, rhs, typeByName)) - } - - if (nodes.length === 0) throw new TableQueryValidationError('Filter has no conditions') - return { all: nodes } -} - -/** Parses a PostgREST `order` value (`col.desc,col2.asc`) into a SortSpec. */ -export function parsePostgrestOrder(input: string, columns: ColumnDefinition[]): SortSpec { - const valid = new Set(columns.map((c) => c.name).concat(['createdAt', 'updatedAt'])) - const spec: SortSpec = [] - for (const part of input.split(',')) { - const seg = part.trim() - if (seg === '') continue - const segments = seg.split('.') - if (segments.length > 2) { - throw new TableQueryValidationError(`Malformed sort "${seg}" — use column or column.desc`) - } - const [field, dirRaw = 'asc'] = segments - validateField(field) - if (!valid.has(field)) { - throw new TableQueryValidationError(`Unknown sort column "${field}"`) - } - // Strict: a typo'd direction must not silently sort ascending. - if (dirRaw !== 'asc' && dirRaw !== 'desc') { - throw new TableQueryValidationError( - `Unknown sort direction "${dirRaw}" on "${field}" — use asc or desc (lowercase)` - ) - } - const direction: SortDirection = dirRaw - spec.push({ field, direction }) - } - return spec -} - -/* ------------------------------ serializing ------------------------------ */ - -function leafToOpSpec(p: Predicate): string { - const v = p.value - switch (p.op) { - case 'eq': - return `eq.${serializeValue(v)}` - case 'ne': - return `neq.${serializeValue(v)}` - case 'gt': - case 'gte': - case 'lt': - case 'lte': - return `${p.op}.${serializeValue(v)}` - case 'in': - return `in.(${(v as JsonValue[]).map(serializeValue).join(',')})` - case 'nin': - return `not.in.(${(v as JsonValue[]).map(serializeValue).join(',')})` - case 'like': - case 'ilike': - case 'match': - case 'imatch': - return `${p.op}.${serializeValue(v)}` - case 'nlike': - return `not.like.${serializeValue(v)}` - case 'nilike': - return `not.ilike.${serializeValue(v)}` - // Builder-only substring ops: build the whole wildcard pattern FIRST, then - // serialize it as ONE value, so quoting wraps the entire pattern and the - // parser's whole-value unquote recovers it (`ilike."*example.com*"`). - case 'contains': - return `ilike.${serializeValue(`*${asText(v)}*`)}` - case 'ncontains': - return `not.ilike.${serializeValue(`*${asText(v)}*`)}` - case 'startsWith': - return `ilike.${serializeValue(`${asText(v)}*`)}` - case 'endsWith': - return `ilike.${serializeValue(`*${asText(v)}`)}` - case 'isNull': - return 'is.null' - case 'isNotNull': - return 'not.is.null' - default: - // isEmpty/isNotEmpty are desugared into groups before leaves serialize. - throw new TableQueryValidationError(`Cannot serialize operator "${p.op}"`) - } -} - -function asText(value: JsonValue | undefined): string { - return value === null || value === undefined ? '' : String(value) -} - -/** - * Quotes values containing reserved chars (and the empty string) so they - * survive a round-trip; literal `"`/`\` inside a quoted value are escaped. - * Inverse of {@link unquote}. - */ -function serializeValue(value: JsonValue | undefined): string { - const s = value === null || value === undefined ? 'null' : String(value) - if (s === '') return '""' - if (!/[,.()&="\\]/.test(s)) return s - return `"${s.replace(/\\/g, '\\\\').replace(/"/g, '\\"')}"` -} - -function isGroup(node: PredicateNode): node is TablePredicate { - return 'all' in node || 'any' in node -} - -/** - * Rewrites emptiness leaves into their PostgREST-expressible groups, preserving - * the legacy "null OR empty string" semantics through the string round-trip: - * isEmpty → `(f is null OR f = '')`, isNotEmpty → `(f not null AND f <> '')`. - */ -function desugarEmptiness(node: PredicateNode): PredicateNode { - if (isGroup(node)) { - return 'all' in node - ? { all: node.all.map(desugarEmptiness) } - : { any: node.any.map(desugarEmptiness) } - } - if (node.op === 'isEmpty') { - return { - any: [ - { field: node.field, op: 'isNull' }, - { field: node.field, op: 'eq', value: '' }, - ], - } - } - if (node.op === 'isNotEmpty') { - return { - all: [ - { field: node.field, op: 'isNotNull' }, - { field: node.field, op: 'ne', value: '' }, - ], - } - } - return node -} - -/** Serializes a node as a group item (dot-form leaves, function-form groups). */ -function nodeToGroupItem(node: PredicateNode): string { - if (isGroup(node)) { - const isAll = 'all' in node - const members = isAll ? node.all : node.any - // A single-member group adds no logic — flatten it so a builder-emitted - // `{ all: [leaf] }` serializes to the leaf, not a redundant `and(leaf)`. - if (members.length === 1) return nodeToGroupItem(members[0]) - return `${isAll ? 'and' : 'or'}(${members.map(nodeToGroupItem).join(',')})` - } - return `${node.field}.${leafToOpSpec(node)}` -} - -/** - * Serializes a `TablePredicate` to a PostgREST querystring fragment (for the - * builder UI). A top-level `all` becomes `&`-joined params; `any` becomes - * `or=(...)`; nested groups use the function form. - */ -export function predicateToPostgrest(predicate: TablePredicate): string { - const desugared = desugarEmptiness(predicate) as TablePredicate - if ('any' in desugared) { - return `or=(${desugared.any.map(nodeToGroupItem).join(',')})` - } - return desugared.all - .map((node) => - isGroup(node) - ? `${'all' in node ? 'and' : 'or'}=(${(('all' in node ? node.all : node.any) as PredicateNode[]).map(nodeToGroupItem).join(',')})` - : `${node.field}=${leafToOpSpec(node)}` - ) - .join('&') -} - -/** Serializes a SortSpec to a PostgREST `order` value. */ -export function sortSpecToPostgrestOrder(sort: SortSpec): string { - return sort.map((s) => `${s.field}.${s.direction}`).join(',') -} diff --git a/apps/sim/lib/table/query-builder/validate.ts b/apps/sim/lib/table/query-builder/validate.ts new file mode 100644 index 00000000000..e0e781c4ea2 --- /dev/null +++ b/apps/sim/lib/table/query-builder/validate.ts @@ -0,0 +1,113 @@ +import { NAME_PATTERN } from '@/lib/table/constants' +import { TableQueryValidationError } from '@/lib/table/errors' +import type { + ColumnDefinition, + ColumnType, + FilterOp, + Predicate, + PredicateNode, + SortSpec, + TablePredicate, +} from '@/lib/table/types' + +/** + * Schema-aware validation for the typed predicate/sort wire. The engine + * (`buildPredicateClause`) trusts its input, so this is the boundary gate that + * every caller-supplied filter/sort passes through — the same checks the old + * PostgREST parser did inline, now grammar-agnostic and applied to the object + * directly (predicates are column-NAME-keyed at the boundary, translated to ids + * afterwards). + */ + +/** Equality/containment ops that are meaningless on a `json` column (they never match). */ +const CONTAINMENT_OPS = new Set(['eq', 'ne', 'in', 'nin']) + +/** System timestamp columns are filterable/sortable but are not in `schema.columns`. */ +const SYSTEM_COLUMN_TYPES: ReadonlyArray<[string, ColumnType]> = [ + ['createdAt', 'date'], + ['updatedAt', 'date'], +] + +function buildTypeByName(columns: ColumnDefinition[]): Map { + const typeByName = new Map(columns.map((c) => [c.name, c.type])) + for (const [name, type] of SYSTEM_COLUMN_TYPES) typeByName.set(name, type) + return typeByName +} + +function validateFieldName(field: string): void { + if (!NAME_PATTERN.test(field)) { + throw new TableQueryValidationError( + `Invalid filter column "${field}". Use a column name (letters, digits, underscore).`, + 'INVALID_FILTER' + ) + } +} + +function validateLeaf(leaf: Predicate, typeByName: Map): void { + validateFieldName(leaf.field) + if (!typeByName.has(leaf.field)) { + throw new TableQueryValidationError( + `Unknown filter column "${leaf.field}". It is not a column on this table.`, + 'INVALID_FILTER' + ) + } + if (typeByName.get(leaf.field) === 'json' && CONTAINMENT_OPS.has(leaf.op)) { + throw new TableQueryValidationError( + `Operator "${leaf.op}" is not supported on json column "${leaf.field}" — use like/ilike for text match or is.null.`, + 'INVALID_FILTER' + ) + } + if ( + (leaf.op === 'in' || leaf.op === 'nin') && + (!Array.isArray(leaf.value) || leaf.value.length === 0) + ) { + throw new TableQueryValidationError( + `Operator "${leaf.op}" on column "${leaf.field}" requires a non-empty array value.`, + 'INVALID_FILTER' + ) + } +} + +function validateNode(node: PredicateNode, typeByName: Map): void { + // Guard before the `in` checks below: an untrusted caller (copilot args, a raw + // block value) can hand us a string/number/null, where `'all' in node` throws + // a raw TypeError. Fail with a clean, actionable message instead. + if (typeof node !== 'object' || node === null) { + throw new TableQueryValidationError( + 'Filter must be a predicate object ({ all | any: [...] }).', + 'INVALID_FILTER' + ) + } + if ('all' in node || 'any' in node) { + const members = 'all' in node ? node.all : node.any + if (!Array.isArray(members)) { + throw new TableQueryValidationError( + 'A filter group ({ all | any }) must be an array of conditions.', + 'INVALID_FILTER' + ) + } + for (const child of members) validateNode(child, typeByName) + return + } + validateLeaf(node as Predicate, typeByName) +} + +/** + * Validates a name-keyed predicate against the table schema: every leaf field + * exists, no equality/containment op targets a `json` column, `in`/`nin` carry a + * non-empty array. Throws {@link TableQueryValidationError} (`INVALID_FILTER`). + */ +export function validatePredicate(predicate: TablePredicate, columns: ColumnDefinition[]): void { + validateNode(predicate, buildTypeByName(columns)) +} + +/** Validates a name-keyed sort spec: every field is a real or system column. */ +export function validateSortSpec(spec: SortSpec, columns: ColumnDefinition[]): void { + const typeByName = buildTypeByName(columns) + for (const { field } of spec) { + validateFieldName(field) + if (!typeByName.has(field)) { + throw new TableQueryValidationError(`Unknown sort column "${field}"`, 'INVALID_ORDER') + } + } +} diff --git a/apps/sim/lib/table/rows/__tests__/cursor.test.ts b/apps/sim/lib/table/rows/__tests__/cursor.test.ts index 80a5163ec45..baf9a3434f6 100644 --- a/apps/sim/lib/table/rows/__tests__/cursor.test.ts +++ b/apps/sim/lib/table/rows/__tests__/cursor.test.ts @@ -54,10 +54,27 @@ describe('cursor codec', () => { expect(token).toMatch(/^[A-Za-z0-9_-]+$/) }) + it('stamps a version field and rejects any other version', () => { + const token = encodeCursor({ + lastRow: { id: 'r', orderKey: 'a' }, + keysetValid: true, + nextOffset: 0, + }) + expect(JSON.parse(Buffer.from(token, 'base64url').toString('utf8')).v).toBe(1) + // A future/unknown version fails cleanly instead of being misread. + expect(() => decodeCursor(Buffer.from('{"v":2,"o":0}').toString('base64url'))).toThrow( + 'Invalid cursor' + ) + // A pre-version token (no `v`) is no longer accepted. + expect(() => decodeCursor(Buffer.from('{"o":0}').toString('base64url'))).toThrow( + 'Invalid cursor' + ) + }) + it('throws on a malformed cursor', () => { expect(() => decodeCursor('not-base64-$$$')).toThrow('Invalid cursor') // Valid base64url but wrong shape. - expect(() => decodeCursor(Buffer.from('{"x":1}').toString('base64url'))).toThrow( + expect(() => decodeCursor(Buffer.from('{"v":1,"x":1}').toString('base64url'))).toThrow( 'Invalid cursor' ) }) @@ -69,10 +86,10 @@ describe('cursor codec', () => { }) it('rejects negative and non-integer offsets', () => { - expect(() => decodeCursor(Buffer.from('{"o":-1}').toString('base64url'))).toThrow( + expect(() => decodeCursor(Buffer.from('{"v":1,"o":-1}').toString('base64url'))).toThrow( 'Invalid cursor' ) - expect(() => decodeCursor(Buffer.from('{"o":1.5}').toString('base64url'))).toThrow( + expect(() => decodeCursor(Buffer.from('{"v":1,"o":1.5}').toString('base64url'))).toThrow( 'Invalid cursor' ) }) diff --git a/apps/sim/lib/table/rows/cursor.ts b/apps/sim/lib/table/rows/cursor.ts index 2016c9d0c84..05fa15a8ecd 100644 --- a/apps/sim/lib/table/rows/cursor.ts +++ b/apps/sim/lib/table/rows/cursor.ts @@ -13,9 +13,22 @@ * the unkeyed rows consumed after it. */ +import { TableQueryValidationError } from '@/lib/table/errors' import type { TableRow, TableRowsCursor } from '@/lib/table/types' -type CursorPayload = { k: string; i: string } | { o: number } | { k: string; i: string; o: number } +/** + * Cursor payload version. Every encoded token carries `v`; decode rejects any + * other value so a future shape change (new `v`) fails cleanly instead of being + * misread against the current field set. + */ +const CURSOR_VERSION = 1 + +type CursorBody = { k: string; i: string } | { o: number } | { k: string; i: string; o: number } +type CursorPayload = CursorBody & { v: number } + +function invalidCursor(): never { + throw new TableQueryValidationError('Invalid cursor', 'INVALID_CURSOR') +} function toBase64Url(json: string): string { return Buffer.from(json, 'utf8').toString('base64url') @@ -43,22 +56,23 @@ export function encodeCursor(args: { nextOffset: number seekBase?: { anchor: TableRowsCursor; offsetFromAnchor: number } }): string { - let payload: CursorPayload + let body: CursorBody if (args.keysetValid && args.lastRow.orderKey) { - payload = { k: args.lastRow.orderKey, i: args.lastRow.id } + body = { k: args.lastRow.orderKey, i: args.lastRow.id } } else if (args.seekBase) { // An anchor is in effect (inbound seek or last keyed row) but a plain // keyset can't stand alone — resume by seeking the anchor then offsetting // past the rows consumed after it. Never valid under a custom sort, where // callers must not pass a seekBase. - payload = { + body = { k: args.seekBase.anchor.orderKey, i: args.seekBase.anchor.id, o: args.seekBase.offsetFromAnchor, } } else { - payload = { o: args.nextOffset } + body = { o: args.nextOffset } } + const payload: CursorPayload = { ...body, v: CURSOR_VERSION } return toBase64Url(JSON.stringify(payload)) } @@ -68,13 +82,14 @@ export function decodeCursor(token: string): { after?: TableRowsCursor; offset?: try { payload = JSON.parse(fromBase64Url(token)) } catch { - throw new Error('Invalid cursor') + invalidCursor() } if (typeof payload !== 'object' || payload === null || Array.isArray(payload)) { - throw new Error('Invalid cursor') + invalidCursor() } const record = payload as Record + if (record.v !== CURSOR_VERSION) invalidCursor() const hasKeyset = typeof record.k === 'string' && typeof record.i === 'string' const hasOffset = typeof record.o === 'number' && Number.isInteger(record.o) && record.o >= 0 @@ -90,5 +105,5 @@ export function decodeCursor(token: string): { after?: TableRowsCursor; offset?: if (hasOffset) { return { offset: record.o as number } } - throw new Error('Invalid cursor') + invalidCursor() } diff --git a/apps/sim/lib/table/rows/service.ts b/apps/sim/lib/table/rows/service.ts index 783d5cc85d6..52d4045ea14 100644 --- a/apps/sim/lib/table/rows/service.ts +++ b/apps/sim/lib/table/rows/service.ts @@ -28,7 +28,12 @@ import { getColumnId } from '@/lib/table/column-keys' import { TABLE_LIMITS, USER_TABLE_ROWS_SQL_NAME } from '@/lib/table/constants' import { TableQueryValidationError } from '@/lib/table/errors' import { nKeysBetween } from '@/lib/table/order-key' -import { type DbExecutor, type DbTransaction, withSeqscanOff } from '@/lib/table/planner' +import { + type DbExecutor, + type DbTransaction, + withReadGuards, + withSeqscanOff, +} from '@/lib/table/planner' import { encodeCursor } from '@/lib/table/rows/cursor' import { applyExecutionsPatch, @@ -1030,11 +1035,15 @@ export async function queryRows( const countPromise = includeTotal ? hasFilter ? countRowsTenantBounded(whereClause) - : db - .select({ count: count() }) - .from(userTableRows) - .where(whereClause ?? baseConditions) - .then((r) => Number(r[0].count)) + : // Unfiltered count plans an index-only scan on the table_id prefix, but + // still runs under the read timeout so it can't pin a connection. + withReadGuards(async (trx) => { + const [r] = await trx + .select({ count: count() }) + .from(userTableRows) + .where(whereClause ?? baseConditions) + return Number(r.count) + }) : null // The inbound seek is honored whenever there's no custom sort (matching the @@ -1184,12 +1193,13 @@ async function fetchRowsBounded(params: BoundedFetchParams): Promise 0 ? query.offset(batchOffset) : query } - // Custom column sorts order by `data->>'col'` — unestimatable, so left - // alone the planner seq-scans and sorts the whole shared relation on every - // page (9.7s measured on a 1M-row table; 0.76s tenant-bounded). One tx per - // batch: SET LOCAL dies with the tx, and holding a tx across JS accounting - // between batches would pin a pooled connection. - return sorted ? withSeqscanOff(async (trx) => buildQuery(trx)) : buildQuery(db) + // One tx per batch (SET LOCAL dies with it; holding a tx across JS + // accounting between batches would pin a pooled connection). Custom sorts + // order by `data->>'col'` — unestimatable — so they also penalize seq scans + // (9.7s→0.76s on a 1M-row table); default-order pages stream the index and + // just need the read timeout. Either way the batch runs under a statement + // timeout so a pathological filter can't scan unbounded. + return withReadGuards(async (trx) => buildQuery(trx), { seqscanOff: sorted }) } for (let iteration = 0; iteration < MAX_QUERY_BATCHES; iteration++) { @@ -1208,7 +1218,8 @@ async function fetchRowsBounded(params: BoundedFetchParams): Promise>` builders below, which would + // silently match nothing (the key never exists in `data`). + if (isSystemColumn(field)) { + return buildSystemColumnClause(tableName, field, op, value) + } + switch (op) { case 'eq': return buildContainmentClause(tableName, field, value as JsonValue) @@ -523,6 +530,67 @@ function buildLogicalClause( return sql`(${sql.join(clauses, sql.raw(` ${operator} `))})` } +/** + * Row-level system columns exposed for filter/sort: their wire name → the real + * `timestamptz` column. Not stored in `data`, so they need real-column SQL, not + * the `data->>'field'` extraction every other builder uses. + */ +const SYSTEM_COLUMNS: Readonly> = { + createdAt: 'created_at', + updatedAt: 'updated_at', +} + +function isSystemColumn(field: string): boolean { + return Object.hasOwn(SYSTEM_COLUMNS, field) +} + +/** + * Builds a predicate against a `timestamptz` system column. Values are ISO + * strings cast to `timestamptz`. Only comparison/equality/null ops make sense; + * text/pattern ops are rejected with an actionable error. + */ +function buildSystemColumnClause( + tableName: string, + field: string, + op: FilterOp, + value: JsonValue | undefined +): SQL | undefined { + const col = sql.raw(`${tableName}.${SYSTEM_COLUMNS[field]}`) + const ts = (v: JsonValue | undefined) => sql`${String(v)}::timestamptz` + switch (op) { + case 'eq': + return sql`${col} = ${ts(value)}` + case 'ne': + return sql`${col} <> ${ts(value)}` + case 'gt': + return sql`${col} > ${ts(value)}` + case 'gte': + return sql`${col} >= ${ts(value)}` + case 'lt': + return sql`${col} < ${ts(value)}` + case 'lte': + return sql`${col} <= ${ts(value)}` + case 'in': { + if (!Array.isArray(value) || value.length === 0) return undefined + return sql`${col} IN (${sql.join(value.map(ts), sql.raw(', '))})` + } + case 'nin': { + if (!Array.isArray(value) || value.length === 0) return undefined + return sql`${col} NOT IN (${sql.join(value.map(ts), sql.raw(', '))})` + } + case 'isNull': + case 'isEmpty': + return sql`${col} IS NULL` + case 'isNotNull': + case 'isNotEmpty': + return sql`${col} IS NOT NULL` + default: + throw new TableQueryValidationError( + `Operator "${op}" is not supported on the timestamp column "${field}" — use eq, neq, gt, gte, lt, lte, in, is.null.` + ) + } +} + /** Builds JSONB containment clause: `data @> '{"field": value}'::jsonb` (uses GIN index) */ function buildContainmentClause(tableName: string, field: string, value: JsonValue): SQL { const jsonObj = JSON.stringify({ [field]: value }) @@ -715,8 +783,8 @@ function buildSortFieldClause( const escapedField = field.replace(/'/g, "''") const directionSql = direction.toUpperCase() - if (field === 'createdAt' || field === 'updatedAt') { - return sql.raw(`${tableName}.${escapedField} ${directionSql}`) + if (isSystemColumn(field)) { + return sql.raw(`${tableName}.${SYSTEM_COLUMNS[field]} ${directionSql}`) } const jsonbExtract = `${tableName}.data->>'${escapedField}'` diff --git a/apps/sim/lib/table/types.ts b/apps/sim/lib/table/types.ts index 53528f958af..b6c6e932e6a 100644 --- a/apps/sim/lib/table/types.ts +++ b/apps/sim/lib/table/types.ts @@ -2,7 +2,7 @@ * Type definitions for user-defined tables. */ -import type { COLUMN_TYPES } from '@/lib/table/constants' +import type { COLUMN_TYPES, FILTER_OPS } from '@/lib/table/constants' export type ColumnValue = string | number | boolean | null | Date export type JsonValue = ColumnValue | JsonValue[] | { [key: string]: JsonValue } @@ -432,29 +432,7 @@ export interface Filter { * This is the canonical operator set the shared `fieldPredicate` leaf * understands; the legacy `$`-operators normalize onto it. */ -export type FilterOp = - | 'eq' - | 'ne' - | 'gt' - | 'gte' - | 'lt' - | 'lte' - | 'in' - | 'nin' - | 'contains' - | 'ncontains' - | 'startsWith' - | 'endsWith' - | 'like' - | 'ilike' - | 'nlike' - | 'nilike' - | 'match' - | 'imatch' - | 'isEmpty' - | 'isNotEmpty' - | 'isNull' - | 'isNotNull' +export type FilterOp = (typeof FILTER_OPS)[number] /** A single v2 leaf predicate: `field op value`. `value` is omitted for `isEmpty`/`isNotEmpty`. */ export interface Predicate { diff --git a/apps/sim/tools/table/query_rows_v2.ts b/apps/sim/tools/table/query_rows_v2.ts index 8874e313658..6f589de7450 100644 --- a/apps/sim/tools/table/query_rows_v2.ts +++ b/apps/sim/tools/table/query_rows_v2.ts @@ -2,18 +2,18 @@ import type { TableQueryV2Response, TableRowQueryV2Params } from '@/tools/table/ import type { ToolConfig } from '@/tools/types' /** - * v2 row query: PostgREST filter grammar + opaque cursor pagination (no offset). + * v2 row query: typed predicate grammar + opaque cursor pagination (no offset). * Hits POST `/api/table/[tableId]/query`. */ export const tableQueryRowsV2Tool: ToolConfig = { id: 'table_query_rows_v2', name: 'Query Rows', description: - 'Query rows with a PostgREST filter string and cursor pagination. ' + - 'Filter is a querystring fragment: `wins=gte.10&status=in.(active,pending)` (top-level params AND; ' + - '`or=(a.eq.1,b.eq.2)` / `and=(...)` for groups). Operators: eq, neq, gt, gte, lt, lte, in, like, ilike, ' + - 'match, imatch, is.null (negate with not., e.g. not.in, not.is.null). ' + - 'Order is PostgREST `order` (e.g. `wins.desc,name.asc`). Omit limit to return the entire result — ' + + 'Query rows with a typed predicate filter and cursor pagination. ' + + 'Filter is a predicate tree: `{"all":[{"field":"wins","op":"gte","value":10},{"field":"status","op":"in","value":["active","pending"]}]}` ' + + '(`all` = AND, `any` = OR; groups nest). Operators: eq, ne, gt, gte, lt, lte, in, nin, like, ilike, ' + + 'nlike, nilike, contains, startsWith, endsWith, isNull, isNotNull, isEmpty, isNotEmpty. ' + + 'Order is a sort spec, e.g. `[{"field":"wins","direction":"desc"}]`. Omit limit to return the entire result — ' + 'the query fails if it exceeds the 5MB budget (narrow with a filter or set a limit). With a limit, ' + 'a page can end early at the byte budget: a non-null nextCursor means more rows exist — pass it back ' + 'as cursor to continue; never infer completion from page size.', @@ -27,16 +27,16 @@ export const tableQueryRowsV2Tool: ToolConfig Date: Fri, 24 Jul 2026 17:40:03 -0700 Subject: [PATCH 05/26] feat(table): gate the v2 tables API behind the tables-v2-api flag Adds a runtime feature flag gating the three v2 HTTP surfaces (GET /api/v2/tables, POST /api/v2/tables/[tableId]/query, POST /api/table/[tableId]/query), returning 404 when off. Gated by userId + the workspace's org cohort via AppConfig; off-AppConfig falls back to the TABLES_V2_API secret (off by default). Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_011M563cbFy2S74GvSDf2C3R --- .../api/table/[tableId]/query/route.test.ts | 1 + .../app/api/table/[tableId]/query/route.ts | 5 +++- apps/sim/app/api/table/utils.ts | 16 +++++++++++ .../v2/tables/[tableId]/query/route.test.ts | 18 ++++++++++--- .../api/v2/tables/[tableId]/query/route.ts | 5 +++- apps/sim/app/api/v2/tables/route.test.ts | 5 ++++ apps/sim/app/api/v2/tables/route.ts | 5 +++- apps/sim/lib/core/config/env.ts | 1 + .../sim/lib/core/config/feature-flags.test.ts | 27 +++++++++++++++++++ apps/sim/lib/core/config/feature-flags.ts | 8 ++++++ apps/sim/lib/workspaces/utils.ts | 15 +++++++++++ 11 files changed, 99 insertions(+), 7 deletions(-) diff --git a/apps/sim/app/api/table/[tableId]/query/route.test.ts b/apps/sim/app/api/table/[tableId]/query/route.test.ts index b6b6ba6e2c5..99cff93635d 100644 --- a/apps/sim/app/api/table/[tableId]/query/route.test.ts +++ b/apps/sim/app/api/table/[tableId]/query/route.test.ts @@ -21,6 +21,7 @@ vi.mock('@/app/api/table/utils', async () => { checkAccess: mockCheckAccess, accessError: (result: { status: number }) => NextResponse.json({ error: 'Access denied' }, { status: result.status }), + tablesV2GateError: () => null, } }) diff --git a/apps/sim/app/api/table/[tableId]/query/route.ts b/apps/sim/app/api/table/[tableId]/query/route.ts index 7cfd2954497..c412f732695 100644 --- a/apps/sim/app/api/table/[tableId]/query/route.ts +++ b/apps/sim/app/api/table/[tableId]/query/route.ts @@ -13,7 +13,7 @@ import { validatePredicate, validateSortSpec } from '@/lib/table/query-builder/v import { decodeCursor } from '@/lib/table/rows/cursor' import { queryRows } from '@/lib/table/rows/service' import { rowWireTranslators } from '@/app/api/table/row-wire' -import { accessError, checkAccess } from '@/app/api/table/utils' +import { accessError, checkAccess, tablesV2GateError } from '@/app/api/table/utils' const logger = createLogger('TableRowQueryAPI') @@ -40,6 +40,9 @@ export const POST = withRouteHandler(async (request: NextRequest, context: RowQu const { params, body } = parsed.data const { tableId } = params + const gateError = await tablesV2GateError(authResult.userId, body.workspaceId) + if (gateError) return gateError + const accessResult = await checkAccess(tableId, authResult.userId, 'read') if (!accessResult.ok) return accessError(accessResult, requestId, tableId) const { table } = accessResult diff --git a/apps/sim/app/api/table/utils.ts b/apps/sim/app/api/table/utils.ts index 7424258ad0e..8a4260ac10b 100644 --- a/apps/sim/app/api/table/utils.ts +++ b/apps/sim/app/api/table/utils.ts @@ -7,11 +7,27 @@ import { deleteTableColumnBodySchema, updateTableColumnBodySchema, } from '@/lib/api/contracts/tables' +import { isFeatureEnabled } from '@/lib/core/config/feature-flags' import type { MultipartError } from '@/lib/core/utils/multipart' import type { ColumnDefinition, Filter, TableDefinition } from '@/lib/table' import { buildFilterClause, getTableById, TableQueryValidationError } from '@/lib/table' import { USER_TABLE_ROWS_SQL_NAME } from '@/lib/table/constants' import { getUserEntityPermissions } from '@/lib/workspaces/permissions/utils' +import { getWorkspaceOrganizationId } from '@/lib/workspaces/utils' + +/** + * Gate for the v2 tables HTTP API (`tables-v2-api` flag). Returns a 404 response + * when the flag is off for the caller — the surface behaves as if it doesn't + * exist — or `null` to proceed. Gated by userId + the workspace's org cohort. + */ +export async function tablesV2GateError( + userId: string, + workspaceId: string +): Promise { + const orgId = await getWorkspaceOrganizationId(workspaceId) + if (await isFeatureEnabled('tables-v2-api', { userId, orgId })) return null + return NextResponse.json({ error: 'Not found' }, { status: 404 }) +} /** * Validates a `filter` against the table's column schema, returning a 400 response on a bad field diff --git a/apps/sim/app/api/v2/tables/[tableId]/query/route.test.ts b/apps/sim/app/api/v2/tables/[tableId]/query/route.test.ts index f60aac10be5..d03b295b192 100644 --- a/apps/sim/app/api/v2/tables/[tableId]/query/route.test.ts +++ b/apps/sim/app/api/v2/tables/[tableId]/query/route.test.ts @@ -9,14 +9,14 @@ import { NextRequest } from 'next/server' import { beforeEach, describe, expect, it, vi } from 'vitest' import type { TableDefinition } from '@/lib/table/types' -const { mockCheckAccess, mockQueryRows, mockCheckRateLimit, mockCheckWorkspaceScope } = vi.hoisted( - () => ({ +const { mockCheckAccess, mockQueryRows, mockCheckRateLimit, mockCheckWorkspaceScope, mockGate } = + vi.hoisted(() => ({ mockCheckAccess: vi.fn(), mockQueryRows: vi.fn(), mockCheckRateLimit: vi.fn(), mockCheckWorkspaceScope: vi.fn(), - }) -) + mockGate: vi.fn(), + })) vi.mock('@/app/api/v1/middleware', async () => { const { NextResponse } = await import('next/server') @@ -37,6 +37,7 @@ vi.mock('@/app/api/table/utils', async () => { checkAccess: mockCheckAccess, accessError: (result: { status: number }) => NextResponse.json({ error: 'Access denied' }, { status: result.status }), + tablesV2GateError: mockGate, } }) @@ -103,6 +104,15 @@ describe('POST /api/v2/tables/[tableId]/query', () => { mockCheckWorkspaceScope.mockResolvedValue(null) mockCheckAccess.mockResolvedValue({ ok: true, table: buildTable() }) mockQueryRows.mockResolvedValue(EMPTY_RESULT) + mockGate.mockResolvedValue(null) + }) + + it('returns 404 when the tables-v2-api flag is off', async () => { + const { NextResponse } = await import('next/server') + mockGate.mockResolvedValue(NextResponse.json({ error: 'Not found' }, { status: 404 })) + const res = await callQuery({ workspaceId: 'workspace-1' }) + expect(res.status).toBe(404) + expect(mockQueryRows).not.toHaveBeenCalled() }) it('translates a name-keyed predicate to storage ids', async () => { diff --git a/apps/sim/app/api/v2/tables/[tableId]/query/route.ts b/apps/sim/app/api/v2/tables/[tableId]/query/route.ts index 0041ff9f4ba..706443d6fef 100644 --- a/apps/sim/app/api/v2/tables/[tableId]/query/route.ts +++ b/apps/sim/app/api/v2/tables/[tableId]/query/route.ts @@ -16,7 +16,7 @@ import { TableQueryValidationError } from '@/lib/table/errors' import { validatePredicate, validateSortSpec } from '@/lib/table/query-builder/validate' import { decodeCursor } from '@/lib/table/rows/cursor' import { queryRows } from '@/lib/table/rows/service' -import { accessError, checkAccess } from '@/app/api/table/utils' +import { accessError, checkAccess, tablesV2GateError } from '@/app/api/table/utils' import { checkRateLimit, checkWorkspaceScope, @@ -72,6 +72,9 @@ export const POST = withRouteHandler(async (request: NextRequest, context: Query const { tableId } = parsed.data.params const { workspaceId, sort, cursor: cursorToken, limit } = parsed.data.body + const gateError = await tablesV2GateError(userId, workspaceId) + if (gateError) return gateError + const scopeError = await checkWorkspaceScope(rateLimit, workspaceId) if (scopeError) return scopeError diff --git a/apps/sim/app/api/v2/tables/route.test.ts b/apps/sim/app/api/v2/tables/route.test.ts index 4b2711653da..f4356b16f5b 100644 --- a/apps/sim/app/api/v2/tables/route.test.ts +++ b/apps/sim/app/api/v2/tables/route.test.ts @@ -31,6 +31,11 @@ vi.mock('@/lib/table', async () => { return { ...actual, listTables: mockListTables } }) +vi.mock('@/app/api/table/utils', () => ({ + normalizeColumn: (col: Record) => col, + tablesV2GateError: () => null, +})) + import { GET } from '@/app/api/v2/tables/route' function buildTable(): TableDefinition { diff --git a/apps/sim/app/api/v2/tables/route.ts b/apps/sim/app/api/v2/tables/route.ts index 87b4da8d014..07e79730b76 100644 --- a/apps/sim/app/api/v2/tables/route.ts +++ b/apps/sim/app/api/v2/tables/route.ts @@ -5,7 +5,7 @@ import { parseRequest, validationErrorResponseFromError } from '@/lib/api/server import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { listTables, type TableSchema } from '@/lib/table' -import { normalizeColumn } from '@/app/api/table/utils' +import { normalizeColumn, tablesV2GateError } from '@/app/api/table/utils' import { checkRateLimit, createRateLimitResponse, @@ -34,6 +34,9 @@ export const GET = withRouteHandler(async (request: NextRequest) => { const { workspaceId } = parsed.data.query + const gateError = await tablesV2GateError(userId, workspaceId) + if (gateError) return gateError + const accessError = await validateWorkspaceAccess(rateLimit, userId, workspaceId) if (accessError) return accessError diff --git a/apps/sim/lib/core/config/env.ts b/apps/sim/lib/core/config/env.ts index dd4f1d60980..3affe27ccdd 100644 --- a/apps/sim/lib/core/config/env.ts +++ b/apps/sim/lib/core/config/env.ts @@ -420,6 +420,7 @@ export const env = createEnv({ DATA_DRAINS_ENABLED: z.boolean().optional(), // Enable data drains on self-hosted (bypasses hosted requirements) FORKING_ENABLED: z.boolean().optional(), // Enable workspace forking on self-hosted (bypasses hosted requirements) DEPLOY_AS_BLOCK: z.boolean().optional(), // Enable deploy-as-block (publish a workflow as a reusable org-wide custom block) + TABLES_V2_API: z.boolean().optional(), // Enable the v2 tables HTTP API (public /api/v2/tables + internal /api/table/[tableId]/query predicate-grammar route) // Organizations - for self-hosted deployments ORGANIZATIONS_ENABLED: z.boolean().optional(), // Enable organizations on self-hosted (bypasses plan requirements) diff --git a/apps/sim/lib/core/config/feature-flags.test.ts b/apps/sim/lib/core/config/feature-flags.test.ts index 8c9b95a7556..79128647e3f 100644 --- a/apps/sim/lib/core/config/feature-flags.test.ts +++ b/apps/sim/lib/core/config/feature-flags.test.ts @@ -12,6 +12,7 @@ const { mockFetch, mockIsPlatformAdmin, envRef, flagRef } = vi.hoisted(() => ({ APPCONFIG_ENVIRONMENT: 'staging' as string | undefined, FORKING_ENABLED: undefined as boolean | undefined, DEPLOY_AS_BLOCK: undefined as boolean | undefined, + TABLES_V2_API: undefined as boolean | undefined, }, flagRef: { isAppConfigEnabled: false }, })) @@ -210,3 +211,29 @@ describe('isFeatureEnabled', () => { }) }) }) + +describe('tables-v2-api flag', () => { + beforeEach(() => { + vi.clearAllMocks() + flagRef.isAppConfigEnabled = false + envRef.TABLES_V2_API = undefined + }) + + it('is off by default off-AppConfig, on when the fallback secret is set', async () => { + expect(await isFeatureEnabled('tables-v2-api')).toBe(false) + envRef.TABLES_V2_API = true + expect(await isFeatureEnabled('tables-v2-api')).toBe(true) + }) + + it('gates by org cohort via AppConfig', async () => { + withAppConfig({ 'tables-v2-api': { orgIds: ['org-1'] } }) + expect(await isFeatureEnabled('tables-v2-api', { orgId: 'org-1' })).toBe(true) + expect(await isFeatureEnabled('tables-v2-api', { orgId: 'org-2' })).toBe(false) + expect(await isFeatureEnabled('tables-v2-api', { userId: 'u1' })).toBe(false) + }) + + it('global enabled turns it on for everyone', async () => { + withAppConfig({ 'tables-v2-api': { enabled: true } }) + expect(await isFeatureEnabled('tables-v2-api')).toBe(true) + }) +}) diff --git a/apps/sim/lib/core/config/feature-flags.ts b/apps/sim/lib/core/config/feature-flags.ts index 8a44624ba78..111c8665e42 100644 --- a/apps/sim/lib/core/config/feature-flags.ts +++ b/apps/sim/lib/core/config/feature-flags.ts @@ -118,6 +118,14 @@ const FEATURE_FLAGS = { 'custom-block publish/list routes. Off-AppConfig falls back to DEPLOY_AS_BLOCK.', fallback: 'DEPLOY_AS_BLOCK', }, + 'tables-v2-api': { + description: + 'Gate the v2 tables HTTP API — the public read API (GET /api/v2/tables, POST ' + + '/api/v2/tables/[tableId]/query) and the internal predicate-grammar query route (POST ' + + '/api/table/[tableId]/query). When off, those routes return 404 as if the surface does not ' + + 'exist. Gated by userId/orgId/admins via AppConfig; off-AppConfig falls back to TABLES_V2_API.', + fallback: 'TABLES_V2_API', + }, } satisfies Record /** diff --git a/apps/sim/lib/workspaces/utils.ts b/apps/sim/lib/workspaces/utils.ts index 4c65d5e5f35..a12fe7ac6ec 100644 --- a/apps/sim/lib/workspaces/utils.ts +++ b/apps/sim/lib/workspaces/utils.ts @@ -47,6 +47,21 @@ export async function getWorkspaceBilledAccountUserId(workspaceId: string): Prom return settings?.billedAccountUserId ?? null } +/** + * The organization that owns a workspace (null for personal workspaces). Used to + * gate features by org cohort — the flag model allowlists org ids, and API routes + * only carry a `workspaceId`, so this resolves the one from the other. + */ +export async function getWorkspaceOrganizationId(workspaceId: string): Promise { + if (!workspaceId) return null + const rows = await db + .select({ organizationId: workspaceTable.organizationId }) + .from(workspaceTable) + .where(eq(workspaceTable.id, workspaceId)) + .limit(1) + return rows[0]?.organizationId ?? null +} + /** * Workspaces the user administers purely through organization owner/admin role, * with no explicit permission row required. Empty when the user is not an org From 210633268b8837e6e814ca8664b657394f2c009a Mon Sep 17 00:00:00 2001 From: Theodore Li Date: Fri, 24 Jul 2026 18:00:31 -0700 Subject: [PATCH 06/26] feat(table): ship table_v2 as a preview block, unhide v1 Table MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit table_v2 was registered ungated in the toolbar while v1 Table was hidden, so a deployment with tables-v2-api off left users with only a Table block whose default Query Rows operation 404s behind the flag. Marks table_v2 `preview: true` — hidden from every discovery surface until revealed via the hosted block-visibility AppConfig document or PREVIEW_BLOCKS, fail-closed, with execution of placed instances never gated. Drops the premature `hideFromToolbar` from v1 Table so it stays available during rollout; v1 gets marked superseded at table_v2 GA, alongside its BlockMeta and docs. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_011M563cbFy2S74GvSDf2C3R --- apps/sim/blocks/blocks/table.ts | 3 --- apps/sim/blocks/blocks/table_v2.ts | 5 +++++ 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/apps/sim/blocks/blocks/table.ts b/apps/sim/blocks/blocks/table.ts index 3d8bb5a3ba8..281bcd077e7 100644 --- a/apps/sim/blocks/blocks/table.ts +++ b/apps/sim/blocks/blocks/table.ts @@ -181,9 +181,6 @@ export const TableBlock: BlockConfig = { type: 'table', name: 'Table', description: 'User-defined data tables', - // Superseded by the v2 Table block (`table_v2`). Existing v1 blocks keep - // working; new workflows pick it up from the toolbar instead. - hideFromToolbar: true, longDescription: 'Create and manage custom data tables. Store, query, and manipulate structured data within workflows.', docsLink: 'https://docs.simstudio.ai/tools/table', diff --git a/apps/sim/blocks/blocks/table_v2.ts b/apps/sim/blocks/blocks/table_v2.ts index ce9a92656cc..8b60a942a14 100644 --- a/apps/sim/blocks/blocks/table_v2.ts +++ b/apps/sim/blocks/blocks/table_v2.ts @@ -192,6 +192,11 @@ export const TableV2Block: BlockConfig = { - Columns are scalar (string/number/boolean/date) or opaque json — there are no array columns; for substring use ilike with *x*.`, docsLink: 'https://docs.simstudio.ai/tools/table', category: 'blocks', + // Unreleased: hidden from every discovery surface until revealed via the hosted + // `block-visibility` AppConfig document or the `PREVIEW_BLOCKS` env allowlist. + // Placed instances always execute. At GA: drop this, add the BlockMeta + docs, + // and mark v1 `table` superseded. + preview: true, bgColor: '#10B981', icon: TableIcon, subBlocks: [ From 295d98f73581f928c704f0bce5d221066cdb9dc4 Mon Sep 17 00:00:00 2001 From: Theodore Li Date: Fri, 24 Jul 2026 18:07:50 -0700 Subject: [PATCH 07/26] fix(table): stop keyset paging from silently dropping unkeyed rows MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `order_key` is nullable — rows predating the backfill script-migration, and forked rows that inherit a NULL key. A bare `(order_key, id) > (:k, :i)` row comparison evaluates to NULL for those rows, so WHERE drops them; because NULLs sort LAST, the whole unkeyed tail became unreachable and the drain terminated early reporting `hasMore: false`. Reproduced on a 121-row table: 52 rows returned, `nextCursor: null`, no error. Affected the grid, CSV export, the snapshot cache, the v2 API, and copilot queries. Admits `order_key IS NULL` in both seeks (`fetchRowsBounded`'s drain and `selectExportRowPage`), which restores the tail and makes the compound `{k,i,o}` cursor's offsetFromAnchor accounting resolve — it was unreachable before. `selectExportRowPage` additionally seeks by anchor kind, since its anchor is the previous page's last row and can itself be unkeyed; its return type no longer casts the nullable column to `string`. Also stops workspace forking minting new NULLs: it spread `...row` into a fresh tableId that the one-shot backfill never revisits, so copied rows now get keys appended after the source's max, preserving visual order. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_011M563cbFy2S74GvSDf2C3R --- .../lib/copy/copy-resources.ts | 17 ++++++++++++++ apps/sim/lib/table/jobs/service.ts | 23 ++++++++++++------- apps/sim/lib/table/rows/cursor.ts | 8 ++++--- apps/sim/lib/table/rows/service.ts | 12 ++++++++-- 4 files changed, 47 insertions(+), 13 deletions(-) diff --git a/apps/sim/ee/workspace-forking/lib/copy/copy-resources.ts b/apps/sim/ee/workspace-forking/lib/copy/copy-resources.ts index 63c32d56549..74fee1d31c7 100644 --- a/apps/sim/ee/workspace-forking/lib/copy/copy-resources.ts +++ b/apps/sim/ee/workspace-forking/lib/copy/copy-resources.ts @@ -27,6 +27,7 @@ import { } from '@/lib/billing/storage' import { mapWithConcurrency } from '@/lib/core/utils/concurrency' import type { DbOrTx } from '@/lib/db/types' +import { nKeysBetween } from '@/lib/table/order-key' import type { TableSchema } from '@/lib/table/types' import { deleteFile, @@ -822,6 +823,21 @@ export async function copyForkResourceContent(params: { try { let copied = 0 let afterId: string | null = null + // `order_key` is nullable, and spreading `...row` would inherit NULLs into a + // brand-new tableId that the one-shot backfill script-migration never revisits + // (it snapshots the pending set up front) — leaving rows the keyset pager has to + // special-case forever. Mint keys for the unkeyed ones instead. They sort last in + // the source (NULLS LAST, id tiebreak) and this loop pages by id, so consuming a + // pre-generated run appended after the source's max key preserves visual order. + const [{ maxKey = null, unkeyed = 0 } = {}] = await db + .select({ + maxKey: sql`max(${userTableRows.orderKey})`, + unkeyed: sql`count(*) filter (where ${userTableRows.orderKey} is null)`, + }) + .from(userTableRows) + .where(eq(userTableRows.tableId, table.sourceId)) + const mintedKeys = unkeyed > 0 ? nKeysBetween(maxKey, null, Number(unkeyed)) : [] + let mintedIdx = 0 for (;;) { const where: SQL | undefined = afterId === null @@ -840,6 +856,7 @@ export async function copyForkResourceContent(params: { id: generateId(), tableId: table.childId, workspaceId: childWorkspaceId, + orderKey: row.orderKey ?? mintedKeys[mintedIdx++] ?? null, // Repoint resource-chip URLs in cell data at the child copies (no-op when no maps). data: contentRefMaps ? remapTableRowResourceUrls(row.data, contentRefMaps) : row.data, })) diff --git a/apps/sim/lib/table/jobs/service.ts b/apps/sim/lib/table/jobs/service.ts index 7b1706e32da..dcf09c65507 100644 --- a/apps/sim/lib/table/jobs/service.ts +++ b/apps/sim/lib/table/jobs/service.ts @@ -209,16 +209,17 @@ export async function getJobProgress(tableId: string, jobId: string): Promise> { +): Promise> { const deleteMask = await pendingDeleteMask(table) const rows = await db .select({ id: userTableRows.id, data: userTableRows.data, orderKey: userTableRows.orderKey }) @@ -228,14 +229,20 @@ export async function selectExportRowPage( eq(userTableRows.tableId, table.id), eq(userTableRows.workspaceId, table.workspaceId), deleteMask, + // `order_key` is nullable and sorts LAST, so the page order is "keyed rows by + // key, then the unkeyed tail by id". A bare row-constructor comparison is NULL + // for unkeyed rows (dropping them) and NULL for an unkeyed anchor (dropping + // everything), which silently truncates exports. Seek per anchor kind instead. after - ? sql`(${userTableRows.orderKey}, ${userTableRows.id}) > (${after.orderKey}, ${after.id})` + ? after.orderKey === null + ? sql`${userTableRows.orderKey} IS NULL AND ${userTableRows.id} > ${after.id}` + : sql`(${userTableRows.orderKey} IS NULL OR (${userTableRows.orderKey}, ${userTableRows.id}) > (${after.orderKey}, ${after.id}))` : undefined ) ) .orderBy(asc(userTableRows.orderKey), asc(userTableRows.id)) .limit(limit) - return rows as Array<{ id: string; data: RowData; orderKey: string }> + return rows } /** How long a terminal export stays listable (and re-downloadable from the tray). */ diff --git a/apps/sim/lib/table/rows/cursor.ts b/apps/sim/lib/table/rows/cursor.ts index 05fa15a8ecd..1d51ab2c69d 100644 --- a/apps/sim/lib/table/rows/cursor.ts +++ b/apps/sim/lib/table/rows/cursor.ts @@ -8,9 +8,11 @@ * - Default order → keyset on `(order_key, id)` (`{ k, i }`), an index seek. * - Sorted views → whole-view offset (`{ o }`), because `(order_key, id)` * keyset can't seek a data-column ordering. - * - Keyset page whose last row lacks an `orderKey` (not yet backfilled) → - * compound (`{ k, i, o }`): seek to the last keyed anchor, then OFFSET past - * the unkeyed rows consumed after it. + * - Keyset page whose last row lacks an `orderKey` (rows predating the backfill, + * or forked rows that inherited a NULL key) → compound (`{ k, i, o }`): seek to + * the last keyed anchor, then OFFSET past the unkeyed rows consumed after it. + * This only resolves correctly because the seek admits `order_key IS NULL` + * rows; a bare `(order_key, id) > (…)` excludes them and strands the tail. */ import { TableQueryValidationError } from '@/lib/table/errors' diff --git a/apps/sim/lib/table/rows/service.ts b/apps/sim/lib/table/rows/service.ts index 6dab4b06454..e3424f0e470 100644 --- a/apps/sim/lib/table/rows/service.ts +++ b/apps/sim/lib/table/rows/service.ts @@ -1008,7 +1008,8 @@ export async function queryRows( } // Keyset seeks are only authoritative when the page order IS the - // `(order_key, id)` index order: no custom sort (order keys are always present). + // `(order_key, id)` index order: no custom sort. Order keys are NOT guaranteed + // present — unkeyed rows sort last and the seek admits them explicitly. const keysetValid = !sort // Count and page drain are independent reads — run them concurrently so the @@ -1164,10 +1165,17 @@ async function fetchRowsBounded(params: BoundedFetchParams): Promise { const buildQuery = (executor: DbExecutor) => { + // `order_key` is nullable (rows predating the backfill, and forked rows that + // inherit a NULL key). A bare row-constructor comparison evaluates to NULL for + // those rows, so they are dropped by WHERE — and because NULLs sort LAST under + // `ORDER BY order_key, id`, the entire unkeyed tail becomes unreachable and the + // drain terminates early reporting `hasMore: false`. Admitting NULLs keeps the + // seek set exactly "the tail after the anchor", which is also what the compound + // `{k,i,o}` cursor's `offsetFromAnchor` accounting assumes. const seekWhere = batchSeek ? and( baseWhere, - sql`(${userTableRows.orderKey}, ${userTableRows.id}) > (${batchSeek.orderKey}, ${batchSeek.id})` + sql`(${userTableRows.orderKey} IS NULL OR (${userTableRows.orderKey}, ${userTableRows.id}) > (${batchSeek.orderKey}, ${batchSeek.id}))` ) : baseWhere const query = executor From de74b64d7ebcf912fb480f8e00e40cbd606ddcb2 Mon Sep 17 00:00:00 2001 From: Theodore Li Date: Fri, 24 Jul 2026 18:13:49 -0700 Subject: [PATCH 08/26] fix(table): close the filter paths that widen a bulk delete to the whole table MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `runTableDelete` built its WHERE as `filter ? buildFilterClause(...) : undefined` with no guard, unlike `runTableUpdate` and the inline paths. A filter that was supplied but compiled to nothing therefore deleted every row past the cutoff — `and()` drops an undefined clause silently. Tables at or under the inline bulk cap 400'd, so only larger tables were affected. Three inputs reached that state while passing every check: - `predicateToFilter` claimed to be lossless but emitted leaves that `buildFilterClause` discards: `op:'eq'` with an array value (the realistic trigger — an LLM reaching for `in` and writing `eq`) and a value-taking op with no value. It now throws instead. - `predicateSchema` allowed an empty `all`/`any` group; both now require `.min(1)`. - `validateLeaf` only checked `in`/`nin` emptiness. It now also rejects a missing value and an array on a scalar op, so the copilot path — which has no Zod — fails the same way the HTTP boundary does, and caps `in`/`nin` list length at 1000 (each element becomes its own containment clause). Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_011M563cbFy2S74GvSDf2C3R --- apps/sim/lib/api/contracts/tables.ts | 16 +++++++- apps/sim/lib/table/delete-runner.ts | 5 +++ .../__tests__/converters.test.ts | 15 +++++++ .../query-builder/__tests__/validate.test.ts | 37 +++++++++++++++++ .../sim/lib/table/query-builder/converters.ts | 23 ++++++++++- apps/sim/lib/table/query-builder/validate.ts | 41 ++++++++++++++++--- 6 files changed, 129 insertions(+), 8 deletions(-) diff --git a/apps/sim/lib/api/contracts/tables.ts b/apps/sim/lib/api/contracts/tables.ts index 5b663b5f52d..a0e2e5743dc 100644 --- a/apps/sim/lib/api/contracts/tables.ts +++ b/apps/sim/lib/api/contracts/tables.ts @@ -288,8 +288,20 @@ const predicateNodeSchema: z.ZodType = z.lazy(() => export const predicateSchema: z.ZodType = z.lazy(() => z.union([ - z.object({ all: z.array(predicateNodeSchema).max(MAX_PREDICATE_GROUP_SIZE) }), - z.object({ any: z.array(predicateNodeSchema).max(MAX_PREDICATE_GROUP_SIZE) }), + // `.min(1)`: an empty group compiles to no WHERE clause, which on the bulk + // delete/update paths reads as "match everything" rather than "match nothing". + z.object({ + all: z + .array(predicateNodeSchema) + .min(1, 'A filter group must contain at least one condition') + .max(MAX_PREDICATE_GROUP_SIZE), + }), + z.object({ + any: z + .array(predicateNodeSchema) + .min(1, 'A filter group must contain at least one condition') + .max(MAX_PREDICATE_GROUP_SIZE), + }), ]) ) const predicateGroupSchema = predicateSchema diff --git a/apps/sim/lib/table/delete-runner.ts b/apps/sim/lib/table/delete-runner.ts index 024daab6afc..6e5a5ecaa1a 100644 --- a/apps/sim/lib/table/delete-runner.ts +++ b/apps/sim/lib/table/delete-runner.ts @@ -69,6 +69,11 @@ export async function runTableDelete(payload: TableDeletePayload): Promise const filterClause = filter ? buildFilterClause(filter, USER_TABLE_ROWS_SQL_NAME, table.schema.columns) : undefined + // A filter that was SUPPLIED but compiles to no clause must never widen into + // "delete every row" — `and()` silently drops an undefined clause downstream. + // Mirrors the guard in update-runner and the inline deleteRowsByFilter path; + // an absent filter is still legitimate (delete-all is an explicit caller mode). + if (filter && !filterClause) throw new Error('Filter is required for bulk delete') const excluded = new Set(excludeRowIds ?? []) // Resume the persisted count: a retried attempt's earlier batches are already committed, diff --git a/apps/sim/lib/table/query-builder/__tests__/converters.test.ts b/apps/sim/lib/table/query-builder/__tests__/converters.test.ts index ec3dd911995..d4da9aaadad 100644 --- a/apps/sim/lib/table/query-builder/__tests__/converters.test.ts +++ b/apps/sim/lib/table/query-builder/__tests__/converters.test.ts @@ -219,6 +219,21 @@ describe('predicateToFilter (v2 → legacy)', () => { $or: [{ a: { $empty: true } }, { b: { $empty: false } }], }) }) + + /** + * The conversion is only safe if it is lossless: `buildFilterClause` skips an + * `undefined` condition and any array-valued one, so a leaf that converts to + * either DISAPPEARS from the WHERE — and a predicate whose only leaf disappears + * compiles to no WHERE at all, turning a bulk delete into a table wipe. + */ + it('throws rather than emit a leaf the legacy compiler would discard', () => { + expect(() => + predicateToFilter({ all: [{ field: 'status', op: 'eq', value: ['a', 'b'] }] }) + ).toThrow(/does not accept an array/) + expect(() => predicateToFilter({ all: [{ field: 'status', op: 'eq' }] })).toThrow( + /requires a value/ + ) + }) }) describe('sortRulesToSortSpec (v2)', () => { diff --git a/apps/sim/lib/table/query-builder/__tests__/validate.test.ts b/apps/sim/lib/table/query-builder/__tests__/validate.test.ts index 0b721aa52b0..b729ce59f2a 100644 --- a/apps/sim/lib/table/query-builder/__tests__/validate.test.ts +++ b/apps/sim/lib/table/query-builder/__tests__/validate.test.ts @@ -97,3 +97,40 @@ describe('validateSortSpec', () => { } }) }) + +/** + * These shapes pass structural validation but compile to a clause the legacy + * `$`-grammar discards — which turns a bulk delete/update into "match everything". + */ +describe('validatePredicate — leaves that would silently widen a bulk write', () => { + it('rejects a scalar op given an array (the eq-instead-of-in mistake)', () => { + expect(() => + validatePredicate({ all: [{ field: 'status', op: 'eq', value: ['a', 'b'] }] }, COLS) + ).toThrow(/does not accept an array — use "in"/) + }) + + it('rejects a value-taking op with no value', () => { + expect(() => validatePredicate({ all: [{ field: 'status', op: 'eq' }] }, COLS)).toThrow( + /requires a value/ + ) + expect(() => validatePredicate({ all: [{ field: 'wins', op: 'gte' }] }, COLS)).toThrow( + /requires a value/ + ) + }) + + it('still allows the valueless ops', () => { + for (const op of ['isNull', 'isNotNull', 'isEmpty', 'isNotEmpty'] as const) { + expect(() => validatePredicate({ all: [{ field: 'status', op }] }, COLS)).not.toThrow() + } + }) + + it('caps in/nin list length', () => { + const huge = Array.from({ length: 1001 }, (_, i) => `v${i}`) + expect(() => + validatePredicate({ all: [{ field: 'status', op: 'in', value: huge }] }, COLS) + ).toThrow(/at most 1000 values/) + expect(() => + validatePredicate({ all: [{ field: 'status', op: 'in', value: huge.slice(0, 1000) }] }, COLS) + ).not.toThrow() + }) +}) diff --git a/apps/sim/lib/table/query-builder/converters.ts b/apps/sim/lib/table/query-builder/converters.ts index d95d7dc7035..845a0b720b8 100644 --- a/apps/sim/lib/table/query-builder/converters.ts +++ b/apps/sim/lib/table/query-builder/converters.ts @@ -4,6 +4,7 @@ import { generateShortId } from '@sim/utils/id' import { isRecordLike } from '@sim/utils/object' +import { TableQueryValidationError } from '@/lib/table/errors' import type { Filter, FilterOp, @@ -323,7 +324,27 @@ function predicateLeafToFilterValue(p: Predicate): Filter[string] { // Valueless null checks carry a dummy `true` so the key survives JSON transport. if (p.op === 'isNull') return { $isNull: true } as Filter[string] if (p.op === 'isNotNull') return { $isNotNull: true } as Filter[string] - if (p.op === 'eq') return p.value as Filter[string] + + // Fail loud rather than emit a leaf `buildFilterClause` silently discards. It skips + // an `undefined` condition and any array-valued one, so a dropped leaf WIDENS the + // result — and on the bulk delete/update paths a predicate whose only leaf is + // dropped compiles to no WHERE at all. `op:'eq'` with an array is the realistic + // trigger (an LLM reaching for `in` and writing `eq`). + if (!VALUELESS_OPS.has(p.op) && p.value === undefined) { + throw new TableQueryValidationError( + `Operator "${p.op}" on column "${p.field}" requires a value.`, + 'INVALID_FILTER' + ) + } + if (p.op === 'eq') { + if (Array.isArray(p.value)) { + throw new TableQueryValidationError( + `Operator "eq" on column "${p.field}" does not accept an array — use "in" to match any of several values.`, + 'INVALID_FILTER' + ) + } + return p.value as Filter[string] + } return { [`$${p.op}`]: p.value } as Filter[string] } diff --git a/apps/sim/lib/table/query-builder/validate.ts b/apps/sim/lib/table/query-builder/validate.ts index e0e781c4ea2..19d552d3678 100644 --- a/apps/sim/lib/table/query-builder/validate.ts +++ b/apps/sim/lib/table/query-builder/validate.ts @@ -22,6 +22,15 @@ import type { /** Equality/containment ops that are meaningless on a `json` column (they never match). */ const CONTAINMENT_OPS = new Set(['eq', 'ne', 'in', 'nin']) +/** Ops that legitimately carry no `value`. */ +const VALUELESS_OPS = new Set(['isEmpty', 'isNotEmpty', 'isNull', 'isNotNull']) + +/** + * Cap on `in`/`nin` list length. Each element becomes its own containment clause, + * so an unbounded list is a cheap memory/CPU amplifier from a small request body. + */ +const MAX_IN_LIST_SIZE = 1000 + /** System timestamp columns are filterable/sortable but are not in `schema.columns`. */ const SYSTEM_COLUMN_TYPES: ReadonlyArray<[string, ColumnType]> = [ ['createdAt', 'date'], @@ -57,12 +66,34 @@ function validateLeaf(leaf: Predicate, typeByName: Map): voi 'INVALID_FILTER' ) } - if ( - (leaf.op === 'in' || leaf.op === 'nin') && - (!Array.isArray(leaf.value) || leaf.value.length === 0) - ) { + if (leaf.op === 'in' || leaf.op === 'nin') { + if (!Array.isArray(leaf.value) || leaf.value.length === 0) { + throw new TableQueryValidationError( + `Operator "${leaf.op}" on column "${leaf.field}" requires a non-empty array value.`, + 'INVALID_FILTER' + ) + } + if (leaf.value.length > MAX_IN_LIST_SIZE) { + throw new TableQueryValidationError( + `Operator "${leaf.op}" on column "${leaf.field}" accepts at most ${MAX_IN_LIST_SIZE} values, got ${leaf.value.length}.`, + 'INVALID_FILTER' + ) + } + return + } + // A value-taking op with no value, or a scalar op handed an array, compiles to a + // clause the legacy `$`-grammar silently discards — which WIDENS a bulk delete or + // update. Reject here so the copilot path (no Zod) fails the same way the HTTP + // boundary does. + if (!VALUELESS_OPS.has(leaf.op) && leaf.value === undefined) { + throw new TableQueryValidationError( + `Operator "${leaf.op}" on column "${leaf.field}" requires a value.`, + 'INVALID_FILTER' + ) + } + if (Array.isArray(leaf.value)) { throw new TableQueryValidationError( - `Operator "${leaf.op}" on column "${leaf.field}" requires a non-empty array value.`, + `Operator "${leaf.op}" on column "${leaf.field}" does not accept an array — use "in" to match any of several values.`, 'INVALID_FILTER' ) } From eaf4179962f1fd74750437ab3b27660322160fc8 Mon Sep 17 00:00:00 2001 From: Theodore Li Date: Fri, 24 Jul 2026 18:22:41 -0700 Subject: [PATCH 09/26] fix(table): reject ambiguous and mixed-grammar filter nodes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two shapes let a destructive filter execute differently than it validated. A node carrying both a group key and `field` was read group-first by the engine and validator but leaf-first by predicateToFilter/predicateNamesToIds, so the gate validated one predicate while the bulk-write path executed another — bypassing the unknown-column, json-op, and empty-list checks on the copilot's delete/update path. validateNode now rejects it, and both converters discriminate group-first so unvalidated callers can't disagree either. filterRulesToPredicate skipped any builder rule without `column`, which silently dropped predicate-shaped members when the two grammars were mixed — turning "delete archived rows for tenant acme" into "delete archived rows for every tenant". It now throws and points at the predicate object; a genuinely blank builder row is still ignored. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_011M563cbFy2S74GvSDf2C3R --- apps/sim/lib/table/column-keys.ts | 5 +++-- .../__tests__/converters.test.ts | 20 +++++++++++++++++ .../query-builder/__tests__/validate.test.ts | 15 +++++++++++++ .../sim/lib/table/query-builder/converters.ts | 22 ++++++++++++++++--- apps/sim/lib/table/query-builder/validate.ts | 10 +++++++++ 5 files changed, 67 insertions(+), 5 deletions(-) diff --git a/apps/sim/lib/table/column-keys.ts b/apps/sim/lib/table/column-keys.ts index b89ac3cf352..46aee84f4f8 100644 --- a/apps/sim/lib/table/column-keys.ts +++ b/apps/sim/lib/table/column-keys.ts @@ -184,9 +184,10 @@ export function predicateNamesToIds( idByName: ReadonlyMap ): TablePredicate { const remap = (node: PredicateNode): PredicateNode => { - if ('field' in node) return { ...node, field: idByName.get(node.field) ?? node.field } + // Group-first, matching isPredicateGroup/validateNode/buildPredicateNode. if ('all' in node) return { all: node.all.map(remap) } - return { any: node.any.map(remap) } + if ('any' in node) return { any: node.any.map(remap) } + return { ...node, field: idByName.get(node.field) ?? node.field } } return remap(predicate) as TablePredicate } diff --git a/apps/sim/lib/table/query-builder/__tests__/converters.test.ts b/apps/sim/lib/table/query-builder/__tests__/converters.test.ts index d4da9aaadad..5395afc6a6a 100644 --- a/apps/sim/lib/table/query-builder/__tests__/converters.test.ts +++ b/apps/sim/lib/table/query-builder/__tests__/converters.test.ts @@ -257,3 +257,23 @@ it('does not throw "rules is not iterable" for a non-array builder value', () => expect(filterRulesToPredicate({} as unknown as FilterRule[])).toBeNull() expect(filterRulesToPredicate(undefined as unknown as FilterRule[])).toBeNull() }) + +it('rejects predicate-shaped members rather than dropping them', () => { + // Mixing grammars used to silently discard the predicate-shaped leaf, widening + // the filter — "delete archived rows for tenant acme" became "…for every tenant". + expect(() => + filterRulesToPredicate([ + { field: 'tenant_id', op: 'eq', value: 'acme' }, + rule({ column: 'status', operator: 'eq', value: 'archived' }), + ] as unknown as FilterRule[]) + ).toThrow(/predicate condition/) +}) + +it('still ignores a genuinely blank builder row', () => { + expect( + filterRulesToPredicate([ + rule({ column: '', operator: 'eq', value: '' }), + rule({ column: 'status', operator: 'eq', value: 'archived' }), + ]) + ).toEqual({ all: [{ field: 'status', op: 'eq', value: 'archived' }] }) +}) diff --git a/apps/sim/lib/table/query-builder/__tests__/validate.test.ts b/apps/sim/lib/table/query-builder/__tests__/validate.test.ts index b729ce59f2a..2521d61ad5b 100644 --- a/apps/sim/lib/table/query-builder/__tests__/validate.test.ts +++ b/apps/sim/lib/table/query-builder/__tests__/validate.test.ts @@ -124,6 +124,21 @@ describe('validatePredicate — leaves that would silently widen a bulk write', } }) + it('rejects a hybrid group+leaf node instead of silently picking one', () => { + // Read group-first by the engine, leaf-first by predicateToFilter — so the gate + // would validate one predicate and the bulk-write path execute another. + expect(() => + validatePredicate( + { + all: [{ field: 'status', op: 'eq', value: 'benign' }], + field: 'nope', + op: 'isNotNull', + } as never, + COLS + ) + ).toThrow(/not both/) + }) + it('caps in/nin list length', () => { const huge = Array.from({ length: 1001 }, (_, i) => `v${i}`) expect(() => diff --git a/apps/sim/lib/table/query-builder/converters.ts b/apps/sim/lib/table/query-builder/converters.ts index 845a0b720b8..f90041ee539 100644 --- a/apps/sim/lib/table/query-builder/converters.ts +++ b/apps/sim/lib/table/query-builder/converters.ts @@ -268,7 +268,21 @@ export function filterRulesToPredicate(rules: FilterRule[]): TablePredicate | nu groups.push(current) current = [] } - if (!rule.column) continue + if (!rule.column) { + // A predicate-shaped member ({field, op, value}) means the caller mixed the + // two grammars — most likely an agent writing predicate leaves into the + // visual builder. Silently skipping it would DROP that condition and widen + // the result, which on a bulk delete/update is destructive. + if (isRecordLike(rule) && ('field' in rule || 'op' in rule)) { + throw new TableQueryValidationError( + 'Filter looks like a predicate condition but was supplied as a builder rule. ' + + 'Provide the whole filter as a predicate object ({ all | any: [...] }) instead of mixing shapes.', + 'INVALID_FILTER' + ) + } + // A genuinely blank builder row (no column picked yet) contributes nothing. + continue + } current.push(ruleToPredicate(rule)) } if (current.length > 0) groups.push(current) @@ -356,9 +370,11 @@ function predicateLeafToFilterValue(p: Predicate): Filter[string] { */ export function predicateToFilter(predicate: TablePredicate): Filter { const nodeToFilter = (node: Predicate | TablePredicate): Filter => { - if ('field' in node) return { [node.field]: predicateLeafToFilterValue(node) } + // Group-first, matching isPredicateGroup/validateNode/buildPredicateNode. A + // leaf-first test would execute a different predicate than the one validated. if ('all' in node) return { $and: node.all.map(nodeToFilter) } - return { $or: node.any.map(nodeToFilter) } + if ('any' in node) return { $or: node.any.map(nodeToFilter) } + return { [node.field]: predicateLeafToFilterValue(node) } } return nodeToFilter(predicate) } diff --git a/apps/sim/lib/table/query-builder/validate.ts b/apps/sim/lib/table/query-builder/validate.ts index 19d552d3678..24d913146b3 100644 --- a/apps/sim/lib/table/query-builder/validate.ts +++ b/apps/sim/lib/table/query-builder/validate.ts @@ -109,6 +109,16 @@ function validateNode(node: PredicateNode, typeByName: Map): 'INVALID_FILTER' ) } + // A node carrying BOTH a group key and `field` is ambiguous: the engine and this + // validator read it group-first while `predicateToFilter`/`predicateNamesToIds` + // read it leaf-first, so the gate would validate one predicate and the bulk-write + // path would execute a different one. Reject rather than pick a winner. + if (('all' in node || 'any' in node) && 'field' in node) { + throw new TableQueryValidationError( + 'A filter node must be either a group ({ all | any: [...] }) or a condition ({ field, op, value }), not both.', + 'INVALID_FILTER' + ) + } if ('all' in node || 'any' in node) { const members = 'all' in node ? node.all : node.any if (!Array.isArray(members)) { From 9b5fe212507c1b939929807a5b7e8a5b219b83ed Mon Sep 17 00:00:00 2001 From: Theodore Li Date: Fri, 24 Jul 2026 19:00:38 -0700 Subject: [PATCH 10/26] fix(table): system columns, rollout safety, and DoS bounds for v2 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes #5920 and applies the pre-ship remediation from the branch review. Built-in columns (#5920) - Add `id` as a system column alongside `createdAt`/`updatedAt`. It previously fell through to `data->>'id'` — a JSONB key that never exists — so filtering by id matched nothing and sorting by id ordered by NULL, despite the docs advertising all three as filterable and sortable. - Normalize timestamp bounds with `::timestamptz AT TIME ZONE 'UTC'`. `created_at`/`updated_at` are `timestamp WITHOUT time zone` holding UTC, so a bare `::timestamptz` promoted the column using the session TimeZone GUC and shifted every bound by the server's offset. Verified on a real 121-row table: the reporter's UTC-3 day range returned 114 rows under America/Sao_Paulo vs 112 under UTC; the fix returns 112 in every session timezone. - Range operators on `string` columns now compare as text instead of falling back to a `::numeric` cast that produced the misleading `... (string) requires a number, got string`. `boolean`/`json` ranges are rejected with a message naming the real type. Rollout safety - Move the `tables-v2-api` flag gate below the authz check on all three routes. Ahead of authz it did a primary-DB read on a caller-supplied workspaceId and its 404-vs-403 split leaked which orgs are in the rollout cohort. - Remove `match`/`imatch` entirely. They were newly added to the legacy `$`-allowlist on this branch, making POSIX regex reachable on the *ungated* v1 public API while the gated v2 route rejected it as a pool-pinning risk. Nothing shipped depends on them; the route-local `assertNoRegexOps` goes away with them. - Restore the bounded-page byte cut as opt-in (`TABLE_MAX_PAGE_BYTES`, default off) to match staging. A short page is only safe for clients terminating on `nextCursor === null`; a pre-existing v1 pager stopping at `rows.length < limit` would read the cut as end-of-data. Unbounded queries still fail fast at the 5MB budget rather than return a partial result. DoS bounding - Cap predicate nesting depth (10) and total nodes (500) with an iterative pre-check. The recursive `z.lazy` union overflowed the stack inside `safeParse` on a deeply nested tree — a RangeError escaping a parser is a 500, not a 400. - Cap the row-query body at 1MB (the 50MB platform default let a caller buffer two orders of magnitude more before any schema check ran). - Route the bulk PUT/DELETE bodies through `parseJsonBody` so the destructive surface gets the platform body cap instead of a raw `request.json()`. Hygiene - Rename the query-builder's `SORT_DIRECTIONS` option list to `SORT_DIRECTION_OPTIONS`; it collided with the wire-level tuple and both were star-exported through `lib/table/index.ts`, so the name resolved to nothing. - Drop stale PostgREST references from comments and error messages. Tests - System-column coverage for id (filter, sort, pattern ops), UTC normalization, and the legacy `$`-grammar path. - The NULL-admitting keyset seek — asserted to fail against the pre-fix comparison — plus cursor round-trip continuity across pages. - `nlike`/`nilike` SQL, predicate depth/size rejection, flag-off 404 and gate-ordering on all three v2 routes, and the byte cut being off by default. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_011M563cbFy2S74GvSDf2C3R --- .../api/table/[tableId]/query/route.test.ts | 25 ++- .../app/api/table/[tableId]/query/route.ts | 14 +- .../sim/app/api/table/[tableId]/rows/route.ts | 38 ++--- apps/sim/app/api/table/utils.ts | 5 + .../v2/tables/[tableId]/query/route.test.ts | 14 +- .../api/v2/tables/[tableId]/query/route.ts | 36 ++--- apps/sim/app/api/v2/tables/route.test.ts | 34 +++- apps/sim/app/api/v2/tables/route.ts | 8 +- .../components/sort-builder/sort-builder.tsx | 4 +- .../api/contracts/tables-predicate.test.ts | 47 ++++++ apps/sim/lib/api/contracts/tables.ts | 63 +++++++- .../service-filter-threading.test.ts | 97 ++++++++++- apps/sim/lib/table/__tests__/sql.test.ts | 144 ++++++++++++++++- apps/sim/lib/table/constants.ts | 17 +- apps/sim/lib/table/errors.ts | 2 +- apps/sim/lib/table/query-builder/constants.ts | 7 +- .../sim/lib/table/query-builder/converters.ts | 9 +- .../table/query-builder/use-query-builder.ts | 4 +- apps/sim/lib/table/query-builder/validate.ts | 12 +- apps/sim/lib/table/rows/service.ts | 35 +++- apps/sim/lib/table/sql.ts | 150 +++++++++++------- apps/sim/lib/table/types.ts | 4 +- apps/sim/tools/registry.minimal.ts | 26 +-- 23 files changed, 627 insertions(+), 168 deletions(-) diff --git a/apps/sim/app/api/table/[tableId]/query/route.test.ts b/apps/sim/app/api/table/[tableId]/query/route.test.ts index 99cff93635d..fb727cd53f3 100644 --- a/apps/sim/app/api/table/[tableId]/query/route.test.ts +++ b/apps/sim/app/api/table/[tableId]/query/route.test.ts @@ -1,7 +1,7 @@ /** * @vitest-environment node * - * v2 query route: PostgREST string parsing, unconditional name→id translation + * v2 query route: predicate parsing, unconditional name→id translation * (session auth included — the string grammar is name-keyed for every caller), * cursor validation, and the response envelope. */ @@ -10,9 +10,10 @@ import { NextRequest } from 'next/server' import { beforeEach, describe, expect, it, vi } from 'vitest' import type { TableDefinition } from '@/lib/table/types' -const { mockCheckAccess, mockQueryRows } = vi.hoisted(() => ({ +const { mockCheckAccess, mockQueryRows, mockGate } = vi.hoisted(() => ({ mockCheckAccess: vi.fn(), mockQueryRows: vi.fn(), + mockGate: vi.fn(), })) vi.mock('@/app/api/table/utils', async () => { @@ -21,7 +22,7 @@ vi.mock('@/app/api/table/utils', async () => { checkAccess: mockCheckAccess, accessError: (result: { status: number }) => NextResponse.json({ error: 'Access denied' }, { status: result.status }), - tablesV2GateError: () => null, + tablesV2GateError: mockGate, } }) @@ -91,6 +92,24 @@ describe('POST /api/table/[tableId]/query', () => { vi.clearAllMocks() mockCheckAccess.mockResolvedValue({ ok: true, table: buildTable() }) mockQueryRows.mockResolvedValue(EMPTY_RESULT) + mockGate.mockResolvedValue(null) + }) + + it('returns 404 when the tables-v2-api flag is off', async () => { + const { NextResponse } = await import('next/server') + authAs('session') + mockGate.mockResolvedValue(NextResponse.json({ error: 'Not found' }, { status: 404 })) + const res = await callQuery({ workspaceId: 'workspace-1' }) + expect(res.status).toBe(404) + expect(mockQueryRows).not.toHaveBeenCalled() + }) + + it('runs the flag gate only after the access check, so it cannot leak a cohort oracle', async () => { + authAs('session') + mockCheckAccess.mockResolvedValue({ ok: false, status: 403 }) + const res = await callQuery({ workspaceId: 'workspace-1' }) + expect(res.status).toBe(403) + expect(mockGate).not.toHaveBeenCalled() }) it('translates predicate/sort column names to storage ids for SESSION auth too', async () => { diff --git a/apps/sim/app/api/table/[tableId]/query/route.ts b/apps/sim/app/api/table/[tableId]/query/route.ts index c412f732695..6debbf32e4e 100644 --- a/apps/sim/app/api/table/[tableId]/query/route.ts +++ b/apps/sim/app/api/table/[tableId]/query/route.ts @@ -1,6 +1,6 @@ import { createLogger } from '@sim/logger' import { type NextRequest, NextResponse } from 'next/server' -import { rowQueryContract } from '@/lib/api/contracts/tables' +import { rowQueryContract, TABLE_QUERY_MAX_BODY_BYTES } from '@/lib/api/contracts/tables' import { parseRequest } from '@/lib/api/server' import { isZodError, validationErrorResponse } from '@/lib/api/server/validation' import { checkSessionOrInternalAuth } from '@/lib/auth/hybrid' @@ -35,14 +35,13 @@ export const POST = withRouteHandler(async (request: NextRequest, context: RowQu return NextResponse.json({ error: 'Authentication required' }, { status: 401 }) } - const parsed = await parseRequest(rowQueryContract, request, context) + const parsed = await parseRequest(rowQueryContract, request, context, { + maxBodyBytes: TABLE_QUERY_MAX_BODY_BYTES, + }) if (!parsed.success) return parsed.response const { params, body } = parsed.data const { tableId } = params - const gateError = await tablesV2GateError(authResult.userId, body.workspaceId) - if (gateError) return gateError - const accessResult = await checkAccess(tableId, authResult.userId, 'read') if (!accessResult.ok) return accessError(accessResult, requestId, tableId) const { table } = accessResult @@ -54,6 +53,11 @@ export const POST = withRouteHandler(async (request: NextRequest, context: RowQu return NextResponse.json({ error: 'Invalid workspace ID' }, { status: 400 }) } + // After authz: the gate reads the workspace's org off the primary DB, and its + // 404 would otherwise distinguish "not in the rollout cohort" from "no access". + const gateError = await tablesV2GateError(authResult.userId, body.workspaceId) + if (gateError) return gateError + const schema = table.schema as TableSchema const wire = rowWireTranslators(authResult.authType, schema) const cursor = body.cursor ? decodeCursor(body.cursor) : undefined diff --git a/apps/sim/app/api/table/[tableId]/rows/route.ts b/apps/sim/app/api/table/[tableId]/rows/route.ts index 3c47bf3d03b..af6c558870f 100644 --- a/apps/sim/app/api/table/[tableId]/rows/route.ts +++ b/apps/sim/app/api/table/[tableId]/rows/route.ts @@ -9,7 +9,7 @@ import { updateRowsByFilterBodySchema, } from '@/lib/api/contracts/tables' import { parseRequest } from '@/lib/api/server' -import { isZodError, validationErrorResponse } from '@/lib/api/server/validation' +import { isZodError, parseJsonBody, validationErrorResponse } from '@/lib/api/server/validation' import { type AuthTypeValue, checkSessionOrInternalAuth } from '@/lib/auth/hybrid' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' @@ -344,12 +344,12 @@ export const PUT = withRouteHandler( return NextResponse.json({ error: 'Authentication required' }, { status: 401 }) } - let body: unknown - try { - body = await request.json() - } catch { - return NextResponse.json({ error: 'Request body must be valid JSON' }, { status: 400 }) - } + // Bulk write bodies carry caller-supplied row data, so they can legitimately + // be large — but not unbounded. `parseJsonBody` applies the platform cap + // (413 past it) that a raw `request.json()` on this destructive surface skips. + const parsedBody = await parseJsonBody(request) + if (!parsedBody.success) return parsedBody.response + const body = parsedBody.data const validated = updateRowsByFilterBodySchema.parse(body) @@ -442,12 +442,12 @@ export const DELETE = withRouteHandler( return NextResponse.json({ error: 'Authentication required' }, { status: 401 }) } - let body: unknown - try { - body = await request.json() - } catch { - return NextResponse.json({ error: 'Request body must be valid JSON' }, { status: 400 }) - } + // Bulk write bodies carry caller-supplied row data, so they can legitimately + // be large — but not unbounded. `parseJsonBody` applies the platform cap + // (413 past it) that a raw `request.json()` on this destructive surface skips. + const parsedBody = await parseJsonBody(request) + if (!parsedBody.success) return parsedBody.response + const body = parsedBody.data const validated = deleteTableRowsBodySchema.parse(body) @@ -539,12 +539,12 @@ export const PATCH = withRouteHandler( return NextResponse.json({ error: 'Authentication required' }, { status: 401 }) } - let body: unknown - try { - body = await request.json() - } catch { - return NextResponse.json({ error: 'Request body must be valid JSON' }, { status: 400 }) - } + // Bulk write bodies carry caller-supplied row data, so they can legitimately + // be large — but not unbounded. `parseJsonBody` applies the platform cap + // (413 past it) that a raw `request.json()` on this destructive surface skips. + const parsedBody = await parseJsonBody(request) + if (!parsedBody.success) return parsedBody.response + const body = parsedBody.data const validated = batchUpdateTableRowsBodySchema.parse(body) diff --git a/apps/sim/app/api/table/utils.ts b/apps/sim/app/api/table/utils.ts index 8a4260ac10b..02328b13163 100644 --- a/apps/sim/app/api/table/utils.ts +++ b/apps/sim/app/api/table/utils.ts @@ -19,6 +19,11 @@ import { getWorkspaceOrganizationId } from '@/lib/workspaces/utils' * Gate for the v2 tables HTTP API (`tables-v2-api` flag). Returns a 404 response * when the flag is off for the caller — the surface behaves as if it doesn't * exist — or `null` to proceed. Gated by userId + the workspace's org cohort. + * + * **Call this AFTER the authz check, never before.** Ahead of authz it does a + * primary-DB read keyed on a caller-supplied `workspaceId`, and the 404-vs-403 + * split tells an unauthorized caller whether that workspace's org is in the + * rollout cohort. */ export async function tablesV2GateError( userId: string, diff --git a/apps/sim/app/api/v2/tables/[tableId]/query/route.test.ts b/apps/sim/app/api/v2/tables/[tableId]/query/route.test.ts index d03b295b192..37d8e774d8b 100644 --- a/apps/sim/app/api/v2/tables/[tableId]/query/route.test.ts +++ b/apps/sim/app/api/v2/tables/[tableId]/query/route.test.ts @@ -2,7 +2,7 @@ * @vitest-environment node * * Public v2 query POST: typed predicate name→id translation, bounded-default vs - * explicit-unbounded limit, regex gating, cursor validation, workspace scoping, + * explicit-unbounded limit, cursor validation, workspace scoping, * and name-keyed row output. */ import { NextRequest } from 'next/server' @@ -115,6 +115,13 @@ describe('POST /api/v2/tables/[tableId]/query', () => { expect(mockQueryRows).not.toHaveBeenCalled() }) + it('runs the flag gate only after the access check, so it cannot leak a cohort oracle', async () => { + mockCheckAccess.mockResolvedValue({ ok: false, status: 403 }) + const res = await callQuery({ workspaceId: 'workspace-1' }) + expect(res.status).toBe(403) + expect(mockGate).not.toHaveBeenCalled() + }) + it('translates a name-keyed predicate to storage ids', async () => { const res = await callQuery({ workspaceId: 'workspace-1', @@ -150,13 +157,14 @@ describe('POST /api/v2/tables/[tableId]/query', () => { expect(mockQueryRows).not.toHaveBeenCalled() }) - it('gates regex ops (match/imatch) off the public surface', async () => { + it('rejects the removed regex ops at the contract boundary', async () => { + // `match`/`imatch` are no longer in FILTER_OPS — a catastrophic-backtracking + // pattern can pin a shared-pool connection, and nothing shipped depends on them. const res = await callQuery({ workspaceId: 'workspace-1', predicate: { all: [{ field: 'name', op: 'match', value: '^jo' }] }, }) expect(res.status).toBe(400) - expect((await res.json()).error).toMatch(/Regex filters.*not supported/) expect(mockQueryRows).not.toHaveBeenCalled() }) diff --git a/apps/sim/app/api/v2/tables/[tableId]/query/route.ts b/apps/sim/app/api/v2/tables/[tableId]/query/route.ts index 706443d6fef..c1da014a442 100644 --- a/apps/sim/app/api/v2/tables/[tableId]/query/route.ts +++ b/apps/sim/app/api/v2/tables/[tableId]/query/route.ts @@ -1,10 +1,11 @@ import { createLogger } from '@sim/logger' import { type NextRequest, NextResponse } from 'next/server' +import { TABLE_QUERY_MAX_BODY_BYTES } from '@/lib/api/contracts/tables' import { V2_DEFAULT_ROW_LIMIT, v2QueryRowsContract } from '@/lib/api/contracts/v2/tables' import { parseRequest, validationErrorResponseFromError } from '@/lib/api/server' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import type { PredicateNode, Sort, TablePredicate, TableSchema } from '@/lib/table' +import type { Sort, TablePredicate, TableSchema } from '@/lib/table' import { buildIdByName, buildNameById, @@ -35,28 +36,10 @@ interface QueryRouteParams { params: Promise<{ tableId: string }> } -/** - * Regex ops (`match`/`imatch`) are gated off the public surface — a - * catastrophic-backtracking pattern can pin a shared-pool connection. Walks the - * predicate tree and throws on any regex leaf. - */ -function assertNoRegexOps(node: PredicateNode): void { - if ('all' in node || 'any' in node) { - for (const child of 'all' in node ? node.all : node.any) assertNoRegexOps(child) - return - } - if (node.op === 'match' || node.op === 'imatch') { - throw new TableQueryValidationError( - 'Regex filters (match/imatch) are not supported on the public API — use like/ilike for pattern match.', - 'INVALID_FILTER' - ) - } -} - /** * POST /api/v2/tables/[tableId]/query — public row query. Typed `predicate`/`sort` * objects + opaque cursor pagination. Default page {@link V2_DEFAULT_ROW_LIMIT}; - * `limit=0` = unbounded (whole result or 400). Regex ops are gated off. + * `limit=0` = unbounded (whole result or 400). */ export const POST = withRouteHandler(async (request: NextRequest, context: QueryRouteParams) => { const requestId = generateRequestId() @@ -66,15 +49,14 @@ export const POST = withRouteHandler(async (request: NextRequest, context: Query if (!rateLimit.allowed) return createRateLimitResponse(rateLimit) const userId = rateLimit.userId! - const parsed = await parseRequest(v2QueryRowsContract, request, context) + const parsed = await parseRequest(v2QueryRowsContract, request, context, { + maxBodyBytes: TABLE_QUERY_MAX_BODY_BYTES, + }) if (!parsed.success) return parsed.response const { tableId } = parsed.data.params const { workspaceId, sort, cursor: cursorToken, limit } = parsed.data.body - const gateError = await tablesV2GateError(userId, workspaceId) - if (gateError) return gateError - const scopeError = await checkWorkspaceScope(rateLimit, workspaceId) if (scopeError) return scopeError @@ -86,6 +68,11 @@ export const POST = withRouteHandler(async (request: NextRequest, context: Query return NextResponse.json({ error: 'Invalid workspace ID' }, { status: 400 }) } + // After authz: the gate reads the workspace's org off the primary DB, and its + // 404 would otherwise distinguish "not in the rollout cohort" from "no access". + const gateError = await tablesV2GateError(userId, workspaceId) + if (gateError) return gateError + const schema = table.schema as TableSchema const cursor = cursorToken ? decodeCursor(cursorToken) : undefined @@ -105,7 +92,6 @@ export const POST = withRouteHandler(async (request: NextRequest, context: Query const nameById = buildNameById(schema) let predicate: TablePredicate | undefined = parsed.data.body.predicate if (predicate) { - assertNoRegexOps(predicate) validatePredicate(predicate, schema.columns) predicate = predicateNamesToIds(predicate, idByName) } diff --git a/apps/sim/app/api/v2/tables/route.test.ts b/apps/sim/app/api/v2/tables/route.test.ts index f4356b16f5b..2880355c8df 100644 --- a/apps/sim/app/api/v2/tables/route.test.ts +++ b/apps/sim/app/api/v2/tables/route.test.ts @@ -7,11 +7,14 @@ import { NextRequest } from 'next/server' import { beforeEach, describe, expect, it, vi } from 'vitest' import type { TableDefinition } from '@/lib/table/types' -const { mockListTables, mockCheckRateLimit, mockValidateWorkspaceAccess } = vi.hoisted(() => ({ - mockListTables: vi.fn(), - mockCheckRateLimit: vi.fn(), - mockValidateWorkspaceAccess: vi.fn(), -})) +const { mockListTables, mockCheckRateLimit, mockValidateWorkspaceAccess, mockGate } = vi.hoisted( + () => ({ + mockListTables: vi.fn(), + mockCheckRateLimit: vi.fn(), + mockValidateWorkspaceAccess: vi.fn(), + mockGate: vi.fn(), + }) +) vi.mock('@/app/api/v1/middleware', async () => { const { NextResponse } = await import('next/server') @@ -33,7 +36,7 @@ vi.mock('@/lib/table', async () => { vi.mock('@/app/api/table/utils', () => ({ normalizeColumn: (col: Record) => col, - tablesV2GateError: () => null, + tablesV2GateError: mockGate, })) import { GET } from '@/app/api/v2/tables/route' @@ -66,6 +69,25 @@ describe('GET /api/v2/tables', () => { mockCheckRateLimit.mockResolvedValue({ allowed: true, userId: 'user-1', keyType: 'workspace' }) mockValidateWorkspaceAccess.mockResolvedValue(null) mockListTables.mockResolvedValue([buildTable()]) + mockGate.mockResolvedValue(null) + }) + + it('returns 404 when the tables-v2-api flag is off', async () => { + const { NextResponse } = await import('next/server') + mockGate.mockResolvedValue(NextResponse.json({ error: 'Not found' }, { status: 404 })) + const res = await callList('workspaceId=workspace-1') + expect(res.status).toBe(404) + expect(mockListTables).not.toHaveBeenCalled() + }) + + it('runs the flag gate only after the access check, so it cannot leak a cohort oracle', async () => { + const { NextResponse } = await import('next/server') + mockValidateWorkspaceAccess.mockResolvedValue( + NextResponse.json({ error: 'Access denied' }, { status: 403 }) + ) + const res = await callList('workspaceId=workspace-1') + expect(res.status).toBe(403) + expect(mockGate).not.toHaveBeenCalled() }) it('returns a typed table summary with a private cache header', async () => { diff --git a/apps/sim/app/api/v2/tables/route.ts b/apps/sim/app/api/v2/tables/route.ts index 07e79730b76..be0c59c32b6 100644 --- a/apps/sim/app/api/v2/tables/route.ts +++ b/apps/sim/app/api/v2/tables/route.ts @@ -34,12 +34,14 @@ export const GET = withRouteHandler(async (request: NextRequest) => { const { workspaceId } = parsed.data.query - const gateError = await tablesV2GateError(userId, workspaceId) - if (gateError) return gateError - const accessError = await validateWorkspaceAccess(rateLimit, userId, workspaceId) if (accessError) return accessError + // After authz: the gate reads the workspace's org off the primary DB, and its + // 404 would otherwise distinguish "not in the rollout cohort" from "no access". + const gateError = await tablesV2GateError(userId, workspaceId) + if (gateError) return gateError + const tables = await listTables(workspaceId) return NextResponse.json( diff --git a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/sort-builder/sort-builder.tsx b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/sort-builder/sort-builder.tsx index d11ec546900..9427c7a8ca5 100644 --- a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/sort-builder/sort-builder.tsx +++ b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/sort-builder/sort-builder.tsx @@ -5,7 +5,7 @@ import { Button, type ComboboxOption } from '@sim/emcn' import { generateId } from '@sim/utils/id' import { Plus } from 'lucide-react' import { useTableColumns } from '@/lib/table/hooks' -import { SORT_DIRECTIONS, type SortRule } from '@/lib/table/query-builder/constants' +import { SORT_DIRECTION_OPTIONS, type SortRule } from '@/lib/table/query-builder/constants' import { useCanonicalSubBlockValue } from '@/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/hooks/use-canonical-sub-block-value' import { useSubBlockValue } from '@/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/hooks/use-sub-block-value' import { SortRuleRow } from './components/sort-rule-row' @@ -47,7 +47,7 @@ export function SortBuilder({ }, [propColumns, dynamicColumns]) const directionOptions = useMemo( - () => SORT_DIRECTIONS.map((dir) => ({ value: dir.value, label: dir.label })), + () => SORT_DIRECTION_OPTIONS.map((dir) => ({ value: dir.value, label: dir.label })), [] ) diff --git a/apps/sim/lib/api/contracts/tables-predicate.test.ts b/apps/sim/lib/api/contracts/tables-predicate.test.ts index f576af54cf9..1ce365ea669 100644 --- a/apps/sim/lib/api/contracts/tables-predicate.test.ts +++ b/apps/sim/lib/api/contracts/tables-predicate.test.ts @@ -8,6 +8,7 @@ import { describe, expect, it } from 'vitest' import { deleteTableRowsBodySchema, + predicateSchema, rowQueryBodySchema, updateRowsByFilterBodySchema, } from '@/lib/api/contracts/tables' @@ -102,3 +103,49 @@ describe('bulk schemas accept either a predicate tree or the legacy filter objec ).toBe(true) }) }) + +/** + * The predicate tree is parsed by a recursive `z.lazy` union. A few thousand + * nested groups overflow the stack inside `safeParse`, and a `RangeError` + * escaping a parser is a 500 on a public endpoint, not a 400. + */ +describe('predicate depth / size guard', () => { + it('rejects a deeply nested tree with a validation issue, not a RangeError', () => { + let node: unknown = { all: [{ field: 'a', op: 'eq', value: 1 }] } + for (let i = 0; i < 5000; i++) node = { all: [node] } + + const result = predicateSchema.safeParse(node) + + expect(result.success).toBe(false) + expect(JSON.stringify(result.error?.issues)).toMatch(/nesting is too deep/) + }) + + it('rejects a wide-but-shallow tree past the node cap', () => { + const node = { + all: Array.from({ length: 60 }, () => ({ + all: Array.from({ length: 60 }, () => ({ field: 'a', op: 'eq', value: 1 })), + })), + } + + const result = predicateSchema.safeParse(node) + + expect(result.success).toBe(false) + expect(JSON.stringify(result.error?.issues)).toMatch(/too many conditions/) + }) + + it('still accepts a realistic nested predicate', () => { + expect( + predicateSchema.safeParse({ + all: [ + { field: 'status', op: 'eq', value: 'active' }, + { + any: [ + { field: 'wins', op: 'gte', value: 10 }, + { field: 'name', op: 'contains', value: 'jo' }, + ], + }, + ], + }).success + ).toBe(true) + }) +}) diff --git a/apps/sim/lib/api/contracts/tables.ts b/apps/sim/lib/api/contracts/tables.ts index a0e2e5743dc..142ec914ffe 100644 --- a/apps/sim/lib/api/contracts/tables.ts +++ b/apps/sim/lib/api/contracts/tables.ts @@ -265,10 +265,53 @@ const filterSchema = domainObjectSchema() /* --------------------------- v2 predicate grammar --------------------------- */ +/** + * Body cap for the row-query routes. A query body is a predicate tree plus a + * cursor; the largest legitimate one — a 100-member predicate with 1000-element + * `in` lists — is comfortably under 1 MB. The 50 MB platform default would let a + * caller buffer two orders of magnitude more before any schema check runs. + */ +export const TABLE_QUERY_MAX_BODY_BYTES = 1024 * 1024 + /** Max members in one `all`/`any` group — a generous bound against pathological trees. */ const MAX_PREDICATE_GROUP_SIZE = 100 /** Max sort keys — more than a few is already a smell. */ const MAX_SORT_KEYS = 16 +/** Max nesting levels of `all`/`any` groups. Ten is already unreadable. */ +const MAX_PREDICATE_DEPTH = 10 +/** Max nodes in the whole tree, so a wide-but-shallow tree can't amplify either. */ +const MAX_PREDICATE_NODES = 500 + +/** + * Iterative depth/size walk over an unvalidated predicate tree. Runs BEFORE the + * recursive Zod schema: a few thousand nested `{all:[...]}` levels overflow the + * stack inside `safeParse`, and a `RangeError` from a parser is a 500, not a 400. + * The walk itself must stay iterative for the same reason. + */ +function predicateTreeTooLarge(root: unknown): string | null { + const stack: Array<{ node: unknown; depth: number }> = [{ node: root, depth: 1 }] + let nodes = 0 + + while (stack.length > 0) { + const { node, depth } = stack.pop()! + if (++nodes > MAX_PREDICATE_NODES) { + return `Filter has too many conditions (max ${MAX_PREDICATE_NODES})` + } + if (depth > MAX_PREDICATE_DEPTH) { + return `Filter nesting is too deep (max ${MAX_PREDICATE_DEPTH} levels)` + } + if (typeof node !== 'object' || node === null) continue + const group = node as { all?: unknown; any?: unknown } + const members = Array.isArray(group.all) + ? group.all + : Array.isArray(group.any) + ? group.any + : null + if (!members) continue + for (const member of members) stack.push({ node: member, depth: depth + 1 }) + } + return null +} /** * v2 filter wire format: the typed `{ all | any: [...] }` predicate tree (same @@ -286,7 +329,7 @@ const predicateNodeSchema: z.ZodType = z.lazy(() => z.union([predicateGroupSchema, predicateLeafSchema]) ) -export const predicateSchema: z.ZodType = z.lazy(() => +const predicateTreeSchema: z.ZodType = z.lazy(() => z.union([ // `.min(1)`: an empty group compiles to no WHERE clause, which on the bulk // delete/update paths reads as "match everything" rather than "match nothing". @@ -304,7 +347,23 @@ export const predicateSchema: z.ZodType = z.lazy(() => }), ]) ) -const predicateGroupSchema = predicateSchema +const predicateGroupSchema = predicateTreeSchema + +/** + * The boundary predicate schema: depth/size guard first, then the recursive + * structural parse. The guard is only applied at the top level — every nested + * group is strictly shallower, so re-checking inside the recursion would be + * redundant work on the hot path. + */ +export const predicateSchema = z + .unknown() + .superRefine((value, ctx) => { + const problem = predicateTreeTooLarge(value) + if (problem) ctx.addIssue({ code: 'custom', message: problem }) + }) + // double-cast-allowed: the pipe's inferred input is `unknown`, and letting TS + // widen the recursive lazy union through it makes typecheck OOM + .pipe(predicateTreeSchema) as unknown as z.ZodType /** v2 sort wire format: an ordered list of `{ field, direction }`. */ export const sortSpecSchema: z.ZodType = z diff --git a/apps/sim/lib/table/__tests__/service-filter-threading.test.ts b/apps/sim/lib/table/__tests__/service-filter-threading.test.ts index ca6ff7b428c..e794c5a8780 100644 --- a/apps/sim/lib/table/__tests__/service-filter-threading.test.ts +++ b/apps/sim/lib/table/__tests__/service-filter-threading.test.ts @@ -7,7 +7,7 @@ * timestamp for dates) are always available at the SQL builder layer — the * latent bug that PR #4657 was originally fixing. */ -import { dbChainMockFns, resetDbChainMock } from '@sim/testing' +import { dbChainMockFns, resetDbChainMock, setEnv } from '@sim/testing' import { sql } from 'drizzle-orm' import { beforeEach, describe, expect, it, vi } from 'vitest' import { TABLE_LIMITS } from '@/lib/table/constants' @@ -159,6 +159,9 @@ describe('queryRows byte budget', () => { beforeEach(() => { vi.clearAllMocks() resetDbChainMock() + // The bounded-page byte cut is opt-in; pin it on rather than inheriting + // whatever the developer's local `.env` happens to set. + setEnv({ TABLE_MAX_PAGE_BYTES: TABLE_LIMITS.MAX_QUERY_RESULT_BYTES }) }) const row = (i: number, blobBytes: number) => ({ @@ -217,6 +220,36 @@ describe('queryRows byte budget', () => { }) }) + it('does NOT byte-cut a bounded page when TABLE_MAX_PAGE_BYTES is unset', async () => { + // Default-off: a short page is only safe for a client that terminates on + // `nextCursor === null`. A pre-existing v1 pager stopping at + // `rows.length < limit` would read the cut as end-of-data and truncate. + setEnv({ TABLE_MAX_PAGE_BYTES: undefined }) + const perRow = Math.floor(TABLE_LIMITS.MAX_QUERY_RESULT_BYTES * 0.6) + dbChainMockFns.limit.mockResolvedValueOnce([]) + dbChainMockFns.limit.mockResolvedValueOnce([row(1, perRow), row(2, perRow)]) + + const result = await queryRows( + TABLE, + { limit: 5, includeTotal: false, withExecutions: false }, + 'req-1' + ) + + expect(result.rows).toHaveLength(2) + expect(result.nextCursor).toBeNull() + }) + + it('still fails fast on an UNBOUNDED query with TABLE_MAX_PAGE_BYTES unset', async () => { + setEnv({ TABLE_MAX_PAGE_BYTES: undefined }) + const perRow = Math.floor(TABLE_LIMITS.MAX_QUERY_RESULT_BYTES * 0.6) + dbChainMockFns.limit.mockResolvedValueOnce([]) + dbChainMockFns.limit.mockResolvedValueOnce([row(1, perRow), row(2, perRow)]) + + await expect( + queryRows(TABLE, { includeTotal: false, withExecutions: false }, 'req-1') + ).rejects.toThrow(/exceeds the 5MB limit/) + }) + it('returns a single over-budget row alone on a bounded page', async () => { dbChainMockFns.limit.mockResolvedValueOnce([]) dbChainMockFns.limit.mockResolvedValueOnce([ @@ -241,6 +274,68 @@ describe('queryRows byte budget', () => { Math.floor((4 * TABLE_LIMITS.MAX_QUERY_RESULT_BYTES) / TABLE_LIMITS.MAX_ROW_SIZE_BYTES) ) + /** + * The seek predicate that pages the drain. `order_key` is nullable (rows + * predating the backfill, forked rows), and a bare + * `(order_key, id) > (:k, :i)` evaluates to NULL for those rows, so WHERE drops + * them. Because NULLs sort last, the whole unkeyed tail then becomes + * unreachable and the drain reports `hasMore: false` — 52 of 121 rows returned + * with no error, reproduced against a real table. + */ + it('seeks with a NULL-admitting comparison so unkeyed rows stay reachable', async () => { + dbChainMockFns.limit.mockResolvedValueOnce([]) + dbChainMockFns.limit.mockResolvedValueOnce([]) + + await queryRows( + TABLE, + { + after: { orderKey: 'a5', id: 'row_5' }, + limit: 10, + includeTotal: false, + withExecutions: false, + }, + 'req-1' + ) + + const seekWhere = JSON.stringify(dbChainMockFns.where.mock.calls.at(-1)) + expect(seekWhere).toMatch(/is null/i) + expect(seekWhere).toContain('a5') + expect(seekWhere).toContain('row_5') + }) + + it('resumes exactly at the last returned row when the cursor is fed back', async () => { + dbChainMockFns.limit.mockResolvedValueOnce([]) + // limit 2 + witness row_3 → page ends at row_2 with a resume cursor. + dbChainMockFns.limit.mockResolvedValueOnce([row(1, 8), row(2, 8), row(3, 8)]) + + const page1 = await queryRows( + TABLE, + { limit: 2, includeTotal: false, withExecutions: false }, + 'req-1' + ) + expect(page1.rows.map((r) => r.id)).toEqual(['row_1', 'row_2']) + + const cursor = decodeCursor(page1.nextCursor as string) + expect(cursor).toEqual({ after: { orderKey: 'a2', id: 'row_2' } }) + + vi.clearAllMocks() + resetDbChainMock() + dbChainMockFns.limit.mockResolvedValueOnce([]) + dbChainMockFns.limit.mockResolvedValueOnce([row(3, 8), row(4, 8)]) + + const page2 = await queryRows( + TABLE, + { ...cursor, limit: 2, includeTotal: false, withExecutions: false }, + 'req-2' + ) + + // No gap (row_3 is the first row past the anchor) and no duplicate of row_2. + expect(page2.rows.map((r) => r.id)).toEqual(['row_3', 'row_4']) + const seekWhere = JSON.stringify(dbChainMockFns.where.mock.calls.at(-1)) + expect(seekWhere).toContain('a2') + expect(seekWhere).toContain('row_2') + }) + it('caps the first unbounded batch and asks limit+1 when a limit is set', async () => { await queryRows(TABLE, { includeTotal: false, withExecutions: false }, 'req-1') // Call 1 = pendingDeleteMask probe (limit 1); call 2 = first drain batch. diff --git a/apps/sim/lib/table/__tests__/sql.test.ts b/apps/sim/lib/table/__tests__/sql.test.ts index 0841c788b97..129170084e7 100644 --- a/apps/sim/lib/table/__tests__/sql.test.ts +++ b/apps/sim/lib/table/__tests__/sql.test.ts @@ -546,13 +546,38 @@ describe('fieldPredicate (shared leaf)', () => { expect(render(fieldPredicate(TABLE, 'name', 'like', '50%*', undefined))).toContain('50\\%%') }) - it('match / imatch emit POSIX regex (~ / ~*)', () => { - const m = render(fieldPredicate(TABLE, 'name', 'match', '^jo.*n$', undefined)) - expect(m).toContain("data->>'name'") - expect(m).toContain('~') - expect(m).not.toContain('~*') - expect(m).toContain('^jo.*n$') - expect(render(fieldPredicate(TABLE, 'name', 'imatch', 'foo', undefined))).toContain('~*') + it('nlike / nilike negate the match and keep null cells', () => { + // "does not match X" must retain rows where the cell is absent — otherwise a + // negated filter silently drops every row with an empty value for that column. + const nlike = render(fieldPredicate(TABLE, 'name', 'nlike', 'jo*n', undefined)) + expect(nlike).toContain('NOT LIKE') + expect(nlike).not.toContain('NOT ILIKE') + expect(nlike).toContain('jo%n') + expect(nlike).toContain('IS NULL') + + const nilike = render(fieldPredicate(TABLE, 'name', 'nilike', '*foo*', undefined)) + expect(nilike).toContain('NOT ILIKE') + expect(nilike).toContain('IS NULL') + }) + + it('reaches nlike / nilike through the legacy $-grammar too', () => { + expect(render(buildFilterClause({ name: { $nlike: 'jo*' } }, TABLE, NO_COLUMNS))).toContain( + 'NOT LIKE' + ) + expect(render(buildFilterClause({ name: { $nilike: 'jo*' } }, TABLE, NO_COLUMNS))).toContain( + 'NOT ILIKE' + ) + }) + + it('no longer accepts the regex ops (removed from FILTER_OPS)', () => { + for (const op of ['match', 'imatch'] as const) { + expect(() => fieldPredicate(TABLE, 'name', op as never, '^jo', undefined)).toThrow( + 'Invalid operator' + ) + } + expect(() => buildFilterClause({ name: { $match: '^jo' } }, TABLE, NO_COLUMNS)).toThrow( + 'Invalid operator' + ) }) it('validates the field name', () => { @@ -590,6 +615,111 @@ describe('fieldPredicate (shared leaf)', () => { }) }) +/** + * Regression coverage for #5920 — "filtering on built-in columns silently + * returns zero rows". Three distinct defects fed that report: the missing + * system-column dispatch (fixed above), `id` never being a system column at + * all, and the session-dependent `timestamptz` promotion that shifts every + * bound by the server's `TimeZone` GUC. + */ +describe('system columns (#5920)', () => { + it('normalizes timestamp bounds to UTC wall clock, not the session TimeZone', () => { + // `created_at`/`updated_at` are `timestamp WITHOUT time zone` holding UTC. + // Without `AT TIME ZONE 'UTC'` the comparison result depends on the server's + // TimeZone setting, so a UTC-3 day range (the reporter's exact query) lands + // off by the offset at both boundaries. + for (const field of ['createdAt', 'updatedAt'] as const) { + const out = render(fieldPredicate(TABLE, field, 'gte', '2026-07-24T03:00:00.000Z', 'date')) + expect(out).toContain("::timestamptz AT TIME ZONE 'UTC'") + } + }) + + it('filters id as the real text column, with no timestamptz cast', () => { + const out = render(fieldPredicate(TABLE, 'id', 'eq', 'row-123', 'string')) + expect(out).toContain(`${TABLE}.id =`) + expect(out).not.toContain("data->>'id'") + expect(out).not.toContain('timestamptz') + }) + + it('supports the pattern ops on id (text), which are meaningless on timestamps', () => { + expect(render(fieldPredicate(TABLE, 'id', 'contains', 'abc', 'string'))).toContain( + `${TABLE}.id ILIKE` + ) + expect(render(fieldPredicate(TABLE, 'id', 'startsWith', 'abc', 'string'))).toContain( + `${TABLE}.id ILIKE` + ) + expect(render(fieldPredicate(TABLE, 'id', 'like', 'ab*', 'string'))).toContain( + `${TABLE}.id LIKE` + ) + expect(render(fieldPredicate(TABLE, 'id', 'ncontains', 'abc', 'string'))).toContain('NOT (') + + expect(() => fieldPredicate(TABLE, 'createdAt', 'contains', 'abc', 'date')).toThrow( + /not supported on the built-in column "createdAt"/ + ) + }) + + it('rejects an empty pattern on id rather than matching every row', () => { + expect(() => fieldPredicate(TABLE, 'id', 'contains', '', 'string')).toThrow( + /requires a non-empty value/ + ) + }) + + it('supports id in ranges and membership', () => { + expect(render(fieldPredicate(TABLE, 'id', 'gt', 'row-100', 'string'))).toContain( + `${TABLE}.id >` + ) + const inClause = render(fieldPredicate(TABLE, 'id', 'in', ['a', 'b'], 'string')) + expect(inClause).toContain(`${TABLE}.id IN (`) + }) + + it('sorts id as a direct column ref', () => { + expect(render(buildSortClause({ id: 'asc' }, TABLE, NO_COLUMNS))).toBe(`${TABLE}.id ASC`) + }) + + it('maps isEmpty/isNotEmpty to IS NULL / IS NOT NULL (not inverted)', () => { + expect(render(fieldPredicate(TABLE, 'createdAt', 'isEmpty', undefined, 'date'))).toContain( + 'IS NULL' + ) + expect(render(fieldPredicate(TABLE, 'createdAt', 'isNotEmpty', undefined, 'date'))).toContain( + 'IS NOT NULL' + ) + }) + + it('reaches the same clause through the legacy $-grammar', () => { + // The v1 API path in the bug report goes through buildFilterClause, so the + // shared `fieldPredicate` leaf must cover it identically. + const out = render( + buildFilterClause( + { createdAt: { $gte: '2026-07-24T03:00:00.000Z', $lte: '2026-07-25T02:59:59.999Z' } }, + TABLE, + NO_COLUMNS + ) + ) + expect(out).toContain(`${TABLE}.created_at >=`) + expect(out).toContain(`${TABLE}.created_at <=`) + expect(out).toContain("AT TIME ZONE 'UTC'") + expect(out).not.toContain('requires a number') + }) +}) + +describe('range operators on non-numeric column types', () => { + it('compares string columns lexicographically as text (no numeric cast)', () => { + const cols: ColumnDefinition[] = [{ name: 'name', type: 'string' }] + const out = render(buildFilterClause({ name: { $gte: 'M' } }, TABLE, cols)) + expect(out).toContain(`${TABLE}.data->>'name' >=`) + expect(out).not.toContain('::numeric') + }) + + it('rejects ranges on boolean / json columns with a type-naming message', () => { + for (const type of ['boolean', 'json'] as const) { + const cols: ColumnDefinition[] = [{ name: 'flag', type }] + expect(() => buildFilterClause({ flag: { $gt: 1 } }, TABLE, cols)).toThrow( + new RegExp(`\\(${type}\\) is not supported`) + ) + } + }) +}) + describe('buildPredicateClause (v2 grammar)', () => { it('all joins members with AND', () => { const p: TablePredicate = { diff --git a/apps/sim/lib/table/constants.ts b/apps/sim/lib/table/constants.ts index 53ca282ee0d..8b441c61aff 100644 --- a/apps/sim/lib/table/constants.ts +++ b/apps/sim/lib/table/constants.ts @@ -72,6 +72,21 @@ export const DEFAULT_TABLE_PLAN_LIMITS = { }, } as const +/** + * Byte budget at which a **bounded** page (one with an explicit `limit`) is cut + * short, or `null` when disabled — the default. Opt in with `TABLE_MAX_PAGE_BYTES`. + * + * Off by default because a short page is only safe for a client that terminates + * on `nextCursor === null`; a pre-existing v1 pager terminating on + * `rows.length < limit` would read the cut as end-of-data and silently truncate. + * Unbounded queries (no `limit`) are unaffected — they always fail fast at + * `TABLE_LIMITS.MAX_QUERY_RESULT_BYTES` rather than return a partial result. + */ +export function getMaxPageBytes(): number | null { + const value = envNumber(env.TABLE_MAX_PAGE_BYTES, 0, { min: 0, integer: true }) + return value > 0 ? value : null +} + /** * Maximum serialized size in bytes of a single row. Defaults to * `TABLE_LIMITS.MAX_ROW_SIZE_BYTES`; overridable via the @@ -171,8 +186,6 @@ export const FILTER_OPS = [ 'ilike', 'nlike', 'nilike', - 'match', - 'imatch', 'isEmpty', 'isNotEmpty', 'isNull', diff --git a/apps/sim/lib/table/errors.ts b/apps/sim/lib/table/errors.ts index bb47bdf4367..6b80b6a7260 100644 --- a/apps/sim/lib/table/errors.ts +++ b/apps/sim/lib/table/errors.ts @@ -14,7 +14,7 @@ export type TableQueryErrorCode = * Routes should map this to HTTP 400 with the message preserved. * * Lives outside `sql.ts` so client-bundled modules (the block definitions pull - * in the PostgREST serializers) can reference it without dragging drizzle-orm + * in the query-builder converters) can reference it without dragging drizzle-orm * into the browser chunk. */ export class TableQueryValidationError extends Error { diff --git a/apps/sim/lib/table/query-builder/constants.ts b/apps/sim/lib/table/query-builder/constants.ts index ada8bc8c882..82b464e9892 100644 --- a/apps/sim/lib/table/query-builder/constants.ts +++ b/apps/sim/lib/table/query-builder/constants.ts @@ -33,7 +33,12 @@ export const LOGICAL_OPERATORS = [ { value: 'or', label: 'or' }, ] as const -export const SORT_DIRECTIONS = [ +/** + * Direction picker options for the sort-builder UI. Distinct from the wire-level + * `SORT_DIRECTIONS` tuple in `lib/table/constants.ts` — both are star-exported + * through `lib/table/index.ts`, and a duplicate name resolves to nothing there. + */ +export const SORT_DIRECTION_OPTIONS = [ { value: 'asc', label: 'ascending' }, { value: 'desc', label: 'descending' }, ] as const diff --git a/apps/sim/lib/table/query-builder/converters.ts b/apps/sim/lib/table/query-builder/converters.ts index f90041ee539..ca9c7a204d3 100644 --- a/apps/sim/lib/table/query-builder/converters.ts +++ b/apps/sim/lib/table/query-builder/converters.ts @@ -146,14 +146,7 @@ function applyLogicalOperators(groups: FilterRule[][]): FilterRule[] { } const ARRAY_OPERATORS = new Set(['in', 'nin']) -const TEXT_MATCH_OPERATORS = new Set([ - 'contains', - 'ncontains', - 'startsWith', - 'endsWith', - 'match', - 'imatch', -]) +const TEXT_MATCH_OPERATORS = new Set(['contains', 'ncontains', 'startsWith', 'endsWith']) function parseValue(value: string, operator: string): JsonValue { if (ARRAY_OPERATORS.has(operator)) { diff --git a/apps/sim/lib/table/query-builder/use-query-builder.ts b/apps/sim/lib/table/query-builder/use-query-builder.ts index 623ec4cd8fd..2d86ae236fd 100644 --- a/apps/sim/lib/table/query-builder/use-query-builder.ts +++ b/apps/sim/lib/table/query-builder/use-query-builder.ts @@ -8,7 +8,7 @@ import { COMPARISON_OPERATORS, type FilterRule, LOGICAL_OPERATORS, - SORT_DIRECTIONS, + SORT_DIRECTION_OPTIONS, type SortRule, } from '@/lib/table/query-builder/constants' import type { ColumnOption } from '@/lib/table/types' @@ -23,7 +23,7 @@ const logicalOptions: ColumnOption[] = LOGICAL_OPERATORS.map((op) => ({ label: op.label, })) -const sortDirectionOptions: ColumnOption[] = SORT_DIRECTIONS.map((d) => ({ +const sortDirectionOptions: ColumnOption[] = SORT_DIRECTION_OPTIONS.map((d) => ({ value: d.value, label: d.label, })) diff --git a/apps/sim/lib/table/query-builder/validate.ts b/apps/sim/lib/table/query-builder/validate.ts index 24d913146b3..6a6ab10aec4 100644 --- a/apps/sim/lib/table/query-builder/validate.ts +++ b/apps/sim/lib/table/query-builder/validate.ts @@ -14,7 +14,7 @@ import type { * Schema-aware validation for the typed predicate/sort wire. The engine * (`buildPredicateClause`) trusts its input, so this is the boundary gate that * every caller-supplied filter/sort passes through — the same checks the old - * PostgREST parser did inline, now grammar-agnostic and applied to the object + * parser used to do inline, now grammar-agnostic and applied to the object * directly (predicates are column-NAME-keyed at the boundary, translated to ids * afterwards). */ @@ -31,10 +31,16 @@ const VALUELESS_OPS = new Set(['isEmpty', 'isNotEmpty', 'isNull', 'isN */ const MAX_IN_LIST_SIZE = 1000 -/** System timestamp columns are filterable/sortable but are not in `schema.columns`. */ +/** + * Row-level system columns are filterable/sortable but are not in + * `schema.columns`. Must stay in sync with `SYSTEM_COLUMNS` in `lib/table/sql.ts` + * — a name here with no SQL dispatch there compiles to a `data->>` extraction + * that silently matches nothing. + */ const SYSTEM_COLUMN_TYPES: ReadonlyArray<[string, ColumnType]> = [ ['createdAt', 'date'], ['updatedAt', 'date'], + ['id', 'string'], ] function buildTypeByName(columns: ColumnDefinition[]): Map { @@ -62,7 +68,7 @@ function validateLeaf(leaf: Predicate, typeByName: Map): voi } if (typeByName.get(leaf.field) === 'json' && CONTAINMENT_OPS.has(leaf.op)) { throw new TableQueryValidationError( - `Operator "${leaf.op}" is not supported on json column "${leaf.field}" — use like/ilike for text match or is.null.`, + `Operator "${leaf.op}" is not supported on json column "${leaf.field}" — use like/ilike for text match, or isNull/isNotNull.`, 'INVALID_FILTER' ) } diff --git a/apps/sim/lib/table/rows/service.ts b/apps/sim/lib/table/rows/service.ts index e3424f0e470..f695317c4fc 100644 --- a/apps/sim/lib/table/rows/service.ts +++ b/apps/sim/lib/table/rows/service.ts @@ -24,7 +24,7 @@ import { wouldExceedRowLimit, } from '@/lib/table/billing' import { getColumnId } from '@/lib/table/column-keys' -import { TABLE_LIMITS, USER_TABLE_ROWS_SQL_NAME } from '@/lib/table/constants' +import { getMaxPageBytes, TABLE_LIMITS, USER_TABLE_ROWS_SQL_NAME } from '@/lib/table/constants' import { TableQueryValidationError } from '@/lib/table/errors' import { nKeysBetween } from '@/lib/table/order-key' import { @@ -1044,6 +1044,7 @@ export async function queryRows( startOffset: offset, limit, budgetBytes: TABLE_LIMITS.MAX_QUERY_RESULT_BYTES, + pageCutBytes: getMaxPageBytes() ?? undefined, }) const [fetched, totalCount] = await Promise.all([drainPromise, countPromise]) @@ -1109,7 +1110,13 @@ interface BoundedFetchParams { /** Inbound offset — the whole-view offset, or the past-anchor offset of a compound cursor. */ startOffset: number limit?: number + /** Drain ceiling: sizes batches, and the fail-fast bound for an unbounded query. */ budgetBytes: number + /** + * Opt-in byte cut for a **bounded** page (`TABLE_MAX_PAGE_BYTES`); `undefined` + * disables it, so a bounded page always returns its full `limit`. + */ + pageCutBytes?: number } interface BoundedFetchResult { @@ -1128,10 +1135,16 @@ const MAX_QUERY_BATCHES = 1000 /** * Drains rows in adaptively-sized bounded batches until the caller's `limit` - * or the byte budget cuts the page. Never issues an unbounded SELECT: the + * or the byte ceiling ends the page. Never issues an unbounded SELECT: the * first batch is capped so its worst-case bytes stay within ~4× the budget at * the max row size, and later batches are sized from the observed average. * + * Byte ceiling: an **unbounded** query (no `limit`) always fails fast at + * `budgetBytes` — returning part of a result that promised everything would be + * silent truncation. A **bounded** page cuts short only when `pageCutBytes` is + * set (`TABLE_MAX_PAGE_BYTES`), because a short page is only safe for clients + * that terminate on `nextCursor === null` rather than on page fullness. + * * Advance strategy: when `keysetValid`, the loop re-anchors on each consumed * keyed row and seeks `(order_key, id) > (anchor)` — delete-tolerant and an * index seek. Otherwise (custom sort, flag off) it advances by OFFSET from the @@ -1142,10 +1155,14 @@ const MAX_QUERY_BATCHES = 1000 * alone exceeds the budget. */ async function fetchRowsBounded(params: BoundedFetchParams): Promise { - const { baseWhere, orderBy, sorted, keysetValid, limit, budgetBytes } = params + const { baseWhere, orderBy, sorted, keysetValid, limit, budgetBytes, pageCutBytes } = params const firstBatchCap = Math.max(1, Math.floor((4 * budgetBytes) / TABLE_LIMITS.MAX_ROW_SIZE_BYTES)) + // The byte ceiling that ends the drain: an unbounded query fails fast at the + // budget; a bounded page cuts only when the operator opted in. + const cutBytes = limit === undefined ? budgetBytes : pageCutBytes + const rows: Array = [] let bytes = 0 let maxRowBytes = 0 @@ -1157,7 +1174,11 @@ async function fetchRowsBounded(params: BoundedFetchParams): Promise { if (rows.length === 0) return Math.min(limit ?? firstBatchCap, firstBatchCap) const avg = Math.max(1, bytes / rows.length) - const remaining = budgetBytes - bytes + // Bytes we may still fetch this batch. When a cut is active it's the + // remainder of that cut; otherwise each batch gets a fresh budget's worth, + // so a large bounded page keeps draining in real steps instead of degrading + // to one row per query once cumulative bytes pass the budget. + const remaining = Math.max(1, cutBytes === undefined ? budgetBytes : cutBytes - bytes) const byAverage = Math.ceil(remaining / avg) + 1 const varianceCap = Math.ceil((8 * remaining) / Math.max(maxRowBytes, 1)) return Math.max(1, Math.min(byAverage, TABLE_LIMITS.QUERY_BATCH_MAX_ROWS, varianceCap)) @@ -1205,17 +1226,17 @@ async function fetchRowsBounded(params: BoundedFetchParams): Promise 0 && bytes + rowBytes > budgetBytes) { + if (cutBytes !== undefined && rows.length > 0 && bytes + rowBytes > cutBytes) { // Unbounded queries promise the ENTIRE result — a partial page would be // silent truncation, so fail fast instead (the drain has only fetched // ~budget bytes at this point, never the whole table). if (limit === undefined) { throw new TableQueryValidationError( - `Query result exceeds the ${Math.floor(budgetBytes / (1024 * 1024))}MB limit. Add a filter or a limit to narrow the result.`, + `Query result exceeds the ${Math.floor(cutBytes / (1024 * 1024))}MB limit. Add a filter or a limit to narrow the result.`, 'TABLE_QUERY_RESULT_TOO_LARGE' ) } - // Bounded page: byte cut with `row` as the witness. Requires a + // Bounded page, byte cut opted in: `row` is the witness. Requires a // non-empty page so a single over-budget row is still returned alone. hasMore = true cut = true diff --git a/apps/sim/lib/table/sql.ts b/apps/sim/lib/table/sql.ts index f17a84b9d5a..6a9fd1865ce 100644 --- a/apps/sim/lib/table/sql.ts +++ b/apps/sim/lib/table/sql.ts @@ -73,8 +73,6 @@ const ALLOWED_OPERATORS = new Set([ '$ilike', '$nlike', '$nilike', - '$match', - '$imatch', '$empty', '$isNull', '$isNotNull', @@ -479,11 +477,6 @@ export function fieldPredicate( case 'isNotEmpty': return buildEmptyClause(tableName, field, false) - case 'match': - return buildRegexClause(tableName, field, value as string, false) - case 'imatch': - return buildRegexClause(tableName, field, value as string, true) - case 'isNull': return buildNullClause(tableName, field, true) case 'isNotNull': @@ -536,13 +529,15 @@ function buildLogicalClause( } /** - * Row-level system columns exposed for filter/sort: their wire name → the real - * `timestamptz` column. Not stored in `data`, so they need real-column SQL, not - * the `data->>'field'` extraction every other builder uses. + * Row columns that are addressable in filters/sorts but live on the row itself + * rather than inside the JSONB `data` blob. The docs advertise all three as + * filterable and sortable; without this dispatch they compile to a `data->>'…'` + * extraction of a key that never exists, so they silently match nothing. */ -const SYSTEM_COLUMNS: Readonly> = { - createdAt: 'created_at', - updatedAt: 'updated_at', +const SYSTEM_COLUMNS: Readonly> = { + createdAt: { column: 'created_at', kind: 'timestamp' }, + updatedAt: { column: 'updated_at', kind: 'timestamp' }, + id: { column: 'id', kind: 'text' }, } function isSystemColumn(field: string): boolean { @@ -550,9 +545,9 @@ function isSystemColumn(field: string): boolean { } /** - * Builds a predicate against a `timestamptz` system column. Values are ISO - * strings cast to `timestamptz`. Only comparison/equality/null ops make sense; - * text/pattern ops are rejected with an actionable error. + * Builds a predicate against a system column. Timestamp columns bind ISO strings + * normalized to UTC wall clock; the text column (`id`) binds as text and also + * accepts the pattern ops. Anything else is rejected with an actionable error. */ function buildSystemColumnClause( tableName: string, @@ -560,28 +555,83 @@ function buildSystemColumnClause( op: FilterOp, value: JsonValue | undefined ): SQL | undefined { - const col = sql.raw(`${tableName}.${SYSTEM_COLUMNS[field]}`) - const ts = (v: JsonValue | undefined) => sql`${String(v)}::timestamptz` + const spec = SYSTEM_COLUMNS[field] + const col = sql.raw(`${tableName}.${spec.column}`) + // `created_at`/`updated_at` are `timestamp WITHOUT time zone` holding UTC wall + // clock. A bare `::timestamptz` comparison promotes the column using the session + // `TimeZone` GUC, so identical queries return different rows per environment and + // day-boundary ranges land off by the offset. Normalizing the bound to UTC wall + // clock is session-independent and still honors an explicit offset in the input. + const ts = (v: JsonValue | undefined) => sql`${String(v)}::timestamptz AT TIME ZONE 'UTC'` + const bind = spec.kind === 'timestamp' ? ts : (v: JsonValue | undefined) => sql`${String(v)}` + /** + * Mirrors the JSONB pattern builders: `*` is the caller's only wildcard, an + * empty pattern is rejected (it would collapse to `%` and match every row), + * and the negated forms keep NULL cells so "does not contain X" retains them. + */ + const like = ( + v: JsonValue | undefined, + pattern: (escaped: string) => string, + ci: boolean, + negate = false + ) => { + const text = String(v ?? '') + if (text.length === 0) { + throw new TableQueryValidationError( + `Operator "${op}" on column "${field}" requires a non-empty value` + ) + } + const p = pattern(escapeLikePattern(text)) + const match = ci ? sql`${col} ILIKE ${p}` : sql`${col} LIKE ${p}` + return negate ? sql`NOT (${match})` : match + } + + // The text system column (`id`) additionally supports the pattern ops; + // timestamps fall through to the unsupported-operator error below. + if (spec.kind === 'text') { + const star = (e: string) => e.replace(/\*/g, '%') + switch (op) { + case 'like': + return like(value, star, false) + case 'ilike': + return like(value, star, true) + case 'nlike': + return like(value, star, false, true) + case 'nilike': + return like(value, star, true, true) + case 'contains': + return like(value, (e) => `%${e}%`, true) + case 'ncontains': + return like(value, (e) => `%${e}%`, true, true) + case 'startsWith': + return like(value, (e) => `${e}%`, true) + case 'endsWith': + return like(value, (e) => `%${e}`, true) + default: + break + } + } + switch (op) { case 'eq': - return sql`${col} = ${ts(value)}` + return sql`${col} = ${bind(value)}` case 'ne': - return sql`${col} <> ${ts(value)}` + return sql`${col} <> ${bind(value)}` case 'gt': - return sql`${col} > ${ts(value)}` + return sql`${col} > ${bind(value)}` case 'gte': - return sql`${col} >= ${ts(value)}` + return sql`${col} >= ${bind(value)}` case 'lt': - return sql`${col} < ${ts(value)}` + return sql`${col} < ${bind(value)}` case 'lte': - return sql`${col} <= ${ts(value)}` + return sql`${col} <= ${bind(value)}` case 'in': { if (!Array.isArray(value) || value.length === 0) return undefined - return sql`${col} IN (${sql.join(value.map(ts), sql.raw(', '))})` + return sql`${col} IN (${sql.join(value.map(bind), sql.raw(', '))})` } case 'nin': { if (!Array.isArray(value) || value.length === 0) return undefined - return sql`${col} NOT IN (${sql.join(value.map(ts), sql.raw(', '))})` + return sql`${col} NOT IN (${sql.join(value.map(bind), sql.raw(', '))})` } case 'isNull': case 'isEmpty': @@ -591,7 +641,7 @@ function buildSystemColumnClause( return sql`${col} IS NOT NULL` default: throw new TableQueryValidationError( - `Operator "${op}" is not supported on the timestamp column "${field}" — use eq, neq, gt, gte, lt, lte, in, is.null.` + `Operator "${op}" is not supported on the built-in column "${field}" — use eq, ne, gt, gte, lt, lte, in, nin, isNull, isNotNull.` ) } } @@ -609,11 +659,12 @@ function buildContainmentClause(tableName: string, field: string, value: JsonVal * to `timestamptz` so date strings compare chronologically and timezone offsets * in ISO strings (e.g. `2024-01-01T00:00:00Z`) are preserved rather than * silently stripped (which would make results depend on the server's TimeZone - * setting). Unknown/other types - * fall back to `numeric` (legacy default — preserves behavior for ad-hoc fields - * with no schema entry). The right-hand value is cast explicitly because - * drizzle parameterizes it as `text`; without the cast, Postgres would compare - * `text text` and silently produce lexicographic results. + * setting). `string` columns compare lexicographically as text. `boolean`/`json` + * columns have no meaningful ordering and are rejected. Columns with no schema + * entry fall back to `numeric` (legacy default — preserves behavior for ad-hoc + * fields). The right-hand value is cast explicitly because drizzle parameterizes + * it as `text`; without the cast, Postgres would compare `text text` and + * silently produce lexicographic results. * * Cannot use the GIN index — falls back to a sequential scan over the table's * rows (bounded by the btree prefix on `table_id`). @@ -626,6 +677,18 @@ function buildComparisonClause( columnType: ColumnType | undefined ): SQL { const escapedField = field.replace(/'/g, "''") + + if (columnType === 'boolean' || columnType === 'json') { + throw new TableQueryValidationError( + `Range operator on column "${field}" (${columnType}) is not supported — ${columnType} values have no ordering.` + ) + } + + if (columnType === 'string') { + const cell = sql.raw(`${tableName}.data->>'${escapedField}'`) + return sql`${cell} ${sql.raw(operator)} ${String(value)}` + } + const cast = jsonbCastForType(columnType) ?? 'numeric' validateComparisonValue(field, columnType, cast, value) const cell = sql.raw(`(${tableName}.data->>'${escapedField}')::${cast}`) @@ -640,7 +703,7 @@ export function escapeLikePattern(value: string): string { } /** - * General LIKE/ILIKE pattern match (PostgREST `like`/`ilike`). The caller's `*` + * General LIKE/ILIKE pattern match (the `like`/`ilike` ops). The caller's `*` * is the only wildcard — it maps to SQL `%`; any literal `%`/`_`/`\` in the * value is escaped so it matches itself. Empty/exact patterns are allowed (an * empty pattern matches only the empty string, not every row, so it's not the @@ -738,25 +801,6 @@ function buildEmptyClause(tableName: string, field: string, isEmpty: boolean): S : sql`(${cell} IS NOT NULL AND ${cell} <> '')` } -/** - * Builds a POSIX regex match against a JSONB cell (`~` case-sensitive, `~*` - * case-insensitive). The pattern is parameterized (no interpolation); Postgres - * validates it at runtime. Negation surfaces null cells, mirroring `$ne`/ - * `$ncontains`. Cannot use the GIN index — sequential scan bounded by the - * `table_id` btree prefix. - */ -function buildRegexClause( - tableName: string, - field: string, - value: string, - caseInsensitive: boolean -): SQL { - const escapedField = field.replace(/'/g, "''") - const cell = sql.raw(`${tableName}.data->>'${escapedField}'`) - const pattern = String(value) - return caseInsensitive ? sql`${cell} ~* ${pattern}` : sql`${cell} ~ ${pattern}` -} - /** * Strict null check on a JSONB cell — distinct from `isEmpty`/`isNotEmpty`, * which also treat the empty string as empty. `isNull` matches an absent key or @@ -789,7 +833,7 @@ function buildSortFieldClause( const directionSql = direction.toUpperCase() if (isSystemColumn(field)) { - return sql.raw(`${tableName}.${SYSTEM_COLUMNS[field]} ${directionSql}`) + return sql.raw(`${tableName}.${SYSTEM_COLUMNS[field].column} ${directionSql}`) } const jsonbExtract = `${tableName}.data->>'${escapedField}'` diff --git a/apps/sim/lib/table/types.ts b/apps/sim/lib/table/types.ts index 5fac83143c8..65ddad502b1 100644 --- a/apps/sim/lib/table/types.ts +++ b/apps/sim/lib/table/types.ts @@ -426,8 +426,8 @@ export interface Filter { /** * v2 filter operators (bare, no `$`). Equality and `in`/`nin` are case-sensitive * (JSONB containment, GIN-indexed); the text ops `contains`/`ncontains`/ - * `startsWith`/`endsWith` are ILIKE (case-insensitive); `match`/`imatch` are - * POSIX regex (`~` / `~*`). `isEmpty`/`isNotEmpty` match null OR empty string; + * `startsWith`/`endsWith` are ILIKE (case-insensitive). `isEmpty`/`isNotEmpty` + * match null OR empty string; * `isNull`/`isNotNull` are strict null checks. The four `is*` ops are valueless. * This is the canonical operator set the shared `fieldPredicate` leaf * understands; the legacy `$`-operators normalize onto it. diff --git a/apps/sim/tools/registry.minimal.ts b/apps/sim/tools/registry.minimal.ts index f5ad29cf0c8..cd4ef77d640 100644 --- a/apps/sim/tools/registry.minimal.ts +++ b/apps/sim/tools/registry.minimal.ts @@ -27,19 +27,6 @@ import { } from '@/tools/gmail' import { guardrailsValidateTool } from '@/tools/guardrails' import { httpRequestTool } from '@/tools/http' -import { - tableBatchInsertRowsTool, - tableDeleteRowsByFilterTool, - tableDeleteRowTool, - tableGetRowTool, - tableGetSchemaTool, - tableInsertRowTool, - tableQueryRowsTool, - tableQueryRowsV2Tool, - tableUpdateRowsByFilterTool, - tableUpdateRowTool, - tableUpsertRowTool, -} from '@/tools/table' import { slackAddReactionTool, slackArchiveConversationTool, @@ -84,6 +71,19 @@ import { slackUpdateMessageTool, slackUpdateViewTool, } from '@/tools/slack' +import { + tableBatchInsertRowsTool, + tableDeleteRowsByFilterTool, + tableDeleteRowTool, + tableGetRowTool, + tableGetSchemaTool, + tableInsertRowTool, + tableQueryRowsTool, + tableQueryRowsV2Tool, + tableUpdateRowsByFilterTool, + tableUpdateRowTool, + tableUpsertRowTool, +} from '@/tools/table' import type { ToolConfig } from '@/tools/types' import { customBlockExecutorTool, workflowExecutorTool } from '@/tools/workflow' From ead639130879a1dcbbf119737a1dab27a3d2021b Mon Sep 17 00:00:00 2001 From: Theodore Li Date: Fri, 24 Jul 2026 19:09:43 -0700 Subject: [PATCH 11/26] fix(table): name the grammar mistake when a legacy $-filter reaches the v2 gate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A `{ status: { $eq: 'x' } }` filter is neither a group nor a leaf, so it fell through to `validateLeaf` with `field: undefined` and came back as `Unknown filter column "undefined"` — a message that sends an LLM caller retrying column names instead of switching grammars. This is reachable today: the copilot's `query_user_table` catalog entry still advertises `filter: MongoDB-style filter for query_rows` plus `offset`/`sort`, while the tool now routes through `validatePredicate` (rejects the $-grammar) and reads only `order`/`cursor` (so `offset`/`sort` silently no-op). Fixing the catalog belongs upstream in the mothership repo; this makes the failure legible in the meantime. Also: - Point both table blocks' docsLink at docs.sim.ai/integrations/table. They were the last two blocks in the repo still on the dead docs.simstudio.ai domain. - Fix the openapi `sort` example, which showed `{"created_at": "desc"}`. The built-in column is `createdAt`; the snake_case form is treated as a user column, so anyone copying the example sorted by a JSONB key that never exists and got NULL ordering — the same silent-wrong-answer class as #5920, on the already-shipped v1 surface. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_011M563cbFy2S74GvSDf2C3R --- apps/docs/openapi.json | 2 +- apps/sim/blocks/blocks/table.ts | 2 +- apps/sim/blocks/blocks/table_v2.ts | 2 +- .../query-builder/__tests__/validate.test.ts | 26 +++++++++++++++++++ apps/sim/lib/table/query-builder/validate.ts | 17 ++++++++++++ 5 files changed, 46 insertions(+), 3 deletions(-) diff --git a/apps/docs/openapi.json b/apps/docs/openapi.json index 49d6e7ed91b..2537f15832c 100644 --- a/apps/docs/openapi.json +++ b/apps/docs/openapi.json @@ -2956,7 +2956,7 @@ "name": "sort", "in": "query", "required": false, - "description": "JSON-encoded sort object. Example: {\"created_at\": \"desc\"}.", + "description": "JSON-encoded sort object. Example: {\"createdAt\": \"desc\"}. Built-in columns are camelCase: id, createdAt, updatedAt.", "schema": { "type": "string" } diff --git a/apps/sim/blocks/blocks/table.ts b/apps/sim/blocks/blocks/table.ts index 281bcd077e7..3e2a99f88d7 100644 --- a/apps/sim/blocks/blocks/table.ts +++ b/apps/sim/blocks/blocks/table.ts @@ -183,7 +183,7 @@ export const TableBlock: BlockConfig = { description: 'User-defined data tables', longDescription: 'Create and manage custom data tables. Store, query, and manipulate structured data within workflows.', - docsLink: 'https://docs.simstudio.ai/tools/table', + docsLink: 'https://docs.sim.ai/integrations/table', category: 'blocks', bgColor: '#10B981', icon: TableIcon, diff --git a/apps/sim/blocks/blocks/table_v2.ts b/apps/sim/blocks/blocks/table_v2.ts index 8b60a942a14..ec352b91dbb 100644 --- a/apps/sim/blocks/blocks/table_v2.ts +++ b/apps/sim/blocks/blocks/table_v2.ts @@ -190,7 +190,7 @@ export const TableV2Block: BlockConfig = { - Omit Limit to get the entire matching result in one response — the query fails with a clear error if it exceeds 5MB (narrow with a filter or set a Limit). - With a Limit, pages can end at the Limit or the 5MB byte budget, whichever comes first — pass nextCursor back as the cursor and loop until it is null; never infer completion from page size. - Columns are scalar (string/number/boolean/date) or opaque json — there are no array columns; for substring use ilike with *x*.`, - docsLink: 'https://docs.simstudio.ai/tools/table', + docsLink: 'https://docs.sim.ai/integrations/table', category: 'blocks', // Unreleased: hidden from every discovery surface until revealed via the hosted // `block-visibility` AppConfig document or the `PREVIEW_BLOCKS` env allowlist. diff --git a/apps/sim/lib/table/query-builder/__tests__/validate.test.ts b/apps/sim/lib/table/query-builder/__tests__/validate.test.ts index 2521d61ad5b..9c7a7855024 100644 --- a/apps/sim/lib/table/query-builder/__tests__/validate.test.ts +++ b/apps/sim/lib/table/query-builder/__tests__/validate.test.ts @@ -149,3 +149,29 @@ describe('validatePredicate — leaves that would silently widen a bulk write', ).not.toThrow() }) }) + +/** + * The copilot's `query_user_table` catalog entry still advertises the legacy + * `$`-grammar, so the agent sends `{ status: { $eq: 'x' } }`. That shape hit + * `validateLeaf` with `field: undefined` and came back as + * `Unknown filter column "undefined"` — a message that sends an LLM retrying + * column names instead of switching grammars. + */ +describe('validatePredicate — legacy $-grammar diagnostics', () => { + it('names the grammar mistake instead of blaming a phantom column', () => { + for (const legacy of [ + { status: { $eq: 'active' } }, + { $or: [{ status: 'a' }, { status: 'b' }] }, + { wins: { $gte: 10 } }, + ]) { + expect(() => validatePredicate(legacy as never, COLS)).toThrow(/legacy operator-object/) + expect(() => validatePredicate(legacy as never, COLS)).not.toThrow(/undefined/) + } + }) + + it('still rejects a plain non-predicate object clearly', () => { + expect(() => validatePredicate({ nonsense: 'x' } as never, COLS)).toThrow( + /must be a group .* or a condition/ + ) + }) +}) diff --git a/apps/sim/lib/table/query-builder/validate.ts b/apps/sim/lib/table/query-builder/validate.ts index 6a6ab10aec4..c66f1ea9b59 100644 --- a/apps/sim/lib/table/query-builder/validate.ts +++ b/apps/sim/lib/table/query-builder/validate.ts @@ -1,3 +1,4 @@ +import { isRecordLike } from '@sim/utils/object' import { NAME_PATTERN } from '@/lib/table/constants' import { TableQueryValidationError } from '@/lib/table/errors' import type { @@ -136,6 +137,22 @@ function validateNode(node: PredicateNode, typeByName: Map): for (const child of members) validateNode(child, typeByName) return } + // Neither a group nor a leaf. Overwhelmingly this is the legacy `$`-grammar + // (`{ status: { $eq: 'x' } }`) — a shape `validateLeaf` would reject as + // `Unknown filter column "undefined"`, which tells an LLM caller nothing and + // sends it retrying column names forever. Name the actual mistake. + if (!('field' in node)) { + const keys = Object.keys(node) + const looksLegacy = keys.some( + (k) => k.startsWith('$') || isRecordLike((node as Record)[k]) + ) + throw new TableQueryValidationError( + looksLegacy + ? 'Filter uses the legacy operator-object grammar. Use a predicate tree instead: { all: [{ field, op, value }] } (or "any" for OR), with bare operators like eq/gte/contains/in.' + : 'A filter node must be a group ({ all | any: [...] }) or a condition ({ field, op, value }).', + 'INVALID_FILTER' + ) + } validateLeaf(node as Predicate, typeByName) } From 705ea6f4d07fd3c352c4c81ab18c8109ae945b49 Mon Sep 17 00:00:00 2001 From: Theodore Li Date: Sat, 25 Jul 2026 12:18:38 -0700 Subject: [PATCH 12/26] fix(table): reject hybrid predicate nodes at the wire instead of silently narrowing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Found by running the HTTP suite: a node carrying BOTH a group key and a leaf's `field`/`op`/`value` returned 200, not 400. Zod strips unrecognized keys by default, so `{ all: [...], field, op, value }` parsed clean against the group branch with the leaf half quietly deleted. The hybrid guard added in eaf4179962 could never fire — the keys were gone before `validatePredicate` ran. On the bulk paths that turns "delete archived rows for tenant acme" into "delete EVERY row for tenant acme". Both node shapes are now `strictObject`. Strict on the group alone would be worse than the bug: the union would fall through to the leaf branch, which is the more dangerous reading of the two. The bulk schemas are unaffected by design — their legacy `$`-object branch accepts any non-empty object, so it absorbs the hybrid WITHOUT stripping, the route's `isTablePredicate` check routes it back to `validatePredicate`, and the runtime guard rejects it there. Tests now pin both layers so removing either one fails loudly. Verified against a running server on the `hello` table: 18/18 HTTP checks pass, including the previously-failing hybrid case. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_011M563cbFy2S74GvSDf2C3R --- .../api/contracts/tables-predicate.test.ts | 74 +++++++++++++++++++ apps/sim/lib/api/contracts/tables.ts | 16 +++- 2 files changed, 87 insertions(+), 3 deletions(-) diff --git a/apps/sim/lib/api/contracts/tables-predicate.test.ts b/apps/sim/lib/api/contracts/tables-predicate.test.ts index 1ce365ea669..5d6c915990f 100644 --- a/apps/sim/lib/api/contracts/tables-predicate.test.ts +++ b/apps/sim/lib/api/contracts/tables-predicate.test.ts @@ -12,6 +12,7 @@ import { rowQueryBodySchema, updateRowsByFilterBodySchema, } from '@/lib/api/contracts/tables' +import { validatePredicate } from '@/lib/table/query-builder/validate' describe('rowQueryBodySchema', () => { it('accepts a predicate/sort object, leaves limit unbounded, has no offset', () => { @@ -149,3 +150,76 @@ describe('predicate depth / size guard', () => { ).toBe(true) }) }) + +/** + * Zod strips unrecognized keys by default, so before the schemas were made + * strict a hybrid node parsed clean against the group branch with its leaf half + * silently deleted — turning "delete archived rows for tenant acme" into + * "delete EVERY row for tenant acme". `validatePredicate`'s hybrid guard could + * not catch it: the keys were gone before it ran. + */ +describe('hybrid group+leaf nodes are rejected, not silently narrowed', () => { + const hybrid = { + all: [{ field: 'tenant_id', op: 'eq', value: 'acme' }], + field: 'status', + op: 'eq', + value: 'archived', + } + + it('rejects rather than dropping the leaf half', () => { + const result = predicateSchema.safeParse(hybrid) + expect(result.success).toBe(false) + // The dangerous outcome: parsing succeeds having quietly widened the filter. + expect(result.success ? result.data : null).not.toEqual({ all: hybrid.all }) + }) + + /** + * The bulk schemas union the predicate tree with the legacy `$`-object, and + * that legacy branch accepts any non-empty object — so it absorbs the hybrid + * and the SCHEMA cannot reject it. Crucially the legacy branch does NOT strip, + * so `all` survives, the route's `isTablePredicate` check routes it back to + * `validatePredicate`, and the hybrid guard there rejects it (→ 400 via + * `route.ts:325`). Asserted here so a future change to either layer that + * removes one of them fails loudly. + */ + it('keeps the hybrid intact through the bulk schemas so the runtime guard can see it', () => { + for (const parsed of [ + deleteTableRowsBodySchema.safeParse({ workspaceId: 'ws-1', filter: hybrid }), + updateRowsByFilterBodySchema.safeParse({ + workspaceId: 'ws-1', + filter: hybrid, + data: { active: false }, + }), + ]) { + expect(parsed.success).toBe(true) + // The leaf half must NOT have been silently dropped on the way through. + expect(parsed.success && parsed.data.filter).toMatchObject({ + all: hybrid.all, + field: 'status', + }) + } + }) + + it('and validatePredicate then rejects it', () => { + expect(() => + validatePredicate(hybrid as never, [ + { name: 'tenant_id', type: 'string' }, + { name: 'status', type: 'string' }, + ]) + ).toThrow(/not both/) + }) + + it('rejects an unknown key on a leaf (a typo must not be dropped)', () => { + expect( + predicateSchema.safeParse({ all: [{ field: 'a', op: 'eq', vlaue: 'typo' }] }).success + ).toBe(false) + }) + + it('still accepts well-formed nodes', () => { + expect( + predicateSchema.safeParse({ + all: [{ field: 'a', op: 'eq', value: 1 }, { any: [{ field: 'b', op: 'isNull' }] }], + }).success + ).toBe(true) + }) +}) diff --git a/apps/sim/lib/api/contracts/tables.ts b/apps/sim/lib/api/contracts/tables.ts index 142ec914ffe..cfc2382e199 100644 --- a/apps/sim/lib/api/contracts/tables.ts +++ b/apps/sim/lib/api/contracts/tables.ts @@ -319,7 +319,17 @@ function predicateTreeTooLarge(root: unknown): string | null { * (unknown column, json-op rejection) is enforced server-side by * `validatePredicate` once the table's columns are known. */ -const predicateLeafSchema = z.object({ +/** + * Both node shapes are `strictObject`, and that is load-bearing rather than + * fussiness. Zod strips unrecognized keys by default, so a hybrid node carrying + * BOTH a group key and a leaf's `field`/`op`/`value` parsed clean against the + * group branch with the leaf half silently deleted. On the bulk paths that turns + * "delete archived rows for tenant acme" into "delete every row for tenant acme". + * `validatePredicate`'s hybrid guard could never catch it — the keys were gone + * before it ran. Strict on BOTH branches is required: strict on the group alone + * would just fall through to the leaf branch, which is the more dangerous reading. + */ +const predicateLeafSchema = z.strictObject({ field: z.string().min(1, 'field is required').max(128), op: z.enum(FILTER_OPS), value: z.unknown().optional(), @@ -333,13 +343,13 @@ const predicateTreeSchema: z.ZodType = z.lazy(() => z.union([ // `.min(1)`: an empty group compiles to no WHERE clause, which on the bulk // delete/update paths reads as "match everything" rather than "match nothing". - z.object({ + z.strictObject({ all: z .array(predicateNodeSchema) .min(1, 'A filter group must contain at least one condition') .max(MAX_PREDICATE_GROUP_SIZE), }), - z.object({ + z.strictObject({ any: z .array(predicateNodeSchema) .min(1, 'A filter group must contain at least one condition') From 0df4d58f31f1266b92ad70d3f44925cfd6ecc9c8 Mon Sep 17 00:00:00 2001 From: Theodore Li Date: Mon, 27 Jul 2026 19:52:40 -0700 Subject: [PATCH 13/26] fix(table): reject a v2 predicate at the legacy filter compiler MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cross-version safety net. If a client speaking the predicate grammar reaches a server that predates it, the predicate arrives at the legacy `$`-compiler as `{ all: [...] }`. That was skipped as "an array on a regular field", so the filter compiled to NO WHERE CLAUSE — which on a bulk delete means every row rather than none. `update-runner` has always had an `if (!filterClause) throw`; `delete-runner` does not, so the background delete path (tables over 1000 rows) was the one that could actually wipe a table. `buildFilterClause` is the single choke point every filter path shares — `queryRows`, `update-runner`, `delete-runner`, inline and background — so one guard there covers all of them, and it names the mismatch instead of failing with a generic "filter required". Scoped to the `all`/`any` discriminators specifically: an ordinary column that happens to hold an array stays a silent skip, so no working legacy filter changes behaviour. This matters for deploy ordering. The copilot and sim deploy independently, and if the copilot ships the new grammar first it starts sending predicates to a sim that cannot parse them. With this guard that is a loud 400 on every path instead of a silent table wipe on one. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_011M563cbFy2S74GvSDf2C3R --- apps/sim/lib/table/__tests__/sql.test.ts | 47 ++++++++++++++++++++++++ apps/sim/lib/table/sql.ts | 12 ++++++ 2 files changed, 59 insertions(+) diff --git a/apps/sim/lib/table/__tests__/sql.test.ts b/apps/sim/lib/table/__tests__/sql.test.ts index 129170084e7..67d1f9bf6b8 100644 --- a/apps/sim/lib/table/__tests__/sql.test.ts +++ b/apps/sim/lib/table/__tests__/sql.test.ts @@ -774,3 +774,50 @@ describe('buildPredicateClause (v2 grammar)', () => { expect(() => buildPredicateClause(p, TABLE, [])).toThrow('Invalid field name') }) }) + +/** + * Cross-version safety. If a client speaking the v2 predicate grammar reaches a + * server that predates it, the predicate arrives at the LEGACY `$`-compiler as + * `{ all: [...] }`. That used to be skipped as "an array on a regular field", + * compiling to no WHERE clause — which on a bulk delete means every row rather + * than none. The guard turns that into a loud, self-describing failure, and it + * sits at the one choke point every filter path shares (`queryRows`, + * `update-runner`, `delete-runner`, inline and background). + */ +describe('legacy compiler rejects a v2 predicate (version-mismatch fail-fast)', () => { + it('throws on a top-level all/any group instead of emitting no clause', () => { + for (const group of ['all', 'any'] as const) { + expect(() => + buildFilterClause( + { [group]: [{ field: 'tenant_id', op: 'eq', value: 'acme' }] } as unknown as Filter, + TABLE, + NO_COLUMNS + ) + ).toThrow(/v2 predicate tree/) + } + }) + + it('catches one nested inside a legacy $or', () => { + expect(() => + buildFilterClause( + { + $or: [{ status: 'a' }, { all: [{ field: 'tenant_id', op: 'eq', value: 'acme' }] }], + } as unknown as Filter, + TABLE, + NO_COLUMNS + ) + ).toThrow(/v2 predicate tree/) + }) + + it('leaves legitimate legacy filters alone', () => { + expect(buildFilterClause({ status: 'archived' }, TABLE, NO_COLUMNS)).toBeDefined() + expect( + buildFilterClause({ $or: [{ status: 'a' }, { status: 'b' }] }, TABLE, NO_COLUMNS) + ).toBeDefined() + // An ordinary column holding an array stays a silent skip — only the + // predicate discriminators `all`/`any` are treated as a version mismatch. + expect(() => + buildFilterClause({ status: ['a', 'b'] } as unknown as Filter, TABLE, NO_COLUMNS) + ).not.toThrow() + }) +}) diff --git a/apps/sim/lib/table/sql.ts b/apps/sim/lib/table/sql.ts index 6a9fd1865ce..024ee133cda 100644 --- a/apps/sim/lib/table/sql.ts +++ b/apps/sim/lib/table/sql.ts @@ -157,6 +157,18 @@ function buildFilterClauseInternal( } // Skip arrays for regular fields - arrays are only valid for $or and $and. + // A v2 predicate tree (`{ all | any: [...] }`) that reaches this legacy + // compiler is a VERSION MISMATCH — a caller speaking the newer grammar + // against an older server. Skipping it as "an array on a regular field" + // compiles to no WHERE clause at all, which on a bulk delete means every + // row rather than none. Fail fast and name the mismatch instead. + if ((field === 'all' || field === 'any') && Array.isArray(condition)) { + throw new TableQueryValidationError( + `Filter looks like a v2 predicate tree ("${field}" group) but reached the legacy filter compiler. ` + + 'This usually means a client is sending the predicate grammar to a server that predates it.' + ) + } + // If we encounter an array here, it's likely malformed input (e.g., { name: [filter1, filter2] }) // which doesn't have a clear semantic meaning, so we skip it. if (Array.isArray(condition)) { From b3f3196338cdfbdd764efbcfe0d0943daa286483 Mon Sep 17 00:00:00 2001 From: Theodore Li Date: Tue, 28 Jul 2026 11:29:24 -0700 Subject: [PATCH 14/26] fix(table): treat a blank table_v2 filter/order editor field as absent Clearing the Filter or Order field in Editor mode is the ordinary way to say 'no filter'. `JSON.parse('')` throws, so it surfaced at run time as `Invalid JSON in Filter: Unexpected end of JSON input` plus a quoting hint that has nothing to do with the actual problem. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_011M563cbFy2S74GvSDf2C3R --- apps/sim/blocks/blocks/table_v2.test.ts | 26 +++++++++++++++++++++++++ apps/sim/blocks/blocks/table_v2.ts | 3 +++ 2 files changed, 29 insertions(+) diff --git a/apps/sim/blocks/blocks/table_v2.test.ts b/apps/sim/blocks/blocks/table_v2.test.ts index b333d77542c..1288d912e82 100644 --- a/apps/sim/blocks/blocks/table_v2.test.ts +++ b/apps/sim/blocks/blocks/table_v2.test.ts @@ -91,3 +91,29 @@ describe('table_v2 bulk transformers', () => { expect(out.filter).toEqual({ all: [{ field: 'name', op: 'eq', value: 'x' }] }) }) }) + +/** + * The Filter/Order fields are canonical pairs: a builder array in Builder mode, + * a JSON string in Editor mode. Clearing the editor field is the ordinary way to + * say "no filter" — it must not surface as a JSON parse error at run time. + */ +describe('table_v2 blank and malformed editor inputs', () => { + const base = { operation: 'query_rows', tableId: 'tbl_1', limit: '10' } + + it('treats a blank / whitespace filter or order as absent', () => { + for (const value of ['', ' ', '\n']) { + expect(params({ ...base, filterInput: value }).filter).toBeUndefined() + expect(params({ ...base, sortInput: value }).order).toBeUndefined() + } + }) + + it('treats an empty builder array as absent', () => { + expect(params({ ...base, filterInput: [] }).filter).toBeUndefined() + expect(params({ ...base, sortInput: [] }).order).toBeUndefined() + }) + + it('still reports genuinely malformed JSON', () => { + expect(() => params({ ...base, filterInput: '{not json}' })).toThrow(/Invalid JSON in Filter/) + expect(() => params({ ...base, sortInput: '{not json}' })).toThrow(/Invalid JSON in Sort/) + }) +}) diff --git a/apps/sim/blocks/blocks/table_v2.ts b/apps/sim/blocks/blocks/table_v2.ts index ec352b91dbb..74dffc275a7 100644 --- a/apps/sim/blocks/blocks/table_v2.ts +++ b/apps/sim/blocks/blocks/table_v2.ts @@ -18,6 +18,9 @@ import { getTrigger } from '@/triggers' function parseJSON(value: string | unknown, fieldName: string): unknown { if (typeof value !== 'string') return value + // A blank editor field means "no filter/order", not malformed JSON. Without + // this, clearing the field throws `Unexpected end of JSON input` at run time. + if (value.trim() === '') return undefined try { return JSON.parse(value) } catch (error) { From 4f9c1042d3d058beed352f2e86c95f6ca518e5e0 Mon Sep 17 00:00:00 2001 From: Theodore Li Date: Tue, 28 Jul 2026 16:16:12 -0700 Subject: [PATCH 15/26] fix(table): resolve select operands for contains/ncontains on multi-select MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Caught by driving mothership at a real multi-select table. It sent a correctly formed predicate — {field:'Color', op:'contains', value:'Teal'} — and got zero rows with success, against a table where 15 rows hold Teal. resolvePredicateSelectValues only resolved eq/ne/in/nin. I excluded contains/ncontains as 'pattern ops that match the raw stored cell', which holds for a string column but not for a multi-select: there the cell is an array of option ids and those two ops express MEMBERSHIP, so their operand is an option name that has to become an option id. It is the primary way to filter a multi-select, so the one op that mattered most was the one left out. The $-grammar sibling resolveFilterSelectValues has always handled $contains/$ncontains for this exact reason; the omission was mine, porting it. The remaining pattern ops (like/startsWith/...) never reach here — fieldPredicate's select allowlist rejects them on select columns first. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_011M563cbFy2S74GvSDf2C3R --- .../lib/table/__tests__/select-values.test.ts | 128 ++++++++++++++++++ apps/sim/lib/table/select-values.ts | 14 +- 2 files changed, 137 insertions(+), 5 deletions(-) create mode 100644 apps/sim/lib/table/__tests__/select-values.test.ts diff --git a/apps/sim/lib/table/__tests__/select-values.test.ts b/apps/sim/lib/table/__tests__/select-values.test.ts new file mode 100644 index 00000000000..32c0891e30c --- /dev/null +++ b/apps/sim/lib/table/__tests__/select-values.test.ts @@ -0,0 +1,128 @@ +/** + * @vitest-environment node + * + * Select-column operand resolution. A select cell stores option IDs, so a filter + * written with the option NAME must be rewritten before it reaches SQL — + * otherwise it compares a name against an id and matches nothing while reporting + * success. Both wire grammars have to do this identically. + */ +import { describe, expect, it } from 'vitest' +import { resolveFilterSelectValues, resolvePredicateSelectValues } from '@/lib/table/select-values' +import type { ColumnDefinition } from '@/lib/table/types' + +const MULTI: ColumnDefinition = { + id: 'col_color', + name: 'Color', + type: 'select', + multiple: true, + options: [ + { id: 'opt_teal', name: 'Teal' }, + { id: 'opt_green', name: 'Green' }, + ], +} +const SINGLE: ColumnDefinition = { + id: 'col_status', + name: 'Status', + type: 'select', + options: [{ id: 'opt_open', name: 'Open' }], +} +const PLAIN: ColumnDefinition = { id: 'col_name', name: 'name', type: 'string' } +const COLS = [MULTI, SINGLE, PLAIN] + +const leaf = (p: unknown) => (p as { all: Array<{ value: unknown }> }).all[0] + +describe('resolvePredicateSelectValues', () => { + /** + * Regression: `contains`/`ncontains` were excluded as "pattern ops". On a + * multi-select they are not pattern ops — the cell is an array of ids and they + * express membership. Mothership sent exactly this and silently got zero rows. + */ + it('resolves contains / ncontains on a MULTI-select (membership, not pattern)', () => { + for (const op of ['contains', 'ncontains'] as const) { + const out = resolvePredicateSelectValues( + { all: [{ field: 'col_color', op, value: 'Teal' }] }, + COLS + ) + expect(leaf(out).value).toBe('opt_teal') + } + }) + + it('resolves eq / ne / in / nin on a single select', () => { + expect( + leaf( + resolvePredicateSelectValues( + { all: [{ field: 'col_status', op: 'eq', value: 'Open' }] }, + COLS + ) + ).value + ).toBe('opt_open') + expect( + leaf( + resolvePredicateSelectValues( + { all: [{ field: 'col_status', op: 'in', value: ['Open'] }] }, + COLS + ) + ).value + ).toEqual(['opt_open']) + }) + + it('matches option names case-insensitively and passes ids through', () => { + expect( + leaf( + resolvePredicateSelectValues( + { all: [{ field: 'col_color', op: 'contains', value: 'teal' }] }, + COLS + ) + ).value + ).toBe('opt_teal') + expect( + leaf( + resolvePredicateSelectValues( + { all: [{ field: 'col_color', op: 'contains', value: 'opt_teal' }] }, + COLS + ) + ).value + ).toBe('opt_teal') + }) + + it('leaves non-select columns and unknown option names alone', () => { + expect( + leaf( + resolvePredicateSelectValues( + { all: [{ field: 'col_name', op: 'contains', value: 'Teal' }] }, + COLS + ) + ).value + ).toBe('Teal') + expect( + leaf( + resolvePredicateSelectValues( + { all: [{ field: 'col_color', op: 'contains', value: 'Nope' }] }, + COLS + ) + ).value + ).toBe('Nope') + }) + + it('recurses through nested groups', () => { + const out = resolvePredicateSelectValues( + { any: [{ all: [{ field: 'col_color', op: 'contains', value: 'Green' }] }] }, + COLS + ) as { any: Array<{ all: Array<{ value: unknown }> }> } + expect(out.any[0].all[0].value).toBe('opt_green') + }) + + /** The two grammars must resolve the same operand set, or they drift. */ + it('agrees with the $-grammar sibling on the same filter', () => { + const legacy = resolveFilterSelectValues({ col_color: { $contains: 'Teal' } }, COLS) + expect((legacy.col_color as { $contains: unknown }).$contains).toBe('opt_teal') + expect( + leaf( + resolvePredicateSelectValues( + { all: [{ field: 'col_color', op: 'contains', value: 'Teal' }] }, + COLS + ) + ).value + ).toBe('opt_teal') + }) +}) diff --git a/apps/sim/lib/table/select-values.ts b/apps/sim/lib/table/select-values.ts index f7916e30af4..2957f23889c 100644 --- a/apps/sim/lib/table/select-values.ts +++ b/apps/sim/lib/table/select-values.ts @@ -116,10 +116,14 @@ export function resolveFilterSelectValues(filter: Filter, columns: ColumnDefinit * op:'eq', value:'Open'}`) compares the option NAME against the stored option * ID and silently matches nothing. * - * Only value-carrying comparison ops are resolved. Pattern ops (`contains`, - * `like`, …) are deliberately left alone: they match against the raw stored - * cell, and rewriting their operand to an id would change what the user asked - * for. The valueless ops carry nothing to resolve. + * Mirrors the `$`-grammar set exactly: equality/membership on a single select + * (`eq`/`ne`/`in`/`nin`) AND `contains`/`ncontains`, which on a MULTI-select are + * not pattern matches at all — the cell is an array of option ids, so those ops + * express membership and their operand is an option, not a substring. Leaving + * them out is what made a correctly-formed multi-select filter match nothing. + * The remaining pattern ops (`like`, `startsWith`, …) are rejected on select + * columns by `fieldPredicate`'s allowlist, so they never reach here. The + * valueless ops carry nothing to resolve. */ export function resolvePredicateSelectValues( predicate: TablePredicate, @@ -130,7 +134,7 @@ export function resolvePredicateSelectValues( ) if (selectById.size === 0) return predicate - const RESOLVED_OPS = new Set(['eq', 'ne', 'in', 'nin']) + const RESOLVED_OPS = new Set(['eq', 'ne', 'in', 'nin', 'contains', 'ncontains']) const walk = (node: PredicateNode): PredicateNode => { if ('all' in node) return { all: node.all.map(walk) } From e8858541a9802005202d4025d66f103017628c96 Mon Sep 17 00:00:00 2001 From: Theodore Li Date: Tue, 28 Jul 2026 18:00:52 -0700 Subject: [PATCH 16/26] fix(tools): stop normalizeToolId eating the _v2 version suffix MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit table_query_rows_v2 starts with 'table_query_rows_', so the resource-suffix strip normalized it to the v1 tool. The executor then logged the v2 id while issuing v1's request shape — GET /rows?filter= instead of POST /query with a predicate body — so a correctly configured table_v2 block 400'd with 'Filter looks like a v2 predicate tree but reached the legacy filter compiler'. The guard was right; the tool resolution was wrong. A trailing _v is a version marker, not a resource id, so it is no longer stripped. Versioned ops are matched longest-first and listed alongside their unversioned form, so table_query_rows_v2_ still normalizes to table_query_rows_v2 rather than collapsing to v1. Applies to the knowledge ops too — same loop shape, same latent trap the first time one of them is versioned. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_011M563cbFy2S74GvSDf2C3R --- apps/sim/tools/normalize.test.ts | 40 ++++++++++++++++++++++++++++++++ apps/sim/tools/normalize.ts | 38 ++++++++++++++++++++++-------- 2 files changed, 68 insertions(+), 10 deletions(-) create mode 100644 apps/sim/tools/normalize.test.ts diff --git a/apps/sim/tools/normalize.test.ts b/apps/sim/tools/normalize.test.ts new file mode 100644 index 00000000000..3f2f845f5ea --- /dev/null +++ b/apps/sim/tools/normalize.test.ts @@ -0,0 +1,40 @@ +/** + * @vitest-environment node + */ +import { describe, expect, it } from 'vitest' +import { normalizeToolId } from '@/tools/normalize' + +describe('normalizeToolId', () => { + it('strips a resource-id suffix', () => { + expect(normalizeToolId('table_query_rows_tbl_1a2c1741')).toBe('table_query_rows') + expect(normalizeToolId('knowledge_search_5cc56998-3e1d-4d91')).toBe('knowledge_search') + expect(normalizeToolId('workflow_executor_f3d81b32')).toBe('workflow_executor') + expect(normalizeToolId('deployed_block_executor_custom_block_9')).toBe( + 'deployed_block_executor' + ) + }) + + it('leaves a bare tool id alone', () => { + expect(normalizeToolId('table_query_rows')).toBe('table_query_rows') + expect(normalizeToolId('gmail_send')).toBe('gmail_send') + }) + + /** + * Regression: `table_query_rows_v2` starts with `table_query_rows_`, so the + * resource-suffix strip turned it into the v1 tool. The executor then logged + * the v2 id while issuing v1's `GET /rows?filter=` — which reached + * the legacy filter compiler and 400'd on a correctly configured v2 block. + */ + it('does not mistake a version suffix for a resource id', () => { + expect(normalizeToolId('table_query_rows_v2')).toBe('table_query_rows_v2') + }) + + it('still strips a resource id from a VERSIONED op', () => { + expect(normalizeToolId('table_query_rows_v2_tbl_1a2c1741')).toBe('table_query_rows_v2') + }) + + it('is not fooled by a table id that merely starts with v', () => { + expect(normalizeToolId('table_query_rows_v2x')).toBe('table_query_rows') + expect(normalizeToolId('table_query_rows_version')).toBe('table_query_rows') + }) +}) diff --git a/apps/sim/tools/normalize.ts b/apps/sim/tools/normalize.ts index 177c11aa140..c01dceaf33b 100644 --- a/apps/sim/tools/normalize.ts +++ b/apps/sim/tools/normalize.ts @@ -6,6 +6,29 @@ * * Pure string utility — no server dependencies, safe to import in client components. */ + +/** + * A trailing `_v2`-style segment is a VERSION marker, not a resource id, so it + * must not be stripped: `table_query_rows_v2` is its own registered tool, and + * normalizing it to `table_query_rows` silently executes the v1 tool's request + * shape under the v2 tool's name. + */ +const VERSION_SUFFIX = /^v\d+$/ + +/** + * Longest id first, so a versioned op claims its own suffixed ids before the + * unversioned prefix can match them (`table_query_rows_v2_` must + * normalize to `table_query_rows_v2`, not `table_query_rows`). + */ +function stripResourceSuffix(toolId: string, ops: string[]): string | null { + for (const op of [...ops].sort((a, b) => b.length - a.length)) { + if (!toolId.startsWith(`${op}_`) || toolId.length <= op.length + 1) continue + if (VERSION_SUFFIX.test(toolId.slice(op.length + 1))) continue + return op + } + return null +} + export function normalizeToolId(toolId: string): string { // Custom (deploy-as-block) tools: 'deployed_block_executor_custom_block_' -> // 'deployed_block_executor'. Note the id deliberately does NOT start with @@ -23,14 +46,12 @@ export function normalizeToolId(toolId: string): string { } const knowledgeOps = ['knowledge_search', 'knowledge_upload_chunk', 'knowledge_create_document'] - for (const op of knowledgeOps) { - if (toolId.startsWith(`${op}_`) && toolId.length > op.length + 1) { - return op - } - } + const knowledge = stripResourceSuffix(toolId, knowledgeOps) + if (knowledge) return knowledge const tableOps = [ 'table_query_rows', + 'table_query_rows_v2', 'table_insert_row', 'table_batch_insert_rows', 'table_update_row', @@ -41,11 +62,8 @@ export function normalizeToolId(toolId: string): string { 'table_delete_row', 'table_get_schema', ] - for (const op of tableOps) { - if (toolId.startsWith(`${op}_`) && toolId.length > op.length + 1) { - return op - } - } + const table = stripResourceSuffix(toolId, tableOps) + if (table) return table return toolId } From f2d964f7d290e92c026f1e2327f248b4596299aa Mon Sep 17 00:00:00 2001 From: Theodore Li Date: Tue, 28 Jul 2026 18:16:58 -0700 Subject: [PATCH 17/26] =?UTF-8?q?fix(table):=20make=20name=E2=86=92storage?= =?UTF-8?q?=20predicate=20translation=20one=20operation?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The block's own filter — {"all":[{"field":"Color","op":"contains","value":"Teal"}]} — returned 0 rows against a table where 15 rows hold Teal. Translating a name-keyed predicate to storage keys is two steps: column names → column ids, and select operands → option ids. Both are required, neither is useful alone, but they were two separate calls each boundary had to remember to pair. Three did not: the internal query route (the table_v2 block's own path), the bulk update/delete resolver, and — before this branch — nothing else needed it, so the gap was invisible until select columns landed. Replaced with a single `predicateToStorage(predicate, schema)` and migrated every call site, so the pair cannot be split again. `predicateNamesToIds` now has no direct callers outside it. Also fixes four type errors that a failed inference in contracts/tables.ts was masking — once the leaf schema type-checked, tsc surfaced the rest: - `ColumnType` was imported from lib/table/types but never exported there (it lived as a local alias in sql.ts). Now exported once, next to ColumnDefinition. - rows/service.ts referenced TableRowsCursor in three signatures without importing it. - export-runner and snapshot-cache still declared their paging cursor's orderKey as non-null, after selectExportRowPage was corrected to return the nullable it always had. - the predicate leaf's `z.unknown()` value infers wider than Predicate['value']; narrowed with an annotated cast, runtime unchanged. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_011M563cbFy2S74GvSDf2C3R --- .../app/api/table/[tableId]/query/route.ts | 5 +++-- .../sim/app/api/table/[tableId]/rows/route.ts | 4 ++-- apps/sim/app/api/table/row-wire.ts | 6 ++--- .../api/v2/tables/[tableId]/query/route.ts | 9 +++----- apps/sim/lib/api/contracts/tables.ts | 6 ++++- .../copilot/tools/server/table/user-table.ts | 22 ++++--------------- apps/sim/lib/table/export-runner.ts | 4 +++- apps/sim/lib/table/jobs/service.ts | 4 +++- apps/sim/lib/table/rows/service.ts | 1 + apps/sim/lib/table/select-values.ts | 20 ++++++++++++++++- apps/sim/lib/table/snapshot-cache.ts | 4 +++- apps/sim/lib/table/types.ts | 3 +++ 12 files changed, 51 insertions(+), 37 deletions(-) diff --git a/apps/sim/app/api/table/[tableId]/query/route.ts b/apps/sim/app/api/table/[tableId]/query/route.ts index 6debbf32e4e..896d60d1873 100644 --- a/apps/sim/app/api/table/[tableId]/query/route.ts +++ b/apps/sim/app/api/table/[tableId]/query/route.ts @@ -7,11 +7,12 @@ import { checkSessionOrInternalAuth } from '@/lib/auth/hybrid' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import type { Sort, TableSchema } from '@/lib/table' -import { buildIdByName, predicateNamesToIds, sortSpecNamesToIds } from '@/lib/table/column-keys' +import { buildIdByName, sortSpecNamesToIds } from '@/lib/table/column-keys' import { TableQueryValidationError } from '@/lib/table/errors' import { validatePredicate, validateSortSpec } from '@/lib/table/query-builder/validate' import { decodeCursor } from '@/lib/table/rows/cursor' import { queryRows } from '@/lib/table/rows/service' +import { predicateToStorage } from '@/lib/table/select-values' import { rowWireTranslators } from '@/app/api/table/row-wire' import { accessError, checkAccess, tablesV2GateError } from '@/app/api/table/utils' @@ -82,7 +83,7 @@ export const POST = withRouteHandler(async (request: NextRequest, context: RowQu let predicate = body.predicate if (predicate) { validatePredicate(predicate, schema.columns) - predicate = predicateNamesToIds(predicate, idByName) + predicate = predicateToStorage(predicate, schema) } let sortSpec = body.sort if (sortSpec?.length) { diff --git a/apps/sim/app/api/table/[tableId]/rows/route.ts b/apps/sim/app/api/table/[tableId]/rows/route.ts index 2ec652bd8a8..887f169fdf2 100644 --- a/apps/sim/app/api/table/[tableId]/rows/route.ts +++ b/apps/sim/app/api/table/[tableId]/rows/route.ts @@ -25,11 +25,11 @@ import { validateRowData, validateRowSize, } from '@/lib/table' -import { buildIdByName, predicateNamesToIds } from '@/lib/table/column-keys' import { TableQueryValidationError } from '@/lib/table/errors' import { predicateToFilter } from '@/lib/table/query-builder/converters' import { validatePredicate } from '@/lib/table/query-builder/validate' import { queryRows } from '@/lib/table/rows/service' +import { predicateToStorage } from '@/lib/table/select-values' import type { TablePredicate } from '@/lib/table/types' import { type RowWireTranslators, rowWireTranslators } from '@/app/api/table/row-wire' import { accessError, checkAccess, rowWriteErrorResponse } from '@/app/api/table/utils' @@ -54,7 +54,7 @@ function resolveBulkFilter( ): Filter { if (isTablePredicate(raw)) { validatePredicate(raw, schema.columns) - return predicateToFilter(predicateNamesToIds(raw, buildIdByName(schema))) + return predicateToFilter(predicateToStorage(raw, schema)) } return wire.filterIn(raw) } diff --git a/apps/sim/app/api/table/row-wire.ts b/apps/sim/app/api/table/row-wire.ts index 705240afa49..f4c0a85d988 100644 --- a/apps/sim/app/api/table/row-wire.ts +++ b/apps/sim/app/api/table/row-wire.ts @@ -4,12 +4,11 @@ import { namedRowMapper } from '@/lib/table/cell-format' import { buildIdByName, filterNamesToIds, - predicateNamesToIds, rowDataNameToId, sortNamesToIds, sortSpecNamesToIds, } from '@/lib/table/column-keys' -import { resolveFilterSelectValues, resolvePredicateSelectValues } from '@/lib/table/select-values' +import { predicateToStorage, resolveFilterSelectValues } from '@/lib/table/select-values' export interface RowWireTranslators { /** Inbound row data: wire keys → storage column ids. */ @@ -58,8 +57,7 @@ export function rowWireTranslators( filterIn: (filter) => resolveFilterSelectValues(filterNamesToIds(filter, idByName), schema.columns), sortIn: (sort) => sortNamesToIds(sort, idByName), - predicateIn: (predicate) => - resolvePredicateSelectValues(predicateNamesToIds(predicate, idByName), schema.columns), + predicateIn: (predicate) => predicateToStorage(predicate, schema), sortSpecIn: (sort) => sortSpecNamesToIds(sort, idByName), } } diff --git a/apps/sim/app/api/v2/tables/[tableId]/query/route.ts b/apps/sim/app/api/v2/tables/[tableId]/query/route.ts index 67e8c4ccfc0..1ddcfcb27e4 100644 --- a/apps/sim/app/api/v2/tables/[tableId]/query/route.ts +++ b/apps/sim/app/api/v2/tables/[tableId]/query/route.ts @@ -6,13 +6,13 @@ import { parseRequest, validationErrorResponseFromError } from '@/lib/api/server import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import type { Sort, TablePredicate, TableSchema } from '@/lib/table' -import { buildIdByName, predicateNamesToIds, sortSpecNamesToIds } from '@/lib/table' +import { buildIdByName, sortSpecNamesToIds } from '@/lib/table' import { namedRowMapper } from '@/lib/table/cell-format' import { TableQueryValidationError } from '@/lib/table/errors' import { validatePredicate, validateSortSpec } from '@/lib/table/query-builder/validate' import { decodeCursor } from '@/lib/table/rows/cursor' import { queryRows } from '@/lib/table/rows/service' -import { resolvePredicateSelectValues } from '@/lib/table/select-values' +import { predicateToStorage } from '@/lib/table/select-values' import { accessError, checkAccess, tablesV2GateError } from '@/app/api/table/utils' import { checkRateLimit, @@ -91,10 +91,7 @@ export const POST = withRouteHandler(async (request: NextRequest, context: Query let predicate: TablePredicate | undefined = parsed.data.body.predicate if (predicate) { validatePredicate(predicate, schema.columns) - predicate = resolvePredicateSelectValues( - predicateNamesToIds(predicate, idByName), - schema.columns - ) + predicate = predicateToStorage(predicate, schema) } let sortSpec = sort if (sortSpec?.length) { diff --git a/apps/sim/lib/api/contracts/tables.ts b/apps/sim/lib/api/contracts/tables.ts index c3289f5a3a3..1100dfd608c 100644 --- a/apps/sim/lib/api/contracts/tables.ts +++ b/apps/sim/lib/api/contracts/tables.ts @@ -6,6 +6,7 @@ import type { CsvHeaderMapping, EnrichmentRunDetail, Filter, + Predicate, PredicateNode, RowData, Sort, @@ -429,11 +430,14 @@ function predicateTreeTooLarge(root: unknown): string | null { * before it ran. Strict on BOTH branches is required: strict on the group alone * would just fall through to the leaf branch, which is the more dangerous reading. */ +// double-cast-allowed: `z.unknown()` keeps the runtime permissive (a leaf value +// is arbitrary JSON), but infers `unknown`, which is wider than +// `Predicate['value']`. The narrowing is type-level only — nothing is coerced. const predicateLeafSchema = z.strictObject({ field: z.string().min(1, 'field is required').max(128), op: z.enum(FILTER_OPS), value: z.unknown().optional(), -}) +}) as unknown as z.ZodType const predicateNodeSchema: z.ZodType = z.lazy(() => z.union([predicateGroupSchema, predicateLeafSchema]) diff --git a/apps/sim/lib/copilot/tools/server/table/user-table.ts b/apps/sim/lib/copilot/tools/server/table/user-table.ts index 653abd31ec1..13517e0dd46 100644 --- a/apps/sim/lib/copilot/tools/server/table/user-table.ts +++ b/apps/sim/lib/copilot/tools/server/table/user-table.ts @@ -30,7 +30,6 @@ import { namedRowMapper } from '@/lib/table/cell-format' import { buildIdByName, columnMatchesRef, - predicateNamesToIds, rowDataNameToId, sortSpecNamesToIds, } from '@/lib/table/column-keys' @@ -64,7 +63,7 @@ import { updateRow, updateRowsByFilter, } from '@/lib/table/rows/service' -import { resolvePredicateSelectValues } from '@/lib/table/select-values' +import { predicateToStorage } from '@/lib/table/select-values' import { createTable, deleteTable, getTableById, renameTable } from '@/lib/table/service' import type { ColumnDefinition, @@ -677,10 +676,7 @@ export const userTableServerTool: BaseServerTool let predicate: TablePredicate | undefined if (args.filter) { validatePredicate(args.filter, table.schema.columns) - predicate = resolvePredicateSelectValues( - predicateNamesToIds(args.filter, idByName), - table.schema.columns - ) + predicate = predicateToStorage(args.filter, table.schema) } let orderSpec = args.order as SortSpec | undefined if (orderSpec?.length) { @@ -859,12 +855,7 @@ export const userTableServerTool: BaseServerTool // the bulk engine (same fieldPredicate leaf → identical SQL). Select // operands arrive as option NAMES and must resolve to stored ids. validatePredicate(args.filter, table.schema.columns) - const idFilter = predicateToFilter( - resolvePredicateSelectValues( - predicateNamesToIds(args.filter, idByName), - table.schema.columns - ) - ) + const idFilter = predicateToFilter(predicateToStorage(args.filter, table.schema)) const idData = rowDataNameToId(args.data, idByName) // Inline handles up to MAX_BULK_OPERATION_SIZE rows in one request; a larger operation @@ -966,12 +957,7 @@ export const userTableServerTool: BaseServerTool // the bulk engine (same fieldPredicate leaf → identical SQL). Select // operands arrive as option NAMES and must resolve to stored ids. validatePredicate(args.filter, table.schema.columns) - const idFilter = predicateToFilter( - resolvePredicateSelectValues( - predicateNamesToIds(args.filter, idByName), - table.schema.columns - ) - ) + const idFilter = predicateToFilter(predicateToStorage(args.filter, table.schema)) // Inline handles up to MAX_BULK_OPERATION_SIZE rows; a larger delete (an explicit limit // above the cap, or unbounded "delete everything matching") hands off to the background diff --git a/apps/sim/lib/table/export-runner.ts b/apps/sim/lib/table/export-runner.ts index 8af3628194c..e8a9cf5fb54 100644 --- a/apps/sim/lib/table/export-runner.ts +++ b/apps/sim/lib/table/export-runner.ts @@ -80,7 +80,9 @@ export async function runTableExport(payload: TableExportPayload): Promise let exported = 0 let firstJsonRow = true - let after: { orderKey: string; id: string } | null = null + // `order_key` is nullable (rows predating the backfill), and the page query + // seeks NULLs explicitly — so the cursor has to carry a null too. + let after: { orderKey: string | null; id: string } | null = null while (true) { // Ownership gate before every page: a canceled job stops within one batch. const owns = await updateJobProgress(tableId, exported, jobId) diff --git a/apps/sim/lib/table/jobs/service.ts b/apps/sim/lib/table/jobs/service.ts index dcf09c65507..ca916e689b8 100644 --- a/apps/sim/lib/table/jobs/service.ts +++ b/apps/sim/lib/table/jobs/service.ts @@ -242,7 +242,9 @@ export async function selectExportRowPage( ) .orderBy(asc(userTableRows.orderKey), asc(userTableRows.id)) .limit(limit) - return rows + // drizzle types a jsonb column as `unknown`; every writer goes through the + // row-data validators, so narrowing here is a projection, not an assumption. + return rows.map((r) => ({ ...r, data: r.data as RowData })) } /** How long a terminal export stays listable (and re-downloadable from the tray). */ diff --git a/apps/sim/lib/table/rows/service.ts b/apps/sim/lib/table/rows/service.ts index 4938d8fa085..6e8df7c4840 100644 --- a/apps/sim/lib/table/rows/service.ts +++ b/apps/sim/lib/table/rows/service.ts @@ -87,6 +87,7 @@ import type { TableDefinition, TableDeleteJobPayload, TableRow, + TableRowsCursor, UpdateRowData, UpsertResult, UpsertRowData, diff --git a/apps/sim/lib/table/select-values.ts b/apps/sim/lib/table/select-values.ts index 2957f23889c..c120a989ed7 100644 --- a/apps/sim/lib/table/select-values.ts +++ b/apps/sim/lib/table/select-values.ts @@ -9,7 +9,7 @@ * both the legacy `$` grammar and the v2 predicate tree. */ -import { getColumnId } from '@/lib/table/column-keys' +import { buildIdByName, getColumnId, predicateNamesToIds } from '@/lib/table/column-keys' import type { ColumnDefinition, ConditionOperators, @@ -19,6 +19,7 @@ import type { Predicate, PredicateNode, TablePredicate, + TableSchema, } from '@/lib/table/types' import { resolveSelectOptionId } from '@/lib/table/validation' @@ -153,3 +154,20 @@ export function resolvePredicateSelectValues( return walk(predicate) as TablePredicate } + +/** + * The complete name-keyed → storage-keyed translation for a v2 predicate: + * column names become column ids AND select operands become option ids. + * + * Both halves are required and neither is useful alone, but they lived as two + * separate calls that every boundary had to remember to pair — and three of them + * did not, so a filter on a select column compared an option NAME against a + * stored option ID and returned zero rows while reporting success. Call this + * instead of `predicateNamesToIds` so the pair cannot be split again. + */ +export function predicateToStorage(predicate: TablePredicate, schema: TableSchema): TablePredicate { + return resolvePredicateSelectValues( + predicateNamesToIds(predicate, buildIdByName(schema)), + schema.columns + ) +} diff --git a/apps/sim/lib/table/snapshot-cache.ts b/apps/sim/lib/table/snapshot-cache.ts index 0c24eccfb39..e99869b4adf 100644 --- a/apps/sim/lib/table/snapshot-cache.ts +++ b/apps/sim/lib/table/snapshot-cache.ts @@ -103,7 +103,9 @@ async function materialize(table: TableDefinition, key: string): Promise bytes += Buffer.byteLength(header) await handle.write(header) - let after: { orderKey: string; id: string } | null = null + // `order_key` is nullable (rows predating the backfill), and the page query + // seeks NULLs explicitly — so the cursor has to carry a null too. + let after: { orderKey: string | null; id: string } | null = null while (true) { const page = await selectExportRowPage(table, after, SNAPSHOT_BATCH_SIZE) if (page.length === 0) break diff --git a/apps/sim/lib/table/types.ts b/apps/sim/lib/table/types.ts index ec80debfa3d..a8322ee5aba 100644 --- a/apps/sim/lib/table/types.ts +++ b/apps/sim/lib/table/types.ts @@ -63,6 +63,9 @@ export interface ColumnDefinition { multiple?: boolean } +/** The column `type` discriminator, named so callers don't index into the interface. */ +export type ColumnType = ColumnDefinition['type'] + /** One group output → one plain column. */ export interface WorkflowGroupOutput { /** Source block id within the configured workflow. `''` for enrichment groups. */ From e268562d9ae7b87adaa29f4cfe51e46afd5c5054 Mon Sep 17 00:00:00 2001 From: Theodore Li Date: Wed, 29 Jul 2026 12:33:32 -0700 Subject: [PATCH 18/26] fix(merge): restore staging's env-flags exports dropped by the merge commit The 79d0dd0408 merge recorded the pre-merge env-flags.ts: the path was reset out of the index mid-merge to keep a local debug edit unstaged, which also discarded staging's isSessionPoliciesEnabled / isCopilotToolPermissionsEnabled exports and broke six importers added by the desktop-app PR (#5998). Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_011M563cbFy2S74GvSDf2C3R --- apps/sim/lib/core/config/env-flags.ts | 173 +++++++++++++++++++++----- 1 file changed, 139 insertions(+), 34 deletions(-) diff --git a/apps/sim/lib/core/config/env-flags.ts b/apps/sim/lib/core/config/env-flags.ts index 23960cea260..115242cbfbc 100644 --- a/apps/sim/lib/core/config/env-flags.ts +++ b/apps/sim/lib/core/config/env-flags.ts @@ -1,7 +1,12 @@ /** * Environment utility functions for consistent environment detection across the application */ -import { env, getEnv, isFalsy, isTruthy } from './env' +import { + ENTERPRISE_FEATURE_LEGACY_DEFAULTS, + type EnterpriseFeature, + resolveEnterpriseEntitlement, +} from './enterprise-entitlements' +import { env, envBoolean, getEnv, isFalsy, isTruthy } from './env' /** * Is the application running in production mode @@ -45,6 +50,18 @@ export const isCopilotBillingAttributionV1Enabled = isTruthy( */ export const isCopilotBillingProtocolRequired = isTruthy(env.COPILOT_BILLING_PROTOCOL_REQUIRED) +/** + * Holds tools the catalog marks `requiresApproval` — shell commands, workflow + * runs, sandboxed code, deployments, integration calls — behind an explicit + * Allow / Skip prompt, blocking the mothership turn until the user answers. + * + * Off by default: turning it on makes the copilot prompt on its most frequently + * used tools, so it is an opt-in change in how the product feels, not just a + * safety toggle. With it off nothing is stamped, gated, or persisted, and an + * approval stamp arriving from Go is cleared on the way to the client. + */ +export const isCopilotToolPermissionsEnabled = isTruthy(env.COPILOT_TOOL_PERMISSIONS_ENABLED) + /** * Is billing enforcement enabled. * @@ -172,26 +189,78 @@ export const isSlackExtendedScopesEnabled = export const isTriggerDevEnabled = isTruthy(env.TRIGGER_DEV_ENABLED) /** - * Is SSO enabled for enterprise authentication + * Turns on the whole enterprise suite for a deployment that does not run + * billing. Individual feature flags below still win where they are set, so an + * operator can enable everything and then switch one feature back off. + * + * Server code reads `ENTERPRISE_ENABLED`; the browser reads the + * `NEXT_PUBLIC_ENTERPRISE_ENABLED` twin (see {@link isBillingEnabled}). + * Deployments must set both together. + */ +export const isEnterpriseEnabled = + typeof window === 'undefined' + ? isTruthy(env.ENTERPRISE_ENABLED) + : isTruthy(getEnv('NEXT_PUBLIC_ENTERPRISE_ENABLED')) + +/** + * Reads a feature's own flag as a tri-state, picking the server var or its + * browser twin for the current runtime. `undefined` means the operator left it + * unset, which is what lets the master switch and legacy default apply. */ -export const isSsoEnabled = isTruthy(env.SSO_ENABLED) +function explicitEnterpriseFlag( + serverValue: boolean | string | undefined, + clientKey: string +): boolean | undefined { + return typeof window === 'undefined' ? envBoolean(serverValue) : envBoolean(getEnv(clientKey)) +} /** - * Is access control (permission groups) enabled via env var override. - * This bypasses plan requirements for self-hosted deployments. + * Resolves one enterprise feature for this deployment. * - * Server code reads `ACCESS_CONTROL_ENABLED`; the browser reads the - * `NEXT_PUBLIC_ACCESS_CONTROL_ENABLED` twin (see {@link isBillingEnabled}). + * When billing runs, subscription plans decide entitlement and these flags are + * only explicit overrides — so an unset flag stays `false` and never widens + * access on Sim Cloud. When billing is off there is no plan to consult, so + * resolution falls through the master switch to the feature's legacy default + * (see {@link ENTERPRISE_FEATURE_LEGACY_DEFAULTS}). + */ +function enterpriseFeatureEnabled( + feature: EnterpriseFeature, + serverValue: boolean | string | undefined, + clientKey: string +): boolean { + const explicit = explicitEnterpriseFlag(serverValue, clientKey) + if (isBillingEnabled) return explicit ?? false + return resolveEnterpriseEntitlement({ + explicit, + masterEnabled: isEnterpriseEnabled, + legacyDefault: ENTERPRISE_FEATURE_LEGACY_DEFAULTS[feature], + }) +} + +/** + * Is SSO enabled for enterprise authentication */ -export const isAccessControlEnabled = - typeof window === 'undefined' - ? isTruthy(env.ACCESS_CONTROL_ENABLED) - : isTruthy(getEnv('NEXT_PUBLIC_ACCESS_CONTROL_ENABLED')) +export const isSsoEnabled = enterpriseFeatureEnabled( + 'sso', + env.SSO_ENABLED, + 'NEXT_PUBLIC_SSO_ENABLED' +) + +/** + * Is access control (permission groups) enabled. + * Required for permission-group enforcement to run at all off-hosted. + */ +export const isAccessControlEnabled = enterpriseFeatureEnabled( + 'accessControl', + env.ACCESS_CONTROL_ENABLED, + 'NEXT_PUBLIC_ACCESS_CONTROL_ENABLED' +) /** * Is organizations enabled. - * True if billing is enabled (orgs come with billing), OR explicitly enabled via env var, - * OR if access control is enabled (access control requires organizations). + * True if billing is enabled (orgs come with billing), OR resolved on for this + * deployment, OR if access control is enabled (access control requires + * organizations). * * Each term resolves through its `NEXT_PUBLIC_*` twin in the browser (see * {@link isBillingEnabled}), so client code — e.g. the better-auth @@ -199,46 +268,82 @@ export const isAccessControlEnabled = */ export const isOrganizationsEnabled = isBillingEnabled || - (typeof window === 'undefined' - ? isTruthy(env.ORGANIZATIONS_ENABLED) - : isTruthy(getEnv('NEXT_PUBLIC_ORGANIZATIONS_ENABLED'))) || + enterpriseFeatureEnabled( + 'organizations', + env.ORGANIZATIONS_ENABLED, + 'NEXT_PUBLIC_ORGANIZATIONS_ENABLED' + ) || isAccessControlEnabled /** - * Is inbox (Sim Mailer) enabled via env var override - * This bypasses hosted requirements for self-hosted deployments + * Is inbox (Sim Mailer) enabled */ -export const isInboxEnabled = isTruthy(env.INBOX_ENABLED) +export const isInboxEnabled = enterpriseFeatureEnabled( + 'inbox', + env.INBOX_ENABLED, + 'NEXT_PUBLIC_INBOX_ENABLED' +) /** - * Is whitelabeling enabled via env var override - * This bypasses hosted requirements for self-hosted deployments + * Is whitelabeling enabled */ -export const isWhitelabelingEnabled = isTruthy(env.WHITELABELING_ENABLED) +export const isWhitelabelingEnabled = enterpriseFeatureEnabled( + 'whitelabeling', + env.WHITELABELING_ENABLED, + 'NEXT_PUBLIC_WHITELABELING_ENABLED' +) /** - * Is audit logs enabled via env var override - * This bypasses hosted requirements for self-hosted deployments + * Is audit log reading enabled. + * + * Off-hosted this replaces the enterprise-subscription check that audit access + * used to require, which no billing-free deployment could ever satisfy. */ -export const isAuditLogsEnabled = isTruthy(env.AUDIT_LOGS_ENABLED) +export const isAuditLogsEnabled = enterpriseFeatureEnabled( + 'auditLogs', + env.AUDIT_LOGS_ENABLED, + 'NEXT_PUBLIC_AUDIT_LOGS_ENABLED' +) + +/** + * Is retention *deletion* enabled. + * + * Configuring retention has always been possible with billing off; this flag + * governs whether the cleanup pass actually expires data. Opt-in on purpose — + * see the note on `dataRetention` in {@link ENTERPRISE_FEATURE_LEGACY_DEFAULTS}. + */ +export const isDataRetentionEnabled = enterpriseFeatureEnabled( + 'dataRetention', + env.DATA_RETENTION_ENABLED, + 'NEXT_PUBLIC_DATA_RETENTION_ENABLED' +) /** - * Is data retention enabled via env var override - * This bypasses hosted requirements for self-hosted deployments + * Is data drains enabled */ -export const isDataRetentionEnabled = isTruthy(env.DATA_RETENTION_ENABLED) +export const isDataDrainsEnabled = enterpriseFeatureEnabled( + 'dataDrains', + env.DATA_DRAINS_ENABLED, + 'NEXT_PUBLIC_DATA_DRAINS_ENABLED' +) /** - * Is data drains enabled via env var override - * This bypasses hosted requirements for self-hosted deployments + * Are organization session policies enabled */ -export const isDataDrainsEnabled = isTruthy(env.DATA_DRAINS_ENABLED) +export const isSessionPoliciesEnabled = enterpriseFeatureEnabled( + 'sessionPolicies', + env.SESSION_POLICIES_ENABLED, + 'NEXT_PUBLIC_SESSION_POLICIES_ENABLED' +) /** - * Is workspace forking enabled via env var override - * This bypasses hosted (Enterprise) requirements for self-hosted deployments + * Is workspace forking enabled */ -export const isForkingEnabled = isTruthy(env.FORKING_ENABLED) +export const isForkingEnabled = enterpriseFeatureEnabled( + 'forking', + env.FORKING_ENABLED, + 'NEXT_PUBLIC_FORKING_ENABLED' +) /** * The selected remote sandbox provider (`SANDBOX_PROVIDER`), defaulting to E2B. From 1d5020854097bb4e6f9fb8ae4e312477352feaeb Mon Sep 17 00:00:00 2001 From: Theodore Li Date: Wed, 29 Jul 2026 12:37:42 -0700 Subject: [PATCH 19/26] refactor(table): store saved views in the v2 predicate grammar MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Views shipped (#5961) storing the legacy `$`-object filter and `{col: dir}` sort record — a brand-new persistent store of the grammar this branch is retiring, created days before the wire moved to predicates. The feature is still dark (`table-views` is UI-only and off), so the stored shape can change now without a data migration; once the flag flips, it cannot. `TableViewConfig` now carries `TablePredicate` + `SortSpec`. The wire contract uses `predicateSchema`/`sortSpecSchema`, which also brings the strict-object node shapes and depth/size bounds to the view routes — previously a view's filter was accepted as an arbitrary domain object. The grid still runs on the legacy pair internally; translation happens at the view boundary. Apply: `predicateToFilter` (total here — stored predicates are builder-authored). Save: `filterToRules ∘ filterRulesToPredicate`, the builder round-trip. SortSpec keeps priority order the record never could. Dev-era rows written before the switch are normalized on read: legacy filters convert through the builder round-trip and are dropped if the result's leaf fields fail the column-name pattern — the rule converters accept garbage (`{$bogus: …}` becomes a rule on a column literally named `$bogus`), so the conversion is validated rather than trusted. Also folds `sortQuery`'s single-entry record out of the save path in favour of the sort params directly, so a saved view records the same thing the URL says. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_011M563cbFy2S74GvSDf2C3R --- .../[workspaceId]/tables/[tableId]/table.tsx | 32 +++++++--- apps/sim/lib/api/contracts/tables.ts | 7 ++- apps/sim/lib/table/types.ts | 4 +- apps/sim/lib/table/views/service.test.ts | 42 ++++++++++++-- apps/sim/lib/table/views/service.ts | 58 ++++++++++++++++--- 5 files changed, 121 insertions(+), 22 deletions(-) diff --git a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/table.tsx b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/table.tsx index 0540b5346b5..e8f20a55d4b 100644 --- a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/table.tsx +++ b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/table.tsx @@ -22,6 +22,11 @@ import type { } from '@/lib/table' import { getColumnId } from '@/lib/table/column-keys' import { TABLE_LIMITS } from '@/lib/table/constants' +import { + filterRulesToPredicate, + filterToRules, + predicateToFilter, +} from '@/lib/table/query-builder/converters' import { type BreadcrumbItem, type ColumnOption, @@ -423,13 +428,17 @@ export function Table({ config: TableViewConfig | null, keep?: { sort?: boolean; filter?: boolean; hiddenColumns?: boolean } ) => { - if (!keep?.filter) setFilter(config?.filter ?? null) + // Stored views speak the v2 grammar; the grid's runtime state is still the + // legacy Filter/Sort pair, so translate at this boundary. A stored predicate + // is always builder-authored (the save path converts from builder output), + // so the legacy projection is total here. + if (!keep?.filter) setFilter(config?.filter ? predicateToFilter(config.filter) : null) if (!keep?.hiddenColumns) setHiddenColumns(config?.hiddenColumns ?? []) if (keep?.sort) return - const sortEntry = config?.sort ? Object.entries(config.sort)[0] : undefined + const sortEntry = config?.sort?.[0] setTableParams({ - sort: sortEntry ? sortEntry[0] : null, - dir: sortEntry ? (sortEntry[1] as SortDirection) : null, + sort: sortEntry ? sortEntry.field : null, + dir: sortEntry ? (sortEntry.direction as SortDirection) : null, }) }, [setTableParams] @@ -651,11 +660,20 @@ export function Table({ const currentViewConfig = useMemo( () => ({ ...(activeView?.config ?? tableData?.metadata), - filter: effectiveFilter, - sort: sortQuery, + // Views store the v2 grammar; the grid runs on the legacy pair. The filter + // is builder-authored, so the rule round-trip is lossless here. + filter: effectiveFilter ? filterRulesToPredicate(filterToRules(effectiveFilter)) : null, + sort: sortColumn ? [{ field: sortColumn, direction: sortDirection }] : null, hiddenColumns: effectiveHiddenColumns, }), - [activeView, tableData?.metadata, effectiveFilter, sortQuery, effectiveHiddenColumns] + [ + activeView, + tableData?.metadata, + effectiveFilter, + sortColumn, + sortDirection, + effectiveHiddenColumns, + ] ) /** diff --git a/apps/sim/lib/api/contracts/tables.ts b/apps/sim/lib/api/contracts/tables.ts index cb18201b6b4..89cb5d78727 100644 --- a/apps/sim/lib/api/contracts/tables.ts +++ b/apps/sim/lib/api/contracts/tables.ts @@ -1717,8 +1717,11 @@ export const tableEventStreamContract = defineRouteContract({ * never invalidates a view. */ export const tableViewConfigSchema = tableMetadataSchema.extend({ - filter: filterSchema.nullable().optional(), - sort: domainObjectSchema().nullable().optional(), + // The v2 predicate/sort grammar — same wire as the query routes, so a saved + // view gets the same strictness and depth bounds as a live filter, and its + // config can later feed the v2 surfaces without conversion. + filter: predicateSchema.nullable().optional(), + sort: sortSpecSchema.nullable().optional(), }) satisfies z.ZodType export const tableViewSchema = z.object({ diff --git a/apps/sim/lib/table/types.ts b/apps/sim/lib/table/types.ts index 01f17465527..1c6b5e39913 100644 --- a/apps/sim/lib/table/types.ts +++ b/apps/sim/lib/table/types.ts @@ -286,8 +286,8 @@ export interface TableMetadata { * user resizes, reorders, pins, or hides columns. */ export interface TableViewConfig extends TableMetadata { - filter?: Filter | null - sort?: Sort | null + filter?: TablePredicate | null + sort?: SortSpec | null } /** Async background-job lifecycle state for a table. NULL/undefined = idle (no job). */ diff --git a/apps/sim/lib/table/views/service.test.ts b/apps/sim/lib/table/views/service.test.ts index 5b048954313..3405f73bd90 100644 --- a/apps/sim/lib/table/views/service.test.ts +++ b/apps/sim/lib/table/views/service.test.ts @@ -3,7 +3,7 @@ */ import { describe, expect, it } from 'vitest' import type { ColumnDefinition, TableViewConfig } from '@/lib/table/types' -import { pruneViewConfig } from '@/lib/table/views/service' +import { normalizeStoredViewConfig, pruneViewConfig } from '@/lib/table/views/service' const columns: ColumnDefinition[] = [ { id: 'col_a', name: 'Name', type: 'text' }, @@ -28,14 +28,18 @@ describe('pruneViewConfig', () => { }) it('drops a sort on a deleted column and collapses to null when none remain', () => { - expect(pruneViewConfig({ sort: { col_gone: 'asc' } }, columns).sort).toBeNull() - expect(pruneViewConfig({ sort: { col_a: 'desc' } }, columns).sort).toEqual({ col_a: 'desc' }) + expect( + pruneViewConfig({ sort: [{ field: 'col_gone', direction: 'asc' }] }, columns).sort + ).toBeNull() + expect( + pruneViewConfig({ sort: [{ field: 'col_a', direction: 'desc' }] }, columns).sort + ).toEqual([{ field: 'col_a', direction: 'desc' }]) }) it('leaves the filter untouched even when it references a deleted column', () => { // Pruning a predicate would silently widen the view's row set — surfacing a // stale condition the user can see and remove is the safer failure. - const filter = { col_gone: { $eq: 'x' } } + const filter = { all: [{ field: 'col_gone', op: 'eq' as const, value: 'x' }] } expect(pruneViewConfig({ filter }, columns).filter).toEqual(filter) }) @@ -50,3 +54,33 @@ describe('pruneViewConfig', () => { ]) }) }) + +/** + * Reads written before the grammar switch: the feature never released, so + * legacy-shaped configs exist only from pre-refactor testing — but they must + * come back as v2, not render broken. + */ +describe('normalizeStoredViewConfig', () => { + it('converts a legacy $-object filter to a predicate tree', () => { + const out = normalizeStoredViewConfig({ filter: { col_a: { $eq: 'x' } } }) + expect(out.filter).toEqual({ all: [{ field: 'col_a', op: 'eq', value: 'x' }] }) + }) + + it('converts a legacy {col: dir} sort record to an ordered spec', () => { + const out = normalizeStoredViewConfig({ sort: { col_a: 'desc' } }) + expect(out.sort).toEqual([{ field: 'col_a', direction: 'desc' }]) + }) + + it('passes v2-shaped configs through untouched', () => { + const config = { + filter: { all: [{ field: 'col_a', op: 'eq', value: 'x' }] }, + sort: [{ field: 'col_a', direction: 'asc' }], + } + expect(normalizeStoredViewConfig(config)).toEqual(config) + }) + + it('drops an unconvertible legacy filter rather than surfacing it broken', () => { + const out = normalizeStoredViewConfig({ filter: { $bogus: [{ nested: true }] } }) + expect(out.filter).toBeNull() + }) +}) diff --git a/apps/sim/lib/table/views/service.ts b/apps/sim/lib/table/views/service.ts index 1b575bb618a..1c8684a414c 100644 --- a/apps/sim/lib/table/views/service.ts +++ b/apps/sim/lib/table/views/service.ts @@ -15,7 +15,15 @@ import { createLogger } from '@sim/logger' import { generateId } from '@sim/utils/id' import { and, asc, eq, ne, sql } from 'drizzle-orm' import { getColumnId } from '@/lib/table/column-keys' -import type { ColumnDefinition, TableViewConfig } from '@/lib/table/types' +import { NAME_PATTERN } from '@/lib/table/constants' +import { filterRulesToPredicate, filterToRules } from '@/lib/table/query-builder/converters' +import type { + ColumnDefinition, + Filter, + Predicate, + PredicateNode, + TableViewConfig, +} from '@/lib/table/types' const logger = createLogger('TableViewsService') @@ -69,22 +77,58 @@ export function pruneViewConfig( pruned.columnWidths = widths } if (config.sort) { - const sort: Record = {} - for (const [id, direction] of Object.entries(config.sort)) { - if (live.has(id)) sort[id] = direction - } - pruned.sort = Object.keys(sort).length > 0 ? sort : null + const sort = config.sort.filter((s) => live.has(s.field)) + pruned.sort = sort.length > 0 ? sort : null } return pruned } +/** + * Migrates a config stored before the grammar switch. The feature never + * released, so legacy-shaped rows exist only from pre-refactor testing: a + * `$`-object filter converts through the builder-rule round-trip (its exact + * authoring domain), and a `{col: dir}` sort record becomes an ordered spec. + * Anything unconvertible is dropped rather than surfaced broken. + */ + +/** Every leaf field in the tree is a plausible column id. */ +function predicateFieldsAreValid(node: PredicateNode): boolean { + if ('all' in node) return node.all.every(predicateFieldsAreValid) + if ('any' in node) return node.any.every(predicateFieldsAreValid) + return NAME_PATTERN.test((node as Predicate).field) +} + +export function normalizeStoredViewConfig(raw: Record): TableViewConfig { + const config = { ...raw } as TableViewConfig + const filter = raw.filter as Record | null | undefined + if (filter && !('all' in filter) && !('any' in filter)) { + try { + const converted = filterRulesToPredicate(filterToRules(filter as Filter)) + // The rule converters don't reject garbage — an unknown `$op` becomes a + // rule on a column literally named `$op`. A converted leaf whose field + // fails the column-name pattern proves the input wasn't builder-authored. + config.filter = converted && predicateFieldsAreValid(converted) ? converted : null + } catch { + config.filter = null + } + } + const sort = raw.sort as Record | unknown[] | null | undefined + if (sort && !Array.isArray(sort)) { + config.sort = Object.entries(sort).map(([field, direction]) => ({ field, direction })) + } + return config +} + function toTableView(row: typeof tableViews.$inferSelect, columns: ColumnDefinition[]): TableView { return { id: row.id, tableId: row.tableId, name: row.name, - config: pruneViewConfig((row.config ?? {}) as TableViewConfig, columns), + config: pruneViewConfig( + normalizeStoredViewConfig((row.config ?? {}) as Record), + columns + ), isDefault: row.isDefault, createdBy: row.createdBy, createdAt: row.createdAt, From 2546a0915046527f81684b4d8164f2c4463e6244 Mon Sep 17 00:00:00 2001 From: Theodore Li Date: Wed, 29 Jul 2026 15:52:08 -0700 Subject: [PATCH 20/26] refactor(table): move the grid onto the v2 predicate grammar (Track C) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The grid was the last surface authoring the legacy `$`-grammar, which forced views to downgrade on apply and upgrade on save, and kept five wire fields legacy-only. Its runtime state, filter bar, and every request it makes now speak `TablePredicate`/`SortSpec`. Wire: the five grid-carrying fields (rows GET filter+sort, find filter+sort, delete-async, cancel-runs, columns-run) accept a dual-grammar union — strict predicate tree first, legacy fallback — so external v1 callers are untouched. The rows read path takes predicates NATIVELY into queryRows (no downgrade); the job/dispatch routes downgrade via predicateToFilter at entry, which throws on any leaf the legacy compiler would silently discard, so persisted job payloads stay legacy and the runners are untouched. Grid: filter state is TablePredicate, the filter bar converts rules with filterRulesToPredicate — now select-aware (a numeric-looking option id is no longer scalar-coerced, matching filterRulesToFilter) — and stale-operator pruning uses a new prunePredicateForColumns that fails CLOSED to "no filter" on malformed values instead of taking the page down. The view apply/save boundary conversions added earlier are deleted: views and grid now share one grammar end to end. isTablePredicate moved from a route-local into converters as the shared dual-wire discriminator; toLegacyFilter/toLegacySort live there too (pure grammar code — keeping them in app/api/table/utils broke every test that wholesale-mocks that module). Legacy converters now have exactly one live consumer: the v1 table block, whose tools still speak the $-wire by contract. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_011M563cbFy2S74GvSDf2C3R --- .../api/table/[tableId]/cancel-runs/route.ts | 6 +- .../api/table/[tableId]/columns/run/route.ts | 15 +++- .../api/table/[tableId]/delete-async/route.ts | 6 +- .../api/table/[tableId]/rows/find/route.ts | 10 ++- .../api/table/[tableId]/rows/route.test.ts | 36 +++++++++ .../sim/app/api/table/[tableId]/rows/route.ts | 21 +++-- .../components/table-filter/table-filter.tsx | 15 ++-- .../components/table-grid/table-grid.tsx | 10 +-- .../tables/[tableId]/hooks/use-table.test.ts | 2 +- .../tables/[tableId]/hooks/use-table.ts | 8 +- .../[workspaceId]/tables/[tableId]/table.tsx | 53 +++++------- .../[workspaceId]/tables/[tableId]/types.ts | 6 +- apps/sim/hooks/queries/tables.ts | 32 +++++--- apps/sim/lib/api/contracts/tables.ts | 17 ++-- .../__tests__/converters.test.ts | 81 +++++++++++++++++++ apps/sim/lib/table/query-builder/constants.ts | 3 +- .../sim/lib/table/query-builder/converters.ts | 68 +++++++++++++++- 17 files changed, 300 insertions(+), 89 deletions(-) diff --git a/apps/sim/app/api/table/[tableId]/cancel-runs/route.ts b/apps/sim/app/api/table/[tableId]/cancel-runs/route.ts index ce656d6be50..7e9d332a1e0 100644 --- a/apps/sim/app/api/table/[tableId]/cancel-runs/route.ts +++ b/apps/sim/app/api/table/[tableId]/cancel-runs/route.ts @@ -5,6 +5,7 @@ import { parseRequest } from '@/lib/api/server' import { checkSessionOrInternalAuth } from '@/lib/auth/hybrid' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' +import { toLegacyFilter } from '@/lib/table/query-builder/converters' import { cancelWorkflowGroupRuns } from '@/lib/table/workflow-columns' import { accessError, checkAccess, tableFilterError } from '@/app/api/table/utils' @@ -32,7 +33,10 @@ export const POST = withRouteHandler(async (request: NextRequest, { params }: Ro const parsed = await parseRequest(cancelTableRunsContract, request, { params }) if (!parsed.success) return parsed.response const { tableId } = parsed.data.params - const { workspaceId, scope, rowId, filter, excludeRowIds } = parsed.data.body + const { workspaceId, scope, rowId, filter: wireFilter, excludeRowIds } = parsed.data.body + // Dual-grammar wire: a predicate downgrades losslessly-or-throws to the + // legacy Filter the runners/persisted payloads still compile. + const filter = toLegacyFilter(wireFilter) const result = await checkAccess(tableId, authResult.userId, 'write') if (!result.ok) return accessError(result, requestId, tableId) diff --git a/apps/sim/app/api/table/[tableId]/columns/run/route.ts b/apps/sim/app/api/table/[tableId]/columns/run/route.ts index 00856ae4a1a..6bdfaea5ac2 100644 --- a/apps/sim/app/api/table/[tableId]/columns/run/route.ts +++ b/apps/sim/app/api/table/[tableId]/columns/run/route.ts @@ -5,6 +5,7 @@ import { parseRequest } from '@/lib/api/server' import { checkSessionOrInternalAuth } from '@/lib/auth/hybrid' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' +import { toLegacyFilter } from '@/lib/table/query-builder/converters' import { runWorkflowColumn } from '@/lib/table/workflow-columns' import { accessError, checkAccess, tableFilterError } from '@/app/api/table/utils' @@ -25,8 +26,18 @@ export const POST = withRouteHandler(async (request: NextRequest, { params }: Ro const parsed = await parseRequest(runColumnContract, request, { params }) if (!parsed.success) return parsed.response const { tableId } = parsed.data.params - const { workspaceId, groupIds, runMode, rowIds, filter, excludeRowIds, limit } = - parsed.data.body + const { + workspaceId, + groupIds, + runMode, + rowIds, + filter: wireFilter, + excludeRowIds, + limit, + } = parsed.data.body + // Dual-grammar wire: downgrade a predicate to the legacy Filter the + // dispatcher and scheduled runs still compile. + const filter = toLegacyFilter(wireFilter) const access = await checkAccess(tableId, auth.userId, 'write') if (!access.ok) return accessError(access, requestId, tableId) diff --git a/apps/sim/app/api/table/[tableId]/delete-async/route.ts b/apps/sim/app/api/table/[tableId]/delete-async/route.ts index 236eb556240..291a985ed93 100644 --- a/apps/sim/app/api/table/[tableId]/delete-async/route.ts +++ b/apps/sim/app/api/table/[tableId]/delete-async/route.ts @@ -11,6 +11,7 @@ import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { markTableDeleteFailed, runTableDelete } from '@/lib/table/delete-runner' import { markTableJobRunning, releaseJobClaim } from '@/lib/table/jobs/service' import { assertRowDelete } from '@/lib/table/mutation-locks' +import { toLegacyFilter } from '@/lib/table/query-builder/converters' import type { TableDeleteJobPayload } from '@/lib/table/types' import { accessError, checkAccess, tableFilterError } from '@/app/api/table/utils' @@ -43,7 +44,10 @@ export const POST = withRouteHandler(async (request: NextRequest, { params }: Ro const parsed = await parseRequest(deleteTableRowsAsyncContract, request, { params }) if (!parsed.success) return parsed.response const { tableId } = parsed.data.params - const { workspaceId, filter, excludeRowIds, estimatedCount } = parsed.data.body + const { workspaceId, filter: wireFilter, excludeRowIds, estimatedCount } = parsed.data.body + // Dual-grammar wire: a predicate downgrades losslessly-or-throws to the + // legacy Filter the runners/persisted payloads still compile. + const filter = toLegacyFilter(wireFilter) const access = await checkAccess(tableId, userId, 'write') if (!access.ok) return accessError(access, requestId, tableId) diff --git a/apps/sim/app/api/table/[tableId]/rows/find/route.ts b/apps/sim/app/api/table/[tableId]/rows/find/route.ts index 81e5500bd9a..5c10c5ae471 100644 --- a/apps/sim/app/api/table/[tableId]/rows/find/route.ts +++ b/apps/sim/app/api/table/[tableId]/rows/find/route.ts @@ -5,8 +5,9 @@ import { isZodError, validationErrorResponse } from '@/lib/api/server/validation import { checkSessionOrInternalAuth } from '@/lib/auth/hybrid' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import type { Sort } from '@/lib/table' +import type { Filter, Sort, SortSpec, TablePredicate } from '@/lib/table' import { TableQueryValidationError } from '@/lib/table/errors' +import { toLegacyFilter, toLegacySort } from '@/lib/table/query-builder/converters' import { findRowMatches } from '@/lib/table/rows/service' import { accessError, checkAccess } from '@/app/api/table/utils' @@ -60,7 +61,12 @@ export const GET = withRouteHandler( const { matches, truncated } = await findRowMatches( table, - { q: validated.q, filter: validated.filter, sort: validated.sort }, + { + q: validated.q, + // Dual-grammar wire: findRowMatches compiles the legacy pair. + filter: toLegacyFilter(validated.filter as Filter | TablePredicate | undefined), + sort: toLegacySort(validated.sort as Sort | SortSpec | undefined), + }, requestId ) diff --git a/apps/sim/app/api/table/[tableId]/rows/route.test.ts b/apps/sim/app/api/table/[tableId]/rows/route.test.ts index 17596ca3735..2b3fa64ea4b 100644 --- a/apps/sim/app/api/table/[tableId]/rows/route.test.ts +++ b/apps/sim/app/api/table/[tableId]/rows/route.test.ts @@ -226,6 +226,42 @@ describe('GET /api/table/[tableId]/rows', () => { const body = await res.json() expect(body.data.rows[0].data).toEqual({ col_aaa: 'Ada', col_bbb: 36 }) }) + + /** + * The grid now speaks the v2 grammar on this route: a predicate-shaped filter + * takes the NATIVE predicate path into queryRows (not a downgrade), and an + * ordered sort spec compiles to the record the engine's sort builder takes. + */ + it('routes a predicate filter + spec sort natively for session callers', async () => { + authAs('session') + + const res = await callGet({ + workspaceId: 'workspace-1', + filter: JSON.stringify({ all: [{ field: 'col_aaa', op: 'eq', value: 'Ada' }] }), + sort: JSON.stringify([{ field: 'col_bbb', direction: 'desc' }]), + }) + + expect(res.status).toBe(200) + const options = mockQueryRows.mock.calls[0][1] + expect(options.predicate).toEqual({ all: [{ field: 'col_aaa', op: 'eq', value: 'Ada' }] }) + expect(options.filter).toBeUndefined() + expect(options.sort).toEqual({ col_bbb: 'desc' }) + }) + + it('translates a name-keyed predicate for internal-JWT callers', async () => { + authAs('internal_jwt') + + const res = await callGet({ + workspaceId: 'workspace-1', + filter: JSON.stringify({ all: [{ field: 'Name', op: 'eq', value: 'Ada' }] }), + sort: JSON.stringify([{ field: 'Age', direction: 'asc' }]), + }) + + expect(res.status).toBe(200) + const options = mockQueryRows.mock.calls[0][1] + expect(options.predicate).toEqual({ all: [{ field: 'col_aaa', op: 'eq', value: 'Ada' }] }) + expect(options.sort).toEqual({ col_bbb: 'asc' }) + }) }) describe('PUT/DELETE /api/table/[tableId]/rows — predicate filters', () => { diff --git a/apps/sim/app/api/table/[tableId]/rows/route.ts b/apps/sim/app/api/table/[tableId]/rows/route.ts index 887f169fdf2..443a9a865d1 100644 --- a/apps/sim/app/api/table/[tableId]/rows/route.ts +++ b/apps/sim/app/api/table/[tableId]/rows/route.ts @@ -13,7 +13,7 @@ import { isZodError, parseJsonBody, validationErrorResponse } from '@/lib/api/se import { type AuthTypeValue, checkSessionOrInternalAuth } from '@/lib/auth/hybrid' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import type { Filter, RowData, Sort, TableRowsCursor, TableSchema } from '@/lib/table' +import type { Filter, RowData, Sort, SortSpec, TableRowsCursor, TableSchema } from '@/lib/table' import { batchInsertRows, batchUpdateRows, @@ -26,7 +26,7 @@ import { validateRowSize, } from '@/lib/table' import { TableQueryValidationError } from '@/lib/table/errors' -import { predicateToFilter } from '@/lib/table/query-builder/converters' +import { isTablePredicate, predicateToFilter } from '@/lib/table/query-builder/converters' import { validatePredicate } from '@/lib/table/query-builder/validate' import { queryRows } from '@/lib/table/rows/service' import { predicateToStorage } from '@/lib/table/select-values' @@ -36,8 +36,15 @@ import { accessError, checkAccess, rowWriteErrorResponse } from '@/app/api/table const logger = createLogger('TableRowsAPI') -function isTablePredicate(raw: TablePredicate | Filter): raw is TablePredicate { - return 'all' in raw || 'any' in raw +/** Dual-grammar sort: an ordered spec (v2) or the legacy record, either keying. */ +function resolveWireSort( + sort: Sort | SortSpec | undefined, + wire: RowWireTranslators +): Sort | undefined { + if (!sort) return undefined + if (!Array.isArray(sort)) return wire.sortIn(sort) + const spec = wire.sortSpecIn(sort) + return spec.length > 0 ? Object.fromEntries(spec.map((s) => [s.field, s.direction])) : undefined } /** @@ -286,8 +293,10 @@ export const GET = withRouteHandler( const result = await queryRows( table, { - filter: validated.filter ? wire.filterIn(validated.filter as Filter) : undefined, - sort: validated.sort ? wire.sortIn(validated.sort) : undefined, + ...(validated.filter && isTablePredicate(validated.filter as Filter | TablePredicate) + ? { predicate: wire.predicateIn(validated.filter as TablePredicate) } + : { filter: validated.filter ? wire.filterIn(validated.filter as Filter) : undefined }), + sort: resolveWireSort(validated.sort as Sort | SortSpec | undefined, wire), limit: validated.limit, offset: validated.offset, after: validated.after, diff --git a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-filter/table-filter.tsx b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-filter/table-filter.tsx index a0af627d296..61643e48d4e 100644 --- a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-filter/table-filter.tsx +++ b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-filter/table-filter.tsx @@ -4,7 +4,7 @@ import { memo, useCallback, useMemo, useRef, useState } from 'react' import { Button, ChipDropdown, ChipInput } from '@sim/emcn' import { Plus, X } from '@sim/emcn/icons' import { generateShortId } from '@sim/utils/id' -import type { ColumnDefinition, Filter, FilterRule } from '@/lib/table' +import type { ColumnDefinition, FilterRule, TablePredicate } from '@/lib/table' import { getColumnId } from '@/lib/table/column-keys' import { COMPARISON_OPERATORS, @@ -12,7 +12,10 @@ import { SINGLE_SELECT_FILTER_OPERATORS, VALUELESS_OPERATORS, } from '@/lib/table/query-builder/constants' -import { filterRulesToFilter, filterToRules } from '@/lib/table/query-builder/converters' +import { + filterRulesToPredicate, + predicateToFilterRules, +} from '@/lib/table/query-builder/converters' const SINGLE_SELECT_COMPARISON_OPERATORS = COMPARISON_OPERATORS.filter((o) => SINGLE_SELECT_FILTER_OPERATORS.has(o.value) @@ -27,14 +30,14 @@ function selectFilterOperators(column: ColumnDefinition | undefined): Set void + filter: TablePredicate | null + onApply: (filter: TablePredicate | null) => void onClose: () => void } export function TableFilter({ columns, filter, onApply, onClose }: TableFilterProps) { const [rules, setRules] = useState(() => { - const fromFilter = filterToRules(filter) + const fromFilter = predicateToFilterRules(filter) return fromFilter.length > 0 ? fromFilter : [createRule(columns)] }) @@ -112,7 +115,7 @@ export function TableFilter({ columns, filter, onApply, onClose }: TableFilterPr const validRules = rulesRef.current.filter( (r) => r.column && (r.value || VALUELESS_OPERATORS.has(r.operator)) ) - onApply(filterRulesToFilter(validRules, columns)) + onApply(filterRulesToPredicate(validRules, columns)) }, [columns, onApply]) const handleClear = useCallback(() => { diff --git a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/table-grid.tsx b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/table-grid.tsx index ed93cdaca87..d7bf866e721 100644 --- a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/table-grid.tsx +++ b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/table-grid.tsx @@ -13,9 +13,9 @@ import type { RunLimit, RunMode, TableFindMatch } from '@/lib/api/contracts/tabl import { captureEvent } from '@/lib/posthog/client' import type { ColumnDefinition, - Filter, TableLocks, TableMetadata, + TablePredicate, TableRow as TableRowType, WorkflowGroup, } from '@/lib/table' @@ -121,7 +121,7 @@ export interface SelectionSnapshot { allRows: boolean rowCount: number /** Active filter when `allRows` is set — lets a filtered "select all" run only matching rows. */ - filter?: Filter + filter?: TablePredicate /** Deselected rows when `allRows` is set — runs/stops skip them. */ excludeRowIds?: string[] } | null @@ -199,7 +199,7 @@ interface TableGridProps { runMode: RunMode, rowIds?: string[], limit?: RunLimit, - filter?: Filter, + filter?: TablePredicate, excludeRowIds?: string[] ) => void /** Fire every runnable column on a single row (per-row gutter Play). */ @@ -209,14 +209,14 @@ interface TableGridProps { onRunRows: ( rowIds: string[] | undefined, runMode: RunMode, - filter?: Filter, + filter?: TablePredicate, excludeRowIds?: string[] ) => void /** Stop running workflows on `rowIds`. Per-row gutter Stop also funnels through here. */ onStopRows: (rowIds: string[]) => void /** Select-all Stop: table-wide, or scoped to the active filter when one is set. * `excludeRowIds` (deselected rows) keep running. */ - onStopAllRows: (filter?: Filter, excludeRowIds?: string[]) => void + onStopAllRows: (filter?: TablePredicate, excludeRowIds?: string[]) => void /** Single-row stop for the per-row gutter button. */ onStopRow: (rowId: string) => void /** diff --git a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/hooks/use-table.test.ts b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/hooks/use-table.test.ts index 8026fbfa630..0fed31ca065 100644 --- a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/hooks/use-table.test.ts +++ b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/hooks/use-table.test.ts @@ -218,7 +218,7 @@ describe('useTable – ensureAllRowsLoaded', () => { }) it('encodes queryOptions.filter into the queryKey passed to getQueryData', async () => { - const filter = { column: 'name', operator: 'eq', value: 'Alice' } as never + const filter = { all: [{ field: 'name', op: 'eq' as const, value: 'Alice' }] } mockGetQueryData.mockReturnValue({ pages: makePages([3], 3) }) const { ensureAllRowsLoaded } = makeHook({ filter, sort: null }) await ensureAllRowsLoaded() diff --git a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/hooks/use-table.ts b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/hooks/use-table.ts index 9fcf4dd7db5..be473ae71d3 100644 --- a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/hooks/use-table.ts +++ b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/hooks/use-table.ts @@ -4,13 +4,13 @@ import { useCallback, useMemo } from 'react' import { useQueryClient } from '@tanstack/react-query' import type { ColumnDefinition, - Filter, TableDefinition, + TablePredicate, TableRow, WorkflowGroup, } from '@/lib/table' import { TABLE_LIMITS } from '@/lib/table/constants' -import { pruneFilterForColumns } from '@/lib/table/query-builder/converters' +import { prunePredicateForColumns } from '@/lib/table/query-builder/converters' import type { FlattenOutputsBlockInput } from '@/lib/workflows/blocks/flatten-outputs' import { getBlock } from '@/blocks' import { @@ -51,7 +51,7 @@ export interface UseTableReturn { * select-all run/stop/delete — must scope with THIS, not the raw filter, or * the action targets a predicate the grid isn't displaying. */ - filter: Filter | null + filter: TablePredicate | null isLoadingRows: boolean refetchRows: () => void /** @@ -98,7 +98,7 @@ export function useTable({ workspaceId, tableId, queryOptions }: UseTableParams) // here, above every consumer of the rows query key, so the paged helpers below // can't rebuild the key from the unpruned filter and drift. const filter = useMemo( - () => pruneFilterForColumns(queryOptions.filter ?? null, tableData?.schema?.columns ?? []), + () => prunePredicateForColumns(queryOptions.filter ?? null, tableData?.schema?.columns ?? []), [queryOptions.filter, tableData?.schema?.columns] ) diff --git a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/table.tsx b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/table.tsx index e8f20a55d4b..146c02c103a 100644 --- a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/table.tsx +++ b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/table.tsx @@ -12,21 +12,16 @@ import type { RunLimit, RunMode, TableViewWire } from '@/lib/api/contracts/table import { captureEvent } from '@/lib/posthog/client' import type { ColumnDefinition, - Filter, - Sort, SortDirection, + SortSpec, TableMetadata, + TablePredicate, TableRow as TableRowType, TableViewConfig, WorkflowGroup, } from '@/lib/table' import { getColumnId } from '@/lib/table/column-keys' import { TABLE_LIMITS } from '@/lib/table/constants' -import { - filterRulesToPredicate, - filterToRules, - predicateToFilter, -} from '@/lib/table/query-builder/converters' import { type BreadcrumbItem, type ColumnOption, @@ -260,7 +255,7 @@ export function Table({ selectionStats: { hasIncompleteOrFailed: false, hasCompleted: false, hasInFlight: false }, singleWorkflowCell: null, }) - const [filter, setFilter] = useState(null) + const [filter, setFilter] = useState(null) const [filterOpen, setFilterOpen] = useState(false) /** Hidden **column ids**. Lives here (not in the grid) because the filter * panel's Columns section edits it and the active view persists it. */ @@ -276,9 +271,9 @@ export function Table({ const hiddenColumnsRef = useRef(hiddenColumns) hiddenColumnsRef.current = hiddenColumns - /** Resolved single-column sort, or `null` when no column is active. */ - const sortQuery = useMemo( - () => (sortColumn ? { [sortColumn]: sortDirection } : null), + /** Resolved single-column sort as an ordered spec, or `null` when none is active. */ + const sortQuery = useMemo( + () => (sortColumn ? [{ field: sortColumn, direction: sortDirection }] : null), [sortColumn, sortDirection] ) @@ -428,11 +423,7 @@ export function Table({ config: TableViewConfig | null, keep?: { sort?: boolean; filter?: boolean; hiddenColumns?: boolean } ) => { - // Stored views speak the v2 grammar; the grid's runtime state is still the - // legacy Filter/Sort pair, so translate at this boundary. A stored predicate - // is always builder-authored (the save path converts from builder output), - // so the legacy projection is total here. - if (!keep?.filter) setFilter(config?.filter ? predicateToFilter(config.filter) : null) + if (!keep?.filter) setFilter(config?.filter ?? null) if (!keep?.hiddenColumns) setHiddenColumns(config?.hiddenColumns ?? []) if (keep?.sort) return const sortEntry = config?.sort?.[0] @@ -660,20 +651,11 @@ export function Table({ const currentViewConfig = useMemo( () => ({ ...(activeView?.config ?? tableData?.metadata), - // Views store the v2 grammar; the grid runs on the legacy pair. The filter - // is builder-authored, so the rule round-trip is lossless here. - filter: effectiveFilter ? filterRulesToPredicate(filterToRules(effectiveFilter)) : null, - sort: sortColumn ? [{ field: sortColumn, direction: sortDirection }] : null, + filter: effectiveFilter ?? null, + sort: sortQuery, hiddenColumns: effectiveHiddenColumns, }), - [ - activeView, - tableData?.metadata, - effectiveFilter, - sortColumn, - sortDirection, - effectiveHiddenColumns, - ] + [activeView, tableData?.metadata, effectiveFilter, sortQuery, effectiveHiddenColumns] ) /** @@ -855,7 +837,7 @@ export function Table({ (args: { groupIds: string[] rowIds?: string[] - filter?: Filter + filter?: TablePredicate excludeRowIds?: string[] runMode: RunMode limit?: RunLimit @@ -894,7 +876,7 @@ export function Table({ runMode: RunMode, rowIds?: string[], limit?: RunLimit, - filter?: Filter, + filter?: TablePredicate, excludeRowIds?: string[] ) => { runScope({ @@ -911,7 +893,12 @@ export function Table({ ) const onRunRows = useCallback( - (rowIds: string[] | undefined, runMode: RunMode, filter?: Filter, excludeRowIds?: string[]) => { + ( + rowIds: string[] | undefined, + runMode: RunMode, + filter?: TablePredicate, + excludeRowIds?: string[] + ) => { runScope({ groupIds: tableWorkflowGroups.map((g) => g.id), rowIds, @@ -979,7 +966,7 @@ export function Table({ /** Select-all Stop — filter-scoped when a filter is active; deselected rows keep running. */ const onStopAllRows = useCallback( - (filter?: Filter, excludeRowIds?: string[]) => { + (filter?: TablePredicate, excludeRowIds?: string[]) => { // `sort` scopes the optimistic flip to the active view's cache (filtered stops // only cancel matching rows server-side). cancelRunsMutate({ scope: 'all', filter, sort: queryOptions.sort, excludeRowIds }) @@ -1079,7 +1066,7 @@ export function Table({ [columnOptions, sortColumn, sortDirection, setTableParams] ) - const handleFilterApply = (next: Filter | null) => { + const handleFilterApply = (next: TablePredicate | null) => { setFilter(next) } diff --git a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/types.ts b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/types.ts index d7c7553c2ce..34618913acf 100644 --- a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/types.ts +++ b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/types.ts @@ -1,4 +1,4 @@ -import type { Filter, Sort, TableRow } from '@/lib/table' +import type { SortSpec, TablePredicate, TableRow } from '@/lib/table' /** * Reason the inline editor completed, used to determine navigation after save @@ -9,8 +9,8 @@ export type SaveReason = 'enter' | 'tab' | 'shift-tab' | 'blur' * Query options for filtering and sorting table data */ export interface QueryOptions { - filter: Filter | null - sort: Sort | null + filter: TablePredicate | null + sort: SortSpec | null } /** diff --git a/apps/sim/hooks/queries/tables.ts b/apps/sim/hooks/queries/tables.ts index 0f447344bd0..a1f13eba635 100644 --- a/apps/sim/hooks/queries/tables.ts +++ b/apps/sim/hooks/queries/tables.ts @@ -88,13 +88,13 @@ import { buildUpgradeHref } from '@/lib/billing/upgrade-reasons' import type { CsvHeaderMapping, EnrichmentRunDetail, - Filter, RowData, RowExecutionMetadata, RowExecutions, - Sort, + SortSpec, TableDefinition, TableMetadata, + TablePredicate, TableRow, WorkflowGroup, WorkflowGroupDependencies, @@ -129,8 +129,8 @@ export const TABLE_EXPORT_JOBS_STALE_TIME = 5 * 1000 type TableRowsParams = Omit & TableIdParamsInput & { - filter?: Filter | null - sort?: Sort | null + filter?: TablePredicate | null + sort?: SortSpec | null } export type TableRowsResponse = Pick< @@ -416,8 +416,8 @@ interface InfiniteTableRowsParams { workspaceId: string tableId: string pageSize: number - filter?: Filter | null - sort?: Sort | null + filter?: TablePredicate | null + sort?: SortSpec | null enabled?: boolean } @@ -433,8 +433,8 @@ interface FindTableRowsParams { workspaceId: string tableId: string q: string - filter?: Filter | null - sort?: Sort | null + filter?: TablePredicate | null + sort?: SortSpec | null } export interface TableFindResult { @@ -1225,7 +1225,7 @@ interface DeleteTableRowsAsyncVariables { filter?: DeleteTableRowsAsyncBody['filter'] /** Active sort — together with `filter` it identifies the exact rows query to optimistically * strip, so we don't clear unrelated cached views (other filters/sorts). */ - sort?: Sort | null + sort?: SortSpec | null /** Rows deselected after "select all" — spared by the job. */ excludeRowIds?: string[] /** Doomed-row estimate shown in the confirm — persisted on the job so server counts can @@ -1259,7 +1259,13 @@ export function useDeleteTableRowsAsync({ workspaceId, tableId }: RowMutationCon // Target the exact infinite-rows query for the view the user is on — not every cached view. const activeKey = tableKeys.infiniteRows( tableId, - tableRowsParamsKey({ pageSize: TABLE_LIMITS.MAX_QUERY_LIMIT, filter: filter ?? null, sort }) + tableRowsParamsKey({ + pageSize: TABLE_LIMITS.MAX_QUERY_LIMIT, + // The wire type is the dual-grammar union; the grid only ever sends its + // own predicate state, so the cache key narrows to that shape. + filter: (filter as TablePredicate | undefined) ?? null, + sort, + }) ) await queryClient.cancelQueries({ queryKey: activeKey }) const previousRows = @@ -1557,10 +1563,10 @@ interface CancelRunsParams { scope: 'all' | 'row' rowId?: string /** Scope-`all` only: cancel just the cells on rows matching this filter (filtered select-all Stop). */ - filter?: Filter + filter?: TablePredicate /** Active sort — with `filter` it identifies the exact rows query whose cells the optimistic * cancel may flip (other cached views contain rows the server won't touch). */ - sort?: Sort | null + sort?: SortSpec | null /** Scope-`all` only: deselected rows whose cells keep running. */ excludeRowIds?: string[] } @@ -2192,7 +2198,7 @@ interface RunColumnVariables { /** "Select all under a filter" — run every row matching this filter (mutually exclusive with * `rowIds`). Optimistic stamping is skipped (like `limit`) since the matching set isn't known * client-side; the dispatcher's real pending stamps drive the UI. */ - filter?: Filter + filter?: TablePredicate /** Select-all scope only: deselected rows — skipped by the dispatcher and the optimistic stamp. */ excludeRowIds?: string[] /** Cap the run to the first `max` eligible rows. Omit for an unbounded run. diff --git a/apps/sim/lib/api/contracts/tables.ts b/apps/sim/lib/api/contracts/tables.ts index 89cb5d78727..c0af27b34c9 100644 --- a/apps/sim/lib/api/contracts/tables.ts +++ b/apps/sim/lib/api/contracts/tables.ts @@ -548,8 +548,11 @@ export const deleteTableRowsBodySchema = z /** Unrefined base so v1 contracts can `.extend()` — consumers use {@link tableRowsQuerySchema}. */ export const tableRowsQueryBaseSchema = z.object({ workspaceId: workspaceIdSchema, - filter: domainObjectSchema().optional(), - sort: domainObjectSchema().optional(), + // Dual-grammar during the v2 transition: the strict predicate tree wins the + // union; anything else falls through to the legacy `$`-object. Same for sort: + // an ordered spec array vs the legacy `{col: dir}` record. + filter: z.union([predicateSchema, domainObjectSchema()]).optional(), + sort: z.union([sortSpecSchema, domainObjectSchema()]).optional(), /** * Keyset cursor `(orderKey, id)` for the default row order — each page is an index seek * instead of OFFSET's scan-and-discard. Mutually exclusive with `sort` (cursors only make @@ -862,8 +865,8 @@ export type RowQueryResponse = ContractJsonResponse export const findTableRowsQuerySchema = z.object({ workspaceId: workspaceIdSchema, q: requiredFieldSchema('Search query is required'), - filter: domainObjectSchema().optional(), - sort: domainObjectSchema().optional(), + filter: z.union([predicateSchema, domainObjectSchema()]).optional(), + sort: z.union([sortSpecSchema, domainObjectSchema()]).optional(), }) /** One matching cell: its 0-based ordinal in the filtered+sorted view, its row id, and the column name. */ @@ -1292,7 +1295,7 @@ export const deleteTableRowsContract = defineRouteContract({ */ export const deleteTableRowsAsyncBodySchema = z.object({ workspaceId: workspaceIdSchema, - filter: nonEmptyFilterSchema.optional(), + filter: bulkFilterSchema.optional(), excludeRowIds: z .array(z.string().min(1)) .max( @@ -1487,7 +1490,7 @@ export const cancelTableRunsBodySchema = z workspaceId: workspaceIdSchema, scope: z.enum(['all', 'row']), rowId: z.string().min(1).optional(), - filter: domainObjectSchema().optional(), + filter: z.union([predicateSchema, domainObjectSchema()]).optional(), /** Scope-`all` only: rows deselected from the selection — their cells keep running. */ excludeRowIds: z .array(z.string().min(1)) @@ -1591,7 +1594,7 @@ export const runColumnBodySchema = z rowIds: z.array(z.string().min(1)).min(1).optional(), /** "Select all under a filter" — run every row matching this filter instead of `rowIds`. The * dispatcher walks only matching rows (paginated), so no id list is materialized. */ - filter: nonEmptyFilterSchema.optional(), + filter: bulkFilterSchema.optional(), /** Select-all scope only: rows deselected from the selection — the dispatcher skips them. */ excludeRowIds: z .array(z.string().min(1)) diff --git a/apps/sim/lib/table/query-builder/__tests__/converters.test.ts b/apps/sim/lib/table/query-builder/__tests__/converters.test.ts index d7d6602668b..e978b184b2d 100644 --- a/apps/sim/lib/table/query-builder/__tests__/converters.test.ts +++ b/apps/sim/lib/table/query-builder/__tests__/converters.test.ts @@ -10,9 +10,11 @@ import { filterRulesToFilter, filterRulesToPredicate, filterToRules, + isTablePredicate, predicateToFilter, predicateToFilterRules, pruneFilterForColumns, + prunePredicateForColumns, sortRulesToSortSpec, } from '@/lib/table/query-builder/converters' import type { ColumnDefinition, FilterRule, SortRule, TablePredicate } from '@/lib/table/types' @@ -366,3 +368,82 @@ describe('pruneFilterForColumns', () => { expect(pruneFilterForColumns(null, [single])).toBeNull() }) }) + +describe('filterRulesToPredicate select-awareness (grid parity)', () => { + const SELECT_COLS: ColumnDefinition[] = [ + { + id: 'col_status', + name: 'col_status', + type: 'select', + options: [{ id: '123', name: 'One Two Three' }], + }, + ] + + it('never scalar-coerces a select operand (a numeric-looking option id stays a string)', () => { + const p = filterRulesToPredicate( + [rule({ column: 'col_status', operator: 'eq', value: '123' })], + SELECT_COLS + ) + expect(p).toEqual({ all: [{ field: 'col_status', op: 'eq', value: '123' }] }) + }) + + it('keeps select in-list members as strings', () => { + const p = filterRulesToPredicate( + [rule({ column: 'col_status', operator: 'in', value: '123, 456' })], + SELECT_COLS + ) + expect(p).toEqual({ all: [{ field: 'col_status', op: 'in', value: ['123', '456'] }] }) + }) + + it('still coerces non-select columns', () => { + const p = filterRulesToPredicate( + [rule({ column: 'wins', operator: 'eq', value: '123' })], + SELECT_COLS + ) + expect(p).toEqual({ all: [{ field: 'wins', op: 'eq', value: 123 }] }) + }) +}) + +describe('prunePredicateForColumns', () => { + const COLS: ColumnDefinition[] = [ + { id: 'col_s', name: 'col_s', type: 'select', options: [{ id: 'o1', name: 'One' }] }, + { id: 'col_n', name: 'col_n', type: 'number' }, + ] + + it('drops an operator a select column no longer accepts, keeps the rest', () => { + const p = prunePredicateForColumns( + { + all: [ + { field: 'col_s', op: 'contains', value: 'o1' }, // single-select: contains not allowed + { field: 'col_n', op: 'gte', value: 5 }, + ], + }, + COLS + ) + expect(p).toEqual({ all: [{ field: 'col_n', op: 'gte', value: 5 }] }) + }) + + it('returns the same reference when nothing is dropped', () => { + const p = { all: [{ field: 'col_n', op: 'gte' as const, value: 5 }] } + expect(prunePredicateForColumns(p, COLS)).toBe(p) + }) + + it('collapses to null when every condition is dropped', () => { + expect( + prunePredicateForColumns({ all: [{ field: 'col_s', op: 'contains', value: 'o1' }] }, COLS) + ).toBeNull() + }) +}) + +describe('isTablePredicate', () => { + it('discriminates the two grammars group-first', () => { + expect(isTablePredicate({ all: [] })).toBe(true) + expect(isTablePredicate({ any: [] })).toBe(true) + expect(isTablePredicate({ status: 'active' })).toBe(false) + expect(isTablePredicate({ $or: [{ a: 1 }] } as never)).toBe(false) + }) +}) + +it('prunePredicateForColumns fails closed on a malformed value instead of throwing', () => { + expect(prunePredicateForColumns({ column: 'name', operator: 'eq' } as never, [])).toBeNull() +}) diff --git a/apps/sim/lib/table/query-builder/constants.ts b/apps/sim/lib/table/query-builder/constants.ts index fc4fbb7f123..8a5b9af4117 100644 --- a/apps/sim/lib/table/query-builder/constants.ts +++ b/apps/sim/lib/table/query-builder/constants.ts @@ -34,7 +34,8 @@ export const VALUELESS_OPERATORS = new Set(['isEmpty', 'isNotEmpty']) * against the whole array can never be true. * * These must stay in step with the server whitelist in `lib/table/sql.ts`: the - * filter picker offers exactly these, and `pruneFilterForColumns` DROPS + * filter picker offers exactly these, and `pruneFilterForColumns` / + * `prunePredicateForColumns` DROP * anything outside them, so an operator missing here is silently discarded from * a filter the server would have accepted. `sql.test.ts` asserts the two sets * agree through `UI_TO_WIRE_OPERATOR` rather than leaving it to a comment. diff --git a/apps/sim/lib/table/query-builder/converters.ts b/apps/sim/lib/table/query-builder/converters.ts index 0841fb68b67..a965b17e194 100644 --- a/apps/sim/lib/table/query-builder/converters.ts +++ b/apps/sim/lib/table/query-builder/converters.ts @@ -117,6 +117,40 @@ export function pruneFilterForColumns( return filterRulesToFilter(kept, columns) } +/** + * Predicate-grammar sibling of {@link pruneFilterForColumns}: drops conditions a + * `select` column no longer accepts (operator stranded by a type/`multiple` + * change), so a stale applied filter can't fail every subsequent rows query. + */ +export function prunePredicateForColumns( + predicate: TablePredicate | null, + columns: ColumnDefinition[] +): TablePredicate | null { + if (!predicate) return null + // A malformed value (corrupt persisted state, a stale cast) fails CLOSED to + // "no filter" — throwing here would take down the whole table page render. + if (!('all' in predicate) && !('any' in predicate)) return null + + const rules = predicateToFilterRules(predicate) + const kept = rules.filter((rule) => { + const column = columns.find((c) => columnMatchesRef(c, rule.column)) + if (column?.type !== 'select') return true + const allowed = column.multiple ? MULTI_SELECT_FILTER_OPERATORS : SINGLE_SELECT_FILTER_OPERATORS + return allowed.has(rule.operator) + }) + + if (kept.length === rules.length) return predicate + return filterRulesToPredicate(kept, columns) +} + +/** + * Discriminates the v2 predicate tree from the legacy `$`-object on dual-grammar + * wire fields. Group-first, matching every other discrimination site. + */ +export function isTablePredicate(value: Filter | TablePredicate): value is TablePredicate { + return 'all' in value || 'any' in value +} + /** Converts a single UI sort rule to a Sort object for API queries. */ export function sortRuleToSort(rule: SortRule | null): Sort | null { if (!rule || !rule.column) return null @@ -290,10 +324,10 @@ function normalizeSortDirection(direction: string): SortDirection { const VALUELESS_OPS = new Set(['isEmpty', 'isNotEmpty', 'isNull', 'isNotNull']) -function ruleToPredicate(rule: FilterRule): Predicate { +function ruleToPredicate(rule: FilterRule, keepAsText = false): Predicate { const op = rule.operator as FilterOp if (VALUELESS_OPS.has(op)) return { field: rule.column, op } - return { field: rule.column, op, value: parseValue(rule.value, rule.operator) } + return { field: rule.column, op, value: parseValue(rule.value, rule.operator, keepAsText) } } /** @@ -301,7 +335,10 @@ function ruleToPredicate(rule: FilterRule): Predicate { * boundary form an `all` group; multiple groups are combined under `any`. * Mirrors {@link filterRulesToFilter} but emits the bare-operator grammar. */ -export function filterRulesToPredicate(rules: FilterRule[]): TablePredicate | null { +export function filterRulesToPredicate( + rules: FilterRule[], + columns: ColumnDefinition[] = [] +): TablePredicate | null { // Tolerate a non-array (the builder value can arrive malformed from an agent // that doesn't speak the rule shape) instead of throwing "rules is not iterable". if (!Array.isArray(rules) || rules.length === 0) return null @@ -329,7 +366,11 @@ export function filterRulesToPredicate(rules: FilterRule[]): TablePredicate | nu // A genuinely blank builder row (no column picked yet) contributes nothing. continue } - current.push(ruleToPredicate(rule)) + // A select value is an opaque option id — never scalar-coerce it (an id + // that happens to look numeric would silently become a number and match + // nothing). Same rule as filterRulesToFilter. + const isSelect = columns.find((c) => columnMatchesRef(c, rule.column))?.type === 'select' + current.push(ruleToPredicate(rule, isSelect)) } if (current.length > 0) groups.push(current) @@ -424,3 +465,22 @@ export function predicateToFilter(predicate: TablePredicate): Filter { } return nodeToFilter(predicate) } + +/** + * Downgrades a dual-grammar wire filter to the legacy `Filter` the job runners + * and search service still compile. The predicate path throws (via + * `predicateToFilter`) on any leaf the legacy compiler would silently discard, + * so a downgraded filter can never widen. Grammar-only: field keys pass through + * untranslated (session callers already speak column ids). + */ +export function toLegacyFilter(filter: Filter | TablePredicate | undefined): Filter | undefined { + if (!filter) return undefined + return isTablePredicate(filter) ? predicateToFilter(filter) : filter +} + +/** Downgrades a dual-grammar wire sort (ordered spec array or legacy record). */ +export function toLegacySort(sort: Sort | SortSpec | undefined): Sort | undefined { + if (!sort) return undefined + if (!Array.isArray(sort)) return sort + return sort.length > 0 ? Object.fromEntries(sort.map((s) => [s.field, s.direction])) : undefined +} From 3dfaaad2f455238ec608f703cacf9f1f45d322a5 Mon Sep 17 00:00:00 2001 From: Theodore Li Date: Wed, 29 Jul 2026 17:21:39 -0700 Subject: [PATCH 21/26] chore(tools): carry every Sim table op in the minimal dev registry MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit table_create and table_list were the only two of the 13 Sim table tools absent from minimal mode. No block routes to them today, but minimal mode should not silently lack table operations the full registry has — that asymmetry is exactly how the earlier 'table block missing in minimal registry' round trips happened. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_011M563cbFy2S74GvSDf2C3R --- apps/sim/tools/registry.minimal.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/apps/sim/tools/registry.minimal.ts b/apps/sim/tools/registry.minimal.ts index cd4ef77d640..899124191ce 100644 --- a/apps/sim/tools/registry.minimal.ts +++ b/apps/sim/tools/registry.minimal.ts @@ -73,11 +73,13 @@ import { } from '@/tools/slack' import { tableBatchInsertRowsTool, + tableCreateTool, tableDeleteRowsByFilterTool, tableDeleteRowTool, tableGetRowTool, tableGetSchemaTool, tableInsertRowTool, + tableListTool, tableQueryRowsTool, tableQueryRowsV2Tool, tableUpdateRowsByFilterTool, @@ -112,6 +114,8 @@ export const tools: Record = { table_query_rows_v2: tableQueryRowsV2Tool, table_get_row: tableGetRowTool, table_get_schema: tableGetSchemaTool, + table_create: tableCreateTool, + table_list: tableListTool, guardrails_validate: guardrailsValidateTool, // Needed so workflow-as-tool and custom (deploy-as-block) tools resolve their // config in minimal-registry dev mode (both route through `workflow_executor`). From d62ad32f44277ca703639771a72c5987ea4aa540 Mon Sep 17 00:00:00 2001 From: Theodore Li Date: Wed, 29 Jul 2026 18:12:17 -0700 Subject: [PATCH 22/26] docs(tables): OpenAPI spec for the v2 tables read API MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Documents the two public v2 endpoints exactly as implemented — the predicate grammar (full operator enum with per-type and select-cardinality semantics, the strict-node and depth/size bounds), cursor pagination with its null-only termination contract, limit=0 unbounded semantics with the 5MB fail-fast, the camelCase built-in columns, and the flag-off 404 behaviour. Deliberately NOT wired into the docs site loader: the surface is dark behind tables-v2-api, and publishing reference docs for an endpoint that 404s would be premature. The filename matches PR #5273's multi-spec layout so adoption is a one-line loader change at GA. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_011M563cbFy2S74GvSDf2C3R --- apps/docs/openapi-v2-tables.json | 381 +++++++++++++++++++++++++++++++ 1 file changed, 381 insertions(+) create mode 100644 apps/docs/openapi-v2-tables.json diff --git a/apps/docs/openapi-v2-tables.json b/apps/docs/openapi-v2-tables.json new file mode 100644 index 00000000000..0641eaaaef3 --- /dev/null +++ b/apps/docs/openapi-v2-tables.json @@ -0,0 +1,381 @@ +{ + "openapi": "3.1.0", + "info": { + "title": "Sim Tables API v2", + "version": "2.0.0-preview", + "description": "Read access to Sim tables with the typed predicate filter grammar and opaque cursor pagination. This surface is feature-gated (`tables-v2-api`): when the flag is off for the caller, every endpoint returns 404 as if it does not exist. Filters are predicate trees — `{\"all\": [...]}` (AND) or `{\"any\": [...]}` (OR) groups whose members are `{field, op, value}` conditions or nested groups. Built-in columns `id`, `createdAt`, and `updatedAt` (camelCase) are filterable and sortable alongside user columns." + }, + "servers": [{ "url": "https://www.sim.ai" }], + "security": [{ "apiKey": [] }], + "paths": { + "/api/v2/tables": { + "get": { + "operationId": "v2ListTables", + "summary": "List Tables", + "description": "List every table in a workspace with its column schema and row count.", + "tags": ["Tables v2"], + "parameters": [ + { + "name": "workspaceId", + "in": "query", + "required": true, + "schema": { "type": "string", "minLength": 1 } + } + ], + "responses": { + "200": { + "description": "Tables in the workspace. Served with `Cache-Control: private, no-store`.", + "content": { + "application/json": { + "schema": { + "type": "object", + "required": ["success", "data"], + "properties": { + "success": { "const": true }, + "data": { + "type": "object", + "required": ["tables", "totalCount"], + "properties": { + "tables": { + "type": "array", + "items": { "$ref": "#/components/schemas/TableSummary" } + }, + "totalCount": { "type": "integer" } + } + } + } + } + } + } + }, + "400": { "$ref": "#/components/responses/ValidationError" }, + "401": { "$ref": "#/components/responses/Unauthorized" }, + "403": { "$ref": "#/components/responses/Forbidden" }, + "404": { "$ref": "#/components/responses/NotFoundOrGated" }, + "429": { "$ref": "#/components/responses/RateLimited" } + } + } + }, + "/api/v2/tables/{tableId}/query": { + "post": { + "operationId": "v2QueryTableRows", + "summary": "Query Rows", + "description": "Query rows with a typed predicate filter, an ordered sort spec, and opaque cursor pagination. Row `data` is keyed by column NAME; `select` cells return option names, and filter operands on select columns accept option names (resolved case-insensitively).\n\n**Pagination contract:** page by passing the previous response's `nextCursor` back as `cursor`, and stop only when it is `null` — a page may return fewer than `limit` rows and still have more behind it, so page fullness is never a termination signal. A cursor encodes a position in the default row order and cannot be combined with `sort` (400 `CURSOR_SORT_CONFLICT`). `totalCount` is computed on the first page only (requests with a `cursor` return `totalCount: null`).", + "tags": ["Tables v2"], + "parameters": [ + { + "name": "tableId", + "in": "path", + "required": true, + "schema": { "type": "string", "minLength": 1 } + } + ], + "requestBody": { + "required": true, + "description": "Bodies over 1 MB are rejected with 413.", + "content": { + "application/json": { + "schema": { + "type": "object", + "required": ["workspaceId"], + "properties": { + "workspaceId": { "type": "string", "minLength": 1 }, + "predicate": { "$ref": "#/components/schemas/Predicate" }, + "sort": { + "type": "array", + "maxItems": 16, + "description": "Ordered sort spec, highest priority first.", + "items": { + "type": "object", + "required": ["field", "direction"], + "properties": { + "field": { "type": "string" }, + "direction": { "enum": ["asc", "desc"] } + } + } + }, + "limit": { + "type": "integer", + "minimum": 0, + "maximum": 1000, + "default": 100, + "description": "Omitted → 100. `1..1000` → page size. `0` → the ENTIRE matching result in one response; fails with 400 `TABLE_QUERY_RESULT_TOO_LARGE` if it exceeds the 5 MB row-data budget (narrow the predicate or page instead)." + }, + "cursor": { + "type": "string", + "description": "Opaque token from a previous response's `nextCursor`. Pass back verbatim. Mutually exclusive with `sort`." + } + } + }, + "examples": { + "filtered": { + "summary": "Multi-select membership + negated pattern", + "value": { + "workspaceId": "ws_123", + "predicate": { + "all": [ + { "field": "Color", "op": "contains", "value": "Purple" }, + { "field": "name", "op": "nlike", "value": "G*" } + ] + }, + "limit": 100 + } + }, + "builtinColumns": { + "summary": "Built-in column range (UTC, timezone-independent)", + "value": { + "workspaceId": "ws_123", + "predicate": { + "all": [ + { "field": "createdAt", "op": "gte", "value": "2026-07-24T03:00:00.000Z" }, + { "field": "createdAt", "op": "lte", "value": "2026-07-25T02:59:59.999Z" } + ] + } + } + } + } + } + } + }, + "responses": { + "200": { + "description": "A page of rows. Served with `Cache-Control: private, no-store`.", + "content": { + "application/json": { + "schema": { + "type": "object", + "required": ["success", "data"], + "properties": { + "success": { "const": true }, + "data": { + "type": "object", + "required": ["rows", "rowCount", "nextCursor"], + "properties": { + "rows": { + "type": "array", + "items": { + "type": "object", + "required": ["id", "data", "createdAt", "updatedAt"], + "properties": { + "id": { "type": "string" }, + "data": { + "type": "object", + "description": "Column-NAME-keyed cell values.", + "additionalProperties": true + }, + "createdAt": { "type": "string", "format": "date-time" }, + "updatedAt": { "type": "string", "format": "date-time" } + } + } + }, + "rowCount": { "type": "integer", "description": "Rows in THIS page." }, + "totalCount": { + "type": ["integer", "null"], + "description": "Rows matching the predicate across all pages. First page only; null when a cursor was supplied." + }, + "limit": { "type": ["integer", "null"] }, + "nextCursor": { + "type": ["string", "null"], + "description": "Non-null ⇒ more rows exist. The ONLY termination signal is null." + } + } + } + } + } + } + } + }, + "400": { + "description": "Validation failure. Machine-readable `code` values include `INVALID_FILTER` (unknown column, operator/type mismatch, malformed tree), `INVALID_ORDER`, `INVALID_CURSOR`, `CURSOR_SORT_CONFLICT`, and `TABLE_QUERY_RESULT_TOO_LARGE` (unbounded result exceeded the 5 MB budget).", + "content": { + "application/json": { + "schema": { "$ref": "#/components/schemas/ErrorBody" } + } + } + }, + "401": { "$ref": "#/components/responses/Unauthorized" }, + "403": { "$ref": "#/components/responses/Forbidden" }, + "404": { "$ref": "#/components/responses/NotFoundOrGated" }, + "413": { + "description": "Request body exceeded the 1 MB cap.", + "content": { + "application/json": { "schema": { "$ref": "#/components/schemas/ErrorBody" } } + } + }, + "429": { "$ref": "#/components/responses/RateLimited" } + } + } + } + }, + "components": { + "securitySchemes": { + "apiKey": { "type": "apiKey", "in": "header", "name": "X-API-Key" } + }, + "schemas": { + "Predicate": { + "description": "A predicate tree: exactly one of `all` (every member must match) or `any` (at least one must). Members are conditions or nested groups; nesting expresses mixed AND/OR logic. Groups must be non-empty (1–100 members), trees at most 10 levels deep and 500 nodes total. Nodes are STRICT: unknown keys, or a node carrying both a group key and condition keys, are rejected rather than ignored.", + "oneOf": [ + { + "type": "object", + "required": ["all"], + "additionalProperties": false, + "properties": { + "all": { + "type": "array", + "minItems": 1, + "maxItems": 100, + "items": { "$ref": "#/components/schemas/PredicateNode" } + } + } + }, + { + "type": "object", + "required": ["any"], + "additionalProperties": false, + "properties": { + "any": { + "type": "array", + "minItems": 1, + "maxItems": 100, + "items": { "$ref": "#/components/schemas/PredicateNode" } + } + } + } + ] + }, + "PredicateNode": { + "oneOf": [ + { "$ref": "#/components/schemas/Predicate" }, + { "$ref": "#/components/schemas/Condition" } + ] + }, + "Condition": { + "type": "object", + "required": ["field", "op"], + "additionalProperties": false, + "properties": { + "field": { + "type": "string", + "maxLength": 128, + "description": "Column name, or a built-in: `id`, `createdAt`, `updatedAt` (camelCase — snake_case is treated as a user column and matches nothing)." + }, + "op": { + "enum": [ + "eq", + "ne", + "gt", + "gte", + "lt", + "lte", + "in", + "nin", + "contains", + "ncontains", + "startsWith", + "endsWith", + "like", + "ilike", + "nlike", + "nilike", + "isEmpty", + "isNotEmpty", + "isNull", + "isNotNull" + ], + "description": "`eq`/`ne`/`in`/`nin` are case-sensitive equality/membership. `contains`/`ncontains`/`startsWith`/`endsWith` are case-insensitive text matches — except on a multi-select column, where `contains`/`ncontains` mean set membership by option name. `like`/`nlike` are case-sensitive and `ilike`/`nilike` case-insensitive patterns with `*` as the only wildcard (literal `%`/`_` match themselves). `isEmpty`/`isNotEmpty` treat null and empty string as empty; `isNull`/`isNotNull` are strict null checks. The four `is*` operators take no `value`. Negated text matches retain rows where the cell is absent. `in`/`nin` require a non-empty array of at most 1000 values; other value-taking operators reject arrays. Select columns accept only equality/membership operators appropriate to their cardinality (single: eq/ne/in/nin; multi: contains/ncontains; both: the `is*` checks)." + }, + "value": { + "description": "Operand. Omit for the `is*` operators. Ranges on `number` columns require numbers, on `date` columns ISO strings (compared as UTC, independent of any session timezone); ranges on `boolean`/`json` columns are rejected." + } + } + }, + "TableSummary": { + "type": "object", + "required": ["id", "name", "schema", "rowCount", "maxRows", "createdAt", "updatedAt"], + "properties": { + "id": { "type": "string" }, + "name": { "type": "string" }, + "description": { "type": ["string", "null"] }, + "schema": { + "type": "object", + "properties": { + "columns": { + "type": "array", + "items": { + "type": "object", + "required": ["name", "type"], + "properties": { + "id": { "type": "string" }, + "name": { "type": "string" }, + "type": { "enum": ["string", "number", "boolean", "date", "json", "select"] }, + "required": { "type": "boolean" }, + "unique": { "type": "boolean" }, + "options": { + "type": "array", + "description": "Declared choices on a `select` column.", + "items": { + "type": "object", + "properties": { "id": { "type": "string" }, "name": { "type": "string" } } + } + }, + "multiple": { "type": "boolean" } + } + } + } + } + }, + "rowCount": { "type": "integer" }, + "maxRows": { "type": "integer" }, + "createdAt": { "type": "string", "format": "date-time" }, + "updatedAt": { "type": "string", "format": "date-time" } + } + }, + "ErrorBody": { + "type": "object", + "required": ["error"], + "properties": { + "error": { + "type": "string", + "description": "Human-readable message naming the failing field/operator." + }, + "code": { + "type": "string", + "description": "Machine-readable code, present on domain validation failures." + } + } + } + }, + "responses": { + "ValidationError": { + "description": "Malformed request.", + "content": { + "application/json": { "schema": { "$ref": "#/components/schemas/ErrorBody" } } + } + }, + "Unauthorized": { + "description": "Missing or invalid API key.", + "content": { + "application/json": { "schema": { "$ref": "#/components/schemas/ErrorBody" } } + } + }, + "Forbidden": { + "description": "The key's workspace scope does not cover this workspace, or the caller lacks read access.", + "content": { + "application/json": { "schema": { "$ref": "#/components/schemas/ErrorBody" } } + } + }, + "NotFoundOrGated": { + "description": "Table not found — or the `tables-v2-api` feature flag is off for this caller, in which case the entire surface answers 404. The gate is evaluated after authorization, so a 404 never distinguishes rollout cohort from missing resource for callers without access.", + "content": { + "application/json": { "schema": { "$ref": "#/components/schemas/ErrorBody" } } + } + }, + "RateLimited": { + "description": "Rate limit exceeded for this key.", + "content": { + "application/json": { "schema": { "$ref": "#/components/schemas/ErrorBody" } } + } + } + } + } +} From 9d8fa6cf1e208fa1238247dafdb30577a47bfe13 Mon Sep 17 00:00:00 2001 From: Theodore Li Date: Wed, 29 Jul 2026 18:55:31 -0700 Subject: [PATCH 23/26] fix(table): close the review findings on the dual-grammar boundaries MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit From the PR #6067 review (greptile + bugbot), all verified before fixing: 1. Hybrid nodes on the async destructive routes (P1/High). The dual union's legacy branch accepts any non-empty object WITHOUT stripping, so a node carrying both a group key and leaf keys reached toLegacyFilter Zod-approved — and predicateToFilter converted it group-first, silently DROPPING the leaf and widening a select-all delete. predicateToFilter now throws on hybrids (lossless-or-throw, like its other rules), toLegacyFilter shape-validates first, and the GET native-predicate path shape-validates too. 2. A column literally named all/any (P1). NAME_PATTERN allows it, and isTablePredicate routed any object with those keys to the predicate compiler. It now requires the group value to be an ARRAY: the legacy equality shorthand and operator objects on such a column keep compiling as legacy, and an array-valued legacy condition was always a dropped no-op, so predicate precedence on arrays regresses nothing. 3. Bulk keying (P2). resolveBulkFilter validated predicates as NAME-keyed and translated unconditionally — wrong for the ID-keyed grid (session wire is identity). Validation now runs AFTER wire translation against STORAGE keys (new validateStoragePredicate), which is keying-correct for every caller and keeps the property that a typo'd column on a destructive path is a 400, not a silent match-nothing no-op. 4. 500s on downgrade rejection (Medium). delete-async called toLegacyFilter outside its try, and cancel-runs/columns-run mapped the throw to the generic 500. All three now return the validation message as a 400. 5. SortSpec broke the wire (High). requestJson threw on arrays of objects, so any active grid sort died client-side before the request — and the server contract rejected string-encoded values anyway, on both grammars. Arrays containing objects now travel as one JSON-string param (exactly what the serializer's own guard comment prescribed), and the rows/find query contracts decode JSON-string filter/sort/after before the union runs. Proven end-to-end with a real NextRequest for both grammars. Session bulk predicates are now id-keyed pass-through (matching the grid); name-keyed translation remains for INTERNAL_JWT workflow tools — tests updated to the corrected contract and extended for every finding. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_011M563cbFy2S74GvSDf2C3R --- .../api/table/[tableId]/cancel-runs/route.ts | 6 +++ .../api/table/[tableId]/columns/run/route.ts | 6 +++ .../[tableId]/delete-async/route.test.ts | 21 ++++++++ .../api/table/[tableId]/delete-async/route.ts | 16 +++++- .../api/table/[tableId]/rows/route.test.ts | 32 +++++++++++- .../sim/app/api/table/[tableId]/rows/route.ts | 26 ++++++++-- apps/sim/lib/api/client/request.test.ts | 30 ++++++++--- apps/sim/lib/api/client/request.ts | 20 ++++---- .../api/contracts/tables-predicate.test.ts | 36 +++++++++++++ apps/sim/lib/api/contracts/tables.ts | 27 ++++++++-- .../__tests__/converters.test.ts | 50 +++++++++++++++++++ .../sim/lib/table/query-builder/converters.ts | 28 ++++++++++- apps/sim/lib/table/query-builder/validate.ts | 36 +++++++++++-- 13 files changed, 298 insertions(+), 36 deletions(-) diff --git a/apps/sim/app/api/table/[tableId]/cancel-runs/route.ts b/apps/sim/app/api/table/[tableId]/cancel-runs/route.ts index 7e9d332a1e0..7dc24f1850a 100644 --- a/apps/sim/app/api/table/[tableId]/cancel-runs/route.ts +++ b/apps/sim/app/api/table/[tableId]/cancel-runs/route.ts @@ -5,6 +5,7 @@ import { parseRequest } from '@/lib/api/server' import { checkSessionOrInternalAuth } from '@/lib/auth/hybrid' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' +import { TableQueryValidationError } from '@/lib/table/errors' import { toLegacyFilter } from '@/lib/table/query-builder/converters' import { cancelWorkflowGroupRuns } from '@/lib/table/workflow-columns' import { accessError, checkAccess, tableFilterError } from '@/app/api/table/utils' @@ -61,6 +62,11 @@ export const POST = withRouteHandler(async (request: NextRequest, { params }: Ro return NextResponse.json({ success: true, data: { cancelled } }) } catch (error) { + // A predicate that Zod accepts but the downgrade rejects (hybrid node, + // eq-with-array, valueless op) is caller error, not a server fault. + if (error instanceof TableQueryValidationError) { + return NextResponse.json({ error: error.message }, { status: 400 }) + } logger.error(`[${requestId}] cancel-runs failed:`, error) return NextResponse.json({ error: 'Failed to cancel runs' }, { status: 500 }) } diff --git a/apps/sim/app/api/table/[tableId]/columns/run/route.ts b/apps/sim/app/api/table/[tableId]/columns/run/route.ts index 6bdfaea5ac2..e45a8de1a70 100644 --- a/apps/sim/app/api/table/[tableId]/columns/run/route.ts +++ b/apps/sim/app/api/table/[tableId]/columns/run/route.ts @@ -5,6 +5,7 @@ import { parseRequest } from '@/lib/api/server' import { checkSessionOrInternalAuth } from '@/lib/auth/hybrid' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' +import { TableQueryValidationError } from '@/lib/table/errors' import { toLegacyFilter } from '@/lib/table/query-builder/converters' import { runWorkflowColumn } from '@/lib/table/workflow-columns' import { accessError, checkAccess, tableFilterError } from '@/app/api/table/utils' @@ -60,6 +61,11 @@ export const POST = withRouteHandler(async (request: NextRequest, { params }: Ro return NextResponse.json({ success: true, data: { dispatchId } }) } catch (error) { + // A predicate that Zod accepts but the downgrade rejects (hybrid node, + // eq-with-array, valueless op) is caller error, not a server fault. + if (error instanceof TableQueryValidationError) { + return NextResponse.json({ error: error.message }, { status: 400 }) + } if (error instanceof Error && error.message === 'Invalid workspace ID') { return NextResponse.json({ error: 'Invalid workspace ID' }, { status: 400 }) } diff --git a/apps/sim/app/api/table/[tableId]/delete-async/route.test.ts b/apps/sim/app/api/table/[tableId]/delete-async/route.test.ts index 38b24e06fb0..cffb5dc8810 100644 --- a/apps/sim/app/api/table/[tableId]/delete-async/route.test.ts +++ b/apps/sim/app/api/table/[tableId]/delete-async/route.test.ts @@ -208,4 +208,25 @@ describe('POST /api/table/[tableId]/delete-async', () => { expect(mockReleaseJobClaim).toHaveBeenCalledWith('tbl_1', 'job-id-xyz') expect(mockRunTableDelete).not.toHaveBeenCalled() }) + + /** + * PR #6067 review finding (greptile P1 / bugbot High): a hybrid filter — group + * key AND leaf keys on one node — passes the dual-grammar union via the + * non-stripping legacy branch, and the downgrade used to convert group-first, + * silently dropping the leaf and WIDENING an async select-all delete. + */ + it('rejects a hybrid group+leaf filter with 400 instead of widening the delete', async () => { + const response = await makeRequest({ + workspaceId: 'workspace-1', + filter: { + all: [{ field: 'tenant_id', op: 'eq', value: 'acme' }], + field: 'status', + op: 'eq', + value: 'archived', + }, + }) + expect(response.status).toBe(400) + const body = await response.json() + expect(body.error).toMatch(/not both/) + }) }) diff --git a/apps/sim/app/api/table/[tableId]/delete-async/route.ts b/apps/sim/app/api/table/[tableId]/delete-async/route.ts index 291a985ed93..5cd7d295e3a 100644 --- a/apps/sim/app/api/table/[tableId]/delete-async/route.ts +++ b/apps/sim/app/api/table/[tableId]/delete-async/route.ts @@ -8,7 +8,9 @@ import { isTriggerDevEnabled } from '@/lib/core/config/env-flags' import { runDetached } from '@/lib/core/utils/background' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' +import type { Filter } from '@/lib/table' import { markTableDeleteFailed, runTableDelete } from '@/lib/table/delete-runner' +import { TableQueryValidationError } from '@/lib/table/errors' import { markTableJobRunning, releaseJobClaim } from '@/lib/table/jobs/service' import { assertRowDelete } from '@/lib/table/mutation-locks' import { toLegacyFilter } from '@/lib/table/query-builder/converters' @@ -46,8 +48,18 @@ export const POST = withRouteHandler(async (request: NextRequest, { params }: Ro const { tableId } = parsed.data.params const { workspaceId, filter: wireFilter, excludeRowIds, estimatedCount } = parsed.data.body // Dual-grammar wire: a predicate downgrades losslessly-or-throws to the - // legacy Filter the runners/persisted payloads still compile. - const filter = toLegacyFilter(wireFilter) + // legacy Filter the runners/persisted payloads still compile. A shape the + // union accepted but the downgrade rejects (hybrid node, eq-with-array) is + // caller error — 400, never the generic 500. + let filter: Filter | undefined + try { + filter = toLegacyFilter(wireFilter) + } catch (error) { + if (error instanceof TableQueryValidationError) { + return NextResponse.json({ error: error.message }, { status: 400 }) + } + throw error + } const access = await checkAccess(tableId, userId, 'write') if (!access.ok) return accessError(access, requestId, tableId) diff --git a/apps/sim/app/api/table/[tableId]/rows/route.test.ts b/apps/sim/app/api/table/[tableId]/rows/route.test.ts index 2b3fa64ea4b..9a83aef3920 100644 --- a/apps/sim/app/api/table/[tableId]/rows/route.test.ts +++ b/apps/sim/app/api/table/[tableId]/rows/route.test.ts @@ -290,13 +290,43 @@ describe('PUT/DELETE /api/table/[tableId]/rows — predicate filters', () => { return DELETE(req, { params: Promise.resolve({ tableId: 'tbl_1' }) }) } - it('PUT translates a name-keyed predicate to an id-keyed filter under SESSION auth', async () => { + /** + * Keying follows the caller (PR #6067 review): the grid authors ID-keyed + * predicates and the session wire is identity, so ids pass through — and a + * NAME under session auth is just an unknown storage key, rejected like any + * other typo rather than half-translated. + */ + it('PUT passes an id-keyed predicate through untouched under SESSION auth', async () => { + authAs('session') + const res = await callPut({ + workspaceId: 'workspace-1', + filter: { all: [{ field: 'col_aaa', op: 'eq', value: 'Ada' }] }, + data: { col_aaa: 'Grace' }, + }) + + expect(res.status).toBe(200) + const args = mockUpdateRowsByFilter.mock.calls[0][1] + expect(args.filter).toEqual({ $and: [{ col_aaa: 'Ada' }] }) + }) + + it('PUT rejects an unknown storage key under SESSION auth with 400', async () => { authAs('session') const res = await callPut({ workspaceId: 'workspace-1', filter: { all: [{ field: 'Name', op: 'eq', value: 'Ada' }] }, data: { col_aaa: 'Grace' }, }) + expect(res.status).toBe(400) + expect(mockUpdateRowsByFilter).not.toHaveBeenCalled() + }) + + it('PUT translates a name-keyed predicate for INTERNAL_JWT callers', async () => { + authAs('internal_jwt') + const res = await callPut({ + workspaceId: 'workspace-1', + filter: { all: [{ field: 'Name', op: 'eq', value: 'Ada' }] }, + data: { Name: 'Grace' }, + }) expect(res.status).toBe(200) const args = mockUpdateRowsByFilter.mock.calls[0][1] diff --git a/apps/sim/app/api/table/[tableId]/rows/route.ts b/apps/sim/app/api/table/[tableId]/rows/route.ts index 443a9a865d1..67652f86d82 100644 --- a/apps/sim/app/api/table/[tableId]/rows/route.ts +++ b/apps/sim/app/api/table/[tableId]/rows/route.ts @@ -27,9 +27,11 @@ import { } from '@/lib/table' import { TableQueryValidationError } from '@/lib/table/errors' import { isTablePredicate, predicateToFilter } from '@/lib/table/query-builder/converters' -import { validatePredicate } from '@/lib/table/query-builder/validate' +import { + validatePredicateShape, + validateStoragePredicate, +} from '@/lib/table/query-builder/validate' import { queryRows } from '@/lib/table/rows/service' -import { predicateToStorage } from '@/lib/table/select-values' import type { TablePredicate } from '@/lib/table/types' import { type RowWireTranslators, rowWireTranslators } from '@/app/api/table/row-wire' import { accessError, checkAccess, rowWriteErrorResponse } from '@/app/api/table/utils' @@ -60,8 +62,15 @@ function resolveBulkFilter( wire: RowWireTranslators ): Filter { if (isTablePredicate(raw)) { - validatePredicate(raw, schema.columns) - return predicateToFilter(predicateToStorage(raw, schema)) + // Shape first (keying-agnostic: hybrid nodes, leaf value rules), then let + // the wire translate — identity for the ID-keyed grid, names→ids for + // workflow tools — and validate the RESULT against storage keys. Post- + // translation, any unresolved field is a typo in the caller's own keying, + // and on a destructive path a typo must 400, not silently match nothing. + validatePredicateShape(raw) + const translated = wire.predicateIn(raw) + validateStoragePredicate(translated, schema.columns) + return predicateToFilter(translated) } return wire.filterIn(raw) } @@ -294,7 +303,14 @@ export const GET = withRouteHandler( table, { ...(validated.filter && isTablePredicate(validated.filter as Filter | TablePredicate) - ? { predicate: wire.predicateIn(validated.filter as TablePredicate) } + ? { + predicate: (() => { + // Shape-check first: nothing upstream validates this branch, and + // an unchecked hybrid node would silently widen the result. + validatePredicateShape(validated.filter as TablePredicate) + return wire.predicateIn(validated.filter as TablePredicate) + })(), + } : { filter: validated.filter ? wire.filterIn(validated.filter as Filter) : undefined }), sort: resolveWireSort(validated.sort as Sort | SortSpec | undefined, wire), limit: validated.limit, diff --git a/apps/sim/lib/api/client/request.test.ts b/apps/sim/lib/api/client/request.test.ts index e28ed0837b2..f7a03a53574 100644 --- a/apps/sim/lib/api/client/request.test.ts +++ b/apps/sim/lib/api/client/request.test.ts @@ -54,18 +54,36 @@ describe('requestJson query serialization', () => { expect(calledUrl).not.toContain('[object Object]') }) - it('throws instead of silently corrupting an array-of-objects query param', async () => { - mockFetchReturning({ ok: true }) + it('JSON-encodes an array-of-objects query param instead of corrupting or throwing', async () => { + // A SortSpec ([{field, direction}]) is the everyday case: repeat-append + // would send "[object Object]", so the whole array travels as ONE JSON + // string param, mirroring plain objects; the server contract decodes it. + const fetchMock = mockFetchReturning({ ok: true }) - const badContract = defineRouteContract({ + const contract = defineRouteContract({ method: 'GET', path: '/api/test', query: z.object({ items: z.array(z.object({ a: z.string() })) }), response: { mode: 'json', schema: z.object({ ok: z.boolean() }) }, }) - await expect(requestJson(badContract, { query: { items: [{ a: 'x' }] } })).rejects.toThrow( - /arrays of objects are not URL-safe/ - ) + await requestJson(contract, { query: { items: [{ a: 'x' }] } }) + const url = String(fetchMock.mock.calls[0][0]) + expect(decodeURIComponent(url)).toContain('items=[{"a":"x"}]') + }) + + it('keeps repeat-append for scalar arrays', async () => { + const fetchMock = mockFetchReturning({ ok: true }) + + const contract = defineRouteContract({ + method: 'GET', + path: '/api/test', + query: z.object({ tags: z.array(z.string()) }), + response: { mode: 'json', schema: z.object({ ok: z.boolean() }) }, + }) + + await requestJson(contract, { query: { tags: ['a', 'b'] } }) + const url = String(fetchMock.mock.calls[0][0]) + expect(url).toContain('tags=a&tags=b') }) }) diff --git a/apps/sim/lib/api/client/request.ts b/apps/sim/lib/api/client/request.ts index a68eb96fd13..cf4b5cef479 100644 --- a/apps/sim/lib/api/client/request.ts +++ b/apps/sim/lib/api/client/request.ts @@ -67,19 +67,17 @@ function appendQuery(path: string, query: unknown): string { if (value === undefined || value === null || value === '') continue if (Array.isArray(value)) { + // An array of objects (e.g. a SortSpec) is not repeat-append-able — each + // item would stringify to "[object Object]" and silently corrupt the + // request (the knowledge tagFilters bug). Encode the WHOLE array as one + // JSON string param, mirroring how plain objects are sent below; the + // server-side contract decodes it. Scalar arrays keep repeat-append. + if (value.some((item) => item !== null && typeof item === 'object')) { + searchParams.set(key, JSON.stringify(value)) + continue + } for (const item of value) { if (item === undefined || item === null || item === '') continue - // A non-scalar in a query array would stringify to "[object Object]" and - // silently corrupt the request. Encode such values as a single JSON - // string param and decode them server-side instead. Failing loudly here - // keeps the boundary honest (this is how the knowledge tagFilters bug - // shipped undetected). - if (typeof item === 'object') { - throw new Error( - `Cannot serialize query param "${key}": arrays of objects are not URL-safe — ` + - 'encode the value as a JSON string param and decode it server-side.' - ) - } searchParams.append(key, String(item)) } continue diff --git a/apps/sim/lib/api/contracts/tables-predicate.test.ts b/apps/sim/lib/api/contracts/tables-predicate.test.ts index 5d6c915990f..8f45d758822 100644 --- a/apps/sim/lib/api/contracts/tables-predicate.test.ts +++ b/apps/sim/lib/api/contracts/tables-predicate.test.ts @@ -10,6 +10,7 @@ import { deleteTableRowsBodySchema, predicateSchema, rowQueryBodySchema, + tableRowsQuerySchema, updateRowsByFilterBodySchema, } from '@/lib/api/contracts/tables' import { validatePredicate } from '@/lib/table/query-builder/validate' @@ -223,3 +224,38 @@ describe('hybrid group+leaf nodes are rejected, not silently narrowed', () => { ).toBe(true) }) }) + +/** + * Wire transport: requestJson serializes structured query params as JSON + * strings; jsonQueryValue decodes them before the union runs. Without it, a + * sorted or filtered grid request 400s at the boundary. + */ +describe('query-string JSON transport (jsonQueryValue)', () => { + it('decodes string-encoded predicate, legacy filter, and sort spec', () => { + const parsed = rowQueryStringSchemaProbe({ + workspaceId: 'ws-1', + filter: JSON.stringify({ all: [{ field: 'a', op: 'eq', value: 1 }] }), + sort: JSON.stringify([{ field: 'a', direction: 'asc' }]), + }) + expect(parsed.filter).toEqual({ all: [{ field: 'a', op: 'eq', value: 1 }] }) + expect(parsed.sort).toEqual([{ field: 'a', direction: 'asc' }]) + + const legacy = rowQueryStringSchemaProbe({ + workspaceId: 'ws-1', + filter: JSON.stringify({ status: { $eq: 'x' } }), + sort: JSON.stringify({ status: 'desc' }), + }) + expect(legacy.filter).toEqual({ status: { $eq: 'x' } }) + expect(legacy.sort).toEqual({ status: 'desc' }) + }) + + it('still rejects a non-JSON garbage string with the real schema error', () => { + expect(() => rowQueryStringSchemaProbe({ workspaceId: 'ws-1', filter: 'not json' })).toThrow() + }) +}) + +function rowQueryStringSchemaProbe(input: Record) { + const result = tableRowsQuerySchema.safeParse(input) + if (!result.success) throw new Error(JSON.stringify(result.error.issues[0])) + return result.data +} diff --git a/apps/sim/lib/api/contracts/tables.ts b/apps/sim/lib/api/contracts/tables.ts index c0af27b34c9..5bccaa52069 100644 --- a/apps/sim/lib/api/contracts/tables.ts +++ b/apps/sim/lib/api/contracts/tables.ts @@ -545,20 +545,37 @@ export const deleteTableRowsBodySchema = z message: 'Provide either filter or rowIds, but not both', }) +/** + * Query-param transport for a structured value: `requestJson` serializes + * objects/arrays into a single JSON-string param, and this decodes it before + * the real schema runs. Non-JSON strings pass through untouched so the inner + * schema produces the real error; already-parsed values (POST bodies reusing a + * schema) are untouched too. + */ +const jsonQueryValue = (schema: S) => + z.preprocess((value) => { + if (typeof value !== 'string' || value === '') return value + try { + return JSON.parse(value) + } catch { + return value + } + }, schema) + /** Unrefined base so v1 contracts can `.extend()` — consumers use {@link tableRowsQuerySchema}. */ export const tableRowsQueryBaseSchema = z.object({ workspaceId: workspaceIdSchema, // Dual-grammar during the v2 transition: the strict predicate tree wins the // union; anything else falls through to the legacy `$`-object. Same for sort: // an ordered spec array vs the legacy `{col: dir}` record. - filter: z.union([predicateSchema, domainObjectSchema()]).optional(), - sort: z.union([sortSpecSchema, domainObjectSchema()]).optional(), + filter: jsonQueryValue(z.union([predicateSchema, domainObjectSchema()])).optional(), + sort: jsonQueryValue(z.union([sortSpecSchema, domainObjectSchema()])).optional(), /** * Keyset cursor `(orderKey, id)` for the default row order — each page is an index seek * instead of OFFSET's scan-and-discard. Mutually exclusive with `sort` (cursors only make * sense on the default order); takes precedence over `offset`. */ - after: domainObjectSchema().optional(), + after: jsonQueryValue(domainObjectSchema()).optional(), limit: z .preprocess( (value) => @@ -865,8 +882,8 @@ export type RowQueryResponse = ContractJsonResponse export const findTableRowsQuerySchema = z.object({ workspaceId: workspaceIdSchema, q: requiredFieldSchema('Search query is required'), - filter: z.union([predicateSchema, domainObjectSchema()]).optional(), - sort: z.union([sortSpecSchema, domainObjectSchema()]).optional(), + filter: jsonQueryValue(z.union([predicateSchema, domainObjectSchema()])).optional(), + sort: jsonQueryValue(z.union([sortSpecSchema, domainObjectSchema()])).optional(), }) /** One matching cell: its 0-based ordinal in the filtered+sorted view, its row id, and the column name. */ diff --git a/apps/sim/lib/table/query-builder/__tests__/converters.test.ts b/apps/sim/lib/table/query-builder/__tests__/converters.test.ts index e978b184b2d..06e3fb11659 100644 --- a/apps/sim/lib/table/query-builder/__tests__/converters.test.ts +++ b/apps/sim/lib/table/query-builder/__tests__/converters.test.ts @@ -16,6 +16,7 @@ import { pruneFilterForColumns, prunePredicateForColumns, sortRulesToSortSpec, + toLegacyFilter, } from '@/lib/table/query-builder/converters' import type { ColumnDefinition, FilterRule, SortRule, TablePredicate } from '@/lib/table/types' @@ -447,3 +448,52 @@ describe('isTablePredicate', () => { it('prunePredicateForColumns fails closed on a malformed value instead of throwing', () => { expect(prunePredicateForColumns({ column: 'name', operator: 'eq' } as never, [])).toBeNull() }) + +/** + * Review findings from PR #6067 (greptile P1 ×2, bugbot High). The dual-grammar + * union's legacy branch accepts any non-empty object WITHOUT stripping, so a + * hybrid node reaches the downgrade Zod-approved — and predicateToFilter used + * to convert it group-first, silently DROPPING the leaf half and widening the + * filter on the async destructive routes (delete-async, cancel-runs, columns/run). + */ +describe('toLegacyFilter / predicateToFilter hybrid safety', () => { + const hybrid = { + all: [{ field: 'tenant_id', op: 'eq', value: 'acme' }], + field: 'status', + op: 'eq', + value: 'archived', + } as never + + it('predicateToFilter throws on a hybrid node instead of dropping the leaf', () => { + expect(() => predicateToFilter(hybrid)).toThrow(/not both/) + }) + + it('toLegacyFilter shape-validates before converting', () => { + expect(() => toLegacyFilter(hybrid)).toThrow(/not both/) + // malformed group value is also caught, not crashed on + expect(() => toLegacyFilter({ all: [{ all: 'x' }] } as never)).toThrow() + }) + + it('toLegacyFilter passes a well-formed predicate and a legacy filter through', () => { + expect(toLegacyFilter({ all: [{ field: 'a', op: 'eq', value: 1 }] })).toEqual({ + $and: [{ a: 1 }], + }) + expect(toLegacyFilter({ status: 'archived' })).toEqual({ status: 'archived' }) + }) +}) + +describe('isTablePredicate vs columns literally named all/any', () => { + it('routes a non-array all/any value to the legacy compiler (a real column)', () => { + // NAME_PATTERN allows a column named "all". Equality shorthand and operator + // objects on it are legacy filters, not predicate groups. + expect(isTablePredicate({ all: 'x' } as never)).toBe(false) + expect(isTablePredicate({ any: { $eq: 'x' } } as never)).toBe(false) + }) + + it('array-valued all/any is a predicate group (documented precedence)', () => { + // A legacy filter with an ARRAY on a regular field was always a dropped + // no-op condition, so predicate precedence here regresses nothing. + expect(isTablePredicate({ all: [] })).toBe(true) + expect(isTablePredicate({ any: [{ field: 'a', op: 'eq', value: 1 }] })).toBe(true) + }) +}) diff --git a/apps/sim/lib/table/query-builder/converters.ts b/apps/sim/lib/table/query-builder/converters.ts index a965b17e194..df27b488e3c 100644 --- a/apps/sim/lib/table/query-builder/converters.ts +++ b/apps/sim/lib/table/query-builder/converters.ts @@ -10,6 +10,7 @@ import { MULTI_SELECT_FILTER_OPERATORS, SINGLE_SELECT_FILTER_OPERATORS, } from '@/lib/table/query-builder/constants' +import { validatePredicateShape } from '@/lib/table/query-builder/validate' import type { ColumnDefinition, Filter, @@ -148,7 +149,14 @@ export function prunePredicateForColumns( * wire fields. Group-first, matching every other discrimination site. */ export function isTablePredicate(value: Filter | TablePredicate): value is TablePredicate { - return 'all' in value || 'any' in value + // Require the group value to be an ARRAY: a legacy filter on a real column + // that happens to be named `all`/`any` (allowed by NAME_PATTERN) uses the + // equality shorthand ({ all: "x" }) or an operator object — neither is + // array-valued, so both keep routing to the legacy compiler. A column named + // all/any holding a literal array was already a dropped no-op condition in + // the legacy grammar, so predicate precedence on arrays regresses nothing. + const v = value as Record + return ('all' in v && Array.isArray(v.all)) || ('any' in v && Array.isArray(v.any)) } /** Converts a single UI sort rule to a Sort object for API queries. */ @@ -457,6 +465,16 @@ function predicateLeafToFilterValue(p: Predicate): Filter[string] { */ export function predicateToFilter(predicate: TablePredicate): Filter { const nodeToFilter = (node: Predicate | TablePredicate): Filter => { + // A hybrid node (group key AND leaf keys) would convert group-first here, + // silently DROPPING the leaf half — which widens the filter, and on the + // bulk delete/update paths that is destructive. Lossless-or-throw, like + // every other non-representable shape in this converter. + if (('all' in node || 'any' in node) && 'field' in node) { + throw new TableQueryValidationError( + 'A filter node must be either a group ({ all | any: [...] }) or a condition ({ field, op, value }), not both.', + 'INVALID_FILTER' + ) + } // Group-first, matching isPredicateGroup/validateNode/buildPredicateNode. A // leaf-first test would execute a different predicate than the one validated. if ('all' in node) return { $and: node.all.map(nodeToFilter) } @@ -475,7 +493,13 @@ export function predicateToFilter(predicate: TablePredicate): Filter { */ export function toLegacyFilter(filter: Filter | TablePredicate | undefined): Filter | undefined { if (!filter) return undefined - return isTablePredicate(filter) ? predicateToFilter(filter) : filter + if (!isTablePredicate(filter)) return filter + // Shape-validate before converting: the dual-grammar union's legacy branch + // accepts any non-empty object without stripping, so a hybrid or malformed + // tree reaches here Zod-approved. The predicate may be name- OR id-keyed + // (grid vs tools), so only the keying-agnostic checks apply. + validatePredicateShape(filter) + return predicateToFilter(filter) } /** Downgrades a dual-grammar wire sort (ordered spec array or legacy record). */ diff --git a/apps/sim/lib/table/query-builder/validate.ts b/apps/sim/lib/table/query-builder/validate.ts index c66f1ea9b59..ea6a44c2933 100644 --- a/apps/sim/lib/table/query-builder/validate.ts +++ b/apps/sim/lib/table/query-builder/validate.ts @@ -1,4 +1,5 @@ import { isRecordLike } from '@sim/utils/object' +import { getColumnId } from '@/lib/table/column-keys' import { NAME_PATTERN } from '@/lib/table/constants' import { TableQueryValidationError } from '@/lib/table/errors' import type { @@ -59,15 +60,15 @@ function validateFieldName(field: string): void { } } -function validateLeaf(leaf: Predicate, typeByName: Map): void { +function validateLeaf(leaf: Predicate, typeByName: Map | null): void { validateFieldName(leaf.field) - if (!typeByName.has(leaf.field)) { + if (typeByName && !typeByName.has(leaf.field)) { throw new TableQueryValidationError( `Unknown filter column "${leaf.field}". It is not a column on this table.`, 'INVALID_FILTER' ) } - if (typeByName.get(leaf.field) === 'json' && CONTAINMENT_OPS.has(leaf.op)) { + if (typeByName?.get(leaf.field) === 'json' && CONTAINMENT_OPS.has(leaf.op)) { throw new TableQueryValidationError( `Operator "${leaf.op}" is not supported on json column "${leaf.field}" — use like/ilike for text match, or isNull/isNotNull.`, 'INVALID_FILTER' @@ -106,7 +107,17 @@ function validateLeaf(leaf: Predicate, typeByName: Map): voi } } -function validateNode(node: PredicateNode, typeByName: Map): void { +/** + * Structure-only validation: hybrid nodes, group shapes, leaf value rules — + * everything that does not require knowing the table's columns. Used by the + * dual-grammar boundaries where the predicate may be NAME- or ID-keyed, so a + * column-existence check against either keying would be wrong. + */ +export function validatePredicateShape(predicate: TablePredicate): void { + validateNode(predicate, null) +} + +function validateNode(node: PredicateNode, typeByName: Map | null): void { // Guard before the `in` checks below: an untrusted caller (copilot args, a raw // block value) can hand us a string/number/null, where `'all' in node` throws // a raw TypeError. Fail with a clean, actionable message instead. @@ -175,3 +186,20 @@ export function validateSortSpec(spec: SortSpec, columns: ColumnDefinition[]): v } } } + +/** + * Validates a STORAGE-keyed predicate — leaf fields are column ids (plus the + * system columns, which keep their names). Runs AFTER wire translation, which + * makes it keying-correct for every caller: a session caller's ids are already + * storage keys, and a workflow tool's names have just been translated — so any + * field left unresolved here is a typo, and on the bulk write paths a typo must + * 400 rather than compile to a filter that silently matches nothing. + */ +export function validateStoragePredicate( + predicate: TablePredicate, + columns: ColumnDefinition[] +): void { + const typeById = new Map(columns.map((c) => [getColumnId(c), c.type])) + for (const [name, type] of SYSTEM_COLUMN_TYPES) typeById.set(name, type) + validateNode(predicate, typeById) +} From 3ebb6398f4bf3722fb4dd1a45c1ed734b53b7006 Mon Sep 17 00:00:00 2001 From: Theodore Li Date: Wed, 29 Jul 2026 19:20:06 -0700 Subject: [PATCH 24/26] fix(table): reject empty filter groups and bind cursors to their sort MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bugbot round 2 on PR #6067, both verified: Empty groups (High). `{all: []}` fails the strict predicate branch (.min(1)) but slips the dual union via the legacy branch — an empty ARRAY inside a non-empty OBJECT — then downgraded to `{$and: []}`, which compiles to no WHERE clause: a run/cancel/delete scope silently widened to every row. validatePredicateShape now mirrors the contract's .min(1), which closes it at every dual-grammar boundary at once (toLegacyFilter, resolveBulkFilter, the GET native path). Cursor↔sort binding (Medium). CURSOR_SORT_CONFLICT only fired for keyset cursors; offset cursors — the shape sorted views actually emit — carried no record of their ordering, so one minted under sort A replayed under sort B (or none) silently paged the wrong sequence. Offset cursors are now stamped with a canonical fingerprint of their sort at mint (queryRows), and a shared assertCursorSortBinding enforces the match at all three consumers (both query routes and the copilot executor), replacing the three hand-rolled keyset-only checks. Keyset/compound cursors stay default-order-only by construction. The OpenAPI cursor wording now states the binding rather than overclaiming. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_011M563cbFy2S74GvSDf2C3R --- apps/docs/openapi-v2-tables.json | 2 +- .../app/api/table/[tableId]/query/route.ts | 19 ++--- .../api/v2/tables/[tableId]/query/route.ts | 19 ++--- .../copilot/tools/server/table/user-table.ts | 16 ++-- .../service-filter-threading.test.ts | 7 +- .../__tests__/converters.test.ts | 4 + .../query-builder/__tests__/validate.test.ts | 26 ++++++- apps/sim/lib/table/query-builder/validate.ts | 9 +++ apps/sim/lib/table/rows/cursor.test.ts | 78 +++++++++++++++++++ apps/sim/lib/table/rows/cursor.ts | 70 +++++++++++++++-- apps/sim/lib/table/rows/service.ts | 1 + 11 files changed, 209 insertions(+), 42 deletions(-) create mode 100644 apps/sim/lib/table/rows/cursor.test.ts diff --git a/apps/docs/openapi-v2-tables.json b/apps/docs/openapi-v2-tables.json index 0641eaaaef3..15757b19f1e 100644 --- a/apps/docs/openapi-v2-tables.json +++ b/apps/docs/openapi-v2-tables.json @@ -60,7 +60,7 @@ "post": { "operationId": "v2QueryTableRows", "summary": "Query Rows", - "description": "Query rows with a typed predicate filter, an ordered sort spec, and opaque cursor pagination. Row `data` is keyed by column NAME; `select` cells return option names, and filter operands on select columns accept option names (resolved case-insensitively).\n\n**Pagination contract:** page by passing the previous response's `nextCursor` back as `cursor`, and stop only when it is `null` — a page may return fewer than `limit` rows and still have more behind it, so page fullness is never a termination signal. A cursor encodes a position in the default row order and cannot be combined with `sort` (400 `CURSOR_SORT_CONFLICT`). `totalCount` is computed on the first page only (requests with a `cursor` return `totalCount: null`).", + "description": "Query rows with a typed predicate filter, an ordered sort spec, and opaque cursor pagination. Row `data` is keyed by column NAME; `select` cells return option names, and filter operands on select columns accept option names (resolved case-insensitively).\n\n**Pagination contract:** page by passing the previous response's `nextCursor` back as `cursor`, and stop only when it is `null` — a page may return fewer than `limit` rows and still have more behind it, so page fullness is never a termination signal. A cursor is bound to the exact query shape it was minted under: keyset cursors to the default row order, offset cursors (sorted views) to that sort. Replaying one under a different `sort` returns 400 `CURSOR_SORT_CONFLICT`. `totalCount` is computed on the first page only (requests with a `cursor` return `totalCount: null`).", "tags": ["Tables v2"], "parameters": [ { diff --git a/apps/sim/app/api/table/[tableId]/query/route.ts b/apps/sim/app/api/table/[tableId]/query/route.ts index 896d60d1873..34a162200d1 100644 --- a/apps/sim/app/api/table/[tableId]/query/route.ts +++ b/apps/sim/app/api/table/[tableId]/query/route.ts @@ -10,7 +10,7 @@ import type { Sort, TableSchema } from '@/lib/table' import { buildIdByName, sortSpecNamesToIds } from '@/lib/table/column-keys' import { TableQueryValidationError } from '@/lib/table/errors' import { validatePredicate, validateSortSpec } from '@/lib/table/query-builder/validate' -import { decodeCursor } from '@/lib/table/rows/cursor' +import { assertCursorSortBinding, decodeCursor } from '@/lib/table/rows/cursor' import { queryRows } from '@/lib/table/rows/service' import { predicateToStorage } from '@/lib/table/select-values' import { rowWireTranslators } from '@/app/api/table/row-wire' @@ -63,19 +63,6 @@ export const POST = withRouteHandler(async (request: NextRequest, context: RowQu const wire = rowWireTranslators(authResult.authType, schema) const cursor = body.cursor ? decodeCursor(body.cursor) : undefined - // A keyset cursor encodes a position on the DEFAULT `(order_key, id)` - // order; combining it with a custom order would silently return page 1 - // of the sorted view relabeled as a next page. - if (cursor?.after && body.sort?.length) { - return NextResponse.json( - { - error: 'Cursor is not valid for a sorted query. Restart paging without the cursor.', - code: 'CURSOR_SORT_CONFLICT', - }, - { status: 400 } - ) - } - // Predicate/sort fields are column-NAME-keyed by construction (the caller // authors names), so validate against the schema then translate names → // storage ids unconditionally — unlike row data, this is not authType-dependent. @@ -94,6 +81,10 @@ export const POST = withRouteHandler(async (request: NextRequest, context: RowQu ? Object.fromEntries(sortSpec.map((s) => [s.field, s.direction])) : undefined + // Cursor↔sort binding: keyset cursors are default-order only; an offset + // cursor must be replayed under the exact sort it was minted with. + if (cursor) assertCursorSortBinding(cursor, sort) + const result = await queryRows( table, { diff --git a/apps/sim/app/api/v2/tables/[tableId]/query/route.ts b/apps/sim/app/api/v2/tables/[tableId]/query/route.ts index 1ddcfcb27e4..4337c0b402c 100644 --- a/apps/sim/app/api/v2/tables/[tableId]/query/route.ts +++ b/apps/sim/app/api/v2/tables/[tableId]/query/route.ts @@ -10,7 +10,7 @@ import { buildIdByName, sortSpecNamesToIds } from '@/lib/table' import { namedRowMapper } from '@/lib/table/cell-format' import { TableQueryValidationError } from '@/lib/table/errors' import { validatePredicate, validateSortSpec } from '@/lib/table/query-builder/validate' -import { decodeCursor } from '@/lib/table/rows/cursor' +import { assertCursorSortBinding, decodeCursor } from '@/lib/table/rows/cursor' import { queryRows } from '@/lib/table/rows/service' import { predicateToStorage } from '@/lib/table/select-values' import { accessError, checkAccess, tablesV2GateError } from '@/app/api/table/utils' @@ -72,18 +72,6 @@ export const POST = withRouteHandler(async (request: NextRequest, context: Query const schema = table.schema as TableSchema const cursor = cursorToken ? decodeCursor(cursorToken) : undefined - // A keyset cursor is bound to the DEFAULT `(order_key, id)` order; combining - // it with a custom sort would relabel page 1 of the sorted view as a next page. - if (cursor?.after && sort?.length) { - return NextResponse.json( - { - error: 'Cursor is not valid for a sorted query. Restart paging without the cursor.', - code: 'CURSOR_SORT_CONFLICT', - }, - { status: 400, headers: PRIVATE_NO_STORE } - ) - } - const idByName = buildIdByName(schema) // Fuses the id→name key remap with select-cell value formatting, so a select // cell surfaces its option NAME rather than the stored option id. @@ -102,6 +90,11 @@ export const POST = withRouteHandler(async (request: NextRequest, context: Query ? Object.fromEntries(sortSpec.map((s) => [s.field, s.direction])) : undefined + // A cursor is only valid for the query shape it was minted under: keyset + // cursors bind to the default order, offset cursors to their sort. Runs on + // the STORAGE-keyed sort so the fingerprint matches what queryRows stamped. + if (cursor) assertCursorSortBinding(cursor, sortObj) + // Public default is a bounded page (unlike the internal surface's unbounded // omit). `limit=0` is the explicit unbounded opt-in. const effectiveLimit = diff --git a/apps/sim/lib/copilot/tools/server/table/user-table.ts b/apps/sim/lib/copilot/tools/server/table/user-table.ts index eb811b49587..9f66adcb9da 100644 --- a/apps/sim/lib/copilot/tools/server/table/user-table.ts +++ b/apps/sim/lib/copilot/tools/server/table/user-table.ts @@ -1,6 +1,6 @@ import { AuditAction, AuditResourceType, recordAudit } from '@sim/audit' import { createLogger } from '@sim/logger' -import { toError } from '@sim/utils/errors' +import { getErrorMessage, toError } from '@sim/utils/errors' import { generateId, generateShortId } from '@sim/utils/id' import { UserTable } from '@/lib/copilot/generated/tool-catalog-v1' import { @@ -49,7 +49,7 @@ import { markTableJobRunning, releaseJobClaim } from '@/lib/table/jobs/service' import { assertRowDelete, assertRowUpdate, patchColumnIds } from '@/lib/table/mutation-locks' import { predicateToFilter } from '@/lib/table/query-builder/converters' import { validatePredicate, validateSortSpec } from '@/lib/table/query-builder/validate' -import { decodeCursor } from '@/lib/table/rows/cursor' +import { assertCursorSortBinding, decodeCursor } from '@/lib/table/rows/cursor' import { batchInsertRows, batchUpdateRows, @@ -700,11 +700,13 @@ export const userTableServerTool: BaseServerTool // physically can't page). A keyset cursor is bound to the default order, // so it can't be combined with a fresh sort. const cursor = args.cursor ? decodeCursor(args.cursor) : undefined - if (cursor?.after && sort) { - return { - success: false, - message: - 'Cursor is not valid for a sorted query. Restart paging without the cursor (omit it on the first sorted page).', + if (cursor) { + try { + // Keyset cursors bind to the default order; offset cursors to the + // exact sort they were minted under. + assertCursorSortBinding(cursor, sort) + } catch (bindError) { + return { success: false, message: getErrorMessage(bindError, 'Invalid cursor') } } } diff --git a/apps/sim/lib/table/__tests__/service-filter-threading.test.ts b/apps/sim/lib/table/__tests__/service-filter-threading.test.ts index 290a709eb23..889e75de2d8 100644 --- a/apps/sim/lib/table/__tests__/service-filter-threading.test.ts +++ b/apps/sim/lib/table/__tests__/service-filter-threading.test.ts @@ -403,7 +403,12 @@ describe('queryRows byte budget', () => { expect(dbChainMockFns.offset).toHaveBeenCalledWith(40) expect(result.rows).toHaveLength(1) - expect(decodeCursor(result.nextCursor as string)).toEqual({ offset: 41 }) + // The offset cursor is stamped with the sort it was minted under, so a + // replay against a different ordering is rejected instead of paging wrong. + expect(decodeCursor(result.nextCursor as string)).toEqual({ + offset: 41, + sortKey: JSON.stringify([['name', 'asc']]), + }) }) it('emits a compound cursor when rows past the inbound anchor are unkeyed', async () => { diff --git a/apps/sim/lib/table/query-builder/__tests__/converters.test.ts b/apps/sim/lib/table/query-builder/__tests__/converters.test.ts index 06e3fb11659..e389b0ad062 100644 --- a/apps/sim/lib/table/query-builder/__tests__/converters.test.ts +++ b/apps/sim/lib/table/query-builder/__tests__/converters.test.ts @@ -497,3 +497,7 @@ describe('isTablePredicate vs columns literally named all/any', () => { expect(isTablePredicate({ any: [{ field: 'a', op: 'eq', value: 1 }] })).toBe(true) }) }) + +it('toLegacyFilter rejects an empty group instead of downgrading it to no WHERE', () => { + expect(() => toLegacyFilter({ all: [] })).toThrow(/at least one condition/) +}) diff --git a/apps/sim/lib/table/query-builder/__tests__/validate.test.ts b/apps/sim/lib/table/query-builder/__tests__/validate.test.ts index 9c7a7855024..cad250b0aa1 100644 --- a/apps/sim/lib/table/query-builder/__tests__/validate.test.ts +++ b/apps/sim/lib/table/query-builder/__tests__/validate.test.ts @@ -3,7 +3,11 @@ */ import { describe, expect, it } from 'vitest' import { TableQueryValidationError } from '@/lib/table/errors' -import { validatePredicate, validateSortSpec } from '@/lib/table/query-builder/validate' +import { + validatePredicate, + validatePredicateShape, + validateSortSpec, +} from '@/lib/table/query-builder/validate' import type { ColumnDefinition } from '@/lib/table/types' const COLS: ColumnDefinition[] = [ @@ -175,3 +179,23 @@ describe('validatePredicate — legacy $-grammar diagnostics', () => { ) }) }) + +/** + * Bugbot round 2: `{all: []}` passes the dual union via the non-empty-OBJECT + * legacy branch, then compiled to no WHERE clause — silently widening a run / + * cancel / delete scope to every row. The shape validator now mirrors the Zod + * contract's .min(1). + */ +describe('empty groups are rejected at every layer', () => { + it('validatePredicateShape rejects an empty group', () => { + expect(() => validatePredicateShape({ all: [] })).toThrow(/at least one condition/) + expect(() => validatePredicateShape({ any: [] })).toThrow(/at least one condition/) + expect(() => validatePredicateShape({ all: [{ any: [] }] } as never)).toThrow( + /at least one condition/ + ) + }) + + it('validatePredicate rejects it too', () => { + expect(() => validatePredicate({ all: [] }, COLS)).toThrow(/at least one condition/) + }) +}) diff --git a/apps/sim/lib/table/query-builder/validate.ts b/apps/sim/lib/table/query-builder/validate.ts index ea6a44c2933..8448668117a 100644 --- a/apps/sim/lib/table/query-builder/validate.ts +++ b/apps/sim/lib/table/query-builder/validate.ts @@ -145,6 +145,15 @@ function validateNode(node: PredicateNode, typeByName: Map | 'INVALID_FILTER' ) } + // Mirrors the Zod contract's .min(1). An empty group slips the strict union + // (falling to the non-empty-OBJECT legacy branch) and compiles to no WHERE + // clause — which on the run/cancel/delete scopes silently means EVERY row. + if (members.length === 0) { + throw new TableQueryValidationError( + 'A filter group must contain at least one condition.', + 'INVALID_FILTER' + ) + } for (const child of members) validateNode(child, typeByName) return } diff --git a/apps/sim/lib/table/rows/cursor.test.ts b/apps/sim/lib/table/rows/cursor.test.ts new file mode 100644 index 00000000000..bae39a35c66 --- /dev/null +++ b/apps/sim/lib/table/rows/cursor.test.ts @@ -0,0 +1,78 @@ +/** + * @vitest-environment node + * + * Opaque cursor encode/decode and the cursor↔sort binding. A cursor encodes a + * position in one specific ordering; replaying it under any other ordering + * silently pages the wrong sequence, so binding violations must throw + * CURSOR_SORT_CONFLICT rather than return wrong rows. + */ +import { describe, expect, it } from 'vitest' +import { TableQueryValidationError } from '@/lib/table/errors' +import { + assertCursorSortBinding, + canonicalSortKey, + decodeCursor, + encodeCursor, +} from '@/lib/table/rows/cursor' + +const ROW = { id: 'row_1', orderKey: 'a1' } + +describe('cursor↔sort binding (bugbot round 2)', () => { + it('stamps an offset cursor with the sort it was minted under', () => { + const token = encodeCursor({ + lastRow: { id: 'row_1', orderKey: null }, + keysetValid: false, + nextOffset: 100, + sort: { col_a: 'desc' }, + }) + const decoded = decodeCursor(token) + expect(decoded.offset).toBe(100) + expect(decoded.sortKey).toBe(canonicalSortKey({ col_a: 'desc' })) + }) + + it('accepts replay under the identical sort', () => { + const decoded = { offset: 100, sortKey: canonicalSortKey({ col_a: 'desc' }) } + expect(() => assertCursorSortBinding(decoded, { col_a: 'desc' })).not.toThrow() + }) + + it('rejects replay under a DIFFERENT sort', () => { + const decoded = { offset: 100, sortKey: canonicalSortKey({ col_a: 'desc' }) } + for (const sort of [{ col_a: 'asc' as const }, { col_b: 'desc' as const }, undefined]) { + expect(() => assertCursorSortBinding(decoded, sort)).toThrow(TableQueryValidationError) + try { + assertCursorSortBinding(decoded, sort) + } catch (e) { + expect((e as TableQueryValidationError).code).toBe('CURSOR_SORT_CONFLICT') + } + } + }) + + it('rejects adding a sort to an unsorted offset cursor', () => { + const token = encodeCursor({ + lastRow: { id: 'row_1', orderKey: null }, + keysetValid: false, + nextOffset: 50, + }) + const decoded = decodeCursor(token) + expect(decoded.sortKey).toBeUndefined() + expect(() => assertCursorSortBinding(decoded, { col_a: 'asc' })).toThrow( + /different sort|sorted query/ + ) + expect(() => assertCursorSortBinding(decoded, undefined)).not.toThrow() + }) + + it('keyset cursors stay default-order only and never carry a sort stamp', () => { + const token = encodeCursor({ lastRow: ROW, keysetValid: true, nextOffset: 10 }) + const decoded = decodeCursor(token) + expect(decoded.after).toEqual({ orderKey: 'a1', id: 'row_1' }) + expect(decoded.sortKey).toBeUndefined() + expect(() => assertCursorSortBinding(decoded, { col_a: 'asc' })).toThrow(/sorted query/) + expect(() => assertCursorSortBinding(decoded, undefined)).not.toThrow() + }) + + it('sort key order is significant (priority is part of the identity)', () => { + expect(canonicalSortKey({ a: 'asc', b: 'desc' })).not.toBe( + canonicalSortKey({ b: 'desc', a: 'asc' }) + ) + }) +}) diff --git a/apps/sim/lib/table/rows/cursor.ts b/apps/sim/lib/table/rows/cursor.ts index 1d51ab2c69d..0d2ab4ab206 100644 --- a/apps/sim/lib/table/rows/cursor.ts +++ b/apps/sim/lib/table/rows/cursor.ts @@ -16,7 +16,7 @@ */ import { TableQueryValidationError } from '@/lib/table/errors' -import type { TableRow, TableRowsCursor } from '@/lib/table/types' +import type { Sort, TableRow, TableRowsCursor } from '@/lib/table/types' /** * Cursor payload version. Every encoded token carries `v`; decode rejects any @@ -26,7 +26,50 @@ import type { TableRow, TableRowsCursor } from '@/lib/table/types' const CURSOR_VERSION = 1 type CursorBody = { k: string; i: string } | { o: number } | { k: string; i: string; o: number } -type CursorPayload = CursorBody & { v: number } +type SortBinding = { s?: string } +type CursorPayload = CursorBody & SortBinding & { v: number } + +/** + * Canonical fingerprint of a sort for cursor binding. Entry order is the sort + * priority (built from the ordered spec upstream), so stringifying entries is + * deterministic for equal sorts and distinct for different ones. + */ +export function canonicalSortKey(sort: Sort | null | undefined): string | undefined { + if (!sort) return undefined + const entries = Object.entries(sort) + return entries.length > 0 ? JSON.stringify(entries) : undefined +} + +/** + * A cursor is only valid for the exact query shape it was minted under: + * keyset/compound cursors encode a position in the DEFAULT `(order_key, id)` + * order, and an offset cursor from a sorted view encodes a position in THAT + * sort. Replaying either against a different ordering silently pages the wrong + * sequence — rows skipped or duplicated with no error. Throws + * `CURSOR_SORT_CONFLICT` so callers restart paging without the cursor. + */ +export function assertCursorSortBinding( + decoded: { after?: TableRowsCursor; offset?: number; sortKey?: string }, + sort: Sort | null | undefined +): void { + const requested = canonicalSortKey(sort) + if (decoded.after && requested !== undefined) { + throw new TableQueryValidationError( + 'Cursor is not valid for a sorted query. Restart paging without the cursor.', + 'CURSOR_SORT_CONFLICT' + ) + } + if ( + decoded.after === undefined && + decoded.offset !== undefined && + decoded.sortKey !== requested + ) { + throw new TableQueryValidationError( + 'Cursor was created under a different sort. Restart paging without the cursor.', + 'CURSOR_SORT_CONFLICT' + ) + } +} function invalidCursor(): never { throw new TableQueryValidationError('Invalid cursor', 'INVALID_CURSOR') @@ -57,6 +100,8 @@ export function encodeCursor(args: { keysetValid: boolean nextOffset: number seekBase?: { anchor: TableRowsCursor; offsetFromAnchor: number } + /** The sort the page was produced under — stamps offset cursors so they can't be replayed against a different ordering. */ + sort?: Sort | null }): string { let body: CursorBody if (args.keysetValid && args.lastRow.orderKey) { @@ -74,12 +119,24 @@ export function encodeCursor(args: { } else { body = { o: args.nextOffset } } - const payload: CursorPayload = { ...body, v: CURSOR_VERSION } + const sortKey = canonicalSortKey(args.sort) + const payload: CursorPayload = { + ...body, + // Only the pure-offset shape can exist under a custom sort; keyset and + // compound shapes are default-order by construction and carry no binding. + ...('k' in body || sortKey === undefined ? {} : { s: sortKey }), + v: CURSOR_VERSION, + } return toBase64Url(JSON.stringify(payload)) } /** Decodes an opaque cursor into the `queryRows` paging inputs it stands for. */ -export function decodeCursor(token: string): { after?: TableRowsCursor; offset?: number } { +export function decodeCursor(token: string): { + after?: TableRowsCursor + offset?: number + /** Sort fingerprint an offset cursor was minted under; absent = default order. */ + sortKey?: string +} { let payload: unknown try { payload = JSON.parse(fromBase64Url(token)) @@ -105,7 +162,10 @@ export function decodeCursor(token: string): { after?: TableRowsCursor; offset?: return { after: { orderKey: record.k as string, id: record.i as string } } } if (hasOffset) { - return { offset: record.o as number } + return { + offset: record.o as number, + ...(typeof record.s === 'string' ? { sortKey: record.s } : {}), + } } invalidCursor() } diff --git a/apps/sim/lib/table/rows/service.ts b/apps/sim/lib/table/rows/service.ts index 6e8df7c4840..dfcc3a4c282 100644 --- a/apps/sim/lib/table/rows/service.ts +++ b/apps/sim/lib/table/rows/service.ts @@ -1145,6 +1145,7 @@ export async function queryRows( seekBase: fetched.anchor ? { anchor: fetched.anchor, offsetFromAnchor: fetched.anchorOffset } : undefined, + sort, }) : null From c66ecef4cd332e5b2be0b1e16a805fc8de784052 Mon Sep 17 00:00:00 2001 From: Theodore Li Date: Wed, 29 Jul 2026 19:50:43 -0700 Subject: [PATCH 25/26] fix(table): storage-validate async-route filters, reject dual-group nodes, resolve coerced select names MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bugbot round 3 on PR #6067, all three verified: Async routes (Medium). delete-async / cancel-runs / columns/run validated only the toLegacyFilter downgrade, which compiles a typo'd field into a clause that silently matches nothing — a filtered delete/run/stop no-ops where the sync bulk routes 400. tableFilterError is now grammar-aware and takes the WIRE filter: predicates go through validateStoragePredicate (same keying the sync routes enforce), legacy filters keep the buildFilterClause check. Dual all/any node (High). {all:[...], any:[...]} fails both strictObject branches, survives the legacy union, and every group-first traversal reads `all` and silently DROPS `any` — half the conditions vanish, widening a bulk delete/update. validateNode (covering every boundary and the copilot tool's validatePredicate) and predicateToFilter (lossless-or-throw) both reject it; nesting expresses the same intent unambiguously. Coerced select names (Medium). The block builder serializes without schema access, so an option NAME that looks numeric/boolean ("123") arrives scalar-coerced and resolveSelectOptionId bailed on non-strings — the filter compared 123 against the stored option id and matched nothing. Stringify scalar operands before matching, fixing every name-keyed caller (v1 and v2 blocks) at the resolution seam instead of per-surface. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_011M563cbFy2S74GvSDf2C3R --- .../api/table/[tableId]/cancel-runs/route.ts | 2 +- .../api/table/[tableId]/columns/run/route.ts | 2 +- .../api/table/[tableId]/delete-async/route.ts | 2 +- apps/sim/app/api/table/utils.test.ts | 50 +++++++++++++++++- apps/sim/app/api/table/utils.ts | 27 +++++++--- .../__tests__/converters.test.ts | 9 ++++ .../query-builder/__tests__/validate.test.ts | 14 +++++ .../sim/lib/table/query-builder/converters.ts | 8 +++ apps/sim/lib/table/query-builder/validate.ts | 11 ++++ apps/sim/lib/table/select-values.test.ts | 52 ++++++++++++++++++- apps/sim/lib/table/validation.ts | 17 ++++-- 11 files changed, 179 insertions(+), 15 deletions(-) diff --git a/apps/sim/app/api/table/[tableId]/cancel-runs/route.ts b/apps/sim/app/api/table/[tableId]/cancel-runs/route.ts index 7dc24f1850a..acace4561c5 100644 --- a/apps/sim/app/api/table/[tableId]/cancel-runs/route.ts +++ b/apps/sim/app/api/table/[tableId]/cancel-runs/route.ts @@ -47,7 +47,7 @@ export const POST = withRouteHandler(async (request: NextRequest, { params }: Ro return NextResponse.json({ error: 'Invalid workspace ID' }, { status: 400 }) } - const filterError = tableFilterError(filter, table.schema.columns) + const filterError = tableFilterError(wireFilter, table.schema.columns) if (filterError) return filterError const cancelled = await cancelWorkflowGroupRuns(tableId, scope === 'row' ? rowId : undefined, { diff --git a/apps/sim/app/api/table/[tableId]/columns/run/route.ts b/apps/sim/app/api/table/[tableId]/columns/run/route.ts index e45a8de1a70..824ce73ddb4 100644 --- a/apps/sim/app/api/table/[tableId]/columns/run/route.ts +++ b/apps/sim/app/api/table/[tableId]/columns/run/route.ts @@ -43,7 +43,7 @@ export const POST = withRouteHandler(async (request: NextRequest, { params }: Ro if (!access.ok) return accessError(access, requestId, tableId) // Validate the filter up front (the dispatcher reuses it) so a bad field fails fast. - const filterError = tableFilterError(filter, access.table.schema.columns) + const filterError = tableFilterError(wireFilter, access.table.schema.columns) if (filterError) return filterError const { dispatchId } = await runWorkflowColumn({ diff --git a/apps/sim/app/api/table/[tableId]/delete-async/route.ts b/apps/sim/app/api/table/[tableId]/delete-async/route.ts index 5cd7d295e3a..83382ea2ddd 100644 --- a/apps/sim/app/api/table/[tableId]/delete-async/route.ts +++ b/apps/sim/app/api/table/[tableId]/delete-async/route.ts @@ -78,7 +78,7 @@ export const POST = withRouteHandler(async (request: NextRequest, { params }: Ro assertRowDelete(table) // Validate the filter up front so the caller gets immediate feedback (the worker reuses it). - const filterError = tableFilterError(filter, table.schema.columns) + const filterError = tableFilterError(wireFilter, table.schema.columns) if (filterError) return filterError // Rows inserted after this instant are spared (created_at <= cutoff in the worker). diff --git a/apps/sim/app/api/table/utils.test.ts b/apps/sim/app/api/table/utils.test.ts index 2b23d45ffe9..99d0ce0c5a5 100644 --- a/apps/sim/app/api/table/utils.test.ts +++ b/apps/sim/app/api/table/utils.test.ts @@ -3,7 +3,8 @@ */ import { describe, expect, it } from 'vitest' import { TableRowLimitError } from '@/lib/table/billing' -import { rootErrorMessage, rowWriteErrorResponse } from '@/app/api/table/utils' +import type { ColumnDefinition } from '@/lib/table/types' +import { rootErrorMessage, rowWriteErrorResponse, tableFilterError } from '@/app/api/table/utils' /** Mimics drizzle's DrizzleQueryError: message is the failed SQL, real error on `cause`. */ function wrapLikeDrizzle(cause: Error): Error { @@ -53,3 +54,50 @@ describe('rowWriteErrorResponse', () => { expect(rowWriteErrorResponse(wrapLikeDrizzle(new Error('deadlock detected')))).toBeNull() }) }) + +/** + * The async destructive routes (delete-async, cancel-runs, columns/run) + * validate the WIRE filter here. The predicate branch must reject unknown + * storage keys the way the sync bulk routes do — the `toLegacyFilter` + * downgrade compiles a typo'd field into a clause that silently matches + * nothing, turning a scoped delete/run into a no-op. + */ +describe('tableFilterError', () => { + const columns: ColumnDefinition[] = [{ id: 'col_status', name: 'status', type: 'string' }] + + it('returns null for an absent filter and a valid id-keyed predicate', () => { + expect(tableFilterError(undefined, columns)).toBeNull() + expect( + tableFilterError({ all: [{ field: 'col_status', op: 'eq', value: 'x' }] }, columns) + ).toBeNull() + expect(tableFilterError({ all: [{ field: 'createdAt', op: 'isNotNull' }] }, columns)).toBeNull() + }) + + it('400s a predicate naming an unknown storage key', async () => { + const response = tableFilterError( + { all: [{ field: 'statuss', op: 'eq', value: 'x' }] }, + columns + ) + expect(response?.status).toBe(400) + const body = await response?.json() + expect(body.error).toMatch(/Unknown filter column "statuss"/) + }) + + it('400s a structurally invalid predicate (empty group, dual group keys)', () => { + expect(tableFilterError({ all: [] } as never, columns)?.status).toBe(400) + expect( + tableFilterError( + { + all: [{ field: 'col_status', op: 'eq', value: 'a' }], + any: [{ field: 'col_status', op: 'eq', value: 'b' }], + } as never, + columns + )?.status + ).toBe(400) + }) + + it('still validates the legacy grammar through buildFilterClause', () => { + expect(tableFilterError({ col_status: 'x' }, columns)).toBeNull() + expect(tableFilterError({ col_status: { $regex: 'x' } } as never, columns)?.status).toBe(400) + }) +}) diff --git a/apps/sim/app/api/table/utils.ts b/apps/sim/app/api/table/utils.ts index 3259a6620f4..835866d829a 100644 --- a/apps/sim/app/api/table/utils.ts +++ b/apps/sim/app/api/table/utils.ts @@ -9,10 +9,12 @@ import { } from '@/lib/api/contracts/tables' import { isFeatureEnabled } from '@/lib/core/config/feature-flags' import type { MultipartError } from '@/lib/core/utils/multipart' -import type { ColumnDefinition, Filter, TableDefinition } from '@/lib/table' +import type { ColumnDefinition, Filter, TableDefinition, TablePredicate } from '@/lib/table' import { buildFilterClause, getTableById, TableQueryValidationError } from '@/lib/table' import { USER_TABLE_ROWS_SQL_NAME } from '@/lib/table/constants' import { TableLockedError } from '@/lib/table/mutation-locks' +import { isTablePredicate } from '@/lib/table/query-builder/converters' +import { validateStoragePredicate } from '@/lib/table/query-builder/validate' import { getUserEntityPermissions } from '@/lib/workspaces/permissions/utils' import { getWorkspaceOrganizationId } from '@/lib/workspaces/utils' @@ -54,17 +56,30 @@ export function tableLockErrorResponse(error: unknown): NextResponse | null { } /** - * Validates a `filter` against the table's column schema, returning a 400 response on a bad field - * (or `null` when the filter is valid or absent). Shared by the routes that accept a filter - * (`delete-async`, `columns/run`) so a bad field fails fast with a clear message. + * Validates a wire `filter` (either grammar) against the table's column schema, + * returning a 400 response on a bad field (or `null` when the filter is valid or + * absent). Shared by the routes that accept a filter (`delete-async`, + * `cancel-runs`, `columns/run`) so a bad field fails fast with a clear message. + * + * Pass the WIRE filter, not the `toLegacyFilter` downgrade: the downgrade + * compiles cleanly through `buildFilterClause` even when a predicate leaf names + * a column that doesn't exist, so validating only the downgraded form lets a + * typo'd field become a clause that silently matches nothing — a no-op where + * the sync bulk routes 400. */ export function tableFilterError( - filter: Filter | undefined, + filter: Filter | TablePredicate | undefined, columns: ColumnDefinition[] ): NextResponse | null { if (!filter) return null try { - buildFilterClause(filter, USER_TABLE_ROWS_SQL_NAME, columns) + if (isTablePredicate(filter)) { + // These routes speak storage keys (session grid uses column ids; system + // columns keep their names) — same keying the sync bulk routes validate. + validateStoragePredicate(filter, columns) + } else { + buildFilterClause(filter, USER_TABLE_ROWS_SQL_NAME, columns) + } return null } catch (error) { if (error instanceof TableQueryValidationError) { diff --git a/apps/sim/lib/table/query-builder/__tests__/converters.test.ts b/apps/sim/lib/table/query-builder/__tests__/converters.test.ts index e389b0ad062..fa9dd5fb93e 100644 --- a/apps/sim/lib/table/query-builder/__tests__/converters.test.ts +++ b/apps/sim/lib/table/query-builder/__tests__/converters.test.ts @@ -468,6 +468,15 @@ describe('toLegacyFilter / predicateToFilter hybrid safety', () => { expect(() => predicateToFilter(hybrid)).toThrow(/not both/) }) + it('throws on a node with both group keys instead of dropping the "any" half', () => { + const dual = { + all: [{ field: 'tenant_id', op: 'eq', value: 'acme' }], + any: [{ field: 'status', op: 'eq', value: 'archived' }], + } as never + expect(() => predicateToFilter(dual)).toThrow(/either "all" or "any"/) + expect(() => toLegacyFilter(dual)).toThrow(/either "all" or "any"/) + }) + it('toLegacyFilter shape-validates before converting', () => { expect(() => toLegacyFilter(hybrid)).toThrow(/not both/) // malformed group value is also caught, not crashed on diff --git a/apps/sim/lib/table/query-builder/__tests__/validate.test.ts b/apps/sim/lib/table/query-builder/__tests__/validate.test.ts index cad250b0aa1..12eea2427a0 100644 --- a/apps/sim/lib/table/query-builder/__tests__/validate.test.ts +++ b/apps/sim/lib/table/query-builder/__tests__/validate.test.ts @@ -143,6 +143,20 @@ describe('validatePredicate — leaves that would silently widen a bulk write', ).toThrow(/not both/) }) + it('rejects a node carrying both group keys instead of dropping the "any" half', () => { + // Every group-first traversal reads `all` and drops `any` — half the + // caller's conditions would vanish, widening a bulk delete/update. + expect(() => + validatePredicate( + { + all: [{ field: 'status', op: 'eq', value: 'archived' }], + any: [{ field: 'status', op: 'eq', value: 'stale' }], + } as never, + COLS + ) + ).toThrow(/either "all" or "any"/) + }) + it('caps in/nin list length', () => { const huge = Array.from({ length: 1001 }, (_, i) => `v${i}`) expect(() => diff --git a/apps/sim/lib/table/query-builder/converters.ts b/apps/sim/lib/table/query-builder/converters.ts index df27b488e3c..5e938c64731 100644 --- a/apps/sim/lib/table/query-builder/converters.ts +++ b/apps/sim/lib/table/query-builder/converters.ts @@ -475,6 +475,14 @@ export function predicateToFilter(predicate: TablePredicate): Filter { 'INVALID_FILTER' ) } + // A node with BOTH group keys would convert `all` and silently drop `any` + // — same widening failure as the hybrid above. Lossless-or-throw. + if ('all' in node && 'any' in node) { + throw new TableQueryValidationError( + 'A filter group must use either "all" or "any", not both — nest one group inside the other instead.', + 'INVALID_FILTER' + ) + } // Group-first, matching isPredicateGroup/validateNode/buildPredicateNode. A // leaf-first test would execute a different predicate than the one validated. if ('all' in node) return { $and: node.all.map(nodeToFilter) } diff --git a/apps/sim/lib/table/query-builder/validate.ts b/apps/sim/lib/table/query-builder/validate.ts index 8448668117a..f508cd56b57 100644 --- a/apps/sim/lib/table/query-builder/validate.ts +++ b/apps/sim/lib/table/query-builder/validate.ts @@ -137,6 +137,17 @@ function validateNode(node: PredicateNode, typeByName: Map | 'INVALID_FILTER' ) } + // Same ambiguity with BOTH group keys: every group-first traversal + // (`predicateNamesToIds`, `predicateToFilter`, `buildPredicateNode`) reads + // `all` and silently DROPS `any`, so half the caller's conditions vanish — + // on a bulk delete/update that widens the matched set. Reject rather than + // pick a winner; nesting expresses the intent unambiguously. + if ('all' in node && 'any' in node) { + throw new TableQueryValidationError( + 'A filter group must use either "all" or "any", not both — nest one group inside the other instead.', + 'INVALID_FILTER' + ) + } if ('all' in node || 'any' in node) { const members = 'all' in node ? node.all : node.any if (!Array.isArray(members)) { diff --git a/apps/sim/lib/table/select-values.test.ts b/apps/sim/lib/table/select-values.test.ts index 7c4d4b456da..041b2f61242 100644 --- a/apps/sim/lib/table/select-values.test.ts +++ b/apps/sim/lib/table/select-values.test.ts @@ -2,7 +2,11 @@ * @vitest-environment node */ import { describe, expect, it } from 'vitest' -import { resolveFilterSelectValues, selectValueToNames } from '@/lib/table/select-values' +import { + resolveFilterSelectValues, + resolvePredicateSelectValues, + selectValueToNames, +} from '@/lib/table/select-values' import type { ColumnDefinition } from '@/lib/table/types' const status: ColumnDefinition = { @@ -100,3 +104,49 @@ describe('resolveFilterSelectValues', () => { ).toEqual({ $or: [{ col_status: 'opt_open' }, { col_title: 'x' }] }) }) }) + +/** + * The block builder serializes without schema access, so an option NAME that + * looks numeric or boolean arrives scalar-coerced ("123" → 123). Resolution + * must still find the option, or a correctly-authored builder filter compares + * a number against the stored id string and matches nothing. + */ +describe('resolvePredicateSelectValues — scalar-coerced option names', () => { + const numericStatus: ColumnDefinition = { + id: 'col_code', + name: 'code', + type: 'select', + options: [ + { id: 'opt_123', name: '123' }, + { id: 'opt_true', name: 'true' }, + ], + } + const columns = [numericStatus] + + it('resolves a coerced numeric name to its option id', () => { + expect( + resolvePredicateSelectValues({ all: [{ field: 'col_code', op: 'eq', value: 123 }] }, columns) + ).toEqual({ all: [{ field: 'col_code', op: 'eq', value: 'opt_123' }] }) + }) + + it('resolves a coerced boolean name, including inside in/contains', () => { + expect( + resolvePredicateSelectValues( + { any: [{ field: 'col_code', op: 'in', value: [true, 123] }] }, + columns + ) + ).toEqual({ any: [{ field: 'col_code', op: 'in', value: ['opt_true', 'opt_123'] }] }) + expect( + resolvePredicateSelectValues( + { all: [{ field: 'col_code', op: 'contains', value: 123 }] }, + columns + ) + ).toEqual({ all: [{ field: 'col_code', op: 'contains', value: 'opt_123' }] }) + }) + + it('leaves a scalar with no matching option name as-is', () => { + expect( + resolvePredicateSelectValues({ all: [{ field: 'col_code', op: 'eq', value: 999 }] }, columns) + ).toEqual({ all: [{ field: 'col_code', op: 'eq', value: 999 }] }) + }) +}) diff --git a/apps/sim/lib/table/validation.ts b/apps/sim/lib/table/validation.ts index 83a8eec8523..14275ea74eb 100644 --- a/apps/sim/lib/table/validation.ts +++ b/apps/sim/lib/table/validation.ts @@ -299,12 +299,21 @@ function optionIds(column: ColumnDefinition): Set { * actually fit the target option set. */ export function resolveSelectOptionId(value: JsonValue, options: SelectOption[]): string | null { - if (typeof value !== 'string') return null - const byId = options.find((o) => o.id === value) + // The block builder serializes without schema access, so an option NAME that + // looks numeric or boolean ("123", "true") arrives scalar-coerced. Stringify + // scalars so the name still resolves; arrays/objects stay unresolvable. + const text = + typeof value === 'string' + ? value + : typeof value === 'number' || typeof value === 'boolean' + ? String(value) + : null + if (text === null) return null + const byId = options.find((o) => o.id === text) if (byId) return byId.id const byName = - options.find((o) => o.name === value) ?? - options.find((o) => o.name.toLowerCase() === value.toLowerCase()) + options.find((o) => o.name === text) ?? + options.find((o) => o.name.toLowerCase() === text.toLowerCase()) return byName ? byName.id : null } From 808bc0d40bbde139a2812e74e66e9620df1d1dfc Mon Sep 17 00:00:00 2001 From: Theodore Li Date: Wed, 29 Jul 2026 22:40:50 -0700 Subject: [PATCH 26/26] fix(table): storage-validate read-path predicates on GET rows and find MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bugbot round 4: the native predicate path on GET /rows ran only the shape check before wire translation, and find ran none before its toLegacyFilter downgrade — so a typo'd field compiled to a clause matching nothing and read back as a plausible empty page, where the bulk write paths 400. GET now runs validateStoragePredicate post-translation (name-keyed JWT callers validate their translated form, same recipe as resolveBulkFilter); find reuses the grammar-aware tableFilterError gate. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_011M563cbFy2S74GvSDf2C3R --- .../table/[tableId]/rows/find/route.test.ts | 20 ++++++++++++++++++- .../api/table/[tableId]/rows/find/route.ts | 11 +++++++++- .../api/table/[tableId]/rows/route.test.ts | 19 ++++++++++++++++++ .../sim/app/api/table/[tableId]/rows/route.ts | 7 ++++++- 4 files changed, 54 insertions(+), 3 deletions(-) diff --git a/apps/sim/app/api/table/[tableId]/rows/find/route.test.ts b/apps/sim/app/api/table/[tableId]/rows/find/route.test.ts index 111324b14de..4b892a39340 100644 --- a/apps/sim/app/api/table/[tableId]/rows/find/route.test.ts +++ b/apps/sim/app/api/table/[tableId]/rows/find/route.test.ts @@ -6,14 +6,16 @@ import { NextRequest } from 'next/server' import { beforeEach, describe, expect, it, vi } from 'vitest' import type { TableDefinition } from '@/lib/table' -const { mockCheckAccess, mockFindRowMatches } = vi.hoisted(() => ({ +const { mockCheckAccess, mockFindRowMatches, mockTableFilterError } = vi.hoisted(() => ({ mockCheckAccess: vi.fn(), mockFindRowMatches: vi.fn(), + mockTableFilterError: vi.fn(), })) vi.mock('@/app/api/table/utils', async () => { const { NextResponse } = await import('next/server') return { + tableFilterError: mockTableFilterError, checkAccess: mockCheckAccess, accessError: (result: { status: number }) => NextResponse.json( @@ -67,6 +69,7 @@ describe('GET /api/table/[tableId]/rows/find', () => { authType: 'session', }) mockCheckAccess.mockResolvedValue({ ok: true, table: buildTable() }) + mockTableFilterError.mockReturnValue(null) mockFindRowMatches.mockResolvedValue({ matches: [{ ordinal: 4, rowId: 'row_4', column: 'name' }], truncated: false, @@ -120,6 +123,21 @@ describe('GET /api/table/[tableId]/rows/find', () => { ) }) + it('short-circuits with the filter gate response before searching', async () => { + const { NextResponse } = await import('next/server') + mockTableFilterError.mockReturnValueOnce( + NextResponse.json({ error: 'Unknown filter column "nope"' }, { status: 400 }) + ) + const res = await callGet({ + workspaceId: 'workspace-1', + q: 'foo', + filter: JSON.stringify({ all: [{ field: 'nope', op: 'eq', value: 1 }] }), + }) + expect(res.status).toBe(400) + expect((await res.json()).error).toMatch(/Unknown filter column/) + expect(mockFindRowMatches).not.toHaveBeenCalled() + }) + it('maps a TableQueryValidationError to 400', async () => { const { TableQueryValidationError } = await import('@/lib/table/errors') mockFindRowMatches.mockRejectedValueOnce(new TableQueryValidationError('Invalid field name')) diff --git a/apps/sim/app/api/table/[tableId]/rows/find/route.ts b/apps/sim/app/api/table/[tableId]/rows/find/route.ts index 5c10c5ae471..f72c86633f6 100644 --- a/apps/sim/app/api/table/[tableId]/rows/find/route.ts +++ b/apps/sim/app/api/table/[tableId]/rows/find/route.ts @@ -9,7 +9,7 @@ import type { Filter, Sort, SortSpec, TablePredicate } from '@/lib/table' import { TableQueryValidationError } from '@/lib/table/errors' import { toLegacyFilter, toLegacySort } from '@/lib/table/query-builder/converters' import { findRowMatches } from '@/lib/table/rows/service' -import { accessError, checkAccess } from '@/app/api/table/utils' +import { accessError, checkAccess, tableFilterError } from '@/app/api/table/utils' const logger = createLogger('TableRowsFindAPI') @@ -59,6 +59,15 @@ export const GET = withRouteHandler( return NextResponse.json({ error: 'Invalid workspace ID' }, { status: 400 }) } + // Same gate as the bulk/async routes: the predicate branch rejects unknown + // storage keys (a typo'd field must 400, not return zero matches), the + // legacy branch keeps its compile check. + const filterError = tableFilterError( + validated.filter as Filter | TablePredicate | undefined, + table.schema.columns + ) + if (filterError) return filterError + const { matches, truncated } = await findRowMatches( table, { diff --git a/apps/sim/app/api/table/[tableId]/rows/route.test.ts b/apps/sim/app/api/table/[tableId]/rows/route.test.ts index 9a83aef3920..b6162110b57 100644 --- a/apps/sim/app/api/table/[tableId]/rows/route.test.ts +++ b/apps/sim/app/api/table/[tableId]/rows/route.test.ts @@ -262,6 +262,25 @@ describe('GET /api/table/[tableId]/rows', () => { expect(options.predicate).toEqual({ all: [{ field: 'col_aaa', op: 'eq', value: 'Ada' }] }) expect(options.sort).toEqual({ col_bbb: 'asc' }) }) + + /** + * Storage validation runs post-translation, mirroring bulk PUT/DELETE: a + * typo'd field must 400, not compile to a clause that matches nothing and + * read back as a plausible empty page. + */ + it('400s a predicate naming an unknown column instead of returning an empty page', async () => { + authAs('session') + + const res = await callGet({ + workspaceId: 'workspace-1', + filter: JSON.stringify({ all: [{ field: 'col_nope', op: 'eq', value: 'Ada' }] }), + }) + + expect(res.status).toBe(400) + const body = await res.json() + expect(body.error).toMatch(/Unknown filter column "col_nope"/) + expect(mockQueryRows).not.toHaveBeenCalled() + }) }) describe('PUT/DELETE /api/table/[tableId]/rows — predicate filters', () => { diff --git a/apps/sim/app/api/table/[tableId]/rows/route.ts b/apps/sim/app/api/table/[tableId]/rows/route.ts index 67652f86d82..748c7138b50 100644 --- a/apps/sim/app/api/table/[tableId]/rows/route.ts +++ b/apps/sim/app/api/table/[tableId]/rows/route.ts @@ -308,7 +308,12 @@ export const GET = withRouteHandler( // Shape-check first: nothing upstream validates this branch, and // an unchecked hybrid node would silently widen the result. validatePredicateShape(validated.filter as TablePredicate) - return wire.predicateIn(validated.filter as TablePredicate) + const translated = wire.predicateIn(validated.filter as TablePredicate) + // Post-translation storage check, mirroring the bulk PUT/DELETE + // paths: a typo'd field must 400, not compile to a clause that + // matches nothing and read back as an empty page. + validateStoragePredicate(translated, (table.schema as TableSchema).columns) + return translated })(), } : { filter: validated.filter ? wire.filterIn(validated.filter as Filter) : undefined }),