diff --git a/.clang-tidy b/.clang-tidy index 187512b..7fce2ae 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -1,5 +1,5 @@ --- -# clang-tidy for pg_clickhouse +# clang-tidy for pg_chdb # # PostgreSQL extensions use palloc/pfree, longjmp-based error handling, # and C89-ish idioms — checks are tuned accordingly. diff --git a/.gitattributes b/.gitattributes index 97b7613..7028ae8 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,3 +1,4 @@ .gitignore export-ignore .gitattributes export-ignore .github export-ignore +.gitmodules export-ignore diff --git a/CHANGELOG.md b/CHANGELOG.md index d4404fa..e5a1a21 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,11 @@ All notable changes to this project will be documented in this file. It uses the [Semantic Versioning]: https://semver.org/spec/v2.0.0.html "Semantic Versioning 2.0.0" -## [v0.1.1] — Unreleased +## [v0.1.1] — 2026-09-06 + +This release makes binary-only changes. Once installed and loaded, any +existing use of chdb and chdb_hook v0.1 will get the v0.1.1 benefits without +needing to `ALTER EXTENSION UPDATE`. ### ⚡ Improvements @@ -41,6 +45,8 @@ All notable changes to this project will be documented in this file. It uses the document the ISO-8601 format of plain text exports of `timestamp` and `timestamptz` values, as well as the impact of the `timezone` setting on exported and imported values. +* Added a [benchmark](dev/benchmark/) and documented its results in the + README. [v0.1.1]: https://github.com/clickhouse/pg_chdb/compare/v0.1.0...v0.1.1 [#76]: https://github.com/clickhouse/pg_chdb/issues/76 diff --git a/META.json b/META.json index 5a57642..34e4dee 100644 --- a/META.json +++ b/META.json @@ -2,7 +2,7 @@ "name": "chdb", "abstract": "Execute queries in chDB.", "description": "Execute queries in chDB.", - "version": "0.1.0", + "version": "0.1.1", "maintainer": "David E. Wheeler ", "license": "apache_2_0", "provides": { @@ -10,13 +10,13 @@ "abstract": "Execute queries in chDB.", "file": "chdb.control", "docfile": "doc/chdb.md", - "version": "0.1.0" + "version": "0.1.1" }, "chdb_hook": { "abstract": "Hook into PostgreSQL commands to execute chDB queries.", "file": "src/hook/hook.c", "docfile": "doc/chdb_hook.md", - "version": "0.1.0" + "version": "0.1.1" } }, "prereqs": { @@ -26,6 +26,9 @@ } } }, + "no_index": { + "directory": ["dev"] + }, "resources": { "bugtracker": { "web": "http://github.com/ClickHouse/pg_chdb/issues/" diff --git a/Makefile b/Makefile index 76fc839..97beb7d 100644 --- a/Makefile +++ b/Makefile @@ -180,5 +180,7 @@ release-notes: CHANGELOG.md $(EXTENSION)-$(DISTVERSION).zip: git archive-all -v --prefix "$(EXTENSION)-$(DISTVERSION)/" --force-submodules $(EXTENSION)-$(DISTVERSION).zip +zip: $(EXTENSION)-$(DISTVERSION).zip + # Run make print-VARIABLE_NAME to print VARIABLE_NAME's flavor and value. print-% : ; $(info $* is $(flavor $*) variable set to "$($*)") @true diff --git a/README.md b/README.md index c6432bf..d4ef531 100644 --- a/README.md +++ b/README.md @@ -89,6 +89,43 @@ CREATE TABLE reviews () WITH ( See the [chdb_hook documentation](doc/chdb_hook.md) for details. +Benchmarking Formats +-------------------- + +The contents of [dev/benchmark](dev/benchmark/) compare the performance of +[chdb_hook] `COPY` to that of [aws_s3], [pg_duckdb], and [pg_lake] for ca. 1m +rows of [NYC Taxi dataset] in a variety of formats. + + + + NYC Taxi Data Benchmark + + +Of the four extensions, [chdb] exhibits the most consistent performance. +[pg_duckdb] and [pg_lake], both backed by [DuckDB], take around 2-3x as long +to import data from CSV, JSON, and Parquet. Only [aws_s3] approaches +[chdb_hook]'s performance, but it supports a much more limited array of data +formats: + +| Extension | Compression | Data Formats +| --------- | ------------------------------------ | -------------------------------- | +| aws_s3 | none | Text (TSV), CSV, Postgres Binary | +| pg_lake | gzip, zstd, snappy (Parquet only) | CSV, JSON, Parquet | +| pg_duckdb | gzip, zstd, snappy (Parquet only) | CSV, JSON, Parquet | +| chdb | tzip, zstd, lz4, bz2, snappy, brotli | TSV, CSV, JSON, BSON, Prometheus, Protobuf, Avro, Parquet, Arrow, XML, CapnProto, Markdown, MsgPack, ORC, and [more][formats]! | + +Additional benchmarking demonstrates relatively consistent performance +importing the [NYC Taxi dataset] in a variety of formats: + + + + Import Benchmark + + +The benchmark uses the [JSONCompact] format for compatibility with the other +extensions; Other JSON formats, such as [JSONCompactEachRow], will more +closely approximate the performance of the other formats. + Architecture ------------ @@ -114,10 +151,10 @@ A helper that dies triggers an error only in the backend that started it, leaving other sessions untouched. > [!IMPORTANT] -> For each query, the helper connects to a new in-memory database to execute -> it. As a consequence, each query currently runs in complete isolation from -> all other queries. Don't create a table and expect to query it in a -> subsequent query. +> For each query, the helper connects to a temporary chDB database on disk to +> execute it. As a consequence, each query currently runs in complete +> isolation from all other queries. Don't create a table and expect to query +> it in a subsequent query. Dependencies ------------ @@ -246,3 +283,12 @@ Copyright (c) 2026, ClickHouse "Postgres Docs: CREATE TABLE" [lib.chdb.io]: https://lib.chdb.io "curl -sL https://lib.chdb.io | bash" [`postgresql.conf` parameters]: https://www.postgresql.org/docs/devel/runtime-config-client.html#RUNTIME-CONFIG-CLIENT-OTHER + [chdb_hook]: https://pgxn.org/dist/chdb/doc/chdb_hook.html "chdb_hook Docs on PGXN" + [aws_s3]: https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/USER_PostgreSQL.S3Import.html + "Importing data from Amazon S3 into an RDS for PostgreSQL DB instance" + [pg_duckdb]: https://github.com/duckdb/pg_duckdb "DuckDB-powered Postgres for high performance apps & analytics" + [pg_lake]: https://github.com/Snowflake-Labs/pg_lake "pg_lake: Postgres with Iceberg and data lake access" + [formats]: https://clickhouse.com/docs/reference/formats/index + "ClickHouse Docs: Formats for input and output data" + [NYC Taxi dataset]: https://clickhouse.com/docs/get-started/quickstarts/tutorial + "ClickHouse Docs: Advanced tutorial" diff --git a/dev/benchmark/README.md b/dev/benchmark/README.md index 41fec3b..0e0a249 100644 --- a/dev/benchmark/README.md +++ b/dev/benchmark/README.md @@ -20,15 +20,24 @@ Test configurations: ### NYC Taxi Data -![NYC Taxi Data Benchmark](taxi-bench.png) + + + NYC Taxi Data Benchmark + ### "Logs" Data -!["Logs" Data Benchmark](logs-bench.png) + + + “Logs” Taxi Data Benchmark + ### chdb Data Formats -![chdb Data Formats](chdb-bench.png) + + + Import Benchmark + ## Data Sets diff --git a/dev/benchmark/chdb-bench-dark.png b/dev/benchmark/chdb-bench-dark.png new file mode 100644 index 0000000..6a05424 Binary files /dev/null and b/dev/benchmark/chdb-bench-dark.png differ diff --git a/dev/benchmark/chdb-bench.png b/dev/benchmark/chdb-bench.png index 6a05424..7c414b0 100644 Binary files a/dev/benchmark/chdb-bench.png and b/dev/benchmark/chdb-bench.png differ diff --git a/dev/benchmark/logs-bench-dark.png b/dev/benchmark/logs-bench-dark.png new file mode 100644 index 0000000..d122f32 Binary files /dev/null and b/dev/benchmark/logs-bench-dark.png differ diff --git a/dev/benchmark/logs-bench.png b/dev/benchmark/logs-bench.png index d122f32..57dbddc 100644 Binary files a/dev/benchmark/logs-bench.png and b/dev/benchmark/logs-bench.png differ diff --git a/dev/benchmark/taxi-bench-dark.png b/dev/benchmark/taxi-bench-dark.png new file mode 100644 index 0000000..8a13624 Binary files /dev/null and b/dev/benchmark/taxi-bench-dark.png differ diff --git a/dev/benchmark/taxi-bench.png b/dev/benchmark/taxi-bench.png index 8a13624..a4eeefd 100644 Binary files a/dev/benchmark/taxi-bench.png and b/dev/benchmark/taxi-bench.png differ diff --git a/doc/chdb.md b/doc/chdb.md index 74c9e82..2ca8acf 100644 --- a/doc/chdb.md +++ b/doc/chdb.md @@ -1,5 +1,5 @@ -chdb 0.1.0 -=========== +chdb 0.1.1 +========== ## Synopsis @@ -153,8 +153,8 @@ Once installed, PostgreSQL tracks two variations of the version: function. * The extension version (defined in the control file) includes only the major and minor versions, visible in the `pg_catalog.pg_extension` table, - the output of the `pg_available_extension_versions()` function, and `\dx - pg_clickhouse`. + the output of the `pg_available_extension_versions()` function, and + `\dx chdb`. In practice this means that a release that increments the patch version, e.g. from `v0.1.0` to `v0.1.1`, benefits all databases that have loaded `v0.1` and @@ -162,8 +162,8 @@ do not need to run `ALTER EXTENSION` to benefit from the upgrade. A release that increments the minor or major versions, on the other hand, will be accompanied by SQL upgrade scripts, and all existing database that contain -the extension must run `ALTER EXTENSION pg_clickhouse UPDATE` to benefit from -the upgrade. +the extension must run `ALTER EXTENSION chdb UPDATE` to benefit from the +upgrade. ## Authors diff --git a/doc/chdb_hook.md b/doc/chdb_hook.md index 81826fd..5a8dec8 100644 --- a/doc/chdb_hook.md +++ b/doc/chdb_hook.md @@ -1,4 +1,4 @@ -chdb_hook 0.1.0 +chdb_hook 0.1.1 =============== ## Synopsis @@ -152,7 +152,7 @@ s3://{bucket}/{path} Or of an object URL: ``` -s3://{bucket}.{region}.amazonaws.com/{path} +s3://{bucket}.s3.{region}.amazonaws.com/{path} ``` #### GCS @@ -689,11 +689,11 @@ chdb_hook adheres to [Semantic Versioning] for its public releases. * The minor version increments for backward compatible SQL changes * The patch version increments for binary-only changes -Once installed, PostgreSQL the version via the the Postgres +Once installed, PostgreSQL the version via the the Postgres 18 [`pg_get_loaded_modules()`] function. ```sql -SELECT version FROM pg_get_loaded_modules() WHERE module_name = 'chdb'; +SELECT version FROM pg_get_loaded_modules() WHERE module_name = 'chdb_hook'; ``` ## Authors diff --git a/src/hook/copy.c b/src/hook/copy.c index ae31b40..5a74c21 100644 --- a/src/hook/copy.c +++ b/src/hook/copy.c @@ -277,7 +277,7 @@ make_ch_query(chdbCopyContext* ctx, StringInfo query, char** names, char** value if (ctx->scheme == s3_scheme) { if (slash && slash - uri >= strlen(AWS_HOST) && !pg_strncasecmp(slash - strlen(AWS_HOST), AWS_HOST, strlen(AWS_HOST))) { - /* s3://{bucket}.{region}.amazonaws.com/{path} */ + /* s3://{bucket}.s3.{region}.amazonaws.com/{path} */ PARAM("{url:String}", "url", psprintf("https://%s", uri)); } else { /* s3://{bucket}/{path} */