From b155e71919905df9e44606d69753f807c67ee961 Mon Sep 17 00:00:00 2001 From: David Meng Date: Tue, 7 Jul 2026 19:58:20 -0700 Subject: [PATCH 1/3] Add clickhouse-trash-schema benchmark The same as ClickBench, but the `hits` table is split into a hundred tables with two columns each: `id` and one of the original columns, and `hits` becomes a view joining all of them by `id`. See https://github.com/ClickHouse/ClickHouse/issues/109426 --- clickhouse-trash-schema/README.md | 7 + clickhouse-trash-schema/benchmark.sh | 4 + clickhouse-trash-schema/check | 4 + clickhouse-trash-schema/create.sql | 433 ++++++++++++++++++++++++++ clickhouse-trash-schema/data-size | 4 + clickhouse-trash-schema/install | 16 + clickhouse-trash-schema/load | 18 ++ clickhouse-trash-schema/queries.sql | 43 +++ clickhouse-trash-schema/query | 8 + clickhouse-trash-schema/split.sql | 106 +++++++ clickhouse-trash-schema/start | 4 + clickhouse-trash-schema/stop | 3 + clickhouse-trash-schema/template.json | 11 + 13 files changed, 661 insertions(+) create mode 100644 clickhouse-trash-schema/README.md create mode 100755 clickhouse-trash-schema/benchmark.sh create mode 100755 clickhouse-trash-schema/check create mode 100644 clickhouse-trash-schema/create.sql create mode 100755 clickhouse-trash-schema/data-size create mode 100755 clickhouse-trash-schema/install create mode 100755 clickhouse-trash-schema/load create mode 100644 clickhouse-trash-schema/queries.sql create mode 100755 clickhouse-trash-schema/query create mode 100644 clickhouse-trash-schema/split.sql create mode 100755 clickhouse-trash-schema/start create mode 100755 clickhouse-trash-schema/stop create mode 100644 clickhouse-trash-schema/template.json diff --git a/clickhouse-trash-schema/README.md b/clickhouse-trash-schema/README.md new file mode 100644 index 0000000000..39df39c7bb --- /dev/null +++ b/clickhouse-trash-schema/README.md @@ -0,0 +1,7 @@ +# Trash Schema Benchmark + +The same as ClickBench, but the `hits` table is split into a hundred tables with two columns each: `id` and one of the original columns. `hits` becomes a view that joins all of them by `id`, so every query has to join a hundred tables. + +This models data science and AI workloads where there is often a large dataset of objects (images, comments, HTML pages) with iteratively computed properties (classification, embeddings, annotations), each stored in a separate "parallel" table sharing the same primary key. The goal is to find opportunities to optimize queries on top of such a schema. + +The queries and settings are identical to ClickBench. `create.sql` defines the per-column tables sorted by `id` and the `hits` view; `split.sql` distributes the loaded data from a staging table into the per-column tables. diff --git a/clickhouse-trash-schema/benchmark.sh b/clickhouse-trash-schema/benchmark.sh new file mode 100755 index 0000000000..12a00b626c --- /dev/null +++ b/clickhouse-trash-schema/benchmark.sh @@ -0,0 +1,4 @@ +#!/bin/bash +# Thin shim — actual flow is in lib/benchmark-common.sh. +export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-partitioned" +exec ../lib/benchmark-common.sh diff --git a/clickhouse-trash-schema/check b/clickhouse-trash-schema/check new file mode 100755 index 0000000000..febe4e0de2 --- /dev/null +++ b/clickhouse-trash-schema/check @@ -0,0 +1,4 @@ +#!/bin/bash +set -e + +clickhouse-client --query "SELECT 1" >/dev/null diff --git a/clickhouse-trash-schema/create.sql b/clickhouse-trash-schema/create.sql new file mode 100644 index 0000000000..c68ab96c31 --- /dev/null +++ b/clickhouse-trash-schema/create.sql @@ -0,0 +1,433 @@ +-- Same table settings as in ../clickhouse/create.sql. + +CREATE OR REPLACE TABLE hits_staging +( + id BIGINT NOT NULL, + WatchID BIGINT NOT NULL, + JavaEnable SMALLINT NOT NULL, + Title TEXT NOT NULL, + GoodEvent SMALLINT NOT NULL, + EventTime TIMESTAMP NOT NULL, + EventDate Date NOT NULL, + CounterID INTEGER NOT NULL, + ClientIP INTEGER NOT NULL, + RegionID INTEGER NOT NULL, + UserID BIGINT NOT NULL, + CounterClass SMALLINT NOT NULL, + OS SMALLINT NOT NULL, + UserAgent SMALLINT NOT NULL, + URL TEXT NOT NULL, + Referer TEXT NOT NULL, + IsRefresh SMALLINT NOT NULL, + RefererCategoryID SMALLINT NOT NULL, + RefererRegionID INTEGER NOT NULL, + URLCategoryID SMALLINT NOT NULL, + URLRegionID INTEGER NOT NULL, + ResolutionWidth SMALLINT NOT NULL, + ResolutionHeight SMALLINT NOT NULL, + ResolutionDepth SMALLINT NOT NULL, + FlashMajor SMALLINT NOT NULL, + FlashMinor SMALLINT NOT NULL, + FlashMinor2 TEXT NOT NULL, + NetMajor SMALLINT NOT NULL, + NetMinor SMALLINT NOT NULL, + UserAgentMajor SMALLINT NOT NULL, + UserAgentMinor VARCHAR(255) NOT NULL, + CookieEnable SMALLINT NOT NULL, + JavascriptEnable SMALLINT NOT NULL, + IsMobile SMALLINT NOT NULL, + MobilePhone SMALLINT NOT NULL, + MobilePhoneModel TEXT NOT NULL, + Params TEXT NOT NULL, + IPNetworkID INTEGER NOT NULL, + TraficSourceID SMALLINT NOT NULL, + SearchEngineID SMALLINT NOT NULL, + SearchPhrase TEXT NOT NULL, + AdvEngineID SMALLINT NOT NULL, + IsArtifical SMALLINT NOT NULL, + WindowClientWidth SMALLINT NOT NULL, + WindowClientHeight SMALLINT NOT NULL, + ClientTimeZone SMALLINT NOT NULL, + ClientEventTime TIMESTAMP NOT NULL, + SilverlightVersion1 SMALLINT NOT NULL, + SilverlightVersion2 SMALLINT NOT NULL, + SilverlightVersion3 INTEGER NOT NULL, + SilverlightVersion4 SMALLINT NOT NULL, + PageCharset TEXT NOT NULL, + CodeVersion INTEGER NOT NULL, + IsLink SMALLINT NOT NULL, + IsDownload SMALLINT NOT NULL, + IsNotBounce SMALLINT NOT NULL, + FUniqID BIGINT NOT NULL, + OriginalURL TEXT NOT NULL, + HID INTEGER NOT NULL, + IsOldCounter SMALLINT NOT NULL, + IsEvent SMALLINT NOT NULL, + IsParameter SMALLINT NOT NULL, + DontCountHits SMALLINT NOT NULL, + WithHash SMALLINT NOT NULL, + HitColor CHAR NOT NULL, + LocalEventTime TIMESTAMP NOT NULL, + Age SMALLINT NOT NULL, + Sex SMALLINT NOT NULL, + Income SMALLINT NOT NULL, + Interests SMALLINT NOT NULL, + Robotness SMALLINT NOT NULL, + RemoteIP INTEGER NOT NULL, + WindowName INTEGER NOT NULL, + OpenerName INTEGER NOT NULL, + HistoryLength SMALLINT NOT NULL, + BrowserLanguage TEXT NOT NULL, + BrowserCountry TEXT NOT NULL, + SocialNetwork TEXT NOT NULL, + SocialAction TEXT NOT NULL, + HTTPError SMALLINT NOT NULL, + SendTiming INTEGER NOT NULL, + DNSTiming INTEGER NOT NULL, + ConnectTiming INTEGER NOT NULL, + ResponseStartTiming INTEGER NOT NULL, + ResponseEndTiming INTEGER NOT NULL, + FetchTiming INTEGER NOT NULL, + SocialSourceNetworkID SMALLINT NOT NULL, + SocialSourcePage TEXT NOT NULL, + ParamPrice BIGINT NOT NULL, + ParamOrderID TEXT NOT NULL, + ParamCurrency TEXT NOT NULL, + ParamCurrencyID SMALLINT NOT NULL, + OpenstatServiceName TEXT NOT NULL, + OpenstatCampaignID TEXT NOT NULL, + OpenstatAdID TEXT NOT NULL, + OpenstatSourceID TEXT NOT NULL, + UTMSource TEXT NOT NULL, + UTMMedium TEXT NOT NULL, + UTMCampaign TEXT NOT NULL, + UTMContent TEXT NOT NULL, + UTMTerm TEXT NOT NULL, + FromTag TEXT NOT NULL, + HasGCLID SMALLINT NOT NULL, + RefererHash BIGINT NOT NULL, + URLHash BIGINT NOT NULL, + CLID INTEGER NOT NULL, + PRIMARY KEY (id) +) +ENGINE = MergeTree +SETTINGS fsync_after_insert = 1, auto_statistics_types = ''; + +CREATE OR REPLACE TABLE hits_WatchID (id BIGINT NOT NULL, WatchID BIGINT NOT NULL, PRIMARY KEY (id)) ENGINE = MergeTree SETTINGS fsync_after_insert = 1, auto_statistics_types = ''; +CREATE OR REPLACE TABLE hits_JavaEnable (id BIGINT NOT NULL, JavaEnable SMALLINT NOT NULL, PRIMARY KEY (id)) ENGINE = MergeTree SETTINGS fsync_after_insert = 1, auto_statistics_types = ''; +CREATE OR REPLACE TABLE hits_Title (id BIGINT NOT NULL, Title TEXT NOT NULL, PRIMARY KEY (id)) ENGINE = MergeTree SETTINGS fsync_after_insert = 1, auto_statistics_types = ''; +CREATE OR REPLACE TABLE hits_GoodEvent (id BIGINT NOT NULL, GoodEvent SMALLINT NOT NULL, PRIMARY KEY (id)) ENGINE = MergeTree SETTINGS fsync_after_insert = 1, auto_statistics_types = ''; +CREATE OR REPLACE TABLE hits_EventTime (id BIGINT NOT NULL, EventTime TIMESTAMP NOT NULL, PRIMARY KEY (id)) ENGINE = MergeTree SETTINGS fsync_after_insert = 1, auto_statistics_types = ''; +CREATE OR REPLACE TABLE hits_EventDate (id BIGINT NOT NULL, EventDate Date NOT NULL, PRIMARY KEY (id)) ENGINE = MergeTree SETTINGS fsync_after_insert = 1, auto_statistics_types = ''; +CREATE OR REPLACE TABLE hits_CounterID (id BIGINT NOT NULL, CounterID INTEGER NOT NULL, PRIMARY KEY (id)) ENGINE = MergeTree SETTINGS fsync_after_insert = 1, auto_statistics_types = ''; +CREATE OR REPLACE TABLE hits_ClientIP (id BIGINT NOT NULL, ClientIP INTEGER NOT NULL, PRIMARY KEY (id)) ENGINE = MergeTree SETTINGS fsync_after_insert = 1, auto_statistics_types = ''; +CREATE OR REPLACE TABLE hits_RegionID (id BIGINT NOT NULL, RegionID INTEGER NOT NULL, PRIMARY KEY (id)) ENGINE = MergeTree SETTINGS fsync_after_insert = 1, auto_statistics_types = ''; +CREATE OR REPLACE TABLE hits_UserID (id BIGINT NOT NULL, UserID BIGINT NOT NULL, PRIMARY KEY (id)) ENGINE = MergeTree SETTINGS fsync_after_insert = 1, auto_statistics_types = ''; +CREATE OR REPLACE TABLE hits_CounterClass (id BIGINT NOT NULL, CounterClass SMALLINT NOT NULL, PRIMARY KEY (id)) ENGINE = MergeTree SETTINGS fsync_after_insert = 1, auto_statistics_types = ''; +CREATE OR REPLACE TABLE hits_OS (id BIGINT NOT NULL, OS SMALLINT NOT NULL, PRIMARY KEY (id)) ENGINE = MergeTree SETTINGS fsync_after_insert = 1, auto_statistics_types = ''; +CREATE OR REPLACE TABLE hits_UserAgent (id BIGINT NOT NULL, UserAgent SMALLINT NOT NULL, PRIMARY KEY (id)) ENGINE = MergeTree SETTINGS fsync_after_insert = 1, auto_statistics_types = ''; +CREATE OR REPLACE TABLE hits_URL (id BIGINT NOT NULL, URL TEXT NOT NULL, PRIMARY KEY (id)) ENGINE = MergeTree SETTINGS fsync_after_insert = 1, auto_statistics_types = ''; +CREATE OR REPLACE TABLE hits_Referer (id BIGINT NOT NULL, Referer TEXT NOT NULL, PRIMARY KEY (id)) ENGINE = MergeTree SETTINGS fsync_after_insert = 1, auto_statistics_types = ''; +CREATE OR REPLACE TABLE hits_IsRefresh (id BIGINT NOT NULL, IsRefresh SMALLINT NOT NULL, PRIMARY KEY (id)) ENGINE = MergeTree SETTINGS fsync_after_insert = 1, auto_statistics_types = ''; +CREATE OR REPLACE TABLE hits_RefererCategoryID (id BIGINT NOT NULL, RefererCategoryID SMALLINT NOT NULL, PRIMARY KEY (id)) ENGINE = MergeTree SETTINGS fsync_after_insert = 1, auto_statistics_types = ''; +CREATE OR REPLACE TABLE hits_RefererRegionID (id BIGINT NOT NULL, RefererRegionID INTEGER NOT NULL, PRIMARY KEY (id)) ENGINE = MergeTree SETTINGS fsync_after_insert = 1, auto_statistics_types = ''; +CREATE OR REPLACE TABLE hits_URLCategoryID (id BIGINT NOT NULL, URLCategoryID SMALLINT NOT NULL, PRIMARY KEY (id)) ENGINE = MergeTree SETTINGS fsync_after_insert = 1, auto_statistics_types = ''; +CREATE OR REPLACE TABLE hits_URLRegionID (id BIGINT NOT NULL, URLRegionID INTEGER NOT NULL, PRIMARY KEY (id)) ENGINE = MergeTree SETTINGS fsync_after_insert = 1, auto_statistics_types = ''; +CREATE OR REPLACE TABLE hits_ResolutionWidth (id BIGINT NOT NULL, ResolutionWidth SMALLINT NOT NULL, PRIMARY KEY (id)) ENGINE = MergeTree SETTINGS fsync_after_insert = 1, auto_statistics_types = ''; +CREATE OR REPLACE TABLE hits_ResolutionHeight (id BIGINT NOT NULL, ResolutionHeight SMALLINT NOT NULL, PRIMARY KEY (id)) ENGINE = MergeTree SETTINGS fsync_after_insert = 1, auto_statistics_types = ''; +CREATE OR REPLACE TABLE hits_ResolutionDepth (id BIGINT NOT NULL, ResolutionDepth SMALLINT NOT NULL, PRIMARY KEY (id)) ENGINE = MergeTree SETTINGS fsync_after_insert = 1, auto_statistics_types = ''; +CREATE OR REPLACE TABLE hits_FlashMajor (id BIGINT NOT NULL, FlashMajor SMALLINT NOT NULL, PRIMARY KEY (id)) ENGINE = MergeTree SETTINGS fsync_after_insert = 1, auto_statistics_types = ''; +CREATE OR REPLACE TABLE hits_FlashMinor (id BIGINT NOT NULL, FlashMinor SMALLINT NOT NULL, PRIMARY KEY (id)) ENGINE = MergeTree SETTINGS fsync_after_insert = 1, auto_statistics_types = ''; +CREATE OR REPLACE TABLE hits_FlashMinor2 (id BIGINT NOT NULL, FlashMinor2 TEXT NOT NULL, PRIMARY KEY (id)) ENGINE = MergeTree SETTINGS fsync_after_insert = 1, auto_statistics_types = ''; +CREATE OR REPLACE TABLE hits_NetMajor (id BIGINT NOT NULL, NetMajor SMALLINT NOT NULL, PRIMARY KEY (id)) ENGINE = MergeTree SETTINGS fsync_after_insert = 1, auto_statistics_types = ''; +CREATE OR REPLACE TABLE hits_NetMinor (id BIGINT NOT NULL, NetMinor SMALLINT NOT NULL, PRIMARY KEY (id)) ENGINE = MergeTree SETTINGS fsync_after_insert = 1, auto_statistics_types = ''; +CREATE OR REPLACE TABLE hits_UserAgentMajor (id BIGINT NOT NULL, UserAgentMajor SMALLINT NOT NULL, PRIMARY KEY (id)) ENGINE = MergeTree SETTINGS fsync_after_insert = 1, auto_statistics_types = ''; +CREATE OR REPLACE TABLE hits_UserAgentMinor (id BIGINT NOT NULL, UserAgentMinor VARCHAR(255) NOT NULL, PRIMARY KEY (id)) ENGINE = MergeTree SETTINGS fsync_after_insert = 1, auto_statistics_types = ''; +CREATE OR REPLACE TABLE hits_CookieEnable (id BIGINT NOT NULL, CookieEnable SMALLINT NOT NULL, PRIMARY KEY (id)) ENGINE = MergeTree SETTINGS fsync_after_insert = 1, auto_statistics_types = ''; +CREATE OR REPLACE TABLE hits_JavascriptEnable (id BIGINT NOT NULL, JavascriptEnable SMALLINT NOT NULL, PRIMARY KEY (id)) ENGINE = MergeTree SETTINGS fsync_after_insert = 1, auto_statistics_types = ''; +CREATE OR REPLACE TABLE hits_IsMobile (id BIGINT NOT NULL, IsMobile SMALLINT NOT NULL, PRIMARY KEY (id)) ENGINE = MergeTree SETTINGS fsync_after_insert = 1, auto_statistics_types = ''; +CREATE OR REPLACE TABLE hits_MobilePhone (id BIGINT NOT NULL, MobilePhone SMALLINT NOT NULL, PRIMARY KEY (id)) ENGINE = MergeTree SETTINGS fsync_after_insert = 1, auto_statistics_types = ''; +CREATE OR REPLACE TABLE hits_MobilePhoneModel (id BIGINT NOT NULL, MobilePhoneModel TEXT NOT NULL, PRIMARY KEY (id)) ENGINE = MergeTree SETTINGS fsync_after_insert = 1, auto_statistics_types = ''; +CREATE OR REPLACE TABLE hits_Params (id BIGINT NOT NULL, Params TEXT NOT NULL, PRIMARY KEY (id)) ENGINE = MergeTree SETTINGS fsync_after_insert = 1, auto_statistics_types = ''; +CREATE OR REPLACE TABLE hits_IPNetworkID (id BIGINT NOT NULL, IPNetworkID INTEGER NOT NULL, PRIMARY KEY (id)) ENGINE = MergeTree SETTINGS fsync_after_insert = 1, auto_statistics_types = ''; +CREATE OR REPLACE TABLE hits_TraficSourceID (id BIGINT NOT NULL, TraficSourceID SMALLINT NOT NULL, PRIMARY KEY (id)) ENGINE = MergeTree SETTINGS fsync_after_insert = 1, auto_statistics_types = ''; +CREATE OR REPLACE TABLE hits_SearchEngineID (id BIGINT NOT NULL, SearchEngineID SMALLINT NOT NULL, PRIMARY KEY (id)) ENGINE = MergeTree SETTINGS fsync_after_insert = 1, auto_statistics_types = ''; +CREATE OR REPLACE TABLE hits_SearchPhrase (id BIGINT NOT NULL, SearchPhrase TEXT NOT NULL, PRIMARY KEY (id)) ENGINE = MergeTree SETTINGS fsync_after_insert = 1, auto_statistics_types = ''; +CREATE OR REPLACE TABLE hits_AdvEngineID (id BIGINT NOT NULL, AdvEngineID SMALLINT NOT NULL, PRIMARY KEY (id)) ENGINE = MergeTree SETTINGS fsync_after_insert = 1, auto_statistics_types = ''; +CREATE OR REPLACE TABLE hits_IsArtifical (id BIGINT NOT NULL, IsArtifical SMALLINT NOT NULL, PRIMARY KEY (id)) ENGINE = MergeTree SETTINGS fsync_after_insert = 1, auto_statistics_types = ''; +CREATE OR REPLACE TABLE hits_WindowClientWidth (id BIGINT NOT NULL, WindowClientWidth SMALLINT NOT NULL, PRIMARY KEY (id)) ENGINE = MergeTree SETTINGS fsync_after_insert = 1, auto_statistics_types = ''; +CREATE OR REPLACE TABLE hits_WindowClientHeight (id BIGINT NOT NULL, WindowClientHeight SMALLINT NOT NULL, PRIMARY KEY (id)) ENGINE = MergeTree SETTINGS fsync_after_insert = 1, auto_statistics_types = ''; +CREATE OR REPLACE TABLE hits_ClientTimeZone (id BIGINT NOT NULL, ClientTimeZone SMALLINT NOT NULL, PRIMARY KEY (id)) ENGINE = MergeTree SETTINGS fsync_after_insert = 1, auto_statistics_types = ''; +CREATE OR REPLACE TABLE hits_ClientEventTime (id BIGINT NOT NULL, ClientEventTime TIMESTAMP NOT NULL, PRIMARY KEY (id)) ENGINE = MergeTree SETTINGS fsync_after_insert = 1, auto_statistics_types = ''; +CREATE OR REPLACE TABLE hits_SilverlightVersion1 (id BIGINT NOT NULL, SilverlightVersion1 SMALLINT NOT NULL, PRIMARY KEY (id)) ENGINE = MergeTree SETTINGS fsync_after_insert = 1, auto_statistics_types = ''; +CREATE OR REPLACE TABLE hits_SilverlightVersion2 (id BIGINT NOT NULL, SilverlightVersion2 SMALLINT NOT NULL, PRIMARY KEY (id)) ENGINE = MergeTree SETTINGS fsync_after_insert = 1, auto_statistics_types = ''; +CREATE OR REPLACE TABLE hits_SilverlightVersion3 (id BIGINT NOT NULL, SilverlightVersion3 INTEGER NOT NULL, PRIMARY KEY (id)) ENGINE = MergeTree SETTINGS fsync_after_insert = 1, auto_statistics_types = ''; +CREATE OR REPLACE TABLE hits_SilverlightVersion4 (id BIGINT NOT NULL, SilverlightVersion4 SMALLINT NOT NULL, PRIMARY KEY (id)) ENGINE = MergeTree SETTINGS fsync_after_insert = 1, auto_statistics_types = ''; +CREATE OR REPLACE TABLE hits_PageCharset (id BIGINT NOT NULL, PageCharset TEXT NOT NULL, PRIMARY KEY (id)) ENGINE = MergeTree SETTINGS fsync_after_insert = 1, auto_statistics_types = ''; +CREATE OR REPLACE TABLE hits_CodeVersion (id BIGINT NOT NULL, CodeVersion INTEGER NOT NULL, PRIMARY KEY (id)) ENGINE = MergeTree SETTINGS fsync_after_insert = 1, auto_statistics_types = ''; +CREATE OR REPLACE TABLE hits_IsLink (id BIGINT NOT NULL, IsLink SMALLINT NOT NULL, PRIMARY KEY (id)) ENGINE = MergeTree SETTINGS fsync_after_insert = 1, auto_statistics_types = ''; +CREATE OR REPLACE TABLE hits_IsDownload (id BIGINT NOT NULL, IsDownload SMALLINT NOT NULL, PRIMARY KEY (id)) ENGINE = MergeTree SETTINGS fsync_after_insert = 1, auto_statistics_types = ''; +CREATE OR REPLACE TABLE hits_IsNotBounce (id BIGINT NOT NULL, IsNotBounce SMALLINT NOT NULL, PRIMARY KEY (id)) ENGINE = MergeTree SETTINGS fsync_after_insert = 1, auto_statistics_types = ''; +CREATE OR REPLACE TABLE hits_FUniqID (id BIGINT NOT NULL, FUniqID BIGINT NOT NULL, PRIMARY KEY (id)) ENGINE = MergeTree SETTINGS fsync_after_insert = 1, auto_statistics_types = ''; +CREATE OR REPLACE TABLE hits_OriginalURL (id BIGINT NOT NULL, OriginalURL TEXT NOT NULL, PRIMARY KEY (id)) ENGINE = MergeTree SETTINGS fsync_after_insert = 1, auto_statistics_types = ''; +CREATE OR REPLACE TABLE hits_HID (id BIGINT NOT NULL, HID INTEGER NOT NULL, PRIMARY KEY (id)) ENGINE = MergeTree SETTINGS fsync_after_insert = 1, auto_statistics_types = ''; +CREATE OR REPLACE TABLE hits_IsOldCounter (id BIGINT NOT NULL, IsOldCounter SMALLINT NOT NULL, PRIMARY KEY (id)) ENGINE = MergeTree SETTINGS fsync_after_insert = 1, auto_statistics_types = ''; +CREATE OR REPLACE TABLE hits_IsEvent (id BIGINT NOT NULL, IsEvent SMALLINT NOT NULL, PRIMARY KEY (id)) ENGINE = MergeTree SETTINGS fsync_after_insert = 1, auto_statistics_types = ''; +CREATE OR REPLACE TABLE hits_IsParameter (id BIGINT NOT NULL, IsParameter SMALLINT NOT NULL, PRIMARY KEY (id)) ENGINE = MergeTree SETTINGS fsync_after_insert = 1, auto_statistics_types = ''; +CREATE OR REPLACE TABLE hits_DontCountHits (id BIGINT NOT NULL, DontCountHits SMALLINT NOT NULL, PRIMARY KEY (id)) ENGINE = MergeTree SETTINGS fsync_after_insert = 1, auto_statistics_types = ''; +CREATE OR REPLACE TABLE hits_WithHash (id BIGINT NOT NULL, WithHash SMALLINT NOT NULL, PRIMARY KEY (id)) ENGINE = MergeTree SETTINGS fsync_after_insert = 1, auto_statistics_types = ''; +CREATE OR REPLACE TABLE hits_HitColor (id BIGINT NOT NULL, HitColor CHAR NOT NULL, PRIMARY KEY (id)) ENGINE = MergeTree SETTINGS fsync_after_insert = 1, auto_statistics_types = ''; +CREATE OR REPLACE TABLE hits_LocalEventTime (id BIGINT NOT NULL, LocalEventTime TIMESTAMP NOT NULL, PRIMARY KEY (id)) ENGINE = MergeTree SETTINGS fsync_after_insert = 1, auto_statistics_types = ''; +CREATE OR REPLACE TABLE hits_Age (id BIGINT NOT NULL, Age SMALLINT NOT NULL, PRIMARY KEY (id)) ENGINE = MergeTree SETTINGS fsync_after_insert = 1, auto_statistics_types = ''; +CREATE OR REPLACE TABLE hits_Sex (id BIGINT NOT NULL, Sex SMALLINT NOT NULL, PRIMARY KEY (id)) ENGINE = MergeTree SETTINGS fsync_after_insert = 1, auto_statistics_types = ''; +CREATE OR REPLACE TABLE hits_Income (id BIGINT NOT NULL, Income SMALLINT NOT NULL, PRIMARY KEY (id)) ENGINE = MergeTree SETTINGS fsync_after_insert = 1, auto_statistics_types = ''; +CREATE OR REPLACE TABLE hits_Interests (id BIGINT NOT NULL, Interests SMALLINT NOT NULL, PRIMARY KEY (id)) ENGINE = MergeTree SETTINGS fsync_after_insert = 1, auto_statistics_types = ''; +CREATE OR REPLACE TABLE hits_Robotness (id BIGINT NOT NULL, Robotness SMALLINT NOT NULL, PRIMARY KEY (id)) ENGINE = MergeTree SETTINGS fsync_after_insert = 1, auto_statistics_types = ''; +CREATE OR REPLACE TABLE hits_RemoteIP (id BIGINT NOT NULL, RemoteIP INTEGER NOT NULL, PRIMARY KEY (id)) ENGINE = MergeTree SETTINGS fsync_after_insert = 1, auto_statistics_types = ''; +CREATE OR REPLACE TABLE hits_WindowName (id BIGINT NOT NULL, WindowName INTEGER NOT NULL, PRIMARY KEY (id)) ENGINE = MergeTree SETTINGS fsync_after_insert = 1, auto_statistics_types = ''; +CREATE OR REPLACE TABLE hits_OpenerName (id BIGINT NOT NULL, OpenerName INTEGER NOT NULL, PRIMARY KEY (id)) ENGINE = MergeTree SETTINGS fsync_after_insert = 1, auto_statistics_types = ''; +CREATE OR REPLACE TABLE hits_HistoryLength (id BIGINT NOT NULL, HistoryLength SMALLINT NOT NULL, PRIMARY KEY (id)) ENGINE = MergeTree SETTINGS fsync_after_insert = 1, auto_statistics_types = ''; +CREATE OR REPLACE TABLE hits_BrowserLanguage (id BIGINT NOT NULL, BrowserLanguage TEXT NOT NULL, PRIMARY KEY (id)) ENGINE = MergeTree SETTINGS fsync_after_insert = 1, auto_statistics_types = ''; +CREATE OR REPLACE TABLE hits_BrowserCountry (id BIGINT NOT NULL, BrowserCountry TEXT NOT NULL, PRIMARY KEY (id)) ENGINE = MergeTree SETTINGS fsync_after_insert = 1, auto_statistics_types = ''; +CREATE OR REPLACE TABLE hits_SocialNetwork (id BIGINT NOT NULL, SocialNetwork TEXT NOT NULL, PRIMARY KEY (id)) ENGINE = MergeTree SETTINGS fsync_after_insert = 1, auto_statistics_types = ''; +CREATE OR REPLACE TABLE hits_SocialAction (id BIGINT NOT NULL, SocialAction TEXT NOT NULL, PRIMARY KEY (id)) ENGINE = MergeTree SETTINGS fsync_after_insert = 1, auto_statistics_types = ''; +CREATE OR REPLACE TABLE hits_HTTPError (id BIGINT NOT NULL, HTTPError SMALLINT NOT NULL, PRIMARY KEY (id)) ENGINE = MergeTree SETTINGS fsync_after_insert = 1, auto_statistics_types = ''; +CREATE OR REPLACE TABLE hits_SendTiming (id BIGINT NOT NULL, SendTiming INTEGER NOT NULL, PRIMARY KEY (id)) ENGINE = MergeTree SETTINGS fsync_after_insert = 1, auto_statistics_types = ''; +CREATE OR REPLACE TABLE hits_DNSTiming (id BIGINT NOT NULL, DNSTiming INTEGER NOT NULL, PRIMARY KEY (id)) ENGINE = MergeTree SETTINGS fsync_after_insert = 1, auto_statistics_types = ''; +CREATE OR REPLACE TABLE hits_ConnectTiming (id BIGINT NOT NULL, ConnectTiming INTEGER NOT NULL, PRIMARY KEY (id)) ENGINE = MergeTree SETTINGS fsync_after_insert = 1, auto_statistics_types = ''; +CREATE OR REPLACE TABLE hits_ResponseStartTiming (id BIGINT NOT NULL, ResponseStartTiming INTEGER NOT NULL, PRIMARY KEY (id)) ENGINE = MergeTree SETTINGS fsync_after_insert = 1, auto_statistics_types = ''; +CREATE OR REPLACE TABLE hits_ResponseEndTiming (id BIGINT NOT NULL, ResponseEndTiming INTEGER NOT NULL, PRIMARY KEY (id)) ENGINE = MergeTree SETTINGS fsync_after_insert = 1, auto_statistics_types = ''; +CREATE OR REPLACE TABLE hits_FetchTiming (id BIGINT NOT NULL, FetchTiming INTEGER NOT NULL, PRIMARY KEY (id)) ENGINE = MergeTree SETTINGS fsync_after_insert = 1, auto_statistics_types = ''; +CREATE OR REPLACE TABLE hits_SocialSourceNetworkID (id BIGINT NOT NULL, SocialSourceNetworkID SMALLINT NOT NULL, PRIMARY KEY (id)) ENGINE = MergeTree SETTINGS fsync_after_insert = 1, auto_statistics_types = ''; +CREATE OR REPLACE TABLE hits_SocialSourcePage (id BIGINT NOT NULL, SocialSourcePage TEXT NOT NULL, PRIMARY KEY (id)) ENGINE = MergeTree SETTINGS fsync_after_insert = 1, auto_statistics_types = ''; +CREATE OR REPLACE TABLE hits_ParamPrice (id BIGINT NOT NULL, ParamPrice BIGINT NOT NULL, PRIMARY KEY (id)) ENGINE = MergeTree SETTINGS fsync_after_insert = 1, auto_statistics_types = ''; +CREATE OR REPLACE TABLE hits_ParamOrderID (id BIGINT NOT NULL, ParamOrderID TEXT NOT NULL, PRIMARY KEY (id)) ENGINE = MergeTree SETTINGS fsync_after_insert = 1, auto_statistics_types = ''; +CREATE OR REPLACE TABLE hits_ParamCurrency (id BIGINT NOT NULL, ParamCurrency TEXT NOT NULL, PRIMARY KEY (id)) ENGINE = MergeTree SETTINGS fsync_after_insert = 1, auto_statistics_types = ''; +CREATE OR REPLACE TABLE hits_ParamCurrencyID (id BIGINT NOT NULL, ParamCurrencyID SMALLINT NOT NULL, PRIMARY KEY (id)) ENGINE = MergeTree SETTINGS fsync_after_insert = 1, auto_statistics_types = ''; +CREATE OR REPLACE TABLE hits_OpenstatServiceName (id BIGINT NOT NULL, OpenstatServiceName TEXT NOT NULL, PRIMARY KEY (id)) ENGINE = MergeTree SETTINGS fsync_after_insert = 1, auto_statistics_types = ''; +CREATE OR REPLACE TABLE hits_OpenstatCampaignID (id BIGINT NOT NULL, OpenstatCampaignID TEXT NOT NULL, PRIMARY KEY (id)) ENGINE = MergeTree SETTINGS fsync_after_insert = 1, auto_statistics_types = ''; +CREATE OR REPLACE TABLE hits_OpenstatAdID (id BIGINT NOT NULL, OpenstatAdID TEXT NOT NULL, PRIMARY KEY (id)) ENGINE = MergeTree SETTINGS fsync_after_insert = 1, auto_statistics_types = ''; +CREATE OR REPLACE TABLE hits_OpenstatSourceID (id BIGINT NOT NULL, OpenstatSourceID TEXT NOT NULL, PRIMARY KEY (id)) ENGINE = MergeTree SETTINGS fsync_after_insert = 1, auto_statistics_types = ''; +CREATE OR REPLACE TABLE hits_UTMSource (id BIGINT NOT NULL, UTMSource TEXT NOT NULL, PRIMARY KEY (id)) ENGINE = MergeTree SETTINGS fsync_after_insert = 1, auto_statistics_types = ''; +CREATE OR REPLACE TABLE hits_UTMMedium (id BIGINT NOT NULL, UTMMedium TEXT NOT NULL, PRIMARY KEY (id)) ENGINE = MergeTree SETTINGS fsync_after_insert = 1, auto_statistics_types = ''; +CREATE OR REPLACE TABLE hits_UTMCampaign (id BIGINT NOT NULL, UTMCampaign TEXT NOT NULL, PRIMARY KEY (id)) ENGINE = MergeTree SETTINGS fsync_after_insert = 1, auto_statistics_types = ''; +CREATE OR REPLACE TABLE hits_UTMContent (id BIGINT NOT NULL, UTMContent TEXT NOT NULL, PRIMARY KEY (id)) ENGINE = MergeTree SETTINGS fsync_after_insert = 1, auto_statistics_types = ''; +CREATE OR REPLACE TABLE hits_UTMTerm (id BIGINT NOT NULL, UTMTerm TEXT NOT NULL, PRIMARY KEY (id)) ENGINE = MergeTree SETTINGS fsync_after_insert = 1, auto_statistics_types = ''; +CREATE OR REPLACE TABLE hits_FromTag (id BIGINT NOT NULL, FromTag TEXT NOT NULL, PRIMARY KEY (id)) ENGINE = MergeTree SETTINGS fsync_after_insert = 1, auto_statistics_types = ''; +CREATE OR REPLACE TABLE hits_HasGCLID (id BIGINT NOT NULL, HasGCLID SMALLINT NOT NULL, PRIMARY KEY (id)) ENGINE = MergeTree SETTINGS fsync_after_insert = 1, auto_statistics_types = ''; +CREATE OR REPLACE TABLE hits_RefererHash (id BIGINT NOT NULL, RefererHash BIGINT NOT NULL, PRIMARY KEY (id)) ENGINE = MergeTree SETTINGS fsync_after_insert = 1, auto_statistics_types = ''; +CREATE OR REPLACE TABLE hits_URLHash (id BIGINT NOT NULL, URLHash BIGINT NOT NULL, PRIMARY KEY (id)) ENGINE = MergeTree SETTINGS fsync_after_insert = 1, auto_statistics_types = ''; +CREATE OR REPLACE TABLE hits_CLID (id BIGINT NOT NULL, CLID INTEGER NOT NULL, PRIMARY KEY (id)) ENGINE = MergeTree SETTINGS fsync_after_insert = 1, auto_statistics_types = ''; + +CREATE OR REPLACE VIEW hits AS +SELECT + WatchID, + JavaEnable, + Title, + GoodEvent, + EventTime, + EventDate, + CounterID, + ClientIP, + RegionID, + UserID, + CounterClass, + OS, + UserAgent, + URL, + Referer, + IsRefresh, + RefererCategoryID, + RefererRegionID, + URLCategoryID, + URLRegionID, + ResolutionWidth, + ResolutionHeight, + ResolutionDepth, + FlashMajor, + FlashMinor, + FlashMinor2, + NetMajor, + NetMinor, + UserAgentMajor, + UserAgentMinor, + CookieEnable, + JavascriptEnable, + IsMobile, + MobilePhone, + MobilePhoneModel, + Params, + IPNetworkID, + TraficSourceID, + SearchEngineID, + SearchPhrase, + AdvEngineID, + IsArtifical, + WindowClientWidth, + WindowClientHeight, + ClientTimeZone, + ClientEventTime, + SilverlightVersion1, + SilverlightVersion2, + SilverlightVersion3, + SilverlightVersion4, + PageCharset, + CodeVersion, + IsLink, + IsDownload, + IsNotBounce, + FUniqID, + OriginalURL, + HID, + IsOldCounter, + IsEvent, + IsParameter, + DontCountHits, + WithHash, + HitColor, + LocalEventTime, + Age, + Sex, + Income, + Interests, + Robotness, + RemoteIP, + WindowName, + OpenerName, + HistoryLength, + BrowserLanguage, + BrowserCountry, + SocialNetwork, + SocialAction, + HTTPError, + SendTiming, + DNSTiming, + ConnectTiming, + ResponseStartTiming, + ResponseEndTiming, + FetchTiming, + SocialSourceNetworkID, + SocialSourcePage, + ParamPrice, + ParamOrderID, + ParamCurrency, + ParamCurrencyID, + OpenstatServiceName, + OpenstatCampaignID, + OpenstatAdID, + OpenstatSourceID, + UTMSource, + UTMMedium, + UTMCampaign, + UTMContent, + UTMTerm, + FromTag, + HasGCLID, + RefererHash, + URLHash, + CLID +FROM hits_WatchID +JOIN hits_JavaEnable USING (id) +JOIN hits_Title USING (id) +JOIN hits_GoodEvent USING (id) +JOIN hits_EventTime USING (id) +JOIN hits_EventDate USING (id) +JOIN hits_CounterID USING (id) +JOIN hits_ClientIP USING (id) +JOIN hits_RegionID USING (id) +JOIN hits_UserID USING (id) +JOIN hits_CounterClass USING (id) +JOIN hits_OS USING (id) +JOIN hits_UserAgent USING (id) +JOIN hits_URL USING (id) +JOIN hits_Referer USING (id) +JOIN hits_IsRefresh USING (id) +JOIN hits_RefererCategoryID USING (id) +JOIN hits_RefererRegionID USING (id) +JOIN hits_URLCategoryID USING (id) +JOIN hits_URLRegionID USING (id) +JOIN hits_ResolutionWidth USING (id) +JOIN hits_ResolutionHeight USING (id) +JOIN hits_ResolutionDepth USING (id) +JOIN hits_FlashMajor USING (id) +JOIN hits_FlashMinor USING (id) +JOIN hits_FlashMinor2 USING (id) +JOIN hits_NetMajor USING (id) +JOIN hits_NetMinor USING (id) +JOIN hits_UserAgentMajor USING (id) +JOIN hits_UserAgentMinor USING (id) +JOIN hits_CookieEnable USING (id) +JOIN hits_JavascriptEnable USING (id) +JOIN hits_IsMobile USING (id) +JOIN hits_MobilePhone USING (id) +JOIN hits_MobilePhoneModel USING (id) +JOIN hits_Params USING (id) +JOIN hits_IPNetworkID USING (id) +JOIN hits_TraficSourceID USING (id) +JOIN hits_SearchEngineID USING (id) +JOIN hits_SearchPhrase USING (id) +JOIN hits_AdvEngineID USING (id) +JOIN hits_IsArtifical USING (id) +JOIN hits_WindowClientWidth USING (id) +JOIN hits_WindowClientHeight USING (id) +JOIN hits_ClientTimeZone USING (id) +JOIN hits_ClientEventTime USING (id) +JOIN hits_SilverlightVersion1 USING (id) +JOIN hits_SilverlightVersion2 USING (id) +JOIN hits_SilverlightVersion3 USING (id) +JOIN hits_SilverlightVersion4 USING (id) +JOIN hits_PageCharset USING (id) +JOIN hits_CodeVersion USING (id) +JOIN hits_IsLink USING (id) +JOIN hits_IsDownload USING (id) +JOIN hits_IsNotBounce USING (id) +JOIN hits_FUniqID USING (id) +JOIN hits_OriginalURL USING (id) +JOIN hits_HID USING (id) +JOIN hits_IsOldCounter USING (id) +JOIN hits_IsEvent USING (id) +JOIN hits_IsParameter USING (id) +JOIN hits_DontCountHits USING (id) +JOIN hits_WithHash USING (id) +JOIN hits_HitColor USING (id) +JOIN hits_LocalEventTime USING (id) +JOIN hits_Age USING (id) +JOIN hits_Sex USING (id) +JOIN hits_Income USING (id) +JOIN hits_Interests USING (id) +JOIN hits_Robotness USING (id) +JOIN hits_RemoteIP USING (id) +JOIN hits_WindowName USING (id) +JOIN hits_OpenerName USING (id) +JOIN hits_HistoryLength USING (id) +JOIN hits_BrowserLanguage USING (id) +JOIN hits_BrowserCountry USING (id) +JOIN hits_SocialNetwork USING (id) +JOIN hits_SocialAction USING (id) +JOIN hits_HTTPError USING (id) +JOIN hits_SendTiming USING (id) +JOIN hits_DNSTiming USING (id) +JOIN hits_ConnectTiming USING (id) +JOIN hits_ResponseStartTiming USING (id) +JOIN hits_ResponseEndTiming USING (id) +JOIN hits_FetchTiming USING (id) +JOIN hits_SocialSourceNetworkID USING (id) +JOIN hits_SocialSourcePage USING (id) +JOIN hits_ParamPrice USING (id) +JOIN hits_ParamOrderID USING (id) +JOIN hits_ParamCurrency USING (id) +JOIN hits_ParamCurrencyID USING (id) +JOIN hits_OpenstatServiceName USING (id) +JOIN hits_OpenstatCampaignID USING (id) +JOIN hits_OpenstatAdID USING (id) +JOIN hits_OpenstatSourceID USING (id) +JOIN hits_UTMSource USING (id) +JOIN hits_UTMMedium USING (id) +JOIN hits_UTMCampaign USING (id) +JOIN hits_UTMContent USING (id) +JOIN hits_UTMTerm USING (id) +JOIN hits_FromTag USING (id) +JOIN hits_HasGCLID USING (id) +JOIN hits_RefererHash USING (id) +JOIN hits_URLHash USING (id) +JOIN hits_CLID USING (id); diff --git a/clickhouse-trash-schema/data-size b/clickhouse-trash-schema/data-size new file mode 100755 index 0000000000..61711eb364 --- /dev/null +++ b/clickhouse-trash-schema/data-size @@ -0,0 +1,4 @@ +#!/bin/bash +set -e + +clickhouse-client --query "SELECT sum(total_bytes) FROM system.tables WHERE database = 'default' AND name LIKE 'hits\_%'" diff --git a/clickhouse-trash-schema/install b/clickhouse-trash-schema/install new file mode 100755 index 0000000000..4b46afd44c --- /dev/null +++ b/clickhouse-trash-schema/install @@ -0,0 +1,16 @@ +#!/bin/bash +set -e + +if [ ! -x /usr/bin/clickhouse ]; then + curl https://clickhouse.com/ | sh + sudo ./clickhouse install --noninteractive +fi + +# Eager startup loading, same as ../clickhouse/install. +sudo mkdir -p /etc/clickhouse-server/config.d +sudo tee /etc/clickhouse-server/config.d/eager_load.yaml >/dev/null <<'EOF' +async_load_databases: false +merge_tree: + primary_key_lazy_load: 0 + columns_and_secondary_indices_sizes_lazy_calculation: 0 +EOF diff --git a/clickhouse-trash-schema/load b/clickhouse-trash-schema/load new file mode 100755 index 0000000000..a172e85578 --- /dev/null +++ b/clickhouse-trash-schema/load @@ -0,0 +1,18 @@ +#!/bin/bash +set -e + +clickhouse-client < create.sql + +# Symlink the parquet files into user_files, same as ../clickhouse/load. +for f in hits_*.parquet; do + sudo ln -sf "$PWD/$f" /var/lib/clickhouse/user_files/"$f" +done +sudo chown -h clickhouse:clickhouse /var/lib/clickhouse/user_files/hits_*.parquet + +# row_number() OVER (), not rowNumberInAllBlocks(): the latter is per-thread and produces duplicate ids under parallel reads. +clickhouse-client --query "INSERT INTO hits_staging SELECT row_number() OVER () AS id, * FROM file('hits_*.parquet')" + +clickhouse-client < split.sql + +sudo rm -f /var/lib/clickhouse/user_files/hits_*.parquet +sync diff --git a/clickhouse-trash-schema/queries.sql b/clickhouse-trash-schema/queries.sql new file mode 100644 index 0000000000..31f65fc898 --- /dev/null +++ b/clickhouse-trash-schema/queries.sql @@ -0,0 +1,43 @@ +SELECT COUNT(*) FROM hits; +SELECT COUNT(*) FROM hits WHERE AdvEngineID <> 0; +SELECT SUM(AdvEngineID), COUNT(*), AVG(ResolutionWidth) FROM hits; +SELECT AVG(UserID) FROM hits; +SELECT COUNT(DISTINCT UserID) FROM hits; +SELECT COUNT(DISTINCT SearchPhrase) FROM hits; +SELECT MIN(EventDate), MAX(EventDate) FROM hits; +SELECT AdvEngineID, COUNT(*) FROM hits WHERE AdvEngineID <> 0 GROUP BY AdvEngineID ORDER BY COUNT(*) DESC; +SELECT RegionID, COUNT(DISTINCT UserID) AS u FROM hits GROUP BY RegionID ORDER BY u DESC LIMIT 10; +SELECT RegionID, SUM(AdvEngineID), COUNT(*) AS c, AVG(ResolutionWidth), COUNT(DISTINCT UserID) FROM hits GROUP BY RegionID ORDER BY c DESC LIMIT 10; +SELECT MobilePhoneModel, COUNT(DISTINCT UserID) AS u FROM hits WHERE MobilePhoneModel <> '' GROUP BY MobilePhoneModel ORDER BY u DESC LIMIT 10; +SELECT MobilePhone, MobilePhoneModel, COUNT(DISTINCT UserID) AS u FROM hits WHERE MobilePhoneModel <> '' GROUP BY MobilePhone, MobilePhoneModel ORDER BY u DESC LIMIT 10; +SELECT SearchPhrase, COUNT(*) AS c FROM hits WHERE SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY c DESC LIMIT 10; +SELECT SearchPhrase, COUNT(DISTINCT UserID) AS u FROM hits WHERE SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY u DESC LIMIT 10; +SELECT SearchEngineID, SearchPhrase, COUNT(*) AS c FROM hits WHERE SearchPhrase <> '' GROUP BY SearchEngineID, SearchPhrase ORDER BY c DESC LIMIT 10; +SELECT UserID, COUNT(*) FROM hits GROUP BY UserID ORDER BY COUNT(*) DESC LIMIT 10; +SELECT UserID, SearchPhrase, COUNT(*) FROM hits GROUP BY UserID, SearchPhrase ORDER BY COUNT(*) DESC LIMIT 10; +SELECT UserID, SearchPhrase, COUNT(*) FROM hits GROUP BY UserID, SearchPhrase LIMIT 10; +SELECT UserID, extract(minute FROM EventTime) AS m, SearchPhrase, COUNT(*) FROM hits GROUP BY UserID, m, SearchPhrase ORDER BY COUNT(*) DESC LIMIT 10; +SELECT UserID FROM hits WHERE UserID = 435090932899640449; +SELECT COUNT(*) FROM hits WHERE URL LIKE '%google%'; +SELECT SearchPhrase, MIN(URL), COUNT(*) AS c FROM hits WHERE URL LIKE '%google%' AND SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY c DESC LIMIT 10; +SELECT SearchPhrase, MIN(URL), MIN(Title), COUNT(*) AS c, COUNT(DISTINCT UserID) FROM hits WHERE Title LIKE '%Google%' AND URL NOT LIKE '%.google.%' AND SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY c DESC LIMIT 10; +SELECT * FROM hits WHERE URL LIKE '%google%' ORDER BY EventTime LIMIT 10; +SELECT SearchPhrase FROM hits WHERE SearchPhrase <> '' ORDER BY EventTime LIMIT 10; +SELECT SearchPhrase FROM hits WHERE SearchPhrase <> '' ORDER BY SearchPhrase LIMIT 10; +SELECT SearchPhrase FROM hits WHERE SearchPhrase <> '' ORDER BY EventTime, SearchPhrase LIMIT 10; +SELECT CounterID, AVG(length(URL)) AS l, COUNT(*) AS c FROM hits WHERE URL <> '' GROUP BY CounterID HAVING COUNT(*) > 100000 ORDER BY l DESC LIMIT 25; +SELECT REGEXP_REPLACE(Referer, '^https?://(?:www\.)?([^/]+)/.*$', '\1') AS k, AVG(length(Referer)) AS l, COUNT(*) AS c, MIN(Referer) FROM hits WHERE Referer <> '' GROUP BY k HAVING COUNT(*) > 100000 ORDER BY l DESC LIMIT 25; +SELECT SUM(ResolutionWidth), SUM(ResolutionWidth + 1), SUM(ResolutionWidth + 2), SUM(ResolutionWidth + 3), SUM(ResolutionWidth + 4), SUM(ResolutionWidth + 5), SUM(ResolutionWidth + 6), SUM(ResolutionWidth + 7), SUM(ResolutionWidth + 8), SUM(ResolutionWidth + 9), SUM(ResolutionWidth + 10), SUM(ResolutionWidth + 11), SUM(ResolutionWidth + 12), SUM(ResolutionWidth + 13), SUM(ResolutionWidth + 14), SUM(ResolutionWidth + 15), SUM(ResolutionWidth + 16), SUM(ResolutionWidth + 17), SUM(ResolutionWidth + 18), SUM(ResolutionWidth + 19), SUM(ResolutionWidth + 20), SUM(ResolutionWidth + 21), SUM(ResolutionWidth + 22), SUM(ResolutionWidth + 23), SUM(ResolutionWidth + 24), SUM(ResolutionWidth + 25), SUM(ResolutionWidth + 26), SUM(ResolutionWidth + 27), SUM(ResolutionWidth + 28), SUM(ResolutionWidth + 29), SUM(ResolutionWidth + 30), SUM(ResolutionWidth + 31), SUM(ResolutionWidth + 32), SUM(ResolutionWidth + 33), SUM(ResolutionWidth + 34), SUM(ResolutionWidth + 35), SUM(ResolutionWidth + 36), SUM(ResolutionWidth + 37), SUM(ResolutionWidth + 38), SUM(ResolutionWidth + 39), SUM(ResolutionWidth + 40), SUM(ResolutionWidth + 41), SUM(ResolutionWidth + 42), SUM(ResolutionWidth + 43), SUM(ResolutionWidth + 44), SUM(ResolutionWidth + 45), SUM(ResolutionWidth + 46), SUM(ResolutionWidth + 47), SUM(ResolutionWidth + 48), SUM(ResolutionWidth + 49), SUM(ResolutionWidth + 50), SUM(ResolutionWidth + 51), SUM(ResolutionWidth + 52), SUM(ResolutionWidth + 53), SUM(ResolutionWidth + 54), SUM(ResolutionWidth + 55), SUM(ResolutionWidth + 56), SUM(ResolutionWidth + 57), SUM(ResolutionWidth + 58), SUM(ResolutionWidth + 59), SUM(ResolutionWidth + 60), SUM(ResolutionWidth + 61), SUM(ResolutionWidth + 62), SUM(ResolutionWidth + 63), SUM(ResolutionWidth + 64), SUM(ResolutionWidth + 65), SUM(ResolutionWidth + 66), SUM(ResolutionWidth + 67), SUM(ResolutionWidth + 68), SUM(ResolutionWidth + 69), SUM(ResolutionWidth + 70), SUM(ResolutionWidth + 71), SUM(ResolutionWidth + 72), SUM(ResolutionWidth + 73), SUM(ResolutionWidth + 74), SUM(ResolutionWidth + 75), SUM(ResolutionWidth + 76), SUM(ResolutionWidth + 77), SUM(ResolutionWidth + 78), SUM(ResolutionWidth + 79), SUM(ResolutionWidth + 80), SUM(ResolutionWidth + 81), SUM(ResolutionWidth + 82), SUM(ResolutionWidth + 83), SUM(ResolutionWidth + 84), SUM(ResolutionWidth + 85), SUM(ResolutionWidth + 86), SUM(ResolutionWidth + 87), SUM(ResolutionWidth + 88), SUM(ResolutionWidth + 89) FROM hits; +SELECT SearchEngineID, ClientIP, COUNT(*) AS c, SUM(IsRefresh), AVG(ResolutionWidth) FROM hits WHERE SearchPhrase <> '' GROUP BY SearchEngineID, ClientIP ORDER BY c DESC LIMIT 10; +SELECT WatchID, ClientIP, COUNT(*) AS c, SUM(IsRefresh), AVG(ResolutionWidth) FROM hits WHERE SearchPhrase <> '' GROUP BY WatchID, ClientIP ORDER BY c DESC LIMIT 10; +SELECT WatchID, ClientIP, COUNT(*) AS c, SUM(IsRefresh), AVG(ResolutionWidth) FROM hits GROUP BY WatchID, ClientIP ORDER BY c DESC LIMIT 10; +SELECT URL, COUNT(*) AS c FROM hits GROUP BY URL ORDER BY c DESC LIMIT 10; +SELECT 1, URL, COUNT(*) AS c FROM hits GROUP BY 1, URL ORDER BY c DESC LIMIT 10; +SELECT ClientIP, ClientIP - 1, ClientIP - 2, ClientIP - 3, COUNT(*) AS c FROM hits GROUP BY ClientIP, ClientIP - 1, ClientIP - 2, ClientIP - 3 ORDER BY c DESC LIMIT 10; +SELECT URL, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND DontCountHits = 0 AND IsRefresh = 0 AND URL <> '' GROUP BY URL ORDER BY PageViews DESC LIMIT 10; +SELECT Title, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND DontCountHits = 0 AND IsRefresh = 0 AND Title <> '' GROUP BY Title ORDER BY PageViews DESC LIMIT 10; +SELECT URL, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND IsRefresh = 0 AND IsLink <> 0 AND IsDownload = 0 GROUP BY URL ORDER BY PageViews DESC LIMIT 10 OFFSET 1000; +SELECT TraficSourceID, SearchEngineID, AdvEngineID, CASE WHEN (SearchEngineID = 0 AND AdvEngineID = 0) THEN Referer ELSE '' END AS Src, URL AS Dst, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND IsRefresh = 0 GROUP BY TraficSourceID, SearchEngineID, AdvEngineID, Src, Dst ORDER BY PageViews DESC LIMIT 10 OFFSET 1000; +SELECT URLHash, EventDate, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND IsRefresh = 0 AND TraficSourceID IN (-1, 6) AND RefererHash = 3594120000172545465 GROUP BY URLHash, EventDate ORDER BY PageViews DESC LIMIT 10 OFFSET 100; +SELECT WindowClientWidth, WindowClientHeight, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND IsRefresh = 0 AND DontCountHits = 0 AND URLHash = 2868770270353813622 GROUP BY WindowClientWidth, WindowClientHeight ORDER BY PageViews DESC LIMIT 10 OFFSET 10000; +SELECT DATE_TRUNC('minute', EventTime) AS M, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-14' AND EventDate <= '2013-07-15' AND IsRefresh = 0 AND DontCountHits = 0 GROUP BY DATE_TRUNC('minute', EventTime) ORDER BY DATE_TRUNC('minute', EventTime) LIMIT 10 OFFSET 1000; diff --git a/clickhouse-trash-schema/query b/clickhouse-trash-schema/query new file mode 100755 index 0000000000..6d5c47fb11 --- /dev/null +++ b/clickhouse-trash-schema/query @@ -0,0 +1,8 @@ +#!/bin/bash +# Reads a SQL query from stdin, runs it via clickhouse-client. +# Stdout: query result (default format). +# Stderr: query runtime in fractional seconds on the last line (from --time). +# Exit non-zero on error. +set -e + +clickhouse-client --time --format=Pretty diff --git a/clickhouse-trash-schema/split.sql b/clickhouse-trash-schema/split.sql new file mode 100644 index 0000000000..1feea7f882 --- /dev/null +++ b/clickhouse-trash-schema/split.sql @@ -0,0 +1,106 @@ +INSERT INTO hits_WatchID SELECT id, WatchID FROM hits_staging; +INSERT INTO hits_JavaEnable SELECT id, JavaEnable FROM hits_staging; +INSERT INTO hits_Title SELECT id, Title FROM hits_staging; +INSERT INTO hits_GoodEvent SELECT id, GoodEvent FROM hits_staging; +INSERT INTO hits_EventTime SELECT id, EventTime FROM hits_staging; +INSERT INTO hits_EventDate SELECT id, EventDate FROM hits_staging; +INSERT INTO hits_CounterID SELECT id, CounterID FROM hits_staging; +INSERT INTO hits_ClientIP SELECT id, ClientIP FROM hits_staging; +INSERT INTO hits_RegionID SELECT id, RegionID FROM hits_staging; +INSERT INTO hits_UserID SELECT id, UserID FROM hits_staging; +INSERT INTO hits_CounterClass SELECT id, CounterClass FROM hits_staging; +INSERT INTO hits_OS SELECT id, OS FROM hits_staging; +INSERT INTO hits_UserAgent SELECT id, UserAgent FROM hits_staging; +INSERT INTO hits_URL SELECT id, URL FROM hits_staging; +INSERT INTO hits_Referer SELECT id, Referer FROM hits_staging; +INSERT INTO hits_IsRefresh SELECT id, IsRefresh FROM hits_staging; +INSERT INTO hits_RefererCategoryID SELECT id, RefererCategoryID FROM hits_staging; +INSERT INTO hits_RefererRegionID SELECT id, RefererRegionID FROM hits_staging; +INSERT INTO hits_URLCategoryID SELECT id, URLCategoryID FROM hits_staging; +INSERT INTO hits_URLRegionID SELECT id, URLRegionID FROM hits_staging; +INSERT INTO hits_ResolutionWidth SELECT id, ResolutionWidth FROM hits_staging; +INSERT INTO hits_ResolutionHeight SELECT id, ResolutionHeight FROM hits_staging; +INSERT INTO hits_ResolutionDepth SELECT id, ResolutionDepth FROM hits_staging; +INSERT INTO hits_FlashMajor SELECT id, FlashMajor FROM hits_staging; +INSERT INTO hits_FlashMinor SELECT id, FlashMinor FROM hits_staging; +INSERT INTO hits_FlashMinor2 SELECT id, FlashMinor2 FROM hits_staging; +INSERT INTO hits_NetMajor SELECT id, NetMajor FROM hits_staging; +INSERT INTO hits_NetMinor SELECT id, NetMinor FROM hits_staging; +INSERT INTO hits_UserAgentMajor SELECT id, UserAgentMajor FROM hits_staging; +INSERT INTO hits_UserAgentMinor SELECT id, UserAgentMinor FROM hits_staging; +INSERT INTO hits_CookieEnable SELECT id, CookieEnable FROM hits_staging; +INSERT INTO hits_JavascriptEnable SELECT id, JavascriptEnable FROM hits_staging; +INSERT INTO hits_IsMobile SELECT id, IsMobile FROM hits_staging; +INSERT INTO hits_MobilePhone SELECT id, MobilePhone FROM hits_staging; +INSERT INTO hits_MobilePhoneModel SELECT id, MobilePhoneModel FROM hits_staging; +INSERT INTO hits_Params SELECT id, Params FROM hits_staging; +INSERT INTO hits_IPNetworkID SELECT id, IPNetworkID FROM hits_staging; +INSERT INTO hits_TraficSourceID SELECT id, TraficSourceID FROM hits_staging; +INSERT INTO hits_SearchEngineID SELECT id, SearchEngineID FROM hits_staging; +INSERT INTO hits_SearchPhrase SELECT id, SearchPhrase FROM hits_staging; +INSERT INTO hits_AdvEngineID SELECT id, AdvEngineID FROM hits_staging; +INSERT INTO hits_IsArtifical SELECT id, IsArtifical FROM hits_staging; +INSERT INTO hits_WindowClientWidth SELECT id, WindowClientWidth FROM hits_staging; +INSERT INTO hits_WindowClientHeight SELECT id, WindowClientHeight FROM hits_staging; +INSERT INTO hits_ClientTimeZone SELECT id, ClientTimeZone FROM hits_staging; +INSERT INTO hits_ClientEventTime SELECT id, ClientEventTime FROM hits_staging; +INSERT INTO hits_SilverlightVersion1 SELECT id, SilverlightVersion1 FROM hits_staging; +INSERT INTO hits_SilverlightVersion2 SELECT id, SilverlightVersion2 FROM hits_staging; +INSERT INTO hits_SilverlightVersion3 SELECT id, SilverlightVersion3 FROM hits_staging; +INSERT INTO hits_SilverlightVersion4 SELECT id, SilverlightVersion4 FROM hits_staging; +INSERT INTO hits_PageCharset SELECT id, PageCharset FROM hits_staging; +INSERT INTO hits_CodeVersion SELECT id, CodeVersion FROM hits_staging; +INSERT INTO hits_IsLink SELECT id, IsLink FROM hits_staging; +INSERT INTO hits_IsDownload SELECT id, IsDownload FROM hits_staging; +INSERT INTO hits_IsNotBounce SELECT id, IsNotBounce FROM hits_staging; +INSERT INTO hits_FUniqID SELECT id, FUniqID FROM hits_staging; +INSERT INTO hits_OriginalURL SELECT id, OriginalURL FROM hits_staging; +INSERT INTO hits_HID SELECT id, HID FROM hits_staging; +INSERT INTO hits_IsOldCounter SELECT id, IsOldCounter FROM hits_staging; +INSERT INTO hits_IsEvent SELECT id, IsEvent FROM hits_staging; +INSERT INTO hits_IsParameter SELECT id, IsParameter FROM hits_staging; +INSERT INTO hits_DontCountHits SELECT id, DontCountHits FROM hits_staging; +INSERT INTO hits_WithHash SELECT id, WithHash FROM hits_staging; +INSERT INTO hits_HitColor SELECT id, HitColor FROM hits_staging; +INSERT INTO hits_LocalEventTime SELECT id, LocalEventTime FROM hits_staging; +INSERT INTO hits_Age SELECT id, Age FROM hits_staging; +INSERT INTO hits_Sex SELECT id, Sex FROM hits_staging; +INSERT INTO hits_Income SELECT id, Income FROM hits_staging; +INSERT INTO hits_Interests SELECT id, Interests FROM hits_staging; +INSERT INTO hits_Robotness SELECT id, Robotness FROM hits_staging; +INSERT INTO hits_RemoteIP SELECT id, RemoteIP FROM hits_staging; +INSERT INTO hits_WindowName SELECT id, WindowName FROM hits_staging; +INSERT INTO hits_OpenerName SELECT id, OpenerName FROM hits_staging; +INSERT INTO hits_HistoryLength SELECT id, HistoryLength FROM hits_staging; +INSERT INTO hits_BrowserLanguage SELECT id, BrowserLanguage FROM hits_staging; +INSERT INTO hits_BrowserCountry SELECT id, BrowserCountry FROM hits_staging; +INSERT INTO hits_SocialNetwork SELECT id, SocialNetwork FROM hits_staging; +INSERT INTO hits_SocialAction SELECT id, SocialAction FROM hits_staging; +INSERT INTO hits_HTTPError SELECT id, HTTPError FROM hits_staging; +INSERT INTO hits_SendTiming SELECT id, SendTiming FROM hits_staging; +INSERT INTO hits_DNSTiming SELECT id, DNSTiming FROM hits_staging; +INSERT INTO hits_ConnectTiming SELECT id, ConnectTiming FROM hits_staging; +INSERT INTO hits_ResponseStartTiming SELECT id, ResponseStartTiming FROM hits_staging; +INSERT INTO hits_ResponseEndTiming SELECT id, ResponseEndTiming FROM hits_staging; +INSERT INTO hits_FetchTiming SELECT id, FetchTiming FROM hits_staging; +INSERT INTO hits_SocialSourceNetworkID SELECT id, SocialSourceNetworkID FROM hits_staging; +INSERT INTO hits_SocialSourcePage SELECT id, SocialSourcePage FROM hits_staging; +INSERT INTO hits_ParamPrice SELECT id, ParamPrice FROM hits_staging; +INSERT INTO hits_ParamOrderID SELECT id, ParamOrderID FROM hits_staging; +INSERT INTO hits_ParamCurrency SELECT id, ParamCurrency FROM hits_staging; +INSERT INTO hits_ParamCurrencyID SELECT id, ParamCurrencyID FROM hits_staging; +INSERT INTO hits_OpenstatServiceName SELECT id, OpenstatServiceName FROM hits_staging; +INSERT INTO hits_OpenstatCampaignID SELECT id, OpenstatCampaignID FROM hits_staging; +INSERT INTO hits_OpenstatAdID SELECT id, OpenstatAdID FROM hits_staging; +INSERT INTO hits_OpenstatSourceID SELECT id, OpenstatSourceID FROM hits_staging; +INSERT INTO hits_UTMSource SELECT id, UTMSource FROM hits_staging; +INSERT INTO hits_UTMMedium SELECT id, UTMMedium FROM hits_staging; +INSERT INTO hits_UTMCampaign SELECT id, UTMCampaign FROM hits_staging; +INSERT INTO hits_UTMContent SELECT id, UTMContent FROM hits_staging; +INSERT INTO hits_UTMTerm SELECT id, UTMTerm FROM hits_staging; +INSERT INTO hits_FromTag SELECT id, FromTag FROM hits_staging; +INSERT INTO hits_HasGCLID SELECT id, HasGCLID FROM hits_staging; +INSERT INTO hits_RefererHash SELECT id, RefererHash FROM hits_staging; +INSERT INTO hits_URLHash SELECT id, URLHash FROM hits_staging; +INSERT INTO hits_CLID SELECT id, CLID FROM hits_staging; +DROP TABLE hits_staging; diff --git a/clickhouse-trash-schema/start b/clickhouse-trash-schema/start new file mode 100755 index 0000000000..a3aa66fe75 --- /dev/null +++ b/clickhouse-trash-schema/start @@ -0,0 +1,4 @@ +#!/bin/bash +set -e + +sudo clickhouse start diff --git a/clickhouse-trash-schema/stop b/clickhouse-trash-schema/stop new file mode 100755 index 0000000000..ea9d529c3e --- /dev/null +++ b/clickhouse-trash-schema/stop @@ -0,0 +1,3 @@ +#!/bin/bash + +sudo clickhouse stop || true diff --git a/clickhouse-trash-schema/template.json b/clickhouse-trash-schema/template.json new file mode 100644 index 0000000000..b446c9c66b --- /dev/null +++ b/clickhouse-trash-schema/template.json @@ -0,0 +1,11 @@ +{ + "system": "ClickHouse (trash schema)", + "proprietary": "no", + "hardware": "cpu", + "tuned": "no", + "tags": [ + "C++", + "column-oriented", + "ClickHouse derivative" + ] +} From 4ed5468056c5cb192c8a71ee3c96e74032ce009d Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Thu, 9 Jul 2026 20:59:13 +0000 Subject: [PATCH 2/3] Minor fixup --- clickhouse-trash-schema/benchmark.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/clickhouse-trash-schema/benchmark.sh b/clickhouse-trash-schema/benchmark.sh index 12a00b626c..d5cf12db78 100755 --- a/clickhouse-trash-schema/benchmark.sh +++ b/clickhouse-trash-schema/benchmark.sh @@ -1,4 +1,3 @@ #!/bin/bash -# Thin shim — actual flow is in lib/benchmark-common.sh. export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-partitioned" exec ../lib/benchmark-common.sh From 269f03f011aa46f991b0566369cc210d95ba9755 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Thu, 9 Jul 2026 21:06:32 +0000 Subject: [PATCH 3/3] Update README --- clickhouse-trash-schema/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/clickhouse-trash-schema/README.md b/clickhouse-trash-schema/README.md index 39df39c7bb..203e3f1a43 100644 --- a/clickhouse-trash-schema/README.md +++ b/clickhouse-trash-schema/README.md @@ -1,5 +1,7 @@ # Trash Schema Benchmark +Inspired by https://github.com/ClickHouse/ClickHouse/issues/109426 + The same as ClickBench, but the `hits` table is split into a hundred tables with two columns each: `id` and one of the original columns. `hits` becomes a view that joins all of them by `id`, so every query has to join a hundred tables. This models data science and AI workloads where there is often a large dataset of objects (images, comments, HTML pages) with iteratively computed properties (classification, embeddings, annotations), each stored in a separate "parallel" table sharing the same primary key. The goal is to find opportunities to optimize queries on top of such a schema.