From 78816e62ddf6bc2c31f6188ece531c92e21b9e9d Mon Sep 17 00:00:00 2001 From: Serge Klochkov <3175289+slvrtrn@users.noreply.github.com> Date: Mon, 2 Sep 2024 18:22:00 +0300 Subject: [PATCH] Add minimal Variant/Dynamic/JSON examples and tests using JSONEachRow format (#305) --- .docker/clickhouse/single_node_tls/Dockerfile | 2 +- docker-compose.cluster.yml | 4 +- docker-compose.yml | 2 +- examples/README.md | 4 + examples/dynamic_variant_json.ts | 68 +++++++++++++++ .../__tests__/integration/data_types.test.ts | 87 +++++++++++++++---- 6 files changed, 146 insertions(+), 21 deletions(-) create mode 100644 examples/dynamic_variant_json.ts diff --git a/.docker/clickhouse/single_node_tls/Dockerfile b/.docker/clickhouse/single_node_tls/Dockerfile index b027edb3..0fd1a6f5 100644 --- a/.docker/clickhouse/single_node_tls/Dockerfile +++ b/.docker/clickhouse/single_node_tls/Dockerfile @@ -1,4 +1,4 @@ -FROM clickhouse/clickhouse-server:24.3-alpine +FROM clickhouse/clickhouse-server:24.8-alpine COPY .docker/clickhouse/single_node_tls/certificates /etc/clickhouse-server/certs RUN chown clickhouse:clickhouse -R /etc/clickhouse-server/certs \ && chmod 600 /etc/clickhouse-server/certs/* \ diff --git a/docker-compose.cluster.yml b/docker-compose.cluster.yml index 6955999b..8c70140d 100644 --- a/docker-compose.cluster.yml +++ b/docker-compose.cluster.yml @@ -2,7 +2,7 @@ version: '2.3' services: clickhouse1: - image: 'clickhouse/clickhouse-server:${CLICKHOUSE_VERSION-24.3-alpine}' + image: 'clickhouse/clickhouse-server:${CLICKHOUSE_VERSION-24.8-alpine}' ulimits: nofile: soft: 262144 @@ -19,7 +19,7 @@ services: - './.docker/clickhouse/users.xml:/etc/clickhouse-server/users.xml' clickhouse2: - image: 'clickhouse/clickhouse-server:${CLICKHOUSE_VERSION-24.3-alpine}' + image: 'clickhouse/clickhouse-server:${CLICKHOUSE_VERSION-24.8-alpine}' ulimits: nofile: soft: 262144 diff --git a/docker-compose.yml b/docker-compose.yml index 83efe85a..15a1bd55 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,7 +1,7 @@ version: '3.8' services: clickhouse: - image: 'clickhouse/clickhouse-server:${CLICKHOUSE_VERSION-24.3-alpine}' + image: 'clickhouse/clickhouse-server:${CLICKHOUSE_VERSION-24.8-alpine}' container_name: 'clickhouse-js-clickhouse-server' ports: - '8123:8123' diff --git a/examples/README.md b/examples/README.md index 64d3d0ee..50baa1f5 100644 --- a/examples/README.md +++ b/examples/README.md @@ -58,6 +58,10 @@ If something is missing, or you found a mistake in one of these examples, please - [select_streaming_json_each_row_for_await.ts](node/select_streaming_json_each_row_for_await.ts) - (Node.js only) similar to [select_streaming_json_each_row.ts](node/select_streaming_json_each_row.ts), but using the `for await` loop syntax. - [select_streaming_text_line_by_line.ts](node/select_streaming_text_line_by_line.ts) - (Node.js only) streaming text formats from ClickHouse and processing it line by line. In this example, CSV format is used. +#### Data types + +- [dynamic_variant_json.ts](./dynamic_variant_json.ts) - using experimental [Dynamic](https://clickhouse.com/docs/en/sql-reference/data-types/dynamic)/[Variant](https://clickhouse.com/docs/en/sql-reference/data-types/variant)/[JSON](https://clickhouse.com/docs/en/sql-reference/data-types/newjson) data types with [JSONEachRow](https://clickhouse.com/docs/en/interfaces/formats#jsoneachrow) format. + #### Special cases - [default_format_setting.ts](default_format_setting.ts) - sending queries using `exec` method without a `FORMAT` clause; the default format will be set from the client settings. diff --git a/examples/dynamic_variant_json.ts b/examples/dynamic_variant_json.ts new file mode 100644 index 00000000..c395e6bc --- /dev/null +++ b/examples/dynamic_variant_json.ts @@ -0,0 +1,68 @@ +import { createClient } from '@clickhouse/client' // or '@clickhouse/client-web' + +void (async () => { + const tableName = `chjs_dynamic_variant_json` + const client = createClient({ + clickhouse_settings: { + // Since ClickHouse 24.1 + allow_experimental_variant_type: 1, + // Since ClickHouse 24.5 + allow_experimental_dynamic_type: 1, + // Since ClickHouse 24.8 + allow_experimental_json_type: 1, + }, + }) + await client.command({ + query: ` + CREATE OR REPLACE TABLE ${tableName} + ( + id UInt64, + var Variant(Int64, String), + dynamic Dynamic, + json JSON + ) + ENGINE MergeTree + ORDER BY id + `, + }) + // Sample representation in JSONEachRow format + const values = [ + { + id: 1, + var: 42, + dynamic: 'foo', + json: { + foo: 'x', + }, + }, + { + id: 2, + var: 'str', + // defaults to Int64; will be represented as a string in JSON* family formats + // this behavior can be changed with `output_format_json_quote_64bit_integers` setting (default is 1). + // see https://clickhouse.com/docs/en/operations/settings/formats#output_format_json_quote_64bit_integers + dynamic: 144, + json: { + bar: 10, + }, + }, + ] + await client.insert({ + table: tableName, + format: 'JSONEachRow', + values, + }) + const rs = await client.query({ + query: ` + SELECT *, + variantType(var), + dynamicType(dynamic), + dynamicType(json.foo), + dynamicType(json.bar) + FROM ${tableName} + `, + format: 'JSONEachRow', + }) + console.log(await rs.json()) + await client.close() +})() diff --git a/packages/client-common/__tests__/integration/data_types.test.ts b/packages/client-common/__tests__/integration/data_types.test.ts index 0c6f0c55..ee6a6b26 100644 --- a/packages/client-common/__tests__/integration/data_types.test.ts +++ b/packages/client-common/__tests__/integration/data_types.test.ts @@ -129,7 +129,9 @@ describe('data types', () => { }) const result = await client .query({ - query: `SELECT * FROM ${table} ORDER BY id ASC`, + query: `SELECT * + FROM ${table} + ORDER BY id ASC`, format: 'TabSeparated', }) .then((r) => r.text()) @@ -238,7 +240,9 @@ describe('data types', () => { await insertData(table, values) const result = await client .query({ - query: `SELECT CAST(e1, 'Int8') AS e1, CAST(e2, 'Int8') AS e2 FROM ${table} ORDER BY id ASC`, + query: `SELECT CAST(e1, 'Int8') AS e1, CAST(e2, 'Int8') AS e2 + FROM ${table} + ORDER BY id ASC`, format: 'JSONEachRow', }) .then((r) => r.json()) @@ -426,7 +430,8 @@ describe('data types', () => { expect( await client .query({ - query: `SELECT sum(distance) FROM ${table}`, + query: `SELECT sum(distance) + FROM ${table}`, format: 'TabSeparated', }) .then((r) => r.text()), @@ -434,7 +439,8 @@ describe('data types', () => { expect( await client .query({ - query: `SELECT max(distance) FROM ${table}`, + query: `SELECT max(distance) + FROM ${table}`, format: 'TabSeparated', }) .then((r) => r.text()), @@ -442,7 +448,8 @@ describe('data types', () => { expect( await client .query({ - query: `SELECT uniqExact(distance) FROM ${table}`, + query: `SELECT uniqExact(distance) + FROM ${table}`, format: 'TabSeparated', }) .then((r) => r.text()), @@ -504,9 +511,10 @@ describe('data types', () => { await insertAndAssert(table, values) }) - // JSON cannot be used on a "modern" Cloud instance - whenOnEnv(TestEnv.LocalSingleNode, TestEnv.LocalCluster, TestEnv.Cloud).it( - 'should work with JSON', + // New experimental JSON type + // https://clickhouse.com/docs/en/sql-reference/data-types/newjson + whenOnEnv(TestEnv.LocalSingleNode, TestEnv.LocalCluster).it( + 'should work with (new) JSON', async () => { const values = [ { @@ -516,8 +524,40 @@ describe('data types', () => { o: { a: 2, b: { c: 3, d: [4, 5, 6] } }, }, ] - const table = await createTableWithFields(client, `o Object('json')`, { - allow_experimental_object_type: 1, + const table = await createTableWithFields(client, `o JSON`, { + allow_experimental_json_type: 1, + }) + await insertAndAssert(table, values, { + output_format_json_quote_64bit_integers: 0, + }) + }, + ) + + // New experimental Variant type + // https://clickhouse.com/docs/en/sql-reference/data-types/variant + whenOnEnv(TestEnv.LocalSingleNode, TestEnv.LocalCluster).it( + 'should work with Variant', + async () => { + const values = [{ var: 'foo' }, { var: 42 }] + const table = await createTableWithFields( + client, + `var Variant(String, Int32)`, + { + allow_experimental_variant_type: 1, + }, + ) + await insertAndAssert(table, values) + }, + ) + + // New experimental Dynamic type + // https://clickhouse.com/docs/en/sql-reference/data-types/dynamic + whenOnEnv(TestEnv.LocalSingleNode, TestEnv.LocalCluster).it( + 'should work with Dynamic', + async () => { + const values = [{ dyn: 'foo' }, { dyn: { bar: 'qux' } }] + const table = await createTableWithFields(client, `dyn Dynamic`, { + allow_experimental_dynamic_type: 1, }) await insertAndAssert(table, values) }, @@ -655,7 +695,9 @@ describe('data types', () => { }) const result = await client .query({ - query: `SELECT n.id, n.name, n.createdAt, n.roles FROM ${table} ORDER BY id ASC`, + query: `SELECT n.id, n.name, n.createdAt, n.roles + FROM ${table} + ORDER BY id ASC`, format: 'JSONEachRow', }) .then((r) => r.json()) @@ -683,25 +725,36 @@ describe('data types', () => { ) { const values = data.map((v, i) => ({ ...v, id: i + 1 })) await client.insert({ + format: 'JSONEachRow', table, values, - format: 'JSONEachRow', clickhouse_settings, }) } - async function assertData(table: string, data: T[]) { + async function assertData( + table: string, + data: T[], + clickhouse_settings: ClickHouseSettings = {}, + ) { const result = await client .query({ - query: `SELECT * EXCEPT (id) FROM ${table} ORDER BY id ASC`, + query: `SELECT * EXCEPT (id) + FROM ${table} + ORDER BY id ASC`, format: 'JSONEachRow', + clickhouse_settings, }) .then((r) => r.json()) expect(result).toEqual(data) } - async function insertAndAssert(table: string, data: T[]) { - await insertData(table, data) - await assertData(table, data) + async function insertAndAssert( + table: string, + data: T[], + clickhouse_settings: ClickHouseSettings = {}, + ) { + await insertData(table, data, clickhouse_settings) + await assertData(table, data, clickhouse_settings) } })