From aafc649261acda914245cebcd3ac7ee4d7038b10 Mon Sep 17 00:00:00 2001 From: Serge Klochkov <3175289+slvrtrn@users.noreply.github.com> Date: Fri, 12 Jan 2024 11:55:53 +0100 Subject: [PATCH] Allow to specify or exclude insert columns (#218) --- .eslintignore | 1 + CHANGELOG.md | 39 ++ examples/insert_ephemeral_columns.ts | 60 +++ examples/insert_exclude_columns.ts | 65 +++ .../insert_specific_columns.test.ts | 387 ++++++++++++++++++ packages/client-common/src/client.ts | 55 ++- packages/client-common/src/version.ts | 2 +- packages/client-node/src/version.ts | 2 +- packages/client-web/src/version.ts | 2 +- 9 files changed, 606 insertions(+), 7 deletions(-) create mode 100644 examples/insert_ephemeral_columns.ts create mode 100644 examples/insert_exclude_columns.ts create mode 100644 packages/client-common/__tests__/integration/insert_specific_columns.test.ts diff --git a/.eslintignore b/.eslintignore index bd862fdb..4781ec11 100644 --- a/.eslintignore +++ b/.eslintignore @@ -1,3 +1,4 @@ +out dist node_modules webpack diff --git a/CHANGELOG.md b/CHANGELOG.md index c66e4159..70383eeb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,42 @@ +## 0.2.8 (Common, Node.js, Web) + +### New features + +- (Web only) Allow to modify Keep-Alive setting (previously always disabled). + Keep-Alive setting **is now enabled by default** for the Web version. + +```ts +import { createClient } from '@clickhouse/client-web' +const client = createClient({ keep_alive: { enabled: true } }) +``` + +- (Node.js & Web) It is now possible to either specify a list of columns to insert the data into or a list of excluded columns: + +```ts +// Generated query: INSERT INTO mytable (message) FORMAT JSONEachRow +await client.insert({ + table: 'mytable', + format: 'JSONEachRow', + values: [{ message: 'foo' }], + columns: ['message'], +}) + +// Generated query: INSERT INTO mytable (* EXCEPT (message)) FORMAT JSONEachRow +await client.insert({ + table: 'mytable', + format: 'JSONEachRow', + values: [{ id: 42 }], + columns: { except: ['message'] }, +}) +``` + +See also the new examples: + +- [Including specific columns or excluding certain ones instead](./examples/insert_exclude_columns.ts) +- [Leveraging this feature](./examples/insert_ephemeral_columns.ts) when working with + [ephemeral columns](https://clickhouse.com/docs/en/sql-reference/statements/create/table#ephemeral) + ([#217](https://github.com/ClickHouse/clickhouse-js/issues/217)) + ## 0.2.7 (Common, Node.js, Web) ### New features diff --git a/examples/insert_ephemeral_columns.ts b/examples/insert_ephemeral_columns.ts new file mode 100644 index 00000000..c621be10 --- /dev/null +++ b/examples/insert_ephemeral_columns.ts @@ -0,0 +1,60 @@ +import { createClient } from '@clickhouse/client' // or '@clickhouse/client-web' + +// Ephemeral columns documentation: https://clickhouse.com/docs/en/sql-reference/statements/create/table#ephemeral +// This example is inspired by https://github.com/ClickHouse/clickhouse-js/issues/217 +void (async () => { + const tableName = 'insert_ephemeral_columns' + const client = createClient({ + clickhouse_settings: { + allow_experimental_object_type: 1, // allows JSON type usage + }, + }) + + await client.command({ + query: ` + CREATE OR REPLACE TABLE ${tableName} + ( + event_type LowCardinality(String) DEFAULT JSONExtractString(message_raw, 'type'), + repo_name LowCardinality(String) DEFAULT JSONExtractString(message_raw, 'repo', 'name'), + message JSON DEFAULT message_raw, + message_raw String EPHEMERAL + ) + ENGINE MergeTree() + ORDER BY (event_type, repo_name) + `, + }) + + await client.insert({ + table: tableName, + values: [ + { + message_raw: { + type: 'MyEventType', + repo: { + name: 'foo', + }, + }, + }, + { + message_raw: { + type: 'SomeOtherType', + repo: { + name: 'bar', + }, + }, + }, + ], + format: 'JSONEachRow', + // The name of the ephemeral column has to be specified here + // to trigger the default values logic for the rest of the columns + columns: ['message_raw'], + }) + + const rows = await client.query({ + query: `SELECT * FROM ${tableName}`, + format: 'JSONCompactEachRowWithNames', + }) + + console.info(await rows.text()) + await client.close() +})() diff --git a/examples/insert_exclude_columns.ts b/examples/insert_exclude_columns.ts new file mode 100644 index 00000000..5a1ff865 --- /dev/null +++ b/examples/insert_exclude_columns.ts @@ -0,0 +1,65 @@ +import { createClient } from '@clickhouse/client' // or '@clickhouse/client-web' + +void (async () => { + const tableName = 'insert_exclude_columns' + const client = createClient() + + await client.command({ + query: ` + CREATE OR REPLACE TABLE ${tableName} + (id UInt32, message String) + ENGINE MergeTree() + ORDER BY (id) + `, + }) + + /** + * Explicitly specifying a list of columns to insert the data into + */ + await client.insert({ + table: tableName, + values: [{ message: 'foo' }], + format: 'JSONEachRow', + // `id` column value for this row will be zero + columns: ['message'], + }) + + await client.insert({ + table: tableName, + values: [{ id: 42 }], + format: 'JSONEachRow', + // `message` column value for this row will be an empty string + columns: ['id'], + }) + + /** + * Alternatively, it is possible to exclude certain columns instead + */ + await client.insert({ + table: tableName, + values: [{ message: 'bar' }], + format: 'JSONEachRow', + // `id` column value for this row will be zero + columns: { + except: ['id'], + }, + }) + + await client.insert({ + table: tableName, + values: [{ id: 144 }], + format: 'JSONEachRow', + // `message` column value for this row will be an empty string + columns: { + except: ['message'], + }, + }) + + const rows = await client.query({ + query: `SELECT * FROM ${tableName} ORDER BY id, message DESC`, + format: 'JSONEachRow', + }) + + console.info(await rows.json()) + await client.close() +})() diff --git a/packages/client-common/__tests__/integration/insert_specific_columns.test.ts b/packages/client-common/__tests__/integration/insert_specific_columns.test.ts new file mode 100644 index 00000000..0dd1f657 --- /dev/null +++ b/packages/client-common/__tests__/integration/insert_specific_columns.test.ts @@ -0,0 +1,387 @@ +import { type ClickHouseClient } from '@clickhouse/client-common' +import { createTableWithFields } from '@test/fixtures/table_with_fields' +import { createTestClient } from '../utils' + +describe('Insert with specific columns', () => { + let client: ClickHouseClient + let table: string + + beforeEach(async () => { + client = createTestClient({ + clickhouse_settings: { + allow_experimental_object_type: 1, + }, + }) + }) + afterEach(async () => { + await client.close() + }) + + // Inspired by https://github.com/ClickHouse/clickhouse-js/issues/217 + // Specifying a particular insert column is especially useful with ephemeral types + describe('list of columns', () => { + beforeEach(async () => { + // `message_raw` will be used as a source for default values here + table = await createTableWithFields( + client, + ` + event_type LowCardinality(String) DEFAULT JSONExtractString(message_raw, 'type'), + repo_name LowCardinality(String) DEFAULT JSONExtractString(message_raw, 'repo', 'name'), + message JSON DEFAULT message_raw, + message_raw String EPHEMERAL + ` // `id UInt32` will be added as well + ) + }) + + it('should work with a single column', async () => { + await client.insert({ + table, + values: [ + { + message_raw: { + type: 'MyEventType', + repo: { + name: 'foo', + }, + }, + }, + { + message_raw: { + type: 'SomeOtherType', + repo: { + name: 'bar', + }, + }, + }, + ], + format: 'JSONEachRow', + columns: ['message_raw'], + }) + + const result = await client + .query({ + query: `SELECT * FROM ${table} ORDER BY repo_name DESC`, + format: 'JSONEachRow', + }) + .then((r) => r.json()) + + expect(result).toEqual([ + { + id: 0, // defaults for everything are taken from `message_raw` + event_type: 'MyEventType', + repo_name: 'foo', + message: { + type: 'MyEventType', + repo: { + name: 'foo', + }, + }, + }, + { + id: 0, + event_type: 'SomeOtherType', + repo_name: 'bar', + message: { + type: 'SomeOtherType', + repo: { + name: 'bar', + }, + }, + }, + ]) + }) + + it('should work with multiple columns', async () => { + await client.insert({ + table, + values: [ + { + id: 42, + message_raw: { + type: 'MyEventType', + repo: { + name: 'foo', + }, + }, + }, + { + id: 144, + message_raw: { + type: 'SomeOtherType', + repo: { + name: 'bar', + }, + }, + }, + ], + format: 'JSONEachRow', + columns: ['id', 'message_raw'], + }) + + const result = await client + .query({ + query: `SELECT * FROM ${table} ORDER BY id`, + format: 'JSONEachRow', + }) + .then((r) => r.json()) + + expect(result).toEqual([ + { + id: 42, // all defaults except `id` are taken from `message_raw` + event_type: 'MyEventType', + repo_name: 'foo', + message: { + type: 'MyEventType', + repo: { + name: 'foo', + }, + }, + }, + { + id: 144, + event_type: 'SomeOtherType', + repo_name: 'bar', + message: { + type: 'SomeOtherType', + repo: { + name: 'bar', + }, + }, + }, + ]) + }) + + // In this case, `message_raw` will be ignored, as expected + it('should work when all the columns are specified', async () => { + const value1 = { + id: 42, + event_type: 'MyEventType', + repo_name: 'foo', + message: { foo: 'bar' }, + } + const value2 = { + id: 42, + event_type: 'MyEventType', + repo_name: 'foo', + message: { foo: 'bar' }, + } + await client.insert({ + table, + values: [ + { ...value1, message_raw: '{ "i": 42 }' }, + { ...value2, message_raw: '{ "j": 255 }' }, + ], + format: 'JSONEachRow', + columns: ['id', 'event_type', 'repo_name', 'message', 'message_raw'], + }) + + const result = await client + .query({ + query: `SELECT * FROM ${table} ORDER BY id`, + format: 'JSONEachRow', + }) + .then((r) => r.json()) + + expect(result).toEqual([value1, value2]) + }) + + it('should fail when an unknown column is specified', async () => { + await expectAsync( + client.insert({ + table, + values: [ + { + message_raw: { + type: 'MyEventType', + repo: { + name: 'foo', + }, + }, + }, + { + message_raw: { + type: 'SomeOtherType', + repo: { + name: 'bar', + }, + }, + }, + ], + format: 'JSONEachRow', + columns: ['foobar', 'message_raw'], + }) + ).toBeRejectedWith( + jasmine.objectContaining({ + message: jasmine.stringContaining('No such column foobar'), + }) + ) + }) + }) + + // This is impossible to test with ephemeral columns (need to send at least `message_raw`), + // so for this corner case the tests are simplified. Essentially, just a fallback to the "normal" insert behavior. + describe('list of columns corner cases', () => { + beforeEach(async () => { + table = await createTableWithFields( + client, + `s String, b Boolean` // `id UInt32` will be added as well + ) + }) + + it('should work when the list is empty', async () => { + const values = [ + { id: 144, s: 'foo', b: true }, + { id: 255, s: 'bar', b: false }, + ] + + await client.insert({ + table, + values, + format: 'JSONEachRow', + // Prohibited by the type system, but the client can be used from the JS + columns: [] as unknown as [string, ...string[]], + }) + + const result = await client + .query({ + query: `SELECT * FROM ${table} ORDER BY id`, + format: 'JSONEachRow', + }) + .then((r) => r.json()) + + expect(result).toEqual(values) + }) + }) + + // TODO: For some reason, ephemeral columns don't work well with EXCEPT (even from the CLI) - to be investigated. + // Thus, the tests for this case are simplified. + describe('list of excluded columns', () => { + beforeEach(async () => { + table = await createTableWithFields( + client, + `s String, b Boolean` // `id UInt32` will be added as well + ) + }) + + it('should work with a single excluded column', async () => { + await client.insert({ + table, + values: [ + { id: 144, b: true }, + { id: 255, b: false }, + ], + format: 'JSONEachRow', + columns: { + except: ['s'], + }, + }) + + const result = await client + .query({ + query: `SELECT * FROM ${table} ORDER BY id`, + format: 'JSONEachRow', + }) + .then((r) => r.json()) + + expect(result).toEqual([ + { id: 144, s: '', b: true }, + { id: 255, s: '', b: false }, + ]) + }) + + it('should work with multiple excluded columns', async () => { + await client.insert({ + table, + values: [{ s: 'foo' }, { s: 'bar' }], + format: 'JSONEachRow', + columns: { + except: ['id', 'b'], + }, + }) + + const result = await client + .query({ + query: `SELECT * FROM ${table} ORDER BY s DESC`, + format: 'JSONEachRow', + }) + .then((r) => r.json()) + + expect(result).toEqual([ + { id: 0, s: 'foo', b: false }, + { id: 0, s: 'bar', b: false }, + ]) + }) + + it('should work when the list is empty', async () => { + const values = [ + { id: 144, s: 'foo', b: true }, + { id: 255, s: 'bar', b: false }, + ] + await client.insert({ + table, + values, + format: 'JSONEachRow', + columns: { + // Prohibited by the type system, but the client can be used from the JS + except: [] as unknown as [string, ...string[]], + }, + }) + + const result = await client + .query({ + query: `SELECT * FROM ${table}`, + format: 'JSONEachRow', + }) + .then((r) => r.json()) + + expect(result).toEqual(values) + }) + + it('should work when all the columns are excluded', async () => { + await client.insert({ + table, + values: [{}, {}], + format: 'JSONEachRow', + columns: { + except: ['id', 's', 'b'], + }, + }) + + const result = await client + .query({ + query: `SELECT * FROM ${table}`, + format: 'JSONEachRow', + }) + .then((r) => r.json()) + + // While ClickHouse allows that via HTTP, the data won't be actually inserted + expect(result).toEqual([]) + }) + + // Surprisingly, `EXCEPT some_unknown_column` does not fail, even from the CLI + it('should still work when an unknown column is specified', async () => { + const values = [ + { id: 144, s: 'foo', b: true }, + { id: 255, s: 'bar', b: false }, + ] + + await client.insert({ + table, + values, + format: 'JSONEachRow', + columns: { + except: ['foobar'], + }, + }) + + const result = await client + .query({ + query: `SELECT * FROM ${table}`, + format: 'JSONEachRow', + }) + .then((r) => r.json()) + + expect(result).toEqual(values) + }) + }) +}) diff --git a/packages/client-common/src/client.ts b/packages/client-common/src/client.ts index 41b55c82..7936f105 100644 --- a/packages/client-common/src/client.ts +++ b/packages/client-common/src/client.ts @@ -3,10 +3,10 @@ import type { ClickHouseSettings, Connection, ConnectionParams, + ConnExecResult, ConnInsertResult, Logger, WithClickHouseSummary, - ConnExecResult, } from '@clickhouse/client-common' import { type DataFormat, @@ -137,14 +137,34 @@ export type InsertValues = | InputJSON | InputJSONObjectEachRow +type NonEmptyArray = [T, ...T[]] + +/** {@link except} field contains a non-empty list of columns to exclude when generating `(* EXCEPT (...))` clause */ +export interface InsertColumnsExcept { + except: NonEmptyArray +} + export interface InsertParams extends BaseQueryParams { /** Name of a table to insert into. */ table: string /** A dataset to insert. */ values: InsertValues - /** Format of the dataset to insert. */ + /** Format of the dataset to insert. Default: `JSONCompactEachRow` */ format?: DataFormat + /** + * Allows to specify which columns the data will be inserted into. + * Accepts either an array of strings (column names) or an object of {@link InsertColumnsExcept} type. + * Examples of generated queries: + * + * - An array such as `['a', 'b']` will generate: `INSERT INTO table (a, b) FORMAT DataFormat` + * - An object such as `{ except: ['a', 'b'] }` will generate: `INSERT INTO table (* EXCEPT (a, b)) FORMAT DataFormat` + * + * By default, the data is inserted into all columns of the {@link InsertParams.table}, + * and the generated statement will be: `INSERT INTO table FORMAT DataFormat`. + * + * See also: https://clickhouse.com/docs/en/sql-reference/statements/insert-into */ + columns?: NonEmptyArray | InsertColumnsExcept } export class ClickHouseClient { @@ -229,10 +249,9 @@ export class ClickHouseClient { */ async insert(params: InsertParams): Promise { const format = params.format || 'JSONCompactEachRow' - this.valuesEncoder.validateInsertValues(params.values, format) - const query = `INSERT INTO ${params.table.trim()} FORMAT ${format}` + const query = getInsertQuery(params, format) return await this.connection.insert({ query, values: this.valuesEncoder.encodeValues(params.values, format), @@ -318,3 +337,31 @@ function getConnectionParams( ), } } + +function isInsertColumnsExcept(obj: unknown): obj is InsertColumnsExcept { + return ( + obj !== undefined && + obj !== null && + typeof obj === 'object' && + // Avoiding ESLint no-prototype-builtins error + Object.prototype.hasOwnProperty.call(obj, 'except') + ) +} + +function getInsertQuery( + params: InsertParams, + format: DataFormat +): string { + let columnsPart = '' + if (params.columns !== undefined) { + if (Array.isArray(params.columns) && params.columns.length > 0) { + columnsPart = ` (${params.columns.join(', ')})` + } else if ( + isInsertColumnsExcept(params.columns) && + params.columns.except.length > 0 + ) { + columnsPart = ` (* EXCEPT (${params.columns.except.join(', ')}))` + } + } + return `INSERT INTO ${params.table.trim()}${columnsPart} FORMAT ${format}` +} diff --git a/packages/client-common/src/version.ts b/packages/client-common/src/version.ts index 08ea5251..746c2b11 100644 --- a/packages/client-common/src/version.ts +++ b/packages/client-common/src/version.ts @@ -1 +1 @@ -export default '0.2.7' +export default '0.2.8' diff --git a/packages/client-node/src/version.ts b/packages/client-node/src/version.ts index 08ea5251..746c2b11 100644 --- a/packages/client-node/src/version.ts +++ b/packages/client-node/src/version.ts @@ -1 +1 @@ -export default '0.2.7' +export default '0.2.8' diff --git a/packages/client-web/src/version.ts b/packages/client-web/src/version.ts index 08ea5251..746c2b11 100644 --- a/packages/client-web/src/version.ts +++ b/packages/client-web/src/version.ts @@ -1 +1 @@ -export default '0.2.7' +export default '0.2.8'