From e41d3409aec6a6a9ccaadde363e68cd0ca1ebf40 Mon Sep 17 00:00:00 2001 From: slvrtrn Date: Fri, 1 Dec 2023 18:11:52 +0100 Subject: [PATCH 1/3] Drain insert response stream in Web version Add async_insert examples --- examples/insert_cloud.ts | 58 +++++++++++++++++++ .../__tests__/integration/insert.test.ts | 14 +++++ .../src/connection/web_connection.ts | 5 +- 3 files changed, 76 insertions(+), 1 deletion(-) create mode 100644 examples/insert_cloud.ts diff --git a/examples/insert_cloud.ts b/examples/insert_cloud.ts new file mode 100644 index 00000000..e25d462a --- /dev/null +++ b/examples/insert_cloud.ts @@ -0,0 +1,58 @@ +import { createClient } from '@clickhouse/client' // or '@clickhouse/client-web' + +void (async () => { + const client = createClient({ + host: getFromEnv('CLICKHOUSE_HOST'), + password: getFromEnv('CLICKHOUSE_PASSWORD'), + // See https://clickhouse.com/docs/en/optimize/asynchronous-inserts + clickhouse_settings: { + async_insert: 1, + wait_for_async_insert: 1, + }, + }) + + // Create the table if necessary + const table = 'async_insert_example' + await client.command({ + query: ` + CREATE TABLE IF NOT EXISTS ${table} + (id UInt32, data String) + ENGINE MergeTree + ORDER BY id + `, + // Tell the server to send the response only when the DDL is fully executed + clickhouse_settings: { + wait_end_of_query: 1, + }, + }) + + // Generate some random data for the sake of example... + const batch = [...new Array(1_000).keys()].map(() => ({ + id: Math.floor(Math.random() * 100_000) + 1, + data: Math.random().toString(36).slice(2), + })) + + await client.insert({ + table, + format: 'JSONEachRow', // or other, depends on your data + values: batch, + }) + + const res = await client + .query({ + query: `SELECT count(*) FROM ${table}`, + format: 'JSONEachRow', + }) + .then((r) => r.json()) + + console.info(res) +})() + +// Node.js only +function getFromEnv(key: string) { + if (process.env[key]) { + return process.env[key] + } + console.error(`${key} environment variable should be set`) + process.exit(1) +} diff --git a/packages/client-common/__tests__/integration/insert.test.ts b/packages/client-common/__tests__/integration/insert.test.ts index 05ad1226..876d05ae 100644 --- a/packages/client-common/__tests__/integration/insert.test.ts +++ b/packages/client-common/__tests__/integration/insert.test.ts @@ -137,4 +137,18 @@ describe('insert', () => { }) ) }) + + it('should work with async inserts', async () => { + await client.insert({ + table: tableName, + values: jsonValues, + format: 'JSONEachRow', + // See https://clickhouse.com/docs/en/optimize/asynchronous-inserts + clickhouse_settings: { + async_insert: 1, + wait_for_async_insert: 1, + }, + }) + await assertJsonValues(client, tableName) + }) }) diff --git a/packages/client-web/src/connection/web_connection.ts b/packages/client-web/src/connection/web_connection.ts index 1c88bb78..847ec65b 100644 --- a/packages/client-web/src/connection/web_connection.ts +++ b/packages/client-web/src/connection/web_connection.ts @@ -92,11 +92,14 @@ export class WebConnection implements Connection { session_id: params.session_id, query_id, }) - await this.request({ + const res = await this.request({ values: params.values, params, searchParams, }) + if (res.body !== null) { + await res.text() // drain the response (it's empty anyway) + } return { query_id, } From 245f0d3d7ca17cc10f79555e3717cf5c7aae5780 Mon Sep 17 00:00:00 2001 From: slvrtrn Date: Fri, 1 Dec 2023 18:14:50 +0100 Subject: [PATCH 2/3] Add CHANGELOG entry --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 90604404..82950bf3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +## 0.2.7 (Web only) + +### Bug fixes + +- Drain insert response stream in Web version - required to properly work with `async_insert`, especially in the Cloudflare Workers context. + ## 0.2.6 (Common, Node.js) ### New features From 2d959e1edfe5751a2668c60e7ee47730f87e5235 Mon Sep 17 00:00:00 2001 From: slvrtrn Date: Fri, 1 Dec 2023 18:16:26 +0100 Subject: [PATCH 3/3] Bump version --- packages/client-common/src/version.ts | 2 +- packages/client-node/src/version.ts | 2 +- packages/client-web/src/version.ts | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/client-common/src/version.ts b/packages/client-common/src/version.ts index 42fb5c8c..08ea5251 100644 --- a/packages/client-common/src/version.ts +++ b/packages/client-common/src/version.ts @@ -1 +1 @@ -export default '0.2.6' +export default '0.2.7' diff --git a/packages/client-node/src/version.ts b/packages/client-node/src/version.ts index 42fb5c8c..08ea5251 100644 --- a/packages/client-node/src/version.ts +++ b/packages/client-node/src/version.ts @@ -1 +1 @@ -export default '0.2.6' +export default '0.2.7' diff --git a/packages/client-web/src/version.ts b/packages/client-web/src/version.ts index 42fb5c8c..08ea5251 100644 --- a/packages/client-web/src/version.ts +++ b/packages/client-web/src/version.ts @@ -1 +1 @@ -export default '0.2.6' +export default '0.2.7'