From 8e14c3ccdde44c2b7dbccb2bb0cc54bb41f41d6d Mon Sep 17 00:00:00 2001 From: KATT Date: Fri, 13 Oct 2023 11:33:09 +0200 Subject: [PATCH 01/17] fix: `tsonAsyncIterator` -> `tsonAsyncIterable` --- examples/async/src/shared.ts | 4 ++-- src/async/deserializeAsync.test.ts | 20 ++++++++++---------- src/async/handlers/tsonAsyncIterable.ts | 10 +++++----- src/async/serializeAsync.test.ts | 6 +++--- src/extend/openai.test.ts | 4 ++-- 5 files changed, 22 insertions(+), 22 deletions(-) diff --git a/examples/async/src/shared.ts b/examples/async/src/shared.ts index a84353cf..cc973f9d 100644 --- a/examples/async/src/shared.ts +++ b/examples/async/src/shared.ts @@ -1,6 +1,6 @@ import { TsonAsyncOptions, - tsonAsyncIterator, + tsonAsyncIterable, tsonBigint, tsonPromise, } from "tupleson"; @@ -14,7 +14,7 @@ export const tsonOptions: TsonAsyncOptions = { // Allow serialization of promises tsonPromise, // Allow serialization of async iterators - tsonAsyncIterator, + tsonAsyncIterable, // Allow serialization of bigints tsonBigint, ], diff --git a/src/async/deserializeAsync.test.ts b/src/async/deserializeAsync.test.ts index eca5d930..166a78ea 100644 --- a/src/async/deserializeAsync.test.ts +++ b/src/async/deserializeAsync.test.ts @@ -6,7 +6,7 @@ import { TsonType, createTsonAsync, createTsonParseAsync, - tsonAsyncIterator, + tsonAsyncIterable, tsonBigint, tsonPromise, } from "../index.js"; @@ -24,7 +24,7 @@ import { mapIterable, readableStreamToAsyncIterable } from "./iterableUtils.js"; test("deserialize variable chunk length", async () => { const tson = createTsonAsync({ nonce: () => "__tson", - types: [tsonAsyncIterator, tsonPromise, tsonBigint], + types: [tsonAsyncIterable, tsonPromise, tsonBigint], }); { const iterable = (async function* () { @@ -62,7 +62,7 @@ test("deserialize variable chunk length", async () => { test("deserialize async iterable", async () => { const tson = createTsonAsync({ nonce: () => "__tson", - types: [tsonAsyncIterator, tsonPromise, tsonBigint], + types: [tsonAsyncIterable, tsonPromise, tsonBigint], }); { @@ -95,7 +95,7 @@ test("deserialize async iterable", async () => { test("stringify async iterable + promise", async () => { const tson = createTsonAsync({ nonce: () => "__tson", - types: [tsonAsyncIterator, tsonPromise, tsonBigint], + types: [tsonAsyncIterable, tsonPromise, tsonBigint], }); const parseOptions = { @@ -158,7 +158,7 @@ test("e2e: stringify async iterable and promise over the network", async () => { // ------------- server ------------------- const opts: TsonAsyncOptions = { - types: [tsonPromise, tsonAsyncIterator, tsonBigint], + types: [tsonPromise, tsonAsyncIterable, tsonBigint], }; const server = await createTestServer({ @@ -259,7 +259,7 @@ test("iterator error", async () => { // ------------- server ------------------- const opts: TsonAsyncOptions = { - types: [tsonPromise, tsonAsyncIterator, tsonCustomError], + types: [tsonPromise, tsonAsyncIterable, tsonCustomError], }; const server = await createTestServer({ @@ -351,7 +351,7 @@ test("values missing when stream ends", async () => { } const opts = { - types: [tsonPromise, tsonAsyncIterator], + types: [tsonPromise, tsonAsyncIterable], } satisfies TsonAsyncOptions; const parseOptions = { @@ -477,7 +477,7 @@ test("1 iterator completed but another never finishes", async () => { } const opts = { - types: [tsonPromise, tsonAsyncIterator], + types: [tsonPromise, tsonAsyncIterable], } satisfies TsonAsyncOptions; const parseOptions = { @@ -566,7 +566,7 @@ test("e2e: simulated server crash", async () => { // ------------- server ------------------- const opts = { - types: [tsonPromise, tsonAsyncIterator], + types: [tsonPromise, tsonAsyncIterable], } satisfies TsonAsyncOptions; const parseOptions = { @@ -666,7 +666,7 @@ test("e2e: client aborted request", async () => { type MockObj = ReturnType; const opts = { nonce: () => "__tson", - types: [tsonPromise, tsonAsyncIterator], + types: [tsonPromise, tsonAsyncIterable], } satisfies TsonAsyncOptions; const parseOptions = { diff --git a/src/async/handlers/tsonAsyncIterable.ts b/src/async/handlers/tsonAsyncIterable.ts index 0117d04b..8a99cfc1 100644 --- a/src/async/handlers/tsonAsyncIterable.ts +++ b/src/async/handlers/tsonAsyncIterable.ts @@ -7,11 +7,11 @@ import { TsonAsyncType } from "../asyncTypes.js"; const ITERATOR_VALUE = 0; const ITERATOR_ERROR = 1; const ITERATOR_DONE = 2; -type SerializedIteratorResult = +type SerializedIterableResult = | [typeof ITERATOR_DONE] | [typeof ITERATOR_ERROR, unknown] | [typeof ITERATOR_VALUE, unknown]; -function isAsyncIterator(value: unknown): value is AsyncIterable { +function isAsyncIterable(value: unknown): value is AsyncIterable { return ( !!value && typeof value === "object" && @@ -19,9 +19,9 @@ function isAsyncIterator(value: unknown): value is AsyncIterable { ); } -export const tsonAsyncIterator: TsonAsyncType< +export const tsonAsyncIterable: TsonAsyncType< AsyncIterable, - SerializedIteratorResult + SerializedIterableResult > = { async: true, deserialize: (opts) => { @@ -65,5 +65,5 @@ export const tsonAsyncIterator: TsonAsyncType< yield [ITERATOR_ERROR, err]; } }, - test: isAsyncIterator, + test: isAsyncIterable, }; diff --git a/src/async/serializeAsync.test.ts b/src/async/serializeAsync.test.ts index d47a50c1..fff5c652 100644 --- a/src/async/serializeAsync.test.ts +++ b/src/async/serializeAsync.test.ts @@ -1,6 +1,6 @@ import { expect, test } from "vitest"; -import { tsonAsyncIterator, tsonBigint, tsonPromise } from "../index.js"; +import { tsonAsyncIterable, tsonBigint, tsonPromise } from "../index.js"; import { sleep } from "../internals/testUtils.js"; import { createAsyncTsonSerialize, @@ -103,7 +103,7 @@ test("serialize 2 promises", async () => { test("serialize async iterable", async () => { const serialize = createAsyncTsonSerialize({ nonce: () => "__tson", - types: [tsonAsyncIterator], + types: [tsonAsyncIterable], }); async function* iterable() { @@ -160,7 +160,7 @@ test("serialize async iterable", async () => { test("stringify async iterable + promise", async () => { const stringify = createTsonStringifyAsync({ nonce: () => "__tson", - types: [tsonAsyncIterator, tsonPromise, tsonBigint], + types: [tsonAsyncIterable, tsonPromise, tsonBigint], }); async function* iterable() { diff --git a/src/extend/openai.test.ts b/src/extend/openai.test.ts index 88c6c07b..6ba0d466 100644 --- a/src/extend/openai.test.ts +++ b/src/extend/openai.test.ts @@ -1,7 +1,7 @@ import OpenAI from "openai"; import { expect, test } from "vitest"; -import { createTsonAsync, tsonAsyncIterator, tsonPromise } from "../index.js"; +import { createTsonAsync, tsonAsyncIterable, tsonPromise } from "../index.js"; import { assert } from "../internals/assert.js"; const apiKey = process.env["OPENAI_API_KEY"]; @@ -15,7 +15,7 @@ test.skipIf(!apiKey)("openai", async () => { const tson = createTsonAsync({ nonce: () => "__tson", - types: [tsonAsyncIterator, tsonPromise], + types: [tsonAsyncIterable, tsonPromise], }); const stringified = tson.stringify({ From f154adb3fd504ebd8c3ec06a0ef0fb6e38afeb6a Mon Sep 17 00:00:00 2001 From: KATT Date: Fri, 13 Oct 2023 13:49:32 +0200 Subject: [PATCH 02/17] cool --- src/async/deserializeAsync.test.ts | 12 +- src/async/deserializeAsync.ts | 199 ++++++++++++++----------- src/async/handlers/tsonPromise.test.ts | 3 +- 3 files changed, 117 insertions(+), 97 deletions(-) diff --git a/src/async/deserializeAsync.test.ts b/src/async/deserializeAsync.test.ts index 166a78ea..d1694089 100644 --- a/src/async/deserializeAsync.test.ts +++ b/src/async/deserializeAsync.test.ts @@ -381,7 +381,7 @@ test("values missing when stream ends", async () => { assert(err); expect(err.message).toMatchInlineSnapshot( - '"Stream interrupted: Stream ended unexpectedly"', + '"Stream interrupted: Stream ended unexpectedly (state 1)"', ); } @@ -390,7 +390,7 @@ test("values missing when stream ends", async () => { const err = await waitError(result.promise); expect(err).toMatchInlineSnapshot( - "[TsonStreamInterruptedError: Stream interrupted: Stream ended unexpectedly]", + '[TsonStreamInterruptedError: Stream interrupted: Stream ended unexpectedly (state 1)]', ); } @@ -398,7 +398,7 @@ test("values missing when stream ends", async () => { expect(parseOptions.onStreamError.mock.calls).toMatchInlineSnapshot(` [ [ - [TsonStreamInterruptedError: Stream interrupted: Stream ended unexpectedly], + [TsonStreamInterruptedError: Stream interrupted: Stream ended unexpectedly (state 1)], ], ] `); @@ -439,7 +439,7 @@ test("async: missing values of promise", async () => { }); expect(parseOptions.onStreamError.mock.calls[0]![0]!).toMatchInlineSnapshot( - "[TsonStreamInterruptedError: Stream interrupted: Stream ended unexpectedly]", + '[TsonStreamInterruptedError: Stream interrupted: Stream ended unexpectedly (state 1)]', ); }); @@ -523,7 +523,7 @@ test("1 iterator completed but another never finishes", async () => { `); expect(err.message).toMatchInlineSnapshot( - '"Stream interrupted: Stream ended unexpectedly"', + '"Stream interrupted: Stream ended unexpectedly (state 1)"', ); } @@ -532,7 +532,7 @@ test("1 iterator completed but another never finishes", async () => { expect(parseOptions.onStreamError.mock.calls).toMatchInlineSnapshot(` [ [ - [TsonStreamInterruptedError: Stream interrupted: Stream ended unexpectedly], + [TsonStreamInterruptedError: Stream interrupted: Stream ended unexpectedly (state 1)], ], ] `); diff --git a/src/async/deserializeAsync.ts b/src/async/deserializeAsync.ts index 251dee80..0a270a08 100644 --- a/src/async/deserializeAsync.ts +++ b/src/async/deserializeAsync.ts @@ -1,4 +1,4 @@ -/* eslint-disable @typescript-eslint/no-unused-vars */ +/* eslint-disable @typescript-eslint/no-non-null-assertion */ import { TsonError } from "../errors.js"; import { assert } from "../internals/assert.js"; @@ -37,7 +37,10 @@ type TsonParseAsync = ( opts?: TsonParseAsyncOptions, ) => Promise; -export function createTsonParseAsyncInner(opts: TsonAsyncOptions) { +type TsonDeserializeIterable = AsyncIterable< + TsonAsyncValueTuple | TsonSerialized +>; +export function createTsonDeserializer(opts: TsonAsyncOptions) { const typeByKey: Record = {}; for (const handler of opts.types) { @@ -52,10 +55,9 @@ export function createTsonParseAsyncInner(opts: TsonAsyncOptions) { } return async ( - iterable: AsyncIterable, + iterable: TsonDeserializeIterable, parseOptions: TsonParseAsyncOptions, ) => { - // this is an awful hack to get around making a some sort of pipeline const cache = new Map< TsonAsyncIndex, ReadableStreamDefaultController @@ -106,36 +108,15 @@ export function createTsonParseAsyncInner(opts: TsonAsyncOptions) { return walk; }; - async function getStreamedValues( - lines: string[], - accumulator: string, - walk: WalkFn, - ) { - // - let streamEnded = false; - // - - function readLine(str: string) { - // console.log("got str", str); - str = str.trimStart(); - - if (str.startsWith(",")) { - // ignore leading comma - str = str.slice(1); - } - - if (str === "" || str === "[" || str === ",") { - // beginning of values array or empty string - return; - } - - if (str === "]]") { - // end of values and stream - streamEnded = true; - return; + async function getStreamedValues(walk: WalkFn) { + // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition + while (true) { + const nextValue = await iterator.next(); + if (nextValue.done) { + break; } - const [index, result] = JSON.parse(str) as TsonAsyncValueTuple; + const [index, result] = nextValue.value as TsonAsyncValueTuple; const controller = cache.get(index); @@ -146,68 +127,25 @@ export function createTsonParseAsyncInner(opts: TsonAsyncOptions) { // resolving deferred controller.enqueue(walkedResult); } - - do { - lines.forEach(readLine); - lines.length = 0; - - const nextValue = await iterator.next(); - if (!nextValue.done) { - accumulator += nextValue.value; - const parts = accumulator.split("\n"); - accumulator = parts.pop() ?? ""; - lines.push(...parts); - } else if (accumulator) { - readLine(accumulator); - } - } while (lines.length); - - assert(streamEnded, "Stream ended unexpectedly"); } async function init() { - let accumulator = ""; - // get the head of the JSON + const nextValue = await iterator.next(); + if (nextValue.done) { + throw new TsonError("Unexpected end of stream before head"); + } - const lines: string[] = []; - do { - const nextValue = await iterator.next(); - if (nextValue.done) { - throw new TsonError("Unexpected end of stream before head"); - } - - accumulator += nextValue.value; - - const parts = accumulator.split("\n"); - accumulator = parts.pop() ?? ""; - lines.push(...parts); - } while (lines.length < 2); - - const [ - /** - * First line is just a `[` - */ - _firstLine, - /** - * Second line is the shape of the JSON - */ - headLine, - // .. third line is a `,` - // .. fourth line is the start of the values array - ...buffer - ] = lines; - - assert(headLine, "No head line found"); - - const head = JSON.parse(headLine) as TsonSerialized; + const head = nextValue.value as TsonSerialized; const walk = walker(head.nonce); try { - return walk(head.json); + const walked = walk(head.json); + + return walked; } finally { - getStreamedValues(buffer, accumulator, walk).catch((cause) => { + getStreamedValues(walk).catch((cause) => { // Something went wrong while getting the streamed values const err = new TsonStreamInterruptedError(cause); @@ -222,19 +160,102 @@ export function createTsonParseAsyncInner(opts: TsonAsyncOptions) { } } - const result = await init().catch((cause: unknown) => { + return await init().catch((cause: unknown) => { throw new TsonError("Failed to initialize TSON stream", { cause }); }); - return [result, cache] as const; }; } +function lineAccumulator() { + let accumulator = ""; + const lines: string[] = []; + + return { + lines, + push(chunk: string) { + accumulator += chunk; + + const parts = accumulator.split("\n"); + accumulator = parts.pop() ?? ""; + lines.push(...parts); + }, + }; +} + +async function* stringIterableToTsonIterable( + iterable: AsyncIterable, +): TsonDeserializeIterable { + // get the head of the JSON + const acc = lineAccumulator(); + + // state of stream + const AWAITING_HEAD = 0; + const STREAMING_VALUES = 1; + const ENDED = 2; + + let state: typeof AWAITING_HEAD | typeof ENDED | typeof STREAMING_VALUES = + AWAITING_HEAD; + + // iterate values & yield them + + for await (const str of iterable) { + acc.push(str); + + if (state === AWAITING_HEAD && acc.lines.length >= 2) { + /** + * First line is just a `[` + */ + acc.lines.shift(); + + // Second line is the head + const headLine = acc.lines.shift(); + + assert(headLine, "No head line found"); + + const head = JSON.parse(headLine) as TsonSerialized; + + yield head; + + state = STREAMING_VALUES; + } + + if (state === STREAMING_VALUES) { + while (acc.lines.length) { + let str = acc.lines.shift()!; + + // console.log("got str", str); + str = str.trimStart(); + + if (str.startsWith(",")) { + // ignore leading comma + str = str.slice(1); + } + + if (str === "" || str === "[" || str === ",") { + // beginning of values array or empty string + continue; + } + + if (str === "]]") { + // end of values and stream + state = ENDED; + continue; + } + + yield JSON.parse(str) as TsonAsyncValueTuple; + } + } + } + + assert(state === ENDED, `Stream ended unexpectedly (state ${state})`); +} + export function createTsonParseAsync(opts: TsonAsyncOptions): TsonParseAsync { - const instance = createTsonParseAsyncInner(opts); + const instance = createTsonDeserializer(opts); return (async (iterable, opts) => { - const [result] = await instance(iterable, opts ?? {}); + const tsonIterable = stringIterableToTsonIterable(iterable); - return result; + return await instance(tsonIterable, opts ?? {}); }) as TsonParseAsync; } diff --git a/src/async/handlers/tsonPromise.test.ts b/src/async/handlers/tsonPromise.test.ts index 06d18a35..7e264146 100644 --- a/src/async/handlers/tsonPromise.test.ts +++ b/src/async/handlers/tsonPromise.test.ts @@ -15,7 +15,6 @@ import { waitError, } from "../../internals/testUtils.js"; import { createPromise } from "../../internals/testUtils.js"; -import { createTsonParseAsyncInner } from "../deserializeAsync.js"; import { mapIterable, readableStreamToAsyncIterable, @@ -453,7 +452,7 @@ test("does not crash node when it receives a promise rejection", async () => { }; const stringify = createTsonStringifyAsync(opts); - const parse = createTsonParseAsyncInner(opts); + const parse = createTsonParseAsync(opts); const original = { foo: createPromise(() => { From 0aa88df83f7a2df2159505df4f94f74e3ee82090 Mon Sep 17 00:00:00 2001 From: KATT Date: Fri, 13 Oct 2023 14:03:38 +0200 Subject: [PATCH 03/17] cool --- src/async/createTsonAsync.ts | 5 +++++ src/async/deserializeAsync.test.ts | 6 +++--- src/async/handlers/tsonPromise.test.ts | 2 +- src/extend/openai.test.ts | 3 ++- src/index.ts | 1 - 5 files changed, 11 insertions(+), 6 deletions(-) diff --git a/src/async/createTsonAsync.ts b/src/async/createTsonAsync.ts index 6bbd33ad..910b1884 100644 --- a/src/async/createTsonAsync.ts +++ b/src/async/createTsonAsync.ts @@ -2,6 +2,11 @@ import { TsonAsyncOptions } from "./asyncTypes.js"; import { createTsonParseAsync } from "./deserializeAsync.js"; import { createTsonStringifyAsync } from "./serializeAsync.js"; +/** + * @internal + * + * Only used for testing - when using the async you gotta pick which one you want + */ export const createTsonAsync = (opts: TsonAsyncOptions) => ({ parse: createTsonParseAsync(opts), stringify: createTsonStringifyAsync(opts), diff --git a/src/async/deserializeAsync.test.ts b/src/async/deserializeAsync.test.ts index d1694089..a19565d4 100644 --- a/src/async/deserializeAsync.test.ts +++ b/src/async/deserializeAsync.test.ts @@ -4,7 +4,6 @@ import { TsonAsyncOptions, TsonParseAsyncOptions, TsonType, - createTsonAsync, createTsonParseAsync, tsonAsyncIterable, tsonBigint, @@ -19,6 +18,7 @@ import { waitFor, } from "../internals/testUtils.js"; import { TsonSerialized } from "../sync/syncTypes.js"; +import { createTsonAsync } from "./createTsonAsync.js"; import { mapIterable, readableStreamToAsyncIterable } from "./iterableUtils.js"; test("deserialize variable chunk length", async () => { @@ -390,7 +390,7 @@ test("values missing when stream ends", async () => { const err = await waitError(result.promise); expect(err).toMatchInlineSnapshot( - '[TsonStreamInterruptedError: Stream interrupted: Stream ended unexpectedly (state 1)]', + "[TsonStreamInterruptedError: Stream interrupted: Stream ended unexpectedly (state 1)]", ); } @@ -439,7 +439,7 @@ test("async: missing values of promise", async () => { }); expect(parseOptions.onStreamError.mock.calls[0]![0]!).toMatchInlineSnapshot( - '[TsonStreamInterruptedError: Stream interrupted: Stream ended unexpectedly (state 1)]', + "[TsonStreamInterruptedError: Stream interrupted: Stream ended unexpectedly (state 1)]", ); }); diff --git a/src/async/handlers/tsonPromise.test.ts b/src/async/handlers/tsonPromise.test.ts index 7e264146..10b8ed76 100644 --- a/src/async/handlers/tsonPromise.test.ts +++ b/src/async/handlers/tsonPromise.test.ts @@ -4,7 +4,6 @@ import { TsonAsyncOptions, TsonType, createAsyncTsonSerialize, - createTsonAsync, createTsonParseAsync, createTsonStringifyAsync, tsonPromise, @@ -15,6 +14,7 @@ import { waitError, } from "../../internals/testUtils.js"; import { createPromise } from "../../internals/testUtils.js"; +import { createTsonAsync } from "../createTsonAsync.js"; import { mapIterable, readableStreamToAsyncIterable, diff --git a/src/extend/openai.test.ts b/src/extend/openai.test.ts index 6ba0d466..6e131616 100644 --- a/src/extend/openai.test.ts +++ b/src/extend/openai.test.ts @@ -1,7 +1,8 @@ import OpenAI from "openai"; import { expect, test } from "vitest"; -import { createTsonAsync, tsonAsyncIterable, tsonPromise } from "../index.js"; +import { createTsonAsync } from "../async/createTsonAsync.js"; +import { tsonAsyncIterable, tsonPromise } from "../index.js"; import { assert } from "../internals/assert.js"; const apiKey = process.env["OPENAI_API_KEY"]; diff --git a/src/index.ts b/src/index.ts index 42cc0781..4d4b0946 100644 --- a/src/index.ts +++ b/src/index.ts @@ -21,7 +21,6 @@ export * from "./sync/handlers/tsonSymbol.js"; // --- async -- export type { TsonAsyncOptions } from "./async/asyncTypes.js"; -export { createTsonAsync } from "./async/createTsonAsync.js"; export { type TsonParseAsyncOptions, createTsonParseAsync, From d8a740cee56c25d7b3ea0ce8b39cd4b9d8a6fc5e Mon Sep 17 00:00:00 2001 From: KATT Date: Fri, 13 Oct 2023 14:07:05 +0200 Subject: [PATCH 04/17] cool --- examples/async/src/server.ts | 4 +-- src/async/createTsonAsync.ts | 6 ++-- src/async/deserializeAsync.test.ts | 42 +++++++++++++++----------- src/async/handlers/tsonPromise.test.ts | 24 +++++++-------- src/async/serializeAsync.test.ts | 4 +-- src/async/serializeAsync.ts | 2 +- src/extend/openai.test.ts | 4 +-- src/index.ts | 2 +- 8 files changed, 47 insertions(+), 41 deletions(-) diff --git a/examples/async/src/server.ts b/examples/async/src/server.ts index fd1fe70e..c40bb773 100644 --- a/examples/async/src/server.ts +++ b/examples/async/src/server.ts @@ -1,9 +1,9 @@ import http from "node:http"; -import { createTsonStringifyAsync } from "tupleson"; +import { createTsonStreamAsync } from "tupleson"; import { tsonOptions } from "./shared.js"; -const tsonStringifyAsync = createTsonStringifyAsync(tsonOptions); +const tsonStringifyAsync = createTsonStreamAsync(tsonOptions); const randomNumber = (min: number, max: number) => Math.floor(Math.random() * (max - min + 1) + min); diff --git a/src/async/createTsonAsync.ts b/src/async/createTsonAsync.ts index 910b1884..348e75fb 100644 --- a/src/async/createTsonAsync.ts +++ b/src/async/createTsonAsync.ts @@ -1,6 +1,6 @@ import { TsonAsyncOptions } from "./asyncTypes.js"; import { createTsonParseAsync } from "./deserializeAsync.js"; -import { createTsonStringifyAsync } from "./serializeAsync.js"; +import { createTsonStreamAsync } from "./serializeAsync.js"; /** * @internal @@ -8,6 +8,6 @@ import { createTsonStringifyAsync } from "./serializeAsync.js"; * Only used for testing - when using the async you gotta pick which one you want */ export const createTsonAsync = (opts: TsonAsyncOptions) => ({ - parse: createTsonParseAsync(opts), - stringify: createTsonStringifyAsync(opts), + parseJsonStream: createTsonParseAsync(opts), + stringifyJsonStream: createTsonStreamAsync(opts), }); diff --git a/src/async/deserializeAsync.test.ts b/src/async/deserializeAsync.test.ts index a19565d4..28581aa6 100644 --- a/src/async/deserializeAsync.test.ts +++ b/src/async/deserializeAsync.test.ts @@ -32,7 +32,7 @@ test("deserialize variable chunk length", async () => { yield '[\n{"json":{"foo":"bar"},"nonce":"__tson"}'; yield "\n,\n[\n]\n]"; })(); - const result = await tson.parse(iterable); + const result = await tson.parseJsonStream(iterable); expect(result).toEqual({ foo: "bar" }); } @@ -41,7 +41,7 @@ test("deserialize variable chunk length", async () => { await sleep(1); yield '[\n{"json":{"foo":"bar"},"nonce":"__tson"}\n,\n[\n]\n]'; })(); - const result = await tson.parse(iterable); + const result = await tson.parseJsonStream(iterable); expect(result).toEqual({ foo: "bar" }); } @@ -54,7 +54,7 @@ test("deserialize variable chunk length", async () => { yield "[\n]\n"; yield "]"; })(); - const result = await tson.parse(iterable); + const result = await tson.parseJsonStream(iterable); expect(result).toEqual({ foo: "bar" }); } }); @@ -71,9 +71,9 @@ test("deserialize async iterable", async () => { foo: "bar", }; - const strIterable = tson.stringify(obj); + const strIterable = tson.stringifyJsonStream(obj); - const result = await tson.parse(strIterable); + const result = await tson.parseJsonStream(strIterable); expect(result).toEqual(obj); } @@ -84,9 +84,9 @@ test("deserialize async iterable", async () => { foo: Promise.resolve("bar"), }; - const strIterable = tson.stringify(obj); + const strIterable = tson.stringifyJsonStream(obj); - const result = await tson.parse(strIterable); + const result = await tson.parseJsonStream(strIterable); expect(await result.foo).toEqual("bar"); } @@ -120,9 +120,9 @@ test("stringify async iterable + promise", async () => { promise: Promise.resolve(42), }; - const strIterable = tson.stringify(input); + const strIterable = tson.stringifyJsonStream(input); - const output = await tson.parse(strIterable, parseOptions); + const output = await tson.parseJsonStream(strIterable, parseOptions); expect(output.foo).toEqual("bar"); @@ -166,7 +166,7 @@ test("e2e: stringify async iterable and promise over the network", async () => { const tson = createTsonAsync(opts); const obj = createMockObj(); - const strIterarable = tson.stringify(obj, 4); + const strIterarable = tson.stringifyJsonStream(obj, 4); for await (const value of strIterarable) { res.write(value); @@ -192,7 +192,7 @@ test("e2e: stringify async iterable and promise over the network", async () => { (v) => textDecoder.decode(v), ); - const parsed = await tson.parse(stringIterator); + const parsed = await tson.parseJsonStream(stringIterator); expect(parsed.foo).toEqual("bar"); @@ -267,7 +267,7 @@ test("iterator error", async () => { const tson = createTsonAsync(opts); const obj = createMockObj(); - const strIterarable = tson.stringify(obj, 4); + const strIterarable = tson.stringifyJsonStream(obj, 4); for await (const value of strIterarable) { res.write(value); @@ -291,7 +291,7 @@ test("iterator error", async () => { (v) => textDecoder.decode(v), ); - const parsed = await tson.parse(stringIterator); + const parsed = await tson.parseJsonStream(stringIterator); expect(await parsed.promise).toEqual(42); const results = []; @@ -432,7 +432,7 @@ test("async: missing values of promise", async () => { await createTsonAsync({ types: [tsonPromise], - }).parse(generator(), parseOptions); + }).parseJsonStream(generator(), parseOptions); await waitFor(() => { expect(parseOptions.onStreamError).toHaveBeenCalledTimes(1); @@ -578,7 +578,7 @@ test("e2e: simulated server crash", async () => { const tson = createTsonAsync(opts); const obj = createMockObj(); - const strIterarable = tson.stringify(obj, 4); + const strIterarable = tson.stringifyJsonStream(obj, 4); void crashedDeferred.promise.then(() => { // destroy the response stream @@ -607,7 +607,10 @@ test("e2e: simulated server crash", async () => { (v) => textDecoder.decode(v), ); - const parsed = await tson.parse(stringIterator, parseOptions); + const parsed = await tson.parseJsonStream( + stringIterator, + parseOptions, + ); { // check the iterator const results = []; @@ -678,7 +681,7 @@ test("e2e: client aborted request", async () => { const tson = createTsonAsync(opts); const obj = createMockObj(); - const strIterarable = tson.stringify(obj, 4); + const strIterarable = tson.stringifyJsonStream(obj, 4); for await (const value of strIterarable) { serverSentChunks.push(value.trimEnd()); @@ -707,7 +710,10 @@ test("e2e: client aborted request", async () => { (v) => textDecoder.decode(v), ); - const parsed = await tson.parse(stringIterator, parseOptions); + const parsed = await tson.parseJsonStream( + stringIterator, + parseOptions, + ); { // check the iterator const results = []; diff --git a/src/async/handlers/tsonPromise.test.ts b/src/async/handlers/tsonPromise.test.ts index 10b8ed76..fa4de76b 100644 --- a/src/async/handlers/tsonPromise.test.ts +++ b/src/async/handlers/tsonPromise.test.ts @@ -5,7 +5,7 @@ import { TsonType, createAsyncTsonSerialize, createTsonParseAsync, - createTsonStringifyAsync, + createTsonStreamAsync, tsonPromise, } from "../../index.js"; import { @@ -193,7 +193,7 @@ test("stringifier - no promises", async () => { const buffer: string[] = []; - for await (const value of tson.stringify(obj, 4)) { + for await (const value of tson.stringifyJsonStream(obj, 4)) { buffer.push(value.trimEnd()); } @@ -230,7 +230,7 @@ test("stringifier - with promise", async () => { const buffer: string[] = []; - for await (const value of tson.stringify(obj, 4)) { + for await (const value of tson.stringifyJsonStream(obj, 4)) { buffer.push(value.trimEnd()); } @@ -264,7 +264,7 @@ test("stringifier - promise in promise", async () => { const buffer: string[] = []; - for await (const value of tson.stringify(obj, 2)) { + for await (const value of tson.stringifyJsonStream(obj, 2)) { buffer.push(value.trimEnd()); } @@ -332,9 +332,9 @@ test("pipe stringifier to parser", async () => { types: [tsonPromise], }); - const strIterarable = tson.stringify(obj, 4); + const strIterarable = tson.stringifyJsonStream(obj, 4); - const value = await tson.parse(strIterarable); + const value = await tson.parseJsonStream(strIterarable); expect(value).toHaveProperty("foo"); expect(await value.foo).toBe("bar"); @@ -355,9 +355,9 @@ test("stringify and parse promise with a promise", async () => { types: [tsonPromise], }); - const strIterarable = tson.stringify(obj, 4); + const strIterarable = tson.stringifyJsonStream(obj, 4); - const value = await tson.parse(strIterarable); + const value = await tson.parseJsonStream(strIterarable); const firstPromise = await value.promise; @@ -396,7 +396,7 @@ test("stringify and parse promise with a promise over a network connection", asy }; }, 3), }; - const strIterarable = tson.stringify(obj, 4); + const strIterarable = tson.stringifyJsonStream(obj, 4); for await (const value of strIterarable) { res.write(value); @@ -422,7 +422,7 @@ test("stringify and parse promise with a promise over a network connection", asy (v) => textDecoder.decode(v), ); - const value = await tson.parse(stringIterator); + const value = await tson.parseJsonStream(stringIterator); const asObj = value as Obj; const firstPromise = await asObj.promise; @@ -450,7 +450,7 @@ test("does not crash node when it receives a promise rejection", async () => { nonce: () => "__tson", types: [tsonPromise], }; - const stringify = createTsonStringifyAsync(opts); + const stringify = createTsonStreamAsync(opts); const parse = createTsonParseAsync(opts); @@ -471,7 +471,7 @@ test("stringify promise rejection", async () => { nonce: () => "__tson", types: [tsonPromise, tsonError], }; - const stringify = createTsonStringifyAsync(opts); + const stringify = createTsonStreamAsync(opts); const parse = createTsonParseAsync(opts); diff --git a/src/async/serializeAsync.test.ts b/src/async/serializeAsync.test.ts index fff5c652..7f24b02d 100644 --- a/src/async/serializeAsync.test.ts +++ b/src/async/serializeAsync.test.ts @@ -4,7 +4,7 @@ import { tsonAsyncIterable, tsonBigint, tsonPromise } from "../index.js"; import { sleep } from "../internals/testUtils.js"; import { createAsyncTsonSerialize, - createTsonStringifyAsync, + createTsonStreamAsync, } from "./serializeAsync.js"; test("serialize promise", async () => { @@ -158,7 +158,7 @@ test("serialize async iterable", async () => { }); test("stringify async iterable + promise", async () => { - const stringify = createTsonStringifyAsync({ + const stringify = createTsonStreamAsync({ nonce: () => "__tson", types: [tsonAsyncIterable, tsonPromise, tsonBigint], }); diff --git a/src/async/serializeAsync.ts b/src/async/serializeAsync.ts index 3f8e6dd8..7bab69e3 100644 --- a/src/async/serializeAsync.ts +++ b/src/async/serializeAsync.ts @@ -207,7 +207,7 @@ export function createAsyncTsonSerialize( }; } -export function createTsonStringifyAsync( +export function createTsonStreamAsync( opts: TsonAsyncOptions, ): TsonAsyncStringifier { const indent = (length: number) => " ".repeat(length); diff --git a/src/extend/openai.test.ts b/src/extend/openai.test.ts index 6e131616..3566d107 100644 --- a/src/extend/openai.test.ts +++ b/src/extend/openai.test.ts @@ -19,7 +19,7 @@ test.skipIf(!apiKey)("openai", async () => { types: [tsonAsyncIterable, tsonPromise], }); - const stringified = tson.stringify({ + const stringified = tson.stringifyJsonStream({ stream: await openai.chat.completions.create({ messages: [{ content: "Say this is a test", role: "user" }], model: "gpt-4", @@ -27,7 +27,7 @@ test.skipIf(!apiKey)("openai", async () => { }), }); - const parsed = await tson.parse(stringified); + const parsed = await tson.parseJsonStream(stringified); let buffer = ""; for await (const out of parsed.stream) { diff --git a/src/index.ts b/src/index.ts index 4d4b0946..910d942c 100644 --- a/src/index.ts +++ b/src/index.ts @@ -27,7 +27,7 @@ export { } from "./async/deserializeAsync.js"; export { createAsyncTsonSerialize, - createTsonStringifyAsync, + createTsonStreamAsync, } from "./async/serializeAsync.js"; export * from "./async/asyncErrors.js"; From bb0e688dba5611ff4db877fb3abbdc4b023a2aa5 Mon Sep 17 00:00:00 2001 From: KATT Date: Fri, 13 Oct 2023 14:25:13 +0200 Subject: [PATCH 05/17] cool SSE bruv --- src/async/asyncTypes.ts | 4 ++++ src/async/serializeAsync.ts | 38 +++++++++++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+) diff --git a/src/async/asyncTypes.ts b/src/async/asyncTypes.ts index 5ed6bb2a..830e0931 100644 --- a/src/async/asyncTypes.ts +++ b/src/async/asyncTypes.ts @@ -10,6 +10,10 @@ export type TsonAsyncStringifierIterable = AsyncIterable & { [serialized]: TValue; }; +export type BrandSerialized = TType & { + [serialized]: TValue; +}; + export type TsonAsyncStringifier = ( value: TValue, space?: number, diff --git a/src/async/serializeAsync.ts b/src/async/serializeAsync.ts index 7bab69e3..c80d7cf7 100644 --- a/src/async/serializeAsync.ts +++ b/src/async/serializeAsync.ts @@ -13,6 +13,7 @@ import { TsonTypeTesterPrimitive, } from "../sync/syncTypes.js"; import { + BrandSerialized, TsonAsyncIndex, TsonAsyncOptions, TsonAsyncStringifier, @@ -246,3 +247,40 @@ export function createTsonStreamAsync( return stringifier as TsonAsyncStringifier; } + +export function crateTsonSSEResponse(opts: TsonAsyncOptions) { + const serialize = createAsyncTsonSerialize(opts); + + return (value: TValue) => { + let controller: ReadableStreamDefaultController = + null as unknown as ReadableStreamDefaultController; + const readable = new ReadableStream({ + start(c) { + controller = c; + }, + }); + + async function iterate() { + const [head, iterable] = serialize(value); + + controller.enqueue(`data: ${JSON.stringify(head)}\n\n`); + for await (const chunk of iterable) { + controller.enqueue(`data: ${JSON.stringify(chunk)}`); + } + } + + iterate().catch((err) => { + controller.error(err); + }); + + const res = new Response(readable, { + headers: { + "Cache-Control": "no-cache", + Connection: "keep-alive", + "Content-Type": "text/event-stream", + }, + status: 200, + }); + return res as BrandSerialized; + }; +} From ec8920d6cc094ab6bec0454d0ef4ed04e11367ae Mon Sep 17 00:00:00 2001 From: KATT Date: Fri, 13 Oct 2023 14:29:15 +0200 Subject: [PATCH 06/17] wip --- README.md | 2 +- src/async/serializeAsync.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 70dc1430..c1d19b6a 100644 --- a/README.md +++ b/README.md @@ -40,7 +40,7 @@ Serialize almost[^1] anything! - 🌊 Serialize & stream things like `Promise`s or async iterators > [!IMPORTANT] -> _Though well-tested, this package might undergo big changes, stay tuned!_ +> _Though well-tested, this package might undergo big changes and **does not** follow semver whilst on `0.x.y`-version, stay tuned!_ ### 👀 Example diff --git a/src/async/serializeAsync.ts b/src/async/serializeAsync.ts index c80d7cf7..80f610aa 100644 --- a/src/async/serializeAsync.ts +++ b/src/async/serializeAsync.ts @@ -265,7 +265,7 @@ export function crateTsonSSEResponse(opts: TsonAsyncOptions) { controller.enqueue(`data: ${JSON.stringify(head)}\n\n`); for await (const chunk of iterable) { - controller.enqueue(`data: ${JSON.stringify(chunk)}`); + controller.enqueue(`data: ${JSON.stringify(chunk)}\n\n`); } } From 1df1b91c90f10185226581ac2c1ea0698d905169 Mon Sep 17 00:00:00 2001 From: KATT Date: Fri, 13 Oct 2023 14:55:29 +0200 Subject: [PATCH 07/17] cool --- examples/sse/package.json | 25 ++++++++++ examples/sse/src/index.html | 16 +++++++ examples/sse/src/server.ts | 91 ++++++++++++++++++++++++++++++++++++ examples/sse/src/shared.ts | 21 +++++++++ examples/sse/tsconfig.json | 11 +++++ src/async/createTsonAsync.ts | 6 ++- src/async/serializeAsync.ts | 2 +- src/async/sse.test.ts | 85 +++++++++++++++++++++++++++++++++ src/index.ts | 1 + 9 files changed, 256 insertions(+), 2 deletions(-) create mode 100644 examples/sse/package.json create mode 100644 examples/sse/src/index.html create mode 100644 examples/sse/src/server.ts create mode 100644 examples/sse/src/shared.ts create mode 100644 examples/sse/tsconfig.json create mode 100644 src/async/sse.test.ts diff --git a/examples/sse/package.json b/examples/sse/package.json new file mode 100644 index 00000000..0e1f612f --- /dev/null +++ b/examples/sse/package.json @@ -0,0 +1,25 @@ +{ + "name": "@examples/minimal", + "version": "10.38.5", + "private": true, + "description": "An example project for tupleson", + "license": "MIT", + "module": "module", + "workspaces": [ + "client", + "server" + ], + "scripts": { + "build": "tsc", + "dev:server": "tsx watch src/server", + "dev": "run-p dev:* --print-label" + }, + "devDependencies": { + "@types/node": "^18.16.16", + "npm-run-all": "^4.1.5", + "tsx": "^3.12.7", + "tupleson": "latest", + "typescript": "^5.1.3", + "wait-port": "^1.0.1" + } +} diff --git a/examples/sse/src/index.html b/examples/sse/src/index.html new file mode 100644 index 00000000..aec9b317 --- /dev/null +++ b/examples/sse/src/index.html @@ -0,0 +1,16 @@ +

SSE test

+ +See log output / inspector tools + + diff --git a/examples/sse/src/server.ts b/examples/sse/src/server.ts new file mode 100644 index 00000000..15dbfd06 --- /dev/null +++ b/examples/sse/src/server.ts @@ -0,0 +1,91 @@ +import fs from "node:fs"; +import http from "node:http"; +import { createTsonSSEResponse } from "tupleson"; + +import { tsonOptions } from "./shared.js"; + +const createResponse = createTsonSSEResponse(tsonOptions); + +const randomNumber = (min: number, max: number) => + Math.floor(Math.random() * (max - min + 1) + min); + +const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); + +/** + * This function returns the object we will be sending to the client. + */ +export function getResponseShape() { + async function* bigintGenerator() { + const iterate = new Array(10).fill(0).map((_, i) => BigInt(i)); + for (const number of iterate) { + await sleep(randomNumber(1, 400)); + yield number; + } + } + + async function* numberGenerator() { + const iterate = new Array(10).fill(0).map((_, i) => i); + for (const number of iterate) { + await sleep(randomNumber(1, 400)); + yield number; + } + } + + return { + bigints: bigintGenerator(), + foo: "bar", + numbers: numberGenerator(), + promise: new Promise((resolve) => + setTimeout(() => { + resolve(42); + }, 1), + ), + rejectedPromise: new Promise((_, reject) => + setTimeout(() => { + reject(new Error("Rejected promise")); + }, 1), + ), + }; +} + +export type ResponseShape = ReturnType; +async function handleRequest( + req: http.IncomingMessage, + res: http.ServerResponse, +) { + if (req.url?.startsWith("/sse")) { + const obj = getResponseShape(); + const response = createResponse(obj); + + // Stream the response to the client + for (const [key, value] of response.headers) { + res.setHeader(key, value); + } + + for await (const value of response.body as any) { + res.write(value); + } + + res.end(); + return; + } + + const data = fs.readFileSync(__dirname + "/index.html"); + res.write(data.toString()); + res.end(); +} + +const server = http.createServer( + (req: http.IncomingMessage, res: http.ServerResponse) => { + handleRequest(req, res).catch((err) => { + console.error(err); + res.writeHead(500, { "Content-Type": "text/plain" }); + res.end("Internal Server Error\n"); + }); + }, +); + +const port = 3000; +server.listen(port); + +console.log(`Server running at http://localhost:${port}`); diff --git a/examples/sse/src/shared.ts b/examples/sse/src/shared.ts new file mode 100644 index 00000000..cc973f9d --- /dev/null +++ b/examples/sse/src/shared.ts @@ -0,0 +1,21 @@ +import { + TsonAsyncOptions, + tsonAsyncIterable, + tsonBigint, + tsonPromise, +} from "tupleson"; + +/** + * Shared tupleSON options for the server and client. + */ +export const tsonOptions: TsonAsyncOptions = { + // We need to specify the types we want to allow serialization of + types: [ + // Allow serialization of promises + tsonPromise, + // Allow serialization of async iterators + tsonAsyncIterable, + // Allow serialization of bigints + tsonBigint, + ], +}; diff --git a/examples/sse/tsconfig.json b/examples/sse/tsconfig.json new file mode 100644 index 00000000..95c6bc3b --- /dev/null +++ b/examples/sse/tsconfig.json @@ -0,0 +1,11 @@ +{ + "compilerOptions": { + "esModuleInterop": true, + "module": "esnext", + "moduleResolution": "node", + "outDir": "dist", + "strict": true, + "target": "esnext" + }, + "include": ["src"] +} diff --git a/src/async/createTsonAsync.ts b/src/async/createTsonAsync.ts index 348e75fb..c23c490c 100644 --- a/src/async/createTsonAsync.ts +++ b/src/async/createTsonAsync.ts @@ -1,6 +1,9 @@ import { TsonAsyncOptions } from "./asyncTypes.js"; import { createTsonParseAsync } from "./deserializeAsync.js"; -import { createTsonStreamAsync } from "./serializeAsync.js"; +import { + createTsonSSEResponse, + createTsonStreamAsync, +} from "./serializeAsync.js"; /** * @internal @@ -10,4 +13,5 @@ import { createTsonStreamAsync } from "./serializeAsync.js"; export const createTsonAsync = (opts: TsonAsyncOptions) => ({ parseJsonStream: createTsonParseAsync(opts), stringifyJsonStream: createTsonStreamAsync(opts), + toSSEResponse: createTsonSSEResponse(opts), }); diff --git a/src/async/serializeAsync.ts b/src/async/serializeAsync.ts index 80f610aa..f8747cc5 100644 --- a/src/async/serializeAsync.ts +++ b/src/async/serializeAsync.ts @@ -248,7 +248,7 @@ export function createTsonStreamAsync( return stringifier as TsonAsyncStringifier; } -export function crateTsonSSEResponse(opts: TsonAsyncOptions) { +export function createTsonSSEResponse(opts: TsonAsyncOptions) { const serialize = createAsyncTsonSerialize(opts); return (value: TValue) => { diff --git a/src/async/sse.test.ts b/src/async/sse.test.ts new file mode 100644 index 00000000..84f16133 --- /dev/null +++ b/src/async/sse.test.ts @@ -0,0 +1,85 @@ +import { expect, test, vitest } from "vitest"; + +import { + TsonAsyncOptions, + TsonParseAsyncOptions, + TsonType, + createTsonParseAsync, + tsonAsyncIterable, + tsonBigint, + tsonPromise, +} from "../index.js"; +import { assert } from "../internals/assert.js"; +import { + createDeferred, + createTestServer, + sleep, + waitError, + waitFor, +} from "../internals/testUtils.js"; +import { TsonSerialized } from "../sync/syncTypes.js"; +import { createTsonAsync } from "./createTsonAsync.js"; +import { mapIterable, readableStreamToAsyncIterable } from "./iterableUtils.js"; + +test("SSE response test", async () => { + function createMockObj() { + async function* generator() { + for (let i = 0; i < 10; i++) { + yield i; + await sleep(1); + + await sleep(1); + } + } + + return { + foo: "bar", + iterable: generator(), + promise: Promise.resolve(42), + rejectedPromise: Promise.reject(new Error("rejected promise")), + }; + } + + type MockObj = ReturnType; + + // ------------- server ------------------- + const opts = { + types: [tsonPromise, tsonAsyncIterable], + } satisfies TsonAsyncOptions; + + const server = await createTestServer({ + handleRequest: async (_req, res) => { + const tson = createTsonAsync(opts); + + const obj = createMockObj(); + const response = tson.toSSEResponse(obj); + + for (const [key, value] of response.headers) { + res.setHeader(key, value); + } + + for await (const value of response.body as any) { + res.write(value); + } + + res.end(); + }, + }); + + // ------------- client ------------------- + const tson = createTsonAsync(opts); + + // do a streamed fetch request + const sse = new EventSource(server.url); + + let messages: MessageEvent[]; + await new Promise(() => { + sse.onmessage = (msg) => { + messages.push(msg); + }; + + sse.addEventListener("error", () => { + console.error("error"); + }); + }); +}); diff --git a/src/index.ts b/src/index.ts index 910d942c..d3cd0d64 100644 --- a/src/index.ts +++ b/src/index.ts @@ -27,6 +27,7 @@ export { } from "./async/deserializeAsync.js"; export { createAsyncTsonSerialize, + createTsonSSEResponse, createTsonStreamAsync, } from "./async/serializeAsync.js"; export * from "./async/asyncErrors.js"; From 7a6d46afbdebe617a0c023b55ec22969884c522a Mon Sep 17 00:00:00 2001 From: KATT Date: Fri, 13 Oct 2023 14:57:16 +0200 Subject: [PATCH 08/17] lock --- pnpm-lock.yaml | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 213768c3..5e01b61c 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -168,6 +168,27 @@ importers: specifier: ^1.0.1 version: 1.1.0 + examples/sse: + devDependencies: + '@types/node': + specifier: ^18.16.16 + version: 18.18.3 + npm-run-all: + specifier: ^4.1.5 + version: 4.1.5 + tsx: + specifier: ^3.12.7 + version: 3.13.0 + tupleson: + specifier: link:../.. + version: link:../.. + typescript: + specifier: ^5.1.3 + version: 5.2.2 + wait-port: + specifier: ^1.0.1 + version: 1.1.0 + packages: /@aashutoshrathi/word-wrap@1.2.6: From 703be261a127eec5d21d3d8f5735f4ae0d360489 Mon Sep 17 00:00:00 2001 From: KATT Date: Fri, 13 Oct 2023 14:58:46 +0200 Subject: [PATCH 09/17] fix --- examples/async/package.json | 2 +- examples/sse/package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/async/package.json b/examples/async/package.json index 9d6ea04e..4b6914ed 100644 --- a/examples/async/package.json +++ b/examples/async/package.json @@ -1,5 +1,5 @@ { - "name": "@examples/minimal", + "name": "@examples/json-stream", "version": "10.38.5", "private": true, "description": "An example project for tupleson", diff --git a/examples/sse/package.json b/examples/sse/package.json index 0e1f612f..5179843a 100644 --- a/examples/sse/package.json +++ b/examples/sse/package.json @@ -1,5 +1,5 @@ { - "name": "@examples/minimal", + "name": "@examples/sse", "version": "10.38.5", "private": true, "description": "An example project for tupleson", From 820220195756f7961482b8e9cacbf885d2ee30c6 Mon Sep 17 00:00:00 2001 From: KATT Date: Fri, 13 Oct 2023 16:29:27 +0200 Subject: [PATCH 10/17] close the controller --- src/async/serializeAsync.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/async/serializeAsync.ts b/src/async/serializeAsync.ts index f8747cc5..5c2d3c94 100644 --- a/src/async/serializeAsync.ts +++ b/src/async/serializeAsync.ts @@ -267,6 +267,8 @@ export function createTsonSSEResponse(opts: TsonAsyncOptions) { for await (const chunk of iterable) { controller.enqueue(`data: ${JSON.stringify(chunk)}\n\n`); } + + controller.close(); } iterate().catch((err) => { From d7c501a24534f12166e20cb47a117c037100a8b0 Mon Sep 17 00:00:00 2001 From: KATT Date: Fri, 13 Oct 2023 16:49:52 +0200 Subject: [PATCH 11/17] cool --- examples/sse/src/index.html | 30 +++++++++-------- package.json | 1 + pnpm-lock.yaml | 7 ++++ src/async/serializeAsync.ts | 5 ++- src/async/sse.test.ts | 65 ++++++++++++++++++------------------- src/internals/testUtils.ts | 1 - 6 files changed, 60 insertions(+), 49 deletions(-) diff --git a/examples/sse/src/index.html b/examples/sse/src/index.html index aec9b317..88aa0c37 100644 --- a/examples/sse/src/index.html +++ b/examples/sse/src/index.html @@ -1,16 +1,18 @@ -

SSE test

+ +

SSE test

-See log output / inspector tools + See log output / inspector tools - + + diff --git a/package.json b/package.json index 580b61bb..1fe6d810 100644 --- a/package.json +++ b/package.json @@ -61,6 +61,7 @@ "eslint-plugin-regexp": "^1.15.0", "eslint-plugin-vitest": "^0.3.1", "eslint-plugin-yml": "^1.9.0", + "event-source-polyfill": "^1.0.31", "jsonc-eslint-parser": "^2.3.0", "knip": "^2.31.0", "markdownlint": "^0.31.1", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 5e01b61c..aca1712a 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -77,6 +77,9 @@ importers: eslint-plugin-yml: specifier: ^1.9.0 version: 1.9.0(eslint@8.50.0) + event-source-polyfill: + specifier: ^1.0.31 + version: 1.0.31 jsonc-eslint-parser: specifier: ^2.3.0 version: 2.3.0 @@ -3138,6 +3141,10 @@ packages: engines: {node: '>=0.10.0'} dev: true + /event-source-polyfill@1.0.31: + resolution: {integrity: sha512-4IJSItgS/41IxN5UVAVuAyczwZF7ZIEsM1XAoUzIHA6A+xzusEZUutdXz2Nr+MQPLxfTiCvqE79/C8HT8fKFvA==} + dev: true + /event-target-shim@5.0.1: resolution: {integrity: sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==} engines: {node: '>=6'} diff --git a/src/async/serializeAsync.ts b/src/async/serializeAsync.ts index 5c2d3c94..37b65257 100644 --- a/src/async/serializeAsync.ts +++ b/src/async/serializeAsync.ts @@ -12,6 +12,7 @@ import { TsonTypeTesterCustom, TsonTypeTesterPrimitive, } from "../sync/syncTypes.js"; +import { TsonStreamInterruptedError } from "./asyncErrors.js"; import { BrandSerialized, TsonAsyncIndex, @@ -268,7 +269,9 @@ export function createTsonSSEResponse(opts: TsonAsyncOptions) { controller.enqueue(`data: ${JSON.stringify(chunk)}\n\n`); } - controller.close(); + controller.error( + new TsonStreamInterruptedError(new Error("SSE stream ended")), + ); } iterate().catch((err) => { diff --git a/src/async/sse.test.ts b/src/async/sse.test.ts index 84f16133..7879e6bb 100644 --- a/src/async/sse.test.ts +++ b/src/async/sse.test.ts @@ -1,34 +1,19 @@ -import { expect, test, vitest } from "vitest"; +/* eslint-disable @typescript-eslint/no-unnecessary-condition */ +import { EventSourcePolyfill, NativeEventSource } from "event-source-polyfill"; +import { expect, test } from "vitest"; +global.EventSource = NativeEventSource || EventSourcePolyfill; -import { - TsonAsyncOptions, - TsonParseAsyncOptions, - TsonType, - createTsonParseAsync, - tsonAsyncIterable, - tsonBigint, - tsonPromise, -} from "../index.js"; -import { assert } from "../internals/assert.js"; -import { - createDeferred, - createTestServer, - sleep, - waitError, - waitFor, -} from "../internals/testUtils.js"; -import { TsonSerialized } from "../sync/syncTypes.js"; +import { TsonAsyncOptions, tsonAsyncIterable, tsonPromise } from "../index.js"; +import { createTestServer, sleep } from "../internals/testUtils.js"; import { createTsonAsync } from "./createTsonAsync.js"; -import { mapIterable, readableStreamToAsyncIterable } from "./iterableUtils.js"; test("SSE response test", async () => { function createMockObj() { async function* generator() { - for (let i = 0; i < 10; i++) { - yield i; - await sleep(1); - - await sleep(1); + let i = 0; + while (true) { + yield i++; + await sleep(100); } } @@ -44,11 +29,12 @@ test("SSE response test", async () => { // ------------- server ------------------- const opts = { + nonce: () => "__tson", types: [tsonPromise, tsonAsyncIterable], } satisfies TsonAsyncOptions; const server = await createTestServer({ - handleRequest: async (_req, res) => { + handleRequest: async (req, res) => { const tson = createTsonAsync(opts); const obj = createMockObj(); @@ -72,14 +58,27 @@ test("SSE response test", async () => { // do a streamed fetch request const sse = new EventSource(server.url); - let messages: MessageEvent[]; - await new Promise(() => { + const messages: MessageEvent["data"][] = []; + await new Promise((resolve) => { sse.onmessage = (msg) => { - messages.push(msg); - }; + // console.log(sse.readyState); + // console.log({ msg }); + messages.push(msg.data); - sse.addEventListener("error", () => { - console.error("error"); - }); + if (messages.length === 5) { + sse.close(); + resolve(); + } + }; }); + + expect(messages).toMatchInlineSnapshot(` + [ + "{\\"json\\":{\\"foo\\":\\"bar\\",\\"iterable\\":[\\"AsyncIterable\\",0,\\"__tson\\"],\\"promise\\":[\\"Promise\\",1,\\"__tson\\"],\\"rejectedPromise\\":[\\"Promise\\",2,\\"__tson\\"]},\\"nonce\\":\\"__tson\\"}", + "[0,[0,0]]", + "[1,[0,42]]", + "[2,[1,{}]]", + "[0,[0,1]]", + ] + `); }); diff --git a/src/internals/testUtils.ts b/src/internals/testUtils.ts index 2fa6c2ea..f8e07362 100644 --- a/src/internals/testUtils.ts +++ b/src/internals/testUtils.ts @@ -51,7 +51,6 @@ export async function createTestServer(opts: { const server = http.createServer((req, res) => { Promise.resolve(opts.handleRequest(req, res)).catch((err) => { console.error(err); - res.statusCode = 500; res.end(); }); }); From 19e6dcdcee97985b1644bdc75c7490bf283e9135 Mon Sep 17 00:00:00 2001 From: KATT Date: Fri, 13 Oct 2023 17:17:41 +0200 Subject: [PATCH 12/17] wip --- src/async/deserializeAsync.ts | 42 +++++++++++++++++++++++++++++++++++ src/async/iterableUtils.ts | 16 +++++++++++++ src/async/sse.test.ts | 6 +++-- 3 files changed, 62 insertions(+), 2 deletions(-) diff --git a/src/async/deserializeAsync.ts b/src/async/deserializeAsync.ts index 0a270a08..7fb3e856 100644 --- a/src/async/deserializeAsync.ts +++ b/src/async/deserializeAsync.ts @@ -11,11 +11,17 @@ import { } from "../sync/syncTypes.js"; import { TsonStreamInterruptedError } from "./asyncErrors.js"; import { + BrandSerialized, TsonAsyncIndex, TsonAsyncOptions, TsonAsyncStringifierIterable, TsonAsyncType, } from "./asyncTypes.js"; +import { + createReadableStream, + mapIterable, + readableStreamToAsyncIterable, +} from "./iterableUtils.js"; import { TsonAsyncValueTuple } from "./serializeAsync.js"; type WalkFn = (value: unknown) => unknown; @@ -259,3 +265,39 @@ export function createTsonParseAsync(opts: TsonAsyncOptions): TsonParseAsync { return await instance(tsonIterable, opts ?? {}); }) as TsonParseAsync; } + +// export function createTsonParseEventSource(opts: TsonAsyncOptions) { +// const instance = createTsonDeserializer(opts); + +// return async ( +// url: string, +// parseOpts?: TsonParseAsyncOptions & { +// abortSignal: AbortSignal; +// }, +// ) => { +// const [stream, controller] = createReadableStream(); +// const eventSource = new EventSource(url); + +// const onAbort = () => { +// eventSource.close(); +// controller.close(); +// parseOpts?.abortSignal.removeEventListener("abort", onAbort); +// }; + +// parseOpts?.abortSignal.addEventListener("abort", onAbort); + +// eventSource.onmessage = (msg) => { +// // eslint-disable-next-line @typescript-eslint/no-unsafe-argument +// controller.enqueue(msg.data); +// }; + +// const iterable = mapIterable( +// readableStreamToAsyncIterable(stream), +// (msg) => { +// return JSON.parse(msg) as TsonAsyncValueTuple | TsonSerialized; +// }, +// ); + +// return (await instance(iterable, parseOpts ?? {})) as TValue; +// }; +// } diff --git a/src/async/iterableUtils.ts b/src/async/iterableUtils.ts index 27850c95..bcddad4f 100644 --- a/src/async/iterableUtils.ts +++ b/src/async/iterableUtils.ts @@ -1,3 +1,5 @@ +import { assert } from "../internals/assert"; + export async function* readableStreamToAsyncIterable( stream: ReadableStream, ): AsyncIterable { @@ -31,3 +33,17 @@ export async function* mapIterable( yield fn(value); } } + +export function createReadableStream() { + let controller: ReadableStreamDefaultController = + null as unknown as ReadableStreamDefaultController; + const stream = new ReadableStream({ + start(c) { + controller = c; + }, + }); + + assert(controller, `Could not find controller - this is a bug`); + + return [stream, controller] as const; +} diff --git a/src/async/sse.test.ts b/src/async/sse.test.ts index 7879e6bb..27785561 100644 --- a/src/async/sse.test.ts +++ b/src/async/sse.test.ts @@ -25,7 +25,7 @@ test("SSE response test", async () => { }; } - type MockObj = ReturnType; + // type MockObj = ReturnType; // ------------- server ------------------- const opts = { @@ -53,7 +53,7 @@ test("SSE response test", async () => { }); // ------------- client ------------------- - const tson = createTsonAsync(opts); + // const tson = createTsonAsync(opts); // do a streamed fetch request const sse = new EventSource(server.url); @@ -82,3 +82,5 @@ test("SSE response test", async () => { ] `); }); + +test.todo("parse SSE response"); From 46e26c2fe534d9aebc0ca1d1b3d1f0943781eee0 Mon Sep 17 00:00:00 2001 From: KATT Date: Fri, 13 Oct 2023 17:22:48 +0200 Subject: [PATCH 13/17] cool --- examples/sse/package.json | 25 ---------- examples/sse/src/index.html | 18 ------- examples/sse/src/server.ts | 91 ----------------------------------- examples/sse/src/shared.ts | 21 -------- examples/sse/tsconfig.json | 11 ----- pnpm-lock.yaml | 7 +++ src/async/createTsonAsync.ts | 3 +- src/async/deserializeAsync.ts | 16 ++---- src/async/iterableUtils.ts | 4 +- src/async/serializeAsync.ts | 9 +--- src/async/sse.test.ts | 2 +- 11 files changed, 16 insertions(+), 191 deletions(-) delete mode 100644 examples/sse/package.json delete mode 100644 examples/sse/src/index.html delete mode 100644 examples/sse/src/server.ts delete mode 100644 examples/sse/src/shared.ts delete mode 100644 examples/sse/tsconfig.json diff --git a/examples/sse/package.json b/examples/sse/package.json deleted file mode 100644 index 5179843a..00000000 --- a/examples/sse/package.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "name": "@examples/sse", - "version": "10.38.5", - "private": true, - "description": "An example project for tupleson", - "license": "MIT", - "module": "module", - "workspaces": [ - "client", - "server" - ], - "scripts": { - "build": "tsc", - "dev:server": "tsx watch src/server", - "dev": "run-p dev:* --print-label" - }, - "devDependencies": { - "@types/node": "^18.16.16", - "npm-run-all": "^4.1.5", - "tsx": "^3.12.7", - "tupleson": "latest", - "typescript": "^5.1.3", - "wait-port": "^1.0.1" - } -} diff --git a/examples/sse/src/index.html b/examples/sse/src/index.html deleted file mode 100644 index 88aa0c37..00000000 --- a/examples/sse/src/index.html +++ /dev/null @@ -1,18 +0,0 @@ - -

SSE test

- - See log output / inspector tools - - - diff --git a/examples/sse/src/server.ts b/examples/sse/src/server.ts deleted file mode 100644 index 15dbfd06..00000000 --- a/examples/sse/src/server.ts +++ /dev/null @@ -1,91 +0,0 @@ -import fs from "node:fs"; -import http from "node:http"; -import { createTsonSSEResponse } from "tupleson"; - -import { tsonOptions } from "./shared.js"; - -const createResponse = createTsonSSEResponse(tsonOptions); - -const randomNumber = (min: number, max: number) => - Math.floor(Math.random() * (max - min + 1) + min); - -const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); - -/** - * This function returns the object we will be sending to the client. - */ -export function getResponseShape() { - async function* bigintGenerator() { - const iterate = new Array(10).fill(0).map((_, i) => BigInt(i)); - for (const number of iterate) { - await sleep(randomNumber(1, 400)); - yield number; - } - } - - async function* numberGenerator() { - const iterate = new Array(10).fill(0).map((_, i) => i); - for (const number of iterate) { - await sleep(randomNumber(1, 400)); - yield number; - } - } - - return { - bigints: bigintGenerator(), - foo: "bar", - numbers: numberGenerator(), - promise: new Promise((resolve) => - setTimeout(() => { - resolve(42); - }, 1), - ), - rejectedPromise: new Promise((_, reject) => - setTimeout(() => { - reject(new Error("Rejected promise")); - }, 1), - ), - }; -} - -export type ResponseShape = ReturnType; -async function handleRequest( - req: http.IncomingMessage, - res: http.ServerResponse, -) { - if (req.url?.startsWith("/sse")) { - const obj = getResponseShape(); - const response = createResponse(obj); - - // Stream the response to the client - for (const [key, value] of response.headers) { - res.setHeader(key, value); - } - - for await (const value of response.body as any) { - res.write(value); - } - - res.end(); - return; - } - - const data = fs.readFileSync(__dirname + "/index.html"); - res.write(data.toString()); - res.end(); -} - -const server = http.createServer( - (req: http.IncomingMessage, res: http.ServerResponse) => { - handleRequest(req, res).catch((err) => { - console.error(err); - res.writeHead(500, { "Content-Type": "text/plain" }); - res.end("Internal Server Error\n"); - }); - }, -); - -const port = 3000; -server.listen(port); - -console.log(`Server running at http://localhost:${port}`); diff --git a/examples/sse/src/shared.ts b/examples/sse/src/shared.ts deleted file mode 100644 index cc973f9d..00000000 --- a/examples/sse/src/shared.ts +++ /dev/null @@ -1,21 +0,0 @@ -import { - TsonAsyncOptions, - tsonAsyncIterable, - tsonBigint, - tsonPromise, -} from "tupleson"; - -/** - * Shared tupleSON options for the server and client. - */ -export const tsonOptions: TsonAsyncOptions = { - // We need to specify the types we want to allow serialization of - types: [ - // Allow serialization of promises - tsonPromise, - // Allow serialization of async iterators - tsonAsyncIterable, - // Allow serialization of bigints - tsonBigint, - ], -}; diff --git a/examples/sse/tsconfig.json b/examples/sse/tsconfig.json deleted file mode 100644 index 95c6bc3b..00000000 --- a/examples/sse/tsconfig.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "compilerOptions": { - "esModuleInterop": true, - "module": "esnext", - "moduleResolution": "node", - "outDir": "dist", - "strict": true, - "target": "esnext" - }, - "include": ["src"] -} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index aca1712a..6fd40ef6 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -23,6 +23,9 @@ importers: '@types/eslint': specifier: ^8.44.3 version: 8.44.3 + '@types/event-source-polyfill': + specifier: ^1.0.2 + version: 1.0.2 '@typescript-eslint/eslint-plugin': specifier: ^6.7.3 version: 6.7.3(@typescript-eslint/parser@6.7.3)(eslint@8.50.0)(typescript@5.2.2) @@ -1238,6 +1241,10 @@ packages: resolution: {integrity: sha512-VeiPZ9MMwXjO32/Xu7+OwflfmeoRwkE/qzndw42gGtgJwZopBnzy2gD//NN1+go1mADzkDcqf/KnFRSjTJ8xJA==} dev: true + /@types/event-source-polyfill@1.0.2: + resolution: {integrity: sha512-qE5zrFd73BRs5oSjVys6g/5GboqOMbzLRTUFPAhfULvvvbRAOXw9m4Wk+p1BtoZm4JgW7TljGGfVabBqvi3eig==} + dev: true + /@types/http-cache-semantics@4.0.2: resolution: {integrity: sha512-FD+nQWA2zJjh4L9+pFXqWOi0Hs1ryBCfI+985NjluQ1p8EYtoLvjLOKidXBtZ4/IcxDX4o8/E8qDS3540tNliw==} dev: true diff --git a/src/async/createTsonAsync.ts b/src/async/createTsonAsync.ts index c23c490c..7009ccac 100644 --- a/src/async/createTsonAsync.ts +++ b/src/async/createTsonAsync.ts @@ -6,9 +6,8 @@ import { } from "./serializeAsync.js"; /** - * @internal - * * Only used for testing - when using the async you gotta pick which one you want + * @internal */ export const createTsonAsync = (opts: TsonAsyncOptions) => ({ parseJsonStream: createTsonParseAsync(opts), diff --git a/src/async/deserializeAsync.ts b/src/async/deserializeAsync.ts index 7fb3e856..bad12800 100644 --- a/src/async/deserializeAsync.ts +++ b/src/async/deserializeAsync.ts @@ -11,17 +11,12 @@ import { } from "../sync/syncTypes.js"; import { TsonStreamInterruptedError } from "./asyncErrors.js"; import { - BrandSerialized, TsonAsyncIndex, TsonAsyncOptions, TsonAsyncStringifierIterable, TsonAsyncType, } from "./asyncTypes.js"; -import { - createReadableStream, - mapIterable, - readableStreamToAsyncIterable, -} from "./iterableUtils.js"; +import { createReadableStream } from "./iterableUtils.js"; import { TsonAsyncValueTuple } from "./serializeAsync.js"; type WalkFn = (value: unknown) => unknown; @@ -85,13 +80,8 @@ export function createTsonDeserializer(opts: TsonAsyncOptions) { const idx = serializedValue as TsonAsyncIndex; - let controller: ReadableStreamDefaultController = - null as unknown as ReadableStreamDefaultController; - const readable = new ReadableStream({ - start(c) { - controller = c; - }, - }); + const [readable, controller] = createReadableStream(); + // the `start` method is called "immediately when the object is constructed" // [MDN](http://developer.mozilla.org/en-US/docs/Web/API/ReadableStream/ReadableStream) // so we're guaranteed that the controller is set in the cache diff --git a/src/async/iterableUtils.ts b/src/async/iterableUtils.ts index bcddad4f..a820ad66 100644 --- a/src/async/iterableUtils.ts +++ b/src/async/iterableUtils.ts @@ -1,4 +1,4 @@ -import { assert } from "../internals/assert"; +import { assert } from "../internals/assert.js"; export async function* readableStreamToAsyncIterable( stream: ReadableStream, @@ -34,7 +34,7 @@ export async function* mapIterable( } } -export function createReadableStream() { +export function createReadableStream() { let controller: ReadableStreamDefaultController = null as unknown as ReadableStreamDefaultController; const stream = new ReadableStream({ diff --git a/src/async/serializeAsync.ts b/src/async/serializeAsync.ts index 37b65257..df26b401 100644 --- a/src/async/serializeAsync.ts +++ b/src/async/serializeAsync.ts @@ -19,6 +19,7 @@ import { TsonAsyncOptions, TsonAsyncStringifier, } from "./asyncTypes.js"; +import { createReadableStream } from "./iterableUtils.js"; type WalkFn = (value: unknown) => unknown; @@ -253,13 +254,7 @@ export function createTsonSSEResponse(opts: TsonAsyncOptions) { const serialize = createAsyncTsonSerialize(opts); return (value: TValue) => { - let controller: ReadableStreamDefaultController = - null as unknown as ReadableStreamDefaultController; - const readable = new ReadableStream({ - start(c) { - controller = c; - }, - }); + const [readable, controller] = createReadableStream(); async function iterate() { const [head, iterable] = serialize(value); diff --git a/src/async/sse.test.ts b/src/async/sse.test.ts index 27785561..a380aa2a 100644 --- a/src/async/sse.test.ts +++ b/src/async/sse.test.ts @@ -1,7 +1,7 @@ /* eslint-disable @typescript-eslint/no-unnecessary-condition */ import { EventSourcePolyfill, NativeEventSource } from "event-source-polyfill"; import { expect, test } from "vitest"; -global.EventSource = NativeEventSource || EventSourcePolyfill; +(global as any).EventSource = NativeEventSource || EventSourcePolyfill; import { TsonAsyncOptions, tsonAsyncIterable, tsonPromise } from "../index.js"; import { createTestServer, sleep } from "../internals/testUtils.js"; From edb1bb0782c17bb0604e02af6fee65d0aa6f9ccb Mon Sep 17 00:00:00 2001 From: KATT Date: Fri, 13 Oct 2023 17:24:16 +0200 Subject: [PATCH 14/17] fix --- pnpm-lock.yaml | 28 ---------------------------- 1 file changed, 28 deletions(-) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 6fd40ef6..b06109a0 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -23,9 +23,6 @@ importers: '@types/eslint': specifier: ^8.44.3 version: 8.44.3 - '@types/event-source-polyfill': - specifier: ^1.0.2 - version: 1.0.2 '@typescript-eslint/eslint-plugin': specifier: ^6.7.3 version: 6.7.3(@typescript-eslint/parser@6.7.3)(eslint@8.50.0)(typescript@5.2.2) @@ -174,27 +171,6 @@ importers: specifier: ^1.0.1 version: 1.1.0 - examples/sse: - devDependencies: - '@types/node': - specifier: ^18.16.16 - version: 18.18.3 - npm-run-all: - specifier: ^4.1.5 - version: 4.1.5 - tsx: - specifier: ^3.12.7 - version: 3.13.0 - tupleson: - specifier: link:../.. - version: link:../.. - typescript: - specifier: ^5.1.3 - version: 5.2.2 - wait-port: - specifier: ^1.0.1 - version: 1.1.0 - packages: /@aashutoshrathi/word-wrap@1.2.6: @@ -1241,10 +1217,6 @@ packages: resolution: {integrity: sha512-VeiPZ9MMwXjO32/Xu7+OwflfmeoRwkE/qzndw42gGtgJwZopBnzy2gD//NN1+go1mADzkDcqf/KnFRSjTJ8xJA==} dev: true - /@types/event-source-polyfill@1.0.2: - resolution: {integrity: sha512-qE5zrFd73BRs5oSjVys6g/5GboqOMbzLRTUFPAhfULvvvbRAOXw9m4Wk+p1BtoZm4JgW7TljGGfVabBqvi3eig==} - dev: true - /@types/http-cache-semantics@4.0.2: resolution: {integrity: sha512-FD+nQWA2zJjh4L9+pFXqWOi0Hs1ryBCfI+985NjluQ1p8EYtoLvjLOKidXBtZ4/IcxDX4o8/E8qDS3540tNliw==} dev: true From 94ab818ac9ba2bf8f6b0454a941704713509d355 Mon Sep 17 00:00:00 2001 From: KATT Date: Fri, 13 Oct 2023 17:25:06 +0200 Subject: [PATCH 15/17] fix --- src/async/deserializeAsync.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/async/deserializeAsync.ts b/src/async/deserializeAsync.ts index bad12800..08627dbf 100644 --- a/src/async/deserializeAsync.ts +++ b/src/async/deserializeAsync.ts @@ -41,7 +41,7 @@ type TsonParseAsync = ( type TsonDeserializeIterable = AsyncIterable< TsonAsyncValueTuple | TsonSerialized >; -export function createTsonDeserializer(opts: TsonAsyncOptions) { +function createTsonDeserializer(opts: TsonAsyncOptions) { const typeByKey: Record = {}; for (const handler of opts.types) { From 14f44dff566575c5e6b8a1195460b1abda19a9cc Mon Sep 17 00:00:00 2001 From: KATT Date: Fri, 13 Oct 2023 17:26:12 +0200 Subject: [PATCH 16/17] cool --- src/index.test.ts | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/index.test.ts b/src/index.test.ts index f17c20d4..5cacd798 100644 --- a/src/index.test.ts +++ b/src/index.test.ts @@ -1,6 +1,7 @@ import { expect, test } from "vitest"; -import { TsonOptions, TsonType, createTson, createTsonAsync } from "./index.js"; +import { createTsonAsync } from "./async/createTsonAsync.js"; +import { TsonOptions, TsonType, createTson } from "./index.js"; import { expectError, waitError } from "./internals/testUtils.js"; test("multiple handlers for primitive string found", () => { @@ -88,7 +89,7 @@ test("async: duplicate keys", async () => { const gen = generator(); await createTsonAsync({ types: [stringHandler, stringHandler], - }).parse(gen); + }).parseJsonStream(gen); }); expect(err).toMatchInlineSnapshot( @@ -104,7 +105,7 @@ test("async: multiple handlers for primitive string found", async () => { const err = await waitError(async () => { const iterator = createTsonAsync({ types: [stringHandler, stringHandler], - }).stringify({}); + }).stringifyJsonStream({}); // eslint-disable-next-line @typescript-eslint/no-unused-vars for await (const _ of iterator) { @@ -129,7 +130,7 @@ test("async: bad init", async () => { const gen = generator(); await createTsonAsync({ types: [], - }).parse(gen); + }).parseJsonStream(gen); }); expect(err).toMatchInlineSnapshot( From 79b1676f900a0f60caf8ef752ddc25c89195500d Mon Sep 17 00:00:00 2001 From: KATT Date: Fri, 13 Oct 2023 17:29:16 +0200 Subject: [PATCH 17/17] fix --- package.json | 1 + pnpm-lock.yaml | 7 +++++++ src/async/sse.test.ts | 2 +- 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/package.json b/package.json index 1fe6d810..ae8f1e95 100644 --- a/package.json +++ b/package.json @@ -43,6 +43,7 @@ "@release-it/conventional-changelog": "^7.0.2", "@tsconfig/strictest": "^2.0.2", "@types/eslint": "^8.44.3", + "@types/event-source-polyfill": "^1.0.2", "@typescript-eslint/eslint-plugin": "^6.7.3", "@typescript-eslint/parser": "^6.7.3", "@vitest/coverage-v8": "^0.34.6", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index b06109a0..b049df88 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -23,6 +23,9 @@ importers: '@types/eslint': specifier: ^8.44.3 version: 8.44.3 + '@types/event-source-polyfill': + specifier: ^1.0.2 + version: 1.0.2 '@typescript-eslint/eslint-plugin': specifier: ^6.7.3 version: 6.7.3(@typescript-eslint/parser@6.7.3)(eslint@8.50.0)(typescript@5.2.2) @@ -1217,6 +1220,10 @@ packages: resolution: {integrity: sha512-VeiPZ9MMwXjO32/Xu7+OwflfmeoRwkE/qzndw42gGtgJwZopBnzy2gD//NN1+go1mADzkDcqf/KnFRSjTJ8xJA==} dev: true + /@types/event-source-polyfill@1.0.2: + resolution: {integrity: sha512-qE5zrFd73BRs5oSjVys6g/5GboqOMbzLRTUFPAhfULvvvbRAOXw9m4Wk+p1BtoZm4JgW7TljGGfVabBqvi3eig==} + dev: true + /@types/http-cache-semantics@4.0.2: resolution: {integrity: sha512-FD+nQWA2zJjh4L9+pFXqWOi0Hs1ryBCfI+985NjluQ1p8EYtoLvjLOKidXBtZ4/IcxDX4o8/E8qDS3540tNliw==} dev: true diff --git a/src/async/sse.test.ts b/src/async/sse.test.ts index a380aa2a..b21336f4 100644 --- a/src/async/sse.test.ts +++ b/src/async/sse.test.ts @@ -34,7 +34,7 @@ test("SSE response test", async () => { } satisfies TsonAsyncOptions; const server = await createTestServer({ - handleRequest: async (req, res) => { + handleRequest: async (_req, res) => { const tson = createTsonAsync(opts); const obj = createMockObj();