feat: sync back-references

benchmark/index.js

@@ -1,11 +1,15 @@
 // taken from https://github.com/Rich-Harris/superjson-and-devalue
 
 import ARSON from "arson";
-import { parse, stringify, uneval } from "devalue";
+import * as devalue from "devalue";
 import c from "kleur";
 import * as superjson from "superjson";
 import { createTson, tsonDate, tsonRegExp, tsonSet } from "tupleson";
 
+const time_formatter = new Intl.NumberFormat('en-US', { unit: 'millisecond', style: 'unit' });
+const size_formatter = new Intl.NumberFormat('en-US', { unit: 'byte', style: 'unit' });
+const number_formatter = new Intl.NumberFormat('en-US');
+
 const obj = {
 	array: [{ foo: 1 }, { bar: 2 }, { baz: 3 }],
 	date: new Date(),
@@ -16,35 +20,37 @@ const obj = {
 };
 
 // circular references are not supported by tupleson
-// obj.self = obj;
+obj.self = obj;
 
 const tson = createTson({
 	types: [tsonDate, tsonRegExp, tsonSet],
 });
 
 const superjson_serialized = superjson.stringify(obj);
-const devalue_unevaled = uneval(obj);
-const devalue_stringified = stringify(obj);
+const devalue_unevaled = devalue.uneval(obj);
+const devalue_stringified = devalue.stringify(obj);
 const arson_stringified = ARSON.stringify(obj);
 const tson_serialized = tson.stringify(obj);
 
+console.log('-- SERIALIZED SIZE --\n')
+
 console.log(
-	`superjson output: ${c.bold().cyan(superjson_serialized.length)} bytes`,
+	`superjson output: ${c.bold().cyan(size_formatter.format(superjson_serialized.length))}`,
 );
 
-console.log(`tson output: ${c.bold().cyan(tson_serialized.length)} bytes`);
+console.log(`tson output: ${c.bold().cyan(size_formatter.format(tson_serialized.length))}`);
 // console.log(superjson_serialized);
 console.log(
-	`devalue.uneval output: ${c.bold().cyan(devalue_unevaled.length)} bytes`,
+	`devalue.uneval output: ${c.bold().cyan(size_formatter.format(devalue_unevaled.length))}`,
 );
 // console.log(devalue_unevaled);
 console.log(
 	`devalue.stringify output: ${c
 		.bold()
-		.cyan(devalue_stringified.length)} bytes`,
+		.cyan(size_formatter.format(devalue_stringified.length))}`,
 );
 // console.log(devalue_stringified);
-console.log(`arson output: ${c.bold().cyan(arson_stringified.length)} bytes`);
+console.log(`arson output: ${c.bold().cyan(size_formatter.format(arson_stringified.length))}`);
 // console.log(arson_stringified);
 
 // const superjson_deserialized = superjson.parse(superjson_serialized);
@@ -53,31 +59,46 @@ console.log(`arson output: ${c.bold().cyan(arson_stringified.length)} bytes`);
 const iterations = 1e6;
 
 function test(fn, label = fn.toString()) {
-	const start = Date.now();
 	console.log();
 	console.log(c.bold(label));
+	global.gc(); // force garbage collection before each test
 	let i = iterations;
+	const before_snap = process.memoryUsage();
+	const start = Date.now();
 	while (i--) {
 		fn();
 	}
-
+	const delta = Date.now() - start;
+	const after_snap = process.memoryUsage();
 	console.log(
-		`${iterations} iterations in ${c.bold().cyan(Date.now() - start)}ms`,
+		`${number_formatter.format(iterations)} iterations in ${c.bold().cyan(time_formatter.format(delta))}`,
 	);
+	// log memory usage delta
+	for (const key in after_snap) {
+		const before = before_snap[key];
+		const after = after_snap[key];
+		const diff = after - before;
+		const color = diff < 0 ? c.green : c.red;
+		console.log(`  ${key}: ${color(size_formatter.format(diff))}`);
+	}
 }
 
+console.log('\n-- SERIALIZATION DURATION --')
+
 // serialization
 test(() => superjson.stringify(obj));
 test(() => tson.stringify(obj));
-test(() => uneval(obj));
-test(() => stringify(obj));
+test(() => devalue.uneval(obj));
+test(() => devalue.stringify(obj));
 test(() => ARSON.stringify(obj));
 
+console.log('\n-- DESERIALIZATION DURATION --')
+
 // deserialization
 test(() => superjson.parse(superjson_serialized));
 test(() => tson.parse(tson_serialized));
 test(() => eval(`(${devalue_unevaled})`));
 test(() => ARSON.parse(arson_stringified));
-test(() => parse(devalue_stringified));
+test(() => devalue.parse(devalue_stringified));
 
 console.log();

benchmark/package.json

@@ -7,7 +7,7 @@
 	"type": "module",
 	"scripts": {
 		"postinstall": "cd ../ && pnpm run build",
-		"start": "node index.js"
+		"start": "node --expose-gc --max-old-space-size=8192 index.js"
 	},
 	"dependencies": {
 		"arson": "^0.2.6",

src/index.test.ts

@@ -1,6 +1,6 @@
 import { expect, test } from "vitest";
 
-import { TsonOptions, TsonType, createTson, createTsonAsync } from "./index.js";
+import { TsonOptions, TsonType, createTson, createTsonAsync, tsonDate, tsonPromise } from "./index.js";
 import { expectError, waitError } from "./internals/testUtils.js";
 
 test("multiple handlers for primitive string found", () => {
@@ -33,20 +33,77 @@ test("duplicate keys", () => {
 	);
 });
 
-test("no max call stack", () => {
+test("back-reference: circular object reference", () => {
 	const t = createTson({
 		types: [],
 	});
 
 	const expected: Record<string, unknown> = {};
 	expected["a"] = expected;
+	expected["b"] = expected;
 
-	// stringify should fail b/c of JSON limitations
-	const err = expectError(() => t.stringify(expected));
+	const str = t.stringify(expected)
+	const res = t.parse(str);
 
-	expect(err.message).toMatchInlineSnapshot('"Circular reference detected"');
+	expect(res).toEqual(expected);
 });
 
+test("back-reference: circular array reference", () => {
+	const t = createTson({
+		types: [],
+	});
+
+	const expected: unknown[] = [];
+	expected[0] = expected;
+	expected[1] = expected;
+
+	const str = t.stringify(expected)
+	const res = t.parse(str);
+
+	expect(res).toEqual(expected);
+});
+
+test("back-reference: non-circular complex reference", () => {
+	const t = createTson({
+		types: [tsonDate],
+	});
+
+	const expected: Record<string, unknown> = {};
+	expected["a"] = {}
+	expected["b"] = expected["a"]
+	expected["c"] = new Date()
+	expected["d"] = expected["c"]
+
+	const str = t.stringify(expected)
+	const res = t.parse(str);
+
+	expect(res["b"]).toBe(res["a"]);
+	expect(res["d"]).toBe(res["c"]);
+});
+
+/**
+ * WILL NOT WORK: the async serialize/deserialize functions haven't
+ * been adapted to handle back-references yet
+ */
+// test("async: back-reference", async () => {
+// 	const t = createTsonAsync({
+// 		types: [tsonPromise],
+// 	});
+
+// 	const needle = {}
+
+// 	const expected = {
+// 		a: needle,
+// 		b: Promise.resolve(needle),
+// 	};
+
+// 	const str = await t.stringify(expected);
+// 	const res = await t.parse(str);
+
+// 	expect(res).toEqual(expected);
+// 	expect(res.a).toBe(await res.b);
+// })
+
 test("allow duplicate objects", () => {
 	const t = createTson({
 		types: [],

src/sync/deserialize.ts

@@ -9,7 +9,7 @@ import {
 	TsonTransformerSerializeDeserialize,
 } from "./syncTypes.js";
 
-type WalkFn = (value: unknown) => unknown;
+type WalkFn = (value: unknown, path?: (string|number)[]) => unknown;
 type WalkerFactory = (nonce: TsonNonce) => WalkFn;
 
 type AnyTsonTransformerSerializeDeserialize =
@@ -30,17 +30,52 @@ export function createTsonDeserialize(opts: TsonOptions): TsonDeserializeFn {
 	}
 
 	const walker: WalkerFactory = (nonce) => {
-		const walk: WalkFn = (value) => {
+		const seen = new Map<string, unknown>();
+		const backrefs: [circular_key: string, origin_key: string][] = [];
+
+		const coreWalk: WalkFn = (value, path = []) => {
+			const key = path.join(nonce);
 			if (isTsonTuple(value, nonce)) {
 				const [type, serializedValue] = value;
+				if (type === 'CIRCULAR') {
+					backrefs.push([key, serializedValue as string]);
+					return;
+				}
 				// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
 				const transformer = typeByKey[type]!;
-				return transformer.deserialize(walk(serializedValue));
+				const parsed = transformer.deserialize(coreWalk(serializedValue, path));
+				seen.set(key, parsed);
+				return parsed;
 			}
 
-			return mapOrReturn(value, walk);
+			const parsed = mapOrReturn(value, (value, key) => coreWalk(value, [...path, key]));
+			if (parsed && typeof parsed === 'object') {
+				seen.set(key, parsed)
+			}
+			return parsed;
 		};
 
+		const walk: WalkFn = (value) => {
+			const res = coreWalk(value);
+			for (const [key, ref] of backrefs) {
+				const prev = seen.get(ref);
+				if (!prev) {
+					throw new Error(`Back-reference ${ref.split(nonce).join('.')} not found`);
+				}
+				const path = key.split(nonce);
+				let insertAt = res as any
+				try {
+					while (path.length > 1) {
+						insertAt = insertAt[path.shift()!];
+					}
+					insertAt[path[0]!] = prev
+				} catch (cause) {
+					throw new Error(`Invalid path to back-reference ${ref.split(nonce).join('.')}`, { cause });
+				}
+			}
+			return res
+		}
+
 		return walk;
 	};
 

src/sync/serialize.ts

@@ -1,4 +1,4 @@
-import { TsonCircularReferenceError } from "../errors.js";
+// import { TsonCircularReferenceError } from "../errors.js";
 import { GetNonce, getDefaultNonce } from "../internals/getNonce.js";
 import { mapOrReturn } from "../internals/mapOrReturn.js";
 import {
@@ -13,7 +13,7 @@ import {
 	TsonTypeTesterPrimitive,
 } from "./syncTypes.js";
 
-type WalkFn = (value: unknown) => unknown;
+type WalkFn = (value: unknown, path?: (string|number)[]) => unknown;
 type WalkerFactory = (nonce: TsonNonce) => WalkFn;
 
 function getHandlers(opts: TsonOptions) {
@@ -56,26 +56,13 @@ export function createTsonSerialize(opts: TsonOptions): TsonSerializeFn {
 	const [getNonce, nonPrimitive, byPrimitive] = getHandlers(opts);
 
 	const walker: WalkerFactory = (nonce) => {
-		const seen = new WeakSet();
+		const seen = new WeakMap<object, (string|number)[]>();
 		const cache = new WeakMap<object, unknown>();
 
-		const walk: WalkFn = (value) => {
+		const walk: WalkFn = (value, path = []) => {
 			const type = typeof value;
 			const isComplex = !!value && type === "object";
 
-			if (isComplex) {
-				if (seen.has(value)) {
-					const cached = cache.get(value);
-					if (!cached) {
-						throw new TsonCircularReferenceError(value);
-					}
-
-					return cached;
-				}
-
-				seen.add(value);
-			}
-
 			const cacheAndReturn = (result: unknown) => {
 				if (isComplex) {
 					cache.set(value, result);
@@ -84,6 +71,19 @@ export function createTsonSerialize(opts: TsonOptions): TsonSerializeFn {
 				return result;
 			};
 
+			if (isComplex) {
+				const prev = seen.get(value);
+				if (prev) {
+					return [
+						"CIRCULAR",
+						prev.join(nonce),
+						nonce,
+					] as TsonTuple;
+				}
+
+				seen.set(value, path)
+			}
+
 			const primitiveHandler = byPrimitive[type];
 			if (
 				primitiveHandler &&
@@ -92,7 +92,7 @@ export function createTsonSerialize(opts: TsonOptions): TsonSerializeFn {
 				return cacheAndReturn([
 					primitiveHandler.key,
 					// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
-					walk(primitiveHandler.serialize!(value)),
+					walk(primitiveHandler.serialize!(value), path),
 					nonce,
 				] as TsonTuple);
 			}
@@ -102,13 +102,13 @@ export function createTsonSerialize(opts: TsonOptions): TsonSerializeFn {
 					return cacheAndReturn([
 						handler.key,
 						// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
-						walk(handler.serialize!(value)),
+						walk(handler.serialize!(value), path),
 						nonce,
 					] as TsonTuple);
 				}
 			}
 
-			return cacheAndReturn(mapOrReturn(value, walk));
+			return cacheAndReturn(mapOrReturn(value, (value, key) => walk(value, [...path, key])));
 		};
 
 		return walk;