From fecfedd1a4bce3846a312fb4a3fb6264d1d542a5 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Thu, 31 Mar 2022 09:33:11 +0100 Subject: [PATCH] feat: compute buffer size ahead of time MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The current implementation loops over message fields and each field encoder creates a `Uint8Array` which is added to a `Uint8ArrayList` and stitched together at the end of the process. Other implementations compute the expected length of the serialized message, allocate a buffer that size then have encoders write their data into it at the correct offsets. This PR makes protons compute the final message buffer size first in the same way. Thing is, it doesn't seem to make a huge amount of difference to performance. Before: ``` Running "Encode/Decode" suite... Progress: 100% pbjs: 12 166 ops/s, ±3.92% | 5.12% slower protons: 9 755 ops/s, ±2.19% | slowest, 23.93% slower protobufjs: 12 823 ops/s, ±2.02% | fastest Finished 3 cases! Fastest: protobufjs Slowest: protons ``` After: ``` Running "Encode/Decode" suite... Progress: 100% pbjs: 11 866 ops/s, ±3.43% | 2.05% slower protons: 9 356 ops/s, ±2.45% | slowest, 22.77% slower protobufjs: 12 114 ops/s, ±2.16% | fastest Finished 3 cases! Fastest: protobufjs Slowest: protons ``` --- packages/protons-runtime/src/codecs/bool.ts | 6 +- packages/protons-runtime/src/codecs/bytes.ts | 10 +-- packages/protons-runtime/src/codecs/codec.ts | 2 +- packages/protons-runtime/src/codecs/double.ts | 8 +- packages/protons-runtime/src/codecs/enum.ts | 7 +- .../protons-runtime/src/codecs/fixed32.ts | 8 +- .../protons-runtime/src/codecs/fixed64.ts | 8 +- packages/protons-runtime/src/codecs/float.ts | 8 +- packages/protons-runtime/src/codecs/int32.ts | 7 +- packages/protons-runtime/src/codecs/int64.ts | 7 +- .../protons-runtime/src/codecs/message.ts | 85 ++++++++++++------- .../protons-runtime/src/codecs/sfixed32.ts | 8 +- .../protons-runtime/src/codecs/sfixed64.ts | 8 +- packages/protons-runtime/src/codecs/sint32.ts | 8 +- packages/protons-runtime/src/codecs/sint64.ts | 7 +- packages/protons-runtime/src/codecs/string.ts | 12 ++- packages/protons-runtime/src/codecs/uint32.ts | 12 +-- packages/protons-runtime/src/codecs/uint64.ts | 8 +- packages/protons-runtime/src/decode.ts | 4 +- packages/protons-runtime/src/encode.ts | 11 ++- .../protons-runtime/src/utils/big-varint.ts | 19 ++--- .../protons-runtime/src/utils/long-bits.ts | 4 +- packages/protons-runtime/src/utils/varint.ts | 22 ++--- 23 files changed, 132 insertions(+), 147 deletions(-) diff --git a/packages/protons-runtime/src/codecs/bool.ts b/packages/protons-runtime/src/codecs/bool.ts index aa3c0d1..ab45ca0 100644 --- a/packages/protons-runtime/src/codecs/bool.ts +++ b/packages/protons-runtime/src/codecs/bool.ts @@ -4,8 +4,10 @@ const encodingLength: EncodingLengthFunction = function boolEncodingLen return 1 } -const encode: EncodeFunction = function boolEncode (value) { - return Uint8Array.from([value ? 1 : 0]) +const encode: EncodeFunction = function boolEncode (val, buf, offset) { + buf.set(offset, val ? 1 : 0) + + return offset + encodingLength(val) } const decode: DecodeFunction = function boolDecode (buffer, offset) { diff --git a/packages/protons-runtime/src/codecs/bytes.ts b/packages/protons-runtime/src/codecs/bytes.ts index 37123f2..cd36bf6 100644 --- a/packages/protons-runtime/src/codecs/bytes.ts +++ b/packages/protons-runtime/src/codecs/bytes.ts @@ -1,5 +1,4 @@ -import { Uint8ArrayList } from 'uint8arraylist' import { unsigned } from '../utils/varint.js' import { DecodeFunction, EncodeFunction, createCodec, EncodingLengthFunction, CODEC_TYPES } from './codec.js' @@ -8,12 +7,11 @@ const encodingLength: EncodingLengthFunction = function bytesEncodin return unsigned.encodingLength(len) + len } -const encode: EncodeFunction = function bytesEncode (val) { - const prefix = new Uint8Array(unsigned.encodingLength(val.byteLength)) +const encode: EncodeFunction = function bytesEncode (val, buf, offset) { + offset = unsigned.encode(val.byteLength, buf, offset) + buf.write(val, offset) - unsigned.encode(val.byteLength, prefix) - - return new Uint8ArrayList(prefix, val) + return offset + val.byteLength } const decode: DecodeFunction = function bytesDecode (buf, offset) { diff --git a/packages/protons-runtime/src/codecs/codec.ts b/packages/protons-runtime/src/codecs/codec.ts index 7332fd7..52c9d1c 100644 --- a/packages/protons-runtime/src/codecs/codec.ts +++ b/packages/protons-runtime/src/codecs/codec.ts @@ -11,7 +11,7 @@ export enum CODEC_TYPES { } export interface EncodeFunction { - (value: T): Uint8Array | Uint8ArrayList + (value: T, buf: Uint8ArrayList, offset: number): number } export interface DecodeFunction { diff --git a/packages/protons-runtime/src/codecs/double.ts b/packages/protons-runtime/src/codecs/double.ts index 1d887fc..a93c772 100644 --- a/packages/protons-runtime/src/codecs/double.ts +++ b/packages/protons-runtime/src/codecs/double.ts @@ -1,15 +1,13 @@ -import { Uint8ArrayList } from 'uint8arraylist' import { DecodeFunction, EncodeFunction, createCodec, EncodingLengthFunction, CODEC_TYPES } from './codec.js' const encodingLength: EncodingLengthFunction = function doubleEncodingLength () { return 8 } -const encode: EncodeFunction = function doubleEncode (val) { - const buf = new Uint8ArrayList(new Uint8Array(encodingLength(val))) - buf.setFloat64(0, val, true) +const encode: EncodeFunction = function doubleEncode (val, buf, offset) { + buf.setFloat64(offset, val, true) - return buf + return offset + encodingLength(val) } const decode: DecodeFunction = function doubleDecode (buf, offset) { diff --git a/packages/protons-runtime/src/codecs/enum.ts b/packages/protons-runtime/src/codecs/enum.ts index 26f8578..fb2d3ca 100644 --- a/packages/protons-runtime/src/codecs/enum.ts +++ b/packages/protons-runtime/src/codecs/enum.ts @@ -10,14 +10,11 @@ export function enumeration (e: T): Codec { return unsigned.encodingLength(index) } - const encode: EncodeFunction = function enumEncode (val) { + const encode: EncodeFunction = function enumEncode (val, buf, offset) { const keys = Object.keys(e) const index = keys.indexOf(val) - const buf = new Uint8Array(unsigned.encodingLength(index)) - unsigned.encode(index, buf) - - return buf + return unsigned.encode(index, buf, offset) } const decode: DecodeFunction = function enumDecode (buf, offset) { diff --git a/packages/protons-runtime/src/codecs/fixed32.ts b/packages/protons-runtime/src/codecs/fixed32.ts index cd29e3d..50a7478 100644 --- a/packages/protons-runtime/src/codecs/fixed32.ts +++ b/packages/protons-runtime/src/codecs/fixed32.ts @@ -1,15 +1,13 @@ -import { Uint8ArrayList } from 'uint8arraylist' import { DecodeFunction, EncodeFunction, createCodec, EncodingLengthFunction, CODEC_TYPES } from './codec.js' const encodingLength: EncodingLengthFunction = function fixed32EncodingLength () { return 4 } -const encode: EncodeFunction = function fixed32Encode (val) { - const buf = new Uint8ArrayList(new Uint8Array(encodingLength(val))) - buf.setInt32(0, val, true) +const encode: EncodeFunction = function fixed32Encode (val, buf, offset) { + buf.setInt32(offset, val, true) - return buf + return offset + encodingLength(val) } const decode: DecodeFunction = function fixed32Decode (buf, offset) { diff --git a/packages/protons-runtime/src/codecs/fixed64.ts b/packages/protons-runtime/src/codecs/fixed64.ts index d8113f1..84a3148 100644 --- a/packages/protons-runtime/src/codecs/fixed64.ts +++ b/packages/protons-runtime/src/codecs/fixed64.ts @@ -1,15 +1,13 @@ -import { Uint8ArrayList } from 'uint8arraylist' import { DecodeFunction, EncodeFunction, createCodec, EncodingLengthFunction, CODEC_TYPES } from './codec.js' const encodingLength: EncodingLengthFunction = function int64EncodingLength (val) { return 8 } -const encode: EncodeFunction = function int64Encode (val) { - const buf = new Uint8ArrayList(new Uint8Array(encodingLength(val))) - buf.setBigInt64(0, val, true) +const encode: EncodeFunction = function int64Encode (val, buf, offset) { + buf.setBigInt64(offset, val, true) - return buf + return offset + encodingLength(val) } const decode: DecodeFunction = function int64Decode (buf, offset) { diff --git a/packages/protons-runtime/src/codecs/float.ts b/packages/protons-runtime/src/codecs/float.ts index 10270af..18698c1 100644 --- a/packages/protons-runtime/src/codecs/float.ts +++ b/packages/protons-runtime/src/codecs/float.ts @@ -1,15 +1,13 @@ -import { Uint8ArrayList } from 'uint8arraylist' import { DecodeFunction, EncodeFunction, createCodec, EncodingLengthFunction, CODEC_TYPES } from './codec.js' const encodingLength: EncodingLengthFunction = function floatEncodingLength () { return 4 } -const encode: EncodeFunction = function floatEncode (val) { - const buf = new Uint8ArrayList(new Uint8Array(encodingLength(1))) - buf.setFloat32(0, val, true) +const encode: EncodeFunction = function floatEncode (val, buf, offset) { + buf.setFloat32(offset, val, true) - return buf + return offset + encodingLength(val) } const decode: DecodeFunction = function floatDecode (buf, offset) { diff --git a/packages/protons-runtime/src/codecs/int32.ts b/packages/protons-runtime/src/codecs/int32.ts index 0889d58..17ec510 100644 --- a/packages/protons-runtime/src/codecs/int32.ts +++ b/packages/protons-runtime/src/codecs/int32.ts @@ -5,11 +5,8 @@ const encodingLength: EncodingLengthFunction = function int32EncodingLen return signed.encodingLength(val) } -const encode: EncodeFunction = function int32Encode (val) { - const buf = new Uint8Array(encodingLength(val)) - signed.encode(val, buf) - - return buf +const encode: EncodeFunction = function int32Encode (val, buf, offset) { + return signed.encode(val, buf, offset) } const decode: DecodeFunction = function int32Decode (buf, offset) { diff --git a/packages/protons-runtime/src/codecs/int64.ts b/packages/protons-runtime/src/codecs/int64.ts index d58c0a7..161ff97 100644 --- a/packages/protons-runtime/src/codecs/int64.ts +++ b/packages/protons-runtime/src/codecs/int64.ts @@ -5,11 +5,8 @@ const encodingLength: EncodingLengthFunction = function int64EncodingLen return signed.encodingLength(val) } -const encode: EncodeFunction = function int64Encode (val) { - const buf = new Uint8Array(encodingLength(val)) - signed.encode(val, buf) - - return buf +const encode: EncodeFunction = function int64Encode (val, buf, offset) { + return signed.encode(val, buf, offset) } const decode: DecodeFunction = function int64Decode (buf, offset) { diff --git a/packages/protons-runtime/src/codecs/message.ts b/packages/protons-runtime/src/codecs/message.ts index 64cf995..13de2a0 100644 --- a/packages/protons-runtime/src/codecs/message.ts +++ b/packages/protons-runtime/src/codecs/message.ts @@ -1,7 +1,6 @@ import { unsigned } from '../utils/varint.js' import type { FieldDef, FieldDefs } from '../index.js' import { DecodeFunction, EncodeFunction, createCodec, EncodingLengthFunction, Codec, CODEC_TYPES } from './codec.js' -import { Uint8ArrayList } from 'uint8arraylist' export interface Factory { new (obj: A): T @@ -9,56 +8,91 @@ export interface Factory { export function message (fieldDefs: FieldDefs): Codec { const encodingLength: EncodingLengthFunction = function messageEncodingLength (val: Record) { + function valueEncodingLength (value: any, fieldNumber: number, fieldDef: FieldDef): number { + if (value == null) { + if (fieldDef.optional === true) { + return 0 + } + + throw new Error(`Non optional field "${fieldDef.name}" was ${value === null ? 'null' : 'undefined'}`) + } + + const key = (fieldNumber << 3) | fieldDef.codec.type + const prefixLength = unsigned.encodingLength(key) + + return fieldDef.codec.encodingLength(value) + prefixLength + } + let length = 0 - for (const fieldDef of Object.values(fieldDefs)) { - length += fieldDef.codec.encodingLength(val[fieldDef.name]) + for (const [fieldNumberStr, fieldDef] of Object.entries(fieldDefs)) { + const fieldNumber = parseInt(fieldNumberStr) + const value = val[fieldDef.name] + + if (value == null && !(fieldDef.optional === true)) { + throw new Error(`Field ${fieldDef.name} cannot be ${value === null ? 'null' : 'undefined'}`) + } + + if (fieldDef.repeats === true) { + if (!Array.isArray(value)) { + throw new Error(`Repeating field ${fieldDef.name} was not an array`) + } + + for (const entry of value) { + length += valueEncodingLength(entry, fieldNumber, fieldDef) + } + + continue + } + + length += valueEncodingLength(value, fieldNumber, fieldDef) } return unsigned.encodingLength(length) + length } - const encode: EncodeFunction> = function messageEncode (val) { - const bytes = new Uint8ArrayList() - - function encodeValue (value: any, fieldNumber: number, fieldDef: FieldDef) { + const encode: EncodeFunction = function messageEncode (val, buf, offset): number { + function encodeValue (value: any, fieldNumber: number, fieldDef: FieldDef, offset: number): number { if (value == null) { if (fieldDef.optional === true) { - return + return offset } throw new Error(`Non optional field "${fieldDef.name}" was ${value === null ? 'null' : 'undefined'}`) } const key = (fieldNumber << 3) | fieldDef.codec.type - const prefix = new Uint8Array(unsigned.encodingLength(key)) - unsigned.encode(key, prefix) - const encoded = fieldDef.codec.encode(value) + offset = unsigned.encode(key, buf, offset) + offset = fieldDef.codec.encode(value, buf, offset) - bytes.append(prefix) - bytes.append(encoded) + return offset } + const length = encodingLength(val) + offset = unsigned.encode(length - unsigned.encodingLength(length), buf, offset) + for (const [fieldNumberStr, fieldDef] of Object.entries(fieldDefs)) { const fieldNumber = parseInt(fieldNumberStr) if (fieldDef.repeats === true) { + // @ts-expect-error cannot use strings to index T if (!Array.isArray(val[fieldDef.name])) { throw new Error(`Repeating field "${fieldDef.name}" was not an array`) } + // @ts-expect-error cannot use strings to index T for (const value of val[fieldDef.name]) { - encodeValue(value, fieldNumber, fieldDef) + offset = encodeValue(value, fieldNumber, fieldDef, offset) } - } else { - encodeValue(val[fieldDef.name], fieldNumber, fieldDef) + + continue } - } - const prefix = new Uint8Array(unsigned.encodingLength(bytes.length)) - unsigned.encode(bytes.length, prefix) + // @ts-expect-error cannot use strings to index T + offset = encodeValue(val[fieldDef.name], fieldNumber, fieldDef, offset) + } - return new Uint8ArrayList(prefix, bytes) + return offset } const decode: DecodeFunction = function messageDecode (buffer, offset) { @@ -78,10 +112,7 @@ export function message (fieldDefs: FieldDefs): Codec { const fieldDef = fieldDefs[fieldNumber] let fieldLength = 0 - // console.info('fieldNumber', fieldNumber, 'wireType', wireType, 'offset', offset) - if (wireType === CODEC_TYPES.VARINT) { - // console.info('decode varint') if (fieldDef != null) { // use the codec if it is available as this could be a bigint const value = fieldDef.codec.decode(buffer, offset) @@ -91,14 +122,11 @@ export function message (fieldDefs: FieldDefs): Codec { fieldLength = unsigned.encodingLength(value) } } else if (wireType === CODEC_TYPES.BIT64) { - // console.info('decode 64bit') fieldLength = 8 } else if (wireType === CODEC_TYPES.LENGTH_DELIMITED) { - // console.info('decode length delimited') const valueLength = unsigned.decode(buffer, offset) fieldLength = valueLength + unsigned.encodingLength(valueLength) } else if (wireType === CODEC_TYPES.BIT32) { - // console.info('decode 32 bit') fieldLength = 4 } else if (wireType === CODEC_TYPES.START_GROUP) { throw new Error('Unsupported wire type START_GROUP') @@ -106,10 +134,7 @@ export function message (fieldDefs: FieldDefs): Codec { throw new Error('Unsupported wire type END_GROUP') } - // console.info('fieldLength', fieldLength) - if (fieldDef != null) { - // console.info('decode', fieldDef.codec.name, fieldDef.name, 'at offset', offset) const value = fieldDef.codec.decode(buffer, offset) if (fieldDef.repeats === true) { @@ -121,8 +146,6 @@ export function message (fieldDefs: FieldDefs): Codec { } else { fields[fieldDef.name] = value } - - // console.info('decoded', value) } offset += fieldLength diff --git a/packages/protons-runtime/src/codecs/sfixed32.ts b/packages/protons-runtime/src/codecs/sfixed32.ts index f165b90..49adbad 100644 --- a/packages/protons-runtime/src/codecs/sfixed32.ts +++ b/packages/protons-runtime/src/codecs/sfixed32.ts @@ -1,15 +1,13 @@ -import { Uint8ArrayList } from 'uint8arraylist' import { DecodeFunction, EncodeFunction, createCodec, EncodingLengthFunction, CODEC_TYPES } from './codec.js' const encodingLength: EncodingLengthFunction = function sfixed32EncodingLength () { return 4 } -const encode: EncodeFunction = function sfixed32Encode (val) { - const buf = new Uint8ArrayList(new Uint8Array(encodingLength(val))) - buf.setInt32(0, val, true) +const encode: EncodeFunction = function sfixed32Encode (val, buf, offset) { + buf.setInt32(offset, val, true) - return buf + return offset + encodingLength(val) } const decode: DecodeFunction = function sfixed32Decode (buf, offset) { diff --git a/packages/protons-runtime/src/codecs/sfixed64.ts b/packages/protons-runtime/src/codecs/sfixed64.ts index 68359b0..440b29f 100644 --- a/packages/protons-runtime/src/codecs/sfixed64.ts +++ b/packages/protons-runtime/src/codecs/sfixed64.ts @@ -1,15 +1,13 @@ -import { Uint8ArrayList } from 'uint8arraylist' import { DecodeFunction, EncodeFunction, createCodec, EncodingLengthFunction, CODEC_TYPES } from './codec.js' const encodingLength: EncodingLengthFunction = function sfixed64EncodingLength () { return 8 } -const encode: EncodeFunction = function sfixed64Encode (val) { - const buf = new Uint8ArrayList(new Uint8Array(encodingLength(val))) - buf.setBigInt64(0, val, true) +const encode: EncodeFunction = function sfixed64Encode (val, buf, offset) { + buf.setBigInt64(offset, val, true) - return buf + return offset + encodingLength(val) } const decode: DecodeFunction = function sfixed64Decode (buf, offset) { diff --git a/packages/protons-runtime/src/codecs/sint32.ts b/packages/protons-runtime/src/codecs/sint32.ts index 3c72c2c..b5f14c2 100644 --- a/packages/protons-runtime/src/codecs/sint32.ts +++ b/packages/protons-runtime/src/codecs/sint32.ts @@ -5,12 +5,8 @@ const encodingLength: EncodingLengthFunction = function sint32EncodingLe return zigzag.encodingLength(val) } -const encode: EncodeFunction = function svarintEncode (val) { - const buf = new Uint8Array(encodingLength(val)) - - zigzag.encode(val, buf) - - return buf +const encode: EncodeFunction = function svarintEncode (val, buf, offset) { + return zigzag.encode(val, buf, offset) } const decode: DecodeFunction = function svarintDecode (buf, offset) { diff --git a/packages/protons-runtime/src/codecs/sint64.ts b/packages/protons-runtime/src/codecs/sint64.ts index 2dc4550..1c0a25c 100644 --- a/packages/protons-runtime/src/codecs/sint64.ts +++ b/packages/protons-runtime/src/codecs/sint64.ts @@ -5,11 +5,8 @@ const encodingLength: EncodingLengthFunction = function int64EncodingLen return zigzag.encodingLength(val) } -const encode: EncodeFunction = function int64Encode (val) { - const buf = new Uint8Array(encodingLength(val)) - zigzag.encode(val, buf) - - return buf +const encode: EncodeFunction = function int64Encode (val, buf, offset) { + return zigzag.encode(val, buf, offset) } const decode: DecodeFunction = function int64Decode (buf, offset) { diff --git a/packages/protons-runtime/src/codecs/string.ts b/packages/protons-runtime/src/codecs/string.ts index 04216dd..0416850 100644 --- a/packages/protons-runtime/src/codecs/string.ts +++ b/packages/protons-runtime/src/codecs/string.ts @@ -2,20 +2,18 @@ import { unsigned } from '../utils/varint.js' import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string' import { toString as uint8ArrayToString } from 'uint8arrays/to-string' import { DecodeFunction, EncodeFunction, createCodec, EncodingLengthFunction, CODEC_TYPES } from './codec.js' -import { Uint8ArrayList } from 'uint8arraylist' const encodingLength: EncodingLengthFunction = function stringEncodingLength (val) { const len = uint8ArrayFromString(val).byteLength - return unsigned.encodingLength(len) + len + return unsigned.encodingLength(len) + val.length } -const encode: EncodeFunction = function stringEncode (val) { +const encode: EncodeFunction = function stringEncode (val, buf, offset) { const asBuf = uint8ArrayFromString(val) - const prefix = new Uint8Array(unsigned.encodingLength(asBuf.byteLength)) + offset = unsigned.encode(asBuf.length, buf, offset) + buf.write(asBuf, offset) - unsigned.encode(asBuf.length, prefix) - - return new Uint8ArrayList(prefix, asBuf) + return offset + asBuf.byteLength } const decode: DecodeFunction = function stringDecode (buf, offset) { diff --git a/packages/protons-runtime/src/codecs/uint32.ts b/packages/protons-runtime/src/codecs/uint32.ts index 57aa1a7..976f26c 100644 --- a/packages/protons-runtime/src/codecs/uint32.ts +++ b/packages/protons-runtime/src/codecs/uint32.ts @@ -5,20 +5,12 @@ const encodingLength: EncodingLengthFunction = function uint32EncodingLe return unsigned.encodingLength(val) } -const encode: EncodeFunction = function uint32Encode (val) { - // val = val < 0 ? val + 4294967296 : val - - const buf = new Uint8Array(encodingLength(val)) - - unsigned.encode(val, buf) - - return buf +const encode: EncodeFunction = function uint32Encode (val, buf, offset) { + return unsigned.encode(val, buf, offset) } const decode: DecodeFunction = function uint32Decode (buf, offset) { return unsigned.decode(buf, offset) - - // return value > 2147483647 ? value - 4294967296 : value } export const uint32 = createCodec('uint32', CODEC_TYPES.VARINT, encode, decode, encodingLength) diff --git a/packages/protons-runtime/src/codecs/uint64.ts b/packages/protons-runtime/src/codecs/uint64.ts index 4753114..d163105 100644 --- a/packages/protons-runtime/src/codecs/uint64.ts +++ b/packages/protons-runtime/src/codecs/uint64.ts @@ -5,12 +5,8 @@ const encodingLength: EncodingLengthFunction = function uint64EncodingLe return unsigned.encodingLength(val) } -const encode: EncodeFunction = function uint64Encode (val) { - const buf = new Uint8Array(unsigned.encodingLength(val)) - - unsigned.encode(val, buf) - - return buf +const encode: EncodeFunction = function uint64Encode (val, buf, offset) { + return unsigned.encode(val, buf, offset) } const decode: DecodeFunction = function uint64Decode (buf, offset) { diff --git a/packages/protons-runtime/src/decode.ts b/packages/protons-runtime/src/decode.ts index 99b427d..b2436fb 100644 --- a/packages/protons-runtime/src/decode.ts +++ b/packages/protons-runtime/src/decode.ts @@ -4,8 +4,8 @@ import type { Codec } from './codecs/codec.js' export function decodeMessage (buf: Uint8Array, codec: Codec) { // wrap root message - const prefix = new Uint8Array(unsigned.encodingLength(buf.length)) - unsigned.encode(buf.length, prefix) + const prefix = Buffer.allocUnsafe(unsigned.encodingLength(buf.length)) + unsigned.encode(buf.length, prefix, 0) return codec.decode(new Uint8ArrayList(prefix, buf), 0) } diff --git a/packages/protons-runtime/src/encode.ts b/packages/protons-runtime/src/encode.ts index 83f8bae..2cc4aa6 100644 --- a/packages/protons-runtime/src/encode.ts +++ b/packages/protons-runtime/src/encode.ts @@ -1,10 +1,13 @@ +import { Uint8ArrayList } from 'uint8arraylist' import type { Codec } from './codecs/codec.js' import { unsigned } from './utils/varint.js' export function encodeMessage (message: T, codec: Codec) { - // unwrap root message - const encoded = codec.encode(message) - const skip = unsigned.encodingLength(unsigned.decode(encoded)) + const len = codec.encodingLength(message) + const buf = new Uint8ArrayList(Buffer.allocUnsafe(len)) - return encoded.slice(skip) + // unwrap root message + codec.encode(message, buf, 0) + const skip = unsigned.encodingLength(unsigned.decode(buf, 0)) + return buf.slice(skip) } diff --git a/packages/protons-runtime/src/utils/big-varint.ts b/packages/protons-runtime/src/utils/big-varint.ts index a899e1e..d8d5417 100644 --- a/packages/protons-runtime/src/utils/big-varint.ts +++ b/packages/protons-runtime/src/utils/big-varint.ts @@ -14,19 +14,20 @@ export const unsigned = { return i + 1 }, - encode (value: bigint, buf: Uint8ArrayList | Uint8Array) { + encode (value: bigint, buf: Uint8ArrayList | Uint8Array, offset: number): number { const access = accessor(buf) - let offset = 0 while (LIMIT < value) { access.set(offset++, Number(value & LIMIT) | 0x80) value >>= 7n } access.set(offset, Number(value)) + + return offset + 1 }, - decode (buf: Uint8ArrayList | Uint8Array, offset = 0) { + decode (buf: Uint8ArrayList | Uint8Array, offset = 0): bigint { return LongBits.fromBytes(buf, offset).toBigInt(true) } } @@ -40,14 +41,12 @@ export const signed = { return unsigned.encodingLength(value) }, - encode (value: bigint, buf: Uint8ArrayList | Uint8Array, offset = 0) { + encode (value: bigint, buf: Uint8ArrayList | Uint8Array, offset: number): number { if (value < 0n) { - LongBits.fromBigInt(value).toBytes(buf, offset) - - return + return LongBits.fromBigInt(value).write(buf, offset) } - return unsigned.encode(value, buf) + return unsigned.encode(value, buf, offset) }, decode (buf: Uint8ArrayList | Uint8Array, offset = 0) { @@ -60,8 +59,8 @@ export const zigzag = { return unsigned.encodingLength(value >= 0 ? value * 2n : value * -2n - 1n) }, - encode (value: bigint, buf: Uint8ArrayList | Uint8Array, offset = 0) { - LongBits.fromBigInt(value).zzEncode().toBytes(buf, offset) + encode (value: bigint, buf: Uint8ArrayList | Uint8Array, offset: number): number { + return LongBits.fromBigInt(value).zzEncode().write(buf, offset) }, decode (buf: Uint8ArrayList | Uint8Array, offset = 0) { diff --git a/packages/protons-runtime/src/utils/long-bits.ts b/packages/protons-runtime/src/utils/long-bits.ts index ad1ddcd..3b6f085 100644 --- a/packages/protons-runtime/src/utils/long-bits.ts +++ b/packages/protons-runtime/src/utils/long-bits.ts @@ -47,7 +47,7 @@ export class LongBits { return new LongBits(hi, lo) } - toBytes (buf: Uint8ArrayList | Uint8Array, offset = 0) { + write (buf: Uint8ArrayList | Uint8Array, offset = 0): number { const access = accessor(buf) while (this.hi > 0) { @@ -62,6 +62,8 @@ export class LongBits { } access.set(offset++, this.lo) + + return offset } static fromBigInt (value: bigint) { diff --git a/packages/protons-runtime/src/utils/varint.ts b/packages/protons-runtime/src/utils/varint.ts index f3b0738..2ca5bf4 100644 --- a/packages/protons-runtime/src/utils/varint.ts +++ b/packages/protons-runtime/src/utils/varint.ts @@ -57,8 +57,7 @@ export const unsigned = { return 10 }, - encode (value: number, buf: Uint8ArrayList | Uint8Array) { - let offset = 0 + encode (value: number, buf: Uint8ArrayList | Uint8Array, offset: number): number { const access = accessor(buf) while (value >= INT) { @@ -72,9 +71,11 @@ export const unsigned = { } access.set(offset, value | 0) + + return offset + 1 }, - decode (buf: Uint8ArrayList | Uint8Array, offset: number = 0) { + decode (buf: Uint8ArrayList | Uint8Array, offset: number) { const access = accessor(buf) let value = 4294967295 // optimizer type-hint, tends to deopt otherwise (?!) @@ -125,9 +126,8 @@ export const signed = { return unsigned.encodingLength(value) }, - encode (value: number, buf: Uint8ArrayList | Uint8Array) { + encode (value: number, buf: Uint8ArrayList | Uint8Array, offset: number): number { if (value < 0) { - let offset = 0 const access = accessor(buf) const bits = LongBits.fromNumber(value) @@ -144,13 +144,13 @@ export const signed = { access.set(offset++, bits.lo) - return + return offset } - unsigned.encode(value, buf) + return unsigned.encode(value, buf, offset) }, - decode (data: Uint8ArrayList | Uint8Array, offset = 0) { + decode (data: Uint8ArrayList | Uint8Array, offset: number) { return unsigned.decode(data, offset) | 0 } } @@ -161,12 +161,12 @@ export const zigzag = { return unsigned.encodingLength(value) }, - encode (value: number, buf: Uint8ArrayList | Uint8Array, offset = 0) { + encode (value: number, buf: Uint8ArrayList | Uint8Array, offset: number): number { value = (value << 1 ^ value >> 31) >>> 0 - return unsigned.encode(value, buf) + return unsigned.encode(value, buf, offset) }, - decode (data: Uint8ArrayList | Uint8Array, offset = 0) { + decode (data: Uint8ArrayList | Uint8Array, offset: number) { const value = unsigned.decode(data, offset) return value >>> 1 ^ -(value & 1) | 0 }