feat: compute buffer size ahead of time

packages/protons-runtime/src/codecs/bool.ts

@@ -4,8 +4,10 @@ const encodingLength: EncodingLengthFunction<boolean> = function boolEncodingLen
   return 1
 }
 
-const encode: EncodeFunction<boolean> = function boolEncode (value) {
-  return Uint8Array.from([value ? 1 : 0])
+const encode: EncodeFunction<boolean> = function boolEncode (val, buf, offset) {
+  buf.set(offset, val ? 1 : 0)
+
+  return offset + encodingLength(val)
 }
 
 const decode: DecodeFunction<boolean> = function boolDecode (buffer, offset) {

packages/protons-runtime/src/codecs/bytes.ts

@@ -1,5 +1,4 @@
 
-import { Uint8ArrayList } from 'uint8arraylist'
 import { unsigned } from '../utils/varint.js'
 import { DecodeFunction, EncodeFunction, createCodec, EncodingLengthFunction, CODEC_TYPES } from './codec.js'
 
@@ -8,12 +7,11 @@ const encodingLength: EncodingLengthFunction<Uint8Array> = function bytesEncodin
   return unsigned.encodingLength(len) + len
 }
 
-const encode: EncodeFunction<Uint8Array> = function bytesEncode (val) {
-  const prefix = new Uint8Array(unsigned.encodingLength(val.byteLength))
+const encode: EncodeFunction<Uint8Array> = function bytesEncode (val, buf, offset) {
+  offset = unsigned.encode(val.byteLength, buf, offset)
+  buf.write(val, offset)
 
-  unsigned.encode(val.byteLength, prefix)
-
-  return new Uint8ArrayList(prefix, val)
+  return offset + val.byteLength
 }
 
 const decode: DecodeFunction<Uint8Array> = function bytesDecode (buf, offset) {

packages/protons-runtime/src/codecs/codec.ts

@@ -11,7 +11,7 @@ export enum CODEC_TYPES {
 }
 
 export interface EncodeFunction<T> {
-  (value: T): Uint8Array | Uint8ArrayList
+  (value: T, buf: Uint8ArrayList, offset: number): number
 }
 
 export interface DecodeFunction<T> {

packages/protons-runtime/src/codecs/double.ts

@@ -1,15 +1,13 @@
-import { Uint8ArrayList } from 'uint8arraylist'
 import { DecodeFunction, EncodeFunction, createCodec, EncodingLengthFunction, CODEC_TYPES } from './codec.js'
 
 const encodingLength: EncodingLengthFunction<number> = function doubleEncodingLength () {
   return 8
 }
 
-const encode: EncodeFunction<number> = function doubleEncode (val) {
-  const buf = new Uint8ArrayList(new Uint8Array(encodingLength(val)))
-  buf.setFloat64(0, val, true)
+const encode: EncodeFunction<number> = function doubleEncode (val, buf, offset) {
+  buf.setFloat64(offset, val, true)
 
-  return buf
+  return offset + encodingLength(val)
 }
 
 const decode: DecodeFunction<number> = function doubleDecode (buf, offset) {

packages/protons-runtime/src/codecs/enum.ts

@@ -10,14 +10,11 @@ export function enumeration <T> (e: T): Codec<T> {
     return unsigned.encodingLength(index)
   }
 
-  const encode: EncodeFunction<string> = function enumEncode (val) {
+  const encode: EncodeFunction<string> = function enumEncode (val, buf, offset) {
     const keys = Object.keys(e)
     const index = keys.indexOf(val)
-    const buf = new Uint8Array(unsigned.encodingLength(index))
 
-    unsigned.encode(index, buf)
-
-    return buf
+    return unsigned.encode(index, buf, offset)
   }
 
   const decode: DecodeFunction<string> = function enumDecode (buf, offset) {

packages/protons-runtime/src/codecs/fixed32.ts

@@ -1,15 +1,13 @@
-import { Uint8ArrayList } from 'uint8arraylist'
 import { DecodeFunction, EncodeFunction, createCodec, EncodingLengthFunction, CODEC_TYPES } from './codec.js'
 
 const encodingLength: EncodingLengthFunction<number> = function fixed32EncodingLength () {
   return 4
 }
 
-const encode: EncodeFunction<number> = function fixed32Encode (val) {
-  const buf = new Uint8ArrayList(new Uint8Array(encodingLength(val)))
-  buf.setInt32(0, val, true)
+const encode: EncodeFunction<number> = function fixed32Encode (val, buf, offset) {
+  buf.setInt32(offset, val, true)
 
-  return buf
+  return offset + encodingLength(val)
 }
 
 const decode: DecodeFunction<number> = function fixed32Decode (buf, offset) {

packages/protons-runtime/src/codecs/fixed64.ts

@@ -1,15 +1,13 @@
-import { Uint8ArrayList } from 'uint8arraylist'
 import { DecodeFunction, EncodeFunction, createCodec, EncodingLengthFunction, CODEC_TYPES } from './codec.js'
 
 const encodingLength: EncodingLengthFunction<bigint> = function int64EncodingLength (val) {
   return 8
 }
 
-const encode: EncodeFunction<bigint> = function int64Encode (val) {
-  const buf = new Uint8ArrayList(new Uint8Array(encodingLength(val)))
-  buf.setBigInt64(0, val, true)
+const encode: EncodeFunction<bigint> = function int64Encode (val, buf, offset) {
+  buf.setBigInt64(offset, val, true)
 
-  return buf
+  return offset + encodingLength(val)
 }
 
 const decode: DecodeFunction<bigint> = function int64Decode (buf, offset) {

packages/protons-runtime/src/codecs/float.ts

@@ -1,15 +1,13 @@
-import { Uint8ArrayList } from 'uint8arraylist'
 import { DecodeFunction, EncodeFunction, createCodec, EncodingLengthFunction, CODEC_TYPES } from './codec.js'
 
 const encodingLength: EncodingLengthFunction<number> = function floatEncodingLength () {
   return 4
 }
 
-const encode: EncodeFunction<number> = function floatEncode (val) {
-  const buf = new Uint8ArrayList(new Uint8Array(encodingLength(1)))
-  buf.setFloat32(0, val, true)
+const encode: EncodeFunction<number> = function floatEncode (val, buf, offset) {
+  buf.setFloat32(offset, val, true)
 
-  return buf
+  return offset + encodingLength(val)
 }
 
 const decode: DecodeFunction<number> = function floatDecode (buf, offset) {

packages/protons-runtime/src/codecs/int32.ts

@@ -5,11 +5,8 @@ const encodingLength: EncodingLengthFunction<number> = function int32EncodingLen
   return signed.encodingLength(val)
 }
 
-const encode: EncodeFunction<number> = function int32Encode (val) {
-  const buf = new Uint8Array(encodingLength(val))
-  signed.encode(val, buf)
-
-  return buf
+const encode: EncodeFunction<number> = function int32Encode (val, buf, offset) {
+  return signed.encode(val, buf, offset)
 }
 
 const decode: DecodeFunction<number> = function int32Decode (buf, offset) {

packages/protons-runtime/src/codecs/int64.ts

@@ -5,11 +5,8 @@ const encodingLength: EncodingLengthFunction<bigint> = function int64EncodingLen
   return signed.encodingLength(val)
 }
 
-const encode: EncodeFunction<bigint> = function int64Encode (val) {
-  const buf = new Uint8Array(encodingLength(val))
-  signed.encode(val, buf)
-
-  return buf
+const encode: EncodeFunction<bigint> = function int64Encode (val, buf, offset) {
+  return signed.encode(val, buf, offset)
 }
 
 const decode: DecodeFunction<bigint> = function int64Decode (buf, offset) {

packages/protons-runtime/src/codecs/message.ts

@@ -1,64 +1,98 @@
 import { unsigned } from '../utils/varint.js'
 import type { FieldDef, FieldDefs } from '../index.js'
 import { DecodeFunction, EncodeFunction, createCodec, EncodingLengthFunction, Codec, CODEC_TYPES } from './codec.js'
-import { Uint8ArrayList } from 'uint8arraylist'
 
 export interface Factory<A, T> {
   new (obj: A): T
 }
 
 export function message <T> (fieldDefs: FieldDefs): Codec<T> {
   const encodingLength: EncodingLengthFunction<T> = function messageEncodingLength (val: Record<string, any>) {
+    function valueEncodingLength (value: any, fieldNumber: number, fieldDef: FieldDef): number {
+      if (value == null) {
+        if (fieldDef.optional === true) {
+          return 0
+        }
+
+        throw new Error(`Non optional field "${fieldDef.name}" was ${value === null ? 'null' : 'undefined'}`)
+      }
+
+      const key = (fieldNumber << 3) | fieldDef.codec.type
+      const prefixLength = unsigned.encodingLength(key)
+
+      return fieldDef.codec.encodingLength(value) + prefixLength
+    }
+
     let length = 0
 
-    for (const fieldDef of Object.values(fieldDefs)) {
-      length += fieldDef.codec.encodingLength(val[fieldDef.name])
+    for (const [fieldNumberStr, fieldDef] of Object.entries(fieldDefs)) {
+      const fieldNumber = parseInt(fieldNumberStr)
+      const value = val[fieldDef.name]
+
+      if (value == null && !(fieldDef.optional === true)) {
+        throw new Error(`Field ${fieldDef.name} cannot be ${value === null ? 'null' : 'undefined'}`)
+      }
+
+      if (fieldDef.repeats === true) {
+        if (!Array.isArray(value)) {
+          throw new Error(`Repeating field ${fieldDef.name} was not an array`)
+        }
+
+        for (const entry of value) {
+          length += valueEncodingLength(entry, fieldNumber, fieldDef)
+        }
+
+        continue
+      }
+
+      length += valueEncodingLength(value, fieldNumber, fieldDef)
     }
 
     return unsigned.encodingLength(length) + length
   }
 
-  const encode: EncodeFunction<Record<string, any>> = function messageEncode (val) {
-    const bytes = new Uint8ArrayList()
-
-    function encodeValue (value: any, fieldNumber: number, fieldDef: FieldDef) {
+  const encode: EncodeFunction<T> = function messageEncode (val, buf, offset): number {
+    function encodeValue (value: any, fieldNumber: number, fieldDef: FieldDef, offset: number): number {
       if (value == null) {
         if (fieldDef.optional === true) {
-          return
+          return offset
         }
 
         throw new Error(`Non optional field "${fieldDef.name}" was ${value === null ? 'null' : 'undefined'}`)
       }
 
       const key = (fieldNumber << 3) | fieldDef.codec.type
-      const prefix = new Uint8Array(unsigned.encodingLength(key))
-      unsigned.encode(key, prefix)
-      const encoded = fieldDef.codec.encode(value)
+      offset = unsigned.encode(key, buf, offset)
+      offset = fieldDef.codec.encode(value, buf, offset)
 
-      bytes.append(prefix)
-      bytes.append(encoded)
+      return offset
     }
 
+    const length = encodingLength(val)
+    offset = unsigned.encode(length - unsigned.encodingLength(length), buf, offset)
+
     for (const [fieldNumberStr, fieldDef] of Object.entries(fieldDefs)) {
       const fieldNumber = parseInt(fieldNumberStr)
 
       if (fieldDef.repeats === true) {
+        // @ts-expect-error cannot use strings to index T
         if (!Array.isArray(val[fieldDef.name])) {
           throw new Error(`Repeating field "${fieldDef.name}" was not an array`)
         }
 
+        // @ts-expect-error cannot use strings to index T
         for (const value of val[fieldDef.name]) {
-          encodeValue(value, fieldNumber, fieldDef)
+          offset = encodeValue(value, fieldNumber, fieldDef, offset)
         }
-      } else {
-        encodeValue(val[fieldDef.name], fieldNumber, fieldDef)
+
+        continue
       }
-    }
 
-    const prefix = new Uint8Array(unsigned.encodingLength(bytes.length))
-    unsigned.encode(bytes.length, prefix)
+      // @ts-expect-error cannot use strings to index T
+      offset = encodeValue(val[fieldDef.name], fieldNumber, fieldDef, offset)
+    }
 
-    return new Uint8ArrayList(prefix, bytes)
+    return offset
   }
 
   const decode: DecodeFunction<T> = function messageDecode (buffer, offset) {
@@ -78,10 +112,7 @@ export function message <T> (fieldDefs: FieldDefs): Codec<T> {
       const fieldDef = fieldDefs[fieldNumber]
       let fieldLength = 0
 
-      // console.info('fieldNumber', fieldNumber, 'wireType', wireType, 'offset', offset)
-
       if (wireType === CODEC_TYPES.VARINT) {
-        // console.info('decode varint')
         if (fieldDef != null) {
           // use the codec if it is available as this could be a bigint
           const value = fieldDef.codec.decode(buffer, offset)
@@ -91,25 +122,19 @@ export function message <T> (fieldDefs: FieldDefs): Codec<T> {
           fieldLength = unsigned.encodingLength(value)
         }
       } else if (wireType === CODEC_TYPES.BIT64) {
-        // console.info('decode 64bit')
         fieldLength = 8
       } else if (wireType === CODEC_TYPES.LENGTH_DELIMITED) {
-        // console.info('decode length delimited')
         const valueLength = unsigned.decode(buffer, offset)
         fieldLength = valueLength + unsigned.encodingLength(valueLength)
       } else if (wireType === CODEC_TYPES.BIT32) {
-        // console.info('decode 32 bit')
         fieldLength = 4
       } else if (wireType === CODEC_TYPES.START_GROUP) {
         throw new Error('Unsupported wire type START_GROUP')
       } else if (wireType === CODEC_TYPES.END_GROUP) {
         throw new Error('Unsupported wire type END_GROUP')
       }
 
-      // console.info('fieldLength', fieldLength)
-
       if (fieldDef != null) {
-        // console.info('decode', fieldDef.codec.name, fieldDef.name, 'at offset', offset)
         const value = fieldDef.codec.decode(buffer, offset)
 
         if (fieldDef.repeats === true) {
@@ -121,8 +146,6 @@ export function message <T> (fieldDefs: FieldDefs): Codec<T> {
         } else {
           fields[fieldDef.name] = value
         }
-
-        // console.info('decoded', value)
       }
 
       offset += fieldLength

packages/protons-runtime/src/codecs/sfixed32.ts

@@ -1,15 +1,13 @@
-import { Uint8ArrayList } from 'uint8arraylist'
 import { DecodeFunction, EncodeFunction, createCodec, EncodingLengthFunction, CODEC_TYPES } from './codec.js'
 
 const encodingLength: EncodingLengthFunction<number> = function sfixed32EncodingLength () {
   return 4
 }
 
-const encode: EncodeFunction<number> = function sfixed32Encode (val) {
-  const buf = new Uint8ArrayList(new Uint8Array(encodingLength(val)))
-  buf.setInt32(0, val, true)
+const encode: EncodeFunction<number> = function sfixed32Encode (val, buf, offset) {
+  buf.setInt32(offset, val, true)
 
-  return buf
+  return offset + encodingLength(val)
 }
 
 const decode: DecodeFunction<number> = function sfixed32Decode (buf, offset) {

packages/protons-runtime/src/codecs/sfixed64.ts

@@ -1,15 +1,13 @@
-import { Uint8ArrayList } from 'uint8arraylist'
 import { DecodeFunction, EncodeFunction, createCodec, EncodingLengthFunction, CODEC_TYPES } from './codec.js'
 
 const encodingLength: EncodingLengthFunction<bigint> = function sfixed64EncodingLength () {
   return 8
 }
 
-const encode: EncodeFunction<bigint> = function sfixed64Encode (val) {
-  const buf = new Uint8ArrayList(new Uint8Array(encodingLength(val)))
-  buf.setBigInt64(0, val, true)
+const encode: EncodeFunction<bigint> = function sfixed64Encode (val, buf, offset) {
+  buf.setBigInt64(offset, val, true)
 
-  return buf
+  return offset + encodingLength(val)
 }
 
 const decode: DecodeFunction<bigint> = function sfixed64Decode (buf, offset) {

packages/protons-runtime/src/codecs/sint32.ts

@@ -5,12 +5,8 @@ const encodingLength: EncodingLengthFunction<number> = function sint32EncodingLe
   return zigzag.encodingLength(val)
 }
 
-const encode: EncodeFunction<number> = function svarintEncode (val) {
-  const buf = new Uint8Array(encodingLength(val))
-
-  zigzag.encode(val, buf)
-
-  return buf
+const encode: EncodeFunction<number> = function svarintEncode (val, buf, offset) {
+  return zigzag.encode(val, buf, offset)
 }
 
 const decode: DecodeFunction<number> = function svarintDecode (buf, offset) {

packages/protons-runtime/src/codecs/sint64.ts

@@ -5,11 +5,8 @@ const encodingLength: EncodingLengthFunction<bigint> = function int64EncodingLen
   return zigzag.encodingLength(val)
 }
 
-const encode: EncodeFunction<bigint> = function int64Encode (val) {
-  const buf = new Uint8Array(encodingLength(val))
-  zigzag.encode(val, buf)
-
-  return buf
+const encode: EncodeFunction<bigint> = function int64Encode (val, buf, offset) {
+  return zigzag.encode(val, buf, offset)
 }
 
 const decode: DecodeFunction<bigint> = function int64Decode (buf, offset) {