Added --- next TLS record --- markers to commentedString

docs/index.html

@@ -26,7 +26,9 @@ <h1 id="heading">See this page fetch itself, byte by byte, over TLS</h1>
       <li id="description">This page performs a live, annotated HTTPS request for its own source. It’s inspired by <a href="https://tls13.xargs.org/">The Illustrated TLS 1.3 Connection</a> and Julia Evans’ <a href="https://jvns.ca/blog/2022/03/23/a-toy-version-of-tls/">toy TLS 1.3</a>.</li>
       <li>It’s built on <a href="https://github.com/jawj/subtls">subtls</a>, a pure-JS TLS 1.3 implementation that depends only on <a href="https://developer.mozilla.org/en-US/docs/Web/API/SubtleCrypto">SubtleCrypto</a>. Raw TCP traffic is carried via a simple <a href="https://github.com/jawj/subtls/blob/main/subtls-wsproxy/worker.ts">WebSocket proxy</a>.</li>
     </ul>
-    <p><span class="hex" style="color: #111"><span style="color: #fff; background: #777"> Key </span> Raw bytes in hexadecimal.</span> <span class="hex" style="color: #8cc">Outgoing message annotations.</span> <span class="hex" style="color: #88c">Incoming message annotations.</span></p>
+    <p>
+      <span class="hex" style="color: #111"><span style="color: #fff; background: #777"> Key </span> Raw bytes in hexadecimal or ++ = new TLS record begins.</span> <span class="hex" style="color: #8cc">Outgoing message annotations.</span> <span class="hex" style="color: #88c">Incoming message annotations.</span>
+    </p>
     <p><input id="go" type="button" value="Get this page, byte by byte"></p>
   </div>
 

docs/index.js

@@ -425,9 +425,11 @@ var Bytes = class {
     __publicField(this, "data");
     __publicField(this, "comments");
     __publicField(this, "indents");
+    __publicField(this, "fetchPoints");
     this.endOfReadableData = this.offset = 0;
     this.comments = {};
     this.indents = { 0: indent };
+    this.fetchPoints = /* @__PURE__ */ new Set();
     if (typeof data === "number") {
       this.data = new Uint8Array(data);
     } else if (data === void 0 || typeof data === "function") {
@@ -466,6 +468,7 @@ var Bytes = class {
       e._bytes_error_reason = "EOF";
       throw e;
     }
+    for (const fetchPoint of newData.fetchPoints ?? []) this.fetchPoints.add(fetchPoint + this.endOfReadableData);
     this.data.set(newData, this.endOfReadableData);
     this.endOfReadableData += newData.length;
   }
@@ -770,20 +773,24 @@ var Bytes = class {
         if (i < len - 1) s += `
 ${indentChars.repeat(indent)}`;
       }
+      if (this.fetchPoints.has(i + 1)) s += "--- next TLS record ---\n";
     }
     return s;
   }
 };
 
 // src/presentation/highlights.ts
-var regex = new RegExp(`  .+|^(${indentChars})+`, "gm");
-var dotColour = "color: #ddd";
+var regex = new RegExp(`  .+|^(${indentChars})+|--- next TLS record ---`, "gm");
+var dotColour = "color: #ccc";
 var textColour = "color: #111";
 var mutedColour = "color: #777";
 function highlightBytes(s, colour) {
   const css = [textColour];
   s = "%c" + s.replace(regex, (m) => {
-    css.push(m.startsWith(indentChars) ? dotColour : `color: ${colour}`, textColour);
+    css.push(
+      m === "--- next TLS record ---" || m.startsWith(indentChars) ? dotColour : `color: ${colour}`,
+      textColour
+    );
     return `%c\u200B${m}\u200B%c`;
   });
   return [s, ...css];
@@ -1690,15 +1697,19 @@ var LazyReadFunctionReadQueue = class extends ReadQueue {
     __publicField(this, "dataIsExhausted", false);
   }
   async read(bytes, readMode = 0 /* CONSUME */) {
+    const fetchPoints = [];
     while (this.bytesInQueue() < bytes) {
-      const data = await this.readFn();
-      if (data === void 0) {
+      fetchPoints.push(this.bytesInQueue());
+      const data2 = await this.readFn();
+      if (data2 === void 0) {
         this.dataIsExhausted = true;
         break;
       }
-      if (data.length > 0) this.enqueue(data);
+      if (data2.length > 0) this.enqueue(data2);
     }
-    return super.read(bytes, readMode);
+    const data = await super.read(bytes, readMode);
+    data.fetchPoints = fetchPoints;
+    return data;
   }
   moreDataMayFollow() {
     return !this.dataIsExhausted;

index.d.ts

@@ -29,6 +29,7 @@ export declare class Bytes {
     data: Uint8Array;
     comments: Record<number, string>;
     indents: Record<number, number>;
+    fetchPoints: Set<number>;
     /**
      * @param data -
      * * If data is a `Uint8Array`, this is the initial data
@@ -196,7 +197,7 @@ export declare class LazyReadFunctionReadQueue extends ReadQueue {
     protected readFn: () => Promise<Uint8Array | undefined>;
     protected dataIsExhausted: boolean;
     constructor(readFn: () => Promise<Uint8Array | undefined>);
-    read(bytes: number, readMode?: ReadMode): Promise<Uint8Array<ArrayBufferLike> | undefined>;
+    read(bytes: number, readMode?: ReadMode): Promise<Uint8ArrayWithFetchPoints>;
     moreDataMayFollow(): boolean;
 }
 
@@ -290,6 +291,10 @@ export declare class TrustedCert extends Cert {
 
 export declare function u8FromHex(hex: string): Uint8Array<ArrayBuffer>;
 
+export declare interface Uint8ArrayWithFetchPoints extends Uint8Array {
+    fetchPoints?: number[];
+}
+
 export declare interface WebSocketOptions {
     close?: () => void;
 }

index.js

@@ -550,9 +550,11 @@ var Bytes = class {
     __publicField(this, "data");
     __publicField(this, "comments");
     __publicField(this, "indents");
+    __publicField(this, "fetchPoints");
     this.endOfReadableData = this.offset = 0;
     this.comments = {};
     this.indents = { 0: indent };
+    this.fetchPoints = /* @__PURE__ */ new Set();
     if (typeof data === "number") {
       this.data = new Uint8Array(data);
     } else if (data === void 0 || typeof data === "function") {
@@ -592,6 +594,7 @@ var Bytes = class {
       e._bytes_error_reason = "EOF";
       throw e;
     }
+    for (const fetchPoint of newData.fetchPoints ?? []) this.fetchPoints.add(fetchPoint + this.endOfReadableData);
     this.data.set(newData, this.endOfReadableData);
     this.endOfReadableData += newData.length;
   }
@@ -886,6 +889,7 @@ lows"}` : `${length === 0 ? "no" : length} bytes${comment ? ` of ${comment}` : "
         if (i < len - 1) s += `
 ${indentChars.repeat(indent)}`;
       }
+      if (this.fetchPoints.has(i + 1)) s += "--- next TLS record ---\n";
     }
     return s;
   }
@@ -1554,7 +1558,7 @@ string", comment);
 var appendLog = Symbol("append");
 
 // src/presentation/highlights.ts
-var regex = new RegExp(`  .+|^(${indentChars})+`, "gm");
+var regex = new RegExp(`  .+|^(${indentChars})+|--- next TLS record ---`, "gm");
 
 // src/util/readQueue.ts
 var ReadMode = /* @__PURE__ */ ((ReadMode2) => {
@@ -1663,15 +1667,19 @@ var LazyReadFunctionReadQueue = class extends ReadQueue {
     __publicField(this, "dataIsExhausted", false);
   }
   async read(bytes, readMode = 0 /* CONSUME */) {
+    const fetchPoints = [];
     while (this.bytesInQueue() < bytes) {
-      const data = await this.readFn();
-      if (data === void 0) {
+      fetchPoints.push(this.bytesInQueue());
+      const data2 = await this.readFn();
+      if (data2 === void 0) {
         this.dataIsExhausted = true;
         break;
       }
-      if (data.length > 0) this.enqueue(data);
+      if (data2.length > 0) this.enqueue(data2);
     }
-    return super.read(bytes, readMode);
+    const data = await super.read(bytes, readMode);
+    data.fetchPoints = fetchPoints;
+    return data;
   }
   moreDataMayFollow() {
     return !this.dataIsExhausted;

src/export.ts

@@ -2,7 +2,7 @@ export * from 'hextreme';
 export { startTls } from './tls/startTls';
 export { Cert, TrustedCert, RootCertsDatabase, RootCertsIndex, RootCertsData, DistinguishedName, OID, CertJSON, allKeyUsages } from './tls/cert';
 export { getRootCertsDatabase } from './util/rootCerts';
-export { WebSocketReadQueue, SocketReadQueue, LazyReadFunctionReadQueue, ReadQueue, ReadMode, DataRequest } from './util/readQueue';
+export { WebSocketReadQueue, SocketReadQueue, LazyReadFunctionReadQueue, ReadQueue, ReadMode, DataRequest, type Uint8ArrayWithFetchPoints } from './util/readQueue';
 export { hexFromU8, u8FromHex } from './util/hex';
 export { default as stableStringify } from './util/stableStringify';
 export { Bytes } from './util/bytes';

src/presentation/highlights.ts

@@ -1,15 +1,18 @@
 
 import { indentChars } from './appearance';
 
-const regex = new RegExp(`  .+|^(${indentChars})+`, 'gm');
-const dotColour = 'color: #ddd';
+const regex = new RegExp(`  .+|^(${indentChars})+|--- next TLS record ---`, 'gm');
+const dotColour = 'color: #ccc';
 export const textColour = 'color: #111';
 export const mutedColour = 'color: #777';
 
 export function highlightBytes(s: string, colour: string) {
   const css: string[] = [textColour];
   s = '%c' + s.replace(regex, m => {
-    css.push(m.startsWith(indentChars) ? dotColour : `color: ${colour}`, textColour);
+    css.push(
+      m === '--- next TLS record ---' || m.startsWith(indentChars) ? dotColour : `color: ${colour}`,
+      textColour
+    );
     return `%c\u200b${m}\u200b%c`;  // note: the zero-length spaces, \u200b, prevent URLs getting mangled
   });
   return [s, ...css];

src/util/bytes.ts

@@ -1,5 +1,6 @@
 import { equal } from './array';
 import { indentChars } from '../presentation/appearance';
+import type { Uint8ArrayWithFetchPoints } from './readQueue';
 
 const initialSize = 1024;
 const growthFactor = 2;
@@ -19,6 +20,7 @@ export class Bytes {
   data: Uint8Array;
   comments: Record<number, string>;
   indents: Record<number, number>;
+  fetchPoints: Set<number>;
 
   /**
    * @param data -
@@ -31,6 +33,7 @@ export class Bytes {
 
     this.comments = {};
     this.indents = { 0: indent };
+    this.fetchPoints = new Set();
 
     if (typeof data === 'number') {
       this.data = new Uint8Array(data);
@@ -68,12 +71,13 @@ export class Bytes {
       );
       this.resizeTo(newSize);
     }
-    const newData = await this.fetchFn(bytes);
+    const newData = await this.fetchFn(bytes) as Uint8ArrayWithFetchPoints;
     if (newData === undefined || newData.length < bytes) {
       const e = new Error(`Not enough data: requested ${bytes} byte(s), received ${newData === undefined ? 'EOF' : `${newData.length} byte(s)`}`);
       (e as any)._bytes_error_reason = 'EOF';
       throw e;
     }
+    for (const fetchPoint of newData.fetchPoints ?? []) this.fetchPoints.add(fetchPoint + this.endOfReadableData);
     this.data.set(newData, this.endOfReadableData);
     this.endOfReadableData += newData.length;
   }
@@ -432,6 +436,7 @@ export class Bytes {
         s += ` ${comment}`;
         if (i < len - 1) s += `\n${indentChars.repeat(indent)}`;
       }
+      if (this.fetchPoints.has(i + 1)) s += '--- next TLS record ---\n';
     }
     return s;
   }

src/util/readQueue.ts

@@ -134,6 +134,10 @@ export class SocketReadQueue extends ReadQueue {
   }
 }
 
+export interface Uint8ArrayWithFetchPoints extends Uint8Array {
+  fetchPoints?: number[];
+}
+
 export class LazyReadFunctionReadQueue extends ReadQueue {
   protected dataIsExhausted = false;
 
@@ -142,15 +146,19 @@ export class LazyReadFunctionReadQueue extends ReadQueue {
   }
 
   override async read(bytes: number, readMode = ReadMode.CONSUME) {
+    const fetchPoints = [];
     while (this.bytesInQueue() < bytes) {
+      fetchPoints.push(this.bytesInQueue());
       const data = await this.readFn();
       if (data === undefined) {
         this.dataIsExhausted = true;
         break;
       }
       if (data.length > 0) this.enqueue(data);
     }
-    return super.read(bytes, readMode);
+    const data = await super.read(bytes, readMode) as Uint8ArrayWithFetchPoints;
+    data.fetchPoints = fetchPoints;
+    return data;
   }
 
   moreDataMayFollow(): boolean {