From 3d0d0c68caae86d75541182f5560f6a970c3e420 Mon Sep 17 00:00:00 2001 From: Alicia Machado Date: Mon, 27 Jan 2025 09:49:37 +0100 Subject: [PATCH] Clarify some TODOs and comments on the code. --- animated-transformer/src/lib/transformer/gpt2.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/animated-transformer/src/lib/transformer/gpt2.ts b/animated-transformer/src/lib/transformer/gpt2.ts index 0f27d3a..052f1dd 100644 --- a/animated-transformer/src/lib/transformer/gpt2.ts +++ b/animated-transformer/src/lib/transformer/gpt2.ts @@ -490,7 +490,6 @@ export function computePredictionWithLoadedTokenizer( // We tokenize directly with the preprocessing function from gpt-tokenizer. inputs: string[], generator: RandomStream - // TODO(@aliciafmachado): save the input as well, and split in two functions: one that tokenizes and one that doesn't. ): string[] { // Encode inputs using the r50k_base.encode which is the tokenizer used for GPT2. // TODO(@aliciafmachado): There is no clear padding in the vocabulary of GPT2. We are currently using