fix: Tokenization will consider the file type

getappmap · Jan 21, 2025 · eb0a840 · eb0a840
1 parent 5db40a8
commit eb0a840
Show file tree

Hide file tree

Showing 2 changed files with 4 additions and 2 deletions.
diff --git a/packages/search/src/build-file-index.ts b/packages/search/src/build-file-index.ts
@@ -37,7 +37,8 @@ async function indexFile(context: Context, filePath: string) {
     fileContents.length,
     fileContents.slice(0, 40)
   );
-  const tokens = context.tokenizer(fileContents, filePath);
+  const fileExtension = filePath.split('.').pop() ?? '';
+  const tokens = context.tokenizer(fileContents, fileExtension);
   const symbols = tokens.symbols.join(' ');
   const words = tokens.words.join(' ');
 

diff --git a/packages/search/src/build-snippet-index.ts b/packages/search/src/build-snippet-index.ts
@@ -28,7 +28,8 @@ async function indexFile(context: Context, file: File) {
   chunks.forEach((chunk) => {
     const { content, startLine } = chunk;
     const snippetId = fileChunkSnippetId(filePath, startLine);
-    const { symbols, words } = context.tokenizer(content, file.filePath);
+    const fileExtension = file.filePath.split('.').pop() ?? '';
+    const { symbols, words } = context.tokenizer(content, fileExtension);
     context.snippetIndex.indexSnippet(
       snippetId,
       file.directory,