From fa2c2e5d3f27f18b836cb7f579628ea2efd662f2 Mon Sep 17 00:00:00 2001 From: Stephan Janssen Date: Fri, 13 Dec 2024 18:34:55 +0100 Subject: [PATCH] Fix #387 : Use countTokensOrdinary and encodeOrdinary --- .../controller/ProjectContextController.java | 4 +- .../projectscanner/ProjectScannerService.java | 39 +++++++++---------- src/main/resources/application.properties | 2 +- 3 files changed, 22 insertions(+), 23 deletions(-) diff --git a/src/main/java/com/devoxx/genie/controller/ProjectContextController.java b/src/main/java/com/devoxx/genie/controller/ProjectContextController.java index 0778c57..db437c7 100644 --- a/src/main/java/com/devoxx/genie/controller/ProjectContextController.java +++ b/src/main/java/com/devoxx/genie/controller/ProjectContextController.java @@ -69,7 +69,9 @@ public void addProjectContext() { projectContext = "Project Context:\n" + projectContent.getContent(); isProjectContextAdded = true; ApplicationManager.getApplication().invokeLater(() -> { - tokenCount = Encodings.newDefaultEncodingRegistry().getEncoding(EncodingType.CL100K_BASE).countTokens(projectContent.getContent()); + tokenCount = Encodings.newDefaultEncodingRegistry() + .getEncoding(EncodingType.CL100K_BASE) + .countTokensOrdinary(projectContent.getContent()); actionButtonsPanel.updateAddProjectButton(isProjectContextAdded, tokenCount); actionButtonsPanel.setAddProjectButtonEnabled(true); actionButtonsPanel.updateTokenUsageBar(tokenCount, tokenLimit); diff --git a/src/main/java/com/devoxx/genie/service/projectscanner/ProjectScannerService.java b/src/main/java/com/devoxx/genie/service/projectscanner/ProjectScannerService.java index e620aad..84a3fa2 100644 --- a/src/main/java/com/devoxx/genie/service/projectscanner/ProjectScannerService.java +++ b/src/main/java/com/devoxx/genie/service/projectscanner/ProjectScannerService.java @@ -28,7 +28,6 @@ import java.util.Arrays; import java.util.Objects; import java.util.concurrent.CompletableFuture; -import java.util.concurrent.atomic.AtomicInteger; public class ProjectScannerService { @@ -60,20 +59,20 @@ public CompletableFuture scanProject(Project project, if (startDirectory == null) { result.append("Directory Structure:\n"); - fullContent = getContentFromModules(project, windowContextMaxTokens, result, scanContentResult); + fullContent = getContentFromModules(project, result, scanContentResult); } else if (startDirectory.isDirectory()) { result.append("Directory Structure:\n"); - fullContent = processDirectory(project, startDirectory, result, scanContentResult, windowContextMaxTokens); + fullContent = processDirectory(project, startDirectory, result, scanContentResult); } else { // Handle the case where startDirectory is a file result.append("File:\n"); - fullContent = processSingleFile(project, startDirectory, result, scanContentResult); + fullContent = processSingleFile(startDirectory, result, scanContentResult); } String content = isTokenCalculation ? fullContent.toString() : truncateToTokens(fullContent.toString(), windowContextMaxTokens, isTokenCalculation); - scanContentResult.setTokenCount(ENCODING.countTokens(content)); + scanContentResult.setTokenCount(ENCODING.countTokensOrdinary(content)); scanContentResult.setContent(content); return scanContentResult; @@ -98,14 +97,14 @@ public ScanContentResult scanProjectSynchronously(Project project, VirtualFile s if (startDirectory == null) { result.append("Directory Structure:\n"); - fullContent = getContentFromModules(project, windowContextMaxTokens, result, scanContentResult); + fullContent = getContentFromModules(project, result, scanContentResult); } else if (startDirectory.isDirectory()) { result.append("Directory Structure:\n"); - fullContent = processDirectory(project, startDirectory, result, scanContentResult, windowContextMaxTokens); + fullContent = processDirectory(project, startDirectory, result, scanContentResult); } else { // Handle the case where startDirectory is a file result.append("File:\n"); - fullContent = processSingleFile(project, startDirectory, result, scanContentResult); + fullContent = processSingleFile(startDirectory, result, scanContentResult); } String content = isTokenCalculation ? fullContent.toString() : truncateToTokens(fullContent.toString(), windowContextMaxTokens, isTokenCalculation); @@ -136,7 +135,6 @@ private void initGitignoreParser(Project project, VirtualFile startDirectory) { } private StringBuilder getContentFromModules(Project project, - int windowContextMaxTokens, StringBuilder result, ScanContentResult scanContentResult) { @@ -157,16 +155,14 @@ private StringBuilder getContentFromModules(Project project, .map(highestCommonRoot -> processDirectory(project, highestCommonRoot, result, - scanContentResult, - windowContextMaxTokens)) + scanContentResult)) .orElseThrow(); } private @NotNull StringBuilder processDirectory(Project project, VirtualFile startDirectory, @NotNull StringBuilder result, - ScanContentResult scanContentResult, - int windowContextMaxTokens) { + ScanContentResult scanContentResult) { result.append(generateSourceTreeRecursive(startDirectory, 0)); result.append("\n\nFile Contents:\n"); @@ -174,14 +170,13 @@ private StringBuilder getContentFromModules(Project project, ProjectFileIndex fileIndex = ProjectFileIndex.getInstance(project); StringBuilder fullContent = new StringBuilder(result); - AtomicInteger currentTokens = new AtomicInteger(0); - walkThroughDirectory(startDirectory, fileIndex, fullContent, currentTokens, scanContentResult); + walkThroughDirectory(startDirectory, fileIndex, fullContent, scanContentResult); + return fullContent; } - private @NotNull StringBuilder processSingleFile(Project project, - VirtualFile file, + private @NotNull StringBuilder processSingleFile(@NotNull VirtualFile file, @NotNull StringBuilder result, ScanContentResult scanContentResult) { result.append(file.getName()).append("\n"); @@ -200,7 +195,6 @@ private StringBuilder getContentFromModules(Project project, private void walkThroughDirectory(@NotNull VirtualFile directory, @NotNull ProjectFileIndex fileIndex, @NotNull StringBuilder fullContent, - @NotNull AtomicInteger currentTokens, @NotNull ScanContentResult scanContentResult) { VfsUtilCore.visitChildrenRecursively(directory, new VirtualFileVisitor() { @@ -224,7 +218,9 @@ public boolean visitFile(@NotNull VirtualFile file) { }); } - private void readFileContent(@NotNull VirtualFile file, @NotNull StringBuilder fullContent, @NotNull ScanContentResult scanContentResult) { + private void readFileContent(@NotNull VirtualFile file, + @NotNull StringBuilder fullContent, + @NotNull ScanContentResult scanContentResult) { scanContentResult.incrementFileCount(); scanContentResult.addFile(Paths.get(file.getPath())); @@ -249,10 +245,11 @@ private void readFileContent(@NotNull VirtualFile file, @NotNull StringBuilder f } } - private String truncateToTokens(String text, + private String truncateToTokens(@NotNull String text, int windowContext, boolean isTokenCalculation) { - IntArrayList tokens = ENCODING.encode(text); + + IntArrayList tokens = ENCODING.encodeOrdinary(text); if (tokens.size() <= windowContext) { return text; } diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index 7319b37..1ddbcc4 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -1,2 +1,2 @@ -#Thu Dec 12 19:55:03 CET 2024 +#Fri Dec 13 18:33:45 CET 2024 version=0.4.5