diff --git a/frontend/src/app/modules/agents/agent/agent-details/agent-details.component.ts b/frontend/src/app/modules/agents/agent/agent-details/agent-details.component.ts index 73ba82d4..abd886ad 100644 --- a/frontend/src/app/modules/agents/agent/agent-details/agent-details.component.ts +++ b/frontend/src/app/modules/agents/agent/agent-details/agent-details.component.ts @@ -236,10 +236,9 @@ export class AgentDetailsComponent implements OnInit { } getLlmName(llmId: string): string { - if (!llmId) { - return 'Unknown'; - } - return this.llmNameMap.get(llmId) || `Unknown LLM (${llmId})`; + if (!llmId) return 'Unknown'; + + return this.llmNameMap.get(llmId) || llmId; } openFunctionEditModal(): void { diff --git a/src/agent/agentContextLocalStorage.ts b/src/agent/agentContextLocalStorage.ts index 02ea7490..7e9c1855 100644 --- a/src/agent/agentContextLocalStorage.ts +++ b/src/agent/agentContextLocalStorage.ts @@ -25,7 +25,7 @@ export function llms(): AgentLLMs { export function addCost(cost: number) { const store = agentContextStorage.getStore(); if (!store) return; - logger.debug(`Adding cost $${cost}`); + logger.debug(`Adding cost $${cost.toFixed(6)}`); store.cost += cost; store.budgetRemaining -= cost; } @@ -81,6 +81,7 @@ export function createContext(config: RunAgentConfig): AgentContext { memory: {}, invoking: [], lastUpdate: Date.now(), + liveFiles: [], }; return context; } diff --git a/src/agent/agentContextTypes.ts b/src/agent/agentContextTypes.ts index adf4d9d2..598d3212 100644 --- a/src/agent/agentContextTypes.ts +++ b/src/agent/agentContextTypes.ts @@ -133,4 +133,6 @@ export interface AgentContext { functionCallHistory: FunctionCallResult[]; /** How many iterations of the autonomous agent control loop to require human input to continue */ hilCount; + /** Files which are always provided in the agent control loop prompt */ + liveFiles: string[]; } diff --git a/src/agent/agentSerialization.ts b/src/agent/agentSerialization.ts index 760d7009..349c99ed 100644 --- a/src/agent/agentSerialization.ts +++ b/src/agent/agentSerialization.ts @@ -23,7 +23,7 @@ export function serializeContext(context: AgentContext): Record { else if (typeof context[key] === 'string' || typeof context[key] === 'number' || typeof context[key] === 'boolean') { serialized[key] = context[key]; } - // Assume arrays (functionCallHistory) can be directly de(serialised) to JSON + // Assume arrays (functionCallHistory, liveFiles) can be directly de(serialised) to JSON else if (Array.isArray(context[key])) { serialized[key] = context[key]; } diff --git a/src/agent/liveFileFunctions.ts b/src/agent/liveFileFunctions.ts new file mode 100644 index 00000000..7d53f547 --- /dev/null +++ b/src/agent/liveFileFunctions.ts @@ -0,0 +1,37 @@ +import { agentContext } from '#agent/agentContextLocalStorage'; +import { func, funcClass } from '#functionSchema/functionDecorators'; + +export const LIVE_FILES_ADD = 'LiveFiles_addFiles'; + +export const LIVE_FILES_REMOVE = 'LiveFiles_removeFiles'; + +/** + * Functions for the agent to add/remove files which always displays the current file contents in the agent control prompt + */ +@funcClass(__filename) +export class LiveFileFunctions { + /** + * Add files which will always have their current contents displayed in the section (increasing LLM token costs) + * @param {string[]} files the files to always include the current contents of in the prompt + */ + @func() + async addFiles(files: string[]): Promise { + const agent = agentContext(); + agent.liveFiles = Array.from(new Set(...agent.liveFiles, ...files)); + return ''; + } + + /** + * Remove files from the section which are no longer required to reduce LLM token costs. + * @param {string[]} files The files to remove + */ + @func() + async removeFiles(files: string[]): Promise { + const agent = agentContext(); + const liveFiles = new Set(agent.liveFiles); + for (const f of files) { + liveFiles.delete(f); + } + agent.liveFiles = Array.from(liveFiles); + } +} diff --git a/src/agent/xmlAgentRunner.ts b/src/agent/xmlAgentRunner.ts index 6911de7e..cd97ad1a 100644 --- a/src/agent/xmlAgentRunner.ts +++ b/src/agent/xmlAgentRunner.ts @@ -9,6 +9,7 @@ import { humanInTheLoop, notifySupervisor } from '#agent/humanInTheLoop'; import { getServiceName } from '#fastify/trace-init/trace-init'; import { FunctionSchema, getAllFunctionSchemas } from '#functionSchema/functions'; import { FunctionResponse } from '#llm/llm'; +import { parseFunctionCallsXml } from '#llm/responseParsers'; import { logger } from '#o11y/logger'; import { withActiveSpan } from '#o11y/trace'; import { envVar } from '#utils/env-var'; @@ -110,18 +111,27 @@ export async function runXmlAgent(agent: AgentContext): Promise } let functionResponse: FunctionResponse; + let llmResponse: string; try { - functionResponse = await agentLLM.generateFunctionResponse(systemPromptWithFunctions, currentPrompt, { + llmResponse = await agentLLM.generateText(systemPromptWithFunctions, currentPrompt, { id: 'generateFunctionCalls', stopSequences, }); + functionResponse = { + textResponse: llmResponse, + functions: parseFunctionCallsXml(llmResponse), + }; } catch (e) { // Should just catch parse error const retryPrompt = `${currentPrompt}\nNote: Your previous response did not contain the response in the required format of .... You must reply in the correct response format.`; - functionResponse = await agentLLM.generateFunctionResponse(systemPromptWithFunctions, retryPrompt, { + llmResponse = await agentLLM.generateText(systemPromptWithFunctions, retryPrompt, { id: 'generateFunctionCalls-retryError', stopSequences, }); + functionResponse = { + textResponse: llmResponse, + functions: parseFunctionCallsXml(llmResponse), + }; } currentPrompt = buildFunctionCallHistoryPrompt('history') + buildMemoryPrompt() + filePrompt + userRequestXml + functionResponse.textResponse; const functionCalls = functionResponse.functions.functionCalls; @@ -132,10 +142,15 @@ export async function runXmlAgent(agent: AgentContext): Promise const retryPrompt = `${currentPrompt} Note: Your previous response did not contain a function call. If you are able to answer/complete the question/task, then call the ${AGENT_COMPLETED_NAME} function with the appropriate response. If you are unsure what to do next then call the ${AGENT_REQUEST_FEEDBACK} function with a clarifying question.`; - const functionCallResponse: FunctionResponse = await agentLLM.generateFunctionResponse(systemPromptWithFunctions, retryPrompt, { + + llmResponse = await agentLLM.generateText(systemPromptWithFunctions, retryPrompt, { id: 'generateFunctionCalls-retryNoFunctions', stopSequences, }); + const functionCallResponse: FunctionResponse = { + textResponse: llmResponse, + functions: parseFunctionCallsXml(llmResponse), + }; // retrying currentPrompt = buildFunctionCallHistoryPrompt('history') + buildMemoryPrompt() + filePrompt + userRequestXml + functionCallResponse.textResponse; const functionCalls = functionCallResponse.functions.functionCalls; diff --git a/src/functionSchema/functionSchemaParser.ts b/src/functionSchema/functionSchemaParser.ts index 5e8bc685..94a888f4 100644 --- a/src/functionSchema/functionSchemaParser.ts +++ b/src/functionSchema/functionSchemaParser.ts @@ -85,10 +85,11 @@ export function functionSchemaParser(sourceFilePath: string): Record sourceUpdatedTimestamp) { try { const json = readFileSync(`${cachedPath}.json`).toString(); - logger.debug(`Loading cached function schemas from ${cachedPath}.json`); + if (logger) logger.debug(`Loading cached function schemas from ${cachedPath}.json`); return JSON.parse(json); } catch (e) { - logger.info('Error loading cached function schemas: ', e.message); + if (logger) logger.info(`Error loading cached function schemas: ${e.message}`); + else console.log(`Error loading cached function schemas: ${e.message}`); } } diff --git a/src/functions/storage/fileSystemService.ts b/src/functions/storage/fileSystemService.ts index bd4ffffd..21b02def 100644 --- a/src/functions/storage/fileSystemService.ts +++ b/src/functions/storage/fileSystemService.ts @@ -12,7 +12,7 @@ import { VersionControlSystem } from '#functions/scm/versionControlSystem'; import { LlmTools } from '#functions/util'; import { logger } from '#o11y/logger'; import { getActiveSpan, span } from '#o11y/trace'; -import { spawnCommand } from '#utils/exec'; +import { execCmd, spawnCommand } from '#utils/exec'; import { CDATA_END, CDATA_START, needsCDATA } from '#utils/xml-utils'; import { SOPHIA_FS } from '../../appVars'; @@ -189,15 +189,20 @@ export class FileSystemService { * @returns the list of file and folder names */ async listFilesInDirectory(dirPath = '.'): Promise { - // const rootPath = path.join(this.basePath, dirPath); const filter: FileFilter = (name) => true; const ig = ignore(); - // TODO should go up the directories to the base path looking for .gitignore files - const gitIgnorePath = path.join(this.getWorkingDirectory(), dirPath, '.gitignore'); - // console.log(gitIgnorePath); + + // Determine the correct path based on whether dirPath is absolute or relative + let readdirPath: string; + if (path.isAbsolute(dirPath)) { + readdirPath = dirPath; + } else { + readdirPath = path.join(this.getWorkingDirectory(), dirPath); + } + + // Load .gitignore rules if present + const gitIgnorePath = path.join(readdirPath, '.gitignore'); if (existsSync(gitIgnorePath)) { - // read the gitignore file into a string array - // console.log(`Found ${gitIgnorePath}`); let lines = await fs.readFile(gitIgnorePath, 'utf8').then((data) => data.split('\n')); lines = lines.map((line) => line.trim()).filter((line) => line.length && !line.startsWith('#'), filter); ig.add(lines); @@ -206,17 +211,22 @@ export class FileSystemService { const files: string[] = []; - const readdirPath = join(this.getWorkingDirectory(), dirPath); - const dirents = await fs.readdir(readdirPath, { withFileTypes: true }); - for (const dirent of dirents) { - const direntName = dirent.isDirectory() ? `${dirent.name}/` : dirent.name; - const relativePath = path.relative(this.getWorkingDirectory(), path.join(this.getWorkingDirectory(), dirPath, direntName)); + try { + const dirents = await fs.readdir(readdirPath, { withFileTypes: true }); + for (const dirent of dirents) { + const direntName = dirent.isDirectory() ? `${dirent.name}/` : dirent.name; + const relativePath = path.relative(this.getWorkingDirectory(), path.join(readdirPath, direntName)); - if (!ig.ignores(relativePath)) { - files.push(dirent.name); + if (!ig.ignores(relativePath)) { + files.push(dirent.name); + } } + } catch (error) { + console.error('Error reading directory:', error); + throw error; // Re-throw the error to be caught by the caller } - return files; //files.map((file) => file.substring(file.lastIndexOf(path.sep, file.length - 1))); + + return files; } /** @@ -227,7 +237,7 @@ export class FileSystemService { async listFilesRecursively(dirPath = './', useGitIgnore = true): Promise { this.log.debug(`listFilesRecursively cwd: ${this.workingDirectory}`); - const startPath = path.join(this.getWorkingDirectory(), dirPath); + const startPath = path.isAbsolute(dirPath) ? dirPath : path.join(this.getWorkingDirectory(), dirPath); // TODO check isnt going higher than this.basePath const ig = useGitIgnore ? await this.loadGitignoreRules(startPath) : ignore(); @@ -441,7 +451,9 @@ export class FileSystemService { async listFolders(dirPath = './'): Promise { const workingDir = this.getWorkingDirectory(); - dirPath = path.join(workingDir, dirPath); + if (!path.isAbsolute(dirPath)) { + dirPath = path.join(workingDir, dirPath); + } try { const items = await fs.readdir(dirPath); const folders: string[] = []; @@ -450,8 +462,7 @@ export class FileSystemService { const itemPath = path.join(dirPath, item); const stat = await fs.stat(itemPath); if (stat.isDirectory()) { - const relativePath = path.relative(workingDir, itemPath); - folders.push(relativePath); + folders.push(item); // Return only the subfolder name } } return folders; @@ -568,6 +579,24 @@ export class FileSystemService { return tree; } + + async getGitRoot(): Promise { + try { + // Use git rev-parse to get the root directory + const result = await execCmd('git rev-parse --show-toplevel'); + + // If command succeeds, return the trimmed stdout (git root path) + if (!result.error) { + return result.stdout.trim(); + } + + // If git command fails, return null + return null; + } catch { + // Any unexpected errors also result in null + return null; + } + } } /** diff --git a/src/llm/base-llm.ts b/src/llm/base-llm.ts index c5f90771..e6a8d106 100644 --- a/src/llm/base-llm.ts +++ b/src/llm/base-llm.ts @@ -1,24 +1,29 @@ import { StreamTextResult } from 'ai'; import { AgentContext } from '#agent/agentContextTypes'; import { countTokens } from '#llm/tokens'; -import { FunctionResponse, GenerateFunctionOptions, GenerateJsonOptions, GenerateTextOptions, LLM, LlmMessage } from './llm'; -import { extractJsonResult, extractStringResult, parseFunctionCallsXml } from './responseParsers'; +import { GenerateJsonOptions, GenerateTextOptions, LLM, LlmMessage } from './llm'; +import { extractJsonResult, extractStringResult } from './responseParsers'; export interface SerializedLLM { service: string; model: string; } +export type InputCostFunction = (input: string, inputTokens: number, usage?: any) => number; +export type OutputCostFunction = (output: string, outputTokens: number) => number; + +export function perMilTokens(dollarsPerMillionTokens: number): InputCostFunction { + return (_, tokens) => (tokens * dollarsPerMillionTokens) / 1_000_000; +} + export abstract class BaseLLM implements LLM { constructor( protected readonly displayName: string, protected readonly service: string, protected readonly model: string, - private maxInputTokens: number, - /** Needed for Aider when we only have the text size */ - public readonly calculateInputCost: (input: string) => number, - /** Needed for Aider when we only have the text size */ - public readonly calculateOutputCost: (output: string) => number, + protected maxInputTokens: number, + readonly calculateInputCost: InputCostFunction, + readonly calculateOutputCost: OutputCostFunction, ) {} protected _generateText(systemPrompt: string | undefined, userPrompt: string, opts?: GenerateTextOptions): Promise { @@ -85,14 +90,6 @@ export abstract class BaseLLM implements LLM { return this.generateTextFromMessages(messages, options); } - async generateFunctionResponse(systemPrompt: string, prompt: string, opts?: GenerateFunctionOptions): Promise { - const response = await this._generateText(systemPrompt, prompt, opts); - return { - textResponse: response, - functions: parseFunctionCallsXml(response), - }; - } - generateTextWithResult(userPrompt: string, opts?: GenerateTextOptions): Promise; generateTextWithResult(systemPrompt: string, userPrompt: string, opts?: GenerateTextOptions): Promise; generateTextWithResult(messages: LlmMessage[], opts?: GenerateTextOptions): Promise; @@ -169,13 +166,6 @@ export abstract class BaseLLM implements LLM { return this.displayName; } - calculateCost(input: string, output: string): [totalCost: number, inputCost: number, outputCost: number] { - const inputCost = this.calculateInputCost(input); - const outputCost = this.calculateOutputCost(output); - const totalCost = inputCost + outputCost; - return [totalCost, inputCost, outputCost]; - } - countTokens(text: string): Promise { // defaults to gpt4o token parser return countTokens(text); diff --git a/src/llm/llm.ts b/src/llm/llm.ts index e6613c6e..5a9a89e5 100644 --- a/src/llm/llm.ts +++ b/src/llm/llm.ts @@ -146,15 +146,7 @@ export interface LLM { */ generateTextWithResult(prompt: string, opts?: GenerateTextOptions): Promise; generateTextWithResult(systemPrompt: string, prompt: string, opts?: GenerateTextOptions): Promise; - generateTextWithResult(messages: LlmMessage[], opts?: GenerateTextOptions): Promise; - - /** - * Generates a response expecting to contain the element matching the FunctionResponse type - * @param systemPrompt - * @param userPrompt - * @param opts - */ - generateFunctionResponse(systemPrompt: string, userPrompt: string, opts?: GenerateFunctionOptions): Promise; + generateTextWithResult(messages: LlmMessage[], opts?: GenerateTextOptions): Promise; /** * Streams text from the LLM @@ -185,13 +177,6 @@ export interface LLM { /** The maximum number of input tokens */ getMaxInputTokens(): number; - /** - * Calculate costs for generation - * @param input the input text - * @param output the output text - */ - calculateCost(input: string, output: string): [totalCost: number, inputCost: number, outputCost: number]; - /** * @param text * @returns the number of tokens in the text for this LLM diff --git a/src/llm/services/ai-llm.ts b/src/llm/services/ai-llm.ts index 618262cc..0bc5733e 100644 --- a/src/llm/services/ai-llm.ts +++ b/src/llm/services/ai-llm.ts @@ -5,6 +5,7 @@ import { GenerateTextResult, LanguageModelUsage, LanguageModelV1, + ProviderMetadata, StreamTextResult, generateText as aiGenerateText, streamText as aiStreamText, @@ -23,21 +24,6 @@ import { appContext } from '../../applicationContext'; export abstract class AiLLM extends BaseLLM { protected aiProvider: Provider | undefined; - /** Model id form the Vercel AI package */ - - constructor( - readonly displayName: string, - readonly service: string, - readonly model: string, - maxInputTokens: number, - /** Needed for Aider when we only have the text size */ - readonly calculateInputCost: (input: string) => number, - /** Needed for Aider when we only have the text size */ - readonly calculateOutputCost: (output: string) => number, - ) { - super(displayName, service, model, maxInputTokens, calculateInputCost, calculateOutputCost); - } - protected abstract provider(): Provider; protected abstract apiKey(): string | undefined; @@ -54,14 +40,13 @@ export abstract class AiLLM extends BaseLLM { return true; } + protected processMessages(llmMessages: LlmMessage[]): LlmMessage[] { + return llmMessages; + } + async generateTextFromMessages(llmMessages: LlmMessage[], opts?: GenerateTextOptions): Promise { return withActiveSpan(`generateTextFromMessages ${opts?.id ?? ''}`, async (span) => { - const messages: CoreMessage[] = llmMessages.map((msg) => { - if (msg.cache === 'ephemeral') { - msg.experimental_providerMetadata = { anthropic: { cacheControl: { type: 'ephemeral' } } }; - } - return msg; - }); + const messages: CoreMessage[] = this.processMessages(llmMessages); const prompt = messages.map((m) => m.content).join('\n'); span.setAttributes({ @@ -92,12 +77,12 @@ export abstract class AiLLM extends BaseLLM { const finishTime = Date.now(); const llmCall: LlmCall = await llmCallSave; - const inputCost = this.calculateInputCost(prompt); - const outputCost = this.calculateOutputCost(responseText); + const inputCost = this.calculateInputCost('', result.usage.promptTokens, result.experimental_providerMetadata); + const outputCost = this.calculateOutputCost(responseText, result.usage.completionTokens); const cost = inputCost + outputCost; llmCall.responseText = responseText; - llmCall.timeToFirstToken = null; // Not available in this implementation + llmCall.timeToFirstToken = finishTime - requestTime; llmCall.totalTime = finishTime - requestTime; llmCall.cost = cost; llmCall.inputTokens = result.usage.promptTokens; @@ -153,7 +138,7 @@ export abstract class AiLLM extends BaseLLM { const requestTime = Date.now(); try { - const result = await aiStreamText({ + const result = aiStreamText({ model: this.aiModel(), messages, temperature: opts?.temperature, @@ -165,15 +150,10 @@ export abstract class AiLLM extends BaseLLM { onChunk({ string: textPart }); } - const response = await result.response; - - // TODO calculate costs from response tokens const usage: LanguageModelUsage = await result.usage; - usage.totalTokens; - usage.promptTokens; - usage.completionTokens; - const inputCost = this.calculateInputCost(prompt); - const outputCost = this.calculateOutputCost(await result.text); + const metadata: ProviderMetadata = await result.experimental_providerMetadata; + const inputCost = this.calculateInputCost('', usage.promptTokens, metadata); + const outputCost = this.calculateOutputCost(await result.text, usage.completionTokens); const cost = inputCost + outputCost; addCost(cost); diff --git a/src/llm/services/anthropic-vertex.ts b/src/llm/services/anthropic-vertex.ts index c8121e69..defcaf41 100644 --- a/src/llm/services/anthropic-vertex.ts +++ b/src/llm/services/anthropic-vertex.ts @@ -9,7 +9,7 @@ import { currentUser } from '#user/userService/userContext'; import { envVar } from '#utils/env-var'; import { appContext } from '../../applicationContext'; import { RetryableError, cacheRetry } from '../../cache/cacheRetry'; -import { BaseLLM } from '../base-llm'; +import { BaseLLM, InputCostFunction, perMilTokens } from '../base-llm'; import { MaxTokensError } from '../errors'; import { GenerateTextOptions, LLM, LlmMessage } from '../llm'; @@ -33,25 +33,18 @@ export function anthropicVertexLLMRegistry(): Record LLM> { // Supported image types image/jpeg', 'image/png', 'image/gif' or 'image/webp' export function Claude3_5_Sonnet_Vertex() { - return new AnthropicVertexLLM( - 'Claude 3.5 Sonnet (Vertex)', - 'claude-3-5-sonnet-v2@20241022', - 3, - 15, - (input: string) => (input.length * 3) / (1_000_000 * 3.5), - (output: string) => (output.length * 15) / (1_000_000 * 3.5), - ); + return new AnthropicVertexLLM('Claude 3.5 Sonnet (Vertex)', 'claude-3-5-sonnet-v2@20241022', 3, 15); } export function Claude3_5_Haiku_Vertex() { - return new AnthropicVertexLLM( - 'Claude 3.5 Haiku (Vertex)', - 'claude-3-5-haiku@20241022', - 1, - 5, - (input: string) => (input.length * 0.25) / (1_000_000 * 3.5), - (output: string) => (output.length * 1.25) / (1_000_000 * 3.5), - ); + return new AnthropicVertexLLM('Claude 3.5 Haiku (Vertex)', 'claude-3-5-haiku@20241022', 1, 5); +} + +function inputCostFunction(dollarsPerMillionTokens: number): InputCostFunction { + return (input: string, tokens: number, usage: any) => + (tokens * dollarsPerMillionTokens) / 1_000_000 + + (usage.cache_creation_input_tokens * dollarsPerMillionTokens * 1.25) / 1_000_000 + + (usage.cache_read_input_tokens * dollarsPerMillionTokens * 0.1) / 1_000_000; } // export function Claude3_Opus_Vertex() { @@ -85,19 +78,15 @@ class AnthropicVertexLLM extends BaseLLM { model: string, private inputTokensMil: number, private outputTokenMil: number, - calculateInputCost: (input: string) => number, - calculateOutputCost: (output: string) => number, ) { - super(displayName, ANTHROPIC_VERTEX_SERVICE, model, 200_000, calculateInputCost, calculateOutputCost); + super(displayName, ANTHROPIC_VERTEX_SERVICE, model, 200_000, inputCostFunction(inputTokensMil), perMilTokens(outputTokenMil)); } private api(): AnthropicVertex { - if (!this.client) { - this.client = new AnthropicVertex({ - projectId: currentUser().llmConfig.vertexProjectId ?? envVar('GCLOUD_PROJECT'), - region: currentUser().llmConfig.vertexRegion || process.env.GCLOUD_CLAUDE_REGION || envVar('GCLOUD_REGION'), - }); - } + this.client ??= new AnthropicVertex({ + projectId: currentUser().llmConfig.vertexProjectId ?? envVar('GCLOUD_PROJECT'), + region: currentUser().llmConfig.vertexRegion || process.env.GCLOUD_CLAUDE_REGION || envVar('GCLOUD_REGION'), + }); return this.client; } @@ -290,8 +279,10 @@ class AnthropicVertexLLM extends BaseLLM { const inputTokens = message.usage.input_tokens; const outputTokens = message.usage.output_tokens; + const usage = message.usage; + + const inputCost = this.calculateInputCost(null, inputTokens, usage); - const inputCost = (inputTokens * this.inputTokensMil) / 1_000_000; const outputCost = (outputTokens * this.outputTokenMil) / 1_000_000; const cost = inputCost + outputCost; addCost(cost); @@ -306,6 +297,7 @@ class AnthropicVertexLLM extends BaseLLM { span.setAttributes({ inputTokens, outputTokens, + cachedInputTokens: usage.cache_read_input_tokens, response: responseText, inputCost: inputCost.toFixed(4), outputCost: outputCost.toFixed(4), diff --git a/src/llm/services/anthropic.ts b/src/llm/services/anthropic.ts index 3733ab35..aba33488 100644 --- a/src/llm/services/anthropic.ts +++ b/src/llm/services/anthropic.ts @@ -1,8 +1,9 @@ import { AnthropicProvider, createAnthropic } from '@ai-sdk/anthropic'; import { AgentLLMs } from '#agent/agentContextTypes'; +import { InputCostFunction, OutputCostFunction, perMilTokens } from '#llm/base-llm'; import { AiLLM } from '#llm/services/ai-llm'; import { currentUser } from '#user/userService/userContext'; -import { LLM } from '../llm'; +import { LLM, LlmMessage } from '../llm'; import { MultiLLM } from '../multi-llm'; export const ANTHROPIC_SERVICE = 'anthropic'; @@ -11,44 +12,22 @@ export function anthropicLLMRegistry(): Record LLM> { return { [`${ANTHROPIC_SERVICE}:claude-3-5-haiku`]: Claude3_5_Haiku, [`${ANTHROPIC_SERVICE}:claude-3-5-sonnet`]: Claude3_5_Sonnet, - // [`${ANTHROPIC_SERVICE}:claude-3-opus`]: Claude3_Opus, }; } -// https://docs.anthropic.com/en/docs/glossary#tokens -// For Claude, a token approximately represents 3.5 English characters -// export function Claude3_Opus() { -// return new Anthropic( -// 'Claude 3 Opus', -// 'claude-3-opus-20240229', -// (input: string) => (input.length * 15) / (1_000_000 * 3.5), -// (output: string) => (output.length * 75) / (1_000_000 * 3.5), -// ); -// } - export function Claude3_5_Sonnet() { - return new Anthropic( - 'Claude 3.5 Sonnet', - 'claude-3-5-sonnet-20241022', - (input: string) => (input.length * 3) / (1_000_000 * 3.5), - (output: string) => (output.length * 15) / (1_000_000 * 3.5), - ); + return new Anthropic('Claude 3.5 Sonnet', 'claude-3-5-sonnet-20241022', 3, 15); } export function Claude3_5_Haiku() { - return new Anthropic( - 'Claude 3.5 Haiku', - 'claude-3-5-haiku-20241022', - (input: string) => (input.length * 0.25) / (1_000_000 * 3.5), - (output: string) => (output.length * 1.25) / (1_000_000 * 3.5), - ); + return new Anthropic('Claude 3.5 Haiku', 'claude-3-5-haiku-20241022', 1, 5); } -export function anthropicLLmFromModel(model: string): LLM | null { - if (model.startsWith('claude-3-5-sonnet-')) return Claude3_5_Sonnet(); - if (model.startsWith('claude-3-5-haiku-')) return Claude3_5_Haiku(); - // if (model.startsWith('claude-3-opus-')) return Claude3_Opus(); - return null; +function inputCostFunction(dollarsPerMillionTokens: number): InputCostFunction { + return (_: string, tokens: number, metadata: any) => + (tokens * dollarsPerMillionTokens) / 1_000_000 + + (metadata.anthropic.cacheCreationInputTokens * dollarsPerMillionTokens * 1.25) / 1_000_000 + + (metadata.anthropic.cacheReadInputTokens * dollarsPerMillionTokens * 0.1) / 1_000_000; } export function ClaudeLLMs(): AgentLLMs { @@ -62,20 +41,28 @@ export function ClaudeLLMs(): AgentLLMs { } export class Anthropic extends AiLLM { - constructor(displayName: string, model: string, calculateInputCost: (input: string) => number, calculateOutputCost: (output: string) => number) { - super(displayName, ANTHROPIC_SERVICE, model, 200_000, calculateInputCost, calculateOutputCost); + constructor(displayName: string, model: string, inputMilTokens: number, outputMilTokens: number) { + super(displayName, ANTHROPIC_SERVICE, model, 200_000, inputCostFunction(inputMilTokens), perMilTokens(outputMilTokens)); } protected apiKey(): string { return currentUser().llmConfig.anthropicKey || process.env.ANTHROPIC_API_KEY; } + protected processMessages(llmMessages: LlmMessage[]): LlmMessage[] { + return llmMessages.map((msg) => { + const clone = { ...msg }; + if (msg.cache === 'ephemeral') { + clone.experimental_providerMetadata = { anthropic: { cacheControl: { type: 'ephemeral' } } }; + } + return clone; + }); + } + provider(): AnthropicProvider { - if (!this.aiProvider) { - this.aiProvider = createAnthropic({ - apiKey: this.apiKey(), - }); - } + this.aiProvider ??= createAnthropic({ + apiKey: this.apiKey(), + }); return this.aiProvider; } } diff --git a/src/llm/services/deepseek.ts b/src/llm/services/deepseek.ts index da91c6cf..e6296484 100644 --- a/src/llm/services/deepseek.ts +++ b/src/llm/services/deepseek.ts @@ -99,9 +99,9 @@ export class DeepseekLLM extends BaseLLM { const responseText = response.data.choices[0].message.content; - const inputCacheHitTokens = response.data.prompt_cache_hit_tokens; - const inputCacheMissTokens = response.data.prompt_cache_miss_tokens; - const outputTokens = response.data.completion_tokens; + const inputCacheHitTokens = response.data.usage.prompt_cache_hit_tokens; + const inputCacheMissTokens = response.data.usage.prompt_cache_miss_tokens; + const outputTokens = response.data.usage.completion_tokens; const timeToFirstToken = Date.now() - requestTime; const finishTime = Date.now(); diff --git a/src/llm/services/mock-llm.ts b/src/llm/services/mock-llm.ts index af16cfcb..7fdd7cbc 100644 --- a/src/llm/services/mock-llm.ts +++ b/src/llm/services/mock-llm.ts @@ -87,8 +87,8 @@ export class MockLLM extends BaseLLM { const finishTime = Date.now(); const llmCall: LlmCall = await llmCallSave; - const inputCost = this.calculateInputCost(prompt); - const outputCost = this.calculateOutputCost(responseText); + const inputCost = this.calculateInputCost(prompt, await this.countTokens(prompt)); + const outputCost = this.calculateOutputCost(responseText, await this.countTokens(responseText)); const cost = inputCost + outputCost; addCost(cost); diff --git a/src/llm/services/vertexai.ts b/src/llm/services/vertexai.ts index 61d14a28..b84c42dd 100644 --- a/src/llm/services/vertexai.ts +++ b/src/llm/services/vertexai.ts @@ -68,13 +68,23 @@ export function Gemini_1_5_Flash() { ); } +// export function Gemini_1_5_Flash_8B() { +// return new VertexLLM( +// 'Gemini 1.5 Flash 8B', +// 'gemini-1.5-flash-8b', +// 1_000_000, +// (input: string) => (input.length * 0.000125) / 1000, +// (output: string) => (output.length * 0.000375) / 1000, +// ); +// } + export function Gemini_2_0_Flash() { return new VertexLLM( 'Gemini 2.0 Flash Experimental', 'gemini-2.0-flash-exp', 1_000_000, - (input: string) => 0, //(input.length * 0.000125) / 1000, - (output: string) => 0, //(output.length * 0.000375) / 1000, + (input: string) => (input.length * 0.000125) / 1000, + (output: string) => (output.length * 0.000375) / 1000, ); } @@ -83,8 +93,8 @@ export function Gemini_2_0_Flash_Thinking() { 'Gemini 2.0 Flash Thinking Experimental', 'gemini-2.0-flash-thinking-exp-1219', 1_000_000, - (input: string) => 0, //(input.length * 0.000125) / 1000, - (output: string) => 0, //(output.length * 0.000375) / 1000, + (input: string) => (input.length * 0.000125) / 1000, + (output: string) => (output.length * 0.000375) / 1000, ); } diff --git a/src/swe/codeEditingAgent.ts b/src/swe/codeEditingAgent.ts index f9afa8ad..da8a22b6 100644 --- a/src/swe/codeEditingAgent.ts +++ b/src/swe/codeEditingAgent.ts @@ -5,8 +5,9 @@ import { Perplexity } from '#functions/web/perplexity'; import { logger } from '#o11y/logger'; import { span } from '#o11y/trace'; import { CompileErrorAnalysis, CompileErrorAnalysisDetails, analyzeCompileErrors } from '#swe/analyzeCompileErrors'; -import { getRepositoryOverview, getTopLevelSummary } from '#swe/documentationBuilder'; +import { SelectedFile, selectFilesAgent } from '#swe/discovery/selectFilesAgent'; import { includeAlternativeAiToolFiles } from '#swe/includeAlternativeAiToolFiles'; +import { getRepositoryOverview, getTopLevelSummary } from '#swe/repoIndexDocBuilder'; import { reviewChanges } from '#swe/reviewChanges'; import { supportingInformation } from '#swe/supportingInformation'; import { execCommand, runShellCommand } from '#utils/exec'; @@ -74,8 +75,10 @@ export class CodeEditingAgent { let fileSelection: string[] = altOptions.fileSelection || []; if (!fileSelection) { // Find the initial set of files required for editing - const filesResponse: SelectFilesResponse = await this.selectFilesToEdit(requirements, projectInfo); - fileSelection = [...filesResponse.primaryFiles.map((selected) => selected.path), ...filesResponse.secondaryFiles.map((selected) => selected.path)]; + // const filesResponse: SelectFilesResponse = await this.selectFilesToEdit(requirements, projectInfo); + // fileSelection = [...filesResponse.primaryFiles.map((selected) => selected.path), ...filesResponse.secondaryFiles.map((selected) => selected.path)]; + const selectFiles = await this.selectFiles(requirements, projectInfo); + fileSelection = selectFiles.map((sf) => sf.path); } await includeAlternativeAiToolFiles(fileSelection); @@ -87,6 +90,7 @@ export class CodeEditingAgent { const repositoryOverview: string = await getRepositoryOverview(); const installedPackages: string = await projectInfo.languageTools.getInstalledPackages(); + // TODO don't need this if we use the architect mode in Aider const implementationDetailsPrompt = `${repositoryOverview}${installedPackages}${await fs.readFilesAsXml(fileSelection)} ${requirements} You are a senior software engineer. Your task is to review the provided user requirements against the code provided and produce a detailed, comprehensive implementation design specification to give to a developer to implement the changes in the provided files. @@ -342,6 +346,11 @@ Then respond in following format: return await selectFilesToEdit(requirements, projectInfo); } + @cacheRetry() + async selectFiles(requirements: string, projectInfo: ProjectInfo): Promise { + return await selectFilesAgent(requirements, projectInfo); + } + async runStaticAnalysis(projectInfo: ProjectInfo): Promise { if (!projectInfo.staticAnalysis) return; const { exitCode, stdout, stderr } = await execCommand(projectInfo.staticAnalysis); diff --git a/src/swe/discovery/codebaseQuery.ts b/src/swe/discovery/codebaseQuery.ts index f8374dd7..9a5918a5 100644 --- a/src/swe/discovery/codebaseQuery.ts +++ b/src/swe/discovery/codebaseQuery.ts @@ -1,8 +1,7 @@ import { getFileSystem, llms } from '#agent/agentContextLocalStorage'; -import { LlmMessage } from '#llm/llm'; import { logger } from '#o11y/logger'; -import { getTopLevelSummary } from '#swe/documentationBuilder'; import { ProjectInfo, getProjectInfo } from '#swe/projectDetection'; +import { getTopLevelSummary } from '#swe/repoIndexDocBuilder'; import { RepositoryMaps, generateRepositoryMaps } from '#swe/repositoryMap'; interface FileSelection { @@ -21,7 +20,7 @@ async function firstPass(query: string): Promise { const prompt = `${await getTopLevelSummary()} -${projectMaps.fileSystemTreeWithSummaries.text} +${projectMaps.fileSystemTreeWithFileSummaries.text} @@ -55,7 +54,7 @@ async function secondPass(query: string, filePaths: string[]): Promise const fileContents = await getFileSystem().readFilesAsXml(filePaths); const prompt = `${await getTopLevelSummary()} -${projectMaps.fileSystemTreeWithSummaries.text} +${projectMaps.fileSystemTreeWithFileSummaries.text} ${query} diff --git a/src/swe/discovery/selectFilesAgent.ts b/src/swe/discovery/selectFilesAgent.ts index f0cf2c64..72e7c0f4 100644 --- a/src/swe/discovery/selectFilesAgent.ts +++ b/src/swe/discovery/selectFilesAgent.ts @@ -2,22 +2,26 @@ import path from 'path'; import { getFileSystem, llms } from '#agent/agentContextLocalStorage'; import { LlmMessage } from '#llm/llm'; import { logger } from '#o11y/logger'; -import { getRepositoryOverview } from '#swe/documentationBuilder'; -import { ProjectInfo, getProjectInfo } from '#swe/projectDetection'; +import { ProjectInfo, detectProjectInfo, getProjectInfo } from '#swe/projectDetection'; +import { getRepositoryOverview } from '#swe/repoIndexDocBuilder'; import { RepositoryMaps, generateRepositoryMaps } from '#swe/repositoryMap'; -// WORK IN PROGRESS ------ +/* +Agent which iteratively loads files to find the file set required for a task/query. -interface AssistantAction { +After each iteration the agent should accept or ignore each of the new files loaded. + +This agent is designed to utilise LLM prompt caching. +*/ + +interface InitialResponse { inspectFiles?: string[]; - selectFiles?: SelectedFile[]; - ignoreFiles?: string[]; - complete?: boolean; } -export interface FileSelection { - files: SelectedFile[]; - extracts?: FileExtract[]; +interface IterationResponse { + keepFiles?: SelectedFile[]; + ignoreFiles?: SelectedFile[]; + inspectFiles?: string[]; } export interface SelectedFile { @@ -26,7 +30,7 @@ export interface SelectedFile { /** The reason why this file needs to in the file selection */ reason: string; /** If the file should not need to be modified when implementing the task. Only relevant when the task is for making changes, and not just a query. */ - readonly: boolean; + readonly?: boolean; } export interface FileExtract { @@ -36,52 +40,115 @@ export interface FileExtract { extract: string; } -function getStageInstructions(stage: 'initial' | 'post_inspect' | 'all_inspected'): string { - if (stage === 'initial') { - return ` -At this stage, you should decide which files to inspect based on the requirements and project map. - -**Valid Actions**: -- Request to inspect files by providing "inspectFiles": ["file1", "file2"] - -**Response Format**: -Respond with a JSON object wrapped in ... tags, containing only the **"inspectFiles"** property. +async function initializeFileSelectionAgent(requirements: string, projectInfo?: ProjectInfo): Promise { + // Ensure projectInfo is available + projectInfo ??= (await detectProjectInfo())[0]; + + // Generate repository maps and overview + const projectMaps: RepositoryMaps = await generateRepositoryMaps([projectInfo]); + const repositoryOverview: string = await getRepositoryOverview(); + const fileSystemWithSummaries: string = `\n${projectMaps.fileSystemTreeWithFileSummaries.text}\n\n`; + + // Construct the initial prompt + const systemPrompt = `${repositoryOverview}${fileSystemWithSummaries} + +Your task is to select the minimal, complete file set that will be required for completing the task/query in the requirements. + +Always respond only in the format/instructions requested. + +# Process Files Response Instructions + +When requested to respond as per the Proces Files Response Instructions you will need to keep/ignore each of the files you previously selected to inspect, giving a reason why. +Then you can select additional files to read and inspect if required from the provided in the system instructions. + +## Response Format +Your response must finish in the following format: + + + + + + + + + + + +## Response Format Contents + +The final part of the response should be a JSON object in the following format: + +{ + keepFiles:[ + {"path": "dir/file1", "reason": "..."} + ], + ignoreFiles:[ + {"path": "dir/file1", "reason": "..."} + ], + inspectFiles: [ + "dir1/dir2/file2" + ] +} + -Do not include file contents unless they have been provided to you. +If you believe that you have all the files required for the requirements task/query, then return an empty array for inspectFiles. `; - } - if (stage === 'post_inspect') { - return ` -You have received the contents of the files you requested to inspect. + // Do not include file contents unless they have been provided to you. + const initialUserPrompt = `\n${requirements}\n + +# Initial Response Instructions + +For this initial file selection step respond in the following format: + + + + + + + +Your response must end with a JSON object wrapped in tags in the following format: + +{ + "inspectFiles": ["dir/file1", "dir1/dir2/file2"] +} + +`; + return [ + { role: 'system', content: systemPrompt, cache: 'ephemeral' }, + { role: 'user', content: initialUserPrompt, cache: 'ephemeral' }, + ]; +} -**Valid Actions**: -- Decide to select or ignore the inspected files by providing: - - "selectFiles": [{"path": "file1", "reason": "...", "readonly": false}, ...] - - "ignoreFiles": ["file2", ...] +async function generateFileSelectionProcessingResponse(messages: LlmMessage[], pendingFiles: string[]): Promise { + const prompt = ` +${(await readFileContents(pendingFiles)).contents} -**Response Format**: -Respond with a JSON object wrapped in ... tags, containing **"selectFiles"** and/or **"ignoreFiles"** properties. +The files that must be included in either the keepFiles or ignoreFiles properties are: +${pendingFiles.join('\n')} -Do not include file contents unless they have been provided to you. +Respond only as per the Process Files Response Instructions. `; - } - if (stage === 'all_inspected') { - return ` -You have processed all inspected files. + const iterationMessages: LlmMessage[] = [...messages, { role: 'user', content: prompt }]; -**Valid Actions**: -- Request to inspect more files by providing "inspectFiles": ["file3", "file4"] -- If you have all the necessary files, complete the selection by responding with "complete": true + return await llms().medium.generateTextWithJson(iterationMessages); +} -**Response Format**: -Respond with a JSON object wrapped in ... tags, containing either: -- **"inspectFiles"** property, or -- **"complete": true** +async function processedIterativeStepUserPrompt(response: IterationResponse): Promise { + const ignored = response.ignoreFiles?.map((s) => s.path) ?? []; + const kept = response.keepFiles?.map((s) => s.path) ?? []; -Do not include file contents unless they have been provided to you. -`; + let ignoreText = ''; + if (ignored.length) { + ignoreText = '\nRemoved the following ignored files:'; + for (const ig of response.ignoreFiles) { + ignoreText += `\n${ig.path} - ${ig.reason}`; + } } - return ''; + + return { + role: 'user', + content: `${(await readFileContents(kept)).contents}${ignoreText}`, + }; } /** @@ -115,7 +182,7 @@ Do not include file contents unless they have been provided to you. * [index] - [role]: [message] * * Messages #1 - * 0 - USER : given and and select initial files for the task. + * 0 - SYSTEM/USER : given and and select initial files for the task. * * Messages #2 * 1 - ASSISTANT: { "inspectFiles": ["file1", "file2"] } @@ -139,7 +206,6 @@ Do not include file contents unless they have been provided to you. * 0 - USER : given and and select initial files for the task. * * - * * The history of the actions will be kept, and always included in final message to the LLM. * * All files staged in a previous step must be processed in the next step (ie. added, extracted or removed) @@ -147,129 +213,97 @@ Do not include file contents unless they have been provided to you. * @param requirements * @param projectInfo */ -export async function selectFilesAgent(requirements: string, projectInfo?: ProjectInfo): Promise { - try { - projectInfo ??= await getProjectInfo(); - const projectMaps: RepositoryMaps = await generateRepositoryMaps([projectInfo]); - const repositoryOverview: string = await getRepositoryOverview(); - const fileSystemWithSummaries: string = `\n${projectMaps.fileSystemTreeWithSummaries.text}\n\n`; - - const messages: LlmMessage[] = []; - const fileSelection: FileSelection = { files: [], extracts: [] }; - let stagedFiles: string[] = []; - let isComplete = false; - - const initialPrompt = `${repositoryOverview} - ${fileSystemWithSummaries} - - ${requirements} - `; - - messages.push({ role: 'user', content: initialPrompt }); - - const maxIterations = 5; - let iterationCount = 0; - - while (!isComplete) { - iterationCount++; - if (iterationCount > maxIterations) { - throw new Error('Maximum interaction iterations reached.'); - } - - // Determine the current stage - let currentStage: 'initial' | 'post_inspect' | 'all_inspected'; - if (iterationCount === 1) { - // First iteration - currentStage = 'initial'; - } else if (stagedFiles.length > 0) { - // Just provided file contents; expecting select or ignore - currentStage = 'post_inspect'; - } else { - // After processing inspected files - currentStage = 'all_inspected'; - } - - // Get the stage-specific instructions - const stageInstructions = getStageInstructions(currentStage); - - // Construct the current prompt by appending stage instructions - const currentPrompt = ` - -Your task is to select files from the to fulfill the given requirements. - -Before responding, please follow these steps: -1. **Observations**: Make observations about the project and requirements. -2. **Thoughts**: Think about which files are necessary. -3. **Reasoning**: Provide reasoning for your choices. -4. **Response**: Finally, respond according to the instructions below. - -${stageInstructions} -`; - - // Add the current prompt to messages - messages.push({ role: 'user', content: currentPrompt }); - - // Call the LLM with the current messages - const assistantResponse = await llms().medium.generateJson(messages); - - // Add the assistant's response to the conversation history - messages.push({ role: 'assistant', content: JSON.stringify(assistantResponse) }); - - // Handle the assistant's response based on the current stage - if (currentStage === 'initial' && assistantResponse.inspectFiles) { - // Read and provide the contents of the requested files - const fileContents = await readFileContents(assistantResponse.inspectFiles); - messages.push({ role: 'user', content: fileContents }); - stagedFiles = assistantResponse.inspectFiles; - } else if (currentStage === 'post_inspect' && (assistantResponse.selectFiles || assistantResponse.ignoreFiles)) { - // Process selected files and remove ignored files from staging - if (assistantResponse.selectFiles) { - fileSelection.files.push(...assistantResponse.selectFiles); - } - if (assistantResponse.ignoreFiles) { - stagedFiles = stagedFiles.filter((f) => !assistantResponse.ignoreFiles.includes(f)); - } - // Ensure all staged files have been processed - if (stagedFiles.length > 0) { - const message = `Please respond with select or ignore for the remaining files in the same JSON format as before.\n${JSON.stringify(stagedFiles)}`; - messages.push({ role: 'user', content: message }); - } else { - // Move to next stage - stagedFiles = []; - } - } else if (currentStage === 'all_inspected') { - if (assistantResponse.inspectFiles) { - // Read and provide the contents of the requested files - const fileContents = await readFileContents(assistantResponse.inspectFiles); - messages.push({ role: 'user', content: fileContents }); - stagedFiles = assistantResponse.inspectFiles; - } else if (assistantResponse.complete) { - // Mark the selection process as complete - isComplete = true; - } else { - throw new Error('Invalid response in all_inspected stage.'); - } - } else { - throw new Error('Unexpected response from assistant.'); - } - } - - if (fileSelection.files.length === 0) { - throw new Error('No files were selected to fulfill the requirements.'); +async function selectFilesCore( + requirements: string, + projectInfo?: ProjectInfo, +): Promise<{ + messages: LlmMessage[]; + selectedFiles: SelectedFile[]; +}> { + const messages: LlmMessage[] = await initializeFileSelectionAgent(requirements, projectInfo); + + const maxIterations = 10; + let iterationCount = 0; + + const initialResponse: InitialResponse = await llms().medium.generateTextWithJson(messages); + messages.push({ role: 'assistant', content: JSON.stringify(initialResponse) }); + + let filesToInspect = initialResponse.inspectFiles || []; + + const keptFiles = new Set<{ path: string; reason: string }>(); + const ignoredFiles = new Set<{ path: string; reason: string }>(); + + while (filesToInspect.length > 0) { + iterationCount++; + if (iterationCount > maxIterations) throw new Error('Maximum interaction iterations reached.'); + + const response: IterationResponse = await generateFileSelectionProcessingResponse(messages, filesToInspect); + logger.info(response); + for (const ignored of response.ignoreFiles ?? []) ignoredFiles.add(ignored); + for (const kept of response.keepFiles ?? []) keptFiles.add(kept); + + messages.push(await processedIterativeStepUserPrompt(response)); + // Don't cache the final result as it would only potentially be used once when generating a query answer + const cache = filesToInspect.length ? 'ephemeral' : undefined; + messages.push({ + role: 'assistant', + content: JSON.stringify(response), + cache, + }); + + // Max of 4 cache tags with Anthropic. Clear the first one after the cached system prompt + const cachedMessages = messages.filter((msg) => msg.cache === 'ephemeral'); + if (cachedMessages.length > 4) { + logger.info('Removing cache tag'); + cachedMessages[1].cache = undefined; } - logger.info(`Selected files: ${fileSelection.files.map((f) => f.path).join(', ')}`); + filesToInspect = response.inspectFiles; - return fileSelection; - } catch (error) { - logger.error('Error in selectFilesAgent:', error); - throw error; + // TODO if keepFiles and ignoreFiles doesnt have all of the files in filesToInspect, then + // filesToInspect = filesToInspect.filter((path) => !keptFiles.has(path) && !ignoredFiles.has(path)); } + + if (keptFiles.size === 0) throw new Error('No files were selected to fulfill the requirements.'); + + const selectedFiles = Array.from(keptFiles.values()); + + return { messages, selectedFiles }; +} + +export async function selectFilesAgent(requirements: string, projectInfo?: ProjectInfo): Promise { + const { selectedFiles } = await selectFilesCore(requirements, projectInfo); + return selectedFiles; } -async function readFileContents(filePaths: string[]): Promise { +export async function queryWorkflow(query: string, projectInfo?: ProjectInfo): Promise { + const { messages, selectedFiles } = await selectFilesCore(query, projectInfo); + + // Construct the final prompt for answering the query + const finalPrompt = ` +${query} + + +Please provide a detailed answer to the query using the information from the available file contents, and including citations to the files where the relevant information was found. +Respond in the following format (Note only the contents of the result tag will be returned to the user): + + + + + `; + + messages.push({ role: 'user', content: finalPrompt }); + + // Perform the additional LLM call to get the answer + const answer = await llms().medium.generateTextWithResult(messages); + return answer.trim(); +} + +async function readFileContents(filePaths: string[]): Promise<{ contents: string; invalidPaths: string[] }> { const fileSystem = getFileSystem(); - let contents = ''; + let contents = '\n'; + + const invalidPaths = []; for (const filePath of filePaths) { const fullPath = path.join(fileSystem.getWorkingDirectory(), filePath); @@ -281,9 +315,9 @@ ${fileContent} `; } catch (e) { logger.info(`Couldn't read ${filePath}`); - contents += `Couldn't read ${filePath}\n`; + contents += `Invalid path ${filePath}\n`; + invalidPaths.push(filePath); } } - - return contents; + return { contents: `${contents}`, invalidPaths }; } diff --git a/src/swe/discovery/selectFilesToEdit.ts b/src/swe/discovery/selectFilesToEdit.ts index e245e972..a27fc819 100644 --- a/src/swe/discovery/selectFilesToEdit.ts +++ b/src/swe/discovery/selectFilesToEdit.ts @@ -3,7 +3,7 @@ import path from 'path'; import { createByModelName } from '@microsoft/tiktokenizer'; import { getFileSystem, llms } from '#agent/agentContextLocalStorage'; import { logger } from '#o11y/logger'; -import { getRepositoryOverview } from '#swe/documentationBuilder'; +import { getRepositoryOverview } from '#swe/repoIndexDocBuilder'; import { RepositoryMaps, generateRepositoryMaps } from '#swe/repositoryMap'; import { ProjectInfo, getProjectInfo } from '../projectDetection'; @@ -27,13 +27,13 @@ export async function selectFilesToEdit(requirements: string, projectInfo?: Proj const projectMaps: RepositoryMaps = await generateRepositoryMaps([projectInfo]); const tokenizer = await createByModelName('gpt-4o'); // TODO model specific tokenizing - const fileSystemTreeTokens = tokenizer.encode(projectMaps.fileSystemTreeWithSummaries.text).length; + const fileSystemTreeTokens = tokenizer.encode(projectMaps.fileSystemTreeWithFolderSummaries.text).length; logger.info(`FileSystem tree tokens: ${fileSystemTreeTokens}`); if (projectInfo.fileSelection) requirements += `\nAdditional note: ${projectInfo.fileSelection}`; const repositoryOverview: string = await getRepositoryOverview(); - const fileSystemWithSummaries: string = `\n${projectMaps.fileSystemTreeWithSummaries.text}\n\n`; + const fileSystemWithSummaries: string = `\n${projectMaps.fileSystemTreeWithFolderSummaries.text}\n\n`; const prompt = `${repositoryOverview} ${fileSystemWithSummaries} @@ -46,12 +46,13 @@ The selected files will be passed to the AI code agent for impementation, and th You will select: 1. The primary files which you anticipate will need to be edited, and their corresponding test files. -2. The secondary supporting files which contain documentation and type information (interfaces, types, classes, function, consts etc) that will be required to correctly makes the changes. Include any files imported by the primary files. If the requirements reference any files relevant to the changes then include them too. +2. The secondary supporting files which contain documentation and type information (interfaces, types, classes, function, consts etc) that will be required to correctly makes the changes. This may include any files imported by the primary files. If the requirements reference any files relevant to the changes then include them too. If there are any instructions related to file selection in the requirements, then those instructions take priority. -Your response MUST ONLY be a JSON object in the format of the following example: -The file paths MUST exist in the file_contents path attributes. +Explain your observations and reasoning. +Then finally your response must end with a JSON object in the format of the following example wrapped in tags: +The file paths must exist in the file_contents path attributes. { @@ -71,7 +72,7 @@ The file paths MUST exist in the file_contents path attributes. `; - let selectedFiles = (await llms().medium.generateJson(prompt, { id: 'selectFilesToEdit' })) as SelectFilesResponse; + let selectedFiles = (await llms().medium.generateTextWithJson(prompt, { id: 'selectFilesToEdit' })) as SelectFilesResponse; selectedFiles = removeLockFiles(selectedFiles); @@ -96,7 +97,7 @@ async function secondPass(requirements: string, initialSelection: SelectFilesRes const projectMaps: RepositoryMaps = await generateRepositoryMaps([projectInfo]); - const fileSystemWithSummaries: string = `\n${projectMaps.fileSystemTreeWithSummaries.text}\n\n`; + const fileSystemWithSummaries: string = `\n${projectMaps.fileSystemTreeWithFolderSummaries.text}\n\n`; const repositoryOverview: string = await getRepositoryOverview(); const prompt = `${repositoryOverview} @@ -190,7 +191,7 @@ This file was selected as a candidate file to implement the requirements based o Now that you can view the full contents of ths file, reassess if the file does need to be included in the fileset for the requirements. We want to ensure we have the minimal required files to reduce costs and focus on the necessary files. A file is considered related if it would be modified to implement the requirements, or contains essential details (code, types, configuration, documentation etc) that would be required to known when implementing the requirements. - + Output the following: 1. If there are any instructions related to file selection in the requirements, then details how that relates to the file_path and file_contents. diff --git a/src/swe/documentationBuilder.ts b/src/swe/documentationBuilder.ts index 591939b4..e69de29b 100644 --- a/src/swe/documentationBuilder.ts +++ b/src/swe/documentationBuilder.ts @@ -1,582 +0,0 @@ -import { promises as fs } from 'node:fs'; -import { basename, dirname, join } from 'path'; -import { Span } from '@opentelemetry/api'; -import { getFileSystem, llms } from '#agent/agentContextLocalStorage'; -import { logger } from '#o11y/logger'; -import { getActiveSpan, withActiveSpan } from '#o11y/trace'; -import { sleep } from '#utils/async-utils'; -import { errorToString } from '#utils/errors'; -import { sophiaDirName } from '../appVars'; - -/** - * This module build summary documentation for a project/repository, to assist with searching in the repository. - * This should generally be run in the root folder of a project/repository. - * The documentation summaries are saved in a parallel directory structure under the.sophia/docs folder - * - * The documentation is generated bottom-up, and takes into account the parent folder summaries available upto the repository root. - * Given initially there isn't any folder level summaries, two passes are initially required. - * - * It's advisable to manually create the top level summary before running this. - */ - -/** Summary documentation for a file/folder */ -export interface Summary { - /** Path to the file/folder */ - path: string; - /** A short of the file/folder */ - short: string; - /** A longer summary of the file/folder */ - long: string; -} - -/** - * This auto-generates documentation for a project/repository, to assist with searching in the repository. - * This should generally be run in the root folder of a project/repository. - * The documentation summaries are saved in a parallel directory structure under the .sophia/docs folder - */ -export async function buildSummaryDocs( - fileFilter: (path: string) => boolean = (file) => (file.endsWith('.tf') || file.endsWith('.ts') || file.endsWith('.py')) && !file.endsWith('test.ts'), -): Promise { - logger.info('Building summary docs'); - - await withActiveSpan('buildSummaryDocs', async (span: Span) => { - await withActiveSpan('buildFileDocs', async (span: Span) => { - // In the first pass we generate the summaries for the individual files - await buildFileDocs(fileFilter); - }); - await withActiveSpan('buildFolderDocs', async (span: Span) => { - // In the second pass we build the folder-level summaries from the bottom up - await buildFolderDocs(); - }); - await withActiveSpan('generateTopLevelSummary', async (span: Span) => { - // Generate a project-level summary from the folder summaries - await generateTopLevelSummary(); - }); - }); -} - -// Utils ----------------------------------------------------------- - -/** - * Returns the summary file path for a given source file path - * @param filePath source file path - * @returns summary file path - */ -function getSummaryFileName(filePath: string): string { - const fileName = basename(filePath); - const dirPath = dirname(filePath); - return join(sophiaDirName, 'docs', dirPath, `${fileName}.json`); -} - -// ----------------------------------------------------------------------------- -// File-level summaries -// ----------------------------------------------------------------------------- - -export async function buildFileDocs(fileFilter: (path: string) => boolean): Promise { - const files: string[] = await getFileSystem().listFilesRecursively(); - const cwd = getFileSystem().getWorkingDirectory(); - const easyLlm = llms().easy; - - const filteredFiles: string[] = files.filter(fileFilter); - - logger.info(`Building summary file docs for ${filteredFiles.length} files matching filters`); - - const docGenOperations = filteredFiles.map((file) => async () => { - const parentSummaries: Summary[] = []; - - const fileContents = await fs.readFile(join(cwd, file)); - try { - let parentSummary = ''; - if (parentSummaries.length) { - parentSummary = ''; - for (const summary of parentSummaries) { - parentSummary += `\n${summary.long}\n}\n`; - } - } - - const prompt = ` -Analyze the following file contents and parent summaries (if available): - -${parentSummary} - -${fileContents} - - -Task: Generate concise and informative summaries for this file to be used as an index for searching the codebase. - -1. Key Questions: - List 3-5 specific questions that this file's contents would help answer. - -2. File Summary: - Provide two summaries in JSON format: - a) A short, terse overview capturing the keywords of the file's main purpose and key details. - b) A longer, tersely keyworded outline highlighting: - - Main functions, classes, or interfaces exported - - Key algorithms or data structures implemented - - Important dependencies or relationships with other parts of the codebase - - Unique or noteworthy aspects of the implementation - - This should be proportional to the length of the file. About one sentence of summary for every 100 lines of the file_contents. - - Written in a terse keyword-heavy style without any filler words. - -The summaries should be in a very terse, gramatically shortened writing style that is incorrect English grammer. This will be only used by LLMs for search, so we want to minimise the tokens. - -Note: Avoid duplicating information from parent summaries. Focus on what's unique to this file. - -Provide terse values for short and long values, in a similar style to the example. - -Respond with the questions and then JSON in this format: - -{ - "short": "Key details", - "long": "Extended details. Key points. Identifiers" -} - - - - -When the filename is variables.tf or output.tf and just has variable declarations respond like the following. Variables which are common to all (ie. project_id, project_number, region) dont require any description. - -variable "project_id" { - description = "The project id where the resources will be created" - type = string -} - -variable "region" { - description = "The region where all resources will be deployed" - type = string -} - -variable "run_sa" { - description = "Cloud Run Service Account" - type = string -} - - - -# Possible questions: -- What services are regional? -- What service accounts are used by Cloud Runs? - - -{ - "short": "project_id, region, run_sa", - "long": "project_id, region, run_sa: Cloud Run Service Account", -} - - - - - - -When a file has terraform resources respond like this example. - -terraform { - required_version = ">= 1.1.4" - required_providers { - google = { - source = "hashicorp/google" - version = "~> 4.46" - } - } -} - -resource "google_cloud_run_service" "foo-service" { - name = "foo-service" - location = var.region - project = var.project_id - - template { - spec { - containers { - image = "gcr.io/cloudrun/hello" - } - service_account_name = var.run_sa - } - } - - lifecycle { - ignore_changes = [ - template - ] - } -} - - - - -# Possible questions: -- What services are regional? -- Why isn't the foo-service changing when the template is updated? - - -{ - "short": "Cloud run service foo-service", - "long": "Cloud run service foo-service with region, project_id, run_sa vars. Ignores changes to template." -} - - - -`; - - logger.info(`Generating summary for ${file}`); - // TODO batch these in parallel - const doc = (await easyLlm.generateJson(prompt, { id: 'Generate file summary' })) as Summary; - doc.path = file; - logger.info(doc); - // Save the documentation summary files in a parallel directory structure under the .sophia/docs folder - await fs.mkdir(join(cwd, sophiaDirName, 'docs', dirname(file)), { recursive: true }); - const summaryFilePath = join(cwd, sophiaDirName, 'docs', `${file}.json`); - logger.info(`Writing summary to ${summaryFilePath}`); - await fs.writeFile(summaryFilePath, JSON.stringify(doc, null, 2)); - } catch (e) { - logger.error(e, `Failed to write documentation for file ${file}`); - } - }); - const all: Promise[] = []; - // Need a way to run in parallel, but then wait and re-try if hitting quotas - for (const op of docGenOperations) { - await op(); - // all.push(op()) - } - try { - await Promise.all(all); - } catch (e) { - logger.error(e); - } - logger.info('Files done'); - await sleep(2000); -} - -// ----------------------------------------------------------------------------- -// Folder-level summaries -// ----------------------------------------------------------------------------- - -/** - * Builds the folder level summaries bottom-up - */ -export async function buildFolderDocs(): Promise { - const fileSystem = getFileSystem(); - logger.info(`Building summary docs for ${fileSystem.getWorkingDirectory()}`); - - const easyLlm = llms().easy; - - const folders = await fileSystem.getAllFoldersRecursively(); - // sorted bottom-up - const sortedFolders = sortFoldersByDepth(folders); - - for (const folderPath of sortedFolders) { - // TODO batch these in parallel, make sure the ordering is ok - let filesAndSubFoldersCombinedSummary: string; - try { - const fileSummaries: Summary[] = await getFileSummaries(folderPath); - const subFolderSummaries: Summary[] = await getSubFolderSummaries(folderPath); - - if (!fileSummaries.length && !sortedFolders.length) continue; - - filesAndSubFoldersCombinedSummary = combineFileAndSubFoldersSummaries(fileSummaries, subFolderSummaries); - - const parentSummaries = await getParentSummaries(folderPath); - const folderSummary: Summary = await generateFolderSummary(easyLlm, filesAndSubFoldersCombinedSummary, parentSummaries); - folderSummary.path = folderPath; - await saveFolderSummary(folderPath, folderSummary); - } catch (e) { - logger.error(e, `Failed to generate summary for folder ${folderPath}`); - logger.error(filesAndSubFoldersCombinedSummary); - } - } -} - -/** - * Sort by depth for bottom-up building of the docs - * @param folders - */ -function sortFoldersByDepth(folders: string[]): string[] { - return folders.sort((a, b) => b.split('/').length - a.split('/').length); -} - -async function getFileSummaries(folderPath: string): Promise { - const fileSystem = getFileSystem(); - const fileNames = await fileSystem.listFilesInDirectory(folderPath); - const summaries: Summary[] = []; - - for (const fileName of fileNames) { - const summaryPath = getSummaryFileName(join(folderPath, fileName)); - logger.info(`File summary path ${summaryPath}`); - try { - const summaryContent = await fs.readFile(summaryPath, 'utf-8'); - summaries.push(JSON.parse(summaryContent)); - } catch (e) { - logger.warn(`Failed to read summary for file ${fileName}`); - } - } - - return summaries; -} - -async function getSubFolderSummaries(folder: string): Promise { - const fileSystem = getFileSystem(); - const subFolders = await fileSystem.listFolders(folder); - const summaries: Summary[] = []; - - for (const subFolder of subFolders) { - const folderName = subFolder.split('/').pop(); - const summaryPath = join('.sophia', 'docs', subFolder, `_${folderName}.json`); - logger.info(`Folder summary path ${summaryPath}`); - try { - const summaryContent = await fs.readFile(summaryPath, 'utf-8'); - summaries.push(JSON.parse(summaryContent)); - } catch (e) { - // logger.warn(`Failed to read summary for subfolder ${subFolder}`); - } - } - - return summaries; -} - -/** - * Formats the summaries of the files and folders into the following format: - * - * dir/dir2 - * paragraph summary - * - * dir/file1 - * paragraph summary - * - * @param fileSummaries - * @param subFolderSummaries - */ -function combineFileAndSubFoldersSummaries(fileSummaries: Summary[], subFolderSummaries: Summary[]): string { - const allSummaries = [...subFolderSummaries, ...fileSummaries]; - return allSummaries.map((summary) => `${summary.path}\n${summary.long}`).join('\n\n'); -} - -async function generateFolderSummary(llm: any, combinedSummary: string, parentSummaries: Summary[] = []): Promise { - let parentSummary = ''; - if (parentSummaries.length) { - parentSummary = '\n'; - for (const summary of parentSummaries) { - parentSummary += `\n${summary.long}\n\n`; - } - parentSummary += '\n\n'; - } - - const prompt = ` -Analyze the following summaries of files and subfolders within this directory: - -${parentSummary} - -${combinedSummary} - - -Task: Generate a cohesive summary for this folder that captures its role in the larger project. - -1. Key Topics: - List 3-5 main topics or functionalities this folder addresses. - -2. Folder Summary: - Provide two summaries in JSON format: - a) A one-sentence overview of the folder's purpose and contents. - b) A paragraph-length description highlighting: - - The folder's role in the project architecture - - Main components or modules contained - - Key functionalities implemented in this folder - - Relationships with other parts of the codebase - - Any patterns or principles evident in the folder's organization - -Note: Focus on the folder's unique contributions. Avoid repeating information from parent summaries. - -Respond only with JSON in this format: - -{ - "sentence": "Concise one-sentence folder summary", - "paragraph": "Detailed paragraph summarizing the folder's contents and significance" -} - -`; - - return await llm.generateJson(prompt, { id: 'Generate folder summary' }); -} - -/** - * Saves the summaries about a folder to /.sophia/docs/folder/_folder.json - * @param folder - * @param summary - */ -async function saveFolderSummary(folder: string, summary: Summary): Promise { - const folderName = basename(folder); - const summaryPath = join('.sophia', 'docs', folder, `_${folderName}.json`); - await fs.mkdir(dirname(summaryPath), { recursive: true }); - await fs.writeFile(summaryPath, JSON.stringify(summary, null, 2)); -} - -// ----------------------------------------------------------------------------- -// Top-level summary -// ----------------------------------------------------------------------------- - -export async function generateTopLevelSummary(): Promise { - const fileSystem = getFileSystem(); - const cwd = fileSystem.getWorkingDirectory(); - - // Get all folder-level summaries - const folderSummaries = await getAllFolderSummaries(cwd); - - // Combine all folder summaries - const combinedSummary = folderSummaries.map((summary) => `${summary.path}:\n${summary.long}`).join('\n\n'); - - // Generate the top-level summary using LLM - const topLevelSummary = await llms().easy.generateText(generateDetailedSummaryPrompt(combinedSummary), { id: 'Generate top level summary' }); - - // Save the top-level summary - await saveTopLevelSummary(cwd, topLevelSummary); - - return topLevelSummary; -} - -async function getAllFolderSummaries(rootDir: string): Promise { - const fileSystem = getFileSystem(); - const folders = await fileSystem.getAllFoldersRecursively(); - const summaries: Summary[] = []; - - for (const folder of folders) { - const folderName = folder.split('/').pop(); - const summaryPath = join(rootDir, '.sophia', 'docs', folder, `_${folderName}.json`); - try { - const summaryContent = await fs.readFile(summaryPath, 'utf-8'); - summaries.push(JSON.parse(summaryContent)); - } catch (e) { - // logger.warn(`Failed to read summary for folder ${folder}`); - } - } - - return summaries; -} - -function generateDetailedSummaryPrompt(combinedSummary: string): string { - return `Based on the following folder summaries, create a comprehensive overview of the entire project: - -${combinedSummary} - -Generate a detailed Markdown summary that includes: - -1. Project Overview: - - The project's primary purpose and goals - -2. Architecture and Structure: - - High-level architecture of the project - - Key directories and their roles - - Main modules or components and their interactions - -3. Core Functionalities: - - List and briefly describe the main features with their location in the project - -4. Technologies and Patterns: - - Primary programming languages used - - Key frameworks, libraries, or tools - - Notable design patterns or architectural decisions - -Ensure the summary is well-structured, using appropriate Markdown formatting for readability. -Include folder path names and file paths where applicable to help readers navigate through the project. -`; -} - -async function saveTopLevelSummary(rootDir: string, summary: string): Promise { - const summaryPath = join(rootDir, sophiaDirName, 'docs', '_summary'); - await fs.writeFile(summaryPath, JSON.stringify(summary, null, 2)); -} - -export async function getTopLevelSummary(): Promise { - try { - return (await fs.readFile(join(sophiaDirName, 'docs', '_summary'))).toString(); - } catch (e) { - return ''; - } -} - -export async function getRepositoryOverview(): Promise { - const repositoryOverview: string = await getTopLevelSummary(); - return repositoryOverview ? `\n${repositoryOverview}\n\n` : ''; -} - -async function getParentSummaries(folderPath: string): Promise { - // TODO should walk up to the git root folder - const parentSummaries: Summary[] = []; - let currentPath = dirname(folderPath); - - while (currentPath !== '.') { - const folderName = basename(currentPath); - const summaryPath = join(sophiaDirName, 'docs', currentPath, `_${folderName}.json`); - try { - const summaryContent = await fs.readFile(summaryPath, 'utf-8'); - parentSummaries.unshift(JSON.parse(summaryContent)); - } catch (e) { - // If we can't read a summary, we've reached the top of the summarized hierarchy - break; - } - currentPath = dirname(currentPath); - } - - return parentSummaries; -} - -/** - * Loads build documentation summaries from the specified directory. - * - * @param {boolean} [createIfNotExits=true] - If true, creates the documentation directory if it doesn't exist. - * @returns {Promise>} A promise that resolves to a Map of file paths to their corresponding Summary objects. - * @throws {Error} If there's an error listing files in the docs directory. - * - * @description - * This function performs the following steps: - * 1. Checks if the docs directory exists, creating it if necessary. - * 2. Lists all JSON files in the docs directory recursively. - * 3. Reads and parses each JSON file, storing the resulting Summary objects in a Map. - * - * @example - * const summaries = await loadBuildDocsSummaries(); - * console.log(`Loaded ${summaries.size} summaries`); - */ -export async function loadBuildDocsSummaries(createIfNotExits = true): Promise> { - const summaries = new Map(); - const fileSystem = getFileSystem(); - const docsDir = join(sophiaDirName, 'docs'); - logger.info(`Load summaries from ${docsDir}`); - - try { - const dirExists = await fileSystem.fileExists(docsDir); - if (!dirExists && !createIfNotExits) { - logger.warn(`The ${docsDir} directory does not exist.`); - return summaries; - } - if (!dirExists) { - await buildSummaryDocs(); - } - - const files = await fileSystem.listFilesRecursively(docsDir, false); - logger.info(`Found ${files.length} files in ${docsDir}`); - - if (files.length === 0) { - logger.warn(`No files found in ${docsDir}. Directory might be empty.`); - return summaries; - } - - for (const file of files) { - if (file.endsWith('.json')) { - try { - if (await fileSystem.fileExists(file)) { - const content = await fileSystem.readFile(file); - const summary: Summary = JSON.parse(content); - summaries.set(summary.path, summary); - } - } catch (error) { - logger.warn(`Failed to read or parse summary file: ${file}. ${errorToString(error)}`); - } - } - } - } catch (error) { - logger.error(`Error listing files in ${docsDir}: ${error.message}`); - throw error; - } - - logger.info(`Loaded ${summaries.size} summaries`); - return summaries; -} diff --git a/src/swe/lang/nodejs/typescriptTools.ts b/src/swe/lang/nodejs/typescriptTools.ts index 68081a08..e9b2e513 100644 --- a/src/swe/lang/nodejs/typescriptTools.ts +++ b/src/swe/lang/nodejs/typescriptTools.ts @@ -35,8 +35,9 @@ export class TypescriptTools implements LanguageTools { async generateProjectMap(): Promise { // Note that the project needs to be in a compilable state otherwise this will fail logger.info('Generating TypeScript project map'); - // TODO dtsFolder needs to be the repository root of the FileSystem workingDirectory - const dtsFolder = `${sophiaDirName}/dts/`; + const fss = getFileSystem(); + const rootFolder = (await fss.getGitRoot()) ?? fss.getWorkingDirectory(); + const dtsFolder = join(rootFolder, sophiaDirName, 'dts'); const tsConfigExists = await getFileSystem().fileExists('tsconfig.json'); if (!tsConfigExists) throw new Error(`tsconfig.json not found in ${getFileSystem().getWorkingDirectory()}`); diff --git a/src/swe/projectDetection.ts b/src/swe/projectDetection.ts index b43a3939..e79a82d1 100644 --- a/src/swe/projectDetection.ts +++ b/src/swe/projectDetection.ts @@ -38,6 +38,8 @@ export interface ProjectInfo extends ProjectScripts { devBranch: string; /** Note to include in the file selection prompts. e.g. "Do not include the files XYZ unless explicitly instructed" */ fileSelection: string; + /** GLob paths of which files should be processed by the buildIndexDocs function in repoIndexDocBuilder.ts */ + indexDocs: string[]; } export async function getProjectInfo(): Promise { @@ -186,6 +188,7 @@ Explain your reasoning, then output a Markdown JSON block, with the JSON formatt ...projectScripts, fileSelection: 'Do not include package manager lock files', languageTools: getLanguageTools(projectDetection.language), + indexDocs: [], }; logger.info(projectInfo, 'ProjectInfo detected'); await getFileSystem().writeFile('projectInfo.json', JSON.stringify([projectInfo], null, 2)); diff --git a/src/swe/repoIndexDocBuilder.ts b/src/swe/repoIndexDocBuilder.ts new file mode 100644 index 00000000..4899ba51 --- /dev/null +++ b/src/swe/repoIndexDocBuilder.ts @@ -0,0 +1,615 @@ +import { promises as fs } from 'node:fs'; +import path, { basename, dirname, join } from 'path'; +import { Span } from '@opentelemetry/api'; +import micromatch from 'micromatch'; +import { getFileSystem, llms } from '#agent/agentContextLocalStorage'; +import { logger } from '#o11y/logger'; +import { withActiveSpan } from '#o11y/trace'; +import { errorToString } from '#utils/errors'; +import { sophiaDirName } from '../appVars'; + +/** + * This module builds summary documentation for a project/repository, to assist with searching in the repository. + * This should generally be run in the root folder of a project/repository. + * The documentation summaries are saved in a parallel directory structure under the.sophia/docs folder + * + * The documentation is generated bottom-up, and takes into account the parent folder summaries available upto the repository root. + * Given initially there isn't any folder level summaries, two passes are initially required. + * + * It's advisable to manually create the top level summary before running this. + */ + +/** Summary documentation for a file/folder */ +export interface Summary { + /** Path to the file/folder */ + path: string; + /** A short of the file/folder */ + short: string; + /** A longer summary of the file/folder */ + long: string; +} + +// Configuration constants +const BATCH_SIZE = 10; + +/** + * This auto-generates summary documentation for a project/repository, to assist with searching in the repository. + * This should generally be run in the root folder of a project/repository. + * The documentation summaries are saved in a parallel directory structure under the .sophia/docs folder + */ +export async function buildIndexDocs(): Promise { + logger.info('Building index docs'); + + await withActiveSpan('Build index docs', async (span: Span) => { + try { + // Load and parse projectInfo.json + const projectInfoPath = path.join(process.cwd(), 'projectInfo.json'); + const projectInfoData = await fs.readFile(projectInfoPath, 'utf-8'); + const projectInfos = JSON.parse(projectInfoData); + + // Assuming you have only one project in the array + const projectInfo = projectInfos[0]; + + // Extract indexDocs patterns + const indexDocsPatterns: string[] = projectInfo.indexDocs || []; + + const fss = getFileSystem(); + // Define fileMatchesIndexDocs function inside buildIndexDocs + function fileMatchesIndexDocs(filePath: string): boolean { + const fss = getFileSystem(); + + // If filePath is absolute, make it relative to the working directory + if (path.isAbsolute(filePath)) { + filePath = path.relative(fss.getWorkingDirectory(), filePath); + } + + // Normalize file path to use forward slashes + const normalizedPath = filePath.split(path.sep).join('/'); + + logger.info(`Checking indexDocs matching for ${normalizedPath}`); + + return micromatch.isMatch(normalizedPath, indexDocsPatterns); + } + + // Define folderMatchesIndexDocs function inside buildIndexDocs + function folderMatchesIndexDocs(folderPath: string): boolean { + const fss = getFileSystem(); + + // Convert absolute folderPath to a relative path + if (path.isAbsolute(folderPath)) { + folderPath = path.relative(fss.getWorkingDirectory(), folderPath); + } + + // Normalize paths to use forward slashes + const normalizedFolderPath = folderPath.split(path.sep).join('/'); + + // Ensure folder path ends with a slash + const folderPathWithSlash = normalizedFolderPath.endsWith('/') ? normalizedFolderPath : `${normalizedFolderPath}/`; + + // Extract directory portions from the patterns + const patternDirs = indexDocsPatterns.map((pattern) => { + const index = pattern.indexOf('**'); + let dir = index !== -1 ? pattern.substring(0, index) : pattern; + dir = dir.endsWith('/') ? dir : `${dir}/`; + return dir; + }); + + // Check if the folder path starts with any of the pattern directories + return patternDirs.some((patternDir) => folderPathWithSlash.startsWith(patternDir)); + } + + const startFolder = getFileSystem().getWorkingDirectory(); + await processFolderRecursively(startFolder, fileMatchesIndexDocs, folderMatchesIndexDocs); + await withActiveSpan('generateTopLevelSummary', async (span: Span) => { + // Generate a project-level summary from the folder summaries + await generateTopLevelSummary(); + }); + } catch (error) { + logger.error(`Failed to build summary docs: ${errorToString(error)}`); + throw error; + } + }); +} + +/** + * Process a single file to generate its documentation summary + */ +async function processFile(filePath: string, easyLlm: any): Promise { + const fileContents = await fs.readFile(filePath, 'utf-8'); + const parentSummaries = await getParentSummaries(dirname(filePath)); + const doc = await generateFileSummary(fileContents, parentSummaries, easyLlm); + const relativeFilePath = path.relative(getFileSystem().getWorkingDirectory(), filePath); + doc.path = relativeFilePath; + + const summaryFilePath = getSummaryFileName(relativeFilePath); + await fs.mkdir(dirname(summaryFilePath), { recursive: true }); + await fs.writeFile(summaryFilePath, JSON.stringify(doc, null, 2)); + logger.info(`Completed summary for ${relativeFilePath}`); +} + +/** + * Process all matching files within a single folder. + * Files are processed in batches to manage memory and API usage. + */ +async function processFilesInFolder(folderPath: string, fileMatchesIndexDocs: (filePath: string) => boolean): Promise { + const fileSystem = getFileSystem(); + const files = await fileSystem.listFilesInDirectory(folderPath); + + // Use the full relative path for matching + const filteredFiles = files.filter((file) => { + const fullRelativePath = path.relative(fileSystem.getWorkingDirectory(), path.join(folderPath, file)); + return fileMatchesIndexDocs(fullRelativePath); + }); + + if (filteredFiles.length === 0) { + logger.info(`No files to process in folder ${folderPath}`); + return; + } + + logger.info(`Processing ${filteredFiles.length} files in folder ${folderPath}`); + const easyLlm = llms().easy; + const errors: Array<{ file: string; error: Error }> = []; + + await withActiveSpan('processFilesInBatches', async (span: Span) => { + // Process files in batches within the folder + for (let i = 0; i < filteredFiles.length; i += BATCH_SIZE) { + const batch = filteredFiles.slice(i, i + BATCH_SIZE); + await Promise.all( + batch.map(async (file) => { + const filePath = join(folderPath, file); + try { + await processFile(filePath, easyLlm); + } catch (e) { + logger.error(e, `Failed to process file ${filePath}`); + errors.push({ file: filePath, error: e }); + } + }), + ); + logger.info(`Completed batch ${Math.floor(i / BATCH_SIZE) + 1} of ${Math.ceil(filteredFiles.length / BATCH_SIZE)}`); + } + }); + + if (errors.length > 0) { + logger.error(`Failed to process ${errors.length} files in folder ${folderPath}`); + errors.forEach(({ file, error }) => logger.error(`${file}: ${errorToString(error)}`)); + } +} + +/** + * Process a folder and its contents recursively in depth-first order. + * First processes all subfolders, then files in the current folder, + * and finally builds the folder summary. + */ +async function processFolderRecursively( + folderPath: string, + fileMatchesIndexDocs: (filePath: string) => boolean, + folderMatchesIndexDocs: (folderPath: string) => boolean, +): Promise { + logger.info(`Processing folder: ${folderPath}`); + + await withActiveSpan('processFolderRecursively', async (span: Span) => { + try { + // Get subfolder names (already updated to return names only) + const subFolders = await getFileSystem().listFolders(folderPath); + + // Process subfolders + for (const subFolder of subFolders) { + const subFolderPath = path.join(folderPath, subFolder); + + // Ensure relative path is correctly calculated + const relativeSubFolderPath = path.relative(getFileSystem().getWorkingDirectory(), subFolderPath); + + if (folderMatchesIndexDocs(relativeSubFolderPath)) { + await processFolderRecursively(subFolderPath, fileMatchesIndexDocs, folderMatchesIndexDocs); + } else { + logger.info(`Skipping folder ${subFolderPath} as it does not match any indexDocs patterns`); + } + } + + // Process files in the current folder + await processFilesInFolder(folderPath, fileMatchesIndexDocs); + + // Build folder summary if any files were processed + const hasProcessedFiles = await checkIfFolderHasProcessedFiles(folderPath, fileMatchesIndexDocs); + if (hasProcessedFiles) { + await buildFolderSummary(folderPath); + } + } catch (error) { + logger.error(`Error processing folder ${folderPath}: ${errorToString(error)}`); + throw error; + } + }); +} + +async function checkIfFolderHasProcessedFiles(folderPath: string, fileMatchesIndexDocs: (filePath: string) => boolean): Promise { + const fileSystem = getFileSystem(); + const files = await fileSystem.listFilesInDirectory(folderPath); + const processedFiles = files.filter((file) => { + const fullRelativePath = path.relative(fileSystem.getWorkingDirectory(), path.join(folderPath, file)); + return fileMatchesIndexDocs(fullRelativePath); + }); + return processedFiles.length > 0; +} + +/** + * Generate a summary for a single file + */ +async function generateFileSummary(fileContents: string, parentSummaries: Summary[], llm: any): Promise { + let parentSummary = ''; + if (parentSummaries.length) { + parentSummary = '\n'; + for (const summary of parentSummaries) { + parentSummary += `\n${summary.long}\n\n`; + } + parentSummary += '\n\n'; + } + + const prompt = ` +Analyze this source code file and generate a summary that captures its purpose and functionality: + +${parentSummary} + +${fileContents} + + +Generate two summaries in JSON format: +1. A one-sentence overview of the file's purpose +2. A detailed paragraph describing: + - The file's main functionality and features + - Key classes/functions/components + - Its role in the larger codebase + - Important dependencies or relationships + - Notable patterns or implementation details + +Focus on unique aspects not covered in parent summaries. + +Respond only with JSON in this format: + +{ + "short": "One-sentence file summary", + "long": "Detailed paragraph describing the file" +} +`; + + return await llm.generateJson(prompt, { id: 'Generate file summary' }); +} + +// Utils ----------------------------------------------------------- + +/** + * Returns the summary file path for a given source file path + * @param filePath source file path + * @returns summary file path + */ +function getSummaryFileName(filePath: string): string { + const relativeFilePath = path.relative(getFileSystem().getWorkingDirectory(), filePath); + const fileName = basename(relativeFilePath); + const dirPath = dirname(relativeFilePath); + return join(sophiaDirName, 'docs', dirPath, `${fileName}.json`); +} + +// ----------------------------------------------------------------------------- +// File-level summaries +// ----------------------------------------------------------------------------- + +// ----------------------------------------------------------------------------- +// Folder-level summaries +// ----------------------------------------------------------------------------- + +/** + * Builds a summary for the current folder using its files and subfolders + */ +async function buildFolderSummary(folderPath: string): Promise { + const fileSummaries = await getFileSummaries(folderPath); + const subFolderSummaries = await getSubFolderSummaries(folderPath); + + if (!fileSummaries.length && !subFolderSummaries.length) { + logger.info(`No summaries to build for folder ${folderPath}`); + return; + } + + try { + const combinedSummary = combineFileAndSubFoldersSummaries(fileSummaries, subFolderSummaries); + const parentSummaries = await getParentSummaries(folderPath); + const folderSummary = await generateFolderSummary(llms().easy, combinedSummary, parentSummaries); + const relativeFolderPath = path.relative(getFileSystem().getWorkingDirectory(), folderPath); + folderSummary.path = relativeFolderPath; + + const folderName = basename(folderPath); + const summaryPath = join(sophiaDirName, 'docs', relativeFolderPath, `_${folderName}.json`); + await fs.mkdir(dirname(summaryPath), { recursive: true }); + await fs.writeFile(summaryPath, JSON.stringify(folderSummary, null, 2)); + logger.info(`Generated summary for folder ${relativeFolderPath}`); + } catch (error) { + logger.error(`Failed to generate summary for folder ${folderPath}: ${errorToString(error)}`); + throw error; + } +} + +/** + * Sort by depth for bottom-up building of the docs + * @param folders + */ +function sortFoldersByDepth(folders: string[]): string[] { + return folders.sort((a, b) => b.split('/').length - a.split('/').length); +} + +async function getFileSummaries(folderPath: string): Promise { + const fileSystem = getFileSystem(); + const fileNames = await fileSystem.listFilesInDirectory(folderPath); + const summaries: Summary[] = []; + + for (const fileName of fileNames) { + const summaryPath = getSummaryFileName(join(folderPath, fileName)); + logger.info(`File summary path ${summaryPath}`); + try { + const summaryContent = await fs.readFile(summaryPath, 'utf-8'); + summaries.push(JSON.parse(summaryContent)); + } catch (e) { + logger.warn(`Failed to read summary for file ${fileName}`); + } + } + + return summaries; +} + +async function getSubFolderSummaries(folder: string): Promise { + const fileSystem = getFileSystem(); + const subFolders = await fileSystem.listFolders(folder); + const summaries: Summary[] = []; + + for (const subFolder of subFolders) { + const folderName = subFolder.split('/').pop(); + const relativeSubFolder = path.relative(fileSystem.getWorkingDirectory(), path.join(folder, subFolder)); + const summaryPath = join('.sophia', 'docs', relativeSubFolder, `_${folderName}.json`); + logger.info(`Folder summary path ${summaryPath}`); + try { + const summaryContent = await fs.readFile(summaryPath, 'utf-8'); + summaries.push(JSON.parse(summaryContent)); + } catch (e) { + // logger.warn(`Failed to read summary for subfolder ${subFolder}`); + } + } + + return summaries; +} + +/** + * Formats the summaries of the files and folders into the following format: + * + * dir/dir2 + * paragraph summary + * + * dir/file1 + * paragraph summary + * + * @param fileSummaries + * @param subFolderSummaries + */ +function combineFileAndSubFoldersSummaries(fileSummaries: Summary[], subFolderSummaries: Summary[]): string { + const allSummaries = [...subFolderSummaries, ...fileSummaries]; + return allSummaries.map((summary) => `${summary.path}\n${summary.long}`).join('\n\n'); +} + +async function generateFolderSummary(llm: any, combinedSummary: string, parentSummaries: Summary[] = []): Promise { + let parentSummary = ''; + if (parentSummaries.length) { + parentSummary = '\n'; + for (const summary of parentSummaries) { + parentSummary += `\n${summary.long}\n\n`; + } + parentSummary += '\n\n'; + } + + const prompt = ` +Analyze the following summaries of files and subfolders within this directory: + +${parentSummary} + +${combinedSummary} + + +Task: Generate a cohesive summary for this folder that captures its role in the larger project. + +1. Key Topics: + List 3-5 main topics or functionalities this folder addresses. + +2. Folder Summary: + Provide two summaries in JSON format: + a) A one-sentence overview of the folder's purpose and contents. + b) A paragraph-length description highlighting: + - The folder's role in the project architecture + - Main components or modules contained + - Key functionalities implemented in this folder + - Relationships with other parts of the codebase + - Any patterns or principles evident in the folder's organization + +Note: Focus on the folder's unique contributions. Avoid repeating information from parent summaries. + +Respond only with JSON in this format: + +{ + "short": "Concise one-sentence folder summary", + "long": "Detailed paragraph summarizing the folder's contents and significance" +} + +`; + + return await llm.generateJson(prompt, { id: 'Generate folder summary' }); +} + +// ----------------------------------------------------------------------------- +// Top-level summary +// ----------------------------------------------------------------------------- + +export async function generateTopLevelSummary(): Promise { + const fileSystem = getFileSystem(); + const cwd = fileSystem.getWorkingDirectory(); + + // Get all folder-level summaries + const folderSummaries = await getAllFolderSummaries(cwd); + + // Combine all folder summaries + const combinedSummary = folderSummaries.map((summary) => `${summary.path}:\n${summary.long}`).join('\n\n'); + + // Generate the top-level summary using LLM + const topLevelSummary = await llms().easy.generateText(generateDetailedSummaryPrompt(combinedSummary), { id: 'Generate top level summary' }); + + // Save the top-level summary + await saveTopLevelSummary(cwd, topLevelSummary); + + return topLevelSummary; +} + +async function getAllFolderSummaries(rootDir: string): Promise { + const fileSystem = getFileSystem(); + const folders = await fileSystem.getAllFoldersRecursively(); + const summaries: Summary[] = []; + + for (const folder of folders) { + const folderName = folder.split('/').pop(); + const summaryPath = join(rootDir, '.sophia', 'docs', folder, `_${folderName}.json`); + try { + const summaryContent = await fs.readFile(summaryPath, 'utf-8'); + summaries.push(JSON.parse(summaryContent)); + } catch (e) { + // logger.warn(`Failed to read summary for folder ${folder}`); + } + } + + return summaries; +} + +function generateDetailedSummaryPrompt(combinedSummary: string): string { + return `Based on the following folder summaries, create a comprehensive overview of the entire project: + +${combinedSummary} + +Generate a detailed Markdown summary that includes: + +1. Project Overview: + - The project's primary purpose and goals + +2. Architecture and Structure: + - High-level architecture of the project + - Key directories and their roles + - Main modules or components and their interactions + +3. Core Functionalities: + - List and briefly describe the main features with their location in the project + +4. Technologies and Patterns: + - Primary programming languages used + - Key frameworks, libraries, or tools + - Notable design patterns or architectural decisions + +Ensure the summary is well-structured, using appropriate Markdown formatting for readability. +Include folder path names and file paths where applicable to help readers navigate through the project. +`; +} + +async function saveTopLevelSummary(rootDir: string, summary: string): Promise { + const summaryPath = join(sophiaDirName, 'docs', '_summary'); + await fs.writeFile(summaryPath, JSON.stringify(summary, null, 2)); +} + +export async function getTopLevelSummary(): Promise { + try { + return (await fs.readFile(join(sophiaDirName, 'docs', '_summary'))).toString(); + } catch (e) { + return ''; + } +} + +export async function getRepositoryOverview(): Promise { + const repositoryOverview: string = await getTopLevelSummary(); + return repositoryOverview ? `\n${repositoryOverview}\n\n` : ''; +} + +async function getParentSummaries(folderPath: string): Promise { + // TODO should walk up to the git root folder + const parentSummaries: Summary[] = []; + let currentPath = dirname(folderPath); + + while (currentPath !== '.') { + const folderName = basename(currentPath); + const summaryPath = join(sophiaDirName, 'docs', currentPath, `_${folderName}.json`); + try { + const summaryContent = await fs.readFile(summaryPath, 'utf-8'); + parentSummaries.unshift(JSON.parse(summaryContent)); + } catch (e) { + // If we can't read a summary, we've reached the top of the summarized hierarchy + break; + } + currentPath = dirname(currentPath); + } + + return parentSummaries; +} + +/** + * Loads build documentation summaries from the specified directory. + * + * @param {boolean} [createIfNotExits=true] - If true, creates the documentation directory if it doesn't exist. + * @returns {Promise>} A promise that resolves to a Map of file paths to their corresponding Summary objects. + * @throws {Error} If there's an error listing files in the docs directory. + * + * @description + * This function performs the following steps: + * 1. Checks if the docs directory exists, creating it if necessary. + * 2. Lists all JSON files in the docs directory recursively. + * 3. Reads and parses each JSON file, storing the resulting Summary objects in a Map. + * + * @example + * const summaries = await loadBuildDocsSummaries(); + * console.log(`Loaded ${summaries.size} summaries`); + */ +export async function loadBuildDocsSummaries(createIfNotExits = false): Promise> { + const summaries = new Map(); + + const fss = getFileSystem(); + // If in a git repo use the repo root to store the summary index files + const repoFolder = (await fss.getGitRoot()) ?? fss.getWorkingDirectory(); + + const docsDir = join(repoFolder, sophiaDirName, 'docs'); + logger.info(`Load summaries from ${docsDir}`); + + try { + const dirExists = await fss.fileExists(docsDir); + if (!dirExists && !createIfNotExits) { + logger.warn(`The ${docsDir} directory does not exist.`); + return summaries; + } + if (!dirExists) { + await buildIndexDocs(); + } + + const files = await fss.listFilesRecursively(docsDir, false); + logger.info(`Found ${files.length} files in ${docsDir}`); + + if (files.length === 0) { + logger.warn(`No files found in ${docsDir}. Directory might be empty.`); + return summaries; + } + + for (const file of files) { + if (file.endsWith('.json')) { + try { + if (await fss.fileExists(file)) { + const content = await fss.readFile(file); + const summary: Summary = JSON.parse(content); + summaries.set(summary.path, summary); + } + } catch (error) { + logger.warn(`Failed to read or parse summary file: ${file}. ${errorToString(error)}`); + } + } + } + } catch (error) { + logger.error(`Error listing files in ${docsDir}: ${error.message}`); + throw error; + } + + logger.info(`Loaded ${summaries.size} summaries`); + return summaries; +} diff --git a/src/swe/repositoryMap.ts b/src/swe/repositoryMap.ts index ab56847e..b639fc53 100644 --- a/src/swe/repositoryMap.ts +++ b/src/swe/repositoryMap.ts @@ -2,7 +2,7 @@ import { getFileSystem } from '#agent/agentContextLocalStorage'; import { countTokens } from '#llm/tokens'; import { logger } from '#o11y/logger'; import { ProjectInfo } from '#swe/projectDetection'; -import { Summary, getTopLevelSummary, loadBuildDocsSummaries } from './documentationBuilder'; +import { Summary, getTopLevelSummary, loadBuildDocsSummaries } from './repoIndexDocBuilder'; interface RepositoryMap { text: string; @@ -12,8 +12,9 @@ interface RepositoryMap { export interface RepositoryMaps { repositorySummary: string; fileSystemTree: RepositoryMap; - fileSystemTreeWithSummaries: RepositoryMap; folderSystemTreeWithSummaries: RepositoryMap; + fileSystemTreeWithFolderSummaries: RepositoryMap; + fileSystemTreeWithFileSummaries: RepositoryMap; languageProjectMap: RepositoryMap; } @@ -38,13 +39,15 @@ export async function generateRepositoryMaps(projectInfos: ProjectInfo[]): Promi const fileSystemTree = await getFileSystem().getFileSystemTree(); - const fileSystemTreeWithSummaries = await generateFileSystemTreeWithSummaries(summaries, false); const folderSystemTreeWithSummaries = await generateFolderTreeWithSummaries(summaries); + const fileSystemTreeWithFolderSummaries = await generateFileSystemTreeWithSummaries(summaries, false); + const fileSystemTreeWithFileSummaries = await generateFileSystemTreeWithSummaries(summaries, true); return { fileSystemTree: { text: fileSystemTree, tokens: await countTokens(fileSystemTree) }, folderSystemTreeWithSummaries: { text: folderSystemTreeWithSummaries, tokens: await countTokens(folderSystemTreeWithSummaries) }, - fileSystemTreeWithSummaries: { text: fileSystemTreeWithSummaries, tokens: await countTokens(fileSystemTreeWithSummaries) }, + fileSystemTreeWithFolderSummaries: { text: fileSystemTreeWithFolderSummaries, tokens: await countTokens(fileSystemTreeWithFolderSummaries) }, + fileSystemTreeWithFileSummaries: { text: fileSystemTreeWithFileSummaries, tokens: await countTokens(fileSystemTreeWithFileSummaries) }, repositorySummary: await getTopLevelSummary(), languageProjectMap: { text: languageProjectMap, tokens: await countTokens(languageProjectMap) }, }; @@ -75,6 +78,7 @@ async function generateFileSystemTreeWithSummaries(summaries: Map