Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: add claude-3-7-sonnet-20250219 #369

Draft
wants to merge 1 commit into
base: main
Choose a base branch
from
Draft

feat: add claude-3-7-sonnet-20250219 #369

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions packages/shortest/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,9 +58,9 @@
"node": ">=18"
},
"peerDependencies": {
"@ai-sdk/anthropic": "^1.1.9",
"@ai-sdk/provider": "^1.0.8",
"ai": "^4.1.45",
"@ai-sdk/anthropic": "^1.1.11",
"@ai-sdk/provider": "^1.0.9",
"ai": "^4.1.46",
"@babel/parser": "^7.26.9",
"@babel/traverse": "^7.26.9",
"@babel/types": "^7.26.9",
Expand Down
98 changes: 89 additions & 9 deletions packages/shortest/src/ai/client.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ import { anthropic } from "@ai-sdk/anthropic";
import { LanguageModelV1FinishReason } from "@ai-sdk/provider";
import {
CoreMessage,
CoreTool,
Tool,
generateText,
LanguageModelV1,
NoSuchToolError,
Expand All @@ -20,6 +20,7 @@ import { getConfig } from "@/index";
import { getLogger, Log } from "@/log";
import { ToolResult } from "@/types";
import { TokenUsage, TokenUsageSchema } from "@/types/ai";
import { ANTHROPIC_MODELS } from "@/types/config";
import {
getErrorDetails,
AIError,
Expand All @@ -28,6 +29,46 @@ import {
} from "@/utils/errors";
import { sleep } from "@/utils/sleep";

const CLAUDE_3_7_TOOLS = {
computer: "20250124",
bash: "20241022",
text: "20241022",
};

const CLAUDE_3_5_TOOLS = {
computer: "20241022",
bash: "20241022",
text: "20241022",
};

const toolTypeSchema = z.object({
computer: z.string(),
bash: z.string(),
text: z.string(),
});

const anthropicModelConfigSchema = z.record(
z.enum(ANTHROPIC_MODELS),
toolTypeSchema,
);

const ANTHROPIC_MODEL_TOOLS = {
"claude-3-7-sonnet-latest": CLAUDE_3_7_TOOLS,
"claude-3-7-sonnet-20250219": CLAUDE_3_7_TOOLS,
"claude-3-5-sonnet-latest": CLAUDE_3_5_TOOLS,
"claude-3-5-sonnet-20241022": CLAUDE_3_5_TOOLS,
};

const ANTHROPIC_MODEL_CONFIG = {
models: anthropicModelConfigSchema.parse(ANTHROPIC_MODEL_TOOLS),

tools: {
computer: (version: string) => `computer_${version}`,
bash: (version: string) => `bash_${version}`,
text: (version: string) => `text_${version}`,
},
};

/**
* Response type for AI client operations.
*
Expand Down Expand Up @@ -87,7 +128,9 @@ export class AIClient {
private log: Log;
private usage: TokenUsage;
private apiRequestCount: number = 0;
private _tools: Record<string, CoreTool> | null = null;
private _tools: Record<string, Tool> | null = null;
private modelName: string;
private toolVersion: typeof CLAUDE_3_7_TOOLS | typeof CLAUDE_3_5_TOOLS;

constructor({
browserTool,
Expand All @@ -98,10 +141,21 @@ export class AIClient {
}) {
this.log = getLogger();
this.log.trace("Initializing AIClient");
this.client = createProvider(getConfig().ai);
const aiConfig = getConfig().ai;
this.client = createProvider(aiConfig);
this.browserTool = browserTool;
this.testCache = testCache;
this.usage = TokenUsageSchema.parse({});

this.modelName = aiConfig.model;
this.toolVersion = this.getToolVersionForModel(
this.modelName as keyof typeof ANTHROPIC_MODEL_TOOLS,
);

this.log.trace("Using model", {
model: this.modelName,
toolVersion: this.toolVersion,
});
this.log.trace(
"Available tools",
Object.fromEntries(
Expand All @@ -113,6 +167,27 @@ export class AIClient {
);
}

/**
* Determines which tool version to use based on the model name
*/
private getToolVersionForModel(
modelName: keyof typeof ANTHROPIC_MODEL_TOOLS,
): typeof CLAUDE_3_7_TOOLS | typeof CLAUDE_3_5_TOOLS {
return ANTHROPIC_MODEL_CONFIG.models[modelName]!;
}

/**
* Gets the appropriate tool factory function for a given tool type and version
*/
private getToolFactory(
toolType: keyof typeof ANTHROPIC_MODEL_CONFIG.tools,
): any {
const toolName = ANTHROPIC_MODEL_CONFIG.tools[toolType](
this.toolVersion[toolType],
);
return (anthropic.tools as any)[toolName];
}

/**
* Executes an AI action with retry logic and error handling.
* Manages conversation flow and caches results for successful tests.
Expand Down Expand Up @@ -290,28 +365,33 @@ export class AIClient {
* Retrieves or initializes the set of available tools for AI interactions.
* Includes browser automation, bash execution, and specialized testing tools.
*
* @returns {Record<string, CoreTool>} Map of available tools
* @returns {Record<string, Tool>} Map of available tools
*
* @see {@link BrowserTool} for web automation tools
* @see {@link BashTool} for shell command execution
*
* @private
*/
private get tools(): Record<string, CoreTool> {
private get tools(): Record<string, Tool> {
if (this._tools) return this._tools;

// Get appropriate tool factory functions based on version
const computerTool = this.getToolFactory("computer");
const bashTool = this.getToolFactory("bash");

this._tools = {
computer: anthropic.tools.computer_20241022({
computer: computerTool({
displayWidthPx: 1920,
displayHeightPx: 1080,
displayNumber: 0,
execute: this.browserTool.execute.bind(this.browserTool),
experimental_toToolResultContent:
this.browserToolResultToToolResultContent,
}),
bash: anthropic.tools.bash_20241022({
execute: async ({ command }) => await new BashTool().execute(command),
experimental_toToolResultContent(result) {
bash: bashTool({
execute: async ({ command }: { command: string }) =>
await new BashTool().execute(command),
experimental_toToolResultContent(result: string) {
return [
{
type: "text",
Expand Down
7 changes: 6 additions & 1 deletion packages/shortest/src/types/config.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,12 @@ export const cliOptionsSchema = z.object({
});
export type CLIOptions = z.infer<typeof cliOptionsSchema>;

const ANTHROPIC_MODELS = ["claude-3-5-sonnet-20241022"] as const;
export const ANTHROPIC_MODELS = [
"claude-3-7-sonnet-20250219",
"claude-3-7-sonnet-latest",
"claude-3-5-sonnet-latest",
"claude-3-5-sonnet-20241022",
] as const;

const aiSchema = z
.object({
Expand Down
2 changes: 1 addition & 1 deletion packages/shortest/tests/e2e/test-loop.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@
// await new Promise((resolve) => setTimeout(resolve, 1000));

// const response = await anthropic.beta.messages.create({
// model: "claude-3-5-sonnet-20241022",
// model: "claude-3-7-sonnet-20250219",
// max_tokens: 1024,
// messages,
// system: SYSTEM_PROMPT,
Expand Down
6 changes: 3 additions & 3 deletions packages/shortest/tests/unit/ai/provider.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,18 +11,18 @@ describe("createProvider", () => {
const config: AIConfig = {
provider: "anthropic",
apiKey: "test-key",
model: "claude-3-5-sonnet-20241022",
model: "claude-3-7-sonnet-20250219",
};

const provider = createProvider(config);
expect(provider).toEqual({ model: "claude-3-5-sonnet-20241022" });
expect(provider).toEqual({ model: "claude-3-7-sonnet-20250219" });
});

it("throws AIError for unsupported provider", () => {
const config = {
provider: "unsupported",
apiKey: "test-key",
model: "claude-3-5-sonnet-20241022",
model: "claude-3-7-sonnet-20250219",
} as unknown as AIConfig;

expect(() => createProvider(config)).toThrow(
Expand Down
12 changes: 6 additions & 6 deletions packages/shortest/tests/unit/config.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@
expect(config.testPattern).toBe("**/*.test.ts");
expect(config.ai).toEqual({
apiKey: "foo",
model: "claude-3-5-sonnet-20241022",
model: "claude-3-7-sonnet-20250219",
provider: "anthropic",
});
expect(config.caching).toEqual({
Expand Down Expand Up @@ -112,7 +112,7 @@
expect(config.ai).toEqual({
apiKey: "env-api-key",
provider: "anthropic",
model: "claude-3-5-sonnet-20241022",
model: "claude-3-7-sonnet-20250219",
});
});
});
Expand All @@ -134,7 +134,7 @@
expect(config.ai).toEqual({
apiKey: "shortest-env-api-key",
provider: "anthropic",
model: "claude-3-5-sonnet-20241022",
model: "claude-3-7-sonnet-20250219",
});
});
});
Expand All @@ -146,7 +146,7 @@
expect(config.ai).toEqual({
apiKey: "explicit-api-key",
provider: "anthropic",
model: "claude-3-5-sonnet-20241022",
model: "claude-3-7-sonnet-20250219",
});
});
});
Expand Down Expand Up @@ -185,8 +185,8 @@
...baseConfig,
ai: { ...baseConfig.ai, model: "invalid-model" as any },
};
expect(() => parseConfig(userConfig)).toThrowError(

Check failure on line 188 in packages/shortest/tests/unit/config.test.ts

View workflow job for this annotation

GitHub Actions / Unit

tests/unit/config.test.ts > Config parsing > with config.ai > with invalid ai.model > throws an error 

AssertionError: expected [Function] to throw error matching /Invalid shortest\.config\n(?:\u001b\[…/ but got 'Invalid shortest.config\n\u001b[36mai…'

- Expected: 
/Invalid shortest\.config\n(?:\u001b\[\d+m)?ai\.model(?:\u001b\[\d+m)?: Invalid enum value\. Expected 'claude-3-7-sonnet-20250219', received 'invalid-model'(?:\s\(received: "invalid-model"\))?/

+ Received: 
"Invalid shortest.config
ai.model: Invalid enum value. Expected 'claude-3-7-sonnet-20250219' | 'claude-3-7-sonnet-latest' | 'claude-3-5-sonnet-latest' | 'claude-3-5-sonnet-20241022', received 'invalid-model' (received: \"invalid-model\")"

 ❯ tests/unit/config.test.ts:188:47
/Invalid shortest\.config\n(?:\u001b\[\d+m)?ai\.model(?:\u001b\[\d+m)?: Invalid enum value\. Expected 'claude-3-5-sonnet-20241022', received 'invalid-model'(?:\s\(received: "invalid-model"\))?/,
/Invalid shortest\.config\n(?:\u001b\[\d+m)?ai\.model(?:\u001b\[\d+m)?: Invalid enum value\. Expected 'claude-3-7-sonnet-20250219', received 'invalid-model'(?:\s\(received: "invalid-model"\))?/,
);
});
});
Expand All @@ -212,7 +212,7 @@
expect(config.ai).toEqual({
provider: "anthropic",
apiKey: "deprecated-api-key",
model: "claude-3-5-sonnet-20241022",
model: "claude-3-7-sonnet-20250219",
});
});
});
Expand Down
8 changes: 4 additions & 4 deletions packages/shortest/tests/unit/initialize-config.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ describe("initializeConfig", () => {
ai: {
provider: "anthropic",
apiKey: "test-key",
model: "claude-3-5-sonnet-20241022",
model: "claude-3-7-sonnet-20250219",
},
caching: {
enabled: true,
Expand All @@ -59,7 +59,7 @@ describe("initializeConfig", () => {
ai: {
provider: "anthropic",
apiKey: "test-key",
model: "claude-3-5-sonnet-20241022",
model: "claude-3-7-sonnet-20250219",
},
}
`,
Expand All @@ -74,7 +74,7 @@ describe("initializeConfig", () => {
ai: {
provider: "anthropic",
apiKey: "test-key",
model: "claude-3-5-sonnet-20241022",
model: "claude-3-7-sonnet-20250219",
},
caching: {
enabled: true,
Expand Down Expand Up @@ -158,7 +158,7 @@ describe("initializeConfig", () => {
ai: {
provider: "anthropic",
apiKey: "test-key",
model: "claude-3-5-sonnet-20241022",
model: "claude-3-7-sonnet-20250219",
},
caching: {
enabled: false,
Expand Down
6 changes: 3 additions & 3 deletions pnpm-lock.yaml
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading