download agent protocol spec and enable AP toolkit

apps/nextjs/src/features/WaggleDance/types/WaggleDanceAgentExecutor.ts

@@ -1,4 +1,5 @@
 import { type MutableRefObject } from "react";
+import { type JsonSpec } from "langchain/tools";
 import { stringify } from "yaml";
 
 import {
@@ -16,6 +17,7 @@ import planTasks from "../utils/planTasks";
 import { sleep } from "../utils/sleep";
 import {
   mapAgentSettingsToCreationProps,
+  type ExecuteRequestBody,
   type GraphDataState,
   type WaggleDanceResult,
 } from "./types";
@@ -67,9 +69,19 @@ class WaggleDanceAgentExecutor {
     this.taskResults = {};
     this.error = null;
     const initialNode = rootPlanNode(this.goalPrompt);
+    let agentProtocolOpenAPISpec: JsonSpec | undefined;
+
+    try {
+      agentProtocolOpenAPISpec = (await (
+        await fetch("/api/ap")
+      ).json()) as JsonSpec;
+    } catch {
+      console.error("Failed to fetch agent protocol spec");
+    }
+
     void (async () => {
       try {
-        const result = await this.planAndSetDAG();
+        const result = await this.planAndSetDAG(agentProtocolOpenAPISpec);
         console.debug("done planning");
         if (this.error) {
           return;
@@ -199,7 +211,9 @@ class WaggleDanceAgentExecutor {
     };
   }
 
-  async planAndSetDAG(): Promise<DraftExecutionGraph | null> {
+  async planAndSetDAG(
+    agentProtocolOpenAPISpec?: JsonSpec,
+  ): Promise<DraftExecutionGraph | null> {
     const creationProps = mapAgentSettingsToCreationProps(
       this.agentSettings["plan"],
     );
@@ -212,6 +226,7 @@ class WaggleDanceAgentExecutor {
       log: this.log,
       injectAgentPacket: this.injectAgentPacket,
       abortSignal: this.abortController.signal,
+      agentProtocolOpenAPISpec,
     });
 
     // Call the check methods here
@@ -255,7 +270,7 @@ class WaggleDanceAgentExecutor {
       const creationProps = mapAgentSettingsToCreationProps(
         this.agentSettings["execute"],
       );
-      const executeRequest = {
+      const executeRequest: ExecuteRequestBody = {
         goalPrompt: this.goalPrompt,
         goalId: this.goalId,
         executionId: this.executionId,

apps/nextjs/src/features/WaggleDance/types/types.ts

@@ -1,5 +1,6 @@
 // WaggleDance/types.ts
 import { type Dispatch, type SetStateAction } from "react";
+import { type JsonSpec } from "langchain/tools";
 
 import {
   type AgentSettings,
@@ -75,6 +76,7 @@ export interface BaseRequestBody {
   creationProps: ModelCreationProps;
   goalId: string;
   goalPrompt: string;
+  agentProtocolOpenAPISpec?: JsonSpec;
 }
 
 export interface RefineRequestBody {

apps/nextjs/src/features/WaggleDance/utils/planTasks.ts

@@ -1,5 +1,7 @@
 // features/WaggleDance/utils/planTasks.ts
 
+import { type JsonSpec } from "langchain/tools";
+
 import { type DraftExecutionGraph, type DraftExecutionNode } from "@acme/db";
 
 import {
@@ -26,6 +28,7 @@ export type PlanTasksProps = {
     task: DraftExecutionNode,
     dag: DraftExecutionGraph,
   ) => Promise<void>;
+  agentProtocolOpenAPISpec?: JsonSpec;
 };
 
 export default async function planTasks({
@@ -38,6 +41,7 @@ export default async function planTasks({
   injectAgentPacket,
   startFirstTask,
   abortSignal,
+  agentProtocolOpenAPISpec,
 }: PlanTasksProps): Promise<DraftExecutionGraph | undefined> {
   const intervalHandler = new PlanUpdateIntervalHandler(100); // update at most every...
   const parseWorker = new Worker(new URL("./parseWorker.ts", import.meta.url));
@@ -48,7 +52,14 @@ export default async function planTasks({
 
     let partialDAG: DraftExecutionGraph = dag;
     let hasFirstTaskStarted = false;
-    const data = { goalPrompt, goalId, executionId, creationProps };
+    const data = {
+      goalPrompt,
+      goalId,
+      executionId,
+      creationProps,
+      agentProtocolOpenAPISpec,
+    };
+    // TODO: merge w/ api.execution.createPlan
     const res = await fetch("/api/agent/plan", {
       method: "POST",
       headers: { "Content-Type": "application/json" },

apps/nextjs/src/pages/api/agent/plan.ts

@@ -39,6 +39,7 @@ export default async function PlanStream(req: NextRequest) {
       goalPrompt: parsedGoalPrompt,
       goalId: parsedGoalId,
       executionId: parsedExecutionId,
+      agentProtocolOpenAPISpec,
     } = (await req.json()) as PlanRequestBody;
     goalPrompt = parsedGoalPrompt;
     goalId = parsedGoalId;
@@ -92,6 +93,7 @@ export default async function PlanStream(req: NextRequest) {
           goalId!,
           abortController.signal,
           `${goalId}_${executionId}`,
+          agentProtocolOpenAPISpec,
         );
 
         if (planResult instanceof Error) {

packages/agent/src/strategy/callExecutionAgent.ts

@@ -11,6 +11,7 @@ import { loadEvaluator } from "langchain/evaluation";
 import { PlanAndExecuteAgentExecutor } from "langchain/experimental/plan_and_execute";
 import { type OpenAI } from "langchain/llms/openai";
 import { type AgentStep } from "langchain/schema";
+import { type JsonSpec } from "langchain/tools";
 import { parse } from "yaml";
 
 import { type DraftExecutionGraph } from "@acme/db";
@@ -51,6 +52,7 @@ export async function callExecutionAgent(creation: {
   contentType: "application/json" | "application/yaml";
   abortSignal: AbortSignal;
   namespace: string;
+  agentProtocolOpenAPISpec?: JsonSpec;
   geo?: Geo;
 }): Promise<string | Error> {
   const {
@@ -64,6 +66,7 @@ export async function callExecutionAgent(creation: {
     abortSignal,
     namespace,
     contentType,
+    agentProtocolOpenAPISpec,
     geo,
   } = creation;
   const callbacks = creationProps.callbacks;
@@ -123,6 +126,7 @@ export async function callExecutionAgent(creation: {
     isCriticism,
     taskObj.id,
     returnType,
+    agentProtocolOpenAPISpec,
     geo,
   );
 

packages/agent/src/strategy/callPlanningAgent.ts

@@ -2,6 +2,7 @@
 
 import { encodingForModel, type Tiktoken } from "js-tiktoken";
 import { LLMChain } from "langchain/chains";
+import { type JsonSpec } from "langchain/tools";
 import { parse as jsonParse, stringify as jsonStringify } from "superjson";
 import { parse as yamlParse, stringify as yamlStringify } from "yaml";
 
@@ -26,6 +27,7 @@ export async function callPlanningAgent(
   goalId: string,
   signal: AbortSignal,
   namespace: string,
+  agentProtocolOpenAPISpec?: JsonSpec,
   returnType: "YAML" | "JSON" = "YAML",
 ): Promise<string | Error> {
   const tags = [
@@ -44,6 +46,7 @@ export async function callPlanningAgent(
     false,
     rootPlanId,
     returnType,
+    agentProtocolOpenAPISpec,
   );
   const prompt = createPlanPrompt({
     goalPrompt,

packages/agent/src/utils/skills.ts

@@ -5,12 +5,14 @@ import {
   SearchApi,
   SerpAPI,
   WolframAlphaTool,
+  type JsonSpec,
   type SearchApiParameters,
   type StructuredTool,
 } from "langchain/tools";
 import { WebBrowser } from "langchain/tools/webbrowser";
 
 import cca2Map from "../lib/cca2Map.json";
+import { AgentProtocolToolkit } from "../skills/AgentProtocolSkill";
 import requestUserHelpSkill from "../skills/requestUserHelpSkill";
 import retrieveMemoriesSkill from "../skills/retrieveMemories";
 import saveMemoriesSkill from "../skills/saveMemories";
@@ -169,6 +171,7 @@ function createSkills(
   isCriticism: boolean,
   taskId: string,
   returnType: "YAML" | "JSON",
+  agentProtocolOpenAPISpec?: JsonSpec,
   geo?: Geo,
 ): StructuredTool[] {
   const tools = [
@@ -224,7 +227,10 @@ function createSkills(
     );
   }
 
-  console.debug(tools.map((tool) => tool.name));
+  if (agentProtocolOpenAPISpec) {
+    const toolkit = new AgentProtocolToolkit(agentProtocolOpenAPISpec, llm, {});
+    tools.push(...toolkit.tools);
+  }
   return tools as StructuredTool[];
 }