fix: demo updates

command-snapshot.json

@@ -7,14 +7,6 @@
     "flags": ["api-version", "flags-dir", "json", "name", "spec", "target-org"],
     "plugin": "@salesforce/plugin-agent"
   },
-  {
-    "alias": [],
-    "command": "agent:generate:definition",
-    "flagAliases": [],
-    "flagChars": [],
-    "flags": ["flags-dir"],
-    "plugin": "@salesforce/plugin-agent"
-  },
   {
     "alias": [],
     "command": "agent:generate:spec",
@@ -37,7 +29,15 @@
   },
   {
     "alias": [],
-    "command": "agent:generate:testset",
+    "command": "agent:generate:test-definition",
+    "flagAliases": [],
+    "flagChars": [],
+    "flags": ["flags-dir"],
+    "plugin": "@salesforce/plugin-agent"
+  },
+  {
+    "alias": [],
+    "command": "agent:generate:test-set",
     "flagAliases": [],
     "flagChars": [],
     "flags": ["flags-dir"],
@@ -63,15 +63,15 @@
     "alias": [],
     "command": "agent:test:results",
     "flagAliases": [],
-    "flagChars": ["f", "i", "o"],
+    "flagChars": ["d", "i", "o"],
     "flags": ["api-version", "flags-dir", "job-id", "json", "output-dir", "result-format", "target-org"],
     "plugin": "@salesforce/plugin-agent"
   },
   {
     "alias": [],
     "command": "agent:test:resume",
     "flagAliases": [],
-    "flagChars": ["f", "i", "o", "r", "w"],
+    "flagChars": ["d", "i", "o", "r", "w"],
     "flags": [
       "api-version",
       "flags-dir",
@@ -89,7 +89,7 @@
     "alias": [],
     "command": "agent:test:run",
     "flagAliases": [],
-    "flagChars": ["f", "n", "o", "w"],
+    "flagChars": ["d", "n", "o", "w"],
     "flags": ["api-version", "flags-dir", "json", "name", "output-dir", "result-format", "target-org", "wait"],
     "plugin": "@salesforce/plugin-agent"
   }

messages/agent.generate.definition.md → messages/agent.generate.test-definition.md

@@ -1,6 +1,6 @@
 # summary
 
-Interactively generate a new AiEvaluationDefinition.
+Interactively generate a new AI Evaluation Test Definition.
 
 # description
 

messages/agent.generate.testset.md → messages/agent.generate.test-set.md

@@ -1,6 +1,6 @@
 # summary
 
-Interactively generate an AiEvaluationTestSet.
+Interactively generate a new Set of AI Evaluation test cases.
 
 # description
 

package.json

@@ -14,6 +14,7 @@
     "@salesforce/kit": "^3.2.1",
     "@salesforce/sf-plugins-core": "^12.1.0",
     "ansis": "^3.3.2",
+    "fast-xml-parser": "^4.5.1",
     "ink": "^5.0.1",
     "ink-text-input": "^6.0.0",
     "react": "^18.3.1"

src/commands/agent/generate/definition.ts → src/commands/agent/generate/test-definition.ts

@@ -13,9 +13,9 @@ import { theme } from '../../../inquirer-theme.js';
 import { readDir } from '../../../read-dir.js';
 
 Messages.importMessagesDirectoryFromMetaUrl(import.meta.url);
-const messages = Messages.loadMessages('@salesforce/plugin-agent', 'agent.generate.definition');
+const messages = Messages.loadMessages('@salesforce/plugin-agent', 'agent.generate.test-definition');
 
-export default class AgentGenerateDefinition extends SfCommand<void> {
+export default class AgentGenerateTestDefinition extends SfCommand<void> {
   public static readonly summary = messages.getMessage('summary');
   public static readonly description = messages.getMessage('description');
   public static readonly examples = messages.getMessages('examples');

src/commands/agent/generate/testset.ts → src/commands/agent/generate/test-set.ts

@@ -5,20 +5,21 @@
  * For full license text, see LICENSE.txt file in the repo root or https://opensource.org/licenses/BSD-3-Clause
  */
 import { dirname, join } from 'node:path';
-import { mkdir, writeFile } from 'node:fs/promises';
+import { mkdir, readFile, writeFile } from 'node:fs/promises';
 import { SfCommand } from '@salesforce/sf-plugins-core';
 import { Messages } from '@salesforce/core';
 import { select, input, confirm, checkbox } from '@inquirer/prompts';
+import { XMLParser } from 'fast-xml-parser';
 import { theme } from '../../../inquirer-theme.js';
 import { readDir } from '../../../read-dir.js';
-
 Messages.importMessagesDirectoryFromMetaUrl(import.meta.url);
-const messages = Messages.loadMessages('@salesforce/plugin-agent', 'agent.generate.testset');
+const messages = Messages.loadMessages('@salesforce/plugin-agent', 'agent.generate.test-set');
 
-export const FORTY_CHAR_API_NAME_REGEX =
-  /^(?=.{1,57}$)[a-zA-Z]([a-zA-Z0-9]|_(?!_)){0,14}(__[a-zA-Z]([a-zA-Z0-9]|_(?!_)){0,39})?$/;
-export const EIGHTY_CHAR_API_NAME_REGEX =
-  /^(?=.{1,97}$)[a-zA-Z]([a-zA-Z0-9]|_(?!_)){0,14}(__[a-zA-Z]([a-zA-Z0-9]|_(?!_)){0,79})?$/;
+// TODO: add these back once we refine the regex
+// export const FORTY_CHAR_API_NAME_REGEX =
+//   /^(?=.{1,57}$)[a-zA-Z]([a-zA-Z0-9]|_(?!_)){0,14}(__[a-zA-Z]([a-zA-Z0-9]|_(?!_)){0,39})?$/;
+// export const EIGHTY_CHAR_API_NAME_REGEX =
+//   /^(?=.{1,97}$)[a-zA-Z]([a-zA-Z0-9]|_(?!_)){0,14}(__[a-zA-Z]([a-zA-Z0-9]|_(?!_)){0,79})?$/;
 
 export type TestSetInputs = {
   utterance: string;
@@ -27,7 +28,7 @@ export type TestSetInputs = {
   topicSequenceExpectedValue: string;
 };
 
-async function promptForTestCase({ topics, actions }: { topics: string[]; actions: string[] }): Promise<TestSetInputs> {
+async function promptForTestCase(genAiPlugins: Record<string, string>): Promise<TestSetInputs> {
   const utterance = await input({
     message: 'Utterance',
     validate: (d: string): boolean | string => d.length > 0 || 'utterance cannot be empty',
@@ -36,63 +37,82 @@ async function promptForTestCase({ topics, actions }: { topics: string[]; action
 
   const customKey = '<OTHER>';
 
-  let topicSequenceExpectedValue = await select<string>({
-    message: 'Expected topic',
-    choices: [...topics, customKey],
-    theme,
-  });
+  const topics = Object.keys(genAiPlugins);
 
-  if (topicSequenceExpectedValue === customKey) {
-    topicSequenceExpectedValue = await input({
-      message: 'Expected topic',
+  const askForOtherActions = async (): Promise<string[]> =>
+    (
+      await input({
+        message: 'Expected action(s)',
+        validate: (d: string): boolean | string => {
+          if (!d.length) {
+            return 'expected value cannot be empty';
+          }
+          return true;
+        },
+        theme,
+      })
+    )
+      .split(',')
+      .map((a) => a.trim());
+
+  const askForBotRating = async (): Promise<string> =>
+    input({
+      message: 'Expected response',
       validate: (d: string): boolean | string => {
         if (!d.length) {
           return 'expected value cannot be empty';
         }
+
         return true;
       },
       theme,
     });
-  }
 
-  let actionSequenceExpectedValue = await checkbox<string>({
-    message: 'Expected action(s)',
-    choices: [...actions, customKey],
+  const topicSequenceExpectedValue = await select<string>({
+    message: 'Expected topic',
+    choices: [...topics, customKey],
     theme,
-    required: true,
   });
 
-  if (actionSequenceExpectedValue.includes(customKey)) {
-    const additional = (
-      await input({
-        message: 'Expected action(s)',
+  if (topicSequenceExpectedValue === customKey) {
+    return {
+      utterance,
+      topicSequenceExpectedValue: await input({
+        message: 'Expected topic',
         validate: (d: string): boolean | string => {
           if (!d.length) {
             return 'expected value cannot be empty';
           }
           return true;
         },
         theme,
-      })
-    )
-      .split(',')
-      .map((a) => a.trim());
-
-    actionSequenceExpectedValue = [...actionSequenceExpectedValue.filter((a) => a !== customKey), ...additional];
+      }),
+      // If the user selects OTHER for the topic, then we don't have a genAiPlugin to get actions from so we ask for them for custom input
+      actionSequenceExpectedValue: await askForOtherActions(),
+      botRatingExpectedValue: await askForBotRating(),
+    };
   }
 
-  const botRatingExpectedValue = await input({
-    message: 'Expected response',
-    validate: (d: string): boolean | string => {
-      if (!d.length) {
-        return 'expected value cannot be empty';
-      }
+  const genAiPluginXml = await readFile(genAiPlugins[topicSequenceExpectedValue], 'utf-8');
+  const parser = new XMLParser();
+  const parsed = parser.parse(genAiPluginXml) as { GenAiPlugin: { genAiFunctions: Array<{ functionName: string }> } };
+  const actions = parsed.GenAiPlugin.genAiFunctions.map((f) => f.functionName);
 
-      return true;
-    },
+  let actionSequenceExpectedValue = await checkbox<string>({
+    message: 'Expected action(s)',
+    choices: [...actions, customKey],
     theme,
+    required: true,
   });
 
+  if (actionSequenceExpectedValue.includes(customKey)) {
+    const additional = await askForOtherActions();
+
+    actionSequenceExpectedValue = [...actionSequenceExpectedValue.filter((a) => a !== customKey), ...additional];
+  }
+
+  const botRatingExpectedValue = await askForBotRating();
+
   return {
     utterance,
     actionSequenceExpectedValue,
@@ -139,32 +159,31 @@ export default class AgentGenerateTestset extends SfCommand<void> {
 
   public async run(): Promise<void> {
     const testSetName = await input({
-      message: 'What is the name of the test set',
-      validate(d: string): boolean | string {
-        // check against FORTY_CHAR_API_NAME_REGEX
-        if (!FORTY_CHAR_API_NAME_REGEX.test(d)) {
-          return 'The non-namespaced portion an API name must begin with a letter, contain only letters, numbers, and underscores, not contain consecutive underscores, and not end with an underscore.';
-        }
-        return true;
-      },
+      message: 'What is the name of this set of test cases',
+      // TODO: add back validation once we refine the regex
+      // validate(d: string): boolean | string {
+      //   // check against FORTY_CHAR_API_NAME_REGEX
+      //   if (!FORTY_CHAR_API_NAME_REGEX.test(d)) {
+      //     return 'The non-namespaced portion an API name must begin with a letter, contain only letters, numbers, and underscores, not contain consecutive underscores, and not end with an underscore.';
+      //   }
+      //   return true;
+      // },
     });
 
     const genAiPluginDir = join('force-app', 'main', 'default', 'genAiPlugins');
-    const genAiPlugins = (await readDir(genAiPluginDir)).map((genAiPlugin) =>
-      genAiPlugin.replace('.genAiPlugin-meta.xml', '')
-    );
-
-    const genAiFunctionsDir = join('force-app', 'main', 'default', 'genAiFunctions');
-    const genAiFunctions = (await readDir(genAiFunctionsDir)).map((genAiFunction) =>
-      genAiFunction.replace('.genAiFunction-meta.xml', '')
+    const genAiPlugins = Object.fromEntries(
+      (await readDir(genAiPluginDir)).map((genAiPlugin) => [
+        genAiPlugin.replace('.genAiPlugin-meta.xml', ''),
+        join(genAiPluginDir, genAiPlugin),
+      ])
     );
 
     const testCases = [];
     do {
       this.log();
       this.styledHeader(`Adding test case #${testCases.length + 1}`);
       // eslint-disable-next-line no-await-in-loop
-      testCases.push(await promptForTestCase({ topics: genAiPlugins, actions: genAiFunctions }));
+      testCases.push(await promptForTestCase(genAiPlugins));
     } while ( // eslint-disable-next-line no-await-in-loop
       await confirm({
         message: 'Would you like to add another test case',

src/flags.ts

@@ -17,7 +17,7 @@ export const resultFormatFlag = Flags.option({
 });
 
 export const testOutputDirFlag = Flags.custom<string>({
-  char: 'f',
+  char: 'd',
   description: messages.getMessage('flags.output-dir.description'),
   summary: messages.getMessage('flags.output-dir.summary'),
 });

src/handleTestResults.ts

@@ -38,33 +38,45 @@ export async function handleTestResults({
 
   if (format === 'human') {
     const formatted = await humanFormat(results);
-    ux.log(formatted);
     if (outputDir) {
-      await writeFileToDir(outputDir, `test-result-${id}.txt`, formatted);
+      const file = `test-result-${id}.txt`;
+      await writeFileToDir(outputDir, file, formatted);
+      ux.log(`Created human-readable file at ${join(outputDir, file)}`);
+    } else {
+      ux.log(formatted);
     }
   }
 
   if (format === 'json') {
     const formatted = await jsonFormat(results);
-    ux.log(formatted);
     if (outputDir) {
-      await writeFileToDir(outputDir, `test-result-${id}.json`, formatted);
+      const file = `test-result-${id}.json`;
+      await writeFileToDir(outputDir, file, formatted);
+      ux.log(`Created JSON file at ${join(outputDir, file)}`);
+    } else {
+      ux.log(formatted);
     }
   }
 
   if (format === 'junit') {
     const formatted = await junitFormat(results);
-    ux.log(formatted);
     if (outputDir) {
-      await writeFileToDir(outputDir, `test-result-${id}.xml`, formatted);
+      const file = `test-result-${id}.xml`;
+      await writeFileToDir(outputDir, file, formatted);
+      ux.log(`Created JUnit file at ${join(outputDir, file)}`);
+    } else {
+      ux.log(formatted);
     }
   }
 
   if (format === 'tap') {
     const formatted = await tapFormat(results);
-    ux.log(formatted);
     if (outputDir) {
-      await writeFileToDir(outputDir, `test-result-${id}.txt`, formatted);
+      const file = `test-result-${id}.txt`;
+      await writeFileToDir(outputDir, file, formatted);
+      ux.log(`Created TAP file at ${join(outputDir, file)}`);
+    } else {
+      ux.log(formatted);
     }
   }
 }

test/commands/agent/generate/testset.test.ts → test/commands/agent/generate/test-set.test.ts

@@ -5,7 +5,7 @@
  * For full license text, see LICENSE.txt file in the repo root or https://opensource.org/licenses/BSD-3-Clause
  */
 import { expect } from 'chai';
-import { type TestSetInputs, constructTestSetXML } from '../../../../src/commands/agent/generate/testset.js';
+import { type TestSetInputs, constructTestSetXML } from '../../../../src/commands/agent/generate/test-set.js';
 
 describe('constructTestSetXML', () => {
   it('should return a valid test set XML', () => {