chore: upgrade integration test-suite to enable performance testing

.changeset/smooth-teachers-invent.md

@@ -0,0 +1,2 @@
+---
+---

packages/integration-tests/src/process-controller/execa_process_killer.ts

@@ -0,0 +1,22 @@
+import { ExecaChildProcess, execa } from 'execa';
+
+/**
+ * Kills the given process (equivalent of sending CTRL-C)
+ * @param processInstance an instance of execa child process
+ */
+export const killExecaProcess = async (processInstance: ExecaChildProcess) => {
+  if (process.platform.startsWith('win')) {
+    if (typeof processInstance.pid !== 'number') {
+      throw new Error('Cannot kill the process that does not have pid');
+    }
+    // Wait X milliseconds before sending kill in hopes of draining the node event queue
+    await new Promise((resolve) => setTimeout(resolve, 1500));
+    // turns out killing child process on Windows is a huge PITA
+    // https://stackoverflow.com/questions/23706055/why-can-i-not-kill-my-child-process-in-nodejs-on-windows
+    // https://github.com/sindresorhus/execa#killsignal-options
+    // eslint-disable-next-line spellcheck/spell-checker
+    await execa('taskkill', ['/pid', `${processInstance.pid}`, '/f', '/t']);
+  } else {
+    processInstance.kill('SIGINT');
+  }
+};

packages/integration-tests/src/process-controller/predicated_action.ts

@@ -0,0 +1,62 @@
+import { ExecaChildProcess } from 'execa';
+
+/**
+ * Type of actions a user can take with their app.
+ */
+export enum ActionType {
+  SEND_INPUT_TO_PROCESS,
+  UPDATE_FILE_CONTENT,
+  ASSERT_ON_PROCESS_OUTPUT,
+  KILL_PROCESS,
+}
+
+type SendInputToProcessAction = {
+  actionType: ActionType.SEND_INPUT_TO_PROCESS;
+  action: (execaProcess: ExecaChildProcess<string>) => Promise<void>;
+};
+type KillProcess = {
+  actionType: ActionType.KILL_PROCESS;
+  action: (execaProcess: ExecaChildProcess<string>) => Promise<void>;
+};
+type UpdateFileContentAction = {
+  actionType: ActionType.UPDATE_FILE_CONTENT;
+  action: () => Promise<void>;
+};
+type AssertOnProcessOutputAction = {
+  actionType: ActionType.ASSERT_ON_PROCESS_OUTPUT;
+  action: (processOutputLine: string) => void;
+};
+
+export type Action =
+  | SendInputToProcessAction
+  | KillProcess
+  | UpdateFileContentAction
+  | AssertOnProcessOutputAction;
+
+/**
+ * Type of predicates based on which to execute actions. An action can only be associated with a predicate
+ */
+export enum PredicateType {
+  MATCHES_STRING_PREDICATE,
+}
+
+type MatchesStringPredicate = {
+  predicateType: PredicateType.MATCHES_STRING_PREDICATE;
+  predicate: (line: string) => boolean;
+};
+
+export type Predicate = MatchesStringPredicate;
+
+/**
+ * Contains a predicate that the process controller should evaluate,
+ * then an optional action is executed by the process controller if predicate is true. IFTTT
+ */
+export type PredicatedAction = {
+  ifThis: Predicate;
+  /**
+   * String that should be sent once the predicate is true
+   *
+   * If we need to do things like send multiple keystrokes in response to a single prompt, we will likely need to expand this to an array of values to send
+   */
+  then?: Action;
+};

packages/integration-tests/src/process-controller/predicated_action_macros.ts

@@ -0,0 +1,64 @@
+import { PredicatedActionBuilder } from './predicated_action_queue_builder.js';
+
+/**
+ * Convenience predicated actions that can be used to build up more complex CLI flows.
+ * By composing flows from reusable macros we will hopefully avoid the situation in the
+ * classic CLI E2E tests where changing one CLI prompt requires updates to 97742 different E2E prompts
+ */
+
+/**
+ * Reusable predicates: Wait for sandbox to finish and emit "✨  Total time: xx.xxs"
+ */
+export const waitForSandboxDeploymentToPrintTotalTime = () =>
+  new PredicatedActionBuilder().waitForLineIncludes('Total time');
+
+/**
+ * Reusable predicates: Wait for sandbox to become idle and emit "Watching for file changes..."
+ */
+export const waitForSandboxToBecomeIdle = () =>
+  new PredicatedActionBuilder().waitForLineIncludes(
+    'Watching for file changes...'
+  );
+
+/**
+ * Reusable predicated action: Wait for sandbox delete to prompt to delete all the resource and respond with yes
+ */
+export const confirmDeleteSandbox = () =>
+  new PredicatedActionBuilder()
+    .waitForLineIncludes(
+      'Are you sure you want to delete all the resources in your sandbox environment'
+    )
+    .sendYes();
+
+/**
+ * Reusable predicated action: Wait for sandbox to prompt on quitting to delete all the resource and respond with no
+ */
+export const rejectCleanupSandbox = () =>
+  new PredicatedActionBuilder()
+    .waitForLineIncludes(
+      'Would you like to delete all the resources in your sandbox environment'
+    )
+    .sendNo();
+
+/**
+ * Reusable predicated action: Wait for sandbox to become idle and then update the
+ * backend code which should trigger sandbox again
+ */
+export const updateFileContent = (from: URL, to: URL) => {
+  return waitForSandboxToBecomeIdle().updateFileContent(from, to);
+};
+
+/**
+ * Reusable predicated action: Wait for sandbox to become idle and then quit it (CTRL-C)
+ */
+export const interruptSandbox = () => waitForSandboxToBecomeIdle().sendCtrlC();
+
+/**
+ * Reusable predicated action: Wait for sandbox to finish deployment and assert that the deployment time is less
+ * than the threshold.
+ */
+export const ensureDeploymentTimeLessThan = (seconds: number) => {
+  return waitForSandboxDeploymentToPrintTotalTime().ensureDeploymentTimeLessThan(
+    seconds
+  );
+};

packages/integration-tests/src/process-controller/predicated_action_queue_builder.ts

@@ -0,0 +1,167 @@
+import {
+  ActionType,
+  PredicateType,
+  PredicatedAction,
+} from './predicated_action.js';
+import os from 'os';
+import fs from 'fs/promises';
+
+import { killExecaProcess } from './execa_process_killer.js';
+import { ExecaChildProcess } from 'execa';
+
+export const CONTROL_C = '\x03';
+/**
+ * Builder for a queue of Actions
+ */
+export class PredicatedActionBuilder {
+  private readonly predicatedActionQueue: PredicatedAction[] = [];
+
+  /**
+   * Append the action queue from another builder to this builder
+   */
+  append = (builder: PredicatedActionBuilder) => {
+    this.predicatedActionQueue.push(...builder.getPredicatedActionQueue());
+    return this;
+  };
+
+  /**
+   * Add a new predicated action to the queue with a predicate that matches a given string
+   */
+  waitForLineIncludes = (str: string) => {
+    this.predicatedActionQueue.push({
+      ifThis: {
+        predicateType: PredicateType.MATCHES_STRING_PREDICATE,
+        predicate: (line) => line.includes(str),
+      },
+    });
+    return this;
+  };
+
+  /**
+   * Adds a wait for ms milliseconds
+   */
+  waitFor = async (ms: number) => {
+    await new Promise((resolve) => setTimeout(resolve, ms));
+  };
+
+  /**
+   * Update the last predicated action to send str to the running process with no newline
+   */
+  send = (str: string) => {
+    this.getLastPredicatedAction().then = {
+      actionType:
+        str === CONTROL_C
+          ? ActionType.KILL_PROCESS
+          : ActionType.SEND_INPUT_TO_PROCESS,
+      action: async (execaProcess: ExecaChildProcess) => {
+        if (str === CONTROL_C) {
+          await killExecaProcess(execaProcess);
+        } else {
+          execaProcess.stdin?.write(str);
+        }
+      },
+    };
+    return this;
+  };
+
+  /**
+   * Update the last predicated action to update backend code by copying files from
+   * `from` location to `to` location.
+   */
+  updateFileContent = (from: URL, to: URL) => {
+    this.getLastPredicatedAction().then = {
+      actionType: ActionType.UPDATE_FILE_CONTENT,
+      action: async () => {
+        await fs.cp(from, to, {
+          recursive: true,
+        });
+      },
+    };
+    return this;
+  };
+
+  /**
+   * Update the last predicated action to validate that the deployment time is less than the one specified
+   */
+  ensureDeploymentTimeLessThan = (seconds: number) => {
+    this.getLastPredicatedAction().then = {
+      actionType: ActionType.ASSERT_ON_PROCESS_OUTPUT,
+      action: (strWithDeploymentTime: string) => {
+        const regex = /^✨ {2}Total time: (\d*.\d*).*$/;
+        const deploymentTime = strWithDeploymentTime.match(regex);
+        if (
+          deploymentTime &&
+          deploymentTime.length > 1 &&
+          !isNaN(+deploymentTime[1])
+        ) {
+          if (+deploymentTime[1] <= seconds) {
+            return;
+          }
+          throw new Error(
+            `Deployment time ${+deploymentTime[1]} seconds is higher than the threshold of ${seconds}`
+          );
+        } else {
+          throw new Error(
+            `Could not determine the deployment time. String was ${strWithDeploymentTime}`
+          );
+        }
+      },
+    };
+    return this;
+  };
+
+  /**
+   * Send line with a newline at the end
+   */
+  sendLine = (line: string) => {
+    this.send(`${line}${os.EOL}`);
+    return this;
+  };
+
+  /**
+   * Send `N\n`
+   */
+  sendNo = () => {
+    this.sendLine('N');
+    return this;
+  };
+
+  /**
+   * Send `Y\n`
+   */
+  sendYes = () => {
+    this.sendLine('Y');
+    return this;
+  };
+
+  /**
+   * Send SIGINT to the child process
+   */
+  sendCtrlC = () => {
+    this.send(CONTROL_C);
+    return this;
+  };
+
+  /**
+   * Get the currently queued actions
+   */
+  getPredicatedActionQueue = (): PredicatedAction[] => {
+    return this.predicatedActionQueue;
+  };
+
+  getLastPredicatedAction = () => {
+    if (this.predicatedActionQueue.length === 0) {
+      throw new Error('Must have a predicate to execute the action');
+    }
+    // this assertion is safe because we checked the length above
+    // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+    const lastPredicatedAction = this.predicatedActionQueue.at(-1)!;
+    if (typeof lastPredicatedAction.then === 'function') {
+      throw new Error(
+        'An action is already registered to the last predicate in the queue. Update the same action.'
+      );
+    }
+
+    return lastPredicatedAction;
+  };
+}