feat: Add usage data metrics for sandbox

.changeset/nervous-emus-deny.md

@@ -0,0 +1,11 @@
+---
+'@aws-amplify/backend-output-storage': patch
+'@aws-amplify/integration-tests': patch
+'@aws-amplify/backend-deployer': patch
+'create-amplify': patch
+'@aws-amplify/platform-core': patch
+'@aws-amplify/sandbox': patch
+'@aws-amplify/backend-cli': patch
+---
+
+Add usage data metrics

.eslint_dictionary.json

@@ -26,6 +26,7 @@
   "deployer",
   "disambiguator",
   "downlevel",
+  "durations",
   "dynamodb",
   "ecma",
   "enum",
@@ -46,6 +47,7 @@
   "graphql",
   "homedir",
   "hotswap",
+  "hotswapped",
   "iamv2",
   "identitypool",
   "idps",
@@ -92,6 +94,8 @@
   "schema",
   "schemas",
   "searchable",
+  "semver",
+  "serializable",
   "shopify",
   "shortstat",
   "sigint",

packages/backend-deployer/API.md

@@ -5,12 +5,13 @@
 ```ts
 
 import { BackendIdentifier } from '@aws-amplify/plugin-types';
+import { DeploymentTimes } from '@aws-amplify/platform-core';
 import { DeploymentType } from '@aws-amplify/plugin-types';
 
 // @public
 export type BackendDeployer = {
-    deploy: (backendId?: BackendIdentifier, deployProps?: DeployProps) => Promise<void>;
-    destroy: (backendId?: BackendIdentifier, destroyProps?: DestroyProps) => Promise<void>;
+    deploy: (backendId?: BackendIdentifier, deployProps?: DeployProps) => Promise<DeployResult>;
+    destroy: (backendId?: BackendIdentifier, destroyProps?: DestroyProps) => Promise<DestroyResult>;
 };
 
 // @public
@@ -25,11 +26,22 @@ export type DeployProps = {
     validateAppSources?: boolean;
 };
 
+// @public (undocumented)
+export type DeployResult = {
+    hotswapped?: boolean;
+    deploymentTimes: DeploymentTimes;
+};
+
 // @public (undocumented)
 export type DestroyProps = {
     deploymentType?: DeploymentType;
 };
 
+// @public (undocumented)
+export type DestroyResult = {
+    deploymentTimes: DeploymentTimes;
+};
+
 // (No @packageDocumentation comment for this package)
 
 ```

packages/backend-deployer/src/cdk_deployer.test.ts

@@ -25,7 +25,7 @@ void describe('invokeCDKCommand', () => {
 
   const invoker = new CDKDeployer(new CdkErrorMapper(), backendLocator);
   const execaMock = mock.method(invoker, 'executeChildProcess', () =>
-    Promise.resolve()
+    Promise.resolve('some cdk output')
   );
 
   beforeEach(() => {

packages/backend-deployer/src/cdk_deployer.ts

@@ -7,7 +7,11 @@ import {
 } from './cdk_deployer_singleton_factory.js';
 import { CdkErrorMapper } from './cdk_error_mapper.js';
 import { BackendIdentifier, DeploymentType } from '@aws-amplify/plugin-types';
-import { BackendLocator, CDKContextKey } from '@aws-amplify/platform-core';
+import {
+  BackendLocator,
+  CDKContextKey,
+  DeploymentTimes,
+} from '@aws-amplify/platform-core';
 import { BackendDeployerEnvironmentVariables } from './environment_variables.js';
 
 /**
@@ -47,12 +51,17 @@ export class CDKDeployer implements BackendDeployer {
       }
     }
 
-    await this.invokeCdk(
+    const cdkOutput = await this.invokeCdk(
       InvokableCommand.DEPLOY,
       backendId,
       deployProps?.deploymentType,
       cdkCommandArgs
     );
+
+    return {
+      deploymentTimes: this.getDeploymentTimes(cdkOutput),
+      hotswapped: this.deploymentHotswapped(cdkOutput),
+    };
   };
 
   /**
@@ -62,12 +71,15 @@ export class CDKDeployer implements BackendDeployer {
     backendId?: BackendIdentifier,
     destroyProps?: DestroyProps
   ) => {
-    await this.invokeCdk(
+    const cdkOutput = await this.invokeCdk(
       InvokableCommand.DESTROY,
       backendId,
       destroyProps?.deploymentType,
       ['--force']
     );
+    return {
+      deploymentTimes: this.getDeploymentTimes(cdkOutput),
+    };
   };
 
   private invokeTsc = async (deployProps?: DeployProps) => {
@@ -104,7 +116,7 @@ export class CDKDeployer implements BackendDeployer {
     backendId?: BackendIdentifier,
     deploymentType?: DeploymentType,
     additionalArguments?: string[]
-  ) => {
+  ): Promise<string> => {
     // Basic args
     const cdkCommandArgs = [
       'cdk',
@@ -145,7 +157,7 @@ export class CDKDeployer implements BackendDeployer {
     }
 
     try {
-      await this.executeChildProcess('npx', cdkCommandArgs);
+      return await this.executeChildProcess('npx', cdkCommandArgs);
     } catch (err) {
       throw this.cdkErrorMapper.getHumanReadableError(err as Error);
     }
@@ -161,23 +173,54 @@ export class CDKDeployer implements BackendDeployer {
     // actionable errors being hidden among the stdout. Moreover execa errors are
     // useless when calling CLIs unless you made execa calling error.
     let aggregatedStderr = '';
-    const aggregatorStream = new stream.Writable();
-    aggregatorStream._write = function (chunk, encoding, done) {
+    let aggregatedStdout = '';
+    const aggregatorStderrStream = new stream.Writable();
+    const aggregatorStdoutStream = new stream.Writable();
+    aggregatorStderrStream._write = function (chunk, encoding, done) {
       aggregatedStderr += chunk;
       done();
     };
+    aggregatorStdoutStream._write = function (chunk, encoding, done) {
+      aggregatedStdout += chunk;
+      done();
+    };
     const childProcess = execa(command, cdkCommandArgs, {
       stdin: 'inherit',
-      stdout: 'inherit',
+      stdout: 'pipe',
       stderr: 'pipe',
     });
-    childProcess.stderr?.pipe(aggregatorStream);
-
+    childProcess.stderr?.pipe(aggregatorStderrStream);
+    childProcess.stdout?.pipe(aggregatorStdoutStream);
+    childProcess.stdout?.pipe(process.stdout);
     try {
       await childProcess;
+      return aggregatedStdout;
     } catch (error) {
       // swallow execa error which is not really helpful, rather throw stderr
       throw new Error(aggregatedStderr);
     }
   };
+
+  private getDeploymentTimes = (cdkOutput: string) => {
+    const deploymentTimes: DeploymentTimes = {};
+    // the time can be in fractional or whole seconds. 24.3, 24, 24.22 etc.
+    const regexTotalTime = /✨ {2}Total time: (\d*\.*\d*)s.*/;
+    const totalTime = cdkOutput.match(regexTotalTime);
+    if (totalTime && totalTime.length > 1 && !isNaN(+totalTime[1])) {
+      deploymentTimes.totalTime = +totalTime[1];
+    }
+
+    const regexSynthTime = /✨ {2}Synthesis time: (\d*\.*\d*)s/;
+    const synthTime = cdkOutput.match(regexSynthTime);
+    if (synthTime && synthTime.length > 1 && !isNaN(+synthTime[1])) {
+      deploymentTimes.synthesisTime = +synthTime[1];
+    }
+
+    return deploymentTimes;
+  };
+
+  private deploymentHotswapped = (cdkOutput: string) => {
+    const regexTotalTime = /✨.*hotswapped!/;
+    return regexTotalTime.test(cdkOutput);
+  };
 }

packages/backend-deployer/src/cdk_deployer_singleton_factory.ts

@@ -1,30 +1,39 @@
 import { BackendIdentifier, DeploymentType } from '@aws-amplify/plugin-types';
 import { CDKDeployer } from './cdk_deployer.js';
 import { CdkErrorMapper } from './cdk_error_mapper.js';
-import { BackendLocator } from '@aws-amplify/platform-core';
+import { BackendLocator, DeploymentTimes } from '@aws-amplify/platform-core';
 
 export type DeployProps = {
   deploymentType?: DeploymentType;
   secretLastUpdated?: Date;
   validateAppSources?: boolean;
 };
 
+export type DeployResult = {
+  hotswapped?: boolean;
+  deploymentTimes: DeploymentTimes;
+};
+
 export type DestroyProps = {
   deploymentType?: DeploymentType;
 };
 
+export type DestroyResult = {
+  deploymentTimes: DeploymentTimes;
+};
+
 /**
  * Invokes an invokable command
  */
 export type BackendDeployer = {
   deploy: (
     backendId?: BackendIdentifier,
     deployProps?: DeployProps
-  ) => Promise<void>;
+  ) => Promise<DeployResult>;
   destroy: (
     backendId?: BackendIdentifier,
     destroyProps?: DestroyProps
-  ) => Promise<void>;
+  ) => Promise<DestroyResult>;
 };
 
 /**

packages/backend-deployer/src/cdk_error_mapper.test.ts

@@ -5,61 +5,76 @@ import { CdkErrorMapper } from './cdk_error_mapper.js';
 const testErrorMappings = [
   {
     errorMessage: 'UnknownError',
-    expectedString: 'UnknownError',
+    expectedTopLevelErrorMessage: 'UnknownError',
+    expectedDownstreamErrorMessage: undefined,
   },
   {
     errorMessage: 'ExpiredToken',
-    expectedString:
+    expectedTopLevelErrorMessage:
       '[ExpiredToken]: The security token included in the request is invalid.',
+    expectedDownstreamErrorMessage: 'ExpiredToken',
   },
   {
     errorMessage: 'Access Denied',
-    expectedString:
+    expectedTopLevelErrorMessage:
       '[AccessDenied]: The deployment role does not have sufficient permissions to perform this deployment.',
+    expectedDownstreamErrorMessage: 'Access Denied',
   },
   {
     errorMessage: 'Has the environment been bootstrapped',
-    expectedString:
+    expectedTopLevelErrorMessage:
       '[BootstrapFailure]: This AWS account and region has not been bootstrapped. Run `cdk bootstrap aws://{YOUR_ACCOUNT_ID}/{YOUR_REGION}` locally to resolve this.',
+    expectedDownstreamErrorMessage: 'Has the environment been bootstrapped',
   },
   {
     errorMessage: 'amplify/backend.ts',
-    expectedString:
+    expectedTopLevelErrorMessage:
       '[SynthError]: Unable to build Amplify backend. Check your backend definition in the `amplify` folder.',
+    expectedDownstreamErrorMessage: 'amplify/backend.ts',
   },
   {
-    errorMessage: 'ROLLBACK_COMPLETE',
-    expectedString:
-      '[CloudFormationFailure]: The CloudFormation deployment has failed. Find more information in the CloudFormation AWS Console for this stack.',
-  },
-  {
-    errorMessage: 'ROLLBACK_FAILED',
-    expectedString:
+    errorMessage: '❌ Deployment failed: something bad happened\n',
+    expectedTopLevelErrorMessage:
       '[CloudFormationFailure]: The CloudFormation deployment has failed. Find more information in the CloudFormation AWS Console for this stack.',
+    expectedDownstreamErrorMessage: 'something bad happened',
   },
   {
     errorMessage:
       'CFN error happened: Updates are not allowed for property: some property',
-    expectedString:
+    expectedTopLevelErrorMessage:
       '[UpdateNotSupported]: The changes that you are trying to apply are not supported.',
+    expectedDownstreamErrorMessage:
+      'CFN error happened: Updates are not allowed for property: some property',
   },
   {
     errorMessage:
       'CFN error happened: Invalid AttributeDataType input, consider using the provided AttributeDataType enum',
-    expectedString:
+    expectedTopLevelErrorMessage:
       '[UpdateNotSupported]: User pool attributes cannot be changed after a user pool has been created.',
+    expectedDownstreamErrorMessage:
+      'CFN error happened: Invalid AttributeDataType input, consider using the provided AttributeDataType enum',
   },
 ];
 
 void describe('invokeCDKCommand', { concurrency: 1 }, () => {
   const cdkErrorMapper = new CdkErrorMapper();
-  testErrorMappings.forEach(({ errorMessage, expectedString }) => {
-    void it(`handles ${errorMessage} error`, () => {
-      const humanReadableError = cdkErrorMapper.getHumanReadableError(
-        new Error(errorMessage)
-      );
-      assert.equal(humanReadableError.message, expectedString);
-      assert.equal((humanReadableError.cause as Error).message, errorMessage);
-    });
-  });
+  testErrorMappings.forEach(
+    ({
+      errorMessage,
+      expectedTopLevelErrorMessage,
+      expectedDownstreamErrorMessage,
+    }) => {
+      void it(`handles ${errorMessage} error`, () => {
+        const humanReadableError = cdkErrorMapper.getHumanReadableError(
+          new Error(errorMessage)
+        );
+        assert.equal(humanReadableError.message, expectedTopLevelErrorMessage);
+        expectedDownstreamErrorMessage &&
+          assert.equal(
+            (humanReadableError.cause as Error).message,
+            expectedDownstreamErrorMessage
+          );
+      });
+    }
+  );
 });

packages/backend-deployer/src/cdk_error_mapper.ts

@@ -23,10 +23,15 @@ export class CdkErrorMapper {
     },
     {
       // the backend entry point file is referenced in the stack indicating a problem in customer code
-      errorRegex: /amplify\/backend.ts/,
+      errorRegex: /amplify\/backend/,
       humanReadableError:
         '[SynthError]: Unable to build Amplify backend. Check your backend definition in the `amplify` folder.',
     },
+    {
+      errorRegex: /SyntaxError:(.*)\n/,
+      humanReadableError:
+        '[SyntaxError]: Unable to build Amplify backend. Check your backend definition in the `amplify` folder.',
+    },
     {
       errorRegex: /Updates are not allowed for property/,
       humanReadableError:
@@ -43,7 +48,7 @@ export class CdkErrorMapper {
     },
     {
       // Note that the order matters, this should be the last as it captures generic CFN error
-      errorRegex: /ROLLBACK_(COMPLETE|FAILED)/,
+      errorRegex: /❌ Deployment failed: (.*)\n/,
       humanReadableError:
         '[CloudFormationFailure]: The CloudFormation deployment has failed. Find more information in the CloudFormation AWS Console for this stack.',
     },
@@ -54,8 +59,14 @@ export class CdkErrorMapper {
       knownError.errorRegex.test(error.message)
     );
 
-    return new Error(matchingError?.humanReadableError || error.message, {
-      cause: error,
-    });
+    if (matchingError) {
+      const underlyingMessage = error.message.match(matchingError.errorRegex);
+      error.message =
+        underlyingMessage && underlyingMessage.length == 2
+          ? underlyingMessage[1]
+          : error.message;
+      return new Error(matchingError.humanReadableError, { cause: error });
+    }
+    return error;
   };
 }