feat: Add usage data metrics for sandbox (#642)

* feat: Add usage data metrics for sandbox

* update api

* add more tests

* add changeset

* fix automated tests

* Refactor packageJsonReader

* update tsconfig

* add comments

* Add usage data emitter factory

* Update interface of usage metrics to take metrics and dimensions

* is it better?

* remove extra variable

* changing to 'on' data listener

* update snapshots

* minor changes

* change to use readline

* PR updates

* Move the uuid types dep

* PR updates

* update to use __dirname

* update snapshots

* Replace package json reeader in create-amplify

* small rename

* remove packageJson.name from installationId

.changeset/nervous-emus-deny.md

@@ -0,0 +1,11 @@
+---
+'@aws-amplify/backend-output-storage': patch
+'@aws-amplify/integration-tests': patch
+'@aws-amplify/backend-deployer': patch
+'create-amplify': patch
+'@aws-amplify/platform-core': patch
+'@aws-amplify/sandbox': patch
+'@aws-amplify/backend-cli': patch
+---
+
+Add usage data metrics

.eslint_dictionary.json

@@ -26,6 +26,7 @@
   "deployer",
   "disambiguator",
   "downlevel",
+  "durations",
   "dynamodb",
   "ecma",
   "enum",
@@ -46,6 +47,7 @@
   "graphql",
   "homedir",
   "hotswap",
+  "hotswapped",
   "iamv2",
   "identitypool",
   "idps",
@@ -92,6 +94,8 @@
   "schema",
   "schemas",
   "searchable",
+  "semver",
+  "serializable",
   "shopify",
   "shortstat",
   "sigint",

packages/backend-deployer/API.md

@@ -9,27 +9,43 @@ import { DeploymentType } from '@aws-amplify/plugin-types';
 
 // @public
 export type BackendDeployer = {
-    deploy: (backendId?: BackendIdentifier, deployProps?: DeployProps) => Promise<void>;
-    destroy: (backendId?: BackendIdentifier, destroyProps?: DestroyProps) => Promise<void>;
+    deploy: (backendId?: BackendIdentifier, deployProps?: DeployProps) => Promise<DeployResult>;
+    destroy: (backendId?: BackendIdentifier, destroyProps?: DestroyProps) => Promise<DestroyResult>;
 };
 
 // @public
 export class BackendDeployerFactory {
     static getInstance: () => BackendDeployer;
 }
 
+// @public (undocumented)
+export type DeploymentTimes = {
+    synthesisTime?: number;
+    totalTime?: number;
+};
+
 // @public (undocumented)
 export type DeployProps = {
     deploymentType?: DeploymentType;
     secretLastUpdated?: Date;
     validateAppSources?: boolean;
 };
 
+// @public (undocumented)
+export type DeployResult = {
+    deploymentTimes: DeploymentTimes;
+};
+
 // @public (undocumented)
 export type DestroyProps = {
     deploymentType?: DeploymentType;
 };
 
+// @public (undocumented)
+export type DestroyResult = {
+    deploymentTimes: DeploymentTimes;
+};
+
 // (No @packageDocumentation comment for this package)
 
 ```

packages/backend-deployer/src/cdk_deployer.ts

@@ -1,9 +1,12 @@
 import { execa } from 'execa';
 import stream from 'stream';
+import readline from 'readline';
 import {
   BackendDeployer,
   DeployProps,
+  DeployResult,
   DestroyProps,
+  DestroyResult,
 } from './cdk_deployer_singleton_factory.js';
 import { CdkErrorMapper } from './cdk_error_mapper.js';
 import { BackendIdentifier, DeploymentType } from '@aws-amplify/plugin-types';
@@ -47,7 +50,7 @@ export class CDKDeployer implements BackendDeployer {
       }
     }
 
-    await this.invokeCdk(
+    return this.invokeCdk(
       InvokableCommand.DEPLOY,
       backendId,
       deployProps?.deploymentType,
@@ -62,7 +65,7 @@ export class CDKDeployer implements BackendDeployer {
     backendId?: BackendIdentifier,
     destroyProps?: DestroyProps
   ) => {
-    await this.invokeCdk(
+    return this.invokeCdk(
       InvokableCommand.DESTROY,
       backendId,
       destroyProps?.deploymentType,
@@ -104,7 +107,7 @@ export class CDKDeployer implements BackendDeployer {
     backendId?: BackendIdentifier,
     deploymentType?: DeploymentType,
     additionalArguments?: string[]
-  ) => {
+  ): Promise<DeployResult | DestroyResult> => {
     // Basic args
     const cdkCommandArgs = [
       'cdk',
@@ -145,7 +148,7 @@ export class CDKDeployer implements BackendDeployer {
     }
 
     try {
-      await this.executeChildProcess('npx', cdkCommandArgs);
+      return await this.executeChildProcess('npx', cdkCommandArgs);
     } catch (err) {
       throw this.cdkErrorMapper.getHumanReadableError(err as Error);
     }
@@ -161,23 +164,52 @@ export class CDKDeployer implements BackendDeployer {
     // actionable errors being hidden among the stdout. Moreover execa errors are
     // useless when calling CLIs unless you made execa calling error.
     let aggregatedStderr = '';
-    const aggregatorStream = new stream.Writable();
-    aggregatorStream._write = function (chunk, encoding, done) {
+    const aggregatorStderrStream = new stream.Writable();
+    aggregatorStderrStream._write = function (chunk, encoding, done) {
       aggregatedStderr += chunk;
       done();
     };
     const childProcess = execa(command, cdkCommandArgs, {
       stdin: 'inherit',
-      stdout: 'inherit',
+      stdout: 'pipe',
       stderr: 'pipe',
     });
-    childProcess.stderr?.pipe(aggregatorStream);
+    childProcess.stderr?.pipe(aggregatorStderrStream);
+    childProcess.stdout?.pipe(process.stdout);
+
+    const cdkOutput = { deploymentTimes: {} };
+    if (childProcess.stdout) {
+      await this.populateCDKOutputFromStdout(cdkOutput, childProcess.stdout);
+    }
 
     try {
       await childProcess;
+      return cdkOutput;
     } catch (error) {
       // swallow execa error which is not really helpful, rather throw stderr
       throw new Error(aggregatedStderr);
     }
   };
+
+  private populateCDKOutputFromStdout = async (
+    output: DeployResult | DestroyResult,
+    stdout: stream.Readable
+  ) => {
+    const regexTotalTime = /✨ {2}Total time: (\d*\.*\d*)s.*/;
+    const regexSynthTime = /✨ {2}Synthesis time: (\d*\.*\d*)s/;
+    const reader = readline.createInterface(stdout);
+    for await (const line of reader) {
+      if (line.includes('✨')) {
+        // Good chance that it contains timing information
+        const totalTime = line.match(regexTotalTime);
+        if (totalTime && totalTime.length > 1 && !isNaN(+totalTime[1])) {
+          output.deploymentTimes.totalTime = +totalTime[1];
+        }
+        const synthTime = line.match(regexSynthTime);
+        if (synthTime && synthTime.length > 1 && !isNaN(+synthTime[1])) {
+          output.deploymentTimes.synthesisTime = +synthTime[1];
+        }
+      }
+    }
+  };
 }

packages/backend-deployer/src/cdk_deployer_singleton_factory.ts

@@ -9,22 +9,35 @@ export type DeployProps = {
   validateAppSources?: boolean;
 };
 
+export type DeployResult = {
+  deploymentTimes: DeploymentTimes;
+};
+
 export type DestroyProps = {
   deploymentType?: DeploymentType;
 };
 
+export type DestroyResult = {
+  deploymentTimes: DeploymentTimes;
+};
+
+export type DeploymentTimes = {
+  synthesisTime?: number;
+  totalTime?: number;
+};
+
 /**
  * Invokes an invokable command
  */
 export type BackendDeployer = {
   deploy: (
     backendId?: BackendIdentifier,
     deployProps?: DeployProps
-  ) => Promise<void>;
+  ) => Promise<DeployResult>;
   destroy: (
     backendId?: BackendIdentifier,
     destroyProps?: DestroyProps
-  ) => Promise<void>;
+  ) => Promise<DestroyResult>;
 };
 
 /**

packages/backend-deployer/src/cdk_error_mapper.test.ts

@@ -5,61 +5,76 @@ import { CdkErrorMapper } from './cdk_error_mapper.js';
 const testErrorMappings = [
   {
     errorMessage: 'UnknownError',
-    expectedString: 'UnknownError',
+    expectedTopLevelErrorMessage: 'UnknownError',
+    expectedDownstreamErrorMessage: undefined,
   },
   {
     errorMessage: 'ExpiredToken',
-    expectedString:
+    expectedTopLevelErrorMessage:
       '[ExpiredToken]: The security token included in the request is invalid.',
+    expectedDownstreamErrorMessage: 'ExpiredToken',
   },
   {
     errorMessage: 'Access Denied',
-    expectedString:
+    expectedTopLevelErrorMessage:
       '[AccessDenied]: The deployment role does not have sufficient permissions to perform this deployment.',
+    expectedDownstreamErrorMessage: 'Access Denied',
   },
   {
     errorMessage: 'Has the environment been bootstrapped',
-    expectedString:
+    expectedTopLevelErrorMessage:
       '[BootstrapFailure]: This AWS account and region has not been bootstrapped. Run `cdk bootstrap aws://{YOUR_ACCOUNT_ID}/{YOUR_REGION}` locally to resolve this.',
+    expectedDownstreamErrorMessage: 'Has the environment been bootstrapped',
   },
   {
     errorMessage: 'amplify/backend.ts',
-    expectedString:
+    expectedTopLevelErrorMessage:
       '[SynthError]: Unable to build Amplify backend. Check your backend definition in the `amplify` folder.',
+    expectedDownstreamErrorMessage: 'amplify/backend.ts',
   },
   {
-    errorMessage: 'ROLLBACK_COMPLETE',
-    expectedString:
-      '[CloudFormationFailure]: The CloudFormation deployment has failed. Find more information in the CloudFormation AWS Console for this stack.',
-  },
-  {
-    errorMessage: 'ROLLBACK_FAILED',
-    expectedString:
+    errorMessage: '❌ Deployment failed: something bad happened\n',
+    expectedTopLevelErrorMessage:
       '[CloudFormationFailure]: The CloudFormation deployment has failed. Find more information in the CloudFormation AWS Console for this stack.',
+    expectedDownstreamErrorMessage: 'something bad happened',
   },
   {
     errorMessage:
       'CFN error happened: Updates are not allowed for property: some property',
-    expectedString:
+    expectedTopLevelErrorMessage:
       '[UpdateNotSupported]: The changes that you are trying to apply are not supported.',
+    expectedDownstreamErrorMessage:
+      'CFN error happened: Updates are not allowed for property: some property',
   },
   {
     errorMessage:
       'CFN error happened: Invalid AttributeDataType input, consider using the provided AttributeDataType enum',
-    expectedString:
+    expectedTopLevelErrorMessage:
       '[UpdateNotSupported]: User pool attributes cannot be changed after a user pool has been created.',
+    expectedDownstreamErrorMessage:
+      'CFN error happened: Invalid AttributeDataType input, consider using the provided AttributeDataType enum',
   },
 ];
 
 void describe('invokeCDKCommand', { concurrency: 1 }, () => {
   const cdkErrorMapper = new CdkErrorMapper();
-  testErrorMappings.forEach(({ errorMessage, expectedString }) => {
-    void it(`handles ${errorMessage} error`, () => {
-      const humanReadableError = cdkErrorMapper.getHumanReadableError(
-        new Error(errorMessage)
-      );
-      assert.equal(humanReadableError.message, expectedString);
-      assert.equal((humanReadableError.cause as Error).message, errorMessage);
-    });
-  });
+  testErrorMappings.forEach(
+    ({
+      errorMessage,
+      expectedTopLevelErrorMessage,
+      expectedDownstreamErrorMessage,
+    }) => {
+      void it(`handles ${errorMessage} error`, () => {
+        const humanReadableError = cdkErrorMapper.getHumanReadableError(
+          new Error(errorMessage)
+        );
+        assert.equal(humanReadableError.message, expectedTopLevelErrorMessage);
+        expectedDownstreamErrorMessage &&
+          assert.equal(
+            (humanReadableError.cause as Error).message,
+            expectedDownstreamErrorMessage
+          );
+      });
+    }
+  );
 });

packages/backend-deployer/src/cdk_error_mapper.ts

@@ -23,10 +23,15 @@ export class CdkErrorMapper {
     },
     {
       // the backend entry point file is referenced in the stack indicating a problem in customer code
-      errorRegex: /amplify\/backend.ts/,
+      errorRegex: /amplify\/backend/,
       humanReadableError:
         '[SynthError]: Unable to build Amplify backend. Check your backend definition in the `amplify` folder.',
     },
+    {
+      errorRegex: /SyntaxError:(.*)\n/,
+      humanReadableError:
+        '[SyntaxError]: Unable to build Amplify backend. Check your backend definition in the `amplify` folder.',
+    },
     {
       errorRegex: /Updates are not allowed for property/,
       humanReadableError:
@@ -43,7 +48,7 @@ export class CdkErrorMapper {
     },
     {
       // Note that the order matters, this should be the last as it captures generic CFN error
-      errorRegex: /ROLLBACK_(COMPLETE|FAILED)/,
+      errorRegex: /❌ Deployment failed: (.*)\n/,
       humanReadableError:
         '[CloudFormationFailure]: The CloudFormation deployment has failed. Find more information in the CloudFormation AWS Console for this stack.',
     },
@@ -54,8 +59,14 @@ export class CdkErrorMapper {
       knownError.errorRegex.test(error.message)
     );
 
-    return new Error(matchingError?.humanReadableError || error.message, {
-      cause: error,
-    });
+    if (matchingError) {
+      const underlyingMessage = error.message.match(matchingError.errorRegex);
+      error.message =
+        underlyingMessage && underlyingMessage.length == 2
+          ? underlyingMessage[1]
+          : error.message;
+      return new Error(matchingError.humanReadableError, { cause: error });
+    }
+    return error;
   };
 }

packages/backend-output-storage/API.md

@@ -8,8 +8,8 @@
 
 import { BackendOutputEntry } from '@aws-amplify/plugin-types';
 import { BackendOutputStorageStrategy } from '@aws-amplify/plugin-types';
-import * as _fs from 'fs';
 import * as _os from 'os';
+import { PackageJsonReader } from '@aws-amplify/platform-core';
 import { Stack } from 'aws-cdk-lib';
 
 // @public (undocumented)
@@ -23,7 +23,7 @@ export type AttributionMetadata = {
 
 // @public
 export class AttributionMetadataStorage {
-    constructor(fs?: typeof _fs, os?: typeof _os);
+    constructor(os?: typeof _os, packageJsonReader?: PackageJsonReader);
     storeAttributionMetadata: (stack: Stack, stackType: string, libraryPackageJsonAbsolutePath: string, additionalMetadata?: Record<string, string>) => void;
 }