fix(cli): ensure Pulumi refresh if webiny watch command has been run (

#4537)

packages/cli-plugin-deploy-pulumi/src/commands/deploy.ts

@@ -4,6 +4,7 @@ import { PackagesBuilder } from "./buildPackages/PackagesBuilder";
 import { pulumiLoginSelectStack } from "./deploy/pulumiLoginSelectStack";
 import { executeDeploy } from "./deploy/executeDeploy";
 import { executePreview } from "./deploy/executePreview";
+import { executeRefresh } from "~/commands/deploy/executeRefresh";
 import { setTimeout } from "node:timers/promises";
 import type { CliContext } from "@webiny/cli/types";
 
@@ -87,6 +88,10 @@ export const deployCommand = (params: IDeployParams, context: CliContext) => {
 
             console.log();
 
+            // A Pulumi refresh might be executed before the deploy. For example,
+            // this is needed if the user run the watch command prior to the deploy.
+            await executeRefresh(commandParams);
+
             if (inputs.preview) {
                 await executePreview(commandParams);
             } else {

packages/cli-plugin-deploy-pulumi/src/commands/deploy/executeRefresh.ts

@@ -0,0 +1,78 @@
+import { Context, IPulumi, IUserCommandInput } from "~/types";
+import { measureDuration } from "~/utils";
+import ora from "ora";
+import { isCI } from "ci-info";
+import { getMustRefreshBeforeDeploy } from "~/utils";
+import {
+    createEnvConfiguration,
+    withEnv,
+    withEnvVariant,
+    withProjectName,
+    withRegion
+} from "~/utils/env";
+
+export interface IExecuteRefreshParams {
+    inputs: IUserCommandInput;
+    context: Context;
+    pulumi: Pick<IPulumi, "run">;
+}
+
+export const executeRefresh = async ({ inputs, context, pulumi }: IExecuteRefreshParams) => {
+    const mustRefreshBeforeDeploy = getMustRefreshBeforeDeploy(context);
+    if (!mustRefreshBeforeDeploy) {
+        return;
+    }
+
+    // We always show deployment logs when doing previews.
+    const showDeploymentLogs = isCI || inputs.deploymentLogs;
+
+    const getDeploymentDuration = measureDuration();
+
+    const spinner = ora("Refreshing Pulumi state...");
+
+    try {
+        const subprocess = pulumi.run({
+            command: ["refresh", "--yes"],
+            execa: {
+                args: {
+                    debug: !!inputs.debug
+                },
+                env: createEnvConfiguration({
+                    configurations: [
+                        withRegion(inputs),
+                        withEnv(inputs),
+                        withEnvVariant(inputs),
+                        withProjectName(context)
+                    ]
+                })
+            }
+        });
+
+        if (showDeploymentLogs) {
+            subprocess.stdout!.pipe(process.stdout);
+            subprocess.stderr!.pipe(process.stderr);
+            await subprocess;
+        } else {
+            spinner.start();
+            await subprocess;
+        }
+
+        const message = `Pulumi state refreshed in ${getDeploymentDuration()}.`;
+
+        if (showDeploymentLogs) {
+            context.success(message);
+        } else {
+            spinner.succeed(message);
+        }
+    } catch (e) {
+        // If the deployment logs were already shown, we don't want to do anything.
+        if (showDeploymentLogs) {
+            throw e;
+        }
+
+        spinner.fail(`Refresh failed. For more details, please check the error logs below.`);
+        console.log();
+        console.log(e.stderr || e.stdout || e.message);
+        throw e;
+    }
+};

packages/cli-plugin-deploy-pulumi/src/commands/index.ts

@@ -233,7 +233,7 @@ export const commands: CliCommandPlugin[] = [
                         yargs.option("inspect", {
                             alias: "i",
                             describe:
-                                "Enable Node debugger (used with local AWS Lambda development)",
+                                "[EXPERIMENTAL] Enable Node debugger (used with local AWS Lambda development)",
                             type: "boolean"
                         });
                         yargs.option("depth", {

packages/cli-plugin-deploy-pulumi/src/commands/newWatch.ts

@@ -4,14 +4,14 @@ import path from "path";
 import { getProject, getProjectApplication } from "@webiny/cli/utils";
 import get from "lodash/get";
 import merge from "lodash/merge";
-import { getDeploymentId, loadEnvVariables, runHook } from "~/utils";
+import type inspectorType from "inspector";
+import { getDeploymentId, loadEnvVariables, runHook, setMustRefreshBeforeDeploy } from "~/utils";
 import { getIotEndpoint } from "./newWatch/getIotEndpoint";
 import { listLambdaFunctions } from "./newWatch/listLambdaFunctions";
 import { listPackages } from "./newWatch/listPackages";
 import { PackagesWatcher } from "./newWatch/watchers/PackagesWatcher";
 import { initInvocationForwarding } from "./newWatch/initInvocationForwarding";
 import { replaceLambdaFunctions } from "./newWatch/replaceLambdaFunctions";
-import type inspectorType from "inspector";
 
 // Do not allow watching "prod" and "production" environments. On the Pulumi CLI side, the command
 // is still in preview mode, so it's definitely not wise to use it on production environments.
@@ -103,25 +103,42 @@ export const newWatch = async (inputs: IUserCommandInput, context: Context) => {
         );
     }
 
-    let lambdaFunctions = listLambdaFunctions(inputs);
+    const functionsList = listLambdaFunctions({
+        env: inputs.env,
+        folder: inputs.folder,
+        variant: inputs.variant,
+        whitelist: inputs.function
+    });
 
-    // Let's filter out the authorizer function, as it's not needed for the watch command.
-    if (projectApplication.id === "core") {
-        lambdaFunctions = lambdaFunctions.filter(fn => {
-            const isAuthorizerFunction = fn.name.includes("watch-command-iot-authorizer");
-            return !isAuthorizerFunction;
-        });
-    }
+    const deployCommand = `yarn webiny deploy ${projectApplication.id} --env ${inputs.env}`;
+    const learnMoreLink = "https://webiny.link/local-aws-lambda-development";
+    const troubleshootingLink = learnMoreLink + "#troubleshooting";
+
+    if (functionsList.meta.count === 0) {
+        // If functions exist, but none are selected for watching, show a warning.
+        if (functionsList.meta.totalCount > 0) {
+            context.info(
+                [
+                    "No AWS Lambda functions will be invoked locally. If this is unexpected, you can try the following:",
+                    " ‣ stop the current development session",
+                    " ‣ redeploy the %s application by running %s command",
+                    " ‣ start a new %s session by rerunning %s command",
+                    "",
+                    "Learn more: %s"
+                ].join("\n"),
+                projectApplication.name,
+                deployCommand,
+                "webiny watch",
+                "webiny watch",
+                troubleshootingLink
+            );
+            console.log();
+        }
 
-    if (!lambdaFunctions.length) {
-        context.debug("No AWS Lambda functions will be invoked locally.");
         await packagesWatcher.watch();
         return;
     }
 
-    const deployCommand = `yarn webiny deploy ${projectApplication.id} --env ${inputs.env}`;
-    const learnMoreLink = "https://webiny.link/local-aws-lambda-development";
-
     context.info(`Local AWS Lambda development session started.`);
     context.warning(
         `Note that once the session is terminated, the %s application will no longer work. To fix this, you %s redeploy it via the %s command. Learn more: %s.`,
@@ -133,7 +150,7 @@ export const newWatch = async (inputs: IUserCommandInput, context: Context) => {
 
     context.debug(
         "The events for following AWS Lambda functions will be forwarded locally: ",
-        lambdaFunctions.map(fn => fn.name)
+        functionsList.list.map(fn => fn.name)
     );
 
     console.log();
@@ -165,12 +182,20 @@ export const newWatch = async (inputs: IUserCommandInput, context: Context) => {
     const sessionId = new Date().getTime();
     const increaseTimeout = inputs.increaseTimeout;
 
+    // We want to ensure a Pulumi refresh is made before the next deploy.
+    setMustRefreshBeforeDeploy(context);
+
     // Ignore promise, we don't need to wait for this to finish.
     replaceLambdaFunctions({
+        context,
+        env: inputs.env,
+        folder: inputs.folder,
+        variant: inputs.variant,
+
         iotEndpoint,
         iotEndpointTopic,
         sessionId,
-        lambdaFunctions,
+        functionsList,
         increaseTimeout
     });
 
@@ -189,7 +214,7 @@ export const newWatch = async (inputs: IUserCommandInput, context: Context) => {
     initInvocationForwarding({
         iotEndpoint,
         iotEndpointTopic,
-        lambdaFunctions,
+        functionsList,
         sessionId
     });
 

packages/cli-plugin-deploy-pulumi/src/commands/newWatch/initInvocationForwarding.ts

@@ -2,6 +2,7 @@ import path from "path";
 import { Worker } from "worker_threads";
 import { compress, decompress } from "@webiny/utils/compression/gzip";
 import mqtt from "mqtt";
+import type { listLambdaFunctions } from "~/commands/newWatch/listLambdaFunctions";
 
 const WEBINY_WATCH_FN_INVOCATION_EVENT = "webiny.watch.functionInvocation";
 const WEBINY_WATCH_FN_INVOCATION_RESULT_EVENT = "webiny.watch.functionInvocationResult";
@@ -33,14 +34,14 @@ export interface IInitInvocationForwardingParams {
     iotEndpoint: string;
     iotEndpointTopic: string;
     sessionId: number;
-    lambdaFunctions: IInitInvocationForwardingParamsLambdaFunction[];
+    functionsList: ReturnType<typeof listLambdaFunctions>;
 }
 
 export const initInvocationForwarding = async ({
     iotEndpoint,
     iotEndpointTopic,
     sessionId,
-    lambdaFunctions
+    functionsList
 }: IInitInvocationForwardingParams) => {
     const client = await mqtt.connectAsync(iotEndpoint);
 
@@ -57,7 +58,7 @@ export const initInvocationForwarding = async ({
             return;
         }
 
-        const invokedLambdaFunction = lambdaFunctions.find(
+        const invokedLambdaFunction = functionsList.list.find(
             lambdaFunction => lambdaFunction.name === payload.data.functionName
         );
         if (!invokedLambdaFunction) {

packages/cli-plugin-deploy-pulumi/src/commands/newWatch/listLambdaFunctions.ts

@@ -5,53 +5,81 @@ import minimatch from "minimatch";
 export interface IListLambdaFunctionsParams {
     folder: string;
     env: string;
-    variant: string | undefined;
-    function?: string | string[];
+    variant?: string;
+    whitelist?: string | string[];
+}
+
+interface ListLambdaFunctionsResult {
+    list: { name: string; path: string }[];
+    meta: {
+        count: number;
+        totalCount: number;
+    };
 }
 
 export const listLambdaFunctions = ({
     folder,
     env,
     variant,
-    function: fn
-}: IListLambdaFunctionsParams) => {
+    whitelist
+}: IListLambdaFunctionsParams): ListLambdaFunctionsResult => {
     const stackExport = getStackExport({ folder, env, variant });
     if (!stackExport) {
         // If no stack export is found, return an empty array. This is a valid scenario.
         // For example, watching the Admin app locally, but not deploying it.
-        return [];
+        return {
+            list: [],
+            meta: {
+                count: 0,
+                totalCount: 0
+            }
+        };
     }
 
-    const functionsList = stackExport.deployment.resources
+    const allFunctionsList = stackExport.deployment.resources
         .filter(r => r.type === "aws:lambda/function:Function")
-        // This filter ensures that Lambda@Edge functions are excluded.
-        .filter(lambdaFunctionResource => {
-            return "." in lambdaFunctionResource.outputs.code.assets;
+        // We don't need to watch the authorizer function.
+        .filter(resource => {
+            const isAuthorizerFunction = resource.inputs.name.includes(
+                "watch-command-iot-authorizer"
+            );
+            return !isAuthorizerFunction;
+        });
+
+    let functionsList = allFunctionsList
+        // This filter ensures that Lambda@Edge functions are excluded. It also ensures a
+        // functions is filtered out if a `pulumi refresh` was called, because, when called,
+        // the paths in Pulumi state file disappear, and we can't determine the path to the handler.
+        .filter(resource => {
+            return "." in resource.inputs.code.assets;
         })
-        .map(lambdaFunctionResource => {
-            const fnName = lambdaFunctionResource.outputs.name;
-            const handlerBuildFolderPath = lambdaFunctionResource.outputs.code.assets["."].path;
+        .map(resource => {
+            const fnName = resource.inputs.name;
+            const handlerBuildFolderPath = resource.inputs.code.assets["."].path;
             const handlerPath = path.join(handlerBuildFolderPath, "handler.js");
             return {
                 name: fnName,
                 path: handlerPath
             };
         });
 
-    if (!fn) {
-        return functionsList;
-    }
-
-    const functionNamesToMatch = Array.isArray(fn) ? fn : [fn];
+    if (whitelist) {
+        const functionNamesToMatch = Array.isArray(whitelist) ? whitelist : [whitelist];
 
-    // `functionNamesToWatch` is an array of glob patterns, which denote which functions to watch.
-    return functionsList.filter(fn => {
-        return functionNamesToMatch.some(pattern => {
-            if (pattern.includes("*")) {
-                return minimatch(fn.name, pattern);
-            }
+        // `functionNamesToWatch` is an array of glob patterns, which denote which functions to watch.
+        functionsList = functionsList.filter(fn => {
+            return functionNamesToMatch.some(pattern => {
+                if (pattern.includes("*")) {
+                    return minimatch(fn.name, pattern);
+                }
 
-            return fn.name.includes(pattern);
+                return fn.name.includes(pattern);
+            });
         });
-    });
+    }
+
+    return {
+        list: functionsList,
+        meta: { count: functionsList.length, totalCount: allFunctionsList.length }
+    };
 };