feat: build packages in parallel threads to use all CPUs

.eslintignore

@@ -5,7 +5,4 @@
 **/.out/
 **/*.d.ts
 idea.js
-scripts
-packages/ui/src/RichTextEditor/editorjs/**
-packages-v6/pb-editor/**
-packages-v6/core/**/artifacts/**
+scripts/**/*.js

package.json

@@ -16,7 +16,8 @@
       "apps/api/migration",
       "apps/api/pageBuilder/updateSettings",
       "apps/api/pageBuilder/import/*",
-      "apps/api/pageBuilder/export/*"
+      "apps/api/pageBuilder/export/*",
+      "scripts/buildPackages"
     ]
   },
   "author": "Webiny Ltd.",
@@ -121,8 +122,8 @@
     "check-ts-configs": "node scripts/checkTsConfigs.js",
     "eslint": "eslint \"**/*.{js,jsx,ts,tsx}\" --max-warnings=0",
     "eslint:fix": "yarn eslint --fix",
-    "build": "node scripts/buildWithCache.js",
-    "build:quick": "node scripts/buildWithCache.js --build-overrides='{\"tsConfig\":{\"compilerOptions\":{\"skipLibCheck\":true}}}'",
+    "build": "node scripts/buildPackages",
+    "build:quick": "node scripts/buildPackages --build-overrides='{\"tsConfig\":{\"compilerOptions\":{\"skipLibCheck\":true}}}'",
     "build:apps": "yarn webiny ws run build --scope='@webiny/app*'",
     "build:api": "yarn webiny ws run build --scope='@webiny/api*' --scope='@webiny/handler*'",
     "watch:apps": "yarn webiny ws run watch --scope='@webiny/app*'",

packages/api-page-builder-aco/package.json

@@ -36,6 +36,7 @@
     "@webiny/api-security-so-ddb": "0.0.0",
     "@webiny/api-tenancy": "0.0.0",
     "@webiny/api-tenancy-so-ddb": "0.0.0",
+    "@webiny/cli": "0.0.0",
     "@webiny/handler-aws": "0.0.0",
     "@webiny/handler-graphql": "0.0.0",
     "@webiny/plugins": "0.0.0",

packages/project-utils/packages/buildPackage.js

@@ -15,6 +15,12 @@ module.exports = async options => {
     rimraf.sync(join(cwd, "*.tsbuildinfo"));
 
     options.logs !== false && console.log("Building...");
+
+    // Make sure `overrides` is an object.
+    if (options.overrides && typeof options.overrides === "string") {
+        options.overrides = JSON.parse(options.overrides);
+    }
+
     await Promise.all([tsCompile(options), babelCompile(options)]);
 
     options.logs !== false && console.log("Copying meta files...");

scripts/buildPackages/package.json

@@ -0,0 +1,23 @@
+{
+  "private": true,
+  "name": "@webiny-scripts/build-packages",
+  "version": "0.0.0",
+  "sideEffects": false,
+  "bin": "./src/index.js",
+  "main": "./src/index.js",
+  "dependencies": {
+    "chalk": "^4.1.0",
+    "execa": "^5.1.1",
+    "folder-hash": "^4.0.4",
+    "fs-extra": "^7.0.1",
+    "listr2": "^5.0.8",
+    "load-json-file": "^6.2.0",
+    "write-json-file": "^4.3.0",
+    "yargs": "^17.3.1"
+  },
+  "devDependencies": {
+    "@types/folder-hash": "^4.0.2",
+    "@types/yargs": "^17.0.8",
+    "ts-node": "^10.5.0"
+  }
+}

scripts/buildPackages/src/buildPackages.ts

@@ -0,0 +1,102 @@
+import { green } from "chalk";
+import yargs from "yargs";
+import writeJson from "write-json-file";
+import { Listr, ListrTask } from "listr2";
+import { getBatches } from "./getBatches";
+import { META_FILE_PATH } from "./constants";
+import { getPackageSourceHash } from "./getPackageSourceHash";
+import { getBuildMeta } from "./getBuildMeta";
+import { buildPackageInNewProcess } from "./buildSinglePackage";
+import { MetaJSON, Package } from "./types";
+
+interface BuildOptions {
+    debug?: boolean;
+    cache?: boolean;
+    buildOverrides?: string;
+}
+
+interface BuildContext {
+    [key: string]: boolean;
+}
+
+export const buildPackages = async () => {
+    const options = yargs.argv as BuildOptions;
+
+    const { batches, packagesNoCache, allPackages } = await getBatches({
+        cache: options.cache ?? true
+    });
+
+    if (!packagesNoCache.length) {
+        console.log("There are no packages that need to be built.");
+        return;
+    }
+
+    if (packagesNoCache.length > 10) {
+        console.log(`\nRunning build for ${green(packagesNoCache.length)} packages.`);
+    } else {
+        console.log("\nRunning build for the following packages:");
+        for (let i = 0; i < packagesNoCache.length; i++) {
+            const item = packagesNoCache[i];
+            console.log(`‣ ${green(item.packageJson.name)}`);
+        }
+    }
+
+    console.log(
+        `\nThe build process will be performed in ${green(batches.length)} ${
+            batches.length > 1 ? "batches" : "batch"
+        }.\n`
+    );
+    const metaJson = getBuildMeta();
+
+    const tasks = new Listr<BuildContext>(
+        batches.map<ListrTask>((packageNames, index) => {
+            const id = `${index + 1}`.padStart(2, "0");
+            const title = `[${id}/${batches.length}] Batch #${id} (${packageNames.length} packages)`;
+
+            return {
+                title,
+                task: (ctx, task): Listr => {
+                    const packages = allPackages.filter(pkg => packageNames.includes(pkg.name));
+
+                    const batchTasks = task.newListr([], {
+                        concurrent: true,
+                        exitOnError: true
+                    });
+
+                    packages.forEach(pkg => {
+                        batchTasks.add(createPackageTask(pkg, options, metaJson));
+                    });
+
+                    return batchTasks;
+                }
+            };
+        }),
+        { concurrent: false, rendererOptions: { showTimer: true, collapse: true } }
+    );
+
+    const start = Date.now();
+    const ctx = {};
+    await tasks.run(ctx);
+    const duration = (Date.now() - start) / 1000;
+
+    console.log(`\nBuild finished in ${green(duration)} seconds.`);
+};
+
+const createPackageTask = (pkg: Package, options: BuildOptions, metaJson: MetaJSON) => {
+    return {
+        title: `${pkg.name}`,
+        task: async () => {
+            try {
+                await buildPackageInNewProcess(pkg, options.buildOverrides);
+
+                // Store package hash
+                const sourceHash = await getPackageSourceHash(pkg);
+                metaJson.packages[pkg.packageJson.name] = { sourceHash };
+
+                await writeJson(META_FILE_PATH, metaJson);
+            } catch (err) {
+                throw new Error(`[${pkg.packageJson.name}] ${err.message}`);
+            }
+        }
+    };
+};

scripts/buildPackages/src/buildSinglePackage.ts

@@ -0,0 +1,18 @@
+import path from "path";
+import fs from "fs-extra";
+import execa from "execa";
+import { Package } from "./types";
+import { getBuildOutputFolder } from "./getBuildOutputFolder";
+import { CACHE_FOLDER_PATH } from "./constants";
+
+export const buildPackageInNewProcess = async (pkg: Package, buildOverrides = "{}") => {
+    // Run build script using execa
+    await execa("yarn", ["build", "--overrides", buildOverrides], {
+        cwd: pkg.packageFolder
+    });
+
+    const cacheFolderPath = path.join(CACHE_FOLDER_PATH, pkg.packageJson.name);
+
+    const buildFolder = getBuildOutputFolder(pkg);
+    fs.copySync(buildFolder, cacheFolderPath);
+};

scripts/buildPackages/src/constants.ts

@@ -0,0 +1,4 @@
+import path from "path";
+
+export const CACHE_FOLDER_PATH = ".webiny/cached-packages";
+export const META_FILE_PATH = path.join(CACHE_FOLDER_PATH, "meta.json");

scripts/buildPackages/src/getBatches.ts

@@ -0,0 +1,135 @@
+import fs from "fs-extra";
+import execa from "execa";
+import path from "path";
+import { green } from "chalk";
+import { getPackages } from "../../utils/getPackages";
+import { Package } from "./types";
+import { CACHE_FOLDER_PATH } from "./constants";
+import { getBuildOutputFolder } from "./getBuildOutputFolder";
+import { getPackageSourceHash } from "./getPackageSourceHash";
+import { getBuildMeta } from "./getBuildMeta";
+import { getPackageCacheFolderPath } from "./getPackageCacheFolderPath";
+
+interface GetBatchesOptions {
+    cache?: boolean;
+}
+
+export async function getBatches(options: GetBatchesOptions = {}) {
+    const metaJson = getBuildMeta();
+
+    const packagesNoCache: Package[] = [];
+    const packagesUseCache: Package[] = [];
+
+    const workspacesPackages = (
+        getPackages({
+            includes: ["/packages/", "/packages-v6/"]
+        }) as Package[]
+    )
+        .filter(item => item.isTs)
+        .filter(pkg => {
+            // Check if packages has a build script
+            return pkg.packageJson.scripts && "build" in pkg.packageJson.scripts;
+        });
+
+    console.log(`There is a total of ${green(workspacesPackages.length)} packages.`);
+    const useCache = options.cache ?? false;
+
+    // 1. Determine for which packages we can use the cached built code, and for which we need to execute build.
+    if (!useCache) {
+        workspacesPackages.forEach(pkg => packagesNoCache.push(pkg));
+    } else {
+        for (const workspacePackage of workspacesPackages) {
+            const cacheFolderPath = getPackageCacheFolderPath(workspacePackage);
+            if (!fs.existsSync(cacheFolderPath)) {
+                packagesNoCache.push(workspacePackage);
+                continue;
+            }
+
+            const sourceHash = await getPackageSourceHash(workspacePackage);
+
+            const packageMeta = metaJson.packages[workspacePackage.packageJson.name] || {};
+
+            if (packageMeta.sourceHash === sourceHash) {
+                packagesUseCache.push(workspacePackage);
+            } else {
+                packagesNoCache.push(workspacePackage);
+            }
+        }
+    }
+
+    // 2. Let's use cached built code where possible.
+    if (packagesUseCache.length) {
+        if (packagesUseCache.length > 10) {
+            console.log(`Using cache for ${green(packagesUseCache.length)} packages.`);
+            console.log(
+                `To build all packages regardless of cache, use the ${green("--no-cache")} flag.`
+            );
+        } else {
+            console.log("Using cache for following packages:");
+            for (let i = 0; i < packagesUseCache.length; i++) {
+                const item = packagesUseCache[i];
+                console.log(green(item.packageJson.name));
+            }
+        }
+
+        for (let i = 0; i < packagesUseCache.length; i++) {
+            const workspacePackage = packagesUseCache[i];
+            const cacheFolderPath = path.join(CACHE_FOLDER_PATH, workspacePackage.packageJson.name);
+            fs.copySync(cacheFolderPath, getBuildOutputFolder(workspacePackage));
+        }
+    } else {
+        if (useCache) {
+            console.log("Cache is empty, all packages need to be built.");
+        } else {
+            console.log("Skipping cache.");
+        }
+    }
+
+    // 3. Where needed, let's build and update the cache.
+    if (packagesNoCache.length === 0) {
+        return { batches: [], packagesNoCache, allPackages: workspacesPackages };
+    }
+
+    // Building all packages - we're respecting the dependency graph.
+    // Note: lists only packages in "packages" folder (check `lerna.json` config).
+    const rawPackagesList: Record<string, string[]> = await execa("lerna", [
+        "list",
+        "--toposort",
+        "--graph",
+        "--all"
+    ]).then(({ stdout }) => JSON.parse(stdout));
+
+    const packagesList: Record<string, string[]> = {};
+
+    for (const packageName in rawPackagesList) {
+        // If in cache, skip.
+        if (packagesUseCache.find(item => item.name === packageName)) {
+            continue;
+        }
+
+        // If not a TS package, skip.
+        if (!workspacesPackages.find(item => item.name === packageName)) {
+            continue;
+        }
+
+        packagesList[packageName] = rawPackagesList[packageName];
+    }
+
+    const batches: string[][] = [[]];
+    for (const packageName in packagesList) {
+        const dependencies = packagesList[packageName];
+        const latestBatch = batches[batches.length - 1];
+        const canEnterCurrentBatch = !dependencies.find(name => latestBatch.includes(name));
+        if (canEnterCurrentBatch) {
+            latestBatch.push(packageName);
+        } else {
+            batches.push([packageName]);
+        }
+    }
+
+    return {
+        batches,
+        packagesNoCache,
+        allPackages: workspacesPackages
+    };
+}

scripts/buildPackages/src/getBuildMeta.ts

@@ -0,0 +1,14 @@
+import loadJson from "load-json-file";
+import { MetaJSON } from "./types";
+import { META_FILE_PATH } from "./constants";
+
+export function getBuildMeta() {
+    let metaJson: MetaJSON = { packages: {} };
+    try {
+        metaJson = loadJson.sync(META_FILE_PATH);
+    } catch {
+        // An error means there's no meta file, so we start a fresh build.
+    }
+
+    return metaJson;
+}

scripts/buildPackages/src/getBuildOutputFolder.ts

@@ -0,0 +1,20 @@
+import path from "path";
+import { PackageJSON } from "./types";
+
+export function getBuildOutputFolder({
+    packageJson,
+    packageFolder
+}: {
+    packageJson: PackageJSON;
+    packageFolder: string;
+}) {
+    const webinyConfig = packageJson.webiny;
+    // `dist` is the default output folder for v5 packages.
+    let buildFolder = "dist";
+    if (webinyConfig) {
+        // Until the need arises, let's just use the default `lib` folder.
+        buildFolder = "lib";
+    }
+
+    return path.join(packageFolder, buildFolder);
+}

scripts/buildPackages/src/getPackageCacheFolderPath.ts

@@ -0,0 +1,7 @@
+import path from "path";
+import { Package } from "./types";
+import { CACHE_FOLDER_PATH } from "./constants";
+
+export function getPackageCacheFolderPath(workspacePackage: Package) {
+    return path.join(CACHE_FOLDER_PATH, workspacePackage.packageJson.name);
+}

scripts/buildPackages/src/getPackageSourceHash.ts

@@ -0,0 +1,11 @@
+import { hashElement } from "folder-hash";
+import { Package } from "./types";
+
+export async function getPackageSourceHash(workspacePackage: Package) {
+    const { hash } = await hashElement(workspacePackage.packageFolder, {
+        folders: { exclude: ["dist", "lib"] },
+        files: { exclude: ["tsconfig.build.tsbuildinfo"] }
+    });
+
+    return hash;
+}