From 62f3a7bb45d6df556e95e2cfceeb631f7687f6a4 Mon Sep 17 00:00:00 2001 From: usimd <11619247+usimd@users.noreply.github.com> Date: Fri, 23 Feb 2024 03:19:48 +0100 Subject: [PATCH] Add caching of pi-gen artifacts --- .github/workflows/integration-test.yml | 1 + __test__/actions.test.ts | 39 +-------- action.yml | 3 +- package-lock.json | 115 +++++++++++++++++++------ package.json | 5 +- src/actions.ts | 38 +++----- src/cache.ts | 54 ++++-------- src/host-dependencies.ts | 2 +- src/main.ts | 4 +- src/pi-gen.ts | 3 +- src/post.ts | 4 +- src/pre.ts | 3 - 12 files changed, 131 insertions(+), 140 deletions(-) delete mode 100644 src/pre.ts diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index be2de1e..609ee89 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -99,6 +99,7 @@ jobs: pubkey-ssh-first-user: ${{ env.CONFIG_PUBLIC_KEY }} increase-runner-disk-size: ${{ github.event_name != 'workflow_dispatch' || inputs.increase-runner-disk }} apt-proxy: http://172.17.0.1:9999 + enable-pigen-cache: true - name: Move APT cache continue-on-error: true diff --git a/__test__/actions.test.ts b/__test__/actions.test.ts index d1b99db..b54db38 100644 --- a/__test__/actions.test.ts +++ b/__test__/actions.test.ts @@ -1,9 +1,8 @@ import * as core from '@actions/core' import {DEFAULT_CONFIG} from '../src/pi-gen-config' import * as actions from '../src/actions' -import {removeContainer} from '../src/remove-container' -import {build} from '../src/build' import {removeRunnerComponents} from '../src/increase-runner-disk-size' +import * as removeContainer from '../src/remove-container' jest.mock('../src/configure', () => ({ configure: jest.fn().mockReturnValue(DEFAULT_CONFIG) @@ -29,28 +28,11 @@ describe('Actions', () => { it('should only increase disk space if requested', async () => { jest.spyOn(core, 'getBooleanInput').mockReturnValueOnce(true) - await actions.piGen() + await actions.build() expect(removeRunnerComponents).toHaveBeenCalled() }) - it('does not run build function twice but invokes cleanup', async () => { - jest - .spyOn(core, 'getState') - .mockReturnValueOnce('') - .mockReturnValueOnce('true') - .mockReturnValueOnce('true') - process.env['INPUT_INCREASE-RUNNER-DISK-SIZE'] = 'false' - - // expect build here - await actions.run() - // expect cleanup here - await actions.run() - - expect(build).toHaveBeenCalledTimes(1) - expect(removeContainer).toHaveBeenCalledTimes(1) - }) - const errorMessage = 'any error' it.each([new Error(errorMessage), errorMessage])( 'should catch errors thrown during build and set build safely as failed', @@ -61,7 +43,7 @@ describe('Actions', () => { }) jest.spyOn(core, 'setFailed') - await expect(actions.piGen()).resolves.not.toThrow() + await expect(actions.build()).resolves.not.toThrow() expect(core.setFailed).toHaveBeenLastCalledWith(errorMessage) } ) @@ -69,24 +51,11 @@ describe('Actions', () => { it.each([new Error(errorMessage), errorMessage])( 'should gracefully catch errors thrown during cleanup and emit a warning message', async error => { - jest.spyOn(core, 'getState').mockImplementation(name => { - throw error - }) + jest.spyOn(removeContainer, 'removeContainer').mockRejectedValue(error) jest.spyOn(core, 'warning') await expect(actions.cleanup()).resolves.not.toThrow() expect(core.warning).toHaveBeenLastCalledWith(errorMessage) } ) - - describe('cleanup', () => { - it.each(['', 'true'])( - 'tries to remove container only if build has started = %s', - async buildStarted => { - jest.spyOn(core, 'getState').mockReturnValueOnce(buildStarted) - await actions.cleanup() - expect(removeContainer).toHaveBeenCalledTimes(buildStarted ? 1 : 0) - } - ) - }) }) diff --git a/action.yml b/action.yml index 8a22e0e..70b786c 100644 --- a/action.yml +++ b/action.yml @@ -149,6 +149,8 @@ inputs: This shall increase the available disk space so that also large images can be compiled on a free GHA runner (benchmark is the full image including a desktop environment). If any packages are missing during the build consider adding them to the `extra-host-dependencies` list. + enable-pigen-cache: + description: Enables caching of pi-gen work artifacts to GitHub action cache to speed up repetitive builds. required: false default: false pi-gen-dir: @@ -172,7 +174,6 @@ outputs: runs: using: node20 - pre: dist/pre.js main: dist/index.js post: dist/post.js diff --git a/package-lock.json b/package-lock.json index 38889c8..bfa7f96 100644 --- a/package-lock.json +++ b/package-lock.json @@ -25,6 +25,7 @@ "@types/jest": "29.5.14", "@types/js-yaml": "4.0.9", "@types/node": "20.17.5", + "@types/object-hash": "^3.0.6", "@types/semver": "7.5.8", "@types/tmp": "0.2.6", "@typescript-eslint/eslint-plugin": "7.18.0", @@ -40,6 +41,7 @@ "jest-mock-extended": "3.0.7", "js-yaml": "4.1.0", "markdown-replace-section": "0.4.0", + "move-file-cli": "3.0.0", "prettier": "3.3.3", "semver": "7.6.3", "ts-jest": "29.2.5", @@ -1923,11 +1925,48 @@ "version": "20.17.5", "resolved": "https://registry.npmjs.org/@types/node/-/node-20.17.5.tgz", "integrity": "sha512-n8FYY/pRxu496441gIcAQFZPKXbhsd6VZygcq+PTSZ75eMh/Ke0hCAROdUa21qiFqKNsPPYic46yXDO1JGiPBQ==", - "dev": true, "dependencies": { "undici-types": "~6.19.2" } }, + "node_modules/@types/node-fetch": { + "version": "2.6.11", + "resolved": "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.6.11.tgz", + "integrity": "sha512-24xFj9R5+rfQJLRyM56qh+wnVSYhyXC2tkoBndtY0U+vubqNsYXGjufB2nn8Q6gt0LrARwL6UBtMCSVCwl4B1g==", + "license": "MIT", + "dependencies": { + "@types/node": "*", + "form-data": "^4.0.0" + } + }, + "node_modules/@types/node-fetch/node_modules/form-data": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.1.tgz", + "integrity": "sha512-tzN8e4TX8+kkxGPK8D5u0FNmjPUjw3lwC9lSLxxoB/+GtsJG91CO8bSWy73APlgAZzZbXEYZJuxjkHH2w+Ezhw==", + "license": "MIT", + "dependencies": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.8", + "mime-types": "^2.1.12" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/@types/normalize-package-data": { + "version": "2.4.4", + "resolved": "https://registry.npmjs.org/@types/normalize-package-data/-/normalize-package-data-2.4.4.tgz", + "integrity": "sha512-37i+OaWTh9qeK4LSHPsyRC7NahnGotNuZvjLSgcPzblpHB3rrCJxAOgI5gCdKm7coonsaX1Of0ILiTcnZjbfxA==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/object-hash": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/@types/object-hash/-/object-hash-3.0.6.tgz", + "integrity": "sha512-fOBV8C1FIu2ELinoILQ+ApxcUKz4ngq+IWUYrxSGjXzzjUALijilampwkMgEtJ+h2njAW3pi853QpzNVCHB73w==", + "dev": true, + "license": "MIT" + }, "node_modules/@types/semver": { "version": "7.5.8", "resolved": "https://registry.npmjs.org/@types/semver/-/semver-7.5.8.tgz", @@ -4799,13 +4838,6 @@ "node": "^10.12.0 || >=12.0.0" } }, - "node_modules/file-uri-to-path": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz", - "integrity": "sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==", - "dev": true, - "optional": true - }, "node_modules/filelist": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/filelist/-/filelist-1.0.4.tgz", @@ -10699,8 +10731,7 @@ "node_modules/undici-types": { "version": "6.19.8", "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.19.8.tgz", - "integrity": "sha512-ve2KP6f/JnbPBFyobGHuerC9g1FYGn/F8n1LWTwNxCEzd6IfqTwUQcNXgEtmmQ6DlRrC1hrSrBnCZPokRrDHjw==", - "dev": true + "integrity": "sha512-ve2KP6f/JnbPBFyobGHuerC9g1FYGn/F8n1LWTwNxCEzd6IfqTwUQcNXgEtmmQ6DlRrC1hrSrBnCZPokRrDHjw==" }, "node_modules/unherit": { "version": "1.1.3", @@ -10932,6 +10963,15 @@ "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==", "dev": true }, + "node_modules/uuid": { + "version": "8.3.2", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", + "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==", + "license": "MIT", + "bin": { + "uuid": "dist/bin/uuid" + } + }, "node_modules/v8-compile-cache-lib": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/v8-compile-cache-lib/-/v8-compile-cache-lib-3.0.1.tgz", @@ -12690,12 +12730,6 @@ "integrity": "sha512-nn5ozdjYQpUCZlWGuxcJY/KpxkWQs4DcbMCmKojjyrYDEAGy4Ce19NN4v5MduafTwJlbKc99UA8YhSVqq9yPZA==", "dev": true }, - "@nolyfill/is-core-module": { - "version": "1.0.39", - "resolved": "https://registry.npmjs.org/@nolyfill/is-core-module/-/is-core-module-1.0.39.tgz", - "integrity": "sha512-nn5ozdjYQpUCZlWGuxcJY/KpxkWQs4DcbMCmKojjyrYDEAGy4Ce19NN4v5MduafTwJlbKc99UA8YhSVqq9yPZA==", - "dev": true - }, "@octokit/auth-token": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/@octokit/auth-token/-/auth-token-4.0.0.tgz", @@ -12951,11 +12985,43 @@ "version": "20.17.5", "resolved": "https://registry.npmjs.org/@types/node/-/node-20.17.5.tgz", "integrity": "sha512-n8FYY/pRxu496441gIcAQFZPKXbhsd6VZygcq+PTSZ75eMh/Ke0hCAROdUa21qiFqKNsPPYic46yXDO1JGiPBQ==", - "dev": true, "requires": { "undici-types": "~6.19.2" } }, + "@types/node-fetch": { + "version": "2.6.11", + "resolved": "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.6.11.tgz", + "integrity": "sha512-24xFj9R5+rfQJLRyM56qh+wnVSYhyXC2tkoBndtY0U+vubqNsYXGjufB2nn8Q6gt0LrARwL6UBtMCSVCwl4B1g==", + "requires": { + "@types/node": "*", + "form-data": "^4.0.0" + }, + "dependencies": { + "form-data": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.1.tgz", + "integrity": "sha512-tzN8e4TX8+kkxGPK8D5u0FNmjPUjw3lwC9lSLxxoB/+GtsJG91CO8bSWy73APlgAZzZbXEYZJuxjkHH2w+Ezhw==", + "requires": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.8", + "mime-types": "^2.1.12" + } + } + } + }, + "@types/normalize-package-data": { + "version": "2.4.4", + "resolved": "https://registry.npmjs.org/@types/normalize-package-data/-/normalize-package-data-2.4.4.tgz", + "integrity": "sha512-37i+OaWTh9qeK4LSHPsyRC7NahnGotNuZvjLSgcPzblpHB3rrCJxAOgI5gCdKm7coonsaX1Of0ILiTcnZjbfxA==", + "dev": true + }, + "@types/object-hash": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/@types/object-hash/-/object-hash-3.0.6.tgz", + "integrity": "sha512-fOBV8C1FIu2ELinoILQ+ApxcUKz4ngq+IWUYrxSGjXzzjUALijilampwkMgEtJ+h2njAW3pi853QpzNVCHB73w==", + "dev": true + }, "@types/semver": { "version": "7.5.8", "resolved": "https://registry.npmjs.org/@types/semver/-/semver-7.5.8.tgz", @@ -15060,13 +15126,6 @@ "flat-cache": "^3.0.4" } }, - "file-uri-to-path": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz", - "integrity": "sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==", - "dev": true, - "optional": true - }, "filelist": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/filelist/-/filelist-1.0.4.tgz", @@ -19499,8 +19558,7 @@ "undici-types": { "version": "6.19.8", "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.19.8.tgz", - "integrity": "sha512-ve2KP6f/JnbPBFyobGHuerC9g1FYGn/F8n1LWTwNxCEzd6IfqTwUQcNXgEtmmQ6DlRrC1hrSrBnCZPokRrDHjw==", - "dev": true + "integrity": "sha512-ve2KP6f/JnbPBFyobGHuerC9g1FYGn/F8n1LWTwNxCEzd6IfqTwUQcNXgEtmmQ6DlRrC1hrSrBnCZPokRrDHjw==" }, "unherit": { "version": "1.1.3", @@ -19677,6 +19735,11 @@ "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==", "dev": true }, + "uuid": { + "version": "8.3.2", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", + "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==" + }, "v8-compile-cache-lib": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/v8-compile-cache-lib/-/v8-compile-cache-lib-3.0.1.tgz", diff --git a/package.json b/package.json index b30333c..125c690 100644 --- a/package.json +++ b/package.json @@ -7,9 +7,8 @@ "scripts": { "build": "tsc", "lint": "eslint src/**/*.ts", - "package": "npm run package-pre && npm run package-main && npm run package-post", + "package": "npm run package-main && npm run package-post", "package-main": "ncc build src/main.ts -m --no-source-map-register --license licenses.txt", - "package-pre": "ncc build src/pre.ts -m --no-source-map-register --out dist/pre && move-file dist/pre/index.js dist/pre.js", "package-post": "ncc build src/post.ts -m --no-source-map-register --out dist/post && move-file dist/post/index.js dist/post.js", "test": "jest", "all": "npm run build && npm run format && npm run lint && npm test && npm run package && npm run update-readme", @@ -48,6 +47,7 @@ "@types/jest": "29.5.14", "@types/js-yaml": "4.0.9", "@types/node": "20.17.5", + "@types/object-hash": "^3.0.6", "@types/semver": "7.5.8", "@types/tmp": "0.2.6", "@typescript-eslint/eslint-plugin": "7.18.0", @@ -63,6 +63,7 @@ "jest-mock-extended": "3.0.7", "js-yaml": "4.1.0", "markdown-replace-section": "0.4.0", + "move-file-cli": "3.0.0", "prettier": "3.3.3", "semver": "7.6.3", "ts-jest": "29.2.5", diff --git a/src/actions.ts b/src/actions.ts index eb89cf4..bc2dac9 100644 --- a/src/actions.ts +++ b/src/actions.ts @@ -1,16 +1,13 @@ import * as core from '@actions/core' -import {Cache} from './cache' import {configure} from './configure' import {installHostDependencies} from './install-dependencies' -import {build} from './build' +import {build as piGenBuild} from './build' import {clonePigen} from './clone-pigen' import {removeContainer} from './remove-container' import {removeRunnerComponents} from './increase-runner-disk-size' +import {Cache} from './cache' -const piGenBuildStartedState = 'pi-gen-build-started' -const cache = new Cache() - -export async function piGen(): Promise { +export async function build(): Promise { try { // Need to force color output for chalk, until https://github.com/actions/runner/issues/241 is resolved. // See also https://github.com/chalk/supports-color/issues/106 @@ -40,9 +37,7 @@ export async function piGen(): Promise { core.getInput('extra-host-modules'), piGenDirectory ) - - core.saveState(piGenBuildStartedState, true) - await build(piGenDirectory, userConfig) + await piGenBuild(piGenDirectory, userConfig) } catch (error) { core.setFailed((error as Error)?.message ?? error) } @@ -50,29 +45,18 @@ export async function piGen(): Promise { export async function cleanup(): Promise { try { - if (core.getState(piGenBuildStartedState)) { - await removeContainer('pigen_work') - } else { - core.info('No build started, nothing to clean') - } + await removeContainer('pigen_work') } catch (error) { core.warning((error as Error)?.message ?? error) } } -export async function run(): Promise { - if (core.getState('main-executed')) { - await cleanup() - } else { - core.saveState('main-executed', true) - await piGen() - } -} - -export async function restoreCache(): Promise { - await cache.restoreCache() -} +export async function restoreCache(): Promise {} export async function saveCache(): Promise { - await cache.saveCache() + if (core.getBooleanInput('enable-pigen-cache')) { + core.startGroup('Cache pi-gen container') + const cache = new Cache() + cache.cacheContainer() + } } diff --git a/src/cache.ts b/src/cache.ts index 43e8ac2..6eab837 100644 --- a/src/cache.ts +++ b/src/cache.ts @@ -1,47 +1,23 @@ -import * as core from '@actions/core' import * as cache from '@actions/cache' import * as exec from '@actions/exec' -import objectHash from 'object-hash' +import * as io from '@actions/io' export class Cache { - private cacheKeyPrefix = 'pigen-work-' - private cacheKey: string - private restoreKeys: string[] = [this.cacheKeyPrefix] + public async cacheContainer(containerName = 'pigen_work') { + const docker = await io.which('docker', true) + const lrzip = await io.which('lrzip', true) + const containerExportResult = await exec.getExecOutput( + docker, + ['container', 'export', '--output=pi-gen.tar', containerName], + {silent: true, ignoreReturnCode: true} + ) - constructor() { - core.debug('Adapting permissions of tar') - exec.exec('sudo', ['chown', 'root', '/usr/bin/tar'], {ignoreReturnCode: true, silent: true}) - exec.exec('sudo', ['chown', 'root', '/bin/tar'], {ignoreReturnCode: true, silent: true}) - exec.exec('sudo', ['chmod', 'u+s', '/usr/bin/tar'], {ignoreReturnCode: true, silent: true}) - exec.exec('sudo', ['chmod', 'u+s', '/usr/bin/tar'], {ignoreReturnCode: true, silent: true}) + const lrzipResult = await exec.getExecOutput( + lrzip, + ['--level=9', 'pi-gen.tar'], + {silent: true, ignoreReturnCode: true} + ) - const inputVars = Object.keys(process.env) - .filter(envName => envName.startsWith('INPUT_')) - .reduce((obj: {[key: string]: string}, envName: string) => { - obj[envName] = process.env[envName]! - return obj - }, {}) - const inputHash = objectHash.sha1(inputVars) - this.cacheKey = `${this.cacheKeyPrefix}${inputHash}` - - core.debug(`Computed cache key: ${this.cacheKey}`) - } - - getCacheKey() { - return this.cacheKey - } - - async restoreCache(): Promise { - if (core.getBooleanInput('enable-pigen-cache')) { - core.info('Restoring pi-gen cache if exists') - await cache.restoreCache(['pigen-work'], this.cacheKey, this.restoreKeys) - } - } - - async saveCache(): Promise { - if (core.getBooleanInput('enable-pigen-cache')) { - core.info('Saving pi-gen work directory to GitHub cache') - await cache.saveCache(['pigen-work'], this.cacheKey) - } + cache.saveCache(['pi-gen.tar.lrz'], 'test') } } diff --git a/src/host-dependencies.ts b/src/host-dependencies.ts index 8c6bcd5..a340ed4 100644 --- a/src/host-dependencies.ts +++ b/src/host-dependencies.ts @@ -1,4 +1,4 @@ export const hostDependencies = { - packages: ['binfmt-support'], + packages: ['binfmt-support', 'lrzip'], modules: ['binfmt_misc'] } diff --git a/src/main.ts b/src/main.ts index e61434a..44717bb 100644 --- a/src/main.ts +++ b/src/main.ts @@ -1,3 +1,3 @@ -import {run} from './actions' +import {build} from './actions' -run() +build() diff --git a/src/pi-gen.ts b/src/pi-gen.ts index ab7597e..79365d8 100644 --- a/src/pi-gen.ts +++ b/src/pi-gen.ts @@ -53,10 +53,9 @@ export class PiGen { } async build(verbose = false): Promise { - fs.mkdirSync('./pigen-work') // By default, we'll pass all user stages as mounts to the Docker run and we'll configure // apt to not report progress (which can become excessive). - let dockerOpts = `${this.getStagesAsDockerMounts()} -e DEBIAN_FRONTEND=noninteractive -v ${fs.realpathSync('./pigen-work')}:/pi-gen/work` + let dockerOpts = `${this.getStagesAsDockerMounts()} -e DEBIAN_FRONTEND=noninteractive` if (this.config.dockerOpts !== undefined && this.config.dockerOpts !== '') { dockerOpts = `${this.config.dockerOpts} ${dockerOpts}` diff --git a/src/post.ts b/src/post.ts index c3cfcdf..c35c2c9 100644 --- a/src/post.ts +++ b/src/post.ts @@ -1,3 +1,3 @@ -import {saveCache} from './actions' +import {cleanup, saveCache} from './actions' -saveCache() +saveCache().finally(() => cleanup()) diff --git a/src/pre.ts b/src/pre.ts deleted file mode 100644 index 3c9e8cc..0000000 --- a/src/pre.ts +++ /dev/null @@ -1,3 +0,0 @@ -import {restoreCache} from './actions' - -restoreCache()