diff --git a/__test__/actions.test.ts b/__test__/actions.test.ts index d1b99db..f533773 100644 --- a/__test__/actions.test.ts +++ b/__test__/actions.test.ts @@ -4,11 +4,14 @@ import * as actions from '../src/actions' import {removeContainer} from '../src/remove-container' import {build} from '../src/build' import {removeRunnerComponents} from '../src/increase-runner-disk-size' +import {installHostDependencies} from '../src/install-dependencies' jest.mock('../src/configure', () => ({ configure: jest.fn().mockReturnValue(DEFAULT_CONFIG) })) -jest.mock('../src/install-dependencies') +jest.mock('../src/install-dependencies', () => ({ + installHostDependencies: jest.fn().mockReturnValue(Promise.resolve()) +})) jest.mock('../src/build') jest.mock('../src/clone-pigen') jest.mock('../src/remove-container') @@ -27,7 +30,7 @@ describe('Actions', () => { }) it('should only increase disk space if requested', async () => { - jest.spyOn(core, 'getBooleanInput').mockReturnValueOnce(true) + jest.spyOn(core, 'getBooleanInput').mockReturnValue(true) await actions.piGen() @@ -40,7 +43,7 @@ describe('Actions', () => { .mockReturnValueOnce('') .mockReturnValueOnce('true') .mockReturnValueOnce('true') - process.env['INPUT_INCREASE-RUNNER-DISK-SIZE'] = 'false' + jest.spyOn(core, 'getBooleanInput').mockReturnValue(false) // expect build here await actions.run() @@ -56,7 +59,8 @@ describe('Actions', () => { 'should catch errors thrown during build and set build safely as failed', async error => { const errorMessage = 'any error' - jest.spyOn(core, 'getInput').mockImplementation((name, options) => { + jest.spyOn(core, 'getBooleanInput').mockReturnValue(false) + jest.spyOn(core, 'getInput').mockImplementation((name, falseptions) => { throw error }) jest.spyOn(core, 'setFailed') diff --git a/__test__/increase-runner-disk-size.test.ts b/__test__/increase-runner-disk-size.test.ts index 852c459..d637ca9 100644 --- a/__test__/increase-runner-disk-size.test.ts +++ b/__test__/increase-runner-disk-size.test.ts @@ -30,13 +30,11 @@ describe('Increasing runner disk size', () => { expect(exec.getExecOutput).toHaveBeenCalledWith( 'sudo', - expect.arrayContaining(['apt-get', 'autoremove']), - expect.anything() - ) - - expect(exec.getExecOutput).toHaveBeenCalledWith( - 'sudo', - expect.arrayContaining(['apt-get', 'autoclean']), + expect.arrayContaining([ + 'sh', + '-c', + 'apt-get autoremove && apt-get autoclean' + ]), expect.anything() ) diff --git a/package.json b/package.json index 8bc9fa5..8c04b20 100644 --- a/package.json +++ b/package.json @@ -5,11 +5,10 @@ "author": "Simon Domke", "main": "dist/index.js", "scripts": { - "build": "tsc", "lint": "eslint src/**/*.ts", "package": "ncc build src/main.ts -m --no-source-map-register --license licenses.txt", "test": "jest", - "all": "npm run build && npm run format && npm run lint && npm test && npm run package && npm run update-readme", + "all": "npm run format && npm run lint && npm test && npm run package && npm run update-readme", "format": "prettier --write '**/*.ts'", "format-check": "prettier --check '**/*.ts'", "update-readme": "ts-node src/misc/update-readme.ts" diff --git a/src/actions.ts b/src/actions.ts index 6df5496..fc09999 100644 --- a/src/actions.ts +++ b/src/actions.ts @@ -5,6 +5,7 @@ import {build} from './build' import {clonePigen} from './clone-pigen' import {removeContainer} from './remove-container' import {removeRunnerComponents} from './increase-runner-disk-size' +import {printLogGroup} from './log' const piGenBuildStartedState = 'pi-gen-build-started' @@ -14,6 +15,8 @@ export async function piGen(): Promise { // See also https://github.com/chalk/supports-color/issues/106 process.env['FORCE_COLOR'] = '2' + const verbose = core.getBooleanInput('verbose-output') + const piGenDirectory = core.getInput('pi-gen-dir') core.debug(`Using pi-gen directory: ${piGenDirectory}`) @@ -27,17 +30,31 @@ export async function piGen(): Promise { const userConfig = await configure() - if (increaseRunnerDiskSize) { - core.info('Removing unused runner components to increase disk space') - await removeRunnerComponents() - } - await clonePigen(piGenRepo, piGenDirectory, core.getInput('pi-gen-version')) - await installHostDependencies( - core.getInput('extra-host-dependencies'), - core.getInput('extra-host-modules'), - piGenDirectory - ) + + const prepareActions = [ + installHostDependencies( + core.getInput('extra-host-dependencies'), + core.getInput('extra-host-modules'), + piGenDirectory + ).then(async logOutput => { + await core.group('Installing build dependencies on host', async () => { + printLogGroup(logOutput, verbose) + }) + if (increaseRunnerDiskSize) { + return removeRunnerComponents().then(async logOutput => { + await core.group( + 'Removing runner components to increase disk build space', + async () => { + printLogGroup(logOutput, verbose) + } + ) + }) + } + }) + ] + + await Promise.all(prepareActions) core.saveState(piGenBuildStartedState, true) await build(piGenDirectory, userConfig) diff --git a/src/increase-runner-disk-size.ts b/src/increase-runner-disk-size.ts index 59db128..d65204e 100644 --- a/src/increase-runner-disk-size.ts +++ b/src/increase-runner-disk-size.ts @@ -1,117 +1,130 @@ import * as exec from '@actions/exec' -import * as core from '@actions/core' +import {LogOutput} from './log' -export async function removeRunnerComponents(): Promise { - try { - core.startGroup('Removing runner components to increase disk build space') +// See https://github.com/actions/runner-images/issues/2840#issuecomment-2272410832 +const HOST_PATHS_TO_REMOVE = [ + '/opt/google/chrome', + '/opt/microsoft/msedge', + '/opt/microsoft/powershell', + '/opt/mssql-tools', + '/opt/hostedtoolcache', + '/opt/pipx', + '/usr/lib/mono', + '/usr/local/julia*', + '/usr/local/lib/android', + '/usr/local/lib/node_modules', + '/usr/local/share/chromium', + '/usr/local/share/powershell', + '/usr/share/dotnet', + '/usr/share/swift', + '/var/cache/snapd', + '/var/lib/snapd', + '/tmp/*', + '/usr/share/doc' +] - const availableDiskSizeBeforeCleanup = await getAvailableDiskSize() - core.debug( - `Available disk space before cleanup: ${availableDiskSizeBeforeCleanup / 1024 / 1024}G` - ) +export async function removeRunnerComponents(): Promise { + const availableDiskSizeBeforeCleanup = await getAvailableDiskSize() - await exec - .getExecOutput( - 'sudo', - ['docker', 'system', 'prune', '--all', '--force'], - { - silent: true, - failOnStdErr: false, - ignoreReturnCode: true - } - ) - .then((returnValue: exec.ExecOutput) => core.debug(returnValue.stdout)) + // Running processes in parallel will mangle their logs. This will keep output in a common + // list with a prefix indicating from which process they stemmed (similar to what one gets + // from docker compose logs). + const log = {log: [], debug: []} as LogOutput - await exec - .getExecOutput( - 'sudo', - [ - 'sh', - '-c', - 'snap list | sed 1d | cut -d" " -f1 | xargs -I{} snap remove {}' - ], - { - silent: true, - failOnStdErr: false, - ignoreReturnCode: true + const actions = [] + + actions.push( + exec.getExecOutput( + 'sudo', + ['docker', 'system', 'prune', '--all', '--force'], + { + silent: true, + listeners: { + errline: line => log.log.push(`docker-system-prune: ${line}`), + stdline: line => log.log.push(`docker-system-prune: ${line}`) } - ) - .then((returnValue: exec.ExecOutput) => core.debug(returnValue.stdout)) + } + ) + ) - await exec + actions.push( + exec .getExecOutput('sudo', ['swapoff', '-a'], { silent: true, - failOnStdErr: false, - ignoreReturnCode: true + listeners: { + errline: line => log.log.push(`swapoff: ${line}`), + stdline: line => log.log.push(`swapoff: ${line}`) + } }) - .then((returnValue: exec.ExecOutput) => core.debug(returnValue.stdout)) - - // See https://github.com/actions/runner-images/issues/2840#issuecomment-2272410832 - const hostPathsToRemove = [ - '/opt/google/chrome', - '/opt/microsoft/msedge', - '/opt/microsoft/powershell', - '/opt/mssql-tools', - '/opt/hostedtoolcache', - '/opt/pipx', - '/usr/lib/mono', - '/usr/local/julia*', - '/usr/local/lib/android', - '/usr/local/lib/node_modules', - '/usr/local/share/chromium', - '/usr/local/share/powershell', - '/usr/share/dotnet', - '/usr/share/swift', - '/mnt/swapfile', - '/swapfile', - '/var/cache/snapd', - '/var/lib/snapd', - '/tmp/*', - '/usr/share/doc' - ] - - await exec - .getExecOutput('sudo', ['rm', '-rf', ...hostPathsToRemove], { - silent: true, - ignoreReturnCode: true, - failOnStdErr: false + .then(result => { + return exec.getExecOutput( + 'sudo', + ['rm', '-rf', '/mnt/swapfile', '/swapfile'], + { + silent: true, + listeners: { + errline: line => log.log.push(`rm-swapfile: ${line}`), + stdline: line => log.log.push(`rm-swapfile: ${line}`) + } + } + ) }) - .then((returnValue: exec.ExecOutput) => core.debug(returnValue.stdout)) + ) - await exec - .getExecOutput( - 'sudo', - ['apt', 'purge', 'snapd', 'php8*', 'r-base', 'imagemagick'], - { - silent: true, - ignoreReturnCode: true - } - ) - .then((returnValue: exec.ExecOutput) => core.debug(returnValue.stdout)) - await exec - .getExecOutput('sudo', ['apt-get', 'autoremove'], { + actions.push( + exec + .getExecOutput('sudo', ['rm', '-rf', ...HOST_PATHS_TO_REMOVE], { silent: true, - ignoreReturnCode: true + listeners: { + errline: line => log.log.push(`rm-host-paths: ${line}`), + stdline: line => log.log.push(`rm-host-paths: ${line}`) + } }) - .then((returnValue: exec.ExecOutput) => core.debug(returnValue.stdout)) - await exec - .getExecOutput('sudo', ['apt-get', 'autoclean'], { - silent: true, - ignoreReturnCode: true + .then((returnValue: exec.ExecOutput) => { + return exec + .getExecOutput( + 'sudo', + ['apt', 'purge', 'snapd', 'php8*', 'r-base', 'imagemagick'], + { + silent: true, + listeners: { + stdline: line => log.log.push(`apt-purge-packages: ${line}`), + errline: line => log.log.push(`apt-purge-packages: ${line}`) + } + } + ) + .then((returnValue: exec.ExecOutput) => { + return exec.getExecOutput( + 'sudo', + ['sh', '-c', 'apt-get autoremove && apt-get autoclean'], + { + silent: true, + listeners: { + stdline: line => + log.log.push(`apt-autoremove-autoclean: ${line}`), + errline: line => + log.log.push(`apt-autoremove-autoclean: ${line}`) + } + } + ) + }) }) - .then((returnValue: exec.ExecOutput) => core.debug(returnValue.stdout)) + ) + return Promise.all(actions).then(async outputs => { + log.debug.push( + `Available disk space before cleanup: ${availableDiskSizeBeforeCleanup / 1024 / 1024}G` + ) const availableDiskSizeAfterCleanup = await getAvailableDiskSize() - core.debug( + log.debug.push( `Available disk space after cleanup: ${availableDiskSizeAfterCleanup / 1024 / 1024}G` ) - - core.info( + log.log.push( `Reclaimed runner disk space: ${((availableDiskSizeAfterCleanup - availableDiskSizeBeforeCleanup) / 1024 / 1024).toFixed(2)}G` ) - } finally { - core.endGroup() - } + + return log + }) } async function getAvailableDiskSize(): Promise { diff --git a/src/install-dependencies.ts b/src/install-dependencies.ts index 91cd104..3bee685 100644 --- a/src/install-dependencies.ts +++ b/src/install-dependencies.ts @@ -1,100 +1,110 @@ import * as fs from 'fs/promises' -import * as core from '@actions/core' import * as exec from '@actions/exec' import * as io from '@actions/io' import {hostDependencies} from './host-dependencies' +import {LogOutput} from './log' export async function installHostDependencies( packages: string, modules: string, piGenDirectory: string -): Promise { +): Promise { let execOutput: exec.ExecOutput | undefined - try { - core.startGroup('Installing build dependencies on host') - const verbose = core.getBooleanInput('verbose-output') + const log = {log: [], debug: []} as LogOutput - const piGenDependencies = await resolvePiGenDependencies(piGenDirectory) + const piGenDependencies = await resolvePiGenDependencies(piGenDirectory, log) - const installPackages = [ - ...new Set([ - ...hostDependencies.packages, - ...packages.split(/[\s,]/), - ...piGenDependencies - ]) - ].filter(p => p) - const hostModules = [ - ...new Set([...hostDependencies.modules, ...modules.split(/[\s,]/)]) - ].filter(m => m) - core.debug( - `Installing additional host packages '${installPackages.join(' ')}'` - ) - core.debug(`Loading additional host modules '${hostModules.join(' ')}'`) + const installPackages = [ + ...new Set([ + ...hostDependencies.packages, + ...packages.split(/[\s,]/), + ...piGenDependencies + ]) + ].filter(p => p) + const hostModules = [ + ...new Set([...hostDependencies.modules, ...modules.split(/[\s,]/)]) + ].filter(m => m) + log.debug.push( + `Installing additional host packages '${installPackages.join(' ')}'` + ) + log.debug.push(`Loading additional host modules '${hostModules.join(' ')}'`) - const sudoPath = await io.which('sudo', true) + const sudoPath = await io.which('sudo', true) - execOutput = await exec.getExecOutput( - sudoPath, - ['-E', 'apt-get', '-y', '-qq', '-o', 'Dpkg::Use-Pty=0', 'update'], - { - silent: !verbose, - env: { - DEBIAN_FRONTEND: 'noninteractive' - } + execOutput = await exec.getExecOutput( + sudoPath, + ['-E', 'apt-get', '-y', '-qq', '-o', 'Dpkg::Use-Pty=0', 'update'], + { + silent: true, + listeners: { + errline: line => log.log.push(line), + stdline: line => log.log.push(line) + }, + env: { + DEBIAN_FRONTEND: 'noninteractive' } - ) + } + ) - execOutput = await exec.getExecOutput( - sudoPath, - [ - '-E', - 'apt-get', - '-qq', - '-o', - 'Dpkg::Use-Pty=0', - '--no-install-recommends', - '--no-install-suggests', - 'install', - '-y', - ...installPackages - ], - { - silent: !verbose, - env: { - DEBIAN_FRONTEND: 'noninteractive' - } + execOutput = await exec.getExecOutput( + sudoPath, + [ + '-E', + 'apt-get', + '-qq', + '-o', + 'Dpkg::Use-Pty=0', + '--no-install-recommends', + '--no-install-suggests', + 'install', + '-y', + ...installPackages + ], + { + silent: true, + listeners: { + errline: line => log.log.push(line), + stdline: line => log.log.push(line) + }, + env: { + DEBIAN_FRONTEND: 'noninteractive' } - ) - execOutput = await exec.getExecOutput( - sudoPath, - ['modprobe', '-a', ...hostModules], - {silent: !verbose} - ) + } + ) + execOutput = await exec.getExecOutput( + sudoPath, + ['modprobe', '-a', ...hostModules], + { + silent: true, + listeners: { + errline: line => log.log.push(line), + stdline: line => log.log.push(line) + } + } + ) - core.info(`Installed packages on host: ${installPackages.join(' ')}`) - core.info(`Loaded modules on host: ${hostModules.join(' ')}`) - } catch (error) { - throw new Error(execOutput?.stderr || (error as Error).message) - } finally { - core.endGroup() - } + log.log.push(`Installed packages on host: ${installPackages.join(' ')}`) + log.log.push(`Loaded modules on host: ${hostModules.join(' ')}`) + + return log } async function resolvePiGenDependencies( - piGenDirectory: string + piGenDirectory: string, + log: LogOutput ): Promise { let piGenDependencies: string[] = [] try { const dependenciesFile = `${piGenDirectory}/depends` const dependenciesStat = await fs.stat(dependenciesFile) if (dependenciesStat.isFile()) { - core.debug(`pi-gen dependencies file found at ${dependenciesFile}`) + log.debug.push(`pi-gen dependencies file found at ${dependenciesFile}`) piGenDependencies = (await fs.readFile(dependenciesFile)) .toString() .split(/\n/) .map(dependency => dependency.substring(dependency.indexOf(':') + 1)) - core.debug( + log.debug.push( `Installing the following dependencies from pi-gen's dependency file: ${piGenDependencies}` ) } diff --git a/src/log.ts b/src/log.ts new file mode 100644 index 0000000..c066116 --- /dev/null +++ b/src/log.ts @@ -0,0 +1,13 @@ +import * as core from '@actions/core' + +export interface LogOutput { + log: string[] + debug: string[] +} + +export async function printLogGroup(logs: LogOutput, verbose: boolean = false) { + if (verbose) { + logs?.log?.forEach(line => core.info(line)) + } + logs?.debug?.forEach(line => core.debug(line)) +}