From 54e0f36d36ff7b6cac12f129a3481a6c338b7923 Mon Sep 17 00:00:00 2001 From: Ryan Kienstra Date: Wed, 11 Aug 2021 08:24:32 -0500 Subject: [PATCH 1/9] Use the correct image for the final container There could be multiple images, so use the same one that was committed. --- src/utils.ts | 33 +++++++++++++++++++++++++-------- 1 file changed, 25 insertions(+), 8 deletions(-) diff --git a/src/utils.ts b/src/utils.ts index 082a2f30..5f4393ef 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -159,7 +159,16 @@ export async function runJob(jobName: string): Promise { message: `Running the CircleCI job ${jobName}`, }); - const tmpPath = '/tmp/circleci'; + const tmpPath = '/tmp/local-ci'; + const checkoutJobs = getCheckoutJobs(`${tmpPath}/${processFile}`); + + // If this is the only job with a checkout, rm the tmp/ directory for this repo. + // If there are files there from another run, they will probably cause an error + // When attempting to overwrite them. + if (checkoutJobs.includes(jobName) && checkoutJobs.length === 1) { + execSync(`rm -rf ${tmpPath}/${getDirectory()}`); + } + try { execSync(`mkdir -p ${tmpPath}`); execSync(`rm -f ${tmpPath}/${processFile}`); @@ -173,9 +182,7 @@ export async function runJob(jobName: string): Promise { ); } - const checkoutJobs = getCheckoutJobs(`${tmpPath}/${processFile}`); - const directoryMatches = getRootPath().match(/[^/]+$/); - const directory = directoryMatches ? directoryMatches[0] : ''; + const directory = getDirectory(); const configFile = getConfigFile(`${tmpPath}/${processFile}`); const attachWorkspaceSteps = configFile?.jobs[jobName]?.steps?.length @@ -219,7 +226,6 @@ export async function runJob(jobName: string): Promise { name: debuggingTerminalName, message: 'This is inside the running container', }); - const latestContainer = '$(docker ps -lq)'; const committedContainerBase = 'local-ci-'; // Once the container is available, start an interactive bash session within the container. @@ -246,9 +252,15 @@ export async function runJob(jobName: string): Promise { }); function commitContainer(): void { - finalTerminal.sendText( - `docker commit ${latestContainer} ${committedContainerBase}${jobName}` - ); + finalTerminal.sendText(` + for container in $(docker ps -q) + do + if [[ $(docker inspect --format='{{.Config.Image}}' $container) == ${dockerImage} ]]; then + echo "Inside the job's container:" + docker commit $container ${committedContainerBase}${jobName} + break + fi + done`); } // Commit the latest container so that this can open an interactive session when it finishes. @@ -291,6 +303,11 @@ export async function runJob(jobName: string): Promise { }); } +export function getDirectory(): string { + const directoryMatches = getRootPath().match(/[^/]+$/); + return directoryMatches ? directoryMatches[0] : ''; +} + export function getDefaultWorkspace(imageName: string): string { try { const stdout = execSync( From 028bb4ee9ddaac8f0a80ab628d77b06ed7312d5d Mon Sep 17 00:00:00 2001 From: Ryan Kienstra Date: Wed, 11 Aug 2021 10:14:11 -0500 Subject: [PATCH 2/9] Define a helper function get_container() This will get the container that has a given image. --- src/utils.ts | 31 ++++++++++++++++--------------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/src/utils.ts b/src/utils.ts index 5f4393ef..09abc30f 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -230,18 +230,26 @@ export async function runJob(jobName: string): Promise { // Once the container is available, start an interactive bash session within the container. debuggingTerminal.sendText(` - until [[ -n $(docker ps -q) && $(docker inspect -f '{{ .Config.Image}}' $(docker ps -q) | grep ${dockerImage}) ]] - do - sleep 2 - done - for container in $(docker ps -q) + get_container() { + IMAGE=$1 + for container in $(docker ps -q) do - if [[ $(docker inspect --format='{{.Config.Image}}' $container) == ${dockerImage} ]]; then + if [[ $(docker inspect --format='{{.Config.Image}}' $container) == $IMAGE ]]; then echo "Inside the job's container:" docker exec -it $container /bin/sh || exit 1 break fi - done`); + done + } + + until [[ -n $(docker ps -q) && $(docker inspect -f '{{ .Config.Image}}' $(docker ps -q) | grep ${dockerImage}) ]] + do + sleep 2 + done + echo "Inside the job's container:" + # Todo: check to see there is a container + docker exec -it get_container(${dockerImage}) /bin/sh || exit 1 + `); debuggingTerminal.show(); @@ -253,14 +261,7 @@ export async function runJob(jobName: string): Promise { function commitContainer(): void { finalTerminal.sendText(` - for container in $(docker ps -q) - do - if [[ $(docker inspect --format='{{.Config.Image}}' $container) == ${dockerImage} ]]; then - echo "Inside the job's container:" - docker commit $container ${committedContainerBase}${jobName} - break - fi - done`); + docker commit get_container(${dockerImage}) ${committedContainerBase}${jobName}`); } // Commit the latest container so that this can open an interactive session when it finishes. From 31802d3482ada90887bd0b33f711d7de36fc7ed6 Mon Sep 17 00:00:00 2001 From: Ryan Kienstra Date: Wed, 11 Aug 2021 10:26:43 -0500 Subject: [PATCH 3/9] Rename function to getDirectoryBasename() It actually gets the basename, not the entire directory. --- src/utils.ts | 39 ++++++++++++++++++++------------------- 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/src/utils.ts b/src/utils.ts index 09abc30f..c6393ce4 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -166,7 +166,7 @@ export async function runJob(jobName: string): Promise { // If there are files there from another run, they will probably cause an error // When attempting to overwrite them. if (checkoutJobs.includes(jobName) && checkoutJobs.length === 1) { - execSync(`rm -rf ${tmpPath}/${getDirectory()}`); + execSync(`rm -rf ${tmpPath}/${getDirectoryBasename()}`); } try { @@ -182,7 +182,7 @@ export async function runJob(jobName: string): Promise { ); } - const directory = getDirectory(); + const directory = getDirectoryBasename(); const configFile = getConfigFile(`${tmpPath}/${processFile}`); const attachWorkspaceSteps = configFile?.jobs[jobName]?.steps?.length @@ -235,19 +235,18 @@ export async function runJob(jobName: string): Promise { for container in $(docker ps -q) do if [[ $(docker inspect --format='{{.Config.Image}}' $container) == $IMAGE ]]; then - echo "Inside the job's container:" - docker exec -it $container /bin/sh || exit 1 - break + return $container fi done } + # @todo: this might be wrong. It might only be doing docker inspect for one image. until [[ -n $(docker ps -q) && $(docker inspect -f '{{ .Config.Image}}' $(docker ps -q) | grep ${dockerImage}) ]] do sleep 2 done echo "Inside the job's container:" - # Todo: check to see there is a container + # @todo: check to see there is a container docker exec -it get_container(${dockerImage}) /bin/sh || exit 1 `); @@ -260,12 +259,13 @@ export async function runJob(jobName: string): Promise { }); function commitContainer(): void { - finalTerminal.sendText(` - docker commit get_container(${dockerImage}) ${committedContainerBase}${jobName}`); + finalTerminal.sendText( + `docker commit get_container(${dockerImage}) ${committedContainerBase}${jobName}` + ); } // Commit the latest container so that this can open an interactive session when it finishes. - // Contianers exit when they finish. + // Containers exit when they finish. // So this creates an alternative container for shell access. commitContainer(); const interval = setInterval(commitContainer, 5000); @@ -286,25 +286,26 @@ export async function runJob(jobName: string): Promise { ); finalTerminal.show(); - } - - if (closedTerminal.name === finalTerminalName) { + } else if (closedTerminal.name === finalTerminalName) { // Remove the container and image that were copies of the running CircleCI job container. const imageName = `${committedContainerBase}${jobName}`; execSync( - `for container in $(docker ps -q) - do - if [[ $(docker inspect --format='{{.Config.Image}}' $container) == ${imageName} ]]; then - docker rm -f $container - fi - done + `docker rm -f get_container(${imageName}) docker rmi -f ${imageName}` ); } }); } -export function getDirectory(): string { +/** + * Gets the basename of the directory. + * + * If the directory is 'example/foo/bar/', + * The basename will be 'bar'. + * + * @returns {string} The basename of the directory. + */ +export function getDirectoryBasename(): string { const directoryMatches = getRootPath().match(/[^/]+$/); return directoryMatches ? directoryMatches[0] : ''; } From ab80f47a32783c2d17b502589c7c9f8cb11ecfcb Mon Sep 17 00:00:00 2001 From: Ryan Kienstra Date: Sat, 14 Aug 2021 14:10:32 -0500 Subject: [PATCH 4/9] If Docker Desktop isn't open, show a message --- src/utils.ts | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/src/utils.ts b/src/utils.ts index c6393ce4..6391b54e 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -152,6 +152,11 @@ export function getRootPath(): string { } export async function runJob(jobName: string): Promise { + if (!isDockerDaemonAvailable()) { + vscode.window.showErrorMessage(`Please open Docker Desktop`); + return; + } + const processFile = 'process.yml'; const terminal = vscode.window.createTerminal({ @@ -160,7 +165,8 @@ export async function runJob(jobName: string): Promise { }); const tmpPath = '/tmp/local-ci'; - const checkoutJobs = getCheckoutJobs(`${tmpPath}/${processFile}`); + const configFileName = '.circleci/config.yml'; + const checkoutJobs = getCheckoutJobs(`${getRootPath()}/${configFileName}`); // If this is the only job with a checkout, rm the tmp/ directory for this repo. // If there are files there from another run, they will probably cause an error @@ -173,7 +179,7 @@ export async function runJob(jobName: string): Promise { execSync(`mkdir -p ${tmpPath}`); execSync(`rm -f ${tmpPath}/${processFile}`); execSync( - `circleci config process ${getRootPath()}/.circleci/config.yml > ${tmpPath}/${processFile}` + `circleci config process ${getRootPath()}/${configFileName} > ${tmpPath}/${processFile}` ); changeCheckoutJob(`${tmpPath}/${processFile}`); } catch (e) { @@ -259,6 +265,7 @@ export async function runJob(jobName: string): Promise { }); function commitContainer(): void { + // @todo: only commit this if get_container() returns an image. finalTerminal.sendText( `docker commit get_container(${dockerImage}) ${committedContainerBase}${jobName}` ); @@ -272,6 +279,7 @@ export async function runJob(jobName: string): Promise { vscode.window.onDidCloseTerminal((closedTerminal) => { clearTimeout(interval); + if ( closedTerminal.name === debuggingTerminalName && closedTerminal?.exitStatus?.code @@ -315,8 +323,9 @@ export function getDefaultWorkspace(imageName: string): string { const stdout = execSync( `docker image inspect ${imageName} --format='{{.Config.User}}'` ); + const userName = stdout.toString().trim() || 'circleci'; - return `/home/${stdout.toString().trim() || 'circleci'}/project`; + return `/home/${userName}/project`; } catch (e) { vscode.window.showErrorMessage( `There was an error getting the default workspace: ${e.message}` @@ -325,3 +334,12 @@ export function getDefaultWorkspace(imageName: string): string { return ''; } } + +export function isDockerDaemonAvailable(): boolean { + try { + execSync(`docker ps`); + return true; + } catch (e) { + return false; + } +} From ac67e3741cb5c69bc61f6f03fb48c404ab1a2250 Mon Sep 17 00:00:00 2001 From: Ryan Kienstra Date: Sun, 15 Aug 2021 15:39:25 -0500 Subject: [PATCH 5/9] Prevent an error on the checkout step When checking out, there was an error: Unexpected environment preparation error: failed to create runner binary: Error response from daemon: mkdir /var/lib/docker/overlay2/f344355b5c446587196e6bb3c881f3d2449cc328cb1e66b0434e1a9807f4d8a6/merged/tmp/_circleci_local_build_repo: file exists --- src/utils.ts | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/src/utils.ts b/src/utils.ts index 6391b54e..0cc16f46 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -176,7 +176,6 @@ export async function runJob(jobName: string): Promise { } try { - execSync(`mkdir -p ${tmpPath}`); execSync(`rm -f ${tmpPath}/${processFile}`); execSync( `circleci config process ${getRootPath()}/${configFileName} > ${tmpPath}/${processFile}` @@ -212,14 +211,14 @@ export async function runJob(jobName: string): Promise { const localVolume = `${tmpPath}/${directory}`; const volume = checkoutJobs.includes(jobName) - ? `${localVolume}:/tmp` + ? `${localVolume}:/${getHomeDirectory(dockerImage)}` : `${localVolume}/${getCheckoutDirectoryBasename( `${tmpPath}/${processFile}` )}:${attachWorkspace}`; const debuggingTerminalName = `local-ci debugging ${jobName}`; const finalTerminalName = 'local-ci final terminal'; - terminal.sendText(`mkdir -p ${localVolume}`); + execSync(`mkdir -p ${localVolume}`); terminal.sendText( `circleci local execute --job ${jobName} --config ${tmpPath}/${processFile} --debug -v ${volume}` ); @@ -267,7 +266,7 @@ export async function runJob(jobName: string): Promise { function commitContainer(): void { // @todo: only commit this if get_container() returns an image. finalTerminal.sendText( - `docker commit get_container(${dockerImage}) ${committedContainerBase}${jobName}` + `docker commit $(get_container ${dockerImage}) ${committedContainerBase}${jobName})` ); } @@ -318,14 +317,21 @@ export function getDirectoryBasename(): string { return directoryMatches ? directoryMatches[0] : ''; } -export function getDefaultWorkspace(imageName: string): string { +/** Gets the directory in /home/ that the job uses, like /home/circleci/. */ +export function getHomeDirectory(imageName: string): string { try { + execSync( + `if [[ -z $(docker image ls | grep ${imageName}) ]]; then + docker image pull ${imageName} + fi` + ); + const stdout = execSync( `docker image inspect ${imageName} --format='{{.Config.User}}'` ); const userName = stdout.toString().trim() || 'circleci'; - return `/home/${userName}/project`; + return `/home/${userName}`; } catch (e) { vscode.window.showErrorMessage( `There was an error getting the default workspace: ${e.message}` @@ -335,6 +341,11 @@ export function getDefaultWorkspace(imageName: string): string { } } +export function getDefaultWorkspace(imageName: string): string { + const homeDirectory = getHomeDirectory(imageName); + return homeDirectory ? `${homeDirectory}/project` : ''; +} + export function isDockerDaemonAvailable(): boolean { try { execSync(`docker ps`); From 3a1cdf70e5e29a2fa19f78e40aea570e2007d148 Mon Sep 17 00:00:00 2001 From: Ryan Kienstra Date: Sun, 15 Aug 2021 16:39:11 -0500 Subject: [PATCH 6/9] Make the get_container() function work In bash, it looks like there really isn't a return statement needed. So echo instead. --- src/utils.ts | 33 +++++++++++++++++---------------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/src/utils.ts b/src/utils.ts index 0cc16f46..59d8d25f 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -232,27 +232,27 @@ export async function runJob(jobName: string): Promise { message: 'This is inside the running container', }); const committedContainerBase = 'local-ci-'; + const getContainerDefinition = `get_container() { + IMAGE=$1 + for container in $(docker ps -q) + do + if [[ $(docker inspect --format='{{.Config.Image}}' "$container") == $IMAGE ]]; then + echo $container + fi + done + }`; // Once the container is available, start an interactive bash session within the container. debuggingTerminal.sendText(` - get_container() { - IMAGE=$1 - for container in $(docker ps -q) - do - if [[ $(docker inspect --format='{{.Config.Image}}' $container) == $IMAGE ]]; then - return $container - fi - done - } - - # @todo: this might be wrong. It might only be doing docker inspect for one image. - until [[ -n $(docker ps -q) && $(docker inspect -f '{{ .Config.Image}}' $(docker ps -q) | grep ${dockerImage}) ]] + ${getContainerDefinition} + until [[ ! -z $(docker ps -q) && ! -z $(docker inspect -f '{{ .Config.Image}}' $(docker ps -q) | grep ${dockerImage}) ]] do sleep 2 done echo "Inside the job's container:" - # @todo: check to see there is a container - docker exec -it get_container(${dockerImage}) /bin/sh || exit 1 + CONTAINER=$(get_container ${dockerImage}) + echo "The container is $CONTAINER" + docker exec -it $CONTAINER /bin/sh || exit 1 `); debuggingTerminal.show(); @@ -266,7 +266,8 @@ export async function runJob(jobName: string): Promise { function commitContainer(): void { // @todo: only commit this if get_container() returns an image. finalTerminal.sendText( - `docker commit $(get_container ${dockerImage}) ${committedContainerBase}${jobName})` + `${getContainerDefinition} + docker commit $(get_container ${dockerImage}) ${committedContainerBase}${jobName}` ); } @@ -297,7 +298,7 @@ export async function runJob(jobName: string): Promise { // Remove the container and image that were copies of the running CircleCI job container. const imageName = `${committedContainerBase}${jobName}`; execSync( - `docker rm -f get_container(${imageName}) + `docker rm -f $(get_container ${imageName}) docker rmi -f ${imageName}` ); } From b7ba0ae29a5a52f7defc9d155f7311a4e5f564f4 Mon Sep 17 00:00:00 2001 From: Ryan Kienstra Date: Sun, 15 Aug 2021 18:24:00 -0500 Subject: [PATCH 7/9] Handle a case where they don't persist the entire workspace In that case, there should be a /tmp directory that the volume uses, to selectively copy to the local. --- src/utils.ts | 45 ++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 40 insertions(+), 5 deletions(-) diff --git a/src/utils.ts b/src/utils.ts index 59d8d25f..97efe354 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -112,7 +112,10 @@ export function changeCheckoutJob(processFile: string): void { configFile.jobs[checkoutJob].steps = configFile?.jobs[ checkoutJob ]?.steps?.map((step) => { - if (!step?.persist_to_workspace) { + if ( + !step?.persist_to_workspace || + doesPersistEntireWorkspace(configFile, checkoutJob) + ) { return step; } @@ -211,7 +214,7 @@ export async function runJob(jobName: string): Promise { const localVolume = `${tmpPath}/${directory}`; const volume = checkoutJobs.includes(jobName) - ? `${localVolume}:/${getHomeDirectory(dockerImage)}` + ? `${localVolume}:${getCheckoutJobDirectory(jobName, configFile)}` : `${localVolume}/${getCheckoutDirectoryBasename( `${tmpPath}/${processFile}` )}:${attachWorkspace}`; @@ -250,9 +253,7 @@ export async function runJob(jobName: string): Promise { sleep 2 done echo "Inside the job's container:" - CONTAINER=$(get_container ${dockerImage}) - echo "The container is $CONTAINER" - docker exec -it $CONTAINER /bin/sh || exit 1 + docker exec -it $(get_container ${dockerImage}) /bin/sh || exit 1 `); debuggingTerminal.show(); @@ -318,6 +319,40 @@ export function getDirectoryBasename(): string { return directoryMatches ? directoryMatches[0] : ''; } +export function getCheckoutJobDirectory( + jobName: string, + configFile: ConfigFile +): string { + if (!configFile || !configFile.jobs[jobName]?.steps) { + return ''; + } + + const imageName = configFile.jobs[jobName]?.docker[0]?.image; + + return doesPersistEntireWorkspace(configFile, jobName) + ? getDefaultWorkspace(imageName) + : `/tmp`; +} + +export function doesPersistEntireWorkspace( + configFile: ConfigFile, + jobName: string +): boolean | undefined { + const persistToWorkspaceSteps = configFile?.jobs[jobName]?.steps?.filter( + (step) => step?.persist_to_workspace + ); + + if (!persistToWorkspaceSteps?.length) { + return false; + } + + return persistToWorkspaceSteps.every( + (step) => + '.' === step?.persist_to_workspace?.root && + step.persist_to_workspace?.paths?.every((path) => '.' === path) + ); +} + /** Gets the directory in /home/ that the job uses, like /home/circleci/. */ export function getHomeDirectory(imageName: string): string { try { From 823c6b1219a69416978c822997d8a632a475976e Mon Sep 17 00:00:00 2001 From: Ryan Kienstra Date: Sun, 15 Aug 2021 18:25:40 -0500 Subject: [PATCH 8/9] Return '/tmp' if the job has no steps (unlikely) --- src/utils.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/utils.ts b/src/utils.ts index 97efe354..c33427d7 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -324,14 +324,14 @@ export function getCheckoutJobDirectory( configFile: ConfigFile ): string { if (!configFile || !configFile.jobs[jobName]?.steps) { - return ''; + return '/tmp'; } const imageName = configFile.jobs[jobName]?.docker[0]?.image; return doesPersistEntireWorkspace(configFile, jobName) ? getDefaultWorkspace(imageName) - : `/tmp`; + : '/tmp'; } export function doesPersistEntireWorkspace( From 3a0f28d73c563c9a5189ae4201e7a7dca68c3224 Mon Sep 17 00:00:00 2001 From: Ryan Kienstra Date: Sun, 15 Aug 2021 23:28:20 -0500 Subject: [PATCH 9/9] Remove getCheckoutDirectoryBasename() It looks like that's not needed, at least with the repos I've been testing. --- src/utils.ts | 41 +---------------------------------------- 1 file changed, 1 insertion(+), 40 deletions(-) diff --git a/src/utils.ts b/src/utils.ts index c33427d7..3aa43731 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -54,43 +54,6 @@ export function getCheckoutJobs(inputFile = ''): string[] { ); } -export function getCheckoutDirectoryBasename(processFile: string): string { - const checkoutJobs = getCheckoutJobs(processFile); - const configFile = getConfigFile(processFile); - - if (!configFile || !checkoutJobs.length) { - return ''; - } - - const checkoutJob = checkoutJobs[0]; - if (!configFile.jobs[checkoutJob]?.steps) { - return ''; - } - - const stepWithPersist = configFile?.jobs[checkoutJob]?.steps?.find( - (step) => step?.persist_to_workspace - ); - - const persistToWorkspacePath = stepWithPersist?.persist_to_workspace?.paths - ?.length - ? stepWithPersist.persist_to_workspace.paths[0] - : ''; - - const pathBase = - !stepWithPersist?.persist_to_workspace?.root || - '.' === stepWithPersist.persist_to_workspace.root - ? configFile.jobs[checkoutJob]?.working_directory ?? - getDefaultWorkspace(configFile.jobs[checkoutJob]?.docker[0]?.image) - : stepWithPersist.persist_to_workspace.root; - - const pathMatches = - !persistToWorkspacePath || persistToWorkspacePath === '.' - ? pathBase.match(/[^/]+$/) - : persistToWorkspacePath.match(/[^/]+$/); - - return pathMatches ? pathMatches[0] : ''; -} - export function changeCheckoutJob(processFile: string): void { const checkoutJobs = getCheckoutJobs(processFile); const configFile = getConfigFile(processFile); @@ -215,9 +178,7 @@ export async function runJob(jobName: string): Promise { const localVolume = `${tmpPath}/${directory}`; const volume = checkoutJobs.includes(jobName) ? `${localVolume}:${getCheckoutJobDirectory(jobName, configFile)}` - : `${localVolume}/${getCheckoutDirectoryBasename( - `${tmpPath}/${processFile}` - )}:${attachWorkspace}`; + : `${localVolume}:${attachWorkspace}`; const debuggingTerminalName = `local-ci debugging ${jobName}`; const finalTerminalName = 'local-ci final terminal';