From 726cc1b99718d23be86fb65d91aa4acf19eb5f12 Mon Sep 17 00:00:00 2001 From: Bryan Kendall Date: Tue, 24 Feb 2015 14:22:00 -0800 Subject: [PATCH 1/7] better race prevention, sort tests a little better --- lib/dockerLayerArchive.sh | 6 + lib/steps.js | 142 ++++---- package.json | 5 +- test/dockerBuildTest.js | 448 -------------------------- test/steps/checkForRequiredEnvVars.js | 52 +++ test/steps/chmodAllKeys.js | 129 ++++++++ test/steps/downloadBuildFiles.js | 158 +++++++++ test/steps/downloadDeployKeys.js | 93 ++++++ test/steps/getRepositories.js | 318 ++++++++++++++++++ test/steps/makeWorkingFolders.js | 122 +++++++ test/steps/parseBuildLogAndHistory.js | 124 +++++++ test/steps/parseDockerfile.js | 172 ++++++++++ test/steps/runDockerBuild.js | 125 +++++++ 13 files changed, 1369 insertions(+), 525 deletions(-) create mode 100644 test/steps/checkForRequiredEnvVars.js create mode 100644 test/steps/chmodAllKeys.js create mode 100644 test/steps/downloadBuildFiles.js create mode 100644 test/steps/downloadDeployKeys.js create mode 100644 test/steps/getRepositories.js create mode 100644 test/steps/makeWorkingFolders.js create mode 100644 test/steps/parseBuildLogAndHistory.js create mode 100644 test/steps/parseDockerfile.js create mode 100644 test/steps/runDockerBuild.js diff --git a/lib/dockerLayerArchive.sh b/lib/dockerLayerArchive.sh index 78c583c..3118d7b 100755 --- a/lib/dockerLayerArchive.sh +++ b/lib/dockerLayerArchive.sh @@ -13,6 +13,10 @@ if [ ! "$CACHED_LAYER" ]; then >&2 echo "Need CACHED_LAYER" exit 1 fi +if [ ! "$CACHED_LAYER_HASH" ]; then + >&2 echo "Need CACHED_LAYER_HASH" + exit 1 +fi if [ ! "$IMAGE_ID" ]; then >&2 echo "Need IMAGE_ID" exit 1 @@ -24,6 +28,8 @@ docker -H "$RUNNABLE_DOCKER" save -o "$tar_name" "$IMAGE_ID" cache_layer_name=$(echo "$RUNNABLE_DOCKERTAG" | awk '{split($0,a,":"); print a[1];}') tar --extract --file "$tar_name" --directory="$tar_dir" "$CACHED_LAYER"/layer.tar mkdir -p /layer-cache/"$cache_layer_name" +rm -rf /layer-cache/"$cache_layer_name"/hash.* mv "$tar_dir"/"$CACHED_LAYER"/layer.tar /layer-cache/"$cache_layer_name"/layer.tar +touch /layer-cache/"$cache_layer_name"/hash."$CACHED_LAYER_HASH" echo "Saved $cache_layer_name/layer.tar" diff --git a/lib/steps.js b/lib/steps.js index 1119f90..7537285 100644 --- a/lib/steps.js +++ b/lib/steps.js @@ -1,8 +1,7 @@ 'use strict'; var colors = require('colors'); -var exec = require('child_process').exec; -var spawn = require('child_process').spawn; +var childProcess = require('child_process'); var async = require('async'); var fs = require('fs'); var path = require('path'); @@ -53,7 +52,7 @@ var steps = module.exports = { }); function mkDir (template, cb) { - exec( + childProcess.exec( 'mktemp -d ' + template, function (err, stdout) { if (err) { return cb(err); } @@ -61,7 +60,7 @@ var steps = module.exports = { }); } function mkFile (template, cb) { - exec( + childProcess.exec( 'mktemp ' + template, function (err, stdout) { if (err) { return cb(err); } @@ -83,7 +82,7 @@ var steps = module.exports = { '--file ' + key, '--dest ' + steps.dirs.keyDirectory ].join(' '); - exec('node downloadS3Files.js ' + args, + childProcess.exec('node downloadS3Files.js ' + args, { cwd: thisDir }, @@ -97,7 +96,7 @@ var steps = module.exports = { if (!process.env.RUNNABLE_DEPLOYKEY || process.env.RUNNABLE_DEPLOYKEY.length === 0) { return cb(); } - exec('chmod -R 600 *', + childProcess.exec('chmod -R 600 *', { cwd: steps.dirs.keyDirectory }, @@ -120,7 +119,6 @@ var steps = module.exports = { } catch (err) { return cb(new Error('RUNNABLE_FILES is poorly formatted JSON.')); } - console.log('Downloading build files...'.bold.yellow); // one call to downloadS3Files with this object var args = [ '--bucket ' + process.env.RUNNABLE_FILES_BUCKET, @@ -128,7 +126,7 @@ var steps = module.exports = { '--prefix "' + process.env.RUNNABLE_PREFIX + '"', '--dest ' + steps.dirs.dockerContext ].join(' '); - exec('node downloadS3Files.js ' + args, + childProcess.exec('node downloadS3Files.js ' + args, { cwd: thisDir }, @@ -137,15 +135,13 @@ var steps = module.exports = { }, getRepositories: function (cb) { - if (!process.env.RUNNABLE_REPO || process.env.RUNNABLE_REPO.length === 0) { + if (!process.env.RUNNABLE_REPO) { return cb(); - } else if (process.env.RUNNABLE_REPO && - (!process.env.RUNNABLE_COMMITISH || - process.env.RUNNABLE_COMMITISH.length === 0)) { + } else if (!process.env.RUNNABLE_COMMITISH) { return cb(new Error('RUNNABLE_COMMITISH is missing.')); } // make sure we have a cache directory - exec('mkdir -p ' + cacheDir, function (err) { + childProcess.exec('mkdir -p ' + cacheDir, function (err) { if (err) { return cb(err); } var reposAndBranches = zip( process.env.RUNNABLE_REPO.split(';'), @@ -170,29 +166,38 @@ var steps = module.exports = { colors.yellow.bold('Cloning %s into ./%s'), repoFullName, repoName); async.waterfall([ function makeRepoDir (cb) { - exec('mkdir -p ' + repoFullName, steps.saveToLogs(function (err) { - cb(err); - })); + childProcess.exec( + 'mkdir -p ' + repoFullName, + steps.saveToLogs(function (err) { + cb(err); + }) + ); }, function addKeyToAgent (cb) { - exec('ssh-add -D', function (err) { + childProcess.exec('ssh-add -D', function (err) { if (err) { return cb(err); } - exec('ssh-add ' + repoKeyPath, steps.saveToLogs(function (err) { - cb(err); - })); + childProcess.exec( + 'ssh-add ' + repoKeyPath, + steps.saveToLogs(function (err) { + cb(err); + }) + ); }); }, function tryForLock (cb) { // need to make sure the directory for the username is there... - exec('mkdir -p ' + path.dirname(lockfileName), function (err) { - if (err) { return cb(err); } - // TODO no steps and retries yet - lockfile.lock(lockfileName, function (err) { - // if no err, then we got the lock - aquiredLock = !err; - cb(null, aquiredLock); - }); - }); + childProcess.exec( + 'mkdir -p ' + path.dirname(lockfileName), + function (err) { + if (err) { return cb(err); } + // TODO no steps and retries yet + lockfile.lock(lockfileName, function (err) { + // if no err, then we got the lock + aquiredLock = !err; + cb(null, aquiredLock); + }); + } + ); }, function cacheOrClone (lockAquired, cb) { if (lockAquired) { @@ -203,13 +208,13 @@ var steps = module.exports = { // repo exists, just will update cb(); } else { - exec( + childProcess.exec( 'git clone -q ' + repo + ' ' + repoCacheDir, steps.saveToLogs(cb)); } }, function getHEADCommit (cb) { - exec('git rev-parse HEAD', + childProcess.exec('git rev-parse HEAD', { cwd: repoCacheDir }, @@ -222,7 +227,7 @@ var steps = module.exports = { }, function updateCachedRepo (cb) { if (fetchAndCheckout) { - exec('git fetch --all', + childProcess.exec('git fetch --all', { cwd: repoCacheDir }, @@ -234,7 +239,7 @@ var steps = module.exports = { }, function checkoutCommitish (cb) { if (fetchAndCheckout) { - exec('git checkout -q ' + commitish, + childProcess.exec('git checkout -q ' + commitish, { cwd: repoCacheDir }, @@ -253,7 +258,7 @@ var steps = module.exports = { repoCacheDir, repoTargetDir ].join(' '); - exec('cp ' + args, steps.saveToLogs(cb)); + childProcess.exec('cp ' + args, steps.saveToLogs(cb)); }, // TODO touch all the things (fix times after copy) // function touchAllTheThings (cb) {} @@ -262,11 +267,11 @@ var steps = module.exports = { cb(err); }); } else { - exec( + childProcess.exec( 'git clone -q ' + repo + ' ' + repoTargetDir, steps.saveToLogs(function (err) { if (err) { return cb(err); } - exec('git checkout -q ' + commitish, + childProcess.exec('git checkout -q ' + commitish, { cwd: repoTargetDir }, @@ -330,7 +335,7 @@ var steps = module.exports = { path.join(steps.dirs.dockerContext, 'Dockerfile'), dockerfile); steps.data.usingCache = true; - exec( + childProcess.exec( 'cp -p ' + layerTar + ' ' + steps.dirs.dockerContext, steps.saveToLogs(cb)); } else { cb(); } @@ -351,8 +356,7 @@ var steps = module.exports = { 'build', '--tag=' + process.env.RUNNABLE_DOCKERTAG ]; - if (process.env.RUNNABLE_DOCKER_BUILDOPTIONS && - process.env.RUNNABLE_DOCKER_BUILDOPTIONS !== '') { + if (process.env.RUNNABLE_DOCKER_BUILDOPTIONS !== '') { args.push.apply( args, process.env.RUNNABLE_DOCKER_BUILDOPTIONS.split(' ')); } @@ -360,7 +364,7 @@ var steps = module.exports = { steps.dirs.dockerContext ); var count = createCount(3, cb); - var build = spawn('docker', args); + var build = childProcess.spawn('docker', args); var stdoutHandler = concat(function (data) { fs.appendFileSync(steps.logs.dockerBuild, data); steps.saveToLogs(count.next)(null, data, ''); @@ -403,25 +407,28 @@ var steps = module.exports = { '--no-trunc', imageId ].join(' '); - exec('docker ' + historyArgs, steps.saveToLogs(function (err, stdout) { - if (err) { return cb(err); } - var lines = stdout.toString().split('\n'); - var layer; - for (var i = 0; i < lines.length; ++i) { - /* jshint -W101 */ - if (/\# ?[R|r][U|u][N|n][N|n][A|a][B|b][L|l][E|e]-[C|c][A|a][C|c][H|h][E|e]/ - .test(lines[i].trim())) { - layer = lines[i].trim().split(' ').shift(); - break; + childProcess.exec( + 'docker ' + historyArgs, + steps.saveToLogs(function (err, stdout) { + if (err) { return cb(err); } + var lines = stdout.toString().split('\n'); + var layer; + for (var i = 0; i < lines.length; ++i) { + /* jshint -W101 */ + if (/\# ?[R|r][U|u][N|n][N|n][A|a][B|b][L|l][E|e]-[C|c][A|a][C|c][H|h][E|e]/ + .test(lines[i].trim())) { + layer = lines[i].trim().split(' ').shift(); + break; + } + /* jshint +W101 */ } - /* jshint +W101 */ - } - if (layer) { - steps.data.getLayerFromThisImage = imageId; - steps.data.cacheThisLayer = layer; - } - cb(); - })); + if (layer) { + steps.data.getLayerFromThisImage = imageId; + steps.data.cacheThisLayer = layer; + } + cb(); + }) + ); }, copyLayer: function (cb) { @@ -434,6 +441,7 @@ var steps = module.exports = { '-d', '-e "IMAGE_ID=' + steps.data.getLayerFromThisImage + '"', '-e "CACHED_LAYER=' + steps.data.cacheThisLayer + '"', + '-e "CACHED_LAYER_HASH=' + steps.data.createdByHash + '"', '-e "RUNNABLE_DOCKER=' + process.env.RUNNABLE_DOCKER + '"', '-e "RUNNABLE_DOCKERTAG=' + process.env.RUNNABLE_DOCKERTAG + '"', '-v "' + process.env.DOCKER_IMAGE_BUILDER_LAYER_CACHE + ':' + @@ -443,23 +451,7 @@ var steps = module.exports = { process.env.RUNNABLE_IMAGE_BUILDER_TAG, './dockerLayerArchive.sh' ].join(' '); - exec('docker ' + args, steps.saveToLogs(function (err) { - if (err) { return cb(err); } - var hashDirectory = path.join( - layerCacheDir, - process.env.RUNNABLE_DOCKERTAG.split(':')[0]); - var allHashFiles = path.join( - hashDirectory, - 'hash.*'); - var filenameToTouch = path.join( - hashDirectory, - 'hash.' + steps.data.createdByHash); - exec( - 'mkdir -p ' + hashDirectory + ' && ' + - 'rm -f ' + allHashFiles + ' && ' + - 'touch ' + filenameToTouch, - steps.saveToLogs(cb)); - })); + childProcess.exec('docker ' + args, steps.saveToLogs(cb)); }, saveToLogs: function (cb) { diff --git a/package.json b/package.json index 3e42f2e..1893510 100644 --- a/package.json +++ b/package.json @@ -18,9 +18,10 @@ }, "devDependencies": { "code": "^1.3.0", - "docker-mock": "^0.6.0", + "docker-mock": "^1.0.0", "jshint": "^2.6.0", - "lab": "^5.2.1" + "lab": "^5.2.1", + "sinon": "^1.12.2" }, "scripts": { "test": "./node_modules/.bin/lab --verbose", diff --git a/test/dockerBuildTest.js b/test/dockerBuildTest.js index 99296f9..469da96 100644 --- a/test/dockerBuildTest.js +++ b/test/dockerBuildTest.js @@ -4,26 +4,6 @@ var Lab = require('lab'); var lab = exports.lab = Lab.script(); var expect = require('code').expect; -var fs = require('fs'); -var path = require('path'); -var exec = require('child_process').exec; -var dockerMock = require('docker-mock'); -dockerMock.listen(5555); - -var cacheDir = process.env.CACHE_DIR; -if (!cacheDir) { - cacheDir = process.env.CACHE_DIR = '/tmp/cache'; -} -var layerCacheDir = process.env.LAYER_CACHE_DIR; -if (!layerCacheDir) { - layerCacheDir = process.env.LAYER_CACHE_DIR = '/tmp/layer-cache'; -} -exec('mkdir -p ', cacheDir); -exec('mkdir -p ', layerCacheDir); - -// require this after we have now changed the env for the directories -var steps = require('../lib/steps'); - lab.test('we need environment variables to run the tests', function (done) { var keys = [ 'AWS_ACCESS_KEY', @@ -34,431 +14,3 @@ lab.test('we need environment variables to run the tests', function (done) { }); done(); }); - -var requiredEnvVars = { - RUNNABLE_AWS_ACCESS_KEY: process.env.AWS_ACCESS_KEY, - RUNNABLE_AWS_SECRET_KEY: process.env.AWS_SECRET_KEY -}; -lab.beforeEach(function (done) { - Object.keys(requiredEnvVars).forEach(function (key) { - process.env[key] = requiredEnvVars[key]; - }); - done(); -}); - -lab.experiment('checkForRequiredEnvVars', function () { - lab.experiment('fails', function () { - lab.beforeEach(function (done) { - Object.keys(requiredEnvVars).forEach( - function (key) { delete process.env[key]; }); - done(); - }); - lab.test('when required env vars are missing', function (done) { - steps.checkForRequiredEnvVars(function (err) { - expect(!!err).to.be.true(); - expect(err.message).to.match(/Missing credentials./); - done(null); - }); - }); - }); - lab.experiment('succeeds', function () { - lab.test('when all env vars are present', function (done) { - steps.checkForRequiredEnvVars(function (err) { - if (err) { return done(err); } - done(err); - }); - }); - }); -}); - -lab.experiment('makeWorkingFolders', function () { - lab.experiment('succeeds', function () { - var createdFolders = [ - 'dockerContext', - 'keyDirectory' - ]; - lab.test('to create all folders', function (done) { - steps.makeWorkingFolders(function (err) { - if (err) { return done(err); } - createdFolders.forEach(function (dirName) { - expect(steps.dirs[dirName]).to.not.be.undefined(); - expect(fs.existsSync(steps.dirs[dirName])).to.be.true(); - }); - done(); - }); - }); - }); -}); - -lab.experiment('downloadDeployKeys', function () { - var requiredEnvVars = { - RUNNABLE_KEYS_BUCKET: 'runnable.image-builder', - RUNNABLE_DEPLOYKEY: 'flaming-octo-nemesis.key' - }; - lab.beforeEach(function (done) { - Object.keys(requiredEnvVars).forEach( - function (key) { process.env[key] = requiredEnvVars[key]; }); - done(); - }); - lab.beforeEach(steps.makeWorkingFolders.bind(steps)); - - lab.experiment('fails', function () { - lab.beforeEach(function (done) { - delete process.env.RUNNABLE_KEYS_BUCKET; - done(); - }); - lab.test('when the required env vars are missing', function (done) { - steps.downloadDeployKeys(function (err) { - expect(!!err).to.be.true(); - expect(err.message).to.match(/RUNNABLE_KEYS_BUCKET is missing/); - done(); - }); - }); - }); - lab.experiment('succeeds', function () { - lab.experiment('if there are no keys', function () { - lab.beforeEach(function (done) { - delete process.env.RUNNABLE_DEPLOYKEY; - done(); - }); - lab.afterEach(function (done) { - process.env.RUNNABLE_DEPLOYKEY = requiredEnvVars.RUNNABLE_DEPLOYKEY; - done(); - }); - lab.test('it should be fine', function (done) { - steps.downloadDeployKeys(function (err) { - if (err) { return done(err); } - // TODO check for empty directory - done(); - }); - }); - }); - lab.experiment('with keys to download', function () { - lab.test('to download the keys', { timeout: 5000 }, function (done) { - steps.downloadDeployKeys(function (err) { - if (err) { return done(err); } - var keyPath = path.join( - steps.dirs.keyDirectory, - requiredEnvVars.RUNNABLE_DEPLOYKEY); - expect(fs.existsSync(keyPath)).to.be.true(); - done(); - }); - }); - }); - }); -}); - -lab.experiment('chmodAllKeys', function () { - var requiredEnvVars = { - RUNNABLE_KEYS_BUCKET: 'runnable.image-builder', - RUNNABLE_DEPLOYKEY: 'flaming-octo-nemesis.key' - }; - lab.beforeEach(function (done) { - Object.keys(requiredEnvVars).forEach( - function (key) { process.env[key] = requiredEnvVars[key]; }); - done(); - }); - lab.beforeEach(steps.makeWorkingFolders.bind(steps)); - - lab.experiment('succeeds', function () { - lab.experiment('when there are keys', function () { - lab.beforeEach({ timeout: 5000 }, steps.downloadDeployKeys.bind(steps)); - - lab.test('to set the permissions on the keys', function (done) { - steps.chmodAllKeys(function (err) { - if (err) { return done(err); } - var keyPath = path.join( - steps.dirs.keyDirectory, - requiredEnvVars.RUNNABLE_DEPLOYKEY); - fs.stat(keyPath, function (err, stats) { - if (err) { return done(err); } - expect(stats.mode).to.equal(33152); - done(); - }); - }); - }); - }); - }); -}); - -lab.experiment('downloadBuildFiles', function () { - var requiredEnvVars = { - RUNNABLE_FILES: '{ "Dockerfile": "K6cluDupwQdFRsuTPJ0SFUrxUB4lmF_Q" }', - RUNNABLE_FILES_BUCKET: 'runnable.image-builder' - }; - lab.beforeEach(function (done) { - Object.keys(requiredEnvVars).forEach( - function (key) { process.env[key] = requiredEnvVars[key]; }); - done(); - }); - lab.beforeEach(steps.makeWorkingFolders.bind(steps)); - - lab.experiment('fails', function () { - lab.beforeEach(function (done) { - delete process.env.RUNNABLE_FILES_BUCKET; - done(); - }); - lab.test('when the required env vars are missing', function (done) { - steps.downloadBuildFiles(function (err) { - expect(!!err).to.be.true(); - expect(err.message).to.match(/RUNNABLE_FILES_BUCKET is missing/); - done(); - }); - }); - }); - lab.experiment('succeeds', function () { - lab.experiment('if there are no files', function () { - lab.beforeEach(function (done) { - delete process.env.RUNNABLE_FILES; - done(); - }); - lab.afterEach(function (done) { - process.env.RUNNABLE_FILES = requiredEnvVars.RUNNABLE_FILES; - done(); - }); - lab.test('it should be fine', function (done) { - steps.downloadBuildFiles(function (err) { - if (err) { return done(err); } - // TODO check for empty directory - done(); - }); - }); - }); - lab.experiment('with keys to download', function () { - lab.test('to download the keys', function (done) { - steps.downloadBuildFiles(function (err) { - if (err) { return done(err); } - var dockerfilePath = path.join( - steps.dirs.dockerContext, - 'Dockerfile'); - expect(fs.existsSync(dockerfilePath)).to.be.true(); - done(); - }); - }); - }); - }); -}); - -lab.experiment('getRepositories', function () { - var requiredEnvVars = { - RUNNABLE_REPO: 'git@github.com:bkendall/flaming-octo-nemesis', - RUNNABLE_COMMITISH: 'master', - RUNNABLE_KEYS_BUCKET: 'runnable.image-builder', - RUNNABLE_DEPLOYKEY: 'flaming-octo-nemesis.key' - }; - lab.beforeEach(function (done) { - Object.keys(requiredEnvVars).forEach( - function (key) { process.env[key] = requiredEnvVars[key]; }); - done(); - }); - lab.beforeEach(function (done) { - exec('rm -rf ' + cacheDir + '/*', done); - }); - lab.experiment('succeeds', function () { - lab.beforeEach(steps.makeWorkingFolders.bind(steps)); - lab.beforeEach({ timeout: 5000 }, steps.downloadDeployKeys.bind(steps)); - lab.beforeEach(steps.chmodAllKeys.bind(steps)); - - lab.experiment('when there is a repo', function () { - /* github can be slow to respond. long timeout */ - lab.test('to download the repo', { timeout: 10000 }, function (done) { - steps.getRepositories(function (err) { - if (err) { return done(err); } - var repoCacheDir = path.join( - cacheDir, - 'bkendall/flaming-octo-nemesis'); - var repoTargetDir = path.join( - steps.dirs.dockerContext, - 'flaming-octo-nemesis'); - expect(fs.existsSync(repoCacheDir)).to.be.true(); - expect(fs.existsSync(repoTargetDir)).to.be.true(); - done(); - }); - }); - }); - lab.experiment('when a lock already exists for the repo', function () { - lab.beforeEach(function (done) { - var cmds = [ - 'mkdir', - '-p', - cacheDir + '/bkendall', - '&& touch', - cacheDir + '/bkendall/flaming-octo-nemesis.lock' - ].join(' '); - exec(cmds, done); - }); - /* github can be slow to respond. long timeout */ - lab.test('to download the repo', { timeout: 10000 }, function (done) { - steps.getRepositories(function (err) { - if (err) { return done(err); } - var repoCacheDir = path.join( - cacheDir, - 'bkendall/flaming-octo-nemesis'); - var repoCacheDirLock = path.join( - cacheDir, - 'bkendall/flaming-octo-nemesis.lock'); - var repoTargetDir = path.join( - steps.dirs.dockerContext, - 'flaming-octo-nemesis'); - expect(fs.existsSync(repoCacheDir)).to.be.false(); - expect(fs.existsSync(repoCacheDirLock)).to.be.true(); - expect(fs.existsSync(repoTargetDir)).to.be.true(); - done(); - }); - }); - }); - }); -}); - -lab.experiment('parseDockerfile', function () { - var requiredEnvVars = { - RUNNABLE_DOCKERTAG: 'test-docker-tag', - RUNNABLE_FILES: '{ "Dockerfile": "AolcUvaTfKOFJg74ABqL9NN08333MS_t" }', - RUNNABLE_FILES_BUCKET: 'runnable.image-builder' - }; - lab.beforeEach(function (done) { - Object.keys(requiredEnvVars).forEach( - function (key) { process.env[key] = requiredEnvVars[key]; }); - done(); - }); - lab.experiment('succeeds', function () { - lab.beforeEach(steps.makeWorkingFolders.bind(steps)); - - lab.experiment('with runnable-cache', function () { - lab.beforeEach({ timeout: 5000 }, steps.downloadBuildFiles.bind(steps)); - - lab.test('should catch the cache line', function (done) { - steps.parseDockerfile(function (err) { - if (err) { return done(err); } - expect(!!steps.data.usingCache).to.be.false(); - expect(steps.data.cachedLine).to.not.be.undefined(); - expect(steps.data.createdByHash).to.not.be.undefined(); - done(); - }); - }); - }); - lab.experiment('with funky Runnable-cache', function () { - lab.beforeEach(function (done) { - process.env.RUNNABLE_FILES = - '{ "Dockerfile": "KKUneazEu5iFAJAkfOIHe0C2jaeGgZpn" }'; - done(); - }); - lab.beforeEach({ timeout: 5000 }, steps.downloadBuildFiles.bind(steps)); - - lab.test('should find the cached line', function (done) { - steps.parseDockerfile(function (err) { - if (err) { return done(err); } - expect(!!steps.data.usingCache).to.be.false(); - expect(steps.data.cachedLine).to.not.be.undefined(); - expect(steps.data.createdByHash).to.not.be.undefined(); - done(); - }); - }); - }); - }); -}); - -lab.experiment('runDockerBuild', function () { - var requiredEnvVars = { - RUNNABLE_DOCKER: 'tcp://localhost:5555', - RUNNABLE_DOCKERTAG: 'test-docker-tag', - RUNNABLE_FILES: '{ "Dockerfile": "K6cluDupwQdFRsuTPJ0SFUrxUB4lmF_Q" }', - RUNNABLE_FILES_BUCKET: 'runnable.image-builder' - }; - lab.beforeEach(function (done) { - Object.keys(requiredEnvVars).forEach( - function (key) { process.env[key] = requiredEnvVars[key]; }); - done(); - }); - lab.experiment('fails', function () { - lab.beforeEach(function (done) { - delete process.env.RUNNABLE_DOCKERTAG; - done(); - }); - lab.test('when the required env vars are missing', function (done) { - steps.runDockerBuild(function (err) { - expect(!!err).to.be.true(); - expect(err.message).to.match(/RUNNABLE_DOCKERTAG is missing/); - done(); - }); - }); - }); - lab.experiment('succeeds', function () { - lab.beforeEach(steps.makeWorkingFolders.bind(steps)); - lab.beforeEach({ timeout: 5000 }, steps.downloadBuildFiles.bind(steps)); - - lab.experiment('if there is no dockerhost', function () { - lab.beforeEach(function (done) { - delete process.env.RUNNABLE_DOCKER; - done(); - }); - lab.test('should do nothing', function (done) { - steps.runDockerBuild(function (err) { - if (err) { return done(err); } - done(); - }); - }); - }); - lab.experiment('to call out to docker and run the build', function () { - lab.test('should do nothing', function (done) { - steps.runDockerBuild(function (err) { - if (err) { return done(err); } - var dockerArgs = [ - '--host ' + requiredEnvVars.RUNNABLE_DOCKER, - 'images' - ].join(' '); - exec('docker ' + dockerArgs, function (err, stdout) { - if (err) { return done(err); } - expect(stdout.trim().split('\n')).to.have.length(2); - done(err); - }); - }); - }); - }); - }); -}); - -lab.experiment('parseBuildLogAndHistory', function () { - var requiredEnvVars = { - RUNNABLE_DOCKER: 'tcp://localhost:5555', - RUNNABLE_DOCKERTAG: 'test-docker-tag', - RUNNABLE_FILES: '{ "Dockerfile": "AolcUvaTfKOFJg74ABqL9NN08333MS_t" }', - RUNNABLE_FILES_BUCKET: 'runnable.image-builder' - }; - lab.beforeEach(function (done) { - Object.keys(requiredEnvVars).forEach( - function (key) { process.env[key] = requiredEnvVars[key]; }); - done(); - }); - lab.experiment('succeeds', function () { - lab.beforeEach(steps.makeWorkingFolders.bind(steps)); - - lab.experiment('with runnable-cache', function () { - lab.beforeEach({ timeout: 5000 }, steps.downloadBuildFiles.bind(steps)); - lab.beforeEach(steps.runDockerBuild.bind(steps)); - lab.test('should catch the cached layer', function (done) { - steps.parseBuildLogAndHistory(function (err) { - if (err) { return done(err); } - expect(steps.data.cacheThisLayer).to.be.match(/^[a-f0-9]+$/); - done(); - }); - }); - }); - lab.experiment('with funky runnable-cache', function () { - lab.beforeEach(function (done) { - process.env.RUNNABLE_FILES = - '{ "Dockerfile": "KKUneazEu5iFAJAkfOIHe0C2jaeGgZpn" }'; - done(); - }); - lab.beforeEach({ timeout: 5000 }, steps.downloadBuildFiles.bind(steps)); - lab.beforeEach(steps.runDockerBuild.bind(steps)); - lab.test('should catch the cached layer', function (done) { - steps.parseBuildLogAndHistory(function (err) { - if (err) { return done(err); } - expect(steps.data.cacheThisLayer).to.be.match(/^[a-f0-9]+$/); - done(); - }); - }); - }); - }); -}); diff --git a/test/steps/checkForRequiredEnvVars.js b/test/steps/checkForRequiredEnvVars.js new file mode 100644 index 0000000..23f610b --- /dev/null +++ b/test/steps/checkForRequiredEnvVars.js @@ -0,0 +1,52 @@ +'use strict'; + +var Lab = require('lab'); +var lab = exports.lab = Lab.script(); +var expect = require('code').expect; + +var cacheDir = process.env.CACHE_DIR; +if (!cacheDir) { + cacheDir = process.env.CACHE_DIR = '/tmp/cache'; +} +var layerCacheDir = process.env.LAYER_CACHE_DIR; +if (!layerCacheDir) { + layerCacheDir = process.env.LAYER_CACHE_DIR = '/tmp/layer-cache'; +} +// require this after we have now changed the env for the directories +var steps = require('../../lib/steps'); + +var requiredEnvVars = { + RUNNABLE_AWS_ACCESS_KEY: process.env.AWS_ACCESS_KEY, + RUNNABLE_AWS_SECRET_KEY: process.env.AWS_SECRET_KEY +}; +lab.beforeEach(function (done) { + Object.keys(requiredEnvVars).forEach(function (key) { + process.env[key] = requiredEnvVars[key]; + }); + done(); +}); + +lab.experiment('checkForRequiredEnvVars', function () { + lab.experiment('fails', function () { + lab.beforeEach(function (done) { + Object.keys(requiredEnvVars).forEach( + function (key) { delete process.env[key]; }); + done(); + }); + lab.test('when required env vars are missing', function (done) { + steps.checkForRequiredEnvVars(function (err) { + expect(!!err).to.be.true(); + expect(err.message).to.match(/Missing credentials./); + done(null); + }); + }); + }); + lab.experiment('succeeds', function () { + lab.test('when all env vars are present', function (done) { + steps.checkForRequiredEnvVars(function (err) { + if (err) { return done(err); } + done(err); + }); + }); + }); +}); diff --git a/test/steps/chmodAllKeys.js b/test/steps/chmodAllKeys.js new file mode 100644 index 0000000..2b44ad4 --- /dev/null +++ b/test/steps/chmodAllKeys.js @@ -0,0 +1,129 @@ +'use strict'; + +var Lab = require('lab'); +var lab = exports.lab = Lab.script(); +var expect = require('code').expect; + +var childProcess = require('child_process'); +var path = require('path'); +var fs = require('fs'); +var sinon = require('sinon'); + +var cacheDir = process.env.CACHE_DIR; +if (!cacheDir) { + cacheDir = process.env.CACHE_DIR = '/tmp/cache'; +} +var layerCacheDir = process.env.LAYER_CACHE_DIR; +if (!layerCacheDir) { + layerCacheDir = process.env.LAYER_CACHE_DIR = '/tmp/layer-cache'; +} +// require this after we have now changed the env for the directories +var steps = require('../../lib/steps'); + +var requiredEnvVars = { + RUNNABLE_AWS_ACCESS_KEY: process.env.AWS_ACCESS_KEY, + RUNNABLE_AWS_SECRET_KEY: process.env.AWS_SECRET_KEY +}; +lab.beforeEach(function (done) { + Object.keys(requiredEnvVars).forEach(function (key) { + process.env[key] = requiredEnvVars[key]; + }); + done(); +}); + +lab.experiment('chmodAllKeys', function () { + var requiredEnvVars = { + RUNNABLE_KEYS_BUCKET: 'runnable.image-builder', + RUNNABLE_DEPLOYKEY: 'flaming-octo-nemesis.key' + }; + lab.beforeEach(function (done) { + Object.keys(requiredEnvVars).forEach( + function (key) { process.env[key] = requiredEnvVars[key]; }); + done(); + }); + lab.beforeEach(steps.makeWorkingFolders.bind(steps)); + + lab.experiment('succeeds', function () { + // provide options to check that exec was called + lab.beforeEach(function (done) { + sinon.spy(childProcess, 'exec'); + done(); + }); + lab.afterEach(function (done) { + childProcess.exec.restore(); + done(); + }); + + lab.experiment('when there are keys', function () { + lab.beforeEach({ timeout: 5000 }, steps.downloadDeployKeys.bind(steps)); + lab.beforeEach(function (done) { childProcess.exec.reset(); done(); }); + + lab.test('to set the permissions on the keys', function (done) { + steps.chmodAllKeys(function (err) { + if (err) { return done(err); } + expect(childProcess.exec.calledOnce).to.be.true(); + expect(childProcess.exec.calledWith('chmod -R 600 *')).to.be.true(); + var keyPath = path.join( + steps.dirs.keyDirectory, + requiredEnvVars.RUNNABLE_DEPLOYKEY); + fs.stat(keyPath, function (err, stats) { + if (err) { return done(err); } + expect(stats.mode).to.equal(33152); + done(); + }); + }); + }); + }); + + lab.experiment('when there are no keys', function () { + var oldDeployKey; + lab.beforeEach(function (done) { + oldDeployKey = process.env.RUNNABLE_DEPLOYKEY; + delete process.env.RUNNABLE_DEPLOYKEY; + done(); + }); + lab.afterEach(function (done) { + process.env.RUNNABLE_DEPLOYKEY = oldDeployKey; + done(); + }); + lab.beforeEach(function (done) { childProcess.exec.reset(); done(); }); + + lab.test('to just move on', function (done) { + steps.chmodAllKeys(function (err) { + if (err) { return done(err); } + expect(childProcess.exec.callCount).to.equal(0); + done(); + }); + }); + }); + }); + + lab.experiment('fails', function () { + lab.beforeEach({ timeout: 5000 }, steps.downloadDeployKeys.bind(steps)); + + lab.experiment('when the exec call fails', function () { + lab.beforeEach(function (done) { + sinon.stub(childProcess, 'exec', function () { + var cb = Array.prototype.slice.call(arguments).pop(); + cb( + new Error('Command failed'), + '', + 'chmod: cannot access ‘*’: No such file or directory\n'); + }); + done(); + }); + lab.afterEach(function (done) { + childProcess.exec.restore(); + done(); + }); + + lab.it('should return an error', function (done) { + steps.chmodAllKeys(function (err) { + expect(require('child_process').exec.calledOnce).to.be.true(); + expect(err).to.exist(); + done(); + }); + }); + }); + }); +}); diff --git a/test/steps/downloadBuildFiles.js b/test/steps/downloadBuildFiles.js new file mode 100644 index 0000000..9a31cc9 --- /dev/null +++ b/test/steps/downloadBuildFiles.js @@ -0,0 +1,158 @@ +'use strict'; + +var Lab = require('lab'); +var lab = exports.lab = Lab.script(); +var expect = require('code').expect; + +var path = require('path'); +var fs = require('fs'); + +var cacheDir = process.env.CACHE_DIR; +if (!cacheDir) { + cacheDir = process.env.CACHE_DIR = '/tmp/cache'; +} +var layerCacheDir = process.env.LAYER_CACHE_DIR; +if (!layerCacheDir) { + layerCacheDir = process.env.LAYER_CACHE_DIR = '/tmp/layer-cache'; +} +// require this after we have now changed the env for the directories +var steps = require('../../lib/steps'); + +var requiredEnvVars = { + RUNNABLE_AWS_ACCESS_KEY: process.env.AWS_ACCESS_KEY, + RUNNABLE_AWS_SECRET_KEY: process.env.AWS_SECRET_KEY +}; +lab.before(function (done) { + Object.keys(requiredEnvVars).forEach(function (key) { + process.env[key] = requiredEnvVars[key]; + }); + done(); +}); + +lab.experiment('downloadBuildFiles', function () { + var requiredEnvVars = { + RUNNABLE_FILES: '{ "Dockerfile": "K6cluDupwQdFRsuTPJ0SFUrxUB4lmF_Q" }', + RUNNABLE_FILES_BUCKET: 'runnable.image-builder', + RUNNABLE_PREFIX: '' + }; + lab.beforeEach(function (done) { + Object.keys(requiredEnvVars).forEach( + function (key) { process.env[key] = requiredEnvVars[key]; }); + done(); + }); + lab.before(steps.makeWorkingFolders.bind(steps)); + + lab.experiment('fails', function () { + lab.beforeEach(function (done) { + delete process.env.RUNNABLE_FILES_BUCKET; + done(); + }); + + lab.test('when the required env vars are missing', function (done) { + steps.downloadBuildFiles(function (err) { + expect(!!err).to.be.true(); + expect(err.message).to.match(/RUNNABLE_FILES_BUCKET is missing/); + done(); + }); + }); + }); + + lab.experiment('succeeds', function () { + + lab.experiment('if prefix is missing', function () { + lab.beforeEach(function (done) { + delete process.env.RUNNABLE_PREFIX; + done(); + }); + lab.afterEach(function (done) { + process.env.RUNNABLE_PREFIX = requiredEnvVars.RUNNABLE_PREFIX; + done(); + }); + + lab.test('it should be fine', function (done) { + steps.downloadBuildFiles(function (err) { + if (err) { return done(err); } + // TODO check directory + done(); + }); + }); + }); + + lab.experiment('using a prefix', function () { + lab.beforeEach(function (done) { + process.env.RUNNABLE_FILES = + '{ "test-prefix/Dockerfile": "VHSTXUoj_1n9970ysq69lu6V6owCzarr" }'; + process.env.RUNNABLE_PREFIX = 'test-prefix/'; + done(); + }); + lab.afterEach(function (done) { + process.env.RUNNABLE_PREFIX = requiredEnvVars.RUNNABLE_PREFIX; + process.env.RUNNABLE_FILES = requiredEnvVars.RUNNABLE_FILES; + done(); + }); + + lab.it('should remove the prefix', function (done) { + steps.downloadBuildFiles(function (err) { + if (err) { return done(err); } + // TODO check for empty directory + done(); + }); + }); + }); + + lab.experiment('if there are no files', function () { + lab.beforeEach(function (done) { + delete process.env.RUNNABLE_FILES; + done(); + }); + lab.afterEach(function (done) { + process.env.RUNNABLE_FILES = requiredEnvVars.RUNNABLE_FILES; + done(); + }); + + lab.test('it should be fine', function (done) { + steps.downloadBuildFiles(function (err) { + if (err) { return done(err); } + // TODO check for empty directory + done(); + }); + }); + }); + + lab.experiment('with files to download', function () { + + lab.test('to download the files', function (done) { + steps.downloadBuildFiles(function (err) { + if (err) { return done(err); } + var dockerfilePath = path.join( + steps.dirs.dockerContext, + 'Dockerfile'); + expect(fs.existsSync(dockerfilePath)).to.be.true(); + done(); + }); + }); + }); + }); + + lab.experiment('fails', function () { + + lab.experiment('when the files are not formatted well', function () { + lab.beforeEach(function (done) { + process.env.RUNNABLE_FILES ='{ "Dockerfile": \'versionid\' }'; + done(); + }); + lab.afterEach(function (done) { + process.env.RUNNABLE_FILES = requiredEnvVars.RUNNABLE_FILES; + done(); + }); + + lab.it('should throw an error', function (done) { + steps.downloadBuildFiles(function (err) { + expect(err).to.exist(); + expect(err.message).to.match(/poorly formatted/); + done(); + }); + }); + }); + }); +}); diff --git a/test/steps/downloadDeployKeys.js b/test/steps/downloadDeployKeys.js new file mode 100644 index 0000000..d8f8260 --- /dev/null +++ b/test/steps/downloadDeployKeys.js @@ -0,0 +1,93 @@ +'use strict'; + +var Lab = require('lab'); +var lab = exports.lab = Lab.script(); +var expect = require('code').expect; + +var path = require('path'); +var fs = require('fs'); + +var cacheDir = process.env.CACHE_DIR; +if (!cacheDir) { + cacheDir = process.env.CACHE_DIR = '/tmp/cache'; +} +var layerCacheDir = process.env.LAYER_CACHE_DIR; +if (!layerCacheDir) { + layerCacheDir = process.env.LAYER_CACHE_DIR = '/tmp/layer-cache'; +} +// require this after we have now changed the env for the directories +var steps = require('../../lib/steps'); + +var requiredEnvVars = { + RUNNABLE_AWS_ACCESS_KEY: process.env.AWS_ACCESS_KEY, + RUNNABLE_AWS_SECRET_KEY: process.env.AWS_SECRET_KEY +}; +lab.beforeEach(function (done) { + Object.keys(requiredEnvVars).forEach(function (key) { + process.env[key] = requiredEnvVars[key]; + }); + done(); +}); + +lab.experiment('downloadDeployKeys', function () { + var requiredEnvVars = { + RUNNABLE_KEYS_BUCKET: 'runnable.image-builder', + RUNNABLE_DEPLOYKEY: 'flaming-octo-nemesis.key' + }; + lab.beforeEach(function (done) { + Object.keys(requiredEnvVars).forEach( + function (key) { process.env[key] = requiredEnvVars[key]; }); + done(); + }); + lab.beforeEach(steps.makeWorkingFolders.bind(steps)); + + lab.experiment('fails', function () { + lab.beforeEach(function (done) { + delete process.env.RUNNABLE_KEYS_BUCKET; + done(); + }); + + lab.test('when the required env vars are missing', function (done) { + steps.downloadDeployKeys(function (err) { + expect(!!err).to.be.true(); + expect(err.message).to.match(/RUNNABLE_KEYS_BUCKET is missing/); + done(); + }); + }); + }); + lab.experiment('succeeds', function () { + + lab.experiment('if there are no keys', function () { + lab.beforeEach(function (done) { + delete process.env.RUNNABLE_DEPLOYKEY; + done(); + }); + lab.afterEach(function (done) { + process.env.RUNNABLE_DEPLOYKEY = requiredEnvVars.RUNNABLE_DEPLOYKEY; + done(); + }); + + lab.test('it should be fine', function (done) { + steps.downloadDeployKeys(function (err) { + if (err) { return done(err); } + // TODO check for empty directory + done(); + }); + }); + }); + + lab.experiment('with keys to download', function () { + + lab.test('to download the keys', { timeout: 5000 }, function (done) { + steps.downloadDeployKeys(function (err) { + if (err) { return done(err); } + var keyPath = path.join( + steps.dirs.keyDirectory, + requiredEnvVars.RUNNABLE_DEPLOYKEY); + expect(fs.existsSync(keyPath)).to.be.true(); + done(); + }); + }); + }); + }); +}); diff --git a/test/steps/getRepositories.js b/test/steps/getRepositories.js new file mode 100644 index 0000000..709a149 --- /dev/null +++ b/test/steps/getRepositories.js @@ -0,0 +1,318 @@ +'use strict'; + +var Lab = require('lab'); +var lab = exports.lab = Lab.script(); +var expect = require('code').expect; + +var childProcess = require('child_process'); +var path = require('path'); +var fs = require('fs'); +var sinon = require('sinon'); + +var lockfile = require('lockfile'); + +var cacheDir = process.env.CACHE_DIR; +if (!cacheDir) { + cacheDir = process.env.CACHE_DIR = '/tmp/cache'; +} +var layerCacheDir = process.env.LAYER_CACHE_DIR; +if (!layerCacheDir) { + layerCacheDir = process.env.LAYER_CACHE_DIR = '/tmp/layer-cache'; +} +// require this after we have now changed the env for the directories +var steps = require('../../lib/steps'); + +var requiredEnvVars = { + RUNNABLE_AWS_ACCESS_KEY: process.env.AWS_ACCESS_KEY, + RUNNABLE_AWS_SECRET_KEY: process.env.AWS_SECRET_KEY +}; +lab.before(function (done) { + Object.keys(requiredEnvVars).forEach(function (key) { + process.env[key] = requiredEnvVars[key]; + }); + done(); +}); + +lab.experiment('getRepositories', function () { + var requiredEnvVars = { + RUNNABLE_REPO: 'git@github.com:bkendall/flaming-octo-nemesis', + RUNNABLE_COMMITISH: '34a728c59e713b7fbf5b0d6ed3a8e4f4e2c695c5', + RUNNABLE_KEYS_BUCKET: 'runnable.image-builder', + RUNNABLE_DEPLOYKEY: 'flaming-octo-nemesis.key' + }; + lab.beforeEach(function (done) { + Object.keys(requiredEnvVars).forEach( + function (key) { process.env[key] = requiredEnvVars[key]; }); + done(); + }); + lab.beforeEach(function (done) { + childProcess.exec('rm -rf ' + cacheDir + '/*', done); + }); + lab.beforeEach(steps.makeWorkingFolders.bind(steps)); + lab.beforeEach({ timeout: 5000 }, steps.downloadDeployKeys.bind(steps)); + lab.beforeEach(steps.chmodAllKeys.bind(steps)); + + lab.experiment('succeeds', function () { + + lab.experiment('when there is a repo', function () { + /* github can be slow to respond. long timeout */ + lab.test('to download the repo', { timeout: 10000 }, function (done) { + steps.getRepositories(function (err) { + if (err) { return done(err); } + var repoCacheDir = path.join( + cacheDir, + 'bkendall/flaming-octo-nemesis'); + var repoTargetDir = path.join( + steps.dirs.dockerContext, + 'flaming-octo-nemesis'); + expect(fs.existsSync(repoCacheDir)).to.be.true(); + expect(fs.existsSync(repoTargetDir)).to.be.true(); + done(); + }); + }); + }); + + lab.experiment('when there is no repo', function () { + lab.beforeEach(function (done) { + delete process.env.RUNNABLE_REPO; + done(); + }); + lab.afterEach(function (done) { + process.env.RUNNABLE_REPO = requiredEnvVars.RUNNABLE_REPO; + done(); + }); + + lab.it('just continues', function (done) { + steps.getRepositories(function (err) { + if (err) { return done(err); } + // TODO look for no repos + done(); + }); + }); + }); + + lab.experiment('when a lock already exists for the repo', function () { + lab.beforeEach(function (done) { + var cmds = [ + 'mkdir', + '-p', + cacheDir + '/bkendall', + '&& touch', + cacheDir + '/bkendall/flaming-octo-nemesis.lock' + ].join(' '); + childProcess.exec(cmds, done); + }); + + /* github can be slow to respond. long timeout */ + lab.test('to download the repo', { timeout: 10000 }, function (done) { + steps.getRepositories(function (err) { + if (err) { return done(err); } + var repoCacheDir = path.join( + cacheDir, + 'bkendall/flaming-octo-nemesis'); + var repoCacheDirLock = path.join( + cacheDir, + 'bkendall/flaming-octo-nemesis.lock'); + var repoTargetDir = path.join( + steps.dirs.dockerContext, + 'flaming-octo-nemesis'); + expect(fs.existsSync(repoCacheDir)).to.be.false(); + expect(fs.existsSync(repoCacheDirLock)).to.be.true(); + expect(fs.existsSync(repoTargetDir)).to.be.true(); + done(); + }); + }); + }); + + lab.experiment('when the repo has already been cached', function () { + lab.beforeEach({ timeout: 10000 }, function (done) { + var cmds = [ + 'mkdir', + '-p', + cacheDir + '/bkendall', + '&& git clone https://github.com/bkendall/flaming-octo-nemesis ' + + cacheDir + '/bkendall/flaming-octo-nemesis' + ].join(' '); + childProcess.exec(cmds, done); + }); + + /* github can be slow to respond. long timeout */ + lab.test('to still complete', { timeout: 10000 }, function (done) { + steps.getRepositories(function (err) { + if (err) { return done(err); } + var repoCacheDir = path.join( + cacheDir, + 'bkendall/flaming-octo-nemesis'); + var repoGitDir = path.join( + cacheDir, + 'bkendall/flaming-octo-nemesis', + '.git'); + var repoCacheDirLock = path.join( + cacheDir, + 'bkendall/flaming-octo-nemesis.lock'); + var repoTargetDir = path.join( + steps.dirs.dockerContext, + 'flaming-octo-nemesis'); + expect(fs.existsSync(repoCacheDir)).to.be.true(); + expect(fs.existsSync(repoGitDir)).to.be.true(); + expect(fs.existsSync(repoCacheDirLock)).to.be.false(); + expect(fs.existsSync(repoTargetDir)).to.be.true(); + done(); + }); + }); + + lab.experiment('but needing to be updated', function () { + lab.beforeEach(function (done) { + var cmd = 'git checkout 04d07787dd44b4f2167e26532e95471871a9b233'; + var cwd = cacheDir + '/bkendall/flaming-octo-nemesis'; + childProcess.exec(cmd, { cwd: cwd }, done); + }); + lab.beforeEach(function (done) { + sinon.spy(childProcess, 'exec'); + done(); + }); + lab.afterEach(function (done) { + childProcess.exec.restore(); + done(); + }); + + lab.test('to updated and complete', function (done) { + steps.getRepositories(function (err) { + if (err) { return done(err); } + expect(childProcess.exec.calledWith('git fetch --all')) + .to.be.true(); + expect( + childProcess.exec.calledWith('git checkout -q ' + + '34a728c59e713b7fbf5b0d6ed3a8e4f4e2c695c5')) + .to.be.true(); + done(); + }); + }); + }); + }); + }); + + lab.experiment('fails', function () { + + lab.experiment('when there is a repo, but no commitish', function () { + lab.beforeEach(function (done) { + delete process.env.RUNNABLE_COMMITISH; + done(); + }); + lab.afterEach(function (done) { + process.env.RUNNABLE_COMMITISH = requiredEnvVars.RUNNABLE_COMMITISH; + done(); + }); + + lab.it('just continues', function (done) { + steps.getRepositories(function (err) { + expect(err).to.exist(); + expect(err.message).to.match(/COMMITISH is missing/); + done(); + }); + }); + }); + + lab.experiment('to make sure the cache directory exists', function () { + lab.beforeEach(function (done) { + sinon.stub(childProcess, 'exec') + .withArgs('mkdir -p /tmp/cache') + .callsArgWith( + 1, + new Error('Command failed'), + '', + ''); + done(); + }); + lab.afterEach(function (done) { + childProcess.exec.restore(); + done(); + }); + + lab.it('returns an error', function (done) { + steps.getRepositories(function (err) { + expect(err).to.exist(); + expect(err.message).to.match(/Command failed/); + done(); + }); + }); + }); + + lab.experiment('to remove all the ssh keys from agent', function () { + lab.beforeEach(function (done) { + sinon.stub(childProcess, 'exec'); + childProcess.exec + .withArgs('ssh-add -D') + .callsArgWith( + 1, + new Error('Command failed'), + '', + ''); + childProcess.exec.yields(null, '', ''); + done(); + }); + lab.afterEach(function (done) { + childProcess.exec.restore(); + done(); + }); + + lab.it('returns an error', function (done) { + steps.getRepositories(function (err) { + expect(err).to.exist(); + expect(err.message).to.match(/Command failed/); + done(); + }); + }); + }); + + lab.experiment('to make the lock file directory', function () { + lab.beforeEach(function (done) { + sinon.stub(childProcess, 'exec'); + childProcess.exec + .withArgs('mkdir -p /tmp/cache/bkendall') + .callsArgWith( + 1, + new Error('Command failed'), + '', + ''); + childProcess.exec.yields(null, '', ''); + done(); + }); + lab.afterEach(function (done) { + childProcess.exec.restore(); + done(); + }); + + lab.it('returns an error', function (done) { + steps.getRepositories(function (err) { + expect(err).to.exist(); + expect(err.message).to.match(/Command failed/); + done(); + }); + }); + }); + + lab.experiment('to release the lock', function () { + lab.beforeEach(function (done) { + sinon.stub(lockfile, 'unlock'); + lockfile.unlock + .callsArgWith(1, new Error('could not unlock')); + done(); + }); + lab.afterEach(function (done) { + lockfile.unlock.restore(); + done(); + }); + + lab.it('returns an error', { timeout: 10000 }, function (done) { + steps.getRepositories(function (err) { + expect(err).to.exist(); + expect(err.message).to.match(/could not unlock/); + // should have tried to unlock it twice + expect(lockfile.unlock.callCount).to.equal(2); + done(); + }); + }); + }); + }); +}); diff --git a/test/steps/makeWorkingFolders.js b/test/steps/makeWorkingFolders.js new file mode 100644 index 0000000..b23ea0b --- /dev/null +++ b/test/steps/makeWorkingFolders.js @@ -0,0 +1,122 @@ +'use strict'; + +var Lab = require('lab'); +var lab = exports.lab = Lab.script(); +var expect = require('code').expect; + +var childProcess = require('child_process'); +var fs = require('fs'); +var sinon = require('sinon'); + +var cacheDir = process.env.CACHE_DIR; +if (!cacheDir) { + cacheDir = process.env.CACHE_DIR = '/tmp/cache'; +} +var layerCacheDir = process.env.LAYER_CACHE_DIR; +if (!layerCacheDir) { + layerCacheDir = process.env.LAYER_CACHE_DIR = '/tmp/layer-cache'; +} +// require this after we have now changed the env for the directories +var steps = require('../../lib/steps'); + +var requiredEnvVars = { + RUNNABLE_AWS_ACCESS_KEY: process.env.AWS_ACCESS_KEY, + RUNNABLE_AWS_SECRET_KEY: process.env.AWS_SECRET_KEY +}; +lab.beforeEach(function (done) { + Object.keys(requiredEnvVars).forEach(function (key) { + process.env[key] = requiredEnvVars[key]; + }); + done(); +}); + +lab.experiment('makeWorkingFolders', function () { + + lab.experiment('succeeds', function () { + lab.beforeEach(function (done) { + sinon.spy(childProcess, 'exec'); + done(); + }); + lab.afterEach(function (done) { + childProcess.exec.restore(); + done(); + }); + + var createdFolders = [ + 'dockerContext', + 'keyDirectory' + ]; + lab.test('to create all folders', function (done) { + steps.makeWorkingFolders(function (err) { + if (err) { return done(err); } + expect(childProcess.exec.callCount).to.equal(5); + createdFolders.forEach(function (dirName) { + expect(steps.dirs[dirName]).to.not.be.undefined(); + expect(fs.existsSync(steps.dirs[dirName])).to.be.true(); + }); + done(); + }); + }); + }); + + lab.experiment('fails', function () { + + lab.experiment('to create a folder', function () { + lab.beforeEach(function (done) { + sinon.stub(childProcess, 'exec', function () { + var args = Array.prototype.slice.call(arguments); + var cb = args.pop(); + if (/mktemp \-d/.test(args[0])) { + cb( + new Error('Command failed'), + '', + ''); + } else { + cb(null, '', ''); + } + }); + done(); + }); + lab.afterEach(function (done) { + childProcess.exec.restore(); + done(); + }); + + lab.test('on an exec call', function (done) { + steps.makeWorkingFolders(function (err) { + expect(err).to.exist(); + done(); + }); + }); + }); + + lab.experiment('to create a file', function () { + lab.beforeEach(function (done) { + sinon.stub(childProcess, 'exec', function () { + var args = Array.prototype.slice.call(arguments); + var cb = args.pop(); + if (/mktemp \//.test(args[0])) { + cb( + new Error('Command failed'), + '', + ''); + } else { + cb(null, '', ''); + } + }); + done(); + }); + lab.afterEach(function (done) { + childProcess.exec.restore(); + done(); + }); + + lab.test('on an exec call', function (done) { + steps.makeWorkingFolders(function (err) { + expect(err).to.exist(); + done(); + }); + }); + }); + }); +}); diff --git a/test/steps/parseBuildLogAndHistory.js b/test/steps/parseBuildLogAndHistory.js new file mode 100644 index 0000000..981e5a6 --- /dev/null +++ b/test/steps/parseBuildLogAndHistory.js @@ -0,0 +1,124 @@ +'use strict'; + +var Lab = require('lab'); +var lab = exports.lab = Lab.script(); +var expect = require('code').expect; + +var childProcess = require('child_process'); +var sinon = require('sinon'); + +var dockerMock = require('docker-mock'); + +var cacheDir = process.env.CACHE_DIR; +if (!cacheDir) { + cacheDir = process.env.CACHE_DIR = '/tmp/cache'; +} +var layerCacheDir = process.env.LAYER_CACHE_DIR; +if (!layerCacheDir) { + layerCacheDir = process.env.LAYER_CACHE_DIR = '/tmp/layer-cache'; +} +// require this after we have now changed the env for the directories +var steps = require('../../lib/steps'); + +var requiredEnvVars = { + RUNNABLE_AWS_ACCESS_KEY: process.env.AWS_ACCESS_KEY, + RUNNABLE_AWS_SECRET_KEY: process.env.AWS_SECRET_KEY +}; +lab.before(function (done) { + Object.keys(requiredEnvVars).forEach(function (key) { + process.env[key] = requiredEnvVars[key]; + }); + done(); +}); + +lab.experiment('parseBuildLogAndHistory', function () { + var dockerMockServer; + lab.before(function (done) { + dockerMockServer = dockerMock.listen(5555, done); + }); + lab.after(function (done) { + dockerMockServer.close(done); + }); + var requiredEnvVars = { + RUNNABLE_DOCKER: 'tcp://localhost:5555', + RUNNABLE_DOCKERTAG: 'test-docker-tag', + RUNNABLE_FILES: '{ "Dockerfile": "AolcUvaTfKOFJg74ABqL9NN08333MS_t" }', + RUNNABLE_FILES_BUCKET: 'runnable.image-builder' + }; + lab.beforeEach(function (done) { + Object.keys(requiredEnvVars).forEach( + function (key) { process.env[key] = requiredEnvVars[key]; }); + done(); + }); + + lab.experiment('succeeds', function () { + lab.beforeEach(steps.makeWorkingFolders.bind(steps)); + + lab.experiment('with runnable-cache', function () { + lab.beforeEach({ timeout: 5000 }, steps.downloadBuildFiles.bind(steps)); + lab.beforeEach(steps.runDockerBuild.bind(steps)); + + lab.test('should catch the cached layer', function (done) { + steps.parseBuildLogAndHistory(function (err) { + if (err) { return done(err); } + expect(steps.data.cacheThisLayer).to.be.match(/^[a-f0-9]+$/); + done(); + }); + }); + }); + + lab.experiment('with funky runnable-cache', function () { + lab.beforeEach(function (done) { + process.env.RUNNABLE_FILES = + '{ "Dockerfile": "KKUneazEu5iFAJAkfOIHe0C2jaeGgZpn" }'; + done(); + }); + lab.beforeEach({ timeout: 5000 }, steps.downloadBuildFiles.bind(steps)); + lab.beforeEach(steps.runDockerBuild.bind(steps)); + + lab.test('should catch the cached layer', function (done) { + steps.parseBuildLogAndHistory(function (err) { + if (err) { return done(err); } + expect(steps.data.cacheThisLayer).to.be.match(/^[a-f0-9]+$/); + done(); + }); + }); + }); + + lab.experiment('with an available layer cache', function () { + lab.beforeEach(function (done) { + var cmds = [ + 'mkdir -p /tmp/layer-cache/test-docker-tag', + 'touch /tmp/layer-cache/test-docker-tag/' + + 'hash.93f7657e7c42734aac70d134cecf53d3', + 'touch /tmp/layer-cache/test-docker-tag/layer.tar' + ].join(' && '); + childProcess.exec(cmds, done); + }); + lab.beforeEach({ timeout: 5000 }, steps.downloadBuildFiles.bind(steps)); + lab.beforeEach(steps.parseDockerfile.bind(steps)); + lab.beforeEach(steps.runDockerBuild.bind(steps)); + lab.beforeEach(function (done) { + expect(!!steps.data.usingCache).to.be.true(); + done(); + }); + lab.beforeEach(function (done) { + sinon.spy(childProcess, 'exec'); + done(); + }); + lab.afterEach(function (done) { + childProcess.exec.restore(); + done(); + }); + + lab.test('should just skip through if using cache', function (done) { + steps.parseBuildLogAndHistory(function (err) { + if (err) { return done(err); } + // shouldn't call to docker to get the layer + expect(childProcess.exec.callCount).to.equal(0); + done(); + }); + }); + }); + }); +}); diff --git a/test/steps/parseDockerfile.js b/test/steps/parseDockerfile.js new file mode 100644 index 0000000..ba81d9d --- /dev/null +++ b/test/steps/parseDockerfile.js @@ -0,0 +1,172 @@ +'use strict'; + +var Lab = require('lab'); +var lab = exports.lab = Lab.script(); +var expect = require('code').expect; + +var childProcess = require('child_process'); +var sinon = require('sinon'); + +var cacheDir = process.env.CACHE_DIR; +if (!cacheDir) { + cacheDir = process.env.CACHE_DIR = '/tmp/cache'; +} +var layerCacheDir = process.env.LAYER_CACHE_DIR; +if (!layerCacheDir) { + layerCacheDir = process.env.LAYER_CACHE_DIR = '/tmp/layer-cache'; +} +// require this after we have now changed the env for the directories +var steps = require('../../lib/steps'); + +var requiredEnvVars = { + RUNNABLE_AWS_ACCESS_KEY: process.env.AWS_ACCESS_KEY, + RUNNABLE_AWS_SECRET_KEY: process.env.AWS_SECRET_KEY +}; +lab.before(function (done) { + Object.keys(requiredEnvVars).forEach(function (key) { + process.env[key] = requiredEnvVars[key]; + }); + done(); +}); + +lab.experiment('parseDockerfile', function () { + var requiredEnvVars = { + RUNNABLE_DOCKERTAG: 'test-docker-tag', + RUNNABLE_FILES: '{ "Dockerfile": "AolcUvaTfKOFJg74ABqL9NN08333MS_t" }', + RUNNABLE_FILES_BUCKET: 'runnable.image-builder' + }; + lab.beforeEach(function (done) { + Object.keys(requiredEnvVars).forEach( + function (key) { process.env[key] = requiredEnvVars[key]; }); + done(); + }); + lab.beforeEach(function (done) { + // have to do this to keep the steps.data bit clean between tests + Object.keys(steps.data).forEach(function (key) { + delete steps.data[key]; + }); + done(); + }); + lab.beforeEach(function (done) { + childProcess.exec('rm -rf /tmp/layer-cache/*', done); + }); + + lab.experiment('succeeds', function () { + lab.beforeEach(steps.makeWorkingFolders.bind(steps)); + + lab.experiment('with runnable-cache', function () { + lab.beforeEach({ timeout: 5000 }, steps.downloadBuildFiles.bind(steps)); + + lab.test('should catch the cache line', function (done) { + steps.parseDockerfile(function (err) { + if (err) { return done(err); } + expect(!!steps.data.usingCache).to.be.false(); + expect(steps.data.cachedLine).to.not.be.undefined(); + expect(steps.data.createdByHash).to.not.be.undefined(); + done(); + }); + }); + }); + + lab.experiment('with funky Runnable-cache', function () { + lab.beforeEach(function (done) { + process.env.RUNNABLE_FILES = + '{ "Dockerfile": "KKUneazEu5iFAJAkfOIHe0C2jaeGgZpn" }'; + done(); + }); + lab.afterEach(function (done) { + process.env.RUNNABLE_FILES = requiredEnvVars.RUNNABLE_FILES; + done(); + }); + lab.beforeEach({ timeout: 5000 }, steps.downloadBuildFiles.bind(steps)); + lab.beforeEach(function (done) { + sinon.spy(childProcess, 'exec'); + done(); + }); + lab.afterEach(function (done) { + childProcess.exec.restore(); + done(); + }); + + lab.test('should find the cached line', function (done) { + steps.parseDockerfile(function (err) { + if (err) { return done(err); } + // not copying a cache + expect(childProcess.exec.callCount).to.equal(0); + expect(!!steps.data.usingCache).to.be.false(); + expect(steps.data.cachedLine).to.not.be.undefined(); + expect(steps.data.createdByHash).to.not.be.undefined(); + done(); + }); + }); + }); + + lab.experiment('with no runnable-cache', function () { + lab.beforeEach(function (done) { + process.env.RUNNABLE_FILES = + '{ "test-prefix/Dockerfile": "ir8FJ0g6CH9P608k4O0lscYNuAz6Yt5q" }'; + process.env.RUNNABLE_PREFIX = 'test-prefix/'; + done(); + }); + lab.afterEach(function (done) { + process.env.RUNNABLE_FILES = requiredEnvVars.RUNNABLE_FILES; + delete process.env.RUNNABLE_PREFIX; + done(); + }); + lab.beforeEach({ timeout: 5000 }, steps.downloadBuildFiles.bind(steps)); + lab.beforeEach(function (done) { + sinon.spy(childProcess, 'exec'); + done(); + }); + lab.afterEach(function (done) { + childProcess.exec.restore(); + done(); + }); + + lab.test('should not do any caching', function (done) { + steps.parseDockerfile(function (err) { + if (err) { return done(err); } + // not copying a cache + expect(childProcess.exec.callCount).to.equal(0); + expect(!!steps.data.usingCache).to.be.false(); + expect(steps.data.cachedLine).to.be.undefined(); + expect(steps.data.createdByHash).to.be.undefined(); + done(); + }); + }); + }); + + lab.experiment('with an available layer cache', function () { + lab.beforeEach(function (done) { + var cmds = [ + 'mkdir -p /tmp/layer-cache/test-docker-tag', + 'touch /tmp/layer-cache/test-docker-tag/' + + 'hash.93f7657e7c42734aac70d134cecf53d3', + 'touch /tmp/layer-cache/test-docker-tag/layer.tar' + ].join(' && '); + childProcess.exec(cmds, done); + }); + lab.beforeEach({ timeout: 5000 }, steps.downloadBuildFiles.bind(steps)); + lab.beforeEach(function (done) { + sinon.spy(childProcess, 'exec'); + done(); + }); + lab.afterEach(function (done) { + childProcess.exec.restore(); + done(); + }); + + lab.test('should not do any caching', function (done) { + steps.parseDockerfile(function (err) { + if (err) { return done(err); } + expect(!!steps.data.usingCache).to.be.true(); + expect(steps.data.cachedLine).to.not.be.undefined(); + expect(steps.data.createdByHash).to.not.be.undefined(); + // using a cache! + expect(childProcess.exec.callCount).to.equal(1); + done(); + }); + }); + }); + }); +}); diff --git a/test/steps/runDockerBuild.js b/test/steps/runDockerBuild.js new file mode 100644 index 0000000..c2d57d4 --- /dev/null +++ b/test/steps/runDockerBuild.js @@ -0,0 +1,125 @@ +'use strict'; + +var Lab = require('lab'); +var lab = exports.lab = Lab.script(); +var expect = require('code').expect; + +var childProcess = require('child_process'); + +var dockerMock = require('docker-mock'); + +var cacheDir = process.env.CACHE_DIR; +if (!cacheDir) { + cacheDir = process.env.CACHE_DIR = '/tmp/cache'; +} +var layerCacheDir = process.env.LAYER_CACHE_DIR; +if (!layerCacheDir) { + layerCacheDir = process.env.LAYER_CACHE_DIR = '/tmp/layer-cache'; +} +// require this after we have now changed the env for the directories +var steps = require('../../lib/steps'); + +var requiredEnvVars = { + RUNNABLE_AWS_ACCESS_KEY: process.env.AWS_ACCESS_KEY, + RUNNABLE_AWS_SECRET_KEY: process.env.AWS_SECRET_KEY +}; +lab.before(function (done) { + Object.keys(requiredEnvVars).forEach(function (key) { + process.env[key] = requiredEnvVars[key]; + }); + done(); +}); + +lab.experiment('runDockerBuild', function () { + var dockerMockServer; + lab.before(function (done) { + dockerMockServer = dockerMock.listen(5555, done); + }); + lab.after(function (done) { + dockerMockServer.close(done); + }); + var requiredEnvVars = { + RUNNABLE_DOCKER: 'tcp://localhost:5555', + RUNNABLE_DOCKERTAG: 'test-docker-tag', + RUNNABLE_FILES: '{ "Dockerfile": "K6cluDupwQdFRsuTPJ0SFUrxUB4lmF_Q" }', + RUNNABLE_FILES_BUCKET: 'runnable.image-builder', + RUNNABLE_DOCKER_BUILDOPTIONS: '--quiet=false' + }; + lab.beforeEach(function (done) { + Object.keys(requiredEnvVars).forEach( + function (key) { process.env[key] = requiredEnvVars[key]; }); + done(); + }); + + lab.experiment('succeeds', function () { + lab.beforeEach(steps.makeWorkingFolders.bind(steps)); + lab.beforeEach({ timeout: 5000 }, steps.downloadBuildFiles.bind(steps)); + + lab.experiment('if there is no dockerhost', function () { + lab.beforeEach(function (done) { + delete process.env.RUNNABLE_DOCKER; + done(); + }); + + lab.test('should do nothing', function (done) { + steps.runDockerBuild(function (err) { + if (err) { return done(err); } + done(); + }); + }); + }); + + lab.experiment('if there is no docker build options', function () { + lab.beforeEach(function (done) { + delete process.env.RUNNABLE_DOCKER_BUILDOPTIONS; + done(); + }); + + lab.test('should build fine', function (done) { + steps.runDockerBuild(function (err) { + if (err) { return done(err); } + var dockerArgs = [ + '--host ' + requiredEnvVars.RUNNABLE_DOCKER, + 'images' + ].join(' '); + childProcess.exec('docker ' + dockerArgs, function (err, stdout) { + if (err) { return done(err); } + expect(stdout.trim().split('\n')).to.have.length(2); + done(err); + }); + }); + }); + }); + + lab.experiment('with all the envs', function () { + lab.test('should call to docker', function (done) { + steps.runDockerBuild(function (err) { + if (err) { return done(err); } + var dockerArgs = [ + '--host ' + requiredEnvVars.RUNNABLE_DOCKER, + 'images' + ].join(' '); + childProcess.exec('docker ' + dockerArgs, function (err, stdout) { + if (err) { return done(err); } + expect(stdout.trim().split('\n')).to.have.length(2); + done(err); + }); + }); + }); + }); + }); + + lab.experiment('fails', function () { + lab.beforeEach(function (done) { + delete process.env.RUNNABLE_DOCKERTAG; + done(); + }); + lab.test('when the required env vars are missing', function (done) { + steps.runDockerBuild(function (err) { + expect(!!err).to.be.true(); + expect(err.message).to.match(/RUNNABLE_DOCKERTAG is missing/); + done(); + }); + }); + }); +}); From 6c7b8319737e4fdc46bafec8436b639efa455f83 Mon Sep 17 00:00:00 2001 From: Bryan Kendall Date: Tue, 24 Feb 2015 14:23:56 -0800 Subject: [PATCH 2/7] add coverage threshold --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 1893510..0f5e350 100644 --- a/package.json +++ b/package.json @@ -24,7 +24,7 @@ "sinon": "^1.12.2" }, "scripts": { - "test": "./node_modules/.bin/lab --verbose", + "test": "./node_modules/.bin/lab --verbose --coverage --threshold 95", "lint": "./node_modules/.bin/jshint --verbose ." }, "repository": { From aad2a7c29cc858502e7b04d5ec4268707c8b3433 Mon Sep 17 00:00:00 2001 From: Bryan Kendall Date: Tue, 24 Feb 2015 17:17:44 -0800 Subject: [PATCH 3/7] use hash as layer file name --- lib/dockerLayerArchive.sh | 6 ++---- lib/steps.js | 36 ++++++++++++++++++----------------- test/steps/parseDockerfile.js | 11 +++++++---- 3 files changed, 28 insertions(+), 25 deletions(-) diff --git a/lib/dockerLayerArchive.sh b/lib/dockerLayerArchive.sh index 3118d7b..39c1502 100755 --- a/lib/dockerLayerArchive.sh +++ b/lib/dockerLayerArchive.sh @@ -28,8 +28,6 @@ docker -H "$RUNNABLE_DOCKER" save -o "$tar_name" "$IMAGE_ID" cache_layer_name=$(echo "$RUNNABLE_DOCKERTAG" | awk '{split($0,a,":"); print a[1];}') tar --extract --file "$tar_name" --directory="$tar_dir" "$CACHED_LAYER"/layer.tar mkdir -p /layer-cache/"$cache_layer_name" -rm -rf /layer-cache/"$cache_layer_name"/hash.* -mv "$tar_dir"/"$CACHED_LAYER"/layer.tar /layer-cache/"$cache_layer_name"/layer.tar -touch /layer-cache/"$cache_layer_name"/hash."$CACHED_LAYER_HASH" +mv "$tar_dir"/"$CACHED_LAYER"/layer.tar /layer-cache/"$cache_layer_name"/"$CACHED_LAYER_HASH".tar -echo "Saved $cache_layer_name/layer.tar" +echo "Saved $cache_layer_name/$CACHED_LAYER_HASH.tar" diff --git a/lib/steps.js b/lib/steps.js index 7537285..ea95ab0 100644 --- a/lib/steps.js +++ b/lib/steps.js @@ -319,26 +319,28 @@ var steps = module.exports = { steps.data.createdByHash = md5sum.digest('hex'); // if the cache layer hash exists, use the layer.tar that should be there - var filename = path.join( - layerCacheDir, - process.env.RUNNABLE_DOCKERTAG.split(':')[0], - 'hash.' + steps.data.createdByHash); var layerTar = path.join( layerCacheDir, process.env.RUNNABLE_DOCKERTAG.split(':')[0], - 'layer.tar'); - if (fs.existsSync(filename)) { - dockerfile = dockerfile.replace( - match[1], - 'ADD layer.tar /\n' + match[1]); - fs.writeFileSync( - path.join(steps.dirs.dockerContext, 'Dockerfile'), - dockerfile); - steps.data.usingCache = true; - childProcess.exec( - 'cp -p ' + layerTar + ' ' + steps.dirs.dockerContext, - steps.saveToLogs(cb)); - } else { cb(); } + steps.data.createdByHash + '.tar'); + childProcess.exec( + 'cp -p ' + layerTar + ' ' + steps.dirs.dockerContext, + steps.saveToLogs(function (err) { + if (err) { + // if this error'd, just don't use the layer + cb(); + } else { + dockerfile = dockerfile.replace( + match[1], + 'ADD layer.tar /\n' + match[1]); + fs.writeFileSync( + path.join(steps.dirs.dockerContext, 'Dockerfile'), + dockerfile); + steps.data.usingCache = true; + cb(); + } + }) + ); } else { cb(); } }, diff --git a/test/steps/parseDockerfile.js b/test/steps/parseDockerfile.js index ba81d9d..4c13770 100644 --- a/test/steps/parseDockerfile.js +++ b/test/steps/parseDockerfile.js @@ -80,7 +80,11 @@ lab.experiment('parseDockerfile', function () { }); lab.beforeEach({ timeout: 5000 }, steps.downloadBuildFiles.bind(steps)); lab.beforeEach(function (done) { - sinon.spy(childProcess, 'exec'); + sinon.stub(childProcess, 'exec') + .yields( + new Error('Command failed: cp: /tmp/empty/file:' + + 'No such file or directory'), + '', ''); done(); }); lab.afterEach(function (done) { @@ -92,7 +96,7 @@ lab.experiment('parseDockerfile', function () { steps.parseDockerfile(function (err) { if (err) { return done(err); } // not copying a cache - expect(childProcess.exec.callCount).to.equal(0); + expect(childProcess.exec.callCount).to.equal(1); expect(!!steps.data.usingCache).to.be.false(); expect(steps.data.cachedLine).to.not.be.undefined(); expect(steps.data.createdByHash).to.not.be.undefined(); @@ -141,8 +145,7 @@ lab.experiment('parseDockerfile', function () { var cmds = [ 'mkdir -p /tmp/layer-cache/test-docker-tag', 'touch /tmp/layer-cache/test-docker-tag/' + - 'hash.93f7657e7c42734aac70d134cecf53d3', - 'touch /tmp/layer-cache/test-docker-tag/layer.tar' + '93f7657e7c42734aac70d134cecf53d3.tar' ].join(' && '); childProcess.exec(cmds, done); }); From 24c87fd882575f221d9117657fcf16aba8434a82 Mon Sep 17 00:00:00 2001 From: Bryan Kendall Date: Tue, 24 Feb 2015 17:26:44 -0800 Subject: [PATCH 4/7] updated functional tests for new files --- scripts/test-circle-08.sh | 2 +- scripts/test-circle-09.sh | 2 +- scripts/test-circle-10.sh | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/test-circle-08.sh b/scripts/test-circle-08.sh index 7a3e257..8fdb1f0 100755 --- a/scripts/test-circle-08.sh +++ b/scripts/test-circle-08.sh @@ -52,4 +52,4 @@ echo "checking layer cache status" # layer-cache tests # directory and layer should exist test -d ./test-"$test_num"/layer-cache/test/test-built-image || (echo "directory for layer should exist" && false) -test -f ./test-"$test_num"/layer-cache/test/test-built-image/layer.tar || (echo "layer.tar should exist" && false) +ls ./test-"$test_num"/layer-cache/test/test-built-image/*.tar 1> /dev/null 2>&1 || (echo "layer.tar should exist" && false) diff --git a/scripts/test-circle-09.sh b/scripts/test-circle-09.sh index 785425c..bec69a6 100755 --- a/scripts/test-circle-09.sh +++ b/scripts/test-circle-09.sh @@ -62,7 +62,7 @@ checks () { # layer-cache tests # directory and layer should exist test -d ./test-"$test_num"/layer-cache/test/test-built-image || (echo "directory for layer should exist" && false) - test -f ./test-"$test_num"/layer-cache/test/test-built-image/layer.tar || (echo "layer.tar should exist" && false) + ls ./test-"$test_num"/layer-cache/test/test-built-image/*.tar 1> /dev/null 2>&1 || (echo "layer.tar should exist" && false) } first_log=$(mktemp /tmp/log.XXXX) diff --git a/scripts/test-circle-10.sh b/scripts/test-circle-10.sh index bf896e4..933e925 100755 --- a/scripts/test-circle-10.sh +++ b/scripts/test-circle-10.sh @@ -62,7 +62,7 @@ checks () { # layer-cache tests # directory and layer should exist test -d ./test-"$test_num"/layer-cache/test/test-built-image || (echo "directory for layer should exist" && false) - test -f ./test-"$test_num"/layer-cache/test/test-built-image/layer.tar || (echo "layer.tar should exist" && false) + ls ./test-"$test_num"/layer-cache/test/test-built-image/*.tar 1> /dev/null 2>&1 || (echo "layer.tar should exist" && false) } first_log=$(mktemp /tmp/log.XXXX) From e2e5fef52b0674e669fbebfa112136bd433cb00a Mon Sep 17 00:00:00 2001 From: Bryan Kendall Date: Wed, 25 Feb 2015 17:07:44 -0800 Subject: [PATCH 5/7] add layer-cache to env for circle --- circle.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/circle.yml b/circle.yml index 125b4ed..4b31216 100644 --- a/circle.yml +++ b/circle.yml @@ -12,9 +12,11 @@ test: - docker build -t test-image-builder . - npm install - mkdir -p /tmp/cache + - mkdir -p /tmp/layer-cache override: - npm run lint - npm test: environment: CACHE_DIR: /tmp/cache + LAYER_CACHE_DIR: /tmp/layer-cache - ./scripts/run-tests.sh From 2c7764aec01a9ef6bef1313ce663c6273de6693d Mon Sep 17 00:00:00 2001 From: Bryan Kendall Date: Wed, 25 Feb 2015 17:20:49 -0800 Subject: [PATCH 6/7] use hash in dockerfile for layer --- lib/steps.js | 2 +- test/steps/parseBuildLogAndHistory.js | 10 ++++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/lib/steps.js b/lib/steps.js index ea95ab0..742c31b 100644 --- a/lib/steps.js +++ b/lib/steps.js @@ -332,7 +332,7 @@ var steps = module.exports = { } else { dockerfile = dockerfile.replace( match[1], - 'ADD layer.tar /\n' + match[1]); + 'ADD ' + steps.data.createdByHash + '.tar /\n' + match[1]); fs.writeFileSync( path.join(steps.dirs.dockerContext, 'Dockerfile'), dockerfile); diff --git a/test/steps/parseBuildLogAndHistory.js b/test/steps/parseBuildLogAndHistory.js index 981e5a6..ab93990 100644 --- a/test/steps/parseBuildLogAndHistory.js +++ b/test/steps/parseBuildLogAndHistory.js @@ -50,6 +50,9 @@ lab.experiment('parseBuildLogAndHistory', function () { function (key) { process.env[key] = requiredEnvVars[key]; }); done(); }); + lab.beforeEach(function (done) { + childProcess.exec('rm -rf ' + layerCacheDir + '/*', done); + }); lab.experiment('succeeds', function () { lab.beforeEach(steps.makeWorkingFolders.bind(steps)); @@ -88,10 +91,9 @@ lab.experiment('parseBuildLogAndHistory', function () { lab.experiment('with an available layer cache', function () { lab.beforeEach(function (done) { var cmds = [ - 'mkdir -p /tmp/layer-cache/test-docker-tag', - 'touch /tmp/layer-cache/test-docker-tag/' + - 'hash.93f7657e7c42734aac70d134cecf53d3', - 'touch /tmp/layer-cache/test-docker-tag/layer.tar' + 'mkdir -p ' + layerCacheDir + '/test-docker-tag', + 'touch ' + layerCacheDir + '/test-docker-tag/' + + '93f7657e7c42734aac70d134cecf53d3.tar' ].join(' && '); childProcess.exec(cmds, done); }); From 776535e555fd8ea0990e04f1819cb58f4639d108 Mon Sep 17 00:00:00 2001 From: Bryan Kendall Date: Wed, 25 Feb 2015 17:36:56 -0800 Subject: [PATCH 7/7] change tests to look for hashes --- scripts/test-circle-09.sh | 4 ++-- scripts/test-circle-10.sh | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/test-circle-09.sh b/scripts/test-circle-09.sh index bec69a6..ad60b6a 100755 --- a/scripts/test-circle-09.sh +++ b/scripts/test-circle-09.sh @@ -71,8 +71,8 @@ second_log=$(mktemp /tmp/log.XXXX) echo "FIRST BUILD" build $first_log checks 1 -grep -vq "ADD layer.tar /" "$first_log" || (echo "should not have added the layer in the first build" && false) +grep -vqE "ADD [0-9a-f]+\.tar /" "$first_log" || (echo "should not have added the layer in the first build" && false) echo "SECOND BUILD" build $second_log checks 0 -grep -q "ADD layer.tar /" "$second_log" || (echo "should have added the layer in the second build" && false) +grep -qE "ADD [0-9a-f]+\.tar /" "$second_log" || (echo "should have added the layer in the second build" && false) diff --git a/scripts/test-circle-10.sh b/scripts/test-circle-10.sh index 933e925..29ba2fa 100755 --- a/scripts/test-circle-10.sh +++ b/scripts/test-circle-10.sh @@ -71,8 +71,8 @@ second_log=$(mktemp /tmp/log.XXXX) echo "FIRST BUILD" build $first_log checks 1 -grep -vq "ADD layer.tar /" "$first_log" || (echo "should not have added the layer in the first build" && false) +grep -vqE "ADD [0-9a-f]+\.tar /" "$first_log" || (echo "should not have added the layer in the first build" && false) echo "SECOND BUILD" build $second_log checks 0 -grep -q "ADD layer.tar /" "$second_log" || (echo "should have added the layer in the second build" && false) +grep -qE "ADD [0-9a-f]+\.tar /" "$second_log" || (echo "should have added the layer in the second build" && false)