Merge pull request #676 from jossemarGT/improve-dockerignore-support

Improve dockerignore support

.gitignore

@@ -4,3 +4,4 @@ npm-debug.log
 *~
 *.log
 .idea
+.vscode

Vagrantfile

@@ -1,8 +1,9 @@
 # -*- mode: ruby -*-
 # vi: set ft=ruby :
 
-BOX_NAME = ENV['BOX_NAME'] || "ubuntu/xenial64"
+BOX_NAME = ENV['BOX_NAME'] || "ubuntu/focal64"
 SSH_PRIVKEY_PATH = ENV["SSH_PRIVKEY_PATH"]
+NODE_MAJOR_VERSION = "14"
 
 $script = <<SCRIPT
 user="$1"
@@ -11,23 +12,42 @@ if [ -z "$user" ]; then
 fi
 
 apt-get update -q
-apt-get install -q -y apt-transport-https ca-certificates
 
-apt-key adv --keyserver hkp://ha.pool.sks-keyservers.net:80 --recv-keys 58118E89F3A912897C070ADBF76221572C52609D
-echo "deb https://apt.dockerproject.org/repo ubuntu-xenial main" > /etc/apt/sources.list.d/docker.list
+apt-get install -yq \
+    ca-certificates \
+    curl \
+    gnupg \
+    lsb-release \
+    apt-transport-https
 
-apt-get update -q
-apt-get install -q -y linux-image-extra-$(uname -r) linux-image-extra-virtual docker-engine
-service docker start
+curl -sL https://deb.nodesource.com/setup_#{NODE_MAJOR_VERSION}.x | sudo -E bash -
+
+apt-get install -yq \
+  gcc \
+  g++ \
+  make \
+  python-is-python3 \
+  nodejs
 
-curl -sL https://deb.nodesource.com/setup_6.x | sudo -E bash -
+mkdir -p /etc/apt/keyrings
+curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg
+echo \
+  "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu \
+  $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
 
 apt-get update -q
-apt-get install -q -y nodejs python-software-properties python g++ make software-properties-common
+apt-get install -yq \
+  docker-ce \
+  docker-ce-cli \
+  containerd.io \
+  linux-image-extra-virtual
+
+grep -q docker /etc/group || groupadd docker
+usermod -a -G docker vagrant
 
+systemctl start docker
 
-usermod -a -G docker ubuntu
-docker pull ubuntu
+docker image pull ubuntu
 
 printf "#"'!'"/bin/bash\ndocker rm -f "'$'"(docker ps -a -q)" > /usr/bin/clearcontainers
 chmod +x /usr/bin/clearcontainers
@@ -46,7 +66,7 @@ end
 
 Vagrant::VERSION >= "1.1.0" and Vagrant.configure("2") do |config|
   config.vm.provider :virtualbox do |vb, override|
-    override.vm.provision :shell, :inline => $script
+    override.vm.provision :shell, :run => :once, :inline => $script
     vb.customize ["modifyvm", :id, "--natdnshostresolver1", "on"]
     vb.customize ["modifyvm", :id, "--natdnsproxy1", "on"]
   end

lib/docker.js

@@ -1,7 +1,5 @@
 var EventEmitter = require('events').EventEmitter,
   Modem = require('docker-modem'),
-  tar = require('tar-fs'),
-  zlib = require('zlib'),
   Container = require('./container'),
   Image = require('./image'),
   Volume = require('./volume'),
@@ -14,11 +12,7 @@ var EventEmitter = require('events').EventEmitter,
   Node = require('./node'),
   Exec = require('./exec'),
   util = require('./util'),
-  extend = util.extend,
-  path = require('path'),
-  extend = util.extend,
-  ignore = require('@balena/dockerignore'),
-  fs = require('fs');
+  extend = util.extend;
 
 var Docker = function(opts) {
   if (!(this instanceof Docker)) return new Docker(opts);
@@ -262,64 +256,55 @@ Docker.prototype.buildImage = function(file, opts, callback) {
     opts = null;
   }
 
-  function build(file) {
-    var optsf = {
-      path: '/build?',
-      method: 'POST',
-      file: file,
-      options: opts,
-      abortSignal: opts && opts.abortSignal,
-      isStream: true,
-      statusCodes: {
-        200: true,
-        500: 'server error'
-      }
-    };
+  var optsf = {
+    path: '/build?',
+    method: 'POST',
+    file: undefined,
+    options: opts,
+    abortSignal: opts && opts.abortSignal,
+    isStream: true,
+    statusCodes: {
+      200: true,
+      500: 'server error'
+    }
+  };
 
-    if (opts) {
-      if (opts.registryconfig) {
-        optsf.registryconfig = optsf.options.registryconfig;
-        delete optsf.options.registryconfig;
-      }
+  if (opts) {
+    if (opts.registryconfig) {
+      optsf.registryconfig = optsf.options.registryconfig;
+      delete optsf.options.registryconfig;
+    }
 
-      //undocumented?
-      if (opts.authconfig) {
-        optsf.authconfig = optsf.options.authconfig;
-        delete optsf.options.authconfig;
-      }
+    //undocumented?
+    if (opts.authconfig) {
+      optsf.authconfig = optsf.options.authconfig;
+      delete optsf.options.authconfig;
     }
+  }
+
+  if (callback === undefined) {
+    const prepareCtxPromise = new self.modem.Promise(function(resolve, _) {
+      util.prepareBuildContext(file, resolve)
+    });
 
-    if (callback === undefined) {
+    return prepareCtxPromise.then((ctx)=> {
       return new self.modem.Promise(function(resolve, reject) {
+        optsf.file = ctx;
         self.modem.dial(optsf, function(err, data) {
           if (err) {
             return reject(err);
           }
           resolve(data);
         });
       });
-    } else {
+    })
+  } else {
+    util.prepareBuildContext(file, (ctx) => {
+      optsf.file = ctx;
       self.modem.dial(optsf, function(err, data) {
         callback(err, data);
       });
-    }
-  }
-
-  if (file && file.context) {
-    let entries = file.src
-
-    const dockerignorePath = path.join(file.context, '.dockerignore')
-    if (fs.existsSync(dockerignorePath)) {
-      const dockerIgnore = ignore({ ignorecase: false }).add(fs.readFileSync(dockerignorePath).toString())
-      entries = (entries || []).filter(dockerIgnore.createFilter())
-    }
-
-    var pack = tar.pack(file.context, {
-      entries: file.src.slice()
-    });
-    return build(pack.pipe(zlib.createGzip()));
-  } else {
-    return build(file);
+    })
   }
 };
 

lib/util.js

@@ -1,3 +1,9 @@
+var DockerIgnore = require('@balena/dockerignore');
+var fs = require('fs');
+var path = require('path');
+var tar = require('tar-fs');
+var zlib = require('zlib');
+
 // https://github.com/HenrikJoreteg/extend-object/blob/v0.1.0/extend-object.js
 
 var arr = [];
@@ -68,3 +74,33 @@ module.exports.parseRepositoryTag = function(input) {
     repository: input
   };
 };
+
+
+module.exports.prepareBuildContext = function(file, next) {
+  if (file && file.context) {
+    fs.readFile(path.join(file.context, '.dockerignore'), (err, data) => {
+      let ignoreFn;
+      let filterFn;
+
+      if (!err) {
+        const dockerIgnore = DockerIgnore({ ignorecase: false }).add(data.toString());
+
+        filterFn = dockerIgnore.createFilter();
+        ignoreFn = (path) => {
+          return !filterFn(path);
+        }
+      }
+
+      const entries = file.src.slice() || []
+
+      const pack = tar.pack(file.context, {
+        entries: filterFn ? entries.filter(filterFn) : entries,
+        ignore: ignoreFn // Only works on directories
+      });
+
+      next(pack.pipe(zlib.createGzip()));
+    })
+  } else {
+    next(file);
+  }
+}

test/docker.js

@@ -3,6 +3,7 @@
 var Bluebird = require('bluebird'),
   expect = require('chai').expect,
   assert = require('assert'),
+  path = require('path'),
   Docker = require('../lib/docker');
 
 var docker = require('./spec_helper').docker;
@@ -128,7 +129,29 @@ describe("#docker", function() {
       docker.buildImage({
         context: __dirname,
         src: ['Dockerfile']
-      }, {}, handler);
+      }, { t: 'multiple-files' }, handler);
+    });
+
+    it("should build image from multiple files while respecting the .dockerignore file", function(done) {
+      this.timeout(60000);
+
+      function handler(err, stream) {
+        expect(err).to.be.null;
+        expect(stream).to.be.ok;
+
+        stream.pipe(process.stdout, {
+          end: true
+        });
+
+        stream.on('end', function() {
+          done();
+        });
+      }
+
+      docker.buildImage({
+        context: path.join(__dirname, 'fixtures', 'dockerignore'),
+        src: ['Dockerfile', 'MC-hammer.txt', 'ignore-dir', 'foo.txt']
+      }, { t: 'honor-dockerignore' }, handler);
     });
 
     it("should build image from multiple files while respecting the dockerignore file", function(done) {

test/fixtures/dockerignore/.dockerignore

@@ -0,0 +1,6 @@
+# Ignore dirs
+ignore-dir
+# Ignore blobs
+*.txt
+# Override rules
+!MC-hammer.txt

test/fixtures/dockerignore/Dockerfile

@@ -0,0 +1,5 @@
+FROM scratch
+
+COPY . .
+
+CMD ["bash"]

test/fixtures/dockerignore/MC-hammer.txt

@@ -0,0 +1 @@
+Can't touch this

test/util.js

@@ -1,11 +1,16 @@
 var expect = require('chai').expect;
 var util = require('../lib/util');
+var path = require('path');
+var zlib = require('zlib');
+var tar = require('tar-fs');
+var fs = require('fs');
+var os = require('os');
 
-describe('util', function() {
+describe('util', function () {
 
-  describe('#parseRepositoryTag', function() {
+  describe('#parseRepositoryTag', function () {
     function validate(input, expected) {
-      it('should parse "' + input + '"', function() {
+      it('should parse "' + input + '"', function () {
         expect(util.parseRepositoryTag(input)).to.eql(expected);
       });
     }
@@ -46,11 +51,50 @@ describe('util', function() {
   });
 
   // https://github.com/HenrikJoreteg/extend-object/blob/v0.1.0/test.js
-  describe('.extend', function() {
+  describe('.extend', function () {
     it('accepts multiple object arguments', function () {
       var start = {};
-      expect(util.extend(start, {name: 'test'}, {hello: 'test'})).to.deep.equal({name: 'test', hello: 'test'});
+      expect(util.extend(start, { name: 'test' }, { hello: 'test' })).to.deep.equal({ name: 'test', hello: 'test' });
       expect(start).to.eql(util.extend(start, {}));
     });
   });
+
+  describe('.prepareBuildContext', function () {
+
+    it("should pass the options through when there is no context", function () {
+      const dummy = {};
+      util.prepareBuildContext(dummy, function (ctx) {
+        expect(ctx).to.be.equal(dummy);
+      })
+    });
+
+    it("bundle the context and source as a single tar.gz stream", function (done) {
+      this.timeout(60000);
+
+      function handler(stream) {
+        expect(stream).to.be.ok;
+
+        const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'dockerode-'));
+        const z = zlib.createGunzip();
+
+        stream
+          .pipe(z)
+          .pipe(tar.extract(tmp), { end: true })
+          .on('finish', function () {
+            const files = fs.readdirSync(tmp);
+
+            expect(files.length).to.be.equal(2);
+            expect(files).to.have.members(['Dockerfile', 'MC-hammer.txt']);
+
+            fs.rm(tmp, { recursive: true });
+            done();
+          });
+      }
+
+      util.prepareBuildContext({
+        context: path.join(__dirname, 'fixtures', 'dockerignore'),
+        src: ['Dockerfile', 'MC-hammer.txt', 'ignore-dir', 'foo.txt']
+      }, handler);
+    });
+  });
 });