Reduce image size produced by machine_image resource (chef-boneyard#26

)

Signed-off-by: Maksim Chizhov <[email protected]>

lib/chef/provisioning/docker_driver/docker_container_machine.rb

@@ -20,6 +20,9 @@ def initialize(machine_spec, transport, convergence_strategy, opts = {})
       @transport = transport
     end
 
+    # Expose volumes to be able to access it from DockerConvergenceStrategy.
+    attr_reader :volumes
+
     def execute_always(command, options = {})
       transport.execute(command, { :read_only => true }.merge(options))
     end

lib/chef/provisioning/docker_driver/docker_convergence_strategy.rb

@@ -0,0 +1,96 @@
+require 'chef/provisioning/convergence_strategy/precreate_chef_objects'
+require 'chef/provisioning/convergence_strategy/install_cached'
+
+# We don't want to install chef-client omnibus into container. Instead we'll 
+# mount volume with exiting installation or install it on that volume if missing.
+class Chef
+module Provisioning
+module DockerDriver
+  class ConvergenceStrategy < Chef::Provisioning::ConvergenceStrategy::InstallCached
+
+    # Initializes instance, sets volume to install chef-client to 
+    # "/tmp/opt/chef:/opt/chef" by default.
+    def initialize(convergence_options, config)
+      super(convergence_options, config)
+      # Chef obmibus installation directory on host machine.
+      @chef_volume = (convergence_options[:chef_volume] || '/tmp/opt/chef') + ":/opt/chef"
+    end
+
+    # Inlike Chef::Provisioning::ConvergenceStrategy::InstallCached, will not 
+    # upload package to machine, but mount it as volume instead.
+    def setup_convergence(action_handler, machine)
+      # We still need some basic setup from InstallCached parent class.
+      PrecreateChefObjects.instance_method(:setup_convergence).bind(self).call(action_handler, machine)
+
+      # Check for existing chef client.
+      version = machine.execute_always('chef-client -v', :volumes => [@chef_volume])
+
+      # Don't do install/upgrade if a chef client exists and
+      # no chef version is defined by user configs or
+      # the chef client's version already matches user config
+      if version.exitstatus == 0
+        version = version.stdout.strip
+        if !chef_version
+          return
+        elsif version =~ /Chef: #{chef_version}$/
+          Chef::Log.debug "Already installed chef version #{version}"
+          return
+        elsif version.include?(chef_version)
+          Chef::Log.warn "Installed chef version #{version} contains desired version #{chef_version}.  " +
+            "If you see this message on consecutive chef runs tighten your desired version constraint to prevent " +
+            "multiple convergence."
+        end
+      end
+
+      # Install chef client
+      platform, platform_version, machine_architecture = machine.detect_os(action_handler)
+      package_file = download_package_for_platform(action_handler, machine, platform, platform_version, machine_architecture)
+      remote_package_file = "#{@tmp_dir}/#{File.basename(package_file)}"
+      install_package(action_handler, machine, remote_package_file)
+    end
+
+    # Converge machine using volumes from machine options and chef installalation
+    # from @chef_volume (default "/tmp/opt/chef:/opt/chef").
+    # This implementation also allows to mount volumes during recipes execution 
+    # including machine images creation.
+    def converge(action_handler, machine)
+      Chef::Log.debug("Converge machine using machine volumes: #{machine.volumes}")
+      PrecreateChefObjects.instance_method(:converge).bind(self).call(action_handler, machine)
+
+      action_handler.open_stream(machine.node['name']) do |stdout|
+        action_handler.open_stream(machine.node['name']) do |stderr|
+          command_line = "chef-client"
+          command_line << " -l #{config[:log_level].to_s}" if config[:log_level]
+          machine.execute(action_handler, command_line,
+            :stream_stdout => stdout,
+            :stream_stderr => stderr,
+            :timeout => @chef_client_timeout,
+	    # In case we've specified volumes in machine options, it needs 
+	    # to be accessible during converge too.
+            :volumes => Array(machine.volumes) << @chef_volume)
+        end
+      end
+    end
+
+    # Installs chef-client omnibus inside container on mounted volume 
+    # (see @chef_volume), package is also mounted as ro volume.
+    def install_package(action_handler, machine, remote_package_file)
+
+      local_package_file = File.join(@package_cache_path, File.basename(remote_package_file))
+      opts = {:volumes => [@chef_volume, "#{local_package_file}:#{remote_package_file}:ro"]}
+      extension = File.extname(remote_package_file)
+      result = case extension
+      when '.rpm'
+        machine.execute(action_handler, "rpm -Uvh --oldpackage --replacepkgs \"#{remote_package_file}\"", opts)
+      when '.deb'
+        machine.execute(action_handler, "dpkg -i \"#{remote_package_file}\"", opts)
+      when '.sh'
+        machine.execute(action_handler, "sh \"#{remote_package_file}\"", opts)
+      else
+        raise "Unknown package extension '#{extension}' for file #{remote_package_file}"
+      end
+    end
+  end
+end
+end
+end

lib/chef/provisioning/docker_driver/driver.rb

@@ -3,7 +3,7 @@
 require 'chef/provisioning/docker_driver/version'
 require 'chef/provisioning/docker_driver/docker_transport'
 require 'chef/provisioning/docker_driver/docker_container_machine'
-require 'chef/provisioning/convergence_strategy/install_cached'
+require 'chef/provisioning/docker_driver/docker_convergence_strategy'
 require 'chef/provisioning/convergence_strategy/no_converge'
 require 'chef/mash'
 
@@ -95,7 +95,7 @@ def build_container(machine_spec, machine_options)
       docker_options = machine_options[:docker_options]
 
       base_image = docker_options[:base_image]
-      if !base_image
+      if base_image.nil?
         Chef::Log.debug("No base images specified in docker options.")
         base_image = base_image_for(machine_spec)
       end
@@ -182,7 +182,7 @@ def destroy_machine(action_handler, machine_spec, machine_options)
       Chef::Log.debug("Removing #{container_name}")
       container.delete
 
-      if !machine_spec.attrs[:keep_image] && !machine_options[:keep_image]
+      unless machine_spec.attrs[:keep_image] || machine_options[:keep_image]
         Chef::Log.debug("Destroying image: chef:#{container_name}")
         image = Docker::Image.get("chef:#{container_name}")
         image.delete
@@ -205,6 +205,14 @@ def find_image(repository, tag)
       }.first
     end
 
+    def find_container(name)
+      begin
+        Docker::Container.get(name, @connection)
+      rescue Docker::Error::NotFoundError
+        Chef::Log.warn("Container #{name} not found.")
+      end
+    end
+
     def driver_url
       "docker:#{Docker.url}"
     end
@@ -213,9 +221,8 @@ def start_machine(action_handler, machine_spec, machine_options, base_image_name
       # Spin up a docker instance if needed, otherwise use the existing one
       container_name = machine_spec.location['container_name']
 
-      begin
-        Docker::Container.get(container_name, @connection)
-      rescue Docker::Error::NotFoundError
+      container = find_container(container_name)
+      if container.nil?
         docker_options = machine_options[:docker_options]
         Chef::Log.debug("Start machine for container #{container_name} using base image #{base_image_name} with options #{docker_options.inspect}")
         image = image_named(base_image_name)
@@ -256,8 +263,9 @@ def machine_for(machine_spec, machine_options, base_image_name)
 
     def convergence_strategy_for(machine_spec, machine_options)
       @unix_convergence_strategy ||= begin
-        Chef::Provisioning::ConvergenceStrategy::InstallCached.
+        Chef::Provisioning::DockerDriver::ConvergenceStrategy.
             new(machine_options[:convergence_options], config)
+
       end
     end
 

spec/integration/primitives_spec.rb

@@ -18,12 +18,12 @@
           },
         }
 
-      let(:iso_date) { Time.now.iso8601.gsub(':', '') }
+      let(:timestamp) { Time.now.to_i }
       docker_driver = driver
 
       context "machine_image resource" do
 
-        let(:spec_image_tag) { "docker_image_spec_#{iso_date}" }
+        let(:spec_image_tag) { "docker_image_spec_#{timestamp}" }
 
         after(:each) {
           image = docker_driver.find_image("chef", spec_image_tag)
@@ -72,4 +72,4 @@
       end
     end
   end
-end
+end

spec/integration/volumes_spec.rb

@@ -0,0 +1,113 @@
+require 'spec_helper'
+require 'time'
+require 'fileutils'
+
+describe "chef-provisioning-docker" do
+  extend DockerSupport
+  include DockerConfig
+
+  when_the_chef_12_server "exists", organization: "foo", server_scope: :context, port: 8900..9000 do
+    with_docker "volumes supported" do
+
+      # owing to how RSpec works, things defined by let() are not accessible in the recipes we define inside
+      # expect_converge{}.
+      ubuntu_options = {
+          :base_image => {
+            :name => 'ubuntu',
+            :repository => 'ubuntu',
+            :tag => '14.04'
+          }
+        }
+
+      let(:timestamp) { Time.now.to_i }
+      docker_driver = driver
+
+      context "machine_image resource" do
+
+        let(:spec_image_tag) { "docker_image_spec_#{timestamp}" }
+
+        after(:each) {
+          image = docker_driver.find_image("chef", "#{spec_image_tag}")
+          image.delete(force: true) unless image.nil?
+          FileUtils.rm_rf("/tmp/opt")
+          FileUtils.rm_rf("/tmp/#{spec_image_tag}")
+        }
+
+        it "installs chef omnibus on the volume" do
+          tag = spec_image_tag
+
+          expect_converge {
+
+            machine_image tag do
+              machine_options :docker_options => ubuntu_options
+              action :create
+            end
+          }.not_to raise_error
+
+          expect(File.exists?("/tmp/opt/chef/bin/chef-client")).to be_truthy
+        end
+
+        it "installs chef omnibus on specific volume" do
+          tag = spec_image_tag
+
+          expect_converge {
+
+            machine_image tag do
+              machine_options :docker_options => ubuntu_options,
+                :convergence_options => {
+                  :chef_volume => "/tmp/#{tag}/chef"
+                }
+              action :create
+            end
+          }.not_to raise_error
+
+          expect(File.exists?("/tmp/#{tag}/chef/bin/chef-client")).to be_truthy
+        end
+
+
+        it "mounts volume during converge" do
+          tag = spec_image_tag
+
+          expect_converge {
+
+            machine_image tag do
+              machine_options :docker_options => ubuntu_options.merge({:volumes => "/tmp/#{tag}:/var/chef/cache/"})
+              action :create
+            end
+          }.not_to raise_error
+
+          expect(File.exists?("/tmp/#{tag}/chef-client-running.pid")).to be_truthy
+        end
+      end
+
+      context "machine resource" do
+        let(:spec_machine_name) { "docker_machine_spec_#{timestamp}" }
+
+        after(:each) {
+          container = docker_driver.find_container(spec_machine_name)
+          container.delete(:force => true) unless container.nil?
+          FileUtils.rm_rf("/tmp/opt/chef")
+          FileUtils.rm_rf("/tmp/#{spec_machine_name}")
+        }
+
+        it "mounts volume" do
+          name = spec_machine_name
+
+          expect_converge {
+
+            machine name do
+              machine_options :docker_options => ubuntu_options.merge({
+                :volumes => "/tmp/#{name}:/tmp/spec",
+                :command => "/bin/bash -c 'echo \"Hi there!\" > /tmp/spec/i_exist' ; sleep 10000",
+              })
+              action :converge
+            end
+          }.not_to raise_error
+
+          expect(docker_driver.find_container(spec_machine_name)).not_to be_nil
+          expect(File.exists?("/tmp/#{name}/i_exist")).to be_truthy
+        end
+      end
+    end
+  end
+end