cvmanager

#!/usr/bin/env ruby

# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
# MA 02110-1301, USA.

require 'optparse'
require 'yaml'
require 'apipie-bindings'
require 'highline/import'
require 'time'

@defaults = {
  :noop        => false,
  :keep        => 5,
  :uri         => 'https://localhost/',
  :timeout     => 300,
  :user        => 'admin',
  :pass        => nil,
  :org         => 1,
  :lifecycle   => 1,
  :force       => false,
  :wait        => false,
  :sequential  => 0,
  :promote_cvs => false,
  :checkrepos  => false,
  :verbose     => false,
  :description => 'autopublish',
  :verify_ssl  => true,
}

@options = {
  :yamlfile  => 'cvmanager.yaml',
}

optparse = OptionParser.new do |opts|
  opts.banner = "Usage: #{opts.program_name} ACTION [options]"
  opts.version = "0.2"

  opts.separator ""
  opts.separator "#{opts.summary_indent}ACTION can be any of [clean,update,publish,promote]"
  opts.separator ""

  opts.on("-U", "--uri=URI", "URI to the Satellite") do |u|
    @options[:uri] = u
  end
  opts.on("-t", "--timeout=TIMEOUT", OptionParser::DecimalInteger, "Timeout value in seconds for any API calls. -1 means never timeout") do |t|
    @options[:timeout] = t
  end
  opts.on("-u", "--user=USER", "User to log in to Satellite") do |u|
    @options[:user] = u
  end
  opts.on("-p", "--pass=PASS", "Password to log in to Satellite") do |p|
    @options[:pass] = p
  end
  opts.on("-o", "--organization-id=ID", "ID of the Organization to manage CVs in") do |o|
    @options[:org] = o
  end
  opts.on("-k", "--keep=NUM", OptionParser::DecimalInteger, "how many unused versions should be kept") do |k|
    @options[:keep] = k
  end
  opts.on("-c", "--config=FILE", "configuration in YAML format") do |c|
    @options[:yamlfile] = c
  end
  opts.on("-l", "--to-lifecycle-environment=ID", OptionParser::DecimalInteger, "which LE should the promote be done to") do |l|
    @options[:lifecycle] = l
  end
  opts.on("-d", "--description=STRING", "Description to use for publish operations") do |d|
    @options[:description] = d
  end
  opts.on("-n", "--noop", "do not actually execute anything") do
    @options[:noop] = true
  end
  opts.on("-f", "--force", "force actions that otherwise would have been skipped") do
    @options[:force] = true
  end
  opts.on("--wait", "wait for started tasks to finish") do
    @options[:wait] = true
  end
  opts.on("--sequential [NUM]", OptionParser::DecimalInteger, "wait for each (or NUM) started task(s) to finish before starting the next one") do |s|
    @options[:wait] = true
    @options[:sequential] = s || 1
  end
  opts.on("--checkrepos", "check repository content was changed before publish") do
    @options[:checkrepos] = true
  end
  opts.on("--verbose", "Get verbose logs from cvmanager") do
    @options[:verbose] = true
  end
  opts.on("--no-verify-ssl", "don't verify SSL certs") do
    @options[:verify_ssl] = false
  end
end
optparse.parse!

if ARGV.empty?
  puts optparse.help
  exit
end

@yaml = YAML.load_file(@options[:yamlfile])

if @yaml.has_key?(:settings) and @yaml[:settings].is_a?(Hash)
  @yaml[:settings].each do |key,val|
    if not @options.has_key?(key)
      @options[key] = val
    end
  end
end

@defaults.each do |key,val|
  if not @options.has_key?(key)
    @options[key] = val
  end
end

if not @options[:user]
  @options[:user] = ask('Satellite username: ')
end

if not @options[:pass]
  @options[:pass] = ask('Satellite password: ') { |q| q.echo = false }
end

# sanitize non-complete config files
[:cv, :ccv].each do |key|
  if not @yaml.has_key?(key)
    @yaml[key] = {}
  end
end
[:publish, :promote].each do |key|
  if not @yaml.has_key?(key)
    @yaml[key] = []
  end
end

@api = ApipieBindings::API.new({:uri => @options[:uri], :username => @options[:user], :password => @options[:pass], :api_version => '2', :timeout => @options[:timeout]}, {:verify_ssl => @options[:verify_ssl]})

def puts_verbose(message)
  if @options[:verbose]
    puts "    [VERBOSE] #{message}"
  end
end

def clean()
  tasks = []
  cvs = []
  firstcv = true
  req = @api.resource(:content_views).call(:index, {:organization_id => @options[:org], :full_results => true})
  cvs.concat(req['results'])
  while (req['results'].length == req['per_page'].to_i)
    req = @api.resource(:content_views).call(:index, {:organization_id => @options[:org], :full_results => true, :per_page => req['per_page'], :page => req['page'].to_i+1})
    cvs.concat(req['results'])
  end

  # Sort the results so that we first process all Composite Content Views
  # and then the Content Views. This cleans up CCVs that might reference CVs.
  cvs.sort_by! { |cv| cv["composite"] ? 0 : 1 }

  # Parse the CV
  cvs.each do |cv|
    keep = []
    puts "Inspecting #{cv['label']}"
    # The first Content View we encounter indicates that we processed all Composite Content Views.
    # Wait until the CCV tasks are finished.
    if not cv["composite"] and firstcv and tasks.any?
      puts "First Content View, waiting for the Composite Content View tasks to be completed"
      firstcv = false
      wait(tasks)
    end
    cv['versions'].sort_by { |v| v['version'].to_f }.reverse.each do |version|
      if not version['environment_ids'].empty?
        puts_verbose " #{cv['label']} v#{version['version']} is published to the following environments: #{version['environment_ids']}, skipping."
        next
      else
        puts_verbose " #{cv['label']} v#{version['version']} is not used by any environment."
      end
      version_details = @api.resource(:content_view_versions).call(:show, {:id => version['id']})
      if not version_details['composite_content_view_ids'].empty?
        puts_verbose " #{cv['label']} v#{version['version']} is used by the following composite contentviews: #{version_details['composite_content_view_ids']}, skipping."
        next
      else
        puts_verbose " #{cv['label']} v#{version['version']} is not used by any composite contentviews."
      end
      if keep.length < @options[:keep]
        keep.push(version)
        puts " keeping #{version['version']}"
      else
        puts " removing #{version['version']}"
        if not @options[:noop]
          begin
            req = @api.resource(:content_view_versions).call(:destroy, {:id => version['id']})
            tasks << req['id']
          rescue RestClient::ExceptionWithResponse => err
            puts " removal of #{cv['label']}, id #{cv['id']} v#{version['version']} failed. Error message '#{err.response}'"
          end
          if @options[:sequential] > 0 and tasks.length >= @options[:sequential]
            tasks = wait(tasks)
            puts "  removed content view version with id #{version['id']}"
          else
            puts "  [task enqueued] removed content view version with id #{version['id']}"
          end
        else
          puts "  [noop] removed content view version with id #{version['id']}"
        end
      end
    end
  end

  wait(tasks)
end

def checktask(task, last_date)
  task_completed_at = Time.xmlschema(task['ended_at']) rescue Time.parse(task['ended_at'])
  if task_completed_at >= last_date
    puts_verbose "Past task was completed at #{task_completed_at}, which is after #{last_date}"
    if task['humanized']['output'] == "No new packages."
      puts_verbose "#{task['humanized']['output']} This past task will NOT trigger a Publish."
      return false
    else
      puts_verbose "#{task['humanized']['output']} This past task will trigger a Publish."
      return true
    end
  end
  puts_verbose "#{task_completed_at} is before #{last_date}, will NOT trigger a Publish."
  return false
end

def checkoldtask(task, last_date)
  task_completed_at = Time.xmlschema(task['ended_at']) rescue Time.parse(task['ended_at'])
  if task_completed_at >= last_date
    puts_verbose "Past task was completed at #{task_completed_at}, which is after #{last_date}. Continue to inspect."
    return false
  end
  puts_verbose "#{task_completed_at} is before #{last_date}, no sense to continue."
  return true
end

def update()
  tasks = []

  ccvs = []
  req = @api.resource(:content_views).call(:index, {:organization_id => @options[:org], :full_results => true})
  ccvs.concat(req['results'])
  while (req['results'].length == req['per_page'].to_i)
    req = @api.resource(:content_views).call(:index, {:organization_id => @options[:org], :full_results => true, :per_page => req['per_page'], :page => req['page'].to_i+1})
    ccvs.concat(req['results'])
  end

  ccvs.each do |ccv|
    next if ! ccv['composite']

    was_updated = false

    puts "Inspecting #{ccv['label']}"

    # loop through the components and check if they are uptodate
    ids = Array.new(ccv['component_ids'])
    ccv['components'].each do |component|
      puts " Checking #{component['content_view']['label']}"

      # get the desired version for this component from the YAML
      # either the version for the component in this CCV is set
      # or it is set globally
      # never touch non-mentioned components
      if @yaml[:ccv].is_a?(Hash) and @yaml[:ccv].has_key?(ccv['label']) and @yaml[:ccv][ccv['label']].has_key?(component['content_view']['label'])
        desired_version = @yaml[:ccv][ccv['label']][component['content_view']['label']]
        puts_verbose "  Desired version #{desired_version} found in CCV"
      elsif @yaml[:cv].is_a?(Hash) and @yaml[:cv].has_key?(component['content_view']['label'])
        desired_version = @yaml[:cv][component['content_view']['label']]
        puts_verbose "  Desired version #{desired_version} found in CV"
      else
        puts_verbose "  Desired version not found, skipping"
        next
      end

      # instead of hard-coding the versions, the user can also specify "latest"
      if desired_version == 'latest'
        cvversions = @api.resource(:content_view_versions).call(:index, {:content_view_id => component['content_view']['id']})
        cvversions = cvversions['results'].sort_by { |v| v['version'].to_f }.reverse
        desired_version = cvversions[0]['version']
        puts_verbose "  Found #{desired_version} as the 'latest' version"
      end

      # if the version of the component does not match the one the user requested update it
      if component['version'].to_s != desired_version.to_s
        puts "  Updating from #{component['version']} to #{desired_version}"
        oldids = ids.dup
        ids.delete(component['id'])
        cvversions = @api.resource(:content_view_versions).call(:index, {:content_view_id => component['content_view']['id'], :version => desired_version})
        desired_version_id = cvversions['results'][0]['id']
        ids.push(desired_version_id)
        puts "  Old components: #{oldids}"
        puts "  New components: #{ids}"
        # do the update
        was_updated = true
      end
    end

    if was_updated
      #Change the member content view versions; We do this once at the end, so if there was multiple CV changes, its only one call
      puts " Committing new content view versions"
      if not @options[:noop]
        @api.resource(:content_views).call(:update, {:id => ccv['id'], :component_ids => ids })
      else
        puts "  [noop] updating CCV #{ccv['id']} to #{ids}"
      end
      puts " Publishing new version as CCV had changes"
      # do the publish
      if not @options[:noop]
        req = @api.resource(:content_views).call(:publish, {:id => ccv['id'], :description => @options[:description]})
        tasks << req['id']
        if @options[:sequential] > 0 and tasks.length >= @options[:sequential]
          tasks = wait(tasks)
        end
      else
        puts "  [noop] publishing CCV #{ccv['id']}"
      end
    end
  end

  wait(tasks)
end

def promote()
  tasks = []

  ccvs = []
  req = @api.resource(:content_views).call(:index, {:organization_id => @options[:org], :full_results => true})
  ccvs.concat(req['results'])
  while (req['results'].length == req['per_page'].to_i)
    req = @api.resource(:content_views).call(:index, {:organization_id => @options[:org], :full_results => true, :per_page => req['per_page'], :page => req['page'].to_i+1})
    ccvs.concat(req['results'])
  end

  ccvs.each do |ccv|
    next if not ccv['composite'] and not @options[:promote_cvs]
    next if not @yaml[:promote].include?(ccv['label']) and not @yaml[:promote].include?("all")

    puts "Inspecting #{ccv['label']}"

    latest_version = ccv['versions'].sort_by { |v| v['version'].to_f }.reverse[0]
    next if ! latest_version

    if not latest_version['environment_ids'].include?(@options[:lifecycle])
      puts " Promoting latest version to lifecycle-environment #{@options[:lifecycle]}"
      if not @options[:noop]
        req = @api.resource(:content_view_versions).call(:promote, {:id => latest_version['id'], :environment_id => @options[:lifecycle], :force => @options[:force]})
        tasks << req['id']
        wait([req['id']]) if @options[:sequential]
      else
        puts " [noop] Promoting #{latest_version['id']} to lifecycle-environment #{@options[:lifecycle]}"
      end
    end
  end

  wait(tasks)
end

def publish()
  tasks = []

  cvs = []
  req = @api.resource(:content_views).call(:index, {:organization_id => @options[:org], :full_results => true})
  cvs.concat(req['results'])
  while (req['results'].length == req['per_page'].to_i)
    req = @api.resource(:content_views).call(:index, {:organization_id => @options[:org], :full_results => true, :per_page => req['per_page'], :page => req['page'].to_i+1})
    cvs.concat(req['results'])
  end

  cvs.each do |cv|
    # if CV is not listed in csv, skip
    puts_verbose "Checking Content View #{cv['label']}"
    next if not @yaml[:publish].include?(cv['label'])

    # if the CV is listed, write it
    puts "Inspecting #{cv['label']} as listed in CSV"

    # initialize variables
    needs_publish = false
    # check if this CV has ever been published
    if cv.has_key?('versions') and cv['versions'].length > 0
      last_ver_published = cv['versions'].sort_by{|ver| ver['published']}.last['published']
      # if published with a version, take last version published time
      cv_last_published = Time.xmlschema(last_ver_published) rescue Time.parse(last_ver_published)
    elsif cv.has_key?('last_published') and cv['last_published']
      # if published without version, save last published time
      last_ver_published = cv['versions'].sort_by{|ver| ver['published']}.last['published']
        cv_last_published = Time.xmlschema(cv['last_published']) rescue Time.parse(cv['last_published'])
    else
      # if not published, save 0 as published time
      cv_last_published = Time.new(0)
    end
    # Check every repo in the CV
    cv['repository_ids'].each do |repo_id|
      # get repo data
      repo = @api.resource(:repositories).call(:show, {:id => repo_id})
      # check if the last sync has been ever completed
      if repo.has_key?('last_sync') and repo['last_sync'] and repo['last_sync'].has_key?('ended_at') and repo['last_sync']['ended_at']
        # if sync completed, save last end sync time
        repo_last_sync = Time.xmlschema(repo['last_sync']['ended_at']) rescue Time.parse(repo['last_sync']['ended_at'])
      else
        # if sync never completed, save 0 as sync time
        repo_last_sync = Time.new(0)
      end
      # check if last repo sync time happened after last CV publish
      if repo_last_sync > cv_last_published
        # if checkrepo option is on, a deeper check will be done
        if @options[:checkrepos]
          # write some info about repo that we are checking
          puts " repo #{repo['label']} (id: #{repo['id']}) seems newer than CV #{cv['label']} (id: #{cv['id']}), checking if sync contains new packages."
          # get last sync repo output from foreman task
          sync_task = @api.resource(:foreman_tasks).call(:show, {:id => repo['last_sync']['id']})
          # check if the package contains "No new package.". The opposite is a number of packages.
          if sync_task['humanized']['output'] == "No new packages."
            # no new packages in the last sync. getting the older
            puts " '#{sync_task['humanized']['output']}' Found in last sync task, will search now for past sync tasks for packages not in the CV."
            # Initialize to start the iteration, garbage collection needed and performed below.
            taskreq = nil
            begin
              # loop over tasks. first round is nil
              while (taskreq == nil or taskreq['results'].length == taskreq['per_page'].to_i)
                # if nil, new data will be taken from zero. otherwise take next page
                if (taskreq == nil)
                  taskreq = @api.resource(:foreman_tasks).call(:index, {:search => 'label=Actions::Katello::Repository::Sync and result=success', :full_results => true, :per_page => 10, :sort_by => :ended_at})
                  puts_verbose "Inspecing sync tasks to #10"
                else
                  taskreq = @api.resource(:foreman_tasks).call(:index, {:search => 'label=Actions::Katello::Repository::Sync and result=success', :full_results => true, :per_page => taskreq['per_page'], :sort_by => :ended_at, :page => taskreq['page'].to_i+1})
                  puts_verbose "Inspecing sync tasks to ##{taskreq['per_page']}"
                end
                # iterate over the results
                taskreq['results'].each do |tasker|
                  # if the task id is the same as the task extracted before loop, continue
                  next if tasker['id'] == sync_task['id']
                    # if task has been completed after cv publish, check if the task that we are checking are for the repo id we are looking for
                    if tasker['input']['repository']['id'] == repo['id']
                      puts_verbose "Found past task that matches repo id: #{tasker['input']['repository']['id']}"
                      # Avoid to check old publications
                      if ( checkoldtask(tasker, cv_last_published) )
                         raise "skip_parsed"
                      end
                      if ( checktask(tasker, cv_last_published) )
                        needs_publish = true
                        raise "publish"
                      end
                    end
                end
              end
              # if we reach this point, all of the tasks has been parsed:
              # no repo that would trigger a publish has been found AND the task date is newer then last publish
              puts "  No more tasks are found, Publish will be SKIPPED."
              raise "skip_parsed"
            rescue => error
              puts_verbose "  Repo: '#{repo['label']}'. Exiting from loop, reason: #{error.inspect}"
            end
            # forcing also garbage collection to run. 250MB of memory estimated.
            taskreq = nil
          else
            puts " #{sync_task['humanized']['output']} Will Publish."
            needs_publish = true
          end
        else
          puts " repo #{repo['label']} (id: #{repo['id']}) seems newer than CV #{cv['label']} (id: #{cv['id']}) (#{repo_last_sync} > #{cv_last_published}), lets publish"
          needs_publish = true
        end
      end
    end
    # check if --force has been used. if so, force publish
    if not needs_publish and @options[:force]
      needs_publish = true
      puts " forced publish, even if there were no changes"
    end
    # finally if the CV has to be published, do it
    if needs_publish
      puts "Publishing #{cv['label']}"
      if not @options[:noop]
        req = @api.resource(:content_views).call(:publish, {:id => cv['id'], :description => @options[:description]})
        tasks << req['id']
        if @options[:sequential] > 0 and tasks.length >= @options[:sequential]
          tasks = wait(tasks)
        end
      else
        puts " [noop] published #{cv['label']}"
      end
    end
  end
  wait(tasks)
end

def wait(tasks)
  need_wait = tasks
  resumed_tasks = []
  if @options[:wait]
    last_need_wait = []
    silence = false
    wait_secs = 0
    while not need_wait.empty?
      if wait_secs < 60
        wait_secs += 10
      end
      puts "waiting #{wait_secs} for pending tasks: #{need_wait}" unless silence
      sleep wait_secs
      last_need_wait = need_wait
      need_wait = []
      tasks.each do |task_id|
        req = @api.resource(:foreman_tasks).call(:show, {:id => task_id})
        if req['pending']
          need_wait << task_id
        end
        if req['state'] == 'paused' and not resumed_tasks.include?(task_id)
          puts "Resuming paused task #{task_id}"
          @api.resource(:foreman_tasks).call(:bulk_resume, {:task_ids => [task_id]})
          resumed_tasks << task_id
        end
      end
      if (wait_secs >= 60 and (last_need_wait.sort == need_wait.sort))
          puts "Silencing output until there's a task status change..." unless silence
          silence = true
      else
          silence = false
      end
    end
  end
  return need_wait
end

action = ARGV.shift

if action == 'clean'
  clean
elsif action == 'update'
  update
elsif action == 'promote'
  promote
elsif action == 'publish'
  publish
end