From 06812f640aa0c4632ec24b3d1433d7ccb9b66822 Mon Sep 17 00:00:00 2001 From: Scott Parker Date: Fri, 27 Apr 2018 10:29:50 -0600 Subject: [PATCH 1/7] Updated to work with Satellite 6.3 which is now automatically updating composite views with the latest versions of content views. --- cvmanager | 503 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 503 insertions(+) diff --git a/cvmanager b/cvmanager index d4a78d7..8280f11 100755 --- a/cvmanager +++ b/cvmanager @@ -1,5 +1,10 @@ #!/usr/bin/env ruby +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU GeneralPublic License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version.#!/usr/bin/env ruby + # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or @@ -14,6 +19,504 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, # MA 02110-1301, USA. +# +# Updated to work with Satellite 6.3 [update composite views section] + +require 'optparse' +require 'yaml' +require 'apipie-bindings' +require 'highline/import' +require 'time' + +@defaults = { + :noop => false, + :keep => 5, + :uri => 'https://localhost/', + :timeout => 300, + :user => 'admin', + :pass => nil, + :org => 1, + :lifecycle => 1, + :force => false, + :wait => false, + :sequential => 0, + :promote_cvs => false, + :checkrepos => false, + :verbose => false, + :description => 'autopublish', + :verify_ssl => true, +} + +@options = { + :yamlfile => 'cvmanager.yaml', +} + +optparse = OptionParser.new do |opts| + opts.banner = "Usage: #{opts.program_name} ACTION [options]" + opts.version = "0.1" + + opts.separator "" + opts.separator "#{opts.summary_indent}ACTION can be any of [clean,update,publish,promote]" + opts.separator "" + + opts.on("-U", "--uri=URI", "URI to the Satellite") do |u| + @options[:uri] = u + end + opts.on("-t", "--timeout=TIMEOUT", OptionParser::DecimalInteger, "Timeout value in seconds for any API calls. -1 means never timeout") do |t| + @options[:timeout] = t + end + opts.on("-u", "--user=USER", "User to log in to Satellite") do |u| + @options[:user] = u + end + opts.on("-p", "--pass=PASS", "Password to log in to Satellite") do |p| + @options[:pass] = p + end + opts.on("-o", "--organization-id=ID", "ID of the Organization to manage CVs in") do |o| + @options[:org] = o + end + opts.on("-k", "--keep=NUM", OptionParser::DecimalInteger, "how many unused versions should be kept") do |k| + @options[:keep] = k + end + opts.on("-c", "--config=FILE", "configuration in YAML format") do |c| + @options[:yamlfile] = c + end + opts.on("-l", "--to-lifecycle-environment=ID", OptionParser::DecimalInteger, "which LE should the promote be done to") do |l| + @options[:lifecycle] = l + end + opts.on("-d", "--description=STRING", "Description to use for publish operations") do |d| + @options[:description] = d + end + opts.on("-n", "--noop", "do not actually execute anything") do + @options[:noop] = true + end + opts.on("-f", "--force", "force actions that otherwise would have been skipped") do + @options[:force] = true + end + opts.on("--wait", "wait for started tasks to finish") do + @options[:wait] = true + end + opts.on("--sequential [NUM]", OptionParser::DecimalInteger, "wait for each (or NUM) started task(s) to finish before starting the next one") do |s| + @options[:wait] = true + @options[:sequential] = s || 1 + end + opts.on("--checkrepos", "check repository content was changed before publish") do + @options[:checkrepos] = true + end + opts.on("--verbose", "Get verbose logs from cvmanager") do + @options[:verbose] = true + end + opts.on("--no-verify-ssl", "don't verify SSL certs") do + @options[:verify_ssl] = false + end +end +optparse.parse! + +if ARGV.empty? + puts optparse.help + exit +end + +@yaml = YAML.load_file(@options[:yamlfile]) + +if @yaml.has_key?(:settings) and @yaml[:settings].is_a?(Hash) + @yaml[:settings].each do |key,val| + if not @options.has_key?(key) + @options[key] = val + end + end +end + +@defaults.each do |key,val| + if not @options.has_key?(key) + @options[key] = val + end +end + +if not @options[:user] + @options[:user] = ask('Satellite username: ') +end + +if not @options[:pass] + @options[:pass] = ask('Satellite password: ') { |q| q.echo = false } +end + +# sanitize non-complete config files +[:cv, :ccv].each do |key| + if not @yaml.has_key?(key) + @yaml[key] = {} + end +end +[:publish, :promote].each do |key| + if not @yaml.has_key?(key) + @yaml[key] = [] + end +end + +@api = ApipieBindings::API.new({:uri => @options[:uri], :username => @options[:user], :password => @options[:pass], :api_version => '2', :timeout => @options[:timeout]}, {:verify_ssl => @options[:verify_ssl]}) + +def puts_verbose(message) + if @options[:verbose] + puts " [VERBOSE] #{message}" + end +end + +def clean() + tasks = [] + cvs = [] + req = @api.resource(:content_views).call(:index, {:organization_id => @options[:org], :full_results => true}) + cvs.concat(req['results']) + while (req['results'].length == req['per_page'].to_i) + req = @api.resource(:content_views).call(:index, {:organization_id => @options[:org], :full_results => true, :per_page => req['per_page'], :page => req['page'].to_i+1}) + cvs.concat(req['results']) + end + + cvs.each do |cv| + keep = [] + puts "Inspecting #{cv['name']}" + cv['versions'].sort_by { |v| v['version'].to_f }.reverse.each do |version| + if not version['environment_ids'].empty? + puts_verbose " #{cv['name']} v#{version['version']} is published to the following environments: #{version['environment_ids']}, skipping." + next + end + version_details = @api.resource(:content_view_versions).call(:show, {:id => version['id']}) + if not version_details['composite_content_view_ids'].empty? + puts_verbose " #{cv['name']} v#{version['version']} is used by the following composite contentviews: #{version_details['composite_content_view_ids']}, skipping." + next + end + if keep.length < @options[:keep] + keep.push(version) + puts " keeping #{version['version']}" + else + puts " removing #{version['version']}" + if not @options[:noop] + req = @api.resource(:content_view_versions).call(:destroy, {:id => version['id']}) + tasks << req['id'] + if @options[:sequential] > 0 and tasks.length >= @options[:sequential] + tasks = wait(tasks) + end + else + puts " [noop] would delete content view version with id #{version['id']}" + end + end + end + end + + wait(tasks) +end + +def checktask(task, last_date) + task_completed_at = Time.xmlschema(task['ended_at']) rescue Time.parse(task['ended_at']) + if task_completed_at >= last_date + puts_verbose "Past task was completed at #{task_completed_at}, which is after #{last_date}" + if task['humanized']['output'] == "No new packages." + puts_verbose "#{task['humanized']['output']} This past task will NOT trigger a Publish." + return false + else + puts_verbose "#{task['humanized']['output']} This past task will trigger a Publish." + return true + end + end + puts_verbose "#{task_completed_at} is before #{last_date}, will NOT trigger a Publish." + return false +end + +def checkoldtask(task, last_date) + task_completed_at = Time.xmlschema(task['ended_at']) rescue Time.parse(task['ended_at']) + if task_completed_at >= last_date + puts_verbose "Past task was completed at #{task_completed_at}, which is after #{last_date}. Continue to inspect." + return false + end + puts_verbose "#{task_completed_at} is before #{last_date}, no sense to continue." + return true +end + +def update() + tasks = [] + + ccvs = [] + req = @api.resource(:content_views).call(:index, {:organization_id => @options[:org], :full_results => true}) + ccvs.concat(req['results']) + while (req['results'].length == req['per_page'].to_i) + req = @api.resource(:content_views).call(:index, {:organization_id => @options[:org], :full_results => true, :per_page => req['per_page'], :page => req['page'].to_i+1}) + ccvs.concat(req['results']) + end + + ccvs.each do |ccv| + next if ! ccv['composite'] + + puts "Inspecting #{ccv['name']}" + + # loop through the components and check if they are uptodate + ids = Array.new(ccv['component_ids']) + ccv['components'].each do |component| + puts " Checking #{component['content_view']['name']}" + + # get the desired version for this component from the YAML + # either the version for the component in this CCV is set + # or it is set globally + # never touch non-mentioned components + if @yaml[:ccv].is_a?(Hash) and @yaml[:ccv].has_key?(ccv['label']) and @yaml[:ccv][ccv['label']].has_key?(component['content_view']['name']) + desired_version = @yaml[:ccv][ccv['label']][component['content_view']['name']] + puts_verbose " Desired version #{desired_version} found in CCV" + elsif @yaml[:cv].is_a?(Hash) and @yaml[:cv].has_key?(component['content_view']['name']) + desired_version = @yaml[:cv][component['content_view']['name']] + puts_verbose " Desired version #{desired_version} found in CV" + else + puts_verbose " Desired version not found, skipping" + next + end + + # instead of hard-coding the versions, the user can also specify "latest" + if desired_version == 'latest' + cvversions = @api.resource(:content_view_versions).call(:index, {:content_view_id => component['content_view']['id']}) + cvversions = cvversions['results'].sort_by { |v| v['version'].to_f }.reverse + desired_version = cvversions[0]['version'] + puts_verbose " Found #{desired_version} as the 'latest' version" + end + # end loop + end + + #Change the member content view versions; We do this once at the end, so if there was multiple CV changes, its only one call + puts " Committing new content view versions" + if not @options[:noop] + @api.resource(:content_views).call(:update, {:id => ccv['id'], :component_ids => ids } + else + puts " [noop] updating CCV #{ccv['id']} to #{ids}" + end + puts " Publishing new version as CCV had changes" + # do the publish + if not @options[:noop] + req = @api.resource(:content_views).call(:publish, {:id => ccv['id'], :description => @options[:description]}) + tasks << req['id'] + if @options[:sequential] > 0 and tasks.length >= @options[:sequential] + tasks = wait(tasks) + end + else + puts " [noop] publishing CCV #{ccv['id']}" + end + # end loop + end + + wait(tasks) +end + +def promote() + tasks = [] + + ccvs = [] + req = @api.resource(:content_views).call(:index, {:organization_id => @options[:org], :full_results => true}) + ccvs.concat(req['results']) + while (req['results'].length == req['per_page'].to_i) + req = @api.resource(:content_views).call(:index, {:organization_id => @options[:org], :full_results => true, :per_page => req['per_page'], :page => req['page'].to_i+1}) + ccvs.concat(req['results']) + end + + ccvs.each do |ccv| + next if not ccv['composite'] and not @options[:promote_cvs] + next if not @yaml[:promote].include?(ccv['name']) and not @yaml[:promote].include?("all") + + puts "Inspecting #{ccv['name']}" + + latest_version = ccv['versions'].sort_by { |v| v['version'].to_f }.reverse[0] + next if ! latest_version + + if not latest_version['environment_ids'].include?(@options[:lifecycle]) + puts " Promoting latest version to lifecycle-environment #{@options[:lifecycle]}" + if not @options[:noop] + req = @api.resource(:content_view_versions).call(:promote, {:id => latest_version['id'], :environment_id => @options[:lifecycle], :force => @options[:force]}) + tasks << req['id'] + wait([req['id']]) if @options[:sequential] + else + puts " [noop] Promoting #{latest_version['id']} to lifecycle-environment #{@options[:lifecycle]}" + end + end + end + + wait(tasks) +end + +def publish() + tasks = [] + + cvs = [] + req = @api.resource(:content_views).call(:index, {:organization_id => @options[:org], :full_results => true}) + cvs.concat(req['results']) + while (req['results'].length == req['per_page'].to_i) + req = @api.resource(:content_views).call(:index, {:organization_id => @options[:org], :full_results => true, :per_page => req['per_page'], :page => req['page'].to_i+1}) + cvs.concat(req['results']) + end + + cvs.each do |cv| + # if CV is not listed in csv, skip + puts_verbose "Checking Content View #{cv['name']}" + next if not @yaml[:publish].include?(cv['name']) + + # if the CV is listed, write it + puts "Inspecting #{cv['name']} as listed in CSV" + + # initialize variables + needs_publish = false + # check if this CV has ever been published + if cv.has_key?('versions') and cv['versions'].length > 0 + last_ver_published = cv['versions'].sort_by{|ver| ver['published']}.last['published'] + # if published with a version, take last version published time + cv_last_published = Time.xmlschema(last_ver_published) rescue Time.parse(last_ver_published) + elsif cv.has_key?('last_published') and cv['last_published'] + # if published without version, save last published time + last_ver_published = cv['versions'].sort_by{|ver| ver['published']}.last['published'] + cv_last_published = Time.xmlschema(cv['last_published']) rescue Time.parse(cv['last_published']) + else + # if not published, save 0 as published time + cv_last_published = Time.new(0) + end + # Check every repo in the CV + cv['repository_ids'].each do |repo_id| + # get repo data + repo = @api.resource(:repositories).call(:show, {:id => repo_id}) + # check if the last sync has been ever completed + if repo.has_key?('last_sync') and repo['last_sync'] and repo['last_sync'].has_key?('ended_at') and repo['last_sync']['ended_at'] + # if sync completed, save last end sync time + repo_last_sync = Time.xmlschema(repo['last_sync']['ended_at']) rescue Time.parse(repo['last_sync']['ended_at']) + else + # if sync never completed, save 0 as sync time + repo_last_sync = Time.new(0) + end + # check if last repo sync time happened after last CV publish + if repo_last_sync > cv_last_published + # if checkrepo option is on, a deeper check will be done + if @options[:checkrepos] + # write some info about repo that we are checking + puts " repo #{repo['label']} (id: #{repo['id']}) seems newer than CV #{cv['name']} (id: #{cv['id']}), checking if sync contains new packages." + # get last sync repo output from foreman task + sync_task = @api.resource(:foreman_tasks).call(:show, {:id => repo['last_sync']['id']}) + # check if the package contains "No new package.". The opposite is a number of packages. + if sync_task['humanized']['output'] == "No new packages." + # no new packages in the last sync. getting the older + puts " '#{sync_task['humanized']['output']}' Found in last sync task, will search now for past sync tasks for packages not in the CV." + # Initialize to start the iteration, garbage collection needed and performed below. + taskreq = nil + begin + # loop over tasks. first round is nil + while (taskreq == nil or taskreq['results'].length == taskreq['per_page'].to_i) + # if nil, new data will be taken from zero. otherwise take next page + if (taskreq == nil) + taskreq = @api.resource(:foreman_tasks).call(:index, {:search => 'label=Actions::Katello::Repository::Sync and result=success', :full_results => true, :per_page => 10, :sort_by => :ended_at}) + puts_verbose "Inspecing sync tasks to #10" + else + taskreq = @api.resource(:foreman_tasks).call(:index, {:search => 'label=Actions::Katello::Repository::Sync and result=success', :full_results => true, :per_page => taskreq['per_page'], :sort_by => :ended_at, :page => taskreq['page'].to_i+1}) + puts_verbose "Inspecing sync tasks to ##{taskreq['per_page']}" + end + # iterate over the results + taskreq['results'].each do |tasker| + # if the task id is the same as the task extracted before loop, continue + next if tasker['id'] == sync_task['id'] + # if task has been completed after cv publish, check if the task that we are checking are for the repo id we are looking for + if tasker['input']['repository']['id'] == repo['id'] + puts_verbose "Found past task that matches repo id: #{tasker['input']['repository']['id']}" + # Avoid to check old publications + if ( checkoldtask(tasker, cv_last_published) ) + raise "skip_parsed" + end + if ( checktask(tasker, cv_last_published) ) + needs_publish = true + raise "publish" + end + end + end + end + # if we reach this point, all of the tasks has been parsed: + # no repo that would trigger a publish has been found AND the task date is newer then last publish + puts " No more tasks are found, Publish will be SKIPPED." + raise "skip_parsed" + rescue => error + puts_verbose " Repo: '#{repo['label']}'. Exiting from loop, reason: #{error.inspect}" + end + # forcing also garbage collection to run. 250MB of memory estimated. + taskreq = nil + else + puts " #{sync_task['humanized']['output']} Will Publish." + needs_publish = true + end + else + puts " repo #{repo['label']} (id: #{repo['id']}) seems newer than CV #{cv['name']} (id: #{cv['id']}) (#{repo_last_sync} > #{cv_last_published}), lets publish" + needs_publish = true + end + end + end + # check if --force has been used. if so, force publish + if not needs_publish and @options[:force] + needs_publish = true + puts " forced publish, even if there were no changes" + end + # finally if the CV has to be published, do it + if needs_publish + puts "Publishing #{cv['name']}" + if not @options[:noop] + req = @api.resource(:content_views).call(:publish, {:id => cv['id'], :description => @options[:description]}) + tasks << req['id'] + if @options[:sequential] > 0 and tasks.length >= @options[:sequential] + tasks = wait(tasks) + end + else + puts " [noop] published #{cv['name']}" + end + end + end + wait(tasks) +end + +def wait(tasks) + need_wait = tasks + if @options[:wait] + last_need_wait = [] + silence = false + wait_secs = 0 + while not need_wait.empty? + if wait_secs < 60 + wait_secs += 10 + end + puts "waiting #{wait_secs} for pending tasks: #{need_wait}" unless silence + sleep wait_secs + last_need_wait = need_wait + need_wait = [] + tasks.each do |task_id| + req = @api.resource(:foreman_tasks).call(:show, {:id => task_id}) + if req['pending'] + need_wait << task_id + end + end + if (wait_secs >= 60 and (last_need_wait.sort == need_wait.sort)) + puts "Silencing output until there's a task status change..." unless silence + silence = true + else + silence = false + end + end + end + return need_wait +end + +action = ARGV.shift + +if action == 'clean' + clean +elsif action == 'update' + update +elsif action == 'promote' + promote +elsif action == 'publish' + publish +end +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, +# MA 02110-1301, USA. require 'optparse' require 'yaml' From e298e1447570ef9604561aefaab658e5c844c9ab Mon Sep 17 00:00:00 2001 From: Scott Parker Date: Fri, 27 Apr 2018 10:32:15 -0600 Subject: [PATCH 2/7] added comments about using cvmanager with Satellite 6.3. --- README.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/README.md b/README.md index 8df9054..2181455 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,11 @@ +# Satellite 6.3 Update + +Satellite 6.3 no longer provide apipie via the repositories. To use this utility, install apipie. + +``` +gem install apipie +``` + # cvmanager For automation of some common tasks related to Content Views we created a tool called `cvmanager`. It consists of a Ruby script (`cvmanager`) and a YAML-formatted configuration file (`cvmanager.yaml`). The various features are described in the following chapters. From 375c7a47fd5f4dfdec2373e352060232c936c834 Mon Sep 17 00:00:00 2001 From: Scott Parker Date: Fri, 27 Apr 2018 10:33:47 -0600 Subject: [PATCH 3/7] add comment about this version will not work with 6.2 or less. --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 2181455..a22d685 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,8 @@ Satellite 6.3 no longer provide apipie via the repositories. To use this utilit gem install apipie ``` +This version will no longer work with Satellite 6.2 or less. + # cvmanager For automation of some common tasks related to Content Views we created a tool called `cvmanager`. It consists of a Ruby script (`cvmanager`) and a YAML-formatted configuration file (`cvmanager.yaml`). The various features are described in the following chapters. From 54d2a87dfd7fd09fcf5bdb0a2ae42548df52f3a3 Mon Sep 17 00:00:00 2001 From: Scott Parker Date: Fri, 27 Apr 2018 10:36:56 -0600 Subject: [PATCH 4/7] add origin to show where the original code came from. --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index a22d685..0ec0497 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,10 @@ gem install apipie This version will no longer work with Satellite 6.2 or less. +# Origin + +This was forked from katello-cvmanger https://github.com/RedHatSatellite/katello-cvmanager and updated to work with Satellite 6.3. + # cvmanager For automation of some common tasks related to Content Views we created a tool called `cvmanager`. It consists of a Ruby script (`cvmanager`) and a YAML-formatted configuration file (`cvmanager.yaml`). The various features are described in the following chapters. From 361bd77c9891f6be026845a92c1532778b097904 Mon Sep 17 00:00:00 2001 From: Scott Parker Date: Fri, 27 Apr 2018 17:12:15 -0600 Subject: [PATCH 5/7] re-uploaded the file --- cvmanager | 525 +----------------------------------------------------- 1 file changed, 5 insertions(+), 520 deletions(-) diff --git a/cvmanager b/cvmanager index 8280f11..454142b 100755 --- a/cvmanager +++ b/cvmanager @@ -1,10 +1,5 @@ #!/usr/bin/env ruby -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU GeneralPublic License as published by -# the Free Software Foundation; either version 2 of the License, or -# (at your option) any later version.#!/usr/bin/env ruby - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or @@ -19,8 +14,6 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, # MA 02110-1301, USA. -# -# Updated to work with Satellite 6.3 [update composite views section] require 'optparse' require 'yaml' @@ -54,7 +47,7 @@ require 'time' optparse = OptionParser.new do |opts| opts.banner = "Usage: #{opts.program_name} ACTION [options]" opts.version = "0.1" - + opts.separator "" opts.separator "#{opts.summary_indent}ACTION can be any of [clean,update,publish,promote]" opts.separator "" @@ -243,7 +236,9 @@ def update() ccvs.each do |ccv| next if ! ccv['composite'] - + + was_updated = false + puts "Inspecting #{ccv['name']}" # loop through the components and check if they are uptodate @@ -276,517 +271,7 @@ def update() # end loop end - #Change the member content view versions; We do this once at the end, so if there was multiple CV changes, its only one call - puts " Committing new content view versions" - if not @options[:noop] - @api.resource(:content_views).call(:update, {:id => ccv['id'], :component_ids => ids } - else - puts " [noop] updating CCV #{ccv['id']} to #{ids}" - end - puts " Publishing new version as CCV had changes" - # do the publish - if not @options[:noop] - req = @api.resource(:content_views).call(:publish, {:id => ccv['id'], :description => @options[:description]}) - tasks << req['id'] - if @options[:sequential] > 0 and tasks.length >= @options[:sequential] - tasks = wait(tasks) - end - else - puts " [noop] publishing CCV #{ccv['id']}" - end - # end loop - end - - wait(tasks) -end - -def promote() - tasks = [] - - ccvs = [] - req = @api.resource(:content_views).call(:index, {:organization_id => @options[:org], :full_results => true}) - ccvs.concat(req['results']) - while (req['results'].length == req['per_page'].to_i) - req = @api.resource(:content_views).call(:index, {:organization_id => @options[:org], :full_results => true, :per_page => req['per_page'], :page => req['page'].to_i+1}) - ccvs.concat(req['results']) - end - - ccvs.each do |ccv| - next if not ccv['composite'] and not @options[:promote_cvs] - next if not @yaml[:promote].include?(ccv['name']) and not @yaml[:promote].include?("all") - - puts "Inspecting #{ccv['name']}" - - latest_version = ccv['versions'].sort_by { |v| v['version'].to_f }.reverse[0] - next if ! latest_version - - if not latest_version['environment_ids'].include?(@options[:lifecycle]) - puts " Promoting latest version to lifecycle-environment #{@options[:lifecycle]}" - if not @options[:noop] - req = @api.resource(:content_view_versions).call(:promote, {:id => latest_version['id'], :environment_id => @options[:lifecycle], :force => @options[:force]}) - tasks << req['id'] - wait([req['id']]) if @options[:sequential] - else - puts " [noop] Promoting #{latest_version['id']} to lifecycle-environment #{@options[:lifecycle]}" - end - end - end - - wait(tasks) -end - -def publish() - tasks = [] - - cvs = [] - req = @api.resource(:content_views).call(:index, {:organization_id => @options[:org], :full_results => true}) - cvs.concat(req['results']) - while (req['results'].length == req['per_page'].to_i) - req = @api.resource(:content_views).call(:index, {:organization_id => @options[:org], :full_results => true, :per_page => req['per_page'], :page => req['page'].to_i+1}) - cvs.concat(req['results']) - end - - cvs.each do |cv| - # if CV is not listed in csv, skip - puts_verbose "Checking Content View #{cv['name']}" - next if not @yaml[:publish].include?(cv['name']) - - # if the CV is listed, write it - puts "Inspecting #{cv['name']} as listed in CSV" - - # initialize variables - needs_publish = false - # check if this CV has ever been published - if cv.has_key?('versions') and cv['versions'].length > 0 - last_ver_published = cv['versions'].sort_by{|ver| ver['published']}.last['published'] - # if published with a version, take last version published time - cv_last_published = Time.xmlschema(last_ver_published) rescue Time.parse(last_ver_published) - elsif cv.has_key?('last_published') and cv['last_published'] - # if published without version, save last published time - last_ver_published = cv['versions'].sort_by{|ver| ver['published']}.last['published'] - cv_last_published = Time.xmlschema(cv['last_published']) rescue Time.parse(cv['last_published']) - else - # if not published, save 0 as published time - cv_last_published = Time.new(0) - end - # Check every repo in the CV - cv['repository_ids'].each do |repo_id| - # get repo data - repo = @api.resource(:repositories).call(:show, {:id => repo_id}) - # check if the last sync has been ever completed - if repo.has_key?('last_sync') and repo['last_sync'] and repo['last_sync'].has_key?('ended_at') and repo['last_sync']['ended_at'] - # if sync completed, save last end sync time - repo_last_sync = Time.xmlschema(repo['last_sync']['ended_at']) rescue Time.parse(repo['last_sync']['ended_at']) - else - # if sync never completed, save 0 as sync time - repo_last_sync = Time.new(0) - end - # check if last repo sync time happened after last CV publish - if repo_last_sync > cv_last_published - # if checkrepo option is on, a deeper check will be done - if @options[:checkrepos] - # write some info about repo that we are checking - puts " repo #{repo['label']} (id: #{repo['id']}) seems newer than CV #{cv['name']} (id: #{cv['id']}), checking if sync contains new packages." - # get last sync repo output from foreman task - sync_task = @api.resource(:foreman_tasks).call(:show, {:id => repo['last_sync']['id']}) - # check if the package contains "No new package.". The opposite is a number of packages. - if sync_task['humanized']['output'] == "No new packages." - # no new packages in the last sync. getting the older - puts " '#{sync_task['humanized']['output']}' Found in last sync task, will search now for past sync tasks for packages not in the CV." - # Initialize to start the iteration, garbage collection needed and performed below. - taskreq = nil - begin - # loop over tasks. first round is nil - while (taskreq == nil or taskreq['results'].length == taskreq['per_page'].to_i) - # if nil, new data will be taken from zero. otherwise take next page - if (taskreq == nil) - taskreq = @api.resource(:foreman_tasks).call(:index, {:search => 'label=Actions::Katello::Repository::Sync and result=success', :full_results => true, :per_page => 10, :sort_by => :ended_at}) - puts_verbose "Inspecing sync tasks to #10" - else - taskreq = @api.resource(:foreman_tasks).call(:index, {:search => 'label=Actions::Katello::Repository::Sync and result=success', :full_results => true, :per_page => taskreq['per_page'], :sort_by => :ended_at, :page => taskreq['page'].to_i+1}) - puts_verbose "Inspecing sync tasks to ##{taskreq['per_page']}" - end - # iterate over the results - taskreq['results'].each do |tasker| - # if the task id is the same as the task extracted before loop, continue - next if tasker['id'] == sync_task['id'] - # if task has been completed after cv publish, check if the task that we are checking are for the repo id we are looking for - if tasker['input']['repository']['id'] == repo['id'] - puts_verbose "Found past task that matches repo id: #{tasker['input']['repository']['id']}" - # Avoid to check old publications - if ( checkoldtask(tasker, cv_last_published) ) - raise "skip_parsed" - end - if ( checktask(tasker, cv_last_published) ) - needs_publish = true - raise "publish" - end - end - end - end - # if we reach this point, all of the tasks has been parsed: - # no repo that would trigger a publish has been found AND the task date is newer then last publish - puts " No more tasks are found, Publish will be SKIPPED." - raise "skip_parsed" - rescue => error - puts_verbose " Repo: '#{repo['label']}'. Exiting from loop, reason: #{error.inspect}" - end - # forcing also garbage collection to run. 250MB of memory estimated. - taskreq = nil - else - puts " #{sync_task['humanized']['output']} Will Publish." - needs_publish = true - end - else - puts " repo #{repo['label']} (id: #{repo['id']}) seems newer than CV #{cv['name']} (id: #{cv['id']}) (#{repo_last_sync} > #{cv_last_published}), lets publish" - needs_publish = true - end - end - end - # check if --force has been used. if so, force publish - if not needs_publish and @options[:force] - needs_publish = true - puts " forced publish, even if there were no changes" - end - # finally if the CV has to be published, do it - if needs_publish - puts "Publishing #{cv['name']}" - if not @options[:noop] - req = @api.resource(:content_views).call(:publish, {:id => cv['id'], :description => @options[:description]}) - tasks << req['id'] - if @options[:sequential] > 0 and tasks.length >= @options[:sequential] - tasks = wait(tasks) - end - else - puts " [noop] published #{cv['name']}" - end - end - end - wait(tasks) -end - -def wait(tasks) - need_wait = tasks - if @options[:wait] - last_need_wait = [] - silence = false - wait_secs = 0 - while not need_wait.empty? - if wait_secs < 60 - wait_secs += 10 - end - puts "waiting #{wait_secs} for pending tasks: #{need_wait}" unless silence - sleep wait_secs - last_need_wait = need_wait - need_wait = [] - tasks.each do |task_id| - req = @api.resource(:foreman_tasks).call(:show, {:id => task_id}) - if req['pending'] - need_wait << task_id - end - end - if (wait_secs >= 60 and (last_need_wait.sort == need_wait.sort)) - puts "Silencing output until there's a task status change..." unless silence - silence = true - else - silence = false - end - end - end - return need_wait -end - -action = ARGV.shift - -if action == 'clean' - clean -elsif action == 'update' - update -elsif action == 'promote' - promote -elsif action == 'publish' - publish -end -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, -# MA 02110-1301, USA. - -require 'optparse' -require 'yaml' -require 'apipie-bindings' -require 'highline/import' -require 'time' - -@defaults = { - :noop => false, - :keep => 5, - :uri => 'https://localhost/', - :timeout => 300, - :user => 'admin', - :pass => nil, - :org => 1, - :lifecycle => 1, - :force => false, - :wait => false, - :sequential => 0, - :promote_cvs => false, - :checkrepos => false, - :verbose => false, - :description => 'autopublish', - :verify_ssl => true, -} - -@options = { - :yamlfile => 'cvmanager.yaml', -} - -optparse = OptionParser.new do |opts| - opts.banner = "Usage: #{opts.program_name} ACTION [options]" - opts.version = "0.1" - - opts.separator "" - opts.separator "#{opts.summary_indent}ACTION can be any of [clean,update,publish,promote]" - opts.separator "" - - opts.on("-U", "--uri=URI", "URI to the Satellite") do |u| - @options[:uri] = u - end - opts.on("-t", "--timeout=TIMEOUT", OptionParser::DecimalInteger, "Timeout value in seconds for any API calls. -1 means never timeout") do |t| - @options[:timeout] = t - end - opts.on("-u", "--user=USER", "User to log in to Satellite") do |u| - @options[:user] = u - end - opts.on("-p", "--pass=PASS", "Password to log in to Satellite") do |p| - @options[:pass] = p - end - opts.on("-o", "--organization-id=ID", "ID of the Organization to manage CVs in") do |o| - @options[:org] = o - end - opts.on("-k", "--keep=NUM", OptionParser::DecimalInteger, "how many unused versions should be kept") do |k| - @options[:keep] = k - end - opts.on("-c", "--config=FILE", "configuration in YAML format") do |c| - @options[:yamlfile] = c - end - opts.on("-l", "--to-lifecycle-environment=ID", OptionParser::DecimalInteger, "which LE should the promote be done to") do |l| - @options[:lifecycle] = l - end - opts.on("-d", "--description=STRING", "Description to use for publish operations") do |d| - @options[:description] = d - end - opts.on("-n", "--noop", "do not actually execute anything") do - @options[:noop] = true - end - opts.on("-f", "--force", "force actions that otherwise would have been skipped") do - @options[:force] = true - end - opts.on("--wait", "wait for started tasks to finish") do - @options[:wait] = true - end - opts.on("--sequential [NUM]", OptionParser::DecimalInteger, "wait for each (or NUM) started task(s) to finish before starting the next one") do |s| - @options[:wait] = true - @options[:sequential] = s || 1 - end - opts.on("--checkrepos", "check repository content was changed before publish") do - @options[:checkrepos] = true - end - opts.on("--verbose", "Get verbose logs from cvmanager") do - @options[:verbose] = true - end - opts.on("--no-verify-ssl", "don't verify SSL certs") do - @options[:verify_ssl] = false - end -end -optparse.parse! - -if ARGV.empty? - puts optparse.help - exit -end - -@yaml = YAML.load_file(@options[:yamlfile]) - -if @yaml.has_key?(:settings) and @yaml[:settings].is_a?(Hash) - @yaml[:settings].each do |key,val| - if not @options.has_key?(key) - @options[key] = val - end - end -end - -@defaults.each do |key,val| - if not @options.has_key?(key) - @options[key] = val - end -end - -if not @options[:user] - @options[:user] = ask('Satellite username: ') -end - -if not @options[:pass] - @options[:pass] = ask('Satellite password: ') { |q| q.echo = false } -end - -# sanitize non-complete config files -[:cv, :ccv].each do |key| - if not @yaml.has_key?(key) - @yaml[key] = {} - end -end -[:publish, :promote].each do |key| - if not @yaml.has_key?(key) - @yaml[key] = [] - end -end - -@api = ApipieBindings::API.new({:uri => @options[:uri], :username => @options[:user], :password => @options[:pass], :api_version => '2', :timeout => @options[:timeout]}, {:verify_ssl => @options[:verify_ssl]}) - -def puts_verbose(message) - if @options[:verbose] - puts " [VERBOSE] #{message}" - end -end - -def clean() - tasks = [] - cvs = [] - req = @api.resource(:content_views).call(:index, {:organization_id => @options[:org], :full_results => true}) - cvs.concat(req['results']) - while (req['results'].length == req['per_page'].to_i) - req = @api.resource(:content_views).call(:index, {:organization_id => @options[:org], :full_results => true, :per_page => req['per_page'], :page => req['page'].to_i+1}) - cvs.concat(req['results']) - end - - cvs.each do |cv| - keep = [] - puts "Inspecting #{cv['name']}" - cv['versions'].sort_by { |v| v['version'].to_f }.reverse.each do |version| - if not version['environment_ids'].empty? - puts_verbose " #{cv['name']} v#{version['version']} is published to the following environments: #{version['environment_ids']}, skipping." - next - end - version_details = @api.resource(:content_view_versions).call(:show, {:id => version['id']}) - if not version_details['composite_content_view_ids'].empty? - puts_verbose " #{cv['name']} v#{version['version']} is used by the following composite contentviews: #{version_details['composite_content_view_ids']}, skipping." - next - end - if keep.length < @options[:keep] - keep.push(version) - puts " keeping #{version['version']}" - else - puts " removing #{version['version']}" - if not @options[:noop] - req = @api.resource(:content_view_versions).call(:destroy, {:id => version['id']}) - tasks << req['id'] - if @options[:sequential] > 0 and tasks.length >= @options[:sequential] - tasks = wait(tasks) - end - else - puts " [noop] would delete content view version with id #{version['id']}" - end - end - end - end - - wait(tasks) -end - -def checktask(task, last_date) - task_completed_at = Time.xmlschema(task['ended_at']) rescue Time.parse(task['ended_at']) - if task_completed_at >= last_date - puts_verbose "Past task was completed at #{task_completed_at}, which is after #{last_date}" - if task['humanized']['output'] == "No new packages." - puts_verbose "#{task['humanized']['output']} This past task will NOT trigger a Publish." - return false - else - puts_verbose "#{task['humanized']['output']} This past task will trigger a Publish." - return true - end - end - puts_verbose "#{task_completed_at} is before #{last_date}, will NOT trigger a Publish." - return false -end - -def checkoldtask(task, last_date) - task_completed_at = Time.xmlschema(task['ended_at']) rescue Time.parse(task['ended_at']) - if task_completed_at >= last_date - puts_verbose "Past task was completed at #{task_completed_at}, which is after #{last_date}. Continue to inspect." - return false - end - puts_verbose "#{task_completed_at} is before #{last_date}, no sense to continue." - return true -end - -def update() - tasks = [] - - ccvs = [] - req = @api.resource(:content_views).call(:index, {:organization_id => @options[:org], :full_results => true}) - ccvs.concat(req['results']) - while (req['results'].length == req['per_page'].to_i) - req = @api.resource(:content_views).call(:index, {:organization_id => @options[:org], :full_results => true, :per_page => req['per_page'], :page => req['page'].to_i+1}) - ccvs.concat(req['results']) - end - - ccvs.each do |ccv| - next if ! ccv['composite'] - - was_updated = false - - puts "Inspecting #{ccv['name']}" - - # loop through the components and check if they are uptodate - ids = Array.new(ccv['component_ids']) - ccv['components'].each do |component| - puts " Checking #{component['content_view']['name']}" - - # get the desired version for this component from the YAML - # either the version for the component in this CCV is set - # or it is set globally - # never touch non-mentioned components - if @yaml[:ccv].is_a?(Hash) and @yaml[:ccv].has_key?(ccv['label']) and @yaml[:ccv][ccv['label']].has_key?(component['content_view']['label']) - desired_version = @yaml[:ccv][ccv['label']][component['content_view']['label']] - puts_verbose " Desired version #{desired_version} found in CCV" - elsif @yaml[:cv].is_a?(Hash) and @yaml[:cv].has_key?(component['content_view']['label']) - desired_version = @yaml[:cv][component['content_view']['label']] - puts_verbose " Desired version #{desired_version} found in CV" - else - puts_verbose " Desired version not found, skipping" - next - end - - # instead of hard-coding the versions, the user can also specify "latest" - if desired_version == 'latest' - cvversions = @api.resource(:content_view_versions).call(:index, {:content_view_id => component['content_view']['id']}) - cvversions = cvversions['results'].sort_by { |v| v['version'].to_f }.reverse - desired_version = cvversions[0]['version'] - puts_verbose " Found #{desired_version} as the 'latest' version" - end - - # if the version of the component does not match the one the user requested update it - if component['version'].to_s != desired_version.to_s - puts " Updating from #{component['version']} to #{desired_version}" - oldids = ids.dup - ids.delete(component['id']) - cvversions = @api.resource(:content_view_versions).call(:index, {:content_view_id => component['content_view']['id'], :version => desired_version}) - desired_version_id = cvversions['results'][0]['id'] - ids.push(desired_version_id) - puts " Old components: #{oldids}" - puts " New components: #{ids}" - # do the update - was_updated = true - end - end - + was_updated = true if was_updated #Change the member content view versions; We do this once at the end, so if there was multiple CV changes, its only one call puts " Committing new content view versions" From 7572c36fabcded2fc567b9fb5a78f2dfdd4302a0 Mon Sep 17 00:00:00 2001 From: Scott Parker Date: Tue, 1 May 2018 09:43:36 -0600 Subject: [PATCH 6/7] changed ruby to tfm-ruby changed ruby to tfm-ruby --- cvmanager | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cvmanager b/cvmanager index 454142b..c02d72f 100755 --- a/cvmanager +++ b/cvmanager @@ -1,4 +1,4 @@ -#!/usr/bin/env ruby +#!/usr/bin/env tfm-ruby # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by From 7576a29f9e51e707f188379b4dffed27af33a648 Mon Sep 17 00:00:00 2001 From: Scott Parker Date: Wed, 2 May 2018 08:14:48 -0600 Subject: [PATCH 7/7] removed tfm from ruby removed the use of tfm-ruby --- cvmanager | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cvmanager b/cvmanager index c02d72f..454142b 100755 --- a/cvmanager +++ b/cvmanager @@ -1,4 +1,4 @@ -#!/usr/bin/env tfm-ruby +#!/usr/bin/env ruby # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by