From 252b33443d7905247f7f67d0d72695463b9cc216 Mon Sep 17 00:00:00 2001 From: ggatward Date: Wed, 3 Aug 2016 15:19:44 +1000 Subject: [PATCH] Rework import script to handle repolist pickle --- helpers.py | 6 +- install_scripts/sat62_setup.txt | 60 ++++++++++++++ install_scripts/sat62disc_setup.txt | 77 ++++++++++++++++- sat_export.py | 8 +- sat_import.py | 123 +++++++++++++++++----------- 5 files changed, 217 insertions(+), 57 deletions(-) diff --git a/helpers.py b/helpers.py index 46f0903..21d1750 100644 --- a/helpers.py +++ b/helpers.py @@ -177,12 +177,12 @@ def get_org_id(org_name): return org_id -def wait_for_task(task_id): +def wait_for_task(task_id, label): """ Wait for the given task ID to complete This displays a message without CR/LF waiting for an OK/FAIL status to be shown """ - msg = "Waiting for export to complete... " + msg = "Waiting for " + label + " to complete... " print msg, log_msg(msg, 'INFO') # Force the status message to be shown to the user @@ -190,7 +190,7 @@ def wait_for_task(task_id): while True: info = get_json(FOREMAN_API + "tasks/" + str(task_id)) if info['state'] == 'paused' and info['result'] == 'error': - msg = "Error with Content View Update " + str(task_id) + msg = "Error with " + label + " " + str(task_id) log_msg(msg, 'ERROR') break if info['pending'] != 1: diff --git a/install_scripts/sat62_setup.txt b/install_scripts/sat62_setup.txt index 001ff40..17248b1 100644 --- a/install_scripts/sat62_setup.txt +++ b/install_scripts/sat62_setup.txt @@ -125,6 +125,66 @@ satellite-installer --foreman-ipa-authentication=true # Enables IPA for Aut satellite-installer --foreman-pam-service=satellite-prod # Enables HBAC +##################### +# If using LDAP for Auth: +# +setsebool -P authlogin_nsswitch_use_ldap=1 + +# We need the certs that make up the CA chain: +cat << EOF > /etc/pki/tls/certs/CA.pem +-----BEGIN CERTIFICATE----- +... +-----END CERTIFICATE----- +EOF + +cat << EOF > /etc/pki/tls/certs/rootca.pem +-----BEGIN CERTIFICATE----- +... +-----END CERTIFICATE----- +EOF + +cd /etc/pki/tls/certs +ln -s rootca.pem /etc/pki/tls/certs/$(openssl x509 -noout -hash -in /etc/pki/tls/certs/rootca.pem).0 +ln -s CA.pem /etc/pki/tls/certs/$(openssl x509 -noout -hash -in /etc/pki/tls/certs/CA.pem).0 + +hammer auth-source ldap create --name 'LDAP Server' --host ldap.example.org -server-type posix \ + --tls yes --port 636 --base-dn ou=People,ou=example,o=com,c=au --attr-login uid + +# Verify the auth source +hammer auth-source ldap list + + +###################### +# Configure Email delivery +# +# If we need to insert a classification header on the email: +cat << EOF > /etc/postfix/classification +/^Content-Transfer-Encoding/i PREPEND x-classification: UNCLASSIFIED +/^Content-Transfer-Encoding/i PREPEND x-protective-marking: VER=2012.3, NS=example.org, SEC=UNCLASSIFIED, ORIGIN=satellite@example.org +EOF +sed -i 's/^#header_checks = /header_checks = regexp:\/etc\/postfix\/classification/' /etc/postfix/main.cf + +# Set Satellite to send mail to the local postfix instance: +cat << EOF > /etc/foreman/email.yaml +# Outgoing email settings +production: + delivery_method: :smtp + smtp_settings: + address: localhost + port: 25 + enable_starttls_auto: false +EOF +sed -i 's/^#relayhost = uucphost/relayhost = smtp.example.org/' /etc/postfix/main.cf +systemctl restart postfix + +# Set the email addresses in Satellite itself +hammer settings set --name email_reply_address --value "someone@example.org" +hammer settings set --name email_subject_prefix --value "[Satellite 6]" + +# Restart Satellite +katello-service restart + + ####################### # Configure content export location hammer settings set --name pulp_export_destination --value /var/sat-export diff --git a/install_scripts/sat62disc_setup.txt b/install_scripts/sat62disc_setup.txt index 424bf5a..53203f6 100644 --- a/install_scripts/sat62disc_setup.txt +++ b/install_scripts/sat62disc_setup.txt @@ -111,6 +111,7 @@ hammer host update --location="$LOC" --name sat62disc.example.org ###################### # If using IPA for Auth: +# ## ON IPA SERVER: ipa service-add HTTP/sat62disc.example.org @@ -133,6 +134,65 @@ satellite-installer --foreman-ipa-authentication=true # Enables IPA for Aut satellite-installer --foreman-pam-service=satellite-prod # Enables HBAC +###################### +# If using LDAP for Auth: +# +setsebool -P authlogin_nsswitch_use_ldap=1 + +# We need the certs that make up the CA chain: +cat << EOF > /etc/pki/tls/certs/CA.pem +-----BEGIN CERTIFICATE----- +... +-----END CERTIFICATE----- +EOF + +cat << EOF > /etc/pki/tls/certs/rootca.pem +-----BEGIN CERTIFICATE----- +... +-----END CERTIFICATE----- +EOF + +cd /etc/pki/tls/certs +ln -s rootca.pem /etc/pki/tls/certs/$(openssl x509 -noout -hash -in /etc/pki/tls/certs/rootca.pem).0 +ln -s CA.pem /etc/pki/tls/certs/$(openssl x509 -noout -hash -in /etc/pki/tls/certs/CA.pem).0 + +hammer auth-source ldap create --name 'LDAP Server' --host ldap.example.org -server-type posix \ + --tls yes --port 636 --base-dn ou=People,ou=example,o=com,c=au --attr-login uid + +# Verify the auth source +hammer auth-source ldap list + + +###################### +# Configure Email delivery +# +# If we need to insert a classification header on the email: +cat << EOF > /etc/postfix/classification +/^Content-Transfer-Encoding/i PREPEND x-classification: UNCLASSIFIED +/^Content-Transfer-Encoding/i PREPEND x-protective-marking: VER=2012.3, NS=example.org, SEC=UNCLASSIFIED, ORIGIN=satellite@example.org +EOF +sed -i 's/^#header_checks = /header_checks = regexp:\/etc\/postfix\/classification/' /etc/postfix/main.cf + +# Set Satellite to send mail to the local postfix instance: +cat << EOF > /etc/foreman/email.yaml +# Outgoing email settings +production: + delivery_method: :smtp + smtp_settings: + address: localhost + port: 25 + enable_starttls_auto: false +EOF +sed -i 's/^#relayhost = uucphost/relayhost = smtp.example.org/' /etc/postfix/main.cf +systemctl restart postfix + +# Set the email addresses in Satellite itself +hammer settings set --name email_reply_address --value "someone@example.org" +hammer settings set --name email_subject_prefix --value "[Satellite 6]" + +# Restart Satellite +katello-service restart + ###################### # Configure Content Import Mount @@ -277,6 +337,21 @@ for i in $(hammer --csv repository list --organization-id 1 | grep -vi '^ID' | a done +########################## +# Self-register the Satellite to itself +rpm -ivh /var/www/html/pub/katello-ca-consumer-latest.noarch.rpm +subscription-manager register --user admin --pass ChangeMe --org $ORG + +#-- You will need to attach RHCI or Satellite subscription to host via the WebUI + + + +########################## +# Install/Configure virt-who +yum -y install virt-who + + + ##################################### # Locations etc @@ -409,8 +484,6 @@ hammer activation-key create --content-view 'RHEL Server' --lifecycle-environmen - - ######################### # Puppet # (May need selinux module for nfs_t) diff --git a/sat_export.py b/sat_export.py index 5498677..4753ddf 100755 --- a/sat_export.py +++ b/sat_export.py @@ -1,13 +1,13 @@ #!/usr/bin/python #title :sat_export.py -#description :Exports Satellite 6 Default Content View for disconnected environments +#description :Exports Satellite 6 Content for disconnected environments #URL :https://github.com/ggatward/sat6_scripts #author :Geoff Gatward #notes :This script is NOT SUPPORTED by Red Hat Global Support Services. #license :GPLv3 #============================================================================== """ -Exports Default Org Content View. +Exports Satellite 6 yum content. """ import sys, argparse, datetime, os, shutil, pickle @@ -541,7 +541,7 @@ def main(): export_id = export_cv(dov_ver, last_export, export_type) # Now we need to wait for the export to complete - helpers.wait_for_task(export_id) + helpers.wait_for_task(export_id, 'export') # Check if the export completed OK. If not we exit the script. tinfo = helpers.get_task_status(export_id) @@ -592,7 +592,7 @@ def main(): export_id = export_repo(repo_result['id'], last_export, export_type) # Now we need to wait for the export to complete - helpers.wait_for_task(export_id) + helpers.wait_for_task(export_id, 'export') # Check if the export completed OK. If not we exit the script. tinfo = helpers.get_task_status(export_id) diff --git a/sat_import.py b/sat_import.py index 10854fc..5396518 100755 --- a/sat_import.py +++ b/sat_import.py @@ -1,16 +1,16 @@ #!/usr/bin/python #title :sat_import.py -#description :Imports Satellite 6 Default Content View for disconnected environments +#description :Imports Satellite 6 Content for disconnected environments #URL :https://github.com/ggatward/sat6_scripts #author :Geoff Gatward #notes :This script is NOT SUPPORTED by Red Hat Global Support Services. #license :GPLv3 #============================================================================== """ -Imports Default Org Content View exported by sat_export.py +Imports Satellite 6 yum content exported by sat_export.py """ -import sys, argparse, os +import sys, argparse, os, pickle import simplejson as json import helpers @@ -66,54 +66,58 @@ def extract_content(basename): # rm basename + '_*' -def sync_content(org_id): +def sync_content(org_id, imported_repos): """ Synchronize the repositories Triggers a sync of all repositories belonging to the configured sync plan """ - # Check that the configured sync plan exists - splans = helpers.get_json( - helpers.KATELLO_API + "organizations/" + str(org_id) + "/sync_plans/") - for sp_result in splans['results']: - if sp_result['name'] == helpers.SYNCPLAN: - sp_id = sp_result['id'] - msg = "Sync plan '" + helpers.SYNCPLAN + "' ID: " + str(sp_id) + repos_to_sync = [] + delete_override = False + + # Get a listing of repositories in this Satellite + enabled_repos = helpers.get_p_json( + helpers.KATELLO_API + "/repositories/", \ + json.dumps( + { + "organization_id": org_id, + } + )) + + # Loop through each repo to be imported/synced + for repo in imported_repos: + do_import = False + for repo_result in enabled_repos['results']: + if repo in repo_result['label']: + do_import = True + repos_to_sync.append(repo_result['id']) + if do_import: + msg = "Repo " + repo + " found in Satellite" helpers.log_msg(msg, 'DEBUG') + else: + msg = "Repo " + repo + " is not enabled in Satellite" + # If the repo is not enabled, don't delete the input files. + # This gives the admin a chance to manually enable the repo and re-import + delete_override = True + helpers.log_msg(msg, 'WARNING') + # TODO: We could go on here and try to enable the Red Hat repo ..... + + msg = "Repo ids to sync: " + str(repos_to_sync) + helpers.log_msg(msg, 'DEBUG') + + msg = "Syncing repositories" + helpers.log_msg(msg, 'INFO') + print msg + task_id = helpers.post_json( + helpers.KATELLO_API + "repositories/bulk/sync", \ + json.dumps( + { + "ids": repos_to_sync, + } + ))["id"] + msg = "Repo sync task id = " + task_id + helpers.log_msg(msg, 'DEBUG') - if not sp_id: - msg = "Sync plan '" + helpers.SYNCPLAN + "' not found" - helpers.log_msg(msg, 'ERROR') - sys.exit(-1) - else: - # Check that no sync tasks are already running - helpers.check_running_sync() - - msg = "No existing running or paused sync tasks detected" - helpers.log_msg(msg, 'DEBUG') - - # Run the sync plan - task_id = helpers.put_json( - helpers.KATELLO_API + "organizations/" + str(org_id) + "/sync_plans/" + str(sp_id) \ - + "/sync", json.dumps( - { - } - ))["id"] - -# Possibly need to loop through repo ID's and use this one instead -# task_id = helpers.put_json( -# helpers.KATELLO_API + "repositories/" + str(repo_id) + "/sync", +\ -# json.dumps( -# { -# source_url: 'http://localhost/pub/.cdn', -# incremental: true -# } -# ))["id"] - - - msg = "Sync plan started - task_id " + task_id - helpers.log_msg(msg, 'DEBUG') - - return task_id + return task_id, delete_override def main(): @@ -142,6 +146,8 @@ def main(): required=True) parser.add_argument('-n', '--nosync', help='Do not trigger a sync after extracting content', required=False, action="store_true") + parser.add_argument('-r', '--remove', help='Remove input files after import has completed', + required=False, action="store_true") args = parser.parse_args() # Set our script variables from the input args @@ -154,21 +160,42 @@ def main(): # Figure out if we have the specified input fileset basename = get_inputfiles(expdate) + # Cleanup from any previous imports + os.system("rm -rf " + helpers.IMPORTDIR + "/{content,custom,listing,*.pkl}") + # Extract the input files extract_content(basename) # Trigger a sync of the content into the Library if args.nosync: print helpers.GREEN + "Import complete.\n" + helpers.ENDC + msg = "Repository sync was requested to be skipped" + helpers.log_msg(msg, 'WARNING') print 'Please synchronise all repositories to make new content available for publishing.' + delete_override = False else: - sync_content(org_id) + # We need to figure out which repos to sync. This comes to us via a pickle containing + # a list of repositories that were exported + imported_repos = pickle.load(open('exported_repos.pkl', 'rb')) + + # Run a repo sync on each imported repo + (task_id, delete_override) = sync_content(org_id, imported_repos) + + # Now we need to wait for the sync to complete + helpers.wait_for_task(task_id, 'sync') + print helpers.GREEN + "Import complete.\n" + helpers.ENDC - print 'Please wait for sync to complete, then publish content views to make new' \ - 'content available.' + print 'Please publish content views to make new content available.' + + if args.remove and not delete_override: + msg = "Removing " + helpers.IMPORTDIR + "/sat6_export_" + expdate + "* input files" + helpers.log_msg(msg, 'DEBUG') +# os.system("rm -f " + helpers.IMPORTDIR + "/sat6_export_" + expdate) + "*" + msg = "Import Complete" helpers.log_msg(msg, 'INFO') if __name__ == "__main__": main() +