diff --git a/Gemfile b/Gemfile index c8c89c9a5..6730b6ee4 100644 --- a/Gemfile +++ b/Gemfile @@ -51,6 +51,7 @@ group :development do gem 'capistrano-git-with-submodules', '2.0.3' gem 'capistrano-service' gem 'awesome_print' + gem 'net-sftp' # gem 'listen', '~> 3.1.5' # gem 'spring-watcher-listen', '~> 2.0.0' # diff --git a/Gemfile.lock b/Gemfile.lock index 6bfa7ecc7..e79ca5879 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1128,6 +1128,8 @@ GEM thor (~> 0.19) net-scp (2.0.0) net-ssh (>= 2.6.5, < 6.0.0) + net-sftp (3.0.0) + net-ssh (>= 5.0.0, < 7.0.0) net-ssh (5.2.0) nio4r (2.5.2) nokogiri (1.10.7) @@ -1341,6 +1343,7 @@ DEPENDENCIES minitest (~> 5.10, != 5.10.2) mocha (~> 1.0.0) neat + net-sftp nokogiri (~> 1.10.4) pg (~> 0.21) phantompdf (~> 1.2.2) diff --git a/lib/modules/sync_seeds.rb b/lib/modules/sync_seeds.rb new file mode 100644 index 000000000..0fe132cab --- /dev/null +++ b/lib/modules/sync_seeds.rb @@ -0,0 +1,157 @@ +class SyncSeeds + require 'net/ssh' + require 'net/scp' + + def initialize(server, username) + @username = username + @server = server + end + + def start_session + Net::SSH.start(@server, @username) do |session| + @session = session + yield + end + end + + def list_local_files(location) + Dir.glob('*', base: location) + end + + def list_remote_files(location) + @session.sftp.dir.glob(location, '*') + end + + def delete_files(files, location) + files.each do |file| + puts "Removing #{file} as it no longer exists remotely" + FileUtils.rm_rf(File.join(location, file)) + end + end + + def files_for_deletion(local_list, remote_list, location) + files = local_list - remote_list + + if files.empty? + puts "No files to delete" + else + delete_files(files, location) + end + end + + def download_files(source, dest) + puts "Downloading #{source} as it's newer or not found locally" + return @session.scp.download!(source, dest) if File.file?(dest) + @session.scp.download!(source, dest, recursive: true) + end + + def create_paths(relative_path) + { local_path: File.join(@local_base, relative_path), remote_path: File.join(@remote_base, relative_path) } + end + + def check_if_newer(parent_folder:, local_item:, remote_item:, remote_path:, local_path:, base: @local_base) + downloaded = false + + if parent_folder.include?(local_item) + yield if block_given? + + is_newer = Time.at(remote_item.attributes.mtime) >= File.stat(local_path).mtime + # If there are any outdated files, will trigger download + if is_newer + if Dir.glob('*', base: @local_base).include?(local_item) + download_files(remote_path, local_path) + # Will be hit if local_item is a file or folder inside a directory + elsif Dir.glob('**/*', base: base).include?(local_item) + download_files(remote_path, local_path) + downloaded = true + end + end + else + # Just download it if it doesn't exist at all + download_files(remote_path, @local_base) + downloaded = true + end + + downloaded + end + + def compare_folders(wildcard:, local:, remote:, base:) + puts "Checking to see what files need to be deleted from #{local}" + + remote_list = @session.sftp.dir.glob(remote, wildcard).map do |f| + f.name.force_encoding('UTF-8') + end + + local_list = Dir.glob(wildcard, base: local) + + files_for_deletion(local_list, remote_list, base) + end + + # When folders need to be checked recursively + def check_inside_folder(folder, local_list) + paths = create_paths(folder) + + compare_folders(wildcard: '**/*', local: paths[:local_path], remote: paths[:remote_path], base: paths[:local_path]) + + local_folder_content = Dir.glob('**/*', base: paths[:local_path]) + remote_folder_content = @session.sftp.dir.glob(paths[:remote_path], '**/*') + + remote_folder_content.each do |file| + # Go through the various files and folders and check to see if they exist locally + local_file = file.name.force_encoding('UTF-8') + + check = check_if_newer( + parent_folder: local_folder_content, + local_item: local_file, + remote_item: file, + remote_path: paths[:remote_path], + local_path: paths[:local_path], + base: paths[:local_path] + ) + + # We don't want to download the whole folder again if it's already been re-downloaded once + # Break out of loop if already downloaded + break if check == true + end + end + + def main_task(local_list:, remote_list:, local_base:, remote_base:) + @local_base = local_base + @remote_base = remote_base + + remote_list.each do |object| + # There are files with non-ASCII characters (i.e. accented) in the CMS files + name = object.name.force_encoding('UTF-8') + paths = create_paths(name) + + check_if_newer( + parent_folder: local_list, + local_item: name, + remote_item: object, + remote_path: paths[:remote_path], + local_path: paths[:local_path] + ) do + check_inside_folder(name, local_list) if object.attributes.directory? + end + end + end + + # Piggybacks on existing Comfy modules + # TODO - the arguments used here for the site and folder + # (e.g. protectedplanet or protected-planet) can be different + # for different environments, need to change both this and the staging seeds rake task + def commence_comfy_import(answer) + logger = ComfortableMexicanSofa.logger + ComfortableMexicanSofa.logger = Logger.new(STDOUT) + + if answer == 'all' + Rake::Task["comfy:cms_seeds:import"].invoke('protected-planet', 'protectedplanet') + else + module_name = "ComfortableMexicanSofa::Seeds::#{answer.singularize}::Importer".constantize + module_name.new('protected-planet', 'protectedplanet').import! + end + + ComfortableMexicanSofa.logger = logger + end + +end \ No newline at end of file diff --git a/lib/tasks/staging_seeds.rake b/lib/tasks/staging_seeds.rake new file mode 100644 index 000000000..44eecec0e --- /dev/null +++ b/lib/tasks/staging_seeds.rake @@ -0,0 +1,78 @@ +# frozen_string_literal: true + +namespace :comfy do + # The methods that this rake task calls are in lib/modules/sync_seeds.rb + SOURCE = 'ProtectedPlanet/current/db/cms_seeds/protected-planet'.freeze + PP_STAGING = 'new-web.pp-staging.linode.protectedplanet.net'.freeze + PP_USER = 'wcmc'.freeze + + def user_input + valid_answers = %w(all files layouts pages nothing) + puts question = "What would you like to import? 'all/files/layouts/pages' or 'nothing' to quit" + answer = STDIN.gets.chomp.downcase + + until valid_answers.include?(answer) + puts question + answer = STDIN.gets.chomp.downcase + end + + abort('Goodbye') if answer == 'nothing' + + { answer: answer, destination: File.join(ComfortableMexicanSofa.config.seeds_path, 'protected-planet') } + end + + desc "Import CMS Seed data from staging. Can be run with arguments + [destination folder - 'protected-planet' by convention, files/pages/layouts/all] + or can accept user input if no argument is supplied" + task :staging_import, %i[dest folder] => [:environment] do |_t, args| + dest = nil + answer = nil + + if args[:dest].nil? && args[:folder].nil? + answers = user_input + dest = answers[:destination] + answer = answers[:answer] + else + dest = File.join(ComfortableMexicanSofa.config.seeds_path, args[:dest]) + answer = args[:folder].downcase + end + + # Creates folder under db/cms_seeds if it doesn't exist + unless Dir.exist?(dest) + FileUtils.mkdir_p(dest) + end + + puts "Importing CMS Seed data from Staging Folder to #{dest} ..." + + new_session = SyncSeeds.new(PP_STAGING, PP_USER) + + # SSH into staging server with Net::SSH + new_session.start_session do |session| + # First get rid of any local top-level (i.e. which exist in the main + # directory of REMOTE) folders/files that don't exist remotely + new_session.compare_folders(wildcard: '*', local: dest, remote: SOURCE, base: dest) + + local_list = new_session.list_local_files(dest) + remote_list = new_session.list_remote_files(SOURCE) + + if answer == 'all' + puts "Downloading all folders..." + else + local_list.filter! { |f| f == answer } + remote_list.filter! do |f| + f.name.force_encoding('UTF-8') == answer + end + + puts "Downloading a new set of #{answer}..." + end + + new_session.main_task(local_list: local_list, remote_list: remote_list, local_base: dest, remote_base: SOURCE) + + puts "Finished downloads, now replacing your local seed data with your selection..." + + + # new_session.commence_comfy_import(answer) + end + end + +end \ No newline at end of file