From 604d8a8aff18bcc51add114cbc5c7205f3300819 Mon Sep 17 00:00:00 2001 From: Stanley Liu Date: Thu, 23 Jul 2020 18:20:06 +0100 Subject: [PATCH 01/49] added a new rake task for syncing seeds from staging --- lib/tasks/staging_seeds.rake | 39 ++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 lib/tasks/staging_seeds.rake diff --git a/lib/tasks/staging_seeds.rake b/lib/tasks/staging_seeds.rake new file mode 100644 index 000000000..25ec0b806 --- /dev/null +++ b/lib/tasks/staging_seeds.rake @@ -0,0 +1,39 @@ +# frozen_string_literal: true + +namespace :comfy do + desc "Import CMS Seed data from staging" + + task :seed_import => :environment do |_t| + require 'net/ssh' + require 'net/scp' + + puts "Importing CMS Seed data from Staging Folder to #{local} ..." + + # Locally stored seeds + local = ComfortableMexicanSofa.config.seeds_path + + # SSH into staging server with Net::SSH + # Check to see if local CMS seeds are newer than local and download any + Net::SSH.start(ENV['PP_STAGING'], ENV['PP_USER']) do |ssh| + ssh.sftp.dir.foreach('ProtectedPlanet/current/db/cms_seeds/protected-planet') do |f| + # Go through the various files and folders and check to see if they exist locally + if Dir.children(local).include?(f) + if File.stat(f).file? + local_file_changed = File.stat(f).mtime > Time.at(sftp.stat(f).mtime) + elsif File.stat(f).directory + ssh.sftp.dir.foreach(f) do |file| + local_file_changed = File.stat(file).mtime > Time.at(sftp.stat(file).mtime) + end + end + else + + end + end + # ssh.scp.download!('ProtectedPlanet/current/db/cms_seeds', local, recursive: true) + end + + puts "Replacing your local seed data..." + + Rake::Task["comfy:cms_seeds:import[protected-planet, protectedplanet]"].invoke + end +end \ No newline at end of file From c1151058e13c434c032a6415b50e77b7773cd9fb Mon Sep 17 00:00:00 2001 From: Stanley Liu Date: Fri, 24 Jul 2020 12:55:56 +0100 Subject: [PATCH 02/49] wrote a rake task to selectively update cms seeds locally from staging if staging is newer --- Gemfile | 1 + Gemfile.lock | 3 +++ lib/tasks/staging_seeds.rake | 46 +++++++++++++++++++++++------------- 3 files changed, 33 insertions(+), 17 deletions(-) diff --git a/Gemfile b/Gemfile index 0076fea4a..fd87d91b4 100644 --- a/Gemfile +++ b/Gemfile @@ -50,6 +50,7 @@ group :development do gem 'capistrano-git-with-submodules', '2.0.3' gem 'capistrano-service' gem 'awesome_print' + gem 'net-sftp' # gem 'listen', '~> 3.1.5' # gem 'spring-watcher-listen', '~> 2.0.0' # diff --git a/Gemfile.lock b/Gemfile.lock index 6bfa7ecc7..e79ca5879 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1128,6 +1128,8 @@ GEM thor (~> 0.19) net-scp (2.0.0) net-ssh (>= 2.6.5, < 6.0.0) + net-sftp (3.0.0) + net-ssh (>= 5.0.0, < 7.0.0) net-ssh (5.2.0) nio4r (2.5.2) nokogiri (1.10.7) @@ -1341,6 +1343,7 @@ DEPENDENCIES minitest (~> 5.10, != 5.10.2) mocha (~> 1.0.0) neat + net-sftp nokogiri (~> 1.10.4) pg (~> 0.21) phantompdf (~> 1.2.2) diff --git a/lib/tasks/staging_seeds.rake b/lib/tasks/staging_seeds.rake index 25ec0b806..20d5b0786 100644 --- a/lib/tasks/staging_seeds.rake +++ b/lib/tasks/staging_seeds.rake @@ -6,34 +6,46 @@ namespace :comfy do task :seed_import => :environment do |_t| require 'net/ssh' require 'net/scp' - - puts "Importing CMS Seed data from Staging Folder to #{local} ..." - # Locally stored seeds - local = ComfortableMexicanSofa.config.seeds_path + def check_timestamp(file, session, local, local_file, remote_file) + newer = Time.at(file.attributes.mtime) >= File.stat(local_file).mtime + if newer + puts "#{file.name} is newer than the local copy, downloading..." + session.scp.download!(remote_file, local) + end + end + + # Locally stored seeds - assumes you already have the local folder - + # it won't create it + local = ComfortableMexicanSofa.config.seeds_path + '/protected-planet' + remote = 'ProtectedPlanet/current/db/cms_seeds/protected-planet' + puts "Importing CMS Seed data from Staging Folder to #{local} ..." + # SSH into staging server with Net::SSH - # Check to see if local CMS seeds are newer than local and download any - Net::SSH.start(ENV['PP_STAGING'], ENV['PP_USER']) do |ssh| - ssh.sftp.dir.foreach('ProtectedPlanet/current/db/cms_seeds/protected-planet') do |f| + Net::SSH.start(ENV['PP_STAGING'], ENV['PP_USER']) do |session| + session.sftp.dir.glob(remote, '**/*').each do |file| # Go through the various files and folders and check to see if they exist locally - if Dir.children(local).include?(f) - if File.stat(f).file? - local_file_changed = File.stat(f).mtime > Time.at(sftp.stat(f).mtime) - elsif File.stat(f).directory - ssh.sftp.dir.foreach(f) do |file| - local_file_changed = File.stat(file).mtime > Time.at(sftp.stat(file).mtime) - end + remote_file = File.join(remote, file.name) + local_file = File.join(local, file.name) + + # There are files with non-ASCII characters (i.e. accented) in the CMS files + if Dir.glob('**/*', base: local).include?(file.name.force_encoding('UTF-8')) + if File.file?(File.join(local, file.name)) + check_timestamp(file, session, local, local_file, remote_file) end else - + puts "#{file.name} doesn\'t exist locally, downloading" + # File doesn't exist locally, so download it (in any folder required) + session.scp.download!(remote_file, local, recursive: true) end end - # ssh.scp.download!('ProtectedPlanet/current/db/cms_seeds', local, recursive: true) end - puts "Replacing your local seed data..." + puts "Finished downloads, now replacing your local seed data..." Rake::Task["comfy:cms_seeds:import[protected-planet, protectedplanet]"].invoke + + end end \ No newline at end of file From 10819705e74334665c5bd802e5c62d58b3708be7 Mon Sep 17 00:00:00 2001 From: Stanley Liu Date: Sun, 26 Jul 2020 10:53:08 +0100 Subject: [PATCH 03/49] fixed task not downloading to correct locations --- lib/tasks/staging_seeds.rake | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/tasks/staging_seeds.rake b/lib/tasks/staging_seeds.rake index 20d5b0786..9524ed45a 100644 --- a/lib/tasks/staging_seeds.rake +++ b/lib/tasks/staging_seeds.rake @@ -11,7 +11,7 @@ namespace :comfy do newer = Time.at(file.attributes.mtime) >= File.stat(local_file).mtime if newer puts "#{file.name} is newer than the local copy, downloading..." - session.scp.download!(remote_file, local) + session.scp.download!(remote_file, local_file) end end @@ -28,7 +28,7 @@ namespace :comfy do # Go through the various files and folders and check to see if they exist locally remote_file = File.join(remote, file.name) local_file = File.join(local, file.name) - + # There are files with non-ASCII characters (i.e. accented) in the CMS files if Dir.glob('**/*', base: local).include?(file.name.force_encoding('UTF-8')) if File.file?(File.join(local, file.name)) @@ -37,7 +37,7 @@ namespace :comfy do else puts "#{file.name} doesn\'t exist locally, downloading" # File doesn't exist locally, so download it (in any folder required) - session.scp.download!(remote_file, local, recursive: true) + session.scp.download!(remote_file, local_file) end end end From d5bba27c23d0e556ad7a2ba2b60fc753ceaa03e2 Mon Sep 17 00:00:00 2001 From: Stanley Liu Date: Sun, 26 Jul 2020 10:57:06 +0100 Subject: [PATCH 04/49] can't invoke import of cms seeds within rake task properly --- lib/tasks/staging_seeds.rake | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/tasks/staging_seeds.rake b/lib/tasks/staging_seeds.rake index 9524ed45a..ad552858a 100644 --- a/lib/tasks/staging_seeds.rake +++ b/lib/tasks/staging_seeds.rake @@ -31,7 +31,7 @@ namespace :comfy do # There are files with non-ASCII characters (i.e. accented) in the CMS files if Dir.glob('**/*', base: local).include?(file.name.force_encoding('UTF-8')) - if File.file?(File.join(local, file.name)) + if File.file?(local_file) check_timestamp(file, session, local, local_file, remote_file) end else @@ -44,7 +44,7 @@ namespace :comfy do puts "Finished downloads, now replacing your local seed data..." - Rake::Task["comfy:cms_seeds:import[protected-planet, protectedplanet]"].invoke + Rake::Task["'comfy:cms_seeds:import[protected-planet, protectedplanet]'"].invoke end From e5117a2de54175194ef44daa80f5321c9a2fedde Mon Sep 17 00:00:00 2001 From: Stanley Liu Date: Sun, 26 Jul 2020 11:05:35 +0100 Subject: [PATCH 05/49] now invokes the existing import task correctly --- lib/tasks/staging_seeds.rake | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/lib/tasks/staging_seeds.rake b/lib/tasks/staging_seeds.rake index ad552858a..8a741b8c4 100644 --- a/lib/tasks/staging_seeds.rake +++ b/lib/tasks/staging_seeds.rake @@ -44,8 +44,6 @@ namespace :comfy do puts "Finished downloads, now replacing your local seed data..." - Rake::Task["'comfy:cms_seeds:import[protected-planet, protectedplanet]'"].invoke - - + Rake::Task["comfy:cms_seeds:import"].invoke('protected-planet', 'protectedplanet') end end \ No newline at end of file From ef422608e7a8449c191e1b2305db9beac90ba55d Mon Sep 17 00:00:00 2001 From: Stanley Liu Date: Sun, 26 Jul 2020 11:07:41 +0100 Subject: [PATCH 06/49] got rid of unused argument --- lib/tasks/staging_seeds.rake | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/tasks/staging_seeds.rake b/lib/tasks/staging_seeds.rake index 8a741b8c4..079269f64 100644 --- a/lib/tasks/staging_seeds.rake +++ b/lib/tasks/staging_seeds.rake @@ -7,7 +7,7 @@ namespace :comfy do require 'net/ssh' require 'net/scp' - def check_timestamp(file, session, local, local_file, remote_file) + def check_timestamp(file, session, local_file, remote_file) newer = Time.at(file.attributes.mtime) >= File.stat(local_file).mtime if newer puts "#{file.name} is newer than the local copy, downloading..." @@ -32,7 +32,7 @@ namespace :comfy do # There are files with non-ASCII characters (i.e. accented) in the CMS files if Dir.glob('**/*', base: local).include?(file.name.force_encoding('UTF-8')) if File.file?(local_file) - check_timestamp(file, session, local, local_file, remote_file) + check_timestamp(file, session, local_file, remote_file) end else puts "#{file.name} doesn\'t exist locally, downloading" From f68da01edabb213b33ff3493d4ecfb53ac3c0542 Mon Sep 17 00:00:00 2001 From: Stanley Liu Date: Mon, 27 Jul 2020 15:35:26 +0100 Subject: [PATCH 07/49] symlinked cms seeds folder --- config/deploy.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/deploy.rb b/config/deploy.rb index 19ef672fb..78e4feb79 100644 --- a/config/deploy.rb +++ b/config/deploy.rb @@ -23,7 +23,7 @@ set :linked_files, %w{config/database.yml .env} -set :linked_dirs, fetch(:linked_dirs, []).push('log', 'tmp/pids', 'tmp/cache', 'tmp/sockets', 'vendor/bundle', 'public/system', 'public/.well-known') +set :linked_dirs, fetch(:linked_dirs, []).push('log', 'tmp/pids', 'tmp/cache', 'tmp/sockets', 'vendor/bundle', 'public/system', 'public/.well-known', 'db/cms_seeds') set :keep_releases, 5 From 3d8266a898209c07897792a52fe8068b91344edd Mon Sep 17 00:00:00 2001 From: Stanley Liu Date: Tue, 28 Jul 2020 11:55:47 +0100 Subject: [PATCH 08/49] move function and constant variables outside of task scope --- lib/tasks/staging_seeds.rake | 36 +++++++++++++++++++----------------- 1 file changed, 19 insertions(+), 17 deletions(-) diff --git a/lib/tasks/staging_seeds.rake b/lib/tasks/staging_seeds.rake index 079269f64..d5557552d 100644 --- a/lib/tasks/staging_seeds.rake +++ b/lib/tasks/staging_seeds.rake @@ -3,34 +3,34 @@ namespace :comfy do desc "Import CMS Seed data from staging" + def check_timestamp(file, session, local_file, remote_file) + is_newer = Time.at(file.attributes.mtime) >= File.stat(local_file).mtime + if is_newer + puts "#{file.name} is newer than the local copy, downloading..." + session.scp.download!(remote_file, local_file) + end + end + + # Locally stored seeds - assumes you already have the local folder - + # it won't create it + LOCAL = (ComfortableMexicanSofa.config.seeds_path + '/protected-planet').freeze + REMOTE = 'ProtectedPlanet/current/db/cms_seeds/protected-planet'.freeze + task :seed_import => :environment do |_t| require 'net/ssh' - require 'net/scp' - - def check_timestamp(file, session, local_file, remote_file) - newer = Time.at(file.attributes.mtime) >= File.stat(local_file).mtime - if newer - puts "#{file.name} is newer than the local copy, downloading..." - session.scp.download!(remote_file, local_file) - end - end - - # Locally stored seeds - assumes you already have the local folder - - # it won't create it - local = ComfortableMexicanSofa.config.seeds_path + '/protected-planet' - remote = 'ProtectedPlanet/current/db/cms_seeds/protected-planet' + require 'net/scp' puts "Importing CMS Seed data from Staging Folder to #{local} ..." # SSH into staging server with Net::SSH Net::SSH.start(ENV['PP_STAGING'], ENV['PP_USER']) do |session| - session.sftp.dir.glob(remote, '**/*').each do |file| + session.sftp.dir.glob(REMOTE, '**/*').each do |file| # Go through the various files and folders and check to see if they exist locally remote_file = File.join(remote, file.name) - local_file = File.join(local, file.name) + local_file = File.join(LOCAL, file.name) # There are files with non-ASCII characters (i.e. accented) in the CMS files - if Dir.glob('**/*', base: local).include?(file.name.force_encoding('UTF-8')) + if Dir.glob('**/*', base: LOCAL).include?(file.name.force_encoding('UTF-8')) if File.file?(local_file) check_timestamp(file, session, local_file, remote_file) end @@ -45,5 +45,7 @@ namespace :comfy do puts "Finished downloads, now replacing your local seed data..." Rake::Task["comfy:cms_seeds:import"].invoke('protected-planet', 'protectedplanet') + + # Todo: get this working with AWS bucket end end \ No newline at end of file From 7ba1e6a5ef296d46d218af9241a5f54489dbd544 Mon Sep 17 00:00:00 2001 From: Stanley Liu Date: Tue, 28 Jul 2020 16:53:08 +0100 Subject: [PATCH 09/49] working on changing it to use less scp --- lib/tasks/staging_seeds.rake | 69 +++++++++++++++++++++++------------- 1 file changed, 44 insertions(+), 25 deletions(-) diff --git a/lib/tasks/staging_seeds.rake b/lib/tasks/staging_seeds.rake index d5557552d..9ffb75be3 100644 --- a/lib/tasks/staging_seeds.rake +++ b/lib/tasks/staging_seeds.rake @@ -3,14 +3,6 @@ namespace :comfy do desc "Import CMS Seed data from staging" - def check_timestamp(file, session, local_file, remote_file) - is_newer = Time.at(file.attributes.mtime) >= File.stat(local_file).mtime - if is_newer - puts "#{file.name} is newer than the local copy, downloading..." - session.scp.download!(remote_file, local_file) - end - end - # Locally stored seeds - assumes you already have the local folder - # it won't create it LOCAL = (ComfortableMexicanSofa.config.seeds_path + '/protected-planet').freeze @@ -20,32 +12,59 @@ namespace :comfy do require 'net/ssh' require 'net/scp' - puts "Importing CMS Seed data from Staging Folder to #{local} ..." + puts "Importing CMS Seed data from Staging Folder to #{LOCAL} ..." # SSH into staging server with Net::SSH Net::SSH.start(ENV['PP_STAGING'], ENV['PP_USER']) do |session| - session.sftp.dir.glob(REMOTE, '**/*').each do |file| - # Go through the various files and folders and check to see if they exist locally - remote_file = File.join(remote, file.name) - local_file = File.join(LOCAL, file.name) - - # There are files with non-ASCII characters (i.e. accented) in the CMS files - if Dir.glob('**/*', base: LOCAL).include?(file.name.force_encoding('UTF-8')) - if File.file?(local_file) - check_timestamp(file, session, local_file, remote_file) + # Map the top-level folders and check top-level files + top_level_folders = session.sftp.dir.glob(REMOTE,'*').filter do |item| + item.attributes.directory? + end + + session.sftp.dir.glob(REMOTE, '*').each do |file| + remote_folder = File.join(REMOTE, file.name) + local_folder = File.join(LOCAL, file.name) + + unless top_level_folders.include?(file) + is_newer = Time.at(file.attributes.mtime) >= File.stat(local_folder).mtime + session.scp.download!(remote_folder, local_folder) if is_newer + end + end + + top_level_folders.each do |folder| + parent_remote = File.join(REMOTE, folder.name) + parent_local = File.join(LOCAL, folder.name) + + unless Dir.glob('**/*', base: LOCAL).include?(folder.name.force_encoding('UTF-8')) + puts "#{folder.name} doesn\'t exist locally, downloading" + # Folder doesn't exist locally, so download it + session.scp.download!(parent_remote, parent_local, recursive: true) + end + + files = [] + + session.sftp.dir.glob(parent_remote, '**/*').each do |file| + # Go through the various files and folders and check to see if they exist locally + local_folder = File.join(LOCAL, file.name) + + # There are files with non-ASCII characters (i.e. accented) in the CMS files + if Dir.glob('**/*', base: local_folder).include?(file.name.force_encoding('UTF-8')) + is_newer = Time.at(file.attributes.mtime) >= File.stat(local_folder).mtime + files << file if is_newer end - else - puts "#{file.name} doesn\'t exist locally, downloading" - # File doesn't exist locally, so download it (in any folder required) - session.scp.download!(remote_file, local_file) + end + + if files.length >= 1 + puts "Downloading a newer version of #{folder.name}" + session.scp.download!(parent_local, parent_remote, recursive: true) end end - end - puts "Finished downloads, now replacing your local seed data..." + # puts "Finished downloads, now replacing your local seed data..." - Rake::Task["comfy:cms_seeds:import"].invoke('protected-planet', 'protectedplanet') + # Rake::Task["comfy:cms_seeds:import"].invoke('protected-planet', 'protectedplanet') # Todo: get this working with AWS bucket end + end end \ No newline at end of file From 2eb1e4731c13a5055ae8813d5a722b03872c27f1 Mon Sep 17 00:00:00 2001 From: Stanley Liu Date: Tue, 28 Jul 2020 18:10:24 +0100 Subject: [PATCH 10/49] reworked so that it will download entire folders at a time --- lib/tasks/staging_seeds.rake | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/lib/tasks/staging_seeds.rake b/lib/tasks/staging_seeds.rake index 9ffb75be3..22f3dc347 100644 --- a/lib/tasks/staging_seeds.rake +++ b/lib/tasks/staging_seeds.rake @@ -24,10 +24,17 @@ namespace :comfy do session.sftp.dir.glob(REMOTE, '*').each do |file| remote_folder = File.join(REMOTE, file.name) local_folder = File.join(LOCAL, file.name) - - unless top_level_folders.include?(file) - is_newer = Time.at(file.attributes.mtime) >= File.stat(local_folder).mtime - session.scp.download!(remote_folder, local_folder) if is_newer + + # only files + unless top_level_folders.find { |f| file.name == f.name } + if File.exist?(local_folder) + is_newer = Time.at(file.attributes.mtime) >= File.stat(local_folder).mtime + puts "Downloading a newer version of #{file.name}" + session.scp.download!(remote_folder, local_folder) if is_newer + else + puts "#{file.name} doesn't exist locally, downloading..." + session.scp.download!(remote_folder, local_folder) + end end end @@ -38,7 +45,7 @@ namespace :comfy do unless Dir.glob('**/*', base: LOCAL).include?(folder.name.force_encoding('UTF-8')) puts "#{folder.name} doesn\'t exist locally, downloading" # Folder doesn't exist locally, so download it - session.scp.download!(parent_remote, parent_local, recursive: true) + session.scp.download!(parent_remote, LOCAL, recursive: true) end files = [] @@ -51,18 +58,20 @@ namespace :comfy do if Dir.glob('**/*', base: local_folder).include?(file.name.force_encoding('UTF-8')) is_newer = Time.at(file.attributes.mtime) >= File.stat(local_folder).mtime files << file if is_newer + else + files << file end end if files.length >= 1 puts "Downloading a newer version of #{folder.name}" - session.scp.download!(parent_local, parent_remote, recursive: true) + session.scp.download!(parent_remote, LOCAL, recursive: true) end end - # puts "Finished downloads, now replacing your local seed data..." + puts "Finished downloads, now replacing your local seed data..." - # Rake::Task["comfy:cms_seeds:import"].invoke('protected-planet', 'protectedplanet') + Rake::Task["comfy:cms_seeds:import"].invoke('protected-planet', 'protectedplanet') # Todo: get this working with AWS bucket end From 5ec326e58f178111fd5d5f52784b88ee49ac3d2e Mon Sep 17 00:00:00 2001 From: Stanley Liu Date: Tue, 28 Jul 2020 21:49:09 +0100 Subject: [PATCH 11/49] added env variables back into script --- lib/tasks/staging_seeds.rake | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/tasks/staging_seeds.rake b/lib/tasks/staging_seeds.rake index 22f3dc347..978ea679a 100644 --- a/lib/tasks/staging_seeds.rake +++ b/lib/tasks/staging_seeds.rake @@ -7,6 +7,8 @@ namespace :comfy do # it won't create it LOCAL = (ComfortableMexicanSofa.config.seeds_path + '/protected-planet').freeze REMOTE = 'ProtectedPlanet/current/db/cms_seeds/protected-planet'.freeze + PP_STAGING = 'new-web.pp-staging.linode.protectedplanet.net'.freeze + PP_USER = 'wcmc'.freeze task :seed_import => :environment do |_t| require 'net/ssh' @@ -15,7 +17,7 @@ namespace :comfy do puts "Importing CMS Seed data from Staging Folder to #{LOCAL} ..." # SSH into staging server with Net::SSH - Net::SSH.start(ENV['PP_STAGING'], ENV['PP_USER']) do |session| + Net::SSH.start(PP_STAGING, PP_USER) do |session| # Map the top-level folders and check top-level files top_level_folders = session.sftp.dir.glob(REMOTE,'*').filter do |item| item.attributes.directory? From 8fb31429f390d6189247a9d32dffcaec21230f82 Mon Sep 17 00:00:00 2001 From: Stanley Liu Date: Tue, 28 Jul 2020 21:50:00 +0100 Subject: [PATCH 12/49] renamed task in prep for the later task to import from S3 --- lib/tasks/staging_seeds.rake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/tasks/staging_seeds.rake b/lib/tasks/staging_seeds.rake index 978ea679a..c350a64f9 100644 --- a/lib/tasks/staging_seeds.rake +++ b/lib/tasks/staging_seeds.rake @@ -10,7 +10,7 @@ namespace :comfy do PP_STAGING = 'new-web.pp-staging.linode.protectedplanet.net'.freeze PP_USER = 'wcmc'.freeze - task :seed_import => :environment do |_t| + task :staging_import => :environment do |_t| require 'net/ssh' require 'net/scp' From c353d2af9dd730a4ea088aa56e3eb035078703c7 Mon Sep 17 00:00:00 2001 From: Stanley Liu Date: Wed, 29 Jul 2020 16:04:47 +0100 Subject: [PATCH 13/49] included a function to check for and remove any top-level folders/files no longer present remotely --- lib/tasks/staging_seeds.rake | 47 +++++++++++++++++++++--------------- 1 file changed, 28 insertions(+), 19 deletions(-) diff --git a/lib/tasks/staging_seeds.rake b/lib/tasks/staging_seeds.rake index c350a64f9..c0a1b5a34 100644 --- a/lib/tasks/staging_seeds.rake +++ b/lib/tasks/staging_seeds.rake @@ -10,6 +10,11 @@ namespace :comfy do PP_STAGING = 'new-web.pp-staging.linode.protectedplanet.net'.freeze PP_USER = 'wcmc'.freeze + def files_to_be_deleted(remote_list) + files = Dir.glob('*', base: LOCAL) - remote_list.map { |f| f.name } + files.each { |file| FileUtils.rm_rf(File.join(LOCAL, file)) } + end + task :staging_import => :environment do |_t| require 'net/ssh' require 'net/scp' @@ -18,28 +23,32 @@ namespace :comfy do # SSH into staging server with Net::SSH Net::SSH.start(PP_STAGING, PP_USER) do |session| + remote_list = session.sftp.dir.glob(REMOTE,'*') + + # First get rid of any local top-level (i.e. which exist in the main + # directory of REMOTE) folders/files that don't exist remotely + files_to_be_deleted(remote_list) + # Map the top-level folders and check top-level files - top_level_folders = session.sftp.dir.glob(REMOTE,'*').filter do |item| - item.attributes.directory? - end - - session.sftp.dir.glob(REMOTE, '*').each do |file| - remote_folder = File.join(REMOTE, file.name) - local_folder = File.join(LOCAL, file.name) - - # only files - unless top_level_folders.find { |f| file.name == f.name } - if File.exist?(local_folder) - is_newer = Time.at(file.attributes.mtime) >= File.stat(local_folder).mtime - puts "Downloading a newer version of #{file.name}" - session.scp.download!(remote_folder, local_folder) if is_newer - else - puts "#{file.name} doesn't exist locally, downloading..." - session.scp.download!(remote_folder, local_folder) - end - end + top_level_folders = remote_list.filter { |item| item.attributes.directory? } + top_level_files = remote_list.filter { |item| item.attributes.file? } + + # download only files + top_level_files.each do |file| + remote_file = File.join(REMOTE, file.name) + local_file = File.join(LOCAL, file.name) + + if File.exist?(local_file) + is_newer = Time.at(file.attributes.mtime) >= File.stat(local_file).mtime + puts "Downloading a newer version of #{file.name}" + session.scp.download!(remote_file, local_file) if is_newer + else + puts "#{file.name} doesn't exist locally, downloading..." + session.scp.download!(remote_file, local_file) + end end + # Start recursively delving into the folders top_level_folders.each do |folder| parent_remote = File.join(REMOTE, folder.name) parent_local = File.join(LOCAL, folder.name) From a9d4acc1a841b03a40629ca33952018f3e5cf8ea Mon Sep 17 00:00:00 2001 From: Stanley Liu Date: Wed, 29 Jul 2020 16:40:45 +0100 Subject: [PATCH 14/49] added new functionality to delete files recursively for subdirectories --- lib/tasks/staging_seeds.rake | 32 +++++++++++++++++++++++--------- 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/lib/tasks/staging_seeds.rake b/lib/tasks/staging_seeds.rake index c0a1b5a34..c1a886c6c 100644 --- a/lib/tasks/staging_seeds.rake +++ b/lib/tasks/staging_seeds.rake @@ -10,9 +10,22 @@ namespace :comfy do PP_STAGING = 'new-web.pp-staging.linode.protectedplanet.net'.freeze PP_USER = 'wcmc'.freeze - def files_to_be_deleted(remote_list) + def delete_top_level_files(remote_list) files = Dir.glob('*', base: LOCAL) - remote_list.map { |f| f.name } - files.each { |file| FileUtils.rm_rf(File.join(LOCAL, file)) } + + files.each do |file| + puts "Removing #{file} as it no longer exists remotely" + FileUtils.rm_rf(File.join(LOCAL, file)) + end + end + + def delete_files_recursively(parent_remote, parent_local) + files = Dir.glob('**/*', base: parent_local) - session.sftp.dir.glob(parent_remote, '**/*').map { |f| f.name } + + files.each do |file| + puts "Removing #{file} as it no longer exists remotely" + FileUtils.rm_rf(File.join(LOCAL, file)) + end end task :staging_import => :environment do |_t| @@ -27,7 +40,7 @@ namespace :comfy do # First get rid of any local top-level (i.e. which exist in the main # directory of REMOTE) folders/files that don't exist remotely - files_to_be_deleted(remote_list) + delete_top_level_files(remote_list) # Map the top-level folders and check top-level files top_level_folders = remote_list.filter { |item| item.attributes.directory? } @@ -53,9 +66,10 @@ namespace :comfy do parent_remote = File.join(REMOTE, folder.name) parent_local = File.join(LOCAL, folder.name) - unless Dir.glob('**/*', base: LOCAL).include?(folder.name.force_encoding('UTF-8')) + delete_files_recursively(parent_remote, parent_local) + + unless Dir.glob('*', base: LOCAL).include?(folder.name) puts "#{folder.name} doesn\'t exist locally, downloading" - # Folder doesn't exist locally, so download it session.scp.download!(parent_remote, LOCAL, recursive: true) end @@ -63,12 +77,12 @@ namespace :comfy do session.sftp.dir.glob(parent_remote, '**/*').each do |file| # Go through the various files and folders and check to see if they exist locally - local_folder = File.join(LOCAL, file.name) + local_folder = File.join(folder.name, file.name) # There are files with non-ASCII characters (i.e. accented) in the CMS files - if Dir.glob('**/*', base: local_folder).include?(file.name.force_encoding('UTF-8')) - is_newer = Time.at(file.attributes.mtime) >= File.stat(local_folder).mtime - files << file if is_newer + if Dir.glob('**/*', base: LOCAL).include?(local_folder.force_encoding('UTF-8')) + is_newer = Time.at(file.attributes.mtime) >= File.stat(File.join(LOCAL, local_folder)).mtime + files << file if is_newer else files << file end From ff6363fc36427ed5024cda4a394d2c052f90a3a8 Mon Sep 17 00:00:00 2001 From: Stanley Liu Date: Wed, 29 Jul 2020 16:47:07 +0100 Subject: [PATCH 15/49] bit of refactoring to tidy things up --- lib/tasks/staging_seeds.rake | 32 ++++++++++++++++++-------------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/lib/tasks/staging_seeds.rake b/lib/tasks/staging_seeds.rake index c1a886c6c..c1f2e6295 100644 --- a/lib/tasks/staging_seeds.rake +++ b/lib/tasks/staging_seeds.rake @@ -10,22 +10,23 @@ namespace :comfy do PP_STAGING = 'new-web.pp-staging.linode.protectedplanet.net'.freeze PP_USER = 'wcmc'.freeze - def delete_top_level_files(remote_list) - files = Dir.glob('*', base: LOCAL) - remote_list.map { |f| f.name } - + def delete_files(files) files.each do |file| puts "Removing #{file} as it no longer exists remotely" FileUtils.rm_rf(File.join(LOCAL, file)) end end - def delete_files_recursively(parent_remote, parent_local) - files = Dir.glob('**/*', base: parent_local) - session.sftp.dir.glob(parent_remote, '**/*').map { |f| f.name } + def delete_top_level_files(local_list, remote_list) + files = local_list - remote_list.map { |f| f.name } + + delete_files(files) + end - files.each do |file| - puts "Removing #{file} as it no longer exists remotely" - FileUtils.rm_rf(File.join(LOCAL, file)) - end + def delete_files_recursively(parent_local, remote_folder_content) + files = Dir.glob('**/*', base: parent_local) - remote_folder_content.map { |f| f.name } + + delete_files(files) end task :staging_import => :environment do |_t| @@ -36,11 +37,12 @@ namespace :comfy do # SSH into staging server with Net::SSH Net::SSH.start(PP_STAGING, PP_USER) do |session| - remote_list = session.sftp.dir.glob(REMOTE,'*') + remote_list = session.sftp.dir.glob(REMOTE, '*') + local_list = Dir.glob('*', base: LOCAL) # First get rid of any local top-level (i.e. which exist in the main # directory of REMOTE) folders/files that don't exist remotely - delete_top_level_files(remote_list) + delete_top_level_files(local_list, remote_list) # Map the top-level folders and check top-level files top_level_folders = remote_list.filter { |item| item.attributes.directory? } @@ -65,17 +67,18 @@ namespace :comfy do top_level_folders.each do |folder| parent_remote = File.join(REMOTE, folder.name) parent_local = File.join(LOCAL, folder.name) + remote_folder_content = session.sftp.dir.glob(parent_remote, '**/*') - delete_files_recursively(parent_remote, parent_local) + delete_files_recursively(parent_local, remote_folder_content) - unless Dir.glob('*', base: LOCAL).include?(folder.name) + unless local_list.include?(folder.name) puts "#{folder.name} doesn\'t exist locally, downloading" session.scp.download!(parent_remote, LOCAL, recursive: true) end files = [] - session.sftp.dir.glob(parent_remote, '**/*').each do |file| + remote_folder.each do |file| # Go through the various files and folders and check to see if they exist locally local_folder = File.join(folder.name, file.name) @@ -88,6 +91,7 @@ namespace :comfy do end end + # If there are any outdated files, will trigger download if files.length >= 1 puts "Downloading a newer version of #{folder.name}" session.scp.download!(parent_remote, LOCAL, recursive: true) From f9b5ad1cdeebcab3a6eb66d02dd6f876290b96bc Mon Sep 17 00:00:00 2001 From: Stanley Liu Date: Thu, 30 Jul 2020 10:01:09 +0100 Subject: [PATCH 16/49] task should delete files from the right location --- lib/tasks/staging_seeds.rake | 21 ++++++++------------- 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/lib/tasks/staging_seeds.rake b/lib/tasks/staging_seeds.rake index c1f2e6295..b2a0be218 100644 --- a/lib/tasks/staging_seeds.rake +++ b/lib/tasks/staging_seeds.rake @@ -10,23 +10,17 @@ namespace :comfy do PP_STAGING = 'new-web.pp-staging.linode.protectedplanet.net'.freeze PP_USER = 'wcmc'.freeze - def delete_files(files) + def delete_files(files, location) files.each do |file| puts "Removing #{file} as it no longer exists remotely" - FileUtils.rm_rf(File.join(LOCAL, file)) + FileUtils.rm_rf(File.join(location, file)) end end - def delete_top_level_files(local_list, remote_list) - files = local_list - remote_list.map { |f| f.name } + def delete_files(list_of_files_1, list_of_files_2, location = LOCAL) + files = list_of_files_1 - list_of_files_2.map { |f| f.name } - delete_files(files) - end - - def delete_files_recursively(parent_local, remote_folder_content) - files = Dir.glob('**/*', base: parent_local) - remote_folder_content.map { |f| f.name } - - delete_files(files) + delete_files(files, location) end task :staging_import => :environment do |_t| @@ -42,7 +36,7 @@ namespace :comfy do # First get rid of any local top-level (i.e. which exist in the main # directory of REMOTE) folders/files that don't exist remotely - delete_top_level_files(local_list, remote_list) + delete_files(local_list, remote_list) # Map the top-level folders and check top-level files top_level_folders = remote_list.filter { |item| item.attributes.directory? } @@ -67,9 +61,10 @@ namespace :comfy do top_level_folders.each do |folder| parent_remote = File.join(REMOTE, folder.name) parent_local = File.join(LOCAL, folder.name) + local_folder_content = Dir.glob('**/*', base: parent_local) remote_folder_content = session.sftp.dir.glob(parent_remote, '**/*') - delete_files_recursively(parent_local, remote_folder_content) + delete_files(local_folder_content, remote_folder_content, parent_local) unless local_list.include?(folder.name) puts "#{folder.name} doesn\'t exist locally, downloading" From 25c993eb9204f23e6f3022e639f192e8c1e5f8a8 Mon Sep 17 00:00:00 2001 From: Stanley Liu Date: Thu, 30 Jul 2020 10:19:11 +0100 Subject: [PATCH 17/49] added condition to handle case of when no files are actually needed to be deleted --- lib/tasks/staging_seeds.rake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/tasks/staging_seeds.rake b/lib/tasks/staging_seeds.rake index b2a0be218..9e5022d75 100644 --- a/lib/tasks/staging_seeds.rake +++ b/lib/tasks/staging_seeds.rake @@ -20,7 +20,7 @@ namespace :comfy do def delete_files(list_of_files_1, list_of_files_2, location = LOCAL) files = list_of_files_1 - list_of_files_2.map { |f| f.name } - delete_files(files, location) + delete_files(files, location) unless files.empty? end task :staging_import => :environment do |_t| From eeac3092ad91a2604fc41f6fb0a62d2729ec1358 Mon Sep 17 00:00:00 2001 From: Stanley Liu Date: Thu, 30 Jul 2020 18:58:19 +0100 Subject: [PATCH 18/49] fixed a small reference issue and added comments to refactor this into a more intelligent task --- lib/tasks/staging_seeds.rake | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/lib/tasks/staging_seeds.rake b/lib/tasks/staging_seeds.rake index 9e5022d75..d09ba347e 100644 --- a/lib/tasks/staging_seeds.rake +++ b/lib/tasks/staging_seeds.rake @@ -73,7 +73,7 @@ namespace :comfy do files = [] - remote_folder.each do |file| + remote_folder_content.each do |file| # Go through the various files and folders and check to see if they exist locally local_folder = File.join(folder.name, file.name) @@ -100,4 +100,11 @@ namespace :comfy do # Todo: get this working with AWS bucket end end -end \ No newline at end of file +end + +# Start from the top-level folder, and check each object in the folder in turn +# If the object is a folder, check each object within, if it is a folder check it and so on. +# For both files and folders, +# if it exists on remote and local, check the timestamp of the local version +# if it exists on remote, but not locally, download it +# if it doesn't exist on remote, but exists locally, delete it \ No newline at end of file From a48feb85ce9fe55edbaa9333b48c5649b0bca4c8 Mon Sep 17 00:00:00 2001 From: Stanley Liu Date: Fri, 31 Jul 2020 18:15:24 +0100 Subject: [PATCH 19/49] refactored to hopefully improve performance --- lib/tasks/staging_seeds.rake | 123 +++++++++++++++++------------------ 1 file changed, 61 insertions(+), 62 deletions(-) diff --git a/lib/tasks/staging_seeds.rake b/lib/tasks/staging_seeds.rake index d09ba347e..2ad77e9ec 100644 --- a/lib/tasks/staging_seeds.rake +++ b/lib/tasks/staging_seeds.rake @@ -17,10 +17,51 @@ namespace :comfy do end end - def delete_files(list_of_files_1, list_of_files_2, location = LOCAL) - files = list_of_files_1 - list_of_files_2.map { |f| f.name } - - delete_files(files, location) unless files.empty? + def files_for_deletion(list_of_files_1, list_of_files_2, location = LOCAL) + files = list_of_files_1 - list_of_files_2 + + begin + delete_files(files, location) + rescue TypeError + puts 'Nothing to delete locally, moving on to the next set of files' + end + end + + def download_file(source, dest, session) + puts "Downloading #{source} as it's newer or not found locally" + return session.scp.download!(source, dest) if File.file?(dest) + session.scp.download!(source, dest, recursive: true) + end + + def create_paths(relative_path) + { local_path: File.join(LOCAL, relative_path), remote_path: File.join(REMOTE, relative_path) } + end + + def folder_delving(folder, local_list, session) + paths = create_paths(folder) + local_folder_content = Dir.glob('**/*', base: paths[:local_path]) + + remote_folder_content = session.sftp.dir.glob(paths[:remote_path], '**/*') + + remote_content_names = remote_folder_content.map {|f| f.name.force_encoding('UTF-8') } + + files_for_deletion(local_folder_content, remote_content_names, paths[:local_path]) + + files = [] + + remote_folder_content.each do |file| + # Go through the various files and folders and check to see if they exist locally + local_folder = File.join(folder, file.name) + + # There are files with non-ASCII characters (i.e. accented) in the CMS files + if Dir.glob('**/*', base: LOCAL).include?(local_folder) + is_newer = Time.at(file.attributes.mtime) >= File.stat(File.join(LOCAL, local_folder)).mtime + # If there are any outdated files, will trigger download + download_file(paths[:remote_path], LOCAL, session) if is_newer + else + download_file(paths[:local_path], LOCAL, session) + end + end end task :staging_import => :environment do |_t| @@ -34,62 +75,27 @@ namespace :comfy do remote_list = session.sftp.dir.glob(REMOTE, '*') local_list = Dir.glob('*', base: LOCAL) + remote_list_names = remote_list.map { |f| f.name } + # First get rid of any local top-level (i.e. which exist in the main # directory of REMOTE) folders/files that don't exist remotely - delete_files(local_list, remote_list) - - # Map the top-level folders and check top-level files - top_level_folders = remote_list.filter { |item| item.attributes.directory? } - top_level_files = remote_list.filter { |item| item.attributes.file? } - - # download only files - top_level_files.each do |file| - remote_file = File.join(REMOTE, file.name) - local_file = File.join(LOCAL, file.name) - - if File.exist?(local_file) - is_newer = Time.at(file.attributes.mtime) >= File.stat(local_file).mtime - puts "Downloading a newer version of #{file.name}" - session.scp.download!(remote_file, local_file) if is_newer - else - puts "#{file.name} doesn't exist locally, downloading..." - session.scp.download!(remote_file, local_file) - end - end + files_for_deletion(local_list, remote_list_names) - # Start recursively delving into the folders - top_level_folders.each do |folder| - parent_remote = File.join(REMOTE, folder.name) - parent_local = File.join(LOCAL, folder.name) - local_folder_content = Dir.glob('**/*', base: parent_local) - remote_folder_content = session.sftp.dir.glob(parent_remote, '**/*') + remote_list.each do |object| + name = object.name.force_encoding('UTF-8') + paths = create_paths(name) - delete_files(local_folder_content, remote_folder_content, parent_local) - - unless local_list.include?(folder.name) - puts "#{folder.name} doesn\'t exist locally, downloading" - session.scp.download!(parent_remote, LOCAL, recursive: true) - end - - files = [] - - remote_folder_content.each do |file| - # Go through the various files and folders and check to see if they exist locally - local_folder = File.join(folder.name, file.name) - - # There are files with non-ASCII characters (i.e. accented) in the CMS files - if Dir.glob('**/*', base: LOCAL).include?(local_folder.force_encoding('UTF-8')) - is_newer = Time.at(file.attributes.mtime) >= File.stat(File.join(LOCAL, local_folder)).mtime - files << file if is_newer - else - files << file + if local_list.include?(name) + # If folder, look inside it to any files that don't exist any more remotely and delete them + if object.attributes.directory? + folder_delving(name, local_list, session) end - end - # If there are any outdated files, will trigger download - if files.length >= 1 - puts "Downloading a newer version of #{folder.name}" - session.scp.download!(parent_remote, LOCAL, recursive: true) + is_newer = Time.at(object.attributes.mtime) >= File.stat(paths[:local_path]).mtime + download_file(paths[:remote_path], paths[:local_path], session) if is_newer + else + # file doesn't exist locally so download it + download_file(paths[:remote_path], LOCAL, session) end end @@ -100,11 +106,4 @@ namespace :comfy do # Todo: get this working with AWS bucket end end -end - -# Start from the top-level folder, and check each object in the folder in turn -# If the object is a folder, check each object within, if it is a folder check it and so on. -# For both files and folders, -# if it exists on remote and local, check the timestamp of the local version -# if it exists on remote, but not locally, download it -# if it doesn't exist on remote, but exists locally, delete it \ No newline at end of file +end \ No newline at end of file From e1b9235e41091cd918b5fe90f0fb8ca3071abcd0 Mon Sep 17 00:00:00 2001 From: Stanley Liu Date: Mon, 3 Aug 2020 08:50:45 +0100 Subject: [PATCH 20/49] remove the cms seeds shared folder from the set of linked dirs --- config/deploy.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/deploy.rb b/config/deploy.rb index 78e4feb79..19ef672fb 100644 --- a/config/deploy.rb +++ b/config/deploy.rb @@ -23,7 +23,7 @@ set :linked_files, %w{config/database.yml .env} -set :linked_dirs, fetch(:linked_dirs, []).push('log', 'tmp/pids', 'tmp/cache', 'tmp/sockets', 'vendor/bundle', 'public/system', 'public/.well-known', 'db/cms_seeds') +set :linked_dirs, fetch(:linked_dirs, []).push('log', 'tmp/pids', 'tmp/cache', 'tmp/sockets', 'vendor/bundle', 'public/system', 'public/.well-known') set :keep_releases, 5 From cecf96a7ea89f39d1c01ab3c5c0743532d0e4c4c Mon Sep 17 00:00:00 2001 From: Stanley Liu Date: Mon, 3 Aug 2020 14:36:01 +0100 Subject: [PATCH 21/49] moved code block to check for files into a separate method --- lib/tasks/staging_seeds.rake | 85 +++++++++++++++++++++--------------- 1 file changed, 49 insertions(+), 36 deletions(-) diff --git a/lib/tasks/staging_seeds.rake b/lib/tasks/staging_seeds.rake index 2ad77e9ec..cf284bbbb 100644 --- a/lib/tasks/staging_seeds.rake +++ b/lib/tasks/staging_seeds.rake @@ -1,15 +1,16 @@ # frozen_string_literal: true namespace :comfy do - desc "Import CMS Seed data from staging" - - # Locally stored seeds - assumes you already have the local folder - - # it won't create it - LOCAL = (ComfortableMexicanSofa.config.seeds_path + '/protected-planet').freeze - REMOTE = 'ProtectedPlanet/current/db/cms_seeds/protected-planet'.freeze - PP_STAGING = 'new-web.pp-staging.linode.protectedplanet.net'.freeze - PP_USER = 'wcmc'.freeze - + + # Locally stored seeds - assumes you already have the local folder - + # it won't create it + LOCAL = (ComfortableMexicanSofa.config.seeds_path + '/protected-planet').freeze + REMOTE = 'ProtectedPlanet/current/db/cms_seeds/protected-planet'.freeze + PP_STAGING = 'new-web.pp-staging.linode.protectedplanet.net'.freeze + PP_USER = 'wcmc'.freeze + + desc "Import CMS Seed data from staging" + def delete_files(files, location) files.each do |file| puts "Removing #{file} as it no longer exists remotely" @@ -27,7 +28,7 @@ namespace :comfy do end end - def download_file(source, dest, session) + def download_files(source, dest, session) puts "Downloading #{source} as it's newer or not found locally" return session.scp.download!(source, dest) if File.file?(dest) session.scp.download!(source, dest, recursive: true) @@ -37,7 +38,27 @@ namespace :comfy do { local_path: File.join(LOCAL, relative_path), remote_path: File.join(REMOTE, relative_path) } end - def folder_delving(folder, local_list, session) + def check_if_newer(parent_folder, local_item, remote_item, remote_path, local_path, session, downloaded = false) + if parent_folder.include?(local_item) + yield if block_given? + + is_newer = Time.at(remote_item.attributes.mtime) >= File.stat(local_path).mtime + # If there are any outdated files, will trigger download + if is_newer + if Dir.glob('*', base: LOCAL).include?(local_item) + download_files(remote_path, local_path, session) + elsif Dir.glob('**/*', base: LOCAL).include?(local_item) + download_files(remote_path, LOCAL, session) + downloaded = true + end + end + else + download_files(remote_path, LOCAL, session) + downloaded = true + end + end + + def check_inside_folder(folder, local_list, session) paths = create_paths(folder) local_folder_content = Dir.glob('**/*', base: paths[:local_path]) @@ -47,20 +68,18 @@ namespace :comfy do files_for_deletion(local_folder_content, remote_content_names, paths[:local_path]) - files = [] - remote_folder_content.each do |file| # Go through the various files and folders and check to see if they exist locally - local_folder = File.join(folder, file.name) + local_file = file.name + absolute_path = File.join(paths[:local_path], file.name) + + # We don't want to download the whole folder again if it's already been re-downloaded once + downloaded_once = false - # There are files with non-ASCII characters (i.e. accented) in the CMS files - if Dir.glob('**/*', base: LOCAL).include?(local_folder) - is_newer = Time.at(file.attributes.mtime) >= File.stat(File.join(LOCAL, local_folder)).mtime - # If there are any outdated files, will trigger download - download_file(paths[:remote_path], LOCAL, session) if is_newer - else - download_file(paths[:local_path], LOCAL, session) - end + check_if_newer(local_folder_content, local_file, file, paths[:remote_path], absolute_path, session, downloaded_once) + + # Break out of loop if already downloaded + break if downloaded_once == true end end @@ -82,28 +101,22 @@ namespace :comfy do files_for_deletion(local_list, remote_list_names) remote_list.each do |object| + # There are files with non-ASCII characters (i.e. accented) in the CMS files name = object.name.force_encoding('UTF-8') paths = create_paths(name) - - if local_list.include?(name) - # If folder, look inside it to any files that don't exist any more remotely and delete them - if object.attributes.directory? - folder_delving(name, local_list, session) - end - - is_newer = Time.at(object.attributes.mtime) >= File.stat(paths[:local_path]).mtime - download_file(paths[:remote_path], paths[:local_path], session) if is_newer - else - # file doesn't exist locally so download it - download_file(paths[:remote_path], LOCAL, session) + + check_if_newer(local_list, name, object, paths[:remote_path], paths[:local_path], session) do + check_inside_folder(name, local_list, session) if object.attributes.directory? end end - puts "Finished downloads, now replacing your local seed data..." + # puts "Finished downloads, now replacing your local seed data..." - Rake::Task["comfy:cms_seeds:import"].invoke('protected-planet', 'protectedplanet') + # Rake::Task["comfy:cms_seeds:import"].invoke('protected-planet', 'protectedplanet') # Todo: get this working with AWS bucket end end + + desc "" end \ No newline at end of file From 8f829218270824ea9d0e6848ec110c4dd837e678 Mon Sep 17 00:00:00 2001 From: Stanley Liu Date: Mon, 3 Aug 2020 15:16:40 +0100 Subject: [PATCH 22/49] moved file comparison between local and remote into a method as well --- lib/tasks/staging_seeds.rake | 33 ++++++++++++++++++--------------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/lib/tasks/staging_seeds.rake b/lib/tasks/staging_seeds.rake index cf284bbbb..f88d9fa8f 100644 --- a/lib/tasks/staging_seeds.rake +++ b/lib/tasks/staging_seeds.rake @@ -7,6 +7,7 @@ namespace :comfy do LOCAL = (ComfortableMexicanSofa.config.seeds_path + '/protected-planet').freeze REMOTE = 'ProtectedPlanet/current/db/cms_seeds/protected-planet'.freeze PP_STAGING = 'new-web.pp-staging.linode.protectedplanet.net'.freeze + PP_PRODUCTION = 'new-web.pp-production.linode.protectedplanet.net'.freeze PP_USER = 'wcmc'.freeze desc "Import CMS Seed data from staging" @@ -18,7 +19,7 @@ namespace :comfy do end end - def files_for_deletion(list_of_files_1, list_of_files_2, location = LOCAL) + def files_for_deletion(list_of_files_1, list_of_files_2, location) files = list_of_files_1 - list_of_files_2 begin @@ -47,26 +48,33 @@ namespace :comfy do if is_newer if Dir.glob('*', base: LOCAL).include?(local_item) download_files(remote_path, local_path, session) + # Will be hit if local_item is a file or folder inside a directory elsif Dir.glob('**/*', base: LOCAL).include?(local_item) download_files(remote_path, LOCAL, session) downloaded = true end end else + # Just download it if it doesn't exist at all download_files(remote_path, LOCAL, session) downloaded = true end end - def check_inside_folder(folder, local_list, session) - paths = create_paths(folder) - local_folder_content = Dir.glob('**/*', base: paths[:local_path]) + def compare_folders(wildcard, local, remote, base = LOCAL) + remote_list = session.sftp.dir.glob(remote, wildcard).map do |f| + f.name.force_encoding('UTF-8') + end - remote_folder_content = session.sftp.dir.glob(paths[:remote_path], '**/*') + local_list = Dir.glob(wildcard, base: local) - remote_content_names = remote_folder_content.map {|f| f.name.force_encoding('UTF-8') } + files_for_deletion(local_list, remote_list_names, base) + end - files_for_deletion(local_folder_content, remote_content_names, paths[:local_path]) + def check_inside_folder(folder, local_list, session) + paths = create_paths(folder) + + compare_folders('**/*', paths[:local_path], paths[:remote_path], paths[:local_path]) remote_folder_content.each do |file| # Go through the various files and folders and check to see if they exist locally @@ -91,15 +99,10 @@ namespace :comfy do # SSH into staging server with Net::SSH Net::SSH.start(PP_STAGING, PP_USER) do |session| - remote_list = session.sftp.dir.glob(REMOTE, '*') - local_list = Dir.glob('*', base: LOCAL) - - remote_list_names = remote_list.map { |f| f.name } - - # First get rid of any local top-level (i.e. which exist in the main + # First get rid of any local top-level (i.e. which exist in the main # directory of REMOTE) folders/files that don't exist remotely - files_for_deletion(local_list, remote_list_names) - + compare_folders('*', LOCAL, REMOTE) + remote_list.each do |object| # There are files with non-ASCII characters (i.e. accented) in the CMS files name = object.name.force_encoding('UTF-8') From be8c3f4b23faf5775256d9567be213a64615ab34 Mon Sep 17 00:00:00 2001 From: Stanley Liu Date: Mon, 3 Aug 2020 15:34:54 +0100 Subject: [PATCH 23/49] set up a router style interface --- lib/tasks/staging_seeds.rake | 42 +++++++++++++++++++++++++++--------- 1 file changed, 32 insertions(+), 10 deletions(-) diff --git a/lib/tasks/staging_seeds.rake b/lib/tasks/staging_seeds.rake index f88d9fa8f..9f2507fa3 100644 --- a/lib/tasks/staging_seeds.rake +++ b/lib/tasks/staging_seeds.rake @@ -91,6 +91,18 @@ namespace :comfy do end end + def main_task(local_list, remote_list, session) + remote_list.each do |object| + # There are files with non-ASCII characters (i.e. accented) in the CMS files + name = object.name.force_encoding('UTF-8') + paths = create_paths(name) + + check_if_newer(local_list, name, object, paths[:remote_path], paths[:local_path], session) do + check_inside_folder(name, local_list, session) if object.attributes.directory? + end + end + end + task :staging_import => :environment do |_t| require 'net/ssh' require 'net/scp' @@ -102,17 +114,28 @@ namespace :comfy do # First get rid of any local top-level (i.e. which exist in the main # directory of REMOTE) folders/files that don't exist remotely compare_folders('*', LOCAL, REMOTE) - - remote_list.each do |object| - # There are files with non-ASCII characters (i.e. accented) in the CMS files - name = object.name.force_encoding('UTF-8') - paths = create_paths(name) - - check_if_newer(local_list, name, object, paths[:remote_path], paths[:local_path], session) do - check_inside_folder(name, local_list, session) if object.attributes.directory? - end + + puts question = "What would you like to import? 'All/Files/Layouts' or 'Nothing' to quit" + valid_answers = ['All', 'Files', 'Layouts', 'Pages', 'Nothing'] + answer = STDIN.gets.chomp.downcase.capitalise + until valid_answers.include?(answer) + puts question + answer = STDIN.gets.chomp.downcase.capitalise end + local_list = Dir.glob('*', base: LOCAL) + remote_list = session.sftp.dir.glob(LOCAL, '*') + + case answer + when 'All' + main_task(local_list, remote_list, session) + when 'Nothing' + abort('Goodbye') + else + remote_list.filter! { |f| f == answer.downcase } + main_task(local_list, remote_list, session) + end + end # puts "Finished downloads, now replacing your local seed data..." # Rake::Task["comfy:cms_seeds:import"].invoke('protected-planet', 'protectedplanet') @@ -121,5 +144,4 @@ namespace :comfy do end end - desc "" end \ No newline at end of file From 8c4bac36aa7d6bee31c9355bd744a0d56b161461 Mon Sep 17 00:00:00 2001 From: Stanley Liu Date: Mon, 3 Aug 2020 16:53:44 +0100 Subject: [PATCH 24/49] fixed issues with router --- lib/tasks/staging_seeds.rake | 83 +++++++++++++++++++++--------------- 1 file changed, 49 insertions(+), 34 deletions(-) diff --git a/lib/tasks/staging_seeds.rake b/lib/tasks/staging_seeds.rake index 9f2507fa3..134f5325f 100644 --- a/lib/tasks/staging_seeds.rake +++ b/lib/tasks/staging_seeds.rake @@ -22,10 +22,10 @@ namespace :comfy do def files_for_deletion(list_of_files_1, list_of_files_2, location) files = list_of_files_1 - list_of_files_2 - begin + if files.empty? + puts "No files to delete" + else delete_files(files, location) - rescue TypeError - puts 'Nothing to delete locally, moving on to the next set of files' end end @@ -39,7 +39,7 @@ namespace :comfy do { local_path: File.join(LOCAL, relative_path), remote_path: File.join(REMOTE, relative_path) } end - def check_if_newer(parent_folder, local_item, remote_item, remote_path, local_path, session, downloaded = false) + def check_if_newer(parent_folder, local_item, remote_item, remote_path, local_path, session, base = LOCAL) if parent_folder.include?(local_item) yield if block_given? @@ -49,45 +49,52 @@ namespace :comfy do if Dir.glob('*', base: LOCAL).include?(local_item) download_files(remote_path, local_path, session) # Will be hit if local_item is a file or folder inside a directory - elsif Dir.glob('**/*', base: LOCAL).include?(local_item) - download_files(remote_path, LOCAL, session) + elsif Dir.glob('**/*', base: base).include?(local_item) + download_files(remote_path, base, session) downloaded = true - end + end end else # Just download it if it doesn't exist at all download_files(remote_path, LOCAL, session) downloaded = true end + + downloaded end - def compare_folders(wildcard, local, remote, base = LOCAL) + def compare_folders(wildcard, local, remote, session, base = LOCAL) + puts "Checking to see what files need to be deleted from #{base}" + remote_list = session.sftp.dir.glob(remote, wildcard).map do |f| f.name.force_encoding('UTF-8') end local_list = Dir.glob(wildcard, base: local) - files_for_deletion(local_list, remote_list_names, base) + files_for_deletion(local_list, remote_list, base) end def check_inside_folder(folder, local_list, session) paths = create_paths(folder) - compare_folders('**/*', paths[:local_path], paths[:remote_path], paths[:local_path]) + compare_folders('**/*', paths[:local_path], paths[:remote_path], session, paths[:local_path]) + + local_folder_content = Dir.glob('**/*', base: paths[:local_path]) + remote_folder_content = session.sftp.dir.glob(paths[:remote_path], '**/*') + + # We don't want to download the whole folder again if it's already been re-downloaded once + remote_folder_content.each do |file| # Go through the various files and folders and check to see if they exist locally local_file = file.name absolute_path = File.join(paths[:local_path], file.name) - - # We don't want to download the whole folder again if it's already been re-downloaded once - downloaded_once = false - check_if_newer(local_folder_content, local_file, file, paths[:remote_path], absolute_path, session, downloaded_once) - + check = check_if_newer(local_folder_content, local_file, file, paths[:remote_path], absolute_path, session, paths[:local_path]) + # Break out of loop if already downloaded - break if downloaded_once == true + break if check == true end end @@ -97,6 +104,7 @@ namespace :comfy do name = object.name.force_encoding('UTF-8') paths = create_paths(name) + check_if_newer(local_list, name, object, paths[:remote_path], paths[:local_path], session) do check_inside_folder(name, local_list, session) if object.attributes.directory? end @@ -109,33 +117,40 @@ namespace :comfy do puts "Importing CMS Seed data from Staging Folder to #{LOCAL} ..." + puts question = "What would you like to import? 'All/Files/Layouts/Pages' or 'Nothing' to quit" + valid_answers = ['All', 'Files', 'Layouts', 'Pages', 'Nothing'] + answer = STDIN.gets.chomp.downcase.capitalize + until valid_answers.include?(answer) + puts question + answer = STDIN.gets.chomp.downcase.capitalize + end + + # Fast-tracked unhappy path + abort('Goodbye') if answer == 'Nothing' + # SSH into staging server with Net::SSH Net::SSH.start(PP_STAGING, PP_USER) do |session| # First get rid of any local top-level (i.e. which exist in the main # directory of REMOTE) folders/files that don't exist remotely - compare_folders('*', LOCAL, REMOTE) - - puts question = "What would you like to import? 'All/Files/Layouts' or 'Nothing' to quit" - valid_answers = ['All', 'Files', 'Layouts', 'Pages', 'Nothing'] - answer = STDIN.gets.chomp.downcase.capitalise - until valid_answers.include?(answer) - puts question - answer = STDIN.gets.chomp.downcase.capitalise - end + compare_folders('*', LOCAL, REMOTE, session) local_list = Dir.glob('*', base: LOCAL) - remote_list = session.sftp.dir.glob(LOCAL, '*') - - case answer - when 'All' - main_task(local_list, remote_list, session) - when 'Nothing' - abort('Goodbye') + remote_list = session.sftp.dir.glob(REMOTE, '*') + + if answer == 'All' + main_task(local_list, remote_list, session) else - remote_list.filter! { |f| f == answer.downcase } + local_list.filter! { |f| f == answer.downcase } + remote_list.filter! do |f| + f.name.force_encoding('UTF-8') == answer.downcase + end + + puts "Downloading a new set of #{answer.downcase}..." + main_task(local_list, remote_list, session) end - end + + # puts "Finished downloads, now replacing your local seed data..." # Rake::Task["comfy:cms_seeds:import"].invoke('protected-planet', 'protectedplanet') From 58dcc5fb9de836c8cdfc7e5d00e87224aea2fe15 Mon Sep 17 00:00:00 2001 From: Stanley Liu Date: Tue, 4 Aug 2020 16:37:20 +0100 Subject: [PATCH 25/49] placed the methods into a module --- lib/modules/sync_seeds.rb | 107 +++++++++++++++++++++++ lib/tasks/staging_seeds.rake | 164 +++++++---------------------------- 2 files changed, 139 insertions(+), 132 deletions(-) create mode 100644 lib/modules/sync_seeds.rb diff --git a/lib/modules/sync_seeds.rb b/lib/modules/sync_seeds.rb new file mode 100644 index 000000000..05dc08d04 --- /dev/null +++ b/lib/modules/sync_seeds.rb @@ -0,0 +1,107 @@ +class SyncSeeds + require 'net/ssh' + require 'net/scp' + + def delete_files(files, location) + files.each do |file| + puts "Removing #{file} as it no longer exists remotely" + FileUtils.rm_rf(File.join(location, file)) + end + end + + def files_for_deletion(list_of_files_1, list_of_files_2, location) + files = list_of_files_1 - list_of_files_2 + + if files.empty? + puts "No files to delete" + else + delete_files(files, location) + end + end + + def download_files(source, dest, session) + puts "Downloading #{source} as it's newer or not found locally" + return session.scp.download!(source, dest) if File.file?(dest) + session.scp.download!(source, dest, recursive: true) + end + + def create_paths(relative_path) + { local_path: File.join(LOCAL, relative_path), remote_path: File.join(REMOTE, relative_path) } + end + + def check_if_newer(parent_folder, local_item, remote_item, remote_path, local_path, session, base = LOCAL) + if parent_folder.include?(local_item) + yield if block_given? + + is_newer = Time.at(remote_item.attributes.mtime) >= File.stat(local_path).mtime + # If there are any outdated files, will trigger download + if is_newer + if Dir.glob('*', base: LOCAL).include?(local_item) + download_files(remote_path, local_path, session) + # Will be hit if local_item is a file or folder inside a directory + elsif Dir.glob('**/*', base: base).include?(local_item) + download_files(remote_path, base, session) + downloaded = true + end + end + else + # Just download it if it doesn't exist at all + download_files(remote_path, LOCAL, session) + downloaded = true + end + + downloaded + end + + def compare_folders(wildcard, local, remote, session, base = LOCAL) + puts "Checking to see what files need to be deleted from #{base}" + + remote_list = session.sftp.dir.glob(remote, wildcard).map do |f| + f.name.force_encoding('UTF-8') + end + + local_list = Dir.glob(wildcard, base: local) + + files_for_deletion(local_list, remote_list, base) + end + + def check_inside_folder(folder, local_list, session) + paths = create_paths(folder) + + compare_folders('**/*', paths[:local_path], paths[:remote_path], session, paths[:local_path]) + + local_folder_content = Dir.glob('**/*', base: paths[:local_path]) + remote_folder_content = session.sftp.dir.glob(paths[:remote_path], '**/*') + + # We don't want to download the whole folder again if it's already been re-downloaded once + + + remote_folder_content.each do |file| + # Go through the various files and folders and check to see if they exist locally + local_file = file.name + absolute_path = File.join(paths[:local_path], file.name) + + check = check_if_newer(local_folder_content, local_file, file, paths[:remote_path], absolute_path, session, paths[:local_path]) + + # Break out of loop if already downloaded + break if check == true + end + end + + def main_task(local_list, remote_list, session) + remote_list.each do |object| + # There are files with non-ASCII characters (i.e. accented) in the CMS files + name = object.name.force_encoding('UTF-8') + paths = create_paths(name) + + + check_if_newer(local_list, name, object, paths[:remote_path], paths[:local_path], session) do + check_inside_folder(name, local_list, session) if object.attributes.directory? + end + end + end + + def start_session(server, username) + Net::SSH.start(server, username) { yield } + end +end \ No newline at end of file diff --git a/lib/tasks/staging_seeds.rake b/lib/tasks/staging_seeds.rake index 134f5325f..db615a406 100644 --- a/lib/tasks/staging_seeds.rake +++ b/lib/tasks/staging_seeds.rake @@ -11,152 +11,52 @@ namespace :comfy do PP_USER = 'wcmc'.freeze desc "Import CMS Seed data from staging" - - def delete_files(files, location) - files.each do |file| - puts "Removing #{file} as it no longer exists remotely" - FileUtils.rm_rf(File.join(location, file)) - end - end - - def files_for_deletion(list_of_files_1, list_of_files_2, location) - files = list_of_files_1 - list_of_files_2 - - if files.empty? - puts "No files to delete" - else - delete_files(files, location) - end - end - - def download_files(source, dest, session) - puts "Downloading #{source} as it's newer or not found locally" - return session.scp.download!(source, dest) if File.file?(dest) - session.scp.download!(source, dest, recursive: true) - end + task :staging_import => :environment do |_t| - def create_paths(relative_path) - { local_path: File.join(LOCAL, relative_path), remote_path: File.join(REMOTE, relative_path) } - end - - def check_if_newer(parent_folder, local_item, remote_item, remote_path, local_path, session, base = LOCAL) - if parent_folder.include?(local_item) - yield if block_given? - - is_newer = Time.at(remote_item.attributes.mtime) >= File.stat(local_path).mtime - # If there are any outdated files, will trigger download - if is_newer - if Dir.glob('*', base: LOCAL).include?(local_item) - download_files(remote_path, local_path, session) - # Will be hit if local_item is a file or folder inside a directory - elsif Dir.glob('**/*', base: base).include?(local_item) - download_files(remote_path, base, session) - downloaded = true - end - end - else - # Just download it if it doesn't exist at all - download_files(remote_path, LOCAL, session) - downloaded = true - end - - downloaded - end - - def compare_folders(wildcard, local, remote, session, base = LOCAL) - puts "Checking to see what files need to be deleted from #{base}" - - remote_list = session.sftp.dir.glob(remote, wildcard).map do |f| - f.name.force_encoding('UTF-8') - end + puts "Importing CMS Seed data from Staging Folder to #{LOCAL} ..." - local_list = Dir.glob(wildcard, base: local) - - files_for_deletion(local_list, remote_list, base) + puts question = "What would you like to import? 'All/Files/Layouts/Pages' or 'Nothing' to quit" + valid_answers = ['All', 'Files', 'Layouts', 'Pages', 'Nothing'] + answer = STDIN.gets.chomp.downcase.capitalize + until valid_answers.include?(answer) + puts question + answer = STDIN.gets.chomp.downcase.capitalize end - def check_inside_folder(folder, local_list, session) - paths = create_paths(folder) - - compare_folders('**/*', paths[:local_path], paths[:remote_path], session, paths[:local_path]) + # Fast-tracked unhappy path + abort('Goodbye') if answer == 'Nothing' - local_folder_content = Dir.glob('**/*', base: paths[:local_path]) - remote_folder_content = session.sftp.dir.glob(paths[:remote_path], '**/*') - - # We don't want to download the whole folder again if it's already been re-downloaded once + SyncSeeds.start_session(PP_STAGING, PP_USER) + # SSH into staging server with Net::SSH + Net::SSH.start(PP_STAGING, PP_USER) do |session| + # First get rid of any local top-level (i.e. which exist in the main + # directory of REMOTE) folders/files that don't exist remotely + SyncSeeds.compare_folders('*', LOCAL, REMOTE, session) - remote_folder_content.each do |file| - # Go through the various files and folders and check to see if they exist locally - local_file = file.name - absolute_path = File.join(paths[:local_path], file.name) - - check = check_if_newer(local_folder_content, local_file, file, paths[:remote_path], absolute_path, session, paths[:local_path]) - - # Break out of loop if already downloaded - break if check == true - end - end + local_list = Dir.glob('*', base: LOCAL) + remote_list = session.sftp.dir.glob(REMOTE, '*') - def main_task(local_list, remote_list, session) - remote_list.each do |object| - # There are files with non-ASCII characters (i.e. accented) in the CMS files - name = object.name.force_encoding('UTF-8') - paths = create_paths(name) - - - check_if_newer(local_list, name, object, paths[:remote_path], paths[:local_path], session) do - check_inside_folder(name, local_list, session) if object.attributes.directory? + if answer == 'All' + SyncSeeds.main_task(local_list, remote_list, session) + else + local_list.filter! { |f| f == answer.downcase } + remote_list.filter! do |f| + f.name.force_encoding('UTF-8') == answer.downcase end - end - end - task :staging_import => :environment do |_t| - require 'net/ssh' - require 'net/scp' - - puts "Importing CMS Seed data from Staging Folder to #{LOCAL} ..." + puts "Downloading a new set of #{answer.downcase}..." - puts question = "What would you like to import? 'All/Files/Layouts/Pages' or 'Nothing' to quit" - valid_answers = ['All', 'Files', 'Layouts', 'Pages', 'Nothing'] - answer = STDIN.gets.chomp.downcase.capitalize - until valid_answers.include?(answer) - puts question - answer = STDIN.gets.chomp.downcase.capitalize + SyncSeeds.main_task(local_list, remote_list, session) end + + + # puts "Finished downloads, now replacing your local seed data..." - # Fast-tracked unhappy path - abort('Goodbye') if answer == 'Nothing' - - # SSH into staging server with Net::SSH - Net::SSH.start(PP_STAGING, PP_USER) do |session| - # First get rid of any local top-level (i.e. which exist in the main - # directory of REMOTE) folders/files that don't exist remotely - compare_folders('*', LOCAL, REMOTE, session) - - local_list = Dir.glob('*', base: LOCAL) - remote_list = session.sftp.dir.glob(REMOTE, '*') - - if answer == 'All' - main_task(local_list, remote_list, session) - else - local_list.filter! { |f| f == answer.downcase } - remote_list.filter! do |f| - f.name.force_encoding('UTF-8') == answer.downcase - end - - puts "Downloading a new set of #{answer.downcase}..." - - main_task(local_list, remote_list, session) - end - - - # puts "Finished downloads, now replacing your local seed data..." - - # Rake::Task["comfy:cms_seeds:import"].invoke('protected-planet', 'protectedplanet') + # Rake::Task["comfy:cms_seeds:import"].invoke('protected-planet', 'protectedplanet') - # Todo: get this working with AWS bucket - end + # Todo: get this working with AWS bucket + end end end \ No newline at end of file From 027e372c96722f43bd68f7fc256fba01d9e761c2 Mon Sep 17 00:00:00 2001 From: Stanley Liu Date: Tue, 4 Aug 2020 16:44:40 +0100 Subject: [PATCH 26/49] adding temporary gdal version 3.0 --- Gemfile | 4 +++- Gemfile.lock | 4 ++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/Gemfile b/Gemfile index fd87d91b4..c4b32dc9c 100644 --- a/Gemfile +++ b/Gemfile @@ -30,7 +30,9 @@ gem 'vuejs-rails', '~> 2.3.2' gem 'sprockets-vue', '~> 0.1.0' gem 'rails-controller-testing' -gem 'gdal', '~> 2.0' +# gem 'gdal', '~> 2.0' +# TODO Stanley - Need to change this version of gdal back to 2.0 (and remove from gitignore) when I reinstall GDAL to 2.4.4 +gem 'gdal', '~> 3.0' # group :production, :staging do # gem 'unicorn' diff --git a/Gemfile.lock b/Gemfile.lock index e79ca5879..6bbf04532 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1065,7 +1065,7 @@ GEM multipart-post (>= 1.2, < 3) fastercsv (1.5.5) ffi (1.12.1) - gdal (2.0.0) + gdal (3.0.0) globalid (0.4.2) activesupport (>= 4.2.0) haml (5.1.2) @@ -1335,7 +1335,7 @@ DEPENDENCIES elasticsearch (~> 7.2.0) exception_notification (~> 4.3.0) factory_girl_rails (~> 4.4.1) - gdal (~> 2.0) + gdal (~> 3.0) httmultiparty (~> 0.3.14) httparty (~> 0.15.1) jquery-rails (~> 4.3.3) From a13845adfbfcb4c9c560f370e02c7f16dfd40d3d Mon Sep 17 00:00:00 2001 From: Stanley Liu Date: Wed, 5 Aug 2020 10:28:49 +0100 Subject: [PATCH 27/49] initialize syncseeds method written --- lib/modules/sync_seeds.rb | 57 ++++++++++++++++++++++++------------ lib/tasks/staging_seeds.rake | 17 ++++++----- 2 files changed, 47 insertions(+), 27 deletions(-) diff --git a/lib/modules/sync_seeds.rb b/lib/modules/sync_seeds.rb index 05dc08d04..c92af8796 100644 --- a/lib/modules/sync_seeds.rb +++ b/lib/modules/sync_seeds.rb @@ -2,6 +2,26 @@ class SyncSeeds require 'net/ssh' require 'net/scp' + def initialize(server, username) + @username = username + @server = server + end + + def start_session + Net::SSH.start(@server, @username) do |session| + @session = session + yield + end + end + + def list_local_files(location) + Dir.glob('*', base: location) + end + + def list_remote_files(location) + @session.sftp.dir.glob(location, '*') + end + def delete_files(files, location) files.each do |file| puts "Removing #{file} as it no longer exists remotely" @@ -19,17 +39,17 @@ def files_for_deletion(list_of_files_1, list_of_files_2, location) end end - def download_files(source, dest, session) + def download_files(source, dest) puts "Downloading #{source} as it's newer or not found locally" - return session.scp.download!(source, dest) if File.file?(dest) - session.scp.download!(source, dest, recursive: true) + return @session.scp.download!(source, dest) if File.file?(dest) + @session.scp.download!(source, dest, recursive: true) end def create_paths(relative_path) { local_path: File.join(LOCAL, relative_path), remote_path: File.join(REMOTE, relative_path) } end - def check_if_newer(parent_folder, local_item, remote_item, remote_path, local_path, session, base = LOCAL) + def check_if_newer(parent_folder, local_item, remote_item, remote_path, local_path, base = LOCAL) if parent_folder.include?(local_item) yield if block_given? @@ -37,26 +57,26 @@ def check_if_newer(parent_folder, local_item, remote_item, remote_path, local_pa # If there are any outdated files, will trigger download if is_newer if Dir.glob('*', base: LOCAL).include?(local_item) - download_files(remote_path, local_path, session) + download_files(remote_path, local_path) # Will be hit if local_item is a file or folder inside a directory elsif Dir.glob('**/*', base: base).include?(local_item) - download_files(remote_path, base, session) + download_files(remote_path, base) downloaded = true end end else # Just download it if it doesn't exist at all - download_files(remote_path, LOCAL, session) + download_files(remote_path, LOCAL) downloaded = true end downloaded end - def compare_folders(wildcard, local, remote, session, base = LOCAL) + def compare_folders(wildcard, local, remote, base = LOCAL) puts "Checking to see what files need to be deleted from #{base}" - remote_list = session.sftp.dir.glob(remote, wildcard).map do |f| + remote_list = @session.sftp.dir.glob(remote, wildcard).map do |f| f.name.force_encoding('UTF-8') end @@ -65,13 +85,13 @@ def compare_folders(wildcard, local, remote, session, base = LOCAL) files_for_deletion(local_list, remote_list, base) end - def check_inside_folder(folder, local_list, session) + def check_inside_folder(folder, local_list) paths = create_paths(folder) - compare_folders('**/*', paths[:local_path], paths[:remote_path], session, paths[:local_path]) + compare_folders('**/*', paths[:local_path], paths[:remote_path], paths[:local_path]) local_folder_content = Dir.glob('**/*', base: paths[:local_path]) - remote_folder_content = session.sftp.dir.glob(paths[:remote_path], '**/*') + remote_folder_content = @session.sftp.dir.glob(paths[:remote_path], '**/*') # We don't want to download the whole folder again if it's already been re-downloaded once @@ -81,27 +101,26 @@ def check_inside_folder(folder, local_list, session) local_file = file.name absolute_path = File.join(paths[:local_path], file.name) - check = check_if_newer(local_folder_content, local_file, file, paths[:remote_path], absolute_path, session, paths[:local_path]) + check = check_if_newer(local_folder_content, local_file, file, paths[:remote_path], absolute_path, paths[:local_path]) # Break out of loop if already downloaded break if check == true end end - def main_task(local_list, remote_list, session) + def main_task(local_list, remote_list) remote_list.each do |object| # There are files with non-ASCII characters (i.e. accented) in the CMS files name = object.name.force_encoding('UTF-8') paths = create_paths(name) - check_if_newer(local_list, name, object, paths[:remote_path], paths[:local_path], session) do - check_inside_folder(name, local_list, session) if object.attributes.directory? + check_if_newer(local_list, name, object, paths[:remote_path], paths[:local_path]) do + check_inside_folder(name, local_list) if object.attributes.directory? end end end - def start_session(server, username) - Net::SSH.start(server, username) { yield } - end + + end \ No newline at end of file diff --git a/lib/tasks/staging_seeds.rake b/lib/tasks/staging_seeds.rake index db615a406..932a46953 100644 --- a/lib/tasks/staging_seeds.rake +++ b/lib/tasks/staging_seeds.rake @@ -26,19 +26,20 @@ namespace :comfy do # Fast-tracked unhappy path abort('Goodbye') if answer == 'Nothing' - SyncSeeds.start_session(PP_STAGING, PP_USER) + new_session = SyncSeeds.new(PP_STAGING, PP_USER) # SSH into staging server with Net::SSH - Net::SSH.start(PP_STAGING, PP_USER) do |session| + new_session.start_session do |session| # First get rid of any local top-level (i.e. which exist in the main # directory of REMOTE) folders/files that don't exist remotely - SyncSeeds.compare_folders('*', LOCAL, REMOTE, session) + new_session.compare_folders('*', LOCAL, REMOTE) - local_list = Dir.glob('*', base: LOCAL) - remote_list = session.sftp.dir.glob(REMOTE, '*') + + local_list = new_session.list_local_files(LOCAL) + remote_list = new_session.list_remote_files(REMOTE) if answer == 'All' - SyncSeeds.main_task(local_list, remote_list, session) + new_session.main_task(local_list, remote_list) else local_list.filter! { |f| f == answer.downcase } remote_list.filter! do |f| @@ -47,7 +48,7 @@ namespace :comfy do puts "Downloading a new set of #{answer.downcase}..." - SyncSeeds.main_task(local_list, remote_list, session) + new_session.main_task(local_list, remote_list) end @@ -56,7 +57,7 @@ namespace :comfy do # Rake::Task["comfy:cms_seeds:import"].invoke('protected-planet', 'protectedplanet') # Todo: get this working with AWS bucket - end + end end end \ No newline at end of file From 555fc770029dc3871d09958a28fd08fb76d9d854 Mon Sep 17 00:00:00 2001 From: Stanley Liu Date: Wed, 5 Aug 2020 15:51:26 +0100 Subject: [PATCH 28/49] reverted back to gdal v2 and added gdal v3 gemfile to gitignore --- .gitignore | 3 +-- Gemfile | 4 +--- Gemfile.lock | 4 ++-- 3 files changed, 4 insertions(+), 7 deletions(-) diff --git a/.gitignore b/.gitignore index b5f4dced2..dd0542a51 100644 --- a/.gitignore +++ b/.gitignore @@ -37,5 +37,4 @@ tmp/cache/webpacker .vscode/ -# Special gemfile with GDAL v3 -Gemfile_GDALV3 \ No newline at end of file +Gemfile_GDALV3 diff --git a/Gemfile b/Gemfile index fcb113ab7..93197d4ff 100644 --- a/Gemfile +++ b/Gemfile @@ -35,9 +35,7 @@ gem 'vuejs-rails', '~> 2.3.2' gem 'sprockets-vue', '~> 0.1.0' gem 'rails-controller-testing' -# gem 'gdal', '~> 2.0' -# TODO Stanley - Need to change this version of gdal back to 2.0 (and remove from gitignore) when I reinstall GDAL to 2.4.4 -gem 'gdal', '~> 3.0' +gem 'gdal', '~> 2.0' # group :production, :staging do # gem 'unicorn' diff --git a/Gemfile.lock b/Gemfile.lock index d933b0b20..5bf8b49d1 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1066,7 +1066,7 @@ GEM multipart-post (>= 1.2, < 3) fastercsv (1.5.5) ffi (1.12.1) - gdal (3.0.0) + gdal (2.0.0) globalid (0.4.2) activesupport (>= 4.2.0) haml (5.1.2) @@ -1337,7 +1337,7 @@ DEPENDENCIES elasticsearch (~> 7.2.0) exception_notification (~> 4.3.0) factory_girl_rails (~> 4.4.1) - gdal (~> 3.0) + gdal (~> 2.0) httmultiparty (~> 0.3.14) httparty (~> 0.15.1) jquery-rails (~> 4.3.3) From 77ed987a9f4c578f228cac9dca538ceddb64a519 Mon Sep 17 00:00:00 2001 From: Stanley Liu Date: Wed, 5 Aug 2020 16:28:02 +0100 Subject: [PATCH 29/49] removed separate gemfile from .gitignore as it's a faff --- .gitignore | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index dd0542a51..2b4fc0a9d 100644 --- a/.gitignore +++ b/.gitignore @@ -35,6 +35,4 @@ tmp/cache/webpacker .byebug_history -.vscode/ - -Gemfile_GDALV3 +.vscode/ \ No newline at end of file From 4f72e3e03bd29f0e01745a13cc38c5acd4f55e77 Mon Sep 17 00:00:00 2001 From: Stanley Liu Date: Fri, 7 Aug 2020 09:16:21 +0100 Subject: [PATCH 30/49] task now imports the selection automatically after checking --- lib/modules/sync_seeds.rb | 17 +++++++------- lib/tasks/staging_seeds.rake | 45 +++++++++++++++++++++--------------- 2 files changed, 36 insertions(+), 26 deletions(-) diff --git a/lib/modules/sync_seeds.rb b/lib/modules/sync_seeds.rb index c92af8796..eb3d32a3b 100644 --- a/lib/modules/sync_seeds.rb +++ b/lib/modules/sync_seeds.rb @@ -52,20 +52,22 @@ def create_paths(relative_path) def check_if_newer(parent_folder, local_item, remote_item, remote_path, local_path, base = LOCAL) if parent_folder.include?(local_item) yield if block_given? - + is_newer = Time.at(remote_item.attributes.mtime) >= File.stat(local_path).mtime # If there are any outdated files, will trigger download if is_newer if Dir.glob('*', base: LOCAL).include?(local_item) download_files(remote_path, local_path) - # Will be hit if local_item is a file or folder inside a directory + # Will be hit if local_item is a file or folder inside a directory elsif Dir.glob('**/*', base: base).include?(local_item) - download_files(remote_path, base) + byebug + download_files(remote_path, local_path) downloaded = true end end else # Just download it if it doesn't exist at all + byebug download_files(remote_path, LOCAL) downloaded = true end @@ -98,11 +100,10 @@ def check_inside_folder(folder, local_list) remote_folder_content.each do |file| # Go through the various files and folders and check to see if they exist locally - local_file = file.name - absolute_path = File.join(paths[:local_path], file.name) - - check = check_if_newer(local_folder_content, local_file, file, paths[:remote_path], absolute_path, paths[:local_path]) - + local_file = file.name.force_encoding('UTF-8') + + check = check_if_newer(local_folder_content, local_file, file, paths[:remote_path], paths[:local_path], paths[:local_path]) + # Break out of loop if already downloaded break if check == true end diff --git a/lib/tasks/staging_seeds.rake b/lib/tasks/staging_seeds.rake index 932a46953..83683fba7 100644 --- a/lib/tasks/staging_seeds.rake +++ b/lib/tasks/staging_seeds.rake @@ -7,24 +7,27 @@ namespace :comfy do LOCAL = (ComfortableMexicanSofa.config.seeds_path + '/protected-planet').freeze REMOTE = 'ProtectedPlanet/current/db/cms_seeds/protected-planet'.freeze PP_STAGING = 'new-web.pp-staging.linode.protectedplanet.net'.freeze - PP_PRODUCTION = 'new-web.pp-production.linode.protectedplanet.net'.freeze PP_USER = 'wcmc'.freeze - desc "Import CMS Seed data from staging" - task :staging_import => :environment do |_t| + desc "Import CMS Seed data from staging. Can be run with user inputs or arguments" + task :staging_import, [:seed_type] => [:environment] do |_t, args| + answer = args[:seed_type] - puts "Importing CMS Seed data from Staging Folder to #{LOCAL} ..." - - puts question = "What would you like to import? 'All/Files/Layouts/Pages' or 'Nothing' to quit" - valid_answers = ['All', 'Files', 'Layouts', 'Pages', 'Nothing'] - answer = STDIN.gets.chomp.downcase.capitalize - until valid_answers.include?(answer) - puts question + if answer.nil? + puts question = "What would you like to import? 'All/Files/Layouts/Pages' or 'Nothing' to quit" + valid_answers = ['All', 'Files', 'Layouts', 'Pages', 'Nothing'] answer = STDIN.gets.chomp.downcase.capitalize + until valid_answers.include?(answer) + puts question + end + + # Fast-tracked unhappy path + abort('Goodbye') if answer == 'Nothing' + else + answer = answer.downcase.capitalize! end - - # Fast-tracked unhappy path - abort('Goodbye') if answer == 'Nothing' + + puts "Importing CMS Seed data from Staging Folder to #{LOCAL} ..." new_session = SyncSeeds.new(PP_STAGING, PP_USER) @@ -34,7 +37,6 @@ namespace :comfy do # directory of REMOTE) folders/files that don't exist remotely new_session.compare_folders('*', LOCAL, REMOTE) - local_list = new_session.list_local_files(LOCAL) remote_list = new_session.list_remote_files(REMOTE) @@ -50,14 +52,21 @@ namespace :comfy do new_session.main_task(local_list, remote_list) end + end - - # puts "Finished downloads, now replacing your local seed data..." + puts "Finished downloads, now replacing your local seed data with your selection..." - # Rake::Task["comfy:cms_seeds:import"].invoke('protected-planet', 'protectedplanet') + logger = ComfortableMexicanSofa.logger + ComfortableMexicanSofa.logger = Logger.new(STDOUT) - # Todo: get this working with AWS bucket + if answer == 'All' + Rake::Task["comfy:cms_seeds:import"].invoke('protected-planet', 'protectedplanet') + else + module_name = "ComfortableMexicanSofa::Seeds::#{answer.singularize}::Importer".constantize + module_name.new('protected-planet', 'protectedplanet').import! end + + ComfortableMexicanSofa.logger = logger end end \ No newline at end of file From f84d0d71029d262c0ca3bf9e0c9e76e6b3c7b627 Mon Sep 17 00:00:00 2001 From: Stanley Liu Date: Fri, 7 Aug 2020 09:19:26 +0100 Subject: [PATCH 31/49] moved comfy import task into its own method in the module --- lib/modules/sync_seeds.rb | 12 ++++++++++++ lib/tasks/staging_seeds.rake | 16 +++------------- 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/lib/modules/sync_seeds.rb b/lib/modules/sync_seeds.rb index eb3d32a3b..08fb7314a 100644 --- a/lib/modules/sync_seeds.rb +++ b/lib/modules/sync_seeds.rb @@ -122,6 +122,18 @@ def main_task(local_list, remote_list) end end + def commence_import(answer) + logger = ComfortableMexicanSofa.logger + ComfortableMexicanSofa.logger = Logger.new(STDOUT) + if answer == 'All' + Rake::Task["comfy:cms_seeds:import"].invoke('protected-planet', 'protectedplanet') + else + module_name = "ComfortableMexicanSofa::Seeds::#{answer.singularize}::Importer".constantize + module_name.new('protected-planet', 'protectedplanet').import! + end + + ComfortableMexicanSofa.logger = logger + end end \ No newline at end of file diff --git a/lib/tasks/staging_seeds.rake b/lib/tasks/staging_seeds.rake index 83683fba7..8a8ac5715 100644 --- a/lib/tasks/staging_seeds.rake +++ b/lib/tasks/staging_seeds.rake @@ -41,7 +41,7 @@ namespace :comfy do remote_list = new_session.list_remote_files(REMOTE) if answer == 'All' - new_session.main_task(local_list, remote_list) + new_session.main_task(local_list, remote_list) else local_list.filter! { |f| f == answer.downcase } remote_list.filter! do |f| @@ -52,21 +52,11 @@ namespace :comfy do new_session.main_task(local_list, remote_list) end - end - - puts "Finished downloads, now replacing your local seed data with your selection..." - logger = ComfortableMexicanSofa.logger - ComfortableMexicanSofa.logger = Logger.new(STDOUT) + puts "Finished downloads, now replacing your local seed data with your selection..." - if answer == 'All' - Rake::Task["comfy:cms_seeds:import"].invoke('protected-planet', 'protectedplanet') - else - module_name = "ComfortableMexicanSofa::Seeds::#{answer.singularize}::Importer".constantize - module_name.new('protected-planet', 'protectedplanet').import! + new_session.commence_import(answer) end - - ComfortableMexicanSofa.logger = logger end end \ No newline at end of file From 9d188ce6577eee9ff795b88fa1e14dc7e5b46ab0 Mon Sep 17 00:00:00 2001 From: Stanley Liu Date: Fri, 7 Aug 2020 09:20:28 +0100 Subject: [PATCH 32/49] made description more informative --- lib/tasks/staging_seeds.rake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/tasks/staging_seeds.rake b/lib/tasks/staging_seeds.rake index 8a8ac5715..202b2c6a6 100644 --- a/lib/tasks/staging_seeds.rake +++ b/lib/tasks/staging_seeds.rake @@ -9,7 +9,7 @@ namespace :comfy do PP_STAGING = 'new-web.pp-staging.linode.protectedplanet.net'.freeze PP_USER = 'wcmc'.freeze - desc "Import CMS Seed data from staging. Can be run with user inputs or arguments" + desc "Import CMS Seed data from staging. Can be run with argument or can accept user input if no argument is supplied" task :staging_import, [:seed_type] => [:environment] do |_t, args| answer = args[:seed_type] From df148d26686f53d77869b55aaa6a9a82c395e21a Mon Sep 17 00:00:00 2001 From: Stanley Liu Date: Fri, 7 Aug 2020 09:22:51 +0100 Subject: [PATCH 33/49] no byebugs --- lib/modules/sync_seeds.rb | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/modules/sync_seeds.rb b/lib/modules/sync_seeds.rb index 08fb7314a..13c661aab 100644 --- a/lib/modules/sync_seeds.rb +++ b/lib/modules/sync_seeds.rb @@ -60,14 +60,12 @@ def check_if_newer(parent_folder, local_item, remote_item, remote_path, local_pa download_files(remote_path, local_path) # Will be hit if local_item is a file or folder inside a directory elsif Dir.glob('**/*', base: base).include?(local_item) - byebug download_files(remote_path, local_path) downloaded = true end end else # Just download it if it doesn't exist at all - byebug download_files(remote_path, LOCAL) downloaded = true end From 41e75ab73b0b2b08bddb1f96c11607acdb97af6c Mon Sep 17 00:00:00 2001 From: Stanley Liu Date: Fri, 7 Aug 2020 09:38:38 +0100 Subject: [PATCH 34/49] added a comment directing devs on which file to look in for the methods --- lib/tasks/staging_seeds.rake | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/lib/tasks/staging_seeds.rake b/lib/tasks/staging_seeds.rake index 202b2c6a6..f715a2ae6 100644 --- a/lib/tasks/staging_seeds.rake +++ b/lib/tasks/staging_seeds.rake @@ -1,15 +1,13 @@ # frozen_string_literal: true namespace :comfy do - - # Locally stored seeds - assumes you already have the local folder - - # it won't create it + # The methods that this rake task calls are in lib/modules/sync_seeds.rb LOCAL = (ComfortableMexicanSofa.config.seeds_path + '/protected-planet').freeze REMOTE = 'ProtectedPlanet/current/db/cms_seeds/protected-planet'.freeze PP_STAGING = 'new-web.pp-staging.linode.protectedplanet.net'.freeze PP_USER = 'wcmc'.freeze - desc "Import CMS Seed data from staging. Can be run with argument or can accept user input if no argument is supplied" + desc "Import CMS Seed data from staging. Can be run with argument (files/layouts/pages/all) or can accept user input if no argument is supplied" task :staging_import, [:seed_type] => [:environment] do |_t, args| answer = args[:seed_type] From 52ac70b9601bfca91c26e0386c55ee9bd854a727 Mon Sep 17 00:00:00 2001 From: Stanley Liu Date: Fri, 7 Aug 2020 14:23:55 +0100 Subject: [PATCH 35/49] destination may be called as an argument when invoking the task --- lib/tasks/staging_seeds.rake | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/lib/tasks/staging_seeds.rake b/lib/tasks/staging_seeds.rake index f715a2ae6..8acf6db7d 100644 --- a/lib/tasks/staging_seeds.rake +++ b/lib/tasks/staging_seeds.rake @@ -2,16 +2,16 @@ namespace :comfy do # The methods that this rake task calls are in lib/modules/sync_seeds.rb - LOCAL = (ComfortableMexicanSofa.config.seeds_path + '/protected-planet').freeze - REMOTE = 'ProtectedPlanet/current/db/cms_seeds/protected-planet'.freeze + FROM = 'ProtectedPlanet/current/db/cms_seeds/protected-planet'.freeze PP_STAGING = 'new-web.pp-staging.linode.protectedplanet.net'.freeze PP_USER = 'wcmc'.freeze - desc "Import CMS Seed data from staging. Can be run with argument (files/layouts/pages/all) or can accept user input if no argument is supplied" - task :staging_import, [:seed_type] => [:environment] do |_t, args| - answer = args[:seed_type] + desc "Import CMS Seed data from staging. Can be run with arguments (, files/pages/layouts/all) or can accept user input if no argument is supplied" + task :staging_import, %i[to folder] => [:environment] do |_t, args| + to = File.join(ComfortableMexicanSofa.config.seeds_path, args[:to]) + answer = args[:folder] - if answer.nil? + if args.nil? puts question = "What would you like to import? 'All/Files/Layouts/Pages' or 'Nothing' to quit" valid_answers = ['All', 'Files', 'Layouts', 'Pages', 'Nothing'] answer = STDIN.gets.chomp.downcase.capitalize @@ -25,7 +25,7 @@ namespace :comfy do answer = answer.downcase.capitalize! end - puts "Importing CMS Seed data from Staging Folder to #{LOCAL} ..." + puts "Importing CMS Seed data from Staging Folder to #{to} ..." new_session = SyncSeeds.new(PP_STAGING, PP_USER) @@ -33,10 +33,10 @@ namespace :comfy do new_session.start_session do |session| # First get rid of any local top-level (i.e. which exist in the main # directory of REMOTE) folders/files that don't exist remotely - new_session.compare_folders('*', LOCAL, REMOTE) + new_session.compare_folders('*', to, FROM) - local_list = new_session.list_local_files(LOCAL) - remote_list = new_session.list_remote_files(REMOTE) + local_list = new_session.list_local_files(to) + remote_list = new_session.list_remote_files(FROM) if answer == 'All' new_session.main_task(local_list, remote_list) From c874f1cc71c9781897d62b90db35487dd34c63f8 Mon Sep 17 00:00:00 2001 From: Stanley Liu Date: Mon, 10 Aug 2020 16:13:52 +0100 Subject: [PATCH 36/49] have to rework compare_folders method as it no longer relies on LOCAL --- lib/modules/sync_seeds.rb | 2 +- lib/tasks/staging_seeds.rake | 11 +++++------ 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/lib/modules/sync_seeds.rb b/lib/modules/sync_seeds.rb index 13c661aab..5d4b70893 100644 --- a/lib/modules/sync_seeds.rb +++ b/lib/modules/sync_seeds.rb @@ -73,7 +73,7 @@ def check_if_newer(parent_folder, local_item, remote_item, remote_path, local_pa downloaded end - def compare_folders(wildcard, local, remote, base = LOCAL) + def compare_folders(wildcard, local, remote, base) puts "Checking to see what files need to be deleted from #{base}" remote_list = @session.sftp.dir.glob(remote, wildcard).map do |f| diff --git a/lib/tasks/staging_seeds.rake b/lib/tasks/staging_seeds.rake index 8acf6db7d..fce3dc1fd 100644 --- a/lib/tasks/staging_seeds.rake +++ b/lib/tasks/staging_seeds.rake @@ -8,10 +8,8 @@ namespace :comfy do desc "Import CMS Seed data from staging. Can be run with arguments (, files/pages/layouts/all) or can accept user input if no argument is supplied" task :staging_import, %i[to folder] => [:environment] do |_t, args| - to = File.join(ComfortableMexicanSofa.config.seeds_path, args[:to]) - answer = args[:folder] - - if args.nil? + + if args.length.nil? puts question = "What would you like to import? 'All/Files/Layouts/Pages' or 'Nothing' to quit" valid_answers = ['All', 'Files', 'Layouts', 'Pages', 'Nothing'] answer = STDIN.gets.chomp.downcase.capitalize @@ -22,7 +20,8 @@ namespace :comfy do # Fast-tracked unhappy path abort('Goodbye') if answer == 'Nothing' else - answer = answer.downcase.capitalize! + to = File.join(ComfortableMexicanSofa.config.seeds_path, args[:to]) + answer = args[:folder].downcase.capitalize! end puts "Importing CMS Seed data from Staging Folder to #{to} ..." @@ -33,7 +32,7 @@ namespace :comfy do new_session.start_session do |session| # First get rid of any local top-level (i.e. which exist in the main # directory of REMOTE) folders/files that don't exist remotely - new_session.compare_folders('*', to, FROM) + new_session.compare_folders('*', to, FROM, to) local_list = new_session.list_local_files(to) remote_list = new_session.list_remote_files(FROM) From 0f56eaca3ca210c281f67c1797aafdf372019da2 Mon Sep 17 00:00:00 2001 From: Stanley Liu Date: Tue, 11 Aug 2020 12:34:44 +0100 Subject: [PATCH 37/49] moved router into a separate method --- lib/modules/sync_seeds.rb | 9 +++++---- lib/tasks/staging_seeds.rake | 37 ++++++++++++++++++------------------ 2 files changed, 24 insertions(+), 22 deletions(-) diff --git a/lib/modules/sync_seeds.rb b/lib/modules/sync_seeds.rb index 5d4b70893..cef53b0a1 100644 --- a/lib/modules/sync_seeds.rb +++ b/lib/modules/sync_seeds.rb @@ -73,8 +73,8 @@ def check_if_newer(parent_folder, local_item, remote_item, remote_path, local_pa downloaded end - def compare_folders(wildcard, local, remote, base) - puts "Checking to see what files need to be deleted from #{base}" + def compare_folders(wildcard, local, remote) + puts "Checking to see what files need to be deleted from #{local}" remote_list = @session.sftp.dir.glob(remote, wildcard).map do |f| f.name.force_encoding('UTF-8') @@ -82,7 +82,7 @@ def compare_folders(wildcard, local, remote, base) local_list = Dir.glob(wildcard, base: local) - files_for_deletion(local_list, remote_list, base) + files_for_deletion(local_list, remote_list, local) end def check_inside_folder(folder, local_list) @@ -120,7 +120,8 @@ def main_task(local_list, remote_list) end end - def commence_import(answer) + # Piggybacks on existing Comfy modules + def commence_comfy_import(answer) logger = ComfortableMexicanSofa.logger ComfortableMexicanSofa.logger = Logger.new(STDOUT) diff --git a/lib/tasks/staging_seeds.rake b/lib/tasks/staging_seeds.rake index fce3dc1fd..de1312203 100644 --- a/lib/tasks/staging_seeds.rake +++ b/lib/tasks/staging_seeds.rake @@ -6,24 +6,25 @@ namespace :comfy do PP_STAGING = 'new-web.pp-staging.linode.protectedplanet.net'.freeze PP_USER = 'wcmc'.freeze - desc "Import CMS Seed data from staging. Can be run with arguments (, files/pages/layouts/all) or can accept user input if no argument is supplied" - task :staging_import, %i[to folder] => [:environment] do |_t, args| - - if args.length.nil? - puts question = "What would you like to import? 'All/Files/Layouts/Pages' or 'Nothing' to quit" - valid_answers = ['All', 'Files', 'Layouts', 'Pages', 'Nothing'] - answer = STDIN.gets.chomp.downcase.capitalize - until valid_answers.include?(answer) - puts question - end - - # Fast-tracked unhappy path - abort('Goodbye') if answer == 'Nothing' - else - to = File.join(ComfortableMexicanSofa.config.seeds_path, args[:to]) - answer = args[:folder].downcase.capitalize! + def user_input + puts question = "What would you like to import? 'All/Files/Layouts/Pages' or 'Nothing' to quit" + valid_answers = ['All', 'Files', 'Layouts', 'Pages', 'Nothing'] + answer = STDIN.gets.chomp.downcase.capitalize + until valid_answers.include?(answer) + puts question end + # Fast-tracked unhappy path + abort('Goodbye') if answer == 'Nothing' + + { answer: answer, destination: File.join(ComfortableMexicanSofa.config.seeds_path, 'protected-planet') } + end + + desc "Import CMS Seed data from staging. Can be run with arguments [, files/pages/layouts/all] or can accept user input if no argument is supplied" + task :staging_import, %i[to folder] => [:environment] do |_t, args| + to = args[:to].nil? ? user_input[:destination] : File.join(ComfortableMexicanSofa.config.seeds_path, args[:to]) + answer = args[:answer].nil? ? user_input[:answer] : args[:folder].downcase.capitalize! + puts "Importing CMS Seed data from Staging Folder to #{to} ..." new_session = SyncSeeds.new(PP_STAGING, PP_USER) @@ -32,7 +33,7 @@ namespace :comfy do new_session.start_session do |session| # First get rid of any local top-level (i.e. which exist in the main # directory of REMOTE) folders/files that don't exist remotely - new_session.compare_folders('*', to, FROM, to) + new_session.compare_folders('*', to, FROM) local_list = new_session.list_local_files(to) remote_list = new_session.list_remote_files(FROM) @@ -52,7 +53,7 @@ namespace :comfy do puts "Finished downloads, now replacing your local seed data with your selection..." - new_session.commence_import(answer) + # new_session.commence_comfy_import(answer) end end From c2a2464efd5ca58c09ba456a804e3fc726e49df4 Mon Sep 17 00:00:00 2001 From: Stanley Liu Date: Tue, 11 Aug 2020 15:52:06 +0100 Subject: [PATCH 38/49] fixed bug where router would prompt twice --- lib/modules/sync_seeds.rb | 17 +++++++++------- lib/tasks/staging_seeds.rake | 38 +++++++++++++++++++++++------------- 2 files changed, 34 insertions(+), 21 deletions(-) diff --git a/lib/modules/sync_seeds.rb b/lib/modules/sync_seeds.rb index cef53b0a1..25c9bc207 100644 --- a/lib/modules/sync_seeds.rb +++ b/lib/modules/sync_seeds.rb @@ -46,17 +46,17 @@ def download_files(source, dest) end def create_paths(relative_path) - { local_path: File.join(LOCAL, relative_path), remote_path: File.join(REMOTE, relative_path) } + { local_path: File.join(@local_base, relative_path), remote_path: File.join(@remote_base, relative_path) } end - def check_if_newer(parent_folder, local_item, remote_item, remote_path, local_path, base = LOCAL) + def check_if_newer(parent_folder, local_item, remote_item, remote_path, local_path, base = @local_base) if parent_folder.include?(local_item) yield if block_given? is_newer = Time.at(remote_item.attributes.mtime) >= File.stat(local_path).mtime # If there are any outdated files, will trigger download if is_newer - if Dir.glob('*', base: LOCAL).include?(local_item) + if Dir.glob('*', base: @local_base).include?(local_item) download_files(remote_path, local_path) # Will be hit if local_item is a file or folder inside a directory elsif Dir.glob('**/*', base: base).include?(local_item) @@ -66,14 +66,14 @@ def check_if_newer(parent_folder, local_item, remote_item, remote_path, local_pa end else # Just download it if it doesn't exist at all - download_files(remote_path, LOCAL) + download_files(remote_path, @local_base) downloaded = true end downloaded end - def compare_folders(wildcard, local, remote) + def compare_folders(wildcard, local, remote, base = @local_base) puts "Checking to see what files need to be deleted from #{local}" remote_list = @session.sftp.dir.glob(remote, wildcard).map do |f| @@ -82,7 +82,7 @@ def compare_folders(wildcard, local, remote) local_list = Dir.glob(wildcard, base: local) - files_for_deletion(local_list, remote_list, local) + files_for_deletion(local_list, remote_list, base) end def check_inside_folder(folder, local_list) @@ -107,7 +107,10 @@ def check_inside_folder(folder, local_list) end end - def main_task(local_list, remote_list) + def main_task(local_list, remote_list, local_base, remote_base) + @local_base = local_base + @remote_base = remote_base + remote_list.each do |object| # There are files with non-ASCII characters (i.e. accented) in the CMS files name = object.name.force_encoding('UTF-8') diff --git a/lib/tasks/staging_seeds.rake b/lib/tasks/staging_seeds.rake index de1312203..8c7c269f2 100644 --- a/lib/tasks/staging_seeds.rake +++ b/lib/tasks/staging_seeds.rake @@ -7,24 +7,34 @@ namespace :comfy do PP_USER = 'wcmc'.freeze def user_input - puts question = "What would you like to import? 'All/Files/Layouts/Pages' or 'Nothing' to quit" - valid_answers = ['All', 'Files', 'Layouts', 'Pages', 'Nothing'] - answer = STDIN.gets.chomp.downcase.capitalize + valid_answers = %w(all files layouts pages nothing) + puts question = "What would you like to import? 'all/files/layouts/pages' or 'nothing' to quit" + answer = STDIN.gets.chomp.downcase + until valid_answers.include?(answer) puts question + answer = STDIN.gets.chomp.downcase end - # Fast-tracked unhappy path - abort('Goodbye') if answer == 'Nothing' + abort('Goodbye') if answer == 'nothing' { answer: answer, destination: File.join(ComfortableMexicanSofa.config.seeds_path, 'protected-planet') } end desc "Import CMS Seed data from staging. Can be run with arguments [, files/pages/layouts/all] or can accept user input if no argument is supplied" task :staging_import, %i[to folder] => [:environment] do |_t, args| - to = args[:to].nil? ? user_input[:destination] : File.join(ComfortableMexicanSofa.config.seeds_path, args[:to]) - answer = args[:answer].nil? ? user_input[:answer] : args[:folder].downcase.capitalize! - + to = nil + answer = nil + + if args.length.nil? + answers = user_input + to = answers[:destination] + answer = answers[:answer] + else + to = File.join(ComfortableMexicanSofa.config.seeds_path, args[:to]) + answer = args[:folder].downcase + end + puts "Importing CMS Seed data from Staging Folder to #{to} ..." new_session = SyncSeeds.new(PP_STAGING, PP_USER) @@ -38,17 +48,17 @@ namespace :comfy do local_list = new_session.list_local_files(to) remote_list = new_session.list_remote_files(FROM) - if answer == 'All' - new_session.main_task(local_list, remote_list) + if answer == 'all' + new_session.main_task(local_list, remote_list, to, FROM) else - local_list.filter! { |f| f == answer.downcase } + local_list.filter! { |f| f == answer } remote_list.filter! do |f| - f.name.force_encoding('UTF-8') == answer.downcase + f.name.force_encoding('UTF-8') == answer end - puts "Downloading a new set of #{answer.downcase}..." + puts "Downloading a new set of #{answer}..." - new_session.main_task(local_list, remote_list) + new_session.main_task(local_list, remote_list, to, FROM) end puts "Finished downloads, now replacing your local seed data with your selection..." From 5347ccec11955ecf8482a647de23dc9a1c4fc2b5 Mon Sep 17 00:00:00 2001 From: Stanley Liu Date: Tue, 11 Aug 2020 16:32:33 +0100 Subject: [PATCH 39/49] kwargs instead of positional args for some methods --- lib/modules/sync_seeds.rb | 33 +++++++++++++++++++++++---------- lib/tasks/staging_seeds.rake | 10 +++++----- 2 files changed, 28 insertions(+), 15 deletions(-) diff --git a/lib/modules/sync_seeds.rb b/lib/modules/sync_seeds.rb index 25c9bc207..ec7e87188 100644 --- a/lib/modules/sync_seeds.rb +++ b/lib/modules/sync_seeds.rb @@ -29,8 +29,8 @@ def delete_files(files, location) end end - def files_for_deletion(list_of_files_1, list_of_files_2, location) - files = list_of_files_1 - list_of_files_2 + def files_for_deletion(local_list, remote_list, location) + files = local_list - remote_list if files.empty? puts "No files to delete" @@ -49,7 +49,7 @@ def create_paths(relative_path) { local_path: File.join(@local_base, relative_path), remote_path: File.join(@remote_base, relative_path) } end - def check_if_newer(parent_folder, local_item, remote_item, remote_path, local_path, base = @local_base) + def check_if_newer(parent_folder:, local_item:, remote_item:, remote_path:, local_path:, base: @local_base) if parent_folder.include?(local_item) yield if block_given? @@ -73,7 +73,7 @@ def check_if_newer(parent_folder, local_item, remote_item, remote_path, local_pa downloaded end - def compare_folders(wildcard, local, remote, base = @local_base) + def compare_folders(wildcard:, local:, remote:, base: @local_base) puts "Checking to see what files need to be deleted from #{local}" remote_list = @session.sftp.dir.glob(remote, wildcard).map do |f| @@ -88,7 +88,7 @@ def compare_folders(wildcard, local, remote, base = @local_base) def check_inside_folder(folder, local_list) paths = create_paths(folder) - compare_folders('**/*', paths[:local_path], paths[:remote_path], paths[:local_path]) + compare_folders(wildcard: '**/*', local: paths[:local_path], remote: paths[:remote_path], base: paths[:local_path]) local_folder_content = Dir.glob('**/*', base: paths[:local_path]) remote_folder_content = @session.sftp.dir.glob(paths[:remote_path], '**/*') @@ -100,14 +100,21 @@ def check_inside_folder(folder, local_list) # Go through the various files and folders and check to see if they exist locally local_file = file.name.force_encoding('UTF-8') - check = check_if_newer(local_folder_content, local_file, file, paths[:remote_path], paths[:local_path], paths[:local_path]) + check = check_if_newer( + parent_folder: local_folder_content, + local_item: local_file, + remote_item: file, + remote_path: paths[:remote_path], + local_path: paths[:local_path], + base: paths[:local_path] + ) # Break out of loop if already downloaded break if check == true end end - def main_task(local_list, remote_list, local_base, remote_base) + def main_task(local_list:, remote_list:, local_base:, remote_base:) @local_base = local_base @remote_base = remote_base @@ -115,9 +122,15 @@ def main_task(local_list, remote_list, local_base, remote_base) # There are files with non-ASCII characters (i.e. accented) in the CMS files name = object.name.force_encoding('UTF-8') paths = create_paths(name) - - - check_if_newer(local_list, name, object, paths[:remote_path], paths[:local_path]) do + + check_if_newer( + parent_folder: local_list, + local_item: name, + remote_item: object, + remote_path: paths[:remote_path], + local_path: paths[:local_path] + ) + do check_inside_folder(name, local_list) if object.attributes.directory? end end diff --git a/lib/tasks/staging_seeds.rake b/lib/tasks/staging_seeds.rake index 8c7c269f2..291c4ce52 100644 --- a/lib/tasks/staging_seeds.rake +++ b/lib/tasks/staging_seeds.rake @@ -43,13 +43,13 @@ namespace :comfy do new_session.start_session do |session| # First get rid of any local top-level (i.e. which exist in the main # directory of REMOTE) folders/files that don't exist remotely - new_session.compare_folders('*', to, FROM) + new_session.compare_folders(wildcard: '*', local: to, remote: FROM) local_list = new_session.list_local_files(to) remote_list = new_session.list_remote_files(FROM) if answer == 'all' - new_session.main_task(local_list, remote_list, to, FROM) + puts "Downloading all folders..." else local_list.filter! { |f| f == answer } remote_list.filter! do |f| @@ -57,10 +57,10 @@ namespace :comfy do end puts "Downloading a new set of #{answer}..." - - new_session.main_task(local_list, remote_list, to, FROM) end - + + new_session.main_task(local_list: local_list, remote_list: remote_list, local_base: to, remote_base: FROM) + puts "Finished downloads, now replacing your local seed data with your selection..." # new_session.commence_comfy_import(answer) From 2d488c9bfc3e7efc6e04803dfe36d39dc42dfddd Mon Sep 17 00:00:00 2001 From: Stanley Liu Date: Tue, 11 Aug 2020 17:25:18 +0100 Subject: [PATCH 40/49] moved a couple of guider comments around --- lib/modules/sync_seeds.rb | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/lib/modules/sync_seeds.rb b/lib/modules/sync_seeds.rb index ec7e87188..22ec5fd0f 100644 --- a/lib/modules/sync_seeds.rb +++ b/lib/modules/sync_seeds.rb @@ -85,6 +85,7 @@ def compare_folders(wildcard:, local:, remote:, base: @local_base) files_for_deletion(local_list, remote_list, base) end + # When folders need to be checked recursively def check_inside_folder(folder, local_list) paths = create_paths(folder) @@ -93,13 +94,10 @@ def check_inside_folder(folder, local_list) local_folder_content = Dir.glob('**/*', base: paths[:local_path]) remote_folder_content = @session.sftp.dir.glob(paths[:remote_path], '**/*') - # We don't want to download the whole folder again if it's already been re-downloaded once - - remote_folder_content.each do |file| # Go through the various files and folders and check to see if they exist locally local_file = file.name.force_encoding('UTF-8') - + check = check_if_newer( parent_folder: local_folder_content, local_item: local_file, @@ -108,7 +106,8 @@ def check_inside_folder(folder, local_list) local_path: paths[:local_path], base: paths[:local_path] ) - + + # We don't want to download the whole folder again if it's already been re-downloaded once # Break out of loop if already downloaded break if check == true end From e2eb8637989785c885a267c4309f827bbce4027b Mon Sep 17 00:00:00 2001 From: Stanley Liu Date: Wed, 12 Aug 2020 09:32:59 +0100 Subject: [PATCH 41/49] fixed the do block of the main task method --- lib/modules/sync_seeds.rb | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/modules/sync_seeds.rb b/lib/modules/sync_seeds.rb index 22ec5fd0f..30fa1530a 100644 --- a/lib/modules/sync_seeds.rb +++ b/lib/modules/sync_seeds.rb @@ -128,8 +128,7 @@ def main_task(local_list:, remote_list:, local_base:, remote_base:) remote_item: object, remote_path: paths[:remote_path], local_path: paths[:local_path] - ) - do + ) do check_inside_folder(name, local_list) if object.attributes.directory? end end From e15d64b26d5b008b47dd8e6329ba79ccd2244f88 Mon Sep 17 00:00:00 2001 From: Stanley Liu Date: Wed, 12 Aug 2020 10:55:47 +0100 Subject: [PATCH 42/49] corrected a typo --- lib/modules/sync_seeds.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/modules/sync_seeds.rb b/lib/modules/sync_seeds.rb index 30fa1530a..c48e7149e 100644 --- a/lib/modules/sync_seeds.rb +++ b/lib/modules/sync_seeds.rb @@ -139,7 +139,7 @@ def commence_comfy_import(answer) logger = ComfortableMexicanSofa.logger ComfortableMexicanSofa.logger = Logger.new(STDOUT) - if answer == 'All' + if answer == 'all' Rake::Task["comfy:cms_seeds:import"].invoke('protected-planet', 'protectedplanet') else module_name = "ComfortableMexicanSofa::Seeds::#{answer.singularize}::Importer".constantize From 0493598b8325a42c93df19544a55990f178a0fcf Mon Sep 17 00:00:00 2001 From: Stanley Liu Date: Mon, 17 Aug 2020 18:37:33 +0100 Subject: [PATCH 43/49] fixed an issue with instance variable being referenced before it was set --- lib/modules/sync_seeds.rb | 6 +++--- lib/tasks/staging_seeds.rake | 20 ++++++++++---------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/lib/modules/sync_seeds.rb b/lib/modules/sync_seeds.rb index c48e7149e..ee3f60509 100644 --- a/lib/modules/sync_seeds.rb +++ b/lib/modules/sync_seeds.rb @@ -25,7 +25,7 @@ def list_remote_files(location) def delete_files(files, location) files.each do |file| puts "Removing #{file} as it no longer exists remotely" - FileUtils.rm_rf(File.join(location, file)) + FileUtils.rm_rf(File.join(location, file)) end end @@ -73,7 +73,7 @@ def check_if_newer(parent_folder:, local_item:, remote_item:, remote_path:, loca downloaded end - def compare_folders(wildcard:, local:, remote:, base: @local_base) + def compare_folders(wildcard:, local:, remote:, base:) puts "Checking to see what files need to be deleted from #{local}" remote_list = @session.sftp.dir.glob(remote, wildcard).map do |f| @@ -81,7 +81,7 @@ def compare_folders(wildcard:, local:, remote:, base: @local_base) end local_list = Dir.glob(wildcard, base: local) - + files_for_deletion(local_list, remote_list, base) end diff --git a/lib/tasks/staging_seeds.rake b/lib/tasks/staging_seeds.rake index 291c4ce52..95bcaebb8 100644 --- a/lib/tasks/staging_seeds.rake +++ b/lib/tasks/staging_seeds.rake @@ -2,7 +2,7 @@ namespace :comfy do # The methods that this rake task calls are in lib/modules/sync_seeds.rb - FROM = 'ProtectedPlanet/current/db/cms_seeds/protected-planet'.freeze + SOURCE = 'ProtectedPlanet/current/db/cms_seeds/protected-planet'.freeze PP_STAGING = 'new-web.pp-staging.linode.protectedplanet.net'.freeze PP_USER = 'wcmc'.freeze @@ -22,20 +22,20 @@ namespace :comfy do end desc "Import CMS Seed data from staging. Can be run with arguments [, files/pages/layouts/all] or can accept user input if no argument is supplied" - task :staging_import, %i[to folder] => [:environment] do |_t, args| - to = nil + task :staging_import, %i[dest folder] => [:environment] do |_t, args| + dest = nil answer = nil if args.length.nil? answers = user_input - to = answers[:destination] + dest = answers[:destination] answer = answers[:answer] else - to = File.join(ComfortableMexicanSofa.config.seeds_path, args[:to]) + dest = File.join(ComfortableMexicanSofa.config.seeds_path, args[:to]) answer = args[:folder].downcase end - puts "Importing CMS Seed data from Staging Folder to #{to} ..." + puts "Importing CMS Seed data from Staging Folder to #{dest} ..." new_session = SyncSeeds.new(PP_STAGING, PP_USER) @@ -43,10 +43,10 @@ namespace :comfy do new_session.start_session do |session| # First get rid of any local top-level (i.e. which exist in the main # directory of REMOTE) folders/files that don't exist remotely - new_session.compare_folders(wildcard: '*', local: to, remote: FROM) + new_session.compare_folders(wildcard: '*', local: dest, remote: SOURCE, base: dest) - local_list = new_session.list_local_files(to) - remote_list = new_session.list_remote_files(FROM) + local_list = new_session.list_local_files(dest) + remote_list = new_session.list_remote_files(SOURCE) if answer == 'all' puts "Downloading all folders..." @@ -59,7 +59,7 @@ namespace :comfy do puts "Downloading a new set of #{answer}..." end - new_session.main_task(local_list: local_list, remote_list: remote_list, local_base: to, remote_base: FROM) + new_session.main_task(local_list: local_list, remote_list: remote_list, local_base: dest, remote_base: SOURCE) puts "Finished downloads, now replacing your local seed data with your selection..." From 111ce880661470ab4793530142e04701c28f37d7 Mon Sep 17 00:00:00 2001 From: Stanley Liu Date: Tue, 18 Aug 2020 10:13:23 +0100 Subject: [PATCH 44/49] initialised downloaded var in check if newer method --- lib/modules/sync_seeds.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/modules/sync_seeds.rb b/lib/modules/sync_seeds.rb index ee3f60509..9315e5011 100644 --- a/lib/modules/sync_seeds.rb +++ b/lib/modules/sync_seeds.rb @@ -50,6 +50,8 @@ def create_paths(relative_path) end def check_if_newer(parent_folder:, local_item:, remote_item:, remote_path:, local_path:, base: @local_base) + downloaded = false + if parent_folder.include?(local_item) yield if block_given? From 21767f27106236dcfbaf305f252c3f2db46c5936 Mon Sep 17 00:00:00 2001 From: Stanley Liu Date: Tue, 18 Aug 2020 10:18:42 +0100 Subject: [PATCH 45/49] corrected typo in args key --- lib/tasks/staging_seeds.rake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/tasks/staging_seeds.rake b/lib/tasks/staging_seeds.rake index 95bcaebb8..f7d65d93f 100644 --- a/lib/tasks/staging_seeds.rake +++ b/lib/tasks/staging_seeds.rake @@ -31,7 +31,7 @@ namespace :comfy do dest = answers[:destination] answer = answers[:answer] else - dest = File.join(ComfortableMexicanSofa.config.seeds_path, args[:to]) + dest = File.join(ComfortableMexicanSofa.config.seeds_path, args[:dest]) answer = args[:folder].downcase end From 28b2e3c0f052fddef7a8a4f1e6a9dc1a073572bf Mon Sep 17 00:00:00 2001 From: Stanley Liu Date: Tue, 18 Aug 2020 10:43:51 +0100 Subject: [PATCH 46/49] corrected the way it checks for args --- lib/tasks/staging_seeds.rake | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/tasks/staging_seeds.rake b/lib/tasks/staging_seeds.rake index f7d65d93f..40efd1017 100644 --- a/lib/tasks/staging_seeds.rake +++ b/lib/tasks/staging_seeds.rake @@ -25,8 +25,9 @@ namespace :comfy do task :staging_import, %i[dest folder] => [:environment] do |_t, args| dest = nil answer = nil + byebug - if args.length.nil? + if args[:dest].nil? && args[:folder].nil? answers = user_input dest = answers[:destination] answer = answers[:answer] From 2ef81f471c878122c24a454c2f01a29441361510 Mon Sep 17 00:00:00 2001 From: Stanley Liu Date: Tue, 18 Aug 2020 10:52:07 +0100 Subject: [PATCH 47/49] creates new folder if it doesn't exist --- lib/tasks/staging_seeds.rake | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/tasks/staging_seeds.rake b/lib/tasks/staging_seeds.rake index 40efd1017..a9c32c791 100644 --- a/lib/tasks/staging_seeds.rake +++ b/lib/tasks/staging_seeds.rake @@ -25,7 +25,6 @@ namespace :comfy do task :staging_import, %i[dest folder] => [:environment] do |_t, args| dest = nil answer = nil - byebug if args[:dest].nil? && args[:folder].nil? answers = user_input @@ -35,6 +34,11 @@ namespace :comfy do dest = File.join(ComfortableMexicanSofa.config.seeds_path, args[:dest]) answer = args[:folder].downcase end + + # Creates folder under db/cms_seeds if it doesn't exist + unless Dir.exist?(dest) + FileUtils.mkdir(dest) + end puts "Importing CMS Seed data from Staging Folder to #{dest} ..." From 9c16448f88fc157342298f4e8a5594309b228882 Mon Sep 17 00:00:00 2001 From: Stanley Liu Date: Tue, 18 Aug 2020 10:56:45 +0100 Subject: [PATCH 48/49] swapped mkdir for mkdir p for the full path --- lib/tasks/staging_seeds.rake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/tasks/staging_seeds.rake b/lib/tasks/staging_seeds.rake index a9c32c791..1666031d5 100644 --- a/lib/tasks/staging_seeds.rake +++ b/lib/tasks/staging_seeds.rake @@ -37,7 +37,7 @@ namespace :comfy do # Creates folder under db/cms_seeds if it doesn't exist unless Dir.exist?(dest) - FileUtils.mkdir(dest) + FileUtils.mkdir_p(dest) end puts "Importing CMS Seed data from Staging Folder to #{dest} ..." From e8db6e91d7aa932cb568a0f09a7a8d07465e56fe Mon Sep 17 00:00:00 2001 From: Stanley Liu Date: Tue, 18 Aug 2020 11:18:56 +0100 Subject: [PATCH 49/49] added todo to change commenc import method --- lib/modules/sync_seeds.rb | 3 +++ lib/tasks/staging_seeds.rake | 5 ++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/lib/modules/sync_seeds.rb b/lib/modules/sync_seeds.rb index 9315e5011..0fe132cab 100644 --- a/lib/modules/sync_seeds.rb +++ b/lib/modules/sync_seeds.rb @@ -137,6 +137,9 @@ def main_task(local_list:, remote_list:, local_base:, remote_base:) end # Piggybacks on existing Comfy modules + # TODO - the arguments used here for the site and folder + # (e.g. protectedplanet or protected-planet) can be different + # for different environments, need to change both this and the staging seeds rake task def commence_comfy_import(answer) logger = ComfortableMexicanSofa.logger ComfortableMexicanSofa.logger = Logger.new(STDOUT) diff --git a/lib/tasks/staging_seeds.rake b/lib/tasks/staging_seeds.rake index 1666031d5..44eecec0e 100644 --- a/lib/tasks/staging_seeds.rake +++ b/lib/tasks/staging_seeds.rake @@ -21,7 +21,9 @@ namespace :comfy do { answer: answer, destination: File.join(ComfortableMexicanSofa.config.seeds_path, 'protected-planet') } end - desc "Import CMS Seed data from staging. Can be run with arguments [, files/pages/layouts/all] or can accept user input if no argument is supplied" + desc "Import CMS Seed data from staging. Can be run with arguments + [destination folder - 'protected-planet' by convention, files/pages/layouts/all] + or can accept user input if no argument is supplied" task :staging_import, %i[dest folder] => [:environment] do |_t, args| dest = nil answer = nil @@ -68,6 +70,7 @@ namespace :comfy do puts "Finished downloads, now replacing your local seed data with your selection..." + # new_session.commence_comfy_import(answer) end end