Skip to content

Commit

Permalink
placed the methods into a module
Browse files Browse the repository at this point in the history
  • Loading branch information
@stanleypliu
stanleypliu committed Aug 4, 2020
1 parent 8c4bac3 commit 58dcc5f
Show file tree
Hide file tree
Showing 2 changed files with 139 additions and 132 deletions.
107 changes: 107 additions & 0 deletions lib/modules/sync_seeds.rb
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,107 @@
class SyncSeeds
require 'net/ssh'
require 'net/scp'

def delete_files(files, location)
files.each do |file|
puts "Removing #{file} as it no longer exists remotely"
FileUtils.rm_rf(File.join(location, file))
end
end

def files_for_deletion(list_of_files_1, list_of_files_2, location)
files = list_of_files_1 - list_of_files_2

if files.empty?
puts "No files to delete"
else
delete_files(files, location)
end
end

def download_files(source, dest, session)
puts "Downloading #{source} as it's newer or not found locally"
return session.scp.download!(source, dest) if File.file?(dest)
session.scp.download!(source, dest, recursive: true)
end

def create_paths(relative_path)
{ local_path: File.join(LOCAL, relative_path), remote_path: File.join(REMOTE, relative_path) }
end

def check_if_newer(parent_folder, local_item, remote_item, remote_path, local_path, session, base = LOCAL)
if parent_folder.include?(local_item)
yield if block_given?

is_newer = Time.at(remote_item.attributes.mtime) >= File.stat(local_path).mtime
# If there are any outdated files, will trigger download
if is_newer
if Dir.glob('*', base: LOCAL).include?(local_item)
download_files(remote_path, local_path, session)
# Will be hit if local_item is a file or folder inside a directory
elsif Dir.glob('**/*', base: base).include?(local_item)
download_files(remote_path, base, session)
downloaded = true
end
end
else
# Just download it if it doesn't exist at all
download_files(remote_path, LOCAL, session)
downloaded = true
end

downloaded
end

def compare_folders(wildcard, local, remote, session, base = LOCAL)
puts "Checking to see what files need to be deleted from #{base}"

remote_list = session.sftp.dir.glob(remote, wildcard).map do |f|
f.name.force_encoding('UTF-8')
end

local_list = Dir.glob(wildcard, base: local)

files_for_deletion(local_list, remote_list, base)
end

def check_inside_folder(folder, local_list, session)
paths = create_paths(folder)

compare_folders('**/*', paths[:local_path], paths[:remote_path], session, paths[:local_path])

local_folder_content = Dir.glob('**/*', base: paths[:local_path])
remote_folder_content = session.sftp.dir.glob(paths[:remote_path], '**/*')

# We don't want to download the whole folder again if it's already been re-downloaded once


remote_folder_content.each do |file|
# Go through the various files and folders and check to see if they exist locally
local_file = file.name
absolute_path = File.join(paths[:local_path], file.name)

check = check_if_newer(local_folder_content, local_file, file, paths[:remote_path], absolute_path, session, paths[:local_path])

# Break out of loop if already downloaded
break if check == true
end
end

def main_task(local_list, remote_list, session)
remote_list.each do |object|
# There are files with non-ASCII characters (i.e. accented) in the CMS files
name = object.name.force_encoding('UTF-8')
paths = create_paths(name)


check_if_newer(local_list, name, object, paths[:remote_path], paths[:local_path], session) do
check_inside_folder(name, local_list, session) if object.attributes.directory?
end
end
end

def start_session(server, username)
Net::SSH.start(server, username) { yield }
end
end
164 changes: 32 additions & 132 deletions lib/tasks/staging_seeds.rake
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,152 +11,52 @@ namespace :comfy do
PP_USER = 'wcmc'.freeze

desc "Import CMS Seed data from staging"

def delete_files(files, location)
files.each do |file|
puts "Removing #{file} as it no longer exists remotely"
FileUtils.rm_rf(File.join(location, file))
end
end

def files_for_deletion(list_of_files_1, list_of_files_2, location)
files = list_of_files_1 - list_of_files_2

if files.empty?
puts "No files to delete"
else
delete_files(files, location)
end
end

def download_files(source, dest, session)
puts "Downloading #{source} as it's newer or not found locally"
return session.scp.download!(source, dest) if File.file?(dest)
session.scp.download!(source, dest, recursive: true)
end
task :staging_import => :environment do |_t|

def create_paths(relative_path)
{ local_path: File.join(LOCAL, relative_path), remote_path: File.join(REMOTE, relative_path) }
end

def check_if_newer(parent_folder, local_item, remote_item, remote_path, local_path, session, base = LOCAL)
if parent_folder.include?(local_item)
yield if block_given?

is_newer = Time.at(remote_item.attributes.mtime) >= File.stat(local_path).mtime
# If there are any outdated files, will trigger download
if is_newer
if Dir.glob('*', base: LOCAL).include?(local_item)
download_files(remote_path, local_path, session)
# Will be hit if local_item is a file or folder inside a directory
elsif Dir.glob('**/*', base: base).include?(local_item)
download_files(remote_path, base, session)
downloaded = true
end
end
else
# Just download it if it doesn't exist at all
download_files(remote_path, LOCAL, session)
downloaded = true
end

downloaded
end

def compare_folders(wildcard, local, remote, session, base = LOCAL)
puts "Checking to see what files need to be deleted from #{base}"

remote_list = session.sftp.dir.glob(remote, wildcard).map do |f|
f.name.force_encoding('UTF-8')
end
puts "Importing CMS Seed data from Staging Folder to #{LOCAL} ..."

local_list = Dir.glob(wildcard, base: local)

files_for_deletion(local_list, remote_list, base)
puts question = "What would you like to import? 'All/Files/Layouts/Pages' or 'Nothing' to quit"
valid_answers = ['All', 'Files', 'Layouts', 'Pages', 'Nothing']
answer = STDIN.gets.chomp.downcase.capitalize
until valid_answers.include?(answer)
puts question
answer = STDIN.gets.chomp.downcase.capitalize
end

def check_inside_folder(folder, local_list, session)
paths = create_paths(folder)

compare_folders('**/*', paths[:local_path], paths[:remote_path], session, paths[:local_path])
# Fast-tracked unhappy path
abort('Goodbye') if answer == 'Nothing'

local_folder_content = Dir.glob('**/*', base: paths[:local_path])
remote_folder_content = session.sftp.dir.glob(paths[:remote_path], '**/*')

# We don't want to download the whole folder again if it's already been re-downloaded once
SyncSeeds.start_session(PP_STAGING, PP_USER)

# SSH into staging server with Net::SSH
Net::SSH.start(PP_STAGING, PP_USER) do |session|
# First get rid of any local top-level (i.e. which exist in the main
# directory of REMOTE) folders/files that don't exist remotely
SyncSeeds.compare_folders('*', LOCAL, REMOTE, session)

remote_folder_content.each do |file|
# Go through the various files and folders and check to see if they exist locally
local_file = file.name
absolute_path = File.join(paths[:local_path], file.name)

check = check_if_newer(local_folder_content, local_file, file, paths[:remote_path], absolute_path, session, paths[:local_path])

# Break out of loop if already downloaded
break if check == true
end
end
local_list = Dir.glob('*', base: LOCAL)
remote_list = session.sftp.dir.glob(REMOTE, '*')

def main_task(local_list, remote_list, session)
remote_list.each do |object|
# There are files with non-ASCII characters (i.e. accented) in the CMS files
name = object.name.force_encoding('UTF-8')
paths = create_paths(name)


check_if_newer(local_list, name, object, paths[:remote_path], paths[:local_path], session) do
check_inside_folder(name, local_list, session) if object.attributes.directory?
if answer == 'All'
SyncSeeds.main_task(local_list, remote_list, session)
else
local_list.filter! { |f| f == answer.downcase }
remote_list.filter! do |f|
f.name.force_encoding('UTF-8') == answer.downcase
end
end
end

task :staging_import => :environment do |_t|
require 'net/ssh'
require 'net/scp'

puts "Importing CMS Seed data from Staging Folder to #{LOCAL} ..."
puts "Downloading a new set of #{answer.downcase}..."

puts question = "What would you like to import? 'All/Files/Layouts/Pages' or 'Nothing' to quit"
valid_answers = ['All', 'Files', 'Layouts', 'Pages', 'Nothing']
answer = STDIN.gets.chomp.downcase.capitalize
until valid_answers.include?(answer)
puts question
answer = STDIN.gets.chomp.downcase.capitalize
SyncSeeds.main_task(local_list, remote_list, session)
end


# puts "Finished downloads, now replacing your local seed data..."

# Fast-tracked unhappy path
abort('Goodbye') if answer == 'Nothing'

# SSH into staging server with Net::SSH
Net::SSH.start(PP_STAGING, PP_USER) do |session|
# First get rid of any local top-level (i.e. which exist in the main
# directory of REMOTE) folders/files that don't exist remotely
compare_folders('*', LOCAL, REMOTE, session)

local_list = Dir.glob('*', base: LOCAL)
remote_list = session.sftp.dir.glob(REMOTE, '*')

if answer == 'All'
main_task(local_list, remote_list, session)
else
local_list.filter! { |f| f == answer.downcase }
remote_list.filter! do |f|
f.name.force_encoding('UTF-8') == answer.downcase
end

puts "Downloading a new set of #{answer.downcase}..."

main_task(local_list, remote_list, session)
end


# puts "Finished downloads, now replacing your local seed data..."

# Rake::Task["comfy:cms_seeds:import"].invoke('protected-planet', 'protectedplanet')
# Rake::Task["comfy:cms_seeds:import"].invoke('protected-planet', 'protectedplanet')

# Todo: get this working with AWS bucket
end
# Todo: get this working with AWS bucket
end
end

end

0 comments on commit 58dcc5f

Please sign in to comment.