Skip to content

Commit

Permalink
initialize syncseeds method written
Browse files Browse the repository at this point in the history
  • Loading branch information
stanleypliu committed Aug 5, 2020
1 parent 027e372 commit a13845a
Show file tree
Hide file tree
Showing 2 changed files with 47 additions and 27 deletions.
57 changes: 38 additions & 19 deletions lib/modules/sync_seeds.rb
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,26 @@ class SyncSeeds
require 'net/ssh'
require 'net/scp'

def initialize(server, username)
@username = username
@server = server
end

def start_session
Net::SSH.start(@server, @username) do |session|
@session = session
yield
end
end

def list_local_files(location)
Dir.glob('*', base: location)
end

def list_remote_files(location)
@session.sftp.dir.glob(location, '*')
end

def delete_files(files, location)
files.each do |file|
puts "Removing #{file} as it no longer exists remotely"
Expand All @@ -19,44 +39,44 @@ def files_for_deletion(list_of_files_1, list_of_files_2, location)
end
end

def download_files(source, dest, session)
def download_files(source, dest)
puts "Downloading #{source} as it's newer or not found locally"
return session.scp.download!(source, dest) if File.file?(dest)
session.scp.download!(source, dest, recursive: true)
return @session.scp.download!(source, dest) if File.file?(dest)
@session.scp.download!(source, dest, recursive: true)
end

def create_paths(relative_path)
{ local_path: File.join(LOCAL, relative_path), remote_path: File.join(REMOTE, relative_path) }
end

def check_if_newer(parent_folder, local_item, remote_item, remote_path, local_path, session, base = LOCAL)
def check_if_newer(parent_folder, local_item, remote_item, remote_path, local_path, base = LOCAL)
if parent_folder.include?(local_item)
yield if block_given?

is_newer = Time.at(remote_item.attributes.mtime) >= File.stat(local_path).mtime
# If there are any outdated files, will trigger download
if is_newer
if Dir.glob('*', base: LOCAL).include?(local_item)
download_files(remote_path, local_path, session)
download_files(remote_path, local_path)
# Will be hit if local_item is a file or folder inside a directory
elsif Dir.glob('**/*', base: base).include?(local_item)
download_files(remote_path, base, session)
download_files(remote_path, base)
downloaded = true
end
end
else
# Just download it if it doesn't exist at all
download_files(remote_path, LOCAL, session)
download_files(remote_path, LOCAL)
downloaded = true
end

downloaded
end

def compare_folders(wildcard, local, remote, session, base = LOCAL)
def compare_folders(wildcard, local, remote, base = LOCAL)
puts "Checking to see what files need to be deleted from #{base}"

remote_list = session.sftp.dir.glob(remote, wildcard).map do |f|
remote_list = @session.sftp.dir.glob(remote, wildcard).map do |f|
f.name.force_encoding('UTF-8')
end

Expand All @@ -65,13 +85,13 @@ def compare_folders(wildcard, local, remote, session, base = LOCAL)
files_for_deletion(local_list, remote_list, base)
end

def check_inside_folder(folder, local_list, session)
def check_inside_folder(folder, local_list)
paths = create_paths(folder)

compare_folders('**/*', paths[:local_path], paths[:remote_path], session, paths[:local_path])
compare_folders('**/*', paths[:local_path], paths[:remote_path], paths[:local_path])

local_folder_content = Dir.glob('**/*', base: paths[:local_path])
remote_folder_content = session.sftp.dir.glob(paths[:remote_path], '**/*')
remote_folder_content = @session.sftp.dir.glob(paths[:remote_path], '**/*')

# We don't want to download the whole folder again if it's already been re-downloaded once

Expand All @@ -81,27 +101,26 @@ def check_inside_folder(folder, local_list, session)
local_file = file.name
absolute_path = File.join(paths[:local_path], file.name)

check = check_if_newer(local_folder_content, local_file, file, paths[:remote_path], absolute_path, session, paths[:local_path])
check = check_if_newer(local_folder_content, local_file, file, paths[:remote_path], absolute_path, paths[:local_path])

# Break out of loop if already downloaded
break if check == true
end
end

def main_task(local_list, remote_list, session)
def main_task(local_list, remote_list)
remote_list.each do |object|
# There are files with non-ASCII characters (i.e. accented) in the CMS files
name = object.name.force_encoding('UTF-8')
paths = create_paths(name)


check_if_newer(local_list, name, object, paths[:remote_path], paths[:local_path], session) do
check_inside_folder(name, local_list, session) if object.attributes.directory?
check_if_newer(local_list, name, object, paths[:remote_path], paths[:local_path]) do
check_inside_folder(name, local_list) if object.attributes.directory?
end
end
end

def start_session(server, username)
Net::SSH.start(server, username) { yield }
end


end
17 changes: 9 additions & 8 deletions lib/tasks/staging_seeds.rake
Original file line number Diff line number Diff line change
Expand Up @@ -26,19 +26,20 @@ namespace :comfy do
# Fast-tracked unhappy path
abort('Goodbye') if answer == 'Nothing'

SyncSeeds.start_session(PP_STAGING, PP_USER)
new_session = SyncSeeds.new(PP_STAGING, PP_USER)

# SSH into staging server with Net::SSH
Net::SSH.start(PP_STAGING, PP_USER) do |session|
new_session.start_session do |session|
# First get rid of any local top-level (i.e. which exist in the main
# directory of REMOTE) folders/files that don't exist remotely
SyncSeeds.compare_folders('*', LOCAL, REMOTE, session)
new_session.compare_folders('*', LOCAL, REMOTE)

local_list = Dir.glob('*', base: LOCAL)
remote_list = session.sftp.dir.glob(REMOTE, '*')

local_list = new_session.list_local_files(LOCAL)
remote_list = new_session.list_remote_files(REMOTE)

if answer == 'All'
SyncSeeds.main_task(local_list, remote_list, session)
new_session.main_task(local_list, remote_list)
else
local_list.filter! { |f| f == answer.downcase }
remote_list.filter! do |f|
Expand All @@ -47,7 +48,7 @@ namespace :comfy do

puts "Downloading a new set of #{answer.downcase}..."

SyncSeeds.main_task(local_list, remote_list, session)
new_session.main_task(local_list, remote_list)
end


Expand All @@ -56,7 +57,7 @@ namespace :comfy do
# Rake::Task["comfy:cms_seeds:import"].invoke('protected-planet', 'protectedplanet')

# Todo: get this working with AWS bucket
end
end
end

end

0 comments on commit a13845a

Please sign in to comment.