Merge branch 'refresh' of github.com:unepwcmc/ProtectedPlanet into re…

…fresh

Gemfile

@@ -51,6 +51,7 @@ group :development do
   gem 'capistrano-git-with-submodules', '2.0.3'
   gem 'capistrano-service'
   gem 'awesome_print'
+  gem 'net-sftp'
   # gem 'listen', '~> 3.1.5'
   # gem 'spring-watcher-listen', '~> 2.0.0'
   #

Gemfile.lock

@@ -1128,6 +1128,8 @@ GEM
       thor (~> 0.19)
     net-scp (2.0.0)
       net-ssh (>= 2.6.5, < 6.0.0)
+    net-sftp (3.0.0)
+      net-ssh (>= 5.0.0, < 7.0.0)
     net-ssh (5.2.0)
     nio4r (2.5.2)
     nokogiri (1.10.7)
@@ -1341,6 +1343,7 @@ DEPENDENCIES
   minitest (~> 5.10, != 5.10.2)
   mocha (~> 1.0.0)
   neat
+  net-sftp
   nokogiri (~> 1.10.4)
   pg (~> 0.21)
   phantompdf (~> 1.2.2)

lib/modules/sync_seeds.rb

@@ -0,0 +1,157 @@
+class SyncSeeds
+  require 'net/ssh'
+  require 'net/scp' 
+
+  def initialize(server, username)
+    @username = username
+    @server = server
+  end
+
+  def start_session
+    Net::SSH.start(@server, @username) do |session| 
+      @session = session
+      yield
+    end
+  end
+
+  def list_local_files(location)
+    Dir.glob('*', base: location)
+  end
+
+  def list_remote_files(location)
+    @session.sftp.dir.glob(location, '*')
+  end
+
+  def delete_files(files, location)
+    files.each do |file| 
+      puts "Removing #{file} as it no longer exists remotely"
+      FileUtils.rm_rf(File.join(location, file))
+    end
+  end
+
+  def files_for_deletion(local_list, remote_list, location)
+    files = local_list - remote_list
+
+    if files.empty?
+      puts "No files to delete"
+    else
+      delete_files(files, location)
+    end
+  end
+
+  def download_files(source, dest)
+    puts "Downloading #{source} as it's newer or not found locally"
+    return @session.scp.download!(source, dest) if File.file?(dest)
+    @session.scp.download!(source, dest, recursive: true)
+  end
+
+  def create_paths(relative_path)
+    { local_path: File.join(@local_base, relative_path), remote_path: File.join(@remote_base, relative_path) }
+  end
+
+  def check_if_newer(parent_folder:, local_item:, remote_item:, remote_path:, local_path:, base: @local_base)
+    downloaded = false
+
+    if parent_folder.include?(local_item)
+      yield if block_given?
+
+      is_newer = Time.at(remote_item.attributes.mtime) >= File.stat(local_path).mtime  
+      # If there are any outdated files, will trigger download
+      if is_newer
+        if Dir.glob('*', base: @local_base).include?(local_item)
+          download_files(remote_path, local_path) 
+          # Will be hit if local_item is a file or folder inside a directory
+        elsif Dir.glob('**/*', base: base).include?(local_item)
+          download_files(remote_path, local_path)
+          downloaded = true  
+        end              
+      end
+    else
+      # Just download it if it doesn't exist at all
+      download_files(remote_path, @local_base)
+      downloaded = true
+    end
+
+    downloaded
+  end
+
+  def compare_folders(wildcard:, local:, remote:, base:)
+    puts "Checking to see what files need to be deleted from #{local}" 
+
+    remote_list = @session.sftp.dir.glob(remote, wildcard).map do |f|  
+      f.name.force_encoding('UTF-8')
+    end
+
+    local_list = Dir.glob(wildcard, base: local)
+
+    files_for_deletion(local_list, remote_list, base)
+  end
+
+  # When folders need to be checked recursively
+  def check_inside_folder(folder, local_list)
+    paths = create_paths(folder)
+
+    compare_folders(wildcard: '**/*', local: paths[:local_path], remote: paths[:remote_path], base: paths[:local_path])
+
+    local_folder_content = Dir.glob('**/*', base: paths[:local_path])
+    remote_folder_content = @session.sftp.dir.glob(paths[:remote_path], '**/*')
+
+    remote_folder_content.each do |file|
+      # Go through the various files and folders and check to see if they exist locally
+      local_file = file.name.force_encoding('UTF-8')
+
+      check = check_if_newer(
+        parent_folder: local_folder_content, 
+        local_item: local_file, 
+        remote_item: file, 
+        remote_path: paths[:remote_path], 
+        local_path: paths[:local_path], 
+        base: paths[:local_path]
+      )
+
+      # We don't want to download the whole folder again if it's already been re-downloaded once
+      # Break out of loop if already downloaded
+      break if check == true
+    end
+  end
+
+  def main_task(local_list:, remote_list:, local_base:, remote_base:)
+    @local_base = local_base
+    @remote_base = remote_base
+
+    remote_list.each do |object|
+      # There are files with non-ASCII characters (i.e. accented) in the CMS files
+      name = object.name.force_encoding('UTF-8')
+      paths = create_paths(name)
+
+      check_if_newer(
+        parent_folder: local_list, 
+        local_item: name, 
+        remote_item: object, 
+        remote_path: paths[:remote_path], 
+        local_path: paths[:local_path]
+      ) do 
+        check_inside_folder(name, local_list) if object.attributes.directory?
+      end
+    end
+  end
+
+  # Piggybacks on existing Comfy modules 
+  # TODO - the arguments used here for the site and folder
+  #  (e.g. protectedplanet or protected-planet) can be different 
+  # for different environments, need to change both this and the staging seeds rake task
+  def commence_comfy_import(answer)
+    logger = ComfortableMexicanSofa.logger
+    ComfortableMexicanSofa.logger = Logger.new(STDOUT)
+
+    if answer == 'all'
+      Rake::Task["comfy:cms_seeds:import"].invoke('protected-planet', 'protectedplanet')     
+    else
+      module_name = "ComfortableMexicanSofa::Seeds::#{answer.singularize}::Importer".constantize
+      module_name.new('protected-planet', 'protectedplanet').import!
+    end
+
+    ComfortableMexicanSofa.logger = logger
+  end
+
+end

lib/tasks/staging_seeds.rake

@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+namespace :comfy do
+  # The methods that this rake task calls are in lib/modules/sync_seeds.rb
+  SOURCE  = 'ProtectedPlanet/current/db/cms_seeds/protected-planet'.freeze
+  PP_STAGING = 'new-web.pp-staging.linode.protectedplanet.net'.freeze
+  PP_USER = 'wcmc'.freeze
+
+  def user_input
+    valid_answers = %w(all files layouts pages nothing)
+    puts question = "What would you like to import? 'all/files/layouts/pages' or 'nothing' to quit"
+    answer = STDIN.gets.chomp.downcase
+
+    until valid_answers.include?(answer)
+      puts question  
+      answer = STDIN.gets.chomp.downcase  
+    end
+
+    abort('Goodbye') if answer == 'nothing'
+
+    { answer: answer, destination: File.join(ComfortableMexicanSofa.config.seeds_path, 'protected-planet') }
+  end
+
+  desc "Import CMS Seed data from staging. Can be run with arguments 
+        [destination folder - 'protected-planet' by convention, files/pages/layouts/all] 
+         or can accept user input if no argument is supplied"
+  task :staging_import, %i[dest folder] => [:environment] do |_t, args| 
+    dest = nil
+    answer = nil
+
+    if args[:dest].nil? && args[:folder].nil?
+      answers = user_input
+      dest = answers[:destination]
+      answer = answers[:answer]
+    else
+      dest = File.join(ComfortableMexicanSofa.config.seeds_path, args[:dest])
+      answer = args[:folder].downcase
+    end
+
+    # Creates folder under db/cms_seeds if it doesn't exist
+    unless Dir.exist?(dest)
+      FileUtils.mkdir_p(dest)
+    end
+
+    puts "Importing CMS Seed data from Staging Folder to #{dest} ..."
+
+    new_session =  SyncSeeds.new(PP_STAGING, PP_USER)
+
+    # SSH into staging server with Net::SSH
+    new_session.start_session do |session|
+      # First get rid of any local top-level (i.e. which exist in the main 
+      # directory of REMOTE) folders/files that don't exist remotely
+      new_session.compare_folders(wildcard: '*', local: dest, remote: SOURCE, base: dest)
+
+      local_list = new_session.list_local_files(dest)
+      remote_list = new_session.list_remote_files(SOURCE)
+
+      if answer == 'all'
+        puts "Downloading all folders..."
+      else
+        local_list.filter! { |f| f == answer } 
+        remote_list.filter! do |f|
+          f.name.force_encoding('UTF-8') == answer
+        end
+
+        puts "Downloading a new set of #{answer}..."
+      end
+
+      new_session.main_task(local_list: local_list, remote_list: remote_list, local_base: dest, remote_base: SOURCE)
+
+      puts "Finished downloads, now replacing your local seed data with your selection..."
+
+
+      # new_session.commence_comfy_import(answer) 
+    end
+  end
+
+end