Merge pull request #533 from unepwcmc/import-ready-fixes

Import ready fixes

Gemfile

@@ -32,6 +32,7 @@ gem 'sprockets-vue', '~> 0.1.0'
 gem 'rails-controller-testing'
 
 gem 'gdal', '~> 2.0'
+gem 'net-sftp'
 #
 group :production, :staging do
 #  gem 'unicorn'
@@ -51,7 +52,6 @@ group :development do
   gem 'capistrano-git-with-submodules', '2.0.3'
   gem 'capistrano-service'
   gem 'awesome_print'
-  gem 'net-sftp'
   # gem 'listen', '~> 3.1.5'
   # gem 'spring-watcher-listen', '~> 2.0.0'
   #

app/models/protected_area.rb

@@ -24,8 +24,6 @@ class ProtectedArea < ApplicationRecord
 
   after_create :create_slug
 
-  validate :oecm_attributes
-
   scope :all_except, -> (pa) { where.not(id: pa) }
 
   scope :oecms, -> { where(is_oecm: true) }
@@ -313,11 +311,4 @@ def self.with_valid_iucn_categories
       "iucn_categories.name IN (#{valid_categories})"
     )
   end
-
-  OECM_ATTRS_ERROR = "'conservation_objectives' and 'supplementary_info' can only be assigned if this is an OECM area".freeze
-  def oecm_attributes
-    return if is_oecm
-
-    errors.add(:oecm_attributes, OECM_ATTRS_ERROR) if supplementary_info || conservation_objectives
-  end
 end

config/secrets.yml

@@ -112,6 +112,7 @@ staging:
   aws_bucket: 'pp-import-staging'
   aws_files_bucket: 'pp-files-staging'
   aws_s3_url: 'https://pp-downloads-staging.s3.amazonaws.com'
+  staging_ssh_key: <%= ENV['STAGING_SSH_KEY'] %>
 
 
   related_sources_base_urls:
@@ -132,6 +133,7 @@ production:
   aws_bucket: 'pp-import-production'
   aws_files_bucket: 'pp-files-production'
   aws_s3_url: 'http://d1gam3xoknrgr2.cloudfront.net'
+  staging_ssh_key: <%= ENV['STAGING_SSH_KEY'] %>
 
 
   elasticsearch:

lib/modules/download/queries.rb

@@ -9,8 +9,8 @@ module Queries
       :no_tk_area, :status, :status_yr,
       :gov_type, :own_type, :mang_auth,
       :mang_plan, :verif, :metadataid,
-      :sub_loc, :parent_iso3, :iso3#,
-      #:supp_info, :cons_obj #TODO Ensure these OECM attributes are working ok here after importing DB with OECMs
+      :sub_loc, :parent_iso3, :iso3,
+      :supp_info, :cons_obj
     ]
 
     POLYGONS_COLUMNS = POINTS_COLUMNS.clone

lib/modules/search/index.rb

@@ -3,6 +3,14 @@ class Search::Index
   MAPPINGS_TEMPLATE = File.read(File.join(TEMPLATE_DIRECTORY, 'mappings.json'))
 
   def self.create
+    page_relation = Comfy::Cms::SearchablePage.includes([
+      :fragments_for_index,
+      {:translations_for_index => :fragments_for_index},
+      :categories
+    ])
+    cms_index = Search::Index.new Search::CMS_INDEX, page_relation
+    cms_index.create
+
     pa_relation = ProtectedArea.without_geometry.includes([
       {:countries_for_index => :region_for_index},
       :sub_locations,
@@ -11,29 +19,19 @@ def self.create
       :governance
     ])
 
-
     region_index = Search::Index.new Search::REGION_INDEX, Region.without_geometry.all
     region_index.create
     country_index = Search::Index.new Search::COUNTRY_INDEX, Country.without_geometry.all
     country_index.create
     pa_index = Search::Index.new Search::PA_INDEX, pa_relation
     pa_index.create
+
+    cms_index.index
     region_index.index
     country_index.index
     pa_index.index
   end
 
-  def self.create_cms_fragments
-    page_relation = Comfy::Cms::SearchablePage.includes([
-      :fragments_for_index,
-      {:translations_for_index => :fragments_for_index},
-      :categories
-    ])
-    cms_index = Search::Index.new Search::CMS_INDEX, page_relation
-    cms_index.create
-    cms_index.index
-  end
-
   def self.count
     self.new(Search::REGION_INDEX).count +
     self.new(Search::COUNTRY_INDEX).count +

lib/modules/sync_seeds.rb

@@ -8,7 +8,13 @@ def initialize(server, username)
   end
 
   def start_session
-    Net::SSH.start(@server, @username) do |session| 
+    opts = Rails.env.development? ? {} : {keys: [Rails.application.secrets.staging_ssh_key]}
+
+    Net::SSH.start(
+      @server,
+      @username,
+      opts
+    ) do |session|
       @session = session
       yield
     end

lib/modules/wdpa/data_standard.rb

@@ -26,8 +26,8 @@ class Wdpa::DataStandard
     :metadataid   => {name: :sources, type: :integer, label: 'Source'},
     :own_type     => {name: :owner_type, type: :string, label: 'Owner Type'},
     :pa_def       => {name: :is_oecm, type: :oecm, label: 'PA Def'},
-    :supp_info    => {name: :supp_info, type: :string, label: 'Supplementary Info'},
-    :cons_obj     => {name: :cons_obj, type: :string, label: 'Conservation objectives'}
+    :supp_info    => {name: :supplementary_info, type: :string, label: 'Supplementary Info'},
+    :cons_obj     => {name: :conservation_objectives, type: :string, label: 'Conservation objectives'}
   }
 
   POLYGON_ATTRIBUTES = [
@@ -37,11 +37,6 @@ class Wdpa::DataStandard
     :shape_length
   ]
 
-  OECM_ATTRIBUTES = [
-    :supp_info,
-    :cons_obj
-  ]
-
   module Matchers
     GEOMETRY_TABLE = /wdpa_?(wdoecm_)?po/i
     POLYGON_TABLE  = /poly/i
@@ -61,9 +56,8 @@ def self.standard_attributes
     STANDARD_ATTRIBUTES
   end
 
-  # TODO Ensure OECM attributes are included when importing a release with OECM areas
   def self.common_attributes
-    STANDARD_ATTRIBUTES.keys - POLYGON_ATTRIBUTES - OECM_ATTRIBUTES
+    STANDARD_ATTRIBUTES.keys - POLYGON_ATTRIBUTES
   end
 
   def self.standard_geometry_attributes

lib/modules/wdpa/protected_area_importer/attribute_importer.rb

@@ -20,6 +20,11 @@ def self.create_protected_area(attributes)
     protected_area_id = nil
     begin
       ActiveRecord::Base.transaction(requires_new: true) do
+        # Below lines were needed on staging as the ProtectedArea model
+        # wasn't getting the new columns information after switching from the
+        # postgres db to the newly created db
+        #ProtectedArea.connection.schema_cache.clear!
+        #ProtectedArea.reset_column_information
         protected_area_id = ProtectedArea.create!(standardised_attributes).id
       end
     rescue

lib/tasks/staging_seeds.rake

@@ -12,43 +12,45 @@ namespace :comfy do
     answer = STDIN.gets.chomp.downcase
 
     until valid_answers.include?(answer)
-      puts question  
-      answer = STDIN.gets.chomp.downcase  
+      puts question
+      answer = STDIN.gets.chomp.downcase
     end
-    
+
     abort('Goodbye') if answer == 'nothing'
 
     { answer: answer, destination: File.join(ComfortableMexicanSofa.config.seeds_path, 'protected-planet') }
   end
 
-  desc "Import CMS Seed data from staging. Can be run with arguments 
-        [destination folder - 'protected-planet' by convention, files/pages/layouts/all] 
+  desc "Import CMS Seed data from staging. Can be run with arguments
+        [destination folder - 'protected-planet' by convention, files/pages/layouts/all]
          or can accept user input if no argument is supplied"
-  task :staging_import, %i[dest folder] => [:environment] do |_t, args| 
+  task :staging_import, %i[dest folder] => [:environment] do |_t, args|
     dest = nil
-    answer = nil # answer is synonymous with folder 
+    answer = nil # answer is synonymous with folder
 
     if args[:dest].nil? && args[:folder].nil?
       answers = user_input
       dest = answers[:destination]
       answer = answers[:answer]
-    else
+    elsif args[:dest] && args[:folder]
       dest = File.join(ComfortableMexicanSofa.config.seeds_path, args[:dest])
       answer = args[:folder].downcase
+    else
+      abort('Please specify both destination and folder or leave both blank')
     end
 
     # Creates folder under db/cms_seeds if it doesn't exist
     unless Dir.exist?(dest)
       FileUtils.mkdir_p(dest)
     end
-      
+
     puts "Importing CMS Seed data from Staging Folder to #{dest} ..."
 
     new_session =  SyncSeeds.new(PP_STAGING, PP_USER)
 
     # SSH into staging server with Net::SSH
-    new_session.start_session do |session|
-      # First get rid of any local top-level (i.e. which exist in the main 
+    new_session.start_session do 
+      # First get rid of any local top-level (i.e. which exist in the main
       # directory of dest) folders/files that don't exist remotely
       new_session.compare_folders(wildcard: '*', local: dest, remote: SOURCE, base: dest)
 
@@ -58,21 +60,21 @@ namespace :comfy do
       if answer == 'all'
         puts "Downloading all folders..."
       else
-        local_list.filter! { |f| f == answer } 
+        local_list.filter! { |f| f == answer }
         remote_list.filter! do |f|
           f.name.force_encoding('UTF-8') == answer
         end
 
         puts "Downloading a new set of #{answer}..."
       end
-      
+
       new_session.main_task(local_list: local_list, remote_list: remote_list, local_base: dest, remote_base: SOURCE)
-      
+
       puts "Finished downloads, now replacing your local seed data with your selection..."
 
 
-      new_session.commence_comfy_import(answer, dest) 
+      new_session.commence_comfy_import(answer, dest)
     end
   end
 
-end
+end