From ac57b7973ba6fe7d2c0bbe0aedc3313dd121b6ae Mon Sep 17 00:00:00 2001 From: Ferdinando Primerano Date: Fri, 18 Sep 2020 11:14:47 +0100 Subject: [PATCH 1/6] Fix and adapt OECM attributes for import --- app/models/protected_area.rb | 9 --------- lib/modules/download/queries.rb | 4 ++-- lib/modules/wdpa/data_standard.rb | 12 +++--------- 3 files changed, 5 insertions(+), 20 deletions(-) diff --git a/app/models/protected_area.rb b/app/models/protected_area.rb index 815ea6bba..730272298 100644 --- a/app/models/protected_area.rb +++ b/app/models/protected_area.rb @@ -24,8 +24,6 @@ class ProtectedArea < ApplicationRecord after_create :create_slug - validate :oecm_attributes - scope :all_except, -> (pa) { where.not(id: pa) } scope :oecms, -> { where(is_oecm: true) } @@ -306,11 +304,4 @@ def self.with_valid_iucn_categories "iucn_categories.name IN (#{valid_categories})" ) end - - OECM_ATTRS_ERROR = "'conservation_objectives' and 'supplementary_info' can only be assigned if this is an OECM area".freeze - def oecm_attributes - return if is_oecm - - errors.add(:oecm_attributes, OECM_ATTRS_ERROR) if supplementary_info || conservation_objectives - end end diff --git a/lib/modules/download/queries.rb b/lib/modules/download/queries.rb index bd0d31fd3..7f4f8711b 100644 --- a/lib/modules/download/queries.rb +++ b/lib/modules/download/queries.rb @@ -9,8 +9,8 @@ module Queries :no_tk_area, :status, :status_yr, :gov_type, :own_type, :mang_auth, :mang_plan, :verif, :metadataid, - :sub_loc, :parent_iso3, :iso3#, - #:supp_info, :cons_obj #TODO Ensure these OECM attributes are working ok here after importing DB with OECMs + :sub_loc, :parent_iso3, :iso3, + :supp_info, :cons_obj ] POLYGONS_COLUMNS = POINTS_COLUMNS.clone diff --git a/lib/modules/wdpa/data_standard.rb b/lib/modules/wdpa/data_standard.rb index e5450790d..fc9c47fce 100644 --- a/lib/modules/wdpa/data_standard.rb +++ b/lib/modules/wdpa/data_standard.rb @@ -26,8 +26,8 @@ class Wdpa::DataStandard :metadataid => {name: :sources, type: :integer, label: 'Source'}, :own_type => {name: :owner_type, type: :string, label: 'Owner Type'}, :pa_def => {name: :is_oecm, type: :oecm, label: 'PA Def'}, - :supp_info => {name: :supp_info, type: :string, label: 'Supplementary Info'}, - :cons_obj => {name: :cons_obj, type: :string, label: 'Conservation objectives'} + :supp_info => {name: :supplementary_info, type: :string, label: 'Supplementary Info'}, + :cons_obj => {name: :conservation_objectives, type: :string, label: 'Conservation objectives'} } POLYGON_ATTRIBUTES = [ @@ -37,11 +37,6 @@ class Wdpa::DataStandard :shape_length ] - OECM_ATTRIBUTES = [ - :supp_info, - :cons_obj - ] - module Matchers GEOMETRY_TABLE = /wdpa_?(wdoecm_)?po/i POLYGON_TABLE = /poly/i @@ -61,9 +56,8 @@ def self.standard_attributes STANDARD_ATTRIBUTES end - # TODO Ensure OECM attributes are included when importing a release with OECM areas def self.common_attributes - STANDARD_ATTRIBUTES.keys - POLYGON_ATTRIBUTES - OECM_ATTRIBUTES + STANDARD_ATTRIBUTES.keys - POLYGON_ATTRIBUTES end def self.standard_geometry_attributes From 5b2190fdb364dae28ff79d1812bdaf44b8633990 Mon Sep 17 00:00:00 2001 From: Ferdinando Primerano Date: Fri, 18 Sep 2020 11:15:01 +0100 Subject: [PATCH 2/6] Add useful comments for importer --- .../wdpa/protected_area_importer/attribute_importer.rb | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lib/modules/wdpa/protected_area_importer/attribute_importer.rb b/lib/modules/wdpa/protected_area_importer/attribute_importer.rb index b9fc395d5..9fc18dcb3 100644 --- a/lib/modules/wdpa/protected_area_importer/attribute_importer.rb +++ b/lib/modules/wdpa/protected_area_importer/attribute_importer.rb @@ -20,6 +20,11 @@ def self.create_protected_area(attributes) protected_area_id = nil begin ActiveRecord::Base.transaction(requires_new: true) do + # Below lines were needed on staging as the ProtectedArea model + # wasn't getting the new columns information after switching from the + # postgres db to the newly created db + #ProtectedArea.connection.schema_cache.clear! + #ProtectedArea.reset_column_information protected_area_id = ProtectedArea.create!(standardised_attributes).id end rescue From 83b5554695937a4f839c592793a8ab5c13776ba4 Mon Sep 17 00:00:00 2001 From: Ferdinando Primerano Date: Fri, 18 Sep 2020 11:15:19 +0100 Subject: [PATCH 3/6] Fix staging seeds rake task --- Gemfile | 2 +- lib/tasks/staging_seeds.rake | 32 +++++++++++++++++--------------- 2 files changed, 18 insertions(+), 16 deletions(-) diff --git a/Gemfile b/Gemfile index ec3a0263b..ab54ed61c 100644 --- a/Gemfile +++ b/Gemfile @@ -32,6 +32,7 @@ gem 'sprockets-vue', '~> 0.1.0' gem 'rails-controller-testing' gem 'gdal', '~> 2.0' +gem 'net-sftp' # group :production, :staging do # gem 'unicorn' @@ -51,7 +52,6 @@ group :development do gem 'capistrano-git-with-submodules', '2.0.3' gem 'capistrano-service' gem 'awesome_print' - gem 'net-sftp' # gem 'listen', '~> 3.1.5' # gem 'spring-watcher-listen', '~> 2.0.0' # diff --git a/lib/tasks/staging_seeds.rake b/lib/tasks/staging_seeds.rake index be0e873b7..1a72e283e 100644 --- a/lib/tasks/staging_seeds.rake +++ b/lib/tasks/staging_seeds.rake @@ -12,43 +12,45 @@ namespace :comfy do answer = STDIN.gets.chomp.downcase until valid_answers.include?(answer) - puts question - answer = STDIN.gets.chomp.downcase + puts question + answer = STDIN.gets.chomp.downcase end - + abort('Goodbye') if answer == 'nothing' { answer: answer, destination: File.join(ComfortableMexicanSofa.config.seeds_path, 'protected-planet') } end - desc "Import CMS Seed data from staging. Can be run with arguments - [destination folder - 'protected-planet' by convention, files/pages/layouts/all] + desc "Import CMS Seed data from staging. Can be run with arguments + [destination folder - 'protected-planet' by convention, files/pages/layouts/all] or can accept user input if no argument is supplied" - task :staging_import, %i[dest folder] => [:environment] do |_t, args| + task :staging_import, %i[dest folder] => [:environment] do |_t, args| dest = nil - answer = nil # answer is synonymous with folder + answer = nil # answer is synonymous with folder if args[:dest].nil? && args[:folder].nil? answers = user_input dest = answers[:destination] answer = answers[:answer] - else + elsif args[:dest] && args[:folder] dest = File.join(ComfortableMexicanSofa.config.seeds_path, args[:dest]) answer = args[:folder].downcase + else + abort('Please specify both destination and folder or leave both blank') end # Creates folder under db/cms_seeds if it doesn't exist unless Dir.exist?(dest) FileUtils.mkdir_p(dest) end - + puts "Importing CMS Seed data from Staging Folder to #{dest} ..." new_session = SyncSeeds.new(PP_STAGING, PP_USER) # SSH into staging server with Net::SSH new_session.start_session do |session| - # First get rid of any local top-level (i.e. which exist in the main + # First get rid of any local top-level (i.e. which exist in the main # directory of dest) folders/files that don't exist remotely new_session.compare_folders(wildcard: '*', local: dest, remote: SOURCE, base: dest) @@ -58,21 +60,21 @@ namespace :comfy do if answer == 'all' puts "Downloading all folders..." else - local_list.filter! { |f| f == answer } + local_list.filter! { |f| f == answer } remote_list.filter! do |f| f.name.force_encoding('UTF-8') == answer end puts "Downloading a new set of #{answer}..." end - + new_session.main_task(local_list: local_list, remote_list: remote_list, local_base: dest, remote_base: SOURCE) - + puts "Finished downloads, now replacing your local seed data with your selection..." - new_session.commence_comfy_import(answer, dest) + new_session.commence_comfy_import(answer, dest) end end -end \ No newline at end of file +end From 1e8e36670a6e2432dc7dfaf446d22b4fcd5b0d93 Mon Sep 17 00:00:00 2001 From: Ferdinando Primerano Date: Fri, 18 Sep 2020 11:31:55 +0100 Subject: [PATCH 4/6] Merge CMS fragments indexing into main create method --- lib/modules/search/index.rb | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/lib/modules/search/index.rb b/lib/modules/search/index.rb index 8f790cc6c..0fbcec74f 100644 --- a/lib/modules/search/index.rb +++ b/lib/modules/search/index.rb @@ -3,6 +3,14 @@ class Search::Index MAPPINGS_TEMPLATE = File.read(File.join(TEMPLATE_DIRECTORY, 'mappings.json')) def self.create + page_relation = Comfy::Cms::SearchablePage.includes([ + :fragments_for_index, + {:translations_for_index => :fragments_for_index}, + :categories + ]) + cms_index = Search::Index.new Search::CMS_INDEX, page_relation + cms_index.create + pa_relation = ProtectedArea.without_geometry.includes([ {:countries_for_index => :region_for_index}, :sub_locations, @@ -11,29 +19,19 @@ def self.create :governance ]) - region_index = Search::Index.new Search::REGION_INDEX, Region.without_geometry.all region_index.create country_index = Search::Index.new Search::COUNTRY_INDEX, Country.without_geometry.all country_index.create pa_index = Search::Index.new Search::PA_INDEX, pa_relation pa_index.create + + cms_index.index region_index.index country_index.index pa_index.index end - def self.create_cms_fragments - page_relation = Comfy::Cms::SearchablePage.includes([ - :fragments_for_index, - {:translations_for_index => :fragments_for_index}, - :categories - ]) - cms_index = Search::Index.new Search::CMS_INDEX, page_relation - cms_index.create - cms_index.index - end - def self.count self.new(Search::REGION_INDEX).count + self.new(Search::COUNTRY_INDEX).count + From 098a7b41d8f2f57f8d8d114b0a93b8b6abfb63a9 Mon Sep 17 00:00:00 2001 From: Stanley Liu Date: Tue, 22 Sep 2020 09:03:04 +0100 Subject: [PATCH 5/6] got rid of unused session var --- lib/tasks/staging_seeds.rake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/tasks/staging_seeds.rake b/lib/tasks/staging_seeds.rake index 1a72e283e..a237ce2f0 100644 --- a/lib/tasks/staging_seeds.rake +++ b/lib/tasks/staging_seeds.rake @@ -49,7 +49,7 @@ namespace :comfy do new_session = SyncSeeds.new(PP_STAGING, PP_USER) # SSH into staging server with Net::SSH - new_session.start_session do |session| + new_session.start_session do # First get rid of any local top-level (i.e. which exist in the main # directory of dest) folders/files that don't exist remotely new_session.compare_folders(wildcard: '*', local: dest, remote: SOURCE, base: dest) From 7c6b6a87253e73c55eb69bba686fefd2380ee2fa Mon Sep 17 00:00:00 2001 From: Ferdinando Primerano Date: Tue, 22 Sep 2020 09:57:12 +0100 Subject: [PATCH 6/6] Specify ssh key when not in development --- config/secrets.yml | 2 ++ lib/modules/sync_seeds.rb | 8 +++++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/config/secrets.yml b/config/secrets.yml index 27f815fc4..9106b13d8 100644 --- a/config/secrets.yml +++ b/config/secrets.yml @@ -112,6 +112,7 @@ staging: aws_bucket: 'pp-import-staging' aws_files_bucket: 'pp-files-staging' aws_s3_url: 'https://pp-downloads-staging.s3.amazonaws.com' + staging_ssh_key: <%= ENV['STAGING_SSH_KEY'] %> related_sources_base_urls: @@ -132,6 +133,7 @@ production: aws_bucket: 'pp-import-production' aws_files_bucket: 'pp-files-production' aws_s3_url: 'http://d1gam3xoknrgr2.cloudfront.net' + staging_ssh_key: <%= ENV['STAGING_SSH_KEY'] %> elasticsearch: diff --git a/lib/modules/sync_seeds.rb b/lib/modules/sync_seeds.rb index 9099afec2..16ca35ce2 100644 --- a/lib/modules/sync_seeds.rb +++ b/lib/modules/sync_seeds.rb @@ -8,7 +8,13 @@ def initialize(server, username) end def start_session - Net::SSH.start(@server, @username) do |session| + opts = Rails.env.development? ? {} : {keys: [Rails.application.secrets.staging_ssh_key]} + + Net::SSH.start( + @server, + @username, + opts + ) do |session| @session = session yield end