From eab45feed52c702ac6cab3fa17642ddcbe4482ba Mon Sep 17 00:00:00 2001 From: Andrew Hobden Date: Tue, 23 Aug 2016 17:28:00 +0200 Subject: [PATCH 01/51] Listen on port 3000 --- config/unicorn.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/unicorn.rb b/config/unicorn.rb index 788779f98..8ff5d30ee 100644 --- a/config/unicorn.rb +++ b/config/unicorn.rb @@ -25,7 +25,7 @@ # This is where we specify the socket. # We will point the upstream Nginx module to this socket later on -listen "#{shared_path}/unicorn.sock", backlog: 1024 +listen 3000, :backlog => 1024 pid "#{shared_path}/pids/unicorn.pid" From ebabcc63f75cc06350c76af38734b3c3363a806d Mon Sep 17 00:00:00 2001 From: Andrew Hobden Date: Fri, 5 Aug 2016 12:09:12 -0700 Subject: [PATCH 02/51] Log to stdout --- config/environments/production.rb | 2 +- config/environments/staging.rb | 2 +- config/initializers/delayed_job_config.rb | 2 +- lib/rack_request_logger.rb | 10 +++++----- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/config/environments/production.rb b/config/environments/production.rb index 7a3f8a460..d3fc74abf 100644 --- a/config/environments/production.rb +++ b/config/environments/production.rb @@ -42,7 +42,7 @@ # See everything in the log (default is :info) config.log_level = :info - config.logger = Logger.new('log/production.log', 3, 1_073_741_824) # 1GB + config.logger = Logger.new($stdout) # Use a different cache store in production # default is: diff --git a/config/environments/staging.rb b/config/environments/staging.rb index 28bd17813..97cf8864e 100644 --- a/config/environments/staging.rb +++ b/config/environments/staging.rb @@ -39,7 +39,7 @@ # Use a different logger for distributed setups # config.logger = SyslogLogger.new - config.logger = Logger.new('log/staging.log', 3, 1_073_741_824) # 1GB + config.logger = Logger.new($stdout) # Use a different cache store in staging # default is: diff --git a/config/initializers/delayed_job_config.rb b/config/initializers/delayed_job_config.rb index dddfe4a7b..db7dd8242 100644 --- a/config/initializers/delayed_job_config.rb +++ b/config/initializers/delayed_job_config.rb @@ -3,7 +3,7 @@ Delayed::Worker.sleep_delay = 5 Delayed::Worker.max_attempts = 20 Delayed::Worker.max_run_time = 5.minutes -Delayed::Worker.logger = Logger.new("#{Rails.root}/log/delayed_job.log") +Delayed::Worker.logger = Logger.new($stdout) # require 'delayed_job_plugin' # Delayed::Worker.plugins << DelayedJobPlugin diff --git a/lib/rack_request_logger.rb b/lib/rack_request_logger.rb index 710854214..63e85e088 100644 --- a/lib/rack_request_logger.rb +++ b/lib/rack_request_logger.rb @@ -3,17 +3,17 @@ module Rack # especially to investigate XML stuff going through the API class RequestLogger def initialize(app) - @logger = ::Logger.new(Rails.root.join('log/rack_requests.log')) + @logger = ::Logger.new($stdout) @app = app end def call(env) if env['HTTP_USER_AGENT'] =~ /Wheelmap/ - @logger.debug('>>>>>>>>>>>>>>>>>>>>>>>>>>') + @logger.debug(">>>>>>>>>>>>>>>>>>>>>>>>>>") @logger.debug("#{Time.now.utc} new request:") - @logger.debug('----------- REQUEST HEADERS -----------') - headers = env.select { |k, _v| k =~ /^HTTP_/ } || [] - @logger.debug(headers.map { |k, v| "#{k}: #{v}" }.join("\n")) + @logger.debug("----------- REQUEST HEADERS -----------") + headers = env.select { |k, v| k =~ /^HTTP_/ } || [] + @logger.debug(headers.map { |k, v| "#{k}: #{v}"}.join("\n")) request = Rack::Request.new env @logger.debug('----------- REQUEST BODY (RAW_POST_DATA) -----------') From cf9ebab9e798d158123da691faca5b1695817d4c Mon Sep 17 00:00:00 2001 From: Felis Gilcher Date: Tue, 23 Aug 2016 22:00:38 +0000 Subject: [PATCH 03/51] new deployment config add app servers, workers are still commented out wwq --- Capfile | 2 +- config/deploy/production.rb | 29 ++++++++++++++++------------- 2 files changed, 17 insertions(+), 14 deletions(-) diff --git a/Capfile b/Capfile index 9ef566ef3..de9a56ce4 100644 --- a/Capfile +++ b/Capfile @@ -14,7 +14,7 @@ require 'capistrano/deploy' # https://github.com/capistrano/bundler # https://github.com/capistrano/rails # -require 'capistrano/rbenv' +#require 'capistrano/rbenv' require 'capistrano/bundler' require 'capistrano/npm' require 'capistrano/rails/assets' diff --git a/config/deploy/production.rb b/config/deploy/production.rb index 8ec07b93e..84e96b65a 100644 --- a/config/deploy/production.rb +++ b/config/deploy/production.rb @@ -6,7 +6,7 @@ set :stage, :production set :deploy_to, "/var/apps/#{fetch(:application)}/#{fetch(:stage)}" -set :branch, :production +set :branch, :"feature/infrastructure-rebuild/deployment" set :rev, proc { `git rev-parse --short #{fetch(:branch)}`.chomp } # Simple Role Syntax @@ -15,10 +15,10 @@ # is considered to be the first unless any hosts have the primary # property set. Don't declare `role :all`, it's a meta role. -role :app, %w(176.9.63.170) -role :web, %w(176.9.63.170) -role :db, %w(176.9.63.170) -role :worker, %w(176.9.63.170) +#role :app, %w{176.9.63.170} +#role :web, %w{176.9.63.170} +#role :db, %w{176.9.63.170} +#role :worker, %w{176.9.63.170} # Extended Server Syntax # ====================== @@ -26,7 +26,10 @@ # server list. The second argument is a, or duck-types, Hash and is # used to set extended properties on the server. -server '176.9.63.170', user: 'rails', roles: %w(web app db worker), port: 22_022 +server 'app0.node.consul', user: 'deploy', roles: %w{web app}, port: 22 +server 'app1.node.consul', user: 'deploy', roles: %w{web app}, port: 22 +#server 'worker.node.consul', user: 'deploy', roles: %w{worker}, port: 22 +#server 'mysql.node.consul', user: 'wheelmap', roles: %w{mysql}, port: 22 # Custom SSH Options # ================== @@ -35,13 +38,13 @@ # # Global options # -------------- -set :ssh_options, - keys: %w(~/.ssh/wheelmap_rsa), - forward_agent: true, - config: true, - port: 22_022 -# auth_methods: %w(password) - +#set :ssh_options, { +# keys: %w(~/.ssh/wheelmap_rsa), +# forward_agent: true, +# config: true, +# port: 22022 +# # auth_methods: %w(password) +#} # # And/or per server (overrides global) # ------------------------------------ From b2b9df914f5c0e764ae8c22b543e468dc90c459c Mon Sep 17 00:00:00 2001 From: Felis Gilcher Date: Tue, 23 Aug 2016 22:37:13 +0000 Subject: [PATCH 04/51] remove stage from the deployment path since we have separate development/production machines we don't need that w --- config/deploy/production.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/deploy/production.rb b/config/deploy/production.rb index 84e96b65a..08cf2708b 100644 --- a/config/deploy/production.rb +++ b/config/deploy/production.rb @@ -4,7 +4,7 @@ set :rails_env, 'production' # added for delayed job set :stage, :production -set :deploy_to, "/var/apps/#{fetch(:application)}/#{fetch(:stage)}" +set :deploy_to, "/var/apps/#{fetch(:application)}/" set :branch, :"feature/infrastructure-rebuild/deployment" set :rev, proc { `git rev-parse --short #{fetch(:branch)}`.chomp } From 89530414b8941e54bceab36f3e9b885a3c279861 Mon Sep 17 00:00:00 2001 From: Felis Gilcher Date: Tue, 23 Aug 2016 23:01:37 +0000 Subject: [PATCH 05/51] add asset server in the web role --- config/deploy/production.rb | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/config/deploy/production.rb b/config/deploy/production.rb index 08cf2708b..6dbccebe4 100644 --- a/config/deploy/production.rb +++ b/config/deploy/production.rb @@ -26,8 +26,9 @@ # server list. The second argument is a, or duck-types, Hash and is # used to set extended properties on the server. -server 'app0.node.consul', user: 'deploy', roles: %w{web app}, port: 22 -server 'app1.node.consul', user: 'deploy', roles: %w{web app}, port: 22 +server 'app0.node.consul', user: 'deploy', roles: %w{app}, port: 22 +server 'app1.node.consul', user: 'deploy', roles: %w{app}, port: 22 +server 'asset.node.consul', user: 'deploy', roles: %w{web}, port: 22 #server 'worker.node.consul', user: 'deploy', roles: %w{worker}, port: 22 #server 'mysql.node.consul', user: 'wheelmap', roles: %w{mysql}, port: 22 From d5c7f26827c9eaf14dd25c14c4a7b0abd9683a2e Mon Sep 17 00:00:00 2001 From: Felis Gilcher Date: Tue, 23 Aug 2016 23:05:18 +0000 Subject: [PATCH 06/51] comment asset server the asset server is lacking the shared configs. deployment is not possible on that machine --- config/deploy/production.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/deploy/production.rb b/config/deploy/production.rb index 6dbccebe4..57507ff51 100644 --- a/config/deploy/production.rb +++ b/config/deploy/production.rb @@ -28,7 +28,7 @@ server 'app0.node.consul', user: 'deploy', roles: %w{app}, port: 22 server 'app1.node.consul', user: 'deploy', roles: %w{app}, port: 22 -server 'asset.node.consul', user: 'deploy', roles: %w{web}, port: 22 +#server 'asset.node.consul', user: 'deploy', roles: %w{web}, port: 22 #server 'worker.node.consul', user: 'deploy', roles: %w{worker}, port: 22 #server 'mysql.node.consul', user: 'wheelmap', roles: %w{mysql}, port: 22 From 4c78e7399f48ffc16f5e987a28635c6b6ebce6cd Mon Sep 17 00:00:00 2001 From: Felis Gilcher Date: Thu, 25 Aug 2016 05:10:23 +0000 Subject: [PATCH 07/51] change tld from consul to production --- config/deploy/production.rb | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/config/deploy/production.rb b/config/deploy/production.rb index 57507ff51..0baa42cb5 100644 --- a/config/deploy/production.rb +++ b/config/deploy/production.rb @@ -26,11 +26,11 @@ # server list. The second argument is a, or duck-types, Hash and is # used to set extended properties on the server. -server 'app0.node.consul', user: 'deploy', roles: %w{app}, port: 22 -server 'app1.node.consul', user: 'deploy', roles: %w{app}, port: 22 -#server 'asset.node.consul', user: 'deploy', roles: %w{web}, port: 22 -#server 'worker.node.consul', user: 'deploy', roles: %w{worker}, port: 22 -#server 'mysql.node.consul', user: 'wheelmap', roles: %w{mysql}, port: 22 +server 'app0.node.production', user: 'deploy', roles: %w{app}, port: 22 +server 'app1.node.production', user: 'deploy', roles: %w{app}, port: 22 +server 'asset.node.production', user: 'deploy', roles: %w{web}, port: 22 +server 'worker.node.production', user: 'deploy', roles: %w{worker}, port: 22 +#server 'mysql.node.production', user: 'wheelmap', roles: %w{mysql}, port: 22 # Custom SSH Options # ================== From cea5d66941826259012781fcc0359ecd2050e072 Mon Sep 17 00:00:00 2001 From: Felis Gilcher Date: Fri, 26 Aug 2016 04:53:50 +0000 Subject: [PATCH 08/51] precompile locally instead of on the server --- config/deploy.rb | 42 ++++++++++++++++++++++++++++++++++--- config/deploy/production.rb | 2 +- 2 files changed, 40 insertions(+), 4 deletions(-) diff --git a/config/deploy.rb b/config/deploy.rb index 0595fbb0f..adfa4e115 100644 --- a/config/deploy.rb +++ b/config/deploy.rb @@ -45,27 +45,63 @@ # Default value for keep_releases is 5 set :keep_releases, 5 + +# Clear existing task so we can replace it rather than "add" to it. +Rake::Task["deploy:compile_assets"].clear + namespace :deploy do + + desc 'Compile assets' + task :compile_assets => [:set_rails_env] do + # invoke 'deploy:assets:precompile' + invoke 'deploy:assets:precompile_local' + invoke 'deploy:assets:backup_manifest' + end + + namespace :assets do + + desc "Precompile assets locally and then rsync to web servers" + task :precompile_local do + # compile assets locally + run_locally do + execute "RAILS_ENV=#{fetch(:stage)} bundle exec rake assets:precompile" + end + + # rsync to each server + local_dir = "./public/assets/" + on roles(:asset) do + # this needs to be done outside run_locally in order for host to exist + remote_dir = "#{host.user}@#{host.hostname}:#{release_path}/public/assets/" + + run_locally { execute "rsync -av --delete #{local_dir} #{remote_dir}" } + end + + # clean up + run_locally { execute "rm -rf #{local_dir}" } + end + + end + desc 'Restart application' task :restart do on roles(:app), in: :sequence, wait: 5 do # Your restart mechanism here, for example: # execute :touch, release_path.join('tmp/restart.txt') - sudo "/etc/init.d/unicorn_#{fetch(:stage)} upgrade" + sudo "systemctl restart unicorn.service" end end desc 'Stopp application' task :stop do on roles(:app), in: :sequence, wait: 5 do - sudo "/etc/init.d/unicorn_#{fetch(:stage)} stop" + sudo "systemctl stop unicorn.service" end end desc 'Start application' task :start do on roles(:app), in: :sequence, wait: 5 do - sudo "/etc/init.d/unicorn_#{fetch(:stage)} start" + sudo "systemctl start unicorn.service" end end diff --git a/config/deploy/production.rb b/config/deploy/production.rb index 0baa42cb5..f61f9c023 100644 --- a/config/deploy/production.rb +++ b/config/deploy/production.rb @@ -28,7 +28,7 @@ server 'app0.node.production', user: 'deploy', roles: %w{app}, port: 22 server 'app1.node.production', user: 'deploy', roles: %w{app}, port: 22 -server 'asset.node.production', user: 'deploy', roles: %w{web}, port: 22 +server 'asset.node.production', user: 'deploy', roles: %w{asset}, port: 22 server 'worker.node.production', user: 'deploy', roles: %w{worker}, port: 22 #server 'mysql.node.production', user: 'wheelmap', roles: %w{mysql}, port: 22 From c2ed2e1ff2c3a0ff38010f01fea10f12cbdd54d1 Mon Sep 17 00:00:00 2001 From: Felis Gilcher Date: Mon, 29 Aug 2016 20:03:13 +0000 Subject: [PATCH 09/51] make deployment mostly work this deploys the proper parts on the proper machines. it currently does not link the required directories (uploaded fotos etc) on assets. Still broken as well: * the sudoers config on app * the sudores config on worker * the systemd unit on worker --- Capfile | 2 +- config/deploy.rb | 27 +++++++++++++++--------- lib/capistrano/tasks/delayed_job.rake | 30 ++++----------------------- 3 files changed, 22 insertions(+), 37 deletions(-) diff --git a/Capfile b/Capfile index de9a56ce4..1b5ebcf34 100644 --- a/Capfile +++ b/Capfile @@ -16,7 +16,7 @@ require 'capistrano/deploy' # #require 'capistrano/rbenv' require 'capistrano/bundler' -require 'capistrano/npm' +#require 'capistrano/npm' require 'capistrano/rails/assets' require 'capistrano/rails/migrations' require 'capistrano/deploytags' diff --git a/config/deploy.rb b/config/deploy.rb index adfa4e115..9bf9e557b 100644 --- a/config/deploy.rb +++ b/config/deploy.rb @@ -23,7 +23,15 @@ set :pty, true # Default value for :linked_files is [] -set :linked_files, %w(config/database.yml config/open_street_map.yml config/metrics.yml config/librato.yml config/newrelic.yml .env config/secrets.yml) +set :linked_files, -> { + if [:app, :worker].include?(fetch(:role)) + %w{ config/database.yml config/open_street_map.yml config/metrics.yml config/librato.yml config/newrelic.yml .env config/secrets.yml} + else + [] + end +} + +set :bundle_roles, [:app, :worker] # Default value for :bundle_without is %w{development test}.join(' ') set :bundle_without, %w(development test metrics deployment).join(' ') @@ -33,14 +41,13 @@ # Default value for linked_dirs is [] # set :linked_dirs, %w{bin log tmp/pids tmp/cache tmp/sockets vendor/bundle public/system} -set :linked_dirs, %w(log tmp/var tmp/osmosis-working-dir tmp/cache tmp/sockets tmp/pids vendor/bundle public/system public/assets node_modules) - -set :rbenv_type, :system # :user or :system, depends on your rbenv setup -set :rbenv_ruby, '2.2.2' -set :rbenv_custom_path, '/opt/rbenv' - -# Default value for default_env is {} -# set :default_env, { path: "/opt/ruby/bin:$PATH" } +set :linked_dirs, -> { + if [:app, :worker].include?(fetch(:role)) + %w{ log tmp/var tmp/osmosis-working-dir tmp/cache tmp/sockets tmp/pids vendor/bundle public/system public/assets node_modules } + else + [] + end +} # Default value for keep_releases is 5 set :keep_releases, 5 @@ -67,7 +74,7 @@ execute "RAILS_ENV=#{fetch(:stage)} bundle exec rake assets:precompile" end - # rsync to each server + # rsync to asset server local_dir = "./public/assets/" on roles(:asset) do # this needs to be done outside run_locally in order for host to exist diff --git a/lib/capistrano/tasks/delayed_job.rake b/lib/capistrano/tasks/delayed_job.rake index 29255d853..35214858b 100644 --- a/lib/capistrano/tasks/delayed_job.rake +++ b/lib/capistrano/tasks/delayed_job.rake @@ -4,49 +4,27 @@ namespace :delayed_job do end def delayed_job_roles - fetch(:delayed_job_server_role, :app) + fetch(:delayed_job_server_role, :worker) end desc 'Stop the delayed_job process' task :stop do on roles(delayed_job_roles) do - within release_path do - with rails_env: fetch(:rails_env) do - begin - sudo '/usr/bin/monit unmonitor delayed_job' - rescue - nil - end - execute :bundle, :exec, :'script/delayed_job', :stop - end - end + sudo "systemctl stop worker.service" end end desc 'Start the delayed_job process' task :start do on roles(delayed_job_roles) do - within release_path do - with rails_env: fetch(:rails_env) do - execute :bundle, :exec, :'script/delayed_job', args, :start - begin - sudo '/usr/bin/monit monitor delayed_job' - rescue - nil - end - end - end + sudo "systemctl start worker.service" end end desc 'Restart the delayed_job process' task :restart do on roles(delayed_job_roles) do - within release_path do - with rails_env: fetch(:rails_env) do - execute :bundle, :exec, :'bin/delayed_job', args, :restart - end - end + sudo "systemctl restart worker.service" end end end From e973fa9241f838f044d85996911649a2814440f1 Mon Sep 17 00:00:00 2001 From: Felis Gilcher Date: Mon, 29 Aug 2016 21:30:30 +0000 Subject: [PATCH 10/51] unicorn -> webapp --- config/deploy.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/config/deploy.rb b/config/deploy.rb index 9bf9e557b..5d5f3616a 100644 --- a/config/deploy.rb +++ b/config/deploy.rb @@ -94,21 +94,21 @@ on roles(:app), in: :sequence, wait: 5 do # Your restart mechanism here, for example: # execute :touch, release_path.join('tmp/restart.txt') - sudo "systemctl restart unicorn.service" + sudo "systemctl restart webapp.service" end end desc 'Stopp application' task :stop do on roles(:app), in: :sequence, wait: 5 do - sudo "systemctl stop unicorn.service" + sudo "systemctl stop webapp.service" end end desc 'Start application' task :start do on roles(:app), in: :sequence, wait: 5 do - sudo "systemctl start unicorn.service" + sudo "systemctl start webapp.service" end end From bb23675650668a4678810c62a3afe67b69a618c1 Mon Sep 17 00:00:00 2001 From: Felis Gilcher Date: Tue, 30 Aug 2016 05:00:07 +0000 Subject: [PATCH 11/51] fix unicorn config * fix application path * log stdout/stderr to stdout/stderr --- config/unicorn.rb | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/config/unicorn.rb b/config/unicorn.rb index 8ff5d30ee..e90c9bb39 100644 --- a/config/unicorn.rb +++ b/config/unicorn.rb @@ -2,9 +2,9 @@ # Set environment to development unless something else is specified env = ENV['RAILS_ENV'] || 'staging' -worker_processes env == 'production' ? 9 : 1 -base_dir = "/var/apps/wheelmap/#{env}/current" -shared_path = "/var/apps/wheelmap/#{env}/shared" +worker_processes 4 +base_dir = "/var/apps/wheelmap/current" +shared_path = "/var/apps/wheelmap/shared" working_directory base_dir # This loads the application in the master process before forking @@ -23,15 +23,13 @@ timeout env == 'production' ? 300 : 30 -# This is where we specify the socket. -# We will point the upstream Nginx module to this socket later on listen 3000, :backlog => 1024 -pid "#{shared_path}/pids/unicorn.pid" +pid = "#{shared_path}/pids/unicorn.pid" -# Set the path of the log files inside the log folder of the testapp -stdout_path "#{shared_path}/log/unicorn.stdout.log" -stderr_path "#{shared_path}/log/unicorn.stderr.log" +## Set the path of the log files inside the log folder of the testapp +#stdout_path "#{shared_path}/log/unicorn.stdout.log" +#stderr_path "#{shared_path}/log/unicorn.stderr.log" # The new check_client_connection option allows unicorn to detect # most disconnected local clients before potentially expensive From 96f261989a5b399f5d8464c973ef36912751370f Mon Sep 17 00:00:00 2001 From: Felis Gilcher Date: Tue, 30 Aug 2016 05:48:02 +0000 Subject: [PATCH 12/51] bump capistrano version to 3.6.1 --- Gemfile.lock | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index 02665eea6..73ca10c8e 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -79,6 +79,8 @@ GEM airbrake (4.3.6) builder multi_json + airbrussh (1.1.0) + sshkit (>= 1.6.1, != 1.7.0) apipie-rails (0.3.6) json arbre (1.0.3) @@ -112,15 +114,18 @@ GEM cacheable_flash (0.3.4) json stackable_flash (>= 0.0.7) - capistrano (3.4.0) + capistrano (3.6.1) + airbrussh (>= 1.0.0) + capistrano-harrow i18n rake (>= 10.0.0) - sshkit (~> 1.3) + sshkit (>= 1.9.0) capistrano-bundler (1.1.4) capistrano (~> 3.1) sshkit (~> 1.2) capistrano-deploytags (1.0.4) capistrano (>= 3.2.0) + capistrano-harrow (0.5.3) capistrano-npm (1.0.1) capistrano (>= 3.0.0) capistrano-rails (1.1.6) @@ -342,7 +347,7 @@ GEM nenv (0.3.0) net-scp (1.2.1) net-ssh (>= 2.6.5) - net-ssh (3.0.2) + net-ssh (3.2.0) newrelic_rpm (3.15.0.314) nokogiri (1.6.7.2) mini_portile2 (~> 2.0.0.rc2) @@ -504,7 +509,7 @@ GEM actionpack (>= 3.0) activesupport (>= 3.0) sprockets (>= 2.8, < 4.0) - sshkit (1.9.0) + sshkit (1.11.2) net-scp (>= 1.1.2) net-ssh (>= 2.8.0) stackable_flash (0.0.7) From 1bb9f888fa3f229b743ffe42047753e9df0c2e47 Mon Sep 17 00:00:00 2001 From: Felis Gilcher Date: Tue, 30 Aug 2016 05:48:31 +0000 Subject: [PATCH 13/51] add bundle config --- config/deploy.rb | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/config/deploy.rb b/config/deploy.rb index 5d5f3616a..f56cfaa7f 100644 --- a/config/deploy.rb +++ b/config/deploy.rb @@ -32,12 +32,9 @@ } set :bundle_roles, [:app, :worker] - -# Default value for :bundle_without is %w{development test}.join(' ') -set :bundle_without, %w(development test metrics deployment).join(' ') - -# Default value for :bundle_jobs is: nil +set :bundle_without, %w{ development test metrics deployment }.join(' ') set :bundle_jobs, 4 +set :bundle_binstubs, -> { shared_path.join('bin') } # Default value for linked_dirs is [] # set :linked_dirs, %w{bin log tmp/pids tmp/cache tmp/sockets vendor/bundle public/system} @@ -141,6 +138,6 @@ end end - after :finished, 'airbrake:deploy' - after :finished, 'newrelic:notice_deployment' +# after :finished, 'airbrake:deploy' +# after :finished, 'newrelic:notice_deployment' end From 8790aeaeada3ef1672da2435940b19635376fb45 Mon Sep 17 00:00:00 2001 From: Felis Gilcher Date: Tue, 30 Aug 2016 06:07:44 +0000 Subject: [PATCH 14/51] fix various deployement-related issues --- config/deploy.rb | 18 +++--------------- config/deploy/production.rb | 2 +- config/unicorn.rb | 2 +- 3 files changed, 5 insertions(+), 17 deletions(-) diff --git a/config/deploy.rb b/config/deploy.rb index f56cfaa7f..55122c81d 100644 --- a/config/deploy.rb +++ b/config/deploy.rb @@ -23,13 +23,7 @@ set :pty, true # Default value for :linked_files is [] -set :linked_files, -> { - if [:app, :worker].include?(fetch(:role)) - %w{ config/database.yml config/open_street_map.yml config/metrics.yml config/librato.yml config/newrelic.yml .env config/secrets.yml} - else - [] - end -} +set :linked_files, %w{ config/database.yml config/open_street_map.yml config/metrics.yml config/librato.yml config/newrelic.yml .env config/secrets.yml} set :bundle_roles, [:app, :worker] set :bundle_without, %w{ development test metrics deployment }.join(' ') @@ -38,13 +32,7 @@ # Default value for linked_dirs is [] # set :linked_dirs, %w{bin log tmp/pids tmp/cache tmp/sockets vendor/bundle public/system} -set :linked_dirs, -> { - if [:app, :worker].include?(fetch(:role)) - %w{ log tmp/var tmp/osmosis-working-dir tmp/cache tmp/sockets tmp/pids vendor/bundle public/system public/assets node_modules } - else - [] - end -} +set :linked_dirs, %w{ log tmp/var tmp/osmosis-working-dir tmp/cache tmp/sockets tmp/pids vendor/bundle public/system public/assets node_modules } # Default value for keep_releases is 5 set :keep_releases, 5 @@ -76,7 +64,7 @@ on roles(:asset) do # this needs to be done outside run_locally in order for host to exist remote_dir = "#{host.user}@#{host.hostname}:#{release_path}/public/assets/" - + execute "mkdir -p #{release_path}/public/assets/" run_locally { execute "rsync -av --delete #{local_dir} #{remote_dir}" } end diff --git a/config/deploy/production.rb b/config/deploy/production.rb index f61f9c023..0a8bfde94 100644 --- a/config/deploy/production.rb +++ b/config/deploy/production.rb @@ -28,7 +28,7 @@ server 'app0.node.production', user: 'deploy', roles: %w{app}, port: 22 server 'app1.node.production', user: 'deploy', roles: %w{app}, port: 22 -server 'asset.node.production', user: 'deploy', roles: %w{asset}, port: 22 +server 'asset.node.production', user: 'deploy', roles: %w{asset}, port: 22, :no_release => true server 'worker.node.production', user: 'deploy', roles: %w{worker}, port: 22 #server 'mysql.node.production', user: 'wheelmap', roles: %w{mysql}, port: 22 diff --git a/config/unicorn.rb b/config/unicorn.rb index e90c9bb39..2f5b630c7 100644 --- a/config/unicorn.rb +++ b/config/unicorn.rb @@ -25,7 +25,7 @@ listen 3000, :backlog => 1024 -pid = "#{shared_path}/pids/unicorn.pid" +pid "#{shared_path}/pids/unicorn.pid" ## Set the path of the log files inside the log folder of the testapp #stdout_path "#{shared_path}/log/unicorn.stdout.log" From 718d69f7c2e34389b1ffe00ca575ba52f7efd6f4 Mon Sep 17 00:00:00 2001 From: Andrew Hobden Date: Wed, 31 Aug 2016 14:10:33 +0200 Subject: [PATCH 15/51] Use tmp as shared instead of specific subfolders --- config/deploy.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/deploy.rb b/config/deploy.rb index 55122c81d..81bfead3b 100644 --- a/config/deploy.rb +++ b/config/deploy.rb @@ -32,7 +32,7 @@ # Default value for linked_dirs is [] # set :linked_dirs, %w{bin log tmp/pids tmp/cache tmp/sockets vendor/bundle public/system} -set :linked_dirs, %w{ log tmp/var tmp/osmosis-working-dir tmp/cache tmp/sockets tmp/pids vendor/bundle public/system public/assets node_modules } +set :linked_dirs, %w{ log tmp vendor/bundle public/system public/assets node_modules } # Default value for keep_releases is 5 set :keep_releases, 5 From 1f03658f72b68d263bbb1ee7d746a1e2a2a6b2c6 Mon Sep 17 00:00:00 2001 From: Felis Gilcher Date: Tue, 30 Aug 2016 18:16:05 +0000 Subject: [PATCH 16/51] symlink assets after deploy --- config/deploy.rb | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/config/deploy.rb b/config/deploy.rb index 81bfead3b..e8fdf0dd2 100644 --- a/config/deploy.rb +++ b/config/deploy.rb @@ -66,6 +66,12 @@ remote_dir = "#{host.user}@#{host.hostname}:#{release_path}/public/assets/" execute "mkdir -p #{release_path}/public/assets/" run_locally { execute "rsync -av --delete #{local_dir} #{remote_dir}" } + + # stolen from https://github.com/capistrano/capistrano/blob/master/lib/capistrano/tasks/deploy.rake#L101 + # we can't directly invoke the task since that runs on deploy roles only + tmp_current_path = release_path.parent.join(current_path.basename) + execute :ln, "-s", release_path, tmp_current_path + execute :mv, tmp_current_path, current_path.parent end # clean up From bd8b0301375fe7879268dfced18c1f7bb673bda8 Mon Sep 17 00:00:00 2001 From: Andrew Hobden Date: Thu, 1 Sep 2016 09:17:24 +0200 Subject: [PATCH 17/51] Update prod branch to the main infra-rebuild since the merge happened --- config/deploy/production.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/deploy/production.rb b/config/deploy/production.rb index 0a8bfde94..ec141a72d 100644 --- a/config/deploy/production.rb +++ b/config/deploy/production.rb @@ -6,7 +6,7 @@ set :stage, :production set :deploy_to, "/var/apps/#{fetch(:application)}/" -set :branch, :"feature/infrastructure-rebuild/deployment" +set :branch, :"feature/infrastructure-rebuild" set :rev, proc { `git rev-parse --short #{fetch(:branch)}`.chomp } # Simple Role Syntax From de8977cafcb04a27949480aee631aeed5945d85e Mon Sep 17 00:00:00 2001 From: Andrew Hobden Date: Thu, 1 Sep 2016 09:17:53 +0200 Subject: [PATCH 18/51] Reflect changes to deploy.rb in staging.rb --- config/deploy/staging.rb | 37 +++++++++++++++++++------------------ 1 file changed, 19 insertions(+), 18 deletions(-) diff --git a/config/deploy/staging.rb b/config/deploy/staging.rb index 25f63d6dc..ae22bd8ee 100644 --- a/config/deploy/staging.rb +++ b/config/deploy/staging.rb @@ -3,32 +3,33 @@ set :delayed_job_args, '-p wheelmap_staging' set :rails_env, 'staging' # added for delayed job -set :stage, :staging -set :deploy_to, "/var/apps/#{fetch(:application)}/#{fetch(:stage)}" +set :stage, :production +set :deploy_to, "/var/apps/#{fetch(:application)}/" -set :branch, ENV['BRANCH'] || 'master' +set :branch, :"feature/infrastructure-rebuild" set :rev, proc { `git rev-parse --short #{fetch(:branch)}`.chomp } -set :whenever_identifier, -> { "#{fetch(:application)}_#{fetch(:stage)}" } - # Simple Role Syntax # ================== # Supports bulk-adding hosts to roles, the primary server in each group # is considered to be the first unless any hosts have the primary # property set. Don't declare `role :all`, it's a meta role. -role :app, %w(176.9.63.171) -role :web, %w(176.9.63.171) -role :db, %w(176.9.63.171) -role :worker, %w(176.9.63.171) +#role :app, %w{176.9.63.170} +#role :web, %w{176.9.63.170} +#role :db, %w{176.9.63.170} +#role :worker, %w{176.9.63.170} # Extended Server Syntax # ====================== # This can be used to drop a more detailed server definition into the # server list. The second argument is a, or duck-types, Hash and is # used to set extended properties on the server. - -server '176.9.63.171', user: 'rails', roles: %w(web app), my_property: :my_value +server 'app0.node.staging', user: 'deploy', roles: %w{app}, port: 22 +server 'app1.node.staging', user: 'deploy', roles: %w{app}, port: 22 +server 'asset.node.staging', user: 'deploy', roles: %w{asset}, port: 22, :no_release => true +server 'worker.node.staging', user: 'deploy', roles: %w{worker}, port: 22 +#server 'mysql.node.production', user: 'wheelmap', roles: %w{mysql}, port: 22 # Custom SSH Options # ================== @@ -37,13 +38,13 @@ # # Global options # -------------- -set :ssh_options, - keys: %w(~/.ssh/wheelmap_rsa), - forward_agent: true, - config: true, - port: 22_022 -# auth_methods: %w(password) - +#set :ssh_options, { +# keys: %w(~/.ssh/wheelmap_rsa), +# forward_agent: true, +# config: true, +# port: 22022 +# # auth_methods: %w(password) +#} # # And/or per server (overrides global) # ------------------------------------ From 894fc2c24d32c596fa6a27a12d62f79b3e66b9a1 Mon Sep 17 00:00:00 2001 From: Andrew Hobden Date: Thu, 1 Sep 2016 09:24:20 +0200 Subject: [PATCH 19/51] Forgot stage... --- config/deploy/staging.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/deploy/staging.rb b/config/deploy/staging.rb index ae22bd8ee..af050d723 100644 --- a/config/deploy/staging.rb +++ b/config/deploy/staging.rb @@ -3,7 +3,7 @@ set :delayed_job_args, '-p wheelmap_staging' set :rails_env, 'staging' # added for delayed job -set :stage, :production +set :stage, :staging set :deploy_to, "/var/apps/#{fetch(:application)}/" set :branch, :"feature/infrastructure-rebuild" From 1bc3d8ea5a2c7e8aa3d15f6c8e82b686950cfda7 Mon Sep 17 00:00:00 2001 From: Andrew Hobden Date: Thu, 1 Sep 2016 10:10:46 +0200 Subject: [PATCH 20/51] Don't keep dead code --- config/deploy/staging.rb | 4 ---- 1 file changed, 4 deletions(-) diff --git a/config/deploy/staging.rb b/config/deploy/staging.rb index af050d723..1d933c603 100644 --- a/config/deploy/staging.rb +++ b/config/deploy/staging.rb @@ -15,10 +15,6 @@ # is considered to be the first unless any hosts have the primary # property set. Don't declare `role :all`, it's a meta role. -#role :app, %w{176.9.63.170} -#role :web, %w{176.9.63.170} -#role :db, %w{176.9.63.170} -#role :worker, %w{176.9.63.170} # Extended Server Syntax # ====================== From 7711197a38df9ec3bfedea2d1d9965d4a43da717 Mon Sep 17 00:00:00 2001 From: Andrew Hobden Date: Mon, 5 Sep 2016 10:26:39 +0200 Subject: [PATCH 21/51] Fix branch for staging deployments. --- config/deploy/staging.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/deploy/staging.rb b/config/deploy/staging.rb index 1d933c603..9681b608d 100644 --- a/config/deploy/staging.rb +++ b/config/deploy/staging.rb @@ -6,7 +6,7 @@ set :stage, :staging set :deploy_to, "/var/apps/#{fetch(:application)}/" -set :branch, :"feature/infrastructure-rebuild" +set :branch, :"feature/infra-rebuild" set :rev, proc { `git rev-parse --short #{fetch(:branch)}`.chomp } # Simple Role Syntax From 16dbf955954a851a220b6d90fd019f58e3469e0d Mon Sep 17 00:00:00 2001 From: Andrew Hobden Date: Wed, 7 Sep 2016 09:54:09 +0000 Subject: [PATCH 22/51] Use asset manifest --- .../stylesheets/relaunch/_container.scss | 8 +++---- app/assets/stylesheets/relaunch/_fonts.scss | 4 ++-- app/assets/stylesheets/relaunch/_footer.scss | 2 +- app/assets/stylesheets/relaunch/_header.scss | 12 +++++------ app/assets/stylesheets/relaunch/_icons.scss | 6 +++--- .../relaunch/_leaflet_overrides.scss | 12 +++++------ app/assets/stylesheets/relaunch/_map.scss | 4 ++-- app/assets/stylesheets/relaunch/_overlay.scss | 6 +++--- app/assets/stylesheets/relaunch/_toolbar.scss | 21 +++++++++---------- 9 files changed, 37 insertions(+), 38 deletions(-) diff --git a/app/assets/stylesheets/relaunch/_container.scss b/app/assets/stylesheets/relaunch/_container.scss index 1b14bebdc..a18abdd10 100644 --- a/app/assets/stylesheets/relaunch/_container.scss +++ b/app/assets/stylesheets/relaunch/_container.scss @@ -1,5 +1,5 @@ #container { - background: url('container_bg.png') repeat-x 0 $headerHeight; + background: asset-url('container_bg.png') repeat-x 0 $headerHeight; display: block; padding-bottom: $footerHeight + 10px; padding-top: $headerHeight; @@ -45,10 +45,10 @@ padding-left: 80px; } p.osm { - background: url("/assets/openstreetmap_logo_small.jpg") no-repeat left center; + background: asset-url("/assets/openstreetmap_logo_small.jpg") no-repeat left center; } p.wheelmap { - background: url("/assets/markerhead.jpg") no-repeat 4px center; + background: asset-url("/assets/markerhead.jpg") no-repeat 4px center; } em { @@ -113,4 +113,4 @@ text-shadow: rgba(255, 255, 255, 0.5), 0, 1px, 1px; } } -} \ No newline at end of file +} diff --git a/app/assets/stylesheets/relaunch/_fonts.scss b/app/assets/stylesheets/relaunch/_fonts.scss index 172dc4ed8..f0172c61a 100644 --- a/app/assets/stylesheets/relaunch/_fonts.scss +++ b/app/assets/stylesheets/relaunch/_fonts.scss @@ -3,6 +3,6 @@ $helvetica_light: 'HelveticaNeue-Light','Helvetica Neue Light','Helvetica Neue', @font-face { font-family: 'EntypoRegular'; - src: url('entypo.eot'); - src: url('entypo.eot?#iefix') format("embedded-opentype"), url('entypo.woff') format("woff"), url('entypo.ttf') format("truetype"), url('entypo.svg#EntypoRegular') format("svg"); + src: asset-url('entypo.eot'); + src: asset-url('entypo.eot?#iefix') format("embedded-opentype"), asset-url('entypo.woff') format("woff"), asset-url('entypo.ttf') format("truetype"), asset-url('entypo.svg#EntypoRegular') format("svg"); } diff --git a/app/assets/stylesheets/relaunch/_footer.scss b/app/assets/stylesheets/relaunch/_footer.scss index bba595422..e0c9e80a1 100644 --- a/app/assets/stylesheets/relaunch/_footer.scss +++ b/app/assets/stylesheets/relaunch/_footer.scss @@ -64,7 +64,7 @@ border-left: 0; padding: 1px 4px; margin-left: 5px; - background: url(/assets/bubble-left.png) no-repeat -5px center; + background: asset-url('/assets/bubble-left.png') no-repeat -5px center; } } diff --git a/app/assets/stylesheets/relaunch/_header.scss b/app/assets/stylesheets/relaunch/_header.scss index 46f7bab87..9b5e7744c 100644 --- a/app/assets/stylesheets/relaunch/_header.scss +++ b/app/assets/stylesheets/relaunch/_header.scss @@ -8,10 +8,10 @@ left: 0; z-index: 2; float: left; - background: url('bg_header.png'); + background: asset-url('bg_header.png'); @include if-min-resolution(1.3) { - background: url(bg_header@2x.png); + background: asset-url('bg_header@2x.png'); } .navbar-inner { @@ -31,10 +31,10 @@ margin: 7px 0 0 0; float: left; position: relative; - background: url(bg_logo.png) center left no-repeat; + background: asset-url('bg_logo.png') center left no-repeat; @include if-min-resolution(1.3) { - background: url(bg_logo@2x.png) center left no-repeat; + background: asset-url('bg_logo@2x.png') center left no-repeat; background-size: 150px 32px; } @@ -61,11 +61,11 @@ .christmas & { width: 160px; height: 42px; - background-image: url(bg_logo_christmas.png); + background-image: asset-url('bg_logo_christmas.png'); margin-top: 0; @include if-min-resolution(1.3) { - background-image: url(bg_logo_christmas@2x.png); + background-image: asset-url('bg_logo_christmas@2x.png'); background-size: 150px 39px; } diff --git a/app/assets/stylesheets/relaunch/_icons.scss b/app/assets/stylesheets/relaunch/_icons.scss index 0571f50c8..ad106597d 100644 --- a/app/assets/stylesheets/relaunch/_icons.scss +++ b/app/assets/stylesheets/relaunch/_icons.scss @@ -8,14 +8,14 @@ } .leaflet-marker-icon { - background: transparent url('marker-sprites.png') no-repeat; + background: transparent asset-url('marker-sprites.png') no-repeat; width: 29px; height: 32px; margin-left: -15px; margin-top: -30px; @include if-min-resolution(1.3) { - background: transparent url('marker-sprites@2x.png') no-repeat; + background: transparent asset-url('marker-sprites@2x.png') no-repeat; background-size: 29px 128px; } @@ -58,4 +58,4 @@ $icons: sprite-map("icons/*[^@2x].png"); max-width: 22px; max-height: 22px; } -} \ No newline at end of file +} diff --git a/app/assets/stylesheets/relaunch/_leaflet_overrides.scss b/app/assets/stylesheets/relaunch/_leaflet_overrides.scss index 9e1028328..6e1c2c409 100644 --- a/app/assets/stylesheets/relaunch/_leaflet_overrides.scss +++ b/app/assets/stylesheets/relaunch/_leaflet_overrides.scss @@ -18,7 +18,7 @@ .leaflet-control-locate { a { - background-image: url('locate_btn.png'); + background-image: asset-url('locate_btn.png'); background-position: center center; background-repeat: no-repeat; background-color: rgba(255,255,255,0.8); @@ -28,18 +28,18 @@ }; @include if-min-resolution(1.3) { - background-image: url('locate_btn@2x.png'); + background-image: asset-url('locate_btn@2x.png'); } } &.requesting a { - background-image: url('spinner_locate.gif'); + background-image: asset-url('spinner_locate.gif'); } &.active a { - background-image: url(locate_btn_active.png); + background-image: asset-url('locate_btn_active.png'); @include if-min-resolution(1.3) { - background-image: url('locate_btn_active@2x.png'); + background-image: asset-url('locate_btn_active@2x.png'); } } } @@ -65,4 +65,4 @@ a { white-space: nowrap; } -} \ No newline at end of file +} diff --git a/app/assets/stylesheets/relaunch/_map.scss b/app/assets/stylesheets/relaunch/_map.scss index a96b3b0db..730468951 100644 --- a/app/assets/stylesheets/relaunch/_map.scss +++ b/app/assets/stylesheets/relaunch/_map.scss @@ -36,9 +36,9 @@ text-indent:-9999px; overflow:hidden; background-repeat:no-repeat; - background-image:url('mapbox_logo.png'); + background-image: asset-url('mapbox_logo.png'); @media screen and (max-width:$breakPhone) { bottom: 13px } -} \ No newline at end of file +} diff --git a/app/assets/stylesheets/relaunch/_overlay.scss b/app/assets/stylesheets/relaunch/_overlay.scss index 4d815a62d..ed6a3192e 100644 --- a/app/assets/stylesheets/relaunch/_overlay.scss +++ b/app/assets/stylesheets/relaunch/_overlay.scss @@ -34,7 +34,7 @@ top: 50%; margin: -325px 0 0 -350px; padding: 0 0 20px 0; - background: url('bg-overlay.png') #FFF no-repeat; + background: asset-url('bg-overlay.png') #FFF no-repeat; background-position: 0 219px; @include box-shadow(0 0 10px #333); @include border-radius(3px); @@ -85,7 +85,7 @@ margin: 0 0 40px 0; text-align: center; - background: url('logo-overlay.png') center top no-repeat; + background: asset-url('logo-overlay.png') center top no-repeat; font-size: 16px; line-height: 24px; @@ -290,4 +290,4 @@ margin-top: 5px; } } -} \ No newline at end of file +} diff --git a/app/assets/stylesheets/relaunch/_toolbar.scss b/app/assets/stylesheets/relaunch/_toolbar.scss index e2c4e51a7..955968cc4 100644 --- a/app/assets/stylesheets/relaunch/_toolbar.scss +++ b/app/assets/stylesheets/relaunch/_toolbar.scss @@ -133,7 +133,7 @@ } &--yes { - background-image: url('toolbar_marker_yes.png'); + background-image: asset-url('toolbar_marker_yes.png'); .popover-title { background-color: $wheelchair-yes; @@ -141,7 +141,7 @@ } &--limited { - background-image: url('toolbar_marker_limited.png'); + background-image: asset-url('toolbar_marker_limited.png'); .popover-title { background-color: $wheelchair-limited; @@ -149,7 +149,7 @@ } &--no { - background-image: url('toolbar_marker_no.png'); + background-image: asset-url('toolbar_marker_no.png'); .popover-title { background-color: $wheelchair-limited; @@ -157,7 +157,7 @@ } &--unknown { - background-image: url('toolbar_marker_unknown.png'); + background-image: asset-url('toolbar_marker_unknown.png'); .popover-title { background-color: $wheelchair-unknown; @@ -166,19 +166,19 @@ @include if-min-resolution(1.3) { &--yes { - background-image: url(toolbar_marker_yes@2x.png); + background-image: asset-url('toolbar_marker_yes@2x.png'); } &--limited { - background-image: url(toolbar_marker_limited@2x.png); + background-image: asset-url('toolbar_marker_limited@2x.png'); } &--no { - background-image: url(toolbar_marker_no@2x.png); + background-image: asset-url('toolbar_marker_no@2x.png'); } &--unknown { - background-image: url(toolbar_marker_unknown@2x.png); + background-image: asset-url('toolbar_marker_unknown@2x.png'); } } } @@ -193,7 +193,7 @@ } &-icon { - background: url('toolbar_marker_icon.png') no-repeat; + background: asset-url('toolbar_marker_icon.png') no-repeat; background-size: 100%; width: image_width('toolbar_marker_icon.png'); height: image_height('toolbar_marker_icon.png'); @@ -201,7 +201,7 @@ margin-top: 1px; // Cosmetic surgery @include if-min-resolution(1.3) { - background-image: url('toolbar_marker_icon@2x.png'); + background-image: asset-url('toolbar_marker_icon@2x.png'); } } @@ -616,4 +616,3 @@ } } } - From 3aafeeb29c507aded85c20b28b622c49d4ce2cfe Mon Sep 17 00:00:00 2001 From: Andrew Hobden Date: Tue, 6 Sep 2016 14:04:01 +0200 Subject: [PATCH 23/51] Copy manifest to apps --- config/deploy.rb | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/config/deploy.rb b/config/deploy.rb index e8fdf0dd2..4c4c8dc03 100644 --- a/config/deploy.rb +++ b/config/deploy.rb @@ -74,6 +74,19 @@ execute :mv, tmp_current_path, current_path.parent end + on roles(:app) do + # this needs to be done outside run_locally in order for host to exist + remote_dir = "#{host.user}@#{host.hostname}:#{release_path}/public/assets/" + execute "mkdir -p #{release_path}/public/assets/" + run_locally { execute "rsync -av --delete #{local_dir}/.sprockets-manifest-* #{remote_dir}" } + + # stolen from https://github.com/capistrano/capistrano/blob/master/lib/capistrano/tasks/deploy.rake#L101 + # we can't directly invoke the task since that runs on deploy roles only + tmp_current_path = release_path.parent.join(current_path.basename) + execute :ln, "-s", release_path, tmp_current_path + execute :mv, tmp_current_path, current_path.parent + end + # clean up run_locally { execute "rm -rf #{local_dir}" } end From a9ed9ff3b3058b11bdffa71bb69072b6062e3ab2 Mon Sep 17 00:00:00 2001 From: Andrew Hobden Date: Tue, 6 Sep 2016 14:20:09 +0200 Subject: [PATCH 24/51] App is a deploy role, don't need this code block --- config/deploy.rb | 6 ------ 1 file changed, 6 deletions(-) diff --git a/config/deploy.rb b/config/deploy.rb index 4c4c8dc03..6e3fe559f 100644 --- a/config/deploy.rb +++ b/config/deploy.rb @@ -79,12 +79,6 @@ remote_dir = "#{host.user}@#{host.hostname}:#{release_path}/public/assets/" execute "mkdir -p #{release_path}/public/assets/" run_locally { execute "rsync -av --delete #{local_dir}/.sprockets-manifest-* #{remote_dir}" } - - # stolen from https://github.com/capistrano/capistrano/blob/master/lib/capistrano/tasks/deploy.rake#L101 - # we can't directly invoke the task since that runs on deploy roles only - tmp_current_path = release_path.parent.join(current_path.basename) - execute :ln, "-s", release_path, tmp_current_path - execute :mv, tmp_current_path, current_path.parent end # clean up From b62730610f04dc453fb945fd3c37d56873b2921b Mon Sep 17 00:00:00 2001 From: Andrew Hobden Date: Tue, 6 Sep 2016 14:29:20 +0200 Subject: [PATCH 25/51] Use asset%d.wheelmap.org instead of staging.wheelmap.org --- config/environments/staging.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/environments/staging.rb b/config/environments/staging.rb index 97cf8864e..02f89fe73 100644 --- a/config/environments/staging.rb +++ b/config/environments/staging.rb @@ -56,7 +56,7 @@ config.serve_static_assets = false # Enable serving of images, stylesheets, and javascripts from an asset server - config.action_controller.asset_host = 'staging.wheelmap.org' + config.action_controller.asset_host = 'asset%d.wheelmap.org' # Disable delivery errors, bad email addresses will be ignored # config.action_mailer.raise_delivery_errors = false From 4446f09cb6c9fba7be81830a82049a395fc05ef6 Mon Sep 17 00:00:00 2001 From: Andrew Hobden Date: Tue, 6 Sep 2016 14:54:40 +0200 Subject: [PATCH 26/51] Reflect URL changes as discussed --- config/environments/staging.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/environments/staging.rb b/config/environments/staging.rb index 02f89fe73..1363158ac 100644 --- a/config/environments/staging.rb +++ b/config/environments/staging.rb @@ -56,7 +56,7 @@ config.serve_static_assets = false # Enable serving of images, stylesheets, and javascripts from an asset server - config.action_controller.asset_host = 'asset%d.wheelmap.org' + config.action_controller.asset_host = 'asset%d.staging.wheelmap.org' # Disable delivery errors, bad email addresses will be ignored # config.action_mailer.raise_delivery_errors = false From fe8b796f6f6912e9456792c966ce9b48b25f62f5 Mon Sep 17 00:00:00 2001 From: Andrew Hobden Date: Wed, 7 Sep 2016 09:57:16 +0000 Subject: [PATCH 27/51] Test cap (Do not merge this) --- config/deploy/staging.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/deploy/staging.rb b/config/deploy/staging.rb index 9681b608d..4c63dd776 100644 --- a/config/deploy/staging.rb +++ b/config/deploy/staging.rb @@ -6,7 +6,7 @@ set :stage, :staging set :deploy_to, "/var/apps/#{fetch(:application)}/" -set :branch, :"feature/infra-rebuild" +set :branch, :"bugfix/apps-use-asset-manifest" set :rev, proc { `git rev-parse --short #{fetch(:branch)}`.chomp } # Simple Role Syntax From 0e367a23287480d97af31a61dbaa78bf2c1ed758 Mon Sep 17 00:00:00 2001 From: Andrew Hobden Date: Wed, 7 Sep 2016 10:24:03 +0000 Subject: [PATCH 28/51] public assets should not be shared between deploys. --- config/deploy.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/deploy.rb b/config/deploy.rb index 6e3fe559f..b1e4c681f 100644 --- a/config/deploy.rb +++ b/config/deploy.rb @@ -32,7 +32,7 @@ # Default value for linked_dirs is [] # set :linked_dirs, %w{bin log tmp/pids tmp/cache tmp/sockets vendor/bundle public/system} -set :linked_dirs, %w{ log tmp vendor/bundle public/system public/assets node_modules } +set :linked_dirs, %w{ log tmp vendor/bundle public/system node_modules } # Default value for keep_releases is 5 set :keep_releases, 5 From b023075965bee1f262b3dbd003ff531c2a1bf4e9 Mon Sep 17 00:00:00 2001 From: Andrew Hobden Date: Wed, 7 Sep 2016 13:39:41 +0000 Subject: [PATCH 29/51] noop :set_linked_dirs --- config/deploy.rb | 3 +++ 1 file changed, 3 insertions(+) diff --git a/config/deploy.rb b/config/deploy.rb index b1e4c681f..6b0437a99 100644 --- a/config/deploy.rb +++ b/config/deploy.rb @@ -43,6 +43,9 @@ namespace :deploy do + desc 'Overwrite defailt :set_linked_dirs so it does not make a bad symlink' + task :set_linked_dirs => do true end + desc 'Compile assets' task :compile_assets => [:set_rails_env] do # invoke 'deploy:assets:precompile' From 079f1a793be1749c9c21e000b1b96c57753a27dc Mon Sep 17 00:00:00 2001 From: Andrew Hobden Date: Wed, 7 Sep 2016 13:41:50 +0000 Subject: [PATCH 30/51] Properly noop :set_linked_dirs --- config/deploy.rb | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/config/deploy.rb b/config/deploy.rb index 6b0437a99..f7c4db9ee 100644 --- a/config/deploy.rb +++ b/config/deploy.rb @@ -40,11 +40,12 @@ # Clear existing task so we can replace it rather than "add" to it. Rake::Task["deploy:compile_assets"].clear +Rake::Task["deploy:set_linked_dirs"].clear namespace :deploy do - - desc 'Overwrite defailt :set_linked_dirs so it does not make a bad symlink' - task :set_linked_dirs => do true end + # We need to noop this because otherwise it creates an undesirable symlink + # Ref: https://github.com/capistrano/rails/blob/54da36a2dda5084a3cbe380b4a8f1ba282379f72/lib/capistrano/tasks/assets.rake#L121 + task :set_linked_dirs do end desc 'Compile assets' task :compile_assets => [:set_rails_env] do From d6e4f56d6d71b7b2e1254ab6447d0f7ef7ffb459 Mon Sep 17 00:00:00 2001 From: Andrew Hobden Date: Wed, 7 Sep 2016 13:49:56 +0000 Subject: [PATCH 31/51] Linting --- config/deploy.rb | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/config/deploy.rb b/config/deploy.rb index f7c4db9ee..44237d152 100644 --- a/config/deploy.rb +++ b/config/deploy.rb @@ -43,9 +43,11 @@ Rake::Task["deploy:set_linked_dirs"].clear namespace :deploy do - # We need to noop this because otherwise it creates an undesirable symlink - # Ref: https://github.com/capistrano/rails/blob/54da36a2dda5084a3cbe380b4a8f1ba282379f72/lib/capistrano/tasks/assets.rake#L121 - task :set_linked_dirs do end + + task :set_linked_dirs do + # noop due to cap/rails adding this to the run list + # Ref: https://github.com/capistrano/rails/blob/54da36a2dda5084a3cbe380b4a8f1ba282379f72/lib/capistrano/tasks/assets.rake#L121 + end desc 'Compile assets' task :compile_assets => [:set_rails_env] do From 18dd00d997177d73a9af0989436f46279cba0c0b Mon Sep 17 00:00:00 2001 From: Andrew Hobden Date: Wed, 7 Sep 2016 13:53:17 +0000 Subject: [PATCH 32/51] Revert branch redirect. Safe for merge --- config/deploy/production.rb | 2 +- config/deploy/staging.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/config/deploy/production.rb b/config/deploy/production.rb index ec141a72d..901cc84b6 100644 --- a/config/deploy/production.rb +++ b/config/deploy/production.rb @@ -6,7 +6,7 @@ set :stage, :production set :deploy_to, "/var/apps/#{fetch(:application)}/" -set :branch, :"feature/infrastructure-rebuild" +set :branch, :"feature/infra-rebuild" set :rev, proc { `git rev-parse --short #{fetch(:branch)}`.chomp } # Simple Role Syntax diff --git a/config/deploy/staging.rb b/config/deploy/staging.rb index 4c63dd776..9681b608d 100644 --- a/config/deploy/staging.rb +++ b/config/deploy/staging.rb @@ -6,7 +6,7 @@ set :stage, :staging set :deploy_to, "/var/apps/#{fetch(:application)}/" -set :branch, :"bugfix/apps-use-asset-manifest" +set :branch, :"feature/infra-rebuild" set :rev, proc { `git rev-parse --short #{fetch(:branch)}`.chomp } # Simple Role Syntax From 304d768d1ed0f2be2e0dc2e46b4e33ff0435837c Mon Sep 17 00:00:00 2001 From: Andrew Hobden Date: Thu, 8 Sep 2016 10:10:53 +0000 Subject: [PATCH 33/51] Reflect the new node names. Worker has a number --- config/deploy/production.rb | 2 +- config/deploy/staging.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/config/deploy/production.rb b/config/deploy/production.rb index 901cc84b6..6c9171c7e 100644 --- a/config/deploy/production.rb +++ b/config/deploy/production.rb @@ -29,7 +29,7 @@ server 'app0.node.production', user: 'deploy', roles: %w{app}, port: 22 server 'app1.node.production', user: 'deploy', roles: %w{app}, port: 22 server 'asset.node.production', user: 'deploy', roles: %w{asset}, port: 22, :no_release => true -server 'worker.node.production', user: 'deploy', roles: %w{worker}, port: 22 +server 'worker0.node.production', user: 'deploy', roles: %w{worker}, port: 22 #server 'mysql.node.production', user: 'wheelmap', roles: %w{mysql}, port: 22 # Custom SSH Options diff --git a/config/deploy/staging.rb b/config/deploy/staging.rb index 9681b608d..e9dffff6c 100644 --- a/config/deploy/staging.rb +++ b/config/deploy/staging.rb @@ -24,7 +24,7 @@ server 'app0.node.staging', user: 'deploy', roles: %w{app}, port: 22 server 'app1.node.staging', user: 'deploy', roles: %w{app}, port: 22 server 'asset.node.staging', user: 'deploy', roles: %w{asset}, port: 22, :no_release => true -server 'worker.node.staging', user: 'deploy', roles: %w{worker}, port: 22 +server 'worker0.node.staging', user: 'deploy', roles: %w{worker}, port: 22 #server 'mysql.node.production', user: 'wheelmap', roles: %w{mysql}, port: 22 # Custom SSH Options From f889ce209423a5ffb8572b61f2291ec82bfe5de2 Mon Sep 17 00:00:00 2001 From: Andrew Hobden Date: Tue, 4 Oct 2016 09:08:48 +0000 Subject: [PATCH 34/51] Add ping file for healthchecks --- config/deploy.rb | 3 +++ 1 file changed, 3 insertions(+) diff --git a/config/deploy.rb b/config/deploy.rb index 44237d152..cac45128d 100644 --- a/config/deploy.rb +++ b/config/deploy.rb @@ -72,6 +72,9 @@ remote_dir = "#{host.user}@#{host.hostname}:#{release_path}/public/assets/" execute "mkdir -p #{release_path}/public/assets/" run_locally { execute "rsync -av --delete #{local_dir} #{remote_dir}" } + # We create this file so the consul health check will pass. We can't use an + # existing file since they are all unpredictably named. + execute "touch #{release_path}/public/assets/ping" # stolen from https://github.com/capistrano/capistrano/blob/master/lib/capistrano/tasks/deploy.rake#L101 # we can't directly invoke the task since that runs on deploy roles only From 6d07642e138de2981aaa660e2c81f64c53bf1fbc Mon Sep 17 00:00:00 2001 From: Andrew Hobden Date: Wed, 21 Sep 2016 17:41:18 +0200 Subject: [PATCH 35/51] New dot domains --- config/environments/production.rb | 2 +- config/environments/staging.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/config/environments/production.rb b/config/environments/production.rb index d3fc74abf..8ad5d0aef 100644 --- a/config/environments/production.rb +++ b/config/environments/production.rb @@ -59,7 +59,7 @@ config.serve_static_assets = false # Enable serving of images, stylesheets, and javascripts from an asset server - config.action_controller.asset_host = 'asset%d.wheelmap.org' + config.action_controller.asset_host = "asset%d.production.new.wheelmap.org" # Disable delivery errors, bad email addresses will be ignored # config.action_mailer.raise_delivery_errors = false diff --git a/config/environments/staging.rb b/config/environments/staging.rb index 1363158ac..592f7fc9e 100644 --- a/config/environments/staging.rb +++ b/config/environments/staging.rb @@ -56,7 +56,7 @@ config.serve_static_assets = false # Enable serving of images, stylesheets, and javascripts from an asset server - config.action_controller.asset_host = 'asset%d.staging.wheelmap.org' + config.action_controller.asset_host = 'asset%d.staging.new.wheelmap.org' # Disable delivery errors, bad email addresses will be ignored # config.action_mailer.raise_delivery_errors = false From 828ca394285cf18d41ee65df88702ce92fe4ccd4 Mon Sep 17 00:00:00 2001 From: Andrew Hobden Date: Thu, 13 Oct 2016 09:28:53 +0200 Subject: [PATCH 36/51] Sync the entire public folder to the asset host. This enables things such as `icons.zip` and `favicon.ico` to be served from the static asset server. --- config/deploy.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/config/deploy.rb b/config/deploy.rb index cac45128d..13d8d95ff 100644 --- a/config/deploy.rb +++ b/config/deploy.rb @@ -43,7 +43,7 @@ Rake::Task["deploy:set_linked_dirs"].clear namespace :deploy do - + task :set_linked_dirs do # noop due to cap/rails adding this to the run list # Ref: https://github.com/capistrano/rails/blob/54da36a2dda5084a3cbe380b4a8f1ba282379f72/lib/capistrano/tasks/assets.rake#L121 @@ -66,10 +66,10 @@ end # rsync to asset server - local_dir = "./public/assets/" + local_dir = "./public/" on roles(:asset) do # this needs to be done outside run_locally in order for host to exist - remote_dir = "#{host.user}@#{host.hostname}:#{release_path}/public/assets/" + remote_dir = "#{host.user}@#{host.hostname}:#{release_path}/public/" execute "mkdir -p #{release_path}/public/assets/" run_locally { execute "rsync -av --delete #{local_dir} #{remote_dir}" } # We create this file so the consul health check will pass. We can't use an From 1bb87db6b481a1c1409572052b9547a6d065bc1f Mon Sep 17 00:00:00 2001 From: Andrew Hobden Date: Thu, 13 Oct 2016 09:54:27 +0200 Subject: [PATCH 37/51] Correct paths for sprockets --- config/deploy.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/deploy.rb b/config/deploy.rb index 13d8d95ff..f7159b12b 100644 --- a/config/deploy.rb +++ b/config/deploy.rb @@ -87,7 +87,7 @@ # this needs to be done outside run_locally in order for host to exist remote_dir = "#{host.user}@#{host.hostname}:#{release_path}/public/assets/" execute "mkdir -p #{release_path}/public/assets/" - run_locally { execute "rsync -av --delete #{local_dir}/.sprockets-manifest-* #{remote_dir}" } + run_locally { execute "rsync -av --delete #{local_dir}/assets/.sprockets-manifest-* #{remote_dir}/assets" } end # clean up From 8d4c015e6cd8a8085293f7ba60b3b17865ded63d Mon Sep 17 00:00:00 2001 From: Andrew Hobden Date: Thu, 13 Oct 2016 10:16:02 +0200 Subject: [PATCH 38/51] Correct sprockets location --- config/deploy.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/deploy.rb b/config/deploy.rb index f7159b12b..a763b9188 100644 --- a/config/deploy.rb +++ b/config/deploy.rb @@ -85,7 +85,7 @@ on roles(:app) do # this needs to be done outside run_locally in order for host to exist - remote_dir = "#{host.user}@#{host.hostname}:#{release_path}/public/assets/" + remote_dir = "#{host.user}@#{host.hostname}:#{release_path}/public/" execute "mkdir -p #{release_path}/public/assets/" run_locally { execute "rsync -av --delete #{local_dir}/assets/.sprockets-manifest-* #{remote_dir}/assets" } end From b600d5d81af2445acade7c42531388ed63745ad8 Mon Sep 17 00:00:00 2001 From: Andrew Hobden Date: Thu, 13 Oct 2016 14:07:16 +0200 Subject: [PATCH 39/51] Migrate to mailgun --- config/environments/production.rb | 19 ++++++++++--------- config/environments/staging.rb | 19 ++++++++++--------- 2 files changed, 20 insertions(+), 18 deletions(-) diff --git a/config/environments/production.rb b/config/environments/production.rb index 8ad5d0aef..d6804e36f 100644 --- a/config/environments/production.rb +++ b/config/environments/production.rb @@ -65,17 +65,18 @@ # config.action_mailer.raise_delivery_errors = false config.action_mailer.raise_delivery_errors = true config.action_mailer.delivery_method = :smtp - config.action_mailer.default_url_options = { host: 'wheelmap.org' } + config.action_mailer.default_url_options = { + :host => 'wheelmap.org', + :from => 'noreply@wheelmap.org', + } config.action_mailer.smtp_settings = { - address: 'mail.wheelmap.org', - enable_starttls_auto: false, - port: 25, - domain: 'wheelmap.org', - authentication: :login, - user_name: Rails.application.secrets.mail_user, - password: Rails.application.secrets.mail_password - + :authentication => :plain, + :address => 'smtp.mailgun.org', + :port => 587, + :domain => 'mg.wheelmap.org', + :user_name => Rails.application.secrets.mail_user, + :password => Rails.application.secrets.mail_password } # Enable threaded mode diff --git a/config/environments/staging.rb b/config/environments/staging.rb index 592f7fc9e..ef6f90786 100644 --- a/config/environments/staging.rb +++ b/config/environments/staging.rb @@ -62,17 +62,18 @@ # config.action_mailer.raise_delivery_errors = false config.action_mailer.raise_delivery_errors = true config.action_mailer.delivery_method = :smtp - config.action_mailer.default_url_options = { host: 'staging.wheelmap.org' } + config.action_mailer.default_url_options = { + :host => 'staging.wheelmap.org', + :from => 'noreply@wheelmap.org', + } config.action_mailer.smtp_settings = { - address: 'mail.wheelmap.org', - enable_starttls_auto: false, - port: 25, - domain: 'wheelmap.org', - authentication: :login, - user_name: Rails.application.secrets.mail_user, - password: Rails.application.secrets.mail_password - + :authentication => :plain, + :address => 'smtp.mailgun.org', + :port => 587, + :domain => 'mg.wheelmap.org', + :user_name => Rails.application.secrets.mail_user, + :password => Rails.application.secrets.mail_password } # Enable threaded mode From 879a5759261b181cb7a63d51024516a40ff5002e Mon Sep 17 00:00:00 2001 From: Andrew Hobden Date: Wed, 19 Oct 2016 14:08:09 +0200 Subject: [PATCH 40/51] Correct directories for uploads and link in deploy properly --- app/uploaders/photo_uploader.rb | 4 +++- config/deploy.rb | 7 ++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/app/uploaders/photo_uploader.rb b/app/uploaders/photo_uploader.rb index 98622c999..cb22f250c 100644 --- a/app/uploaders/photo_uploader.rb +++ b/app/uploaders/photo_uploader.rb @@ -18,7 +18,9 @@ class PhotoUploader < CarrierWave::Uploader::Base # This is a sensible default for uploaders that are meant to be mounted: def store_dir File.join('system', 'uploads', model.class.to_s.underscore, subdirs) - # File.join('uploads', model.class.to_s.underscore, mounted_as.to_s, subdirs) + end + def cache_dir + File.join('system', 'uploads', 'tmp', model.class.to_s.underscore) end def subdirs diff --git a/config/deploy.rb b/config/deploy.rb index a763b9188..216b5e291 100644 --- a/config/deploy.rb +++ b/config/deploy.rb @@ -72,6 +72,11 @@ remote_dir = "#{host.user}@#{host.hostname}:#{release_path}/public/" execute "mkdir -p #{release_path}/public/assets/" run_locally { execute "rsync -av --delete #{local_dir} #{remote_dir}" } + + # We need to link the existing shared `/var/apps/wheelmap/public/system` folder + # into the new release. + execute :ln, "-s", "#{deploy_path}/public/system", "#{release_path}/public/system" + # We create this file so the consul health check will pass. We can't use an # existing file since they are all unpredictably named. execute "touch #{release_path}/public/assets/ping" @@ -105,7 +110,7 @@ end end - desc 'Stopp application' + desc 'Stop application' task :stop do on roles(:app), in: :sequence, wait: 5 do sudo "systemctl stop webapp.service" From 8f0ad212ac68688a882001b624f64aa92d989c82 Mon Sep 17 00:00:00 2001 From: Andrew Hobden Date: Wed, 19 Oct 2016 14:23:34 +0200 Subject: [PATCH 41/51] Fix uploads link path --- config/deploy.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/deploy.rb b/config/deploy.rb index 216b5e291..631ce6c6a 100644 --- a/config/deploy.rb +++ b/config/deploy.rb @@ -75,7 +75,7 @@ # We need to link the existing shared `/var/apps/wheelmap/public/system` folder # into the new release. - execute :ln, "-s", "#{deploy_path}/public/system", "#{release_path}/public/system" + execute :ln, "-s", "#{deploy_path}/public/system/uploads", "#{release_path}/public/system" # We create this file so the consul health check will pass. We can't use an # existing file since they are all unpredictably named. From 4ea0a67d0ae74d001650077192b70a8736f116b0 Mon Sep 17 00:00:00 2001 From: Andrew Hobden Date: Thu, 20 Oct 2016 15:21:24 +0200 Subject: [PATCH 42/51] Only log WARN and ERROR on production. --- config/environments/production.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/environments/production.rb b/config/environments/production.rb index d6804e36f..504f4a8dd 100644 --- a/config/environments/production.rb +++ b/config/environments/production.rb @@ -40,7 +40,7 @@ # config.force_ssl = true # See everything in the log (default is :info) - config.log_level = :info + config.log_level = :warn config.logger = Logger.new($stdout) From ecf33b5f832b67899c1a24483cbe8e9930c19fae Mon Sep 17 00:00:00 2001 From: Andrew Hobden Date: Thu, 5 Jan 2017 17:45:16 +0100 Subject: [PATCH 43/51] Correct upload symlink --- config/deploy.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/deploy.rb b/config/deploy.rb index 631ce6c6a..d34ad0554 100644 --- a/config/deploy.rb +++ b/config/deploy.rb @@ -75,7 +75,7 @@ # We need to link the existing shared `/var/apps/wheelmap/public/system` folder # into the new release. - execute :ln, "-s", "#{deploy_path}/public/system/uploads", "#{release_path}/public/system" + execute :ln, "-s", "#{deploy_path}/public/system", "#{release_path}/public/" # We create this file so the consul health check will pass. We can't use an # existing file since they are all unpredictably named. From 971c984973ab3ac50fe9113cc759caf343acb655 Mon Sep 17 00:00:00 2001 From: Andrew Hobden Date: Fri, 6 Jan 2017 11:06:53 +0100 Subject: [PATCH 44/51] Fix whitespace --- config/deploy.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/deploy.rb b/config/deploy.rb index d34ad0554..475b417d1 100644 --- a/config/deploy.rb +++ b/config/deploy.rb @@ -75,7 +75,7 @@ # We need to link the existing shared `/var/apps/wheelmap/public/system` folder # into the new release. - execute :ln, "-s", "#{deploy_path}/public/system", "#{release_path}/public/" + execute :ln, "-s", "#{deploy_path}/public/system", "#{release_path}/public/" # We create this file so the consul health check will pass. We can't use an # existing file since they are all unpredictably named. From d1e9709a5db97a2da2d6dd0d72aab34cc0279cf8 Mon Sep 17 00:00:00 2001 From: Andrew Hobden Date: Fri, 13 Jan 2017 09:49:48 +0100 Subject: [PATCH 45/51] Use memcached on new infra --- config/environments/production.rb | 14 ++++++++------ config/environments/staging.rb | 13 +++++++------ 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/config/environments/production.rb b/config/environments/production.rb index 504f4a8dd..c36bd8e5a 100644 --- a/config/environments/production.rb +++ b/config/environments/production.rb @@ -47,12 +47,14 @@ # Use a different cache store in production # default is: # config.cache_store = :file_store, "tmp/cache/" - config.cache_store = :mem_cache_store, 'localhost', { namespace: "wheelmap/#{Rails.env}/", - c_threshold: 10_000, - compression: true, - debug: Rails.env.development?, - readonly: false, - urlencode: false } + config.cache_store = :mem_cache_store, 'app-database', { :namespace => "wheelmap/#{Rails.env}/", + :c_threshold => 10_000, + :compression => true, + :debug => Rails.env.development?, + :readonly => false, + :urlencode => false + } + # Disable Rails's static asset server # In production, Apache or nginx will already do this diff --git a/config/environments/staging.rb b/config/environments/staging.rb index ef6f90786..1673fe30c 100644 --- a/config/environments/staging.rb +++ b/config/environments/staging.rb @@ -44,12 +44,13 @@ # Use a different cache store in staging # default is: # config.cache_store = :file_store, "tmp/cache/" - config.cache_store = :mem_cache_store, 'localhost', { namespace: "wheelmap/#{Rails.env}/", - c_threshold: 10_000, - compression: true, - debug: Rails.env.staging?, - readonly: false, - urlencode: false } + config.cache_store = :mem_cache_store, 'app-database', { :namespace => "wheelmap/#{Rails.env}/", + :c_threshold => 10_000, + :compression => true, + :debug => Rails.env.staging?, + :readonly => false, + :urlencode => false + } # Disable Rails's static asset server # In production, Apache or nginx will already do this From ca7d30543660108bf7601eb34b9dcbc666b29dc0 Mon Sep 17 00:00:00 2001 From: Andrew Hobden Date: Tue, 25 Apr 2017 15:07:16 +0200 Subject: [PATCH 46/51] syncing changes --- config/deploy.rb | 4 ++-- config/deploy/production.rb | 1 + config/deploy/staging.rb | 1 + lib/tasks/planet_reader.rake | 5 ++--- 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/config/deploy.rb b/config/deploy.rb index 475b417d1..9f4cda8e0 100644 --- a/config/deploy.rb +++ b/config/deploy.rb @@ -25,7 +25,7 @@ # Default value for :linked_files is [] set :linked_files, %w{ config/database.yml config/open_street_map.yml config/metrics.yml config/librato.yml config/newrelic.yml .env config/secrets.yml} -set :bundle_roles, [:app, :worker] +set :bundle_roles, [:app, :worker, :importer] set :bundle_without, %w{ development test metrics deployment }.join(' ') set :bundle_jobs, 4 set :bundle_binstubs, -> { shared_path.join('bin') } @@ -75,7 +75,7 @@ # We need to link the existing shared `/var/apps/wheelmap/public/system` folder # into the new release. - execute :ln, "-s", "#{deploy_path}/public/system", "#{release_path}/public/" + execute :ln, "-s", "#{deploy_path}/public/system/uploads", "#{release_path}/public/system" # We create this file so the consul health check will pass. We can't use an # existing file since they are all unpredictably named. diff --git a/config/deploy/production.rb b/config/deploy/production.rb index 6c9171c7e..570187c95 100644 --- a/config/deploy/production.rb +++ b/config/deploy/production.rb @@ -30,6 +30,7 @@ server 'app1.node.production', user: 'deploy', roles: %w{app}, port: 22 server 'asset.node.production', user: 'deploy', roles: %w{asset}, port: 22, :no_release => true server 'worker0.node.production', user: 'deploy', roles: %w{worker}, port: 22 +server 'osm-database.node.production', user: 'deploy', roles: %w{importer}, port: 22 # Used for replication. #server 'mysql.node.production', user: 'wheelmap', roles: %w{mysql}, port: 22 # Custom SSH Options diff --git a/config/deploy/staging.rb b/config/deploy/staging.rb index e9dffff6c..e261c705d 100644 --- a/config/deploy/staging.rb +++ b/config/deploy/staging.rb @@ -25,6 +25,7 @@ server 'app1.node.staging', user: 'deploy', roles: %w{app}, port: 22 server 'asset.node.staging', user: 'deploy', roles: %w{asset}, port: 22, :no_release => true server 'worker0.node.staging', user: 'deploy', roles: %w{worker}, port: 22 +server 'osm-database.node.staging', user: 'deploy', roles: %w{importer}, port: 22 # Used for replication. #server 'mysql.node.production', user: 'wheelmap', roles: %w{mysql}, port: 22 # Custom SSH Options diff --git a/lib/tasks/planet_reader.rake b/lib/tasks/planet_reader.rake index 9e49d187a..e274fd7c4 100644 --- a/lib/tasks/planet_reader.rake +++ b/lib/tasks/planet_reader.rake @@ -54,9 +54,8 @@ def get_new_replication_file end def get_new_shape_replication_files - puts 'INFO: Fetching shape changes.' - system "ssh -p 22022 osm@176.9.63.171 \"ruby make_osc.rb -d\" 2> /dev/null >> #{SHAPE_FILE}" - puts "INFO: Downloaded #{File.size(SHAPE_FILE)} bytes" + puts "INFO: Fetching shape changes." + system "#{MAKE_OSC} -d >> #{SHAPE_FILE}" end def merge_replication_files From c5bed486c895d86a40b1f4abecd47a386c60aa7f Mon Sep 17 00:00:00 2001 From: Andrew Hobden Date: Tue, 25 Apr 2017 15:14:00 +0200 Subject: [PATCH 47/51] Use bundle --- lib/tasks/planet_reader.rake | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/tasks/planet_reader.rake b/lib/tasks/planet_reader.rake index e274fd7c4..e2fe4ed72 100644 --- a/lib/tasks/planet_reader.rake +++ b/lib/tasks/planet_reader.rake @@ -8,6 +8,7 @@ SHAPE_FILE = "#{VAR_DIR}/shapes.osc".freeze STATE_FILE = "#{WORKING_DIR}/state.txt".freeze BACKUP_FILE = "#{WORKING_DIR}/state.old".freeze DOWNLOAD_LOCK = "#{WORKING_DIR}/download.lock".freeze +MAKE_OSC_DIR = "/var/apps/make_osc/".freeze require 'rake' @@ -55,7 +56,7 @@ end def get_new_shape_replication_files puts "INFO: Fetching shape changes." - system "#{MAKE_OSC} -d >> #{SHAPE_FILE}" + system "BUNDLE_GEMFILE=#{MAKE_OSC_DIR}/Gemfile bundle exec #{MAKE_OSC_DIR}/make_osc.rb -d >> #{SHAPE_FILE}" end def merge_replication_files From ee2e8baee8830d966c246f20b2edcb9b9d38882a Mon Sep 17 00:00:00 2001 From: Andrew Hobden Date: Tue, 23 May 2017 15:10:51 +0200 Subject: [PATCH 48/51] Correct rsync and symbolic link for asset --- config/deploy.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/deploy.rb b/config/deploy.rb index 9f4cda8e0..9542eb1a1 100644 --- a/config/deploy.rb +++ b/config/deploy.rb @@ -71,11 +71,11 @@ # this needs to be done outside run_locally in order for host to exist remote_dir = "#{host.user}@#{host.hostname}:#{release_path}/public/" execute "mkdir -p #{release_path}/public/assets/" - run_locally { execute "rsync -av --delete #{local_dir} #{remote_dir}" } + run_locally { execute "rsync -av --delete --exclude system #{local_dir} #{remote_dir}" } # We need to link the existing shared `/var/apps/wheelmap/public/system` folder # into the new release. - execute :ln, "-s", "#{deploy_path}/public/system/uploads", "#{release_path}/public/system" + execute :ln, "-s", "#{deploy_path}/public/system", "#{release_path}/public/system" # We create this file so the consul health check will pass. We can't use an # existing file since they are all unpredictably named. From 2608cb15e126771aba64761a8c795f98eb285e70 Mon Sep 17 00:00:00 2001 From: Andrew Hobden Date: Wed, 19 Jul 2017 12:39:04 +0200 Subject: [PATCH 49/51] Remap asset servers --- config/environments/production.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/environments/production.rb b/config/environments/production.rb index c36bd8e5a..3c2da59b1 100644 --- a/config/environments/production.rb +++ b/config/environments/production.rb @@ -61,7 +61,7 @@ config.serve_static_assets = false # Enable serving of images, stylesheets, and javascripts from an asset server - config.action_controller.asset_host = "asset%d.production.new.wheelmap.org" + config.action_controller.asset_host = "asset%d-new-prod.wheelmap.org" # Disable delivery errors, bad email addresses will be ignored # config.action_mailer.raise_delivery_errors = false From efb5ef92e7a840b7dc5f7b86e3b34e70daa9c2a3 Mon Sep 17 00:00:00 2001 From: Andrew Hobden Date: Wed, 19 Jul 2017 14:10:35 +0200 Subject: [PATCH 50/51] Use only 1 asset host --- config/environments/production.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/environments/production.rb b/config/environments/production.rb index 3c2da59b1..3bbc1e85c 100644 --- a/config/environments/production.rb +++ b/config/environments/production.rb @@ -61,7 +61,7 @@ config.serve_static_assets = false # Enable serving of images, stylesheets, and javascripts from an asset server - config.action_controller.asset_host = "asset%d-new-prod.wheelmap.org" + config.action_controller.asset_host = "asset-new-prod.wheelmap.org" # Disable delivery errors, bad email addresses will be ignored # config.action_mailer.raise_delivery_errors = false From 8e55336f3ffd6b5709b90b29430d55c669c19bf4 Mon Sep 17 00:00:00 2001 From: Andrew Hobden Date: Wed, 19 Jul 2017 15:26:25 +0200 Subject: [PATCH 51/51] Refine domains a bit --- config/environments/production.rb | 2 +- config/environments/staging.rb | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/config/environments/production.rb b/config/environments/production.rb index 3bbc1e85c..6df959d28 100644 --- a/config/environments/production.rb +++ b/config/environments/production.rb @@ -61,7 +61,7 @@ config.serve_static_assets = false # Enable serving of images, stylesheets, and javascripts from an asset server - config.action_controller.asset_host = "asset-new-prod.wheelmap.org" + config.action_controller.asset_host = "new-production-asset.wheelmap.org" # Disable delivery errors, bad email addresses will be ignored # config.action_mailer.raise_delivery_errors = false diff --git a/config/environments/staging.rb b/config/environments/staging.rb index 1673fe30c..b11a8bab2 100644 --- a/config/environments/staging.rb +++ b/config/environments/staging.rb @@ -57,13 +57,13 @@ config.serve_static_assets = false # Enable serving of images, stylesheets, and javascripts from an asset server - config.action_controller.asset_host = 'asset%d.staging.new.wheelmap.org' + config.action_controller.asset_host = 'new-staging-asset.wheelmap.org' # Disable delivery errors, bad email addresses will be ignored # config.action_mailer.raise_delivery_errors = false config.action_mailer.raise_delivery_errors = true config.action_mailer.delivery_method = :smtp - config.action_mailer.default_url_options = { + config.action_mailer.default_url_options = { :host => 'staging.wheelmap.org', :from => 'noreply@wheelmap.org', }