Merge pull request #9 from NuCivic/rc-1.12.11.0

Upgrades to DKAN 1.11

build-dkan.make

@@ -8,8 +8,8 @@ includes[core] = drupal-org-core.make
 projects[dkan][type] = profile
 projects[dkan][download][type] = git
 projects[dkan][download][url] = https://github.com/NuCivic/dkan.git
-projects[dkan][download][tag] = 7.x-1.12.10
+projects[dkan][download][tag] = 7.x-1.12.11
 
 ; Un-comment if diff is not empty.
-projects[dkan][patch][1] = https://github.com/NuCivic/dkan/compare/7.x-1.12.10...release-1-12.diff
+projects[dkan][patch][1] = https://github.com/NuCivic/dkan/compare/7.x-1.12.11...release-1-12.diff
 

circle.yml

@@ -73,8 +73,9 @@ dependencies:
 test:
   override:
     - ruby dkan/.ahoy/.scripts/circle-behat.rb docroot/profiles/dkan/test/features --tags='~@customizable&&~@fixme&&~@testBug':
-         parallel: true
-
+        timeout: 1200
+        parallel: true
+
   post:
     - echo $CIRCLE_ARTIFACTS; cp -av dkan/test/assets $CIRCLE_ARTIFACTS:
         parallel: true

config/config.php

@@ -22,6 +22,17 @@
       'queue' => '50MB',
     ),
   ),
+  'private' => 
+  array (
+    'aws' => 
+    array (
+      'scrubbed_data_url' => 'CHANGE ME',
+    ),
+    'probo' => 
+    array (
+      'password' => 'CHANGE ME',
+    ),
+  ),
   'acquia' => 
   array (
     'subscription' => 'professional',

dkan/.ahoy/.scripts/behat-parse-params.rb

@@ -51,7 +51,7 @@ def behat_parse_params args
       params.push param
     end
   end
-
+  params.push '--colors' 
   {:files => files, :params => params}
 end
 

dkan/.ahoy/.scripts/circle-behat-balancer.rb

@@ -0,0 +1,87 @@
+require "pathname"
+
+TOKENS = {
+  "s" => (1),
+  "m" => (60),
+  "h" => (60 * 60),
+  "d" => (60 * 60 * 24)
+}
+
+
+def parse_time(time_string, measure = "m")
+  time = 0 
+  time_string.scan(/([-+]?[0-9]*\.?[0-9]*)(\w)/).each do |amount, measure|
+    begin
+      time += amount.to_i * TOKENS[measure]
+    rescue TypeError => te
+      puts "Missing time header"
+    rescue StandardError => se
+      puts "An error happen trying to parse the time header"
+    end
+  end
+  return time
+end
+
+def parse_header(file) 
+  file = Pathname(file).realpath.to_s
+  first_line = File.open(file, &:gets)
+  first_line.sub! "#", ""
+  if first_line.include? "time"
+    time = first_line.split(":")[1].strip
+    time = parse_time(time)
+  else 
+    time = 60 # For missing headers default to 60s
+  end 
+  return time
+end 
+
+def sort_files(files)
+  sorted_files = {}
+  files.each_index do |i|
+    sorted_files[files[i]] = parse_header(files[i])
+  end
+  return sorted_files.sort_by {|k, v| v}.reverse
+end
+
+def node_total()
+  unless ENV.has_key?("CIRCLE_NODE_TOTAL")
+    return 1
+  else 
+    return ENV["CIRCLE_NODE_TOTAL"].to_i
+  end
+end
+
+def node_index()
+  unless ENV.has_key?("CIRCLE_NODE_INDEX")
+    return 0
+  else 
+    return ENV["CIRCLE_NODE_INDEX"].to_i
+  end
+end
+
+def get_containers()
+  return Array.new(node_total(), 0)
+end
+
+def get_distribution()
+  distribution = {}
+  to = node_total() - 1;
+  for i in 0..to
+    distribution[i] = []
+  end
+  return distribution
+end 
+
+def balance(files)
+  containers = get_containers()
+  distribution = get_distribution()
+  files = sort_files(files)
+  files.each_index do |i|
+    emptier = containers.each_with_index.min[1]
+    file, weight = files[i]
+    containers[emptier] += weight
+    distribution[emptier].push(file)
+  end
+  return distribution
+end
+

dkan/.ahoy/.scripts/circle-behat.rb

@@ -4,41 +4,39 @@
 # Ex. There are 3 VMs setup: (3)
 require "pathname"
 require "./dkan/.ahoy/.scripts/behat-parse-params.rb"
+require "./dkan/.ahoy/.scripts/circle-behat-balancer.rb"
 require "pp"
 
-unless ENV.has_key?("CIRCLE_NODE_INDEX")
-  puts "No parrallelism found, setting defaults to run all tests."
-  CIRCLE_NODE_TOTAL=1
-  CIRCLE_NODE_INDEX=0
-else 
-  CIRCLE_NODE_TOTAL = ENV['CIRCLE_NODE_TOTAL'].to_i
-  CIRCLE_NODE_INDEX = ENV['CIRCLE_NODE_INDEX'].to_i
-end
-
 CIRCLE_ARTIFACTS =  ENV.has_key?("CIRCLE_ARTIFACTS") ? ENV['CIRCLE_ARTIFACTS'] : "assets"
+CIRCLE_NODE_TOTAL = node_total()
+CIRCLE_NODE_INDEX = node_index()
+TOKENS = {
+  "s" => (1),
+  "m" => (60),
+  "h" => (60 * 60),
+  "d" => (60 * 60 * 24)
+}
 
 puts "CIRCLE_NODE_TOTAL = #{CIRCLE_NODE_TOTAL}"
 puts "CIRCLE_NODE_INDEX = #{CIRCLE_NODE_INDEX}"
 
 
 error = 0
 parsed = behat_parse_params(ARGV)
-files = parsed[:files]
 params = behat_join_params parsed[:params]
+files = balance(parsed[:files])
 
-files.each_index do |i|
-  if (i % CIRCLE_NODE_TOTAL)  == CIRCLE_NODE_INDEX
-    file = Pathname(files[i]).realpath.to_s
-    puts "RUNNING: ahoy dkan test #{file} --format=pretty --out=std --format=junit --out='#{CIRCLE_ARTIFACTS}/junit' #{params} --colors"
-    IO.popen(
-    "ahoy dkan test #{file} --format=pretty --out=std --format=junit --out='#{CIRCLE_ARTIFACTS}/junit' #{params} --colors"
-    ) do |io|
-      while line = io.gets
-        print line
-      end
-      io.close
-      error = 1 unless $?.success?
+files[CIRCLE_NODE_INDEX].each_index do |i|
+  file = Pathname(files[CIRCLE_NODE_INDEX][i]).realpath.to_s
+  puts "RUNNING: ahoy dkan test #{file} --format=pretty --out=std --format=junit --out='#{CIRCLE_ARTIFACTS}/junit' #{params} --colors"
+  IO.popen(
+  "ahoy dkan test #{file} --format=pretty --out=std --format=junit --out='#{CIRCLE_ARTIFACTS}/junit' #{params} --colors"
+  ) do |io|
+    while line = io.gets
+      print line
     end
+    io.close
+    error = 1 unless $?.success?
   end
 end
 

dkan/.ahoy/.scripts/dkan-make.sh

@@ -0,0 +1,13 @@
+if [ $(uname) == "Darwin" ]; then
+  concurrency=`sysctl -n hw.ncpu`
+else
+  concurrency=`grep -c ^processor /proc/cpuinfo`
+fi
+
+if [ "$AHOY_CMD_PROXY" = "DOCKER" ]; then
+  working_copy="--working-copy"
+else
+  working_copy=""
+fi
+
+drush --root=docroot -y make --no-core --contrib-destination=./ dkan/drupal-org.make --no-recursion --no-cache --verbose $working_copy --concurrency=$concurrency docroot/profiles/dkan $@

dkan/.ahoy/.scripts/dkan-si.sh

@@ -0,0 +1,4 @@
+drush si dkan --root=docroot --verbose --account-pass='admin' "$1" --site-name='DKAN' install_configure_form.update_status_module='array(FALSE,FALSE)' --yes && \
+  drush --root=docroot sql-dump > backups/last_install.sql.tmp && \
+  mv backups/last_install.sql.tmp backups/last_install.sql && \
+  echo "Installed DKAN and created a new backup at backups/last_install.sql"

dkan/.ahoy/.scripts/drupal-rebuild.sh

@@ -0,0 +1,16 @@
+if [ $(uname) == "Darwin" ]; then
+  concurrency=`sysctl -n hw.ncpu`
+else
+  concurrency=`grep -c ^processor /proc/cpuinfo`
+fi
+
+if [ "$AHOY_CMD_PROXY" = "DOCKER" ]; then
+  working_copy="--working-copy"
+else
+  working_copy=""
+fi
+
+drush --root=docroot  make --concurrency=$concurrency --prepare-install dkan/drupal-org-core.make docroot --yes
+
+drush --root=docroot -y --verbose $root si minimal --sites-subdir=default --account-pass='admin' --db-url=$1 install_configure_form.update_status_module='array(false,false)' &&
+  ln -s ../../dkan docroot/profiles/dkan

dkan/.ahoy/dkan.ahoy.yml

@@ -4,7 +4,12 @@ commands:
   drupal-rebuild:
     usage: 'Builds a drupal site in ./docroot. Requires database url.'
     cmd: |
-      if [[ "{{args}}" != mysql* ]]; then
+      if [ "$AHOY_CMD_PROXY" != "DOCKER" ]; then
+        ARGS="{{args}}"
+      else
+        ARGS=`ahoy docker mysql-url`
+      fi
+      if [[ "$ARGS" != mysql* ]]; then
           echo "You need to specify the drupal db url to use in the form mysql://root:root@localhost/db as an argument."
           exit 1
       fi
@@ -14,20 +19,23 @@ commands:
           ahoy confirm "./docroot folder alredy exists. Delete it and reinstall drupal?" && chmod -R 777 docroot/sites/default && rm -rf docroot ||
           { echo ".. skipping installation"; exit 1;}
       fi
-      ahoy drush make --prepare-install dkan/drupal-org-core.make docroot --yes
-      ln -s ../../dkan docroot/profiles/dkan &&
-      ahoy drush "-y --verbose si minimal --sites-subdir=default --account-pass='admin' --db-url={{args}} install_configure_form.update_status_module=\"'array\(FALSE,FALSE\)'\""
+      ahoy cmd-proxy bash dkan/.ahoy/.scripts/drupal-rebuild.sh $ARGS
 
   remake:
     usage: Rebuild all the dkan projects from the drupal-org.make file using drush make.
     # Note that we tried using --overwrite, but that seems to delete the entire modules/dkan folder which includes both checked-in modules and modules from make.
     # So, we delete the specific folders.
     cmd: |
-      rm -rf docroot/profiles/dkan/modules/dkan/dkan_dataset ;
-      rm -rf docroot/profiles/dkan/modules/dkan/dkan_datastore ;
-      rm -rf docroot/profiles/dkan/modules/contrib ;
-      echo "removed dkan/modules folders created by make and running a fresh drush make."
-      ahoy drush -y make --no-core --contrib-destination=./ dkan/drupal-org.make --no-recursion --no-cache --verbose --working-copy docroot/profiles/dkan
+      ARGS="{{args}}"
+      if [ -z "$ARGS" ]; then
+        rm -rf docroot/profiles/dkan/modules/contrib ;
+        ARGS=''
+        echo "removed dkan/modules folders created by make and running a fresh drush make."
+      else
+        ARGS="--projects=$ARGS"
+        echo "Running drush make for the following modules: $ARGS."
+      fi
+      ahoy cmd-proxy bash dkan/.ahoy/.scripts/dkan-make.sh "$ARGS"
 
   reinstall:
     usage: Reinstall the dkan install profile (db only).
@@ -41,11 +49,15 @@ commands:
         ahoy dkan sqlc <  backups/last_install.sql && \
         echo "Installed dkan from backup"
       else
-        ahoy drush "si dkan --verbose --account-pass='admin' --site-name='DKAN' install_configure_form.update_status_module=\"\'array\(FALSE,FALSE\)\'\" --yes" && \
-        ahoy drush sql-dump > backups/last_install.sql.tmp && \
-        mv backups/last_install.sql.tmp backups/last_install.sql && \
-        echo "Installed DKAN and created a new backup at backups/last_install.sql"
+        if [ "$AHOY_CMD_PROXY" == "DOCKER" ]; then
+          MYSQL=`ahoy docker mysql-url`
+          MYSQL="--db-url=$MYSQL"
+        else
+          MYSQL=''
+        fi
+        ahoy cmd-proxy bash dkan/.ahoy/.scripts/dkan-si.sh $MYSQL
       fi
+
   module-fetch:
     usage: Checkout a dkan module from the NuCivic github repo.
     cmd: |

dkan/.ahoy/docker-compose.yml

@@ -43,7 +43,7 @@ db:
 # Used for all console commands and tools.
 cli:
   hostname: cli
-  image: nuams/drupal-cli
+  image: nuams/drupal-cli:2016-10-16
   environment:
     - XDEBUG_CONFIG=idekey=cli
     - PHP_IDE_CONFIG=serverName=dkan.docker
@@ -88,7 +88,7 @@ memcached:
 # Uncomment the service definition section below and the link in the web service above to start using selenium2 driver for Behat tests requiring JS support.
 browser:
   hostname: browser
-  image: selenium/standalone-chrome-debug:2.49.1
+  image: selenium/standalone-chrome-debug:2.53.1
   # This helps keep selenium-chrome from crashing because it uses shared memory.
   volumes:
     - /dev/shm:/dev/shm

dkan/.ahoy/docker.ahoy.yml

@@ -1,10 +1,6 @@
 ahoyapi: v1
 version: 0.0.0
 commands:
-  diagnose:
-    usage: A series of ahoy-docker setup diagnosis commands.
-    import: dkan/.ahoy/diagnose.ahoy.yml
-    hide: true
   env:
     cmd: |
       cat << EOF
@@ -67,8 +63,13 @@ commands:
         # if the input is not empty, then don't use tty
         docker exec -i $id bash -c "$command $args"
       fi
-    usage: run a command in the docker-compose cli or in any other docker service container.
-
+    usage: run a command in the docker-compose cli service container.
+  mysql-url:
+    cmd: |
+      MYSQL=`docker exec -it $(ahoy docker compose ps -q cli) bash -c 'echo "mysql://$DB_1_ENV_MYSQL_USER:$DB_1_ENV_MYSQL_PASSWORD@$DB_1_PORT_3306_TCP_ADDR/$DB_1_ENV_MYSQL_DATABASE"'`
+      MYSQL="$(echo -e "${MYSQL}" | tr -d '[[:space:]]')"
+      echo "$MYSQL"
+    usage: Outputs a mysql-url
   mysql:
     cmd: "docker exec -it $(ahoy docker compose ps -q cli) bash -c 'mysql -uroot -p$DB_ENV_MYSQL_ROOT_PASSWORD -h$DB_PORT_3306_TCP_ADDR $DB_ENV_MYSQL_DATABASE'"
     usage: Connect to the default mysql database as the root user.

dkan/.ahoy/site/.ahoy.yml

@@ -15,6 +15,15 @@ commands:
     hide: true
     import: dkan/.ahoy/dkan.ahoy.yml
 
+  diagnose:
+    usage: A series of ahoy-docker setup diagnosis commands.
+    import: dkan/.ahoy/diagnose.ahoy.yml
+    hide: true
+
+  doctor:
+    usage: A series of ahoy-docker setup diagnosis commands.
+    import: dkan/.ahoy/diagnose.ahoy.yml
+    hide: true
 
   confirm:
     cmd: ahoy -f dkan/.ahoy/utils.ahoy.yml confirm {{args}}
@@ -23,7 +32,7 @@ commands:
   docker:
     usage: A series of docker commands for dkan development (experimental)
     import: dkan/.ahoy/docker.ahoy.yml
-    hide: false 
+    hide: true
 
   init:
     cmd: echo "ahoy.yml file already exists."

dkan/.ahoy/site/build.ahoy.yml

@@ -163,8 +163,8 @@ commands:
       set -e
       AHOY_CMD_PROXY=''
       DIRNAME=${PWD##*/}
-      if [[ "$DIRNAME" != "dkan_starter" ]]; then
-        echo "This command should only be use from a dkan_starter working copy"
+      if [[ "$DIRNAME" != "data_starter_private" ]]; then
+        echo "This command should only be use from a data_starter_private working copy"
         exit 0
       fi
 
@@ -179,7 +179,7 @@ commands:
       fi
       # No ahoy commands after cd-ing out of project folder.
       cd ..
-      git clone '[email protected]:NuCivic/dkan_starter.git' {{args}} --depth=1
+      git clone '[email protected]:NuCivic/data_starter_private.git' {{args}} --depth=1
       cd {{args}}
       rm -rf .git build.make build-dkan.make drupal-org-core.make
       echo "Site {{args}} initiated at ../{{args}}"