Rework of the CI logic in (mostly) Ruby.

This refactoring had a number of goals:

* Replace fragile shell script constructs with more robust Ruby code. Ruby
  has all the tools needed for shell scripting, but is a proper programming
  language.

* Various hacks to avoid encoding issues with Python's handling of strings
  can be avoided, especially when output is not legal Unicode. Ruby is more
  forgiving of malformed Unicode strings and can pass them through as long
  as you don't try to process their contents.

* The Python implementation had issues with truncating output when processes
  were killed from timeouts, as the signaling mechanism prevented output
  from being captured.

* Templating of the Docker file was easiest with `erb`.

In addition, we also incorporate the GitHub status page layout in this
repository so that we don't need to update multiple repos when the layout
changes.

Jenkinsfile

@@ -77,7 +77,7 @@ node {
             }
 	    updateTimestamp()
 	    get url: metarepo, dir: "meta"
-	    sh "meta/prepare.sh"
+	    sh "meta/prepare.rb"
             // Update repositories
             if (rebuild != "none") {
 		get url: "https://github.com/julialang/julia",
@@ -116,12 +116,12 @@ node {
         stage('Build') {
             if (rebuild != "none") {
 	        run_in_docker {
-		    sh "meta/install/install-julia.sh"
-		    sh "meta/install/install-oscar.sh"
-		    sh "meta/install/install-jupyter.sh"
-		    sh "meta/install/install-gap.sh"
-		    sh "meta/install/install-gap-packages.sh"
-		    sh "meta/install/install-finalize.sh"
+		    sh "meta/install/install-julia.rb"
+		    sh "meta/install/install-oscar.rb"
+		    sh "meta/install/install-jupyter.rb"
+		    sh "meta/install/install-gap.rb"
+		    sh "meta/install/install-gap-packages.rb"
+		    sh "meta/install/install-finalize.rb"
 		}
             } else {
                 // skip build stage
@@ -130,7 +130,7 @@ node {
         }
         stage('Test') {
 	    run_in_docker {
-		sh "meta/run-tests.sh"
+		sh "meta/run-tests.rb"
 	    }
         }
     } finally {

bin/create-stdjob

@@ -1,20 +1,26 @@
-#!/bin/bash
+#!/usr/bin/env ruby
+require_relative "../jenkins-env.rb"
+require "fileutils"
 
-if [ ! -d "$JENKINS_HOME/jobs" ]; then
-  echo "Jenkins has not been setup yet."
+def fatal(msg)
+  puts msg
   exit 1
-fi
+end
 
-if [ -z "$1" ]; then
-  echo "usage: create-stdjob <jobname>"
-  exit 1
-fi
+if not Dir.exist?("#{$JENKINS_HOME}/jobs") then
+  fatal "Jenkins has not been setup yet."
+end
 
-if [ -d "$JENKINS_HOME/jobs/$1" ]; then
-  echo "Job '$1' already exists."
-  exit 1
-fi
+if ARGV.size != 1 or ARGV.first == "" then
+  fatal "usage: create-stdjob <jobname>"
+end
+
+job = ARGV.first
+jobdir = "#{$JENKINS_HOME}/jobs/#{job}"
+
+if Dir.exist?(jobdir) then
+  fatal "Job #{job} already exists."
+end
 
-JOBDIR="$JENKINS_HOME/jobs/$1"
-mkdir -p $JOBDIR
-cp -f $HOME/develop/ci-meta/config.xml $JOBDIR
+FileUtils.mkdir_p jobdir
+FileUtils.cp_f File.expand_path("#{__dir__}/../config.xml"), jobdir

bin/docker-cli

@@ -1,5 +1,3 @@
-#!/bin/bash
-set -e
-BASEDIR="$(realpath "$(dirname "$0")/..")"
-source "$BASEDIR/jenkins-env.sh"
-docker run -it "$OSCAR_CI_IMAGE" "$@"
+#!/usr/bin/env ruby
+require_relative "../jenkins-env.rb"
+exec "docker", "run", "-it", $OSCAR_CI_IMAGE, *ARGV

bin/get-jenkins

@@ -1,7 +1,5 @@
-#!/bin/bash
-set -e
-BASEDIR="$(realpath "$(dirname "$0")/..")"
-source "$BASEDIR/jenkins-env.sh"
-JENKINS_VER="${JENKINS_VER:-latest}"
-JENKINS_URL="http://mirrors.jenkins.io/war-stable/${JENKINS_VER}/jenkins.war"
-wget -O "$JENKINS_WAR" "$JENKINS_URL"
+#!/usr/bin/env ruby
+require_relative "../jenkins-env.rb"
+$JENKINS_VER = ENV["JENKINS_VER"] || "latest"
+$JENKINS_URL="http://mirrors.jenkins.io/war-stable/#{$JENKINS_VER}/jenkins.war"
+system "wget", "-O", $JENKINS_WAR, $JENKINS_URL

bin/jenkins

@@ -1,10 +1,9 @@
-#!/bin/bash
-set -e
-BASEDIR="$(realpath "$(dirname "$0")/..")"
-source "$BASEDIR/jenkins-env.sh"
-test -f "$JENKINS_WAR" || {
-  echo "Please install $JENKINS_WAR first."
-  echo "You can use $BASEDIR/bin/get-jenkins for that."
+#!/usr/bin/env ruby
+require_relative "../jenkins-env.rb"
+require "shellwords"
+if not File.exist?($JENKINS_WAR) then
+  puts "Please install $JENKINS_WAR first."
+  puts "You can use bin/get-jenkins for that."
   exit 1
-}
-java -Dorg.jenkinsci.plugins.durabletask.BourneShellScript.HEARTBEAT_CHECK_INTERVAL=600 -jar "$JENKINS_WAR" "$@"
+end
+exec "java -Dorg.jenkinsci.plugins.durabletask.BourneShellScript.HEARTBEAT_CHECK_INTERVAL=600 -jar #{$JENKINS_WAR} #{Shellwords.shelljoin(ARGV)}"

bin/show-image-name

@@ -1,5 +1,3 @@
-#!/bin/bash
-set -e
-BASEDIR="$(realpath "$(dirname "$0")/..")"
-source "$BASEDIR/jenkins-env.sh"
-echo "$OSCAR_CI_IMAGE"
+#!/usr/bin/env ruby
+require_relative "../jenkins-env"
+puts $OSCAR_CI_IMAGE

check-julia-notebook.rb

@@ -0,0 +1,32 @@
+#!/usr/bin/env ruby
+require_relative "settings.rb"
+require "json"
+
+env = { "LC_ALL" => "en_US.UTF-8" }
+notebook = ARGV.shift
+basename = File.basename(notebook)
+kernel = nil
+kernelspecs = JSON.load(%x{jupyter kernelspec list --json})["kernelspecs"]
+for name in kernelspecs.keys do
+  if name.start_with?("julia-") then
+    kernel = name
+    break
+  end
+end
+if not kernel then
+  puts "=== Error: no Julia Jupyter kernel found"
+  exit 1
+end
+
+success = system env, "jupyter", "nbconvert",
+  "--ExecutePreprocessor.kernel_name=#{kernel}",
+  "--ExecutePreprocessor.timeout=600",
+  "--to=notebook",
+  "--output-dir=#{$WORKSPACE}/notebooks-out",
+  "--execute",
+  notebook
+
+puts "=== notebook diff for #{basename}"
+system env, "meta/nb-diff", "-w", notebook, "notebooks-out/#{basename}"
+
+exit success

install/install-finalize.rb

@@ -0,0 +1,3 @@
+#!/usr/bin/env ruby
+require_relative "../settings.rb"
+system %q{julia -e 'include("meta/safepkg.jl"); SafePkg.precompile()'}

install/install-gap-packages.rb

@@ -0,0 +1,11 @@
+#!/usr/bin/env ruby
+require_relative "../settings.rb"
+require "fileutils"
+
+packages = [ "NemoLinearAlgebraForCAP" ]
+pkgdir = %x{julia meta/gappkgpath.jl}.chomp
+
+for pkg in packages do
+  FileUtils.rm_f "#{pkgdir}/#{pkg}"
+  FileUtils.ln_sf File.realpath(pkg), pkgdir
+end

install/install-gap.rb

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+require_relative "../settings.rb"
+require "fileutils"
+
+FileUtils.ln_sf %x{julia meta/gappath.jl}.chomp, "local/bin/gap"

install/install-julia.rb

@@ -0,0 +1,6 @@
+#!/usr/bin/env ruby
+require_relative "../settings.rb"
+require "fileutils"
+
+system "make -C julia -j#{ENV['BUILDJOBS'] || 4}"
+FileUtils.ln_sf "#{$WORKSPACE}/julia/julia", "local/bin"

install/install-jupyter.rb

@@ -0,0 +1,15 @@
+#!/usr/bin/env ruby
+require_relative "../settings.rb"
+require "fileutils"
+
+FileUtils.mkdir_p $JUPYTER_BASE
+system "python3", "-m", "venv", $IPYTHON
+system "#{$IPYTHON}/bin/pip", "install", "--cache-dir",
+  "#{$JUPYTER_BASE}/.pip-cache", "jupyter", "notebook"
+system "julia", "meta/install/install-jupyter.jl"
+jupyter = "#{$IPYTHON}/bin/jupyter"
+FileUtils.ln_sf jupyter, "#{$WORKSPACE}/local/bin"
+jupyter_data_dir = ENV["JUPYTER_DATA_DIR"]
+kernel_json = Dir.glob("#{jupyter_data_dir}/**/julia-*/kernel.json",
+  File::FNM_DOTMATCH).first
+system "sed", "-i", "-e", "/--project=/d", kernel_json

install/install-oscar.rb

@@ -0,0 +1,6 @@
+#!/usr/bin/env ruby
+require_relative "../settings.rb"
+require "fileutils"
+
+FileUtils.rm_rf $JULIA_ENV
+system "julia", "meta/install/install-oscar.jl"

jenkins-env.rb

@@ -0,0 +1,6 @@
+$OSCAR_CI_NAME = ENV["OSCAR_CI_NAME"] || "oscar-ci"
+$OSCAR_CI_IMAGE = ENV["OSCAR_CI_IMAGE"] || $OSCAR_CI_NAME
+$JENKINS_HOME = ENV["JENKINS_HOME"] ||
+ "#{ENV["HOME"]}/jenkins/#{ENV["OSCAR_CI_NAME"]}"
+ENV["JENKINS_HOME"] = $JENKINS_HOME
+$JENKINS_WAR = ENV["JENKINS_WAR"] || File.expand_path("#{__dir__}/jenkins.war")

layout/.gitignore

@@ -0,0 +1,5 @@
+_site
+.sass-cache
+.jekyll-cache
+.jekyll-metadata
+vendor

layout/404.html

@@ -0,0 +1,25 @@
+---
+permalink: /404.html
+layout: default
+---
+
+<style type="text/css" media="screen">
+  .container {
+    margin: 10px auto;
+    max-width: 600px;
+    text-align: center;
+  }
+  h1 {
+    margin: 30px 0;
+    font-size: 4em;
+    line-height: 1;
+    letter-spacing: -1px;
+  }
+</style>
+
+<div class="container">
+  <h1>404</h1>
+
+  <p><strong>Page not found :(</strong></p>
+  <p>The requested page could not be found.</p>
+</div>

layout/Gemfile

@@ -0,0 +1,30 @@
+source "https://rubygems.org"
+# Hello! This is where you manage which Jekyll version is used to run.
+# When you want to use a different version, change it below, save the
+# file and run `bundle install`. Run Jekyll with `bundle exec`, like so:
+#
+#     bundle exec jekyll serve
+#
+# This will help ensure the proper Jekyll version is running.
+# Happy Jekylling!
+# gem "jekyll", "~> 4.0.0"
+# This is the default theme for new Jekyll sites. You may change this to anything you like.
+# gem "minima", "~> 2.5"
+# If you want to use GitHub Pages, remove the "gem "jekyll"" above and
+# uncomment the line below. To upgrade, run `bundle update github-pages`.
+gem "github-pages", group: :jekyll_plugins
+# If you have any plugins, put them here!
+group :jekyll_plugins do
+  gem "jekyll-feed", "~> 0.12"
+end
+
+# Windows and JRuby does not include zoneinfo files, so bundle the tzinfo-data gem
+# and associated library.
+install_if -> { RUBY_PLATFORM =~ %r!mingw|mswin|java! } do
+  gem "tzinfo", "~> 1.2"
+  gem "tzinfo-data"
+end
+
+# Performance-booster for watching directories on Windows
+gem "wdm", "~> 0.1.1", :install_if => Gem.win_platform?
+