dockernize Travis build for Linux

.travis.yml

@@ -1,124 +1,50 @@
 dist: trusty
 sudo: required
 language: cpp
+services:
+  - docker
 addons:
 matrix:
     include:
         - os: osx
+          env: PLATFORM=macosx-x86_64
         - os: osx
-          env: STATIC=1
+          env: PLATFORM=macosx-x86_64 LINK_STATIC=true
         - os: linux
-          addons:
-              apt:
-                  sources:
-                      - ubuntu-toolchain-r-test
-                      - sourceline: 'ppa:maarten-fonville/protobuf'
-                  packages:
-                      - gcc-7
-                      - g++-7
-                      - cmake-data
-                      - cmake
-                      - libopencv-dev
-                      - libprotobuf-dev
-                      - protobuf-compiler
-              coverity_scan:
-                  project:
-                      name: "pfnet-research/menoh"
-                      description: "Menoh: DNN inference library"
-                  notification_email: [email protected]
-                  build_command_prepend: >-
-                      cov-configure --compiler /usr/bin/g++-7 --comptype g++ -- -march=native -fPIC -std=gnu++14 &&
-                      cmake .
-                  build_command: make
-                  branch_pattern: coverity_scan
+          env: PLATFORM=linux-x86_64 BUILDENV_IMAGE=okapies/buildenv:linux-x64-devtoolset-6
+#          addons:
+#              coverity_scan:
+#                  project:
+#                      name: "pfnet-research/menoh"
+#                      description: "Menoh: DNN inference library"
+#                  notification_email: [email protected]
+#                  build_command_prepend: >-
+#                      cov-configure --compiler /usr/bin/g++-7 --comptype g++ -- -march=native -fPIC -std=gnu++14 &&
+#                      cmake .
+#                  build_command: make
+#                  branch_pattern: coverity_scan
         - os: linux
-          env: STATIC=1
-          addons:
-              apt:
-                  sources:
-                      - ubuntu-toolchain-r-test
-                  packages:
-                      - gcc-7
-                      - g++-7
-                      - cmake-data
-                      - cmake
-                      - libopencv-dev
+          env: PLATFORM=linux-x86_64 BUILDENV_IMAGE=okapies/buildenv:linux-x64-devtoolset-6 LINK_STATIC=true
+
 env:
     global:
+        - PROTOBUF_VERSION: 3.6.1
+        - MKLDNN_VERSION: 0.16
+        - MAKE_JOBS: 2
         # The next declaration is the encrypted COVERITY_SCAN_TOKEN, created
         #   via the "travis encrypt" command using the project repo's public key
-        - secure: "q1I4YsB5VcNaF9Unmm6T92ht9/KwOGbxZVCpXIp5XUVulpaZq7sTd3rL1v3r1mUCYaabkcy9N4UPQjJZsuOlU4jc8zPzPxPir7hOER5umlkfSMuc1RhmShT8cK9naznqv7FLSTIjTZIao85Lrgxgw0B6xzcWc0kSeJPJVAmS5kwmC/FCQS2MPQpyhfE5JjpUrePOT+lRTB6Psm5bWyEww8bPsatO2k5b8DDdmUJIxmuJ1UTCx5rj/ZcTJLWAsj8D7u9aUfCmOhV5+hqHBvJd/06FLt254SNmvzmVLW9CVU/aZvuTtRECgBYCVndR7NxWpRHo1SBKqgLu+cNOFoFyt++1V+FAbpxj9JMktZNyxWp22c/FvBBdHynOsxBxVFdGIzhcwhQMiHFLOK3pnyiByabtINhERqrszkbpztOepBE3o8PGpjOz8iIx1TtLgmWwAw5D6WXx8FeP5FMkJwpXckCMI5tX5wPoU8cpZIwPjCxG3Z+ojHw+80pQWCrMZnEDfcf9zskJNsmv/GbiWGEvI8xVG0gst5VmjaAXK7JhC0cKvPOEmCFRGY+BWdjD3dkYIIElUmBRfTRDpcDJV6j5r1xMv7QKRFDfAjnC33KLJo2aALZTrkRPveIP2h2jU13ZbemN8GKWwEWNzidmwtCbH4rpe80rFqASWkyfii7HrEI="
-before_install:
-    - |
-      if [ "$TRAVIS_OS_NAME" = "linux" ]; then
-        echo -n | openssl s_client -connect scan.coverity.com:443 | sed -ne '/-BEGIN CERTIFICATE-/,/-END CERTIFICATE-/p' | sudo tee -a /etc/ssl/certs/ca-
-      fi
-install:
-    - |
-      if [ "$TRAVIS_OS_NAME" = "linux" ]; then
-        curl -LO https://github.com/pfnet-research/menoh/releases/download/v1.0.3/ubuntu1404_mkl-dnn_0.16-1_amd64.deb
-        curl -LO https://github.com/pfnet-research/menoh/releases/download/v1.0.3/ubuntu1404_mkl-dnn-dev_0.16-1_amd64.deb
-        sudo dpkg -i *.deb
-      fi
-    - |
-      if [ "$TRAVIS_OS_NAME" = "osx" ]; then
-        brew update
-        brew upgrade python
-        export PATH=/usr/local/opt/python/libexec/bin:$PATH
-        brew install numpy || true
-        brew install opencv mkl-dnn
-        if [ -z "$STATIC" ]; then brew install protobuf; fi
-      else
-        pyenv local 3.6
-      fi
-    - if [ "$TRAVIS_OS_NAME" = "linux" -a "$CXX" = "g++" ]; then export CXX="g++-7" CC="gcc-7"; fi
-    - mkdir -p data
-    - pip install --user chainer
-    - python retrieve_data.py
-    - python gen_test_data.py
-before_script:
-    - |
-      echo 'Checking tools and libraries version:'
-      if [ "$TRAVIS_OS_NAME" = "linux" ]; then
-        apt list --installed
-        ldconfig -p
-      fi
-      if [ "$TRAVIS_OS_NAME" = "osx" ]; then
-        brew list --versions
-      fi
-      cmake --version
-      make --version
+#        - secure: "q1I4YsB5VcNaF9Unmm6T92ht9/KwOGbxZVCpXIp5XUVulpaZq7sTd3rL1v3r1mUCYaabkcy9N4UPQjJZsuOlU4jc8zPzPxPir7hOER5umlkfSMuc1RhmShT8cK9naznqv7FLSTIjTZIao85Lrgxgw0B6xzcWc0kSeJPJVAmS5kwmC/FCQS2MPQpyhfE5JjpUrePOT+lRTB6Psm5bWyEww8bPsatO2k5b8DDdmUJIxmuJ1UTCx5rj/ZcTJLWAsj8D7u9aUfCmOhV5+hqHBvJd/06FLt254SNmvzmVLW9CVU/aZvuTtRECgBYCVndR7NxWpRHo1SBKqgLu+cNOFoFyt++1V+FAbpxj9JMktZNyxWp22c/FvBBdHynOsxBxVFdGIzhcwhQMiHFLOK3pnyiByabtINhERqrszkbpztOepBE3o8PGpjOz8iIx1TtLgmWwAw5D6WXx8FeP5FMkJwpXckCMI5tX5wPoU8cpZIwPjCxG3Z+ojHw+80pQWCrMZnEDfcf9zskJNsmv/GbiWGEvI8xVG0gst5VmjaAXK7JhC0cKvPOEmCFRGY+BWdjD3dkYIIElUmBRfTRDpcDJV6j5r1xMv7QKRFDfAjnC33KLJo2aALZTrkRPveIP2h2jU13ZbemN8GKWwEWNzidmwtCbH4rpe80rFqASWkyfii7HrEI="
+
+cache:
+  directories:
+    - ${HOME}/downloads
+    - ${HOME}/build/protobuf-${PROTOBUF_VERSION}
+    - ${HOME}/build/mkl-dnn-${MKLDNN_VERSION}
+
+#before_install:
+#    - |
+#      if [ "$TRAVIS_OS_NAME" = "linux" ]; then
+#        echo -n | openssl s_client -connect scan.coverity.com:443 | sed -ne '/-BEGIN CERTIFICATE-/,/-END CERTIFICATE-/p' | sudo tee -a /etc/ssl/certs/ca-
+#      fi
 script:
-    #- if [ -f cov-int/build-log.txt ]; then cat cov-int/build-log.txt; fi
-    # CMakeCache.txt generated for coverity_scan build hinders out-of-source build
-    - if [ -f CMakeCache.txt ]; then rm CMakeCache.txt; fi
-    - |
-      if [ -n "$STATIC" ]; then
-        if [ "$TRAVIS_OS_NAME" = "osx" ]; then
-          STATIC_OPTION="-DLINK_STATIC_LIBPROTOBUF=ON"
-        else
-          STATIC_OPTION="-DLINK_STATIC_LIBPROTOBUF=ON -DLINK_STATIC_LIBSTDCXX=ON -DLINK_STATIC_LIBGCC=ON"
-        fi
-      else
-        STATIC_OPTION=""
-      fi
-    - mkdir build
-    - cd build
-    - |
-      if [ "$TRAVIS_OS_NAME" == "osx" ]; then
-        cmake -DENABLE_TEST=ON \
-        $STATIC_OPTION \
-        ..
-      else
-        cmake -DENABLE_TEST=ON \
-        $STATIC_OPTION \
-        ..
-      fi
-    - make
-    - ./test/menoh_test
-    - |
-      if [ "$TRAVIS_OS_NAME" == "osx" ]; then
-        otool -L menoh/libmenoh.dylib
-      else
-        ldd menoh/libmenoh.so
-      fi
+    - bash -ex ${TRAVIS_BUILD_DIR}/.travis/run-build.sh

.travis/README.md

@@ -0,0 +1,50 @@
+# Build scripts for Travis
+
+## Prerequisites
+It requires a container image with the following softwares:
+
+- GCC 5.2 or later
+- CMake 3.1 or later
+- Python 3.6
+- pip
+- OpenCV 2.4 or later
+
+Currently it uses [okapies/buildenv](https://hub.docker.com/r/okapies/buildenv/) image for linux-x86_64 platform.
+
+## Architecture
+`.travis.yml` -> `run-build.sh` -> `build.sh` -> `install-*.sh` & `build-menoh.sh`
+
+1. `run-build.sh` starts a Docker container for building the project
+    - You can access `${HOME}` (`/home/travis`) directory from the container transparently because it maps `${HOME}` in the Travis environment to container's `/home/travis`
+2. `build.sh` runs a build workflow *in the container*
+    - (All commands are run by `docker_exec` and `docker_exec_cmd` functions)
+    - Install the prerequisites
+    - Run a build
+    - Run a test
+3. Release and clean up
+
+## Directory
+- `/home/travis` (= `${HOME}`)
+    - `/downloads` (cache)
+    - `/build`
+        - `/protobuf-<ver>` (cache)
+        - `/mkl-dnn-<ver>` (cache)
+        - `/<user>/<repo>` (= `${TRAVIS_BUILD_DIR}`)
+            - `/menoh`
+            - `/test`
+            - `/cmake`
+            - `CMakeLists.txt`
+            - ...
+            - `/build`
+                - `/menoh`
+                - `/test`
+                - ...
+
+## Cache
+```yaml
+cache:
+  directories:
+    - ${HOME}/downloads
+    - ${HOME}/build/protobuf-${PROTOBUF_VERSION}
+    - ${HOME}/build/mkl-dnn-${MKLDNN_VERSION}
+```

.travis/init-build-linux.sh

@@ -0,0 +1,79 @@
+# check if variables are set
+test -n "${DOCKER_CONTAINER_ID}" || { echo "DOCKER_CONTAINER_ID does not exist"; exit 1; }
+test -n "${PROTOBUF_VERSION}" || { echo "PROTOBUF_VERSION does not exist"; exit 1; }
+test -n "${MKLDNN_VERSION}" || { echo "MKLDNN_VERSION does not exist"; exit 1; }
+test -n "${MAKE_JOBS}" || { echo "MAKE_JOBS does not exist"; exit 1; }
+
+test -n "${LINK_STATIC}" || LINK_STATIC=false
+
+# TODO: make them configurable for outside Travis
+export WORK_DIR=${HOME}
+export PROJ_DIR=${TRAVIS_BUILD_DIR} # = ${HOME}/build/${TRAVIS_REPO_SLUG}
+
+export PROTOBUF_INSTALL_DIR=/usr/local
+export MKLDNN_INSTALL_DIR=/usr/local
+
+# define shared functions for Linux-based platforms
+function docker_exec() {
+    docker exec -it ${DOCKER_CONTAINER_ID} /bin/bash -xec "$1"
+}
+
+function docker_exec_cmd() {
+    docker exec -it ${DOCKER_CONTAINER_ID} /bin/bash -xe $@
+}
+
+function install_protobuf() {
+    docker_exec_cmd \
+        ${PROJ_DIR}/.travis/install-protobuf.sh \
+            --version ${PROTOBUF_VERSION} \
+            --download-dir ${WORK_DIR}/downloads \
+            --build-dir ${WORK_DIR}/build \
+            --install-dir ${PROTOBUF_INSTALL_DIR} \
+            --parallel ${MAKE_JOBS}
+}
+
+function install_mkldnn() {
+    docker_exec_cmd \
+        ${PROJ_DIR}/.travis/install-mkldnn.sh \
+            --version ${MKLDNN_VERSION} \
+            --download-dir ${WORK_DIR}/downloads \
+            --build-dir ${WORK_DIR}/build \
+            --install-dir ${MKLDNN_INSTALL_DIR} \
+            --parallel ${MAKE_JOBS}
+}
+
+function prepare_menoh_data() {
+    echo -e "\e[33;1mPreparing data/ for Menoh\e[0m"
+    docker_exec "$(cat << EOS
+        cd ${PROJ_DIR}/menoh && \
+        ([ -d "data" ] || mkdir -p data) && \
+        python3 retrieve_data.py && \
+        python3 gen_test_data.py
+EOS
+)"
+}
+
+function build_menoh() {
+    # CMakeCache.txt generated for coverity_scan build hinders out-of-source build
+    docker_exec "if [ -f \"CMakeCache.txt\" ]; then cd ${PROJ_DIR} && rm CMakeCache.txt; fi"
+    if [ "${LINK_STATIC}" != "true" ]; then
+        docker_exec_cmd \
+            ${PROJ_DIR}/.travis/build-menoh.sh \
+                --source-dir ${PROJ_DIR}
+    else
+        docker_exec_cmd \
+            ${PROJ_DIR}/.travis/build-menoh.sh \
+                --source-dir ${PROJ_DIR} \
+                --link-static-libgcc ON \
+                --link-static-libstdcxx ON \
+                --link-static-libprotobuf ON
+    fi
+}
+
+function test_menoh() {
+    docker_exec "cd ${PROJ_DIR}/menoh/build && ./test/menoh_test"
+}
+
+function check_menoh_artifact() {
+    ldd ${PROJ_DIR}/menoh/build/menoh/libmenoh.so
+}

.travis/init-build-osx.sh

@@ -0,0 +1,36 @@
+# check if variables are set
+test -n "${MAKE_JOBS}" || { echo "MAKE_JOBS does not exist"; exit 1; }
+
+# TODO: make them configurable for outside Travis
+export WORK_DIR=${HOME}
+export PROJ_DIR=${TRAVIS_BUILD_DIR} # = ${HOME}/build/${TRAVIS_REPO_SLUG}
+
+function prepare_menoh_data() {
+    echo -e "\e[33;1mPreparing data/ for Menoh\e[0m"
+
+    cd ${PROJ_DIR}/menoh
+    [ -d "data" ] || mkdir -p data
+
+    python retrieve_data.py
+    python gen_test_data.py
+}
+
+function build_menoh() {
+    if [ "${LINK_STATIC}" != "true" ]; then
+        ${PROJ_DIR}/.travis/build-menoh.sh \
+            --source-dir ${PROJ_DIR}
+    else
+    	# Does not set --link-static-libgcc and --link-static-libstdcxx in macOS
+        ${PROJ_DIR}/.travis/build-menoh.sh \
+            --source-dir ${PROJ_DIR} \
+            --link-static-libprotobuf ON
+    fi
+}
+
+function test_menoh() {
+    cd ${PROJ_DIR}/menoh/build && ./test/menoh_test
+}
+
+function check_menoh_artifact() {
+    otool -L ${PROJ_DIR}/menoh/build/menoh/libmenoh.dylib
+}