Merge remote-tracking branch 'origin/master' into feature/configure-f…

…ix-iface
pfnet-research · Oct 5, 2018 · cc57e9d · cc57e9d
2 parents 2375e7d + 8fed3d0
commit cc57e9d
Show file tree

Hide file tree

Showing 64 changed files with 2,739 additions and 1,979 deletions.
diff --git a/.gitmodules b/.gitmodules
@@ -1,6 +1,10 @@
 [submodule "external/onnx"]
 	path = external/onnx
-	url = https://github.com/onnx/onnx
+	url = https://github.com/onnx/onnx.git
+	ignore = dirty
 [submodule "test/lib/googletest"]
 	path = test/lib/googletest
 	url = https://github.com/google/googletest.git
+[submodule "test/lib/filesystem"]
+	path = test/lib/filesystem
+	url = https://github.com/wjakob/filesystem.git
diff --git a/.travis.yml b/.travis.yml
@@ -60,12 +60,15 @@ matrix:
           install:
               - if [ "$TRAVIS_OS_NAME" = "linux" -a "$CXX" = "g++" ]; then export CXX="g++-7" CC="gcc-7"; fi
               - |
-                bash -ex ${TRAVIS_BUILD_DIR}/.travis/install-mkldnn.sh \
+                bash -ex ${TRAVIS_BUILD_DIR}/scripts/build-mkldnn.sh \
                     --version ${MKLDNN_VERSION} \
                     --download-dir ${HOME}/downloads \
-                    --build-dir ${HOME}/build \
+                    --extract-dir ${HOME}/build \
                     --install-dir ${HOME}/mkl-dnn-${MKLDNN_VERSION} \
                     --parallel ${MAKE_JOBS}
+              - |
+                bash -ex ${TRAVIS_BUILD_DIR}/scripts/install-mkldnn.sh \
+                    --build-dir ${HOME}/build/mkl-dnn-${MKLDNN_VERSION}/build
           script: true # skip build and test
 
 script:

diff --git a/.travis/README.md b/.travis/README.md
@@ -12,7 +12,7 @@ Linux-based platforms requires a container image with the following softwares:
 Currently it uses [okapies/buildenv](https://hub.docker.com/r/okapies/buildenv/) image for linux-x86_64 platform.
 
 ## Architecture
-`.travis.yml` -> `run-build.sh` -> `build.sh` -> `install-*.sh` & `build-menoh.sh`
+`.travis.yml` -> `run-build.sh` -> `build.sh` -> `scripts/*.sh`
 
 1. `run-build.sh` just calls a platform's `build.sh` for running the actual build workflow
 2. `build.sh`

diff --git a/.travis/build-menoh.sh b/.travis/build-menoh.sh
diff --git a/.travis/init-build-linux.sh b/.travis/init-build-linux.sh
@@ -16,8 +16,8 @@ export MKLDNN_INSTALL_DIR=/usr/local
 # $HOME:$HOME = /home/travis                     : /home/travis
 #               /home/travis/build               : /home/travis/build
 #               /home/travis/build/<user>/<repo> : /home/travis/build/<user>/<repo> (= ${TRAVIS_BUILD_DIR})
-SHARED_SCRIPT_DIR=$(cd $(dirname ${BASH_SOURCE:-$0}); pwd)
-source ${SHARED_SCRIPT_DIR}/run-container.sh --image ${BUILDENV_IMAGE} --work-dir ${WORK_DIR}
+SOURCE_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}/..")"; pwd)
+source ${SOURCE_DIR}/scripts/run-container.sh --image ${BUILDENV_IMAGE} --work-dir ${WORK_DIR}
 test -n "${BUILDENV_CONTAINER_ID}" || { echo "BUILDENV_CONTAINER_ID can't be empty" 1>&2; exit 1; }
 
 ## define shared functions for Linux-based platforms
@@ -34,52 +34,68 @@ function docker_exec_script() {
     return $?
 }
 
-function install_protobuf() {
+function build_protobuf() {
     docker_exec_script \
-        ${PROJ_DIR}/.travis/install-protobuf.sh \
+        "${PROJ_DIR}/scripts/build-protobuf.sh" \
             --version ${PROTOBUF_VERSION} \
-            --download-dir ${WORK_DIR}/downloads \
-            --build-dir ${WORK_DIR}/build \
-            --install-dir ${PROTOBUF_INSTALL_DIR} \
+            --download-dir "${WORK_DIR}/downloads" \
+            --extract-dir "${WORK_DIR}/build" \
+            --install-dir "${PROTOBUF_INSTALL_DIR}" \
             --parallel ${MAKE_JOBS}
 }
 
-function install_mkldnn() {
+function install_protobuf() {
+    docker_exec_script \
+        "${PROJ_DIR}/scripts/install-protobuf.sh" \
+            --build-dir "${WORK_DIR}/build/protobuf-${PROTOBUF_VERSION}"
+}
+
+function build_mkldnn() {
     docker_exec_script \
-        ${PROJ_DIR}/.travis/install-mkldnn.sh \
+        "${PROJ_DIR}/scripts/build-mkldnn.sh" \
             --version ${MKLDNN_VERSION} \
-            --download-dir ${WORK_DIR}/downloads \
-            --build-dir ${WORK_DIR}/build \
-            --install-dir ${MKLDNN_INSTALL_DIR} \
+            --download-dir "${WORK_DIR}/downloads" \
+            --extract-dir "${WORK_DIR}/build" \
+            --install-dir "${MKLDNN_INSTALL_DIR}" \
             --parallel ${MAKE_JOBS}
 }
 
+function install_mkldnn() {
+    docker_exec_script \
+        "${PROJ_DIR}/scripts/install-mkldnn.sh" \
+            --build-dir "${WORK_DIR}/build/mkl-dnn-${MKLDNN_VERSION}/build"
+}
+
 function prepare_menoh_data() {
     docker_exec_script \
-        ${PROJ_DIR}/.travis/prepare-menoh-data.sh \
-            --source-dir ${PROJ_DIR} \
+        "${PROJ_DIR}/scripts/prepare-menoh-data.sh" \
+            --source-dir "${PROJ_DIR}" \
             --python-executable python3
 }
 
 function build_menoh() {
     if [ "${LINK_STATIC}" != "true" ]; then
         docker_exec_script \
-            ${PROJ_DIR}/.travis/build-menoh.sh \
-                --source-dir ${PROJ_DIR}
+            "${PROJ_DIR}/scripts/build-menoh.sh" \
+                --build-type Release \
+                --source-dir "${PROJ_DIR}" \
+                --python-executable python3
     else
         docker_exec_script \
-            ${PROJ_DIR}/.travis/build-menoh.sh \
-                --source-dir ${PROJ_DIR} \
+            "${PROJ_DIR}/scripts/build-menoh.sh" \
+                --build-type Release \
+                --source-dir "${PROJ_DIR}" \
+                --python-executable python3 \
                 --link-static-libgcc ON \
                 --link-static-libstdcxx ON \
                 --link-static-libprotobuf ON
     fi
 }
 
 function test_menoh() {
-    docker_exec "cd ${PROJ_DIR}/build && ./test/menoh_test"
+    docker_exec "cd \"${PROJ_DIR}/build\" && ./test/menoh_test"
 }
 
 function check_menoh_artifact() {
-    docker_exec "ldd ${PROJ_DIR}/build/menoh/libmenoh.so"
+    docker_exec "ldd \"${PROJ_DIR}/build/menoh/libmenoh.so\""
 }
diff --git a/.travis/init-build-osx.sh b/.travis/init-build-osx.sh
@@ -8,28 +8,32 @@ export PROJ_DIR=${TRAVIS_BUILD_DIR} # = ${HOME}/build/${TRAVIS_REPO_SLUG}
 ## define shared functions for macOS (OSX) platforms
 
 function prepare_menoh_data() {
-    bash -ex ${PROJ_DIR}/.travis/prepare-menoh-data.sh \
-        --source-dir ${PROJ_DIR} \
+    bash -ex "${PROJ_DIR}/scripts/prepare-menoh-data.sh" \
+        --source-dir "${PROJ_DIR}" \
         --python-executable python
 }
 
 function build_menoh() {
     if [ "${LINK_STATIC}" != "true" ]; then
-        bash -ex ${PROJ_DIR}/.travis/build-menoh.sh \
-            --source-dir ${PROJ_DIR}
+        bash -ex "${PROJ_DIR}/scripts/build-menoh.sh" \
+            --build-type Release \
+            --source-dir "${PROJ_DIR}" \
+            --python-executable python
     else
         # Does not set --link-static-libgcc and --link-static-libstdcxx in macOS
-        bash -ex ${PROJ_DIR}/.travis/build-menoh.sh \
-            --source-dir ${PROJ_DIR} \
+        bash -ex "${PROJ_DIR}/scripts/build-menoh.sh" \
+            --build-type Release \
+            --source-dir "${PROJ_DIR}" \
+            --python-executable python \
             --link-static-libprotobuf ON
     fi
 }
 
 function test_menoh() {
-    cd ${PROJ_DIR}/build
+    cd "${PROJ_DIR}/build"
     ./test/menoh_test
 }
 
 function check_menoh_artifact() {
-    otool -L ${PROJ_DIR}/build/menoh/libmenoh.dylib
+    otool -L "${PROJ_DIR}/build/menoh/libmenoh.dylib"
 }
diff --git a/.travis/linux-x86_64/build.sh b/.travis/linux-x86_64/build.sh
@@ -1,6 +1,6 @@
 #!/bin/bash -ex
 
-BASE_DIR=$(cd $(dirname ${BASH_SOURCE:-$0}); pwd)
+BASE_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd)
 
 # initialize this script
 source ${BASE_DIR}/../init-build-linux.sh
@@ -13,7 +13,9 @@ docker_exec "ls -l ${WORK_DIR}/build/${TRAVIS_REPO_SLUG}"
 docker_exec "(printenv | grep PATH) && make --version && cmake --version && g++ --version && ldd --version"
 
 # build and install prerequisites
+build_protobuf
 install_protobuf
+build_mkldnn
 install_mkldnn
 
 docker_exec "pip3 install --user chainer" # for generating test data

diff --git a/.travis/macosx-x86_64/build.sh b/.travis/macosx-x86_64/build.sh
@@ -1,6 +1,6 @@
 #!/bin/bash
 
-BASE_DIR=$(cd $(dirname ${BASH_SOURCE:-$0}); pwd)
+BASE_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd)
 
 # initialize this script
 source ${BASE_DIR}/../init-build-osx.sh

diff --git a/BUILDING.md b/BUILDING.md
@@ -5,18 +5,25 @@ You need to install [prerequisites](#prerequisites) for your platform before [bu
 To build Menoh, you require the following toolchains:
 
 Unix:
-- CMake 3.1 or later
 - GCC 4.9 or later
+- CMake 3.1 or later
+- Python 2.7 or later
 
 macOS (OSX):
 - XCode
 - [Homebrew](https://brew.sh/)
+- CMake 3.1 or later
+- Python 2.7 or later
 
 Windows:
 - Visual Studio 2015
+- CMake 3.1 or later
+- Python 2.7 or later
 
 Windows (MINGW):
 - [MSYS2](http://www.msys2.org/)
+- CMake 3.1 or later
+- Python 2.7 or later
 
 You also need to install the dependent libraries on your system:
 
@@ -25,7 +32,7 @@ You also need to install the dependent libraries on your system:
 
 `protobuf` can be installed through most package managers instead of building it yourself. `mkl-dnn` package, unfortunatelly, is not available in many environments at the moment (except for `brew` in macOS).
 
-Note that you can use ProtoBuf either version 2 or 3, but, for example, if you build Menoh with `protoc` ver 3 you should use the binary with runtime ver 3.
+Note that you can use ProtoBuf either version 2 or 3, but the runtime version should be the same as `protoc` in your system.
 
 ### Debian/Ubuntu
 ```
@@ -53,10 +60,9 @@ Download and unzip https://github.com/protocolbuffers/protobuf/releases/download
 
 ```
 cd protobuf-3.6.1/cmake
-mdir build
+mkdir build
 cd build
 cmake .. -G "Visual Studio 14" -A x64 -Dprotobuf_MSVC_STATIC_RUNTIME=OFF -Dprotobuf_BUILD_TESTS=OFF -DCMAKE_INSTALL_PREFIX=(CMake_Install_Dir)
-cmake --build . --config Debug --target install
 cmake --build . --config Release --target install
 cd ../../..
 ```
@@ -67,26 +73,25 @@ git clone https://github.com/intel/mkl-dnn.git
 cd mkl-dnn/scripts
 .\prepare_mkl.bat
 cd ..
-mdir build
+mkdir build
 cd build
 cmake .. -G "Visual Studio 14 Win64"  -DCMAKE_INSTALL_PREFIX=(CMake_Install_Dir)
-cmake --build . --config Debug --target install
 cmake --build . --config Release --target install
 cd ../..
 ```
 
 ### Windows (MINGW)
 ```
 pacman -S mingw-w64-x86_64-toolchain
-pacman -S git
+pacman -S git make
 pacman -S mingw-w64-x86_64-cmake
 pacman -S mingw-w64-x86_64-protobuf mingw-w64-x86_64-protobuf-c
 ```
 
 #### Installing MKL-DNN from binary package
 ```
 curl -omingw-w64-x86_64-mkl-dnn-0.15-1-x86_64.pkg.tar.xz -L https://github.com/pfnet-research/menoh/releases/download/v1.0.3/mingw-w64-x86_64-mkl-dnn-0.15-1-x86_64.pkg.tar.xz
-pacman -S --noconfirm mingw-w64-x86_64-mkl-dnn-0.15-1-x86_64.pkg.tar.xz
+pacman -U --noconfirm mingw-w64-x86_64-mkl-dnn-0.15-1-x86_64.pkg.tar.xz
 ```
 
 #### Installing MKL-DNN from source
@@ -124,7 +129,7 @@ make install # as root
 To run the example, you also need to download model data:
 
 ```
-python retrieve_data.py
+python scripts/retrieve_data.py
 ```
 
 #### Static linking
@@ -177,10 +182,9 @@ Please replace `(CMake_Install_Dir)` in the following with your working director
 ```
 git clone https://github.com/pfnet-research/menoh.git
 cd menoh
-mdir build
+mkdir build
 cd build
-cmake .. -G "Visual Studio 14 Win64" -DCMAKE_PREFIX_PATH=CMake_Install_Dir) -DCMAKE_INSTALL_PREFIX=CMake_Install_Dir) -DENABLE_TEST=OFF -DENABLE_BENCHMARK=OFF -DENABLE_EXAMPLE=OFF -DENABLE_TOOL=OFF
-cmake --build . --config Debug --target install
+cmake .. -G "Visual Studio 14 Win64" -DCMAKE_PREFIX_PATH=(CMake_Install_Dir) -DCMAKE_INSTALL_PREFIX=(CMake_Install_Dir) -DENABLE_TEST=OFF -DENABLE_BENCHMARK=OFF -DENABLE_EXAMPLE=OFF
 cmake --build . --config Release --target install
 ```
 
@@ -194,3 +198,12 @@ MSYS2_ARG_CONV_EXCL="-DCMAKE_INSTALL_PREFIX=" \
   cmake -G "MSYS Makefiles" -DCMAKE_INSTALL_PREFIX=/mingw64
 make
 ```
+
+### Note
+
+#### Python command name
+Menoh requires `python` command to generate source codes at build time. Add `PYTHON_EXECUTABLE` option to `cmake` if you want to use `python` command with non-standard name (e.g. `python3`).
+
+```bash
+cmake -DPYTHON_EXECUTABLE=python3 ..
+```