Merge pull request #120 from pfnet-research/feature/libonnx

Switch to use libonnx directly

.gitmodules

@@ -1,6 +1,7 @@
 [submodule "external/onnx"]
 	path = external/onnx
-	url = https://github.com/onnx/onnx
+	url = https://github.com/onnx/onnx.git
+	ignore = dirty
 [submodule "test/lib/googletest"]
 	path = test/lib/googletest
 	url = https://github.com/google/googletest.git

.travis/init-build-linux.sh

@@ -78,12 +78,14 @@ function build_menoh() {
         docker_exec_script \
             "${PROJ_DIR}/scripts/build-menoh.sh" \
                 --build-type Release \
-                --source-dir "${PROJ_DIR}"
+                --source-dir "${PROJ_DIR}" \
+                --python-executable python3
     else
         docker_exec_script \
             "${PROJ_DIR}/scripts/build-menoh.sh" \
                 --build-type Release \
                 --source-dir "${PROJ_DIR}" \
+                --python-executable python3 \
                 --link-static-libgcc ON \
                 --link-static-libstdcxx ON \
                 --link-static-libprotobuf ON

.travis/init-build-osx.sh

@@ -17,12 +17,14 @@ function build_menoh() {
     if [ "${LINK_STATIC}" != "true" ]; then
         bash -ex "${PROJ_DIR}/scripts/build-menoh.sh" \
             --build-type Release \
-            --source-dir "${PROJ_DIR}"
+            --source-dir "${PROJ_DIR}" \
+            --python-executable python
     else
         # Does not set --link-static-libgcc and --link-static-libstdcxx in macOS
         bash -ex "${PROJ_DIR}/scripts/build-menoh.sh" \
             --build-type Release \
             --source-dir "${PROJ_DIR}" \
+            --python-executable python \
             --link-static-libprotobuf ON
     fi
 }

BUILDING.md

@@ -5,18 +5,25 @@ You need to install [prerequisites](#prerequisites) for your platform before [bu
 To build Menoh, you require the following toolchains:
 
 Unix:
-- CMake 3.1 or later
 - GCC 4.9 or later
+- CMake 3.1 or later
+- Python 2.7 or later
 
 macOS (OSX):
 - XCode
 - [Homebrew](https://brew.sh/)
+- CMake 3.1 or later
+- Python 2.7 or later
 
 Windows:
 - Visual Studio 2015
+- CMake 3.1 or later
+- Python 2.7 or later
 
 Windows (MINGW):
 - [MSYS2](http://www.msys2.org/)
+- CMake 3.1 or later
+- Python 2.7 or later
 
 You also need to install the dependent libraries on your system:
 
@@ -53,10 +60,9 @@ Download and unzip https://github.com/protocolbuffers/protobuf/releases/download
 
 ```
 cd protobuf-3.6.1/cmake
-mdir build
+mkdir build
 cd build
 cmake .. -G "Visual Studio 14" -A x64 -Dprotobuf_MSVC_STATIC_RUNTIME=OFF -Dprotobuf_BUILD_TESTS=OFF -DCMAKE_INSTALL_PREFIX=(CMake_Install_Dir)
-cmake --build . --config Debug --target install
 cmake --build . --config Release --target install
 cd ../../..
 ```
@@ -67,10 +73,9 @@ git clone https://github.com/intel/mkl-dnn.git
 cd mkl-dnn/scripts
 .\prepare_mkl.bat
 cd ..
-mdir build
+mkdir build
 cd build
 cmake .. -G "Visual Studio 14 Win64"  -DCMAKE_INSTALL_PREFIX=(CMake_Install_Dir)
-cmake --build . --config Debug --target install
 cmake --build . --config Release --target install
 cd ../..
 ```
@@ -177,10 +182,9 @@ Please replace `(CMake_Install_Dir)` in the following with your working director
 ```
 git clone https://github.com/pfnet-research/menoh.git
 cd menoh
-mdir build
+mkdir build
 cd build
-cmake .. -G "Visual Studio 14 Win64" -DCMAKE_PREFIX_PATH=CMake_Install_Dir) -DCMAKE_INSTALL_PREFIX=CMake_Install_Dir) -DENABLE_TEST=OFF -DENABLE_BENCHMARK=OFF -DENABLE_EXAMPLE=OFF -DENABLE_TOOL=OFF
-cmake --build . --config Debug --target install
+cmake .. -G "Visual Studio 14 Win64" -DCMAKE_PREFIX_PATH=(CMake_Install_Dir) -DCMAKE_INSTALL_PREFIX=(CMake_Install_Dir) -DENABLE_TEST=OFF -DENABLE_BENCHMARK=OFF -DENABLE_EXAMPLE=OFF
 cmake --build . --config Release --target install
 ```
 
@@ -194,3 +198,12 @@ MSYS2_ARG_CONV_EXCL="-DCMAKE_INSTALL_PREFIX=" \
   cmake -G "MSYS Makefiles" -DCMAKE_INSTALL_PREFIX=/mingw64
 make
 ```
+
+### Note
+
+#### Python command name
+Menoh requires `python` command to generate source codes at build time. Add `PYTHON_EXECUTABLE` option to `cmake` if you want to use `python` command with non-standard name (e.g. `python3`).
+
+```bash
+cmake -DPYTHON_EXECUTABLE=python3 ..
+```

CMakeLists.txt

@@ -7,8 +7,6 @@ set(MENOH_MINOR_VERSION 1)
 set(MENOH_PATCH_VERSION 0)
 
 # Options
-option(BUILD_SHARED_LIBS "Build shared libs" ON)
-
 option(USE_OLD_GLIBCXX_ABI "Generate binaries for the old libstdc++ ABI" OFF)
 option(LINK_STATIC_LIBPROTOBUF "Link static libprotobuf to libmenoh" OFF)
 
@@ -52,11 +50,37 @@ mark_as_advanced(DOWNLOAD_LOCATION)
 # Enable ExternalProject_Add
 include(ExternalProject)
 
-# Setup protobuf
-include(SetupProtobuf)
+# Setup protobuf (it is used in ONNX and Menoh)
+if(LINK_STATIC_LIBPROTOBUF)
+    # Note: We can't use `set(PROTOBUF_BUILD_SHARED_LIBS OFF)` in `FindProtobuf` module
+    # because `libprotobuf.a` produced by the package manager is not PIC. So we need to
+    # build it by ourselves.
+    if(UNIX OR MINGW)
+        include(BuildProtobuf)
+    else()
+        message(FATAL_ERROR "LINK_STATIC_LIBPROTOBUF is supported only in UNIX-like environments")
+    endif()
+else()
+    # Note: It may conflict with the loading mechanism in onnx's CMake configuration.
+    # See external/onnx/CMakeLists.txt for more details.
+    include(FindProtobuf)
+    find_package(Protobuf ${PROTOBUF_VERSION} REQUIRED)
+endif()
+
+# Build libonnx.a
+message(STATUS "Adding external/onnx")
+
+set(ONNX_SRC_DIR ${EXTERNAL_DIR}/onnx)
+execute_process(COMMAND git submodule update --init -- ${ONNX_SRC_DIR} WORKING_DIRECTORY ${CMAKE_SOURCE_DIR})
+
+# see https://github.com/onnx/onnx/pull/1466
+message(STATUS "Patching to external/onnx")
+configure_file(${EXTERNAL_DIR}/onnx-v1.3.0-patch_CMakeLists.txt ${EXTERNAL_DIR}/onnx/CMakeLists.txt COPYONLY)
 
-# Generate source codes from ONNX protobuf schema
-include(GenerateOnnxSrc)
+# TODO: enable the following option when it is ready for migrating to onnx-ml
+#set(ONNX_ML 1)
+add_subdirectory(external/onnx EXCLUDE_FROM_ALL) # Note: BUILD_SHARED_LIBS must be OFF in this place
+include_directories("${ONNX_INCLUDE_DIRS}")
 
 # Setup MKLDNN
 find_package(MKLDNN "0.14")
@@ -67,24 +91,24 @@ endif()
 include_directories("${MKLDNN_INCLUDE_DIR}")
 
 if(${ENABLE_TEST})
-    message(STATUS "# add_subdirectory\(test\)")
+    message(STATUS "Adding test")
     add_subdirectory(test)
 endif()
 
 if(${ENABLE_BENCHMARK})
-    message(STATUS "# add_subdirectory\(benchmark\)")
+    message(STATUS "Adding benchmark")
     add_subdirectory(benchmark)
 endif()
 
 if(${ENABLE_EXAMPLE})
-    message(STATUS "# add_subdirectory\(example\)")
+    message(STATUS "Adding example")
     add_subdirectory(example)
 endif()
 
-message(STATUS "# add_subdirectory\(menoh\)")
+message(STATUS "Adding menoh")
 add_subdirectory(menoh)
 
-message(STATUS "# add_subdirectory\(include\)")
+message(STATUS "Adding include")
 add_subdirectory(include)
 
 if(SHOW_ALL_VARIABLES)

appveyor.yml

@@ -72,7 +72,7 @@ build_script:
     cmake -G "MSYS Makefiles" -DENABLE_TEST=ON %STATIC_OPTION% -DCMAKE_INSTALL_PREFIX=/mingw64 .. &&
     make
   ) else (
-    cmake -G "Visual Studio 14 Win64" -DENABLE_TEST=OFF -DENABLE_BENCHMARK=OFF -DENABLE_EXAMPLE=OFF -DENABLE_TOOL=OFF -DCMAKE_INSTALL_PREFIX=c:\menoh-%MENOH_REV%-msvc .. &&
+    cmake -G "Visual Studio 14 Win64" -DENABLE_TEST=OFF -DENABLE_BENCHMARK=OFF -DENABLE_EXAMPLE=OFF -DCMAKE_INSTALL_PREFIX=c:\menoh-%MENOH_REV%-msvc .. &&
     cmake --build . --config Release --target install
   )
 

cmake/BuildProtobuf.cmake

@@ -0,0 +1,44 @@
+set(PROTOBUF_VERSION_STATIC "3.6.1")
+
+set(PROTOBUF_DIR ${CMAKE_CURRENT_BINARY_DIR}/protobuf-${PROTOBUF_VERSION_STATIC})
+set(PROTOBUF_URL "https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOBUF_VERSION_STATIC}/protobuf-cpp-${PROTOBUF_VERSION_STATIC}.tar.gz")
+set(PROTOBUF_HASH MD5=406d5b8636576b1c86730ca5cbd1e576)
+
+# Requires `-fPIC` for linking with a shared library
+set(PROTOBUF_CFLAGS -fPIC)
+set(PROTOBUF_CXXFLAGS -fPIC)
+if(USE_OLD_GLIBCXX_ABI)
+    set(PROTOBUF_CXXFLAGS "${PROTOBUF_CXXFLAGS} -D_GLIBCXX_USE_CXX11_ABI=0")
+endif()
+
+ExternalProject_Add(Protobuf
+    PREFIX ${PROTOBUF_DIR}
+    URL ${PROTOBUF_URL}
+    URL_HASH ${PROTOBUF_HASH}
+    DOWNLOAD_DIR "${DOWNLOAD_LOCATION}"
+    BUILD_IN_SOURCE 1
+    CONFIGURE_COMMAND bash -ex ${CMAKE_MODULE_PATH}/configure-helper.sh ${CMAKE_C_COMPILER} ${CMAKE_CXX_COMPILER} ${PROTOBUF_DIR} ${PROTOBUF_CFLAGS} ${PROTOBUF_CXXFLAGS}
+    BUILD_COMMAND make -j4
+    INSTALL_COMMAND make install
+)
+
+set(PROTOBUF_LIBRARY_STATIC ${PROTOBUF_DIR}/lib/libprotobuf.a)
+set(PROTOBUF_LIBRARY_SHARED ${PROTOBUF_DIR}/lib/libprotobuf.so)
+
+# Mimic the behavior of `FindProtobuf` module
+# Use the old variable names to ensure backward compatibility
+set(PROTOBUF_INCLUDE_DIR ${PROTOBUF_DIR}/include)
+set(PROTOBUF_INCLUDE_DIRS ${PROTOBUF_INCLUDE_DIR})
+set(PROTOBUF_LIBRARY ${PROTOBUF_LIBRARY_STATIC}) # use the static library
+set(PROTOBUF_LIBRARIES ${PROTOBUF_LIBRARY})
+set(PROTOBUF_PROTOC_EXECUTABLE ${PROTOBUF_DIR}/bin/protoc)
+set(PROTOBUF_FOUND TRUE)
+
+add_library(protobuf::libprotobuf UNKNOWN IMPORTED)
+# Note: INTERFACE_INCLUDE_DIRECTORIES can't set in this place because include/ is
+# not installed during executing `cmake`
+set_target_properties(protobuf::libprotobuf PROPERTIES
+    IMPORTED_LOCATION "${PROTOBUF_LIBRARY_STATIC}")
+add_executable(protobuf::protoc IMPORTED)
+set_target_properties(protobuf::protoc PROPERTIES
+    IMPORTED_LOCATION "${PROTOBUF_PROTOC_EXECUTABLE}")

cmake/ConfigureMenoh.cmake

@@ -8,13 +8,13 @@ macro(menoh_link_libraries TARGET_NAME SCOPE)
         target_link_libraries(${TARGET_NAME} ${SCOPE} -static-libstdc++)
     endif()
 
+    target_link_libraries(${TARGET_NAME} ${SCOPE} onnx) # onnx also contains protobuf
+
     if(NOT ${SCOPE})
         # PUBLIC will add transitive dependencies (`mklml_intel` and `iomp5`) to the link interface
         # Note: change it to PRIVATE after building mkldnn itself
         target_link_libraries(${TARGET_NAME} PUBLIC ${MKLDNN_LIBRARIES})
     else()
         target_link_libraries(${TARGET_NAME} ${MKLDNN_LIBRARIES})
     endif()
-
-    target_link_libraries(${TARGET_NAME} ${SCOPE} ${PROTOBUF_LIBRARIES})
 endmacro()

external/cmdline.h

@@ -39,8 +39,12 @@
 
 #ifdef _MSC_VER
 #define NOMINMAX
-#include <DbgHelp.h>
+/*
+  You must include Windows.h before DbgHelp.h.
+  See https://stackoverflow.com/a/43283926/1014818 for more details.
+*/
 #include <Windows.h>
+#include <DbgHelp.h>
 #else
 #include <cxxabi.h>
 #endif