From 08289fcf5b99e470e11eb09669ec13a57a4f811c Mon Sep 17 00:00:00 2001 From: Yuta Okamoto Date: Mon, 1 Oct 2018 10:29:14 +0900 Subject: [PATCH 01/14] add cross building support for protobuf --- scripts/build-boost.sh | 120 ++++++++++++++++++++++++++++++++++++++ scripts/build-menoh.sh | 4 +- scripts/build-protobuf.sh | 18 +++++- scripts/install-boost.sh | 35 +++++++++++ 4 files changed, 174 insertions(+), 3 deletions(-) create mode 100644 scripts/build-boost.sh create mode 100644 scripts/install-boost.sh diff --git a/scripts/build-boost.sh b/scripts/build-boost.sh new file mode 100644 index 0000000..183247f --- /dev/null +++ b/scripts/build-boost.sh @@ -0,0 +1,120 @@ +#!/bin/bash -e + +# retrieve arguments +while [[ $# != 0 ]]; do + case $1 in + --) + shift + break + ;; + --version) + readonly ARG_VERSION="$2" + shift 2 + ;; + --download-dir) + readonly ARG_DOWNLOAD_DIR="$2" + shift 2 + ;; + --extract-dir) + readonly ARG_EXTRACT_DIR="$2" + shift 2 + ;; + --build-dir) + readonly ARG_BUILD_DIR="$2" + shift 2 + ;; + --stage-dir) + readonly ARG_STAGE_DIR="$2" + shift 2 + ;; + --install-dir) + readonly ARG_INSTALL_DIR="$2" + shift 2 + ;; + --toolset) + readonly ARG_TOOLSET="$2" + shift 2 + ;; + --cxx) + readonly ARG_CXX_CMD="$2" + shift 2 + ;; + --parallel) + readonly ARG_PARALLEL="$2" + shift 2 + ;; + -*) + echo Unknown option \"$1\" 1>&2 + exit + ;; + *) + break + ;; + esac +done + +# validate the arguments +test -n "${ARG_DOWNLOAD_DIR}" || { echo "--download-dir is not specified" 1>&2; exit 1; } +test -n "${ARG_EXTRACT_DIR}" || { echo "--extract-dir is not specified" 1>&2; exit 1; } + +# options that have default value +test -n "${ARG_VERSION}" || readonly ARG_VERSION=1.64.0 + +readonly LIBRARY_NAME=boost_$(echo ${ARG_VERSION} | sed -e 's/\./_/g') # e.g. boost_1_64_0 +readonly SOURCE_DIR="${ARG_EXTRACT_DIR}/${LIBRARY_NAME}" + +test -n "${ARG_BUILD_DIR}" || readonly ARG_BUILD_DIR="${SOURCE_DIR}/build" +test -n "${ARG_STAGE_DIR}" || readonly ARG_STAGE_DIR="${SOURCE_DIR}/stage" +test -n "${ARG_INSTALL_DIR}" || readonly ARG_INSTALL_DIR=/usr/local +test -n "${ARG_PARALLEL}" || readonly ARG_PARALLEL=1 + +# options for cross compiling +if [ -n "${ARG_TOOLSET}" ]; then # e.g. gcc-arm + # requires a location of g++ command + test -n "${ARG_CXX_CMD}" || { echo "--cxx is not specified" 1>&2; exit 1; } + + readonly USER_CONFIG_JAM="using $(echo ${ARG_TOOLSET} | sed -e 's/-/ : /') : ${ARG_CXX_CMD} ;" + readonly OPT_TOOLSET=toolset=${ARG_TOOLSET} +fi + +# download (if it isn't cached) +if [ ! -e "${SOURCE_DIR}/INSTALL" ]; then + echo -e "\e[33;1mDownloading libboost\e[0m" + + [ -d "${ARG_DOWNLOAD_DIR}" ] || mkdir -p "${ARG_DOWNLOAD_DIR}" + + cd "${ARG_DOWNLOAD_DIR}" + if [ ! -e "${LIBRARY_NAME}.tar.gz" ]; then + download_dir="https://dl.bintray.com/boostorg/release/${ARG_VERSION}/source/${LIBRARY_NAME}.tar.gz" + curl -LO ${download_dir} # wget doesn't work for bintray with 403 errpr + fi + tar -zxf ${LIBRARY_NAME}.tar.gz -C "${ARG_EXTRACT_DIR}" + + echo -e "\e[32;1mlibboost was successfully downloaded.\e[0m" +else + echo -e "\e[32;1mlibboost has been downloaded.\e[0m" +fi + +# build (if it isn't cached) +if [ ! -e "${ARG_STAGE_DIR}/lib/libboost_system.a" ]; then + echo -e "\e[33;1mBuilding libprotobuf\e[0m" + + cd "${SOURCE_DIR}" + + # bootstrap + ./bootstrap.sh --prefix="${ARG_INSTALL_DIR}" + + # configure options + if [ -n "${USER_CONFIG_JAM}" ]; then + echo "${USER_CONFIG_JAM}" > ${SOURCE_DIR}/user_config.jam + readonly OPT_USER_CONFIG_JAM=--user-config=${SOURCE_DIR}/user_config.jam + fi + echo "-j${ARG_PARALLEL} ${OPT_TOOLSET} ${OPT_USER_CONFIG_JAM} link=static cflags=-fPIC cxxflags=-fPIC --with-filesystem --with-test --with-log --with-program_options --build-dir=${ARG_BUILD_DIR} --stage-dir=${ARG_STAGE_DIR}" > b2_opts.txt + + # run a build + ./b2 stage $(cat b2_opts.txt) + + echo -e "\e[32;1mlibboost was successfully built.\e[0m" +else + echo -e "\e[32;1mlibboost has been built.\e[0m" +fi diff --git a/scripts/build-menoh.sh b/scripts/build-menoh.sh index 6f7f80b..38c4fed 100755 --- a/scripts/build-menoh.sh +++ b/scripts/build-menoh.sh @@ -59,8 +59,8 @@ test -n "${ARG_BUILD_DIR}" || readonly ARG_BUILD_DIR="${ARG_SOURCE_DIR}/build" test -n "${ARG_INSTALL_DIR}" || readonly ARG_INSTALL_DIR=/usr/local if [ -n "${ARG_MKLDNN_DIR}" ]; then - OPT_MKLDNN_INCLUDE_DIR=-DMKLDNN_INCLUDE_DIR=${ARG_MKLDNN_DIR}/include - OPT_MKLDNN_LIBRARY=-DMKLDNN_LIBRARY=${ARG_MKLDNN_DIR}/lib/libmkldnn.so + readonly OPT_MKLDNN_INCLUDE_DIR=-DMKLDNN_INCLUDE_DIR=${ARG_MKLDNN_DIR}/include + readonly OPT_MKLDNN_LIBRARY=-DMKLDNN_LIBRARY=${ARG_MKLDNN_DIR}/lib/libmkldnn.so fi test -n "${ARG_LINK_STATIC_LIBGCC}" || readonly ARG_LINK_STATIC_LIBGCC='OFF' diff --git a/scripts/build-protobuf.sh b/scripts/build-protobuf.sh index f694b8b..f1fdb34 100755 --- a/scripts/build-protobuf.sh +++ b/scripts/build-protobuf.sh @@ -27,6 +27,14 @@ while [[ $# != 0 ]]; do readonly ARG_INSTALL_DIR="$2" shift 2 ;; + --host) + readonly ARG_HOST="$2" + shift 2 + ;; + --with-protoc) + readonly ARG_WITH_PROTOC="$2" + shift 2 + ;; --parallel) readonly ARG_PARALLEL="$2" shift 2 @@ -55,6 +63,14 @@ test -n "${ARG_BUILD_DIR}" || readonly ARG_BUILD_DIR="${SOURCE_DIR}" test -n "${ARG_INSTALL_DIR}" || readonly ARG_INSTALL_DIR=/usr/local test -n "${ARG_PARALLEL}" || readonly ARG_PARALLEL=1 +# options for cross compiling +if [ -n "${ARG_HOST}" ]; then + readonly OPT_HOST=--host=${ARG_HOST} +fi +if [ -n "${ARG_WITH_PROTOC}" ]; then + readonly OPT_WITH_PROTOC=--with-protoc=${ARG_WITH_PROTOC} +fi + # download (if it isn't cached) if [ ! -e "${SOURCE_DIR}/LICENSE" ]; then echo -e "\e[33;1mDownloading libprotobuf\e[0m" @@ -80,7 +96,7 @@ if [ ! -e "${ARG_BUILD_DIR}/src/libprotobuf.la" ]; then [ -d "${ARG_BUILD_DIR}" ] || mkdir -p "${ARG_BUILD_DIR}" cd "${ARG_BUILD_DIR}" - "${SOURCE_DIR}/configure" --prefix="${ARG_INSTALL_DIR}" CFLAGS="-g -O2 -fPIC" CXXFLAGS="-g -O2 -fPIC" + "${SOURCE_DIR}/configure" --prefix="${ARG_INSTALL_DIR}" CFLAGS="-g -O2 -fPIC" CXXFLAGS="-g -O2 -fPIC" "${OPT_HOST}" "${OPT_WITH_PROTOC}" make -j${ARG_PARALLEL} echo -e "\e[32;1mlibprotobuf was successfully built.\e[0m" diff --git a/scripts/install-boost.sh b/scripts/install-boost.sh new file mode 100644 index 0000000..fdb3a13 --- /dev/null +++ b/scripts/install-boost.sh @@ -0,0 +1,35 @@ +#!/bin/bash -e + +# retrieve arguments +while [[ $# != 0 ]]; do + case $1 in + --) + shift + break + ;; + --source-dir) + readonly ARG_SOURCE_DIR="$2" + shift 2 + ;; + --dest-dir) + readonly ARG_DESTDIR="$2" + shift 2 + ;; + -*) + echo Unknown option \"$1\" 1>&2 + exit + ;; + *) + break + ;; + esac +done + +# validate the arguments +test -n "${ARG_SOURCE_DIR}" || { echo "--source-dir is not specified" 1>&2; exit 1; } + +# install (always) +echo -e "\e[33;1mInstalling libboost\e[0m" + +cd "${ARG_SOURCE_DIR}" +./b2 install $(cat ${ARG_SOURCE_DIR}/b2_opts.txt) From a28b4f31913243d5226285ec132dd2f738f53ffa Mon Sep 17 00:00:00 2001 From: Yuta Okamoto Date: Mon, 1 Oct 2018 17:34:07 +0900 Subject: [PATCH 02/14] switch to use libonnx directly --- CMakeLists.txt | 42 ++++++++++++++++++++-------- cmake/BuildProtobuf.cmake | 44 +++++++++++++++++++++++++++++ cmake/ConfigureMenoh.cmake | 5 ++-- cmake/GenerateOnnxSrc.cmake | 25 ----------------- cmake/SetupProtobuf.cmake | 56 ------------------------------------- menoh/CMakeLists.txt | 14 ---------- test/CMakeLists.txt | 8 ++---- 7 files changed, 80 insertions(+), 114 deletions(-) create mode 100644 cmake/BuildProtobuf.cmake delete mode 100644 cmake/GenerateOnnxSrc.cmake delete mode 100644 cmake/SetupProtobuf.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index 751fe2f..a000058 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -7,8 +7,6 @@ set(MENOH_MINOR_VERSION 1) set(MENOH_PATCH_VERSION 0) # Options -option(BUILD_SHARED_LIBS "Build shared libs" ON) - option(USE_OLD_GLIBCXX_ABI "Generate binaries for the old libstdc++ ABI" OFF) option(LINK_STATIC_LIBPROTOBUF "Link static libprotobuf to libmenoh" OFF) @@ -52,11 +50,33 @@ mark_as_advanced(DOWNLOAD_LOCATION) # Enable ExternalProject_Add include(ExternalProject) -# Setup protobuf -include(SetupProtobuf) +# Setup protobuf (MUST run before building ONNX) +if(LINK_STATIC_LIBPROTOBUF) + # Note: We can't use `set(PROTOBUF_BUILD_SHARED_LIBS OFF)` in `FindProtobuf` module + # because `libprotobuf.a` produced by the package manager is not PIC. So we need to + # build it by ourselves. + if(UNIX OR MINGW) + include(BuildProtobuf) + else() + message(FATAL_ERROR "LINK_STATIC_LIBPROTOBUF is supported only in UNIX-like environments") + endif() +else() + include(FindProtobuf) + find_package(Protobuf ${PROTOBUF_VERSION} REQUIRED) +endif() + +include_directories(${PROTOBUF_INCLUDE_DIRS}) + +# Build libonnx.a +message(STATUS "Adding external/onnx") + +set(ONNX_SRC_DIR ${EXTERNAL_DIR}/onnx) +execute_process(COMMAND git submodule update --init -- ${ONNX_SRC_DIR} WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}) -# Generate source codes from ONNX protobuf schema -include(GenerateOnnxSrc) +# TODO: enable the following option when it is ready for migrating to onnx-ml +#set(ONNX_ML 1) +add_subdirectory(external/onnx) # Note: BUILD_SHARED_LIBS must be OFF in this place +include_directories("${ONNX_INCLUDE_DIRS}") # Setup MKLDNN find_package(MKLDNN "0.14") @@ -67,24 +87,24 @@ endif() include_directories("${MKLDNN_INCLUDE_DIR}") if(${ENABLE_TEST}) - message(STATUS "# add_subdirectory\(test\)") + message(STATUS "Adding test") add_subdirectory(test) endif() if(${ENABLE_BENCHMARK}) - message(STATUS "# add_subdirectory\(benchmark\)") + message(STATUS "Adding benchmark") add_subdirectory(benchmark) endif() if(${ENABLE_EXAMPLE}) - message(STATUS "# add_subdirectory\(example\)") + message(STATUS "Adding example") add_subdirectory(example) endif() -message(STATUS "# add_subdirectory\(menoh\)") +message(STATUS "Adding menoh") add_subdirectory(menoh) -message(STATUS "# add_subdirectory\(include\)") +message(STATUS "Adding include") add_subdirectory(include) if(SHOW_ALL_VARIABLES) diff --git a/cmake/BuildProtobuf.cmake b/cmake/BuildProtobuf.cmake new file mode 100644 index 0000000..3822f96 --- /dev/null +++ b/cmake/BuildProtobuf.cmake @@ -0,0 +1,44 @@ +set(PROTOBUF_VERSION_STATIC "3.6.1") + +set(PROTOBUF_DIR ${CMAKE_CURRENT_BINARY_DIR}/protobuf-${PROTOBUF_VERSION_STATIC}) +set(PROTOBUF_URL "https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOBUF_VERSION_STATIC}/protobuf-cpp-${PROTOBUF_VERSION_STATIC}.tar.gz") +set(PROTOBUF_HASH MD5=406d5b8636576b1c86730ca5cbd1e576) + +# Requires `-fPIC` for linking with a shared library +set(PROTOBUF_CFLAGS -fPIC) +set(PROTOBUF_CXXFLAGS -fPIC) +if(USE_OLD_GLIBCXX_ABI) + set(PROTOBUF_CXXFLAGS "${PROTOBUF_CXXFLAGS} -D_GLIBCXX_USE_CXX11_ABI=0") +endif() + +ExternalProject_Add(Protobuf + PREFIX ${PROTOBUF_DIR} + URL ${PROTOBUF_URL} + URL_HASH ${PROTOBUF_HASH} + DOWNLOAD_DIR "${DOWNLOAD_LOCATION}" + BUILD_IN_SOURCE 1 + CONFIGURE_COMMAND bash -ex ${CMAKE_MODULE_PATH}/configure-helper.sh ${CMAKE_C_COMPILER} ${CMAKE_CXX_COMPILER} ${PROTOBUF_DIR} ${PROTOBUF_CFLAGS} ${PROTOBUF_CXXFLAGS} + BUILD_COMMAND make -j4 + INSTALL_COMMAND make install +) + +set(PROTOBUF_LIBRARY_STATIC ${PROTOBUF_DIR}/lib/libprotobuf.a) +set(PROTOBUF_LIBRARY_SHARED ${PROTOBUF_DIR}/lib/libprotobuf.so) + +# Mimic the behavior of `FindProtobuf` module +# Use the old variable names to ensure backward compatibility +set(PROTOBUF_INCLUDE_DIR ${PROTOBUF_DIR}/include) +set(PROTOBUF_INCLUDE_DIRS ${PROTOBUF_INCLUDE_DIR}) +set(PROTOBUF_LIBRARY ${PROTOBUF_LIBRARY_STATIC}) # use the static library +set(PROTOBUF_LIBRARIES ${PROTOBUF_LIBRARY}) +set(PROTOBUF_PROTOC_EXECUTABLE ${PROTOBUF_DIR}/bin/protoc) +set(PROTOBUF_FOUND TRUE) + +add_library(protobuf::libprotobuf UNKNOWN IMPORTED) +# Note: INTERFACE_INCLUDE_DIRECTORIES can't set in this place because include/ is +# not installed during executing `cmake` +set_target_properties(protobuf::libprotobuf PROPERTIES + IMPORTED_LOCATION "${PROTOBUF_LIBRARY_STATIC}") +add_executable(protobuf::protoc IMPORTED) +set_target_properties(protobuf::protoc PROPERTIES + IMPORTED_LOCATION "${PROTOBUF_PROTOC_EXECUTABLE}") diff --git a/cmake/ConfigureMenoh.cmake b/cmake/ConfigureMenoh.cmake index a2f956a..fbbe2f0 100644 --- a/cmake/ConfigureMenoh.cmake +++ b/cmake/ConfigureMenoh.cmake @@ -8,6 +8,9 @@ macro(menoh_link_libraries TARGET_NAME SCOPE) target_link_libraries(${TARGET_NAME} ${SCOPE} -static-libstdc++) endif() + target_link_libraries(${TARGET_NAME} ${SCOPE} ${PROTOBUF_LIBRARIES}) + target_link_libraries(${TARGET_NAME} ${SCOPE} onnx) + if(NOT ${SCOPE}) # PUBLIC will add transitive dependencies (`mklml_intel` and `iomp5`) to the link interface # Note: change it to PRIVATE after building mkldnn itself @@ -15,6 +18,4 @@ macro(menoh_link_libraries TARGET_NAME SCOPE) else() target_link_libraries(${TARGET_NAME} ${MKLDNN_LIBRARIES}) endif() - - target_link_libraries(${TARGET_NAME} ${SCOPE} ${PROTOBUF_LIBRARIES}) endmacro() diff --git a/cmake/GenerateOnnxSrc.cmake b/cmake/GenerateOnnxSrc.cmake deleted file mode 100644 index 0c2d8b5..0000000 --- a/cmake/GenerateOnnxSrc.cmake +++ /dev/null @@ -1,25 +0,0 @@ -set(ONNX_OUTPUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/onnx) - -if(NOT EXISTS "${ONNX_OUTPUT_DIR}") - file(MAKE_DIRECTORY "${ONNX_OUTPUT_DIR}") -endif() - -set(ONNX_SRC_DIR ${EXTERNAL_DIR}/onnx) -execute_process(COMMAND git submodule update --init -- ${ONNX_SRC_DIR} WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}) - -set(ONNX_PROTO_HEADER ${ONNX_OUTPUT_DIR}/onnx/onnx.pb.h) -set(ONNX_PROTO_SRC ${ONNX_OUTPUT_DIR}/onnx/onnx.pb.cc) - -set(ONNX_GENERATED_OUTPUTS ${ONNX_PROTO_HEADER} ${ONNX_PROTO_SRC}) - -add_custom_target(gen_onnx_outputs DEPENDS ${ONNX_GENERATED_OUTPUTS}) -add_custom_command( - OUTPUT ${ONNX_GENERATED_OUTPUTS} - COMMAND ${PROTOBUF_PROTOC_EXECUTABLE} - ARGS -I ${ONNX_SRC_DIR} --cpp_out . ${ONNX_SRC_DIR}/onnx/onnx.proto - DEPENDS ${PROTOBUF_PROTOC_EXECUTABLE} ${ONNX_SRC_DIR}/onnx/onnx.proto - COMMENT "Generating ONNX source files" - WORKING_DIRECTORY ${ONNX_OUTPUT_DIR} - VERBATIM) - -include_directories(${ONNX_OUTPUT_DIR}) # for ONNX_PROTO_HEADER diff --git a/cmake/SetupProtobuf.cmake b/cmake/SetupProtobuf.cmake deleted file mode 100644 index 327edf4..0000000 --- a/cmake/SetupProtobuf.cmake +++ /dev/null @@ -1,56 +0,0 @@ -set(PROTOBUF_VERSION "2.6.1") - -if(LINK_STATIC_LIBPROTOBUF) - # Note: We can't use `set(PROTOBUF_BUILD_SHARED_LIBS OFF)` in `FindProtobuf` module - # because `libprotobuf.a` produced by the package manager is not PIC. So we need to - # build it by ourselves. - - if(UNIX OR MINGW) - set(PROTOBUF_VERSION_STATIC "3.6.1") - set(PROTOBUF_DIR ${CMAKE_CURRENT_BINARY_DIR}/protobuf-${PROTOBUF_VERSION_STATIC}) - set(PROTOBUF_URL "https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOBUF_VERSION_STATIC}/protobuf-cpp-${PROTOBUF_VERSION_STATIC}.tar.gz") - set(PROTOBUF_HASH MD5=406d5b8636576b1c86730ca5cbd1e576) - - # Requires `-fPIC` for linking with a shared library - set(PROTOBUF_CFLAGS -fPIC) - set(PROTOBUF_CXXFLAGS -fPIC) - if(USE_OLD_GLIBCXX_ABI) - set(PROTOBUF_CXXFLAGS "${PROTOBUF_CXXFLAGS} -D_GLIBCXX_USE_CXX11_ABI=0") - endif() - - ExternalProject_Add(Protobuf - PREFIX ${PROTOBUF_DIR} - URL ${PROTOBUF_URL} - URL_HASH ${PROTOBUF_HASH} - DOWNLOAD_DIR "${DOWNLOAD_LOCATION}" - BUILD_IN_SOURCE 1 - CONFIGURE_COMMAND bash -ex ${CMAKE_MODULE_PATH}/configure-helper.sh ${CMAKE_C_COMPILER} ${CMAKE_CXX_COMPILER} ${PROTOBUF_DIR} ${PROTOBUF_CFLAGS} ${PROTOBUF_CXXFLAGS} - BUILD_COMMAND make -j4 - INSTALL_COMMAND make install - ) - - set(PROTOBUF_LIBRARY_STATIC ${PROTOBUF_DIR}/lib/libprotobuf.a) - set(PROTOBUF_LIBRARY_SHARED ${PROTOBUF_DIR}/lib/libprotobuf.so) - - # Mimic the behavior of `FindProtobuf` module - # Use the old variable names to ensure backward compatibility - set(PROTOBUF_INCLUDE_DIR ${PROTOBUF_DIR}/include) - set(PROTOBUF_LIBRARY ${PROTOBUF_LIBRARY_STATIC}) # use the static library - set(PROTOBUF_LIBRARIES ${PROTOBUF_LIBRARY}) - set(PROTOBUF_PROTOC_EXECUTABLE ${PROTOBUF_DIR}/bin/protoc) - set(PROTOBUF_FOUND TRUE) - - add_library(protobuf::libprotobuf UNKNOWN IMPORTED) - # Note: INTERFACE_INCLUDE_DIRECTORIES can't set in this place because include/ is - # not installed during executing `cmake` - set_target_properties(protobuf::libprotobuf PROPERTIES - IMPORTED_LOCATION "${PROTOBUF_LIBRARY_STATIC}") - else() - message(FATAL_ERROR "LINK_STATIC_LIBPROTOBUF is supported only in UNIX-like environments") - endif() -else() - include(FindProtobuf) - find_package(Protobuf ${PROTOBUF_VERSION} REQUIRED) -endif() - -include_directories(${PROTOBUF_INCLUDE_DIR}) diff --git a/menoh/CMakeLists.txt b/menoh/CMakeLists.txt index 9bd995b..06e9f3a 100644 --- a/menoh/CMakeLists.txt +++ b/menoh/CMakeLists.txt @@ -2,13 +2,6 @@ option(LINK_STATIC_LIBGCC "Link static libgcc to libmenoh" OFF) option(LINK_STATIC_LIBSTDCXX "Link static libstdc++ to libmenoh" OFF) -if(NOT DEFINED ONNX_PROTO_SRC) - message(FATAL_ERROR "ONNX_PROTO_SRC is not found") -endif() -if(NOT DEFINED ONNX_PROTO_HEADER) - message(FATAL_ERROR "ONNX_PROTO_HEADER is not found") -endif() - # Note: The libraries can be static (.a) or shared (.so) if(NOT DEFINED MKLDNN_LIBRARIES) message(FATAL_ERROR "MKLDNN_LIBRARIES is not found") @@ -21,10 +14,6 @@ file(GLOB_RECURSE SOURCES "." "*.cpp") # Create a object library for generating shared library add_library(menoh_objlib OBJECT ${SOURCES}) - -add_dependencies(menoh_objlib gen_onnx_outputs) -target_sources(menoh_objlib PRIVATE ${ONNX_PROTO_SRC}) - set_target_properties(menoh_objlib PROPERTIES POSITION_INDEPENDENT_CODE ON) include(ConfigureMenoh) @@ -43,9 +32,6 @@ menoh_link_libraries(menoh PRIVATE) add_library(menoh_test_target SHARED $) menoh_link_libraries(menoh_test_target PRIVATE) -set_source_files_properties(${ONNX_PROTO_SRC} PROPERTIES GENERATED TRUE) -set_source_files_properties(${ONNX_PROTO_HEADER} PROPERTIES GENERATED TRUE) - install(TARGETS menoh RUNTIME DESTINATION "bin" LIBRARY DESTINATION "lib" diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 04f9813..3085c5a 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -3,7 +3,7 @@ enable_testing() # GTest setup set(GTEST_DIR "lib/googletest") execute_process(COMMAND git submodule update --init -- test/${GTEST_DIR} WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}) -message(STATUS "# add_subdirectory\(${GTEST_DIR}\)") +message(STATUS "Adding_${GTEST_DIR}") add_subdirectory(${GTEST_DIR}) # filesystem setup @@ -25,11 +25,7 @@ add_executable(menoh_test ) -if(LINK_STATIC_LIBPROTOBUF) - target_link_libraries(menoh_test gtest_main menoh_test_target) -else() - target_link_libraries(menoh_test gtest_main menoh_test_target ${PROTOBUF_LIBRARY}) -endif() +target_link_libraries(menoh_test gtest_main menoh_test_target onnx_proto) if(UNIX AND NOT APPLE) set_property( From 8d373ea59300b6d1e329cf3dab4bc74acc9f2e4f Mon Sep 17 00:00:00 2001 From: Yuta Okamoto Date: Mon, 1 Oct 2018 18:08:11 +0900 Subject: [PATCH 03/14] add PYTHON_EXECUTABLE option --- .travis/init-build-linux.sh | 4 +++- .travis/init-build-osx.sh | 4 +++- scripts/build-menoh.sh | 6 ++++++ 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/.travis/init-build-linux.sh b/.travis/init-build-linux.sh index a5e5f16..ae5f0db 100644 --- a/.travis/init-build-linux.sh +++ b/.travis/init-build-linux.sh @@ -78,12 +78,14 @@ function build_menoh() { docker_exec_script \ "${PROJ_DIR}/scripts/build-menoh.sh" \ --build-type Release \ - --source-dir "${PROJ_DIR}" + --source-dir "${PROJ_DIR}" \ + --python-executable python3 else docker_exec_script \ "${PROJ_DIR}/scripts/build-menoh.sh" \ --build-type Release \ --source-dir "${PROJ_DIR}" \ + --python-executable python3 \ --link-static-libgcc ON \ --link-static-libstdcxx ON \ --link-static-libprotobuf ON diff --git a/.travis/init-build-osx.sh b/.travis/init-build-osx.sh index c1228d8..afe593a 100644 --- a/.travis/init-build-osx.sh +++ b/.travis/init-build-osx.sh @@ -17,12 +17,14 @@ function build_menoh() { if [ "${LINK_STATIC}" != "true" ]; then bash -ex "${PROJ_DIR}/scripts/build-menoh.sh" \ --build-type Release \ - --source-dir "${PROJ_DIR}" + --source-dir "${PROJ_DIR}" \ + --python-executable python else # Does not set --link-static-libgcc and --link-static-libstdcxx in macOS bash -ex "${PROJ_DIR}/scripts/build-menoh.sh" \ --build-type Release \ --source-dir "${PROJ_DIR}" \ + --python-executable python \ --link-static-libprotobuf ON fi } diff --git a/scripts/build-menoh.sh b/scripts/build-menoh.sh index 38c4fed..5184c4e 100755 --- a/scripts/build-menoh.sh +++ b/scripts/build-menoh.sh @@ -29,6 +29,10 @@ while [[ $# != 0 ]]; do readonly ARG_MKLDNN_DIR="$2" shift 2 ;; + --python-executable) + ARG_PYTHON_EXECUTABLE="$2" + shift 2 + ;; --link-static-libgcc) readonly ARG_LINK_STATIC_LIBGCC="$2" shift 2 @@ -63,6 +67,7 @@ if [ -n "${ARG_MKLDNN_DIR}" ]; then readonly OPT_MKLDNN_LIBRARY=-DMKLDNN_LIBRARY=${ARG_MKLDNN_DIR}/lib/libmkldnn.so fi +test -n "${ARG_PYTHON_EXECUTABLE}" || readonly ARG_PYTHON_EXECUTABLE=python test -n "${ARG_LINK_STATIC_LIBGCC}" || readonly ARG_LINK_STATIC_LIBGCC='OFF' test -n "${ARG_LINK_STATIC_LIBSTDCXX}" || readonly ARG_LINK_STATIC_LIBSTDCXX='OFF' test -n "${ARG_LINK_STATIC_LIBPROTOBUF}" || readonly ARG_LINK_STATIC_LIBPROTOBUF='OFF' @@ -77,6 +82,7 @@ cmake \ "-DCMAKE_INSTALL_PREFIX=${ARG_INSTALL_DIR}" \ "${OPT_MKLDNN_INCLUDE_DIR}" \ "${OPT_MKLDNN_LIBRARY}" \ + -DPYTHON_EXECUTABLE=${ARG_PYTHON_EXECUTABLE} \ -DLINK_STATIC_LIBGCC=${ARG_LINK_STATIC_LIBGCC} \ -DLINK_STATIC_LIBSTDCXX=${ARG_LINK_STATIC_LIBSTDCXX} \ -DLINK_STATIC_LIBPROTOBUF=${ARG_LINK_STATIC_LIBPROTOBUF} \ From 6a7413e24ac59c5fded4e85e32ec7341583b3742 Mon Sep 17 00:00:00 2001 From: Yuta Okamoto Date: Tue, 2 Oct 2018 11:25:42 +0900 Subject: [PATCH 04/14] fix to link protobuf in test for shared library mode --- CMakeLists.txt | 4 +++- cmake/ConfigureMenoh.cmake | 3 +-- menoh/CMakeLists.txt | 3 --- test/CMakeLists.txt | 6 +++++- 4 files changed, 9 insertions(+), 7 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index a000058..4ed72f3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -50,7 +50,7 @@ mark_as_advanced(DOWNLOAD_LOCATION) # Enable ExternalProject_Add include(ExternalProject) -# Setup protobuf (MUST run before building ONNX) +# Setup protobuf (it is used in ONNX and Menoh) if(LINK_STATIC_LIBPROTOBUF) # Note: We can't use `set(PROTOBUF_BUILD_SHARED_LIBS OFF)` in `FindProtobuf` module # because `libprotobuf.a` produced by the package manager is not PIC. So we need to @@ -61,6 +61,8 @@ if(LINK_STATIC_LIBPROTOBUF) message(FATAL_ERROR "LINK_STATIC_LIBPROTOBUF is supported only in UNIX-like environments") endif() else() + # Note: It may conflict with the loading mechanism in onnx's CMake configuration. + # See external/onnx/CMakeLists.txt for more details. include(FindProtobuf) find_package(Protobuf ${PROTOBUF_VERSION} REQUIRED) endif() diff --git a/cmake/ConfigureMenoh.cmake b/cmake/ConfigureMenoh.cmake index fbbe2f0..1752799 100644 --- a/cmake/ConfigureMenoh.cmake +++ b/cmake/ConfigureMenoh.cmake @@ -8,8 +8,7 @@ macro(menoh_link_libraries TARGET_NAME SCOPE) target_link_libraries(${TARGET_NAME} ${SCOPE} -static-libstdc++) endif() - target_link_libraries(${TARGET_NAME} ${SCOPE} ${PROTOBUF_LIBRARIES}) - target_link_libraries(${TARGET_NAME} ${SCOPE} onnx) + target_link_libraries(${TARGET_NAME} ${SCOPE} onnx) # onnx also contains protobuf if(NOT ${SCOPE}) # PUBLIC will add transitive dependencies (`mklml_intel` and `iomp5`) to the link interface diff --git a/menoh/CMakeLists.txt b/menoh/CMakeLists.txt index 06e9f3a..398d48f 100644 --- a/menoh/CMakeLists.txt +++ b/menoh/CMakeLists.txt @@ -6,9 +6,6 @@ option(LINK_STATIC_LIBSTDCXX "Link static libstdc++ to libmenoh" OFF) if(NOT DEFINED MKLDNN_LIBRARIES) message(FATAL_ERROR "MKLDNN_LIBRARIES is not found") endif() -if(NOT DEFINED PROTOBUF_LIBRARIES) - message(FATAL_ERROR "PROTOBUF_LIBRARIES is not found") -endif() file(GLOB_RECURSE SOURCES "." "*.cpp") diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 3085c5a..2ca5a7c 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -25,7 +25,11 @@ add_executable(menoh_test ) -target_link_libraries(menoh_test gtest_main menoh_test_target onnx_proto) +if(LINK_STATIC_LIBPROTOBUF) + target_link_libraries(menoh_test gtest_main menoh_test_target) +else() + target_link_libraries(menoh_test gtest_main menoh_test_target ${PROTOBUF_LIBRARIES}) +endif() if(UNIX AND NOT APPLE) set_property( From 8770abd750b9a8558c8b2c1be0899f51b2fe0ce9 Mon Sep 17 00:00:00 2001 From: Yuta Okamoto Date: Tue, 2 Oct 2018 12:08:40 +0900 Subject: [PATCH 05/14] add explicit dependency to gen_onnx_proto --- appveyor.yml | 2 +- menoh/CMakeLists.txt | 2 ++ test/CMakeLists.txt | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 7c67464..9951f2c 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -72,7 +72,7 @@ build_script: cmake -G "MSYS Makefiles" -DENABLE_TEST=ON %STATIC_OPTION% -DCMAKE_INSTALL_PREFIX=/mingw64 .. && make ) else ( - cmake -G "Visual Studio 14 Win64" -DENABLE_TEST=OFF -DENABLE_BENCHMARK=OFF -DENABLE_EXAMPLE=OFF -DENABLE_TOOL=OFF -DCMAKE_INSTALL_PREFIX=c:\menoh-%MENOH_REV%-msvc .. && + cmake -G "Visual Studio 14 Win64" -DENABLE_TEST=OFF -DENABLE_BENCHMARK=OFF -DENABLE_EXAMPLE=OFF -DCMAKE_INSTALL_PREFIX=c:\menoh-%MENOH_REV%-msvc .. && cmake --build . --config Release --target install ) diff --git a/menoh/CMakeLists.txt b/menoh/CMakeLists.txt index 398d48f..d420997 100644 --- a/menoh/CMakeLists.txt +++ b/menoh/CMakeLists.txt @@ -11,6 +11,8 @@ file(GLOB_RECURSE SOURCES "." "*.cpp") # Create a object library for generating shared library add_library(menoh_objlib OBJECT ${SOURCES}) +add_dependencies(menoh_objlib gen_onnx_proto) # custom target defined in onnx + set_target_properties(menoh_objlib PROPERTIES POSITION_INDEPENDENT_CODE ON) include(ConfigureMenoh) diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 2ca5a7c..2b9f9e4 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -3,7 +3,7 @@ enable_testing() # GTest setup set(GTEST_DIR "lib/googletest") execute_process(COMMAND git submodule update --init -- test/${GTEST_DIR} WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}) -message(STATUS "Adding_${GTEST_DIR}") +message(STATUS "Adding ${GTEST_DIR}") add_subdirectory(${GTEST_DIR}) # filesystem setup From 99a77bcca9a44c85382eaa84f3ba2b5817808895 Mon Sep 17 00:00:00 2001 From: Yuta Okamoto Date: Tue, 2 Oct 2018 14:52:43 +0900 Subject: [PATCH 06/14] add PYTHON_EXECUTABLE option --- BUILDING.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/BUILDING.md b/BUILDING.md index eb6fea1..a308f45 100644 --- a/BUILDING.md +++ b/BUILDING.md @@ -194,3 +194,12 @@ MSYS2_ARG_CONV_EXCL="-DCMAKE_INSTALL_PREFIX=" \ cmake -G "MSYS Makefiles" -DCMAKE_INSTALL_PREFIX=/mingw64 make ``` + +### Note + +#### Python command name +Menoh requires `python` command to generate source codes at build time. Add `PYTHON_EXECUTABLE` option to `cmake` if you want to use `python` command with non-standard name (e.g. `python3`). + +```bash +cmake -DPYTHON_EXECUTABLE=python3 .. +``` From db60d6695ee018467e195ae56a3b49011d41620a Mon Sep 17 00:00:00 2001 From: Yuta Okamoto Date: Wed, 3 Oct 2018 12:48:08 +0900 Subject: [PATCH 07/14] add a patch for CMake configuration in onnx --- CMakeLists.txt | 6 +- external/onnx-v1.3.0-patch_CMakeLists.txt | 565 ++++++++++++++++++++++ menoh/CMakeLists.txt | 5 + test/CMakeLists.txt | 2 + 4 files changed, 576 insertions(+), 2 deletions(-) create mode 100644 external/onnx-v1.3.0-patch_CMakeLists.txt diff --git a/CMakeLists.txt b/CMakeLists.txt index 4ed72f3..6cce354 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -67,14 +67,16 @@ else() find_package(Protobuf ${PROTOBUF_VERSION} REQUIRED) endif() -include_directories(${PROTOBUF_INCLUDE_DIRS}) - # Build libonnx.a message(STATUS "Adding external/onnx") set(ONNX_SRC_DIR ${EXTERNAL_DIR}/onnx) execute_process(COMMAND git submodule update --init -- ${ONNX_SRC_DIR} WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}) +# see https://github.com/onnx/onnx/pull/1466 +message(STATUS "Patching to external/onnx") +configure_file(${EXTERNAL_DIR}/onnx-v1.3.0-patch_CMakeLists.txt ${EXTERNAL_DIR}/onnx/CMakeLists.txt COPYONLY) + # TODO: enable the following option when it is ready for migrating to onnx-ml #set(ONNX_ML 1) add_subdirectory(external/onnx) # Note: BUILD_SHARED_LIBS must be OFF in this place diff --git a/external/onnx-v1.3.0-patch_CMakeLists.txt b/external/onnx-v1.3.0-patch_CMakeLists.txt new file mode 100644 index 0000000..23ec68b --- /dev/null +++ b/external/onnx-v1.3.0-patch_CMakeLists.txt @@ -0,0 +1,565 @@ +# Minimum CMake required +cmake_minimum_required(VERSION 3.1) +include(cmake/Utils.cmake) +# Set default build type +if(NOT CMAKE_BUILD_TYPE) + message(STATUS "Build type not set - defaulting to Release") + set( + CMAKE_BUILD_TYPE "Release" + CACHE + STRING + "Choose the type of build from: Debug Release RelWithDebInfo MinSizeRel Coverage." + FORCE) +endif() +cmake_policy(SET CMP0063 NEW) + +# Project +project(onnx C CXX) +option(ONNX_BUILD_BENCHMARKS "Build ONNX micro-benchmarks" OFF) + +option(BUILD_ONNX_PYTHON "Build Python binaries" OFF) +option(ONNX_GEN_PB_TYPE_STUBS "Generate protobuf python type stubs" ON) +option(ONNX_WERROR "Build with Werror" OFF) +option(ONNX_COVERAGE "Build with coverage instrumentation" OFF) +option(ONNX_BUILD_TESTS "Build ONNX C++ APIs Tests" OFF) +option(ONNX_USE_LITE_PROTO "Use lite protobuf instead of full." OFF) + +set(ONNX_NAMESPACE "onnx" CACHE STRING "onnx namespace") + +# Set C++11 as standard for the whole project +if(NOT MSVC) + set(CMAKE_CXX_STANDARD 11) +endif(NOT MSVC) + +set(ONNX_ROOT ${PROJECT_SOURCE_DIR}) + +# Read ONNX version +file(READ "${PROJECT_SOURCE_DIR}/VERSION_NUMBER" ONNX_VERSION) +string(STRIP "${ONNX_VERSION}" ONNX_VERSION) + +if(NOT MSVC) + set(CMAKE_C_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -O0") + set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -O0") + if(ONNX_COVERAGE) + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fprofile-arcs -ftest-coverage") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fprofile-arcs -ftest-coverage") + endif() +endif() + +if(ONNX_BUILD_TESTS) + list(APPEND CMAKE_MODULE_PATH ${ONNX_ROOT}/cmake/external) + include(googletest) +endif() + +if((ONNX_USE_LITE_PROTO AND TARGET protobuf::libprotobuf-lite) OR ((NOT ONNX_USE_LITE_PROTO) AND TARGET protobuf::libprotobuf)) + # Sometimes we need to use protoc compiled for host architecture while linking + # libprotobuf against target architecture. See https://github.com/caffe2/caffe + # 2/blob/96f35ad75480b25c1a23d6e9e97bccae9f7a7f9c/cmake/ProtoBuf.cmake#L92-L99 + if(EXISTS "${ONNX_CUSTOM_PROTOC_EXECUTABLE}") + message(STATUS "Using custom protoc executable") + set(ONNX_PROTOC_EXECUTABLE ${ONNX_CUSTOM_PROTOC_EXECUTABLE}) + else() + set(ONNX_PROTOC_EXECUTABLE $) + endif() +else() + # Customized version of find Protobuf. We need to avoid situations mentioned + # in https://github.com/caffe2/caffe2/blob/b7d983f255ef5496474f1ea188edb5e0ac4 + # 42761/cmake/ProtoBuf.cmake#L82-L92 The following section is stolen from + # cmake/ProtoBuf.cmake in Caffe2 + find_program(Protobuf_PROTOC_EXECUTABLE + NAMES protoc + DOC "The Google Protocol Buffers Compiler") + + # Only if protoc was found, seed the include directories and libraries. We + # assume that protoc is installed at PREFIX/bin. We use get_filename_component + # to resolve PREFIX. + if(Protobuf_PROTOC_EXECUTABLE) + set(ONNX_PROTOC_EXECUTABLE ${Protobuf_PROTOC_EXECUTABLE}) + get_filename_component(_PROTOBUF_INSTALL_PREFIX + ${Protobuf_PROTOC_EXECUTABLE} DIRECTORY) + get_filename_component(_PROTOBUF_INSTALL_PREFIX + ${_PROTOBUF_INSTALL_PREFIX}/.. REALPATH) + find_library(Protobuf_PROTOC_LIBRARY + NAMES protoc + PATHS ${_PROTOBUF_INSTALL_PREFIX}/lib + NO_DEFAULT_PATH) + if(ONNX_USE_LITE_PROTO) + find_library(Protobuf_LITE_LIBRARY + NAMES protobuf-lite + PATHS ${_PROTOBUF_INSTALL_PREFIX}/lib + NO_DEFAULT_PATH) + else(ONNX_USE_LITE_PROTO) + find_library(Protobuf_LIBRARY + NAMES protobuf + PATHS ${_PROTOBUF_INSTALL_PREFIX}/lib + NO_DEFAULT_PATH) + endif(ONNX_USE_LITE_PROTO) + find_path(Protobuf_INCLUDE_DIR google/protobuf/service.h + PATHS ${_PROTOBUF_INSTALL_PREFIX}/include + NO_DEFAULT_PATH) + find_package(Protobuf REQUIRED) + endif() +endif() + +# Build the libraries with -fPIC +set(CMAKE_POSITION_INDEPENDENT_CODE ON) + +# function(RELATIVE_PROTOBUF_GENERATE_CPP SRCS HDRS ROOT_DIR) from https://githu +# b.com/tensorflow/tensorflow/blob/d2c3b873c6f8ff999a2e4ee707a84ff00d9c15a5/tens +# orflow/contrib/cmake/tf_core_framework.cmake to solve the problem that +# customized dir can't be specified when calling PROTOBUF_GENERATE_CPP. +function(RELATIVE_PROTOBUF_GENERATE_CPP NAME SRCS HDRS ROOT_DIR DEPEND) + if(NOT ARGN) + message( + SEND_ERROR + "Error: RELATIVE_PROTOBUF_GENERATE_CPP() called without any proto files" + ) + return() + endif() + + if(MSVC AND BUILD_SHARED_LIBS) + set(ONNX_DLLEXPORT_STR "dllexport_decl=ONNX_API:") + else() + set(ONNX_DLLEXPORT_STR "") + endif() + + set(${SRCS}) + set(${HDRS}) + + set(GEN_PROTO_PY ${ROOT_DIR}/onnx/gen_proto.py) + foreach(INFILE ${ARGN}) + set(ABS_FILE ${ROOT_DIR}/${INFILE}) + get_filename_component(FILE_DIR ${ABS_FILE} DIRECTORY) + get_filename_component(FILE_WE ${INFILE} NAME_WE) + if(ONNX_ML) + if(ONNX_NAMESPACE STREQUAL "onnx") + set(GENERATED_FILE_WE "${FILE_WE}-ml") + else() + set(GENERATED_FILE_WE "${FILE_WE}_${ONNX_NAMESPACE}-ml") + endif() + else() + if(ONNX_NAMESPACE STREQUAL "onnx") + set(GENERATED_FILE_WE "${FILE_WE}") + else() + set(GENERATED_FILE_WE "${FILE_WE}_${ONNX_NAMESPACE}") + endif() + endif() + file(RELATIVE_PATH REL_DIR ${ROOT_DIR} ${FILE_DIR}) + set(OUTPUT_PROTO_DIR "${CMAKE_CURRENT_BINARY_DIR}/${REL_DIR}") + + set(OUTPUT_PB_HEADER "${OUTPUT_PROTO_DIR}/${GENERATED_FILE_WE}.pb.h") + set(OUTPUT_PB_SRC "${OUTPUT_PROTO_DIR}/${GENERATED_FILE_WE}.pb.cc") + set(GENERATED_PROTO "${OUTPUT_PROTO_DIR}/${GENERATED_FILE_WE}.proto") + if(NOT (ONNX_NAMESPACE STREQUAL "onnx")) + # We need this dummy header generated by gen_proto.py when ONNX_NAMESPACE + # is not onnx + list(APPEND ${HDRS} "${OUTPUT_PROTO_DIR}/${GENERATED_FILE_WE}.pb.h") + endif() + list(APPEND ${SRCS} "${OUTPUT_PB_SRC}") + list(APPEND ${HDRS} "${OUTPUT_PB_HEADER}") + + if(NOT EXISTS "${OUTPUT_PROTO_DIR}") + file(MAKE_DIRECTORY "${OUTPUT_PROTO_DIR}") + endif() + + if("${PYTHON_EXECUTABLE}" STREQUAL "") + set(_python_exe "python") + else() + set(_python_exe "${PYTHON_EXECUTABLE}") + endif() + set(GEN_PROTO_ARGS + -p + "${ONNX_NAMESPACE}" + -o + "${OUTPUT_PROTO_DIR}" + "${FILE_WE}") + if(ONNX_ML) + list(APPEND GEN_PROTO_ARGS -m) + endif() + if(ONNX_USE_LITE_PROTO) + list(APPEND GEN_PROTO_ARGS -l) + endif() + add_custom_command(OUTPUT "${GENERATED_PROTO}" + COMMAND "${_python_exe}" "${GEN_PROTO_PY}" + ARGS ${GEN_PROTO_ARGS} + DEPENDS ${INFILE} + COMMENT "Running gen_proto.py on ${INFILE}" + VERBATIM) + + set(PROTOC_ARGS + ${GENERATED_PROTO} + -I + ${CMAKE_CURRENT_BINARY_DIR} + --cpp_out + ${ONNX_DLLEXPORT_STR}${CMAKE_CURRENT_BINARY_DIR}) + if(BUILD_ONNX_PYTHON) + list(APPEND PROTOC_ARGS --python_out + ${ONNX_DLLEXPORT_STR}${CMAKE_CURRENT_BINARY_DIR}) + if(ONNX_GEN_PB_TYPE_STUBS) + # Haven't figured out how to generate mypy stubs on Windows yet + if(NOT WIN32) + # If onnx was packaged to pypi from Windows, protoc-gen-mypy.py is + # missing the +x flag. Add it. + execute_process(COMMAND chmod +x ${ROOT_DIR}/tools/protoc-gen-mypy.py) + set(PROTOC_MYPY_PLUGIN_FILE ${ROOT_DIR}/tools/protoc-gen-mypy.py) + else(NOT WIN32) + set(PROTOC_MYPY_PLUGIN_FILE ${ROOT_DIR}/tools/protoc-gen-mypy.bat) + endif() + list(APPEND PROTOC_ARGS + --plugin + protoc-gen-mypy=${PROTOC_MYPY_PLUGIN_FILE} + --mypy_out + ${ONNX_DLLEXPORT_STR}${CMAKE_CURRENT_BINARY_DIR}) + endif() + endif() + if(NOT ONNX_PROTOC_EXECUTABLE) + message(FATAL_ERROR "Protobuf compiler not found") + endif() + if(ONNX_PROTO_POST_BUILD_SCRIPT) + add_custom_command( + OUTPUT "${OUTPUT_PB_SRC}" "${OUTPUT_PB_HEADER}" + COMMAND ${ONNX_PROTOC_EXECUTABLE} ARGS ${PROTOC_ARGS} + COMMAND "${CMAKE_COMMAND}" -DFILENAME=${OUTPUT_PB_HEADER} + -DNAMESPACES=${ONNX_NAMESPACE} -P + ${ONNX_PROTO_POST_BUILD_SCRIPT} + COMMAND "${CMAKE_COMMAND}" -DFILENAME=${OUTPUT_PB_SRC} + -DNAMESPACES=${ONNX_NAMESPACE} -P + ${ONNX_PROTO_POST_BUILD_SCRIPT} + DEPENDS ${GENERATED_PROTO} ${DEPEND} + COMMENT "Running C++ protocol buffer compiler on ${GENERATED_PROTO}" + VERBATIM) + else() + add_custom_command( + OUTPUT "${OUTPUT_PB_SRC}" "${OUTPUT_PB_HEADER}" + COMMAND ${ONNX_PROTOC_EXECUTABLE} ARGS ${PROTOC_ARGS} + DEPENDS ${GENERATED_PROTO} ${DEPEND} + COMMENT "Running C++ protocol buffer compiler on ${GENERATED_PROTO}" + VERBATIM) + endif() + add_custom_target(${NAME} DEPENDS ${OUTPUT_PB_SRC} ${OUTPUT_PB_HEADER}) + endforeach() + + set_source_files_properties(${${SRCS}} ${${HDRS}} PROPERTIES GENERATED TRUE) + set(${SRCS} ${${SRCS}} PARENT_SCOPE) + set(${HDRS} ${${HDRS}} PARENT_SCOPE) +endfunction() + +relative_protobuf_generate_cpp(gen_onnx_proto + PROTO_SRCS + PROTO_HDRS + ${ONNX_ROOT} + "" + onnx/onnx.in.proto) +relative_protobuf_generate_cpp(gen_onnx_operators_proto + PROTO_SRCS2 + PROTO_HDRS2 + ${ONNX_ROOT} + gen_onnx_proto + onnx/onnx-operators.in.proto) +list(APPEND PROTO_SRCS ${PROTO_SRCS2}) +list(APPEND PROTO_HDRS ${PROTO_HDRS2}) + +file(GLOB_RECURSE onnx_src "${ONNX_ROOT}/onnx/*.h" "${ONNX_ROOT}/onnx/*.cc") +file(GLOB_RECURSE onnx_gtests_src "${ONNX_ROOT}/onnx/test/cpp/*.h" + "${ONNX_ROOT}/onnx/test/cpp/*.cc") +list(REMOVE_ITEM onnx_src "${ONNX_ROOT}/onnx/cpp2py_export.cc") +list(REMOVE_ITEM onnx_src ${onnx_gtests_src}) + +add_library(onnx_proto ${PROTO_SRCS} ${PROTO_HDRS}) +target_include_directories(onnx_proto PUBLIC + $ + $ + $) + + +if(ONNX_USE_LITE_PROTO) + if(TARGET protobuf::libprotobuf-lite) + target_link_libraries(onnx_proto PUBLIC protobuf::libprotobuf-lite) + else() + target_link_libraries(onnx_proto PUBLIC ${PROTOBUF_LITE_LIBRARIES}) + endif() +else() + if(TARGET protobuf::libprotobuf) + target_link_libraries(onnx_proto PUBLIC protobuf::libprotobuf) + else() + target_link_libraries(onnx_proto PUBLIC ${PROTOBUF_LIBRARIES}) + endif() +endif() +add_onnx_global_defines(onnx_proto) + +add_library(onnx ${onnx_src}) +target_include_directories(onnx PUBLIC + $ + $ + $) +target_link_libraries(onnx PUBLIC onnx_proto) +add_onnx_global_defines(onnx) + +if(BUILD_ONNX_PYTHON) + if("${PY_EXT_SUFFIX}" STREQUAL "") + if(MSVC) + set(PY_EXT_SUFFIX ".pyd") + else() + set(PY_EXT_SUFFIX ".so") + endif() + endif() + + add_library(onnx_cpp2py_export MODULE "${ONNX_ROOT}/onnx/cpp2py_export.cc") + set_target_properties(onnx_cpp2py_export PROPERTIES PREFIX "") + set_target_properties(onnx_cpp2py_export + PROPERTIES COMPILE_FLAGS "-fvisibility=hidden") + set_target_properties(onnx_cpp2py_export PROPERTIES SUFFIX ${PY_EXT_SUFFIX}) + set_target_properties(onnx_cpp2py_export + PROPERTIES LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}) + target_include_directories(onnx_cpp2py_export PRIVATE + $ + $ + $ + ${PYTHON_INCLUDE_DIR}) + + # pybind11 is a header only lib + find_package(pybind11 2.2) + if(pybind11_FOUND) + target_include_directories(onnx_cpp2py_export PUBLIC + ${pybind11_INCLUDE_DIRS}) + else() + if(EXISTS ${ONNX_ROOT}/third_party/pybind11/include/pybind11/pybind11.h) + target_include_directories(onnx_cpp2py_export PUBLIC + ${ONNX_ROOT}/third_party/pybind11/include) + else() + message(FATAL_ERROR "cannot find pybind") + endif() + endif() + + if(APPLE) + set_target_properties(onnx_cpp2py_export + PROPERTIES LINK_FLAGS "-undefined dynamic_lookup") + target_link_libraries(onnx_cpp2py_export + PRIVATE -Wl,-force_load,$) + elseif(MSVC) + # In MSVC, we will add whole archive in default + target_link_libraries(onnx_cpp2py_export + PRIVATE -WHOLEARCHIVE:$) + else() + # Assume everything else is like gcc + target_link_libraries(onnx_cpp2py_export + PRIVATE "-Wl,--whole-archive" $ + "-Wl,--no-whole-archive") + endif() + + target_link_libraries(onnx_cpp2py_export PRIVATE onnx) + + if(MSVC) + if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang") + set(MP_FLAG "-Xclang" "-fopenmp") + set(EXTRA_FLAGS + "-Wno-implicit-function-declaration" + "-Wno-undefined-inline" + "-Wno-incompatible-pointer-types" + "-Wno-dllexport-explicit-instantiation-decl" + "-Wno-microsoft-unqualified-friend" + "-Wno-absolute-value" + "-Wno-unused-variable" + "-Wno-writable-strings" + "-Qunused-arguments") + else() + set(MP_FLAG "/MP") + set(EXTRA_FLAGS "") + endif() + find_package(PythonInterp ${PY_VERSION} REQUIRED) + find_package(PythonLibs ${PY_VERSION} REQUIRED) + target_link_libraries(onnx_cpp2py_export PRIVATE ${PYTHON_LIBRARIES}) + target_compile_options(onnx_cpp2py_export + PRIVATE ${MP_FLAG} + /WX + /wd4800 # disable warning type' : forcing + # value to bool 'true' or 'false' + # (performance warning) + /wd4503 # identifier' : decorated name length + # exceeded, name was truncated + /wd4146 # unary minus operator applied to + # unsigned type, result still + # unsigned from include\google\protob + # uf\wire_format_lite.h + ${EXTRA_FLAGS}) + target_compile_options(onnx_cpp2py_export PRIVATE /MT) + add_onnx_global_defines(onnx_cpp2py_export) + endif() +endif() + +if(ONNX_BUILD_BENCHMARKS) + if(NOT TARGET benchmark) + # We will not need to test benchmark lib itself. + set(BENCHMARK_ENABLE_TESTING OFF + CACHE BOOL "Disable benchmark testing as we don't need it.") + # We will not need to install benchmark since we link it statically. + set(BENCHMARK_ENABLE_INSTALL OFF + CACHE BOOL + "Disable benchmark install to avoid overwriting vendor install.") + add_subdirectory(${PROJECT_SOURCE_DIR}/third_party/benchmark) + endif() + + add_executable(protobuf-bench tools/protobuf-bench.cc) + target_include_directories(protobuf-bench PUBLIC + $ + $ + $ + $) + target_link_libraries(protobuf-bench onnx_proto benchmark) +endif() + +# Export include directories +set(ONNX_INCLUDE_DIRS "${ONNX_ROOT}" "${CMAKE_CURRENT_BINARY_DIR}") +get_directory_property(hasParent PARENT_DIRECTORY) +if(hasParent) + set(ONNX_INCLUDE_DIRS ${ONNX_INCLUDE_DIRS} PARENT_SCOPE) +endif() + +if(MSVC) + target_compile_options(onnx_proto + PRIVATE ${MP_FLAG} + /WX + /wd4800 # disable warning type' : forcing value + # to bool 'true' or 'false' + # (performance warning) + /wd4503 # identifier' : decorated name length + # exceeded, name was truncated + /wd4146 # unary minus operator applied to + # unsigned type, result still unsigned: + # include\google\protobuf\wire_format_l + # ite.h + ${EXTRA_FLAGS}) + target_compile_options(onnx + PRIVATE ${MP_FLAG} + /WX + /wd4800 # disable warning type' : forcing value + # to bool 'true' or 'false' + # (performance warning) + /wd4503 # identifier' : decorated name length + # exceeded, name was truncated + /wd4146 # unary minus operator applied to + # unsigned type, result still unsigned + ${EXTRA_FLAGS}) + add_msvc_runtime_flag(onnx_proto) + add_msvc_runtime_flag(onnx) + set(onnx_static_library_flags + -IGNORE:4221 # LNK4221: This object file does not define any previously + # undefined public symbols, so it will not be used by any + # link operation that consumes this library + ) + set_target_properties(onnx + PROPERTIES STATIC_LIBRARY_FLAGS + "${onnx_static_library_flags}") +elseif(APPLE) + +else() + if(${ONNX_WERROR}) + target_compile_options(onnx PRIVATE -Werror=sign-compare -Werror=conversion) + endif() +endif() + +if(APPLE) + set_target_properties(onnx PROPERTIES LINK_FLAGS "-undefined dynamic_lookup") +endif() + +# ---[ ONNX Interface for Framework Integratin (ONNXIFI) +add_library(onnxifi INTERFACE) +target_include_directories(onnxifi INTERFACE + $ + $) + +# ---[ ONNXIFI loader +add_library(onnxifi_loader STATIC onnx/onnxifi_loader.c) +target_include_directories(onnxifi_loader PUBLIC + $ + $) +# Users of ONNX backend API would compile it with their toolchain, so it is +# implemented in standard C89 for maximum compatibility +set_target_properties(onnxifi_loader + PROPERTIES C_STANDARD + 90 + C_EXTENSIONS + NO) +target_link_libraries(onnxifi_loader PUBLIC onnxifi ${CMAKE_DL_LIBS}) +if(MSVC) + add_msvc_runtime_flag(onnxifi_loader) +endif() + +if (NOT ANDROID AND NOT IOS) + # ---[ ONNXIFI wrapper + add_library(onnxifi_wrapper MODULE onnx/onnxifi_wrapper.c) + target_include_directories(onnxifi_wrapper PRIVATE + $ + $) + set_target_properties(onnxifi_wrapper PROPERTIES + C_STANDARD 99 + C_EXTENSIONS NO + OUTPUT_NAME "onnxifi" + POSITION_INDEPENDENT_CODE YES) + target_link_libraries(onnxifi_wrapper PRIVATE onnxifi_loader onnxifi) + if(DEFINED ONNXIFI_SEARCH_DIR) + target_compile_definitions(onnxifi_wrapper PRIVATE "ONNXIFI_SEARCH_DIR=\"${ONNXIFI_SEARCH_DIR}\"") + endif() + if(WIN32) + if(MSVC) + target_compile_definitions(onnxifi_wrapper PRIVATE "ONNXIFI_PUBLIC=__declspec(dllexport)") + else() + target_compile_definitions(onnxifi_wrapper PRIVATE "ONNXIFI_PUBLIC=__attribute__((__dllexport__))") + endif() + endif() + if(APPLE) + # By default CMake would use .so suffix on Mac + set_target_properties(onnxifi_wrapper PROPERTIES SUFFIX ".dylib") + endif() +endif() + +# ---[ ONNXIFI dummy backend +add_library(onnxifi_dummy SHARED onnx/onnxifi_dummy.c) +target_include_directories(onnxifi_dummy PRIVATE + $ + $) +target_link_libraries(onnxifi_dummy PUBLIC onnxifi ${CMAKE_DL_LIBS}) +target_compile_definitions(onnxifi_dummy PRIVATE ONNXIFI_BUILD_LIBRARY=TRUE) +if(MSVC) + add_msvc_runtime_flag(onnxifi_dummy) +endif() + +install(DIRECTORY ${ONNX_ROOT}/onnx + DESTINATION include + FILES_MATCHING + PATTERN "*.h") +install(DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/onnx + DESTINATION include + FILES_MATCHING + PATTERN "*.h") + +configure_file( + ${PROJECT_SOURCE_DIR}/cmake/ONNXConfigVersion.cmake.in + ${PROJECT_BINARY_DIR}/ONNXConfigVersion.cmake + @ONLY) +configure_file( + ${PROJECT_SOURCE_DIR}/cmake/ONNXConfig.cmake.in + ${PROJECT_BINARY_DIR}/ONNXConfig.cmake + @ONLY) +install(FILES + ${PROJECT_BINARY_DIR}/ONNXConfigVersion.cmake + ${PROJECT_BINARY_DIR}/ONNXConfig.cmake + DESTINATION share/cmake/ONNX + COMPONENT dev) +install(EXPORT ONNXTargets DESTINATION share/cmake/ONNX) +install(TARGETS + onnx onnx_proto + onnxifi onnxifi_dummy onnxifi_loader + EXPORT ONNXTargets DESTINATION lib) + +if(NOT ANDROID AND NOT IOS) + install(TARGETS onnxifi_wrapper + EXPORT ONNXTargets DESTINATION lib) +endif() + +if(ONNX_BUILD_TESTS) + include(${ONNX_ROOT}/cmake/unittest.cmake) +endif() + +include(cmake/summary.cmake) +onnx_print_configuration_summary() diff --git a/menoh/CMakeLists.txt b/menoh/CMakeLists.txt index d420997..a4fe9e0 100644 --- a/menoh/CMakeLists.txt +++ b/menoh/CMakeLists.txt @@ -3,6 +3,9 @@ option(LINK_STATIC_LIBGCC "Link static libgcc to libmenoh" OFF) option(LINK_STATIC_LIBSTDCXX "Link static libstdc++ to libmenoh" OFF) # Note: The libraries can be static (.a) or shared (.so) +if(NOT DEFINED PROTOBUF_INCLUDE_DIRS) + message(FATAL_ERROR "PROTOBUF_INCLUDE_DIRS is not found") +endif() if(NOT DEFINED MKLDNN_LIBRARIES) message(FATAL_ERROR "MKLDNN_LIBRARIES is not found") endif() @@ -11,7 +14,9 @@ file(GLOB_RECURSE SOURCES "." "*.cpp") # Create a object library for generating shared library add_library(menoh_objlib OBJECT ${SOURCES}) + add_dependencies(menoh_objlib gen_onnx_proto) # custom target defined in onnx +target_include_directories(menoh_objlib PUBLIC $) set_target_properties(menoh_objlib PROPERTIES POSITION_INDEPENDENT_CODE ON) diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index f118d7f..96c18ff 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -25,6 +25,8 @@ add_executable(menoh_test #vgg16.cpp ) +target_include_directories(menoh_test PUBLIC $) + if(LINK_STATIC_LIBPROTOBUF) target_link_libraries(menoh_test gtest_main menoh_test_target) else() From 7ce597e4232066bd511efa703c851fe2fce54921 Mon Sep 17 00:00:00 2001 From: Yuta Okamoto Date: Wed, 3 Oct 2018 15:11:50 +0900 Subject: [PATCH 08/14] add EXCLUDE_FROM_ALL to external projects --- CMakeLists.txt | 2 +- test/CMakeLists.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 6cce354..6aa884e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -79,7 +79,7 @@ configure_file(${EXTERNAL_DIR}/onnx-v1.3.0-patch_CMakeLists.txt ${EXTERNAL_DIR}/ # TODO: enable the following option when it is ready for migrating to onnx-ml #set(ONNX_ML 1) -add_subdirectory(external/onnx) # Note: BUILD_SHARED_LIBS must be OFF in this place +add_subdirectory(external/onnx EXCLUDE_FROM_ALL) # Note: BUILD_SHARED_LIBS must be OFF in this place include_directories("${ONNX_INCLUDE_DIRS}") # Setup MKLDNN diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 96c18ff..866092c 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -4,7 +4,7 @@ enable_testing() set(GTEST_DIR "lib/googletest") execute_process(COMMAND git submodule update --init -- test/${GTEST_DIR} WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}) message(STATUS "Adding ${GTEST_DIR}") -add_subdirectory(${GTEST_DIR}) +add_subdirectory(${GTEST_DIR} EXCLUDE_FROM_ALL) # filesystem setup set(FILESYSTEM_DIR "lib/filesystem") From 9d91a14f3c24d9d9333d7f2bcfb9fba327b20842 Mon Sep 17 00:00:00 2001 From: Yuta Okamoto Date: Wed, 3 Oct 2018 15:16:08 +0900 Subject: [PATCH 09/14] add ignore=dirty option to external/onnx --- .gitmodules | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitmodules b/.gitmodules index 17edb59..056710b 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,6 +1,7 @@ [submodule "external/onnx"] path = external/onnx - url = https://github.com/onnx/onnx + url = https://github.com/onnx/onnx.git + ignore = dirty [submodule "test/lib/googletest"] path = test/lib/googletest url = https://github.com/google/googletest.git From 6a0fb845f599a282fc3d0b2cc9ed242d639313f0 Mon Sep 17 00:00:00 2001 From: Yuta Okamoto Date: Wed, 3 Oct 2018 17:54:37 +0900 Subject: [PATCH 10/14] update BUILDING.md --- BUILDING.md | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/BUILDING.md b/BUILDING.md index a308f45..19d8c63 100644 --- a/BUILDING.md +++ b/BUILDING.md @@ -53,10 +53,9 @@ Download and unzip https://github.com/protocolbuffers/protobuf/releases/download ``` cd protobuf-3.6.1/cmake -mdir build +mkdir build cd build cmake .. -G "Visual Studio 14" -A x64 -Dprotobuf_MSVC_STATIC_RUNTIME=OFF -Dprotobuf_BUILD_TESTS=OFF -DCMAKE_INSTALL_PREFIX=(CMake_Install_Dir) -cmake --build . --config Debug --target install cmake --build . --config Release --target install cd ../../.. ``` @@ -67,10 +66,9 @@ git clone https://github.com/intel/mkl-dnn.git cd mkl-dnn/scripts .\prepare_mkl.bat cd .. -mdir build +mkdir build cd build cmake .. -G "Visual Studio 14 Win64" -DCMAKE_INSTALL_PREFIX=(CMake_Install_Dir) -cmake --build . --config Debug --target install cmake --build . --config Release --target install cd ../.. ``` @@ -177,10 +175,9 @@ Please replace `(CMake_Install_Dir)` in the following with your working director ``` git clone https://github.com/pfnet-research/menoh.git cd menoh -mdir build +mkdir build cd build -cmake .. -G "Visual Studio 14 Win64" -DCMAKE_PREFIX_PATH=CMake_Install_Dir) -DCMAKE_INSTALL_PREFIX=CMake_Install_Dir) -DENABLE_TEST=OFF -DENABLE_BENCHMARK=OFF -DENABLE_EXAMPLE=OFF -DENABLE_TOOL=OFF -cmake --build . --config Debug --target install +cmake .. -G "Visual Studio 14 Win64" -DCMAKE_PREFIX_PATH=(CMake_Install_Dir) -DCMAKE_INSTALL_PREFIX=(CMake_Install_Dir) -DENABLE_TEST=OFF -DENABLE_BENCHMARK=OFF -DENABLE_EXAMPLE=OFF cmake --build . --config Release --target install ``` From c1573ed1084f4a6be12921b3e0766e9fbdf50ab7 Mon Sep 17 00:00:00 2001 From: Yuta Okamoto Date: Thu, 4 Oct 2018 19:41:30 +0900 Subject: [PATCH 11/14] fix some environment specific bugs --- external/cmdline.h | 6 +++++- test/CMakeLists.txt | 6 ++++++ test/lib/googletest | 2 +- 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/external/cmdline.h b/external/cmdline.h index f498b66..f6b3b9b 100644 --- a/external/cmdline.h +++ b/external/cmdline.h @@ -39,8 +39,12 @@ #ifdef _MSC_VER #define NOMINMAX -#include +/* + You must include Windows.h before DbgHelp.h. + See https://stackoverflow.com/a/43283926/1014818 for more details. +*/ #include +#include #else #include #endif diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 866092c..6224acb 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -3,7 +3,13 @@ enable_testing() # GTest setup set(GTEST_DIR "lib/googletest") execute_process(COMMAND git submodule update --init -- test/${GTEST_DIR} WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}) + message(STATUS "Adding ${GTEST_DIR}") + +# Prevent overriding the parent project's compiler/linker settings on Windows +# See https://github.com/google/googletest/blob/master/googletest/README.md#visual-studio-dynamic-vs-static-runtimes +set(gtest_force_shared_crt ON CACHE BOOL "" FORCE) + add_subdirectory(${GTEST_DIR} EXCLUDE_FROM_ALL) # filesystem setup diff --git a/test/lib/googletest b/test/lib/googletest index ba96d0b..2fe3bd9 160000 --- a/test/lib/googletest +++ b/test/lib/googletest @@ -1 +1 @@ -Subproject commit ba96d0b1161f540656efdaed035b3c062b60e006 +Subproject commit 2fe3bd994b3189899d93f1d5a881e725e046fdc2 From 38f65a8433e11edef039ecba47404264ed96936d Mon Sep 17 00:00:00 2001 From: Yuta Okamoto Date: Fri, 5 Oct 2018 14:02:13 +0900 Subject: [PATCH 12/14] Revert "add cross building support for protobuf" This reverts commit 08289fcf5b99e470e11eb09669ec13a57a4f811c. --- scripts/build-boost.sh | 120 -------------------------------------- scripts/build-menoh.sh | 4 +- scripts/build-protobuf.sh | 18 +----- scripts/install-boost.sh | 35 ----------- 4 files changed, 3 insertions(+), 174 deletions(-) delete mode 100644 scripts/build-boost.sh delete mode 100644 scripts/install-boost.sh diff --git a/scripts/build-boost.sh b/scripts/build-boost.sh deleted file mode 100644 index 183247f..0000000 --- a/scripts/build-boost.sh +++ /dev/null @@ -1,120 +0,0 @@ -#!/bin/bash -e - -# retrieve arguments -while [[ $# != 0 ]]; do - case $1 in - --) - shift - break - ;; - --version) - readonly ARG_VERSION="$2" - shift 2 - ;; - --download-dir) - readonly ARG_DOWNLOAD_DIR="$2" - shift 2 - ;; - --extract-dir) - readonly ARG_EXTRACT_DIR="$2" - shift 2 - ;; - --build-dir) - readonly ARG_BUILD_DIR="$2" - shift 2 - ;; - --stage-dir) - readonly ARG_STAGE_DIR="$2" - shift 2 - ;; - --install-dir) - readonly ARG_INSTALL_DIR="$2" - shift 2 - ;; - --toolset) - readonly ARG_TOOLSET="$2" - shift 2 - ;; - --cxx) - readonly ARG_CXX_CMD="$2" - shift 2 - ;; - --parallel) - readonly ARG_PARALLEL="$2" - shift 2 - ;; - -*) - echo Unknown option \"$1\" 1>&2 - exit - ;; - *) - break - ;; - esac -done - -# validate the arguments -test -n "${ARG_DOWNLOAD_DIR}" || { echo "--download-dir is not specified" 1>&2; exit 1; } -test -n "${ARG_EXTRACT_DIR}" || { echo "--extract-dir is not specified" 1>&2; exit 1; } - -# options that have default value -test -n "${ARG_VERSION}" || readonly ARG_VERSION=1.64.0 - -readonly LIBRARY_NAME=boost_$(echo ${ARG_VERSION} | sed -e 's/\./_/g') # e.g. boost_1_64_0 -readonly SOURCE_DIR="${ARG_EXTRACT_DIR}/${LIBRARY_NAME}" - -test -n "${ARG_BUILD_DIR}" || readonly ARG_BUILD_DIR="${SOURCE_DIR}/build" -test -n "${ARG_STAGE_DIR}" || readonly ARG_STAGE_DIR="${SOURCE_DIR}/stage" -test -n "${ARG_INSTALL_DIR}" || readonly ARG_INSTALL_DIR=/usr/local -test -n "${ARG_PARALLEL}" || readonly ARG_PARALLEL=1 - -# options for cross compiling -if [ -n "${ARG_TOOLSET}" ]; then # e.g. gcc-arm - # requires a location of g++ command - test -n "${ARG_CXX_CMD}" || { echo "--cxx is not specified" 1>&2; exit 1; } - - readonly USER_CONFIG_JAM="using $(echo ${ARG_TOOLSET} | sed -e 's/-/ : /') : ${ARG_CXX_CMD} ;" - readonly OPT_TOOLSET=toolset=${ARG_TOOLSET} -fi - -# download (if it isn't cached) -if [ ! -e "${SOURCE_DIR}/INSTALL" ]; then - echo -e "\e[33;1mDownloading libboost\e[0m" - - [ -d "${ARG_DOWNLOAD_DIR}" ] || mkdir -p "${ARG_DOWNLOAD_DIR}" - - cd "${ARG_DOWNLOAD_DIR}" - if [ ! -e "${LIBRARY_NAME}.tar.gz" ]; then - download_dir="https://dl.bintray.com/boostorg/release/${ARG_VERSION}/source/${LIBRARY_NAME}.tar.gz" - curl -LO ${download_dir} # wget doesn't work for bintray with 403 errpr - fi - tar -zxf ${LIBRARY_NAME}.tar.gz -C "${ARG_EXTRACT_DIR}" - - echo -e "\e[32;1mlibboost was successfully downloaded.\e[0m" -else - echo -e "\e[32;1mlibboost has been downloaded.\e[0m" -fi - -# build (if it isn't cached) -if [ ! -e "${ARG_STAGE_DIR}/lib/libboost_system.a" ]; then - echo -e "\e[33;1mBuilding libprotobuf\e[0m" - - cd "${SOURCE_DIR}" - - # bootstrap - ./bootstrap.sh --prefix="${ARG_INSTALL_DIR}" - - # configure options - if [ -n "${USER_CONFIG_JAM}" ]; then - echo "${USER_CONFIG_JAM}" > ${SOURCE_DIR}/user_config.jam - readonly OPT_USER_CONFIG_JAM=--user-config=${SOURCE_DIR}/user_config.jam - fi - echo "-j${ARG_PARALLEL} ${OPT_TOOLSET} ${OPT_USER_CONFIG_JAM} link=static cflags=-fPIC cxxflags=-fPIC --with-filesystem --with-test --with-log --with-program_options --build-dir=${ARG_BUILD_DIR} --stage-dir=${ARG_STAGE_DIR}" > b2_opts.txt - - # run a build - ./b2 stage $(cat b2_opts.txt) - - echo -e "\e[32;1mlibboost was successfully built.\e[0m" -else - echo -e "\e[32;1mlibboost has been built.\e[0m" -fi diff --git a/scripts/build-menoh.sh b/scripts/build-menoh.sh index 5184c4e..5740377 100755 --- a/scripts/build-menoh.sh +++ b/scripts/build-menoh.sh @@ -63,8 +63,8 @@ test -n "${ARG_BUILD_DIR}" || readonly ARG_BUILD_DIR="${ARG_SOURCE_DIR}/build" test -n "${ARG_INSTALL_DIR}" || readonly ARG_INSTALL_DIR=/usr/local if [ -n "${ARG_MKLDNN_DIR}" ]; then - readonly OPT_MKLDNN_INCLUDE_DIR=-DMKLDNN_INCLUDE_DIR=${ARG_MKLDNN_DIR}/include - readonly OPT_MKLDNN_LIBRARY=-DMKLDNN_LIBRARY=${ARG_MKLDNN_DIR}/lib/libmkldnn.so + OPT_MKLDNN_INCLUDE_DIR=-DMKLDNN_INCLUDE_DIR=${ARG_MKLDNN_DIR}/include + OPT_MKLDNN_LIBRARY=-DMKLDNN_LIBRARY=${ARG_MKLDNN_DIR}/lib/libmkldnn.so fi test -n "${ARG_PYTHON_EXECUTABLE}" || readonly ARG_PYTHON_EXECUTABLE=python diff --git a/scripts/build-protobuf.sh b/scripts/build-protobuf.sh index f1fdb34..f694b8b 100755 --- a/scripts/build-protobuf.sh +++ b/scripts/build-protobuf.sh @@ -27,14 +27,6 @@ while [[ $# != 0 ]]; do readonly ARG_INSTALL_DIR="$2" shift 2 ;; - --host) - readonly ARG_HOST="$2" - shift 2 - ;; - --with-protoc) - readonly ARG_WITH_PROTOC="$2" - shift 2 - ;; --parallel) readonly ARG_PARALLEL="$2" shift 2 @@ -63,14 +55,6 @@ test -n "${ARG_BUILD_DIR}" || readonly ARG_BUILD_DIR="${SOURCE_DIR}" test -n "${ARG_INSTALL_DIR}" || readonly ARG_INSTALL_DIR=/usr/local test -n "${ARG_PARALLEL}" || readonly ARG_PARALLEL=1 -# options for cross compiling -if [ -n "${ARG_HOST}" ]; then - readonly OPT_HOST=--host=${ARG_HOST} -fi -if [ -n "${ARG_WITH_PROTOC}" ]; then - readonly OPT_WITH_PROTOC=--with-protoc=${ARG_WITH_PROTOC} -fi - # download (if it isn't cached) if [ ! -e "${SOURCE_DIR}/LICENSE" ]; then echo -e "\e[33;1mDownloading libprotobuf\e[0m" @@ -96,7 +80,7 @@ if [ ! -e "${ARG_BUILD_DIR}/src/libprotobuf.la" ]; then [ -d "${ARG_BUILD_DIR}" ] || mkdir -p "${ARG_BUILD_DIR}" cd "${ARG_BUILD_DIR}" - "${SOURCE_DIR}/configure" --prefix="${ARG_INSTALL_DIR}" CFLAGS="-g -O2 -fPIC" CXXFLAGS="-g -O2 -fPIC" "${OPT_HOST}" "${OPT_WITH_PROTOC}" + "${SOURCE_DIR}/configure" --prefix="${ARG_INSTALL_DIR}" CFLAGS="-g -O2 -fPIC" CXXFLAGS="-g -O2 -fPIC" make -j${ARG_PARALLEL} echo -e "\e[32;1mlibprotobuf was successfully built.\e[0m" diff --git a/scripts/install-boost.sh b/scripts/install-boost.sh deleted file mode 100644 index fdb3a13..0000000 --- a/scripts/install-boost.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/bin/bash -e - -# retrieve arguments -while [[ $# != 0 ]]; do - case $1 in - --) - shift - break - ;; - --source-dir) - readonly ARG_SOURCE_DIR="$2" - shift 2 - ;; - --dest-dir) - readonly ARG_DESTDIR="$2" - shift 2 - ;; - -*) - echo Unknown option \"$1\" 1>&2 - exit - ;; - *) - break - ;; - esac -done - -# validate the arguments -test -n "${ARG_SOURCE_DIR}" || { echo "--source-dir is not specified" 1>&2; exit 1; } - -# install (always) -echo -e "\e[33;1mInstalling libboost\e[0m" - -cd "${ARG_SOURCE_DIR}" -./b2 install $(cat ${ARG_SOURCE_DIR}/b2_opts.txt) From 8e8e487d97c8056699ba8f2f149606a2565b671e Mon Sep 17 00:00:00 2001 From: Yuta Okamoto Date: Fri, 5 Oct 2018 14:06:38 +0900 Subject: [PATCH 13/14] fix an indent --- .gitmodules | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitmodules b/.gitmodules index 056710b..19ec233 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,7 +1,7 @@ [submodule "external/onnx"] path = external/onnx url = https://github.com/onnx/onnx.git - ignore = dirty + ignore = dirty [submodule "test/lib/googletest"] path = test/lib/googletest url = https://github.com/google/googletest.git From 2d0d5f42ae189f46ddc1b69c31ce8103767b742c Mon Sep 17 00:00:00 2001 From: Yuta Okamoto Date: Fri, 5 Oct 2018 14:59:50 +0900 Subject: [PATCH 14/14] add python to prerequisites --- BUILDING.md | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/BUILDING.md b/BUILDING.md index 19d8c63..492fb45 100644 --- a/BUILDING.md +++ b/BUILDING.md @@ -5,18 +5,25 @@ You need to install [prerequisites](#prerequisites) for your platform before [bu To build Menoh, you require the following toolchains: Unix: -- CMake 3.1 or later - GCC 4.9 or later +- CMake 3.1 or later +- Python 2.7 or later macOS (OSX): - XCode - [Homebrew](https://brew.sh/) +- CMake 3.1 or later +- Python 2.7 or later Windows: - Visual Studio 2015 +- CMake 3.1 or later +- Python 2.7 or later Windows (MINGW): - [MSYS2](http://www.msys2.org/) +- CMake 3.1 or later +- Python 2.7 or later You also need to install the dependent libraries on your system: