diff --git a/.github/workflows/flex.yml b/.github/workflows/flex.yml index b6871e55e74e..6a9c6747a896 100644 --- a/.github/workflows/flex.yml +++ b/.github/workflows/flex.yml @@ -22,11 +22,39 @@ concurrency: cancel-in-progress: true jobs: - build-flex: + test-build-flex: + runs-on: ubuntu-latest + + strategy: + matrix: + os: [ubuntu-20.04, ubuntu-22.04] + platform: [x86_64] + + steps: + - name: Checkout code + uses: actions/checkout@v3 + + - name: Install dependencies + run: | + cd ${GITHUB_WORKSPACE}/flex + bash ./scripts/install_dependencies.sh 4 + + - name: Setup tmate session + if: false + uses: mxschmitt/action-tmate@v2 + + - name: Build + run: | + cd ${GITHUB_WORKSPACE}/flex + git submodule update --init + mkdir build && cd build # only test default build + cmake .. -DCMAKE_BUILD_TYPE=DEBUG -DBUILD_DOC=OFF && sudo make -j 4 + + test-flex: runs-on: ubuntu-20.04 if: ${{ github.repository == 'alibaba/GraphScope' }} container: - image: registry.cn-hongkong.aliyuncs.com/graphscope/hqps-server-base:v0.0.8 + image: registry.cn-hongkong.aliyuncs.com/graphscope/hqps-server-base:v0.0.9 steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/hqps-db-ci.yml b/.github/workflows/hqps-db-ci.yml index 786973a39c8c..17b5ccfa02f3 100644 --- a/.github/workflows/hqps-db-ci.yml +++ b/.github/workflows/hqps-db-ci.yml @@ -28,7 +28,7 @@ jobs: runs-on: ubuntu-20.04 if: ${{ github.repository == 'alibaba/GraphScope' }} container: - image: registry.cn-hongkong.aliyuncs.com/graphscope/hqps-server-base:v0.0.8 + image: registry.cn-hongkong.aliyuncs.com/graphscope/hqps-server-base:v0.0.9 steps: - uses: actions/checkout@v3 diff --git a/flex/.devcontainer.json b/flex/.devcontainer.json index d1b8c725dca5..4457e7560a8a 100644 --- a/flex/.devcontainer.json +++ b/flex/.devcontainer.json @@ -3,7 +3,7 @@ { "name": "GraphScope", // Or use a Dockerfile or Docker Compose file. More info: https://containers.dev/guide/dockerfile - "image": "registry.cn-hongkong.aliyuncs.com/graphscope/hqps-server-base:v0.0.8", + "image": "registry.cn-hongkong.aliyuncs.com/graphscope/hqps-server-base:v0.0.9", // Features to add to the dev container. More info: https://containers.dev/features. "features": { "ghcr.io/devcontainers/features/common-utils:2": { diff --git a/flex/Dockerfile b/flex/Dockerfile index 7d5874a60dd9..328a07694296 100644 --- a/flex/Dockerfile +++ b/flex/Dockerfile @@ -39,7 +39,7 @@ RUN apt-get install -y protobuf-compiler libprotobuf-dev RUN cd /tmp && apt-get install -y -V ca-certificates lsb-release wget && \ curl -o apache-arrow-apt-source-latest.deb https://apache.jfrog.io/artifactory/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb && \ apt-get install -y ./apache-arrow-apt-source-latest.deb && \ - apt-get update && apt-get install -y libarrow-dev=6.0.1-1 + apt-get update && apt-get install -y libarrow-dev=8.0.0-1 RUN apt-get install -y sudo @@ -51,6 +51,9 @@ RUN useradd -m graphscope -u 1001 && \ USER graphscope WORKDIR /home/graphscope +# set HOME +ENV HOME /home/graphscope + RUN curl -sf -L https://static.rust-lang.org/rustup.sh | \ sh -s -- -y --profile minimal --default-toolchain=1.70.0 && \ chmod +x "$HOME/.cargo/env" && \ diff --git a/flex/engines/graph_db/app/app_base.h b/flex/engines/graph_db/app/app_base.h index f5f57eda3cc9..5fd64991a834 100644 --- a/flex/engines/graph_db/app/app_base.h +++ b/flex/engines/graph_db/app/app_base.h @@ -23,6 +23,7 @@ #include #include +#include #include diff --git a/flex/engines/hqps_db/core/utils/hqps_type.h b/flex/engines/hqps_db/core/utils/hqps_type.h index 36173129cf0c..a0451638a37e 100644 --- a/flex/engines/hqps_db/core/utils/hqps_type.h +++ b/flex/engines/hqps_db/core/utils/hqps_type.h @@ -1,6 +1,7 @@ #ifndef ENGINES_HQPS_ENGINE_UTILS_TYPE_UTILS_H_ #define ENGINES_HQPS_ENGINE_UTILS_TYPE_UTILS_H_ +#include #include "flex/storages/rt_mutable_graph/types.h" namespace gs { diff --git a/flex/engines/http_server/codegen_proxy.h b/flex/engines/http_server/codegen_proxy.h index 9677cecd3283..44a0a6994ecb 100644 --- a/flex/engines/http_server/codegen_proxy.h +++ b/flex/engines/http_server/codegen_proxy.h @@ -21,6 +21,7 @@ #include #include #include +#include #include "glog/logging.h" diff --git a/flex/interactive/docker/interactive-runtime.Dockerfile b/flex/interactive/docker/interactive-runtime.Dockerfile index 0604b0d4111f..fb1b414656f9 100644 --- a/flex/interactive/docker/interactive-runtime.Dockerfile +++ b/flex/interactive/docker/interactive-runtime.Dockerfile @@ -11,7 +11,7 @@ SHELL ["/bin/bash", "-c"] RUN cd /tmp && sudo apt-get update && sudo apt-get install -y -V ca-certificates lsb-release wget && \ curl -o apache-arrow-apt-source-latest.deb https://apache.jfrog.io/artifactory/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb && \ sudo apt-get install -y ./apache-arrow-apt-source-latest.deb && \ - sudo apt-get update && sudo apt-get install -y libarrow-dev=6.0.1-1 + sudo apt-get update && sudo apt-get install -y libarrow-dev=8.0.0-1 RUN curl -sf -L https://static.rust-lang.org/rustup.sh | \ sh -s -- -y --profile minimal --default-toolchain=1.70.0 && \ diff --git a/flex/scripts/install_dependencies.sh b/flex/scripts/install_dependencies.sh index 112a404409f3..1ad610aabe8d 100644 --- a/flex/scripts/install_dependencies.sh +++ b/flex/scripts/install_dependencies.sh @@ -1,34 +1,49 @@ -apt update -y +#!/bin/bash +set -e +# accept at most one argument, parallelism +if [ $# -gt 1 ]; then + echo "Usage: $0 [parallelism]" + exit 1 +fi +parallelism=$(nproc) +if [ $# -eq 1 ]; then + parallelism=$1 +fi +echo "parallelism: $parallelism" -apt install -y \ +sudo apt update -y + +sudo apt install -y \ ninja-build ragel libhwloc-dev libnuma-dev libpciaccess-dev vim wget curl \ - git g++ libgoogle-glog-dev cmake libopenmpi-dev default-jdk libcrypto++-dev \ - libboost-all-dev libxml2-dev -apt install -y xfslibs-dev libgnutls28-dev liblz4-dev maven openssl pkg-config \ + git g++ libunwind-dev libgoogle-glog-dev cmake libopenmpi-dev default-jdk libcrypto++-dev \ + libboost-all-dev libxml2-dev protobuf-compiler libprotobuf-dev +sudo apt install -y xfslibs-dev libgnutls28-dev liblz4-dev maven openssl pkg-config \ libsctp-dev gcc make python3 systemtap-sdt-dev libtool libyaml-cpp-dev \ libc-ares-dev stow libfmt-dev diffutils valgrind doxygen python3-pip net-tools graphviz +pushd /tmp git clone https://github.com/alibaba/libgrape-lite.git cd libgrape-lite git checkout v0.3.2 -mkdir build && cd build && cmake .. -make -j && make install -cp /usr/local/lib/libgrape-lite.so /usr/lib/libgrape-lite.so +mkdir build && cd build && cmake .. -DBUILD_LIBGRAPELITE_TESTS=OFF +make -j ${parallelism} && sudo make install +sudo cp /usr/local/lib/libgrape-lite.so /usr/lib/libgrape-lite.so +popd && rm -rf /tmp/libgrape-lite -pushd /tmp && apt-get install -y -V ca-certificates lsb-release wget +pushd /tmp && sudo apt-get install -y -V ca-certificates lsb-release wget curl -o apache-arrow-apt-source-latest.deb https://apache.jfrog.io/artifactory/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb -apt-get install -y ./apache-arrow-apt-source-latest.deb -apt-get update && apt-get install -y libarrow-dev=6.0.1-1 -popd +sudo apt-get install -y ./apache-arrow-apt-source-latest.deb +sudo apt-get update && sudo apt-get install -y libarrow-dev=8.0.0-1 +popd && rm -rf /tmp/apache-arrow-apt-source-latest.deb -cd ../.. -git clone https://github.com/alibaba/hiactor.git -cd hiactor && git checkout e16949ca53 -git submodule update --init --recursive -./seastar/seastar/install-dependencies.sh +pushd /tmp +git clone https://github.com/alibaba/hiactor.git -b v0.1.1 --single-branch +cd hiactor && git submodule update --init --recursive +sudo bash ./seastar/seastar/install-dependencies.sh mkdir build && cd build cmake -DHiactor_DEMOS=OFF -DHiactor_TESTING=OFF -DHiactor_DPDK=OFF -DHiactor_CXX_DIALECT=gnu++17 -DSeastar_CXX_FLAGS="-DSEASTAR_DEFAULT_ALLOCATOR -mno-avx512" .. -make -j && make install +make -j ${parallelism} && sudo make install +popd && rm -rf /tmp/hiactor -echo "fs.aio-max-nr = 1048576" >> /etc/sysctl.conf -sysctl -p /etc/sysctl.conf \ No newline at end of file +sudo sh -c 'echo "fs.aio-max-nr = 1048576" >> /etc/sysctl.conf' +sudo sysctl -p /etc/sysctl.conf \ No newline at end of file diff --git a/flex/utils/arrow_utils.h b/flex/utils/arrow_utils.h index 8b2678dd9f16..d17c00629136 100644 --- a/flex/utils/arrow_utils.h +++ b/flex/utils/arrow_utils.h @@ -32,7 +32,8 @@ class LDBCTimeStampParser : public arrow::TimestampParser { ~LDBCTimeStampParser() override {} bool operator()(const char* s, size_t length, arrow::TimeUnit::type out_unit, - int64_t* out) const override { + int64_t* out, + bool* out_zone_offset_present = NULLPTR) const override { using seconds_type = std::chrono::duration; // We allow the following formats for all units: @@ -154,7 +155,8 @@ class LDBCLongDateParser : public arrow::TimestampParser { ~LDBCLongDateParser() override {} bool operator()(const char* s, size_t length, arrow::TimeUnit::type out_unit, - int64_t* out) const override { + int64_t* out, + bool* out_zone_offset_present = NULLPTR) const override { uint64_t seconds; // convert (s, s + length - 4) to seconds_since_epoch if (ARROW_PREDICT_FALSE(