ci: Reuse the compiled libraries/binaries between jobs for interactiv…

…e CI (#4304) Reuse compiled libraries and binaries across CI jobs for Interactive CI.
alibaba · Oct 29, 2024 · 49b2643 · 49b2643
1 parent e10d273
commit 49b2643
Show file tree

Hide file tree

Showing 5 changed files with 74 additions and 19 deletions.
diff --git a/.github/workflows/interactive.yml b/.github/workflows/interactive.yml
@@ -24,8 +24,52 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
+  build-interactive:
+    runs-on: ubuntu-20.04
+    if: ${{ github.repository == 'alibaba/GraphScope' }}
+    container:
+      image: registry.cn-hongkong.aliyuncs.com/graphscope/graphscope-dev:v0.24.2-amd64
+    steps:
+    - uses: actions/checkout@v4
+    - name: Install latest libgrape-lite
+      if: false
+      run: |
+        git clone --single-branch https://github.com/alibaba/libgrape-lite.git /tmp/libgrape-lite
+        cd /tmp/libgrape-lite
+        mkdir -p build && cd build
+        cmake ..
+        make -j$(nproc)
+        make install
+
+    - name: Setup tmate session
+      if: false
+      uses: mxschmitt/action-tmate@v3
+
+    - name: Build Interactive
+      env:
+        GIE_HOME: ${{ github.workspace }}/interactive_engine/
+        HOME: /home/graphscope/
+      run: |
+        . /home/graphscope/.graphscope_env
+        cd ${GITHUB_WORKSPACE}/
+        git submodule update --init
+        cd ${GITHUB_WORKSPACE}/flex
+        mkdir build && cd build
+        cmake .. -DCMAKE_INSTALL_PREFIX=/opt/graphscope && sudo make -j$(nproc)
+        # package the build artifacts
+        cd .. && tar -zcf build.tar.gz build
+    
+    - name: Upload Artifacts
+      uses: actions/upload-artifact@v4
+      with:
+        name: interactive_build-${{ github.sha }}
+        path: |
+          ${{ github.workspace }}/flex/build.tar.gz
+
+
   test-hqps-engine:
     runs-on: ubuntu-20.04
+    needs: build-interactive
     if: ${{ github.repository == 'alibaba/GraphScope' }}
     container:
       image: registry.cn-hongkong.aliyuncs.com/graphscope/graphscope-dev:v0.24.2-amd64
@@ -58,23 +102,28 @@ jobs:
         cmake ..
         make -j$(nproc)
         make install
+    
+    - name: Download Artifacts
+      uses: actions/download-artifact@v4
+      with:
+        name: interactive_build-${{ github.sha }}
 
     - name: Setup tmate session
       if: false
       uses: mxschmitt/action-tmate@v3
 
+    - name: Extract build artifacts
+      run: |
+        cd ${GITHUB_WORKSPACE}
+        tar zxf build.tar.gz -C flex && rm build.tar.gz
+
     - name: Build
       env:
         GIE_HOME: ${{ github.workspace }}/interactive_engine/
         HOME: /home/graphscope/
       run: |
         . /home/graphscope/.graphscope_env
         cd ${GITHUB_WORKSPACE}/
-        git submodule update --init
-        cd ${GITHUB_WORKSPACE}/flex
-        mkdir build && cd build
-        cmake .. && sudo make -j$(nproc)
-        sudo make install
 
         # cargo
         . /home/graphscope/.cargo/env
@@ -400,6 +449,7 @@ jobs:
 
   test-flex:
     runs-on: ubuntu-20.04
+    needs: build-interactive
     if: ${{ github.repository == 'alibaba/GraphScope' }}
     container:
       image: registry.cn-hongkong.aliyuncs.com/graphscope/graphscope-dev:v0.24.2-amd64
@@ -416,14 +466,15 @@ jobs:
         make -j$(nproc)
         make install
 
-    - name: Build
-      env:
-        HOME: /home/graphscope/
+    - name: Download Artifacts
+      uses: actions/download-artifact@v4
+      with:
+        name: interactive_build-${{ github.sha }}
+
+    - name: Extract build artifacts
       run: |
-        cd ${GITHUB_WORKSPACE}/flex
-        git submodule update --init
-        mkdir build && cd build
-        cmake .. && sudo make -j$(nproc)
+        cd ${GITHUB_WORKSPACE}
+        tar zxf build.tar.gz -C flex && rm build.tar.gz
 
     - name: Test GRIN on mutable csr 
       run: |

diff --git a/flex/engines/graph_db/grin/CMakeLists.txt b/flex/engines/graph_db/grin/CMakeLists.txt
@@ -63,6 +63,11 @@ include_directories(${CMAKE_CURRENT_SOURCE_DIR}/../../../..)
 include_directories(${CMAKE_CURRENT_SOURCE_DIR}/../../../storages/rt_mutable_graph)
 include_directories(${CMAKE_CURRENT_SOURCE_DIR}/../../../utils/property)
 include_directories(${CMAKE_CURRENT_SOURCE_DIR}/include)
+if (EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/../../../build/utils)
+    include_directories(${CMAKE_CURRENT_SOURCE_DIR}/../../../build/utils)
+else ()
+    message(FATAL_ERROR "Please build flex first")
+endif ()
 
 file(GLOB_RECURSE FILES_NEED_FORMAT "src/*.cc")
 

diff --git a/flex/engines/http_server/CMakeLists.txt b/flex/engines/http_server/CMakeLists.txt
@@ -4,7 +4,7 @@ if (Hiactor_FOUND)
 
   hiactor_codegen (server_actor_autogen server_actor_autogen_files
         SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/
-        INCLUDE_PATHS ${Hiactor_INCLUDE_DIR},${CMAKE_CURRENT_SOURCE_DIR}/../../../)
+        INCLUDE_PATHS ${Hiactor_INCLUDE_DIR},${CMAKE_CURRENT_SOURCE_DIR}/../../../,${CMAKE_CURRENT_BINARY_DIR}/../../utils/)
 
 
   # get all .cc files in current directory, except for generated/
@@ -32,7 +32,7 @@ if (Hiactor_FOUND)
   endif ()
   set_target_properties(Hiactor::seastar PROPERTIES INTERFACE_COMPILE_OPTIONS "${seastar_options}")
 
-  target_include_directories(flex_server PUBLIC ${CMAKE_CURRENT_BINARY_DIR}/../hqps/)
+  target_include_directories(flex_server PUBLIC ${CMAKE_CURRENT_BINARY_DIR}/../../utils/)
   target_link_libraries(flex_server flex_plan_proto)
   if (OPENTELEMETRY_CPP_FOUND)
     target_link_libraries(flex_server otel)

diff --git a/flex/utils/CMakeLists.txt b/flex/utils/CMakeLists.txt
@@ -6,7 +6,7 @@ set(INTERACTIVE_ERROR_PROTO_FILES
         ${GRAPHSCOPE_ERROR_PROTO_DIR}/interactive.proto
 )
 #create directory first
-file(MAKE_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/error_pb)
+file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/flex/utils/error_pb)
 
 # proto gen for gie proto
 protobuf_generate(APPEND_PATH
@@ -15,7 +15,7 @@ protobuf_generate(APPEND_PATH
         OUT_VAR ERROR_PROTO_SRCS
         PROTOS ${INTERACTIVE_ERROR_PROTO_FILES}
         IMPORT_DIRS ${GRAPHSCOPE_ERROR_PROTO_DIR}
-        PROTOC_OUT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/error_pb
+        PROTOC_OUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/flex/utils/error_pb
 )
 
 
@@ -56,8 +56,8 @@ target_link_libraries(flex_plan_proto PUBLIC ${Protobuf_LIBRARIES})
 install_flex_target(flex_plan_proto)
 
 #install proto_generated_files
-install(DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/flex/proto_generated_gie/
-        DESTINATION include/flex/proto_generated_gie/
+install(DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/flex
+        DESTINATION include
         FILES_MATCHING
         PATTERN "*.h"
 )

diff --git a/flex/utils/service_utils.h b/flex/utils/service_utils.h
@@ -32,7 +32,6 @@
 #include <vector>
 
 #include "flex/utils/property/types.h"
-#include "flex/utils/result.h"
 #include "flex/utils/yaml_utils.h"
 
 #include <glog/logging.h>