diff --git a/docs/flex/interactive/development/dev_and_test.md b/docs/flex/interactive/development/dev_and_test.md index 8663b354c75a..3a99b3035ce9 100644 --- a/docs/flex/interactive/development/dev_and_test.md +++ b/docs/flex/interactive/development/dev_and_test.md @@ -57,6 +57,21 @@ The Interactive Query Engine code is organized in the `flex` folder as follows: +### Dependency Graph + +Interactive follows the lego-like building concept of GraphScope Flex, comprising multiple modules. A dependency graph illustrates the relationships among these modules, although only a subset of third-party dependencies is included for clarity. + + +:::{figure-md} + +Dependency Graph between modules + +Dependency Graph between modules +::: + + ### Compiler The Compiler is crucial in Interactive as it converts graph queries written in graph query languages (Cypher/Gremlin) into physical query plans using GAIA IR. diff --git a/docs/images/flex_interactive_dep_graph.png b/docs/images/flex_interactive_dep_graph.png new file mode 100644 index 000000000000..8980b6e3c8dd Binary files /dev/null and b/docs/images/flex_interactive_dep_graph.png differ diff --git a/flex/CMakeLists.txt b/flex/CMakeLists.txt index e0ba1169dd6e..2ebfa4f0e6b2 100644 --- a/flex/CMakeLists.txt +++ b/flex/CMakeLists.txt @@ -224,7 +224,7 @@ add_subdirectory(storages) add_subdirectory(engines) add_subdirectory(bin) if (OPENTELEMETRY_CPP_FOUND) - add_subdirectory(otel) + add_subdirectory(otel) endif() if (BUILD_TEST) add_subdirectory(tests) diff --git a/flex/bin/CMakeLists.txt b/flex/bin/CMakeLists.txt index 34f92b96bd2e..a9e93f445196 100644 --- a/flex/bin/CMakeLists.txt +++ b/flex/bin/CMakeLists.txt @@ -27,7 +27,7 @@ install_without_export_flex_target(flex_analytical_engine) if(Hiactor_FOUND) add_executable(interactive_server interactive_server.cc) - target_link_libraries(interactive_server flex_utils flex_graph_db flex_server hqps_plan_proto flex_utils ${GLOG_LIBRARIES} ${GFLAGS_LIBRARIES}) + target_link_libraries(interactive_server flex_utils flex_graph_db flex_server flex_plan_proto flex_utils ${GLOG_LIBRARIES} ${GFLAGS_LIBRARIES}) if (OPENTELEMETRY_CPP_FOUND) target_link_libraries(interactive_server otel) endif() diff --git a/flex/codegen/CMakeLists.txt b/flex/codegen/CMakeLists.txt index 02b5d0b6f0ea..55e679406880 100644 --- a/flex/codegen/CMakeLists.txt +++ b/flex/codegen/CMakeLists.txt @@ -1,8 +1,3 @@ - -find_package(Boost REQUIRED COMPONENTS system filesystem - context program_options regex thread) -include_directories(SYSTEM ${Boost_INCLUDE_DIRS}) - add_executable(gen_code_from_plan gen_code_from_plan.cc) -target_link_libraries(gen_code_from_plan flex_rt_mutable_graph flex_utils hqps_plan_proto ${GLOG_LIBRARIES} ${Boost_LIBRARIES}) +target_link_libraries(gen_code_from_plan flex_rt_mutable_graph flex_utils flex_plan_proto ${GLOG_LIBRARIES} ${Boost_LIBRARIES}) install_flex_target(gen_code_from_plan) diff --git a/flex/engines/graph_db/CMakeLists.txt b/flex/engines/graph_db/CMakeLists.txt index f280a20a5940..6618b6abf5dc 100644 --- a/flex/engines/graph_db/CMakeLists.txt +++ b/flex/engines/graph_db/CMakeLists.txt @@ -5,9 +5,8 @@ file(GLOB_RECURSE GRAPH_DB_SRC_FILES "${CMAKE_CURRENT_SOURCE_DIR}/app/*.cc" add_library(flex_graph_db SHARED ${GRAPH_DB_SRC_FILES}) target_include_directories(flex_graph_db PUBLIC $) -target_link_libraries(flex_graph_db flex_rt_mutable_graph flex_utils ${GLOG_LIBRARIES} ${LIBGRAPELITE_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT}) -target_link_libraries(flex_graph_db hqps_plan_proto) -target_link_libraries(flex_graph_db runtime_adhoc) +target_link_libraries(flex_graph_db flex_rt_mutable_graph flex_utils ${LIBGRAPELITE_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT}) +target_link_libraries(flex_graph_db flex_plan_proto runtime_adhoc) install_flex_target(flex_graph_db) install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/database/graph_db.h diff --git a/flex/engines/graph_db/runtime/CMakeLists.txt b/flex/engines/graph_db/runtime/CMakeLists.txt index 8868a143501a..99d58d16d9ce 100644 --- a/flex/engines/graph_db/runtime/CMakeLists.txt +++ b/flex/engines/graph_db/runtime/CMakeLists.txt @@ -1,12 +1,16 @@ file(GLOB_RECURSE COMMON_SOURCES "common/*.cc") add_library(runtime_common SHARED ${COMMON_SOURCES}) -target_link_libraries(runtime_common ${Boost_LIBRARIES} flex_rt_mutable_graph flex_utils hqps_plan_proto) +target_link_libraries(runtime_common ${Boost_LIBRARIES} flex_utils flex_plan_proto) install_flex_target(runtime_common) file(GLOB_RECURSE ADHOC_SOURCES "adhoc/*.cc") add_library(runtime_adhoc SHARED ${ADHOC_SOURCES}) target_link_libraries(runtime_adhoc runtime_common) -target_link_libraries(runtime_adhoc Boost::headers) install_flex_target(runtime_adhoc) +install(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} + DESTINATION include/flex/engines/graph_db + FILES_MATCHING + PATTERN "*.h" +) \ No newline at end of file diff --git a/flex/engines/hqps_db/CMakeLists.txt b/flex/engines/hqps_db/CMakeLists.txt index ff9e8a821999..97a34295cf5a 100644 --- a/flex/engines/hqps_db/CMakeLists.txt +++ b/flex/engines/hqps_db/CMakeLists.txt @@ -1,46 +1,5 @@ -set(GIE_COMPILER_PROTO_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../../../interactive_engine/executor/ir/proto/) -set(GIE_COMPILER_PROTO_JOB_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../../../interactive_engine/executor/engine/pegasus/server/proto/) -# proto files of gie compiler -set(CODE_GEN_PROTOBUF_FILES - ${GIE_COMPILER_PROTO_DIR}/algebra.proto - ${GIE_COMPILER_PROTO_DIR}/common.proto - ${GIE_COMPILER_PROTO_DIR}/expr.proto - ${GIE_COMPILER_PROTO_DIR}/physical.proto - ${GIE_COMPILER_PROTO_DIR}/results.proto - ${GIE_COMPILER_PROTO_DIR}/schema.proto - ${GIE_COMPILER_PROTO_DIR}/type.proto - ${GIE_COMPILER_PROTO_DIR}/stored_procedure.proto - ${GIE_COMPILER_PROTO_JOB_DIR}/job_service.proto -) - -#create directory first -file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/flex/proto_generated_gie) - -# proto gen for gie proto -protobuf_generate(APPEND_PATH - TARGET ${LOCAL_EXE_NAME} - LANGUAGE cpp - OUT_VAR PROTO_SRCS_GIE - PROTOS ${CODE_GEN_PROTOBUF_FILES} - IMPORT_DIRS ${GIE_COMPILER_PROTO_DIR} - PROTOC_OUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/flex/proto_generated_gie -) - -add_library(hqps_plan_proto SHARED ${PROTO_SRCS_GIE}) -target_include_directories(hqps_plan_proto PUBLIC - $ - $) -target_link_libraries(hqps_plan_proto PUBLIC ${Protobuf_LIBRARIES}) -install_flex_target(hqps_plan_proto) - install(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} DESTINATION include/flex/engines/ FILES_MATCHING PATTERN "*.h" -) -#install proto_generated_files -install(DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/flex/proto_generated_gie/ - DESTINATION include/flex/proto_generated_gie/ - FILES_MATCHING - PATTERN "*.h" -) +) \ No newline at end of file diff --git a/flex/engines/http_server/CMakeLists.txt b/flex/engines/http_server/CMakeLists.txt index c87b9b542583..2d65b452cf12 100644 --- a/flex/engines/http_server/CMakeLists.txt +++ b/flex/engines/http_server/CMakeLists.txt @@ -33,7 +33,7 @@ if (Hiactor_FOUND) set_target_properties(Hiactor::seastar PROPERTIES INTERFACE_COMPILE_OPTIONS "${seastar_options}") target_include_directories(flex_server PUBLIC ${CMAKE_CURRENT_BINARY_DIR}/../hqps/) - target_link_libraries(flex_server hqps_plan_proto) + target_link_libraries(flex_server flex_plan_proto) if (OPENTELEMETRY_CPP_FOUND) target_link_libraries(flex_server otel) endif() diff --git a/flex/flex-config.in.cmake b/flex/flex-config.in.cmake index 10d87bb46e60..d24de2a826eb 100644 --- a/flex/flex-config.in.cmake +++ b/flex/flex-config.in.cmake @@ -9,7 +9,7 @@ set(FLEX_HOME "${CMAKE_CURRENT_LIST_DIR}/../../..") include("${CMAKE_CURRENT_LIST_DIR}/flex-targets.cmake") -set(FLEX_LIBRARIES flex::flex_utils flex::flex_rt_mutable_graph flex::flex_graph_db flex::flex_bsp flex::flex_immutable_graph flex::hqps_plan_proto) +set(FLEX_LIBRARIES flex::flex_utils flex::flex_rt_mutable_graph flex::flex_graph_db flex::flex_bsp flex::flex_immutable_graph flex::flex_plan_proto) set(FLEX_INCLUDE_DIR "${FLEX_HOME}/include") set(FLEX_INCLUDE_DIRS "${FLEX_INCLUDE_DIR}") diff --git a/flex/storages/metadata/CMakeLists.txt b/flex/storages/metadata/CMakeLists.txt index 5a146c480e45..dc2dcfd657b5 100644 --- a/flex/storages/metadata/CMakeLists.txt +++ b/flex/storages/metadata/CMakeLists.txt @@ -2,7 +2,7 @@ file(GLOB_RECURSE METADATA_SRC_FILES "${CMAKE_CURRENT_SOURCE_DIR}/*.cc") add_library(flex_metadata_store SHARED ${METADATA_SRC_FILES}) -target_link_libraries(flex_metadata_store ${YAML_CPP_LIBRARIES} flex_utils) +target_link_libraries(flex_metadata_store flex_utils) install_flex_target(flex_metadata_store) diff --git a/flex/storages/rt_mutable_graph/CMakeLists.txt b/flex/storages/rt_mutable_graph/CMakeLists.txt index 2e6775c67cd3..9c761c476826 100644 --- a/flex/storages/rt_mutable_graph/CMakeLists.txt +++ b/flex/storages/rt_mutable_graph/CMakeLists.txt @@ -9,7 +9,7 @@ if (NOT BUILD_ODPS_FRAGMENT_LOADER) list(REMOVE_ITEM RT_MUTABLE_GRAPH_SRC_FILES "${CMAKE_CURRENT_SOURCE_DIR}/loader/odps_fragment_loader.cc") endif() add_library(flex_rt_mutable_graph SHARED ${RT_MUTABLE_GRAPH_SRC_FILES}) -target_link_libraries(flex_rt_mutable_graph flex_utils ${LIBGRAPELITE_LIBRARIES} ${YAML_CPP_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT}) +target_link_libraries(flex_rt_mutable_graph flex_utils ${LIBGRAPELITE_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT}) install_flex_target(flex_rt_mutable_graph) install(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} diff --git a/flex/tests/hqps/CMakeLists.txt b/flex/tests/hqps/CMakeLists.txt index 4938217e4466..fefe2f346cc2 100644 --- a/flex/tests/hqps/CMakeLists.txt +++ b/flex/tests/hqps/CMakeLists.txt @@ -7,5 +7,5 @@ foreach(f ${GS_TEST_FILES}) set(T_NAME ${CMAKE_MATCH_1}) message(STATUS "Found graphscope test - " ${T_NAME}) add_executable(${T_NAME} ${CMAKE_CURRENT_SOURCE_DIR}/${T_NAME}.cc) - target_link_libraries(${T_NAME} hqps_plan_proto flex_rt_mutable_graph flex_graph_db flex_utils ${GLOG_LIBRARIES} ${LIBGRAPELITE_LIBRARIES}) + target_link_libraries(${T_NAME} flex_plan_proto flex_rt_mutable_graph flex_graph_db flex_utils ${GLOG_LIBRARIES} ${LIBGRAPELITE_LIBRARIES}) endforeach() diff --git a/flex/tests/rt_mutable_graph/CMakeLists.txt b/flex/tests/rt_mutable_graph/CMakeLists.txt index 8c9b0036472c..eec6bc2137be 100644 --- a/flex/tests/rt_mutable_graph/CMakeLists.txt +++ b/flex/tests/rt_mutable_graph/CMakeLists.txt @@ -1,5 +1,4 @@ -# file(GLOB_RECURSE GS_TEST_FILES "${CMAKE_CURRENT_SOURCE_DIR}/*.cc") file(GLOB GS_TEST_FILES RELATIVE "${CMAKE_CURRENT_SOURCE_DIR}" "${CMAKE_CURRENT_SOURCE_DIR}/*.cc") diff --git a/flex/utils/CMakeLists.txt b/flex/utils/CMakeLists.txt index b28da92a2083..4da3916c18b5 100644 --- a/flex/utils/CMakeLists.txt +++ b/flex/utils/CMakeLists.txt @@ -1,3 +1,4 @@ +############################################ Generate Error Code Proto ############################################ # The error code of FLEX Interactive is defined in $GRAPHSCOPE_HOME/proto/error/interactive.proto set(GRAPHSCOPE_ERROR_PROTO_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../../proto/error/) # proto files of gie compiler @@ -16,19 +17,59 @@ protobuf_generate(APPEND_PATH IMPORT_DIRS ${GRAPHSCOPE_ERROR_PROTO_DIR} PROTOC_OUT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/error_pb ) -file(GLOB_RECURSE UTILS_SRC_FILES "${CMAKE_CURRENT_SOURCE_DIR}/*.cc") -add_library(flex_utils SHARED ${UTILS_SRC_FILES} ${ERROR_PROTO_SRCS} ${Protobuf_LIBRARIES}) -target_include_directories(flex_utils PUBLIC +############################################ Generate Physical Plan Proto ############################################ +set(GIE_COMPILER_PROTO_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../../interactive_engine/executor/ir/proto/) +set(GIE_COMPILER_PROTO_JOB_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../../interactive_engine/executor/engine/pegasus/server/proto/) +# proto files of gie compiler +set(CODE_GEN_PROTOBUF_FILES + ${GIE_COMPILER_PROTO_DIR}/algebra.proto + ${GIE_COMPILER_PROTO_DIR}/common.proto + ${GIE_COMPILER_PROTO_DIR}/expr.proto + ${GIE_COMPILER_PROTO_DIR}/physical.proto + ${GIE_COMPILER_PROTO_DIR}/results.proto + ${GIE_COMPILER_PROTO_DIR}/schema.proto + ${GIE_COMPILER_PROTO_DIR}/type.proto + ${GIE_COMPILER_PROTO_DIR}/stored_procedure.proto + ${GIE_COMPILER_PROTO_JOB_DIR}/job_service.proto +) + +#create directory first +file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/flex/proto_generated_gie) + +# proto gen for gie proto +protobuf_generate(APPEND_PATH + TARGET ${LOCAL_EXE_NAME} + LANGUAGE cpp + OUT_VAR PROTO_SRCS_GIE + PROTOS ${CODE_GEN_PROTOBUF_FILES} + IMPORT_DIRS ${GIE_COMPILER_PROTO_DIR} + PROTOC_OUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/flex/proto_generated_gie +) + +add_library(flex_plan_proto SHARED ${PROTO_SRCS_GIE}) +target_include_directories(flex_plan_proto PUBLIC $ $) -# Link the static library of arrow, to save the trouble of linking the shared library of arrow -target_link_libraries(flex_utils ${ARROW_LIB}) - +target_link_libraries(flex_plan_proto PUBLIC ${Protobuf_LIBRARIES}) +install_flex_target(flex_plan_proto) -target_link_libraries(flex_utils ${YAML_CPP_LIBRARIES} ${Boost_LIBRARIES} ${Protobuf_LIBRARIES}) +#install proto_generated_files +install(DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/flex/proto_generated_gie/ + DESTINATION include/flex/proto_generated_gie/ + FILES_MATCHING + PATTERN "*.h" +) +############################################ Build Utils Library ############################################ +file(GLOB_RECURSE UTILS_SRC_FILES "${CMAKE_CURRENT_SOURCE_DIR}/*.cc") +add_library(flex_utils SHARED ${UTILS_SRC_FILES} ${ERROR_PROTO_SRCS}) +target_include_directories(flex_utils PUBLIC + $ + $) +# Link the static library of arrow, to save the trouble of linking the shared library of arrow +target_link_libraries(flex_utils ${ARROW_LIB} ${YAML_CPP_LIBRARIES} ${Boost_LIBRARIES} ${GLOG_LIBRARIES}) install_flex_target(flex_utils) install(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} diff --git a/k8s/dockerfiles/flex-interactive.Dockerfile b/k8s/dockerfiles/flex-interactive.Dockerfile index 8a978c0ec842..18e07560c23a 100644 --- a/k8s/dockerfiles/flex-interactive.Dockerfile +++ b/k8s/dockerfiles/flex-interactive.Dockerfile @@ -8,6 +8,7 @@ FROM $REGISTRY/graphscope/graphscope-dev:$VINEYARD_VERSION-$ARCH AS builder ARG ENABLE_COORDINATOR="false" ARG OPTIMIZE_FOR_HOST=OFF ARG ENABLE_OPENTELMETRY=false +ARG PARALLEL=8 RUN sudo mkdir -p /opt/flex && sudo chown -R graphscope:graphscope /opt/flex/ USER graphscope @@ -26,7 +27,7 @@ COPY --chown=graphscope:graphscope . /home/graphscope/GraphScope # install flex RUN . ${HOME}/.cargo/env && cd ${HOME}/GraphScope/flex && \ - git submodule update --init && mkdir build && cd build && cmake .. -DCMAKE_INSTALL_PREFIX=/opt/flex -DBUILD_DOC=OFF -DBUILD_TEST=OFF -DOPTIMIZE_FOR_HOST=${OPTIMIZE_FOR_HOST} && make -j && make install && \ + git submodule update --init && mkdir build && cd build && cmake .. -DCMAKE_INSTALL_PREFIX=/opt/flex -DBUILD_DOC=OFF -DBUILD_TEST=OFF -DOPTIMIZE_FOR_HOST=${OPTIMIZE_FOR_HOST} && make -j ${PARALLEL} && make install && \ cd ~/GraphScope/interactive_engine/ && mvn clean package -Pexperimental -DskipTests && \ cd ~/GraphScope/interactive_engine/compiler && cp target/compiler-0.0.1-SNAPSHOT.jar /opt/flex/lib/ && \ cp target/libs/*.jar /opt/flex/lib/ && \