From 07a50d789abf2182ae89397eda5fba56fc23852b Mon Sep 17 00:00:00 2001 From: "xiaolei.zl" Date: Wed, 13 Sep 2023 17:22:02 +0800 Subject: [PATCH] check ci --- .github/workflows/hqps-db-ci.yml | 24 +++++------ flex/bin/sync_server.cc | 4 +- flex/codegen/src/hqps/hqps_scan_builder.h | 50 ++++++++++++++++++++++- flex/interactive/bin/gs_interactive | 26 ++++++------ flex/tests/hqps/hqps_cypher_test.sh | 23 ++--------- 5 files changed, 79 insertions(+), 48 deletions(-) diff --git a/.github/workflows/hqps-db-ci.yml b/.github/workflows/hqps-db-ci.yml index 8f875b64ae36..036baa3c3a2c 100644 --- a/.github/workflows/hqps-db-ci.yml +++ b/.github/workflows/hqps-db-ci.yml @@ -86,7 +86,7 @@ jobs: GS_TEST_DIR: ${{ github.workspace }}/gstest run: | # download dataset - git clone -b master --single-branch --depth=1 https://github.com/Graphscope/gstest.git ${GS_TEST_DIR} + git clone -b master --single-branch --depth=1 https://github.com/zhanglei1949/gstest.git ${GS_TEST_DIR} - name: Sample Query test env: @@ -106,21 +106,21 @@ jobs: GIE_HOME=${GITHUB_WORKSPACE}/interactive_engine # create tmp ir.compiler.properties - touch /tmp/ir.compiler.properties - echo "engine.type: hiactor" >> /tmp/ir.compiler.properties - echo "graph.schema: ${GS_TEST_DIR}/flex/ldbc-sf01-long-date/ldbc_schema_csr_ic.json" >> /tmp/ir.compiler.properties - echo "graph.store: exp" >> /tmp/ir.compiler.properties - echo "graph.planner.is.on: true" >> /tmp/ir.compiler.properties - echo "graph.planner.opt: RBO" >> /tmp/ir.compiler.properties - echo "graph.planner.rules: FilterMatchRule" >> /tmp/ir.compiler.properties + # touch /tmp/ir.compiler.properties + # echo "engine.type: hiactor" >> /tmp/ir.compiler.properties + # echo "graph.schema: ${GS_TEST_DIR}/flex/ldbc-sf01-long-date/ldbc_schema_csr_ic.json" >> /tmp/ir.compiler.properties + # echo "graph.store: exp" >> /tmp/ir.compiler.properties + # echo "graph.planner.is.on: true" >> /tmp/ir.compiler.properties + # echo "graph.planner.opt: RBO" >> /tmp/ir.compiler.properties + # echo "graph.planner.rules: FilterMatchRule" >> /tmp/ir.compiler.properties cd ${GITHUB_WORKSPACE}/flex/bin for i in 2 3 5 6 8 9 11 12; do cmd="./load_plan_and_gen.sh -e=hqps -i=../resources/queries/ic/adhoc/ic${i}_adhoc.cypher -w=/tmp/codgen/" - cmd=${cmd}" -o=/tmp/plugin --ir_conf=/tmp/ir.compiler.properties " - cmd=${cmd}" --graph_schema_path=${GS_TEST_DIR}/flex/ldbc-sf01-long-date/ldbc_schema_csr_ic.json" + cmd=${cmd}" -o=/tmp/plugin --ir_conf=${GS_TEST_DIR}/flex/ldbc-sf01-long-date/engine_config.yaml " + cmd=${cmd}" --graph_schema_path=${GS_TEST_DIR}/flex/ldbc-sf01-long-date/audit_graph_schema.yaml" cmd=${cmd}" --gie_home=${GIE_HOME}" echo $cmd eval ${cmd} @@ -129,8 +129,8 @@ jobs: for i in 1 2 3 4 5 6 7 8 9; do cmd="./load_plan_and_gen.sh -e=hqps -i=../resources/queries/ic/adhoc/simple_match_${i}.cypher -w=/tmp/codgen/" - cmd=${cmd}" -o=/tmp/plugin --ir_conf=/tmp/ir.compiler.properties " - cmd=${cmd}" --graph_schema_path=${GS_TEST_DIR}/flex/ldbc-sf01-long-date/ldbc_schema_csr_ic.json" + cmd=${cmd}" -o=/tmp/plugin --ir_conf=${GS_TEST_DIR}/flex/ldbc-sf01-long-date/engine_config.yaml " + cmd=${cmd}" --graph_schema_path=${GS_TEST_DIR}/flex/ldbc-sf01-long-date/audit_graph_schema.yaml" cmd=${cmd}" --gie_home=${GIE_HOME}" echo $cmd eval ${cmd} diff --git a/flex/bin/sync_server.cc b/flex/bin/sync_server.cc index 13c8736e735a..616a6ca5e0e5 100644 --- a/flex/bin/sync_server.cc +++ b/flex/bin/sync_server.cc @@ -175,8 +175,8 @@ void init_codegen_proxy(const bpo::variables_map& vm, LOG(FATAL) << "gie-home not exists: " << gie_home; } } - server::CodegenProxy::get().Init(codegen_dir, codegen_bin, graph_schema_file, - engine_config_file, gie_home); + server::CodegenProxy::get().Init(codegen_dir, codegen_bin, engine_config_file, + graph_schema_file, gie_home); } } // namespace gs diff --git a/flex/codegen/src/hqps/hqps_scan_builder.h b/flex/codegen/src/hqps/hqps_scan_builder.h index 27e27bd92526..405a5fed2bf7 100644 --- a/flex/codegen/src/hqps/hqps_scan_builder.h +++ b/flex/codegen/src/hqps/hqps_scan_builder.h @@ -89,6 +89,52 @@ class ScanOpBuilder { return *this; } + ScanOpBuilder& idx_predicate(const algebra::IndexPredicate& predicate) { + // check query_params not has predicate. + if (query_params_.has_predicate()) { + throw std::runtime_error( + std::string("Currently only support one predicate")); + } + // Currently we only support one predicate. + if (predicate.or_predicates_size() < 1) { + return *this; + } + if (predicate.or_predicates_size() != 1) { + throw std::runtime_error( + std::string("Currently only support one predicate")); + } + auto or_predicate = predicate.or_predicates(0); + if (or_predicate.predicates_size() != 1) { + throw std::runtime_error( + std::string("Currently only support one and predicate")); + } + auto triplet = or_predicate.predicates(0); + // add index predicate to query params + auto* new_predicate = query_params_.mutable_predicate(); + { + auto first_op = new_predicate->add_operators(); + common::Variable variable; + auto& property = triplet.key(); + *(variable.mutable_property()) = property; + variable.mutable_node_type()->set_data_type(common::DataType::INT64); + *(first_op->mutable_var()) = variable; + } + { + auto second = new_predicate->add_operators(); + second->set_logical(common::Logical::EQ); + second->mutable_node_type()->set_data_type(common::DataType::BOOLEAN); + } + { + auto third = new_predicate->add_operators(); + auto& value = triplet.value(); + third->mutable_node_type()->set_data_type(common::DataType::INT64); + *(third->mutable_const_()) = value; + } + VLOG(10) << "Add index predicate to query params: " + << query_params_.DebugString(); + return *this; + } + std::string Build() const { std::string label_name; std::vector labels_ids; @@ -275,7 +321,9 @@ static std::string BuildScanOp( } else { builder.resAlias(-1); } - return builder.queryParams(scan_pb.params()).Build(); + return builder.queryParams(scan_pb.params()) + .idx_predicate(scan_pb.idx_predicate()) + .Build(); } } // namespace gs diff --git a/flex/interactive/bin/gs_interactive b/flex/interactive/bin/gs_interactive index 47c1aa8b2164..df907199123a 100755 --- a/flex/interactive/bin/gs_interactive +++ b/flex/interactive/bin/gs_interactive @@ -389,21 +389,21 @@ function update_engine_config_from_yaml(){ function init_usage() { cat << EOF - db_admin.sh init -c [--config] + gs_interactive init -c [--config] Init the database, create the containers. Specify the database version and volume mounting in the config yaml. EOF } function destroy_usage() { cat << EOF - db_admin.sh destroy + gs_interactive destroy Destroy the current database, remove the container. EOF } function create_usage() { cat << EOF - db_admin.sh database create -g [--graph] -c [--config] + gs_interactive database create -g [--graph] -c [--config] Create a graph in database, with the provided schema file. User should import data to the created graph. EOF @@ -411,14 +411,14 @@ EOF function remove_usage() { cat << EOF - db_admin.sh database remove -g [--graph] + gs_interactive database remove -g [--graph] Remove the database with the given graph. EOF } function import_usage() { cat << EOF - db_admin.sh database import -g [--graph] -c [--config] + gs_interactive database import -g [--graph] -c [--config] Load the raw data specified in bulk load file to the specified graph. EOF } @@ -432,28 +432,28 @@ function database_usage(){ function start_usage() { cat << EOF - db_admin.sh service start -g [--graph] -c [--config] + gs_interactive service start -g [--graph] -c [--config] Start the graph service on the specified graph, with the provided engine config file. EOF } function stop_usage() { cat << EOF - db_admin.sh service stop + gs_interactive service stop Stop the database with the given graph. EOF } function restart_usage() { cat << EOF - db_admin.sh service restart -c [--config] [engine config file] + gs_interactive service restart -c [--config] [engine config file] Restart the database with current running graph. Can update with new engine config file. EOF } function get_log_usage() { cat << EOF - db_admin.sh service get_log -o [--output] output directory + gs_interactive service get_log -o [--output] output directory Get the log of the specified service/compiler, and write to the output file. EOF } @@ -467,7 +467,7 @@ function services_usage(){ function compile_usage(){ cat << EOF - db_admin.sh procedure compile -g[--graph] -i -i -d [--description] --compile_only Compile cypher/.cc to dynamic library, according to the schema of graph. The output library will be placed at ./data/{graph_name}/lib. @@ -479,7 +479,7 @@ EOF function enable_proc_usage(){ cat << EOF - db_admin.sh procedure enable -g[--graph] -n[--name] + gs_interactive procedure enable -g[--graph] -n[--name] -c[--config] Enable the stored procedure in the given graph, with the provided library. stored_procedures.yaml contains the the stored procedure names at each line. @@ -488,7 +488,7 @@ EOF function disable_proc_usage(){ cat << EOF - db_admin.sh procedure disable -g[--graph] -n[--name] + gs_interactive procedure disable -g[--graph] -n[--name] -c[--config] Disable the stored procedure in the given graph, with the provided library. stored_procedures.yaml contains the the stored procedure names at each line. @@ -497,7 +497,7 @@ EOF function show_stored_procedure_usage(){ cat << EOF - db_admin.sh procedure show -g[--graph] graph_name + gs_interactive procedure show -g[--graph] graph_name Show all stored procedure for the given graph. EOF } diff --git a/flex/tests/hqps/hqps_cypher_test.sh b/flex/tests/hqps/hqps_cypher_test.sh index b06a70cc69d5..d8f4d152560c 100644 --- a/flex/tests/hqps/hqps_cypher_test.sh +++ b/flex/tests/hqps/hqps_cypher_test.sh @@ -36,7 +36,6 @@ GRAPH_SCHEMA_YAML=${GS_TEST_DIR}/flex/ldbc-sf01-long-date/audit_graph_schema.yam GRAPH_BULK_LOAD_YAML=${GS_TEST_DIR}/flex/ldbc-sf01-long-date/audit_bulk_load.yaml COMPILER_GRAPH_SCHEMA=${GS_TEST_DIR}/flex/ldbc-sf01-long-date/audit_graph_schema.yaml GRAPH_CSR_DATA_DIR=${HOME}/csr-data-dir/ -HQPS_IR_CONF=/tmp/hqps.ir.properties # check if GRAPH_SCHEMA_YAML exists if [ ! -f ${GRAPH_SCHEMA_YAML} ]; then echo "GRAPH_SCHEMA_YAML: ${GRAPH_SCHEMA_YAML} not found" @@ -79,21 +78,6 @@ kill_service(){ # kill service when exit trap kill_service EXIT -create_ir_conf(){ - rm ${HQPS_IR_CONF} || true - echo "engine.type: hiactor" >> ${HQPS_IR_CONF} - echo "hiactor.hosts: localhost:10000" >> ${HQPS_IR_CONF} - echo "graph.store: exp" >> ${HQPS_IR_CONF} - echo "graph.schema: ${GS_TEST_DIR}/flex/ldbc-sf01-long-date/ldbc_schema_csr_ic.json" >> ${HQPS_IR_CONF} - echo "graph.planner.is.on: true" >> ${HQPS_IR_CONF} - echo "graph.planner.opt: RBO" >> ${HQPS_IR_CONF} - echo "graph.planner.rules: FilterMatchRule" >> ${HQPS_IR_CONF} - echo "gremlin.server.disabled: true" >> ${HQPS_IR_CONF} - echo "neo4j.bolt.server.port: 7687" >> ${HQPS_IR_CONF} - - echo "Finish generate HQPS_IR_CONF" - cat ${HQPS_IR_CONF} -} # start engine service and load ldbc graph start_engine_service(){ @@ -105,7 +89,7 @@ start_engine_service(){ # export FLEX_DATA_DIR export FLEX_DATA_DIR=${GS_TEST_DIR}/flex/ldbc-sf01-long-date/ - cmd="${SERVER_BIN} -c ${ENGINE_CONFIG_PATH} -g ${GRAPH_SCHEMA_YAML} " + cmd="${SERVER_BIN} -c ${DST_ENGINE_CONFIG_PATH} -g ${GRAPH_SCHEMA_YAML} " cmd="${cmd} --data-path ${GRAPH_CSR_DATA_DIR} -l ${GRAPH_BULK_LOAD_YAML} " cmd="${cmd} --gie-home ${GIE_HOME}" @@ -122,7 +106,7 @@ start_engine_service(){ start_compiler_service(){ echo "try to start compiler service" pushd ${GIE_HOME}/compiler - cmd="make run graph.schema:=${COMPILER_GRAPH_SCHEMA} config.path=${HQPS_IR_CONF}" + cmd="make run graph.schema=${COMPILER_GRAPH_SCHEMA} config.path=${DST_ENGINE_CONFIG_PATH}" echo "Start compiler service with command: ${cmd}" ${cmd} & sleep 5 @@ -153,10 +137,9 @@ run_simple_test(){ } kill_service -create_ir_conf start_engine_service start_compiler_service -#run_ldbc_test +run_ldbc_test run_simple_test kill_service