Skip to content

Commit

Permalink
check ci
Browse files Browse the repository at this point in the history
  • Loading branch information
@zhanglei1949
zhanglei1949 committed Sep 13, 2023
1 parent cae326f commit 07a50d7
Show file tree
Hide file tree
Showing 5 changed files with 79 additions and 48 deletions.
24 changes: 12 additions & 12 deletions .github/workflows/hqps-db-ci.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -86,7 +86,7 @@ jobs:
GS_TEST_DIR: ${{ github.workspace }}/gstest
run: |
# download dataset
git clone -b master --single-branch --depth=1 https://github.com/Graphscope/gstest.git ${GS_TEST_DIR}
git clone -b master --single-branch --depth=1 https://github.com/zhanglei1949/gstest.git ${GS_TEST_DIR}
- name: Sample Query test
env:
Expand All @@ -106,21 +106,21 @@ jobs:
GIE_HOME=${GITHUB_WORKSPACE}/interactive_engine
# create tmp ir.compiler.properties
touch /tmp/ir.compiler.properties
echo "engine.type: hiactor" >> /tmp/ir.compiler.properties
echo "graph.schema: ${GS_TEST_DIR}/flex/ldbc-sf01-long-date/ldbc_schema_csr_ic.json" >> /tmp/ir.compiler.properties
echo "graph.store: exp" >> /tmp/ir.compiler.properties
echo "graph.planner.is.on: true" >> /tmp/ir.compiler.properties
echo "graph.planner.opt: RBO" >> /tmp/ir.compiler.properties
echo "graph.planner.rules: FilterMatchRule" >> /tmp/ir.compiler.properties
# touch /tmp/ir.compiler.properties
# echo "engine.type: hiactor" >> /tmp/ir.compiler.properties
# echo "graph.schema: ${GS_TEST_DIR}/flex/ldbc-sf01-long-date/ldbc_schema_csr_ic.json" >> /tmp/ir.compiler.properties
# echo "graph.store: exp" >> /tmp/ir.compiler.properties
# echo "graph.planner.is.on: true" >> /tmp/ir.compiler.properties
# echo "graph.planner.opt: RBO" >> /tmp/ir.compiler.properties
# echo "graph.planner.rules: FilterMatchRule" >> /tmp/ir.compiler.properties
cd ${GITHUB_WORKSPACE}/flex/bin
for i in 2 3 5 6 8 9 11 12;
do
cmd="./load_plan_and_gen.sh -e=hqps -i=../resources/queries/ic/adhoc/ic${i}_adhoc.cypher -w=/tmp/codgen/"
cmd=${cmd}" -o=/tmp/plugin --ir_conf=/tmp/ir.compiler.properties "
cmd=${cmd}" --graph_schema_path=${GS_TEST_DIR}/flex/ldbc-sf01-long-date/ldbc_schema_csr_ic.json"
cmd=${cmd}" -o=/tmp/plugin --ir_conf=${GS_TEST_DIR}/flex/ldbc-sf01-long-date/engine_config.yaml "
cmd=${cmd}" --graph_schema_path=${GS_TEST_DIR}/flex/ldbc-sf01-long-date/audit_graph_schema.yaml"
cmd=${cmd}" --gie_home=${GIE_HOME}"
echo $cmd
eval ${cmd}
Expand All @@ -129,8 +129,8 @@ jobs:
for i in 1 2 3 4 5 6 7 8 9;
do
cmd="./load_plan_and_gen.sh -e=hqps -i=../resources/queries/ic/adhoc/simple_match_${i}.cypher -w=/tmp/codgen/"
cmd=${cmd}" -o=/tmp/plugin --ir_conf=/tmp/ir.compiler.properties "
cmd=${cmd}" --graph_schema_path=${GS_TEST_DIR}/flex/ldbc-sf01-long-date/ldbc_schema_csr_ic.json"
cmd=${cmd}" -o=/tmp/plugin --ir_conf=${GS_TEST_DIR}/flex/ldbc-sf01-long-date/engine_config.yaml "
cmd=${cmd}" --graph_schema_path=${GS_TEST_DIR}/flex/ldbc-sf01-long-date/audit_graph_schema.yaml"
cmd=${cmd}" --gie_home=${GIE_HOME}"
echo $cmd
eval ${cmd}
Expand Down
4 changes: 2 additions & 2 deletions flex/bin/sync_server.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -175,8 +175,8 @@ void init_codegen_proxy(const bpo::variables_map& vm,
LOG(FATAL) << "gie-home not exists: " << gie_home;
}
}
server::CodegenProxy::get().Init(codegen_dir, codegen_bin, graph_schema_file,
engine_config_file, gie_home);
server::CodegenProxy::get().Init(codegen_dir, codegen_bin, engine_config_file,
graph_schema_file, gie_home);
}
} // namespace gs

Expand Down
50 changes: 49 additions & 1 deletion flex/codegen/src/hqps/hqps_scan_builder.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -89,6 +89,52 @@ class ScanOpBuilder {
return *this;
}

ScanOpBuilder& idx_predicate(const algebra::IndexPredicate& predicate) {
// check query_params not has predicate.
if (query_params_.has_predicate()) {
throw std::runtime_error(
std::string("Currently only support one predicate"));
}
// Currently we only support one predicate.
if (predicate.or_predicates_size() < 1) {
return *this;
}
if (predicate.or_predicates_size() != 1) {
throw std::runtime_error(
std::string("Currently only support one predicate"));
}
auto or_predicate = predicate.or_predicates(0);
if (or_predicate.predicates_size() != 1) {
throw std::runtime_error(
std::string("Currently only support one and predicate"));
}
auto triplet = or_predicate.predicates(0);
// add index predicate to query params
auto* new_predicate = query_params_.mutable_predicate();
{
auto first_op = new_predicate->add_operators();
common::Variable variable;
auto& property = triplet.key();
*(variable.mutable_property()) = property;
variable.mutable_node_type()->set_data_type(common::DataType::INT64);
*(first_op->mutable_var()) = variable;
}
{
auto second = new_predicate->add_operators();
second->set_logical(common::Logical::EQ);
second->mutable_node_type()->set_data_type(common::DataType::BOOLEAN);
}
{
auto third = new_predicate->add_operators();
auto& value = triplet.value();
third->mutable_node_type()->set_data_type(common::DataType::INT64);
*(third->mutable_const_()) = value;
}
VLOG(10) << "Add index predicate to query params: "
<< query_params_.DebugString();
return *this;
}

std::string Build() const {
std::string label_name;
std::vector<int32_t> labels_ids;
Expand Down Expand Up @@ -275,7 +321,9 @@ static std::string BuildScanOp(
} else {
builder.resAlias(-1);
}
return builder.queryParams(scan_pb.params()).Build();
return builder.queryParams(scan_pb.params())
.idx_predicate(scan_pb.idx_predicate())
.Build();
}

} // namespace gs
Expand Down
26 changes: 13 additions & 13 deletions flex/interactive/bin/gs_interactive
View file
Original file line number Diff line number Diff line change
Expand Up @@ -389,36 +389,36 @@ function update_engine_config_from_yaml(){

function init_usage() {
cat << EOF
db_admin.sh init -c [--config] <config file>
gs_interactive init -c [--config] <config file>
Init the database, create the containers. Specify the database version and volume mounting in the config yaml.
EOF
}

function destroy_usage() {
cat << EOF
db_admin.sh destroy
gs_interactive destroy
Destroy the current database, remove the container.
EOF
}

function create_usage() {
cat << EOF
db_admin.sh database create -g [--graph] <graph_name> -c [--config] <schema file>
gs_interactive database create -g [--graph] <graph_name> -c [--config] <schema file>
Create a graph in database, with the provided schema file.
User should import data to the created graph.
EOF
}

function remove_usage() {
cat << EOF
db_admin.sh database remove -g [--graph] <graph_name>
gs_interactive database remove -g [--graph] <graph_name>
Remove the database with the given graph.
EOF
}

function import_usage() {
cat << EOF
db_admin.sh database import -g [--graph] <graph_name> -c [--config] <bulk load file>
gs_interactive database import -g [--graph] <graph_name> -c [--config] <bulk load file>
Load the raw data specified in bulk load file to the specified graph.
EOF
}
Expand All @@ -432,28 +432,28 @@ function database_usage(){

function start_usage() {
cat << EOF
db_admin.sh service start -g [--graph] <graph_name> -c [--config] <engine config file>
gs_interactive service start -g [--graph] <graph_name> -c [--config] <engine config file>
Start the graph service on the specified graph, with the provided engine config file.
EOF
}

function stop_usage() {
cat << EOF
db_admin.sh service stop
gs_interactive service stop
Stop the database with the given graph.
EOF
}

function restart_usage() {
cat << EOF
db_admin.sh service restart -c [--config] [engine config file]
gs_interactive service restart -c [--config] [engine config file]
Restart the database with current running graph. Can update with new engine config file.
EOF
}

function get_log_usage() {
cat << EOF
db_admin.sh service get_log -o [--output] output directory
gs_interactive service get_log -o [--output] output directory
Get the log of the specified service/compiler, and write to the output file.
EOF
}
Expand All @@ -467,7 +467,7 @@ function services_usage(){

function compile_usage(){
cat << EOF
db_admin.sh procedure compile -g[--graph] <graph_name> -i <sourcefile[.cc, .cypher, directory]
gs_interactive procedure compile -g[--graph] <graph_name> -i <sourcefile[.cc, .cypher, directory]
-n[--name] <name of the stored procedure> -d [--description] <description of the stored procedure>
--compile_only
Compile cypher/.cc to dynamic library, according to the schema of graph. The output library will be placed at ./data/{graph_name}/lib.
Expand All @@ -479,7 +479,7 @@ EOF

function enable_proc_usage(){
cat << EOF
db_admin.sh procedure enable -g[--graph] <graph_name> -n[--name] <library names,seprated by ','>
gs_interactive procedure enable -g[--graph] <graph_name> -n[--name] <library names,seprated by ','>
-c[--config] <stored_procedures.yaml>
Enable the stored procedure in the given graph, with the provided library.
stored_procedures.yaml contains the the stored procedure names at each line.
Expand All @@ -488,7 +488,7 @@ EOF

function disable_proc_usage(){
cat << EOF
db_admin.sh procedure disable -g[--graph] <graph_name> -n[--name] <library names,seprated by ','>
gs_interactive procedure disable -g[--graph] <graph_name> -n[--name] <library names,seprated by ','>
-c[--config] <stored_procedures.yaml>
Disable the stored procedure in the given graph, with the provided library.
stored_procedures.yaml contains the the stored procedure names at each line.
Expand All @@ -497,7 +497,7 @@ EOF

function show_stored_procedure_usage(){
cat << EOF
db_admin.sh procedure show -g[--graph] graph_name
gs_interactive procedure show -g[--graph] graph_name
Show all stored procedure for the given graph.
EOF
}
Expand Down
23 changes: 3 additions & 20 deletions flex/tests/hqps/hqps_cypher_test.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,6 @@ GRAPH_SCHEMA_YAML=${GS_TEST_DIR}/flex/ldbc-sf01-long-date/audit_graph_schema.yam
GRAPH_BULK_LOAD_YAML=${GS_TEST_DIR}/flex/ldbc-sf01-long-date/audit_bulk_load.yaml
COMPILER_GRAPH_SCHEMA=${GS_TEST_DIR}/flex/ldbc-sf01-long-date/audit_graph_schema.yaml
GRAPH_CSR_DATA_DIR=${HOME}/csr-data-dir/
HQPS_IR_CONF=/tmp/hqps.ir.properties
# check if GRAPH_SCHEMA_YAML exists
if [ ! -f ${GRAPH_SCHEMA_YAML} ]; then
echo "GRAPH_SCHEMA_YAML: ${GRAPH_SCHEMA_YAML} not found"
Expand Down Expand Up @@ -79,21 +78,6 @@ kill_service(){
# kill service when exit
trap kill_service EXIT

create_ir_conf(){
rm ${HQPS_IR_CONF} || true
echo "engine.type: hiactor" >> ${HQPS_IR_CONF}
echo "hiactor.hosts: localhost:10000" >> ${HQPS_IR_CONF}
echo "graph.store: exp" >> ${HQPS_IR_CONF}
echo "graph.schema: ${GS_TEST_DIR}/flex/ldbc-sf01-long-date/ldbc_schema_csr_ic.json" >> ${HQPS_IR_CONF}
echo "graph.planner.is.on: true" >> ${HQPS_IR_CONF}
echo "graph.planner.opt: RBO" >> ${HQPS_IR_CONF}
echo "graph.planner.rules: FilterMatchRule" >> ${HQPS_IR_CONF}
echo "gremlin.server.disabled: true" >> ${HQPS_IR_CONF}
echo "neo4j.bolt.server.port: 7687" >> ${HQPS_IR_CONF}

echo "Finish generate HQPS_IR_CONF"
cat ${HQPS_IR_CONF}
}

# start engine service and load ldbc graph
start_engine_service(){
Expand All @@ -105,7 +89,7 @@ start_engine_service(){
# export FLEX_DATA_DIR
export FLEX_DATA_DIR=${GS_TEST_DIR}/flex/ldbc-sf01-long-date/

cmd="${SERVER_BIN} -c ${ENGINE_CONFIG_PATH} -g ${GRAPH_SCHEMA_YAML} "
cmd="${SERVER_BIN} -c ${DST_ENGINE_CONFIG_PATH} -g ${GRAPH_SCHEMA_YAML} "
cmd="${cmd} --data-path ${GRAPH_CSR_DATA_DIR} -l ${GRAPH_BULK_LOAD_YAML} "
cmd="${cmd} --gie-home ${GIE_HOME}"

Expand All @@ -122,7 +106,7 @@ start_engine_service(){
start_compiler_service(){
echo "try to start compiler service"
pushd ${GIE_HOME}/compiler
cmd="make run graph.schema:=${COMPILER_GRAPH_SCHEMA} config.path=${HQPS_IR_CONF}"
cmd="make run graph.schema=${COMPILER_GRAPH_SCHEMA} config.path=${DST_ENGINE_CONFIG_PATH}"
echo "Start compiler service with command: ${cmd}"
${cmd} &
sleep 5
Expand Down Expand Up @@ -153,10 +137,9 @@ run_simple_test(){
}

kill_service
create_ir_conf
start_engine_service
start_compiler_service
#run_ldbc_test
run_ldbc_test
run_simple_test
kill_service

Expand Down

0 comments on commit 07a50d7

Please sign in to comment.