From f129398d3cd4fa0a78f482fce5b5f76618580e03 Mon Sep 17 00:00:00 2001 From: shirly121 Date: Fri, 8 Sep 2023 18:00:37 +0800 Subject: [PATCH] parent 6ab796e5721e5c58f9c0665c609eabf22b84821a author shirly121 1694167237 +0800 committer xiaolei.zl 1695348300 +0800 parent 6ab796e5721e5c58f9c0665c609eabf22b84821a author shirly121 1694167237 +0800 committer xiaolei.zl 1695348286 +0800 [GIE Compiler] fix bugs of columnId in schema refactor(flex): Replace the Adhoc csv reader with Arrow CSV reader (#3154) 1. Use Arrow CSV Reader to replace current adhoc csv reader, to support more configurable options in `bulk_load.yaml`. 2. Introduce `CSVFragmentLoader`, `BasicFragmentLoader` for `MutablePropertyFragment`. With this PR merged, `MutablePropertyFragment` will support loading fragment from csv with options: - delimeter: default '|' - header_row: default true - quoting: default false - quoting_char: default '"' - escaping: default false - escaping_char: default'\\' - batch_size: the batch size of when reading file into memory, default 1MB. - batch_reader: default false. If set to true, `arrow::csv::StreamingReader` will be used to parse the input file. Otherwise, `arrow::TableReader` will be used. With this PR merged, the performance of graph loading will be improved. The Adhoc Reader denote the current implemented csv parser, 1,2,4,8 denotes the parallelism of graph loading, i.e. how many labels of vertex/edge are concurrently processed. Note that TableReader is around 10x faster than StreamingReader. The possible reason could be the multi-threading is used. See [arrow-csv-doc](https://arrow.apache.org/docs/cpp/csv.html) for details. | Reader | Phase | 1 | 2 | 4 | 8 | | --------- | -------------- | ---------- |---------- |---------- |---------- | | Adhoc Reader | ReadFile\+LoadGraph |805s| 468s| 349s| 313s| | Adhoc Reader | Serialization | 126s| 126s| 126s| 126s| | Adhoc Reader | **Total** |931s| 594s| 475s| 439s| | Table Reader | ReadFile | 9s |9s |9s| 9s| | Table Reader | LoadGraph |455s| 280s| 211s| 182s| | Table Reader |Serialization |126s| 126s| 126s| 126s| | Table Reader | **Total** | 600s| 415s| 346s| 317s| | Streaming Reader | ReadFile |91s| 91s| 91s| 91s| | Streaming Reader | LoadGraph | 555s| 289s| 196s| 149s| | Streaming Reader | Serialization |126s| 126s| 126s| 126s| | Streaming Reader | **Total** | 772s| 506s| 413s| 366s| | Reader | Phase | 1 | 2 | 4 | 8 | | --------- | -------------- | ---------- |---------- |---------- |---------- | | Adhoc Reader | ReadFile\+LoadGraph |2720s| 1548s| 1176s| 948s| | Adhoc Reader | Serialization | 409s| 409s| 409s| 409s| | Adhoc Reader | **Total** | 3129s| 1957s| 1585s| 1357s| | Table Reader | ReadFile |24s| 24s| 24s| 24s| | Table Reader | LoadGraph |1576s| 949s| 728s| 602s| | Table Reader |Serialization |409s| 409s| 409s| 409s| | Table Reader | **Total** | 2009s| 1382s| 1161s| 1035s| | Streaming Reader | ReadFile |300s| 300s| 300s| 300s| | Streaming Reader | LoadGraph | 1740s| 965s| 669s| 497s| | Streaming Reader | Serialization | 409s| 409s| 409s| 409s| | Streaming Reader | **Total** | 2539s| 1674s| 1378s| 1206s| | Reader | Phase | 1 | 2 | 4 | 8 | | --------- | -------------- | ---------- |---------- |---------- |---------- | | Adhoc Reader | ReadFile\+LoadGraph | 8260s| 4900s |3603s |2999s| | Adhoc Reader | Serialization | 1201s | 1201s| 1201s |1201s| | Adhoc Reader | **Total** | 9461s| 6101s | 4804s |4200s| | Table Reader | ReadFile | 73s |73s| 96s| 96s| | Table Reader | LoadGraph |4650s| 2768s| 2155s |1778s| | Table Reader |Serialization | 1201s | 1201s| 1201s |1201s| | Table Reader | **Total** | 5924s| 4042s| 3452s| 3075s| | Streaming Reader | ReadFile | 889s |889s | 889s| 889s| | Streaming Reader | LoadGraph | 5589s| 3005s| 2200s| 1712s| | Streaming Reader | Serialization | 1201s| 1201s| 1201s |1201s | | Streaming Reader | **Total** | 7679s | 5095s |4290s| 3802s| FIx #3116 minor fix and move modern graph fix grin test todo: do_start fix fix stash fix fix make rules unique dockerfile stash minor change remove plugin-dir fix minor fix debug debug fix fix fix bulk_load.yaml bash format some fix fix format fix grin test some fi check ci fix ci set fix ci fix dd f disable tmate fix some bug fix fix refactor fix fix fix minor some fix fix support default src_dst primarykey mapping in bulk load fix fix fix fix Ci rename fix java and add get_person_name.cypher [GIE Compiler] minor fix use graphscope gstest format add movie queries dd debug add movie test format format fix script debug fix test script minor sort query results minor minor format fix ci format gstest Add License fix bugs --- .github/workflows/hqps-db-ci.yml | 40 ++- flex/interactive/bin/gs_interactive | 142 +++++--- flex/interactive/conf/engine_config.yaml | 1 - .../modern_graph/get_person_name.cypher | 2 +- flex/tests/hqps/hqps_cypher_test.sh | 79 +++-- flex/tests/hqps/queries/movie/query1.cypher | 1 + flex/tests/hqps/queries/movie/query10.cypher | 4 + flex/tests/hqps/queries/movie/query11.cypher | 4 + flex/tests/hqps/queries/movie/query12.cypher | 2 + flex/tests/hqps/queries/movie/query13.cypher | 4 + flex/tests/hqps/queries/movie/query14.cypher | 6 + flex/tests/hqps/queries/movie/query15.cypher | 4 + flex/tests/hqps/queries/movie/query2.cypher | 1 + flex/tests/hqps/queries/movie/query3.cypher | 1 + flex/tests/hqps/queries/movie/query4.cypher | 2 + flex/tests/hqps/queries/movie/query5.cypher | 6 + flex/tests/hqps/queries/movie/query6.cypher | 2 + flex/tests/hqps/queries/movie/query7.cypher | 3 + flex/tests/hqps/queries/movie/query8.cypher | 2 + flex/tests/hqps/queries/movie/query9.cypher | 2 + interactive_engine/compiler/pom.xml | 1 + .../ir/meta/reader/LocalMetaDataReader.java | 2 +- .../integration/suite/ldbc/MovieQueries.java | 302 ++++++++++++++++++ .../cypher/integration/movie/MovieTest.java | 131 ++++++++ 24 files changed, 658 insertions(+), 86 deletions(-) create mode 100644 flex/tests/hqps/queries/movie/query1.cypher create mode 100644 flex/tests/hqps/queries/movie/query10.cypher create mode 100644 flex/tests/hqps/queries/movie/query11.cypher create mode 100644 flex/tests/hqps/queries/movie/query12.cypher create mode 100644 flex/tests/hqps/queries/movie/query13.cypher create mode 100644 flex/tests/hqps/queries/movie/query14.cypher create mode 100644 flex/tests/hqps/queries/movie/query15.cypher create mode 100644 flex/tests/hqps/queries/movie/query2.cypher create mode 100644 flex/tests/hqps/queries/movie/query3.cypher create mode 100644 flex/tests/hqps/queries/movie/query4.cypher create mode 100644 flex/tests/hqps/queries/movie/query5.cypher create mode 100644 flex/tests/hqps/queries/movie/query6.cypher create mode 100644 flex/tests/hqps/queries/movie/query7.cypher create mode 100644 flex/tests/hqps/queries/movie/query8.cypher create mode 100644 flex/tests/hqps/queries/movie/query9.cypher create mode 100644 interactive_engine/compiler/src/main/java/com/alibaba/graphscope/cypher/integration/suite/ldbc/MovieQueries.java create mode 100644 interactive_engine/compiler/src/test/java/com/alibaba/graphscope/cypher/integration/movie/MovieTest.java diff --git a/.github/workflows/hqps-db-ci.yml b/.github/workflows/hqps-db-ci.yml index ef480ea7fab8..8a9cd009fad5 100644 --- a/.github/workflows/hqps-db-ci.yml +++ b/.github/workflows/hqps-db-ci.yml @@ -78,8 +78,8 @@ jobs: which cargo # build compiler - cd ${GIE_HOME}/compiler - make build + cd ${GIE_HOME}/ + mvn clean install -Pexperimental -DskipTests - name: Prepare dataset and workspace env: @@ -91,6 +91,8 @@ jobs: mkdir -p ${INTERACTIVE_WORKSPACE}/data/ldbc GRAPH_SCHEMA_YAML=${GS_TEST_DIR}/flex/ldbc-sf01-long-date/audit_graph_schema.yaml cp ${GRAPH_SCHEMA_YAML} ${INTERACTIVE_WORKSPACE}/data/ldbc/graph.yaml + mkdir -p ${INTERACTIVE_WORKSPACE}/data/movies + cp ${GS_TEST_DIR}/flex/movies/movies_schema.yaml ${INTERACTIVE_WORKSPACE}/data/movies/graph.yaml - name: Sample Query test env: @@ -129,7 +131,19 @@ jobs: eval ${cmd} done - - name: Run End-to-End cypher adhoc query test + # test movie graph, 8,9,10 are not supported now + # change the default_graph config in ${GS_TEST_DIR}/flex/ldbc-sf01-long-date/engine_config.yaml to movies + sed -i 's/default_graph: ldbc/default_graph: movies/g' ${GS_TEST_DIR}/flex/ldbc-sf01-long-date/engine_config.yaml + for i in 1 2 3 4 5 6 7 11 12 13 14 15; + do + cmd="./load_plan_and_gen.sh -e=hqps -i=../tests/hqps/queries/movie/query${i}.cypher -w=/tmp/codgen/" + cmd=${cmd}" -o=/tmp/plugin --ir_conf=${GS_TEST_DIR}/flex/ldbc-sf01-long-date/engine_config.yaml " + cmd=${cmd}" --graph_schema_path=${INTERACTIVE_WORKSPACE}/data/movies/graph.yaml" + echo $cmd + eval ${cmd} + done + + - name: Run End-to-End cypher adhoc ldbc query test env: GS_TEST_DIR: ${{ github.workspace }}/gstest HOME : /home/graphscope/ @@ -138,5 +152,23 @@ jobs: cd ${GITHUB_WORKSPACE}/flex/tests/hqps/ export FLEX_DATA_DIR=${GS_TEST_DIR}/flex/ldbc-sf01-long-date export ENGINE_TYPE=hiactor - bash hqps_cypher_test.sh ${GS_TEST_DIR} ${INTERACTIVE_WORKSPACE} + # change the default_graph config in ${GS_TEST_DIR}/flex/ldbc-sf01-long-date/engine_config.yaml to ldbc + sed -i 's/default_graph: movies/default_graph: ldbc/g' ${GS_TEST_DIR}/flex/ldbc-sf01-long-date/engine_config.yaml + bash hqps_cypher_test.sh ${INTERACTIVE_WORKSPACE} ldbc ${GS_TEST_DIR}/flex/ldbc-sf01-long-date/audit_bulk_load.yaml \ + ${GS_TEST_DIR}/flex/ldbc-sf01-long-date/engine_config.yaml + + - name: Run End-to-End cypher adhoc movie query test + env: + GS_TEST_DIR: ${{ github.workspace }}/gstest + HOME : /home/graphscope/ + INTERACTIVE_WORKSPACE: /tmp/interactive_workspace + run: | + cd ${GITHUB_WORKSPACE}/flex/tests/hqps/ + export FLEX_DATA_DIR=../../interactive/examples/movies/ + export ENGINE_TYPE=hiactor + # change the default_graph config in ${GS_TEST_DIR}/flex/ldbc-sf01-long-date/engine_config.yaml to movies + sed -i 's/default_graph: ldbc/default_graph: movies/g' ${GS_TEST_DIR}/flex/ldbc-sf01-long-date/engine_config.yaml + bash hqps_cypher_test.sh ${INTERACTIVE_WORKSPACE} movies ${GS_TEST_DIR}/flex/movies/movies_import.yaml \ + ${GS_TEST_DIR}/flex/ldbc-sf01-long-date/engine_config.yaml + diff --git a/flex/interactive/bin/gs_interactive b/flex/interactive/bin/gs_interactive index 2fc9993005ca..42012d403f18 100755 --- a/flex/interactive/bin/gs_interactive +++ b/flex/interactive/bin/gs_interactive @@ -370,6 +370,10 @@ function update_engine_config_from_yaml(){ if [[ -n "${default_graph}" ]]; then DATABASE_CURRENT_GRAPH_NAME="${default_graph}" fi + # update hiactor shard num + if [[ -n "${compute_engine_shard_num}" ]]; then + DATABASE_COMPUTE_ENGINE_SHARD_NUM="${compute_engine_shard_num}" + fi # compiler if [[ -n ${compiler_planner_is_on} ]]; then DATABASE_COMPILER_PLANNER_IS_ON="${compiler_planner_is_on}" @@ -380,6 +384,7 @@ function update_engine_config_from_yaml(){ fi # append the founded compiler planner rules to DATABASE_COMPILER_PLANNER_RULES x=1 + CURRENT_DATABASE_COMPILER_PLANNER_RULES="" while true; do compiler_planner_rules_x_key="compiler_planner_rules_${x}" compiler_planner_rules_x=$(eval echo "\$${compiler_planner_rules_x_key}") @@ -387,11 +392,17 @@ function update_engine_config_from_yaml(){ break fi # check compiler_planner_rules_x present in DATABASE_COMPILER_PLANNER_RULES, if not, append - if [[ ! "${DATABASE_COMPILER_PLANNER_RULES}" =~ "${compiler_planner_rules_x}" ]]; then - DATABASE_COMPILER_PLANNER_RULES="${DATABASE_COMPILER_PLANNER_RULES},${compiler_planner_rules_x}" + if [[ ! "${CURRENT_DATABASE_COMPILER_PLANNER_RULES}" =~ "${compiler_planner_rules_x}" ]]; then + CURRENT_DATABASE_COMPILER_PLANNER_RULES="${CURRENT_DATABASE_COMPILER_PLANNER_RULES},${compiler_planner_rules_x}" fi x=$((x + 1)) done + # if CURRENT_DATABASE_COMPILER_PLANNER_RULES is not empty,override DATABASE_COMPILER_PLANNER_RULES + if [[ -n "${CURRENT_DATABASE_COMPILER_PLANNER_RULES}" ]]; then + # remove the first ',' + CURRENT_DATABASE_COMPILER_PLANNER_RULES=$(echo "${CURRENT_DATABASE_COMPILER_PLANNER_RULES}" | sed 's/^,//g') + DATABASE_COMPILER_PLANNER_RULES="${CURRENT_DATABASE_COMPILER_PLANNER_RULES}" + fi if [[ -n "${compiler_endpoint_address}" ]]; then DATABASE_COMPILER_ENDPOINT_ADDRESS="${compiler_endpoint_address}" fi @@ -1106,7 +1117,7 @@ function do_import(){ info "Graph Schema exists" # copy the bulk_load_file to container bulk_load_file_name=$(basename "${bulk_load_file}") - docker_bulk_load_file="/tmp/${bulk_load_file_name}" + docker_bulk_load_file="${HOST_DB_TMP_DIR}/${bulk_load_file_name}" docker cp "${bulk_load_file}" "${GIE_DB_CONTAINER_NAME}:${docker_bulk_load_file}" docker_graph_data_dir="${DATABASE_WORKSPACE}/data//${graph_name}/indices" @@ -1186,6 +1197,26 @@ function do_destroy() { if [ -f "${HOST_DB_ENV_FILE}" ]; then rm ${HOST_DB_ENV_FILE} fi + # rm the temp files used + if [ -f "${HOST_DB_TMP_DIR}/graph0.yaml" ]; then + rm ${HOST_DB_TMP_DIR}/graph0.yaml + fi + if [ -f "${HOST_DB_TMP_DIR}/.enable" ]; then + rm ${HOST_DB_TMP_DIR}/.enable + fi + # rm ${HOST_DB_TMP_DIR}/engine_config.yaml ${HOST_DB_TMP_DIR}/real_engine_config.yaml, ${HOST_DB_TMP_DIR}/graph_exists ${HOST_DB_TMP_DIR}/graph_loaded + if [ -f "${HOST_DB_TMP_DIR}/engine_config.yaml" ]; then + rm ${HOST_DB_TMP_DIR}/engine_config.yaml + fi + if [ -f "${HOST_DB_TMP_DIR}/real_engine_config.yaml" ]; then + rm ${HOST_DB_TMP_DIR}/real_engine_config.yaml + fi + if [ -f "${HOST_DB_TMP_DIR}/graph_exists" ]; then + rm ${HOST_DB_TMP_DIR}/graph_exists + fi + if [ -f "${HOST_DB_TMP_DIR}/graph_loaded" ]; then + rm ${HOST_DB_TMP_DIR}/graph_loaded + fi info "Finish destroy database" } @@ -1242,7 +1273,7 @@ function do_start(){ esac done # try parse default_graph from engine_config_file - # generate real engine config file, put it at /tmp/real_engine_config.yaml + # generate real engine config file, put it at ${HOST_DB_TMP_DIR}/real_engine_config.yaml if [ -z "${graph_name}" ]; then graph_name=${DATABASE_CURRENT_GRAPH_NAME} else @@ -1255,7 +1286,7 @@ function do_start(){ exit 1 fi - real_engine_config_file="/tmp/real_engine_config.yaml" + real_engine_config_file="${HOST_DB_TMP_DIR}/real_engine_config.yaml" if [ -z "${engine_config_file}" ]; then if ! generate_real_engine_conf "${real_engine_config_file}"; then err "generate engine config file failed" @@ -1276,22 +1307,22 @@ function do_start(){ # check if modern_graph exists in container, get the result as bool docker_graph_schema_file="${DATABASE_WORKSPACE}/data/${graph_name}/graph.yaml" wal_file="${DATABASE_WORKSPACE}/data/${graph_name}/indices/init_snapshot.bin" - if [ -f "/tmp/graph_exists" ]; then - if ! rm /tmp/graph_exists; then - err "fail to remove /tmp/graph_exists, please remove it manually" + if [ -f "${HOST_DB_TMP_DIR}/graph_exists" ]; then + if ! rm "${HOST_DB_TMP_DIR}/graph_exists"; then + err "fail to remove ${HOST_DB_TMP_DIR}/graph_exists, please remove it manually, maybe sudo rm -rf ${HOST_DB_TMP_DIR}/graph_exists" exit 1 fi fi - if [ -f "/tmp/graph_loaded" ]; then - if ! rm /tmp/graph_loaded; then - err "fail to remove /tmp/graph_loaded, please remove it manually" + if [ -f "${HOST_DB_TMP_DIR}/graph_loaded" ]; then + if ! rm "${HOST_DB_TMP_DIR}/graph_loaded"; then + err "fail to remove ${HOST_DB_TMP_DIR}/graph_loaded, please remove it manually, maybe sudo rm -rf ${HOST_DB_TMP_DIR}/graph_loaded" exit 1 fi fi - docker exec "${GIE_DB_CONTAINER_NAME}" bash -c "( [ -f ${docker_graph_schema_file} ] && echo \"true\" e) || echo \"false\"" > /tmp/graph_exists - docker exec "${GIE_DB_CONTAINER_NAME}" bash -c "( [ -f ${wal_file} ] && echo \"true\" e) || echo \"false\"" > /tmp/graph_loaded - graph_exists=$(cat /tmp/graph_exists) - graph_loaded=$(cat /tmp/graph_loaded) + docker exec "${GIE_DB_CONTAINER_NAME}" bash -c "( [ -f ${docker_graph_schema_file} ] && echo \"true\" e) || echo \"false\"" > ${HOST_DB_TMP_DIR}/graph_exists + docker exec "${GIE_DB_CONTAINER_NAME}" bash -c "( [ -f ${wal_file} ] && echo \"true\" e) || echo \"false\"" > ${HOST_DB_TMP_DIR}/graph_loaded + graph_exists=$(cat ${HOST_DB_TMP_DIR}/graph_exists) + graph_loaded=$(cat ${HOST_DB_TMP_DIR}/graph_loaded) if [ "${graph_exists}" = "false" ]; then # if graph_name is default_graph, we should create it first # otherwise, we should tell user to create it first @@ -1584,7 +1615,7 @@ function do_compile() { . ${HOST_DB_ENV_FILE} fi - real_engine_config_file="/tmp/real_engine_config.yaml" + real_engine_config_file="${HOST_DB_TMP_DIR}/real_engine_config.yaml" # update default graph name DATABASE_CURRENT_GRAPH_NAME=${graph_name} if ! generate_real_engine_conf "${real_engine_config_file}"; then @@ -1598,9 +1629,30 @@ function do_compile() { docker_graph_dir="${DATABASE_WORKSPACE}/data/${graph_name}" docker_graph_schema="${docker_graph_dir}/graph.yaml" docker exec "${GIE_DB_CONTAINER_NAME}" bash -c "[ -d ${docker_graph_dir} ] || (echo -e \"${RED} Graph [${graph_name}] not exists, please create it first.${NC}\" && exit 1)" + # Fetch current installed procedures, and check if the procedure is already installed + # if not compile_only, we should add the stored_procedure_name to .enable + docker_graph_enable_file="${docker_graph_dir}/plugins/.enable" + # copy container to host + if [ -f "${HOST_DB_TMP_DIR}/.enable" ]; then + if ! rm -f ${HOST_DB_TMP_DIR}/.enable; then + err "fail to remove ${HOST_DB_TMP_DIR}/.enable, please remove it manually, maybe sudo rm -rf ${HOST_DB_TMP_DIR}/.enable" + exit 1 + fi + fi + # if docker_graph_enable_file exists. copy it to host + docker exec "${GIE_DB_CONTAINER_NAME}" test -e "${docker_graph_enable_file}" && (docker cp "${GIE_DB_CONTAINER_NAME}:${docker_graph_enable_file}" "${HOST_DB_TMP_DIR}/.enable") + + if [ ! -f "${HOST_DB_TMP_DIR}/.enable" ]; then + touch "${HOST_DB_TMP_DIR}/.enable" + fi + # check if the stored_procedure_name is already in .enable + if grep -q "${stored_procedure_name}" "${HOST_DB_TMP_DIR}/.enable"; then + err "stored_procedure_name [${stored_procedure_name}] already exists, please use another name" + exit 1 + fi container_output_dir="${DATABASE_WORKSPACE}/data/${graph_name}/plugins" - cotainer_input_path="/tmp/${file_name}" + cotainer_input_path="${HOST_DB_TMP_DIR}/${file_name}" # docker cp file to container cmd="docker cp ${real_file_path} ${GIE_DB_CONTAINER_NAME}:${cotainer_input_path}" eval ${cmd} || exit 1 @@ -1631,22 +1683,12 @@ function do_compile() { fi info "success generate dynamic lib ${output_file}." - # if not compile_only, we should add the stored_procedure_name to .enable - docker_graph_enable_file="${docker_graph_dir}/plugins/.enable" - # copy container to host - rm -f /tmp/.enable - # if docker_graph_enable_file exists. copy it to host - docker exec "${GIE_DB_CONTAINER_NAME}" test -e "${docker_graph_enable_file}" && (docker cp "${GIE_DB_CONTAINER_NAME}:${docker_graph_enable_file}" "/tmp/.enable") - - if [ ! -f "/tmp/.enable" ]; then - touch "/tmp/.enable" - fi # if compile_only equal to false if [ "${compile_only}" = false ]; then - echo "${stored_procedure_name}" >> /tmp/.enable + echo "${stored_procedure_name}" >> ${HOST_DB_TMP_DIR}/.enable fi # copy back - docker cp "/tmp/.enable" "${GIE_DB_CONTAINER_NAME}:${docker_graph_enable_file}" || exit 1 + docker cp "${HOST_DB_TMP_DIR}/.enable" "${GIE_DB_CONTAINER_NAME}:${docker_graph_enable_file}" || exit 1 } function do_enable(){ @@ -1716,26 +1758,31 @@ function do_enable(){ docker exec "${GIE_DB_CONTAINER_NAME}" bash -c "[ -d ${docker_graph_dir} ] || (echo -e \"${RED} Graph ${graph_name} not exists, please create it first.${NC}\" && exit 1)" docker_graph_plugin_dir="${docker_graph_dir}/plugins" docker_graph_enable_file="${docker_graph_plugin_dir}/.enable" - rm -f /tmp/.enable + if [ -f "${HOST_DB_TMP_DIR}/.enable" ]; then + if ! rm -f ${HOST_DB_TMP_DIR}/.enable; then + err "fail to remove ${HOST_DB_TMP_DIR}/.enable, please remove it manually, maybe sudo rm -rf ${HOST_DB_TMP_DIR}/.enable" + exit 1 + fi + fi # copy the .enable file to host, and append the stored_procedure_names to it; if the stored_procedure_names already exists, do nothing - docker cp "${GIE_DB_CONTAINER_NAME}:${docker_graph_enable_file}" "/tmp/.enable" || true - if [ ! -f "/tmp/.enable" ]; then - touch "/tmp/.enable" + docker cp "${GIE_DB_CONTAINER_NAME}:${docker_graph_enable_file}" "${HOST_DB_TMP_DIR}/.enable" || true + if [ ! -f "${HOST_DB_TMP_DIR}/.enable" ]; then + touch "${HOST_DB_TMP_DIR}/.enable" fi - old_line_num=$(wc -l < /tmp/.enable) + old_line_num=$(wc -l < ${HOST_DB_TMP_DIR}/.enable) # split the stored_procedure_names by ',' and append them to .enable file IFS=',' read -ra stored_procedure_names_array <<< "${stored_procedure_names}" for stored_procedure_name in "${stored_procedure_names_array[@]}"; do # check if the stored_procedure_name already exists in .enable file - if grep -q "${stored_procedure_name}" "/tmp/.enable"; then + if grep -q "${stored_procedure_name}" "${HOST_DB_TMP_DIR}/.enable"; then info "stored_procedure_name ${stored_procedure_name} already exists in .enable file, skip" else - echo "${stored_procedure_name}" >> /tmp/.enable + echo "${stored_procedure_name}" >> ${HOST_DB_TMP_DIR}/.enable fi done # copy the .enable file back to container - docker cp "/tmp/.enable" "${GIE_DB_CONTAINER_NAME}:${docker_graph_enable_file}" || exit 1 - new_line_num=$(wc -l < /tmp/.enable) + docker cp "${HOST_DB_TMP_DIR}/.enable" "${GIE_DB_CONTAINER_NAME}:${docker_graph_enable_file}" || exit 1 + new_line_num=$(wc -l < ${HOST_DB_TMP_DIR}/.enable) info "Successfuly enable stored_procedures ${stored_procedure_names} for graph [${graph_name}], ${old_line_num} -> ${new_line_num}" } @@ -1823,16 +1870,16 @@ function do_disable(){ # add the names to .enable file for graph_name # copy the .enable file to host, and remove the stored_procedure_names from it - docker cp "${GIE_DB_CONTAINER_NAME}:${docker_graph_enable_file}" "/tmp/.enable" || exit 1 - old_line_num=$(wc -l < /tmp/.enable) + docker cp "${GIE_DB_CONTAINER_NAME}:${docker_graph_enable_file}" "${HOST_DB_TMP_DIR}/.enable" || exit 1 + old_line_num=$(wc -l < ${HOST_DB_TMP_DIR}/.enable) # split the stored_procedure_names by ',' and remove them from .enable file IFS=',' read -ra stored_procedure_names_array <<< "${stored_procedure_names}" for stored_procedure_name in "${stored_procedure_names_array[@]}"; do - sed -i "/${stored_procedure_name}/d" /tmp/.enable + sed -i "/${stored_procedure_name}/d" ${HOST_DB_TMP_DIR}/.enable done # copy the .enable file back to container - docker cp "/tmp/.enable" "${GIE_DB_CONTAINER_NAME}:${docker_graph_enable_file}" || exit 1 - new_line_num=$(wc -l < /tmp/.enable) + docker cp "${HOST_DB_TMP_DIR}/.enable" "${GIE_DB_CONTAINER_NAME}:${docker_graph_enable_file}" || exit 1 + new_line_num=$(wc -l < ${HOST_DB_TMP_DIR}/.enable) info "Successfuly disable stored_procedures ${stored_procedure_names} for graph [${graph_name}], ${old_line_num} -> ${new_line_num}" } @@ -1880,10 +1927,9 @@ function do_show(){ # check if docker_graph_yaml exists, if not ,exit docker exec "${GIE_DB_CONTAINER_NAME}" bash -c "[ -f ${docker_graph_yaml} ] || (echo -e \"${RED}Graph [${graph_name}] not exists, please create it first. ${NC}\" && exit 1)" || exit 1 # copy to host - docker cp "${GIE_DB_CONTAINER_NAME}:${docker_graph_yaml}" "/tmp/graph.yaml" || (echo "fail to copy" && exit 1) - # read /tmp/graph.yaml find the enabled_list list, and print the following lines + docker cp "${GIE_DB_CONTAINER_NAME}:${docker_graph_yaml}" "${HOST_DB_TMP_DIR}/graph.yaml" || (echo "fail to copy" && exit 1) # parse /tmp/graph.yaml and get stored_procedures_enable_lists array - eval $(parse_yaml "/tmp/graph.yaml") + eval $(parse_yaml "${HOST_DB_TMP_DIR}/graph.yaml") x=1 stored_procedures_enable_lists_array=() while true; do @@ -1903,14 +1949,14 @@ function do_show(){ docker_graph_enable_file="${docker_graph_plugin_dir}/.enable" # check if docker_graph_enable_file exists, if not ,exit docker exec "${GIE_DB_CONTAINER_NAME}" bash -c "[ -f ${docker_graph_enable_file} ] || (echo -e \"${RED}Graph [${graph_name}] has no procedures registered. ${NC}\" && exit 1)" || exit 1 - docker cp "${GIE_DB_CONTAINER_NAME}:${docker_graph_enable_file}" "/tmp/.enable" || exit 1 + docker cp "${GIE_DB_CONTAINER_NAME}:${docker_graph_enable_file}" "${HOST_DB_TMP_DIR}/.enable" || exit 1 disabled_list=() # iterate the .enable file, for each line, check if it is in stored_procedures_enable_lists_array, if not, add it to disabled_list while read line; do if [[ ! " ${stored_procedures_enable_lists_array[@]} " =~ " ${line} " ]]; then disabled_list+=("${line}") fi - done < /tmp/.enable + done < ${HOST_DB_TMP_DIR}/.enable info "Disabled Size: ${#disabled_list[@]}" # print the enabled_list and disabled_list diff --git a/flex/interactive/conf/engine_config.yaml b/flex/interactive/conf/engine_config.yaml index 7a8b9eb358b6..3e1d6cefe7cf 100644 --- a/flex/interactive/conf/engine_config.yaml +++ b/flex/interactive/conf/engine_config.yaml @@ -1,5 +1,4 @@ log_level: INFO # default INFO -default_graph: modern # configure the graph to be loaded while starting the service, if graph name not specified compute_engine: shard_num: 1 # the number of shared workers, default 1 compiler: diff --git a/flex/interactive/examples/modern_graph/get_person_name.cypher b/flex/interactive/examples/modern_graph/get_person_name.cypher index f8c9f275ae4e..fbb3197f3849 100644 --- a/flex/interactive/examples/modern_graph/get_person_name.cypher +++ b/flex/interactive/examples/modern_graph/get_person_name.cypher @@ -1 +1 @@ -MATCH(p : person {id: $personId}) RETURN p.name; \ No newline at end of file +MATCH(p : person {id: $personId}) RETURN p.name; diff --git a/flex/tests/hqps/hqps_cypher_test.sh b/flex/tests/hqps/hqps_cypher_test.sh index fb8fd13a6ce9..a6876c862356 100644 --- a/flex/tests/hqps/hqps_cypher_test.sh +++ b/flex/tests/hqps/hqps_cypher_test.sh @@ -19,47 +19,50 @@ SERVER_BIN=${FLEX_HOME}/build/bin/sync_server GIE_HOME=${FLEX_HOME}/../interactive_engine/ # -if [ $# -lt 2 ]; then - echo "only receives: $# args, need 2" - echo "Usage: $0 " +if [ ! $# -eq 4 ]; then + echo "only receives: $# args, need 4" + echo "Usage: $0 " exit 1 fi -GS_TEST_DIR=$1 -INTERACTIVE_WORKSPACE=$2 -if [ ! -d ${GS_TEST_DIR} ]; then - echo "GS_TEST_DIR: ${GS_TEST_DIR} not exists" - exit 1 -fi +INTERACTIVE_WORKSPACE=$1 +GRAPH_NAME=$2 +GRAPH_BULK_LOAD_YAML=$3 +ENGINE_CONFIG_PATH=$4 if [ ! -d ${INTERACTIVE_WORKSPACE} ]; then echo "INTERACTIVE_WORKSPACE: ${INTERACTIVE_WORKSPACE} not exists" exit 1 fi - -ENGINE_CONFIG_PATH=${GS_TEST_DIR}/flex/ldbc-sf01-long-date/engine_config.yaml -ORI_GRAPH_SCHEMA_YAML=${GS_TEST_DIR}/flex/ldbc-sf01-long-date/audit_graph_schema.yaml -GRAPH_SCHEMA_YAML=${INTERACTIVE_WORKSPACE}/data/ldbc/graph.yaml -GRAPH_BULK_LOAD_YAML=${GS_TEST_DIR}/flex/ldbc-sf01-long-date/audit_bulk_load.yaml -COMPILER_GRAPH_SCHEMA=${GS_TEST_DIR}/flex/ldbc-sf01-long-date/audit_graph_schema.yaml -GRAPH_CSR_DATA_DIR=${HOME}/csr-data-dir/ -# check if GRAPH_SCHEMA_YAML exists -if [ ! -f ${GRAPH_SCHEMA_YAML} ]; then - echo "GRAPH_SCHEMA_YAML: ${GRAPH_SCHEMA_YAML} not found" +# check graph is ldbc or movies +if [ ${GRAPH_NAME} != "ldbc" ] && [ ${GRAPH_NAME} != "movies" ]; then + echo "GRAPH_NAME: ${GRAPH_NAME} not supported, use movies or ldbc" + exit 1 +fi +if [ ! -d ${INTERACTIVE_WORKSPACE}/data/${GRAPH_NAME} ]; then + echo "GRAPH: ${GRAPH_NAME} not exists" + exit 1 +fi +if [ ! -f ${INTERACTIVE_WORKSPACE}/data/${GRAPH_NAME}/graph.yaml ]; then + echo "GRAPH_SCHEMA_FILE: ${BULK_LOAD_FILE} not exists" exit 1 fi - -# check if GRAPH_BULK_LOAD_YAML exists if [ ! -f ${GRAPH_BULK_LOAD_YAML} ]; then - echo "GRAPH_BULK_LOAD_YAML: ${GRAPH_BULK_LOAD_YAML} not found" + echo "GRAPH_BULK_LOAD_YAML: ${GRAPH_BULK_LOAD_YAML} not exists" exit 1 fi - -# check if COMPILER_GRAPH_SCHEMA exists -if [ ! -f ${COMPILER_GRAPH_SCHEMA} ]; then - echo "COMPILER_GRAPH_SCHEMA: ${COMPILER_GRAPH_SCHEMA} not found" +if [ ! -f ${ENGINE_CONFIG_PATH} ]; then + echo "ENGINE_CONFIG: ${ENGINE_CONFIG_PATH} not exists" exit 1 fi +GRAPH_SCHEMA_YAML=${INTERACTIVE_WORKSPACE}/data/${GRAPH_NAME}/graph.yaml +GRAPH_CSR_DATA_DIR=${HOME}/csr-data-dir/ +# rm data dir if exists +if [ -d ${GRAPH_CSR_DATA_DIR} ]; then + rm -rf ${GRAPH_CSR_DATA_DIR} +fi + + RED='\033[0;31m' GREEN='\033[0;32m' NC='\033[0m' # No Color @@ -92,8 +95,6 @@ start_engine_service(){ err "SERVER_BIN not found" exit 1 fi - # export FLEX_DATA_DIR - export FLEX_DATA_DIR=${GS_TEST_DIR}/flex/ldbc-sf01-long-date/ cmd="${SERVER_BIN} -c ${ENGINE_CONFIG_PATH} -g ${GRAPH_SCHEMA_YAML} " cmd="${cmd} --data-path ${GRAPH_CSR_DATA_DIR} -l ${GRAPH_BULK_LOAD_YAML} " @@ -111,7 +112,7 @@ start_engine_service(){ start_compiler_service(){ echo "try to start compiler service" pushd ${GIE_HOME}/compiler - cmd="make run graph.schema=${COMPILER_GRAPH_SCHEMA} config.path=${ENGINE_CONFIG_PATH}" + cmd="make run graph.schema=${GRAPH_SCHEMA_YAML} config.path=${ENGINE_CONFIG_PATH}" echo "Start compiler service with command: ${cmd}" ${cmd} & sleep 5 @@ -141,11 +142,27 @@ run_simple_test(){ popd } +run_movie_test(){ + echo "run movie test" + pushd ${GIE_HOME}/compiler + cmd="mvn test -Dtest=com.alibaba.graphscope.cypher.integration.movie.MovieTest" + echo "Start movie test: ${cmd}" + ${cmd} + info "Finish movie test" + popd +} + kill_service start_engine_service start_compiler_service -run_ldbc_test -run_simple_test +# if GRAPH_NAME equals ldbc +if [ "${GRAPH_NAME}" == "ldbc" ]; then + run_ldbc_test + run_simple_test +else + run_movie_test +fi + kill_service diff --git a/flex/tests/hqps/queries/movie/query1.cypher b/flex/tests/hqps/queries/movie/query1.cypher new file mode 100644 index 000000000000..d08ec9d27a68 --- /dev/null +++ b/flex/tests/hqps/queries/movie/query1.cypher @@ -0,0 +1 @@ +MATCH (tom:Person) WHERE tom.name = "Tom Hanks" RETURN tom.born AS bornYear,tom.name AS personName; \ No newline at end of file diff --git a/flex/tests/hqps/queries/movie/query10.cypher b/flex/tests/hqps/queries/movie/query10.cypher new file mode 100644 index 000000000000..4787fc5f03ec --- /dev/null +++ b/flex/tests/hqps/queries/movie/query10.cypher @@ -0,0 +1,4 @@ +MATCH p=shortestPath( + (bacon:Person {name:"Kevin Bacon"})-[*]-(meg:Person {name:"Meg Ryan"}) +) +RETURN p; \ No newline at end of file diff --git a/flex/tests/hqps/queries/movie/query11.cypher b/flex/tests/hqps/queries/movie/query11.cypher new file mode 100644 index 000000000000..f4e32e1b70f8 --- /dev/null +++ b/flex/tests/hqps/queries/movie/query11.cypher @@ -0,0 +1,4 @@ +MATCH (tom:Person {name: 'Tom Hanks'})-[r:ACTED_IN]->(movie:Movie) +WITH movie.title as movieTitle, movie.released as movieReleased +ORDER BY movieReleased DESC, movieTitle ASC LIMIT 10 +return movieTitle, movieReleased; \ No newline at end of file diff --git a/flex/tests/hqps/queries/movie/query12.cypher b/flex/tests/hqps/queries/movie/query12.cypher new file mode 100644 index 000000000000..9d70234aa885 --- /dev/null +++ b/flex/tests/hqps/queries/movie/query12.cypher @@ -0,0 +1,2 @@ +MATCH (tom:Person {name: 'Tom Hanks'})-[:ACTED_IN]->(:Movie)<-[:ACTED_IN]-(coActor:Person) +WITH DISTINCT coActor.name AS coActorName ORDER BY coActorName ASC LIMIT 10 return coActorName; \ No newline at end of file diff --git a/flex/tests/hqps/queries/movie/query13.cypher b/flex/tests/hqps/queries/movie/query13.cypher new file mode 100644 index 000000000000..e8f5b9ddd6b1 --- /dev/null +++ b/flex/tests/hqps/queries/movie/query13.cypher @@ -0,0 +1,4 @@ +MATCH (tom:Person {name: 'Tom Hanks'})-[:ACTED_IN]->(movie1:Movie)<-[:ACTED_IN]-(coActor:Person)-[:ACTED_IN]->(movie2:Movie)<-[:ACTED_IN]-(coCoActor:Person) +WHERE tom <> coCoActor +AND NOT (tom)-[:ACTED_IN]->(:Movie)<-[:ACTED_IN]-(coCoActor) +RETURN coCoActor.name AS coCoActorName ORDER BY coCoActorName ASC LIMIT 10; \ No newline at end of file diff --git a/flex/tests/hqps/queries/movie/query14.cypher b/flex/tests/hqps/queries/movie/query14.cypher new file mode 100644 index 000000000000..368c65daca62 --- /dev/null +++ b/flex/tests/hqps/queries/movie/query14.cypher @@ -0,0 +1,6 @@ +MATCH (tom:Person {name: 'Tom Hanks'})-[:ACTED_IN]->(movie1:Movie)<-[:ACTED_IN]-(coActor:Person)-[:ACTED_IN]->(movie2:Movie)<-[:ACTED_IN]-(coCoActor:Person) +WHERE tom <> coCoActor +AND NOT (tom)-[:ACTED_IN]->(:Movie)<-[:ACTED_IN]-(coCoActor) +RETURN coCoActor.name AS coCoActorName, count(coCoActor) AS frequency +ORDER BY frequency DESC, coCoActorName ASC +LIMIT 5; \ No newline at end of file diff --git a/flex/tests/hqps/queries/movie/query15.cypher b/flex/tests/hqps/queries/movie/query15.cypher new file mode 100644 index 000000000000..f0af5c5ac45f --- /dev/null +++ b/flex/tests/hqps/queries/movie/query15.cypher @@ -0,0 +1,4 @@ +MATCH (tom:Person {name: 'Tom Hanks'})-[:ACTED_IN]->(movie1:Movie)<-[:ACTED_IN]-(coActor:Person)-[:ACTED_IN]->(movie2:Movie)<-[:ACTED_IN]-(cruise:Person {name: 'Tom Cruise'}) +WHERE NOT (tom)-[:ACTED_IN]->(:Movie)<-[:ACTED_IN]-(cruise) +RETURN tom.name AS actorName, movie1.title AS movie1Title, coActor.name AS coActorName, movie2.title AS movie2Title, cruise.name AS coCoActorName +ORDER BY movie1Title ASC, movie2Title ASC LIMIT 10; \ No newline at end of file diff --git a/flex/tests/hqps/queries/movie/query2.cypher b/flex/tests/hqps/queries/movie/query2.cypher new file mode 100644 index 000000000000..f5a2260cb74e --- /dev/null +++ b/flex/tests/hqps/queries/movie/query2.cypher @@ -0,0 +1 @@ +MATCH (cloudAtlas:Movie {title: "Cloud Atlas"}) RETURN cloudAtlas.tagline AS tagline, cloudAtlas.released AS releasedYear,cloudAtlas.title AS title; \ No newline at end of file diff --git a/flex/tests/hqps/queries/movie/query3.cypher b/flex/tests/hqps/queries/movie/query3.cypher new file mode 100644 index 000000000000..d73ccef0f7e1 --- /dev/null +++ b/flex/tests/hqps/queries/movie/query3.cypher @@ -0,0 +1 @@ +MATCH (people:Person) RETURN people.name AS personName ORDER BY personName ASC LIMIT 10; \ No newline at end of file diff --git a/flex/tests/hqps/queries/movie/query4.cypher b/flex/tests/hqps/queries/movie/query4.cypher new file mode 100644 index 000000000000..e3af07a9f751 --- /dev/null +++ b/flex/tests/hqps/queries/movie/query4.cypher @@ -0,0 +1,2 @@ +MATCH (nineties:Movie) WHERE nineties.released >= 1990 AND nineties.released < 2000 +RETURN nineties.title AS ninetiesTitle ORDER BY ninetiesTitle DESC LIMIT 10; \ No newline at end of file diff --git a/flex/tests/hqps/queries/movie/query5.cypher b/flex/tests/hqps/queries/movie/query5.cypher new file mode 100644 index 000000000000..37a84b105aa1 --- /dev/null +++ b/flex/tests/hqps/queries/movie/query5.cypher @@ -0,0 +1,6 @@ +MATCH (tom:Person {name: "Tom Hanks"})-[:ACTED_IN]->(tomHanksMovies) +RETURN tom.born AS bornYear, + tomHanksMovies.tagline AS movieTagline, + tomHanksMovies.title AS movieTitle, + tomHanksMovies.released AS releaseYear +ORDER BY releaseYear DESC, movieTitle ASC LIMIT 10; \ No newline at end of file diff --git a/flex/tests/hqps/queries/movie/query6.cypher b/flex/tests/hqps/queries/movie/query6.cypher new file mode 100644 index 000000000000..334377f7dddd --- /dev/null +++ b/flex/tests/hqps/queries/movie/query6.cypher @@ -0,0 +1,2 @@ +MATCH (cloudAtlas:Movie {title: "Cloud Atlas"})<-[:DIRECTED]-(directors) +RETURN directors.name AS directorsName ORDER BY directorsName ASC LIMIT 10; \ No newline at end of file diff --git a/flex/tests/hqps/queries/movie/query7.cypher b/flex/tests/hqps/queries/movie/query7.cypher new file mode 100644 index 000000000000..4de99e41b88e --- /dev/null +++ b/flex/tests/hqps/queries/movie/query7.cypher @@ -0,0 +1,3 @@ +MATCH (tom:Person {name:"Tom Hanks"})-[:ACTED_IN]->(m)<-[:ACTED_IN]-(coActors) +RETURN m.title AS movieTitle, m.released AS releasedYear, coActors.name AS coActorName +ORDER BY releasedYear DESC, movieTitle ASC LIMIT 10; \ No newline at end of file diff --git a/flex/tests/hqps/queries/movie/query8.cypher b/flex/tests/hqps/queries/movie/query8.cypher new file mode 100644 index 000000000000..3335a4621dfd --- /dev/null +++ b/flex/tests/hqps/queries/movie/query8.cypher @@ -0,0 +1,2 @@ +MATCH (people:Person)-[relatedTo]-(:Movie {title: "Cloud Atlas"}) +RETURN people.name, type(relatedTo), relatedTo \ No newline at end of file diff --git a/flex/tests/hqps/queries/movie/query9.cypher b/flex/tests/hqps/queries/movie/query9.cypher new file mode 100644 index 000000000000..951cef1edf3c --- /dev/null +++ b/flex/tests/hqps/queries/movie/query9.cypher @@ -0,0 +1,2 @@ +MATCH (bacon:Person {name:"Kevin Bacon"})-[*1..3]-(hollywood) +RETURN DISTINCT bacon, hollywood \ No newline at end of file diff --git a/interactive_engine/compiler/pom.xml b/interactive_engine/compiler/pom.xml index 1af1b5bb93bb..590a7928cccf 100644 --- a/interactive_engine/compiler/pom.xml +++ b/interactive_engine/compiler/pom.xml @@ -253,6 +253,7 @@ **/IrLdbcTest.java **/SimpleMatchTest.java **/IrPatternTest.java + **/MovieTest.java diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/reader/LocalMetaDataReader.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/reader/LocalMetaDataReader.java index ab76af784fe1..b8ce6a9dd960 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/reader/LocalMetaDataReader.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/reader/LocalMetaDataReader.java @@ -99,7 +99,7 @@ private Map getProcedureNameWithInputStream(File procedureD logger.debug("load procedure {}", procedureName); } } catch (Exception e) { - logger.warn( + logger.debug( "procedure config {} has invalid format, error msg: {}", file.getName(), e); } } diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/cypher/integration/suite/ldbc/MovieQueries.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/cypher/integration/suite/ldbc/MovieQueries.java new file mode 100644 index 000000000000..6cdb43fc0a34 --- /dev/null +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/cypher/integration/suite/ldbc/MovieQueries.java @@ -0,0 +1,302 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.cypher.integration.suite.ldbc; + +import java.util.Arrays; +import java.util.List; + +public class MovieQueries { + public static QueryContext get_movie_query1_test() { + String query = + "MATCH (tom:Person) WHERE tom.name = \"Tom Hanks\" RETURN tom.born AS" + + " bornYear,tom.name AS personName;"; + List expected = + Arrays.asList("Record<{bornYear: 1956, personName: \"Tom Hanks\"}>"); + return new QueryContext(query, expected); + } + + public static QueryContext get_movie_query2_test() { + String query = + "MATCH (cloudAtlas:Movie {title: \"Cloud Atlas\"}) RETURN cloudAtlas.tagline AS" + + " tagline, cloudAtlas.released AS releasedYear,cloudAtlas.title AS title;"; + List expected = + Arrays.asList( + "Record<{tagline: \"Everything is connected\", releasedYear: 2012, title:" + + " \"Cloud Atlas\"}>"); + return new QueryContext(query, expected); + } + + public static QueryContext get_movie_query3_test() { + String query = + "MATCH (people:Person) RETURN people.name AS personName ORDER BY personName ASC" + + " LIMIT 10;"; + List expected = + Arrays.asList( + "Record<{personName: \"Aaron Sorkin\"}>", + "Record<{personName: \"Al Pacino\"}>", + "Record<{personName: \"Angela Scope\"}>", + "Record<{personName: \"Annabella Sciorra\"}>", + "Record<{personName: \"Anthony Edwards\"}>", + "Record<{personName: \"Audrey Tautou\"}>", + "Record<{personName: \"Ben Miles\"}>", + "Record<{personName: \"Bill Paxton\"}>", + "Record<{personName: \"Bill Pullman\"}>", + "Record<{personName: \"Billy Crystal\"}>"); + return new QueryContext(query, expected); + } + + public static QueryContext get_movie_query4_test() { + String query = + "MATCH (nineties:Movie) WHERE nineties.released >= 1990 AND nineties.released <" + + " 2000\n" + + "RETURN nineties.title AS ninetiesTitle ORDER BY ninetiesTitle DESC LIMIT" + + " 10;"; + List expected = + Arrays.asList( + "Record<{ninetiesTitle: \"You've Got Mail\"}>", + "Record<{ninetiesTitle: \"When Harry Met Sally\"}>", + "Record<{ninetiesTitle: \"What Dreams May Come\"}>", + "Record<{ninetiesTitle: \"Unforgiven\"}>", + "Record<{ninetiesTitle: \"Twister\"}>", + "Record<{ninetiesTitle: \"The Matrix\"}>", + "Record<{ninetiesTitle: \"The Green Mile\"}>", + "Record<{ninetiesTitle: \"The Devil's Advocate\"}>", + "Record<{ninetiesTitle: \"The Birdcage\"}>", + "Record<{ninetiesTitle: \"That Thing You Do\"}>"); + return new QueryContext(query, expected); + } + + public static QueryContext get_movie_query5_test() { + String query = + "MATCH (tom:Person {name: \"Tom Hanks\"})-[:ACTED_IN]->(tomHanksMovies)\n" + + "RETURN tom.born AS bornYear,\n" + + "tomHanksMovies.tagline AS movieTagline,\n" + + "tomHanksMovies.title AS movieTitle,\n" + + "tomHanksMovies.released AS releaseYear\n" + + "ORDER BY releaseYear DESC, movieTitle ASC LIMIT 10;"; + List expected = + Arrays.asList( + "Record<{bornYear: 1956, movieTagline: \"Everything is connected\"," + + " movieTitle: \"Cloud Atlas\", releaseYear: 2012}>", + "Record<{bornYear: 1956, movieTagline: \"A stiff drink. A little mascara. A" + + " lot of nerve. Who said they couldn't bring down the Soviet" + + " empire.\", movieTitle: \"Charlie Wilson's War\", releaseYear:" + + " 2007}>", + "Record<{bornYear: 1956, movieTagline: \"Break The Codes\", movieTitle:" + + " \"The Da Vinci Code\", releaseYear: 2006}>", + "Record<{bornYear: 1956, movieTagline: \"This Holiday Season... Believe\"," + + " movieTitle: \"The Polar Express\", releaseYear: 2004}>", + "Record<{bornYear: 1956, movieTagline: \"At the edge of the world, his" + + " journey begins.\", movieTitle: \"Cast Away\", releaseYear: 2000}>", + "Record<{bornYear: 1956, movieTagline: \"Walk a mile you'll never" + + " forget.\", movieTitle: \"The Green Mile\", releaseYear: 1999}>", + "Record<{bornYear: 1956, movieTagline: \"At odds in life... in love" + + " on-line.\", movieTitle: \"You've Got Mail\", releaseYear: 1998}>", + "Record<{bornYear: 1956, movieTagline: \"In every life there comes a time" + + " when that thing you dream becomes that thing you do\", movieTitle:" + + " \"That Thing You Do\", releaseYear: 1996}>", + "Record<{bornYear: 1956, movieTagline: \"Houston, we have a problem.\"," + + " movieTitle: \"Apollo 13\", releaseYear: 1995}>", + "Record<{bornYear: 1956, movieTagline: \"What if someone you never met," + + " someone you never saw, someone you never knew was the only someone" + + " for you?\", movieTitle: \"Sleepless in Seattle\", releaseYear:" + + " 1993}>"); + return new QueryContext(query, expected); + } + + public static QueryContext get_movie_query6_test() { + String query = + "MATCH (cloudAtlas:Movie {title: \"Cloud Atlas\"})<-[:DIRECTED]-(directors)\n" + + "RETURN directors.name AS directorsName ORDER BY directorsName ASC LIMIT 10;"; + List expected = + Arrays.asList( + "Record<{directorsName: \"Lana Wachowski\"}>", + "Record<{directorsName: \"Lilly Wachowski\"}>", + "Record<{directorsName: \"Tom Tykwer\"}>"); + return new QueryContext(query, expected); + } + + public static QueryContext get_movie_query7_test() { + String query = + "MATCH (tom:Person {name:\"Tom Hanks\"})-[:ACTED_IN]->(m)<-[:ACTED_IN]-(coActors)\n" + + "RETURN m.title AS movieTitle, m.released AS releasedYear, coActors.name AS" + + " coActorName\n" + + "ORDER BY releasedYear DESC, movieTitle ASC LIMIT 10;"; + List expected = + Arrays.asList( + "Record<{movieTitle: \"Cloud Atlas\", releasedYear: 2012, coActorName:" + + " \"Hugo Weaving\"}>", + "Record<{movieTitle: \"Cloud Atlas\", releasedYear: 2012, coActorName:" + + " \"Jim Broadbent\"}>", + "Record<{movieTitle: \"Cloud Atlas\", releasedYear: 2012, coActorName:" + + " \"Halle Berry\"}>", + "Record<{movieTitle: \"Cloud Atlas\", releasedYear: 2012, coActorName:" + + " \"Tom Hanks\"}>", + "Record<{movieTitle: \"Charlie Wilson's War\", releasedYear: 2007," + + " coActorName: \"Julia Roberts\"}>", + "Record<{movieTitle: \"Charlie Wilson's War\", releasedYear: 2007," + + " coActorName: \"Tom Hanks\"}>", + "Record<{movieTitle: \"Charlie Wilson's War\", releasedYear: 2007," + + " coActorName: \"Philip Seymour Hoffman\"}>", + "Record<{movieTitle: \"The Da Vinci Code\", releasedYear: 2006," + + " coActorName: \"Tom Hanks\"}>", + "Record<{movieTitle: \"The Da Vinci Code\", releasedYear: 2006," + + " coActorName: \"Ian McKellen\"}>", + "Record<{movieTitle: \"The Da Vinci Code\", releasedYear: 2006," + + " coActorName: \"Audrey Tautou\"}>"); + return new QueryContext(query, expected); + } + + public static QueryContext get_movie_query8_test() { + String query = + "MATCH (people:Person)-[relatedTo]-(:Movie {title: \"Cloud Atlas\"})\n" + + "RETURN personName, type(relatedTo), relatedTo"; + List expected = Arrays.asList(); + return new QueryContext(query, expected); + } + + public static QueryContext get_movie_query9_test() { + String query = + "MATCH (bacon:Person {name:\"Kevin Bacon\"})-[*1..3]-(hollywood)\n" + + "RETURN DISTINCT bacon, hollywood"; + List expected = Arrays.asList(); + return new QueryContext(query, expected); + } + + public static QueryContext get_movie_query10_test() { + String query = + "MATCH p=shortestPath(\n" + + " (bacon:Person {name:\"Kevin Bacon\"})-[*]-(meg:Person {name:\"Meg" + + " Ryan\"})\n" + + ")\n" + + "RETURN p;"; + List expected = Arrays.asList(); + return new QueryContext(query, expected); + } + + public static QueryContext get_movie_query11_test() { + String query = + "MATCH (tom:Person {name: 'Tom Hanks'})-[r:ACTED_IN]->(movie:Movie)\n" + + "WITH movie.title as movieTitle, movie.released as movieReleased\n" + + "ORDER BY movieReleased DESC, movieTitle ASC LIMIT 10\n" + + "return movieTitle, movieReleased;"; + List expected = + Arrays.asList( + "Record<{movieTitle: \"Cloud Atlas\", movieReleased: 2012}>", + "Record<{movieTitle: \"Charlie Wilson's War\", movieReleased: 2007}>", + "Record<{movieTitle: \"The Da Vinci Code\", movieReleased: 2006}>", + "Record<{movieTitle: \"The Polar Express\", movieReleased: 2004}>", + "Record<{movieTitle: \"Cast Away\", movieReleased: 2000}>", + "Record<{movieTitle: \"The Green Mile\", movieReleased: 1999}>", + "Record<{movieTitle: \"You've Got Mail\", movieReleased: 1998}>", + "Record<{movieTitle: \"That Thing You Do\", movieReleased: 1996}>", + "Record<{movieTitle: \"Apollo 13\", movieReleased: 1995}>", + "Record<{movieTitle: \"Sleepless in Seattle\", movieReleased: 1993}>"); + return new QueryContext(query, expected); + } + + public static QueryContext get_movie_query12_test() { + String query = + "MATCH (tom:Person {name: 'Tom" + + " Hanks'})-[:ACTED_IN]->(:Movie)<-[:ACTED_IN]-(coActor:Person)\n" + + "WITH DISTINCT coActor.name AS coActorName ORDER BY coActorName ASC LIMIT 10" + + " return coActorName;"; + List expected = + Arrays.asList( + "Record<{coActorName: \"Audrey Tautou\"}>", + "Record<{coActorName: \"Bill Paxton\"}>", + "Record<{coActorName: \"Bill Pullman\"}>", + "Record<{coActorName: \"Bonnie Hunt\"}>", + "Record<{coActorName: \"Charlize Theron\"}>", + "Record<{coActorName: \"Dave Chappelle\"}>", + "Record<{coActorName: \"David Morse\"}>", + "Record<{coActorName: \"Ed Harris\"}>", + "Record<{coActorName: \"Gary Sinise\"}>", + "Record<{coActorName: \"Geena Davis\"}>"); + return new QueryContext(query, expected); + } + + public static QueryContext get_movie_query13_test() { + String query = + "MATCH (tom:Person {name: 'Tom" + + " Hanks'})-[:ACTED_IN]->(movie1:Movie)<-[:ACTED_IN]-(coActor:Person)-[:ACTED_IN]->(movie2:Movie)<-[:ACTED_IN]-(coCoActor:Person)\n" + + "WHERE tom <> coCoActor\n" + + "AND NOT (tom)-[:ACTED_IN]->(:Movie)<-[:ACTED_IN]-(coCoActor)\n" + + "RETURN coCoActor.name AS coCoActorName ORDER BY coCoActorName ASC LIMIT 10;"; + List expected = + Arrays.asList( + "Record<{coCoActorName: \"Aaron Sorkin\"}>", + "Record<{coCoActorName: \"Al Pacino\"}>", + "Record<{coCoActorName: \"Anthony Edwards\"}>", + "Record<{coCoActorName: \"Anthony Edwards\"}>", + "Record<{coCoActorName: \"Anthony Edwards\"}>", + "Record<{coCoActorName: \"Ben Miles\"}>", + "Record<{coCoActorName: \"Billy Crystal\"}>", + "Record<{coCoActorName: \"Billy Crystal\"}>", + "Record<{coCoActorName: \"Billy Crystal\"}>", + "Record<{coCoActorName: \"Bruno Kirby\"}>"); + return new QueryContext(query, expected); + } + + public static QueryContext get_movie_query14_test() { + String query = + "MATCH (tom:Person {name: 'Tom" + + " Hanks'})-[:ACTED_IN]->(movie1:Movie)<-[:ACTED_IN]-(coActor:Person)-[:ACTED_IN]->(movie2:Movie)<-[:ACTED_IN]-(coCoActor:Person)\n" + + "WHERE tom <> coCoActor\n" + + "AND NOT (tom)-[:ACTED_IN]->(:Movie)<-[:ACTED_IN]-(coCoActor)\n" + + "RETURN coCoActor.name AS coCoActorName, count(coCoActor) AS frequency\n" + + "ORDER BY frequency DESC, coCoActorName ASC\n" + + "LIMIT 5;"; + List expected = + Arrays.asList( + "Record<{coCoActorName: \"Tom Cruise\", frequency: 5}>", + "Record<{coCoActorName: \"Zach Grenier\", frequency: 5}>", + "Record<{coCoActorName: \"Cuba Gooding Jr.\", frequency: 4}>", + "Record<{coCoActorName: \"Keanu Reeves\", frequency: 4}>", + "Record<{coCoActorName: \"Anthony Edwards\", frequency: 3}>"); + return new QueryContext(query, expected); + } + + public static QueryContext get_movie_query15_test() { + String query = + "MATCH (tom:Person {name: 'Tom" + + " Hanks'})-[:ACTED_IN]->(movie1:Movie)<-[:ACTED_IN]-(coActor:Person)-[:ACTED_IN]->(movie2:Movie)<-[:ACTED_IN]-(cruise:Person" + + " {name: 'Tom Cruise'})\n" + + "WHERE NOT (tom)-[:ACTED_IN]->(:Movie)<-[:ACTED_IN]-(cruise)\n" + + "RETURN tom.name AS actorName, movie1.title AS movie1Title, coActor.name AS" + + " coActorName, movie2.title AS movie2Title, cruise.name AS coCoActorName\n" + + "ORDER BY movie1Title ASC, movie2Title ASC LIMIT 10;"; + List expected = + Arrays.asList( + "Record<{actorName: \"Tom Hanks\", movie1Title: \"Apollo 13\", coActorName:" + + " \"Kevin Bacon\", movie2Title: \"A Few Good Men\", coCoActorName:" + + " \"Tom Cruise\"}>", + "Record<{actorName: \"Tom Hanks\", movie1Title: \"Joe Versus the Volcano\"," + + " coActorName: \"Meg Ryan\", movie2Title: \"Top Gun\", coCoActorName:" + + " \"Tom Cruise\"}>", + "Record<{actorName: \"Tom Hanks\", movie1Title: \"Sleepless in Seattle\"," + + " coActorName: \"Meg Ryan\", movie2Title: \"Top Gun\", coCoActorName:" + + " \"Tom Cruise\"}>", + "Record<{actorName: \"Tom Hanks\", movie1Title: \"The Green Mile\"," + + " coActorName: \"Bonnie Hunt\", movie2Title: \"Jerry Maguire\"," + + " coCoActorName: \"Tom Cruise\"}>", + "Record<{actorName: \"Tom Hanks\", movie1Title: \"You've Got Mail\"," + + " coActorName: \"Meg Ryan\", movie2Title: \"Top Gun\", coCoActorName:" + + " \"Tom Cruise\"}>"); + return new QueryContext(query, expected); + } +} diff --git a/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/cypher/integration/movie/MovieTest.java b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/cypher/integration/movie/MovieTest.java new file mode 100644 index 000000000000..44932da81601 --- /dev/null +++ b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/cypher/integration/movie/MovieTest.java @@ -0,0 +1,131 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.cypher.integration.movie; + +import com.alibaba.graphscope.cypher.integration.suite.ldbc.MovieQueries; +import com.alibaba.graphscope.cypher.integration.suite.ldbc.QueryContext; + +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; +import org.neo4j.driver.GraphDatabase; +import org.neo4j.driver.Result; +import org.neo4j.driver.Session; + +public class MovieTest { + + private static Session session; + + @BeforeClass + public static void beforeClass() { + String neo4jServerUrl = + System.getProperty("neo4j.bolt.server.url", "neo4j://localhost:7687"); + session = GraphDatabase.driver(neo4jServerUrl).session(); + } + + @Test + public void run_movie_query1_test() { + QueryContext testQuery = MovieQueries.get_movie_query1_test(); + Result result = session.run(testQuery.getQuery()); + Assert.assertEquals(testQuery.getExpectedResult().toString(), result.list().toString()); + } + + @Test + public void run_movie_query2_test() { + QueryContext testQuery = MovieQueries.get_movie_query2_test(); + Result result = session.run(testQuery.getQuery()); + Assert.assertEquals(testQuery.getExpectedResult().toString(), result.list().toString()); + } + + @Test + public void run_movie_query3_test() { + QueryContext testQuery = MovieQueries.get_movie_query3_test(); + Result result = session.run(testQuery.getQuery()); + Assert.assertEquals(testQuery.getExpectedResult().toString(), result.list().toString()); + } + + @Test + public void run_movie_query4_test() { + QueryContext testQuery = MovieQueries.get_movie_query4_test(); + Result result = session.run(testQuery.getQuery()); + Assert.assertEquals(testQuery.getExpectedResult().toString(), result.list().toString()); + } + + @Test + public void run_movie_query5_test() { + QueryContext testQuery = MovieQueries.get_movie_query5_test(); + Result result = session.run(testQuery.getQuery()); + Assert.assertEquals(testQuery.getExpectedResult().toString(), result.list().toString()); + } + + @Test + public void run_movie_query6_test() { + QueryContext testQuery = MovieQueries.get_movie_query6_test(); + Result result = session.run(testQuery.getQuery()); + Assert.assertEquals(testQuery.getExpectedResult().toString(), result.list().toString()); + } + + @Test + public void run_movie_query7_test() { + QueryContext testQuery = MovieQueries.get_movie_query7_test(); + Result result = session.run(testQuery.getQuery()); + Assert.assertEquals(testQuery.getExpectedResult().toString(), result.list().toString()); + } + + @Test + public void run_movie_query11_test() { + QueryContext testQuery = MovieQueries.get_movie_query11_test(); + Result result = session.run(testQuery.getQuery()); + Assert.assertEquals(testQuery.getExpectedResult().toString(), result.list().toString()); + } + + @Test + public void run_movie_query12_test() { + QueryContext testQuery = MovieQueries.get_movie_query12_test(); + Result result = session.run(testQuery.getQuery()); + Assert.assertEquals(testQuery.getExpectedResult().toString(), result.list().toString()); + } + + @Test + public void run_movie_query13_test() { + QueryContext testQuery = MovieQueries.get_movie_query13_test(); + Result result = session.run(testQuery.getQuery()); + Assert.assertEquals(testQuery.getExpectedResult().toString(), result.list().toString()); + } + + @Test + public void run_movie_query14_test() { + QueryContext testQuery = MovieQueries.get_movie_query14_test(); + Result result = session.run(testQuery.getQuery()); + Assert.assertEquals(testQuery.getExpectedResult().toString(), result.list().toString()); + } + + @Test + public void run_movie_query15_test() { + QueryContext testQuery = MovieQueries.get_movie_query15_test(); + Result result = session.run(testQuery.getQuery()); + Assert.assertEquals(testQuery.getExpectedResult().toString(), result.list().toString()); + } + + @AfterClass + public static void afterClass() { + if (session != null) { + session.close(); + } + } +}