From 524491b5f8d8c3351df328518bc84d1b7f39db5a Mon Sep 17 00:00:00 2001 From: "xiaolei.zl" Date: Fri, 15 Sep 2023 17:34:45 +0800 Subject: [PATCH] fix some bug --- flex/interactive/bin/gs_interactive | 237 +++++++++++++++--- .../examples/modern_graph/bulk_load.yaml | 2 +- 2 files changed, 197 insertions(+), 42 deletions(-) diff --git a/flex/interactive/bin/gs_interactive b/flex/interactive/bin/gs_interactive index df907199123a..13e2290f342b 100755 --- a/flex/interactive/bin/gs_interactive +++ b/flex/interactive/bin/gs_interactive @@ -20,6 +20,7 @@ DB_PROD_NAME="interactive" # colored error and info functions to wrap messages. RED='\033[0;31m' GREEN='\033[0;32m' +YELLOW='\033[0;33m' NC='\033[0m' # No Color err() { echo -e "${RED}[$(date +'%Y-%m-%d %H:%M:%S')] -ERROR- $* ${NC}" >&2 @@ -29,6 +30,10 @@ info() { echo -e "${GREEN}[$(date +'%Y-%m-%d %H:%M:%S')] -INFO- $* ${NC}" } +emph(){ + echo -e "${YELLOW}[$(date +'%Y-%m-%d %H:%M:%S')] -INFO- $* ${NC}" +} + ################## Some Util Functions ################## # source: https://github.com/mrbaseman/parse_yaml.git @@ -91,10 +96,10 @@ function parse_yaml { sed -ne "s|^\($s\):|\1|" \ -e "s|^\($s\)\(---\)\($s\)||" \ -e "s|^\($s\)\(\.\.\.\)\($s\)||" \ - -e "s|^\($s\)-$s[\"']\(.*\)[\"']$s\$|\1$fs$fs\2|p;t" \ - -e "s|^\($s\)\($w\)$s:$s[\"']\(.*\)[\"']$s\$|\1$fs\2$fs\3|p;t" \ + -e "s|^\($s\)-${s}[\"']\(.*\)[\"']$s\$|\1$fs$fs\2|p;t" \ + -e "s|^\($s\)\($w\)$s:${s}[\"']\(.*\)[\"']$s\$|\1$fs\2$fs\3|p;t" \ -e "s|^\($s\)-$s\(.*\)$s\$|\1$fs$fs\2|" \ - -e "s|^\($s\)\($w\)$s:$s[\"']\?\(.*\)$s\$|\1$fs\2$fs\3|" \ + -e "s|^\($s\)\($w\)$s:${s}[\"']\?\(.*\)$s\$|\1$fs\2$fs\3|" \ -e "s|^\($s\)[\"']\?\([^&][^$fs]\+\)[\"']$s\$|\1$fs$fs$fs\2|" \ -e "s|^\($s\)[\"']\?\([^&][^$fs]\+\)$s\$|\1$fs$fs$fs\2|" \ -e "s|$s\$||p" | \ @@ -190,9 +195,9 @@ HOST_DB_TMP_DIR="/tmp" #################### DEFINE DEFAULT CONSTATNS #################### DATABASE_VERSION="v0.0.2" -DATABASE_DEFAULT_GRAPH_NAME="modern" -DATABASE_CURRENT_GRAPH_NAME="modern" -DATABASE_DEFAULT_GRAPH_DOCKER_PATH="/home/graphscope/default_graph/${DEFAULT_GRAPH_NAME}" +DATABASE_DEFAULT_GRAPH_NAME="default_graph" +DATABASE_CURRENT_GRAPH_NAME=${DATABASE_DEFAULT_GRAPH_NAME} +DATABASE_DEFAULT_GRAPH_DOCKER_PATH="/home/graphscope/${DATABASE_DEFAULT_GRAPH_NAME}" DATABASE_DEFAULT_GRAPH_MOUNT_CMD="${HOST_DB_HOME}/examples/modern_graph/:${DATABASE_DEFAULT_GRAPH_DOCKER_PATH}" DATABASE_VOLUMES="${DATABASE_DEFAULT_GRAPH_MOUNT_CMD}" DATABASE_LOG_LEVEL="INFO" @@ -229,6 +234,19 @@ GIE_DB_CONTAINER_NAME="${DB_PROD_NAME}-server" info "Finish create log dir" #################### DEFINE FUNCTIONS #################### + +function check_graph_name_valid() { + if [ $# -ne 1 ]; then + err "Expect graph name given." + exit 1 + fi + local graph_name=$1 + # check graph_name can be a valid directory name + if [[ ! "${graph_name}" =~ ^[a-zA-Z0-9_]+$ ]]; then + err "graph name ${graph_name} is not valid, only [a-zA-Z0-9_] are allowed." + exit 1 + fi +} function check_running_containers_and_exit(){ # check if there is any running containers info "Check running containers and exit" @@ -423,10 +441,18 @@ function import_usage() { EOF } +function list_usage() { + cat << EOF + gs_interactive database list + List all graphs in the database. +EOF +} + function database_usage(){ create_usage remove_usage import_usage + list_usage } @@ -777,6 +803,16 @@ function do_init(){ docker exec -u graphscope "${GIE_DB_CONTAINER_NAME}" bash -c "sudo chown -R graphscope:graphscope ${DATABASE_WORKSPACE}" || exit 1 docker exec -u graphscope "${GIE_DB_CONTAINER_NAME}" bash -c "mkdir -p ${DATABASE_WORKSPACE}/logs" || exit 1 docker exec -u graphscope "${GIE_DB_CONTAINER_NAME}" bash -c "mkdir -p ${DATABASE_WORKSPACE}/conf" || exit 1 + + # create default_graph + do_create -g ${DATABASE_DEFAULT_GRAPH_NAME} -c ${HOST_DB_HOME}/examples/modern_graph/modern_graph.yaml + do_import -g ${DATABASE_DEFAULT_GRAPH_NAME} -c ${HOST_DB_HOME}/examples/modern_graph/bulk_load.yaml + # check whether do_import success + if [ $? -ne 0 ]; then + err "Fail to import default graph" + exit 1 + fi + info "Successfuly create and import default graph: ${DATABASE_DEFAULT_GRAPH_NAME}" } @@ -808,9 +844,10 @@ function do_create(){ create_usage exit 1 fi + check_graph_name_valid "${graph_name}" check_file_exists "${schema_file}" # check graph is running inside docker - check_graph_not_running ${graph_name} || err "Can not create graph ${graph_name}, since a graph with same nameing running." + check_graph_not_running ${graph_name} || (err "Can not create graph ${graph_name}, since a graph with same nameing running." && exit 1) # create the graph directory in the docker's workspace . ${HOST_DB_ENV_FILE} docker_graph_dir="${DATABASE_WORKSPACE}/data/${graph_name}" @@ -852,12 +889,40 @@ function do_remove(){ remove_usage exit 1 fi + check_graph_name_valid "${graph_name}" + # check if the graph is created + if [ ! -d "${HOST_DB_HOME}/data/${graph_name}" ]; then + err "graph [${graph_name}] can not be removed, since it is not created." + exit 1 + fi # check graph is running inside docker check_graph_not_running ${graph_name} || err "Can not remove graph ${graph_name}, since a graph with same nameing running." . ${HOST_DB_ENV_FILE} docker_graph_dir="${DATABASE_WORKSPACE}/data/${graph_name}" + ### let user confirm to remove the graph + read -p "Are you sure to remove graph ${graph_name}? [y/n]" -n 1 -r + echo + if [[ ! $REPLY =~ ^[Yy]$ ]]; then + info "Abort removing graph ${graph_name}" + exit 1 + fi + # rm -rf the graph directory in the docker's workspace - docker exec "${GIE_DB_CONTAINER_NAME}" bash -c "rm -rf ${docker_graph_dir}" || exit 1 + docker exec "${GIE_DB_CONTAINER_NAME}" bash -c "rm -rf ${docker_graph_dir}" || ( echo "Fail to delete graph ${graph_name}" && exit 1) + info "Successfuly removed graph ${graph_name}" + # if graph_name is current graph, set current graph to .running + if [ "${DATABASE_CURRENT_GRAPH_NAME}" = "${graph_name}" ]; then + info "Remove current graph ${graph_name}, set current graph to default_graph" + echo "export DATABASE_CURRENT_GRAPH_NAME=${DATABASE_DEFAULT_GRAPH_NAME}" >> ${HOST_DB_ENV_FILE} + fi + #local graph_name=$(sed -n '1p' ${HOST_DB_RUNNING_FILE} | cut -d '=' -f 2) + # if the removed graph is in .running, remove it + if [ -f "${HOST_DB_RUNNING_FILE}" ]; then + local graph_name=$(sed -n '1p' ${HOST_DB_RUNNING_FILE} | cut -d '=' -f 2) + if [ "${graph_name}" = "${graph_name}" ]; then + rm ${HOST_DB_RUNNING_FILE} + fi + fi } #################### Import #################### @@ -882,6 +947,13 @@ function do_import(){ ;; esac done + # check graph_name is set + if [ -z "${graph_name}" ]; then + err "graph name is not specified" + import_usage + exit 1 + fi + check_graph_name_valid "${graph_name}" info "Import data to graph ${graph_name} from ${bulk_load_file}" # check if the container is running check_container_running @@ -913,9 +985,45 @@ function do_import(){ info "Successfuly import data to graph ${graph_name}" } +#################### List Graph #################### +function do_list() { + # check if the container is running + check_container_running + # get all graph names + . ${HOST_DB_ENV_FILE} + host_data_dir="${HOST_DB_HOME}/${DATABASE_DATA_DIR_NAME}" + # get all graph names into a array + graph_names=($(ls ${host_data_dir})) + # if graph_names is empty, no graph exists + if [ ${#graph_names[@]} -eq 0 ]; then + info "No graph has been created." + exit 0 + fi + # print all graph names + for graph_name in "${graph_names[@]}"; do + echo "Found graph: ${graph_name}" + done + +} + #################### Destroy #################### function do_destroy() { - info "Destroy database" + + # if container not exists, exit + if ! docker ps -a --format '{{.Names}}' | grep -Eq "^${GIE_DB_CONTAINER_NAME}$"; then + info "Database has not been created, exit" + exit 0 + fi + + # let user confirm to destroy the database + read -p "Are you sure to destroy the database? [y/n]" -n 1 -r + echo + if [[ ! $REPLY =~ ^[Yy]$ ]]; then + info "Abort destroying database" + exit 1 + fi + + info "Destroying database..." docker stop "${GIE_DB_CONTAINER_NAME}" docker rm "${GIE_DB_CONTAINER_NAME}" . ${HOST_DB_ENV_FILE} @@ -923,9 +1031,12 @@ function do_destroy() { sudo rm -rf ${HOST_DB_HOME}/data/* #rm .running - rm ${HOST_DB_RUNNING_FILE} - rm ${HOST_DB_ENV_FILE} - + if [ -f "${HOST_DB_RUNNING_FILE}" ]; then + rm ${HOST_DB_RUNNING_FILE} + fi + if [ -f "${HOST_DB_ENV_FILE}" ]; then + rm ${HOST_DB_ENV_FILE} + fi info "Finish destroy database" } @@ -936,8 +1047,22 @@ function do_start(){ info "Starting database..." # check whether the .running file exists, if exists, exit - check_process_not_running_in_container "${GIE_DB_CONTAINER_NAME}" "${DOCKER_DB_SERVER_BIN}" "Database is already running" - check_process_not_running_in_container "${GIE_DB_CONTAINER_NAME}" "${DOCKER_DB_COMPILER_BIN}" "Compiler is already running" + local compiler_process_id=$(docker top "${GIE_DB_CONTAINER_NAME}" | grep "${DOCKER_DB_COMPILER_BIN}" | awk '{print $2}\') + local server_process_id=$(docker top "${GIE_DB_CONTAINER_NAME}" | grep "${DOCKER_DB_SERVER_BIN}" | awk '{print $2}\') + # if both process are running, exit + # if only one process is running, kill the process + if [ -n "${compiler_process_id}" ] && [ -n "${server_process_id}" ]; then + local old_graph_name=$(sed -n '1p' ${HOST_DB_RUNNING_FILE} | cut -d '=' -f 2) + err "Database is already running on [${old_graph_name}], please stop it first" + exit 1 + fi + if [ -n "${compiler_process_id}" ]; then + docker exec "${GIE_DB_CONTAINER_NAME}" bash -c "kill -9 ${compiler_process_id}" + fi + if [ -n "${server_process_id}" ]; then + docker exec "${GIE_DB_CONTAINER_NAME}" bash -c "kill -9 ${server_process_id}" + fi + # set trap to do_stop trap do_stop SIGINT SIGTERM @@ -968,10 +1093,11 @@ function do_start(){ # generate real engine config file, put it at /tmp/real_engine_config.yaml if [ -z "${graph_name}" ]; then graph_name=${DATABASE_CURRENT_GRAPH_NAME} - info "Using user specified graph ${graph_name}" else DATABASE_CURRENT_GRAPH_NAME=${graph_name} + info "Using user specified graph ${graph_name}" fi + check_graph_name_valid "${graph_name}" real_engine_config_file="/tmp/real_engine_config.yaml" if [ -z "${engine_config_file}" ]; then @@ -985,26 +1111,20 @@ function do_start(){ docker cp "${real_engine_config_file}" "${GIE_DB_CONTAINER_NAME}:${dst_engine_config_file}" || (echo "fail to copy $engine_config_file to container" && exit 1) - if [ -z "${graph_name}" ]; then - info "graph name is not specified" - info "Using default graph [modern]" - graph_name="modern" - fi - # check if modern_graph exists in container, get the result as bool docker_graph_schema_file="${DATABASE_WORKSPACE}/data/${graph_name}/graph.yaml" wal_file="${DATABASE_WORKSPACE}/data/${graph_name}/indices/init_snapshot.bin" docker exec "${GIE_DB_CONTAINER_NAME}" bash -c "( [ -f ${docker_graph_schema_file} ] && [ -f ${wal_file} ] && echo \"true\" e) || echo \"false\"" > /tmp/graph_exists graph_exists=$(cat /tmp/graph_exists) if [ "${graph_exists}" = "false" ]; then - info "graph ${graph_name} not exists, create it first" - # remove the data/${graph_name} directory in container - docker exec "${GIE_DB_CONTAINER_NAME}" bash -c "rm -rf ${DATABASE_WORKSPACE}/data/${graph_name}" - do_create -g ${graph_name} -c ${HOST_DB_HOME}/examples/modern_graph/modern_graph.yaml - do_import -g ${graph_name} -c ${HOST_DB_HOME}/examples/modern_graph/bulk_load.yaml - info "Successfuly create and import graph ${graph_name}" + # if graph_name is default_graph, we should create it first + # otherwise, we should tell user to create it first + if [ "${graph_name}" != "${DATABASE_DEFAULT_GRAPH_NAME}" ]; then + err "graph ${graph_name} not exists, please create it first" + exit 1 + fi else - info "graph ${graph_name} exists, skip create and import" + info "graph ${graph_name} exists, start it" fi do_stop @@ -1048,7 +1168,7 @@ function do_start(){ info "Successfuly start compiler" # get cypher port from engine config file # bolt_connector_port=$(parse_yaml "${engine_config_file}" | grep "compiler_endpoint_bolt_connector_port" | awk -F "=" '{print $2}') - info "DataBase service is running..., port is open on :${DATABASE_COMPILER_BOLT_PORT}" + emph "DataBase service is running, port is open on :${DATABASE_COMPILER_BOLT_PORT}" # if do_start success, we should write current args to ${HOST_DB_RUNNING_FILE} echo "GRAPH_NAME=${graph_name}" > ${HOST_DB_RUNNING_FILE} @@ -1068,10 +1188,15 @@ function do_stop(){ info "No running database found, do nothing" fi # get graph_name from ${HOST_DB_RUNNING_FILE} - local graph_name=$(sed -n '1p' ${HOST_DB_RUNNING_FILE} | cut -d '=' -f 2) - docker_graph_lock_file="${DATABASE_WORKSPACE}/data/${graph_name}/.lock" - docker exec "${GIE_DB_CONTAINER_NAME}" bash -c "rm -f ${docker_graph_lock_file}" || exit 1 - info "Successfuly remove ${docker_graph_lock_file} file" + if [ -f "${HOST_DB_RUNNING_FILE}" ]; then + local graph_name=$(sed -n '1p' ${HOST_DB_RUNNING_FILE} | cut -d '=' -f 2) + docker_graph_lock_file="${DATABASE_WORKSPACE}/data/${graph_name}/.lock" + docker exec "${GIE_DB_CONTAINER_NAME}" bash -c "rm -f ${docker_graph_lock_file}" || exit 1 + info "Successfuly remove ${docker_graph_lock_file} file" + else + info "No graph is running" + fi + # stop the SERVER_BIN process and graph_server process docker exec "${GIE_DB_CONTAINER_NAME}" bash -c "pkill -f ${DOCKER_DB_SERVER_BIN}" docker exec "${GIE_DB_CONTAINER_NAME}" bash -c "pkill -f ${DOCKER_DB_COMPILER_BIN}" @@ -1098,7 +1223,7 @@ function do_status() { # copy the engine config file to host's tmp directory docker cp "${GIE_DB_CONTAINER_NAME}:${docker_engine_config_file}" "${HOST_DB_TMP_DIR}/engine_config.yaml" || exit 1 eval $(parse_yaml "${HOST_DB_TMP_DIR}/engine_config.yaml") - info "Database service is running..., port is open on :${compiler_endpoint_bolt_connector_port}" + emph "Database service is running, port is open on :${compiler_endpoint_bolt_connector_port}" } @@ -1229,10 +1354,17 @@ function do_compile() { # check graph_name if [ -z "${graph_name}" ]; then - err "graph_name is empty" - compile_usage - exit 1 + # let user confirm that no graph_name is specified, will use default graph. + read -p "No graph_name is specified, will use default graph, continue? [y/n]" -n 1 -r + echo + if [[ ! $REPLY =~ ^[Yy]$ ]]; then + info "Abort compile stored_procedure" + exit 1 + fi + info "Using default graph [${DATABASE_DEFAULT_GRAPH_NAME}]" + graph_name=${DATABASE_DEFAULT_GRAPH_NAME} fi + check_graph_name_valid "${graph_name}" # check file_path check_file_exists "${file_path}" @@ -1287,7 +1419,9 @@ function do_compile() { echo "Running cmd: ${cmd}" eval ${cmd} || exit 1 # check output exists - output_file="${HOST_DB_HOME}/data/${graph_name}/plugins/lib${stored_procedure_name}.so" + # remove the suffix of file_name + real_file_name="${file_name%.*}" + output_file="${HOST_DB_HOME}/data/${graph_name}/plugins/lib${real_file_name}.so" if [ ! -f "${output_file}" ]; then err "output file ${output_file} not exist, compilation failed" @@ -1299,7 +1433,9 @@ function do_compile() { docker_graph_enable_file="${docker_graph_dir}/plugins/.enable" # copy container to host rm -f /tmp/.enable - docker cp "${GIE_DB_CONTAINER_NAME}:${docker_graph_enable_file}" "/tmp/.enable" || true + # if docker_graph_enable_file exists. copy it to host + docker exec "${GIE_DB_CONTAINER_NAME}" test -e "${docker_graph_enable_file}" && (docker cp "${GIE_DB_CONTAINER_NAME}:${docker_graph_enable_file}" "/tmp/.enable") + if [ ! -f "/tmp/.enable" ]; then touch "/tmp/.enable" fi @@ -1337,6 +1473,13 @@ function do_enable(){ ;; esac done + # check graph_name not empty + if [ -z "${graph_name}" ]; then + err "graph_name is empty" + enable_proc_usage + exit 1 + fi + check_graph_name_valid "${graph_name}" # --name and --config can not be set at the same time if [ ! -z "${stored_procedure_names}" ] && [ ! -z "${stored_procedure_names_yaml}" ]; then @@ -1425,6 +1568,7 @@ function do_disable(){ disable_proc_usage exit 1 fi + check_graph_name_valid "${graph_name}" info "graph_name = ${graph_name}" docker_graph_dir="${DATABASE_WORKSPACE}/data/${graph_name}" docker exec "${GIE_DB_CONTAINER_NAME}" bash -c "[ -d ${docker_graph_dir} ] || (echo \"graph ${graph_name} not exists, please create it first\" && exit 1)" @@ -1484,14 +1628,20 @@ function do_show(){ err "graph_name is empty" exit 1 fi + check_graph_name_valid "${graph_name}" info "graph_name = ${graph_name}" docker_graph_dir="${DATABASE_WORKSPACE}/data/${graph_name}" - docker exec "${GIE_DB_CONTAINER_NAME}" bash -c "[ -d ${docker_graph_dir} ] || (echo \"graph ${graph_name} not exists, please create it first\" && exit 1)" || exit 1 + docker exec "${GIE_DB_CONTAINER_NAME}" bash -c "[ -d ${docker_graph_dir} ] || (echo -e \"${RED}Graph ${graph_name} not exists, please create it first. ${NC}\" && exit 1)" || exit 1 docker_graph_plugin_dir="${docker_graph_dir}/plugins" docker_graph_enable_file="${docker_graph_plugin_dir}/.enable" + # check if docker_graph_enable_file exists, if not ,exit + docker exec "${GIE_DB_CONTAINER_NAME}" bash -c "[ -f ${docker_graph_enable_file} ] || (echo -e \"${RED}Graph [${graph_name}] has no procedures registered. ${NC}\" && exit 1)" || exit 1 docker cp "${GIE_DB_CONTAINER_NAME}:${docker_graph_enable_file}" "/tmp/.enable" || exit 1 info "Enabled stored_procedures for graph ${graph_name}:" - cat /tmp/.enable + # iterate the .enable file and print the stored_procedure_name + while read line; do + emph "Procedure: ${line}" + done < /tmp/.enable } function do_database(){ @@ -1513,6 +1663,11 @@ function do_database(){ do_import "$@" exit 0 ;; + list) + shift + do_list "$@" + exit 0 + ;; *) err "unknown option $1" database_usage diff --git a/flex/interactive/examples/modern_graph/bulk_load.yaml b/flex/interactive/examples/modern_graph/bulk_load.yaml index b1647ff4a294..2963e105e945 100644 --- a/flex/interactive/examples/modern_graph/bulk_load.yaml +++ b/flex/interactive/examples/modern_graph/bulk_load.yaml @@ -2,7 +2,7 @@ graph: modern loading_config: data_source: scheme: file # file, oss, s3, hdfs; only file is supported now - # location: /home/graphscope/default_graph/ + location: /home/graphscope/default_graph/ import_option: init # append, overwrite, only init is supported now format: type: csv