diff --git a/.github/workflows/interactive.yml b/.github/workflows/interactive.yml index 3505505d0fb5..97f666d60dcb 100644 --- a/.github/workflows/interactive.yml +++ b/.github/workflows/interactive.yml @@ -82,7 +82,7 @@ jobs: # build compiler cd ${GIE_HOME}/ - mvn clean install -Pexperimental -DskipTests + mvn clean install -Pexperimental -DskipTests -q - name: Prepare dataset and workspace env: @@ -166,6 +166,13 @@ jobs: bash hqps_sdk_test.sh ${TMP_INTERACTIVE_WORKSPACE} ./engine_config_test.yaml python sed -i 's/temp_workspace/interactive_workspace/g' ./engine_config_test.yaml sed -i 's/thread_num_per_worker: 4/thread_num_per_worker: 1/g' ./engine_config_test.yaml + + - name: Robustness test + env: + INTERACTIVE_WORKSPACE: /tmp/interactive_workspace + run: | + cd ${GITHUB_WORKSPACE}/flex/tests/hqps + bash hqps_robust_test.sh ${INTERACTIVE_WORKSPACE} ./engine_config_test.yaml - name: Sample Query test env: @@ -305,7 +312,7 @@ jobs: sdk_version=$(grep -oPm1 "(?<=)[^<]+" ${GITHUB_WORKSPACE}/flex/interactive/sdk/java/pom.xml) sed -i "s/.*<\/interactive.sdk.version>/${sdk_version}<\/interactive.sdk.version>/" ${GITHUB_WORKSPACE}/interactive_engine/pom.xml cd ${GITHUB_WORKSPACE}/interactive_engine/ - mvn clean install -Pexperimental -DskipTests + mvn clean install -Pexperimental -DskipTests -q - name: Run End-to-End cypher adhoc ldbc query test env: diff --git a/flex/engines/hqps_db/core/operator/sink.h b/flex/engines/hqps_db/core/operator/sink.h index 01ab8a782b6b..9ff64bf309b2 100644 --- a/flex/engines/hqps_db/core/operator/sink.h +++ b/flex/engines/hqps_db/core/operator/sink.h @@ -114,9 +114,8 @@ void template_set_value(common::Value* value, T v) { } template ) && - (!std::is_same_v)>::type* = nullptr> + typename std::enable_if<(std::is_same_v) &&( + !std::is_same_v)>::type* = nullptr> void template_set_value(common::Value* value, T v) { value->set_i64(v); } @@ -429,8 +428,12 @@ class SinkOp { // get all property for two labels vertex auto& schema = graph.schema(); std::array, 2> prop_names; - prop_names[0] = schema.get_vertex_property_names(labels[0]); - prop_names[1] = schema.get_vertex_property_names(labels[1]); + if (labels[0] < schema.vertex_label_num()) { + prop_names[0] = schema.get_vertex_property_names(labels[0]); + } + if (labels[1] < schema.vertex_label_num()) { + prop_names[1] = schema.get_vertex_property_names(labels[1]); + } // get all properties std::array>, 2> column_ptrs; for (size_t i = 0; i < prop_names[0].size(); ++i) { @@ -479,7 +482,7 @@ class SinkOp { for (size_t i : repeat_offsets) { num_rows += i; } - CHECK(num_rows == results_vec.results_size()) + CHECK((int32_t) num_rows == results_vec.results_size()) << num_rows << " " << results_vec.results_size(); } size_t cur_ind = 0; @@ -526,7 +529,9 @@ class SinkOp { results::CollectiveResults& results_vec, const std::vector& repeat_offsets, int32_t tag_id) { auto& schema = graph.schema(); - auto prop_names = schema.get_vertex_property_names(label); + auto prop_names = label < schema.vertex_label_num() + ? schema.get_vertex_property_names(label) + : std::vector(); // get all properties std::vector> column_ptrs; for (size_t i = 0; i < prop_names.size(); ++i) { diff --git a/flex/engines/hqps_db/structures/multi_edge_set/untyped_edge_set.h b/flex/engines/hqps_db/structures/multi_edge_set/untyped_edge_set.h index f0befaace3ef..8285fa160b60 100644 --- a/flex/engines/hqps_db/structures/multi_edge_set/untyped_edge_set.h +++ b/flex/engines/hqps_db/structures/multi_edge_set/untyped_edge_set.h @@ -359,7 +359,7 @@ class UnTypedEdgeSet { std::vector get_directions() const { std::vector res; auto edge_triplet = get_edge_triplets(); - for (auto src_label_ind = 0; src_label_ind < src_labels_.size(); + for (size_t src_label_ind = 0; src_label_ind < src_labels_.size(); ++src_label_ind) { auto src_label = src_labels_[src_label_ind]; std::vector> tmp; @@ -627,14 +627,13 @@ class UnTypedEdgeSet { auto src_vid = src_vertices_[i]; auto& cur_edge_iters = edge_iters[i]; auto src_label_ind = label_indices_[i]; - auto src_label = src_labels_[src_label_ind]; for (size_t j = 0; j < cur_edge_iters.size(); ++j) { auto& cur_iter = cur_edge_iters[j]; while (cur_iter.IsValid()) { auto dst_vid = cur_iter.GetDstId(); auto data = cur_iter.GetData(); - for (auto k = 0; k < repeat_array[cur_ind]; ++k) { + for (size_t k = 0; k < repeat_array[cur_ind]; ++k) { dst_eles.emplace_back(std::make_tuple(src_vid, dst_vid, data)); label_triplet_indices.emplace_back(sizes[src_label_ind] + j); } @@ -716,7 +715,7 @@ class UnTypedEdgeSet { std::vector>> get_edge_triplets() const { std::vector>> ret; - for (auto src_label_ind = 0; src_label_ind < src_labels_.size(); + for (size_t src_label_ind = 0; src_label_ind < src_labels_.size(); ++src_label_ind) { auto src_label = src_labels_[src_label_ind]; std::vector> tmp; diff --git a/flex/engines/http_server/workdir_manipulator.cc b/flex/engines/http_server/workdir_manipulator.cc index 5e70d76b56ab..e3e67444b377 100644 --- a/flex/engines/http_server/workdir_manipulator.cc +++ b/flex/engines/http_server/workdir_manipulator.cc @@ -16,6 +16,10 @@ #include "flex/engines/http_server/workdir_manipulator.h" #include "flex/engines/http_server/codegen_proxy.h" +#include // uuid class +#include // generators +#include // streaming operators etc. + // Write a macro to define the function, to check whether a filed presents in a // json object. #define CHECK_JSON_FIELD(json, field) \ @@ -668,8 +672,8 @@ gs::Result WorkDirManipulator::CreateFile( } // get the timestamp as the file name - auto time_stamp = std::to_string(gs::GetCurrentTimeStamp()); - auto file_name = GetUploadDir() + "/" + time_stamp; + boost::uuids::uuid uuid = boost::uuids::random_generator()(); + auto file_name = GetUploadDir() + "/" + boost::uuids::to_string(uuid); std::ofstream fout(file_name); if (!fout.is_open()) { return {gs::Status(gs::StatusCode::PermissionError, "Fail to open file")}; diff --git a/flex/interactive/sdk/python/gs_interactive/client/driver.py b/flex/interactive/sdk/python/gs_interactive/client/driver.py index d67d07b3addc..78539f157135 100644 --- a/flex/interactive/sdk/python/gs_interactive/client/driver.py +++ b/flex/interactive/sdk/python/gs_interactive/client/driver.py @@ -51,6 +51,14 @@ def __init__( self._gremlin_endpoint = gremlin_endpoint self._session = None self.init_host_and_port() + self._neo4j_driver = None + + def close(self): + if self._neo4j_driver is not None: + self._neo4j_driver.close() + + def __del(self): + self.close() def init_host_and_port(self): # prepend http:// to self._admin_endpoint @@ -118,7 +126,9 @@ def get_port(self) -> int: def getNeo4jSessionImpl(self, **config) -> Neo4jSession: if self._cypher_endpoint is None: self._cypher_endpoint = self.getNeo4jEndpoint() - return GraphDatabase.driver(self._cypher_endpoint, auth=None).session(**config) + if self._neo4j_driver is None: + self._neo4j_driver = GraphDatabase.driver(self._cypher_endpoint, auth=None) + return self._neo4j_driver.session(**config) def getNeo4jEndpoint(self) -> str: """ diff --git a/flex/interactive/sdk/python/gs_interactive/tests/__init__.py b/flex/interactive/sdk/python/gs_interactive/tests/__init__.py new file mode 100644 index 000000000000..9a43cecbaa96 --- /dev/null +++ b/flex/interactive/sdk/python/gs_interactive/tests/__init__.py @@ -0,0 +1,18 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# Copyright 2020 Alibaba Group Holding Limited. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + diff --git a/flex/interactive/sdk/python/gs_interactive/tests/conftest.py b/flex/interactive/sdk/python/gs_interactive/tests/conftest.py new file mode 100644 index 000000000000..b589b8b92b1f --- /dev/null +++ b/flex/interactive/sdk/python/gs_interactive/tests/conftest.py @@ -0,0 +1,444 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# Copyright 2020 Alibaba Group Holding Limited. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import time +from gs_interactive.client.driver import Driver +from gs_interactive.models import * +from gs_interactive.client.status import StatusCode +from gs_interactive.client.session import Session +from neo4j import Session as Neo4jSession +import pytest + +# get the directory of the current file +import os +import sys + +cur_dir=os.path.dirname(os.path.abspath(__file__)) +MODERN_GRAPH_DATA_DIR=os.path.abspath(os.path.join(cur_dir, "../../../../examples/modern_graph")) +print("MODERN_GRAPH_DATA_DIR: ", MODERN_GRAPH_DATA_DIR) + + +modern_graph_full = { + "name": "modern_graph", + "description": "This is a test graph", + "schema": { + "vertex_types": [ + { + "type_name": "person", + "properties": [ + { + "property_name": "id", + "property_type": {"primitive_type": "DT_SIGNED_INT64"}, + }, + { + "property_name": "name", + "property_type": {"string": {"long_text": ""}}, + }, + { + "property_name": "age", + "property_type": {"primitive_type": "DT_SIGNED_INT32"}, + }, + ], + "primary_keys": ["id"], + }, + { + "type_name": "software", + "properties": [ + { + "property_name": "id", + "property_type": {"primitive_type": "DT_SIGNED_INT64"}, + }, + { + "property_name": "name", + "property_type": {"string": {"long_text": ""}}, + }, + { + "property_name": "lang", + "property_type": {"string": {"long_text": ""}}, + }, + ], + "primary_keys": ["id"], + } + ], + "edge_types": [ + { + "type_name": "knows", + "vertex_type_pair_relations": [ + { + "source_vertex": "person", + "destination_vertex": "person", + "relation": "MANY_TO_MANY", + } + ], + "properties": [ + { + "property_name": "weight", + "property_type": {"primitive_type": "DT_DOUBLE"}, + } + ], + "primary_keys": [], + }, + { + "type_name": "created", + "vertex_type_pair_relations": [ + { + "source_vertex": "person", + "destination_vertex": "software", + "relation": "MANY_TO_MANY", + } + ], + "properties": [ + { + "property_name": "weight", + "property_type": {"primitive_type": "DT_DOUBLE"}, + } + ], + "primary_keys": [], + } + ], + }, +} + +modern_graph_vertex_only = { + "name": "modern_graph", + "description": "This is a test graph, only contains vertex", + "schema": { + "vertex_types": [ + { + "type_name": "person", + "properties": [ + { + "property_name": "id", + "property_type": {"primitive_type": "DT_SIGNED_INT64"}, + }, + { + "property_name": "name", + "property_type": {"string": {"long_text": ""}}, + }, + { + "property_name": "age", + "property_type": {"primitive_type": "DT_SIGNED_INT32"}, + }, + ], + "primary_keys": ["id"], + } + ], + "edge_types": [], + }, +} + +modern_graph_partial = { + "name": "modern_graph", + "description": "This is a test graph", + "schema": { + "vertex_types": [ + { + "type_name": "person", + "properties": [ + { + "property_name": "id", + "property_type": {"primitive_type": "DT_SIGNED_INT64"}, + }, + { + "property_name": "name", + "property_type": {"string": {"long_text": ""}}, + }, + { + "property_name": "age", + "property_type": {"primitive_type": "DT_SIGNED_INT32"}, + }, + ], + "primary_keys": ["id"], + } + ], + "edge_types": [ + { + "type_name": "knows", + "vertex_type_pair_relations": [ + { + "source_vertex": "person", + "destination_vertex": "person", + "relation": "MANY_TO_MANY", + } + ], + "properties": [ + { + "property_name": "weight", + "property_type": {"primitive_type": "DT_DOUBLE"}, + } + ], + "primary_keys": [], + } + ], + }, +} + + +modern_graph_full_import_config = { + "loading_config": { + "data_source": { + "scheme": "file", + "location": "@" + MODERN_GRAPH_DATA_DIR + }, + "import_option": "init", + "format": { + "type": "csv", + "metadata": { + "delimiter": "|", + } + } + }, + "vertex_mappings": [ + { + "type_name": "person", + "inputs": [ + "person.csv" + ], + }, + { + "type_name": "software", + "inputs": [ + "software.csv" + ], + } + ], + "edge_mappings": [ + { + "type_triplet": { + "edge": "knows", + "source_vertex": "person", + "destination_vertex": "person" + }, + "inputs": [ + "person_knows_person.csv" + ], + }, + { + "type_triplet": { + "edge": "created", + "source_vertex": "person", + "destination_vertex": "software" + }, + "inputs": [ + "person_created_software.csv" + ], + } + ] +} + + +modern_graph_partial_import_config = { + "loading_config": { + "data_source": { + "scheme": "file", + "location": "@" + MODERN_GRAPH_DATA_DIR + }, + "import_option": "init", + "format": { + "type": "csv", + "metadata": { + "delimiter": "|", + } + } + }, + "vertex_mappings": [ + { + "type_name": "person", + "inputs": [ + "person.csv" + ], + }, + ], + "edge_mappings": [ + { + "type_triplet": { + "edge": "knows", + "source_vertex": "person", + "destination_vertex": "person" + }, + "inputs": [ + "person_knows_person.csv" + ], + } + ] +} + +modern_graph_vertex_only_import_config = { + "loading_config": { + "data_source": { + "scheme": "file", + "location": "@" + MODERN_GRAPH_DATA_DIR + }, + "import_option": "init", + "format": { + "type": "csv", + "metadata": { + "delimiter": "|", + } + } + }, + "vertex_mappings": [ + { + "type_name": "person", + "inputs": [ + "person.csv" + ], + } + ] +} +@pytest.fixture(scope="module") +def interactive_driver(): + driver = Driver() + yield driver + driver.close() + + +@pytest.fixture(scope="module") +def interactive_session(interactive_driver): + yield interactive_driver.session() + +@pytest.fixture(scope="module") +def neo4j_session(interactive_driver): + _neo4j_sess = interactive_driver.getNeo4jSession() + yield _neo4j_sess + _neo4j_sess.close() + +@pytest.fixture(scope="module") +def create_modern_graph(interactive_session): + create_graph_request = CreateGraphRequest.from_dict(modern_graph_full) + resp = interactive_session.create_graph(create_graph_request) + assert resp.is_ok() + graph_id = resp.get_value().graph_id + yield graph_id + delete_running_graph(interactive_session, graph_id) + + +@pytest.fixture(scope="module") +def create_vertex_only_modern_graph(interactive_session): + create_graph_request = CreateGraphRequest.from_dict(modern_graph_vertex_only) + resp = interactive_session.create_graph(create_graph_request) + assert resp.is_ok() + graph_id = resp.get_value().graph_id + yield graph_id + delete_running_graph(interactive_session, graph_id) + +@pytest.fixture(scope="module") +def create_partial_modern_graph(interactive_session): + create_graph_request = CreateGraphRequest.from_dict(modern_graph_partial) + resp = interactive_session.create_graph(create_graph_request) + assert resp.is_ok() + graph_id = resp.get_value().graph_id + yield graph_id + delete_running_graph(interactive_session, graph_id) + +def wait_job_finish(sess : Session, job_id: str): + assert job_id is not None + while True: + resp = sess.get_job(job_id) + assert resp.is_ok() + status = resp.get_value().status + print("job status: ", status) + if status == "SUCCESS": + return True + elif status == "FAILED": + return False + else: + time.sleep(1) + +def import_data_to_vertex_only_modern_graph(sess: Session, graph_id: str): + schema_mapping = SchemaMapping.from_dict(modern_graph_vertex_only_import_config) + resp = sess.bulk_loading(graph_id, schema_mapping) + assert resp.is_ok() + job_id = resp.get_value().job_id + assert wait_job_finish(sess, job_id) + +def import_data_to_partial_modern_graph(sess: Session, graph_id: str): + schema_mapping = SchemaMapping.from_dict(modern_graph_partial_import_config) + resp = sess.bulk_loading(graph_id, schema_mapping) + assert resp.is_ok() + job_id = resp.get_value().job_id + assert wait_job_finish(sess, job_id) + +def import_data_to_full_modern_graph(sess: Session, graph_id: str): + schema_mapping = SchemaMapping.from_dict(modern_graph_full_import_config) + resp = sess.bulk_loading(graph_id, schema_mapping) + assert resp.is_ok() + job_id = resp.get_value().job_id + assert wait_job_finish(sess, job_id) + +def submit_query_via_neo4j_endpoint(neo4j_sess : Neo4jSession, graph_id: str, query: str): + query = "MATCH(n) return n" + result = neo4j_sess.run(query) + #check have 1 records, result 0 + result_cnt = 0 + for record in result: + print("record: ", record) + result_cnt += 1 + print("result count: ", result_cnt , " for query ", query) + + +def run_cypher_test_suite(neo4j_sess : Neo4jSession, graph_id: str, queries: list[str]): + for query in queries: + submit_query_via_neo4j_endpoint(neo4j_sess, graph_id, query) + +def call_procedure(neo4j_sess : Neo4jSession, graph_id: str, proc_name: str): + query = "CALL " + proc_name + "()" + result = neo4j_sess.run(query) + for record in result: + print(record) + + +def delete_running_graph(sess: Session, graph_id: str): + # restart the service on graph "1" + print("delete running graph {}", graph_id) + # first the the service status, to get the graph id + service_status = sess.get_service_status() + assert service_status.is_ok() + running_graph_id = service_status.get_value().graph.id + if running_graph_id is None: + print("No running graph") + return + if running_graph_id != graph_id: + print("The request graph is not running, safe to delete") + else: + resp = sess.start_service(StartServiceRequest(graph_id="1")) + assert resp.is_ok() + # drop the graph + resp = sess.delete_graph(graph_id) + assert resp.is_ok() + +def create_procedure(sess: Session, graph_id: str, name: str, query: str, description = "test proc"): + request = CreateProcedureRequest( + name=name, + description=description, + type="cypher", + query=query) + + resp = sess.create_procedure(graph_id, request) + if not resp.is_ok(): + print("Failed to create procedure: ", resp.get_status_message()) + raise Exception("Failed to create procedure, status: ", resp.get_status_message()) + proc_id = resp.get_value().procedure_id + return proc_id + + +def start_service_on_graph(interactive_session, graph_id : str): + resp = interactive_session.start_service(StartServiceRequest(graph_id=graph_id)) + assert resp.is_ok() + # wait one second to let compiler get the new graph + time.sleep(1) \ No newline at end of file diff --git a/flex/interactive/sdk/python/test/test_driver.py b/flex/interactive/sdk/python/gs_interactive/tests/test_driver.py similarity index 98% rename from flex/interactive/sdk/python/test/test_driver.py rename to flex/interactive/sdk/python/gs_interactive/tests/test_driver.py index 335075ce2601..7db0830e2a56 100644 --- a/flex/interactive/sdk/python/test/test_driver.py +++ b/flex/interactive/sdk/python/gs_interactive/tests/test_driver.py @@ -21,7 +21,8 @@ import time import unittest -import pytest +import sys +sys.path.append(os.path.join(os.path.dirname(__file__), "../../")) from gs_interactive.client.driver import Driver from gs_interactive.models import * @@ -355,9 +356,8 @@ def createCypherProcedure(self): def createCppProcedure(self): self._cpp_proc_name = "test_procedure_cpp" - # read strings from file ../../java/src/test/resources/sample_app.cc app_path = os.path.join( - os.path.dirname(__file__), "../../java/src/test/resources/sample_app.cc" + os.path.dirname(__file__), "../../../java/src/test/resources/sample_app.cc" ) if not os.path.exists(app_path): raise Exception("sample_app.cc not found") @@ -380,8 +380,6 @@ def restart(self): ) assert resp.is_ok() print("restart: ", resp.get_value()) - # wait 5 seconds - time.sleep(5) # get service status resp = self._sess.get_service_status() assert resp.is_ok() @@ -477,7 +475,6 @@ def restartOnNewGraph(self): ) assert start_service_res.is_ok() print("finish restartOnNewGraph") - time.sleep(5) def getStatistics(self): resp = self._sess.get_graph_statistics(self._graph_id) @@ -504,8 +501,6 @@ def callPrcedureWithServiceStop(self): start_service_request=StartServiceRequest(graph_id=self._graph_id) ) assert start_res.is_ok() - # wait 5 seconds - time.sleep(5) def callProcedureWithHttp(self): req = QueryRequest( diff --git a/flex/interactive/sdk/python/gs_interactive/tests/test_robustness.py b/flex/interactive/sdk/python/gs_interactive/tests/test_robustness.py new file mode 100644 index 000000000000..f3ba08fa7349 --- /dev/null +++ b/flex/interactive/sdk/python/gs_interactive/tests/test_robustness.py @@ -0,0 +1,119 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# Copyright 2020 Alibaba Group Holding Limited. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import os +import sys + +import pytest +sys.path.append(os.path.join(os.path.dirname(__file__), "../../")) + +from gs_interactive.client.driver import Driver +from gs_interactive.client.session import Session +from gs_interactive.models import * + + +from gs_interactive.tests.conftest import create_vertex_only_modern_graph, start_service_on_graph,interactive_driver +from gs_interactive.tests.conftest import create_procedure, delete_running_graph, create_modern_graph, create_partial_modern_graph,run_cypher_test_suite, call_procedure +from gs_interactive.tests.conftest import import_data_to_vertex_only_modern_graph, import_data_to_partial_modern_graph, import_data_to_full_modern_graph + + +cypher_queries = [ + "MATCH(n) return count(n)", + "MATCH(n) return n", + "MATCH(n) return n limit 10", + "MATCH()-[r]->() return count(r)", + "MATCH(a)-[b]->(c) return count(b)", + "MATCH(a)-[b]->(c) return b", + "MATCH(a)-[b]->(c) return c.id", +] + +def test_query_on_vertex_only_graph(interactive_session, neo4j_session, create_vertex_only_modern_graph): + """ + Test Query on a graph with only a vertex-only schema defined, no data is imported. + """ + print("[Query on vertex only graph]") + start_service_on_graph(interactive_session, create_vertex_only_modern_graph) + run_cypher_test_suite(neo4j_session, create_vertex_only_modern_graph, cypher_queries) + + start_service_on_graph(interactive_session,"1") + import_data_to_vertex_only_modern_graph(interactive_session, create_vertex_only_modern_graph) + run_cypher_test_suite(neo4j_session, create_vertex_only_modern_graph, cypher_queries) + +def test_query_on_partial_graph(interactive_session,neo4j_session, create_partial_modern_graph): + """ + Test Query on a graph with the partial schema of modern graph defined, no data is imported. + """ + print("[Query on partial graph]") + # start service on new graph + start_service_on_graph(interactive_session, create_partial_modern_graph) + # try to query on the graph + run_cypher_test_suite(neo4j_session, create_partial_modern_graph, cypher_queries) + start_service_on_graph(interactive_session,"1") + import_data_to_partial_modern_graph(interactive_session, create_partial_modern_graph) + run_cypher_test_suite(neo4j_session, create_partial_modern_graph, cypher_queries) + +def test_query_on_full_modern_graph(interactive_session, neo4j_session, create_modern_graph): + """ + Test Query on a graph with full schema of modern graph defined, no data is imported. + """ + print("[Query on full modern graph]") + start_service_on_graph(interactive_session,create_modern_graph) + # try to query on the graph + run_cypher_test_suite(neo4j_session, create_modern_graph, cypher_queries) + start_service_on_graph(interactive_session,"1") + import_data_to_full_modern_graph(interactive_session, create_modern_graph) + run_cypher_test_suite(neo4j_session, create_modern_graph, cypher_queries) + + +def test_service_switching(interactive_session,neo4j_session, create_modern_graph, create_vertex_only_modern_graph ): + """ + Create a procedure on graph a, and create graph b, and create a procedure with same procedure name. + Then restart graph on b, and query on graph a's procedure a. + """ + print("[Cross query]") + + # create procedure on graph_a_id + a_proc_id = create_procedure(interactive_session, create_modern_graph, "test_proc", "MATCH(n: software) return count(n);") + print("Procedure id: ", a_proc_id) + start_service_on_graph(interactive_session, create_modern_graph) + call_procedure(neo4j_session, create_modern_graph, a_proc_id) + + # create procedure on graph_b_id + b_proc_id = create_procedure(interactive_session, create_vertex_only_modern_graph, "test_proc", "MATCH(n: person) return count(n);") + start_service_on_graph(interactive_session, create_vertex_only_modern_graph) + call_procedure(neo4j_session, create_vertex_only_modern_graph, b_proc_id) + + +def test_procedure_creation(interactive_session, neo4j_session, create_modern_graph): + print("[Test procedure creation]") + + # create procedure with description contains spaces,',', and special characters '!','@','#','$','%','^','&','*','(',')' + a_proc_id = create_procedure(interactive_session, create_modern_graph, "test_proc_1", "MATCH(n: software) return count(n);", "This is a test procedure, with special characters: !@#$%^&*()") + print("Procedure id: ", a_proc_id) + start_service_on_graph(interactive_session, create_modern_graph) + call_procedure(neo4j_session, create_modern_graph, a_proc_id) + + # create procedure with name containing space, should fail, expect to raise exception + with pytest.raises(Exception): + create_procedure(interactive_session, create_modern_graph, "test proc", "MATCH(n: software) return count(n);") + + + # create procedure with invalid cypher query, should fail, expect to raise exception + with pytest.raises(Exception): + create_procedure(interactive_session, create_modern_graph, "test_proc2", "MATCH(n: IDONTKOWN) return count(n)") + diff --git a/flex/interactive/sdk/python/test/test_utils.py b/flex/interactive/sdk/python/gs_interactive/tests/test_utils.py similarity index 94% rename from flex/interactive/sdk/python/test/test_utils.py rename to flex/interactive/sdk/python/gs_interactive/tests/test_utils.py index f30abb0e4f22..5e926af93ca5 100644 --- a/flex/interactive/sdk/python/test/test_utils.py +++ b/flex/interactive/sdk/python/gs_interactive/tests/test_utils.py @@ -22,6 +22,9 @@ import pytest +import sys +sys.path.append(os.path.join(os.path.dirname(__file__), "../../")) + from gs_interactive.client.driver import Driver from gs_interactive.models import * from gs_interactive.client.utils import InputFormat, append_format_byte diff --git a/flex/tests/hqps/hqps_robust_test.sh b/flex/tests/hqps/hqps_robust_test.sh new file mode 100644 index 000000000000..b93e3a65f94d --- /dev/null +++ b/flex/tests/hqps/hqps_robust_test.sh @@ -0,0 +1,103 @@ +#!/bin/bash +# Copyright 2020 Alibaba Group Holding Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +set -e +SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd) +FLEX_HOME=${SCRIPT_DIR}/../../ +SERVER_BIN=${FLEX_HOME}/build/bin/interactive_server +ADMIN_PORT=7777 +QUERY_PORT=10000 +CYPHER_PORT=7687 + +if [ ! $# -eq 2 ]; then + echo "only receives: $# args, need 2" + echo "Usage: $0 " + exit 1 +fi + +INTERACTIVE_WORKSPACE=$1 +ENGINE_CONFIG_PATH=$2 + +if [ ! -d ${INTERACTIVE_WORKSPACE} ]; then + echo "INTERACTIVE_WORKSPACE: ${INTERACTIVE_WORKSPACE} not exists" + exit 1 +fi +if [ ! -f ${ENGINE_CONFIG_PATH} ]; then + echo "ENGINE_CONFIG: ${ENGINE_CONFIG_PATH} not exists" + exit 1 +fi + +RED='\033[0;31m' +GREEN='\033[0;32m' +NC='\033[0m' # No Color +err() { + echo -e "${RED}[$(date +'%Y-%m-%d %H:%M:%S')] -ERROR- $* ${NC}" >&2 +} + +info() { + echo -e "${GREEN}[$(date +'%Y-%m-%d %H:%M:%S')] -INFO- $* ${NC}" +} + +kill_service(){ + info "Kill Service first" + ps -ef | grep "interactive_server" | awk '{print $2}' | xargs kill -9 || true + ps -ef | grep "compiler" | awk '{print $2}' | xargs kill -9 || true + sleep 3 + # check if service is killed + info "Kill Service success" +} + +# kill service when exit +trap kill_service EXIT + +# start engine service +start_engine_service(){ + #check SERVER_BIN exists + if [ ! -f ${SERVER_BIN} ]; then + err "SERVER_BIN not found" + exit 1 + fi + + cmd="${SERVER_BIN} -c ${ENGINE_CONFIG_PATH} --enable-admin-service true " + cmd="${cmd} -w ${INTERACTIVE_WORKSPACE} --start-compiler true &" + + echo "Start engine service with command: ${cmd}" + eval ${cmd} + sleep 10 + #check interactive_server is running, if not, exit + ps -ef | grep "interactive_server" | grep -v grep + + info "Start engine service success" +} + + +run_robust_test(){ + pushd ${FLEX_HOME}/interactive/sdk/python/gs_interactive + cmd="python3 -m pytest -s tests/test_robustness.py" + echo "Run robust test with command: ${cmd}" + eval ${cmd} || (err "Run robust test failed"; exit 1) + info "Run robust test success" + popd +} + +kill_service +start_engine_service +export INTERACTIVE_ADMIN_ENDPOINT=http://localhost:${ADMIN_PORT} +export INTERACTIVE_STORED_PROC_ENDPOINT=http://localhost:${QUERY_PORT} +export INTERACTIVE_CYPHER_ENDPOINT=neo4j://localhost:${CYPHER_PORT} +export INTERACTIVE_GREMLIN_ENDPOINT=ws://localhost:${GREMLIN_PORT}/gremlin + +run_robust_test + +kill_service \ No newline at end of file diff --git a/flex/tests/hqps/hqps_sdk_test.sh b/flex/tests/hqps/hqps_sdk_test.sh index f9b1f6df01cc..892fc2798248 100644 --- a/flex/tests/hqps/hqps_sdk_test.sh +++ b/flex/tests/hqps/hqps_sdk_test.sh @@ -104,11 +104,11 @@ run_java_sdk_test(){ run_python_sdk_test(){ echo "run python sdk test" - pushd ${FLEX_HOME}/interactive/sdk/python/ - cmd="python3 -m pytest -s test/test_driver.py" + pushd ${FLEX_HOME}/interactive/sdk/python/gs_interactive + cmd="python3 -m pytest -s tests/test_driver.py" echo "Run python sdk test: ${cmd}" eval ${cmd} || (err "test_driver failed" && exit 1) - cmd="python3 -m pytest -s test/test_utils.py" + cmd="python3 -m pytest -s tests/test_utils.py" echo "Run python sdk test: ${cmd}" eval ${cmd} || (err "test_utils failed" && exit 1) info "Finish python sdk test" diff --git a/flex/tests/hqps/match_query.h b/flex/tests/hqps/match_query.h index 6517641c37ac..20db0ce5f537 100644 --- a/flex/tests/hqps/match_query.h +++ b/flex/tests/hqps/match_query.h @@ -16,6 +16,7 @@ #ifndef TESTS_HQPS_MATCH_QUERY_H_ #define TESTS_HQPS_MATCH_QUERY_H_ +#include "flex/engines/hqps_db/app/interactive_app_base.h" #include "flex/engines/hqps_db/core/sync_engine.h" #include "flex/utils/app_utils.h" @@ -1027,5 +1028,92 @@ class MatchQuery15 : public ReadAppBase { } }; +// Auto generated expression class definition +struct MatchQuery16expr0 { + public: + using result_t = bool; + static constexpr bool filter_null = true; + MatchQuery16expr0() {} + + inline auto operator()(LabelKey label) const { + return (label std::array{0, 2}); + } + + private: +}; + +// Auto generated query class definition +class MatchQuery16 : public ReadAppBase { + public: + using Engine = SyncEngine; + using label_id_t = typename gs::MutableCSRInterface::label_id_t; + using vertex_id_t = typename gs::MutableCSRInterface::vertex_id_t; + using gid_t = typename gs::MutableCSRInterface::gid_t; + // constructor + MatchQuery16() {} + // Query function for query class + results::CollectiveResults Query(gs::MutableCSRInterface& graph) const { + auto expr0 = gs::make_filter(MatchQuery16expr0(), + gs::PropertySelector("label")); + auto ctx0 = Engine::template ScanVertex( + graph, std::array{0, 2}, std::move(expr0)); + + auto edge_expand_opt0 = gs::make_edge_expand_multie_opt< + label_id_t, std::tuple, std::tuple, + std::tuple, std::tuple, + std::tuple, std::tuple>( + gs::Direction::Out, + std::array, 6>{ + std::array{0, 1, 0}, + std::array{2, 1, 5}, + std::array{0, 1, 2}, + std::array{0, 1, 3}, + std::array{2, 0, 4}, + std::array{0, 1, 1}}, + std::tuple{PropTupleArrayT>{}, + PropTupleArrayT>{"rating"}, + PropTupleArrayT>{}, + PropTupleArrayT>{}, + PropTupleArrayT>{}, + PropTupleArrayT>{}}); + auto ctx1 = + Engine::template EdgeExpandE( + graph, std::move(ctx0), std::move(edge_expand_opt0)); + + auto get_v_opt1 = make_getv_opt( + gs::VOpt::End, + std::array{(label_id_t) 0, (label_id_t) 1}); + auto ctx2 = Engine::template GetV( + graph, std::move(ctx1), std::move(get_v_opt1)); + auto ctx3 = Engine::Project( + graph, std::move(ctx2), + std::tuple{gs::make_mapper_with_variable( + gs::PropertySelector("")), + gs::make_mapper_with_variable( + gs::PropertySelector("")), + gs::make_mapper_with_variable( + gs::PropertySelector(""))}); + return Engine::Sink(graph, ctx3, std::array{0, 1, 2}); + } + + // Wrapper query function for query class + bool Query(const GraphDBSession& sess, Decoder& decoder, + Encoder& encoder) override { + // decoding params from decoder, and call real query func + + gs::MutableCSRInterface graph(sess); + auto res = Query(graph); + // dump results to string + std::string res_str = res.SerializeAsString(); + // encode results to encoder + if (!res_str.empty()) { + encoder.put_string_view(res_str); + } + return true; + } + // private members + private: +}; + } // namespace gs #endif // TESTS_HQPS_MATCH_QUERY_H_ \ No newline at end of file diff --git a/flex/tests/hqps/query_test.cc b/flex/tests/hqps/query_test.cc index 1b1c79a6f392..649024e23e59 100644 --- a/flex/tests/hqps/query_test.cc +++ b/flex/tests/hqps/query_test.cc @@ -241,5 +241,17 @@ int main(int argc, char** argv) { LOG(INFO) << "Finish MatchQuery15 test"; } + { + gs::MatchQuery16 query; + std::vector encoder_array; + gs::Encoder input_encoder(encoder_array); + std::vector output_array; + gs::Encoder output(output_array); + gs::Decoder input(encoder_array.data(), encoder_array.size()); + + query.Query(sess, input, output); + LOG(INFO) << "Finish MatchQuery16 test"; + } + LOG(INFO) << "Finish context test."; }