From ce0ae5e2ac11e95358e0a2762891f7c15c180ffa Mon Sep 17 00:00:00 2001 From: Zhang Lei Date: Fri, 20 Dec 2024 13:58:45 +0800 Subject: [PATCH] fix(interactive): Fix yaml node parsing (#4377) As titled. --- .../sdk/python/gs_interactive/client/result.py | 3 +-- .../sdk/python/gs_interactive/client/session.py | 12 ++++++------ .../sdk/python/gs_interactive/client/status.py | 8 ++++---- flex/tests/hqps/interactive_config_test.yaml | 3 ++- flex/utils/yaml_utils.cc | 14 +++++++++++--- .../compiler/conf/ir.compiler.properties | 5 ++++- .../graphscope/common/config/GraphConfig.java | 3 +++ .../graphscope/common/config/YamlConfigs.java | 3 +++ .../common/ir/meta/reader/HttpIrMetaReader.java | 14 ++++++++++---- 9 files changed, 44 insertions(+), 21 deletions(-) diff --git a/flex/interactive/sdk/python/gs_interactive/client/result.py b/flex/interactive/sdk/python/gs_interactive/client/result.py index f5aea2b6417f..79ec0d126bb5 100644 --- a/flex/interactive/sdk/python/gs_interactive/client/result.py +++ b/flex/interactive/sdk/python/gs_interactive/client/result.py @@ -20,9 +20,8 @@ from typing import TypeVar from gs_interactive.api_response import ApiResponse -from gs_interactive.exceptions import ApiException - from gs_interactive.client.status import Status +from gs_interactive.exceptions import ApiException # Define a generic type placeholder T = TypeVar("T") diff --git a/flex/interactive/sdk/python/gs_interactive/client/session.py b/flex/interactive/sdk/python/gs_interactive/client/session.py index 96da79ed7007..95ff753502a2 100644 --- a/flex/interactive/sdk/python/gs_interactive/client/session.py +++ b/flex/interactive/sdk/python/gs_interactive/client/session.py @@ -23,6 +23,11 @@ from typing import Optional from typing import Union +from pydantic import Field +from pydantic import StrictBytes +from pydantic import StrictStr +from typing_extensions import Annotated + from gs_interactive.api import AdminServiceGraphManagementApi from gs_interactive.api import AdminServiceJobManagementApi from gs_interactive.api import AdminServiceProcedureManagementApi @@ -32,18 +37,13 @@ from gs_interactive.api import QueryServiceApi from gs_interactive.api import UtilsApi from gs_interactive.api_client import ApiClient -from gs_interactive.configuration import Configuration -from pydantic import Field -from pydantic import StrictBytes -from pydantic import StrictStr -from typing_extensions import Annotated - from gs_interactive.client.generated.results_pb2 import CollectiveResults from gs_interactive.client.result import Result from gs_interactive.client.status import Status from gs_interactive.client.status import StatusCode from gs_interactive.client.utils import InputFormat from gs_interactive.client.utils import append_format_byte +from gs_interactive.configuration import Configuration from gs_interactive.models import CreateGraphRequest from gs_interactive.models import CreateGraphResponse from gs_interactive.models import CreateProcedureRequest diff --git a/flex/interactive/sdk/python/gs_interactive/client/status.py b/flex/interactive/sdk/python/gs_interactive/client/status.py index 21564752ecdf..26745c230f5f 100644 --- a/flex/interactive/sdk/python/gs_interactive/client/status.py +++ b/flex/interactive/sdk/python/gs_interactive/client/status.py @@ -16,16 +16,16 @@ # limitations under the License. # +from urllib3.exceptions import MaxRetryError +from urllib3.exceptions import ProtocolError + from gs_interactive.api_response import ApiResponse +from gs_interactive.client.generated.interactive_pb2 import Code as StatusCode from gs_interactive.exceptions import ApiException from gs_interactive.exceptions import BadRequestException from gs_interactive.exceptions import ForbiddenException from gs_interactive.exceptions import NotFoundException from gs_interactive.exceptions import ServiceException -from urllib3.exceptions import MaxRetryError -from urllib3.exceptions import ProtocolError - -from gs_interactive.client.generated.interactive_pb2 import Code as StatusCode from gs_interactive.models.api_response_with_code import APIResponseWithCode diff --git a/flex/tests/hqps/interactive_config_test.yaml b/flex/tests/hqps/interactive_config_test.yaml index 0e1f9f15ed21..6013b0079d90 100644 --- a/flex/tests/hqps/interactive_config_test.yaml +++ b/flex/tests/hqps/interactive_config_test.yaml @@ -5,7 +5,7 @@ compute_engine: type: hiactor workers: - localhost:10000 - thread_num_per_worker: 1 + thread_num_per_worker: 4 store: type: cpp-mcsr metadata_store: @@ -26,6 +26,7 @@ compiler: statistics: uri: http://localhost:7777/v1/graph/%s/statistics interval: 86400000 # ms + timeout: 1000 # ms endpoint: default_listen_address: localhost bolt_connector: diff --git a/flex/utils/yaml_utils.cc b/flex/utils/yaml_utils.cc index 775c4245ea2a..3c36a0e85535 100644 --- a/flex/utils/yaml_utils.cc +++ b/flex/utils/yaml_utils.cc @@ -52,12 +52,20 @@ void convert_yaml_node_to_json(const YAML::Node& node, json.SetInt(node.as()); } catch (const YAML::BadConversion& e) { try { - json.SetDouble(node.as()); + json.SetInt64(node.as()); } catch (const YAML::BadConversion& e) { try { - json.SetBool(node.as()); + json.SetUint64(node.as()); } catch (const YAML::BadConversion& e) { - json.SetString(node.as().c_str(), allocator); + try { + json.SetDouble(node.as()); + } catch (const YAML::BadConversion& e) { + try { + json.SetBool(node.as()); + } catch (const YAML::BadConversion& e) { + json.SetString(node.as().c_str(), allocator); + } + } } } } diff --git a/interactive_engine/compiler/conf/ir.compiler.properties b/interactive_engine/compiler/conf/ir.compiler.properties index 992f828a2b2e..2ab6058f579c 100644 --- a/interactive_engine/compiler/conf/ir.compiler.properties +++ b/interactive_engine/compiler/conf/ir.compiler.properties @@ -68,5 +68,8 @@ calcite.default.charset: UTF-8 # set the interval in milliseconds to fetch graph schema # graph.meta.schema.fetch.interval.ms: 1000 -# set the timeout in milliseconds to fetch graph statistics +# set the interval in milliseconds to fetch graph statistics # graph.meta.statistics.fetch.interval.ms: 86400000l + +# set the timeout in milliseconds to fetch graph statistics +# graph.meta.fetch.timeout.ms: 1000 diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/config/GraphConfig.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/config/GraphConfig.java index 9ab065f61366..74a7086599bc 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/config/GraphConfig.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/config/GraphConfig.java @@ -31,6 +31,9 @@ public class GraphConfig { public static final Config GRAPH_META_STATISTICS_FETCH_INTERVAL_MS = Config.longConfig("graph.meta.statistics.fetch.interval.ms", 24 * 3600 * 1000l); + public static final Config GRAPH_META_FETCH_TIMEOUT_MS = + Config.longConfig("graph.meta.fetch.timeout.ms", 1000); + // an intermediate solution to support foreign key, will be integrated into schema public static final Config GRAPH_FOREIGN_KEY_URI = Config.stringConfig("graph.foreign.key", ""); diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/config/YamlConfigs.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/config/YamlConfigs.java index e55682a86cdb..ec35991327ad 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/config/YamlConfigs.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/config/YamlConfigs.java @@ -72,6 +72,9 @@ public class YamlConfigs extends Configs { "graph.meta.statistics.fetch.interval.ms", (Configs configs) -> configs.get("compiler.meta.reader.statistics.interval")) + .put( + "graph.meta.fetch.timeout.ms", + (Configs configs) -> configs.get("compiler.meta.reader.timeout")) .put( "graph.store", (Configs configs) -> { diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/reader/HttpIrMetaReader.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/reader/HttpIrMetaReader.java index 2e12792d7bf1..ba6cfb089d87 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/reader/HttpIrMetaReader.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/reader/HttpIrMetaReader.java @@ -57,7 +57,9 @@ public HttpIrMetaReader(Configs configs) { public IrMeta readMeta() throws IOException { try { HttpResponse response = - sendRequest(GraphConfig.GRAPH_META_SCHEMA_URI.get(configs)); + sendRequest( + GraphConfig.GRAPH_META_SCHEMA_URI.get(configs), + GraphConfig.GRAPH_META_FETCH_TIMEOUT_MS.get(configs)); String res = response.body(); Preconditions.checkArgument( response.statusCode() == 200, @@ -91,7 +93,8 @@ public IrGraphStatistics readStats(GraphId graphId) throws IOException { sendRequest( String.format( GraphConfig.GRAPH_META_STATISTICS_URI.get(configs), - graphId.getId())); + graphId.getId()), + GraphConfig.GRAPH_META_FETCH_TIMEOUT_MS.get(configs)); String res = response.body(); Preconditions.checkArgument( response.statusCode() == 200, @@ -109,7 +112,9 @@ public IrGraphStatistics readStats(GraphId graphId) throws IOException { public boolean syncStatsEnabled(GraphId graphId) throws IOException { try { HttpResponse response = - sendRequest(GraphConfig.GRAPH_META_SCHEMA_URI.get(configs)); + sendRequest( + GraphConfig.GRAPH_META_SCHEMA_URI.get(configs), + GraphConfig.GRAPH_META_FETCH_TIMEOUT_MS.get(configs)); String res = response.body(); Preconditions.checkArgument( response.statusCode() == 200, @@ -122,13 +127,14 @@ public boolean syncStatsEnabled(GraphId graphId) throws IOException { } } - private HttpResponse sendRequest(String requestUri) + private HttpResponse sendRequest(String requestUri, long timeOut) throws IOException, InterruptedException { HttpRequest request = HttpRequest.newBuilder() .uri(URI.create(requestUri)) .headers(CONTENT_TYPE, APPLICATION_JSON) .GET() + .timeout(java.time.Duration.ofMillis(timeOut)) .build(); return httpClient.send(request, HttpResponse.BodyHandlers.ofString()); }