From 044c9ef2cf13d9575174e729154697329c4e12f0 Mon Sep 17 00:00:00 2001 From: Siyuan Zhang Date: Thu, 28 Nov 2024 10:56:50 +0800 Subject: [PATCH] fix(python): refresh channel incase the service becomes available after initial heart beat (#4338) also bump up rustup toolchain version to 1.76.0 to avoid some rust packages compilation failure. --- k8s/dockerfiles/graphscope-store.Dockerfile | 1 + k8s/dockerfiles/interactive.Dockerfile | 1 + k8s/internal/Makefile | 1 + python/graphscope/client/rpc.py | 13 ++++++++++--- 4 files changed, 13 insertions(+), 3 deletions(-) diff --git a/k8s/dockerfiles/graphscope-store.Dockerfile b/k8s/dockerfiles/graphscope-store.Dockerfile index 1d8372eac547..c079b2aac43b 100644 --- a/k8s/dockerfiles/graphscope-store.Dockerfile +++ b/k8s/dockerfiles/graphscope-store.Dockerfile @@ -14,6 +14,7 @@ COPY --chown=graphscope:graphscope . /home/graphscope/graphscope COPY --chown=graphscope:graphscope ./interactive_engine/assembly/src/conf/maven.settings.xml /home/graphscope/.m2/settings.xml USER graphscope +RUN rustup toolchain install 1.76.0 && rustup default 1.76.0 RUN cd /home/graphscope/graphscope \ && . ~/.graphscope_env \ diff --git a/k8s/dockerfiles/interactive.Dockerfile b/k8s/dockerfiles/interactive.Dockerfile index 4d1881e84ef9..4a818174e5f3 100644 --- a/k8s/dockerfiles/interactive.Dockerfile +++ b/k8s/dockerfiles/interactive.Dockerfile @@ -18,6 +18,7 @@ RUN cd /home/graphscope/GraphScope/ && \ else \ mkdir /home/graphscope/install; \ . /home/graphscope/.graphscope_env; \ + rustup toolchain install 1.76.0 && rustup default 1.76.0; \ make interactive-install BUILD_TYPE="$profile" INSTALL_PREFIX=/home/graphscope/install; \ fi diff --git a/k8s/internal/Makefile b/k8s/internal/Makefile index cd7fb8dbbe8b..81736ddf4b52 100644 --- a/k8s/internal/Makefile +++ b/k8s/internal/Makefile @@ -110,6 +110,7 @@ graphscope-manylinux2014-py3-nodocker: sudo sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-* && \ sudo yum install java-11-openjdk-devel -y && \ sudo yum remove java-1.8.0-openjdk-devel java-1.8.0-openjdk java-1.8.0-openjdk-headless -y && \ + rustup toolchain install 1.76.0 && rustup default 1.76.0 && \ cd $(WORKING_DIR)/../.. && \ if [[ "${PLATFORM}" == "aarch64" ]]; then \ export AUDITWHEEL_PLAT=manylinux2014_${PLATFORM}; \ diff --git a/python/graphscope/client/rpc.py b/python/graphscope/client/rpc.py index 58f5c131a5fa..6bfa8297aaad 100644 --- a/python/graphscope/client/rpc.py +++ b/python/graphscope/client/rpc.py @@ -41,19 +41,23 @@ class GRPCClient(object): def __init__(self, launcher, endpoint, reconnect=False): """Connect to GRAPE engine at the given :code:`endpoint`.""" # create the gRPC stub - options = [ + self._options = [ ("grpc.max_send_message_length", GS_GRPC_MAX_MESSAGE_LENGTH), ("grpc.max_receive_message_length", GS_GRPC_MAX_MESSAGE_LENGTH), ("grpc.max_metadata_size", GS_GRPC_MAX_MESSAGE_LENGTH), ] + self._endpoint = endpoint self._launcher = launcher self._grpc_utils = GRPCUtils() - self._channel = grpc.insecure_channel(endpoint, options=options) - self._stub = coordinator_service_pb2_grpc.CoordinatorServiceStub(self._channel) + self._stub = self._get_stub() self._session_id = None self._logs_fetching_thread = None self._reconnect = reconnect + def _get_stub(self): + channel = grpc.insecure_channel(self._endpoint, options=self._options) + return coordinator_service_pb2_grpc.CoordinatorServiceStub(channel) + def waiting_service_ready(self, timeout_seconds=60): begin_time = time.time() request = message_pb2.HeartBeatRequest() @@ -76,6 +80,9 @@ def waiting_service_ready(self, timeout_seconds=60): logger.warning("Heart beat analytical engine failed, %s", msg) if time.time() - begin_time >= timeout_seconds: raise ConnectionError(f"Connect coordinator timeout, {msg}") + # refresh the channel incase the server became available + if e.code() == grpc.StatusCode.UNAVAILABLE: + self._stub = self._get_stub() time.sleep(1) def connect(self, cleanup_instance=True, dangling_timeout_seconds=60):