chore: Fix the CI by skip loading unused images (#3283)

Fixes #3281

.github/workflows/k8s-ci.yml

@@ -375,7 +375,11 @@ jobs:
         run: |
           cd ${GITHUB_WORKSPACE}/k8s
           # Use a dummy builder image (tag=ci, which is actually a busybox) to reduce time and space to pull the builder
-          make graphscope CI=true VERSION=${SHORT_SHA} REGISTRY=registry-vpc.cn-hongkong.aliyuncs.com BUILDER_VERSION=ci
+          make analytical CI=true VERSION=${SHORT_SHA} REGISTRY=registry-vpc.cn-hongkong.aliyuncs.com BUILDER_VERSION=ci
+          make interactive CI=true VERSION=${SHORT_SHA} REGISTRY=registry-vpc.cn-hongkong.aliyuncs.com BUILDER_VERSION=ci
+          make coordinator CI=true VERSION=${SHORT_SHA} REGISTRY=registry-vpc.cn-hongkong.aliyuncs.com BUILDER_VERSION=ci
+
+          docker images
 
       - name: Install Python dependencies
         run: |
@@ -428,8 +432,8 @@ jobs:
           echo "Loaded frontend"
           minikube image load graphscope/interactive-executor:${SHORT_SHA}
           echo "Loaded executor"
-          minikube image load graphscope/learning:${SHORT_SHA}
-          echo "loaded learning"
+          # minikube image load graphscope/learning:${SHORT_SHA}
+          # echo "loaded learning"
 
       - uses: dashanji/kubernetes-log-export-action@v5
         env:
@@ -566,7 +570,11 @@ jobs:
         run: |
           cd ${GITHUB_WORKSPACE}/k8s
           # Use a dummy builder image (tag=ci, which is actually a busybox) to reduce time and space to pull the builder
-          make graphscope CI=true VERSION=${SHORT_SHA} REGISTRY=registry-vpc.cn-hongkong.aliyuncs.com BUILDER_VERSION=ci
+          make analytical CI=true VERSION=${SHORT_SHA} REGISTRY=registry-vpc.cn-hongkong.aliyuncs.com BUILDER_VERSION=ci
+          make interactive CI=true VERSION=${SHORT_SHA} REGISTRY=registry-vpc.cn-hongkong.aliyuncs.com BUILDER_VERSION=ci
+          make coordinator CI=true VERSION=${SHORT_SHA} REGISTRY=registry-vpc.cn-hongkong.aliyuncs.com BUILDER_VERSION=ci
+
+          docker images
 
       - name: Install Python dependencies
         run: |
@@ -581,7 +589,7 @@ jobs:
         run: |
           export GS_TEST_DIR=${GITHUB_WORKSPACE}/interactive_engine/tests/src/main/resources
           minikube start --base-image='registry-vpc.cn-hongkong.aliyuncs.com/graphscope/kicbase:v0.0.30' \
-                         --cpus='12' --memory='32000mb' --disk-size='40000mb' \
+                         --cpus='12' --memory='32000mb' --disk-size='60000mb' \
                          --mount=true --mount-string="${GS_TEST_DIR}:${GS_TEST_DIR}"
 
           export GS_REGISTRY=""
@@ -590,7 +598,6 @@ jobs:
           minikube image load graphscope/analytical:${SHORT_SHA}
           minikube image load graphscope/interactive-frontend:${SHORT_SHA}
           minikube image load graphscope/interactive-executor:${SHORT_SHA}
-          minikube image load graphscope/learning:${SHORT_SHA}
 
           export PYTHONPATH=${GITHUB_WORKSPACE}/python:${PYTHONPATH}
           cd ${GITHUB_WORKSPACE}/interactive_engine && mvn clean install --quiet -DskipTests -Drust.compile.skip=true -P graphscope
@@ -683,7 +690,7 @@ jobs:
 
           # prepare minikube cluster
           minikube start --base-image='registry-vpc.cn-hongkong.aliyuncs.com/graphscope/kicbase:v0.0.30' \
-          --cpus='12' --memory='32000mb' --disk-size='40000mb' \
+          --cpus='12' --memory='32000mb' --disk-size='60000mb' \
           --mount=true --mount-string="${STORE_DATA_PATH}:${STORE_DATA_PATH}"
           minikube image load ${GIE_IMAGE}:${SHORT_SHA}
 

charts/graphscope/values.yaml

@@ -65,7 +65,7 @@ engines:
   # Available options of log_level: INFO, DEBUG
   log_level: INFO
 
-  enabled_engines: "analytical,interactive,learning"
+  enabled_engines: "analytical,interactive"
   gae:
     resources:
       requests:

interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/SnapshotCache.java

@@ -92,7 +92,7 @@ public synchronized long advanceQuerySnapshotId(long snapshotId, GraphDef graphD
                         || graphDef.getSchemaVersion() > oldGraphDef.getVersion())) {
             newSnapshotInfoBuilder.setGraphDef(graphDef);
             logger.info("schema updated. schema version [" + graphDef.getVersion() + "]");
-            logger.info(graphDef.toProto().toString());
+            logger.debug(graphDef.toProto().toString());
         }
         this.snapshotWithSchemaRef.set(newSnapshotInfoBuilder.build());
         logger.debug("snapshotId update to [" + snapshotId + "]");

interactive_engine/tests/function_test.sh

@@ -28,7 +28,7 @@ function _start {
     curl -XPOST http://localhost:${_port} -d 'graphscope.set_option(show_log=True)'
     curl -XPOST http://localhost:${_port} -d 'from graphscope.framework.loader import Loader'
     curl -XPOST http://localhost:${_port} -d 'from graphscope.dataset import load_modern_graph'
-    curl_sess="curl -XPOST http://localhost:${_port} -d 'session = graphscope.session(num_workers=${workers}, k8s_volumes={\"data\": {\"type\": \"hostPath\", \"field\": {\"path\": \"${GS_TEST_DIR}\", \"type\": \"Directory\"}, \"mounts\": {\"mountPath\": \"/testingdata\"}}}, k8s_coordinator_cpu=1.0, k8s_coordinator_mem='\''4Gi'\'', k8s_vineyard_cpu=1.0, k8s_vineyard_mem='\''4Gi'\'', vineyard_shared_mem='\''4Gi'\'', k8s_engine_cpu=1.0, k8s_engine_mem='\''4Gi'\'', k8s_image_registry='\''${GS_REGISTRY}'\'', k8s_image_tag='\''${GS_TAG}'\'')' --write-out %{http_code} --silent --output ./curl.tmp"
+    curl_sess="curl -XPOST http://localhost:${_port} -d 'session = graphscope.session(num_workers=${workers}, k8s_volumes={\"data\": {\"type\": \"hostPath\", \"field\": {\"path\": \"${GS_TEST_DIR}\", \"type\": \"Directory\"}, \"mounts\": {\"mountPath\": \"/testingdata\"}}}, k8s_coordinator_cpu=1.0, k8s_coordinator_mem='\''4Gi'\'', k8s_vineyard_cpu=1.0, k8s_vineyard_mem='\''4Gi'\'', enabled_engines='\''gae,gie'\'', k8s_engine_cpu=1.0, k8s_engine_mem='\''4Gi'\'', k8s_image_registry='\''${GS_REGISTRY}'\'', k8s_image_tag='\''${GS_TAG}'\'')' --write-out %{http_code} --silent --output ./curl.tmp"
 
     echo $curl_sess
     code=$(sh -c "$curl_sess")

python/graphscope/config.py

@@ -121,15 +121,15 @@ class DatasetConfig:
 class EngineConfig:
     """Engine configuration"""
 
-    enabled_engines: str = "gae,gie,gle"  # A set of engines to enable.
+    enabled_engines: str = "gae,gie"  # A set of engines to enable.
     # Node selector for engine pods, default is None.
     node_selector: Union[str, None] = None
 
-    enable_gae: bool = True  # Enable or disable analytical engine.
+    enable_gae: bool = False  # Enable or disable analytical engine.
     # Enable or disable analytical engine with java support.
     enable_gae_java: bool = False
-    enable_gie: bool = True  # Enable or disable interactive engine.
-    enable_gle: bool = True  # Enable or disable learning engine.
+    enable_gie: bool = False  # Enable or disable interactive engine.
+    enable_gle: bool = False  # Enable or disable learning engine.
 
     preemptive: bool = True
 

python/graphscope/tests/kubernetes/test_demo_script.py

@@ -822,6 +822,6 @@ def test_modualize():
         num_workers=1,
         k8s_image_registry=get_gs_registry_on_ci_env(),
         k8s_image_tag=get_gs_tag_on_ci_env(),
-        enabled_engines="interactive",
+        enabled_engines="analytical",
     )
     sess.close()

python/setup.py

@@ -136,9 +136,10 @@ def run(self):
 
 class BuildGLTorchExt(torch.utils.cpp_extension.BuildExtension if torch else build_ext):
     def run(self):
-        assert (
-            torch
-        ), "Building graphlearn-torch extension requires installing pytorch first. Let WITH_GLTORCH=OFF if you don't need it."
+        if torch is None:
+            print("Building graphlearn-torch extension requires pytorch")
+            print("Set WITH_GLTORCH=OFF if you don't need it.")
+            return
         self.extensions = [
             ext
             for ext in self.extensions