diff --git a/charts/graphscope-store/values.yaml b/charts/graphscope-store/values.yaml index 883131b45be2..063a81baa478 100644 --- a/charts/graphscope-store/values.yaml +++ b/charts/graphscope-store/values.yaml @@ -497,7 +497,7 @@ javaOpts: "" graphName: "graphscope" -rpcMaxBytesMb: 4 +rpcMaxBytesMb: 20 engineType: "gaia" diff --git a/docs/storage_engine/groot.md b/docs/storage_engine/groot.md index ac2a5750c002..60ac625dc89c 100644 --- a/docs/storage_engine/groot.md +++ b/docs/storage_engine/groot.md @@ -430,7 +430,7 @@ skip.header=true load.after.build=true # This is not required when load.after.build=true # hadoop.endpoint=127.0.0.1:9000 -# ``` +``` Details of the parameters are listed below: diff --git a/interactive_engine/data-load-tool/src/main/java/com/alibaba/graphscope/groot/dataload/databuild/OfflineBuildOdps.java b/interactive_engine/data-load-tool/src/main/java/com/alibaba/graphscope/groot/dataload/databuild/OfflineBuildOdps.java index d8663df4e8d2..79a98ba8722a 100644 --- a/interactive_engine/data-load-tool/src/main/java/com/alibaba/graphscope/groot/dataload/databuild/OfflineBuildOdps.java +++ b/interactive_engine/data-load-tool/src/main/java/com/alibaba/graphscope/groot/dataload/databuild/OfflineBuildOdps.java @@ -224,6 +224,11 @@ public static void main(String[] args) throws IOException { client.clearIngest(uniquePath); throw ex; } + try { + client.clearIngest(uniquePath); + } catch (Exception ex) { + logger.error("Clear ingest failed, ignored."); + } } } diff --git a/interactive_engine/groot-client/pom.xml b/interactive_engine/groot-client/pom.xml index 01d2b7c3f0dd..9b46320d90ed 100644 --- a/interactive_engine/groot-client/pom.xml +++ b/interactive_engine/groot-client/pom.xml @@ -181,7 +181,6 @@ com.alibaba.graphscope.groot.sdk.example.RealtimeWrite com.alibaba.graphscope.groot.sdk.example.RealtimeWrite - 1.11 -1 diff --git a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/coordinator/SchemaManager.java b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/coordinator/SchemaManager.java index b183ca71736b..9632d482092d 100644 --- a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/coordinator/SchemaManager.java +++ b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/coordinator/SchemaManager.java @@ -106,6 +106,7 @@ private void recoverInternal() throws IOException, ExecutionException, Interrupt this.graphDefRef.set(graphDef); this.ready = true; logger.info("SchemaManager recovered. version [" + graphDef.getVersion() + "]"); + logger.info(graphDef.toProto().toString()); } public void stop() { diff --git a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/frontend/ClientService.java b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/frontend/ClientService.java index 7c20e804f23f..de816a69cff4 100644 --- a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/frontend/ClientService.java +++ b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/frontend/ClientService.java @@ -380,9 +380,11 @@ public void clearIngest( int storeCount = this.metaService.getStoreCount(); AtomicInteger counter = new AtomicInteger(storeCount); AtomicBoolean finished = new AtomicBoolean(false); + String dataPath = request.getDataPath(); for (int i = 0; i < storeCount; i++) { this.storeIngestor.clearIngest( i, + dataPath, new CompletionCallback() { @Override public void onCompleted(Void res) { diff --git a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/frontend/StoreIngestClient.java b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/frontend/StoreIngestClient.java index b1daa9d9f1de..9a8f3e1b70bb 100644 --- a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/frontend/StoreIngestClient.java +++ b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/frontend/StoreIngestClient.java @@ -58,9 +58,9 @@ public void onCompleted() {} }); } - public void storeClearIngest(CompletionCallback callback) { + public void storeClearIngest(String path, CompletionCallback callback) { this.stub.storeClearIngest( - StoreClearIngestRequest.newBuilder().build(), + StoreClearIngestRequest.newBuilder().setDataPath(path).build(), new StreamObserver() { @Override public void onNext(StoreClearIngestResponse storeClearIngestResponse) { diff --git a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/frontend/StoreIngestClients.java b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/frontend/StoreIngestClients.java index 2bb4b5d8f208..ea2d2008b291 100644 --- a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/frontend/StoreIngestClients.java +++ b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/frontend/StoreIngestClients.java @@ -48,7 +48,7 @@ public void ingest( } @Override - public void clearIngest(int storeId, CompletionCallback callback) { - this.getClient(storeId).storeClearIngest(callback); + public void clearIngest(int storeId, String path, CompletionCallback callback) { + this.getClient(storeId).storeClearIngest(path, callback); } } diff --git a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/frontend/StoreIngestor.java b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/frontend/StoreIngestor.java index 8a99e8dd6b9d..285ca06d3620 100644 --- a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/frontend/StoreIngestor.java +++ b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/frontend/StoreIngestor.java @@ -26,5 +26,5 @@ void ingest( Map config, CompletionCallback callback); - void clearIngest(int storeId, CompletionCallback callback); + void clearIngest(int storeId, String path, CompletionCallback callback); } diff --git a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/store/StoreService.java b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/store/StoreService.java index 475a62a713cd..6e9c2d6e61ae 100644 --- a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/store/StoreService.java +++ b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/store/StoreService.java @@ -224,9 +224,11 @@ public boolean batchWrite(StoreDataBatch storeDataBatch) long snapshotId = storeDataBatch.getSnapshotId(); List> dataBatch = storeDataBatch.getDataBatch(); AtomicBoolean hasDdl = new AtomicBoolean(false); + int maxRetry = 10; for (Map partitionToBatch : dataBatch) { - while (!shouldStop && partitionToBatch.size() != 0) { + while (!shouldStop && partitionToBatch.size() != 0 && maxRetry > 0) { partitionToBatch = writeStore(snapshotId, partitionToBatch, hasDdl); + maxRetry--; } } return hasDdl.get(); @@ -364,12 +366,11 @@ private void ingestDataInternal( } public void clearIngest(String dataPath) throws IOException { - String dataRoot = StoreConfig.STORE_DATA_PATH.get(storeConfigs); if (dataPath == null || dataPath.isEmpty()) { logger.warn("Must set a sub-path for clearing."); return; } - + String dataRoot = StoreConfig.STORE_DATA_PATH.get(storeConfigs); Path downloadPath = Paths.get(dataRoot, "download", dataPath); try { logger.info("Clearing directory {}", downloadPath); diff --git a/interactive_engine/groot-server/src/main/java/com/alibaba/graphscope/groot/servers/ir/IrServiceProducer.java b/interactive_engine/groot-server/src/main/java/com/alibaba/graphscope/groot/servers/ir/IrServiceProducer.java index bf8dde2f6005..c7d7ac83fc8a 100644 --- a/interactive_engine/groot-server/src/main/java/com/alibaba/graphscope/groot/servers/ir/IrServiceProducer.java +++ b/interactive_engine/groot-server/src/main/java/com/alibaba/graphscope/groot/servers/ir/IrServiceProducer.java @@ -56,7 +56,7 @@ public AbstractService makeGraphService( ChannelFetcher channelFetcher = new RpcChannelManagerFetcher(channelManager, executorCount, RoleType.GAIA_RPC); com.alibaba.graphscope.common.config.Configs irConfigs = getConfigs(); - logger.info("IR Config: {}", irConfigs); + logger.info("IR configs: {}", irConfigs); IrMetaFetcher irMetaFetcher = new GrootMetaFetcher(schemaFetcher); SnapshotUpdateCommitter updateCommitter = new SnapshotUpdateCommitter(channelManager); int frontendId = CommonConfig.NODE_IDX.get(configs); diff --git a/interactive_engine/pom.xml b/interactive_engine/pom.xml index 61733289326d..f20ecc0a98a1 100644 --- a/interactive_engine/pom.xml +++ b/interactive_engine/pom.xml @@ -114,6 +114,13 @@ + + maven-compiler-plugin + + 1.8 + 1.8 + + org.apache.maven.plugins maven-gpg-plugin @@ -216,6 +223,9 @@ 3.0.0 3.2.2 2.7 + 1.8 + 1.8 + 4.4.0