From 201970c17b342263176ec3c0d73f1a8015b3a883 Mon Sep 17 00:00:00 2001 From: Thang PHAM Date: Mon, 2 Dec 2024 12:56:48 +0100 Subject: [PATCH 1/5] Computing debug mode Signed-off-by: Thang PHAM --- .../AbstractComputationRunContext.java | 3 ++ .../service/AbstractResultContext.java | 7 +-- .../service/AbstractWorkerService.java | 43 ++++++++++++++++++- .../computation/service/ExecutionService.java | 5 --- 4 files changed, 48 insertions(+), 10 deletions(-) diff --git a/src/main/java/com/powsybl/ws/commons/computation/service/AbstractComputationRunContext.java b/src/main/java/com/powsybl/ws/commons/computation/service/AbstractComputationRunContext.java index 6962d4f..a224508 100644 --- a/src/main/java/com/powsybl/ws/commons/computation/service/AbstractComputationRunContext.java +++ b/src/main/java/com/powsybl/ws/commons/computation/service/AbstractComputationRunContext.java @@ -7,6 +7,7 @@ package com.powsybl.ws.commons.computation.service; import com.powsybl.commons.report.ReportNode; +import com.powsybl.computation.ComputationManager; import com.powsybl.iidm.network.Network; import com.powsybl.ws.commons.computation.dto.ReportInfos; import lombok.Getter; @@ -30,6 +31,8 @@ public abstract class AbstractComputationRunContext

{ private P parameters; private ReportNode reportNode; private Network network; + private boolean debug; + private ComputationManager computationManager; protected AbstractComputationRunContext(UUID networkUuid, String variantId, String receiver, ReportInfos reportInfos, String userId, String provider, P parameters) { diff --git a/src/main/java/com/powsybl/ws/commons/computation/service/AbstractResultContext.java b/src/main/java/com/powsybl/ws/commons/computation/service/AbstractResultContext.java index 7ef4170..84ccc68 100644 --- a/src/main/java/com/powsybl/ws/commons/computation/service/AbstractResultContext.java +++ b/src/main/java/com/powsybl/ws/commons/computation/service/AbstractResultContext.java @@ -17,9 +17,7 @@ import java.util.Objects; import java.util.UUID; -import static com.powsybl.ws.commons.computation.service.NotificationService.HEADER_PROVIDER; -import static com.powsybl.ws.commons.computation.service.NotificationService.HEADER_RECEIVER; -import static com.powsybl.ws.commons.computation.service.NotificationService.HEADER_USER_ID; +import static com.powsybl.ws.commons.computation.service.NotificationService.*; /** * @author Mathieu Deharbe @@ -42,6 +40,8 @@ public abstract class AbstractResultContext toMessage(ObjectMapper objectMapper) { .setHeader(REPORT_UUID_HEADER, runContext.getReportInfos().reportUuid() != null ? runContext.getReportInfos().reportUuid().toString() : null) .setHeader(REPORTER_ID_HEADER, runContext.getReportInfos().reporterId()) .setHeader(REPORT_TYPE_HEADER, runContext.getReportInfos().computationType()) + .setHeader(DEBUG_HEADER, runContext.isDebug()) .copyHeaders(getSpecificMsgHeaders(objectMapper)) .build(); } diff --git a/src/main/java/com/powsybl/ws/commons/computation/service/AbstractWorkerService.java b/src/main/java/com/powsybl/ws/commons/computation/service/AbstractWorkerService.java index 4787ad4..4359b6c 100644 --- a/src/main/java/com/powsybl/ws/commons/computation/service/AbstractWorkerService.java +++ b/src/main/java/com/powsybl/ws/commons/computation/service/AbstractWorkerService.java @@ -8,7 +8,11 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.powsybl.commons.PowsyblException; +import com.powsybl.commons.io.FileUtil; import com.powsybl.commons.report.ReportNode; +import com.powsybl.computation.ComputationManager; +import com.powsybl.computation.local.LocalComputationConfig; +import com.powsybl.computation.local.LocalComputationManager; import com.powsybl.iidm.network.Network; import com.powsybl.iidm.network.VariantManagerConstants; import com.powsybl.network.store.client.NetworkStoreService; @@ -20,6 +24,10 @@ import org.springframework.messaging.Message; import org.springframework.web.server.ResponseStatusException; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.Map; import java.util.Objects; import java.util.UUID; @@ -165,6 +173,11 @@ public Consumer> consumeRun() { protected void clean(AbstractResultContext resultContext) { futures.remove(resultContext.getResultUuid()); cancelComputationRequests.remove(resultContext.getResultUuid()); + // clean working directory + if (!resultContext.getRunContext().isDebug()) { + Path workDir = resultContext.getRunContext().getComputationManager().getLocalDir(); + removeWorkingDirectory(workDir); + } } /** @@ -199,10 +212,11 @@ protected void publishFail(AbstractResultContext resultContext, String messag /** * Do some extra task before running the computation, e.g. print log or init extra data for the run context - * @param ignoredRunContext This context may be used for further computation in overriding classes + * @param runContext This context may be used for further computation in overriding classes */ - protected void preRun(C ignoredRunContext) { + protected void preRun(C runContext) { LOGGER.info("Run {} computation...", getComputationType()); + runContext.setComputationManager(createComputationManager()); } protected R run(C runContext, UUID resultUuid, AtomicReference rootReporter) throws Exception { @@ -262,4 +276,29 @@ protected CompletableFuture runAsync( protected abstract String getComputationType(); protected abstract CompletableFuture getCompletableFuture(C runContext, String provider, UUID resultUuid); + + private ComputationManager createComputationManager() { + LocalComputationConfig localComputationConfig = LocalComputationConfig.load(); + Path localDir = localComputationConfig.getLocalDir(); + try { + String workDirPrefix = getComputationType().replaceAll("\\s+", "_").toLowerCase() + "_"; + Path workDir = Files.createTempDirectory(localDir, workDirPrefix); + return new LocalComputationManager(new LocalComputationConfig(workDir, localComputationConfig.getAvailableCore()), executionService.getExecutorService()); + } catch (IOException e) { + throw new UncheckedIOException(String.format("Error occurred while creating a working directory inside the local directory %s", + localDir.toAbsolutePath()), e); + } + } + + private void removeWorkingDirectory(Path workDir) { + if (workDir != null) { + try { + FileUtil.removeDir(workDir); + } catch (IOException e) { + LOGGER.error(String.format("%s: Error occurred while cleaning working directory at %s", getComputationType(), workDir.toAbsolutePath()), e); + } + } else { + LOGGER.info("{}: No working directory to clean", getComputationType()); + } + } } diff --git a/src/main/java/com/powsybl/ws/commons/computation/service/ExecutionService.java b/src/main/java/com/powsybl/ws/commons/computation/service/ExecutionService.java index f6a124a..77c06ef 100644 --- a/src/main/java/com/powsybl/ws/commons/computation/service/ExecutionService.java +++ b/src/main/java/com/powsybl/ws/commons/computation/service/ExecutionService.java @@ -7,8 +7,6 @@ package com.powsybl.ws.commons.computation.service; -import com.powsybl.computation.ComputationManager; -import com.powsybl.computation.local.LocalComputationManager; import jakarta.annotation.PostConstruct; import jakarta.annotation.PreDestroy; import lombok.Getter; @@ -28,13 +26,10 @@ public class ExecutionService { private ExecutorService executorService; - private ComputationManager computationManager; - @SneakyThrows @PostConstruct private void postConstruct() { executorService = Executors.newCachedThreadPool(); - computationManager = new LocalComputationManager(getExecutorService()); } @PreDestroy From 2af529d35e7a051cdd012d3a31f8f4b515ee5970 Mon Sep 17 00:00:00 2001 From: Thang PHAM Date: Mon, 2 Dec 2024 13:14:48 +0100 Subject: [PATCH 2/5] add debug into run context constructor Signed-off-by: Thang PHAM --- .../computation/service/AbstractComputationRunContext.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/powsybl/ws/commons/computation/service/AbstractComputationRunContext.java b/src/main/java/com/powsybl/ws/commons/computation/service/AbstractComputationRunContext.java index a224508..3c5282e 100644 --- a/src/main/java/com/powsybl/ws/commons/computation/service/AbstractComputationRunContext.java +++ b/src/main/java/com/powsybl/ws/commons/computation/service/AbstractComputationRunContext.java @@ -35,7 +35,7 @@ public abstract class AbstractComputationRunContext

{ private ComputationManager computationManager; protected AbstractComputationRunContext(UUID networkUuid, String variantId, String receiver, ReportInfos reportInfos, - String userId, String provider, P parameters) { + String userId, String provider, P parameters, boolean debug) { this.networkUuid = networkUuid; this.variantId = variantId; this.receiver = receiver; @@ -45,5 +45,6 @@ protected AbstractComputationRunContext(UUID networkUuid, String variantId, Stri this.parameters = parameters; this.reportNode = ReportNode.NO_OP; this.network = null; + this.debug = debug; } } From 770a2e044e3f8e4fc238e1498af9cae90a2e975d Mon Sep 17 00:00:00 2001 From: Thang PHAM Date: Mon, 2 Dec 2024 16:21:13 +0100 Subject: [PATCH 3/5] config executor service for test Signed-off-by: Thang PHAM --- .../commons/computation/ComputationTest.java | 32 +++++++------------ 1 file changed, 11 insertions(+), 21 deletions(-) diff --git a/src/test/java/com/powsybl/ws/commons/computation/ComputationTest.java b/src/test/java/com/powsybl/ws/commons/computation/ComputationTest.java index 8dfd238..253ed9a 100644 --- a/src/test/java/com/powsybl/ws/commons/computation/ComputationTest.java +++ b/src/test/java/com/powsybl/ws/commons/computation/ComputationTest.java @@ -6,17 +6,7 @@ import com.powsybl.network.store.client.NetworkStoreService; import com.powsybl.network.store.client.PreloadingStrategy; import com.powsybl.ws.commons.computation.dto.ReportInfos; -import com.powsybl.ws.commons.computation.service.AbstractComputationObserver; -import com.powsybl.ws.commons.computation.service.AbstractComputationResultService; -import com.powsybl.ws.commons.computation.service.AbstractComputationRunContext; -import com.powsybl.ws.commons.computation.service.AbstractComputationService; -import com.powsybl.ws.commons.computation.service.AbstractResultContext; -import com.powsybl.ws.commons.computation.service.AbstractWorkerService; -import com.powsybl.ws.commons.computation.service.CancelContext; -import com.powsybl.ws.commons.computation.service.ExecutionService; -import com.powsybl.ws.commons.computation.service.NotificationService; -import com.powsybl.ws.commons.computation.service.ReportService; -import com.powsybl.ws.commons.computation.service.UuidGeneratorService; +import com.powsybl.ws.commons.computation.service.*; import io.micrometer.core.instrument.MeterRegistry; import io.micrometer.core.instrument.simple.SimpleMeterRegistry; import io.micrometer.observation.ObservationRegistry; @@ -39,17 +29,13 @@ import java.util.Map; import java.util.UUID; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; -import static com.powsybl.ws.commons.computation.service.NotificationService.HEADER_RECEIVER; -import static com.powsybl.ws.commons.computation.service.NotificationService.HEADER_RESULT_UUID; -import static com.powsybl.ws.commons.computation.service.NotificationService.HEADER_USER_ID; +import static com.powsybl.ws.commons.computation.service.NotificationService.*; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.ArgumentMatchers.isA; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; +import static org.mockito.ArgumentMatchers.*; +import static org.mockito.Mockito.*; @ExtendWith({ MockitoExtension.class }) @Slf4j @@ -61,7 +47,9 @@ class ComputationTest implements WithAssertions { private NetworkStoreService networkStoreService; @Mock private ReportService reportService; + @Mock private final ExecutionService executionService = new ExecutionService(); + private final ExecutorService executorService = Executors.newCachedThreadPool(); private final UuidGeneratorService uuidGeneratorService = new UuidGeneratorService(); @Mock private StreamBridge publisher; @@ -125,7 +113,7 @@ private static class MockComputationRunContext extends AbstractComputationRunCon protected MockComputationRunContext(UUID networkUuid, String variantId, String receiver, ReportInfos reportInfos, String userId, String provider, Object parameters) { - super(networkUuid, variantId, receiver, reportInfos, userId, provider, parameters); + super(networkUuid, variantId, receiver, reportInfos, userId, provider, parameters, false); } } @@ -235,6 +223,7 @@ private void initComputationExecution() { when(networkStoreService.getNetwork(eq(networkUuid), any(PreloadingStrategy.class))) .thenReturn(network); when(network.getVariantManager()).thenReturn(variantManager); + when(executionService.getExecutorService()).thenReturn(executorService); } @Test @@ -261,6 +250,7 @@ void testComputationFailed() { // test the course verify(notificationService.getPublisher(), times(1)).send(eq("publishFailed-out-0"), isA(Message.class)); + executionService.getExecutorService().shutdown(); } @Test From 55c7fc434d25a61ab2dbac77685492994bc8e427 Mon Sep 17 00:00:00 2001 From: Thang PHAM Date: Mon, 2 Dec 2024 18:32:52 +0100 Subject: [PATCH 4/5] rectify Signed-off-by: Thang PHAM --- .../service/AbstractWorkerService.java | 27 ++++++------------- 1 file changed, 8 insertions(+), 19 deletions(-) diff --git a/src/main/java/com/powsybl/ws/commons/computation/service/AbstractWorkerService.java b/src/main/java/com/powsybl/ws/commons/computation/service/AbstractWorkerService.java index 4359b6c..6be58cb 100644 --- a/src/main/java/com/powsybl/ws/commons/computation/service/AbstractWorkerService.java +++ b/src/main/java/com/powsybl/ws/commons/computation/service/AbstractWorkerService.java @@ -8,7 +8,6 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.powsybl.commons.PowsyblException; -import com.powsybl.commons.io.FileUtil; import com.powsybl.commons.report.ReportNode; import com.powsybl.computation.ComputationManager; import com.powsybl.computation.local.LocalComputationConfig; @@ -30,6 +29,7 @@ import java.nio.file.Path; import java.util.Map; import java.util.Objects; +import java.util.Optional; import java.util.UUID; import java.util.concurrent.CancellationException; import java.util.concurrent.CompletableFuture; @@ -173,11 +173,8 @@ public Consumer> consumeRun() { protected void clean(AbstractResultContext resultContext) { futures.remove(resultContext.getResultUuid()); cancelComputationRequests.remove(resultContext.getResultUuid()); - // clean working directory - if (!resultContext.getRunContext().isDebug()) { - Path workDir = resultContext.getRunContext().getComputationManager().getLocalDir(); - removeWorkingDirectory(workDir); - } + + Optional.ofNullable(resultContext.getRunContext().getComputationManager()).ifPresent(ComputationManager::close); } /** @@ -277,7 +274,11 @@ protected CompletableFuture runAsync( protected abstract CompletableFuture getCompletableFuture(C runContext, String provider, UUID resultUuid); - private ComputationManager createComputationManager() { + /** + * set method as public to mock DockerLocalComputationManager when testing with test container + * @return a computation manager + */ + public ComputationManager createComputationManager() { LocalComputationConfig localComputationConfig = LocalComputationConfig.load(); Path localDir = localComputationConfig.getLocalDir(); try { @@ -289,16 +290,4 @@ private ComputationManager createComputationManager() { localDir.toAbsolutePath()), e); } } - - private void removeWorkingDirectory(Path workDir) { - if (workDir != null) { - try { - FileUtil.removeDir(workDir); - } catch (IOException e) { - LOGGER.error(String.format("%s: Error occurred while cleaning working directory at %s", getComputationType(), workDir.toAbsolutePath()), e); - } - } else { - LOGGER.info("{}: No working directory to clean", getComputationType()); - } - } } From 662c686906a20bc0c0700681882afdb78d7bc598 Mon Sep 17 00:00:00 2001 From: Thang PHAM Date: Mon, 2 Dec 2024 19:35:07 +0100 Subject: [PATCH 5/5] add debug dir in the notification Signed-off-by: Thang PHAM --- .../service/AbstractWorkerService.java | 33 ++++++++++++++----- .../service/NotificationService.java | 1 + 2 files changed, 26 insertions(+), 8 deletions(-) diff --git a/src/main/java/com/powsybl/ws/commons/computation/service/AbstractWorkerService.java b/src/main/java/com/powsybl/ws/commons/computation/service/AbstractWorkerService.java index 6be58cb..2b9b648 100644 --- a/src/main/java/com/powsybl/ws/commons/computation/service/AbstractWorkerService.java +++ b/src/main/java/com/powsybl/ws/commons/computation/service/AbstractWorkerService.java @@ -27,10 +27,7 @@ import java.io.UncheckedIOException; import java.nio.file.Files; import java.nio.file.Path; -import java.util.Map; -import java.util.Objects; -import java.util.Optional; -import java.util.UUID; +import java.util.*; import java.util.concurrent.CancellationException; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; @@ -40,6 +37,8 @@ import java.util.concurrent.locks.ReentrantLock; import java.util.function.Consumer; +import static com.powsybl.ws.commons.computation.service.NotificationService.HEADER_DEBUG_DIR; + /** * @author Mathieu Deharbe * @param powsybl Result class specific to the computation @@ -197,14 +196,32 @@ public Consumer> consumeCancel() { protected abstract void saveResult(Network network, AbstractResultContext resultContext, R result); - protected void sendResultMessage(AbstractResultContext resultContext, R ignoredResult) { + private Map getAdditionalHeaders(AbstractResultContext resultContext, R ignoredResult) { + Map additionalHeaders = new HashMap<>(); + if (resultContext.getRunContext().isDebug() && resultContext.getRunContext().getComputationManager() != null) { + additionalHeaders.put(HEADER_DEBUG_DIR, resultContext.getRunContext().getComputationManager().getLocalDir().toAbsolutePath().toString()); + } + return additionalHeaders; + } + + public Map getResultHeaders(AbstractResultContext resultContext, R result) { + return getAdditionalHeaders(resultContext, result); + } + + public Map getFailHeaders(AbstractResultContext resultContext, R result) { + return getAdditionalHeaders(resultContext, result); + } + + private void sendResultMessage(AbstractResultContext resultContext, R result) { + Map additionalHeaders = getResultHeaders(resultContext, result); notificationService.sendResultMessage(resultContext.getResultUuid(), resultContext.getRunContext().getReceiver(), - resultContext.getRunContext().getUserId(), null); + resultContext.getRunContext().getUserId(), additionalHeaders); } - protected void publishFail(AbstractResultContext resultContext, String message) { + private void publishFail(AbstractResultContext resultContext, String message) { + Map additionalHeaders = getFailHeaders(resultContext, null); notificationService.publishFail(resultContext.getResultUuid(), resultContext.getRunContext().getReceiver(), - message, resultContext.getRunContext().getUserId(), getComputationType(), null); + message, resultContext.getRunContext().getUserId(), getComputationType(), additionalHeaders); } /** diff --git a/src/main/java/com/powsybl/ws/commons/computation/service/NotificationService.java b/src/main/java/com/powsybl/ws/commons/computation/service/NotificationService.java index 59d9a01..d86fd80 100644 --- a/src/main/java/com/powsybl/ws/commons/computation/service/NotificationService.java +++ b/src/main/java/com/powsybl/ws/commons/computation/service/NotificationService.java @@ -48,6 +48,7 @@ public class NotificationService { public static final String HEADER_PROVIDER = "provider"; public static final String HEADER_MESSAGE = "message"; public static final String HEADER_USER_ID = "userId"; + public static final String HEADER_DEBUG_DIR = "debugDir"; public static final String SENDING_MESSAGE = "Sending message : {}";