From 9ad4c9473f118d8fd351b66a7b36b646dfcdd02c Mon Sep 17 00:00:00 2001 From: IvanBorislavovDimitrov Date: Tue, 21 Jan 2025 14:27:44 +0200 Subject: [PATCH] Add unit tests --- .../health/ApplicationHealthCalculator.java | 8 +- .../DatabaseWaitingLocksAnalyzer.java | 21 ++- .../ApplicationHealthCalculatorTest.java | 143 ++++++++++++++++++ .../DatabaseWaitingLocksAnalyzerTest.java | 81 ++++++++++ .../services/DatabaseHealthServiceTest.java | 42 +++++ .../DatabaseMonitoringServiceTest.java | 43 ++++++ 6 files changed, 332 insertions(+), 6 deletions(-) create mode 100644 multiapps-controller-core/src/test/java/org/cloudfoundry/multiapps/controller/core/application/health/ApplicationHealthCalculatorTest.java create mode 100644 multiapps-controller-core/src/test/java/org/cloudfoundry/multiapps/controller/core/application/health/database/DatabaseWaitingLocksAnalyzerTest.java create mode 100644 multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/DatabaseHealthServiceTest.java create mode 100644 multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/DatabaseMonitoringServiceTest.java diff --git a/multiapps-controller-core/src/main/java/org/cloudfoundry/multiapps/controller/core/application/health/ApplicationHealthCalculator.java b/multiapps-controller-core/src/main/java/org/cloudfoundry/multiapps/controller/core/application/health/ApplicationHealthCalculator.java index c3a55e69b6..c6433f29cc 100644 --- a/multiapps-controller-core/src/main/java/org/cloudfoundry/multiapps/controller/core/application/health/ApplicationHealthCalculator.java +++ b/multiapps-controller-core/src/main/java/org/cloudfoundry/multiapps/controller/core/application/health/ApplicationHealthCalculator.java @@ -63,10 +63,14 @@ public ApplicationHealthCalculator(@Autowired(required = false) ObjectStoreFileS this.databaseHealthService = databaseHealthService; this.databaseMonitoringService = databaseMonitoringService; this.databaseWaitingLocksAnalyzer = databaseWaitingLocksAnalyzer; + scheduleRegularHealthUpdate(); + } + + protected void scheduleRegularHealthUpdate() { scheduler.scheduleAtFixedRate(this::updateHealthStatus, 0, UPDATE_HEALTH_CHECK_STATUS_PERIOD_IN_SECONDS, TimeUnit.SECONDS); } - private void updateHealthStatus() { + protected void updateHealthStatus() { List> tasks = List.of(this::isObjectStoreFileStorageHealthy, this::isDatabaseHealthy, databaseWaitingLocksAnalyzer::hasIncreasedDbLocks); try { @@ -172,7 +176,7 @@ private boolean isDatabaseHealthy() { } } - private ResilientOperationExecutor getResilienceExecutor() { + protected ResilientOperationExecutor getResilienceExecutor() { return new ResilientOperationExecutor(); } diff --git a/multiapps-controller-core/src/main/java/org/cloudfoundry/multiapps/controller/core/application/health/database/DatabaseWaitingLocksAnalyzer.java b/multiapps-controller-core/src/main/java/org/cloudfoundry/multiapps/controller/core/application/health/database/DatabaseWaitingLocksAnalyzer.java index c88b46213d..d4103a2307 100644 --- a/multiapps-controller-core/src/main/java/org/cloudfoundry/multiapps/controller/core/application/health/database/DatabaseWaitingLocksAnalyzer.java +++ b/multiapps-controller-core/src/main/java/org/cloudfoundry/multiapps/controller/core/application/health/database/DatabaseWaitingLocksAnalyzer.java @@ -29,6 +29,7 @@ public class DatabaseWaitingLocksAnalyzer { private static final Duration ANOMALY_DETECTION_THRESHOLD_IN_MINUTES = Duration.ofMinutes(5); private static final int MAXIMUM_VALUE_OF_NORMAL_LOCKS_COUNT = 5; private static final double MAXIMAL_ACCEPTABLE_INCREMENTAL_LOCKS_DEVIATION_INDEX = 0.5; + private static final int MINIMUM_REQUIRED_INCREASED_SAMPLES_REQUIRED_FOR_LOGGING = 5; private final ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor(); private final List> waitingLocksSamples = new LinkedList<>(); @@ -41,10 +42,14 @@ public DatabaseWaitingLocksAnalyzer(DatabaseMonitoringService databaseMonitoring ApplicationConfiguration applicationConfiguration) { this.databaseMonitoringService = databaseMonitoringService; this.applicationConfiguration = applicationConfiguration; + scheduleRegularLocksRefresh(); + } + + protected void scheduleRegularLocksRefresh() { executor.scheduleAtFixedRate(this::refreshLockInfo, 0, POLLING_LOCKS_INTERVAL_IN_SECONDS, TimeUnit.SECONDS); } - private synchronized void refreshLockInfo() { + protected synchronized void refreshLockInfo() { deleteObsoleteSamples(); takeLocksSample(); } @@ -65,9 +70,10 @@ public synchronized boolean hasIncreasedDbLocks() { minimumRequiredSamplesCount)); return false; } - boolean hasIncreasedLocks = calculateIncreasingOrEqualIndex() >= MAXIMAL_ACCEPTABLE_INCREMENTAL_LOCKS_DEVIATION_INDEX + double calculatedIncreasingOrEqualIndex = calculateIncreasingOrEqualIndex(); + boolean hasIncreasedLocks = calculatedIncreasingOrEqualIndex >= MAXIMAL_ACCEPTABLE_INCREMENTAL_LOCKS_DEVIATION_INDEX && checkIfLastOneThirdOfSequenceHasIncreasedOrIsEqualComparedToFirstOneThird(minimumRequiredSamplesCount); - if (hasIncreasedLocks) { + if (shouldLogValues()) { LOGGER.info(MessageFormat.format(Messages.VALUES_IN_INSTANCE_IN_THE_WAITING_FOR_LOCKS_SAMPLES, applicationConfiguration.getApplicationInstanceIndex(), waitingLocksSamples.stream() .map(CachedObject::get) @@ -76,7 +82,7 @@ public synchronized boolean hasIncreasedDbLocks() { return hasIncreasedLocks; } - public double calculateIncreasingOrEqualIndex() { + private double calculateIncreasingOrEqualIndex() { int increasingOrEqualCount = 0; int decreasingCount = 0; int totalComparisons = waitingLocksSamples.size() - 1; @@ -119,4 +125,11 @@ private boolean checkIfLastOneThirdOfSequenceHasIncreasedOrIsEqualComparedToFirs .sum(); return sumOfLastOneThird >= sumOfFirstOneThird; } + + private boolean shouldLogValues() { + return waitingLocksSamples.stream() + .map(CachedObject::get) + .filter(value -> value >= MAXIMUM_VALUE_OF_NORMAL_LOCKS_COUNT) + .count() >= MINIMUM_REQUIRED_INCREASED_SAMPLES_REQUIRED_FOR_LOGGING; + } } diff --git a/multiapps-controller-core/src/test/java/org/cloudfoundry/multiapps/controller/core/application/health/ApplicationHealthCalculatorTest.java b/multiapps-controller-core/src/test/java/org/cloudfoundry/multiapps/controller/core/application/health/ApplicationHealthCalculatorTest.java new file mode 100644 index 0000000000..e067eb7e88 --- /dev/null +++ b/multiapps-controller-core/src/test/java/org/cloudfoundry/multiapps/controller/core/application/health/ApplicationHealthCalculatorTest.java @@ -0,0 +1,143 @@ +package org.cloudfoundry.multiapps.controller.core.application.health; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.cloudfoundry.multiapps.common.SLException; +import org.cloudfoundry.multiapps.controller.client.util.ResilientOperationExecutor; +import org.cloudfoundry.multiapps.controller.core.application.health.database.DatabaseWaitingLocksAnalyzer; +import org.cloudfoundry.multiapps.controller.core.application.health.model.ApplicationHealthResult; +import org.cloudfoundry.multiapps.controller.core.util.ApplicationConfiguration; +import org.cloudfoundry.multiapps.controller.persistence.services.DatabaseHealthService; +import org.cloudfoundry.multiapps.controller.persistence.services.DatabaseMonitoringService; +import org.cloudfoundry.multiapps.controller.persistence.services.ObjectStoreFileStorage; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.MockitoAnnotations; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; + +class ApplicationHealthCalculatorTest { + + @Mock + private ObjectStoreFileStorage objectStoreFileStorage; + @Mock + private ApplicationConfiguration applicationConfiguration; + @Mock + private DatabaseHealthService databaseHealthService; + @Mock + private DatabaseMonitoringService databaseMonitoringService; + @Mock + private DatabaseWaitingLocksAnalyzer databaseWaitingLocksAnalyzer; + + private ApplicationHealthCalculator applicationHealthCalculator; + + @BeforeEach + void setUp() throws Exception { + MockitoAnnotations.openMocks(this) + .close(); + Mockito.when(applicationConfiguration.isHealthCheckEnabled()) + .thenReturn(true); + applicationHealthCalculator = new ApplicationHealthCalculatorMock(objectStoreFileStorage, + applicationConfiguration, + databaseHealthService, + databaseMonitoringService, + databaseWaitingLocksAnalyzer); + } + + @Test + void testUpdateWithFailingObjectStore() { + Mockito.doThrow(new SLException("Object store not working")) + .when(objectStoreFileStorage) + .testConnection(); + applicationHealthCalculator.updateHealthStatus(); + ResponseEntity applicationHealthResultResponseEntity = applicationHealthCalculator.calculateApplicationHealth(); + assertEquals(HttpStatus.SERVICE_UNAVAILABLE, applicationHealthResultResponseEntity.getStatusCode()); + assertEquals(ApplicationHealthResult.Status.DOWN, applicationHealthResultResponseEntity.getBody() + .getStatus()); + assertFalse(applicationHealthResultResponseEntity.getBody() + .hasIncreasedLocks()); + } + + @Test + void testUpdateWithFailingDatabase() { + Mockito.doThrow(new SLException("Database not working")) + .when(databaseHealthService) + .testDatabaseConnection(); + applicationHealthCalculator.updateHealthStatus(); + ResponseEntity applicationHealthResultResponseEntity = applicationHealthCalculator.calculateApplicationHealth(); + assertEquals(HttpStatus.SERVICE_UNAVAILABLE, applicationHealthResultResponseEntity.getStatusCode()); + assertEquals(ApplicationHealthResult.Status.DOWN, applicationHealthResultResponseEntity.getBody() + .getStatus()); + assertFalse(applicationHealthResultResponseEntity.getBody() + .hasIncreasedLocks()); + } + + @Test + void testSuccessfulHealthCheck() { + applicationHealthCalculator.updateHealthStatus(); + ResponseEntity applicationHealthResultResponseEntity = applicationHealthCalculator.calculateApplicationHealth(); + assertEquals(HttpStatus.OK, applicationHealthResultResponseEntity.getStatusCode()); + assertEquals(ApplicationHealthResult.Status.UP, applicationHealthResultResponseEntity.getBody() + .getStatus()); + assertFalse(applicationHealthResultResponseEntity.getBody() + .hasIncreasedLocks()); + } + + @Test + void testUpdateWithIncreasedDatabaseLocks() { + Mockito.when(databaseWaitingLocksAnalyzer.hasIncreasedDbLocks()) + .thenReturn(true); + applicationHealthCalculator.updateHealthStatus(); + ResponseEntity applicationHealthResultResponseEntity = applicationHealthCalculator.calculateApplicationHealth(); + assertEquals(HttpStatus.OK, applicationHealthResultResponseEntity.getStatusCode()); + assertEquals(ApplicationHealthResult.Status.DOWN, applicationHealthResultResponseEntity.getBody() + .getStatus()); + assertTrue(applicationHealthResultResponseEntity.getBody() + .hasIncreasedLocks()); + } + + @Test + void testSuccessfulUpdateWithMissingObjectStore() { + applicationHealthCalculator = new ApplicationHealthCalculatorMock(null, + applicationConfiguration, + databaseHealthService, + databaseMonitoringService, + databaseWaitingLocksAnalyzer); + applicationHealthCalculator.updateHealthStatus(); + ResponseEntity applicationHealthResultResponseEntity = applicationHealthCalculator.calculateApplicationHealth(); + assertEquals(HttpStatus.OK, applicationHealthResultResponseEntity.getStatusCode()); + assertEquals(ApplicationHealthResult.Status.UP, applicationHealthResultResponseEntity.getBody() + .getStatus()); + assertFalse(applicationHealthResultResponseEntity.getBody() + .hasIncreasedLocks()); + } + + private static class ApplicationHealthCalculatorMock extends ApplicationHealthCalculator { + public ApplicationHealthCalculatorMock(ObjectStoreFileStorage objectStoreFileStorage, + ApplicationConfiguration applicationConfiguration, + DatabaseHealthService databaseHealthService, + DatabaseMonitoringService databaseMonitoringService, + DatabaseWaitingLocksAnalyzer databaseWaitingLocksAnalyzer) { + super(objectStoreFileStorage, + applicationConfiguration, + databaseHealthService, + databaseMonitoringService, + databaseWaitingLocksAnalyzer); + } + + @Override + protected ResilientOperationExecutor getResilienceExecutor() { + return new ResilientOperationExecutor().withRetryCount(0) + .withWaitTimeBetweenRetriesInMillis(0); + } + + @Override + protected void scheduleRegularHealthUpdate() { + // Do nothing + } + } +} diff --git a/multiapps-controller-core/src/test/java/org/cloudfoundry/multiapps/controller/core/application/health/database/DatabaseWaitingLocksAnalyzerTest.java b/multiapps-controller-core/src/test/java/org/cloudfoundry/multiapps/controller/core/application/health/database/DatabaseWaitingLocksAnalyzerTest.java new file mode 100644 index 0000000000..b00fa3bf1b --- /dev/null +++ b/multiapps-controller-core/src/test/java/org/cloudfoundry/multiapps/controller/core/application/health/database/DatabaseWaitingLocksAnalyzerTest.java @@ -0,0 +1,81 @@ +package org.cloudfoundry.multiapps.controller.core.application.health.database; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.ArgumentMatchers.anyString; + +import java.util.stream.Stream; + +import org.cloudfoundry.multiapps.controller.core.util.ApplicationConfiguration; +import org.cloudfoundry.multiapps.controller.persistence.services.DatabaseMonitoringService; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.MockitoAnnotations; + +class DatabaseWaitingLocksAnalyzerTest { + + @Mock + private DatabaseMonitoringService databaseMonitoringService; + @Mock + private ApplicationConfiguration applicationConfiguration; + + private DatabaseWaitingLocksAnalyzer databaseWaitingLocksAnalyzer; + + @BeforeEach + void setUp() throws Exception { + MockitoAnnotations.openMocks(this) + .close(); + databaseWaitingLocksAnalyzer = new DatabaseWaitingLocksAnalyzerMock(databaseMonitoringService, applicationConfiguration); + } + + // @formatter:off + static Stream testIncreasedLocks() { + return Stream.of(Arguments.of(new long[] { 1, 2, 3, 4, 5, 6, 6, 6, 7, 7, 6, 5, 4, 3, 2, 1, 1, 2, 3, 4, 5, 6, 7, 8, 10, 5, 14, 16, 4, 3, 2, 1 }, false), // has values under the minimal threshold + Arguments.of(new long[] { 6, 7, 12, 12, 12, 12, 12, 12, 60, 59, 58, 57, 56, 55, 54, 53, 52, 51, 50, 49, 48, 47, 46, 45, 44, 43, 42, 41, 40, 41, 35, 30 }, false), // most of the values are decreasing, assuming that they will continue to decrease + Arguments.of(new long[] { 24, 24, 26, 29, 30, 30, 30, 31, 30, 14, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 11, 12, 13, 14, 14, 15, 16, 17, 18, 20 }, false), // the sum of the last on third of the sequence is smaller compared to the sum of the first one third + Arguments.of(new long[] { 24, 24, 26, 29, 30, 30, 30, 31, 30, 14, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 24, 29, 31, 32, 14, 19, 36, 38, 40, 45 }, true) // the index of increase is bigger than the threshold and the sum of the last sequence is bigger than the sum of the first sequence + ); + } + // @formatter:on + + @ParameterizedTest + @MethodSource + void testIncreasedLocks(long[] lockSamples, boolean hasIncreasedLocksExpectation) { + mockDatabaseWaitingLocksAnalyzer(lockSamples); + refreshLocksSamples(lockSamples); + assertEquals(hasIncreasedLocksExpectation, databaseWaitingLocksAnalyzer.hasIncreasedDbLocks()); + } + + private void mockDatabaseWaitingLocksAnalyzer(long[] lockSamples) { + Mockito.when(databaseMonitoringService.getProcessesWaitingForLocks(anyString())) + .thenAnswer(invocation -> { + int callIndex = Mockito.mockingDetails(databaseMonitoringService) + .getInvocations() + .size() + - 1; + return lockSamples[callIndex]; + }); + } + + private void refreshLocksSamples(long[] lockSamples) { + for (int i = 0; i < lockSamples.length; i++) { + databaseWaitingLocksAnalyzer.refreshLockInfo(); + } + } + + private static class DatabaseWaitingLocksAnalyzerMock extends DatabaseWaitingLocksAnalyzer { + + public DatabaseWaitingLocksAnalyzerMock(DatabaseMonitoringService databaseMonitoringService, + ApplicationConfiguration applicationConfiguration) { + super(databaseMonitoringService, applicationConfiguration); + } + + @Override + protected void scheduleRegularLocksRefresh() { + // Do nothing + } + } +} diff --git a/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/DatabaseHealthServiceTest.java b/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/DatabaseHealthServiceTest.java new file mode 100644 index 0000000000..887ae28a0a --- /dev/null +++ b/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/DatabaseHealthServiceTest.java @@ -0,0 +1,42 @@ +package org.cloudfoundry.multiapps.controller.persistence.services; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import jakarta.persistence.EntityManager; +import jakarta.persistence.EntityManagerFactory; +import jakarta.persistence.Query; + +class DatabaseHealthServiceTest { + + private EntityManagerFactory entityManagerFactory; + private EntityManager entityManager; + private Query query; + private DatabaseHealthService databaseHealthService; + + @BeforeEach + void setUp() { + entityManagerFactory = mock(EntityManagerFactory.class); + entityManager = mock(EntityManager.class); + query = mock(Query.class); + databaseHealthService = new DatabaseHealthService(entityManagerFactory); + + when(entityManagerFactory.createEntityManager()).thenReturn(entityManager); + when(entityManager.createNativeQuery(anyString())).thenReturn(query); + when(query.getSingleResult()).thenReturn(10); + } + + @Test + void testDatabaseConnection() { + databaseHealthService.testDatabaseConnection(); + + verify(entityManager).close(); + assertEquals(10, query.getSingleResult()); + } +} diff --git a/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/DatabaseMonitoringServiceTest.java b/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/DatabaseMonitoringServiceTest.java new file mode 100644 index 0000000000..32d3686f32 --- /dev/null +++ b/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/DatabaseMonitoringServiceTest.java @@ -0,0 +1,43 @@ +package org.cloudfoundry.multiapps.controller.persistence.services; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import jakarta.persistence.EntityManager; +import jakarta.persistence.EntityManagerFactory; +import jakarta.persistence.Query; + +class DatabaseMonitoringServiceTest { + + private EntityManagerFactory entityManagerFactory; + private EntityManager entityManager; + private Query query; + private DatabaseMonitoringService databaseMonitoringService; + + @BeforeEach + void setUp() { + entityManagerFactory = mock(EntityManagerFactory.class); + entityManager = mock(EntityManager.class); + query = mock(Query.class); + databaseMonitoringService = new DatabaseMonitoringService(entityManagerFactory); + + when(entityManagerFactory.createEntityManager()).thenReturn(entityManager); + when(entityManager.createNativeQuery(anyString())).thenReturn(query); + when(query.getSingleResult()).thenReturn(10L); + } + + @Test + void testGetProcessesWaitingForLocks() { + long result = databaseMonitoringService.getProcessesWaitingForLocks("testApp"); + + assertEquals(10L, result); + verify(entityManager).close(); + } + +}