diff --git a/tracker/src/main/java/org/edu_sharing/elasticsearch/TrackerAvailabilityService.java b/tracker/src/main/java/org/edu_sharing/elasticsearch/TrackerAvailabilityService.java new file mode 100644 index 00000000..3c0109f4 --- /dev/null +++ b/tracker/src/main/java/org/edu_sharing/elasticsearch/TrackerAvailabilityService.java @@ -0,0 +1,28 @@ +package org.edu_sharing.elasticsearch; + +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.actuate.health.Health; +import org.springframework.boot.availability.*; +import org.springframework.stereotype.Component; + +@Component +public class TrackerAvailabilityService extends ApplicationAvailabilityBean implements TrackerAvailabilityTickService { + // curl localhost:8081/actuator/health/liveness + private long lastTrackingEvent = System.currentTimeMillis(); + @Value("${management.endpoint.health.trackingTimeoutThreshold}") + private long trackingTimeoutThreshold; + + @Override + public void tick() { + lastTrackingEvent = System.currentTimeMillis(); + } + + @Override + public S getState(Class stateType) { + S state = super.getState(stateType); + if(state instanceof LivenessState && (System.currentTimeMillis() - lastTrackingEvent) > trackingTimeoutThreshold * 1000 * 60) { + return (S) LivenessState.BROKEN; + } + return state; + } +} \ No newline at end of file diff --git a/tracker/src/main/java/org/edu_sharing/elasticsearch/TrackerAvailabilityTickService.java b/tracker/src/main/java/org/edu_sharing/elasticsearch/TrackerAvailabilityTickService.java new file mode 100644 index 00000000..86cda0c8 --- /dev/null +++ b/tracker/src/main/java/org/edu_sharing/elasticsearch/TrackerAvailabilityTickService.java @@ -0,0 +1,10 @@ +package org.edu_sharing.elasticsearch; + +public interface TrackerAvailabilityTickService { + + /** + * tell the application that the tracker is still alive and has just triggered a new run + * Used for liveness probes + */ + void tick(); +} diff --git a/tracker/src/main/java/org/edu_sharing/elasticsearch/elasticsearch/config/mode/DefaultConfiguration.java b/tracker/src/main/java/org/edu_sharing/elasticsearch/elasticsearch/config/mode/DefaultConfiguration.java index 67327f5b..76229769 100644 --- a/tracker/src/main/java/org/edu_sharing/elasticsearch/elasticsearch/config/mode/DefaultConfiguration.java +++ b/tracker/src/main/java/org/edu_sharing/elasticsearch/elasticsearch/config/mode/DefaultConfiguration.java @@ -1,6 +1,7 @@ package org.edu_sharing.elasticsearch.elasticsearch.config.mode; import lombok.RequiredArgsConstructor; +import org.edu_sharing.elasticsearch.TrackerAvailabilityTickService; import org.edu_sharing.elasticsearch.elasticsearch.core.migration.MigrationService; import org.edu_sharing.elasticsearch.elasticsearch.core.migration.WaitForMigrationJob; import org.edu_sharing.elasticsearch.jobs.AclTrackerJob; @@ -24,18 +25,18 @@ public WaitForMigrationJob waitForMigrationJob(MigrationService migrationService } @Bean - public TransactionTrackerJob transactionTrackerJob(TransactionTracker transactionTracker){ - return new TransactionTrackerJob(transactionTracker); + public TransactionTrackerJob transactionTrackerJob(TransactionTracker transactionTracker, TrackerAvailabilityTickService tickService){ + return new TransactionTrackerJob(transactionTracker, tickService); } @Bean - public AclTrackerJob aclTrackerJob(AclTracker aclTracker){ - return new AclTrackerJob(aclTracker); + public AclTrackerJob aclTrackerJob(AclTracker aclTracker, TrackerAvailabilityTickService tickService){ + return new AclTrackerJob(aclTracker, tickService); } @Bean @ConditionalOnProperty(prefix = "statistic", name = "enabled", havingValue = "true") - public StatisticsTrackerJob statisticsTrackerJob(StatisticsTracker statisticsTracker){ - return new StatisticsTrackerJob(statisticsTracker); + public StatisticsTrackerJob statisticsTrackerJob(StatisticsTracker statisticsTracker, TrackerAvailabilityTickService tickService){ + return new StatisticsTrackerJob(statisticsTracker, tickService); } } diff --git a/tracker/src/main/java/org/edu_sharing/elasticsearch/jobs/AclTrackerJob.java b/tracker/src/main/java/org/edu_sharing/elasticsearch/jobs/AclTrackerJob.java index 743ec2f0..a22bdf73 100644 --- a/tracker/src/main/java/org/edu_sharing/elasticsearch/jobs/AclTrackerJob.java +++ b/tracker/src/main/java/org/edu_sharing/elasticsearch/jobs/AclTrackerJob.java @@ -2,6 +2,7 @@ import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.edu_sharing.elasticsearch.TrackerAvailabilityTickService; import org.edu_sharing.elasticsearch.elasticsearch.core.migration.MigrationCompletedAware; import org.edu_sharing.elasticsearch.tracker.AclTracker; import org.springframework.scheduling.annotation.Scheduled; @@ -11,11 +12,14 @@ public class AclTrackerJob implements MigrationCompletedAware { private final AclTracker aclTracker; + private final TrackerAvailabilityTickService tickService; private boolean migrated = false; + @Scheduled(fixedDelayString = "${tracker.delay}") public void track() { + tickService.tick(); if (!migrated) { return; } diff --git a/tracker/src/main/java/org/edu_sharing/elasticsearch/jobs/StatisticsTrackerJob.java b/tracker/src/main/java/org/edu_sharing/elasticsearch/jobs/StatisticsTrackerJob.java index 461ea151..8ed1caec 100644 --- a/tracker/src/main/java/org/edu_sharing/elasticsearch/jobs/StatisticsTrackerJob.java +++ b/tracker/src/main/java/org/edu_sharing/elasticsearch/jobs/StatisticsTrackerJob.java @@ -2,6 +2,8 @@ import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.edu_sharing.elasticsearch.TrackerAvailabilityService; +import org.edu_sharing.elasticsearch.TrackerAvailabilityTickService; import org.edu_sharing.elasticsearch.elasticsearch.core.migration.MigrationCompletedAware; import org.edu_sharing.elasticsearch.tracker.StatisticsTracker; import org.springframework.scheduling.annotation.Scheduled; @@ -11,6 +13,8 @@ public class StatisticsTrackerJob implements MigrationCompletedAware { private final StatisticsTracker statisticsTracker; + private final TrackerAvailabilityTickService tickService; + private boolean migrated = false; /** @@ -19,6 +23,7 @@ public class StatisticsTrackerJob implements MigrationCompletedAware { */ @Scheduled(fixedDelayString = "${statistic.delay}") public void track() { + tickService.tick(); if (!migrated) { return; } diff --git a/tracker/src/main/java/org/edu_sharing/elasticsearch/jobs/TransactionTrackerJob.java b/tracker/src/main/java/org/edu_sharing/elasticsearch/jobs/TransactionTrackerJob.java index d289aebf..ff2c9b59 100644 --- a/tracker/src/main/java/org/edu_sharing/elasticsearch/jobs/TransactionTrackerJob.java +++ b/tracker/src/main/java/org/edu_sharing/elasticsearch/jobs/TransactionTrackerJob.java @@ -3,6 +3,8 @@ import lombok.RequiredArgsConstructor; import lombok.Setter; import lombok.extern.slf4j.Slf4j; +import org.edu_sharing.elasticsearch.TrackerAvailabilityService; +import org.edu_sharing.elasticsearch.TrackerAvailabilityTickService; import org.edu_sharing.elasticsearch.elasticsearch.core.migration.MigrationCompletedAware; import org.edu_sharing.elasticsearch.tracker.TransactionTracker; import org.springframework.beans.factory.annotation.Value; @@ -16,6 +18,8 @@ public class TransactionTrackerJob implements MigrationCompletedAware, ApplicationContextAware { private final TransactionTracker transactionTracker; + private final TrackerAvailabilityTickService tickService; + private boolean migrated = false; @Value("${shutdown.on.exception}") @@ -26,6 +30,7 @@ public class TransactionTrackerJob implements MigrationCompletedAware, Applicati @Scheduled(fixedDelayString = "${tracker.delay}") public void track() { + tickService.tick(); if (!migrated) { return; } diff --git a/tracker/src/main/resources/application.properties b/tracker/src/main/resources/application.properties index ca81eafe..e8606bc2 100644 --- a/tracker/src/main/resources/application.properties +++ b/tracker/src/main/resources/application.properties @@ -52,6 +52,9 @@ management.endpoint.health.probes.enabled = true management.endpoint.health.show-details = always management.endpoint.health.show-components = always +# timeout since which a last method run is considered acceptable, in minutes +management.endpoint.health.trackingTimeoutThreshold=360 + management.endpoints.web.exposure.include = health,metrics,prometheus management.server.address=0.0.0.0