diff --git a/CHANGELOG.md b/CHANGELOG.md
index bfc802ca1..b8c15901f 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,3 +1,7 @@
+## [3.11.5] - 2023-10-23
+### Fixed
+- Added timeout on `MetastoreMappingImpl.isAvailable()` calls to prevent long waits on unresponsive metastores.
+
## [3.11.4] - 2023-08-23
### Changed
- Metrics have been incorporated into Waggle Dance with the inclusion of tags, which will facilitate filtering within Datadog.
diff --git a/pom.xml b/pom.xml
index 24d972078..d4874f87b 100644
--- a/pom.xml
+++ b/pom.xml
@@ -42,7 +42,6 @@
2.3.7
4.13.1
3.5.15
- 3.1.5
1.9
1.8.9
4.0.0
diff --git a/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/mapping/model/MetaStoreMappingImpl.java b/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/mapping/model/MetaStoreMappingImpl.java
index fc9711f5f..30f008e04 100644
--- a/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/mapping/model/MetaStoreMappingImpl.java
+++ b/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/mapping/model/MetaStoreMappingImpl.java
@@ -1,5 +1,5 @@
/**
- * Copyright (C) 2016-2021 Expedia, Inc.
+ * Copyright (C) 2016-2023 Expedia, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -19,6 +19,11 @@
import java.util.Collections;
import java.util.List;
import java.util.Locale;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
import org.apache.hadoop.hive.metastore.MetaStoreFilterHook;
import org.apache.hadoop.hive.metastore.api.AlreadyExistsException;
@@ -39,6 +44,9 @@ class MetaStoreMappingImpl implements MetaStoreMapping {
private final static Logger log = LoggerFactory.getLogger(MetaStoreMappingImpl.class);
+ // MilliSeconds
+ static final long DEFAULT_AVAILABILITY_TIMEOUT = 500;
+
private final String databasePrefix;
private final CloseableThriftHiveMetastoreIface client;
private final AccessControlHandler accessControlHandler;
@@ -106,18 +114,32 @@ public void close() throws IOException {
client.close();
}
+ /**
+ * This is potentially slow so a best effort is made and false is returned after a timeout.
+ */
@Override
public boolean isAvailable() {
- try {
- boolean isOpen = client.isOpen();
- if (isOpen && connectionType == ConnectionType.TUNNELED) {
- client.getStatus();
+ Future future = CompletableFuture.supplyAsync(() -> {
+ try {
+ boolean isOpen = client.isOpen();
+ if (isOpen && connectionType == ConnectionType.TUNNELED) {
+ client.getStatus();
+ }
+ return isOpen;
+ } catch (Exception e) {
+ log.error("Metastore Mapping {} unavailable", name, e);
+ return false;
}
- return isOpen;
- } catch (Exception e) {
- log.error("Metastore Mapping {} unavailable", name, e);
- return false;
+ });
+ try {
+ return future.get(DEFAULT_AVAILABILITY_TIMEOUT + getLatency(), TimeUnit.MILLISECONDS);
+ } catch (TimeoutException e) {
+ log.info("Took too long to check availability of '" + name + "', assuming unavailable");
+ future.cancel(true);
+ } catch (InterruptedException | ExecutionException e) {
+ log.error("Error while checking availability '" + name + "', assuming unavailable");
}
+ return false;
}
@Override
diff --git a/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/mapping/model/MetaStoreMappingImplTest.java b/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/mapping/model/MetaStoreMappingImplTest.java
index a65f893cd..57b8692e6 100644
--- a/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/mapping/model/MetaStoreMappingImplTest.java
+++ b/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/mapping/model/MetaStoreMappingImplTest.java
@@ -1,5 +1,5 @@
/**
- * Copyright (C) 2016-2021 Expedia, Inc.
+ * Copyright (C) 2016-2023 Expedia, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -19,12 +19,14 @@
import static org.hamcrest.CoreMatchers.sameInstance;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.junit.Assert.fail;
+import static org.mockito.AdditionalAnswers.answersWithDelay;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import static com.hotels.bdp.waggledance.api.model.ConnectionType.DIRECT;
import static com.hotels.bdp.waggledance.api.model.ConnectionType.TUNNELED;
+import static com.hotels.bdp.waggledance.mapping.model.MetaStoreMappingImpl.DEFAULT_AVAILABILITY_TIMEOUT;
import java.io.IOException;
@@ -115,6 +117,12 @@ public void isNotAvailable() {
assertThat(metaStoreMapping.isAvailable(), is(false));
}
+ @Test
+ public void isNotAvailableTimeout() throws Exception {
+ when(client.isOpen()).then(answersWithDelay(DEFAULT_AVAILABILITY_TIMEOUT+LATENCY+10, a -> true));
+ assertThat(metaStoreMapping.isAvailable(), is(false));
+ }
+
@Test
public void isAvailableTunnelled() throws Exception {
when(client.isOpen()).thenReturn(true);