diff --git a/CHANGELOG.md b/CHANGELOG.md index bfc802ca1..b8c15901f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +## [3.11.5] - 2023-10-23 +### Fixed +- Added timeout on `MetastoreMappingImpl.isAvailable()` calls to prevent long waits on unresponsive metastores. + ## [3.11.4] - 2023-08-23 ### Changed - Metrics have been incorporated into Waggle Dance with the inclusion of tags, which will facilitate filtering within Datadog. diff --git a/pom.xml b/pom.xml index 24d972078..d4874f87b 100644 --- a/pom.xml +++ b/pom.xml @@ -42,7 +42,6 @@ 2.3.7 4.13.1 3.5.15 - 3.1.5 1.9 1.8.9 4.0.0 diff --git a/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/mapping/model/MetaStoreMappingImpl.java b/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/mapping/model/MetaStoreMappingImpl.java index fc9711f5f..30f008e04 100644 --- a/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/mapping/model/MetaStoreMappingImpl.java +++ b/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/mapping/model/MetaStoreMappingImpl.java @@ -1,5 +1,5 @@ /** - * Copyright (C) 2016-2021 Expedia, Inc. + * Copyright (C) 2016-2023 Expedia, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,6 +19,11 @@ import java.util.Collections; import java.util.List; import java.util.Locale; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import org.apache.hadoop.hive.metastore.MetaStoreFilterHook; import org.apache.hadoop.hive.metastore.api.AlreadyExistsException; @@ -39,6 +44,9 @@ class MetaStoreMappingImpl implements MetaStoreMapping { private final static Logger log = LoggerFactory.getLogger(MetaStoreMappingImpl.class); + // MilliSeconds + static final long DEFAULT_AVAILABILITY_TIMEOUT = 500; + private final String databasePrefix; private final CloseableThriftHiveMetastoreIface client; private final AccessControlHandler accessControlHandler; @@ -106,18 +114,32 @@ public void close() throws IOException { client.close(); } + /** + * This is potentially slow so a best effort is made and false is returned after a timeout. + */ @Override public boolean isAvailable() { - try { - boolean isOpen = client.isOpen(); - if (isOpen && connectionType == ConnectionType.TUNNELED) { - client.getStatus(); + Future future = CompletableFuture.supplyAsync(() -> { + try { + boolean isOpen = client.isOpen(); + if (isOpen && connectionType == ConnectionType.TUNNELED) { + client.getStatus(); + } + return isOpen; + } catch (Exception e) { + log.error("Metastore Mapping {} unavailable", name, e); + return false; } - return isOpen; - } catch (Exception e) { - log.error("Metastore Mapping {} unavailable", name, e); - return false; + }); + try { + return future.get(DEFAULT_AVAILABILITY_TIMEOUT + getLatency(), TimeUnit.MILLISECONDS); + } catch (TimeoutException e) { + log.info("Took too long to check availability of '" + name + "', assuming unavailable"); + future.cancel(true); + } catch (InterruptedException | ExecutionException e) { + log.error("Error while checking availability '" + name + "', assuming unavailable"); } + return false; } @Override diff --git a/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/mapping/model/MetaStoreMappingImplTest.java b/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/mapping/model/MetaStoreMappingImplTest.java index a65f893cd..57b8692e6 100644 --- a/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/mapping/model/MetaStoreMappingImplTest.java +++ b/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/mapping/model/MetaStoreMappingImplTest.java @@ -1,5 +1,5 @@ /** - * Copyright (C) 2016-2021 Expedia, Inc. + * Copyright (C) 2016-2023 Expedia, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,12 +19,14 @@ import static org.hamcrest.CoreMatchers.sameInstance; import static org.hamcrest.MatcherAssert.assertThat; import static org.junit.Assert.fail; +import static org.mockito.AdditionalAnswers.answersWithDelay; import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import static com.hotels.bdp.waggledance.api.model.ConnectionType.DIRECT; import static com.hotels.bdp.waggledance.api.model.ConnectionType.TUNNELED; +import static com.hotels.bdp.waggledance.mapping.model.MetaStoreMappingImpl.DEFAULT_AVAILABILITY_TIMEOUT; import java.io.IOException; @@ -115,6 +117,12 @@ public void isNotAvailable() { assertThat(metaStoreMapping.isAvailable(), is(false)); } + @Test + public void isNotAvailableTimeout() throws Exception { + when(client.isOpen()).then(answersWithDelay(DEFAULT_AVAILABILITY_TIMEOUT+LATENCY+10, a -> true)); + assertThat(metaStoreMapping.isAvailable(), is(false)); + } + @Test public void isAvailableTunnelled() throws Exception { when(client.isOpen()).thenReturn(true);