From 4b77d3fb3ae06f620c0cff3c47dd3643b6272f10 Mon Sep 17 00:00:00 2001 From: lakde Date: Mon, 19 Aug 2024 11:38:26 -0500 Subject: [PATCH 01/39] Initial Changes. Signed-off-by: lakde --- .../lib/core/models/ExtensionConfig.java | 12 +-- .../lib/core/models/features/Extension.java | 12 +++ .../naksha/lib/extmanager/ExtensionCache.java | 19 +++-- .../extmanager/helpers/AmazonS3Helper.java | 17 ---- .../here/naksha/lib/extmanager/BaseSetup.java | 2 +- .../com/here/naksha/lib/hub/NakshaHub.java | 81 +++++++++++++------ 6 files changed, 81 insertions(+), 62 deletions(-) diff --git a/here-naksha-lib-core/src/main/java/com/here/naksha/lib/core/models/ExtensionConfig.java b/here-naksha-lib-core/src/main/java/com/here/naksha/lib/core/models/ExtensionConfig.java index 36c38e25e..1d7a101a7 100644 --- a/here-naksha-lib-core/src/main/java/com/here/naksha/lib/core/models/ExtensionConfig.java +++ b/here-naksha-lib-core/src/main/java/com/here/naksha/lib/core/models/ExtensionConfig.java @@ -32,7 +32,6 @@ public class ExtensionConfig { public static final String EXPIRY = "expiry"; public static final String EXTENSIONS = "extensions"; public static final String WHITELIST_DELEGATE_CLASSES = "whitelistDelegateClasses"; - public static final String ENV_NAME = "env"; @JsonProperty(EXPIRY) long expiry; @@ -43,19 +42,14 @@ public class ExtensionConfig { @JsonProperty(WHITELIST_DELEGATE_CLASSES) List whitelistDelegateClasses; - @JsonProperty(ENV_NAME) - String env; - @JsonCreator public ExtensionConfig( @JsonProperty(EXPIRY) @NotNull Long expiry, @JsonProperty(EXTENSIONS) @Nullable List extensions, - @JsonProperty(WHITELIST_DELEGATE_CLASSES) @Nullable List whitelistDelegateClasses, - @JsonProperty(ENV_NAME) @NotNull String env) { + @JsonProperty(WHITELIST_DELEGATE_CLASSES) @Nullable List whitelistDelegateClasses) { this.expiry = expiry; this.extensions = extensions; this.whitelistDelegateClasses = whitelistDelegateClasses; - this.env = env; } public long getExpiry() { @@ -69,8 +63,4 @@ public List getExtensions() { public List getWhilelistDelegateClass() { return whitelistDelegateClasses; } - - public String getEnv() { - return env; - } } diff --git a/here-naksha-lib-core/src/main/java/com/here/naksha/lib/core/models/features/Extension.java b/here-naksha-lib-core/src/main/java/com/here/naksha/lib/core/models/features/Extension.java index 29aad5f86..031ed3868 100644 --- a/here-naksha-lib-core/src/main/java/com/here/naksha/lib/core/models/features/Extension.java +++ b/here-naksha-lib-core/src/main/java/com/here/naksha/lib/core/models/features/Extension.java @@ -36,6 +36,7 @@ public class Extension extends XyzFeature { public static final String URL = "url"; public static final String VERSION = "version"; public static final String INIT_CLASS_NAME = "initClassName"; + public static final String ENV = "env"; @JsonProperty(URL) String url; @@ -46,6 +47,9 @@ public class Extension extends XyzFeature { @JsonProperty(INIT_CLASS_NAME) String initClassName; + @JsonProperty(ENV) + String env; + /** * Create an extension. * @@ -78,4 +82,12 @@ public String getVersion() { public String getInitClassName() { return initClassName; } + + public void setEnv(String env) { + this.env = env; + } + + public String getEnv() { + return env; + } } diff --git a/here-naksha-lib-ext-manager/src/main/java/com/here/naksha/lib/extmanager/ExtensionCache.java b/here-naksha-lib-ext-manager/src/main/java/com/here/naksha/lib/extmanager/ExtensionCache.java index ec1fdc17b..5118cb5b3 100644 --- a/here-naksha-lib-ext-manager/src/main/java/com/here/naksha/lib/extmanager/ExtensionCache.java +++ b/here-naksha-lib-ext-manager/src/main/java/com/here/naksha/lib/extmanager/ExtensionCache.java @@ -83,9 +83,10 @@ protected void buildExtensionCache(ExtensionConfig extensionConfig) { publishIntoCache(result, extensionConfig); }); - // Removing existing extension which has been removed from the configuration - List extIds = - extensionConfig.getExtensions().stream().map(Extension::getId).toList(); + // Removing existing extension which has been removed from the configuration. + List extIds = extensionConfig.getExtensions().stream() + .map(extension -> extension.getEnv() + ":" + extension.getId()) + .toList(); for (String key : loaderCache.keySet()) { if (!extIds.contains(key)) { @@ -100,6 +101,7 @@ protected void buildExtensionCache(ExtensionConfig extensionConfig) { private void publishIntoCache(KVPair result, ExtensionConfig extensionConfig) { if (result != null && result.getValue() != null) { final Extension extension = result.getKey(); + final String extensionIdWthEnv = extension.getEnv() + ":" + extension.getId(); final File jarFile = result.getValue(); ClassLoader loader; try { @@ -121,7 +123,7 @@ private void publishIntoCache(KVPair result, ExtensionConfig ex logger.error( "Failed to instantiate class {} for extension {} ", extension.getInitClassName(), - extension.getId(), + extensionIdWthEnv, e); return; } @@ -129,21 +131,22 @@ private void publishIntoCache(KVPair result, ExtensionConfig ex if (!isNullOrEmpty(extension.getInitClassName())) logger.info( "Extension {} initialization using initClassName {} done successfully.", - extension.getId(), + extensionIdWthEnv, extension.getInitClassName()); loaderCache.put(extension.getId(), new KVPair(extension, loader)); PluginCache.removeExtensionCache(extension.getId()); logger.info( "Extension id={}, version={} is successfully loaded into the cache, using Jar at {} for env={}.", - extension.getId(), + extensionIdWthEnv, extension.getVersion(), extension.getUrl().substring(extension.getUrl().lastIndexOf("/") + 1), - extensionConfig.getEnv()); + extension.getEnv()); } } private boolean isLoaderMappingExist(Extension extension) { - KVPair existingMapping = loaderCache.get(extension.getId()); + final String extensionIdWthEnv = extension.getEnv() + ":" + extension.getId(); + KVPair existingMapping = loaderCache.get(extensionIdWthEnv); if (existingMapping == null) return false; final Extension exExtension = existingMapping.getKey(); diff --git a/here-naksha-lib-ext-manager/src/main/java/com/here/naksha/lib/extmanager/helpers/AmazonS3Helper.java b/here-naksha-lib-ext-manager/src/main/java/com/here/naksha/lib/extmanager/helpers/AmazonS3Helper.java index 5ed3364b3..c811cc035 100644 --- a/here-naksha-lib-ext-manager/src/main/java/com/here/naksha/lib/extmanager/helpers/AmazonS3Helper.java +++ b/here-naksha-lib-ext-manager/src/main/java/com/here/naksha/lib/extmanager/helpers/AmazonS3Helper.java @@ -26,14 +26,11 @@ import java.io.InputStream; import java.io.InputStreamReader; import java.net.URI; -import java.util.List; import org.jetbrains.annotations.NotNull; import software.amazon.awssdk.services.s3.S3Client; import software.amazon.awssdk.services.s3.S3Uri; import software.amazon.awssdk.services.s3.model.GetObjectRequest; import software.amazon.awssdk.services.s3.model.GetObjectRequest.Builder; -import software.amazon.awssdk.services.s3.model.ListObjectsRequest; -import software.amazon.awssdk.services.s3.model.ListObjectsResponse; public class AmazonS3Helper implements FileClient { private static S3Client s3Client; @@ -93,20 +90,6 @@ public String getFileContent(String url) throws IOException { } } - public List listKeysInBucket(String url) { - S3Uri fileUri = getS3Uri(url); - String delimiter = "/"; - - ListObjectsRequest.Builder listObjectsRequestBuilder = - ListObjectsRequest.builder().bucket(fileUri.bucket().get()).delimiter(delimiter); - - if (fileUri.key().isPresent()) - listObjectsRequestBuilder.prefix(fileUri.key().get()); - - ListObjectsResponse response = getS3Client().listObjects(listObjectsRequestBuilder.build()); - return response.commonPrefixes().stream().map(cm -> cm.prefix()).toList(); - } - public S3Uri getS3Uri(String url) { URI uri = URI.create(url); return getS3Client().utilities().parseUri(uri); diff --git a/here-naksha-lib-ext-manager/src/test/java/com/here/naksha/lib/extmanager/BaseSetup.java b/here-naksha-lib-ext-manager/src/test/java/com/here/naksha/lib/extmanager/BaseSetup.java index f997e0f67..fc5b546e1 100644 --- a/here-naksha-lib-ext-manager/src/test/java/com/here/naksha/lib/extmanager/BaseSetup.java +++ b/here-naksha-lib-ext-manager/src/test/java/com/here/naksha/lib/extmanager/BaseSetup.java @@ -25,6 +25,6 @@ public ExtensionConfig getExtensionConfig() { } catch (IOException e) { throw new RuntimeException(e); } - return new ExtensionConfig(System.currentTimeMillis() + 6000, list,whitelistUrls,"test"); + return new ExtensionConfig(System.currentTimeMillis() + 6000, list,whitelistUrls); } } diff --git a/here-naksha-lib-hub/src/main/java/com/here/naksha/lib/hub/NakshaHub.java b/here-naksha-lib-hub/src/main/java/com/here/naksha/lib/hub/NakshaHub.java index be3f1e175..06c183600 100644 --- a/here-naksha-lib-hub/src/main/java/com/here/naksha/lib/hub/NakshaHub.java +++ b/here-naksha-lib-hub/src/main/java/com/here/naksha/lib/hub/NakshaHub.java @@ -18,6 +18,7 @@ */ package com.here.naksha.lib.hub; +import static com.here.naksha.lib.core.NakshaAdminCollection.EVENT_HANDLERS; import static com.here.naksha.lib.core.exceptions.UncheckedException.unchecked; import static com.here.naksha.lib.core.models.PluginCache.getStorageConstructor; import static com.here.naksha.lib.core.util.storage.RequestHelper.createFeatureRequest; @@ -25,6 +26,7 @@ import static com.here.naksha.lib.core.util.storage.RequestHelper.readFeaturesByIdsRequest; import static com.here.naksha.lib.core.util.storage.RequestHelper.upsertFeaturesRequest; import static com.here.naksha.lib.core.util.storage.ResultHelper.readFeatureFromResult; +import static com.here.naksha.lib.core.util.storage.ResultHelper.readFeaturesFromResult; import com.fasterxml.jackson.databind.ObjectMapper; import com.here.naksha.lib.core.*; @@ -35,19 +37,16 @@ import com.here.naksha.lib.core.models.XyzError; import com.here.naksha.lib.core.models.features.Extension; import com.here.naksha.lib.core.models.geojson.implementation.XyzFeature; +import com.here.naksha.lib.core.models.naksha.EventHandler; import com.here.naksha.lib.core.models.naksha.Storage; import com.here.naksha.lib.core.models.naksha.XyzCollection; -import com.here.naksha.lib.core.models.storage.EExecutedOp; -import com.here.naksha.lib.core.models.storage.ErrorResult; -import com.here.naksha.lib.core.models.storage.ForwardCursor; -import com.here.naksha.lib.core.models.storage.Result; -import com.here.naksha.lib.core.models.storage.WriteXyzCollections; -import com.here.naksha.lib.core.models.storage.XyzCollectionCodec; +import com.here.naksha.lib.core.models.storage.*; import com.here.naksha.lib.core.storage.IReadSession; import com.here.naksha.lib.core.storage.IStorage; import com.here.naksha.lib.core.storage.IWriteSession; import com.here.naksha.lib.core.util.IoHelp; import com.here.naksha.lib.core.util.json.Json; +import com.here.naksha.lib.core.util.storage.RequestHelper; import com.here.naksha.lib.core.util.storage.ResultHelper; import com.here.naksha.lib.core.view.ViewDeserialize; import com.here.naksha.lib.extmanager.ExtensionManager; @@ -56,9 +55,9 @@ import com.here.naksha.lib.hub.storages.NHAdminStorage; import com.here.naksha.lib.hub.storages.NHSpaceStorage; import com.here.naksha.lib.psql.PsqlStorage; -import java.util.ArrayList; -import java.util.List; -import java.util.NoSuchElementException; + +import java.util.*; + import org.jetbrains.annotations.ApiStatus; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; @@ -98,6 +97,11 @@ public class NakshaHub implements INaksha { */ protected @NotNull IExtensionManager extensionManager; + /** + * The extensionId property path in handler json. + */ + protected static final String[] EXTN_ID_PROP_PATH = {"extensionId"}; + @ApiStatus.AvailableSince(NakshaVersion.v2_0_7) public NakshaHub( final @NotNull String appName, @@ -239,7 +243,7 @@ private static WriteXyzCollections createAdminCollectionsRequest() { } else { try { List nakshaHubConfigs = - ResultHelper.readFeaturesFromResult(rdResult, NakshaHubConfig.class); + readFeaturesFromResult(rdResult, NakshaHubConfig.class); for (final NakshaHubConfig cfg : nakshaHubConfigs) { if (cfg.getId().equals(configId)) { customDbCfg = cfg; @@ -292,28 +296,57 @@ private static WriteXyzCollections createAdminCollectionsRequest() { @Override public @NotNull ExtensionConfig getExtensionConfig() { + // Create ReadFeatures Request to read all handlers where extensionId not null from Admin DB + final NakshaContext nakshaContext = new NakshaContext().withAppId(NakshaHubConfig.defaultAppName()); + final ReadFeatures request = new ReadFeatures(EVENT_HANDLERS); + final PRef pref = RequestHelper.pRefFromPropPath(EXTN_ID_PROP_PATH); + POp notNullCondition = POp.exists(pref); + request.setPropertyOp(notNullCondition); + + final Result rdResult; + try (IReadSession readSession = getAdminStorage().newReadSession(nakshaContext, false)) { + rdResult = readSession.execute(request); + } catch (Exception e) { + logger.error("Failed reading from collections {}. ", request.getCollections(), e); + throw new RuntimeException("Failed to execute single read session", e); + } + final List eventHandlers; + try { + eventHandlers = readFeaturesFromResult(rdResult, EventHandler.class); + } catch (NoCursor e) { + logger.error("NoCursor exception encountered", e); + throw new RuntimeException("Failed to open cursor", e); + } + + Set extensionIds = new HashSet<>(); + for (EventHandler eventHandler : eventHandlers) { + String extensionId = eventHandler.getExtensionId(); + if (extensionId != null && extensionId.contains(":")) { + extensionIds.add(extensionId); + } else { + logger.error("Environment is missing for an extension Id"); + } + } + final ExtensionConfigParams extensionConfigParams = nakshaHubConfig.extensionConfigParams; if (!extensionConfigParams.extensionRootPath.startsWith("s3://")) throw new UnsupportedOperationException( "ExtensionRootPath must be a valid s3 bucket url which should be prefixed with s3://"); - List extList = loadExtensionConfigFromS3(extensionConfigParams.getExtensionRootPath()); + List extList = loadExtensionConfigFromS3(extensionConfigParams.getExtensionRootPath(), extensionIds); return new ExtensionConfig( System.currentTimeMillis() + extensionConfigParams.getIntervalMs(), extList, - extensionConfigParams.getWhiteListClasses(), - this.nakshaHubConfig.env.toLowerCase()); + extensionConfigParams.getWhiteListClasses()); } - private List loadExtensionConfigFromS3(String extensionRootPath) { + private List loadExtensionConfigFromS3(String extensionRootPath, Set extensionIds) { AmazonS3Helper s3Helper = new AmazonS3Helper(); - final String bucketName = s3Helper.getS3Uri(extensionRootPath).bucket().get(); - - List list = s3Helper.listKeysInBucket(extensionRootPath); List extList = new ArrayList<>(); - list.stream().forEach(extensionPath -> { - String filePath = - "s3://" + bucketName + "/" + extensionPath + "latest-" + nakshaHubConfig.env.toLowerCase() + ".txt"; + extensionIds.forEach(extensionId -> { + String env = extensionId.split(":")[0]; + String extensionIdWotEnv = extensionId.split(":")[1]; + String filePath = extensionRootPath + extensionIdWotEnv + "/" + "latest-" + env.toLowerCase() + ".txt"; String version; try { version = s3Helper.getFileContent(filePath); @@ -322,11 +355,8 @@ private List loadExtensionConfigFromS3(String extensionRootPath) { return; } - String bits[] = extensionPath.split("/"); - String extensionId = bits[bits.length - 1]; - - filePath = "s3://" + bucketName + "/" + extensionPath + extensionId + "-" + version + "." - + nakshaHubConfig.env.toLowerCase().toLowerCase() + ".json"; + filePath = extensionRootPath + extensionIdWotEnv + "/" + extensionIdWotEnv + "-" + version + "." + + env.toLowerCase() + ".json"; String exJson; try { exJson = s3Helper.getFileContent(filePath); @@ -337,6 +367,7 @@ private List loadExtensionConfigFromS3(String extensionRootPath) { Extension extension; try { extension = new ObjectMapper().readValue(exJson, Extension.class); + extension.setEnv(env); extList.add(extension); } catch (Exception e) { logger.error("Failed to convert extension meta data to Extension object. {} ", exJson, e); From ba85d9d82c36d6e4a44b2bb872253fb4bf7222c7 Mon Sep 17 00:00:00 2001 From: lakde Date: Mon, 19 Aug 2024 16:41:21 -0500 Subject: [PATCH 02/39] Fix Unit Tests Signed-off-by: lakde --- .../here/naksha/lib/extmanager/ExtensionCache.java | 14 +++++++------- .../naksha/lib/extmanager/ExtensionCacheTest.java | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/here-naksha-lib-ext-manager/src/main/java/com/here/naksha/lib/extmanager/ExtensionCache.java b/here-naksha-lib-ext-manager/src/main/java/com/here/naksha/lib/extmanager/ExtensionCache.java index 5118cb5b3..e45229fb1 100644 --- a/here-naksha-lib-ext-manager/src/main/java/com/here/naksha/lib/extmanager/ExtensionCache.java +++ b/here-naksha-lib-ext-manager/src/main/java/com/here/naksha/lib/extmanager/ExtensionCache.java @@ -85,8 +85,8 @@ protected void buildExtensionCache(ExtensionConfig extensionConfig) { // Removing existing extension which has been removed from the configuration. List extIds = extensionConfig.getExtensions().stream() - .map(extension -> extension.getEnv() + ":" + extension.getId()) - .toList(); + .map(extension -> extension.getEnv() + ":" + extension.getId()) + .toList(); for (String key : loaderCache.keySet()) { if (!extIds.contains(key)) { @@ -123,7 +123,7 @@ private void publishIntoCache(KVPair result, ExtensionConfig ex logger.error( "Failed to instantiate class {} for extension {} ", extension.getInitClassName(), - extensionIdWthEnv, + extensionIdWthEnv, e); return; } @@ -131,13 +131,13 @@ private void publishIntoCache(KVPair result, ExtensionConfig ex if (!isNullOrEmpty(extension.getInitClassName())) logger.info( "Extension {} initialization using initClassName {} done successfully.", - extensionIdWthEnv, + extensionIdWthEnv, extension.getInitClassName()); - loaderCache.put(extension.getId(), new KVPair(extension, loader)); - PluginCache.removeExtensionCache(extension.getId()); + loaderCache.put(extensionIdWthEnv, new KVPair(extension, loader)); + PluginCache.removeExtensionCache(extensionIdWthEnv); logger.info( "Extension id={}, version={} is successfully loaded into the cache, using Jar at {} for env={}.", - extensionIdWthEnv, + extensionIdWthEnv, extension.getVersion(), extension.getUrl().substring(extension.getUrl().lastIndexOf("/") + 1), extension.getEnv()); diff --git a/here-naksha-lib-ext-manager/src/test/java/com/here/naksha/lib/extmanager/ExtensionCacheTest.java b/here-naksha-lib-ext-manager/src/test/java/com/here/naksha/lib/extmanager/ExtensionCacheTest.java index 6c203456e..91796031d 100644 --- a/here-naksha-lib-ext-manager/src/test/java/com/here/naksha/lib/extmanager/ExtensionCacheTest.java +++ b/here-naksha-lib-ext-manager/src/test/java/com/here/naksha/lib/extmanager/ExtensionCacheTest.java @@ -76,7 +76,7 @@ public void testGetClassLoaderById() throws IOException { extensionCache.buildExtensionCache(extensionConfig); Assertions.assertEquals(2,extensionCache.getCacheLength()); - ClassLoader loader=extensionCache.getClassLoaderById(extensionConfig.getExtensions().get(0).getId()); + ClassLoader loader=extensionCache.getClassLoaderById(extensionConfig.getExtensions().get(0).getEnv()+":"+extensionConfig.getExtensions().get(0).getId()); Assertions.assertNotNull(loader); Assertions.assertEquals(classLoader,loader); } From 1befc37be24dd8da48ddefaf348bff937b0a168d Mon Sep 17 00:00:00 2001 From: lakde Date: Wed, 18 Sep 2024 09:32:57 -0500 Subject: [PATCH 03/39] Review Fixes Signed-off-by: lakde --- gradle.properties | 2 +- .../src/main/resources/swagger/openapi.yaml | 2 +- .../src/main/java/com/here/naksha/lib/hub/NakshaHub.java | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/gradle.properties b/gradle.properties index 98293820d..92a7e516e 100644 --- a/gradle.properties +++ b/gradle.properties @@ -11,4 +11,4 @@ mavenPassword=YourPassword # - here-naksha-lib-core/NakshaVersion (static property: latest) # - here-naksha-lib-psql/resources/naksha_plpgsql.sql (method: naksha_version) # - here-naksha-app-service/src/main/resources/swagger/openapi.yaml (info.version property) -version=2.1.0 +version=2.2.0 diff --git a/here-naksha-app-service/src/main/resources/swagger/openapi.yaml b/here-naksha-app-service/src/main/resources/swagger/openapi.yaml index 915b3ed00..5a080a407 100644 --- a/here-naksha-app-service/src/main/resources/swagger/openapi.yaml +++ b/here-naksha-app-service/src/main/resources/swagger/openapi.yaml @@ -12,7 +12,7 @@ servers: info: title: "Naskha Hub-API" description: "Naksha Hub-API is a REST API to provide simple access to geo data." - version: "2.1.0" + version: "2.2.0" security: - AccessToken: [ ] diff --git a/here-naksha-lib-hub/src/main/java/com/here/naksha/lib/hub/NakshaHub.java b/here-naksha-lib-hub/src/main/java/com/here/naksha/lib/hub/NakshaHub.java index 06c183600..a6f453bf2 100644 --- a/here-naksha-lib-hub/src/main/java/com/here/naksha/lib/hub/NakshaHub.java +++ b/here-naksha-lib-hub/src/main/java/com/here/naksha/lib/hub/NakshaHub.java @@ -307,8 +307,8 @@ private static WriteXyzCollections createAdminCollectionsRequest() { try (IReadSession readSession = getAdminStorage().newReadSession(nakshaContext, false)) { rdResult = readSession.execute(request); } catch (Exception e) { - logger.error("Failed reading from collections {}. ", request.getCollections(), e); - throw new RuntimeException("Failed to execute single read session", e); + logger.error("Failed during reading extension handler configurations from collections {}. ", request.getCollections(), e); + throw new RuntimeException("Failed reading extension handler configurations", e); } final List eventHandlers; try { From 58995d321ea67e817a1681e214a11abfa7883c38 Mon Sep 17 00:00:00 2001 From: Rohit Lakde <37445331+rlakde@users.noreply.github.com> Date: Fri, 25 Oct 2024 08:56:14 -0700 Subject: [PATCH 04/39] Update openapi.yaml --- here-naksha-app-service/src/main/resources/swagger/openapi.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/here-naksha-app-service/src/main/resources/swagger/openapi.yaml b/here-naksha-app-service/src/main/resources/swagger/openapi.yaml index 4cf34add4..55ee704f1 100644 --- a/here-naksha-app-service/src/main/resources/swagger/openapi.yaml +++ b/here-naksha-app-service/src/main/resources/swagger/openapi.yaml @@ -12,7 +12,7 @@ servers: info: title: "Naskha Hub-API" description: "Naksha Hub-API is a REST API to provide simple access to geo data." - version: "2.2.0" + version: "2.1.5" security: - AccessToken: [ ] From 7ec0f8624f9b7d58a3ba560cfe737c0c76a4612d Mon Sep 17 00:00:00 2001 From: Rohit Lakde <37445331+rlakde@users.noreply.github.com> Date: Fri, 25 Oct 2024 08:56:40 -0700 Subject: [PATCH 05/39] Update gradle.properties --- gradle.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle.properties b/gradle.properties index 20ca314ce..793f190a6 100644 --- a/gradle.properties +++ b/gradle.properties @@ -12,4 +12,4 @@ mavenPassword=YourPassword # - here-naksha-lib-psql/resources/naksha_plpgsql.sql (method: naksha_version) # - here-naksha-app-service/src/main/resources/swagger/openapi.yaml (info.version property) -version=2.2.0 +version=2.1.5 From acc577027755159a4ff7c2706190f4bc2e35959f Mon Sep 17 00:00:00 2001 From: Rohit Lakde <37445331+rlakde@users.noreply.github.com> Date: Fri, 25 Oct 2024 08:58:41 -0700 Subject: [PATCH 06/39] Update gradle.properties --- gradle.properties | 1 - 1 file changed, 1 deletion(-) diff --git a/gradle.properties b/gradle.properties index 793f190a6..9b66cc554 100644 --- a/gradle.properties +++ b/gradle.properties @@ -11,5 +11,4 @@ mavenPassword=YourPassword # - here-naksha-lib-core/NakshaVersion (static property: latest) # - here-naksha-lib-psql/resources/naksha_plpgsql.sql (method: naksha_version) # - here-naksha-app-service/src/main/resources/swagger/openapi.yaml (info.version property) - version=2.1.5 From b07480f42478b0a3c79be8c03b3409a7f0d6120f Mon Sep 17 00:00:00 2001 From: Rohit Lakde <37445331+rlakde@users.noreply.github.com> Date: Fri, 25 Oct 2024 08:59:17 -0700 Subject: [PATCH 07/39] Rebase CASL-321 (#371) * Fixed array patching (#365) * Fixed array patching * corrected patcher test expectation * Fix thread stuck issue + addtnl logs (#368) * Added logs to troubleshoot thread hanging issue * CASL-258 unreleased lock fix (#358) * Fixed array patching (#365) (#367) * corrected patcher test expectation * updated version * updated changelog --------- Co-authored-by: Pawel Mazurek <52866094+cyberhead-pl@users.noreply.github.com> --------- Co-authored-by: Hiren Patel <80465571+hirenkp2000@users.noreply.github.com> Co-authored-by: Pawel Mazurek <52866094+cyberhead-pl@users.noreply.github.com> --- CHANGELOG.md | 10 ++ .../http/tasks/WriteFeatureApiTask.java | 4 +- .../src/main/resources/swagger/openapi.yaml | 2 +- .../here/naksha/lib/core/AbstractTask.java | 17 +++ .../here/naksha/lib/core/NakshaContext.java | 1 + .../naksha/lib/core/storage/IStorage.java | 8 +- .../com/here/naksha/lib/core/util/FibMap.java | 12 ++ .../lib/core/util/diff/PatcherUtils.java | 27 +++-- .../core/util/fib/FibLinearProbeTable.java | 2 +- .../here/naksha/lib/core/util/fib/FibSet.java | 12 ++ .../lib/core/util/json/JsonFieldBool.java | 5 + .../lib/core/util/json/JsonFieldByte.java | 5 + .../lib/core/util/json/JsonFieldChar.java | 5 + .../lib/core/util/json/JsonFieldShort.java | 5 + .../naksha/lib/core/util/json/JsonMap.java | 9 ++ .../lib/core/util/diff/PatcherTest.java | 33 +++++- .../util/fib/FibLinearProbeTableTest.java | 41 +++++++ .../feature_3_patched_to_4_no_remove.json | 4 +- .../resources/patcher/topology/existing.json | 109 ++++++++++++++++++ .../resources/patcher/topology/expected.json | 109 ++++++++++++++++++ .../resources/patcher/topology/input.json | 92 +++++++++++++++ 21 files changed, 487 insertions(+), 25 deletions(-) create mode 100644 here-naksha-lib-core/src/test/resources/patcher/topology/existing.json create mode 100644 here-naksha-lib-core/src/test/resources/patcher/topology/expected.json create mode 100644 here-naksha-lib-core/src/test/resources/patcher/topology/input.json diff --git a/CHANGELOG.md b/CHANGELOG.md index c7922bd32..ec3443aaf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,13 @@ +## Naksha_2.1.5 + +- Fixed thread hanging issue fixed by avoiding indefinite locking in `FibLinearProbeTable.java` + +## Naksha_2.1.4 + +- Fixes: + - Patch API (POST /features) fixed to `replace` entire array instead of `append` nodes during patch operation, which otherwise prevents removal of the node even-though is desired + + ## Naksha_1.1.1 - Fixes: diff --git a/here-naksha-app-service/src/main/java/com/here/naksha/app/service/http/tasks/WriteFeatureApiTask.java b/here-naksha-app-service/src/main/java/com/here/naksha/app/service/http/tasks/WriteFeatureApiTask.java index 628630f59..4ab53e69e 100644 --- a/here-naksha-app-service/src/main/java/com/here/naksha/app/service/http/tasks/WriteFeatureApiTask.java +++ b/here-naksha-app-service/src/main/java/com/here/naksha/app/service/http/tasks/WriteFeatureApiTask.java @@ -20,7 +20,7 @@ import static com.here.naksha.app.service.http.apis.ApiParams.*; import static com.here.naksha.common.http.apis.ApiParamsConst.*; -import static com.here.naksha.lib.core.util.diff.PatcherUtils.removeAllRemoveOp; +import static com.here.naksha.lib.core.util.diff.PatcherUtils.removeAllRemoveOpExceptForList; import static com.here.naksha.lib.core.util.storage.ResultHelper.readFeaturesFromResult; import com.fasterxml.jackson.core.JsonProcessingException; @@ -389,7 +389,7 @@ private List performInMemoryPatching( if (inputFeature.getId().equals(storageFeature.getId())) { // we found matching feature in storage, so we take patched version of the feature final Difference difference = Patcher.getDifference(storageFeature, inputFeature); - final Difference diffNoRemoveOp = removeAllRemoveOp(difference); + final Difference diffNoRemoveOp = removeAllRemoveOpExceptForList(difference); featureToPatch = Patcher.patch(storageFeature, diffNoRemoveOp); break; } diff --git a/here-naksha-app-service/src/main/resources/swagger/openapi.yaml b/here-naksha-app-service/src/main/resources/swagger/openapi.yaml index 55ee704f1..f3b01152a 100644 --- a/here-naksha-app-service/src/main/resources/swagger/openapi.yaml +++ b/here-naksha-app-service/src/main/resources/swagger/openapi.yaml @@ -10,7 +10,7 @@ servers: - url: "https://naksha-v2.ext.mapcreator.here.com/" description: "PRD" info: - title: "Naskha Hub-API" + title: "Naksha Hub-API" description: "Naksha Hub-API is a REST API to provide simple access to geo data." version: "2.1.5" diff --git a/here-naksha-lib-core/src/main/java/com/here/naksha/lib/core/AbstractTask.java b/here-naksha-lib-core/src/main/java/com/here/naksha/lib/core/AbstractTask.java index b0c7c7503..949eb0476 100644 --- a/here-naksha-lib-core/src/main/java/com/here/naksha/lib/core/AbstractTask.java +++ b/here-naksha-lib-core/src/main/java/com/here/naksha/lib/core/AbstractTask.java @@ -613,6 +613,9 @@ private void incInstanceLevelUsage(String actorId, long limit) { break; } // Failed, conflict, repeat + log.info( + "Concurrency conflict while incrementing instance level threadCount from {}. Will retry...", + threadCount); } } @@ -649,6 +652,9 @@ private void incActorLevelUsage(String actorId, long limit) { if (counter == null) { Long existing = actorUsageMap.putIfAbsent(actorId, 1L); if (existing != null) { + log.info( + "Concurrency conflict while initializing threadCount to 1 for actorId [{}]. Will retry...", + actorId); continue; // Repeat, conflict with other thread } return; @@ -669,6 +675,10 @@ private void incActorLevelUsage(String actorId, long limit) { break; } // Failed, conflict, repeat + log.info( + "Concurrency conflict while incrementing actor level threadCount from {} for actorId [{}]. Will retry...", + counter, + actorId); } } @@ -693,6 +703,9 @@ private void decActorLevelUsage(String actorId) { log.error("Invalid actor usage value for actor: " + actorId + " value: " + current); } if (!actorUsageMap.remove(actorId, current)) { + log.info( + "Concurrency conflict while removing actor level threadCount for actorId [{}]. Will retry...", + actorId); continue; } break; @@ -700,6 +713,10 @@ private void decActorLevelUsage(String actorId) { break; } // Failed, repeat, conflict with other thread + log.info( + "Concurrency conflict while decrementing actor level threadCount from {} for actorId [{}]. Will retry...", + current, + actorId); } } } diff --git a/here-naksha-lib-core/src/main/java/com/here/naksha/lib/core/NakshaContext.java b/here-naksha-lib-core/src/main/java/com/here/naksha/lib/core/NakshaContext.java index d1302cd25..ff3062dcb 100644 --- a/here-naksha-lib-core/src/main/java/com/here/naksha/lib/core/NakshaContext.java +++ b/here-naksha-lib-core/src/main/java/com/here/naksha/lib/core/NakshaContext.java @@ -221,6 +221,7 @@ public long startNanos() { return newValue; } // Conflict, two threads seem to want to update the same key the same time! + logger.info("Concurrency conflict while updating attachment map for key {}", valueClass); } } diff --git a/here-naksha-lib-core/src/main/java/com/here/naksha/lib/core/storage/IStorage.java b/here-naksha-lib-core/src/main/java/com/here/naksha/lib/core/storage/IStorage.java index f0313a810..c0043bc84 100644 --- a/here-naksha-lib-core/src/main/java/com/here/naksha/lib/core/storage/IStorage.java +++ b/here-naksha-lib-core/src/main/java/com/here/naksha/lib/core/storage/IStorage.java @@ -32,6 +32,8 @@ import org.jetbrains.annotations.ApiStatus.AvailableSince; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * Storage API to gain access to storages. @@ -39,6 +41,8 @@ @AvailableSince(NakshaVersion.v2_0_6) public interface IStorage extends AutoCloseable { + Logger logger = LoggerFactory.getLogger(IStorage.class); + /** * Initializes the storage, create the transaction table, install needed scripts and extensions. * @@ -191,10 +195,12 @@ default void close() { try { shutdown(null).get(); return; - } catch (InterruptedException ignore) { + } catch (InterruptedException ie) { + logger.warn("Exception while shutting down IStorage. ", ie); } catch (Exception e) { throw unchecked(e); } + logger.info("Unable to shutdown IStorage. Will retry..."); } } diff --git a/here-naksha-lib-core/src/main/java/com/here/naksha/lib/core/util/FibMap.java b/here-naksha-lib-core/src/main/java/com/here/naksha/lib/core/util/FibMap.java index 904298d01..444fbbd96 100644 --- a/here-naksha-lib-core/src/main/java/com/here/naksha/lib/core/util/FibMap.java +++ b/here-naksha-lib-core/src/main/java/com/here/naksha/lib/core/util/FibMap.java @@ -29,6 +29,8 @@ import java.util.List; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * Functional implementation of a recursive, thread safe, growing map, based upon iterator = listdiff.iterator(); - while (iterator.hasNext()) { - Difference next = iterator.next(); - if (next == null) continue; - next = removeAllRemoveOp(next); - if (next == null) iterator.remove(); - } - return listdiff; } else if (difference instanceof MapDiff) { final MapDiff mapdiff = (MapDiff) difference; final Iterator> iterator = mapdiff.entrySet().iterator(); while (iterator.hasNext()) { Entry next = iterator.next(); - next.setValue(removeAllRemoveOp(next.getValue())); + next.setValue(removeAllRemoveOpExceptForList(next.getValue())); if (next.getValue() == null) iterator.remove(); } return mapdiff; } + // NOTE - we avoid removal of nodes for ListDiff, to ensure List is always replaced during patch and not + // appended + /*else if (difference instanceof ListDiff) { + final ListDiff listdiff = (ListDiff) difference; + final Iterator iterator = listdiff.iterator(); + while (iterator.hasNext()) { + Difference next = iterator.next(); + if (next == null) continue; + next = removeAllRemoveOpExceptForList(next); + if (next == null) iterator.remove(); + } + return listdiff; + }*/ return difference; } } diff --git a/here-naksha-lib-core/src/main/java/com/here/naksha/lib/core/util/fib/FibLinearProbeTable.java b/here-naksha-lib-core/src/main/java/com/here/naksha/lib/core/util/fib/FibLinearProbeTable.java index a9f39515a..d74d70706 100644 --- a/here-naksha-lib-core/src/main/java/com/here/naksha/lib/core/util/fib/FibLinearProbeTable.java +++ b/here-naksha-lib-core/src/main/java/com/here/naksha/lib/core/util/fib/FibLinearProbeTable.java @@ -82,7 +82,7 @@ ENTRY execute(final @NotNull FibSetOp op, final @NotNull KEY key, final @NotNull if (raw instanceof Reference) { Reference ref = (Reference) raw; entry = (ENTRY) ref.get(); - if (entry == null && lock.tryLock()) { + if (entry == null && (lock.isHeldByCurrentThread() || lock.tryLock())) { locked = true; array[i] = null; SIZE.getAndAdd(fibSet, -1L); diff --git a/here-naksha-lib-core/src/main/java/com/here/naksha/lib/core/util/fib/FibSet.java b/here-naksha-lib-core/src/main/java/com/here/naksha/lib/core/util/fib/FibSet.java index 7033ad47c..928e31af1 100644 --- a/here-naksha-lib-core/src/main/java/com/here/naksha/lib/core/util/fib/FibSet.java +++ b/here-naksha-lib-core/src/main/java/com/here/naksha/lib/core/util/fib/FibSet.java @@ -37,6 +37,8 @@ import org.jetbrains.annotations.ApiStatus.AvailableSince; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * A recursive, thread safe, weak/soft/strong referencing set, based upon > { + private static final Logger logger = LoggerFactory.getLogger(FibSet.class); + /** * The empty array used by default, so that empty maps do not consume memory. */ @@ -452,6 +456,9 @@ ENTRY _execute( if (raw_entry == null) { if (!ARRAY.compareAndSet(array, index, ref, null)) { // Race condition, another thread modified the array slot. + logger.info( + "Concurrency conflict while initializing array value at index {}. Will retry...", + index); continue; } SIZE.getAndAdd(this, -1L); @@ -487,6 +494,7 @@ ENTRY _execute( return (ENTRY) entry; } // Race condition, other thread updated the reference concurrently. + logger.info("Concurrency conflict while setting array value at index {}. Will retry...", index); continue; } assert op == REMOVE; @@ -495,6 +503,7 @@ ENTRY _execute( return (ENTRY) entry; } // Race condition, other thread updated the reference concurrently. + logger.info("Concurrency conflict while nullifying array value at index {}. Will retry...", index); continue; } @@ -516,6 +525,7 @@ ENTRY _execute( return _execute(op, key, key_hash, refType, sub_array, depth + 1); } // Race condition, another thread modified concurrently. + logger.info("Concurrency conflict while setting array value at index {}. Will retry...", index); continue; } @@ -529,6 +539,7 @@ ENTRY _execute( return new_entry; } // Race condition, another thread modified concurrently. + logger.info("Concurrency conflict while setting array value at index {}. Will retry...", index); continue; } @@ -546,6 +557,7 @@ ENTRY _execute( return new_entry; } // Race condition, another thread modified concurrently. + logger.info("Concurrency conflict while initializing array value at index {}. Will retry...", index); } } diff --git a/here-naksha-lib-core/src/main/java/com/here/naksha/lib/core/util/json/JsonFieldBool.java b/here-naksha-lib-core/src/main/java/com/here/naksha/lib/core/util/json/JsonFieldBool.java index f0a96c6c5..61e056c75 100644 --- a/here-naksha-lib-core/src/main/java/com/here/naksha/lib/core/util/json/JsonFieldBool.java +++ b/here-naksha-lib-core/src/main/java/com/here/naksha/lib/core/util/json/JsonFieldBool.java @@ -23,9 +23,13 @@ import java.nio.ByteOrder; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public final class JsonFieldBool extends JsonField { + private static final Logger logger = LoggerFactory.getLogger(JsonFieldBool.class); + JsonFieldBool( @NotNull JsonClass jsonClass, @NotNull Field javaField, @@ -94,6 +98,7 @@ public boolean _compareAndSwap(@NotNull OBJECT object, Boolean expected, Boolean return true; } // We need to loop, because possibly some code modified bytes we're not interested in. + logger.info("Concurrency conflict while setting value at offset {}. Will retry...", offset); } } diff --git a/here-naksha-lib-core/src/main/java/com/here/naksha/lib/core/util/json/JsonFieldByte.java b/here-naksha-lib-core/src/main/java/com/here/naksha/lib/core/util/json/JsonFieldByte.java index ce17f34b1..b84c07e0b 100644 --- a/here-naksha-lib-core/src/main/java/com/here/naksha/lib/core/util/json/JsonFieldByte.java +++ b/here-naksha-lib-core/src/main/java/com/here/naksha/lib/core/util/json/JsonFieldByte.java @@ -23,9 +23,13 @@ import java.nio.ByteOrder; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public final class JsonFieldByte extends JsonField { + private static final Logger logger = LoggerFactory.getLogger(JsonFieldByte.class); + JsonFieldByte( @NotNull JsonClass jsonClass, @NotNull Field javaField, @@ -100,6 +104,7 @@ public boolean _compareAndSwap(@NotNull OBJECT object, Byte expected, Byte value return true; } // We need to loop, because possibly some code modified bytes we're not interested in. + logger.info("Concurrency conflict while setting value at offset {}. Will retry...", offset); } } diff --git a/here-naksha-lib-core/src/main/java/com/here/naksha/lib/core/util/json/JsonFieldChar.java b/here-naksha-lib-core/src/main/java/com/here/naksha/lib/core/util/json/JsonFieldChar.java index fe42df8e9..72cb31fe2 100644 --- a/here-naksha-lib-core/src/main/java/com/here/naksha/lib/core/util/json/JsonFieldChar.java +++ b/here-naksha-lib-core/src/main/java/com/here/naksha/lib/core/util/json/JsonFieldChar.java @@ -23,9 +23,13 @@ import java.nio.ByteOrder; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public final class JsonFieldChar extends JsonField { + private static final Logger logger = LoggerFactory.getLogger(JsonFieldChar.class); + JsonFieldChar( @NotNull JsonClass jsonClass, @NotNull Field javaField, @@ -101,6 +105,7 @@ public boolean _compareAndSwap(@NotNull OBJECT object, Character expected, Chara return true; } // We need to loop, because possibly some code modified bytes we're not interested in. + logger.info("Concurrency conflict while setting value at offset {}. Will retry...", offset); } } diff --git a/here-naksha-lib-core/src/main/java/com/here/naksha/lib/core/util/json/JsonFieldShort.java b/here-naksha-lib-core/src/main/java/com/here/naksha/lib/core/util/json/JsonFieldShort.java index e47d7059b..408d78de6 100644 --- a/here-naksha-lib-core/src/main/java/com/here/naksha/lib/core/util/json/JsonFieldShort.java +++ b/here-naksha-lib-core/src/main/java/com/here/naksha/lib/core/util/json/JsonFieldShort.java @@ -23,9 +23,13 @@ import java.nio.ByteOrder; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public final class JsonFieldShort extends JsonField { + private static final Logger logger = LoggerFactory.getLogger(JsonFieldShort.class); + JsonFieldShort( @NotNull JsonClass jsonClass, @NotNull Field javaField, @@ -102,6 +106,7 @@ public boolean _compareAndSwap(@NotNull OBJECT object, Short expected, Short val return true; } // We need to loop, because possibly some code modified bytes we're not interested in. + logger.info("Concurrency conflict while setting value at offset {}. Will retry...", offset); } } diff --git a/here-naksha-lib-core/src/main/java/com/here/naksha/lib/core/util/json/JsonMap.java b/here-naksha-lib-core/src/main/java/com/here/naksha/lib/core/util/json/JsonMap.java index 9129ef13c..d689e1872 100644 --- a/here-naksha-lib-core/src/main/java/com/here/naksha/lib/core/util/json/JsonMap.java +++ b/here-naksha-lib-core/src/main/java/com/here/naksha/lib/core/util/json/JsonMap.java @@ -42,6 +42,8 @@ import org.jetbrains.annotations.ApiStatus.AvailableSince; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * A map that uses {@link String} key and arbitrary values. The map is thread safe for concurrent @@ -55,6 +57,8 @@ public class JsonMap implements Map<@NotNull String, @Nullable Object>, Iterable> { + private static final Logger logger = LoggerFactory.getLogger(JsonMap.class); + /** * Create a new empty map. * @@ -278,6 +282,7 @@ public boolean containsKey(@Nullable Object key) { final Object result = FibMap.put(key, oldValue, newValue, true, rootMutable(), this::intern, this::conflict); if (result instanceof FibMapConflict) { + logger.info("Concurrency conflict while setting value for key {}. Will retry...", key); continue; } assert result == oldValue; @@ -305,6 +310,7 @@ public boolean containsKey(@Nullable Object key) { final Object result = FibMap.put(key, original, newValue, true, rootMutable(), this::intern, this::conflict); if (result instanceof FibMapConflict) { + logger.info("Concurrency conflict while setting value for key {}. Will retry...", key); continue; } assert result == original; @@ -339,6 +345,7 @@ public boolean containsKey(@Nullable Object key) { final Object result = FibMap.put(key, original, UNDEFINED, true, rootMutable(), this::intern, this::conflict); if (result instanceof FibMapConflict) { + logger.info("Concurrency conflict while removing value for key {}. Will retry...", key); continue; } assert result == UNDEFINED; @@ -350,6 +357,7 @@ public boolean containsKey(@Nullable Object key) { final Object result = FibMap.put(key, original, newValue, true, rootMutable(), this::intern, this::conflict); if (result instanceof FibMapConflict) { + logger.info("Concurrency conflict while setting value for key {}. Will retry...", key); continue; } assert result == original; @@ -468,6 +476,7 @@ public void clear() { SIZE.getAndAdd(this, -oldSize); return; } + logger.info("Concurrency conflict while clearing map. Will retry..."); } } diff --git a/here-naksha-lib-core/src/test/java/com/here/naksha/lib/core/util/diff/PatcherTest.java b/here-naksha-lib-core/src/test/java/com/here/naksha/lib/core/util/diff/PatcherTest.java index 4b0955425..c30fc5322 100644 --- a/here-naksha-lib-core/src/test/java/com/here/naksha/lib/core/util/diff/PatcherTest.java +++ b/here-naksha-lib-core/src/test/java/com/here/naksha/lib/core/util/diff/PatcherTest.java @@ -24,15 +24,16 @@ import com.here.naksha.lib.core.util.IoHelp; import com.here.naksha.lib.core.util.json.JsonObject; import com.here.naksha.lib.core.util.json.JsonSerializable; -import java.util.ArrayList; + import java.util.Arrays; import java.util.List; import java.util.stream.Stream; + +import com.here.naksha.test.common.FileUtil; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import org.json.JSONException; import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Named; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; @@ -42,7 +43,7 @@ import java.util.Map; -import static com.here.naksha.lib.core.util.diff.PatcherUtils.removeAllRemoveOp; +import static com.here.naksha.lib.core.util.diff.PatcherUtils.removeAllRemoveOpExceptForList; import static org.junit.jupiter.api.Assertions.*; import static org.junit.jupiter.params.provider.Arguments.arguments; @@ -113,7 +114,7 @@ void testCompareBasicNestedJson() throws JSONException { assertTrue(((MapDiff) nestedArrayDiff34.get(2)).get("willBeDeletedProperty") instanceof RemoveOp); // Modify the whole difference to get rid of all RemoveOp - Difference newDiff34 = removeAllRemoveOp(mapDiff34); + Difference newDiff34 = removeAllRemoveOpExceptForList(mapDiff34); final JsonObject patchedf3 = Patcher.patch(f3,newDiff34); assertNotNull(patchedf3); @@ -158,6 +159,28 @@ void testCompareSameArrayDifferentOrder() throws JSONException { assertNull(newDiff); } + @Test + void testCompareArrayRemovalOfNode() throws JSONException { + final XyzFeature input = + FileUtil.parseJsonFileOrFail("src/test/resources/", "patcher/topology/input.json", XyzFeature.class); + //JsonSerializable.deserialize(IoHelp.readResource("patcher/topology/input.json"), XyzFeature.class); + assertNotNull(input); + final XyzFeature existing = + FileUtil.parseJsonFileOrFail("src/test/resources/", "patcher/topology/existing.json", XyzFeature.class); + //JsonSerializable.deserialize(IoHelp.readResource("patcher/topology/existing.json"), XyzFeature.class); + assertNotNull(existing); + final XyzFeature expected = + FileUtil.parseJsonFileOrFail("src/test/resources/", "patcher/topology/expected.json", XyzFeature.class); + assertNotNull(expected); + + final Difference difference = Patcher.getDifference(existing, input); + final Difference diffNoRemoveOp = removeAllRemoveOpExceptForList(difference); + final XyzFeature patchedFeature = Patcher.patch(existing, diffNoRemoveOp); + assertNotNull(patchedFeature); + + JSONAssert.assertEquals(expected.toString(), patchedFeature.toString(), JSONCompareMode.STRICT); + } + @Test void testPatchingOnlyShuffledArrayProvided() throws JSONException { final JsonObject f3 = @@ -171,7 +194,7 @@ void testPatchingOnlyShuffledArrayProvided() throws JSONException { final Difference diff36 = Patcher.getDifference(f3, f6); assertNotNull(diff36); // Simulate REST API behaviour, ignore all RemoveOp type of Difference - final Difference diff36NoRemove = removeAllRemoveOp(diff36); + final Difference diff36NoRemove = removeAllRemoveOpExceptForList(diff36); final JsonObject patchedf3Tof6 = Patcher.patch(f3, diff36NoRemove); final JsonObject expectedPatchedf3 = diff --git a/here-naksha-lib-core/src/test/java/com/here/naksha/lib/core/util/fib/FibLinearProbeTableTest.java b/here-naksha-lib-core/src/test/java/com/here/naksha/lib/core/util/fib/FibLinearProbeTableTest.java index 1e93c2339..a78bcdf73 100644 --- a/here-naksha-lib-core/src/test/java/com/here/naksha/lib/core/util/fib/FibLinearProbeTableTest.java +++ b/here-naksha-lib-core/src/test/java/com/here/naksha/lib/core/util/fib/FibLinearProbeTableTest.java @@ -29,9 +29,16 @@ import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.fail; import java.lang.ref.SoftReference; import java.lang.ref.WeakReference; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import org.junit.jupiter.api.Test; class FibLinearProbeTableTest { @@ -151,4 +158,38 @@ void test_expansion() { assertEquals(TOTAL_SIZE - i, SET.size); } } + + @Test + void test_Lock() { + final FibSet> SET = new FibSet<>(FibMapEntry::new); + final FibLinearProbeTable> lpt = new FibLinearProbeTable<>(SET, 0); + + /** + * First we put 2 weak references to lpt, then we call gc() - in such scenario next GET call should remove + * all empty references, but to do this it has to acquire lock, and release it at the end. + */ + + lpt.execute(PUT, "foo", WEAK); + lpt.execute(PUT, "foo1", WEAK); + System.gc(); + lpt.execute(GET, "foo1", WEAK); + + // then + ExecutorService executor = Executors.newSingleThreadExecutor(); + Future future = executor.submit(() -> { + lpt.execute(PUT, "foo3", WEAK); + return "done"; + }); + + /** + * Now we try to put new value to lpt in another thread - if previous locks were not released it should + * throw timeout exception. + */ + try { + future.get(1, TimeUnit.SECONDS); + } catch (TimeoutException | InterruptedException | ExecutionException e) { + fail("lock not released! " + e); + } + executor.shutdownNow(); + } } diff --git a/here-naksha-lib-core/src/test/resources/patcher/feature_3_patched_to_4_no_remove.json b/here-naksha-lib-core/src/test/resources/patcher/feature_3_patched_to_4_no_remove.json index ef6767850..a5d12d73b 100644 --- a/here-naksha-lib-core/src/test/resources/patcher/feature_3_patched_to_4_no_remove.json +++ b/here-naksha-lib-core/src/test/resources/patcher/feature_3_patched_to_4_no_remove.json @@ -15,10 +15,8 @@ { "id": "element1", "nestedShouldBeUpdated": "updated", - "willBeDeletedProperty": "not yet", "isAddedProperty": "yes" - }, - "willBeDeletedElement" + } ], "speedLimit": [ { diff --git a/here-naksha-lib-core/src/test/resources/patcher/topology/existing.json b/here-naksha-lib-core/src/test/resources/patcher/topology/existing.json new file mode 100644 index 000000000..57f6ae503 --- /dev/null +++ b/here-naksha-lib-core/src/test/resources/patcher/topology/existing.json @@ -0,0 +1,109 @@ +{ + "id": "778407118", + "bbox": [ + 14.20594, + 50.02377, + 14.20667, + 50.02386 + ], + "type": "Feature", + "momType": "Topology", + "properties": { + "rightAdmin": [ + { + "range": { + "endOffset": 1.0, + "startOffset": 0.0 + }, + "value": { + "id": "urn:here::here:Admin:20445433", + "featureType": "Admin", + "referenceName": "adminId" + }, + "confidence": { + "score": 0.0, + "scoreType": "EXISTENCE", + "updatedOn": "2021-02-17T21:44:27.863Z", + "featureType": "ROAD_ADMIN_INFORMATION", + "updateSource": "INJECTION_TMOB" + } + } + ], + "isLongHaul": [ + { + "range": { + "endOffset": 1.0, + "startOffset": 0.0 + }, + "value": false + } + ], + "isMotorway": [ + { + "range": { + "endOffset": 1, + "startOffset": 0 + }, + "value": false, + "confidence": { + "score": 0.9, + "sources": [ + "BMW_RSD" + ], + "scoreType": "GENERIC", + "updatedOn": "2024-10-15T16:39:51.329Z", + "featureType": "UNDEFINED", + "updateSource": "MODERATED", + "triggeringReferences": [ + { + "momFeatureType": "LogicalRoadSign", + "genericFeatureId": "urn:here::here:logicalroadsign:1622677077480690247" + } + ] + } + }, + { + "range": { + "endOffset": 1.0, + "startOffset": 0.6918789974069862 + }, + "value": true, + "confidence": { + "score": 0.75, + "sources": [ + "BMW_RSD" + ], + "scoreType": "GENERIC", + "updatedOn": "2024-10-05T20:59:55.69448Z", + "featureType": "UNDEFINED", + "updateSource": "WALLE", + "triggeringReferences": [ + { + "momFeatureType": "LogicalRoadSign", + "genericFeatureId": "urn:here::here:logicalroadsign:1622677077480690247" + }, + { + "momFeatureType": "LogicalRoadSign", + "genericFeatureId": "urn:here::here:logicalroadsign:1622677077990856712" + } + ] + } + }, + { + "range": { + "endOffset": 0.6873084947544703, + "startOffset": 0.0 + }, + "value": false + } + ], + "referencePoint": { + "type": "Point", + "coordinates": [ + 14.20667, + 50.02379, + 416.38 + ] + } + } +} \ No newline at end of file diff --git a/here-naksha-lib-core/src/test/resources/patcher/topology/expected.json b/here-naksha-lib-core/src/test/resources/patcher/topology/expected.json new file mode 100644 index 000000000..390e16cb6 --- /dev/null +++ b/here-naksha-lib-core/src/test/resources/patcher/topology/expected.json @@ -0,0 +1,109 @@ +{ + "momType": "Topology", + "referencePoint": { + "type": "Point", + "coordinates": [ + 14.20667, + 50.02379, + 416.38 + ] + }, + "id": "778407118", + "type": "Feature", + "bbox": [ + 14.20594, + 50.02377, + 14.20667, + 50.02386 + ], + "properties": { + "isLongHaul": [ + { + "range": { + "endOffset": 1.0, + "startOffset": 0.0 + }, + "value": false + } + ], + "isMotorway": [ + { + "range": { + "endOffset": 1, + "startOffset": 0 + }, + "value": false, + "confidence": { + "score": 0.75, + "sources": [ + "BMW_RSD" + ], + "scoreType": "GENERIC", + "updatedOn": "2024-10-15T16:39:51.329Z", + "featureType": "UNDEFINED", + "updateSource": "MODERATED", + "triggeringReferences": [ + { + "momFeatureType": "LogicalRoadSign", + "genericFeatureId": "urn:here::here:logicalroadsign:1622677077480690247" + } + ] + } + } + ], + "rightAdmin": [ + { + "range": { + "endOffset": 1, + "startOffset": 0 + }, + "value": { + "id": "urn:here::here:Admin:20445433", + "featureType": "Admin", + "referenceName": "adminId" + }, + "confidence": { + "score": 0, + "scoreType": "EXISTENCE", + "updatedOn": "2021-02-17T21:44:27.863Z", + "featureType": "ROAD_ADMIN_INFORMATION", + "updateSource": "INJECTION_TMOB" + } + } + ], + "referencePoint": { + "type": "Point", + "coordinates": [ + 14.20667, + 50.02379, + 416.38 + ] + } + }, + "geometry": { + "type": "LineString", + "coordinates": [ + [ + 14.20667, + 50.02379, + 416.38 + ], + [ + 14.20636, + 50.02377, + 417.17 + ], + [ + 14.20617, + 50.02379, + 417.38 + ], + [ + 14.20594, + 50.02386, + 417.16 + ] + ] + }, + "class": "NAVLINK" +} \ No newline at end of file diff --git a/here-naksha-lib-core/src/test/resources/patcher/topology/input.json b/here-naksha-lib-core/src/test/resources/patcher/topology/input.json new file mode 100644 index 000000000..bfa527fa4 --- /dev/null +++ b/here-naksha-lib-core/src/test/resources/patcher/topology/input.json @@ -0,0 +1,92 @@ +{ + "momType": "Topology", + "referencePoint": { + "type": "Point", + "coordinates": [ + 14.20667, + 50.02379, + 416.38 + ] + }, + "id": "778407118", + "type": "Feature", + "bbox": [ + 14.20594, + 50.02377, + 14.20667, + 50.02386 + ], + "properties": { + "isMotorway": [ + { + "range": { + "endOffset": 1, + "startOffset": 0 + }, + "value": false, + "confidence": { + "score": 0.75, + "sources": [ + "BMW_RSD" + ], + "scoreType": "GENERIC", + "updatedOn": "2024-10-15T16:39:51.329Z", + "featureType": "UNDEFINED", + "updateSource": "MODERATED", + "triggeringReferences": [ + { + "momFeatureType": "LogicalRoadSign", + "genericFeatureId": "urn:here::here:logicalroadsign:1622677077480690247" + } + ] + } + } + ], + "rightAdmin": [ + { + "range": { + "endOffset": 1, + "startOffset": 0 + }, + "value": { + "id": "urn:here::here:Admin:20445433", + "featureType": "Admin", + "referenceName": "adminId" + }, + "confidence": { + "score": 0, + "scoreType": "EXISTENCE", + "updatedOn": "2021-02-17T21:44:27.863Z", + "featureType": "ROAD_ADMIN_INFORMATION", + "updateSource": "INJECTION_TMOB" + } + } + ] + }, + "geometry": { + "type": "LineString", + "coordinates": [ + [ + 14.20667, + 50.02379, + 416.38 + ], + [ + 14.20636, + 50.02377, + 417.17 + ], + [ + 14.20617, + 50.02379, + 417.38 + ], + [ + 14.20594, + 50.02386, + 417.16 + ] + ] + }, + "class": "NAVLINK" +} \ No newline at end of file From 3e55ff93438756b275577163e144df46ce178fde Mon Sep 17 00:00:00 2001 From: Rohit Lakde <37445331+rlakde@users.noreply.github.com> Date: Mon, 28 Oct 2024 10:07:06 -0500 Subject: [PATCH 08/39] Update reusable-build-and-publish.yml --- .github/workflows/reusable-build-and-publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/reusable-build-and-publish.yml b/.github/workflows/reusable-build-and-publish.yml index d7ec83b11..1c2c5030e 100644 --- a/.github/workflows/reusable-build-and-publish.yml +++ b/.github/workflows/reusable-build-and-publish.yml @@ -17,7 +17,7 @@ jobs: runs-on: ubuntu-latest services: postgres: - image: postgis/postgis + image: postgis/postgis:16-master env: POSTGRES_PASSWORD: password POSTGRES_USER: postgres From dbd6e7965fb75bf53f0c6fade5109c2790ace381 Mon Sep 17 00:00:00 2001 From: Rohit Lakde <37445331+rlakde@users.noreply.github.com> Date: Mon, 28 Oct 2024 10:31:10 -0500 Subject: [PATCH 09/39] Update gradle.properties --- gradle.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle.properties b/gradle.properties index 9b66cc554..92a7e516e 100644 --- a/gradle.properties +++ b/gradle.properties @@ -11,4 +11,4 @@ mavenPassword=YourPassword # - here-naksha-lib-core/NakshaVersion (static property: latest) # - here-naksha-lib-psql/resources/naksha_plpgsql.sql (method: naksha_version) # - here-naksha-app-service/src/main/resources/swagger/openapi.yaml (info.version property) -version=2.1.5 +version=2.2.0 From 9ea31d5932c0eafd165b0ebc55ff0a9ddbb4bfbe Mon Sep 17 00:00:00 2001 From: Rohit Lakde <37445331+rlakde@users.noreply.github.com> Date: Mon, 28 Oct 2024 10:31:35 -0500 Subject: [PATCH 10/39] Update openapi.yaml --- here-naksha-app-service/src/main/resources/swagger/openapi.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/here-naksha-app-service/src/main/resources/swagger/openapi.yaml b/here-naksha-app-service/src/main/resources/swagger/openapi.yaml index f3b01152a..061878e6d 100644 --- a/here-naksha-app-service/src/main/resources/swagger/openapi.yaml +++ b/here-naksha-app-service/src/main/resources/swagger/openapi.yaml @@ -12,7 +12,7 @@ servers: info: title: "Naksha Hub-API" description: "Naksha Hub-API is a REST API to provide simple access to geo data." - version: "2.1.5" + version: "2.2.0" security: - AccessToken: [ ] From 0b204a59ad93bea06fb93ecc246500b6c578a958 Mon Sep 17 00:00:00 2001 From: Rohit Lakde <37445331+rlakde@users.noreply.github.com> Date: Tue, 29 Oct 2024 15:42:44 -0500 Subject: [PATCH 11/39] Add Interface IExtensionInit --- .../here/naksha/lib/core/IExtensionInit.java | 43 +++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 here-naksha-lib-core/src/main/java/com/here/naksha/lib/core/IExtensionInit.java diff --git a/here-naksha-lib-core/src/main/java/com/here/naksha/lib/core/IExtensionInit.java b/here-naksha-lib-core/src/main/java/com/here/naksha/lib/core/IExtensionInit.java new file mode 100644 index 000000000..4e2cab31a --- /dev/null +++ b/here-naksha-lib-core/src/main/java/com/here/naksha/lib/core/IExtensionInit.java @@ -0,0 +1,43 @@ +/* + * Copyright (C) 2017-2024 HERE Europe B.V. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * License-Filename: LICENSE + */ +package com.here.naksha.lib.core; + +import com.here.naksha.lib.core.models.features.Extension; + +/** + * Naksha Extension Interface for all extensions providing initClassName. + */ +public interface IExtensionInit { + + /** + * Initializes the extension with the specified hub and extension parameters. + * This method should be called to set up any necessary resources or configurations + * required by the extension to operate correctly. + * + * @param hub The hub instance to be used by the extension. + * @param extension The extension instance being initialized. + */ + void init(INaksha hub, Extension extension); + + /** + * Closes the extension. This method should be called to ensure proper + * cleanup when the extension is no longer needed. + */ + void close(); +} From 528c9507067a2795e7f4ece77965350efad94c3a Mon Sep 17 00:00:00 2001 From: Rohit Lakde <37445331+rlakde@users.noreply.github.com> Date: Tue, 29 Oct 2024 15:50:37 -0500 Subject: [PATCH 12/39] Add instanceCache logic. --- .../naksha/lib/extmanager/ExtensionCache.java | 60 +++++++++++++++---- 1 file changed, 47 insertions(+), 13 deletions(-) diff --git a/here-naksha-lib-ext-manager/src/main/java/com/here/naksha/lib/extmanager/ExtensionCache.java b/here-naksha-lib-ext-manager/src/main/java/com/here/naksha/lib/extmanager/ExtensionCache.java index e45229fb1..b2e838d84 100644 --- a/here-naksha-lib-ext-manager/src/main/java/com/here/naksha/lib/extmanager/ExtensionCache.java +++ b/here-naksha-lib-ext-manager/src/main/java/com/here/naksha/lib/extmanager/ExtensionCache.java @@ -18,6 +18,7 @@ */ package com.here.naksha.lib.extmanager; +import com.here.naksha.lib.core.IExtensionInit; import com.here.naksha.lib.core.INaksha; import com.here.naksha.lib.core.SimpleTask; import com.here.naksha.lib.core.models.ExtensionConfig; @@ -29,7 +30,6 @@ import com.here.naksha.lib.extmanager.models.KVPair; import java.io.File; import java.io.IOException; -import java.lang.reflect.InvocationTargetException; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -50,6 +50,7 @@ public class ExtensionCache { new ConcurrentHashMap<>(); private static final Map jarClientMap = new HashMap<>(); private final @NotNull INaksha naksha; + private Map instanceCache = new ConcurrentHashMap<>(); static { jarClientMap.put(JarClientType.S3.getType(), new AmazonS3Helper()); @@ -65,6 +66,7 @@ public ExtensionCache(@NotNull INaksha naksha) { * Also it removes existing mapping from cache which is not available in config store anymore */ protected void buildExtensionCache(ExtensionConfig extensionConfig) { + IExtensionInit instance = null; List>> futures = extensionConfig.getExtensions().stream() .filter(extension -> !this.isLoaderMappingExist(extension)) .map(extension -> { @@ -90,9 +92,31 @@ protected void buildExtensionCache(ExtensionConfig extensionConfig) { for (String key : loaderCache.keySet()) { if (!extIds.contains(key)) { - loaderCache.remove(key); - PluginCache.removeExtensionCache(key); - logger.info("Extension {} removed from cache.", key); + KVPair kvPair = loaderCache.get(key); + Object instanceObj = kvPair.getValue(); + Extension extension = kvPair.getKey(); + final String extensionIdWthEnv = extension.getEnv() + ":" + extension.getId(); + if (!isNullOrEmpty(extension.getInitClassName())) { + if (instanceObj instanceof IExtensionInit initInstance) { + try { + initInstance.close(); + instance = initInstance; + logger.info("Extension {} closed successfully.", extensionIdWthEnv); + } catch (Exception e) { + logger.error("Failed to close extension {}", extensionIdWthEnv, e); + } + } else { + logger.error("Instance is not of type IExtensionInit for extension {}", extensionIdWthEnv); + } + } + synchronized (this) { + if (instance != null) { + instanceCache.remove(key); + } + loaderCache.remove(key); + PluginCache.removeExtensionCache(key); + logger.info("Extension {} removed from cache.", key); + } } } logger.info("Extension cache size " + loaderCache.size()); @@ -103,6 +127,7 @@ private void publishIntoCache(KVPair result, ExtensionConfig ex final Extension extension = result.getKey(); final String extensionIdWthEnv = extension.getEnv() + ":" + extension.getId(); final File jarFile = result.getValue(); + IExtensionInit instance = null; ClassLoader loader; try { loader = ClassLoaderHelper.getClassLoader(jarFile, extensionConfig.getWhilelistDelegateClass()); @@ -114,12 +139,16 @@ private void publishIntoCache(KVPair result, ExtensionConfig ex if (!isNullOrEmpty(extension.getInitClassName())) { try { Class clz = loader.loadClass(extension.getInitClassName()); - clz.getConstructor(INaksha.class, Extension.class).newInstance(naksha, extension); - } catch (ClassNotFoundException - | InvocationTargetException - | InstantiationException - | NoSuchMethodException - | IllegalAccessException e) { + Object obj = clz.getConstructor(INaksha.class, Extension.class).newInstance(naksha, extension); + if (obj instanceof IExtensionInit initInstance) { + initInstance.close(); + initInstance.init(naksha, extension); + instance = initInstance; + } else { + logger.error("Extension does not implement IExtensionInit for extension {}", extension.getId()); + return; + } + } catch (Exception e) { logger.error( "Failed to instantiate class {} for extension {} ", extension.getInitClassName(), @@ -133,8 +162,13 @@ private void publishIntoCache(KVPair result, ExtensionConfig ex "Extension {} initialization using initClassName {} done successfully.", extensionIdWthEnv, extension.getInitClassName()); - loaderCache.put(extensionIdWthEnv, new KVPair(extension, loader)); - PluginCache.removeExtensionCache(extensionIdWthEnv); + synchronized (this) { + if (instance != null) { + instanceCache.put(extensionIdWthEnv, instance); + } + loaderCache.put(extensionIdWthEnv, new KVPair(extension, loader)); + PluginCache.removeExtensionCache(extensionIdWthEnv); + } logger.info( "Extension id={}, version={} is successfully loaded into the cache, using Jar at {} for env={}.", extensionIdWthEnv, @@ -160,7 +194,7 @@ private boolean isLoaderMappingExist(Extension extension) { } /** - * Lamda function which will initiate the downloading for extension jar + * Lambda function which will initiate the downloading for extension jar */ private KVPair downloadJar(Extension extension) { logger.info("Downloading jar {} with version {} ", extension.getId(), extension.getVersion()); From 7325ec87bcb64b14f7d0764572e81935e04c2d70 Mon Sep 17 00:00:00 2001 From: Rohit Lakde <37445331+rlakde@users.noreply.github.com> Date: Mon, 4 Nov 2024 12:35:48 -0600 Subject: [PATCH 13/39] Create ValueTuple.java --- .../naksha/lib/extmanager/ValueTuple.java | 46 +++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 here-naksha-lib-ext-manager/src/main/java/com/here/naksha/lib/extmanager/ValueTuple.java diff --git a/here-naksha-lib-ext-manager/src/main/java/com/here/naksha/lib/extmanager/ValueTuple.java b/here-naksha-lib-ext-manager/src/main/java/com/here/naksha/lib/extmanager/ValueTuple.java new file mode 100644 index 000000000..6eca51d76 --- /dev/null +++ b/here-naksha-lib-ext-manager/src/main/java/com/here/naksha/lib/extmanager/ValueTuple.java @@ -0,0 +1,46 @@ +/* + * Copyright (C) 2017-2024 HERE Europe B.V. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * License-Filename: LICENSE + */ +package com.here.naksha.lib.extmanager; + +import com.here.naksha.lib.core.IExtensionInit; +import com.here.naksha.lib.core.models.features.Extension; + +public class ValueTuple { + private final Extension extension; + private final ClassLoader classLoader; + private final IExtensionInit instance; + + public ValueTuple(Extension extension, ClassLoader classLoader, IExtensionInit instance) { + this.extension = extension; + this.classLoader = classLoader; + this.instance = instance; + } + + public Extension getExtension() { + return extension; + } + + public ClassLoader getClassLoader() { + return classLoader; + } + + public IExtensionInit getInstance() { + return instance; + } +} From 94431001603c2fa098845dc560ca4f0f859f5373 Mon Sep 17 00:00:00 2001 From: Rohit Lakde <37445331+rlakde@users.noreply.github.com> Date: Mon, 4 Nov 2024 12:43:28 -0600 Subject: [PATCH 14/39] loaderCache Changes --- .../naksha/lib/extmanager/ExtensionCache.java | 74 +++++++++---------- 1 file changed, 33 insertions(+), 41 deletions(-) diff --git a/here-naksha-lib-ext-manager/src/main/java/com/here/naksha/lib/extmanager/ExtensionCache.java b/here-naksha-lib-ext-manager/src/main/java/com/here/naksha/lib/extmanager/ExtensionCache.java index b2e838d84..b58858119 100644 --- a/here-naksha-lib-ext-manager/src/main/java/com/here/naksha/lib/extmanager/ExtensionCache.java +++ b/here-naksha-lib-ext-manager/src/main/java/com/here/naksha/lib/extmanager/ExtensionCache.java @@ -46,11 +46,9 @@ */ public class ExtensionCache { private static final @NotNull Logger logger = LoggerFactory.getLogger(ExtensionCache.class); - private static final ConcurrentHashMap> loaderCache = - new ConcurrentHashMap<>(); + private static final ConcurrentHashMap loaderCache = new ConcurrentHashMap<>(); private static final Map jarClientMap = new HashMap<>(); private final @NotNull INaksha naksha; - private Map instanceCache = new ConcurrentHashMap<>(); static { jarClientMap.put(JarClientType.S3.getType(), new AmazonS3Helper()); @@ -66,7 +64,6 @@ public ExtensionCache(@NotNull INaksha naksha) { * Also it removes existing mapping from cache which is not available in config store anymore */ protected void buildExtensionCache(ExtensionConfig extensionConfig) { - IExtensionInit instance = null; List>> futures = extensionConfig.getExtensions().stream() .filter(extension -> !this.isLoaderMappingExist(extension)) .map(extension -> { @@ -92,32 +89,8 @@ protected void buildExtensionCache(ExtensionConfig extensionConfig) { for (String key : loaderCache.keySet()) { if (!extIds.contains(key)) { - KVPair kvPair = loaderCache.get(key); - Object instanceObj = kvPair.getValue(); - Extension extension = kvPair.getKey(); - final String extensionIdWthEnv = extension.getEnv() + ":" + extension.getId(); - if (!isNullOrEmpty(extension.getInitClassName())) { - if (instanceObj instanceof IExtensionInit initInstance) { - try { - initInstance.close(); - instance = initInstance; - logger.info("Extension {} closed successfully.", extensionIdWthEnv); - } catch (Exception e) { - logger.error("Failed to close extension {}", extensionIdWthEnv, e); - } - } else { - logger.error("Instance is not of type IExtensionInit for extension {}", extensionIdWthEnv); - } + removeExtensionFromCache(key); } - synchronized (this) { - if (instance != null) { - instanceCache.remove(key); - } - loaderCache.remove(key); - PluginCache.removeExtensionCache(key); - logger.info("Extension {} removed from cache.", key); - } - } } logger.info("Extension cache size " + loaderCache.size()); } @@ -162,13 +135,10 @@ private void publishIntoCache(KVPair result, ExtensionConfig ex "Extension {} initialization using initClassName {} done successfully.", extensionIdWthEnv, extension.getInitClassName()); - synchronized (this) { - if (instance != null) { - instanceCache.put(extensionIdWthEnv, instance); - } - loaderCache.put(extensionIdWthEnv, new KVPair(extension, loader)); - PluginCache.removeExtensionCache(extensionIdWthEnv); - } + + loaderCache.put(extensionIdWthEnv, new KVPair(extension, loader)); + PluginCache.removeExtensionCache(extensionIdWthEnv); + logger.info( "Extension id={}, version={} is successfully loaded into the cache, using Jar at {} for env={}.", extensionIdWthEnv, @@ -178,12 +148,34 @@ private void publishIntoCache(KVPair result, ExtensionConfig ex } } + private void removeExtensionFromCache(String extensionId) { + ValueTuple valueTuple = loaderCache.get(extensionId); + if (valueTuple != null) { + IExtensionInit instance = valueTuple.getInstance(); + Extension extension = valueTuple.getExtension(); + final String extensionIdWithEnv = extension.getEnv() + ":" + extension.getId(); + + if (instance != null) { + try { + instance.close(); + logger.info("Extension {} closed successfully.", extensionIdWithEnv); + } catch (Exception e) { + logger.error("Failed to close extension {}", extensionIdWithEnv, e); + } + } + + loaderCache.remove(extensionId); + PluginCache.removeExtensionCache(extensionId); + logger.info("Extension {} removed from cache.", extensionId); + } + } + private boolean isLoaderMappingExist(Extension extension) { final String extensionIdWthEnv = extension.getEnv() + ":" + extension.getId(); - KVPair existingMapping = loaderCache.get(extensionIdWthEnv); + ValueTuple existingMapping = loaderCache.get(extensionIdWthEnv); if (existingMapping == null) return false; - final Extension exExtension = existingMapping.getKey(); + final Extension exExtension = existingMapping.getExtension(); final String exInitClassName = isNullOrEmpty(exExtension.getInitClassName()) ? "" : exExtension.getInitClassName(); final String initClassName = isNullOrEmpty(extension.getInitClassName()) ? "" : extension.getInitClassName(); @@ -218,8 +210,8 @@ protected FileClient getJarClient(String url) { } protected ClassLoader getClassLoaderById(@NotNull String extensionId) { - KVPair mappedLoader = loaderCache.get(extensionId); - return mappedLoader == null ? null : mappedLoader.getValue(); + ValueTuple mappedLoader = loaderCache.get(extensionId); + return mappedLoader == null ? null : mappedLoader.getClassLoader(); } public int getCacheLength() { @@ -227,7 +219,7 @@ public int getCacheLength() { } public List getCachedExtensions() { - return loaderCache.values().stream().map(KVPair::getKey).toList(); + return loaderCache.values().stream().map(ValueTuple::getExtension).toList(); } private boolean isNullOrEmpty(String value) { From 32c1b9e36b74cdbd0bef0740a91ed92e59423e9d Mon Sep 17 00:00:00 2001 From: Rohit Lakde <37445331+rlakde@users.noreply.github.com> Date: Mon, 4 Nov 2024 12:49:00 -0600 Subject: [PATCH 15/39] Correct logger messages --- .../src/main/java/com/here/naksha/lib/hub/NakshaHub.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/here-naksha-lib-hub/src/main/java/com/here/naksha/lib/hub/NakshaHub.java b/here-naksha-lib-hub/src/main/java/com/here/naksha/lib/hub/NakshaHub.java index a6f453bf2..4fc0a3455 100644 --- a/here-naksha-lib-hub/src/main/java/com/here/naksha/lib/hub/NakshaHub.java +++ b/here-naksha-lib-hub/src/main/java/com/here/naksha/lib/hub/NakshaHub.java @@ -314,8 +314,8 @@ private static WriteXyzCollections createAdminCollectionsRequest() { try { eventHandlers = readFeaturesFromResult(rdResult, EventHandler.class); } catch (NoCursor e) { - logger.error("NoCursor exception encountered", e); - throw new RuntimeException("Failed to open cursor", e); + logger.error("NoCursor exception encountered while reading Extension based Handlers", e); + throw new RuntimeException("Failed to open Cursor while reading Extension based Handlers", e); } Set extensionIds = new HashSet<>(); From 3bd4148ec2e956e6a792b1a27714b60ca1c369db Mon Sep 17 00:00:00 2001 From: Rohit Lakde <37445331+rlakde@users.noreply.github.com> Date: Mon, 4 Nov 2024 12:50:45 -0600 Subject: [PATCH 16/39] Update IExtensionInit description --- .../main/java/com/here/naksha/lib/core/IExtensionInit.java | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/here-naksha-lib-core/src/main/java/com/here/naksha/lib/core/IExtensionInit.java b/here-naksha-lib-core/src/main/java/com/here/naksha/lib/core/IExtensionInit.java index 4e2cab31a..5b038ff6d 100644 --- a/here-naksha-lib-core/src/main/java/com/here/naksha/lib/core/IExtensionInit.java +++ b/here-naksha-lib-core/src/main/java/com/here/naksha/lib/core/IExtensionInit.java @@ -26,10 +26,8 @@ public interface IExtensionInit { /** - * Initializes the extension with the specified hub and extension parameters. - * This method should be called to set up any necessary resources or configurations - * required by the extension to operate correctly. - * + * This method should be called to set up any necessary configurations + * Extension configuration supplied as part of deployment pipeline for respective Extension and sub-env. * @param hub The hub instance to be used by the extension. * @param extension The extension instance being initialized. */ From d718694fa1c8630d26d911c4cb55d1d9ee607368 Mon Sep 17 00:00:00 2001 From: Rohit Lakde <37445331+rlakde@users.noreply.github.com> Date: Mon, 4 Nov 2024 12:55:47 -0600 Subject: [PATCH 17/39] Fix bug --- .../java/com/here/naksha/lib/extmanager/ExtensionCache.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/here-naksha-lib-ext-manager/src/main/java/com/here/naksha/lib/extmanager/ExtensionCache.java b/here-naksha-lib-ext-manager/src/main/java/com/here/naksha/lib/extmanager/ExtensionCache.java index b58858119..fa235f106 100644 --- a/here-naksha-lib-ext-manager/src/main/java/com/here/naksha/lib/extmanager/ExtensionCache.java +++ b/here-naksha-lib-ext-manager/src/main/java/com/here/naksha/lib/extmanager/ExtensionCache.java @@ -136,7 +136,7 @@ private void publishIntoCache(KVPair result, ExtensionConfig ex extensionIdWthEnv, extension.getInitClassName()); - loaderCache.put(extensionIdWthEnv, new KVPair(extension, loader)); + loaderCache.put(extensionIdWthEnv, new ValueTuple(extension, loader, instance)); PluginCache.removeExtensionCache(extensionIdWthEnv); logger.info( From f9314f63212197deba91afa963bce6215656ad06 Mon Sep 17 00:00:00 2001 From: Rohit Lakde <37445331+rlakde@users.noreply.github.com> Date: Mon, 4 Nov 2024 13:02:48 -0600 Subject: [PATCH 18/39] Correct logger in Hub. --- .../src/main/java/com/here/naksha/lib/hub/NakshaHub.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/here-naksha-lib-hub/src/main/java/com/here/naksha/lib/hub/NakshaHub.java b/here-naksha-lib-hub/src/main/java/com/here/naksha/lib/hub/NakshaHub.java index 4fc0a3455..ccb4376a0 100644 --- a/here-naksha-lib-hub/src/main/java/com/here/naksha/lib/hub/NakshaHub.java +++ b/here-naksha-lib-hub/src/main/java/com/here/naksha/lib/hub/NakshaHub.java @@ -324,7 +324,7 @@ private static WriteXyzCollections createAdminCollectionsRequest() { if (extensionId != null && extensionId.contains(":")) { extensionIds.add(extensionId); } else { - logger.error("Environment is missing for an extension Id"); + logger.error("Environment is missing for an extension Id {}", extensionId); } } From a8cfeaad5cf679eac9ceb1014deee11bac84ec68 Mon Sep 17 00:00:00 2001 From: Rohit Lakde <37445331+rlakde@users.noreply.github.com> Date: Mon, 4 Nov 2024 19:35:17 -0600 Subject: [PATCH 19/39] Update IExtensionInit Description. --- .../main/java/com/here/naksha/lib/core/IExtensionInit.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/here-naksha-lib-core/src/main/java/com/here/naksha/lib/core/IExtensionInit.java b/here-naksha-lib-core/src/main/java/com/here/naksha/lib/core/IExtensionInit.java index 5b038ff6d..33d21a305 100644 --- a/here-naksha-lib-core/src/main/java/com/here/naksha/lib/core/IExtensionInit.java +++ b/here-naksha-lib-core/src/main/java/com/here/naksha/lib/core/IExtensionInit.java @@ -26,10 +26,11 @@ public interface IExtensionInit { /** - * This method should be called to set up any necessary configurations - * Extension configuration supplied as part of deployment pipeline for respective Extension and sub-env. + * Initializes the extension with the specified hub and extension parameters. + * This method should be called to set up any necessary resources or configurations + * required by the extension to operate correctly. * @param hub The hub instance to be used by the extension. - * @param extension The extension instance being initialized. + * @param extension Extension configuration supplied as part of deployment pipeline for respective Extension and sub-env. */ void init(INaksha hub, Extension extension); From f05cd53e9d60d389af80eedd6774aa0b567dbdd5 Mon Sep 17 00:00:00 2001 From: Rohit Lakde <37445331+rlakde@users.noreply.github.com> Date: Mon, 4 Nov 2024 19:38:38 -0600 Subject: [PATCH 20/39] Correct removeExtensionFromCache. --- .../com/here/naksha/lib/extmanager/ExtensionCache.java | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/here-naksha-lib-ext-manager/src/main/java/com/here/naksha/lib/extmanager/ExtensionCache.java b/here-naksha-lib-ext-manager/src/main/java/com/here/naksha/lib/extmanager/ExtensionCache.java index fa235f106..e38b9729b 100644 --- a/here-naksha-lib-ext-manager/src/main/java/com/here/naksha/lib/extmanager/ExtensionCache.java +++ b/here-naksha-lib-ext-manager/src/main/java/com/here/naksha/lib/extmanager/ExtensionCache.java @@ -112,9 +112,8 @@ private void publishIntoCache(KVPair result, ExtensionConfig ex if (!isNullOrEmpty(extension.getInitClassName())) { try { Class clz = loader.loadClass(extension.getInitClassName()); - Object obj = clz.getConstructor(INaksha.class, Extension.class).newInstance(naksha, extension); + Object obj = clz.getConstructor().newInstance(); if (obj instanceof IExtensionInit initInstance) { - initInstance.close(); initInstance.init(naksha, extension); instance = initInstance; } else { @@ -152,15 +151,12 @@ private void removeExtensionFromCache(String extensionId) { ValueTuple valueTuple = loaderCache.get(extensionId); if (valueTuple != null) { IExtensionInit instance = valueTuple.getInstance(); - Extension extension = valueTuple.getExtension(); - final String extensionIdWithEnv = extension.getEnv() + ":" + extension.getId(); - if (instance != null) { try { instance.close(); - logger.info("Extension {} closed successfully.", extensionIdWithEnv); + logger.info("Extension {} closed successfully.", extensionId); } catch (Exception e) { - logger.error("Failed to close extension {}", extensionIdWithEnv, e); + logger.error("Failed to close extension {}", extensionId, e); } } From 5fda3754bd1f6f72632527b57f076e90062aef21 Mon Sep 17 00:00:00 2001 From: Rohit Lakde <37445331+rlakde@users.noreply.github.com> Date: Tue, 5 Nov 2024 01:55:39 -0600 Subject: [PATCH 21/39] Review Fixes for ExtensionCache --- .../naksha/lib/extmanager/ExtensionCache.java | 57 ++++++++++--------- 1 file changed, 30 insertions(+), 27 deletions(-) diff --git a/here-naksha-lib-ext-manager/src/main/java/com/here/naksha/lib/extmanager/ExtensionCache.java b/here-naksha-lib-ext-manager/src/main/java/com/here/naksha/lib/extmanager/ExtensionCache.java index e38b9729b..1c8fb6223 100644 --- a/here-naksha-lib-ext-manager/src/main/java/com/here/naksha/lib/extmanager/ExtensionCache.java +++ b/here-naksha-lib-ext-manager/src/main/java/com/here/naksha/lib/extmanager/ExtensionCache.java @@ -100,12 +100,12 @@ private void publishIntoCache(KVPair result, ExtensionConfig ex final Extension extension = result.getKey(); final String extensionIdWthEnv = extension.getEnv() + ":" + extension.getId(); final File jarFile = result.getValue(); - IExtensionInit instance = null; + IExtensionInit initObj = null; ClassLoader loader; try { loader = ClassLoaderHelper.getClassLoader(jarFile, extensionConfig.getWhilelistDelegateClass()); } catch (Exception e) { - logger.error("Failed to load extension jar " + extension.getId(), e); + logger.error("Failed to load extension jar " + extensionIdWthEnv, e); return; } @@ -115,9 +115,13 @@ private void publishIntoCache(KVPair result, ExtensionConfig ex Object obj = clz.getConstructor().newInstance(); if (obj instanceof IExtensionInit initInstance) { initInstance.init(naksha, extension); - instance = initInstance; + initObj = initInstance; + logger.info( + "Extension {} initialization using initClassName {} done successfully.", + extensionIdWthEnv, + extension.getInitClassName()); } else { - logger.error("Extension does not implement IExtensionInit for extension {}", extension.getId()); + logger.error("Extension does not implement IExtensionInit for extension {}", extensionIdWthEnv); return; } } catch (Exception e) { @@ -129,40 +133,39 @@ private void publishIntoCache(KVPair result, ExtensionConfig ex return; } } - if (!isNullOrEmpty(extension.getInitClassName())) - logger.info( - "Extension {} initialization using initClassName {} done successfully.", - extensionIdWthEnv, - extension.getInitClassName()); - loaderCache.put(extensionIdWthEnv, new ValueTuple(extension, loader, instance)); - PluginCache.removeExtensionCache(extensionIdWthEnv); + ValueTuple previousValue = loaderCache.put(extensionIdWthEnv, new ValueTuple(extension, loader, initObj)); + if (previousValue != null) { + IExtensionInit previousInitObj = previousValue.getInstance(); + closeExtensionInstance(extensionIdWthEnv, previousInitObj); + } logger.info( - "Extension id={}, version={} is successfully loaded into the cache, using Jar at {} for env={}.", + "Extension id={}, version={} is successfully loaded into the cache, using Jar at {}.", extensionIdWthEnv, extension.getVersion(), - extension.getUrl().substring(extension.getUrl().lastIndexOf("/") + 1), - extension.getEnv()); + extension.getUrl().substring(extension.getUrl().lastIndexOf("/") + 1)); } } private void removeExtensionFromCache(String extensionId) { - ValueTuple valueTuple = loaderCache.get(extensionId); + ValueTuple valueTuple = loaderCache.remove(extensionId); + PluginCache.removeExtensionCache(extensionId); + logger.info("Extension {} removed from cache.", extensionId); if (valueTuple != null) { - IExtensionInit instance = valueTuple.getInstance(); - if (instance != null) { - try { - instance.close(); - logger.info("Extension {} closed successfully.", extensionId); - } catch (Exception e) { - logger.error("Failed to close extension {}", extensionId, e); - } - } + IExtensionInit initObj = valueTuple.getInstance(); + closeExtensionInstance(extensionId, initObj); + } + } - loaderCache.remove(extensionId); - PluginCache.removeExtensionCache(extensionId); - logger.info("Extension {} removed from cache.", extensionId); + private void closeExtensionInstance(String extensionId, IExtensionInit initObj) { + if (initObj != null) { + try { + initObj.close(); + logger.info("Extension {} closed successfully.", extensionId); + } catch (Exception e) { + logger.error("Failed to close extension {}", extensionId, e); + } } } From 6acf5218a884e8c2437656ac689da3a3f4d20ccf Mon Sep 17 00:00:00 2001 From: Rohit Lakde <37445331+rlakde@users.noreply.github.com> Date: Thu, 12 Dec 2024 19:00:04 +0530 Subject: [PATCH 22/39] Update ExtensionCache.java --- .../naksha/lib/extmanager/ExtensionCache.java | 85 +++++++------------ 1 file changed, 30 insertions(+), 55 deletions(-) diff --git a/here-naksha-lib-ext-manager/src/main/java/com/here/naksha/lib/extmanager/ExtensionCache.java b/here-naksha-lib-ext-manager/src/main/java/com/here/naksha/lib/extmanager/ExtensionCache.java index 1c8fb6223..e45229fb1 100644 --- a/here-naksha-lib-ext-manager/src/main/java/com/here/naksha/lib/extmanager/ExtensionCache.java +++ b/here-naksha-lib-ext-manager/src/main/java/com/here/naksha/lib/extmanager/ExtensionCache.java @@ -18,7 +18,6 @@ */ package com.here.naksha.lib.extmanager; -import com.here.naksha.lib.core.IExtensionInit; import com.here.naksha.lib.core.INaksha; import com.here.naksha.lib.core.SimpleTask; import com.here.naksha.lib.core.models.ExtensionConfig; @@ -30,6 +29,7 @@ import com.here.naksha.lib.extmanager.models.KVPair; import java.io.File; import java.io.IOException; +import java.lang.reflect.InvocationTargetException; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -46,7 +46,8 @@ */ public class ExtensionCache { private static final @NotNull Logger logger = LoggerFactory.getLogger(ExtensionCache.class); - private static final ConcurrentHashMap loaderCache = new ConcurrentHashMap<>(); + private static final ConcurrentHashMap> loaderCache = + new ConcurrentHashMap<>(); private static final Map jarClientMap = new HashMap<>(); private final @NotNull INaksha naksha; @@ -89,8 +90,10 @@ protected void buildExtensionCache(ExtensionConfig extensionConfig) { for (String key : loaderCache.keySet()) { if (!extIds.contains(key)) { - removeExtensionFromCache(key); - } + loaderCache.remove(key); + PluginCache.removeExtensionCache(key); + logger.info("Extension {} removed from cache.", key); + } } logger.info("Extension cache size " + loaderCache.size()); } @@ -100,31 +103,23 @@ private void publishIntoCache(KVPair result, ExtensionConfig ex final Extension extension = result.getKey(); final String extensionIdWthEnv = extension.getEnv() + ":" + extension.getId(); final File jarFile = result.getValue(); - IExtensionInit initObj = null; ClassLoader loader; try { loader = ClassLoaderHelper.getClassLoader(jarFile, extensionConfig.getWhilelistDelegateClass()); } catch (Exception e) { - logger.error("Failed to load extension jar " + extensionIdWthEnv, e); + logger.error("Failed to load extension jar " + extension.getId(), e); return; } if (!isNullOrEmpty(extension.getInitClassName())) { try { Class clz = loader.loadClass(extension.getInitClassName()); - Object obj = clz.getConstructor().newInstance(); - if (obj instanceof IExtensionInit initInstance) { - initInstance.init(naksha, extension); - initObj = initInstance; - logger.info( - "Extension {} initialization using initClassName {} done successfully.", - extensionIdWthEnv, - extension.getInitClassName()); - } else { - logger.error("Extension does not implement IExtensionInit for extension {}", extensionIdWthEnv); - return; - } - } catch (Exception e) { + clz.getConstructor(INaksha.class, Extension.class).newInstance(naksha, extension); + } catch (ClassNotFoundException + | InvocationTargetException + | InstantiationException + | NoSuchMethodException + | IllegalAccessException e) { logger.error( "Failed to instantiate class {} for extension {} ", extension.getInitClassName(), @@ -133,48 +128,28 @@ private void publishIntoCache(KVPair result, ExtensionConfig ex return; } } - - ValueTuple previousValue = loaderCache.put(extensionIdWthEnv, new ValueTuple(extension, loader, initObj)); - if (previousValue != null) { - IExtensionInit previousInitObj = previousValue.getInstance(); - closeExtensionInstance(extensionIdWthEnv, previousInitObj); - } - + if (!isNullOrEmpty(extension.getInitClassName())) + logger.info( + "Extension {} initialization using initClassName {} done successfully.", + extensionIdWthEnv, + extension.getInitClassName()); + loaderCache.put(extensionIdWthEnv, new KVPair(extension, loader)); + PluginCache.removeExtensionCache(extensionIdWthEnv); logger.info( - "Extension id={}, version={} is successfully loaded into the cache, using Jar at {}.", + "Extension id={}, version={} is successfully loaded into the cache, using Jar at {} for env={}.", extensionIdWthEnv, extension.getVersion(), - extension.getUrl().substring(extension.getUrl().lastIndexOf("/") + 1)); - } - } - - private void removeExtensionFromCache(String extensionId) { - ValueTuple valueTuple = loaderCache.remove(extensionId); - PluginCache.removeExtensionCache(extensionId); - logger.info("Extension {} removed from cache.", extensionId); - if (valueTuple != null) { - IExtensionInit initObj = valueTuple.getInstance(); - closeExtensionInstance(extensionId, initObj); - } - } - - private void closeExtensionInstance(String extensionId, IExtensionInit initObj) { - if (initObj != null) { - try { - initObj.close(); - logger.info("Extension {} closed successfully.", extensionId); - } catch (Exception e) { - logger.error("Failed to close extension {}", extensionId, e); - } + extension.getUrl().substring(extension.getUrl().lastIndexOf("/") + 1), + extension.getEnv()); } } private boolean isLoaderMappingExist(Extension extension) { final String extensionIdWthEnv = extension.getEnv() + ":" + extension.getId(); - ValueTuple existingMapping = loaderCache.get(extensionIdWthEnv); + KVPair existingMapping = loaderCache.get(extensionIdWthEnv); if (existingMapping == null) return false; - final Extension exExtension = existingMapping.getExtension(); + final Extension exExtension = existingMapping.getKey(); final String exInitClassName = isNullOrEmpty(exExtension.getInitClassName()) ? "" : exExtension.getInitClassName(); final String initClassName = isNullOrEmpty(extension.getInitClassName()) ? "" : extension.getInitClassName(); @@ -185,7 +160,7 @@ private boolean isLoaderMappingExist(Extension extension) { } /** - * Lambda function which will initiate the downloading for extension jar + * Lamda function which will initiate the downloading for extension jar */ private KVPair downloadJar(Extension extension) { logger.info("Downloading jar {} with version {} ", extension.getId(), extension.getVersion()); @@ -209,8 +184,8 @@ protected FileClient getJarClient(String url) { } protected ClassLoader getClassLoaderById(@NotNull String extensionId) { - ValueTuple mappedLoader = loaderCache.get(extensionId); - return mappedLoader == null ? null : mappedLoader.getClassLoader(); + KVPair mappedLoader = loaderCache.get(extensionId); + return mappedLoader == null ? null : mappedLoader.getValue(); } public int getCacheLength() { @@ -218,7 +193,7 @@ public int getCacheLength() { } public List getCachedExtensions() { - return loaderCache.values().stream().map(ValueTuple::getExtension).toList(); + return loaderCache.values().stream().map(KVPair::getKey).toList(); } private boolean isNullOrEmpty(String value) { From ec3f3fc2785f7e5b5766739af7579bed05e8c7a6 Mon Sep 17 00:00:00 2001 From: Rohit Lakde <37445331+rlakde@users.noreply.github.com> Date: Thu, 12 Dec 2024 19:01:04 +0530 Subject: [PATCH 23/39] Update NakshaHub.java --- .../src/main/java/com/here/naksha/lib/hub/NakshaHub.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/here-naksha-lib-hub/src/main/java/com/here/naksha/lib/hub/NakshaHub.java b/here-naksha-lib-hub/src/main/java/com/here/naksha/lib/hub/NakshaHub.java index ccb4376a0..a6f453bf2 100644 --- a/here-naksha-lib-hub/src/main/java/com/here/naksha/lib/hub/NakshaHub.java +++ b/here-naksha-lib-hub/src/main/java/com/here/naksha/lib/hub/NakshaHub.java @@ -314,8 +314,8 @@ private static WriteXyzCollections createAdminCollectionsRequest() { try { eventHandlers = readFeaturesFromResult(rdResult, EventHandler.class); } catch (NoCursor e) { - logger.error("NoCursor exception encountered while reading Extension based Handlers", e); - throw new RuntimeException("Failed to open Cursor while reading Extension based Handlers", e); + logger.error("NoCursor exception encountered", e); + throw new RuntimeException("Failed to open cursor", e); } Set extensionIds = new HashSet<>(); @@ -324,7 +324,7 @@ private static WriteXyzCollections createAdminCollectionsRequest() { if (extensionId != null && extensionId.contains(":")) { extensionIds.add(extensionId); } else { - logger.error("Environment is missing for an extension Id {}", extensionId); + logger.error("Environment is missing for an extension Id"); } } From 5311364bb4499c69e271bdfb33a7f1a6d1ae6615 Mon Sep 17 00:00:00 2001 From: Rohit Lakde <37445331+rlakde@users.noreply.github.com> Date: Thu, 12 Dec 2024 19:01:52 +0530 Subject: [PATCH 24/39] Extension Sub Environment Changes (#330) (#392) * Initial Changes. * Fix Unit Tests * Review Fixes * Rebase CASL-321 (#371) * Fixed array patching (#365) * Fix thread stuck issue + addtnl logs (#368) * CASL-258 unreleased lock fix (#358) * updated version * updated changelog * Update reusable-build-and-publish.yml * Update gradle.properties * Update openapi.yaml --------- Signed-off-by: lakde From a80f6a49263013601c07b0c152d9fef1c53c0d13 Mon Sep 17 00:00:00 2001 From: Rohit Lakde <37445331+rlakde@users.noreply.github.com> Date: Thu, 12 Dec 2024 19:03:57 +0530 Subject: [PATCH 25/39] Update ExtensionCache.java --- .../naksha/lib/extmanager/ExtensionCache.java | 85 ++++++++++++------- 1 file changed, 55 insertions(+), 30 deletions(-) diff --git a/here-naksha-lib-ext-manager/src/main/java/com/here/naksha/lib/extmanager/ExtensionCache.java b/here-naksha-lib-ext-manager/src/main/java/com/here/naksha/lib/extmanager/ExtensionCache.java index e45229fb1..1c8fb6223 100644 --- a/here-naksha-lib-ext-manager/src/main/java/com/here/naksha/lib/extmanager/ExtensionCache.java +++ b/here-naksha-lib-ext-manager/src/main/java/com/here/naksha/lib/extmanager/ExtensionCache.java @@ -18,6 +18,7 @@ */ package com.here.naksha.lib.extmanager; +import com.here.naksha.lib.core.IExtensionInit; import com.here.naksha.lib.core.INaksha; import com.here.naksha.lib.core.SimpleTask; import com.here.naksha.lib.core.models.ExtensionConfig; @@ -29,7 +30,6 @@ import com.here.naksha.lib.extmanager.models.KVPair; import java.io.File; import java.io.IOException; -import java.lang.reflect.InvocationTargetException; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -46,8 +46,7 @@ */ public class ExtensionCache { private static final @NotNull Logger logger = LoggerFactory.getLogger(ExtensionCache.class); - private static final ConcurrentHashMap> loaderCache = - new ConcurrentHashMap<>(); + private static final ConcurrentHashMap loaderCache = new ConcurrentHashMap<>(); private static final Map jarClientMap = new HashMap<>(); private final @NotNull INaksha naksha; @@ -90,10 +89,8 @@ protected void buildExtensionCache(ExtensionConfig extensionConfig) { for (String key : loaderCache.keySet()) { if (!extIds.contains(key)) { - loaderCache.remove(key); - PluginCache.removeExtensionCache(key); - logger.info("Extension {} removed from cache.", key); - } + removeExtensionFromCache(key); + } } logger.info("Extension cache size " + loaderCache.size()); } @@ -103,23 +100,31 @@ private void publishIntoCache(KVPair result, ExtensionConfig ex final Extension extension = result.getKey(); final String extensionIdWthEnv = extension.getEnv() + ":" + extension.getId(); final File jarFile = result.getValue(); + IExtensionInit initObj = null; ClassLoader loader; try { loader = ClassLoaderHelper.getClassLoader(jarFile, extensionConfig.getWhilelistDelegateClass()); } catch (Exception e) { - logger.error("Failed to load extension jar " + extension.getId(), e); + logger.error("Failed to load extension jar " + extensionIdWthEnv, e); return; } if (!isNullOrEmpty(extension.getInitClassName())) { try { Class clz = loader.loadClass(extension.getInitClassName()); - clz.getConstructor(INaksha.class, Extension.class).newInstance(naksha, extension); - } catch (ClassNotFoundException - | InvocationTargetException - | InstantiationException - | NoSuchMethodException - | IllegalAccessException e) { + Object obj = clz.getConstructor().newInstance(); + if (obj instanceof IExtensionInit initInstance) { + initInstance.init(naksha, extension); + initObj = initInstance; + logger.info( + "Extension {} initialization using initClassName {} done successfully.", + extensionIdWthEnv, + extension.getInitClassName()); + } else { + logger.error("Extension does not implement IExtensionInit for extension {}", extensionIdWthEnv); + return; + } + } catch (Exception e) { logger.error( "Failed to instantiate class {} for extension {} ", extension.getInitClassName(), @@ -128,28 +133,48 @@ private void publishIntoCache(KVPair result, ExtensionConfig ex return; } } - if (!isNullOrEmpty(extension.getInitClassName())) - logger.info( - "Extension {} initialization using initClassName {} done successfully.", - extensionIdWthEnv, - extension.getInitClassName()); - loaderCache.put(extensionIdWthEnv, new KVPair(extension, loader)); - PluginCache.removeExtensionCache(extensionIdWthEnv); + + ValueTuple previousValue = loaderCache.put(extensionIdWthEnv, new ValueTuple(extension, loader, initObj)); + if (previousValue != null) { + IExtensionInit previousInitObj = previousValue.getInstance(); + closeExtensionInstance(extensionIdWthEnv, previousInitObj); + } + logger.info( - "Extension id={}, version={} is successfully loaded into the cache, using Jar at {} for env={}.", + "Extension id={}, version={} is successfully loaded into the cache, using Jar at {}.", extensionIdWthEnv, extension.getVersion(), - extension.getUrl().substring(extension.getUrl().lastIndexOf("/") + 1), - extension.getEnv()); + extension.getUrl().substring(extension.getUrl().lastIndexOf("/") + 1)); + } + } + + private void removeExtensionFromCache(String extensionId) { + ValueTuple valueTuple = loaderCache.remove(extensionId); + PluginCache.removeExtensionCache(extensionId); + logger.info("Extension {} removed from cache.", extensionId); + if (valueTuple != null) { + IExtensionInit initObj = valueTuple.getInstance(); + closeExtensionInstance(extensionId, initObj); + } + } + + private void closeExtensionInstance(String extensionId, IExtensionInit initObj) { + if (initObj != null) { + try { + initObj.close(); + logger.info("Extension {} closed successfully.", extensionId); + } catch (Exception e) { + logger.error("Failed to close extension {}", extensionId, e); + } } } private boolean isLoaderMappingExist(Extension extension) { final String extensionIdWthEnv = extension.getEnv() + ":" + extension.getId(); - KVPair existingMapping = loaderCache.get(extensionIdWthEnv); + ValueTuple existingMapping = loaderCache.get(extensionIdWthEnv); if (existingMapping == null) return false; - final Extension exExtension = existingMapping.getKey(); + final Extension exExtension = existingMapping.getExtension(); final String exInitClassName = isNullOrEmpty(exExtension.getInitClassName()) ? "" : exExtension.getInitClassName(); final String initClassName = isNullOrEmpty(extension.getInitClassName()) ? "" : extension.getInitClassName(); @@ -160,7 +185,7 @@ private boolean isLoaderMappingExist(Extension extension) { } /** - * Lamda function which will initiate the downloading for extension jar + * Lambda function which will initiate the downloading for extension jar */ private KVPair downloadJar(Extension extension) { logger.info("Downloading jar {} with version {} ", extension.getId(), extension.getVersion()); @@ -184,8 +209,8 @@ protected FileClient getJarClient(String url) { } protected ClassLoader getClassLoaderById(@NotNull String extensionId) { - KVPair mappedLoader = loaderCache.get(extensionId); - return mappedLoader == null ? null : mappedLoader.getValue(); + ValueTuple mappedLoader = loaderCache.get(extensionId); + return mappedLoader == null ? null : mappedLoader.getClassLoader(); } public int getCacheLength() { @@ -193,7 +218,7 @@ public int getCacheLength() { } public List getCachedExtensions() { - return loaderCache.values().stream().map(KVPair::getKey).toList(); + return loaderCache.values().stream().map(ValueTuple::getExtension).toList(); } private boolean isNullOrEmpty(String value) { From ee35c849b564d41e55679ce1b60b4bac64905ab7 Mon Sep 17 00:00:00 2001 From: Rohit Lakde <37445331+rlakde@users.noreply.github.com> Date: Thu, 12 Dec 2024 19:04:42 +0530 Subject: [PATCH 26/39] Update NakshaHub.java --- .../src/main/java/com/here/naksha/lib/hub/NakshaHub.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/here-naksha-lib-hub/src/main/java/com/here/naksha/lib/hub/NakshaHub.java b/here-naksha-lib-hub/src/main/java/com/here/naksha/lib/hub/NakshaHub.java index a6f453bf2..ccb4376a0 100644 --- a/here-naksha-lib-hub/src/main/java/com/here/naksha/lib/hub/NakshaHub.java +++ b/here-naksha-lib-hub/src/main/java/com/here/naksha/lib/hub/NakshaHub.java @@ -314,8 +314,8 @@ private static WriteXyzCollections createAdminCollectionsRequest() { try { eventHandlers = readFeaturesFromResult(rdResult, EventHandler.class); } catch (NoCursor e) { - logger.error("NoCursor exception encountered", e); - throw new RuntimeException("Failed to open cursor", e); + logger.error("NoCursor exception encountered while reading Extension based Handlers", e); + throw new RuntimeException("Failed to open Cursor while reading Extension based Handlers", e); } Set extensionIds = new HashSet<>(); @@ -324,7 +324,7 @@ private static WriteXyzCollections createAdminCollectionsRequest() { if (extensionId != null && extensionId.contains(":")) { extensionIds.add(extensionId); } else { - logger.error("Environment is missing for an extension Id"); + logger.error("Environment is missing for an extension Id {}", extensionId); } } From 1d73a389398dd0d3b938010b39869d85add6f371 Mon Sep 17 00:00:00 2001 From: Rohit Lakde <37445331+rlakde@users.noreply.github.com> Date: Fri, 20 Dec 2024 13:49:48 +0530 Subject: [PATCH 27/39] Update gradle.properties --- gradle.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle.properties b/gradle.properties index fee820e8e..fc1620840 100644 --- a/gradle.properties +++ b/gradle.properties @@ -11,5 +11,5 @@ mavenPassword=YourPassword # - here-naksha-lib-core/NakshaVersion (static property: latest) # - here-naksha-lib-psql/resources/naksha_plpgsql.sql (method: naksha_version) # - here-naksha-app-service/src/main/resources/swagger/openapi.yaml (info.version property) -version=2.2.0 +version=2.2.1 From 9834cc1071d4deb5803b3c5c36d6c3c8f7ce0edf Mon Sep 17 00:00:00 2001 From: Rohit Lakde <37445331+rlakde@users.noreply.github.com> Date: Fri, 20 Dec 2024 13:51:05 +0530 Subject: [PATCH 28/39] Update openapi.yaml --- here-naksha-app-service/src/main/resources/swagger/openapi.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/here-naksha-app-service/src/main/resources/swagger/openapi.yaml b/here-naksha-app-service/src/main/resources/swagger/openapi.yaml index 061878e6d..bb3e48c6a 100644 --- a/here-naksha-app-service/src/main/resources/swagger/openapi.yaml +++ b/here-naksha-app-service/src/main/resources/swagger/openapi.yaml @@ -12,7 +12,7 @@ servers: info: title: "Naksha Hub-API" description: "Naksha Hub-API is a REST API to provide simple access to geo data." - version: "2.2.0" + version: "2.2.1" security: - AccessToken: [ ] From 807706c4f4b715fb28d41f61c829e06c4089442e Mon Sep 17 00:00:00 2001 From: Rohit Lakde <37445331+rlakde@users.noreply.github.com> Date: Fri, 20 Dec 2024 14:16:39 +0530 Subject: [PATCH 29/39] Update NakshaVersion.java --- .../src/main/java/com/here/naksha/lib/core/NakshaVersion.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/here-naksha-lib-core/src/main/java/com/here/naksha/lib/core/NakshaVersion.java b/here-naksha-lib-core/src/main/java/com/here/naksha/lib/core/NakshaVersion.java index fa997276c..7e46c80f5 100644 --- a/here-naksha-lib-core/src/main/java/com/here/naksha/lib/core/NakshaVersion.java +++ b/here-naksha-lib-core/src/main/java/com/here/naksha/lib/core/NakshaVersion.java @@ -57,12 +57,13 @@ public class NakshaVersion implements Comparable { public static final String v2_0_20 = "2.0.20"; public static final String v2_1_0 = "2.1.0"; public static final String v2_1_1 = "2.1.1"; + public static final String v2_2_1 = "2.2.1"; /** * The latest version of the naksha-extension stored in the resources. */ @AvailableSince(v2_0_5) - public static final NakshaVersion latest = of(v2_1_1); + public static final NakshaVersion latest = of(v2_2_1); private final int major; private final int minor; From 26ae8127b134b04b2ad72ca911dc2b3b4165a734 Mon Sep 17 00:00:00 2001 From: Rohit Lakde <37445331+rlakde@users.noreply.github.com> Date: Fri, 20 Dec 2024 14:16:59 +0530 Subject: [PATCH 30/39] Update gradle.properties --- gradle.properties | 1 - 1 file changed, 1 deletion(-) diff --git a/gradle.properties b/gradle.properties index fc1620840..0c9001e49 100644 --- a/gradle.properties +++ b/gradle.properties @@ -9,7 +9,6 @@ mavenPassword=YourPassword # When updating the version, please as well consider: # - here-naksha-lib-core/NakshaVersion (static property: latest) -# - here-naksha-lib-psql/resources/naksha_plpgsql.sql (method: naksha_version) # - here-naksha-app-service/src/main/resources/swagger/openapi.yaml (info.version property) version=2.2.1 From d364a212e38464ad3203d6b737374b1f3e1743a2 Mon Sep 17 00:00:00 2001 From: Rohit Lakde <37445331+rlakde@users.noreply.github.com> Date: Fri, 3 Jan 2025 14:46:15 -0600 Subject: [PATCH 31/39] Update gradle.properties --- gradle.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle.properties b/gradle.properties index 0c9001e49..cb06ea2e9 100644 --- a/gradle.properties +++ b/gradle.properties @@ -8,7 +8,7 @@ mavenUser=YourUserName mavenPassword=YourPassword # When updating the version, please as well consider: -# - here-naksha-lib-core/NakshaVersion (static property: latest) +# - here-naksha-lib-core/src/main/com/here/naksha/lib/core/NakshaVersion (static property: latest) # - here-naksha-app-service/src/main/resources/swagger/openapi.yaml (info.version property) version=2.2.1 From 6c461d37ef4ef2211a1ba9fdaf2242935ec1a894 Mon Sep 17 00:00:00 2001 From: Rohit Lakde <37445331+rlakde@users.noreply.github.com> Date: Fri, 3 Jan 2025 14:49:38 -0600 Subject: [PATCH 32/39] Update NakshaVersion.java --- .../src/main/java/com/here/naksha/lib/core/NakshaVersion.java | 1 + 1 file changed, 1 insertion(+) diff --git a/here-naksha-lib-core/src/main/java/com/here/naksha/lib/core/NakshaVersion.java b/here-naksha-lib-core/src/main/java/com/here/naksha/lib/core/NakshaVersion.java index 7e46c80f5..0cfa490c6 100644 --- a/here-naksha-lib-core/src/main/java/com/here/naksha/lib/core/NakshaVersion.java +++ b/here-naksha-lib-core/src/main/java/com/here/naksha/lib/core/NakshaVersion.java @@ -57,6 +57,7 @@ public class NakshaVersion implements Comparable { public static final String v2_0_20 = "2.0.20"; public static final String v2_1_0 = "2.1.0"; public static final String v2_1_1 = "2.1.1"; + public static final String v2_2_0 = "2.2.0"; public static final String v2_2_1 = "2.2.1"; /** From 896c98c675dcf0573fa40111f0f5b4cae4481529 Mon Sep 17 00:00:00 2001 From: Rohit Lakde <37445331+rlakde@users.noreply.github.com> Date: Fri, 3 Jan 2025 14:52:02 -0600 Subject: [PATCH 33/39] Update NakshaHub.java --- .../com/here/naksha/lib/hub/NakshaHub.java | 24 ++++++++++--------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/here-naksha-lib-hub/src/main/java/com/here/naksha/lib/hub/NakshaHub.java b/here-naksha-lib-hub/src/main/java/com/here/naksha/lib/hub/NakshaHub.java index ccb4376a0..0e30d9539 100644 --- a/here-naksha-lib-hub/src/main/java/com/here/naksha/lib/hub/NakshaHub.java +++ b/here-naksha-lib-hub/src/main/java/com/here/naksha/lib/hub/NakshaHub.java @@ -47,7 +47,6 @@ import com.here.naksha.lib.core.util.IoHelp; import com.here.naksha.lib.core.util.json.Json; import com.here.naksha.lib.core.util.storage.RequestHelper; -import com.here.naksha.lib.core.util.storage.ResultHelper; import com.here.naksha.lib.core.view.ViewDeserialize; import com.here.naksha.lib.extmanager.ExtensionManager; import com.here.naksha.lib.extmanager.IExtensionManager; @@ -55,9 +54,7 @@ import com.here.naksha.lib.hub.storages.NHAdminStorage; import com.here.naksha.lib.hub.storages.NHSpaceStorage; import com.here.naksha.lib.psql.PsqlStorage; - import java.util.*; - import org.jetbrains.annotations.ApiStatus; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; @@ -242,8 +239,7 @@ private static WriteXyzCollections createAdminCollectionsRequest() { new Exception("Unable to read custom/default config from Admin DB. " + er, er.exception)); } else { try { - List nakshaHubConfigs = - readFeaturesFromResult(rdResult, NakshaHubConfig.class); + List nakshaHubConfigs = readFeaturesFromResult(rdResult, NakshaHubConfig.class); for (final NakshaHubConfig cfg : nakshaHubConfigs) { if (cfg.getId().equals(configId)) { customDbCfg = cfg; @@ -307,15 +303,21 @@ private static WriteXyzCollections createAdminCollectionsRequest() { try (IReadSession readSession = getAdminStorage().newReadSession(nakshaContext, false)) { rdResult = readSession.execute(request); } catch (Exception e) { - logger.error("Failed during reading extension handler configurations from collections {}. ", request.getCollections(), e); + logger.error( + "Failed during reading extension handler configurations from collections {}. ", + request.getCollections(), + e); throw new RuntimeException("Failed reading extension handler configurations", e); } final List eventHandlers; try { eventHandlers = readFeaturesFromResult(rdResult, EventHandler.class); - } catch (NoCursor e) { - logger.error("NoCursor exception encountered while reading Extension based Handlers", e); - throw new RuntimeException("Failed to open Cursor while reading Extension based Handlers", e); + } catch (NoCursor | NoSuchElementException e) { + logger.info("No relevant handlers found for Extension loading", e); + return new ExtensionConfig( + System.currentTimeMillis() + nakshaHubConfig.extensionConfigParams.getIntervalMs(), + Collections.emptyList(), + null); } Set extensionIds = new HashSet<>(); @@ -324,7 +326,7 @@ private static WriteXyzCollections createAdminCollectionsRequest() { if (extensionId != null && extensionId.contains(":")) { extensionIds.add(extensionId); } else { - logger.error("Environment is missing for an extension Id {}", extensionId); + logger.error("Environment is missing for an extension Id"); } } @@ -356,7 +358,7 @@ private List loadExtensionConfigFromS3(String extensionRootPath, Set< } filePath = extensionRootPath + extensionIdWotEnv + "/" + extensionIdWotEnv + "-" + version + "." - + env.toLowerCase() + ".json"; + + env.toLowerCase() + ".json"; String exJson; try { exJson = s3Helper.getFileContent(filePath); From 44aab834a3909e3016cde1d5fd31daeeea763e22 Mon Sep 17 00:00:00 2001 From: Rohit Lakde <37445331+rlakde@users.noreply.github.com> Date: Fri, 3 Jan 2025 14:57:10 -0600 Subject: [PATCH 34/39] Support multiple JWT public keys (#395) (#396) * Updated doc about existing JWT loading * added support for additional pub key * updated readme * Updated version to 2.2.1 * Updated doc for pvt and pub key * Added constants in NakshaHubConfig constructor * addressed review comments Co-authored-by: Hiren Patel <80465571+hirenkp2000@users.noreply.github.com> --- README.md | 26 +++++-- build.gradle.kts | 4 +- docker/Dockerfile | 1 + docker/README.md | 6 +- docker/cloud-config.json | 6 +- docker/run-app.sh | 28 ++++++- .../here/naksha/app/service/NakshaApp.java | 26 +++---- .../http/auth/NakshaJwtAuthHandler.java | 5 +- .../src/main/resources/mock-config.json | 13 ---- .../test-config-with-extensions.json | 9 +-- .../src/main/resources/test-config.json | 9 +-- .../here/naksha/lib/hub/NakshaHubConfig.java | 76 +++++++++++++------ .../resources/unit_test_data/mock_config.json | 2 +- 13 files changed, 134 insertions(+), 77 deletions(-) delete mode 100644 here-naksha-app-service/src/main/resources/mock-config.json diff --git a/README.md b/README.md index 2247681f5..84e331a96 100644 --- a/README.md +++ b/README.md @@ -103,9 +103,9 @@ java -jar # Example 1 : Start service with test config against default Database URL (useful for local env) java -jar build/libs/naksha-2.0.6-all.jar test-config -# Example 2 : Start service with given custom config and custom database URL (useful for cloud env) +# Example 2 : Start service with given custom config (using NAKSHA_CONFIG_PATH) and custom database URL (useful for cloud env) java -jar build/libs/naksha-2.0.6-all.jar cloud-config 'jdbc:postgresql://localhost:5432/postgres?user=postgres&password=pswd&schema=naksha&app=naksha_local&id=naksha_admin_db' -# Example 3 : Start service with given custom config and default (local) database URL +# Example 3 : Start service with given custom config (using NAKSHA_CONFIG_PATH) and default (local) database URL java -jar build/libs/naksha-2.0.6-all.jar custom-config ``` @@ -126,17 +126,16 @@ Once application is UP, the OpenAPI specification is accessible at `http(s)://{h The service persists out of modules with a bootstrap code to start the service. Service provides default configuration in [default-config.json](here-naksha-lib-hub/src/main/resources/config/default-config.json). The custom (external) configuration file can be supplied by modifying environment variable or by creating the `default-config.json` file in the corresponding configuration folder. -The exact configuration folder is platform dependent, but generally follows the [XGD user configuration directory](https://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html), standard, so on Linux being by default `~/.config/naksha/v{x.x.x}/`. For Windows the files will reside in the [CSIDL_PROFILE](https://learn.microsoft.com/en-us/windows/win32/shell/csidl?redirectedfrom=MSDN) folder, by default `C:\Users\{username}\.config\naksha\v{x.x.x}\`. -Here `{x.x.x}` is the Naksha application version (for example, if version is `2.0.7`, then path will be `...\.config\naksha\v2.0.7`) +The exact configuration folder is platform dependent, but generally follows the [XGD user configuration directory](https://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html), standard, so on Linux being by default `~/.config/naksha/`. For Windows the files will reside in the [CSIDL_PROFILE](https://learn.microsoft.com/en-us/windows/win32/shell/csidl?redirectedfrom=MSDN) folder, by default `C:\Users\{username}\.config\naksha\`. -Next to this, an explicit location can be specified via the environment variable `NAKSHA_CONFIG_PATH`, this path will not be extended by the `naksha/v{x.x.x}` folder, so you can directly specify where to keep the config files. This is important when you want to start multiple versions of the service: `NAKSHA_CONFIG_PATH=~/.config/naksha/ java -jar naksha.jar {arguments}`. +Next to this, an explicit location can be specified via the environment variable `NAKSHA_CONFIG_PATH`, this path will not be extended by the `naksha/` folder, so you can directly specify where to keep the config files. This is important when you want to start multiple versions of the service: `NAKSHA_CONFIG_PATH=~/.config/naksha/ java -jar naksha.jar {arguments}`. In the custom config file, the name of the individual properties can be set as per source code here [NakshaHubConfig](here-naksha-lib-hub/src/main/java/com/here/naksha/lib/hub/NakshaHubConfig.java). All properties annotated with `@JsonProperty` can be set in custom config file. Config file is loaded using `{config-id}` supplied as CLI argument, as per following precedence on file location (first match wins): 1. using env variable `NAKSHA_CONFIG_PATH` (full path will be `$NAKSHA_CONFIG_PATH/{config-id}.json`) -2. as per user's home directory `user.home` (full path will be `{user-home}/.config/naksha/v{x.x.x}/{config-id}.json` ) +2. as per user's home directory `user.home` (full path will be `{user-home}/.config/naksha/{config-id}.json` ) 3. as per config previously loaded in Naksha Admin Storage (PostgreSQL database) 4. default config loaded from jar (`here-naksha-lib-hub/src/main/resources/config/default-config.json`) @@ -154,6 +153,21 @@ vi $NAKSHA_CONFIG_PATH/default-config.json java -jar naksha.jar default-config ``` +The config also accepts custom RSA256 Private key and multiple Public key files (in PEM format) to support JWT signing/verification operations. + +* If custom Private key not provided, default will be loaded from Jar bundled resource [here-naksha-app-service/src/main/resources/auth/jwt.key](here-naksha-app-service/src/main/resources/auth/jwt.key). +* If custom Public key not provided, default will be loaded from Jar bundled resource [here-naksha-app-service/src/main/resources/auth/jwt.pub](here-naksha-app-service/src/main/resources/auth/jwt.pub). + +Sample commands to generate the custom key files: + +```bash +# Generate private key +openssl genrsa -out ./custom_rsa256.key 2048 + +# Generate public key (using above private key) +openssl rsa -in ./custom_rsa256.key -pubout -outform PEM -out ./custom_rsa256.pub +``` + # Usage Start using the service by creating a _space_: diff --git a/build.gradle.kts b/build.gradle.kts index 4534d59da..0895a31b4 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -158,7 +158,9 @@ subprojects { // excluding tests where Builder pattern gets broken by palantir targetExclude("src/test/**") encoding("UTF-8") - val YEAR = LocalDateTime.now().format(DateTimeFormatter.ofPattern("yyyy")) + // TODO - Disabling auto-correction to 2025 for now. Will handle via separate change. + //val YEAR = LocalDateTime.now().format(DateTimeFormatter.ofPattern("yyyy")) + val YEAR = 2024 licenseHeader(""" /* * Copyright (C) 2017-$YEAR HERE Europe B.V. diff --git a/docker/Dockerfile b/docker/Dockerfile index 2c5b9a58a..9b5fdef0d 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -24,6 +24,7 @@ ENV NAKSHA_ADMIN_DB_URL 'jdbc:postgresql://host.docker.internal:5432/postgres?us ENV NAKSHA_EXTENSION_S3_BUCKET 'naksha-pvt-releases' ENV NAKSHA_JWT_PVT_KEY '' ENV NAKSHA_JWT_PUB_KEY '' +ENV NAKSHA_JWT_PUB_KEY_2 '' ENV JAVA_OPTS '' # Execute Shell Script diff --git a/docker/README.md b/docker/README.md index 7d9ed86d2..0e92394bd 100644 --- a/docker/README.md +++ b/docker/README.md @@ -30,8 +30,9 @@ To get Naksha container running, one must do the following: use, `jdbc:postgresql://host.docker.internal:5432/postgres?user=postgres&password=password&schema=naksha&app=naksha_local&id=naksha_admin_db` by default - `NAKSHA_EXTENSION_S3_BUCKET`: S3 bucket name or S3 bucket access point.The default value is `naksha-pvt-releases`. - - `NAKSHA_JWT_PVT_KEY`: Naksha JWT private key. If not provided then it will load it from `here-naksha-app-service/src/main/resources/auth/jwt.key`. - - `NAKSHA_JWT_PUB_KEY`: Naksha JWT public key. If not provided then it will load it from `here-naksha-app-service/src/main/resources/auth/jwt.pub`. + - `NAKSHA_JWT_PVT_KEY`: Naksha JWT private key. + - `NAKSHA_JWT_PUB_KEY`: Naksha JWT public key. + - `NAKSHA_JWT_PUB_KEY_2`: Additional Naksha JWT public key, if needed to validate the JWT signed by some other application's PVT key. - `JAVA_OPTS`: Any custom java options like `-Xms1024m -Xmx2048m` When connecting Naksha app to database, one has to consider container networking - if your @@ -61,6 +62,7 @@ To get Naksha container running, one must do the following: --env NAKSHA_EXTENSION_S3_BUCKET= \ --env NAKSHA_JWT_PVT_KEY= \ --env NAKSHA_JWT_PUB_KEY= \ + --env NAKSHA_JWT_PUB_KEY_2= \ --env JAVA_OPTS="-Xms1024m -Xmx2048m" \ -p 8080:8080 \ local-naksha-app diff --git a/docker/cloud-config.json b/docker/cloud-config.json index db60ca862..bb7db8f3a 100644 --- a/docker/cloud-config.json +++ b/docker/cloud-config.json @@ -1,11 +1,13 @@ { "id": "cloud-config", - "type": "Config", + "type": "Feature", "httpPort": 7080, "requestBodyLimit": 25, + "authMode": "JWT", + "jwtPvtKeyPath": "SOME_PVT_KEY_PATH", + "jwtPubKeyPaths": "SOME_PUB_KEY_PATHS", "maxParallelRequestsPerCPU": 100, "maxPctParallelRequestsPerActor": 25, - "authMode": "JWT", "extensionConfigParams": { "whitelistClasses": [ "java.*", "javax.*", "com.here.*", "jdk.internal.reflect.*", "com.sun.*", "org.w3c.dom.*", "sun.misc.*","org.locationtech.jts.*"], "intervalms": 30000, diff --git a/docker/run-app.sh b/docker/run-app.sh index 1b342a029..51b9e3c14 100644 --- a/docker/run-app.sh +++ b/docker/run-app.sh @@ -3,13 +3,15 @@ # Set the NAKSHA_CONFIG_PATH export NAKSHA_CONFIG_PATH=/home/naksha/app/config/ -# Replace placeholder in cloud-config.json -sed -i "s+SOME_S3_BUCKET+${NAKSHA_EXTENSION_S3_BUCKET}+g" /home/naksha/app/config/cloud-config.json +NAKSHA_PVT_KEY_PATH="" +NAKSHA_PUB_KEY_PATHS="" # Check if NAKSHA_JWT_PVT_KEY is set and create jwt.key file if it is if [ -n "$NAKSHA_JWT_PVT_KEY" ]; then mkdir -p ${NAKSHA_CONFIG_PATH}auth/ - echo "$NAKSHA_JWT_PVT_KEY" | sed 's/\\n/\n/g' > ${NAKSHA_CONFIG_PATH}auth/jwt.key + KEY_FILE_PATH=auth/jwt.key + echo "$NAKSHA_JWT_PVT_KEY" | sed 's/\\n/\n/g' > ${NAKSHA_CONFIG_PATH}${KEY_FILE_PATH} + NAKSHA_PVT_KEY_PATH=${KEY_FILE_PATH} echo "Using custom JWT private key" else echo "No custom JWT private key supplied" @@ -18,11 +20,29 @@ fi # Check if NAKSHA_JWT_PUB_KEY is set and create jwt.pub file if it is if [ -n "$NAKSHA_JWT_PUB_KEY" ]; then mkdir -p ${NAKSHA_CONFIG_PATH}auth/ - echo "$NAKSHA_JWT_PUB_KEY" | sed 's/\\n/\n/g' > ${NAKSHA_CONFIG_PATH}auth/jwt.pub + KEY_FILE_PATH=auth/jwt.pub + echo "$NAKSHA_JWT_PUB_KEY" | sed 's/\\n/\n/g' > ${NAKSHA_CONFIG_PATH}${KEY_FILE_PATH} + NAKSHA_PUB_KEY_PATHS=${KEY_FILE_PATH} echo "Using custom JWT public key" else echo "No custom JWT public key supplied" fi +# Check if NAKSHA_JWT_PUB_KEY_2 is set and create jwt_2.pub file if it is +if [ -n "$NAKSHA_JWT_PUB_KEY_2" ]; then + mkdir -p ${NAKSHA_CONFIG_PATH}auth/ + KEY_FILE_PATH=auth/jwt_2.pub + echo "$NAKSHA_JWT_PUB_KEY_2" | sed 's/\\n/\n/g' > ${NAKSHA_CONFIG_PATH}${KEY_FILE_PATH} + NAKSHA_PUB_KEY_PATHS=${NAKSHA_PUB_KEY_PATHS},${KEY_FILE_PATH} + echo "Using custom JWT public key 2" +else + echo "No custom JWT public key 2 supplied" +fi + +# Replace all placeholders in cloud-config.json +sed -i "s+SOME_S3_BUCKET+${NAKSHA_EXTENSION_S3_BUCKET}+g" /home/naksha/app/config/cloud-config.json +sed -i "s+SOME_PVT_KEY_PATH+${NAKSHA_PVT_KEY_PATH}+g" /home/naksha/app/config/cloud-config.json +sed -i "s+SOME_PUB_KEY_PATHS+${NAKSHA_PUB_KEY_PATHS}+g" /home/naksha/app/config/cloud-config.json + # Start the application java $JAVA_OPTS -jar /home/naksha/app/naksha-*-all.jar $NAKSHA_CONFIG_ID $NAKSHA_ADMIN_DB_URL \ No newline at end of file diff --git a/here-naksha-app-service/src/main/java/com/here/naksha/app/service/NakshaApp.java b/here-naksha-app-service/src/main/java/com/here/naksha/app/service/NakshaApp.java index d34a72217..e9bc98429 100644 --- a/here-naksha-app-service/src/main/java/com/here/naksha/app/service/NakshaApp.java +++ b/here-naksha-app-service/src/main/java/com/here/naksha/app/service/NakshaApp.java @@ -102,13 +102,13 @@ private static void printUsage() { err.println("Examples:"); err.println(" "); err.println(" Example 1 : Start service with given config and default (local) database URL"); - err.println(" java -jar naksha.jar default-config"); + err.println(" java -jar naksha.jar test-config"); err.println(" "); err.println(" Example 2 : Start service with given config and custom database URL"); - err.println(" java -jar naksha.jar default-config '" + DEFAULT_URL + "'"); + err.println(" java -jar naksha.jar test-config '" + DEFAULT_URL + "'"); err.println(" "); - err.println(" Example 3 : Start service with mock config (with in-memory hub)"); - err.println(" java -jar naksha.jar mock-config"); + err.println(" Example 3 : Start service with custom config (using custom NAKSHA_CONFIG_PATH)"); + err.println(" java -jar naksha.jar custom-config"); err.println(" "); err.flush(); } @@ -220,20 +220,20 @@ public NakshaApp( } this.vertx = Vertx.vertx(this.vertxOptions); - final String jwtKey; - final String jwtPub; + final List keyOptions = new ArrayList<>(); + // read JWT pvt key { - final String path = "auth/" + config.jwtName + ".key"; - jwtKey = readAuthKeyFile(path, NakshaHubConfig.APP_NAME); + final String keyContent = readAuthKeyFile(config.jwtPvtKeyPath, NakshaHubConfig.APP_NAME); + keyOptions.add(new PubSecKeyOptions().setAlgorithm("RS256").setBuffer(keyContent)); } - { - final String path = "auth/" + config.jwtName + ".pub"; - jwtPub = readAuthKeyFile(path, NakshaHubConfig.APP_NAME); + // read JWT pub keys + for (final String keyPath : config.jwtPubKeyPaths.split(",")) { + final String keyContent = readAuthKeyFile(keyPath, NakshaHubConfig.APP_NAME); + keyOptions.add(new PubSecKeyOptions().setAlgorithm("RS256").setBuffer(keyContent)); } this.authOptions = new JWTAuthOptions() .setJWTOptions(new JWTOptions().setAlgorithm("RS256")) - .addPubSecKey(new PubSecKeyOptions().setAlgorithm("RS256").setBuffer(jwtKey)) - .addPubSecKey(new PubSecKeyOptions().setAlgorithm("RS256").setBuffer(jwtPub)); + .setPubSecKeys(keyOptions); this.authProvider = new NakshaAuthProvider(this.vertx, this.authOptions); final WebClientOptions webClientOptions = new WebClientOptions(); diff --git a/here-naksha-app-service/src/main/java/com/here/naksha/app/service/http/auth/NakshaJwtAuthHandler.java b/here-naksha-app-service/src/main/java/com/here/naksha/app/service/http/auth/NakshaJwtAuthHandler.java index cb730f77a..1d2a558bb 100644 --- a/here-naksha-app-service/src/main/java/com/here/naksha/app/service/http/auth/NakshaJwtAuthHandler.java +++ b/here-naksha-app-service/src/main/java/com/here/naksha/app/service/http/auth/NakshaJwtAuthHandler.java @@ -44,7 +44,7 @@ public class NakshaJwtAuthHandler extends JWTAuthHandlerImpl { /** * The master JWT used for testing. */ - private final String MASTER_JWT = authProvider.generateToken(MASTER_JWT_PAYLOAD); + private static String MASTER_JWT = null; public NakshaJwtAuthHandler( @NotNull JWTAuth authProvider, @NotNull NakshaHubConfig hubConfig, @Nullable String realm) { @@ -57,6 +57,9 @@ public void authenticate(@NotNull RoutingContext context, @NotNull Handler<@NotN if (hubConfig.authMode == AuthorizationMode.DUMMY && !context.request().headers().contains(HttpHeaders.AUTHORIZATION)) { // Use the master JWT for testing in DUMMY auth mode with no JWT provided in request + if (MASTER_JWT == null) { + MASTER_JWT = authProvider.generateToken(MASTER_JWT_PAYLOAD); + } context.request().headers().set(HttpHeaders.AUTHORIZATION, "Bearer " + MASTER_JWT); } // TODO: If compressed JWTs are supported diff --git a/here-naksha-app-service/src/main/resources/mock-config.json b/here-naksha-app-service/src/main/resources/mock-config.json deleted file mode 100644 index a6679c724..000000000 --- a/here-naksha-app-service/src/main/resources/mock-config.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "id": "mock-config", - "type": "Config", - - "httpPort": 8080, - "env": "local", - "jwtName": "jwt", - "hubClassName": "com.here.naksha.lib.hub.mock.NakshaHubMock", - "maintenanceInitialDelayInMins": 60, - "maintenanceIntervalInMins": 720, - "maintenancePoolCoreSize": 5, - "maintenancePoolMaxSize": 20 -} \ No newline at end of file diff --git a/here-naksha-app-service/src/main/resources/test-config-with-extensions.json b/here-naksha-app-service/src/main/resources/test-config-with-extensions.json index 5be333f36..1ffeb2100 100644 --- a/here-naksha-app-service/src/main/resources/test-config-with-extensions.json +++ b/here-naksha-app-service/src/main/resources/test-config-with-extensions.json @@ -1,16 +1,13 @@ { "id": "test-config", - "type": "Config", + "type": "Feature", "httpPort": 8080, "env": "local", "requestBodyLimit": 25, "authMode": "DUMMY", - "jwtName": "jwt", - "maintenanceInitialDelayInMins": 60, - "maintenanceIntervalInMins": 720, - "maintenancePoolCoreSize": 5, - "maintenancePoolMaxSize": 20, + "jwtPvtKeyPath": "auth/jwt.key", + "jwtPubKeyPaths": "auth/jwt.pub", "maxParallelRequestsPerCPU": 30, "maxPctParallelRequestsPerActor": 100, "storageParams": { diff --git a/here-naksha-app-service/src/main/resources/test-config.json b/here-naksha-app-service/src/main/resources/test-config.json index c5d31cad8..b0e9d2f6d 100644 --- a/here-naksha-app-service/src/main/resources/test-config.json +++ b/here-naksha-app-service/src/main/resources/test-config.json @@ -1,16 +1,13 @@ { "id": "test-config", - "type": "Config", + "type": "Feature", "httpPort": 8080, "env": "local", "requestBodyLimit": 25, "authMode": "DUMMY", - "jwtName": "jwt", - "maintenanceInitialDelayInMins": 60, - "maintenanceIntervalInMins": 720, - "maintenancePoolCoreSize": 5, - "maintenancePoolMaxSize": 20, + "jwtPvtKeyPath": "auth/jwt.key", + "jwtPubKeyPaths": "auth/jwt.pub", "maxParallelRequestsPerCPU": 30, "maxPctParallelRequestsPerActor": 100, "storageParams": { diff --git a/here-naksha-lib-hub/src/main/java/com/here/naksha/lib/hub/NakshaHubConfig.java b/here-naksha-lib-hub/src/main/java/com/here/naksha/lib/hub/NakshaHubConfig.java index aa8a8f320..eab00743d 100644 --- a/here-naksha-lib-hub/src/main/java/com/here/naksha/lib/hub/NakshaHubConfig.java +++ b/here-naksha-lib-hub/src/main/java/com/here/naksha/lib/hub/NakshaHubConfig.java @@ -22,7 +22,6 @@ import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonInclude.Include; import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.annotation.JsonTypeName; import com.here.naksha.lib.core.NakshaVersion; import com.here.naksha.lib.core.models.geojson.implementation.XyzFeature; import com.here.naksha.lib.core.util.json.JsonSerializable; @@ -39,7 +38,6 @@ /** * The Naksha-Hub service configuration. */ -@JsonTypeName(value = "Config") public final class NakshaHubConfig extends XyzFeature implements JsonSerializable { private static final Logger logger = LoggerFactory.getLogger(NakshaHubConfig.class); @@ -81,21 +79,40 @@ public final class NakshaHubConfig extends XyzFeature implements JsonSerializabl return NakshaHub.class.getName(); } + /** + * Returns a default relative path to Private Key useful for JWT signing + * + * @return The default private key path + */ + public static @NotNull String defaultJwtPvtKeyPath() { + return "auth/jwt.key"; + } + + /** + * Returns default relative paths to Public Keys useful for JWT signature verification + * + * @return The default public key paths + */ + public static @NotNull String defaultJwtPubKeyPaths() { + return "auth/jwt.pub"; + } + @JsonCreator NakshaHubConfig( @JsonProperty("id") @NotNull String id, - @JsonProperty("hubClassName") @Nullable String hubClassName, - @JsonProperty("userAgent") @Nullable String userAgent, - @JsonProperty("appId") @Nullable String appId, - @JsonProperty("author") @Nullable String author, - @JsonProperty("httpPort") @Nullable Integer httpPort, - @JsonProperty("hostname") @Nullable String hostname, - @JsonProperty("endpoint") @Nullable String endpoint, - @JsonProperty("env") @Nullable String env, - @JsonProperty("webRoot") @Nullable String webRoot, + @JsonProperty(HUB_CLASS_NAME) @Nullable String hubClassName, + @JsonProperty(USER_AGENT) @Nullable String userAgent, + @JsonProperty(APP_ID) @Nullable String appId, + @JsonProperty(AUTHOR) @Nullable String author, + @JsonProperty(HTTP_PORT) @Nullable Integer httpPort, + @JsonProperty(HOSTNAME) @Nullable String hostname, + @JsonProperty(ENDPOINT) @Nullable String endpoint, + @JsonProperty(ENV) @Nullable String env, + @JsonProperty(WEB_ROOT) @Nullable String webRoot, @JsonProperty(NAKSHA_AUTH) @Nullable AuthorizationMode authMode, - @JsonProperty("jwtName") @Nullable String jwtName, - @JsonProperty("debug") @Nullable Boolean debug, + @JsonProperty(JWT_PVT_KEY_PATH) @Nullable String jwtPvtKeyPath, + @JsonProperty(JWT_PUB_KEY_PATHS) @Nullable String jwtPubKeyPaths, + @JsonProperty(DEBUG) @Nullable Boolean debug, @JsonProperty("maintenanceIntervalInMins") @Nullable Integer maintenanceIntervalInMins, @JsonProperty("maintenanceInitialDelayInMins") @Nullable Integer maintenanceInitialDelayInMins, @JsonProperty("maintenancePoolCoreSize") @Nullable Integer maintenancePoolCoreSize, @@ -107,8 +124,8 @@ public final class NakshaHubConfig extends XyzFeature implements JsonSerializabl @JsonProperty("maxPctParallelRequestsPerActor") @Nullable Integer maxPctParallelRequestsPerActor) { super(id); if (httpPort != null && (httpPort < 0 || httpPort > 65535)) { - logger.atError() - .setMessage("Invalid port in Naksha configuration: {}") + logger.atWarn() + .setMessage("Invalid port in Naksha configuration: {}, falling back to default \"8080\"") .addArgument(httpPort) .log(); httpPort = 8080; @@ -119,8 +136,8 @@ public final class NakshaHubConfig extends XyzFeature implements JsonSerializabl try { hostname = InetAddress.getLocalHost().getHostAddress(); } catch (UnknownHostException e) { - logger.atError() - .setMessage("Unable to resolve the hostname using Java's API.") + logger.atWarn() + .setMessage("Unable to resolve the hostname using Java's API, using default \"localhost\".") .setCause(e) .log(); hostname = "localhost"; @@ -167,7 +184,10 @@ public final class NakshaHubConfig extends XyzFeature implements JsonSerializabl this.env = env; this.webRoot = webRoot; this.authMode = (authMode == null) ? AuthorizationMode.JWT : authMode; - this.jwtName = jwtName != null && !jwtName.isEmpty() ? jwtName : "jwt"; + this.jwtPvtKeyPath = + (jwtPvtKeyPath != null && !jwtPvtKeyPath.isEmpty()) ? jwtPvtKeyPath : defaultJwtPvtKeyPath(); + this.jwtPubKeyPaths = + (jwtPubKeyPaths != null && !jwtPubKeyPaths.isEmpty()) ? jwtPubKeyPaths : defaultJwtPubKeyPaths(); this.userAgent = userAgent != null && !userAgent.isEmpty() ? userAgent : defaultAppName(); this.debug = Boolean.TRUE.equals(debug); this.maintenanceIntervalInMins = @@ -263,13 +283,25 @@ private String getEnv(String env) { @JsonProperty(WEB_ROOT) public final @Nullable String webRoot; - public static final String JWT_NAME = "jwtName"; + public static final String JWT_PVT_KEY_PATH = "jwtPvtKeyPath"; + /** + * The relative path to Private key file to support JWT signing (e.g. {@code "auth/jwt.key"}). + * The path should be relative to the directory where config file is supplied. + * For example - if config file is {@code "/home/config/cloud-config.json"} then the key path {@code "auth/jwt.key"} + * will be considered relative to {@code "/home/config"} folder, resulting into absolute path as {@code "/home/config/auth/jwt.key}" + */ + @JsonProperty(JWT_PVT_KEY_PATH) + public final @NotNull String jwtPvtKeyPath; + public static final String JWT_PUB_KEY_PATHS = "jwtPubKeyPaths"; /** - * The JWT key files to be read from the disk ({@code "~/.config/naksha/auth/$.(key|pub)"}). + * The comma separated relative paths to Public key files to support JWT signature verification (e.g. {@code "auth/jwt.pub,auth/jwt_2.pub"}). + * The path should be relative to the directory where config file is supplied. + * For example - if config file is {@code "/home/config/cloud-config.json"} then the key path {@code "auth/jwt.pub"} + * will be considered relative to {@code "/home/config"} folder, resulting into absolute path as {@code "/home/config/auth/jwt.pub}" */ - @JsonProperty(JWT_NAME) - public final @NotNull String jwtName; + @JsonProperty(JWT_PUB_KEY_PATHS) + public final @NotNull String jwtPubKeyPaths; public static final String USER_AGENT = "userAgent"; diff --git a/here-naksha-lib-hub/src/test/resources/unit_test_data/mock_config.json b/here-naksha-lib-hub/src/test/resources/unit_test_data/mock_config.json index 076b651fc..fe31543a6 100644 --- a/here-naksha-lib-hub/src/test/resources/unit_test_data/mock_config.json +++ b/here-naksha-lib-hub/src/test/resources/unit_test_data/mock_config.json @@ -1,6 +1,6 @@ { "id": "mock-config", - "type": "Config", + "type": "Feature", "httpPort": 8080, "//hostname": "some-instance.aws.com", From 7a70459db5d833d0769b519c4258ab1f255fb81e Mon Sep 17 00:00:00 2001 From: Rohit Lakde <37445331+rlakde@users.noreply.github.com> Date: Sun, 5 Jan 2025 18:29:55 -0600 Subject: [PATCH 35/39] Update ExtensionCache.java --- .../com/here/naksha/lib/extmanager/ExtensionCache.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/here-naksha-lib-ext-manager/src/main/java/com/here/naksha/lib/extmanager/ExtensionCache.java b/here-naksha-lib-ext-manager/src/main/java/com/here/naksha/lib/extmanager/ExtensionCache.java index 1c8fb6223..847331982 100644 --- a/here-naksha-lib-ext-manager/src/main/java/com/here/naksha/lib/extmanager/ExtensionCache.java +++ b/here-naksha-lib-ext-manager/src/main/java/com/here/naksha/lib/extmanager/ExtensionCache.java @@ -49,6 +49,7 @@ public class ExtensionCache { private static final ConcurrentHashMap loaderCache = new ConcurrentHashMap<>(); private static final Map jarClientMap = new HashMap<>(); private final @NotNull INaksha naksha; + private static final String WHITE_LIST_CLASSES = "whitelistClasses"; static { jarClientMap.put(JarClientType.S3.getType(), new AmazonS3Helper()); @@ -103,7 +104,11 @@ private void publishIntoCache(KVPair result, ExtensionConfig ex IExtensionInit initObj = null; ClassLoader loader; try { - loader = ClassLoaderHelper.getClassLoader(jarFile, extensionConfig.getWhilelistDelegateClass()); + @SuppressWarnings("unchecked") + List whitelistClasses = (List) extension + .getProperties() + .getOrDefault(WHITE_LIST_CLASSES, extensionConfig.getWhilelistDelegateClass()); + loader = ClassLoaderHelper.getClassLoader(jarFile, whitelistClasses); } catch (Exception e) { logger.error("Failed to load extension jar " + extensionIdWthEnv, e); return; From 7c5dbb0efcd338fe66d7235b867d0e9e98c3b4fe Mon Sep 17 00:00:00 2001 From: Rohit Lakde <37445331+rlakde@users.noreply.github.com> Date: Tue, 14 Jan 2025 14:37:15 -0600 Subject: [PATCH 36/39] Update gradle.properties --- gradle.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle.properties b/gradle.properties index cb06ea2e9..1cf9f87a5 100644 --- a/gradle.properties +++ b/gradle.properties @@ -10,5 +10,5 @@ mavenPassword=YourPassword # When updating the version, please as well consider: # - here-naksha-lib-core/src/main/com/here/naksha/lib/core/NakshaVersion (static property: latest) # - here-naksha-app-service/src/main/resources/swagger/openapi.yaml (info.version property) -version=2.2.1 +version=2.2.2 From 698fc56b189eac6c31904f94cfba8d9ce523e547 Mon Sep 17 00:00:00 2001 From: Rohit Lakde <37445331+rlakde@users.noreply.github.com> Date: Tue, 14 Jan 2025 14:38:34 -0600 Subject: [PATCH 37/39] Update openapi.yaml --- here-naksha-app-service/src/main/resources/swagger/openapi.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/here-naksha-app-service/src/main/resources/swagger/openapi.yaml b/here-naksha-app-service/src/main/resources/swagger/openapi.yaml index bb3e48c6a..8a110d2fa 100644 --- a/here-naksha-app-service/src/main/resources/swagger/openapi.yaml +++ b/here-naksha-app-service/src/main/resources/swagger/openapi.yaml @@ -12,7 +12,7 @@ servers: info: title: "Naksha Hub-API" description: "Naksha Hub-API is a REST API to provide simple access to geo data." - version: "2.2.1" + version: "2.2.2" security: - AccessToken: [ ] From cbc078d64afab3c8dca7e3184e761ed3b4c5652c Mon Sep 17 00:00:00 2001 From: Rohit Lakde <37445331+rlakde@users.noreply.github.com> Date: Tue, 14 Jan 2025 14:39:58 -0600 Subject: [PATCH 38/39] Update NakshaVersion.java --- .../src/main/java/com/here/naksha/lib/core/NakshaVersion.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/here-naksha-lib-core/src/main/java/com/here/naksha/lib/core/NakshaVersion.java b/here-naksha-lib-core/src/main/java/com/here/naksha/lib/core/NakshaVersion.java index 0cfa490c6..900ce433e 100644 --- a/here-naksha-lib-core/src/main/java/com/here/naksha/lib/core/NakshaVersion.java +++ b/here-naksha-lib-core/src/main/java/com/here/naksha/lib/core/NakshaVersion.java @@ -59,12 +59,13 @@ public class NakshaVersion implements Comparable { public static final String v2_1_1 = "2.1.1"; public static final String v2_2_0 = "2.2.0"; public static final String v2_2_1 = "2.2.1"; + public static final String v2_2_2 = "2.2.2"; /** * The latest version of the naksha-extension stored in the resources. */ @AvailableSince(v2_0_5) - public static final NakshaVersion latest = of(v2_2_1); + public static final NakshaVersion latest = of(v2_2_2); private final int major; private final int minor; From b0943f1671f5bc5501622327e90ba88e1f665b26 Mon Sep 17 00:00:00 2001 From: Rohit Lakde Date: Wed, 15 Jan 2025 09:03:14 -0600 Subject: [PATCH 39/39] Update logger as per review comment. --- .../java/com/here/naksha/lib/extmanager/ExtensionCache.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/here-naksha-lib-ext-manager/src/main/java/com/here/naksha/lib/extmanager/ExtensionCache.java b/here-naksha-lib-ext-manager/src/main/java/com/here/naksha/lib/extmanager/ExtensionCache.java index 847331982..77b1b2d66 100644 --- a/here-naksha-lib-ext-manager/src/main/java/com/here/naksha/lib/extmanager/ExtensionCache.java +++ b/here-naksha-lib-ext-manager/src/main/java/com/here/naksha/lib/extmanager/ExtensionCache.java @@ -108,6 +108,7 @@ private void publishIntoCache(KVPair result, ExtensionConfig ex List whitelistClasses = (List) extension .getProperties() .getOrDefault(WHITE_LIST_CLASSES, extensionConfig.getWhilelistDelegateClass()); + logger.info("Whitelist classes in use for extension {} are {}", extensionIdWthEnv, whitelistClasses); loader = ClassLoaderHelper.getClassLoader(jarFile, whitelistClasses); } catch (Exception e) { logger.error("Failed to load extension jar " + extensionIdWthEnv, e); @@ -126,7 +127,7 @@ private void publishIntoCache(KVPair result, ExtensionConfig ex extensionIdWthEnv, extension.getInitClassName()); } else { - logger.error("Extension does not implement IExtensionInit for extension {}", extensionIdWthEnv); + logger.error("InitClassName {} does not implement IExtensionInit for Extension {}", extension.getInitClassName(), extensionIdWthEnv); return; } } catch (Exception e) {