From 618344eec897046583da843e69497519164a37bb Mon Sep 17 00:00:00 2001 From: Lajos Veres Date: Thu, 12 Oct 2023 14:57:42 +0100 Subject: [PATCH 01/81] CORE-17371 fix CPI signerSummaryHash and fileChecksum in FlowContextPropertys (#4847) --- .../net/corda/flow/pipeline/runner/impl/FlowRunnerImpl.kt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/flow/flow-service/src/main/kotlin/net/corda/flow/pipeline/runner/impl/FlowRunnerImpl.kt b/components/flow/flow-service/src/main/kotlin/net/corda/flow/pipeline/runner/impl/FlowRunnerImpl.kt index f941bcb36ea..539674297b2 100644 --- a/components/flow/flow-service/src/main/kotlin/net/corda/flow/pipeline/runner/impl/FlowRunnerImpl.kt +++ b/components/flow/flow-service/src/main/kotlin/net/corda/flow/pipeline/runner/impl/FlowRunnerImpl.kt @@ -196,8 +196,8 @@ class FlowRunnerImpl @Activate constructor( this[FlowContextPropertyKeys.CPI_NAME] = metadata.cpiId.name this[FlowContextPropertyKeys.CPI_VERSION] = metadata.cpiId.version this[FlowContextPropertyKeys.CPI_SIGNER_SUMMARY_HASH] = - metadata.cpiId.signerSummaryHash.toHexString() - this[FlowContextPropertyKeys.CPI_FILE_CHECKSUM] = metadata.fileChecksum.toHexString() + metadata.cpiId.signerSummaryHash.toString() + this[FlowContextPropertyKeys.CPI_FILE_CHECKSUM] = metadata.fileChecksum.toString() this[FlowContextPropertyKeys.INITIAL_PLATFORM_VERSION] = platformInfoProvider.localWorkerPlatformVersion.toString() From 5340472afcd85b93ace20c60f82b23ffebf66af1 Mon Sep 17 00:00:00 2001 From: Dries Samyn Date: Thu, 12 Oct 2023 14:58:57 +0100 Subject: [PATCH 02/81] CORE-17623 - move-token-selection-processor (#4835) Move the Token Selection Processor in the stand-alone worker. --- .../workers/release/db-worker/build.gradle | 1 - .../corda/applications/workers/db/DBWorker.kt | 6 ---- .../workers/db/test/ConfigTests.kt | 5 --- .../token-selection-worker/build.gradle | 2 ++ .../token/selection/TokenSelectionWorker.kt | 15 +++++++-- .../services/TokenCacheEventProcessor.kt | 5 +++ processors/db-processor/build.gradle | 2 +- processors/token-cache-processor/build.gradle | 17 ++++++++++ .../cache/internal/TokenCacheProcessorImpl.kt | 33 ++++++++++++++++++- 9 files changed, 70 insertions(+), 16 deletions(-) diff --git a/applications/workers/release/db-worker/build.gradle b/applications/workers/release/db-worker/build.gradle index c20f73805a9..5b2477f22af 100644 --- a/applications/workers/release/db-worker/build.gradle +++ b/applications/workers/release/db-worker/build.gradle @@ -28,7 +28,6 @@ dependencies { // Processor list must be kept in sync with workersForProcessor in net.corda.cli.plugins.topicconfig.Create implementation project(':processors:db-processor') implementation project(':processors:scheduler-processor') - implementation project(':processors:token-cache-processor') implementation 'org.jetbrains.kotlin:kotlin-osgi-bundle' implementation "info.picocli:picocli:$picocliVersion" implementation 'net.corda:corda-base' diff --git a/applications/workers/release/db-worker/src/main/kotlin/net/corda/applications/workers/db/DBWorker.kt b/applications/workers/release/db-worker/src/main/kotlin/net/corda/applications/workers/db/DBWorker.kt index 2204c868f56..317e21fd1f3 100644 --- a/applications/workers/release/db-worker/src/main/kotlin/net/corda/applications/workers/db/DBWorker.kt +++ b/applications/workers/release/db-worker/src/main/kotlin/net/corda/applications/workers/db/DBWorker.kt @@ -18,7 +18,6 @@ import net.corda.osgi.api.Application import net.corda.osgi.api.Shutdown import net.corda.processors.db.DBProcessor import net.corda.processors.scheduler.SchedulerProcessor -import net.corda.processors.token.cache.TokenCacheProcessor import net.corda.schema.configuration.BootConfig.BOOT_DB import net.corda.tracing.configureTracing import net.corda.tracing.shutdownTracing @@ -36,8 +35,6 @@ import picocli.CommandLine.Option class DBWorker @Activate constructor( @Reference(service = DBProcessor::class) private val processor: DBProcessor, - @Reference(service = TokenCacheProcessor::class) - private val tokenCacheProcessor: TokenCacheProcessor, @Reference(service = SchedulerProcessor::class) private val schedulerProcessor: SchedulerProcessor, @Reference(service = Shutdown::class) @@ -69,7 +66,6 @@ class DBWorker @Activate constructor( JavaSerialisationFilter.install() - val params = getParams(args, DBWorkerParams()) if (printHelpOrVersion(params.defaultParams, DBWorker::class.java, shutDownService)) return @@ -86,7 +82,6 @@ class DBWorker @Activate constructor( ) webServer.start(params.defaultParams.workerServerPort) processor.start(config) - tokenCacheProcessor.start(config) schedulerProcessor.start(config) } @@ -94,7 +89,6 @@ class DBWorker @Activate constructor( logger.info("DB worker stopping.") processor.stop() webServer.stop() - tokenCacheProcessor.stop() schedulerProcessor.stop() shutdownTracing() } diff --git a/applications/workers/release/db-worker/src/test/kotlin/net/corda/applications/workers/db/test/ConfigTests.kt b/applications/workers/release/db-worker/src/test/kotlin/net/corda/applications/workers/db/test/ConfigTests.kt index 7e1e9faca85..708bb7a5add 100644 --- a/applications/workers/release/db-worker/src/test/kotlin/net/corda/applications/workers/db/test/ConfigTests.kt +++ b/applications/workers/release/db-worker/src/test/kotlin/net/corda/applications/workers/db/test/ConfigTests.kt @@ -54,7 +54,6 @@ class ConfigTests { val dbWorker = DBWorker( dbProcessor, mock(), - mock(), DummyShutdown(), DummyLifecycleRegistry(), DummyWebServer(), @@ -99,7 +98,6 @@ class ConfigTests { val dbWorker = DBWorker( dbProcessor, mock(), - mock(), DummyShutdown(), DummyLifecycleRegistry(), DummyWebServer(), @@ -133,7 +131,6 @@ class ConfigTests { val dbWorker = DBWorker( dbProcessor, mock(), - mock(), DummyShutdown(), DummyLifecycleRegistry(), DummyWebServer(), @@ -166,7 +163,6 @@ class ConfigTests { val dbWorker = DBWorker( dbProcessor, mock(), - mock(), DummyShutdown(), DummyLifecycleRegistry(), DummyWebServer(), @@ -193,7 +189,6 @@ class ConfigTests { val dbWorker = DBWorker( dbProcessor, mock(), - mock(), DummyShutdown(), DummyLifecycleRegistry(), DummyWebServer(), diff --git a/applications/workers/release/token-selection-worker/build.gradle b/applications/workers/release/token-selection-worker/build.gradle index b7e4e1324c5..46b60623fac 100644 --- a/applications/workers/release/token-selection-worker/build.gradle +++ b/applications/workers/release/token-selection-worker/build.gradle @@ -24,6 +24,7 @@ dependencies { implementation project(':libs:platform-info') implementation project(':libs:tracing') implementation project(':libs:web:web') + implementation project(':processors:token-cache-processor') implementation 'org.jetbrains.kotlin:kotlin-osgi-bundle' implementation "info.picocli:picocli:$picocliVersion" implementation 'net.corda:corda-base' @@ -36,6 +37,7 @@ dependencies { runtimeOnly "org.osgi:org.osgi.util.function:$osgiUtilFunctionVersion" runtimeOnly "org.osgi:org.osgi.util.promise:$osgiUtilPromiseVersion" + runtimeOnly project(':libs:messaging:kafka-message-bus-impl') runtimeOnly project(':libs:tracing-impl') testImplementation 'org.osgi:osgi.core' diff --git a/applications/workers/release/token-selection-worker/src/main/kotlin/net/corda/applications/workers/token/selection/TokenSelectionWorker.kt b/applications/workers/release/token-selection-worker/src/main/kotlin/net/corda/applications/workers/token/selection/TokenSelectionWorker.kt index d2e444a2e3f..832270a743a 100644 --- a/applications/workers/release/token-selection-worker/src/main/kotlin/net/corda/applications/workers/token/selection/TokenSelectionWorker.kt +++ b/applications/workers/release/token-selection-worker/src/main/kotlin/net/corda/applications/workers/token/selection/TokenSelectionWorker.kt @@ -5,6 +5,7 @@ import net.corda.applications.workers.workercommon.DefaultWorkerParams import net.corda.applications.workers.workercommon.Health import net.corda.applications.workers.workercommon.JavaSerialisationFilter import net.corda.applications.workers.workercommon.Metrics +import net.corda.applications.workers.workercommon.WorkerHelpers import net.corda.applications.workers.workercommon.WorkerHelpers.Companion.getParams import net.corda.applications.workers.workercommon.WorkerHelpers.Companion.loggerStartupInfo import net.corda.applications.workers.workercommon.WorkerHelpers.Companion.printHelpOrVersion @@ -14,6 +15,7 @@ import net.corda.libs.platform.PlatformInfoProvider import net.corda.lifecycle.registry.LifecycleRegistry import net.corda.osgi.api.Application import net.corda.osgi.api.Shutdown +import net.corda.processors.token.cache.TokenCacheProcessor import net.corda.schema.configuration.BootConfig import net.corda.tracing.configureTracing import net.corda.tracing.shutdownTracing @@ -42,6 +44,8 @@ class TokenSelectionWorker @Activate constructor( private val configurationValidatorFactory: ConfigurationValidatorFactory, @Reference(service = SecretsServiceFactoryResolver::class) val secretsServiceFactoryResolver: SecretsServiceFactoryResolver, + @Reference(service = TokenCacheProcessor::class) + private val tokenCacheProcessor: TokenCacheProcessor, ) : Application { private companion object { @@ -64,14 +68,21 @@ class TokenSelectionWorker @Activate constructor( configureTracing("Token selection Worker", params.defaultParams.zipkinTraceUrl, params.defaultParams.traceSamplesPerSecond) - webServer.start(params.defaultParams.workerServerPort) + val config = WorkerHelpers.getBootstrapConfig( + secretsServiceFactoryResolver, + params.defaultParams, + configurationValidatorFactory.createConfigValidator(), + listOf(WorkerHelpers.createConfigFromParams(BootConfig.BOOT_DB, params.databaseParams)) + ) - // This is a placeholder worker the processor is still todo + webServer.start(params.defaultParams.workerServerPort) + tokenCacheProcessor.start(config) } override fun shutdown() { logger.info("Token selection worker stopping.") webServer.stop() + tokenCacheProcessor.stop() shutdownTracing() } } diff --git a/components/ledger/ledger-utxo-token-cache/src/main/kotlin/net/corda/ledger/utxo/token/cache/services/TokenCacheEventProcessor.kt b/components/ledger/ledger-utxo-token-cache/src/main/kotlin/net/corda/ledger/utxo/token/cache/services/TokenCacheEventProcessor.kt index 335827b09df..a326a77ae56 100644 --- a/components/ledger/ledger-utxo-token-cache/src/main/kotlin/net/corda/ledger/utxo/token/cache/services/TokenCacheEventProcessor.kt +++ b/components/ledger/ledger-utxo-token-cache/src/main/kotlin/net/corda/ledger/utxo/token/cache/services/TokenCacheEventProcessor.kt @@ -15,6 +15,7 @@ import net.corda.messaging.api.processor.StateAndEventProcessor import net.corda.messaging.api.processor.StateAndEventProcessor.State import net.corda.messaging.api.records.Record import net.corda.tracing.traceStateAndEventExecution +import net.corda.utilities.debug import org.slf4j.LoggerFactory @Suppress("LongParameterList") @@ -52,6 +53,8 @@ class TokenCacheEventProcessor( markForDLQ = true) } + log.debug { "Token event received: $tokenEvent" } + return traceStateAndEventExecution(event, "Token Event - ${tokenEvent.javaClass.simpleName}") { try { tokenSelectionMetrics.recordProcessingTime(tokenEvent) { @@ -85,6 +88,8 @@ class TokenCacheEventProcessor( val result = handler.handle(tokenCache, poolCacheState, tokenEvent) + log.debug { "sending token response: $result" } + if (result == null) { StateAndEventProcessor.Response( State(poolCacheState.toAvro(), metadata = state?.metadata), diff --git a/processors/db-processor/build.gradle b/processors/db-processor/build.gradle index 5f87ac291a8..c4fb4634917 100644 --- a/processors/db-processor/build.gradle +++ b/processors/db-processor/build.gradle @@ -68,7 +68,6 @@ dependencies { runtimeOnly project(':components:configuration:configuration-write-service-impl') runtimeOnly project(':components:configuration:configuration-read-service-impl') runtimeOnly project(':components:db:db-connection-manager-impl') - runtimeOnly project(':libs:flows:external-event-responses-impl') runtimeOnly project(':components:reconciliation:reconciliation-impl') runtimeOnly project(':components:uniqueness:uniqueness-checker-impl') runtimeOnly project(':components:virtual-node:cpi-info-read-service-impl') @@ -88,6 +87,7 @@ dependencies { runtimeOnly project(":libs:application:application-impl") runtimeOnly project(':libs:crypto:cipher-suite-impl') runtimeOnly project(':libs:db:db-orm-impl') + runtimeOnly project(':libs:flows:external-event-responses-impl') runtimeOnly project(':libs:lifecycle:lifecycle-impl') runtimeOnly project(':libs:membership:membership-impl') runtimeOnly project(':libs:messaging:messaging-impl') diff --git a/processors/token-cache-processor/build.gradle b/processors/token-cache-processor/build.gradle index 34493d92155..d5df1946809 100644 --- a/processors/token-cache-processor/build.gradle +++ b/processors/token-cache-processor/build.gradle @@ -14,11 +14,28 @@ dependencies { implementation 'org.jetbrains.kotlin:kotlin-osgi-bundle' implementation 'org.slf4j:slf4j-api' + implementation 'net.corda:corda-config-schema' + implementation 'net.corda:corda-db-schema' + implementation project(':components:ledger:ledger-utxo-token-cache') implementation project(":components:configuration:configuration-read-service") + implementation project(':components:db:db-connection-manager') + implementation project(':components:virtual-node:virtual-node-info-read-service') + implementation project(':libs:configuration:configuration-datamodel') implementation project(":libs:lifecycle:lifecycle") implementation project(':libs:utilities') runtimeOnly project(":components:configuration:configuration-read-service-impl") runtimeOnly project(":libs:web:web-impl") + + runtimeOnly project(':components:configuration:configuration-read-service-impl') + runtimeOnly project(':components:db:db-connection-manager-impl') + runtimeOnly project(":libs:application:application-impl") + runtimeOnly project(':libs:crypto:cipher-suite-impl') + runtimeOnly project(':libs:db:db-orm-impl') + runtimeOnly project(':libs:flows:external-event-responses-impl') + runtimeOnly project(':libs:lifecycle:lifecycle-impl') + runtimeOnly project(':libs:messaging:messaging-impl') + runtimeOnly project(':libs:schema-registry:schema-registry-impl') + runtimeOnly project(":libs:web:web-impl") } diff --git a/processors/token-cache-processor/src/main/kotlin/net/corda/processors/token/cache/internal/TokenCacheProcessorImpl.kt b/processors/token-cache-processor/src/main/kotlin/net/corda/processors/token/cache/internal/TokenCacheProcessorImpl.kt index f4e976d5221..9cf2e6b0981 100644 --- a/processors/token-cache-processor/src/main/kotlin/net/corda/processors/token/cache/internal/TokenCacheProcessorImpl.kt +++ b/processors/token-cache-processor/src/main/kotlin/net/corda/processors/token/cache/internal/TokenCacheProcessorImpl.kt @@ -1,9 +1,12 @@ package net.corda.processors.token.cache.internal import net.corda.configuration.read.ConfigurationReadService +import net.corda.db.connection.manager.DbConnectionManager +import net.corda.db.schema.CordaDb import net.corda.ledger.utxo.token.cache.factories.TokenCacheComponentFactory import net.corda.ledger.utxo.token.cache.services.TokenCacheComponent import net.corda.libs.configuration.SmartConfig +import net.corda.libs.configuration.datamodel.ConfigurationEntities import net.corda.lifecycle.DependentComponents import net.corda.lifecycle.LifecycleCoordinator import net.corda.lifecycle.LifecycleCoordinatorFactory @@ -12,8 +15,11 @@ import net.corda.lifecycle.RegistrationStatusChangeEvent import net.corda.lifecycle.StartEvent import net.corda.lifecycle.StopEvent import net.corda.lifecycle.createCoordinator +import net.corda.orm.JpaEntitiesRegistry import net.corda.processors.token.cache.TokenCacheProcessor +import net.corda.schema.configuration.BootConfig.BOOT_DB import net.corda.utilities.debug +import net.corda.virtualnode.read.VirtualNodeInfoReadService import org.osgi.service.component.annotations.Activate import org.osgi.service.component.annotations.Component import org.osgi.service.component.annotations.Reference @@ -28,8 +34,27 @@ class TokenCacheProcessorImpl @Activate constructor( @Reference(service = ConfigurationReadService::class) private val configurationReadService: ConfigurationReadService, @Reference(service = TokenCacheComponentFactory::class) - private val tokenCacheComponentFactory: TokenCacheComponentFactory + private val tokenCacheComponentFactory: TokenCacheComponentFactory, + @Reference(service = DbConnectionManager::class) + private val dbConnectionManager: DbConnectionManager, + @Reference(service = VirtualNodeInfoReadService::class) + private val virtualNodeInfoReadService: VirtualNodeInfoReadService, + @Reference(service = JpaEntitiesRegistry::class) + private val entitiesRegistry: JpaEntitiesRegistry, ) : TokenCacheProcessor { + init { + // define the different DB Entity Sets + // entities can be in different packages, but all JPA classes must be passed in. + entitiesRegistry.register( + CordaDb.CordaCluster.persistenceUnitName, + ConfigurationEntities.classes + ) + entitiesRegistry.register( + CordaDb.Vault.persistenceUnitName, + // Token selection uses native queries, so no JPA entities to register. + emptySet() + ) + } private companion object { val log: Logger = LoggerFactory.getLogger(this::class.java.enclosingClass) @@ -37,6 +62,8 @@ class TokenCacheProcessorImpl @Activate constructor( private val dependentComponents = DependentComponents.of( ::configurationReadService, + ::dbConnectionManager, + ::virtualNodeInfoReadService, ).with(tokenCacheComponentFactory.create(), TokenCacheComponent::class.java) private val lifecycleCoordinator = @@ -65,6 +92,10 @@ class TokenCacheProcessorImpl @Activate constructor( coordinator.updateStatus(event.status) } is BootConfigEvent -> { + val bootstrapConfig = event.config + log.info("Bootstrapping DB connection Manager") + dbConnectionManager.bootstrap(bootstrapConfig.getConfig(BOOT_DB)) + log.info("Bootstrapping config read service") configurationReadService.bootstrapConfig(event.config) } is StopEvent -> { From 5f3f3475270fae9beaf2ffb776a19455e0db6cd8 Mon Sep 17 00:00:00 2001 From: Viktor Kolomeyko Date: Thu, 12 Oct 2023 16:45:14 +0100 Subject: [PATCH 03/81] Revert "CORE-17429 State metadata support in message processor (#4828)" This reverts commit e2aab1c35bc05075321facd29631290f5144a157. --- .../integration/TestFlowMessageProcessor.kt | 5 +- .../executor/FlowMapperMessageProcessor.kt | 14 ++--- .../FlowMapperMessageProcessorTest.kt | 24 +++------ .../testing/context/FlowServiceTestContext.kt | 8 +-- .../testing/context/OutputAssertionsImpl.kt | 12 ++--- .../impl/FlowEventContextConverterImpl.kt | 3 +- .../factory/FlowEventPipelineFactory.kt | 5 +- .../impl/FlowEventPipelineFactoryImpl.kt | 8 ++- .../pipeline/impl/FlowEventProcessorImpl.kt | 12 ++--- .../corda/flow/RequestHandlerTestContext.kt | 3 +- .../FlowEventContextConverterImplTest.kt | 2 +- .../FlowEventPipelineFactoryImplTest.kt | 9 +--- .../impl/FlowEventProcessorImplTest.kt | 22 ++++---- .../flow/test/utils/FlowEventContextHelper.kt | 6 +-- .../services/TokenCacheEventProcessor.kt | 19 +++---- .../services/TokenCacheEventProcessorTest.kt | 36 ++++--------- .../linkmanager/delivery/DeliveryTracker.kt | 30 ++++------- .../delivery/DeliveryTrackerTest.kt | 8 +-- .../MembershipPersistenceAsyncProcessor.kt | 24 ++++----- ...MembershipPersistenceAsyncProcessorTest.kt | 53 +++++++------------ .../service/impl/CommandsRetryManager.kt | 5 +- .../dynamic/RegistrationProcessor.kt | 30 +++++------ .../dynamic/RegistrationProcessorTest.kt | 32 ++++------- .../flow/pipeline/events/FlowEventContext.kt | 5 +- .../corda/messaging/mediator/ProcessorTask.kt | 13 ++--- .../messaging/mediator/StateManagerHelper.kt | 9 ++-- .../StateAndEventSubscriptionImpl.kt | 10 +--- .../MultiSourceEventMediatorImplTest.kt | 2 +- .../messaging/mediator/ProcessorTaskTest.kt | 21 +++----- .../mediator/StateManagerHelperTest.kt | 23 +++----- .../factory/MediatorComponentFactoryTest.kt | 7 +-- .../StateAndEventSubscriptionImplTest.kt | 5 +- libs/messaging/messaging/build.gradle | 2 - .../api/processor/StateAndEventProcessor.kt | 18 ++----- .../processors/TestStateEventProcessor.kt | 10 +--- .../TestStateEventProcessorStrings.kt | 10 +--- ...ateAndEventSubscriptionIntegrationTests.kt | 47 ++++------------ .../stateandevent/EventSubscription.kt | 12 ++--- .../stateandevent/EventSubscriptionTest.kt | 12 ++--- .../stateandevent/StateSubscriptionTest.kt | 5 +- 40 files changed, 187 insertions(+), 394 deletions(-) diff --git a/components/flow/flow-mapper-service/src/integrationTest/kotlin/net/corda/session/mapper/service/integration/TestFlowMessageProcessor.kt b/components/flow/flow-mapper-service/src/integrationTest/kotlin/net/corda/session/mapper/service/integration/TestFlowMessageProcessor.kt index 7c45b556ccf..a8c9d6bd04d 100644 --- a/components/flow/flow-mapper-service/src/integrationTest/kotlin/net/corda/session/mapper/service/integration/TestFlowMessageProcessor.kt +++ b/components/flow/flow-mapper-service/src/integrationTest/kotlin/net/corda/session/mapper/service/integration/TestFlowMessageProcessor.kt @@ -3,7 +3,6 @@ package net.corda.session.mapper.service.integration import net.corda.data.flow.event.FlowEvent import net.corda.data.flow.state.checkpoint.Checkpoint import net.corda.messaging.api.processor.StateAndEventProcessor -import net.corda.messaging.api.processor.StateAndEventProcessor.State import net.corda.messaging.api.records.Record import org.junit.jupiter.api.fail import java.util.concurrent.CountDownLatch @@ -19,8 +18,8 @@ class TestFlowMessageProcessor( var eventsReceived: MutableList> = mutableListOf() override fun onNext( - state: State?, - event: Record, + state: Checkpoint?, + event: Record ): StateAndEventProcessor.Response { eventsReceived.add(event) diff --git a/components/flow/flow-mapper-service/src/main/kotlin/net/corda/session/mapper/service/executor/FlowMapperMessageProcessor.kt b/components/flow/flow-mapper-service/src/main/kotlin/net/corda/session/mapper/service/executor/FlowMapperMessageProcessor.kt index 87db6bec60f..f0ffc75ab88 100644 --- a/components/flow/flow-mapper-service/src/main/kotlin/net/corda/session/mapper/service/executor/FlowMapperMessageProcessor.kt +++ b/components/flow/flow-mapper-service/src/main/kotlin/net/corda/session/mapper/service/executor/FlowMapperMessageProcessor.kt @@ -7,7 +7,6 @@ import net.corda.data.flow.state.mapper.FlowMapperState import net.corda.flow.mapper.factory.FlowMapperEventExecutorFactory import net.corda.libs.configuration.SmartConfig import net.corda.messaging.api.processor.StateAndEventProcessor -import net.corda.messaging.api.processor.StateAndEventProcessor.State import net.corda.messaging.api.records.Record import net.corda.metrics.CordaMetrics import net.corda.schema.configuration.FlowConfig @@ -39,14 +38,14 @@ class FlowMapperMessageProcessor( private val clock = UTCClock() override fun onNext( - state: State?, - event: Record, + state: FlowMapperState?, + event: Record ): StateAndEventProcessor.Response { val key = event.key logger.trace { "Received event. Key: $key Event: ${event.value}" } val value = event.value ?: return StateAndEventProcessor.Response(state, emptyList()) - val eventType = value.payload?.javaClass?.simpleName ?: "Unknown" + val eventType = value.payload?.let { it.javaClass.simpleName } ?: "Unknown" CordaMetrics.Metric.FlowMapperEventLag.builder() @@ -58,12 +57,9 @@ class FlowMapperMessageProcessor( return traceStateAndEventExecution(event, "Flow Mapper Event - $eventType") { eventProcessingTimer.recordCallable { if (!isExpiredSessionEvent(value)) { - val executor = flowMapperEventExecutorFactory.create(key, value, state?.value, flowConfig) + val executor = flowMapperEventExecutorFactory.create(key, value, state, flowConfig) val result = executor.execute() - StateAndEventProcessor.Response( - State(result.flowMapperState, state?.metadata), - result.outputEvents - ) + StateAndEventProcessor.Response(result.flowMapperState, result.outputEvents) } else { logger.debug { "This event is expired and will be ignored. Event: $event State: $state" } CordaMetrics.Metric.FlowMapperExpiredSessionEventCount.builder() diff --git a/components/flow/flow-mapper-service/src/test/kotlin/net/corda/session/mapper/service/executor/FlowMapperMessageProcessorTest.kt b/components/flow/flow-mapper-service/src/test/kotlin/net/corda/session/mapper/service/executor/FlowMapperMessageProcessorTest.kt index f72a72056b4..babb72cf8e4 100644 --- a/components/flow/flow-mapper-service/src/test/kotlin/net/corda/session/mapper/service/executor/FlowMapperMessageProcessorTest.kt +++ b/components/flow/flow-mapper-service/src/test/kotlin/net/corda/session/mapper/service/executor/FlowMapperMessageProcessorTest.kt @@ -13,12 +13,9 @@ import net.corda.flow.mapper.FlowMapperResult import net.corda.flow.mapper.executor.FlowMapperEventExecutor import net.corda.flow.mapper.factory.FlowMapperEventExecutorFactory import net.corda.libs.configuration.SmartConfigImpl -import net.corda.libs.statemanager.api.Metadata -import net.corda.messaging.api.processor.StateAndEventProcessor.State import net.corda.messaging.api.records.Record import net.corda.schema.configuration.FlowConfig import net.corda.test.flow.util.buildSessionEvent -import org.assertj.core.api.Assertions.assertThat import org.junit.jupiter.api.Test import org.mockito.kotlin.any import org.mockito.kotlin.anyOrNull @@ -38,15 +35,12 @@ class FlowMapperMessageProcessorTest { private val config = SmartConfigImpl.empty().withValue(FlowConfig.SESSION_P2P_TTL, ConfigValueFactory.fromAnyRef(10000)) private val flowMapperMessageProcessor = FlowMapperMessageProcessor(flowMapperEventExecutorFactory, config) - private fun buildMapperState(status: FlowMapperStateType, metadata: Metadata = Metadata()) : State { - return State( - FlowMapperState.newBuilder() - .setStatus(status) - .setFlowId("flowId") - .setExpiryTime(Instant.now().toEpochMilli()) - .build(), - metadata = metadata, - ) + private fun buildMapperState(status: FlowMapperStateType) : FlowMapperState { + return FlowMapperState.newBuilder() + .setStatus(status) + .setFlowId("flowId") + .setExpiryTime(Instant.now().toEpochMilli()) + .build() } private fun buildMapperEvent(payload: Any) : Record { @@ -75,12 +69,8 @@ class FlowMapperMessageProcessorTest { @Test fun `when state is OPEN new session events are processed`() { - val metadata = Metadata(mapOf("foo" to "bar")) - val output = flowMapperMessageProcessor.onNext( - buildMapperState(FlowMapperStateType.OPEN, metadata),buildMapperEvent(buildSessionEvent()) - ) + flowMapperMessageProcessor.onNext(buildMapperState(FlowMapperStateType.OPEN), buildMapperEvent(buildSessionEvent())) verify(flowMapperEventExecutorFactory, times(1)).create(any(), any(), anyOrNull(), any(), any()) - assertThat(output.updatedState?.metadata).isEqualTo(metadata) } @Test diff --git a/components/flow/flow-service/src/integrationTest/kotlin/net/corda/flow/testing/context/FlowServiceTestContext.kt b/components/flow/flow-service/src/integrationTest/kotlin/net/corda/flow/testing/context/FlowServiceTestContext.kt index c3901d96a73..2a2aadc5a8e 100644 --- a/components/flow/flow-service/src/integrationTest/kotlin/net/corda/flow/testing/context/FlowServiceTestContext.kt +++ b/components/flow/flow-service/src/integrationTest/kotlin/net/corda/flow/testing/context/FlowServiceTestContext.kt @@ -53,7 +53,6 @@ import net.corda.libs.packaging.core.CpkManifest import net.corda.libs.packaging.core.CpkMetadata import net.corda.libs.packaging.core.CpkType import net.corda.messaging.api.processor.StateAndEventProcessor -import net.corda.messaging.api.processor.StateAndEventProcessor.State import net.corda.messaging.api.records.Record import net.corda.schema.Schemas.Flow.FLOW_EVENT_TOPIC import net.corda.schema.configuration.ConfigKeys.FLOW_CONFIG @@ -439,13 +438,10 @@ class FlowServiceTestContext @Activate constructor( log.info("Start test run for input/output set $iteration") flowFiberFactory.fiber.reset() flowFiberFactory.fiber.setIoRequests(testRun.ioRequests) - val response = flowEventProcessor.onNext( - State(lastPublishedState, metadata = null), - testRun.event - ) + val response = flowEventProcessor.onNext(lastPublishedState, testRun.event) testRun.flowContinuation = flowFiberFactory.fiber.flowContinuation testRun.response = response - lastPublishedState = response.updatedState?.value + lastPublishedState = response.updatedState } } diff --git a/components/flow/flow-service/src/integrationTest/kotlin/net/corda/flow/testing/context/OutputAssertionsImpl.kt b/components/flow/flow-service/src/integrationTest/kotlin/net/corda/flow/testing/context/OutputAssertionsImpl.kt index ea711e469b7..d6b00b966a0 100644 --- a/components/flow/flow-service/src/integrationTest/kotlin/net/corda/flow/testing/context/OutputAssertionsImpl.kt +++ b/components/flow/flow-service/src/integrationTest/kotlin/net/corda/flow/testing/context/OutputAssertionsImpl.kt @@ -259,13 +259,13 @@ class OutputAssertionsImpl( override fun hasPendingUserException() { asserts.add { testRun -> - assertThat(testRun.response?.updatedState?.value?.pipelineState?.pendingPlatformError).isNotNull() + assertThat(testRun.response?.updatedState?.pipelineState?.pendingPlatformError).isNotNull() } } override fun noPendingUserException() { asserts.add { testRun -> - assertThat(testRun.response?.updatedState?.value?.pipelineState?.pendingPlatformError).isNull() + assertThat(testRun.response?.updatedState?.pipelineState?.pendingPlatformError).isNull() } } @@ -278,8 +278,8 @@ class OutputAssertionsImpl( override fun checkpointHasRetry(expectedCount: Int) { asserts.add { testRun -> - assertThat(testRun.response?.updatedState?.value?.pipelineState?.retryState).isNotNull - val retry = testRun.response!!.updatedState!!.value?.pipelineState!!.retryState + assertThat(testRun.response?.updatedState?.pipelineState?.retryState).isNotNull + val retry = testRun.response!!.updatedState!!.pipelineState!!.retryState assertThat(retry.retryCount).isEqualTo(expectedCount) @@ -296,7 +296,7 @@ class OutputAssertionsImpl( override fun checkpointDoesNotHaveRetry() { asserts.add { testRun -> - assertThat(testRun.response?.updatedState?.value?.pipelineState?.retryState).isNull() + assertThat(testRun.response?.updatedState?.pipelineState?.retryState).isNull() } } @@ -364,7 +364,7 @@ class OutputAssertionsImpl( override fun nullStateRecord() { asserts.add { - assertNull(it.response?.updatedState?.value, "Expected to receive NULL for output state") + assertNull(it.response?.updatedState, "Expected to receive NULL for output state") } } diff --git a/components/flow/flow-service/src/main/kotlin/net/corda/flow/pipeline/converters/impl/FlowEventContextConverterImpl.kt b/components/flow/flow-service/src/main/kotlin/net/corda/flow/pipeline/converters/impl/FlowEventContextConverterImpl.kt index c1328de65e6..08d4cdb6ec2 100644 --- a/components/flow/flow-service/src/main/kotlin/net/corda/flow/pipeline/converters/impl/FlowEventContextConverterImpl.kt +++ b/components/flow/flow-service/src/main/kotlin/net/corda/flow/pipeline/converters/impl/FlowEventContextConverterImpl.kt @@ -4,7 +4,6 @@ import net.corda.data.flow.state.checkpoint.Checkpoint import net.corda.flow.pipeline.events.FlowEventContext import net.corda.flow.pipeline.converters.FlowEventContextConverter import net.corda.messaging.api.processor.StateAndEventProcessor -import net.corda.messaging.api.processor.StateAndEventProcessor.State import org.osgi.service.component.annotations.Component @Suppress("Unused") @@ -12,7 +11,7 @@ import org.osgi.service.component.annotations.Component class FlowEventContextConverterImpl : FlowEventContextConverter { override fun convert(flowContext: FlowEventContext<*>): StateAndEventProcessor.Response { return StateAndEventProcessor.Response( - State(flowContext.checkpoint.toAvro(), metadata = flowContext.metadata), + flowContext.checkpoint.toAvro(), flowContext.outputRecords, flowContext.sendToDlq ) diff --git a/components/flow/flow-service/src/main/kotlin/net/corda/flow/pipeline/factory/FlowEventPipelineFactory.kt b/components/flow/flow-service/src/main/kotlin/net/corda/flow/pipeline/factory/FlowEventPipelineFactory.kt index e727a966d1c..afd24b79fad 100644 --- a/components/flow/flow-service/src/main/kotlin/net/corda/flow/pipeline/factory/FlowEventPipelineFactory.kt +++ b/components/flow/flow-service/src/main/kotlin/net/corda/flow/pipeline/factory/FlowEventPipelineFactory.kt @@ -4,7 +4,6 @@ import net.corda.data.flow.event.FlowEvent import net.corda.data.flow.state.checkpoint.Checkpoint import net.corda.flow.pipeline.FlowEventPipeline import net.corda.libs.configuration.SmartConfig -import net.corda.messaging.api.processor.StateAndEventProcessor.State import net.corda.tracing.TraceContext /** @@ -16,7 +15,7 @@ interface FlowEventPipelineFactory { /** * Creates a [FlowEventPipeline] instance. * - * @param state The [Checkpoint] and metadata passed through the pipeline. + * @param checkpoint The [Checkpoint] passed through the pipeline. * @param event The [FlowEvent] passed through the pipeline. * @param config The [SmartConfig] containing the settings used in the pipeline factory. * @param mdcProperties properties to set the flow fibers MDC with. @@ -26,7 +25,7 @@ interface FlowEventPipelineFactory { * @return A new [FlowEventPipeline] instance. */ fun create( - state: State?, + checkpoint: Checkpoint?, event: FlowEvent, configs: Map, mdcProperties: Map, diff --git a/components/flow/flow-service/src/main/kotlin/net/corda/flow/pipeline/factory/impl/FlowEventPipelineFactoryImpl.kt b/components/flow/flow-service/src/main/kotlin/net/corda/flow/pipeline/factory/impl/FlowEventPipelineFactoryImpl.kt index b37882a6f1b..be2c575615b 100644 --- a/components/flow/flow-service/src/main/kotlin/net/corda/flow/pipeline/factory/impl/FlowEventPipelineFactoryImpl.kt +++ b/components/flow/flow-service/src/main/kotlin/net/corda/flow/pipeline/factory/impl/FlowEventPipelineFactoryImpl.kt @@ -19,7 +19,6 @@ import net.corda.flow.pipeline.runner.FlowRunner import net.corda.flow.state.impl.FlowCheckpointFactory import net.corda.libs.configuration.SmartConfig import net.corda.libs.configuration.helper.getConfig -import net.corda.messaging.api.processor.StateAndEventProcessor.State import net.corda.schema.configuration.ConfigKeys.FLOW_CONFIG import net.corda.tracing.TraceContext import net.corda.virtualnode.read.VirtualNodeInfoReadService @@ -98,7 +97,7 @@ class FlowEventPipelineFactoryImpl( ) override fun create( - state: State?, + checkpoint: Checkpoint?, event: FlowEvent, configs: Map, mdcProperties: Map, @@ -107,7 +106,7 @@ class FlowEventPipelineFactoryImpl( ): FlowEventPipeline { val flowCheckpoint = flowCheckpointFactory.create( event.flowId, - state?.value, + checkpoint, configs.getConfig(FLOW_CONFIG) ) @@ -122,8 +121,7 @@ class FlowEventPipelineFactoryImpl( outputRecords = emptyList(), mdcProperties = mdcProperties, flowMetrics = metrics, - flowTraceContext = traceContext, - metadata = state?.metadata + flowTraceContext = traceContext ) val flowExecutionPipelineStage = FlowExecutionPipelineStage( diff --git a/components/flow/flow-service/src/main/kotlin/net/corda/flow/pipeline/impl/FlowEventProcessorImpl.kt b/components/flow/flow-service/src/main/kotlin/net/corda/flow/pipeline/impl/FlowEventProcessorImpl.kt index f840c33916f..b048caa9976 100644 --- a/components/flow/flow-service/src/main/kotlin/net/corda/flow/pipeline/impl/FlowEventProcessorImpl.kt +++ b/components/flow/flow-service/src/main/kotlin/net/corda/flow/pipeline/impl/FlowEventProcessorImpl.kt @@ -14,7 +14,6 @@ import net.corda.flow.pipeline.factory.FlowEventPipelineFactory import net.corda.flow.pipeline.handlers.FlowPostProcessingHandler import net.corda.libs.configuration.SmartConfig import net.corda.messaging.api.processor.StateAndEventProcessor -import net.corda.messaging.api.processor.StateAndEventProcessor.State import net.corda.messaging.api.records.Record import net.corda.schema.configuration.ConfigKeys.FLOW_CONFIG import net.corda.schema.configuration.MessagingConfig.Subscription.PROCESSOR_TIMEOUT @@ -52,11 +51,11 @@ class FlowEventProcessorImpl( } override fun onNext( - state: State?, + state: Checkpoint?, event: Record, ): StateAndEventProcessor.Response { val flowEvent = event.value - val mdcProperties = flowMDCService.getMDCLogging(state?.value, flowEvent, event.key) + val mdcProperties = flowMDCService.getMDCLogging(state, flowEvent, event.key) val eventType = event.value?.payload?.javaClass?.simpleName ?: "Unknown" return withMDC(mdcProperties) { traceStateAndEventExecution(event, "Flow Event - $eventType") { @@ -68,16 +67,13 @@ class FlowEventProcessorImpl( private fun getFlowPipelineResponse( flowEvent: FlowEvent?, event: Record, - state: State?, + state: Checkpoint?, mdcProperties: Map, traceContext: TraceContext ): StateAndEventProcessor.Response { if (flowEvent == null) { log.debug { "The incoming event record '${event}' contained a null FlowEvent, this event will be discarded" } - return StateAndEventProcessor.Response( - state, - listOf() - ) + return StateAndEventProcessor.Response(state, listOf()) } diff --git a/components/flow/flow-service/src/test/kotlin/net/corda/flow/RequestHandlerTestContext.kt b/components/flow/flow-service/src/test/kotlin/net/corda/flow/RequestHandlerTestContext.kt index 9a3b779caaa..bf4e67adca6 100644 --- a/components/flow/flow-service/src/test/kotlin/net/corda/flow/RequestHandlerTestContext.kt +++ b/components/flow/flow-service/src/test/kotlin/net/corda/flow/RequestHandlerTestContext.kt @@ -70,7 +70,6 @@ class RequestHandlerTestContext(val payload: PAYLOAD) { recordList, mdcProperties = emptyMap(), flowMetrics = mock(), - flowTraceContext = mock(), - metadata = null + flowTraceContext = mock() ) } diff --git a/components/flow/flow-service/src/test/kotlin/net/corda/flow/pipeline/converters/FlowEventContextConverterImplTest.kt b/components/flow/flow-service/src/test/kotlin/net/corda/flow/pipeline/converters/FlowEventContextConverterImplTest.kt index 83c54c433e8..be7cbf773ca 100644 --- a/components/flow/flow-service/src/test/kotlin/net/corda/flow/pipeline/converters/FlowEventContextConverterImplTest.kt +++ b/components/flow/flow-service/src/test/kotlin/net/corda/flow/pipeline/converters/FlowEventContextConverterImplTest.kt @@ -29,7 +29,7 @@ class FlowEventContextConverterImplTest { val result = converter.convert(context) assertThat(result.markForDLQ).isTrue - assertThat(result.updatedState?.value).isSameAs(avroCheckpoint) + assertThat(result.updatedState).isSameAs(avroCheckpoint) assertThat(result.responseEvents).isSameAs(records) } } \ No newline at end of file diff --git a/components/flow/flow-service/src/test/kotlin/net/corda/flow/pipeline/factory/FlowEventPipelineFactoryImplTest.kt b/components/flow/flow-service/src/test/kotlin/net/corda/flow/pipeline/factory/FlowEventPipelineFactoryImplTest.kt index e91822b592e..b8f908b2f52 100644 --- a/components/flow/flow-service/src/test/kotlin/net/corda/flow/pipeline/factory/FlowEventPipelineFactoryImplTest.kt +++ b/components/flow/flow-service/src/test/kotlin/net/corda/flow/pipeline/factory/FlowEventPipelineFactoryImplTest.kt @@ -19,7 +19,6 @@ import net.corda.flow.pipeline.runner.FlowRunner import net.corda.flow.state.FlowCheckpoint import net.corda.flow.state.impl.FlowCheckpointFactory import net.corda.flow.test.utils.buildFlowEventContext -import net.corda.messaging.api.processor.StateAndEventProcessor.State import net.corda.schema.configuration.ConfigKeys.FLOW_CONFIG import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Test @@ -90,13 +89,7 @@ class FlowEventPipelineFactoryImplTest { flowEventContext, mock(), ) - val result = factory.create( - State(checkpoint, null), - flowEvent, - mapOf(FLOW_CONFIG to config), - emptyMap(), - flowEventContext.flowTraceContext, - 0) + val result = factory.create(checkpoint, flowEvent, mapOf(FLOW_CONFIG to config), emptyMap(), flowEventContext.flowTraceContext, 0) assertEquals(expected.context, result.context) } } \ No newline at end of file diff --git a/components/flow/flow-service/src/test/kotlin/net/corda/flow/pipeline/impl/FlowEventProcessorImplTest.kt b/components/flow/flow-service/src/test/kotlin/net/corda/flow/pipeline/impl/FlowEventProcessorImplTest.kt index b04e605803b..b09507754e4 100644 --- a/components/flow/flow-service/src/test/kotlin/net/corda/flow/pipeline/impl/FlowEventProcessorImplTest.kt +++ b/components/flow/flow-service/src/test/kotlin/net/corda/flow/pipeline/impl/FlowEventProcessorImplTest.kt @@ -31,7 +31,6 @@ import net.corda.flow.pipeline.handlers.FlowPostProcessingHandler import net.corda.flow.state.FlowCheckpoint import net.corda.flow.test.utils.buildFlowEventContext import net.corda.messaging.api.processor.StateAndEventProcessor -import net.corda.messaging.api.processor.StateAndEventProcessor.State import net.corda.messaging.api.records.Record import net.corda.schema.Schemas.Flow.FLOW_EVENT_TOPIC import net.corda.schema.configuration.ConfigKeys.FLOW_CONFIG @@ -85,7 +84,6 @@ class FlowEventProcessorImplTest { private val flowKey = "flow id" private val flowCheckpoint = mock() private val checkpoint: Checkpoint = mock() - private val state = State(checkpoint, metadata = null) private val flowState: FlowState = mock() private val flowStartContext: FlowStartContext = mock() private val externalEventState: ExternalEventState = mock() @@ -163,9 +161,9 @@ class FlowEventProcessorImplTest { fun `Returns the state unaltered if no flow event supplied`() { val inputEvent = getFlowEventRecord(null) - val response = processor.onNext(state, inputEvent) + val response = processor.onNext(checkpoint, inputEvent) - assertThat(response.updatedState?.value).isSameAs(checkpoint) + assertThat(response.updatedState).isSameAs(checkpoint) assertThat(response.responseEvents).isEmpty() verify(flowMDCService, times(0)).getMDCLogging(anyOrNull(), any(), any()) } @@ -174,7 +172,7 @@ class FlowEventProcessorImplTest { fun `Returns a checkpoint and events to send`() { val inputEvent = getFlowEventRecord(FlowEvent(flowKey, payload)) - val response = processor.onNext(state, inputEvent) + val response = processor.onNext(checkpoint, inputEvent) val expectedRecords = updatedContext.outputRecords verify(flowEventContextConverter).convert(argThat { outputRecords == expectedRecords }) @@ -185,7 +183,7 @@ class FlowEventProcessorImplTest { @Test fun `Calls the pipeline steps in order`() { - processor.onNext(state, getFlowEventRecord(FlowEvent(flowKey, payload))) + processor.onNext(checkpoint, getFlowEventRecord(FlowEvent(flowKey, payload))) inOrder(flowEventPipeline) { verify(flowEventPipeline).eventPreProcessing() verify(flowEventPipeline).virtualNodeFlowOperationalChecks() @@ -201,7 +199,7 @@ class FlowEventProcessorImplTest { whenever(flowEventPipeline.eventPreProcessing()).thenThrow(error) whenever(flowEventExceptionProcessor.process(error, flowEventPipeline.context)).thenReturn(errorContext) - val response = processor.onNext(state, getFlowEventRecord(FlowEvent(flowKey, payload))) + val response = processor.onNext(checkpoint, getFlowEventRecord(FlowEvent(flowKey, payload))) assertThat(response).isEqualTo(errorResponse) } @@ -213,7 +211,7 @@ class FlowEventProcessorImplTest { whenever(flowEventPipeline.eventPreProcessing()).thenThrow(error) whenever(flowEventExceptionProcessor.process(error, flowEventPipeline.context)).thenReturn(errorContext) - val response = processor.onNext(state, getFlowEventRecord(FlowEvent(flowKey, payload))) + val response = processor.onNext(checkpoint, getFlowEventRecord(FlowEvent(flowKey, payload))) assertThat(response).isEqualTo(errorResponse) } @@ -225,7 +223,7 @@ class FlowEventProcessorImplTest { whenever(flowEventPipeline.eventPreProcessing()).thenThrow(error) whenever(flowEventExceptionProcessor.process(error, flowEventPipeline.context)).thenReturn(errorContext) - val response = processor.onNext(state, getFlowEventRecord(FlowEvent(flowKey, payload))) + val response = processor.onNext(checkpoint, getFlowEventRecord(FlowEvent(flowKey, payload))) assertThat(response).isEqualTo(errorResponse) } @@ -237,7 +235,7 @@ class FlowEventProcessorImplTest { whenever(flowEventPipeline.eventPreProcessing()).thenThrow(error) whenever(flowEventExceptionProcessor.process(error, flowEventPipeline.context)).thenReturn(errorContext) - val response = processor.onNext(state, getFlowEventRecord(FlowEvent(flowKey, payload))) + val response = processor.onNext(checkpoint, getFlowEventRecord(FlowEvent(flowKey, payload))) assertThat(response).isEqualTo(errorResponse) } @@ -249,7 +247,7 @@ class FlowEventProcessorImplTest { whenever(flowEventPipeline.eventPreProcessing()).thenThrow(error) whenever(flowEventExceptionProcessor.process(error, updatedContext)).thenReturn(errorContext) - val response = processor.onNext(state, getFlowEventRecord(FlowEvent(flowKey, payload))) + val response = processor.onNext(checkpoint, getFlowEventRecord(FlowEvent(flowKey, payload))) assertThat(response).isEqualTo(errorResponse) } @@ -270,7 +268,7 @@ class FlowEventProcessorImplTest { whenever(flowEventExceptionProcessor.process(error, updatedContext)).thenReturn(flowKillErrorContext) whenever(flowEventContextConverter.convert(eq(flowKillErrorContext))).thenReturn(killErrorResponse) - val result = processor.onNext(state, getFlowEventRecord(FlowEvent(flowKey, payload))) + val result = processor.onNext(checkpoint, getFlowEventRecord(FlowEvent(flowKey, payload))) assertThat(result).isEqualTo(killErrorResponse) } diff --git a/components/flow/flow-service/src/test/kotlin/net/corda/flow/test/utils/FlowEventContextHelper.kt b/components/flow/flow-service/src/test/kotlin/net/corda/flow/test/utils/FlowEventContextHelper.kt index 67a78a23ec6..9b7a09c7353 100644 --- a/components/flow/flow-service/src/test/kotlin/net/corda/flow/test/utils/FlowEventContextHelper.kt +++ b/components/flow/flow-service/src/test/kotlin/net/corda/flow/test/utils/FlowEventContextHelper.kt @@ -43,8 +43,7 @@ fun buildFlowEventContext( sendToDlq, emptyMap(), mock(), - mock(), - null + mock() ) } @@ -68,7 +67,6 @@ fun buildFlowEventContext( sendToDlq, emptyMap(), mock(), - mock(), - null + mock() ) } diff --git a/components/ledger/ledger-utxo-token-cache/src/main/kotlin/net/corda/ledger/utxo/token/cache/services/TokenCacheEventProcessor.kt b/components/ledger/ledger-utxo-token-cache/src/main/kotlin/net/corda/ledger/utxo/token/cache/services/TokenCacheEventProcessor.kt index a326a77ae56..36aacf716f1 100644 --- a/components/ledger/ledger-utxo-token-cache/src/main/kotlin/net/corda/ledger/utxo/token/cache/services/TokenCacheEventProcessor.kt +++ b/components/ledger/ledger-utxo-token-cache/src/main/kotlin/net/corda/ledger/utxo/token/cache/services/TokenCacheEventProcessor.kt @@ -12,7 +12,6 @@ import net.corda.ledger.utxo.token.cache.entities.TokenEvent import net.corda.ledger.utxo.token.cache.entities.TokenPoolCache import net.corda.ledger.utxo.token.cache.handlers.TokenEventHandler import net.corda.messaging.api.processor.StateAndEventProcessor -import net.corda.messaging.api.processor.StateAndEventProcessor.State import net.corda.messaging.api.records.Record import net.corda.tracing.traceStateAndEventExecution import net.corda.utilities.debug @@ -39,18 +38,15 @@ class TokenCacheEventProcessor( override val stateValueClass = TokenPoolCacheState::class.java override fun onNext( - state: State?, - event: Record, + state: TokenPoolCacheState?, + event: Record ): StateAndEventProcessor.Response { val tokenEvent = try { eventConverter.convert(event.value) } catch (e: Exception) { log.error("Unexpected error while processing event '${event}'. The event will be sent to the DLQ.", e) - return StateAndEventProcessor.Response( - state, - listOf(), - markForDLQ = true) + return StateAndEventProcessor.Response(state, listOf(), markForDLQ = true) } log.debug { "Token event received: $tokenEvent" } @@ -58,7 +54,7 @@ class TokenCacheEventProcessor( return traceStateAndEventExecution(event, "Token Event - ${tokenEvent.javaClass.simpleName}") { try { tokenSelectionMetrics.recordProcessingTime(tokenEvent) { - val nonNullableState = state?.value ?: TokenPoolCacheState().apply { + val nonNullableState = state ?: TokenPoolCacheState().apply { this.poolKey = event.key this.availableTokens = listOf() this.tokenClaims = listOf() @@ -91,13 +87,10 @@ class TokenCacheEventProcessor( log.debug { "sending token response: $result" } if (result == null) { - StateAndEventProcessor.Response( - State(poolCacheState.toAvro(), metadata = state?.metadata), - listOf() - ) + StateAndEventProcessor.Response(poolCacheState.toAvro(), listOf()) } else { StateAndEventProcessor.Response( - State(poolCacheState.toAvro(), metadata = state?.metadata), + poolCacheState.toAvro(), listOf(result) ) } diff --git a/components/ledger/ledger-utxo-token-cache/src/test/kotlin/net/corda/ledger/utxo/token/cache/impl/services/TokenCacheEventProcessorTest.kt b/components/ledger/ledger-utxo-token-cache/src/test/kotlin/net/corda/ledger/utxo/token/cache/impl/services/TokenCacheEventProcessorTest.kt index cfbbc883616..889a20e774b 100644 --- a/components/ledger/ledger-utxo-token-cache/src/test/kotlin/net/corda/ledger/utxo/token/cache/impl/services/TokenCacheEventProcessorTest.kt +++ b/components/ledger/ledger-utxo-token-cache/src/test/kotlin/net/corda/ledger/utxo/token/cache/impl/services/TokenCacheEventProcessorTest.kt @@ -17,7 +17,6 @@ import net.corda.ledger.utxo.token.cache.impl.POOL_CACHE_KEY import net.corda.ledger.utxo.token.cache.impl.POOL_KEY import net.corda.ledger.utxo.token.cache.services.TokenCacheEventProcessor import net.corda.ledger.utxo.token.cache.services.TokenSelectionMetricsImpl -import net.corda.messaging.api.processor.StateAndEventProcessor.State import net.corda.messaging.api.records.Record import net.corda.utilities.time.UTCClock import org.assertj.core.api.Assertions.assertThat @@ -66,13 +65,10 @@ class TokenCacheEventProcessorTest { val target = createTokenCacheEventProcessor() whenever(eventConverter.convert(any())).thenThrow(IllegalStateException()) - val result = target.onNext( - State(stateIn, metadata = null), - eventIn - ) + val result = target.onNext(stateIn, eventIn) assertThat(result.responseEvents).isEmpty() - assertThat(result.updatedState?.value).isSameAs(stateIn) + assertThat(result.updatedState).isSameAs(stateIn) assertThat(result.markForDLQ).isTrue } @@ -89,10 +85,7 @@ class TokenCacheEventProcessorTest { tokenSelectionMetrics ) - val result = target.onNext( - State(stateIn, metadata = null), - eventIn - ) + val result = target.onNext(stateIn, eventIn) verify(externalEventResponseFactory).platformError( eq( @@ -106,7 +99,7 @@ class TokenCacheEventProcessorTest { ) assertThat(result.responseEvents).isNotEmpty() - assertThat(result.updatedState?.value).isSameAs(stateIn) + assertThat(result.updatedState).isSameAs(stateIn) assertThat(result.markForDLQ).isFalse() } @@ -124,10 +117,7 @@ class TokenCacheEventProcessorTest { tokenSelectionMetrics ) - val result = target.onNext( - State(stateIn, metadata = null), - eventIn - ) + val result = target.onNext(stateIn, eventIn) verify(externalEventResponseFactory).platformError( eq( @@ -141,7 +131,7 @@ class TokenCacheEventProcessorTest { ) assertThat(result.responseEvents).isNotEmpty() - assertThat(result.updatedState?.value).isSameAs(stateIn) + assertThat(result.updatedState).isSameAs(stateIn) assertThat(result.markForDLQ).isFalse() } @@ -166,14 +156,11 @@ class TokenCacheEventProcessorTest { val target = createTokenCacheEventProcessor() - val result = target.onNext( - State(stateIn, metadata = null), - eventIn - ) + val result = target.onNext(stateIn, eventIn) assertThat(result.responseEvents).hasSize(1) assertThat(result.responseEvents.first()).isEqualTo(handlerResponse) - assertThat(result.updatedState?.value).isSameAs(outputState) + assertThat(result.updatedState).isSameAs(outputState) assertThat(result.markForDLQ).isFalse } @@ -204,7 +191,7 @@ class TokenCacheEventProcessorTest { assertThat(result.responseEvents).hasSize(1) assertThat(result.responseEvents.first()).isEqualTo(handlerResponse) - assertThat(result.updatedState?.value).isSameAs(outputState) + assertThat(result.updatedState).isSameAs(outputState) assertThat(result.markForDLQ).isFalse } @@ -229,10 +216,7 @@ class TokenCacheEventProcessorTest { val target = createTokenCacheEventProcessor() - target.onNext( - State(stateIn, metadata = null), - eventIn - ) + target.onNext(stateIn, eventIn) val inOrder = inOrder(cachePoolState, mockHandler) diff --git a/components/link-manager/src/main/kotlin/net/corda/p2p/linkmanager/delivery/DeliveryTracker.kt b/components/link-manager/src/main/kotlin/net/corda/p2p/linkmanager/delivery/DeliveryTracker.kt index 5d74040b65d..4bd6b6a66ff 100644 --- a/components/link-manager/src/main/kotlin/net/corda/p2p/linkmanager/delivery/DeliveryTracker.kt +++ b/components/link-manager/src/main/kotlin/net/corda/p2p/linkmanager/delivery/DeliveryTracker.kt @@ -20,7 +20,6 @@ import net.corda.membership.grouppolicy.GroupPolicyProvider import net.corda.membership.read.MembershipGroupReaderProvider import net.corda.messaging.api.processor.StateAndEventProcessor import net.corda.messaging.api.processor.StateAndEventProcessor.Response -import net.corda.messaging.api.processor.StateAndEventProcessor.State import net.corda.messaging.api.publisher.config.PublisherConfig import net.corda.messaging.api.publisher.factory.PublisherFactory import net.corda.messaging.api.records.Record @@ -151,8 +150,8 @@ internal class DeliveryTracker( val processor = object : StateAndEventProcessor { override fun onNext( - state: State?, - event: Record, + state: AuthenticatedMessageDeliveryState?, + event: Record ): Response { val marker = event.value if (marker == null) { @@ -162,26 +161,18 @@ internal class DeliveryTracker( val markerType = marker.marker val timestamp = marker.timestamp return when (markerType) { - is LinkManagerProcessedMarker -> Response( - State( - AuthenticatedMessageDeliveryState(markerType.message, timestamp), - metadata = state?.metadata - ), - emptyList() - ) + is LinkManagerProcessedMarker -> Response(AuthenticatedMessageDeliveryState(markerType.message, timestamp), emptyList()) is LinkManagerReceivedMarker -> { - val value = state?.value - if (value != null) { + if (state != null) { // if we receive multiple acknowledgements, it is possible the state might have been nullified already. // Only the first one matters for calculating the end-to-end delivery latency anyway. - recordDeliveryLatencyMetric(value) + recordDeliveryLatencyMetric(state) } Response(null, emptyList()) } is TtlExpiredMarker -> { - val value = state?.value - if (value != null) { - recordTtlExpiredMetric(value) + if (state != null) { + recordTtlExpiredMetric(state) } Response(null, emptyList()) } @@ -189,11 +180,8 @@ internal class DeliveryTracker( } } - private fun respond(state: State?): Response { - return Response( - state, - emptyList() - ) + private fun respond(state: AuthenticatedMessageDeliveryState?): Response { + return Response(state, emptyList()) } private fun recordDeliveryLatencyMetric(state: AuthenticatedMessageDeliveryState) { diff --git a/components/link-manager/src/test/kotlin/net/corda/p2p/linkmanager/delivery/DeliveryTrackerTest.kt b/components/link-manager/src/test/kotlin/net/corda/p2p/linkmanager/delivery/DeliveryTrackerTest.kt index 9dbad6396b0..093fa06358a 100644 --- a/components/link-manager/src/test/kotlin/net/corda/p2p/linkmanager/delivery/DeliveryTrackerTest.kt +++ b/components/link-manager/src/test/kotlin/net/corda/p2p/linkmanager/delivery/DeliveryTrackerTest.kt @@ -173,9 +173,9 @@ class DeliveryTrackerTest { tracker.stop() assertEquals(0, response.responseEvents.size) - assertNotNull(response.updatedState?.value) - assertSame(messageAndKey, response.updatedState!!.value!!.message) - assertEquals(timeStamp, response.updatedState!!.value!!.timestamp) + assertNotNull(response.updatedState) + assertSame(messageAndKey, response.updatedState!!.message) + assertEquals(timeStamp, response.updatedState!!.timestamp) } @Test @@ -188,7 +188,7 @@ class DeliveryTrackerTest { tracker.stop() assertEquals(0, response.responseEvents.size) - assertNull(response.updatedState?.value) + assertNull(response.updatedState) } @Test diff --git a/components/membership/membership-persistence-service-impl/src/main/kotlin/net/corda/membership/impl/persistence/service/MembershipPersistenceAsyncProcessor.kt b/components/membership/membership-persistence-service-impl/src/main/kotlin/net/corda/membership/impl/persistence/service/MembershipPersistenceAsyncProcessor.kt index e693c8d9fde..8b16c5fc9a1 100644 --- a/components/membership/membership-persistence-service-impl/src/main/kotlin/net/corda/membership/impl/persistence/service/MembershipPersistenceAsyncProcessor.kt +++ b/components/membership/membership-persistence-service-impl/src/main/kotlin/net/corda/membership/impl/persistence/service/MembershipPersistenceAsyncProcessor.kt @@ -4,7 +4,6 @@ import net.corda.data.membership.db.request.async.MembershipPersistenceAsyncRequ import net.corda.data.membership.db.request.async.MembershipPersistenceAsyncRequestState import net.corda.membership.impl.persistence.service.handler.HandlerFactories import net.corda.messaging.api.processor.StateAndEventProcessor -import net.corda.messaging.api.processor.StateAndEventProcessor.State import net.corda.messaging.api.records.Record import org.slf4j.Logger import org.slf4j.LoggerFactory @@ -19,9 +18,10 @@ internal class MembershipPersistenceAsyncProcessor( const val MAX_RETRIES = 10 } override fun onNext( - state: State?, + state: MembershipPersistenceAsyncRequestState?, event: Record, ): StateAndEventProcessor.Response { + val numberOfRetriesSoFar = state?.numberOfRetriesSoFar ?: 0 val request = event.value if (request == null) { logger.warn("Empty request for ${event.key}") @@ -46,21 +46,21 @@ internal class MembershipPersistenceAsyncProcessor( retry( event.key, e, - state, + numberOfRetriesSoFar, request, ) } catch (e: OptimisticLockException) { retry( event.key, e, - state, + numberOfRetriesSoFar, request, ) } catch (e: RecoverableException) { retry( event.key, e, - state, + numberOfRetriesSoFar, request, ) } catch (e: Exception) { @@ -71,20 +71,16 @@ internal class MembershipPersistenceAsyncProcessor( private fun retry( key: String, e: Exception, - state: State?, + numberOfRetriesSoFar: Int, request: MembershipPersistenceAsyncRequest ): StateAndEventProcessor.Response { - val numberOfRetriesSoFar = state?.value?.numberOfRetriesSoFar ?: 0 return if (numberOfRetriesSoFar < MAX_RETRIES) { logger.warn("Got error while trying to execute $key. Will retry again.", e) StateAndEventProcessor.Response( - updatedState = State( - MembershipPersistenceAsyncRequestState( - request, - numberOfRetriesSoFar + 1, - handlers.persistenceHandlerServices.clock.instant(), - ), - metadata = state?.metadata, + updatedState = MembershipPersistenceAsyncRequestState( + request, + numberOfRetriesSoFar + 1, + handlers.persistenceHandlerServices.clock.instant(), ), responseEvents = emptyList(), markForDLQ = false, diff --git a/components/membership/membership-persistence-service-impl/src/test/kotlin/net/corda/membership/impl/persistence/service/MembershipPersistenceAsyncProcessorTest.kt b/components/membership/membership-persistence-service-impl/src/test/kotlin/net/corda/membership/impl/persistence/service/MembershipPersistenceAsyncProcessorTest.kt index dfa340dc8b8..a9d71b7e136 100644 --- a/components/membership/membership-persistence-service-impl/src/test/kotlin/net/corda/membership/impl/persistence/service/MembershipPersistenceAsyncProcessorTest.kt +++ b/components/membership/membership-persistence-service-impl/src/test/kotlin/net/corda/membership/impl/persistence/service/MembershipPersistenceAsyncProcessorTest.kt @@ -8,7 +8,6 @@ import net.corda.data.membership.db.request.async.MembershipPersistenceAsyncRequ import net.corda.membership.impl.persistence.service.handler.HandlerFactories import net.corda.membership.impl.persistence.service.handler.PersistenceHandlerServices import net.corda.messaging.api.processor.StateAndEventProcessor -import net.corda.messaging.api.processor.StateAndEventProcessor.State import net.corda.messaging.api.records.Record import net.corda.utilities.time.Clock import net.corda.v5.base.exceptions.CordaRuntimeException @@ -60,7 +59,7 @@ class MembershipPersistenceAsyncProcessorTest { ) assertThat(reply).isEqualTo( - StateAndEventProcessor.Response( + StateAndEventProcessor.Response( null, emptyList(), true, @@ -76,7 +75,7 @@ class MembershipPersistenceAsyncProcessorTest { ) assertThat(reply).isEqualTo( - StateAndEventProcessor.Response( + StateAndEventProcessor.Response( null, emptyList(), false, @@ -104,7 +103,7 @@ class MembershipPersistenceAsyncProcessorTest { ) assertThat(reply).isEqualTo( - StateAndEventProcessor.Response( + StateAndEventProcessor.Response( null, emptyList(), true, @@ -123,13 +122,10 @@ class MembershipPersistenceAsyncProcessorTest { assertThat(reply).isEqualTo( StateAndEventProcessor.Response( - State( - MembershipPersistenceAsyncRequestState( - envelope, - 1, - now, - ), - metadata = null + MembershipPersistenceAsyncRequestState( + envelope, + 1, + now, ), emptyList(), false, @@ -142,26 +138,20 @@ class MembershipPersistenceAsyncProcessorTest { whenever(handlers.handle(any())).doThrow(OptimisticLockException("Nop")) val reply = processor.onNext( - State( - MembershipPersistenceAsyncRequestState( - envelope, - 1, - now, - ), - metadata = null + MembershipPersistenceAsyncRequestState( + envelope, + 1, + now, ), Record("topic", "key", envelope) ) assertThat(reply).isEqualTo( StateAndEventProcessor.Response( - State( - MembershipPersistenceAsyncRequestState( - envelope, - 2, - now, - ), - metadata = null + MembershipPersistenceAsyncRequestState( + envelope, + 2, + now, ), emptyList(), false, @@ -174,19 +164,16 @@ class MembershipPersistenceAsyncProcessorTest { whenever(handlers.handle(any())).doThrow(RecoverableException("Nop")) val reply = processor.onNext( - State( - MembershipPersistenceAsyncRequestState( - envelope, - 20, - now, - ), - metadata = null + MembershipPersistenceAsyncRequestState( + envelope, + 20, + now, ), Record("topic", "key", envelope) ) assertThat(reply).isEqualTo( - StateAndEventProcessor.Response( + StateAndEventProcessor.Response( null, emptyList(), true, diff --git a/components/membership/membership-service-impl/src/main/kotlin/net/corda/membership/service/impl/CommandsRetryManager.kt b/components/membership/membership-service-impl/src/main/kotlin/net/corda/membership/service/impl/CommandsRetryManager.kt index 0bbf4acc9c6..3cb6189bfaa 100644 --- a/components/membership/membership-service-impl/src/main/kotlin/net/corda/membership/service/impl/CommandsRetryManager.kt +++ b/components/membership/membership-service-impl/src/main/kotlin/net/corda/membership/service/impl/CommandsRetryManager.kt @@ -8,7 +8,6 @@ import net.corda.data.membership.async.request.SentToMgmWaitingForNetwork import net.corda.libs.configuration.SmartConfig import net.corda.lifecycle.Resource import net.corda.messaging.api.processor.StateAndEventProcessor -import net.corda.messaging.api.processor.StateAndEventProcessor.State import net.corda.messaging.api.publisher.config.PublisherConfig import net.corda.messaging.api.publisher.factory.PublisherFactory import net.corda.messaging.api.records.Record @@ -47,11 +46,11 @@ internal class CommandsRetryManager( private val timers = ConcurrentHashMap>() override fun onNext( - state: State?, + state: MembershipAsyncRequestState?, event: Record, ): StateAndEventProcessor.Response { return StateAndEventProcessor.Response( - updatedState = State(event.value, state?.metadata), + updatedState = event.value, responseEvents = emptyList(), markForDLQ = false, ) diff --git a/components/membership/registration-impl/src/main/kotlin/net/corda/membership/impl/registration/dynamic/RegistrationProcessor.kt b/components/membership/registration-impl/src/main/kotlin/net/corda/membership/impl/registration/dynamic/RegistrationProcessor.kt index 2ada4552ca7..77f251583c3 100644 --- a/components/membership/registration-impl/src/main/kotlin/net/corda/membership/impl/registration/dynamic/RegistrationProcessor.kt +++ b/components/membership/registration-impl/src/main/kotlin/net/corda/membership/impl/registration/dynamic/RegistrationProcessor.kt @@ -34,7 +34,6 @@ import net.corda.membership.persistence.client.MembershipPersistenceClient import net.corda.membership.persistence.client.MembershipQueryClient import net.corda.membership.read.MembershipGroupReaderProvider import net.corda.messaging.api.processor.StateAndEventProcessor -import net.corda.messaging.api.processor.StateAndEventProcessor.State import net.corda.messaging.api.records.Record import net.corda.utilities.time.Clock import org.slf4j.LoggerFactory @@ -130,62 +129,61 @@ class RegistrationProcessor( @Suppress("ComplexMethod") override fun onNext( - state: State?, - event: Record, + state: RegistrationState?, + event: Record ): StateAndEventProcessor.Response { logger.info("Processing registration command for registration ID ${event.key}.") val result = try { - val stateValue = state?.value when (val command = event.value?.command) { is QueueRegistration -> { logger.info("Received queue registration command.") - handlers[QueueRegistration::class.java]?.invoke(stateValue, event) + handlers[QueueRegistration::class.java]?.invoke(state, event) } is CheckForPendingRegistration -> { logger.info("Received check for pending registration command.") - handlers[CheckForPendingRegistration::class.java]?.invoke(stateValue, event) + handlers[CheckForPendingRegistration::class.java]?.invoke(state, event) } is StartRegistration -> { logger.info("Received start registration command.") - handlers[StartRegistration::class.java]?.invoke(stateValue, event) + handlers[StartRegistration::class.java]?.invoke(state, event) } is VerifyMember -> { logger.info("Received verify member during registration command.") - handlers[VerifyMember::class.java]?.invoke(stateValue, event) + handlers[VerifyMember::class.java]?.invoke(state, event) } is ProcessMemberVerificationResponse -> { logger.info("Received process member verification response during registration command.") - handlers[ProcessMemberVerificationResponse::class.java]?.invoke(stateValue, event) + handlers[ProcessMemberVerificationResponse::class.java]?.invoke(state, event) } is ApproveRegistration -> { logger.info("Received approve registration command.") - handlers[ApproveRegistration::class.java]?.invoke(stateValue, event) + handlers[ApproveRegistration::class.java]?.invoke(state, event) } is DeclineRegistration -> { logger.info("Received decline registration command.") logger.warn("Declining registration because: ${command.reason}") - handlers[DeclineRegistration::class.java]?.invoke(stateValue, event) + handlers[DeclineRegistration::class.java]?.invoke(state, event) } is ProcessMemberVerificationRequest -> { logger.info("Received process member verification request during registration command.") - handlers[ProcessMemberVerificationRequest::class.java]?.invoke(stateValue, event) + handlers[ProcessMemberVerificationRequest::class.java]?.invoke(state, event) } is PersistMemberRegistrationState -> { logger.info("Received persist member registration state command.") - handlers[PersistMemberRegistrationState::class.java]?.invoke(stateValue, event) + handlers[PersistMemberRegistrationState::class.java]?.invoke(state, event) } else -> { logger.warn("Unhandled registration command received.") - createEmptyResult(stateValue) + createEmptyResult(state) } } } catch (e: MissingRegistrationStateException) { @@ -196,9 +194,7 @@ class RegistrationProcessor( createEmptyResult() } return StateAndEventProcessor.Response( - result?.updatedState.let { - State(it, state?.metadata) - }, + result?.updatedState, result?.outputStates ?: emptyList() ) } diff --git a/components/membership/registration-impl/src/test/kotlin/net/corda/membership/impl/registration/dynamic/RegistrationProcessorTest.kt b/components/membership/registration-impl/src/test/kotlin/net/corda/membership/impl/registration/dynamic/RegistrationProcessorTest.kt index d76e6991f26..3016721747c 100644 --- a/components/membership/registration-impl/src/test/kotlin/net/corda/membership/impl/registration/dynamic/RegistrationProcessorTest.kt +++ b/components/membership/registration-impl/src/test/kotlin/net/corda/membership/impl/registration/dynamic/RegistrationProcessorTest.kt @@ -42,7 +42,6 @@ import net.corda.membership.lib.MemberInfoExtension.Companion.PARTY_SESSION_KEYS import net.corda.membership.lib.MemberInfoExtension.Companion.PLATFORM_VERSION import net.corda.membership.lib.SelfSignedMemberInfo import net.corda.membership.persistence.client.MembershipPersistenceOperation -import net.corda.messaging.api.processor.StateAndEventProcessor.State import net.corda.test.util.time.TestClock import net.corda.v5.base.types.MemberX500Name import net.corda.v5.membership.EndpointInfo @@ -282,13 +281,8 @@ class RegistrationProcessorTest { null, RegistrationState(registrationId, holdingIdentity, mgmHoldingIdentity, emptyList()) ).forEach { state -> - with( - processor.onNext( - State(state, metadata = null), - Record(testTopic, testTopicKey, RegistrationCommand(Any())) - ) - ) { - assertThat(updatedState?.value).isEqualTo(state) + with(processor.onNext(state, Record(testTopic, testTopicKey, RegistrationCommand(Any())))) { + assertThat(updatedState).isEqualTo(state) assertThat(responseEvents).isEmpty() } } @@ -297,7 +291,7 @@ class RegistrationProcessorTest { @Test fun `queue registration command - onNext can be called`() { val result = processor.onNext(null, Record(testTopic, testTopicKey, queueRegistrationCommand)) - assertThat(result.updatedState?.value).isNull() + assertThat(result.updatedState).isNull() assertThat(result.responseEvents).isNotEmpty.hasSize(2) assertThat(result.responseEvents.firstNotNullOf { it.value as? RegistrationCommand }.command) .isNotNull @@ -307,7 +301,7 @@ class RegistrationProcessorTest { @Test fun `check for pending registration command - onNext can be called`() { val result = processor.onNext(null, Record(testTopic, testTopicKey, checkForPendingRegistrationCommand)) - assertThat(result.updatedState?.value).isNotNull() + assertThat(result.updatedState).isNotNull() assertThat(result.responseEvents).isNotEmpty.hasSize(1) assertThat((result.responseEvents.first().value as? RegistrationCommand)?.command) .isNotNull @@ -317,13 +311,10 @@ class RegistrationProcessorTest { @Test fun `start registration command - onNext can be called for start registration command`() { val result = processor.onNext( - State( - RegistrationState(registrationId, holdingIdentity, mgmHoldingIdentity, emptyList()), - metadata = null - ), + RegistrationState(registrationId, holdingIdentity, mgmHoldingIdentity, emptyList()), Record(testTopic, testTopicKey, startRegistrationCommand) ) - assertThat(result.updatedState?.value).isNotNull + assertThat(result.updatedState).isNotNull val events = result.responseEvents assertThat(events).isNotEmpty.hasSize(2) assertThat(events.firstNotNullOf { it.value as? RegistrationCommand }.command) @@ -334,7 +325,7 @@ class RegistrationProcessorTest { @Test fun `process member verification request command - onNext can be called for command`() { val result = processor.onNext(null, Record(testTopic, testTopicKey, verificationRequestCommand)) - assertThat(result.updatedState?.value).isNull() + assertThat(result.updatedState).isNull() assertThat(result.responseEvents) .hasSize(1) .anySatisfy { @@ -357,11 +348,8 @@ class RegistrationProcessorTest { @Test fun `verify member command - onNext can be called for command`() { - val result = processor.onNext( - State(state, metadata = null), - Record(testTopic, testTopicKey, verifyMemberCommand) - ) - assertThat(result.updatedState?.value).isNotNull + val result = processor.onNext(state, Record(testTopic, testTopicKey, verifyMemberCommand)) + assertThat(result.updatedState).isNotNull assertThat(result.responseEvents).isNotEmpty.hasSize(1) .allMatch { (result.responseEvents.first().value as? AppMessage)?.message as? AuthenticatedMessage != null @@ -371,7 +359,7 @@ class RegistrationProcessorTest { @Test fun `missing RegistrationState results in empty response`() { val result = processor.onNext(null, Record(testTopic, testTopicKey, verifyMemberCommand)) - assertThat(result.updatedState?.value).isNull() + assertThat(result.updatedState).isNull() assertThat(result.responseEvents).isEmpty() } } diff --git a/libs/flows/flow-api/src/main/kotlin/net/corda/flow/pipeline/events/FlowEventContext.kt b/libs/flows/flow-api/src/main/kotlin/net/corda/flow/pipeline/events/FlowEventContext.kt index ba389ce5f34..c2ce2b05957 100644 --- a/libs/flows/flow-api/src/main/kotlin/net/corda/flow/pipeline/events/FlowEventContext.kt +++ b/libs/flows/flow-api/src/main/kotlin/net/corda/flow/pipeline/events/FlowEventContext.kt @@ -4,7 +4,6 @@ import net.corda.data.flow.event.FlowEvent import net.corda.flow.pipeline.metrics.FlowMetrics import net.corda.flow.state.FlowCheckpoint import net.corda.libs.configuration.SmartConfig -import net.corda.libs.statemanager.api.Metadata import net.corda.messaging.api.records.Record import net.corda.tracing.TraceContext @@ -24,7 +23,6 @@ import net.corda.tracing.TraceContext * @param mdcProperties properties to set the flow fibers MDC with. * @param flowMetrics The [FlowMetrics] instance associated with the flow event * @param flowTraceContext The [TraceContext] instance associated with the flow event - * @param metadata Metadata associated with the checkpoint in state storage */ data class FlowEventContext( val checkpoint: FlowCheckpoint, @@ -37,6 +35,5 @@ data class FlowEventContext( val sendToDlq: Boolean = false, val mdcProperties: Map, val flowMetrics: FlowMetrics, - val flowTraceContext: TraceContext, - val metadata: Metadata? + val flowTraceContext: TraceContext ) diff --git a/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/ProcessorTask.kt b/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/ProcessorTask.kt index 1faeb4d28c3..db7016b23a6 100644 --- a/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/ProcessorTask.kt +++ b/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/ProcessorTask.kt @@ -28,23 +28,18 @@ data class ProcessorTask( } override fun call(): Result { - var state = stateManagerHelper.deserializeValue(persistedState)?.let { stateValue -> - StateAndEventProcessor.State( - stateValue, - persistedState?.metadata - ) - } + var stateValue = stateManagerHelper.deserializeValue(persistedState) val outputEvents = events.map { event -> - val response = processor.onNext(state, event) - state = response.updatedState + val response = processor.onNext(stateValue, event) + stateValue = response.updatedState response.responseEvents }.flatten() val updatedState = stateManagerHelper.createOrUpdateState( key.toString(), persistedState, - state, + stateValue ) return Result(this, outputEvents, updatedState) diff --git a/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/StateManagerHelper.kt b/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/StateManagerHelper.kt index 4d6389d7e26..3d43146696b 100644 --- a/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/StateManagerHelper.kt +++ b/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/StateManagerHelper.kt @@ -5,7 +5,6 @@ import net.corda.avro.serialization.CordaAvroSerializer import net.corda.libs.statemanager.api.Metadata import net.corda.libs.statemanager.api.State import net.corda.libs.statemanager.api.StateManager -import net.corda.messaging.api.processor.StateAndEventProcessor /** * Helper for working with [StateManager], used by [MultiSourceEventMediatorImpl]. @@ -21,18 +20,18 @@ class StateManagerHelper( * * @param key Event's key. * @param persistedState State being updated. - * @param newState Updated state. + * @param newValue Updated state value. */ fun createOrUpdateState( key: String, persistedState: State?, - newState: StateAndEventProcessor.State?, - ) = serialize(newState?.value)?.let { serializedValue -> + newValue: S?, + ) = serialize(newValue)?.let { serializedValue -> State( key, serializedValue, persistedState?.version ?: State.VERSION_INITIAL_VALUE, - newState?.metadata ?: Metadata(), + persistedState?.metadata ?: Metadata() ) } diff --git a/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/subscription/StateAndEventSubscriptionImpl.kt b/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/subscription/StateAndEventSubscriptionImpl.kt index c58ed7697c0..2753f0fed70 100644 --- a/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/subscription/StateAndEventSubscriptionImpl.kt +++ b/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/subscription/StateAndEventSubscriptionImpl.kt @@ -13,7 +13,6 @@ import net.corda.messaging.api.chunking.ChunkSerializerService import net.corda.messaging.api.exception.CordaMessageAPIFatalException import net.corda.messaging.api.exception.CordaMessageAPIIntermittentException import net.corda.messaging.api.processor.StateAndEventProcessor -import net.corda.messaging.api.processor.StateAndEventProcessor.State import net.corda.messaging.api.records.Record import net.corda.messaging.api.subscription.StateAndEventSubscription import net.corda.messaging.api.subscription.listener.StateAndEventListener @@ -277,7 +276,7 @@ internal class StateAndEventSubscriptionImpl( val state = updatedStates[event.partition]?.get(event.key) val partitionId = event.partition val thisEventUpdates = getUpdatesForEvent(state, event) - val updatedState = thisEventUpdates?.updatedState?.value + val updatedState = thisEventUpdates?.updatedState when { @@ -330,12 +329,7 @@ internal class StateAndEventSubscriptionImpl( private fun getUpdatesForEvent(state: S?, event: CordaConsumerRecord): StateAndEventProcessor.Response? { val future = stateAndEventConsumer.waitForFunctionToFinish( - { - processor.onNext( - State(state, metadata = null), - event.toRecord() - ) - }, config.processorTimeout.toMillis(), + { processor.onNext(state, event.toRecord()) }, config.processorTimeout.toMillis(), "Failed to finish within the time limit for state: $state and event: $event" ) @Suppress("unchecked_cast") diff --git a/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/MultiSourceEventMediatorImplTest.kt b/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/MultiSourceEventMediatorImplTest.kt index 61e1a2f28a3..e676375909a 100644 --- a/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/MultiSourceEventMediatorImplTest.kt +++ b/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/MultiSourceEventMediatorImplTest.kt @@ -74,7 +74,7 @@ class MultiSourceEventMediatorImplTest { any() ) ).thenAnswer { - StateAndEventProcessor.Response( + StateAndEventProcessor.Response( updatedState = mock(), responseEvents = listOf( Record( diff --git a/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/ProcessorTaskTest.kt b/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/ProcessorTaskTest.kt index 8a35d21b3a3..a0321712b3e 100644 --- a/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/ProcessorTaskTest.kt +++ b/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/ProcessorTaskTest.kt @@ -1,6 +1,5 @@ package net.corda.messaging.mediator -import net.corda.libs.statemanager.api.Metadata import net.corda.libs.statemanager.api.State import net.corda.messaging.api.processor.StateAndEventProcessor import net.corda.messaging.api.records.Record @@ -32,7 +31,7 @@ class ProcessorTaskTest { private val stateManagerHelper = mock>() @Captor - private val stateCaptor = argumentCaptor>() + private val stateCaptor = argumentCaptor() @Captor private val eventCaptor = argumentCaptor>() @@ -40,13 +39,13 @@ class ProcessorTaskTest { @BeforeEach fun setup() { `when`(processor.onNext(anyOrNull(), any())).thenAnswer { invocation -> - val state = invocation.getArgument>(0) - val id = state?.let { it.value!!.id + 1 } ?: 0 + val state = invocation.getArgument(0) + val id = state?.let { it.id + 1 } ?: 0 StateAndEventProcessor.Response( - StateAndEventProcessor.State(StateType(id), Metadata(mapOf("id" to id))), + StateType(id), listOf( EventType("outputEvent$id").toRecord() - ), + ) ) } @@ -73,15 +72,9 @@ class ProcessorTaskTest { val result = task.call() - verify(processor, times(inputEventRecords.size)).onNext( - stateCaptor.capture(), eventCaptor.capture() - ) + verify(processor, times(inputEventRecords.size)).onNext(stateCaptor.capture(), eventCaptor.capture()) val capturedInputStates = stateCaptor.allValues - val expectedInputStates = listOf( - null, - StateAndEventProcessor.State(StateType(0), Metadata(mapOf("id" to 0))), - StateAndEventProcessor.State(StateType(1), Metadata(mapOf("id" to 1))), - ) + val expectedInputStates = listOf(null, StateType(0), StateType(1)) assertEquals(expectedInputStates, capturedInputStates) val capturedInputEventRecords = eventCaptor.allValues assertEquals(inputEventRecords, capturedInputEventRecords) diff --git a/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/StateManagerHelperTest.kt b/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/StateManagerHelperTest.kt index a1c9db1b781..5610a805d5e 100644 --- a/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/StateManagerHelperTest.kt +++ b/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/StateManagerHelperTest.kt @@ -5,7 +5,6 @@ import net.corda.avro.serialization.CordaAvroSerializer import net.corda.libs.statemanager.api.Metadata import net.corda.libs.statemanager.api.State import net.corda.libs.statemanager.api.StateManager -import net.corda.messaging.api.processor.StateAndEventProcessor import org.junit.jupiter.api.Assertions.assertArrayEquals import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Assertions.assertNotNull @@ -52,10 +51,7 @@ class StateManagerHelperTest { fun `successfully creates new state`() { val persistedState: State? = null - val newState = StateAndEventProcessor.State( - StateType(1), - mock(), - ) + val newValue = StateType(1) val stateManagerHelper = StateManagerHelper( stateManager, stateSerializer, @@ -63,14 +59,14 @@ class StateManagerHelperTest { ) val state = stateManagerHelper.createOrUpdateState( - TEST_KEY, persistedState, newState + TEST_KEY, persistedState, newValue ) assertNotNull(state) assertEquals(TEST_KEY, state!!.key) - assertArrayEquals(serialized(newState.value!!), state.value) + assertArrayEquals(serialized(newValue), state.value) assertEquals(State.VERSION_INITIAL_VALUE, state.version) - assertEquals(newState.metadata, state.metadata) + assertNotNull(state.metadata) } @Test @@ -82,10 +78,7 @@ class StateManagerHelperTest { stateVersion, mock() ) - val updatedState = StateAndEventProcessor.State( - StateType(TEST_STATE_VALUE.id + 1), - mock(), - ) + val updatedValue = StateType(TEST_STATE_VALUE.id + 1) val stateManagerHelper = StateManagerHelper( stateManager, stateSerializer, @@ -93,14 +86,14 @@ class StateManagerHelperTest { ) val state = stateManagerHelper.createOrUpdateState( - TEST_KEY, persistedState, updatedState + TEST_KEY, persistedState, updatedValue ) assertNotNull(state) assertEquals(persistedState.key, state!!.key) - assertArrayEquals(serialized(updatedState.value!!), state.value) + assertArrayEquals(serialized(updatedValue), state.value) assertEquals(persistedState.version, state.version) - assertEquals(updatedState.metadata, state.metadata) + assertEquals(persistedState.metadata, state.metadata) } @Test diff --git a/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/factory/MediatorComponentFactoryTest.kt b/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/factory/MediatorComponentFactoryTest.kt index 43b88ad86f8..f111353d0be 100644 --- a/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/factory/MediatorComponentFactoryTest.kt +++ b/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/factory/MediatorComponentFactoryTest.kt @@ -1,6 +1,5 @@ package net.corda.messaging.mediator.factory - import net.corda.messaging.api.mediator.MediatorConsumer import net.corda.messaging.api.mediator.MessageRouter import net.corda.messaging.api.mediator.MessagingClient @@ -11,7 +10,6 @@ import net.corda.messaging.api.mediator.factory.MessageRouterFactory import net.corda.messaging.api.mediator.factory.MessagingClientFactory import net.corda.messaging.api.mediator.factory.MessagingClientFinder import net.corda.messaging.api.processor.StateAndEventProcessor -import net.corda.messaging.api.processor.StateAndEventProcessor.State import net.corda.messaging.api.records.Record import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Assertions.assertNotNull @@ -29,12 +27,9 @@ import org.mockito.kotlin.whenever class MediatorComponentFactoryTest { private lateinit var mediatorComponentFactory: MediatorComponentFactory private val messageProcessor = object : StateAndEventProcessor { - override fun onNext( - state: State?, event: Record - ): StateAndEventProcessor.Response { + override fun onNext(state: String?, event: Record): StateAndEventProcessor.Response { TODO("Not yet implemented") } - override val keyClass get() = String::class.java override val stateValueClass get() = String::class.java override val eventValueClass get() = String::class.java diff --git a/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/subscription/StateAndEventSubscriptionImplTest.kt b/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/subscription/StateAndEventSubscriptionImplTest.kt index f468aec20bc..1f0a14fdb01 100644 --- a/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/subscription/StateAndEventSubscriptionImplTest.kt +++ b/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/subscription/StateAndEventSubscriptionImplTest.kt @@ -17,7 +17,6 @@ import net.corda.messaging.api.chunking.ChunkSerializerService import net.corda.messaging.api.exception.CordaMessageAPIFatalException import net.corda.messaging.api.exception.CordaMessageAPIIntermittentException import net.corda.messaging.api.processor.StateAndEventProcessor -import net.corda.messaging.api.processor.StateAndEventProcessor.State import net.corda.messaging.api.records.Record import net.corda.messaging.constants.SubscriptionType import net.corda.messaging.createResolvedSubscriptionConfig @@ -76,7 +75,7 @@ class StateAndEventSubscriptionImplTest { doAnswer { CompletableFuture.completedFuture( StateAndEventProcessor.Response( - State("newstate", metadata = null), + "newstate", emptyList() ) ) @@ -467,7 +466,7 @@ class StateAndEventSubscriptionImplTest { doAnswer { CompletableFuture.completedFuture( - StateAndEventProcessor.Response( + StateAndEventProcessor.Response( null, listOf(outputRecord), true diff --git a/libs/messaging/messaging/build.gradle b/libs/messaging/messaging/build.gradle index 728525d2f30..18c0c5d1ae5 100644 --- a/libs/messaging/messaging/build.gradle +++ b/libs/messaging/messaging/build.gradle @@ -7,8 +7,6 @@ dependencies { compileOnly 'org.osgi:osgi.annotation' compileOnly "co.paralleluniverse:quasar-osgi-annotations:$quasarVersion" - api project(":libs:state-manager:state-manager-api") - implementation platform("net.corda:corda-api:$cordaApiVersion") implementation 'org.jetbrains.kotlin:kotlin-osgi-bundle' implementation "com.typesafe:config:$typeSafeConfigVersion" diff --git a/libs/messaging/messaging/src/main/kotlin/net/corda/messaging/api/processor/StateAndEventProcessor.kt b/libs/messaging/messaging/src/main/kotlin/net/corda/messaging/api/processor/StateAndEventProcessor.kt index 59d17937cd0..9d8ed5c2bc3 100644 --- a/libs/messaging/messaging/src/main/kotlin/net/corda/messaging/api/processor/StateAndEventProcessor.kt +++ b/libs/messaging/messaging/src/main/kotlin/net/corda/messaging/api/processor/StateAndEventProcessor.kt @@ -1,6 +1,5 @@ package net.corda.messaging.api.processor -import net.corda.libs.statemanager.api.Metadata import net.corda.messaging.api.records.Record /** @@ -18,13 +17,6 @@ import net.corda.messaging.api.records.Record * [CordaFatalException] and will cause the subscription to close. */ interface StateAndEventProcessor { - /** - * State and metadata stored alongside the state. - */ - data class State( - val value: S?, - val metadata: Metadata?, - ) /** * This class encapsulates the responses that will be returned (from [onNext]) to the subscription for @@ -33,12 +25,8 @@ interface StateAndEventProcessor { data class Response( /** * The updated state in response to an incoming event from [onNext]. - * - * Both the state and its associated metadata will be overwritten in storage by this new state when provided by - * the processor. It is the processor's responsibility to ensure that any required metadata is preserved across - * processing. */ - val updatedState: State?, + val updatedState: S?, /** * A list of events to be published in response to an incoming event from [onNext]. @@ -50,7 +38,7 @@ interface StateAndEventProcessor { /** * Flag to indicate processing failed and the State and Event should be moved to the Dead Letter Queue */ - val markForDLQ: Boolean = false, + val markForDLQ: Boolean = false ) /** @@ -67,7 +55,7 @@ interface StateAndEventProcessor { * NOTE: The returned events will be published and the processed events will be consumed atomically as a * single transaction. */ - fun onNext(state: State?, event: Record): Response + fun onNext(state: S?, event: Record): Response /** * [keyClass], [stateValueClass] and [eventValueClass] to easily get the class types the processor operates upon. diff --git a/testing/message-patterns/src/integrationTest/kotlin/net/corda/messaging/integration/processors/TestStateEventProcessor.kt b/testing/message-patterns/src/integrationTest/kotlin/net/corda/messaging/integration/processors/TestStateEventProcessor.kt index 7a4ab2bc19e..7418a4d1fc6 100644 --- a/testing/message-patterns/src/integrationTest/kotlin/net/corda/messaging/integration/processors/TestStateEventProcessor.kt +++ b/testing/message-patterns/src/integrationTest/kotlin/net/corda/messaging/integration/processors/TestStateEventProcessor.kt @@ -5,7 +5,6 @@ import net.corda.data.demo.DemoStateRecord import net.corda.messaging.api.exception.CordaMessageAPIIntermittentException import net.corda.messaging.api.processor.StateAndEventProcessor import net.corda.messaging.api.processor.StateAndEventProcessor.Response -import net.corda.messaging.api.processor.StateAndEventProcessor.State import net.corda.messaging.api.records.Record import org.slf4j.LoggerFactory import java.util.concurrent.CountDownLatch @@ -33,9 +32,7 @@ class TestStateEventProcessor( get() = DemoRecord::class.java - override fun onNext( - state: State?, event: Record - ): Response { + override fun onNext(state: DemoStateRecord?, event: Record): Response { onNextLatch.countDown() log.info("Received record, ${onNextLatch.count} remaining") @@ -62,9 +59,6 @@ class TestStateEventProcessor( emptyList() } - return Response( - State(newState, metadata = null), - outputRecordList - ) + return Response(newState, outputRecordList) } } diff --git a/testing/message-patterns/src/integrationTest/kotlin/net/corda/messaging/integration/processors/TestStateEventProcessorStrings.kt b/testing/message-patterns/src/integrationTest/kotlin/net/corda/messaging/integration/processors/TestStateEventProcessorStrings.kt index 0a38a112d64..3735573c117 100644 --- a/testing/message-patterns/src/integrationTest/kotlin/net/corda/messaging/integration/processors/TestStateEventProcessorStrings.kt +++ b/testing/message-patterns/src/integrationTest/kotlin/net/corda/messaging/integration/processors/TestStateEventProcessorStrings.kt @@ -3,7 +3,6 @@ package net.corda.messaging.integration.processors import net.corda.messaging.api.exception.CordaMessageAPIIntermittentException import net.corda.messaging.api.processor.StateAndEventProcessor import net.corda.messaging.api.processor.StateAndEventProcessor.Response -import net.corda.messaging.api.processor.StateAndEventProcessor.State import net.corda.messaging.api.records.Record import java.util.concurrent.CountDownLatch @@ -24,9 +23,7 @@ class TestStateEventProcessorStrings( override val eventValueClass: Class get() = String::class.java - override fun onNext( - state: State?, event: Record - ): Response { + override fun onNext(state: String?, event: Record): Response { onNextLatch.countDown() if (delayProcessorOnFirst != null) { @@ -51,9 +48,6 @@ class TestStateEventProcessorStrings( emptyList() } - return Response( - State(newState, metadata = null), - outputRecordList - ) + return Response(newState, outputRecordList) } } diff --git a/testing/p2p/inmemory-messaging-impl/src/integrationTest/kotlin/net/corda/messaging/emulation/subscription/stateandevent/InMemoryStateAndEventSubscriptionIntegrationTests.kt b/testing/p2p/inmemory-messaging-impl/src/integrationTest/kotlin/net/corda/messaging/emulation/subscription/stateandevent/InMemoryStateAndEventSubscriptionIntegrationTests.kt index 7432a043613..a805c4dbd4e 100644 --- a/testing/p2p/inmemory-messaging-impl/src/integrationTest/kotlin/net/corda/messaging/emulation/subscription/stateandevent/InMemoryStateAndEventSubscriptionIntegrationTests.kt +++ b/testing/p2p/inmemory-messaging-impl/src/integrationTest/kotlin/net/corda/messaging/emulation/subscription/stateandevent/InMemoryStateAndEventSubscriptionIntegrationTests.kt @@ -63,19 +63,14 @@ class InMemoryStateAndEventSubscriptionIntegrationTests { groupName = "group", ) val processor = object : StateAndEventProcessor { - override fun onNext( - state: StateAndEventProcessor.State?, event: Record - ): StateAndEventProcessor.Response { + override fun onNext(state: State?, event: Record): StateAndEventProcessor.Response { if ((state != null) && (event.value == Event.STOP)) { - states[event.key.type] = state.value!!.number + states[event.key.type] = state.number countDown.countDown() } return if (event.value == Event.CREATE_STATE) { StateAndEventProcessor.Response( - StateAndEventProcessor.State( - State(event.key.type), - metadata = null - ), + State(event.key.type), listOf( Record( subscriptionConfig.eventTopic, @@ -85,13 +80,7 @@ class InMemoryStateAndEventSubscriptionIntegrationTests { ) ) } else { - StateAndEventProcessor.Response( - StateAndEventProcessor.State( - State(event.key.type), - metadata = null - ), - emptyList() - ) + StateAndEventProcessor.Response(State(event.key.type), emptyList()) } } @@ -143,9 +132,7 @@ class InMemoryStateAndEventSubscriptionIntegrationTests { Record(subscriptionConfig.eventTopic, Key(it), Event.SEND_TO_ANOTHER_TOPIC) } val processor = object : StateAndEventProcessor { - override fun onNext( - state: StateAndEventProcessor.State?, event: Record - ): StateAndEventProcessor.Response { + override fun onNext(state: State?, event: Record): StateAndEventProcessor.Response { return StateAndEventProcessor.Response( null, (1..increaseBy).map { @@ -168,9 +155,7 @@ class InMemoryStateAndEventSubscriptionIntegrationTests { subscription.start() val anotherProcessor = object : StateAndEventProcessor { - override fun onNext( - state: StateAndEventProcessor.State?, event: Record - ): StateAndEventProcessor.Response { + override fun onNext(state: String?, event: Record): StateAndEventProcessor.Response { got.add(event) countDown.countDown() return StateAndEventProcessor.Response(null, emptyList()) @@ -231,20 +216,15 @@ class InMemoryStateAndEventSubscriptionIntegrationTests { } val processor = object : StateAndEventProcessor { - override fun onNext( - state: StateAndEventProcessor.State?, event: Record - ): StateAndEventProcessor.Response { + override fun onNext(state: State?, event: Record): StateAndEventProcessor.Response { val newState = when (event.value) { Event.CREATE_STATE -> 1 - Event.INCREASE_STATE -> (state?.value!!.number + 1) + Event.INCREASE_STATE -> (state!!.number + 1) else -> throw Exception("Unexpected event!") } countDown.countDown() return StateAndEventProcessor.Response( - StateAndEventProcessor.State( - State(newState), - metadata = null - ), + State(newState), emptyList() ) } @@ -322,14 +302,9 @@ class InMemoryStateAndEventSubscriptionIntegrationTests { } } val processor = object : StateAndEventProcessor { - override fun onNext( - state: StateAndEventProcessor.State?, event: Record - ): StateAndEventProcessor.Response { + override fun onNext(state: State?, event: Record): StateAndEventProcessor.Response { return StateAndEventProcessor.Response( - StateAndEventProcessor.State( - State(state?.value?.number?.inc() ?: -1), - metadata = null - ), + State(state?.number?.inc() ?: -1), emptyList() ) } diff --git a/testing/p2p/inmemory-messaging-impl/src/main/kotlin/net/corda/messaging/emulation/subscription/stateandevent/EventSubscription.kt b/testing/p2p/inmemory-messaging-impl/src/main/kotlin/net/corda/messaging/emulation/subscription/stateandevent/EventSubscription.kt index ce7295f9d8a..7543f65a160 100644 --- a/testing/p2p/inmemory-messaging-impl/src/main/kotlin/net/corda/messaging/emulation/subscription/stateandevent/EventSubscription.kt +++ b/testing/p2p/inmemory-messaging-impl/src/main/kotlin/net/corda/messaging/emulation/subscription/stateandevent/EventSubscription.kt @@ -1,7 +1,6 @@ package net.corda.messaging.emulation.subscription.stateandevent import net.corda.lifecycle.Lifecycle -import net.corda.messaging.api.processor.StateAndEventProcessor.State import net.corda.messaging.api.records.Record import net.corda.messaging.emulation.topic.model.Consumption import net.corda.messaging.emulation.topic.model.RecordMetadata @@ -36,22 +35,19 @@ internal class EventSubscription( ) if (event != null) { val state = subscription.stateSubscription.getValue(event.key) - val response = subscription.processor.onNext( - State(state, metadata = null), - event - ) - subscription.setValue(event.key, response.updatedState?.value, eventMetaData.partition) + val response = subscription.processor.onNext(state, event) + subscription.setValue(event.key, response.updatedState, eventMetaData.partition) subscription.topicService.addRecords( listOf( Record( subscription.stateSubscriptionConfig.eventTopic, event.key, - response.updatedState?.value + response.updatedState ) ) + response.responseEvents ) - subscription.stateAndEventListener?.onPostCommit(mapOf(event.key to response.updatedState?.value)) + subscription.stateAndEventListener?.onPostCommit(mapOf(event.key to response.updatedState)) } } } diff --git a/testing/p2p/inmemory-messaging-impl/src/test/kotlin/net/corda/messaging/emulation/subscription/stateandevent/EventSubscriptionTest.kt b/testing/p2p/inmemory-messaging-impl/src/test/kotlin/net/corda/messaging/emulation/subscription/stateandevent/EventSubscriptionTest.kt index 57126f97dbc..ca63b347b28 100644 --- a/testing/p2p/inmemory-messaging-impl/src/test/kotlin/net/corda/messaging/emulation/subscription/stateandevent/EventSubscriptionTest.kt +++ b/testing/p2p/inmemory-messaging-impl/src/test/kotlin/net/corda/messaging/emulation/subscription/stateandevent/EventSubscriptionTest.kt @@ -1,7 +1,6 @@ package net.corda.messaging.emulation.subscription.stateandevent import net.corda.messaging.api.processor.StateAndEventProcessor -import net.corda.messaging.api.processor.StateAndEventProcessor.State import net.corda.messaging.api.records.Record import net.corda.messaging.api.subscription.config.SubscriptionConfig import net.corda.messaging.api.subscription.listener.StateAndEventListener @@ -35,14 +34,9 @@ class EventSubscriptionTest { on { stateSubscriptionConfig } doReturn SubscriptionConfig("group1", "topic1") on { stateSubscription } doReturn stateSubscription on { processor } doReturn object : StateAndEventProcessor { - override fun onNext( - state: State?, event: Record - ): StateAndEventProcessor.Response { - received.add(state?.value to event) - return StateAndEventProcessor.Response( - State(newState, metadata = null), - response - ) + override fun onNext(state: String?, event: Record): StateAndEventProcessor.Response { + received.add(state to event) + return StateAndEventProcessor.Response(newState, response) } override val keyClass = String::class.java diff --git a/testing/p2p/inmemory-messaging-impl/src/test/kotlin/net/corda/messaging/emulation/subscription/stateandevent/StateSubscriptionTest.kt b/testing/p2p/inmemory-messaging-impl/src/test/kotlin/net/corda/messaging/emulation/subscription/stateandevent/StateSubscriptionTest.kt index 8bcd0008965..feac82951dd 100644 --- a/testing/p2p/inmemory-messaging-impl/src/test/kotlin/net/corda/messaging/emulation/subscription/stateandevent/StateSubscriptionTest.kt +++ b/testing/p2p/inmemory-messaging-impl/src/test/kotlin/net/corda/messaging/emulation/subscription/stateandevent/StateSubscriptionTest.kt @@ -1,7 +1,6 @@ package net.corda.messaging.emulation.subscription.stateandevent import net.corda.messaging.api.processor.StateAndEventProcessor -import net.corda.messaging.api.processor.StateAndEventProcessor.State import net.corda.messaging.api.records.Record import net.corda.messaging.api.subscription.config.SubscriptionConfig import net.corda.messaging.api.subscription.listener.StateAndEventListener @@ -36,9 +35,7 @@ class StateSubscriptionTest { on { subscriptionConfig } doReturn SubscriptionConfig("group", "topic") on { stateSubscriptionConfig } doReturn SubscriptionConfig("group1", "topic1") on { processor } doReturn object : StateAndEventProcessor { - override fun onNext( - state: State?, event: Record - ): StateAndEventProcessor.Response { + override fun onNext(state: String?, event: Record): StateAndEventProcessor.Response { return StateAndEventProcessor.Response(null, emptyList()) } From 7dcbf244f5552d02ee80412b24e5a6c5a39c40b3 Mon Sep 17 00:00:00 2001 From: Viktor Kolomeyko Date: Thu, 12 Oct 2023 16:45:15 +0100 Subject: [PATCH 04/81] Revert "CORE-16203 Replace State and Event pattern with Multi-Source Event Mediator in FlowWorker (#4832)" This reverts commit 1e949e7907fe651fbe2c1b801f851f4c1fa05af6. --- .../mediator/FlowEventMediatorFactoryImpl.kt | 3 -- .../corda/flow/service/FlowExecutorImpl.kt | 40 +++++++++------ .../flow/service/FlowExecutorImplTest.kt | 50 +++++++++++-------- .../db/consumer/DBCordaConsumerImpl.kt | 29 ++++++----- .../kafka/consumer/CordaKafkaConsumerImpl.kt | 33 +++--------- .../consumer/CordaKafkaConsumerImplTest.kt | 26 +++++++--- .../messagebus/api/consumer/CordaConsumer.kt | 4 +- .../messaging/mediator/MessageBusConsumer.kt | 30 +++++++++-- .../mediator/MultiSourceEventMediatorImpl.kt | 16 +++++- .../mediator/taskmanager/TaskManagerImpl.kt | 21 ++------ .../mediator/MessageBusConsumerTest.kt | 22 +++++--- .../MultiSourceEventMediatorImplTest.kt | 23 ++++++--- .../mediator/TaskManagerHelperTest.kt | 4 +- .../api/mediator/MediatorConsumer.kt | 10 ++-- 14 files changed, 180 insertions(+), 131 deletions(-) diff --git a/components/flow/flow-service/src/main/kotlin/net/corda/flow/messaging/mediator/FlowEventMediatorFactoryImpl.kt b/components/flow/flow-service/src/main/kotlin/net/corda/flow/messaging/mediator/FlowEventMediatorFactoryImpl.kt index 5408e207373..6b90dd93361 100644 --- a/components/flow/flow-service/src/main/kotlin/net/corda/flow/messaging/mediator/FlowEventMediatorFactoryImpl.kt +++ b/components/flow/flow-service/src/main/kotlin/net/corda/flow/messaging/mediator/FlowEventMediatorFactoryImpl.kt @@ -7,7 +7,6 @@ import net.corda.data.flow.event.mapper.FlowMapperEvent import net.corda.data.flow.output.FlowStatus import net.corda.data.flow.state.checkpoint.Checkpoint import net.corda.data.ledger.persistence.LedgerPersistenceRequest -import net.corda.data.ledger.utxo.token.selection.event.TokenPoolCacheEvent import net.corda.data.persistence.EntityRequest import net.corda.data.uniqueness.UniquenessCheckRequestAvro import net.corda.flow.pipeline.factory.FlowEventProcessorFactory @@ -28,7 +27,6 @@ import net.corda.schema.Schemas.Flow.FLOW_MAPPER_EVENT_TOPIC import net.corda.schema.Schemas.Flow.FLOW_STATUS_TOPIC import net.corda.schema.Schemas.Persistence.PERSISTENCE_ENTITY_PROCESSOR_TOPIC import net.corda.schema.Schemas.Persistence.PERSISTENCE_LEDGER_PROCESSOR_TOPIC -import net.corda.schema.Schemas.Services.TOKEN_CACHE_EVENT import net.corda.schema.Schemas.UniquenessChecker.UNIQUENESS_CHECK_TOPIC import net.corda.schema.Schemas.Verification.VERIFICATION_LEDGER_PROCESSOR_TOPIC import org.osgi.service.component.annotations.Activate @@ -96,7 +94,6 @@ class FlowEventMediatorFactoryImpl @Activate constructor( is FlowOpsRequest -> routeTo(messageBusClient, FLOW_OPS_MESSAGE_TOPIC) is FlowStatus -> routeTo(messageBusClient, FLOW_STATUS_TOPIC) is LedgerPersistenceRequest -> routeTo(messageBusClient, PERSISTENCE_LEDGER_PROCESSOR_TOPIC) - is TokenPoolCacheEvent -> routeTo(messageBusClient, TOKEN_CACHE_EVENT) is TransactionVerificationRequest -> routeTo(messageBusClient, VERIFICATION_LEDGER_PROCESSOR_TOPIC) is UniquenessCheckRequestAvro -> routeTo(messageBusClient, UNIQUENESS_CHECK_TOPIC) else -> { diff --git a/components/flow/flow-service/src/main/kotlin/net/corda/flow/service/FlowExecutorImpl.kt b/components/flow/flow-service/src/main/kotlin/net/corda/flow/service/FlowExecutorImpl.kt index 0bef3fa8ef9..cb2dab9393b 100644 --- a/components/flow/flow-service/src/main/kotlin/net/corda/flow/service/FlowExecutorImpl.kt +++ b/components/flow/flow-service/src/main/kotlin/net/corda/flow/service/FlowExecutorImpl.kt @@ -3,7 +3,7 @@ package net.corda.flow.service import com.typesafe.config.ConfigValueFactory import net.corda.data.flow.event.FlowEvent import net.corda.data.flow.state.checkpoint.Checkpoint -import net.corda.flow.messaging.mediator.FlowEventMediatorFactory +import net.corda.flow.pipeline.factory.FlowEventProcessorFactory import net.corda.libs.configuration.SmartConfig import net.corda.libs.configuration.helper.getConfig import net.corda.lifecycle.LifecycleCoordinatorFactory @@ -13,7 +13,10 @@ import net.corda.lifecycle.RegistrationHandle import net.corda.lifecycle.StartEvent import net.corda.lifecycle.StopEvent import net.corda.lifecycle.createCoordinator -import net.corda.messaging.api.mediator.MultiSourceEventMediator +import net.corda.messaging.api.subscription.StateAndEventSubscription +import net.corda.messaging.api.subscription.config.SubscriptionConfig +import net.corda.messaging.api.subscription.factory.SubscriptionFactory +import net.corda.schema.Schemas.Flow.FLOW_EVENT_TOPIC import net.corda.schema.configuration.BootConfig.CRYPTO_WORKER_REST_ENDPOINT import net.corda.schema.configuration.BootConfig.PERSISTENCE_WORKER_REST_ENDPOINT import net.corda.schema.configuration.BootConfig.UNIQUENESS_WORKER_REST_ENDPOINT @@ -33,29 +36,34 @@ import org.slf4j.LoggerFactory @Component(service = [FlowExecutor::class]) class FlowExecutorImpl constructor( coordinatorFactory: LifecycleCoordinatorFactory, - private val flowEventMediatorFactory: FlowEventMediatorFactory, - private val toMessagingConfig: (Map) -> SmartConfig, + private val subscriptionFactory: SubscriptionFactory, + private val flowEventProcessorFactory: FlowEventProcessorFactory, + private val toMessagingConfig: (Map) -> SmartConfig ) : FlowExecutor { @Activate constructor( @Reference(service = LifecycleCoordinatorFactory::class) coordinatorFactory: LifecycleCoordinatorFactory, - @Reference(service = FlowEventMediatorFactory::class) - flowEventMediatorFactory: FlowEventMediatorFactory, + @Reference(service = SubscriptionFactory::class) + subscriptionFactory: SubscriptionFactory, + @Reference(service = FlowEventProcessorFactory::class) + flowEventProcessorFactory: FlowEventProcessorFactory ) : this( coordinatorFactory, - flowEventMediatorFactory, + subscriptionFactory, + flowEventProcessorFactory, { cfg -> cfg.getConfig(MESSAGING_CONFIG) } ) companion object { private val log = LoggerFactory.getLogger(this::class.java.enclosingClass) + private const val CONSUMER_GROUP = "FlowEventConsumer" } private val coordinator = coordinatorFactory.createCoordinator { event, _ -> eventHandler(event) } + private var subscription: StateAndEventSubscription? = null private var subscriptionRegistrationHandle: RegistrationHandle? = null - private var multiSourceEventMediator: MultiSourceEventMediator? = null override fun onConfigChange(config: Map) { try { @@ -64,18 +72,19 @@ class FlowExecutorImpl constructor( // close the lifecycle registration first to prevent down being signaled subscriptionRegistrationHandle?.close() - multiSourceEventMediator?.close() + subscription?.close() - multiSourceEventMediator = flowEventMediatorFactory.create( - updatedConfigs, - messagingConfig, + subscription = subscriptionFactory.createStateAndEventSubscription( + SubscriptionConfig(CONSUMER_GROUP, FLOW_EVENT_TOPIC), + flowEventProcessorFactory.create(updatedConfigs), + messagingConfig ) subscriptionRegistrationHandle = coordinator.followStatusChangesByName( - setOf(multiSourceEventMediator!!.subscriptionName) + setOf(subscription!!.subscriptionName) ) - multiSourceEventMediator?.start() + subscription?.start() } catch (ex: Exception) { val reason = "Failed to configure the flow executor using '${config}'" log.error(reason, ex) @@ -117,11 +126,10 @@ class FlowExecutorImpl constructor( is StartEvent -> { coordinator.updateStatus(LifecycleStatus.UP) } - is StopEvent -> { log.trace { "Flow executor is stopping..." } subscriptionRegistrationHandle?.close() - multiSourceEventMediator?.close() + subscription?.close() log.trace { "Flow executor stopped" } } } diff --git a/components/flow/flow-service/src/test/kotlin/net/corda/flow/service/FlowExecutorImplTest.kt b/components/flow/flow-service/src/test/kotlin/net/corda/flow/service/FlowExecutorImplTest.kt index ab0f58959eb..6a8aad3e000 100644 --- a/components/flow/flow-service/src/test/kotlin/net/corda/flow/service/FlowExecutorImplTest.kt +++ b/components/flow/flow-service/src/test/kotlin/net/corda/flow/service/FlowExecutorImplTest.kt @@ -3,7 +3,6 @@ package net.corda.flow.service import com.typesafe.config.ConfigValueFactory import net.corda.data.flow.event.FlowEvent import net.corda.data.flow.state.checkpoint.Checkpoint -import net.corda.flow.messaging.mediator.FlowEventMediatorFactory import net.corda.flow.pipeline.factory.FlowEventProcessorFactory import net.corda.libs.configuration.SmartConfig import net.corda.libs.configuration.SmartConfigImpl @@ -14,8 +13,9 @@ import net.corda.lifecycle.LifecycleEventHandler import net.corda.lifecycle.LifecycleStatus import net.corda.lifecycle.RegistrationHandle import net.corda.lifecycle.StopEvent -import net.corda.messaging.api.mediator.MultiSourceEventMediator import net.corda.messaging.api.processor.StateAndEventProcessor +import net.corda.messaging.api.subscription.StateAndEventSubscription +import net.corda.messaging.api.subscription.factory.SubscriptionFactory import net.corda.schema.configuration.BootConfig import net.corda.schema.configuration.ConfigKeys.BOOT_CONFIG import net.corda.schema.configuration.ConfigKeys.FLOW_CONFIG @@ -26,6 +26,7 @@ import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Test import org.mockito.Mockito.inOrder import org.mockito.kotlin.any +import org.mockito.kotlin.anyOrNull import org.mockito.kotlin.argumentCaptor import org.mockito.kotlin.eq import org.mockito.kotlin.mock @@ -36,7 +37,7 @@ class FlowExecutorImplTest { private val coordinatorFactory = mock() private val flowEventProcessorFactory = mock() - private val flowEventMediatorFactory = mock() + private val subscriptionFactory = mock() private val toMessagingConfig: (Map) -> SmartConfig = { messagingConfig } @@ -48,18 +49,20 @@ class FlowExecutorImplTest { private val messagingConfig = getMinimalMessagingConfig() private val subscriptionRegistrationHandle = mock() private val flowExecutorCoordinator = mock() - private val multiSourceEventMediator = mock>() + private val subscription = mock>() private val flowEventProcessor = mock>() @BeforeEach fun setup() { whenever(flowEventProcessorFactory.create(any())).thenReturn(flowEventProcessor) whenever( - flowEventMediatorFactory.create( + subscriptionFactory.createStateAndEventSubscription( any(), any(), + any(), + anyOrNull() ) - ).thenReturn(multiSourceEventMediator) + ).thenReturn(subscription) whenever(coordinatorFactory.createCoordinator(any(), any())).thenReturn(flowExecutorCoordinator) whenever(flowExecutorCoordinator.followStatusChangesByName(any())).thenReturn(subscriptionRegistrationHandle) @@ -73,7 +76,7 @@ class FlowExecutorImplTest { } @Test - fun `lifecycle - flow executor signals error if it fails to create event mediator`() { + fun `lifecycle - flow executor signals error if it fails to create a subscription`() { val invalidConfig = mapOf() val flowExecutor = getFlowExecutor() @@ -86,9 +89,9 @@ class FlowExecutorImplTest { } @Test - fun `lifecycle - flow executor signals error if event mediator signals error`() { + fun `lifecycle - flow executor signals error if the subscription signals error`() { val name = LifecycleCoordinatorName("", "") - whenever(multiSourceEventMediator.subscriptionName).thenReturn(name) + whenever(subscription.subscriptionName).thenReturn(name) val flowExecutor = getFlowExecutor() flowExecutor.start() @@ -105,7 +108,7 @@ class FlowExecutorImplTest { } @Test - fun `lifecycle - flow executor stops event mediator when stopped`() { + fun `lifecycle - flow executor stops subscription when stopped`() { val flowExecutor = getFlowExecutor() flowExecutor.onConfigChange(config) @@ -116,7 +119,7 @@ class FlowExecutorImplTest { } verify(subscriptionRegistrationHandle).close() - verify(multiSourceEventMediator).close() + verify(subscription).close() } @Test @@ -124,10 +127,10 @@ class FlowExecutorImplTest { val name1 = LifecycleCoordinatorName("", "") val name2 = LifecycleCoordinatorName("", "") val subscriptionRegistrationHandle2 = mock() - val multiSourceEventMediator2 = mock>() + val subscription2 = mock>() - whenever(multiSourceEventMediator.subscriptionName).thenReturn(name1) - whenever(multiSourceEventMediator2.subscriptionName).thenReturn(name2) + whenever(subscription.subscriptionName).thenReturn(name1) + whenever(subscription2.subscriptionName).thenReturn(name2) // First config change gets us subscribed val flowExecutor = getFlowExecutor() @@ -137,27 +140,29 @@ class FlowExecutorImplTest { // now we change config and should see the subscription registration removed, // the subscription re-created and then the subscription registered again whenever( - flowEventMediatorFactory.create( + subscriptionFactory.createStateAndEventSubscription( + any(), any(), any(), + anyOrNull() ) - ).thenReturn(multiSourceEventMediator2) + ).thenReturn(subscription2) whenever(flowExecutorCoordinator.followStatusChangesByName(any())).thenReturn(subscriptionRegistrationHandle2) flowExecutor.onConfigChange(config) inOrder( - multiSourceEventMediator, - multiSourceEventMediator2, + subscription, + subscription2, subscriptionRegistrationHandle, subscriptionRegistrationHandle2, flowExecutorCoordinator ).apply { verify(subscriptionRegistrationHandle).close() - verify(multiSourceEventMediator).close() + verify(subscription).close() verify(flowExecutorCoordinator).followStatusChangesByName(eq(setOf(name2))) - verify(multiSourceEventMediator2).start() + verify(subscription2).start() } } @@ -170,12 +175,13 @@ class FlowExecutorImplTest { private fun getFlowExecutor(): FlowExecutorImpl { return FlowExecutorImpl( coordinatorFactory, - flowEventMediatorFactory, + subscriptionFactory, + flowEventProcessorFactory, toMessagingConfig ) } - private fun getMinimalMessagingConfig(): SmartConfig { + private fun getMinimalMessagingConfig() : SmartConfig { return SmartConfigImpl.empty() .withValue(PROCESSOR_TIMEOUT, ConfigValueFactory.fromAnyRef(5000)) .withValue(MAX_ALLOWED_MSG_SIZE, ConfigValueFactory.fromAnyRef(1000000000)) diff --git a/libs/messaging/db-message-bus-impl/src/main/kotlin/net/corda/messagebus/db/consumer/DBCordaConsumerImpl.kt b/libs/messaging/db-message-bus-impl/src/main/kotlin/net/corda/messagebus/db/consumer/DBCordaConsumerImpl.kt index 462ab0990e2..937b1415199 100644 --- a/libs/messaging/db-message-bus-impl/src/main/kotlin/net/corda/messagebus/db/consumer/DBCordaConsumerImpl.kt +++ b/libs/messaging/db-message-bus-impl/src/main/kotlin/net/corda/messagebus/db/consumer/DBCordaConsumerImpl.kt @@ -203,18 +203,23 @@ internal class DBCordaConsumerImpl constructor( } } - override fun syncCommitOffsets() { - dbAccess.writeOffsets( - lastReadOffset.map { (cordaTopicPartition, offset) -> - CommittedPositionEntry( - cordaTopicPartition.topic, - groupId, - cordaTopicPartition.partition, - offset, - ATOMIC_TRANSACTION, - ) - } - ) + override fun asyncCommitOffsets(callback: CordaConsumer.Callback?) { + try { + dbAccess.writeOffsets( + lastReadOffset.map { (cordaTopicPartition, offset) -> + CommittedPositionEntry( + cordaTopicPartition.topic, + groupId, + cordaTopicPartition.partition, + offset, + ATOMIC_TRANSACTION, + ) + } + ) + callback?.onCompletion(lastReadOffset, null) + } catch(e: Exception) { + callback?.onCompletion(emptyMap(), e) + } } override fun syncCommitOffsets(event: CordaConsumerRecord, metaData: String?) { diff --git a/libs/messaging/kafka-message-bus-impl/src/main/kotlin/net/corda/messagebus/kafka/consumer/CordaKafkaConsumerImpl.kt b/libs/messaging/kafka-message-bus-impl/src/main/kotlin/net/corda/messagebus/kafka/consumer/CordaKafkaConsumerImpl.kt index 1432c61e4c2..7ccfc2e56a7 100644 --- a/libs/messaging/kafka-message-bus-impl/src/main/kotlin/net/corda/messagebus/kafka/consumer/CordaKafkaConsumerImpl.kt +++ b/libs/messaging/kafka-message-bus-impl/src/main/kotlin/net/corda/messagebus/kafka/consumer/CordaKafkaConsumerImpl.kt @@ -304,31 +304,14 @@ class CordaKafkaConsumerImpl( } } - override fun syncCommitOffsets() { - var attemptCommit = true - - while (attemptCommit) { - try { - consumer.commitSync() - attemptCommit = false - } catch (ex: Exception) { - when (ex::class.java) { - in fatalExceptions -> { - logErrorAndThrowFatalException( - "Error attempting to commitSync offsets.", - ex - ) - } - in transientExceptions -> { - logWarningAndThrowIntermittentException("Failed to commitSync offsets.", ex) - } - else -> { - logErrorAndThrowFatalException( - "Unexpected error attempting to commitSync offsets .", ex - ) - } - } - } + override fun asyncCommitOffsets(callback: CordaConsumer.Callback?) { + consumer.commitAsync { offsets, exception -> + callback?.onCompletion( + offsets.entries.associate { + it.key!!.toCordaTopicPartition(config.topicPrefix) to it.value.offset() + }, + exception + ) } } diff --git a/libs/messaging/kafka-message-bus-impl/src/test/kotlin/net/corda/messagebus/kafka/consumer/CordaKafkaConsumerImplTest.kt b/libs/messaging/kafka-message-bus-impl/src/test/kotlin/net/corda/messagebus/kafka/consumer/CordaKafkaConsumerImplTest.kt index 17160d98864..84a4563bbc0 100644 --- a/libs/messaging/kafka-message-bus-impl/src/test/kotlin/net/corda/messagebus/kafka/consumer/CordaKafkaConsumerImplTest.kt +++ b/libs/messaging/kafka-message-bus-impl/src/test/kotlin/net/corda/messagebus/kafka/consumer/CordaKafkaConsumerImplTest.kt @@ -4,6 +4,7 @@ import io.micrometer.core.instrument.binder.kafka.KafkaClientMetrics import net.corda.data.chunking.Chunk import net.corda.data.chunking.ChunkKey import net.corda.messagebus.api.CordaTopicPartition +import net.corda.messagebus.api.consumer.CordaConsumer import net.corda.messagebus.api.consumer.CordaConsumerRebalanceListener import net.corda.messagebus.api.consumer.CordaConsumerRecord import net.corda.messagebus.api.consumer.CordaOffsetResetStrategy @@ -24,6 +25,7 @@ import org.apache.kafka.clients.consumer.Consumer import org.apache.kafka.clients.consumer.ConsumerRebalanceListener import org.apache.kafka.clients.consumer.MockConsumer import org.apache.kafka.clients.consumer.OffsetAndMetadata +import org.apache.kafka.clients.consumer.OffsetCommitCallback import org.apache.kafka.common.KafkaException import org.apache.kafka.common.TopicPartition import org.apache.kafka.common.errors.AuthenticationException @@ -43,6 +45,7 @@ import org.mockito.kotlin.anyOrNull import org.mockito.kotlin.doAnswer import org.mockito.kotlin.doReturn import org.mockito.kotlin.doThrow +import org.mockito.kotlin.eq import org.mockito.kotlin.mock import org.mockito.kotlin.times import org.mockito.kotlin.verify @@ -177,26 +180,33 @@ class CordaKafkaConsumerImplTest { } @Test - fun testSyncCommitOffsets() { + fun testAsyncCommitOffsets() { + val callback = mock() assertThat(consumer.committed(setOf(partition))).isEmpty() cordaKafkaConsumer.poll(Duration.ZERO) - cordaKafkaConsumer.syncCommitOffsets() + cordaKafkaConsumer.asyncCommitOffsets(callback) val committedPositionAfterPoll = consumer.committed(setOf(partition)) assertThat(committedPositionAfterPoll.values.first().offset()).isEqualTo(numberOfRecords) } @Test - fun testSyncCommitOffsetsException() { + fun testAsyncCommitOffsetsException() { consumer = mock() cordaKafkaConsumer = createConsumer(consumer) + val exception = CommitFailedException() + doAnswer { + val callback = it.arguments[0] as OffsetCommitCallback + callback.onComplete(mock(), exception) + null + }.whenever(consumer).commitAsync(any()) + val callback = mock() - doThrow(CommitFailedException()).whenever(consumer).commitSync() - assertThatExceptionOfType(CordaMessageAPIFatalException::class.java).isThrownBy { - cordaKafkaConsumer.syncCommitOffsets() - } - verify(consumer, times(1)).commitSync() + cordaKafkaConsumer.asyncCommitOffsets(callback) + + verify(consumer, times(1)).commitAsync(any()) + verify(callback, times(1)).onCompletion(any(), eq(exception)) } @Test diff --git a/libs/messaging/message-bus/src/main/kotlin/net/corda/messagebus/api/consumer/CordaConsumer.kt b/libs/messaging/message-bus/src/main/kotlin/net/corda/messagebus/api/consumer/CordaConsumer.kt index 3a686ea9723..2269483455b 100644 --- a/libs/messaging/message-bus/src/main/kotlin/net/corda/messagebus/api/consumer/CordaConsumer.kt +++ b/libs/messaging/message-bus/src/main/kotlin/net/corda/messagebus/api/consumer/CordaConsumer.kt @@ -146,10 +146,10 @@ interface CordaConsumer : AutoCloseable { fun resetToLastCommittedPositions(offsetStrategy: CordaOffsetResetStrategy) /** - * Synchronously commit the consumer offsets. + * Asynchronously commit the consumer offsets. * @throws CordaMessageAPIFatalException fatal error occurred attempting to commit offsets. */ - fun syncCommitOffsets() + fun asyncCommitOffsets(callback: Callback?) /** * Synchronously commit the consumer offset for this [event] back to the topic partition. diff --git a/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/MessageBusConsumer.kt b/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/MessageBusConsumer.kt index 4176b4f5e85..04b4258ed70 100644 --- a/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/MessageBusConsumer.kt +++ b/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/MessageBusConsumer.kt @@ -1,5 +1,8 @@ package net.corda.messaging.mediator +import kotlinx.coroutines.CompletableDeferred +import kotlinx.coroutines.Deferred +import net.corda.messagebus.api.CordaTopicPartition import net.corda.messagebus.api.consumer.CordaConsumer import net.corda.messagebus.api.consumer.CordaConsumerRecord import net.corda.messagebus.api.consumer.CordaOffsetResetStrategy @@ -13,14 +16,33 @@ class MessageBusConsumer( private val topic: String, private val consumer: CordaConsumer, ): MediatorConsumer { - override fun subscribe() = consumer.subscribe(topic) - override fun poll(timeout: Duration): List> = consumer.poll(timeout) + override fun subscribe() = + consumer.subscribe(topic) - override fun syncCommitOffsets() = consumer.syncCommitOffsets() + override fun poll(timeout: Duration): Deferred>> = + CompletableDeferred>>().apply { + try { + complete(consumer.poll(timeout)) + } catch (throwable: Throwable) { + completeExceptionally(throwable) + } + } + + override fun asyncCommitOffsets(): Deferred> = + CompletableDeferred>().apply { + consumer.asyncCommitOffsets { offsets, exception -> + if (exception != null) { + completeExceptionally(exception) + } else { + complete(offsets) + } + } + } override fun resetEventOffsetPosition() = consumer.resetToLastCommittedPositions(CordaOffsetResetStrategy.EARLIEST) - override fun close() = consumer.close() + override fun close() = + consumer.close() } \ No newline at end of file diff --git a/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/MultiSourceEventMediatorImpl.kt b/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/MultiSourceEventMediatorImpl.kt index 0262d264f98..32023c08925 100644 --- a/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/MultiSourceEventMediatorImpl.kt +++ b/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/MultiSourceEventMediatorImpl.kt @@ -1,5 +1,6 @@ package net.corda.messaging.mediator +import kotlinx.coroutines.runBlocking import net.corda.avro.serialization.CordaAvroDeserializer import net.corda.avro.serialization.CordaAvroSerializer import net.corda.libs.statemanager.api.StateManager @@ -150,6 +151,7 @@ class MultiSourceEventMediatorImpl( } private fun processEvents() { + log.debug { "Polling and processing events" } val messages = pollConsumers() if (messages.isNotEmpty()) { val msgGroups = messages.groupBy { it.key } @@ -175,13 +177,23 @@ class MultiSourceEventMediatorImpl( private fun pollConsumers(): List> { return consumers.map { consumer -> - consumer.poll(config.pollTimeout) + taskManager.execute(TaskType.SHORT_RUNNING) { + runBlocking { + consumer.poll(config.pollTimeout).await() + } + } + }.map { + it.join() }.flatten() } private fun commitOffsets() { consumers.map { consumer -> - consumer.syncCommitOffsets() + taskManager.execute(TaskType.SHORT_RUNNING) { + runBlocking { + consumer.asyncCommitOffsets().await() + } + } } } diff --git a/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/taskmanager/TaskManagerImpl.kt b/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/taskmanager/TaskManagerImpl.kt index 305bfb0e752..dec4a07876a 100644 --- a/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/taskmanager/TaskManagerImpl.kt +++ b/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/taskmanager/TaskManagerImpl.kt @@ -4,7 +4,6 @@ import net.corda.messaging.api.mediator.taskmanager.TaskManager import net.corda.messaging.api.mediator.taskmanager.TaskType import org.osgi.service.component.annotations.Activate import org.osgi.service.component.annotations.Component -import org.slf4j.LoggerFactory import java.util.UUID import java.util.concurrent.CompletableFuture import java.util.concurrent.Executors @@ -13,9 +12,6 @@ import kotlin.concurrent.thread // TODO This is used temporarily until Task Manager implementation is finished @Component(service = [TaskManager::class]) class TaskManagerImpl @Activate constructor() : TaskManager { - companion object { - private val log = LoggerFactory.getLogger(this::class.java.enclosingClass) - } private var executorService = Executors.newSingleThreadExecutor() override fun execute(type: TaskType, command: () -> T) = @@ -25,20 +21,11 @@ class TaskManagerImpl @Activate constructor() : TaskManager { } private fun executeShortRunning(command: () -> T): CompletableFuture { - val resultFuture = CompletableFuture() - try { - executorService.execute { - try { - resultFuture.complete(command()) - } catch (t: Throwable) { - log.error("Task error", t) - resultFuture.completeExceptionally(t) - } - } - } catch (t: Throwable) { - log.error("Executor error", t) + val result = CompletableFuture() + executorService.execute { + result.complete(command()) } - return resultFuture + return result } private fun executeLongRunning(command: () -> T): CompletableFuture { diff --git a/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/MessageBusConsumerTest.kt b/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/MessageBusConsumerTest.kt index 20f4c5e45b0..d4d2a466cf4 100644 --- a/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/MessageBusConsumerTest.kt +++ b/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/MessageBusConsumerTest.kt @@ -1,5 +1,6 @@ package net.corda.messaging.mediator +import kotlinx.coroutines.runBlocking import net.corda.messagebus.api.consumer.CordaConsumer import net.corda.v5.base.exceptions.CordaRuntimeException import org.junit.jupiter.api.BeforeEach @@ -50,23 +51,30 @@ class MessageBusConsumerTest { doThrow(CordaRuntimeException("")).whenever(cordaConsumer).poll(any()) assertThrows { - mediatorConsumer.poll(timeout) + runBlocking { + mediatorConsumer.poll(timeout).await() + } } } @Test - fun testSyncCommitOffsets() { - mediatorConsumer.syncCommitOffsets() + fun testCommitAsyncOffsets() { + mediatorConsumer.asyncCommitOffsets() - verify(cordaConsumer).syncCommitOffsets() + verify(cordaConsumer).asyncCommitOffsets(any()) } @Test - fun testSyncCommitOffsetsWithError() { - doThrow(CordaRuntimeException("")).whenever(cordaConsumer).syncCommitOffsets() + fun testCommitAsyncOffsetsWithError() { + whenever(cordaConsumer.asyncCommitOffsets(any())).thenAnswer { invocation -> + val callback = invocation.getArgument(0) + callback.onCompletion(mock(), CordaRuntimeException("")) + } assertThrows { - mediatorConsumer.syncCommitOffsets() + runBlocking { + mediatorConsumer.asyncCommitOffsets().await() + } } } diff --git a/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/MultiSourceEventMediatorImplTest.kt b/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/MultiSourceEventMediatorImplTest.kt index e676375909a..48f558cb4ff 100644 --- a/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/MultiSourceEventMediatorImplTest.kt +++ b/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/MultiSourceEventMediatorImplTest.kt @@ -5,6 +5,7 @@ import net.corda.avro.serialization.CordaAvroDeserializer import net.corda.avro.serialization.CordaAvroSerializer import net.corda.libs.statemanager.api.StateManager import net.corda.lifecycle.LifecycleCoordinatorFactory +import net.corda.messagebus.api.CordaTopicPartition import net.corda.messagebus.api.consumer.CordaConsumerRecord import net.corda.messaging.api.mediator.MediatorConsumer import net.corda.messaging.api.mediator.MediatorMessage @@ -25,6 +26,7 @@ import net.corda.messaging.api.processor.StateAndEventProcessor import net.corda.messaging.api.records.Record import net.corda.test.util.waitWhile import org.junit.jupiter.api.BeforeEach +// import org.junit.jupiter.api.Test import org.mockito.kotlin.any import org.mockito.kotlin.anyOrNull import org.mockito.kotlin.atLeast @@ -117,7 +119,7 @@ class MultiSourceEventMediatorImplTest { ) } - // @Test + //@Test // TODO Test temporarily disabled as it seems to be flaky fun `mediator processes multiples events by key`() { val events = (1..6).map { "event$it" } @@ -134,13 +136,18 @@ class MultiSourceEventMediatorImplTest { ), ) var batchNumber = 0 + whenever(consumer.asyncCommitOffsets()).thenAnswer { + CompletableDeferred(mock>()) + } whenever(consumer.poll(any())).thenAnswer { - if (batchNumber < eventBatches.size) { - eventBatches[batchNumber++] - } else { - Thread.sleep(10) - emptyList() - } + CompletableDeferred( + if (batchNumber < eventBatches.size) { + eventBatches[batchNumber++] + } else { + Thread.sleep(10) + emptyList() + } + ) } mediator.start() @@ -154,7 +161,7 @@ class MultiSourceEventMediatorImplTest { verify(stateManager, times(eventBatches.size)).get(any()) verify(stateManager, times(eventBatches.size)).create(any()) verify(consumer, atLeast(eventBatches.size)).poll(any()) - verify(consumer, times(eventBatches.size)).syncCommitOffsets() + verify(consumer, times(eventBatches.size)).asyncCommitOffsets() verify(messagingClient, times(events.size)).send(any()) } diff --git a/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/TaskManagerHelperTest.kt b/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/TaskManagerHelperTest.kt index 5a664098766..1fbc942b174 100644 --- a/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/TaskManagerHelperTest.kt +++ b/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/TaskManagerHelperTest.kt @@ -7,13 +7,13 @@ import net.corda.messaging.api.mediator.MessageRouter import net.corda.messaging.api.mediator.MessagingClient.Companion.MSG_PROP_KEY import net.corda.messaging.api.mediator.taskmanager.TaskManager import net.corda.messaging.api.processor.StateAndEventProcessor -import net.corda.messaging.api.records.Record import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Test +import org.mockito.kotlin.mock +import net.corda.messaging.api.records.Record import org.mockito.Mockito.`when` import org.mockito.kotlin.any import org.mockito.kotlin.argumentCaptor -import org.mockito.kotlin.mock import org.mockito.kotlin.times import org.mockito.kotlin.verify diff --git a/libs/messaging/messaging/src/main/kotlin/net/corda/messaging/api/mediator/MediatorConsumer.kt b/libs/messaging/messaging/src/main/kotlin/net/corda/messaging/api/mediator/MediatorConsumer.kt index 6e2036891fd..79e53453d2a 100644 --- a/libs/messaging/messaging/src/main/kotlin/net/corda/messaging/api/mediator/MediatorConsumer.kt +++ b/libs/messaging/messaging/src/main/kotlin/net/corda/messaging/api/mediator/MediatorConsumer.kt @@ -1,5 +1,7 @@ package net.corda.messaging.api.mediator +import kotlinx.coroutines.Deferred +import net.corda.messagebus.api.CordaTopicPartition import net.corda.messagebus.api.consumer.CordaConsumerRecord import java.time.Duration @@ -18,12 +20,14 @@ interface MediatorConsumer : AutoCloseable { * * @param timeout - The maximum time to block if there are no available messages. */ - fun poll(timeout: Duration): List> + fun poll(timeout: Duration): Deferred>> /** - * Synchronously commits the consumer offsets. This function should be called only after `poll` was called. + * Asynchronously commit the consumer offsets. This function should be called only after `poll` was called. + * + * @return [Deferred] with committed offsets. */ - fun syncCommitOffsets() + fun asyncCommitOffsets(): Deferred> /** * Resets consumer's offsets to the last committed positions. Next poll will read from the last committed positions. From b97c4d09cb5fec69faadf2ef82705e12a5ba89d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Jos=C3=A9=20Ramos?= Date: Thu, 12 Oct 2023 17:43:33 +0100 Subject: [PATCH 05/81] CORE-16323: Fix State Manager Optimistic Locking (#4846) Fix update and delete SQL statements within the State Manager implementation so only the states with the specified key and version are updated. This prevents race conditions while executing the optimistic locking and also reduces by half the amount of SQL statements that needs be executed for every update and delete. --- .../libs/statemanager/impl/ObjectMapperExt.kt | 7 + .../statemanager/impl/StateManagerImpl.kt | 64 ++++------ .../impl/repository/StateRepository.kt | 10 +- .../repository/impl/AbstractQueryProvider.kt | 2 +- .../repository/impl/PostgresQueryProvider.kt | 2 +- .../repository/impl/StateRepositoryImpl.kt | 52 +++++--- .../statemanager/impl/ObjectMapperExtTest.kt | 33 +++++ .../statemanager/impl/StateManagerImplTest.kt | 120 +++++------------- 8 files changed, 136 insertions(+), 154 deletions(-) create mode 100644 libs/state-manager/state-manager-db-impl/src/main/kotlin/net/corda/libs/statemanager/impl/ObjectMapperExt.kt create mode 100644 libs/state-manager/state-manager-db-impl/src/test/kotlin/net/corda/libs/statemanager/impl/ObjectMapperExtTest.kt diff --git a/libs/state-manager/state-manager-db-impl/src/main/kotlin/net/corda/libs/statemanager/impl/ObjectMapperExt.kt b/libs/state-manager/state-manager-db-impl/src/main/kotlin/net/corda/libs/statemanager/impl/ObjectMapperExt.kt new file mode 100644 index 00000000000..d25488d5797 --- /dev/null +++ b/libs/state-manager/state-manager-db-impl/src/main/kotlin/net/corda/libs/statemanager/impl/ObjectMapperExt.kt @@ -0,0 +1,7 @@ +package net.corda.libs.statemanager.impl + +import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.module.kotlin.readValue +import net.corda.libs.statemanager.api.Metadata + +fun ObjectMapper.convertToMetadata(json: String) = Metadata(readValue(json)) diff --git a/libs/state-manager/state-manager-db-impl/src/main/kotlin/net/corda/libs/statemanager/impl/StateManagerImpl.kt b/libs/state-manager/state-manager-db-impl/src/main/kotlin/net/corda/libs/statemanager/impl/StateManagerImpl.kt index 1f535e9011f..7265b76d62e 100644 --- a/libs/state-manager/state-manager-db-impl/src/main/kotlin/net/corda/libs/statemanager/impl/StateManagerImpl.kt +++ b/libs/state-manager/state-manager-db-impl/src/main/kotlin/net/corda/libs/statemanager/impl/StateManagerImpl.kt @@ -1,9 +1,7 @@ package net.corda.libs.statemanager.impl import com.fasterxml.jackson.databind.ObjectMapper -import com.fasterxml.jackson.module.kotlin.readValue import net.corda.libs.statemanager.api.IntervalFilter -import net.corda.libs.statemanager.api.Metadata import net.corda.libs.statemanager.api.MetadataFilter import net.corda.libs.statemanager.api.State import net.corda.libs.statemanager.api.StateManager @@ -11,11 +9,9 @@ import net.corda.libs.statemanager.impl.model.v1.StateEntity import net.corda.libs.statemanager.impl.repository.StateRepository import net.corda.orm.utils.transaction import org.slf4j.LoggerFactory -import javax.persistence.EntityManager import javax.persistence.EntityManagerFactory // TODO-[CORE-17025]: remove Hibernate. -// TODO-[CORE-16323]: check whether the optimistic locking can be improved / merged into single SQL statement. class StateManagerImpl( private val stateRepository: StateRepository, private val entityManagerFactory: EntityManagerFactory, @@ -32,29 +28,6 @@ class StateManagerImpl( private fun StateEntity.fromPersistentEntity() = State(key, value, version, objectMapper.convertToMetadata(metadata), modifiedTime) - internal fun checkVersionAndPrepareEntitiesForPersistence( - states: Collection, - entityManager: EntityManager - ): Pair, Map> { - val matchedVersions = mutableListOf() - val unmatchedVersions = mutableMapOf() - val persistedStates = stateRepository.get(entityManager, states.map { it.key }) - - states.forEach { st -> - val persisted = persistedStates.find { it.key == st.key } - - persisted?.let { - if (st.version == persisted.version) { - matchedVersions.add(st.toPersistentEntity()) - } else { - unmatchedVersions[it.key] = it.fromPersistentEntity() - } - } - } - - return Pair(matchedVersions, unmatchedVersions) - } - override fun create(states: Collection): Map { val failures = mutableMapOf() @@ -85,28 +58,38 @@ class StateManagerImpl( } override fun update(states: Collection): Map { - entityManagerFactory.transaction { em -> - val (updatable, mismatchVersions) = checkVersionAndPrepareEntitiesForPersistence(states, em) - stateRepository.update(em, updatable) + try { + entityManagerFactory.transaction { em -> + val failedUpdates = stateRepository.update(em, states.map { it.toPersistentEntity() }) - return mismatchVersions.also { - if (it.isNotEmpty()) { - logger.info("Optimistic locking check failed for States ${it.entries.joinToString()}") + return if (failedUpdates.isEmpty()) { + emptyMap() + } else { + logger.warn("Optimistic locking check failed while updating States ${failedUpdates.joinToString()}") + get(failedUpdates) } } + } catch (e: Exception) { + logger.warn("Failed to updated batch of states - ${states.joinToString { it.key }}", e) + throw e } } override fun delete(states: Collection): Map { - entityManagerFactory.transaction { em -> - val (deletable, mismatchVersions) = checkVersionAndPrepareEntitiesForPersistence(states, em) - stateRepository.delete(em, deletable.map { it.key }) + try { + entityManagerFactory.transaction { em -> + val failedDeletes = stateRepository.delete(em, states.map { it.toPersistentEntity() }) - return mismatchVersions.also { - if (it.isNotEmpty()) { - logger.info("Optimistic locking check failed for States ${it.entries.joinToString()}") + return if (failedDeletes.isEmpty()) { + emptyMap() + } else { + logger.warn("Optimistic locking check failed while deleting States ${failedDeletes.joinToString()}") + get(failedDeletes) } } + } catch (e: Exception) { + logger.warn("Failed to delete batch of states - ${states.joinToString { it.key }}", e) + throw e } } @@ -155,6 +138,3 @@ class StateManagerImpl( entityManagerFactory.close() } } - -fun ObjectMapper.convertToMetadata(json: String) = - Metadata(this.readValue(json)) \ No newline at end of file diff --git a/libs/state-manager/state-manager-db-impl/src/main/kotlin/net/corda/libs/statemanager/impl/repository/StateRepository.kt b/libs/state-manager/state-manager-db-impl/src/main/kotlin/net/corda/libs/statemanager/impl/repository/StateRepository.kt index c0762996342..561c0aef064 100644 --- a/libs/state-manager/state-manager-db-impl/src/main/kotlin/net/corda/libs/statemanager/impl/repository/StateRepository.kt +++ b/libs/state-manager/state-manager-db-impl/src/main/kotlin/net/corda/libs/statemanager/impl/repository/StateRepository.kt @@ -35,19 +35,19 @@ interface StateRepository { * * @param entityManager Used to interact with the state manager persistence context. * @param states Collection of states to be updated. - * @return Collection of states found. + * @return Collection of keys for states that could not be updated due to optimistic locking check failure. */ - fun update(entityManager: EntityManager, states: Collection) + fun update(entityManager: EntityManager, states: Collection): Collection /** * Delete states with the given keys from the persistence context. * Transaction should be controlled by the caller. * * @param entityManager Used to interact with the state manager persistence context. - * @param keys Collection of states to delete. - * @return Collection of states found. + * @param states Collection of states to be deleted. + * @return Collection of keys for states that could not be deleted due to optimistic locking check failure. */ - fun delete(entityManager: EntityManager, keys: Collection) + fun delete(entityManager: EntityManager, states: Collection): Collection /** * Retrieve entities that were lastly updated between [IntervalFilter.start] and [IntervalFilter.finish]. diff --git a/libs/state-manager/state-manager-db-impl/src/main/kotlin/net/corda/libs/statemanager/impl/repository/impl/AbstractQueryProvider.kt b/libs/state-manager/state-manager-db-impl/src/main/kotlin/net/corda/libs/statemanager/impl/repository/impl/AbstractQueryProvider.kt index b5a2ac5eceb..244fa5ce42e 100644 --- a/libs/state-manager/state-manager-db-impl/src/main/kotlin/net/corda/libs/statemanager/impl/repository/impl/AbstractQueryProvider.kt +++ b/libs/state-manager/state-manager-db-impl/src/main/kotlin/net/corda/libs/statemanager/impl/repository/impl/AbstractQueryProvider.kt @@ -20,7 +20,7 @@ abstract class AbstractQueryProvider : QueryProvider { override val deleteStatesByKey: String get() = """ - DELETE FROM $STATE_MANAGER_TABLE s WHERE s.key IN (:$KEYS_PARAMETER_NAME) + DELETE FROM $STATE_MANAGER_TABLE s WHERE s.key = :$KEY_PARAMETER_NAME AND s.version = :$VERSION_PARAMETER_NAME """.trimIndent() override val findStatesUpdatedBetween: String diff --git a/libs/state-manager/state-manager-db-impl/src/main/kotlin/net/corda/libs/statemanager/impl/repository/impl/PostgresQueryProvider.kt b/libs/state-manager/state-manager-db-impl/src/main/kotlin/net/corda/libs/statemanager/impl/repository/impl/PostgresQueryProvider.kt index c0475700da1..ee739bdeae0 100644 --- a/libs/state-manager/state-manager-db-impl/src/main/kotlin/net/corda/libs/statemanager/impl/repository/impl/PostgresQueryProvider.kt +++ b/libs/state-manager/state-manager-db-impl/src/main/kotlin/net/corda/libs/statemanager/impl/repository/impl/PostgresQueryProvider.kt @@ -16,7 +16,7 @@ class PostgresQueryProvider : AbstractQueryProvider() { get() = """ UPDATE $STATE_MANAGER_TABLE SET key = :$KEY_PARAMETER_NAME, value = :$VALUE_PARAMETER_NAME, version = version + 1, metadata = CAST(:$METADATA_PARAMETER_NAME as JSONB), modified_time = CURRENT_TIMESTAMP AT TIME ZONE 'UTC' - WHERE key = :$KEY_PARAMETER_NAME + WHERE key = :$KEY_PARAMETER_NAME AND version = :$VERSION_PARAMETER_NAME """.trimIndent() override fun findStatesByMetadataMatchingAll(filters: Collection) = diff --git a/libs/state-manager/state-manager-db-impl/src/main/kotlin/net/corda/libs/statemanager/impl/repository/impl/StateRepositoryImpl.kt b/libs/state-manager/state-manager-db-impl/src/main/kotlin/net/corda/libs/statemanager/impl/repository/impl/StateRepositoryImpl.kt index 84a79d41804..58dda1384d6 100644 --- a/libs/state-manager/state-manager-db-impl/src/main/kotlin/net/corda/libs/statemanager/impl/repository/impl/StateRepositoryImpl.kt +++ b/libs/state-manager/state-manager-db-impl/src/main/kotlin/net/corda/libs/statemanager/impl/repository/impl/StateRepositoryImpl.kt @@ -8,6 +8,7 @@ import org.slf4j.LoggerFactory import javax.persistence.EntityManager import javax.persistence.Query +// TODO-[CORE-17733]: batch update and delete. class StateRepositoryImpl(private val queryProvider: QueryProvider) : StateRepository { private companion object { @@ -33,31 +34,42 @@ class StateRepositoryImpl(private val queryProvider: QueryProvider) : StateRepos .setParameter(KEYS_PARAMETER_NAME, keys) .resultListAsStateEntityCollection() - override fun update(entityManager: EntityManager, states: Collection) = - try { - states.forEach { - entityManager - .createNativeQuery(queryProvider.updateState) - .setParameter(KEY_PARAMETER_NAME, it.key) - .setParameter(VALUE_PARAMETER_NAME, it.value) - .setParameter(METADATA_PARAMETER_NAME, it.metadata) - .executeUpdate() - } - } catch (e: Exception) { - logger.warn("Failed to updated batch of states - ${states.joinToString { it.key }}", e) - throw e + override fun update(entityManager: EntityManager, states: Collection): Collection { + val failedKeys = mutableListOf() + + states.forEach { state -> + entityManager + .createNativeQuery(queryProvider.updateState) + .setParameter(KEY_PARAMETER_NAME, state.key) + .setParameter(VALUE_PARAMETER_NAME, state.value) + .setParameter(VERSION_PARAMETER_NAME, state.version) + .setParameter(METADATA_PARAMETER_NAME, state.metadata) + .executeUpdate().also { + if (it == 0) { + failedKeys.add(state.key) + } + } } - override fun delete(entityManager: EntityManager, keys: Collection) { - try { + return failedKeys + } + + override fun delete(entityManager: EntityManager, states: Collection): Collection { + val failedKeys = mutableListOf() + + states.forEach { state -> entityManager .createNativeQuery(queryProvider.deleteStatesByKey) - .setParameter(KEYS_PARAMETER_NAME, keys) - .executeUpdate() - } catch (e: Exception) { - logger.warn("Failed to delete batch of states - ${keys.joinToString()}", e) - throw e + .setParameter(KEY_PARAMETER_NAME, state.key) + .setParameter(VERSION_PARAMETER_NAME, state.version) + .executeUpdate().also { + if (it == 0) { + failedKeys.add(state.key) + } + } } + + return failedKeys } override fun updatedBetween(entityManager: EntityManager, interval: IntervalFilter): Collection = diff --git a/libs/state-manager/state-manager-db-impl/src/test/kotlin/net/corda/libs/statemanager/impl/ObjectMapperExtTest.kt b/libs/state-manager/state-manager-db-impl/src/test/kotlin/net/corda/libs/statemanager/impl/ObjectMapperExtTest.kt new file mode 100644 index 00000000000..673e742c84d --- /dev/null +++ b/libs/state-manager/state-manager-db-impl/src/test/kotlin/net/corda/libs/statemanager/impl/ObjectMapperExtTest.kt @@ -0,0 +1,33 @@ +package net.corda.libs.statemanager.impl + +import com.fasterxml.jackson.databind.ObjectMapper +import org.assertj.core.api.Assertions.assertThat +import org.assertj.core.api.Assertions.entry +import org.junit.jupiter.api.Test + +class ObjectMapperExtTest { + + @Test + fun convertToMetadataWorksCorrectlyForEmptyJson() { + val metadataJson = "{}" + + val metadata = ObjectMapper().convertToMetadata(metadataJson) + assertThat(metadata).isEmpty() + } + + @Test + fun convertToMetadataWorksCorrectlyForNonEmptyJson() { + val metadataJson = """ + { + "foo": "bar", + "hello": 123 + } + """.trimIndent() + + val metadata = ObjectMapper().convertToMetadata(metadataJson) + assertThat(metadata).containsExactly( + entry("foo", "bar"), + entry("hello", 123) + ) + } +} diff --git a/libs/state-manager/state-manager-db-impl/src/test/kotlin/net/corda/libs/statemanager/impl/StateManagerImplTest.kt b/libs/state-manager/state-manager-db-impl/src/test/kotlin/net/corda/libs/statemanager/impl/StateManagerImplTest.kt index a089b04f655..7bbb0690b4f 100644 --- a/libs/state-manager/state-manager-db-impl/src/test/kotlin/net/corda/libs/statemanager/impl/StateManagerImplTest.kt +++ b/libs/state-manager/state-manager-db-impl/src/test/kotlin/net/corda/libs/statemanager/impl/StateManagerImplTest.kt @@ -1,6 +1,5 @@ package net.corda.libs.statemanager.impl -import com.fasterxml.jackson.databind.ObjectMapper import net.corda.libs.statemanager.api.State import net.corda.libs.statemanager.api.metadata import net.corda.libs.statemanager.impl.model.v1.StateEntity @@ -47,49 +46,6 @@ class StateManagerImplTest { private fun StateEntity.newVersion() = StateEntity(key, value, metadata, version + 1, modifiedTime) - @Test - fun checkVersionAndPrepareEntitiesForPersistenceReturnsEmptyUnmatchedVersionsWhenAllEntitiesHaveCorrectVersionSet() { - whenever(stateRepository.get(any(), any())).thenReturn(listOf(persistentStateOne, persistentStateTwo)) - - val result = - stateManager.checkVersionAndPrepareEntitiesForPersistence(listOf(apiStateOne, apiStateTwo), entityManager) - assertThat(result.first).containsExactly(persistentStateOne, persistentStateTwo) - assertThat(result.second).isEmpty() - } - - @Test - fun checkVersionAndPrepareEntitiesForPersistenceReturnsEmptyMatchedVersionsWhenAllEntitiesHaveIncorrectVersionSet() { - val moreUpToDateStateOne = persistentStateOne.newVersion() - val moreUpToDateStateTwo = persistentStateTwo.newVersion() - whenever(stateRepository.get(any(), any())).thenReturn(listOf(moreUpToDateStateOne, moreUpToDateStateTwo)) - - val result = - stateManager.checkVersionAndPrepareEntitiesForPersistence(listOf(apiStateOne, apiStateTwo), entityManager) - assertThat(result.first).isEmpty() - assertThat(result.second).containsExactlyInAnyOrderEntriesOf( - mutableMapOf( - apiStateOne.key to moreUpToDateStateOne.toState(), - apiStateTwo.key to moreUpToDateStateTwo.toState(), - ) - ) - } - - @Test - fun checkVersionAndPrepareEntitiesForPersistenceReturnsCorrectlyWhenSomeVersionsMatchAndSomeVersionsDoNotMatch() { - val moreUpToDateStateTwo = persistentStateTwo.newVersion() - whenever(stateRepository.get(any(), any())).thenReturn(listOf(persistentStateOne, moreUpToDateStateTwo)) - val result = stateManager.checkVersionAndPrepareEntitiesForPersistence( - listOf(apiStateOne, apiStateTwo, apiStateThree), entityManager - ) - - assertThat(result.first).containsExactly(persistentStateOne) - assertThat(result.second).containsExactlyInAnyOrderEntriesOf( - mutableMapOf( - apiStateTwo.key to moreUpToDateStateTwo.toState(), - ) - ) - } - @Test fun createReturnsEmptyMapWhenAllInsertsSucceed() { assertThat(stateManager.create(listOf(apiStateOne, apiStateTwo))).isEmpty() @@ -109,54 +65,48 @@ class StateManagerImplTest { } @Test - fun updateOnlyPersistsStatesWithMatchingVersions() { - val moreUpToDateStateTwo = persistentStateTwo.newVersion() - val persistentStateFour = StateEntity("key4", "state4".toByteArray(), "{}", 4, Instant.now()) - val apiStateFour = persistentStateFour.toState() - whenever(stateRepository.get(any(), any())).thenReturn( - listOf(persistentStateOne, moreUpToDateStateTwo, persistentStateFour) - ) - - val result = stateManager.update(listOf(apiStateOne, apiStateTwo, apiStateThree, apiStateFour)) - assertThat(result).containsExactly(entry(apiStateTwo.key, moreUpToDateStateTwo.toState())) - verify(stateRepository).get( - entityManager, - listOf(apiStateOne.key, apiStateTwo.key, apiStateThree.key, apiStateFour.key) - ) - verify(stateRepository).update(entityManager, listOf(persistentStateOne, persistentStateFour)) + fun updateReturnsEmptyMapWhenOptimisticLockingCheckSucceedsForAllStates() { + whenever(stateRepository.update(any(), any())).thenReturn(emptyList()) + + val result = stateManager.update(listOf(apiStateOne, apiStateTwo, apiStateThree)) + assertThat(result).isEmpty() + verify(stateRepository).update(entityManager, listOf(persistentStateOne, persistentStateTwo, persistentStateThree)) + verifyNoMoreInteractions(stateRepository) + } + + @Test + fun updateReturnsLatestPersistedViewForStatesThatFailedOptimisticLockingCheck() { + val persistedStateTwo = persistentStateTwo.newVersion() + whenever(stateRepository.get(any(), any())).thenReturn(listOf(persistedStateTwo)) + whenever(stateRepository.update(any(), any())).thenReturn(listOf(persistedStateTwo.key)) + + val result = stateManager.update(listOf(apiStateOne, apiStateTwo, apiStateThree)) + assertThat(result).containsExactly(entry(persistedStateTwo.key, persistedStateTwo.toState())) + verify(stateRepository).get(entityManager, listOf(apiStateTwo.key)) + verify(stateRepository).update(entityManager, listOf(persistentStateOne, persistentStateTwo, persistentStateThree)) verifyNoMoreInteractions(stateRepository) } @Test - fun deleteOnlyPersistsStatesWithMatchingVersions() { - val moreUpToDateStateTwo = persistentStateTwo.newVersion() - val persistentStateFour = StateEntity("key4", "state4".toByteArray(), "{}", 4, Instant.now()) - val apiStateFour = persistentStateFour.toState() - whenever(stateRepository.get(any(), any())).thenReturn( - listOf(persistentStateOne, moreUpToDateStateTwo, persistentStateFour) - ) - - val result = stateManager.delete(listOf(apiStateOne, apiStateTwo, apiStateThree, apiStateFour)) - assertThat(result).containsExactly(entry(apiStateTwo.key, moreUpToDateStateTwo.toState())) - verify(stateRepository).get( - entityManager, - listOf(apiStateOne.key, apiStateTwo.key, apiStateThree.key, apiStateFour.key) - ) - verify(stateRepository).delete(entityManager, listOf(persistentStateOne.key, persistentStateFour.key)) + fun deleteReturnsEmptyMapWhenOptimisticLockingCheckSucceedsForAllStates() { + whenever(stateRepository.delete(any(), any())).thenReturn(emptyList()) + + val result = stateManager.delete(listOf(apiStateOne, apiStateTwo, apiStateThree)) + assertThat(result).isEmpty() + verify(stateRepository).delete(entityManager, listOf(persistentStateOne, persistentStateTwo, persistentStateThree)) verifyNoMoreInteractions(stateRepository) } @Test - fun convertJson() { - val str = """ - { - "foo": "bar", - "hello": 123 - } - """.trimIndent() - - val meta = ObjectMapper().convertToMetadata(str) - assertThat(meta["foo"]).isEqualTo("bar") - assertThat(meta["hello"]).isEqualTo(123) + fun deleteReturnsLatestPersistedViewForStatesThatFailedOptimisticLockingCheck() { + val persistedStateThree = persistentStateThree.newVersion() + whenever(stateRepository.get(any(), any())).thenReturn(listOf(persistedStateThree)) + whenever(stateRepository.delete(any(), any())).thenReturn(listOf(persistedStateThree.key)) + + val result = stateManager.delete(listOf(apiStateOne, apiStateTwo, apiStateThree)) + assertThat(result).containsExactly(entry(persistedStateThree.key, persistedStateThree.toState())) + verify(stateRepository).get(entityManager, listOf(apiStateThree.key)) + verify(stateRepository).delete(entityManager, listOf(persistentStateOne, persistentStateTwo, persistentStateThree)) + verifyNoMoreInteractions(stateRepository) } } From e43fb77a223e58df1ca7939dc2f06fd0f1e3987f Mon Sep 17 00:00:00 2001 From: Miljenko Brkic <97448832+mbrkic-r3@users.noreply.github.com> Date: Fri, 13 Oct 2023 08:10:12 +0100 Subject: [PATCH 06/81] CORE-16203 Removed coroutine usage from Multi-Source Event Mediator. (#4858) --- libs/messaging/messaging-impl/build.gradle | 1 - .../net/corda/messaging/mediator/ClientTask.kt | 9 +++------ .../corda/messaging/mediator/MessageBusClient.kt | 16 ++++------------ .../corda/messaging/mediator/ClientTaskTest.kt | 10 +--------- .../messaging/mediator/MessageBusClientTest.kt | 15 +++++---------- .../mediator/MultiSourceEventMediatorImplTest.kt | 4 +--- libs/messaging/messaging/build.gradle | 1 - .../messaging/api/mediator/MessagingClient.kt | 9 +++------ 8 files changed, 17 insertions(+), 48 deletions(-) diff --git a/libs/messaging/messaging-impl/build.gradle b/libs/messaging/messaging-impl/build.gradle index b4178896dcb..845cc242509 100644 --- a/libs/messaging/messaging-impl/build.gradle +++ b/libs/messaging/messaging-impl/build.gradle @@ -12,7 +12,6 @@ dependencies { implementation project(":libs:chunking:chunking-core") implementation project(":libs:crypto:cipher-suite") implementation project(":libs:crypto:crypto-core") - implementation project(path: ':libs:kotlin-coroutines', configuration: 'bundle') implementation project(":libs:lifecycle:lifecycle") implementation project(":libs:messaging:messaging") implementation project(":libs:messaging:message-bus") diff --git a/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/ClientTask.kt b/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/ClientTask.kt index c93138ee1a3..8182934b459 100644 --- a/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/ClientTask.kt +++ b/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/ClientTask.kt @@ -1,6 +1,5 @@ package net.corda.messaging.mediator -import kotlinx.coroutines.runBlocking import net.corda.messaging.api.mediator.MediatorMessage import net.corda.messaging.api.mediator.MessageRouter import net.corda.messaging.api.mediator.MessagingClient @@ -30,11 +29,9 @@ data class ClientTask( val destination = messageRouter.getDestination(message) @Suppress("UNCHECKED_CAST") - val reply = runBlocking { - with(destination) { - message.addProperty(MSG_PROP_ENDPOINT, endpoint) - client.send(message).await() as MediatorMessage? - } + val reply = with(destination) { + message.addProperty(MSG_PROP_ENDPOINT, endpoint) + client.send(message) as MediatorMessage? } return Result(this, reply) } diff --git a/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/MessageBusClient.kt b/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/MessageBusClient.kt index 66338f569c0..116ca1579dd 100644 --- a/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/MessageBusClient.kt +++ b/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/MessageBusClient.kt @@ -1,7 +1,5 @@ package net.corda.messaging.mediator -import kotlinx.coroutines.CompletableDeferred -import kotlinx.coroutines.Deferred import net.corda.messagebus.api.producer.CordaProducer import net.corda.messagebus.api.producer.CordaProducerRecord import net.corda.messaging.api.mediator.MediatorMessage @@ -20,16 +18,10 @@ class MessageBusClient( private val log: Logger = LoggerFactory.getLogger(this::class.java.enclosingClass) } - override fun send(message: MediatorMessage<*>): Deferred?> = - CompletableDeferred?>().apply { - producer.send(message.toCordaProducerRecord()) { ex -> - if (ex != null) { - completeExceptionally(ex) - } else { - complete(null) - } - } - } + override fun send(message: MediatorMessage<*>): MediatorMessage<*>? { + producer.send(message.toCordaProducerRecord(), null) + return null + } override fun close() { try { diff --git a/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/ClientTaskTest.kt b/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/ClientTaskTest.kt index d28ef8b4b45..d97beeffe6a 100644 --- a/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/ClientTaskTest.kt +++ b/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/ClientTaskTest.kt @@ -1,7 +1,5 @@ package net.corda.messaging.mediator -import kotlinx.coroutines.Deferred -import kotlinx.coroutines.runBlocking import net.corda.messaging.api.mediator.MediatorMessage import net.corda.messaging.api.mediator.MessageRouter import net.corda.messaging.api.mediator.MessagingClient @@ -25,7 +23,6 @@ class ClientTaskTest { private val messageRouter = mock() private val routingDestination = mock() private val messagingClient = mock() - private val clientDeferredReply = mock>>() private val clientReply = mock>() @BeforeEach @@ -37,13 +34,8 @@ class ClientTaskTest { messagingClient ) `when`(messagingClient.send(any())).thenReturn( - clientDeferredReply + clientReply ) - runBlocking { - `when`(clientDeferredReply.await()).thenReturn( - clientReply - ) - } } @Test diff --git a/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/MessageBusClientTest.kt b/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/MessageBusClientTest.kt index 09ae5ee8869..58559d25ba4 100644 --- a/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/MessageBusClientTest.kt +++ b/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/MessageBusClientTest.kt @@ -1,6 +1,5 @@ package net.corda.messaging.mediator -import kotlinx.coroutines.runBlocking import net.corda.messagebus.api.producer.CordaProducer import net.corda.messagebus.api.producer.CordaProducerRecord import net.corda.messaging.api.mediator.MediatorMessage @@ -9,9 +8,10 @@ import net.corda.v5.base.exceptions.CordaRuntimeException import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Test import org.junit.jupiter.api.assertThrows +import org.mockito.Mockito import org.mockito.Mockito.times -import org.mockito.kotlin.any import org.mockito.kotlin.eq +import org.mockito.kotlin.isNull import org.mockito.kotlin.mock import org.mockito.kotlin.verify import org.mockito.kotlin.whenever @@ -50,7 +50,7 @@ class MessageBusClientTest { messageProps.toHeaders(), ) - verify(cordaProducer).send(eq(expected), any()) + verify(cordaProducer).send(eq(expected), isNull()) } @Test @@ -62,14 +62,9 @@ class MessageBusClientTest { messageProps.toHeaders(), ) - whenever(cordaProducer.send(eq(record), any())).thenAnswer { invocation -> - val callback = invocation.getArgument(1) - callback.onCompletion(CordaRuntimeException("")) - } + Mockito.doThrow(CordaRuntimeException("")).whenever(cordaProducer).send(eq(record), isNull()) assertThrows { - runBlocking { - messageBusClient.send(message).await() - } + messageBusClient.send(message) } } diff --git a/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/MultiSourceEventMediatorImplTest.kt b/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/MultiSourceEventMediatorImplTest.kt index 61e1a2f28a3..6eedc2a5d59 100644 --- a/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/MultiSourceEventMediatorImplTest.kt +++ b/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/MultiSourceEventMediatorImplTest.kt @@ -1,13 +1,11 @@ package net.corda.messaging.mediator -import kotlinx.coroutines.CompletableDeferred import net.corda.avro.serialization.CordaAvroDeserializer import net.corda.avro.serialization.CordaAvroSerializer import net.corda.libs.statemanager.api.StateManager import net.corda.lifecycle.LifecycleCoordinatorFactory import net.corda.messagebus.api.consumer.CordaConsumerRecord import net.corda.messaging.api.mediator.MediatorConsumer -import net.corda.messaging.api.mediator.MediatorMessage import net.corda.messaging.api.mediator.MessageRouter import net.corda.messaging.api.mediator.MessagingClient import net.corda.messaging.api.mediator.MultiSourceEventMediator @@ -61,7 +59,7 @@ class MultiSourceEventMediatorImplTest { whenever(mediatorConsumerFactory.create(any>())).thenReturn(consumer) whenever(messagingClient.send(any())).thenAnswer { - CompletableDeferred(null as MediatorMessage?) + null } whenever(messagingClientFactory.create(any())).thenReturn(messagingClient) diff --git a/libs/messaging/messaging/build.gradle b/libs/messaging/messaging/build.gradle index 728525d2f30..5f5e9aa8330 100644 --- a/libs/messaging/messaging/build.gradle +++ b/libs/messaging/messaging/build.gradle @@ -16,7 +16,6 @@ dependencies { implementation "net.corda:corda-base" implementation "net.corda:corda-config-schema" implementation project(":libs:chunking:chunking-core") - implementation project(path: ':libs:kotlin-coroutines', configuration: 'bundle') implementation project(":libs:lifecycle:lifecycle") implementation project(":libs:messaging:message-bus") implementation project(":libs:configuration:configuration-core") diff --git a/libs/messaging/messaging/src/main/kotlin/net/corda/messaging/api/mediator/MessagingClient.kt b/libs/messaging/messaging/src/main/kotlin/net/corda/messaging/api/mediator/MessagingClient.kt index 71ea0f32f24..c148e4c6b4d 100644 --- a/libs/messaging/messaging/src/main/kotlin/net/corda/messaging/api/mediator/MessagingClient.kt +++ b/libs/messaging/messaging/src/main/kotlin/net/corda/messaging/api/mediator/MessagingClient.kt @@ -1,7 +1,5 @@ package net.corda.messaging.api.mediator -import kotlinx.coroutines.Deferred - /** * Multi-source event mediator messaging client. */ @@ -20,11 +18,10 @@ interface MessagingClient : AutoCloseable { val id: String /** - * Asynchronously sends a generic [MediatorMessage], and returns any result/error through a [Deferred] response. + * Sends a generic [MediatorMessage] and returns any result/error through a response. * * @param message The [MediatorMessage] to send. - * @return [Deferred] instance representing the asynchronous computation result, or null if the destination doesn't - * provide a response. + * @return Computation result, or null if the destination doesn't provide a response. * */ - fun send(message: MediatorMessage<*>): Deferred?> + fun send(message: MediatorMessage<*>): MediatorMessage<*>? } From a8c33f7b0beddf31eeb6e25da176e7d5fe370d2d Mon Sep 17 00:00:00 2001 From: David Currie Date: Fri, 13 Oct 2023 09:13:07 +0100 Subject: [PATCH 07/81] CORE-16246 Allow topic configuration overrides (#4857) This commit provides the ability to override topic configuration, including replica and partition counts, on the topic create CLI commands and via the Helm chart. The YAML file output by the topic create preview now includes partitions and replicas fields per topic. These default to the global parameters. These are then consumed by the topic create connect command so can be manually modified between preview and use. The topic create preview and topic create connect commands also both take an --overides/-o option, which is the optional path to a file of the same format as the preview file. This file's contents are merged to override those generated/passed in. The contents of this file can be provided as YAML under bootstrap.kafka.overrides. --- charts/corda-lib/templates/_bootstrap.tpl | 8 +- charts/corda/values.schema.json | 13 ++++ .../corda/cli/plugins/topicconfig/Create.kt | 60 ++++++++++++++- .../cli/plugins/topicconfig/CreateConnect.kt | 6 +- .../plugins/topicconfig/CreateConnectTest.kt | 74 ++++++++++++++++++- .../test/resources/override_topic_config.yaml | 16 ++++ .../src/test/resources/preview_config.yaml | 6 ++ .../short_generated_topic_config.yaml | 4 + 8 files changed, 177 insertions(+), 10 deletions(-) create mode 100644 tools/plugins/topic-config/src/test/resources/override_topic_config.yaml diff --git a/charts/corda-lib/templates/_bootstrap.tpl b/charts/corda-lib/templates/_bootstrap.tpl index a1b7ca0a87b..2684989c7dc 100644 --- a/charts/corda-lib/templates/_bootstrap.tpl +++ b/charts/corda-lib/templates/_bootstrap.tpl @@ -373,6 +373,9 @@ spec: {{- end }} '-r', '{{ .Values.bootstrap.kafka.replicas }}', '-p', '{{ .Values.bootstrap.kafka.partitions }}', + {{- if .Values.bootstrap.kafka.overrides }} + '-o', '/tmp/overrides.yaml', + {{- end }} 'connect', '-w', '{{ .Values.bootstrap.kafka.timeoutSeconds }}' ] @@ -408,7 +411,7 @@ spec: {{- end }} {{- end }} initContainers: - - name: create-trust-store + - name: setup image: {{ include "corda.bootstrapCliImage" . }} imagePullPolicy: {{ .Values.imagePullPolicy }} {{- include "corda.bootstrapResources" . | nindent 10 }} @@ -421,6 +424,9 @@ spec: - -c args: - | + {{- with .Values.bootstrap.kafka.overrides }} + echo -e {{ toYaml . | quote }} > /tmp/overrides.yaml + {{- end }} touch /tmp/config.properties {{- if .Values.kafka.tls.enabled }} {{- if .Values.kafka.sasl.enabled }} diff --git a/charts/corda/values.schema.json b/charts/corda/values.schema.json index 8a2b78438a8..69b8aa46db4 100644 --- a/charts/corda/values.schema.json +++ b/charts/corda/values.schema.json @@ -1442,6 +1442,19 @@ 3 ] }, + "overrides": { + "type": "object", + "default": null, + "title": "overrides for Kafka topic configuration", + "examples": [ + { + "topics": [{ + "name": "avro.schema", + "partitions": 5 + }] + } + ] + }, "timeoutSeconds": { "type": "integer", "default": 60, diff --git a/tools/plugins/topic-config/src/main/kotlin/net/corda/cli/plugins/topicconfig/Create.kt b/tools/plugins/topic-config/src/main/kotlin/net/corda/cli/plugins/topicconfig/Create.kt index a4beed21a21..1c4dbb41f49 100644 --- a/tools/plugins/topic-config/src/main/kotlin/net/corda/cli/plugins/topicconfig/Create.kt +++ b/tools/plugins/topic-config/src/main/kotlin/net/corda/cli/plugins/topicconfig/Create.kt @@ -12,6 +12,8 @@ import java.nio.charset.Charset import java.util.jar.JarEntry import java.util.jar.JarFile import picocli.CommandLine +import java.nio.file.Files +import java.nio.file.Paths @CommandLine.Command( name = "create", @@ -39,6 +41,12 @@ class Create( ) var partitionOverride: Int = 1 + @CommandLine.Option( + names = ["-o", "--overrides"], + description = ["Relative path of override Kafka topic configuration file in YAML format"] + ) + var overrideFilePath: String? = null + @CommandLine.Option( names = ["-u", "--user"], description = ["One or more Corda workers and their respective Kafka users e.g. -u crypto=Charlie -u rest=Rob"] @@ -61,10 +69,25 @@ class Create( val acls: List ) + data class OverrideTopicConfigurations( + val topics: List, + val acls: List + ) + data class PreviewTopicConfiguration( val name: String, + val partitions: Int, + val replicas: Short, val config: Map = emptyMap() ) + + data class OverrideTopicConfiguration( + val name: String, + val partitions: Int?, + val replicas: Short?, + val config: Map = emptyMap() + ) + data class PreviewTopicACL( val topic: String, val users: List @@ -162,16 +185,23 @@ class Create( } fun getTopicConfigsForPreview(): PreviewTopicConfigurations { - return getTopicConfigsForPreview(getTopicConfigs()) + return applyOverrides(getTopicConfigsForPreview(getTopicConfigs())) } + fun applyOverrides(config: PreviewTopicConfigurations) : PreviewTopicConfigurations = + if (overrideFilePath == null) { + config + } else { + mergeConfigurations(config, mapper.readValue(Files.readString(Paths.get(overrideFilePath!!)))) + } + fun getTopicConfigsForPreview(topicConfigurations: List): PreviewTopicConfigurations { val topicConfigs = mutableListOf() val acls = mutableListOf() topicConfigurations.forEach { topicConfig -> val topicName = getTopicName(topicConfig) - topicConfigs.add(PreviewTopicConfiguration(topicName, topicConfig.config)) + topicConfigs.add(PreviewTopicConfiguration(topicName, partitionOverride, replicaOverride, topicConfig.config)) val usersReadAccess = getUsersForProcessors(topicConfig.consumers) val usersWriteAccess = getUsersForProcessors(topicConfig.producers) @@ -190,4 +220,30 @@ class Create( return PreviewTopicConfigurations(topicConfigs, acls) } + + private fun mergeConfigurations(source: PreviewTopicConfigurations, overrides: OverrideTopicConfigurations) = + PreviewTopicConfigurations( + overrides.topics.fold(source.topics, ::mergeTopicConfiguration), + overrides.acls.fold(source.acls, ::mergeTopicACL) + ) + + private fun mergeTopicConfiguration(source: List, override: OverrideTopicConfiguration) = + source.map { + if (it.name == override.name) { + PreviewTopicConfiguration( + it.name, + override.partitions ?: it.partitions, + override.replicas ?: it.replicas, + it.config + override.config + ) + } else { + it + } + }.toList() + + private fun mergeTopicACL(source: List, override: PreviewTopicACL) = + source.map { + if (it.topic == override.topic) PreviewTopicACL(it.topic, it.users + override.users) else it + }.toList() + } diff --git a/tools/plugins/topic-config/src/main/kotlin/net/corda/cli/plugins/topicconfig/CreateConnect.kt b/tools/plugins/topic-config/src/main/kotlin/net/corda/cli/plugins/topicconfig/CreateConnect.kt index 2d002fbe2cb..adc8358651e 100644 --- a/tools/plugins/topic-config/src/main/kotlin/net/corda/cli/plugins/topicconfig/CreateConnect.kt +++ b/tools/plugins/topic-config/src/main/kotlin/net/corda/cli/plugins/topicconfig/CreateConnect.kt @@ -158,14 +158,14 @@ class CreateConnect : Runnable { fun getTopics(topicConfigs: List) = topicConfigs.associate { topicConfig: Create.PreviewTopicConfiguration -> - topicConfig.name to NewTopic(topicConfig.name, create!!.partitionOverride, create!!.replicaOverride) + topicConfig.name to NewTopic(topicConfig.name, topicConfig.partitions, topicConfig.replicas) .configs(topicConfig.config) } fun getGeneratedTopicConfigs(): Create.PreviewTopicConfigurations = if (configFilePath == null) { create!!.getTopicConfigsForPreview() } else { - // Simply read the info from provided file - create!!.mapper.readValue(Files.readString(Paths.get(configFilePath!!))) + // Simply read the info from provided file, applying any overrides + create!!.applyOverrides(create!!.mapper.readValue(Files.readString(Paths.get(configFilePath!!)))) } } diff --git a/tools/plugins/topic-config/src/test/kotlin/net/corda/cli/plugins/topicconfig/CreateConnectTest.kt b/tools/plugins/topic-config/src/test/kotlin/net/corda/cli/plugins/topicconfig/CreateConnectTest.kt index 757d6ce5ad2..08ed0f7b38f 100644 --- a/tools/plugins/topic-config/src/test/kotlin/net/corda/cli/plugins/topicconfig/CreateConnectTest.kt +++ b/tools/plugins/topic-config/src/test/kotlin/net/corda/cli/plugins/topicconfig/CreateConnectTest.kt @@ -26,14 +26,17 @@ class CreateConnectTest { @Test fun `validate new topic with no config`() { - assertThat(getCommandWithGeneratedConfig().getTopics(listOf(Create.PreviewTopicConfiguration("topic", emptyMap())))) - .containsEntry("topic", NewTopic("topic", 1, 1).configs(emptyMap())) + assertThat(getCommandWithGeneratedConfig().getTopics(listOf(Create.PreviewTopicConfiguration("topic", 5, 3, emptyMap())))) + .containsEntry("topic", NewTopic("topic", 5, 3).configs(emptyMap())) } @Test fun `validate new topic with config`() { - assertThat(getCommandWithGeneratedConfig().getTopics(listOf(Create.PreviewTopicConfiguration("topic", mapOf("key" to "value"))))) - .containsEntry("topic", NewTopic("topic", 1, 1).configs(mapOf("key" to "value"))) + assertThat( + getCommandWithGeneratedConfig().getTopics( + listOf(Create.PreviewTopicConfiguration("topic", 5, 3, mapOf("key" to "value"))) + ) + ).containsEntry("topic", NewTopic("topic", 5, 3).configs(mapOf("key" to "value"))) } @Test @@ -60,6 +63,60 @@ class CreateConnectTest { ) } + @Test + fun `validate acls created from config file with overrides`() { + val cmd = getCommandWithConfigAndOverrideFiles() + val acls = cmd.getGeneratedTopicConfigs().acls + assertThat(cmd.getAclBindings(acls)) + .containsExactly( + AclBinding(ResourcePattern(ResourceType.TOPIC, "avro.schema", PatternType.LITERAL), + AccessControlEntry("User:Chris", "*", AclOperation.READ, AclPermissionType.ALLOW)), + AclBinding(ResourcePattern(ResourceType.TOPIC, "avro.schema", PatternType.LITERAL), + AccessControlEntry("User:Chris", "*", AclOperation.WRITE, AclPermissionType.ALLOW)), + AclBinding(ResourcePattern(ResourceType.TOPIC, "avro.schema", PatternType.LITERAL), + AccessControlEntry("User:Chris", "*", AclOperation.DESCRIBE, AclPermissionType.ALLOW)), + AclBinding(ResourcePattern(ResourceType.TOPIC, "avro.schema", PatternType.LITERAL), + AccessControlEntry("User:Mo", "*", AclOperation.READ, AclPermissionType.ALLOW)), + AclBinding(ResourcePattern(ResourceType.TOPIC, "avro.schema", PatternType.LITERAL), + AccessControlEntry("User:Mo", "*", AclOperation.DESCRIBE, AclPermissionType.ALLOW)), + AclBinding(ResourcePattern(ResourceType.TOPIC, "avro.schema", PatternType.LITERAL), + AccessControlEntry("User:Mo", "*", AclOperation.WRITE, AclPermissionType.ALLOW)), + AclBinding(ResourcePattern(ResourceType.TOPIC, "certificates.rpc.ops", PatternType.LITERAL), + AccessControlEntry("User:Dan", "*", AclOperation.READ, AclPermissionType.ALLOW)), + AclBinding(ResourcePattern(ResourceType.TOPIC, "certificates.rpc.ops", PatternType.LITERAL), + AccessControlEntry("User:Dan", "*", AclOperation.DESCRIBE, AclPermissionType.ALLOW)), + AclBinding(ResourcePattern(ResourceType.TOPIC, "certificates.rpc.ops", PatternType.LITERAL), + AccessControlEntry("User:George", "*", AclOperation.READ, AclPermissionType.ALLOW)) + + ) + } + + @Test + fun `validate topics created from config file`() { + val cmd = getCommandWithConfigFile() + val topics = cmd.getGeneratedTopicConfigs().topics + assertThat(cmd.getTopics(topics)) + .containsEntry("avro.schema", NewTopic("avro.schema", 5, 3) + .configs(mapOf("cleanup.policy" to "compact", "segment.ms" to "600000", + "delete.retention.ms" to "300000", "min.compaction.lag.ms" to "60000", + "max.compaction.lag.ms" to "604800000", "min.cleanable.dirty.ratio" to "0.5"))) + .containsEntry("certificates.rpc.ops", NewTopic("certificates.rpc.ops", 4, 2) + .configs(emptyMap())) + } + + @Test + fun `validate topics created from config file with overrides`() { + val cmd = getCommandWithConfigAndOverrideFiles() + val topics = cmd.getGeneratedTopicConfigs().topics + assertThat(cmd.getTopics(topics)) + .containsEntry("avro.schema", NewTopic("avro.schema", 8, 3) + .configs(mapOf("cleanup.policy" to "compact", "segment.ms" to "600000", + "delete.retention.ms" to "300000", "min.compaction.lag.ms" to "60000", + "max.compaction.lag.ms" to "604800000", "min.cleanable.dirty.ratio" to "0.7"))) + .containsEntry("certificates.rpc.ops", NewTopic("certificates.rpc.ops", 4, 2) + .configs(emptyMap())) + } + @Test fun `kafka server address is mandatory`() { val command = CreateConnect().apply { @@ -87,4 +144,13 @@ class CreateConnectTest { create = Create() create!!.topic = TopicPlugin.Topic() } + + private fun getCommandWithConfigAndOverrideFiles() = CreateConnect().apply { + val configFile = this::class.java.classLoader.getResource("short_generated_topic_config.yaml")!!.toURI() + configFilePath = Paths.get(configFile).toString() + create = Create() + create!!.topic = TopicPlugin.Topic() + val overrideFile = this::class.java.classLoader.getResource("override_topic_config.yaml")!!.toURI() + create!!.overrideFilePath = Paths.get(overrideFile).toString() + } } diff --git a/tools/plugins/topic-config/src/test/resources/override_topic_config.yaml b/tools/plugins/topic-config/src/test/resources/override_topic_config.yaml new file mode 100644 index 00000000000..acf67f70054 --- /dev/null +++ b/tools/plugins/topic-config/src/test/resources/override_topic_config.yaml @@ -0,0 +1,16 @@ +topics: + - name: avro.schema + partitions: 8 + config: + min.cleanable.dirty.ratio: 0.7 +acls: + - topic: avro.schema + users: + - name: Mo + operations: + - write + - topic: certificates.rpc.ops + users: + - name: George + operations: + - read \ No newline at end of file diff --git a/tools/plugins/topic-config/src/test/resources/preview_config.yaml b/tools/plugins/topic-config/src/test/resources/preview_config.yaml index 4b79fafc9e7..f81594402c5 100644 --- a/tools/plugins/topic-config/src/test/resources/preview_config.yaml +++ b/tools/plugins/topic-config/src/test/resources/preview_config.yaml @@ -1,9 +1,15 @@ topics: - name: config.management.request + partitions: 1 + replicas: 1 config: {} - name: config.management.request.resp + partitions: 1 + replicas: 1 config: {} - name: config.topic + partitions: 1 + replicas: 1 config: cleanup.policy: compact segment.ms: 600000 diff --git a/tools/plugins/topic-config/src/test/resources/short_generated_topic_config.yaml b/tools/plugins/topic-config/src/test/resources/short_generated_topic_config.yaml index 7b9a0fb1f48..a47beb67ad2 100644 --- a/tools/plugins/topic-config/src/test/resources/short_generated_topic_config.yaml +++ b/tools/plugins/topic-config/src/test/resources/short_generated_topic_config.yaml @@ -1,5 +1,7 @@ topics: - name: avro.schema + partitions: 5 + replicas: 3 config: cleanup.policy: compact segment.ms: 600000 @@ -8,6 +10,8 @@ topics: max.compaction.lag.ms: 604800000 min.cleanable.dirty.ratio: 0.5 - name: certificates.rpc.ops + partitions: 4 + replicas: 2 config: {} acls: - topic: avro.schema From e4888e755807c2149c7ed1b6fb727a8552d3fd32 Mon Sep 17 00:00:00 2001 From: Yash Nabar Date: Fri, 13 Oct 2023 09:37:09 +0100 Subject: [PATCH 08/81] CORE-16778 Fix exception handling for invalid minimum platform version (#4827) Fixes exception handling for minimum platform version submitted using the MGM API to improve the returned error message. --- .../membership/lib/verifiers/GroupParametersUpdateVerifier.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/membership/membership-common/src/main/kotlin/net/corda/membership/lib/verifiers/GroupParametersUpdateVerifier.kt b/libs/membership/membership-common/src/main/kotlin/net/corda/membership/lib/verifiers/GroupParametersUpdateVerifier.kt index 93cbba3879b..2b6af6004e6 100644 --- a/libs/membership/membership-common/src/main/kotlin/net/corda/membership/lib/verifiers/GroupParametersUpdateVerifier.kt +++ b/libs/membership/membership-common/src/main/kotlin/net/corda/membership/lib/verifiers/GroupParametersUpdateVerifier.kt @@ -30,7 +30,7 @@ class GroupParametersUpdateVerifier { } } parameters[MPV_KEY]?.let { - if (it.toInt() !in 50000..99999) { + if ((it.toIntOrNull() ?: -1) !in 50000..99999) { errorMessages += "The minimum platform version (key: $MPV_KEY) has an invalid value.\n" } it From e42b16c7c8facacd18835c12ecaaef9011e29fa1 Mon Sep 17 00:00:00 2001 From: Lajos Veres Date: Fri, 13 Oct 2023 09:50:53 +0100 Subject: [PATCH 09/81] CORE-17431 utxo transaction metadata (#4845) CORE-17431 Store metadata in utxo_transaction_metadata since it is relatively large but only changes occasionally. --- .../tests/UtxoPersistenceServiceImplTest.kt | 51 ++++++++++++------ .../utxo/tests/datamodel/UtxoEntityFactory.kt | 40 ++++++++++++-- .../ledger/persistence/utxo/UtxoRepository.kt | 14 ++++- .../persistence/utxo/UtxoTransactionReader.kt | 3 ++ .../utxo/impl/AbstractUtxoQueryProvider.kt | 9 ++-- .../utxo/impl/PostgresUtxoQueryProvider.kt | 11 +++- .../utxo/impl/UtxoPersistenceServiceImpl.kt | 15 +++++- .../utxo/impl/UtxoQueryProvider.kt | 9 +++- .../utxo/impl/UtxoRepositoryImpl.kt | 36 ++++++++++--- .../utxo/impl/UtxoTransactionReaderImpl.kt | 3 ++ .../impl/UtxoPersistenceServiceImplTest.kt | 29 +++++++--- .../datamodel/utxo/UtxoTransactionEntity.kt | 8 ++- .../utxo/UtxoTransactionMetadataEntity.kt | 53 +++++++++++++++++++ gradle.properties | 2 +- .../ledger/tests/HsqldbVaultNamedQueryTest.kt | 13 ++++- .../ledger/utxo/HsqldbUtxoQueryProvider.kt | 19 +++++-- 16 files changed, 266 insertions(+), 49 deletions(-) create mode 100644 components/ledger/ledger-persistence/testing-datamodel/src/main/kotlin/com/example/ledger/testing/datamodel/utxo/UtxoTransactionMetadataEntity.kt diff --git a/components/ledger/ledger-persistence/src/integrationTest/kotlin/net/corda/ledger/persistence/utxo/tests/UtxoPersistenceServiceImplTest.kt b/components/ledger/ledger-persistence/src/integrationTest/kotlin/net/corda/ledger/persistence/utxo/tests/UtxoPersistenceServiceImplTest.kt index f6f8cf4b871..ecb6ddb5bb9 100644 --- a/components/ledger/ledger-persistence/src/integrationTest/kotlin/net/corda/ledger/persistence/utxo/tests/UtxoPersistenceServiceImplTest.kt +++ b/components/ledger/ledger-persistence/src/integrationTest/kotlin/net/corda/ledger/persistence/utxo/tests/UtxoPersistenceServiceImplTest.kt @@ -406,7 +406,6 @@ class UtxoPersistenceServiceImplTest { // Verify persisted data entityManagerFactory.transaction { em -> val dbTransaction = em.find(entityFactory.utxoTransaction, signedTransaction.id.toString()) - assertThat(dbTransaction).isNotNull val txPrivacySalt = dbTransaction.field("privacySalt") val txAccountId = dbTransaction.field("accountId") @@ -416,19 +415,19 @@ class UtxoPersistenceServiceImplTest { assertThat(txAccountId).isEqualTo(account) assertThat(txCreatedTs).isNotNull - val componentGroupLists = signedTransaction.wireTransaction.componentGroupLists + val componentGroupListsWithoutMetadata = signedTransaction.wireTransaction.componentGroupLists.drop(1) val txComponents = dbTransaction.field?>("components") assertThat(txComponents).isNotNull - .hasSameSizeAs(componentGroupLists.flatten().filter { it.isNotEmpty() }) + .hasSameSizeAs(componentGroupListsWithoutMetadata.flatten().filter { it.isNotEmpty() }) txComponents!! .sortedWith(compareBy { it.field("groupIndex") }.thenBy { it.field("leafIndex") }) .groupBy { it.field("groupIndex") }.values - .zip(componentGroupLists) + .zip(componentGroupListsWithoutMetadata) .forEachIndexed { groupIndex, (dbComponentGroup, componentGroup) -> assertThat(dbComponentGroup).hasSameSizeAs(componentGroup) dbComponentGroup.zip(componentGroup) .forEachIndexed { leafIndex, (dbComponent, component) -> - assertThat(dbComponent.field("groupIndex")).isEqualTo(groupIndex) + assertThat(dbComponent.field("groupIndex")).isEqualTo(groupIndex +1 ) assertThat(dbComponent.field("leafIndex")).isEqualTo(leafIndex) assertThat(dbComponent.field("data")).isEqualTo(component) assertThat(dbComponent.field("hash")).isEqualTo( @@ -437,6 +436,13 @@ class UtxoPersistenceServiceImplTest { } } + val dbMetadata = dbTransaction.field("metadata") + assertThat(dbMetadata).isNotNull + assertThat(dbMetadata.field("canonicalData")) + .isEqualTo(signedTransaction.wireTransaction.componentGroupLists[0][0]) + assertThat(dbMetadata.field("groupParametersHash")).isNotNull + assertThat(dbMetadata.field("cpiFileChecksum")).isNotNull + val dbTransactionOutputs = em.createNamedQuery( "UtxoVisibleTransactionOutputEntity.findByTransactionId", entityFactory.utxoVisibleTransactionOutput @@ -444,7 +450,7 @@ class UtxoPersistenceServiceImplTest { .setParameter("transactionId", signedTransaction.id.toString()) .resultList assertThat(dbTransactionOutputs).isNotNull - .hasSameSizeAs(componentGroupLists[UtxoComponentGroup.OUTPUTS.ordinal]) + .hasSameSizeAs(componentGroupListsWithoutMetadata[UtxoComponentGroup.OUTPUTS.ordinal-1]) dbTransactionOutputs .sortedWith(compareBy { it.field("groupIndex") }.thenBy { it.field("leafIndex") }) .zip(defaultVisibleTransactionOutputs) @@ -556,25 +562,38 @@ class UtxoPersistenceServiceImplTest { createdTs: Instant = testClock.instant(), status: TransactionStatus = UNVERIFIED ): Any { + val metadataBytes = signedTransaction.wireTransaction.componentGroupLists[0][0] + val metadata = entityFactory.createOrFindUtxoTransactionMetadataEntity( + digest("SHA-256", metadataBytes).toString(), + metadataBytes, + "fakeGroupParametersHash", + "fakeCpiFileChecksum" + ) + return entityFactory.createUtxoTransactionEntity( signedTransaction.id.toString(), signedTransaction.wireTransaction.privacySalt.bytes, account, createdTs, status.value, - createdTs + createdTs, + metadata ).also { transaction -> transaction.field>("components").addAll( signedTransaction.wireTransaction.componentGroupLists.flatMapIndexed { groupIndex, componentGroup -> componentGroup.mapIndexed { leafIndex: Int, component -> - entityFactory.createUtxoTransactionComponentEntity( - transaction, - groupIndex, - leafIndex, - component, - digest("SHA-256", component).toString() - ) - } + if (groupIndex != 0 || leafIndex != 0) { + entityFactory.createUtxoTransactionComponentEntity( + transaction, + groupIndex, + leafIndex, + component, + digest("SHA-256", component).toString() + ) + } else { + null + } + }.filterNotNull() } ) transaction.field>("signatures").addAll( @@ -658,6 +677,8 @@ class UtxoPersistenceServiceImplTest { get() = transactionContainer.id override val privacySalt: PrivacySalt get() = transactionContainer.wireTransaction.privacySalt + override val metadata: TransactionMetadataInternal + get() = transactionContainer.wireTransaction.metadata as TransactionMetadataInternal override val rawGroupLists: List> get() = transactionContainer.wireTransaction.componentGroupLists override val signatures: List diff --git a/components/ledger/ledger-persistence/src/integrationTest/kotlin/net/corda/ledger/persistence/utxo/tests/datamodel/UtxoEntityFactory.kt b/components/ledger/ledger-persistence/src/integrationTest/kotlin/net/corda/ledger/persistence/utxo/tests/datamodel/UtxoEntityFactory.kt index 3bbea0fdaa1..7e467461041 100644 --- a/components/ledger/ledger-persistence/src/integrationTest/kotlin/net/corda/ledger/persistence/utxo/tests/datamodel/UtxoEntityFactory.kt +++ b/components/ledger/ledger-persistence/src/integrationTest/kotlin/net/corda/ledger/persistence/utxo/tests/datamodel/UtxoEntityFactory.kt @@ -1,12 +1,14 @@ package net.corda.ledger.persistence.utxo.tests.datamodel +import net.corda.orm.utils.transaction import java.time.Instant import javax.persistence.EntityManagerFactory -class UtxoEntityFactory(entityManagerFactory: EntityManagerFactory) { +class UtxoEntityFactory(private val entityManagerFactory: EntityManagerFactory) { private val entityMap = entityManagerFactory.metamodel.entities.associate { it.name to it.bindableJavaType } val utxoTransaction: Class<*> get() = classFor("UtxoTransactionEntity") + val utxoTransactionMetadata: Class<*> get() = classFor("UtxoTransactionMetadataEntity") val utxoTransactionComponent: Class<*> get() = classFor("UtxoTransactionComponentEntity") val utxoVisibleTransactionOutput: Class<*> get() = classFor("UtxoVisibleTransactionOutputEntity") val utxoTransactionSignature: Class<*> get() = classFor("UtxoTransactionSignatureEntity") @@ -17,10 +19,40 @@ class UtxoEntityFactory(entityManagerFactory: EntityManagerFactory) { accountId: String, created: Instant, status: String, - updated: Instant + updated: Instant, + utxoTransactionMetadata: Any ): Any { - return utxoTransaction.constructors.single { it.parameterCount == 6 }.newInstance( - transactionId, privacySalt, accountId, created, status, updated + return utxoTransaction.constructors.single { it.parameterCount == 7 }.newInstance( + transactionId, privacySalt, accountId, created, status, updated, utxoTransactionMetadata + ) + } + + fun createOrFindUtxoTransactionMetadataEntity( + hash: String, + canonicalData: ByteArray, + groupParametersHash: String, + cpiFileChecksum: String, + ): Any { + return entityManagerFactory.transaction { em -> + em.find(utxoTransactionMetadata, hash) ?: createUtxoTransactionMetadataEntity( + hash, + canonicalData, + groupParametersHash, + cpiFileChecksum, + ).also { + em.persist(it) + } + } + } + + fun createUtxoTransactionMetadataEntity( + hash: String, + canonicalData: ByteArray, + groupParametersHash: String, + cpiFileChecksum: String, + ): Any { + return utxoTransactionMetadata.constructors.single { it.parameterCount == 4 }.newInstance( + hash, canonicalData, groupParametersHash, cpiFileChecksum ) } diff --git a/components/ledger/ledger-persistence/src/main/kotlin/net/corda/ledger/persistence/utxo/UtxoRepository.kt b/components/ledger/ledger-persistence/src/main/kotlin/net/corda/ledger/persistence/utxo/UtxoRepository.kt index b566f85aad0..7626d7b5264 100644 --- a/components/ledger/ledger-persistence/src/main/kotlin/net/corda/ledger/persistence/utxo/UtxoRepository.kt +++ b/components/ledger/ledger-persistence/src/main/kotlin/net/corda/ledger/persistence/utxo/UtxoRepository.kt @@ -26,7 +26,7 @@ interface UtxoRepository { id: String ): SignedTransactionContainer? - /** Retrieves transaction component leafs */ + /** Retrieves transaction component leafs except metadata which is stored separately */ fun findTransactionComponentLeafs( entityManager: EntityManager, transactionId: String @@ -76,7 +76,17 @@ interface UtxoRepository { privacySalt: ByteArray, account: String, timestamp: Instant, - status: TransactionStatus + status: TransactionStatus, + metadataHash: String + ) + + /** Persists transaction metadata (operation is idempotent) */ + fun persistTransactionMetadata( + entityManager: EntityManager, + hash: String, + metadataBytes: ByteArray, + groupParametersHash: String, + cpiFileChecksum: String ) /** Persists transaction component leaf [data] (operation is idempotent) */ diff --git a/components/ledger/ledger-persistence/src/main/kotlin/net/corda/ledger/persistence/utxo/UtxoTransactionReader.kt b/components/ledger/ledger-persistence/src/main/kotlin/net/corda/ledger/persistence/utxo/UtxoTransactionReader.kt index 4da18b7fd18..74ae3183ceb 100644 --- a/components/ledger/ledger-persistence/src/main/kotlin/net/corda/ledger/persistence/utxo/UtxoTransactionReader.kt +++ b/components/ledger/ledger-persistence/src/main/kotlin/net/corda/ledger/persistence/utxo/UtxoTransactionReader.kt @@ -5,6 +5,7 @@ import net.corda.v5.application.crypto.DigitalSignatureAndMetadata import net.corda.v5.crypto.SecureHash import net.corda.v5.ledger.common.transaction.CordaPackageSummary import net.corda.ledger.common.data.transaction.PrivacySalt +import net.corda.ledger.common.data.transaction.TransactionMetadataInternal import net.corda.v5.ledger.utxo.ContractState import net.corda.v5.ledger.utxo.StateAndRef import net.corda.v5.ledger.utxo.StateRef @@ -13,6 +14,8 @@ interface UtxoTransactionReader { val id: SecureHash + val metadata: TransactionMetadataInternal + val account: String val status: TransactionStatus diff --git a/components/ledger/ledger-persistence/src/main/kotlin/net/corda/ledger/persistence/utxo/impl/AbstractUtxoQueryProvider.kt b/components/ledger/ledger-persistence/src/main/kotlin/net/corda/ledger/persistence/utxo/impl/AbstractUtxoQueryProvider.kt index 4a47246e39b..1f91c967eac 100644 --- a/components/ledger/ledger-persistence/src/main/kotlin/net/corda/ledger/persistence/utxo/impl/AbstractUtxoQueryProvider.kt +++ b/components/ledger/ledger-persistence/src/main/kotlin/net/corda/ledger/persistence/utxo/impl/AbstractUtxoQueryProvider.kt @@ -9,10 +9,13 @@ abstract class AbstractUtxoQueryProvider : UtxoQueryProvider { val UNVERIFIED = TransactionStatus.UNVERIFIED.value } - override val findTransactionPrivacySalt: String + override val findTransactionPrivacySaltAndMetadata: String get() = """ - SELECT privacy_salt - FROM {h-schema}utxo_transaction + SELECT privacy_salt, + utm.canonical_data + FROM {h-schema}utxo_transaction AS ut + JOIN {h-schema}utxo_transaction_metadata AS utm + ON ut.metadata_hash = utm.hash WHERE id = :transactionId""" .trimIndent() diff --git a/components/ledger/ledger-persistence/src/main/kotlin/net/corda/ledger/persistence/utxo/impl/PostgresUtxoQueryProvider.kt b/components/ledger/ledger-persistence/src/main/kotlin/net/corda/ledger/persistence/utxo/impl/PostgresUtxoQueryProvider.kt index 101bc0f1527..f9947c75a98 100644 --- a/components/ledger/ledger-persistence/src/main/kotlin/net/corda/ledger/persistence/utxo/impl/PostgresUtxoQueryProvider.kt +++ b/components/ledger/ledger-persistence/src/main/kotlin/net/corda/ledger/persistence/utxo/impl/PostgresUtxoQueryProvider.kt @@ -19,13 +19,20 @@ class PostgresUtxoQueryProvider @Activate constructor( override val persistTransaction: String get() = """ - INSERT INTO {h-schema}utxo_transaction(id, privacy_salt, account_id, created, status, updated) - VALUES (:id, :privacySalt, :accountId, :createdAt, :status, :updatedAt) + INSERT INTO {h-schema}utxo_transaction(id, privacy_salt, account_id, created, status, updated, metadata_hash) + VALUES (:id, :privacySalt, :accountId, :createdAt, :status, :updatedAt, :metadataHash) ON CONFLICT(id) DO UPDATE SET status = EXCLUDED.status, updated = EXCLUDED.updated WHERE utxo_transaction.status = EXCLUDED.status OR utxo_transaction.status = '$UNVERIFIED'""" .trimIndent() + override val persistTransactionMetadata: String + get() = """ + INSERT INTO {h-schema}utxo_transaction_metadata(hash, canonical_data, group_parameters_hash, cpi_file_checksum) + VALUES (:hash, :canonicalData, :groupParametersHash, :cpiFileChecksum) + ON CONFLICT DO NOTHING""" + .trimIndent() + override val persistTransactionComponentLeaf: String get() = """ INSERT INTO {h-schema}utxo_transaction_component(transaction_id, group_idx, leaf_idx, data, hash) diff --git a/components/ledger/ledger-persistence/src/main/kotlin/net/corda/ledger/persistence/utxo/impl/UtxoPersistenceServiceImpl.kt b/components/ledger/ledger-persistence/src/main/kotlin/net/corda/ledger/persistence/utxo/impl/UtxoPersistenceServiceImpl.kt index 0ddde389b71..f26d073ed91 100644 --- a/components/ledger/ledger-persistence/src/main/kotlin/net/corda/ledger/persistence/utxo/impl/UtxoPersistenceServiceImpl.kt +++ b/components/ledger/ledger-persistence/src/main/kotlin/net/corda/ledger/persistence/utxo/impl/UtxoPersistenceServiceImpl.kt @@ -145,6 +145,18 @@ class UtxoPersistenceServiceImpl( val nowUtc = utcClock.instant() val transactionIdString = transaction.id.toString() + val metadataBytes = transaction.rawGroupLists[0][0] + val metadataHash = sandboxDigestService.hash(metadataBytes, DigestAlgorithmName.SHA2_256).toString() + + val metadata = transaction.metadata + repository.persistTransactionMetadata( + em, + metadataHash, + metadataBytes, + requireNotNull(metadata.getMembershipGroupParametersHash()) { "Metadata without membership group parameters hash" }, + requireNotNull(metadata.getCpiMetadata()) { "Metadata without CPI metadata" }.fileChecksum + ) + // Insert the Transaction repository.persistTransaction( em, @@ -152,7 +164,8 @@ class UtxoPersistenceServiceImpl( transaction.privacySalt.bytes, transaction.account, nowUtc, - transaction.status + transaction.status, + metadataHash ) // Insert the Transactions components diff --git a/components/ledger/ledger-persistence/src/main/kotlin/net/corda/ledger/persistence/utxo/impl/UtxoQueryProvider.kt b/components/ledger/ledger-persistence/src/main/kotlin/net/corda/ledger/persistence/utxo/impl/UtxoQueryProvider.kt index 572f9b50ab1..c3bb702a640 100644 --- a/components/ledger/ledger-persistence/src/main/kotlin/net/corda/ledger/persistence/utxo/impl/UtxoQueryProvider.kt +++ b/components/ledger/ledger-persistence/src/main/kotlin/net/corda/ledger/persistence/utxo/impl/UtxoQueryProvider.kt @@ -11,9 +11,9 @@ package net.corda.ledger.persistence.utxo.impl */ interface UtxoQueryProvider { /** - * @property findTransactionPrivacySalt SQL text for [UtxoRepositoryImpl.findTransactionPrivacySalt]. + * @property findTransactionPrivacySaltAndMetadata SQL text for [UtxoRepositoryImpl.findTransactionPrivacySaltAndMetadata]. */ - val findTransactionPrivacySalt: String + val findTransactionPrivacySaltAndMetadata: String /** * @property findTransactionComponentLeafs SQL text for [UtxoRepositoryImpl.findTransactionComponentLeafs]. @@ -61,6 +61,11 @@ interface UtxoQueryProvider { */ val persistTransaction: String + /** + * @property persistTransactionMetadata SQL text for [UtxoRepositoryImpl.persistTransactionMetadata]. + */ + val persistTransactionMetadata: String + /** * @property persistTransactionComponentLeaf SQL text for [UtxoRepositoryImpl.persistTransactionComponentLeaf]. */ diff --git a/components/ledger/ledger-persistence/src/main/kotlin/net/corda/ledger/persistence/utxo/impl/UtxoRepositoryImpl.kt b/components/ledger/ledger-persistence/src/main/kotlin/net/corda/ledger/persistence/utxo/impl/UtxoRepositoryImpl.kt index 6ceed46eae3..da1e4ded418 100644 --- a/components/ledger/ledger-persistence/src/main/kotlin/net/corda/ledger/persistence/utxo/impl/UtxoRepositoryImpl.kt +++ b/components/ledger/ledger-persistence/src/main/kotlin/net/corda/ledger/persistence/utxo/impl/UtxoRepositoryImpl.kt @@ -60,9 +60,9 @@ class UtxoRepositoryImpl @Activate constructor( entityManager: EntityManager, id: String ): SignedTransactionContainer? { - val privacySalt = findTransactionPrivacySalt(entityManager, id) ?: return null + val (privacySalt, metadataBytes) = findTransactionPrivacySaltAndMetadata(entityManager, id) ?: return null val wireTransaction = wireTransactionFactory.create( - findTransactionComponentLeafs(entityManager, id), + mapOf(0 to listOf(metadataBytes)) + findTransactionComponentLeafs(entityManager, id), privacySalt ) return SignedTransactionContainer( @@ -87,14 +87,14 @@ class UtxoRepositoryImpl @Activate constructor( .associate { r -> parseSecureHash(r.get(0) as String) to r.get(1) as String } } - private fun findTransactionPrivacySalt( + private fun findTransactionPrivacySaltAndMetadata( entityManager: EntityManager, transactionId: String - ): PrivacySaltImpl? { - return entityManager.createNativeQuery(queryProvider.findTransactionPrivacySalt, Tuple::class.java) + ): Pair? { + return entityManager.createNativeQuery(queryProvider.findTransactionPrivacySaltAndMetadata, Tuple::class.java) .setParameter("transactionId", transactionId) .resultListAsTuples() - .map { r -> PrivacySaltImpl(r.get(0) as ByteArray) } + .map { r -> Pair(PrivacySaltImpl(r.get(0) as ByteArray), r.get(1) as ByteArray) } .firstOrNull() } @@ -183,7 +183,8 @@ class UtxoRepositoryImpl @Activate constructor( privacySalt: ByteArray, account: String, timestamp: Instant, - status: TransactionStatus + status: TransactionStatus, + metadataHash: String ) { entityManager.createNativeQuery(queryProvider.persistTransaction) .setParameter("id", id) @@ -192,10 +193,27 @@ class UtxoRepositoryImpl @Activate constructor( .setParameter("createdAt", timestamp) .setParameter("status", status.value) .setParameter("updatedAt", timestamp) + .setParameter("metadataHash", metadataHash) .executeUpdate() .logResult("transaction [$id]") } + override fun persistTransactionMetadata( + entityManager: EntityManager, + hash: String, + metadataBytes: ByteArray, + groupParametersHash: String, + cpiFileChecksum: String + ){ + entityManager.createNativeQuery(queryProvider.persistTransactionMetadata) + .setParameter("hash", hash) + .setParameter("canonicalData", metadataBytes) + .setParameter("groupParametersHash", groupParametersHash) + .setParameter("cpiFileChecksum", cpiFileChecksum) + .executeUpdate() + .logResult("transaction metadata [$hash]") + } + override fun persistTransactionComponentLeaf( entityManager: EntityManager, transactionId: String, @@ -204,6 +222,10 @@ class UtxoRepositoryImpl @Activate constructor( data: ByteArray, hash: String ) { + // Metadata is not stored with the other components. See persistTransactionMetadata(). + if (groupIndex == 0 && leafIndex == 0) { + return + } entityManager.createNativeQuery(queryProvider.persistTransactionComponentLeaf) .setParameter("transactionId", transactionId) .setParameter("groupIndex", groupIndex) diff --git a/components/ledger/ledger-persistence/src/main/kotlin/net/corda/ledger/persistence/utxo/impl/UtxoTransactionReaderImpl.kt b/components/ledger/ledger-persistence/src/main/kotlin/net/corda/ledger/persistence/utxo/impl/UtxoTransactionReaderImpl.kt index 372ded0da64..32eb6a4cfe3 100644 --- a/components/ledger/ledger-persistence/src/main/kotlin/net/corda/ledger/persistence/utxo/impl/UtxoTransactionReaderImpl.kt +++ b/components/ledger/ledger-persistence/src/main/kotlin/net/corda/ledger/persistence/utxo/impl/UtxoTransactionReaderImpl.kt @@ -75,6 +75,9 @@ class UtxoTransactionReaderImpl( override val privacySalt: PrivacySalt get() = signedTransaction.wireTransaction.privacySalt + override val metadata: TransactionMetadataInternal + get() = signedTransaction.wireTransaction.metadata as TransactionMetadataInternal + override val rawGroupLists: List> get() = signedTransaction.wireTransaction.componentGroupLists diff --git a/components/ledger/ledger-persistence/src/test/kotlin/net/corda/ledger/persistence/utxo/impl/UtxoPersistenceServiceImplTest.kt b/components/ledger/ledger-persistence/src/test/kotlin/net/corda/ledger/persistence/utxo/impl/UtxoPersistenceServiceImplTest.kt index c4b5ad22538..3b9d93f8fda 100644 --- a/components/ledger/ledger-persistence/src/test/kotlin/net/corda/ledger/persistence/utxo/impl/UtxoPersistenceServiceImplTest.kt +++ b/components/ledger/ledger-persistence/src/test/kotlin/net/corda/ledger/persistence/utxo/impl/UtxoPersistenceServiceImplTest.kt @@ -2,8 +2,10 @@ package net.corda.ledger.persistence.utxo.impl import com.fasterxml.jackson.databind.ObjectMapper import net.corda.application.impl.services.json.JsonMarshallingServiceImpl +import net.corda.crypto.core.SecureHashImpl import net.corda.crypto.testkit.SecureHashUtils.randomSecureHash import net.corda.ledger.common.data.transaction.PrivacySalt +import net.corda.ledger.common.data.transaction.TransactionMetadataImpl import net.corda.ledger.common.data.transaction.TransactionStatus import net.corda.ledger.persistence.json.DefaultContractStateVaultJsonFactory import net.corda.ledger.persistence.json.impl.DefaultContractStateVaultJsonFactoryImpl @@ -12,6 +14,7 @@ import net.corda.ledger.persistence.utxo.CustomRepresentation import net.corda.ledger.persistence.utxo.UtxoRepository import net.corda.ledger.persistence.utxo.UtxoTransactionReader import net.corda.utilities.time.UTCClock +import net.corda.v5.application.crypto.DigestService import net.corda.v5.application.marshalling.JsonMarshallingService import net.corda.v5.base.types.MemberX500Name import net.corda.v5.crypto.SecureHash @@ -55,9 +58,12 @@ class UtxoPersistenceServiceImplTest { persistedJsonStrings[txId] = customRepresentation } - on { persistTransaction(any(), any(), any(), any(), any(), any()) } doAnswer {} + on { persistTransaction(any(), any(), any(), any(), any(), any(), any()) } doAnswer {} on { persistTransactionComponentLeaf(any(), any(), any(), any(), any(), any()) } doAnswer {} } + private val mockDigestService = mock { + on { hash(any(), any())} doAnswer { SecureHashImpl("algo", byteArrayOf(1, 2, 11)) } + } private val mockPrivacySalt = mock { on { bytes } doReturn ByteArray(0) @@ -81,7 +87,7 @@ class UtxoPersistenceServiceImplTest { mockEmFactory, mockRepository, mock(), - mock(), + mockDigestService, storage, DefaultContractStateVaultJsonFactoryImpl(), JsonMarshallingServiceImpl(), // We could mock this but this is basically just a layer on top of Jackson @@ -132,7 +138,7 @@ class UtxoPersistenceServiceImplTest { mockEmFactory, mockRepository, mock(), - mock(), + mockDigestService, storage, emptyDefaultContractStateVaultJsonFactory, JsonMarshallingServiceImpl(), @@ -217,7 +223,7 @@ class UtxoPersistenceServiceImplTest { mockEmFactory, mockRepository, mock(), - mock(), + mockDigestService, ContractStateVaultJsonFactoryRegistryImpl(), // Empty storage DefaultContractStateVaultJsonFactoryImpl(), JsonMarshallingServiceImpl(), @@ -256,7 +262,7 @@ class UtxoPersistenceServiceImplTest { mockEmFactory, mockRepository, mock(), - mock(), + mockDigestService, storage, DefaultContractStateVaultJsonFactoryImpl(), JsonMarshallingServiceImpl(), @@ -290,7 +296,7 @@ class UtxoPersistenceServiceImplTest { private fun createMockTransaction(producedStates: Map>): UtxoTransactionReader { return mock { on { getConsumedStateRefs() } doReturn emptyList() - on { rawGroupLists } doReturn emptyList() + on { rawGroupLists } doReturn listOf(listOf("{}".toByteArray())) on { visibleStatesIndexes } doReturn listOf(0) on { status } doReturn TransactionStatus.UNVERIFIED on { signatures } doReturn emptyList() @@ -298,6 +304,17 @@ class UtxoPersistenceServiceImplTest { on { privacySalt } doReturn mockPrivacySalt on { account } doReturn "" on { getVisibleStates() } doReturn producedStates + on { metadata } doReturn TransactionMetadataImpl( + mapOf( + "membershipGroupParametersHash" to "membershipGroupParametersHash", + "cpiMetadata" to mapOf( + "name" to "name", + "version" to "version", + "signerSummaryHash" to "signerSummaryHash", + "fileChecksum" to "cpiFileChecksum" + ) + ) + ) } } diff --git a/components/ledger/ledger-persistence/testing-datamodel/src/main/kotlin/com/example/ledger/testing/datamodel/utxo/UtxoTransactionEntity.kt b/components/ledger/ledger-persistence/testing-datamodel/src/main/kotlin/com/example/ledger/testing/datamodel/utxo/UtxoTransactionEntity.kt index 83d102365fd..ab41020411b 100644 --- a/components/ledger/ledger-persistence/testing-datamodel/src/main/kotlin/com/example/ledger/testing/datamodel/utxo/UtxoTransactionEntity.kt +++ b/components/ledger/ledger-persistence/testing-datamodel/src/main/kotlin/com/example/ledger/testing/datamodel/utxo/UtxoTransactionEntity.kt @@ -8,6 +8,8 @@ import javax.persistence.Id import javax.persistence.OneToMany import javax.persistence.Table import net.corda.v5.base.annotations.CordaSerializable +import javax.persistence.JoinColumn +import javax.persistence.ManyToOne @CordaSerializable @Entity @@ -30,7 +32,11 @@ data class UtxoTransactionEntity( var status: String, @get:Column(name = "updated", nullable = false) - var updated: Instant + var updated: Instant, + + @get:ManyToOne + @get:JoinColumn(name = "metadata_hash", nullable = false, updatable = false) + var metadata: UtxoTransactionMetadataEntity, ) { @get:OneToMany(mappedBy = "transaction", cascade = [CascadeType.ALL], orphanRemoval = true) var components: MutableList = mutableListOf() diff --git a/components/ledger/ledger-persistence/testing-datamodel/src/main/kotlin/com/example/ledger/testing/datamodel/utxo/UtxoTransactionMetadataEntity.kt b/components/ledger/ledger-persistence/testing-datamodel/src/main/kotlin/com/example/ledger/testing/datamodel/utxo/UtxoTransactionMetadataEntity.kt new file mode 100644 index 00000000000..d325886ec3c --- /dev/null +++ b/components/ledger/ledger-persistence/testing-datamodel/src/main/kotlin/com/example/ledger/testing/datamodel/utxo/UtxoTransactionMetadataEntity.kt @@ -0,0 +1,53 @@ +package com.example.ledger.testing.datamodel.utxo + +import javax.persistence.CascadeType +import javax.persistence.Column +import javax.persistence.Entity +import javax.persistence.Id +import javax.persistence.OneToMany +import javax.persistence.Table +import net.corda.v5.base.annotations.CordaSerializable + +@CordaSerializable +@Entity +@Table(name = "utxo_transaction_metadata") +data class UtxoTransactionMetadataEntity( + @get:Id + @get:Column(name = "hash", nullable = false, updatable = false) + var hash: String, + + @get:Column(name = "canonical_data", nullable = false) + var canonicalData: ByteArray, + + @get:Column(name = "group_parameters_hash", nullable = false) + var groupParametersHash: String, + + @get:Column(name = "cpi_file_checksum", nullable = false) + var cpiFileChecksum: String + +) { + @get:OneToMany(mappedBy = "metadata", cascade = [CascadeType.ALL], orphanRemoval = true) + var transactions: MutableList = mutableListOf() + + override fun equals(other: Any?): Boolean { + if (this === other) return true + if (javaClass != other?.javaClass) return false + + other as UtxoTransactionMetadataEntity + + if (hash != other.hash) return false + if (!canonicalData.contentEquals(other.canonicalData)) return false + if (groupParametersHash != other.groupParametersHash) return false + if (cpiFileChecksum != other.cpiFileChecksum) return false + + return true + } + + override fun hashCode(): Int { + var result = hash.hashCode() + result = 31 * result + canonicalData.contentHashCode() + result = 31 * result + groupParametersHash.hashCode() + result = 31 * result + cpiFileChecksum.hashCode() + return result + } +} diff --git a/gradle.properties b/gradle.properties index 84986c72529..7a66c075b71 100644 --- a/gradle.properties +++ b/gradle.properties @@ -46,7 +46,7 @@ commonsTextVersion = 1.10.0 bouncycastleVersion=1.76 # Corda API libs revision (change in 4th digit indicates a breaking change) # Change to 5.1.0.xx-SNAPSHOT to pick up maven local published copy -cordaApiVersion=5.1.0.32-beta+ +cordaApiVersion=5.1.0.33-beta+ disruptorVersion=3.4.4 felixConfigAdminVersion=1.9.26 diff --git a/testing/ledger/ledger-hsqldb/src/integrationTest/kotlin/net/corda/testing/ledger/tests/HsqldbVaultNamedQueryTest.kt b/testing/ledger/ledger-hsqldb/src/integrationTest/kotlin/net/corda/testing/ledger/tests/HsqldbVaultNamedQueryTest.kt index 350ac5fb32a..ba0388e0c88 100644 --- a/testing/ledger/ledger-hsqldb/src/integrationTest/kotlin/net/corda/testing/ledger/tests/HsqldbVaultNamedQueryTest.kt +++ b/testing/ledger/ledger-hsqldb/src/integrationTest/kotlin/net/corda/testing/ledger/tests/HsqldbVaultNamedQueryTest.kt @@ -3,6 +3,7 @@ package net.corda.testing.ledger.tests import com.example.ledger.testing.datamodel.utxo.UtxoTransactionComponentEntity import com.example.ledger.testing.datamodel.utxo.UtxoTransactionComponentEntityId import com.example.ledger.testing.datamodel.utxo.UtxoTransactionEntity +import com.example.ledger.testing.datamodel.utxo.UtxoTransactionMetadataEntity import com.example.ledger.testing.datamodel.utxo.UtxoVisibleTransactionOutputEntity import com.example.ledger.testing.datamodel.utxo.UtxoVisibleTransactionOutputEntityId import com.example.ledger.testing.datamodel.utxo.UtxoTransactionSignatureEntity @@ -78,6 +79,7 @@ class HsqldbVaultNamedQueryTest { val entities = JpaEntitiesSet.create("utxo-ledger", setOf( UtxoTransactionEntity::class.java, + UtxoTransactionMetadataEntity::class.java, UtxoTransactionComponentEntity::class.java, UtxoTransactionComponentEntityId::class.java, UtxoVisibleTransactionOutputEntity::class.java, @@ -92,13 +94,22 @@ class HsqldbVaultNamedQueryTest { entityManagerFactory.transaction { em -> val timestamp = Instant.now() + val metadata = em.find(UtxoTransactionMetadataEntity::class.java, "hash") ?: UtxoTransactionMetadataEntity( + "hash", + "canonicalData".toByteArray(), + "groupParametersHash", + "cpiFileChecksum" + ).also{ + em.persist(it) + } val tx = UtxoTransactionEntity( id = txId.toString(), privacySalt = byteArrayOf(), accountId = ACCOUNT_ID.toString(), created = timestamp, status = TransactionStatus.VERIFIED.value, - updated = timestamp + updated = timestamp, + metadata = metadata ) val visibleStates = mutableListOf() diff --git a/testing/ledger/ledger-hsqldb/src/main/kotlin/net/corda/testing/ledger/utxo/HsqldbUtxoQueryProvider.kt b/testing/ledger/ledger-hsqldb/src/main/kotlin/net/corda/testing/ledger/utxo/HsqldbUtxoQueryProvider.kt index 0eca180e0c8..daec8adaaf7 100644 --- a/testing/ledger/ledger-hsqldb/src/main/kotlin/net/corda/testing/ledger/utxo/HsqldbUtxoQueryProvider.kt +++ b/testing/ledger/ledger-hsqldb/src/main/kotlin/net/corda/testing/ledger/utxo/HsqldbUtxoQueryProvider.kt @@ -22,12 +22,23 @@ class HsqldbUtxoQueryProvider @Activate constructor( override val persistTransaction: String get() = """ MERGE INTO {h-schema}utxo_transaction AS ut - USING (VALUES :id, CAST(:privacySalt AS VARBINARY(64)), :accountId, CAST(:createdAt AS TIMESTAMP), :status, CAST(:updatedAt AS TIMESTAMP)) - AS x(id, privacy_salt, account_id, created, status, updated) + USING (VALUES :id, CAST(:privacySalt AS VARBINARY(64)), :accountId, CAST(:createdAt AS TIMESTAMP), :status, CAST(:updatedAt AS TIMESTAMP), :metadataHash) + AS x(id, privacy_salt, account_id, created, status, updated, metadata_hash) ON x.id = ut.id WHEN NOT MATCHED THEN - INSERT (id, privacy_salt, account_id, created, status, updated) - VALUES (x.id, x.privacy_salt, x.account_id, x.created, x.status, x.updated)""" + INSERT (id, privacy_salt, account_id, created, status, updated, metadata_hash) + VALUES (x.id, x.privacy_salt, x.account_id, x.created, x.status, x.updated, x.metadata_hash)""" + .trimIndent() + + override val persistTransactionMetadata: String + get() = """ + MERGE INTO {h-schema}utxo_transaction_metadata AS m + USING (VALUES :hash, CAST(:canonicalData AS VARBINARY(1048576)), :groupParametersHash, :cpiFileChecksum) + AS x(hash, canonical_data, group_parameters_hash, cpi_file_checksum) + ON x.hash = m.hash + WHEN NOT MATCHED THEN + INSERT (hash, canonical_data, group_parameters_hash, cpi_file_checksum) + VALUES (x.hash, x.canonical_data, x.group_parameters_hash, x.cpi_file_checksum)""" .trimIndent() override val persistTransactionComponentLeaf: String From 191cd1aa7a65cd8617abbdfed2b8365e9a561aff Mon Sep 17 00:00:00 2001 From: Simon Johnson <106170800+simon-johnson-r3@users.noreply.github.com> Date: Fri, 13 Oct 2023 10:04:20 +0100 Subject: [PATCH 10/81] CORE-17433 Sandbox cache thread safety (#4844) From logs locally and in our resiliency tests it's clear sandbox closing and therefore bundle uninstallation can be performed in multiple threads depending on whether purgeExpiryQueue is called from the cache's executor thread via the removal listener, or as part of some other synchronous cache operation which happens in the thread of the client calling it. The parallelism in SecManagerForkJoinPool is configured to the number of processors. This means we wont just see the members of SandboxGroupContextCacheImpl manipulated across 2 threads, we will see it across many and it could happen concurrently in periods of significant flow activity. This change makes the sandbox cache single threaded, meaning we only ever see manipulation of SandboxGroupContextCacheImpl on the same thread - confirmed by examining the logs. The author of Caffeine states here that using the same thread for the executor is quite acceptable, the Cache itself does little work there: https://stackoverflow.com/questions/68101454/caffeine-combining-both-scheduler-and-executor-service --- .../service/impl/SandboxGroupContextCacheImpl.kt | 6 +++++- .../net/corda/cache/caffeine/CacheFactory.kt | 14 ++++++++++++-- .../net/corda/cache/caffeine/CacheFactoryImpl.kt | 8 +++++++- 3 files changed, 24 insertions(+), 4 deletions(-) diff --git a/components/virtual-node/sandbox-group-context-service/src/main/kotlin/net/corda/sandboxgroupcontext/service/impl/SandboxGroupContextCacheImpl.kt b/components/virtual-node/sandbox-group-context-service/src/main/kotlin/net/corda/sandboxgroupcontext/service/impl/SandboxGroupContextCacheImpl.kt index c6ac53d0bd3..4ce26ffc3bd 100644 --- a/components/virtual-node/sandbox-group-context-service/src/main/kotlin/net/corda/sandboxgroupcontext/service/impl/SandboxGroupContextCacheImpl.kt +++ b/components/virtual-node/sandbox-group-context-service/src/main/kotlin/net/corda/sandboxgroupcontext/service/impl/SandboxGroupContextCacheImpl.kt @@ -80,11 +80,15 @@ internal class SandboxGroupContextCacheImpl private constructor( /** * Builds a cache for the specified SandboxGroup [type] with [capacity] maximum size. + * Uses buildNonAsync in order that the removal listener is called in the same thread as cache interactions. + * The removal listener interacts with both the expiryQueue and the toBeClosed list, neither of which are + * thread safe. This also ensures if purgeExpiryQueue closes any sandboxes they are closed in the same thread + * as would be the case with all other calls to the same method. */ private fun buildSandboxGroupTypeCache( type: SandboxGroupType, capacity: Long - ): Cache = CacheFactoryImpl().build( + ): Cache = CacheFactoryImpl().buildNonAsync( "sandbox-cache-${type}", Caffeine.newBuilder() .maximumSize(capacity) diff --git a/libs/cache/cache-caffeine/src/main/kotlin/net/corda/cache/caffeine/CacheFactory.kt b/libs/cache/cache-caffeine/src/main/kotlin/net/corda/cache/caffeine/CacheFactory.kt index 83ab7533686..f18c6cf9e3b 100644 --- a/libs/cache/cache-caffeine/src/main/kotlin/net/corda/cache/caffeine/CacheFactory.kt +++ b/libs/cache/cache-caffeine/src/main/kotlin/net/corda/cache/caffeine/CacheFactory.kt @@ -10,7 +10,7 @@ import com.github.benmanes.caffeine.cache.LoadingCache */ interface CacheFactory { /** - * Build a Caffeine Cache + * Build a Caffeine Cache. * * @param name Cache name, used for logging and metrics. * @param caffeine Caffeine cache builder. @@ -18,7 +18,17 @@ interface CacheFactory { fun build(name: String, caffeine: Caffeine): Cache /** - * Build a Caffeine Loading Cache + * Build a Caffeine Cache which executes all actions including callbacks synchronously in the same thread + * the calls to the cache are made. The primary use case for this builder is adding non thread safe listeners + * to the cache. + * + * @param name Cache name, used for logging and metrics. + * @param caffeine Caffeine cache builder. + */ + fun buildNonAsync(name: String, caffeine: Caffeine): Cache + + /** + * Build a Caffeine Loading Cache. * * @param name Cache name, used for logging and metrics. * @param caffeine Caffeine cache builder. diff --git a/libs/cache/cache-caffeine/src/main/kotlin/net/corda/cache/caffeine/CacheFactoryImpl.kt b/libs/cache/cache-caffeine/src/main/kotlin/net/corda/cache/caffeine/CacheFactoryImpl.kt index 445fae0d6ce..f8dde549dc8 100644 --- a/libs/cache/cache-caffeine/src/main/kotlin/net/corda/cache/caffeine/CacheFactoryImpl.kt +++ b/libs/cache/cache-caffeine/src/main/kotlin/net/corda/cache/caffeine/CacheFactoryImpl.kt @@ -19,7 +19,13 @@ class CacheFactoryImpl: CacheFactory { .build() return CaffeineCacheMetrics.monitor(CordaMetrics.registry, cache, name) } - + override fun buildNonAsync(name: String, caffeine: Caffeine): Cache { + val cache: Cache = caffeine + .executor(Runnable::run) + .recordStats() + .build() + return CaffeineCacheMetrics.monitor(CordaMetrics.registry, cache, name) + } override fun build(name: String, caffeine: Caffeine, loader: CacheLoader): LoadingCache { val cache: LoadingCache = caffeine .executor(SecManagerForkJoinPool.pool) From 9bafc926600423f5c1bdfa9501a461fe8f36cd84 Mon Sep 17 00:00:00 2001 From: Nikolett Nagy <61757742+nikinagy@users.noreply.github.com> Date: Fri, 13 Oct 2023 10:25:33 +0100 Subject: [PATCH 11/81] CORE-17368 - Revert fix for missing config 'frequencyOfExpirationPoll' in ExpirationProcessor (#4839) We added a new config value to define the polling frequency for the ExpirationProcessor. Previously, the processor failed to reach a healthy lifecycle state when we did a platform upgrade, because there was no mechanism to push the newly introduced config values to Kafka. We added a temporary fix which would set the value of the frequencyOfExpirationPoll to a default in case of a missing config exception happens on upgrade. We now have in place a fix to force initial config reconciliation on platform upgrade scenarios, so this temporary fix should be safe to remove. --- .../dynamic/mgm/ExpirationProcessorImpl.kt | 24 +++++--------- .../dynamic/mgm/ExpirationProcessorTest.kt | 31 ++++++------------- 2 files changed, 18 insertions(+), 37 deletions(-) diff --git a/components/membership/registration-impl/src/main/kotlin/net/corda/membership/impl/registration/dynamic/mgm/ExpirationProcessorImpl.kt b/components/membership/registration-impl/src/main/kotlin/net/corda/membership/impl/registration/dynamic/mgm/ExpirationProcessorImpl.kt index a2cdcd44555..9e84031ce2f 100644 --- a/components/membership/registration-impl/src/main/kotlin/net/corda/membership/impl/registration/dynamic/mgm/ExpirationProcessorImpl.kt +++ b/components/membership/registration-impl/src/main/kotlin/net/corda/membership/impl/registration/dynamic/mgm/ExpirationProcessorImpl.kt @@ -1,6 +1,5 @@ package net.corda.membership.impl.registration.dynamic.mgm -import com.typesafe.config.ConfigException import net.corda.configuration.read.ConfigChangedEvent import net.corda.configuration.read.ConfigurationReadService import net.corda.crypto.core.ShortHash @@ -90,9 +89,6 @@ internal class ExpirationProcessorImpl internal constructor( const val WAIT_FOR_CONFIG_RESOURCE_NAME = "ExpirationProcessor.registerComponentForUpdates" const val PUBLISHER_RESOURCE_NAME = "ExpirationProcessor.publisher" const val PUBLISHER_CLIENT_ID = "expiration-processor" - - const val MAX_DURATION_BETWEEN_EXPIRED_REGISTRATION_REQUESTS_POLLS_DEFAULT = 300L - const val EXPIRATION_DATE_FOR_REGISTRATION_REQUESTS_DEFAULT = 180L } private val coordinatorName = LifecycleCoordinatorName.forComponent() @@ -146,16 +142,14 @@ internal class ExpirationProcessorImpl internal constructor( ) : TimerEvent private inner class ActiveImpl(membershipConfiguration: SmartConfig) : InnerExpirationProcessor { - private val expirationDate = try { - membershipConfiguration.getLong(MAX_DURATION_BETWEEN_EXPIRED_REGISTRATION_REQUESTS_POLLS).toMillis() - } catch(e: ConfigException.Missing) { - MAX_DURATION_BETWEEN_EXPIRED_REGISTRATION_REQUESTS_POLLS_DEFAULT.toMillis() - } - private val timeframe = try { - membershipConfiguration.getLong(EXPIRATION_DATE_FOR_REGISTRATION_REQUESTS).toMillis() - } catch(e: ConfigException.Missing) { - EXPIRATION_DATE_FOR_REGISTRATION_REQUESTS_DEFAULT.toMillis() - } + private val expirationDate = membershipConfiguration + .getLong(MAX_DURATION_BETWEEN_EXPIRED_REGISTRATION_REQUESTS_POLLS).let { + TimeUnit.MINUTES.toMillis(it) + } + private val timeframe = membershipConfiguration + .getLong(EXPIRATION_DATE_FOR_REGISTRATION_REQUESTS).let { + TimeUnit.MINUTES.toMillis(it) + } private val maxNoise = (0.1 * timeframe).toInt() override fun cancelOrScheduleProcessingOfExpiredRequests(mgm: HoldingIdentity): Boolean { @@ -309,6 +303,4 @@ internal class ExpirationProcessorImpl internal constructor( ?.publish(records) ?.forEach { it.join() } } - - private fun Long.toMillis() = TimeUnit.MINUTES.toMillis(this) } \ No newline at end of file diff --git a/components/membership/registration-impl/src/test/kotlin/net/corda/membership/impl/registration/dynamic/mgm/ExpirationProcessorTest.kt b/components/membership/registration-impl/src/test/kotlin/net/corda/membership/impl/registration/dynamic/mgm/ExpirationProcessorTest.kt index d2b4e73afd7..d2225b80fbc 100644 --- a/components/membership/registration-impl/src/test/kotlin/net/corda/membership/impl/registration/dynamic/mgm/ExpirationProcessorTest.kt +++ b/components/membership/registration-impl/src/test/kotlin/net/corda/membership/impl/registration/dynamic/mgm/ExpirationProcessorTest.kt @@ -1,6 +1,5 @@ package net.corda.membership.impl.registration.dynamic.mgm -import com.typesafe.config.ConfigException import net.corda.configuration.read.ConfigChangedEvent import net.corda.configuration.read.ConfigurationReadService import net.corda.data.membership.command.registration.RegistrationCommand @@ -246,36 +245,26 @@ class ExpirationProcessorTest { verify(publisher).start() } - // test required for 5.0 to 5.1 platform upgrade @Test - fun `use hard-coded defaults until newest version of config is received via reconciliation`() { + fun `use values from config to configure the processor`() { + val capturedTimeframes = argumentCaptor() + val capturedEvents = argumentCaptor<(String) -> TimerEvent>() val dummyTime = 100L whenever(membershipConfig.getLong(MAX_DURATION_BETWEEN_EXPIRED_REGISTRATION_REQUESTS_POLLS)) - .thenThrow(ConfigException.Missing("")).thenReturn(dummyTime) + .thenReturn(dummyTime) whenever(membershipConfig.getLong(EXPIRATION_DATE_FOR_REGISTRATION_REQUESTS)) - .thenThrow(ConfigException.Missing("")).thenReturn(dummyTime) + .thenReturn(dummyTime) - val capturedTimeframes = argumentCaptor() - val capturedEvents = argumentCaptor<(String) -> TimerEvent>() - - handler.firstValue.processEvent(configChangedEvent, coordinator) - // new config received handler.firstValue.processEvent(configChangedEvent, coordinator) - verify(coordinator, times(2)).setTimer(any(), capturedTimeframes.capture(), capturedEvents.capture()) + verify(coordinator, times(1)).setTimer(any(), capturedTimeframes.capture(), capturedEvents.capture()) SoftAssertions.assertSoftly { - it.assertThat(capturedTimeframes.firstValue).isLessThanOrEqualTo(TimeUnit.MINUTES.toMillis(180)) - it.assertThat(capturedTimeframes.secondValue).isLessThanOrEqualTo(TimeUnit.MINUTES.toMillis(dummyTime)) - - val firstEvent = capturedEvents.firstValue.invoke("") - it.assertThat(firstEvent).isInstanceOf(ExpirationProcessorImpl.DeclineExpiredRegistrationRequests::class.java) - val declineEvent = firstEvent as ExpirationProcessorImpl.DeclineExpiredRegistrationRequests - it.assertThat(declineEvent.expirationDate).isEqualTo(TimeUnit.MINUTES.toMillis(300)) + it.assertThat(capturedTimeframes.firstValue).isLessThanOrEqualTo(TimeUnit.MINUTES.toMillis(dummyTime)) - val secondEvent = capturedEvents.secondValue.invoke("") - it.assertThat(secondEvent).isInstanceOf(ExpirationProcessorImpl.DeclineExpiredRegistrationRequests::class.java) - val declineEventAfterUpdate = secondEvent as ExpirationProcessorImpl.DeclineExpiredRegistrationRequests + val capturedEvent = capturedEvents.firstValue.invoke("") + it.assertThat(capturedEvent).isInstanceOf(ExpirationProcessorImpl.DeclineExpiredRegistrationRequests::class.java) + val declineEventAfterUpdate = capturedEvent as ExpirationProcessorImpl.DeclineExpiredRegistrationRequests it.assertThat(declineEventAfterUpdate.expirationDate).isEqualTo(TimeUnit.MINUTES.toMillis(dummyTime)) } } From f1be93e60b1df52e23cc87bfca036784acf8c8ad Mon Sep 17 00:00:00 2001 From: Jenny Yang <135036209+jennyang-r3@users.noreply.github.com> Date: Fri, 13 Oct 2023 11:20:25 +0100 Subject: [PATCH 12/81] CORE-17688: Fix sandbox level cache (#4841) Update sandbox level cache to receive virtualNodeContext to fix CPI upgrade failure --- .../testing/context/FlowServiceTestContext.kt | 9 +++- .../crypto/MySigningKeysCacheImpl.kt | 11 ++-- .../fiber/cache/impl/FlowFiberCacheImpl.kt | 14 ++--- .../crypto/MySigningKeysCacheImplTest.kt | 47 ++++++++++++---- .../impl/cache/impl/StateAndRefCacheImpl.kt | 7 ++- .../persistence/GroupParametersCacheImpl.kt | 11 ++-- .../cache/impl/StateAndRefCacheImplTest.kt | 53 +++++++++++++++---- .../GroupParametersCacheImplTest.kt | 40 +++++++++++--- .../sandboxgroupcontext/SandboxedCache.kt | 29 ++++------ 9 files changed, 152 insertions(+), 69 deletions(-) diff --git a/components/flow/flow-service/src/integrationTest/kotlin/net/corda/flow/testing/context/FlowServiceTestContext.kt b/components/flow/flow-service/src/integrationTest/kotlin/net/corda/flow/testing/context/FlowServiceTestContext.kt index c3901d96a73..6394cd846c1 100644 --- a/components/flow/flow-service/src/integrationTest/kotlin/net/corda/flow/testing/context/FlowServiceTestContext.kt +++ b/components/flow/flow-service/src/integrationTest/kotlin/net/corda/flow/testing/context/FlowServiceTestContext.kt @@ -37,6 +37,7 @@ import net.corda.flow.testing.fakes.FakeFlowFiberFactory import net.corda.flow.testing.fakes.FakeMembershipGroupReaderProvider import net.corda.flow.testing.fakes.FakeSandboxGroupContextComponent import net.corda.flow.testing.tests.ALL_TEST_VIRTUAL_NODES +import net.corda.flow.testing.tests.CPK1_CHECKSUM import net.corda.flow.testing.tests.FLOW_NAME import net.corda.flow.testing.tests.SESSION_PROPERTIES import net.corda.flow.utils.KeyValueStore @@ -55,6 +56,8 @@ import net.corda.libs.packaging.core.CpkType import net.corda.messaging.api.processor.StateAndEventProcessor import net.corda.messaging.api.processor.StateAndEventProcessor.State import net.corda.messaging.api.records.Record +import net.corda.sandboxgroupcontext.SandboxGroupType.FLOW +import net.corda.sandboxgroupcontext.VirtualNodeContext import net.corda.schema.Schemas.Flow.FLOW_EVENT_TOPIC import net.corda.schema.configuration.ConfigKeys.FLOW_CONFIG import net.corda.schema.configuration.FlowConfig @@ -421,8 +424,12 @@ class FlowServiceTestContext @Activate constructor( } override fun resetFlowFiberCache() { - ALL_TEST_VIRTUAL_NODES.forEach { flowFiberCache.remove(it.toCorda()) } + ALL_TEST_VIRTUAL_NODES.forEach { + flowFiberCache.remove( + VirtualNodeContext(it.toCorda(), setOf(CPK1_CHECKSUM), FLOW, null) + ) } +} fun clearTestRuns() { testRuns.clear() diff --git a/components/flow/flow-service/src/main/kotlin/net/corda/flow/application/crypto/MySigningKeysCacheImpl.kt b/components/flow/flow-service/src/main/kotlin/net/corda/flow/application/crypto/MySigningKeysCacheImpl.kt index 0bf8752b865..704701fff0a 100644 --- a/components/flow/flow-service/src/main/kotlin/net/corda/flow/application/crypto/MySigningKeysCacheImpl.kt +++ b/components/flow/flow-service/src/main/kotlin/net/corda/flow/application/crypto/MySigningKeysCacheImpl.kt @@ -10,7 +10,6 @@ import net.corda.sandboxgroupcontext.SandboxedCache.CacheKey import net.corda.sandboxgroupcontext.VirtualNodeContext import net.corda.sandboxgroupcontext.service.CacheEviction import net.corda.utilities.debug -import net.corda.virtualnode.HoldingIdentity import org.osgi.service.component.annotations.Activate import org.osgi.service.component.annotations.Component import org.osgi.service.component.annotations.Deactivate @@ -58,10 +57,10 @@ class MySigningKeysCacheImpl @Activate constructor( private fun onEviction(vnc: VirtualNodeContext) { log.debug { - "Evicting cached items from ${cache::class.java} with holding identity: ${vnc.holdingIdentity} and sandbox type: " + - SandboxGroupType.FLOW + "Evicting cached items from ${cache::class.java} with holding identity: ${vnc.holdingIdentity}, " + + "cpkFileChecksums: ${vnc.cpkFileChecksums} and sandbox type: ${SandboxGroupType.FLOW}" } - remove(vnc.holdingIdentity) + remove(vnc) } override fun get(keys: Set): Map { @@ -82,8 +81,8 @@ class MySigningKeysCacheImpl @Activate constructor( } } - override fun remove(holdingIdentity: HoldingIdentity) { - cache.invalidateAll(cache.asMap().keys.filter { it.holdingIdentity == holdingIdentity }) + override fun remove(virtualNodeContext: VirtualNodeContext) { + cache.invalidateAll(cache.asMap().keys.filter { it.virtualNodeContext == virtualNodeContext }) cache.cleanUp() } } \ No newline at end of file diff --git a/components/flow/flow-service/src/main/kotlin/net/corda/flow/fiber/cache/impl/FlowFiberCacheImpl.kt b/components/flow/flow-service/src/main/kotlin/net/corda/flow/fiber/cache/impl/FlowFiberCacheImpl.kt index b98a7fe4e41..93e6d69579d 100644 --- a/components/flow/flow-service/src/main/kotlin/net/corda/flow/fiber/cache/impl/FlowFiberCacheImpl.kt +++ b/components/flow/flow-service/src/main/kotlin/net/corda/flow/fiber/cache/impl/FlowFiberCacheImpl.kt @@ -11,7 +11,6 @@ import net.corda.sandboxgroupcontext.SandboxedCache import net.corda.sandboxgroupcontext.VirtualNodeContext import net.corda.sandboxgroupcontext.service.CacheEviction import net.corda.utilities.debug -import net.corda.virtualnode.HoldingIdentity import net.corda.virtualnode.toAvro import org.osgi.service.component.annotations.Activate import org.osgi.service.component.annotations.Component @@ -59,10 +58,10 @@ class FlowFiberCacheImpl @Activate constructor( private fun onEviction(vnc: VirtualNodeContext) { logger.debug { - "Evicting cached items from ${cache::class.java} with holding identity: ${vnc.holdingIdentity} and sandbox type: " + - SandboxGroupType.FLOW + "Evicting cached items from ${cache::class.java} with holding identity: ${vnc.holdingIdentity}, " + + "cpkFileChecksums: ${vnc.cpkFileChecksums} and sandbox type: ${SandboxGroupType.FLOW}" } - remove(vnc.holdingIdentity) + remove(vnc) } override fun put(key: FlowKey, fiber: FlowFiberImpl) { @@ -83,9 +82,10 @@ class FlowFiberCacheImpl @Activate constructor( cache.cleanUp() } - override fun remove(holdingIdentity: HoldingIdentity) { - logger.debug { "Flow fiber cache removing holdingIdentity $holdingIdentity" } - val holdingIdentityToRemove = holdingIdentity.toAvro() + override fun remove(virtualNodeContext: VirtualNodeContext) { + logger.debug { + "Flow fiber cache removing holdingIdentity ${virtualNodeContext.holdingIdentity}" } + val holdingIdentityToRemove = virtualNodeContext.holdingIdentity.toAvro() val keysToInvalidate = cache.asMap().keys.filter { holdingIdentityToRemove == it.identity } cache.invalidateAll(keysToInvalidate) cache.cleanUp() diff --git a/components/flow/flow-service/src/test/kotlin/net/corda/flow/application/crypto/MySigningKeysCacheImplTest.kt b/components/flow/flow-service/src/test/kotlin/net/corda/flow/application/crypto/MySigningKeysCacheImplTest.kt index ea290818904..d35a7821874 100644 --- a/components/flow/flow-service/src/test/kotlin/net/corda/flow/application/crypto/MySigningKeysCacheImplTest.kt +++ b/components/flow/flow-service/src/test/kotlin/net/corda/flow/application/crypto/MySigningKeysCacheImplTest.kt @@ -1,10 +1,11 @@ package net.corda.flow.application.crypto +import net.corda.crypto.core.SecureHashImpl import net.corda.flow.ALICE_X500_HOLDING_IDENTITY import net.corda.flow.BOB_X500_HOLDING_IDENTITY import net.corda.sandboxgroupcontext.CurrentSandboxGroupContext import net.corda.sandboxgroupcontext.SandboxGroupContext -import net.corda.sandboxgroupcontext.SandboxGroupType +import net.corda.sandboxgroupcontext.SandboxGroupType.FLOW import net.corda.sandboxgroupcontext.VirtualNodeContext import net.corda.sandboxgroupcontext.service.CacheEviction import net.corda.virtualnode.toCorda @@ -22,10 +23,24 @@ class MySigningKeysCacheImplTest { val KEY_B = mock() val KEY_C = mock() val KEY_D = mock() + val CPK1_CHECKSUM = SecureHashImpl("ALG", byteArrayOf(0, 0, 0, 0)) } private val sandbox = mock() private val virtualNodeContext = mock() + private val aliceVirtualNodeContext = VirtualNodeContext( + ALICE_X500_HOLDING_IDENTITY.toCorda(), + setOf(CPK1_CHECKSUM), + FLOW, + null + ) + private val bobVirtualNodeContext = VirtualNodeContext( + BOB_X500_HOLDING_IDENTITY.toCorda(), + setOf(CPK1_CHECKSUM), + FLOW, + null + ) + private val currentSandboxGroupContext = mock() private val cacheEviction = mock() private val mySigningKeysCache = MySigningKeysCacheImpl(currentSandboxGroupContext, cacheEviction) @@ -33,7 +48,7 @@ class MySigningKeysCacheImplTest { @BeforeEach fun beforeEach() { whenever(sandbox.virtualNodeContext).thenReturn(virtualNodeContext) - whenever(virtualNodeContext.sandboxGroupType).thenReturn(SandboxGroupType.FLOW) + whenever(virtualNodeContext.sandboxGroupType).thenReturn(FLOW) whenever(virtualNodeContext.holdingIdentity).thenReturn(ALICE_X500_HOLDING_IDENTITY.toCorda()) whenever(currentSandboxGroupContext.get()).thenReturn(sandbox) } @@ -70,16 +85,28 @@ class MySigningKeysCacheImplTest { } @Test - fun `removes keys by holding identity`() { - whenever(virtualNodeContext.holdingIdentity).thenReturn( - ALICE_X500_HOLDING_IDENTITY.toCorda(), - ALICE_X500_HOLDING_IDENTITY.toCorda(), - BOB_X500_HOLDING_IDENTITY.toCorda(), - BOB_X500_HOLDING_IDENTITY.toCorda() + fun `removes keys by virtual node context`() { + // return vnode in this order in consecutive calls of the function (alice, bob, alice, bob) + whenever(sandbox.virtualNodeContext).thenReturn( + aliceVirtualNodeContext, + bobVirtualNodeContext, + aliceVirtualNodeContext, + bobVirtualNodeContext, + ) + whenever(currentSandboxGroupContext.get()).thenReturn(sandbox) + + // alice put cache + mySigningKeysCache.putAll(mapOf(KEY_A to KEY_A, KEY_B to null)) + // bob put cache + mySigningKeysCache.putAll(mapOf(KEY_C to KEY_C, KEY_D to null)) + mySigningKeysCache.remove(aliceVirtualNodeContext) + + // alice's cache should be empty + assertThat(mySigningKeysCache.get(setOf(KEY_A, KEY_B, KEY_C, KEY_D))).containsExactlyInAnyOrderEntriesOf( + emptyMap() ) - mySigningKeysCache.putAll(mapOf(KEY_A to KEY_A, KEY_B to null, KEY_C to KEY_C, KEY_D to null)) - mySigningKeysCache.remove(ALICE_X500_HOLDING_IDENTITY.toCorda()) + // there should bob's cache only assertThat(mySigningKeysCache.get(setOf(KEY_A, KEY_B, KEY_C, KEY_D))).containsExactlyInAnyOrderEntriesOf( mapOf( KEY_C to KEY_C, diff --git a/components/ledger/ledger-utxo-flow/src/main/kotlin/net/corda/ledger/utxo/flow/impl/cache/impl/StateAndRefCacheImpl.kt b/components/ledger/ledger-utxo-flow/src/main/kotlin/net/corda/ledger/utxo/flow/impl/cache/impl/StateAndRefCacheImpl.kt index e93807c665c..63db7203d84 100644 --- a/components/ledger/ledger-utxo-flow/src/main/kotlin/net/corda/ledger/utxo/flow/impl/cache/impl/StateAndRefCacheImpl.kt +++ b/components/ledger/ledger-utxo-flow/src/main/kotlin/net/corda/ledger/utxo/flow/impl/cache/impl/StateAndRefCacheImpl.kt @@ -12,7 +12,6 @@ import net.corda.sandboxgroupcontext.service.CacheEviction import net.corda.utilities.debug import net.corda.v5.ledger.utxo.StateAndRef import net.corda.v5.ledger.utxo.StateRef -import net.corda.virtualnode.HoldingIdentity import org.osgi.service.component.annotations.Activate import org.osgi.service.component.annotations.Component import org.osgi.service.component.annotations.Deactivate @@ -68,8 +67,8 @@ class StateAndRefCacheImpl @Activate constructor( } } - override fun remove(holdingIdentity: HoldingIdentity) { - cache.invalidateAll(cache.asMap().keys.filter { it.holdingIdentity == holdingIdentity }) + override fun remove(virtualNodeContext: VirtualNodeContext) { + cache.invalidateAll(cache.asMap().keys.filter { it.virtualNodeContext == virtualNodeContext }) cache.cleanUp() } @@ -86,6 +85,6 @@ class StateAndRefCacheImpl @Activate constructor( "Evicting cached items from ${cache::class.java} with holding identity: ${vnc.holdingIdentity} and sandbox type: " + SandboxGroupType.FLOW } - remove(vnc.holdingIdentity) + remove(vnc) } } diff --git a/components/ledger/ledger-utxo-flow/src/main/kotlin/net/corda/ledger/utxo/flow/impl/persistence/GroupParametersCacheImpl.kt b/components/ledger/ledger-utxo-flow/src/main/kotlin/net/corda/ledger/utxo/flow/impl/persistence/GroupParametersCacheImpl.kt index 0d1950a9602..e715eeceb8d 100644 --- a/components/ledger/ledger-utxo-flow/src/main/kotlin/net/corda/ledger/utxo/flow/impl/persistence/GroupParametersCacheImpl.kt +++ b/components/ledger/ledger-utxo-flow/src/main/kotlin/net/corda/ledger/utxo/flow/impl/persistence/GroupParametersCacheImpl.kt @@ -12,7 +12,6 @@ import net.corda.sandboxgroupcontext.VirtualNodeContext import net.corda.sandboxgroupcontext.service.CacheEviction import net.corda.utilities.debug import net.corda.v5.crypto.SecureHash -import net.corda.virtualnode.HoldingIdentity import org.osgi.service.component.annotations.Activate import org.osgi.service.component.annotations.Component import org.osgi.service.component.annotations.Deactivate @@ -58,10 +57,10 @@ class GroupParametersCacheImpl @Activate constructor( private fun onEviction(vnc: VirtualNodeContext) { log.debug { - "Evicting cached items from ${cache::class.java} with holding identity: ${vnc.holdingIdentity} and sandbox type: " + - SandboxGroupType.FLOW + "Evicting cached items from ${cache::class.java} with holding identity: ${vnc.holdingIdentity}, " + + "cpkFileChecksums: ${vnc.cpkFileChecksums} and sandbox type: ${SandboxGroupType.FLOW}" } - remove(vnc.holdingIdentity) + remove(vnc) } override fun get(hash: SecureHash): SignedGroupParameters? { @@ -74,8 +73,8 @@ class GroupParametersCacheImpl @Activate constructor( cache.put(CacheKey(virtualNodeContext, groupParameters.hash), groupParameters) } - override fun remove(holdingIdentity: HoldingIdentity) { - cache.invalidateAll(cache.asMap().keys.filter { it.holdingIdentity == holdingIdentity }) + override fun remove(virtualNodeContext: VirtualNodeContext) { + cache.invalidateAll(cache.asMap().keys.filter { it.virtualNodeContext == virtualNodeContext }) cache.cleanUp() } } \ No newline at end of file diff --git a/components/ledger/ledger-utxo-flow/src/test/kotlin/net/corda/ledger/utxo/flow/impl/cache/impl/StateAndRefCacheImplTest.kt b/components/ledger/ledger-utxo-flow/src/test/kotlin/net/corda/ledger/utxo/flow/impl/cache/impl/StateAndRefCacheImplTest.kt index 1bdc3c16e08..86f7da677df 100644 --- a/components/ledger/ledger-utxo-flow/src/test/kotlin/net/corda/ledger/utxo/flow/impl/cache/impl/StateAndRefCacheImplTest.kt +++ b/components/ledger/ledger-utxo-flow/src/test/kotlin/net/corda/ledger/utxo/flow/impl/cache/impl/StateAndRefCacheImplTest.kt @@ -1,9 +1,10 @@ package net.corda.ledger.utxo.flow.impl.cache.impl +import net.corda.crypto.core.SecureHashImpl import net.corda.data.identity.HoldingIdentity import net.corda.sandboxgroupcontext.CurrentSandboxGroupContext import net.corda.sandboxgroupcontext.SandboxGroupContext -import net.corda.sandboxgroupcontext.SandboxGroupType +import net.corda.sandboxgroupcontext.SandboxGroupType.FLOW import net.corda.sandboxgroupcontext.VirtualNodeContext import net.corda.sandboxgroupcontext.service.CacheEviction import net.corda.v5.ledger.utxo.ContractState @@ -21,16 +22,33 @@ class StateAndRefCacheImplTest { private companion object { val STATE_REF_1 = StateRef(mock(), 0) val STATE_REF_2 = StateRef(mock(), 0) + val STATE_REF_3 = StateRef(mock(), 0) + val STATE_REF_4 = StateRef(mock(), 0) val STATE_AND_REF_1 = mock>() val STATE_AND_REF_2 = mock>() + val STATE_AND_REF_3 = mock>() + val STATE_AND_REF_4 = mock>() val ALICE_X500_HOLDING_IDENTITY = HoldingIdentity("CN=Alice, O=Alice Corp, L=LDN, C=GB", "group1") val BOB_X500_HOLDING_IDENTITY = HoldingIdentity("CN=Bob, O=Alice Corp, L=LDN, C=GB", "group1") + val CPK1_CHECKSUM = SecureHashImpl("ALG", byteArrayOf(0, 0, 0, 0)) } private val sandbox = mock() private val virtualNodeContext = mock() + private val aliceVirtualNodeContext = VirtualNodeContext( + ALICE_X500_HOLDING_IDENTITY.toCorda(), + setOf(CPK1_CHECKSUM), + FLOW, + null + ) + private val bobVirtualNodeContext = VirtualNodeContext( + BOB_X500_HOLDING_IDENTITY.toCorda(), + setOf(CPK1_CHECKSUM), + FLOW, + null + ) private val currentSandboxGroupContext = mock() private val cacheEviction = mock() private val stateAndRefCache = StateAndRefCacheImpl(currentSandboxGroupContext, cacheEviction) @@ -38,12 +56,14 @@ class StateAndRefCacheImplTest { @BeforeEach fun beforeEach() { whenever(sandbox.virtualNodeContext).thenReturn(virtualNodeContext) - whenever(virtualNodeContext.sandboxGroupType).thenReturn(SandboxGroupType.FLOW) + whenever(virtualNodeContext.sandboxGroupType).thenReturn(FLOW) whenever(virtualNodeContext.holdingIdentity).thenReturn(ALICE_X500_HOLDING_IDENTITY.toCorda()) whenever(currentSandboxGroupContext.get()).thenReturn(sandbox) whenever(STATE_AND_REF_1.ref).thenReturn(STATE_REF_1) whenever(STATE_AND_REF_2.ref).thenReturn(STATE_REF_2) + whenever(STATE_AND_REF_3.ref).thenReturn(STATE_REF_3) + whenever(STATE_AND_REF_4.ref).thenReturn(STATE_REF_4) } @Test @@ -87,15 +107,30 @@ class StateAndRefCacheImplTest { } @Test - fun `removes keys by holding identity`() { - whenever(virtualNodeContext.holdingIdentity).thenReturn( - ALICE_X500_HOLDING_IDENTITY.toCorda(), - BOB_X500_HOLDING_IDENTITY.toCorda() + fun `removes keys by virtual node context`() { + // subsequent calls of the function call return in the order of alice, bob, alice, bob + whenever(sandbox.virtualNodeContext).thenReturn( + aliceVirtualNodeContext, + bobVirtualNodeContext, + aliceVirtualNodeContext, + bobVirtualNodeContext ) - stateAndRefCache.putAll(listOf(STATE_AND_REF_1, STATE_AND_REF_2)) + whenever(currentSandboxGroupContext.get()).thenReturn(sandbox) - stateAndRefCache.remove(ALICE_X500_HOLDING_IDENTITY.toCorda()) + whenever(STATE_AND_REF_1.ref).thenReturn(STATE_REF_1) + whenever(STATE_AND_REF_2.ref).thenReturn(STATE_REF_2) - assertThat(stateAndRefCache.get(setOf(STATE_REF_1, STATE_REF_2)).values).containsExactly(STATE_AND_REF_2) + stateAndRefCache.putAll(listOf(STATE_AND_REF_1, STATE_AND_REF_2)) + stateAndRefCache.putAll(listOf(STATE_AND_REF_3, STATE_AND_REF_4)) + stateAndRefCache.remove(aliceVirtualNodeContext) + + // alice's cache should be empty + assertThat( + stateAndRefCache.get(setOf(STATE_REF_1, STATE_REF_2, STATE_REF_3, STATE_REF_4)).values + ).containsExactlyElementsOf(emptyList()) + // bob's cache should exist + assertThat( + stateAndRefCache.get(setOf(STATE_REF_1, STATE_REF_2, STATE_REF_3, STATE_REF_4)).values + ).containsExactlyElementsOf(listOf(STATE_AND_REF_3, STATE_AND_REF_4)) } } diff --git a/components/ledger/ledger-utxo-flow/src/test/kotlin/net/corda/ledger/utxo/flow/impl/persistence/GroupParametersCacheImplTest.kt b/components/ledger/ledger-utxo-flow/src/test/kotlin/net/corda/ledger/utxo/flow/impl/persistence/GroupParametersCacheImplTest.kt index 74489e8a469..5c77b682c6c 100644 --- a/components/ledger/ledger-utxo-flow/src/test/kotlin/net/corda/ledger/utxo/flow/impl/persistence/GroupParametersCacheImplTest.kt +++ b/components/ledger/ledger-utxo-flow/src/test/kotlin/net/corda/ledger/utxo/flow/impl/persistence/GroupParametersCacheImplTest.kt @@ -1,10 +1,11 @@ package net.corda.ledger.utxo.flow.impl.persistence +import net.corda.crypto.core.SecureHashImpl import net.corda.data.identity.HoldingIdentity import net.corda.membership.lib.SignedGroupParameters import net.corda.sandboxgroupcontext.CurrentSandboxGroupContext import net.corda.sandboxgroupcontext.SandboxGroupContext -import net.corda.sandboxgroupcontext.SandboxGroupType +import net.corda.sandboxgroupcontext.SandboxGroupType.FLOW import net.corda.sandboxgroupcontext.VirtualNodeContext import net.corda.sandboxgroupcontext.service.CacheEviction import net.corda.v5.crypto.SecureHash @@ -25,10 +26,24 @@ class GroupParametersCacheImplTest { val ALICE_X500_HOLDING_IDENTITY = HoldingIdentity("CN=Alice, O=Alice Corp, L=LDN, C=GB", "group1") val BOB_X500_HOLDING_IDENTITY = HoldingIdentity("CN=Bob, O=Alice Corp, L=LDN, C=GB", "group1") + + val CPK1_CHECKSUM = SecureHashImpl("ALG", byteArrayOf(0, 0, 0, 0)) } private val sandbox = mock() private val virtualNodeContext = mock() + private val aliceVirtualNodeContext = VirtualNodeContext( + ALICE_X500_HOLDING_IDENTITY.toCorda(), + setOf(CPK1_CHECKSUM), + FLOW, + null + ) + private val bobVirtualNodeContext = VirtualNodeContext( + BOB_X500_HOLDING_IDENTITY.toCorda(), + setOf(CPK1_CHECKSUM), + FLOW, + null + ) private val currentSandboxGroupContext = mock() private val cacheEviction = mock() private val groupParametersCache = GroupParametersCacheImpl(currentSandboxGroupContext, cacheEviction) @@ -36,7 +51,7 @@ class GroupParametersCacheImplTest { @BeforeEach fun beforeEach() { whenever(sandbox.virtualNodeContext).thenReturn(virtualNodeContext) - whenever(virtualNodeContext.sandboxGroupType).thenReturn(SandboxGroupType.FLOW) + whenever(virtualNodeContext.sandboxGroupType).thenReturn(FLOW) whenever(virtualNodeContext.holdingIdentity).thenReturn(ALICE_X500_HOLDING_IDENTITY.toCorda()) whenever(currentSandboxGroupContext.get()).thenReturn(sandbox) whenever(GROUP_PARAMS_1.hash).thenReturn(KEY_1) @@ -57,16 +72,27 @@ class GroupParametersCacheImplTest { } @Test - fun `removes keys by holding identity`() { - whenever(virtualNodeContext.holdingIdentity).thenReturn( - ALICE_X500_HOLDING_IDENTITY.toCorda(), - BOB_X500_HOLDING_IDENTITY.toCorda() + fun `removes keys by virtual node context`() { + // return vnode in this order in consecutive calls of the function + whenever(sandbox.virtualNodeContext).thenReturn( + aliceVirtualNodeContext, + bobVirtualNodeContext, + aliceVirtualNodeContext, + bobVirtualNodeContext ) + whenever(currentSandboxGroupContext.get()).thenReturn(sandbox) + whenever(GROUP_PARAMS_1.hash).thenReturn(KEY_1) + whenever(GROUP_PARAMS_2.hash).thenReturn(KEY_2) + + // alice puts cache groupParametersCache.put(GROUP_PARAMS_1) + // bob puts cache groupParametersCache.put(GROUP_PARAMS_2) - groupParametersCache.remove(ALICE_X500_HOLDING_IDENTITY.toCorda()) + groupParametersCache.remove(aliceVirtualNodeContext) + // alice gets cache and it's null assertThat(groupParametersCache.get(KEY_1)).isNull() + // bob gets cache and it's group param 2 assertThat(groupParametersCache.get(KEY_2)).isEqualTo(GROUP_PARAMS_2) } } \ No newline at end of file diff --git a/libs/virtual-node/sandbox-group-context/src/main/kotlin/net/corda/sandboxgroupcontext/SandboxedCache.kt b/libs/virtual-node/sandbox-group-context/src/main/kotlin/net/corda/sandboxgroupcontext/SandboxedCache.kt index e8ba3aefa39..356dcf6fa58 100644 --- a/libs/virtual-node/sandbox-group-context/src/main/kotlin/net/corda/sandboxgroupcontext/SandboxedCache.kt +++ b/libs/virtual-node/sandbox-group-context/src/main/kotlin/net/corda/sandboxgroupcontext/SandboxedCache.kt @@ -1,17 +1,15 @@ package net.corda.sandboxgroupcontext -import net.corda.virtualnode.HoldingIdentity - /** * Instances of [SandboxedCache] are "sandbox-level" caches. * * These caches should exist as global OSGi singletons. * - * [SandboxedCache]s have invalidate keys related to a sandbox when that sandbox is evicted from the worker's sandbox cache. [remove] is - * called when this occurs. + * [SandboxedCache]s have invalidate keys related to a sandbox when that sandbox is evicted from + * the worker's sandbox cache. [remove] is called when this occurs. * - * All [SandboxedCache]s should contain an internal cache that contains _at least_ the [HoldingIdentity] of the sandbox that put key-value - * pairs into the cache. + * All [SandboxedCache]s should contain an internal cache that contains _at least_ the [VirtualNodeContext] of + * the sandbox that put key-value pairs into the cache. * * Ideally [CacheKey] should be used by a [SandboxedCache]'s internal cache. */ @@ -20,25 +18,18 @@ interface SandboxedCache { /** * A cache key holding sandbox information and the _real_ key to the cache. * - * @property holdingIdentity The [HoldingIdentity] of the sandbox that added the key-value pair. + * @property virtualNodeContext The [VirtualNodeContext] of the sandbox that added the key-value pair. * @property key The real key of the cache. */ - data class CacheKey(val holdingIdentity: HoldingIdentity, val key: T) { - - /** - * @param virtualNodeContext The [VirtualNodeContext] of the sandbox that added the key-value pair. - * @param key The real key of the cache. - */ - constructor(virtualNodeContext: VirtualNodeContext, key: T) : this(virtualNodeContext.holdingIdentity, key) - } + data class CacheKey(val virtualNodeContext: VirtualNodeContext, val key: T) /** - * Removes key-value pairs from the cache based on [HoldingIdentity]. + * Removes key-value pairs from the cache based on [VirtualNodeContext]. * - * Implementations of this method should iterate over the keys to remove the ones that match the [holdingIdentity]. + * Implementations of this method should iterate over the keys to remove the ones that match the [virtualNodeContext]. * - * @param holdingIdentity The [HoldingIdentity] of the keys to remove from the cache. + * @param virtualNodeContext The [VirtualNodeContext] of the keys to remove from the cache. */ - fun remove(holdingIdentity: HoldingIdentity) + fun remove(virtualNodeContext: VirtualNodeContext) } \ No newline at end of file From 040c4be0f254eebad3976e5fad27d15d99da4bc4 Mon Sep 17 00:00:00 2001 From: Emily Bowe Date: Fri, 13 Oct 2023 11:21:06 +0100 Subject: [PATCH 13/81] CORE-16234: For persistence api add endpoint and change naming of existing endpoint (#4774) For the RPC Epic the Persistence api will require two endpoints. One endpoint is already merged which relates to ledger-related persistence. This PR will be making a suggested change to the naming of that endpoint for clarity, and will also add the second endpoint required. When this work is merged we will have the endpoints: /ledger - for ledger persistence /persistence - for entity persistence --- ...rsistenceRequestSubscriptionFactoryImpl.kt | 6 +- .../VerificationSubscriptionFactoryImpl.kt | 4 +- .../build.gradle | 1 + .../impl/tests/PersistenceExceptionTests.kt | 23 +-- .../tests/PersistenceServiceInternalTests.kt | 25 +-- .../impl/tests/helpers/Utils.kt | 14 ++ .../impl/EntityProcessorImpl.kt | 27 --- ...> EntityRequestSubscriptionFactoryImpl.kt} | 37 +++- .../impl/FlowPersistenceServiceImpl.kt | 44 +++-- .../impl/internal/EntityRequestProcessor.kt | 79 ++++++++ .../internal/EntityRpcRequestProcessor.kt | 50 +++++ ...essageProcessor.kt => ProcessorService.kt} | 171 ++++++------------ .../entityprocessor/impl/internal/Utils.kt | 8 + .../impl/EntityProcessorFactoryImplTest.kt | 16 -- ...ntityRequestSubscriptionFactoryImplTest.kt | 81 +++++++++ .../entity-processor-service/build.gradle | 5 + .../corda/entityprocessor/EntityProcessor.kt | 5 - .../entityprocessor/EntityProcessorFactory.kt | 13 -- .../EntityRequestSubscriptionFactory.kt | 26 +++ .../BatchedUniquenessCheckerLifecycleImpl.kt | 4 +- 20 files changed, 406 insertions(+), 233 deletions(-) create mode 100644 components/persistence/entity-processor-service-impl/src/integrationTest/kotlin/net/corda/entityprocessor/impl/tests/helpers/Utils.kt delete mode 100644 components/persistence/entity-processor-service-impl/src/main/kotlin/net/corda/entityprocessor/impl/EntityProcessorImpl.kt rename components/persistence/entity-processor-service-impl/src/main/kotlin/net/corda/entityprocessor/impl/{EntityProcessorFactoryImpl.kt => EntityRequestSubscriptionFactoryImpl.kt} (53%) create mode 100644 components/persistence/entity-processor-service-impl/src/main/kotlin/net/corda/entityprocessor/impl/internal/EntityRequestProcessor.kt create mode 100644 components/persistence/entity-processor-service-impl/src/main/kotlin/net/corda/entityprocessor/impl/internal/EntityRpcRequestProcessor.kt rename components/persistence/entity-processor-service-impl/src/main/kotlin/net/corda/entityprocessor/impl/internal/{EntityMessageProcessor.kt => ProcessorService.kt} (53%) create mode 100644 components/persistence/entity-processor-service-impl/src/main/kotlin/net/corda/entityprocessor/impl/internal/Utils.kt delete mode 100644 components/persistence/entity-processor-service-impl/src/test/kotlin/net/corda/entityprocessor/impl/EntityProcessorFactoryImplTest.kt create mode 100644 components/persistence/entity-processor-service-impl/src/test/kotlin/net/corda/entityprocessor/impl/EntityRequestSubscriptionFactoryImplTest.kt delete mode 100644 components/persistence/entity-processor-service/src/main/kotlin/net/corda/entityprocessor/EntityProcessor.kt delete mode 100644 components/persistence/entity-processor-service/src/main/kotlin/net/corda/entityprocessor/EntityProcessorFactory.kt create mode 100644 components/persistence/entity-processor-service/src/main/kotlin/net/corda/entityprocessor/EntityRequestSubscriptionFactory.kt diff --git a/components/ledger/ledger-persistence/src/main/kotlin/net/corda/ledger/persistence/processor/impl/LedgerPersistenceRequestSubscriptionFactoryImpl.kt b/components/ledger/ledger-persistence/src/main/kotlin/net/corda/ledger/persistence/processor/impl/LedgerPersistenceRequestSubscriptionFactoryImpl.kt index 186ffac292b..4448a05bc6a 100644 --- a/components/ledger/ledger-persistence/src/main/kotlin/net/corda/ledger/persistence/processor/impl/LedgerPersistenceRequestSubscriptionFactoryImpl.kt +++ b/components/ledger/ledger-persistence/src/main/kotlin/net/corda/ledger/persistence/processor/impl/LedgerPersistenceRequestSubscriptionFactoryImpl.kt @@ -36,8 +36,8 @@ class LedgerPersistenceRequestSubscriptionFactoryImpl @Activate constructor( ) : LedgerPersistenceRequestSubscriptionFactory { companion object { internal const val GROUP_NAME = "persistence.ledger.processor" - const val SUBSCRIPTION_NAME = "Persistence" - const val PERSISTENCE_PATH = "/persistence" + const val SUBSCRIPTION_NAME = "Ledger" + const val PATH = "/ledger" } override fun create(config: SmartConfig): Subscription { @@ -67,7 +67,7 @@ class LedgerPersistenceRequestSubscriptionFactoryImpl @Activate constructor( LedgerPersistenceRequest::class.java, FlowEvent::class.java ) - val rpcConfig = SyncRPCConfig(SUBSCRIPTION_NAME, PERSISTENCE_PATH) + val rpcConfig = SyncRPCConfig(SUBSCRIPTION_NAME, PATH) return subscriptionFactory.createHttpRPCSubscription(rpcConfig, processor) } } diff --git a/components/ledger/ledger-verification/src/main/kotlin/net/corda/ledger/verification/processor/impl/VerificationSubscriptionFactoryImpl.kt b/components/ledger/ledger-verification/src/main/kotlin/net/corda/ledger/verification/processor/impl/VerificationSubscriptionFactoryImpl.kt index 8ccce636f81..e667596f645 100644 --- a/components/ledger/ledger-verification/src/main/kotlin/net/corda/ledger/verification/processor/impl/VerificationSubscriptionFactoryImpl.kt +++ b/components/ledger/ledger-verification/src/main/kotlin/net/corda/ledger/verification/processor/impl/VerificationSubscriptionFactoryImpl.kt @@ -31,7 +31,7 @@ class VerificationSubscriptionFactoryImpl @Activate constructor( companion object { internal const val GROUP_NAME = "verification.ledger.processor" const val SUBSCRIPTION_NAME = "Verification" - const val VERIFICATION_PATH = "/verification" + const val PATH = "/verification" } override fun create(config: SmartConfig): Subscription { @@ -61,7 +61,7 @@ class VerificationSubscriptionFactoryImpl @Activate constructor( TransactionVerificationRequest::class.java, FlowEvent::class.java ) - val rpcConfig = SyncRPCConfig(SUBSCRIPTION_NAME, VERIFICATION_PATH) + val rpcConfig = SyncRPCConfig(SUBSCRIPTION_NAME, PATH) return subscriptionFactory.createHttpRPCSubscription(rpcConfig, processor) } diff --git a/components/persistence/entity-processor-service-impl/build.gradle b/components/persistence/entity-processor-service-impl/build.gradle index 17bf5e0b876..c333ff49c66 100644 --- a/components/persistence/entity-processor-service-impl/build.gradle +++ b/components/persistence/entity-processor-service-impl/build.gradle @@ -41,6 +41,7 @@ dependencies { implementation project(":libs:serialization:serialization-avro") implementation 'net.corda:corda-application' + implementation 'net.corda:corda-avro-schema' implementation 'net.corda:corda-config-schema' implementation 'net.corda:corda-db-schema' implementation 'net.corda:corda-topic-schema' diff --git a/components/persistence/entity-processor-service-impl/src/integrationTest/kotlin/net/corda/entityprocessor/impl/tests/PersistenceExceptionTests.kt b/components/persistence/entity-processor-service-impl/src/integrationTest/kotlin/net/corda/entityprocessor/impl/tests/PersistenceExceptionTests.kt index f735b9a9e48..c83c27ed9b3 100644 --- a/components/persistence/entity-processor-service-impl/src/integrationTest/kotlin/net/corda/entityprocessor/impl/tests/PersistenceExceptionTests.kt +++ b/components/persistence/entity-processor-service-impl/src/integrationTest/kotlin/net/corda/entityprocessor/impl/tests/PersistenceExceptionTests.kt @@ -22,7 +22,9 @@ import net.corda.db.persistence.testkit.helpers.Resources import net.corda.db.persistence.testkit.helpers.SandboxHelper.createDog import net.corda.db.persistence.testkit.helpers.SandboxHelper.getDogClass import net.corda.db.schema.DbSchema -import net.corda.entityprocessor.impl.internal.EntityMessageProcessor +import net.corda.entityprocessor.impl.internal.EntityRequestProcessor +import net.corda.entityprocessor.impl.tests.helpers.assertEventResponseWithError +import net.corda.entityprocessor.impl.tests.helpers.assertEventResponseWithoutError import net.corda.flow.external.events.responses.exceptions.CpkNotAvailableException import net.corda.flow.external.events.responses.exceptions.VirtualNodeException import net.corda.flow.utils.toKeyValuePairList @@ -49,8 +51,6 @@ import net.corda.virtualnode.toAvro import org.assertj.core.api.Assertions.assertThat import org.junit.jupiter.api.AfterEach import org.junit.jupiter.api.Assertions.assertEquals -import org.junit.jupiter.api.Assertions.assertNotNull -import org.junit.jupiter.api.Assertions.assertNull import org.junit.jupiter.api.BeforeAll import org.junit.jupiter.api.Test import org.junit.jupiter.api.TestInstance @@ -105,7 +105,7 @@ class PersistenceExceptionTests { private lateinit var dbConnectionManager: FakeDbConnectionManager private lateinit var entitySandboxService: EntitySandboxService - private lateinit var processor: EntityMessageProcessor + private lateinit var processor: EntityRequestProcessor private lateinit var virtualNodeInfo: VirtualNodeInfo private lateinit var cpkFileHashes: Set @@ -157,7 +157,7 @@ class PersistenceExceptionTests { virtualNodeInfoReadService, dbConnectionManager ) - processor = EntityMessageProcessor( + processor = EntityRequestProcessor( currentSandboxGroupContext, entitySandboxService, responseFactory, @@ -217,7 +217,7 @@ class PersistenceExceptionTests { dbConnectionManager ) - val processor = EntityMessageProcessor( + val processor = EntityRequestProcessor( currentSandboxGroupContext, brokenEntitySandboxService, responseFactory, @@ -286,6 +286,8 @@ class PersistenceExceptionTests { createDogDb(DOGS_TABLE_WITHOUT_PK) val persistEntitiesRequest = createDogPersistRequest() + val initialDogDbCount = getDogDbCount(virtualNodeInfo.vaultDmlConnectionId) + val record1 = processor.onNext(listOf(Record(TOPIC, UUID.randomUUID().toString(), persistEntitiesRequest))) assertEventResponseWithoutError(record1.single()) // duplicate request @@ -294,7 +296,7 @@ class PersistenceExceptionTests { val dogDbCount = getDogDbCount(virtualNodeInfo.vaultDmlConnectionId) // There shouldn't be a dog duplicate entry in the DB, i.e. dogs count in the DB should still be 1 - assertEquals(1, dogDbCount) + assertEquals(initialDogDbCount + 1, dogDbCount) } @Test @@ -384,10 +386,3 @@ class PersistenceExceptionTests { } } -private fun assertEventResponseWithoutError(record: Record<*, *>) { - assertNull(((record.value as FlowEvent).payload as ExternalEventResponse).error) -} - -private fun assertEventResponseWithError(record: Record<*, *>) { - assertNotNull(((record.value as FlowEvent).payload as ExternalEventResponse).error) -} \ No newline at end of file diff --git a/components/persistence/entity-processor-service-impl/src/integrationTest/kotlin/net/corda/entityprocessor/impl/tests/PersistenceServiceInternalTests.kt b/components/persistence/entity-processor-service-impl/src/integrationTest/kotlin/net/corda/entityprocessor/impl/tests/PersistenceServiceInternalTests.kt index 15f18d2401f..1e725884113 100644 --- a/components/persistence/entity-processor-service-impl/src/integrationTest/kotlin/net/corda/entityprocessor/impl/tests/PersistenceServiceInternalTests.kt +++ b/components/persistence/entity-processor-service-impl/src/integrationTest/kotlin/net/corda/entityprocessor/impl/tests/PersistenceServiceInternalTests.kt @@ -1,15 +1,14 @@ package net.corda.entityprocessor.impl.tests -import net.corda.cpiinfo.read.CpiInfoReadService -import net.corda.cpk.read.CpkReadService import net.corda.avro.serialization.CordaAvroDeserializer import net.corda.avro.serialization.CordaAvroSerializationFactory +import net.corda.cpiinfo.read.CpiInfoReadService +import net.corda.cpk.read.CpkReadService import net.corda.data.KeyValuePairList import net.corda.data.flow.event.FlowEvent import net.corda.data.flow.event.external.ExternalEventContext import net.corda.data.flow.event.external.ExternalEventResponse import net.corda.data.flow.event.external.ExternalEventResponseErrorType -import net.corda.v5.application.flows.FlowContextPropertyKeys.CPK_FILE_CHECKSUM import net.corda.data.persistence.DeleteEntities import net.corda.data.persistence.DeleteEntitiesById import net.corda.data.persistence.EntityRequest @@ -35,7 +34,7 @@ import net.corda.db.persistence.testkit.helpers.SandboxHelper.getCatClass import net.corda.db.persistence.testkit.helpers.SandboxHelper.getDogClass import net.corda.db.persistence.testkit.helpers.SandboxHelper.getOwnerClass import net.corda.db.schema.DbSchema -import net.corda.entityprocessor.impl.internal.EntityMessageProcessor +import net.corda.entityprocessor.impl.internal.EntityRequestProcessor import net.corda.entityprocessor.impl.internal.PersistenceServiceInternal import net.corda.entityprocessor.impl.internal.getClass import net.corda.entityprocessor.impl.tests.helpers.AnimalCreator.createCats @@ -56,6 +55,7 @@ import net.corda.test.util.dsl.entities.cpx.getCpkFileHashes import net.corda.testing.sandboxes.SandboxSetup import net.corda.testing.sandboxes.fetchService import net.corda.testing.sandboxes.lifecycle.EachTestLifecycle +import net.corda.v5.application.flows.FlowContextPropertyKeys.CPK_FILE_CHECKSUM import net.corda.virtualnode.VirtualNodeInfo import net.corda.virtualnode.read.VirtualNodeInfoReadService import net.corda.virtualnode.toAvro @@ -81,11 +81,6 @@ import java.time.ZoneOffset import java.util.UUID import javax.persistence.EntityManagerFactory -sealed class QuerySetup { - data class NamedQuery(val params: Map, val query: String = "Dog.summon") : QuerySetup() - data class All(val className: String) : QuerySetup() -} - /** * To use Postgres rather than in-memory (HSQL): * @@ -96,6 +91,12 @@ sealed class QuerySetup { * Rather than creating a new serializer in these tests from scratch, * we grab a reference to the one in the sandbox and use that to serialize and de-serialize. */ + +sealed class QuerySetup { + data class NamedQuery(val params: Map, val query: String = "Dog.summon") : QuerySetup() + data class All(val className: String) : QuerySetup() +} + @ExtendWith(ServiceExtension::class, BundleContextExtension::class, DBSetup::class) @TestInstance(TestInstance.Lifecycle.PER_CLASS) class PersistenceServiceInternalTests { @@ -285,7 +286,7 @@ class PersistenceServiceInternalTests { } ) - val processor = EntityMessageProcessor( + val processor = EntityRequestProcessor( currentSandboxGroupContext, myEntitySandboxService, responseFactory, @@ -1015,8 +1016,8 @@ class PersistenceServiceInternalTests { private fun SandboxGroupContext.deserialize(bytes: ByteBuffer) = getSerializationService().deserialize(bytes.array(), Any::class.java) - private fun getMessageProcessor(payloadCheck: (bytes: ByteBuffer) -> ByteBuffer): EntityMessageProcessor { - return EntityMessageProcessor( + private fun getMessageProcessor(payloadCheck: (bytes: ByteBuffer) -> ByteBuffer): EntityRequestProcessor { + return EntityRequestProcessor( currentSandboxGroupContext, entitySandboxService, responseFactory, diff --git a/components/persistence/entity-processor-service-impl/src/integrationTest/kotlin/net/corda/entityprocessor/impl/tests/helpers/Utils.kt b/components/persistence/entity-processor-service-impl/src/integrationTest/kotlin/net/corda/entityprocessor/impl/tests/helpers/Utils.kt new file mode 100644 index 00000000000..5113f3b0526 --- /dev/null +++ b/components/persistence/entity-processor-service-impl/src/integrationTest/kotlin/net/corda/entityprocessor/impl/tests/helpers/Utils.kt @@ -0,0 +1,14 @@ +package net.corda.entityprocessor.impl.tests.helpers + +import net.corda.data.flow.event.FlowEvent +import net.corda.data.flow.event.external.ExternalEventResponse +import net.corda.messaging.api.records.Record +import org.junit.jupiter.api.Assertions + +fun assertEventResponseWithoutError(record: Record<*, *>) { + Assertions.assertNull(((record.value as FlowEvent).payload as ExternalEventResponse).error) +} + +fun assertEventResponseWithError(record: Record<*, *>) { + Assertions.assertNotNull(((record.value as FlowEvent).payload as ExternalEventResponse).error) +} \ No newline at end of file diff --git a/components/persistence/entity-processor-service-impl/src/main/kotlin/net/corda/entityprocessor/impl/EntityProcessorImpl.kt b/components/persistence/entity-processor-service-impl/src/main/kotlin/net/corda/entityprocessor/impl/EntityProcessorImpl.kt deleted file mode 100644 index 62f2c643c07..00000000000 --- a/components/persistence/entity-processor-service-impl/src/main/kotlin/net/corda/entityprocessor/impl/EntityProcessorImpl.kt +++ /dev/null @@ -1,27 +0,0 @@ -package net.corda.entityprocessor.impl - -import net.corda.data.persistence.EntityRequest -import net.corda.entityprocessor.EntityProcessor -import net.corda.messaging.api.subscription.Subscription -import org.osgi.service.component.annotations.Component - -/** - * Entity processor. - * Starts the subscription, which in turn passes the messages to the - * [net.corda.entityprocessor.impl.internal.EntityMessageProcessor]. - */ -@Component(service = [EntityProcessor::class]) -class EntityProcessorImpl( - private val subscription: Subscription -) : - EntityProcessor { - override val isRunning: Boolean - get() = subscription.isRunning - - override fun start() = subscription.start() - - // It is important to call `subscription.close()` rather than `subscription.stop()` as the latter does not remove - // Lifecycle coordinator from the registry, causing it to appear there in `DOWN` state. This will in turn fail - // overall Health check's `status` check. - override fun stop() = subscription.close() -} diff --git a/components/persistence/entity-processor-service-impl/src/main/kotlin/net/corda/entityprocessor/impl/EntityProcessorFactoryImpl.kt b/components/persistence/entity-processor-service-impl/src/main/kotlin/net/corda/entityprocessor/impl/EntityRequestSubscriptionFactoryImpl.kt similarity index 53% rename from components/persistence/entity-processor-service-impl/src/main/kotlin/net/corda/entityprocessor/impl/EntityProcessorFactoryImpl.kt rename to components/persistence/entity-processor-service-impl/src/main/kotlin/net/corda/entityprocessor/impl/EntityRequestSubscriptionFactoryImpl.kt index 124f60cff65..cd6a437449d 100644 --- a/components/persistence/entity-processor-service-impl/src/main/kotlin/net/corda/entityprocessor/impl/EntityProcessorFactoryImpl.kt +++ b/components/persistence/entity-processor-service-impl/src/main/kotlin/net/corda/entityprocessor/impl/EntityRequestSubscriptionFactoryImpl.kt @@ -1,10 +1,15 @@ package net.corda.entityprocessor.impl -import net.corda.entityprocessor.EntityProcessor -import net.corda.entityprocessor.EntityProcessorFactory -import net.corda.entityprocessor.impl.internal.EntityMessageProcessor +import net.corda.data.flow.event.FlowEvent +import net.corda.data.persistence.EntityRequest +import net.corda.entityprocessor.EntityRequestSubscriptionFactory +import net.corda.entityprocessor.impl.internal.EntityRequestProcessor +import net.corda.entityprocessor.impl.internal.EntityRpcRequestProcessor import net.corda.libs.configuration.SmartConfig +import net.corda.messaging.api.subscription.RPCSubscription +import net.corda.messaging.api.subscription.Subscription import net.corda.messaging.api.subscription.config.SubscriptionConfig +import net.corda.messaging.api.subscription.config.SyncRPCConfig import net.corda.messaging.api.subscription.factory.SubscriptionFactory import net.corda.persistence.common.EntitySandboxService import net.corda.persistence.common.PayloadChecker @@ -16,8 +21,8 @@ import org.osgi.service.component.annotations.Component import org.osgi.service.component.annotations.Reference @Suppress("UNUSED") -@Component(service = [EntityProcessorFactory::class]) -class EntityProcessorFactoryImpl @Activate constructor( +@Component(service = [EntityRequestSubscriptionFactory::class]) +class EntityRequestSubscriptionFactoryImpl @Activate constructor( @Reference(service = CurrentSandboxGroupContext::class) private val currentSandboxGroupContext: CurrentSandboxGroupContext, @Reference(service = SubscriptionFactory::class) @@ -26,28 +31,40 @@ class EntityProcessorFactoryImpl @Activate constructor( private val entitySandboxService: EntitySandboxService, @Reference(service = ResponseFactory::class) private val responseFactory: ResponseFactory -) : EntityProcessorFactory { +) : EntityRequestSubscriptionFactory { companion object { internal const val GROUP_NAME = "persistence.entity.processor" + const val SUBSCRIPTION_NAME = "Persistence" + const val PATH = "/persistence" } - override fun create(config: SmartConfig): EntityProcessor { + override fun create(config: SmartConfig): Subscription { val subscriptionConfig = SubscriptionConfig(GROUP_NAME, Schemas.Persistence.PERSISTENCE_ENTITY_PROCESSOR_TOPIC) - val processor = EntityMessageProcessor( + val processor = EntityRequestProcessor( currentSandboxGroupContext, entitySandboxService, responseFactory, PayloadChecker(config)::checkSize ) - val subscription = subscriptionFactory.createDurableSubscription( + return subscriptionFactory.createDurableSubscription( subscriptionConfig, processor, config, null ) + } - return EntityProcessorImpl(subscription) + override fun createRpcSubscription(): RPCSubscription { + val processor = EntityRpcRequestProcessor( + currentSandboxGroupContext, + entitySandboxService, + responseFactory, + EntityRequest::class.java, + FlowEvent::class.java + ) + val rpcConfig = SyncRPCConfig(SUBSCRIPTION_NAME, PATH) + return subscriptionFactory.createHttpRPCSubscription(rpcConfig, processor) } } diff --git a/components/persistence/entity-processor-service-impl/src/main/kotlin/net/corda/entityprocessor/impl/FlowPersistenceServiceImpl.kt b/components/persistence/entity-processor-service-impl/src/main/kotlin/net/corda/entityprocessor/impl/FlowPersistenceServiceImpl.kt index b4b369a791a..fed025b22c4 100644 --- a/components/persistence/entity-processor-service-impl/src/main/kotlin/net/corda/entityprocessor/impl/FlowPersistenceServiceImpl.kt +++ b/components/persistence/entity-processor-service-impl/src/main/kotlin/net/corda/entityprocessor/impl/FlowPersistenceServiceImpl.kt @@ -3,8 +3,8 @@ package net.corda.entityprocessor.impl import net.corda.configuration.read.ConfigChangedEvent import net.corda.configuration.read.ConfigurationReadService import net.corda.cpiinfo.read.CpiInfoReadService -import net.corda.entityprocessor.EntityProcessor -import net.corda.entityprocessor.EntityProcessorFactory +import net.corda.data.persistence.EntityRequest +import net.corda.entityprocessor.EntityRequestSubscriptionFactory import net.corda.entityprocessor.FlowPersistenceService import net.corda.libs.configuration.helper.getConfig import net.corda.lifecycle.DependentComponents @@ -15,8 +15,8 @@ import net.corda.lifecycle.LifecycleStatus import net.corda.lifecycle.RegistrationStatusChangeEvent import net.corda.lifecycle.Resource import net.corda.lifecycle.StartEvent -import net.corda.lifecycle.StopEvent import net.corda.lifecycle.createCoordinator +import net.corda.messaging.api.subscription.Subscription import net.corda.sandboxgroupcontext.service.SandboxGroupContextComponent import net.corda.schema.configuration.ConfigKeys.BOOT_CONFIG import net.corda.schema.configuration.ConfigKeys.MESSAGING_CONFIG @@ -40,14 +40,15 @@ class FlowPersistenceServiceImpl @Activate constructor( private val virtualNodeInfoReadService: VirtualNodeInfoReadService, @Reference(service = CpiInfoReadService::class) private val cpiInfoReadService: CpiInfoReadService, - @Reference(service = EntityProcessorFactory::class) - private val entityProcessorFactory: EntityProcessorFactory + @Reference(service = EntityRequestSubscriptionFactory::class) + private val entityRequestSubscriptionFactory: EntityRequestSubscriptionFactory ) : FlowPersistenceService { private var configHandle: Resource? = null - private var entityProcessor: EntityProcessor? = null + private var entityProcessorSubscription: Subscription? = null companion object { private val logger = LoggerFactory.getLogger(this::class.java.enclosingClass) + const val RPC_SUBSCRIPTION = "RPC_SUBSCRIPTION" } private val dependentComponents = DependentComponents.of( @@ -56,7 +57,7 @@ class FlowPersistenceServiceImpl @Activate constructor( ::virtualNodeInfoReadService, ::cpiInfoReadService, ) - private val coordinator = coordinatorFactory.createCoordinator(dependentComponents, ::eventHandler) + private val lifecycleCoordinator = coordinatorFactory.createCoordinator(dependentComponents, ::eventHandler) private fun eventHandler(event: LifecycleEvent, coordinator: LifecycleCoordinator) { logger.debug { "FlowPersistenceService received: $event" } @@ -66,39 +67,46 @@ class FlowPersistenceServiceImpl @Activate constructor( } is RegistrationStatusChangeEvent -> { if (event.status == LifecycleStatus.UP) { + configHandle?.close() configHandle = configurationReadService.registerComponentForUpdates( coordinator, setOf(BOOT_CONFIG, MESSAGING_CONFIG) ) + initialiseRpcSubscription() } else { - configHandle?.close() + coordinator.updateStatus(event.status) } } is ConfigChangedEvent -> { - entityProcessor?.stop() - val newEntityProcessor = entityProcessorFactory.create( + entityProcessorSubscription?.close() + val newEntityProcessorSubscription = entityRequestSubscriptionFactory.create( event.config.getConfig(MESSAGING_CONFIG) ) logger.debug("Starting EntityProcessor.") - newEntityProcessor.start() - entityProcessor = newEntityProcessor + newEntityProcessorSubscription.start() + entityProcessorSubscription = newEntityProcessorSubscription coordinator.updateStatus(LifecycleStatus.UP) } - is StopEvent -> { - entityProcessor?.stop() - logger.debug { "Stopping EntityProcessor." } + } + } + + private fun initialiseRpcSubscription() { + val subscription = entityRequestSubscriptionFactory.createRpcSubscription() + lifecycleCoordinator.createManagedResource(RPC_SUBSCRIPTION) { + subscription.also { + it.start() } } } override val isRunning: Boolean - get() = coordinator.isRunning + get() = lifecycleCoordinator.isRunning override fun start() { - coordinator.start() + lifecycleCoordinator.start() } override fun stop() { - coordinator.stop() + lifecycleCoordinator.stop() } } diff --git a/components/persistence/entity-processor-service-impl/src/main/kotlin/net/corda/entityprocessor/impl/internal/EntityRequestProcessor.kt b/components/persistence/entity-processor-service-impl/src/main/kotlin/net/corda/entityprocessor/impl/internal/EntityRequestProcessor.kt new file mode 100644 index 00000000000..a9a35613f62 --- /dev/null +++ b/components/persistence/entity-processor-service-impl/src/main/kotlin/net/corda/entityprocessor/impl/internal/EntityRequestProcessor.kt @@ -0,0 +1,79 @@ +package net.corda.entityprocessor.impl.internal + +import net.corda.data.persistence.EntityRequest +import net.corda.data.persistence.EntityResponse +import net.corda.libs.virtualnode.datamodel.repository.RequestsIdsRepository +import net.corda.libs.virtualnode.datamodel.repository.RequestsIdsRepositoryImpl +import net.corda.messaging.api.processor.DurableProcessor +import net.corda.messaging.api.records.Record +import net.corda.metrics.CordaMetrics +import net.corda.persistence.common.EntitySandboxService +import net.corda.persistence.common.ResponseFactory +import net.corda.sandboxgroupcontext.CurrentSandboxGroupContext +import net.corda.tracing.traceEventProcessingNullableSingle +import net.corda.utilities.debug +import org.slf4j.LoggerFactory +import java.nio.ByteBuffer +import java.time.Duration +import java.time.Instant + + +/** + * Handles incoming requests, typically from the flow worker, and sends responses. + * + * The [EntityRequest] contains the request and a typed payload. + * + * The [EntityResponse] contains the response or an exception-like payload whose presence indicates + * an error has occurred. + * + * [payloadCheck] is called against each AMQP payload in the result (not the entire Avro array of results) + */ +class EntityRequestProcessor( + private val currentSandboxGroupContext: CurrentSandboxGroupContext, + private val entitySandboxService: EntitySandboxService, + private val responseFactory: ResponseFactory, + private val payloadCheck: (bytes: ByteBuffer) -> ByteBuffer, + private val requestsIdsRepository: RequestsIdsRepository = RequestsIdsRepositoryImpl() +) : DurableProcessor { + + private companion object { + val logger = LoggerFactory.getLogger(this::class.java.enclosingClass) + val processorService = ProcessorService() + } + + override val keyClass = String::class.java + + override val valueClass = EntityRequest::class.java + + override fun onNext(events: List>): List> { + logger.debug { "onNext processing messages ${events.joinToString(",") { it.key }}" } + + return events.mapNotNull { event -> + val request = event.value + if (request == null) { + // We received a [null] external event therefore we do not know the flow id to respond to. + null + } else { + val eventType = request.request?.javaClass?.simpleName ?: "Unknown" + traceEventProcessingNullableSingle(event, "Crypto Event - $eventType") { + CordaMetrics.Metric.Db.EntityPersistenceRequestLag.builder() + .withTag(CordaMetrics.Tag.OperationName, request.request::class.java.name).build().record( + Duration.ofMillis(Instant.now().toEpochMilli() - event.timestamp) + ) + + // val persistenceServiceInternal = PersistenceServiceInternal(sandbox::getClass, payloadCheck) + + processorService.processEvent( + logger, + request, + entitySandboxService, + currentSandboxGroupContext, + responseFactory, + requestsIdsRepository, + payloadCheck + ) + } + } + } + } +} diff --git a/components/persistence/entity-processor-service-impl/src/main/kotlin/net/corda/entityprocessor/impl/internal/EntityRpcRequestProcessor.kt b/components/persistence/entity-processor-service-impl/src/main/kotlin/net/corda/entityprocessor/impl/internal/EntityRpcRequestProcessor.kt new file mode 100644 index 00000000000..531f323c0d4 --- /dev/null +++ b/components/persistence/entity-processor-service-impl/src/main/kotlin/net/corda/entityprocessor/impl/internal/EntityRpcRequestProcessor.kt @@ -0,0 +1,50 @@ +package net.corda.entityprocessor.impl.internal + +import net.corda.data.flow.event.FlowEvent +import net.corda.data.persistence.EntityRequest +import net.corda.data.persistence.EntityResponse +import net.corda.libs.virtualnode.datamodel.repository.RequestsIdsRepository +import net.corda.libs.virtualnode.datamodel.repository.RequestsIdsRepositoryImpl +import net.corda.messaging.api.processor.SyncRPCProcessor +import net.corda.persistence.common.EntitySandboxService +import net.corda.persistence.common.ResponseFactory +import net.corda.sandboxgroupcontext.CurrentSandboxGroupContext +import net.corda.utilities.debug +import org.slf4j.LoggerFactory + + +/** + * Handles incoming requests, typically from the flow worker, and sends responses. + * + * The [EntityRequest] contains the request and a typed payload. + * + * The [EntityResponse] contains the response or an exception-like payload whose presence indicates + * an error has occurred. + * + * [payloadCheck] is called against each AMQP payload in the result (not the entire Avro array of results) + */ + +@Suppress("LongParameterList") +class EntityRpcRequestProcessor( + private val currentSandboxGroupContext: CurrentSandboxGroupContext, + private val entitySandboxService: EntitySandboxService, + private val responseFactory: ResponseFactory, + override val requestClass: Class, + override val responseClass: Class, + private val requestsIdsRepository: RequestsIdsRepository = RequestsIdsRepositoryImpl() +) : SyncRPCProcessor { + + private companion object { + val logger = LoggerFactory.getLogger(this::class.java.enclosingClass) + val processorService = ProcessorService() + } + + override fun process(request: EntityRequest): FlowEvent { + logger.debug { "process processing request $request" } + + val record = processorService.processEvent( + logger, request, entitySandboxService, currentSandboxGroupContext, responseFactory, requestsIdsRepository + ) { it } + return record.value as FlowEvent + } +} diff --git a/components/persistence/entity-processor-service-impl/src/main/kotlin/net/corda/entityprocessor/impl/internal/EntityMessageProcessor.kt b/components/persistence/entity-processor-service-impl/src/main/kotlin/net/corda/entityprocessor/impl/internal/ProcessorService.kt similarity index 53% rename from components/persistence/entity-processor-service-impl/src/main/kotlin/net/corda/entityprocessor/impl/internal/EntityMessageProcessor.kt rename to components/persistence/entity-processor-service-impl/src/main/kotlin/net/corda/entityprocessor/impl/internal/ProcessorService.kt index 9e9b27a09a1..5324e79f17e 100644 --- a/components/persistence/entity-processor-service-impl/src/main/kotlin/net/corda/entityprocessor/impl/internal/EntityMessageProcessor.kt +++ b/components/persistence/entity-processor-service-impl/src/main/kotlin/net/corda/entityprocessor/impl/internal/ProcessorService.kt @@ -2,7 +2,6 @@ package net.corda.entityprocessor.impl.internal import net.corda.crypto.core.parseSecureHash import net.corda.data.KeyValuePairList -import net.corda.v5.application.flows.FlowContextPropertyKeys.CPK_FILE_CHECKSUM import net.corda.data.flow.event.FlowEvent import net.corda.data.persistence.DeleteEntities import net.corda.data.persistence.DeleteEntitiesById @@ -15,8 +14,6 @@ import net.corda.data.persistence.MergeEntities import net.corda.data.persistence.PersistEntities import net.corda.flow.utils.toMap import net.corda.libs.virtualnode.datamodel.repository.RequestsIdsRepository -import net.corda.libs.virtualnode.datamodel.repository.RequestsIdsRepositoryImpl -import net.corda.messaging.api.processor.DurableProcessor import net.corda.messaging.api.records.Record import net.corda.metrics.CordaMetrics import net.corda.orm.utils.transaction @@ -26,187 +23,140 @@ import net.corda.persistence.common.getEntityManagerFactory import net.corda.persistence.common.getSerializationService import net.corda.sandboxgroupcontext.CurrentSandboxGroupContext import net.corda.sandboxgroupcontext.SandboxGroupContext -import net.corda.tracing.traceEventProcessingNullableSingle import net.corda.utilities.MDC_CLIENT_ID import net.corda.utilities.MDC_EXTERNAL_EVENT_ID -import net.corda.utilities.debug import net.corda.utilities.translateFlowContextToMDC import net.corda.utilities.withMDC +import net.corda.v5.application.flows.FlowContextPropertyKeys import net.corda.v5.base.exceptions.CordaRuntimeException import net.corda.virtualnode.toCorda -import org.slf4j.LoggerFactory +import org.slf4j.Logger import java.nio.ByteBuffer import java.time.Duration -import java.time.Instant import java.util.UUID import javax.persistence.EntityManager import javax.persistence.PersistenceException -fun SandboxGroupContext.getClass(fullyQualifiedClassName: String) = - this.sandboxGroup.loadClassFromMainBundles(fullyQualifiedClassName) - -/** - * Handles incoming requests, typically from the flow worker, and sends responses. - * - * The [EntityRequest] contains the request and a typed payload. - * - * The [EntityResponse] contains the response or an exception-like payload whose presence indicates - * an error has occurred. - * - * [payloadCheck] is called against each AMQP payload in the result (not the entire Avro array of results) - */ -class EntityMessageProcessor( - private val currentSandboxGroupContext: CurrentSandboxGroupContext, - private val entitySandboxService: EntitySandboxService, - private val responseFactory: ResponseFactory, - private val payloadCheck: (bytes: ByteBuffer) -> ByteBuffer, - private val requestsIdsRepository: RequestsIdsRepository = RequestsIdsRepositoryImpl() -) : DurableProcessor { - private companion object { - val logger = LoggerFactory.getLogger(this::class.java.enclosingClass) - } - - override val keyClass = String::class.java - - override val valueClass = EntityRequest::class.java - - override fun onNext(events: List>): List> { - logger.debug { "onNext processing messages ${events.joinToString(",") { it.key }}" } - - return events.mapNotNull { event -> - val request = event.value - if (request == null) { - // We received a [null] external event therefore we do not know the flow id to respond to. - null - } else { - val eventType = request.request?.javaClass?.simpleName ?: "Unknown" - traceEventProcessingNullableSingle(event, "Crypto Event - $eventType") { - CordaMetrics.Metric.Db.EntityPersistenceRequestLag.builder() - .withTag(CordaMetrics.Tag.OperationName, request.request::class.java.name) - .build() - .record( - Duration.ofMillis(Instant.now().toEpochMilli() - event.timestamp) - ) - processEvent(request) - } - } - } - } - - private fun processEvent(request: EntityRequest): Record<*, *> { - val clientRequestId = - request.flowExternalEventContext.contextProperties.toMap()[MDC_CLIENT_ID] ?: "" - - return withMDC( +@SuppressWarnings("LongParameterList") +class ProcessorService { + + fun processEvent( + logger: Logger, + request: EntityRequest, + entitySandboxService: EntitySandboxService, + currentSandboxGroupContext: CurrentSandboxGroupContext, + responseFactory: ResponseFactory, + requestsIdsRepository: RequestsIdsRepository, + payload: (bytes: ByteBuffer) -> ByteBuffer + ): Record<*, *> { + val startTime = System.nanoTime() + val clientRequestId = request.flowExternalEventContext.contextProperties.toMap()[MDC_CLIENT_ID] ?: "" + val holdingIdentity = request.holdingIdentity.toCorda() + + val result = withMDC( mapOf( - MDC_CLIENT_ID to clientRequestId, - MDC_EXTERNAL_EVENT_ID to request.flowExternalEventContext.requestId + MDC_CLIENT_ID to clientRequestId, MDC_EXTERNAL_EVENT_ID to request.flowExternalEventContext.requestId ) + translateFlowContextToMDC(request.flowExternalEventContext.contextProperties.toMap()) ) { - val startTime = System.nanoTime() var requestOutcome = "FAILED" try { - val holdingIdentity = request.holdingIdentity.toCorda() logger.info("Handling ${request.request::class.java.name} for holdingIdentity ${holdingIdentity.shortHash.value}") - val cpkFileHashes = request.flowExternalEventContext.contextProperties.items - .filter { it.key.startsWith(CPK_FILE_CHECKSUM) } - .map { it.value.toSecureHash() } - .toSet() + val cpkFileHashes = request.flowExternalEventContext.contextProperties.items.filter { + it.key.startsWith(FlowContextPropertyKeys.CPK_FILE_CHECKSUM) + }.map { it.value.toSecureHash() }.toSet() val sandbox = entitySandboxService.get(holdingIdentity, cpkFileHashes) currentSandboxGroupContext.set(sandbox) - processRequestWithSandbox(sandbox, request).also { requestOutcome = "SUCCEEDED" } + val persistenceServiceInternal = PersistenceServiceInternal(sandbox::getClass, payload) + + processRequestWithSandbox( + sandbox, request, responseFactory, persistenceServiceInternal, requestsIdsRepository + ).also { requestOutcome = "SUCCEEDED" } } catch (e: Exception) { responseFactory.errorResponse(request.flowExternalEventContext, e) } finally { + currentSandboxGroupContext.remove() + }.also { CordaMetrics.Metric.Db.EntityPersistenceRequestTime.builder() .withTag(CordaMetrics.Tag.OperationName, request.request::class.java.name) - .withTag(CordaMetrics.Tag.OperationStatus, requestOutcome) - .build() + .withTag(CordaMetrics.Tag.OperationStatus, requestOutcome).build() .record(Duration.ofNanos(System.nanoTime() - startTime)) - - currentSandboxGroupContext.remove() } } + return result } + private fun String.toSecureHash() = parseSecureHash(this) + @Suppress("ComplexMethod") private fun processRequestWithSandbox( sandbox: SandboxGroupContext, - request: EntityRequest + request: EntityRequest, + responseFactory: ResponseFactory, + persistenceServiceInternal: PersistenceServiceInternal, + requestsIdsRepository: RequestsIdsRepository ): Record { // get the per-sandbox entity manager and serialization services val entityManagerFactory = sandbox.getEntityManagerFactory() val serializationService = sandbox.getSerializationService() + val entityManager = entityManagerFactory.createEntityManager() - val persistenceServiceInternal = PersistenceServiceInternal(sandbox::getClass, payloadCheck) - - val em = entityManagerFactory.createEntityManager() return when (val entityRequest = request.request) { is PersistEntities -> { val requestId = UUID.fromString(request.flowExternalEventContext.requestId) - val entityResponse = withDeduplicationCheck( - requestId, - em, - onDuplication = { - EntityResponse(emptyList(), KeyValuePairList(emptyList()), null) - } - ) { + val entityResponse = withDeduplicationCheck(requestId, entityManager, onDuplication = { + EntityResponse(emptyList(), KeyValuePairList(emptyList()), null) + }, requestsIdsRepository) { persistenceServiceInternal.persist(serializationService, it, entityRequest) } responseFactory.successResponse( - request.flowExternalEventContext, - entityResponse + request.flowExternalEventContext, entityResponse ) } - is DeleteEntities -> em.transaction { + is DeleteEntities -> entityManager.transaction { responseFactory.successResponse( request.flowExternalEventContext, persistenceServiceInternal.deleteEntities(serializationService, it, entityRequest) ) } - is DeleteEntitiesById -> em.transaction { + is DeleteEntitiesById -> entityManager.transaction { responseFactory.successResponse( - request.flowExternalEventContext, - persistenceServiceInternal.deleteEntitiesByIds( - serializationService, - it, - entityRequest + request.flowExternalEventContext, persistenceServiceInternal.deleteEntitiesByIds( + serializationService, it, entityRequest ) ) } is MergeEntities -> { - val entityResponse = em.transaction { + val entityResponse = entityManager.transaction { persistenceServiceInternal.merge(serializationService, it, entityRequest) } responseFactory.successResponse( - request.flowExternalEventContext, - entityResponse + request.flowExternalEventContext, entityResponse ) } - is FindEntities -> em.transaction { + is FindEntities -> entityManager.transaction { responseFactory.successResponse( request.flowExternalEventContext, persistenceServiceInternal.find(serializationService, it, entityRequest) ) } - is FindAll -> em.transaction { + is FindAll -> entityManager.transaction { responseFactory.successResponse( request.flowExternalEventContext, persistenceServiceInternal.findAll(serializationService, it, entityRequest) ) } - is FindWithNamedQuery -> em.transaction { + is FindWithNamedQuery -> entityManager.transaction { responseFactory.successResponse( request.flowExternalEventContext, persistenceServiceInternal.findWithNamedQuery(serializationService, it, entityRequest) @@ -215,23 +165,21 @@ class EntityMessageProcessor( else -> { responseFactory.fatalErrorResponse( - request.flowExternalEventContext, - CordaRuntimeException("Unknown command") + request.flowExternalEventContext, CordaRuntimeException("Unknown command") ) } } } - private fun String.toSecureHash() = parseSecureHash(this) - // We should require requestId to be a UUID to avoid request ids collisions private fun withDeduplicationCheck( requestId: UUID, - em: EntityManager, + entityManager: EntityManager, onDuplication: () -> EntityResponse, - block: (EntityManager) -> EntityResponse, + requestsIdsRepository: RequestsIdsRepository, + block: (EntityManager) -> EntityResponse ): EntityResponse { - return em.transaction { + return entityManager.transaction { try { requestsIdsRepository.persist(requestId, it) it.flush() @@ -241,7 +189,8 @@ class EntityMessageProcessor( it.transaction.setRollbackOnly() return@transaction onDuplication() } - block(em) + block(entityManager) } } -} + +} \ No newline at end of file diff --git a/components/persistence/entity-processor-service-impl/src/main/kotlin/net/corda/entityprocessor/impl/internal/Utils.kt b/components/persistence/entity-processor-service-impl/src/main/kotlin/net/corda/entityprocessor/impl/internal/Utils.kt new file mode 100644 index 00000000000..ed1ba37809a --- /dev/null +++ b/components/persistence/entity-processor-service-impl/src/main/kotlin/net/corda/entityprocessor/impl/internal/Utils.kt @@ -0,0 +1,8 @@ +package net.corda.entityprocessor.impl.internal + +import net.corda.sandboxgroupcontext.SandboxGroupContext + + +fun SandboxGroupContext.getClass(fullyQualifiedClassName: String) = + this.sandboxGroup.loadClassFromMainBundles(fullyQualifiedClassName) + diff --git a/components/persistence/entity-processor-service-impl/src/test/kotlin/net/corda/entityprocessor/impl/EntityProcessorFactoryImplTest.kt b/components/persistence/entity-processor-service-impl/src/test/kotlin/net/corda/entityprocessor/impl/EntityProcessorFactoryImplTest.kt deleted file mode 100644 index 1eeef77f5d6..00000000000 --- a/components/persistence/entity-processor-service-impl/src/test/kotlin/net/corda/entityprocessor/impl/EntityProcessorFactoryImplTest.kt +++ /dev/null @@ -1,16 +0,0 @@ -package net.corda.entityprocessor.impl - -import net.corda.persistence.common.PayloadChecker -import net.corda.persistence.common.exceptions.KafkaMessageSizeException -import org.junit.jupiter.api.Test -import org.junit.jupiter.api.assertThrows -import java.nio.ByteBuffer - -internal class EntityProcessorFactoryImplTest { - @Test - fun `payload check throws if max bytes exceeded`() { - val maxSize = 1024 * 10 - val bytes = ByteBuffer.wrap(ByteArray(maxSize + 1)) - assertThrows { PayloadChecker(maxSize).checkSize(bytes) } - } -} diff --git a/components/persistence/entity-processor-service-impl/src/test/kotlin/net/corda/entityprocessor/impl/EntityRequestSubscriptionFactoryImplTest.kt b/components/persistence/entity-processor-service-impl/src/test/kotlin/net/corda/entityprocessor/impl/EntityRequestSubscriptionFactoryImplTest.kt new file mode 100644 index 00000000000..7fdef4dc5ba --- /dev/null +++ b/components/persistence/entity-processor-service-impl/src/test/kotlin/net/corda/entityprocessor/impl/EntityRequestSubscriptionFactoryImplTest.kt @@ -0,0 +1,81 @@ +package net.corda.entityprocessor.impl + +import net.corda.data.flow.event.FlowEvent +import net.corda.data.persistence.EntityRequest +import net.corda.libs.configuration.SmartConfig +import net.corda.messaging.api.processor.DurableProcessor +import net.corda.messaging.api.processor.SyncRPCProcessor +import net.corda.messaging.api.subscription.RPCSubscription +import net.corda.messaging.api.subscription.Subscription +import net.corda.messaging.api.subscription.config.SubscriptionConfig +import net.corda.messaging.api.subscription.factory.SubscriptionFactory +import net.corda.persistence.common.PayloadChecker +import net.corda.persistence.common.exceptions.KafkaMessageSizeException +import net.corda.schema.Schemas +import org.assertj.core.api.Assertions +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.assertThrows +import org.mockito.kotlin.any +import org.mockito.kotlin.eq +import org.mockito.kotlin.mock +import org.mockito.kotlin.whenever +import java.nio.ByteBuffer + +internal class EntityRequestSubscriptionFactoryImplTest { + @Test + fun `payload check throws if max bytes exceeded`() { + val maxSize = 1024 * 10 + val bytes = ByteBuffer.wrap(ByteArray(maxSize + 1)) + assertThrows { PayloadChecker(maxSize).checkSize(bytes) } + } + + //TODO - add test for create and createRpcSubscription + + @Test + fun `factory creates kafka subscription`() { + val subscriptionFactory = mock() + val config = mock() + + val expectedSubscription = mock>() + val expectedSubscriptionConfig = SubscriptionConfig( + "persistence.entity.processor", + Schemas.Persistence.PERSISTENCE_ENTITY_PROCESSOR_TOPIC + ) + + whenever( + subscriptionFactory.createDurableSubscription( + eq(expectedSubscriptionConfig), + any>(), + eq(config), + eq(null) + ) + ).thenReturn(expectedSubscription) + + val target = EntityRequestSubscriptionFactoryImpl(mock(), subscriptionFactory, mock(), mock()) + + val result = target.create(config) + + Assertions.assertThat(result).isSameAs(expectedSubscription) + } + + @Test + fun `factory creates rpc subscription`() { + val subscriptionFactory = mock() + + val expectedSubscription = mock>() + + whenever( + subscriptionFactory.createHttpRPCSubscription( + any(), + any>(), + ) + ).thenReturn(expectedSubscription) + + val target = EntityRequestSubscriptionFactoryImpl(mock(), subscriptionFactory, mock(), mock()) + Assertions.assertThat(target).isNotNull + val result = target.createRpcSubscription() + Assertions.assertThat(result).isNotNull + Assertions.assertThat(result).isSameAs(expectedSubscription) + } + +} diff --git a/components/persistence/entity-processor-service/build.gradle b/components/persistence/entity-processor-service/build.gradle index ded51edd9bb..359259a35da 100644 --- a/components/persistence/entity-processor-service/build.gradle +++ b/components/persistence/entity-processor-service/build.gradle @@ -10,5 +10,10 @@ dependencies { api project(':libs:lifecycle:lifecycle') api project(':libs:configuration:configuration-core') + implementation 'org.jetbrains.kotlin:kotlin-osgi-bundle' + implementation platform("net.corda:corda-api:$cordaApiVersion") + implementation project(':libs:messaging:messaging') + + api "net.corda:corda-avro-schema" } diff --git a/components/persistence/entity-processor-service/src/main/kotlin/net/corda/entityprocessor/EntityProcessor.kt b/components/persistence/entity-processor-service/src/main/kotlin/net/corda/entityprocessor/EntityProcessor.kt deleted file mode 100644 index e0af3e9b60b..00000000000 --- a/components/persistence/entity-processor-service/src/main/kotlin/net/corda/entityprocessor/EntityProcessor.kt +++ /dev/null @@ -1,5 +0,0 @@ -package net.corda.entityprocessor - -import net.corda.lifecycle.Lifecycle - -interface EntityProcessor : Lifecycle diff --git a/components/persistence/entity-processor-service/src/main/kotlin/net/corda/entityprocessor/EntityProcessorFactory.kt b/components/persistence/entity-processor-service/src/main/kotlin/net/corda/entityprocessor/EntityProcessorFactory.kt deleted file mode 100644 index 6b477b61e5b..00000000000 --- a/components/persistence/entity-processor-service/src/main/kotlin/net/corda/entityprocessor/EntityProcessorFactory.kt +++ /dev/null @@ -1,13 +0,0 @@ -package net.corda.entityprocessor - -import net.corda.libs.configuration.SmartConfig - -interface EntityProcessorFactory { - /** - * Create a new entity processor. - * - * This should be called from/wired into the db-processor start up. - */ - fun create(config: SmartConfig): EntityProcessor -} - diff --git a/components/persistence/entity-processor-service/src/main/kotlin/net/corda/entityprocessor/EntityRequestSubscriptionFactory.kt b/components/persistence/entity-processor-service/src/main/kotlin/net/corda/entityprocessor/EntityRequestSubscriptionFactory.kt new file mode 100644 index 00000000000..24c96df4640 --- /dev/null +++ b/components/persistence/entity-processor-service/src/main/kotlin/net/corda/entityprocessor/EntityRequestSubscriptionFactory.kt @@ -0,0 +1,26 @@ +package net.corda.entityprocessor + +import net.corda.data.flow.event.FlowEvent +import net.corda.data.persistence.EntityRequest +import net.corda.libs.configuration.SmartConfig +import net.corda.messaging.api.subscription.RPCSubscription +import net.corda.messaging.api.subscription.Subscription + +/** + * The [EntityRequestSubscriptionFactory] creates a new subscription to the durable topic used to receive + * [EntityRequest] messages. + */ +interface EntityRequestSubscriptionFactory { + /** + * Create a new subscription + * + * @param config Configuration for the subscription + * @return A new subscription for [EntityRequest] messages + */ + + fun create(config: SmartConfig): Subscription + + fun createRpcSubscription(): RPCSubscription + +} + diff --git a/components/uniqueness/uniqueness-checker-impl/src/main/kotlin/net/corda/uniqueness/checker/impl/BatchedUniquenessCheckerLifecycleImpl.kt b/components/uniqueness/uniqueness-checker-impl/src/main/kotlin/net/corda/uniqueness/checker/impl/BatchedUniquenessCheckerLifecycleImpl.kt index 06920a86a4f..04619cefe44 100644 --- a/components/uniqueness/uniqueness-checker-impl/src/main/kotlin/net/corda/uniqueness/checker/impl/BatchedUniquenessCheckerLifecycleImpl.kt +++ b/components/uniqueness/uniqueness-checker-impl/src/main/kotlin/net/corda/uniqueness/checker/impl/BatchedUniquenessCheckerLifecycleImpl.kt @@ -52,7 +52,7 @@ class BatchedUniquenessCheckerLifecycleImpl @Activate constructor( const val GROUP_NAME = "uniqueness.checker" const val CONFIG_HANDLE = "CONFIG_HANDLE" const val SUBSCRIPTION_NAME = "Uniqueness Check" - const val UNIQUENESS_CHECKER_PATH = "/uniqueness-checker" + const val PATH = "/uniqueness-checker" const val SUBSCRIPTION = "SUBSCRIPTION" const val RPC_SUBSCRIPTION = "RPC_SUBSCRIPTION" @@ -141,7 +141,7 @@ class BatchedUniquenessCheckerLifecycleImpl @Activate constructor( FlowEvent::class.java ) lifecycleCoordinator.createManagedResource(RPC_SUBSCRIPTION) { - val rpcConfig = SyncRPCConfig(SUBSCRIPTION_NAME, UNIQUENESS_CHECKER_PATH) + val rpcConfig = SyncRPCConfig(SUBSCRIPTION_NAME, PATH) subscriptionFactory.createHttpRPCSubscription(rpcConfig, processor).also { it.start() } From 9141c15ceb7ac7f14b3024a92fd60aaa123d788f Mon Sep 17 00:00:00 2001 From: Viktor Kolomeyko Date: Fri, 13 Oct 2023 11:51:20 +0100 Subject: [PATCH 14/81] Revert "CORE-16203 Removed coroutine usage from Multi-Source Event Mediator. (#4858)" This reverts commit e43fb77a223e58df1ca7939dc2f06fd0f1e3987f. --- libs/messaging/messaging-impl/build.gradle | 1 + .../net/corda/messaging/mediator/ClientTask.kt | 9 ++++++--- .../corda/messaging/mediator/MessageBusClient.kt | 16 ++++++++++++---- .../corda/messaging/mediator/ClientTaskTest.kt | 10 +++++++++- .../messaging/mediator/MessageBusClientTest.kt | 15 ++++++++++----- .../mediator/MultiSourceEventMediatorImplTest.kt | 4 +++- libs/messaging/messaging/build.gradle | 1 + .../messaging/api/mediator/MessagingClient.kt | 9 ++++++--- 8 files changed, 48 insertions(+), 17 deletions(-) diff --git a/libs/messaging/messaging-impl/build.gradle b/libs/messaging/messaging-impl/build.gradle index 845cc242509..b4178896dcb 100644 --- a/libs/messaging/messaging-impl/build.gradle +++ b/libs/messaging/messaging-impl/build.gradle @@ -12,6 +12,7 @@ dependencies { implementation project(":libs:chunking:chunking-core") implementation project(":libs:crypto:cipher-suite") implementation project(":libs:crypto:crypto-core") + implementation project(path: ':libs:kotlin-coroutines', configuration: 'bundle') implementation project(":libs:lifecycle:lifecycle") implementation project(":libs:messaging:messaging") implementation project(":libs:messaging:message-bus") diff --git a/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/ClientTask.kt b/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/ClientTask.kt index 8182934b459..c93138ee1a3 100644 --- a/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/ClientTask.kt +++ b/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/ClientTask.kt @@ -1,5 +1,6 @@ package net.corda.messaging.mediator +import kotlinx.coroutines.runBlocking import net.corda.messaging.api.mediator.MediatorMessage import net.corda.messaging.api.mediator.MessageRouter import net.corda.messaging.api.mediator.MessagingClient @@ -29,9 +30,11 @@ data class ClientTask( val destination = messageRouter.getDestination(message) @Suppress("UNCHECKED_CAST") - val reply = with(destination) { - message.addProperty(MSG_PROP_ENDPOINT, endpoint) - client.send(message) as MediatorMessage? + val reply = runBlocking { + with(destination) { + message.addProperty(MSG_PROP_ENDPOINT, endpoint) + client.send(message).await() as MediatorMessage? + } } return Result(this, reply) } diff --git a/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/MessageBusClient.kt b/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/MessageBusClient.kt index 116ca1579dd..66338f569c0 100644 --- a/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/MessageBusClient.kt +++ b/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/MessageBusClient.kt @@ -1,5 +1,7 @@ package net.corda.messaging.mediator +import kotlinx.coroutines.CompletableDeferred +import kotlinx.coroutines.Deferred import net.corda.messagebus.api.producer.CordaProducer import net.corda.messagebus.api.producer.CordaProducerRecord import net.corda.messaging.api.mediator.MediatorMessage @@ -18,10 +20,16 @@ class MessageBusClient( private val log: Logger = LoggerFactory.getLogger(this::class.java.enclosingClass) } - override fun send(message: MediatorMessage<*>): MediatorMessage<*>? { - producer.send(message.toCordaProducerRecord(), null) - return null - } + override fun send(message: MediatorMessage<*>): Deferred?> = + CompletableDeferred?>().apply { + producer.send(message.toCordaProducerRecord()) { ex -> + if (ex != null) { + completeExceptionally(ex) + } else { + complete(null) + } + } + } override fun close() { try { diff --git a/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/ClientTaskTest.kt b/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/ClientTaskTest.kt index d97beeffe6a..d28ef8b4b45 100644 --- a/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/ClientTaskTest.kt +++ b/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/ClientTaskTest.kt @@ -1,5 +1,7 @@ package net.corda.messaging.mediator +import kotlinx.coroutines.Deferred +import kotlinx.coroutines.runBlocking import net.corda.messaging.api.mediator.MediatorMessage import net.corda.messaging.api.mediator.MessageRouter import net.corda.messaging.api.mediator.MessagingClient @@ -23,6 +25,7 @@ class ClientTaskTest { private val messageRouter = mock() private val routingDestination = mock() private val messagingClient = mock() + private val clientDeferredReply = mock>>() private val clientReply = mock>() @BeforeEach @@ -34,8 +37,13 @@ class ClientTaskTest { messagingClient ) `when`(messagingClient.send(any())).thenReturn( - clientReply + clientDeferredReply ) + runBlocking { + `when`(clientDeferredReply.await()).thenReturn( + clientReply + ) + } } @Test diff --git a/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/MessageBusClientTest.kt b/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/MessageBusClientTest.kt index 58559d25ba4..09ae5ee8869 100644 --- a/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/MessageBusClientTest.kt +++ b/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/MessageBusClientTest.kt @@ -1,5 +1,6 @@ package net.corda.messaging.mediator +import kotlinx.coroutines.runBlocking import net.corda.messagebus.api.producer.CordaProducer import net.corda.messagebus.api.producer.CordaProducerRecord import net.corda.messaging.api.mediator.MediatorMessage @@ -8,10 +9,9 @@ import net.corda.v5.base.exceptions.CordaRuntimeException import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Test import org.junit.jupiter.api.assertThrows -import org.mockito.Mockito import org.mockito.Mockito.times +import org.mockito.kotlin.any import org.mockito.kotlin.eq -import org.mockito.kotlin.isNull import org.mockito.kotlin.mock import org.mockito.kotlin.verify import org.mockito.kotlin.whenever @@ -50,7 +50,7 @@ class MessageBusClientTest { messageProps.toHeaders(), ) - verify(cordaProducer).send(eq(expected), isNull()) + verify(cordaProducer).send(eq(expected), any()) } @Test @@ -62,9 +62,14 @@ class MessageBusClientTest { messageProps.toHeaders(), ) - Mockito.doThrow(CordaRuntimeException("")).whenever(cordaProducer).send(eq(record), isNull()) + whenever(cordaProducer.send(eq(record), any())).thenAnswer { invocation -> + val callback = invocation.getArgument(1) + callback.onCompletion(CordaRuntimeException("")) + } assertThrows { - messageBusClient.send(message) + runBlocking { + messageBusClient.send(message).await() + } } } diff --git a/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/MultiSourceEventMediatorImplTest.kt b/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/MultiSourceEventMediatorImplTest.kt index e3e76b6a12b..48f558cb4ff 100644 --- a/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/MultiSourceEventMediatorImplTest.kt +++ b/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/MultiSourceEventMediatorImplTest.kt @@ -1,5 +1,6 @@ package net.corda.messaging.mediator +import kotlinx.coroutines.CompletableDeferred import net.corda.avro.serialization.CordaAvroDeserializer import net.corda.avro.serialization.CordaAvroSerializer import net.corda.libs.statemanager.api.StateManager @@ -7,6 +8,7 @@ import net.corda.lifecycle.LifecycleCoordinatorFactory import net.corda.messagebus.api.CordaTopicPartition import net.corda.messagebus.api.consumer.CordaConsumerRecord import net.corda.messaging.api.mediator.MediatorConsumer +import net.corda.messaging.api.mediator.MediatorMessage import net.corda.messaging.api.mediator.MessageRouter import net.corda.messaging.api.mediator.MessagingClient import net.corda.messaging.api.mediator.MultiSourceEventMediator @@ -61,7 +63,7 @@ class MultiSourceEventMediatorImplTest { whenever(mediatorConsumerFactory.create(any>())).thenReturn(consumer) whenever(messagingClient.send(any())).thenAnswer { - null + CompletableDeferred(null as MediatorMessage?) } whenever(messagingClientFactory.create(any())).thenReturn(messagingClient) diff --git a/libs/messaging/messaging/build.gradle b/libs/messaging/messaging/build.gradle index cc0a5337e00..18c0c5d1ae5 100644 --- a/libs/messaging/messaging/build.gradle +++ b/libs/messaging/messaging/build.gradle @@ -14,6 +14,7 @@ dependencies { implementation "net.corda:corda-base" implementation "net.corda:corda-config-schema" implementation project(":libs:chunking:chunking-core") + implementation project(path: ':libs:kotlin-coroutines', configuration: 'bundle') implementation project(":libs:lifecycle:lifecycle") implementation project(":libs:messaging:message-bus") implementation project(":libs:configuration:configuration-core") diff --git a/libs/messaging/messaging/src/main/kotlin/net/corda/messaging/api/mediator/MessagingClient.kt b/libs/messaging/messaging/src/main/kotlin/net/corda/messaging/api/mediator/MessagingClient.kt index c148e4c6b4d..71ea0f32f24 100644 --- a/libs/messaging/messaging/src/main/kotlin/net/corda/messaging/api/mediator/MessagingClient.kt +++ b/libs/messaging/messaging/src/main/kotlin/net/corda/messaging/api/mediator/MessagingClient.kt @@ -1,5 +1,7 @@ package net.corda.messaging.api.mediator +import kotlinx.coroutines.Deferred + /** * Multi-source event mediator messaging client. */ @@ -18,10 +20,11 @@ interface MessagingClient : AutoCloseable { val id: String /** - * Sends a generic [MediatorMessage] and returns any result/error through a response. + * Asynchronously sends a generic [MediatorMessage], and returns any result/error through a [Deferred] response. * * @param message The [MediatorMessage] to send. - * @return Computation result, or null if the destination doesn't provide a response. + * @return [Deferred] instance representing the asynchronous computation result, or null if the destination doesn't + * provide a response. * */ - fun send(message: MediatorMessage<*>): MediatorMessage<*>? + fun send(message: MediatorMessage<*>): Deferred?> } From cc2950fd4963b0114a88c38af311ceb55302ec8e Mon Sep 17 00:00:00 2001 From: nkovacsx <57627796+nkovacsx@users.noreply.github.com> Date: Fri, 13 Oct 2023 13:28:58 +0100 Subject: [PATCH 15/81] CORE-17430 Persist reference states into utxo_transaction_sources table and align with column changes (#4849) The utxo_transaction_sources table has been modified. See CORE-17430 Modify the utxo_transaction_sources table by removing unused columns and renaming corda-api#1288. This PR contains the changes to align with those DB changes. Reference states were not persisted in utxo_transaction_sources until now. Added that persisting logic. --- .../tests/UtxoLedgerMessageProcessorTests.kt | 2 +- .../tests/UtxoPersistenceServiceImplTest.kt | 44 +++++++++++++++++ .../utxo/tests/datamodel/UtxoEntityFactory.kt | 1 + .../ledger/persistence/utxo/UtxoRepository.kt | 11 +++++ .../persistence/utxo/UtxoTransactionReader.kt | 2 + .../utxo/impl/PostgresUtxoQueryProvider.kt | 11 ++++- .../utxo/impl/UtxoPersistenceServiceImpl.kt | 24 +++++++++ .../utxo/impl/UtxoQueryProvider.kt | 5 ++ .../utxo/impl/UtxoRepositoryImpl.kt | 18 +++++++ .../utxo/impl/UtxoTransactionReaderImpl.kt | 2 + .../utxo/UtxoTransactionSourceEntity.kt | 49 +++++++++++++++++++ gradle.properties | 2 +- .../ledger/tests/HsqldbVaultNamedQueryTest.kt | 6 ++- .../ledger/utxo/HsqldbUtxoQueryProvider.kt | 12 +++++ 14 files changed, 185 insertions(+), 4 deletions(-) create mode 100644 components/ledger/ledger-persistence/testing-datamodel/src/main/kotlin/com/example/ledger/testing/datamodel/utxo/UtxoTransactionSourceEntity.kt diff --git a/components/ledger/ledger-persistence/src/integrationTest/kotlin/net/corda/ledger/persistence/utxo/tests/UtxoLedgerMessageProcessorTests.kt b/components/ledger/ledger-persistence/src/integrationTest/kotlin/net/corda/ledger/persistence/utxo/tests/UtxoLedgerMessageProcessorTests.kt index 82c7e544f96..c0aa0885359 100644 --- a/components/ledger/ledger-persistence/src/integrationTest/kotlin/net/corda/ledger/persistence/utxo/tests/UtxoLedgerMessageProcessorTests.kt +++ b/components/ledger/ledger-persistence/src/integrationTest/kotlin/net/corda/ledger/persistence/utxo/tests/UtxoLedgerMessageProcessorTests.kt @@ -207,7 +207,7 @@ class UtxoLedgerMessageProcessorTests { listOf("4".toByteArray()), listOf("5".toByteArray()), emptyList(), - listOf("7".toByteArray()), + emptyList(), listOf(outputState), listOf("9".toByteArray()) ), diff --git a/components/ledger/ledger-persistence/src/integrationTest/kotlin/net/corda/ledger/persistence/utxo/tests/UtxoPersistenceServiceImplTest.kt b/components/ledger/ledger-persistence/src/integrationTest/kotlin/net/corda/ledger/persistence/utxo/tests/UtxoPersistenceServiceImplTest.kt index ecb6ddb5bb9..890e45714e8 100644 --- a/components/ledger/ledger-persistence/src/integrationTest/kotlin/net/corda/ledger/persistence/utxo/tests/UtxoPersistenceServiceImplTest.kt +++ b/components/ledger/ledger-persistence/src/integrationTest/kotlin/net/corda/ledger/persistence/utxo/tests/UtxoPersistenceServiceImplTest.kt @@ -443,6 +443,46 @@ class UtxoPersistenceServiceImplTest { assertThat(dbMetadata.field("groupParametersHash")).isNotNull assertThat(dbMetadata.field("cpiFileChecksum")).isNotNull + val dbTransactionSources = em.createNamedQuery( + "UtxoTransactionSourceEntity.findByTransactionId", + entityFactory.utxoTransactionSource + ) + .setParameter("transactionId", signedTransaction.id.toString()) + .resultList + + assertThat(dbTransactionSources).allMatch { + it.field("groupIndex") == UtxoComponentGroup.INPUTS.ordinal + || it.field("groupIndex") == UtxoComponentGroup.REFERENCES.ordinal + } + + val (dbTransactionInputs, dbTransactionReferences) = dbTransactionSources.partition { + it.field("groupIndex") == UtxoComponentGroup.INPUTS.ordinal + } + + assertThat(dbTransactionInputs).isNotNull + .hasSameSizeAs(defaultInputStateRefs) + + assertThat(dbTransactionReferences).isNotNull + .hasSameSizeAs(defaultReferenceStateRefs) + + dbTransactionInputs + .sortedWith(compareBy { it.field("groupIndex") }.thenBy { it.field("leafIndex") }) + .zip(defaultInputStateRefs) + .forEachIndexed { leafIndex, (dbInput, transactionInput) -> + assertThat(dbInput.field("leafIndex")).isEqualTo(leafIndex) + assertThat(dbInput.field("refTransactionId")).isEqualTo(transactionInput.transactionId.toString()) + assertThat(dbInput.field("refLeafIndex")).isEqualTo(transactionInput.index) + } + + dbTransactionReferences + .sortedWith(compareBy { it.field("groupIndex") }.thenBy { it.field("leafIndex") }) + .zip(defaultReferenceStateRefs) + .forEachIndexed { leafIndex, (dbInput, transactionInput) -> + assertThat(dbInput.field("leafIndex")).isEqualTo(leafIndex) + assertThat(dbInput.field("refTransactionId")).isEqualTo(transactionInput.transactionId.toString()) + assertThat(dbInput.field("refLeafIndex")).isEqualTo(transactionInput.index) + } + val dbTransactionOutputs = em.createNamedQuery( "UtxoVisibleTransactionOutputEntity.findByTransactionId", entityFactory.utxoVisibleTransactionOutput @@ -697,6 +737,10 @@ class UtxoPersistenceServiceImplTest { TODO("Not yet implemented") } + override fun getReferenceStateRefs(): List { + return listOf(StateRef(SecureHashImpl("SHA-256", ByteArray(34)), 2)) + } + override fun getConsumedStateRefs(): List { return listOf(StateRef(SecureHashImpl("SHA-256", ByteArray(12)), 1)) } diff --git a/components/ledger/ledger-persistence/src/integrationTest/kotlin/net/corda/ledger/persistence/utxo/tests/datamodel/UtxoEntityFactory.kt b/components/ledger/ledger-persistence/src/integrationTest/kotlin/net/corda/ledger/persistence/utxo/tests/datamodel/UtxoEntityFactory.kt index 7e467461041..b668d991c4e 100644 --- a/components/ledger/ledger-persistence/src/integrationTest/kotlin/net/corda/ledger/persistence/utxo/tests/datamodel/UtxoEntityFactory.kt +++ b/components/ledger/ledger-persistence/src/integrationTest/kotlin/net/corda/ledger/persistence/utxo/tests/datamodel/UtxoEntityFactory.kt @@ -12,6 +12,7 @@ class UtxoEntityFactory(private val entityManagerFactory: EntityManagerFactory) val utxoTransactionComponent: Class<*> get() = classFor("UtxoTransactionComponentEntity") val utxoVisibleTransactionOutput: Class<*> get() = classFor("UtxoVisibleTransactionOutputEntity") val utxoTransactionSignature: Class<*> get() = classFor("UtxoTransactionSignatureEntity") + val utxoTransactionSource: Class<*> get() = classFor("UtxoTransactionSourceEntity") fun createUtxoTransactionEntity( transactionId: String, diff --git a/components/ledger/ledger-persistence/src/main/kotlin/net/corda/ledger/persistence/utxo/UtxoRepository.kt b/components/ledger/ledger-persistence/src/main/kotlin/net/corda/ledger/persistence/utxo/UtxoRepository.kt index 7626d7b5264..ea292006331 100644 --- a/components/ledger/ledger-persistence/src/main/kotlin/net/corda/ledger/persistence/utxo/UtxoRepository.kt +++ b/components/ledger/ledger-persistence/src/main/kotlin/net/corda/ledger/persistence/utxo/UtxoRepository.kt @@ -89,6 +89,17 @@ interface UtxoRepository { cpiFileChecksum: String ) + /** Persists transaction source (operation is idempotent) */ + @Suppress("LongParameterList") + fun persistTransactionSource( + entityManager: EntityManager, + transactionId: String, + groupIndex: Int, + leafIndex: Int, + sourceStateTransactionId: String, + sourceStateIndex: Int + ) + /** Persists transaction component leaf [data] (operation is idempotent) */ @Suppress("LongParameterList") fun persistTransactionComponentLeaf( diff --git a/components/ledger/ledger-persistence/src/main/kotlin/net/corda/ledger/persistence/utxo/UtxoTransactionReader.kt b/components/ledger/ledger-persistence/src/main/kotlin/net/corda/ledger/persistence/utxo/UtxoTransactionReader.kt index 74ae3183ceb..8a632b44308 100644 --- a/components/ledger/ledger-persistence/src/main/kotlin/net/corda/ledger/persistence/utxo/UtxoTransactionReader.kt +++ b/components/ledger/ledger-persistence/src/main/kotlin/net/corda/ledger/persistence/utxo/UtxoTransactionReader.kt @@ -35,4 +35,6 @@ interface UtxoTransactionReader { fun getConsumedStates(persistenceService: UtxoPersistenceService): List> fun getConsumedStateRefs(): List + + fun getReferenceStateRefs(): List } diff --git a/components/ledger/ledger-persistence/src/main/kotlin/net/corda/ledger/persistence/utxo/impl/PostgresUtxoQueryProvider.kt b/components/ledger/ledger-persistence/src/main/kotlin/net/corda/ledger/persistence/utxo/impl/PostgresUtxoQueryProvider.kt index f9947c75a98..151e2c3ccce 100644 --- a/components/ledger/ledger-persistence/src/main/kotlin/net/corda/ledger/persistence/utxo/impl/PostgresUtxoQueryProvider.kt +++ b/components/ledger/ledger-persistence/src/main/kotlin/net/corda/ledger/persistence/utxo/impl/PostgresUtxoQueryProvider.kt @@ -33,10 +33,19 @@ class PostgresUtxoQueryProvider @Activate constructor( ON CONFLICT DO NOTHING""" .trimIndent() + override val persistTransactionSource: String + get() = """ + INSERT INTO {h-schema}utxo_transaction_sources( + transaction_id, group_idx, leaf_idx, source_state_transaction_id, source_state_idx) + VALUES( + :transactionId, :groupIndex, :leafIndex, :sourceStateTransactionId, :sourceStateIndex) + ON CONFLICT DO NOTHING""" + .trimIndent() + override val persistTransactionComponentLeaf: String get() = """ INSERT INTO {h-schema}utxo_transaction_component(transaction_id, group_idx, leaf_idx, data, hash) - VALUES(:transactionId, :groupIndex, :leafIndex, :data, :hash) + VALUES(:transactionId, :groupIndex, :leafIndex, :data, :hash) ON CONFLICT DO NOTHING""" .trimIndent() diff --git a/components/ledger/ledger-persistence/src/main/kotlin/net/corda/ledger/persistence/utxo/impl/UtxoPersistenceServiceImpl.kt b/components/ledger/ledger-persistence/src/main/kotlin/net/corda/ledger/persistence/utxo/impl/UtxoPersistenceServiceImpl.kt index f26d073ed91..e9623b7da30 100644 --- a/components/ledger/ledger-persistence/src/main/kotlin/net/corda/ledger/persistence/utxo/impl/UtxoPersistenceServiceImpl.kt +++ b/components/ledger/ledger-persistence/src/main/kotlin/net/corda/ledger/persistence/utxo/impl/UtxoPersistenceServiceImpl.kt @@ -182,6 +182,30 @@ class UtxoPersistenceServiceImpl( } } + // Insert inputs data + transaction.getConsumedStateRefs().forEachIndexed { index, input -> + repository.persistTransactionSource( + em, + transactionIdString, + UtxoComponentGroup.INPUTS.ordinal, + index, + input.transactionId.toString(), + input.index + ) + } + + // Insert reference data + transaction.getReferenceStateRefs().forEachIndexed { index, reference -> + repository.persistTransactionSource( + em, + transactionIdString, + UtxoComponentGroup.REFERENCES.ordinal, + index, + reference.transactionId.toString(), + reference.index + ) + } + // Insert outputs data transaction.getVisibleStates().entries.forEach { (stateIndex, stateAndRef) -> val utxoToken = utxoTokenMap[stateAndRef.ref] diff --git a/components/ledger/ledger-persistence/src/main/kotlin/net/corda/ledger/persistence/utxo/impl/UtxoQueryProvider.kt b/components/ledger/ledger-persistence/src/main/kotlin/net/corda/ledger/persistence/utxo/impl/UtxoQueryProvider.kt index c3bb702a640..467e6663da2 100644 --- a/components/ledger/ledger-persistence/src/main/kotlin/net/corda/ledger/persistence/utxo/impl/UtxoQueryProvider.kt +++ b/components/ledger/ledger-persistence/src/main/kotlin/net/corda/ledger/persistence/utxo/impl/UtxoQueryProvider.kt @@ -66,6 +66,11 @@ interface UtxoQueryProvider { */ val persistTransactionMetadata: String + /** + * @property persistTransactionSource SQL text for [UtxoRepositoryImpl.persistTransactionSource]. + */ + val persistTransactionSource: String + /** * @property persistTransactionComponentLeaf SQL text for [UtxoRepositoryImpl.persistTransactionComponentLeaf]. */ diff --git a/components/ledger/ledger-persistence/src/main/kotlin/net/corda/ledger/persistence/utxo/impl/UtxoRepositoryImpl.kt b/components/ledger/ledger-persistence/src/main/kotlin/net/corda/ledger/persistence/utxo/impl/UtxoRepositoryImpl.kt index da1e4ded418..c565616e6a2 100644 --- a/components/ledger/ledger-persistence/src/main/kotlin/net/corda/ledger/persistence/utxo/impl/UtxoRepositoryImpl.kt +++ b/components/ledger/ledger-persistence/src/main/kotlin/net/corda/ledger/persistence/utxo/impl/UtxoRepositoryImpl.kt @@ -214,6 +214,24 @@ class UtxoRepositoryImpl @Activate constructor( .logResult("transaction metadata [$hash]") } + override fun persistTransactionSource( + entityManager: EntityManager, + transactionId: String, + groupIndex: Int, + leafIndex: Int, + sourceStateTransactionId: String, + sourceStateIndex: Int + ) { + entityManager.createNativeQuery(queryProvider.persistTransactionSource) + .setParameter("transactionId", transactionId) + .setParameter("groupIndex", groupIndex) + .setParameter("leafIndex", leafIndex) + .setParameter("sourceStateTransactionId", sourceStateTransactionId) + .setParameter("sourceStateIndex", sourceStateIndex) + .executeUpdate() + .logResult("transaction source [$transactionId, $groupIndex, $leafIndex]") + } + override fun persistTransactionComponentLeaf( entityManager: EntityManager, transactionId: String, diff --git a/components/ledger/ledger-persistence/src/main/kotlin/net/corda/ledger/persistence/utxo/impl/UtxoTransactionReaderImpl.kt b/components/ledger/ledger-persistence/src/main/kotlin/net/corda/ledger/persistence/utxo/impl/UtxoTransactionReaderImpl.kt index 32eb6a4cfe3..4a81534b2a9 100644 --- a/components/ledger/ledger-persistence/src/main/kotlin/net/corda/ledger/persistence/utxo/impl/UtxoTransactionReaderImpl.kt +++ b/components/ledger/ledger-persistence/src/main/kotlin/net/corda/ledger/persistence/utxo/impl/UtxoTransactionReaderImpl.kt @@ -121,4 +121,6 @@ class UtxoTransactionReaderImpl( } override fun getConsumedStateRefs(): List = wrappedWireTransaction.inputStateRefs + + override fun getReferenceStateRefs(): List = wrappedWireTransaction.referenceStateRefs } diff --git a/components/ledger/ledger-persistence/testing-datamodel/src/main/kotlin/com/example/ledger/testing/datamodel/utxo/UtxoTransactionSourceEntity.kt b/components/ledger/ledger-persistence/testing-datamodel/src/main/kotlin/com/example/ledger/testing/datamodel/utxo/UtxoTransactionSourceEntity.kt new file mode 100644 index 00000000000..5eaa0810324 --- /dev/null +++ b/components/ledger/ledger-persistence/testing-datamodel/src/main/kotlin/com/example/ledger/testing/datamodel/utxo/UtxoTransactionSourceEntity.kt @@ -0,0 +1,49 @@ +package com.example.ledger.testing.datamodel.utxo + +import net.corda.v5.base.annotations.CordaSerializable +import java.io.Serializable +import javax.persistence.Column +import javax.persistence.Embeddable +import javax.persistence.Entity +import javax.persistence.Id +import javax.persistence.IdClass +import javax.persistence.JoinColumn +import javax.persistence.ManyToOne +import javax.persistence.NamedQuery +import javax.persistence.Table + +@CordaSerializable +@NamedQuery( + name = "UtxoTransactionSourceEntity.findByTransactionId", + query = "from UtxoTransactionSourceEntity where transaction.id = :transactionId" +) +@Entity +@Table(name = "utxo_transaction_sources") +@IdClass(UtxoTransactionSourceEntityId::class) +data class UtxoTransactionSourceEntity( + @get:Id + @get:ManyToOne + @get:JoinColumn(name = "transaction_id", nullable = false, updatable = false) + var transaction: UtxoTransactionEntity, + + @get:Id + @get:Column(name = "group_idx", nullable = false) + var groupIndex: Int, + + @get:Id + @get:Column(name = "leaf_idx", nullable = false) + var leafIndex: Int, + + @get:Column(name = "source_state_transaction_id", nullable = false) + var refTransactionId: String, + + @get:Column(name = "source_state_idx", nullable = false) + var refLeafIndex: Int +) + +@Embeddable +data class UtxoTransactionSourceEntityId( + var transaction: UtxoTransactionEntity, + var groupIndex: Int, + var leafIndex: Int +) : Serializable diff --git a/gradle.properties b/gradle.properties index 7a66c075b71..0bd8bfe6dbc 100644 --- a/gradle.properties +++ b/gradle.properties @@ -46,7 +46,7 @@ commonsTextVersion = 1.10.0 bouncycastleVersion=1.76 # Corda API libs revision (change in 4th digit indicates a breaking change) # Change to 5.1.0.xx-SNAPSHOT to pick up maven local published copy -cordaApiVersion=5.1.0.33-beta+ +cordaApiVersion=5.1.0.34-beta+ disruptorVersion=3.4.4 felixConfigAdminVersion=1.9.26 diff --git a/testing/ledger/ledger-hsqldb/src/integrationTest/kotlin/net/corda/testing/ledger/tests/HsqldbVaultNamedQueryTest.kt b/testing/ledger/ledger-hsqldb/src/integrationTest/kotlin/net/corda/testing/ledger/tests/HsqldbVaultNamedQueryTest.kt index ba0388e0c88..f3b03d8380d 100644 --- a/testing/ledger/ledger-hsqldb/src/integrationTest/kotlin/net/corda/testing/ledger/tests/HsqldbVaultNamedQueryTest.kt +++ b/testing/ledger/ledger-hsqldb/src/integrationTest/kotlin/net/corda/testing/ledger/tests/HsqldbVaultNamedQueryTest.kt @@ -8,6 +8,8 @@ import com.example.ledger.testing.datamodel.utxo.UtxoVisibleTransactionOutputEnt import com.example.ledger.testing.datamodel.utxo.UtxoVisibleTransactionOutputEntityId import com.example.ledger.testing.datamodel.utxo.UtxoTransactionSignatureEntity import com.example.ledger.testing.datamodel.utxo.UtxoTransactionSignatureEntityId +import com.example.ledger.testing.datamodel.utxo.UtxoTransactionSourceEntity +import com.example.ledger.testing.datamodel.utxo.UtxoTransactionSourceEntityId import java.time.Instant import java.util.UUID import javax.persistence.EntityManagerFactory @@ -85,7 +87,9 @@ class HsqldbVaultNamedQueryTest { UtxoVisibleTransactionOutputEntity::class.java, UtxoVisibleTransactionOutputEntityId::class.java, UtxoTransactionSignatureEntity::class.java, - UtxoTransactionSignatureEntityId::class.java + UtxoTransactionSignatureEntityId::class.java, + UtxoTransactionSourceEntity::class.java, + UtxoTransactionSourceEntityId::class.java )) entityManagerFactory = dbConnectionManager.createEntityManagerFactory(dbConnectionId, entities) } diff --git a/testing/ledger/ledger-hsqldb/src/main/kotlin/net/corda/testing/ledger/utxo/HsqldbUtxoQueryProvider.kt b/testing/ledger/ledger-hsqldb/src/main/kotlin/net/corda/testing/ledger/utxo/HsqldbUtxoQueryProvider.kt index daec8adaaf7..864a64d0a1d 100644 --- a/testing/ledger/ledger-hsqldb/src/main/kotlin/net/corda/testing/ledger/utxo/HsqldbUtxoQueryProvider.kt +++ b/testing/ledger/ledger-hsqldb/src/main/kotlin/net/corda/testing/ledger/utxo/HsqldbUtxoQueryProvider.kt @@ -41,6 +41,18 @@ class HsqldbUtxoQueryProvider @Activate constructor( VALUES (x.hash, x.canonical_data, x.group_parameters_hash, x.cpi_file_checksum)""" .trimIndent() + override val persistTransactionSource: String + get() = """ + MERGE INTO {h-schema}utxo_transaction_sources AS uts + USING (VALUES :transactionId, CAST(:groupIndex AS INT), CAST(:leafIndex AS INT), + :sourceStateTransactionId, CAST(:sourceStateIndex AS INT)) + AS x(transaction_id, group_idx, leaf_idx, source_state_transaction_id, source_state_idx) + ON uts.transaction_id = x.transaction_id AND uts.group_idx = x.group_idx AND uts.leaf_idx = x.leaf_idx + WHEN NOT MATCHED THEN + INSERT (transaction_id, group_idx, leaf_idx, source_state_transaction_id, source_state_idx) + VALUES (x.transaction_id, x.group_idx, x.leaf_idx, x.source_state_transaction_id, x.source_state_idx)""" + .trimIndent() + override val persistTransactionComponentLeaf: String get() = """ MERGE INTO {h-schema}utxo_transaction_component AS utc From 2ec635227771ad4ef5336486fc639a4b8953fa32 Mon Sep 17 00:00:00 2001 From: Ben Millar <44114751+ben-millar@users.noreply.github.com> Date: Fri, 13 Oct 2023 13:45:15 +0100 Subject: [PATCH 16/81] CORE-17604 Moving RPC endpoint paths to consts file (#4862) This moves the RPC worker endpoint paths (E.g. `/ledger`) to a common consts file so they can be used to define both client and server config. --- ...gerPersistenceRequestSubscriptionFactoryImpl.kt | 4 ++-- .../impl/VerificationSubscriptionFactoryImpl.kt | 4 ++-- .../impl/EntityRequestSubscriptionFactoryImpl.kt | 4 ++-- .../impl/BatchedUniquenessCheckerLifecycleImpl.kt | 4 ++-- .../messaging/api/constants/WorkerRPCPaths.kt | 14 ++++++++++++++ 5 files changed, 22 insertions(+), 8 deletions(-) create mode 100644 libs/messaging/messaging/src/main/kotlin/net/corda/messaging/api/constants/WorkerRPCPaths.kt diff --git a/components/ledger/ledger-persistence/src/main/kotlin/net/corda/ledger/persistence/processor/impl/LedgerPersistenceRequestSubscriptionFactoryImpl.kt b/components/ledger/ledger-persistence/src/main/kotlin/net/corda/ledger/persistence/processor/impl/LedgerPersistenceRequestSubscriptionFactoryImpl.kt index 4448a05bc6a..c152b531ba0 100644 --- a/components/ledger/ledger-persistence/src/main/kotlin/net/corda/ledger/persistence/processor/impl/LedgerPersistenceRequestSubscriptionFactoryImpl.kt +++ b/components/ledger/ledger-persistence/src/main/kotlin/net/corda/ledger/persistence/processor/impl/LedgerPersistenceRequestSubscriptionFactoryImpl.kt @@ -7,6 +7,7 @@ import net.corda.ledger.persistence.processor.LedgerPersistenceRequestProcessor import net.corda.ledger.persistence.processor.LedgerPersistenceRequestSubscriptionFactory import net.corda.ledger.persistence.processor.LedgerPersistenceRpcRequestProcessor import net.corda.libs.configuration.SmartConfig +import net.corda.messaging.api.constants.WorkerRPCPaths.LEDGER_PATH import net.corda.messaging.api.subscription.RPCSubscription import net.corda.messaging.api.subscription.Subscription import net.corda.messaging.api.subscription.config.SubscriptionConfig @@ -37,7 +38,6 @@ class LedgerPersistenceRequestSubscriptionFactoryImpl @Activate constructor( companion object { internal const val GROUP_NAME = "persistence.ledger.processor" const val SUBSCRIPTION_NAME = "Ledger" - const val PATH = "/ledger" } override fun create(config: SmartConfig): Subscription { @@ -67,7 +67,7 @@ class LedgerPersistenceRequestSubscriptionFactoryImpl @Activate constructor( LedgerPersistenceRequest::class.java, FlowEvent::class.java ) - val rpcConfig = SyncRPCConfig(SUBSCRIPTION_NAME, PATH) + val rpcConfig = SyncRPCConfig(SUBSCRIPTION_NAME, LEDGER_PATH) return subscriptionFactory.createHttpRPCSubscription(rpcConfig, processor) } } diff --git a/components/ledger/ledger-verification/src/main/kotlin/net/corda/ledger/verification/processor/impl/VerificationSubscriptionFactoryImpl.kt b/components/ledger/ledger-verification/src/main/kotlin/net/corda/ledger/verification/processor/impl/VerificationSubscriptionFactoryImpl.kt index e667596f645..6db925a36f2 100644 --- a/components/ledger/ledger-verification/src/main/kotlin/net/corda/ledger/verification/processor/impl/VerificationSubscriptionFactoryImpl.kt +++ b/components/ledger/ledger-verification/src/main/kotlin/net/corda/ledger/verification/processor/impl/VerificationSubscriptionFactoryImpl.kt @@ -6,6 +6,7 @@ import net.corda.ledger.utxo.verification.TransactionVerificationRequest import net.corda.ledger.verification.processor.VerificationSubscriptionFactory import net.corda.ledger.verification.sandbox.VerificationSandboxService import net.corda.libs.configuration.SmartConfig +import net.corda.messaging.api.constants.WorkerRPCPaths.VERIFICATION_PATH import net.corda.messaging.api.subscription.RPCSubscription import net.corda.messaging.api.subscription.Subscription import net.corda.messaging.api.subscription.config.SubscriptionConfig @@ -31,7 +32,6 @@ class VerificationSubscriptionFactoryImpl @Activate constructor( companion object { internal const val GROUP_NAME = "verification.ledger.processor" const val SUBSCRIPTION_NAME = "Verification" - const val PATH = "/verification" } override fun create(config: SmartConfig): Subscription { @@ -61,7 +61,7 @@ class VerificationSubscriptionFactoryImpl @Activate constructor( TransactionVerificationRequest::class.java, FlowEvent::class.java ) - val rpcConfig = SyncRPCConfig(SUBSCRIPTION_NAME, PATH) + val rpcConfig = SyncRPCConfig(SUBSCRIPTION_NAME, VERIFICATION_PATH) return subscriptionFactory.createHttpRPCSubscription(rpcConfig, processor) } diff --git a/components/persistence/entity-processor-service-impl/src/main/kotlin/net/corda/entityprocessor/impl/EntityRequestSubscriptionFactoryImpl.kt b/components/persistence/entity-processor-service-impl/src/main/kotlin/net/corda/entityprocessor/impl/EntityRequestSubscriptionFactoryImpl.kt index cd6a437449d..541b638985a 100644 --- a/components/persistence/entity-processor-service-impl/src/main/kotlin/net/corda/entityprocessor/impl/EntityRequestSubscriptionFactoryImpl.kt +++ b/components/persistence/entity-processor-service-impl/src/main/kotlin/net/corda/entityprocessor/impl/EntityRequestSubscriptionFactoryImpl.kt @@ -6,6 +6,7 @@ import net.corda.entityprocessor.EntityRequestSubscriptionFactory import net.corda.entityprocessor.impl.internal.EntityRequestProcessor import net.corda.entityprocessor.impl.internal.EntityRpcRequestProcessor import net.corda.libs.configuration.SmartConfig +import net.corda.messaging.api.constants.WorkerRPCPaths.PERSISTENCE_PATH import net.corda.messaging.api.subscription.RPCSubscription import net.corda.messaging.api.subscription.Subscription import net.corda.messaging.api.subscription.config.SubscriptionConfig @@ -35,7 +36,6 @@ class EntityRequestSubscriptionFactoryImpl @Activate constructor( companion object { internal const val GROUP_NAME = "persistence.entity.processor" const val SUBSCRIPTION_NAME = "Persistence" - const val PATH = "/persistence" } override fun create(config: SmartConfig): Subscription { @@ -64,7 +64,7 @@ class EntityRequestSubscriptionFactoryImpl @Activate constructor( EntityRequest::class.java, FlowEvent::class.java ) - val rpcConfig = SyncRPCConfig(SUBSCRIPTION_NAME, PATH) + val rpcConfig = SyncRPCConfig(SUBSCRIPTION_NAME, PERSISTENCE_PATH) return subscriptionFactory.createHttpRPCSubscription(rpcConfig, processor) } } diff --git a/components/uniqueness/uniqueness-checker-impl/src/main/kotlin/net/corda/uniqueness/checker/impl/BatchedUniquenessCheckerLifecycleImpl.kt b/components/uniqueness/uniqueness-checker-impl/src/main/kotlin/net/corda/uniqueness/checker/impl/BatchedUniquenessCheckerLifecycleImpl.kt index 04619cefe44..a61c14186bd 100644 --- a/components/uniqueness/uniqueness-checker-impl/src/main/kotlin/net/corda/uniqueness/checker/impl/BatchedUniquenessCheckerLifecycleImpl.kt +++ b/components/uniqueness/uniqueness-checker-impl/src/main/kotlin/net/corda/uniqueness/checker/impl/BatchedUniquenessCheckerLifecycleImpl.kt @@ -16,6 +16,7 @@ import net.corda.lifecycle.RegistrationStatusChangeEvent import net.corda.lifecycle.StartEvent import net.corda.lifecycle.StopEvent import net.corda.lifecycle.createCoordinator +import net.corda.messaging.api.constants.WorkerRPCPaths.UNIQUENESS_PATH import net.corda.messaging.api.subscription.config.SubscriptionConfig import net.corda.messaging.api.subscription.config.SyncRPCConfig import net.corda.messaging.api.subscription.factory.SubscriptionFactory @@ -52,7 +53,6 @@ class BatchedUniquenessCheckerLifecycleImpl @Activate constructor( const val GROUP_NAME = "uniqueness.checker" const val CONFIG_HANDLE = "CONFIG_HANDLE" const val SUBSCRIPTION_NAME = "Uniqueness Check" - const val PATH = "/uniqueness-checker" const val SUBSCRIPTION = "SUBSCRIPTION" const val RPC_SUBSCRIPTION = "RPC_SUBSCRIPTION" @@ -141,7 +141,7 @@ class BatchedUniquenessCheckerLifecycleImpl @Activate constructor( FlowEvent::class.java ) lifecycleCoordinator.createManagedResource(RPC_SUBSCRIPTION) { - val rpcConfig = SyncRPCConfig(SUBSCRIPTION_NAME, PATH) + val rpcConfig = SyncRPCConfig(SUBSCRIPTION_NAME, UNIQUENESS_PATH) subscriptionFactory.createHttpRPCSubscription(rpcConfig, processor).also { it.start() } diff --git a/libs/messaging/messaging/src/main/kotlin/net/corda/messaging/api/constants/WorkerRPCPaths.kt b/libs/messaging/messaging/src/main/kotlin/net/corda/messaging/api/constants/WorkerRPCPaths.kt new file mode 100644 index 00000000000..b4a34f9633c --- /dev/null +++ b/libs/messaging/messaging/src/main/kotlin/net/corda/messaging/api/constants/WorkerRPCPaths.kt @@ -0,0 +1,14 @@ +package net.corda.messaging.api.constants + +/** + * These are the paths which should be appended to the Corda worker service endpoints to create the + * full RPC endpoint URI. E.g.: "${messagingConfig.getString(CRYPTO_WORKER_REST_ENDPOINT)}$CRYPTO_PATH" + * + */ +object WorkerRPCPaths { + const val CRYPTO_PATH = "/crypto" + const val LEDGER_PATH = "/ledger" + const val PERSISTENCE_PATH = "/persistence" + const val UNIQUENESS_PATH = "/uniqueness-checker" + const val VERIFICATION_PATH = "/verification" +} From 237ef6867ef045e22f2a4bb8f95e9ed71b66bf30 Mon Sep 17 00:00:00 2001 From: Viktor Kolomeyko Date: Fri, 13 Oct 2023 13:54:37 +0100 Subject: [PATCH 17/81] CORE-17411: VNode MGM re-registration improvements (#4851) 1. Do not rely on order of the requests returned. 2. Add more logging when re-registration is actually happening. --- .../handlers/VirtualNodeUpgradeOperationHandler.kt | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/components/virtual-node/virtual-node-write-service-impl/src/main/kotlin/net/corda/virtualnode/write/db/impl/writer/asyncoperation/handlers/VirtualNodeUpgradeOperationHandler.kt b/components/virtual-node/virtual-node-write-service-impl/src/main/kotlin/net/corda/virtualnode/write/db/impl/writer/asyncoperation/handlers/VirtualNodeUpgradeOperationHandler.kt index 2c56ad3f749..7553379ced8 100644 --- a/components/virtual-node/virtual-node-write-service-impl/src/main/kotlin/net/corda/virtualnode/write/db/impl/writer/asyncoperation/handlers/VirtualNodeUpgradeOperationHandler.kt +++ b/components/virtual-node/virtual-node-write-service-impl/src/main/kotlin/net/corda/virtualnode/write/db/impl/writer/asyncoperation/handlers/VirtualNodeUpgradeOperationHandler.kt @@ -227,11 +227,11 @@ internal class VirtualNodeUpgradeOperationHandler( val records = if (mgmInfo == null) { logger.info("No MGM information found in group policy. MGM member info not published.") - mutableListOf() + emptyList() } else { val oldMgmMemberInfo = membershipGroupReader.lookup(mgmInfo.name) if (mgmInfo != oldMgmMemberInfo) { - mutableListOf(recordFactory.createMgmInfoRecord(holdingIdentity, mgmInfo)) + listOf(recordFactory.createMgmInfoRecord(holdingIdentity, mgmInfo)) } else { emptyList() } @@ -248,7 +248,7 @@ internal class VirtualNodeUpgradeOperationHandler( if (registrationRequest.payload.isNotEmpty()) { try { // Get the latest registration request - val registrationRequestDetails = registrationRequest.payload.last() + val registrationRequestDetails = registrationRequest.payload.sortedBy { it.serial }.last() val updatedSerial = registrationRequestDetails.serial + 1 val registrationContext = registrationRequestDetails @@ -258,10 +258,9 @@ internal class VirtualNodeUpgradeOperationHandler( registrationContext[MemberInfoExtension.SERIAL] = updatedSerial.toString() - memberResourceClient.startRegistration( - holdingIdentity.shortHash, - registrationContext, - ) + logger.info("Starting MGM re-registration for holdingIdentity=$holdingIdentity, " + + "shortHash=${holdingIdentity.shortHash}, registrationContext=$registrationContext") + memberResourceClient.startRegistration(holdingIdentity.shortHash, registrationContext) } catch (e: ContextDeserializationException) { logger.warn( "Could not deserialize previous registration context for ${holdingIdentity.shortHash}. " + From afab21763ad4fbd094d38bc541966053f0c6d512 Mon Sep 17 00:00:00 2001 From: Conal Smith <68279385+conalsmith-r3@users.noreply.github.com> Date: Fri, 13 Oct 2023 15:09:42 +0100 Subject: [PATCH 18/81] CORE-16419 - template, schema and values yaml changes to support isolated state manager db (#4763) These changes enable an isolated state-manager database to be automatically deployed via the Corda helm chart when db bootstrapping is enabled. A state-manager database can be bootstrapped by providing the `bootstrap.db.stateManager` postgres credentials overrides, and a `stateManager.db.host` override, along with credentials in `stateManager.db`. If no overrides are provided, the state-manager will be bootstrapped in a schema in the cluster database with cluster credentials. Template functions in _helpers.tpl will fall back to cluster JDBC URL and credentials. --- .ci/e2eTests/corda.yaml | 16 ++ charts/corda-lib/templates/_bootstrap.tpl | 51 +++- charts/corda-lib/templates/_helpers.tpl | 254 +++++++++++++++--- charts/corda-lib/templates/_worker.tpl | 13 +- charts/corda/templates/secrets.yaml | 23 ++ charts/corda/values.schema.json | 153 ++++++----- charts/corda/values.yaml | 53 +++- state-manager-postgres.yaml | 39 +++ state-manager.yaml | 49 ++++ .../net/corda/cli/plugins/dbconfig/Spec.kt | 2 +- .../corda/cli/plugins/dbconfig/SpecTest.kt | 20 ++ values-prereqs.yaml | 1 + 12 files changed, 533 insertions(+), 141 deletions(-) create mode 100644 state-manager-postgres.yaml create mode 100644 state-manager.yaml diff --git a/.ci/e2eTests/corda.yaml b/.ci/e2eTests/corda.yaml index 0d59d0f3b00..fce5db5173b 100644 --- a/.ci/e2eTests/corda.yaml +++ b/.ci/e2eTests/corda.yaml @@ -7,6 +7,13 @@ bootstrap: valueFrom: secretKeyRef: key: "postgres-password" + stateManager: + username: + value: "state-manager-user" + password: + valueFrom: + secretKeyRef: + key: "password" kafka: sasl: username: @@ -30,6 +37,15 @@ db: username: value: "user" +stateManager: + db: + username: + value: "state-manager-user" + password: + valueFrom: + secretKeyRef: + key: "password" + kafka: bootstrapServers: "prereqs-kafka:9092" tls: diff --git a/charts/corda-lib/templates/_bootstrap.tpl b/charts/corda-lib/templates/_bootstrap.tpl index 2684989c7dc..3d3c3356f2d 100644 --- a/charts/corda-lib/templates/_bootstrap.tpl +++ b/charts/corda-lib/templates/_bootstrap.tpl @@ -153,12 +153,12 @@ spec: JDBC_URL="jdbc:{{ include "corda.clusterDbType" . }}://{{ required "A db host is required" .Values.db.cluster.host }}:{{ include "corda.clusterDbPort" . }}/{{ include "corda.clusterDbName" . }}" - echo 'Generating DB specification' + echo 'Generating Cluster DB specification' mkdir /tmp/db java -Dpf4j.pluginsDir=/opt/override/plugins -Dlog4j2.debug=false -jar /opt/override/cli.jar database spec \ - -s "config,rbac,crypto,statemanager" \ - -g "config:${DB_CLUSTER_SCHEMA},rbac:${DB_RBAC_SCHEMA},crypto:${DB_CRYPTO_SCHEMA},statemanager:STATE_MANAGER" \ - -u "${PGUSER}" -p "${PGPASSWORD}" \ + -s "config,rbac,crypto" \ + -g "config:${DB_CLUSTER_SCHEMA},rbac:${DB_RBAC_SCHEMA},crypto:${DB_CRYPTO_SCHEMA}" \ + -u "${CLUSTER_PGUSER}" -p "${CLUSTER_PGPASSWORD}" \ --jdbc-url "${JDBC_URL}" \ -c -l /tmp/db @@ -225,6 +225,15 @@ spec: {{- end }} -l /tmp/crypto + echo 'Generating State Manager DB specification' + STATE_MANAGER_JDBC_URL="{{- include "corda.stateManagerJdbcUrl" . -}}" + mkdir /tmp/stateManager + java -Dpf4j.pluginsDir=/opt/override/plugins -Dlog4j2.debug=false -jar /opt/override/cli.jar database spec \ + -s "statemanager" -g "statemanager:state_manager" \ + -u "${STATE_MANAGER_PGUSER}" -p "${STATE_MANAGER_PGPASSWORD}" \ + --jdbc-url "${STATE_MANAGER_JDBC_URL}" \ + -c -l /tmp/stateManager + echo 'Generating REST API user initial configuration' java -Dpf4j.pluginsDir=/opt/override/plugins -Dlog4j2.debug=false -jar /opt/override/cli.jar initial-config create-user-config \ -u "${REST_API_ADMIN_USERNAME}" -p "${REST_API_ADMIN_PASSWORD}" \ @@ -257,6 +266,7 @@ spec: {{- include "corda.restApiAdminSecretEnv" . | nindent 12 }} {{- include "corda.cryptoDbUsernameEnv" . | nindent 12 }} {{- include "corda.cryptoDbPasswordEnv" . | nindent 12 }} + {{- include "corda.bootstrapStateManagerDbEnv" . | nindent 12 }} containers: - name: apply image: {{ include "corda.bootstrapDbClientImage" . }} @@ -270,16 +280,17 @@ spec: set -ev echo 'Applying DB specification' - find /tmp/db -iname "*.sql" | xargs printf -- ' -f %s' | xargs psql -v ON_ERROR_STOP=1 -h "${DB_CLUSTER_HOST}" -p "${DB_CLUSTER_PORT}" --dbname "${DB_CLUSTER_NAME}" + export PGPASSWORD="${CLUSTER_PGPASSWORD}" + find /tmp/db -iname "*.sql" | xargs printf -- ' -f %s' | xargs psql -v ON_ERROR_STOP=1 -h "${DB_CLUSTER_HOST}" -p "${DB_CLUSTER_PORT}" -U "${CLUSTER_PGUSER}" --dbname "${DB_CLUSTER_NAME}" echo 'Applying initial configurations' - psql -v ON_ERROR_STOP=1 -h "${DB_CLUSTER_HOST}" -p "${DB_CLUSTER_PORT}" -f /tmp/rbac/db-config.sql -f /tmp/vnodes/db-config.sql -f /tmp/crypto/db-config.sql -f /tmp/crypto-config.sql --dbname "dbname=${DB_CLUSTER_NAME} options=--search_path=${DB_CLUSTER_SCHEMA}" + psql -v ON_ERROR_STOP=1 -h "${DB_CLUSTER_HOST}" -p "${DB_CLUSTER_PORT}" -U "${CLUSTER_PGUSER}" -f /tmp/rbac/db-config.sql -f /tmp/vnodes/db-config.sql -f /tmp/crypto/db-config.sql -f /tmp/crypto-config.sql --dbname "dbname=${DB_CLUSTER_NAME} options=--search_path=${DB_CLUSTER_SCHEMA}" echo 'Applying initial RBAC configuration' - psql -v ON_ERROR_STOP=1 -h "${DB_CLUSTER_HOST}" -p "${DB_CLUSTER_PORT}" -f /tmp/rbac-config.sql --dbname "dbname=${DB_CLUSTER_NAME} options=--search_path=${DB_RBAC_SCHEMA}" + psql -v ON_ERROR_STOP=1 -h "${DB_CLUSTER_HOST}" -p "${DB_CLUSTER_PORT}" -U "${CLUSTER_PGUSER}" -f /tmp/rbac-config.sql --dbname "dbname=${DB_CLUSTER_NAME} options=--search_path=${DB_RBAC_SCHEMA}" echo 'Creating users and granting permissions' - psql -v ON_ERROR_STOP=1 -h "${DB_CLUSTER_HOST}" -p "${DB_CLUSTER_PORT}" "${DB_CLUSTER_NAME}" << SQL + psql -v ON_ERROR_STOP=1 -h "${DB_CLUSTER_HOST}" -p "${DB_CLUSTER_PORT}" -U "${CLUSTER_PGUSER}" "${DB_CLUSTER_NAME}" << SQL GRANT USAGE ON SCHEMA ${DB_CLUSTER_SCHEMA} TO "${DB_CLUSTER_USERNAME}"; GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA ${DB_CLUSTER_SCHEMA} TO "${DB_CLUSTER_USERNAME}"; GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA ${DB_CLUSTER_SCHEMA} TO "${DB_CLUSTER_USERNAME}"; @@ -289,11 +300,23 @@ spec: DO \$\$ BEGIN IF EXISTS (SELECT FROM pg_catalog.pg_roles WHERE rolname = '${CRYPTO_DB_USER_USERNAME}') THEN RAISE NOTICE 'Role "${CRYPTO_DB_USER_USERNAME}" already exists'; ELSE CREATE USER "${CRYPTO_DB_USER_USERNAME}" WITH ENCRYPTED PASSWORD '$CRYPTO_DB_USER_PASSWORD'; END IF; END \$\$; GRANT USAGE ON SCHEMA ${DB_CRYPTO_SCHEMA} TO "${CRYPTO_DB_USER_USERNAME}"; GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA ${DB_CRYPTO_SCHEMA} TO "${CRYPTO_DB_USER_USERNAME}"; - GRANT USAGE ON SCHEMA STATE_MANAGER TO "${DB_CLUSTER_USERNAME}"; - GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA STATE_MANAGER TO "${DB_CLUSTER_USERNAME}"; SQL echo 'DB Bootstrapped' + + echo 'Applying State Manager Specification' + export PGPASSWORD="${STATE_MANAGER_PGPASSWORD}" + find /tmp/stateManager -iname "*.sql" | xargs printf -- ' -f %s' | xargs psql -v ON_ERROR_STOP=1 -h "${STATE_MANAGER_DB_HOST}" -p "${STATE_MANAGER_DB_PORT}" -U "${STATE_MANAGER_PGUSER}" --dbname "${STATE_MANAGER_DB_NAME}" + + echo 'Creating users and granting permissions for State Manager' + psql -v ON_ERROR_STOP=1 -h "${STATE_MANAGER_DB_HOST}" -p "${STATE_MANAGER_DB_PORT}" -U "${STATE_MANAGER_PGUSER}" "${STATE_MANAGER_DB_NAME}" << SQL + DO \$\$ BEGIN IF EXISTS (SELECT FROM pg_catalog.pg_roles WHERE rolname = '${STATE_MANAGER_DB_USERNAME}') THEN RAISE NOTICE 'Role "${STATE_MANAGER_DB_USERNAME}" already exists'; ELSE CREATE USER "${STATE_MANAGER_DB_USERNAME}" WITH ENCRYPTED PASSWORD '${STATE_MANAGER_DB_PASSWORD}'; END IF; END \$\$; + GRANT USAGE ON SCHEMA STATE_MANAGER TO "${STATE_MANAGER_DB_USERNAME}"; + GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA STATE_MANAGER TO "${STATE_MANAGER_DB_USERNAME}"; + GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA STATE_MANAGER TO "${STATE_MANAGER_DB_USERNAME}"; + SQL + + echo 'State Manager Bootstrapped' volumeMounts: - mountPath: /tmp name: temp @@ -310,11 +333,19 @@ spec: value: {{ .Values.bootstrap.db.rbac.schema | quote }} - name: DB_CRYPTO_SCHEMA value: {{ .Values.bootstrap.db.crypto.schema | quote }} + - name: STATE_MANAGER_DB_HOST + value: {{ include "corda.stateManagerDbHost" . | quote }} + - name: STATE_MANAGER_DB_PORT + value: {{ include "corda.stateManagerDbPort" . | quote }} + - name: STATE_MANAGER_DB_NAME + value: {{ include "corda.stateManagerDbName" . | quote }} {{- include "corda.bootstrapClusterDbEnv" . | nindent 12 }} {{- include "corda.rbacDbUserEnv" . | nindent 12 }} {{- include "corda.cryptoDbUsernameEnv" . | nindent 12 }} {{- include "corda.cryptoDbPasswordEnv" . | nindent 12 }} {{- include "corda.clusterDbEnv" . | nindent 12 }} + {{- include "corda.bootstrapStateManagerDbEnv" . | nindent 12 }} + {{- include "corda.stateManagerDbEnv" . | nindent 12 }} volumes: - name: temp emptyDir: {} diff --git a/charts/corda-lib/templates/_helpers.tpl b/charts/corda-lib/templates/_helpers.tpl index 2a8181f4cf9..04ad4ea475b 100644 --- a/charts/corda-lib/templates/_helpers.tpl +++ b/charts/corda-lib/templates/_helpers.tpl @@ -233,23 +233,31 @@ Cluster DB credentials environment variables - name: DB_CLUSTER_USERNAME valueFrom: secretKeyRef: - {{- if .Values.db.cluster.username.valueFrom.secretKeyRef.name }} - name: {{ .Values.db.cluster.username.valueFrom.secretKeyRef.name | quote }} - key: {{ required "Must specify db.cluster.username.valueFrom.secretKeyRef.key" .Values.db.cluster.username.valueFrom.secretKeyRef.key | quote }} - {{- else }} - name: {{ include "corda.clusterDbDefaultSecretName" . | quote }} - key: "username" - {{- end }} + {{- include "corda.clusterDbUsername" . | nindent 6 }} - name: DB_CLUSTER_PASSWORD valueFrom: secretKeyRef: - {{- if .Values.db.cluster.password.valueFrom.secretKeyRef.name }} - name: {{ .Values.db.cluster.password.valueFrom.secretKeyRef.name | quote }} - key: {{ required "Must specify db.cluster.password.valueFrom.secretKeyRef.key" .Values.db.cluster.password.valueFrom.secretKeyRef.key | quote }} - {{- else }} - name: {{ include "corda.clusterDbDefaultSecretName" . | quote }} - key: "password" - {{- end }} + {{- include "corda.clusterDbPassword" . | nindent 6 }} +{{- end -}} + +{{- define "corda.clusterDbUsername" -}} +{{- if .Values.db.cluster.username.valueFrom.secretKeyRef.name -}} +name: {{ .Values.db.cluster.username.valueFrom.secretKeyRef.name | quote }} +key: {{ required "Must specify db.cluster.username.valueFrom.secretKeyRef.key" .Values.db.cluster.username.valueFrom.secretKeyRef.key | quote }} +{{- else -}} +name: {{ include "corda.clusterDbDefaultSecretName" . | quote }} +key: "username" +{{- end }} +{{- end }} + +{{- define "corda.clusterDbPassword" -}} +{{- if .Values.db.cluster.password.valueFrom.secretKeyRef.name -}} +name: {{ .Values.db.cluster.password.valueFrom.secretKeyRef.name | quote }} +key: {{ required "Must specify db.cluster.password.valueFrom.secretKeyRef.key" .Values.db.cluster.password.valueFrom.secretKeyRef.key | quote }} +{{- else -}} +name: {{ include "corda.clusterDbDefaultSecretName" . | quote }} +key: "password" +{{- end -}} {{- end -}} {{/* @@ -259,42 +267,42 @@ Default name for bootstrap cluster DB secret {{ printf "%s-bootstrap-cluster-db" (include "corda.fullname" .) }} {{- end -}} +{{- define "corda.bootstrapClusterPgUser" -}} +{{- if .Values.bootstrap.db.cluster.username.valueFrom.secretKeyRef.name -}} +name: {{ .Values.bootstrap.db.cluster.username.valueFrom.secretKeyRef.name | quote }} +key: {{ required "Must specify bootstrap.db.cluster.username.valueFrom.secretKeyRef.key" .Values.bootstrap.db.cluster.username.valueFrom.secretKeyRef.key | quote }} +{{- else if .Values.bootstrap.db.cluster.username.value -}} +name: {{ include "corda.bootstrapClusterDbDefaultSecretName" . | quote }} +key: "username" +{{- else -}} +{{ include "corda.clusterDbUsername" . }} +{{- end -}} +{{- end }} + +{{- define "corda.bootstrapClusterPgPassword" -}} +{{- if .Values.bootstrap.db.cluster.password.valueFrom.secretKeyRef.name -}} +name: {{ .Values.bootstrap.db.cluster.password.valueFrom.secretKeyRef.name | quote }} +key: {{ required "Must specify bootstrap.db.cluster.password.valueFrom.secretKeyRef.key" .Values.bootstrap.db.cluster.password.valueFrom.secretKeyRef.key | quote }} +{{- else if .Values.bootstrap.db.cluster.password.value -}} +name: {{ include "corda.bootstrapClusterDbDefaultSecretName" . | quote }} +key: "password" +{{- else -}} +{{ include "corda.clusterDbPassword" . }} +{{- end -}} +{{- end }} + {{/* Bootstrap cluster DB credentials environment variables */}} {{- define "corda.bootstrapClusterDbEnv" -}} -- name: PGUSER +- name: CLUSTER_PGUSER valueFrom: secretKeyRef: - {{- if .Values.bootstrap.db.cluster.username.valueFrom.secretKeyRef.name }} - name: {{ .Values.bootstrap.db.cluster.username.valueFrom.secretKeyRef.name | quote }} - key: {{ required "Must specify bootstrap.db.cluster.username.valueFrom.secretKeyRef.key" .Values.bootstrap.db.cluster.username.valueFrom.secretKeyRef.key | quote }} - {{- else if .Values.bootstrap.db.cluster.username.value }} - name: {{ include "corda.bootstrapClusterDbDefaultSecretName" . | quote }} - key: "username" - {{- else if .Values.db.cluster.username.valueFrom.secretKeyRef.name }} - name: {{ .Values.db.cluster.username.valueFrom.secretKeyRef.name | quote }} - key: {{ required "Must specify db.cluster.username.valueFrom.secretKeyRef.key" .Values.db.cluster.username.valueFrom.secretKeyRef.key | quote }} - {{- else }} - name: {{ include "corda.clusterDbDefaultSecretName" . | quote }} - key: "username" - {{- end }} -- name: PGPASSWORD + {{- include "corda.bootstrapClusterPgUser" . | nindent 6 }} +- name: CLUSTER_PGPASSWORD valueFrom: secretKeyRef: - {{- if .Values.bootstrap.db.cluster.password.valueFrom.secretKeyRef.name }} - name: {{ .Values.bootstrap.db.cluster.password.valueFrom.secretKeyRef.name | quote }} - key: {{ required "Must specify bootstrap.db.cluster.password.valueFrom.secretKeyRef.key" .Values.bootstrap.db.cluster.password.valueFrom.secretKeyRef.key | quote }} - {{- else if .Values.bootstrap.db.cluster.password.value }} - name: {{ include "corda.bootstrapClusterDbDefaultSecretName" . | quote }} - key: "password" - {{- else if .Values.db.cluster.password.valueFrom.secretKeyRef.name }} - name: {{ .Values.db.cluster.password.valueFrom.secretKeyRef.name | quote }} - key: {{ required "Must specify db.cluster.password.valueFrom.secretKeyRef.key" .Values.db.cluster.password.valueFrom.secretKeyRef.key | quote }} - {{- else}} - name: {{ include "corda.clusterDbDefaultSecretName" . | quote }} - key: "password" - {{- end }} + {{- include "corda.bootstrapClusterPgPassword" . | nindent 6 }} {{- end -}} {{/* @@ -615,6 +623,168 @@ data: {{ $caSecretKey }}: {{ $caSecretValue }} {{- end }} +{{/* +Default name for State Manager DB secret +*/}} +{{- define "corda.stateManagerDbDefaultSecretName" -}} +{{- if $.Values.stateManager.db.host -}} +{{ printf "%s-statemanager-db" (include "corda.fullname" .) }} +{{- else -}} +{{ printf "%s" (include "corda.clusterDbDefaultSecretName" .) }} +{{- end -}} +{{- end }} + +{{/* +State Manager JDBC URL. If no state-manager host is provided, it will default to use a state_manager schema in +the Corda cluster. +*/}} +{{- define "corda.stateManagerJdbcUrl" -}} +{{- if $.Values.stateManager.db.host -}} +jdbc:{{- include "corda.stateManagerDbType" . -}}://{{- .Values.stateManager.db.host -}}:{{- include "corda.stateManagerDbPort" . -}}/{{- include "corda.stateManagerDbName" . -}} +{{- else -}} +jdbc:{{- include "corda.clusterDbType" . -}}://{{- .Values.db.cluster.host -}}:{{- include "corda.clusterDbPort" . -}}/{{- include "corda.clusterDbName" . -}} +{{- end -}} +{{- end -}} + +{{/* +State Manager DB type +*/}} +{{- define "corda.stateManagerDbType" -}} +{{- if $.Values.stateManager.db.host -}} +{{- .Values.stateManager.db.type | default "postgresql" -}} +{{- else -}} +{{- include "corda.clusterDbType" . -}} +{{- end -}} +{{- end -}} + +{{/* +State Manager DB host +*/}} +{{- define "corda.stateManagerDbHost" -}} +{{- if .Values.stateManager.db.host -}} +{{- .Values.stateManager.db.host -}} +{{- else -}} +{{- .Values.db.cluster.host -}} +{{- end -}} +{{- end -}} + +{{/* +State Manager DB port +*/}} +{{- define "corda.stateManagerDbPort" -}} +{{- if $.Values.stateManager.db.host }} +{{- .Values.stateManager.db.port | default "5432" -}} +{{- else }} +{{- include "corda.clusterDbPort" . -}} +{{- end }} +{{- end -}} + +{{/* +State Manager DB name +*/}} +{{- define "corda.stateManagerDbName" -}} +{{- if $.Values.stateManager.db.host }} +{{- .Values.stateManager.db.database | default "state-manager" -}} +{{- else }} +{{- include "corda.clusterDbName" . -}} +{{- end }} +{{- end -}} + + +{{/* +State Manager DB credentials environment variables. +STATE_MANAGER_DB_USERNAME falls back to the cluster database if no stateManager credentials are supplied, and if no +stateManager host is supplied. +*/}} +{{- define "corda.stateManagerDbEnv" -}} +- name: STATE_MANAGER_DB_USERNAME + valueFrom: + secretKeyRef: + {{- if .Values.stateManager.db.host }} + {{- if .Values.stateManager.db.username.valueFrom.secretKeyRef.name }} + name: {{ .Values.stateManager.db.username.valueFrom.secretKeyRef.name | quote }} + key: {{ required "Must specify stateManager.db.username.valueFrom.secretKeyRef.key" .Values.stateManager.db.username.valueFrom.secretKeyRef.key | quote }} + {{- else }} + name: {{ include "corda.stateManagerDbDefaultSecretName" . | quote }} + key: "username" + {{- end }} + {{- else }} + {{- include "corda.clusterDbUsername" . | nindent 6 }} + {{- end }} +- name: STATE_MANAGER_DB_PASSWORD + valueFrom: + secretKeyRef: + {{- if .Values.stateManager.db.host }} + {{- if .Values.stateManager.db.password.valueFrom.secretKeyRef.name }} + name: {{ .Values.stateManager.db.password.valueFrom.secretKeyRef.name | quote }} + key: {{ required "Must specify stateManager.db.password.valueFrom.secretKeyRef.key" .Values.stateManager.db.password.valueFrom.secretKeyRef.key | quote }} + {{- else }} + name: {{ include "corda.stateManagerDbDefaultSecretName" . | quote }} + key: "password" + {{- end }} + {{- else }} + {{- include "corda.clusterDbPassword" . | nindent 6 }} + {{- end }} +{{- end -}} + +{{/* +Default name for bootstrap State Manager DB secret +*/}} +{{- define "corda.bootstrapStateManagerDbDefaultSecretName" -}} +{{- if .Values.stateManager.db.host -}} +{{ printf "%s-bootstrap-statemanager-db" (include "corda.fullname" .) }} +{{- else -}} +{{ printf "%s" (include "corda.bootstrapClusterDbDefaultSecretName" .) }} +{{- end }} +{{- end }} + +{{/* +Bootstrap State Manager DB credentials environment variables. If there is no other host defined for state manager, the +cluster db user and password are used. +*/}} +{{- define "corda.bootstrapStateManagerDbEnv" -}} +- name: STATE_MANAGER_PGUSER + valueFrom: + secretKeyRef: + {{- if .Values.stateManager.db.host }} + {{- if .Values.bootstrap.db.stateManager.username.valueFrom.secretKeyRef.name }} + name: {{ .Values.bootstrap.db.stateManager.username.valueFrom.secretKeyRef.name | quote }} + key: {{ required "Must specify bootstrap.db.stateManager.username.valueFrom.secretKeyRef.key" .Values.bootstrap.db.stateManager.username.valueFrom.secretKeyRef.key | quote }} + {{- else if .Values.bootstrap.db.stateManager.username.value }} + name: {{ include "corda.bootstrapStateManagerDbDefaultSecretName" . | quote }} + key: "username" + {{- else if .Values.stateManager.db.username.valueFrom.secretKeyRef.name }} + name: {{ .Values.stateManager.db.username.valueFrom.secretKeyRef.name | quote }} + key: {{ required "Must specify stateManager.db.username.valueFrom.secretKeyRef.key" .Values.stateManager.db.username.valueFrom.secretKeyRef.key | quote }} + {{- else }} + name: {{ include "corda.stateManagerDbDefaultSecretName" . | quote }} + key: "username" + {{- end }} + {{- else }} + {{- include "corda.bootstrapClusterPgUser" . | nindent 6 }} + {{- end }} +- name: STATE_MANAGER_PGPASSWORD + valueFrom: + secretKeyRef: + {{- if .Values.stateManager.db.host }} + {{- if .Values.bootstrap.db.stateManager.password.valueFrom.secretKeyRef.name }} + name: {{ .Values.bootstrap.db.stateManager.password.valueFrom.secretKeyRef.name | quote }} + key: {{ required "Must specify bootstrap.db.stateManager.password.valueFrom.secretKeyRef.key" .Values.bootstrap.db.stateManager.password.valueFrom.secretKeyRef.key | quote }} + {{- else if .Values.bootstrap.db.stateManager.password.value }} + name: {{ include "corda.bootstrapStateManagerDbDefaultSecretName" . | quote }} + key: "password" + {{- else if .Values.stateManager.db.password.valueFrom.secretKeyRef.name }} + name: {{ .Values.stateManager.db.password.valueFrom.secretKeyRef.name | quote }} + key: {{ required "Must specify stateManager.db.password.valueFrom.secretKeyRef.key" .Values.stateManager.db.password.valueFrom.secretKeyRef.key | quote }} + {{- else }} + name: {{ include "corda.stateManagerDbDefaultSecretName" . | quote }} + key: "password" + {{- end }} + {{- else }} + {{- include "corda.bootstrapClusterPgPassword" . | nindent 6 }} + {{- end }} +{{- end -}} + {{/* The port which should be used to connect to Corda worker instances */}} diff --git a/charts/corda-lib/templates/_worker.tpl b/charts/corda-lib/templates/_worker.tpl index 40ea3e6a8b1..e9f348f9943 100644 --- a/charts/corda-lib/templates/_worker.tpl +++ b/charts/corda-lib/templates/_worker.tpl @@ -239,10 +239,12 @@ spec: {{- if not (($.Values).vault).url }} {{- include "corda.configSaltAndPassphraseEnv" $ | nindent 10 }} {{- end }} - {{- /* TODO-[CORE-16419]: isolate StateManager database from the Cluster database */ -}} - {{- if or $optionalArgs.clusterDbAccess $optionalArgs.stateManagerDbAccess }} + {{- if $optionalArgs.clusterDbAccess }} {{- include "corda.clusterDbEnv" $ | nindent 10 }} {{- end }} + {{- if $optionalArgs.stateManagerDbAccess }} + {{- include "corda.stateManagerDbEnv" $ | nindent 10 }} + {{- end }} args: - "--workspace-dir=/work" - "--temp-dir=/tmp" @@ -287,16 +289,15 @@ spec: - "-ddatabase.pool.keepaliveTimeSeconds={{ .clusterDbConnectionPool.keepaliveTimeSeconds }}" - "-ddatabase.pool.validationTimeoutSeconds={{ .clusterDbConnectionPool.validationTimeoutSeconds }}" {{- end }} - {{- /* TODO-[CORE-16419]: isolate StateManager database from the Cluster database */ -}} {{- if $optionalArgs.stateManagerDbAccess }} - "--stateManager" - "type=DATABASE" - "--stateManager" - - "database.user=$(DB_CLUSTER_USERNAME)" + - "database.user=$(STATE_MANAGER_DB_USERNAME)" - "--stateManager" - - "database.pass=$(DB_CLUSTER_PASSWORD)" + - "database.pass=$(STATE_MANAGER_DB_PASSWORD)" - "--stateManager" - - "database.jdbc.url=jdbc:postgresql://{{ required "Must specify db.cluster.host" $.Values.db.cluster.host }}:{{ $.Values.db.cluster.port }}/{{ $.Values.db.cluster.database }}?currentSchema=STATE_MANAGER" + - "database.jdbc.url={{- include "corda.stateManagerJdbcUrl" $ -}}?currentSchema=STATE_MANAGER" - "--stateManager" - "database.jdbc.directory=/opt/jdbc-driver" - "--stateManager" diff --git a/charts/corda/templates/secrets.yaml b/charts/corda/templates/secrets.yaml index d4a68eec4a8..974b72bf9aa 100644 --- a/charts/corda/templates/secrets.yaml +++ b/charts/corda/templates/secrets.yaml @@ -17,6 +17,17 @@ ( dict "username" ( dict "required" true ) "password" ( dict "required" true ) ) ) }} +{{- if .Values.stateManager.db.host }} +{{- include "corda.secret" + ( list + $ + .Values.stateManager.db + "stateManager.db" + ( include "corda.stateManagerDbDefaultSecretName" . ) + ( dict "username" ( dict ) "password" ( dict ) ) + ) +}} +{{- end }} {{- include "corda.secret" ( list $ @@ -47,6 +58,18 @@ ( dict "cleanup" true ) ) }} +{{- if .Values.stateManager.db.host }} +{{- include "corda.secret" + ( list + $ + .Values.bootstrap.db.stateManager + "boostrap.db.stateManager" + ( include "corda.bootstrapStateManagerDbDefaultSecretName" . ) + ( dict "username" ( dict ) "password" ( dict ) ) + ( dict "cleanup" true ) + ) +}} +{{- end }} {{- end }} {{- if or (.Values.bootstrap.db.enabled) (and (.Values.bootstrap.rbac.enabled) (and (or (.Values.bootstrap.restApiAdmin.username.value) (.Values.bootstrap.restApiAdmin.username.valueFrom.secretKeyRef.name)) (or (.Values.bootstrap.restApiAdmin.password.value) (.Values.bootstrap.restApiAdmin.password.valueFrom.secretKeyRef.name)))) }} {{- include "corda.secret" diff --git a/charts/corda/values.schema.json b/charts/corda/values.schema.json index 69b8aa46db4..2ab971a361e 100644 --- a/charts/corda/values.schema.json +++ b/charts/corda/values.schema.json @@ -780,6 +780,91 @@ } }] }, + "stateManager": { + "type": "object", + "default": {}, + "title": "State Manager configuration", + "additionalProperties": true, + "properties": { + "type": { + "type": "string", + "default": "DATABASE", + "title": "The type of State Manager", + "examples": [ + "DATABASE" + ], + "enum": ["DATABASE"] + }, + "db": { + "type": "object", + "default": {}, + "title": "State Manager database configuration", + "additionalProperties": false, + "properties": { + "username": { + "$ref": "#/$defs/config" + }, + "password": { + "$ref": "#/$defs/config" + }, + "host": { + "anyOf": [ + { + "type": "string", + "format": "hostname" + }, + { + "type": "null" + } + ], + "default": null, + "title": "the State Manager database host", + "examples": [ + "postgresql.example.com" + ] + }, + "type": { + "type": "string", + "default": "postgresql", + "title": "the State Manager database type", + "examples": [ + "postgresql" + ], + "enum": ["postgresql"] + }, + "port": { + "type": "integer", + "default": 5432, + "title": "the State Manager database port", + "examples": [ + 5432 + ] + }, + "database": { + "type": "string", + "default": "state-manager", + "title": "the name of the State Manager database", + "examples": [ + "state-manager" + ], + "minLength": 1 + } + }, + "examples": [{ + "host": "postgresql.example.com", + "type": "postgresql", + "port": 5432, + "username": { + "value": "user" + }, + "password": { + "value": "password" + }, + "database": "state-manager" + }] + } + } + }, "kafka": { "type": "object", "default": {}, @@ -1262,77 +1347,12 @@ "type": "object", "default": {}, "title": "State Manager db configuration", - "required": [ - "dbConnectionPool" - ], "properties": { "username": { "$ref": "#/$defs/config" }, "password": { "$ref": "#/$defs/config" - }, - "dbConnectionPool": { - "type": "object", - "default": {}, - "title": "JDBC connection pool configuration for State Manager DB", - "required": [ - "maxSize", - "idleTimeoutSeconds", - "maxLifetimeSeconds", - "keepAliveTimeSeconds", - "validationTimeoutSeconds" - ], - "properties": { - "maxSize": { - "type": "integer", - "default": 5, - "title": "maximum JDBC connection pool size for State Manager DB", - "examples": [ - 5 - ] - }, - "minSize": { - "anyOf": [ - { - "type": "integer", - "minimum": 0 - }, - { - "type": "null" - } - ], - "default": null, - "title": "minimum JDBC connection pool size for State Manager DB; null value means pool's min size will default to pool's max size value" - }, - "idleTimeoutSeconds": { - "type": "integer", - "default": 120, - "minimum": 0, - "title": "maximum time (in seconds) a connection can stay idle in the pool; A value of 0 means that idle connections are never removed from the pool" - }, - "maxLifetimeSeconds": { - "type": "integer", - "default": 1800, - "minimum": 1, - "title": "maximum time (in seconds) a connection can stay in the pool, regardless if it has been idle or has been recently used; If a connection is in-use and has reached \"maxLifetime\" timeout, it will be removed from the pool only when it becomes idle" - }, - "keepAliveTimeSeconds": { - "type": "integer", - "default": 0, - "minimum": 0, - "title": "interval time (in seconds) in which connections will be tested for aliveness; Connections which are no longer alive are removed from the pool; A value of 0 means this check is disabled" - }, - "validationTimeoutSeconds": { - "type": "integer", - "minimum": 1, - "default": 5, - "title": "maximum time (in seconds) that the pool will wait for a connection to be validated as alive" - } - }, - "examples": [{ - "maxSize": 5 - }] } }, "examples": [{ @@ -1341,9 +1361,6 @@ }, "username": { "value": "username" - }, - "dbConnectionPool": { - "maxSize": 5 } }] }, diff --git a/charts/corda/values.yaml b/charts/corda/values.yaml index 7d82dd06330..8b5e9206b14 100644 --- a/charts/corda/values.yaml +++ b/charts/corda/values.yaml @@ -143,6 +143,45 @@ db: # -- the schema in which the cluster database config entities will be stored, passed to workers on startup schema: CONFIG +# State Manager configuration +stateManager: + # Type of State Manager + type: DATABASE + # State Manager database configuration + db: + # -- the State Manager database host + host: null + # -- the State Manager database type + type: "postgresql" + # -- the State Manager database port + port: 5432 + # the State Manager database user configuration + username: + # -- the State Manager database user + value: "" + # the State Manager database user secret configuration; used in preference to value if name is set + valueFrom: + # the State Manager database user secret key reference + secretKeyRef: + # -- the State Manager database user secret name + name: "" + # -- the State Manager database user secret key + key: "" + # the State Manager database password configuration + password: + # -- the State Manager database password + value: "" + # the State Manager database password secret configuration; used in preference to value if name is set + valueFrom: + # the State Manager database password secret key reference + secretKeyRef: + # -- the State Manager database password secret name + name: "" + # -- the State Manager database password secret key + key: "" + # -- the State Manager database name + database: state-manager + # Kafka configuration kafka: # -- comma-separated list of Kafka bootstrap servers (required); for example: "broker1.example.com:9092,broker2.example.com:9092,broker3.example.com:9092" @@ -366,20 +405,6 @@ bootstrap: # State Manager DB Bootstrap Configuration stateManager: - # JDBC connection pool configuration for state-manager DB - dbConnectionPool: - # -- maximum JDBC connection pool size for state-manager DB - maxSize: 5 - # -- minimum JDBC connection pool size for state-manager DB; null value means pool's min size will default to pool's max size value - minSize: null - # -- maximum time (in seconds) a connection can stay idle in the pool; A value of 0 means that idle connections are never removed from the pool - idleTimeoutSeconds: 120 - # -- maximum time (in seconds) a connection can stay in the pool, regardless if it has been idle or has been recently used; If a connection is in-use and has reached "maxLifetime" timeout, it will be removed from the pool only when it becomes idle - maxLifetimeSeconds: 1800 - # -- interval time (in seconds) in which connections will be tested for aliveness; Connections which are no longer alive are removed from the pool; A value of 0 means this check is disabled - keepAliveTimeSeconds: 0 - # -- maximum time (in seconds) that the pool will wait for a connection to be validated as alive - validationTimeoutSeconds: 5 # the username configuration username: # -- the username, defaults to this value diff --git a/state-manager-postgres.yaml b/state-manager-postgres.yaml new file mode 100644 index 00000000000..3eb06d3856f --- /dev/null +++ b/state-manager-postgres.yaml @@ -0,0 +1,39 @@ +# Override file suitable for local deployment of State Manager Bitnami Postgres +# +# NOTE: The below assumes you deploy Kafka and the PostgreSQL database in the same namespace, so that domain names containing just the service name are resolved (i.e. prereqs-postgresql instead of prereqs-postgresql.) +# If that is not the case, you might need to add the namespace as a suffix. +auth: + # -- name of database to be created. + database: state-manager + # -- name of the user to be created. + username: state-manager-user + # -- password of the user to be created (if left blank, auto-generated) + password: +image: + tag: 14.4.0-debian-11-r23 +# values configuring the permissions of volumes. +volumePermissions: + # -- enable/disable an init container which changes ownership of the mounted volumes. + enabled: true +# Configuring TLS +tls: + # -- enable TLS for postgres. + enabled: true + # -- enable automatic certificate generation for postgres. + autoGenerated: true +# primary (read/write instance) configuration. +primary: + # Custom init configuration.. + initdb: + # -- ConfigMap-like object containing scripts to be executed on startup. + # @default -- corda_user_init.sh + scripts: + corda_user_init.sh: | + export PGPASSFILE=/tmp/pgpasswd$$ + touch $PGPASSFILE + chmod 600 $PGPASSFILE + trap "rm $PGPASSFILE" EXIT + echo "localhost:${POSTGRES_PORT_NUMBER:-5432}:$POSTGRESQL_DATABASE:postgres:$POSTGRES_POSTGRES_PASSWORD" > $PGPASSFILE + psql -v ON_ERROR_STOP=1 $POSTGRESQL_DATABASE <<-EOF + ALTER ROLE "$POSTGRES_USER" NOSUPERUSER CREATEDB CREATEROLE INHERIT LOGIN; + EOF \ No newline at end of file diff --git a/state-manager.yaml b/state-manager.yaml new file mode 100644 index 00000000000..5f928b5fd42 --- /dev/null +++ b/state-manager.yaml @@ -0,0 +1,49 @@ +# Override file suitable for local deployment of the Corda Helm chart against version 0.1.0 of the corda-dev prereqs +# Helm chart and a state-manager deployed via bitnami postgres helm chart with the overrides from +# `state-manager-postgres.yaml`. +# +# First use `./gradlew publishOSGiImage --parallel` to create local docker images +# +# Then deploy the prereqs using: +# helm upgrade --install prereqs -n corda \ +# oci://corda-os-docker.software.r3.com/helm-charts/corda-dev-prereqs \ +# --render-subchart-notes \ +# --timeout 10m \ +# +# Then deploy the state manager using the bitnami helm chart +# helm install state-manager-db oci://docker-remotes.software.r3.com/bitnamicharts/postgresql -n corda --version "12.1.0" \ +# -f ./state-manager-postgres.yaml \ +# --timeout 10m \ +# --wait +# +# Then deploy Corda using +# helm upgrade --install corda -n corda \ +# charts/corda \ +# --values values-prereqs.yaml \ +# --values state-manager.yaml \ +# --wait +# +# +# NOTE: This assumes you deploy the above in the same `corda` namespace, so that domain names containing just the service +# name are resolved (i.e. prereqs-postgresql instead of prereqs-postgresql.). If that is not the case, you +# might need to add the namespace as a suffix. +bootstrap: + db: + stateManager: + password: + valueFrom: + secretKeyRef: + name: "state-manager-db-postgresql" + key: "postgres-password" + username: + value: "postgres" +stateManager: + db: + host: "state-manager-db-postgresql" + password: + valueFrom: + secretKeyRef: + name: "state-manager-db-postgresql" + key: "password" + username: + value: "statemanager-user" diff --git a/tools/plugins/db-config/src/main/kotlin/net/corda/cli/plugins/dbconfig/Spec.kt b/tools/plugins/db-config/src/main/kotlin/net/corda/cli/plugins/dbconfig/Spec.kt index 9a9975fceab..b0045e376c0 100644 --- a/tools/plugins/db-config/src/main/kotlin/net/corda/cli/plugins/dbconfig/Spec.kt +++ b/tools/plugins/db-config/src/main/kotlin/net/corda/cli/plugins/dbconfig/Spec.kt @@ -91,7 +91,7 @@ class Spec(private val config: SpecConfig = SpecConfig()) : Runnable { private const val DEFAULT_CHANGELOG_PATH = "./databasechangelog.csv" // These should match the schema sub directory from the LIQUIBASEFILES list - private const val SCHEMA_OPTIONS = "config, messagebus, rbac, crypto" + private const val SCHEMA_OPTIONS = "config, messagebus, rbac, crypto, statemanager" // messagebus deliberately excluded as it's not used in Corda Clusters private val DEFAULT_SCHEMA_OPTIONS = listOf("config", "rbac", "crypto") diff --git a/tools/plugins/db-config/src/test/kotlin/net/corda/cli/plugins/dbconfig/SpecTest.kt b/tools/plugins/db-config/src/test/kotlin/net/corda/cli/plugins/dbconfig/SpecTest.kt index cd6438ce87e..201b88a77ac 100644 --- a/tools/plugins/db-config/src/test/kotlin/net/corda/cli/plugins/dbconfig/SpecTest.kt +++ b/tools/plugins/db-config/src/test/kotlin/net/corda/cli/plugins/dbconfig/SpecTest.kt @@ -120,4 +120,24 @@ class SpecTest { verify(mockLiquibase, times(NUMBER_OF_DEFAULT_SCHEMAS)).update(any(), any()) verify(mockWriter, times(NUMBER_OF_DEFAULT_SCHEMAS)).close() } + + @Test + fun `Verify specifying statemanager schema will generate only statemanager sql`() { + val spec = Spec(specConfig) + + spec.jdbcUrl = JDBC_URL + spec.user = USER + spec.password = PASSWORD + spec.schemasToGenerate = listOf("statemanager") + spec.generateSchemaSql = listOf("statemanager:STATE_MANAGER_SCHEMA") + + spec.run() + + verify(mockConnectionFactory, times(1)).invoke(JDBC_URL, USER, PASSWORD) + verify(mockDatabaseFactory, times(1)).invoke(mockConnection) + verify(mockLiquibaseFactory, times(1)).invoke(any(), eq(mockDatabase)) + + verify(mockLiquibase, times(1)).update(any(), any()) + verify(mockWriter, times(1)).close() + } } diff --git a/values-prereqs.yaml b/values-prereqs.yaml index cd744c21fce..5846751bad1 100644 --- a/values-prereqs.yaml +++ b/values-prereqs.yaml @@ -43,6 +43,7 @@ db: secretKeyRef: name: "prereqs-postgres" key: "corda-password" + kafka: bootstrapServers: "prereqs-kafka:9092" sasl: From debeba4832e2bbc8d4d0294230213bc9f4a9f6f4 Mon Sep 17 00:00:00 2001 From: Christian Sailer Date: Fri, 13 Oct 2023 15:26:54 +0100 Subject: [PATCH 19/81] CORE-17747 remove attachments (#4861) Remove the attachment API from UTXO ledger transactions. --- .../tests/UtxoFilteredTransactionTest.kt | 2 +- ...ilteredTransactionAMQPSerializationTest.kt | 6 +-- .../UtxoBaselinedTransactionBuilder.kt | 12 ------ .../UtxoTransactionBuilderContainer.kt | 2 - .../transaction/UtxoTransactionBuilderData.kt | 1 - .../transaction/UtxoTransactionBuilderImpl.kt | 16 -------- .../UtxoTransactionBuilderInternal.kt | 1 - .../impl/UtxoSignedTransactionFactoryImpl.kt | 2 +- .../flow/impl/UtxoLedgerServiceImplTest.kt | 3 -- ...veAndUpdateTransactionBuilderFlowV1Test.kt | 38 ------------------- .../SendTransactionBuilderDiffFlowV1Test.kt | 35 ----------------- ...UtxoBaselinedTransactionBuilderDiffTest.kt | 23 ----------- .../UtxoLedgerTransactionImplTest.kt | 3 -- .../UtxoTransactionBuilderImplAppendTest.kt | 21 ---------- .../UtxoTransactionBuilderImplTest.kt | 33 ---------------- .../UtxoSignedTransactionSerializerTest.kt | 2 - gradle.properties | 2 +- .../data/transaction/UtxoComponentGroup.kt | 2 +- .../transaction/UtxoLedgerTransactionImpl.kt | 11 ------ .../transaction/WrappedUtxoWireTransaction.kt | 10 ----- 20 files changed, 7 insertions(+), 218 deletions(-) diff --git a/components/ledger/ledger-utxo-flow/src/integrationTest/kotlin/net/corda/ledger/utxo/flow/impl/transaction/filtered/tests/UtxoFilteredTransactionTest.kt b/components/ledger/ledger-utxo-flow/src/integrationTest/kotlin/net/corda/ledger/utxo/flow/impl/transaction/filtered/tests/UtxoFilteredTransactionTest.kt index b183da342fe..a8a65d3731c 100644 --- a/components/ledger/ledger-utxo-flow/src/integrationTest/kotlin/net/corda/ledger/utxo/flow/impl/transaction/filtered/tests/UtxoFilteredTransactionTest.kt +++ b/components/ledger/ledger-utxo-flow/src/integrationTest/kotlin/net/corda/ledger/utxo/flow/impl/transaction/filtered/tests/UtxoFilteredTransactionTest.kt @@ -440,7 +440,7 @@ class UtxoFilteredTransactionTest : UtxoLedgerIntegrationTest() { serializationService.serialize(listOf(MyCommand::class.java.name)).bytes, serializationService.serialize(listOf(MyCommand::class.java.name)).bytes ), - // attachments + // unused (was attachments) emptyList(), // inputs List(numberOfInputStates) { diff --git a/components/ledger/ledger-utxo-flow/src/integrationTest/kotlin/net/corda/ledger/utxo/flow/impl/transaction/serializer/tests/UtxoFilteredTransactionAMQPSerializationTest.kt b/components/ledger/ledger-utxo-flow/src/integrationTest/kotlin/net/corda/ledger/utxo/flow/impl/transaction/serializer/tests/UtxoFilteredTransactionAMQPSerializationTest.kt index d676122a649..cb920e1f495 100644 --- a/components/ledger/ledger-utxo-flow/src/integrationTest/kotlin/net/corda/ledger/utxo/flow/impl/transaction/serializer/tests/UtxoFilteredTransactionAMQPSerializationTest.kt +++ b/components/ledger/ledger-utxo-flow/src/integrationTest/kotlin/net/corda/ledger/utxo/flow/impl/transaction/serializer/tests/UtxoFilteredTransactionAMQPSerializationTest.kt @@ -45,7 +45,7 @@ class UtxoFilteredTransactionAMQPSerializationTest : UtxoLedgerIntegrationTest() serializationService.serialize(outputInfo).bytes ), // output infos emptyList(), // command infos - emptyList(), // attachments + emptyList(), // unused listOf( serializationService.serialize(StateRef(inputHash, 0)).bytes, serializationService.serialize(StateRef(inputHash, 1)).bytes @@ -103,7 +103,7 @@ class UtxoFilteredTransactionAMQPSerializationTest : UtxoLedgerIntegrationTest() serializationService.serialize(outputInfo).bytes ), // output infos emptyList(), // command infos - emptyList(), // attachments + emptyList(), // unused listOf( serializationService.serialize(StateRef(inputHash, 0)).bytes, serializationService.serialize(StateRef(inputHash, 1)).bytes @@ -158,7 +158,7 @@ class UtxoFilteredTransactionAMQPSerializationTest : UtxoLedgerIntegrationTest() serializationService.serialize(outputInfo).bytes ), // output infos emptyList(), // command infos - emptyList(), // attachments + emptyList(), // unused emptyList(), // inputs emptyList(), // references listOf( diff --git a/components/ledger/ledger-utxo-flow/src/main/kotlin/net/corda/ledger/utxo/flow/impl/transaction/UtxoBaselinedTransactionBuilder.kt b/components/ledger/ledger-utxo-flow/src/main/kotlin/net/corda/ledger/utxo/flow/impl/transaction/UtxoBaselinedTransactionBuilder.kt index 4441a090f0d..7da8b2c6b27 100644 --- a/components/ledger/ledger-utxo-flow/src/main/kotlin/net/corda/ledger/utxo/flow/impl/transaction/UtxoBaselinedTransactionBuilder.kt +++ b/components/ledger/ledger-utxo-flow/src/main/kotlin/net/corda/ledger/utxo/flow/impl/transaction/UtxoBaselinedTransactionBuilder.kt @@ -4,7 +4,6 @@ import net.corda.ledger.utxo.flow.impl.timewindow.TimeWindowBetweenImpl import net.corda.ledger.utxo.flow.impl.timewindow.TimeWindowUntilImpl import net.corda.v5.base.annotations.Suspendable import net.corda.v5.base.types.MemberX500Name -import net.corda.v5.crypto.SecureHash import net.corda.v5.ledger.utxo.Command import net.corda.v5.ledger.utxo.ContractState import net.corda.v5.ledger.utxo.StateRef @@ -27,8 +26,6 @@ class UtxoBaselinedTransactionBuilder private constructor( override val timeWindow: TimeWindow? get() = currentTransactionBuilder.timeWindow - override val attachments: List - get() = currentTransactionBuilder.attachments override val commands: List get() = currentTransactionBuilder.commands override val signatories: List @@ -85,7 +82,6 @@ class UtxoBaselinedTransactionBuilder private constructor( && other.notaryName == notaryName && other.notaryKey == notaryKey && other.timeWindow == timeWindow - && other.attachments == attachments && other.commands == commands && other.inputStateRefs == inputStateRefs && other.referenceStateRefs == referenceStateRefs @@ -98,7 +94,6 @@ class UtxoBaselinedTransactionBuilder private constructor( notaryName, notaryKey, timeWindow, - attachments, commands, signatories, inputStateRefs, @@ -110,7 +105,6 @@ class UtxoBaselinedTransactionBuilder private constructor( return "UtxoBaselinedTransactionBuilder(" + "notary=$notaryName (key: $notaryKey) (orig: ${baselineTransactionBuilder.getNotaryName()}), " + "timeWindow=$timeWindow (orig: ${baselineTransactionBuilder.timeWindow}), " + - "attachments=$attachments (orig: ${baselineTransactionBuilder.attachments}), " + "commands=$commands (orig: ${baselineTransactionBuilder.commands}), " + "signatories=$signatories (orig: ${baselineTransactionBuilder.signatories}), " + "inputStateRefs=$inputStateRefs (orig: ${baselineTransactionBuilder.inputStateRefs}), " + @@ -122,11 +116,6 @@ class UtxoBaselinedTransactionBuilder private constructor( // Unfortunately we cannot just simply delegate everything to currentTransactionBuilder since that would return itself // instead of the baselined transaction builder object. - override fun addAttachment(attachmentId: SecureHash): UtxoBaselinedTransactionBuilder { - currentTransactionBuilder.addAttachment(attachmentId) - return this - } - override fun addCommand(command: Command): UtxoBaselinedTransactionBuilder { currentTransactionBuilder.addCommand(command) return this @@ -221,7 +210,6 @@ class UtxoBaselinedTransactionBuilder private constructor( UtxoTransactionBuilderContainer( if (baselineTransactionBuilder.getNotaryName() == null) notaryName else null, if (baselineTransactionBuilder.timeWindow == null) timeWindow else null, - attachments - baselineTransactionBuilder.attachments.toSet(), commands.drop(baselineTransactionBuilder.commands.size), signatories - baselineTransactionBuilder.signatories.toSet(), inputStateRefs - baselineTransactionBuilder.inputStateRefs.toSet(), diff --git a/components/ledger/ledger-utxo-flow/src/main/kotlin/net/corda/ledger/utxo/flow/impl/transaction/UtxoTransactionBuilderContainer.kt b/components/ledger/ledger-utxo-flow/src/main/kotlin/net/corda/ledger/utxo/flow/impl/transaction/UtxoTransactionBuilderContainer.kt index 6762e7da1e3..6f01088c7b7 100644 --- a/components/ledger/ledger-utxo-flow/src/main/kotlin/net/corda/ledger/utxo/flow/impl/transaction/UtxoTransactionBuilderContainer.kt +++ b/components/ledger/ledger-utxo-flow/src/main/kotlin/net/corda/ledger/utxo/flow/impl/transaction/UtxoTransactionBuilderContainer.kt @@ -2,7 +2,6 @@ package net.corda.ledger.utxo.flow.impl.transaction import net.corda.v5.base.annotations.CordaSerializable import net.corda.v5.base.types.MemberX500Name -import net.corda.v5.crypto.SecureHash import net.corda.v5.ledger.utxo.Command import net.corda.v5.ledger.utxo.StateRef import net.corda.v5.ledger.utxo.TimeWindow @@ -12,7 +11,6 @@ import java.security.PublicKey data class UtxoTransactionBuilderContainer( private val notaryName: MemberX500Name? = null, override val timeWindow: TimeWindow? = null, - override val attachments: List = listOf(), override val commands: List = listOf(), override val signatories: List = listOf(), override val inputStateRefs: List = listOf(), diff --git a/components/ledger/ledger-utxo-flow/src/main/kotlin/net/corda/ledger/utxo/flow/impl/transaction/UtxoTransactionBuilderData.kt b/components/ledger/ledger-utxo-flow/src/main/kotlin/net/corda/ledger/utxo/flow/impl/transaction/UtxoTransactionBuilderData.kt index 1eaa574b3fc..7cd638da87b 100644 --- a/components/ledger/ledger-utxo-flow/src/main/kotlin/net/corda/ledger/utxo/flow/impl/transaction/UtxoTransactionBuilderData.kt +++ b/components/ledger/ledger-utxo-flow/src/main/kotlin/net/corda/ledger/utxo/flow/impl/transaction/UtxoTransactionBuilderData.kt @@ -9,7 +9,6 @@ import java.security.PublicKey interface UtxoTransactionBuilderData { val timeWindow: TimeWindow? - val attachments: List val commands: List val signatories: List val inputStateRefs: List diff --git a/components/ledger/ledger-utxo-flow/src/main/kotlin/net/corda/ledger/utxo/flow/impl/transaction/UtxoTransactionBuilderImpl.kt b/components/ledger/ledger-utxo-flow/src/main/kotlin/net/corda/ledger/utxo/flow/impl/transaction/UtxoTransactionBuilderImpl.kt index df2277264a1..c8305204382 100644 --- a/components/ledger/ledger-utxo-flow/src/main/kotlin/net/corda/ledger/utxo/flow/impl/transaction/UtxoTransactionBuilderImpl.kt +++ b/components/ledger/ledger-utxo-flow/src/main/kotlin/net/corda/ledger/utxo/flow/impl/transaction/UtxoTransactionBuilderImpl.kt @@ -6,7 +6,6 @@ import net.corda.ledger.utxo.flow.impl.transaction.factory.UtxoSignedTransaction import net.corda.ledger.utxo.flow.impl.transaction.verifier.UtxoTransactionBuilderVerifier import net.corda.v5.base.annotations.Suspendable import net.corda.v5.base.types.MemberX500Name -import net.corda.v5.crypto.SecureHash import net.corda.v5.ledger.common.NotaryLookup import net.corda.v5.ledger.utxo.Command import net.corda.v5.ledger.utxo.ContractState @@ -25,7 +24,6 @@ class UtxoTransactionBuilderImpl( private var notaryName: MemberX500Name? = null, private var notaryKey: PublicKey? = null, override var timeWindow: TimeWindow? = null, - override val attachments: MutableList = mutableListOf(), override val commands: MutableList = mutableListOf(), override val signatories: MutableList = mutableListOf(), override val inputStateRefs: MutableList = mutableListOf(), @@ -35,14 +33,6 @@ class UtxoTransactionBuilderImpl( private var alreadySigned = false - override fun addAttachment(attachmentId: SecureHash): UtxoTransactionBuilder { - require(attachmentId !in attachments) { - "Duplicating attachments is not allowed." - } - this.attachments += attachmentId - return this - } - override fun addCommand(command: Command): UtxoTransactionBuilder { this.commands += command return this @@ -183,7 +173,6 @@ class UtxoTransactionBuilderImpl( return UtxoTransactionBuilderContainer( notaryName, timeWindow, - attachments.toMutableList(), commands.toMutableList(), signatories.toMutableList(), inputStateRefs.toMutableList(), @@ -213,7 +202,6 @@ class UtxoTransactionBuilderImpl( && other.notaryName == notaryName && other.notaryKey == notaryKey && other.timeWindow == timeWindow - && other.attachments == attachments && other.commands == commands && other.inputStateRefs == inputStateRefs && other.referenceStateRefs == referenceStateRefs @@ -225,7 +213,6 @@ class UtxoTransactionBuilderImpl( notaryName, notaryKey, timeWindow, - attachments, commands, signatories, inputStateRefs, @@ -237,7 +224,6 @@ class UtxoTransactionBuilderImpl( return "UtxoTransactionBuilderImpl(" + "notary=$notaryName (key: $notaryKey), " + "timeWindow=$timeWindow, " + - "attachments=$attachments, " + "commands=$commands, " + "signatories=$signatories, " + "inputStateRefs=$inputStateRefs, " + @@ -251,7 +237,6 @@ class UtxoTransactionBuilderImpl( * Also, notary and time window of the original takes precedence. * Those will not be overwritten regardless of there are new values. * It de-duplicates the - * - attachments * - signatories * - inputStateRefs * - referenceStateRefs @@ -264,7 +249,6 @@ class UtxoTransactionBuilderImpl( this.notaryName ?: other.getNotaryName(), this.notaryKey ?: lookUpNotaryKey(other.getNotaryName()), this.timeWindow ?: other.timeWindow, - (this.attachments + other.attachments).distinct().toMutableList(), (this.commands + other.commands).toMutableList(), (this.signatories + other.signatories).distinct().toMutableList(), (this.inputStateRefs + other.inputStateRefs).distinct().toMutableList(), diff --git a/components/ledger/ledger-utxo-flow/src/main/kotlin/net/corda/ledger/utxo/flow/impl/transaction/UtxoTransactionBuilderInternal.kt b/components/ledger/ledger-utxo-flow/src/main/kotlin/net/corda/ledger/utxo/flow/impl/transaction/UtxoTransactionBuilderInternal.kt index ad55f85a2fd..070da22f96d 100644 --- a/components/ledger/ledger-utxo-flow/src/main/kotlin/net/corda/ledger/utxo/flow/impl/transaction/UtxoTransactionBuilderInternal.kt +++ b/components/ledger/ledger-utxo-flow/src/main/kotlin/net/corda/ledger/utxo/flow/impl/transaction/UtxoTransactionBuilderInternal.kt @@ -15,7 +15,6 @@ interface UtxoTransactionBuilderInternal : UtxoTransactionBuilder, UtxoTransacti * Also, notary and time window of the original takes precedence. * Those will not be overwritten regardless of there are new values. * It de-duplicates the - * - attachments * - signatories * - inputStateRefs * - referenceStateRefs diff --git a/components/ledger/ledger-utxo-flow/src/main/kotlin/net/corda/ledger/utxo/flow/impl/transaction/factory/impl/UtxoSignedTransactionFactoryImpl.kt b/components/ledger/ledger-utxo-flow/src/main/kotlin/net/corda/ledger/utxo/flow/impl/transaction/factory/impl/UtxoSignedTransactionFactoryImpl.kt index 5622bad62f0..18e040e7de7 100644 --- a/components/ledger/ledger-utxo-flow/src/main/kotlin/net/corda/ledger/utxo/flow/impl/transaction/factory/impl/UtxoSignedTransactionFactoryImpl.kt +++ b/components/ledger/ledger-utxo-flow/src/main/kotlin/net/corda/ledger/utxo/flow/impl/transaction/factory/impl/UtxoSignedTransactionFactoryImpl.kt @@ -176,7 +176,7 @@ class UtxoSignedTransactionFactoryImpl @Activate constructor( UtxoComponentGroup.SIGNATORIES -> utxoTransactionBuilder.signatories UtxoComponentGroup.OUTPUTS_INFO -> outputsInfo UtxoComponentGroup.COMMANDS_INFO -> commandsInfo - UtxoComponentGroup.DATA_ATTACHMENTS -> utxoTransactionBuilder.attachments + UtxoComponentGroup.UNUSED -> emptyList() UtxoComponentGroup.INPUTS -> utxoTransactionBuilder.inputStateRefs UtxoComponentGroup.OUTPUTS -> outputTransactionStates.map { it.contractState diff --git a/components/ledger/ledger-utxo-flow/src/test/kotlin/net/corda/ledger/utxo/flow/impl/UtxoLedgerServiceImplTest.kt b/components/ledger/ledger-utxo-flow/src/test/kotlin/net/corda/ledger/utxo/flow/impl/UtxoLedgerServiceImplTest.kt index e7d264aa046..0eb8a36b4a9 100644 --- a/components/ledger/ledger-utxo-flow/src/test/kotlin/net/corda/ledger/utxo/flow/impl/UtxoLedgerServiceImplTest.kt +++ b/components/ledger/ledger-utxo-flow/src/test/kotlin/net/corda/ledger/utxo/flow/impl/UtxoLedgerServiceImplTest.kt @@ -1,6 +1,5 @@ package net.corda.ledger.utxo.flow.impl -import net.corda.crypto.core.SecureHashImpl import net.corda.flow.pipeline.exceptions.FlowFatalException import net.corda.flow.pipeline.sandbox.FlowSandboxGroupContext import net.corda.flow.pipeline.sessions.protocol.FlowProtocolStore @@ -57,7 +56,6 @@ class UtxoLedgerServiceImplTest: UtxoLedgerTest() { .thenReturn(listOf(inputStateAndRef, referenceStateAndRef)) val command = UtxoCommandExample() - val attachment = SecureHashImpl("SHA-256", ByteArray(12)) val signedTransaction = transactionBuilder .setNotary(notaryX500Name) @@ -67,7 +65,6 @@ class UtxoLedgerServiceImplTest: UtxoLedgerTest() { .addReferenceState(referenceStateRef) .addSignatories(listOf(publicKeyExample)) .addCommand(command) - .addAttachment(attachment) .toSignedTransaction() assertIs(signedTransaction) diff --git a/components/ledger/ledger-utxo-flow/src/test/kotlin/net/corda/ledger/utxo/flow/impl/flows/transactionbuilder/v1/ReceiveAndUpdateTransactionBuilderFlowV1Test.kt b/components/ledger/ledger-utxo-flow/src/test/kotlin/net/corda/ledger/utxo/flow/impl/flows/transactionbuilder/v1/ReceiveAndUpdateTransactionBuilderFlowV1Test.kt index 05b40992c19..1f8a195d6be 100644 --- a/components/ledger/ledger-utxo-flow/src/test/kotlin/net/corda/ledger/utxo/flow/impl/flows/transactionbuilder/v1/ReceiveAndUpdateTransactionBuilderFlowV1Test.kt +++ b/components/ledger/ledger-utxo-flow/src/test/kotlin/net/corda/ledger/utxo/flow/impl/flows/transactionbuilder/v1/ReceiveAndUpdateTransactionBuilderFlowV1Test.kt @@ -114,44 +114,6 @@ class ReceiveAndUpdateTransactionBuilderFlowV1Test : UtxoLedgerTest() { verify(mockFlowEngine, never()).subFlow(any()) } - @Test - fun `receiving new attachments appends them`() { - whenever(session.receive(UtxoTransactionBuilderContainer::class.java)).thenReturn( - UtxoTransactionBuilderContainer(attachments = listOf(hash1, hash2)) - ) - - val returnedTransactionBuilder = callSendFlow() - - assertContentEquals(listOf(hash1, hash2), returnedTransactionBuilder.attachments) - verify(mockFlowEngine, never()).subFlow(any()) - } - - @Test - fun `receiving existing attachment does not append it`() { - originalTransactionalBuilder.addAttachment(hash1) - whenever(session.receive(UtxoTransactionBuilderContainer::class.java)).thenReturn( - UtxoTransactionBuilderContainer(attachments = mutableListOf(hash1)) - ) - - val returnedTransactionBuilder = callSendFlow() - - assertContentEquals(listOf(hash1), returnedTransactionBuilder.attachments) - verify(mockFlowEngine, never()).subFlow(any()) - } - - @Test - fun `receiving duplicated attachments appends once`() { - originalTransactionalBuilder.addAttachment(hash1) - whenever(session.receive(UtxoTransactionBuilderContainer::class.java)).thenReturn( - UtxoTransactionBuilderContainer(attachments = mutableListOf(hash1, hash2, hash2)) - ) - - val returnedTransactionBuilder = callSendFlow() - - assertContentEquals(listOf(hash1, hash2), returnedTransactionBuilder.attachments) - verify(mockFlowEngine, never()).subFlow(any()) - } - @Test fun `receiving commands appends them (new, old, duplicated)`() { originalTransactionalBuilder.addCommand(command1) diff --git a/components/ledger/ledger-utxo-flow/src/test/kotlin/net/corda/ledger/utxo/flow/impl/flows/transactionbuilder/v1/SendTransactionBuilderDiffFlowV1Test.kt b/components/ledger/ledger-utxo-flow/src/test/kotlin/net/corda/ledger/utxo/flow/impl/flows/transactionbuilder/v1/SendTransactionBuilderDiffFlowV1Test.kt index 241650132f6..2d1d1b742a4 100644 --- a/components/ledger/ledger-utxo-flow/src/test/kotlin/net/corda/ledger/utxo/flow/impl/flows/transactionbuilder/v1/SendTransactionBuilderDiffFlowV1Test.kt +++ b/components/ledger/ledger-utxo-flow/src/test/kotlin/net/corda/ledger/utxo/flow/impl/flows/transactionbuilder/v1/SendTransactionBuilderDiffFlowV1Test.kt @@ -48,7 +48,6 @@ class SendTransactionBuilderDiffFlowV1Test { whenever(currentTransactionBuilder.notaryName).thenReturn(null) whenever(currentTransactionBuilder.notaryKey).thenReturn(null) whenever(currentTransactionBuilder.timeWindow).thenReturn(null) - whenever(currentTransactionBuilder.attachments).thenReturn(listOf()) whenever(currentTransactionBuilder.commands).thenReturn(listOf()) whenever(currentTransactionBuilder.signatories).thenReturn(listOf()) whenever(currentTransactionBuilder.inputStateRefs).thenReturn(listOf()) @@ -58,7 +57,6 @@ class SendTransactionBuilderDiffFlowV1Test { whenever(originalTransactionalBuilder.getNotaryName()).thenReturn(null) whenever(originalTransactionalBuilder.timeWindow).thenReturn(null) - whenever(originalTransactionalBuilder.attachments).thenReturn(listOf()) whenever(originalTransactionalBuilder.commands).thenReturn(listOf()) whenever(originalTransactionalBuilder.signatories).thenReturn(listOf()) whenever(originalTransactionalBuilder.inputStateRefs).thenReturn(listOf()) @@ -118,39 +116,6 @@ class SendTransactionBuilderDiffFlowV1Test { verify(flowEngine, never()).subFlow(any()) } - @Test - fun `called with the same attachments does not send anything`() { - whenever(originalTransactionalBuilder.attachments).thenReturn(listOf(hash1)) - whenever(currentTransactionBuilder.attachments).thenReturn(listOf(hash1)) - - callSendFlow() - - verify(session).send(UtxoTransactionBuilderContainer()) - verify(flowEngine, never()).subFlow(any()) - } - - @Test - fun `called with the same attachments duplicated does not send anything`() { - whenever(originalTransactionalBuilder.attachments).thenReturn(listOf(hash1)) - whenever(currentTransactionBuilder.attachments).thenReturn(listOf(hash1, hash1)) - - callSendFlow() - - verify(session).send(UtxoTransactionBuilderContainer()) - verify(flowEngine, never()).subFlow(any()) - } - - @Test - fun `called with a new attachments returns the new only`() { - whenever(originalTransactionalBuilder.attachments).thenReturn(listOf(hash1)) - whenever(currentTransactionBuilder.attachments).thenReturn(listOf(hash1, hash2)) - - callSendFlow() - - verify(session).send(UtxoTransactionBuilderContainer(attachments = mutableListOf(hash2))) - verify(flowEngine, never()).subFlow(any()) - } - @Test fun `called with the same commands does not send anything`() { whenever(originalTransactionalBuilder.commands).thenReturn(listOf(command1)) diff --git a/components/ledger/ledger-utxo-flow/src/test/kotlin/net/corda/ledger/utxo/flow/impl/transaction/UtxoBaselinedTransactionBuilderDiffTest.kt b/components/ledger/ledger-utxo-flow/src/test/kotlin/net/corda/ledger/utxo/flow/impl/transaction/UtxoBaselinedTransactionBuilderDiffTest.kt index 4182c5def40..a9ff35d1259 100644 --- a/components/ledger/ledger-utxo-flow/src/test/kotlin/net/corda/ledger/utxo/flow/impl/transaction/UtxoBaselinedTransactionBuilderDiffTest.kt +++ b/components/ledger/ledger-utxo-flow/src/test/kotlin/net/corda/ledger/utxo/flow/impl/transaction/UtxoBaselinedTransactionBuilderDiffTest.kt @@ -73,29 +73,6 @@ class UtxoBaselinedTransactionBuilderDiffTest : UtxoLedgerTest() { assertNull(result.timeWindow) } - @Test - fun `diff - attachments do not get set if they are the same`() { - val result = - UtxoBaselinedTransactionBuilder( - utxoTransactionBuilder.addAttachment(hash1).addAttachment(hash2) as UtxoTransactionBuilderInternal - ) - .diff() - assertEquals(UtxoTransactionBuilderContainer(), result) - } - - - @Test - fun `diff - attachments get set when there is a new one`() { - val result = - UtxoBaselinedTransactionBuilder( - utxoTransactionBuilder.addAttachment(hash1) as UtxoTransactionBuilderInternal - ) - .addAttachment(hash2) - .diff() - - assertContentEquals(listOf(hash2), result.attachments) - } - @Test fun `diff - commands get added regardless of previous same ones or duplications`() { val result = diff --git a/components/ledger/ledger-utxo-flow/src/test/kotlin/net/corda/ledger/utxo/flow/impl/transaction/UtxoLedgerTransactionImplTest.kt b/components/ledger/ledger-utxo-flow/src/test/kotlin/net/corda/ledger/utxo/flow/impl/transaction/UtxoLedgerTransactionImplTest.kt index 62e0eb60ed2..303b1c15491 100644 --- a/components/ledger/ledger-utxo-flow/src/test/kotlin/net/corda/ledger/utxo/flow/impl/transaction/UtxoLedgerTransactionImplTest.kt +++ b/components/ledger/ledger-utxo-flow/src/test/kotlin/net/corda/ledger/utxo/flow/impl/transaction/UtxoLedgerTransactionImplTest.kt @@ -1,6 +1,5 @@ package net.corda.ledger.utxo.flow.impl.transaction -import net.corda.crypto.core.SecureHashImpl import net.corda.ledger.common.testkit.publicKeyExample import net.corda.ledger.utxo.test.UtxoLedgerTest import net.corda.ledger.utxo.testkit.UtxoCommandExample @@ -28,7 +27,6 @@ internal class UtxoLedgerTransactionImplTest : UtxoLedgerTest() { private val referenceStateRef = referenceStateAndRef.ref private val command = UtxoCommandExample() - private val attachment = SecureHashImpl("SHA-256", ByteArray(12)) private lateinit var ledgerTransaction: UtxoLedgerTransaction @@ -48,7 +46,6 @@ internal class UtxoLedgerTransactionImplTest : UtxoLedgerTest() { .addReferenceState(referenceStateRef) .addSignatories(listOf(publicKeyExample)) .addCommand(command) - .addAttachment(attachment) .toSignedTransaction() ledgerTransaction = signedTransaction.toLedgerTransaction() } diff --git a/components/ledger/ledger-utxo-flow/src/test/kotlin/net/corda/ledger/utxo/flow/impl/transaction/UtxoTransactionBuilderImplAppendTest.kt b/components/ledger/ledger-utxo-flow/src/test/kotlin/net/corda/ledger/utxo/flow/impl/transaction/UtxoTransactionBuilderImplAppendTest.kt index 90678e5bf4a..a29db8b053c 100644 --- a/components/ledger/ledger-utxo-flow/src/test/kotlin/net/corda/ledger/utxo/flow/impl/transaction/UtxoTransactionBuilderImplAppendTest.kt +++ b/components/ledger/ledger-utxo-flow/src/test/kotlin/net/corda/ledger/utxo/flow/impl/transaction/UtxoTransactionBuilderImplAppendTest.kt @@ -86,27 +86,6 @@ class UtxoTransactionBuilderImplAppendTest : UtxoLedgerTest() { assertEquals(utxoTimeWindowExample, result.timeWindow) } - @Test - fun `Does not add again already added attachments`() { - originalTransactionalBuilder.addAttachment(hash1) - val result = originalTransactionalBuilder.append( - UtxoTransactionBuilderContainer( - attachments = listOf(hash1) - ) - ) - assertContentEquals(listOf(hash1), result.attachments) - } - - @Test - fun `Appends and deduplicates new attachments`() { - val result = originalTransactionalBuilder.append( - UtxoTransactionBuilderContainer( - attachments = listOf(hash1, hash1, hash2) - ) - ) - assertContentEquals(listOf(hash1, hash2), result.attachments) - } - @Test fun `Appends commands`() { originalTransactionalBuilder.addCommand(command1) diff --git a/components/ledger/ledger-utxo-flow/src/test/kotlin/net/corda/ledger/utxo/flow/impl/transaction/UtxoTransactionBuilderImplTest.kt b/components/ledger/ledger-utxo-flow/src/test/kotlin/net/corda/ledger/utxo/flow/impl/transaction/UtxoTransactionBuilderImplTest.kt index a8784cffe66..912172a0d1f 100644 --- a/components/ledger/ledger-utxo-flow/src/test/kotlin/net/corda/ledger/utxo/flow/impl/transaction/UtxoTransactionBuilderImplTest.kt +++ b/components/ledger/ledger-utxo-flow/src/test/kotlin/net/corda/ledger/utxo/flow/impl/transaction/UtxoTransactionBuilderImplTest.kt @@ -50,7 +50,6 @@ class UtxoTransactionBuilderImplTest : UtxoLedgerTest() { .addReferenceState(referenceStateRef) .addSignatories(listOf(publicKeyExample)) .addCommand(UtxoCommandExample()) - .addAttachment(SecureHashImpl("SHA-256", ByteArray(12))) .toSignedTransaction() assertIs(tx.id) assertEquals(inputStateRef, tx.inputStateRefs.single()) @@ -89,7 +88,6 @@ class UtxoTransactionBuilderImplTest : UtxoLedgerTest() { .addOutputState(getUtxoStateExample()) .addSignatories(listOf(publicKeyExample)) .addCommand(UtxoCommandExample()) - .addAttachment(SecureHashImpl("SHA-256", ByteArray(12))) .toSignedTransaction() as UtxoSignedTransactionImpl val metadata = tx.wireTransaction.metadata as TransactionMetadataInternal @@ -129,7 +127,6 @@ class UtxoTransactionBuilderImplTest : UtxoLedgerTest() { .addOutputState(getUtxoStateExample()) .addSignatories(listOf(publicKeyExample)) .addCommand(UtxoCommandExample()) - .addAttachment(SecureHashImpl("SHA-256", ByteArray(12))) builder.toSignedTransaction() builder.toSignedTransaction() @@ -169,7 +166,6 @@ class UtxoTransactionBuilderImplTest : UtxoLedgerTest() { .addReferenceState(referenceStateRef) .addSignatories(listOf(publicKeyExample)) .addCommand(UtxoCommandExample()) - .addAttachment(SecureHashImpl("SHA-256", ByteArray(12))) .toSignedTransaction() assertThat(tx.outputStateAndRefs).hasSize(7) @@ -229,24 +225,6 @@ class UtxoTransactionBuilderImplTest : UtxoLedgerTest() { ) } - @Test - fun `adding attachments mutates and returns the current builder`() { - val attachmentId = SecureHashImpl("SHA-256", ByteArray(12)) - val originalTransactionBuilder = utxoTransactionBuilder - val mutatedTransactionBuilder = utxoTransactionBuilder.addAttachment(attachmentId) - assertThat((mutatedTransactionBuilder as UtxoTransactionBuilderInternal).attachments).isEqualTo( - listOf( - attachmentId - ) - ) - assertThat(mutatedTransactionBuilder).isEqualTo(originalTransactionBuilder) - assertThat(System.identityHashCode(mutatedTransactionBuilder)).isEqualTo( - System.identityHashCode( - originalTransactionBuilder - ) - ) - } - @Test fun `adding commands mutates and returns the current builder`() { val command = UtxoCommandExample() @@ -431,17 +409,6 @@ class UtxoTransactionBuilderImplTest : UtxoLedgerTest() { ) } - @Test - fun `Duplicating attachments throws`() { - val attachmentId = SecureHashImpl("SHA", byteArrayOf(1, 1, 1, 1)) - utxoTransactionBuilder - .addAttachment(attachmentId) - assertThatThrownBy { - utxoTransactionBuilder - .addAttachment(attachmentId) - }.isInstanceOf(IllegalArgumentException::class.java) - } - @Test fun `Duplicating signatories throws`() { assertThatThrownBy { diff --git a/components/ledger/ledger-utxo-flow/src/test/kotlin/net/corda/ledger/utxo/flow/impl/transaction/serializer/amqp/UtxoSignedTransactionSerializerTest.kt b/components/ledger/ledger-utxo-flow/src/test/kotlin/net/corda/ledger/utxo/flow/impl/transaction/serializer/amqp/UtxoSignedTransactionSerializerTest.kt index 9afb82839f6..4209ddb047e 100644 --- a/components/ledger/ledger-utxo-flow/src/test/kotlin/net/corda/ledger/utxo/flow/impl/transaction/serializer/amqp/UtxoSignedTransactionSerializerTest.kt +++ b/components/ledger/ledger-utxo-flow/src/test/kotlin/net/corda/ledger/utxo/flow/impl/transaction/serializer/amqp/UtxoSignedTransactionSerializerTest.kt @@ -1,6 +1,5 @@ package net.corda.ledger.utxo.flow.impl.transaction.serializer.amqp -import net.corda.crypto.core.SecureHashImpl import net.corda.internal.serialization.amqp.helper.TestSerializationService import net.corda.ledger.common.testkit.publicKeyExample import net.corda.ledger.utxo.test.UtxoLedgerTest @@ -58,7 +57,6 @@ class UtxoSignedTransactionSerializerTest : UtxoLedgerTest() { .addReferenceState(referenceStateRef) .addSignatories(listOf(publicKeyExample)) .addCommand(UtxoCommandExample()) - .addAttachment(SecureHashImpl("SHA-256", ByteArray(12))) .toSignedTransaction() val bytes = serializationService.serialize(signedTx) diff --git a/gradle.properties b/gradle.properties index 0bd8bfe6dbc..87d66d36fb6 100644 --- a/gradle.properties +++ b/gradle.properties @@ -46,7 +46,7 @@ commonsTextVersion = 1.10.0 bouncycastleVersion=1.76 # Corda API libs revision (change in 4th digit indicates a breaking change) # Change to 5.1.0.xx-SNAPSHOT to pick up maven local published copy -cordaApiVersion=5.1.0.34-beta+ +cordaApiVersion=5.1.0.35-beta+ disruptorVersion=3.4.4 felixConfigAdminVersion=1.9.26 diff --git a/libs/ledger/ledger-utxo-data/src/main/kotlin/net/corda/ledger/utxo/data/transaction/UtxoComponentGroup.kt b/libs/ledger/ledger-utxo-data/src/main/kotlin/net/corda/ledger/utxo/data/transaction/UtxoComponentGroup.kt index 840c3df58e3..d5aaac9e2b1 100644 --- a/libs/ledger/ledger-utxo-data/src/main/kotlin/net/corda/ledger/utxo/data/transaction/UtxoComponentGroup.kt +++ b/libs/ledger/ledger-utxo-data/src/main/kotlin/net/corda/ledger/utxo/data/transaction/UtxoComponentGroup.kt @@ -16,7 +16,7 @@ enum class UtxoComponentGroup { SIGNATORIES, OUTPUTS_INFO, COMMANDS_INFO, - DATA_ATTACHMENTS, + UNUSED, // can't remove to keep the Merkle hash stable INPUTS, REFERENCES, OUTPUTS, diff --git a/libs/ledger/ledger-utxo-data/src/main/kotlin/net/corda/ledger/utxo/data/transaction/UtxoLedgerTransactionImpl.kt b/libs/ledger/ledger-utxo-data/src/main/kotlin/net/corda/ledger/utxo/data/transaction/UtxoLedgerTransactionImpl.kt index ebce6aeac39..7e3d5f37269 100644 --- a/libs/ledger/ledger-utxo-data/src/main/kotlin/net/corda/ledger/utxo/data/transaction/UtxoLedgerTransactionImpl.kt +++ b/libs/ledger/ledger-utxo-data/src/main/kotlin/net/corda/ledger/utxo/data/transaction/UtxoLedgerTransactionImpl.kt @@ -6,7 +6,6 @@ import net.corda.ledger.utxo.data.transaction.verifier.verifyMetadata import net.corda.v5.base.types.MemberX500Name import net.corda.v5.crypto.SecureHash import net.corda.v5.ledger.common.transaction.TransactionMetadata -import net.corda.v5.ledger.utxo.Attachment import net.corda.v5.ledger.utxo.Command import net.corda.v5.ledger.utxo.ContractState import net.corda.v5.ledger.utxo.StateAndRef @@ -55,16 +54,6 @@ class UtxoLedgerTransactionImpl( return wrappedWireTransaction.signatories } - override fun getAttachments(): List { - return wrappedWireTransaction.attachments - } - - override fun getAttachment(id: SecureHash): Attachment { - return requireNotNull(attachments.singleOrNull { it.id == id }) { - "Failed to find a single attachment with id: $id." - } - } - override fun getCommands(): List { return wrappedWireTransaction.commands } diff --git a/libs/ledger/ledger-utxo-data/src/main/kotlin/net/corda/ledger/utxo/data/transaction/WrappedUtxoWireTransaction.kt b/libs/ledger/ledger-utxo-data/src/main/kotlin/net/corda/ledger/utxo/data/transaction/WrappedUtxoWireTransaction.kt index ed7eee1be71..ba13b850ed9 100644 --- a/libs/ledger/ledger-utxo-data/src/main/kotlin/net/corda/ledger/utxo/data/transaction/WrappedUtxoWireTransaction.kt +++ b/libs/ledger/ledger-utxo-data/src/main/kotlin/net/corda/ledger/utxo/data/transaction/WrappedUtxoWireTransaction.kt @@ -7,7 +7,6 @@ import net.corda.v5.application.serialization.SerializationService import net.corda.v5.base.types.MemberX500Name import net.corda.v5.crypto.SecureHash import net.corda.v5.ledger.common.transaction.TransactionMetadata -import net.corda.v5.ledger.utxo.Attachment import net.corda.v5.ledger.utxo.Command import net.corda.v5.ledger.utxo.ContractState import net.corda.v5.ledger.utxo.StateAndRef @@ -51,15 +50,6 @@ class WrappedUtxoWireTransaction( deserialize(UtxoComponentGroup.NOTARY, timeWindowIndex) } - val attachmentIds: List by lazy(LazyThreadSafetyMode.PUBLICATION) { - deserialize(UtxoComponentGroup.DATA_ATTACHMENTS) - } - - val attachments: List by lazy(LazyThreadSafetyMode.PUBLICATION) { - //TODO("Not yet implemented.") - emptyList() - } - val commands: List by lazy(LazyThreadSafetyMode.PUBLICATION) { deserialize(UtxoComponentGroup.COMMANDS) } From 8c013bfaca049711040d4781575cc20fbf7f6227 Mon Sep 17 00:00:00 2001 From: Nikolett Nagy <61757742+nikinagy@users.noreply.github.com> Date: Fri, 13 Oct 2023 16:26:58 +0100 Subject: [PATCH 20/81] CORE-16752 - Unclear error message after failed signature verification (#4863) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Original scope was: The below conversation shows that if certificate validation fails, it’s unclear which certificate the error refers to. In this case it must refer to the TLS certificate because the session certificate is not used, but the error message should be clearer regardless. However, we lately merged in a fix where we specify which type of certificate was wrong, so the message should be okay. This change resulted in Internal Server Error 500, which is wrong so modified it to 400. --- .../certificate/client/impl/HostedIdentityEntryFactory.kt | 2 +- .../membership/impl/rest/v1/NetworkRestResourceImpl.kt | 6 +----- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/components/membership/certificates-client-impl/src/main/kotlin/net/corda/membership/certificate/client/impl/HostedIdentityEntryFactory.kt b/components/membership/certificates-client-impl/src/main/kotlin/net/corda/membership/certificate/client/impl/HostedIdentityEntryFactory.kt index a75e29e65c8..dfaf7f72e1c 100644 --- a/components/membership/certificates-client-impl/src/main/kotlin/net/corda/membership/certificate/client/impl/HostedIdentityEntryFactory.kt +++ b/components/membership/certificates-client-impl/src/main/kotlin/net/corda/membership/certificate/client/impl/HostedIdentityEntryFactory.kt @@ -281,7 +281,7 @@ internal class HostedIdentityEntryFactory( } catch (e: SignatureException) { false } - }.firstOrNull() ?: throw CordaRuntimeException( + }.firstOrNull() ?: throw BadRequestException( "The ${certificateType.type.label} certificate was not signed by the correct certificate authority" ) } diff --git a/components/membership/membership-rest-impl/src/main/kotlin/net/corda/membership/impl/rest/v1/NetworkRestResourceImpl.kt b/components/membership/membership-rest-impl/src/main/kotlin/net/corda/membership/impl/rest/v1/NetworkRestResourceImpl.kt index dceb54c0a05..8ab7a0a786d 100644 --- a/components/membership/membership-rest-impl/src/main/kotlin/net/corda/membership/impl/rest/v1/NetworkRestResourceImpl.kt +++ b/components/membership/membership-rest-impl/src/main/kotlin/net/corda/membership/impl/rest/v1/NetworkRestResourceImpl.kt @@ -24,7 +24,6 @@ import org.osgi.service.component.annotations.Activate import org.osgi.service.component.annotations.Component import org.osgi.service.component.annotations.Reference import org.slf4j.LoggerFactory -import java.security.SignatureException @Component(service = [PluggableRestResource::class]) class NetworkRestResourceImpl @Activate constructor( @@ -72,11 +71,8 @@ class NetworkRestResourceImpl @Activate constructor( } catch (e: CertificatesResourceNotFoundException) { throw ResourceNotFoundException(e.message) } catch (e: BadRequestException) { - logger.warn(e.message) - throw e - } catch (e: SignatureException) { logger.warn("Could not $operation", e) - throw BadRequestException("The certificate was not signed by the correct certificate authority. ${e.message}") + throw e } catch (e: CordaRPCAPIPartitionException) { logger.warn("Could not $operation", e) throw ServiceUnavailableException("Could not $operation: Repartition Event!") From d5dc203c08b96aed6462a8bf891c2a93bf9c276d Mon Sep 17 00:00:00 2001 From: Ben Millar <44114751+ben-millar@users.noreply.github.com> Date: Fri, 13 Oct 2023 16:45:21 +0100 Subject: [PATCH 21/81] CORE-16181 Implement RPC Producer that implements RPC client (#4709) This PR adds a `RPCClient` which will, in the future, be used by the Multi-source Event Mediator to make calls to Corda workers via RPC. Also included is a simple `HTTPRetryExecutor` which will retry HTTP calls a configurable number of times with incremental backoff and allows for a user-specified set of retryable exceptions. --- charts/corda-lib/templates/_worker.tpl | 2 +- .../net/corda/messaging/mediator/RPCClient.kt | 130 +++++++++++ .../MessagingClientFactoryFactoryImpl.kt | 12 +- .../mediator/factory/RPCClientFactory.kt | 21 ++ .../corda/messaging/utils/HTTPRetryConfig.kt | 37 +++ .../messaging/utils/HTTPRetryExecutor.kt | 42 ++++ .../corda/messaging/mediator/RPCClientTest.kt | 219 ++++++++++++++++++ .../MessagingClientFactoryFactoryTest.kt | 14 +- .../mediator/factory/RPCClientFactoryTest.kt | 29 +++ .../messaging/utils/HTTPRetryExecutorTest.kt | 77 ++++++ .../api/exception/CordaRestAPIExceptions.kt | 12 + .../factory/MessagingClientFactoryFactory.kt | 11 +- 12 files changed, 602 insertions(+), 4 deletions(-) create mode 100644 libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/RPCClient.kt create mode 100644 libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/factory/RPCClientFactory.kt create mode 100644 libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/utils/HTTPRetryConfig.kt create mode 100644 libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/utils/HTTPRetryExecutor.kt create mode 100644 libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/RPCClientTest.kt create mode 100644 libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/factory/RPCClientFactoryTest.kt create mode 100644 libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/utils/HTTPRetryExecutorTest.kt diff --git a/charts/corda-lib/templates/_worker.tpl b/charts/corda-lib/templates/_worker.tpl index e9f348f9943..97fe731c992 100644 --- a/charts/corda-lib/templates/_worker.tpl +++ b/charts/corda-lib/templates/_worker.tpl @@ -84,7 +84,7 @@ metadata: spec: type: ClusterIP selector: - app: {{ $workerName }} + app.kubernetes.io/component: {{ include "corda.workerComponent" $worker }} ports: - protocol: TCP port: {{ include "corda.workerServicePort" . }} diff --git a/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/RPCClient.kt b/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/RPCClient.kt new file mode 100644 index 00000000000..23bcdd52912 --- /dev/null +++ b/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/RPCClient.kt @@ -0,0 +1,130 @@ +package net.corda.messaging.mediator + +import java.io.IOException +import java.net.URI +import java.net.http.HttpClient +import java.net.http.HttpRequest +import java.net.http.HttpResponse +import java.util.concurrent.TimeoutException +import net.corda.avro.serialization.CordaAvroSerializationFactory +import net.corda.messaging.api.exception.CordaHTTPClientErrorException +import net.corda.messaging.api.exception.CordaHTTPServerErrorException +import net.corda.messaging.api.mediator.MediatorMessage +import net.corda.messaging.api.mediator.MessagingClient +import net.corda.messaging.api.mediator.MessagingClient.Companion.MSG_PROP_ENDPOINT +import net.corda.messaging.api.records.Record +import net.corda.messaging.utils.HTTPRetryConfig +import net.corda.messaging.utils.HTTPRetryExecutor +import org.slf4j.Logger +import org.slf4j.LoggerFactory + +class RPCClient( + override val id: String, + cordaAvroSerializerFactory: CordaAvroSerializationFactory, + private val onSerializationError: ((ByteArray) -> Unit)?, + httpClientFactory: () -> HttpClient = { HttpClient.newBuilder().build() }, + private val retryConfig: HTTPRetryConfig = + HTTPRetryConfig.Builder() + .retryOn(IOException::class, TimeoutException::class) + .build() +) : MessagingClient { + private val httpClient: HttpClient = httpClientFactory() + private val serializer = cordaAvroSerializerFactory.createAvroSerializer {} + private val deserializer = cordaAvroSerializerFactory.createAvroDeserializer({}, Record::class.java) + + + private companion object { + private val log: Logger = LoggerFactory.getLogger(this::class.java.enclosingClass) + } + + override fun send(message: MediatorMessage<*>): MediatorMessage<*>? { + return try { + processMessage(message) + } catch (e: Exception) { + handleExceptions(e) + null + } + } + + private fun processMessage(message: MediatorMessage<*>): MediatorMessage<*> { + val payload = serializePayload(message) + val request = buildHttpRequest(payload, message.endpoint()) + val response = sendWithRetry(request) + + checkResponseStatus(response.statusCode()) + + val deserializedResponse = deserializePayload(response.body()) + return MediatorMessage(deserializedResponse, mutableMapOf("statusCode" to response.statusCode())) + } + + private fun serializePayload(message: MediatorMessage<*>): ByteArray { + val payload = message.payload + + if (payload is ByteArray) return payload + + try { + return serializer.serialize(message.payload as Record<*, *>)!! + } catch (e: Exception) { + val errorMsg = "Failed to serialize instance of class type ${ + message.payload?.let { it::class.java.name } ?: "null" + }." + log.error(errorMsg) + onSerializationError?.invoke(errorMsg.toByteArray()) + throw(e) + } + } + + private fun deserializePayload(payload: ByteArray): Record<*,*> { + return try { + deserializer.deserialize(payload)!! + } catch (e: Exception) { + val errorMsg = "Failed to deserialize payload of size ${payload.size} bytes due to: ${e.message}" + log.error(errorMsg) + onSerializationError?.invoke(errorMsg.toByteArray()) + throw(e) + } + } + + private fun buildHttpRequest(payload: ByteArray, endpoint: String): HttpRequest { + return HttpRequest.newBuilder() + .uri(URI("http://$endpoint")) + .PUT(HttpRequest.BodyPublishers.ofByteArray(payload)) + .build() + } + + private fun sendWithRetry(request: HttpRequest): HttpResponse { + return HTTPRetryExecutor.withConfig(retryConfig) { + httpClient.send(request, HttpResponse.BodyHandlers.ofByteArray()) + } + } + + private fun checkResponseStatus(statusCode: Int) { + when (statusCode) { + in 400..499 -> throw CordaHTTPClientErrorException(statusCode, "Server returned status code $statusCode.") + in 500..599 -> throw CordaHTTPServerErrorException(statusCode, "Server returned status code $statusCode.") + } + } + + private fun handleExceptions(e: Exception) { + when (e) { + is IOException -> log.error("Network or IO operation error in RPCClient: ", e) + is InterruptedException -> log.error("Operation was interrupted in RPCClient: ", e) + is IllegalArgumentException -> log.error("Invalid argument provided in RPCClient call: ", e) + is SecurityException -> log.error("Security violation detected in RPCClient: ", e) + is IllegalStateException -> log.error("Coroutine state error in RPCClient: ", e) + is CordaHTTPClientErrorException -> log.error("Client-side HTTP error in RPCClient: ", e) + is CordaHTTPServerErrorException -> log.error("Server-side HTTP error in RPCClient: ", e) + else -> log.error("Unhandled exception in RPCClient: ", e) + } + + throw e + } + + override fun close() { + // Nothing to do here + } + + private fun MediatorMessage<*>.endpoint(): String { + return getProperty(MSG_PROP_ENDPOINT) + } +} diff --git a/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/factory/MessagingClientFactoryFactoryImpl.kt b/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/factory/MessagingClientFactoryFactoryImpl.kt index 540e01d7240..0eecb634e7c 100644 --- a/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/factory/MessagingClientFactoryFactoryImpl.kt +++ b/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/factory/MessagingClientFactoryFactoryImpl.kt @@ -1,5 +1,6 @@ package net.corda.messaging.mediator.factory +import net.corda.avro.serialization.CordaAvroSerializationFactory import net.corda.libs.configuration.SmartConfig import net.corda.messagebus.api.producer.builder.CordaProducerBuilder import net.corda.messaging.api.mediator.factory.MessagingClientFactoryFactory @@ -14,6 +15,8 @@ import org.osgi.service.component.annotations.Reference class MessagingClientFactoryFactoryImpl @Activate constructor( @Reference(service = CordaProducerBuilder::class) private val cordaProducerBuilder: CordaProducerBuilder, + @Reference(service = CordaAvroSerializationFactory::class) + private val cordaSerializationFactory: CordaAvroSerializationFactory ): MessagingClientFactoryFactory { override fun createMessageBusClientFactory( id: String, @@ -23,4 +26,11 @@ class MessagingClientFactoryFactoryImpl @Activate constructor( messageBusConfig, cordaProducerBuilder, ) -} \ No newline at end of file + + override fun createRPCClientFactory( + id: String + ) = RPCClientFactory( + id, + cordaSerializationFactory + ) +} diff --git a/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/factory/RPCClientFactory.kt b/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/factory/RPCClientFactory.kt new file mode 100644 index 00000000000..cd6ba744268 --- /dev/null +++ b/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/factory/RPCClientFactory.kt @@ -0,0 +1,21 @@ +package net.corda.messaging.mediator.factory + +import net.corda.avro.serialization.CordaAvroSerializationFactory +import net.corda.messaging.api.mediator.MessagingClient +import net.corda.messaging.api.mediator.config.MessagingClientConfig +import net.corda.messaging.api.mediator.factory.MessagingClientFactory +import net.corda.messaging.mediator.RPCClient + +class RPCClientFactory( + private val id: String, + private val cordaSerializationFactory: CordaAvroSerializationFactory +): MessagingClientFactory { + + override fun create(config: MessagingClientConfig): MessagingClient { + return RPCClient( + id, + cordaSerializationFactory, + config.onSerializationError + ) + } +} diff --git a/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/utils/HTTPRetryConfig.kt b/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/utils/HTTPRetryConfig.kt new file mode 100644 index 00000000000..13a791b6531 --- /dev/null +++ b/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/utils/HTTPRetryConfig.kt @@ -0,0 +1,37 @@ +package net.corda.messaging.utils + +import kotlin.reflect.KClass + +/** + * Configuration class for HTTP retry parameters. + * + * @property times The number of a times a retry will be attempted. Default is 3 + * @property initialDelay The initial delay (in milliseconds) before the first retry. Default is 100ms + * @property factor The multiplier used to increase the delay for each subsequent retry. Default is 2.0 + * @property retryOn A set of exception classes that should trigger a retry when caught. + * If an exception not in this list is caught, it will be propagated immediately without retrying. + * Default is the generic [Exception] class, meaning all exceptions will trigger a retry. + */ +data class HTTPRetryConfig( + val times: Int = 3, + val initialDelay: Long = 100, + val factor: Double = 2.0, + val retryOn: Set> = setOf(Exception::class) +) { + class Builder { + private var times: Int = 3 + private var initialDelay: Long = 100 + private var factor: Double = 2.0 + private var retryOn: Set> = setOf(Exception::class) + + fun times(times: Int) = apply { this.times = times } + fun initialDelay(delay: Long) = apply { this.initialDelay = delay } + fun factor(factor: Double) = apply { this.factor = factor } + fun retryOn(vararg exceptions: KClass) = apply { this.retryOn = exceptions.toSet() } + + fun build(): HTTPRetryConfig { + return HTTPRetryConfig(times, initialDelay, factor, retryOn) + + } + } +} diff --git a/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/utils/HTTPRetryExecutor.kt b/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/utils/HTTPRetryExecutor.kt new file mode 100644 index 00000000000..99b6095a932 --- /dev/null +++ b/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/utils/HTTPRetryExecutor.kt @@ -0,0 +1,42 @@ +package net.corda.messaging.utils + +import org.slf4j.Logger +import org.slf4j.LoggerFactory + +class HTTPRetryExecutor { + companion object { + private val log: Logger = LoggerFactory.getLogger(this::class.java.enclosingClass) + + fun withConfig(config: HTTPRetryConfig, block: () -> T): T { + var currentDelay = config.initialDelay + for (i in 0 until config.times - 1) { + try { + log.info("HTTPRetryExecutor making attempt #${i + 1}.") + val result = block() + log.info("Operation successful after #${i + 1} attempt/s.") + return result + } catch (e: Exception) { + if (config.retryOn.none { it.isInstance(e) }) { + log.warn("HTTPRetryExecutor caught a non-retryable exception") + throw e + } + + log.warn("Attempt #${i + 1} failed due to ${e.message}. Retrying in $currentDelay ms...") + Thread.sleep(currentDelay) + currentDelay = (currentDelay * config.factor).toLong() + } + } + + log.warn("All retry attempts exhausted. Making the final call.") + + try { + val result = block() + log.info("Operation successful after #${config.times} attempt/s.") + return result + } catch (e: Exception) { + log.error("Operation failed after ${config.times} attempt/s.") + throw e + } + } + } +} diff --git a/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/RPCClientTest.kt b/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/RPCClientTest.kt new file mode 100644 index 00000000000..762308ef01d --- /dev/null +++ b/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/RPCClientTest.kt @@ -0,0 +1,219 @@ +package net.corda.messaging.mediator + +import java.io.IOException +import java.net.http.HttpClient +import java.net.http.HttpRequest +import java.net.http.HttpResponse +import net.corda.avro.serialization.CordaAvroDeserializer +import net.corda.avro.serialization.CordaAvroSerializationFactory +import net.corda.avro.serialization.CordaAvroSerializer +import net.corda.messaging.api.exception.CordaHTTPClientErrorException +import net.corda.messaging.api.exception.CordaHTTPServerErrorException +import net.corda.messaging.api.mediator.MediatorMessage +import net.corda.messaging.api.mediator.MessagingClient.Companion.MSG_PROP_ENDPOINT +import net.corda.messaging.api.records.Record +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertNotNull +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.assertThrows +import org.mockito.Mockito.times +import org.mockito.kotlin.any +import org.mockito.kotlin.eq +import org.mockito.kotlin.mock +import org.mockito.kotlin.verify +import org.mockito.kotlin.whenever + +class RPCClientTest { + + private lateinit var client: RPCClient + private val message = MediatorMessage( + Record("topic", "key", "testPayload"), + mutableMapOf(MSG_PROP_ENDPOINT to "test-endpoint/test") + ) + + data class Mocks( + val serializer: CordaAvroSerializer, + val deserializer: CordaAvroDeserializer>, + val httpClient: HttpClient, + val httpResponse: HttpResponse + ) + + private inner class MockEnvironment( + val mockSerializer: CordaAvroSerializer = mock(), + val mockDeserializer: CordaAvroDeserializer> = mock(), + val mockHttpClient: HttpClient = mock(), + val mockHttpResponse: HttpResponse = mock() + ) { + init { + whenever(mockSerializer.serialize(any>())) + .thenReturn("testPayload".toByteArray()) + + whenever(mockDeserializer.deserialize(any())) + .thenReturn(Record("topic", "key", "responsePayload")) + + whenever(mockHttpResponse.statusCode()) + .thenReturn(200) + + whenever(mockHttpResponse.body()) + .thenReturn("responsePayload".toByteArray()) + + whenever(mockHttpClient.send(any(), any>())) + .thenReturn(mockHttpResponse) + } + + fun withHttpStatus(status: Int) = apply { + whenever(mockHttpResponse.statusCode()).thenReturn(status) + } + + val mocks: Mocks + get() = Mocks(mockSerializer, mockDeserializer, mockHttpClient, mockHttpResponse) + } + + + private fun createClient( + mocks: Mocks, + onSerializationError: (ByteArray) -> Unit = mock(), + httpClientFactory: () -> HttpClient = { mocks.httpClient } + ): RPCClient { + val mockSerializationFactory: CordaAvroSerializationFactory = mock() + + whenever(mockSerializationFactory.createAvroSerializer(any())) + .thenReturn(mocks.serializer) + + whenever(mockSerializationFactory.createAvroDeserializer(any(), eq(Record::class.java))) + .thenReturn(mocks.deserializer) + + return RPCClient( + "TestRPCClient1", + mockSerializationFactory, + onSerializationError, + httpClientFactory + ) + } + + @BeforeEach + fun setup() { + val environment = MockEnvironment() + client = createClient(environment.mocks) + } + + @Test + fun `send() processes message and returns result`() { + val result = client.send(message) + assertNotNull(result?.payload) + assertEquals( + Record("topic", "key", "responsePayload"), + result!!.payload + ) + } + + @Test + fun `send() handles 4XX error`() { + val environment = MockEnvironment() + .withHttpStatus(404) + + val client = createClient(environment.mocks) + + assertThrows { + client.send(message) + } + } + + @Test + fun `send() handles 5XX error`() { + val environment = MockEnvironment() + .withHttpStatus(500) + + val client = createClient(environment.mocks) + + assertThrows { + client.send(message) + } + } + + @Test + fun `send() handles serialization error`() { + val onSerializationError: (ByteArray) -> Unit = mock() + + val environment = MockEnvironment().apply { + whenever(mockSerializer.serialize(any>())) + .thenThrow(IllegalArgumentException("Serialization error")) + } + + val client = createClient(environment.mocks, onSerializationError) + + assertThrows { + client.send(message) + } + + verify(onSerializationError).invoke(any()) + } + + @Test + fun `send() handles deserialization error`() { + val onSerializationError: (ByteArray) -> Unit = mock() + + val environment = MockEnvironment().apply { + whenever(mockSerializer.serialize(any>())) + .thenThrow(IllegalArgumentException("Deserialization error")) + } + + val client = createClient(environment.mocks, onSerializationError) + + assertThrows { + client.send(message) + } + + verify(onSerializationError).invoke(any()) + } + + @Test + fun `send retries on IOException and eventually succeeds`() { + val environment = MockEnvironment().apply { + whenever(mockHttpClient.send(any(), any>())) + .thenThrow(IOException("Simulated IO exception")) + .thenThrow(IOException("Simulated IO exception")) + .thenReturn(mockHttpResponse) + } + + val client = createClient(environment.mocks) + + val result = client.send(message) + assertEquals( + Record("topic", "key", "responsePayload"), + result!!.payload + ) + } + + @Test + fun `send fails after exhausting all retries`() { + val environment = MockEnvironment().apply { + whenever(mockHttpClient.send(any(), any>())) + .thenThrow(IOException("Simulated IO exception")) + } + + val client = createClient(environment.mocks) + + assertThrows { + client.send(message) + } + } + + @Test + fun `send retries the correct number of times before failing`() { + val environment = MockEnvironment().apply { + whenever(mockHttpClient.send(any(), any>())) + .thenThrow(IOException("Simulated IO exception")) + } + + val client = createClient(environment.mocks) + + assertThrows { + client.send(message) + } + + verify(environment.mockHttpClient, times(3)) + .send(any(), any>()) + } +} diff --git a/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/factory/MessagingClientFactoryFactoryTest.kt b/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/factory/MessagingClientFactoryFactoryTest.kt index e4a68753e9e..569c80cf319 100644 --- a/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/factory/MessagingClientFactoryFactoryTest.kt +++ b/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/factory/MessagingClientFactoryFactoryTest.kt @@ -1,5 +1,6 @@ package net.corda.messaging.mediator.factory +import net.corda.avro.serialization.CordaAvroSerializationFactory import net.corda.libs.configuration.SmartConfig import net.corda.messagebus.api.producer.builder.CordaProducerBuilder import org.junit.jupiter.api.Assertions @@ -10,12 +11,14 @@ import org.mockito.kotlin.mock class MessagingClientFactoryFactoryTest { private lateinit var messagingClientFactoryFactory: MessagingClientFactoryFactoryImpl private val cordaProducerBuilder = mock() + private val cordaAvroSerializationFactory = mock() private val messageBusConfig = mock() @BeforeEach fun beforeEach() { messagingClientFactoryFactory = MessagingClientFactoryFactoryImpl( cordaProducerBuilder, + cordaAvroSerializationFactory ) } @@ -28,4 +31,13 @@ class MessagingClientFactoryFactoryTest { Assertions.assertNotNull(messageBusClientFactory) } -} \ No newline at end of file + + @Test + fun testCreateRPCClientFactory() { + val rpcClientFactory = messagingClientFactoryFactory.createRPCClientFactory( + "rpcClient1" + ) + + Assertions.assertNotNull(rpcClientFactory) + } +} diff --git a/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/factory/RPCClientFactoryTest.kt b/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/factory/RPCClientFactoryTest.kt new file mode 100644 index 00000000000..2ae62f487e1 --- /dev/null +++ b/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/factory/RPCClientFactoryTest.kt @@ -0,0 +1,29 @@ +package net.corda.messaging.mediator.factory + +import net.corda.avro.serialization.CordaAvroSerializationFactory +import net.corda.messaging.api.mediator.config.MessagingClientConfig +import org.junit.jupiter.api.Assertions +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import org.mockito.Mockito.mock + +class RPCClientFactoryTest { + private lateinit var cordaSerializationFactory: CordaAvroSerializationFactory + private lateinit var rpcClientFactory: RPCClientFactory + + @BeforeEach + fun beforeEach() { + cordaSerializationFactory = mock(CordaAvroSerializationFactory::class.java) + rpcClientFactory = RPCClientFactory( + "RPCClient1", + mock(CordaAvroSerializationFactory::class.java) + ) + } + + @Test + fun testCreateRPCClient() { + val config = MessagingClientConfig {} + val rpcClient = rpcClientFactory.create(config) + Assertions.assertNotNull(rpcClient) + } +} diff --git a/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/utils/HTTPRetryExecutorTest.kt b/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/utils/HTTPRetryExecutorTest.kt new file mode 100644 index 00000000000..ddb302e6ade --- /dev/null +++ b/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/utils/HTTPRetryExecutorTest.kt @@ -0,0 +1,77 @@ +package net.corda.messaging.utils + +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.assertThrows + +class HTTPRetryExecutorTest { + private lateinit var retryConfig: HTTPRetryConfig + + @BeforeEach + fun setUp() { + retryConfig = HTTPRetryConfig.Builder() + .times(3) + .initialDelay(100) + .factor(2.0) + .retryOn(RuntimeException::class) + .build() + } + + @Test + fun `successfully returns after first attempt`() { + val result = HTTPRetryExecutor.withConfig(retryConfig) { + "Success" + } + + assertEquals("Success", result) + } + + @Suppress("TooGenericExceptionThrown") + @Test + fun `should retry until successful`() { + var attempt = 0 + + val result = HTTPRetryExecutor.withConfig(retryConfig) { + ++attempt + if (attempt < 3) { + throw RuntimeException("Failed on attempt $attempt") + } + "Success on attempt $attempt" + } + + assertEquals("Success on attempt 3", result) + } + + @Suppress("TooGenericExceptionThrown") + @Test + fun `should throw exception after max attempts`() { + var attempt = 0 + + assertThrows { + HTTPRetryExecutor.withConfig(retryConfig) { + ++attempt + throw RuntimeException("Failed on attempt $attempt") + } + } + } + + @Suppress("TooGenericExceptionThrown") + @Test + fun `should not retry on non-retryable exception`() { + val config = HTTPRetryConfig.Builder() + .times(3) + .initialDelay(100) + .factor(2.0) + .retryOn(SpecificException::class) + .build() + + assertThrows { + HTTPRetryExecutor.withConfig(config) { + throw RuntimeException("I'm not retryable!") + } + } + } + + internal class SpecificException(message: String) : Exception(message) +} \ No newline at end of file diff --git a/libs/messaging/messaging/src/main/kotlin/net/corda/messaging/api/exception/CordaRestAPIExceptions.kt b/libs/messaging/messaging/src/main/kotlin/net/corda/messaging/api/exception/CordaRestAPIExceptions.kt index b173417557e..a369ff75971 100644 --- a/libs/messaging/messaging/src/main/kotlin/net/corda/messaging/api/exception/CordaRestAPIExceptions.kt +++ b/libs/messaging/messaging/src/main/kotlin/net/corda/messaging/api/exception/CordaRestAPIExceptions.kt @@ -19,3 +19,15 @@ class CordaRPCAPIPartitionException(message: String?, exception: Throwable? = nu */ class CordaRPCAPIResponderException(val errorType: String, message: String?, exception: Throwable? = null) : CordaRuntimeException(message, exception) + +/** + * Exception representing a 4XX response from the HTTP server + */ +class CordaHTTPClientErrorException(val statusCode: Int, message: String?, exception: Throwable? = null) : + CordaRuntimeException(message, exception) + +/** + * Exception representing a 5XX response from the HTTP server + */ +class CordaHTTPServerErrorException(val statusCode: Int, message: String?, exception: Throwable? = null) : + CordaRuntimeException(message, exception) diff --git a/libs/messaging/messaging/src/main/kotlin/net/corda/messaging/api/mediator/factory/MessagingClientFactoryFactory.kt b/libs/messaging/messaging/src/main/kotlin/net/corda/messaging/api/mediator/factory/MessagingClientFactoryFactory.kt index 9e69b6e31ea..3c4c7951847 100644 --- a/libs/messaging/messaging/src/main/kotlin/net/corda/messaging/api/mediator/factory/MessagingClientFactoryFactory.kt +++ b/libs/messaging/messaging/src/main/kotlin/net/corda/messaging/api/mediator/factory/MessagingClientFactoryFactory.kt @@ -16,4 +16,13 @@ interface MessagingClientFactoryFactory { id: String, messageBusConfig: SmartConfig, ) : MessagingClientFactory -} \ No newline at end of file + + /** + * Creates an RPC messaging client factory. + * + * @param id RPC client ID. + */ + fun createRPCClientFactory( + id: String + ) : MessagingClientFactory +} From b3b40d3cbbd471a74b2ee786c8a3603fd0d78048 Mon Sep 17 00:00:00 2001 From: Dan Newton Date: Fri, 13 Oct 2023 17:06:27 +0100 Subject: [PATCH 22/81] CORE-15552 Add `findUnconsumedStatesByExactType` paging API (#4837) Add `findUnconsumedStatesByExactType` to `UtxoLedgerService`, allowing developers to retrieve states by type from the vault in pages. This dissuades developers from retrieving every state from the vault in one go and returning them to a flow. Reuse the vault-named query logic to create a `findUnconsumedStatesByExactType` API that supports paging without too much extra work. Remove the non-paged version of `findUnconsumedStatesByExactType` as well. --- .../impl/VaultNamedQueryFactoryProvider.kt | 17 +++-- .../utxo/UtxoPersistenceService.kt | 2 - .../ledger/persistence/utxo/UtxoRepository.kt | 6 -- .../utxo/impl/AbstractUtxoQueryProvider.kt | 26 -------- .../utxo/impl/UtxoPersistenceServiceImpl.kt | 8 --- .../utxo/impl/UtxoQueryProvider.kt | 5 -- .../utxo/impl/UtxoRepositoryImpl.kt | 7 --- .../impl/UtxoRequestHandlerSelectorImpl.kt | 11 ---- ...consumedStatesByExactTypeRequestHandler.kt | 34 ---------- .../utxo/flow/impl/UtxoLedgerServiceImpl.kt | 16 ++++- .../LedgerPersistenceMetricOperationName.kt | 1 - .../UtxoLedgerStateQueryService.kt | 11 ---- .../UtxoLedgerStateQueryServiceImpl.kt | 14 ----- ...edStatesByExactTypeExternalEventFactory.kt | 54 ---------------- ...atesByExactTypeExternalEventFactoryTest.kt | 62 ------------------- gradle.properties | 2 +- 16 files changed, 28 insertions(+), 248 deletions(-) delete mode 100644 components/ledger/ledger-persistence/src/main/kotlin/net/corda/ledger/persistence/utxo/impl/request/handlers/UtxoFindUnconsumedStatesByExactTypeRequestHandler.kt delete mode 100644 components/ledger/ledger-utxo-flow/src/main/kotlin/net/corda/ledger/utxo/flow/impl/persistence/external/events/FindUnconsumedStatesByExactTypeExternalEventFactory.kt delete mode 100644 components/ledger/ledger-utxo-flow/src/test/kotlin/net/corda/ledger/utxo/flow/impl/persistence/external/events/FindUnconsumedStatesByExactTypeExternalEventFactoryTest.kt diff --git a/components/ledger/ledger-persistence/src/main/kotlin/net/corda/ledger/persistence/query/registration/impl/VaultNamedQueryFactoryProvider.kt b/components/ledger/ledger-persistence/src/main/kotlin/net/corda/ledger/persistence/query/registration/impl/VaultNamedQueryFactoryProvider.kt index 9ce4e6ff4e6..b66e5e5b056 100644 --- a/components/ledger/ledger-persistence/src/main/kotlin/net/corda/ledger/persistence/query/registration/impl/VaultNamedQueryFactoryProvider.kt +++ b/components/ledger/ledger-persistence/src/main/kotlin/net/corda/ledger/persistence/query/registration/impl/VaultNamedQueryFactoryProvider.kt @@ -5,6 +5,7 @@ import net.corda.sandbox.type.UsedByPersistence import net.corda.sandboxgroupcontext.CustomMetadataConsumer import net.corda.sandboxgroupcontext.MutableSandboxGroupContext import net.corda.sandboxgroupcontext.getMetadataServices +import net.corda.utilities.debug import net.corda.v5.base.annotations.Suspendable import net.corda.v5.ledger.utxo.query.VaultNamedQueryFactory import net.corda.v5.ledger.utxo.query.registration.VaultNamedQueryBuilderFactory @@ -17,7 +18,7 @@ import org.slf4j.LoggerFactory @Suppress("unused") @Component( - service = [ UsedByPersistence::class ], + service = [UsedByPersistence::class], property = [SandboxConstants.CORDA_MARKER_ONLY_SERVICE], scope = ServiceScope.PROTOTYPE ) @@ -27,19 +28,27 @@ class VaultNamedQueryFactoryProvider @Activate constructor( ) : UsedByPersistence, CustomMetadataConsumer { private companion object { + const val FIND_UNCONSUMED_STATES_BY_EXACT_TYPE = "CORDA_FIND_UNCONSUMED_STATES_BY_EXACT_TYPE" val logger: Logger = LoggerFactory.getLogger(VaultNamedQueryFactoryProvider::class.java) } @Suspendable override fun accept(context: MutableSandboxGroupContext) { + registerPlatformQueries(vaultNamedQueryBuilderFactory) + val metadataServices = context.getMetadataServices() - if (logger.isDebugEnabled) { - logger.debug("Number of vault named queries found: ${metadataServices.size}") - } + logger.debug { "Number of vault named queries found: ${metadataServices.size}" } metadataServices.forEach { it.create(vaultNamedQueryBuilderFactory) } } + + private fun registerPlatformQueries(vaultNamedQueryBuilderFactory: VaultNamedQueryBuilderFactory) { + vaultNamedQueryBuilderFactory + .create(FIND_UNCONSUMED_STATES_BY_EXACT_TYPE) + .whereJson("WHERE visible_states.type = :type") + .register() + } } diff --git a/components/ledger/ledger-persistence/src/main/kotlin/net/corda/ledger/persistence/utxo/UtxoPersistenceService.kt b/components/ledger/ledger-persistence/src/main/kotlin/net/corda/ledger/persistence/utxo/UtxoPersistenceService.kt index 9e5c1b7bfad..3acafd1919d 100644 --- a/components/ledger/ledger-persistence/src/main/kotlin/net/corda/ledger/persistence/utxo/UtxoPersistenceService.kt +++ b/components/ledger/ledger-persistence/src/main/kotlin/net/corda/ledger/persistence/utxo/UtxoPersistenceService.kt @@ -52,8 +52,6 @@ interface UtxoPersistenceService { fun findUnconsumedVisibleStatesByType(stateClass: Class): List - fun findUnconsumedVisibleStatesByExactType(stateClass: Class): List - fun resolveStateRefs(stateRefs: List): List fun persistTransaction( diff --git a/components/ledger/ledger-persistence/src/main/kotlin/net/corda/ledger/persistence/utxo/UtxoRepository.kt b/components/ledger/ledger-persistence/src/main/kotlin/net/corda/ledger/persistence/utxo/UtxoRepository.kt index ea292006331..3132115e9c3 100644 --- a/components/ledger/ledger-persistence/src/main/kotlin/net/corda/ledger/persistence/utxo/UtxoRepository.kt +++ b/components/ledger/ledger-persistence/src/main/kotlin/net/corda/ledger/persistence/utxo/UtxoRepository.kt @@ -37,12 +37,6 @@ interface UtxoRepository { entityManager: EntityManager ): List - /** Retrieves transaction component leaves related to visible unspent states */ - fun findUnconsumedVisibleStatesByExactType( - entityManager: EntityManager, - stateClassType: String - ): List - /** Retrieves transaction component leafs related to specific StateRefs */ fun resolveStateRefs( entityManager: EntityManager, diff --git a/components/ledger/ledger-persistence/src/main/kotlin/net/corda/ledger/persistence/utxo/impl/AbstractUtxoQueryProvider.kt b/components/ledger/ledger-persistence/src/main/kotlin/net/corda/ledger/persistence/utxo/impl/AbstractUtxoQueryProvider.kt index 1f91c967eac..043cc9cb898 100644 --- a/components/ledger/ledger-persistence/src/main/kotlin/net/corda/ledger/persistence/utxo/impl/AbstractUtxoQueryProvider.kt +++ b/components/ledger/ledger-persistence/src/main/kotlin/net/corda/ledger/persistence/utxo/impl/AbstractUtxoQueryProvider.kt @@ -49,32 +49,6 @@ abstract class AbstractUtxoQueryProvider : UtxoQueryProvider { ORDER BY tx.created, tc_output.transaction_id, tc_output.leaf_idx""" .trimIndent() - override val findUnconsumedVisibleStatesByExactType: String - get() = """ - SELECT tc_output.transaction_id, - tc_output.leaf_idx, - tc_output_info.data as output_info_data, - tc_output.data AS output_data - FROM {h-schema}utxo_visible_transaction_output AS vto - JOIN {h-schema}utxo_transaction_component AS tc_output_info - ON tc_output_info.transaction_id = vto.transaction_id - AND tc_output_info.leaf_idx = vto.leaf_idx - AND tc_output_info.group_idx = ${UtxoComponentGroup.OUTPUTS_INFO.ordinal} - JOIN {h-schema}utxo_transaction_component AS tc_output - ON tc_output.transaction_id = tc_output_info.transaction_id - AND tc_output.leaf_idx = tc_output_info.leaf_idx - AND tc_output.group_idx = ${UtxoComponentGroup.OUTPUTS.ordinal} - JOIN {h-schema}utxo_transaction_output AS tx_o - ON tx_o.transaction_id = tc_output.transaction_id - AND tx_o.leaf_idx = tc_output.leaf_idx - JOIN {h-schema}utxo_transaction AS tx - ON tx.transaction_id = tx_o.transaction_id - WHERE tx_o.type = :type - AND vto.consumed IS NULL - AND tx.status = :verified - ORDER BY tx.created, tc_output.transaction_id, tc_output.leaf_idx""" - .trimIndent() - override val findTransactionSignatures: String get() = """ SELECT signature diff --git a/components/ledger/ledger-persistence/src/main/kotlin/net/corda/ledger/persistence/utxo/impl/UtxoPersistenceServiceImpl.kt b/components/ledger/ledger-persistence/src/main/kotlin/net/corda/ledger/persistence/utxo/impl/UtxoPersistenceServiceImpl.kt index e9623b7da30..740c840ea58 100644 --- a/components/ledger/ledger-persistence/src/main/kotlin/net/corda/ledger/persistence/utxo/impl/UtxoPersistenceServiceImpl.kt +++ b/components/ledger/ledger-persistence/src/main/kotlin/net/corda/ledger/persistence/utxo/impl/UtxoPersistenceServiceImpl.kt @@ -117,14 +117,6 @@ class UtxoPersistenceServiceImpl( } } - override fun findUnconsumedVisibleStatesByExactType( - stateClass: Class - ): List { - return entityManagerFactory.transaction { em -> - repository.findUnconsumedVisibleStatesByExactType(em, stateClass.canonicalName) - } - } - override fun resolveStateRefs(stateRefs: List): List { return entityManagerFactory.transaction { em -> repository.resolveStateRefs(em, stateRefs) diff --git a/components/ledger/ledger-persistence/src/main/kotlin/net/corda/ledger/persistence/utxo/impl/UtxoQueryProvider.kt b/components/ledger/ledger-persistence/src/main/kotlin/net/corda/ledger/persistence/utxo/impl/UtxoQueryProvider.kt index 467e6663da2..ea943c5d0be 100644 --- a/components/ledger/ledger-persistence/src/main/kotlin/net/corda/ledger/persistence/utxo/impl/UtxoQueryProvider.kt +++ b/components/ledger/ledger-persistence/src/main/kotlin/net/corda/ledger/persistence/utxo/impl/UtxoQueryProvider.kt @@ -25,11 +25,6 @@ interface UtxoQueryProvider { */ val findUnconsumedVisibleStatesByType: String - /** - * @property findUnconsumedVisibleStatesByExactType SQL text for [UtxoRepositoryImpl.findUnconsumedVisibleStatesByExactType]. - */ - val findUnconsumedVisibleStatesByExactType: String - /** * @property findTransactionSignatures SQL text for [UtxoRepositoryImpl.findTransactionSignatures]. */ diff --git a/components/ledger/ledger-persistence/src/main/kotlin/net/corda/ledger/persistence/utxo/impl/UtxoRepositoryImpl.kt b/components/ledger/ledger-persistence/src/main/kotlin/net/corda/ledger/persistence/utxo/impl/UtxoRepositoryImpl.kt index c565616e6a2..e3a3516127c 100644 --- a/components/ledger/ledger-persistence/src/main/kotlin/net/corda/ledger/persistence/utxo/impl/UtxoRepositoryImpl.kt +++ b/components/ledger/ledger-persistence/src/main/kotlin/net/corda/ledger/persistence/utxo/impl/UtxoRepositoryImpl.kt @@ -129,13 +129,6 @@ class UtxoRepositoryImpl @Activate constructor( return findUnconsumedVisibleStates(entityManager, queryProvider.findUnconsumedVisibleStatesByType, null) } - override fun findUnconsumedVisibleStatesByExactType( - entityManager: EntityManager, - stateClassType: String - ): List { - return findUnconsumedVisibleStates(entityManager, queryProvider.findUnconsumedVisibleStatesByExactType, stateClassType) - } - override fun resolveStateRefs( entityManager: EntityManager, stateRefs: List diff --git a/components/ledger/ledger-persistence/src/main/kotlin/net/corda/ledger/persistence/utxo/impl/UtxoRequestHandlerSelectorImpl.kt b/components/ledger/ledger-persistence/src/main/kotlin/net/corda/ledger/persistence/utxo/impl/UtxoRequestHandlerSelectorImpl.kt index 72cb8c4837c..3a17bb02a1f 100644 --- a/components/ledger/ledger-persistence/src/main/kotlin/net/corda/ledger/persistence/utxo/impl/UtxoRequestHandlerSelectorImpl.kt +++ b/components/ledger/ledger-persistence/src/main/kotlin/net/corda/ledger/persistence/utxo/impl/UtxoRequestHandlerSelectorImpl.kt @@ -3,7 +3,6 @@ package net.corda.ledger.persistence.utxo.impl import net.corda.data.ledger.persistence.FindSignedGroupParameters import net.corda.data.ledger.persistence.FindSignedLedgerTransaction import net.corda.data.ledger.persistence.FindTransaction -import net.corda.data.ledger.persistence.FindUnconsumedStatesByExactType import net.corda.data.ledger.persistence.FindTransactionIdsAndStatuses import net.corda.data.ledger.persistence.FindUnconsumedStatesByType import net.corda.data.ledger.persistence.LedgerPersistenceRequest @@ -25,7 +24,6 @@ import net.corda.ledger.persistence.utxo.impl.request.handlers.UtxoFindTransacti import net.corda.ledger.persistence.utxo.impl.request.handlers.UtxoFindSignedGroupParametersRequestHandler import net.corda.ledger.persistence.utxo.impl.request.handlers.UtxoFindSignedLedgerTransactionRequestHandler import net.corda.ledger.persistence.utxo.impl.request.handlers.UtxoFindTransactionRequestHandler -import net.corda.ledger.persistence.utxo.impl.request.handlers.UtxoFindUnconsumedStatesByExactTypeRequestHandler import net.corda.ledger.persistence.utxo.impl.request.handlers.UtxoFindUnconsumedStatesByTypeRequestHandler import net.corda.ledger.persistence.utxo.impl.request.handlers.UtxoPersistSignedGroupParametersIfDoNotExistRequestHandler import net.corda.ledger.persistence.utxo.impl.request.handlers.UtxoPersistTransactionIfDoesNotExistRequestHandler @@ -100,15 +98,6 @@ class UtxoRequestHandlerSelectorImpl @Activate constructor( outputRecordFactory ) } - is FindUnconsumedStatesByExactType -> { - UtxoFindUnconsumedStatesByExactTypeRequestHandler( - req, - sandbox, - externalEventContext, - persistenceService, - outputRecordFactory - ) - } is ResolveStateRefs -> { UtxoResolveStateRefsRequestHandler( req, diff --git a/components/ledger/ledger-persistence/src/main/kotlin/net/corda/ledger/persistence/utxo/impl/request/handlers/UtxoFindUnconsumedStatesByExactTypeRequestHandler.kt b/components/ledger/ledger-persistence/src/main/kotlin/net/corda/ledger/persistence/utxo/impl/request/handlers/UtxoFindUnconsumedStatesByExactTypeRequestHandler.kt deleted file mode 100644 index c0df0902855..00000000000 --- a/components/ledger/ledger-persistence/src/main/kotlin/net/corda/ledger/persistence/utxo/impl/request/handlers/UtxoFindUnconsumedStatesByExactTypeRequestHandler.kt +++ /dev/null @@ -1,34 +0,0 @@ -package net.corda.ledger.persistence.utxo.impl.request.handlers - -import net.corda.data.flow.event.external.ExternalEventContext -import net.corda.data.ledger.persistence.FindUnconsumedStatesByExactType -import net.corda.ledger.persistence.common.RequestHandler -import net.corda.ledger.persistence.utxo.UtxoOutputRecordFactory -import net.corda.ledger.persistence.utxo.UtxoPersistenceService -import net.corda.messaging.api.records.Record -import net.corda.sandboxgroupcontext.SandboxGroupContext -import net.corda.v5.ledger.utxo.ContractState - -@Suppress("LongParameterList") -class UtxoFindUnconsumedStatesByExactTypeRequestHandler( - private val findUnconsumedStatesByExactType: FindUnconsumedStatesByExactType, - private val sandbox: SandboxGroupContext, - private val externalEventContext: ExternalEventContext, - private val persistenceService: UtxoPersistenceService, - private val utxoOutputRecordFactory: UtxoOutputRecordFactory -) : RequestHandler { - - @Suppress("UNCHECKED_CAST") - override fun execute(): List> { - val stateType = sandbox.sandboxGroup.loadClassFromMainBundles(findUnconsumedStatesByExactType.stateClassName) - require(ContractState::class.java.isAssignableFrom(stateType)) { - "Provided ${findUnconsumedStatesByExactType.stateClassName} is not type of ContractState" - } - - val visibleStates = persistenceService.findUnconsumedVisibleStatesByExactType( - stateType as Class - ) - - return listOf(utxoOutputRecordFactory.getStatesSuccessRecord(visibleStates, externalEventContext)) - } -} diff --git a/components/ledger/ledger-utxo-flow/src/main/kotlin/net/corda/ledger/utxo/flow/impl/UtxoLedgerServiceImpl.kt b/components/ledger/ledger-utxo-flow/src/main/kotlin/net/corda/ledger/utxo/flow/impl/UtxoLedgerServiceImpl.kt index eabc13477c0..00c9c2103d0 100644 --- a/components/ledger/ledger-utxo-flow/src/main/kotlin/net/corda/ledger/utxo/flow/impl/UtxoLedgerServiceImpl.kt +++ b/components/ledger/ledger-utxo-flow/src/main/kotlin/net/corda/ledger/utxo/flow/impl/UtxoLedgerServiceImpl.kt @@ -24,6 +24,7 @@ import net.corda.sandboxgroupcontext.getObjectByKey import net.corda.utilities.time.UTCClock import net.corda.v5.application.flows.FlowEngine import net.corda.v5.application.messaging.FlowSession +import net.corda.v5.application.persistence.PagedQuery import net.corda.v5.base.annotations.Suspendable import net.corda.v5.base.annotations.VisibleForTesting import net.corda.v5.base.exceptions.CordaRuntimeException @@ -49,6 +50,7 @@ import org.osgi.service.component.annotations.Reference import org.osgi.service.component.annotations.ServiceScope.PROTOTYPE import java.security.PrivilegedActionException import java.security.PrivilegedExceptionAction +import java.time.Instant @Suppress("LongParameterList", "TooManyFunctions") @Component(service = [UtxoLedgerService::class, UsedByFlow::class], scope = PROTOTYPE) @@ -65,6 +67,7 @@ class UtxoLedgerServiceImpl @Activate constructor( ) : UtxoLedgerService, UsedByFlow, SingletonSerializeAsToken { private companion object { + const val FIND_UNCONSUMED_STATES_BY_EXACT_TYPE = "CORDA_FIND_UNCONSUMED_STATES_BY_EXACT_TYPE" val clock = UTCClock() } @@ -106,8 +109,17 @@ class UtxoLedgerServiceImpl @Activate constructor( } @Suspendable - override fun findUnconsumedStatesByExactType(type: Class): List> { - return utxoLedgerStateQueryService.findUnconsumedStatesByExactType(type) + override fun findUnconsumedStatesByExactType( + type: Class, + limit: Int, + createdTimestampLimit: Instant + ): PagedQuery.ResultSet> { + @Suppress("UNCHECKED_CAST") + return query(FIND_UNCONSUMED_STATES_BY_EXACT_TYPE, StateAndRef::class.java) + .setParameter("type", type.name) + .setCreatedTimestampLimit(createdTimestampLimit) + .setLimit(limit) + .execute() as PagedQuery.ResultSet> } @Suspendable diff --git a/components/ledger/ledger-utxo-flow/src/main/kotlin/net/corda/ledger/utxo/flow/impl/persistence/LedgerPersistenceMetricOperationName.kt b/components/ledger/ledger-utxo-flow/src/main/kotlin/net/corda/ledger/utxo/flow/impl/persistence/LedgerPersistenceMetricOperationName.kt index 29382a7e9e9..175bd056d40 100644 --- a/components/ledger/ledger-utxo-flow/src/main/kotlin/net/corda/ledger/utxo/flow/impl/persistence/LedgerPersistenceMetricOperationName.kt +++ b/components/ledger/ledger-utxo-flow/src/main/kotlin/net/corda/ledger/utxo/flow/impl/persistence/LedgerPersistenceMetricOperationName.kt @@ -6,7 +6,6 @@ enum class LedgerPersistenceMetricOperationName { FindSignedLedgerTransactionWithStatus, FindTransactionIdsAndStatuses, FindTransactionWithStatus, - FindUnconsumedStatesByExactType, FindUnconsumedStatesByType, FindWithNamedQuery, PersistSignedGroupParametersIfDoNotExist, diff --git a/components/ledger/ledger-utxo-flow/src/main/kotlin/net/corda/ledger/utxo/flow/impl/persistence/UtxoLedgerStateQueryService.kt b/components/ledger/ledger-utxo-flow/src/main/kotlin/net/corda/ledger/utxo/flow/impl/persistence/UtxoLedgerStateQueryService.kt index 2cf004582dc..cfdb213be90 100644 --- a/components/ledger/ledger-utxo-flow/src/main/kotlin/net/corda/ledger/utxo/flow/impl/persistence/UtxoLedgerStateQueryService.kt +++ b/components/ledger/ledger-utxo-flow/src/main/kotlin/net/corda/ledger/utxo/flow/impl/persistence/UtxoLedgerStateQueryService.kt @@ -21,17 +21,6 @@ interface UtxoLedgerStateQueryService { @Suspendable fun findUnconsumedStatesByType(stateClass: Class): List> - /** - * Find unconsumed visible states of type [stateClass]. - * - * @param stateClass The class of the aimed states. - * @return The result [StateAndRef]s. - * - * @throws CordaPersistenceException if an error happens during find operation. - */ - @Suspendable - fun findUnconsumedStatesByExactType(stateClass: Class): List> - /** * Resolve [StateRef]s to [StateAndRef]s * diff --git a/components/ledger/ledger-utxo-flow/src/main/kotlin/net/corda/ledger/utxo/flow/impl/persistence/UtxoLedgerStateQueryServiceImpl.kt b/components/ledger/ledger-utxo-flow/src/main/kotlin/net/corda/ledger/utxo/flow/impl/persistence/UtxoLedgerStateQueryServiceImpl.kt index b04c00935d9..8b84c171a33 100644 --- a/components/ledger/ledger-utxo-flow/src/main/kotlin/net/corda/ledger/utxo/flow/impl/persistence/UtxoLedgerStateQueryServiceImpl.kt +++ b/components/ledger/ledger-utxo-flow/src/main/kotlin/net/corda/ledger/utxo/flow/impl/persistence/UtxoLedgerStateQueryServiceImpl.kt @@ -4,10 +4,8 @@ import io.micrometer.core.instrument.Timer import net.corda.flow.external.events.executor.ExternalEventExecutor import net.corda.flow.fiber.metrics.recordSuspendable import net.corda.ledger.utxo.flow.impl.cache.StateAndRefCache -import net.corda.ledger.utxo.flow.impl.persistence.LedgerPersistenceMetricOperationName.FindUnconsumedStatesByExactType import net.corda.ledger.utxo.flow.impl.persistence.LedgerPersistenceMetricOperationName.FindUnconsumedStatesByType import net.corda.ledger.utxo.flow.impl.persistence.LedgerPersistenceMetricOperationName.ResolveStateRefs -import net.corda.ledger.utxo.flow.impl.persistence.external.events.FindUnconsumedStatesByExactTypeExternalEventFactory import net.corda.ledger.utxo.flow.impl.persistence.external.events.FindUnconsumedStatesByTypeExternalEventFactory import net.corda.ledger.utxo.flow.impl.persistence.external.events.FindUnconsumedStatesByTypeParameters import net.corda.ledger.utxo.flow.impl.persistence.external.events.ResolveStateRefsExternalEventFactory @@ -54,18 +52,6 @@ class UtxoLedgerStateQueryServiceImpl @Activate constructor( } } - @Suspendable - override fun findUnconsumedStatesByExactType(stateClass: Class): List> { - return recordSuspendable({ ledgerPersistenceFlowTimer(FindUnconsumedStatesByExactType) }) @Suspendable { - wrapWithPersistenceException { - externalEventExecutor.execute( - FindUnconsumedStatesByExactTypeExternalEventFactory::class.java, - FindUnconsumedStatesByTypeParameters(stateClass) - ) - }.map { it.toStateAndRef(serializationService) } - } - } - @Suspendable override fun resolveStateRefs(stateRefs: Iterable): List> { return recordSuspendable({ ledgerPersistenceFlowTimer(ResolveStateRefs) }) @Suspendable { diff --git a/components/ledger/ledger-utxo-flow/src/main/kotlin/net/corda/ledger/utxo/flow/impl/persistence/external/events/FindUnconsumedStatesByExactTypeExternalEventFactory.kt b/components/ledger/ledger-utxo-flow/src/main/kotlin/net/corda/ledger/utxo/flow/impl/persistence/external/events/FindUnconsumedStatesByExactTypeExternalEventFactory.kt deleted file mode 100644 index 526c6ef231e..00000000000 --- a/components/ledger/ledger-utxo-flow/src/main/kotlin/net/corda/ledger/utxo/flow/impl/persistence/external/events/FindUnconsumedStatesByExactTypeExternalEventFactory.kt +++ /dev/null @@ -1,54 +0,0 @@ -package net.corda.ledger.utxo.flow.impl.persistence.external.events - -import net.corda.data.flow.event.external.ExternalEventContext -import net.corda.data.ledger.persistence.FindUnconsumedStatesByExactType -import net.corda.data.ledger.persistence.LedgerPersistenceRequest -import net.corda.data.ledger.persistence.LedgerTypes -import net.corda.data.ledger.persistence.UtxoTransactionOutputs -import net.corda.flow.external.events.factory.ExternalEventFactory -import net.corda.flow.external.events.factory.ExternalEventRecord -import net.corda.flow.state.FlowCheckpoint -import net.corda.ledger.utxo.data.transaction.UtxoVisibleTransactionOutputDto -import net.corda.schema.Schemas -import net.corda.virtualnode.toAvro -import org.osgi.service.component.annotations.Activate -import org.osgi.service.component.annotations.Component -import java.time.Clock - -@Component(service = [ExternalEventFactory::class]) -class FindUnconsumedStatesByExactTypeExternalEventFactory( - private val clock: Clock -) : ExternalEventFactory> -{ - @Activate - constructor() : this(Clock.systemUTC()) - - override val responseType = UtxoTransactionOutputs::class.java - - override fun createExternalEvent( - checkpoint: FlowCheckpoint, - flowExternalEventContext: ExternalEventContext, - parameters: FindUnconsumedStatesByTypeParameters - ): ExternalEventRecord { - return ExternalEventRecord( - topic = Schemas.Persistence.PERSISTENCE_LEDGER_PROCESSOR_TOPIC, - payload = LedgerPersistenceRequest.newBuilder() - .setTimestamp(clock.instant()) - .setHoldingIdentity(checkpoint.holdingIdentity.toAvro()) - .setRequest(createRequest(parameters)) - .setFlowExternalEventContext(flowExternalEventContext) - .setLedgerType(LedgerTypes.UTXO) - .build() - ) - } - - private fun createRequest(parameters: FindUnconsumedStatesByTypeParameters): Any { - return FindUnconsumedStatesByExactType(parameters.stateClass.canonicalName) - } - - override fun resumeWith(checkpoint: FlowCheckpoint, response: UtxoTransactionOutputs): List { - return response.transactionOutputs.map { - UtxoVisibleTransactionOutputDto(it.transactionId, it.index, it.info.array(), it.data.array()) - } - } -} \ No newline at end of file diff --git a/components/ledger/ledger-utxo-flow/src/test/kotlin/net/corda/ledger/utxo/flow/impl/persistence/external/events/FindUnconsumedStatesByExactTypeExternalEventFactoryTest.kt b/components/ledger/ledger-utxo-flow/src/test/kotlin/net/corda/ledger/utxo/flow/impl/persistence/external/events/FindUnconsumedStatesByExactTypeExternalEventFactoryTest.kt deleted file mode 100644 index cbc8bf10d85..00000000000 --- a/components/ledger/ledger-utxo-flow/src/test/kotlin/net/corda/ledger/utxo/flow/impl/persistence/external/events/FindUnconsumedStatesByExactTypeExternalEventFactoryTest.kt +++ /dev/null @@ -1,62 +0,0 @@ -package net.corda.ledger.utxo.flow.impl.persistence.external.events - -import net.corda.data.KeyValuePairList -import net.corda.data.flow.event.external.ExternalEventContext -import net.corda.data.ledger.persistence.FindUnconsumedStatesByExactType -import net.corda.data.ledger.persistence.LedgerPersistenceRequest -import net.corda.data.ledger.persistence.LedgerTypes -import net.corda.flow.state.FlowCheckpoint -import net.corda.schema.Schemas -import net.corda.v5.ledger.utxo.ContractState -import net.corda.virtualnode.toCorda -import org.junit.jupiter.api.Assertions.assertEquals -import org.junit.jupiter.api.Assertions.assertNull -import org.junit.jupiter.api.Test -import org.mockito.kotlin.mock -import org.mockito.kotlin.whenever -import java.security.PublicKey -import java.time.Clock -import java.time.Instant -import java.time.ZoneId - -class FindUnconsumedStatesByExactTypeExternalEventFactoryTest { - - class TestContractState : ContractState { - override fun getParticipants(): List { - return emptyList() - } - } - - @Test - fun `creates a record containing an UtxoLedgerRequest with a FindUnconsumedStatesByExactType payload`() { - val checkpoint = mock() - val stateClass = TestContractState()::class.java - val externalEventContext = ExternalEventContext( - "request id", - "flow id", - KeyValuePairList(emptyList()) - ) - val testClock = Clock.fixed(Instant.now(), ZoneId.of("UTC")) - - whenever(checkpoint.holdingIdentity).thenReturn(ALICE_X500_HOLDING_IDENTITY.toCorda()) - - val externalEventRecord = FindUnconsumedStatesByExactTypeExternalEventFactory(testClock).createExternalEvent( - checkpoint, - externalEventContext, - FindUnconsumedStatesByTypeParameters(stateClass) - ) - - assertEquals(Schemas.Persistence.PERSISTENCE_LEDGER_PROCESSOR_TOPIC, externalEventRecord.topic) - assertNull(externalEventRecord.key) - assertEquals( - LedgerPersistenceRequest( - testClock.instant(), - ALICE_X500_HOLDING_IDENTITY, - LedgerTypes.UTXO, - FindUnconsumedStatesByExactType(stateClass.canonicalName), - externalEventContext - ), - externalEventRecord.payload - ) - } -} diff --git a/gradle.properties b/gradle.properties index 87d66d36fb6..5afb4782e13 100644 --- a/gradle.properties +++ b/gradle.properties @@ -46,7 +46,7 @@ commonsTextVersion = 1.10.0 bouncycastleVersion=1.76 # Corda API libs revision (change in 4th digit indicates a breaking change) # Change to 5.1.0.xx-SNAPSHOT to pick up maven local published copy -cordaApiVersion=5.1.0.35-beta+ +cordaApiVersion=5.1.0.36-beta+ disruptorVersion=3.4.4 felixConfigAdminVersion=1.9.26 From 42777ec4249e1894752dd0e55fd89e65e1178fb7 Mon Sep 17 00:00:00 2001 From: Ben Millar <44114751+ben-millar@users.noreply.github.com> Date: Fri, 13 Oct 2023 17:54:03 +0100 Subject: [PATCH 23/81] Revert "CORE-16181 Implement RPC Producer that implements RPC client (#4709)" (#4868) This reverts commit d5dc203c08b96aed6462a8bf891c2a93bf9c276d. --- charts/corda-lib/templates/_worker.tpl | 2 +- .../net/corda/messaging/mediator/RPCClient.kt | 130 ----------- .../MessagingClientFactoryFactoryImpl.kt | 12 +- .../mediator/factory/RPCClientFactory.kt | 21 -- .../corda/messaging/utils/HTTPRetryConfig.kt | 37 --- .../messaging/utils/HTTPRetryExecutor.kt | 42 ---- .../corda/messaging/mediator/RPCClientTest.kt | 219 ------------------ .../MessagingClientFactoryFactoryTest.kt | 14 +- .../mediator/factory/RPCClientFactoryTest.kt | 29 --- .../messaging/utils/HTTPRetryExecutorTest.kt | 77 ------ .../api/exception/CordaRestAPIExceptions.kt | 12 - .../factory/MessagingClientFactoryFactory.kt | 11 +- 12 files changed, 4 insertions(+), 602 deletions(-) delete mode 100644 libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/RPCClient.kt delete mode 100644 libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/factory/RPCClientFactory.kt delete mode 100644 libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/utils/HTTPRetryConfig.kt delete mode 100644 libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/utils/HTTPRetryExecutor.kt delete mode 100644 libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/RPCClientTest.kt delete mode 100644 libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/factory/RPCClientFactoryTest.kt delete mode 100644 libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/utils/HTTPRetryExecutorTest.kt diff --git a/charts/corda-lib/templates/_worker.tpl b/charts/corda-lib/templates/_worker.tpl index 97fe731c992..e9f348f9943 100644 --- a/charts/corda-lib/templates/_worker.tpl +++ b/charts/corda-lib/templates/_worker.tpl @@ -84,7 +84,7 @@ metadata: spec: type: ClusterIP selector: - app.kubernetes.io/component: {{ include "corda.workerComponent" $worker }} + app: {{ $workerName }} ports: - protocol: TCP port: {{ include "corda.workerServicePort" . }} diff --git a/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/RPCClient.kt b/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/RPCClient.kt deleted file mode 100644 index 23bcdd52912..00000000000 --- a/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/RPCClient.kt +++ /dev/null @@ -1,130 +0,0 @@ -package net.corda.messaging.mediator - -import java.io.IOException -import java.net.URI -import java.net.http.HttpClient -import java.net.http.HttpRequest -import java.net.http.HttpResponse -import java.util.concurrent.TimeoutException -import net.corda.avro.serialization.CordaAvroSerializationFactory -import net.corda.messaging.api.exception.CordaHTTPClientErrorException -import net.corda.messaging.api.exception.CordaHTTPServerErrorException -import net.corda.messaging.api.mediator.MediatorMessage -import net.corda.messaging.api.mediator.MessagingClient -import net.corda.messaging.api.mediator.MessagingClient.Companion.MSG_PROP_ENDPOINT -import net.corda.messaging.api.records.Record -import net.corda.messaging.utils.HTTPRetryConfig -import net.corda.messaging.utils.HTTPRetryExecutor -import org.slf4j.Logger -import org.slf4j.LoggerFactory - -class RPCClient( - override val id: String, - cordaAvroSerializerFactory: CordaAvroSerializationFactory, - private val onSerializationError: ((ByteArray) -> Unit)?, - httpClientFactory: () -> HttpClient = { HttpClient.newBuilder().build() }, - private val retryConfig: HTTPRetryConfig = - HTTPRetryConfig.Builder() - .retryOn(IOException::class, TimeoutException::class) - .build() -) : MessagingClient { - private val httpClient: HttpClient = httpClientFactory() - private val serializer = cordaAvroSerializerFactory.createAvroSerializer {} - private val deserializer = cordaAvroSerializerFactory.createAvroDeserializer({}, Record::class.java) - - - private companion object { - private val log: Logger = LoggerFactory.getLogger(this::class.java.enclosingClass) - } - - override fun send(message: MediatorMessage<*>): MediatorMessage<*>? { - return try { - processMessage(message) - } catch (e: Exception) { - handleExceptions(e) - null - } - } - - private fun processMessage(message: MediatorMessage<*>): MediatorMessage<*> { - val payload = serializePayload(message) - val request = buildHttpRequest(payload, message.endpoint()) - val response = sendWithRetry(request) - - checkResponseStatus(response.statusCode()) - - val deserializedResponse = deserializePayload(response.body()) - return MediatorMessage(deserializedResponse, mutableMapOf("statusCode" to response.statusCode())) - } - - private fun serializePayload(message: MediatorMessage<*>): ByteArray { - val payload = message.payload - - if (payload is ByteArray) return payload - - try { - return serializer.serialize(message.payload as Record<*, *>)!! - } catch (e: Exception) { - val errorMsg = "Failed to serialize instance of class type ${ - message.payload?.let { it::class.java.name } ?: "null" - }." - log.error(errorMsg) - onSerializationError?.invoke(errorMsg.toByteArray()) - throw(e) - } - } - - private fun deserializePayload(payload: ByteArray): Record<*,*> { - return try { - deserializer.deserialize(payload)!! - } catch (e: Exception) { - val errorMsg = "Failed to deserialize payload of size ${payload.size} bytes due to: ${e.message}" - log.error(errorMsg) - onSerializationError?.invoke(errorMsg.toByteArray()) - throw(e) - } - } - - private fun buildHttpRequest(payload: ByteArray, endpoint: String): HttpRequest { - return HttpRequest.newBuilder() - .uri(URI("http://$endpoint")) - .PUT(HttpRequest.BodyPublishers.ofByteArray(payload)) - .build() - } - - private fun sendWithRetry(request: HttpRequest): HttpResponse { - return HTTPRetryExecutor.withConfig(retryConfig) { - httpClient.send(request, HttpResponse.BodyHandlers.ofByteArray()) - } - } - - private fun checkResponseStatus(statusCode: Int) { - when (statusCode) { - in 400..499 -> throw CordaHTTPClientErrorException(statusCode, "Server returned status code $statusCode.") - in 500..599 -> throw CordaHTTPServerErrorException(statusCode, "Server returned status code $statusCode.") - } - } - - private fun handleExceptions(e: Exception) { - when (e) { - is IOException -> log.error("Network or IO operation error in RPCClient: ", e) - is InterruptedException -> log.error("Operation was interrupted in RPCClient: ", e) - is IllegalArgumentException -> log.error("Invalid argument provided in RPCClient call: ", e) - is SecurityException -> log.error("Security violation detected in RPCClient: ", e) - is IllegalStateException -> log.error("Coroutine state error in RPCClient: ", e) - is CordaHTTPClientErrorException -> log.error("Client-side HTTP error in RPCClient: ", e) - is CordaHTTPServerErrorException -> log.error("Server-side HTTP error in RPCClient: ", e) - else -> log.error("Unhandled exception in RPCClient: ", e) - } - - throw e - } - - override fun close() { - // Nothing to do here - } - - private fun MediatorMessage<*>.endpoint(): String { - return getProperty(MSG_PROP_ENDPOINT) - } -} diff --git a/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/factory/MessagingClientFactoryFactoryImpl.kt b/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/factory/MessagingClientFactoryFactoryImpl.kt index 0eecb634e7c..540e01d7240 100644 --- a/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/factory/MessagingClientFactoryFactoryImpl.kt +++ b/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/factory/MessagingClientFactoryFactoryImpl.kt @@ -1,6 +1,5 @@ package net.corda.messaging.mediator.factory -import net.corda.avro.serialization.CordaAvroSerializationFactory import net.corda.libs.configuration.SmartConfig import net.corda.messagebus.api.producer.builder.CordaProducerBuilder import net.corda.messaging.api.mediator.factory.MessagingClientFactoryFactory @@ -15,8 +14,6 @@ import org.osgi.service.component.annotations.Reference class MessagingClientFactoryFactoryImpl @Activate constructor( @Reference(service = CordaProducerBuilder::class) private val cordaProducerBuilder: CordaProducerBuilder, - @Reference(service = CordaAvroSerializationFactory::class) - private val cordaSerializationFactory: CordaAvroSerializationFactory ): MessagingClientFactoryFactory { override fun createMessageBusClientFactory( id: String, @@ -26,11 +23,4 @@ class MessagingClientFactoryFactoryImpl @Activate constructor( messageBusConfig, cordaProducerBuilder, ) - - override fun createRPCClientFactory( - id: String - ) = RPCClientFactory( - id, - cordaSerializationFactory - ) -} +} \ No newline at end of file diff --git a/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/factory/RPCClientFactory.kt b/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/factory/RPCClientFactory.kt deleted file mode 100644 index cd6ba744268..00000000000 --- a/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/factory/RPCClientFactory.kt +++ /dev/null @@ -1,21 +0,0 @@ -package net.corda.messaging.mediator.factory - -import net.corda.avro.serialization.CordaAvroSerializationFactory -import net.corda.messaging.api.mediator.MessagingClient -import net.corda.messaging.api.mediator.config.MessagingClientConfig -import net.corda.messaging.api.mediator.factory.MessagingClientFactory -import net.corda.messaging.mediator.RPCClient - -class RPCClientFactory( - private val id: String, - private val cordaSerializationFactory: CordaAvroSerializationFactory -): MessagingClientFactory { - - override fun create(config: MessagingClientConfig): MessagingClient { - return RPCClient( - id, - cordaSerializationFactory, - config.onSerializationError - ) - } -} diff --git a/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/utils/HTTPRetryConfig.kt b/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/utils/HTTPRetryConfig.kt deleted file mode 100644 index 13a791b6531..00000000000 --- a/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/utils/HTTPRetryConfig.kt +++ /dev/null @@ -1,37 +0,0 @@ -package net.corda.messaging.utils - -import kotlin.reflect.KClass - -/** - * Configuration class for HTTP retry parameters. - * - * @property times The number of a times a retry will be attempted. Default is 3 - * @property initialDelay The initial delay (in milliseconds) before the first retry. Default is 100ms - * @property factor The multiplier used to increase the delay for each subsequent retry. Default is 2.0 - * @property retryOn A set of exception classes that should trigger a retry when caught. - * If an exception not in this list is caught, it will be propagated immediately without retrying. - * Default is the generic [Exception] class, meaning all exceptions will trigger a retry. - */ -data class HTTPRetryConfig( - val times: Int = 3, - val initialDelay: Long = 100, - val factor: Double = 2.0, - val retryOn: Set> = setOf(Exception::class) -) { - class Builder { - private var times: Int = 3 - private var initialDelay: Long = 100 - private var factor: Double = 2.0 - private var retryOn: Set> = setOf(Exception::class) - - fun times(times: Int) = apply { this.times = times } - fun initialDelay(delay: Long) = apply { this.initialDelay = delay } - fun factor(factor: Double) = apply { this.factor = factor } - fun retryOn(vararg exceptions: KClass) = apply { this.retryOn = exceptions.toSet() } - - fun build(): HTTPRetryConfig { - return HTTPRetryConfig(times, initialDelay, factor, retryOn) - - } - } -} diff --git a/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/utils/HTTPRetryExecutor.kt b/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/utils/HTTPRetryExecutor.kt deleted file mode 100644 index 99b6095a932..00000000000 --- a/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/utils/HTTPRetryExecutor.kt +++ /dev/null @@ -1,42 +0,0 @@ -package net.corda.messaging.utils - -import org.slf4j.Logger -import org.slf4j.LoggerFactory - -class HTTPRetryExecutor { - companion object { - private val log: Logger = LoggerFactory.getLogger(this::class.java.enclosingClass) - - fun withConfig(config: HTTPRetryConfig, block: () -> T): T { - var currentDelay = config.initialDelay - for (i in 0 until config.times - 1) { - try { - log.info("HTTPRetryExecutor making attempt #${i + 1}.") - val result = block() - log.info("Operation successful after #${i + 1} attempt/s.") - return result - } catch (e: Exception) { - if (config.retryOn.none { it.isInstance(e) }) { - log.warn("HTTPRetryExecutor caught a non-retryable exception") - throw e - } - - log.warn("Attempt #${i + 1} failed due to ${e.message}. Retrying in $currentDelay ms...") - Thread.sleep(currentDelay) - currentDelay = (currentDelay * config.factor).toLong() - } - } - - log.warn("All retry attempts exhausted. Making the final call.") - - try { - val result = block() - log.info("Operation successful after #${config.times} attempt/s.") - return result - } catch (e: Exception) { - log.error("Operation failed after ${config.times} attempt/s.") - throw e - } - } - } -} diff --git a/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/RPCClientTest.kt b/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/RPCClientTest.kt deleted file mode 100644 index 762308ef01d..00000000000 --- a/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/RPCClientTest.kt +++ /dev/null @@ -1,219 +0,0 @@ -package net.corda.messaging.mediator - -import java.io.IOException -import java.net.http.HttpClient -import java.net.http.HttpRequest -import java.net.http.HttpResponse -import net.corda.avro.serialization.CordaAvroDeserializer -import net.corda.avro.serialization.CordaAvroSerializationFactory -import net.corda.avro.serialization.CordaAvroSerializer -import net.corda.messaging.api.exception.CordaHTTPClientErrorException -import net.corda.messaging.api.exception.CordaHTTPServerErrorException -import net.corda.messaging.api.mediator.MediatorMessage -import net.corda.messaging.api.mediator.MessagingClient.Companion.MSG_PROP_ENDPOINT -import net.corda.messaging.api.records.Record -import org.junit.jupiter.api.Assertions.assertEquals -import org.junit.jupiter.api.Assertions.assertNotNull -import org.junit.jupiter.api.BeforeEach -import org.junit.jupiter.api.Test -import org.junit.jupiter.api.assertThrows -import org.mockito.Mockito.times -import org.mockito.kotlin.any -import org.mockito.kotlin.eq -import org.mockito.kotlin.mock -import org.mockito.kotlin.verify -import org.mockito.kotlin.whenever - -class RPCClientTest { - - private lateinit var client: RPCClient - private val message = MediatorMessage( - Record("topic", "key", "testPayload"), - mutableMapOf(MSG_PROP_ENDPOINT to "test-endpoint/test") - ) - - data class Mocks( - val serializer: CordaAvroSerializer, - val deserializer: CordaAvroDeserializer>, - val httpClient: HttpClient, - val httpResponse: HttpResponse - ) - - private inner class MockEnvironment( - val mockSerializer: CordaAvroSerializer = mock(), - val mockDeserializer: CordaAvroDeserializer> = mock(), - val mockHttpClient: HttpClient = mock(), - val mockHttpResponse: HttpResponse = mock() - ) { - init { - whenever(mockSerializer.serialize(any>())) - .thenReturn("testPayload".toByteArray()) - - whenever(mockDeserializer.deserialize(any())) - .thenReturn(Record("topic", "key", "responsePayload")) - - whenever(mockHttpResponse.statusCode()) - .thenReturn(200) - - whenever(mockHttpResponse.body()) - .thenReturn("responsePayload".toByteArray()) - - whenever(mockHttpClient.send(any(), any>())) - .thenReturn(mockHttpResponse) - } - - fun withHttpStatus(status: Int) = apply { - whenever(mockHttpResponse.statusCode()).thenReturn(status) - } - - val mocks: Mocks - get() = Mocks(mockSerializer, mockDeserializer, mockHttpClient, mockHttpResponse) - } - - - private fun createClient( - mocks: Mocks, - onSerializationError: (ByteArray) -> Unit = mock(), - httpClientFactory: () -> HttpClient = { mocks.httpClient } - ): RPCClient { - val mockSerializationFactory: CordaAvroSerializationFactory = mock() - - whenever(mockSerializationFactory.createAvroSerializer(any())) - .thenReturn(mocks.serializer) - - whenever(mockSerializationFactory.createAvroDeserializer(any(), eq(Record::class.java))) - .thenReturn(mocks.deserializer) - - return RPCClient( - "TestRPCClient1", - mockSerializationFactory, - onSerializationError, - httpClientFactory - ) - } - - @BeforeEach - fun setup() { - val environment = MockEnvironment() - client = createClient(environment.mocks) - } - - @Test - fun `send() processes message and returns result`() { - val result = client.send(message) - assertNotNull(result?.payload) - assertEquals( - Record("topic", "key", "responsePayload"), - result!!.payload - ) - } - - @Test - fun `send() handles 4XX error`() { - val environment = MockEnvironment() - .withHttpStatus(404) - - val client = createClient(environment.mocks) - - assertThrows { - client.send(message) - } - } - - @Test - fun `send() handles 5XX error`() { - val environment = MockEnvironment() - .withHttpStatus(500) - - val client = createClient(environment.mocks) - - assertThrows { - client.send(message) - } - } - - @Test - fun `send() handles serialization error`() { - val onSerializationError: (ByteArray) -> Unit = mock() - - val environment = MockEnvironment().apply { - whenever(mockSerializer.serialize(any>())) - .thenThrow(IllegalArgumentException("Serialization error")) - } - - val client = createClient(environment.mocks, onSerializationError) - - assertThrows { - client.send(message) - } - - verify(onSerializationError).invoke(any()) - } - - @Test - fun `send() handles deserialization error`() { - val onSerializationError: (ByteArray) -> Unit = mock() - - val environment = MockEnvironment().apply { - whenever(mockSerializer.serialize(any>())) - .thenThrow(IllegalArgumentException("Deserialization error")) - } - - val client = createClient(environment.mocks, onSerializationError) - - assertThrows { - client.send(message) - } - - verify(onSerializationError).invoke(any()) - } - - @Test - fun `send retries on IOException and eventually succeeds`() { - val environment = MockEnvironment().apply { - whenever(mockHttpClient.send(any(), any>())) - .thenThrow(IOException("Simulated IO exception")) - .thenThrow(IOException("Simulated IO exception")) - .thenReturn(mockHttpResponse) - } - - val client = createClient(environment.mocks) - - val result = client.send(message) - assertEquals( - Record("topic", "key", "responsePayload"), - result!!.payload - ) - } - - @Test - fun `send fails after exhausting all retries`() { - val environment = MockEnvironment().apply { - whenever(mockHttpClient.send(any(), any>())) - .thenThrow(IOException("Simulated IO exception")) - } - - val client = createClient(environment.mocks) - - assertThrows { - client.send(message) - } - } - - @Test - fun `send retries the correct number of times before failing`() { - val environment = MockEnvironment().apply { - whenever(mockHttpClient.send(any(), any>())) - .thenThrow(IOException("Simulated IO exception")) - } - - val client = createClient(environment.mocks) - - assertThrows { - client.send(message) - } - - verify(environment.mockHttpClient, times(3)) - .send(any(), any>()) - } -} diff --git a/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/factory/MessagingClientFactoryFactoryTest.kt b/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/factory/MessagingClientFactoryFactoryTest.kt index 569c80cf319..e4a68753e9e 100644 --- a/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/factory/MessagingClientFactoryFactoryTest.kt +++ b/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/factory/MessagingClientFactoryFactoryTest.kt @@ -1,6 +1,5 @@ package net.corda.messaging.mediator.factory -import net.corda.avro.serialization.CordaAvroSerializationFactory import net.corda.libs.configuration.SmartConfig import net.corda.messagebus.api.producer.builder.CordaProducerBuilder import org.junit.jupiter.api.Assertions @@ -11,14 +10,12 @@ import org.mockito.kotlin.mock class MessagingClientFactoryFactoryTest { private lateinit var messagingClientFactoryFactory: MessagingClientFactoryFactoryImpl private val cordaProducerBuilder = mock() - private val cordaAvroSerializationFactory = mock() private val messageBusConfig = mock() @BeforeEach fun beforeEach() { messagingClientFactoryFactory = MessagingClientFactoryFactoryImpl( cordaProducerBuilder, - cordaAvroSerializationFactory ) } @@ -31,13 +28,4 @@ class MessagingClientFactoryFactoryTest { Assertions.assertNotNull(messageBusClientFactory) } - - @Test - fun testCreateRPCClientFactory() { - val rpcClientFactory = messagingClientFactoryFactory.createRPCClientFactory( - "rpcClient1" - ) - - Assertions.assertNotNull(rpcClientFactory) - } -} +} \ No newline at end of file diff --git a/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/factory/RPCClientFactoryTest.kt b/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/factory/RPCClientFactoryTest.kt deleted file mode 100644 index 2ae62f487e1..00000000000 --- a/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/factory/RPCClientFactoryTest.kt +++ /dev/null @@ -1,29 +0,0 @@ -package net.corda.messaging.mediator.factory - -import net.corda.avro.serialization.CordaAvroSerializationFactory -import net.corda.messaging.api.mediator.config.MessagingClientConfig -import org.junit.jupiter.api.Assertions -import org.junit.jupiter.api.BeforeEach -import org.junit.jupiter.api.Test -import org.mockito.Mockito.mock - -class RPCClientFactoryTest { - private lateinit var cordaSerializationFactory: CordaAvroSerializationFactory - private lateinit var rpcClientFactory: RPCClientFactory - - @BeforeEach - fun beforeEach() { - cordaSerializationFactory = mock(CordaAvroSerializationFactory::class.java) - rpcClientFactory = RPCClientFactory( - "RPCClient1", - mock(CordaAvroSerializationFactory::class.java) - ) - } - - @Test - fun testCreateRPCClient() { - val config = MessagingClientConfig {} - val rpcClient = rpcClientFactory.create(config) - Assertions.assertNotNull(rpcClient) - } -} diff --git a/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/utils/HTTPRetryExecutorTest.kt b/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/utils/HTTPRetryExecutorTest.kt deleted file mode 100644 index ddb302e6ade..00000000000 --- a/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/utils/HTTPRetryExecutorTest.kt +++ /dev/null @@ -1,77 +0,0 @@ -package net.corda.messaging.utils - -import org.junit.jupiter.api.Assertions.assertEquals -import org.junit.jupiter.api.BeforeEach -import org.junit.jupiter.api.Test -import org.junit.jupiter.api.assertThrows - -class HTTPRetryExecutorTest { - private lateinit var retryConfig: HTTPRetryConfig - - @BeforeEach - fun setUp() { - retryConfig = HTTPRetryConfig.Builder() - .times(3) - .initialDelay(100) - .factor(2.0) - .retryOn(RuntimeException::class) - .build() - } - - @Test - fun `successfully returns after first attempt`() { - val result = HTTPRetryExecutor.withConfig(retryConfig) { - "Success" - } - - assertEquals("Success", result) - } - - @Suppress("TooGenericExceptionThrown") - @Test - fun `should retry until successful`() { - var attempt = 0 - - val result = HTTPRetryExecutor.withConfig(retryConfig) { - ++attempt - if (attempt < 3) { - throw RuntimeException("Failed on attempt $attempt") - } - "Success on attempt $attempt" - } - - assertEquals("Success on attempt 3", result) - } - - @Suppress("TooGenericExceptionThrown") - @Test - fun `should throw exception after max attempts`() { - var attempt = 0 - - assertThrows { - HTTPRetryExecutor.withConfig(retryConfig) { - ++attempt - throw RuntimeException("Failed on attempt $attempt") - } - } - } - - @Suppress("TooGenericExceptionThrown") - @Test - fun `should not retry on non-retryable exception`() { - val config = HTTPRetryConfig.Builder() - .times(3) - .initialDelay(100) - .factor(2.0) - .retryOn(SpecificException::class) - .build() - - assertThrows { - HTTPRetryExecutor.withConfig(config) { - throw RuntimeException("I'm not retryable!") - } - } - } - - internal class SpecificException(message: String) : Exception(message) -} \ No newline at end of file diff --git a/libs/messaging/messaging/src/main/kotlin/net/corda/messaging/api/exception/CordaRestAPIExceptions.kt b/libs/messaging/messaging/src/main/kotlin/net/corda/messaging/api/exception/CordaRestAPIExceptions.kt index a369ff75971..b173417557e 100644 --- a/libs/messaging/messaging/src/main/kotlin/net/corda/messaging/api/exception/CordaRestAPIExceptions.kt +++ b/libs/messaging/messaging/src/main/kotlin/net/corda/messaging/api/exception/CordaRestAPIExceptions.kt @@ -19,15 +19,3 @@ class CordaRPCAPIPartitionException(message: String?, exception: Throwable? = nu */ class CordaRPCAPIResponderException(val errorType: String, message: String?, exception: Throwable? = null) : CordaRuntimeException(message, exception) - -/** - * Exception representing a 4XX response from the HTTP server - */ -class CordaHTTPClientErrorException(val statusCode: Int, message: String?, exception: Throwable? = null) : - CordaRuntimeException(message, exception) - -/** - * Exception representing a 5XX response from the HTTP server - */ -class CordaHTTPServerErrorException(val statusCode: Int, message: String?, exception: Throwable? = null) : - CordaRuntimeException(message, exception) diff --git a/libs/messaging/messaging/src/main/kotlin/net/corda/messaging/api/mediator/factory/MessagingClientFactoryFactory.kt b/libs/messaging/messaging/src/main/kotlin/net/corda/messaging/api/mediator/factory/MessagingClientFactoryFactory.kt index 3c4c7951847..9e69b6e31ea 100644 --- a/libs/messaging/messaging/src/main/kotlin/net/corda/messaging/api/mediator/factory/MessagingClientFactoryFactory.kt +++ b/libs/messaging/messaging/src/main/kotlin/net/corda/messaging/api/mediator/factory/MessagingClientFactoryFactory.kt @@ -16,13 +16,4 @@ interface MessagingClientFactoryFactory { id: String, messageBusConfig: SmartConfig, ) : MessagingClientFactory - - /** - * Creates an RPC messaging client factory. - * - * @param id RPC client ID. - */ - fun createRPCClientFactory( - id: String - ) : MessagingClientFactory -} +} \ No newline at end of file From 438bf3771a593c56f4f2583572c9b3063f784af4 Mon Sep 17 00:00:00 2001 From: David Currie Date: Fri, 13 Oct 2023 18:39:21 +0100 Subject: [PATCH 24/81] ES-1477 Enable PodMonitor filtering by metric name (#4866) --- charts/corda-lib/templates/_helpers.tpl | 7 +++++++ charts/corda/values.schema.json | 10 ++++++++++ charts/corda/values.yaml | 3 +++ 3 files changed, 20 insertions(+) diff --git a/charts/corda-lib/templates/_helpers.tpl b/charts/corda-lib/templates/_helpers.tpl index 04ad4ea475b..e7d77c61f3d 100644 --- a/charts/corda-lib/templates/_helpers.tpl +++ b/charts/corda-lib/templates/_helpers.tpl @@ -561,6 +561,13 @@ metadata: spec: podMetricsEndpoints: - port: monitor + {{- with .Values.metrics.podMonitor.keepNames }} + metricRelabelings: + - sourceLabels: + - "__name__" + regex: {{ join "|" . | quote }} + action: "keep" + {{- end }} jobLabel: {{ $.Release.Name }}-{{ include "corda.name" . }} selector: matchLabels: diff --git a/charts/corda/values.schema.json b/charts/corda/values.schema.json index 2ab971a361e..a830a6f8270 100644 --- a/charts/corda/values.schema.json +++ b/charts/corda/values.schema.json @@ -526,6 +526,16 @@ ] }, "examples": [{}] + }, + "keepNames": { + "type": "array", + "items": { + "type": "string", + "format": "regex" + }, + "default": [], + "title": "A list of regular expressions for the names of metrics that Prometheus should keep; if empty, all metrics are kept", + "examples": [[ "jvm_.*" ]] } }, "examples": [ diff --git a/charts/corda/values.yaml b/charts/corda/values.yaml index 8b5e9206b14..a8ed73f42a6 100644 --- a/charts/corda/values.yaml +++ b/charts/corda/values.yaml @@ -78,6 +78,9 @@ metrics: enabled: false # -- Labels that can be used so PodMonitor is discovered by Prometheus labels: {} + # -- A list of regular expressions for the names of metrics that Prometheus should keep; if empty, all metrics are kept + keepNames: [] + # - "jvm_.*" # Distributed tracing configuration tracing: From 25f5bd455fdef60600b994070c18527365a55ae4 Mon Sep 17 00:00:00 2001 From: Ronan Browne Date: Mon, 16 Oct 2023 08:45:43 +0100 Subject: [PATCH 25/81] ES-1491: Update CODEOWNERS file for 5.1 code freeze (#4877) --- .github/CODEOWNERS | 108 +-------------------------------------------- 1 file changed, 1 insertion(+), 107 deletions(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 861051ac939..734204008da 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,107 +1 @@ - -# Build scripts should be audited by BLT - -Jenkinsfile @corda/blt -.ci/** @corda/blt - -*.gradle @corda/blt -gradle.properties @corda/corda5-team-leads -gradle/* @corda/blt - -.github/** @corda/blt -CODEOWNERS @corda/blt @corda/corda5-team-leads - -# Modules to be audited by REST team -/applications/workers/release/rest-worker/ @corda/rest -/components/rest-gateway-comp/ @corda/rest -/components/permissions/ @corda/rest -/components/rbac-security-manager-service/ @corda/rest -/libs/rest/ @corda/rest -/libs/permissions/ @corda/rest -/processors/rest-processor/ @corda/rest -/tools/plugins/initial-rbac/ @corda/rest -/tools/plugins/plugins-rest/ @corda/rest -/tools/plugins/virtual-node/ @corda/rest - -# Corda Helm chart for cluster management team -/charts/corda/ @corda/cluster-management - -# Modules to be audited by the Network team -/applications/workers/release/p2p-gateway-worker/ @corda/corda-platform-network-team -/applications/workers/release/p2p-link-manager-worker/ @corda/corda-platform-network-team -/applications/workers/release/member-worker/ @corda/corda-platform-network-team -/processors/link-manager-processor/ @corda/corda-platform-network-team -/processors/gateway-processor/ @corda/corda-platform-network-team -/processors/member-processor/ @corda/corda-platform-network-team -/components/gateway/ @corda/corda-platform-network-team -/components/link-manager/ @corda/corda-platform-network-team -/components/membership/ @corda/corda-platform-network-team -/libs/membership/ @corda/corda-platform-network-team -/libs/p2p-crypto/ @corda/corda-platform-network-team -/libs/layered-property-map/ @corda/corda-platform-network-team -/tools/plugins/mgm/ @corda/corda-platform-network-team -/tools/plugins/network/ @corda/corda-platform-network-team -/applications/tools/p2p-test/ @corda/corda-platform-network-team - -# Modules to be audited by Sandboxing SMEs -/components/security-manager/ @corda/sandboxing -/components/virtual-node/sandbox-* @corda/sandboxing -/components/sandbox* @corda/sandboxing -/libs/virtual-node/sandbox-* @corda/sandboxing -/osgi-* @corda/sandboxing -/testing/sandboxes/ @corda/sandboxing -/testing/sandboxes-testkit/ @corda/sandboxing -/testing/security-manager-utilities/ @corda/sandboxing - -# Modules to be audited by Crypto SMEs -/components/crypto/ @corda/crypto -/libs/crypto/ @corda/crypto -/processors/crypto/ @corda/crypto - -# Modules to be audited by Packaging SMEs -/components/chunking/ @corda/packaging -/components/virtual-node/cpi-* @corda/packaging -/components/virtual-node/cpk-* @corda/packaging -/libs/chunking/ @corda/packaging -/libs/packaging/ @corda/packaging -/libs/serialization/ @corda/packaging -/libs/virtual-node/cpi-* @corda/packaging -/testing/packaging-test-utilities/ @corda/packaging -/tools/plugins/package @corda/packaging - -# Modules to be audited by DB SMEs -/components/db/ @corda/db -/components/persistence/ @corda/db -/components/reconciliation/ @corda/db -/libs/db/ @corda/db -/processors/db/ @corda/db -/testing/persistence-testkit/ @corda/db -/tools/plugins/db-config @corda/db - -# Modules to be audited by Flow Worker team -/components/flow/ @corda/flow-worker -/libs/flows/ @corda/flow-worker -/libs/lifecycle/ @corda/flow-worker -/libs/messaging/ @corda/flow-worker -/libs/application/application-impl/ @corda/flow-worker -/processors/flow-processor/ @corda/flow-worker -/testing/flow/ @corda/flow-worker -/testing/message-patterns/ @corda/flow-worker -/applications/workers/release/flow-worker @corda/flow-worker - -# Modules to be audited by Ledger SMEs -/components/ledger/ @corda/ledger -/libs/ledger/ @corda/ledger -/testing/ledger/ @corda/ledger - -# Modules to be audited by Notary SMEs -/components/uniqueness/ @corda/notaries -/libs/uniqueness/ @corda/notaries -/notary-plugins/ @corda/notaries -/processors/uniqueness-processor/ @corda/notaries -/testing/uniqueness/ @corda/notaries - -# Ledger token selection files to be reviewed by the REST team -# This needs to be after the ledger rules to partially override those -/components/ledger/ledger-utxo-token-cache @corda/rest -/components/ledger/ledger-utxo-flow/src/main/kotlin/net/corda/ledger/utxo/impl/token @corda/rest +* @driessamyn @jasonbyrner3 @dimosr @ronanbrowne @rick-r3 @simon-johnson-r3 @blsemo @Omar-awad @aditisdesai @vinir3 @vkolomeyko @thiagoviana @Sakpal From 8fab3261acff483e442f8bd77f9dd58dfc57cdc8 Mon Sep 17 00:00:00 2001 From: Yash Nabar Date: Mon, 16 Oct 2023 11:29:44 +0100 Subject: [PATCH 26/81] CORE-17047 Enable duplicate key detection in group policy parsing (#4797) Disallow duplicate key entries in the group policy. Before this change, providing duplicate keys in a static network group policy resulted in the last instance of that key being used (ignoring other instances). --- .../lib/impl/grouppolicy/GroupPolicyParserImpl.kt | 13 ++++++++++--- .../impl/grouppolicy/GroupPolicyParserImplTest.kt | 13 +++++++++++++ 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/libs/membership/membership-impl/src/main/kotlin/net/corda/membership/lib/impl/grouppolicy/GroupPolicyParserImpl.kt b/libs/membership/membership-impl/src/main/kotlin/net/corda/membership/lib/impl/grouppolicy/GroupPolicyParserImpl.kt index 2be336abaed..0aed5db0367 100644 --- a/libs/membership/membership-impl/src/main/kotlin/net/corda/membership/lib/impl/grouppolicy/GroupPolicyParserImpl.kt +++ b/libs/membership/membership-impl/src/main/kotlin/net/corda/membership/lib/impl/grouppolicy/GroupPolicyParserImpl.kt @@ -1,5 +1,6 @@ package net.corda.membership.lib.impl.grouppolicy +import com.fasterxml.jackson.core.JsonParser import com.fasterxml.jackson.databind.JsonNode import com.fasterxml.jackson.databind.ObjectMapper import net.corda.membership.lib.MemberInfoExtension.Companion.CREATION_TIME @@ -35,9 +36,10 @@ class GroupPolicyParserImpl @Activate constructor( const val EMPTY_GROUP_POLICY = "GroupPolicy file is empty." const val NULL_GROUP_POLICY = "GroupPolicy file is null." const val FAILED_PARSING = "GroupPolicy file is incorrectly formatted and parsing failed." + private val duplicateKeyRegex = "Duplicate field '.*'".toRegex(RegexOption.IGNORE_CASE) } - private val objectMapper = ObjectMapper() + private val objectMapper = ObjectMapper().enable(JsonParser.Feature.STRICT_DUPLICATE_DETECTION) private val clock = UTCClock() private val mgmVersions = mapOf( @@ -58,6 +60,11 @@ class GroupPolicyParserImpl @Activate constructor( } ) + private fun failedParsing(errorMessage: String): String = + duplicateKeyRegex.find(errorMessage)?.groupValues?.firstOrNull()?.let { + "$FAILED_PARSING Caused by: $it." + } ?: FAILED_PARSING + @Suppress("ThrowsCount") override fun parse( holdingIdentity: HoldingIdentity, @@ -78,7 +85,7 @@ class GroupPolicyParserImpl @Activate constructor( objectMapper.readTree(groupPolicy) } catch (e: Exception) { logger.error("$FAILED_PARSING Caused by: ${e.message}") - throw BadGroupPolicyException(FAILED_PARSING, e) + throw BadGroupPolicyException(failedParsing("${e.message}"), e) } } } @@ -105,7 +112,7 @@ class GroupPolicyParserImpl @Activate constructor( objectMapper.readTree(groupPolicy) } catch (e: Exception) { logger.error("$FAILED_PARSING Caused by: ${e.message}") - throw BadGroupPolicyException(FAILED_PARSING, e) + throw BadGroupPolicyException(failedParsing("${e.message}"), e) } } } diff --git a/libs/membership/membership-impl/src/test/kotlin/net/corda/membership/lib/impl/grouppolicy/GroupPolicyParserImplTest.kt b/libs/membership/membership-impl/src/test/kotlin/net/corda/membership/lib/impl/grouppolicy/GroupPolicyParserImplTest.kt index 0fe6be90dc8..00667b6ccf0 100644 --- a/libs/membership/membership-impl/src/test/kotlin/net/corda/membership/lib/impl/grouppolicy/GroupPolicyParserImplTest.kt +++ b/libs/membership/membership-impl/src/test/kotlin/net/corda/membership/lib/impl/grouppolicy/GroupPolicyParserImplTest.kt @@ -28,8 +28,10 @@ import net.corda.membership.lib.impl.grouppolicy.v1.TEST_CERT import net.corda.membership.lib.impl.grouppolicy.v1.TEST_FILE_FORMAT_VERSION import net.corda.membership.lib.impl.grouppolicy.v1.TEST_GROUP_ID import net.corda.membership.lib.impl.grouppolicy.v1.buildPersistedProperties +import net.corda.utilities.rootMessage import net.corda.v5.base.types.MemberX500Name import net.corda.virtualnode.HoldingIdentity +import org.assertj.core.api.Assertions.assertThat import org.assertj.core.api.SoftAssertions.assertSoftly import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Test @@ -103,6 +105,17 @@ class GroupPolicyParserImplTest { assertThrows { groupPolicyParser.parse(holdingIdentity, INVALID_FORMAT_GROUP_POLICY) { null } } } + @Test + fun `Duplicate keys in group policy throws corda runtime exception`() { + val groupPolicyWithDuplicateKey = + "{ \"fileFormatVersion\": 5, ${getSampleGroupPolicy(GroupPolicyType.STATIC).substringAfter("{")}" + + val exception = assertThrows { + groupPolicyParser.parse(holdingIdentity, groupPolicyWithDuplicateKey) { null } + } + assertThat(exception.rootMessage).contains("Duplicate field 'fileFormatVersion'") + } + @Test fun `Parse group policy - verify interface properties`() { val result = groupPolicyParser.parse(holdingIdentity, getSampleGroupPolicy(GroupPolicyType.STATIC)) { null } From 4b8440d9c67db5c0eb20bf6e07885b2691722826 Mon Sep 17 00:00:00 2001 From: Miljenko Brkic <97448832+mbrkic-r3@users.noreply.github.com> Date: Thu, 12 Oct 2023 11:26:52 +0100 Subject: [PATCH 27/81] CORE-17429 State metadata support in message processor (#4828) Adds support for state metadata to the state and event processor. Components using the event mediator to process events and states may wish to set metadata alongside the state that is written back to the underlying state storage. This change adds to the StateAndEventProcessor API to facilitate this, as well as migrating existing state and event processors to use this. --------- Co-authored-by: James Higgs --- .../integration/TestFlowMessageProcessor.kt | 5 +- .../executor/FlowMapperMessageProcessor.kt | 14 +++-- .../FlowMapperMessageProcessorTest.kt | 24 ++++++--- .../testing/context/FlowServiceTestContext.kt | 8 ++- .../testing/context/OutputAssertionsImpl.kt | 12 ++--- .../impl/FlowEventContextConverterImpl.kt | 3 +- .../factory/FlowEventPipelineFactory.kt | 5 +- .../impl/FlowEventPipelineFactoryImpl.kt | 8 +-- .../pipeline/impl/FlowEventProcessorImpl.kt | 12 +++-- .../corda/flow/RequestHandlerTestContext.kt | 3 +- .../FlowEventContextConverterImplTest.kt | 2 +- .../FlowEventPipelineFactoryImplTest.kt | 9 +++- .../impl/FlowEventProcessorImplTest.kt | 22 ++++---- .../flow/test/utils/FlowEventContextHelper.kt | 6 ++- .../services/TokenCacheEventProcessor.kt | 19 ++++--- .../services/TokenCacheEventProcessorTest.kt | 36 +++++++++---- .../linkmanager/delivery/DeliveryTracker.kt | 30 +++++++---- .../delivery/DeliveryTrackerTest.kt | 8 +-- .../MembershipPersistenceAsyncProcessor.kt | 24 +++++---- ...MembershipPersistenceAsyncProcessorTest.kt | 53 ++++++++++++------- .../service/impl/CommandsRetryManager.kt | 5 +- .../dynamic/RegistrationProcessor.kt | 30 ++++++----- .../dynamic/RegistrationProcessorTest.kt | 32 +++++++---- .../flow/pipeline/events/FlowEventContext.kt | 5 +- .../corda/messaging/mediator/ProcessorTask.kt | 13 +++-- .../messaging/mediator/StateManagerHelper.kt | 9 ++-- .../StateAndEventSubscriptionImpl.kt | 10 +++- .../MultiSourceEventMediatorImplTest.kt | 2 +- .../messaging/mediator/ProcessorTaskTest.kt | 21 +++++--- .../mediator/StateManagerHelperTest.kt | 23 +++++--- .../factory/MediatorComponentFactoryTest.kt | 7 ++- .../StateAndEventSubscriptionImplTest.kt | 5 +- libs/messaging/messaging/build.gradle | 2 + .../api/processor/StateAndEventProcessor.kt | 18 +++++-- .../processors/TestStateEventProcessor.kt | 10 +++- .../TestStateEventProcessorStrings.kt | 10 +++- ...ateAndEventSubscriptionIntegrationTests.kt | 47 ++++++++++++---- .../stateandevent/EventSubscription.kt | 12 +++-- .../stateandevent/EventSubscriptionTest.kt | 12 +++-- .../stateandevent/StateSubscriptionTest.kt | 5 +- 40 files changed, 394 insertions(+), 187 deletions(-) diff --git a/components/flow/flow-mapper-service/src/integrationTest/kotlin/net/corda/session/mapper/service/integration/TestFlowMessageProcessor.kt b/components/flow/flow-mapper-service/src/integrationTest/kotlin/net/corda/session/mapper/service/integration/TestFlowMessageProcessor.kt index a8c9d6bd04d..7c45b556ccf 100644 --- a/components/flow/flow-mapper-service/src/integrationTest/kotlin/net/corda/session/mapper/service/integration/TestFlowMessageProcessor.kt +++ b/components/flow/flow-mapper-service/src/integrationTest/kotlin/net/corda/session/mapper/service/integration/TestFlowMessageProcessor.kt @@ -3,6 +3,7 @@ package net.corda.session.mapper.service.integration import net.corda.data.flow.event.FlowEvent import net.corda.data.flow.state.checkpoint.Checkpoint import net.corda.messaging.api.processor.StateAndEventProcessor +import net.corda.messaging.api.processor.StateAndEventProcessor.State import net.corda.messaging.api.records.Record import org.junit.jupiter.api.fail import java.util.concurrent.CountDownLatch @@ -18,8 +19,8 @@ class TestFlowMessageProcessor( var eventsReceived: MutableList> = mutableListOf() override fun onNext( - state: Checkpoint?, - event: Record + state: State?, + event: Record, ): StateAndEventProcessor.Response { eventsReceived.add(event) diff --git a/components/flow/flow-mapper-service/src/main/kotlin/net/corda/session/mapper/service/executor/FlowMapperMessageProcessor.kt b/components/flow/flow-mapper-service/src/main/kotlin/net/corda/session/mapper/service/executor/FlowMapperMessageProcessor.kt index f0ffc75ab88..87db6bec60f 100644 --- a/components/flow/flow-mapper-service/src/main/kotlin/net/corda/session/mapper/service/executor/FlowMapperMessageProcessor.kt +++ b/components/flow/flow-mapper-service/src/main/kotlin/net/corda/session/mapper/service/executor/FlowMapperMessageProcessor.kt @@ -7,6 +7,7 @@ import net.corda.data.flow.state.mapper.FlowMapperState import net.corda.flow.mapper.factory.FlowMapperEventExecutorFactory import net.corda.libs.configuration.SmartConfig import net.corda.messaging.api.processor.StateAndEventProcessor +import net.corda.messaging.api.processor.StateAndEventProcessor.State import net.corda.messaging.api.records.Record import net.corda.metrics.CordaMetrics import net.corda.schema.configuration.FlowConfig @@ -38,14 +39,14 @@ class FlowMapperMessageProcessor( private val clock = UTCClock() override fun onNext( - state: FlowMapperState?, - event: Record + state: State?, + event: Record, ): StateAndEventProcessor.Response { val key = event.key logger.trace { "Received event. Key: $key Event: ${event.value}" } val value = event.value ?: return StateAndEventProcessor.Response(state, emptyList()) - val eventType = value.payload?.let { it.javaClass.simpleName } ?: "Unknown" + val eventType = value.payload?.javaClass?.simpleName ?: "Unknown" CordaMetrics.Metric.FlowMapperEventLag.builder() @@ -57,9 +58,12 @@ class FlowMapperMessageProcessor( return traceStateAndEventExecution(event, "Flow Mapper Event - $eventType") { eventProcessingTimer.recordCallable { if (!isExpiredSessionEvent(value)) { - val executor = flowMapperEventExecutorFactory.create(key, value, state, flowConfig) + val executor = flowMapperEventExecutorFactory.create(key, value, state?.value, flowConfig) val result = executor.execute() - StateAndEventProcessor.Response(result.flowMapperState, result.outputEvents) + StateAndEventProcessor.Response( + State(result.flowMapperState, state?.metadata), + result.outputEvents + ) } else { logger.debug { "This event is expired and will be ignored. Event: $event State: $state" } CordaMetrics.Metric.FlowMapperExpiredSessionEventCount.builder() diff --git a/components/flow/flow-mapper-service/src/test/kotlin/net/corda/session/mapper/service/executor/FlowMapperMessageProcessorTest.kt b/components/flow/flow-mapper-service/src/test/kotlin/net/corda/session/mapper/service/executor/FlowMapperMessageProcessorTest.kt index babb72cf8e4..f72a72056b4 100644 --- a/components/flow/flow-mapper-service/src/test/kotlin/net/corda/session/mapper/service/executor/FlowMapperMessageProcessorTest.kt +++ b/components/flow/flow-mapper-service/src/test/kotlin/net/corda/session/mapper/service/executor/FlowMapperMessageProcessorTest.kt @@ -13,9 +13,12 @@ import net.corda.flow.mapper.FlowMapperResult import net.corda.flow.mapper.executor.FlowMapperEventExecutor import net.corda.flow.mapper.factory.FlowMapperEventExecutorFactory import net.corda.libs.configuration.SmartConfigImpl +import net.corda.libs.statemanager.api.Metadata +import net.corda.messaging.api.processor.StateAndEventProcessor.State import net.corda.messaging.api.records.Record import net.corda.schema.configuration.FlowConfig import net.corda.test.flow.util.buildSessionEvent +import org.assertj.core.api.Assertions.assertThat import org.junit.jupiter.api.Test import org.mockito.kotlin.any import org.mockito.kotlin.anyOrNull @@ -35,12 +38,15 @@ class FlowMapperMessageProcessorTest { private val config = SmartConfigImpl.empty().withValue(FlowConfig.SESSION_P2P_TTL, ConfigValueFactory.fromAnyRef(10000)) private val flowMapperMessageProcessor = FlowMapperMessageProcessor(flowMapperEventExecutorFactory, config) - private fun buildMapperState(status: FlowMapperStateType) : FlowMapperState { - return FlowMapperState.newBuilder() - .setStatus(status) - .setFlowId("flowId") - .setExpiryTime(Instant.now().toEpochMilli()) - .build() + private fun buildMapperState(status: FlowMapperStateType, metadata: Metadata = Metadata()) : State { + return State( + FlowMapperState.newBuilder() + .setStatus(status) + .setFlowId("flowId") + .setExpiryTime(Instant.now().toEpochMilli()) + .build(), + metadata = metadata, + ) } private fun buildMapperEvent(payload: Any) : Record { @@ -69,8 +75,12 @@ class FlowMapperMessageProcessorTest { @Test fun `when state is OPEN new session events are processed`() { - flowMapperMessageProcessor.onNext(buildMapperState(FlowMapperStateType.OPEN), buildMapperEvent(buildSessionEvent())) + val metadata = Metadata(mapOf("foo" to "bar")) + val output = flowMapperMessageProcessor.onNext( + buildMapperState(FlowMapperStateType.OPEN, metadata),buildMapperEvent(buildSessionEvent()) + ) verify(flowMapperEventExecutorFactory, times(1)).create(any(), any(), anyOrNull(), any(), any()) + assertThat(output.updatedState?.metadata).isEqualTo(metadata) } @Test diff --git a/components/flow/flow-service/src/integrationTest/kotlin/net/corda/flow/testing/context/FlowServiceTestContext.kt b/components/flow/flow-service/src/integrationTest/kotlin/net/corda/flow/testing/context/FlowServiceTestContext.kt index 0659567a21c..6394cd846c1 100644 --- a/components/flow/flow-service/src/integrationTest/kotlin/net/corda/flow/testing/context/FlowServiceTestContext.kt +++ b/components/flow/flow-service/src/integrationTest/kotlin/net/corda/flow/testing/context/FlowServiceTestContext.kt @@ -54,6 +54,7 @@ import net.corda.libs.packaging.core.CpkManifest import net.corda.libs.packaging.core.CpkMetadata import net.corda.libs.packaging.core.CpkType import net.corda.messaging.api.processor.StateAndEventProcessor +import net.corda.messaging.api.processor.StateAndEventProcessor.State import net.corda.messaging.api.records.Record import net.corda.sandboxgroupcontext.SandboxGroupType.FLOW import net.corda.sandboxgroupcontext.VirtualNodeContext @@ -445,10 +446,13 @@ class FlowServiceTestContext @Activate constructor( log.info("Start test run for input/output set $iteration") flowFiberFactory.fiber.reset() flowFiberFactory.fiber.setIoRequests(testRun.ioRequests) - val response = flowEventProcessor.onNext(lastPublishedState, testRun.event) + val response = flowEventProcessor.onNext( + State(lastPublishedState, metadata = null), + testRun.event + ) testRun.flowContinuation = flowFiberFactory.fiber.flowContinuation testRun.response = response - lastPublishedState = response.updatedState + lastPublishedState = response.updatedState?.value } } diff --git a/components/flow/flow-service/src/integrationTest/kotlin/net/corda/flow/testing/context/OutputAssertionsImpl.kt b/components/flow/flow-service/src/integrationTest/kotlin/net/corda/flow/testing/context/OutputAssertionsImpl.kt index d6b00b966a0..ea711e469b7 100644 --- a/components/flow/flow-service/src/integrationTest/kotlin/net/corda/flow/testing/context/OutputAssertionsImpl.kt +++ b/components/flow/flow-service/src/integrationTest/kotlin/net/corda/flow/testing/context/OutputAssertionsImpl.kt @@ -259,13 +259,13 @@ class OutputAssertionsImpl( override fun hasPendingUserException() { asserts.add { testRun -> - assertThat(testRun.response?.updatedState?.pipelineState?.pendingPlatformError).isNotNull() + assertThat(testRun.response?.updatedState?.value?.pipelineState?.pendingPlatformError).isNotNull() } } override fun noPendingUserException() { asserts.add { testRun -> - assertThat(testRun.response?.updatedState?.pipelineState?.pendingPlatformError).isNull() + assertThat(testRun.response?.updatedState?.value?.pipelineState?.pendingPlatformError).isNull() } } @@ -278,8 +278,8 @@ class OutputAssertionsImpl( override fun checkpointHasRetry(expectedCount: Int) { asserts.add { testRun -> - assertThat(testRun.response?.updatedState?.pipelineState?.retryState).isNotNull - val retry = testRun.response!!.updatedState!!.pipelineState!!.retryState + assertThat(testRun.response?.updatedState?.value?.pipelineState?.retryState).isNotNull + val retry = testRun.response!!.updatedState!!.value?.pipelineState!!.retryState assertThat(retry.retryCount).isEqualTo(expectedCount) @@ -296,7 +296,7 @@ class OutputAssertionsImpl( override fun checkpointDoesNotHaveRetry() { asserts.add { testRun -> - assertThat(testRun.response?.updatedState?.pipelineState?.retryState).isNull() + assertThat(testRun.response?.updatedState?.value?.pipelineState?.retryState).isNull() } } @@ -364,7 +364,7 @@ class OutputAssertionsImpl( override fun nullStateRecord() { asserts.add { - assertNull(it.response?.updatedState, "Expected to receive NULL for output state") + assertNull(it.response?.updatedState?.value, "Expected to receive NULL for output state") } } diff --git a/components/flow/flow-service/src/main/kotlin/net/corda/flow/pipeline/converters/impl/FlowEventContextConverterImpl.kt b/components/flow/flow-service/src/main/kotlin/net/corda/flow/pipeline/converters/impl/FlowEventContextConverterImpl.kt index 08d4cdb6ec2..c1328de65e6 100644 --- a/components/flow/flow-service/src/main/kotlin/net/corda/flow/pipeline/converters/impl/FlowEventContextConverterImpl.kt +++ b/components/flow/flow-service/src/main/kotlin/net/corda/flow/pipeline/converters/impl/FlowEventContextConverterImpl.kt @@ -4,6 +4,7 @@ import net.corda.data.flow.state.checkpoint.Checkpoint import net.corda.flow.pipeline.events.FlowEventContext import net.corda.flow.pipeline.converters.FlowEventContextConverter import net.corda.messaging.api.processor.StateAndEventProcessor +import net.corda.messaging.api.processor.StateAndEventProcessor.State import org.osgi.service.component.annotations.Component @Suppress("Unused") @@ -11,7 +12,7 @@ import org.osgi.service.component.annotations.Component class FlowEventContextConverterImpl : FlowEventContextConverter { override fun convert(flowContext: FlowEventContext<*>): StateAndEventProcessor.Response { return StateAndEventProcessor.Response( - flowContext.checkpoint.toAvro(), + State(flowContext.checkpoint.toAvro(), metadata = flowContext.metadata), flowContext.outputRecords, flowContext.sendToDlq ) diff --git a/components/flow/flow-service/src/main/kotlin/net/corda/flow/pipeline/factory/FlowEventPipelineFactory.kt b/components/flow/flow-service/src/main/kotlin/net/corda/flow/pipeline/factory/FlowEventPipelineFactory.kt index afd24b79fad..e727a966d1c 100644 --- a/components/flow/flow-service/src/main/kotlin/net/corda/flow/pipeline/factory/FlowEventPipelineFactory.kt +++ b/components/flow/flow-service/src/main/kotlin/net/corda/flow/pipeline/factory/FlowEventPipelineFactory.kt @@ -4,6 +4,7 @@ import net.corda.data.flow.event.FlowEvent import net.corda.data.flow.state.checkpoint.Checkpoint import net.corda.flow.pipeline.FlowEventPipeline import net.corda.libs.configuration.SmartConfig +import net.corda.messaging.api.processor.StateAndEventProcessor.State import net.corda.tracing.TraceContext /** @@ -15,7 +16,7 @@ interface FlowEventPipelineFactory { /** * Creates a [FlowEventPipeline] instance. * - * @param checkpoint The [Checkpoint] passed through the pipeline. + * @param state The [Checkpoint] and metadata passed through the pipeline. * @param event The [FlowEvent] passed through the pipeline. * @param config The [SmartConfig] containing the settings used in the pipeline factory. * @param mdcProperties properties to set the flow fibers MDC with. @@ -25,7 +26,7 @@ interface FlowEventPipelineFactory { * @return A new [FlowEventPipeline] instance. */ fun create( - checkpoint: Checkpoint?, + state: State?, event: FlowEvent, configs: Map, mdcProperties: Map, diff --git a/components/flow/flow-service/src/main/kotlin/net/corda/flow/pipeline/factory/impl/FlowEventPipelineFactoryImpl.kt b/components/flow/flow-service/src/main/kotlin/net/corda/flow/pipeline/factory/impl/FlowEventPipelineFactoryImpl.kt index be2c575615b..b37882a6f1b 100644 --- a/components/flow/flow-service/src/main/kotlin/net/corda/flow/pipeline/factory/impl/FlowEventPipelineFactoryImpl.kt +++ b/components/flow/flow-service/src/main/kotlin/net/corda/flow/pipeline/factory/impl/FlowEventPipelineFactoryImpl.kt @@ -19,6 +19,7 @@ import net.corda.flow.pipeline.runner.FlowRunner import net.corda.flow.state.impl.FlowCheckpointFactory import net.corda.libs.configuration.SmartConfig import net.corda.libs.configuration.helper.getConfig +import net.corda.messaging.api.processor.StateAndEventProcessor.State import net.corda.schema.configuration.ConfigKeys.FLOW_CONFIG import net.corda.tracing.TraceContext import net.corda.virtualnode.read.VirtualNodeInfoReadService @@ -97,7 +98,7 @@ class FlowEventPipelineFactoryImpl( ) override fun create( - checkpoint: Checkpoint?, + state: State?, event: FlowEvent, configs: Map, mdcProperties: Map, @@ -106,7 +107,7 @@ class FlowEventPipelineFactoryImpl( ): FlowEventPipeline { val flowCheckpoint = flowCheckpointFactory.create( event.flowId, - checkpoint, + state?.value, configs.getConfig(FLOW_CONFIG) ) @@ -121,7 +122,8 @@ class FlowEventPipelineFactoryImpl( outputRecords = emptyList(), mdcProperties = mdcProperties, flowMetrics = metrics, - flowTraceContext = traceContext + flowTraceContext = traceContext, + metadata = state?.metadata ) val flowExecutionPipelineStage = FlowExecutionPipelineStage( diff --git a/components/flow/flow-service/src/main/kotlin/net/corda/flow/pipeline/impl/FlowEventProcessorImpl.kt b/components/flow/flow-service/src/main/kotlin/net/corda/flow/pipeline/impl/FlowEventProcessorImpl.kt index b048caa9976..f840c33916f 100644 --- a/components/flow/flow-service/src/main/kotlin/net/corda/flow/pipeline/impl/FlowEventProcessorImpl.kt +++ b/components/flow/flow-service/src/main/kotlin/net/corda/flow/pipeline/impl/FlowEventProcessorImpl.kt @@ -14,6 +14,7 @@ import net.corda.flow.pipeline.factory.FlowEventPipelineFactory import net.corda.flow.pipeline.handlers.FlowPostProcessingHandler import net.corda.libs.configuration.SmartConfig import net.corda.messaging.api.processor.StateAndEventProcessor +import net.corda.messaging.api.processor.StateAndEventProcessor.State import net.corda.messaging.api.records.Record import net.corda.schema.configuration.ConfigKeys.FLOW_CONFIG import net.corda.schema.configuration.MessagingConfig.Subscription.PROCESSOR_TIMEOUT @@ -51,11 +52,11 @@ class FlowEventProcessorImpl( } override fun onNext( - state: Checkpoint?, + state: State?, event: Record, ): StateAndEventProcessor.Response { val flowEvent = event.value - val mdcProperties = flowMDCService.getMDCLogging(state, flowEvent, event.key) + val mdcProperties = flowMDCService.getMDCLogging(state?.value, flowEvent, event.key) val eventType = event.value?.payload?.javaClass?.simpleName ?: "Unknown" return withMDC(mdcProperties) { traceStateAndEventExecution(event, "Flow Event - $eventType") { @@ -67,13 +68,16 @@ class FlowEventProcessorImpl( private fun getFlowPipelineResponse( flowEvent: FlowEvent?, event: Record, - state: Checkpoint?, + state: State?, mdcProperties: Map, traceContext: TraceContext ): StateAndEventProcessor.Response { if (flowEvent == null) { log.debug { "The incoming event record '${event}' contained a null FlowEvent, this event will be discarded" } - return StateAndEventProcessor.Response(state, listOf()) + return StateAndEventProcessor.Response( + state, + listOf() + ) } diff --git a/components/flow/flow-service/src/test/kotlin/net/corda/flow/RequestHandlerTestContext.kt b/components/flow/flow-service/src/test/kotlin/net/corda/flow/RequestHandlerTestContext.kt index bf4e67adca6..9a3b779caaa 100644 --- a/components/flow/flow-service/src/test/kotlin/net/corda/flow/RequestHandlerTestContext.kt +++ b/components/flow/flow-service/src/test/kotlin/net/corda/flow/RequestHandlerTestContext.kt @@ -70,6 +70,7 @@ class RequestHandlerTestContext(val payload: PAYLOAD) { recordList, mdcProperties = emptyMap(), flowMetrics = mock(), - flowTraceContext = mock() + flowTraceContext = mock(), + metadata = null ) } diff --git a/components/flow/flow-service/src/test/kotlin/net/corda/flow/pipeline/converters/FlowEventContextConverterImplTest.kt b/components/flow/flow-service/src/test/kotlin/net/corda/flow/pipeline/converters/FlowEventContextConverterImplTest.kt index be7cbf773ca..83c54c433e8 100644 --- a/components/flow/flow-service/src/test/kotlin/net/corda/flow/pipeline/converters/FlowEventContextConverterImplTest.kt +++ b/components/flow/flow-service/src/test/kotlin/net/corda/flow/pipeline/converters/FlowEventContextConverterImplTest.kt @@ -29,7 +29,7 @@ class FlowEventContextConverterImplTest { val result = converter.convert(context) assertThat(result.markForDLQ).isTrue - assertThat(result.updatedState).isSameAs(avroCheckpoint) + assertThat(result.updatedState?.value).isSameAs(avroCheckpoint) assertThat(result.responseEvents).isSameAs(records) } } \ No newline at end of file diff --git a/components/flow/flow-service/src/test/kotlin/net/corda/flow/pipeline/factory/FlowEventPipelineFactoryImplTest.kt b/components/flow/flow-service/src/test/kotlin/net/corda/flow/pipeline/factory/FlowEventPipelineFactoryImplTest.kt index b8f908b2f52..e91822b592e 100644 --- a/components/flow/flow-service/src/test/kotlin/net/corda/flow/pipeline/factory/FlowEventPipelineFactoryImplTest.kt +++ b/components/flow/flow-service/src/test/kotlin/net/corda/flow/pipeline/factory/FlowEventPipelineFactoryImplTest.kt @@ -19,6 +19,7 @@ import net.corda.flow.pipeline.runner.FlowRunner import net.corda.flow.state.FlowCheckpoint import net.corda.flow.state.impl.FlowCheckpointFactory import net.corda.flow.test.utils.buildFlowEventContext +import net.corda.messaging.api.processor.StateAndEventProcessor.State import net.corda.schema.configuration.ConfigKeys.FLOW_CONFIG import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Test @@ -89,7 +90,13 @@ class FlowEventPipelineFactoryImplTest { flowEventContext, mock(), ) - val result = factory.create(checkpoint, flowEvent, mapOf(FLOW_CONFIG to config), emptyMap(), flowEventContext.flowTraceContext, 0) + val result = factory.create( + State(checkpoint, null), + flowEvent, + mapOf(FLOW_CONFIG to config), + emptyMap(), + flowEventContext.flowTraceContext, + 0) assertEquals(expected.context, result.context) } } \ No newline at end of file diff --git a/components/flow/flow-service/src/test/kotlin/net/corda/flow/pipeline/impl/FlowEventProcessorImplTest.kt b/components/flow/flow-service/src/test/kotlin/net/corda/flow/pipeline/impl/FlowEventProcessorImplTest.kt index b09507754e4..b04e605803b 100644 --- a/components/flow/flow-service/src/test/kotlin/net/corda/flow/pipeline/impl/FlowEventProcessorImplTest.kt +++ b/components/flow/flow-service/src/test/kotlin/net/corda/flow/pipeline/impl/FlowEventProcessorImplTest.kt @@ -31,6 +31,7 @@ import net.corda.flow.pipeline.handlers.FlowPostProcessingHandler import net.corda.flow.state.FlowCheckpoint import net.corda.flow.test.utils.buildFlowEventContext import net.corda.messaging.api.processor.StateAndEventProcessor +import net.corda.messaging.api.processor.StateAndEventProcessor.State import net.corda.messaging.api.records.Record import net.corda.schema.Schemas.Flow.FLOW_EVENT_TOPIC import net.corda.schema.configuration.ConfigKeys.FLOW_CONFIG @@ -84,6 +85,7 @@ class FlowEventProcessorImplTest { private val flowKey = "flow id" private val flowCheckpoint = mock() private val checkpoint: Checkpoint = mock() + private val state = State(checkpoint, metadata = null) private val flowState: FlowState = mock() private val flowStartContext: FlowStartContext = mock() private val externalEventState: ExternalEventState = mock() @@ -161,9 +163,9 @@ class FlowEventProcessorImplTest { fun `Returns the state unaltered if no flow event supplied`() { val inputEvent = getFlowEventRecord(null) - val response = processor.onNext(checkpoint, inputEvent) + val response = processor.onNext(state, inputEvent) - assertThat(response.updatedState).isSameAs(checkpoint) + assertThat(response.updatedState?.value).isSameAs(checkpoint) assertThat(response.responseEvents).isEmpty() verify(flowMDCService, times(0)).getMDCLogging(anyOrNull(), any(), any()) } @@ -172,7 +174,7 @@ class FlowEventProcessorImplTest { fun `Returns a checkpoint and events to send`() { val inputEvent = getFlowEventRecord(FlowEvent(flowKey, payload)) - val response = processor.onNext(checkpoint, inputEvent) + val response = processor.onNext(state, inputEvent) val expectedRecords = updatedContext.outputRecords verify(flowEventContextConverter).convert(argThat { outputRecords == expectedRecords }) @@ -183,7 +185,7 @@ class FlowEventProcessorImplTest { @Test fun `Calls the pipeline steps in order`() { - processor.onNext(checkpoint, getFlowEventRecord(FlowEvent(flowKey, payload))) + processor.onNext(state, getFlowEventRecord(FlowEvent(flowKey, payload))) inOrder(flowEventPipeline) { verify(flowEventPipeline).eventPreProcessing() verify(flowEventPipeline).virtualNodeFlowOperationalChecks() @@ -199,7 +201,7 @@ class FlowEventProcessorImplTest { whenever(flowEventPipeline.eventPreProcessing()).thenThrow(error) whenever(flowEventExceptionProcessor.process(error, flowEventPipeline.context)).thenReturn(errorContext) - val response = processor.onNext(checkpoint, getFlowEventRecord(FlowEvent(flowKey, payload))) + val response = processor.onNext(state, getFlowEventRecord(FlowEvent(flowKey, payload))) assertThat(response).isEqualTo(errorResponse) } @@ -211,7 +213,7 @@ class FlowEventProcessorImplTest { whenever(flowEventPipeline.eventPreProcessing()).thenThrow(error) whenever(flowEventExceptionProcessor.process(error, flowEventPipeline.context)).thenReturn(errorContext) - val response = processor.onNext(checkpoint, getFlowEventRecord(FlowEvent(flowKey, payload))) + val response = processor.onNext(state, getFlowEventRecord(FlowEvent(flowKey, payload))) assertThat(response).isEqualTo(errorResponse) } @@ -223,7 +225,7 @@ class FlowEventProcessorImplTest { whenever(flowEventPipeline.eventPreProcessing()).thenThrow(error) whenever(flowEventExceptionProcessor.process(error, flowEventPipeline.context)).thenReturn(errorContext) - val response = processor.onNext(checkpoint, getFlowEventRecord(FlowEvent(flowKey, payload))) + val response = processor.onNext(state, getFlowEventRecord(FlowEvent(flowKey, payload))) assertThat(response).isEqualTo(errorResponse) } @@ -235,7 +237,7 @@ class FlowEventProcessorImplTest { whenever(flowEventPipeline.eventPreProcessing()).thenThrow(error) whenever(flowEventExceptionProcessor.process(error, flowEventPipeline.context)).thenReturn(errorContext) - val response = processor.onNext(checkpoint, getFlowEventRecord(FlowEvent(flowKey, payload))) + val response = processor.onNext(state, getFlowEventRecord(FlowEvent(flowKey, payload))) assertThat(response).isEqualTo(errorResponse) } @@ -247,7 +249,7 @@ class FlowEventProcessorImplTest { whenever(flowEventPipeline.eventPreProcessing()).thenThrow(error) whenever(flowEventExceptionProcessor.process(error, updatedContext)).thenReturn(errorContext) - val response = processor.onNext(checkpoint, getFlowEventRecord(FlowEvent(flowKey, payload))) + val response = processor.onNext(state, getFlowEventRecord(FlowEvent(flowKey, payload))) assertThat(response).isEqualTo(errorResponse) } @@ -268,7 +270,7 @@ class FlowEventProcessorImplTest { whenever(flowEventExceptionProcessor.process(error, updatedContext)).thenReturn(flowKillErrorContext) whenever(flowEventContextConverter.convert(eq(flowKillErrorContext))).thenReturn(killErrorResponse) - val result = processor.onNext(checkpoint, getFlowEventRecord(FlowEvent(flowKey, payload))) + val result = processor.onNext(state, getFlowEventRecord(FlowEvent(flowKey, payload))) assertThat(result).isEqualTo(killErrorResponse) } diff --git a/components/flow/flow-service/src/test/kotlin/net/corda/flow/test/utils/FlowEventContextHelper.kt b/components/flow/flow-service/src/test/kotlin/net/corda/flow/test/utils/FlowEventContextHelper.kt index 9b7a09c7353..67a78a23ec6 100644 --- a/components/flow/flow-service/src/test/kotlin/net/corda/flow/test/utils/FlowEventContextHelper.kt +++ b/components/flow/flow-service/src/test/kotlin/net/corda/flow/test/utils/FlowEventContextHelper.kt @@ -43,7 +43,8 @@ fun buildFlowEventContext( sendToDlq, emptyMap(), mock(), - mock() + mock(), + null ) } @@ -67,6 +68,7 @@ fun buildFlowEventContext( sendToDlq, emptyMap(), mock(), - mock() + mock(), + null ) } diff --git a/components/ledger/ledger-utxo-token-cache/src/main/kotlin/net/corda/ledger/utxo/token/cache/services/TokenCacheEventProcessor.kt b/components/ledger/ledger-utxo-token-cache/src/main/kotlin/net/corda/ledger/utxo/token/cache/services/TokenCacheEventProcessor.kt index 36aacf716f1..a326a77ae56 100644 --- a/components/ledger/ledger-utxo-token-cache/src/main/kotlin/net/corda/ledger/utxo/token/cache/services/TokenCacheEventProcessor.kt +++ b/components/ledger/ledger-utxo-token-cache/src/main/kotlin/net/corda/ledger/utxo/token/cache/services/TokenCacheEventProcessor.kt @@ -12,6 +12,7 @@ import net.corda.ledger.utxo.token.cache.entities.TokenEvent import net.corda.ledger.utxo.token.cache.entities.TokenPoolCache import net.corda.ledger.utxo.token.cache.handlers.TokenEventHandler import net.corda.messaging.api.processor.StateAndEventProcessor +import net.corda.messaging.api.processor.StateAndEventProcessor.State import net.corda.messaging.api.records.Record import net.corda.tracing.traceStateAndEventExecution import net.corda.utilities.debug @@ -38,15 +39,18 @@ class TokenCacheEventProcessor( override val stateValueClass = TokenPoolCacheState::class.java override fun onNext( - state: TokenPoolCacheState?, - event: Record + state: State?, + event: Record, ): StateAndEventProcessor.Response { val tokenEvent = try { eventConverter.convert(event.value) } catch (e: Exception) { log.error("Unexpected error while processing event '${event}'. The event will be sent to the DLQ.", e) - return StateAndEventProcessor.Response(state, listOf(), markForDLQ = true) + return StateAndEventProcessor.Response( + state, + listOf(), + markForDLQ = true) } log.debug { "Token event received: $tokenEvent" } @@ -54,7 +58,7 @@ class TokenCacheEventProcessor( return traceStateAndEventExecution(event, "Token Event - ${tokenEvent.javaClass.simpleName}") { try { tokenSelectionMetrics.recordProcessingTime(tokenEvent) { - val nonNullableState = state ?: TokenPoolCacheState().apply { + val nonNullableState = state?.value ?: TokenPoolCacheState().apply { this.poolKey = event.key this.availableTokens = listOf() this.tokenClaims = listOf() @@ -87,10 +91,13 @@ class TokenCacheEventProcessor( log.debug { "sending token response: $result" } if (result == null) { - StateAndEventProcessor.Response(poolCacheState.toAvro(), listOf()) + StateAndEventProcessor.Response( + State(poolCacheState.toAvro(), metadata = state?.metadata), + listOf() + ) } else { StateAndEventProcessor.Response( - poolCacheState.toAvro(), + State(poolCacheState.toAvro(), metadata = state?.metadata), listOf(result) ) } diff --git a/components/ledger/ledger-utxo-token-cache/src/test/kotlin/net/corda/ledger/utxo/token/cache/impl/services/TokenCacheEventProcessorTest.kt b/components/ledger/ledger-utxo-token-cache/src/test/kotlin/net/corda/ledger/utxo/token/cache/impl/services/TokenCacheEventProcessorTest.kt index 889a20e774b..cfbbc883616 100644 --- a/components/ledger/ledger-utxo-token-cache/src/test/kotlin/net/corda/ledger/utxo/token/cache/impl/services/TokenCacheEventProcessorTest.kt +++ b/components/ledger/ledger-utxo-token-cache/src/test/kotlin/net/corda/ledger/utxo/token/cache/impl/services/TokenCacheEventProcessorTest.kt @@ -17,6 +17,7 @@ import net.corda.ledger.utxo.token.cache.impl.POOL_CACHE_KEY import net.corda.ledger.utxo.token.cache.impl.POOL_KEY import net.corda.ledger.utxo.token.cache.services.TokenCacheEventProcessor import net.corda.ledger.utxo.token.cache.services.TokenSelectionMetricsImpl +import net.corda.messaging.api.processor.StateAndEventProcessor.State import net.corda.messaging.api.records.Record import net.corda.utilities.time.UTCClock import org.assertj.core.api.Assertions.assertThat @@ -65,10 +66,13 @@ class TokenCacheEventProcessorTest { val target = createTokenCacheEventProcessor() whenever(eventConverter.convert(any())).thenThrow(IllegalStateException()) - val result = target.onNext(stateIn, eventIn) + val result = target.onNext( + State(stateIn, metadata = null), + eventIn + ) assertThat(result.responseEvents).isEmpty() - assertThat(result.updatedState).isSameAs(stateIn) + assertThat(result.updatedState?.value).isSameAs(stateIn) assertThat(result.markForDLQ).isTrue } @@ -85,7 +89,10 @@ class TokenCacheEventProcessorTest { tokenSelectionMetrics ) - val result = target.onNext(stateIn, eventIn) + val result = target.onNext( + State(stateIn, metadata = null), + eventIn + ) verify(externalEventResponseFactory).platformError( eq( @@ -99,7 +106,7 @@ class TokenCacheEventProcessorTest { ) assertThat(result.responseEvents).isNotEmpty() - assertThat(result.updatedState).isSameAs(stateIn) + assertThat(result.updatedState?.value).isSameAs(stateIn) assertThat(result.markForDLQ).isFalse() } @@ -117,7 +124,10 @@ class TokenCacheEventProcessorTest { tokenSelectionMetrics ) - val result = target.onNext(stateIn, eventIn) + val result = target.onNext( + State(stateIn, metadata = null), + eventIn + ) verify(externalEventResponseFactory).platformError( eq( @@ -131,7 +141,7 @@ class TokenCacheEventProcessorTest { ) assertThat(result.responseEvents).isNotEmpty() - assertThat(result.updatedState).isSameAs(stateIn) + assertThat(result.updatedState?.value).isSameAs(stateIn) assertThat(result.markForDLQ).isFalse() } @@ -156,11 +166,14 @@ class TokenCacheEventProcessorTest { val target = createTokenCacheEventProcessor() - val result = target.onNext(stateIn, eventIn) + val result = target.onNext( + State(stateIn, metadata = null), + eventIn + ) assertThat(result.responseEvents).hasSize(1) assertThat(result.responseEvents.first()).isEqualTo(handlerResponse) - assertThat(result.updatedState).isSameAs(outputState) + assertThat(result.updatedState?.value).isSameAs(outputState) assertThat(result.markForDLQ).isFalse } @@ -191,7 +204,7 @@ class TokenCacheEventProcessorTest { assertThat(result.responseEvents).hasSize(1) assertThat(result.responseEvents.first()).isEqualTo(handlerResponse) - assertThat(result.updatedState).isSameAs(outputState) + assertThat(result.updatedState?.value).isSameAs(outputState) assertThat(result.markForDLQ).isFalse } @@ -216,7 +229,10 @@ class TokenCacheEventProcessorTest { val target = createTokenCacheEventProcessor() - target.onNext(stateIn, eventIn) + target.onNext( + State(stateIn, metadata = null), + eventIn + ) val inOrder = inOrder(cachePoolState, mockHandler) diff --git a/components/link-manager/src/main/kotlin/net/corda/p2p/linkmanager/delivery/DeliveryTracker.kt b/components/link-manager/src/main/kotlin/net/corda/p2p/linkmanager/delivery/DeliveryTracker.kt index 4bd6b6a66ff..5d74040b65d 100644 --- a/components/link-manager/src/main/kotlin/net/corda/p2p/linkmanager/delivery/DeliveryTracker.kt +++ b/components/link-manager/src/main/kotlin/net/corda/p2p/linkmanager/delivery/DeliveryTracker.kt @@ -20,6 +20,7 @@ import net.corda.membership.grouppolicy.GroupPolicyProvider import net.corda.membership.read.MembershipGroupReaderProvider import net.corda.messaging.api.processor.StateAndEventProcessor import net.corda.messaging.api.processor.StateAndEventProcessor.Response +import net.corda.messaging.api.processor.StateAndEventProcessor.State import net.corda.messaging.api.publisher.config.PublisherConfig import net.corda.messaging.api.publisher.factory.PublisherFactory import net.corda.messaging.api.records.Record @@ -150,8 +151,8 @@ internal class DeliveryTracker( val processor = object : StateAndEventProcessor { override fun onNext( - state: AuthenticatedMessageDeliveryState?, - event: Record + state: State?, + event: Record, ): Response { val marker = event.value if (marker == null) { @@ -161,18 +162,26 @@ internal class DeliveryTracker( val markerType = marker.marker val timestamp = marker.timestamp return when (markerType) { - is LinkManagerProcessedMarker -> Response(AuthenticatedMessageDeliveryState(markerType.message, timestamp), emptyList()) + is LinkManagerProcessedMarker -> Response( + State( + AuthenticatedMessageDeliveryState(markerType.message, timestamp), + metadata = state?.metadata + ), + emptyList() + ) is LinkManagerReceivedMarker -> { - if (state != null) { + val value = state?.value + if (value != null) { // if we receive multiple acknowledgements, it is possible the state might have been nullified already. // Only the first one matters for calculating the end-to-end delivery latency anyway. - recordDeliveryLatencyMetric(state) + recordDeliveryLatencyMetric(value) } Response(null, emptyList()) } is TtlExpiredMarker -> { - if (state != null) { - recordTtlExpiredMetric(state) + val value = state?.value + if (value != null) { + recordTtlExpiredMetric(value) } Response(null, emptyList()) } @@ -180,8 +189,11 @@ internal class DeliveryTracker( } } - private fun respond(state: AuthenticatedMessageDeliveryState?): Response { - return Response(state, emptyList()) + private fun respond(state: State?): Response { + return Response( + state, + emptyList() + ) } private fun recordDeliveryLatencyMetric(state: AuthenticatedMessageDeliveryState) { diff --git a/components/link-manager/src/test/kotlin/net/corda/p2p/linkmanager/delivery/DeliveryTrackerTest.kt b/components/link-manager/src/test/kotlin/net/corda/p2p/linkmanager/delivery/DeliveryTrackerTest.kt index 093fa06358a..9dbad6396b0 100644 --- a/components/link-manager/src/test/kotlin/net/corda/p2p/linkmanager/delivery/DeliveryTrackerTest.kt +++ b/components/link-manager/src/test/kotlin/net/corda/p2p/linkmanager/delivery/DeliveryTrackerTest.kt @@ -173,9 +173,9 @@ class DeliveryTrackerTest { tracker.stop() assertEquals(0, response.responseEvents.size) - assertNotNull(response.updatedState) - assertSame(messageAndKey, response.updatedState!!.message) - assertEquals(timeStamp, response.updatedState!!.timestamp) + assertNotNull(response.updatedState?.value) + assertSame(messageAndKey, response.updatedState!!.value!!.message) + assertEquals(timeStamp, response.updatedState!!.value!!.timestamp) } @Test @@ -188,7 +188,7 @@ class DeliveryTrackerTest { tracker.stop() assertEquals(0, response.responseEvents.size) - assertNull(response.updatedState) + assertNull(response.updatedState?.value) } @Test diff --git a/components/membership/membership-persistence-service-impl/src/main/kotlin/net/corda/membership/impl/persistence/service/MembershipPersistenceAsyncProcessor.kt b/components/membership/membership-persistence-service-impl/src/main/kotlin/net/corda/membership/impl/persistence/service/MembershipPersistenceAsyncProcessor.kt index 8b16c5fc9a1..e693c8d9fde 100644 --- a/components/membership/membership-persistence-service-impl/src/main/kotlin/net/corda/membership/impl/persistence/service/MembershipPersistenceAsyncProcessor.kt +++ b/components/membership/membership-persistence-service-impl/src/main/kotlin/net/corda/membership/impl/persistence/service/MembershipPersistenceAsyncProcessor.kt @@ -4,6 +4,7 @@ import net.corda.data.membership.db.request.async.MembershipPersistenceAsyncRequ import net.corda.data.membership.db.request.async.MembershipPersistenceAsyncRequestState import net.corda.membership.impl.persistence.service.handler.HandlerFactories import net.corda.messaging.api.processor.StateAndEventProcessor +import net.corda.messaging.api.processor.StateAndEventProcessor.State import net.corda.messaging.api.records.Record import org.slf4j.Logger import org.slf4j.LoggerFactory @@ -18,10 +19,9 @@ internal class MembershipPersistenceAsyncProcessor( const val MAX_RETRIES = 10 } override fun onNext( - state: MembershipPersistenceAsyncRequestState?, + state: State?, event: Record, ): StateAndEventProcessor.Response { - val numberOfRetriesSoFar = state?.numberOfRetriesSoFar ?: 0 val request = event.value if (request == null) { logger.warn("Empty request for ${event.key}") @@ -46,21 +46,21 @@ internal class MembershipPersistenceAsyncProcessor( retry( event.key, e, - numberOfRetriesSoFar, + state, request, ) } catch (e: OptimisticLockException) { retry( event.key, e, - numberOfRetriesSoFar, + state, request, ) } catch (e: RecoverableException) { retry( event.key, e, - numberOfRetriesSoFar, + state, request, ) } catch (e: Exception) { @@ -71,16 +71,20 @@ internal class MembershipPersistenceAsyncProcessor( private fun retry( key: String, e: Exception, - numberOfRetriesSoFar: Int, + state: State?, request: MembershipPersistenceAsyncRequest ): StateAndEventProcessor.Response { + val numberOfRetriesSoFar = state?.value?.numberOfRetriesSoFar ?: 0 return if (numberOfRetriesSoFar < MAX_RETRIES) { logger.warn("Got error while trying to execute $key. Will retry again.", e) StateAndEventProcessor.Response( - updatedState = MembershipPersistenceAsyncRequestState( - request, - numberOfRetriesSoFar + 1, - handlers.persistenceHandlerServices.clock.instant(), + updatedState = State( + MembershipPersistenceAsyncRequestState( + request, + numberOfRetriesSoFar + 1, + handlers.persistenceHandlerServices.clock.instant(), + ), + metadata = state?.metadata, ), responseEvents = emptyList(), markForDLQ = false, diff --git a/components/membership/membership-persistence-service-impl/src/test/kotlin/net/corda/membership/impl/persistence/service/MembershipPersistenceAsyncProcessorTest.kt b/components/membership/membership-persistence-service-impl/src/test/kotlin/net/corda/membership/impl/persistence/service/MembershipPersistenceAsyncProcessorTest.kt index a9d71b7e136..dfa340dc8b8 100644 --- a/components/membership/membership-persistence-service-impl/src/test/kotlin/net/corda/membership/impl/persistence/service/MembershipPersistenceAsyncProcessorTest.kt +++ b/components/membership/membership-persistence-service-impl/src/test/kotlin/net/corda/membership/impl/persistence/service/MembershipPersistenceAsyncProcessorTest.kt @@ -8,6 +8,7 @@ import net.corda.data.membership.db.request.async.MembershipPersistenceAsyncRequ import net.corda.membership.impl.persistence.service.handler.HandlerFactories import net.corda.membership.impl.persistence.service.handler.PersistenceHandlerServices import net.corda.messaging.api.processor.StateAndEventProcessor +import net.corda.messaging.api.processor.StateAndEventProcessor.State import net.corda.messaging.api.records.Record import net.corda.utilities.time.Clock import net.corda.v5.base.exceptions.CordaRuntimeException @@ -59,7 +60,7 @@ class MembershipPersistenceAsyncProcessorTest { ) assertThat(reply).isEqualTo( - StateAndEventProcessor.Response( + StateAndEventProcessor.Response( null, emptyList(), true, @@ -75,7 +76,7 @@ class MembershipPersistenceAsyncProcessorTest { ) assertThat(reply).isEqualTo( - StateAndEventProcessor.Response( + StateAndEventProcessor.Response( null, emptyList(), false, @@ -103,7 +104,7 @@ class MembershipPersistenceAsyncProcessorTest { ) assertThat(reply).isEqualTo( - StateAndEventProcessor.Response( + StateAndEventProcessor.Response( null, emptyList(), true, @@ -122,10 +123,13 @@ class MembershipPersistenceAsyncProcessorTest { assertThat(reply).isEqualTo( StateAndEventProcessor.Response( - MembershipPersistenceAsyncRequestState( - envelope, - 1, - now, + State( + MembershipPersistenceAsyncRequestState( + envelope, + 1, + now, + ), + metadata = null ), emptyList(), false, @@ -138,20 +142,26 @@ class MembershipPersistenceAsyncProcessorTest { whenever(handlers.handle(any())).doThrow(OptimisticLockException("Nop")) val reply = processor.onNext( - MembershipPersistenceAsyncRequestState( - envelope, - 1, - now, + State( + MembershipPersistenceAsyncRequestState( + envelope, + 1, + now, + ), + metadata = null ), Record("topic", "key", envelope) ) assertThat(reply).isEqualTo( StateAndEventProcessor.Response( - MembershipPersistenceAsyncRequestState( - envelope, - 2, - now, + State( + MembershipPersistenceAsyncRequestState( + envelope, + 2, + now, + ), + metadata = null ), emptyList(), false, @@ -164,16 +174,19 @@ class MembershipPersistenceAsyncProcessorTest { whenever(handlers.handle(any())).doThrow(RecoverableException("Nop")) val reply = processor.onNext( - MembershipPersistenceAsyncRequestState( - envelope, - 20, - now, + State( + MembershipPersistenceAsyncRequestState( + envelope, + 20, + now, + ), + metadata = null ), Record("topic", "key", envelope) ) assertThat(reply).isEqualTo( - StateAndEventProcessor.Response( + StateAndEventProcessor.Response( null, emptyList(), true, diff --git a/components/membership/membership-service-impl/src/main/kotlin/net/corda/membership/service/impl/CommandsRetryManager.kt b/components/membership/membership-service-impl/src/main/kotlin/net/corda/membership/service/impl/CommandsRetryManager.kt index 3cb6189bfaa..0bbf4acc9c6 100644 --- a/components/membership/membership-service-impl/src/main/kotlin/net/corda/membership/service/impl/CommandsRetryManager.kt +++ b/components/membership/membership-service-impl/src/main/kotlin/net/corda/membership/service/impl/CommandsRetryManager.kt @@ -8,6 +8,7 @@ import net.corda.data.membership.async.request.SentToMgmWaitingForNetwork import net.corda.libs.configuration.SmartConfig import net.corda.lifecycle.Resource import net.corda.messaging.api.processor.StateAndEventProcessor +import net.corda.messaging.api.processor.StateAndEventProcessor.State import net.corda.messaging.api.publisher.config.PublisherConfig import net.corda.messaging.api.publisher.factory.PublisherFactory import net.corda.messaging.api.records.Record @@ -46,11 +47,11 @@ internal class CommandsRetryManager( private val timers = ConcurrentHashMap>() override fun onNext( - state: MembershipAsyncRequestState?, + state: State?, event: Record, ): StateAndEventProcessor.Response { return StateAndEventProcessor.Response( - updatedState = event.value, + updatedState = State(event.value, state?.metadata), responseEvents = emptyList(), markForDLQ = false, ) diff --git a/components/membership/registration-impl/src/main/kotlin/net/corda/membership/impl/registration/dynamic/RegistrationProcessor.kt b/components/membership/registration-impl/src/main/kotlin/net/corda/membership/impl/registration/dynamic/RegistrationProcessor.kt index 77f251583c3..2ada4552ca7 100644 --- a/components/membership/registration-impl/src/main/kotlin/net/corda/membership/impl/registration/dynamic/RegistrationProcessor.kt +++ b/components/membership/registration-impl/src/main/kotlin/net/corda/membership/impl/registration/dynamic/RegistrationProcessor.kt @@ -34,6 +34,7 @@ import net.corda.membership.persistence.client.MembershipPersistenceClient import net.corda.membership.persistence.client.MembershipQueryClient import net.corda.membership.read.MembershipGroupReaderProvider import net.corda.messaging.api.processor.StateAndEventProcessor +import net.corda.messaging.api.processor.StateAndEventProcessor.State import net.corda.messaging.api.records.Record import net.corda.utilities.time.Clock import org.slf4j.LoggerFactory @@ -129,61 +130,62 @@ class RegistrationProcessor( @Suppress("ComplexMethod") override fun onNext( - state: RegistrationState?, - event: Record + state: State?, + event: Record, ): StateAndEventProcessor.Response { logger.info("Processing registration command for registration ID ${event.key}.") val result = try { + val stateValue = state?.value when (val command = event.value?.command) { is QueueRegistration -> { logger.info("Received queue registration command.") - handlers[QueueRegistration::class.java]?.invoke(state, event) + handlers[QueueRegistration::class.java]?.invoke(stateValue, event) } is CheckForPendingRegistration -> { logger.info("Received check for pending registration command.") - handlers[CheckForPendingRegistration::class.java]?.invoke(state, event) + handlers[CheckForPendingRegistration::class.java]?.invoke(stateValue, event) } is StartRegistration -> { logger.info("Received start registration command.") - handlers[StartRegistration::class.java]?.invoke(state, event) + handlers[StartRegistration::class.java]?.invoke(stateValue, event) } is VerifyMember -> { logger.info("Received verify member during registration command.") - handlers[VerifyMember::class.java]?.invoke(state, event) + handlers[VerifyMember::class.java]?.invoke(stateValue, event) } is ProcessMemberVerificationResponse -> { logger.info("Received process member verification response during registration command.") - handlers[ProcessMemberVerificationResponse::class.java]?.invoke(state, event) + handlers[ProcessMemberVerificationResponse::class.java]?.invoke(stateValue, event) } is ApproveRegistration -> { logger.info("Received approve registration command.") - handlers[ApproveRegistration::class.java]?.invoke(state, event) + handlers[ApproveRegistration::class.java]?.invoke(stateValue, event) } is DeclineRegistration -> { logger.info("Received decline registration command.") logger.warn("Declining registration because: ${command.reason}") - handlers[DeclineRegistration::class.java]?.invoke(state, event) + handlers[DeclineRegistration::class.java]?.invoke(stateValue, event) } is ProcessMemberVerificationRequest -> { logger.info("Received process member verification request during registration command.") - handlers[ProcessMemberVerificationRequest::class.java]?.invoke(state, event) + handlers[ProcessMemberVerificationRequest::class.java]?.invoke(stateValue, event) } is PersistMemberRegistrationState -> { logger.info("Received persist member registration state command.") - handlers[PersistMemberRegistrationState::class.java]?.invoke(state, event) + handlers[PersistMemberRegistrationState::class.java]?.invoke(stateValue, event) } else -> { logger.warn("Unhandled registration command received.") - createEmptyResult(state) + createEmptyResult(stateValue) } } } catch (e: MissingRegistrationStateException) { @@ -194,7 +196,9 @@ class RegistrationProcessor( createEmptyResult() } return StateAndEventProcessor.Response( - result?.updatedState, + result?.updatedState.let { + State(it, state?.metadata) + }, result?.outputStates ?: emptyList() ) } diff --git a/components/membership/registration-impl/src/test/kotlin/net/corda/membership/impl/registration/dynamic/RegistrationProcessorTest.kt b/components/membership/registration-impl/src/test/kotlin/net/corda/membership/impl/registration/dynamic/RegistrationProcessorTest.kt index 3016721747c..d76e6991f26 100644 --- a/components/membership/registration-impl/src/test/kotlin/net/corda/membership/impl/registration/dynamic/RegistrationProcessorTest.kt +++ b/components/membership/registration-impl/src/test/kotlin/net/corda/membership/impl/registration/dynamic/RegistrationProcessorTest.kt @@ -42,6 +42,7 @@ import net.corda.membership.lib.MemberInfoExtension.Companion.PARTY_SESSION_KEYS import net.corda.membership.lib.MemberInfoExtension.Companion.PLATFORM_VERSION import net.corda.membership.lib.SelfSignedMemberInfo import net.corda.membership.persistence.client.MembershipPersistenceOperation +import net.corda.messaging.api.processor.StateAndEventProcessor.State import net.corda.test.util.time.TestClock import net.corda.v5.base.types.MemberX500Name import net.corda.v5.membership.EndpointInfo @@ -281,8 +282,13 @@ class RegistrationProcessorTest { null, RegistrationState(registrationId, holdingIdentity, mgmHoldingIdentity, emptyList()) ).forEach { state -> - with(processor.onNext(state, Record(testTopic, testTopicKey, RegistrationCommand(Any())))) { - assertThat(updatedState).isEqualTo(state) + with( + processor.onNext( + State(state, metadata = null), + Record(testTopic, testTopicKey, RegistrationCommand(Any())) + ) + ) { + assertThat(updatedState?.value).isEqualTo(state) assertThat(responseEvents).isEmpty() } } @@ -291,7 +297,7 @@ class RegistrationProcessorTest { @Test fun `queue registration command - onNext can be called`() { val result = processor.onNext(null, Record(testTopic, testTopicKey, queueRegistrationCommand)) - assertThat(result.updatedState).isNull() + assertThat(result.updatedState?.value).isNull() assertThat(result.responseEvents).isNotEmpty.hasSize(2) assertThat(result.responseEvents.firstNotNullOf { it.value as? RegistrationCommand }.command) .isNotNull @@ -301,7 +307,7 @@ class RegistrationProcessorTest { @Test fun `check for pending registration command - onNext can be called`() { val result = processor.onNext(null, Record(testTopic, testTopicKey, checkForPendingRegistrationCommand)) - assertThat(result.updatedState).isNotNull() + assertThat(result.updatedState?.value).isNotNull() assertThat(result.responseEvents).isNotEmpty.hasSize(1) assertThat((result.responseEvents.first().value as? RegistrationCommand)?.command) .isNotNull @@ -311,10 +317,13 @@ class RegistrationProcessorTest { @Test fun `start registration command - onNext can be called for start registration command`() { val result = processor.onNext( - RegistrationState(registrationId, holdingIdentity, mgmHoldingIdentity, emptyList()), + State( + RegistrationState(registrationId, holdingIdentity, mgmHoldingIdentity, emptyList()), + metadata = null + ), Record(testTopic, testTopicKey, startRegistrationCommand) ) - assertThat(result.updatedState).isNotNull + assertThat(result.updatedState?.value).isNotNull val events = result.responseEvents assertThat(events).isNotEmpty.hasSize(2) assertThat(events.firstNotNullOf { it.value as? RegistrationCommand }.command) @@ -325,7 +334,7 @@ class RegistrationProcessorTest { @Test fun `process member verification request command - onNext can be called for command`() { val result = processor.onNext(null, Record(testTopic, testTopicKey, verificationRequestCommand)) - assertThat(result.updatedState).isNull() + assertThat(result.updatedState?.value).isNull() assertThat(result.responseEvents) .hasSize(1) .anySatisfy { @@ -348,8 +357,11 @@ class RegistrationProcessorTest { @Test fun `verify member command - onNext can be called for command`() { - val result = processor.onNext(state, Record(testTopic, testTopicKey, verifyMemberCommand)) - assertThat(result.updatedState).isNotNull + val result = processor.onNext( + State(state, metadata = null), + Record(testTopic, testTopicKey, verifyMemberCommand) + ) + assertThat(result.updatedState?.value).isNotNull assertThat(result.responseEvents).isNotEmpty.hasSize(1) .allMatch { (result.responseEvents.first().value as? AppMessage)?.message as? AuthenticatedMessage != null @@ -359,7 +371,7 @@ class RegistrationProcessorTest { @Test fun `missing RegistrationState results in empty response`() { val result = processor.onNext(null, Record(testTopic, testTopicKey, verifyMemberCommand)) - assertThat(result.updatedState).isNull() + assertThat(result.updatedState?.value).isNull() assertThat(result.responseEvents).isEmpty() } } diff --git a/libs/flows/flow-api/src/main/kotlin/net/corda/flow/pipeline/events/FlowEventContext.kt b/libs/flows/flow-api/src/main/kotlin/net/corda/flow/pipeline/events/FlowEventContext.kt index c2ce2b05957..ba389ce5f34 100644 --- a/libs/flows/flow-api/src/main/kotlin/net/corda/flow/pipeline/events/FlowEventContext.kt +++ b/libs/flows/flow-api/src/main/kotlin/net/corda/flow/pipeline/events/FlowEventContext.kt @@ -4,6 +4,7 @@ import net.corda.data.flow.event.FlowEvent import net.corda.flow.pipeline.metrics.FlowMetrics import net.corda.flow.state.FlowCheckpoint import net.corda.libs.configuration.SmartConfig +import net.corda.libs.statemanager.api.Metadata import net.corda.messaging.api.records.Record import net.corda.tracing.TraceContext @@ -23,6 +24,7 @@ import net.corda.tracing.TraceContext * @param mdcProperties properties to set the flow fibers MDC with. * @param flowMetrics The [FlowMetrics] instance associated with the flow event * @param flowTraceContext The [TraceContext] instance associated with the flow event + * @param metadata Metadata associated with the checkpoint in state storage */ data class FlowEventContext( val checkpoint: FlowCheckpoint, @@ -35,5 +37,6 @@ data class FlowEventContext( val sendToDlq: Boolean = false, val mdcProperties: Map, val flowMetrics: FlowMetrics, - val flowTraceContext: TraceContext + val flowTraceContext: TraceContext, + val metadata: Metadata? ) diff --git a/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/ProcessorTask.kt b/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/ProcessorTask.kt index db7016b23a6..1faeb4d28c3 100644 --- a/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/ProcessorTask.kt +++ b/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/ProcessorTask.kt @@ -28,18 +28,23 @@ data class ProcessorTask( } override fun call(): Result { - var stateValue = stateManagerHelper.deserializeValue(persistedState) + var state = stateManagerHelper.deserializeValue(persistedState)?.let { stateValue -> + StateAndEventProcessor.State( + stateValue, + persistedState?.metadata + ) + } val outputEvents = events.map { event -> - val response = processor.onNext(stateValue, event) - stateValue = response.updatedState + val response = processor.onNext(state, event) + state = response.updatedState response.responseEvents }.flatten() val updatedState = stateManagerHelper.createOrUpdateState( key.toString(), persistedState, - stateValue + state, ) return Result(this, outputEvents, updatedState) diff --git a/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/StateManagerHelper.kt b/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/StateManagerHelper.kt index 3d43146696b..4d6389d7e26 100644 --- a/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/StateManagerHelper.kt +++ b/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/StateManagerHelper.kt @@ -5,6 +5,7 @@ import net.corda.avro.serialization.CordaAvroSerializer import net.corda.libs.statemanager.api.Metadata import net.corda.libs.statemanager.api.State import net.corda.libs.statemanager.api.StateManager +import net.corda.messaging.api.processor.StateAndEventProcessor /** * Helper for working with [StateManager], used by [MultiSourceEventMediatorImpl]. @@ -20,18 +21,18 @@ class StateManagerHelper( * * @param key Event's key. * @param persistedState State being updated. - * @param newValue Updated state value. + * @param newState Updated state. */ fun createOrUpdateState( key: String, persistedState: State?, - newValue: S?, - ) = serialize(newValue)?.let { serializedValue -> + newState: StateAndEventProcessor.State?, + ) = serialize(newState?.value)?.let { serializedValue -> State( key, serializedValue, persistedState?.version ?: State.VERSION_INITIAL_VALUE, - persistedState?.metadata ?: Metadata() + newState?.metadata ?: Metadata(), ) } diff --git a/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/subscription/StateAndEventSubscriptionImpl.kt b/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/subscription/StateAndEventSubscriptionImpl.kt index 2753f0fed70..c58ed7697c0 100644 --- a/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/subscription/StateAndEventSubscriptionImpl.kt +++ b/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/subscription/StateAndEventSubscriptionImpl.kt @@ -13,6 +13,7 @@ import net.corda.messaging.api.chunking.ChunkSerializerService import net.corda.messaging.api.exception.CordaMessageAPIFatalException import net.corda.messaging.api.exception.CordaMessageAPIIntermittentException import net.corda.messaging.api.processor.StateAndEventProcessor +import net.corda.messaging.api.processor.StateAndEventProcessor.State import net.corda.messaging.api.records.Record import net.corda.messaging.api.subscription.StateAndEventSubscription import net.corda.messaging.api.subscription.listener.StateAndEventListener @@ -276,7 +277,7 @@ internal class StateAndEventSubscriptionImpl( val state = updatedStates[event.partition]?.get(event.key) val partitionId = event.partition val thisEventUpdates = getUpdatesForEvent(state, event) - val updatedState = thisEventUpdates?.updatedState + val updatedState = thisEventUpdates?.updatedState?.value when { @@ -329,7 +330,12 @@ internal class StateAndEventSubscriptionImpl( private fun getUpdatesForEvent(state: S?, event: CordaConsumerRecord): StateAndEventProcessor.Response? { val future = stateAndEventConsumer.waitForFunctionToFinish( - { processor.onNext(state, event.toRecord()) }, config.processorTimeout.toMillis(), + { + processor.onNext( + State(state, metadata = null), + event.toRecord() + ) + }, config.processorTimeout.toMillis(), "Failed to finish within the time limit for state: $state and event: $event" ) @Suppress("unchecked_cast") diff --git a/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/MultiSourceEventMediatorImplTest.kt b/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/MultiSourceEventMediatorImplTest.kt index 48f558cb4ff..db31cd73257 100644 --- a/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/MultiSourceEventMediatorImplTest.kt +++ b/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/MultiSourceEventMediatorImplTest.kt @@ -76,7 +76,7 @@ class MultiSourceEventMediatorImplTest { any() ) ).thenAnswer { - StateAndEventProcessor.Response( + StateAndEventProcessor.Response( updatedState = mock(), responseEvents = listOf( Record( diff --git a/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/ProcessorTaskTest.kt b/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/ProcessorTaskTest.kt index a0321712b3e..8a35d21b3a3 100644 --- a/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/ProcessorTaskTest.kt +++ b/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/ProcessorTaskTest.kt @@ -1,5 +1,6 @@ package net.corda.messaging.mediator +import net.corda.libs.statemanager.api.Metadata import net.corda.libs.statemanager.api.State import net.corda.messaging.api.processor.StateAndEventProcessor import net.corda.messaging.api.records.Record @@ -31,7 +32,7 @@ class ProcessorTaskTest { private val stateManagerHelper = mock>() @Captor - private val stateCaptor = argumentCaptor() + private val stateCaptor = argumentCaptor>() @Captor private val eventCaptor = argumentCaptor>() @@ -39,13 +40,13 @@ class ProcessorTaskTest { @BeforeEach fun setup() { `when`(processor.onNext(anyOrNull(), any())).thenAnswer { invocation -> - val state = invocation.getArgument(0) - val id = state?.let { it.id + 1 } ?: 0 + val state = invocation.getArgument>(0) + val id = state?.let { it.value!!.id + 1 } ?: 0 StateAndEventProcessor.Response( - StateType(id), + StateAndEventProcessor.State(StateType(id), Metadata(mapOf("id" to id))), listOf( EventType("outputEvent$id").toRecord() - ) + ), ) } @@ -72,9 +73,15 @@ class ProcessorTaskTest { val result = task.call() - verify(processor, times(inputEventRecords.size)).onNext(stateCaptor.capture(), eventCaptor.capture()) + verify(processor, times(inputEventRecords.size)).onNext( + stateCaptor.capture(), eventCaptor.capture() + ) val capturedInputStates = stateCaptor.allValues - val expectedInputStates = listOf(null, StateType(0), StateType(1)) + val expectedInputStates = listOf( + null, + StateAndEventProcessor.State(StateType(0), Metadata(mapOf("id" to 0))), + StateAndEventProcessor.State(StateType(1), Metadata(mapOf("id" to 1))), + ) assertEquals(expectedInputStates, capturedInputStates) val capturedInputEventRecords = eventCaptor.allValues assertEquals(inputEventRecords, capturedInputEventRecords) diff --git a/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/StateManagerHelperTest.kt b/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/StateManagerHelperTest.kt index 5610a805d5e..a1c9db1b781 100644 --- a/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/StateManagerHelperTest.kt +++ b/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/StateManagerHelperTest.kt @@ -5,6 +5,7 @@ import net.corda.avro.serialization.CordaAvroSerializer import net.corda.libs.statemanager.api.Metadata import net.corda.libs.statemanager.api.State import net.corda.libs.statemanager.api.StateManager +import net.corda.messaging.api.processor.StateAndEventProcessor import org.junit.jupiter.api.Assertions.assertArrayEquals import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Assertions.assertNotNull @@ -51,7 +52,10 @@ class StateManagerHelperTest { fun `successfully creates new state`() { val persistedState: State? = null - val newValue = StateType(1) + val newState = StateAndEventProcessor.State( + StateType(1), + mock(), + ) val stateManagerHelper = StateManagerHelper( stateManager, stateSerializer, @@ -59,14 +63,14 @@ class StateManagerHelperTest { ) val state = stateManagerHelper.createOrUpdateState( - TEST_KEY, persistedState, newValue + TEST_KEY, persistedState, newState ) assertNotNull(state) assertEquals(TEST_KEY, state!!.key) - assertArrayEquals(serialized(newValue), state.value) + assertArrayEquals(serialized(newState.value!!), state.value) assertEquals(State.VERSION_INITIAL_VALUE, state.version) - assertNotNull(state.metadata) + assertEquals(newState.metadata, state.metadata) } @Test @@ -78,7 +82,10 @@ class StateManagerHelperTest { stateVersion, mock() ) - val updatedValue = StateType(TEST_STATE_VALUE.id + 1) + val updatedState = StateAndEventProcessor.State( + StateType(TEST_STATE_VALUE.id + 1), + mock(), + ) val stateManagerHelper = StateManagerHelper( stateManager, stateSerializer, @@ -86,14 +93,14 @@ class StateManagerHelperTest { ) val state = stateManagerHelper.createOrUpdateState( - TEST_KEY, persistedState, updatedValue + TEST_KEY, persistedState, updatedState ) assertNotNull(state) assertEquals(persistedState.key, state!!.key) - assertArrayEquals(serialized(updatedValue), state.value) + assertArrayEquals(serialized(updatedState.value!!), state.value) assertEquals(persistedState.version, state.version) - assertEquals(persistedState.metadata, state.metadata) + assertEquals(updatedState.metadata, state.metadata) } @Test diff --git a/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/factory/MediatorComponentFactoryTest.kt b/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/factory/MediatorComponentFactoryTest.kt index f111353d0be..43b88ad86f8 100644 --- a/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/factory/MediatorComponentFactoryTest.kt +++ b/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/factory/MediatorComponentFactoryTest.kt @@ -1,5 +1,6 @@ package net.corda.messaging.mediator.factory + import net.corda.messaging.api.mediator.MediatorConsumer import net.corda.messaging.api.mediator.MessageRouter import net.corda.messaging.api.mediator.MessagingClient @@ -10,6 +11,7 @@ import net.corda.messaging.api.mediator.factory.MessageRouterFactory import net.corda.messaging.api.mediator.factory.MessagingClientFactory import net.corda.messaging.api.mediator.factory.MessagingClientFinder import net.corda.messaging.api.processor.StateAndEventProcessor +import net.corda.messaging.api.processor.StateAndEventProcessor.State import net.corda.messaging.api.records.Record import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Assertions.assertNotNull @@ -27,9 +29,12 @@ import org.mockito.kotlin.whenever class MediatorComponentFactoryTest { private lateinit var mediatorComponentFactory: MediatorComponentFactory private val messageProcessor = object : StateAndEventProcessor { - override fun onNext(state: String?, event: Record): StateAndEventProcessor.Response { + override fun onNext( + state: State?, event: Record + ): StateAndEventProcessor.Response { TODO("Not yet implemented") } + override val keyClass get() = String::class.java override val stateValueClass get() = String::class.java override val eventValueClass get() = String::class.java diff --git a/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/subscription/StateAndEventSubscriptionImplTest.kt b/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/subscription/StateAndEventSubscriptionImplTest.kt index 1f0a14fdb01..f468aec20bc 100644 --- a/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/subscription/StateAndEventSubscriptionImplTest.kt +++ b/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/subscription/StateAndEventSubscriptionImplTest.kt @@ -17,6 +17,7 @@ import net.corda.messaging.api.chunking.ChunkSerializerService import net.corda.messaging.api.exception.CordaMessageAPIFatalException import net.corda.messaging.api.exception.CordaMessageAPIIntermittentException import net.corda.messaging.api.processor.StateAndEventProcessor +import net.corda.messaging.api.processor.StateAndEventProcessor.State import net.corda.messaging.api.records.Record import net.corda.messaging.constants.SubscriptionType import net.corda.messaging.createResolvedSubscriptionConfig @@ -75,7 +76,7 @@ class StateAndEventSubscriptionImplTest { doAnswer { CompletableFuture.completedFuture( StateAndEventProcessor.Response( - "newstate", + State("newstate", metadata = null), emptyList() ) ) @@ -466,7 +467,7 @@ class StateAndEventSubscriptionImplTest { doAnswer { CompletableFuture.completedFuture( - StateAndEventProcessor.Response( + StateAndEventProcessor.Response( null, listOf(outputRecord), true diff --git a/libs/messaging/messaging/build.gradle b/libs/messaging/messaging/build.gradle index 18c0c5d1ae5..728525d2f30 100644 --- a/libs/messaging/messaging/build.gradle +++ b/libs/messaging/messaging/build.gradle @@ -7,6 +7,8 @@ dependencies { compileOnly 'org.osgi:osgi.annotation' compileOnly "co.paralleluniverse:quasar-osgi-annotations:$quasarVersion" + api project(":libs:state-manager:state-manager-api") + implementation platform("net.corda:corda-api:$cordaApiVersion") implementation 'org.jetbrains.kotlin:kotlin-osgi-bundle' implementation "com.typesafe:config:$typeSafeConfigVersion" diff --git a/libs/messaging/messaging/src/main/kotlin/net/corda/messaging/api/processor/StateAndEventProcessor.kt b/libs/messaging/messaging/src/main/kotlin/net/corda/messaging/api/processor/StateAndEventProcessor.kt index 9d8ed5c2bc3..59d17937cd0 100644 --- a/libs/messaging/messaging/src/main/kotlin/net/corda/messaging/api/processor/StateAndEventProcessor.kt +++ b/libs/messaging/messaging/src/main/kotlin/net/corda/messaging/api/processor/StateAndEventProcessor.kt @@ -1,5 +1,6 @@ package net.corda.messaging.api.processor +import net.corda.libs.statemanager.api.Metadata import net.corda.messaging.api.records.Record /** @@ -17,6 +18,13 @@ import net.corda.messaging.api.records.Record * [CordaFatalException] and will cause the subscription to close. */ interface StateAndEventProcessor { + /** + * State and metadata stored alongside the state. + */ + data class State( + val value: S?, + val metadata: Metadata?, + ) /** * This class encapsulates the responses that will be returned (from [onNext]) to the subscription for @@ -25,8 +33,12 @@ interface StateAndEventProcessor { data class Response( /** * The updated state in response to an incoming event from [onNext]. + * + * Both the state and its associated metadata will be overwritten in storage by this new state when provided by + * the processor. It is the processor's responsibility to ensure that any required metadata is preserved across + * processing. */ - val updatedState: S?, + val updatedState: State?, /** * A list of events to be published in response to an incoming event from [onNext]. @@ -38,7 +50,7 @@ interface StateAndEventProcessor { /** * Flag to indicate processing failed and the State and Event should be moved to the Dead Letter Queue */ - val markForDLQ: Boolean = false + val markForDLQ: Boolean = false, ) /** @@ -55,7 +67,7 @@ interface StateAndEventProcessor { * NOTE: The returned events will be published and the processed events will be consumed atomically as a * single transaction. */ - fun onNext(state: S?, event: Record): Response + fun onNext(state: State?, event: Record): Response /** * [keyClass], [stateValueClass] and [eventValueClass] to easily get the class types the processor operates upon. diff --git a/testing/message-patterns/src/integrationTest/kotlin/net/corda/messaging/integration/processors/TestStateEventProcessor.kt b/testing/message-patterns/src/integrationTest/kotlin/net/corda/messaging/integration/processors/TestStateEventProcessor.kt index 7418a4d1fc6..7a4ab2bc19e 100644 --- a/testing/message-patterns/src/integrationTest/kotlin/net/corda/messaging/integration/processors/TestStateEventProcessor.kt +++ b/testing/message-patterns/src/integrationTest/kotlin/net/corda/messaging/integration/processors/TestStateEventProcessor.kt @@ -5,6 +5,7 @@ import net.corda.data.demo.DemoStateRecord import net.corda.messaging.api.exception.CordaMessageAPIIntermittentException import net.corda.messaging.api.processor.StateAndEventProcessor import net.corda.messaging.api.processor.StateAndEventProcessor.Response +import net.corda.messaging.api.processor.StateAndEventProcessor.State import net.corda.messaging.api.records.Record import org.slf4j.LoggerFactory import java.util.concurrent.CountDownLatch @@ -32,7 +33,9 @@ class TestStateEventProcessor( get() = DemoRecord::class.java - override fun onNext(state: DemoStateRecord?, event: Record): Response { + override fun onNext( + state: State?, event: Record + ): Response { onNextLatch.countDown() log.info("Received record, ${onNextLatch.count} remaining") @@ -59,6 +62,9 @@ class TestStateEventProcessor( emptyList() } - return Response(newState, outputRecordList) + return Response( + State(newState, metadata = null), + outputRecordList + ) } } diff --git a/testing/message-patterns/src/integrationTest/kotlin/net/corda/messaging/integration/processors/TestStateEventProcessorStrings.kt b/testing/message-patterns/src/integrationTest/kotlin/net/corda/messaging/integration/processors/TestStateEventProcessorStrings.kt index 3735573c117..0a38a112d64 100644 --- a/testing/message-patterns/src/integrationTest/kotlin/net/corda/messaging/integration/processors/TestStateEventProcessorStrings.kt +++ b/testing/message-patterns/src/integrationTest/kotlin/net/corda/messaging/integration/processors/TestStateEventProcessorStrings.kt @@ -3,6 +3,7 @@ package net.corda.messaging.integration.processors import net.corda.messaging.api.exception.CordaMessageAPIIntermittentException import net.corda.messaging.api.processor.StateAndEventProcessor import net.corda.messaging.api.processor.StateAndEventProcessor.Response +import net.corda.messaging.api.processor.StateAndEventProcessor.State import net.corda.messaging.api.records.Record import java.util.concurrent.CountDownLatch @@ -23,7 +24,9 @@ class TestStateEventProcessorStrings( override val eventValueClass: Class get() = String::class.java - override fun onNext(state: String?, event: Record): Response { + override fun onNext( + state: State?, event: Record + ): Response { onNextLatch.countDown() if (delayProcessorOnFirst != null) { @@ -48,6 +51,9 @@ class TestStateEventProcessorStrings( emptyList() } - return Response(newState, outputRecordList) + return Response( + State(newState, metadata = null), + outputRecordList + ) } } diff --git a/testing/p2p/inmemory-messaging-impl/src/integrationTest/kotlin/net/corda/messaging/emulation/subscription/stateandevent/InMemoryStateAndEventSubscriptionIntegrationTests.kt b/testing/p2p/inmemory-messaging-impl/src/integrationTest/kotlin/net/corda/messaging/emulation/subscription/stateandevent/InMemoryStateAndEventSubscriptionIntegrationTests.kt index a805c4dbd4e..7432a043613 100644 --- a/testing/p2p/inmemory-messaging-impl/src/integrationTest/kotlin/net/corda/messaging/emulation/subscription/stateandevent/InMemoryStateAndEventSubscriptionIntegrationTests.kt +++ b/testing/p2p/inmemory-messaging-impl/src/integrationTest/kotlin/net/corda/messaging/emulation/subscription/stateandevent/InMemoryStateAndEventSubscriptionIntegrationTests.kt @@ -63,14 +63,19 @@ class InMemoryStateAndEventSubscriptionIntegrationTests { groupName = "group", ) val processor = object : StateAndEventProcessor { - override fun onNext(state: State?, event: Record): StateAndEventProcessor.Response { + override fun onNext( + state: StateAndEventProcessor.State?, event: Record + ): StateAndEventProcessor.Response { if ((state != null) && (event.value == Event.STOP)) { - states[event.key.type] = state.number + states[event.key.type] = state.value!!.number countDown.countDown() } return if (event.value == Event.CREATE_STATE) { StateAndEventProcessor.Response( - State(event.key.type), + StateAndEventProcessor.State( + State(event.key.type), + metadata = null + ), listOf( Record( subscriptionConfig.eventTopic, @@ -80,7 +85,13 @@ class InMemoryStateAndEventSubscriptionIntegrationTests { ) ) } else { - StateAndEventProcessor.Response(State(event.key.type), emptyList()) + StateAndEventProcessor.Response( + StateAndEventProcessor.State( + State(event.key.type), + metadata = null + ), + emptyList() + ) } } @@ -132,7 +143,9 @@ class InMemoryStateAndEventSubscriptionIntegrationTests { Record(subscriptionConfig.eventTopic, Key(it), Event.SEND_TO_ANOTHER_TOPIC) } val processor = object : StateAndEventProcessor { - override fun onNext(state: State?, event: Record): StateAndEventProcessor.Response { + override fun onNext( + state: StateAndEventProcessor.State?, event: Record + ): StateAndEventProcessor.Response { return StateAndEventProcessor.Response( null, (1..increaseBy).map { @@ -155,7 +168,9 @@ class InMemoryStateAndEventSubscriptionIntegrationTests { subscription.start() val anotherProcessor = object : StateAndEventProcessor { - override fun onNext(state: String?, event: Record): StateAndEventProcessor.Response { + override fun onNext( + state: StateAndEventProcessor.State?, event: Record + ): StateAndEventProcessor.Response { got.add(event) countDown.countDown() return StateAndEventProcessor.Response(null, emptyList()) @@ -216,15 +231,20 @@ class InMemoryStateAndEventSubscriptionIntegrationTests { } val processor = object : StateAndEventProcessor { - override fun onNext(state: State?, event: Record): StateAndEventProcessor.Response { + override fun onNext( + state: StateAndEventProcessor.State?, event: Record + ): StateAndEventProcessor.Response { val newState = when (event.value) { Event.CREATE_STATE -> 1 - Event.INCREASE_STATE -> (state!!.number + 1) + Event.INCREASE_STATE -> (state?.value!!.number + 1) else -> throw Exception("Unexpected event!") } countDown.countDown() return StateAndEventProcessor.Response( - State(newState), + StateAndEventProcessor.State( + State(newState), + metadata = null + ), emptyList() ) } @@ -302,9 +322,14 @@ class InMemoryStateAndEventSubscriptionIntegrationTests { } } val processor = object : StateAndEventProcessor { - override fun onNext(state: State?, event: Record): StateAndEventProcessor.Response { + override fun onNext( + state: StateAndEventProcessor.State?, event: Record + ): StateAndEventProcessor.Response { return StateAndEventProcessor.Response( - State(state?.number?.inc() ?: -1), + StateAndEventProcessor.State( + State(state?.value?.number?.inc() ?: -1), + metadata = null + ), emptyList() ) } diff --git a/testing/p2p/inmemory-messaging-impl/src/main/kotlin/net/corda/messaging/emulation/subscription/stateandevent/EventSubscription.kt b/testing/p2p/inmemory-messaging-impl/src/main/kotlin/net/corda/messaging/emulation/subscription/stateandevent/EventSubscription.kt index 7543f65a160..ce7295f9d8a 100644 --- a/testing/p2p/inmemory-messaging-impl/src/main/kotlin/net/corda/messaging/emulation/subscription/stateandevent/EventSubscription.kt +++ b/testing/p2p/inmemory-messaging-impl/src/main/kotlin/net/corda/messaging/emulation/subscription/stateandevent/EventSubscription.kt @@ -1,6 +1,7 @@ package net.corda.messaging.emulation.subscription.stateandevent import net.corda.lifecycle.Lifecycle +import net.corda.messaging.api.processor.StateAndEventProcessor.State import net.corda.messaging.api.records.Record import net.corda.messaging.emulation.topic.model.Consumption import net.corda.messaging.emulation.topic.model.RecordMetadata @@ -35,19 +36,22 @@ internal class EventSubscription( ) if (event != null) { val state = subscription.stateSubscription.getValue(event.key) - val response = subscription.processor.onNext(state, event) - subscription.setValue(event.key, response.updatedState, eventMetaData.partition) + val response = subscription.processor.onNext( + State(state, metadata = null), + event + ) + subscription.setValue(event.key, response.updatedState?.value, eventMetaData.partition) subscription.topicService.addRecords( listOf( Record( subscription.stateSubscriptionConfig.eventTopic, event.key, - response.updatedState + response.updatedState?.value ) ) + response.responseEvents ) - subscription.stateAndEventListener?.onPostCommit(mapOf(event.key to response.updatedState)) + subscription.stateAndEventListener?.onPostCommit(mapOf(event.key to response.updatedState?.value)) } } } diff --git a/testing/p2p/inmemory-messaging-impl/src/test/kotlin/net/corda/messaging/emulation/subscription/stateandevent/EventSubscriptionTest.kt b/testing/p2p/inmemory-messaging-impl/src/test/kotlin/net/corda/messaging/emulation/subscription/stateandevent/EventSubscriptionTest.kt index ca63b347b28..57126f97dbc 100644 --- a/testing/p2p/inmemory-messaging-impl/src/test/kotlin/net/corda/messaging/emulation/subscription/stateandevent/EventSubscriptionTest.kt +++ b/testing/p2p/inmemory-messaging-impl/src/test/kotlin/net/corda/messaging/emulation/subscription/stateandevent/EventSubscriptionTest.kt @@ -1,6 +1,7 @@ package net.corda.messaging.emulation.subscription.stateandevent import net.corda.messaging.api.processor.StateAndEventProcessor +import net.corda.messaging.api.processor.StateAndEventProcessor.State import net.corda.messaging.api.records.Record import net.corda.messaging.api.subscription.config.SubscriptionConfig import net.corda.messaging.api.subscription.listener.StateAndEventListener @@ -34,9 +35,14 @@ class EventSubscriptionTest { on { stateSubscriptionConfig } doReturn SubscriptionConfig("group1", "topic1") on { stateSubscription } doReturn stateSubscription on { processor } doReturn object : StateAndEventProcessor { - override fun onNext(state: String?, event: Record): StateAndEventProcessor.Response { - received.add(state to event) - return StateAndEventProcessor.Response(newState, response) + override fun onNext( + state: State?, event: Record + ): StateAndEventProcessor.Response { + received.add(state?.value to event) + return StateAndEventProcessor.Response( + State(newState, metadata = null), + response + ) } override val keyClass = String::class.java diff --git a/testing/p2p/inmemory-messaging-impl/src/test/kotlin/net/corda/messaging/emulation/subscription/stateandevent/StateSubscriptionTest.kt b/testing/p2p/inmemory-messaging-impl/src/test/kotlin/net/corda/messaging/emulation/subscription/stateandevent/StateSubscriptionTest.kt index feac82951dd..8bcd0008965 100644 --- a/testing/p2p/inmemory-messaging-impl/src/test/kotlin/net/corda/messaging/emulation/subscription/stateandevent/StateSubscriptionTest.kt +++ b/testing/p2p/inmemory-messaging-impl/src/test/kotlin/net/corda/messaging/emulation/subscription/stateandevent/StateSubscriptionTest.kt @@ -1,6 +1,7 @@ package net.corda.messaging.emulation.subscription.stateandevent import net.corda.messaging.api.processor.StateAndEventProcessor +import net.corda.messaging.api.processor.StateAndEventProcessor.State import net.corda.messaging.api.records.Record import net.corda.messaging.api.subscription.config.SubscriptionConfig import net.corda.messaging.api.subscription.listener.StateAndEventListener @@ -35,7 +36,9 @@ class StateSubscriptionTest { on { subscriptionConfig } doReturn SubscriptionConfig("group", "topic") on { stateSubscriptionConfig } doReturn SubscriptionConfig("group1", "topic1") on { processor } doReturn object : StateAndEventProcessor { - override fun onNext(state: String?, event: Record): StateAndEventProcessor.Response { + override fun onNext( + state: State?, event: Record + ): StateAndEventProcessor.Response { return StateAndEventProcessor.Response(null, emptyList()) } From f02e3c4094dd5d233720f3faad53506c8648abee Mon Sep 17 00:00:00 2001 From: Miljenko Brkic Date: Fri, 13 Oct 2023 03:36:40 +0100 Subject: [PATCH 28/81] CORE-16203 Removed coroutine usage from Multi-Source Event Mediator. --- libs/messaging/messaging-impl/build.gradle | 1 - .../net/corda/messaging/mediator/ClientTask.kt | 9 +++------ .../corda/messaging/mediator/MessageBusClient.kt | 16 ++++------------ .../corda/messaging/mediator/ClientTaskTest.kt | 10 +--------- .../messaging/mediator/MessageBusClientTest.kt | 15 +++++---------- .../mediator/MultiSourceEventMediatorImplTest.kt | 4 +--- libs/messaging/messaging/build.gradle | 1 - .../messaging/api/mediator/MessagingClient.kt | 9 +++------ 8 files changed, 17 insertions(+), 48 deletions(-) diff --git a/libs/messaging/messaging-impl/build.gradle b/libs/messaging/messaging-impl/build.gradle index b4178896dcb..845cc242509 100644 --- a/libs/messaging/messaging-impl/build.gradle +++ b/libs/messaging/messaging-impl/build.gradle @@ -12,7 +12,6 @@ dependencies { implementation project(":libs:chunking:chunking-core") implementation project(":libs:crypto:cipher-suite") implementation project(":libs:crypto:crypto-core") - implementation project(path: ':libs:kotlin-coroutines', configuration: 'bundle') implementation project(":libs:lifecycle:lifecycle") implementation project(":libs:messaging:messaging") implementation project(":libs:messaging:message-bus") diff --git a/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/ClientTask.kt b/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/ClientTask.kt index c93138ee1a3..8182934b459 100644 --- a/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/ClientTask.kt +++ b/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/ClientTask.kt @@ -1,6 +1,5 @@ package net.corda.messaging.mediator -import kotlinx.coroutines.runBlocking import net.corda.messaging.api.mediator.MediatorMessage import net.corda.messaging.api.mediator.MessageRouter import net.corda.messaging.api.mediator.MessagingClient @@ -30,11 +29,9 @@ data class ClientTask( val destination = messageRouter.getDestination(message) @Suppress("UNCHECKED_CAST") - val reply = runBlocking { - with(destination) { - message.addProperty(MSG_PROP_ENDPOINT, endpoint) - client.send(message).await() as MediatorMessage? - } + val reply = with(destination) { + message.addProperty(MSG_PROP_ENDPOINT, endpoint) + client.send(message) as MediatorMessage? } return Result(this, reply) } diff --git a/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/MessageBusClient.kt b/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/MessageBusClient.kt index 66338f569c0..116ca1579dd 100644 --- a/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/MessageBusClient.kt +++ b/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/MessageBusClient.kt @@ -1,7 +1,5 @@ package net.corda.messaging.mediator -import kotlinx.coroutines.CompletableDeferred -import kotlinx.coroutines.Deferred import net.corda.messagebus.api.producer.CordaProducer import net.corda.messagebus.api.producer.CordaProducerRecord import net.corda.messaging.api.mediator.MediatorMessage @@ -20,16 +18,10 @@ class MessageBusClient( private val log: Logger = LoggerFactory.getLogger(this::class.java.enclosingClass) } - override fun send(message: MediatorMessage<*>): Deferred?> = - CompletableDeferred?>().apply { - producer.send(message.toCordaProducerRecord()) { ex -> - if (ex != null) { - completeExceptionally(ex) - } else { - complete(null) - } - } - } + override fun send(message: MediatorMessage<*>): MediatorMessage<*>? { + producer.send(message.toCordaProducerRecord(), null) + return null + } override fun close() { try { diff --git a/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/ClientTaskTest.kt b/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/ClientTaskTest.kt index d28ef8b4b45..d97beeffe6a 100644 --- a/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/ClientTaskTest.kt +++ b/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/ClientTaskTest.kt @@ -1,7 +1,5 @@ package net.corda.messaging.mediator -import kotlinx.coroutines.Deferred -import kotlinx.coroutines.runBlocking import net.corda.messaging.api.mediator.MediatorMessage import net.corda.messaging.api.mediator.MessageRouter import net.corda.messaging.api.mediator.MessagingClient @@ -25,7 +23,6 @@ class ClientTaskTest { private val messageRouter = mock() private val routingDestination = mock() private val messagingClient = mock() - private val clientDeferredReply = mock>>() private val clientReply = mock>() @BeforeEach @@ -37,13 +34,8 @@ class ClientTaskTest { messagingClient ) `when`(messagingClient.send(any())).thenReturn( - clientDeferredReply + clientReply ) - runBlocking { - `when`(clientDeferredReply.await()).thenReturn( - clientReply - ) - } } @Test diff --git a/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/MessageBusClientTest.kt b/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/MessageBusClientTest.kt index 09ae5ee8869..58559d25ba4 100644 --- a/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/MessageBusClientTest.kt +++ b/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/MessageBusClientTest.kt @@ -1,6 +1,5 @@ package net.corda.messaging.mediator -import kotlinx.coroutines.runBlocking import net.corda.messagebus.api.producer.CordaProducer import net.corda.messagebus.api.producer.CordaProducerRecord import net.corda.messaging.api.mediator.MediatorMessage @@ -9,9 +8,10 @@ import net.corda.v5.base.exceptions.CordaRuntimeException import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Test import org.junit.jupiter.api.assertThrows +import org.mockito.Mockito import org.mockito.Mockito.times -import org.mockito.kotlin.any import org.mockito.kotlin.eq +import org.mockito.kotlin.isNull import org.mockito.kotlin.mock import org.mockito.kotlin.verify import org.mockito.kotlin.whenever @@ -50,7 +50,7 @@ class MessageBusClientTest { messageProps.toHeaders(), ) - verify(cordaProducer).send(eq(expected), any()) + verify(cordaProducer).send(eq(expected), isNull()) } @Test @@ -62,14 +62,9 @@ class MessageBusClientTest { messageProps.toHeaders(), ) - whenever(cordaProducer.send(eq(record), any())).thenAnswer { invocation -> - val callback = invocation.getArgument(1) - callback.onCompletion(CordaRuntimeException("")) - } + Mockito.doThrow(CordaRuntimeException("")).whenever(cordaProducer).send(eq(record), isNull()) assertThrows { - runBlocking { - messageBusClient.send(message).await() - } + messageBusClient.send(message) } } diff --git a/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/MultiSourceEventMediatorImplTest.kt b/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/MultiSourceEventMediatorImplTest.kt index db31cd73257..acd83cafaad 100644 --- a/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/MultiSourceEventMediatorImplTest.kt +++ b/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/MultiSourceEventMediatorImplTest.kt @@ -1,6 +1,5 @@ package net.corda.messaging.mediator -import kotlinx.coroutines.CompletableDeferred import net.corda.avro.serialization.CordaAvroDeserializer import net.corda.avro.serialization.CordaAvroSerializer import net.corda.libs.statemanager.api.StateManager @@ -8,7 +7,6 @@ import net.corda.lifecycle.LifecycleCoordinatorFactory import net.corda.messagebus.api.CordaTopicPartition import net.corda.messagebus.api.consumer.CordaConsumerRecord import net.corda.messaging.api.mediator.MediatorConsumer -import net.corda.messaging.api.mediator.MediatorMessage import net.corda.messaging.api.mediator.MessageRouter import net.corda.messaging.api.mediator.MessagingClient import net.corda.messaging.api.mediator.MultiSourceEventMediator @@ -63,7 +61,7 @@ class MultiSourceEventMediatorImplTest { whenever(mediatorConsumerFactory.create(any>())).thenReturn(consumer) whenever(messagingClient.send(any())).thenAnswer { - CompletableDeferred(null as MediatorMessage?) + null } whenever(messagingClientFactory.create(any())).thenReturn(messagingClient) diff --git a/libs/messaging/messaging/build.gradle b/libs/messaging/messaging/build.gradle index 728525d2f30..5f5e9aa8330 100644 --- a/libs/messaging/messaging/build.gradle +++ b/libs/messaging/messaging/build.gradle @@ -16,7 +16,6 @@ dependencies { implementation "net.corda:corda-base" implementation "net.corda:corda-config-schema" implementation project(":libs:chunking:chunking-core") - implementation project(path: ':libs:kotlin-coroutines', configuration: 'bundle') implementation project(":libs:lifecycle:lifecycle") implementation project(":libs:messaging:message-bus") implementation project(":libs:configuration:configuration-core") diff --git a/libs/messaging/messaging/src/main/kotlin/net/corda/messaging/api/mediator/MessagingClient.kt b/libs/messaging/messaging/src/main/kotlin/net/corda/messaging/api/mediator/MessagingClient.kt index 71ea0f32f24..c148e4c6b4d 100644 --- a/libs/messaging/messaging/src/main/kotlin/net/corda/messaging/api/mediator/MessagingClient.kt +++ b/libs/messaging/messaging/src/main/kotlin/net/corda/messaging/api/mediator/MessagingClient.kt @@ -1,7 +1,5 @@ package net.corda.messaging.api.mediator -import kotlinx.coroutines.Deferred - /** * Multi-source event mediator messaging client. */ @@ -20,11 +18,10 @@ interface MessagingClient : AutoCloseable { val id: String /** - * Asynchronously sends a generic [MediatorMessage], and returns any result/error through a [Deferred] response. + * Sends a generic [MediatorMessage] and returns any result/error through a response. * * @param message The [MediatorMessage] to send. - * @return [Deferred] instance representing the asynchronous computation result, or null if the destination doesn't - * provide a response. + * @return Computation result, or null if the destination doesn't provide a response. * */ - fun send(message: MediatorMessage<*>): Deferred?> + fun send(message: MediatorMessage<*>): MediatorMessage<*>? } From 867ec061884b4b4e11c96aded927044111d37ae1 Mon Sep 17 00:00:00 2001 From: Miljenko Brkic <97448832+mbrkic-r3@users.noreply.github.com> Date: Thu, 12 Oct 2023 09:27:13 +0100 Subject: [PATCH 29/81] CORE-16203 Replace State and Event pattern with Multi-Source Event Mediator in FlowWorker (#4832) Using Multi-Source Event Mediator instead of State and Event Subscription in FlowWorker. Polling consumers and committing offsets performed in mediator thread in a synchronous way (without using coroutines) --- .../mediator/FlowEventMediatorFactoryImpl.kt | 3 ++ .../corda/flow/service/FlowExecutorImpl.kt | 40 ++++++--------- .../flow/service/FlowExecutorImplTest.kt | 50 ++++++++----------- .../db/consumer/DBCordaConsumerImpl.kt | 29 +++++------ .../kafka/consumer/CordaKafkaConsumerImpl.kt | 33 +++++++++--- .../consumer/CordaKafkaConsumerImplTest.kt | 26 +++------- .../messagebus/api/consumer/CordaConsumer.kt | 4 +- .../messaging/mediator/MessageBusConsumer.kt | 30 ++--------- .../mediator/MultiSourceEventMediatorImpl.kt | 16 +----- .../mediator/taskmanager/TaskManagerImpl.kt | 21 ++++++-- .../mediator/MessageBusConsumerTest.kt | 22 +++----- .../MultiSourceEventMediatorImplTest.kt | 23 +++------ .../mediator/TaskManagerHelperTest.kt | 4 +- .../api/mediator/MediatorConsumer.kt | 10 ++-- 14 files changed, 131 insertions(+), 180 deletions(-) diff --git a/components/flow/flow-service/src/main/kotlin/net/corda/flow/messaging/mediator/FlowEventMediatorFactoryImpl.kt b/components/flow/flow-service/src/main/kotlin/net/corda/flow/messaging/mediator/FlowEventMediatorFactoryImpl.kt index 6b90dd93361..5408e207373 100644 --- a/components/flow/flow-service/src/main/kotlin/net/corda/flow/messaging/mediator/FlowEventMediatorFactoryImpl.kt +++ b/components/flow/flow-service/src/main/kotlin/net/corda/flow/messaging/mediator/FlowEventMediatorFactoryImpl.kt @@ -7,6 +7,7 @@ import net.corda.data.flow.event.mapper.FlowMapperEvent import net.corda.data.flow.output.FlowStatus import net.corda.data.flow.state.checkpoint.Checkpoint import net.corda.data.ledger.persistence.LedgerPersistenceRequest +import net.corda.data.ledger.utxo.token.selection.event.TokenPoolCacheEvent import net.corda.data.persistence.EntityRequest import net.corda.data.uniqueness.UniquenessCheckRequestAvro import net.corda.flow.pipeline.factory.FlowEventProcessorFactory @@ -27,6 +28,7 @@ import net.corda.schema.Schemas.Flow.FLOW_MAPPER_EVENT_TOPIC import net.corda.schema.Schemas.Flow.FLOW_STATUS_TOPIC import net.corda.schema.Schemas.Persistence.PERSISTENCE_ENTITY_PROCESSOR_TOPIC import net.corda.schema.Schemas.Persistence.PERSISTENCE_LEDGER_PROCESSOR_TOPIC +import net.corda.schema.Schemas.Services.TOKEN_CACHE_EVENT import net.corda.schema.Schemas.UniquenessChecker.UNIQUENESS_CHECK_TOPIC import net.corda.schema.Schemas.Verification.VERIFICATION_LEDGER_PROCESSOR_TOPIC import org.osgi.service.component.annotations.Activate @@ -94,6 +96,7 @@ class FlowEventMediatorFactoryImpl @Activate constructor( is FlowOpsRequest -> routeTo(messageBusClient, FLOW_OPS_MESSAGE_TOPIC) is FlowStatus -> routeTo(messageBusClient, FLOW_STATUS_TOPIC) is LedgerPersistenceRequest -> routeTo(messageBusClient, PERSISTENCE_LEDGER_PROCESSOR_TOPIC) + is TokenPoolCacheEvent -> routeTo(messageBusClient, TOKEN_CACHE_EVENT) is TransactionVerificationRequest -> routeTo(messageBusClient, VERIFICATION_LEDGER_PROCESSOR_TOPIC) is UniquenessCheckRequestAvro -> routeTo(messageBusClient, UNIQUENESS_CHECK_TOPIC) else -> { diff --git a/components/flow/flow-service/src/main/kotlin/net/corda/flow/service/FlowExecutorImpl.kt b/components/flow/flow-service/src/main/kotlin/net/corda/flow/service/FlowExecutorImpl.kt index cb2dab9393b..0bef3fa8ef9 100644 --- a/components/flow/flow-service/src/main/kotlin/net/corda/flow/service/FlowExecutorImpl.kt +++ b/components/flow/flow-service/src/main/kotlin/net/corda/flow/service/FlowExecutorImpl.kt @@ -3,7 +3,7 @@ package net.corda.flow.service import com.typesafe.config.ConfigValueFactory import net.corda.data.flow.event.FlowEvent import net.corda.data.flow.state.checkpoint.Checkpoint -import net.corda.flow.pipeline.factory.FlowEventProcessorFactory +import net.corda.flow.messaging.mediator.FlowEventMediatorFactory import net.corda.libs.configuration.SmartConfig import net.corda.libs.configuration.helper.getConfig import net.corda.lifecycle.LifecycleCoordinatorFactory @@ -13,10 +13,7 @@ import net.corda.lifecycle.RegistrationHandle import net.corda.lifecycle.StartEvent import net.corda.lifecycle.StopEvent import net.corda.lifecycle.createCoordinator -import net.corda.messaging.api.subscription.StateAndEventSubscription -import net.corda.messaging.api.subscription.config.SubscriptionConfig -import net.corda.messaging.api.subscription.factory.SubscriptionFactory -import net.corda.schema.Schemas.Flow.FLOW_EVENT_TOPIC +import net.corda.messaging.api.mediator.MultiSourceEventMediator import net.corda.schema.configuration.BootConfig.CRYPTO_WORKER_REST_ENDPOINT import net.corda.schema.configuration.BootConfig.PERSISTENCE_WORKER_REST_ENDPOINT import net.corda.schema.configuration.BootConfig.UNIQUENESS_WORKER_REST_ENDPOINT @@ -36,34 +33,29 @@ import org.slf4j.LoggerFactory @Component(service = [FlowExecutor::class]) class FlowExecutorImpl constructor( coordinatorFactory: LifecycleCoordinatorFactory, - private val subscriptionFactory: SubscriptionFactory, - private val flowEventProcessorFactory: FlowEventProcessorFactory, - private val toMessagingConfig: (Map) -> SmartConfig + private val flowEventMediatorFactory: FlowEventMediatorFactory, + private val toMessagingConfig: (Map) -> SmartConfig, ) : FlowExecutor { @Activate constructor( @Reference(service = LifecycleCoordinatorFactory::class) coordinatorFactory: LifecycleCoordinatorFactory, - @Reference(service = SubscriptionFactory::class) - subscriptionFactory: SubscriptionFactory, - @Reference(service = FlowEventProcessorFactory::class) - flowEventProcessorFactory: FlowEventProcessorFactory + @Reference(service = FlowEventMediatorFactory::class) + flowEventMediatorFactory: FlowEventMediatorFactory, ) : this( coordinatorFactory, - subscriptionFactory, - flowEventProcessorFactory, + flowEventMediatorFactory, { cfg -> cfg.getConfig(MESSAGING_CONFIG) } ) companion object { private val log = LoggerFactory.getLogger(this::class.java.enclosingClass) - private const val CONSUMER_GROUP = "FlowEventConsumer" } private val coordinator = coordinatorFactory.createCoordinator { event, _ -> eventHandler(event) } - private var subscription: StateAndEventSubscription? = null private var subscriptionRegistrationHandle: RegistrationHandle? = null + private var multiSourceEventMediator: MultiSourceEventMediator? = null override fun onConfigChange(config: Map) { try { @@ -72,19 +64,18 @@ class FlowExecutorImpl constructor( // close the lifecycle registration first to prevent down being signaled subscriptionRegistrationHandle?.close() - subscription?.close() + multiSourceEventMediator?.close() - subscription = subscriptionFactory.createStateAndEventSubscription( - SubscriptionConfig(CONSUMER_GROUP, FLOW_EVENT_TOPIC), - flowEventProcessorFactory.create(updatedConfigs), - messagingConfig + multiSourceEventMediator = flowEventMediatorFactory.create( + updatedConfigs, + messagingConfig, ) subscriptionRegistrationHandle = coordinator.followStatusChangesByName( - setOf(subscription!!.subscriptionName) + setOf(multiSourceEventMediator!!.subscriptionName) ) - subscription?.start() + multiSourceEventMediator?.start() } catch (ex: Exception) { val reason = "Failed to configure the flow executor using '${config}'" log.error(reason, ex) @@ -126,10 +117,11 @@ class FlowExecutorImpl constructor( is StartEvent -> { coordinator.updateStatus(LifecycleStatus.UP) } + is StopEvent -> { log.trace { "Flow executor is stopping..." } subscriptionRegistrationHandle?.close() - subscription?.close() + multiSourceEventMediator?.close() log.trace { "Flow executor stopped" } } } diff --git a/components/flow/flow-service/src/test/kotlin/net/corda/flow/service/FlowExecutorImplTest.kt b/components/flow/flow-service/src/test/kotlin/net/corda/flow/service/FlowExecutorImplTest.kt index 6a8aad3e000..ab0f58959eb 100644 --- a/components/flow/flow-service/src/test/kotlin/net/corda/flow/service/FlowExecutorImplTest.kt +++ b/components/flow/flow-service/src/test/kotlin/net/corda/flow/service/FlowExecutorImplTest.kt @@ -3,6 +3,7 @@ package net.corda.flow.service import com.typesafe.config.ConfigValueFactory import net.corda.data.flow.event.FlowEvent import net.corda.data.flow.state.checkpoint.Checkpoint +import net.corda.flow.messaging.mediator.FlowEventMediatorFactory import net.corda.flow.pipeline.factory.FlowEventProcessorFactory import net.corda.libs.configuration.SmartConfig import net.corda.libs.configuration.SmartConfigImpl @@ -13,9 +14,8 @@ import net.corda.lifecycle.LifecycleEventHandler import net.corda.lifecycle.LifecycleStatus import net.corda.lifecycle.RegistrationHandle import net.corda.lifecycle.StopEvent +import net.corda.messaging.api.mediator.MultiSourceEventMediator import net.corda.messaging.api.processor.StateAndEventProcessor -import net.corda.messaging.api.subscription.StateAndEventSubscription -import net.corda.messaging.api.subscription.factory.SubscriptionFactory import net.corda.schema.configuration.BootConfig import net.corda.schema.configuration.ConfigKeys.BOOT_CONFIG import net.corda.schema.configuration.ConfigKeys.FLOW_CONFIG @@ -26,7 +26,6 @@ import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Test import org.mockito.Mockito.inOrder import org.mockito.kotlin.any -import org.mockito.kotlin.anyOrNull import org.mockito.kotlin.argumentCaptor import org.mockito.kotlin.eq import org.mockito.kotlin.mock @@ -37,7 +36,7 @@ class FlowExecutorImplTest { private val coordinatorFactory = mock() private val flowEventProcessorFactory = mock() - private val subscriptionFactory = mock() + private val flowEventMediatorFactory = mock() private val toMessagingConfig: (Map) -> SmartConfig = { messagingConfig } @@ -49,20 +48,18 @@ class FlowExecutorImplTest { private val messagingConfig = getMinimalMessagingConfig() private val subscriptionRegistrationHandle = mock() private val flowExecutorCoordinator = mock() - private val subscription = mock>() + private val multiSourceEventMediator = mock>() private val flowEventProcessor = mock>() @BeforeEach fun setup() { whenever(flowEventProcessorFactory.create(any())).thenReturn(flowEventProcessor) whenever( - subscriptionFactory.createStateAndEventSubscription( + flowEventMediatorFactory.create( any(), any(), - any(), - anyOrNull() ) - ).thenReturn(subscription) + ).thenReturn(multiSourceEventMediator) whenever(coordinatorFactory.createCoordinator(any(), any())).thenReturn(flowExecutorCoordinator) whenever(flowExecutorCoordinator.followStatusChangesByName(any())).thenReturn(subscriptionRegistrationHandle) @@ -76,7 +73,7 @@ class FlowExecutorImplTest { } @Test - fun `lifecycle - flow executor signals error if it fails to create a subscription`() { + fun `lifecycle - flow executor signals error if it fails to create event mediator`() { val invalidConfig = mapOf() val flowExecutor = getFlowExecutor() @@ -89,9 +86,9 @@ class FlowExecutorImplTest { } @Test - fun `lifecycle - flow executor signals error if the subscription signals error`() { + fun `lifecycle - flow executor signals error if event mediator signals error`() { val name = LifecycleCoordinatorName("", "") - whenever(subscription.subscriptionName).thenReturn(name) + whenever(multiSourceEventMediator.subscriptionName).thenReturn(name) val flowExecutor = getFlowExecutor() flowExecutor.start() @@ -108,7 +105,7 @@ class FlowExecutorImplTest { } @Test - fun `lifecycle - flow executor stops subscription when stopped`() { + fun `lifecycle - flow executor stops event mediator when stopped`() { val flowExecutor = getFlowExecutor() flowExecutor.onConfigChange(config) @@ -119,7 +116,7 @@ class FlowExecutorImplTest { } verify(subscriptionRegistrationHandle).close() - verify(subscription).close() + verify(multiSourceEventMediator).close() } @Test @@ -127,10 +124,10 @@ class FlowExecutorImplTest { val name1 = LifecycleCoordinatorName("", "") val name2 = LifecycleCoordinatorName("", "") val subscriptionRegistrationHandle2 = mock() - val subscription2 = mock>() + val multiSourceEventMediator2 = mock>() - whenever(subscription.subscriptionName).thenReturn(name1) - whenever(subscription2.subscriptionName).thenReturn(name2) + whenever(multiSourceEventMediator.subscriptionName).thenReturn(name1) + whenever(multiSourceEventMediator2.subscriptionName).thenReturn(name2) // First config change gets us subscribed val flowExecutor = getFlowExecutor() @@ -140,29 +137,27 @@ class FlowExecutorImplTest { // now we change config and should see the subscription registration removed, // the subscription re-created and then the subscription registered again whenever( - subscriptionFactory.createStateAndEventSubscription( - any(), + flowEventMediatorFactory.create( any(), any(), - anyOrNull() ) - ).thenReturn(subscription2) + ).thenReturn(multiSourceEventMediator2) whenever(flowExecutorCoordinator.followStatusChangesByName(any())).thenReturn(subscriptionRegistrationHandle2) flowExecutor.onConfigChange(config) inOrder( - subscription, - subscription2, + multiSourceEventMediator, + multiSourceEventMediator2, subscriptionRegistrationHandle, subscriptionRegistrationHandle2, flowExecutorCoordinator ).apply { verify(subscriptionRegistrationHandle).close() - verify(subscription).close() + verify(multiSourceEventMediator).close() verify(flowExecutorCoordinator).followStatusChangesByName(eq(setOf(name2))) - verify(subscription2).start() + verify(multiSourceEventMediator2).start() } } @@ -175,13 +170,12 @@ class FlowExecutorImplTest { private fun getFlowExecutor(): FlowExecutorImpl { return FlowExecutorImpl( coordinatorFactory, - subscriptionFactory, - flowEventProcessorFactory, + flowEventMediatorFactory, toMessagingConfig ) } - private fun getMinimalMessagingConfig() : SmartConfig { + private fun getMinimalMessagingConfig(): SmartConfig { return SmartConfigImpl.empty() .withValue(PROCESSOR_TIMEOUT, ConfigValueFactory.fromAnyRef(5000)) .withValue(MAX_ALLOWED_MSG_SIZE, ConfigValueFactory.fromAnyRef(1000000000)) diff --git a/libs/messaging/db-message-bus-impl/src/main/kotlin/net/corda/messagebus/db/consumer/DBCordaConsumerImpl.kt b/libs/messaging/db-message-bus-impl/src/main/kotlin/net/corda/messagebus/db/consumer/DBCordaConsumerImpl.kt index 937b1415199..462ab0990e2 100644 --- a/libs/messaging/db-message-bus-impl/src/main/kotlin/net/corda/messagebus/db/consumer/DBCordaConsumerImpl.kt +++ b/libs/messaging/db-message-bus-impl/src/main/kotlin/net/corda/messagebus/db/consumer/DBCordaConsumerImpl.kt @@ -203,23 +203,18 @@ internal class DBCordaConsumerImpl constructor( } } - override fun asyncCommitOffsets(callback: CordaConsumer.Callback?) { - try { - dbAccess.writeOffsets( - lastReadOffset.map { (cordaTopicPartition, offset) -> - CommittedPositionEntry( - cordaTopicPartition.topic, - groupId, - cordaTopicPartition.partition, - offset, - ATOMIC_TRANSACTION, - ) - } - ) - callback?.onCompletion(lastReadOffset, null) - } catch(e: Exception) { - callback?.onCompletion(emptyMap(), e) - } + override fun syncCommitOffsets() { + dbAccess.writeOffsets( + lastReadOffset.map { (cordaTopicPartition, offset) -> + CommittedPositionEntry( + cordaTopicPartition.topic, + groupId, + cordaTopicPartition.partition, + offset, + ATOMIC_TRANSACTION, + ) + } + ) } override fun syncCommitOffsets(event: CordaConsumerRecord, metaData: String?) { diff --git a/libs/messaging/kafka-message-bus-impl/src/main/kotlin/net/corda/messagebus/kafka/consumer/CordaKafkaConsumerImpl.kt b/libs/messaging/kafka-message-bus-impl/src/main/kotlin/net/corda/messagebus/kafka/consumer/CordaKafkaConsumerImpl.kt index 7ccfc2e56a7..1432c61e4c2 100644 --- a/libs/messaging/kafka-message-bus-impl/src/main/kotlin/net/corda/messagebus/kafka/consumer/CordaKafkaConsumerImpl.kt +++ b/libs/messaging/kafka-message-bus-impl/src/main/kotlin/net/corda/messagebus/kafka/consumer/CordaKafkaConsumerImpl.kt @@ -304,14 +304,31 @@ class CordaKafkaConsumerImpl( } } - override fun asyncCommitOffsets(callback: CordaConsumer.Callback?) { - consumer.commitAsync { offsets, exception -> - callback?.onCompletion( - offsets.entries.associate { - it.key!!.toCordaTopicPartition(config.topicPrefix) to it.value.offset() - }, - exception - ) + override fun syncCommitOffsets() { + var attemptCommit = true + + while (attemptCommit) { + try { + consumer.commitSync() + attemptCommit = false + } catch (ex: Exception) { + when (ex::class.java) { + in fatalExceptions -> { + logErrorAndThrowFatalException( + "Error attempting to commitSync offsets.", + ex + ) + } + in transientExceptions -> { + logWarningAndThrowIntermittentException("Failed to commitSync offsets.", ex) + } + else -> { + logErrorAndThrowFatalException( + "Unexpected error attempting to commitSync offsets .", ex + ) + } + } + } } } diff --git a/libs/messaging/kafka-message-bus-impl/src/test/kotlin/net/corda/messagebus/kafka/consumer/CordaKafkaConsumerImplTest.kt b/libs/messaging/kafka-message-bus-impl/src/test/kotlin/net/corda/messagebus/kafka/consumer/CordaKafkaConsumerImplTest.kt index 84a4563bbc0..17160d98864 100644 --- a/libs/messaging/kafka-message-bus-impl/src/test/kotlin/net/corda/messagebus/kafka/consumer/CordaKafkaConsumerImplTest.kt +++ b/libs/messaging/kafka-message-bus-impl/src/test/kotlin/net/corda/messagebus/kafka/consumer/CordaKafkaConsumerImplTest.kt @@ -4,7 +4,6 @@ import io.micrometer.core.instrument.binder.kafka.KafkaClientMetrics import net.corda.data.chunking.Chunk import net.corda.data.chunking.ChunkKey import net.corda.messagebus.api.CordaTopicPartition -import net.corda.messagebus.api.consumer.CordaConsumer import net.corda.messagebus.api.consumer.CordaConsumerRebalanceListener import net.corda.messagebus.api.consumer.CordaConsumerRecord import net.corda.messagebus.api.consumer.CordaOffsetResetStrategy @@ -25,7 +24,6 @@ import org.apache.kafka.clients.consumer.Consumer import org.apache.kafka.clients.consumer.ConsumerRebalanceListener import org.apache.kafka.clients.consumer.MockConsumer import org.apache.kafka.clients.consumer.OffsetAndMetadata -import org.apache.kafka.clients.consumer.OffsetCommitCallback import org.apache.kafka.common.KafkaException import org.apache.kafka.common.TopicPartition import org.apache.kafka.common.errors.AuthenticationException @@ -45,7 +43,6 @@ import org.mockito.kotlin.anyOrNull import org.mockito.kotlin.doAnswer import org.mockito.kotlin.doReturn import org.mockito.kotlin.doThrow -import org.mockito.kotlin.eq import org.mockito.kotlin.mock import org.mockito.kotlin.times import org.mockito.kotlin.verify @@ -180,33 +177,26 @@ class CordaKafkaConsumerImplTest { } @Test - fun testAsyncCommitOffsets() { - val callback = mock() + fun testSyncCommitOffsets() { assertThat(consumer.committed(setOf(partition))).isEmpty() cordaKafkaConsumer.poll(Duration.ZERO) - cordaKafkaConsumer.asyncCommitOffsets(callback) + cordaKafkaConsumer.syncCommitOffsets() val committedPositionAfterPoll = consumer.committed(setOf(partition)) assertThat(committedPositionAfterPoll.values.first().offset()).isEqualTo(numberOfRecords) } @Test - fun testAsyncCommitOffsetsException() { + fun testSyncCommitOffsetsException() { consumer = mock() cordaKafkaConsumer = createConsumer(consumer) - val exception = CommitFailedException() - doAnswer { - val callback = it.arguments[0] as OffsetCommitCallback - callback.onComplete(mock(), exception) - null - }.whenever(consumer).commitAsync(any()) - val callback = mock() - - cordaKafkaConsumer.asyncCommitOffsets(callback) - verify(consumer, times(1)).commitAsync(any()) - verify(callback, times(1)).onCompletion(any(), eq(exception)) + doThrow(CommitFailedException()).whenever(consumer).commitSync() + assertThatExceptionOfType(CordaMessageAPIFatalException::class.java).isThrownBy { + cordaKafkaConsumer.syncCommitOffsets() + } + verify(consumer, times(1)).commitSync() } @Test diff --git a/libs/messaging/message-bus/src/main/kotlin/net/corda/messagebus/api/consumer/CordaConsumer.kt b/libs/messaging/message-bus/src/main/kotlin/net/corda/messagebus/api/consumer/CordaConsumer.kt index 2269483455b..3a686ea9723 100644 --- a/libs/messaging/message-bus/src/main/kotlin/net/corda/messagebus/api/consumer/CordaConsumer.kt +++ b/libs/messaging/message-bus/src/main/kotlin/net/corda/messagebus/api/consumer/CordaConsumer.kt @@ -146,10 +146,10 @@ interface CordaConsumer : AutoCloseable { fun resetToLastCommittedPositions(offsetStrategy: CordaOffsetResetStrategy) /** - * Asynchronously commit the consumer offsets. + * Synchronously commit the consumer offsets. * @throws CordaMessageAPIFatalException fatal error occurred attempting to commit offsets. */ - fun asyncCommitOffsets(callback: Callback?) + fun syncCommitOffsets() /** * Synchronously commit the consumer offset for this [event] back to the topic partition. diff --git a/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/MessageBusConsumer.kt b/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/MessageBusConsumer.kt index 04b4258ed70..4176b4f5e85 100644 --- a/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/MessageBusConsumer.kt +++ b/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/MessageBusConsumer.kt @@ -1,8 +1,5 @@ package net.corda.messaging.mediator -import kotlinx.coroutines.CompletableDeferred -import kotlinx.coroutines.Deferred -import net.corda.messagebus.api.CordaTopicPartition import net.corda.messagebus.api.consumer.CordaConsumer import net.corda.messagebus.api.consumer.CordaConsumerRecord import net.corda.messagebus.api.consumer.CordaOffsetResetStrategy @@ -16,33 +13,14 @@ class MessageBusConsumer( private val topic: String, private val consumer: CordaConsumer, ): MediatorConsumer { + override fun subscribe() = consumer.subscribe(topic) - override fun subscribe() = - consumer.subscribe(topic) + override fun poll(timeout: Duration): List> = consumer.poll(timeout) - override fun poll(timeout: Duration): Deferred>> = - CompletableDeferred>>().apply { - try { - complete(consumer.poll(timeout)) - } catch (throwable: Throwable) { - completeExceptionally(throwable) - } - } - - override fun asyncCommitOffsets(): Deferred> = - CompletableDeferred>().apply { - consumer.asyncCommitOffsets { offsets, exception -> - if (exception != null) { - completeExceptionally(exception) - } else { - complete(offsets) - } - } - } + override fun syncCommitOffsets() = consumer.syncCommitOffsets() override fun resetEventOffsetPosition() = consumer.resetToLastCommittedPositions(CordaOffsetResetStrategy.EARLIEST) - override fun close() = - consumer.close() + override fun close() = consumer.close() } \ No newline at end of file diff --git a/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/MultiSourceEventMediatorImpl.kt b/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/MultiSourceEventMediatorImpl.kt index 32023c08925..0262d264f98 100644 --- a/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/MultiSourceEventMediatorImpl.kt +++ b/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/MultiSourceEventMediatorImpl.kt @@ -1,6 +1,5 @@ package net.corda.messaging.mediator -import kotlinx.coroutines.runBlocking import net.corda.avro.serialization.CordaAvroDeserializer import net.corda.avro.serialization.CordaAvroSerializer import net.corda.libs.statemanager.api.StateManager @@ -151,7 +150,6 @@ class MultiSourceEventMediatorImpl( } private fun processEvents() { - log.debug { "Polling and processing events" } val messages = pollConsumers() if (messages.isNotEmpty()) { val msgGroups = messages.groupBy { it.key } @@ -177,23 +175,13 @@ class MultiSourceEventMediatorImpl( private fun pollConsumers(): List> { return consumers.map { consumer -> - taskManager.execute(TaskType.SHORT_RUNNING) { - runBlocking { - consumer.poll(config.pollTimeout).await() - } - } - }.map { - it.join() + consumer.poll(config.pollTimeout) }.flatten() } private fun commitOffsets() { consumers.map { consumer -> - taskManager.execute(TaskType.SHORT_RUNNING) { - runBlocking { - consumer.asyncCommitOffsets().await() - } - } + consumer.syncCommitOffsets() } } diff --git a/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/taskmanager/TaskManagerImpl.kt b/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/taskmanager/TaskManagerImpl.kt index dec4a07876a..305bfb0e752 100644 --- a/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/taskmanager/TaskManagerImpl.kt +++ b/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/taskmanager/TaskManagerImpl.kt @@ -4,6 +4,7 @@ import net.corda.messaging.api.mediator.taskmanager.TaskManager import net.corda.messaging.api.mediator.taskmanager.TaskType import org.osgi.service.component.annotations.Activate import org.osgi.service.component.annotations.Component +import org.slf4j.LoggerFactory import java.util.UUID import java.util.concurrent.CompletableFuture import java.util.concurrent.Executors @@ -12,6 +13,9 @@ import kotlin.concurrent.thread // TODO This is used temporarily until Task Manager implementation is finished @Component(service = [TaskManager::class]) class TaskManagerImpl @Activate constructor() : TaskManager { + companion object { + private val log = LoggerFactory.getLogger(this::class.java.enclosingClass) + } private var executorService = Executors.newSingleThreadExecutor() override fun execute(type: TaskType, command: () -> T) = @@ -21,11 +25,20 @@ class TaskManagerImpl @Activate constructor() : TaskManager { } private fun executeShortRunning(command: () -> T): CompletableFuture { - val result = CompletableFuture() - executorService.execute { - result.complete(command()) + val resultFuture = CompletableFuture() + try { + executorService.execute { + try { + resultFuture.complete(command()) + } catch (t: Throwable) { + log.error("Task error", t) + resultFuture.completeExceptionally(t) + } + } + } catch (t: Throwable) { + log.error("Executor error", t) } - return result + return resultFuture } private fun executeLongRunning(command: () -> T): CompletableFuture { diff --git a/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/MessageBusConsumerTest.kt b/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/MessageBusConsumerTest.kt index d4d2a466cf4..20f4c5e45b0 100644 --- a/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/MessageBusConsumerTest.kt +++ b/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/MessageBusConsumerTest.kt @@ -1,6 +1,5 @@ package net.corda.messaging.mediator -import kotlinx.coroutines.runBlocking import net.corda.messagebus.api.consumer.CordaConsumer import net.corda.v5.base.exceptions.CordaRuntimeException import org.junit.jupiter.api.BeforeEach @@ -51,30 +50,23 @@ class MessageBusConsumerTest { doThrow(CordaRuntimeException("")).whenever(cordaConsumer).poll(any()) assertThrows { - runBlocking { - mediatorConsumer.poll(timeout).await() - } + mediatorConsumer.poll(timeout) } } @Test - fun testCommitAsyncOffsets() { - mediatorConsumer.asyncCommitOffsets() + fun testSyncCommitOffsets() { + mediatorConsumer.syncCommitOffsets() - verify(cordaConsumer).asyncCommitOffsets(any()) + verify(cordaConsumer).syncCommitOffsets() } @Test - fun testCommitAsyncOffsetsWithError() { - whenever(cordaConsumer.asyncCommitOffsets(any())).thenAnswer { invocation -> - val callback = invocation.getArgument(0) - callback.onCompletion(mock(), CordaRuntimeException("")) - } + fun testSyncCommitOffsetsWithError() { + doThrow(CordaRuntimeException("")).whenever(cordaConsumer).syncCommitOffsets() assertThrows { - runBlocking { - mediatorConsumer.asyncCommitOffsets().await() - } + mediatorConsumer.syncCommitOffsets() } } diff --git a/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/MultiSourceEventMediatorImplTest.kt b/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/MultiSourceEventMediatorImplTest.kt index acd83cafaad..6eedc2a5d59 100644 --- a/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/MultiSourceEventMediatorImplTest.kt +++ b/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/MultiSourceEventMediatorImplTest.kt @@ -4,7 +4,6 @@ import net.corda.avro.serialization.CordaAvroDeserializer import net.corda.avro.serialization.CordaAvroSerializer import net.corda.libs.statemanager.api.StateManager import net.corda.lifecycle.LifecycleCoordinatorFactory -import net.corda.messagebus.api.CordaTopicPartition import net.corda.messagebus.api.consumer.CordaConsumerRecord import net.corda.messaging.api.mediator.MediatorConsumer import net.corda.messaging.api.mediator.MessageRouter @@ -24,7 +23,6 @@ import net.corda.messaging.api.processor.StateAndEventProcessor import net.corda.messaging.api.records.Record import net.corda.test.util.waitWhile import org.junit.jupiter.api.BeforeEach -// import org.junit.jupiter.api.Test import org.mockito.kotlin.any import org.mockito.kotlin.anyOrNull import org.mockito.kotlin.atLeast @@ -117,7 +115,7 @@ class MultiSourceEventMediatorImplTest { ) } - //@Test + // @Test // TODO Test temporarily disabled as it seems to be flaky fun `mediator processes multiples events by key`() { val events = (1..6).map { "event$it" } @@ -134,18 +132,13 @@ class MultiSourceEventMediatorImplTest { ), ) var batchNumber = 0 - whenever(consumer.asyncCommitOffsets()).thenAnswer { - CompletableDeferred(mock>()) - } whenever(consumer.poll(any())).thenAnswer { - CompletableDeferred( - if (batchNumber < eventBatches.size) { - eventBatches[batchNumber++] - } else { - Thread.sleep(10) - emptyList() - } - ) + if (batchNumber < eventBatches.size) { + eventBatches[batchNumber++] + } else { + Thread.sleep(10) + emptyList() + } } mediator.start() @@ -159,7 +152,7 @@ class MultiSourceEventMediatorImplTest { verify(stateManager, times(eventBatches.size)).get(any()) verify(stateManager, times(eventBatches.size)).create(any()) verify(consumer, atLeast(eventBatches.size)).poll(any()) - verify(consumer, times(eventBatches.size)).asyncCommitOffsets() + verify(consumer, times(eventBatches.size)).syncCommitOffsets() verify(messagingClient, times(events.size)).send(any()) } diff --git a/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/TaskManagerHelperTest.kt b/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/TaskManagerHelperTest.kt index 1fbc942b174..5a664098766 100644 --- a/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/TaskManagerHelperTest.kt +++ b/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/TaskManagerHelperTest.kt @@ -7,13 +7,13 @@ import net.corda.messaging.api.mediator.MessageRouter import net.corda.messaging.api.mediator.MessagingClient.Companion.MSG_PROP_KEY import net.corda.messaging.api.mediator.taskmanager.TaskManager import net.corda.messaging.api.processor.StateAndEventProcessor +import net.corda.messaging.api.records.Record import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Test -import org.mockito.kotlin.mock -import net.corda.messaging.api.records.Record import org.mockito.Mockito.`when` import org.mockito.kotlin.any import org.mockito.kotlin.argumentCaptor +import org.mockito.kotlin.mock import org.mockito.kotlin.times import org.mockito.kotlin.verify diff --git a/libs/messaging/messaging/src/main/kotlin/net/corda/messaging/api/mediator/MediatorConsumer.kt b/libs/messaging/messaging/src/main/kotlin/net/corda/messaging/api/mediator/MediatorConsumer.kt index 79e53453d2a..6e2036891fd 100644 --- a/libs/messaging/messaging/src/main/kotlin/net/corda/messaging/api/mediator/MediatorConsumer.kt +++ b/libs/messaging/messaging/src/main/kotlin/net/corda/messaging/api/mediator/MediatorConsumer.kt @@ -1,7 +1,5 @@ package net.corda.messaging.api.mediator -import kotlinx.coroutines.Deferred -import net.corda.messagebus.api.CordaTopicPartition import net.corda.messagebus.api.consumer.CordaConsumerRecord import java.time.Duration @@ -20,14 +18,12 @@ interface MediatorConsumer : AutoCloseable { * * @param timeout - The maximum time to block if there are no available messages. */ - fun poll(timeout: Duration): Deferred>> + fun poll(timeout: Duration): List> /** - * Asynchronously commit the consumer offsets. This function should be called only after `poll` was called. - * - * @return [Deferred] with committed offsets. + * Synchronously commits the consumer offsets. This function should be called only after `poll` was called. */ - fun asyncCommitOffsets(): Deferred> + fun syncCommitOffsets() /** * Resets consumer's offsets to the last committed positions. Next poll will read from the last committed positions. From c69a6773cc1903b6e1d8eed2f63eadf5973208ed Mon Sep 17 00:00:00 2001 From: Miljenko Brkic Date: Sat, 14 Oct 2023 09:39:16 +0100 Subject: [PATCH 30/81] CORE-16203 Set linger.ms to 0 for stateAndEvent producer in kafka-messaging-defaults.conf --- .../src/main/resources/kafka-messaging-defaults.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/messaging/kafka-message-bus-impl/src/main/resources/kafka-messaging-defaults.conf b/libs/messaging/kafka-message-bus-impl/src/main/resources/kafka-messaging-defaults.conf index 611db668ece..f105646011c 100644 --- a/libs/messaging/kafka-message-bus-impl/src/main/resources/kafka-messaging-defaults.conf +++ b/libs/messaging/kafka-message-bus-impl/src/main/resources/kafka-messaging-defaults.conf @@ -78,7 +78,7 @@ roles { # within this pattern is transactional by default, and the transaction commit also causes a flush. producer = ${producer} { # Maximum time to wait for additional messages before sending the current batch. - linger.ms = 1000 + linger.ms = 0 # Maximum amount of memory in bytes (not messages) that will be used for each batch. Can not be higher than # the value configured for "message.max.bytes" on the broker side (1mb by default). batch.size = 750000 From bbb46dfcba4617af4534df1fa6179b83d6ee3c7c Mon Sep 17 00:00:00 2001 From: Ramzi El-Yafi Date: Mon, 16 Oct 2023 15:30:10 +0100 Subject: [PATCH 31/81] CORE-17782 Suppress deprecation warning on `findUnconsumedStatesByType` (#4884) --- gradle.properties | 2 +- .../r3/corda/testing/packagingverification/ReportStatesFlow.kt | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/gradle.properties b/gradle.properties index 5afb4782e13..5cf01b55410 100644 --- a/gradle.properties +++ b/gradle.properties @@ -46,7 +46,7 @@ commonsTextVersion = 1.10.0 bouncycastleVersion=1.76 # Corda API libs revision (change in 4th digit indicates a breaking change) # Change to 5.1.0.xx-SNAPSHOT to pick up maven local published copy -cordaApiVersion=5.1.0.36-beta+ +cordaApiVersion=5.1.0.37-beta+ disruptorVersion=3.4.4 felixConfigAdminVersion=1.9.26 diff --git a/testing/cpbs/packaging-verification-app-v1/src/main/kotlin/com/r3/corda/testing/packagingverification/ReportStatesFlow.kt b/testing/cpbs/packaging-verification-app-v1/src/main/kotlin/com/r3/corda/testing/packagingverification/ReportStatesFlow.kt index b6e31f95519..4ddab4646ef 100644 --- a/testing/cpbs/packaging-verification-app-v1/src/main/kotlin/com/r3/corda/testing/packagingverification/ReportStatesFlow.kt +++ b/testing/cpbs/packaging-verification-app-v1/src/main/kotlin/com/r3/corda/testing/packagingverification/ReportStatesFlow.kt @@ -13,6 +13,7 @@ class ReportStatesFlow : ClientStartableFlow { @Suspendable override fun call(requestBody: ClientRequestBody): String { + @Suppress("Deprecation") // Call to be replaced in CORE-17745 val unconsumedStates = utxoLedgerService.findUnconsumedStatesByType(SimpleState::class.java) val result = unconsumedStates.fold(0L) { acc, stateAndRef -> acc + stateAndRef.state.contractState.value From aa14419f563a7510f1f7d27a130fda55834e2228 Mon Sep 17 00:00:00 2001 From: Yash Nabar Date: Mon, 16 Oct 2023 15:44:15 +0100 Subject: [PATCH 32/81] CORE-17625 MGM lookups return active platform version in its own MemberInfo (#4855) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit After performing a platform upgrade, an MGM’s view of its own MemberInfo is not refreshed, so any lookups performed by such an MGM would continue to show it to be on the older version. This change updates the MembershipGroupReader to decorate the returned MGM MemberInfo (when requested by the MGM) with the active platform version from PlatformInfoProvider. This will be removed on completion of CORE-17811, once support for MGM re-registration is available. --- .../membership-group-read-impl/build.gradle | 1 + .../read/MembershipGroupReaderProviderImpl.kt | 8 +- .../reader/MembershipGroupReaderFactory.kt | 8 +- .../read/reader/MembershipGroupReaderImpl.kt | 44 ++++++++- .../MembershipGroupReaderProviderImplTest.kt | 5 +- .../MembershipGroupReaderFactoryTest.kt | 8 +- .../reader/MembershipGroupReaderImplTest.kt | 96 ++++++++++++++++++- 7 files changed, 160 insertions(+), 10 deletions(-) diff --git a/components/membership/membership-group-read-impl/build.gradle b/components/membership/membership-group-read-impl/build.gradle index 42568e44d22..b6c1f663dc1 100644 --- a/components/membership/membership-group-read-impl/build.gradle +++ b/components/membership/membership-group-read-impl/build.gradle @@ -17,6 +17,7 @@ dependencies { implementation project(":libs:lifecycle:lifecycle") implementation project(":libs:membership:membership-impl") implementation project(":libs:messaging:messaging") + implementation project(':libs:platform-info') implementation project(":libs:serialization:serialization-avro") implementation project(":libs:virtual-node:virtual-node-info") diff --git a/components/membership/membership-group-read-impl/src/main/kotlin/net/corda/membership/impl/read/MembershipGroupReaderProviderImpl.kt b/components/membership/membership-group-read-impl/src/main/kotlin/net/corda/membership/impl/read/MembershipGroupReaderProviderImpl.kt index 4cb609d3329..26f57604175 100644 --- a/components/membership/membership-group-read-impl/src/main/kotlin/net/corda/membership/impl/read/MembershipGroupReaderProviderImpl.kt +++ b/components/membership/membership-group-read-impl/src/main/kotlin/net/corda/membership/impl/read/MembershipGroupReaderProviderImpl.kt @@ -1,6 +1,7 @@ package net.corda.membership.impl.read import net.corda.configuration.read.ConfigurationReadService +import net.corda.libs.platform.PlatformInfoProvider import net.corda.lifecycle.LifecycleCoordinatorFactory import net.corda.lifecycle.LifecycleStatus import net.corda.lifecycle.StartEvent @@ -44,6 +45,8 @@ class MembershipGroupReaderProviderImpl @Activate constructor( private val memberInfoFactory: MemberInfoFactory, @Reference(service = GroupParametersReaderService::class) private val groupParametersReaderService: GroupParametersReaderService, + @Reference(service = PlatformInfoProvider::class) + private val platformInfoProvider: PlatformInfoProvider, ) : MembershipGroupReaderProvider { companion object { @@ -113,8 +116,9 @@ class MembershipGroupReaderProviderImpl @Activate constructor( private val membershipGroupReadCache: MembershipGroupReadCache, ) : InnerMembershipGroupReaderProvider { // Factory responsible for creating group readers or taking existing instances from the cache. - private val membershipGroupReaderFactory: MembershipGroupReaderFactory = - MembershipGroupReaderFactory.Impl(membershipGroupReadCache, groupParametersReaderService) + private val membershipGroupReaderFactory: MembershipGroupReaderFactory = MembershipGroupReaderFactory.Impl( + membershipGroupReadCache, groupParametersReaderService, memberInfoFactory, platformInfoProvider + ) /** * Get the [MembershipGroupReader] instance for the given holding identity. diff --git a/components/membership/membership-group-read-impl/src/main/kotlin/net/corda/membership/impl/read/reader/MembershipGroupReaderFactory.kt b/components/membership/membership-group-read-impl/src/main/kotlin/net/corda/membership/impl/read/reader/MembershipGroupReaderFactory.kt index b2f524c0de2..e2fcab86d7e 100644 --- a/components/membership/membership-group-read-impl/src/main/kotlin/net/corda/membership/impl/read/reader/MembershipGroupReaderFactory.kt +++ b/components/membership/membership-group-read-impl/src/main/kotlin/net/corda/membership/impl/read/reader/MembershipGroupReaderFactory.kt @@ -1,6 +1,8 @@ package net.corda.membership.impl.read.reader +import net.corda.libs.platform.PlatformInfoProvider import net.corda.membership.impl.read.cache.MembershipGroupReadCache +import net.corda.membership.lib.MemberInfoFactory import net.corda.membership.read.GroupParametersReaderService import net.corda.membership.read.MembershipGroupReader import net.corda.virtualnode.HoldingIdentity @@ -23,6 +25,8 @@ interface MembershipGroupReaderFactory { class Impl( private val membershipGroupReadCache: MembershipGroupReadCache, private val groupParametersReaderService: GroupParametersReaderService, + private val memberInfoFactory: MemberInfoFactory, + private val platformInfoProvider: PlatformInfoProvider, ) : MembershipGroupReaderFactory { private val groupReaderCache get() = membershipGroupReadCache.groupReaderCache @@ -35,7 +39,9 @@ interface MembershipGroupReaderFactory { ) = MembershipGroupReaderImpl( holdingIdentity, membershipGroupReadCache, - groupParametersReaderService + groupParametersReaderService, + memberInfoFactory, + platformInfoProvider, ).apply { groupReaderCache.put(holdingIdentity, this) } diff --git a/components/membership/membership-group-read-impl/src/main/kotlin/net/corda/membership/impl/read/reader/MembershipGroupReaderImpl.kt b/components/membership/membership-group-read-impl/src/main/kotlin/net/corda/membership/impl/read/reader/MembershipGroupReaderImpl.kt index 30c9b999c00..f6796cdc371 100644 --- a/components/membership/membership-group-read-impl/src/main/kotlin/net/corda/membership/impl/read/reader/MembershipGroupReaderImpl.kt +++ b/components/membership/membership-group-read-impl/src/main/kotlin/net/corda/membership/impl/read/reader/MembershipGroupReaderImpl.kt @@ -1,20 +1,27 @@ package net.corda.membership.impl.read.reader import net.corda.data.p2p.app.MembershipStatusFilter +import net.corda.libs.platform.PlatformInfoProvider import net.corda.membership.impl.read.cache.MembershipGroupReadCache import net.corda.membership.lib.InternalGroupParameters import net.corda.membership.lib.MemberInfoExtension.Companion.MEMBER_STATUS_ACTIVE import net.corda.membership.lib.MemberInfoExtension.Companion.MEMBER_STATUS_PENDING import net.corda.membership.lib.MemberInfoExtension.Companion.MEMBER_STATUS_SUSPENDED +import net.corda.membership.lib.MemberInfoExtension.Companion.PLATFORM_VERSION +import net.corda.membership.lib.MemberInfoExtension.Companion.isMgm import net.corda.membership.lib.MemberInfoExtension.Companion.ledgerKeyHashes import net.corda.membership.lib.MemberInfoExtension.Companion.sessionKeyHashes import net.corda.membership.lib.MemberInfoExtension.Companion.status +import net.corda.membership.lib.MemberInfoFactory import net.corda.membership.lib.SignedGroupParameters +import net.corda.membership.lib.toSortedMap +import net.corda.membership.lib.toWire import net.corda.membership.read.GroupParametersReaderService import net.corda.membership.read.MembershipGroupReader import net.corda.membership.read.NotaryVirtualNodeLookup import net.corda.v5.base.types.MemberX500Name import net.corda.v5.crypto.SecureHash +import net.corda.v5.membership.MemberContext import net.corda.v5.membership.MemberInfo import net.corda.virtualnode.HoldingIdentity @@ -22,6 +29,8 @@ class MembershipGroupReaderImpl( private val holdingIdentity: HoldingIdentity, private val membershipGroupReadCache: MembershipGroupReadCache, private val groupParametersReaderService: GroupParametersReaderService, + private val memberInfoFactory: MemberInfoFactory, + private val platformInfoProvider: PlatformInfoProvider, ) : MembershipGroupReader { override val groupId: String = holdingIdentity.groupId override val owningMember: MemberX500Name = holdingIdentity.x500Name @@ -39,20 +48,24 @@ class MembershipGroupReaderImpl( get() = groupParametersReaderService.getSigned(holdingIdentity) override fun lookup(filter: MembershipStatusFilter): Collection = - memberList.filterBy(filter) + withCurrentMgmPlatformVersion(memberList.filterBy(filter)) override fun lookupByLedgerKey(ledgerKeyHash: SecureHash, filter: MembershipStatusFilter): MemberInfo? = memberList.filterBy(filter).singleOrNull { ledgerKeyHash in it.ledgerKeyHashes } override fun lookupBySessionKey(sessionKeyHash: SecureHash, filter: MembershipStatusFilter): MemberInfo? = - memberList.filterBy(filter).singleOrNull { it.sessionKeyHashes.contains(sessionKeyHash) } + memberList.filterBy(filter).singleOrNull { it.sessionKeyHashes.contains(sessionKeyHash) }?.let { + withCurrentMgmPlatformVersion(setOf(it)).first() + } override val notaryVirtualNodeLookup: NotaryVirtualNodeLookup by lazy { NotaryVirtualNodeLookupImpl(this) } override fun lookup(name: MemberX500Name, filter: MembershipStatusFilter): MemberInfo? = - memberList.filterBy(filter).singleOrNull { it.name == name } + memberList.filterBy(filter).singleOrNull { it.name == name }?.let { + withCurrentMgmPlatformVersion(setOf(it)).first() + } private fun List.filterBy(filter: MembershipStatusFilter): List { return when (filter) { @@ -72,4 +85,29 @@ class MembershipGroupReaderImpl( else -> this.filter { it.status == MEMBER_STATUS_ACTIVE || it.status == MEMBER_STATUS_SUSPENDED } } } + + /** + * Note: This method should be called on every lookup result returned by the following methods: [lookup], [lookup], + * [lookupBySessionKey]. + * + * This method refreshes an MGM's view on its own [MemberInfo] to reflect the correct platform version, as + * retrieved from [PlatformInfoProvider]. Without this, an MGM would continue to see the older platform version + * in its [MemberInfo] after performing a platform upgrade. + */ + private fun withCurrentMgmPlatformVersion(members: Collection): Collection = + members.firstOrNull { it.isMgm && it.name == owningMember }?.let { mgm -> + members + .minus(mgm) + .plus( + memberInfoFactory.createMemberInfo( + mgm.memberProvidedContext.refreshPlatformVersion().toSortedMap(), + mgm.mgmProvidedContext.toWire().toSortedMap() + ) + ) + } ?: members + + private fun MemberContext.refreshPlatformVersion(): Map = + entries.filterNot { it.key == PLATFORM_VERSION } + .associate { it.key to it.value } + .plus(PLATFORM_VERSION to platformInfoProvider.activePlatformVersion.toString()) } diff --git a/components/membership/membership-group-read-impl/src/test/kotlin/net/corda/membership/impl/read/MembershipGroupReaderProviderImplTest.kt b/components/membership/membership-group-read-impl/src/test/kotlin/net/corda/membership/impl/read/MembershipGroupReaderProviderImplTest.kt index 51de43c315b..bf13dd4f6d9 100644 --- a/components/membership/membership-group-read-impl/src/test/kotlin/net/corda/membership/impl/read/MembershipGroupReaderProviderImplTest.kt +++ b/components/membership/membership-group-read-impl/src/test/kotlin/net/corda/membership/impl/read/MembershipGroupReaderProviderImplTest.kt @@ -1,6 +1,7 @@ package net.corda.membership.impl.read import net.corda.configuration.read.ConfigurationReadService +import net.corda.libs.platform.PlatformInfoProvider import net.corda.lifecycle.LifecycleCoordinator import net.corda.lifecycle.LifecycleCoordinatorFactory import net.corda.lifecycle.LifecycleStatus @@ -58,6 +59,7 @@ class MembershipGroupReaderProviderImplTest { } private val memberInfoFactory: MemberInfoFactory = mock() private val groupParametersReaderService: GroupParametersReaderService = mock() + private val platformInfoProvider: PlatformInfoProvider = mock() @BeforeEach fun setUp() { @@ -66,7 +68,8 @@ class MembershipGroupReaderProviderImplTest { subscriptionFactory, lifecycleCoordinatorFactory, memberInfoFactory, - groupParametersReaderService + groupParametersReaderService, + platformInfoProvider, ) } diff --git a/components/membership/membership-group-read-impl/src/test/kotlin/net/corda/membership/impl/read/reader/MembershipGroupReaderFactoryTest.kt b/components/membership/membership-group-read-impl/src/test/kotlin/net/corda/membership/impl/read/reader/MembershipGroupReaderFactoryTest.kt index 7d048473c4d..855369140dd 100644 --- a/components/membership/membership-group-read-impl/src/test/kotlin/net/corda/membership/impl/read/reader/MembershipGroupReaderFactoryTest.kt +++ b/components/membership/membership-group-read-impl/src/test/kotlin/net/corda/membership/impl/read/reader/MembershipGroupReaderFactoryTest.kt @@ -1,10 +1,12 @@ package net.corda.membership.impl.read.reader +import net.corda.libs.platform.PlatformInfoProvider import net.corda.membership.impl.read.TestProperties.Companion.GROUP_ID_1 import net.corda.membership.impl.read.TestProperties.Companion.aliceName import net.corda.membership.impl.read.TestProperties.Companion.bobName import net.corda.membership.impl.read.cache.MemberDataCache import net.corda.membership.impl.read.cache.MembershipGroupReadCache +import net.corda.membership.lib.MemberInfoFactory import net.corda.membership.read.GroupParametersReaderService import net.corda.membership.read.MembershipGroupReader import net.corda.virtualnode.HoldingIdentity @@ -40,10 +42,14 @@ class MembershipGroupReaderFactoryTest { } private val groupParametersReaderService: GroupParametersReaderService = mock() + private val memberInfoFactory: MemberInfoFactory = mock() + private val platformInfoProvider: PlatformInfoProvider = mock() @BeforeEach fun setUp() { - membershipGroupReaderFactory = MembershipGroupReaderFactory.Impl(cache, groupParametersReaderService) + membershipGroupReaderFactory = MembershipGroupReaderFactory.Impl( + cache, groupParametersReaderService, memberInfoFactory, platformInfoProvider + ) } @Test diff --git a/components/membership/membership-group-read-impl/src/test/kotlin/net/corda/membership/impl/read/reader/MembershipGroupReaderImplTest.kt b/components/membership/membership-group-read-impl/src/test/kotlin/net/corda/membership/impl/read/reader/MembershipGroupReaderImplTest.kt index aa47e3e069c..13ed7a733ec 100644 --- a/components/membership/membership-group-read-impl/src/test/kotlin/net/corda/membership/impl/read/reader/MembershipGroupReaderImplTest.kt +++ b/components/membership/membership-group-read-impl/src/test/kotlin/net/corda/membership/impl/read/reader/MembershipGroupReaderImplTest.kt @@ -4,16 +4,29 @@ import net.corda.crypto.cipher.suite.sha256Bytes import net.corda.crypto.core.SecureHashImpl import net.corda.data.p2p.app.MembershipStatusFilter import net.corda.data.p2p.app.MembershipStatusFilter.ACTIVE_OR_SUSPENDED_IF_PRESENT_OR_PENDING +import net.corda.libs.platform.PlatformInfoProvider import net.corda.membership.impl.read.TestProperties import net.corda.membership.impl.read.TestProperties.Companion.GROUP_ID_1 import net.corda.membership.impl.read.cache.MemberListCache import net.corda.membership.impl.read.cache.MembershipGroupReadCache +import net.corda.membership.lib.MemberInfoExtension.Companion.GROUP_ID +import net.corda.membership.lib.MemberInfoExtension.Companion.IS_MGM import net.corda.membership.lib.MemberInfoExtension.Companion.LEDGER_KEYS import net.corda.membership.lib.MemberInfoExtension.Companion.MEMBER_STATUS_ACTIVE import net.corda.membership.lib.MemberInfoExtension.Companion.MEMBER_STATUS_PENDING import net.corda.membership.lib.MemberInfoExtension.Companion.MEMBER_STATUS_SUSPENDED +import net.corda.membership.lib.MemberInfoExtension.Companion.MODIFIED_TIME +import net.corda.membership.lib.MemberInfoExtension.Companion.PARTY_NAME +import net.corda.membership.lib.MemberInfoExtension.Companion.PARTY_SESSION_KEYS +import net.corda.membership.lib.MemberInfoExtension.Companion.PLATFORM_VERSION +import net.corda.membership.lib.MemberInfoExtension.Companion.PROTOCOL_VERSION +import net.corda.membership.lib.MemberInfoExtension.Companion.SERIAL import net.corda.membership.lib.MemberInfoExtension.Companion.SESSION_KEYS +import net.corda.membership.lib.MemberInfoExtension.Companion.SOFTWARE_VERSION import net.corda.membership.lib.MemberInfoExtension.Companion.STATUS +import net.corda.membership.lib.MemberInfoExtension.Companion.URL_KEY +import net.corda.membership.lib.MemberInfoExtension.Companion.isMgm +import net.corda.membership.lib.MemberInfoFactory import net.corda.membership.lib.SignedGroupParameters import net.corda.membership.lib.UnsignedGroupParameters import net.corda.membership.read.GroupParametersReaderService @@ -26,15 +39,21 @@ import net.corda.virtualnode.HoldingIdentity import org.assertj.core.api.Assertions.assertThat import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Assertions.assertNull +import org.junit.jupiter.api.BeforeAll import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Nested import org.junit.jupiter.api.Test +import org.junit.jupiter.api.TestInstance import org.junit.jupiter.api.assertThrows +import org.mockito.kotlin.any import org.mockito.kotlin.doReturn import org.mockito.kotlin.eq import org.mockito.kotlin.mock import org.mockito.kotlin.verify import org.mockito.kotlin.whenever import java.security.PublicKey +import java.time.Instant +import java.util.SortedMap class MembershipGroupReaderImplTest { private lateinit var membershipGroupReaderImpl: MembershipGroupReaderImpl @@ -129,7 +148,9 @@ class MembershipGroupReaderImplTest { membershipGroupReaderImpl = MembershipGroupReaderImpl( aliceIdGroup1, membershipGroupCache, - groupParametersReaderService + groupParametersReaderService, + mock(), + mock(), ) } @@ -345,9 +366,80 @@ class MembershipGroupReaderImplTest { val bobGroupReader = MembershipGroupReaderImpl( bobIdGroup1, membershipGroupCache, - groupParametersReaderService + groupParametersReaderService, + mock(), + mock(), ) assertThat(bobGroupReader.signedGroupParameters).isEqualTo(signedGroupParameters) verify(groupParametersReaderService).getSigned(eq(bobIdGroup1)) } + + @TestInstance(TestInstance.Lifecycle.PER_CLASS) + @Nested + inner class MGMLookupTests { + private val mgmName = TestProperties.charlieName + private val mgmIdGroup1 = HoldingIdentity(mgmName, GROUP_ID_1) + private val platformInfoProvider: PlatformInfoProvider = mock { + on { activePlatformVersion } doReturn 50100 + } + private val memberContextMap: SortedMap = sortedMapOf( + PARTY_NAME to mgmName.toString(), + String.format(PARTY_SESSION_KEYS, 0) to "1234", + GROUP_ID to GROUP_ID_1, + URL_KEY.format(0) to "https://corda5.r3.com:10000", + PROTOCOL_VERSION.format(0) to "1", + SOFTWARE_VERSION to "5.0.0", + PLATFORM_VERSION to platformInfoProvider.activePlatformVersion.toString(), + ) + private val mgmContextMap = sortedMapOf( + STATUS to MEMBER_STATUS_ACTIVE, + MODIFIED_TIME to Instant.now().toString(), + IS_MGM to "true", + SERIAL to "1", + ) + private val mgmMemberInfo: MemberInfo = mock { + on { name } doReturn mgmName + on { memberProvidedContext } doReturn mockedActiveMemberProvidedContext + on { mgmProvidedContext } doReturn mockedActiveMgmProvidedContext + on { isActive } doReturn true + on { isMgm } doReturn true + } + private val mgmWithLatestPlatformVersion: MemberInfo = mock() + private val memberInfoFactory: MemberInfoFactory = mock { + on { createMemberInfo(eq(memberContextMap), any()) } doReturn mgmWithLatestPlatformVersion + } + private lateinit var mgmGroupReader: MembershipGroupReaderImpl + + @BeforeAll + fun setup() { + whenever(mockedActiveMemberProvidedContext.entries).doReturn(memberContextMap.entries) + whenever(mockedActiveMgmProvidedContext.entries).doReturn(mgmContextMap.entries) + mgmGroupReader = MembershipGroupReaderImpl( + mgmIdGroup1, + membershipGroupCache, + groupParametersReaderService, + memberInfoFactory, + platformInfoProvider, + ) + } + + @Test + fun `lookup performed by MGM returns active platform version in its own MemberInfo`() { + whenever(memberCache.get(eq(mgmIdGroup1))).thenReturn(listOf(aliceActiveMemberInfo, mgmMemberInfo)) + assertThat(mgmGroupReader.lookup()) + .containsExactlyInAnyOrder(mgmWithLatestPlatformVersion, aliceActiveMemberInfo) + } + + @Test + fun `lookup performed by MGM based on name returns active platform version in its own MemberInfo`() { + whenever(memberCache.get(eq(mgmIdGroup1))).thenReturn(listOf(aliceActiveMemberInfo, mgmMemberInfo)) + assertThat(mgmGroupReader.lookup(mgmName)).isEqualTo(mgmWithLatestPlatformVersion) + } + + @Test + fun `lookup performed by MGM based on session key hash returns active platform version in its own MemberInfo`() { + whenever(memberCache.get(eq(mgmIdGroup1))).thenReturn(listOf(mgmMemberInfo)) + assertThat(mgmGroupReader.lookupBySessionKey(mockSessionKeyHash)).isEqualTo(mgmWithLatestPlatformVersion) + } + } } From 93771d89ace27d9d713bd7ea5ca37d8507819274 Mon Sep 17 00:00:00 2001 From: Dan Newton Date: Mon, 16 Oct 2023 17:07:05 +0100 Subject: [PATCH 33/81] CORE-17487 Parallel flow pipeline and Task Manager (#4731) Add support for parallel flows executing within the flow pipeline. This includes: - Temporary multi-thread implementation of the state and event pattern. - Using the `SameThreadExecutor` for flow fibers, which causes the fiber to run on the thread that started the fiber. - Using a Kryo Pool for Kryo instances, as kryo instances are not thread-safe themselves. Add `TaskManager` and a `TaskManagerFactory` which is a wrapper around a `ScheduledExecutorService` used for gathering metrics on executing tasks. The built-in Micrometer metrics for executor services are used, along with some additional metrics in the `TaskManager` that are more specific to our use-cases. --------- Co-authored-by: Conal Smith --- .../FlowMapperEventMediatorFactoryImpl.kt | 5 + .../FlowMapperEventMediatorFactoryImplTest.kt | 6 +- .../fiber/factory/FlowFiberFactoryImpl.kt | 27 ++-- .../mediator/FlowEventMediatorFactoryImpl.kt | 7 + .../FlowEventMediatorFactoryImplTest.kt | 11 +- ...edgerTransactionVerificationServiceImpl.kt | 2 +- .../kafka/producer/CordaKafkaProducerImpl.kt | 2 +- libs/messaging/messaging-impl/build.gradle | 1 + .../mediator/MultiSourceEventMediatorImpl.kt | 5 +- .../messaging/mediator/TaskManagerHelper.kt | 25 ++-- .../MultiSourceEventMediatorFactoryImpl.kt | 31 +++- .../mediator/statemanager/StateManagerImpl.kt | 19 ++- .../mediator/taskmanager/TaskManagerImpl.kt | 58 -------- .../StateAndEventSubscriptionImpl.kt | 2 +- .../consumer/StateAndEventConsumerImpl.kt | 2 +- .../MultiSourceEventMediatorImplTest.kt | 4 +- .../mediator/TaskManagerHelperTest.kt | 73 +++++---- .../MultiSourceEventMediatorFactoryTest.kt | 10 +- .../mediator/config/EventMediatorConfig.kt | 2 + .../config/EventMediatorConfigBuilder.kt | 22 ++- .../api/mediator/taskmanager/TaskManager.kt | 12 -- .../kotlin/net/corda/metrics/CordaMetrics.kt | 22 +++ .../KryoCheckpointSerializer.kt | 19 ++- .../KryoCheckpointSerializerBuilderImpl.kt | 28 ++-- .../KryoCheckpointSerializerTest.kt | 31 ++-- ...SingletonSerializeAsTokenSerializerTest.kt | 12 +- libs/task-manager/build.gradle | 20 +++ .../net/corda/taskmanager/package-info.java | 4 + .../net/corda/taskmanager/TaskManager.kt | 13 ++ .../corda/taskmanager/TaskManagerFactory.kt | 16 ++ .../impl/CordaExecutorServiceWrapper.kt | 13 ++ .../impl/TaskManagerFactoryImpl.kt | 43 ++++++ .../corda/taskmanager/impl/TaskManagerImpl.kt | 132 +++++++++++++++++ .../taskmanager/impl/TaskManagerImplTest.kt | 139 ++++++++++++++++++ settings.gradle | 1 + 35 files changed, 627 insertions(+), 192 deletions(-) delete mode 100644 libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/taskmanager/TaskManagerImpl.kt delete mode 100644 libs/messaging/messaging/src/main/kotlin/net/corda/messaging/api/mediator/taskmanager/TaskManager.kt create mode 100644 libs/task-manager/build.gradle create mode 100644 libs/task-manager/src/main/java/net/corda/taskmanager/package-info.java create mode 100644 libs/task-manager/src/main/kotlin/net/corda/taskmanager/TaskManager.kt create mode 100644 libs/task-manager/src/main/kotlin/net/corda/taskmanager/TaskManagerFactory.kt create mode 100644 libs/task-manager/src/main/kotlin/net/corda/taskmanager/impl/CordaExecutorServiceWrapper.kt create mode 100644 libs/task-manager/src/main/kotlin/net/corda/taskmanager/impl/TaskManagerFactoryImpl.kt create mode 100644 libs/task-manager/src/main/kotlin/net/corda/taskmanager/impl/TaskManagerImpl.kt create mode 100644 libs/task-manager/src/test/kotlin/net/corda/taskmanager/impl/TaskManagerImplTest.kt diff --git a/components/flow/flow-mapper-service/src/main/kotlin/net/corda/session/mapper/messaging/mediator/FlowMapperEventMediatorFactoryImpl.kt b/components/flow/flow-mapper-service/src/main/kotlin/net/corda/session/mapper/messaging/mediator/FlowMapperEventMediatorFactoryImpl.kt index 22c7937428f..3bbdb0523af 100644 --- a/components/flow/flow-mapper-service/src/main/kotlin/net/corda/session/mapper/messaging/mediator/FlowMapperEventMediatorFactoryImpl.kt +++ b/components/flow/flow-mapper-service/src/main/kotlin/net/corda/session/mapper/messaging/mediator/FlowMapperEventMediatorFactoryImpl.kt @@ -17,6 +17,7 @@ import net.corda.messaging.api.processor.StateAndEventProcessor import net.corda.schema.Schemas.Flow.FLOW_EVENT_TOPIC import net.corda.schema.Schemas.Flow.FLOW_MAPPER_EVENT_TOPIC import net.corda.schema.Schemas.P2P.P2P_OUT_TOPIC +import net.corda.schema.configuration.FlowConfig import net.corda.session.mapper.service.executor.FlowMapperMessageProcessor import org.osgi.service.component.annotations.Activate import org.osgi.service.component.annotations.Component @@ -43,12 +44,14 @@ class FlowMapperEventMediatorFactoryImpl @Activate constructor( messagingConfig: SmartConfig, ) = eventMediatorFactory.create( createEventMediatorConfig( + flowConfig, messagingConfig, FlowMapperMessageProcessor(flowMapperEventExecutorFactory, flowConfig), ) ) private fun createEventMediatorConfig( + flowConfig: SmartConfig, messagingConfig: SmartConfig, messageProcessor: StateAndEventProcessor, ) = EventMediatorConfigBuilder() @@ -66,6 +69,8 @@ class FlowMapperEventMediatorFactoryImpl @Activate constructor( ) .messageProcessor(messageProcessor) .messageRouterFactory(createMessageRouterFactory()) + .threads(flowConfig.getInt(FlowConfig.PROCESSING_THREAD_POOL_SIZE)) + .threadName("flow-mapper-event-mediator") .build() private fun createMessageRouterFactory() = MessageRouterFactory { clientFinder -> diff --git a/components/flow/flow-mapper-service/src/test/kotlin/net/corda/session/mapper/service/messaging/mediator/FlowMapperEventMediatorFactoryImplTest.kt b/components/flow/flow-mapper-service/src/test/kotlin/net/corda/session/mapper/service/messaging/mediator/FlowMapperEventMediatorFactoryImplTest.kt index b8b3b751c93..149fd9d9a22 100644 --- a/components/flow/flow-mapper-service/src/test/kotlin/net/corda/session/mapper/service/messaging/mediator/FlowMapperEventMediatorFactoryImplTest.kt +++ b/components/flow/flow-mapper-service/src/test/kotlin/net/corda/session/mapper/service/messaging/mediator/FlowMapperEventMediatorFactoryImplTest.kt @@ -3,10 +3,12 @@ package net.corda.session.mapper.service.messaging.mediator import net.corda.data.flow.event.mapper.FlowMapperEvent import net.corda.data.flow.state.mapper.FlowMapperState import net.corda.flow.mapper.factory.FlowMapperEventExecutorFactory +import net.corda.libs.configuration.SmartConfig import net.corda.messaging.api.mediator.config.EventMediatorConfig import net.corda.messaging.api.mediator.factory.MediatorConsumerFactoryFactory import net.corda.messaging.api.mediator.factory.MessagingClientFactoryFactory import net.corda.messaging.api.mediator.factory.MultiSourceEventMediatorFactory +import net.corda.schema.configuration.FlowConfig import net.corda.session.mapper.messaging.mediator.FlowMapperEventMediatorFactory import net.corda.session.mapper.messaging.mediator.FlowMapperEventMediatorFactoryImpl import org.junit.jupiter.api.Assertions.assertNotNull @@ -22,11 +24,13 @@ class FlowMapperEventMediatorFactoryImplTest { private val mediatorConsumerFactoryFactory = mock() private val messagingClientFactoryFactory = mock() private val multiSourceEventMediatorFactory = mock() + private val flowConfig = mock() @BeforeEach fun beforeEach() { `when`(multiSourceEventMediatorFactory.create(any>())) .thenReturn(mock()) + `when`(flowConfig.getInt(FlowConfig.PROCESSING_THREAD_POOL_SIZE)).thenReturn(10) flowMapperEventMediatorFactory = FlowMapperEventMediatorFactoryImpl( flowMapperEventExecutorFactory, @@ -38,7 +42,7 @@ class FlowMapperEventMediatorFactoryImplTest { @Test fun `successfully creates event mediator`() { - val mediator = flowMapperEventMediatorFactory.create(mock(), mock()) + val mediator = flowMapperEventMediatorFactory.create(flowConfig, mock()) assertNotNull(mediator) } diff --git a/components/flow/flow-service/src/main/kotlin/net/corda/flow/fiber/factory/FlowFiberFactoryImpl.kt b/components/flow/flow-service/src/main/kotlin/net/corda/flow/fiber/factory/FlowFiberFactoryImpl.kt index 499f1b5982b..d6679fb6ed5 100644 --- a/components/flow/flow-service/src/main/kotlin/net/corda/flow/fiber/factory/FlowFiberFactoryImpl.kt +++ b/components/flow/flow-service/src/main/kotlin/net/corda/flow/fiber/factory/FlowFiberFactoryImpl.kt @@ -1,10 +1,7 @@ package net.corda.flow.fiber.factory -import co.paralleluniverse.concurrent.util.ScheduledSingleThreadExecutor +import co.paralleluniverse.common.util.SameThreadExecutor import co.paralleluniverse.fibers.FiberExecutorScheduler -import co.paralleluniverse.fibers.FiberScheduler -import java.util.UUID -import java.util.concurrent.ExecutorService import net.corda.flow.fiber.FiberExceptionConstants import net.corda.flow.fiber.FiberFuture import net.corda.flow.fiber.FlowContinuation @@ -16,9 +13,9 @@ import net.corda.flow.pipeline.exceptions.FlowFatalException import net.corda.metrics.CordaMetrics import org.osgi.service.component.annotations.Activate import org.osgi.service.component.annotations.Component -import org.osgi.service.component.annotations.Deactivate import org.osgi.service.component.annotations.Reference import org.slf4j.LoggerFactory +import java.util.UUID @Component @Suppress("Unused") @@ -31,10 +28,12 @@ class FlowFiberFactoryImpl @Activate constructor( private val logger = LoggerFactory.getLogger(this::class.java.enclosingClass) } - private val currentScheduler: FiberScheduler = FiberExecutorScheduler( - "Same thread scheduler", - ScheduledSingleThreadExecutor() - ) + private val currentThreadFiberExecutor = object : FiberExecutorScheduler("Flow Fiber scheduler", SameThreadExecutor.getExecutor()) { + + override fun isCurrentThreadInScheduler(): Boolean { + return true + } + } override fun createAndStartFlowFiber( flowFiberExecutionContext: FlowFiberExecutionContext, @@ -47,7 +46,7 @@ class FlowFiberFactoryImpl @Activate constructor( throw FlowFatalException(FiberExceptionConstants.INVALID_FLOW_KEY.format(flowId), e) } try { - val flowFiber = FlowFiberImpl(id, logic, currentScheduler) + val flowFiber = FlowFiberImpl(id, logic, currentThreadFiberExecutor) return FiberFuture(flowFiber, flowFiber.startFlow(flowFiberExecutionContext)) } catch (e: Throwable) { throw FlowFatalException(FiberExceptionConstants.UNABLE_TO_EXECUTE.format(e.message ?: "No exception message provided."), e) @@ -66,7 +65,7 @@ class FlowFiberFactoryImpl @Activate constructor( getFromCacheOrDeserialize(flowFiberExecutionContext) }!! - return FiberFuture(fiber, fiber.resume(flowFiberExecutionContext, suspensionOutcome, currentScheduler)) + return FiberFuture(fiber, fiber.resume(flowFiberExecutionContext, suspensionOutcome, currentThreadFiberExecutor)) } private fun getFromCacheOrDeserialize(flowFiberExecutionContext: FlowFiberExecutionContext): FlowFiberImpl { @@ -81,10 +80,4 @@ class FlowFiberFactoryImpl @Activate constructor( FlowFiberImpl::class.java ) } - - @Deactivate - fun shutdown() { - currentScheduler.shutdown() - (currentScheduler.executor as? ExecutorService)?.shutdownNow() - } } \ No newline at end of file diff --git a/components/flow/flow-service/src/main/kotlin/net/corda/flow/messaging/mediator/FlowEventMediatorFactoryImpl.kt b/components/flow/flow-service/src/main/kotlin/net/corda/flow/messaging/mediator/FlowEventMediatorFactoryImpl.kt index 5408e207373..d4b81c53bf1 100644 --- a/components/flow/flow-service/src/main/kotlin/net/corda/flow/messaging/mediator/FlowEventMediatorFactoryImpl.kt +++ b/components/flow/flow-service/src/main/kotlin/net/corda/flow/messaging/mediator/FlowEventMediatorFactoryImpl.kt @@ -13,6 +13,7 @@ import net.corda.data.uniqueness.UniquenessCheckRequestAvro import net.corda.flow.pipeline.factory.FlowEventProcessorFactory import net.corda.ledger.utxo.verification.TransactionVerificationRequest import net.corda.libs.configuration.SmartConfig +import net.corda.libs.configuration.helper.getConfig import net.corda.messaging.api.mediator.MediatorMessage import net.corda.messaging.api.mediator.MessageRouter import net.corda.messaging.api.mediator.RoutingDestination.Companion.routeTo @@ -31,6 +32,8 @@ import net.corda.schema.Schemas.Persistence.PERSISTENCE_LEDGER_PROCESSOR_TOPIC import net.corda.schema.Schemas.Services.TOKEN_CACHE_EVENT import net.corda.schema.Schemas.UniquenessChecker.UNIQUENESS_CHECK_TOPIC import net.corda.schema.Schemas.Verification.VERIFICATION_LEDGER_PROCESSOR_TOPIC +import net.corda.schema.configuration.ConfigKeys +import net.corda.schema.configuration.FlowConfig import org.osgi.service.component.annotations.Activate import org.osgi.service.component.annotations.Component import org.osgi.service.component.annotations.Reference @@ -60,12 +63,14 @@ class FlowEventMediatorFactoryImpl @Activate constructor( messagingConfig: SmartConfig, ) = eventMediatorFactory.create( createEventMediatorConfig( + configs, messagingConfig, flowEventProcessorFactory.create(configs), ) ) private fun createEventMediatorConfig( + configs: Map, messagingConfig: SmartConfig, messageProcessor: StateAndEventProcessor, ) = EventMediatorConfigBuilder() @@ -83,6 +88,8 @@ class FlowEventMediatorFactoryImpl @Activate constructor( ) .messageProcessor(messageProcessor) .messageRouterFactory(createMessageRouterFactory()) + .threads(configs.getConfig(ConfigKeys.FLOW_CONFIG).getInt(FlowConfig.PROCESSING_THREAD_POOL_SIZE)) + .threadName("flow-event-mediator") .build() private fun createMessageRouterFactory() = MessageRouterFactory { clientFinder -> diff --git a/components/flow/flow-service/src/test/kotlin/net/corda/flow/messaging/FlowEventMediatorFactoryImplTest.kt b/components/flow/flow-service/src/test/kotlin/net/corda/flow/messaging/FlowEventMediatorFactoryImplTest.kt index 4f5af4bfb6d..d0fa7d377f9 100644 --- a/components/flow/flow-service/src/test/kotlin/net/corda/flow/messaging/FlowEventMediatorFactoryImplTest.kt +++ b/components/flow/flow-service/src/test/kotlin/net/corda/flow/messaging/FlowEventMediatorFactoryImplTest.kt @@ -6,10 +6,13 @@ import net.corda.data.flow.state.checkpoint.Checkpoint import net.corda.flow.messaging.mediator.FlowEventMediatorFactory import net.corda.flow.messaging.mediator.FlowEventMediatorFactoryImpl import net.corda.flow.pipeline.factory.FlowEventProcessorFactory +import net.corda.libs.configuration.SmartConfig import net.corda.messaging.api.mediator.config.EventMediatorConfig import net.corda.messaging.api.mediator.factory.MediatorConsumerFactoryFactory import net.corda.messaging.api.mediator.factory.MessagingClientFactoryFactory import net.corda.messaging.api.mediator.factory.MultiSourceEventMediatorFactory +import net.corda.schema.configuration.ConfigKeys +import net.corda.schema.configuration.FlowConfig import org.junit.jupiter.api.Assertions.assertNotNull import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Test @@ -24,6 +27,7 @@ class FlowEventMediatorFactoryImplTest { private val messagingClientFactoryFactory = mock() private val multiSourceEventMediatorFactory = mock() private val cordaAvroSerializationFactory = mock() + private val flowConfig = mock() @BeforeEach fun beforeEach() { @@ -33,6 +37,8 @@ class FlowEventMediatorFactoryImplTest { `when`(multiSourceEventMediatorFactory.create(any>())) .thenReturn(mock()) + `when`(flowConfig.getInt(FlowConfig.PROCESSING_THREAD_POOL_SIZE)).thenReturn(10) + flowEventMediatorFactory = FlowEventMediatorFactoryImpl( flowEventProcessorFactory, mediatorConsumerFactoryFactory, @@ -44,7 +50,10 @@ class FlowEventMediatorFactoryImplTest { @Test fun `successfully creates event mediator`() { - val mediator = flowEventMediatorFactory.create(mock(), mock()) + val mediator = flowEventMediatorFactory.create( + mapOf(ConfigKeys.FLOW_CONFIG to flowConfig), + mock() + ) assertNotNull(mediator) } diff --git a/components/ledger/ledger-utxo-flow/src/main/kotlin/net/corda/ledger/utxo/flow/impl/transaction/verifier/UtxoLedgerTransactionVerificationServiceImpl.kt b/components/ledger/ledger-utxo-flow/src/main/kotlin/net/corda/ledger/utxo/flow/impl/transaction/verifier/UtxoLedgerTransactionVerificationServiceImpl.kt index 3d866ca6e8f..666ffe09cbf 100644 --- a/components/ledger/ledger-utxo-flow/src/main/kotlin/net/corda/ledger/utxo/flow/impl/transaction/verifier/UtxoLedgerTransactionVerificationServiceImpl.kt +++ b/components/ledger/ledger-utxo-flow/src/main/kotlin/net/corda/ledger/utxo/flow/impl/transaction/verifier/UtxoLedgerTransactionVerificationServiceImpl.kt @@ -7,9 +7,9 @@ import net.corda.ledger.common.data.transaction.TransactionMetadataInternal import net.corda.ledger.utxo.data.transaction.TransactionVerificationStatus import net.corda.ledger.utxo.data.transaction.UtxoLedgerTransactionContainer import net.corda.ledger.utxo.data.transaction.UtxoLedgerTransactionInternal +import net.corda.ledger.utxo.flow.impl.groupparameters.verifier.SignedGroupParametersVerifier import net.corda.ledger.utxo.flow.impl.transaction.verifier.external.events.TransactionVerificationExternalEventFactory import net.corda.ledger.utxo.flow.impl.transaction.verifier.external.events.TransactionVerificationParameters -import net.corda.ledger.utxo.flow.impl.groupparameters.verifier.SignedGroupParametersVerifier import net.corda.membership.lib.SignedGroupParameters import net.corda.metrics.CordaMetrics import net.corda.sandbox.type.SandboxConstants.CORDA_SYSTEM_SERVICE diff --git a/libs/messaging/kafka-message-bus-impl/src/main/kotlin/net/corda/messagebus/kafka/producer/CordaKafkaProducerImpl.kt b/libs/messaging/kafka-message-bus-impl/src/main/kotlin/net/corda/messagebus/kafka/producer/CordaKafkaProducerImpl.kt index b3c9abc1e8f..e23e8480739 100644 --- a/libs/messaging/kafka-message-bus-impl/src/main/kotlin/net/corda/messagebus/kafka/producer/CordaKafkaProducerImpl.kt +++ b/libs/messaging/kafka-message-bus-impl/src/main/kotlin/net/corda/messagebus/kafka/producer/CordaKafkaProducerImpl.kt @@ -391,7 +391,7 @@ class CordaKafkaProducerImpl( throw CordaMessageAPIProducerRequiresReset("Error occurred $errorString", ex) } - in transientExceptions -> { + in transientExceptions -> { if (abortTransaction) { abortTransaction() } diff --git a/libs/messaging/messaging-impl/build.gradle b/libs/messaging/messaging-impl/build.gradle index 845cc242509..ce8c06fcb68 100644 --- a/libs/messaging/messaging-impl/build.gradle +++ b/libs/messaging/messaging-impl/build.gradle @@ -17,6 +17,7 @@ dependencies { implementation project(":libs:messaging:message-bus") implementation project(":libs:metrics") implementation project(":libs:schema-registry:schema-registry") + implementation project(":libs:task-manager") implementation project(":libs:tracing") implementation project(":libs:configuration:configuration-core") implementation project(':libs:utilities') diff --git a/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/MultiSourceEventMediatorImpl.kt b/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/MultiSourceEventMediatorImpl.kt index 0262d264f98..f0d6e9d1423 100644 --- a/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/MultiSourceEventMediatorImpl.kt +++ b/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/MultiSourceEventMediatorImpl.kt @@ -14,9 +14,8 @@ import net.corda.messaging.api.mediator.MessageRouter import net.corda.messaging.api.mediator.MessagingClient import net.corda.messaging.api.mediator.MultiSourceEventMediator import net.corda.messaging.api.mediator.config.EventMediatorConfig -import net.corda.messaging.api.mediator.taskmanager.TaskManager -import net.corda.messaging.api.mediator.taskmanager.TaskType import net.corda.messaging.mediator.factory.MediatorComponentFactory +import net.corda.taskmanager.TaskManager import net.corda.utilities.debug import org.slf4j.LoggerFactory import java.util.UUID @@ -58,7 +57,7 @@ class MultiSourceEventMediatorImpl( override fun start() { log.debug { "Starting multi-source event mediator with config: $config" } lifecycleCoordinator.start() - taskManager.execute(TaskType.LONG_RUNNING, ::run) + taskManager.executeLongRunningTask(::run) } private fun stop() = Thread.currentThread().interrupt() diff --git a/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/TaskManagerHelper.kt b/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/TaskManagerHelper.kt index bc822ef7285..6e065c5190b 100644 --- a/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/TaskManagerHelper.kt +++ b/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/TaskManagerHelper.kt @@ -6,11 +6,10 @@ import net.corda.messagebus.api.consumer.CordaConsumerRecord import net.corda.messaging.api.mediator.MediatorMessage import net.corda.messaging.api.mediator.MessageRouter import net.corda.messaging.api.mediator.MessagingClient.Companion.MSG_PROP_KEY -import net.corda.messaging.api.mediator.taskmanager.TaskManager -import net.corda.messaging.api.mediator.taskmanager.TaskType import net.corda.messaging.api.processor.StateAndEventProcessor import net.corda.messaging.api.records.Record import net.corda.messaging.utils.toRecord +import net.corda.taskmanager.TaskManager /** * Helper that creates and executes various tasks used by [MultiSourceEventMediatorImpl]. @@ -99,7 +98,7 @@ internal class TaskManagerHelper( processorTasks: Collection> ): List> { return processorTasks.map { processorTask -> - taskManager.execute(TaskType.SHORT_RUNNING, processorTask::call) + taskManager.executeShortRunningTask(processorTask::call) }.map { it.join() } @@ -116,30 +115,32 @@ internal class TaskManagerHelper( fun createClientTasks( processorTaskResults: List>, messageRouter: MessageRouter, - ): List> { - return processorTaskResults.map { result -> - result.outputEvents.map { event -> + ): Map>> { + return processorTaskResults.associate { result -> + result.key to result.outputEvents.map { event -> ClientTask( event.toMessage(), messageRouter, result, ) } - }.flatten() + } } /** * Executes given [ClientTask]s and waits for all to finish. * - * @param clientTasks Tasks to execute. + * @param clientTaskMap Tasks to execute. * @return Result of task executions. */ fun executeClientTasks( - clientTasks: Collection> + clientTaskMap: Map>> ): List> { - return clientTasks.map { clientTask -> - taskManager.execute(TaskType.SHORT_RUNNING, clientTask::call) - }.map { + return clientTaskMap.map { (_, clientTasks) -> + taskManager.executeShortRunningTask { + clientTasks.map { it.call() } + } + }.flatMap { it.join() } } diff --git a/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/factory/MultiSourceEventMediatorFactoryImpl.kt b/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/factory/MultiSourceEventMediatorFactoryImpl.kt index fb160de51a5..d9a34c6a071 100644 --- a/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/factory/MultiSourceEventMediatorFactoryImpl.kt +++ b/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/factory/MultiSourceEventMediatorFactoryImpl.kt @@ -6,24 +6,35 @@ import net.corda.lifecycle.LifecycleCoordinatorFactory import net.corda.messaging.api.mediator.MultiSourceEventMediator import net.corda.messaging.api.mediator.config.EventMediatorConfig import net.corda.messaging.api.mediator.factory.MultiSourceEventMediatorFactory -import net.corda.messaging.api.mediator.taskmanager.TaskManager import net.corda.messaging.mediator.MultiSourceEventMediatorImpl +import net.corda.taskmanager.TaskManagerFactory import org.osgi.service.component.annotations.Activate import org.osgi.service.component.annotations.Component import org.osgi.service.component.annotations.Reference @Component(service = [MultiSourceEventMediatorFactory::class]) -class MultiSourceEventMediatorFactoryImpl @Activate constructor( - @Reference(service = CordaAvroSerializationFactory::class) +class MultiSourceEventMediatorFactoryImpl( private val cordaAvroSerializationFactory: CordaAvroSerializationFactory, - @Reference(service = TaskManager::class) - private val taskManager: TaskManager, - @Reference(service = StateManager::class) private val stateManager: StateManager, - @Reference(service = LifecycleCoordinatorFactory::class) private val lifecycleCoordinatorFactory: LifecycleCoordinatorFactory, + private val taskManagerFactory: TaskManagerFactory ): MultiSourceEventMediatorFactory { + @Activate + constructor( + @Reference(service = CordaAvroSerializationFactory::class) + cordaAvroSerializationFactory: CordaAvroSerializationFactory, + @Reference(service = StateManager::class) + stateManager: StateManager, + @Reference(service = LifecycleCoordinatorFactory::class) + lifecycleCoordinatorFactory: LifecycleCoordinatorFactory + ) : this( + cordaAvroSerializationFactory, + stateManager, + lifecycleCoordinatorFactory, + TaskManagerFactory.INSTANCE + ) + override fun create( eventMediatorConfig: EventMediatorConfig, ): MultiSourceEventMediator { @@ -37,7 +48,11 @@ class MultiSourceEventMediatorFactoryImpl @Activate constructor( stateSerializer, stateDeserializer, stateManager, - taskManager, + taskManagerFactory.createThreadPoolTaskManager( + name = eventMediatorConfig.name, + threadName = eventMediatorConfig.threadName, + threads = eventMediatorConfig.threads + ), lifecycleCoordinatorFactory, ) } diff --git a/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/statemanager/StateManagerImpl.kt b/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/statemanager/StateManagerImpl.kt index 3210bff48fe..a8d2823977d 100644 --- a/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/statemanager/StateManagerImpl.kt +++ b/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/statemanager/StateManagerImpl.kt @@ -6,11 +6,12 @@ import net.corda.libs.statemanager.api.State import net.corda.libs.statemanager.api.StateManager import org.osgi.service.component.annotations.Activate import org.osgi.service.component.annotations.Component +import java.util.concurrent.ConcurrentHashMap // TODO This is used temporarily until State Manager implementation is finished @Component(service = [StateManager::class]) class StateManagerImpl @Activate constructor() : StateManager { - private val storage = mutableMapOf() + private val storage = ConcurrentHashMap() override fun create(states: Collection): Map { return states.mapNotNull { @@ -24,14 +25,16 @@ class StateManagerImpl @Activate constructor() : StateManager { override fun update(states: Collection): Map { return states.mapNotNull { - val existingState = storage[it.key] - if (existingState?.version == it.version) { - val updatedState = it.copy(version = it.version + 1) - storage[it.key] = updatedState - null - } else { - it + var output: State? = null + storage.compute(it.key) { _, existingState -> + if (existingState?.version == it.version) { + it.copy(version = it.version + 1) + } else { + output = it + it + } } + output }.associateBy { it.key } } diff --git a/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/taskmanager/TaskManagerImpl.kt b/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/taskmanager/TaskManagerImpl.kt deleted file mode 100644 index 305bfb0e752..00000000000 --- a/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/taskmanager/TaskManagerImpl.kt +++ /dev/null @@ -1,58 +0,0 @@ -package net.corda.messaging.mediator.taskmanager - -import net.corda.messaging.api.mediator.taskmanager.TaskManager -import net.corda.messaging.api.mediator.taskmanager.TaskType -import org.osgi.service.component.annotations.Activate -import org.osgi.service.component.annotations.Component -import org.slf4j.LoggerFactory -import java.util.UUID -import java.util.concurrent.CompletableFuture -import java.util.concurrent.Executors -import kotlin.concurrent.thread - -// TODO This is used temporarily until Task Manager implementation is finished -@Component(service = [TaskManager::class]) -class TaskManagerImpl @Activate constructor() : TaskManager { - companion object { - private val log = LoggerFactory.getLogger(this::class.java.enclosingClass) - } - private var executorService = Executors.newSingleThreadExecutor() - - override fun execute(type: TaskType, command: () -> T) = - when (type) { - TaskType.SHORT_RUNNING -> executeShortRunning(command) - TaskType.LONG_RUNNING -> executeLongRunning(command) - } - - private fun executeShortRunning(command: () -> T): CompletableFuture { - val resultFuture = CompletableFuture() - try { - executorService.execute { - try { - resultFuture.complete(command()) - } catch (t: Throwable) { - log.error("Task error", t) - resultFuture.completeExceptionally(t) - } - } - } catch (t: Throwable) { - log.error("Executor error", t) - } - return resultFuture - } - - private fun executeLongRunning(command: () -> T): CompletableFuture { - val uniqueId = UUID.randomUUID() - val result = CompletableFuture() - thread( - start = true, - isDaemon = true, - contextClassLoader = null, - name = "Task Manager - $uniqueId", - priority = -1, - ) { - result.complete(command()) - } - return result - } -} \ No newline at end of file diff --git a/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/subscription/StateAndEventSubscriptionImpl.kt b/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/subscription/StateAndEventSubscriptionImpl.kt index c58ed7697c0..a489df829cb 100644 --- a/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/subscription/StateAndEventSubscriptionImpl.kt +++ b/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/subscription/StateAndEventSubscriptionImpl.kt @@ -380,4 +380,4 @@ internal class StateAndEventSubscriptionImpl( throw CordaMessageAPIIntermittentException(message, ex) } } -} +} \ No newline at end of file diff --git a/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/subscription/consumer/StateAndEventConsumerImpl.kt b/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/subscription/consumer/StateAndEventConsumerImpl.kt index b23e9d8b8fe..485d2fdcdcd 100644 --- a/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/subscription/consumer/StateAndEventConsumerImpl.kt +++ b/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/subscription/consumer/StateAndEventConsumerImpl.kt @@ -374,7 +374,7 @@ internal class StateAndEventConsumerImpl( //will never be null, created on assignment in rebalance listener val currentStatesByPartition = currentStates[partitionId] ?: throw CordaMessageAPIFatalException("Current State map for " + - "group ${config.group} on topic $stateTopic[$partitionId] is null.") + "group ${config.group} on topic $stateTopic[$partitionId] is null.") updatedStatesByKey[key] = value if (value != null) { currentStatesByPartition[key] = Pair(clock.instant().toEpochMilli(), value) diff --git a/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/MultiSourceEventMediatorImplTest.kt b/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/MultiSourceEventMediatorImplTest.kt index 6eedc2a5d59..5e08607170f 100644 --- a/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/MultiSourceEventMediatorImplTest.kt +++ b/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/MultiSourceEventMediatorImplTest.kt @@ -18,9 +18,9 @@ import net.corda.messaging.api.mediator.factory.MediatorConsumerFactory import net.corda.messaging.api.mediator.factory.MessageRouterFactory import net.corda.messaging.api.mediator.factory.MessagingClientFactory import net.corda.messaging.api.mediator.factory.MessagingClientFinder -import net.corda.messaging.api.mediator.taskmanager.TaskManager import net.corda.messaging.api.processor.StateAndEventProcessor import net.corda.messaging.api.records.Record +import net.corda.taskmanager.TaskManager import net.corda.test.util.waitWhile import org.junit.jupiter.api.BeforeEach import org.mockito.kotlin.any @@ -89,7 +89,7 @@ class MultiSourceEventMediatorImplTest { whenever(stateSerializer.serialize(any())).thenAnswer { ByteArray(0) } - whenever(taskManager.execute(any(), any<() -> Any>())).thenAnswer { invocation -> + whenever(taskManager.executeLongRunningTask (any<() -> Any>())).thenAnswer { invocation -> val command = invocation.getArgument<() -> Any>(1) CompletableFuture.supplyAsync(command) } diff --git a/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/TaskManagerHelperTest.kt b/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/TaskManagerHelperTest.kt index 5a664098766..fd81671121f 100644 --- a/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/TaskManagerHelperTest.kt +++ b/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/TaskManagerHelperTest.kt @@ -5,9 +5,10 @@ import net.corda.messagebus.api.consumer.CordaConsumerRecord import net.corda.messaging.api.mediator.MediatorMessage import net.corda.messaging.api.mediator.MessageRouter import net.corda.messaging.api.mediator.MessagingClient.Companion.MSG_PROP_KEY -import net.corda.messaging.api.mediator.taskmanager.TaskManager import net.corda.messaging.api.processor.StateAndEventProcessor import net.corda.messaging.api.records.Record +import net.corda.taskmanager.TaskManager +import org.assertj.core.api.Assertions.assertThat import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Test import org.mockito.Mockito.`when` @@ -16,6 +17,8 @@ import org.mockito.kotlin.argumentCaptor import org.mockito.kotlin.mock import org.mockito.kotlin.times import org.mockito.kotlin.verify +import org.mockito.kotlin.whenever +import java.util.concurrent.CompletableFuture class TaskManagerHelperTest { private companion object { @@ -164,27 +167,33 @@ class TaskManagerHelperTest { val processorTask1 = mock>() val processorTask2 = mock>() - `when`(taskManager.execute(any(), any<() -> ProcessorTask.Result>())).thenReturn(mock()) + `when`(taskManager.executeShortRunningTask(any<() -> ProcessorTask.Result>())).thenReturn(mock()) taskManagerHelper.executeProcessorTasks( listOf(processorTask1, processorTask2) ) val commandCaptor = argumentCaptor<() -> ProcessorTask.Result>() - verify(taskManager, times(2)).execute(any(), commandCaptor.capture()) + verify(taskManager, times(2)).executeShortRunningTask(commandCaptor.capture()) assertEquals(processorTask1::call, commandCaptor.firstValue) assertEquals(processorTask2::call, commandCaptor.secondValue) } @Test fun `successfully creates client tasks from message processor tasks`() { + val key1 = "key1" + val key2 = "key2" val processorTaskResult1 = ProcessorTask.Result( - mock>(), + mock>().apply { + whenever(this.key).thenReturn(key1) + }, outputEvents = listOf(EVENT1).toRecords(KEY1), mock(), ) val processorTaskResult2 = ProcessorTask.Result( - mock>(), + mock>().apply { + whenever(this.key).thenReturn(key2) + }, outputEvents = listOf(EVENT2, EVENT3).toRecords(KEY2), mock(), ) @@ -195,21 +204,25 @@ class TaskManagerHelperTest { messageRouter, ) - val expectedClientTasks = listOf( - ClientTask( - MediatorMessage(EVENT1, mutableMapOf(MSG_PROP_KEY to KEY1)), - messageRouter, - processorTaskResult1, + val expectedClientTasks = mapOf( + key1 to listOf( + ClientTask( + MediatorMessage(EVENT1, mutableMapOf(MSG_PROP_KEY to KEY1)), + messageRouter, + processorTaskResult1, + ) ), - ClientTask( - MediatorMessage(EVENT2, mutableMapOf(MSG_PROP_KEY to KEY2)), - messageRouter, - processorTaskResult2, - ), - ClientTask( - MediatorMessage(EVENT3, mutableMapOf(MSG_PROP_KEY to KEY2)), - messageRouter, - processorTaskResult2, + key2 to listOf( + ClientTask( + MediatorMessage(EVENT2, mutableMapOf(MSG_PROP_KEY to KEY2)), + messageRouter, + processorTaskResult2, + ), + ClientTask( + MediatorMessage(EVENT3, mutableMapOf(MSG_PROP_KEY to KEY2)), + messageRouter, + processorTaskResult2, + ) ), ) assertEquals(expectedClientTasks, clientTasks) @@ -219,17 +232,23 @@ class TaskManagerHelperTest { fun `successfully executes client tasks`() { val clientTask1 = mock>() val clientTask2 = mock>() + val result1 = ClientTask.Result(clientTask1, null) + val result2 = ClientTask.Result(clientTask2, null) + val future1 = mock>>>() + val future2 = mock>>>() + whenever(future1.join()).thenReturn(listOf(result1)) + whenever(future2.join()).thenReturn(listOf(result2)) - `when`(taskManager.execute(any(), any<() -> ClientTask.Result>())).thenReturn(mock()) - - taskManagerHelper.executeClientTasks( - listOf(clientTask1, clientTask2) + `when`(taskManager.executeShortRunningTask(any<() -> List>>())).thenReturn( + future1, + future2 ) - val commandCaptor = argumentCaptor<() -> ClientTask.Result>() - verify(taskManager, times(2)).execute(any(), commandCaptor.capture()) - assertEquals(clientTask1::call, commandCaptor.firstValue) - assertEquals(clientTask2::call, commandCaptor.secondValue) + val results = taskManagerHelper.executeClientTasks( + mapOf("1" to listOf(clientTask1), "2" to listOf(clientTask2)) + ) + assertThat(results).containsOnly(result1, result2) + verify(taskManager, times(2)).executeShortRunningTask(any<() -> List>>()) } private fun List.toCordaConsumerRecords(key: String) = diff --git a/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/factory/MultiSourceEventMediatorFactoryTest.kt b/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/factory/MultiSourceEventMediatorFactoryTest.kt index 2d6df3d6baf..41654a23a9e 100644 --- a/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/factory/MultiSourceEventMediatorFactoryTest.kt +++ b/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/factory/MultiSourceEventMediatorFactoryTest.kt @@ -7,8 +7,9 @@ import net.corda.libs.statemanager.api.StateManager import net.corda.lifecycle.LifecycleCoordinatorFactory import net.corda.messaging.api.mediator.config.EventMediatorConfig import net.corda.messaging.api.mediator.factory.MessageRouterFactory -import net.corda.messaging.api.mediator.taskmanager.TaskManager import net.corda.messaging.api.processor.StateAndEventProcessor +import net.corda.taskmanager.TaskManager +import net.corda.taskmanager.TaskManagerFactory import org.junit.jupiter.api.Assertions import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Test @@ -22,16 +23,18 @@ class MultiSourceEventMediatorFactoryTest { private val cordaAvroSerializationFactory = mock() private val serializer = mock>() private val stateDeserializer = mock>() + private val taskManagerFactory = mock() @BeforeEach fun beforeEach() { doReturn(serializer).`when`(cordaAvroSerializationFactory).createAvroSerializer(anyOrNull()) doReturn(stateDeserializer).`when`(cordaAvroSerializationFactory).createAvroDeserializer(any(), any>()) + doReturn(mock()).`when`(taskManagerFactory).createThreadPoolTaskManager(any(), any(), any()) multiSourceEventMediatorFactory = MultiSourceEventMediatorFactoryImpl( cordaAvroSerializationFactory, - mock(), mock(), mock(), + taskManagerFactory, ) } @@ -43,6 +46,9 @@ class MultiSourceEventMediatorFactoryTest { val config = mock>() doReturn(messageProcessor).`when`(config).messageProcessor doReturn(messageRouterFactory).`when`(config).messageRouterFactory + doReturn("name").`when`(config).name + doReturn(1).`when`(config).threads + doReturn("name").`when`(config).threadName val mediator = multiSourceEventMediatorFactory.create(config) diff --git a/libs/messaging/messaging/src/main/kotlin/net/corda/messaging/api/mediator/config/EventMediatorConfig.kt b/libs/messaging/messaging/src/main/kotlin/net/corda/messaging/api/mediator/config/EventMediatorConfig.kt index b7b446e910c..627a90c26e8 100644 --- a/libs/messaging/messaging/src/main/kotlin/net/corda/messaging/api/mediator/config/EventMediatorConfig.kt +++ b/libs/messaging/messaging/src/main/kotlin/net/corda/messaging/api/mediator/config/EventMediatorConfig.kt @@ -29,6 +29,8 @@ data class EventMediatorConfig( val clientFactories: Collection, val messageProcessor : StateAndEventProcessor, val messageRouterFactory: MessageRouterFactory, + val threads: Int, + val threadName: String ) { /** * Timeout for polling consumers. diff --git a/libs/messaging/messaging/src/main/kotlin/net/corda/messaging/api/mediator/config/EventMediatorConfigBuilder.kt b/libs/messaging/messaging/src/main/kotlin/net/corda/messaging/api/mediator/config/EventMediatorConfigBuilder.kt index a36e3cff590..d5c32e6199e 100644 --- a/libs/messaging/messaging/src/main/kotlin/net/corda/messaging/api/mediator/config/EventMediatorConfigBuilder.kt +++ b/libs/messaging/messaging/src/main/kotlin/net/corda/messaging/api/mediator/config/EventMediatorConfigBuilder.kt @@ -22,6 +22,8 @@ class EventMediatorConfigBuilder { private var clientFactories = emptyArray() private var messageProcessor : StateAndEventProcessor? = null private var messageRouterFactory: MessageRouterFactory? = null + private var threads: Int? = null + private var threadName: String? = null /** Sets name for [MultiSourceEventMediator]. */ fun name(name: String) = @@ -47,21 +49,25 @@ class EventMediatorConfigBuilder { fun messageRouterFactory(messageRouterFactory: MessageRouterFactory) = apply { this.messageRouterFactory = messageRouterFactory } + fun threads(threads: Int) = + apply { this.threads = threads } + + fun threadName(threadName: String) = + apply { this.threadName = threadName } + /** Builds [EventMediatorConfig]. */ fun build(): EventMediatorConfig { - check(name != null) { "Name not set" } - check(messagingConfig != null) { "Messaging configuration not set" } check(consumerFactories.isNotEmpty()) { "At least on consumer factory has to be set" } check(clientFactories.isNotEmpty()) { "At least on messaging client factory has to be set" } - check(messageProcessor != null) { "Message processor not set" } - check(messageRouterFactory != null) { "Message router factory not set" } return EventMediatorConfig( - name!!, - messagingConfig!!, + name = checkNotNull(name) { "Name not set" }, + messagingConfig = checkNotNull(messagingConfig) { "Messaging configuration not set" }, consumerFactories.asList(), clientFactories.asList(), - messageProcessor!!, - messageRouterFactory!! + messageProcessor = checkNotNull(messageProcessor) { "Message processor not set" }, + messageRouterFactory = checkNotNull(messageRouterFactory) { "Message router factory not set" }, + threads = checkNotNull(threads) { "Number of threads not set" }, + threadName = checkNotNull(threadName) { "Thread name not set" } ) } } \ No newline at end of file diff --git a/libs/messaging/messaging/src/main/kotlin/net/corda/messaging/api/mediator/taskmanager/TaskManager.kt b/libs/messaging/messaging/src/main/kotlin/net/corda/messaging/api/mediator/taskmanager/TaskManager.kt deleted file mode 100644 index 01fe5a75863..00000000000 --- a/libs/messaging/messaging/src/main/kotlin/net/corda/messaging/api/mediator/taskmanager/TaskManager.kt +++ /dev/null @@ -1,12 +0,0 @@ -package net.corda.messaging.api.mediator.taskmanager - -import java.util.concurrent.CompletableFuture - -enum class TaskType { - SHORT_RUNNING, LONG_RUNNING -} - -// TODO This is used temporarily until Task Manager implementation is finished -interface TaskManager { - fun execute(type: TaskType, command: () -> T): CompletableFuture -} \ No newline at end of file diff --git a/libs/metrics/src/main/kotlin/net/corda/metrics/CordaMetrics.kt b/libs/metrics/src/main/kotlin/net/corda/metrics/CordaMetrics.kt index 1caecef4e7b..8ae931e9078 100644 --- a/libs/metrics/src/main/kotlin/net/corda/metrics/CordaMetrics.kt +++ b/libs/metrics/src/main/kotlin/net/corda/metrics/CordaMetrics.kt @@ -661,6 +661,18 @@ object CordaMetrics { computation ) } + + object TaskManager { + /** + * Time it took to execute a task, includes time waiting to be scheduled. + */ + object TaskCompletionTime : Metric("taskmanager.completion.time", CordaMetrics::timer) + + /** + * The number of live tasks running or scheduled in the task manager. + */ + class LiveTasks(computation: Supplier) : ComputedValue("taskmanager.live.tasks", computation) + } } /** @@ -797,6 +809,16 @@ object CordaMetrics { */ SignatureSpec("signature.spec"), + /** + * Task manager name. + */ + TaskManagerName("task.manager.name"), + + /** + * Task type. + */ + TaskType("task.type"), + /** * Identifier of a tenant either a virtual node identifier or cluster level tenant id. */ diff --git a/libs/serialization/serialization-kryo/src/main/kotlin/net/corda/kryoserialization/KryoCheckpointSerializer.kt b/libs/serialization/serialization-kryo/src/main/kotlin/net/corda/kryoserialization/KryoCheckpointSerializer.kt index 4996a467e53..c18dab9b82d 100644 --- a/libs/serialization/serialization-kryo/src/main/kotlin/net/corda/kryoserialization/KryoCheckpointSerializer.kt +++ b/libs/serialization/serialization-kryo/src/main/kotlin/net/corda/kryoserialization/KryoCheckpointSerializer.kt @@ -1,12 +1,13 @@ package net.corda.kryoserialization import com.esotericsoftware.kryo.Kryo +import com.esotericsoftware.kryo.util.Pool import net.corda.serialization.checkpoint.CheckpointSerializer import org.slf4j.LoggerFactory import java.io.ByteArrayInputStream class KryoCheckpointSerializer( - private val kryo: Kryo, + private val kryoPool: Pool, ) : CheckpointSerializer { private companion object { @@ -20,7 +21,13 @@ class KryoCheckpointSerializer( return try { kryoInput(ByteArrayInputStream(bytes)) { @Suppress("unchecked_cast") - kryo.readClassAndObject(this) as T + kryoPool.obtain().let { kryo -> + try { + kryo.readClassAndObject(this) as T + } finally { + kryoPool.free(kryo) + } + } } } catch (ex: Exception) { log.error("Failed to deserialize bytes", ex) @@ -31,7 +38,13 @@ class KryoCheckpointSerializer( override fun serialize(obj: T): ByteArray { return try { kryoOutput { - kryo.writeClassAndObject(this, obj) + kryoPool.obtain().let { kryo -> + try { + kryo.writeClassAndObject(this, obj) + } finally { + kryoPool.free(kryo) + } + } } } catch (ex: Exception) { log.error("Failed to serialize", ex) diff --git a/libs/serialization/serialization-kryo/src/main/kotlin/net/corda/kryoserialization/impl/KryoCheckpointSerializerBuilderImpl.kt b/libs/serialization/serialization-kryo/src/main/kotlin/net/corda/kryoserialization/impl/KryoCheckpointSerializerBuilderImpl.kt index e235721110a..ebd8bdcb00b 100644 --- a/libs/serialization/serialization-kryo/src/main/kotlin/net/corda/kryoserialization/impl/KryoCheckpointSerializerBuilderImpl.kt +++ b/libs/serialization/serialization-kryo/src/main/kotlin/net/corda/kryoserialization/impl/KryoCheckpointSerializerBuilderImpl.kt @@ -5,6 +5,7 @@ import co.paralleluniverse.io.serialization.kryo.KryoSerializer import com.esotericsoftware.kryo.ClassResolver import com.esotericsoftware.kryo.Kryo import com.esotericsoftware.kryo.Serializer +import com.esotericsoftware.kryo.util.Pool import net.corda.crypto.cipher.suite.KeyEncodingService import net.corda.kryoserialization.CordaKryoException import net.corda.kryoserialization.DefaultKryoCustomizer @@ -69,9 +70,6 @@ class KryoCheckpointSerializerBuilderImpl( } override fun build(): KryoCheckpointSerializer { - val classResolver = CordaClassResolver(sandboxGroup) - val classSerializer = ClassSerializer(sandboxGroup) - val publicKeySerializers = listOf( PublicKey::class.java, EdDSAPublicKey::class.java, CompositeKey::class.java, BCECPublicKey::class.java, BCRSAPublicKey::class.java, BCSphincs256PublicKey::class.java @@ -82,16 +80,20 @@ class KryoCheckpointSerializerBuilderImpl( X500Principal::class.java to X500PrincipalSerializer() ) - val kryo = DefaultKryoCustomizer.customize( - kryoFactory.apply(classResolver), - serializers + publicKeySerializers + otherCustomSerializers, - classSerializer - ) - - return KryoCheckpointSerializer(kryo).also { - // Clear the builder state - serializers.clear() - singletonInstances.clear() + val pool = object : Pool(true, false, 8) { + override fun create(): Kryo { + val classResolver = CordaClassResolver(sandboxGroup) + val classSerializer = ClassSerializer(sandboxGroup) + return DefaultKryoCustomizer.customize( + kryoFactory.apply(classResolver), + serializers + publicKeySerializers + otherCustomSerializers, + classSerializer + ).also { + classResolver.setKryo(it) + } + } } + + return KryoCheckpointSerializer(pool) } } diff --git a/libs/serialization/serialization-kryo/src/test/kotlin/net/corda/kryoserialization/KryoCheckpointSerializerTest.kt b/libs/serialization/serialization-kryo/src/test/kotlin/net/corda/kryoserialization/KryoCheckpointSerializerTest.kt index b6356f350d6..837a0ca2272 100644 --- a/libs/serialization/serialization-kryo/src/test/kotlin/net/corda/kryoserialization/KryoCheckpointSerializerTest.kt +++ b/libs/serialization/serialization-kryo/src/test/kotlin/net/corda/kryoserialization/KryoCheckpointSerializerTest.kt @@ -6,6 +6,7 @@ import co.paralleluniverse.io.serialization.kryo.KryoSerializer import com.esotericsoftware.kryo.ClassResolver import com.esotericsoftware.kryo.Kryo import com.esotericsoftware.kryo.util.MapReferenceResolver +import com.esotericsoftware.kryo.util.Pool import net.corda.data.flow.state.checkpoint.FlowStackItem import net.corda.kryoserialization.KryoCheckpointSerializerTest.SerializableFunction import net.corda.kryoserialization.TestClass.Companion.TEST_INT @@ -78,7 +79,7 @@ class KryoCheckpointSerializerTest { }, emptyMap(), ClassSerializer(sandboxGroup) - ) + ).toPool() ) assertThatThrownBy { serializer.serialize(FlowStackItem()) } @@ -117,7 +118,7 @@ class KryoCheckpointSerializerTest { Kryo(CordaClassResolver(sandboxGroup), MapReferenceResolver()), emptyMap(), ClassSerializer(sandboxGroup) - ) + ).toPool() ) val tester = TestClass(TEST_INT, TEST_STRING) @@ -140,7 +141,7 @@ class KryoCheckpointSerializerTest { kryo = getQuasarKryo(CordaClassResolver(sandboxGroup)), serializers = emptyMap(), classSerializer = ClassSerializer(sandboxGroup) - ) + ).toPool() ) val tester = Instant.now() @@ -160,7 +161,7 @@ class KryoCheckpointSerializerTest { kryo = getQuasarKryo(CordaClassResolver(sandboxGroup)), serializers = emptyMap(), classSerializer = ClassSerializer(sandboxGroup) - ) + ).toPool() ) val tester = Chronology.getAvailableChronologies().first() @@ -178,7 +179,7 @@ class KryoCheckpointSerializerTest { kryo = getQuasarKryo(CordaClassResolver(sandboxGroup)), serializers = emptyMap(), classSerializer = ClassSerializer(sandboxGroup) - ) + ).toPool() ) val tester = ZoneRules.of(ZoneOffset.UTC) @@ -198,7 +199,7 @@ class KryoCheckpointSerializerTest { kryo = getQuasarKryo(CordaClassResolver(sandboxGroup)), serializers = emptyMap(), classSerializer = ClassSerializer(sandboxGroup) - ) + ).toPool() ) val tester = Function { x -> "Hello $x, hash=${x.hashCode()}" } @@ -219,7 +220,7 @@ class KryoCheckpointSerializerTest { kryo = getQuasarKryo(CordaClassResolver(sandboxGroup)), serializers = emptyMap(), classSerializer = ClassSerializer(sandboxGroup) - ) + ).toPool() ) val obj = LambdaField("Something Extra") @@ -246,7 +247,7 @@ class KryoCheckpointSerializerTest { kryo = getQuasarKryo(CordaClassResolver(sandboxGroup)), serializers = emptyMap(), classSerializer = ClassSerializer(sandboxGroup) - ) + ).toPool() ) val obj = LambdaField("Something Extra") @@ -300,7 +301,7 @@ class KryoCheckpointSerializerTest { runTestWithCollection(LinkedList()) } - private data class TestClassWithIterator(val list: C, val iterator: I) + private data class TestClassWithIterator(val list: C, val iterator: I) private fun runTestWithCollection(collection: MutableCollection) { @@ -310,7 +311,7 @@ class KryoCheckpointSerializerTest { Kryo(CordaClassResolver(sandboxGroup), MapReferenceResolver()), emptyMap(), ClassSerializer(sandboxGroup) - ) + ).toPool() ) for (i in 1..20) { @@ -341,7 +342,7 @@ class KryoCheckpointSerializerTest { Kryo(CordaClassResolver(sandboxGroup), MapReferenceResolver()), emptyMap(), ClassSerializer(sandboxGroup) - ) + ).toPool() ) for (i in 1..20) { @@ -374,4 +375,12 @@ class KryoCheckpointSerializerTest { Class.forName(classCaptor.lastValue) } } + + private fun Kryo.toPool(): Pool { + return object : Pool(true, false, 4) { + override fun create(): Kryo { + return this@toPool + } + } + } } diff --git a/libs/serialization/serialization-kryo/src/test/kotlin/net/corda/kryoserialization/serializers/SingletonSerializeAsTokenSerializerTest.kt b/libs/serialization/serialization-kryo/src/test/kotlin/net/corda/kryoserialization/serializers/SingletonSerializeAsTokenSerializerTest.kt index 72e495e881d..e2061325672 100644 --- a/libs/serialization/serialization-kryo/src/test/kotlin/net/corda/kryoserialization/serializers/SingletonSerializeAsTokenSerializerTest.kt +++ b/libs/serialization/serialization-kryo/src/test/kotlin/net/corda/kryoserialization/serializers/SingletonSerializeAsTokenSerializerTest.kt @@ -2,6 +2,7 @@ package net.corda.kryoserialization.serializers import com.esotericsoftware.kryo.Kryo import com.esotericsoftware.kryo.util.MapReferenceResolver +import com.esotericsoftware.kryo.util.Pool import net.corda.kryoserialization.CordaKryoException import net.corda.kryoserialization.DefaultKryoCustomizer import net.corda.kryoserialization.KryoCheckpointSerializer @@ -39,7 +40,7 @@ class SingletonSerializeAsTokenSerializerTest { Kryo(CordaClassResolver(sandboxGroup), MapReferenceResolver()), mapOf(SingletonSerializeAsToken::class.java to SingletonSerializeAsTokenSerializer(emptyMap())), ClassSerializer(sandboxGroup) - ) + ).toPool() ) assertThatExceptionOfType(CordaKryoException::class.java).isThrownBy { @@ -59,7 +60,7 @@ class SingletonSerializeAsTokenSerializerTest { Kryo(CordaClassResolver(sandboxGroup), MapReferenceResolver()), emptyMap(), ClassSerializer(sandboxGroup) - ) + ).toPool() ) val bytes = serializer.serialize(instance) @@ -68,4 +69,11 @@ class SingletonSerializeAsTokenSerializerTest { } } + private fun Kryo.toPool(): Pool { + return object : Pool(true, false, 4) { + override fun create(): Kryo { + return this@toPool + } + } + } } diff --git a/libs/task-manager/build.gradle b/libs/task-manager/build.gradle new file mode 100644 index 00000000000..92c44653b1f --- /dev/null +++ b/libs/task-manager/build.gradle @@ -0,0 +1,20 @@ +plugins { + id 'corda.common-publishing' + id 'corda.common-library' +} + +description 'Task manager' + +dependencies { + compileOnly 'org.osgi:osgi.annotation' + + implementation platform("net.corda:corda-api:$cordaApiVersion") + implementation project(":libs:metrics") + implementation project(":libs:utilities") + + implementation 'org.jetbrains.kotlin:kotlin-osgi-bundle' + + testImplementation "org.mockito:mockito-core:$mockitoVersion" + testImplementation "org.mockito.kotlin:mockito-kotlin:$mockitoKotlinVersion" + testImplementation "org.assertj:assertj-core:$assertjVersion" +} diff --git a/libs/task-manager/src/main/java/net/corda/taskmanager/package-info.java b/libs/task-manager/src/main/java/net/corda/taskmanager/package-info.java new file mode 100644 index 00000000000..db04d532c0d --- /dev/null +++ b/libs/task-manager/src/main/java/net/corda/taskmanager/package-info.java @@ -0,0 +1,4 @@ +@Export +package net.corda.taskmanager; + +import org.osgi.annotation.bundle.Export; diff --git a/libs/task-manager/src/main/kotlin/net/corda/taskmanager/TaskManager.kt b/libs/task-manager/src/main/kotlin/net/corda/taskmanager/TaskManager.kt new file mode 100644 index 00000000000..70563314f03 --- /dev/null +++ b/libs/task-manager/src/main/kotlin/net/corda/taskmanager/TaskManager.kt @@ -0,0 +1,13 @@ +package net.corda.taskmanager + +import java.util.concurrent.CompletableFuture +import java.util.concurrent.Executor +import java.util.concurrent.TimeUnit + +interface TaskManager : Executor { + fun executeShortRunningTask(command: () -> T): CompletableFuture + fun executeLongRunningTask(command: () -> T): CompletableFuture + fun executeScheduledTask(command: () -> T, delay: Long, unit: TimeUnit): CompletableFuture + + fun shutdown(): CompletableFuture +} \ No newline at end of file diff --git a/libs/task-manager/src/main/kotlin/net/corda/taskmanager/TaskManagerFactory.kt b/libs/task-manager/src/main/kotlin/net/corda/taskmanager/TaskManagerFactory.kt new file mode 100644 index 00000000000..1aaf25a78fc --- /dev/null +++ b/libs/task-manager/src/main/kotlin/net/corda/taskmanager/TaskManagerFactory.kt @@ -0,0 +1,16 @@ +package net.corda.taskmanager + +import net.corda.taskmanager.impl.TaskManagerFactoryImpl + +interface TaskManagerFactory { + + companion object { + val INSTANCE: TaskManagerFactory = TaskManagerFactoryImpl + } + + fun createThreadPoolTaskManager( + name: String, + threadName: String, + threads: Int, + ): TaskManager +} \ No newline at end of file diff --git a/libs/task-manager/src/main/kotlin/net/corda/taskmanager/impl/CordaExecutorServiceWrapper.kt b/libs/task-manager/src/main/kotlin/net/corda/taskmanager/impl/CordaExecutorServiceWrapper.kt new file mode 100644 index 00000000000..9dd23c2e114 --- /dev/null +++ b/libs/task-manager/src/main/kotlin/net/corda/taskmanager/impl/CordaExecutorServiceWrapper.kt @@ -0,0 +1,13 @@ +package net.corda.taskmanager.impl + +import io.micrometer.core.instrument.MeterRegistry +import io.micrometer.core.instrument.binder.jvm.ExecutorServiceMetrics +import java.util.concurrent.ScheduledExecutorService + +internal class CordaExecutorServiceWrapper( + private val name: String, + private val metricPrefix: String, + private val executor: ScheduledExecutorService, + private val registry: MeterRegistry, + private val delegate: ScheduledExecutorService = ExecutorServiceMetrics.monitor(registry, executor, name, metricPrefix) +) : ScheduledExecutorService by delegate \ No newline at end of file diff --git a/libs/task-manager/src/main/kotlin/net/corda/taskmanager/impl/TaskManagerFactoryImpl.kt b/libs/task-manager/src/main/kotlin/net/corda/taskmanager/impl/TaskManagerFactoryImpl.kt new file mode 100644 index 00000000000..bf491814045 --- /dev/null +++ b/libs/task-manager/src/main/kotlin/net/corda/taskmanager/impl/TaskManagerFactoryImpl.kt @@ -0,0 +1,43 @@ +package net.corda.taskmanager.impl + +import net.corda.metrics.CordaMetrics +import net.corda.taskmanager.TaskManager +import net.corda.taskmanager.TaskManagerFactory +import java.util.Locale +import java.util.concurrent.Executors +import java.util.concurrent.ThreadFactory +import java.util.concurrent.atomic.AtomicLong + +internal object TaskManagerFactoryImpl : TaskManagerFactory { + + override fun createThreadPoolTaskManager( + name: String, + threadName: String, + threads: Int, + ): TaskManager { + return TaskManagerImpl( + name = name, + longRunningThreadName = "$threadName-long-running-thread", + executorService = CordaExecutorServiceWrapper( + name, + "corda.taskmanager.", + Executors.newScheduledThreadPool( + threads, + threadFactory(threadName) + ), + CordaMetrics.registry + ) + ) + } + + private fun threadFactory(threadName: String): ThreadFactory { + val backingThreadFactory = Executors.defaultThreadFactory() + val count = AtomicLong(0) + return ThreadFactory { runnable -> + backingThreadFactory.newThread(runnable).apply { + setName(String.format(Locale.ROOT, "$threadName-thread-%d", count.getAndIncrement())) + setDaemon(false) + } + } + } +} \ No newline at end of file diff --git a/libs/task-manager/src/main/kotlin/net/corda/taskmanager/impl/TaskManagerImpl.kt b/libs/task-manager/src/main/kotlin/net/corda/taskmanager/impl/TaskManagerImpl.kt new file mode 100644 index 00000000000..5cea2ea799e --- /dev/null +++ b/libs/task-manager/src/main/kotlin/net/corda/taskmanager/impl/TaskManagerImpl.kt @@ -0,0 +1,132 @@ +package net.corda.taskmanager.impl + +import io.micrometer.core.instrument.Timer +import net.corda.metrics.CordaMetrics +import net.corda.taskmanager.TaskManager +import net.corda.utilities.VisibleForTesting +import java.util.UUID +import java.util.concurrent.CompletableFuture +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.ScheduledExecutorService +import java.util.concurrent.TimeUnit +import kotlin.concurrent.thread + +internal class TaskManagerImpl( + private val name: String, + private val longRunningThreadName: String, + private val executorService: ScheduledExecutorService +) : TaskManager { + + enum class Type { + SHORT_RUNNING, LONG_RUNNING, SCHEDULED + } + + @VisibleForTesting + val liveTaskCounts = ConcurrentHashMap() + + private val shortRunningTaskGauge = CordaMetrics.Metric.TaskManager.LiveTasks { liveTaskCounts[Type.SHORT_RUNNING] ?: 0 }.builder() + .withTag(CordaMetrics.Tag.TaskManagerName, name) + .withTag(CordaMetrics.Tag.TaskType, Type.SHORT_RUNNING.name) + .build() + + private val longRunningTaskGauge = CordaMetrics.Metric.TaskManager.LiveTasks { liveTaskCounts[Type.LONG_RUNNING] ?: 0 }.builder() + .withTag(CordaMetrics.Tag.TaskManagerName, name) + .withTag(CordaMetrics.Tag.TaskType, Type.LONG_RUNNING.name) + .build() + + private val scheduledTaskGauge = CordaMetrics.Metric.TaskManager.LiveTasks { liveTaskCounts[Type.SCHEDULED] ?: 0 }.builder() + .withTag(CordaMetrics.Tag.TaskManagerName, name) + .withTag(CordaMetrics.Tag.TaskType, Type.SCHEDULED.name) + .build() + + override fun executeShortRunningTask(command: () -> T): CompletableFuture { + val start = System.nanoTime() + incrementTaskCount(Type.SHORT_RUNNING) + return CompletableFuture.supplyAsync( + { command() }, + executorService + ).whenComplete { _, _ -> + taskCompletionMeter(Type.SHORT_RUNNING).record(System.nanoTime() - start, TimeUnit.NANOSECONDS) + decrementTaskCount(Type.SHORT_RUNNING) + } + } + + override fun executeLongRunningTask(command: () -> T): CompletableFuture { + val start = System.nanoTime() + val uniqueId = UUID.randomUUID() + val result = CompletableFuture() + incrementTaskCount(Type.LONG_RUNNING) + thread( + start = true, + isDaemon = true, + contextClassLoader = null, + name = "$longRunningThreadName-$uniqueId", + priority = -1, + ) { + try { + result.complete(command()) + } catch (t: Throwable) { + result.completeExceptionally(t) + } + } + return result.whenComplete { _, _ -> + taskCompletionMeter(Type.LONG_RUNNING).record(System.nanoTime() - start, TimeUnit.NANOSECONDS) + decrementTaskCount(Type.LONG_RUNNING) + } + } + + override fun executeScheduledTask(command: () -> T, delay: Long, unit: TimeUnit): CompletableFuture { + incrementTaskCount(Type.SCHEDULED) + val result = CompletableFuture() + executorService.schedule( + { + val start = System.nanoTime() + try { + result.complete(command()) + } catch (t: Throwable) { + result.completeExceptionally(t) + } finally { + // This recording only records the time that the task executed for, not the scheduled time since that seems weird + // for a scheduled task. Consider changing the other ones to not include the scheduling time? + taskCompletionMeter(Type.SCHEDULED).record(System.nanoTime() - start, TimeUnit.NANOSECONDS) + decrementTaskCount(Type.SCHEDULED) + } + }, + delay, + unit + ) + return result + } + + override fun execute(command: Runnable) { + executorService.execute(command) + } + + override fun shutdown(): CompletableFuture { + executorService.shutdown() + CordaMetrics.registry.remove(shortRunningTaskGauge) + CordaMetrics.registry.remove(longRunningTaskGauge) + CordaMetrics.registry.remove(scheduledTaskGauge) + // This [CompletableFuture] must not run on the executor service otherwise it'll never shut down. + // [runAsync] runs this task in the fork join common pool. + val shutdownFuture = CompletableFuture.runAsync { + executorService.awaitTermination(100, TimeUnit.SECONDS) + } + return shutdownFuture + } + + private fun incrementTaskCount(type: Type) { + liveTaskCounts.compute(type) { _, count -> if (count == null) 1 else count + 1 } + } + + private fun decrementTaskCount(type: Type) { + liveTaskCounts.computeIfPresent(type) { _, count -> count - 1 } + } + + private fun taskCompletionMeter(type: Type): Timer { + return CordaMetrics.Metric.TaskManager.TaskCompletionTime.builder() + .withTag(CordaMetrics.Tag.TaskManagerName, name) + .withTag(CordaMetrics.Tag.TaskType, type.name) + .build() + } +} \ No newline at end of file diff --git a/libs/task-manager/src/test/kotlin/net/corda/taskmanager/impl/TaskManagerImplTest.kt b/libs/task-manager/src/test/kotlin/net/corda/taskmanager/impl/TaskManagerImplTest.kt new file mode 100644 index 00000000000..f1c239be5dd --- /dev/null +++ b/libs/task-manager/src/test/kotlin/net/corda/taskmanager/impl/TaskManagerImplTest.kt @@ -0,0 +1,139 @@ +package net.corda.taskmanager.impl + +import org.assertj.core.api.Assertions.assertThat +import org.assertj.core.api.Assertions.assertThatThrownBy +import org.junit.jupiter.api.Test +import org.mockito.Mockito.mock +import org.mockito.kotlin.any +import org.mockito.kotlin.argumentCaptor +import org.mockito.kotlin.verify +import org.mockito.kotlin.whenever +import java.util.concurrent.ExecutionException +import java.util.concurrent.ScheduledExecutorService +import java.util.concurrent.ScheduledFuture +import java.util.concurrent.TimeUnit + +class TaskManagerImplTest { + + private companion object { + const val RESULT = 1 + } + + private val executorService = mock() + private val captor = argumentCaptor() + private val taskManager = TaskManagerImpl("", "", executorService) + + @Test + fun `executeShortRunningTask increments the task count, runs the task and decrements the task count when finished`() { + whenever(executorService.execute(captor.capture())).then { captor.firstValue.run() } + val result = taskManager.executeShortRunningTask { + assertThat(taskManager.liveTaskCounts).containsExactlyEntriesOf( + mapOf(TaskManagerImpl.Type.SHORT_RUNNING to 1) + ) + RESULT + } + assertThat(result.get()).isEqualTo(RESULT) + assertThat(taskManager.liveTaskCounts).containsExactlyEntriesOf( + mapOf(TaskManagerImpl.Type.SHORT_RUNNING to 0) + ) + } + + @Test + fun `executeShortRunningTask increments the task count and decrements the task count when the task fails`() { + whenever(executorService.execute(captor.capture())).then { captor.firstValue.run() } + val result = taskManager.executeShortRunningTask { + assertThat(taskManager.liveTaskCounts).containsExactlyEntriesOf( + mapOf(TaskManagerImpl.Type.SHORT_RUNNING to 1) + ) + throw IllegalStateException("fails") + } + assertThatThrownBy { result.get() } + .isExactlyInstanceOf(ExecutionException::class.java) + .hasCauseExactlyInstanceOf(IllegalStateException::class.java) + assertThat(taskManager.liveTaskCounts).containsExactlyEntriesOf( + mapOf(TaskManagerImpl.Type.SHORT_RUNNING to 0) + ) + } + + @Test + fun `executeLongRunningTask increments the task count, runs the task and decrements the task count when finished`() { + val result = taskManager.executeLongRunningTask { + assertThat(taskManager.liveTaskCounts).containsExactlyEntriesOf( + mapOf(TaskManagerImpl.Type.LONG_RUNNING to 1) + ) + RESULT + } + assertThat(result.get()).isEqualTo(RESULT) + assertThat(taskManager.liveTaskCounts).containsExactlyEntriesOf( + mapOf(TaskManagerImpl.Type.LONG_RUNNING to 0) + ) + } + + @Test + fun `executeLongRunningTask increments the task count and decrements the task count when the task fails`() { + val result = taskManager.executeLongRunningTask { + assertThat(taskManager.liveTaskCounts).containsExactlyEntriesOf( + mapOf(TaskManagerImpl.Type.LONG_RUNNING to 1) + ) + throw IllegalStateException("fails") + } + assertThatThrownBy { result.get() } + .isExactlyInstanceOf(ExecutionException::class.java) + .hasCauseExactlyInstanceOf(IllegalStateException::class.java) + assertThat(taskManager.liveTaskCounts).containsExactlyEntriesOf( + mapOf(TaskManagerImpl.Type.LONG_RUNNING to 0) + ) + } + + @Test + fun `executeScheduledTask increments the task count, runs the task and decrements the task count when finished`() { + whenever(executorService.schedule(captor.capture(), any(), any())).then { + captor.firstValue.run() + mock>() + } + val result = taskManager.executeScheduledTask( + { + assertThat(taskManager.liveTaskCounts).containsExactlyEntriesOf( + mapOf(TaskManagerImpl.Type.SCHEDULED to 1) + ) + RESULT + }, + 1, + TimeUnit.SECONDS + ) + assertThat(result.get()).isEqualTo(RESULT) + assertThat(taskManager.liveTaskCounts).containsExactlyEntriesOf( + mapOf(TaskManagerImpl.Type.SCHEDULED to 0) + ) + } + + @Test + fun `executeScheduledTask increments the task count and decrements the task count when the task fails`() { + whenever(executorService.schedule(captor.capture(), any(), any())).then { + captor.firstValue.run() + mock>() + } + val result = taskManager.executeScheduledTask( + { + assertThat(taskManager.liveTaskCounts).containsExactlyEntriesOf( + mapOf(TaskManagerImpl.Type.SCHEDULED to 1) + ) + throw IllegalStateException("fails") + }, + 1, + TimeUnit.SECONDS + ) + assertThatThrownBy { result.get() } + .isExactlyInstanceOf(ExecutionException::class.java) + .hasCauseExactlyInstanceOf(IllegalStateException::class.java) + assertThat(taskManager.liveTaskCounts).containsExactlyEntriesOf( + mapOf(TaskManagerImpl.Type.SCHEDULED to 0) + ) + } + + @Test + fun `shutdown terminates executor service`() { + taskManager.shutdown().get() + verify(executorService).awaitTermination(any(), any()) + } +} \ No newline at end of file diff --git a/settings.gradle b/settings.gradle index 9e8474b335a..49bd8f09a31 100644 --- a/settings.gradle +++ b/settings.gradle @@ -353,6 +353,7 @@ include 'libs:serialization:serialization-internal' include 'libs:serialization:serialization-kryo' include 'libs:serialization:serialization-kryo:cpks:serializable-cpk-one' include 'libs:serialization:serialization-kryo:cpks:serializable-cpk-two' +include 'libs:task-manager' include 'libs:uniqueness:common' include 'libs:utilities' include 'libs:state-manager:state-manager-api' From 88d17929f1f3f59f7d6127d0f19867586902bd63 Mon Sep 17 00:00:00 2001 From: Miljenko Brkic <97448832+mbrkic-r3@users.noreply.github.com> Date: Sat, 14 Oct 2023 16:38:10 +0100 Subject: [PATCH 34/81] CORE-17661 5.1 Performance integration - Multi-Source Event Mediator integration with StateManager (#4874) Integration of Multi-Source Event Mediator with StateManager. --- .../FlowMapperEventMediatorFactory.kt | 3 + .../FlowMapperEventMediatorFactoryImpl.kt | 5 ++ .../FlowMapperEventMediatorFactoryImplTest.kt | 2 +- .../mediator/FlowEventMediatorFactory.kt | 3 + .../mediator/FlowEventMediatorFactoryImpl.kt | 5 ++ .../corda/flow/service/FlowExecutorImpl.kt | 14 ++++ .../FlowEventMediatorFactoryImplTest.kt | 3 +- .../flow/service/FlowExecutorImplTest.kt | 13 +++- .../MultiSourceEventMediatorFactoryImpl.kt | 7 +- .../mediator/statemanager/StateManagerImpl.kt | 67 ------------------- .../MultiSourceEventMediatorFactoryTest.kt | 3 +- .../mediator/config/EventMediatorConfig.kt | 11 ++- .../config/EventMediatorConfigBuilder.kt | 11 ++- 13 files changed, 66 insertions(+), 81 deletions(-) delete mode 100644 libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/statemanager/StateManagerImpl.kt diff --git a/components/flow/flow-mapper-service/src/main/kotlin/net/corda/session/mapper/messaging/mediator/FlowMapperEventMediatorFactory.kt b/components/flow/flow-mapper-service/src/main/kotlin/net/corda/session/mapper/messaging/mediator/FlowMapperEventMediatorFactory.kt index 89e76f48d08..4685f063d3b 100644 --- a/components/flow/flow-mapper-service/src/main/kotlin/net/corda/session/mapper/messaging/mediator/FlowMapperEventMediatorFactory.kt +++ b/components/flow/flow-mapper-service/src/main/kotlin/net/corda/session/mapper/messaging/mediator/FlowMapperEventMediatorFactory.kt @@ -3,6 +3,7 @@ package net.corda.session.mapper.messaging.mediator import net.corda.data.flow.event.mapper.FlowMapperEvent import net.corda.data.flow.state.mapper.FlowMapperState import net.corda.libs.configuration.SmartConfig +import net.corda.libs.statemanager.api.StateManager import net.corda.messaging.api.mediator.MultiSourceEventMediator /** @@ -14,10 +15,12 @@ interface FlowMapperEventMediatorFactory { * * @param flowConfig Flow configuration. * @param messagingConfig Messaging configuration. + * @param stateManager State manager. */ fun create( flowConfig: SmartConfig, messagingConfig: SmartConfig, + stateManager: StateManager, ): MultiSourceEventMediator } \ No newline at end of file diff --git a/components/flow/flow-mapper-service/src/main/kotlin/net/corda/session/mapper/messaging/mediator/FlowMapperEventMediatorFactoryImpl.kt b/components/flow/flow-mapper-service/src/main/kotlin/net/corda/session/mapper/messaging/mediator/FlowMapperEventMediatorFactoryImpl.kt index 3bbdb0523af..888a6ea3d1a 100644 --- a/components/flow/flow-mapper-service/src/main/kotlin/net/corda/session/mapper/messaging/mediator/FlowMapperEventMediatorFactoryImpl.kt +++ b/components/flow/flow-mapper-service/src/main/kotlin/net/corda/session/mapper/messaging/mediator/FlowMapperEventMediatorFactoryImpl.kt @@ -6,6 +6,7 @@ import net.corda.data.flow.state.mapper.FlowMapperState import net.corda.data.p2p.app.AppMessage import net.corda.flow.mapper.factory.FlowMapperEventExecutorFactory import net.corda.libs.configuration.SmartConfig +import net.corda.libs.statemanager.api.StateManager import net.corda.messaging.api.mediator.MessageRouter import net.corda.messaging.api.mediator.RoutingDestination.Companion.routeTo import net.corda.messaging.api.mediator.config.EventMediatorConfigBuilder @@ -42,11 +43,13 @@ class FlowMapperEventMediatorFactoryImpl @Activate constructor( override fun create( flowConfig: SmartConfig, messagingConfig: SmartConfig, + stateManager: StateManager, ) = eventMediatorFactory.create( createEventMediatorConfig( flowConfig, messagingConfig, FlowMapperMessageProcessor(flowMapperEventExecutorFactory, flowConfig), + stateManager, ) ) @@ -54,6 +57,7 @@ class FlowMapperEventMediatorFactoryImpl @Activate constructor( flowConfig: SmartConfig, messagingConfig: SmartConfig, messageProcessor: StateAndEventProcessor, + stateManager: StateManager, ) = EventMediatorConfigBuilder() .name("FlowMapperEventMediator") .messagingConfig(messagingConfig) @@ -71,6 +75,7 @@ class FlowMapperEventMediatorFactoryImpl @Activate constructor( .messageRouterFactory(createMessageRouterFactory()) .threads(flowConfig.getInt(FlowConfig.PROCESSING_THREAD_POOL_SIZE)) .threadName("flow-mapper-event-mediator") + .stateManager(stateManager) .build() private fun createMessageRouterFactory() = MessageRouterFactory { clientFinder -> diff --git a/components/flow/flow-mapper-service/src/test/kotlin/net/corda/session/mapper/service/messaging/mediator/FlowMapperEventMediatorFactoryImplTest.kt b/components/flow/flow-mapper-service/src/test/kotlin/net/corda/session/mapper/service/messaging/mediator/FlowMapperEventMediatorFactoryImplTest.kt index 149fd9d9a22..da208559925 100644 --- a/components/flow/flow-mapper-service/src/test/kotlin/net/corda/session/mapper/service/messaging/mediator/FlowMapperEventMediatorFactoryImplTest.kt +++ b/components/flow/flow-mapper-service/src/test/kotlin/net/corda/session/mapper/service/messaging/mediator/FlowMapperEventMediatorFactoryImplTest.kt @@ -42,7 +42,7 @@ class FlowMapperEventMediatorFactoryImplTest { @Test fun `successfully creates event mediator`() { - val mediator = flowMapperEventMediatorFactory.create(flowConfig, mock()) + val mediator = flowMapperEventMediatorFactory.create(flowConfig, mock(), mock()) assertNotNull(mediator) } diff --git a/components/flow/flow-service/src/main/kotlin/net/corda/flow/messaging/mediator/FlowEventMediatorFactory.kt b/components/flow/flow-service/src/main/kotlin/net/corda/flow/messaging/mediator/FlowEventMediatorFactory.kt index 332ce01a7d9..16da52e2a92 100644 --- a/components/flow/flow-service/src/main/kotlin/net/corda/flow/messaging/mediator/FlowEventMediatorFactory.kt +++ b/components/flow/flow-service/src/main/kotlin/net/corda/flow/messaging/mediator/FlowEventMediatorFactory.kt @@ -3,6 +3,7 @@ package net.corda.flow.messaging.mediator import net.corda.data.flow.event.FlowEvent import net.corda.data.flow.state.checkpoint.Checkpoint import net.corda.libs.configuration.SmartConfig +import net.corda.libs.statemanager.api.StateManager import net.corda.messaging.api.mediator.MultiSourceEventMediator /** @@ -14,11 +15,13 @@ interface FlowEventMediatorFactory { * * @param configs Map of configurations (keys are API defined configuration keys). * @param messagingConfig Messaging configuration. + * @param stateManager State manager. * @see net.corda.schema.configuration.ConfigKeys */ fun create( configs: Map, messagingConfig: SmartConfig, + stateManager: StateManager, ): MultiSourceEventMediator } \ No newline at end of file diff --git a/components/flow/flow-service/src/main/kotlin/net/corda/flow/messaging/mediator/FlowEventMediatorFactoryImpl.kt b/components/flow/flow-service/src/main/kotlin/net/corda/flow/messaging/mediator/FlowEventMediatorFactoryImpl.kt index d4b81c53bf1..525c46985d4 100644 --- a/components/flow/flow-service/src/main/kotlin/net/corda/flow/messaging/mediator/FlowEventMediatorFactoryImpl.kt +++ b/components/flow/flow-service/src/main/kotlin/net/corda/flow/messaging/mediator/FlowEventMediatorFactoryImpl.kt @@ -14,6 +14,7 @@ import net.corda.flow.pipeline.factory.FlowEventProcessorFactory import net.corda.ledger.utxo.verification.TransactionVerificationRequest import net.corda.libs.configuration.SmartConfig import net.corda.libs.configuration.helper.getConfig +import net.corda.libs.statemanager.api.StateManager import net.corda.messaging.api.mediator.MediatorMessage import net.corda.messaging.api.mediator.MessageRouter import net.corda.messaging.api.mediator.RoutingDestination.Companion.routeTo @@ -61,11 +62,13 @@ class FlowEventMediatorFactoryImpl @Activate constructor( override fun create( configs: Map, messagingConfig: SmartConfig, + stateManager: StateManager, ) = eventMediatorFactory.create( createEventMediatorConfig( configs, messagingConfig, flowEventProcessorFactory.create(configs), + stateManager, ) ) @@ -73,6 +76,7 @@ class FlowEventMediatorFactoryImpl @Activate constructor( configs: Map, messagingConfig: SmartConfig, messageProcessor: StateAndEventProcessor, + stateManager: StateManager, ) = EventMediatorConfigBuilder() .name("FlowEventMediator") .messagingConfig(messagingConfig) @@ -90,6 +94,7 @@ class FlowEventMediatorFactoryImpl @Activate constructor( .messageRouterFactory(createMessageRouterFactory()) .threads(configs.getConfig(ConfigKeys.FLOW_CONFIG).getInt(FlowConfig.PROCESSING_THREAD_POOL_SIZE)) .threadName("flow-event-mediator") + .stateManager(stateManager) .build() private fun createMessageRouterFactory() = MessageRouterFactory { clientFinder -> diff --git a/components/flow/flow-service/src/main/kotlin/net/corda/flow/service/FlowExecutorImpl.kt b/components/flow/flow-service/src/main/kotlin/net/corda/flow/service/FlowExecutorImpl.kt index 0bef3fa8ef9..a4668a1536b 100644 --- a/components/flow/flow-service/src/main/kotlin/net/corda/flow/service/FlowExecutorImpl.kt +++ b/components/flow/flow-service/src/main/kotlin/net/corda/flow/service/FlowExecutorImpl.kt @@ -6,6 +6,7 @@ import net.corda.data.flow.state.checkpoint.Checkpoint import net.corda.flow.messaging.mediator.FlowEventMediatorFactory import net.corda.libs.configuration.SmartConfig import net.corda.libs.configuration.helper.getConfig +import net.corda.libs.statemanager.api.StateManagerFactory import net.corda.lifecycle.LifecycleCoordinatorFactory import net.corda.lifecycle.LifecycleEvent import net.corda.lifecycle.LifecycleStatus @@ -18,6 +19,7 @@ import net.corda.schema.configuration.BootConfig.CRYPTO_WORKER_REST_ENDPOINT import net.corda.schema.configuration.BootConfig.PERSISTENCE_WORKER_REST_ENDPOINT import net.corda.schema.configuration.BootConfig.UNIQUENESS_WORKER_REST_ENDPOINT import net.corda.schema.configuration.BootConfig.VERIFICATION_WORKER_REST_ENDPOINT +import net.corda.schema.configuration.ConfigKeys import net.corda.schema.configuration.ConfigKeys.BOOT_CONFIG import net.corda.schema.configuration.ConfigKeys.FLOW_CONFIG import net.corda.schema.configuration.ConfigKeys.MESSAGING_CONFIG @@ -34,6 +36,7 @@ import org.slf4j.LoggerFactory class FlowExecutorImpl constructor( coordinatorFactory: LifecycleCoordinatorFactory, private val flowEventMediatorFactory: FlowEventMediatorFactory, + private val stateManagerFactory: StateManagerFactory, private val toMessagingConfig: (Map) -> SmartConfig, ) : FlowExecutor { @@ -43,14 +46,18 @@ class FlowExecutorImpl constructor( coordinatorFactory: LifecycleCoordinatorFactory, @Reference(service = FlowEventMediatorFactory::class) flowEventMediatorFactory: FlowEventMediatorFactory, + @Reference(service = StateManagerFactory::class) + stateManagerFactory: StateManagerFactory, ) : this( coordinatorFactory, flowEventMediatorFactory, + stateManagerFactory, { cfg -> cfg.getConfig(MESSAGING_CONFIG) } ) companion object { private val log = LoggerFactory.getLogger(this::class.java.enclosingClass) + // TODO private const val STATE_MANAGER = "flow.engine.state.manager" } private val coordinator = coordinatorFactory.createCoordinator { event, _ -> eventHandler(event) } @@ -61,14 +68,21 @@ class FlowExecutorImpl constructor( try { val messagingConfig = toMessagingConfig(config).withServiceEndpoints(config) val updatedConfigs = updateConfigsWithFlowConfig(config, messagingConfig) + val stateManagerConfig = config.getConfig(ConfigKeys.STATE_MANAGER_CONFIG) // close the lifecycle registration first to prevent down being signaled subscriptionRegistrationHandle?.close() multiSourceEventMediator?.close() + // TODO Create as managed resource once issue with stateManager is fixed + val stateManager = // coordinator.createManagedResource(STATE_MANAGER) { + stateManagerFactory.create(stateManagerConfig) + // } + multiSourceEventMediator = flowEventMediatorFactory.create( updatedConfigs, messagingConfig, + stateManager, ) subscriptionRegistrationHandle = coordinator.followStatusChangesByName( diff --git a/components/flow/flow-service/src/test/kotlin/net/corda/flow/messaging/FlowEventMediatorFactoryImplTest.kt b/components/flow/flow-service/src/test/kotlin/net/corda/flow/messaging/FlowEventMediatorFactoryImplTest.kt index d0fa7d377f9..a1002bbe873 100644 --- a/components/flow/flow-service/src/test/kotlin/net/corda/flow/messaging/FlowEventMediatorFactoryImplTest.kt +++ b/components/flow/flow-service/src/test/kotlin/net/corda/flow/messaging/FlowEventMediatorFactoryImplTest.kt @@ -52,7 +52,8 @@ class FlowEventMediatorFactoryImplTest { fun `successfully creates event mediator`() { val mediator = flowEventMediatorFactory.create( mapOf(ConfigKeys.FLOW_CONFIG to flowConfig), - mock() + mock(), + mock(), ) assertNotNull(mediator) diff --git a/components/flow/flow-service/src/test/kotlin/net/corda/flow/service/FlowExecutorImplTest.kt b/components/flow/flow-service/src/test/kotlin/net/corda/flow/service/FlowExecutorImplTest.kt index ab0f58959eb..95314fd0c74 100644 --- a/components/flow/flow-service/src/test/kotlin/net/corda/flow/service/FlowExecutorImplTest.kt +++ b/components/flow/flow-service/src/test/kotlin/net/corda/flow/service/FlowExecutorImplTest.kt @@ -7,6 +7,8 @@ import net.corda.flow.messaging.mediator.FlowEventMediatorFactory import net.corda.flow.pipeline.factory.FlowEventProcessorFactory import net.corda.libs.configuration.SmartConfig import net.corda.libs.configuration.SmartConfigImpl +import net.corda.libs.statemanager.api.StateManager +import net.corda.libs.statemanager.api.StateManagerFactory import net.corda.lifecycle.LifecycleCoordinator import net.corda.lifecycle.LifecycleCoordinatorFactory import net.corda.lifecycle.LifecycleCoordinatorName @@ -19,6 +21,7 @@ import net.corda.messaging.api.processor.StateAndEventProcessor import net.corda.schema.configuration.BootConfig import net.corda.schema.configuration.ConfigKeys.BOOT_CONFIG import net.corda.schema.configuration.ConfigKeys.FLOW_CONFIG +import net.corda.schema.configuration.ConfigKeys.STATE_MANAGER_CONFIG import net.corda.schema.configuration.MessagingConfig.MAX_ALLOWED_MSG_SIZE import net.corda.schema.configuration.MessagingConfig.Subscription.PROCESSOR_TIMEOUT import org.assertj.core.api.Assertions.assertThat @@ -36,6 +39,7 @@ class FlowExecutorImplTest { private val coordinatorFactory = mock() private val flowEventProcessorFactory = mock() + private val stateManagerFactory = mock() private val flowEventMediatorFactory = mock() private val toMessagingConfig: (Map) -> SmartConfig = { messagingConfig @@ -43,26 +47,31 @@ class FlowExecutorImplTest { private val config = mutableMapOf( BOOT_CONFIG to SmartConfigImpl.empty().withServiceEndpoints(), - FLOW_CONFIG to SmartConfigImpl.empty() + FLOW_CONFIG to SmartConfigImpl.empty(), + STATE_MANAGER_CONFIG to SmartConfigImpl.empty(), ) private val messagingConfig = getMinimalMessagingConfig() private val subscriptionRegistrationHandle = mock() private val flowExecutorCoordinator = mock() private val multiSourceEventMediator = mock>() private val flowEventProcessor = mock>() + private val stateManager = mock() @BeforeEach fun setup() { whenever(flowEventProcessorFactory.create(any())).thenReturn(flowEventProcessor) + whenever(stateManagerFactory.create(any())).thenReturn(stateManager) whenever( flowEventMediatorFactory.create( any(), any(), + any(), ) ).thenReturn(multiSourceEventMediator) whenever(coordinatorFactory.createCoordinator(any(), any())).thenReturn(flowExecutorCoordinator) whenever(flowExecutorCoordinator.followStatusChangesByName(any())).thenReturn(subscriptionRegistrationHandle) + whenever(flowExecutorCoordinator.createManagedResource(any(), any<() -> StateManager>())).thenReturn(stateManager) } @Test @@ -140,6 +149,7 @@ class FlowExecutorImplTest { flowEventMediatorFactory.create( any(), any(), + any(), ) ).thenReturn(multiSourceEventMediator2) @@ -171,6 +181,7 @@ class FlowExecutorImplTest { return FlowExecutorImpl( coordinatorFactory, flowEventMediatorFactory, + stateManagerFactory, toMessagingConfig ) } diff --git a/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/factory/MultiSourceEventMediatorFactoryImpl.kt b/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/factory/MultiSourceEventMediatorFactoryImpl.kt index d9a34c6a071..6c7b3bc573b 100644 --- a/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/factory/MultiSourceEventMediatorFactoryImpl.kt +++ b/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/factory/MultiSourceEventMediatorFactoryImpl.kt @@ -1,7 +1,6 @@ package net.corda.messaging.mediator.factory import net.corda.avro.serialization.CordaAvroSerializationFactory -import net.corda.libs.statemanager.api.StateManager import net.corda.lifecycle.LifecycleCoordinatorFactory import net.corda.messaging.api.mediator.MultiSourceEventMediator import net.corda.messaging.api.mediator.config.EventMediatorConfig @@ -15,7 +14,6 @@ import org.osgi.service.component.annotations.Reference @Component(service = [MultiSourceEventMediatorFactory::class]) class MultiSourceEventMediatorFactoryImpl( private val cordaAvroSerializationFactory: CordaAvroSerializationFactory, - private val stateManager: StateManager, private val lifecycleCoordinatorFactory: LifecycleCoordinatorFactory, private val taskManagerFactory: TaskManagerFactory ): MultiSourceEventMediatorFactory { @@ -24,13 +22,10 @@ class MultiSourceEventMediatorFactoryImpl( constructor( @Reference(service = CordaAvroSerializationFactory::class) cordaAvroSerializationFactory: CordaAvroSerializationFactory, - @Reference(service = StateManager::class) - stateManager: StateManager, @Reference(service = LifecycleCoordinatorFactory::class) lifecycleCoordinatorFactory: LifecycleCoordinatorFactory ) : this( cordaAvroSerializationFactory, - stateManager, lifecycleCoordinatorFactory, TaskManagerFactory.INSTANCE ) @@ -47,7 +42,7 @@ class MultiSourceEventMediatorFactoryImpl( eventMediatorConfig, stateSerializer, stateDeserializer, - stateManager, + eventMediatorConfig.stateManager, taskManagerFactory.createThreadPoolTaskManager( name = eventMediatorConfig.name, threadName = eventMediatorConfig.threadName, diff --git a/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/statemanager/StateManagerImpl.kt b/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/statemanager/StateManagerImpl.kt deleted file mode 100644 index a8d2823977d..00000000000 --- a/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/statemanager/StateManagerImpl.kt +++ /dev/null @@ -1,67 +0,0 @@ -package net.corda.messaging.mediator.statemanager - -import net.corda.libs.statemanager.api.IntervalFilter -import net.corda.libs.statemanager.api.MetadataFilter -import net.corda.libs.statemanager.api.State -import net.corda.libs.statemanager.api.StateManager -import org.osgi.service.component.annotations.Activate -import org.osgi.service.component.annotations.Component -import java.util.concurrent.ConcurrentHashMap - -// TODO This is used temporarily until State Manager implementation is finished -@Component(service = [StateManager::class]) -class StateManagerImpl @Activate constructor() : StateManager { - private val storage = ConcurrentHashMap() - - override fun create(states: Collection): Map { - return states.mapNotNull { - storage.putIfAbsent(it.key, it) - }.associate { it.key to RuntimeException("State already exists [$it]") } - } - - override fun get(keys: Collection): Map { - return keys.mapNotNull { storage[it] }.associateBy { it.key } - } - - override fun update(states: Collection): Map { - return states.mapNotNull { - var output: State? = null - storage.compute(it.key) { _, existingState -> - if (existingState?.version == it.version) { - it.copy(version = it.version + 1) - } else { - output = it - it - } - } - output - }.associateBy { it.key } - } - - override fun delete(states: Collection): Map { - TODO("Not yet implemented") - } - - override fun updatedBetween(interval: IntervalFilter): Map { - TODO("Not yet implemented") - } - - override fun findByMetadataMatchingAll(filters: Collection): Map { - TODO("Not yet implemented") - } - - override fun findByMetadataMatchingAny(filters: Collection): Map { - TODO("Not yet implemented") - } - - override fun findUpdatedBetweenWithMetadataFilter( - intervalFilter: IntervalFilter, - metadataFilter: MetadataFilter - ): Map { - TODO("Not yet implemented") - } - - override fun close() { - TODO("Not yet implemented") - } -} diff --git a/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/factory/MultiSourceEventMediatorFactoryTest.kt b/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/factory/MultiSourceEventMediatorFactoryTest.kt index 41654a23a9e..aa4e1cba0cc 100644 --- a/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/factory/MultiSourceEventMediatorFactoryTest.kt +++ b/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/factory/MultiSourceEventMediatorFactoryTest.kt @@ -32,7 +32,6 @@ class MultiSourceEventMediatorFactoryTest { doReturn(mock()).`when`(taskManagerFactory).createThreadPoolTaskManager(any(), any(), any()) multiSourceEventMediatorFactory = MultiSourceEventMediatorFactoryImpl( cordaAvroSerializationFactory, - mock(), mock(), taskManagerFactory, ) @@ -43,12 +42,14 @@ class MultiSourceEventMediatorFactoryTest { val messageProcessor = mock>() doReturn(Any::class.java).`when`(messageProcessor).stateValueClass val messageRouterFactory = mock() + val stateManager = mock() val config = mock>() doReturn(messageProcessor).`when`(config).messageProcessor doReturn(messageRouterFactory).`when`(config).messageRouterFactory doReturn("name").`when`(config).name doReturn(1).`when`(config).threads doReturn("name").`when`(config).threadName + doReturn(stateManager).`when`(config).stateManager val mediator = multiSourceEventMediatorFactory.create(config) diff --git a/libs/messaging/messaging/src/main/kotlin/net/corda/messaging/api/mediator/config/EventMediatorConfig.kt b/libs/messaging/messaging/src/main/kotlin/net/corda/messaging/api/mediator/config/EventMediatorConfig.kt index 627a90c26e8..26eaa487508 100644 --- a/libs/messaging/messaging/src/main/kotlin/net/corda/messaging/api/mediator/config/EventMediatorConfig.kt +++ b/libs/messaging/messaging/src/main/kotlin/net/corda/messaging/api/mediator/config/EventMediatorConfig.kt @@ -1,6 +1,7 @@ package net.corda.messaging.api.mediator.config import net.corda.libs.configuration.SmartConfig +import net.corda.libs.statemanager.api.StateManager import net.corda.messaging.api.mediator.MultiSourceEventMediator import net.corda.messaging.api.mediator.factory.MediatorConsumerFactory import net.corda.messaging.api.mediator.factory.MessageRouterFactory @@ -21,16 +22,20 @@ import java.time.Duration * @property clientFactories Factories for creating messaging clients. * @property messageProcessor State and event processor. * @property messageRouterFactory Message router factory. + * @property threads Number of threads used by task manager. + * @property threadName Name (prefix) for task manager threads. + * @property stateManager State manager. */ data class EventMediatorConfig( val name: String, - val messagingConfig : SmartConfig, + val messagingConfig: SmartConfig, val consumerFactories: Collection, val clientFactories: Collection, - val messageProcessor : StateAndEventProcessor, + val messageProcessor: StateAndEventProcessor, val messageRouterFactory: MessageRouterFactory, val threads: Int, - val threadName: String + val threadName: String, + val stateManager: StateManager, ) { /** * Timeout for polling consumers. diff --git a/libs/messaging/messaging/src/main/kotlin/net/corda/messaging/api/mediator/config/EventMediatorConfigBuilder.kt b/libs/messaging/messaging/src/main/kotlin/net/corda/messaging/api/mediator/config/EventMediatorConfigBuilder.kt index d5c32e6199e..83a4e0c3f4c 100644 --- a/libs/messaging/messaging/src/main/kotlin/net/corda/messaging/api/mediator/config/EventMediatorConfigBuilder.kt +++ b/libs/messaging/messaging/src/main/kotlin/net/corda/messaging/api/mediator/config/EventMediatorConfigBuilder.kt @@ -1,6 +1,7 @@ package net.corda.messaging.api.mediator.config import net.corda.libs.configuration.SmartConfig +import net.corda.libs.statemanager.api.StateManager import net.corda.messaging.api.mediator.MultiSourceEventMediator import net.corda.messaging.api.mediator.factory.MediatorConsumerFactory import net.corda.messaging.api.mediator.factory.MessageRouterFactory @@ -24,6 +25,7 @@ class EventMediatorConfigBuilder { private var messageRouterFactory: MessageRouterFactory? = null private var threads: Int? = null private var threadName: String? = null + private var stateManager: StateManager? = null /** Sets name for [MultiSourceEventMediator]. */ fun name(name: String) = @@ -49,12 +51,18 @@ class EventMediatorConfigBuilder { fun messageRouterFactory(messageRouterFactory: MessageRouterFactory) = apply { this.messageRouterFactory = messageRouterFactory } + /** Sets number of threads for task manager. */ fun threads(threads: Int) = apply { this.threads = threads } + /** Sets name preix for task manager threads. */ fun threadName(threadName: String) = apply { this.threadName = threadName } + /** Sets state manager. */ + fun stateManager(stateManager: StateManager) = + apply { this.stateManager = stateManager } + /** Builds [EventMediatorConfig]. */ fun build(): EventMediatorConfig { check(consumerFactories.isNotEmpty()) { "At least on consumer factory has to be set" } @@ -67,7 +75,8 @@ class EventMediatorConfigBuilder { messageProcessor = checkNotNull(messageProcessor) { "Message processor not set" }, messageRouterFactory = checkNotNull(messageRouterFactory) { "Message router factory not set" }, threads = checkNotNull(threads) { "Number of threads not set" }, - threadName = checkNotNull(threadName) { "Thread name not set" } + threadName = checkNotNull(threadName) { "Thread name not set" }, + stateManager = checkNotNull(stateManager) { "State manager not set" }, ) } } \ No newline at end of file From 9513272630bd3ef4ebc9ddc7068799eca7d3db0e Mon Sep 17 00:00:00 2001 From: Joseph Zuniga-Daly Date: Mon, 16 Oct 2023 17:30:03 +0100 Subject: [PATCH 35/81] CORE-17483: Fix configuration change smoke test (#4883) Our configuration change test causes the lifecycle components within workers to restart. They receive the updated configuration and restart with their new configuration. We had pushed this test to the end of the run to avoid other tests being affected by the restart. We have identified two issues with this test: * The order we had defined to push the test to the end had not taken effect * The code to test the cluster is in a good state did not handle exceptions that naturally happen when a client tries to connect to a server that is restarting; so the test was not waiting for the cluster to reach a good state To address this we are making the following changes: * Extract the configuration change test into its own test class * Remove the method order annotation from the test, we want class order instead * Increase the `@Order` value to `Int.MAX_VALUE` which is the maximum allowed * The default order value is `1073741823`, we previously used `999`, so we were not passing the default * Change `startRpcFlow` to use `assertWithRetryIgnoringExceptions` which retries on exception --- .../smoketest/flow/ConfigurationChangeTest.kt | 124 ++++++++++++++++++ .../workers/smoketest/flow/FlowTests.kt | 61 --------- .../corda/e2etest/utilities/FlowTestUtils.kt | 2 +- 3 files changed, 125 insertions(+), 62 deletions(-) create mode 100644 applications/workers/workers-smoketest/src/smokeTest/kotlin/net/corda/applications/workers/smoketest/flow/ConfigurationChangeTest.kt diff --git a/applications/workers/workers-smoketest/src/smokeTest/kotlin/net/corda/applications/workers/smoketest/flow/ConfigurationChangeTest.kt b/applications/workers/workers-smoketest/src/smokeTest/kotlin/net/corda/applications/workers/smoketest/flow/ConfigurationChangeTest.kt new file mode 100644 index 00000000000..0497ba972d6 --- /dev/null +++ b/applications/workers/workers-smoketest/src/smokeTest/kotlin/net/corda/applications/workers/smoketest/flow/ConfigurationChangeTest.kt @@ -0,0 +1,124 @@ +package net.corda.applications.workers.smoketest.flow + +import net.corda.applications.workers.smoketest.utils.TEST_CPB_LOCATION +import net.corda.applications.workers.smoketest.utils.TEST_CPI_NAME +import net.corda.e2etest.utilities.DEFAULT_CLUSTER +import net.corda.e2etest.utilities.RPC_FLOW_STATUS_SUCCESS +import net.corda.e2etest.utilities.RpcSmokeTestInput +import net.corda.e2etest.utilities.awaitRestFlowResult +import net.corda.e2etest.utilities.conditionallyUploadCordaPackage +import net.corda.e2etest.utilities.conditionallyUploadCpiSigningCertificate +import net.corda.e2etest.utilities.config.configWithDefaultsNode +import net.corda.e2etest.utilities.config.getConfig +import net.corda.e2etest.utilities.config.managedConfig +import net.corda.e2etest.utilities.config.waitForConfigurationChange +import net.corda.e2etest.utilities.getHoldingIdShortHash +import net.corda.e2etest.utilities.getOrCreateVirtualNodeFor +import net.corda.e2etest.utilities.registerStaticMember +import net.corda.e2etest.utilities.startRpcFlow +import net.corda.schema.configuration.ConfigKeys.MESSAGING_CONFIG +import net.corda.schema.configuration.MessagingConfig.MAX_ALLOWED_MSG_SIZE +import org.assertj.core.api.Assertions.assertThat +import org.junit.jupiter.api.BeforeAll +import org.junit.jupiter.api.Order +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.TestInstance +import org.junit.jupiter.api.TestInstance.Lifecycle +import java.util.UUID + +@Suppress("Unused", "FunctionName") +//The flow tests must go last as one test updates the messaging config which is highly disruptive to subsequent test runs. The real +// solution to this is a larger effort to have components listen to their messaging pattern lifecycle status and for them to go DOWN when +// their patterns are DOWN - CORE-8015 +@Order(Int.MAX_VALUE) +@TestInstance(Lifecycle.PER_CLASS) +class ConfigurationChangeTest { + + companion object { + private val testRunUniqueId = UUID.randomUUID() + private val groupId = UUID.randomUUID().toString() + private val applicationCpiName = "${TEST_CPI_NAME}_$testRunUniqueId" + private val bobX500 = "CN=Bob-$testRunUniqueId, OU=Application, O=R3, L=London, C=GB" + private var bobHoldingId: String = getHoldingIdShortHash(bobX500, groupId) + private val charlyX500 = "CN=Charley-$testRunUniqueId, OU=Application, O=R3, L=London, C=GB" + private var charlieHoldingId: String = getHoldingIdShortHash(charlyX500, groupId) + private val staticMemberList = listOf( + bobX500, + charlyX500, + ) + + @BeforeAll + @JvmStatic + internal fun beforeAll() { + DEFAULT_CLUSTER.conditionallyUploadCpiSigningCertificate() + + // Upload test flows if not already uploaded + conditionallyUploadCordaPackage( + applicationCpiName, TEST_CPB_LOCATION, groupId, staticMemberList + ) + + // Make sure Virtual Nodes are created + val bobActualHoldingId = getOrCreateVirtualNodeFor(bobX500, applicationCpiName) + val charlieActualHoldingId = getOrCreateVirtualNodeFor(charlyX500, applicationCpiName) + + // Just validate the function and actual vnode holding ID hash are in sync + // if this fails the X500_BOB formatting could have changed or the hash implementation might have changed + assertThat(bobActualHoldingId).isEqualTo(bobHoldingId) + assertThat(charlieActualHoldingId).isEqualTo(charlieHoldingId) + + registerStaticMember(bobHoldingId) + registerStaticMember(charlieHoldingId) + } + } + + @Test + fun `cluster configuration changes are picked up and workers continue to operate normally`() { + val currentConfigValue = getConfig(MESSAGING_CONFIG).configWithDefaultsNode()[MAX_ALLOWED_MSG_SIZE].asInt() + val newConfigurationValue = (currentConfigValue * 1.5).toInt() + + managedConfig { configManager -> + println("Set new config") + configManager + .load(MESSAGING_CONFIG, MAX_ALLOWED_MSG_SIZE, newConfigurationValue) + .apply() + // Wait for the rpc-worker to reload the configuration and come back up + println("Wait for the rpc-worker to reload the configuration and come back up") + waitForConfigurationChange(MESSAGING_CONFIG, MAX_ALLOWED_MSG_SIZE, newConfigurationValue.toString(), false) + + // Execute some flows which require functionality from different workers and make sure they succeed + println("Execute some flows which require functionality from different workers and make sure they succeed") + val flowIds = mutableListOf( + startRpcFlow( + bobHoldingId, + RpcSmokeTestInput().apply { + command = "persistence_persist" + data = mapOf("id" to UUID.randomUUID().toString()) + } + ), + + startRpcFlow( + bobHoldingId, + RpcSmokeTestInput().apply { + command = "crypto_sign_and_verify" + data = mapOf("memberX500" to bobX500) + } + ), + + startRpcFlow( + bobHoldingId, + RpcSmokeTestInput().apply { + command = "lookup_member_by_x500_name" + data = mapOf("id" to charlyX500) + } + ) + ) + + println("Check status of flows") + flowIds.forEach { + val flowResult = awaitRestFlowResult(bobHoldingId, it) + assertThat(flowResult.flowError).isNull() + assertThat(flowResult.flowStatus).isEqualTo(RPC_FLOW_STATUS_SUCCESS) + } + } + } +} diff --git a/applications/workers/workers-smoketest/src/smokeTest/kotlin/net/corda/applications/workers/smoketest/flow/FlowTests.kt b/applications/workers/workers-smoketest/src/smokeTest/kotlin/net/corda/applications/workers/smoketest/flow/FlowTests.kt index 6e8cd499744..3d8d28eb4c0 100644 --- a/applications/workers/workers-smoketest/src/smokeTest/kotlin/net/corda/applications/workers/smoketest/flow/FlowTests.kt +++ b/applications/workers/workers-smoketest/src/smokeTest/kotlin/net/corda/applications/workers/smoketest/flow/FlowTests.kt @@ -15,35 +15,21 @@ import net.corda.e2etest.utilities.TEST_NOTARY_CPI_NAME import net.corda.e2etest.utilities.awaitRestFlowResult import net.corda.e2etest.utilities.conditionallyUploadCordaPackage import net.corda.e2etest.utilities.conditionallyUploadCpiSigningCertificate -import net.corda.e2etest.utilities.config.configWithDefaultsNode -import net.corda.e2etest.utilities.config.getConfig -import net.corda.e2etest.utilities.config.managedConfig -import net.corda.e2etest.utilities.config.waitForConfigurationChange import net.corda.e2etest.utilities.getHoldingIdShortHash import net.corda.e2etest.utilities.getOrCreateVirtualNodeFor import net.corda.e2etest.utilities.registerStaticMember import net.corda.e2etest.utilities.startRpcFlow -import net.corda.schema.configuration.ConfigKeys.MESSAGING_CONFIG -import net.corda.schema.configuration.MessagingConfig.MAX_ALLOWED_MSG_SIZE import net.corda.v5.crypto.DigestAlgorithmName import org.assertj.core.api.Assertions.assertThat import org.junit.jupiter.api.Assertions.assertAll import org.junit.jupiter.api.BeforeAll -import org.junit.jupiter.api.MethodOrderer -import org.junit.jupiter.api.Order import org.junit.jupiter.api.Test import org.junit.jupiter.api.TestInstance import org.junit.jupiter.api.TestInstance.Lifecycle -import org.junit.jupiter.api.TestMethodOrder import java.util.UUID import kotlin.text.Typography.quote @Suppress("Unused", "FunctionName") -//The flow tests must go last as one test updates the messaging config which is highly disruptive to subsequent test runs. The real -// solution to this is a larger effort to have components listen to their messaging pattern lifecycle status and for them to go DOWN when -// their patterns are DOWN - CORE-8015 -@Order(999) -@TestMethodOrder(MethodOrderer.OrderAnnotation::class) @TestInstance(Lifecycle.PER_CLASS) class FlowTests { @@ -851,53 +837,6 @@ class FlowTests { } } - @Test - fun `cluster configuration changes are picked up and workers continue to operate normally`() { - val currentConfigValue = getConfig(MESSAGING_CONFIG).configWithDefaultsNode()[MAX_ALLOWED_MSG_SIZE].asInt() - val newConfigurationValue = (currentConfigValue * 1.5).toInt() - - managedConfig { configManager -> - configManager - .load(MESSAGING_CONFIG, MAX_ALLOWED_MSG_SIZE, newConfigurationValue) - .apply() - // Wait for the rpc-worker to reload the configuration and come back up - waitForConfigurationChange(MESSAGING_CONFIG, MAX_ALLOWED_MSG_SIZE, newConfigurationValue.toString(), false) - - // Execute some flows which require functionality from different workers and make sure they succeed - val flowIds = mutableListOf( - startRpcFlow( - bobHoldingId, - RpcSmokeTestInput().apply { - command = "persistence_persist" - data = mapOf("id" to UUID.randomUUID().toString()) - } - ), - - startRpcFlow( - bobHoldingId, - RpcSmokeTestInput().apply { - command = "crypto_sign_and_verify" - data = mapOf("memberX500" to bobX500) - } - ), - - startRpcFlow( - bobHoldingId, - RpcSmokeTestInput().apply { - command = "lookup_member_by_x500_name" - data = mapOf("id" to charlyX500) - } - ) - ) - - flowIds.forEach { - val flowResult = awaitRestFlowResult(bobHoldingId, it) - assertThat(flowResult.flowError).isNull() - assertThat(flowResult.flowStatus).isEqualTo(RPC_FLOW_STATUS_SUCCESS) - } - } - } - @Test fun `Json serialisation`() { val requestBody = RpcSmokeTestInput().apply { diff --git a/testing/e2e-test-utilities/src/main/kotlin/net/corda/e2etest/utilities/FlowTestUtils.kt b/testing/e2e-test-utilities/src/main/kotlin/net/corda/e2etest/utilities/FlowTestUtils.kt index c4ee80f0296..7b34da3bbe5 100644 --- a/testing/e2e-test-utilities/src/main/kotlin/net/corda/e2etest/utilities/FlowTestUtils.kt +++ b/testing/e2e-test-utilities/src/main/kotlin/net/corda/e2etest/utilities/FlowTestUtils.kt @@ -21,7 +21,7 @@ fun startRpcFlow( ): String { return DEFAULT_CLUSTER.cluster { - assertWithRetry { + assertWithRetryIgnoringExceptions { timeout(RETRY_TIMEOUT) command { flowStart( From 33a9bf584c20ad471adf7e71f51e13527eba642f Mon Sep 17 00:00:00 2001 From: Miljenko Brkic <97448832+mbrkic-r3@users.noreply.github.com> Date: Sun, 15 Oct 2023 09:39:25 +0100 Subject: [PATCH 36/81] CORE-16203 5.1 Performance integration - FlowMapper using Multi-Source Event Mediator (#4875) Using Multi-Source Event Mediator instead of State and Event Subscription in FlowMapperWorker. --------- Co-authored-by: James Higgs --- .../FlowMapperServiceIntegrationTest.kt | 14 ++++- .../TestStateManagerFactoryImpl.kt | 21 ++++++- .../messaging/mediator/package-info.java | 4 ++ .../mapper/messaging/package-info.java | 4 ++ .../messaging/mediator/package-info.java | 4 -- .../service/messaging/package-info.java | 4 -- .../mapper/service/FlowMapperService.kt | 52 +++++---------- .../mapper/service/FlowMapperServiceTest.kt | 63 ++++++++++--------- .../flow/flow-mapper-service/test.bndrun | 1 - 9 files changed, 86 insertions(+), 81 deletions(-) create mode 100644 components/flow/flow-mapper-service/src/main/java/net/corda/session/mapper/messaging/mediator/package-info.java create mode 100644 components/flow/flow-mapper-service/src/main/java/net/corda/session/mapper/messaging/package-info.java delete mode 100644 components/flow/flow-mapper-service/src/main/java/net/corda/session/mapper/service/messaging/mediator/package-info.java delete mode 100644 components/flow/flow-mapper-service/src/main/java/net/corda/session/mapper/service/messaging/package-info.java diff --git a/components/flow/flow-mapper-service/src/integrationTest/kotlin/net/corda/session/mapper/service/integration/FlowMapperServiceIntegrationTest.kt b/components/flow/flow-mapper-service/src/integrationTest/kotlin/net/corda/session/mapper/service/integration/FlowMapperServiceIntegrationTest.kt index bb2923bda5e..5324858d6fb 100644 --- a/components/flow/flow-mapper-service/src/integrationTest/kotlin/net/corda/session/mapper/service/integration/FlowMapperServiceIntegrationTest.kt +++ b/components/flow/flow-mapper-service/src/integrationTest/kotlin/net/corda/session/mapper/service/integration/FlowMapperServiceIntegrationTest.kt @@ -11,6 +11,7 @@ import net.corda.data.flow.FlowStartContext import net.corda.data.flow.event.MessageDirection import net.corda.data.flow.event.SessionEvent import net.corda.data.flow.event.StartFlow +import net.corda.data.flow.event.mapper.ExecuteCleanup import net.corda.data.flow.event.mapper.FlowMapperEvent import net.corda.data.flow.event.mapper.ScheduleCleanup import net.corda.data.flow.event.session.SessionCounterpartyInfoRequest @@ -35,7 +36,6 @@ import net.corda.schema.Schemas.Flow.FLOW_EVENT_TOPIC import net.corda.schema.Schemas.Flow.FLOW_MAPPER_EVENT_TOPIC import net.corda.schema.Schemas.P2P.P2P_OUT_TOPIC import net.corda.schema.configuration.BootConfig.BOOT_MAX_ALLOWED_MSG_SIZE -import net.corda.schema.configuration.BootConfig.BOOT_STATE_MANAGER_JDBC_URL import net.corda.schema.configuration.BootConfig.INSTANCE_ID import net.corda.schema.configuration.BootConfig.TOPIC_PREFIX import net.corda.schema.configuration.ConfigKeys.FLOW_CONFIG @@ -243,6 +243,16 @@ class FlowMapperServiceIntegrationTest { assertFalse(flowEventLatch.await(3, TimeUnit.SECONDS)) assertThat(flowEventLatch.count).isEqualTo(1) + // Manually publish an execute cleanup event. Temporary until the full solution has been integrated. + val executeCleanup = Record( + FLOW_MAPPER_EVENT_TOPIC, + testId, + FlowMapperEvent( + ExecuteCleanup(listOf()) + ) + ) + publisher.publish(listOf(executeCleanup)) + //send same key start rpc again publisher.publish(listOf(startRPCEvent)) @@ -444,6 +454,7 @@ class FlowMapperServiceIntegrationTest { } processing { cleanupTime = 10000 + poolSize = 1 } """ @@ -462,6 +473,7 @@ class FlowMapperServiceIntegrationTest { producer { close.timeout = 6000 } + pollTimeout = 100 } """ diff --git a/components/flow/flow-mapper-service/src/integrationTest/kotlin/net/corda/session/mapper/service/integration/TestStateManagerFactoryImpl.kt b/components/flow/flow-mapper-service/src/integrationTest/kotlin/net/corda/session/mapper/service/integration/TestStateManagerFactoryImpl.kt index 702778df178..c65b184a118 100644 --- a/components/flow/flow-mapper-service/src/integrationTest/kotlin/net/corda/session/mapper/service/integration/TestStateManagerFactoryImpl.kt +++ b/components/flow/flow-mapper-service/src/integrationTest/kotlin/net/corda/session/mapper/service/integration/TestStateManagerFactoryImpl.kt @@ -7,6 +7,7 @@ import net.corda.libs.statemanager.api.State import net.corda.libs.statemanager.api.StateManager import net.corda.libs.statemanager.api.StateManagerFactory import org.osgi.service.component.annotations.Component +import java.util.concurrent.ConcurrentHashMap /** * The real state manager implementation requires postgres to run. As a result, it is impossible to plug it into the @@ -20,19 +21,33 @@ class TestStateManagerFactoryImpl : StateManagerFactory { override fun create(config: SmartConfig): StateManager { return object : StateManager { + private val storage = ConcurrentHashMap() override fun close() { } override fun create(states: Collection): Map { - TODO("Not yet implemented") + return states.mapNotNull { + storage.putIfAbsent(it.key, it) + }.associate { it.key to RuntimeException("State already exists [$it]") } } override fun get(keys: Collection): Map { - TODO("Not yet implemented") + return keys.mapNotNull { storage[it] }.associateBy { it.key } } override fun update(states: Collection): Map { - TODO("Not yet implemented") + return states.mapNotNull { + var output: State? = null + storage.compute(it.key) { _, existingState -> + if (existingState?.version == it.version) { + it.copy(version = it.version + 1) + } else { + output = it + it + } + } + output + }.associateBy { it.key } } override fun delete(states: Collection): Map { diff --git a/components/flow/flow-mapper-service/src/main/java/net/corda/session/mapper/messaging/mediator/package-info.java b/components/flow/flow-mapper-service/src/main/java/net/corda/session/mapper/messaging/mediator/package-info.java new file mode 100644 index 00000000000..a4ada847e77 --- /dev/null +++ b/components/flow/flow-mapper-service/src/main/java/net/corda/session/mapper/messaging/mediator/package-info.java @@ -0,0 +1,4 @@ +@Export +package net.corda.session.mapper.messaging.mediator; + +import org.osgi.annotation.bundle.Export; diff --git a/components/flow/flow-mapper-service/src/main/java/net/corda/session/mapper/messaging/package-info.java b/components/flow/flow-mapper-service/src/main/java/net/corda/session/mapper/messaging/package-info.java new file mode 100644 index 00000000000..882e59a06be --- /dev/null +++ b/components/flow/flow-mapper-service/src/main/java/net/corda/session/mapper/messaging/package-info.java @@ -0,0 +1,4 @@ +@Export +package net.corda.session.mapper.messaging; + +import org.osgi.annotation.bundle.Export; diff --git a/components/flow/flow-mapper-service/src/main/java/net/corda/session/mapper/service/messaging/mediator/package-info.java b/components/flow/flow-mapper-service/src/main/java/net/corda/session/mapper/service/messaging/mediator/package-info.java deleted file mode 100644 index 8d183e14ce3..00000000000 --- a/components/flow/flow-mapper-service/src/main/java/net/corda/session/mapper/service/messaging/mediator/package-info.java +++ /dev/null @@ -1,4 +0,0 @@ -@Export -package net.corda.session.mapper.service.messaging.mediator; - -import org.osgi.annotation.bundle.Export; diff --git a/components/flow/flow-mapper-service/src/main/java/net/corda/session/mapper/service/messaging/package-info.java b/components/flow/flow-mapper-service/src/main/java/net/corda/session/mapper/service/messaging/package-info.java deleted file mode 100644 index 2ad66720cc3..00000000000 --- a/components/flow/flow-mapper-service/src/main/java/net/corda/session/mapper/service/messaging/package-info.java +++ /dev/null @@ -1,4 +0,0 @@ -@Export -package net.corda.session.mapper.service.messaging; - -import org.osgi.annotation.bundle.Export; diff --git a/components/flow/flow-mapper-service/src/main/kotlin/net/corda/session/mapper/service/FlowMapperService.kt b/components/flow/flow-mapper-service/src/main/kotlin/net/corda/session/mapper/service/FlowMapperService.kt index 76569db3b21..17e2af7e018 100644 --- a/components/flow/flow-mapper-service/src/main/kotlin/net/corda/session/mapper/service/FlowMapperService.kt +++ b/components/flow/flow-mapper-service/src/main/kotlin/net/corda/session/mapper/service/FlowMapperService.kt @@ -2,9 +2,9 @@ package net.corda.session.mapper.service import net.corda.configuration.read.ConfigChangedEvent import net.corda.configuration.read.ConfigurationReadService -import net.corda.flow.mapper.factory.FlowMapperEventExecutorFactory import net.corda.libs.configuration.SmartConfig import net.corda.libs.configuration.helper.getConfig +import net.corda.libs.statemanager.api.StateManager import net.corda.libs.statemanager.api.StateManagerFactory import net.corda.lifecycle.Lifecycle import net.corda.lifecycle.LifecycleCoordinator @@ -15,21 +15,18 @@ import net.corda.lifecycle.LifecycleStatus import net.corda.lifecycle.RegistrationStatusChangeEvent import net.corda.lifecycle.StartEvent import net.corda.lifecycle.createCoordinator -import net.corda.messaging.api.publisher.config.PublisherConfig -import net.corda.messaging.api.publisher.factory.PublisherFactory +import net.corda.membership.locally.hosted.identities.LocallyHostedIdentitiesService import net.corda.messaging.api.subscription.config.SubscriptionConfig import net.corda.messaging.api.subscription.factory.SubscriptionFactory import net.corda.schema.Schemas.Flow.FLOW_MAPPER_CLEANUP_TOPIC -import net.corda.schema.Schemas.Flow.FLOW_MAPPER_EVENT_TOPIC import net.corda.schema.Schemas.ScheduledTask.SCHEDULED_TASK_TOPIC_MAPPER_PROCESSOR import net.corda.schema.configuration.ConfigKeys.FLOW_CONFIG import net.corda.schema.configuration.ConfigKeys.MESSAGING_CONFIG +import net.corda.schema.configuration.ConfigKeys.STATE_MANAGER_CONFIG import net.corda.schema.configuration.FlowConfig +import net.corda.session.mapper.messaging.mediator.FlowMapperEventMediatorFactory import net.corda.session.mapper.service.executor.CleanupProcessor -import net.corda.session.mapper.service.executor.FlowMapperListener -import net.corda.session.mapper.service.executor.FlowMapperMessageProcessor import net.corda.session.mapper.service.executor.ScheduledTaskProcessor -import net.corda.session.mapper.service.executor.ScheduledTaskState import net.corda.v5.base.exceptions.CordaRuntimeException import org.osgi.service.component.annotations.Activate import org.osgi.service.component.annotations.Component @@ -37,9 +34,6 @@ import org.osgi.service.component.annotations.Deactivate import org.osgi.service.component.annotations.Reference import org.slf4j.LoggerFactory import java.time.Clock -import java.util.concurrent.Executors -import net.corda.membership.locally.hosted.identities.LocallyHostedIdentitiesService -import net.corda.schema.configuration.ConfigKeys.STATE_MANAGER_CONFIG @Suppress("LongParameterList", "ForbiddenComment") @Component(service = [FlowMapperService::class]) @@ -50,10 +44,8 @@ class FlowMapperService @Activate constructor( private val configurationReadService: ConfigurationReadService, @Reference(service = SubscriptionFactory::class) private val subscriptionFactory: SubscriptionFactory, - @Reference(service = PublisherFactory::class) - private val publisherFactory: PublisherFactory, - @Reference(service = FlowMapperEventExecutorFactory::class) - private val flowMapperEventExecutorFactory: FlowMapperEventExecutorFactory, + @Reference(service = FlowMapperEventMediatorFactory::class) + private val flowMapperEventMediatorFactory: FlowMapperEventMediatorFactory, @Reference(service = StateManagerFactory::class) private val stateManagerFactory: StateManagerFactory ) : Lifecycle { @@ -63,8 +55,7 @@ class FlowMapperService @Activate constructor( private const val CONSUMER_GROUP = "FlowMapperConsumer" private const val SCHEDULED_TASK_CONSUMER_GROUP = "$CONSUMER_GROUP.scheduledTasks" private const val CLEANUP_TASK_CONSUMER_GROUP = "$CONSUMER_GROUP.cleanup" - private const val SUBSCRIPTION = "SUBSCRIPTION" - private const val CLEANUP_TASK = "TASK" + private const val EVENT_MEDIATOR = "EVENT_MEDIATOR" private const val REGISTRATION = "REGISTRATION" private const val CONFIG_HANDLE = "CONFIG_HANDLE" private const val SCHEDULED_TASK_PROCESSOR = "flow.mapper.scheduled.task.processor" @@ -117,30 +108,20 @@ class FlowMapperService @Activate constructor( val flowConfig = event.config.getConfig(FLOW_CONFIG) val stateManagerConfig = event.config.getConfig(STATE_MANAGER_CONFIG) - // TODO: This can be removed once the state manager is integrated into the flow mapper and the new cleanup - // tasks work correctly. - val newScheduledTaskState = coordinator.createManagedResource(CLEANUP_TASK) { - ScheduledTaskState( - Executors.newSingleThreadScheduledExecutor(), - publisherFactory.createPublisher( - PublisherConfig("$CONSUMER_GROUP-cleanup-publisher"), - messagingConfig - ), - mutableMapOf() - ) + val stateManager = coordinator.createManagedResource(STATE_MANAGER) { + stateManagerFactory.create(stateManagerConfig) } - coordinator.createManagedResource(SUBSCRIPTION) { - subscriptionFactory.createStateAndEventSubscription( - SubscriptionConfig(CONSUMER_GROUP, FLOW_MAPPER_EVENT_TOPIC), - FlowMapperMessageProcessor(flowMapperEventExecutorFactory, flowConfig), + coordinator.createManagedResource(EVENT_MEDIATOR) { + flowMapperEventMediatorFactory.create( + flowConfig, messagingConfig, - FlowMapperListener(newScheduledTaskState) + stateManager, ) }.also { it.start() } - setupCleanupTasks(messagingConfig, flowConfig, stateManagerConfig) + setupCleanupTasks(messagingConfig, flowConfig, stateManager) coordinator.updateStatus(LifecycleStatus.UP) } catch (e: CordaRuntimeException) { val errorMsg = "Error restarting flow mapper from config change" @@ -152,12 +133,9 @@ class FlowMapperService @Activate constructor( private fun setupCleanupTasks( messagingConfig: SmartConfig, flowConfig: SmartConfig, - stateManagerConfig: SmartConfig + stateManager: StateManager ) { val window = flowConfig.getLong(FlowConfig.PROCESSING_FLOW_CLEANUP_TIME) - val stateManager = coordinator.createManagedResource(STATE_MANAGER) { - stateManagerFactory.create(stateManagerConfig) - } val scheduledTaskProcessor = ScheduledTaskProcessor( stateManager, Clock.systemUTC(), diff --git a/components/flow/flow-mapper-service/src/test/kotlin/net/corda/session/mapper/service/FlowMapperServiceTest.kt b/components/flow/flow-mapper-service/src/test/kotlin/net/corda/session/mapper/service/FlowMapperServiceTest.kt index 2f132e854b2..9d5404958d2 100644 --- a/components/flow/flow-mapper-service/src/test/kotlin/net/corda/session/mapper/service/FlowMapperServiceTest.kt +++ b/components/flow/flow-mapper-service/src/test/kotlin/net/corda/session/mapper/service/FlowMapperServiceTest.kt @@ -15,13 +15,14 @@ import net.corda.lifecycle.LifecycleCoordinatorName import net.corda.lifecycle.test.impl.LifecycleTest import net.corda.membership.locally.hosted.identities.LocallyHostedIdentitiesService import net.corda.messaging.api.exception.CordaMessageAPIConfigException -import net.corda.messaging.api.subscription.StateAndEventSubscription +import net.corda.messaging.api.mediator.MultiSourceEventMediator import net.corda.messaging.api.subscription.Subscription import net.corda.messaging.api.subscription.factory.SubscriptionFactory import net.corda.schema.configuration.ConfigKeys.FLOW_CONFIG import net.corda.schema.configuration.ConfigKeys.MESSAGING_CONFIG import net.corda.schema.configuration.ConfigKeys.STATE_MANAGER_CONFIG import net.corda.schema.configuration.FlowConfig +import net.corda.session.mapper.messaging.mediator.FlowMapperEventMediatorFactory import org.junit.jupiter.api.Test import org.mockito.kotlin.any import org.mockito.kotlin.anyOrNull @@ -57,14 +58,17 @@ internal class FlowMapperServiceTest { @Test fun `flow mapper service correctly responds to dependencies changes`() { val subscriptionFactory = mock().also { - doAnswer { mock>() } - .whenever(it).createStateAndEventSubscription(any(), any(), any(), any()) doAnswer { mock>() } .whenever(it).createDurableSubscription(any(), any(), any(), anyOrNull()) } + val flowMapperEventMediatorFactory = mock().also { + doAnswer { mock>() } + .whenever(it).create(any(), any(), any()) + } val stateManagerFactory = mock().also { doAnswer { mock() }.whenever(it).create(any()) } + LifecycleTest { addDependency() addDependency() @@ -73,9 +77,8 @@ internal class FlowMapperServiceTest { coordinatorFactory, configReadService, subscriptionFactory, - mock(), - mock(), - stateManagerFactory + flowMapperEventMediatorFactory, + stateManagerFactory, ) }.run { @@ -106,7 +109,7 @@ internal class FlowMapperServiceTest { @Test fun `flow mapper service correctly reacts to config changes`() { val subName = LifecycleCoordinatorName("sub") - val subscription = mock>().apply { + val eventMediator = mock>().apply { whenever(subscriptionName).thenReturn(subName) } val scheduledTaskSubName = LifecycleCoordinatorName("scheduledSub") @@ -118,13 +121,15 @@ internal class FlowMapperServiceTest { whenever(subscriptionName).thenReturn(cleanupTaskSubName) } val subscriptionFactory = mock().apply { - whenever(createStateAndEventSubscription(any(), any(), any(), any())) - .thenReturn(subscription) whenever(createDurableSubscription(any(), any(), any(), anyOrNull())) .thenReturn(scheduledTaskSubscription) whenever(createDurableSubscription(any(), any(), any(), anyOrNull())) .thenReturn(cleanupSubscription) } + val flowMapperEventMediatorFactory = mock().apply { + whenever(create(any(), any(), any())) + .thenReturn(eventMediator) + } val stateManagerFactory = mock().also { doAnswer { mock() }.whenever(it).create(any()) } @@ -138,9 +143,8 @@ internal class FlowMapperServiceTest { coordinatorFactory, configReadService, subscriptionFactory, - mock(), - mock(), - stateManagerFactory + flowMapperEventMediatorFactory, + stateManagerFactory, ) }.run { testClass.start() @@ -150,27 +154,25 @@ internal class FlowMapperServiceTest { sendConfigUpdate(configMap) - // Create and start the subscription (using the message config) - verify(subscriptionFactory).createStateAndEventSubscription( - any(), + // Create and start the event mediator (using the message config) + verify(flowMapperEventMediatorFactory).create( any(), eq(messagingConfig), - any() + any(), ) - verify(subscription).start() + verify(eventMediator).start() verifyIsUp() sendConfigUpdate(configMap) - // Close, recreate and start the subscription (using the message config) - verify(subscription).close() - verify(subscriptionFactory, times(2)).createStateAndEventSubscription( - any(), + // Close, recreate and start the event mediator (using the message config) + verify(eventMediator).close() + verify(flowMapperEventMediatorFactory, times(2)).create( any(), eq(messagingConfig), - any() + any(), ) - verify(subscription, times(2)).start() + verify(eventMediator, times(2)).start() verifyIsUp() } } @@ -178,8 +180,9 @@ internal class FlowMapperServiceTest { @Test fun `flow mapper service correctly handles bad config`() { val subName = LifecycleCoordinatorName("sub") - val subscriptionFactory = mock().apply { - whenever(createStateAndEventSubscription(any(), any(), any(), any())) + val subscriptionFactory = mock() + val flowMapperEventMediatorFactory = mock().apply { + whenever(create(any(), any(), any())) .thenThrow(CordaMessageAPIConfigException("Bad config!")) } val stateManagerFactory = mock().also { @@ -195,9 +198,8 @@ internal class FlowMapperServiceTest { coordinatorFactory, configReadService, subscriptionFactory, - mock(), - mock(), - stateManagerFactory + flowMapperEventMediatorFactory, + stateManagerFactory, ) }.run { testClass.start() @@ -208,11 +210,10 @@ internal class FlowMapperServiceTest { sendConfigUpdate(configMap) // Create and start the subscription (using the message config) - verify(subscriptionFactory).createStateAndEventSubscription( - any(), + verify(flowMapperEventMediatorFactory).create( any(), eq(messagingConfig), - any() + any(), ) verifyIsInError() } diff --git a/components/flow/flow-mapper-service/test.bndrun b/components/flow/flow-mapper-service/test.bndrun index f9b85cf75bd..08ad238db17 100644 --- a/components/flow/flow-mapper-service/test.bndrun +++ b/components/flow/flow-mapper-service/test.bndrun @@ -3,7 +3,6 @@ -resolve.effective: resolve,active -runee: JavaSE-17 -runtrace: true -#-runjdb: 5005 # Enable debugging. # -runjdb: 1044 From 73887dcb1eb8fc39324fdc2957c7f6c61b2a89a3 Mon Sep 17 00:00:00 2001 From: Miljenko Brkic Date: Sun, 15 Oct 2023 23:09:50 +0100 Subject: [PATCH 37/81] CORE-17562 Metrics for Multi-Source Event Mediator --- .../messaging/constants/MetricsConstants.kt | 1 + .../mediator/MultiSourceEventMediatorImpl.kt | 18 ++++++++++----- .../messaging/mediator/TaskManagerHelper.kt | 14 +++++++---- .../mediator/metrics/EventMediatorMetrics.kt | 23 +++++++++++++++++++ .../mediator/TaskManagerHelperTest.kt | 12 +++++++--- 5 files changed, 54 insertions(+), 14 deletions(-) create mode 100644 libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/metrics/EventMediatorMetrics.kt diff --git a/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/constants/MetricsConstants.kt b/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/constants/MetricsConstants.kt index f6fd58c0709..fb0848080d6 100644 --- a/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/constants/MetricsConstants.kt +++ b/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/constants/MetricsConstants.kt @@ -7,6 +7,7 @@ object MetricsConstants { const val DURABLE_PATTERN_TYPE = "Durable" const val RPC_PATTERN_TYPE = "RPC" const val STATE_AND_EVENT_PATTERN_TYPE = "StateAndEvent" + const val EVENT_MEDIATOR_TYPE = "EventMediator" // Operation types, to use with OperationName tag const val ON_NEXT_OPERATION = "onNext" diff --git a/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/MultiSourceEventMediatorImpl.kt b/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/MultiSourceEventMediatorImpl.kt index f0d6e9d1423..4fe9f9d5270 100644 --- a/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/MultiSourceEventMediatorImpl.kt +++ b/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/MultiSourceEventMediatorImpl.kt @@ -15,6 +15,7 @@ import net.corda.messaging.api.mediator.MessagingClient import net.corda.messaging.api.mediator.MultiSourceEventMediator import net.corda.messaging.api.mediator.config.EventMediatorConfig import net.corda.messaging.mediator.factory.MediatorComponentFactory +import net.corda.messaging.mediator.metrics.EventMediatorMetrics import net.corda.taskmanager.TaskManager import net.corda.utilities.debug import org.slf4j.LoggerFactory @@ -38,11 +39,12 @@ class MultiSourceEventMediatorImpl( private val mediatorComponentFactory = MediatorComponentFactory( config.messageProcessor, config.consumerFactories, config.clientFactories, config.messageRouterFactory ) + private val metrics = EventMediatorMetrics(config.name) private val stateManagerHelper = StateManagerHelper( stateManager, stateSerializer, stateDeserializer ) private val taskManagerHelper = TaskManagerHelper( - taskManager, stateManagerHelper + taskManager, stateManagerHelper, metrics ) private val uniqueId = UUID.randomUUID().toString() private val lifecycleCoordinatorName = LifecycleCoordinatorName( @@ -173,14 +175,18 @@ class MultiSourceEventMediatorImpl( } private fun pollConsumers(): List> { - return consumers.map { consumer -> - consumer.poll(config.pollTimeout) - }.flatten() + return metrics.pollTimer.recordCallable { + consumers.map { consumer -> + consumer.poll(config.pollTimeout) + }.flatten() + }!! } private fun commitOffsets() { - consumers.map { consumer -> - consumer.syncCommitOffsets() + metrics.commitTimer.recordCallable { + consumers.map { consumer -> + consumer.syncCommitOffsets() + } } } diff --git a/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/TaskManagerHelper.kt b/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/TaskManagerHelper.kt index 6e065c5190b..ec9a14e73fa 100644 --- a/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/TaskManagerHelper.kt +++ b/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/TaskManagerHelper.kt @@ -8,6 +8,7 @@ import net.corda.messaging.api.mediator.MessageRouter import net.corda.messaging.api.mediator.MessagingClient.Companion.MSG_PROP_KEY import net.corda.messaging.api.processor.StateAndEventProcessor import net.corda.messaging.api.records.Record +import net.corda.messaging.mediator.metrics.EventMediatorMetrics import net.corda.messaging.utils.toRecord import net.corda.taskmanager.TaskManager @@ -17,6 +18,7 @@ import net.corda.taskmanager.TaskManager internal class TaskManagerHelper( private val taskManager: TaskManager, private val stateManagerHelper: StateManagerHelper, + private val metrics: EventMediatorMetrics, ) { /** @@ -97,11 +99,13 @@ internal class TaskManagerHelper( fun executeProcessorTasks( processorTasks: Collection> ): List> { - return processorTasks.map { processorTask -> - taskManager.executeShortRunningTask(processorTask::call) - }.map { - it.join() - } + return metrics.processorTimer.recordCallable { + processorTasks.map { processorTask -> + taskManager.executeShortRunningTask(processorTask::call) + }.map { + it.join() + } + }!! } /** diff --git a/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/metrics/EventMediatorMetrics.kt b/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/metrics/EventMediatorMetrics.kt new file mode 100644 index 00000000000..567fcb1bd02 --- /dev/null +++ b/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/metrics/EventMediatorMetrics.kt @@ -0,0 +1,23 @@ +package net.corda.messaging.mediator.metrics + +import net.corda.messaging.constants.MetricsConstants +import net.corda.metrics.CordaMetrics + +class EventMediatorMetrics( + mediatorName: String, +) { + val processorTimer = CordaMetrics.Metric.Messaging.MessageProcessorTime.builder() + .withTag(CordaMetrics.Tag.MessagePatternType, MetricsConstants.EVENT_MEDIATOR_TYPE) + .withTag(CordaMetrics.Tag.MessagePatternClientId, mediatorName) + .withTag(CordaMetrics.Tag.OperationName, MetricsConstants.BATCH_PROCESS_OPERATION) + .build() + + val pollTimer = CordaMetrics.Metric.Messaging.ConsumerPollTime.builder() + .withTag(CordaMetrics.Tag.MessagePatternClientId, mediatorName) + .build() + + val commitTimer = CordaMetrics.Metric.Messaging.MessageCommitTime.builder() + .withTag(CordaMetrics.Tag.MessagePatternType, MetricsConstants.EVENT_MEDIATOR_TYPE) + .withTag(CordaMetrics.Tag.MessagePatternClientId, mediatorName) + .build() +} \ No newline at end of file diff --git a/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/TaskManagerHelperTest.kt b/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/TaskManagerHelperTest.kt index fd81671121f..fc26abcc68e 100644 --- a/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/TaskManagerHelperTest.kt +++ b/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/TaskManagerHelperTest.kt @@ -7,6 +7,7 @@ import net.corda.messaging.api.mediator.MessageRouter import net.corda.messaging.api.mediator.MessagingClient.Companion.MSG_PROP_KEY import net.corda.messaging.api.processor.StateAndEventProcessor import net.corda.messaging.api.records.Record +import net.corda.messaging.mediator.metrics.EventMediatorMetrics import net.corda.taskmanager.TaskManager import org.assertj.core.api.Assertions.assertThat import org.junit.jupiter.api.Assertions.assertEquals @@ -31,7 +32,8 @@ class TaskManagerHelperTest { private val taskManager = mock() private val stateManagerHelper = mock>() - private val taskManagerHelper = TaskManagerHelper(taskManager, stateManagerHelper) + private val eventMediatorMetrics = mock() + private val taskManagerHelper = TaskManagerHelper(taskManager, stateManagerHelper, eventMediatorMetrics) private val messageProcessor = mock>() @Test @@ -167,7 +169,8 @@ class TaskManagerHelperTest { val processorTask1 = mock>() val processorTask2 = mock>() - `when`(taskManager.executeShortRunningTask(any<() -> ProcessorTask.Result>())).thenReturn(mock()) + `when`(taskManager.executeShortRunningTask(any<() -> ProcessorTask.Result>())) + .thenReturn(mock()) taskManagerHelper.executeProcessorTasks( listOf(processorTask1, processorTask2) @@ -248,7 +251,10 @@ class TaskManagerHelperTest { mapOf("1" to listOf(clientTask1), "2" to listOf(clientTask2)) ) assertThat(results).containsOnly(result1, result2) - verify(taskManager, times(2)).executeShortRunningTask(any<() -> List>>()) + verify( + taskManager, + times(2) + ).executeShortRunningTask(any<() -> List>>()) } private fun List.toCordaConsumerRecords(key: String) = From b965fd85d60fe275c8e37eef11040d530cf54892 Mon Sep 17 00:00:00 2001 From: Miljenko Brkic Date: Mon, 16 Oct 2023 00:12:05 +0100 Subject: [PATCH 38/81] CORE-17562 Fixed unit test --- .../net/corda/messaging/mediator/TaskManagerHelperTest.kt | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/TaskManagerHelperTest.kt b/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/TaskManagerHelperTest.kt index fc26abcc68e..b0b768fd9cd 100644 --- a/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/TaskManagerHelperTest.kt +++ b/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/TaskManagerHelperTest.kt @@ -1,5 +1,6 @@ package net.corda.messaging.mediator +import io.micrometer.core.instrument.Timer import net.corda.libs.statemanager.api.State import net.corda.messagebus.api.consumer.CordaConsumerRecord import net.corda.messaging.api.mediator.MediatorMessage @@ -19,6 +20,7 @@ import org.mockito.kotlin.mock import org.mockito.kotlin.times import org.mockito.kotlin.verify import org.mockito.kotlin.whenever +import java.util.concurrent.Callable import java.util.concurrent.CompletableFuture class TaskManagerHelperTest { @@ -171,6 +173,11 @@ class TaskManagerHelperTest { `when`(taskManager.executeShortRunningTask(any<() -> ProcessorTask.Result>())) .thenReturn(mock()) + val timer = mock() + whenever(timer.recordCallable(any>())).thenAnswer { invocation -> + invocation.getArgument>(0).call() + } + whenever(eventMediatorMetrics.processorTimer).thenReturn(timer) taskManagerHelper.executeProcessorTasks( listOf(processorTask1, processorTask2) From 7d2a1332603fa07c67ef0f985f0435edda4cf5c4 Mon Sep 17 00:00:00 2001 From: William Vigor <58432369+williamvigorr3@users.noreply.github.com> Date: Tue, 17 Oct 2023 09:58:28 +0100 Subject: [PATCH 39/81] CORE-17514 Change the Link Manager to handle bad group policy (#4854) This can happen if we can't query the DB worker for the MGM's group policy. --- .../linkmanager/common/MessageConverter.kt | 15 +- .../delivery/InMemorySessionReplayer.kt | 10 +- .../outbound/OutboundMessageProcessor.kt | 9 +- .../sessions/SessionManagerImpl.kt | 39 +++- .../sessions/SessionManagerWarnings.kt | 4 + .../common/MessageConverterTest.kt | 29 ++- .../delivery/InMemorySessionReplayerTest.kt | 32 ++- .../outbound/OutboundMessageProcessorTest.kt | 48 ++++- .../sessions/SessionManagerTest.kt | 199 +++++++++++++++++- .../grouppolicy/GroupPolicyProviderImpl.kt | 14 +- .../GroupPolicyProviderImplTest.kt | 37 +++- .../membership/lib/grouppolicy/GroupPolicy.kt | 26 --- .../impl/grouppolicy/v1/MGMGroupPolicyImpl.kt | 87 +++----- .../grouppolicy/v1/MGMGroupPolicyImplTest.kt | 83 ++++++-- 14 files changed, 510 insertions(+), 122 deletions(-) diff --git a/components/link-manager/src/main/kotlin/net/corda/p2p/linkmanager/common/MessageConverter.kt b/components/link-manager/src/main/kotlin/net/corda/p2p/linkmanager/common/MessageConverter.kt index 984ecad0c92..a45df208519 100644 --- a/components/link-manager/src/main/kotlin/net/corda/p2p/linkmanager/common/MessageConverter.kt +++ b/components/link-manager/src/main/kotlin/net/corda/p2p/linkmanager/common/MessageConverter.kt @@ -34,6 +34,7 @@ import org.apache.avro.AvroRuntimeException import org.slf4j.LoggerFactory import java.io.IOException import java.nio.ByteBuffer +import net.corda.membership.lib.exceptions.BadGroupPolicyException /** * This class contains code which can be used to convert between [LinkOutMessage]/[LinkInMessage] and @@ -231,12 +232,16 @@ class MessageConverter { "which is not in the network map. The message was discarded.") return null } - val p2pParams = groupPolicyProvider.getP2PParameters(source) + val p2pParams = try { + groupPolicyProvider.getP2PParameters(source) + } catch (except: BadGroupPolicyException) { + logger.warn("The group policy data is unavailable or cannot be parsed for our identity = $source. Error:" + + " ${except.message}. The message was discarded.") + return null + } if (p2pParams == null) { - logger.warn( - "Could not find the group info in the " + - "GroupPolicyProvider for our identity = $source. The message was discarded." - ) + logger.warn("Could not find the group info in the GroupPolicyProvider for our identity = $source." + + " The message was discarded.") return null } diff --git a/components/link-manager/src/main/kotlin/net/corda/p2p/linkmanager/delivery/InMemorySessionReplayer.kt b/components/link-manager/src/main/kotlin/net/corda/p2p/linkmanager/delivery/InMemorySessionReplayer.kt index 6b7906b4704..29a9f29c601 100644 --- a/components/link-manager/src/main/kotlin/net/corda/p2p/linkmanager/delivery/InMemorySessionReplayer.kt +++ b/components/link-manager/src/main/kotlin/net/corda/p2p/linkmanager/delivery/InMemorySessionReplayer.kt @@ -8,6 +8,7 @@ import net.corda.lifecycle.domino.logic.ComplexDominoTile import net.corda.lifecycle.domino.logic.LifecycleWithDominoTile import net.corda.lifecycle.domino.logic.util.PublisherWithDominoLogic import net.corda.membership.grouppolicy.GroupPolicyProvider +import net.corda.membership.lib.exceptions.BadGroupPolicyException import net.corda.membership.read.MembershipGroupReaderProvider import net.corda.messaging.api.publisher.config.PublisherConfig import net.corda.messaging.api.publisher.factory.PublisherFactory @@ -140,7 +141,14 @@ internal class InMemorySessionReplayer( return } - val networkType = groupPolicyProvider.getP2PParameters(messageReplay.sessionCounterparties.ourId)?.networkType + val networkType = try { + groupPolicyProvider.getP2PParameters(messageReplay.sessionCounterparties.ourId)?.networkType + } catch (except: BadGroupPolicyException) { + logger.warn("Attempted to replay a session negotiation message (type ${messageReplay.message::class.java.simpleName}) but" + + " the group policy data is unavailable or cannot be parsed for ${messageReplay.sessionCounterparties.ourId}. Error:" + + " ${except.message}. The message was not replayed.") + return + } if (networkType == null) { logger.warn("Attempted to replay a session negotiation message (type ${messageReplay.message::class.java.simpleName}) but" + " could not find the network type in the GroupPolicyProvider for ${messageReplay.sessionCounterparties.ourId}." + diff --git a/components/link-manager/src/main/kotlin/net/corda/p2p/linkmanager/outbound/OutboundMessageProcessor.kt b/components/link-manager/src/main/kotlin/net/corda/p2p/linkmanager/outbound/OutboundMessageProcessor.kt index f33562fc96d..f28b16f4e4c 100644 --- a/components/link-manager/src/main/kotlin/net/corda/p2p/linkmanager/outbound/OutboundMessageProcessor.kt +++ b/components/link-manager/src/main/kotlin/net/corda/p2p/linkmanager/outbound/OutboundMessageProcessor.kt @@ -42,6 +42,7 @@ import net.corda.virtualnode.toCorda import org.slf4j.Logger import org.slf4j.LoggerFactory import java.time.Instant +import net.corda.membership.lib.exceptions.BadGroupPolicyException @Suppress("LongParameterList", "TooManyFunctions") internal class OutboundMessageProcessor( @@ -216,7 +217,13 @@ internal class OutboundMessageProcessor( return listOf(Record(Schemas.P2P.P2P_IN_TOPIC, LinkManager.generateKey(), AppMessage(inboundMessage))) } else { val source = message.header.source.toCorda() - val p2pParams = groupPolicyProvider.getP2PParameters(source) + val p2pParams = try { + groupPolicyProvider.getP2PParameters(source) + } catch (except: BadGroupPolicyException) { + logger.warn("The group policy data is unavailable or cannot be parsed for $source. Error: ${except.message}. The message" + + " ${message.header.messageId} was discarded.") + return emptyList() + } if (p2pParams == null) { logger.warn( "Could not find the p2p parameters in the GroupPolicyProvider for $source. " + diff --git a/components/link-manager/src/main/kotlin/net/corda/p2p/linkmanager/sessions/SessionManagerImpl.kt b/components/link-manager/src/main/kotlin/net/corda/p2p/linkmanager/sessions/SessionManagerImpl.kt index f87e263c988..954d80df55d 100644 --- a/components/link-manager/src/main/kotlin/net/corda/p2p/linkmanager/sessions/SessionManagerImpl.kt +++ b/components/link-manager/src/main/kotlin/net/corda/p2p/linkmanager/sessions/SessionManagerImpl.kt @@ -95,8 +95,10 @@ import java.util.concurrent.ScheduledExecutorService import java.util.concurrent.TimeUnit import java.util.concurrent.atomic.AtomicReference import java.util.concurrent.locks.ReentrantReadWriteLock +import net.corda.membership.lib.exceptions.BadGroupPolicyException import net.corda.p2p.crypto.protocol.api.InvalidSelectedModeError import net.corda.p2p.crypto.protocol.api.NoCommonModeError +import net.corda.p2p.linkmanager.sessions.SessionManagerWarnings.badGroupPolicy import kotlin.concurrent.read import kotlin.concurrent.write @@ -432,7 +434,13 @@ internal class SessionManagerImpl( multiplicity: Int ): List> { - val p2pParams = groupPolicyProvider.getP2PParameters(counterparties.ourId) + val p2pParams = try { + groupPolicyProvider.getP2PParameters(counterparties.ourId) + } catch (except: BadGroupPolicyException) { + logger.warn("The group policy data is unavailable or cannot be parsed for ${counterparties.ourId}. Error: ${except.message}. " + + "The sessionInit message was not sent.") + return emptyList() + } if (p2pParams == null) { logger.warn( "Could not find the p2p parameters in the GroupPolicyProvider for ${counterparties.ourId}." + @@ -549,7 +557,13 @@ internal class SessionManagerImpl( return null } - val p2pParams = groupPolicyProvider.getP2PParameters(sessionCounterparties.ourId) + val p2pParams = try { + groupPolicyProvider.getP2PParameters(sessionCounterparties.ourId) + } catch (except: BadGroupPolicyException) { + logger.warn("The group policy data is unavailable or cannot be parsed for ${sessionCounterparties.ourId}. Error: " + + "${except.message}. The sessionInit message was not sent.") + return emptyList() + } if (p2pParams == null) { logger.warn( "Could not find the group information in the GroupPolicyProvider for ${sessionCounterparties.ourId}." + @@ -649,7 +663,12 @@ internal class SessionManagerImpl( sessionInfo ) - val p2pParams = groupPolicyProvider.getP2PParameters(ourIdentityInfo.holdingIdentity) + val p2pParams = try { + groupPolicyProvider.getP2PParameters(ourIdentityInfo.holdingIdentity) + } catch (except: BadGroupPolicyException) { + logger.badGroupPolicy(message::class.java.simpleName, message.header.sessionId, ourIdentityInfo.holdingIdentity, except.message) + return null + } if (p2pParams == null) { logger.couldNotFindGroupInfo(message::class.java.simpleName, message.header.sessionId, ourIdentityInfo.holdingIdentity) return null @@ -762,7 +781,12 @@ internal class SessionManagerImpl( } val (hostedIdentityInSameGroup, peerMemberInfo) = locallyHostedIdentityWithPeerMemberInfo - val p2pParams = groupPolicyProvider.getP2PParameters(hostedIdentityInSameGroup) + val p2pParams = try { + groupPolicyProvider.getP2PParameters(hostedIdentityInSameGroup) + } catch (except: BadGroupPolicyException) { + logger.badGroupPolicy(message::class.java.simpleName, message.header.sessionId, hostedIdentityInSameGroup, except.message) + return null + } if (p2pParams == null) { logger.couldNotFindGroupInfo(message::class.java.simpleName, message.header.sessionId, hostedIdentityInSameGroup) return null @@ -827,7 +851,12 @@ internal class SessionManagerImpl( return null } - val p2pParams = groupPolicyProvider.getP2PParameters(ourIdentityInfo.holdingIdentity) + val p2pParams = try { + groupPolicyProvider.getP2PParameters(ourIdentityInfo.holdingIdentity) + } catch (except: BadGroupPolicyException) { + logger.badGroupPolicy(message::class.java.simpleName, message.header.sessionId, ourIdentityInfo.holdingIdentity, except.message) + return null + } if (p2pParams == null) { logger.couldNotFindGroupInfo(message::class.java.simpleName, message.header.sessionId, ourIdentityInfo.holdingIdentity) return null diff --git a/components/link-manager/src/main/kotlin/net/corda/p2p/linkmanager/sessions/SessionManagerWarnings.kt b/components/link-manager/src/main/kotlin/net/corda/p2p/linkmanager/sessions/SessionManagerWarnings.kt index cd15130104d..249e7ac0eb0 100644 --- a/components/link-manager/src/main/kotlin/net/corda/p2p/linkmanager/sessions/SessionManagerWarnings.kt +++ b/components/link-manager/src/main/kotlin/net/corda/p2p/linkmanager/sessions/SessionManagerWarnings.kt @@ -50,6 +50,10 @@ internal object SessionManagerWarnings { " The message was discarded." ) } + internal fun Logger.badGroupPolicy(messageName: String, sessionId: String, holdingIdentity: HoldingIdentity, message: String?) { + this.warn("The group policy data is unavailable or cannot be parsed for identity $holdingIdentity. Error: $message. The " + + "$messageName for sessionId $sessionId was discarded.") + } internal fun Logger.couldNotFindGroupInfo(messageName: String, sessionId: String, holdingIdentity: HoldingIdentity) { this.warn( diff --git a/components/link-manager/src/test/kotlin/net/corda/p2p/linkmanager/common/MessageConverterTest.kt b/components/link-manager/src/test/kotlin/net/corda/p2p/linkmanager/common/MessageConverterTest.kt index 9771f8fa0af..65094208b49 100644 --- a/components/link-manager/src/test/kotlin/net/corda/p2p/linkmanager/common/MessageConverterTest.kt +++ b/components/link-manager/src/test/kotlin/net/corda/p2p/linkmanager/common/MessageConverterTest.kt @@ -30,6 +30,8 @@ import org.mockito.kotlin.doReturn import org.mockito.kotlin.doThrow import org.mockito.kotlin.mock import java.nio.ByteBuffer +import net.corda.membership.grouppolicy.GroupPolicyProvider +import net.corda.membership.lib.exceptions.BadGroupPolicyException class MessageConverterTest { @@ -100,7 +102,7 @@ class MessageConverterTest { } @Test - fun `createLinkOutMessageFromFlowMessage returns null (with appropriate logging) if the destination is not in the network map`() { + fun `linkOutMessageFromAuthenticatedMessageAndKey returns null if the destination is not in the network map`() { val mac = mock { on { header } doReturn mockHeader on { mac } doReturn byteArrayOf() @@ -127,7 +129,7 @@ class MessageConverterTest { } @Test - fun `createLinkOutMessageFromFlowMessage returns null if the destination does not have the expected serial`() { + fun `linkOutMessageFromAuthenticatedMessageAndKey returns null if the destination does not have the expected serial`() { val mac = mock { on { header } doReturn mockHeader on { mac } doReturn byteArrayOf() @@ -154,7 +156,7 @@ class MessageConverterTest { } @Test - fun `createLinkOutMessageFromFlowMessage returns null (with appropriate logging) if our network type is not in the network map`() { + fun `linkOutMessageFromAuthenticatedMessageAndKey returns null if our p2p params is not in the group policy provider`() { val mac = mock { on { header } doReturn mockHeader on { mac } doReturn byteArrayOf() @@ -175,6 +177,27 @@ class MessageConverterTest { ) } + @Test + fun `linkOutMessageFromAuthenticatedMessageAndKey returns null, if BadGroupPolicy exception is thrown on group policy look up`() { + val mac = mock { + on { header } doReturn mockHeader + on { mac } doReturn byteArrayOf() + } + val session = mock { + on { createMac(any()) } doReturn mac + } + val groupId = "group-1" + val peer = createTestHoldingIdentity("CN=Impostor, O=Evil Corp, L=LDN, C=GB", groupId) + val us = createTestHoldingIdentity("CN=Bob, O=Bob Corp, L=LDN, C=GB", groupId) + val members = mockMembers(listOf(us, peer)) + val groups = mock { + on { getP2PParameters(us) } doThrow BadGroupPolicyException("Bad group policy.") + } + val flowMessage = authenticatedMessageAndKey(us, peer, ByteBuffer.wrap("DATA".toByteArray())) + assertThat(MessageConverter.linkOutMessageFromAuthenticatedMessageAndKey(flowMessage, session, groups, members, 1)).isNull() + loggingInterceptor.assertSingleWarningContains("Bad group policy.") + } + @Test fun `createLinkOutMessage does not validate serial when it's not provided`(){ val mac = mock { diff --git a/components/link-manager/src/test/kotlin/net/corda/p2p/linkmanager/delivery/InMemorySessionReplayerTest.kt b/components/link-manager/src/test/kotlin/net/corda/p2p/linkmanager/delivery/InMemorySessionReplayerTest.kt index 233af7e05db..52838b3d2d2 100644 --- a/components/link-manager/src/test/kotlin/net/corda/p2p/linkmanager/delivery/InMemorySessionReplayerTest.kt +++ b/components/link-manager/src/test/kotlin/net/corda/p2p/linkmanager/delivery/InMemorySessionReplayerTest.kt @@ -42,6 +42,8 @@ import org.mockito.kotlin.whenever import java.security.KeyPairGenerator import java.security.Security import java.util.UUID +import net.corda.membership.lib.exceptions.BadGroupPolicyException +import org.mockito.kotlin.doThrow class InMemorySessionReplayerTest { @@ -181,15 +183,12 @@ class InMemorySessionReplayerTest { } @Test - fun `The replaySchedular callback logs a warning when our network type is not in the network map`() { + fun `The replaySchedular callback logs a warning when our network type is not in the group policy provider`() { val parameters = mock { on { tlsPki } doReturn GroupPolicyConstants.PolicyValues.P2PParameters.TlsPkiMode.STANDARD } - val groupPolicy = mock { - on { p2pParameters } doReturn parameters - } val groups = mock { - on {getGroupPolicy(any()) } doReturnConsecutively listOf(null, groupPolicy) + on {getP2PParameters(any()) } doReturnConsecutively listOf(null, parameters) } InMemorySessionReplayer(mock(), mock(), mock(), mock(), groups, groupsAndMembers.first, mockTimeFacilitiesProvider.clock) @@ -211,6 +210,29 @@ class InMemorySessionReplayerTest { " $US. The message was not replayed.") } + @Test + fun `The replaySchedular callback logs a warning, if BadGroupPolicyException is thrown on group policy lookup`() { + val groups = mock { + on {getP2PParameters(any()) } doThrow BadGroupPolicyException("Bad group policy.") + } + + InMemorySessionReplayer(mock(), mock(), mock(), mock(), groups, groupsAndMembers.first, mockTimeFacilitiesProvider.clock) + val helloMessage = AuthenticationProtocolInitiator( + id, + setOf(ProtocolMode.AUTHENTICATION_ONLY), + MAX_MESSAGE_SIZE, + KEY_PAIR.public, + GROUP_ID, + CertificateCheckMode.NoCertificate + ).generateInitiatorHello() + + setRunning() + val messageReplay = InMemorySessionReplayer.SessionMessageReplay(helloMessage, id, SESSION_COUNTERPARTIES) { _,_ -> } + replayCallback(messageReplay, "foo-bar") + + loggingInterceptor.assertSingleWarningContains("Bad group policy.") + } + @Test fun `The replaySchedular callback logs a warning when the responder is not in the network map`() { val memberInfo = groupsAndMembers.first.getGroupReader(US).lookup(COUNTER_PARTY.x500Name) diff --git a/components/link-manager/src/test/kotlin/net/corda/p2p/linkmanager/outbound/OutboundMessageProcessorTest.kt b/components/link-manager/src/test/kotlin/net/corda/p2p/linkmanager/outbound/OutboundMessageProcessorTest.kt index dd589981395..65499e605f7 100644 --- a/components/link-manager/src/test/kotlin/net/corda/p2p/linkmanager/outbound/OutboundMessageProcessorTest.kt +++ b/components/link-manager/src/test/kotlin/net/corda/p2p/linkmanager/outbound/OutboundMessageProcessorTest.kt @@ -45,6 +45,8 @@ import org.mockito.kotlin.verify import org.mockito.kotlin.whenever import java.nio.ByteBuffer import java.time.Instant +import net.corda.membership.lib.exceptions.BadGroupPolicyException +import org.mockito.kotlin.doThrow class OutboundMessageProcessorTest { private val myIdentity = createTestHoldingIdentity("CN=PartyA, O=Corp, L=LDN, C=GB", "Group") @@ -576,7 +578,51 @@ class OutboundMessageProcessorTest { @Test fun `unauthenticated messages are dropped if group info is not available`() { val groupPolicyProvider = mock { - on { getP2PParameters(localIdentity) } doReturn null + on { getP2PParameters(myIdentity) } doReturn null + } + + val processor = OutboundMessageProcessor( + sessionManager, + hostingMap, + groupPolicyProvider, + membersAndGroups.first, + assignedListener, + messagesPendingSession, + mockTimeFacilitiesProvider.clock, + networkMessagingValidator, + ) + + val payload = "test" + val unauthenticatedMsg = OutboundUnauthenticatedMessage( + OutboundUnauthenticatedMessageHeader( + remoteIdentity.toAvro(), + myIdentity.toAvro(), + "subsystem", + "messageId", + ), + ByteBuffer.wrap(payload.toByteArray()), + ) + val appMessage = AppMessage(unauthenticatedMsg) + + val records = processor.onNext( + listOf( + EventLogRecord( + Schemas.P2P.P2P_OUT_TOPIC, + "key", + appMessage, + 1, + 0 + ) + ) + ) + + assertThat(records).isEmpty() + } + + @Test + fun `unauthenticated messages are dropped, if BadGroupPolicyException is thrown on group policy lookup`() { + val groupPolicyProvider = mock { + on { getP2PParameters(myIdentity) } doThrow BadGroupPolicyException("Bad Group Policy") } val processor = OutboundMessageProcessor( diff --git a/components/link-manager/src/test/kotlin/net/corda/p2p/linkmanager/sessions/SessionManagerTest.kt b/components/link-manager/src/test/kotlin/net/corda/p2p/linkmanager/sessions/SessionManagerTest.kt index 6a10d3a7a7f..941d7ff2b80 100644 --- a/components/link-manager/src/test/kotlin/net/corda/p2p/linkmanager/sessions/SessionManagerTest.kt +++ b/components/link-manager/src/test/kotlin/net/corda/p2p/linkmanager/sessions/SessionManagerTest.kt @@ -104,6 +104,7 @@ import net.corda.p2p.crypto.protocol.api.NoCommonModeError import net.corda.p2p.linkmanager.grouppolicy.protocolModes import java.util.UUID import java.util.concurrent.TimeUnit +import net.corda.membership.lib.exceptions.BadGroupPolicyException class SessionManagerTest { @@ -461,7 +462,7 @@ class SessionManagerTest { } @Test - fun `when no session exists, if network type is missing from network map no message is sent`() { + fun `when no session exists, if network type is missing from group policy provider no message is sent`() { whenever(outboundSessionPool.constructed().first().getNextSession(counterparties)) .thenReturn(OutboundSessionPool.SessionPoolStatus.NewSessionsNeeded) val initiatorHello = mock() @@ -476,21 +477,90 @@ class SessionManagerTest { loggingInterceptor.assertSingleWarningContains("Could not find the p2p parameters in the GroupPolicyProvider") loggingInterceptor.assertSingleWarningContains("The sessionInit message was not sent.") } + @Test + fun `when no session exists, if group policy is missing from group policy provider, on second lookup no message is sent`() { + whenever(outboundSessionPool.constructed().first().getNextSession(counterparties)) + .thenReturn(OutboundSessionPool.SessionPoolStatus.NewSessionsNeeded) + whenever(groupPolicyProvider.getP2PParameters(OUR_PARTY)) + .thenReturn(parameters) + .thenReturn(null) + val initiatorHello = mock() + whenever(protocolInitiator.generateInitiatorHello()).thenReturn(initiatorHello) + val anotherInitiatorHello = mock() + whenever(secondProtocolInitiator.generateInitiatorHello()).thenReturn(anotherInitiatorHello) + + val sessionState = sessionManager.processOutboundMessage(message) + assertThat(sessionState).isInstanceOf(NewSessionsNeeded::class.java) + + argumentCaptor { + verify(sessionReplayer, times(2)).addMessageForReplay( + any(), + this.capture(), + eq(SessionManager.SessionCounterparties(OUR_PARTY, PEER_PARTY, MembershipStatusFilter.ACTIVE, 1L)) + ) + assertThat(this.allValues.size).isEqualTo(2) + assertThat(this.allValues).extracting { + it.sessionCounterparties.ourId + }.containsOnly(OUR_PARTY) + assertThat(this.allValues).extracting { + it.sessionCounterparties.counterpartyId + }.containsOnly(PEER_PARTY) + assertThat(this.allValues).extracting { it.message as InitiatorHelloMessage } + .containsExactlyInAnyOrder(initiatorHello, anotherInitiatorHello) + } + loggingInterceptor.assertSingleWarningContains("The sessionInit message was not sent.") + } @Test - fun `when no session exists, if protocol mode is missing from network map no message is sent`() { + fun `when no session exists, if BadGroupPolicyException is thrown on group policy lookup, no message is sent`() { whenever(outboundSessionPool.constructed().first().getNextSession(counterparties)) .thenReturn(OutboundSessionPool.SessionPoolStatus.NewSessionsNeeded) val initiatorHello = mock() whenever(protocolInitiator.generateInitiatorHello()).thenReturn(initiatorHello) val anotherInitiatorHello = mock() whenever(secondProtocolInitiator.generateInitiatorHello()).thenReturn(anotherInitiatorHello) - whenever(groupPolicyProvider.getP2PParameters(OUR_PARTY)).thenReturn(null) + whenever(groupPolicyProvider.getP2PParameters(OUR_PARTY)).thenThrow(BadGroupPolicyException("Bad group policy")) val sessionState = sessionManager.processOutboundMessage(message) assertThat(sessionState).isInstanceOf(SessionManager.SessionState.CannotEstablishSession::class.java) - verify(sessionReplayer, never()).addMessageForReplay(any(), any(), any()) - loggingInterceptor.assertSingleWarningContains("Could not find the p2p parameters in the GroupPolicyProvider") + + loggingInterceptor.assertSingleWarningContains("Bad group policy") + loggingInterceptor.assertSingleWarningContains("The sessionInit message was not sent.") + } + + @Test + fun `when no session exists, if BadGroupPolicyException is thrown on group policy lookup again, no message is sent`() { + whenever(outboundSessionPool.constructed().first().getNextSession(counterparties)) + .thenReturn(OutboundSessionPool.SessionPoolStatus.NewSessionsNeeded) + whenever(groupPolicyProvider.getP2PParameters(OUR_PARTY)) + .thenReturn(parameters) + .thenThrow(BadGroupPolicyException("Bad group policy")) + val initiatorHello = mock() + whenever(protocolInitiator.generateInitiatorHello()).thenReturn(initiatorHello) + val anotherInitiatorHello = mock() + whenever(secondProtocolInitiator.generateInitiatorHello()).thenReturn(anotherInitiatorHello) + + val sessionState = sessionManager.processOutboundMessage(message) + assertThat(sessionState).isInstanceOf(SessionManager.SessionState.NewSessionsNeeded::class.java) + + argumentCaptor { + verify(sessionReplayer, times(2)).addMessageForReplay( + any(), + this.capture(), + eq(SessionManager.SessionCounterparties(OUR_PARTY, PEER_PARTY, MembershipStatusFilter.ACTIVE, 1L)) + ) + assertThat(this.allValues.size).isEqualTo(2) + assertThat(this.allValues).extracting { + it.sessionCounterparties.ourId + }.containsOnly(OUR_PARTY) + assertThat(this.allValues).extracting { + it.sessionCounterparties.counterpartyId + }.containsOnly(PEER_PARTY) + assertThat(this.allValues).extracting { it.message as InitiatorHelloMessage } + .containsExactlyInAnyOrder(initiatorHello, anotherInitiatorHello) + } + + loggingInterceptor.assertSingleWarningContains("Bad group policy") loggingInterceptor.assertSingleWarningContains("The sessionInit message was not sent.") } @@ -654,7 +724,26 @@ class SessionManagerTest { } @Test - fun `when an initiator hello is received, but network type is missing from network map, then message is dropped`() { + fun `when an initiator hello is received, if BadGroupPolicyException is thrown on group policy lookup, then the message is dropped`() { + val initiatorKeyHash = messageDigest.hash(PEER_KEY.public.encoded) + val sessionId = "some-session-id" + val responderHello = mock() + whenever(protocolResponder.generateResponderHello()).thenReturn(responderHello) + whenever(groupPolicyProvider.getP2PParameters(OUR_PARTY)).thenThrow(BadGroupPolicyException("Bad group policy")) + + val header = CommonHeader(MessageType.INITIATOR_HELLO, 1, sessionId, 1, Instant.now().toEpochMilli()) + val initiatorHelloMsg = InitiatorHelloMessage(header, ByteBuffer.wrap(PEER_KEY.public.encoded), + InitiatorHandshakeIdentity(ByteBuffer.wrap(initiatorKeyHash), GROUP_ID)) + val responseMessage = sessionManager.processSessionMessage(LinkInMessage(initiatorHelloMsg)) + + assertThat(responseMessage).isNull() + loggingInterceptor.assertSingleWarningContains("Bad group policy") + loggingInterceptor + .assertSingleWarningContains("The ${InitiatorHelloMessage::class.java.simpleName} for sessionId $sessionId was discarded.") + } + + @Test + fun `when an initiator hello is received, but network type is missing from group policy provider, then message is dropped`() { val initiatorKeyHash = messageDigest.hash(PEER_KEY.public.encoded) val sessionId = "some-session-id" val responderHello = mock() @@ -809,7 +898,28 @@ class SessionManagerTest { } @Test - fun `when responder hello is received, but network type is missing from network map, message is dropped`() { + fun `when responder hello is received, if BadGroupPolicyException is thrown on group policy lookup, message is dropped`() { + val sessionId = "some-session" + whenever(outboundSessionPool.constructed().first().getSession(sessionId)).thenReturn( + OutboundSessionPool.SessionType.PendingSession(counterparties, protocolInitiator) + ) + + val initiatorHandshakeMsg = mock() + whenever(protocolInitiator.generateOurHandshakeMessage(eq(PEER_KEY.public), eq(null), any())).thenReturn(initiatorHandshakeMsg) + whenever(groupPolicyProvider.getP2PParameters(OUR_PARTY)).thenThrow(BadGroupPolicyException("Bad group policy")) + val header = CommonHeader(MessageType.RESPONDER_HANDSHAKE, 1, sessionId, 4, Instant.now().toEpochMilli()) + val responderHello = ResponderHelloMessage(header, ByteBuffer.wrap(PEER_KEY.public.encoded)) + val responseMessage = sessionManager.processSessionMessage(LinkInMessage(responderHello)) + + assertThat(responseMessage).isNull() + loggingInterceptor + .assertSingleWarningContains("Bad group policy") + loggingInterceptor + .assertSingleWarningContains("The ${ResponderHelloMessage::class.java.simpleName} for sessionId ${sessionId} was discarded.") + } + + @Test + fun `when responder hello is received, but p2p params are missing from group policy provider, message is dropped`() { val sessionId = "some-session" whenever(outboundSessionPool.constructed().first().getSession(sessionId)).thenReturn( OutboundSessionPool.SessionType.PendingSession(counterparties, protocolInitiator) @@ -1169,7 +1279,36 @@ class SessionManagerTest { } @Test - fun `when initiator handshake is received, but network type is missing from the network map, the message is dropped`() { + fun `when initiator handshake is received, if BadGroupPolicyException is thrown on group policy lookup, the message is dropped`() { + val sessionId = "some-session-id" + val initiatorPublicKeyHash = messageDigest.hash(PEER_KEY.public.encoded) + val responderPublicKeyHash = messageDigest.hash(OUR_KEY.public.encoded) + whenever(protocolResponder.generateResponderHello()).thenReturn(mock()) + + val initiatorHelloHeader = CommonHeader(MessageType.INITIATOR_HELLO, 1, sessionId, 1, Instant.now().toEpochMilli()) + val initiatorHelloMessage = InitiatorHelloMessage(initiatorHelloHeader, ByteBuffer.wrap(PEER_KEY.public.encoded), + InitiatorHandshakeIdentity(ByteBuffer.wrap(messageDigest.hash(PEER_KEY.public.encoded)), GROUP_ID)) + sessionManager.processSessionMessage(LinkInMessage(initiatorHelloMessage)) + + val initiatorHandshakeHeader = CommonHeader(MessageType.INITIATOR_HANDSHAKE, 1, sessionId, 3, Instant.now().toEpochMilli()) + val initiatorHandshake = InitiatorHandshakeMessage(initiatorHandshakeHeader, RANDOM_BYTES, RANDOM_BYTES) + whenever(protocolResponder.getInitiatorIdentity()) + .thenReturn(InitiatorHandshakeIdentity(ByteBuffer.wrap(initiatorPublicKeyHash), GROUP_ID)) + whenever(protocolResponder.validatePeerHandshakeMessage( + initiatorHandshake, + listOf(PEER_KEY.public to SignatureSpecs.ECDSA_SHA256), + )).thenReturn(HandshakeIdentityData(initiatorPublicKeyHash, responderPublicKeyHash, GROUP_ID)) + whenever(groupPolicyProvider.getP2PParameters(OUR_PARTY)).thenThrow(BadGroupPolicyException("Bad Group Policy")) + val responseMessage = sessionManager.processSessionMessage(LinkInMessage(initiatorHandshake)) + + assertThat(responseMessage).isNull() + loggingInterceptor.assertSingleWarningContains("Bad Group Policy.") + loggingInterceptor + .assertSingleWarningContains("The ${InitiatorHandshakeMessage::class.java.simpleName} for sessionId $sessionId was discarded.") + } + + @Test + fun `when initiator handshake is received, but network type is missing from the group policy provider, the message is dropped`() { val sessionId = "some-session-id" val initiatorPublicKeyHash = messageDigest.hash(PEER_KEY.public.encoded) val responderPublicKeyHash = messageDigest.hash(OUR_KEY.public.encoded) @@ -2029,6 +2168,50 @@ class SessionManagerTest { } + @Test + fun `sessions are removed even if BadGroupPolicyException is thrown on group policy lookup`() { + whenever(outboundSessionPool.constructed().first().getSession(protocolInitiator.sessionId)).thenReturn( + OutboundSessionPool.SessionType.PendingSession(counterparties, protocolInitiator) + ) + + whenever(protocolInitiator.generateOurHandshakeMessage(eq(PEER_KEY.public), eq(null), any())).thenReturn(mock()) + + val header = CommonHeader(MessageType.RESPONDER_HANDSHAKE, 1, protocolInitiator.sessionId, 4, Instant.now().toEpochMilli()) + val responderHello = ResponderHelloMessage(header, ByteBuffer.wrap(PEER_KEY.public.encoded)) + + sessionManager.processSessionMessage(LinkInMessage(responderHello)) + val responderHandshakeMessage = ResponderHandshakeMessage(header, RANDOM_BYTES, RANDOM_BYTES) + val session = mock() + + whenever(session.sessionId).doAnswer{protocolInitiator.sessionId} + whenever(protocolInitiator.getSession()).thenReturn(session) + whenever(outboundSessionPool.constructed().last().replaceSession(eq(counterparties), eq(sessionId), any())).thenReturn(true) + whenever(protocolInitiator.generateInitiatorHello()).thenReturn(mock()) + whenever(groupPolicyProvider.getP2PParameters(OUR_PARTY)).thenThrow(BadGroupPolicyException("Bad group policy")) + + assertThat(sessionManager.processSessionMessage(LinkInMessage(responderHandshakeMessage))).isNull() + mockTimeFacilitiesProvider.advanceTime(5.days + 1.minutes) + + loggingInterceptor.assertInfoContains("Outbound session sessionId" + + " (local=HoldingIdentity(x500Name=CN=Alice, O=Alice Corp, L=LDN, C=GB, groupId=myGroup)," + + " remote=HoldingIdentity(x500Name=CN=Bob, O=Bob Corp, L=LDN, C=GB, groupId=myGroup))" + + " timed out to refresh ephemeral keys and it will be cleaned up." + ) + + verify(sessionReplayer, times(2)).removeMessageFromReplay( + "${protocolInitiator.sessionId}_${InitiatorHandshakeMessage::class.java.simpleName}", + counterparties + ) + + verify(sessionReplayer, times(2)).removeMessageFromReplay( + "${protocolInitiator.sessionId}_${InitiatorHelloMessage::class.java.simpleName}", + counterparties + ) + + verify(outboundSessionPool.constructed().last()).removeSessions(counterparties) + + } + @Test fun `recordsForSessionEstablished returns empty list if the message convertor can not create the link out message`() { val session = mock { diff --git a/components/membership/group-policy-impl/src/main/kotlin/net/corda/membership/impl/grouppolicy/GroupPolicyProviderImpl.kt b/components/membership/group-policy-impl/src/main/kotlin/net/corda/membership/impl/grouppolicy/GroupPolicyProviderImpl.kt index 7eaa63b77b8..cbbf3d782e1 100644 --- a/components/membership/group-policy-impl/src/main/kotlin/net/corda/membership/impl/grouppolicy/GroupPolicyProviderImpl.kt +++ b/components/membership/group-policy-impl/src/main/kotlin/net/corda/membership/impl/grouppolicy/GroupPolicyProviderImpl.kt @@ -332,7 +332,7 @@ class GroupPolicyProviderImpl @Activate constructor( ) { val holdingIdentity = member.viewOwningMember.toCorda() val gp = parseGroupPolicy(holdingIdentity) - if (gp is MGMGroupPolicy) { + if (gp is MGMGroupPolicy && gp.validate(holdingIdentity)) { groupPolicies[holdingIdentity] = gp callBack(holdingIdentity, gp) } else { @@ -344,6 +344,18 @@ class GroupPolicyProviderImpl @Activate constructor( } } + private fun MGMGroupPolicy.validate(holdingIdentity: HoldingIdentity): Boolean { + return try { + this.p2pParameters + true + } catch (e: BadGroupPolicyException) { + logger.warn("Something went wrong while parsing the group policy for virtual node with holding identity" + + " [$holdingIdentity]. Error: ${e.message} The group policy will be removed from the cache, to be parsed and cached on" + + " the next read.") + false + } + } + override val keyClass = String::class.java override val valueClass = PersistentMemberInfo::class.java } diff --git a/components/membership/group-policy-impl/src/test/kotlin/net/corda/membership/impl/grouppolicy/GroupPolicyProviderImplTest.kt b/components/membership/group-policy-impl/src/test/kotlin/net/corda/membership/impl/grouppolicy/GroupPolicyProviderImplTest.kt index 3b0c41f8f3a..df4b76a86eb 100644 --- a/components/membership/group-policy-impl/src/test/kotlin/net/corda/membership/impl/grouppolicy/GroupPolicyProviderImplTest.kt +++ b/components/membership/group-policy-impl/src/test/kotlin/net/corda/membership/impl/grouppolicy/GroupPolicyProviderImplTest.kt @@ -653,7 +653,42 @@ class GroupPolicyProviderImplTest { ) verify(groupPolicyParser, times(2)).parse(eq(holdingIdentity5), any(), any()) - // previous value was rmeoved from cache, hence re-calculating + // previous value was removed from cache, hence re-calculating + groupPolicyProvider.getGroupPolicy(holdingIdentity5) + verify(groupPolicyParser, times(3)).parse(eq(holdingIdentity5), any(), any()) + } + + @Test + fun `MGM group policy is removed from cache if exception occurs when validating`() { + postConfigChangedEvent() + startComponentAndDependencies() + + groupPolicyProvider.FinishedRegistrationsProcessor(memberInfoFactory) {_, _ -> } + .onNext( + Record("", "", mgmPersistentMemberInfo), + null, + emptyMap(), + ) + verify(groupPolicyParser, times(1)).parse(eq(holdingIdentity5), any(), any()) + + // returns from cache + groupPolicyProvider.getGroupPolicy(holdingIdentity5) + verify(groupPolicyParser, times(1)).parse(eq(holdingIdentity5), any(), any()) + + // on new event we will fail parsing + val mgmGroupPolicy = mock { + on { p2pParameters } doThrow BadGroupPolicyException("Bad group policy.") + } + whenever(groupPolicyParser.parse(eq(holdingIdentity5), any(), any())).thenReturn(mgmGroupPolicy) + groupPolicyProvider.FinishedRegistrationsProcessor(memberInfoFactory) {_, _ -> } + .onNext( + Record("", "", mgmPersistentMemberInfo), + null, + emptyMap(), + ) + verify(groupPolicyParser, times(2)).parse(eq(holdingIdentity5), any(), any()) + + // previous value was removed from cache, hence re-calculating groupPolicyProvider.getGroupPolicy(holdingIdentity5) verify(groupPolicyParser, times(3)).parse(eq(holdingIdentity5), any(), any()) } diff --git a/libs/membership/membership-common/src/main/kotlin/net/corda/membership/lib/grouppolicy/GroupPolicy.kt b/libs/membership/membership-common/src/main/kotlin/net/corda/membership/lib/grouppolicy/GroupPolicy.kt index 11781aa1fcb..ff55c136433 100644 --- a/libs/membership/membership-common/src/main/kotlin/net/corda/membership/lib/grouppolicy/GroupPolicy.kt +++ b/libs/membership/membership-common/src/main/kotlin/net/corda/membership/lib/grouppolicy/GroupPolicy.kt @@ -88,8 +88,6 @@ interface GroupPolicy { * The policy for session key handling. * [SessionKeyPolicy.COMBINED] means the same key is used for session initiation and ledger signing. * [SessionKeyPolicy.DISTINCT] means separate keys are used for sessions and ledger signing. - * - * @throws [BadGroupPolicyException] if the data is unavailable or cannot be parsed. */ @get:Throws(BadGroupPolicyException::class) val sessionKeyPolicy: SessionKeyPolicy @@ -97,8 +95,6 @@ interface GroupPolicy { /** * Static network member configurations. Only present for static networks. * Extensible map of properties which represent a template to build a static network member. - * - * @throws [BadGroupPolicyException] if the data is unavailable or cannot be parsed. */ @get:Throws(BadGroupPolicyException::class) val staticNetworkMembers: List>? @@ -106,8 +102,6 @@ interface GroupPolicy { /** * Static network group parameters. Only present for static networks. * May include minimum platform version and custom parameters (with "ext." prefix). - * - * @throws [BadGroupPolicyException] if the data is unavailable or cannot be parsed. */ @get:Throws(BadGroupPolicyException::class) val staticNetworkGroupParameters: Map? @@ -117,58 +111,38 @@ interface GroupPolicy { /** * A collection of trust root certificates for session initiation as PEM strigns. * This is optional. If `sessionPki` mode is [SessionPkiMode.NO_PKI] then this will return null. - * - * @throws [BadGroupPolicyException] if the data is unavailable or cannot be parsed. */ - @get:Throws(BadGroupPolicyException::class) val sessionTrustRoots: Collection? /** * A collection of TLS trust root certificates as PEM strings. * Parsing validates for at least one certificate. - * - * @throws [BadGroupPolicyException] if the data is unavailable or cannot be parsed. */ - @get:Throws(BadGroupPolicyException::class) val tlsTrustRoots: Collection /** * The session PKI mode. - * - * @throws [BadGroupPolicyException] if the data is unavailable or cannot be parsed. */ - @get:Throws(BadGroupPolicyException::class) val sessionPki: SessionPkiMode /** * The TLS PKI mode. - * - * @throws [BadGroupPolicyException] if the data is unavailable or cannot be parsed. */ - @get:Throws(BadGroupPolicyException::class) val tlsPki: TlsPkiMode /** * The TLS version to be used. - * - * @throws [BadGroupPolicyException] if the data is unavailable or cannot be parsed. */ val tlsVersion: TlsVersion /** * The P2P protocol mode. - * - * @throws [BadGroupPolicyException] if the data is unavailable or cannot be parsed. */ - @get:Throws(BadGroupPolicyException::class) val protocolMode: ProtocolMode /** * The P2P TLS type. - * - * @throws [BadGroupPolicyException] if the data is unavailable or cannot be parsed. */ - @get:Throws(BadGroupPolicyException::class) val tlsType: TlsType /** diff --git a/libs/membership/membership-impl/src/main/kotlin/net/corda/membership/lib/impl/grouppolicy/v1/MGMGroupPolicyImpl.kt b/libs/membership/membership-impl/src/main/kotlin/net/corda/membership/lib/impl/grouppolicy/v1/MGMGroupPolicyImpl.kt index 6efc0035e45..73f257b3492 100644 --- a/libs/membership/membership-impl/src/main/kotlin/net/corda/membership/lib/impl/grouppolicy/v1/MGMGroupPolicyImpl.kt +++ b/libs/membership/membership-impl/src/main/kotlin/net/corda/membership/lib/impl/grouppolicy/v1/MGMGroupPolicyImpl.kt @@ -64,22 +64,20 @@ class MGMGroupPolicyImpl( override val synchronisationProtocol = rootNode.getMandatoryString(SYNC_PROTOCOL) - override val protocolParameters: GroupPolicy.ProtocolParameters = ProtocolParametersImpl() + override val protocolParameters: GroupPolicy.ProtocolParameters by lazy { + ProtocolParametersImpl() + } - override val p2pParameters: GroupPolicy.P2PParameters = P2PParametersImpl() + override val p2pParameters: GroupPolicy.P2PParameters by lazy { + P2PParametersImpl() + } override val mgmInfo: GroupPolicy.MGMInfo? = null override val cipherSuite: GroupPolicy.CipherSuite = CipherSuiteImpl() internal inner class ProtocolParametersImpl : GroupPolicy.ProtocolParameters { - override val sessionKeyPolicy by lazy { - SessionKeyPolicy.fromString( - getPersistedString( - PropertyKeys.SESSION_KEY_POLICY - ) - ) ?: COMBINED - } + override val sessionKeyPolicy = SessionKeyPolicy.fromString(getPersistedString(PropertyKeys.SESSION_KEY_POLICY)) ?: COMBINED override val staticNetworkMembers: List>? = null @@ -88,55 +86,40 @@ class MGMGroupPolicyImpl( internal inner class P2PParametersImpl : GroupPolicy.P2PParameters { - override val sessionPki by lazy { - SessionPkiMode.fromString( - getPersistedString( - PropertyKeys.SESSION_PKI_MODE - ) - ) ?: NO_PKI - } + override val sessionPki = SessionPkiMode.fromString(getPersistedString(PropertyKeys.SESSION_PKI_MODE)) ?: NO_PKI - override val sessionTrustRoots by lazy { - if (!persistedProperties.entries.any { it.key.startsWith(PropertyKeys.SESSION_TRUST_ROOTS) }) { - null - } else { - persistedProperties.parseList(PropertyKeys.SESSION_TRUST_ROOTS, String::class.java) - } + override val sessionTrustRoots = if (!persistedProperties.entries.any { it.key.startsWith(PropertyKeys.SESSION_TRUST_ROOTS) }) { + null + } else { + persistedProperties.parseList(PropertyKeys.SESSION_TRUST_ROOTS, String::class.java) } - override val tlsTrustRoots by lazy { - if (!persistedProperties.entries.any { it.key.startsWith(PropertyKeys.TLS_TRUST_ROOTS) }) { - emptyList() - } else { - persistedProperties.parseList(PropertyKeys.TLS_TRUST_ROOTS, String::class.java) - } + override val tlsTrustRoots = if (!persistedProperties.entries.any { it.key.startsWith(PropertyKeys.TLS_TRUST_ROOTS) }) { + emptyList() + } else { + persistedProperties.parseList(PropertyKeys.TLS_TRUST_ROOTS, String::class.java) } - override val tlsPki by lazy { - TlsPkiMode.fromString( - getPersistedString(PropertyKeys.TLS_PKI_MODE) - ) ?: STANDARD - } - override val tlsVersion by lazy { - TlsVersion.fromString( - getPersistedString(PropertyKeys.TLS_VERSION) - ) ?: VERSION_1_3 - } - override val protocolMode by lazy { - ProtocolMode.fromString( - getPersistedString(PropertyKeys.P2P_PROTOCOL_MODE) - ) ?: AUTH_ENCRYPT - } - override val tlsType by lazy { - TlsType.fromString( - getPersistedString(PropertyKeys.TLS_TYPE) - ) ?: TlsType.ONE_WAY - } - override val mgmClientCertificateSubject by lazy { - getPersistedString(PropertyKeys.MGM_CLIENT_CERTIFICATE_SUBJECT)?.let { - MemberX500Name.parse(it) - } + override val tlsPki = TlsPkiMode.fromString( + getPersistedString(PropertyKeys.TLS_PKI_MODE) + ) ?: STANDARD + + override val tlsVersion = TlsVersion.fromString( + getPersistedString(PropertyKeys.TLS_VERSION) + ) ?: VERSION_1_3 + + override val protocolMode = ProtocolMode.fromString( + getPersistedString(PropertyKeys.P2P_PROTOCOL_MODE) + ) ?: AUTH_ENCRYPT + + override val tlsType = TlsType.fromString( + getPersistedString(PropertyKeys.TLS_TYPE) + ) ?: TlsType.ONE_WAY + + + override val mgmClientCertificateSubject = getPersistedString(PropertyKeys.MGM_CLIENT_CERTIFICATE_SUBJECT)?.let { + MemberX500Name.parse(it) } } diff --git a/libs/membership/membership-impl/src/test/kotlin/net/corda/membership/lib/impl/grouppolicy/v1/MGMGroupPolicyImplTest.kt b/libs/membership/membership-impl/src/test/kotlin/net/corda/membership/lib/impl/grouppolicy/v1/MGMGroupPolicyImplTest.kt index 1aa0c1d21aa..1762fca634e 100644 --- a/libs/membership/membership-impl/src/test/kotlin/net/corda/membership/lib/impl/grouppolicy/v1/MGMGroupPolicyImplTest.kt +++ b/libs/membership/membership-impl/src/test/kotlin/net/corda/membership/lib/impl/grouppolicy/v1/MGMGroupPolicyImplTest.kt @@ -127,7 +127,7 @@ class MGMGroupPolicyImplTest { } @Test - fun `persisted properties are not queried until they are needed`() { + fun `persisted properties are queried once when the p2pParameters are accessed`() { val propertyQuery: () -> LayeredPropertyMap = mock { on { invoke() } doReturn persistedProperties } @@ -152,26 +152,83 @@ class MGMGroupPolicyImplTest { it.assertThat(groupPolicy.synchronisationProtocol).isEqualTo(TEST_SYNC_PROTOCOL) } verify(propertyQuery, never()).invoke() + + val p2pParameters = groupPolicy.p2pParameters + + verify(propertyQuery).invoke() + val protocolParameters = groupPolicy.protocolParameters + assertSoftly { - it.assertThat(groupPolicy.protocolParameters.sessionKeyPolicy).isEqualTo(DISTINCT) - it.assertThat(groupPolicy.p2pParameters.sessionPki).isEqualTo(SessionPkiMode.STANDARD_EV3) - it.assertThat(groupPolicy.p2pParameters.sessionTrustRoots).isNotNull - it.assertThat(groupPolicy.p2pParameters.sessionTrustRoots?.size).isEqualTo(1) - it.assertThat(groupPolicy.p2pParameters.sessionTrustRoots?.first()).isEqualTo(TEST_CERT) - it.assertThat(groupPolicy.p2pParameters.tlsTrustRoots.size).isEqualTo(1) - it.assertThat(groupPolicy.p2pParameters.tlsTrustRoots.first()).isEqualTo(TEST_CERT) - it.assertThat(groupPolicy.p2pParameters.tlsPki).isEqualTo(TlsPkiMode.STANDARD_EV3) - it.assertThat(groupPolicy.p2pParameters.tlsVersion).isEqualTo(VERSION_1_2) - it.assertThat(groupPolicy.p2pParameters.protocolMode).isEqualTo(AUTH) - it.assertThat(groupPolicy.p2pParameters.mgmClientCertificateSubject).isEqualTo(null) + it.assertThat(protocolParameters.sessionKeyPolicy).isEqualTo(DISTINCT) + it.assertThat(p2pParameters.sessionPki).isEqualTo(SessionPkiMode.STANDARD_EV3) + it.assertThat(p2pParameters.sessionTrustRoots).isNotNull + it.assertThat(p2pParameters.sessionTrustRoots?.size).isEqualTo(1) + it.assertThat(p2pParameters.sessionTrustRoots?.first()).isEqualTo(TEST_CERT) + it.assertThat(p2pParameters.tlsTrustRoots.size).isEqualTo(1) + it.assertThat(p2pParameters.tlsTrustRoots.first()).isEqualTo(TEST_CERT) + it.assertThat(p2pParameters.tlsPki).isEqualTo(TlsPkiMode.STANDARD_EV3) + it.assertThat(p2pParameters.tlsVersion).isEqualTo(VERSION_1_2) + it.assertThat(p2pParameters.protocolMode).isEqualTo(AUTH) + it.assertThat(p2pParameters.mgmClientCertificateSubject).isEqualTo(null) it.assertThat(groupPolicy.mgmInfo).isNull() it.assertThat(groupPolicy.cipherSuite.entries).isEmpty() } verify(propertyQuery).invoke() } - } + @Test + fun `persisted properties are queried once when the protocolParameters are accessed`() { + val propertyQuery: () -> LayeredPropertyMap = mock { + on { invoke() } doReturn persistedProperties + } + val groupPolicy: GroupPolicy = assertDoesNotThrow { + MGMGroupPolicyImpl( + holdingIdentity, + buildGroupPolicyNode( + groupIdOverride = MGM_DEFAULT_GROUP_ID, + protocolParametersOverride = null, + p2pParametersOverride = null, + mgmInfoOverride = null, + cipherSuiteOverride = null + ), + propertyQuery + ) + } + verify(propertyQuery, never()).invoke() + assertSoftly { + it.assertThat(groupPolicy.fileFormatVersion).isEqualTo(TEST_FILE_FORMAT_VERSION) + it.assertThat(groupPolicy.groupId).isEqualTo(TEST_GROUP_ID) + it.assertThat(groupPolicy.registrationProtocol).isEqualTo(TEST_REG_PROTOCOL) + it.assertThat(groupPolicy.synchronisationProtocol).isEqualTo(TEST_SYNC_PROTOCOL) + } + verify(propertyQuery, never()).invoke() + + val protocolParameters = groupPolicy.protocolParameters + + verify(propertyQuery).invoke() + + val p2pParameters = groupPolicy.p2pParameters + + assertSoftly { + it.assertThat(protocolParameters.sessionKeyPolicy).isEqualTo(DISTINCT) + it.assertThat(p2pParameters.sessionPki).isEqualTo(SessionPkiMode.STANDARD_EV3) + it.assertThat(p2pParameters.sessionTrustRoots).isNotNull + it.assertThat(p2pParameters.sessionTrustRoots?.size).isEqualTo(1) + it.assertThat(p2pParameters.sessionTrustRoots?.first()).isEqualTo(TEST_CERT) + it.assertThat(p2pParameters.tlsTrustRoots.size).isEqualTo(1) + it.assertThat(p2pParameters.tlsTrustRoots.first()).isEqualTo(TEST_CERT) + it.assertThat(p2pParameters.tlsPki).isEqualTo(TlsPkiMode.STANDARD_EV3) + it.assertThat(p2pParameters.tlsVersion).isEqualTo(VERSION_1_2) + it.assertThat(p2pParameters.protocolMode).isEqualTo(AUTH) + it.assertThat(p2pParameters.mgmClientCertificateSubject).isEqualTo(null) + + it.assertThat(groupPolicy.mgmInfo).isNull() + it.assertThat(groupPolicy.cipherSuite.entries).isEmpty() + } + verify(propertyQuery).invoke() + } + } @Test fun `mgmClientCertificateSubject is extracted correctly`() { From 704b3c6ef40f3c8215a5fd07ca2ccd3e5c3ca1cb Mon Sep 17 00:00:00 2001 From: Thiago Viana <3837906+thiagoviana@users.noreply.github.com> Date: Mon, 16 Oct 2023 17:16:26 +0100 Subject: [PATCH 40/81] CORE-16242 - Synchronous RPC Pattern API Implementation for Crypto Worker (#4820) HTTP RPC server integration into the Crypto processor. Allows Crypto processor to receive HTTP external event requests sent from the flow engine. Client integration will come later. --- .../smoketest/services/CryptoRPCSmokeTests.kt | 256 +++++++++ .../crypto/service/impl/rpc/package-info.java | 6 + .../impl/rpc/CryptoFlowOpsRpcProcessor.kt | 138 +++++ .../rpc/CryptoFlowOpsRpcProcessorTests.kt | 504 ++++++++++++++++++ .../crypto/internal/CryptoProcessorImpl.kt | 73 ++- 5 files changed, 958 insertions(+), 19 deletions(-) create mode 100644 applications/workers/workers-smoketest/src/smokeTest/kotlin/net/corda/applications/workers/smoketest/services/CryptoRPCSmokeTests.kt create mode 100644 components/crypto/crypto-service-impl/src/main/java/net/corda/crypto/service/impl/rpc/package-info.java create mode 100644 components/crypto/crypto-service-impl/src/main/kotlin/net/corda/crypto/service/impl/rpc/CryptoFlowOpsRpcProcessor.kt create mode 100644 components/crypto/crypto-service-impl/src/test/kotlin/net/corda/crypto/service/impl/rpc/CryptoFlowOpsRpcProcessorTests.kt diff --git a/applications/workers/workers-smoketest/src/smokeTest/kotlin/net/corda/applications/workers/smoketest/services/CryptoRPCSmokeTests.kt b/applications/workers/workers-smoketest/src/smokeTest/kotlin/net/corda/applications/workers/smoketest/services/CryptoRPCSmokeTests.kt new file mode 100644 index 00000000000..1dde1fbfe77 --- /dev/null +++ b/applications/workers/workers-smoketest/src/smokeTest/kotlin/net/corda/applications/workers/smoketest/services/CryptoRPCSmokeTests.kt @@ -0,0 +1,256 @@ +package net.corda.applications.workers.smoketest.services + +import net.corda.applications.workers.smoketest.utils.PLATFORM_VERSION +import net.corda.crypto.core.SecureHashImpl +import net.corda.crypto.core.toAvro +import net.corda.data.KeyValuePair +import net.corda.data.KeyValuePairList +import net.corda.data.crypto.SecureHashes +import net.corda.data.crypto.wire.CryptoRequestContext +import net.corda.data.crypto.wire.CryptoResponseContext +import net.corda.data.crypto.wire.ops.flow.FlowOpsRequest +import net.corda.data.crypto.wire.ops.flow.FlowOpsResponse +import net.corda.data.crypto.wire.ops.flow.queries.ByIdsFlowQuery +import net.corda.data.flow.event.FlowEvent +import net.corda.data.flow.event.external.ExternalEventContext +import net.corda.data.flow.event.external.ExternalEventResponse +import net.corda.e2etest.utilities.DEFAULT_CLUSTER +import net.corda.e2etest.utilities.conditionallyUploadCordaPackage +import net.corda.e2etest.utilities.conditionallyUploadCpiSigningCertificate +import net.corda.e2etest.utilities.getHoldingIdShortHash +import net.corda.e2etest.utilities.getOrCreateVirtualNodeFor +import net.corda.e2etest.utilities.registerStaticMember +import net.corda.messagebus.kafka.serialization.CordaAvroSerializationFactoryImpl +import net.corda.schema.registry.impl.AvroSchemaRegistryImpl +import net.corda.test.util.time.AutoTickTestClock +import org.assertj.core.api.Assertions.assertThat +import org.assertj.core.api.SoftAssertions.assertSoftly +import org.junit.jupiter.api.Assertions +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.BeforeAll +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.TestInstance +import org.slf4j.LoggerFactory +import java.net.URI +import java.net.http.HttpClient +import java.net.http.HttpRequest +import java.net.http.HttpResponse +import java.time.Duration +import java.time.Instant +import java.util.UUID + +/** + * Tests for the Crypto RPC service + */ +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +class CryptoRPCSmokeTests { + private val httpClient: HttpClient = HttpClient.newBuilder() + .connectTimeout(Duration.ofSeconds(30)) + .build() + private val serializationFactory = CordaAvroSerializationFactoryImpl( + AvroSchemaRegistryImpl() + ) + + private val avroSerializer = serializationFactory.createAvroSerializer { } + private val avroFlowEventDeserializer = serializationFactory.createAvroDeserializer({}, FlowEvent::class.java) + private val avroCryptoDeserializer = serializationFactory.createAvroDeserializer({}, FlowOpsResponse::class.java) + + companion object { + const val TEST_CPI_NAME = "ledger-utxo-demo-app" + const val TEST_CPB_LOCATION = "/META-INF/ledger-utxo-demo-app.cpb" + + val logger = LoggerFactory.getLogger(this::class.java.enclosingClass) + } + + private val testRunUniqueId = UUID.randomUUID() + private val requestId = UUID.randomUUID() + private val flowId = UUID.randomUUID() + private val groupId = UUID.randomUUID().toString() + private val cpiName = "${TEST_CPI_NAME}_$testRunUniqueId" + + private val aliceX500 = "CN=Alice-${testRunUniqueId}, OU=Application, O=R3, L=London, C=GB" + + private val aliceHoldingId: String = getHoldingIdShortHash(aliceX500, groupId) + + private val externalEventContext: ExternalEventContext = createExternalEventContext() + private lateinit var cryptoRequestContext: CryptoRequestContext + + private fun createExternalEventContext(): ExternalEventContext { + val simpleContext = KeyValuePairList( + listOf( + KeyValuePair("Hello", "World!") + ) + ) + + return ExternalEventContext.newBuilder() + .setContextProperties(simpleContext) + .setRequestId(requestId.toString()) + .setFlowId(flowId.toString()) + .build() + } + + private val staticMemberList = listOf( + aliceX500 + ) + + @BeforeAll + fun beforeAll() { + DEFAULT_CLUSTER.conditionallyUploadCpiSigningCertificate() + + conditionallyUploadCordaPackage( + cpiName, + TEST_CPB_LOCATION, + groupId, + staticMemberList + ) + val aliceActualHoldingId = getOrCreateVirtualNodeFor(aliceX500, cpiName) + assertThat(aliceActualHoldingId).isEqualTo(aliceHoldingId) + registerStaticMember(aliceHoldingId) + } + + @BeforeEach + fun setup() { + cryptoRequestContext = createRequestContext() + } + + @Test + fun `RPC endpoint accepts a request and returns back a response`() { + val url = "${System.getProperty("cryptoWorkerUrl")}api/$PLATFORM_VERSION/crypto" + + logger.info("crypto url: $url") + val serializedPayload = avroSerializer.serialize(generateByIdsFlowOpsRequest()) + + val request = HttpRequest.newBuilder() + .uri(URI.create(url)) + .headers("Content-Type", "application/octet-stream") + .POST(HttpRequest.BodyPublishers.ofByteArray(serializedPayload)) + .build() + val response = httpClient.send(request, HttpResponse.BodyHandlers.ofByteArray()) + + assertThat(response.statusCode()).isEqualTo(200).withFailMessage("status code on response: ${response.statusCode()} url: $url") + + val responseBody: ByteArray = response.body() + val responseEvent = avroFlowEventDeserializer.deserialize(responseBody) + + assertThat(responseEvent).isNotNull + + val deserializedExternalEventResponse = avroCryptoDeserializer.deserialize((responseEvent?.payload as ExternalEventResponse).payload.array()) + + assertThat(deserializedExternalEventResponse).isNotNull + assertStandardSuccessResponse(deserializedExternalEventResponse!!, testClock) + assertResponseContext(cryptoRequestContext, deserializedExternalEventResponse.context) + } + + @Test + fun `RPC endpoint accepts a request and returns back an error response with 200 status`() { + val url = "${System.getProperty("cryptoWorkerUrl")}api/$PLATFORM_VERSION/crypto" + + logger.info("crypto url: $url") + val serializedPayload = avroSerializer.serialize(generateByIdsFlowOpsRequest(returnError = true)) + + val request = HttpRequest.newBuilder() + .uri(URI.create(url)) + .headers("Content-Type", "application/octet-stream") + .POST(HttpRequest.BodyPublishers.ofByteArray(serializedPayload)) + .build() + val response = httpClient.send(request, HttpResponse.BodyHandlers.ofByteArray()) + + assertThat(response.statusCode()).isEqualTo(200).withFailMessage("status code on response: ${response.statusCode()} url: $url") + + val responseBody: ByteArray = response.body() + val responseEvent = avroFlowEventDeserializer.deserialize(responseBody) + + assertThat(responseEvent).isNotNull + + val externalEventResponse = responseEvent?.payload as ExternalEventResponse + assertThat(externalEventResponse.payload).isNull() + assertThat(externalEventResponse.error).isNotNull() + } + + @Test + fun `RPC endpoint does not accept request and returns back a 500 error`() { + val url = "${System.getProperty("cryptoWorkerUrl")}api/$PLATFORM_VERSION/crypto" + + logger.info("crypto url: $url") + val serializedPayload = avroSerializer.serialize(generateByIdsFlowOpsRequest()) + + val request = HttpRequest.newBuilder() + .uri(URI.create(url)) + .headers("Content-Type", "application/octet-stream") + .PUT(HttpRequest.BodyPublishers.ofByteArray(serializedPayload)) + .build() + val response = httpClient.send(request, HttpResponse.BodyHandlers.ofByteArray()) + + assertThat(response.statusCode()).isEqualTo(404).withFailMessage("status code on response: ${response.statusCode()} url: $url") + } + + private val testClock = AutoTickTestClock(Instant.MAX, Duration.ofSeconds(1)) + + /** + * Generate simple request to lookup for keys by their full key ids. + * Lookup will return no items in the response. + */ + private fun generateByIdsFlowOpsRequest(returnError: Boolean = false) : FlowOpsRequest { + val secureHash = SecureHashImpl("algorithm", "12345678".toByteArray()).toAvro() + val generateByIdsRequest = ByIdsFlowQuery(SecureHashes(listOf(secureHash))) + + if (returnError) { + cryptoRequestContext.tenantId = UUID.randomUUID().toString() + } + + return FlowOpsRequest.newBuilder() + .setContext(cryptoRequestContext) + .setRequest(generateByIdsRequest) + .setFlowExternalEventContext(externalEventContext) + .build() + } + + private fun createRequestContext(): CryptoRequestContext = CryptoRequestContext( + "test-component", + Instant.now(), + UUID.randomUUID().toString(), + aliceHoldingId, + KeyValuePairList( + listOf( + KeyValuePair("key1", "value1"), + KeyValuePair("key2", "value2") + ) + ) + ) + + private fun assertResponseContext(expected: CryptoRequestContext, actual: CryptoResponseContext) { + val now = Instant.now() + assertEquals(expected.tenantId, actual.tenantId) + assertEquals(expected.requestId, actual.requestId) + assertEquals(expected.requestingComponent, actual.requestingComponent) + assertEquals(expected.requestTimestamp, actual.requestTimestamp) + assertThat(actual.responseTimestamp.epochSecond) + .isGreaterThanOrEqualTo(expected.requestTimestamp.epochSecond) + .isLessThanOrEqualTo(now.epochSecond) + assertSoftly { softly -> + softly.assertThat(actual.other.items.size == expected.other.items.size) + softly.assertThat(actual.other.items.containsAll(expected.other.items)) + softly.assertThat(expected.other.items.containsAll(actual.other.items)) + } + } + + private fun assertStandardSuccessResponse( + response: FlowOpsResponse, + clock: AutoTickTestClock? = null + ) = getResultOfType(response) + .run { assertValidTimestamp(response.context.requestTimestamp, clock) } + + private inline fun getResultOfType(response: FlowOpsResponse): T { + Assertions.assertInstanceOf(T::class.java, response) + @Suppress("UNCHECKED_CAST") + return response as T + } + + private fun assertValidTimestamp(timestamp: Instant, clock: AutoTickTestClock? = null) { + assertThat(timestamp).isAfter(Instant.MIN) + if (clock != null) { + assertThat(timestamp).isBeforeOrEqualTo(clock.peekTime()) + } + } +} diff --git a/components/crypto/crypto-service-impl/src/main/java/net/corda/crypto/service/impl/rpc/package-info.java b/components/crypto/crypto-service-impl/src/main/java/net/corda/crypto/service/impl/rpc/package-info.java new file mode 100644 index 00000000000..defd45cc4b0 --- /dev/null +++ b/components/crypto/crypto-service-impl/src/main/java/net/corda/crypto/service/impl/rpc/package-info.java @@ -0,0 +1,6 @@ +@Export +@QuasarIgnoreAllPackages +package net.corda.crypto.service.impl.rpc; + +import co.paralleluniverse.quasar.annotations.QuasarIgnoreAllPackages; +import org.osgi.annotation.bundle.Export; diff --git a/components/crypto/crypto-service-impl/src/main/kotlin/net/corda/crypto/service/impl/rpc/CryptoFlowOpsRpcProcessor.kt b/components/crypto/crypto-service-impl/src/main/kotlin/net/corda/crypto/service/impl/rpc/CryptoFlowOpsRpcProcessor.kt new file mode 100644 index 00000000000..60ff339d598 --- /dev/null +++ b/components/crypto/crypto-service-impl/src/main/kotlin/net/corda/crypto/service/impl/rpc/CryptoFlowOpsRpcProcessor.kt @@ -0,0 +1,138 @@ +package net.corda.crypto.service.impl.rpc + +import net.corda.crypto.cipher.suite.KeyEncodingService +import net.corda.crypto.config.impl.RetryingConfig +import net.corda.crypto.core.CryptoService +import net.corda.crypto.core.SecureHashImpl +import net.corda.crypto.core.ShortHash +import net.corda.crypto.core.publicKeyIdFromBytes +import net.corda.crypto.impl.retrying.BackoffStrategy +import net.corda.crypto.impl.retrying.CryptoRetryingExecutor +import net.corda.crypto.impl.toMap +import net.corda.crypto.impl.toSignatureSpec +import net.corda.data.KeyValuePairList +import net.corda.data.crypto.wire.CryptoRequestContext +import net.corda.data.crypto.wire.CryptoResponseContext +import net.corda.data.crypto.wire.CryptoSignatureWithKey +import net.corda.data.crypto.wire.CryptoSigningKeys +import net.corda.data.crypto.wire.ops.flow.FlowOpsRequest +import net.corda.data.crypto.wire.ops.flow.FlowOpsResponse +import net.corda.data.crypto.wire.ops.flow.commands.SignFlowCommand +import net.corda.data.crypto.wire.ops.flow.queries.ByIdsFlowQuery +import net.corda.data.crypto.wire.ops.flow.queries.FilterMyKeysFlowQuery +import net.corda.data.flow.event.FlowEvent +import net.corda.flow.external.events.responses.factory.ExternalEventResponseFactory +import net.corda.messaging.api.processor.SyncRPCProcessor +import net.corda.metrics.CordaMetrics +import net.corda.utilities.MDC_CLIENT_ID +import net.corda.utilities.MDC_EXTERNAL_EVENT_ID +import net.corda.utilities.MDC_FLOW_ID +import net.corda.utilities.trace +import net.corda.utilities.translateFlowContextToMDC +import net.corda.utilities.withMDC +import org.slf4j.LoggerFactory +import java.nio.ByteBuffer +import java.time.Duration +import java.time.Instant + +@Suppress("LongParameterList") +class CryptoFlowOpsRpcProcessor( + private val cryptoService: CryptoService, + private val externalEventResponseFactory: ExternalEventResponseFactory, + config: RetryingConfig, + private val keyEncodingService: KeyEncodingService, + override val requestClass: Class, + override val responseClass: Class, +) : SyncRPCProcessor { + companion object { + private val logger = LoggerFactory.getLogger(this::class.java.enclosingClass) + } + + private val executor = CryptoRetryingExecutor( + logger, + BackoffStrategy.createBackoff(config.maxAttempts, config.waitBetweenMills) + ) + + override fun process(request: FlowOpsRequest): FlowEvent { + logger.trace { "process just started processing ${request::class.java.name}" } + + val clientRequestId = request.flowExternalEventContext.contextProperties.toMap()[MDC_CLIENT_ID] ?: "" + + val mdc = mapOf( + MDC_FLOW_ID to request.flowExternalEventContext.flowId, + MDC_CLIENT_ID to clientRequestId, + MDC_EXTERNAL_EVENT_ID to request.flowExternalEventContext.requestId + ) + translateFlowContextToMDC(request.flowExternalEventContext.contextProperties.toMap()) + + val result = withMDC(mdc) { + val requestPayload = request.request + val startTime = System.nanoTime() + logger.info("Handling ${requestPayload::class.java.name} for tenant ${request.context.tenantId}") + + try { + val response = executor.executeWithRetry { + handleRequest(requestPayload, request.context) + } + + externalEventResponseFactory.success( + request.flowExternalEventContext, + FlowOpsResponse(createResponseContext(request), response, null) + ) + } catch (throwable: Throwable) { + logger.error( + "Failed to handle ${requestPayload::class.java.name} for tenant ${request.context.tenantId}", + throwable + ) + externalEventResponseFactory.platformError(request.flowExternalEventContext, throwable) + }.also { + CordaMetrics.Metric.Crypto.FlowOpsProcessorExecutionTime.builder() + .withTag(CordaMetrics.Tag.OperationName, requestPayload::class.java.simpleName) + .build() + .record(Duration.ofNanos(System.nanoTime() - startTime)) + } + } + + return result.value as FlowEvent + } + + private fun handleRequest(request: Any, context: CryptoRequestContext): Any { + return when (request) { + is FilterMyKeysFlowQuery -> { + val keys = request.keys.map { ShortHash.of(publicKeyIdFromBytes(it.array())) } + cryptoService.lookupSigningKeysByPublicKeyShortHash(context.tenantId, keys) + } + + is SignFlowCommand -> { + val publicKey = cryptoService.schemeMetadata.decodePublicKey(request.publicKey.array()) + val signature = cryptoService.sign( + context.tenantId, + publicKey, + request.signatureSpec.toSignatureSpec(cryptoService.schemeMetadata), + request.bytes.array(), + request.context.toMap() + ) + CryptoSignatureWithKey( + ByteBuffer.wrap(cryptoService.schemeMetadata.encodeAsByteArray(signature.by)), + ByteBuffer.wrap(signature.bytes) + ) + } + + is ByIdsFlowQuery -> + CryptoSigningKeys(cryptoService.lookupSigningKeysByPublicKeyHashes( + context.tenantId, + request.fullKeyIds.hashes.map { SecureHashImpl(it.algorithm, it.bytes.array()) } + ).map { it.toCryptoSigningKey(keyEncodingService) }) + + else -> throw IllegalArgumentException("Unknown request type ${request::class.java.name}") + } + } + + private fun createResponseContext(request: FlowOpsRequest) = CryptoResponseContext( + request.context.requestingComponent, + request.context.requestTimestamp, + request.context.requestId, + Instant.now(), + request.context.tenantId, + KeyValuePairList(request.context.other.items.toList()) + ) +} diff --git a/components/crypto/crypto-service-impl/src/test/kotlin/net/corda/crypto/service/impl/rpc/CryptoFlowOpsRpcProcessorTests.kt b/components/crypto/crypto-service-impl/src/test/kotlin/net/corda/crypto/service/impl/rpc/CryptoFlowOpsRpcProcessorTests.kt new file mode 100644 index 00000000000..6df1b46566a --- /dev/null +++ b/components/crypto/crypto-service-impl/src/test/kotlin/net/corda/crypto/service/impl/rpc/CryptoFlowOpsRpcProcessorTests.kt @@ -0,0 +1,504 @@ +package net.corda.crypto.service.impl.rpc + +import net.corda.configuration.read.ConfigChangedEvent +import net.corda.crypto.cipher.suite.CipherSchemeMetadata +import net.corda.crypto.cipher.suite.KeyEncodingService +import net.corda.crypto.cipher.suite.SignatureSpecs +import net.corda.crypto.cipher.suite.sha256Bytes +import net.corda.crypto.client.CryptoOpsProxyClient +import net.corda.crypto.config.impl.createDefaultCryptoConfig +import net.corda.crypto.config.impl.retrying +import net.corda.crypto.config.impl.toCryptoConfig +import net.corda.crypto.core.CryptoService +import net.corda.crypto.core.DigitalSignatureWithKey +import net.corda.crypto.core.SecureHashImpl +import net.corda.crypto.core.SigningKeyInfo +import net.corda.crypto.core.fullId +import net.corda.crypto.flow.CryptoFlowOpsTransformer.Companion.REQUEST_OP_KEY +import net.corda.crypto.flow.CryptoFlowOpsTransformer.Companion.REQUEST_TTL_KEY +import net.corda.crypto.flow.CryptoFlowOpsTransformer.Companion.RESPONSE_TOPIC +import net.corda.crypto.flow.impl.CryptoFlowOpsTransformerImpl +import net.corda.crypto.service.impl.infra.ActResult +import net.corda.crypto.service.impl.infra.ActResultTimestamps +import net.corda.crypto.service.impl.infra.act +import net.corda.data.ExceptionEnvelope +import net.corda.data.KeyValuePairList +import net.corda.data.crypto.wire.CryptoResponseContext +import net.corda.data.crypto.wire.CryptoSignatureWithKey +import net.corda.data.crypto.wire.CryptoSigningKey +import net.corda.data.crypto.wire.CryptoSigningKeys +import net.corda.data.crypto.wire.ops.flow.FlowOpsRequest +import net.corda.data.crypto.wire.ops.flow.FlowOpsResponse +import net.corda.data.crypto.wire.ops.flow.commands.SignFlowCommand +import net.corda.data.crypto.wire.ops.flow.queries.ByIdsFlowQuery +import net.corda.data.flow.event.FlowEvent +import net.corda.data.flow.event.external.ExternalEventContext +import net.corda.flow.external.events.responses.factory.ExternalEventResponseFactory +import net.corda.libs.configuration.SmartConfigFactory +import net.corda.messaging.api.records.Record +import net.corda.schema.Schemas +import net.corda.schema.configuration.ConfigKeys +import net.corda.v5.application.crypto.DigestService +import net.corda.v5.crypto.DigestAlgorithmName +import net.corda.v5.crypto.SecureHash +import org.junit.jupiter.api.Assertions.assertInstanceOf +import org.junit.jupiter.api.Assertions.assertNotNull +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import org.mockito.kotlin.any +import org.mockito.kotlin.argumentCaptor +import org.mockito.kotlin.doAnswer +import org.mockito.kotlin.doReturn +import org.mockito.kotlin.eq +import org.mockito.kotlin.mock +import org.mockito.kotlin.whenever +import java.nio.ByteBuffer +import java.security.PublicKey +import java.time.Instant +import java.util.UUID +import kotlin.random.Random +import kotlin.test.assertEquals +import kotlin.test.assertTrue + + class CryptoFlowOpsRpcProcessorTests { + companion object { + private val configEvent = ConfigChangedEvent( + setOf(ConfigKeys.CRYPTO_CONFIG), + mapOf( + ConfigKeys.CRYPTO_CONFIG to + SmartConfigFactory.createWithoutSecurityServices().create( + createDefaultCryptoConfig("pass", "salt") + ) + ) + ) + } + + private lateinit var tenantId: String + private lateinit var componentName: String + private lateinit var eventTopic: String + private lateinit var responseTopic: String + private lateinit var keyEncodingService: KeyEncodingService + private lateinit var cryptoOpsClient: CryptoOpsProxyClient + private lateinit var cryptoService: CryptoService + private lateinit var externalEventResponseFactory: ExternalEventResponseFactory + private lateinit var processor: CryptoFlowOpsRpcProcessor + private lateinit var digestService: DigestService + + private val flowOpsResponseArgumentCaptor = argumentCaptor() + + private fun buildTransformer(ttl: Long = 123): CryptoFlowOpsTransformerImpl = + CryptoFlowOpsTransformerImpl( + serializer = mock(), + requestingComponent = componentName, + responseTopic = responseTopic, + keyEncodingService = keyEncodingService, + digestService = digestService, + requestValidityWindowSeconds = ttl + ) + + private fun mockPublicKey(): PublicKey { + val serialisedPublicKey = Random(Instant.now().toEpochMilli()).nextBytes(256) + return mock { + on { encoded } doReturn serialisedPublicKey + } + } + + private inline fun assertResponseContext( + result: ActResult>, + flowOpsResponse: FlowOpsResponse, + ttl: Long = 123 + ): RESPONSE { + assertNotNull(result.value) + assertInstanceOf(RESPONSE::class.java, flowOpsResponse.response) + val context = flowOpsResponse.context + val resp = flowOpsResponse.response as RESPONSE + assertResponseContext(result, context, ttl) + return resp + } + + private inline fun assertResponseContext( + timestamps: ActResultTimestamps, + context: CryptoResponseContext, + ttl: Long + ) { + timestamps.assertThatIsBetween(context.responseTimestamp) + //timestamps.assertThatIsBetween(context.requestTimestamp) // not always (or not normally?) true, TODO - find some way to cover? + assertEquals(componentName, context.requestingComponent) + assertTrue(context.other.items.size >= 3) + assertTrue { + context.other.items.firstOrNull { + it.key == REQUEST_OP_KEY && it.value == REQUEST::class.java.simpleName + } != null + } + assertTrue { + context.other.items.firstOrNull { + it.key == RESPONSE_TOPIC && it.value == responseTopic + } != null + } + assertTrue { + context.other.items.firstOrNull { + it.key == REQUEST_TTL_KEY && it.value == ttl.toString() + } != null + } + } + + @BeforeEach + fun setup() { + tenantId = UUID.randomUUID().toString() + componentName = UUID.randomUUID().toString() + eventTopic = UUID.randomUUID().toString() + responseTopic = UUID.randomUUID().toString() + keyEncodingService = mock { + on { encodeAsByteArray(any()) } doAnswer { + (it.getArgument(0) as PublicKey).encoded + } + on { decodePublicKey(any()) } doAnswer { sc -> + mock { + on { encoded } doAnswer { + sc.getArgument(0) as ByteArray + } + } + } + } + cryptoOpsClient = mock() + externalEventResponseFactory = mock() + val publicKeyMock = mock { + on { encoded } doReturn byteArrayOf(42) + } + val signatureMock = mock { + on { by } doReturn publicKeyMock + on { bytes } doReturn byteArrayOf(9, 0, 0, 0) + } + val schemeMetadataMock = mock { + on { decodePublicKey(any()) } doReturn publicKeyMock + on { encodeAsByteArray(any()) } doReturn byteArrayOf(42) + } + val singleSigningKeyInfo = listOf(mock { + on { publicKey } doReturn publicKeyMock + }) + cryptoService = mock { + on { sign(any(), any(), any(), any(), any()) } doReturn signatureMock + on { schemeMetadata } doReturn schemeMetadataMock + on { lookupSigningKeysByPublicKeyHashes(any(), any()) } doReturn singleSigningKeyInfo + } + val retryingConfig = configEvent.config.toCryptoConfig().retrying() + processor = CryptoFlowOpsRpcProcessor( + cryptoService, + externalEventResponseFactory, + retryingConfig, + keyEncodingService, + FlowOpsRequest::class.java, + FlowEvent::class.java + ) + digestService = mock().also { + fun capture() { + val bytesCaptor = argumentCaptor() + whenever(it.hash(bytesCaptor.capture(), any())).thenAnswer { + val bytes = bytesCaptor.firstValue + SecureHashImpl(DigestAlgorithmName.SHA2_256.name, bytes.sha256Bytes()).also { + capture() + } + } + } + capture() + } + } + + /** + * Results of a doFlowOperations experiment + * + * @param lookedUpSigningKeys - string form of signing keys looked up in each invocation + * @param successfulFlowOpsResponses - flow ops responses which were successful + * @param transformedResponse - transformed DTO form of the first successful response + * @param capturedTenantIds - tenant IDs stored in flow responses + * @param rawActResult - timing information and a list of raw records + * @param recordKeys - the UUIds of each flow op request + * @param rawFlowOpsResponses - uncast records for flow ops response capture + * @param flowExternalEventContexts - the contexts prepared for each flow op + */ + + data class Results( + val lookedUpSigningKeys: List>, + val successfulFlowOpsResponses: List, + val transformedResponses: List, + val capturedTenantIds: List, + val rawActResult: ActResult>, + val recordKeys: List, + val rawFlowOpsResponses: List, + val flowExternalEventContexts: List, + ) + + /** Run a flow operation in the mocked flow ops bus processor + + * @param P - type parameter for the flow os request + * @param R - type parameter for the flow ops responses + * @param S - type parameter for transformed flow ops responses + * @param myPublicKeys - the set of public keys available from the underlying signing service + * @param flowOpCallbacks - a list of callback to create the flow signing operation required, given a transformer and an event context + * + * @returns Results instance capturing data recorded during the flow operations + */ + private inline fun doFlowOperations( + myPublicKeys: List, + flowOpCallbacks: List<(CryptoFlowOpsTransformerImpl, ExternalEventContext) -> FlowOpsRequest?>, + + ): Results { + val indices = flowOpCallbacks.indices + val capturedTenantIds: MutableList = mutableListOf() + val lookedUpSigningKeys = mutableListOf>() // the secure hashes passed into the signing service + val recordKeys = flowOpCallbacks.map { + UUID.randomUUID().toString() + } // UUIDs for the flow op records that are passed into the crypto flow ops processor + + val flowExternalEventContexts = + recordKeys.map { ExternalEventContext("request id", it, KeyValuePairList(emptyList())) } + + indices.map { + whenever( + externalEventResponseFactory.success( + eq(flowExternalEventContexts[it]), + flowOpsResponseArgumentCaptor.capture() + ) + ).thenReturn( + Record( + Schemas.Flow.FLOW_EVENT_TOPIC, + flowExternalEventContexts[it].flowId, + FlowEvent() + ) + ) + whenever( + externalEventResponseFactory.platformError( + eq(flowExternalEventContexts[it]), + any() + ) + ).thenReturn( + Record( + Schemas.Flow.FLOW_EVENT_TOPIC, + flowExternalEventContexts[it].flowId, + FlowEvent() + ) + ) + whenever( + externalEventResponseFactory.transientError( + eq(flowExternalEventContexts[it]), + any() + ) + ).thenReturn( + Record( + Schemas.Flow.FLOW_EVENT_TOPIC, + flowExternalEventContexts[it].flowId, + FlowEvent() + ) + ) + } + + // capture what is passed in to the signing service operations + doAnswer { + capturedTenantIds.add(it.getArgument(0)) + lookedUpSigningKeys.add(it.getArgument>(1).map { it.toString() }) + myPublicKeys.map { mockSigningKeyInfo(it) } + }.whenever(cryptoService).lookupSigningKeysByPublicKeyHashes(any(), any()) + doAnswer { + capturedTenantIds.add(it.getArgument(0)) + DigitalSignatureWithKey(myPublicKeys.first(), byteArrayOf(42)) + }.whenever(cryptoService).sign(any(), any(), any(), any(), any()) + + val transformer = buildTransformer() + val flowOps = indices.map { flowOpCallbacks[it](transformer, flowExternalEventContexts[it]) } + + val requests = indices.map { + flowOps[it] + } + + // run the flows ops processor + val result = act { + requests.filterNotNull().map { + processor.process(it) + } + } + + val successfulFlowOpsResponses = + flowOpsResponseArgumentCaptor.allValues.map { assertResponseContext(result, it) } + + val transformedResponses = flowOpsResponseArgumentCaptor.allValues.map { + val x = transformer.transform(it) + if (x !is S) throw IllegalArgumentException() + x + } + + return Results( + lookedUpSigningKeys = lookedUpSigningKeys, + successfulFlowOpsResponses = successfulFlowOpsResponses, + transformedResponses = transformedResponses, + capturedTenantIds = capturedTenantIds.toList(), + rawActResult = result, + recordKeys = recordKeys, + rawFlowOpsResponses = flowOpsResponseArgumentCaptor.allValues, + flowExternalEventContexts = flowExternalEventContexts + ) + } + @Suppress("UNCHECKED_CAST") + @Test + fun `Should process filter my keys query`() { + val myPublicKeys = listOf( + mockPublicKey(), + mockPublicKey() + ) + + val notMyKey = mockPublicKey() + + val results = doFlowOperations>( + myPublicKeys, + listOf { transformer, flowExternalEventContext -> + transformer.createFilterMyKeys( + tenantId, + listOf(myPublicKeys[0], myPublicKeys[1], notMyKey), + flowExternalEventContext + ) + } + ) + assertEquals(1, results.lookedUpSigningKeys.size) + val passedSecureHashes = results.lookedUpSigningKeys.first() + assertEquals(3, passedSecureHashes.size) + assertEquals(myPublicKeys[0].fullId(), passedSecureHashes[0]) + assertEquals(myPublicKeys[1].fullId(), passedSecureHashes[1]) + assertEquals(notMyKey.fullId(), passedSecureHashes[2]) + assertNotNull(results.successfulFlowOpsResponses.first().keys) + assertEquals(2, results.successfulFlowOpsResponses.first().keys.size) + assertTrue(results.successfulFlowOpsResponses.first().keys.any { + it.publicKey.array().contentEquals(keyEncodingService.encodeAsByteArray(myPublicKeys[0])) + }) + assertTrue( + results.successfulFlowOpsResponses.first().keys.any { + it.publicKey.array().contentEquals(keyEncodingService.encodeAsByteArray(myPublicKeys[1])) + } + ) + assertEquals(2, results.transformedResponses.first().size) + assertTrue(results.transformedResponses.first().any { it.encoded.contentEquals(myPublicKeys[0].encoded) }) + assertTrue(results.transformedResponses.first().any { it.encoded.contentEquals(myPublicKeys[1].encoded) }) + assertEquals(results.capturedTenantIds, listOf(tenantId)) + } + + + @Test + fun `Should process sign command`() { + val publicKey = mockPublicKey() + val data = UUID.randomUUID().toString().toByteArray() + + doFlowOperations( + listOf(publicKey), listOf { transformer, flowExternalEventContext -> + transformer.createSign( + UUID.randomUUID().toString(), + tenantId, + publicKey.encoded, + SignatureSpecs.EDDSA_ED25519, + data, + mapOf("key1" to "value1"), + flowExternalEventContext + ) + } + ) + } + + @Test + fun `Should process list with valid event and skip event without value`() { + val myPublicKeys = listOf( + mockPublicKey(), + mockPublicKey() + ) + val notMyKey = mockPublicKey() + + val r = doFlowOperations>( + myPublicKeys, listOf( + { _, _ -> null }, + { t, f -> t.createFilterMyKeys(tenantId, listOf(myPublicKeys[0], myPublicKeys[1], notMyKey), f) }, + ) + ) + assertEquals(listOf(tenantId), r.capturedTenantIds) + assertEquals(3, r.lookedUpSigningKeys.first().size) + + // CryptoFlowOpsBusProcessor filters out null requests, since there's no information to send a response, + // so we should expect 1 output not 2 in this case + assertEquals(1, r.transformedResponses.size) + val transformed = r.transformedResponses.first() + assertInstanceOf(List::class.java, transformed) + assertEquals(2, transformed.size) + assertTrue(transformed.any { it.encoded.contentEquals(myPublicKeys[0].encoded) }) + assertTrue(transformed.any { it.encoded.contentEquals(myPublicKeys[1].encoded) }) + } + + @Suppress("UNCHECKED_CAST") + @Test + fun `Should process list with valid event and return error for failed event`() { + val failingTenantId = UUID.randomUUID().toString() + val myPublicKeys = listOf( + mockPublicKey(), + mockPublicKey() + ) + val notMyKey = mockPublicKey() + + val r = doFlowOperations>( + myPublicKeys, listOf( + { t, f -> + t.createFilterMyKeys( + failingTenantId, + listOf(myPublicKeys[0], myPublicKeys[1], notMyKey), + f + ) + }, + { t,f -> t.createFilterMyKeys( tenantId, listOf(myPublicKeys[0], myPublicKeys[1], notMyKey), f) } + )) + assertEquals(2, r.rawActResult.value?.size?:0) + + r.rawFlowOpsResponses[1].let { flowOpsResponse -> + assertInstanceOf(CryptoSigningKeys::class.java, flowOpsResponse.response) + val context1 = flowOpsResponse.context + val response1 = flowOpsResponse.response as CryptoSigningKeys + assertEquals(context1.tenantId, tenantId) + assertNotNull(response1.keys) + assertEquals(2, response1.keys.size) + assertTrue( + response1.keys.any { + it.publicKey.array().contentEquals(keyEncodingService.encodeAsByteArray(myPublicKeys[0])) + } + ) + assertTrue( + response1.keys.any { + it.publicKey.array().contentEquals(keyEncodingService.encodeAsByteArray(myPublicKeys[1])) + } + ) + } + assertEquals(2, r.capturedTenantIds.size) + assertEquals(failingTenantId, r.capturedTenantIds[0]) + assertEquals(tenantId, r.capturedTenantIds[1]) + assertEquals(2, r.lookedUpSigningKeys.size) + val passedList0 = r.lookedUpSigningKeys[0] + assertEquals(3, passedList0.size) + assertEquals(myPublicKeys[0].fullId(), passedList0[0]) + assertEquals(myPublicKeys[1].fullId(), passedList0[1]) + assertEquals(notMyKey.fullId(), passedList0[2]) + val passedList1 = r.lookedUpSigningKeys[0] + assertEquals(3, passedList1.size) + assertEquals(myPublicKeys[0].fullId(), passedList1[0]) + assertEquals(myPublicKeys[1].fullId(), passedList1[1]) + assertEquals(notMyKey.fullId(), passedList1[2]) + assertInstanceOf(List::class.java, r.transformedResponses) + val keys = r.transformedResponses.first() + assertEquals(2, r.transformedResponses.first().size) + assertTrue(keys.any { it.encoded.contentEquals(myPublicKeys[0].encoded) }) + assertTrue(keys.any { it.encoded.contentEquals(myPublicKeys[1].encoded) }) + } + + private fun mockSigningKeyInfo(key0: PublicKey) = mock { + on { id } doAnswer { + mock { + on { value } doAnswer { "id1" } + } + } + on { timestamp } doAnswer { Instant.now() } + on { publicKey } doAnswer { key0 } + on { toCryptoSigningKey(any()) } doAnswer { mock { + on { publicKey } doAnswer { ByteBuffer.wrap(keyEncodingService.encodeAsByteArray(key0)) } + } } + } +} \ No newline at end of file diff --git a/processors/crypto-processor/src/main/kotlin/net/corda/processors/crypto/internal/CryptoProcessorImpl.kt b/processors/crypto-processor/src/main/kotlin/net/corda/processors/crypto/internal/CryptoProcessorImpl.kt index f130618b4e7..fc410fa741a 100644 --- a/processors/crypto-processor/src/main/kotlin/net/corda/processors/crypto/internal/CryptoProcessorImpl.kt +++ b/processors/crypto-processor/src/main/kotlin/net/corda/processors/crypto/internal/CryptoProcessorImpl.kt @@ -30,6 +30,7 @@ import net.corda.crypto.persistence.db.model.CryptoEntities import net.corda.crypto.persistence.getEntityManagerFactory import net.corda.crypto.service.impl.TenantInfoServiceImpl import net.corda.crypto.service.impl.bus.CryptoFlowOpsBusProcessor +import net.corda.crypto.service.impl.rpc.CryptoFlowOpsRpcProcessor import net.corda.crypto.service.impl.bus.CryptoOpsBusProcessor import net.corda.crypto.service.impl.bus.HSMRegistrationBusProcessor import net.corda.crypto.softhsm.TenantInfoService @@ -40,8 +41,10 @@ import net.corda.crypto.softhsm.impl.SoftCryptoService import net.corda.crypto.softhsm.impl.WrappingRepositoryImpl import net.corda.data.crypto.wire.hsm.registration.HSMRegistrationRequest import net.corda.data.crypto.wire.hsm.registration.HSMRegistrationResponse +import net.corda.data.crypto.wire.ops.flow.FlowOpsRequest import net.corda.data.crypto.wire.ops.rpc.RpcOpsRequest import net.corda.data.crypto.wire.ops.rpc.RpcOpsResponse +import net.corda.data.flow.event.FlowEvent import net.corda.db.connection.manager.DbConnectionManager import net.corda.db.schema.CordaDb import net.corda.flow.external.events.responses.factory.ExternalEventResponseFactory @@ -61,6 +64,7 @@ import net.corda.lifecycle.createCoordinator import net.corda.messaging.api.subscription.SubscriptionBase import net.corda.messaging.api.subscription.config.RPCConfig import net.corda.messaging.api.subscription.config.SubscriptionConfig +import net.corda.messaging.api.subscription.config.SyncRPCConfig import net.corda.messaging.api.subscription.factory.SubscriptionFactory import net.corda.orm.JpaEntitiesRegistry import net.corda.processors.crypto.CryptoProcessor @@ -120,8 +124,11 @@ class CryptoProcessorImpl @Activate constructor( const val FLOW_OPS_SUBSCRIPTION = "FLOW_OPS_SUBSCRIPTION" const val RPC_OPS_SUBSCRIPTION = "RPC_OPS_SUBSCRIPTION" const val HSM_REG_SUBSCRIPTION = "HSM_REG_SUBSCRIPTION" - } + const val RPC_SUBSCRIPTION = "RPC_SUBSCRIPTION" + const val SUBSCRIPTION_NAME = "Crypto" + const val CRYPTO_PATH = "/crypto" + } init { jpaEntitiesRegistry.register(CordaDb.Crypto.persistenceUnitName, CryptoEntities.classes) @@ -199,7 +206,8 @@ class CryptoProcessorImpl @Activate constructor( logger.trace("Assigned SOFT HSM for $tenantId:$category") } } - startBusProcessors(event, coordinator) + startBusProcessors(event, coordinator) // to be removed when Event Mediator is fully implemented + startProcessors(event, coordinator) setStatus(LifecycleStatus.UP, coordinator) } } @@ -289,7 +297,7 @@ class CryptoProcessorImpl @Activate constructor( ) } - private fun startTenantInfoService() = TenantInfoServiceImpl({ + private fun startTenantInfoService() = TenantInfoServiceImpl { HSMRepositoryImpl( getEntityManagerFactory( CryptoTenants.CRYPTO, @@ -298,30 +306,35 @@ class CryptoProcessorImpl @Activate constructor( jpaEntitiesRegistry ) ) - }) - - - private fun startBusProcessors(event: ConfigChangedEvent, coordinator: LifecycleCoordinator) { - val cryptoConfig = event.config.getConfig(CRYPTO_CONFIG) + } + private fun startProcessors(event: ConfigChangedEvent, coordinator: LifecycleCoordinator) { + val cryptoConfig = event.config.getConfig(CRYPTO_CONFIG) - // make the processors + // create processors val retryingConfig = cryptoConfig.retrying() - val flowOpsProcessor = - CryptoFlowOpsBusProcessor(cryptoService, externalEventResponseFactory, retryingConfig, keyEncodingService) + val flowOpsProcessor = CryptoFlowOpsRpcProcessor( + cryptoService, + externalEventResponseFactory, + retryingConfig, keyEncodingService, + FlowOpsRequest::class.java, + FlowEvent::class.java + ) val rpcOpsProcessor = CryptoOpsBusProcessor(cryptoService, retryingConfig, keyEncodingService) val hsmRegistrationProcessor = HSMRegistrationBusProcessor(tenantInfoService, cryptoService, retryingConfig) - // now make and start the subscriptions + // create and start subscriptions val messagingConfig = event.config.getConfig(MESSAGING_CONFIG) val flowGroupName = "crypto.ops.flow" - coordinator.createManagedResource(FLOW_OPS_SUBSCRIPTION) { - subscriptionFactory.createDurableSubscription( - subscriptionConfig = SubscriptionConfig(flowGroupName, Schemas.Crypto.FLOW_OPS_MESSAGE_TOPIC), - processor = flowOpsProcessor, - messagingConfig = messagingConfig, - partitionAssignmentListener = null - ) + + val rpcConfig = SyncRPCConfig(SUBSCRIPTION_NAME, CRYPTO_PATH) + coordinator.createManagedResource(RPC_SUBSCRIPTION) { + subscriptionFactory.createHttpRPCSubscription( + rpcConfig = rpcConfig, + processor = flowOpsProcessor + ).also { + it.start() + } } logger.trace("Starting processing on $flowGroupName ${Schemas.Crypto.FLOW_OPS_MESSAGE_TOPIC}") coordinator.getManagedResource(FLOW_OPS_SUBSCRIPTION)!!.start() @@ -362,6 +375,28 @@ class CryptoProcessorImpl @Activate constructor( logger.trace("Starting processing on $hsmRegGroupName ${Schemas.Crypto.RPC_HSM_REGISTRATION_MESSAGE_TOPIC}") coordinator.getManagedResource(HSM_REG_SUBSCRIPTION)!!.start() } + + private fun startBusProcessors(event: ConfigChangedEvent, coordinator: LifecycleCoordinator) { + val cryptoConfig = event.config.getConfig(CRYPTO_CONFIG) + + // create processors + val retryingConfig = cryptoConfig.retrying() + val flowOpsProcessor = CryptoFlowOpsBusProcessor(cryptoService, externalEventResponseFactory, retryingConfig, keyEncodingService) + + // create and start subscriptions + val messagingConfig = event.config.getConfig(MESSAGING_CONFIG) + val flowGroupName = "crypto.ops.flow" + coordinator.createManagedResource(FLOW_OPS_SUBSCRIPTION) { + subscriptionFactory.createDurableSubscription( + subscriptionConfig = SubscriptionConfig(flowGroupName, Schemas.Crypto.FLOW_OPS_MESSAGE_TOPIC), + processor = flowOpsProcessor, + messagingConfig = messagingConfig, + partitionAssignmentListener = null + ) + } + logger.trace("Starting processing on $flowGroupName ${Schemas.Crypto.FLOW_OPS_MESSAGE_TOPIC}") + coordinator.getManagedResource(FLOW_OPS_SUBSCRIPTION)!!.start() + } private fun setStatus(status: LifecycleStatus, coordinator: LifecycleCoordinator) { logger.trace("Crypto processor is set to be $status") From 7465279f34e7022fb2d1da52a18c6054d037c8c7 Mon Sep 17 00:00:00 2001 From: David Currie Date: Tue, 17 Oct 2023 09:01:36 +0100 Subject: [PATCH 41/81] CORE-17822 Apply prefix to topic overrides (#4893) --- .../corda/cli/plugins/topicconfig/Create.kt | 11 ++++- .../plugins/topicconfig/CreateConnectTest.kt | 43 ++++++++++--------- .../cli/plugins/topicconfig/PreviewTest.kt | 1 + .../src/test/resources/preview_config.yaml | 12 +++--- .../short_generated_topic_config.yaml | 8 ++-- 5 files changed, 44 insertions(+), 31 deletions(-) diff --git a/tools/plugins/topic-config/src/main/kotlin/net/corda/cli/plugins/topicconfig/Create.kt b/tools/plugins/topic-config/src/main/kotlin/net/corda/cli/plugins/topicconfig/Create.kt index 1c4dbb41f49..eda3701434a 100644 --- a/tools/plugins/topic-config/src/main/kotlin/net/corda/cli/plugins/topicconfig/Create.kt +++ b/tools/plugins/topic-config/src/main/kotlin/net/corda/cli/plugins/topicconfig/Create.kt @@ -192,9 +192,18 @@ class Create( if (overrideFilePath == null) { config } else { - mergeConfigurations(config, mapper.readValue(Files.readString(Paths.get(overrideFilePath!!)))) + mergeConfigurations( + config, + applyPrefix(mapper.readValue(Files.readString(Paths.get(overrideFilePath!!)))) + ) } + private fun applyPrefix(overrides: OverrideTopicConfigurations): OverrideTopicConfigurations = + OverrideTopicConfigurations( + overrides.topics.map { it.copy(name = topic!!.namePrefix + it.name) }, + overrides.acls.map { it.copy(topic = topic!!.namePrefix + it.topic) } + ) + fun getTopicConfigsForPreview(topicConfigurations: List): PreviewTopicConfigurations { val topicConfigs = mutableListOf() val acls = mutableListOf() diff --git a/tools/plugins/topic-config/src/test/kotlin/net/corda/cli/plugins/topicconfig/CreateConnectTest.kt b/tools/plugins/topic-config/src/test/kotlin/net/corda/cli/plugins/topicconfig/CreateConnectTest.kt index 08ed0f7b38f..b6ffd4709c8 100644 --- a/tools/plugins/topic-config/src/test/kotlin/net/corda/cli/plugins/topicconfig/CreateConnectTest.kt +++ b/tools/plugins/topic-config/src/test/kotlin/net/corda/cli/plugins/topicconfig/CreateConnectTest.kt @@ -45,19 +45,19 @@ class CreateConnectTest { val acls = cmd.getGeneratedTopicConfigs().acls assertThat(cmd.getAclBindings(acls)) .containsExactly( - AclBinding(ResourcePattern(ResourceType.TOPIC, "avro.schema", PatternType.LITERAL), + AclBinding(ResourcePattern(ResourceType.TOPIC, "prefix.avro.schema", PatternType.LITERAL), AccessControlEntry("User:Chris", "*", AclOperation.READ, AclPermissionType.ALLOW)), - AclBinding(ResourcePattern(ResourceType.TOPIC, "avro.schema", PatternType.LITERAL), + AclBinding(ResourcePattern(ResourceType.TOPIC, "prefix.avro.schema", PatternType.LITERAL), AccessControlEntry("User:Chris", "*", AclOperation.WRITE, AclPermissionType.ALLOW)), - AclBinding(ResourcePattern(ResourceType.TOPIC, "avro.schema", PatternType.LITERAL), + AclBinding(ResourcePattern(ResourceType.TOPIC, "prefix.avro.schema", PatternType.LITERAL), AccessControlEntry("User:Chris", "*", AclOperation.DESCRIBE, AclPermissionType.ALLOW)), - AclBinding(ResourcePattern(ResourceType.TOPIC, "avro.schema", PatternType.LITERAL), + AclBinding(ResourcePattern(ResourceType.TOPIC, "prefix.avro.schema", PatternType.LITERAL), AccessControlEntry("User:Mo", "*", AclOperation.READ, AclPermissionType.ALLOW)), - AclBinding(ResourcePattern(ResourceType.TOPIC, "avro.schema", PatternType.LITERAL), + AclBinding(ResourcePattern(ResourceType.TOPIC, "prefix.avro.schema", PatternType.LITERAL), AccessControlEntry("User:Mo", "*", AclOperation.DESCRIBE, AclPermissionType.ALLOW)), - AclBinding(ResourcePattern(ResourceType.TOPIC, "certificates.rpc.ops", PatternType.LITERAL), + AclBinding(ResourcePattern(ResourceType.TOPIC, "prefix.certificates.rpc.ops", PatternType.LITERAL), AccessControlEntry("User:Dan", "*", AclOperation.READ, AclPermissionType.ALLOW)), - AclBinding(ResourcePattern(ResourceType.TOPIC, "certificates.rpc.ops", PatternType.LITERAL), + AclBinding(ResourcePattern(ResourceType.TOPIC, "prefix.certificates.rpc.ops", PatternType.LITERAL), AccessControlEntry("User:Dan", "*", AclOperation.DESCRIBE, AclPermissionType.ALLOW)) ) @@ -69,23 +69,23 @@ class CreateConnectTest { val acls = cmd.getGeneratedTopicConfigs().acls assertThat(cmd.getAclBindings(acls)) .containsExactly( - AclBinding(ResourcePattern(ResourceType.TOPIC, "avro.schema", PatternType.LITERAL), + AclBinding(ResourcePattern(ResourceType.TOPIC, "prefix.avro.schema", PatternType.LITERAL), AccessControlEntry("User:Chris", "*", AclOperation.READ, AclPermissionType.ALLOW)), - AclBinding(ResourcePattern(ResourceType.TOPIC, "avro.schema", PatternType.LITERAL), + AclBinding(ResourcePattern(ResourceType.TOPIC, "prefix.avro.schema", PatternType.LITERAL), AccessControlEntry("User:Chris", "*", AclOperation.WRITE, AclPermissionType.ALLOW)), - AclBinding(ResourcePattern(ResourceType.TOPIC, "avro.schema", PatternType.LITERAL), + AclBinding(ResourcePattern(ResourceType.TOPIC, "prefix.avro.schema", PatternType.LITERAL), AccessControlEntry("User:Chris", "*", AclOperation.DESCRIBE, AclPermissionType.ALLOW)), - AclBinding(ResourcePattern(ResourceType.TOPIC, "avro.schema", PatternType.LITERAL), + AclBinding(ResourcePattern(ResourceType.TOPIC, "prefix.avro.schema", PatternType.LITERAL), AccessControlEntry("User:Mo", "*", AclOperation.READ, AclPermissionType.ALLOW)), - AclBinding(ResourcePattern(ResourceType.TOPIC, "avro.schema", PatternType.LITERAL), + AclBinding(ResourcePattern(ResourceType.TOPIC, "prefix.avro.schema", PatternType.LITERAL), AccessControlEntry("User:Mo", "*", AclOperation.DESCRIBE, AclPermissionType.ALLOW)), - AclBinding(ResourcePattern(ResourceType.TOPIC, "avro.schema", PatternType.LITERAL), + AclBinding(ResourcePattern(ResourceType.TOPIC, "prefix.avro.schema", PatternType.LITERAL), AccessControlEntry("User:Mo", "*", AclOperation.WRITE, AclPermissionType.ALLOW)), - AclBinding(ResourcePattern(ResourceType.TOPIC, "certificates.rpc.ops", PatternType.LITERAL), + AclBinding(ResourcePattern(ResourceType.TOPIC, "prefix.certificates.rpc.ops", PatternType.LITERAL), AccessControlEntry("User:Dan", "*", AclOperation.READ, AclPermissionType.ALLOW)), - AclBinding(ResourcePattern(ResourceType.TOPIC, "certificates.rpc.ops", PatternType.LITERAL), + AclBinding(ResourcePattern(ResourceType.TOPIC, "prefix.certificates.rpc.ops", PatternType.LITERAL), AccessControlEntry("User:Dan", "*", AclOperation.DESCRIBE, AclPermissionType.ALLOW)), - AclBinding(ResourcePattern(ResourceType.TOPIC, "certificates.rpc.ops", PatternType.LITERAL), + AclBinding(ResourcePattern(ResourceType.TOPIC, "prefix.certificates.rpc.ops", PatternType.LITERAL), AccessControlEntry("User:George", "*", AclOperation.READ, AclPermissionType.ALLOW)) ) @@ -96,11 +96,11 @@ class CreateConnectTest { val cmd = getCommandWithConfigFile() val topics = cmd.getGeneratedTopicConfigs().topics assertThat(cmd.getTopics(topics)) - .containsEntry("avro.schema", NewTopic("avro.schema", 5, 3) + .containsEntry("prefix.avro.schema", NewTopic("prefix.avro.schema", 5, 3) .configs(mapOf("cleanup.policy" to "compact", "segment.ms" to "600000", "delete.retention.ms" to "300000", "min.compaction.lag.ms" to "60000", "max.compaction.lag.ms" to "604800000", "min.cleanable.dirty.ratio" to "0.5"))) - .containsEntry("certificates.rpc.ops", NewTopic("certificates.rpc.ops", 4, 2) + .containsEntry("prefix.certificates.rpc.ops", NewTopic("prefix.certificates.rpc.ops", 4, 2) .configs(emptyMap())) } @@ -109,11 +109,11 @@ class CreateConnectTest { val cmd = getCommandWithConfigAndOverrideFiles() val topics = cmd.getGeneratedTopicConfigs().topics assertThat(cmd.getTopics(topics)) - .containsEntry("avro.schema", NewTopic("avro.schema", 8, 3) + .containsEntry("prefix.avro.schema", NewTopic("prefix.avro.schema", 8, 3) .configs(mapOf("cleanup.policy" to "compact", "segment.ms" to "600000", "delete.retention.ms" to "300000", "min.compaction.lag.ms" to "60000", "max.compaction.lag.ms" to "604800000", "min.cleanable.dirty.ratio" to "0.7"))) - .containsEntry("certificates.rpc.ops", NewTopic("certificates.rpc.ops", 4, 2) + .containsEntry("prefix.certificates.rpc.ops", NewTopic("prefix.certificates.rpc.ops", 4, 2) .configs(emptyMap())) } @@ -135,6 +135,7 @@ class CreateConnectTest { private fun getCommandWithGeneratedConfig() = CreateConnect().apply { create = Create() create!!.topic = TopicPlugin.Topic() + create!!.topic!!.namePrefix = "prefix." create!!.kafkaUsers = mapOf("crypto" to "Chris", "db" to "Dan", "flow" to "Fiona", "membership" to "Mo") } @@ -143,6 +144,7 @@ class CreateConnectTest { configFilePath = Paths.get(configFile).toString() create = Create() create!!.topic = TopicPlugin.Topic() + create!!.topic!!.namePrefix = "prefix." } private fun getCommandWithConfigAndOverrideFiles() = CreateConnect().apply { @@ -150,6 +152,7 @@ class CreateConnectTest { configFilePath = Paths.get(configFile).toString() create = Create() create!!.topic = TopicPlugin.Topic() + create!!.topic!!.namePrefix = "prefix." val overrideFile = this::class.java.classLoader.getResource("override_topic_config.yaml")!!.toURI() create!!.overrideFilePath = Paths.get(overrideFile).toString() } diff --git a/tools/plugins/topic-config/src/test/kotlin/net/corda/cli/plugins/topicconfig/PreviewTest.kt b/tools/plugins/topic-config/src/test/kotlin/net/corda/cli/plugins/topicconfig/PreviewTest.kt index a11f205b805..84909809503 100644 --- a/tools/plugins/topic-config/src/test/kotlin/net/corda/cli/plugins/topicconfig/PreviewTest.kt +++ b/tools/plugins/topic-config/src/test/kotlin/net/corda/cli/plugins/topicconfig/PreviewTest.kt @@ -41,6 +41,7 @@ class PreviewTest { "persistence" to "I", "rest" to "J", "uniqueness" to "K") + preview.create!!.topic!!.namePrefix = "prefix." return preview } } diff --git a/tools/plugins/topic-config/src/test/resources/preview_config.yaml b/tools/plugins/topic-config/src/test/resources/preview_config.yaml index f81594402c5..91599b43f5d 100644 --- a/tools/plugins/topic-config/src/test/resources/preview_config.yaml +++ b/tools/plugins/topic-config/src/test/resources/preview_config.yaml @@ -1,13 +1,13 @@ topics: - - name: config.management.request + - name: prefix.config.management.request partitions: 1 replicas: 1 config: {} - - name: config.management.request.resp + - name: prefix.config.management.request.resp partitions: 1 replicas: 1 config: {} - - name: config.topic + - name: prefix.config.topic partitions: 1 replicas: 1 config: @@ -18,7 +18,7 @@ topics: max.compaction.lag.ms: 604800000 min.cleanable.dirty.ratio: 0.5 acls: - - topic: config.management.request + - topic: prefix.config.management.request users: - name: B operations: @@ -28,7 +28,7 @@ acls: operations: - write - describe - - topic: config.management.request.resp + - topic: prefix.config.management.request.resp users: - name: J operations: @@ -38,7 +38,7 @@ acls: operations: - write - describe - - topic: config.topic + - topic: prefix.config.topic users: - name: A operations: diff --git a/tools/plugins/topic-config/src/test/resources/short_generated_topic_config.yaml b/tools/plugins/topic-config/src/test/resources/short_generated_topic_config.yaml index a47beb67ad2..71ac7ee8312 100644 --- a/tools/plugins/topic-config/src/test/resources/short_generated_topic_config.yaml +++ b/tools/plugins/topic-config/src/test/resources/short_generated_topic_config.yaml @@ -1,5 +1,5 @@ topics: - - name: avro.schema + - name: prefix.avro.schema partitions: 5 replicas: 3 config: @@ -9,12 +9,12 @@ topics: min.compaction.lag.ms: 60000 max.compaction.lag.ms: 604800000 min.cleanable.dirty.ratio: 0.5 - - name: certificates.rpc.ops + - name: prefix.certificates.rpc.ops partitions: 4 replicas: 2 config: {} acls: - - topic: avro.schema + - topic: prefix.avro.schema users: - name: Chris operations: @@ -25,7 +25,7 @@ acls: operations: - read - describe - - topic: certificates.rpc.ops + - topic: prefix.certificates.rpc.ops users: - name: Dan operations: From cc61a525783b20bcbcdf5457a9c3746bf0d9d30f Mon Sep 17 00:00:00 2001 From: Nikolett Nagy <61757742+nikinagy@users.noreply.github.com> Date: Tue, 17 Oct 2023 11:18:12 +0100 Subject: [PATCH 42/81] CORE-17045 - Membership platform upgrade test (#4817) Description: used more generic name for cross version testing cluster added ability to define notary service name when onboarding notaries --- .../main/kotlin/net/corda/e2etest/utilities/ClusterInfo.kt | 6 +++--- .../kotlin/net/corda/e2etest/utilities/MembershipUtils.kt | 6 ++++-- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/testing/e2e-test-utilities/src/main/kotlin/net/corda/e2etest/utilities/ClusterInfo.kt b/testing/e2e-test-utilities/src/main/kotlin/net/corda/e2etest/utilities/ClusterInfo.kt index 8f7a3068970..66db2be50ba 100644 --- a/testing/e2e-test-utilities/src/main/kotlin/net/corda/e2etest/utilities/ClusterInfo.kt +++ b/testing/e2e-test-utilities/src/main/kotlin/net/corda/e2etest/utilities/ClusterInfo.kt @@ -104,8 +104,8 @@ object ClusterCInfo : ClusterInfo() { } /** - * Default cluster info for E2E test cluster on which 5.0 deployment will run + * Default cluster info for E2E test cluster on which a previous major version of deployment will run */ -object ClusterFive0Info : ClusterInfo() { - override val id = "FIVE0" +object ClusterDInfo : ClusterInfo() { + override val id = "D" } \ No newline at end of file diff --git a/testing/e2e-test-utilities/src/main/kotlin/net/corda/e2etest/utilities/MembershipUtils.kt b/testing/e2e-test-utilities/src/main/kotlin/net/corda/e2etest/utilities/MembershipUtils.kt index 31306ca61a8..d620dbcf887 100644 --- a/testing/e2e-test-utilities/src/main/kotlin/net/corda/e2etest/utilities/MembershipUtils.kt +++ b/testing/e2e-test-utilities/src/main/kotlin/net/corda/e2etest/utilities/MembershipUtils.kt @@ -33,6 +33,7 @@ const val CERT_ALIAS_P2P = "p2p-tls-cert" const val CERT_ALIAS_SESSION = "p2p-session-cert" const val DEFAULT_KEY_SCHEME = "CORDA.ECDSA.SECP256R1" const val DEFAULT_SIGNATURE_SPEC = "SHA256withECDSA" +const val DEFAULT_NOTARY_SERVICE = "O=NotaryService, L=London, C=GB" /** * Onboard a member by uploading a CPI if it doesn't exist, creating a vnode if it doesn't exist, configuring the @@ -151,7 +152,8 @@ fun ClusterInfo.onboardNotaryMember( x500Name: String, wait: Boolean = true, getAdditionalContext: ((holdingId: String) -> Map)? = null, - tlsCertificateUploadedCallback: (String) -> Unit = {} + tlsCertificateUploadedCallback: (String) -> Unit = {}, + notaryServiceName: String = DEFAULT_NOTARY_SERVICE ) = onboardMember( resourceName, cpiName, @@ -164,7 +166,7 @@ fun ClusterInfo.onboardNotaryMember( mapOf( "corda.roles.0" to "notary", - "corda.notary.service.name" to MemberX500Name.parse("O=NotaryService, L=London, C=GB").toString(), + "corda.notary.service.name" to MemberX500Name.parse(notaryServiceName).toString(), "corda.notary.service.flow.protocol.name" to "com.r3.corda.notary.plugin.nonvalidating", "corda.notary.service.flow.protocol.version.0" to "1", "corda.notary.keys.0.id" to notaryKeyId, From 2a6037ab09208234ccb9346d7ff107fcc33d2ba5 Mon Sep 17 00:00:00 2001 From: James Higgs <45565019+JamesHR3@users.noreply.github.com> Date: Tue, 17 Oct 2023 10:42:58 +0100 Subject: [PATCH 43/81] CORE-17627: Add metadata to flow mapper states for mapper status (#4886) Adds the state status key to flow mapper state metadata and populates it with the current state. The metadata key is used by the mapper cleanup logic to ensure that mapper states are removed. --- .../executor/FlowMapperMessageProcessor.kt | 9 ++++- .../FlowMapperMessageProcessorTest.kt | 38 +++++++++++++++++-- 2 files changed, 43 insertions(+), 4 deletions(-) diff --git a/components/flow/flow-mapper-service/src/main/kotlin/net/corda/session/mapper/service/executor/FlowMapperMessageProcessor.kt b/components/flow/flow-mapper-service/src/main/kotlin/net/corda/session/mapper/service/executor/FlowMapperMessageProcessor.kt index 87db6bec60f..e1bbebbdef0 100644 --- a/components/flow/flow-mapper-service/src/main/kotlin/net/corda/session/mapper/service/executor/FlowMapperMessageProcessor.kt +++ b/components/flow/flow-mapper-service/src/main/kotlin/net/corda/session/mapper/service/executor/FlowMapperMessageProcessor.kt @@ -6,11 +6,13 @@ import net.corda.data.flow.event.mapper.FlowMapperEvent import net.corda.data.flow.state.mapper.FlowMapperState import net.corda.flow.mapper.factory.FlowMapperEventExecutorFactory import net.corda.libs.configuration.SmartConfig +import net.corda.libs.statemanager.api.Metadata import net.corda.messaging.api.processor.StateAndEventProcessor import net.corda.messaging.api.processor.StateAndEventProcessor.State import net.corda.messaging.api.records.Record import net.corda.metrics.CordaMetrics import net.corda.schema.configuration.FlowConfig +import net.corda.session.mapper.service.state.StateMetadataKeys.FLOW_MAPPER_STATUS import net.corda.tracing.traceStateAndEventExecution import net.corda.utilities.debug import net.corda.utilities.time.UTCClock @@ -60,8 +62,13 @@ class FlowMapperMessageProcessor( if (!isExpiredSessionEvent(value)) { val executor = flowMapperEventExecutorFactory.create(key, value, state?.value, flowConfig) val result = executor.execute() + val newMap = result.flowMapperState?.status?.let { + mapOf(FLOW_MAPPER_STATUS to it.toString()) + } ?: mapOf() StateAndEventProcessor.Response( - State(result.flowMapperState, state?.metadata), + State(result.flowMapperState, state?.metadata?.let { + Metadata(it + newMap) + } ?: Metadata(newMap)), result.outputEvents ) } else { diff --git a/components/flow/flow-mapper-service/src/test/kotlin/net/corda/session/mapper/service/executor/FlowMapperMessageProcessorTest.kt b/components/flow/flow-mapper-service/src/test/kotlin/net/corda/session/mapper/service/executor/FlowMapperMessageProcessorTest.kt index f72a72056b4..8ab9ee6ca65 100644 --- a/components/flow/flow-mapper-service/src/test/kotlin/net/corda/session/mapper/service/executor/FlowMapperMessageProcessorTest.kt +++ b/components/flow/flow-mapper-service/src/test/kotlin/net/corda/session/mapper/service/executor/FlowMapperMessageProcessorTest.kt @@ -17,6 +17,7 @@ import net.corda.libs.statemanager.api.Metadata import net.corda.messaging.api.processor.StateAndEventProcessor.State import net.corda.messaging.api.records.Record import net.corda.schema.configuration.FlowConfig +import net.corda.session.mapper.service.state.StateMetadataKeys.FLOW_MAPPER_STATUS import net.corda.test.flow.util.buildSessionEvent import org.assertj.core.api.Assertions.assertThat import org.junit.jupiter.api.Test @@ -76,11 +77,16 @@ class FlowMapperMessageProcessorTest { @Test fun `when state is OPEN new session events are processed`() { val metadata = Metadata(mapOf("foo" to "bar")) + whenever(flowMapperEventExecutor.execute()).thenReturn(FlowMapperResult(FlowMapperState().apply { + status = FlowMapperStateType.OPEN + }, listOf())) val output = flowMapperMessageProcessor.onNext( buildMapperState(FlowMapperStateType.OPEN, metadata),buildMapperEvent(buildSessionEvent()) ) verify(flowMapperEventExecutorFactory, times(1)).create(any(), any(), anyOrNull(), any(), any()) - assertThat(output.updatedState?.metadata).isEqualTo(metadata) + assertThat(output.updatedState?.metadata).isEqualTo( + Metadata(metadata + mapOf(FLOW_MAPPER_STATUS to FlowMapperStateType.OPEN.toString())) + ) } @Test @@ -91,9 +97,12 @@ class FlowMapperMessageProcessorTest { @Test fun `when state is OPEN expired session events are not processed`() { - flowMapperMessageProcessor.onNext(buildMapperState(FlowMapperStateType.OPEN), buildMapperEvent(buildSessionEvent(Instant.now() - .minusSeconds(100000)))) + val metadata = Metadata(mapOf("foo" to "bar")) + val output = flowMapperMessageProcessor.onNext( + buildMapperState(FlowMapperStateType.OPEN, metadata = metadata), + buildMapperEvent(buildSessionEvent(Instant.now().minusSeconds(100000)))) verify(flowMapperEventExecutorFactory, times(0)).create(any(), any(), anyOrNull(), any(), any()) + assertThat(output.updatedState?.metadata).isEqualTo(metadata) } @Test @@ -127,4 +136,27 @@ class FlowMapperMessageProcessorTest { flowMapperMessageProcessor.onNext(buildMapperState(FlowMapperStateType.ERROR), buildMapperEvent(buildSessionEvent(Instant.now()))) verify(flowMapperEventExecutorFactory, times(1)).create(any(), any(), anyOrNull(), any(), any()) } + + @Test + fun `when input event has no value the existing state is returned`() { + val metadata = Metadata(mapOf("foo" to "bar")) + val state = buildMapperState(FlowMapperStateType.OPEN, metadata) + val output = flowMapperMessageProcessor.onNext(state, Record("foo", "foo", null)) + verify(flowMapperEventExecutorFactory, times(0)).create(any(), any(), anyOrNull(), any(), any()) + assertThat(output.updatedState).isEqualTo(state) + } + + @Test + fun `when input metadata is null metadata is still set`() { + whenever(flowMapperEventExecutor.execute()).thenReturn(FlowMapperResult(FlowMapperState().apply { + status = FlowMapperStateType.OPEN + }, listOf())) + val output = flowMapperMessageProcessor.onNext( + buildMapperState(FlowMapperStateType.OPEN),buildMapperEvent(buildSessionEvent()) + ) + verify(flowMapperEventExecutorFactory, times(1)).create(any(), any(), anyOrNull(), any(), any()) + assertThat(output.updatedState?.metadata).isEqualTo( + Metadata(mapOf(FLOW_MAPPER_STATUS to FlowMapperStateType.OPEN.toString())) + ) + } } From 041a355f9145cca96dd91521777f4fe33f578597 Mon Sep 17 00:00:00 2001 From: James Higgs <45565019+JamesHR3@users.noreply.github.com> Date: Tue, 17 Oct 2023 13:33:54 +0100 Subject: [PATCH 44/81] CORE-17388: Add session timeout metadata to store alongside the checkpoint (#4889) Adds session timeout to the metadata stored alongside the checkpoint. Session timeout is implemented using the state storage lookup mechanism, which requires the metadata to be populated with the session timeout key. The earliest timestamp of session expiry for currently open sessions is the value assigned to this property. --- .../SessionTimeoutTaskProcessor.kt | 3 +- .../impl/FlowGlobalPostProcessorImpl.kt | 33 +++++++++- .../flow/state/impl/CheckpointMetadataKeys.kt | 14 +++++ .../SessionTimeoutTaskProcessorTests.kt | 2 +- .../impl/FlowGlobalPostProcessorImplTest.kt | 62 +++++++++++++++++++ .../flow/test/utils/FlowEventContextHelper.kt | 1 + 6 files changed, 111 insertions(+), 4 deletions(-) create mode 100644 components/flow/flow-service/src/main/kotlin/net/corda/flow/state/impl/CheckpointMetadataKeys.kt diff --git a/components/flow/flow-service/src/main/kotlin/net/corda/flow/maintenance/SessionTimeoutTaskProcessor.kt b/components/flow/flow-service/src/main/kotlin/net/corda/flow/maintenance/SessionTimeoutTaskProcessor.kt index 3224b05d9a0..bc6d3172233 100644 --- a/components/flow/flow-service/src/main/kotlin/net/corda/flow/maintenance/SessionTimeoutTaskProcessor.kt +++ b/components/flow/flow-service/src/main/kotlin/net/corda/flow/maintenance/SessionTimeoutTaskProcessor.kt @@ -2,6 +2,7 @@ package net.corda.flow.maintenance import net.corda.data.flow.FlowTimeout import net.corda.data.scheduler.ScheduledTaskTrigger +import net.corda.flow.state.impl.CheckpointMetadataKeys.STATE_META_SESSION_EXPIRY_KEY import net.corda.libs.statemanager.api.MetadataFilter import net.corda.libs.statemanager.api.Operation import net.corda.libs.statemanager.api.StateManager @@ -18,8 +19,6 @@ class SessionTimeoutTaskProcessor( ) : DurableProcessor { companion object { private val logger = LoggerFactory.getLogger(SessionTimeoutTaskProcessor::class.java) - // TODO - this may need to move out somewhere else. - const val STATE_META_SESSION_EXPIRY_KEY = "session.expiry" } override val keyClass: Class get() = String::class.java diff --git a/components/flow/flow-service/src/main/kotlin/net/corda/flow/pipeline/impl/FlowGlobalPostProcessorImpl.kt b/components/flow/flow-service/src/main/kotlin/net/corda/flow/pipeline/impl/FlowGlobalPostProcessorImpl.kt index 1f0334583c1..79e9024931a 100644 --- a/components/flow/flow-service/src/main/kotlin/net/corda/flow/pipeline/impl/FlowGlobalPostProcessorImpl.kt +++ b/components/flow/flow-service/src/main/kotlin/net/corda/flow/pipeline/impl/FlowGlobalPostProcessorImpl.kt @@ -12,10 +12,13 @@ import net.corda.flow.pipeline.events.FlowEventContext import net.corda.flow.pipeline.exceptions.FlowFatalException import net.corda.flow.pipeline.factory.FlowMessageFactory import net.corda.flow.pipeline.factory.FlowRecordFactory +import net.corda.flow.state.impl.CheckpointMetadataKeys.STATE_META_SESSION_EXPIRY_KEY +import net.corda.libs.statemanager.api.Metadata import net.corda.membership.read.MembershipGroupReaderProvider import net.corda.messaging.api.records.Record import net.corda.schema.configuration.FlowConfig.EXTERNAL_EVENT_MESSAGE_RESEND_WINDOW import net.corda.schema.configuration.FlowConfig.SESSION_FLOW_CLEANUP_TIME +import net.corda.schema.configuration.FlowConfig.SESSION_TIMEOUT_WINDOW import net.corda.session.manager.SessionManager import net.corda.utilities.debug import net.corda.v5.base.types.MemberX500Name @@ -55,8 +58,12 @@ class FlowGlobalPostProcessorImpl @Activate constructor( postProcessRetries(context) context.flowMetrics.flowEventCompleted(context.inputEvent.payload::class.java.name) + val metadata = getStateMetadata(context) - return context.copy(outputRecords = context.outputRecords + outputRecords) + return context.copy( + outputRecords = context.outputRecords + outputRecords, + metadata = metadata + ) } private fun getSessionEvents(context: FlowEventContext, now: Instant): List> { @@ -196,4 +203,28 @@ class FlowGlobalPostProcessorImpl @Activate constructor( val status = flowMessageFactory.createFlowStartedStatusMessage(checkpoint) return listOf(flowRecordFactory.createFlowStatusRecord(status)) } + + private fun getStateMetadata(context: FlowEventContext): Metadata? { + val checkpoint = context.checkpoint + // Find the earliest expiry time for any open sessions. + val lastReceivedMessageTime = checkpoint.sessions.filter { + it.status == SessionStateType.CREATED || it.status == SessionStateType.CONFIRMED + }.minByOrNull { it.lastReceivedMessageTime }?.lastReceivedMessageTime + + return if (lastReceivedMessageTime != null) { + // Add the metadata key if there are any open sessions. + val expiryTime = lastReceivedMessageTime + Duration.ofMillis( + context.flowConfig.getLong(SESSION_TIMEOUT_WINDOW) + ) + val newMap = mapOf(STATE_META_SESSION_EXPIRY_KEY to expiryTime.epochSecond) + context.metadata?.let { + Metadata(it + newMap) + } ?: Metadata(newMap) + } else { + // If there are no open sessions, remove the metadata key. + context.metadata?.let { + Metadata(it - STATE_META_SESSION_EXPIRY_KEY) + } + } + } } diff --git a/components/flow/flow-service/src/main/kotlin/net/corda/flow/state/impl/CheckpointMetadataKeys.kt b/components/flow/flow-service/src/main/kotlin/net/corda/flow/state/impl/CheckpointMetadataKeys.kt new file mode 100644 index 00000000000..3c4ee2b638b --- /dev/null +++ b/components/flow/flow-service/src/main/kotlin/net/corda/flow/state/impl/CheckpointMetadataKeys.kt @@ -0,0 +1,14 @@ +package net.corda.flow.state.impl + +/** + * Metadata keys for information stored alongside the flow checkpoint. + */ +object CheckpointMetadataKeys { + /** + * Earliest expiry time of any session still active in this checkpoint. + * + * Note that the time provided here should only take into consideration open sessions. If the checkpoint has no open + * sessions, then this metadata key should be removed. + */ + const val STATE_META_SESSION_EXPIRY_KEY = "session.expiry" +} \ No newline at end of file diff --git a/components/flow/flow-service/src/test/kotlin/net/corda/flow/maintenance/SessionTimeoutTaskProcessorTests.kt b/components/flow/flow-service/src/test/kotlin/net/corda/flow/maintenance/SessionTimeoutTaskProcessorTests.kt index 7639576a6df..9a9303f057c 100644 --- a/components/flow/flow-service/src/test/kotlin/net/corda/flow/maintenance/SessionTimeoutTaskProcessorTests.kt +++ b/components/flow/flow-service/src/test/kotlin/net/corda/flow/maintenance/SessionTimeoutTaskProcessorTests.kt @@ -2,7 +2,7 @@ package net.corda.flow.maintenance import net.corda.data.flow.FlowTimeout import net.corda.data.scheduler.ScheduledTaskTrigger -import net.corda.flow.maintenance.SessionTimeoutTaskProcessor.Companion.STATE_META_SESSION_EXPIRY_KEY +import net.corda.flow.state.impl.CheckpointMetadataKeys.STATE_META_SESSION_EXPIRY_KEY import net.corda.libs.statemanager.api.Metadata import net.corda.libs.statemanager.api.State import net.corda.libs.statemanager.api.StateManager diff --git a/components/flow/flow-service/src/test/kotlin/net/corda/flow/pipeline/impl/FlowGlobalPostProcessorImplTest.kt b/components/flow/flow-service/src/test/kotlin/net/corda/flow/pipeline/impl/FlowGlobalPostProcessorImplTest.kt index 692c6b0a5c9..bd0efde7d66 100644 --- a/components/flow/flow-service/src/test/kotlin/net/corda/flow/pipeline/impl/FlowGlobalPostProcessorImplTest.kt +++ b/components/flow/flow-service/src/test/kotlin/net/corda/flow/pipeline/impl/FlowGlobalPostProcessorImplTest.kt @@ -19,10 +19,13 @@ import net.corda.flow.pipeline.exceptions.FlowFatalException import net.corda.flow.pipeline.factory.FlowMessageFactory import net.corda.flow.pipeline.factory.FlowRecordFactory import net.corda.flow.state.FlowCheckpoint +import net.corda.flow.state.impl.CheckpointMetadataKeys.STATE_META_SESSION_EXPIRY_KEY import net.corda.flow.test.utils.buildFlowEventContext +import net.corda.libs.statemanager.api.Metadata import net.corda.membership.read.MembershipGroupReader import net.corda.membership.read.MembershipGroupReaderProvider import net.corda.messaging.api.records.Record +import net.corda.schema.configuration.FlowConfig.SESSION_TIMEOUT_WINDOW import net.corda.session.manager.SessionManager import net.corda.virtualnode.HoldingIdentity import net.corda.virtualnode.toCorda @@ -41,6 +44,7 @@ import org.mockito.kotlin.never import org.mockito.kotlin.times import org.mockito.kotlin.verify import org.mockito.kotlin.whenever +import java.time.Duration import java.time.Instant class FlowGlobalPostProcessorImplTest { @@ -352,4 +356,62 @@ class FlowGlobalPostProcessorImplTest { verify(sessionManager, times(1)).errorSession(any()) verify(checkpoint, times(0)).putSessionState(any()) } + + @Test + fun `when open session exists session timeout is set in metadata`() { + val earliestInstant = Instant.now().minusSeconds(20) + sessionState1.apply { + lastReceivedMessageTime = earliestInstant + status = SessionStateType.CONFIRMED + } + sessionState2.apply { + lastReceivedMessageTime = earliestInstant.plusSeconds(4) + status = SessionStateType.CONFIRMED + } + whenever(checkpoint.sessions).thenReturn(listOf(sessionState1, sessionState2)) + val output = flowGlobalPostProcessor.postProcess(testContext) + val window = Duration.ofMillis(testContext.flowConfig.getLong(SESSION_TIMEOUT_WINDOW)) + val expectedExpiry = (earliestInstant + window).epochSecond + assertThat(output.metadata).containsEntry(STATE_META_SESSION_EXPIRY_KEY, expectedExpiry) + } + + @Test + fun `when no open session exists and metadata previously had expiry key it is removed`() { + val earliestInstant = Instant.now().minusSeconds(20) + sessionState1.apply { + lastReceivedMessageTime = earliestInstant + status = SessionStateType.CLOSED + } + sessionState2.apply { + lastReceivedMessageTime = earliestInstant.plusSeconds(4) + status = SessionStateType.CLOSED + } + whenever(checkpoint.sessions).thenReturn(listOf(sessionState1, sessionState2)) + val context = testContext.copy( + metadata = Metadata(mapOf(STATE_META_SESSION_EXPIRY_KEY to earliestInstant.epochSecond)) + ) + val output = flowGlobalPostProcessor.postProcess(context) + assertThat(output.metadata).doesNotContainKey(STATE_META_SESSION_EXPIRY_KEY) + } + + @Test + fun `when open session exists previous metadata key is overwritten`() { + val earliestInstant = Instant.now().minusSeconds(20) + sessionState1.apply { + lastReceivedMessageTime = earliestInstant + status = SessionStateType.CONFIRMED + } + sessionState2.apply { + lastReceivedMessageTime = earliestInstant.plusSeconds(4) + status = SessionStateType.CONFIRMED + } + whenever(checkpoint.sessions).thenReturn(listOf(sessionState1, sessionState2)) + val context = testContext.copy( + metadata = Metadata(mapOf(STATE_META_SESSION_EXPIRY_KEY to earliestInstant.epochSecond)) + ) + val output = flowGlobalPostProcessor.postProcess(context) + val window = Duration.ofMillis(testContext.flowConfig.getLong(SESSION_TIMEOUT_WINDOW)) + val expectedExpiry = (earliestInstant + window).epochSecond + assertThat(output.metadata).containsEntry(STATE_META_SESSION_EXPIRY_KEY, expectedExpiry) + } } diff --git a/components/flow/flow-service/src/test/kotlin/net/corda/flow/test/utils/FlowEventContextHelper.kt b/components/flow/flow-service/src/test/kotlin/net/corda/flow/test/utils/FlowEventContextHelper.kt index 67a78a23ec6..b032336bd16 100644 --- a/components/flow/flow-service/src/test/kotlin/net/corda/flow/test/utils/FlowEventContextHelper.kt +++ b/components/flow/flow-service/src/test/kotlin/net/corda/flow/test/utils/FlowEventContextHelper.kt @@ -30,6 +30,7 @@ fun buildFlowEventContext( .withValue(FlowConfig.SESSION_FLOW_CLEANUP_TIME, ConfigValueFactory.fromAnyRef(10000)) .withValue(FlowConfig.PROCESSING_FLOW_CLEANUP_TIME, ConfigValueFactory.fromAnyRef(10000)) .withValue(FlowConfig.EXTERNAL_EVENT_MESSAGE_RESEND_WINDOW, ConfigValueFactory.fromAnyRef(100)) + .withValue(FlowConfig.SESSION_TIMEOUT_WINDOW, ConfigValueFactory.fromAnyRef(5000)) ) return FlowEventContext( From 08daa23bce769116008a654b0e8afe2e618c46fe Mon Sep 17 00:00:00 2001 From: Yiftach Kaplan <67583323+yift-r3@users.noreply.github.com> Date: Tue, 17 Oct 2023 15:27:45 +0100 Subject: [PATCH 45/81] CORE-17863: Close db connection pool after getOwnedKeyRecord (#4904) --- .../crypto/softhsm/impl/SoftCryptoService.kt | 7 +++--- .../impl/SoftCryptoServiceGeneralTests.kt | 22 +++++++++++++++++++ 2 files changed, 25 insertions(+), 4 deletions(-) diff --git a/components/crypto/crypto-softhsm-impl/src/main/kotlin/net/corda/crypto/softhsm/impl/SoftCryptoService.kt b/components/crypto/crypto-softhsm-impl/src/main/kotlin/net/corda/crypto/softhsm/impl/SoftCryptoService.kt index 1880e2fe0ae..b177b084585 100644 --- a/components/crypto/crypto-softhsm-impl/src/main/kotlin/net/corda/crypto/softhsm/impl/SoftCryptoService.kt +++ b/components/crypto/crypto-softhsm-impl/src/main/kotlin/net/corda/crypto/softhsm/impl/SoftCryptoService.kt @@ -542,10 +542,9 @@ open class SoftCryptoService( val keyId = ShortHash.of(requestedFullKeyId) val cacheKey = ShortHashCacheKey(tenantId, keyId) val signingKeyInfo = shortHashCache.getIfPresent(cacheKey) ?: run { - val repo = signingRepositoryFactory.getInstance(tenantId) - val result = repo.findKey(publicKey) - if (result == null) throw IllegalArgumentException("The public key '${publicKey.publicKeyId()}' was not found") - result + signingRepositoryFactory.getInstance(tenantId).use { repo -> + repo.findKey(publicKey) + } ?: throw IllegalArgumentException("The public key '${publicKey.publicKeyId()}' was not found") } return OwnedKeyRecord(publicKey, signingKeyInfo) diff --git a/components/crypto/crypto-softhsm-impl/src/test/kotlin/net/corda/crypto/softhsm/impl/SoftCryptoServiceGeneralTests.kt b/components/crypto/crypto-softhsm-impl/src/test/kotlin/net/corda/crypto/softhsm/impl/SoftCryptoServiceGeneralTests.kt index dc656bc2038..2674a9f7dd5 100644 --- a/components/crypto/crypto-softhsm-impl/src/test/kotlin/net/corda/crypto/softhsm/impl/SoftCryptoServiceGeneralTests.kt +++ b/components/crypto/crypto-softhsm-impl/src/test/kotlin/net/corda/crypto/softhsm/impl/SoftCryptoServiceGeneralTests.kt @@ -435,6 +435,28 @@ class SoftCryptoServiceGeneralTests { assertThat(exception.message).contains("was not found") } + @Test + fun `Should close the repo after use`() { + val repo = mock { + on { findKey(any()) } doReturn null + } + val cryptoService = makeSoftCryptoService(signingRepository = repo) + val publicKey = mock { + on { encoded } doReturn UUID.randomUUID().toString().toByteArray() + } + assertThrows { + cryptoService.sign( + tenantId = tenantId, + publicKey = publicKey, + signatureSpec = SignatureSpecImpl("NONE"), + data = ByteArray(2), + context = emptyMap() + ) + } + + verify(repo).close() + } + private fun mockDigestService() = mock { on { hash(any(), any()) } doReturn SecureHashUtils.randomSecureHash() } From 40516d87972ca19a05e410e456a1d7788c6f55fa Mon Sep 17 00:00:00 2001 From: Viktor Kolomeyko Date: Tue, 17 Oct 2023 17:08:16 +0100 Subject: [PATCH 46/81] CORE-17867: Some extra logging for External Messaging (#4906) During vNode creation and upgrade. --- .../handlers/CreateVirtualNodeOperationHandler.kt | 2 ++ .../handlers/VirtualNodeUpgradeOperationHandler.kt | 2 ++ 2 files changed, 4 insertions(+) diff --git a/components/virtual-node/virtual-node-write-service-impl/src/main/kotlin/net/corda/virtualnode/write/db/impl/writer/asyncoperation/handlers/CreateVirtualNodeOperationHandler.kt b/components/virtual-node/virtual-node-write-service-impl/src/main/kotlin/net/corda/virtualnode/write/db/impl/writer/asyncoperation/handlers/CreateVirtualNodeOperationHandler.kt index e8f3ff84083..f26aa453f49 100644 --- a/components/virtual-node/virtual-node-write-service-impl/src/main/kotlin/net/corda/virtualnode/write/db/impl/writer/asyncoperation/handlers/CreateVirtualNodeOperationHandler.kt +++ b/components/virtual-node/virtual-node-write-service-impl/src/main/kotlin/net/corda/virtualnode/write/db/impl/writer/asyncoperation/handlers/CreateVirtualNodeOperationHandler.kt @@ -95,6 +95,8 @@ internal class CreateVirtualNodeOperationHandler( cpiMetadata.cpiId, cpiMetadata.cpksMetadata ) + + logger.info("Generated new ExternalMessagingRouteConfig as: $externalMessagingRouteConfig") val vNodeConnections = execLog.measureExecTime("persist holding ID and virtual node") { createVirtualNodeService.persistHoldingIdAndVirtualNode( diff --git a/components/virtual-node/virtual-node-write-service-impl/src/main/kotlin/net/corda/virtualnode/write/db/impl/writer/asyncoperation/handlers/VirtualNodeUpgradeOperationHandler.kt b/components/virtual-node/virtual-node-write-service-impl/src/main/kotlin/net/corda/virtualnode/write/db/impl/writer/asyncoperation/handlers/VirtualNodeUpgradeOperationHandler.kt index 7553379ced8..cb00a7be136 100644 --- a/components/virtual-node/virtual-node-write-service-impl/src/main/kotlin/net/corda/virtualnode/write/db/impl/writer/asyncoperation/handlers/VirtualNodeUpgradeOperationHandler.kt +++ b/components/virtual-node/virtual-node-write-service-impl/src/main/kotlin/net/corda/virtualnode/write/db/impl/writer/asyncoperation/handlers/VirtualNodeUpgradeOperationHandler.kt @@ -293,6 +293,8 @@ internal class VirtualNodeUpgradeOperationHandler( targetCpi.cpksMetadata ) + logger.info("Generated upgraded ExternalMessagingRouteConfig as: $externalMessagingRouteConfig") + upgradeVirtualNodeEntity(em, request, requestId, requestTimestamp, targetCpi, externalMessagingRouteConfig) } return Triple(upgradedVNodeInfo, cpkChangelogs, targetCpi) From 034c4fa6e8f67d829da5a753f683904abb63403b Mon Sep 17 00:00:00 2001 From: James Higgs <45565019+JamesHR3@users.noreply.github.com> Date: Wed, 18 Oct 2023 11:06:51 +0100 Subject: [PATCH 47/81] CORE-17882: Add a route for flow events (#4915) Adds a route for flow events to the flow worker event mediator. --- .../flow/messaging/mediator/FlowEventMediatorFactoryImpl.kt | 1 + .../net/corda/flow/service/NewConfigurationReceived.kt | 6 ------ 2 files changed, 1 insertion(+), 6 deletions(-) delete mode 100644 components/flow/flow-service/src/main/kotlin/net/corda/flow/service/NewConfigurationReceived.kt diff --git a/components/flow/flow-service/src/main/kotlin/net/corda/flow/messaging/mediator/FlowEventMediatorFactoryImpl.kt b/components/flow/flow-service/src/main/kotlin/net/corda/flow/messaging/mediator/FlowEventMediatorFactoryImpl.kt index 525c46985d4..6a8e4ff3eb7 100644 --- a/components/flow/flow-service/src/main/kotlin/net/corda/flow/messaging/mediator/FlowEventMediatorFactoryImpl.kt +++ b/components/flow/flow-service/src/main/kotlin/net/corda/flow/messaging/mediator/FlowEventMediatorFactoryImpl.kt @@ -111,6 +111,7 @@ class FlowEventMediatorFactoryImpl @Activate constructor( is TokenPoolCacheEvent -> routeTo(messageBusClient, TOKEN_CACHE_EVENT) is TransactionVerificationRequest -> routeTo(messageBusClient, VERIFICATION_LEDGER_PROCESSOR_TOPIC) is UniquenessCheckRequestAvro -> routeTo(messageBusClient, UNIQUENESS_CHECK_TOPIC) + is FlowEvent -> routeTo(messageBusClient, FLOW_EVENT_TOPIC) else -> { val eventType = event?.let { it::class.java } throw IllegalStateException("No route defined for event type [$eventType]") diff --git a/components/flow/flow-service/src/main/kotlin/net/corda/flow/service/NewConfigurationReceived.kt b/components/flow/flow-service/src/main/kotlin/net/corda/flow/service/NewConfigurationReceived.kt deleted file mode 100644 index dfc8ee21a9c..00000000000 --- a/components/flow/flow-service/src/main/kotlin/net/corda/flow/service/NewConfigurationReceived.kt +++ /dev/null @@ -1,6 +0,0 @@ -package net.corda.flow.service - -import net.corda.libs.configuration.SmartConfig -import net.corda.lifecycle.LifecycleEvent - -data class NewConfigurationReceived(val config: SmartConfig) : LifecycleEvent From ffc0f34e448ce0ea65236fdc63ec0e6be5e43a79 Mon Sep 17 00:00:00 2001 From: Dries Samyn Date: Wed, 18 Oct 2023 15:18:40 +0100 Subject: [PATCH 48/81] Disable test. --- .../workers/smoketest/flow/ConfigurationChangeTest.kt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/applications/workers/workers-smoketest/src/smokeTest/kotlin/net/corda/applications/workers/smoketest/flow/ConfigurationChangeTest.kt b/applications/workers/workers-smoketest/src/smokeTest/kotlin/net/corda/applications/workers/smoketest/flow/ConfigurationChangeTest.kt index 0497ba972d6..115dee37bcf 100644 --- a/applications/workers/workers-smoketest/src/smokeTest/kotlin/net/corda/applications/workers/smoketest/flow/ConfigurationChangeTest.kt +++ b/applications/workers/workers-smoketest/src/smokeTest/kotlin/net/corda/applications/workers/smoketest/flow/ConfigurationChangeTest.kt @@ -20,6 +20,7 @@ import net.corda.schema.configuration.ConfigKeys.MESSAGING_CONFIG import net.corda.schema.configuration.MessagingConfig.MAX_ALLOWED_MSG_SIZE import org.assertj.core.api.Assertions.assertThat import org.junit.jupiter.api.BeforeAll +import org.junit.jupiter.api.Disabled import org.junit.jupiter.api.Order import org.junit.jupiter.api.Test import org.junit.jupiter.api.TestInstance @@ -32,6 +33,7 @@ import java.util.UUID // their patterns are DOWN - CORE-8015 @Order(Int.MAX_VALUE) @TestInstance(Lifecycle.PER_CLASS) +@Disabled("Temporarily disabled, will be re-enabled as part of PR #4899") class ConfigurationChangeTest { companion object { From 0d868f2705e5c064d2d9a1cd74eb7e113af90a30 Mon Sep 17 00:00:00 2001 From: Ben Millar <44114751+ben-millar@users.noreply.github.com> Date: Tue, 17 Oct 2023 15:48:37 +0100 Subject: [PATCH 49/81] CORE-16181 Implementing RPC client, routing external events through RPC (#4885) This PR adds a new implementation of MessagingClient, RPCClient, handling synchronous calls to the Corda workers via RPC/HTTP with configurable retries. Includes also a small bug-fix for the task manager. --------- Co-authored-by: Miljenko Brkic <97448832+mbrkic-r3@users.noreply.github.com> --- charts/corda-lib/templates/_worker.tpl | 2 +- .../mediator/FlowEventMediatorFactoryImpl.kt | 45 ++-- .../FlowEventMediatorFactoryImplTest.kt | 3 + .../net/corda/messaging/mediator/RPCClient.kt | 112 ++++++++++ .../messaging/mediator/TaskManagerHelper.kt | 3 +- .../MessagingClientFactoryFactoryImpl.kt | 12 +- .../mediator/factory/RPCClientFactory.kt | 29 +++ .../corda/messaging/utils/HTTPRetryConfig.kt | 34 +++ .../messaging/utils/HTTPRetryExecutor.kt | 43 ++++ .../corda/messaging/mediator/RPCClientTest.kt | 203 ++++++++++++++++++ .../mediator/TaskManagerHelperTest.kt | 2 +- .../MessagingClientFactoryFactoryTest.kt | 14 +- .../mediator/factory/RPCClientFactoryTest.kt | 29 +++ .../messaging/utils/HTTPRetryExecutorTest.kt | 77 +++++++ .../api/exception/CordaRestAPIExceptions.kt | 12 ++ .../factory/MessagingClientFactoryFactory.kt | 11 +- 16 files changed, 611 insertions(+), 20 deletions(-) create mode 100644 libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/RPCClient.kt create mode 100644 libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/factory/RPCClientFactory.kt create mode 100644 libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/utils/HTTPRetryConfig.kt create mode 100644 libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/utils/HTTPRetryExecutor.kt create mode 100644 libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/RPCClientTest.kt create mode 100644 libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/factory/RPCClientFactoryTest.kt create mode 100644 libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/utils/HTTPRetryExecutorTest.kt diff --git a/charts/corda-lib/templates/_worker.tpl b/charts/corda-lib/templates/_worker.tpl index e9f348f9943..97fe731c992 100644 --- a/charts/corda-lib/templates/_worker.tpl +++ b/charts/corda-lib/templates/_worker.tpl @@ -84,7 +84,7 @@ metadata: spec: type: ClusterIP selector: - app: {{ $workerName }} + app.kubernetes.io/component: {{ include "corda.workerComponent" $worker }} ports: - protocol: TCP port: {{ include "corda.workerServicePort" . }} diff --git a/components/flow/flow-service/src/main/kotlin/net/corda/flow/messaging/mediator/FlowEventMediatorFactoryImpl.kt b/components/flow/flow-service/src/main/kotlin/net/corda/flow/messaging/mediator/FlowEventMediatorFactoryImpl.kt index 6a8e4ff3eb7..f211e3e6021 100644 --- a/components/flow/flow-service/src/main/kotlin/net/corda/flow/messaging/mediator/FlowEventMediatorFactoryImpl.kt +++ b/components/flow/flow-service/src/main/kotlin/net/corda/flow/messaging/mediator/FlowEventMediatorFactoryImpl.kt @@ -14,7 +14,13 @@ import net.corda.flow.pipeline.factory.FlowEventProcessorFactory import net.corda.ledger.utxo.verification.TransactionVerificationRequest import net.corda.libs.configuration.SmartConfig import net.corda.libs.configuration.helper.getConfig +import net.corda.libs.platform.PlatformInfoProvider import net.corda.libs.statemanager.api.StateManager +import net.corda.messaging.api.constants.WorkerRPCPaths.CRYPTO_PATH +import net.corda.messaging.api.constants.WorkerRPCPaths.LEDGER_PATH +import net.corda.messaging.api.constants.WorkerRPCPaths.PERSISTENCE_PATH +import net.corda.messaging.api.constants.WorkerRPCPaths.UNIQUENESS_PATH +import net.corda.messaging.api.constants.WorkerRPCPaths.VERIFICATION_PATH import net.corda.messaging.api.mediator.MediatorMessage import net.corda.messaging.api.mediator.MessageRouter import net.corda.messaging.api.mediator.RoutingDestination.Companion.routeTo @@ -24,21 +30,21 @@ import net.corda.messaging.api.mediator.factory.MessageRouterFactory import net.corda.messaging.api.mediator.factory.MessagingClientFactoryFactory import net.corda.messaging.api.mediator.factory.MultiSourceEventMediatorFactory import net.corda.messaging.api.processor.StateAndEventProcessor -import net.corda.schema.Schemas.Crypto.FLOW_OPS_MESSAGE_TOPIC import net.corda.schema.Schemas.Flow.FLOW_EVENT_TOPIC import net.corda.schema.Schemas.Flow.FLOW_MAPPER_EVENT_TOPIC import net.corda.schema.Schemas.Flow.FLOW_STATUS_TOPIC -import net.corda.schema.Schemas.Persistence.PERSISTENCE_ENTITY_PROCESSOR_TOPIC -import net.corda.schema.Schemas.Persistence.PERSISTENCE_LEDGER_PROCESSOR_TOPIC import net.corda.schema.Schemas.Services.TOKEN_CACHE_EVENT -import net.corda.schema.Schemas.UniquenessChecker.UNIQUENESS_CHECK_TOPIC -import net.corda.schema.Schemas.Verification.VERIFICATION_LEDGER_PROCESSOR_TOPIC +import net.corda.schema.configuration.BootConfig.CRYPTO_WORKER_REST_ENDPOINT +import net.corda.schema.configuration.BootConfig.PERSISTENCE_WORKER_REST_ENDPOINT +import net.corda.schema.configuration.BootConfig.UNIQUENESS_WORKER_REST_ENDPOINT +import net.corda.schema.configuration.BootConfig.VERIFICATION_WORKER_REST_ENDPOINT import net.corda.schema.configuration.ConfigKeys import net.corda.schema.configuration.FlowConfig import org.osgi.service.component.annotations.Activate import org.osgi.service.component.annotations.Component import org.osgi.service.component.annotations.Reference +@Suppress("LongParameterList") @Component(service = [FlowEventMediatorFactory::class]) class FlowEventMediatorFactoryImpl @Activate constructor( @Reference(service = FlowEventProcessorFactory::class) @@ -50,11 +56,14 @@ class FlowEventMediatorFactoryImpl @Activate constructor( @Reference(service = MultiSourceEventMediatorFactory::class) private val eventMediatorFactory: MultiSourceEventMediatorFactory, @Reference(service = CordaAvroSerializationFactory::class) - cordaAvroSerializationFactory: CordaAvroSerializationFactory, + private val cordaAvroSerializationFactory: CordaAvroSerializationFactory, + @Reference(service = PlatformInfoProvider::class) + val platformInfoProvider: PlatformInfoProvider, ) : FlowEventMediatorFactory { companion object { private const val CONSUMER_GROUP = "FlowEventConsumer" private const val MESSAGE_BUS_CLIENT = "MessageBusClient" + private const val RPC_CLIENT = "RpcClient" } private val deserializer = cordaAvroSerializationFactory.createAvroDeserializer({}, Any::class.java) @@ -89,28 +98,36 @@ class FlowEventMediatorFactoryImpl @Activate constructor( messagingClientFactoryFactory.createMessageBusClientFactory( MESSAGE_BUS_CLIENT, messagingConfig ), + messagingClientFactoryFactory.createRPCClientFactory( + RPC_CLIENT + ) ) .messageProcessor(messageProcessor) - .messageRouterFactory(createMessageRouterFactory()) + .messageRouterFactory(createMessageRouterFactory(messagingConfig)) .threads(configs.getConfig(ConfigKeys.FLOW_CONFIG).getInt(FlowConfig.PROCESSING_THREAD_POOL_SIZE)) .threadName("flow-event-mediator") .stateManager(stateManager) .build() - private fun createMessageRouterFactory() = MessageRouterFactory { clientFinder -> + private fun createMessageRouterFactory(messagingConfig: SmartConfig) = MessageRouterFactory { clientFinder -> val messageBusClient = clientFinder.find(MESSAGE_BUS_CLIENT) + val rpcClient = clientFinder.find(RPC_CLIENT) + + fun rpcEndpoint(endpoint: String, path: String) : String { + val platformVersion = platformInfoProvider.localWorkerSoftwareShortVersion + return "http://${messagingConfig.getString(endpoint)}/api/${platformVersion}$path" + } MessageRouter { message -> when (val event = message.event()) { - // TODO Route external events to RPC client after CORE-16181 is done - is EntityRequest -> routeTo(messageBusClient, PERSISTENCE_ENTITY_PROCESSOR_TOPIC) + is EntityRequest -> routeTo(rpcClient, rpcEndpoint(PERSISTENCE_WORKER_REST_ENDPOINT, PERSISTENCE_PATH)) is FlowMapperEvent -> routeTo(messageBusClient, FLOW_MAPPER_EVENT_TOPIC) - is FlowOpsRequest -> routeTo(messageBusClient, FLOW_OPS_MESSAGE_TOPIC) + is FlowOpsRequest -> routeTo(rpcClient, rpcEndpoint(CRYPTO_WORKER_REST_ENDPOINT, CRYPTO_PATH)) is FlowStatus -> routeTo(messageBusClient, FLOW_STATUS_TOPIC) - is LedgerPersistenceRequest -> routeTo(messageBusClient, PERSISTENCE_LEDGER_PROCESSOR_TOPIC) + is LedgerPersistenceRequest -> routeTo(rpcClient, rpcEndpoint(PERSISTENCE_WORKER_REST_ENDPOINT, LEDGER_PATH)) is TokenPoolCacheEvent -> routeTo(messageBusClient, TOKEN_CACHE_EVENT) - is TransactionVerificationRequest -> routeTo(messageBusClient, VERIFICATION_LEDGER_PROCESSOR_TOPIC) - is UniquenessCheckRequestAvro -> routeTo(messageBusClient, UNIQUENESS_CHECK_TOPIC) + is TransactionVerificationRequest -> routeTo(rpcClient, rpcEndpoint(VERIFICATION_WORKER_REST_ENDPOINT, VERIFICATION_PATH)) + is UniquenessCheckRequestAvro -> routeTo(rpcClient, rpcEndpoint(UNIQUENESS_WORKER_REST_ENDPOINT, UNIQUENESS_PATH)) is FlowEvent -> routeTo(messageBusClient, FLOW_EVENT_TOPIC) else -> { val eventType = event?.let { it::class.java } diff --git a/components/flow/flow-service/src/test/kotlin/net/corda/flow/messaging/FlowEventMediatorFactoryImplTest.kt b/components/flow/flow-service/src/test/kotlin/net/corda/flow/messaging/FlowEventMediatorFactoryImplTest.kt index a1002bbe873..cae5a568394 100644 --- a/components/flow/flow-service/src/test/kotlin/net/corda/flow/messaging/FlowEventMediatorFactoryImplTest.kt +++ b/components/flow/flow-service/src/test/kotlin/net/corda/flow/messaging/FlowEventMediatorFactoryImplTest.kt @@ -7,6 +7,7 @@ import net.corda.flow.messaging.mediator.FlowEventMediatorFactory import net.corda.flow.messaging.mediator.FlowEventMediatorFactoryImpl import net.corda.flow.pipeline.factory.FlowEventProcessorFactory import net.corda.libs.configuration.SmartConfig +import net.corda.libs.platform.PlatformInfoProvider import net.corda.messaging.api.mediator.config.EventMediatorConfig import net.corda.messaging.api.mediator.factory.MediatorConsumerFactoryFactory import net.corda.messaging.api.mediator.factory.MessagingClientFactoryFactory @@ -27,6 +28,7 @@ class FlowEventMediatorFactoryImplTest { private val messagingClientFactoryFactory = mock() private val multiSourceEventMediatorFactory = mock() private val cordaAvroSerializationFactory = mock() + private val platformInfoProvider = mock() private val flowConfig = mock() @BeforeEach @@ -45,6 +47,7 @@ class FlowEventMediatorFactoryImplTest { messagingClientFactoryFactory, multiSourceEventMediatorFactory, cordaAvroSerializationFactory, + platformInfoProvider ) } diff --git a/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/RPCClient.kt b/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/RPCClient.kt new file mode 100644 index 00000000000..eaed1ed3b00 --- /dev/null +++ b/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/RPCClient.kt @@ -0,0 +1,112 @@ +package net.corda.messaging.mediator + +import java.io.IOException +import java.net.URI +import java.net.http.HttpClient +import java.net.http.HttpRequest +import java.net.http.HttpResponse +import java.util.concurrent.TimeoutException +import net.corda.avro.serialization.CordaAvroSerializationFactory +import net.corda.messaging.api.exception.CordaHTTPClientErrorException +import net.corda.messaging.api.exception.CordaHTTPServerErrorException +import net.corda.messaging.api.mediator.MediatorMessage +import net.corda.messaging.api.mediator.MessagingClient +import net.corda.messaging.api.mediator.MessagingClient.Companion.MSG_PROP_ENDPOINT +import net.corda.messaging.utils.HTTPRetryConfig +import net.corda.messaging.utils.HTTPRetryExecutor +import net.corda.utilities.trace +import org.slf4j.Logger +import org.slf4j.LoggerFactory + +class RPCClient( + override val id: String, + cordaAvroSerializerFactory: CordaAvroSerializationFactory, + private val onSerializationError: ((ByteArray) -> Unit)?, + private val httpClient: HttpClient, + private val retryConfig: HTTPRetryConfig = + HTTPRetryConfig.Builder() + .retryOn(IOException::class.java, TimeoutException::class.java) + .build() +) : MessagingClient { + private val deserializer = cordaAvroSerializerFactory.createAvroDeserializer({}, Any::class.java) + + private companion object { + private val log: Logger = LoggerFactory.getLogger(this::class.java.enclosingClass) + } + + override fun send(message: MediatorMessage<*>): MediatorMessage<*>? { + return try { + log.trace { "Received RPC external event send request for endpoint ${message.endpoint()}" } + processMessage(message) + } catch (e: Exception) { + handleExceptions(e) + null + } + } + + private fun processMessage(message: MediatorMessage<*>): MediatorMessage<*> { + val request = buildHttpRequest(message) + val response = sendWithRetry(request) + + checkResponseStatus(response.statusCode()) + + val deserializedResponse = deserializePayload(response.body()) + return MediatorMessage(deserializedResponse, mutableMapOf("statusCode" to response.statusCode())) + } + + + private fun deserializePayload(payload: ByteArray): Any { + return try { + deserializer.deserialize(payload)!! + } catch (e: Exception) { + val errorMsg = "Failed to deserialize payload of size ${payload.size} bytes due to: ${e.message}" + log.warn(errorMsg, e) + onSerializationError?.invoke(errorMsg.toByteArray()) + throw e + } + } + + private fun buildHttpRequest(message: MediatorMessage<*>): HttpRequest { + return HttpRequest.newBuilder() + .uri(URI(message.endpoint())) + .POST(HttpRequest.BodyPublishers.ofByteArray(message.payload as ByteArray)) + .build() + } + + private fun sendWithRetry(request: HttpRequest): HttpResponse { + return HTTPRetryExecutor.withConfig(retryConfig) { + httpClient.send(request, HttpResponse.BodyHandlers.ofByteArray()) + } + } + + private fun checkResponseStatus(statusCode: Int) { + log.trace("Received response with status code $statusCode") + when (statusCode) { + in 400..499 -> throw CordaHTTPClientErrorException(statusCode, "Server returned status code $statusCode.") + in 500..599 -> throw CordaHTTPServerErrorException(statusCode, "Server returned status code $statusCode.") + } + } + + private fun handleExceptions(e: Exception) { + when (e) { + is IOException -> log.warn("Network or IO operation error in RPCClient: ", e) + is InterruptedException -> log.warn("Operation was interrupted in RPCClient: ", e) + is IllegalArgumentException -> log.warn("Invalid argument provided in RPCClient call: ", e) + is SecurityException -> log.warn("Security violation detected in RPCClient: ", e) + is IllegalStateException -> log.warn("Coroutine state error in RPCClient: ", e) + is CordaHTTPClientErrorException -> log.warn("Client-side HTTP error in RPCClient: ", e) + is CordaHTTPServerErrorException -> log.warn("Server-side HTTP error in RPCClient: ", e) + else -> log.warn("Unhandled exception in RPCClient: ", e) + } + + throw e + } + + override fun close() { + // Nothing to do here + } + + private fun MediatorMessage<*>.endpoint(): String { + return getProperty(MSG_PROP_ENDPOINT) + } +} diff --git a/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/TaskManagerHelper.kt b/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/TaskManagerHelper.kt index ec9a14e73fa..284ced5dcf6 100644 --- a/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/TaskManagerHelper.kt +++ b/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/TaskManagerHelper.kt @@ -64,8 +64,9 @@ internal class TaskManagerHelper( val groupedEvents = clientTaskResults.map { it.toRecord() } with(clientTaskResults.first()) { val persistedState = processorTaskResult.updatedState!! + val incrementVersion = if (processorTaskResult.processorTask.persistedState == null) 0 else 1 processorTask.copy( - persistedState = persistedState.copy(version = persistedState.version + 1), + persistedState = persistedState.copy(version = persistedState.version + incrementVersion), events = groupedEvents ) } diff --git a/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/factory/MessagingClientFactoryFactoryImpl.kt b/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/factory/MessagingClientFactoryFactoryImpl.kt index 540e01d7240..0eecb634e7c 100644 --- a/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/factory/MessagingClientFactoryFactoryImpl.kt +++ b/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/factory/MessagingClientFactoryFactoryImpl.kt @@ -1,5 +1,6 @@ package net.corda.messaging.mediator.factory +import net.corda.avro.serialization.CordaAvroSerializationFactory import net.corda.libs.configuration.SmartConfig import net.corda.messagebus.api.producer.builder.CordaProducerBuilder import net.corda.messaging.api.mediator.factory.MessagingClientFactoryFactory @@ -14,6 +15,8 @@ import org.osgi.service.component.annotations.Reference class MessagingClientFactoryFactoryImpl @Activate constructor( @Reference(service = CordaProducerBuilder::class) private val cordaProducerBuilder: CordaProducerBuilder, + @Reference(service = CordaAvroSerializationFactory::class) + private val cordaSerializationFactory: CordaAvroSerializationFactory ): MessagingClientFactoryFactory { override fun createMessageBusClientFactory( id: String, @@ -23,4 +26,11 @@ class MessagingClientFactoryFactoryImpl @Activate constructor( messageBusConfig, cordaProducerBuilder, ) -} \ No newline at end of file + + override fun createRPCClientFactory( + id: String + ) = RPCClientFactory( + id, + cordaSerializationFactory + ) +} diff --git a/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/factory/RPCClientFactory.kt b/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/factory/RPCClientFactory.kt new file mode 100644 index 00000000000..b0c80e87dd6 --- /dev/null +++ b/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/factory/RPCClientFactory.kt @@ -0,0 +1,29 @@ +package net.corda.messaging.mediator.factory + +import java.net.http.HttpClient +import java.time.Duration +import net.corda.avro.serialization.CordaAvroSerializationFactory +import net.corda.messaging.api.mediator.MessagingClient +import net.corda.messaging.api.mediator.config.MessagingClientConfig +import net.corda.messaging.api.mediator.factory.MessagingClientFactory +import net.corda.messaging.mediator.RPCClient + +class RPCClientFactory( + private val id: String, + private val cordaSerializationFactory: CordaAvroSerializationFactory +): MessagingClientFactory { + private val httpClient: HttpClient by lazy { + HttpClient.newBuilder() + .connectTimeout(Duration.ofSeconds(30)) + .build() + } + + override fun create(config: MessagingClientConfig): MessagingClient { + return RPCClient( + id, + cordaSerializationFactory, + config.onSerializationError, + httpClient + ) + } +} diff --git a/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/utils/HTTPRetryConfig.kt b/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/utils/HTTPRetryConfig.kt new file mode 100644 index 00000000000..f09edf86226 --- /dev/null +++ b/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/utils/HTTPRetryConfig.kt @@ -0,0 +1,34 @@ +package net.corda.messaging.utils + +/** + * Configuration class for HTTP retry parameters. + * + * @property times The number of a times a retry will be attempted. Default is 3 + * @property initialDelay The initial delay (in milliseconds) before the first retry. Default is 100ms + * @property factor The multiplier used to increase the delay for each subsequent retry. Default is 2.0 + * @property retryOn A set of exception classes that should trigger a retry when caught. + * If an exception not in this list is caught, it will be propagated immediately without retrying. + * Default is the generic [Exception] class, meaning all exceptions will trigger a retry. + */ +data class HTTPRetryConfig( + val times: Int = 3, + val initialDelay: Long = 100, + val factor: Double = 2.0, + val retryOn: Set> = setOf(Exception::class.java) +) { + class Builder { + private var times: Int = 3 + private var initialDelay: Long = 100 + private var factor: Double = 2.0 + private var retryOn: Set> = setOf(Exception::class.java) + + fun times(times: Int) = apply { this.times = times } + fun initialDelay(delay: Long) = apply { this.initialDelay = delay } + fun factor(factor: Double) = apply { this.factor = factor } + fun retryOn(vararg exceptions: Class) = apply { this.retryOn = exceptions.toSet() } + + fun build(): HTTPRetryConfig { + return HTTPRetryConfig(times, initialDelay, factor, retryOn) + } + } +} diff --git a/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/utils/HTTPRetryExecutor.kt b/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/utils/HTTPRetryExecutor.kt new file mode 100644 index 00000000000..1ec6b6b6bc5 --- /dev/null +++ b/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/utils/HTTPRetryExecutor.kt @@ -0,0 +1,43 @@ +package net.corda.messaging.utils + +import net.corda.utilities.trace +import org.slf4j.Logger +import org.slf4j.LoggerFactory + +class HTTPRetryExecutor { + companion object { + private val log: Logger = LoggerFactory.getLogger(this::class.java.enclosingClass) + + fun withConfig(config: HTTPRetryConfig, block: () -> T): T { + var currentDelay = config.initialDelay + for (i in 0 until config.times - 1) { + try { + log.trace { "HTTPRetryExecutor making attempt #${i + 1}." } + val result = block() + log.trace { "Operation successful after #${i + 1} attempt/s." } + return result + } catch (e: Exception) { + if (config.retryOn.none { it.isInstance(e) }) { + log.warn("HTTPRetryExecutor caught a non-retryable exception: ${e.message}", e) + throw e + } + + log.trace { "Attempt #${i + 1} failed due to ${e.message}. Retrying in $currentDelay ms..." } + Thread.sleep(currentDelay) + currentDelay = (currentDelay * config.factor).toLong() + } + } + + log.trace("All retry attempts exhausted. Making the final call.") + + try { + val result = block() + log.trace { "Operation successful after #${config.times} attempt/s." } + return result + } catch (e: Exception) { + log.trace { "Operation failed after ${config.times} attempt/s." } + throw e + } + } + } +} diff --git a/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/RPCClientTest.kt b/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/RPCClientTest.kt new file mode 100644 index 00000000000..e85dd13667f --- /dev/null +++ b/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/RPCClientTest.kt @@ -0,0 +1,203 @@ +package net.corda.messaging.mediator + +import java.io.IOException +import java.net.http.HttpClient +import java.net.http.HttpRequest +import java.net.http.HttpResponse +import net.corda.avro.serialization.CordaAvroDeserializer +import net.corda.avro.serialization.CordaAvroSerializationFactory +import net.corda.avro.serialization.CordaAvroSerializer +import net.corda.data.flow.event.FlowEvent +import net.corda.messaging.api.exception.CordaHTTPClientErrorException +import net.corda.messaging.api.exception.CordaHTTPServerErrorException +import net.corda.messaging.api.mediator.MediatorMessage +import net.corda.messaging.api.mediator.MessagingClient.Companion.MSG_PROP_ENDPOINT +import net.corda.messaging.api.records.Record +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertNotNull +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.assertThrows +import org.mockito.Mockito.times +import org.mockito.kotlin.any +import org.mockito.kotlin.eq +import org.mockito.kotlin.mock +import org.mockito.kotlin.verify +import org.mockito.kotlin.whenever + +class RPCClientTest { + + private lateinit var client: RPCClient + private val payload = "testPayload".toByteArray() + private val message = MediatorMessage( + payload, + mutableMapOf(MSG_PROP_ENDPOINT to "http://test-endpoint/api/5.1/test") + ) + + data class Mocks( + val serializer: CordaAvroSerializer, + val deserializer: CordaAvroDeserializer, + val httpClient: HttpClient, + val httpResponse: HttpResponse + ) + + private inner class MockEnvironment( + val mockSerializer: CordaAvroSerializer = mock(), + val mockDeserializer: CordaAvroDeserializer = mock(), + val mockHttpClient: HttpClient = mock(), + val mockHttpResponse: HttpResponse = mock() + ) { + init { + whenever(mockSerializer.serialize(any>())) + .thenReturn("testPayload".toByteArray()) + + whenever(mockDeserializer.deserialize(any())) + .thenReturn(FlowEvent()) + + whenever(mockHttpResponse.statusCode()) + .thenReturn(200) + + whenever(mockHttpResponse.body()) + .thenReturn("responsePayload".toByteArray()) + + whenever(mockHttpClient.send(any(), any>())) + .thenReturn(mockHttpResponse) + } + + fun withHttpStatus(status: Int) = apply { + whenever(mockHttpResponse.statusCode()).thenReturn(status) + } + + val mocks: Mocks + get() = Mocks(mockSerializer, mockDeserializer, mockHttpClient, mockHttpResponse) + } + + + private fun createClient( + mocks: Mocks, + onSerializationError: (ByteArray) -> Unit = mock(), + ): RPCClient { + val mockSerializationFactory: CordaAvroSerializationFactory = mock() + + whenever(mockSerializationFactory.createAvroSerializer(any())) + .thenReturn(mocks.serializer) + + whenever(mockSerializationFactory.createAvroDeserializer(any(), eq(Any::class.java))) + .thenReturn(mocks.deserializer) + + return RPCClient( + "TestRPCClient1", + mockSerializationFactory, + onSerializationError, + mocks.httpClient + ) + } + + @BeforeEach + fun setup() { + val environment = MockEnvironment() + client = createClient(environment.mocks) + } + + @Test + fun `send() processes message and returns result`() { + val result = client.send(message) + assertNotNull(result?.payload) + assertEquals( + FlowEvent(), + result!!.payload + ) + } + + @Test + fun `send() handles 4XX error`() { + val environment = MockEnvironment() + .withHttpStatus(404) + + val client = createClient(environment.mocks) + + assertThrows { + client.send(message) + } + } + + @Test + fun `send() handles 5XX error`() { + val environment = MockEnvironment() + .withHttpStatus(500) + + val client = createClient(environment.mocks) + + assertThrows { + client.send(message) + } + } + + @Test + fun `send() handles deserialization error`() { + val onSerializationError: (ByteArray) -> Unit = mock() + + val environment = MockEnvironment().apply { + whenever(mockDeserializer.deserialize(any())) + .thenThrow(IllegalArgumentException("Deserialization error")) + } + + val client = createClient(environment.mocks, onSerializationError) + + assertThrows { + client.send(message) + } + + verify(onSerializationError).invoke(any()) + } + + @Test + fun `send retries on IOException and eventually succeeds`() { + val environment = MockEnvironment().apply { + whenever(mockHttpClient.send(any(), any>())) + .thenThrow(IOException("Simulated IO exception")) + .thenThrow(IOException("Simulated IO exception")) + .thenReturn(mockHttpResponse) + } + + val client = createClient(environment.mocks) + val result = client.send(message) + + assertNotNull(result?.payload) + assertEquals( + FlowEvent(), + result!!.payload + ) + } + + @Test + fun `send fails after exhausting all retries`() { + val environment = MockEnvironment().apply { + whenever(mockHttpClient.send(any(), any>())) + .thenThrow(IOException("Simulated IO exception")) + } + + val client = createClient(environment.mocks) + + assertThrows { + client.send(message) + } + } + + @Test + fun `send retries the correct number of times before failing`() { + val environment = MockEnvironment().apply { + whenever(mockHttpClient.send(any(), any>())) + .thenThrow(IOException("Simulated IO exception")) + } + + val client = createClient(environment.mocks) + + assertThrows { + client.send(message) + } + + verify(environment.mockHttpClient, times(3)) + .send(any(), any>()) + } +} diff --git a/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/TaskManagerHelperTest.kt b/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/TaskManagerHelperTest.kt index b0b768fd9cd..837a8905d72 100644 --- a/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/TaskManagerHelperTest.kt +++ b/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/TaskManagerHelperTest.kt @@ -110,7 +110,7 @@ class TaskManagerHelperTest { val expectedProcessorTasks = listOf( ProcessorTask( KEY2, - updateState.copy(version = updateState.version + 1), + updateState.copy(version = updateState.version), listOf(replyMessage.payload!!).toRecords(KEY2), messageProcessor, stateManagerHelper diff --git a/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/factory/MessagingClientFactoryFactoryTest.kt b/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/factory/MessagingClientFactoryFactoryTest.kt index e4a68753e9e..569c80cf319 100644 --- a/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/factory/MessagingClientFactoryFactoryTest.kt +++ b/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/factory/MessagingClientFactoryFactoryTest.kt @@ -1,5 +1,6 @@ package net.corda.messaging.mediator.factory +import net.corda.avro.serialization.CordaAvroSerializationFactory import net.corda.libs.configuration.SmartConfig import net.corda.messagebus.api.producer.builder.CordaProducerBuilder import org.junit.jupiter.api.Assertions @@ -10,12 +11,14 @@ import org.mockito.kotlin.mock class MessagingClientFactoryFactoryTest { private lateinit var messagingClientFactoryFactory: MessagingClientFactoryFactoryImpl private val cordaProducerBuilder = mock() + private val cordaAvroSerializationFactory = mock() private val messageBusConfig = mock() @BeforeEach fun beforeEach() { messagingClientFactoryFactory = MessagingClientFactoryFactoryImpl( cordaProducerBuilder, + cordaAvroSerializationFactory ) } @@ -28,4 +31,13 @@ class MessagingClientFactoryFactoryTest { Assertions.assertNotNull(messageBusClientFactory) } -} \ No newline at end of file + + @Test + fun testCreateRPCClientFactory() { + val rpcClientFactory = messagingClientFactoryFactory.createRPCClientFactory( + "rpcClient1" + ) + + Assertions.assertNotNull(rpcClientFactory) + } +} diff --git a/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/factory/RPCClientFactoryTest.kt b/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/factory/RPCClientFactoryTest.kt new file mode 100644 index 00000000000..2ae62f487e1 --- /dev/null +++ b/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/factory/RPCClientFactoryTest.kt @@ -0,0 +1,29 @@ +package net.corda.messaging.mediator.factory + +import net.corda.avro.serialization.CordaAvroSerializationFactory +import net.corda.messaging.api.mediator.config.MessagingClientConfig +import org.junit.jupiter.api.Assertions +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import org.mockito.Mockito.mock + +class RPCClientFactoryTest { + private lateinit var cordaSerializationFactory: CordaAvroSerializationFactory + private lateinit var rpcClientFactory: RPCClientFactory + + @BeforeEach + fun beforeEach() { + cordaSerializationFactory = mock(CordaAvroSerializationFactory::class.java) + rpcClientFactory = RPCClientFactory( + "RPCClient1", + mock(CordaAvroSerializationFactory::class.java) + ) + } + + @Test + fun testCreateRPCClient() { + val config = MessagingClientConfig {} + val rpcClient = rpcClientFactory.create(config) + Assertions.assertNotNull(rpcClient) + } +} diff --git a/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/utils/HTTPRetryExecutorTest.kt b/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/utils/HTTPRetryExecutorTest.kt new file mode 100644 index 00000000000..e1245ebdc83 --- /dev/null +++ b/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/utils/HTTPRetryExecutorTest.kt @@ -0,0 +1,77 @@ +package net.corda.messaging.utils + +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.assertThrows + +class HTTPRetryExecutorTest { + private lateinit var retryConfig: HTTPRetryConfig + + @BeforeEach + fun setUp() { + retryConfig = HTTPRetryConfig.Builder() + .times(3) + .initialDelay(100) + .factor(2.0) + .retryOn(RuntimeException::class.java) + .build() + } + + @Test + fun `successfully returns after first attempt`() { + val result = HTTPRetryExecutor.withConfig(retryConfig) { + "Success" + } + + assertEquals("Success", result) + } + + @Suppress("TooGenericExceptionThrown") + @Test + fun `should retry until successful`() { + var attempt = 0 + + val result = HTTPRetryExecutor.withConfig(retryConfig) { + ++attempt + if (attempt < 3) { + throw RuntimeException("Failed on attempt $attempt") + } + "Success on attempt $attempt" + } + + assertEquals("Success on attempt 3", result) + } + + @Suppress("TooGenericExceptionThrown") + @Test + fun `should throw exception after max attempts`() { + var attempt = 0 + + assertThrows { + HTTPRetryExecutor.withConfig(retryConfig) { + ++attempt + throw RuntimeException("Failed on attempt $attempt") + } + } + } + + @Suppress("TooGenericExceptionThrown") + @Test + fun `should not retry on non-retryable exception`() { + val config = HTTPRetryConfig.Builder() + .times(3) + .initialDelay(100) + .factor(2.0) + .retryOn(SpecificException::class.java) + .build() + + assertThrows { + HTTPRetryExecutor.withConfig(config) { + throw RuntimeException("I'm not retryable!") + } + } + } + + internal class SpecificException(message: String) : Exception(message) +} \ No newline at end of file diff --git a/libs/messaging/messaging/src/main/kotlin/net/corda/messaging/api/exception/CordaRestAPIExceptions.kt b/libs/messaging/messaging/src/main/kotlin/net/corda/messaging/api/exception/CordaRestAPIExceptions.kt index b173417557e..a369ff75971 100644 --- a/libs/messaging/messaging/src/main/kotlin/net/corda/messaging/api/exception/CordaRestAPIExceptions.kt +++ b/libs/messaging/messaging/src/main/kotlin/net/corda/messaging/api/exception/CordaRestAPIExceptions.kt @@ -19,3 +19,15 @@ class CordaRPCAPIPartitionException(message: String?, exception: Throwable? = nu */ class CordaRPCAPIResponderException(val errorType: String, message: String?, exception: Throwable? = null) : CordaRuntimeException(message, exception) + +/** + * Exception representing a 4XX response from the HTTP server + */ +class CordaHTTPClientErrorException(val statusCode: Int, message: String?, exception: Throwable? = null) : + CordaRuntimeException(message, exception) + +/** + * Exception representing a 5XX response from the HTTP server + */ +class CordaHTTPServerErrorException(val statusCode: Int, message: String?, exception: Throwable? = null) : + CordaRuntimeException(message, exception) diff --git a/libs/messaging/messaging/src/main/kotlin/net/corda/messaging/api/mediator/factory/MessagingClientFactoryFactory.kt b/libs/messaging/messaging/src/main/kotlin/net/corda/messaging/api/mediator/factory/MessagingClientFactoryFactory.kt index 9e69b6e31ea..3c4c7951847 100644 --- a/libs/messaging/messaging/src/main/kotlin/net/corda/messaging/api/mediator/factory/MessagingClientFactoryFactory.kt +++ b/libs/messaging/messaging/src/main/kotlin/net/corda/messaging/api/mediator/factory/MessagingClientFactoryFactory.kt @@ -16,4 +16,13 @@ interface MessagingClientFactoryFactory { id: String, messageBusConfig: SmartConfig, ) : MessagingClientFactory -} \ No newline at end of file + + /** + * Creates an RPC messaging client factory. + * + * @param id RPC client ID. + */ + fun createRPCClientFactory( + id: String + ) : MessagingClientFactory +} From 82df6961a3092a5cfc711de0b5921ab8e2dd3673 Mon Sep 17 00:00:00 2001 From: James Higgs <45565019+JamesHR3@users.noreply.github.com> Date: Wed, 18 Oct 2023 08:35:52 +0100 Subject: [PATCH 50/81] CORE-17626: Add integration test for flow mapper cleanup (#4907) Adds an integration test for mapper cleanup. Also addresses a small bug where the key for the cleanup event was not serialized correctly. --- .../FlowMapperServiceIntegrationTest.kt | 45 +++++++++++++++++++ .../TestStateManagerFactoryImpl.kt | 23 ++++++++-- .../executor/ScheduledTaskProcessor.kt | 2 +- 3 files changed, 66 insertions(+), 4 deletions(-) diff --git a/components/flow/flow-mapper-service/src/integrationTest/kotlin/net/corda/session/mapper/service/integration/FlowMapperServiceIntegrationTest.kt b/components/flow/flow-mapper-service/src/integrationTest/kotlin/net/corda/session/mapper/service/integration/FlowMapperServiceIntegrationTest.kt index 5324858d6fb..d923c331885 100644 --- a/components/flow/flow-mapper-service/src/integrationTest/kotlin/net/corda/session/mapper/service/integration/FlowMapperServiceIntegrationTest.kt +++ b/components/flow/flow-mapper-service/src/integrationTest/kotlin/net/corda/session/mapper/service/integration/FlowMapperServiceIntegrationTest.kt @@ -18,11 +18,16 @@ import net.corda.data.flow.event.session.SessionCounterpartyInfoRequest import net.corda.data.flow.event.session.SessionData import net.corda.data.flow.event.session.SessionError import net.corda.data.flow.event.session.SessionInit +import net.corda.data.flow.state.mapper.FlowMapperStateType import net.corda.data.identity.HoldingIdentity +import net.corda.data.scheduler.ScheduledTaskTrigger import net.corda.db.messagebus.testkit.DBSetup import net.corda.flow.utils.emptyKeyValuePairList import net.corda.libs.configuration.SmartConfigFactory import net.corda.libs.configuration.SmartConfigImpl +import net.corda.libs.statemanager.api.Metadata +import net.corda.libs.statemanager.api.State +import net.corda.libs.statemanager.api.StateManagerFactory import net.corda.membership.locally.hosted.identities.IdentityInfo import net.corda.membership.locally.hosted.identities.LocallyHostedIdentitiesService import net.corda.messaging.api.publisher.Publisher @@ -35,6 +40,7 @@ import net.corda.schema.Schemas.Config.CONFIG_TOPIC import net.corda.schema.Schemas.Flow.FLOW_EVENT_TOPIC import net.corda.schema.Schemas.Flow.FLOW_MAPPER_EVENT_TOPIC import net.corda.schema.Schemas.P2P.P2P_OUT_TOPIC +import net.corda.schema.Schemas.ScheduledTask import net.corda.schema.configuration.BootConfig.BOOT_MAX_ALLOWED_MSG_SIZE import net.corda.schema.configuration.BootConfig.INSTANCE_ID import net.corda.schema.configuration.BootConfig.TOPIC_PREFIX @@ -44,7 +50,9 @@ import net.corda.schema.configuration.ConfigKeys.STATE_MANAGER_CONFIG import net.corda.schema.configuration.MessagingConfig.Bus.BUS_TYPE import net.corda.schema.configuration.MessagingConfig.MAX_ALLOWED_MSG_SIZE import net.corda.session.mapper.service.FlowMapperService +import net.corda.session.mapper.service.state.StateMetadataKeys import net.corda.test.flow.util.buildSessionEvent +import net.corda.test.util.eventually import net.corda.virtualnode.toCorda import org.assertj.core.api.Assertions.assertThat import org.junit.jupiter.api.Assertions.assertFalse @@ -58,6 +66,7 @@ import org.osgi.test.junit5.service.ServiceExtension import java.lang.System.currentTimeMillis import java.nio.ByteBuffer import java.security.KeyPairGenerator +import java.time.Duration import java.time.Instant import java.util.concurrent.CountDownLatch import java.util.concurrent.TimeUnit @@ -90,6 +99,9 @@ class FlowMapperServiceIntegrationTest { @InjectService(timeout = 4000) lateinit var locallyHostedIdentityService: LocallyHostedIdentitiesService + @InjectService(timeout = 4000) + lateinit var stateManagerFactory: StateManagerFactory + private val messagingConfig = SmartConfigImpl.empty() .withValue(INSTANCE_ID, ConfigValueFactory.fromAnyRef(1)) .withValue(TOPIC_PREFIX, ConfigValueFactory.fromAnyRef("")) @@ -404,6 +416,39 @@ class FlowMapperServiceIntegrationTest { assertThat(event.payload).isInstanceOf(SessionError::class.java) } + @Test + fun `mapper state cleanup correctly cleans up old states`() { + + // Create a state in the state manager. Note the modified time has to be further in the past than the configured + // flow processing time. + val stateKey = "foo" + val config = SmartConfigImpl.empty() + val stateManager = stateManagerFactory.create(config) + stateManager.create(listOf( + State( + stateKey, + byteArrayOf(), + metadata = Metadata(mapOf(StateMetadataKeys.FLOW_MAPPER_STATUS to FlowMapperStateType.CLOSING.toString())), + modifiedTime = Instant.now().minusSeconds(20) + ) + )) + + // Publish a scheduled task trigger. + val testId = "test6" + val publisher = publisherFactory.createPublisher(PublisherConfig(testId), messagingConfig) + publisher.publish(listOf( + Record( + ScheduledTask.SCHEDULED_TASK_TOPIC_MAPPER_PROCESSOR, + "foo", + ScheduledTaskTrigger(ScheduledTask.SCHEDULED_TASK_NAME_MAPPER_CLEANUP, Instant.now())) + )) + + eventually(duration = Duration.ofMinutes(1)) { + val states = stateManager.get(listOf(stateKey)) + assertThat(states[stateKey]).isNull() + } + } + private fun setupConfig(publisher: Publisher) { val bootConfig = smartConfigFactory.create(ConfigFactory.parseString(bootConf)) publishConfig(publisher) diff --git a/components/flow/flow-mapper-service/src/integrationTest/kotlin/net/corda/session/mapper/service/integration/TestStateManagerFactoryImpl.kt b/components/flow/flow-mapper-service/src/integrationTest/kotlin/net/corda/session/mapper/service/integration/TestStateManagerFactoryImpl.kt index c65b184a118..c392f4debe9 100644 --- a/components/flow/flow-mapper-service/src/integrationTest/kotlin/net/corda/session/mapper/service/integration/TestStateManagerFactoryImpl.kt +++ b/components/flow/flow-mapper-service/src/integrationTest/kotlin/net/corda/session/mapper/service/integration/TestStateManagerFactoryImpl.kt @@ -19,9 +19,10 @@ import java.util.concurrent.ConcurrentHashMap @Component class TestStateManagerFactoryImpl : StateManagerFactory { + private val storage = ConcurrentHashMap() + override fun create(config: SmartConfig): StateManager { return object : StateManager { - private val storage = ConcurrentHashMap() override fun close() { } @@ -51,7 +52,18 @@ class TestStateManagerFactoryImpl : StateManagerFactory { } override fun delete(states: Collection): Map { - TODO("Not yet implemented") + return states.mapNotNull { + var output: State? = null + storage.compute(it.key) { _, existingState -> + if (existingState?.version == it.version) { + null + } else { + output = it + existingState + } + } + output + }.associateBy { it.key } } override fun updatedBetween(interval: IntervalFilter): Map { @@ -66,11 +78,16 @@ class TestStateManagerFactoryImpl : StateManagerFactory { TODO("Not yet implemented") } + // Only supporting equals for now. override fun findUpdatedBetweenWithMetadataFilter( intervalFilter: IntervalFilter, metadataFilter: MetadataFilter ): Map { - TODO("Not yet implemented") + return storage.filter { (_, state) -> + state.modifiedTime >= intervalFilter.start && state.modifiedTime <= intervalFilter.finish + }.filter { (_, state) -> + state.metadata.containsKey(metadataFilter.key) && state.metadata[metadataFilter.key] == metadataFilter.value + } } } } diff --git a/components/flow/flow-mapper-service/src/main/kotlin/net/corda/session/mapper/service/executor/ScheduledTaskProcessor.kt b/components/flow/flow-mapper-service/src/main/kotlin/net/corda/session/mapper/service/executor/ScheduledTaskProcessor.kt index d753487a9f0..3778b460bb9 100644 --- a/components/flow/flow-mapper-service/src/main/kotlin/net/corda/session/mapper/service/executor/ScheduledTaskProcessor.kt +++ b/components/flow/flow-mapper-service/src/main/kotlin/net/corda/session/mapper/service/executor/ScheduledTaskProcessor.kt @@ -33,7 +33,7 @@ class ScheduledTaskProcessor( override fun onNext(events: List>): List> { return if (events.any { it.value?.name == Schemas.ScheduledTask.SCHEDULED_TASK_NAME_MAPPER_CLEANUP }) { process().map { - Record(Schemas.Flow.FLOW_MAPPER_CLEANUP_TOPIC, UUID.randomUUID(), it) + Record(Schemas.Flow.FLOW_MAPPER_CLEANUP_TOPIC, UUID.randomUUID().toString(), it) } } else { listOf() From aaf7973af228a52e6ccbd1f2107ef096bdd230b1 Mon Sep 17 00:00:00 2001 From: Jacob Scott Date: Wed, 18 Oct 2023 17:10:23 +0100 Subject: [PATCH 51/81] Create Remove Stale Branches GitHub Action (#4909) --- .github/workflows/remove-stale-branches.yml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 .github/workflows/remove-stale-branches.yml diff --git a/.github/workflows/remove-stale-branches.yml b/.github/workflows/remove-stale-branches.yml new file mode 100644 index 00000000000..25dd1b9f8c4 --- /dev/null +++ b/.github/workflows/remove-stale-branches.yml @@ -0,0 +1,19 @@ +name: 'Remove stale branches' +on: + schedule: + - cron: '0 0 * * 1-5' + +jobs: + remove-stale-branches: + name: Remove stale branches + runs-on: ubuntu-latest + steps: + - uses: fpicalausa/remove-stale-branches@v1.5.8 + with: + dry-run: true + days-before-branch-stale: 30 + days-before-branch-delete: 14 + stale-branch-message: "@{author} The branch [{branchName}]({branchUrl}) hasn't been updated in the last 30 days and is marked as stale. It will be removed in 14 days.\r\nIf you want to keep this branch around, delete this comment or add new commits to this branch." + exempt-protected-branches: true + exempt-branches-regex: "^(release\\/|feature\\/|poc\\/).*" + operations-per-run: 30 From bed356e1bba230c47c2a76e8c3824bbe852f64c4 Mon Sep 17 00:00:00 2001 From: Miljenko Brkic <97448832+mbrkic-r3@users.noreply.github.com> Date: Wed, 18 Oct 2023 20:31:39 +0100 Subject: [PATCH 52/81] CORE-17843 Flaky test FlowMapperServiceIntegrationTest.testStartRPCDuplicatesAndCleanup() (#4899) Fixed stopping event mediator. FlowMapperServiceIntegrationTest modified to work with event mediator for flow event processing. --- .../smoketest/flow/ConfigurationChangeTest.kt | 2 - .../FlowMapperServiceIntegrationTest.kt | 94 +++++++++---------- .../TestFlowEventMediatorFactory.kt | 14 +++ .../TestFlowEventMediatorFactoryImpl.kt | 92 ++++++++++++++++++ .../TestStateManagerFactoryImpl.kt | 5 +- .../mediator/MultiSourceEventMediatorImpl.kt | 29 ++++-- 6 files changed, 179 insertions(+), 57 deletions(-) create mode 100644 components/flow/flow-mapper-service/src/integrationTest/kotlin/net/corda/session/mapper/service/integration/TestFlowEventMediatorFactory.kt create mode 100644 components/flow/flow-mapper-service/src/integrationTest/kotlin/net/corda/session/mapper/service/integration/TestFlowEventMediatorFactoryImpl.kt diff --git a/applications/workers/workers-smoketest/src/smokeTest/kotlin/net/corda/applications/workers/smoketest/flow/ConfigurationChangeTest.kt b/applications/workers/workers-smoketest/src/smokeTest/kotlin/net/corda/applications/workers/smoketest/flow/ConfigurationChangeTest.kt index 115dee37bcf..0497ba972d6 100644 --- a/applications/workers/workers-smoketest/src/smokeTest/kotlin/net/corda/applications/workers/smoketest/flow/ConfigurationChangeTest.kt +++ b/applications/workers/workers-smoketest/src/smokeTest/kotlin/net/corda/applications/workers/smoketest/flow/ConfigurationChangeTest.kt @@ -20,7 +20,6 @@ import net.corda.schema.configuration.ConfigKeys.MESSAGING_CONFIG import net.corda.schema.configuration.MessagingConfig.MAX_ALLOWED_MSG_SIZE import org.assertj.core.api.Assertions.assertThat import org.junit.jupiter.api.BeforeAll -import org.junit.jupiter.api.Disabled import org.junit.jupiter.api.Order import org.junit.jupiter.api.Test import org.junit.jupiter.api.TestInstance @@ -33,7 +32,6 @@ import java.util.UUID // their patterns are DOWN - CORE-8015 @Order(Int.MAX_VALUE) @TestInstance(Lifecycle.PER_CLASS) -@Disabled("Temporarily disabled, will be re-enabled as part of PR #4899") class ConfigurationChangeTest { companion object { diff --git a/components/flow/flow-mapper-service/src/integrationTest/kotlin/net/corda/session/mapper/service/integration/FlowMapperServiceIntegrationTest.kt b/components/flow/flow-mapper-service/src/integrationTest/kotlin/net/corda/session/mapper/service/integration/FlowMapperServiceIntegrationTest.kt index d923c331885..1cea67f0b38 100644 --- a/components/flow/flow-mapper-service/src/integrationTest/kotlin/net/corda/session/mapper/service/integration/FlowMapperServiceIntegrationTest.kt +++ b/components/flow/flow-mapper-service/src/integrationTest/kotlin/net/corda/session/mapper/service/integration/FlowMapperServiceIntegrationTest.kt @@ -13,7 +13,6 @@ import net.corda.data.flow.event.SessionEvent import net.corda.data.flow.event.StartFlow import net.corda.data.flow.event.mapper.ExecuteCleanup import net.corda.data.flow.event.mapper.FlowMapperEvent -import net.corda.data.flow.event.mapper.ScheduleCleanup import net.corda.data.flow.event.session.SessionCounterpartyInfoRequest import net.corda.data.flow.event.session.SessionData import net.corda.data.flow.event.session.SessionError @@ -37,7 +36,7 @@ import net.corda.messaging.api.records.Record import net.corda.messaging.api.subscription.config.SubscriptionConfig import net.corda.messaging.api.subscription.factory.SubscriptionFactory import net.corda.schema.Schemas.Config.CONFIG_TOPIC -import net.corda.schema.Schemas.Flow.FLOW_EVENT_TOPIC +import net.corda.schema.Schemas.Flow.FLOW_MAPPER_CLEANUP_TOPIC import net.corda.schema.Schemas.Flow.FLOW_MAPPER_EVENT_TOPIC import net.corda.schema.Schemas.P2P.P2P_OUT_TOPIC import net.corda.schema.Schemas.ScheduledTask @@ -63,7 +62,6 @@ import org.junit.jupiter.api.TestInstance import org.junit.jupiter.api.extension.ExtendWith import org.osgi.test.common.annotation.InjectService import org.osgi.test.junit5.service.ServiceExtension -import java.lang.System.currentTimeMillis import java.nio.ByteBuffer import java.security.KeyPairGenerator import java.time.Duration @@ -90,6 +88,9 @@ class FlowMapperServiceIntegrationTest { @InjectService(timeout = 4000) lateinit var subscriptionFactory: SubscriptionFactory + @InjectService(timeout = 4000) + lateinit var flowEventMediatorFactory: TestFlowEventMediatorFactory + @InjectService(timeout = 4000) lateinit var configService: ConfigurationReadService @@ -108,6 +109,8 @@ class FlowMapperServiceIntegrationTest { .withValue(BUS_TYPE, ConfigValueFactory.fromAnyRef("INMEMORY")) .withValue(MAX_ALLOWED_MSG_SIZE, ConfigValueFactory.fromAnyRef(100000000)) + private val stateManagerConfig = SmartConfigImpl.empty() + private val schemaVersion = ConfigurationSchemaVersion(1, 0) private val aliceHoldingIdentity = HoldingIdentity("CN=Alice, O=Alice Corp, L=LDN, C=GB", "group1") @@ -117,6 +120,7 @@ class FlowMapperServiceIntegrationTest { @BeforeEach fun setup() { + TestStateManagerFactoryImpl.clear() if (!setup) { setup = true val publisher = publisherFactory.createPublisher(PublisherConfig(clientId), messagingConfig) @@ -145,14 +149,17 @@ class FlowMapperServiceIntegrationTest { fun `Test first session event outbound sets up flow mapper state, verify subsequent messages received are passed to flow event topic` () { val testId = "test1" + val testSessionId = "testSession1" + val testFlowId = "testFlow1" + val testCpiId = "testCpi1" val publisher = publisherFactory.createPublisher(PublisherConfig(testId), messagingConfig) - //send 2 session init, 1 is duplicate + //send 2 session init val sessionDataAndInitEvent = Record( - FLOW_MAPPER_EVENT_TOPIC, testId, FlowMapperEvent( + FLOW_MAPPER_EVENT_TOPIC, testSessionId, FlowMapperEvent( buildSessionEvent( - MessageDirection.OUTBOUND, testId, 1, SessionData(ByteBuffer.wrap("bytes".toByteArray()), SessionInit( - testId, testId, emptyKeyValuePairList(), emptyKeyValuePairList() + MessageDirection.OUTBOUND, testSessionId, 1, SessionData(ByteBuffer.wrap("bytes".toByteArray()), SessionInit( + testCpiId, testFlowId, emptyKeyValuePairList(), emptyKeyValuePairList() )), initiatedIdentity = charlieHoldingIdentity, contextSessionProps = emptyKeyValuePairList() @@ -166,7 +173,7 @@ class FlowMapperServiceIntegrationTest { val p2pLatch = CountDownLatch(1) val p2pOutSub = subscriptionFactory.createDurableSubscription( SubscriptionConfig("$testId-p2p-out", P2P_OUT_TOPIC), - TestP2POutProcessor(testId, p2pLatch, 1), messagingConfig, null + TestP2POutProcessor(testSessionId, p2pLatch, 2), messagingConfig, null ) p2pOutSub.start() assertTrue(p2pLatch.await(20, TimeUnit.SECONDS)) @@ -174,10 +181,10 @@ class FlowMapperServiceIntegrationTest { //send data back val sessionDataEvent = Record( - FLOW_MAPPER_EVENT_TOPIC, testId, FlowMapperEvent( + FLOW_MAPPER_EVENT_TOPIC, testSessionId, FlowMapperEvent( buildSessionEvent( MessageDirection.INBOUND, - testId, + testSessionId, 2, SessionData(ByteBuffer.wrap("".toByteArray()), null), contextSessionProps = emptyKeyValuePairList() @@ -189,16 +196,15 @@ class FlowMapperServiceIntegrationTest { //validate flow event topic val flowEventLatch = CountDownLatch(1) val testProcessor = TestFlowMessageProcessor(flowEventLatch, 1, SessionEvent::class.java) - val flowEventSub = subscriptionFactory.createStateAndEventSubscription( - SubscriptionConfig("$testId-flow-event", FLOW_EVENT_TOPIC), - testProcessor, + val flowEventMediator = flowEventMediatorFactory.create( messagingConfig, - null + stateManagerConfig, + testProcessor, ) - flowEventSub.start() + flowEventMediator.start() assertTrue(flowEventLatch.await(5, TimeUnit.SECONDS)) - flowEventSub.close() + flowEventMediator.close() } @Test @@ -234,37 +240,29 @@ class FlowMapperServiceIntegrationTest { //flow event subscription to validate outputs val flowEventLatch = CountDownLatch(2) val testProcessor = TestFlowMessageProcessor(flowEventLatch, 2, StartFlow::class.java) - val flowEventSub = subscriptionFactory.createStateAndEventSubscription( - SubscriptionConfig("$testId-flow-event", FLOW_EVENT_TOPIC), - testProcessor, + val flowEventMediator = flowEventMediatorFactory.create( messagingConfig, - null + stateManagerConfig, + testProcessor, ) - flowEventSub.start() + flowEventMediator.start() - //cleanup - val cleanup = Record( - FLOW_MAPPER_EVENT_TOPIC, testId, FlowMapperEvent( - ScheduleCleanup(currentTimeMillis()) - ) - ) - publisher.publish(listOf(cleanup)) - - //assert duplicate start rpc didn't get processed (and also give Execute cleanup time to run) + //assert duplicate start rpc didn't get processed assertFalse(flowEventLatch.await(3, TimeUnit.SECONDS)) assertThat(flowEventLatch.count).isEqualTo(1) // Manually publish an execute cleanup event. Temporary until the full solution has been integrated. val executeCleanup = Record( - FLOW_MAPPER_EVENT_TOPIC, + FLOW_MAPPER_CLEANUP_TOPIC, testId, - FlowMapperEvent( - ExecuteCleanup(listOf()) - ) + ExecuteCleanup(listOf(testId)) ) publisher.publish(listOf(executeCleanup)) + // give Execute cleanup time to run + assertFalse(flowEventLatch.await(3, TimeUnit.SECONDS)) + //send same key start rpc again publisher.publish(listOf(startRPCEvent)) @@ -276,7 +274,7 @@ class FlowMapperServiceIntegrationTest { ) ).withFailMessage("latch was ${flowEventLatch.count}").isTrue - flowEventSub.close() + flowEventMediator.close() } @Test @@ -312,14 +310,17 @@ class FlowMapperServiceIntegrationTest { @Test fun `flow mapper still works after config update`() { val testId = "test4" + val testSessionId = "testSession4" + val testFlowId = "testFlow4" + val testCpiId = "testCpi4" val publisher = publisherFactory.createPublisher(PublisherConfig(testId), messagingConfig) //send 2 session init, 1 is duplicate val sessionInitEvent = Record( - FLOW_MAPPER_EVENT_TOPIC, testId, FlowMapperEvent( + FLOW_MAPPER_EVENT_TOPIC, testSessionId, FlowMapperEvent( buildSessionEvent( - MessageDirection.OUTBOUND, testId, 1, SessionCounterpartyInfoRequest(SessionInit( - testId, testId, emptyKeyValuePairList(), emptyKeyValuePairList() + MessageDirection.OUTBOUND, testSessionId, 1, SessionCounterpartyInfoRequest(SessionInit( + testCpiId, testFlowId, emptyKeyValuePairList(), emptyKeyValuePairList() )), initiatedIdentity = charlieHoldingIdentity, contextSessionProps = emptyKeyValuePairList() @@ -333,7 +334,7 @@ class FlowMapperServiceIntegrationTest { val p2pLatch = CountDownLatch(1) val p2pOutSub = subscriptionFactory.createDurableSubscription( SubscriptionConfig("$testId-p2p-out", P2P_OUT_TOPIC), - TestP2POutProcessor(testId, p2pLatch, 1), messagingConfig, null + TestP2POutProcessor(testSessionId, p2pLatch, 1), messagingConfig, null ) p2pOutSub.start() assertTrue(p2pLatch.await(10, TimeUnit.SECONDS)) @@ -344,10 +345,10 @@ class FlowMapperServiceIntegrationTest { //send data back val sessionDataEvent = Record( - FLOW_MAPPER_EVENT_TOPIC, testId, FlowMapperEvent( + FLOW_MAPPER_EVENT_TOPIC, testSessionId, FlowMapperEvent( buildSessionEvent( MessageDirection.INBOUND, - testId, + testSessionId, 2, SessionData(ByteBuffer.wrap("".toByteArray()), null), initiatedIdentity = charlieHoldingIdentity, @@ -360,16 +361,15 @@ class FlowMapperServiceIntegrationTest { //validate flow event topic val flowEventLatch = CountDownLatch(1) val testProcessor = TestFlowMessageProcessor(flowEventLatch, 1, SessionEvent::class.java) - val flowEventSub = subscriptionFactory.createStateAndEventSubscription( - SubscriptionConfig("$testId-flow-event", FLOW_EVENT_TOPIC), - testProcessor, + val flowEventMediator = flowEventMediatorFactory.create( messagingConfig, - null + stateManagerConfig, + testProcessor, ) - flowEventSub.start() + flowEventMediator.start() assertTrue(flowEventLatch.await(5, TimeUnit.SECONDS)) - flowEventSub.close() + flowEventMediator.close() } diff --git a/components/flow/flow-mapper-service/src/integrationTest/kotlin/net/corda/session/mapper/service/integration/TestFlowEventMediatorFactory.kt b/components/flow/flow-mapper-service/src/integrationTest/kotlin/net/corda/session/mapper/service/integration/TestFlowEventMediatorFactory.kt new file mode 100644 index 00000000000..3ab14b4683e --- /dev/null +++ b/components/flow/flow-mapper-service/src/integrationTest/kotlin/net/corda/session/mapper/service/integration/TestFlowEventMediatorFactory.kt @@ -0,0 +1,14 @@ +package net.corda.session.mapper.service.integration + +import net.corda.data.flow.event.FlowEvent +import net.corda.data.flow.state.checkpoint.Checkpoint +import net.corda.libs.configuration.SmartConfig +import net.corda.messaging.api.mediator.MultiSourceEventMediator + +interface TestFlowEventMediatorFactory { + fun create( + messagingConfig: SmartConfig, + stateManagerConfig: SmartConfig, + flowEventProcessor: TestFlowMessageProcessor, + ): MultiSourceEventMediator +} \ No newline at end of file diff --git a/components/flow/flow-mapper-service/src/integrationTest/kotlin/net/corda/session/mapper/service/integration/TestFlowEventMediatorFactoryImpl.kt b/components/flow/flow-mapper-service/src/integrationTest/kotlin/net/corda/session/mapper/service/integration/TestFlowEventMediatorFactoryImpl.kt new file mode 100644 index 00000000000..5ef089f6d79 --- /dev/null +++ b/components/flow/flow-mapper-service/src/integrationTest/kotlin/net/corda/session/mapper/service/integration/TestFlowEventMediatorFactoryImpl.kt @@ -0,0 +1,92 @@ +package net.corda.session.mapper.service.integration + +import com.typesafe.config.ConfigValueFactory +import net.corda.data.flow.event.FlowEvent +import net.corda.data.flow.event.mapper.FlowMapperEvent +import net.corda.data.flow.state.checkpoint.Checkpoint +import net.corda.libs.configuration.SmartConfig +import net.corda.libs.statemanager.api.StateManagerFactory +import net.corda.messaging.api.mediator.MessageRouter +import net.corda.messaging.api.mediator.RoutingDestination.Companion.routeTo +import net.corda.messaging.api.mediator.config.EventMediatorConfigBuilder +import net.corda.messaging.api.mediator.factory.MediatorConsumerFactoryFactory +import net.corda.messaging.api.mediator.factory.MessageRouterFactory +import net.corda.messaging.api.mediator.factory.MessagingClientFactoryFactory +import net.corda.messaging.api.mediator.factory.MultiSourceEventMediatorFactory +import net.corda.messaging.api.processor.StateAndEventProcessor +import net.corda.schema.Schemas.Flow.FLOW_EVENT_TOPIC +import net.corda.schema.Schemas.Flow.FLOW_MAPPER_EVENT_TOPIC +import net.corda.schema.configuration.MessagingConfig +import org.osgi.service.component.annotations.Activate +import org.osgi.service.component.annotations.Component +import org.osgi.service.component.annotations.Reference +import java.util.UUID + +@Component(service = [TestFlowEventMediatorFactory::class]) +class TestFlowEventMediatorFactoryImpl @Activate constructor( + @Reference(service = MediatorConsumerFactoryFactory::class) + private val mediatorConsumerFactoryFactory: MediatorConsumerFactoryFactory, + @Reference(service = MessagingClientFactoryFactory::class) + private val messagingClientFactoryFactory: MessagingClientFactoryFactory, + @Reference(service = MultiSourceEventMediatorFactory::class) + private val eventMediatorFactory: MultiSourceEventMediatorFactory, + @Reference(service = StateManagerFactory::class) + private val stateManagerFactory: StateManagerFactory, +) : TestFlowEventMediatorFactory { + companion object { + private const val CONSUMER_GROUP = "FlowEventConsumer" + private const val MESSAGE_BUS_CLIENT = "MessageBusClient" + } + + override fun create( + messagingConfig: SmartConfig, + stateManagerConfig: SmartConfig, + flowEventProcessor: TestFlowMessageProcessor, + ) = eventMediatorFactory.create( + createEventMediatorConfig( + messagingConfig + .withValue(MessagingConfig.Subscription.POLL_TIMEOUT, ConfigValueFactory.fromAnyRef(100)) + .withValue(MessagingConfig.Subscription.PROCESSOR_RETRIES, ConfigValueFactory.fromAnyRef(1)), + flowEventProcessor, + stateManagerConfig, + ) + ) + + private fun createEventMediatorConfig( + messagingConfig: SmartConfig, + messageProcessor: StateAndEventProcessor, + stateManagerConfig: SmartConfig, + ) = EventMediatorConfigBuilder() + .name("FlowEventMediator ${UUID.randomUUID()}") + .messagingConfig(messagingConfig) + .consumerFactories( + mediatorConsumerFactoryFactory.createMessageBusConsumerFactory( + FLOW_EVENT_TOPIC, CONSUMER_GROUP, messagingConfig + ), + ) + .clientFactories( + messagingClientFactoryFactory.createMessageBusClientFactory( + MESSAGE_BUS_CLIENT, messagingConfig + ), + ) + .messageProcessor(messageProcessor) + .messageRouterFactory(createMessageRouterFactory()) + .threads(1) + .threadName("flow-event-mediator") + .stateManager(stateManagerFactory.create(stateManagerConfig)) + .build() + + private fun createMessageRouterFactory() = MessageRouterFactory { clientFinder -> + val messageBusClient = clientFinder.find(MESSAGE_BUS_CLIENT) + + MessageRouter { message -> + when (val event = message.payload) { + is FlowMapperEvent -> routeTo(messageBusClient, FLOW_MAPPER_EVENT_TOPIC) + else -> { + val eventType = event?.let { it::class.java } + throw IllegalStateException("No route defined for event type [$eventType]") + } + } + } + } +} \ No newline at end of file diff --git a/components/flow/flow-mapper-service/src/integrationTest/kotlin/net/corda/session/mapper/service/integration/TestStateManagerFactoryImpl.kt b/components/flow/flow-mapper-service/src/integrationTest/kotlin/net/corda/session/mapper/service/integration/TestStateManagerFactoryImpl.kt index c392f4debe9..0fee00fc7e3 100644 --- a/components/flow/flow-mapper-service/src/integrationTest/kotlin/net/corda/session/mapper/service/integration/TestStateManagerFactoryImpl.kt +++ b/components/flow/flow-mapper-service/src/integrationTest/kotlin/net/corda/session/mapper/service/integration/TestStateManagerFactoryImpl.kt @@ -18,8 +18,11 @@ import java.util.concurrent.ConcurrentHashMap */ @Component class TestStateManagerFactoryImpl : StateManagerFactory { + companion object { + private val storage = ConcurrentHashMap() - private val storage = ConcurrentHashMap() + fun clear() = storage.clear() + } override fun create(config: SmartConfig): StateManager { return object : StateManager { diff --git a/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/MultiSourceEventMediatorImpl.kt b/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/MultiSourceEventMediatorImpl.kt index 4fe9f9d5270..c8a7f2da992 100644 --- a/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/MultiSourceEventMediatorImpl.kt +++ b/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/MultiSourceEventMediatorImpl.kt @@ -19,7 +19,9 @@ import net.corda.messaging.mediator.metrics.EventMediatorMetrics import net.corda.taskmanager.TaskManager import net.corda.utilities.debug import org.slf4j.LoggerFactory +import java.lang.Thread.sleep import java.util.UUID +import java.util.concurrent.atomic.AtomicBoolean @Suppress("LongParameterList") class MultiSourceEventMediatorImpl( @@ -56,25 +58,33 @@ class MultiSourceEventMediatorImpl( override val subscriptionName: LifecycleCoordinatorName get() = lifecycleCoordinatorName + private val stopped = AtomicBoolean(false) + private val running = AtomicBoolean(false) + override fun start() { log.debug { "Starting multi-source event mediator with config: $config" } lifecycleCoordinator.start() taskManager.executeLongRunningTask(::run) } - private fun stop() = Thread.currentThread().interrupt() + private fun stop() = stopped.set(true) - private val stopped get() = Thread.currentThread().isInterrupted + private fun stopped() = stopped.get() override fun close() { + log.debug("Closing multi-source event mediator") stop() + while (running.get()) { + sleep(100) + } lifecycleCoordinator.close() } private fun run() { + running.set(true) var attempts = 0 - while (!stopped) { + while (!stopped()) { attempts++ try { consumers = mediatorComponentFactory.createConsumers(::onSerializationError) @@ -84,7 +94,7 @@ class MultiSourceEventMediatorImpl( consumers.forEach { it.subscribe() } lifecycleCoordinator.updateStatus(LifecycleStatus.UP) - while (!stopped) { + while (!stopped()) { processEventsWithRetries() } @@ -113,11 +123,13 @@ class MultiSourceEventMediatorImpl( closeConsumersAndProducers() } } + running.set(false) } private fun onSerializationError(event: ByteArray) { - log.debug { "Error serializing [$event] "} - TODO("Not yet implemented") + // TODO CORE-17012 Subscription error handling (DLQ) + log.warn("Failed to deserialize event") + log.debug { "Failed to deserialize event: ${event.contentToString()}" } } private fun closeConsumersAndProducers() { @@ -128,7 +140,7 @@ class MultiSourceEventMediatorImpl( private fun processEventsWithRetries() { var attempts = 0 var keepProcessing = true - while (keepProcessing && !stopped) { + while (keepProcessing && !stopped()) { try { processEvents() keepProcessing = false @@ -152,6 +164,9 @@ class MultiSourceEventMediatorImpl( private fun processEvents() { val messages = pollConsumers() + if (stopped()) { + return + } if (messages.isNotEmpty()) { val msgGroups = messages.groupBy { it.key } val persistedStates = stateManager.get(msgGroups.keys.map { it.toString() }) From cb329abdc97d1763f230dd0dc381702fc920678c Mon Sep 17 00:00:00 2001 From: James Higgs <45565019+JamesHR3@users.noreply.github.com> Date: Thu, 19 Oct 2023 09:09:29 +0100 Subject: [PATCH 53/81] CORE-17885: Use explicit conversion of session timeout to long (#4927) Adjusts the session timeout processor to use an explicit conversion to long. When going through a cycle of writing metadata to the state store and reading it out again, the original type of any numeric values is lost. This causes a problem as casts between numerics are not allowed. To address this, the underlying numeric value in the metadata is converted to `Number`, and then explicitly converted to a long. This additionally introduces a more thorough unit test for the flow event mediator factory, which ensures that routes are correct for different event types. --- .../SessionTimeoutTaskProcessor.kt | 8 ++- .../SessionTimeoutTaskProcessorTests.kt | 6 +- .../FlowEventMediatorFactoryImplTest.kt | 56 ++++++++++++++++++- 3 files changed, 62 insertions(+), 8 deletions(-) diff --git a/components/flow/flow-service/src/main/kotlin/net/corda/flow/maintenance/SessionTimeoutTaskProcessor.kt b/components/flow/flow-service/src/main/kotlin/net/corda/flow/maintenance/SessionTimeoutTaskProcessor.kt index bc6d3172233..f8bb5bf39f6 100644 --- a/components/flow/flow-service/src/main/kotlin/net/corda/flow/maintenance/SessionTimeoutTaskProcessor.kt +++ b/components/flow/flow-service/src/main/kotlin/net/corda/flow/maintenance/SessionTimeoutTaskProcessor.kt @@ -10,6 +10,7 @@ import net.corda.messaging.api.processor.DurableProcessor import net.corda.messaging.api.records.Record import net.corda.schema.Schemas.Flow.FLOW_TIMEOUT_TOPIC import net.corda.schema.Schemas.ScheduledTask +import net.corda.utilities.debug import org.slf4j.LoggerFactory import java.time.Instant @@ -39,13 +40,14 @@ class SessionTimeoutTaskProcessor( logger.trace("No flows to time out") emptyList() } else { - // TODO - take log message out when everything plumbed in. - logger.info("Trigger cleanup of $checkpoints") + logger.debug { "Trigger cleanup of $checkpoints" } checkpoints.map { kvp -> + val instant = (kvp.value.metadata[STATE_META_SESSION_EXPIRY_KEY] as Number).toLong() Record(FLOW_TIMEOUT_TOPIC, kvp.key, FlowTimeout( kvp.value.key, - Instant.ofEpochSecond(kvp.value.metadata[STATE_META_SESSION_EXPIRY_KEY] as Long)) + Instant.ofEpochSecond(instant) + ) ) } } diff --git a/components/flow/flow-service/src/test/kotlin/net/corda/flow/maintenance/SessionTimeoutTaskProcessorTests.kt b/components/flow/flow-service/src/test/kotlin/net/corda/flow/maintenance/SessionTimeoutTaskProcessorTests.kt index 9a9303f057c..97c082df4ce 100644 --- a/components/flow/flow-service/src/test/kotlin/net/corda/flow/maintenance/SessionTimeoutTaskProcessorTests.kt +++ b/components/flow/flow-service/src/test/kotlin/net/corda/flow/maintenance/SessionTimeoutTaskProcessorTests.kt @@ -26,7 +26,7 @@ class SessionTimeoutTaskProcessorTests { "foo", randomBytes(), 0, - Metadata(mapOf(STATE_META_SESSION_EXPIRY_KEY to now.minusSeconds(1).epochSecond))) + Metadata(mapOf(STATE_META_SESSION_EXPIRY_KEY to now.minusSeconds(1).epochSecond.toInt()))) private val states = mapOf( state1.key to state1, ) @@ -65,13 +65,15 @@ class SessionTimeoutTaskProcessorTests { fun `when state found return`() { val processor = SessionTimeoutTaskProcessor(stateManager) { now } val output = processor.onNext(listOf(record1)) + val outputInstant = (state1.metadata[STATE_META_SESSION_EXPIRY_KEY] as Number).toLong() assertThat(output).containsExactly( Record( Schemas.Flow.FLOW_TIMEOUT_TOPIC, state1.key, FlowTimeout( state1.key, - Instant.ofEpochSecond(state1.metadata[STATE_META_SESSION_EXPIRY_KEY] as Long)) + Instant.ofEpochSecond(outputInstant) + ) ) ) } diff --git a/components/flow/flow-service/src/test/kotlin/net/corda/flow/messaging/FlowEventMediatorFactoryImplTest.kt b/components/flow/flow-service/src/test/kotlin/net/corda/flow/messaging/FlowEventMediatorFactoryImplTest.kt index cae5a568394..321fad3ca1e 100644 --- a/components/flow/flow-service/src/test/kotlin/net/corda/flow/messaging/FlowEventMediatorFactoryImplTest.kt +++ b/components/flow/flow-service/src/test/kotlin/net/corda/flow/messaging/FlowEventMediatorFactoryImplTest.kt @@ -1,25 +1,47 @@ package net.corda.flow.messaging import net.corda.avro.serialization.CordaAvroSerializationFactory +import net.corda.data.crypto.wire.ops.flow.FlowOpsRequest import net.corda.data.flow.event.FlowEvent +import net.corda.data.flow.event.mapper.FlowMapperEvent +import net.corda.data.flow.output.FlowStatus import net.corda.data.flow.state.checkpoint.Checkpoint +import net.corda.data.ledger.persistence.LedgerPersistenceRequest +import net.corda.data.ledger.utxo.token.selection.event.TokenPoolCacheEvent +import net.corda.data.persistence.EntityRequest +import net.corda.data.uniqueness.UniquenessCheckRequestAvro import net.corda.flow.messaging.mediator.FlowEventMediatorFactory import net.corda.flow.messaging.mediator.FlowEventMediatorFactoryImpl import net.corda.flow.pipeline.factory.FlowEventProcessorFactory +import net.corda.ledger.utxo.verification.TransactionVerificationRequest import net.corda.libs.configuration.SmartConfig import net.corda.libs.platform.PlatformInfoProvider +import net.corda.messaging.api.constants.WorkerRPCPaths.CRYPTO_PATH +import net.corda.messaging.api.constants.WorkerRPCPaths.LEDGER_PATH +import net.corda.messaging.api.constants.WorkerRPCPaths.PERSISTENCE_PATH +import net.corda.messaging.api.constants.WorkerRPCPaths.UNIQUENESS_PATH +import net.corda.messaging.api.constants.WorkerRPCPaths.VERIFICATION_PATH +import net.corda.messaging.api.mediator.MediatorMessage import net.corda.messaging.api.mediator.config.EventMediatorConfig import net.corda.messaging.api.mediator.factory.MediatorConsumerFactoryFactory import net.corda.messaging.api.mediator.factory.MessagingClientFactoryFactory +import net.corda.messaging.api.mediator.factory.MessagingClientFinder import net.corda.messaging.api.mediator.factory.MultiSourceEventMediatorFactory +import net.corda.schema.Schemas.Flow.FLOW_EVENT_TOPIC +import net.corda.schema.Schemas.Flow.FLOW_MAPPER_EVENT_TOPIC +import net.corda.schema.Schemas.Flow.FLOW_STATUS_TOPIC +import net.corda.schema.Schemas.Services.TOKEN_CACHE_EVENT import net.corda.schema.configuration.ConfigKeys import net.corda.schema.configuration.FlowConfig +import org.assertj.core.api.Assertions.assertThat import org.junit.jupiter.api.Assertions.assertNotNull import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Test import org.mockito.Mockito.`when` import org.mockito.kotlin.any +import org.mockito.kotlin.argumentCaptor import org.mockito.kotlin.mock +import org.mockito.kotlin.whenever class FlowEventMediatorFactoryImplTest { private lateinit var flowEventMediatorFactory: FlowEventMediatorFactory @@ -31,12 +53,14 @@ class FlowEventMediatorFactoryImplTest { private val platformInfoProvider = mock() private val flowConfig = mock() + val captor = argumentCaptor>() + @BeforeEach fun beforeEach() { `when`(flowEventProcessorFactory.create(any())) .thenReturn(mock()) - `when`(multiSourceEventMediatorFactory.create(any>())) + `when`(multiSourceEventMediatorFactory.create(captor.capture())) .thenReturn(mock()) `when`(flowConfig.getInt(FlowConfig.PROCESSING_THREAD_POOL_SIZE)).thenReturn(10) @@ -51,14 +75,40 @@ class FlowEventMediatorFactoryImplTest { ) } + private fun endpoint(suffix: String) : String { + // As no config is supplied in these tests, the parameterized parts of the endpoint will be null. + return "http://null/api/null$suffix" + } + @Test - fun `successfully creates event mediator`() { + fun `successfully creates event mediator with expected routes`() { val mediator = flowEventMediatorFactory.create( mapOf(ConfigKeys.FLOW_CONFIG to flowConfig), mock(), mock(), ) - assertNotNull(mediator) + val clientFinder = mock().apply { + whenever(this.find(any())).thenReturn(mock()) + } + val config = captor.firstValue + val router = config.messageRouterFactory.create(clientFinder) + assertThat(router.getDestination(MediatorMessage(FlowEvent())).endpoint).isEqualTo(FLOW_EVENT_TOPIC) + assertThat(router.getDestination(MediatorMessage(FlowMapperEvent())).endpoint) + .isEqualTo(FLOW_MAPPER_EVENT_TOPIC) + assertThat(router.getDestination(MediatorMessage(EntityRequest())).endpoint) + .isEqualTo(endpoint(PERSISTENCE_PATH)) + assertThat(router.getDestination(MediatorMessage(FlowOpsRequest())).endpoint) + .isEqualTo(endpoint(CRYPTO_PATH)) + assertThat(router.getDestination(MediatorMessage(FlowStatus())).endpoint).isEqualTo(FLOW_STATUS_TOPIC) + assertThat(router.getDestination(MediatorMessage(LedgerPersistenceRequest())).endpoint) + .isEqualTo(endpoint(LEDGER_PATH)) + assertThat(router.getDestination(MediatorMessage(TokenPoolCacheEvent())).endpoint).isEqualTo(TOKEN_CACHE_EVENT) + assertThat(router.getDestination(MediatorMessage(TransactionVerificationRequest())).endpoint).isEqualTo( + endpoint(VERIFICATION_PATH) + ) + assertThat(router.getDestination(MediatorMessage(UniquenessCheckRequestAvro())).endpoint).isEqualTo( + endpoint(UNIQUENESS_PATH) + ) } } \ No newline at end of file From 81316bb4ed57591e796498751de23f12fe0e5276 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Waldemar=20=C5=BBurowski?= <45210402+wzur-r3@users.noreply.github.com> Date: Fri, 20 Oct 2023 09:57:30 +0100 Subject: [PATCH 54/81] ES-1518: switch to KubernetesAgent class for Jenkins agent definition (#4939) * use KubernetesAgent to handle creation of `label`, `nodeSelector` and `defaultContainer` in more consistent way across many projects --- .../Jenkinsfile_PublishHelmChart | 25 +++++++++++-------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/.ci/PublishHelmChart/Jenkinsfile_PublishHelmChart b/.ci/PublishHelmChart/Jenkinsfile_PublishHelmChart index 8f62b3c5d84..71e387d7bc0 100644 --- a/.ci/PublishHelmChart/Jenkinsfile_PublishHelmChart +++ b/.ci/PublishHelmChart/Jenkinsfile_PublishHelmChart @@ -1,29 +1,34 @@ #! groovy @Library('corda-shared-build-pipeline-steps@5.1') _ +import com.r3.build.agents.KubernetesAgent import com.r3.build.enums.BuildEnvironment +import com.r3.build.enums.KubernetesCluster import com.r3.build.utils.PublishingUtils -int cpus = 1 -BuildEnvironment buildEnvironment = BuildEnvironment.AMD64_LINUX_JAVA17 +/** + * Jenkins Kubernetes agent + */ +KubernetesAgent k8s = new KubernetesAgent( + BuildEnvironment.AMD64_LINUX_JAVA17, + KubernetesCluster.JenkinsAgents, + 1 +) PublishingUtils publishingUtils = new PublishingUtils(this) pipeline { agent { kubernetes { - cloud "eks-e2e" - yaml kubernetesBuildAgentYaml('build', buildEnvironment, cpus) + cloud k8s.buildCluster.cloudName + yaml k8s.JSON yamlMergeStrategy merge() // important to keep tolerations from the inherited template idleMinutes 15 podRetention always() - label ([ - "gradle-build", - "${cpus}cpus", - "${buildEnvironment.jenkinsLabel}" - ].join('-')) + nodeSelector k8s.nodeSelector + label k8s.jenkinsLabel showRawYaml false - defaultContainer 'build' + defaultContainer k8s.defaultContainer.name } } From ba16ab860327b221548bc686910e74ae2fcaed82 Mon Sep 17 00:00:00 2001 From: Dan Newton Date: Fri, 20 Oct 2023 16:11:59 +0100 Subject: [PATCH 55/81] CORE-17837 Add `UtxoSignedLedgerTransactionKryoSerializer` (#4945) Add a Kryo serializer for `UtxoSignedLedgerTransaction`. This fixes a bug where `UtxoSignedLedgerTransaction` was being serialized with the Kryo serializer for `UtxoSignedTransaction`, due to it implementing the same interface; however, the serializer can only correct serialize `UtxoSignedTransaction`s. Adding a specific serializer means Kryo picks up the correct one to serialize and deserialize with. --- .../tests/UtxoFilteredTransactionTest.kt | 4 +-- ...oLedgerTransactionKryoSerializationTest.kt | 20 +++++++++++ ...dLedgerTransactionKryoSerializationTest.kt | 23 +++++++++++++ ...oSignedTransactionKryoSerializationTest.kt | 11 ++++--- .../UtxoSignedLedgerTransaction.kt | 6 ++++ .../UtxoSignedLedgerTransactionImpl.kt | 2 +- .../factory/UtxoLedgerTransactionFactory.kt | 2 +- .../impl/UtxoLedgerTransactionFactoryImpl.kt | 5 ++- ...xoSignedLedgerTransactionKryoSerializer.kt | 33 +++++++++++++++++++ .../transaction/WrappedUtxoWireTransaction.kt | 17 ++++++++++ .../utxo/testkit/UtxoLedgerIntegrationTest.kt | 18 +++++++--- .../testkit/UtxoSignedTransactionExample.kt | 3 +- 12 files changed, 127 insertions(+), 17 deletions(-) create mode 100644 components/ledger/ledger-utxo-flow/src/integrationTest/kotlin/net/corda/ledger/utxo/flow/impl/transaction/serializer/tests/UtxoLedgerTransactionKryoSerializationTest.kt create mode 100644 components/ledger/ledger-utxo-flow/src/integrationTest/kotlin/net/corda/ledger/utxo/flow/impl/transaction/serializer/tests/UtxoSignedLedgerTransactionKryoSerializationTest.kt create mode 100644 components/ledger/ledger-utxo-flow/src/main/kotlin/net/corda/ledger/utxo/flow/impl/transaction/serializer/kryo/UtxoSignedLedgerTransactionKryoSerializer.kt diff --git a/components/ledger/ledger-utxo-flow/src/integrationTest/kotlin/net/corda/ledger/utxo/flow/impl/transaction/filtered/tests/UtxoFilteredTransactionTest.kt b/components/ledger/ledger-utxo-flow/src/integrationTest/kotlin/net/corda/ledger/utxo/flow/impl/transaction/filtered/tests/UtxoFilteredTransactionTest.kt index a8a65d3731c..4f8ddd953d2 100644 --- a/components/ledger/ledger-utxo-flow/src/integrationTest/kotlin/net/corda/ledger/utxo/flow/impl/transaction/filtered/tests/UtxoFilteredTransactionTest.kt +++ b/components/ledger/ledger-utxo-flow/src/integrationTest/kotlin/net/corda/ledger/utxo/flow/impl/transaction/filtered/tests/UtxoFilteredTransactionTest.kt @@ -4,6 +4,7 @@ import net.corda.crypto.core.parseSecureHash import net.corda.ledger.common.testkit.publicKeyExample import net.corda.ledger.utxo.data.transaction.UtxoOutputInfoComponent import net.corda.ledger.utxo.flow.impl.timewindow.TimeWindowBetweenImpl +import net.corda.ledger.utxo.flow.impl.transaction.UtxoSignedTransactionInternal import net.corda.ledger.utxo.testkit.UtxoLedgerIntegrationTest import net.corda.ledger.utxo.testkit.UtxoStateClassExample import net.corda.ledger.utxo.testkit.createExample @@ -11,7 +12,6 @@ import net.corda.ledger.utxo.testkit.notaryX500Name import net.corda.v5.ledger.utxo.Command import net.corda.v5.ledger.utxo.StateAndRef import net.corda.v5.ledger.utxo.StateRef -import net.corda.v5.ledger.utxo.transaction.UtxoSignedTransaction import net.corda.v5.ledger.utxo.transaction.filtered.UtxoFilteredData import org.assertj.core.api.Assertions.assertThat import org.assertj.core.api.Assertions.assertThatCode @@ -408,7 +408,7 @@ class UtxoFilteredTransactionTest : UtxoLedgerIntegrationTest() { assertThatCode { utxoFilteredTransaction.verify() }.doesNotThrowAnyException() } - private fun createSignedTransaction(numberOfInputStates: Int = 2, numberOfOutputStates: Int = 2): UtxoSignedTransaction { + private fun createSignedTransaction(numberOfInputStates: Int = 2, numberOfOutputStates: Int = 2): UtxoSignedTransactionInternal { val inputHash = parseSecureHash("SHA256:1234567890abcdef") val outputInfo = UtxoOutputInfoComponent( encumbrance = null, diff --git a/components/ledger/ledger-utxo-flow/src/integrationTest/kotlin/net/corda/ledger/utxo/flow/impl/transaction/serializer/tests/UtxoLedgerTransactionKryoSerializationTest.kt b/components/ledger/ledger-utxo-flow/src/integrationTest/kotlin/net/corda/ledger/utxo/flow/impl/transaction/serializer/tests/UtxoLedgerTransactionKryoSerializationTest.kt new file mode 100644 index 00000000000..cf8fc17e9fd --- /dev/null +++ b/components/ledger/ledger-utxo-flow/src/integrationTest/kotlin/net/corda/ledger/utxo/flow/impl/transaction/serializer/tests/UtxoLedgerTransactionKryoSerializationTest.kt @@ -0,0 +1,20 @@ +package net.corda.ledger.utxo.flow.impl.transaction.serializer.tests + +import net.corda.ledger.utxo.data.transaction.UtxoLedgerTransactionInternal +import net.corda.ledger.utxo.testkit.UtxoLedgerIntegrationTest +import org.assertj.core.api.Assertions.assertThat +import org.junit.jupiter.api.Assertions.assertDoesNotThrow +import org.junit.jupiter.api.Test + +class UtxoLedgerTransactionKryoSerializationTest : UtxoLedgerIntegrationTest() { + @Test + fun `correct serialization of a UtxoLedgerTransaction`() { + val bytes = kryoSerializer.serialize(utxoLedgerTransaction) + val deserialized = kryoSerializer.deserialize(bytes, UtxoLedgerTransactionInternal::class.java) + + assertThat(deserialized).isEqualTo(utxoLedgerTransaction) + assertDoesNotThrow { deserialized.id } + assertThat(deserialized.id).isEqualTo(utxoLedgerTransaction.id) + assertThat(deserialized.wireTransaction).isEqualTo(utxoLedgerTransaction.wireTransaction) + } +} diff --git a/components/ledger/ledger-utxo-flow/src/integrationTest/kotlin/net/corda/ledger/utxo/flow/impl/transaction/serializer/tests/UtxoSignedLedgerTransactionKryoSerializationTest.kt b/components/ledger/ledger-utxo-flow/src/integrationTest/kotlin/net/corda/ledger/utxo/flow/impl/transaction/serializer/tests/UtxoSignedLedgerTransactionKryoSerializationTest.kt new file mode 100644 index 00000000000..e15fa60c209 --- /dev/null +++ b/components/ledger/ledger-utxo-flow/src/integrationTest/kotlin/net/corda/ledger/utxo/flow/impl/transaction/serializer/tests/UtxoSignedLedgerTransactionKryoSerializationTest.kt @@ -0,0 +1,23 @@ +package net.corda.ledger.utxo.flow.impl.transaction.serializer.tests + +import net.corda.ledger.utxo.flow.impl.transaction.UtxoSignedLedgerTransaction +import net.corda.ledger.utxo.flow.impl.transaction.UtxoSignedLedgerTransactionImpl +import net.corda.ledger.utxo.testkit.UtxoLedgerIntegrationTest +import org.assertj.core.api.Assertions.assertThat +import org.junit.jupiter.api.Assertions.assertDoesNotThrow +import org.junit.jupiter.api.Test + +class UtxoSignedLedgerTransactionKryoSerializationTest : UtxoLedgerIntegrationTest() { + @Test + fun `correct serialization of a UtxoSignedLedgerTransaction`() { + val utxoSignedLedgerTransaction = UtxoSignedLedgerTransactionImpl(utxoLedgerTransaction, utxoSignedTransaction) + val bytes = kryoSerializer.serialize(utxoSignedLedgerTransaction) + val deserialized = kryoSerializer.deserialize(bytes, UtxoSignedLedgerTransaction::class.java) + + assertThat(deserialized).isEqualTo(utxoSignedLedgerTransaction) + assertDoesNotThrow { deserialized.id } + assertThat(deserialized.id).isEqualTo(utxoSignedLedgerTransaction.id) + assertThat(deserialized.ledgerTransaction).isEqualTo(utxoSignedLedgerTransaction.ledgerTransaction) + assertThat(deserialized.signedTransaction).isEqualTo(utxoSignedLedgerTransaction.signedTransaction) + } +} diff --git a/components/ledger/ledger-utxo-flow/src/integrationTest/kotlin/net/corda/ledger/utxo/flow/impl/transaction/serializer/tests/UtxoSignedTransactionKryoSerializationTest.kt b/components/ledger/ledger-utxo-flow/src/integrationTest/kotlin/net/corda/ledger/utxo/flow/impl/transaction/serializer/tests/UtxoSignedTransactionKryoSerializationTest.kt index ea0d8431bce..12bf4af313e 100644 --- a/components/ledger/ledger-utxo-flow/src/integrationTest/kotlin/net/corda/ledger/utxo/flow/impl/transaction/serializer/tests/UtxoSignedTransactionKryoSerializationTest.kt +++ b/components/ledger/ledger-utxo-flow/src/integrationTest/kotlin/net/corda/ledger/utxo/flow/impl/transaction/serializer/tests/UtxoSignedTransactionKryoSerializationTest.kt @@ -1,21 +1,22 @@ package net.corda.ledger.utxo.flow.impl.transaction.serializer.tests +import net.corda.ledger.utxo.flow.impl.transaction.UtxoSignedTransactionInternal import net.corda.ledger.utxo.testkit.UtxoLedgerIntegrationTest -import net.corda.v5.ledger.utxo.transaction.UtxoSignedTransaction import org.assertj.core.api.Assertions.assertThat import org.junit.jupiter.api.Assertions import org.junit.jupiter.api.Test class UtxoSignedTransactionKryoSerializationTest: UtxoLedgerIntegrationTest() { + @Test - @Suppress("FunctionName") - fun `correct serialization of a utxo Signed Transaction`() { + fun `correct serialization of a UtxoSignedTransaction`() { val bytes = kryoSerializer.serialize(utxoSignedTransaction) - val deserialized = kryoSerializer.deserialize(bytes, UtxoSignedTransaction::class.java) + val deserialized = kryoSerializer.deserialize(bytes, UtxoSignedTransactionInternal::class.java) assertThat(deserialized).isEqualTo(utxoSignedTransaction) Assertions.assertDoesNotThrow { deserialized.id } - Assertions.assertEquals(utxoSignedTransaction.id, deserialized.id) + Assertions.assertEquals(deserialized.id, utxoSignedTransaction.id) + assertThat(deserialized.wireTransaction).isEqualTo(utxoSignedTransaction.wireTransaction) } } diff --git a/components/ledger/ledger-utxo-flow/src/main/kotlin/net/corda/ledger/utxo/flow/impl/transaction/UtxoSignedLedgerTransaction.kt b/components/ledger/ledger-utxo-flow/src/main/kotlin/net/corda/ledger/utxo/flow/impl/transaction/UtxoSignedLedgerTransaction.kt index ed36792bcfa..2286d3d3213 100644 --- a/components/ledger/ledger-utxo-flow/src/main/kotlin/net/corda/ledger/utxo/flow/impl/transaction/UtxoSignedLedgerTransaction.kt +++ b/components/ledger/ledger-utxo-flow/src/main/kotlin/net/corda/ledger/utxo/flow/impl/transaction/UtxoSignedLedgerTransaction.kt @@ -2,6 +2,7 @@ package net.corda.ledger.utxo.flow.impl.transaction import net.corda.ledger.utxo.data.transaction.UtxoLedgerTransactionInternal import net.corda.v5.ledger.utxo.transaction.UtxoLedgerTransaction +import net.corda.v5.ledger.utxo.transaction.UtxoSignedTransaction /** * [UtxoSignedLedgerTransaction] is a wrapper that combines the functionality of [UtxoLedgerTransactionInternal] and @@ -13,4 +14,9 @@ interface UtxoSignedLedgerTransaction : UtxoLedgerTransactionInternal, UtxoSigne * Gets the delegate [UtxoLedgerTransaction] from the [UtxoSignedLedgerTransaction] instance. */ val ledgerTransaction: UtxoLedgerTransaction + + /** + * Gets the delegate [UtxoSignedTransaction] from the [UtxoSignedLedgerTransaction] instance. + */ + val signedTransaction: UtxoSignedTransaction } \ No newline at end of file diff --git a/components/ledger/ledger-utxo-flow/src/main/kotlin/net/corda/ledger/utxo/flow/impl/transaction/UtxoSignedLedgerTransactionImpl.kt b/components/ledger/ledger-utxo-flow/src/main/kotlin/net/corda/ledger/utxo/flow/impl/transaction/UtxoSignedLedgerTransactionImpl.kt index 2e9d5d8b791..7127bd78578 100644 --- a/components/ledger/ledger-utxo-flow/src/main/kotlin/net/corda/ledger/utxo/flow/impl/transaction/UtxoSignedLedgerTransactionImpl.kt +++ b/components/ledger/ledger-utxo-flow/src/main/kotlin/net/corda/ledger/utxo/flow/impl/transaction/UtxoSignedLedgerTransactionImpl.kt @@ -21,7 +21,7 @@ import java.security.PublicKey */ data class UtxoSignedLedgerTransactionImpl( override val ledgerTransaction: UtxoLedgerTransactionInternal, - private val signedTransaction: UtxoSignedTransactionInternal + override val signedTransaction: UtxoSignedTransactionInternal ) : UtxoSignedLedgerTransaction, UtxoLedgerTransaction by ledgerTransaction, UtxoSignedTransactionInternal by signedTransaction { override fun getId(): SecureHash { diff --git a/components/ledger/ledger-utxo-flow/src/main/kotlin/net/corda/ledger/utxo/flow/impl/transaction/factory/UtxoLedgerTransactionFactory.kt b/components/ledger/ledger-utxo-flow/src/main/kotlin/net/corda/ledger/utxo/flow/impl/transaction/factory/UtxoLedgerTransactionFactory.kt index 01b1d6bae73..e473efff78e 100644 --- a/components/ledger/ledger-utxo-flow/src/main/kotlin/net/corda/ledger/utxo/flow/impl/transaction/factory/UtxoLedgerTransactionFactory.kt +++ b/components/ledger/ledger-utxo-flow/src/main/kotlin/net/corda/ledger/utxo/flow/impl/transaction/factory/UtxoLedgerTransactionFactory.kt @@ -20,7 +20,7 @@ interface UtxoLedgerTransactionFactory { @Suspendable fun create( wireTransaction: WireTransaction - ): UtxoLedgerTransaction + ): UtxoLedgerTransactionInternal fun create( wireTransaction: WireTransaction, diff --git a/components/ledger/ledger-utxo-flow/src/main/kotlin/net/corda/ledger/utxo/flow/impl/transaction/factory/impl/UtxoLedgerTransactionFactoryImpl.kt b/components/ledger/ledger-utxo-flow/src/main/kotlin/net/corda/ledger/utxo/flow/impl/transaction/factory/impl/UtxoLedgerTransactionFactoryImpl.kt index b1538b4c30e..4645e1ca655 100644 --- a/components/ledger/ledger-utxo-flow/src/main/kotlin/net/corda/ledger/utxo/flow/impl/transaction/factory/impl/UtxoLedgerTransactionFactoryImpl.kt +++ b/components/ledger/ledger-utxo-flow/src/main/kotlin/net/corda/ledger/utxo/flow/impl/transaction/factory/impl/UtxoLedgerTransactionFactoryImpl.kt @@ -1,13 +1,13 @@ package net.corda.ledger.utxo.flow.impl.transaction.factory.impl import net.corda.crypto.core.parseSecureHash +import net.corda.flow.application.GroupParametersLookupInternal import net.corda.ledger.common.data.transaction.TransactionMetadataInternal import net.corda.ledger.common.data.transaction.WireTransaction import net.corda.ledger.utxo.data.transaction.UtxoLedgerTransactionImpl import net.corda.ledger.utxo.data.transaction.UtxoLedgerTransactionInternal import net.corda.ledger.utxo.data.transaction.UtxoVisibleTransactionOutputDto import net.corda.ledger.utxo.data.transaction.WrappedUtxoWireTransaction -import net.corda.flow.application.GroupParametersLookupInternal import net.corda.ledger.utxo.flow.impl.persistence.UtxoLedgerGroupParametersPersistenceService import net.corda.ledger.utxo.flow.impl.persistence.UtxoLedgerStateQueryService import net.corda.ledger.utxo.flow.impl.transaction.factory.UtxoLedgerTransactionFactory @@ -16,7 +16,6 @@ import net.corda.v5.application.serialization.SerializationService import net.corda.v5.base.annotations.Suspendable import net.corda.v5.base.exceptions.CordaRuntimeException import net.corda.v5.ledger.utxo.ContractState -import net.corda.v5.ledger.utxo.transaction.UtxoLedgerTransaction import net.corda.v5.membership.GroupParameters import net.corda.v5.serialization.SingletonSerializeAsToken import org.osgi.service.component.annotations.Activate @@ -39,7 +38,7 @@ class UtxoLedgerTransactionFactoryImpl @Activate constructor( @Suspendable override fun create( wireTransaction: WireTransaction - ): UtxoLedgerTransaction { + ): UtxoLedgerTransactionInternal { val wrappedUtxoWireTransaction = WrappedUtxoWireTransaction(wireTransaction, serializationService) val allStateRefs = (wrappedUtxoWireTransaction.inputStateRefs + wrappedUtxoWireTransaction.referenceStateRefs) diff --git a/components/ledger/ledger-utxo-flow/src/main/kotlin/net/corda/ledger/utxo/flow/impl/transaction/serializer/kryo/UtxoSignedLedgerTransactionKryoSerializer.kt b/components/ledger/ledger-utxo-flow/src/main/kotlin/net/corda/ledger/utxo/flow/impl/transaction/serializer/kryo/UtxoSignedLedgerTransactionKryoSerializer.kt new file mode 100644 index 00000000000..34632dabaa5 --- /dev/null +++ b/components/ledger/ledger-utxo-flow/src/main/kotlin/net/corda/ledger/utxo/flow/impl/transaction/serializer/kryo/UtxoSignedLedgerTransactionKryoSerializer.kt @@ -0,0 +1,33 @@ +package net.corda.ledger.utxo.flow.impl.transaction.serializer.kryo + +import net.corda.ledger.utxo.data.transaction.UtxoLedgerTransactionInternal +import net.corda.ledger.utxo.flow.impl.transaction.UtxoSignedLedgerTransaction +import net.corda.ledger.utxo.flow.impl.transaction.UtxoSignedLedgerTransactionImpl +import net.corda.ledger.utxo.flow.impl.transaction.UtxoSignedTransactionInternal +import net.corda.sandbox.type.SandboxConstants.CORDA_UNINJECTABLE_SERVICE +import net.corda.sandbox.type.UsedByFlow +import net.corda.serialization.checkpoint.CheckpointInput +import net.corda.serialization.checkpoint.CheckpointInternalCustomSerializer +import net.corda.serialization.checkpoint.CheckpointOutput +import org.osgi.service.component.annotations.Component +import org.osgi.service.component.annotations.ServiceScope.PROTOTYPE + +@Component( + service = [ CheckpointInternalCustomSerializer::class, UsedByFlow::class ], + property = [ CORDA_UNINJECTABLE_SERVICE ], + scope = PROTOTYPE +) +class UtxoSignedLedgerTransactionKryoSerializer : CheckpointInternalCustomSerializer, UsedByFlow { + override val type: Class get() = UtxoSignedLedgerTransaction::class.java + + override fun write(output: CheckpointOutput, obj: UtxoSignedLedgerTransaction) { + output.writeClassAndObject(obj.ledgerTransaction) + output.writeClassAndObject(obj.signedTransaction) + } + + override fun read(input: CheckpointInput, type: Class): UtxoSignedLedgerTransaction { + val ledgerTransaction = input.readClassAndObject() as UtxoLedgerTransactionInternal + val signedTransaction = input.readClassAndObject() as UtxoSignedTransactionInternal + return UtxoSignedLedgerTransactionImpl(ledgerTransaction, signedTransaction) + } +} diff --git a/libs/ledger/ledger-utxo-data/src/main/kotlin/net/corda/ledger/utxo/data/transaction/WrappedUtxoWireTransaction.kt b/libs/ledger/ledger-utxo-data/src/main/kotlin/net/corda/ledger/utxo/data/transaction/WrappedUtxoWireTransaction.kt index ba13b850ed9..f31cc1fcc46 100644 --- a/libs/ledger/ledger-utxo-data/src/main/kotlin/net/corda/ledger/utxo/data/transaction/WrappedUtxoWireTransaction.kt +++ b/libs/ledger/ledger-utxo-data/src/main/kotlin/net/corda/ledger/utxo/data/transaction/WrappedUtxoWireTransaction.kt @@ -89,4 +89,21 @@ class WrappedUtxoWireTransaction( val serializedBytes = wireTransaction.getComponentGroupList(group.ordinal)[index] return serializationService.deserialize(serializedBytes) } + + override fun equals(other: Any?): Boolean { + if (this === other) return true + if (javaClass != other?.javaClass) return false + + other as WrappedUtxoWireTransaction + + return wireTransaction == other.wireTransaction + } + + override fun hashCode(): Int { + return wireTransaction.hashCode() + } + + override fun toString(): String { + return "WrappedUtxoWireTransaction(wireTransaction=$wireTransaction)" + } } diff --git a/testing/ledger/ledger-utxo-testkit/src/main/kotlin/net/corda/ledger/utxo/testkit/UtxoLedgerIntegrationTest.kt b/testing/ledger/ledger-utxo-testkit/src/main/kotlin/net/corda/ledger/utxo/testkit/UtxoLedgerIntegrationTest.kt index 434733df1f2..772bd5643a8 100644 --- a/testing/ledger/ledger-utxo-testkit/src/main/kotlin/net/corda/ledger/utxo/testkit/UtxoLedgerIntegrationTest.kt +++ b/testing/ledger/ledger-utxo-testkit/src/main/kotlin/net/corda/ledger/utxo/testkit/UtxoLedgerIntegrationTest.kt @@ -1,22 +1,26 @@ package net.corda.ledger.utxo.testkit import net.corda.ledger.common.integration.test.CommonLedgerIntegrationTest +import net.corda.ledger.utxo.data.transaction.UtxoLedgerTransactionImpl +import net.corda.ledger.utxo.data.transaction.UtxoLedgerTransactionInternal +import net.corda.ledger.utxo.data.transaction.WrappedUtxoWireTransaction import net.corda.ledger.utxo.flow.impl.persistence.UtxoLedgerPersistenceService +import net.corda.ledger.utxo.flow.impl.transaction.UtxoSignedTransactionInternal import net.corda.ledger.utxo.flow.impl.transaction.factory.UtxoSignedTransactionFactory import net.corda.sandboxgroupcontext.getSandboxSingletonService import net.corda.testing.sandboxes.SandboxSetup import net.corda.v5.ledger.utxo.UtxoLedgerService -import net.corda.v5.ledger.utxo.transaction.UtxoSignedTransaction -abstract class UtxoLedgerIntegrationTest: CommonLedgerIntegrationTest() { +abstract class UtxoLedgerIntegrationTest : CommonLedgerIntegrationTest() { override val testingCpb = "/META-INF/ledger-utxo-state-app.cpb" lateinit var utxoSignedTransactionFactory: UtxoSignedTransactionFactory lateinit var utxoLedgerService: UtxoLedgerService lateinit var utxoLedgerPersistenceService: UtxoLedgerPersistenceService - lateinit var utxoSignedTransaction: UtxoSignedTransaction + lateinit var utxoSignedTransaction: UtxoSignedTransactionInternal + lateinit var utxoLedgerTransaction: UtxoLedgerTransactionInternal - override fun initialize(setup: SandboxSetup){ + override fun initialize(setup: SandboxSetup) { super.initialize(setup) utxoSignedTransactionFactory = sandboxGroupContext.getSandboxSingletonService() @@ -27,5 +31,11 @@ abstract class UtxoLedgerIntegrationTest: CommonLedgerIntegrationTest() { jsonValidator, wireTransactionFactory ) + utxoLedgerTransaction = UtxoLedgerTransactionImpl( + WrappedUtxoWireTransaction(utxoSignedTransaction.wireTransaction, serializationService), + emptyList(), + emptyList(), + null + ) } } \ No newline at end of file diff --git a/testing/ledger/ledger-utxo-testkit/src/main/kotlin/net/corda/ledger/utxo/testkit/UtxoSignedTransactionExample.kt b/testing/ledger/ledger-utxo-testkit/src/main/kotlin/net/corda/ledger/utxo/testkit/UtxoSignedTransactionExample.kt index b4818d608c8..c857e2e354c 100644 --- a/testing/ledger/ledger-utxo-testkit/src/main/kotlin/net/corda/ledger/utxo/testkit/UtxoSignedTransactionExample.kt +++ b/testing/ledger/ledger-utxo-testkit/src/main/kotlin/net/corda/ledger/utxo/testkit/UtxoSignedTransactionExample.kt @@ -11,6 +11,7 @@ import net.corda.ledger.common.testkit.getWireTransactionExample import net.corda.ledger.utxo.data.transaction.UtxoComponentGroup import net.corda.ledger.utxo.data.transaction.UtxoLedgerTransactionImpl import net.corda.ledger.utxo.flow.impl.transaction.UtxoSignedTransactionImpl +import net.corda.ledger.utxo.flow.impl.transaction.UtxoSignedTransactionInternal import net.corda.ledger.utxo.flow.impl.transaction.factory.UtxoLedgerTransactionFactory import net.corda.ledger.utxo.flow.impl.transaction.factory.UtxoSignedTransactionFactory import net.corda.v5.application.crypto.DigestService @@ -24,7 +25,7 @@ fun UtxoSignedTransactionFactory.createExample( wireTransactionFactory: WireTransactionFactory, componentGroups: List> = defaultComponentGroups + List(UtxoComponentGroup.values().size - defaultComponentGroups.size - 1) { emptyList() } -):UtxoSignedTransaction { +): UtxoSignedTransactionInternal { val wireTransaction = wireTransactionFactory.createExample( jsonMarshallingService, jsonValidator, From fb24af4d5909c1008a9e251ffdcca13c2b1aed41 Mon Sep 17 00:00:00 2001 From: "tom.fitzpatrick" Date: Mon, 23 Oct 2023 08:51:20 +0100 Subject: [PATCH 56/81] CORE-17297: Fix incompatible responder flow failure handling (#4949) Set SessionStateType.ERROR when creating session state in failure path --- .../flow/pipeline/handlers/events/SessionEventHandler.kt | 9 +++++++++ .../pipeline/handlers/events/SessionEventHandlerTest.kt | 1 + 2 files changed, 10 insertions(+) diff --git a/components/flow/flow-service/src/main/kotlin/net/corda/flow/pipeline/handlers/events/SessionEventHandler.kt b/components/flow/flow-service/src/main/kotlin/net/corda/flow/pipeline/handlers/events/SessionEventHandler.kt index 7bb0c2b36b8..a2b94295e2d 100644 --- a/components/flow/flow-service/src/main/kotlin/net/corda/flow/pipeline/handlers/events/SessionEventHandler.kt +++ b/components/flow/flow-service/src/main/kotlin/net/corda/flow/pipeline/handlers/events/SessionEventHandler.kt @@ -21,6 +21,7 @@ import net.corda.flow.pipeline.sessions.FlowSessionManager import net.corda.flow.pipeline.sessions.protocol.FlowAndProtocolVersion import net.corda.flow.state.FlowCheckpoint import net.corda.flow.utils.KeyValueStore +import net.corda.flow.utils.emptyKeyValuePairList import net.corda.flow.utils.keyValuePairListOf import net.corda.flow.utils.toMap import net.corda.session.manager.Constants.Companion.FLOW_PROTOCOL @@ -231,6 +232,14 @@ class SessionEventHandler @Activate constructor( sessionId: String, exception: Throwable ) { + context.checkpoint.putSessionState(sessionManager.generateSessionState( + sessionId, + emptyKeyValuePairList(), + (context.inputEventPayload as SessionEvent).initiatingIdentity, + Instant.now(), + SessionStateType.ERROR + )) + context.checkpoint.putSessionStates( flowSessionManager.sendErrorMessages( context.checkpoint, diff --git a/components/flow/flow-service/src/test/kotlin/net/corda/flow/pipeline/handlers/events/SessionEventHandlerTest.kt b/components/flow/flow-service/src/test/kotlin/net/corda/flow/pipeline/handlers/events/SessionEventHandlerTest.kt index 225ecefe1bd..6637a6305bf 100644 --- a/components/flow/flow-service/src/test/kotlin/net/corda/flow/pipeline/handlers/events/SessionEventHandlerTest.kt +++ b/components/flow/flow-service/src/test/kotlin/net/corda/flow/pipeline/handlers/events/SessionEventHandlerTest.kt @@ -159,6 +159,7 @@ class SessionEventHandlerTest { flowSandboxService, sessionManager, fakeCheckpointInitializerService, flowSessionManager) sessionEventHandler.preProcess(inputContext) + verify(sessionManager, times(1)).generateSessionState(any(), any(), anyOrNull(), any(), any()) verify(flowSessionManager, times(1)).sendErrorMessages(any(), any(), anyOrNull(), any()) } From ea1e89366eecab25764517234b4bb3e3f7089405 Mon Sep 17 00:00:00 2001 From: Lorcan Wogan <69468264+LWogan@users.noreply.github.com> Date: Mon, 23 Oct 2023 13:17:12 +0100 Subject: [PATCH 57/81] CORE-17603 move debug log to trace as it is noisey (#4943) --- .../messaging/subscription/StateAndEventSubscriptionImpl.kt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/subscription/StateAndEventSubscriptionImpl.kt b/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/subscription/StateAndEventSubscriptionImpl.kt index a489df829cb..71a3d4bc3da 100644 --- a/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/subscription/StateAndEventSubscriptionImpl.kt +++ b/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/subscription/StateAndEventSubscriptionImpl.kt @@ -30,6 +30,7 @@ import net.corda.metrics.CordaMetrics import net.corda.schema.Schemas.getDLQTopic import net.corda.schema.Schemas.getStateAndEventStateTopic import net.corda.utilities.debug +import net.corda.utilities.trace import org.slf4j.LoggerFactory import java.nio.ByteBuffer import java.time.Clock @@ -187,7 +188,7 @@ internal class StateAndEventSubscriptionImpl( var keepProcessing = true while (keepProcessing && !threadLooper.loopStopped) { try { - log.debug { "Polling and processing events" } + log.trace { "Polling and processing events" } var rebalanceOccurred = false val records = stateAndEventConsumer.pollEvents() val batches = getEventsByBatch(records).iterator() From 667c483799e4ef25c31fb13b771a42c4bd278da8 Mon Sep 17 00:00:00 2001 From: Viktor Kolomeyko Date: Mon, 23 Oct 2023 13:40:21 +0100 Subject: [PATCH 58/81] CORE-17411: Workaround when we encounter enriched MGM request (#4946) So that we will not perform re-registration at all in this case. Also adding some additional logging. --- .../VirtualNodeUpgradeOperationHandler.kt | 27 +++++++++++++++---- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/components/virtual-node/virtual-node-write-service-impl/src/main/kotlin/net/corda/virtualnode/write/db/impl/writer/asyncoperation/handlers/VirtualNodeUpgradeOperationHandler.kt b/components/virtual-node/virtual-node-write-service-impl/src/main/kotlin/net/corda/virtualnode/write/db/impl/writer/asyncoperation/handlers/VirtualNodeUpgradeOperationHandler.kt index cb00a7be136..0f8c35e1690 100644 --- a/components/virtual-node/virtual-node-write-service-impl/src/main/kotlin/net/corda/virtualnode/write/db/impl/writer/asyncoperation/handlers/VirtualNodeUpgradeOperationHandler.kt +++ b/components/virtual-node/virtual-node-write-service-impl/src/main/kotlin/net/corda/virtualnode/write/db/impl/writer/asyncoperation/handlers/VirtualNodeUpgradeOperationHandler.kt @@ -63,6 +63,10 @@ internal class VirtualNodeUpgradeOperationHandler( private companion object { private val logger = LoggerFactory.getLogger(this::class.java.enclosingClass) + + private fun isEnriched(context: Map<*, *>): Boolean { + return context.containsKey(MemberInfoExtension.MEMBER_CPI_NAME) + } } private val keyValuePairListDeserializer: CordaAvroDeserializer by lazy { @@ -214,6 +218,7 @@ internal class VirtualNodeUpgradeOperationHandler( * after the virtual node has been upgraded, so that the member CPI version is up-to-date. * Republishes the MGM's Member Info, if the Group Policy was changed. */ + @Suppress("NestedBlockDepth") private fun reRegisterMember(upgradedVNodeInfo: VirtualNodeInfo, cpiMetadata: CpiMetadata) { val holdingIdentity = upgradedVNodeInfo.holdingIdentity val membershipGroupReader = membershipGroupReaderProvider.getGroupReader(holdingIdentity) @@ -245,22 +250,34 @@ internal class VirtualNodeUpgradeOperationHandler( ) when (registrationRequest) { is MembershipQueryResult.Success -> { - if (registrationRequest.payload.isNotEmpty()) { + val payload = registrationRequest.payload + if (payload.isNotEmpty()) { try { // Get the latest registration request - val registrationRequestDetails = registrationRequest.payload.sortedBy { it.serial }.last() - - val updatedSerial = registrationRequestDetails.serial + 1 + val registrationRequestDetails = payload.sortedBy { it.serial }.last() + val registrationContext = registrationRequestDetails .memberProvidedContext.data.array() .deserializeContext(keyValuePairListDeserializer) .toMutableMap() + if (isEnriched(registrationContext)) { + // In some cases, the registration request contains the platform-transformed data, + // instead of the user-provided context, so we skip re-registration. + // This is applicable only for old registration requests, as it's now fixed. + logger.warn("The platform was not able to automatically re-register the vNode. " + + "Please perform re-registration of vNode $holdingIdentity manually.") + return + } + + val updatedSerial = registrationRequestDetails.serial + 1 registrationContext[MemberInfoExtension.SERIAL] = updatedSerial.toString() logger.info("Starting MGM re-registration for holdingIdentity=$holdingIdentity, " + "shortHash=${holdingIdentity.shortHash}, registrationContext=$registrationContext") - memberResourceClient.startRegistration(holdingIdentity.shortHash, registrationContext) + val registrationProgress = + memberResourceClient.startRegistration(holdingIdentity.shortHash, registrationContext) + logger.info("Registration progress: $registrationProgress") } catch (e: ContextDeserializationException) { logger.warn( "Could not deserialize previous registration context for ${holdingIdentity.shortHash}. " + From e011b42ad8f7918ed99dc378ebe8b59ba9a3b182 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Jos=C3=A9=20Ramos?= Date: Mon, 23 Oct 2023 14:56:18 +0100 Subject: [PATCH 59/81] CORE-17881: Individual State Manager Per Worker Type (#4948) Update Helm Chart so every worker type can configure its own intance of the State Manager. --- .ci/e2eTests/corda.yaml | 47 ++- charts/corda-lib/templates/_bootstrap.tpl | 41 +-- charts/corda-lib/templates/_helpers.tpl | 162 --------- charts/corda-lib/templates/_stateManager.tpl | 231 +++++++++++++ charts/corda-lib/templates/_worker.tpl | 22 +- charts/corda/templates/secrets.yaml | 49 +-- charts/corda/values.schema.json | 328 ++++++++++--------- charts/corda/values.yaml | 255 ++++++++------ state-manager.yaml | 56 +++- 9 files changed, 680 insertions(+), 511 deletions(-) create mode 100644 charts/corda-lib/templates/_stateManager.tpl diff --git a/.ci/e2eTests/corda.yaml b/.ci/e2eTests/corda.yaml index fce5db5173b..1be0bc14a33 100644 --- a/.ci/e2eTests/corda.yaml +++ b/.ci/e2eTests/corda.yaml @@ -8,12 +8,20 @@ bootstrap: secretKeyRef: key: "postgres-password" stateManager: - username: - value: "state-manager-user" - password: - valueFrom: - secretKeyRef: - key: "password" + flow: + username: + value: "state-manager-user" + password: + valueFrom: + secretKeyRef: + key: "password" + flowMapper: + username: + value: "state-manager-user" + password: + valueFrom: + secretKeyRef: + key: "password" kafka: sasl: username: @@ -37,15 +45,6 @@ db: username: value: "user" -stateManager: - db: - username: - value: "state-manager-user" - password: - valueFrom: - secretKeyRef: - key: "password" - kafka: bootstrapServers: "prereqs-kafka:9092" tls: @@ -89,6 +88,15 @@ workers: secretKeyRef: name: "kafka-credentials" key: "flow" + stateManager: + db: + name: state-manager + username: + value: "state-manager-user" + password: + valueFrom: + secretKeyRef: + key: "password" flowMapper: kafka: sasl: @@ -99,6 +107,15 @@ workers: secretKeyRef: name: "kafka-credentials" key: "flowMapper" + stateManager: + db: + name: state-manager + username: + value: "state-manager-user" + password: + valueFrom: + secretKeyRef: + key: "password" verification: kafka: sasl: diff --git a/charts/corda-lib/templates/_bootstrap.tpl b/charts/corda-lib/templates/_bootstrap.tpl index 3d3c3356f2d..d527565a2fb 100644 --- a/charts/corda-lib/templates/_bootstrap.tpl +++ b/charts/corda-lib/templates/_bootstrap.tpl @@ -225,15 +225,6 @@ spec: {{- end }} -l /tmp/crypto - echo 'Generating State Manager DB specification' - STATE_MANAGER_JDBC_URL="{{- include "corda.stateManagerJdbcUrl" . -}}" - mkdir /tmp/stateManager - java -Dpf4j.pluginsDir=/opt/override/plugins -Dlog4j2.debug=false -jar /opt/override/cli.jar database spec \ - -s "statemanager" -g "statemanager:state_manager" \ - -u "${STATE_MANAGER_PGUSER}" -p "${STATE_MANAGER_PGPASSWORD}" \ - --jdbc-url "${STATE_MANAGER_JDBC_URL}" \ - -c -l /tmp/stateManager - echo 'Generating REST API user initial configuration' java -Dpf4j.pluginsDir=/opt/override/plugins -Dlog4j2.debug=false -jar /opt/override/cli.jar initial-config create-user-config \ -u "${REST_API_ADMIN_USERNAME}" -p "${REST_API_ADMIN_PASSWORD}" \ @@ -266,7 +257,15 @@ spec: {{- include "corda.restApiAdminSecretEnv" . | nindent 12 }} {{- include "corda.cryptoDbUsernameEnv" . | nindent 12 }} {{- include "corda.cryptoDbPasswordEnv" . | nindent 12 }} - {{- include "corda.bootstrapStateManagerDbEnv" . | nindent 12 }} + {{- range $workerName, $authConfig := .Values.bootstrap.db.stateManager -}} + {{- $workerConfig := (index $.Values.workers $workerName) -}} + {{/* No point in trying to bootstrap the State Manager for the specific worker if the host has not been configured */}} + {{- if and (not $workerConfig.stateManager.db.host) (or ( $authConfig.username.value ) ( $authConfig.username.valueFrom.secretKeyRef.name ) ( $authConfig.password.value ) ( $authConfig.password.valueFrom.secretKeyRef.name ) ) -}} + {{- fail ( printf "Can only specify bootstrap.db.stateManager.%s when workers.%s.stateManager.host is configured" $workerName $workerName ) -}} + {{- else -}} + {{- include "corda.bootstrapStateManagerDb" ( list $ $workerName $authConfig ) }} + {{- end -}} + {{- end }} containers: - name: apply image: {{ include "corda.bootstrapDbClientImage" . }} @@ -303,20 +302,6 @@ spec: SQL echo 'DB Bootstrapped' - - echo 'Applying State Manager Specification' - export PGPASSWORD="${STATE_MANAGER_PGPASSWORD}" - find /tmp/stateManager -iname "*.sql" | xargs printf -- ' -f %s' | xargs psql -v ON_ERROR_STOP=1 -h "${STATE_MANAGER_DB_HOST}" -p "${STATE_MANAGER_DB_PORT}" -U "${STATE_MANAGER_PGUSER}" --dbname "${STATE_MANAGER_DB_NAME}" - - echo 'Creating users and granting permissions for State Manager' - psql -v ON_ERROR_STOP=1 -h "${STATE_MANAGER_DB_HOST}" -p "${STATE_MANAGER_DB_PORT}" -U "${STATE_MANAGER_PGUSER}" "${STATE_MANAGER_DB_NAME}" << SQL - DO \$\$ BEGIN IF EXISTS (SELECT FROM pg_catalog.pg_roles WHERE rolname = '${STATE_MANAGER_DB_USERNAME}') THEN RAISE NOTICE 'Role "${STATE_MANAGER_DB_USERNAME}" already exists'; ELSE CREATE USER "${STATE_MANAGER_DB_USERNAME}" WITH ENCRYPTED PASSWORD '${STATE_MANAGER_DB_PASSWORD}'; END IF; END \$\$; - GRANT USAGE ON SCHEMA STATE_MANAGER TO "${STATE_MANAGER_DB_USERNAME}"; - GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA STATE_MANAGER TO "${STATE_MANAGER_DB_USERNAME}"; - GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA STATE_MANAGER TO "${STATE_MANAGER_DB_USERNAME}"; - SQL - - echo 'State Manager Bootstrapped' volumeMounts: - mountPath: /tmp name: temp @@ -333,19 +318,11 @@ spec: value: {{ .Values.bootstrap.db.rbac.schema | quote }} - name: DB_CRYPTO_SCHEMA value: {{ .Values.bootstrap.db.crypto.schema | quote }} - - name: STATE_MANAGER_DB_HOST - value: {{ include "corda.stateManagerDbHost" . | quote }} - - name: STATE_MANAGER_DB_PORT - value: {{ include "corda.stateManagerDbPort" . | quote }} - - name: STATE_MANAGER_DB_NAME - value: {{ include "corda.stateManagerDbName" . | quote }} {{- include "corda.bootstrapClusterDbEnv" . | nindent 12 }} {{- include "corda.rbacDbUserEnv" . | nindent 12 }} {{- include "corda.cryptoDbUsernameEnv" . | nindent 12 }} {{- include "corda.cryptoDbPasswordEnv" . | nindent 12 }} {{- include "corda.clusterDbEnv" . | nindent 12 }} - {{- include "corda.bootstrapStateManagerDbEnv" . | nindent 12 }} - {{- include "corda.stateManagerDbEnv" . | nindent 12 }} volumes: - name: temp emptyDir: {} diff --git a/charts/corda-lib/templates/_helpers.tpl b/charts/corda-lib/templates/_helpers.tpl index e7d77c61f3d..8112e7a9450 100644 --- a/charts/corda-lib/templates/_helpers.tpl +++ b/charts/corda-lib/templates/_helpers.tpl @@ -630,168 +630,6 @@ data: {{ $caSecretKey }}: {{ $caSecretValue }} {{- end }} -{{/* -Default name for State Manager DB secret -*/}} -{{- define "corda.stateManagerDbDefaultSecretName" -}} -{{- if $.Values.stateManager.db.host -}} -{{ printf "%s-statemanager-db" (include "corda.fullname" .) }} -{{- else -}} -{{ printf "%s" (include "corda.clusterDbDefaultSecretName" .) }} -{{- end -}} -{{- end }} - -{{/* -State Manager JDBC URL. If no state-manager host is provided, it will default to use a state_manager schema in -the Corda cluster. -*/}} -{{- define "corda.stateManagerJdbcUrl" -}} -{{- if $.Values.stateManager.db.host -}} -jdbc:{{- include "corda.stateManagerDbType" . -}}://{{- .Values.stateManager.db.host -}}:{{- include "corda.stateManagerDbPort" . -}}/{{- include "corda.stateManagerDbName" . -}} -{{- else -}} -jdbc:{{- include "corda.clusterDbType" . -}}://{{- .Values.db.cluster.host -}}:{{- include "corda.clusterDbPort" . -}}/{{- include "corda.clusterDbName" . -}} -{{- end -}} -{{- end -}} - -{{/* -State Manager DB type -*/}} -{{- define "corda.stateManagerDbType" -}} -{{- if $.Values.stateManager.db.host -}} -{{- .Values.stateManager.db.type | default "postgresql" -}} -{{- else -}} -{{- include "corda.clusterDbType" . -}} -{{- end -}} -{{- end -}} - -{{/* -State Manager DB host -*/}} -{{- define "corda.stateManagerDbHost" -}} -{{- if .Values.stateManager.db.host -}} -{{- .Values.stateManager.db.host -}} -{{- else -}} -{{- .Values.db.cluster.host -}} -{{- end -}} -{{- end -}} - -{{/* -State Manager DB port -*/}} -{{- define "corda.stateManagerDbPort" -}} -{{- if $.Values.stateManager.db.host }} -{{- .Values.stateManager.db.port | default "5432" -}} -{{- else }} -{{- include "corda.clusterDbPort" . -}} -{{- end }} -{{- end -}} - -{{/* -State Manager DB name -*/}} -{{- define "corda.stateManagerDbName" -}} -{{- if $.Values.stateManager.db.host }} -{{- .Values.stateManager.db.database | default "state-manager" -}} -{{- else }} -{{- include "corda.clusterDbName" . -}} -{{- end }} -{{- end -}} - - -{{/* -State Manager DB credentials environment variables. -STATE_MANAGER_DB_USERNAME falls back to the cluster database if no stateManager credentials are supplied, and if no -stateManager host is supplied. -*/}} -{{- define "corda.stateManagerDbEnv" -}} -- name: STATE_MANAGER_DB_USERNAME - valueFrom: - secretKeyRef: - {{- if .Values.stateManager.db.host }} - {{- if .Values.stateManager.db.username.valueFrom.secretKeyRef.name }} - name: {{ .Values.stateManager.db.username.valueFrom.secretKeyRef.name | quote }} - key: {{ required "Must specify stateManager.db.username.valueFrom.secretKeyRef.key" .Values.stateManager.db.username.valueFrom.secretKeyRef.key | quote }} - {{- else }} - name: {{ include "corda.stateManagerDbDefaultSecretName" . | quote }} - key: "username" - {{- end }} - {{- else }} - {{- include "corda.clusterDbUsername" . | nindent 6 }} - {{- end }} -- name: STATE_MANAGER_DB_PASSWORD - valueFrom: - secretKeyRef: - {{- if .Values.stateManager.db.host }} - {{- if .Values.stateManager.db.password.valueFrom.secretKeyRef.name }} - name: {{ .Values.stateManager.db.password.valueFrom.secretKeyRef.name | quote }} - key: {{ required "Must specify stateManager.db.password.valueFrom.secretKeyRef.key" .Values.stateManager.db.password.valueFrom.secretKeyRef.key | quote }} - {{- else }} - name: {{ include "corda.stateManagerDbDefaultSecretName" . | quote }} - key: "password" - {{- end }} - {{- else }} - {{- include "corda.clusterDbPassword" . | nindent 6 }} - {{- end }} -{{- end -}} - -{{/* -Default name for bootstrap State Manager DB secret -*/}} -{{- define "corda.bootstrapStateManagerDbDefaultSecretName" -}} -{{- if .Values.stateManager.db.host -}} -{{ printf "%s-bootstrap-statemanager-db" (include "corda.fullname" .) }} -{{- else -}} -{{ printf "%s" (include "corda.bootstrapClusterDbDefaultSecretName" .) }} -{{- end }} -{{- end }} - -{{/* -Bootstrap State Manager DB credentials environment variables. If there is no other host defined for state manager, the -cluster db user and password are used. -*/}} -{{- define "corda.bootstrapStateManagerDbEnv" -}} -- name: STATE_MANAGER_PGUSER - valueFrom: - secretKeyRef: - {{- if .Values.stateManager.db.host }} - {{- if .Values.bootstrap.db.stateManager.username.valueFrom.secretKeyRef.name }} - name: {{ .Values.bootstrap.db.stateManager.username.valueFrom.secretKeyRef.name | quote }} - key: {{ required "Must specify bootstrap.db.stateManager.username.valueFrom.secretKeyRef.key" .Values.bootstrap.db.stateManager.username.valueFrom.secretKeyRef.key | quote }} - {{- else if .Values.bootstrap.db.stateManager.username.value }} - name: {{ include "corda.bootstrapStateManagerDbDefaultSecretName" . | quote }} - key: "username" - {{- else if .Values.stateManager.db.username.valueFrom.secretKeyRef.name }} - name: {{ .Values.stateManager.db.username.valueFrom.secretKeyRef.name | quote }} - key: {{ required "Must specify stateManager.db.username.valueFrom.secretKeyRef.key" .Values.stateManager.db.username.valueFrom.secretKeyRef.key | quote }} - {{- else }} - name: {{ include "corda.stateManagerDbDefaultSecretName" . | quote }} - key: "username" - {{- end }} - {{- else }} - {{- include "corda.bootstrapClusterPgUser" . | nindent 6 }} - {{- end }} -- name: STATE_MANAGER_PGPASSWORD - valueFrom: - secretKeyRef: - {{- if .Values.stateManager.db.host }} - {{- if .Values.bootstrap.db.stateManager.password.valueFrom.secretKeyRef.name }} - name: {{ .Values.bootstrap.db.stateManager.password.valueFrom.secretKeyRef.name | quote }} - key: {{ required "Must specify bootstrap.db.stateManager.password.valueFrom.secretKeyRef.key" .Values.bootstrap.db.stateManager.password.valueFrom.secretKeyRef.key | quote }} - {{- else if .Values.bootstrap.db.stateManager.password.value }} - name: {{ include "corda.bootstrapStateManagerDbDefaultSecretName" . | quote }} - key: "password" - {{- else if .Values.stateManager.db.password.valueFrom.secretKeyRef.name }} - name: {{ .Values.stateManager.db.password.valueFrom.secretKeyRef.name | quote }} - key: {{ required "Must specify stateManager.db.password.valueFrom.secretKeyRef.key" .Values.stateManager.db.password.valueFrom.secretKeyRef.key | quote }} - {{- else }} - name: {{ include "corda.stateManagerDbDefaultSecretName" . | quote }} - key: "password" - {{- end }} - {{- else }} - {{- include "corda.bootstrapClusterPgPassword" . | nindent 6 }} - {{- end }} -{{- end -}} - {{/* The port which should be used to connect to Corda worker instances */}} diff --git a/charts/corda-lib/templates/_stateManager.tpl b/charts/corda-lib/templates/_stateManager.tpl new file mode 100644 index 00000000000..05f1ccc7525 --- /dev/null +++ b/charts/corda-lib/templates/_stateManager.tpl @@ -0,0 +1,231 @@ +{{/* + State Manager Named Templates + Most of the templates verify the value set for the [workerName.stateManager.db.host] property as that's the main + indication of whether the user has configured the stateManager for the individual worker or not. When not configured, + the templates use the cluster database details by default. +*/}} + +{{/* State Manager Database Type */}} +{{- define "corda.stateManagerDbType" -}} +{{- $ := index . 0 }} +{{- $workerConfig := index . 1 }} +{{- if $workerConfig.stateManager.db.host -}} +{{- $workerConfig.stateManager.db.type -}} +{{- else -}} +{{- include "corda.clusterDbType" . -}} +{{- end -}} +{{- end -}} + +{{/* State Manager Database Host */}} +{{- define "corda.stateManagerDbHost" -}} +{{- $ := index . 0 }} +{{- $workerConfig := index . 1 }} +{{- if $workerConfig.stateManager.db.host -}} +{{- $workerConfig.stateManager.db.host -}} +{{- else -}} +{{- $.Values.db.cluster.host -}} +{{- end -}} +{{- end -}} + + +{{/* State Manager Database Port */}} +{{- define "corda.stateManagerDbPort" -}} +{{- $ := index . 0 }} +{{- $workerConfig := index . 1 }} +{{- if $workerConfig.stateManager.db.host -}} +{{- $workerConfig.stateManager.db.port -}} +{{- else }} +{{- include "corda.clusterDbPort" $ -}} +{{- end }} +{{- end -}} + + +{{/* State Manager Database Name */}} +{{- define "corda.stateManagerDbName" -}} +{{- $ := index . 0 }} +{{- $workerConfig := index . 1 }} +{{- if $workerConfig.stateManager.db.host -}} +{{- $workerConfig.stateManager.db.name -}} +{{- else }} +{{- include "corda.clusterDbName" $ -}} +{{- end }} +{{- end -}} + + +{{/* State Manager JDBC URL */}} +{{- define "corda.stateManagerJdbcUrl" -}} +{{- $ := index . 0 }} +{{- $workerConfig := index . 1 }} +{{- if $workerConfig.stateManager.db.host -}} +jdbc:{{- include "corda.stateManagerDbType" (list $ $workerConfig) -}}://{{- $workerConfig.stateManager.db.host -}}:{{- include "corda.stateManagerDbPort" (list $ $workerConfig) -}}/{{- include "corda.stateManagerDbName" (list $ $workerConfig) -}} +{{- else -}} +jdbc:{{- include "corda.clusterDbType" $ -}}://{{- $.Values.db.cluster.host -}}:{{- include "corda.clusterDbPort" $ -}}/{{- include "corda.clusterDbName" $ -}} +{{- end -}} +{{- end -}} + + +{{/* State Manager Default Worker Secret Name */}} +{{- define "corda.stateManagerDefaultSecretName" -}} +{{- $ := index . 0 }} +{{- $workerKey := index . 1 }} +{{- $workerName := printf "%s-worker" ( include "corda.workerTypeKebabCase" $workerKey ) }} +{{- printf "%s-%s-state-manager-secret" (include "corda.fullname" $) $workerName }} +{{- end -}} + + +{{/* State Manager Default Worker Secret Name */}} +{{- define "corda.stateManagerDefaultBootSecretName" -}} +{{- $ := index . 0 }} +{{- $workerKey := index . 1 }} +{{- $workerName := printf "%s-worker" ( include "corda.workerTypeKebabCase" $workerKey ) }} +{{- printf "%s-%s-state-manager-boot-secret" (include "corda.fullname" $) $workerName }} +{{- end -}} + + +{{- define "corda.bootstrapStateManagerDbEnv" -}} +{{- $ := index . 0 -}} +{{- $worker := index . 1 -}} +{{- $workerKey := index . 2 -}} +{{- $bootConfig := index . 3 -}} +- name: STATE_MANAGER_PGUSER + valueFrom: + secretKeyRef: + {{- if $worker.stateManager.db.host }} + {{- if $bootConfig.username.valueFrom.secretKeyRef.name }} + name: {{ $bootConfig.username.valueFrom.secretKeyRef.name | quote }} + key: {{ required (printf "Must specify bootstrap.db.stateManager.%s.username.valueFrom.secretKeyRef.key" $workerKey) $bootConfig.username.valueFrom.secretKeyRef.key | quote }} + {{- else }} + name: {{ include "corda.stateManagerDefaultBootSecretName" ( list $ $workerKey ) | quote }} + key: "username" + {{- end }} + {{- else }} + {{- include "corda.bootstrapClusterPgUser" $ | nindent 6 }} + {{- end }} +- name: STATE_MANAGER_PGPASSWORD + valueFrom: + secretKeyRef: + {{- if $worker.stateManager.db.host }} + {{- if $bootConfig.password.valueFrom.secretKeyRef.name }} + name: {{ $bootConfig.password.valueFrom.secretKeyRef.name | quote }} + key: {{ required (printf "Must specify bootstrap.db.stateManager.%s.password.valueFrom.secretKeyRef.key" $workerKey) $bootConfig.password.valueFrom.secretKeyRef.key | quote }} + {{- else }} + name: {{ include "corda.stateManagerDefaultBootSecretName" ( list $ $workerKey ) | quote }} + key: "password" + {{- end }} + {{- else }} + {{- include "corda.bootstrapClusterPgPassword" $ | nindent 6 }} + {{- end }} +{{- end -}} + + +{{/* State Manager Database Credentials Environment Variables */}} +{{- define "corda.stateManagerDbEnv" -}} +{{- $ := index . 0 -}} +{{- $worker := index . 1 -}} +{{- $workerKey := index . 2 -}} +- name: STATE_MANAGER_USERNAME + valueFrom: + secretKeyRef: + {{- if $worker.stateManager.db.host }} + {{- if $worker.stateManager.db.username.valueFrom.secretKeyRef.name }} + name: {{ $worker.stateManager.db.username.valueFrom.secretKeyRef.name | quote }} + key: {{ required (printf "Must specify workers.%s.stateManager.db.username.valueFrom.secretKeyRef.key" $workerKey) $worker.stateManager.db.username.valueFrom.secretKeyRef.key | quote }} + {{- else }} + name: {{ include "corda.stateManagerDefaultSecretName" ( list $ $workerKey ) | quote }} + key: "username" + {{- end }} + {{- else }} + {{- include "corda.clusterDbUsername" $ | nindent 6 }} + {{- end }} +- name: STATE_MANAGER_PASSWORD + valueFrom: + secretKeyRef: + {{- if $worker.stateManager.db.host }} + {{- if $worker.stateManager.db.password.valueFrom.secretKeyRef.name }} + name: {{ $worker.stateManager.db.password.valueFrom.secretKeyRef.name | quote }} + key: {{ required (printf "Must specify workers.%s.stateManager.db.password.valueFrom.secretKeyRef.key" $workerKey) $worker.stateManager.db.password.valueFrom.secretKeyRef.key | quote }} + {{- else }} + name: {{ include "corda.stateManagerDefaultSecretName" ( list $ $workerKey ) | quote }} + key: "password" + {{- end }} + {{- else }} + {{- include "corda.clusterDbPassword" $ | nindent 6 }} + {{- end }} +{{- end -}} + + +{{/* State Manager Containers to Create & Apply Database Schemas Within The Bootstrap Job */}} +{{- define "corda.bootstrapStateManagerDb" -}} +{{- $ := index . 0 -}} +{{- $workerKey := index . 1 -}} +{{- $authConfig := index . 2 -}} +{{- $worker := (index $.Values.workers $workerKey) -}} +{{- $workerName := printf "%s-worker" ( include "corda.workerTypeKebabCase" $workerKey ) -}} +{{- with index . 0 -}} +{{/* We use two init-containers for serial execution to prevent issues at applying the same Liquibase files at the same time (developer use case where all workers use the same state manager database) */}} + - name: generate-state-manager-{{ $workerName }} + image: {{ include "corda.bootstrapCliImage" . }} + imagePullPolicy: {{ .Values.imagePullPolicy }} + {{- include "corda.bootstrapResources" . | nindent 10 }} + {{- include "corda.containerSecurityContext" . | nindent 10 }} + env: + {{- include "corda.bootstrapCliEnv" . | nindent 12 }} + {{- include "corda.bootstrapStateManagerDbEnv" ( list $ $worker $workerKey $authConfig ) | nindent 12 }} + command: [ 'sh', '-c', '-e' ] + args: + - | + #!/bin/sh + set -ev + + echo 'Generating State Manager DB specification for {{ $workerName }}...' + STATE_MANAGER_JDBC_URL="{{- include "corda.stateManagerJdbcUrl" ( list . $worker ) -}}" + mkdir /tmp/stateManager-{{ $workerKey }} + java -Dpf4j.pluginsDir=/opt/override/plugins -Dlog4j2.debug=false -jar /opt/override/cli.jar database spec \ + -s "statemanager" -g "statemanager:state_manager" \ + -u "${STATE_MANAGER_PGUSER}" -p "${STATE_MANAGER_PGPASSWORD}" \ + --jdbc-url "${STATE_MANAGER_JDBC_URL}" \ + -c -l /tmp/stateManager-{{ $workerKey }} + echo 'Generating State Manager DB specification for {{ $workerName }}... Done' + workingDir: /tmp + volumeMounts: + - mountPath: /tmp + name: temp + {{- include "corda.log4jVolumeMount" . | nindent 12 }} + - name: apply-state-manager-{{ $workerName }} + image: {{ include "corda.bootstrapDbClientImage" . }} + imagePullPolicy: {{ .Values.imagePullPolicy }} + {{- include "corda.bootstrapResources" . | nindent 10 }} + {{- include "corda.containerSecurityContext" . | nindent 10 }} + env: + - name: STATE_MANAGER_DB_HOST + value: {{ include "corda.stateManagerDbHost" ( list . $worker ) | quote }} + - name: STATE_MANAGER_DB_PORT + value: {{ include "corda.stateManagerDbPort" ( list . $worker ) | quote }} + - name: STATE_MANAGER_DB_NAME + value: {{ include "corda.stateManagerDbName" ( list . $worker ) | quote }} + {{- include "corda.stateManagerDbEnv" ( list $ $worker $workerKey ) | nindent 12 }} + {{- include "corda.bootstrapStateManagerDbEnv" ( list $ $worker $workerKey $authConfig ) | nindent 12 }} + command: [ 'sh', '-c', '-e' ] + args: + - | + #!/bin/sh + set -ev + echo 'Applying State Manager Specification for {{ $workerName }}...' + export PGPASSWORD="${STATE_MANAGER_PGPASSWORD}" + find /tmp/stateManager-{{ $workerKey }} -iname "*.sql" | xargs printf -- ' -f %s' | xargs psql -v ON_ERROR_STOP=1 -h "${STATE_MANAGER_DB_HOST}" -p "${STATE_MANAGER_DB_PORT}" -U "${STATE_MANAGER_PGUSER}" --dbname "${STATE_MANAGER_DB_NAME}" + echo 'Applying State Manager Specification for {{ $workerName }}... Done!' + + echo 'Creating users and granting permissions for State Manager in {{ $workerName }}...' + psql -v ON_ERROR_STOP=1 -h "${STATE_MANAGER_DB_HOST}" -p "${STATE_MANAGER_DB_PORT}" -U "${STATE_MANAGER_PGUSER}" "${STATE_MANAGER_DB_NAME}" << SQL + DO \$\$ BEGIN IF EXISTS (SELECT FROM pg_catalog.pg_roles WHERE rolname = '${STATE_MANAGER_USERNAME}') THEN RAISE NOTICE 'Role "${STATE_MANAGER_USERNAME}" already exists'; ELSE CREATE USER "${STATE_MANAGER_USERNAME}" WITH ENCRYPTED PASSWORD '${STATE_MANAGER_PASSWORD}'; END IF; END \$\$; + GRANT USAGE ON SCHEMA STATE_MANAGER TO "${STATE_MANAGER_USERNAME}"; + GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA STATE_MANAGER TO "${STATE_MANAGER_USERNAME}"; + GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA STATE_MANAGER TO "${STATE_MANAGER_USERNAME}"; + SQL + + echo 'Creating users and granting permissions for State Manager in {{ $workerName }}... Done!' + volumeMounts: + - mountPath: /tmp + name: temp +{{- end }} +{{- end }} diff --git a/charts/corda-lib/templates/_worker.tpl b/charts/corda-lib/templates/_worker.tpl index 97fe731c992..6bb13316b1d 100644 --- a/charts/corda-lib/templates/_worker.tpl +++ b/charts/corda-lib/templates/_worker.tpl @@ -243,7 +243,7 @@ spec: {{- include "corda.clusterDbEnv" $ | nindent 10 }} {{- end }} {{- if $optionalArgs.stateManagerDbAccess }} - {{- include "corda.stateManagerDbEnv" $ | nindent 10 }} + {{- include "corda.stateManagerDbEnv" ( list $ . $worker ) | nindent 10 }} {{- end }} args: - "--workspace-dir=/work" @@ -293,29 +293,29 @@ spec: - "--stateManager" - "type=DATABASE" - "--stateManager" - - "database.user=$(STATE_MANAGER_DB_USERNAME)" + - "database.user=$(STATE_MANAGER_USERNAME)" - "--stateManager" - - "database.pass=$(STATE_MANAGER_DB_PASSWORD)" + - "database.pass=$(STATE_MANAGER_PASSWORD)" - "--stateManager" - - "database.jdbc.url={{- include "corda.stateManagerJdbcUrl" $ -}}?currentSchema=STATE_MANAGER" + - "database.jdbc.url={{- include "corda.stateManagerJdbcUrl" ( list $ . ) -}}?currentSchema=STATE_MANAGER" - "--stateManager" - "database.jdbc.directory=/opt/jdbc-driver" - "--stateManager" - "database.jdbc.driver=org.postgresql.Driver" - "--stateManager" - - "database.pool.maxSize={{ .stateManagerDbConnectionPool.maxSize }}" - {{- if .stateManagerDbConnectionPool.minSize }} + - "database.pool.maxSize={{ .stateManager.db.connectionPool.maxSize }}" + {{- if .stateManager.db.connectionPool.minSize }} - "--stateManager" - - "database.pool.minSize={{ .stateManagerDbConnectionPool.minSize }}" + - "database.pool.minSize={{ .stateManager.db.connectionPool.minSize }}" {{- end }} - "--stateManager" - - "database.pool.idleTimeoutSeconds={{ .stateManagerDbConnectionPool.idleTimeoutSeconds }}" + - "database.pool.idleTimeoutSeconds={{ .stateManager.db.connectionPool.idleTimeoutSeconds }}" - "--stateManager" - - "database.pool.maxLifetimeSeconds={{ .stateManagerDbConnectionPool.maxLifetimeSeconds }}" + - "database.pool.maxLifetimeSeconds={{ .stateManager.db.connectionPool.maxLifetimeSeconds }}" - "--stateManager" - - "database.pool.keepAliveTimeSeconds={{ .stateManagerDbConnectionPool.keepAliveTimeSeconds }}" + - "database.pool.keepAliveTimeSeconds={{ .stateManager.db.connectionPool.keepAliveTimeSeconds }}" - "--stateManager" - - "database.pool.validationTimeoutSeconds={{ .stateManagerDbConnectionPool.validationTimeoutSeconds }}" + - "database.pool.validationTimeoutSeconds={{ .stateManager.db.connectionPool.validationTimeoutSeconds }}" {{- end }} {{- if $.Values.tracing.endpoint }} - "--send-trace-to={{ $.Values.tracing.endpoint }}" diff --git a/charts/corda/templates/secrets.yaml b/charts/corda/templates/secrets.yaml index 974b72bf9aa..1ba137a8a6c 100644 --- a/charts/corda/templates/secrets.yaml +++ b/charts/corda/templates/secrets.yaml @@ -17,17 +17,6 @@ ( dict "username" ( dict "required" true ) "password" ( dict "required" true ) ) ) }} -{{- if .Values.stateManager.db.host }} -{{- include "corda.secret" - ( list - $ - .Values.stateManager.db - "stateManager.db" - ( include "corda.stateManagerDbDefaultSecretName" . ) - ( dict "username" ( dict ) "password" ( dict ) ) - ) -}} -{{- end }} {{- include "corda.secret" ( list $ @@ -37,6 +26,20 @@ ( dict "salt" ( dict "generate" 32 ) "passphrase" ( dict "generate" 32 ) ) ) }} +{{/* If host is set at the worker level, user is expected to also set runtime connection credentials */}} +{{- range $workerKey, $workerConfig := .Values.workers }} +{{- if ( $workerConfig.stateManager ).db.host -}} +{{- include "corda.secret" + ( list + $ + $workerConfig.stateManager.db + ( printf "workers.%s.stateManager.db" $workerKey ) + ( include "corda.stateManagerDefaultSecretName" ( list $ $workerKey ) ) + ( dict "username" ( dict "required" true ) "password" ( dict "generate" 12 ) ) + ) +}} +{{- end }} +{{- end }} {{- if .Values.bootstrap.db.enabled }} {{- include "corda.secret" ( list @@ -58,17 +61,21 @@ ( dict "cleanup" true ) ) }} -{{- if .Values.stateManager.db.host }} -{{- include "corda.secret" - ( list - $ - .Values.bootstrap.db.stateManager - "boostrap.db.stateManager" - ( include "corda.bootstrapStateManagerDbDefaultSecretName" . ) - ( dict "username" ( dict ) "password" ( dict ) ) - ( dict "cleanup" true ) - ) +{{/* If host is set at the worker level and bootstrap is enabled, user is expected to also set bootstrap connection credentials */}} +{{- range $workerKey, $authConfig := .Values.bootstrap.db.stateManager }} +{{- $workerConfig := (index $.Values.workers $workerKey) }} +{{- if ( $workerConfig.stateManager ).db.host -}} +{{- include "corda.secret" + ( list + $ + $authConfig + ( printf "bootstrap.db.stateManager.%s" $workerKey ) + ( include "corda.stateManagerDefaultBootSecretName" ( list $ $workerKey ) ) + ( dict "username" ( dict "required" true ) "password" ( dict "required" true ) ) + ( dict "cleanup" true ) + ) }} +{{- end }} {{- end }} {{- end }} {{- if or (.Values.bootstrap.db.enabled) (and (.Values.bootstrap.rbac.enabled) (and (or (.Values.bootstrap.restApiAdmin.username.value) (.Values.bootstrap.restApiAdmin.username.valueFrom.secretKeyRef.name)) (or (.Values.bootstrap.restApiAdmin.password.value) (.Values.bootstrap.restApiAdmin.password.valueFrom.secretKeyRef.name)))) }} diff --git a/charts/corda/values.schema.json b/charts/corda/values.schema.json index a830a6f8270..ae7424561d4 100644 --- a/charts/corda/values.schema.json +++ b/charts/corda/values.schema.json @@ -790,91 +790,6 @@ } }] }, - "stateManager": { - "type": "object", - "default": {}, - "title": "State Manager configuration", - "additionalProperties": true, - "properties": { - "type": { - "type": "string", - "default": "DATABASE", - "title": "The type of State Manager", - "examples": [ - "DATABASE" - ], - "enum": ["DATABASE"] - }, - "db": { - "type": "object", - "default": {}, - "title": "State Manager database configuration", - "additionalProperties": false, - "properties": { - "username": { - "$ref": "#/$defs/config" - }, - "password": { - "$ref": "#/$defs/config" - }, - "host": { - "anyOf": [ - { - "type": "string", - "format": "hostname" - }, - { - "type": "null" - } - ], - "default": null, - "title": "the State Manager database host", - "examples": [ - "postgresql.example.com" - ] - }, - "type": { - "type": "string", - "default": "postgresql", - "title": "the State Manager database type", - "examples": [ - "postgresql" - ], - "enum": ["postgresql"] - }, - "port": { - "type": "integer", - "default": 5432, - "title": "the State Manager database port", - "examples": [ - 5432 - ] - }, - "database": { - "type": "string", - "default": "state-manager", - "title": "the name of the State Manager database", - "examples": [ - "state-manager" - ], - "minLength": 1 - } - }, - "examples": [{ - "host": "postgresql.example.com", - "type": "postgresql", - "port": 5432, - "username": { - "value": "user" - }, - "password": { - "value": "password" - }, - "database": "state-manager" - }] - } - } - }, "kafka": { "type": "object", "default": {}, @@ -1354,25 +1269,20 @@ }] }, "stateManager": { - "type": "object", - "default": {}, - "title": "State Manager db configuration", - "properties": { - "username": { - "$ref": "#/$defs/config" - }, - "password": { - "$ref": "#/$defs/config" - } + "flow": { + "allOf": [ + { + "$ref": "#/$defs/basicAuthConfig" + } + ] }, - "examples": [{ - "password": { - "value": "password" - }, - "username": { - "value": "username" - } - }] + "flowMapper": { + "allOf": [ + { + "$ref": "#/$defs/basicAuthConfig" + } + ] + } }, "clientImage": { "type": "object", @@ -1911,8 +1821,8 @@ }, { "required": [ - "verifyInstrumentation", - "stateManagerDbConnectionPool" + "stateManager", + "verifyInstrumentation" ], "properties": { "image": {}, @@ -1924,8 +1834,8 @@ "profiling": {}, "resources": {}, "kafka": {}, - "stateManagerDbConnectionPool": { - "$ref": "#/$defs/stateManagerDbConnectionPool" + "stateManager": { + "$ref": "#/$defs/stateManager" }, "verifyInstrumentation": { "type": "boolean", @@ -1975,8 +1885,8 @@ }, { "required": [ - "verifyInstrumentation", - "stateManagerDbConnectionPool" + "stateManager", + "verifyInstrumentation" ], "properties": { "image": {}, @@ -1988,8 +1898,8 @@ "profiling": {}, "resources": {}, "kafka": {}, - "stateManagerDbConnectionPool" : { - "$ref": "#/$defs/stateManagerDbConnectionPool" + "stateManager" : { + "$ref": "#/$defs/stateManager" }, "verifyInstrumentation": { "type": "boolean", @@ -3527,63 +3437,163 @@ } } }, - "stateManagerDbConnectionPool": { + "stateManager": { "type": "object", "default": {}, - "title": "JDBC connection pool configuration for State Manager DB", - "required": [ - "maxSize", - "idleTimeoutSeconds", - "maxLifetimeSeconds", - "keepAliveTimeSeconds", - "validationTimeoutSeconds" - ], - "additionalProperties": false, + "title": "State Manager configuration", + "additionalProperties": true, "properties": { - "maxSize": { - "type": "integer", - "default": 5, - "title": "JDBC connection pool size for State Manager DB", + "type": { + "type": "string", + "default": "DATABASE", + "title": "The type of State Manager", "examples": [ - 5 - ] + "DATABASE" + ], + "enum": ["DATABASE"] }, - "minSize": { - "anyOf": [ - { + "db": { + "type": "object", + "default": {}, + "title": "State Manager database configuration", + "additionalProperties": false, + "properties": { + "username": { + "$ref": "#/$defs/config" + }, + "password": { + "$ref": "#/$defs/config" + }, + "host": { + "anyOf": [ + { + "type": "string", + "format": "hostname" + }, + { + "type": "null" + } + ], + "default": null, + "title": "the State Manager database host", + "examples": [ + "postgresql.example.com" + ] + }, + "type": { + "type": "string", + "default": "postgresql", + "title": "the State Manager database type", + "examples": [ + "postgresql" + ], + "enum": ["postgresql"] + }, + "port": { "type": "integer", - "minimum": 0 + "default": 5432, + "title": "the State Manager database port", + "examples": [ + 5432 + ] }, - { - "type": "null" + "name": { + "type": "string", + "default": "state-manager", + "title": "the name of the State Manager database", + "examples": [ + "state-manager" + ], + "minLength": 1 + }, + "connectionPool": { + "type": "object", + "default": {}, + "title": "JDBC connection pool configuration for State Manager DB", + "required": [ + "maxSize", + "idleTimeoutSeconds", + "maxLifetimeSeconds", + "keepAliveTimeSeconds", + "validationTimeoutSeconds" + ], + "additionalProperties": false, + "properties": { + "maxSize": { + "type": "integer", + "default": 5, + "title": "JDBC connection pool size for State Manager DB", + "examples": [ + 5 + ] + }, + "minSize": { + "anyOf": [ + { + "type": "integer", + "minimum": 0 + }, + { + "type": "null" + } + ], + "default": null, + "title": "Minimum JDBC connection pool size for State Manager DB; null value means pool's min size will default to pool's max size value" + }, + "idleTimeoutSeconds": { + "type": "integer", + "default": 120, + "minimum": 0, + "title": "Maximum time (in seconds) a connection can stay idle in the pool; A value of 0 means that idle connections are never removed from the pool" + }, + "maxLifetimeSeconds": { + "type": "integer", + "default": 1800, + "minimum": 1, + "title": "Maximum time (in seconds) a connection can stay in the pool, regardless if it has been idle or has been recently used; If a connection is in-use and has reached \"maxLifetime\" timeout, it will be removed from the pool only when it becomes idle" + }, + "keepAliveTimeSeconds": { + "type": "integer", + "default": 0, + "minimum": 0, + "title": "Interval time (in seconds) in which connections will be tested for aliveness; Connections which are no longer alive are removed from the pool; A value of 0 means this check is disabled" + }, + "validationTimeoutSeconds": { + "type": "integer", + "minimum": 1, + "default": 5, + "title": "Maximum time (in seconds) that the pool will wait for a connection to be validated as alive" + } + } } - ], - "default": null, - "title": "Minimum JDBC connection pool size for State Manager DB; null value means pool's min size will default to pool's max size value" - }, - "idleTimeoutSeconds": { - "type": "integer", - "default": 120, - "minimum": 0, - "title": "Maximum time (in seconds) a connection can stay idle in the pool; A value of 0 means that idle connections are never removed from the pool" - }, - "maxLifetimeSeconds": { - "type": "integer", - "default": 1800, - "minimum": 1, - "title": "Maximum time (in seconds) a connection can stay in the pool, regardless if it has been idle or has been recently used; If a connection is in-use and has reached \"maxLifetime\" timeout, it will be removed from the pool only when it becomes idle" - }, - "keepAliveTimeSeconds": { - "type": "integer", - "default": 0, - "minimum": 0, - "title": "Interval time (in seconds) in which connections will be tested for aliveness; Connections which are no longer alive are removed from the pool; A value of 0 means this check is disabled" + }, + "examples": [{ + "host": "postgresql.example.com", + "type": "postgresql", + "port": 5432, + "username": { + "value": "user" + }, + "password": { + "value": "password" + }, + "database": "state-manager", + "connectionPool": {} + }] + } + } + }, + "basicAuthConfig": { + "type": "object", + "default": {}, + "title": "Configuration for which username and password may be provided inline or via a secret", + "additionalProperties": false, + "properties": { + "username": { + "$ref": "#/$defs/config" }, - "validationTimeoutSeconds": { - "type": "integer", - "minimum": 1, - "default": 5, - "title": "Maximum time (in seconds) that the pool will wait for a connection to be validated as alive" + "password": { + "$ref": "#/$defs/config" } } } diff --git a/charts/corda/values.yaml b/charts/corda/values.yaml index a8ed73f42a6..fc38165d694 100644 --- a/charts/corda/values.yaml +++ b/charts/corda/values.yaml @@ -146,45 +146,6 @@ db: # -- the schema in which the cluster database config entities will be stored, passed to workers on startup schema: CONFIG -# State Manager configuration -stateManager: - # Type of State Manager - type: DATABASE - # State Manager database configuration - db: - # -- the State Manager database host - host: null - # -- the State Manager database type - type: "postgresql" - # -- the State Manager database port - port: 5432 - # the State Manager database user configuration - username: - # -- the State Manager database user - value: "" - # the State Manager database user secret configuration; used in preference to value if name is set - valueFrom: - # the State Manager database user secret key reference - secretKeyRef: - # -- the State Manager database user secret name - name: "" - # -- the State Manager database user secret key - key: "" - # the State Manager database password configuration - password: - # -- the State Manager database password - value: "" - # the State Manager database password secret configuration; used in preference to value if name is set - valueFrom: - # the State Manager database password secret key reference - secretKeyRef: - # -- the State Manager database password secret name - name: "" - # -- the State Manager database password secret key - key: "" - # -- the State Manager database name - database: state-manager - # Kafka configuration kafka: # -- comma-separated list of Kafka bootstrap servers (required); for example: "broker1.example.com:9092,broker2.example.com:9092,broker3.example.com:9092" @@ -254,7 +215,7 @@ kafka: # Configuration for cluster bootstrap bootstrap: - # Configuration for the preinstall check + # Configuration for the pre-installation check preinstallCheck: # -- indicates whether the preinstall check is enabled enabled: true @@ -264,6 +225,7 @@ bootstrap: create: true # -- name of the service account, if not set and create is true, a name is generated from a template name: "" + # Configuration for the REST API admin to be created during cluster bootstrap restApiAdmin: # the username configuration for the REST API admin to be created during cluster bootstrap @@ -290,8 +252,12 @@ bootstrap: name: "" # -- the key for the field in the secret containing the password for the REST API admin created during cluster bootstrap key: "" + # Configuration for database bootstrap db: + # -- Indicates whether DB bootstrap is enabled as part of installation + enabled: true + # Bootstrap cluster database configuration cluster: # the bootstrap cluster database password configuration @@ -318,8 +284,7 @@ bootstrap: name: "" # -- the bootstrap cluster database user secret key key: "" - # -- Indicates whether DB bootstrap is enabled as part of installation - enabled: true + # crypto configuration crypto: # -- the schema in which the cluster database crypto entities will be stored @@ -408,30 +373,56 @@ bootstrap: # State Manager DB Bootstrap Configuration stateManager: - # the username configuration - username: - # -- the username, defaults to this value - value: "state_manager_user" - # the username secret configuration; used in preference to value if name is set - valueFrom: - # the username secret key reference - secretKeyRef: - # -- the username secret name - name: "" - # -- the username secret key - key: "" - # password configuration - password: - # -- the password, defaults to a value randomly-generated on install - value: "" - # the password secret configuration - valueFrom: - # the password secret key reference - secretKeyRef: - # -- the password secret name - name: "" - # -- the password secret key - key: "" + flow: + # the username configuration + username: + # -- the username, defaults to this value + value: "" + # the username secret configuration; used in preference to value if name is set + valueFrom: + # the username secret key reference + secretKeyRef: + # -- the username secret name + name: "" + # -- the username secret key + key: "" + # password configuration + password: + # -- the password, defaults to a value randomly-generated on install + value: "" + # the password secret configuration + valueFrom: + # the password secret key reference + secretKeyRef: + # -- the password secret name + name: "" + # -- the password secret key + key: "" + flowMapper: + # the username configuration + username: + # -- the username, defaults to this value + value: "" + # the username secret configuration; used in preference to value if name is set + valueFrom: + # the username secret key reference + secretKeyRef: + # -- the username secret name + name: "" + # -- the username secret key + key: "" + # password configuration + password: + # -- the password, defaults to a value randomly-generated on install + value: "" + # the password secret configuration + valueFrom: + # the password secret key reference + secretKeyRef: + # -- the password secret name + name: "" + # -- the password secret key + key: "" # Image containing DB client, used to set up the DB clientImage: @@ -738,20 +729,58 @@ workers: limits: {} # -- run flow worker with Quasar's verifyInstrumentation enabled verifyInstrumentation: false - # flow worker JDBC connection pool configuration for State Manager DB - stateManagerDbConnectionPool: - # -- flow worker maximum JDBC connection pool size for state manager DB - maxSize: 5 - # -- flow worker minimum JDBC connection pool size for state manager DB; null value means pool's min size will default to pool's max size value - minSize: null - # -- maximum time (in seconds) a connection can stay idle in the pool; A value of 0 means that idle connections are never removed from the pool - idleTimeoutSeconds: 120 - # -- maximum time (in seconds) a connection can stay in the pool, regardless if it has been idle or has been recently used; If a connection is in-use and has reached "maxLifetime" timeout, it will be removed from the pool only when it becomes idle - maxLifetimeSeconds: 1800 - # -- interval time (in seconds) in which connections will be tested for aliveness; Connections which are no longer alive are removed from the pool; A value of 0 means this check is disabled - keepAliveTimeSeconds: 0 - # -- maximum time (in seconds) that the pool will wait for a connection to be validated as alive - validationTimeoutSeconds: 5 + # Flow Worker State Manager configuration + stateManager: + # Type of State Manager + type: DATABASE + # State Manager database configuration + db: + # -- the State Manager database host + host: null + # -- the State Manager database type + type: "postgresql" + # -- the State Manager database port + port: 5432 + # -- the State Manager database name + name: flow_state_manager + # the State Manager database user configuration + username: + # -- the State Manager database user + value: "flow_state_manager_user" + # the State Manager database user secret configuration; used in preference to value if name is set + valueFrom: + # the State Manager database user secret key reference + secretKeyRef: + # -- the State Manager database user secret name + name: "" + # -- the State Manager database user secret key + key: "" + # the State Manager database password configuration + password: + # -- the State Manager database password + value: "" + # the State Manager database password secret configuration; used in preference to value if name is set + valueFrom: + # the State Manager database password secret key reference + secretKeyRef: + # -- the State Manager database password secret name + name: "" + # -- the State Manager database password secret key + key: "" + # flow worker JDBC connection pool configuration for State Manager DB + connectionPool: + # -- flow worker maximum JDBC connection pool size for state manager DB + maxSize: 5 + # -- flow worker minimum JDBC connection pool size for state manager DB; null value means pool's min size will default to pool's max size value + minSize: null + # -- maximum time (in seconds) a connection can stay idle in the pool; A value of 0 means that idle connections are never removed from the pool + idleTimeoutSeconds: 120 + # -- maximum time (in seconds) a connection can stay in the pool, regardless if it has been idle or has been recently used; If a connection is in-use and has reached "maxLifetime" timeout, it will be removed from the pool only when it becomes idle + maxLifetimeSeconds: 1800 + # -- interval time (in seconds) in which connections will be tested for aliveness; Connections which are no longer alive are removed from the pool; A value of 0 means this check is disabled + keepAliveTimeSeconds: 0 + # -- maximum time (in seconds) that the pool will wait for a connection to be validated as alive + validationTimeoutSeconds: 5 # flow worker Kafka configuration kafka: # if kafka.sasl.enabled, the credentials to connect to Kafka with for the flow worker @@ -820,20 +849,58 @@ workers: limits: { } # -- run flow mapper worker with Quasar's verifyInstrumentation enabled verifyInstrumentation: false - # flow mapper worker JDBC connection pool configuration for State Manager DB - stateManagerDbConnectionPool: - # -- flow mapper worker maximum JDBC connection pool size for state manager DB - maxSize: 5 - # -- flow mapper worker minimum JDBC connection pool size for state manager DB; null value means pool's min size will default to pool's max size value - minSize: null - # -- maximum time (in seconds) a connection can stay idle in the pool; A value of 0 means that idle connections are never removed from the pool - idleTimeoutSeconds: 120 - # -- maximum time (in seconds) a connection can stay in the pool, regardless if it has been idle or has been recently used; If a connection is in-use and has reached "maxLifetime" timeout, it will be removed from the pool only when it becomes idle - maxLifetimeSeconds: 1800 - # -- interval time (in seconds) in which connections will be tested for aliveness; Connections which are no longer alive are removed from the pool; A value of 0 means this check is disabled - keepAliveTimeSeconds: 0 - # -- maximum time (in seconds) that the pool will wait for a connection to be validated as alive - validationTimeoutSeconds: 5 + # Flow Mapper Worker State Manager configuration + stateManager: + # Type of State Manager + type: DATABASE + # State Manager database configuration + db: + # -- the State Manager database host + host: null + # -- the State Manager database type + type: "postgresql" + # -- the State Manager database port + port: 5432 + # -- the State Manager database name + name: flow_mapper_state_manager + # the State Manager database user configuration + username: + # -- the State Manager database user + value: "flow_mapper_state_manager_user" + # the State Manager database user secret configuration; used in preference to value if name is set + valueFrom: + # the State Manager database user secret key reference + secretKeyRef: + # -- the State Manager database user secret name + name: "" + # -- the State Manager database user secret key + key: "" + # the State Manager database password configuration + password: + # -- the State Manager database password + value: "" + # the State Manager database password secret configuration; used in preference to value if name is set + valueFrom: + # the State Manager database password secret key reference + secretKeyRef: + # -- the State Manager database password secret name + name: "" + # -- the State Manager database password secret key + key: "" + # flow mapper worker JDBC connection pool configuration for State Manager DB + connectionPool: + # -- flow mapper worker maximum JDBC connection pool size for state manager DB + maxSize: 5 + # -- flow mapper worker minimum JDBC connection pool size for state manager DB; null value means pool's min size will default to pool's max size value + minSize: null + # -- maximum time (in seconds) a connection can stay idle in the pool; A value of 0 means that idle connections are never removed from the pool + idleTimeoutSeconds: 120 + # -- maximum time (in seconds) a connection can stay in the pool, regardless if it has been idle or has been recently used; If a connection is in-use and has reached "maxLifetime" timeout, it will be removed from the pool only when it becomes idle + maxLifetimeSeconds: 1800 + # -- interval time (in seconds) in which connections will be tested for aliveness; Connections which are no longer alive are removed from the pool; A value of 0 means this check is disabled + keepAliveTimeSeconds: 0 + # -- maximum time (in seconds) that the pool will wait for a connection to be validated as alive + validationTimeoutSeconds: 5 # flow mapper worker Kafka configuration kafka: # if kafka.sasl.enabled, the credentials to connect to Kafka with for the flow mapper worker diff --git a/state-manager.yaml b/state-manager.yaml index 5f928b5fd42..8418b7ec65c 100644 --- a/state-manager.yaml +++ b/state-manager.yaml @@ -30,20 +30,42 @@ bootstrap: db: stateManager: - password: - valueFrom: - secretKeyRef: - name: "state-manager-db-postgresql" - key: "postgres-password" - username: - value: "postgres" -stateManager: - db: - host: "state-manager-db-postgresql" - password: - valueFrom: - secretKeyRef: - name: "state-manager-db-postgresql" - key: "password" - username: - value: "statemanager-user" + flow: + username: + value: "postgres" + password: + valueFrom: + secretKeyRef: + name: "state-manager-db-postgresql" + key: "postgres-password" + flowMapper: + username: + value: "postgres" + password: + valueFrom: + secretKeyRef: + name: "state-manager-db-postgresql" + key: "postgres-password" +workers: + flow: + stateManager: + db: + host: "state-manager-db-postgresql" + username: + value: "statemanager-user" + password: + valueFrom: + secretKeyRef: + name: "state-manager-db-postgresql" + key: "password" + flowMapper: + stateManager: + db: + host: "state-manager-db-postgresql" + username: + value: "statemanager-user" + password: + valueFrom: + secretKeyRef: + name: "state-manager-db-postgresql" + key: "password" From f36fa7fca37b1cd6ddb3974eaec5e6961277c593 Mon Sep 17 00:00:00 2001 From: Joseph Zuniga-Daly Date: Mon, 23 Oct 2023 16:19:28 +0100 Subject: [PATCH 60/81] CORE-17834: Allow empty responses from HTTP RPC (#4937) * CORE-17834: Allow empty responses from HTTP RPC * CORE-17836: Adds the request message key as a custom HTTP header Co-authored-by: Viktor Kolomeyko --- .../net/corda/messaging/mediator/RPCClient.kt | 21 +++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/RPCClient.kt b/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/RPCClient.kt index eaed1ed3b00..886b3e92c60 100644 --- a/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/RPCClient.kt +++ b/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/RPCClient.kt @@ -12,12 +12,15 @@ import net.corda.messaging.api.exception.CordaHTTPServerErrorException import net.corda.messaging.api.mediator.MediatorMessage import net.corda.messaging.api.mediator.MessagingClient import net.corda.messaging.api.mediator.MessagingClient.Companion.MSG_PROP_ENDPOINT +import net.corda.messaging.api.mediator.MessagingClient.Companion.MSG_PROP_KEY import net.corda.messaging.utils.HTTPRetryConfig import net.corda.messaging.utils.HTTPRetryExecutor import net.corda.utilities.trace import org.slf4j.Logger import org.slf4j.LoggerFactory +const val CORDA_REQUEST_KEY_HEADER = "corda-request-key" + class RPCClient( override val id: String, cordaAvroSerializerFactory: CordaAvroSerializationFactory, @@ -55,9 +58,12 @@ class RPCClient( } - private fun deserializePayload(payload: ByteArray): Any { + private fun deserializePayload(payload: ByteArray): Any? { return try { - deserializer.deserialize(payload)!! + when { + payload.isEmpty() -> null + else -> deserializer.deserialize(payload) + } } catch (e: Exception) { val errorMsg = "Failed to deserialize payload of size ${payload.size} bytes due to: ${e.message}" log.warn(errorMsg, e) @@ -67,10 +73,17 @@ class RPCClient( } private fun buildHttpRequest(message: MediatorMessage<*>): HttpRequest { - return HttpRequest.newBuilder() + + val builder = HttpRequest.newBuilder() .uri(URI(message.endpoint())) .POST(HttpRequest.BodyPublishers.ofByteArray(message.payload as ByteArray)) - .build() + + // Add key HTTP header + message.getPropertyOrNull(MSG_PROP_KEY)?.let { value -> + builder.header(CORDA_REQUEST_KEY_HEADER, value.toString()) + } + + return builder.build() } private fun sendWithRetry(request: HttpRequest): HttpResponse { From 8b7e33a458e4a211d3290ea0d87828f9dce79072 Mon Sep 17 00:00:00 2001 From: Filipe Oliveira Date: Mon, 23 Oct 2023 17:53:50 +0100 Subject: [PATCH 61/81] CORE-17867 - Added logging to `VirtualNodeWriteEventHandler` (#4959) An e2e test is flaky and extra logging is required in order to understand what might be causing issues. --- .../write/db/impl/VirtualNodeWriteEventHandler.kt | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/components/virtual-node/virtual-node-write-service-impl/src/main/kotlin/net/corda/virtualnode/write/db/impl/VirtualNodeWriteEventHandler.kt b/components/virtual-node/virtual-node-write-service-impl/src/main/kotlin/net/corda/virtualnode/write/db/impl/VirtualNodeWriteEventHandler.kt index 82adfcb0a1d..689d24f48ac 100644 --- a/components/virtual-node/virtual-node-write-service-impl/src/main/kotlin/net/corda/virtualnode/write/db/impl/VirtualNodeWriteEventHandler.kt +++ b/components/virtual-node/virtual-node-write-service-impl/src/main/kotlin/net/corda/virtualnode/write/db/impl/VirtualNodeWriteEventHandler.kt @@ -17,6 +17,7 @@ import net.corda.schema.configuration.ConfigKeys import net.corda.virtualnode.write.db.VirtualNodeWriteServiceException import net.corda.virtualnode.write.db.impl.writer.VirtualNodeWriter import net.corda.virtualnode.write.db.impl.writer.VirtualNodeWriterFactory +import org.slf4j.LoggerFactory /** Handles incoming [LifecycleCoordinator] events for [VirtualNodeWriteServiceImpl]. */ internal class VirtualNodeWriteEventHandler( @@ -24,6 +25,10 @@ internal class VirtualNodeWriteEventHandler( private val virtualNodeWriterFactory: VirtualNodeWriterFactory ) : LifecycleEventHandler { + private companion object { + val logger = LoggerFactory.getLogger(this::class.java.enclosingClass) + } + private var registrationHandle: AutoCloseable? = null private var configUpdateHandle: AutoCloseable? = null internal var virtualNodeWriter: VirtualNodeWriter? = null @@ -46,11 +51,14 @@ internal class VirtualNodeWriteEventHandler( val externalMsgConfig = event.config.getConfig(ConfigKeys.EXTERNAL_MESSAGING_CONFIG) val vnodeDatasourceConfig = event.config.getConfig(ConfigKeys.VNODE_DATASOURCE_CONFIG) + logger.info("Configuration changed event received") try { virtualNodeWriter?.close() + logger.info("Current virtual node writer has been closed") virtualNodeWriter = virtualNodeWriterFactory .create(msgConfig, externalMsgConfig, vnodeDatasourceConfig) .apply { start() } + logger.info("New virtual node write has been created") coordinator.updateStatus(UP) } catch (e: Exception) { From 565200371a64e3cacccf70be9fa3599ebd35beca Mon Sep 17 00:00:00 2001 From: Ronan Browne Date: Tue, 24 Oct 2023 11:55:41 +0100 Subject: [PATCH 62/81] ES-1524: Remove Jenkins MultiCluster Test file from this repo (#4960) --- .ci/e2eTests/JenkinsfileMultiClusterTest | 10 ---------- 1 file changed, 10 deletions(-) delete mode 100644 .ci/e2eTests/JenkinsfileMultiClusterTest diff --git a/.ci/e2eTests/JenkinsfileMultiClusterTest b/.ci/e2eTests/JenkinsfileMultiClusterTest deleted file mode 100644 index ba8376e3abb..00000000000 --- a/.ci/e2eTests/JenkinsfileMultiClusterTest +++ /dev/null @@ -1,10 +0,0 @@ -@Library('corda-shared-build-pipeline-steps@5.1') _ - -endToEndPipeline( - dailyBuildCron: '0 */12 * * *', - multiCluster: true, - gradleTestTargetsToExecute: ['smokeTest', 'e2eTest'], - usePackagedCordaHelmChart: false, - gradleAdditionalArgs : '-PrunMultiClusterTests -Dscan.tag.MultiCluster', - javaVersion: '17' -) From 549847b3cbe89cda625fa27972c48e45c0c5f8ab Mon Sep 17 00:00:00 2001 From: David Currie Date: Tue, 24 Oct 2023 13:35:24 +0100 Subject: [PATCH 63/81] CORE-17899 Add nginx for token selection sharding (#4950) --- charts/corda-lib/templates/_helpers.tpl | 47 +++- charts/corda-lib/templates/_nginx.tpl | 286 ++++++++++++++++++++++++ charts/corda-lib/templates/_worker.tpl | 66 +++--- charts/corda/templates/workers.yaml | 2 +- charts/corda/values.schema.json | 82 +++++++ charts/corda/values.yaml | 14 ++ 6 files changed, 455 insertions(+), 42 deletions(-) create mode 100644 charts/corda-lib/templates/_nginx.tpl diff --git a/charts/corda-lib/templates/_helpers.tpl b/charts/corda-lib/templates/_helpers.tpl index 8112e7a9450..444765b08ab 100644 --- a/charts/corda-lib/templates/_helpers.tpl +++ b/charts/corda-lib/templates/_helpers.tpl @@ -650,7 +650,48 @@ Get the endpoint argument for a given worker {{- define "corda.getWorkerEndpoint" }} {{- $context := .context }} {{- $worker := .worker }} -{{- $workerName := printf "%s-%s-worker" (include "corda.fullname" $context) (include "corda.workerTypeKebabCase" $worker) }} -{{- $workerServiceName := include "corda.workerInternalServiceName" $workerName }} -{{- printf "endpoints.%s=%s:%s" $worker $workerServiceName (include "corda.workerServicePort" $) }} +{{- $workerValues := ( index $context.Values.workers $worker ) }} +{{- $workerName := printf "%s-%s-worker" ( include "corda.fullname" $context ) ( include "corda.workerTypeKebabCase" $worker ) }} +{{- $workerServiceName := "" }} +{{- if ( ( $workerValues.sharding ).enabled ) }} +{{- $workerServiceName = include "corda.nginxName" $workerName }} +{{- else }} +{{- $workerServiceName = include "corda.workerInternalServiceName" $workerName }} +{{- end }} +{{- printf "endpoints.%s=%s:%s" $worker $workerServiceName ( include "corda.workerServicePort" $context ) }} +{{- end }} + +{{/* +Default pod affinity +*/}} +{{- define "corda.defaultAffinity" -}} +{{- $weight := index . 0 }} +{{- $component := index . 1 }} +weight: {{ $weight}} +podAffinityTerm: + labelSelector: + matchExpressions: + - key: "app.kubernetes.io/component" + operator: In + values: + - {{ $component | quote }} + topologyKey: "kubernetes.io/hostname" +{{- end }} + +{{/* +Pod affinity +*/}} +{{- define "corda.affinity" -}} +{{- $ := index . 0 }} +{{- $component := index . 1 }} +{{- $affinity := default ( deepCopy $.Values.affinity ) dict }} +{{- if not ($affinity.podAntiAffinity) }} +{{- $_ := set $affinity "podAntiAffinity" dict }} +{{- end }} +{{- if not ($affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution) }} +{{- $_ := set $affinity.podAntiAffinity "preferredDuringSchedulingIgnoredDuringExecution" list }} +{{- end }} +{{- $_ := set $affinity.podAntiAffinity "preferredDuringSchedulingIgnoredDuringExecution" ( append $affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution ( fromYaml ( include "corda.defaultAffinity" ( list ( add ( len $affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution ) 1 ) $component ) ) ) ) }} +affinity: +{{- toYaml $affinity | nindent 2 }} {{- end }} diff --git a/charts/corda-lib/templates/_nginx.tpl b/charts/corda-lib/templates/_nginx.tpl new file mode 100644 index 00000000000..744ec3c4cb2 --- /dev/null +++ b/charts/corda-lib/templates/_nginx.tpl @@ -0,0 +1,286 @@ +{{- define "corda.nginxName" -}} +{{- printf "%s-nginx" . }} +{{- end }} + +{{- define "corda.nginxComponent" -}} +{{ printf "%s-nginx" . }} +{{- end }} + +{{- define "corda.nginxLabels" -}} +{{- $workerName := index . 1 }} +{{- with ( index . 0 ) }} +{{- include "corda.labels" . }} +app.kubernetes.io/component: {{ include "corda.nginxComponent" $workerName }} +{{- end }} +{{- end }} + +{{- define "corda.nginxSelectorLabels" -}} +{{- $workerName := index . 1 }} +{{- with ( index . 0 ) }} +{{- include "corda.selectorLabels" . }} +app.kubernetes.io/component: {{ include "corda.nginxComponent" $workerName }} +{{- end }} +{{- end }} + +{{- define "corda.nginx" }} +{{- $workerName := index . 1 }} +{{- $shardingConfig := index . 2 }} +{{- with ( index . 0 ) }} +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + labels: + {{- include "corda.nginxLabels" ( list . $workerName ) | nindent 4 }} + name: {{ include "corda.nginxName" $workerName | quote }} +spec: + selector: + matchLabels: + {{ include "corda.nginxSelectorLabels" ( list . $workerName ) | nindent 6 }} + minAvailable: 1 +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + {{- include "corda.nginxLabels" ( list . $workerName ) | nindent 4 }} + name: {{ include "corda.nginxName" $workerName | quote }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + labels: + {{- include "corda.nginxLabels" ( list . $workerName ) | nindent 4 }} + name: {{ include "corda.nginxName" $workerName | quote }} +data: + allow-snippet-annotations: "false" +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + labels: + {{- include "corda.nginxLabels" ( list . $workerName ) | nindent 4 }} + name: {{ include "corda.nginxName" $workerName | quote }} +rules: + - apiGroups: + - "" + resources: + - namespaces + verbs: + - get + - apiGroups: + - "" + resources: + - configmaps + - pods + - secrets + - endpoints + verbs: + - get + - list + - watch + - apiGroups: + - "" + resources: + - services + verbs: + - get + - list + - watch + - apiGroups: + - networking.k8s.io + resources: + - ingresses + verbs: + - get + - list + - watch + - apiGroups: + - networking.k8s.io + resources: + - ingresses/status + verbs: + - update + - apiGroups: + - networking.k8s.io + resources: + - ingressclasses + verbs: + - get + - list + - watch + - apiGroups: + - coordination.k8s.io + resources: + - leases + resourceNames: + - {{ include "corda.nginxName" $workerName }}-leader + verbs: + - get + - update + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - create + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - list + - watch + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + {{- include "corda.nginxLabels" ( list . $workerName ) | nindent 4 }} + name: {{ include "corda.nginxName" $workerName | quote }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ include "corda.nginxName" $workerName | quote }} +subjects: + - kind: ServiceAccount + name: {{ include "corda.nginxName" $workerName | quote }} + namespace: {{ .Release.Namespace | quote }} +--- +apiVersion: v1 +kind: Service +metadata: + labels: + {{- include "corda.nginxLabels" ( list . $workerName ) | nindent 4 }} + name: {{ include "corda.nginxName" $workerName | quote }} +spec: + type: ClusterIP + ipFamilyPolicy: SingleStack + ipFamilies: + - IPv4 + ports: + - name: http + port: {{ include "corda.workerServicePort" ( list . $workerName ) }} + protocol: TCP + targetPort: http + appProtocol: http + selector: + {{ include "corda.nginxSelectorLabels" ( list . $workerName ) | nindent 4 }} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + {{- include "corda.nginxLabels" ( list . $workerName ) | nindent 4 }} + name: {{ include "corda.nginxName" $workerName | quote }} +spec: + selector: + matchLabels: + {{ include "corda.nginxSelectorLabels" ( list . $workerName ) | nindent 6 }} + replicas: {{ $shardingConfig.replicaCount }} + revisionHistoryLimit: 10 + minReadySeconds: 0 + template: + metadata: + labels: + {{- include "corda.nginxLabels" ( list . $workerName ) | nindent 8 }} + {{- with .Values.annotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + dnsPolicy: ClusterFirst + {{- with .Values.podSecurityContext }} + securityContext: + {{- . | toYaml | nindent 8 }} + {{- end }} + {{- include "corda.imagePullSecrets" . | indent 6 }} + {{- include "corda.tolerations" . | indent 6 }} + {{- with .Values.serviceAccount.name }} + serviceAccountName: {{ . }} + {{- end }} + {{- include "corda.topologySpreadConstraints" . | indent 6 }} + {{- include "corda.affinity" (list . ( include "corda.nginxComponent" $workerName ) ) | indent 6 }} + containers: + - name: controller + {{- with $shardingConfig.image }} + image: {{ ( printf "%s/%s:%s" .registry .repository .tag ) | quote }} + {{- end }} + imagePullPolicy: {{ .Values.imagePullPolicy }} + securityContext: + {{- with .Values.containerSecurityContext -}} + {{- . | toYaml | nindent 12}} + {{- end }} + capabilities: + add: + - NET_BIND_SERVICE + readOnlyRootFilesystem: false + runAsUser: 101 + lifecycle: + preStop: + exec: + command: + - /wait-shutdown + args: + - /nginx-ingress-controller + - --publish-service=$(POD_NAMESPACE)/{{ include "corda.nginxName" $workerName }} + - --election-id={{ include "corda.nginxName" $workerName }}-leader + - --controller-class=k8s.io/{{ include "corda.nginxName" $workerName }} + - --ingress-class={{ include "corda.nginxName" $workerName }} + - --configmap=$(POD_NAMESPACE)/{{ include "corda.nginxName" $workerName }} + - --watch-namespace=$(POD_NAMESPACE) + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: LD_PRELOAD + value: /usr/local/lib/libmimalloc.so + livenessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: 10254 + scheme: HTTP + initialDelaySeconds: 10 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + readinessProbe: + failureThreshold: 3 + httpGet: + path: /healthz + port: 10254 + scheme: HTTP + initialDelaySeconds: 10 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + ports: + - name: http + containerPort: 80 + protocol: TCP + resources: + requests: + cpu: 100m + memory: 90Mi + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "corda.nginxName" $workerName | quote }} + terminationGracePeriodSeconds: 300 +{{- end }} +{{- end }} diff --git a/charts/corda-lib/templates/_worker.tpl b/charts/corda-lib/templates/_worker.tpl index 6bb13316b1d..233e30cfbb1 100644 --- a/charts/corda-lib/templates/_worker.tpl +++ b/charts/corda-lib/templates/_worker.tpl @@ -8,6 +8,9 @@ Worker deployment. {{- $optionalArgs := dict }} {{- if gt (len .) 3 }}{{ $optionalArgs = index . 3 }}{{ end }} {{- with index . 1 }} +{{- if ( ( .sharding ).enabled ) }} + {{- include "corda.nginx" ( list $ $workerName .sharding ) }} +{{- end }} {{- with .ingress }} {{- if gt (len .hosts) 0 }} --- @@ -46,6 +49,29 @@ spec: {{- end }} {{- end }} {{- end }} +{{- with .sharding }} +{{- if .enabled }} +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: {{ ( printf "%s-sharded" $workerName ) | quote }} + labels: + {{- include "corda.workerLabels" ( list $ $worker ) | nindent 4 }} + annotations: + {{- with .annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + kubernetes.io/ingress.class: {{ include "corda.nginxName" $workerName | quote }} + nginx.ingress.kubernetes.io/upstream-hash-by: "$http_corda_request_key" +spec: + defaultBackend: + service: + name: {{ include "corda.workerInternalServiceName" $workerName | quote }} + port: + name: monitor +{{- end }} +{{- end }} {{- with .service }} --- apiVersion: v1 @@ -130,7 +156,7 @@ spec: serviceAccountName: {{ . }} {{- end }} {{- include "corda.topologySpreadConstraints" $ | indent 6 }} - {{- include "corda.affinity" (list $ . $worker ) | indent 6 }} + {{- include "corda.affinity" (list $ ( include "corda.workerComponent" $worker ) ) | indent 6 }} containers: - name: {{ $workerName | quote }} image: {{ include "corda.workerImage" ( list $ . ) }} @@ -325,8 +351,7 @@ spec: {{- end }} {{- if $optionalArgs.servicesAccessed }} {{- range $worker := $optionalArgs.servicesAccessed }} - {{- $endpoint := include "corda.getWorkerEndpoint" (dict "context" $ "worker" $worker) }} - - --serviceEndpoint={{ $endpoint }} + - "--serviceEndpoint={{ include "corda.getWorkerEndpoint" (dict "context" $ "worker" $worker) }}" {{- end }} {{- end }} {{- range $i, $arg := $optionalArgs.additionalWorkerArgs }} @@ -490,38 +515,3 @@ Worker image {{- printf "%s/%s:%s" ( .image.registry | default $.Values.image.registry ) ( .image.repository ) ( .image.tag | default $.Values.image.tag | default $.Chart.AppVersion ) | quote }} {{- end }} {{- end }} - -{{/* -Worker default affinity -*/}} -{{- define "corda.defaultAffinity" -}} -{{- $weight := index . 0 }} -{{- $worker := index . 1 }} -weight: {{ $weight}} -podAffinityTerm: - labelSelector: - matchExpressions: - - key: "app.kubernetes.io/component" - operator: In - values: - - {{ include "corda.workerComponent" $worker }} - topologyKey: "kubernetes.io/hostname" -{{- end }} - -{{/* -Worker affinity -*/}} -{{- define "corda.affinity" -}} -{{- $ := index . 0 }} -{{- $worker := index . 2 }} -{{- $affinity := default ( deepCopy $.Values.affinity ) dict }} -{{- if not ($affinity.podAntiAffinity) }} -{{- $_ := set $affinity "podAntiAffinity" dict }} -{{- end }} -{{- if not ($affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution) }} -{{- $_ := set $affinity.podAntiAffinity "preferredDuringSchedulingIgnoredDuringExecution" list }} -{{- end }} -{{- $_ := set $affinity.podAntiAffinity "preferredDuringSchedulingIgnoredDuringExecution" ( append $affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution ( fromYaml ( include "corda.defaultAffinity" ( list ( add ( len $affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution ) 1 ) $worker ) ) ) ) }} -affinity: -{{- toYaml $affinity | nindent 2 }} -{{- end }} diff --git a/charts/corda/templates/workers.yaml b/charts/corda/templates/workers.yaml index 63d598925c7..7ddb4245965 100644 --- a/charts/corda/templates/workers.yaml +++ b/charts/corda/templates/workers.yaml @@ -5,7 +5,7 @@ ( dict "clusterDbAccess" true ) ) }} {{- include "corda.worker" ( list $ .Values.workers.flow "flow" - ( dict "stateManagerDbAccess" true "servicesAccessed" (list "crypto" "verification" "uniqueness" "persistence") ) + ( dict "stateManagerDbAccess" true "servicesAccessed" ( list "crypto" "verification" "uniqueness" "persistence" "tokenSelection" ) ) ) }} {{- include "corda.worker" ( list $ .Values.workers.flowMapper "flowMapper" ( dict "stateManagerDbAccess" true ) diff --git a/charts/corda/values.schema.json b/charts/corda/values.schema.json index ae7424561d4..f233c892b80 100644 --- a/charts/corda/values.schema.json +++ b/charts/corda/values.schema.json @@ -2581,6 +2581,88 @@ "profiling": {}, "resources": {}, "kafka": {}, + "sharding": { + "type": "object", + "default": { + "enabled": false, + "image": { + "registry": "registry.k8s.io", + "repository": "ingress-nginx/controller", + "tag": "v1.9.3@sha256:8fd21d59428507671ce0fb47f818b1d859c92d2ad07bb7c947268d433030ba98" + }, + "replicaCount": 2 + }, + "title": "token selection sharding configuration", + "required": [ + "enabled", + "image", + "replicaCount" + ], + "additionalProperties": false, + "properties": { + "enabled": { + "type": "boolean", + "default": "false", + "title": "enable sharding of requests" + }, + "image": { + "type": "object", + "default": { + "registry": "registry.k8s.io", + "repository": "ingress-nginx/controller", + "tag": "v1.9.3@sha256:8fd21d59428507671ce0fb47f818b1d859c92d2ad07bb7c947268d433030ba98" + }, + "title": "sharding image configuration", + "required": [ + "registry", + "repository", + "tag" + ], + "additionalProperties": false, + "properties": { + "registry": { + "type": "string", + "default": "registry.k8s.io", + "title": "sharding image registry", + "examples": [ + "registry.k8s.io" + ], + "minLength": 1 + }, + "repository": { + "type": "string", + "title": "sharding image repository", + "examples": [ + "ingress-nginx/controller" + ], + "minLength": 1 + }, + "tag": { + "type": "string", + "default": "v1.9.3@sha256:8fd21d59428507671ce0fb47f818b1d859c92d2ad07bb7c947268d433030ba98", + "title": "sharding image tag", + "examples": [ + "v1.9.3@sha256:8fd21d59428507671ce0fb47f818b1d859c92d2ad07bb7c947268d433030ba98" + ], + "minLength": 1 + } + }, + "examples": [{ + "registry": "corda-os-docker.software.r3.com", + "repository": "corda-os-xxx-worker", + "tag": "" + }] + }, + "replicaCount": { + "type": "integer", + "default": 2, + "title": "sharding replica count", + "examples": [ + 2 + ] + } + } + }, "clusterDbConnectionPool": { "type": "object", "default": {}, diff --git a/charts/corda/values.yaml b/charts/corda/values.yaml index fc38165d694..5a52b291e52 100644 --- a/charts/corda/values.yaml +++ b/charts/corda/values.yaml @@ -1473,6 +1473,20 @@ workers: name: "" # -- the token selection worker SASL password secret key for client connection to Kafka key: "" + # token selection sharding configuration + sharding: + # -- enable sharding of token selection requests + enabled: false + # token selection sharding image configuration + image: + # -- token selection sharding image registry + registry: "registry.k8s.io" + # -- token selection sharding image repository + repository: "ingress-nginx/controller" + # -- token selection sharding image tag + tag: "v1.9.3@sha256:8fd21d59428507671ce0fb47f818b1d859c92d2ad07bb7c947268d433030ba98" + # -- token selection replica count + replicaCount: 2 # uniqueness worker configuration uniqueness: From 2b824f65ff4200d04c1caf3844946bf464084849 Mon Sep 17 00:00:00 2001 From: Owen Stanford <92725587+owenstanford@users.noreply.github.com> Date: Wed, 25 Oct 2023 08:21:21 +0100 Subject: [PATCH 64/81] CORE-17502 - Helm changes to support token selection + state manager (#4966) CORE-17502 - Helm changes to support token selection + state manager --- ...JenkinsfileCombinedWorkerPluginsSmokeTests | 2 +- .ci/JenkinsfileSnykDelta | 2 +- .../Jenkinsfile_PublishHelmChart | 2 +- .ci/dev/forward-merge/JenkinsInteropMerge | 2 +- .ci/e2eTests/Jenkinsfile | 2 +- .ci/e2eTests/JenkinsfileCombinedWorker | 2 +- .ci/e2eTests/JenkinsfileCombinedWorkerLinux | 2 +- .ci/e2eTests/JenkinsfileCombinedWorkerWindows | 2 +- .ci/e2eTests/JenkinsfileEnterpriseSmokeTests | 2 +- .ci/e2eTests/JenkinsfileUnstableTest | 2 +- .ci/e2eTests/corda.yaml | 16 ++++ .ci/e2eTests/ethereum/Jenkinsfile | 2 +- .ci/nightly/JenkinsfileNightly | 2 +- .ci/nightly/JenkinsfileSnykScan | 2 +- .ci/nightly/JenkinsfileUnstableTests | 2 +- .ci/nightly/JenkinsfileWindowsCompatibility | 2 +- .ci/versionCompatibility/Jenkinsfile | 2 +- .../latest-version.Jenkinsfile | 2 +- Jenkinsfile | 2 +- charts/corda/templates/workers.yaml | 2 +- charts/corda/values.schema.json | 10 +++ charts/corda/values.yaml | 79 ++++++++++++++++++- state-manager.yaml | 12 +++ 23 files changed, 135 insertions(+), 20 deletions(-) diff --git a/.ci/JenkinsfileCombinedWorkerPluginsSmokeTests b/.ci/JenkinsfileCombinedWorkerPluginsSmokeTests index 2d3c912e5a4..10d3a447d2d 100644 --- a/.ci/JenkinsfileCombinedWorkerPluginsSmokeTests +++ b/.ci/JenkinsfileCombinedWorkerPluginsSmokeTests @@ -1,4 +1,4 @@ -@Library('corda-shared-build-pipeline-steps@5.1') _ +@Library('corda-shared-build-pipeline-steps@owen/CORE-17957-Add-state-manager-support') _ import groovy.transform.Field import com.r3.build.utils.PipelineUtils diff --git a/.ci/JenkinsfileSnykDelta b/.ci/JenkinsfileSnykDelta index 83b6279ecee..e58dfa2049f 100644 --- a/.ci/JenkinsfileSnykDelta +++ b/.ci/JenkinsfileSnykDelta @@ -1,4 +1,4 @@ -@Library('corda-shared-build-pipeline-steps@5.1') _ +@Library('corda-shared-build-pipeline-steps@owen/CORE-17957-Add-state-manager-support') _ snykDelta( snykOrgId: 'corda5-snyk-org-id', diff --git a/.ci/PublishHelmChart/Jenkinsfile_PublishHelmChart b/.ci/PublishHelmChart/Jenkinsfile_PublishHelmChart index 71e387d7bc0..dca17c5e735 100644 --- a/.ci/PublishHelmChart/Jenkinsfile_PublishHelmChart +++ b/.ci/PublishHelmChart/Jenkinsfile_PublishHelmChart @@ -1,5 +1,5 @@ #! groovy -@Library('corda-shared-build-pipeline-steps@5.1') _ +@Library('corda-shared-build-pipeline-steps@owen/CORE-17957-Add-state-manager-support') _ import com.r3.build.agents.KubernetesAgent import com.r3.build.enums.BuildEnvironment diff --git a/.ci/dev/forward-merge/JenkinsInteropMerge b/.ci/dev/forward-merge/JenkinsInteropMerge index 1c238f2d7ea..fac1087a875 100644 --- a/.ci/dev/forward-merge/JenkinsInteropMerge +++ b/.ci/dev/forward-merge/JenkinsInteropMerge @@ -1,4 +1,4 @@ -@Library('corda-shared-build-pipeline-steps@5.1') _ +@Library('corda-shared-build-pipeline-steps@owen/CORE-17957-Add-state-manager-support') _ /* * Forward merge any changes in current branch to the branch with following version. diff --git a/.ci/e2eTests/Jenkinsfile b/.ci/e2eTests/Jenkinsfile index b17d7e1ca11..45e45092660 100644 --- a/.ci/e2eTests/Jenkinsfile +++ b/.ci/e2eTests/Jenkinsfile @@ -1,4 +1,4 @@ -@Library('corda-shared-build-pipeline-steps@5.1') _ +@Library('corda-shared-build-pipeline-steps@owen/CORE-17957-Add-state-manager-support') _ endToEndPipeline( assembleAndCompile: false, diff --git a/.ci/e2eTests/JenkinsfileCombinedWorker b/.ci/e2eTests/JenkinsfileCombinedWorker index 46f087b1567..55de9c86711 100644 --- a/.ci/e2eTests/JenkinsfileCombinedWorker +++ b/.ci/e2eTests/JenkinsfileCombinedWorker @@ -1,4 +1,4 @@ -@Library('corda-shared-build-pipeline-steps@5.1') _ +@Library('corda-shared-build-pipeline-steps@owen/CORE-17957-Add-state-manager-support') _ import groovy.transform.Field import com.r3.build.utils.PipelineUtils diff --git a/.ci/e2eTests/JenkinsfileCombinedWorkerLinux b/.ci/e2eTests/JenkinsfileCombinedWorkerLinux index 87de095abd8..77bf30cd1c3 100644 --- a/.ci/e2eTests/JenkinsfileCombinedWorkerLinux +++ b/.ci/e2eTests/JenkinsfileCombinedWorkerLinux @@ -1,4 +1,4 @@ -@Library('corda-shared-build-pipeline-steps@5.1') _ +@Library('corda-shared-build-pipeline-steps@owen/CORE-17957-Add-state-manager-support') _ combinedWorkerPipeline( agentOs : 'linux' diff --git a/.ci/e2eTests/JenkinsfileCombinedWorkerWindows b/.ci/e2eTests/JenkinsfileCombinedWorkerWindows index 5ac27d61f25..5256243bca2 100644 --- a/.ci/e2eTests/JenkinsfileCombinedWorkerWindows +++ b/.ci/e2eTests/JenkinsfileCombinedWorkerWindows @@ -1,4 +1,4 @@ -@Library('corda-shared-build-pipeline-steps@5.1') _ +@Library('corda-shared-build-pipeline-steps@owen/CORE-17957-Add-state-manager-support') _ combinedWorkerPipeline( agentOs : 'windows' diff --git a/.ci/e2eTests/JenkinsfileEnterpriseSmokeTests b/.ci/e2eTests/JenkinsfileEnterpriseSmokeTests index 2e83154c6be..a13da96577f 100644 --- a/.ci/e2eTests/JenkinsfileEnterpriseSmokeTests +++ b/.ci/e2eTests/JenkinsfileEnterpriseSmokeTests @@ -2,7 +2,7 @@ * Pipeline to take a packaged helm chart, deploy it and subsequently run the smoke test target * in this repo against these images - Can be used to exercise the C5 Ent images against smoke tests in corda-runtime-os */ -@Library('corda-shared-build-pipeline-steps@5.1') _ +@Library('corda-shared-build-pipeline-steps@owen/CORE-17957-Add-state-manager-support') _ endToEndPipeline( helmVersion: '^5.1.0-beta', diff --git a/.ci/e2eTests/JenkinsfileUnstableTest b/.ci/e2eTests/JenkinsfileUnstableTest index fb4873abfd8..9024981b4f4 100644 --- a/.ci/e2eTests/JenkinsfileUnstableTest +++ b/.ci/e2eTests/JenkinsfileUnstableTest @@ -1,4 +1,4 @@ -@Library('corda-shared-build-pipeline-steps@5.1') _ +@Library('corda-shared-build-pipeline-steps@owen/CORE-17957-Add-state-manager-support') _ endToEndPipeline( assembleAndCompile: true, diff --git a/.ci/e2eTests/corda.yaml b/.ci/e2eTests/corda.yaml index 1be0bc14a33..91186fe7b21 100644 --- a/.ci/e2eTests/corda.yaml +++ b/.ci/e2eTests/corda.yaml @@ -22,6 +22,13 @@ bootstrap: valueFrom: secretKeyRef: key: "password" + tokenSelection: + username: + value: "state-manager-user" + password: + valueFrom: + secretKeyRef: + key: "password" kafka: sasl: username: @@ -176,6 +183,15 @@ workers: secretKeyRef: name: "kafka-credentials" key: "tokenSelection" + stateManager: + db: + name: state-manager + username: + value: "state-manager-user" + password: + valueFrom: + secretKeyRef: + key: "password" rest: kafka: sasl: diff --git a/.ci/e2eTests/ethereum/Jenkinsfile b/.ci/e2eTests/ethereum/Jenkinsfile index 45f3beaef00..3ca104746a7 100644 --- a/.ci/e2eTests/ethereum/Jenkinsfile +++ b/.ci/e2eTests/ethereum/Jenkinsfile @@ -1,4 +1,4 @@ -@Library('corda-shared-build-pipeline-steps@5.1') _ +@Library('corda-shared-build-pipeline-steps@owen/CORE-17957-Add-state-manager-support') _ ethereumInterop( deployEthereum: true, diff --git a/.ci/nightly/JenkinsfileNightly b/.ci/nightly/JenkinsfileNightly index ae8831746ec..2f2cee3f9f0 100644 --- a/.ci/nightly/JenkinsfileNightly +++ b/.ci/nightly/JenkinsfileNightly @@ -1,4 +1,4 @@ -@Library('corda-shared-build-pipeline-steps@5.1') _ +@Library('corda-shared-build-pipeline-steps@owen/CORE-17957-Add-state-manager-support') _ cordaPipelineKubernetesAgent( dailyBuildCron: 'H 03 * * *', diff --git a/.ci/nightly/JenkinsfileSnykScan b/.ci/nightly/JenkinsfileSnykScan index e307baef42d..634f4965f96 100644 --- a/.ci/nightly/JenkinsfileSnykScan +++ b/.ci/nightly/JenkinsfileSnykScan @@ -1,4 +1,4 @@ -@Library('corda-shared-build-pipeline-steps@5.1') _ +@Library('corda-shared-build-pipeline-steps@owen/CORE-17957-Add-state-manager-support') _ cordaSnykScanPipeline ( snykTokenId: 'r3-snyk-corda5', diff --git a/.ci/nightly/JenkinsfileUnstableTests b/.ci/nightly/JenkinsfileUnstableTests index 7481ab5a961..331cc55fc36 100644 --- a/.ci/nightly/JenkinsfileUnstableTests +++ b/.ci/nightly/JenkinsfileUnstableTests @@ -1,5 +1,5 @@ //catch all job for flaky tests -@Library('corda-shared-build-pipeline-steps@5.1') _ +@Library('corda-shared-build-pipeline-steps@owen/CORE-17957-Add-state-manager-support') _ cordaPipelineKubernetesAgent( runIntegrationTests: true, diff --git a/.ci/nightly/JenkinsfileWindowsCompatibility b/.ci/nightly/JenkinsfileWindowsCompatibility index 5a608e02b4f..2d2b8a082ac 100644 --- a/.ci/nightly/JenkinsfileWindowsCompatibility +++ b/.ci/nightly/JenkinsfileWindowsCompatibility @@ -1,4 +1,4 @@ -@Library('corda-shared-build-pipeline-steps@5.1') _ +@Library('corda-shared-build-pipeline-steps@owen/CORE-17957-Add-state-manager-support') _ windowsCompatibility( runIntegrationTests: true, diff --git a/.ci/versionCompatibility/Jenkinsfile b/.ci/versionCompatibility/Jenkinsfile index 6f84690828a..699736f3110 100644 --- a/.ci/versionCompatibility/Jenkinsfile +++ b/.ci/versionCompatibility/Jenkinsfile @@ -1,3 +1,3 @@ -@Library('corda-shared-build-pipeline-steps@5.1') _ +@Library('corda-shared-build-pipeline-steps@owen/CORE-17957-Add-state-manager-support') _ cordaCompatibilityCheckPipeline(javaVersion: '17') \ No newline at end of file diff --git a/.ci/versionCompatibility/latest-version.Jenkinsfile b/.ci/versionCompatibility/latest-version.Jenkinsfile index dafe424944f..7b9a11406a4 100644 --- a/.ci/versionCompatibility/latest-version.Jenkinsfile +++ b/.ci/versionCompatibility/latest-version.Jenkinsfile @@ -1,4 +1,4 @@ -@Library('corda-shared-build-pipeline-steps@5.1') _ +@Library('corda-shared-build-pipeline-steps@owen/CORE-17957-Add-state-manager-support') _ // This build forces using the "very latest" version of the dependencies, regardless of which revision was chosen // This is useful as it gives early indication of a downstream change that may introduce a breaking change diff --git a/Jenkinsfile b/Jenkinsfile index d828e1c4068..23d83ff568b 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,4 +1,4 @@ -@Library('corda-shared-build-pipeline-steps@5.1') _ +@Library('corda-shared-build-pipeline-steps@owen/CORE-17957-Add-state-manager-support') _ cordaPipelineKubernetesAgent( dailyBuildCron: 'H H/6 * * *', diff --git a/charts/corda/templates/workers.yaml b/charts/corda/templates/workers.yaml index 7ddb4245965..3a20d7d30f2 100644 --- a/charts/corda/templates/workers.yaml +++ b/charts/corda/templates/workers.yaml @@ -20,7 +20,7 @@ ( dict "clusterDbAccess" true ) ) }} {{- include "corda.worker" ( list $ .Values.workers.tokenSelection "tokenSelection" - ( dict "clusterDbAccess" true ) + ( dict "clusterDbAccess" true "stateManagerDbAccess" true ) ) }} {{- include "corda.worker" ( list $ .Values.workers.rest "rest" ( dict "httpPort" 8888 "tlsSecretName" ( include "corda.restTlsSecretName" $ ) "additionalWorkerArgs" ( list "-rtls.crt.path=/tls/tls.crt" "-rtls.key.path=/tls/tls.key" "-rtls.ca.crt.path=/tls/ca.crt" ) ) diff --git a/charts/corda/values.schema.json b/charts/corda/values.schema.json index f233c892b80..8e9029f24ed 100644 --- a/charts/corda/values.schema.json +++ b/charts/corda/values.schema.json @@ -1282,6 +1282,13 @@ "$ref": "#/$defs/basicAuthConfig" } ] + }, + "tokenSelection": { + "allOf": [ + { + "$ref": "#/$defs/basicAuthConfig" + } + ] } }, "clientImage": { @@ -2580,6 +2587,9 @@ "logging": {}, "profiling": {}, "resources": {}, + "stateManager": { + "$ref": "#/$defs/stateManager" + }, "kafka": {}, "sharding": { "type": "object", diff --git a/charts/corda/values.yaml b/charts/corda/values.yaml index 5a52b291e52..367ca1ffef7 100644 --- a/charts/corda/values.yaml +++ b/charts/corda/values.yaml @@ -423,7 +423,31 @@ bootstrap: name: "" # -- the password secret key key: "" - + tokenSelection: + # the username configuration + username: + # -- the username, defaults to this value + value: "" + # the username secret configuration; used in preference to value if name is set + valueFrom: + # the username secret key reference + secretKeyRef: + # -- the username secret name + name: "" + # -- the username secret key + key: "" + # password configuration + password: + # -- the password, defaults to a value randomly-generated on install + value: "" + # the password secret configuration + valueFrom: + # the password secret key reference + secretKeyRef: + # -- the password secret name + name: "" + # -- the password secret key + key: "" # Image containing DB client, used to set up the DB clientImage: # -- registry for image containing a db client, used to set up the db @@ -1445,6 +1469,59 @@ workers: keepaliveTimeSeconds: 0 # -- maximum time (in seconds) that the pool will wait for a connection to be validated as alive validationTimeoutSeconds: 5 + + stateManager: + # Type of State Manager + type: DATABASE + # State Manager database configuration + db: + # -- the State Manager database host + host: null + # -- the State Manager database type + type: "postgresql" + # -- the State Manager database port + port: 5432 + # -- the State Manager database name + name: token_selection_state_manager + # the State Manager database user configuration + username: + # -- the State Manager database user + value: "token_selection_state_manager_user" + # the State Manager database user secret configuration; used in preference to value if name is set + valueFrom: + # the State Manager database user secret key reference + secretKeyRef: + # -- the State Manager database user secret name + name: "" + # -- the State Manager database user secret key + key: "" + # the State Manager database password configuration + password: + # -- the State Manager database password + value: "" + # the State Manager database password secret configuration; used in preference to value if name is set + valueFrom: + # the State Manager database password secret key reference + secretKeyRef: + # -- the State Manager database password secret name + name: "" + # -- the State Manager database password secret key + key: "" + # token selection worker JDBC connection pool configuration for State Manager DB + connectionPool: + # -- token selection maximum JDBC connection pool size for state manager DB + maxSize: 5 + # -- token selection minimum JDBC connection pool size for state manager DB; null value means pool's min size will default to pool's max size value + minSize: null + # -- maximum time (in seconds) a connection can stay idle in the pool; A value of 0 means that idle connections are never removed from the pool + idleTimeoutSeconds: 120 + # -- maximum time (in seconds) a connection can stay in the pool, regardless if it has been idle or has been recently used; If a connection is in-use and has reached "maxLifetime" timeout, it will be removed from the pool only when it becomes idle + maxLifetimeSeconds: 1800 + # -- interval time (in seconds) in which connections will be tested for aliveness; Connections which are no longer alive are removed from the pool; A value of 0 means this check is disabled + keepAliveTimeSeconds: 0 + # -- maximum time (in seconds) that the pool will wait for a connection to be validated as alive + validationTimeoutSeconds: 5 + # token selection worker Kafka configuration kafka: # if kafka.sasl.enabled, the credentials to connect to Kafka with for the token selection worker diff --git a/state-manager.yaml b/state-manager.yaml index 8418b7ec65c..b333b5c2f83 100644 --- a/state-manager.yaml +++ b/state-manager.yaml @@ -69,3 +69,15 @@ workers: secretKeyRef: name: "state-manager-db-postgresql" key: "password" + + tokenSelection: + stateManager: + db: + host: "state-manager-db-postgresql" + username: + value: "statemanager-user" + password: + valueFrom: + secretKeyRef: + name: "state-manager-db-postgresql" + key: "password" \ No newline at end of file From 9c7d2ea2e86dff4ab9c79257ca387ebd17cd8063 Mon Sep 17 00:00:00 2001 From: Owen Stanford <92725587+owenstanford@users.noreply.github.com> Date: Wed, 25 Oct 2023 09:35:12 +0100 Subject: [PATCH 65/81] CORE-17502 - revert jenkins import refs (#4968) --- .ci/JenkinsfileCombinedWorkerPluginsSmokeTests | 2 +- .ci/JenkinsfileSnykDelta | 2 +- .ci/PublishHelmChart/Jenkinsfile_PublishHelmChart | 2 +- .ci/dev/forward-merge/JenkinsInteropMerge | 2 +- .ci/e2eTests/Jenkinsfile | 2 +- .ci/e2eTests/JenkinsfileCombinedWorker | 2 +- .ci/e2eTests/JenkinsfileCombinedWorkerLinux | 2 +- .ci/e2eTests/JenkinsfileCombinedWorkerWindows | 2 +- .ci/e2eTests/JenkinsfileEnterpriseSmokeTests | 2 +- .ci/e2eTests/JenkinsfileUnstableTest | 2 +- .ci/e2eTests/ethereum/Jenkinsfile | 2 +- .ci/nightly/JenkinsfileNightly | 2 +- .ci/nightly/JenkinsfileSnykScan | 2 +- .ci/nightly/JenkinsfileUnstableTests | 2 +- .ci/nightly/JenkinsfileWindowsCompatibility | 2 +- .ci/versionCompatibility/Jenkinsfile | 2 +- .ci/versionCompatibility/latest-version.Jenkinsfile | 2 +- Jenkinsfile | 2 +- 18 files changed, 18 insertions(+), 18 deletions(-) diff --git a/.ci/JenkinsfileCombinedWorkerPluginsSmokeTests b/.ci/JenkinsfileCombinedWorkerPluginsSmokeTests index 10d3a447d2d..2d3c912e5a4 100644 --- a/.ci/JenkinsfileCombinedWorkerPluginsSmokeTests +++ b/.ci/JenkinsfileCombinedWorkerPluginsSmokeTests @@ -1,4 +1,4 @@ -@Library('corda-shared-build-pipeline-steps@owen/CORE-17957-Add-state-manager-support') _ +@Library('corda-shared-build-pipeline-steps@5.1') _ import groovy.transform.Field import com.r3.build.utils.PipelineUtils diff --git a/.ci/JenkinsfileSnykDelta b/.ci/JenkinsfileSnykDelta index e58dfa2049f..83b6279ecee 100644 --- a/.ci/JenkinsfileSnykDelta +++ b/.ci/JenkinsfileSnykDelta @@ -1,4 +1,4 @@ -@Library('corda-shared-build-pipeline-steps@owen/CORE-17957-Add-state-manager-support') _ +@Library('corda-shared-build-pipeline-steps@5.1') _ snykDelta( snykOrgId: 'corda5-snyk-org-id', diff --git a/.ci/PublishHelmChart/Jenkinsfile_PublishHelmChart b/.ci/PublishHelmChart/Jenkinsfile_PublishHelmChart index dca17c5e735..71e387d7bc0 100644 --- a/.ci/PublishHelmChart/Jenkinsfile_PublishHelmChart +++ b/.ci/PublishHelmChart/Jenkinsfile_PublishHelmChart @@ -1,5 +1,5 @@ #! groovy -@Library('corda-shared-build-pipeline-steps@owen/CORE-17957-Add-state-manager-support') _ +@Library('corda-shared-build-pipeline-steps@5.1') _ import com.r3.build.agents.KubernetesAgent import com.r3.build.enums.BuildEnvironment diff --git a/.ci/dev/forward-merge/JenkinsInteropMerge b/.ci/dev/forward-merge/JenkinsInteropMerge index fac1087a875..1c238f2d7ea 100644 --- a/.ci/dev/forward-merge/JenkinsInteropMerge +++ b/.ci/dev/forward-merge/JenkinsInteropMerge @@ -1,4 +1,4 @@ -@Library('corda-shared-build-pipeline-steps@owen/CORE-17957-Add-state-manager-support') _ +@Library('corda-shared-build-pipeline-steps@5.1') _ /* * Forward merge any changes in current branch to the branch with following version. diff --git a/.ci/e2eTests/Jenkinsfile b/.ci/e2eTests/Jenkinsfile index 45e45092660..b17d7e1ca11 100644 --- a/.ci/e2eTests/Jenkinsfile +++ b/.ci/e2eTests/Jenkinsfile @@ -1,4 +1,4 @@ -@Library('corda-shared-build-pipeline-steps@owen/CORE-17957-Add-state-manager-support') _ +@Library('corda-shared-build-pipeline-steps@5.1') _ endToEndPipeline( assembleAndCompile: false, diff --git a/.ci/e2eTests/JenkinsfileCombinedWorker b/.ci/e2eTests/JenkinsfileCombinedWorker index 55de9c86711..46f087b1567 100644 --- a/.ci/e2eTests/JenkinsfileCombinedWorker +++ b/.ci/e2eTests/JenkinsfileCombinedWorker @@ -1,4 +1,4 @@ -@Library('corda-shared-build-pipeline-steps@owen/CORE-17957-Add-state-manager-support') _ +@Library('corda-shared-build-pipeline-steps@5.1') _ import groovy.transform.Field import com.r3.build.utils.PipelineUtils diff --git a/.ci/e2eTests/JenkinsfileCombinedWorkerLinux b/.ci/e2eTests/JenkinsfileCombinedWorkerLinux index 77bf30cd1c3..87de095abd8 100644 --- a/.ci/e2eTests/JenkinsfileCombinedWorkerLinux +++ b/.ci/e2eTests/JenkinsfileCombinedWorkerLinux @@ -1,4 +1,4 @@ -@Library('corda-shared-build-pipeline-steps@owen/CORE-17957-Add-state-manager-support') _ +@Library('corda-shared-build-pipeline-steps@5.1') _ combinedWorkerPipeline( agentOs : 'linux' diff --git a/.ci/e2eTests/JenkinsfileCombinedWorkerWindows b/.ci/e2eTests/JenkinsfileCombinedWorkerWindows index 5256243bca2..5ac27d61f25 100644 --- a/.ci/e2eTests/JenkinsfileCombinedWorkerWindows +++ b/.ci/e2eTests/JenkinsfileCombinedWorkerWindows @@ -1,4 +1,4 @@ -@Library('corda-shared-build-pipeline-steps@owen/CORE-17957-Add-state-manager-support') _ +@Library('corda-shared-build-pipeline-steps@5.1') _ combinedWorkerPipeline( agentOs : 'windows' diff --git a/.ci/e2eTests/JenkinsfileEnterpriseSmokeTests b/.ci/e2eTests/JenkinsfileEnterpriseSmokeTests index a13da96577f..2e83154c6be 100644 --- a/.ci/e2eTests/JenkinsfileEnterpriseSmokeTests +++ b/.ci/e2eTests/JenkinsfileEnterpriseSmokeTests @@ -2,7 +2,7 @@ * Pipeline to take a packaged helm chart, deploy it and subsequently run the smoke test target * in this repo against these images - Can be used to exercise the C5 Ent images against smoke tests in corda-runtime-os */ -@Library('corda-shared-build-pipeline-steps@owen/CORE-17957-Add-state-manager-support') _ +@Library('corda-shared-build-pipeline-steps@5.1') _ endToEndPipeline( helmVersion: '^5.1.0-beta', diff --git a/.ci/e2eTests/JenkinsfileUnstableTest b/.ci/e2eTests/JenkinsfileUnstableTest index 9024981b4f4..fb4873abfd8 100644 --- a/.ci/e2eTests/JenkinsfileUnstableTest +++ b/.ci/e2eTests/JenkinsfileUnstableTest @@ -1,4 +1,4 @@ -@Library('corda-shared-build-pipeline-steps@owen/CORE-17957-Add-state-manager-support') _ +@Library('corda-shared-build-pipeline-steps@5.1') _ endToEndPipeline( assembleAndCompile: true, diff --git a/.ci/e2eTests/ethereum/Jenkinsfile b/.ci/e2eTests/ethereum/Jenkinsfile index 3ca104746a7..45f3beaef00 100644 --- a/.ci/e2eTests/ethereum/Jenkinsfile +++ b/.ci/e2eTests/ethereum/Jenkinsfile @@ -1,4 +1,4 @@ -@Library('corda-shared-build-pipeline-steps@owen/CORE-17957-Add-state-manager-support') _ +@Library('corda-shared-build-pipeline-steps@5.1') _ ethereumInterop( deployEthereum: true, diff --git a/.ci/nightly/JenkinsfileNightly b/.ci/nightly/JenkinsfileNightly index 2f2cee3f9f0..ae8831746ec 100644 --- a/.ci/nightly/JenkinsfileNightly +++ b/.ci/nightly/JenkinsfileNightly @@ -1,4 +1,4 @@ -@Library('corda-shared-build-pipeline-steps@owen/CORE-17957-Add-state-manager-support') _ +@Library('corda-shared-build-pipeline-steps@5.1') _ cordaPipelineKubernetesAgent( dailyBuildCron: 'H 03 * * *', diff --git a/.ci/nightly/JenkinsfileSnykScan b/.ci/nightly/JenkinsfileSnykScan index 634f4965f96..e307baef42d 100644 --- a/.ci/nightly/JenkinsfileSnykScan +++ b/.ci/nightly/JenkinsfileSnykScan @@ -1,4 +1,4 @@ -@Library('corda-shared-build-pipeline-steps@owen/CORE-17957-Add-state-manager-support') _ +@Library('corda-shared-build-pipeline-steps@5.1') _ cordaSnykScanPipeline ( snykTokenId: 'r3-snyk-corda5', diff --git a/.ci/nightly/JenkinsfileUnstableTests b/.ci/nightly/JenkinsfileUnstableTests index 331cc55fc36..7481ab5a961 100644 --- a/.ci/nightly/JenkinsfileUnstableTests +++ b/.ci/nightly/JenkinsfileUnstableTests @@ -1,5 +1,5 @@ //catch all job for flaky tests -@Library('corda-shared-build-pipeline-steps@owen/CORE-17957-Add-state-manager-support') _ +@Library('corda-shared-build-pipeline-steps@5.1') _ cordaPipelineKubernetesAgent( runIntegrationTests: true, diff --git a/.ci/nightly/JenkinsfileWindowsCompatibility b/.ci/nightly/JenkinsfileWindowsCompatibility index 2d2b8a082ac..5a608e02b4f 100644 --- a/.ci/nightly/JenkinsfileWindowsCompatibility +++ b/.ci/nightly/JenkinsfileWindowsCompatibility @@ -1,4 +1,4 @@ -@Library('corda-shared-build-pipeline-steps@owen/CORE-17957-Add-state-manager-support') _ +@Library('corda-shared-build-pipeline-steps@5.1') _ windowsCompatibility( runIntegrationTests: true, diff --git a/.ci/versionCompatibility/Jenkinsfile b/.ci/versionCompatibility/Jenkinsfile index 699736f3110..6f84690828a 100644 --- a/.ci/versionCompatibility/Jenkinsfile +++ b/.ci/versionCompatibility/Jenkinsfile @@ -1,3 +1,3 @@ -@Library('corda-shared-build-pipeline-steps@owen/CORE-17957-Add-state-manager-support') _ +@Library('corda-shared-build-pipeline-steps@5.1') _ cordaCompatibilityCheckPipeline(javaVersion: '17') \ No newline at end of file diff --git a/.ci/versionCompatibility/latest-version.Jenkinsfile b/.ci/versionCompatibility/latest-version.Jenkinsfile index 7b9a11406a4..dafe424944f 100644 --- a/.ci/versionCompatibility/latest-version.Jenkinsfile +++ b/.ci/versionCompatibility/latest-version.Jenkinsfile @@ -1,4 +1,4 @@ -@Library('corda-shared-build-pipeline-steps@owen/CORE-17957-Add-state-manager-support') _ +@Library('corda-shared-build-pipeline-steps@5.1') _ // This build forces using the "very latest" version of the dependencies, regardless of which revision was chosen // This is useful as it gives early indication of a downstream change that may introduce a breaking change diff --git a/Jenkinsfile b/Jenkinsfile index 23d83ff568b..d828e1c4068 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,4 +1,4 @@ -@Library('corda-shared-build-pipeline-steps@owen/CORE-17957-Add-state-manager-support') _ +@Library('corda-shared-build-pipeline-steps@5.1') _ cordaPipelineKubernetesAgent( dailyBuildCron: 'H H/6 * * *', From b48fc3cae312cf4af90da8c9c585988a2b851db2 Mon Sep 17 00:00:00 2001 From: David Currie Date: Wed, 25 Oct 2023 09:50:31 +0100 Subject: [PATCH 66/81] ES-1477 Add default metrics (#4953) --- charts/corda-lib/templates/_helpers.tpl | 6 +++++- charts/corda/values.schema.json | 10 +++++++++- charts/corda/values.yaml | 21 +++++++++++++++++++-- 3 files changed, 33 insertions(+), 4 deletions(-) diff --git a/charts/corda-lib/templates/_helpers.tpl b/charts/corda-lib/templates/_helpers.tpl index 444765b08ab..50e90d24161 100644 --- a/charts/corda-lib/templates/_helpers.tpl +++ b/charts/corda-lib/templates/_helpers.tpl @@ -561,13 +561,17 @@ metadata: spec: podMetricsEndpoints: - port: monitor - {{- with .Values.metrics.podMonitor.keepNames }} metricRelabelings: + {{- with .Values.metrics.podMonitor.keepNames }} - sourceLabels: - "__name__" regex: {{ join "|" . | quote }} action: "keep" {{- end }} + {{- with .Values.metrics.podMonitor.dropLabels }} + - regex: {{ join "|" . | quote }} + action: "labeldrop" + {{- end }} jobLabel: {{ $.Release.Name }}-{{ include "corda.name" . }} selector: matchLabels: diff --git a/charts/corda/values.schema.json b/charts/corda/values.schema.json index 8e9029f24ed..2d2c199493e 100644 --- a/charts/corda/values.schema.json +++ b/charts/corda/values.schema.json @@ -533,9 +533,17 @@ "type": "string", "format": "regex" }, - "default": [], "title": "A list of regular expressions for the names of metrics that Prometheus should keep; if empty, all metrics are kept", "examples": [[ "jvm_.*" ]] + }, + "dropLabels": { + "type": "array", + "items": { + "type": "string", + "format": "regex" + }, + "title": "A list of regular expressions for labels that Prometheus should drop across all metrics; if empty, all labels are kept", + "examples": [[ "virtualnode_destination" ]] } }, "examples": [ diff --git a/charts/corda/values.yaml b/charts/corda/values.yaml index 367ca1ffef7..058bc4a449d 100644 --- a/charts/corda/values.yaml +++ b/charts/corda/values.yaml @@ -79,8 +79,25 @@ metrics: # -- Labels that can be used so PodMonitor is discovered by Prometheus labels: {} # -- A list of regular expressions for the names of metrics that Prometheus should keep; if empty, all metrics are kept - keepNames: [] - # - "jvm_.*" + keepNames: + - "corda_flow_execution_time_seconds_(count|sum|max)" + - "corda_http_server_request_time_seconds_(count|sum|max)" + - "corda_p2p_gateway_inbound_request_time_seconds_(count|sum|max)" + - "corda_p2p_gateway_outbound_request_time_seconds_(count|sum|max)" + - "corda_p2p_gateway_outbound_tls_connections_total" + - "corda_p2p_message_outbound_total" + - "corda_p2p_message_outbound_replayed_total" + - "corda_p2p_message_outbound_latency_seconds_(count|sum|max)" + - "corda_p2p_message_inbound_total" + - "corda_p2p_session_outbound_total" + - "corda_p2p_session_inbound_total" + - "corda_membership_actions_handler_time_seconds_(count|sum|max)" + - "jvm_.*" + - "process_cpu_usage" + # -- A list of regular expressions for labels that Prometheus should drop across all metrics; if empty, all labels are kept + dropLabels: + - "virtualnode_destination" + - "virtualnode_source" # Distributed tracing configuration tracing: From 8be576d9e62772eccda108a07e2d211eb0e544c1 Mon Sep 17 00:00:00 2001 From: Ben Millar <44114751+ben-millar@users.noreply.github.com> Date: Wed, 25 Oct 2023 12:21:38 +0100 Subject: [PATCH 67/81] CORE-17883 Moving status code check to HTTPRetryExecutor, retrying on 404 (#4925) Stability improvements for the `RPCClient`. We now retry on 4XX and 5XX status codes, and thrown exceptions are wrapped as either `CordaMessageAPIIntermittentException` or `CordaMessageAPIFatalException` so that the FlowEventMediator can handle them appropriately. --- .../net/corda/messaging/mediator/RPCClient.kt | 49 ++++++++------ .../messaging/utils/HTTPRetryExecutor.kt | 67 +++++++++++++------ .../corda/messaging/mediator/RPCClientTest.kt | 14 ++-- .../messaging/utils/HTTPRetryExecutorTest.kt | 53 ++++++++++++--- 4 files changed, 123 insertions(+), 60 deletions(-) diff --git a/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/RPCClient.kt b/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/RPCClient.kt index 886b3e92c60..b61444ab026 100644 --- a/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/RPCClient.kt +++ b/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/RPCClient.kt @@ -9,6 +9,8 @@ import java.util.concurrent.TimeoutException import net.corda.avro.serialization.CordaAvroSerializationFactory import net.corda.messaging.api.exception.CordaHTTPClientErrorException import net.corda.messaging.api.exception.CordaHTTPServerErrorException +import net.corda.messaging.api.exception.CordaMessageAPIFatalException +import net.corda.messaging.api.exception.CordaMessageAPIIntermittentException import net.corda.messaging.api.mediator.MediatorMessage import net.corda.messaging.api.mediator.MessagingClient import net.corda.messaging.api.mediator.MessagingClient.Companion.MSG_PROP_ENDPOINT @@ -28,7 +30,10 @@ class RPCClient( private val httpClient: HttpClient, private val retryConfig: HTTPRetryConfig = HTTPRetryConfig.Builder() - .retryOn(IOException::class.java, TimeoutException::class.java) + .retryOn(IOException::class.java, + TimeoutException::class.java, + CordaHTTPClientErrorException::class.java, + CordaHTTPServerErrorException::class.java) .build() ) : MessagingClient { private val deserializer = cordaAvroSerializerFactory.createAvroDeserializer({}, Any::class.java) @@ -43,7 +48,6 @@ class RPCClient( processMessage(message) } catch (e: Exception) { handleExceptions(e) - null } } @@ -51,8 +55,6 @@ class RPCClient( val request = buildHttpRequest(message) val response = sendWithRetry(request) - checkResponseStatus(response.statusCode()) - val deserializedResponse = deserializePayload(response.body()) return MediatorMessage(deserializedResponse, mutableMapOf("statusCode" to response.statusCode())) } @@ -92,27 +94,30 @@ class RPCClient( } } - private fun checkResponseStatus(statusCode: Int) { - log.trace("Received response with status code $statusCode") - when (statusCode) { - in 400..499 -> throw CordaHTTPClientErrorException(statusCode, "Server returned status code $statusCode.") - in 500..599 -> throw CordaHTTPServerErrorException(statusCode, "Server returned status code $statusCode.") - } - } + private fun handleExceptions(e: Exception): Nothing { + val exceptionToThrow = when (e) { + is IOException, + is InterruptedException, + is TimeoutException, + is CordaHTTPClientErrorException, + is CordaHTTPServerErrorException -> { + log.warn("Intermittent error in RPCClient: ", e) + CordaMessageAPIIntermittentException(e.message, e) + } - private fun handleExceptions(e: Exception) { - when (e) { - is IOException -> log.warn("Network or IO operation error in RPCClient: ", e) - is InterruptedException -> log.warn("Operation was interrupted in RPCClient: ", e) - is IllegalArgumentException -> log.warn("Invalid argument provided in RPCClient call: ", e) - is SecurityException -> log.warn("Security violation detected in RPCClient: ", e) - is IllegalStateException -> log.warn("Coroutine state error in RPCClient: ", e) - is CordaHTTPClientErrorException -> log.warn("Client-side HTTP error in RPCClient: ", e) - is CordaHTTPServerErrorException -> log.warn("Server-side HTTP error in RPCClient: ", e) - else -> log.warn("Unhandled exception in RPCClient: ", e) + is IllegalArgumentException, + is SecurityException -> { + log.warn("Fatal error in RPCClient: ", e) + CordaMessageAPIFatalException(e.message, e) + } + + else -> { + log.warn("Unhandled exception in RPCClient: ", e) + e + } } - throw e + throw exceptionToThrow } override fun close() { diff --git a/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/utils/HTTPRetryExecutor.kt b/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/utils/HTTPRetryExecutor.kt index 1ec6b6b6bc5..f428d1484b5 100644 --- a/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/utils/HTTPRetryExecutor.kt +++ b/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/utils/HTTPRetryExecutor.kt @@ -1,6 +1,10 @@ package net.corda.messaging.utils +import java.net.http.HttpResponse +import net.corda.messaging.api.exception.CordaHTTPClientErrorException +import net.corda.messaging.api.exception.CordaHTTPServerErrorException import net.corda.utilities.trace +import net.corda.v5.base.exceptions.CordaRuntimeException import org.slf4j.Logger import org.slf4j.LoggerFactory @@ -8,36 +12,55 @@ class HTTPRetryExecutor { companion object { private val log: Logger = LoggerFactory.getLogger(this::class.java.enclosingClass) - fun withConfig(config: HTTPRetryConfig, block: () -> T): T { + fun withConfig(config: HTTPRetryConfig, block: () -> HttpResponse): HttpResponse { var currentDelay = config.initialDelay - for (i in 0 until config.times - 1) { - try { - log.trace { "HTTPRetryExecutor making attempt #${i + 1}." } - val result = block() - log.trace { "Operation successful after #${i + 1} attempt/s." } - return result - } catch (e: Exception) { - if (config.retryOn.none { it.isInstance(e) }) { - log.warn("HTTPRetryExecutor caught a non-retryable exception: ${e.message}", e) - throw e - } - - log.trace { "Attempt #${i + 1} failed due to ${e.message}. Retrying in $currentDelay ms..." } - Thread.sleep(currentDelay) - currentDelay = (currentDelay * config.factor).toLong() - } + for (i in 0 until config.times) { + val result = tryAttempt(i, config, block) + if (result != null) return result + + log.trace { "Attempt #${i + 1} failed. Retrying in $currentDelay ms..." } + Thread.sleep(currentDelay) + currentDelay = (currentDelay * config.factor).toLong() } - log.trace("All retry attempts exhausted. Making the final call.") + val errorMsg = "Retry logic exhausted all attempts without a valid return or rethrow, though this shouldn't be possible." + log.trace { errorMsg } + throw CordaRuntimeException(errorMsg) + } - try { + private fun tryAttempt(i: Int, config: HTTPRetryConfig, block: () -> HttpResponse): HttpResponse? { + return try { + log.trace { "HTTPRetryExecutor making attempt #${i + 1}." } val result = block() - log.trace { "Operation successful after #${config.times} attempt/s." } - return result + checkResponseStatus(result.statusCode()) + log.trace { "Operation successful after #${i + 1} attempt/s." } + result } catch (e: Exception) { - log.trace { "Operation failed after ${config.times} attempt/s." } + handleException(i, config, e) + null + } + } + + private fun handleException(attempt: Int, config: HTTPRetryConfig, e: Exception) { + val isFinalAttempt = attempt == config.times - 1 + val isRetryable = config.retryOn.any { it.isInstance(e) } + + if (!isRetryable || isFinalAttempt) { + val errorMsg = when { + isFinalAttempt -> "Operation failed after ${config.times} attempts." + else -> "HTTPRetryExecutor caught a non-retryable exception: ${e.message}" + } + log.trace { errorMsg } throw e } } + + private fun checkResponseStatus(statusCode: Int) { + log.trace { "Received response with status code $statusCode" } + when (statusCode) { + in 400..499 -> throw CordaHTTPClientErrorException(statusCode, "Server returned status code $statusCode.") + in 500..599 -> throw CordaHTTPServerErrorException(statusCode, "Server returned status code $statusCode.") + } + } } } diff --git a/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/RPCClientTest.kt b/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/RPCClientTest.kt index e85dd13667f..cf8787d6b98 100644 --- a/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/RPCClientTest.kt +++ b/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/mediator/RPCClientTest.kt @@ -8,8 +8,8 @@ import net.corda.avro.serialization.CordaAvroDeserializer import net.corda.avro.serialization.CordaAvroSerializationFactory import net.corda.avro.serialization.CordaAvroSerializer import net.corda.data.flow.event.FlowEvent -import net.corda.messaging.api.exception.CordaHTTPClientErrorException -import net.corda.messaging.api.exception.CordaHTTPServerErrorException +import net.corda.messaging.api.exception.CordaMessageAPIFatalException +import net.corda.messaging.api.exception.CordaMessageAPIIntermittentException import net.corda.messaging.api.mediator.MediatorMessage import net.corda.messaging.api.mediator.MessagingClient.Companion.MSG_PROP_ENDPOINT import net.corda.messaging.api.records.Record @@ -116,7 +116,7 @@ class RPCClientTest { val client = createClient(environment.mocks) - assertThrows { + assertThrows { client.send(message) } } @@ -128,7 +128,7 @@ class RPCClientTest { val client = createClient(environment.mocks) - assertThrows { + assertThrows { client.send(message) } } @@ -144,7 +144,7 @@ class RPCClientTest { val client = createClient(environment.mocks, onSerializationError) - assertThrows { + assertThrows { client.send(message) } @@ -179,7 +179,7 @@ class RPCClientTest { val client = createClient(environment.mocks) - assertThrows { + assertThrows { client.send(message) } } @@ -193,7 +193,7 @@ class RPCClientTest { val client = createClient(environment.mocks) - assertThrows { + assertThrows { client.send(message) } diff --git a/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/utils/HTTPRetryExecutorTest.kt b/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/utils/HTTPRetryExecutorTest.kt index e1245ebdc83..b652ea13114 100644 --- a/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/utils/HTTPRetryExecutorTest.kt +++ b/libs/messaging/messaging-impl/src/test/kotlin/net/corda/messaging/utils/HTTPRetryExecutorTest.kt @@ -1,9 +1,13 @@ package net.corda.messaging.utils +import java.net.http.HttpResponse +import net.corda.messaging.api.exception.CordaHTTPClientErrorException import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Test import org.junit.jupiter.api.assertThrows +import org.mockito.Mockito.mock +import org.mockito.kotlin.whenever class HTTPRetryExecutorTest { private lateinit var retryConfig: HTTPRetryConfig @@ -20,27 +24,33 @@ class HTTPRetryExecutorTest { @Test fun `successfully returns after first attempt`() { - val result = HTTPRetryExecutor.withConfig(retryConfig) { - "Success" + val mockResponse: HttpResponse = mock() + whenever(mockResponse.body()).thenReturn("Success") + + val result: HttpResponse = HTTPRetryExecutor.withConfig(retryConfig) { + mockResponse } - assertEquals("Success", result) + assertEquals("Success", result.body()) } @Suppress("TooGenericExceptionThrown") @Test fun `should retry until successful`() { + val mockResponse: HttpResponse = mock() + whenever(mockResponse.body()).thenReturn("Success on attempt 3") + var attempt = 0 - val result = HTTPRetryExecutor.withConfig(retryConfig) { + val result: HttpResponse = HTTPRetryExecutor.withConfig(retryConfig) { ++attempt if (attempt < 3) { throw RuntimeException("Failed on attempt $attempt") } - "Success on attempt $attempt" + mockResponse } - assertEquals("Success on attempt 3", result) + assertEquals("Success on attempt 3", result.body()) } @Suppress("TooGenericExceptionThrown") @@ -49,7 +59,7 @@ class HTTPRetryExecutorTest { var attempt = 0 assertThrows { - HTTPRetryExecutor.withConfig(retryConfig) { + HTTPRetryExecutor.withConfig(retryConfig) { ++attempt throw RuntimeException("Failed on attempt $attempt") } @@ -67,11 +77,36 @@ class HTTPRetryExecutorTest { .build() assertThrows { - HTTPRetryExecutor.withConfig(config) { + HTTPRetryExecutor.withConfig(config) { throw RuntimeException("I'm not retryable!") } } } + @Test + fun `should retry on client error status code`() { + val mockResponse: HttpResponse = mock() + whenever(mockResponse.body()).thenReturn("Success on attempt 3") + val config = HTTPRetryConfig.Builder() + .times(3) + .initialDelay(100) + .factor(2.0) + .retryOn(SpecificException::class.java) + .retryOn(CordaHTTPClientErrorException::class.java) + .build() + + var attempt = 0 + + val result: HttpResponse = HTTPRetryExecutor.withConfig(config) { + ++attempt + if (attempt < 3) { + throw CordaHTTPClientErrorException(404, "Not Found on attempt $attempt") + } + mockResponse + } + + assertEquals("Success on attempt 3", result.body()) + } + internal class SpecificException(message: String) : Exception(message) -} \ No newline at end of file +} From 56ed03d046255438502f0f41d1bdca222d839580 Mon Sep 17 00:00:00 2001 From: Lorcan Wogan <69468264+LWogan@users.noreply.github.com> Date: Wed, 25 Oct 2023 12:33:25 +0100 Subject: [PATCH 68/81] CORE-17932 ensure the flow service depends on the membership group reader providers lifecycle (#4961) Small fix to ensure the flow engine does not try to run without the membership cache populated and ready. --- .../src/main/kotlin/net/corda/flow/service/FlowService.kt | 2 ++ .../src/test/kotlin/net/corda/flow/service/FlowServiceTest.kt | 3 +++ 2 files changed, 5 insertions(+) diff --git a/components/flow/flow-service/src/main/kotlin/net/corda/flow/service/FlowService.kt b/components/flow/flow-service/src/main/kotlin/net/corda/flow/service/FlowService.kt index 08aed82af94..07c90524d83 100644 --- a/components/flow/flow-service/src/main/kotlin/net/corda/flow/service/FlowService.kt +++ b/components/flow/flow-service/src/main/kotlin/net/corda/flow/service/FlowService.kt @@ -16,6 +16,7 @@ import net.corda.lifecycle.RegistrationStatusChangeEvent import net.corda.lifecycle.StartEvent import net.corda.lifecycle.StopEvent import net.corda.lifecycle.createCoordinator +import net.corda.membership.read.MembershipGroupReaderProvider import net.corda.sandboxgroupcontext.service.SandboxGroupContextComponent import net.corda.schema.configuration.ConfigKeys.BOOT_CONFIG import net.corda.schema.configuration.ConfigKeys.FLOW_CONFIG @@ -69,6 +70,7 @@ class FlowService @Activate constructor( LifecycleCoordinatorName.forComponent(), LifecycleCoordinatorName.forComponent(), LifecycleCoordinatorName.forComponent(), + LifecycleCoordinatorName.forComponent(), ) ) flowMaintenance.start() diff --git a/components/flow/flow-service/src/test/kotlin/net/corda/flow/service/FlowServiceTest.kt b/components/flow/flow-service/src/test/kotlin/net/corda/flow/service/FlowServiceTest.kt index 3edd56f88be..e8f0ce12f4b 100644 --- a/components/flow/flow-service/src/test/kotlin/net/corda/flow/service/FlowServiceTest.kt +++ b/components/flow/flow-service/src/test/kotlin/net/corda/flow/service/FlowServiceTest.kt @@ -7,6 +7,7 @@ import net.corda.flow.MINIMUM_SMART_CONFIG import net.corda.flow.maintenance.FlowMaintenance import net.corda.lifecycle.LifecycleCoordinatorName import net.corda.lifecycle.test.impl.LifecycleTest +import net.corda.membership.read.MembershipGroupReaderProvider import net.corda.sandboxgroupcontext.service.SandboxGroupContextComponent import net.corda.schema.configuration.ConfigKeys import net.corda.virtualnode.read.VirtualNodeInfoReadService @@ -33,6 +34,7 @@ class FlowServiceTest { Arguments.of(LifecycleCoordinatorName.forComponent()), Arguments.of(LifecycleCoordinatorName.forComponent()), Arguments.of(LifecycleCoordinatorName.forComponent()), + Arguments.of(LifecycleCoordinatorName.forComponent()), ) } } @@ -156,6 +158,7 @@ class FlowServiceTest { addDependency() addDependency() addDependency() + addDependency() addDependency() addDependency() From b1d7e318af8ec888a704a78d3687f11b880476d1 Mon Sep 17 00:00:00 2001 From: Dries Samyn Date: Wed, 25 Oct 2023 14:35:33 +0100 Subject: [PATCH 69/81] - Increase max request payload (#4972) - Configure logging and tracing --- libs/web/web-impl/build.gradle | 1 + .../net/corda/web/server/JavalinServer.kt | 17 ++++++++++++++++- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/libs/web/web-impl/build.gradle b/libs/web/web-impl/build.gradle index b313982738b..be37b70d005 100644 --- a/libs/web/web-impl/build.gradle +++ b/libs/web/web-impl/build.gradle @@ -11,6 +11,7 @@ dependencies { implementation project(":libs:lifecycle:lifecycle") api project(':libs:platform-info') implementation project(":libs:rest:rest") + implementation project(':libs:tracing') implementation project(":libs:web:web") implementation 'org.jetbrains.kotlin:kotlin-osgi-bundle' diff --git a/libs/web/web-impl/src/main/kotlin/net/corda/web/server/JavalinServer.kt b/libs/web/web-impl/src/main/kotlin/net/corda/web/server/JavalinServer.kt index 07a30d4d223..6d15c895aee 100644 --- a/libs/web/web-impl/src/main/kotlin/net/corda/web/server/JavalinServer.kt +++ b/libs/web/web-impl/src/main/kotlin/net/corda/web/server/JavalinServer.kt @@ -5,6 +5,7 @@ import net.corda.libs.platform.PlatformInfoProvider import net.corda.lifecycle.LifecycleCoordinatorFactory import net.corda.lifecycle.LifecycleStatus import net.corda.lifecycle.createCoordinator +import net.corda.tracing.configureJavalinForTracing import net.corda.utilities.classload.executeWithThreadContextClassLoader import net.corda.utilities.executeWithStdErrSuppressed import net.corda.web.api.Endpoint @@ -32,7 +33,21 @@ class JavalinServer( coordinatorFactory: LifecycleCoordinatorFactory, @Reference(service = PlatformInfoProvider::class) platformInfoProvider: PlatformInfoProvider, - ) : this(coordinatorFactory, { Javalin.create() }, platformInfoProvider) + ) : this( + coordinatorFactory, + { + Javalin.create { config -> + // hardcode to 100Mb for now + // TODO CORE-17986: make configurable + config.maxRequestSize = 100_000_000L + + if (log.isDebugEnabled) { + config.enableDevLogging() + } + configureJavalinForTracing(config) + } + }, + platformInfoProvider) private companion object { val log: Logger = LoggerFactory.getLogger(this::class.java.enclosingClass) From 8469a8f11a87b416ac375896b0460193becc74d1 Mon Sep 17 00:00:00 2001 From: Lorcan Wogan <69468264+LWogan@users.noreply.github.com> Date: Wed, 25 Oct 2023 14:49:06 +0100 Subject: [PATCH 70/81] CORE-17779 add metric for a HTTP RPC Client request (#4901) Add a metric to record some information about how long it takes for an RPCClient to receive a response, as well as a distribution of response message size --- .../net/corda/messaging/mediator/RPCClient.kt | 53 ++++++++++++++++--- .../kotlin/net/corda/metrics/CordaMetrics.kt | 23 +++++++- 2 files changed, 67 insertions(+), 9 deletions(-) diff --git a/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/RPCClient.kt b/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/RPCClient.kt index b61444ab026..3eb7c5d5304 100644 --- a/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/RPCClient.kt +++ b/libs/messaging/messaging-impl/src/main/kotlin/net/corda/messaging/mediator/RPCClient.kt @@ -1,11 +1,5 @@ package net.corda.messaging.mediator -import java.io.IOException -import java.net.URI -import java.net.http.HttpClient -import java.net.http.HttpRequest -import java.net.http.HttpResponse -import java.util.concurrent.TimeoutException import net.corda.avro.serialization.CordaAvroSerializationFactory import net.corda.messaging.api.exception.CordaHTTPClientErrorException import net.corda.messaging.api.exception.CordaHTTPServerErrorException @@ -17,9 +11,18 @@ import net.corda.messaging.api.mediator.MessagingClient.Companion.MSG_PROP_ENDPO import net.corda.messaging.api.mediator.MessagingClient.Companion.MSG_PROP_KEY import net.corda.messaging.utils.HTTPRetryConfig import net.corda.messaging.utils.HTTPRetryExecutor +import net.corda.metrics.CordaMetrics +import net.corda.utilities.debug import net.corda.utilities.trace import org.slf4j.Logger import org.slf4j.LoggerFactory +import java.io.IOException +import java.net.URI +import java.net.http.HttpClient +import java.net.http.HttpRequest +import java.net.http.HttpResponse +import java.time.Duration +import java.util.concurrent.TimeoutException const val CORDA_REQUEST_KEY_HEADER = "corda-request-key" @@ -40,6 +43,8 @@ class RPCClient( private companion object { private val log: Logger = LoggerFactory.getLogger(this::class.java.enclosingClass) + private const val SUCCESS: String = "SUCCESS" + private const val FAILED: String = "FAILED" } override fun send(message: MediatorMessage<*>): MediatorMessage<*>? { @@ -89,8 +94,40 @@ class RPCClient( } private fun sendWithRetry(request: HttpRequest): HttpResponse { - return HTTPRetryExecutor.withConfig(retryConfig) { - httpClient.send(request, HttpResponse.BodyHandlers.ofByteArray()) + val startTime = System.nanoTime() + return try { + val response = HTTPRetryExecutor.withConfig(retryConfig) { + httpClient.send(request, HttpResponse.BodyHandlers.ofByteArray()) + } + buildMetricForResponse(startTime, SUCCESS, request, response) + response + } catch (ex: Exception) { + log.debug { "Catching exception in HttpClient sendWithRetry in order to log metrics, $ex" } + buildMetricForResponse(startTime, FAILED, request) + throw ex + } + } + + private fun buildMetricForResponse( + startTime: Long, + operationStatus: String, + request: HttpRequest, + response: HttpResponse? = null + ) { + val endTime = System.nanoTime() + val uri = request.method() + request.uri().toString() + CordaMetrics.Metric.Messaging.HTTPRPCResponseTime.builder() + .withTag(CordaMetrics.Tag.OperationStatus, operationStatus) + .withTag(CordaMetrics.Tag.HttpRequestUri, uri) + .withTag(CordaMetrics.Tag.HttpResponseCode, response?.statusCode().toString()) + .build() + .record(Duration.ofNanos(endTime - startTime)) + + if (response != null) { + CordaMetrics.Metric.Messaging.HTTPRPCResponseSize.builder() + .withTag(CordaMetrics.Tag.HttpRequestUri, uri) + .build() + .record(response.body().size.toDouble()) } } diff --git a/libs/metrics/src/main/kotlin/net/corda/metrics/CordaMetrics.kt b/libs/metrics/src/main/kotlin/net/corda/metrics/CordaMetrics.kt index 8ae931e9078..b9cbc6d6cf1 100644 --- a/libs/metrics/src/main/kotlin/net/corda/metrics/CordaMetrics.kt +++ b/libs/metrics/src/main/kotlin/net/corda/metrics/CordaMetrics.kt @@ -7,7 +7,6 @@ import io.micrometer.core.instrument.Meter import io.micrometer.core.instrument.MeterRegistry import io.micrometer.core.instrument.Metrics import io.micrometer.core.instrument.Tags -import io.micrometer.core.instrument.Tag as micrometerTag import io.micrometer.core.instrument.Timer import io.micrometer.core.instrument.binder.BaseUnits import io.micrometer.core.instrument.composite.CompositeMeterRegistry @@ -19,6 +18,7 @@ import java.nio.file.Path import java.util.function.Supplier import java.util.function.ToDoubleFunction import java.util.function.ToLongFunction +import io.micrometer.core.instrument.Tag as micrometerTag object CordaMetrics { @@ -660,6 +660,17 @@ object CordaMetrics { "consumer.partitioned.inmemory.store", computation ) + + /** + * Record how long a HTTP RPC call from the messaging library takes to receive a response + */ + object HTTPRPCResponseTime : Metric("rpc.http.response.time", CordaMetrics::timer) + + /** + * Record the size of HTTP RPC responses + */ + object HTTPRPCResponseSize : Metric("rpc.http.response.size", Metrics::summary) + } object TaskManager { @@ -692,6 +703,16 @@ object CordaMetrics { */ HttpMethod("http.method"), + /** + * The URI that a HTTP request was sent to + */ + HttpRequestUri("http.request.uri"), + + /** + * Response code received for a HTTP response + */ + HttpResponseCode("http.response.code"), + /** * Type of the SandboxGroup to which the metric applies. */ From 6281ecc42b8c9a3b2dbebacc3ffab7ed4cd68915 Mon Sep 17 00:00:00 2001 From: James Higgs <45565019+JamesHR3@users.noreply.github.com> Date: Wed, 25 Oct 2023 14:49:28 +0100 Subject: [PATCH 71/81] CORE-17389: Process flow timeout events and clean up checkpoints (#4951) Problem description The flow engine currently responds to scheduled task triggers to produce flow timeout events when a session has timed out. These need to be processed to remove the timed out checkpoint and send any cleanup events (session errors and mapper cleanup events) to the rest of the system. The flow status also needs to be updated. Solution Add a processor for the flow timeout events. This PR also includes a checkpoint cleanup handler class, which generates the relevant events for when a flow is terminated with an error. This can be reused in the FlowEventExceptionProcessor, although this has not been integrated in this PR. The objective here is to encapsulate this cleanup logic. At present, there is a bug in the flow engine that means not all of this logic is present everywhere it is needed. --- .../executor/ScheduledTaskProcessor.kt | 9 +- .../executor/ScheduledTaskProcessorTest.kt | 24 ++- .../corda/flow/state/impl/package-info.java | 4 + .../maintenance/CheckpointCleanupHandler.kt | 21 +++ .../CheckpointCleanupHandlerImpl.kt | 119 +++++++++++++++ .../FlowMaintenanceHandlersFactory.kt | 27 ++++ .../FlowMaintenanceHandlersFactoryImpl.kt | 43 ++++++ .../flow/maintenance/FlowMaintenanceImpl.kt | 46 ++++-- .../TimeoutEventCleanupProcessor.kt | 63 ++++++++ .../CheckpointCleanupHandlerImplTest.kt | 139 ++++++++++++++++++ .../maintenance/FlowMaintenanceImplTests.kt | 72 ++++++--- .../TimeoutEventCleanupProcessorTest.kt | 109 ++++++++++++++ 12 files changed, 639 insertions(+), 37 deletions(-) create mode 100644 components/flow/flow-service/src/main/java/net/corda/flow/state/impl/package-info.java create mode 100644 components/flow/flow-service/src/main/kotlin/net/corda/flow/maintenance/CheckpointCleanupHandler.kt create mode 100644 components/flow/flow-service/src/main/kotlin/net/corda/flow/maintenance/CheckpointCleanupHandlerImpl.kt create mode 100644 components/flow/flow-service/src/main/kotlin/net/corda/flow/maintenance/FlowMaintenanceHandlersFactory.kt create mode 100644 components/flow/flow-service/src/main/kotlin/net/corda/flow/maintenance/FlowMaintenanceHandlersFactoryImpl.kt create mode 100644 components/flow/flow-service/src/main/kotlin/net/corda/flow/maintenance/TimeoutEventCleanupProcessor.kt create mode 100644 components/flow/flow-service/src/test/kotlin/net/corda/flow/maintenance/CheckpointCleanupHandlerImplTest.kt create mode 100644 components/flow/flow-service/src/test/kotlin/net/corda/flow/maintenance/TimeoutEventCleanupProcessorTest.kt diff --git a/components/flow/flow-mapper-service/src/main/kotlin/net/corda/session/mapper/service/executor/ScheduledTaskProcessor.kt b/components/flow/flow-mapper-service/src/main/kotlin/net/corda/session/mapper/service/executor/ScheduledTaskProcessor.kt index 3778b460bb9..487f958cb00 100644 --- a/components/flow/flow-mapper-service/src/main/kotlin/net/corda/session/mapper/service/executor/ScheduledTaskProcessor.kt +++ b/components/flow/flow-mapper-service/src/main/kotlin/net/corda/session/mapper/service/executor/ScheduledTaskProcessor.kt @@ -51,10 +51,16 @@ class ScheduledTaskProcessor( private fun getExpiredStateIds() : List { val windowExpiry = clock.instant() - Duration.ofMillis(cleanupWindow) - val states = stateManager.findUpdatedBetweenWithMetadataFilter( + val closingStates = stateManager.findUpdatedBetweenWithMetadataFilter( IntervalFilter(Instant.EPOCH, windowExpiry), MetadataFilter(FLOW_MAPPER_STATUS, Operation.Equals, FlowMapperStateType.CLOSING.toString()) ) + val errorStates = stateManager.findUpdatedBetweenWithMetadataFilter( + IntervalFilter(Instant.EPOCH, windowExpiry), + MetadataFilter(FLOW_MAPPER_STATUS, Operation.Equals, FlowMapperStateType.ERROR.toString()) + ) + val states = closingStates + errorStates + return states.map { it.key }.also { @@ -66,7 +72,6 @@ class ScheduledTaskProcessor( return ids.chunked(batchSize) } - override val keyClass = String::class.java override val valueClass = ScheduledTaskTrigger::class.java } \ No newline at end of file diff --git a/components/flow/flow-mapper-service/src/test/kotlin/net/corda/session/mapper/service/executor/ScheduledTaskProcessorTest.kt b/components/flow/flow-mapper-service/src/test/kotlin/net/corda/session/mapper/service/executor/ScheduledTaskProcessorTest.kt index 476921bc775..10eb521a215 100644 --- a/components/flow/flow-mapper-service/src/test/kotlin/net/corda/session/mapper/service/executor/ScheduledTaskProcessorTest.kt +++ b/components/flow/flow-mapper-service/src/test/kotlin/net/corda/session/mapper/service/executor/ScheduledTaskProcessorTest.kt @@ -28,9 +28,14 @@ class ScheduledTaskProcessorTest { private val clock = Clock.fixed(Instant.now(), ZoneId.systemDefault()) private val window = 1000L - private val states = listOf( - createStateEntry("key1", clock.instant().minusMillis(window * 2)), - createStateEntry("key4", clock.instant().minusMillis(window * 3)) + private val closingStates = listOf( + createStateEntry("key1", clock.instant().minusMillis(window * 2), FlowMapperStateType.CLOSING.toString()), + createStateEntry("key4", clock.instant().minusMillis(window * 3), FlowMapperStateType.CLOSING.toString()) + ).toMap() + + private val errorStates = listOf( + createStateEntry("key2", clock.instant().minusMillis(window * 2), FlowMapperStateType.CLOSING.toString()), + createStateEntry("key5", clock.instant().minusMillis(window * 3), FlowMapperStateType.CLOSING.toString()) ).toMap() private val inputEvent = Record( Schemas.ScheduledTask.SCHEDULED_TASK_TOPIC_MAPPER_PROCESSOR, @@ -41,7 +46,7 @@ class ScheduledTaskProcessorTest { @Test fun `when scheduled task handler generates new records, ID of each retrieved state is present in output events`() { val stateManager = mock() - whenever(stateManager.findUpdatedBetweenWithMetadataFilter(any(), any())).thenReturn(states) + whenever(stateManager.findUpdatedBetweenWithMetadataFilter(any(), any())).thenReturn(closingStates+errorStates) val scheduledTaskProcessor = ScheduledTaskProcessor( stateManager, clock, @@ -54,12 +59,16 @@ class ScheduledTaskProcessorTest { IntervalFilter(Instant.EPOCH, clock.instant() - Duration.ofMillis(window)), MetadataFilter(FLOW_MAPPER_STATUS, Operation.Equals, FlowMapperStateType.CLOSING.toString()) ) + verify(stateManager).findUpdatedBetweenWithMetadataFilter( + IntervalFilter(Instant.EPOCH, clock.instant() - Duration.ofMillis(window)), + MetadataFilter(FLOW_MAPPER_STATUS, Operation.Equals, FlowMapperStateType.ERROR.toString()) + ) } @Test fun `when batch size is set to one, a record per id is present in output events`() { val stateManager = mock() - whenever(stateManager.findUpdatedBetweenWithMetadataFilter(any(), any())).thenReturn(states) + whenever(stateManager.findUpdatedBetweenWithMetadataFilter(any(), any())).thenReturn(closingStates) val scheduledTaskProcessor = ScheduledTaskProcessor( stateManager, clock, @@ -107,12 +116,13 @@ class ScheduledTaskProcessorTest { private fun createStateEntry( key: String, - lastUpdated: Instant + lastUpdated: Instant, + status:String ): Pair { val state = State( key, byteArrayOf(), - metadata = metadata(FLOW_MAPPER_STATUS to FlowMapperStateType.CLOSING.toString()), + metadata = metadata(FLOW_MAPPER_STATUS to status), modifiedTime = lastUpdated ) return Pair(key, state) diff --git a/components/flow/flow-service/src/main/java/net/corda/flow/state/impl/package-info.java b/components/flow/flow-service/src/main/java/net/corda/flow/state/impl/package-info.java new file mode 100644 index 00000000000..480a2cba856 --- /dev/null +++ b/components/flow/flow-service/src/main/java/net/corda/flow/state/impl/package-info.java @@ -0,0 +1,4 @@ +@Export +package net.corda.flow.state.impl; + +import org.osgi.annotation.bundle.Export; \ No newline at end of file diff --git a/components/flow/flow-service/src/main/kotlin/net/corda/flow/maintenance/CheckpointCleanupHandler.kt b/components/flow/flow-service/src/main/kotlin/net/corda/flow/maintenance/CheckpointCleanupHandler.kt new file mode 100644 index 00000000000..9140091d10d --- /dev/null +++ b/components/flow/flow-service/src/main/kotlin/net/corda/flow/maintenance/CheckpointCleanupHandler.kt @@ -0,0 +1,21 @@ +package net.corda.flow.maintenance + +import net.corda.flow.state.FlowCheckpoint +import net.corda.libs.configuration.SmartConfig +import net.corda.messaging.api.records.Record +import java.lang.Exception + +interface CheckpointCleanupHandler { + + /** + * Generate any records to clean up the system when a flow is terminated. + * + * This will also modify the checkpoint state to indicate that it should be removed. + * + * @param checkpoint The checkpoint to evaluate in order to generate any cleanup records. + * @param config Flow configuration + * @param exception The exception causing the checkpoint to be cleaned up. + * @return A list of records to be published to the rest of the system to clean up any state for this flow. + */ + fun cleanupCheckpoint(checkpoint: FlowCheckpoint, config: SmartConfig, exception: Exception) : List> +} \ No newline at end of file diff --git a/components/flow/flow-service/src/main/kotlin/net/corda/flow/maintenance/CheckpointCleanupHandlerImpl.kt b/components/flow/flow-service/src/main/kotlin/net/corda/flow/maintenance/CheckpointCleanupHandlerImpl.kt new file mode 100644 index 00000000000..656b47485b9 --- /dev/null +++ b/components/flow/flow-service/src/main/kotlin/net/corda/flow/maintenance/CheckpointCleanupHandlerImpl.kt @@ -0,0 +1,119 @@ +package net.corda.flow.maintenance + +import net.corda.data.flow.FlowInitiatorType +import net.corda.data.flow.event.mapper.ScheduleCleanup +import net.corda.data.flow.state.session.SessionStateType +import net.corda.flow.pipeline.exceptions.FlowMarkedForKillException +import net.corda.flow.pipeline.exceptions.FlowProcessingExceptionTypes.FLOW_FAILED +import net.corda.flow.pipeline.factory.FlowMessageFactory +import net.corda.flow.pipeline.factory.FlowRecordFactory +import net.corda.flow.pipeline.sessions.FlowSessionManager +import net.corda.flow.state.FlowCheckpoint +import net.corda.libs.configuration.SmartConfig +import net.corda.messaging.api.records.Record +import net.corda.schema.configuration.FlowConfig +import org.osgi.service.component.annotations.Activate +import org.osgi.service.component.annotations.Component +import org.osgi.service.component.annotations.Reference +import java.lang.Exception +import java.time.Instant + +@Component(service = [CheckpointCleanupHandler::class]) +class CheckpointCleanupHandlerImpl @Activate constructor( + @Reference(service = FlowRecordFactory::class) + private val flowRecordFactory: FlowRecordFactory, + @Reference(service = FlowSessionManager::class) + private val flowSessionManager: FlowSessionManager, + @Reference(service = FlowMessageFactory::class) + private val flowMessageFactory: FlowMessageFactory +) : CheckpointCleanupHandler { + + override fun cleanupCheckpoint( + checkpoint: FlowCheckpoint, + config: SmartConfig, + exception: Exception + ): List> { + val time = Instant.now() + val records = errorActiveSessions(checkpoint, config, exception, time) + + cleanupSessions(checkpoint, config, time) + + generateStatus(checkpoint, exception) + + cleanupInitiatingFlow(checkpoint, config, time) + checkpoint.markDeleted() + return records + } + + private fun errorActiveSessions( + checkpoint: FlowCheckpoint, + config: SmartConfig, + exception: Exception, + currentTime: Instant + ): List> { + val sessions = checkpoint.sessions.filterNot { + it.status == SessionStateType.CLOSED || it.status == SessionStateType.ERROR + }.map { it.sessionId } + if (sessions.isNotEmpty()) { + checkpoint.putSessionStates( + flowSessionManager.sendErrorMessages( + checkpoint, + sessions, + exception, + currentTime + ) + ) + } + return flowSessionManager.getSessionErrorEventRecords(checkpoint, config, currentTime) + } + + private fun cleanupSessions( + checkpoint: FlowCheckpoint, + config: SmartConfig, + currentTime: Instant + ): List> { + val cleanupTimeWindow = config.getLong(FlowConfig.SESSION_FLOW_CLEANUP_TIME) + val cleanupTime = currentTime.plusMillis(cleanupTimeWindow).toEpochMilli() + return checkpoint.sessions.filterNot { it.hasScheduledCleanup }.map { + it.hasScheduledCleanup = true + flowRecordFactory.createFlowMapperEventRecord( + it.sessionId, + ScheduleCleanup(cleanupTime) + ) + } + } + + private fun generateStatus(checkpoint: FlowCheckpoint, exception: Exception): List> { + return try { + val status = when (exception) { + is FlowMarkedForKillException -> { + flowMessageFactory.createFlowKilledStatusMessage(checkpoint, exception.message) + } + else -> { + flowMessageFactory.createFlowFailedStatusMessage( + checkpoint, + FLOW_FAILED, + exception.message ?: "No message provided" + ) + } + } + listOf(flowRecordFactory.createFlowStatusRecord(status)) + } catch (e: Exception) { + listOf() + } + } + + private fun cleanupInitiatingFlow( + checkpoint: FlowCheckpoint, + config: SmartConfig, + currentTime: Instant + ): List> { + return if (checkpoint.flowStartContext.initiatorType == FlowInitiatorType.RPC) { + val cleanupWindow = config.getLong(FlowConfig.PROCESSING_FLOW_CLEANUP_TIME) + val expiryTime = currentTime.plusMillis(cleanupWindow).toEpochMilli() + listOf(flowRecordFactory.createFlowMapperEventRecord( + checkpoint.flowKey.toString(), + ScheduleCleanup(expiryTime) + )) + } else { + listOf() + } + } +} \ No newline at end of file diff --git a/components/flow/flow-service/src/main/kotlin/net/corda/flow/maintenance/FlowMaintenanceHandlersFactory.kt b/components/flow/flow-service/src/main/kotlin/net/corda/flow/maintenance/FlowMaintenanceHandlersFactory.kt new file mode 100644 index 00000000000..9e02d25384a --- /dev/null +++ b/components/flow/flow-service/src/main/kotlin/net/corda/flow/maintenance/FlowMaintenanceHandlersFactory.kt @@ -0,0 +1,27 @@ +package net.corda.flow.maintenance + +import net.corda.libs.configuration.SmartConfig +import net.corda.libs.statemanager.api.StateManager + +/** + * Factory for generating flow maintenance event handlers. + */ +interface FlowMaintenanceHandlersFactory { + + /** + * Create a handler for scheduled task triggers handling session timeout. + * + * @param stateManager The state manager the handler should use to retrieve states. + * @return A session timeout task processor. + */ + fun createScheduledTaskHandler(stateManager: StateManager): SessionTimeoutTaskProcessor + + /** + * Create a handler for session timeout events. + * + * @param stateManager The state manager the handler should use to retrieve states. + * @param config The flow configuration. + * @return A timeout event handler. + */ + fun createTimeoutEventHandler(stateManager: StateManager, config: SmartConfig): TimeoutEventCleanupProcessor +} \ No newline at end of file diff --git a/components/flow/flow-service/src/main/kotlin/net/corda/flow/maintenance/FlowMaintenanceHandlersFactoryImpl.kt b/components/flow/flow-service/src/main/kotlin/net/corda/flow/maintenance/FlowMaintenanceHandlersFactoryImpl.kt new file mode 100644 index 00000000000..3964f7c05ba --- /dev/null +++ b/components/flow/flow-service/src/main/kotlin/net/corda/flow/maintenance/FlowMaintenanceHandlersFactoryImpl.kt @@ -0,0 +1,43 @@ +package net.corda.flow.maintenance + +import net.corda.avro.serialization.CordaAvroSerializationFactory +import net.corda.data.flow.state.checkpoint.Checkpoint +import net.corda.flow.state.impl.FlowCheckpointFactory +import net.corda.libs.configuration.SmartConfig +import net.corda.libs.statemanager.api.StateManager +import org.osgi.service.component.annotations.Activate +import org.osgi.service.component.annotations.Component +import org.osgi.service.component.annotations.Reference + +/** + * Factory for constructing event handlers for flow maintenance events + */ +@Component(service = [FlowMaintenanceHandlersFactory::class]) +class FlowMaintenanceHandlersFactoryImpl @Activate constructor( + @Reference(service = CordaAvroSerializationFactory::class) + avroSerializationFactory: CordaAvroSerializationFactory, + @Reference(service = CheckpointCleanupHandler::class) + private val checkpointCleanupHandler: CheckpointCleanupHandler, + @Reference(service = FlowCheckpointFactory::class) + private val flowCheckpointFactory: FlowCheckpointFactory +) : FlowMaintenanceHandlersFactory { + + private val checkpointDeserializer = avroSerializationFactory.createAvroDeserializer({}, Checkpoint::class.java) + + override fun createScheduledTaskHandler(stateManager: StateManager): SessionTimeoutTaskProcessor { + return SessionTimeoutTaskProcessor(stateManager) + } + + override fun createTimeoutEventHandler( + stateManager: StateManager, + config: SmartConfig + ): TimeoutEventCleanupProcessor { + return TimeoutEventCleanupProcessor( + checkpointCleanupHandler, + stateManager, + checkpointDeserializer, + flowCheckpointFactory, + config + ) + } +} \ No newline at end of file diff --git a/components/flow/flow-service/src/main/kotlin/net/corda/flow/maintenance/FlowMaintenanceImpl.kt b/components/flow/flow-service/src/main/kotlin/net/corda/flow/maintenance/FlowMaintenanceImpl.kt index 3e135e3b334..2236c316e9f 100644 --- a/components/flow/flow-service/src/main/kotlin/net/corda/flow/maintenance/FlowMaintenanceImpl.kt +++ b/components/flow/flow-service/src/main/kotlin/net/corda/flow/maintenance/FlowMaintenanceImpl.kt @@ -1,8 +1,8 @@ package net.corda.flow.maintenance +import com.typesafe.config.ConfigValueFactory import net.corda.libs.configuration.SmartConfig import net.corda.libs.configuration.helper.getConfig -import net.corda.libs.statemanager.api.StateManager import net.corda.libs.statemanager.api.StateManagerFactory import net.corda.lifecycle.LifecycleCoordinator import net.corda.lifecycle.LifecycleCoordinatorFactory @@ -15,6 +15,8 @@ import net.corda.messaging.api.subscription.config.SubscriptionConfig import net.corda.messaging.api.subscription.factory.SubscriptionFactory import net.corda.schema.Schemas import net.corda.schema.configuration.ConfigKeys +import net.corda.schema.configuration.MessagingConfig.MAX_ALLOWED_MSG_SIZE +import net.corda.schema.configuration.MessagingConfig.Subscription.PROCESSOR_TIMEOUT import net.corda.utilities.debug import net.corda.utilities.trace import org.osgi.service.component.annotations.Activate @@ -30,25 +32,27 @@ class FlowMaintenanceImpl @Activate constructor( private val subscriptionFactory: SubscriptionFactory, @Reference(service = StateManagerFactory::class) private val stateManagerFactory: StateManagerFactory, + @Reference(service = FlowMaintenanceHandlersFactory::class) + private val flowMaintenanceHandlersFactory: FlowMaintenanceHandlersFactory ) : FlowMaintenance { companion object { private val logger = LoggerFactory.getLogger(this::class.java.enclosingClass) } private val coordinator = coordinatorFactory.createCoordinator(::eventHandler) - private var stateManagerConfig: SmartConfig? = null - private var stateManager: StateManager? = null override fun onConfigChange(config: Map) { - // Top level component is using ConfigurationReadService#registerComponentForUpdates, so either both or none of the keys + // Top level component is using ConfigurationReadService#registerComponentForUpdates, so all the below keys // should be present. - if (config.containsKey(ConfigKeys.STATE_MANAGER_CONFIG) && config.containsKey(ConfigKeys.MESSAGING_CONFIG)) { + val requiredKeys = listOf(ConfigKeys.STATE_MANAGER_CONFIG, ConfigKeys.MESSAGING_CONFIG, ConfigKeys.FLOW_CONFIG) + if (requiredKeys.all { config.containsKey(it) }) { val messagingConfig = config.getConfig(ConfigKeys.MESSAGING_CONFIG) val newStateManagerConfig = config.getConfig(ConfigKeys.STATE_MANAGER_CONFIG) + val flowConfig = getFlowConfig(config, messagingConfig) - stateManager?.close() - stateManagerConfig = newStateManagerConfig - stateManager = stateManagerFactory.create(newStateManagerConfig) + val stateManager = coordinator.createManagedResource("STATE_MANAGER") { + stateManagerFactory.create(newStateManagerConfig) + } coordinator.createManagedResource("FLOW_MAINTENANCE_SUBSCRIPTION") { subscriptionFactory.createDurableSubscription( @@ -56,7 +60,19 @@ class FlowMaintenanceImpl @Activate constructor( "flow.maintenance.tasks", Schemas.ScheduledTask.SCHEDULED_TASK_TOPIC_FLOW_PROCESSOR ), - SessionTimeoutTaskProcessor(stateManager!!), + flowMaintenanceHandlersFactory.createScheduledTaskHandler(stateManager), + messagingConfig, + null + ) + }.start() + + coordinator.createManagedResource("FLOW_TIMEOUT_SUBSCRIPTION") { + subscriptionFactory.createDurableSubscription( + SubscriptionConfig( + "flow.timeout.task", + Schemas.Flow.FLOW_TIMEOUT_TOPIC + ), + flowMaintenanceHandlersFactory.createTimeoutEventHandler(stateManager, flowConfig), messagingConfig, null ) @@ -64,6 +80,16 @@ class FlowMaintenanceImpl @Activate constructor( } } + /** + * Flow logic requires some messaging config values + */ + private fun getFlowConfig( + config: Map, + messagingConfig: SmartConfig + ) = config.getConfig(ConfigKeys.FLOW_CONFIG) + .withValue(MAX_ALLOWED_MSG_SIZE, ConfigValueFactory.fromAnyRef(messagingConfig.getLong(MAX_ALLOWED_MSG_SIZE))) + .withValue(PROCESSOR_TIMEOUT, ConfigValueFactory.fromAnyRef(messagingConfig.getLong(PROCESSOR_TIMEOUT))) + override val isRunning: Boolean get() = coordinator.isRunning @@ -72,8 +98,6 @@ class FlowMaintenanceImpl @Activate constructor( } override fun stop() { - stateManager?.close() - stateManager = null coordinator.stop() } diff --git a/components/flow/flow-service/src/main/kotlin/net/corda/flow/maintenance/TimeoutEventCleanupProcessor.kt b/components/flow/flow-service/src/main/kotlin/net/corda/flow/maintenance/TimeoutEventCleanupProcessor.kt new file mode 100644 index 00000000000..0add6f4c643 --- /dev/null +++ b/components/flow/flow-service/src/main/kotlin/net/corda/flow/maintenance/TimeoutEventCleanupProcessor.kt @@ -0,0 +1,63 @@ +package net.corda.flow.maintenance + +import net.corda.avro.serialization.CordaAvroDeserializer +import net.corda.data.flow.FlowTimeout +import net.corda.data.flow.state.checkpoint.Checkpoint +import net.corda.flow.pipeline.exceptions.FlowFatalException +import net.corda.flow.state.impl.FlowCheckpointFactory +import net.corda.libs.configuration.SmartConfig +import net.corda.libs.statemanager.api.StateManager +import net.corda.messaging.api.processor.DurableProcessor +import net.corda.messaging.api.records.Record +import net.corda.utilities.debug +import org.slf4j.LoggerFactory + +class TimeoutEventCleanupProcessor( + private val checkpointCleanupHandler: CheckpointCleanupHandler, + private val stateManager: StateManager, + private val avroDeserializer: CordaAvroDeserializer, + private val flowCheckpointFactory: FlowCheckpointFactory, + private val config: SmartConfig +) : DurableProcessor { + + private companion object { + private val logger = LoggerFactory.getLogger(this::class.java.enclosingClass) + } + + override fun onNext(events: List>): List> { + logger.debug { "Processing ${events.size} flows for timeout" } + val statesToRecords = stateManager.get(events.mapNotNull { + it.value?.checkpointStateKey + }).mapNotNull { (_, state) -> + avroDeserializer.deserialize(state.value)?.let { + state to generateCleanupRecords(it) + } + }.toMap() + if (statesToRecords.size < events.size) { + logger.info( + "Could not process ${events.size - statesToRecords.size} events for flow session timeout cleanup as the " + + "checkpoint did not deserialize cleanly." + ) + } + val undeletedStates = stateManager.delete(statesToRecords.keys) + if (undeletedStates.isNotEmpty()) { + logger.info("Failed to delete ${undeletedStates.size} checkpoints when handling flow session timeout.") + } + val records = statesToRecords.filterKeys { !undeletedStates.containsKey(it.key) }.map { + it.value + }.flatten() + return records + } + + private fun generateCleanupRecords(checkpoint: Checkpoint): List> { + val flowCheckpoint = flowCheckpointFactory.create(checkpoint.flowId, checkpoint, config) + return checkpointCleanupHandler.cleanupCheckpoint( + flowCheckpoint, + config, + FlowFatalException("A session was timed out") + ) + } + + override val keyClass = String::class.java + override val valueClass = FlowTimeout::class.java +} \ No newline at end of file diff --git a/components/flow/flow-service/src/test/kotlin/net/corda/flow/maintenance/CheckpointCleanupHandlerImplTest.kt b/components/flow/flow-service/src/test/kotlin/net/corda/flow/maintenance/CheckpointCleanupHandlerImplTest.kt new file mode 100644 index 00000000000..165bd5ec13e --- /dev/null +++ b/components/flow/flow-service/src/test/kotlin/net/corda/flow/maintenance/CheckpointCleanupHandlerImplTest.kt @@ -0,0 +1,139 @@ +package net.corda.flow.maintenance + +import com.typesafe.config.ConfigValueFactory +import net.corda.data.flow.FlowInitiatorType +import net.corda.data.flow.FlowKey +import net.corda.data.flow.FlowStartContext +import net.corda.data.flow.state.session.SessionState +import net.corda.data.flow.state.session.SessionStateType +import net.corda.data.identity.HoldingIdentity +import net.corda.flow.pipeline.exceptions.FlowFatalException +import net.corda.flow.pipeline.exceptions.FlowMarkedForKillException +import net.corda.flow.pipeline.factory.FlowMessageFactory +import net.corda.flow.pipeline.factory.FlowRecordFactory +import net.corda.flow.pipeline.sessions.FlowSessionManager +import net.corda.flow.state.FlowCheckpoint +import net.corda.libs.configuration.SmartConfigImpl +import net.corda.schema.configuration.FlowConfig +import org.assertj.core.api.Assertions.assertThat +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import org.mockito.Mockito.mock +import org.mockito.kotlin.any +import org.mockito.kotlin.never +import org.mockito.kotlin.verify +import org.mockito.kotlin.whenever + +class CheckpointCleanupHandlerImplTest { + + private val config = SmartConfigImpl.empty() + .withValue(FlowConfig.SESSION_FLOW_CLEANUP_TIME, ConfigValueFactory.fromAnyRef(1000)) + .withValue(FlowConfig.PROCESSING_FLOW_CLEANUP_TIME, ConfigValueFactory.fromAnyRef(1000)) + private val flowRecordFactory = mock() + private val flowSessionManager = mock() + private val flowMessageFactory = mock() + + @BeforeEach + fun setup() { + whenever(flowRecordFactory.createFlowStatusRecord(any())).thenReturn(mock()) + whenever(flowRecordFactory.createFlowMapperEventRecord(any(), any())).thenReturn(mock()) + whenever(flowSessionManager.getSessionErrorEventRecords(any(), any(),any())).thenReturn(listOf(mock())) + whenever(flowMessageFactory.createFlowFailedStatusMessage(any(), any(), any())).thenReturn(mock()) + whenever(flowMessageFactory.createFlowKilledStatusMessage(any(), any())).thenReturn(mock()) + } + + @Test + fun `when there are active sessions and the checkpoint is RPC started correct number of output records are generated`() { + val checkpoint = setupCheckpoint(activeSessions = listOf("a", "b", "c"), rpcStarted = true) + val handler = CheckpointCleanupHandlerImpl(flowRecordFactory, flowSessionManager, flowMessageFactory) + val output = handler.cleanupCheckpoint(checkpoint, config, FlowFatalException("oops")) + // There should be: + // - 1 session error record, as the mock returns a single value + // - 3 session cleanup records, as the mock is called three times for this. + // - 1 status record + // - 1 mapper cleanup record + assertThat(output.size).isEqualTo(6) + verify(flowMessageFactory).createFlowFailedStatusMessage(any(), any(), any()) + verify(checkpoint).putSessionStates(any()) + verify(checkpoint).markDeleted() + } + + @Test + fun `when there are active sessions and the checkpoint is P2P started correct number of output records are generated`() { + val checkpoint = setupCheckpoint(activeSessions = listOf("a", "b", "c"), rpcStarted = false) + val handler = CheckpointCleanupHandlerImpl(flowRecordFactory, flowSessionManager, flowMessageFactory) + val output = handler.cleanupCheckpoint(checkpoint, config, FlowFatalException("oops")) + // There should be: + // - 1 session error record, as the mock returns a single value + // - 3 session cleanup records, as the mock is called three times for this. + // - 1 status record + assertThat(output.size).isEqualTo(5) + verify(flowMessageFactory).createFlowFailedStatusMessage(any(), any(), any()) + verify(checkpoint).putSessionStates(any()) + verify(checkpoint).markDeleted() + } + + @Test + fun `when there are no active sessions the checkpoint still errors any sessions with active errors`() { + val checkpoint = setupCheckpoint( + activeSessions = listOf(), + rpcStarted = false, + inactiveSessions = listOf("a", "b", "c") + ) + val handler = CheckpointCleanupHandlerImpl(flowRecordFactory, flowSessionManager, flowMessageFactory) + val output = handler.cleanupCheckpoint(checkpoint, config, FlowFatalException("oops")) + // There should be: + // - 1 session error record, as the mock returns a single value + // - 3 session cleanup records, as the mock is called three times for this. + // - 1 status record + assertThat(output.size).isEqualTo(5) + verify(flowMessageFactory).createFlowFailedStatusMessage(any(), any(), any()) + verify(checkpoint, never()).putSessionStates(any()) + verify(checkpoint).markDeleted() + } + + @Test + fun `when the flow is killed the right status message is created`() { + val checkpoint = setupCheckpoint(activeSessions = listOf("a", "b", "c"), rpcStarted = true) + val handler = CheckpointCleanupHandlerImpl(flowRecordFactory, flowSessionManager, flowMessageFactory) + val output = handler.cleanupCheckpoint(checkpoint, config, FlowMarkedForKillException("oops")) + // There should be: + // - 1 session error record, as the mock returns a single value + // - 3 session cleanup records, as the mock is called three times for this. + // - 1 status record + // - 1 mapper cleanup record + assertThat(output.size).isEqualTo(6) + verify(flowMessageFactory).createFlowKilledStatusMessage(any(), any()) + verify(checkpoint).putSessionStates(any()) + } + + private fun setupCheckpoint( + activeSessions: List, + rpcStarted: Boolean, + inactiveSessions: List = listOf() + ): FlowCheckpoint { + val checkpoint = mock() + whenever(checkpoint.flowKey).thenReturn(FlowKey("foo", HoldingIdentity("bar", "baz"))) + val howStarted = if (rpcStarted) FlowInitiatorType.RPC else FlowInitiatorType.P2P + val startContext = mock().apply { + whenever(initiatorType).thenReturn(howStarted) + } + whenever(checkpoint.flowStartContext).thenReturn(startContext) + val sessions = activeSessions.map { + mock().apply { + whenever(status).thenReturn(SessionStateType.CONFIRMED) + whenever(hasScheduledCleanup).thenReturn(false) + whenever(sessionId).thenReturn(it) + } + } + inactiveSessions.map { + mock().apply { + whenever(status).thenReturn(SessionStateType.ERROR) + whenever(hasScheduledCleanup).thenReturn(false) + whenever(sessionId).thenReturn(it) + } + } + whenever(checkpoint.sessions).thenReturn(sessions) + return checkpoint + } + +} \ No newline at end of file diff --git a/components/flow/flow-service/src/test/kotlin/net/corda/flow/maintenance/FlowMaintenanceImplTests.kt b/components/flow/flow-service/src/test/kotlin/net/corda/flow/maintenance/FlowMaintenanceImplTests.kt index f825584dc31..65cf844650f 100644 --- a/components/flow/flow-service/src/test/kotlin/net/corda/flow/maintenance/FlowMaintenanceImplTests.kt +++ b/components/flow/flow-service/src/test/kotlin/net/corda/flow/maintenance/FlowMaintenanceImplTests.kt @@ -1,11 +1,13 @@ package net.corda.flow.maintenance +import net.corda.data.flow.FlowTimeout import net.corda.data.scheduler.ScheduledTaskTrigger import net.corda.libs.configuration.SmartConfig import net.corda.libs.statemanager.api.StateManager import net.corda.libs.statemanager.api.StateManagerFactory import net.corda.lifecycle.LifecycleCoordinator import net.corda.lifecycle.LifecycleCoordinatorFactory +import net.corda.lifecycle.Resource import net.corda.messaging.api.subscription.Subscription import net.corda.messaging.api.subscription.factory.SubscriptionFactory import net.corda.schema.Schemas @@ -13,48 +15,72 @@ import net.corda.schema.configuration.ConfigKeys import org.junit.jupiter.api.Test import org.mockito.internal.verification.Times import org.mockito.kotlin.any +import org.mockito.kotlin.anyOrNull import org.mockito.kotlin.argThat import org.mockito.kotlin.argumentCaptor +import org.mockito.kotlin.doAnswer import org.mockito.kotlin.doReturn import org.mockito.kotlin.eq import org.mockito.kotlin.isNull import org.mockito.kotlin.mock import org.mockito.kotlin.never +import org.mockito.kotlin.times import org.mockito.kotlin.verify +import org.mockito.kotlin.whenever class FlowMaintenanceImplTests { + private val stateManager = mock() + private val stateManagerFactory = mock { + on { create(any()) } doReturn (stateManager) + } + private val subscription = mock>() + private val timeoutSubscription = mock>() private val lifecycleCoordinator = mock { - on { createManagedResource(any(), any<() -> Subscription>()) } doReturn (subscription) + val captor = argumentCaptor<() -> Resource>() + on { createManagedResource(any(), captor.capture()) } doAnswer { captor.lastValue.invoke() } } private val lifecycleCoordinatorFactory = mock { on { createCoordinator(any(), any()) } doReturn (lifecycleCoordinator) } private val subscriptionFactory = mock { - on { createDurableSubscription(any(), any(), any(), any()) } doReturn(subscription) - } - private val stateManager = mock() - private val stateManagerFactory = mock { - on { create(any()) } doReturn (stateManager) + on { createDurableSubscription(any(), any(), any(), anyOrNull()) } doReturn(subscription) + on { createDurableSubscription(any(), any(), any(), anyOrNull()) } doReturn (timeoutSubscription) } - private val flowMaintenance = FlowMaintenanceImpl(lifecycleCoordinatorFactory, subscriptionFactory, stateManagerFactory) - private val messagingConfig = mock() + private val messagingConfig = mock().apply { + whenever(getLong(any())).thenReturn(100L) + } private val stateManagerConfig = mock() + private val flowConfig = mock().apply { + whenever(withValue(any(), any())).thenReturn(this) + } private val config = mapOf( ConfigKeys.MESSAGING_CONFIG to messagingConfig, - ConfigKeys.STATE_MANAGER_CONFIG to stateManagerConfig + ConfigKeys.STATE_MANAGER_CONFIG to stateManagerConfig, + ConfigKeys.FLOW_CONFIG to flowConfig + ) + + private val flowMaintenanceHandlersFactory = mock { + on { createScheduledTaskHandler(any()) } doReturn (SessionTimeoutTaskProcessor(stateManager)) + on { createTimeoutEventHandler(any(), any()) } doReturn ( + TimeoutEventCleanupProcessor(mock(), stateManager, mock(), mock(), flowConfig) + ) + } + + private val flowMaintenance = FlowMaintenanceImpl( + lifecycleCoordinatorFactory, + subscriptionFactory, + stateManagerFactory, + flowMaintenanceHandlersFactory ) @Test fun `when config provided create subscription and start it`() { - val captor = argumentCaptor<() -> Subscription>() - flowMaintenance.onConfigChange(config) - verify(lifecycleCoordinator).createManagedResource(any(), captor.capture()) - captor.firstValue() - verify(subscriptionFactory).createDurableSubscription( + verify(lifecycleCoordinator, times(3)).createManagedResource(any(), any<() -> Resource>()) + verify(subscriptionFactory, times(1)).createDurableSubscription( argThat { it -> it.eventTopic == Schemas.ScheduledTask.SCHEDULED_TASK_TOPIC_FLOW_PROCESSOR }, @@ -62,8 +88,17 @@ class FlowMaintenanceImplTests { eq(messagingConfig), isNull() ) + verify(subscriptionFactory, times(1)).createDurableSubscription( + argThat { it -> + it.eventTopic == Schemas.Flow.FLOW_TIMEOUT_TOPIC + }, + any(), + eq(messagingConfig), + isNull() + ) verify(stateManagerFactory).create(stateManagerConfig) verify(subscription).start() + verify(timeoutSubscription).start() } @Test @@ -85,16 +120,19 @@ class FlowMaintenanceImplTests { @Test fun `when new state manager config pushed create another StateManager and close old`() { flowMaintenance.onConfigChange(config) - val newConfig = mock() + val newConfig = mock().apply { + whenever(withValue(any(), any())).thenReturn(this) + } flowMaintenance.onConfigChange( mapOf( ConfigKeys.MESSAGING_CONFIG to newConfig, ConfigKeys.STATE_MANAGER_CONFIG to newConfig, + ConfigKeys.FLOW_CONFIG to newConfig ) ) verify(stateManagerFactory).create(newConfig) - verify(stateManager).close() + verify(lifecycleCoordinator, times(6)).createManagedResource(any(), any<() -> Resource>()) } @Test @@ -102,7 +140,7 @@ class FlowMaintenanceImplTests { flowMaintenance.onConfigChange(config) flowMaintenance.stop() - verify(stateManager).close() + verify(lifecycleCoordinator).stop() } @Test diff --git a/components/flow/flow-service/src/test/kotlin/net/corda/flow/maintenance/TimeoutEventCleanupProcessorTest.kt b/components/flow/flow-service/src/test/kotlin/net/corda/flow/maintenance/TimeoutEventCleanupProcessorTest.kt new file mode 100644 index 00000000000..4991e6e8e29 --- /dev/null +++ b/components/flow/flow-service/src/test/kotlin/net/corda/flow/maintenance/TimeoutEventCleanupProcessorTest.kt @@ -0,0 +1,109 @@ +package net.corda.flow.maintenance + +import net.corda.avro.serialization.CordaAvroDeserializer +import net.corda.data.flow.FlowTimeout +import net.corda.data.flow.state.checkpoint.Checkpoint +import net.corda.flow.state.impl.FlowCheckpointFactory +import net.corda.libs.configuration.SmartConfig +import net.corda.libs.statemanager.api.State +import net.corda.libs.statemanager.api.StateManager +import net.corda.messaging.api.records.Record +import org.assertj.core.api.Assertions.assertThat +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import org.mockito.Mockito.mock +import org.mockito.kotlin.any +import org.mockito.kotlin.whenever +import java.time.Instant + +class TimeoutEventCleanupProcessorTest { + private val checkpointCleanupHandler = mock() + private val stateManager = mock() + private val avroDeserializer = mock>() + private val flowCheckpointFactory = mock() + private val config = mock() + + private val inputRecords = listOf( + buildRecord("foo"), + buildRecord("bar") + ) + + @BeforeEach + fun setup() { + val checkpoint = mock().apply { + whenever(flowId).thenReturn("flowID") + } + whenever(avroDeserializer.deserialize(any())).thenReturn(checkpoint) + whenever(flowCheckpointFactory.create(any(), any(), any())).thenReturn(mock()) + whenever(checkpointCleanupHandler.cleanupCheckpoint(any(), any(), any())).thenReturn(listOf(mock())) + } + + @Test + fun `when timeout processor receives events with states output records are generated`() { + whenever(stateManager.get(any())).thenReturn(inputRecords.associate { it.key to buildState(it.key) }) + whenever(stateManager.delete(any())).thenReturn(mapOf()) + val processor = TimeoutEventCleanupProcessor( + checkpointCleanupHandler, + stateManager, + avroDeserializer, + flowCheckpointFactory, + config + ) + val output = processor.onNext(inputRecords) + assertThat(output.size).isEqualTo(2) + } + + @Test + fun `when timeout processor fails to delete a state no records are output`() { + whenever(stateManager.get(any())).thenReturn(inputRecords.associate { it.key to buildState(it.key) }) + whenever(stateManager.delete(any())).thenReturn(inputRecords.associate { it.key to buildState(it.key) }) + val processor = TimeoutEventCleanupProcessor( + checkpointCleanupHandler, + stateManager, + avroDeserializer, + flowCheckpointFactory, + config + ) + val output = processor.onNext(inputRecords) + assertThat(output).isEmpty() + } + + @Test + fun `when state manager does not have states available no records are output`() { + whenever(stateManager.get(any())).thenReturn(mapOf()) + whenever(stateManager.delete(any())).thenReturn(mapOf()) + val processor = TimeoutEventCleanupProcessor( + checkpointCleanupHandler, + stateManager, + avroDeserializer, + flowCheckpointFactory, + config + ) + val output = processor.onNext(inputRecords) + assertThat(output).isEmpty() + } + + @Test + fun `when avro deserializer fails to deserialize no records are output`() { + whenever(stateManager.get(any())).thenReturn(inputRecords.associate { it.key to buildState(it.key) }) + whenever(avroDeserializer.deserialize(any())).thenReturn(null) + whenever(stateManager.delete(any())).thenReturn(mapOf()) + val processor = TimeoutEventCleanupProcessor( + checkpointCleanupHandler, + stateManager, + avroDeserializer, + flowCheckpointFactory, + config + ) + val output = processor.onNext(inputRecords) + assertThat(output).isEmpty() + } + + private fun buildRecord(key: String) : Record { + return Record("timeout", key, FlowTimeout(key, Instant.now())) + } + + private fun buildState(key: String) : State { + return State(key, byteArrayOf()) + } +} \ No newline at end of file From f8261e32921d1d8f6a411ccce1bf467440fb4eb3 Mon Sep 17 00:00:00 2001 From: Lorcan Wogan <69468264+LWogan@users.noreply.github.com> Date: Wed, 25 Oct 2023 14:49:50 +0100 Subject: [PATCH 72/81] CORE-17950 make start flow events idempotent (#4965) The multi source mediator currently updates the state manager after processing each event in a batch. If there is an error that occurs after this has happened, the consumer will rollback to its last poll position and replay events. The StartFlow event is the only FlowEvent not idempotent. This PR addresses this and will ignore subsequent StartFlowEvents unless the checkpoint is null or we are in a retry state and flow state is null (indicating the StartFlow is what needs to be retried) --- .../corda/flow/testing/tests/StartFlowTest.kt | 1 - .../pipeline/impl/FlowEventProcessorImpl.kt | 14 ++++++- .../impl/FlowEventProcessorImplTest.kt | 37 +++++++++++++++++++ 3 files changed, 50 insertions(+), 2 deletions(-) diff --git a/components/flow/flow-service/src/integrationTest/kotlin/net/corda/flow/testing/tests/StartFlowTest.kt b/components/flow/flow-service/src/integrationTest/kotlin/net/corda/flow/testing/tests/StartFlowTest.kt index a7795e71474..ff76acc2935 100644 --- a/components/flow/flow-service/src/integrationTest/kotlin/net/corda/flow/testing/tests/StartFlowTest.kt +++ b/components/flow/flow-service/src/integrationTest/kotlin/net/corda/flow/testing/tests/StartFlowTest.kt @@ -80,7 +80,6 @@ class StartFlowTest : FlowServiceTestBase() { * Scenario 1 - Fails multiple times but completes before the retry limit * Scenario 2 - Fails multiple times and hits the retry limit failing the flow to the DLQ */ - @Test fun `RPC Start Flow - Retry scenario 1 - Fail then succeeds`() { given { diff --git a/components/flow/flow-service/src/main/kotlin/net/corda/flow/pipeline/impl/FlowEventProcessorImpl.kt b/components/flow/flow-service/src/main/kotlin/net/corda/flow/pipeline/impl/FlowEventProcessorImpl.kt index f840c33916f..f7caeab4e81 100644 --- a/components/flow/flow-service/src/main/kotlin/net/corda/flow/pipeline/impl/FlowEventProcessorImpl.kt +++ b/components/flow/flow-service/src/main/kotlin/net/corda/flow/pipeline/impl/FlowEventProcessorImpl.kt @@ -1,6 +1,7 @@ package net.corda.flow.pipeline.impl import net.corda.data.flow.event.FlowEvent +import net.corda.data.flow.event.StartFlow import net.corda.data.flow.state.checkpoint.Checkpoint import net.corda.flow.pipeline.FlowEventExceptionProcessor import net.corda.flow.pipeline.FlowMDCService @@ -80,7 +81,6 @@ class FlowEventProcessorImpl( ) } - val pipeline = try { log.trace { "Flow [${event.key}] Received event: ${flowEvent.payload::class.java} / ${flowEvent.payload}" } flowEventPipelineFactory.create(state, flowEvent, configs, mdcProperties, traceContext, event.timestamp) @@ -94,6 +94,18 @@ class FlowEventProcessorImpl( ) } + val checkpoint = state?.value + val isInRetryState = pipeline.context.checkpoint.inRetryState && checkpoint?.flowState == null + val flowEventPayload = flowEvent.payload + + if (flowEventPayload is StartFlow && checkpoint != null && !isInRetryState) { + log.debug { "Ignoring duplicate '${StartFlow::class.java}'. Checkpoint has already been initialized" } + return StateAndEventProcessor.Response( + state, + listOf() + ) + } + // flow result timeout must be lower than the processor timeout as the processor thread will be killed by the subscription consumer // thread after this period and so this timeout would never be reached and given a chance to return otherwise. val flowTimeout = (flowConfig.getLong(PROCESSOR_TIMEOUT) * 0.75).toLong() diff --git a/components/flow/flow-service/src/test/kotlin/net/corda/flow/pipeline/impl/FlowEventProcessorImplTest.kt b/components/flow/flow-service/src/test/kotlin/net/corda/flow/pipeline/impl/FlowEventProcessorImplTest.kt index b04e605803b..a3208b8ca9a 100644 --- a/components/flow/flow-service/src/test/kotlin/net/corda/flow/pipeline/impl/FlowEventProcessorImplTest.kt +++ b/components/flow/flow-service/src/test/kotlin/net/corda/flow/pipeline/impl/FlowEventProcessorImplTest.kt @@ -285,6 +285,43 @@ class FlowEventProcessorImplTest { verify(flowMDCService, times(1)).getMDCLogging(anyOrNull(), any(), any()) } + @Test + fun `Execute flow pipeline with a checkpoint and start flow event`() { + val inputEvent = getFlowEventRecord(FlowEvent(flowKey, startFlowEvent)) + + val response = processor.onNext(state, inputEvent) + + assertThat(response).isEqualTo(StateAndEventProcessor.Response(state, emptyList(), false)) + verify(flowMDCService, times(1)).getMDCLogging(anyOrNull(), any(), any()) + verify(flowEventPipelineFactory, times(1)).create(any(),any(),any(),any(),any(),any()) + } + + @Test + fun `Execute flow pipeline with a checkpoint and start flow event in retry mode with no FlowState`() { + val inputEvent = getFlowEventRecord(FlowEvent(flowKey, startFlowEvent)) + whenever(flowCheckpoint.inRetryState).thenReturn(true) + whenever(checkpoint.flowState).thenReturn(null) + + val response = processor.onNext(state, inputEvent) + + assertThat(response).isEqualTo(outputResponse) + verify(flowMDCService, times(1)).getMDCLogging(anyOrNull(), any(), any()) + verify(flowEventPipelineFactory, times(1)).create(any(),any(),any(),any(),any(),any()) + } + + @Test + fun `Execute flow pipeline with a checkpoint and start flow event in retry mode with a FlowState`() { + val inputEvent = getFlowEventRecord(FlowEvent(flowKey, startFlowEvent)) + whenever(flowCheckpoint.inRetryState).thenReturn(true) + whenever(checkpoint.flowState).thenReturn(flowState) + + val response = processor.onNext(state, inputEvent) + + assertThat(response).isEqualTo(StateAndEventProcessor.Response(state, emptyList(), false)) + verify(flowMDCService, times(1)).getMDCLogging(anyOrNull(), any(), any()) + verify(flowEventPipelineFactory, times(1)).create(any(),any(),any(),any(),any(),any()) + } + @Test fun `Execute flow pipeline from null checkpoint and session init event`() { val inputEvent = getFlowEventRecord(FlowEvent(flowKey, sessionInitFlowEvent)) From 4867017bb68e0300d164ff7aa7746e8cd6c42bb1 Mon Sep 17 00:00:00 2001 From: dickon Date: Wed, 25 Oct 2023 14:52:32 +0100 Subject: [PATCH 73/81] CORE-17963: add lock around sandbox (#4973) The flow worker now calls sandbox code from multiple threads. This appears to cause problems in the sandbox code, so for now put a reentrant lock around all the interface methods of SandboxGroupContextServiceImpl. The only potentially issue is that there are callbacks out of the sandbox code when eviction happens, and they will hold the lock. However, this appears to be harmless. Without this test, we saw large network test failures in the first few flows, and with this we are running 14,396 flows without issues. In future we should review all the code used by the flow worker for thread safety. --- .../impl/SandboxGroupContextServiceImpl.kt | 47 ++++++++++--------- 1 file changed, 26 insertions(+), 21 deletions(-) diff --git a/components/virtual-node/sandbox-group-context-service/src/main/kotlin/net/corda/sandboxgroupcontext/service/impl/SandboxGroupContextServiceImpl.kt b/components/virtual-node/sandbox-group-context-service/src/main/kotlin/net/corda/sandboxgroupcontext/service/impl/SandboxGroupContextServiceImpl.kt index 75e1a3a3320..cc42840c6ef 100644 --- a/components/virtual-node/sandbox-group-context-service/src/main/kotlin/net/corda/sandboxgroupcontext/service/impl/SandboxGroupContextServiceImpl.kt +++ b/components/virtual-node/sandbox-group-context-service/src/main/kotlin/net/corda/sandboxgroupcontext/service/impl/SandboxGroupContextServiceImpl.kt @@ -58,6 +58,8 @@ import java.util.SortedMap import java.util.TreeMap import java.util.UUID import java.util.concurrent.CompletableFuture +import java.util.concurrent.locks.ReentrantLock +import kotlin.concurrent.withLock import kotlin.contracts.ExperimentalContracts import kotlin.contracts.InvocationKind import kotlin.contracts.contract @@ -121,40 +123,41 @@ class SandboxGroupContextServiceImpl @Activate constructor( } private val cache = SandboxGroupContextCacheImpl(0) + private val lock = ReentrantLock() - override fun resizeCache(type: SandboxGroupType, capacity: Long) { + override fun resizeCache(type: SandboxGroupType, capacity: Long) = lock.withLock { if (capacity != cache.capacities[type]) { logger.info("Changing Sandbox cache capacity for type {} from {} to {}", type, cache.capacities[type], capacity) cache.resize(type, capacity) } } - override fun flushCache(): CompletableFuture<*> { - return cache.flush() + override fun flushCache(): CompletableFuture<*> = lock.withLock { + cache.flush() } @Throws(InterruptedException::class) - override fun waitFor(completion: CompletableFuture<*>, duration: Duration): Boolean { + override fun waitFor(completion: CompletableFuture<*>, duration: Duration): Boolean = lock.withLock { return cache.waitFor(completion, duration) } - override fun remove(virtualNodeContext: VirtualNodeContext): CompletableFuture<*>? { - return cache.remove(virtualNodeContext) + override fun remove(virtualNodeContext: VirtualNodeContext): CompletableFuture<*>? = lock.withLock { + cache.remove(virtualNodeContext) } - override fun addEvictionListener(type: SandboxGroupType, listener: EvictionListener): Boolean { - return cache.addEvictionListener(type, listener) + override fun addEvictionListener(type: SandboxGroupType, listener: EvictionListener): Boolean = lock.withLock { + cache.addEvictionListener(type, listener) } - override fun removeEvictionListener(type: SandboxGroupType, listener: EvictionListener): Boolean { - return cache.removeEvictionListener(type, listener) + override fun removeEvictionListener(type: SandboxGroupType, listener: EvictionListener): Boolean = lock.withLock{ + cache.removeEvictionListener(type, listener) } override fun getOrCreate( virtualNodeContext: VirtualNodeContext, initializer: SandboxGroupContextInitializer - ): SandboxGroupContext { - return cache.get(virtualNodeContext) { vnc -> + ): SandboxGroupContext = lock.withLock { + cache.get(virtualNodeContext) { vnc -> val sandboxTimer = CordaMetrics.Metric.SandboxCreateTime.builder() .forVirtualNode(vnc.holdingIdentity.shortHash.value) .withTag(CordaMetrics.Tag.SandboxGroupType, vnc.sandboxGroupType.name) @@ -326,7 +329,7 @@ class SandboxGroupContextServiceImpl @Activate constructor( serviceNames: (CpkMetadata) -> Iterable, isMetadataService: (Class<*>) -> Boolean, serviceMarkerType: Class<*> - ): AutoCloseable { + ): AutoCloseable = lock.withLock { val group = sandboxGroupContext.sandboxGroup val services = group.metadata.flatMap { (mainBundle, cpkMetadata) -> // Fetch metadata classes provided by each CPK main bundle. @@ -389,14 +392,16 @@ class SandboxGroupContextServiceImpl @Activate constructor( } override fun acceptCustomMetadata(sandboxGroupContext: MutableSandboxGroupContext) { - sandboxGroupContext.getObjectByKey>(SANDBOX_SINGLETONS) - ?.filterIsInstance() - ?.forEach { customMetadataConsumer -> - customMetadataConsumer.accept(sandboxGroupContext) - } + lock.withLock { + sandboxGroupContext.getObjectByKey>(SANDBOX_SINGLETONS) + ?.filterIsInstance() + ?.forEach { customMetadataConsumer -> + customMetadataConsumer.accept(sandboxGroupContext) + } + } } - override fun hasCpks(cpkChecksums: Set): Boolean { + override fun hasCpks(cpkChecksums: Set): Boolean = lock.withLock { val missingCpks = cpkChecksums.filter { cpkReadService.get(it) == null } @@ -405,11 +410,11 @@ class SandboxGroupContextServiceImpl @Activate constructor( logger.info("CPK(s) not (yet) found in cache: {}", missingCpks) } - return missingCpks.isEmpty() + missingCpks.isEmpty() } @Deactivate - override fun close() { + override fun close() = lock.withLock { cache.close() } From 55af91432e7c61f17373ce77209e8fb30f8abbe7 Mon Sep 17 00:00:00 2001 From: Owen Stanford <92725587+owenstanford@users.noreply.github.com> Date: Wed, 25 Oct 2023 16:40:14 +0100 Subject: [PATCH 74/81] CORE-17502: Add token selection HTTP RPC (#4910) CORE-17502 - Migrate the flow/token selection RPC from Kafka to the new HTTP RPC Model CORE-17503 - Migrate the token selection state model from the Kafka state and event pattern to the new State Manage model --- .ci/e2eTests/JenkinsfileCombinedWorker | 2 +- .run/Combined Worker Local.run.xml | 2 +- .../selection/SimpleHttpRPCPerformanceTest.kt | 214 +++++++++++++++ .../mediator/FlowEventMediatorFactoryImpl.kt | 5 +- .../corda/flow/service/FlowExecutorImpl.kt | 4 +- .../FlowEventMediatorFactoryImplTest.kt | 8 +- .../flow/service/FlowExecutorImplTest.kt | 1 + .../ledger/ledger-utxo-flow/build.gradle | 1 + .../TokenClaimReleasePostProcessingHandler.kt | 12 +- .../factories/TokenClaimFactoryImplTest.kt | 3 +- ...enClaimReleasePostProcessingHandlerTest.kt | 34 ++- .../ledger-utxo-token-cache/build.gradle | 7 +- .../utxo/token/cache/entities/TokenPoolKey.kt | 15 ++ .../factories/TokenCacheComponentFactory.kt | 48 +++- .../TokenCacheEventProcessorFactory.kt | 7 + .../TokenCacheEventProcessorFactoryImpl.kt | 48 ++-- .../handlers/TokenClaimQueryEventHandler.kt | 1 - .../services/BasicClaimStateStoreImpl.kt | 59 +++++ .../token/cache/services/ClaimStateStore.kt | 15 ++ .../cache/services/ClaimStateStoreCache.kt | 7 + .../services/ClaimStateStoreCacheImpl.kt | 75 ++++++ .../cache/services/ClaimStateStoreFactory.kt | 7 + .../services/ClaimStateStoreFactoryImpl.kt | 16 ++ .../PerformanceClaimStateStoreImpl.kt | 114 ++++++++ .../cache/services/StoredPoolClaimState.kt | 10 + .../cache/services/TokenCacheComponent.kt | 9 +- .../services/TokenCacheEventProcessor.kt | 97 +++---- .../TokenPoolCacheStateSerialization.kt | 11 + .../TokenPoolCacheStateSerializationImpl.kt | 21 ++ .../TokenSelectionDelegatedProcessor.kt | 13 + .../TokenSelectionDelegatedProcessorImpl.kt | 86 ++++++ .../TokenSelectionSyncRPCProcessor.kt | 22 ++ .../TokenCacheSubscriptionHandlerImpl.kt | 38 ++- .../utxo/token/cache/impl/ExampleData.kt | 40 ++- .../cache/impl/entities/TokenPoolKeyTest.kt | 31 +++ .../services/BasicClaimStateStoreImplTest.kt | 91 +++++++ .../services/ClaimStateStoreCacheImplTest.kt | 96 +++++++ .../PerformanceClaimStateStoreImplTest.kt | 247 ++++++++++++++++++ .../impl/services/TokenCacheComponentTest.kt | 12 +- .../services/TokenCacheEventProcessorTest.kt | 14 +- .../TokenCacheSubscriptionHandlerImplTest.kt | 54 ++-- ...okenSelectionDelegatedProcessorImplTest.kt | 181 +++++++++++++ .../net/corda/messaging/mediator/RPCClient.kt | 7 +- .../subscription/SyncRPCSubscriptionImpl.kt | 21 +- .../corda/messaging/mediator/RPCClientTest.kt | 18 +- .../SyncRPCSubscriptionImplTest.kt | 18 ++ .../messaging/api/constants/WorkerRPCPaths.kt | 1 + .../api/processor/SyncRPCProcessor.kt | 2 +- processors/token-cache-processor/build.gradle | 1 + 49 files changed, 1707 insertions(+), 139 deletions(-) create mode 100644 applications/workers/workers-smoketest/src/smokeTest/kotlin/net/corda/applications/workers/smoketest/token/selection/SimpleHttpRPCPerformanceTest.kt create mode 100644 components/ledger/ledger-utxo-token-cache/src/main/kotlin/net/corda/ledger/utxo/token/cache/services/BasicClaimStateStoreImpl.kt create mode 100644 components/ledger/ledger-utxo-token-cache/src/main/kotlin/net/corda/ledger/utxo/token/cache/services/ClaimStateStore.kt create mode 100644 components/ledger/ledger-utxo-token-cache/src/main/kotlin/net/corda/ledger/utxo/token/cache/services/ClaimStateStoreCache.kt create mode 100644 components/ledger/ledger-utxo-token-cache/src/main/kotlin/net/corda/ledger/utxo/token/cache/services/ClaimStateStoreCacheImpl.kt create mode 100644 components/ledger/ledger-utxo-token-cache/src/main/kotlin/net/corda/ledger/utxo/token/cache/services/ClaimStateStoreFactory.kt create mode 100644 components/ledger/ledger-utxo-token-cache/src/main/kotlin/net/corda/ledger/utxo/token/cache/services/ClaimStateStoreFactoryImpl.kt create mode 100644 components/ledger/ledger-utxo-token-cache/src/main/kotlin/net/corda/ledger/utxo/token/cache/services/PerformanceClaimStateStoreImpl.kt create mode 100644 components/ledger/ledger-utxo-token-cache/src/main/kotlin/net/corda/ledger/utxo/token/cache/services/StoredPoolClaimState.kt create mode 100644 components/ledger/ledger-utxo-token-cache/src/main/kotlin/net/corda/ledger/utxo/token/cache/services/TokenPoolCacheStateSerialization.kt create mode 100644 components/ledger/ledger-utxo-token-cache/src/main/kotlin/net/corda/ledger/utxo/token/cache/services/TokenPoolCacheStateSerializationImpl.kt create mode 100644 components/ledger/ledger-utxo-token-cache/src/main/kotlin/net/corda/ledger/utxo/token/cache/services/TokenSelectionDelegatedProcessor.kt create mode 100644 components/ledger/ledger-utxo-token-cache/src/main/kotlin/net/corda/ledger/utxo/token/cache/services/TokenSelectionDelegatedProcessorImpl.kt create mode 100644 components/ledger/ledger-utxo-token-cache/src/main/kotlin/net/corda/ledger/utxo/token/cache/services/TokenSelectionSyncRPCProcessor.kt create mode 100644 components/ledger/ledger-utxo-token-cache/src/test/kotlin/net/corda/ledger/utxo/token/cache/impl/entities/TokenPoolKeyTest.kt create mode 100644 components/ledger/ledger-utxo-token-cache/src/test/kotlin/net/corda/ledger/utxo/token/cache/impl/services/BasicClaimStateStoreImplTest.kt create mode 100644 components/ledger/ledger-utxo-token-cache/src/test/kotlin/net/corda/ledger/utxo/token/cache/impl/services/ClaimStateStoreCacheImplTest.kt create mode 100644 components/ledger/ledger-utxo-token-cache/src/test/kotlin/net/corda/ledger/utxo/token/cache/impl/services/PerformanceClaimStateStoreImplTest.kt create mode 100644 components/ledger/ledger-utxo-token-cache/src/test/kotlin/net/corda/ledger/utxo/token/cache/impl/services/TokenSelectionDelegatedProcessorImplTest.kt diff --git a/.ci/e2eTests/JenkinsfileCombinedWorker b/.ci/e2eTests/JenkinsfileCombinedWorker index 46f087b1567..f78de78a232 100644 --- a/.ci/e2eTests/JenkinsfileCombinedWorker +++ b/.ci/e2eTests/JenkinsfileCombinedWorker @@ -90,7 +90,7 @@ pipeline { REST_TLS_PATH = "${env.WORKSPACE}/applications/workers/release/combined-worker/tls/rest/rest_worker.pfx" VM_PARAMETERS = "-Dco.paralleluniverse.fibers.verifyInstrumentation=true" LOG4J_PARAMETERS = "-Dlog4j.configurationFile=log4j2-console.xml" - PROGRAM_PARAMETERS = "--instance-id=0 -mbus.busType=DATABASE -spassphrase=password -ssalt=salt -ddatabase.user=u${postgresDb} -ddatabase.pass=password -ddatabase.jdbc.url=jdbc:postgresql://${postgresHost}:${postgresPort}/${postgresDb} -ddatabase.jdbc.directory=${JDBC_PATH} -rtls.keystore.path=${REST_TLS_PATH} -rtls.keystore.password=mySecretPassword --serviceEndpoint=endpoints.crypto=localhost:7004 --serviceEndpoint=endpoints.verification=localhost:7004 --serviceEndpoint=endpoints.uniqueness=localhost:7004 --serviceEndpoint=endpoints.persistence=localhost:7004" + PROGRAM_PARAMETERS = "--instance-id=0 -mbus.busType=DATABASE -spassphrase=password -ssalt=salt -ddatabase.user=u${postgresDb} -ddatabase.pass=password -ddatabase.jdbc.url=jdbc:postgresql://${postgresHost}:${postgresPort}/${postgresDb} -ddatabase.jdbc.directory=${JDBC_PATH} -rtls.keystore.path=${REST_TLS_PATH} -rtls.keystore.password=mySecretPassword --serviceEndpoint=endpoints.crypto=localhost:7004 --serviceEndpoint=endpoints.verification=localhost:7004 --serviceEndpoint=endpoints.uniqueness=localhost:7004 --serviceEndpoint=endpoints.persistence=localhost:7004 --serviceEndpoint=endpoints.tokenSelection=localhost:7004" WORKING_DIRECTORY = "${env.WORKSPACE}" } steps { diff --git a/.run/Combined Worker Local.run.xml b/.run/Combined Worker Local.run.xml index ec8e493d59d..c518d5853f4 100644 --- a/.run/Combined Worker Local.run.xml +++ b/.run/Combined Worker Local.run.xml @@ -2,7 +2,7 @@