ES-1707: Merging forward updates from release/os/5.1 to release/interop/syntax - 2023-12-29

.ci/dev/forward-merge/JenkinsfileMergeAutomation

@@ -0,0 +1,32 @@
+#! groovy
+@Library('[email protected]') _
+
+/**
+ * Forward merge any changes in current branch to the branch with following version.
+ *
+ * Please note, the branches names are intentionally separated as variables, to minimised conflicts
+ * during automated merges for this file.
+ *
+ * These variables should be updated when a new version is cut
+ */
+
+/**
+ * the branch name of origin branch, it should match the current branch
+ * and it acts as a fail-safe inside {@code forwardMerger} pipeline
+ */
+String originBranch = 'release/os/5.1'
+
+/**
+ * the branch name of target branch, it should be the branch with the next version
+ * after the one in current branch.
+ */
+String targetBranch = 'release/os/5.2'
+
+/**
+ * Forward merge any changes between {@code originBranch} and {@code targetBranch}
+ */
+forwardMerger(
+    targetBranch: targetBranch,
+    originBranch: originBranch,
+    slackChannel: '#c5-forward-merge-bot-notifications'
+)

.github/CODEOWNERS

@@ -1 +1,107 @@
-* @driessamyn @jasonbyrner3 @dimosr @ronanbrowne @rick-r3 @simon-johnson-r3 @blsemo @Omar-awad @aditisdesai @vinir3 @vkolomeyko @thiagoviana @Sakpal
+
+# Build scripts should be audited by BLT
+
+Jenkinsfile        @corda/blt
+.ci/**             @corda/blt
+
+*.gradle           @corda/blt
+gradle.properties  @corda/corda5-team-leads
+gradle/*           @corda/blt
+
+.github/**          @corda/blt
+CODEOWNERS         @corda/blt @corda/corda5-team-leads
+
+# Modules to be audited by REST team
+/applications/workers/release/rest-worker/      @corda/rest
+/components/rest-gateway-comp/                  @corda/rest
+/components/permissions/                        @corda/rest
+/components/rbac-security-manager-service/      @corda/rest
+/libs/rest/                                     @corda/rest
+/libs/permissions/                              @corda/rest
+/processors/rest-processor/                     @corda/rest
+/tools/plugins/initial-rbac/                    @corda/rest
+/tools/plugins/plugins-rest/                    @corda/rest
+/tools/plugins/virtual-node/                    @corda/rest
+
+# Corda Helm chart for cluster management team
+/charts/corda/                                @corda/cluster-management
+
+# Modules to be audited by the Network team
+/applications/workers/release/p2p-gateway-worker/               @corda/corda-platform-network-team
+/applications/workers/release/p2p-link-manager-worker/          @corda/corda-platform-network-team
+/applications/workers/release/member-worker/                    @corda/corda-platform-network-team
+/processors/link-manager-processor/                             @corda/corda-platform-network-team
+/processors/gateway-processor/                                  @corda/corda-platform-network-team
+/processors/member-processor/                                   @corda/corda-platform-network-team
+/components/gateway/                                            @corda/corda-platform-network-team
+/components/link-manager/                                       @corda/corda-platform-network-team
+/components/membership/                                         @corda/corda-platform-network-team
+/libs/membership/                                               @corda/corda-platform-network-team
+/libs/p2p-crypto/                                               @corda/corda-platform-network-team
+/libs/layered-property-map/                                     @corda/corda-platform-network-team
+/tools/plugins/mgm/                                             @corda/corda-platform-network-team
+/tools/plugins/network/                                         @corda/corda-platform-network-team
+/applications/tools/p2p-test/                                   @corda/corda-platform-network-team
+
+# Modules to be audited by Sandboxing SMEs
+/components/security-manager/                                   @corda/sandboxing
+/components/virtual-node/sandbox-*                              @corda/sandboxing
+/components/sandbox*                                            @corda/sandboxing
+/libs/virtual-node/sandbox-*                                    @corda/sandboxing
+/osgi-*                                                         @corda/sandboxing
+/testing/sandboxes/                                             @corda/sandboxing
+/testing/sandboxes-testkit/                                     @corda/sandboxing
+/testing/security-manager-utilities/                            @corda/sandboxing
+
+# Modules to be audited by Crypto SMEs
+/components/crypto/                                             @corda/crypto
+/libs/crypto/                                                   @corda/crypto
+/processors/crypto/                                             @corda/crypto
+
+# Modules to be audited by Packaging SMEs
+/components/chunking/                                           @corda/packaging
+/components/virtual-node/cpi-*                                  @corda/packaging
+/components/virtual-node/cpk-*                                  @corda/packaging
+/libs/chunking/                                                 @corda/packaging
+/libs/packaging/                                                @corda/packaging
+/libs/serialization/                                            @corda/packaging
+/libs/virtual-node/cpi-*                                        @corda/packaging
+/testing/packaging-test-utilities/                              @corda/packaging
+/tools/plugins/package                                          @corda/packaging
+
+# Modules to be audited by DB SMEs
+/components/db/                                                 @corda/db
+/components/persistence/                                        @corda/db
+/components/reconciliation/                                     @corda/db
+/libs/db/                                                       @corda/db
+/processors/db/                                                 @corda/db
+/testing/persistence-testkit/                                   @corda/db
+/tools/plugins/db-config                                        @corda/db
+
+# Modules to be audited by Flow Worker team
+/components/flow/                                               @corda/flow-worker
+/libs/flows/                                                    @corda/flow-worker
+/libs/lifecycle/                                                @corda/flow-worker
+/libs/messaging/                                                @corda/flow-worker
+/libs/application/application-impl/                             @corda/flow-worker
+/processors/flow-processor/                                     @corda/flow-worker
+/testing/flow/                                                  @corda/flow-worker
+/testing/message-patterns/                                      @corda/flow-worker
+/applications/workers/release/flow-worker                       @corda/flow-worker
+
+# Modules to be audited by Ledger SMEs
+/components/ledger/                                             @corda/ledger
+/libs/ledger/                                                   @corda/ledger
+/testing/ledger/                                                @corda/ledger
+
+# Modules to be audited by Notary SMEs
+/components/uniqueness/                                         @corda/notaries
+/libs/uniqueness/                                               @corda/notaries
+/notary-plugins/                                                @corda/notaries
+/processors/uniqueness-processor/                               @corda/notaries
+/testing/uniqueness/                                            @corda/notaries
+
+# Ledger token selection files to be reviewed by the REST team
+# This needs to be after the ledger rules to partially override those
+/components/ledger/ledger-utxo-token-cache                      @corda/rest
+/components/ledger/ledger-utxo-flow/src/main/kotlin/net/corda/ledger/utxo/impl/token @corda/rest

applications/workers/workers-smoketest/src/smokeTest/kotlin/net/corda/applications/workers/smoketest/services/CryptoRPCSmokeTests.kt

@@ -225,9 +225,9 @@ class CryptoRPCSmokeTests {
         assertEquals(expected.requestId, actual.requestId)
         assertEquals(expected.requestingComponent, actual.requestingComponent)
         assertEquals(expected.requestTimestamp, actual.requestTimestamp)
-        assertThat(actual.responseTimestamp.epochSecond)
-            .isGreaterThanOrEqualTo(expected.requestTimestamp.epochSecond)
-            .isLessThanOrEqualTo(now.epochSecond)
+        assertThat(actual.responseTimestamp.toEpochMilli())
+            .isGreaterThanOrEqualTo(expected.requestTimestamp.toEpochMilli())
+            .isLessThanOrEqualTo(now.toEpochMilli())
         assertSoftly { softly ->
             softly.assertThat(actual.other.items.size == expected.other.items.size)
             softly.assertThat(actual.other.items.containsAll(expected.other.items))

components/configuration/configuration-write-service-impl/src/main/kotlin/net/corda/configuration/write/impl/publish/ConfigPublishServiceImpl.kt

@@ -19,6 +19,7 @@ import net.corda.v5.base.versioning.Version
 import org.osgi.service.component.annotations.Activate
 import org.osgi.service.component.annotations.Component
 import org.osgi.service.component.annotations.Reference
+import org.slf4j.LoggerFactory
 
 // This needs to be a `Lifecycle` for reconciliation, maybe not only for that. However it cannot really wait on
 // `ConfigurationReadService`, because it will be used by `ConfigWriteService` which needs to be started before
@@ -36,6 +37,10 @@ class ConfigPublishServiceImpl @Activate constructor(
     configurationValidatorFactory: ConfigurationValidatorFactory
 ) : ConfigPublishService {
 
+    companion object {
+        private val logger = LoggerFactory.getLogger(ConfigPublishServiceImpl::class.java)
+    }
+
     private val handler = ConfigPublishServiceHandler(publisherFactory, configMerger)
 
     override val lifecycleCoordinatorName = LifecycleCoordinatorName.forComponent<ConfigPublishService>()
@@ -102,6 +107,56 @@ class ConfigPublishServiceImpl @Activate constructor(
         TODO("Not yet implemented")
     }
 
+    override fun valuesMisalignedAfterDefaults(
+        recordKey: String,
+        dbRecordValue: Configuration,
+        kafkaRecordValue: Configuration
+    ): Boolean {
+        require(dbRecordValue.version == kafkaRecordValue.version)
+        require(dbRecordValue.schemaVersion.majorVersion == kafkaRecordValue.schemaVersion.majorVersion)
+        val schemaMajorVersion = dbRecordValue.schemaVersion.majorVersion
+        require(dbRecordValue.schemaVersion.minorVersion == kafkaRecordValue.schemaVersion.minorVersion)
+        val schemaMinorVersion = dbRecordValue.schemaVersion.minorVersion
+
+        val dbConfigValueWithDefaults =
+            smartConfigFactory.create(ConfigFactory.parseString(dbRecordValue.value)).run {
+                validator.validate(
+                    recordKey,
+                    Version(schemaMajorVersion, schemaMinorVersion),
+                    this,
+                    applyDefaults = true
+                )
+            }
+
+        val kafkaConfigValue =
+            smartConfigFactory.create(ConfigFactory.parseString(kafkaRecordValue.value)).run {
+                validator.validate(
+                    recordKey,
+                    Version(schemaMajorVersion, schemaMinorVersion),
+                    this,
+                    applyDefaults = false,
+                )
+            }
+
+
+        val configsAreNotEqual = dbConfigValueWithDefaults != kafkaConfigValue
+        if (configsAreNotEqual) {
+            logger.info(
+                "Configuration for key $recordKey is misaligned on Kafka after applying defaults (Kafka will be updated).\n" +
+                        "DB config value: ${
+                            dbConfigValueWithDefaults.toSafeConfig().root()
+                                .render(ConfigRenderOptions.concise().setFormatted(true))
+                        }\n" +
+                        "Kafka config value: ${
+                            kafkaConfigValue.toSafeConfig().root()
+                                .render(ConfigRenderOptions.concise().setFormatted(true))
+                        }"
+            )
+        }
+
+        return configsAreNotEqual
+    }
+
     override val isRunning get() = coordinator.isRunning
 
     override fun start() = coordinator.start()

components/crypto/crypto-service-impl/src/test/kotlin/net/corda/crypto/service/impl/bus/CryptoOpsBusProcessorTests.kt

@@ -165,9 +165,9 @@ class CryptoOpsBusProcessorTests {
         assertEquals(expected.requestId, actual.requestId)
         assertEquals(expected.requestingComponent, actual.requestingComponent)
         assertEquals(expected.requestTimestamp, actual.requestTimestamp)
-        assertThat(actual.responseTimestamp.epochSecond)
-            .isGreaterThanOrEqualTo(expected.requestTimestamp.epochSecond)
-            .isLessThanOrEqualTo(now.epochSecond)
+        assertThat(actual.responseTimestamp.toEpochMilli())
+            .isGreaterThanOrEqualTo(expected.requestTimestamp.toEpochMilli())
+            .isLessThanOrEqualTo(now.toEpochMilli())
         assertTrue(
             actual.other.items.size == expected.other.items.size &&
                     actual.other.items.containsAll(expected.other.items) &&

components/crypto/crypto-service-impl/src/test/kotlin/net/corda/crypto/service/impl/bus/HSMRegistrationBusProcessorTests.kt

@@ -72,9 +72,9 @@ class HSMRegistrationBusProcessorTests {
             assertEquals(expected.requestId, actual.requestId)
             assertEquals(expected.requestingComponent, actual.requestingComponent)
             assertEquals(expected.requestTimestamp, actual.requestTimestamp)
-            assertThat(actual.responseTimestamp.epochSecond)
-                .isGreaterThanOrEqualTo(expected.requestTimestamp.epochSecond)
-                .isLessThanOrEqualTo(now.epochSecond)
+            assertThat(actual.responseTimestamp.toEpochMilli())
+                .isGreaterThanOrEqualTo(expected.requestTimestamp.toEpochMilli())
+                .isLessThanOrEqualTo(now.toEpochMilli())
             assertTrue(
                 actual.other.items.size == expected.other.items.size &&
                         actual.other.items.containsAll(expected.other.items) &&

components/flow/flow-service/src/integrationTest/kotlin/net/corda/flow/testing/tests/ExternalEventAcceptanceTest.kt

@@ -3,17 +3,19 @@ package net.corda.flow.testing.tests
 import net.corda.data.KeyValuePairList
 import net.corda.data.flow.event.external.ExternalEventContext
 import net.corda.data.flow.event.external.ExternalEventResponseErrorType
+import net.corda.data.flow.output.FlowStates
 import net.corda.data.persistence.EntityRequest
 import net.corda.data.persistence.EntityResponse
 import net.corda.data.persistence.FindEntities
 import net.corda.flow.external.events.factory.ExternalEventFactory
 import net.corda.flow.external.events.factory.ExternalEventRecord
 import net.corda.flow.fiber.FlowIORequest
+import net.corda.flow.pipeline.exceptions.FlowProcessingExceptionTypes.FLOW_FAILED
 import net.corda.flow.state.FlowCheckpoint
+import net.corda.flow.testing.context.ALICE_FLOW_KEY_MAPPER
 import net.corda.flow.testing.context.FlowServiceTestBase
 import net.corda.flow.testing.context.flowResumedWithError
 import net.corda.schema.configuration.FlowConfig
-import net.corda.utilities.seconds
 import net.corda.v5.base.exceptions.CordaRuntimeException
 import org.junit.jupiter.api.BeforeEach
 import org.junit.jupiter.api.Test
@@ -400,6 +402,9 @@ class ExternalEventAcceptanceTest : FlowServiceTestBase() {
                 markedForDlq()
                 flowDidNotResume()
                 flowFiberCacheDoesNotContainKey(ALICE_HOLDING_IDENTITY, REQUEST_ID1)
+                scheduleFlowMapperCleanupEvents(ALICE_FLOW_KEY_MAPPER)
+                nullStateRecord()
+                flowStatus(state = FlowStates.FAILED, errorType = FLOW_FAILED, errorMessage = "message")
             }
         }
     }

components/flow/flow-service/src/integrationTest/kotlin/net/corda/flow/testing/tests/FlowKilledAcceptanceTest.kt

@@ -3,6 +3,8 @@ package net.corda.flow.testing.tests
 import net.corda.data.flow.output.FlowStates
 import net.corda.flow.application.sessions.SessionInfo
 import net.corda.flow.fiber.FlowIORequest
+import net.corda.flow.testing.context.ALICE_FLOW_KEY_MAPPER
+import net.corda.flow.testing.context.BOB_FLOW_KEY_MAPPER
 import net.corda.flow.testing.context.FlowServiceTestBase
 import net.corda.virtualnode.OperationalStatus
 import org.junit.jupiter.api.BeforeEach
@@ -41,8 +43,10 @@ class FlowKilledAcceptanceTest : FlowServiceTestBase() {
 
         then {
             expectOutputForFlow(FLOW_ID1) {
+                sessionErrorEvents()
                 nullStateRecord()
                 flowKilledStatus(flowTerminatedReason = "Flow operational status is INACTIVE")
+                scheduleFlowMapperCleanupEvents(BOB_FLOW_KEY_MAPPER)
                 flowFiberCacheDoesNotContainKey(BOB_HOLDING_IDENTITY, REQUEST_ID1)
             }
         }
@@ -94,8 +98,10 @@ class FlowKilledAcceptanceTest : FlowServiceTestBase() {
 
         then {
             expectOutputForFlow(FLOW_ID1) {
+                sessionErrorEvents(SESSION_ID_1)
                 nullStateRecord()
                 flowKilledStatus(flowTerminatedReason = "Flow operational status is INACTIVE")
+                scheduleFlowMapperCleanupEvents(SESSION_ID_1, ALICE_FLOW_KEY_MAPPER)
                 flowFiberCacheDoesNotContainKey(ALICE_HOLDING_IDENTITY, REQUEST_ID1)
             }
         }