CORE-17431 utxo transaction metadata (#4845)

CORE-17431 Store metadata in utxo_transaction_metadata since it is relatively large but only changes occasionally.

components/ledger/ledger-persistence/src/integrationTest/kotlin/net/corda/ledger/persistence/utxo/tests/UtxoPersistenceServiceImplTest.kt

@@ -406,7 +406,6 @@ class UtxoPersistenceServiceImplTest {
         // Verify persisted data
         entityManagerFactory.transaction { em ->
             val dbTransaction = em.find(entityFactory.utxoTransaction, signedTransaction.id.toString())
-
             assertThat(dbTransaction).isNotNull
             val txPrivacySalt = dbTransaction.field<ByteArray>("privacySalt")
             val txAccountId = dbTransaction.field<String>("accountId")
@@ -416,19 +415,19 @@ class UtxoPersistenceServiceImplTest {
             assertThat(txAccountId).isEqualTo(account)
             assertThat(txCreatedTs).isNotNull
 
-            val componentGroupLists = signedTransaction.wireTransaction.componentGroupLists
+            val componentGroupListsWithoutMetadata = signedTransaction.wireTransaction.componentGroupLists.drop(1)
             val txComponents = dbTransaction.field<Collection<Any>?>("components")
             assertThat(txComponents).isNotNull
-                .hasSameSizeAs(componentGroupLists.flatten().filter { it.isNotEmpty() })
+                .hasSameSizeAs(componentGroupListsWithoutMetadata.flatten().filter { it.isNotEmpty() })
             txComponents!!
                 .sortedWith(compareBy<Any> { it.field<Int>("groupIndex") }.thenBy { it.field<Int>("leafIndex") })
                 .groupBy { it.field<Int>("groupIndex") }.values
-                .zip(componentGroupLists)
+                .zip(componentGroupListsWithoutMetadata)
                 .forEachIndexed { groupIndex, (dbComponentGroup, componentGroup) ->
                     assertThat(dbComponentGroup).hasSameSizeAs(componentGroup)
                     dbComponentGroup.zip(componentGroup)
                         .forEachIndexed { leafIndex, (dbComponent, component) ->
-                            assertThat(dbComponent.field<Int>("groupIndex")).isEqualTo(groupIndex)
+                            assertThat(dbComponent.field<Int>("groupIndex")).isEqualTo(groupIndex +1 )
                             assertThat(dbComponent.field<Int>("leafIndex")).isEqualTo(leafIndex)
                             assertThat(dbComponent.field<ByteArray>("data")).isEqualTo(component)
                             assertThat(dbComponent.field<String>("hash")).isEqualTo(
@@ -437,14 +436,21 @@ class UtxoPersistenceServiceImplTest {
                         }
                 }
 
+            val dbMetadata = dbTransaction.field<Any>("metadata")
+            assertThat(dbMetadata).isNotNull
+            assertThat(dbMetadata.field<ByteArray>("canonicalData"))
+                .isEqualTo(signedTransaction.wireTransaction.componentGroupLists[0][0])
+            assertThat(dbMetadata.field<String>("groupParametersHash")).isNotNull
+            assertThat(dbMetadata.field<String>("cpiFileChecksum")).isNotNull
+
             val dbTransactionOutputs = em.createNamedQuery(
                 "UtxoVisibleTransactionOutputEntity.findByTransactionId",
                 entityFactory.utxoVisibleTransactionOutput
             )
                 .setParameter("transactionId", signedTransaction.id.toString())
                 .resultList
             assertThat(dbTransactionOutputs).isNotNull
-                .hasSameSizeAs(componentGroupLists[UtxoComponentGroup.OUTPUTS.ordinal])
+                .hasSameSizeAs(componentGroupListsWithoutMetadata[UtxoComponentGroup.OUTPUTS.ordinal-1])
             dbTransactionOutputs
                 .sortedWith(compareBy<Any> { it.field<Int>("groupIndex") }.thenBy { it.field<Int>("leafIndex") })
                 .zip(defaultVisibleTransactionOutputs)
@@ -556,25 +562,38 @@ class UtxoPersistenceServiceImplTest {
         createdTs: Instant = testClock.instant(),
         status: TransactionStatus = UNVERIFIED
     ): Any {
+        val metadataBytes = signedTransaction.wireTransaction.componentGroupLists[0][0]
+        val metadata = entityFactory.createOrFindUtxoTransactionMetadataEntity(
+            digest("SHA-256", metadataBytes).toString(),
+            metadataBytes,
+            "fakeGroupParametersHash",
+            "fakeCpiFileChecksum"
+        )
+
         return entityFactory.createUtxoTransactionEntity(
             signedTransaction.id.toString(),
             signedTransaction.wireTransaction.privacySalt.bytes,
             account,
             createdTs,
             status.value,
-            createdTs
+            createdTs,
+            metadata
         ).also { transaction ->
             transaction.field<MutableCollection<Any>>("components").addAll(
                 signedTransaction.wireTransaction.componentGroupLists.flatMapIndexed { groupIndex, componentGroup ->
                     componentGroup.mapIndexed { leafIndex: Int, component ->
-                        entityFactory.createUtxoTransactionComponentEntity(
-                            transaction,
-                            groupIndex,
-                            leafIndex,
-                            component,
-                            digest("SHA-256", component).toString()
-                        )
-                    }
+                        if (groupIndex != 0 || leafIndex != 0) {
+                            entityFactory.createUtxoTransactionComponentEntity(
+                                transaction,
+                                groupIndex,
+                                leafIndex,
+                                component,
+                                digest("SHA-256", component).toString()
+                            )
+                        } else {
+                            null
+                        }
+                    }.filterNotNull()
                 }
             )
             transaction.field<MutableCollection<Any>>("signatures").addAll(
@@ -658,6 +677,8 @@ class UtxoPersistenceServiceImplTest {
             get() = transactionContainer.id
         override val privacySalt: PrivacySalt
             get() = transactionContainer.wireTransaction.privacySalt
+        override val metadata: TransactionMetadataInternal
+            get() = transactionContainer.wireTransaction.metadata as TransactionMetadataInternal
         override val rawGroupLists: List<List<ByteArray>>
             get() = transactionContainer.wireTransaction.componentGroupLists
         override val signatures: List<DigitalSignatureAndMetadata>

components/ledger/ledger-persistence/src/integrationTest/kotlin/net/corda/ledger/persistence/utxo/tests/datamodel/UtxoEntityFactory.kt

@@ -1,12 +1,14 @@
 package net.corda.ledger.persistence.utxo.tests.datamodel
 
+import net.corda.orm.utils.transaction
 import java.time.Instant
 import javax.persistence.EntityManagerFactory
 
-class UtxoEntityFactory(entityManagerFactory: EntityManagerFactory) {
+class UtxoEntityFactory(private val entityManagerFactory: EntityManagerFactory) {
     private val entityMap = entityManagerFactory.metamodel.entities.associate { it.name to it.bindableJavaType }
 
     val utxoTransaction: Class<*> get() = classFor("UtxoTransactionEntity")
+    val utxoTransactionMetadata: Class<*> get() = classFor("UtxoTransactionMetadataEntity")
     val utxoTransactionComponent: Class<*> get() = classFor("UtxoTransactionComponentEntity")
     val utxoVisibleTransactionOutput: Class<*> get() = classFor("UtxoVisibleTransactionOutputEntity")
     val utxoTransactionSignature: Class<*> get() = classFor("UtxoTransactionSignatureEntity")
@@ -17,10 +19,40 @@ class UtxoEntityFactory(entityManagerFactory: EntityManagerFactory) {
         accountId: String,
         created: Instant,
         status: String,
-        updated: Instant
+        updated: Instant,
+        utxoTransactionMetadata: Any
     ): Any {
-        return utxoTransaction.constructors.single { it.parameterCount == 6 }.newInstance(
-            transactionId, privacySalt, accountId, created, status, updated
+        return utxoTransaction.constructors.single { it.parameterCount == 7 }.newInstance(
+            transactionId, privacySalt, accountId, created, status, updated, utxoTransactionMetadata
+        )
+    }
+
+    fun createOrFindUtxoTransactionMetadataEntity(
+        hash: String,
+        canonicalData: ByteArray,
+        groupParametersHash: String,
+        cpiFileChecksum: String,
+    ): Any {
+        return entityManagerFactory.transaction { em ->
+            em.find(utxoTransactionMetadata, hash) ?: createUtxoTransactionMetadataEntity(
+                hash,
+                canonicalData,
+                groupParametersHash,
+                cpiFileChecksum,
+            ).also {
+                em.persist(it)
+            }
+        }
+    }
+
+    fun createUtxoTransactionMetadataEntity(
+        hash: String,
+        canonicalData: ByteArray,
+        groupParametersHash: String,
+        cpiFileChecksum: String,
+    ): Any {
+        return utxoTransactionMetadata.constructors.single { it.parameterCount == 4 }.newInstance(
+            hash, canonicalData, groupParametersHash, cpiFileChecksum
         )
     }
 

components/ledger/ledger-persistence/src/main/kotlin/net/corda/ledger/persistence/utxo/UtxoRepository.kt

@@ -26,7 +26,7 @@ interface UtxoRepository {
         id: String
     ): SignedTransactionContainer?
 
-    /** Retrieves transaction component leafs */
+    /** Retrieves transaction component leafs except metadata which is stored separately */
     fun findTransactionComponentLeafs(
         entityManager: EntityManager,
         transactionId: String
@@ -76,7 +76,17 @@ interface UtxoRepository {
         privacySalt: ByteArray,
         account: String,
         timestamp: Instant,
-        status: TransactionStatus
+        status: TransactionStatus,
+        metadataHash: String
+    )
+
+    /** Persists transaction metadata (operation is idempotent) */
+    fun persistTransactionMetadata(
+        entityManager: EntityManager,
+        hash: String,
+        metadataBytes: ByteArray,
+        groupParametersHash: String,
+        cpiFileChecksum: String
     )
 
     /** Persists transaction component leaf [data] (operation is idempotent) */

components/ledger/ledger-persistence/src/main/kotlin/net/corda/ledger/persistence/utxo/UtxoTransactionReader.kt

@@ -5,6 +5,7 @@ import net.corda.v5.application.crypto.DigitalSignatureAndMetadata
 import net.corda.v5.crypto.SecureHash
 import net.corda.v5.ledger.common.transaction.CordaPackageSummary
 import net.corda.ledger.common.data.transaction.PrivacySalt
+import net.corda.ledger.common.data.transaction.TransactionMetadataInternal
 import net.corda.v5.ledger.utxo.ContractState
 import net.corda.v5.ledger.utxo.StateAndRef
 import net.corda.v5.ledger.utxo.StateRef
@@ -13,6 +14,8 @@ interface UtxoTransactionReader {
 
     val id: SecureHash
 
+    val metadata: TransactionMetadataInternal
+
     val account: String
 
     val status: TransactionStatus

components/ledger/ledger-persistence/src/main/kotlin/net/corda/ledger/persistence/utxo/impl/AbstractUtxoQueryProvider.kt

@@ -9,10 +9,13 @@ abstract class AbstractUtxoQueryProvider : UtxoQueryProvider {
         val UNVERIFIED = TransactionStatus.UNVERIFIED.value
     }
 
-    override val findTransactionPrivacySalt: String
+    override val findTransactionPrivacySaltAndMetadata: String
         get() = """
-            SELECT privacy_salt
-            FROM {h-schema}utxo_transaction
+            SELECT privacy_salt,
+            utm.canonical_data
+            FROM {h-schema}utxo_transaction AS ut
+            JOIN {h-schema}utxo_transaction_metadata AS utm
+                ON ut.metadata_hash = utm.hash
             WHERE id = :transactionId"""
             .trimIndent()
 

components/ledger/ledger-persistence/src/main/kotlin/net/corda/ledger/persistence/utxo/impl/PostgresUtxoQueryProvider.kt

@@ -19,13 +19,20 @@ class PostgresUtxoQueryProvider @Activate constructor(
 
     override val persistTransaction: String
         get() = """
-            INSERT INTO {h-schema}utxo_transaction(id, privacy_salt, account_id, created, status, updated)
-                VALUES (:id, :privacySalt, :accountId, :createdAt, :status, :updatedAt)
+            INSERT INTO {h-schema}utxo_transaction(id, privacy_salt, account_id, created, status, updated, metadata_hash)
+                VALUES (:id, :privacySalt, :accountId, :createdAt, :status, :updatedAt, :metadataHash)
             ON CONFLICT(id) DO
                 UPDATE SET status = EXCLUDED.status, updated = EXCLUDED.updated
             WHERE utxo_transaction.status = EXCLUDED.status OR utxo_transaction.status = '$UNVERIFIED'"""
             .trimIndent()
 
+    override val persistTransactionMetadata: String
+        get() = """
+            INSERT INTO {h-schema}utxo_transaction_metadata(hash, canonical_data, group_parameters_hash, cpi_file_checksum)
+                VALUES (:hash, :canonicalData, :groupParametersHash, :cpiFileChecksum)
+            ON CONFLICT DO NOTHING"""
+            .trimIndent()
+
     override val persistTransactionComponentLeaf: String
         get() = """
             INSERT INTO {h-schema}utxo_transaction_component(transaction_id, group_idx, leaf_idx, data, hash)

components/ledger/ledger-persistence/src/main/kotlin/net/corda/ledger/persistence/utxo/impl/UtxoPersistenceServiceImpl.kt

@@ -145,14 +145,27 @@ class UtxoPersistenceServiceImpl(
         val nowUtc = utcClock.instant()
         val transactionIdString = transaction.id.toString()
 
+        val metadataBytes = transaction.rawGroupLists[0][0]
+        val metadataHash = sandboxDigestService.hash(metadataBytes, DigestAlgorithmName.SHA2_256).toString()
+
+        val metadata = transaction.metadata
+        repository.persistTransactionMetadata(
+            em,
+            metadataHash,
+            metadataBytes,
+            requireNotNull(metadata.getMembershipGroupParametersHash()) { "Metadata without membership group parameters hash" },
+            requireNotNull(metadata.getCpiMetadata()) { "Metadata without CPI metadata" }.fileChecksum
+        )
+
         // Insert the Transaction
         repository.persistTransaction(
             em,
             transactionIdString,
             transaction.privacySalt.bytes,
             transaction.account,
             nowUtc,
-            transaction.status
+            transaction.status,
+            metadataHash
         )
 
         // Insert the Transactions components

components/ledger/ledger-persistence/src/main/kotlin/net/corda/ledger/persistence/utxo/impl/UtxoQueryProvider.kt

@@ -11,9 +11,9 @@ package net.corda.ledger.persistence.utxo.impl
  */
 interface UtxoQueryProvider {
     /**
-     * @property findTransactionPrivacySalt SQL text for [UtxoRepositoryImpl.findTransactionPrivacySalt].
+     * @property findTransactionPrivacySaltAndMetadata SQL text for [UtxoRepositoryImpl.findTransactionPrivacySaltAndMetadata].
      */
-    val findTransactionPrivacySalt: String
+    val findTransactionPrivacySaltAndMetadata: String
 
     /**
      * @property findTransactionComponentLeafs SQL text for [UtxoRepositoryImpl.findTransactionComponentLeafs].
@@ -61,6 +61,11 @@ interface UtxoQueryProvider {
      */
     val persistTransaction: String
 
+    /**
+     * @property persistTransactionMetadata SQL text for [UtxoRepositoryImpl.persistTransactionMetadata].
+     */
+    val persistTransactionMetadata: String
+
     /**
      * @property persistTransactionComponentLeaf SQL text for [UtxoRepositoryImpl.persistTransactionComponentLeaf].
      */

components/ledger/ledger-persistence/src/main/kotlin/net/corda/ledger/persistence/utxo/impl/UtxoRepositoryImpl.kt

@@ -60,9 +60,9 @@ class UtxoRepositoryImpl @Activate constructor(
         entityManager: EntityManager,
         id: String
     ): SignedTransactionContainer? {
-        val privacySalt = findTransactionPrivacySalt(entityManager, id) ?: return null
+        val (privacySalt, metadataBytes) = findTransactionPrivacySaltAndMetadata(entityManager, id) ?: return null
         val wireTransaction = wireTransactionFactory.create(
-            findTransactionComponentLeafs(entityManager, id),
+            mapOf(0 to listOf(metadataBytes)) + findTransactionComponentLeafs(entityManager, id),
             privacySalt
         )
         return SignedTransactionContainer(
@@ -87,14 +87,14 @@ class UtxoRepositoryImpl @Activate constructor(
             .associate { r -> parseSecureHash(r.get(0) as String) to r.get(1) as String }
     }
 
-    private fun findTransactionPrivacySalt(
+    private fun findTransactionPrivacySaltAndMetadata(
         entityManager: EntityManager,
         transactionId: String
-    ): PrivacySaltImpl? {
-        return entityManager.createNativeQuery(queryProvider.findTransactionPrivacySalt, Tuple::class.java)
+    ): Pair<PrivacySaltImpl, ByteArray>? {
+        return entityManager.createNativeQuery(queryProvider.findTransactionPrivacySaltAndMetadata, Tuple::class.java)
             .setParameter("transactionId", transactionId)
             .resultListAsTuples()
-            .map { r -> PrivacySaltImpl(r.get(0) as ByteArray) }
+            .map { r -> Pair(PrivacySaltImpl(r.get(0) as ByteArray), r.get(1) as ByteArray) }
             .firstOrNull()
     }
 
@@ -183,7 +183,8 @@ class UtxoRepositoryImpl @Activate constructor(
         privacySalt: ByteArray,
         account: String,
         timestamp: Instant,
-        status: TransactionStatus
+        status: TransactionStatus,
+        metadataHash: String
     ) {
         entityManager.createNativeQuery(queryProvider.persistTransaction)
             .setParameter("id", id)
@@ -192,10 +193,27 @@ class UtxoRepositoryImpl @Activate constructor(
             .setParameter("createdAt", timestamp)
             .setParameter("status", status.value)
             .setParameter("updatedAt", timestamp)
+            .setParameter("metadataHash", metadataHash)
             .executeUpdate()
             .logResult("transaction [$id]")
     }
 
+    override fun persistTransactionMetadata(
+        entityManager: EntityManager,
+        hash: String,
+        metadataBytes: ByteArray,
+        groupParametersHash: String,
+        cpiFileChecksum: String
+    ){
+        entityManager.createNativeQuery(queryProvider.persistTransactionMetadata)
+            .setParameter("hash", hash)
+            .setParameter("canonicalData", metadataBytes)
+            .setParameter("groupParametersHash", groupParametersHash)
+            .setParameter("cpiFileChecksum", cpiFileChecksum)
+            .executeUpdate()
+            .logResult("transaction metadata [$hash]")
+    }
+
     override fun persistTransactionComponentLeaf(
         entityManager: EntityManager,
         transactionId: String,
@@ -204,6 +222,10 @@ class UtxoRepositoryImpl @Activate constructor(
         data: ByteArray,
         hash: String
     ) {
+        // Metadata is not stored with the other components. See persistTransactionMetadata().
+        if (groupIndex == 0 && leafIndex == 0) {
+            return
+        }
         entityManager.createNativeQuery(queryProvider.persistTransactionComponentLeaf)
             .setParameter("transactionId", transactionId)
             .setParameter("groupIndex", groupIndex)

components/ledger/ledger-persistence/src/main/kotlin/net/corda/ledger/persistence/utxo/impl/UtxoTransactionReaderImpl.kt

@@ -75,6 +75,9 @@ class UtxoTransactionReaderImpl(
     override val privacySalt: PrivacySalt
         get() = signedTransaction.wireTransaction.privacySalt
 
+    override val metadata: TransactionMetadataInternal
+        get() = signedTransaction.wireTransaction.metadata as TransactionMetadataInternal
+
     override val rawGroupLists: List<List<ByteArray>>
         get() = signedTransaction.wireTransaction.componentGroupLists