From b0241ecba8859f638414eadb86ca6c70e29a4b4a Mon Sep 17 00:00:00 2001
From: Lajos Veres <lajos.veres@r3.com>
Date: Tue, 10 Oct 2023 10:50:08 +0100
Subject: [PATCH 1/5] CORE-17431 utxo_transaction_metadata table and related
 migrations

---
 .../migration/ledger-utxo-creation-v5.1.xml   | 53 +++++++++++++++++++
 1 file changed, 53 insertions(+)

diff --git a/data/db-schema/src/main/resources/net/corda/db/schema/vnode-vault/migration/ledger-utxo-creation-v5.1.xml b/data/db-schema/src/main/resources/net/corda/db/schema/vnode-vault/migration/ledger-utxo-creation-v5.1.xml
index b958807b00..9aa5e97485 100644
--- a/data/db-schema/src/main/resources/net/corda/db/schema/vnode-vault/migration/ledger-utxo-creation-v5.1.xml
+++ b/data/db-schema/src/main/resources/net/corda/db/schema/vnode-vault/migration/ledger-utxo-creation-v5.1.xml
@@ -121,6 +121,59 @@
 
         <!-- Drop the utxo_cpk table -->
         <dropTable tableName="utxo_cpk"/>
+
+        <createTable tableName="utxo_transaction_metadata">
+            <column name="hash" type="VARCHAR(160)">
+                <constraints nullable="false" primaryKey="true"/>
+            </column>
+            <column name="data" type="${json.column.type}">
+                <constraints nullable="false"/>
+            </column>
+            <column name="group_parameters_hash" type="VARCHAR(160)">
+                <constraints nullable="false"/>
+            </column>
+            <column name="cpi_file_checksum" type="VARCHAR(160)">
+                <constraints nullable="false"/>
+            </column>
+        </createTable>
+
+        <!-- Add utxo_transaction.metadata_hash nullable to let migrate data later -->
+        <addColumn tableName="utxo_transaction">
+            <column name="metadata_hash" type="VARCHAR(160)">
+                <constraints nullable="true" foreignKeyName="fk_utxo_transaction_metadata" references="utxo_transaction_metadata(hash)"/>
+            </column>
+        </addColumn>
+
+        <!-- Populate utxo_transaction_metadata. Postgresql only since, no other dbs were supported before 5.1. So only Postgres can have data which needs to be migrated. -->
+        <!-- ^^ is this correct?? -->
+        <sql dbms="postgresql">
+            INSERT INTO utxo_transaction_metadata
+            SELECT
+                hash,
+                convert_from(data, 'UTF-8')::jsonb AS data,
+                convert_from(data, 'UTF-8')::jsonb->'membershipGroupParametersHash' AS group_parameters_hash,
+                convert_from(data, 'UTF-8')::jsonb->'cpiMetadata'->'fileChecksum' AS cpi_file_checksum
+            FROM
+                utxo_transaction_component
+            WHERE
+                group_idx=0 AND
+                leaf_idx=0;
+        </sql>
+
+        <!-- Migrate utxo_transaction.metadata_hash -->
+        <update tableName="utxo_transaction">
+            <column name="metadata_hash"
+                    valueComputed="(select hash from utxo_transaction_component c where c.transaction_id=id and group_idx=0 and leaf_idx=0)"/>
+        </update>
+
+        <!-- Change utxo_transaction.metadata_hash to not nullable -->
+        <addNotNullConstraint tableName="utxo_transaction" columnName="metadata_hash" columnDataType="VARCHAR(160)"/>
+
+        <!-- Old metadata records are not needed anymore. -->
+        <delete tableName="utxo_transaction_component">
+            <where>group_idx=0 AND leaf_idx=0</where>
+        </delete>
+
     </changeSet>
 
 </databaseChangeLog>
\ No newline at end of file

From 11c85e51a29a8dc23d35fd70866d4652a1bd5b4e Mon Sep 17 00:00:00 2001
From: Lajos Veres <lajos.veres@r3.com>
Date: Wed, 11 Oct 2023 15:10:34 +0100
Subject: [PATCH 2/5] CORE-17431 Store data in both BYTEA and JSONB

---
 .../vnode-vault/migration/ledger-utxo-creation-v5.1.xml   | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/data/db-schema/src/main/resources/net/corda/db/schema/vnode-vault/migration/ledger-utxo-creation-v5.1.xml b/data/db-schema/src/main/resources/net/corda/db/schema/vnode-vault/migration/ledger-utxo-creation-v5.1.xml
index 9aa5e97485..775063323d 100644
--- a/data/db-schema/src/main/resources/net/corda/db/schema/vnode-vault/migration/ledger-utxo-creation-v5.1.xml
+++ b/data/db-schema/src/main/resources/net/corda/db/schema/vnode-vault/migration/ledger-utxo-creation-v5.1.xml
@@ -126,7 +126,10 @@
             <column name="hash" type="VARCHAR(160)">
                 <constraints nullable="false" primaryKey="true"/>
             </column>
-            <column name="data" type="${json.column.type}">
+            <column name="canonical_data" type="VARBINARY(1048576)">
+                <constraints nullable="false"/>
+            </column>
+            <column name="json_data" type="${json.column.type}">
                 <constraints nullable="false"/>
             </column>
             <column name="group_parameters_hash" type="VARCHAR(160)">
@@ -150,7 +153,8 @@
             INSERT INTO utxo_transaction_metadata
             SELECT
                 hash,
-                convert_from(data, 'UTF-8')::jsonb AS data,
+                data as canonical_data,
+                convert_from(data, 'UTF-8')::jsonb AS json_data,
                 convert_from(data, 'UTF-8')::jsonb->'membershipGroupParametersHash' AS group_parameters_hash,
                 convert_from(data, 'UTF-8')::jsonb->'cpiMetadata'->'fileChecksum' AS cpi_file_checksum
             FROM

From 7b594fc69aefad02e03673400a636884e252d668 Mon Sep 17 00:00:00 2001
From: Lajos Veres <lajos.veres@r3.com>
Date: Thu, 12 Oct 2023 10:13:11 +0100
Subject: [PATCH 3/5] CORE-17431 Remove JSONB since the BYTEA can be
 indexed/converted if needed

---
 .../vnode-vault/migration/ledger-utxo-creation-v5.1.xml       | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/data/db-schema/src/main/resources/net/corda/db/schema/vnode-vault/migration/ledger-utxo-creation-v5.1.xml b/data/db-schema/src/main/resources/net/corda/db/schema/vnode-vault/migration/ledger-utxo-creation-v5.1.xml
index 775063323d..7dea5ca4eb 100644
--- a/data/db-schema/src/main/resources/net/corda/db/schema/vnode-vault/migration/ledger-utxo-creation-v5.1.xml
+++ b/data/db-schema/src/main/resources/net/corda/db/schema/vnode-vault/migration/ledger-utxo-creation-v5.1.xml
@@ -129,9 +129,6 @@
             <column name="canonical_data" type="VARBINARY(1048576)">
                 <constraints nullable="false"/>
             </column>
-            <column name="json_data" type="${json.column.type}">
-                <constraints nullable="false"/>
-            </column>
             <column name="group_parameters_hash" type="VARCHAR(160)">
                 <constraints nullable="false"/>
             </column>
@@ -154,7 +151,6 @@
             SELECT
                 hash,
                 data as canonical_data,
-                convert_from(data, 'UTF-8')::jsonb AS json_data,
                 convert_from(data, 'UTF-8')::jsonb->'membershipGroupParametersHash' AS group_parameters_hash,
                 convert_from(data, 'UTF-8')::jsonb->'cpiMetadata'->'fileChecksum' AS cpi_file_checksum
             FROM

From 04fa829b0ebe70dfa9286ba8c23c4278fbfe83e8 Mon Sep 17 00:00:00 2001
From: Lajos Veres <lajos.veres@r3.com>
Date: Thu, 12 Oct 2023 10:19:06 +0100
Subject: [PATCH 4/5] CORE-17431 Bump API

---
 gradle.properties | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/gradle.properties b/gradle.properties
index 73c15af796..37dfbd0648 100644
--- a/gradle.properties
+++ b/gradle.properties
@@ -9,7 +9,7 @@ cordaProductVersion = 5.1.0
 # NOTE: update this each time this module contains a breaking change
 ## NOTE: currently this is a top level revision, so all API versions will line up, but this could be moved to
 ##   a per module property in which case module versions can change independently.
-cordaApiRevision = 32
+cordaApiRevision = 33
 
 # Main
 kotlinVersion = 1.8.21

From 0f8baa40a33c095e9205e8e8327469d5b4f58ddc Mon Sep 17 00:00:00 2001
From: Lajos Veres <lajos.veres@r3.com>
Date: Thu, 12 Oct 2023 16:21:22 +0100
Subject: [PATCH 5/5] CORE-17431 missing distinct

---
 .../schema/vnode-vault/migration/ledger-utxo-creation-v5.1.xml  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/data/db-schema/src/main/resources/net/corda/db/schema/vnode-vault/migration/ledger-utxo-creation-v5.1.xml b/data/db-schema/src/main/resources/net/corda/db/schema/vnode-vault/migration/ledger-utxo-creation-v5.1.xml
index 7dea5ca4eb..156636ce8d 100644
--- a/data/db-schema/src/main/resources/net/corda/db/schema/vnode-vault/migration/ledger-utxo-creation-v5.1.xml
+++ b/data/db-schema/src/main/resources/net/corda/db/schema/vnode-vault/migration/ledger-utxo-creation-v5.1.xml
@@ -148,7 +148,7 @@
         <!-- ^^ is this correct?? -->
         <sql dbms="postgresql">
             INSERT INTO utxo_transaction_metadata
-            SELECT
+            SELECT DISTINCT
                 hash,
                 data as canonical_data,
                 convert_from(data, 'UTF-8')::jsonb->'membershipGroupParametersHash' AS group_parameters_hash,