Merge pull request #655 from oasisprotocol/andrew7234/consensus-tx-body

Andrew7234/consensus tx body

analyzer/consensus/consensus.go

@@ -8,23 +8,39 @@ import (
 	"fmt"
 	"math"
 	"os"
+	"reflect"
 	"strings"
 
 	coreCommon "github.com/oasisprotocol/oasis-core/go/common"
 	"github.com/oasisprotocol/oasis-core/go/common/cbor"
 	"github.com/oasisprotocol/oasis-core/go/common/crypto/signature"
+	"github.com/oasisprotocol/oasis-core/go/common/entity"
 	sdkConfig "github.com/oasisprotocol/oasis-sdk/client-sdk/go/config"
 
-	"github.com/oasisprotocol/nexus/common"
+	beaconCobalt "github.com/oasisprotocol/nexus/coreapi/v21.1.1/beacon/api"
+	governanceCobalt "github.com/oasisprotocol/nexus/coreapi/v21.1.1/governance/api"
+	registryCobalt "github.com/oasisprotocol/nexus/coreapi/v21.1.1/registry/api"
+	roothashCobalt "github.com/oasisprotocol/nexus/coreapi/v21.1.1/roothash/api"
+
+	beacon "github.com/oasisprotocol/nexus/coreapi/v22.2.11/beacon/api"
+	"github.com/oasisprotocol/nexus/coreapi/v22.2.11/common/node"
 	"github.com/oasisprotocol/nexus/coreapi/v22.2.11/consensus/api/transaction"
 	genesis "github.com/oasisprotocol/nexus/coreapi/v22.2.11/genesis/api"
+	governance "github.com/oasisprotocol/nexus/coreapi/v22.2.11/governance/api"
+	keymanager "github.com/oasisprotocol/nexus/coreapi/v22.2.11/keymanager/api"
+	registry "github.com/oasisprotocol/nexus/coreapi/v22.2.11/registry/api"
+	roothash "github.com/oasisprotocol/nexus/coreapi/v22.2.11/roothash/api"
 	staking "github.com/oasisprotocol/nexus/coreapi/v22.2.11/staking/api"
 
+	consensusEden "github.com/oasisprotocol/nexus/coreapi/v23.0/consensus/api"
+	keymanagerEden "github.com/oasisprotocol/nexus/coreapi/v23.0/keymanager/api"
+
 	"github.com/oasisprotocol/nexus/analyzer"
 	"github.com/oasisprotocol/nexus/analyzer/block"
 	"github.com/oasisprotocol/nexus/analyzer/queries"
 	"github.com/oasisprotocol/nexus/analyzer/util"
 	apiTypes "github.com/oasisprotocol/nexus/api/v1/types"
+	"github.com/oasisprotocol/nexus/common"
 	"github.com/oasisprotocol/nexus/config"
 	"github.com/oasisprotocol/nexus/log"
 	"github.com/oasisprotocol/nexus/metrics"
@@ -36,6 +52,46 @@ const (
 	consensusAnalyzerName = "consensus"
 )
 
+// The Cobalt (v21.1.1) version of oasis-core defines some transaction structs
+// that are incompatible with the structs used in Damask onwards. We track
+// those structs here and use them as a backup if the initial unmarshalling fails.
+var bodyTypeForTxMethodCobalt = map[string]interface{}{
+	"governance.SubmitProposal": governanceCobalt.ProposalContent{},
+	"registry.DeregisterEntity": nil,
+	"registry.RegisterRuntime":  registryCobalt.Runtime{},
+	"roothash.ExecutorCommit":   roothashCobalt.ExecutorCommit{},
+	"roothash.Evidence":         roothashCobalt.Evidence{},
+}
+
+var bodyTypeForTxMethod = map[string]interface{}{
+	"beacon.PVSSCommit":                 beaconCobalt.PVSSCommit{},
+	"beacon.PVSSReveal":                 beaconCobalt.PVSSReveal{},
+	"beacon.VRFProve":                   beacon.VRFProve{},
+	"consensus.Meta":                    consensusEden.BlockMetadata{},
+	"governance.SubmitProposal":         governance.ProposalContent{},
+	"governance.CastVote":               governance.ProposalVote{},
+	"keymanager.PublishMasterSecret":    keymanagerEden.SignedEncryptedMasterSecret{},
+	"keymanager.PublishEphemeralSecret": keymanagerEden.SignedEncryptedEphemeralSecret{},
+	"keymanager.UpdatePolicy":           keymanager.SignedPolicySGX{},
+	"registry.RegisterEntity":           entity.SignedEntity{},
+	"registry.DeregisterEntity":         registry.DeregisterEntity{},
+	"registry.RegisterNode":             node.MultiSignedNode{},
+	"registry.UnfreezeNode":             registry.UnfreezeNode{},
+	"registry.RegisterRuntime":          registry.Runtime{},
+	"registry.ProveFreshness":           registry.Runtime{},
+	"roothash.ExecutorCommit":           roothash.ExecutorCommit{},
+	"roothash.ExecutorProposerTimeout":  roothash.ExecutorProposerTimeoutRequest{},
+	"roothash.Evidence":                 roothash.Evidence{},
+	"roothash.SubmitMsg":                roothash.SubmitMsg{},
+	"staking.Transfer":                  staking.Transfer{},
+	"staking.Burn":                      staking.Burn{},
+	"staking.AddEscrow":                 staking.Escrow{},
+	"staking.ReclaimEscrow":             staking.ReclaimEscrow{},
+	"staking.AmendCommissionSchedule":   staking.AmendCommissionSchedule{},
+	"staking.Allow":                     staking.Allow{},
+	"staking.Withdraw":                  staking.Withdraw{},
+}
+
 type EventType = apiTypes.ConsensusEventType // alias for brevity
 
 type parsedEvent struct {
@@ -376,6 +432,31 @@ func (m *processor) queueEpochInserts(batch *storage.QueryBatch, data *consensus
 	return nil
 }
 
+// Adapted from https://github.com/oasisprotocol/oasis-core/blob/master/go/consensus/api/transaction/transaction.go#L58
+func unpackTxBody(t *transaction.Transaction) (interface{}, error) {
+	bodyType, ok := bodyTypeForTxMethod[string(t.Method)]
+	if !ok {
+		return nil, fmt.Errorf("unknown tx method: %s", t.Method)
+	}
+
+	// Deserialize into correct type.
+	v := reflect.New(reflect.TypeOf(bodyType)).Interface()
+	if err := cbor.Unmarshal(t.Body, v); err != nil {
+		// This may be an older tx from Cobalt; attempt to unmarshal into
+		// the appropriate Cobalt tx type.
+		bodyType, ok = bodyTypeForTxMethodCobalt[string(t.Method)]
+		if !ok {
+			return nil, fmt.Errorf("unable to cbor-decode consensus tx body: %w, body: %x", err, t.Body)
+		}
+		v = reflect.New(reflect.TypeOf(bodyType)).Interface()
+		if err2 := cbor.Unmarshal(t.Body, v); err2 != nil {
+			return nil, fmt.Errorf("unable to cbor-decode consensus tx body into Cobalt type (%w) or newer type (%w), body: %x", err, err2, t.Body)
+		}
+	}
+
+	return v, nil
+}
+
 func (m *processor) queueTransactionInserts(batch *storage.QueryBatch, data *consensusBlockData) error {
 	for i, txr := range data.TransactionsWithResults {
 		signedTx := txr.Transaction
@@ -395,10 +476,9 @@ func (m *processor) queueTransactionInserts(batch *storage.QueryBatch, data *con
 			signedTx.Signature.PublicKey,
 		).String()
 
-		bodyBytes, err := tx.Body.MarshalCBOR()
+		body, err := unpackTxBody(tx)
 		if err != nil {
-			m.logger.Warn("failed to marshal transaction body", "err", err, "tx_hash", signedTx.Hash().Hex(), "height", data.Height)
-			bodyBytes = []byte{}
+			m.logger.Warn("failed to unpack tx body", "err", err, "tx_hash", signedTx.Hash().Hex(), "height", data.Height)
 		}
 		var module *string
 		if len(result.Error.Module) > 0 {
@@ -425,7 +505,7 @@ func (m *processor) queueTransactionInserts(batch *storage.QueryBatch, data *con
 			fmt.Sprintf("%d", fee.Gas),
 			tx.Method,
 			sender,
-			bodyBytes,
+			body,
 			module,
 			result.Error.Code,
 			message,

analyzer/consensus/consensus_test.go

@@ -0,0 +1,101 @@
+package consensus
+
+import (
+	"encoding/hex"
+	"encoding/json"
+	"reflect"
+	"testing"
+
+	"github.com/fxamacker/cbor/v2"
+	"github.com/stretchr/testify/require"
+
+	roothashCobalt "github.com/oasisprotocol/nexus/coreapi/v21.1.1/roothash/api"
+	"github.com/oasisprotocol/nexus/coreapi/v22.2.11/consensus/api/transaction"
+	roothashDamask "github.com/oasisprotocol/nexus/coreapi/v22.2.11/roothash/api"
+)
+
+func TestCobaltTx(t *testing.T) {
+	// Setup Cobalt-only roothash.ExecutorCommit transaction
+	bodyBytes, err := hex.DecodeString("a26269645820000000000000000000000000000000000000000000000000000000000000ff0367636f6d6d69747381a2697369676e6174757265a2697369676e61747572655840c6974b7385427e41fea391e67e7795605cd1b9532bbe08acf140ea4cd9548fe9b51c7b7f5bd3e4503fe3c99d1fd91d3c54837f1640382dd86b774778dc8d69096a7075626c69635f6b6579582014de3f20514009d1aa9e9fa144381a5342159d19c82cd53ded21723d7044a64d73756e747275737465645f7261775f76616c75655903d8a666686561646572a565726f756e6419ab3567696f5f726f6f745820f25e088a5f40ee6d26c527bcc60781fd7f655b829e302dc7f68ca3a14f853cc76a73746174655f726f6f7458205e6ae2fc0992eb20bca5457e57bf5e9a3cc414cfc6b40b4f92664dcbee3c95096d6d657373616765735f686173685820c672b8d1ef56ed28ab87c3622c5114069bdd3ad7b8f9737498d0c01ecef0967a6d70726576696f75735f686173685820806469c247e30fb9d2bf7c54a238a195495d389886915803be3dbf64b99b09826772616b5f7369675840000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000006a696e7075745f726f6f74582099265585e51182de59a17ea7333d31c28580652ab7c8af8f69ba86c6955b916d6d74786e5f73636865645f736967a2697369676e6174757265584018494171c4d91da9ba1463b3620500d2eb87690a672aea8dd8d9d156a944fc995a96ed8152d727db3358bad284142642f311480309893c726b7c781db52dc90c6a7075626c69635f6b6579582014de3f20514009d1aa9e9fa144381a5342159d19c82cd53ded21723d7044a64d72696e7075745f73746f726167655f7369677382a2697369676e61747572655840ff6183073ee6811a56cd4633ae936e2acec02696710fa5b145b09834c82bb95856960905da53819d6b035b5dec3e407dd3afad73d970bb93217c1d72a68ff8086a7075626c69635f6b65795820029270158d7ad16ef25a7f26bacfd65e25e4f6ca589b6fdaaf669fb40792312fa2697369676e6174757265584076b5d3152ad353d6c1762dcc524ebfde532fa0423bbf75d88a2bd7f7620aae4f33cd067e37f905b55f29f0f60408f542ca3e831e89c8d493efbd1993feb0d2026a7075626c69635f6b657958206dbcf2330a17e0fc52126cca2c2173b5b6530a17a2006a8339de19fe4a47f75c7273746f726167655f7369676e61747572657382a2697369676e61747572655840ab27361e360ca3911ffcc53df832aaf22833cf05200afbebdb2061adedd5afdb562fa0798492743c4ac39d93486646d9343412daece2c071857f42793f7c69016a7075626c69635f6b65795820029270158d7ad16ef25a7f26bacfd65e25e4f6ca589b6fdaaf669fb40792312fa2697369676e6174757265584020e853374fe0065a9161b43bb39f8ee6f0114d1787b6e8f6d104fde3feab0d530944d5805b4396f6353a799bd1f437f8ab44d9c85f04bef1c3bb9e17262579046a7075626c69635f6b657958206dbcf2330a17e0fc52126cca2c2173b5b6530a17a2006a8339de19fe4a47f75c")
+	require.Nil(t, err)
+	var body cbor.RawMessage
+	err = body.UnmarshalCBOR(bodyBytes)
+	require.Nil(t, err)
+	// Round: 3028327, tx_index: 2
+	tx := transaction.Transaction{
+		Nonce:  0,
+		Method: "roothash.ExecutorCommit",
+		Body:   body,
+	}
+	expected := []byte(`{"id":"000000000000000000000000000000000000000000000000000000000000ff03","commits":[{"untrusted_raw_value":"pmZoZWFkZXKlZXJvdW5kGas1Z2lvX3Jvb3RYIPJeCIpfQO5tJsUnvMYHgf1/ZVuCnjAtx/aMo6FPhTzHanN0YXRlX3Jvb3RYIF5q4vwJkusgvKVFfle/Xpo8xBTPxrQLT5JmTcvuPJUJbW1lc3NhZ2VzX2hhc2hYIMZyuNHvVu0oq4fDYixRFAab3TrXuPlzdJjQwB7O8JZ6bXByZXZpb3VzX2hhc2hYIIBkacJH4w+50r98VKI4oZVJXTiYhpFYA749v2S5mwmCZ3Jha19zaWdYQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABqaW5wdXRfcm9vdFggmSZVheURgt5ZoX6nMz0xwoWAZSq3yK+PabqGxpVbkW1tdHhuX3NjaGVkX3NpZ6Jpc2lnbmF0dXJlWEAYSUFxxNkdqboUY7NiBQDS64dpCmcq6o3Y2dFWqUT8mVqW7YFS1yfbM1i60oQUJkLzEUgDCYk8cmt8eB21LckManB1YmxpY19rZXlYIBTePyBRQAnRqp6foUQ4GlNCFZ0ZyCzVPe0hcj1wRKZNcmlucHV0X3N0b3JhZ2Vfc2lnc4KiaXNpZ25hdHVyZVhA/2GDBz7mgRpWzUYzrpNuKs7AJpZxD6WxRbCYNMgruVhWlgkF2lOBnWsDW13sPkB906+tc9lwu5MhfB1ypo/4CGpwdWJsaWNfa2V5WCACknAVjXrRbvJafya6z9ZeJeT2ylibb9qvZp+0B5IxL6Jpc2lnbmF0dXJlWEB2tdMVKtNT1sF2LcxSTr/eUy+gQju/ddiKK9f3YgquTzPNBn43+QW1Xynw9gQI9ULKPoMeicjUk++9GZP+sNICanB1YmxpY19rZXlYIG288jMKF+D8UhJsyiwhc7W2UwoXogBqgzneGf5KR/dccnN0b3JhZ2Vfc2lnbmF0dXJlc4KiaXNpZ25hdHVyZVhAqyc2HjYMo5Ef/MU9+DKq8igzzwUgCvvr2yBhre3Vr9tWL6B5hJJ0PErDnZNIZkbZNDQS2uziwHGFf0J5P3xpAWpwdWJsaWNfa2V5WCACknAVjXrRbvJafya6z9ZeJeT2ylibb9qvZp+0B5IxL6Jpc2lnbmF0dXJlWEAg6FM3T+AGWpFhtDuzn47m8BFNF4e26PbRBP3j/qsNUwlE1YBbQ5b2NTp5m9H0N/irRNnIXwS+8cO7nhcmJXkEanB1YmxpY19rZXlYIG288jMKF+D8UhJsyiwhc7W2UwoXogBqgzneGf5KR/dc","signature":{"public_key":"FN4/IFFACdGqnp+hRDgaU0IVnRnILNU97SFyPXBEpk0=","signature":"xpdLc4VCfkH+o5HmfneVYFzRuVMrvgis8UDqTNlUj+m1HHt/W9PkUD/jyZ0f2R08VIN/FkA4Ldhrd0d43I1pCQ=="}}]}`)
+
+	// Parse and validate
+	parsed, err := unpackTxBody(&tx)
+	require.Nil(t, err)
+	require.Equal(t, reflect.TypeOf(&roothashCobalt.ExecutorCommit{}).String(), reflect.TypeOf(parsed).String())
+	parsedJson, err := json.Marshal(parsed)
+	require.Nil(t, err)
+	require.Equal(t, expected, parsedJson)
+}
+
+func TestDamaskTx(t *testing.T) {
+	// Setup Damask-only roothash.ExecutorCommit transaction
+	bodyBytes, err := hex.DecodeString("a26269645820000000000000000000000000000000000000000000000000e199119c992377cb67636f6d6d69747381a3637369675840bd1dec77999d5c30b6ed748467c998fd837db67e6252daf9a7b76c56f8df41ae6feffbc12f9285d595b40d6c83e5e979075da4b272da21bf2e023c490cb75f0666686561646572a365726f756e641952d3676661696c757265016d70726576696f75735f68617368582067e13bd5666486c4a050a9caeb8d17fc9728110581426398312b25ce2b3abe20676e6f64655f696458207e7dcb588bccb5aaf2e77029935b2b0c1fc8f52c636e961ff53b766c7dd091fe")
+	require.Nil(t, err)
+	var body cbor.RawMessage
+	err = body.UnmarshalCBOR(bodyBytes)
+	require.Nil(t, err)
+	// Round: 11453160, tx_index: 1
+	tx := transaction.Transaction{
+		Nonce:  0,
+		Method: "roothash.ExecutorCommit",
+		Body:   body,
+	}
+	expected := []byte(`{"id":"000000000000000000000000000000000000000000000000e199119c992377cb","commits":[{"node_id":"fn3LWIvMtary53Apk1srDB/I9SxjbpYf9Tt2bH3Qkf4=","header":{"round":21203,"previous_hash":"67e13bd5666486c4a050a9caeb8d17fc9728110581426398312b25ce2b3abe20","failure":1},"sig":"vR3sd5mdXDC27XSEZ8mY/YN9tn5iUtr5p7dsVvjfQa5v7/vBL5KF1ZW0DWyD5el5B12ksnLaIb8uAjxJDLdfBg=="}]}`)
+
+	// Parse and validate
+	parsed, err := unpackTxBody(&tx)
+	require.Nil(t, err)
+	require.Equal(t, reflect.TypeOf(&roothashDamask.ExecutorCommit{}).String(), reflect.TypeOf(parsed).String())
+	parsedJson, err := json.Marshal(parsed)
+	require.Nil(t, err)
+	require.Equal(t, expected, parsedJson)
+}
+
+func TestMalformedTxBody(t *testing.T) {
+	// Setup malformed transaction
+	bodyBytes, err := hex.DecodeString("deadbeef")
+	require.Nil(t, err)
+	var body cbor.RawMessage
+	err = body.UnmarshalCBOR(bodyBytes)
+	require.Nil(t, err)
+	tx := transaction.Transaction{
+		Nonce:  0,
+		Method: "staking.Allow",
+		Body:   body,
+	}
+
+	parsed, err := unpackTxBody(&tx)
+	require.Nil(t, parsed)
+	require.NotNil(t, err)
+	require.Equal(t, "unable to cbor-decode consensus tx body: cbor: invalid additional information 30 for type tag, body: deadbeef", err.Error())
+}
+
+func TestMalformedTxMethod(t *testing.T) {
+	// Setup malformed transaction
+	bodyBytes, err := hex.DecodeString("deadbeef")
+	require.Nil(t, err)
+	var body cbor.RawMessage
+	err = body.UnmarshalCBOR(bodyBytes)
+	require.Nil(t, err)
+	tx := transaction.Transaction{
+		Nonce:  0,
+		Method: "not_a_valid_method",
+		Body:   body,
+	}
+
+	parsed, err := unpackTxBody(&tx)
+	require.Nil(t, parsed)
+	require.NotNil(t, err)
+	require.Equal(t, "unknown tx method: not_a_valid_method", err.Error())
+}

api/spec/v1.yaml

@@ -1491,11 +1491,12 @@ components:
           allOf: [$ref: '#/components/schemas/ConsensusTxMethod']
           description: The method that was called.
         body:
-          type: string
-          format: byte
-          # TODO: This is a serialzied CBOR blob. Expose as a JSON subtree instead.
-          # Front-end can pull the "recipient" field from here for transactions of a suitable type.
-          description: The method call body.
+          type: object
+          description: The method call body. This spec does not encode the many possible types;
+            instead, see [the Go API](https://pkg.go.dev/github.com/oasisprotocol/oasis-core/go) of oasis-core.
+            This object will conform to one of the types passed to variable instantiations using `NewMethodName` 
+            two levels down the hierarchy, e.g. `MethodTransfer` from `oasis-core/go/staking/api` seen 
+            [here](https://pkg.go.dev/github.com/oasisprotocol/oasis-core/[email protected]/staking/api#pkg-variables).
           example: *tx_body_1
         success:
           type: boolean

storage/migrations/12_consensus_tx_body.up.sql

@@ -0,0 +1,9 @@
+BEGIN;
+
+ALTER TABLE chain.transactions
+    DROP COLUMN body;
+
+ALTER TABLE chain.transactions
+    ADD COLUMN body JSONB;
+
+COMMIT;

tests/e2e_regression/damask/expected/block.body

@@ -1,3 +1,6 @@
 {
-  "msg": "item not found"
+  "hash": "3cabeb4dd3b9a17c17e2447d2456b3446cabf569621886f51545605c272fd31d",
+  "height": 8049500,
+  "num_transactions": 1,
+  "timestamp": "2022-04-11T12:00:33Z"
 }

tests/e2e_regression/damask/expected/block.headers

@@ -1,7 +1,6 @@
-HTTP/1.1 404 Not Found
-Content-Type: application/json; charset=utf-8
+HTTP/1.1 200 OK
+Content-Type: application/json
 Vary: Origin
-X-Content-Type-Options: nosniff
 Date: UNINTERESTING
 Content-Length: UNINTERESTING
 

tests/e2e_regression/damask/expected/tx.body

@@ -1,3 +1,20 @@
 {
-  "msg": "item not found"
+  "block": 8049424,
+  "body": {
+    "amount": "100000",
+    "to": "oasis1qp6pu7sr822u3mecyt3vtzm54wq2rl5ezg5gutvz"
+  },
+  "error": {
+    "code": 9,
+    "message": "staking: amount is lower than the minimum transfer amount",
+    "module": "staking"
+  },
+  "fee": "0",
+  "hash": "f7a03e0912d355901ee794e5fec79a6b4c91363fc27d953596ee6de5c1492798",
+  "index": 0,
+  "method": "staking.Transfer",
+  "nonce": 11,
+  "sender": "oasis1qqnk4au603zs94k0d0n7c0hkx8t4p6r87s60axru",
+  "success": false,
+  "timestamp": "2022-04-11T11:52:24Z"
 }

tests/e2e_regression/damask/expected/tx.headers

@@ -1,7 +1,6 @@
-HTTP/1.1 404 Not Found
-Content-Type: application/json; charset=utf-8
+HTTP/1.1 200 OK
+Content-Type: application/json
 Vary: Origin
-X-Content-Type-Options: nosniff
 Date: UNINTERESTING
 Content-Length: UNINTERESTING