Merge pull request #568 from oasisprotocol/pro-wh/bugfix/chainutf8

evmnfts, evmtokens: utf-8-clean strings from chain

analyzer/evmnfts/evm_nfts.go

@@ -116,12 +116,17 @@ func (p *processor) ProcessItem(ctx context.Context, batch *storage.QueryBatch,
 		staleNFT.Addr,
 		staleNFT.ID,
 		staleNFT.DownloadRound,
-		nftData.MetadataURI,
+		storage.SanitizeStringP(nftData.MetadataURI),
 		nftData.MetadataAccessed,
-		nftData.Metadata,
-		nftData.Name,
-		nftData.Description,
-		nftData.Image,
+		// Metadata has already gone through a round trip of JSON Unmarshal
+		// and Marshal, which should already make it safe for PostgreSQL. But
+		// for consistency, pass it through SanitizeString as well. This
+		// shouldn't have any effect, and if it does, it shouldn't affect the
+		// JSON syntax.
+		storage.SanitizeStringP(nftData.Metadata),
+		storage.SanitizeStringP(nftData.Name),
+		storage.SanitizeStringP(nftData.Description),
+		storage.SanitizeStringP(nftData.Image),
 	)
 	return nil
 }

analyzer/evmtokens/evm_tokens.go

@@ -139,8 +139,8 @@ func (p *processor) ProcessItem(ctx context.Context, batch *storage.QueryBatch,
 			p.runtime,
 			staleToken.Addr,
 			tokenData.Type,
-			tokenData.Name,
-			tokenData.Symbol,
+			storage.SanitizeString(tokenData.Name),
+			storage.SanitizeString(tokenData.Symbol),
 			tokenData.Decimals,
 			totalSupply,
 			staleToken.DownloadRound,

analyzer/runtime/evm/erc721_test.go

@@ -0,0 +1,42 @@
+package evm
+
+import (
+	"encoding/json"
+	"testing"
+	"unicode/utf8"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestJSONRoundTrip(t *testing.T) {
+	for _, s := range []string{
+		// Invalid UTF-8.
+		"{\"a\": \"\xff\"}",
+		"{\"\xff\": \"a\"}",
+		// NUL bytes.
+		"{\"a\": \"\x00\"}",
+		"{\"\x00\": \"a\"}",
+		// Codepoint zero.
+		"{\"a\": \"\\u0000\"}",
+		"{\"\\u0000\": \"a\"}",
+	} {
+		var parsed any
+		err := json.Unmarshal([]byte(s), &parsed)
+		if err != nil {
+			// Erroring out while unmarshalling is fine.
+			t.Logf("%#v -> parsing: %v\n", s, err)
+			continue
+		}
+		roundTripBuf, err := json.Marshal(parsed)
+		if err != nil {
+			// Erroring out while marshalling is uncool, but not a showstopper.
+			t.Logf("%#v -> stringifying: %v\n", s, err)
+			continue
+		}
+		roundTrip := string(roundTripBuf)
+		// If it gets through the round trip, it'd better be clean.
+		require.True(t, utf8.ValidString(roundTrip), "%#v -> invalid UTF-8 %#v", s, roundTrip)
+		require.NotContains(t, roundTrip, 0x0, "%#v -> contains NUL byte %#v", s, roundTrip)
+		t.Logf("%#v -> %#v\n", s, roundTrip)
+	}
+}

storage/api.go

@@ -7,6 +7,8 @@ import (
 	"sync"
 
 	"github.com/jackc/pgx/v5"
+
+	"github.com/oasisprotocol/nexus/common"
 )
 
 type BatchItem struct {
@@ -128,3 +130,10 @@ type TargetStorage interface {
 func SanitizeString(msg string) string {
 	return strings.ToValidUTF8(strings.ReplaceAll(msg, "\x00", "?"), "?")
 }
+
+func SanitizeStringP(msg *string) *string {
+	if msg == nil {
+		return nil
+	}
+	return common.Ptr(SanitizeString(*msg))
+}