Merge pull request #768 from oasisprotocol/andrew7234/evm-verifier-up…

…date-sourcify

update evmverifier to use current sourcify api

.changelog/768.bugfix.md

@@ -0,0 +1 @@
+evmverifier: update to use current sourcify api

.gitignore

@@ -13,7 +13,7 @@ tests/e2e/testnet/net-runner
 tests/e2e_regression/*/actual/*
 
 # KVStore autogenerated backups
-tests/e2e_regression/rpc-cache/*.backup
+tests/e2e_regression/**/rpc-cache/*.backup
 
 # Log output.
 **/*.log

analyzer/evmverifier/sourcify/client.go

@@ -86,35 +86,79 @@ func (s *SourcifyClient) callAPI(ctx context.Context, method string, url *url.UR
 // Note: This uses the free, public server API. If it turns out to be unreliable, we could use the repository API (vis IPFS proxy) instead, e.g.:
 // http://ipfs.default:8080/ipns/repo.sourcify.dev/contracts/full_match/23294
 func (s *SourcifyClient) GetVerifiedContractAddresses(ctx context.Context, runtime common.Runtime) (map[ethCommon.Address]VerificationLevel, error) {
-	// Fetch verified contract addresses.
+	// Build map of addresses.
+	addresses := make(map[ethCommon.Address]VerificationLevel)
+
+	// Fetch fully verified contract addresses.
+	// See https://docs.sourcify.dev/docs/full-vs-partial-match/
 	u := *s.serverUrl
-	u.Path = path.Join(u.Path, "files/contracts", sourcifyChains[s.chain][runtime])
-	body, err := s.callAPI(ctx, http.MethodGet, &u)
+	u.Path = path.Join(u.Path, "files/contracts/full", sourcifyChains[s.chain][runtime])
+	fullyVerified, err := s.GetAllAddressPages(ctx, &u)
 	if err != nil {
-		return nil, fmt.Errorf("failed to fetch verified contract addresses: %w (%s)", err, u.String())
+		return nil, err
 	}
-
-	// Parse response.
-	var response struct {
-		Full    []ethCommon.Address `json:"full"`
-		Partial []ethCommon.Address `json:"partial"` // See https://docs.sourcify.dev/docs/full-vs-partial-match/
+	for _, addr := range fullyVerified {
+		addresses[addr] = VerificationLevelFull
 	}
-	if err := json.Unmarshal(body, &response); err != nil {
-		return nil, fmt.Errorf("failed to parse verified contract addresses: %w (%s)", err, u.String())
+	// Fetch partially verified contract addresses.
+	u = *s.serverUrl
+	u.Path = path.Join(u.Path, "files/contracts/any", sourcifyChains[s.chain][runtime])
+	allContracts, err := s.GetAllAddressPages(ctx, &u)
+	if err != nil {
+		return nil, err
 	}
-
-	// Build map of addresses.
-	addresses := make(map[ethCommon.Address]VerificationLevel)
-	for _, addr := range response.Full {
-		addresses[addr] = VerificationLevelFull
+	for _, addr := range allContracts {
+		if _, exists := addresses[addr]; !exists {
+			addresses[addr] = VerificationLevelPartial
+		}
 	}
-	for _, addr := range response.Partial {
-		addresses[addr] = VerificationLevelPartial
+
+	return addresses, nil
+}
+
+func (s *SourcifyClient) GetAllAddressPages(ctx context.Context, u *url.URL) ([]ethCommon.Address, error) {
+	page := 0
+	addresses := []ethCommon.Address{}
+	for {
+		// https://sourcify.dev/server/api-docs/#/Repository/get_files_contracts_any__chain_
+		q := u.Query()
+		q.Set("page", fmt.Sprintf("%d", page))
+		u.RawQuery = q.Encode()
+
+		body, err := s.callAPI(ctx, http.MethodGet, u)
+		if err != nil {
+			return nil, fmt.Errorf("failed to fetch fully verified contract addresses from %s: %w", u.String(), err)
+		}
+		// Parse response.
+		var response struct {
+			Results    []ethCommon.Address `json:"results"`
+			Pagination SourcifyPagination  `json:"pagination"`
+		}
+
+		if err := json.Unmarshal(body, &response); err != nil {
+			return nil, fmt.Errorf("failed to parse fully verified contract addresses: %w (%s)", err, u.String())
+		}
+		addresses = append(addresses, response.Results...)
+		// Check pagination and increment if necessary.
+		if !response.Pagination.HasNextPage {
+			break
+		}
+		page++
 	}
 
 	return addresses, nil
 }
 
+type SourcifyPagination struct {
+	CurrentPage        uint64 `json:"currentPage"`
+	TotalPages         uint64 `json:"totalPages"`
+	ResultsPerPage     uint64 `json:"resultsPerPage"`
+	ResultsCurrentPage uint64 `json:"resultsCurrentPage"`
+	TotalResults       uint64 `json:"totalResults"`
+	HasNextPage        bool   `json:"hasNextPage"`
+	HasPreviousPage    bool   `json:"hasPreviousPage"`
+}
+
 type SourceFile struct {
 	Name    string `json:"name"`
 	Path    string `json:"path"`

analyzer/evmverifier/sourcify/client_test.go

@@ -18,27 +18,71 @@ import (
 )
 
 var (
-	// Source: https://sourcify.dev/server/files/contracts/23294
-	//
-	//go:embed testdata/get_contract_addresses_response.json
-	mockGetContractAddressesRsponse []byte
-
 	// Source: https://sourcify.dev/server/files/any/23294/0x127c49aE10e3c18be057106F4d16946E3Ae43975
 	//
 	//go:embed testdata/get_contract_source_files_response.json
 	mockGetContractSourceFilesResponse []byte
+
+	// Source: https://sourcify.dev/server/files/contracts/any/23295
+	//
+	//go:embed testdata/get_contract_addresses_any_0.json
+	mockGetContractAddressesAnyPage0Response []byte
+
+	// Source: https://sourcify.dev/server/files/contracts/any/23295?page=1
+	//
+	//go:embed testdata/get_contract_addresses_any_1.json
+	mockGetContractAddressesAnyPage1Response []byte
+
+	// Source: https://sourcify.dev/server/files/contracts/any/23295
+	//
+	//go:embed testdata/get_contract_addresses_full_0.json
+	mockGetContractAddressesFullPage0Response []byte
+
+	// Source: https://sourcify.dev/server/files/contracts/any/23295?page=1
+	//
+	//go:embed testdata/get_contract_addresses_full_1.json
+	mockGetContractAddressesFullPage1Response []byte
+
+	// Source: https://sourcify.dev/server/files/contracts/any/23295?page=2
+	//
+	//go:embed testdata/get_contract_addresses_empty_page.json
+	mockGetContractAddressesEmptyPageResponse []byte
 )
 
 func TestGetVerifiedContractAddresses(t *testing.T) {
 	require := require.New(t)
 
 	testServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		if !strings.HasPrefix(r.URL.Path, "/files/contracts") {
-			w.WriteHeader(http.StatusNotFound)
+		switch {
+		case strings.HasPrefix(r.URL.Path, "/files/contracts/any"):
+			switch {
+			case !r.URL.Query().Has("page") || r.URL.Query().Get("page") == "0":
+				w.WriteHeader(http.StatusOK)
+				_, _ = w.Write(mockGetContractAddressesAnyPage0Response)
+			case r.URL.Query().Get("page") == "1":
+				w.WriteHeader(http.StatusOK)
+				_, _ = w.Write(mockGetContractAddressesAnyPage1Response)
+			default:
+				w.WriteHeader(http.StatusOK)
+				_, _ = w.Write(mockGetContractAddressesEmptyPageResponse)
+			}
 			return
+		case strings.HasPrefix(r.URL.Path, "/files/contracts/full"):
+			switch {
+			case !r.URL.Query().Has("page") || r.URL.Query().Get("page") == "0":
+				w.WriteHeader(http.StatusOK)
+				_, _ = w.Write(mockGetContractAddressesFullPage0Response)
+			case r.URL.Query().Get("page") == "1":
+				w.WriteHeader(http.StatusOK)
+				_, _ = w.Write(mockGetContractAddressesFullPage1Response)
+			default:
+				w.WriteHeader(http.StatusOK)
+				_, _ = w.Write(mockGetContractAddressesEmptyPageResponse)
+			}
+			return
+		default:
+			w.WriteHeader(http.StatusNotFound)
 		}
-		w.WriteHeader(http.StatusOK)
-		_, _ = w.Write(mockGetContractAddressesRsponse)
 	}))
 	defer testServer.Close()
 
@@ -60,8 +104,8 @@ func TestGetVerifiedContractAddresses(t *testing.T) {
 			require.FailNowf("GetVerifiedContractAddresses", "unexpected verification level %s", level)
 		}
 	}
-	require.Equal(15, nFull)
-	require.Equal(3, nPartial)
+	require.Equal(249, nFull)
+	require.Equal(42, nPartial)
 }
 
 func TestGetContractSourceFiles(t *testing.T) {