host/sgx/epid: ensure attestation API calls hit the same proxy

Refactors the IAS proxy client to expose separate clients for each
configured IAS proxy, instead of load-balancing internally between
endpoints on a per request basis.

This is required because the attestation procedure requires three calls to
the IAS endpoint (`GetSPIDInfo`, `GetSigRL`, `VerifyEvidence`) and these
should contact the same endpoint.

Also retry other proxies on failure.

.changelog/5390.bugfix.md

@@ -0,0 +1,9 @@
+host/sgx/epid: ensure consistent IAS proxy usage for attestation
+
+Refactors the IAS proxy client to expose separate clients for each configured
+IAS proxy, instead of load-balancing internally between endpoints on a
+per-request basis.
+
+This is required because the attestation procedure requires three calls to
+the IAS endpoint (`GetSPIDInfo`, `GetSigRL`, `VerifyEvidence`) which should
+all interact with the same endpoint.

go/ias/init.go

@@ -14,7 +14,7 @@ import (
 var logger = logging.GetLogger("ias")
 
 // New creates a new IAS endpoint.
-func New(identity *identity.Identity) (api.Endpoint, error) {
+func New(identity *identity.Identity) ([]api.Endpoint, error) {
 	if cmdFlags.DebugDontBlameOasis() {
 		if config.GlobalConfig.IAS.DebugSkipVerify {
 			logger.Warn("`ias.debug_skip_verify` set, AVR signature validation bypassed")

go/ias/proxy/client/client.go

@@ -8,8 +8,6 @@ import (
 	"strings"
 
 	"google.golang.org/grpc"
-	"google.golang.org/grpc/resolver"
-	"google.golang.org/grpc/resolver/manual"
 
 	"github.com/oasisprotocol/oasis-core/go/common/crypto/signature"
 	cmnGrpc "github.com/oasisprotocol/oasis-core/go/common/grpc"
@@ -20,45 +18,44 @@ import (
 	"github.com/oasisprotocol/oasis-core/go/ias/proxy"
 )
 
-var _ api.Endpoint = (*proxyClient)(nil)
+var _ api.Endpoint = (*mockEndpoint)(nil)
 
-type proxyClient struct {
-	identity *identity.Identity
+type mockEndpoint struct{}
 
-	conn     *grpc.ClientConn
-	endpoint api.Endpoint
+func (m *mockEndpoint) VerifyEvidence(_ context.Context, evidence *api.Evidence) (*ias.AVRBundle, error) {
+	// Generate a mock AVR, under the assumption that the runtime is built to support this.
+	// The runtime will reject the mock AVR if it is not.
+	avr, err := ias.NewMockAVR(evidence.Quote, evidence.Nonce)
+	if err != nil {
+		return nil, err
+	}
+	return &ias.AVRBundle{
+		Body: avr,
+	}, nil
+}
 
-	spidInfo *api.SPIDInfo
+func (m *mockEndpoint) GetSPIDInfo(_ context.Context) (*api.SPIDInfo, error) {
+	spidInfo := &api.SPIDInfo{}
+	_ = spidInfo.SPID.UnmarshalBinary(make([]byte, ias.SPIDSize))
+	return spidInfo, nil
+}
 
-	logger *logging.Logger
+func (m *mockEndpoint) GetSigRL(_ context.Context, _ uint32) ([]byte, error) {
+	return nil, fmt.Errorf("IAS proxy is not configured, mock used")
 }
 
-func (c *proxyClient) fetchSPIDInfo(ctx context.Context) error {
-	if c.spidInfo != nil || c.endpoint == nil {
-		return nil
-	}
+func (m *mockEndpoint) Cleanup() {}
 
-	var err error
-	if c.spidInfo, err = c.endpoint.GetSPIDInfo(ctx); err != nil {
-		return err
-	}
-	return nil
+var _ api.Endpoint = (*proxyClient)(nil)
+
+type proxyClient struct {
+	conn     *grpc.ClientConn
+	endpoint api.Endpoint
+
+	logger *logging.Logger
 }
 
 func (c *proxyClient) VerifyEvidence(ctx context.Context, evidence *api.Evidence) (*ias.AVRBundle, error) {
-	if c.endpoint == nil {
-		// If the IAS proxy is not configured, generate a mock AVR, under the
-		// assumption that the runtime is built to support this.  The runtime
-		// will reject the mock AVR if it is not.
-		avr, err := ias.NewMockAVR(evidence.Quote, evidence.Nonce)
-		if err != nil {
-			return nil, err
-		}
-		return &ias.AVRBundle{
-			Body: avr,
-		}, nil
-	}
-
 	// Ensure the evidence.Quote passes basic sanity/security checks before
 	// even bothering to contact the backend.
 	var untrustedQuote ias.Quote
@@ -73,84 +70,62 @@ func (c *proxyClient) VerifyEvidence(ctx context.Context, evidence *api.Evidence
 }
 
 func (c *proxyClient) GetSPIDInfo(ctx context.Context) (*api.SPIDInfo, error) {
-	if err := c.fetchSPIDInfo(ctx); err != nil {
-		return nil, err
-	}
-	return c.spidInfo, nil
+	return c.endpoint.GetSPIDInfo(ctx)
 }
 
 func (c *proxyClient) GetSigRL(ctx context.Context, epidGID uint32) ([]byte, error) {
-	if c.endpoint == nil {
-		return nil, fmt.Errorf("IAS proxy is not configured, mock used")
-	}
 	return c.endpoint.GetSigRL(ctx, epidGID)
 }
 
 func (c *proxyClient) Cleanup() {
-	if c.conn != nil {
-		_ = c.conn.Close()
-	}
+	_ = c.conn.Close()
 }
 
-// New creates a new IAS proxy client endpoint.
-func New(identity *identity.Identity, addresses []string) (api.Endpoint, error) {
-	c := &proxyClient{
-		identity: identity,
-		logger:   logging.GetLogger("ias/proxyclient"),
-	}
+// New creates a collection of IAS proxy clients (one client per provided address).
+func New(identity *identity.Identity, addresses []string) ([]api.Endpoint, error) {
+	logger := logging.GetLogger("ias/proxyclient")
 
 	if len(addresses) == 0 {
-		c.logger.Warn("IAS proxy is not configured, all reports will be mocked")
-
-		c.spidInfo = &api.SPIDInfo{}
-		_ = c.spidInfo.SPID.UnmarshalBinary(make([]byte, ias.SPIDSize))
-	} else {
-		var resolverState resolver.State
-		pubKeys := make(map[signature.PublicKey]bool)
-		for _, addr := range addresses {
-			spl := strings.Split(addr, "@")
-			if len(spl) != 2 {
-				return nil, fmt.Errorf("missing public key in address '%s'", addr)
-			}
-
-			var pk signature.PublicKey
-			if err := pk.UnmarshalText([]byte(spl[0])); err != nil {
-				return nil, fmt.Errorf("malformed public key in address '%s': %w", addr, err)
-			}
-
-			pubKeys[pk] = true
-			resolverState.Addresses = append(resolverState.Addresses, resolver.Address{Addr: spl[1]})
+		logger.Warn("IAS proxy is not configured, all reports will be mocked")
+		return []api.Endpoint{&mockEndpoint{}}, nil
+	}
+
+	clients := make([]api.Endpoint, 0, len(addresses))
+	for _, addr := range addresses {
+		spl := strings.Split(addr, "@")
+		if len(spl) != 2 {
+			return nil, fmt.Errorf("missing public key in address '%s'", addr)
 		}
 
+		var pk signature.PublicKey
+		if err := pk.UnmarshalText([]byte(spl[0])); err != nil {
+			return nil, fmt.Errorf("malformed public key in address '%s': %w", addr, err)
+		}
 		creds, err := cmnGrpc.NewClientCreds(&cmnGrpc.ClientOptions{
-			ServerPubKeys: pubKeys,
+			ServerPubKeys: map[signature.PublicKey]bool{pk: true},
 			CommonName:    proxy.CommonName,
 			GetClientCertificate: func(cri *tls.CertificateRequestInfo) (*tls.Certificate, error) {
 				return identity.TLSCertificate, nil
 			},
 		})
 		if err != nil {
-			return nil, fmt.Errorf("failed to create client credentials: %w", err)
+			return nil, fmt.Errorf("failed to create client credentials for address '%s': %w", addr, err)
 		}
-
-		manualResolver := manual.NewBuilderWithScheme("oasis-core-resolver")
 		conn, err := cmnGrpc.Dial(
-			"oasis-core-resolver:///",
+			spl[1],
 			grpc.WithTransportCredentials(creds),
-			// https://github.com/grpc/grpc-go/issues/3003
-			grpc.WithDefaultServiceConfig(`{"loadBalancingPolicy":"round_robin"}`),
 			grpc.WithDefaultCallOptions(grpc.WaitForReady(true)),
-			grpc.WithResolvers(manualResolver),
 		)
 		if err != nil {
-			return nil, fmt.Errorf("failed to dial IAS proxy: %w", err)
+			return nil, fmt.Errorf("failed to dial IAS proxy address '%s': %w", addr, err)
 		}
 
-		manualResolver.UpdateState(resolverState)
-
-		c.conn = conn
-		c.endpoint = api.NewEndpointClient(conn)
+		clients = append(clients, &proxyClient{
+			conn:     conn,
+			endpoint: api.NewEndpointClient(conn),
+			logger:   logger,
+		})
 	}
 
-	return c, nil
+	return clients, nil
 }

go/oasis-node/cmd/node/node.go

@@ -74,7 +74,7 @@ type Node struct {
 	Genesis  genesisAPI.Provider
 	Identity *identity.Identity
 	Sentry   sentryAPI.Backend
-	IAS      iasAPI.Endpoint
+	IAS      []iasAPI.Endpoint
 
 	RuntimeRegistry runtimeRegistry.Registry
 
@@ -240,7 +240,6 @@ func (n *Node) initRuntimeWorkers() error {
 		n.Consensus,
 		n.LightClient,
 		n.P2P,
-		n.IAS,
 		n.Consensus.KeyManager(),
 		n.RuntimeRegistry,
 	)

go/runtime/host/sgx/epid.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"encoding/binary"
 	"fmt"
+	"time"
 
 	"github.com/oasisprotocol/oasis-core/go/common"
 	"github.com/oasisprotocol/oasis-core/go/common/cbor"
@@ -20,6 +21,11 @@ type teeStateEPID struct {
 	teeStateImplCommon
 
 	epidGID uint32
+
+	// prevIAS is the index of the IAS server that was used for the last successful attestation.
+	// This is used as a heuristic to first query the IAS server that is likely able to
+	// successfully do the attestation.
+	prevIAS int
 }
 
 func (ep *teeStateEPID) Init(ctx context.Context, sp *sgxProvisioner, runtimeID common.Namespace, version version.Version) ([]byte, error) {
@@ -36,20 +42,57 @@ func (ep *teeStateEPID) Init(ctx context.Context, sp *sgxProvisioner, runtimeID
 }
 
 func (ep *teeStateEPID) Update(ctx context.Context, sp *sgxProvisioner, conn protocol.Connection, report []byte, nonce string) ([]byte, error) {
-	spidInfo, err := sp.ias.GetSPIDInfo(ctx)
-	if err != nil {
-		return nil, fmt.Errorf("error while requesting SPID info: %w", err)
-	}
-
 	// Check if new format of attestations is supported in the consensus layer and use it.
 	regParams, err := sp.consensus.Registry().ConsensusParameters(ctx, consensus.HeightLatest)
 	if err != nil {
 		return nil, fmt.Errorf("unable to determine registry consensus parameters: %w", err)
 	}
 	supportsAttestationV1 := (regParams.TEEFeatures != nil && regParams.TEEFeatures.SGX.PCS)
 
+	// Start with the IAS server that was used for the last successful attestation.
+	// TODO: Could consider implementing a strategy for more optimized endpoint selection with
+	// latency and success rate feedback (in ias/proxy/client.go). But (re-)attestations are
+	// not so frequent and this is the only code that uses the IAS clients, so this is good enough.
+	for i := ep.prevIAS; i < ep.prevIAS+len(sp.ias); i++ {
+		idx := i % len(sp.ias)
+		resp, err := ep.update(ctx, sp, conn, report, nonce, supportsAttestationV1, sp.ias[idx])
+		if err == nil {
+			ep.prevIAS = idx
+			return resp, nil
+		}
+
+		sp.logger.Warn("error obtaining attestation, trying next IAS server", "err", err, "client_idx", idx)
+		if i == ep.prevIAS+len(sp.ias)-1 {
+			return nil, err
+		}
+
+		select {
+		case <-time.After(50 * time.Millisecond):
+			continue
+		case <-ctx.Done():
+			return nil, ctx.Err()
+		}
+	}
+	return nil, fmt.Errorf("no IAS servers configured")
+}
+
+func (ep *teeStateEPID) update(
+	ctx context.Context,
+	sp *sgxProvisioner,
+	conn protocol.Connection,
+	report []byte,
+	nonce string,
+	supportsAttestationV1 bool,
+	iasClient ias.Endpoint,
+) ([]byte, error) {
+	// Obtain SPID info.
+	spidInfo, err := iasClient.GetSPIDInfo(ctx)
+	if err != nil {
+		return nil, fmt.Errorf("error while requesting SPID info: %w", err)
+	}
+
 	// Update the SigRL (Not cached, knowing if revoked is important).
-	sigRL, err := sp.ias.GetSigRL(ctx, ep.epidGID)
+	sigRL, err := iasClient.GetSigRL(ctx, ep.epidGID)
 	if err != nil {
 		return nil, fmt.Errorf("error while requesting SigRL: %w", err)
 	}
@@ -84,7 +127,7 @@ func (ep *teeStateEPID) Update(ctx context.Context, sp *sgxProvisioner, conn pro
 		MinTCBEvaluationDataNumber: quotePolicy.MinTCBEvaluationDataNumber,
 	}
 
-	avrBundle, err := sp.ias.VerifyEvidence(ctx, &evidence)
+	avrBundle, err := iasClient.VerifyEvidence(ctx, &evidence)
 	if err != nil {
 		return nil, fmt.Errorf("error while verifying attestation evidence: %w", err)
 	}
@@ -94,7 +137,7 @@ func (ep *teeStateEPID) Update(ctx context.Context, sp *sgxProvisioner, conn pro
 	if decErr == nil && avr.TCBEvaluationDataNumber < quotePolicy.MinTCBEvaluationDataNumber {
 		// Retry again with early updating.
 		evidence.EarlyTCBUpdate = true
-		avrBundle, err = sp.ias.VerifyEvidence(ctx, &evidence)
+		avrBundle, err = iasClient.VerifyEvidence(ctx, &evidence)
 		if err != nil {
 			return nil, fmt.Errorf("error while verifying attestation evidence with early update: %w", err)
 		}

go/runtime/host/sgx/sgx.go

@@ -59,8 +59,8 @@ type Config struct {
 	// LoaderPath is the path to the runtime loader binary.
 	LoaderPath string
 
-	// IAS is the Intel Attestation Service endpoint.
-	IAS ias.Endpoint
+	// IAS are the Intel Attestation Service endpoint.
+	IAS []ias.Endpoint
 	// PCS is the Intel Provisioning Certification Service client.
 	PCS pcs.Client
 	// Consensus is the consensus layer backend.
@@ -157,7 +157,7 @@ type sgxProvisioner struct {
 	cfg Config
 
 	sandbox   host.Provisioner
-	ias       ias.Endpoint
+	ias       []ias.Endpoint
 	pcs       pcs.Client
 	aesm      *aesm.Client
 	consensus consensus.Backend

go/runtime/host/sgx/sgx_test.go

@@ -12,6 +12,7 @@ import (
 	cmnIAS "github.com/oasisprotocol/oasis-core/go/common/sgx/ias"
 	"github.com/oasisprotocol/oasis-core/go/common/version"
 	cmt "github.com/oasisprotocol/oasis-core/go/consensus/cometbft/api"
+	"github.com/oasisprotocol/oasis-core/go/ias/api"
 	iasHttp "github.com/oasisprotocol/oasis-core/go/ias/http"
 	"github.com/oasisprotocol/oasis-core/go/runtime/bundle"
 	"github.com/oasisprotocol/oasis-core/go/runtime/host"
@@ -76,7 +77,7 @@ func TestProvisionerSGX(t *testing.T) {
 					ConsensusProtocolVersion: version.Versions.ConsensusProtocol,
 				},
 				LoaderPath:            envRuntimeLoaderPath,
-				IAS:                   ias,
+				IAS:                   []api.Endpoint{ias},
 				RuntimeAttestInterval: 2 * time.Second,
 				InsecureNoSandbox:     true,
 				SandboxBinaryPath:     bwrapPath,
@@ -93,7 +94,7 @@ func TestProvisionerSGX(t *testing.T) {
 				},
 				LoaderPath:            envRuntimeLoaderPath,
 				RuntimeAttestInterval: 2 * time.Second,
-				IAS:                   ias,
+				IAS:                   []api.Endpoint{ias},
 				SandboxBinaryPath:     bwrapPath,
 			})
 		}, extraTests)

go/runtime/registry/config.go

@@ -70,7 +70,7 @@ type RuntimeHostConfig struct {
 	Runtimes map[common.Namespace]map[version.Version]*runtimeHost.Config
 }
 
-func newConfig(dataDir string, commonStore *persistent.CommonStore, consensus consensus.Backend, ias ias.Endpoint) (*RuntimeConfig, error) { //nolint: gocyclo
+func newConfig(dataDir string, commonStore *persistent.CommonStore, consensus consensus.Backend, ias []ias.Endpoint) (*RuntimeConfig, error) { //nolint: gocyclo
 	var cfg RuntimeConfig
 
 	haveSetRuntimes := len(config.GlobalConfig.Runtime.Paths) > 0

go/runtime/registry/registry.go

@@ -571,7 +571,7 @@ func newRuntime(
 }
 
 // New creates a new runtime registry.
-func New(ctx context.Context, dataDir string, commonStore *persistent.CommonStore, consensus consensus.Backend, ias ias.Endpoint) (Registry, error) {
+func New(ctx context.Context, dataDir string, commonStore *persistent.CommonStore, consensus consensus.Backend, ias []ias.Endpoint) (Registry, error) {
 	cfg, err := newConfig(dataDir, commonStore, consensus, ias)
 	if err != nil {
 		return nil, err