diff --git a/pilot/pkg/xds/eds.go b/pilot/pkg/xds/eds.go
index 81f662589046..0a5fd9090e61 100644
--- a/pilot/pkg/xds/eds.go
+++ b/pilot/pkg/xds/eds.go
@@ -216,14 +216,6 @@ func (s *DiscoveryServer) deleteEndpointShards(shard model.ShardKey, serviceName
 		epShards := s.EndpointShardsByService[serviceName][namespace]
 		epShards.mutex.Lock()
 		delete(epShards.Shards, shard)
-		epShards.ServiceAccounts = sets.Set{}
-		for _, shard := range epShards.Shards {
-			for _, ep := range shard {
-				if ep.ServiceAccount != "" {
-					epShards.ServiceAccounts.Insert(ep.ServiceAccount)
-				}
-			}
-		}
 		// Clear the cache here to avoid race in cache writes (see edsCacheUpdate for details).
 		s.Cache.Clear(map[model.ConfigKey]struct{}{{
 			Kind:      gvk.ServiceEntry,