From ccfc6ba41b97e79d5b87180e0eabf58f491aee00 Mon Sep 17 00:00:00 2001 From: Yahya Hassanzadeh Date: Wed, 15 Nov 2023 14:01:12 -0800 Subject: [PATCH 01/67] adds GossipSubApplicationSpecificScoreCache --- network/p2p/cache.go | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/network/p2p/cache.go b/network/p2p/cache.go index f764f1c6321..fdb574190aa 100644 --- a/network/p2p/cache.go +++ b/network/p2p/cache.go @@ -1,6 +1,8 @@ package p2p import ( + "time" + "github.com/libp2p/go-libp2p/core/peer" "github.com/libp2p/go-libp2p/core/protocol" ) @@ -69,6 +71,32 @@ type GossipSubSpamRecordCache interface { Has(peerID peer.ID) bool } +// GossipSubApplicationSpecificScoreCache is a cache for storing the application specific score of peers. +// The application specific score of a peer is used to calculate the GossipSub score of the peer; it contains the spam penalty of the peer, staking score, and subscription penalty. +// Note that none of the application specific scores, spam penalties, staking scores, and subscription penalties are shared publicly with other peers. +// Rather they are solely used by the current peer to select the peers to which it will connect on a topic mesh. +// The cache is expected to have an eject policy to remove the least recently used record when the cache is full. +// Implementation must be thread-safe, but can be blocking. +type GossipSubApplicationSpecificScoreCache interface { + // Get returns the application specific score of a peer from the cache. + // Args: + // - peerID: the peer ID of the peer in the GossipSub protocol. + // Returns: + // - float64: the application specific score of the peer. + // - time.Time: the time at which the score was last updated. + // - bool: true if the score was retrieved successfully, false otherwise. + Get(peerID peer.ID) (float64, time.Time, bool) + + // Add adds the application specific score of a peer to the cache. + // The cache is expected to have an eject policy to remove the least recently used record when the cache is full, + // hence Add is expected to succeed always. + // Args: + // - peerID: the peer ID of the peer in the GossipSub protocol. + // - score: the application specific score of the peer. + // - time: the time at which the score was last updated. + Add(peerID peer.ID, score float64, time time.Time) +} + // GossipSubSpamRecord represents spam record of a peer in the GossipSub protocol. // It acts as a penalty card for a peer in the GossipSub protocol that keeps the // spam penalty of the peer as well as its decay factor. From 5e2b1dfaf853bd28d9b5741560d968a51cb858b2 Mon Sep 17 00:00:00 2001 From: Yahya Hassanzadeh Date: Wed, 15 Nov 2023 14:10:47 -0800 Subject: [PATCH 02/67] adds app specific score cache entity --- .../scoring/internal/appSpecificScoreCache.go | 1 + .../internal/appSpecificScoreRecord.go | 42 +++++++++++++++++++ 2 files changed, 43 insertions(+) create mode 100644 network/p2p/scoring/internal/appSpecificScoreCache.go create mode 100644 network/p2p/scoring/internal/appSpecificScoreRecord.go diff --git a/network/p2p/scoring/internal/appSpecificScoreCache.go b/network/p2p/scoring/internal/appSpecificScoreCache.go new file mode 100644 index 00000000000..5bf0569ce8c --- /dev/null +++ b/network/p2p/scoring/internal/appSpecificScoreCache.go @@ -0,0 +1 @@ +package internal diff --git a/network/p2p/scoring/internal/appSpecificScoreRecord.go b/network/p2p/scoring/internal/appSpecificScoreRecord.go new file mode 100644 index 00000000000..24e860bf84b --- /dev/null +++ b/network/p2p/scoring/internal/appSpecificScoreRecord.go @@ -0,0 +1,42 @@ +package internal + +import ( + "time" + + "github.com/libp2p/go-libp2p/core/peer" + + "github.com/onflow/flow-go/model/flow" +) + +// appSpecificScoreRecordEntity is an entity that represents the application specific score of a peer. +type appSpecificScoreRecordEntity struct { + // entityId is the key of the entity in the cache. It is the hash of the peer id. + // It is intentionally encoded as part of the struct to avoid recomputing it. + entityId flow.Identifier + + // PeerID is the peer id of the peer that is the owner of the subscription. + PeerID peer.ID + + // Score is the application specific score of the peer. + Score float64 + + // LastUpdated is the last time the score was updated. + LastUpdated time.Time +} + +var _ flow.Entity = (*appSpecificScoreRecordEntity)(nil) + +// ID returns the entity id of the subscription record, which is the hash of the peer id. +// The ApplicationSpecificScoreCache uses the entity id as the key in the cache. +func (a *appSpecificScoreRecordEntity) ID() flow.Identifier { + if a.entityId == flow.ZeroID { + a.entityId = flow.MakeID(a.PeerID) + } + return a.entityId +} + +// Checksum returns the entity id of the subscription record, which is the hash of the peer id. +// It is of no use in the cache, but it is implemented to satisfy the flow.Entity interface. +func (a *appSpecificScoreRecordEntity) Checksum() flow.Identifier { + return a.ID() +} From af0fb5346385b117107162feac5aae11d635883f Mon Sep 17 00:00:00 2001 From: Yahya Hassanzadeh Date: Wed, 15 Nov 2023 14:26:04 -0800 Subject: [PATCH 03/67] adds cache implementation --- network/p2p/cache.go | 7 +- .../scoring/internal/appSpecificScoreCache.go | 102 ++++++++++++++++++ .../internal/appSpecificScoreRecord.go | 11 +- 3 files changed, 110 insertions(+), 10 deletions(-) diff --git a/network/p2p/cache.go b/network/p2p/cache.go index fdb574190aa..268bf0bdded 100644 --- a/network/p2p/cache.go +++ b/network/p2p/cache.go @@ -88,13 +88,14 @@ type GossipSubApplicationSpecificScoreCache interface { Get(peerID peer.ID) (float64, time.Time, bool) // Add adds the application specific score of a peer to the cache. - // The cache is expected to have an eject policy to remove the least recently used record when the cache is full, - // hence Add is expected to succeed always. + // If the peer already has a score in the cache, the score is updated. // Args: // - peerID: the peer ID of the peer in the GossipSub protocol. // - score: the application specific score of the peer. // - time: the time at which the score was last updated. - Add(peerID peer.ID, score float64, time time.Time) + // Returns: + // - error on failure to add the score. The returned error is irrecoverable and indicates an exception. + Add(peerID peer.ID, score float64, time time.Time) error } // GossipSubSpamRecord represents spam record of a peer in the GossipSub protocol. diff --git a/network/p2p/scoring/internal/appSpecificScoreCache.go b/network/p2p/scoring/internal/appSpecificScoreCache.go index 5bf0569ce8c..5513db07df8 100644 --- a/network/p2p/scoring/internal/appSpecificScoreCache.go +++ b/network/p2p/scoring/internal/appSpecificScoreCache.go @@ -1 +1,103 @@ package internal + +import ( + "fmt" + "time" + + "github.com/libp2p/go-libp2p/core/peer" + "github.com/rs/zerolog" + + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/module" + herocache "github.com/onflow/flow-go/module/mempool/herocache/backdata" + "github.com/onflow/flow-go/module/mempool/herocache/backdata/heropool" + "github.com/onflow/flow-go/module/mempool/stdmap" + "github.com/onflow/flow-go/network/p2p" +) + +// AppSpecificScoreCache is a cache that stores the application specific score of peers. +// The application specific score of a peer is used to calculate the GossipSub score of the peer. +// Note that the application specific score and the GossipSub score are solely used by the current peer to select the peers +// to which it will connect on a topic mesh. +type AppSpecificScoreCache struct { + c *stdmap.Backend +} + +var _ p2p.GossipSubApplicationSpecificScoreCache = (*AppSpecificScoreCache)(nil) + +// NewAppSpecificScoreCache creates a new application specific score cache with the given size limit. +// The cache has an LRU eviction policy. +// Args: +// - sizeLimit: the size limit of the cache. +// - logger: the logger to use for logging. +// - collector: the metrics collector to use for collecting metrics. +// Returns: +// - *AppSpecificScoreCache: the created cache. +func NewAppSpecificScoreCache(sizeLimit uint32, logger zerolog.Logger, collector module.HeroCacheMetrics) *AppSpecificScoreCache { + backData := herocache.NewCache(sizeLimit, + herocache.DefaultOversizeFactor, + heropool.LRUEjection, + logger.With().Str("mempool", "subscription-records").Logger(), + collector) + + return &AppSpecificScoreCache{ + c: stdmap.NewBackend(stdmap.WithBackData(backData)), + } +} + +// Get returns the application specific score of a peer from the cache. +// Args: +// - peerID: the peer ID of the peer in the GossipSub protocol. +// Returns: +// - float64: the application specific score of the peer. +// - time.Time: the time at which the score was last updated. +// - bool: true if the score was retrieved successfully, false otherwise. +func (a *AppSpecificScoreCache) Get(peerID peer.ID) (float64, time.Time, bool) { + e, ok := a.c.ByID(flow.MakeID(peerID)) + if !ok { + return 0, time.Time{}, false + } + return e.(appSpecificScoreRecordEntity).Score, e.(appSpecificScoreRecordEntity).LastUpdated, true +} + +// Add adds the application specific score of a peer to the cache. +// If the peer already has a score in the cache, the score is updated. +// Args: +// - peerID: the peer ID of the peer in the GossipSub protocol. +// - score: the application specific score of the peer. +// - time: the time at which the score was last updated. +// Returns: +// - error on failure to add the score. The returned error is irrecoverable and indicates an exception. +func (a *AppSpecificScoreCache) Add(peerID peer.ID, score float64, time time.Time) error { + entityId := flow.MakeID(peerID) + + // first tries an optimistic add; if it fails, it tries an optimistic update + added := a.c.Add(appSpecificScoreRecordEntity{ + entityId: entityId, + PeerID: peerID, + Score: score, + LastUpdated: time, + }) + if !added { + updated, ok := a.c.Adjust(entityId, func(entity flow.Entity) flow.Entity { + r := entity.(appSpecificScoreRecordEntity) + r.Score = score + r.LastUpdated = time + return r + }) + + if !ok { + return fmt.Errorf("failed to add app specific score record for peer %s", peerID) + } + + u := updated.(appSpecificScoreRecordEntity) + if u.Score != score { + return fmt.Errorf("incorrect update of app specific score record for peer %s, expected score %f, got score %f", peerID, score, u.Score) + } + if u.LastUpdated != time { + return fmt.Errorf("incorrect update of app specific score record for peer %s, expected last updated %s, got last updated %s", peerID, time, u.LastUpdated) + } + } + + return nil +} diff --git a/network/p2p/scoring/internal/appSpecificScoreRecord.go b/network/p2p/scoring/internal/appSpecificScoreRecord.go index 24e860bf84b..5abd11d8922 100644 --- a/network/p2p/scoring/internal/appSpecificScoreRecord.go +++ b/network/p2p/scoring/internal/appSpecificScoreRecord.go @@ -27,16 +27,13 @@ type appSpecificScoreRecordEntity struct { var _ flow.Entity = (*appSpecificScoreRecordEntity)(nil) // ID returns the entity id of the subscription record, which is the hash of the peer id. -// The ApplicationSpecificScoreCache uses the entity id as the key in the cache. -func (a *appSpecificScoreRecordEntity) ID() flow.Identifier { - if a.entityId == flow.ZeroID { - a.entityId = flow.MakeID(a.PeerID) - } +// The AppSpecificScoreCache uses the entity id as the key in the cache. +func (a appSpecificScoreRecordEntity) ID() flow.Identifier { return a.entityId } // Checksum returns the entity id of the subscription record, which is the hash of the peer id. // It is of no use in the cache, but it is implemented to satisfy the flow.Entity interface. -func (a *appSpecificScoreRecordEntity) Checksum() flow.Identifier { - return a.ID() +func (a appSpecificScoreRecordEntity) Checksum() flow.Identifier { + return a.entityId } From e5265a6a4f3a5f09632be437cbbf9d33c4544ae4 Mon Sep 17 00:00:00 2001 From: Yahya Hassanzadeh Date: Wed, 15 Nov 2023 14:34:14 -0800 Subject: [PATCH 04/67] adds TestAppSpecificScoreCache --- .../internal/appSpecificScoreCache_test.go | 47 +++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 network/p2p/scoring/internal/appSpecificScoreCache_test.go diff --git a/network/p2p/scoring/internal/appSpecificScoreCache_test.go b/network/p2p/scoring/internal/appSpecificScoreCache_test.go new file mode 100644 index 00000000000..29ded5d2798 --- /dev/null +++ b/network/p2p/scoring/internal/appSpecificScoreCache_test.go @@ -0,0 +1,47 @@ +package internal_test + +import ( + "testing" + "time" + + "github.com/rs/zerolog" + "github.com/stretchr/testify/require" + + "github.com/onflow/flow-go/module/metrics" + "github.com/onflow/flow-go/network/p2p/scoring/internal" + "github.com/onflow/flow-go/utils/unittest" +) + +// TestAppSpecificScoreCache tests the functionality of AppSpecificScoreCache; +// specifically, it tests the Add and Get methods. +// It does not test the eviction policy of the cache. +func TestAppSpecificScoreCache(t *testing.T) { + logger := zerolog.Nop() + + cache := internal.NewAppSpecificScoreCache(10, logger, metrics.NewNoopCollector()) + require.NotNil(t, cache, "failed to create AppSpecificScoreCache") + + peerID := unittest.PeerIdFixture(t) + score := 5.0 + updateTime := time.Now() + + err := cache.Add(peerID, score, updateTime) + require.Nil(t, err, "failed to add score to cache") + + // retrieve score from cache + retrievedScore, lastUpdated, found := cache.Get(peerID) + require.True(t, found, "failed to find score in cache") + require.Equal(t, score, retrievedScore, "retrieved score does not match expected") + require.Equal(t, updateTime, lastUpdated, "retrieved update time does not match expected") + + // test cache update + newScore := 10.0 + err = cache.Add(peerID, newScore, updateTime.Add(time.Minute)) + require.Nil(t, err, "Failed to update score in cache") + + // retrieve updated score + updatedScore, updatedTime, found := cache.Get(peerID) + require.True(t, found, "failed to find updated score in cache") + require.Equal(t, newScore, updatedScore, "updated score does not match expected") + require.Equal(t, updateTime.Add(time.Minute), updatedTime, "updated time does not match expected") +} From 6b5b5971141d145a9d83c4ed6256e512108c1f14 Mon Sep 17 00:00:00 2001 From: Yahya Hassanzadeh Date: Wed, 15 Nov 2023 14:43:15 -0800 Subject: [PATCH 05/67] adds TestAppSpecificScoreCache_Concurrent_Add_Get_Update --- .../internal/appSpecificScoreCache_test.go | 96 +++++++++++++++++++ 1 file changed, 96 insertions(+) diff --git a/network/p2p/scoring/internal/appSpecificScoreCache_test.go b/network/p2p/scoring/internal/appSpecificScoreCache_test.go index 29ded5d2798..62b812b13c3 100644 --- a/network/p2p/scoring/internal/appSpecificScoreCache_test.go +++ b/network/p2p/scoring/internal/appSpecificScoreCache_test.go @@ -1,6 +1,7 @@ package internal_test import ( + "sync" "testing" "time" @@ -45,3 +46,98 @@ func TestAppSpecificScoreCache(t *testing.T) { require.Equal(t, newScore, updatedScore, "updated score does not match expected") require.Equal(t, updateTime.Add(time.Minute), updatedTime, "updated time does not match expected") } + +// TestAppSpecificScoreCache_Concurrent_Add_Get_Update tests the concurrent functionality of AppSpecificScoreCache; +// specifically, it tests the Add and Get methods under concurrent access. +func TestAppSpecificScoreCache_Concurrent_Add_Get_Update(t *testing.T) { + logger := zerolog.Nop() + + cache := internal.NewAppSpecificScoreCache(10, logger, metrics.NewNoopCollector()) + require.NotNil(t, cache, "failed to create AppSpecificScoreCache") + + peerId1 := unittest.PeerIdFixture(t) + score1 := 5.0 + lastUpdated1 := time.Now() + + peerId2 := unittest.PeerIdFixture(t) + score2 := 10.0 + lastUpdated2 := time.Now().Add(time.Minute) + + wg := sync.WaitGroup{} + wg.Add(2) + go func() { + defer wg.Done() + err := cache.Add(peerId1, score1, lastUpdated1) + require.Nil(t, err, "failed to add score1 to cache") + }() + + go func() { + defer wg.Done() + err := cache.Add(peerId2, score2, lastUpdated2) + require.Nil(t, err, "failed to add score2 to cache") + }() + + unittest.RequireReturnsBefore(t, wg.Wait, 100*time.Millisecond, "failed to add scores to cache") + + // retrieve scores concurrently + wg.Add(2) + go func() { + defer wg.Done() + retrievedScore, lastUpdated, found := cache.Get(peerId1) + require.True(t, found, "failed to find score1 in cache") + require.Equal(t, score1, retrievedScore, "retrieved score1 does not match expected") + require.Equal(t, lastUpdated1, lastUpdated, "retrieved update time1 does not match expected") + }() + + go func() { + defer wg.Done() + retrievedScore, lastUpdated, found := cache.Get(peerId2) + require.True(t, found, "failed to find score2 in cache") + require.Equal(t, score2, retrievedScore, "retrieved score2 does not match expected") + require.Equal(t, lastUpdated2, lastUpdated, "retrieved update time2 does not match expected") + }() + + unittest.RequireReturnsBefore(t, wg.Wait, 100*time.Millisecond, "failed to retrieve scores from cache") + + // test cache update + newScore1 := 15.0 + newScore2 := 20.0 + lastUpdated1 = time.Now().Add(time.Minute) + lastUpdated2 = time.Now().Add(time.Minute) + + wg.Add(2) + go func() { + defer wg.Done() + err := cache.Add(peerId1, newScore1, lastUpdated1) + require.Nil(t, err, "failed to update score1 in cache") + }() + + go func() { + defer wg.Done() + err := cache.Add(peerId2, newScore2, lastUpdated2) + require.Nil(t, err, "failed to update score2 in cache") + }() + + unittest.RequireReturnsBefore(t, wg.Wait, 100*time.Millisecond, "failed to update scores in cache") + + // retrieve updated scores concurrently + wg.Add(2) + + go func() { + defer wg.Done() + updatedScore, updatedTime, found := cache.Get(peerId1) + require.True(t, found, "failed to find updated score1 in cache") + require.Equal(t, newScore1, updatedScore, "updated score1 does not match expected") + require.Equal(t, lastUpdated1, updatedTime, "updated time1 does not match expected") + }() + + go func() { + defer wg.Done() + updatedScore, updatedTime, found := cache.Get(peerId2) + require.True(t, found, "failed to find updated score2 in cache") + require.Equal(t, newScore2, updatedScore, "updated score2 does not match expected") + require.Equal(t, lastUpdated2, updatedTime, "updated time2 does not match expected") + }() + + unittest.RequireReturnsBefore(t, wg.Wait, 100*time.Millisecond, "failed to retrieve updated scores from cache") +} From 078d99d6f6690128599c700b4c2347c93b2189cd Mon Sep 17 00:00:00 2001 From: Yahya Hassanzadeh Date: Wed, 15 Nov 2023 14:49:13 -0800 Subject: [PATCH 06/67] adds TestAppSpecificScoreCache_Eviction --- .../internal/appSpecificScoreCache_test.go | 37 +++++++++++++++---- 1 file changed, 30 insertions(+), 7 deletions(-) diff --git a/network/p2p/scoring/internal/appSpecificScoreCache_test.go b/network/p2p/scoring/internal/appSpecificScoreCache_test.go index 62b812b13c3..415218f3c30 100644 --- a/network/p2p/scoring/internal/appSpecificScoreCache_test.go +++ b/network/p2p/scoring/internal/appSpecificScoreCache_test.go @@ -5,7 +5,6 @@ import ( "testing" "time" - "github.com/rs/zerolog" "github.com/stretchr/testify/require" "github.com/onflow/flow-go/module/metrics" @@ -17,9 +16,7 @@ import ( // specifically, it tests the Add and Get methods. // It does not test the eviction policy of the cache. func TestAppSpecificScoreCache(t *testing.T) { - logger := zerolog.Nop() - - cache := internal.NewAppSpecificScoreCache(10, logger, metrics.NewNoopCollector()) + cache := internal.NewAppSpecificScoreCache(10, unittest.Logger(), metrics.NewNoopCollector()) require.NotNil(t, cache, "failed to create AppSpecificScoreCache") peerID := unittest.PeerIdFixture(t) @@ -50,9 +47,7 @@ func TestAppSpecificScoreCache(t *testing.T) { // TestAppSpecificScoreCache_Concurrent_Add_Get_Update tests the concurrent functionality of AppSpecificScoreCache; // specifically, it tests the Add and Get methods under concurrent access. func TestAppSpecificScoreCache_Concurrent_Add_Get_Update(t *testing.T) { - logger := zerolog.Nop() - - cache := internal.NewAppSpecificScoreCache(10, logger, metrics.NewNoopCollector()) + cache := internal.NewAppSpecificScoreCache(10, unittest.Logger(), metrics.NewNoopCollector()) require.NotNil(t, cache, "failed to create AppSpecificScoreCache") peerId1 := unittest.PeerIdFixture(t) @@ -141,3 +136,31 @@ func TestAppSpecificScoreCache_Concurrent_Add_Get_Update(t *testing.T) { unittest.RequireReturnsBefore(t, wg.Wait, 100*time.Millisecond, "failed to retrieve updated scores from cache") } + +// TestAppSpecificScoreCache_Eviction tests the eviction policy of AppSpecificScoreCache; +// specifically, it tests that the cache evicts the least recently used record when the cache is full. +func TestAppSpecificScoreCache_Eviction(t *testing.T) { + cache := internal.NewAppSpecificScoreCache(10, unittest.Logger(), metrics.NewNoopCollector()) + require.NotNil(t, cache, "failed to create AppSpecificScoreCache") + + peerIds := unittest.PeerIdFixtures(t, 11) + scores := []float64{5.0, 10.0, 15.0, 20.0, 25.0, 30.0, 35.0, -1, -2, -3, -4} + require.Equal(t, len(peerIds), len(scores), "peer ids and scores must have the same length") + + // add scores to cache + for i := 0; i < len(peerIds); i++ { + err := cache.Add(peerIds[i], scores[i], time.Now()) + require.Nil(t, err, "failed to add score to cache") + } + + // retrieve scores from cache; the first score should have been evicted + for i := 1; i < len(peerIds); i++ { + retrievedScore, _, found := cache.Get(peerIds[i]) + require.True(t, found, "failed to find score in cache") + require.Equal(t, scores[i], retrievedScore, "retrieved score does not match expected") + } + + // the first score should not be in the cache + _, _, found := cache.Get(peerIds[0]) + require.False(t, found, "score should not be in cache") +} From 22ca2c24720cad541e2088c7505acaec0cdbd4b4 Mon Sep 17 00:00:00 2001 From: Yahya Hassanzadeh Date: Wed, 15 Nov 2023 14:58:57 -0800 Subject: [PATCH 07/67] moves computing app specific score to another function --- network/p2p/scoring/registry.go | 117 +++++++++++++++++++------------- 1 file changed, 69 insertions(+), 48 deletions(-) diff --git a/network/p2p/scoring/registry.go b/network/p2p/scoring/registry.go index 48167212fd5..d0bd0012a30 100644 --- a/network/p2p/scoring/registry.go +++ b/network/p2p/scoring/registry.go @@ -91,31 +91,38 @@ type GossipSubAppSpecificScoreRegistry struct { // initial application specific penalty record, used to initialize the penalty cache entry. init func() p2p.GossipSubSpamRecord validator p2p.SubscriptionValidator + scoreTTL time.Duration } // GossipSubAppSpecificScoreRegistryConfig is the configuration for the GossipSubAppSpecificScoreRegistry. // The configuration is used to initialize the registry. type GossipSubAppSpecificScoreRegistryConfig struct { - Logger zerolog.Logger + Logger zerolog.Logger `validate:"required"` // Validator is the subscription validator used to validate the subscriptions of peers, and determine if a peer is // authorized to subscribe to a topic. - Validator p2p.SubscriptionValidator + Validator p2p.SubscriptionValidator `validate:"required"` // Penalty encapsulates the penalty unit for each control message type misbehaviour. - Penalty GossipSubCtrlMsgPenaltyValue + Penalty GossipSubCtrlMsgPenaltyValue `validate:"required"` // IdProvider is the identity provider used to translate peer ids at the networking layer to Flow identifiers (if // an authorized peer is found). - IdProvider module.IdentityProvider + IdProvider module.IdentityProvider `validate:"required"` // Init is a factory function that returns a new GossipSubSpamRecord. It is used to initialize the spam record of // a peer when the peer is first observed by the local peer. - Init func() p2p.GossipSubSpamRecord + Init func() p2p.GossipSubSpamRecord `validate:"required"` + + // ScoreTTL is the time to live of the application specific score of a peer; the registry keeps a cached copy of the + // application specific score of a peer for this duration. When the duration expires, the application specific score + // of the peer is updated asynchronously. As long as the update is in progress, the cached copy of the application + // specific score of the peer is used even if it is expired. + ScoreTTL time.Duration `validate:"required"` // CacheFactory is a factory function that returns a new GossipSubSpamRecordCache. It is used to initialize the spamScoreCache. // The cache is used to store the application specific penalty of peers. - CacheFactory func() p2p.GossipSubSpamRecordCache + CacheFactory func() p2p.GossipSubSpamRecordCache `validate:"required"` } // NewGossipSubAppSpecificScoreRegistry returns a new GossipSubAppSpecificScoreRegistry. @@ -134,6 +141,7 @@ func NewGossipSubAppSpecificScoreRegistry(config *GossipSubAppSpecificScoreRegis init: config.Init, validator: config.Validator, idProvider: config.IdProvider, + scoreTTL: config.ScoreTTL, } builder := component.NewComponentManagerBuilder() @@ -163,56 +171,69 @@ var _ p2p.GossipSubInvCtrlMsgNotifConsumer = (*GossipSubAppSpecificScoreRegistry // AppSpecificScoreFunc returns the application specific penalty function that is called by the GossipSub protocol to determine the application specific penalty of a peer. func (r *GossipSubAppSpecificScoreRegistry) AppSpecificScoreFunc() func(peer.ID) float64 { return func(pid peer.ID) float64 { - appSpecificScore := float64(0) + return r.computeAppSpecificScore(pid) + } +} - lg := r.logger.With().Str("peer_id", p2plogging.PeerId(pid)).Logger() - // (1) spam penalty: the penalty is applied to the application specific penalty when a peer conducts a spamming misbehaviour. - spamRecord, err, spamRecordExists := r.spamScoreCache.Get(pid) - if err != nil { - // the error is considered fatal as it means the cache is not working properly. - // we should not continue with the execution as it may lead to routing attack vulnerability. - r.logger.Fatal().Str("peer_id", p2plogging.PeerId(pid)).Err(err).Msg("could not get application specific penalty for peer") - return appSpecificScore // unreachable, but added to avoid proceeding with the execution if log level is changed. - } +// computeAppSpecificScore computes the application specific score of a peer. +// The application specific score is computed based on the spam penalty, staking score, and subscription penalty. +// The spam penalty is the penalty applied to the application specific score when a peer conducts a spamming misbehaviour. +// The staking score is the reward/penalty applied to the application specific score when a peer is staked/unstaked. +// The subscription penalty is the penalty applied to the application specific score when a peer is subscribed to a topic that it is not allowed to subscribe to based on its role. +// Args: +// - pid: the peer ID of the peer in the GossipSub protocol. +// Returns: +// - float64: the application specific score of the peer. +func (r *GossipSubAppSpecificScoreRegistry) computeAppSpecificScore(pid peer.ID) float64 { + appSpecificScore := float64(0) - if spamRecordExists { - lg = lg.With().Float64("spam_penalty", spamRecord.Penalty).Logger() - appSpecificScore += spamRecord.Penalty - } + lg := r.logger.With().Str("peer_id", p2plogging.PeerId(pid)).Logger() + // (1) spam penalty: the penalty is applied to the application specific penalty when a peer conducts a spamming misbehaviour. + spamRecord, err, spamRecordExists := r.spamScoreCache.Get(pid) + if err != nil { + // the error is considered fatal as it means the cache is not working properly. + // we should not continue with the execution as it may lead to routing attack vulnerability. + r.logger.Fatal().Str("peer_id", p2plogging.PeerId(pid)).Err(err).Msg("could not get application specific penalty for peer") + return appSpecificScore // unreachable, but added to avoid proceeding with the execution if log level is changed. + } - // (2) staking score: for staked peers, a default positive reward is applied only if the peer has no penalty on spamming and subscription. - // for unknown peers a negative penalty is applied. - stakingScore, flowId, role := r.stakingScore(pid) - if stakingScore < 0 { - lg = lg.With().Float64("staking_penalty", stakingScore).Logger() - // staking penalty is applied right away. - appSpecificScore += stakingScore - } + if spamRecordExists { + lg = lg.With().Float64("spam_penalty", spamRecord.Penalty).Logger() + appSpecificScore += spamRecord.Penalty + } - if stakingScore >= 0 { - // (3) subscription penalty: the subscription penalty is applied to the application specific penalty when a - // peer is subscribed to a topic that it is not allowed to subscribe to based on its role. - // Note: subscription penalty can be considered only for staked peers, for non-staked peers, we cannot - // determine the role of the peer. - subscriptionPenalty := r.subscriptionPenalty(pid, flowId, role) - lg = lg.With().Float64("subscription_penalty", subscriptionPenalty).Logger() - if subscriptionPenalty < 0 { - appSpecificScore += subscriptionPenalty - } - } + // (2) staking score: for staked peers, a default positive reward is applied only if the peer has no penalty on spamming and subscription. + // for unknown peers a negative penalty is applied. + stakingScore, flowId, role := r.stakingScore(pid) + if stakingScore < 0 { + lg = lg.With().Float64("staking_penalty", stakingScore).Logger() + // staking penalty is applied right away. + appSpecificScore += stakingScore + } - // (4) staking reward: for staked peers, a default positive reward is applied only if the peer has no penalty on spamming and subscription. - if stakingScore > 0 && appSpecificScore == float64(0) { - lg = lg.With().Float64("staking_reward", stakingScore).Logger() - appSpecificScore += stakingScore + if stakingScore >= 0 { + // (3) subscription penalty: the subscription penalty is applied to the application specific penalty when a + // peer is subscribed to a topic that it is not allowed to subscribe to based on its role. + // Note: subscription penalty can be considered only for staked peers, for non-staked peers, we cannot + // determine the role of the peer. + subscriptionPenalty := r.subscriptionPenalty(pid, flowId, role) + lg = lg.With().Float64("subscription_penalty", subscriptionPenalty).Logger() + if subscriptionPenalty < 0 { + appSpecificScore += subscriptionPenalty } + } - lg.Trace(). - Float64("total_app_specific_score", appSpecificScore). - Msg("application specific penalty computed") - - return appSpecificScore + // (4) staking reward: for staked peers, a default positive reward is applied only if the peer has no penalty on spamming and subscription. + if stakingScore > 0 && appSpecificScore == float64(0) { + lg = lg.With().Float64("staking_reward", stakingScore).Logger() + appSpecificScore += stakingScore } + + lg.Trace(). + Float64("total_app_specific_score", appSpecificScore). + Msg("application specific penalty computed") + + return appSpecificScore } func (r *GossipSubAppSpecificScoreRegistry) stakingScore(pid peer.ID) (float64, flow.Identifier, flow.Role) { From 1cc3a4018371e604f35353cb9e5dd1b3a3de5c24 Mon Sep 17 00:00:00 2001 From: Yahya Hassanzadeh Date: Thu, 16 Nov 2023 10:21:48 -0800 Subject: [PATCH 08/67] adds app specific cache to registry --- network/p2p/scoring/registry.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/network/p2p/scoring/registry.go b/network/p2p/scoring/registry.go index d0bd0012a30..6eee3b32d5f 100644 --- a/network/p2p/scoring/registry.go +++ b/network/p2p/scoring/registry.go @@ -92,6 +92,8 @@ type GossipSubAppSpecificScoreRegistry struct { init func() p2p.GossipSubSpamRecord validator p2p.SubscriptionValidator scoreTTL time.Duration + // appScoreCache is a cache that stores the application specific score of peers. + appScoreCache p2p.GossipSubApplicationSpecificScoreCache } // GossipSubAppSpecificScoreRegistryConfig is the configuration for the GossipSubAppSpecificScoreRegistry. From 6df4465a88b1c7c6a07f23b612b3b0a406a9a683 Mon Sep 17 00:00:00 2001 From: Yahya Hassanzadeh Date: Thu, 16 Nov 2023 16:42:33 -0800 Subject: [PATCH 09/67] updates app-specific score function to return the cached score --- network/p2p/scoring/registry.go | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/network/p2p/scoring/registry.go b/network/p2p/scoring/registry.go index b79c5630a8a..35ce7fabce7 100644 --- a/network/p2p/scoring/registry.go +++ b/network/p2p/scoring/registry.go @@ -171,10 +171,33 @@ func NewGossipSubAppSpecificScoreRegistry(config *GossipSubAppSpecificScoreRegis var _ p2p.GossipSubInvCtrlMsgNotifConsumer = (*GossipSubAppSpecificScoreRegistry)(nil) -// AppSpecificScoreFunc returns the application specific penalty function that is called by the GossipSub protocol to determine the application specific penalty of a peer. +// AppSpecificScoreFunc returns the application specific score function that is called by the GossipSub protocol to determine the application specific score of a peer. +// The application specific score is part of the overall score of a peer, and is used to determine the peer's score based +// This function reads the application specific score of a peer from the cache, and if the penalty is not found in the cache, it computes it. +// If the score is not found in the cache, it is computed and added to the cache. +// Also if the score is expired, it is computed and added to the cache. +// Returns: +// - func(peer.ID) float64: the application specific score function. +// Implementation must be thread-safe. func (r *GossipSubAppSpecificScoreRegistry) AppSpecificScoreFunc() func(peer.ID) float64 { return func(pid peer.ID) float64 { - return r.computeAppSpecificScore(pid) + appSpecificScore, lastUpdated, ok := r.appScoreCache.Get(pid) + if !ok || time.Since(lastUpdated) > r.scoreTTL { + appSpecificScore = r.computeAppSpecificScore(pid) + err := r.appScoreCache.Add(pid, appSpecificScore, time.Now()) + if err != nil { + // the error is considered fatal as it means the cache is not working properly. + r.logger.Fatal(). + Str("remote_peer_id", p2plogging.PeerId(pid)). + Err(err). + Msg("could not add application specific penalty for peer") + } + r.logger.Trace(). + Str("remote_peer_id", p2plogging.PeerId(pid)). + Float64("app_specific_score", appSpecificScore). + Msg("application specific penalty computed and cache updated") + } + return appSpecificScore } } From b353bcd137fcd75b24002694531667d26516348a Mon Sep 17 00:00:00 2001 From: Yahya Hassanzadeh Date: Thu, 16 Nov 2023 18:26:28 -0800 Subject: [PATCH 10/67] adds worker pool --- network/p2p/scoring/registry.go | 111 ++++++++++++++++++++++----- network/p2p/scoring/registry_test.go | 9 ++- network/p2p/scoring/score_option.go | 2 +- 3 files changed, 99 insertions(+), 23 deletions(-) diff --git a/network/p2p/scoring/registry.go b/network/p2p/scoring/registry.go index 35ce7fabce7..1cd0b65286d 100644 --- a/network/p2p/scoring/registry.go +++ b/network/p2p/scoring/registry.go @@ -7,10 +7,13 @@ import ( "github.com/libp2p/go-libp2p/core/peer" "github.com/rs/zerolog" + "github.com/onflow/flow-go/engine/common/worker" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module" "github.com/onflow/flow-go/module/component" "github.com/onflow/flow-go/module/irrecoverable" + "github.com/onflow/flow-go/module/mempool/queue" + "github.com/onflow/flow-go/module/metrics" "github.com/onflow/flow-go/network/p2p" netcache "github.com/onflow/flow-go/network/p2p/cache" p2pmsg "github.com/onflow/flow-go/network/p2p/message" @@ -85,15 +88,28 @@ type GossipSubAppSpecificScoreRegistry struct { component.Component logger zerolog.Logger idProvider module.IdentityProvider + // spamScoreCache currently only holds the control message misbehaviour penalty (spam related penalty). spamScoreCache p2p.GossipSubSpamRecordCache - penalty GossipSubCtrlMsgPenaltyValue + + penalty GossipSubCtrlMsgPenaltyValue + // initial application specific penalty record, used to initialize the penalty cache entry. - init func() p2p.GossipSubSpamRecord + init func() p2p.GossipSubSpamRecord + validator p2p.SubscriptionValidator - scoreTTL time.Duration + + // scoreTTL is the time to live of the application specific score of a peer; the registry keeps a cached copy of the + // application specific score of a peer for this duration. When the duration expires, the application specific score + // of the peer is updated asynchronously. As long as the update is in progress, the cached copy of the application + // specific score of the peer is used even if it is expired. + scoreTTL time.Duration + // appScoreCache is a cache that stores the application specific score of peers. appScoreCache p2p.GossipSubApplicationSpecificScoreCache + + // appScoreUpdateWorkerPool is the worker pool for handling the application specific score update of peers in a non-blocking way. + appScoreUpdateWorkerPool *worker.Pool[peer.ID] } // GossipSubAppSpecificScoreRegistryConfig is the configuration for the GossipSubAppSpecificScoreRegistry. @@ -101,6 +117,12 @@ type GossipSubAppSpecificScoreRegistry struct { type GossipSubAppSpecificScoreRegistryConfig struct { Logger zerolog.Logger `validate:"required"` + // AppSpecificScoreNumWorkers is the number of workers in the worker pool for handling the application specific score update of peers in a non-blocking way. + AppSpecificScoreNumWorkers int `validate:"gt=0"` + + // AppSpecificScoreWorkerPoolSize is the size of the worker pool for handling the application specific score update of peers in a non-blocking way. + AppSpecificScoreWorkerPoolSize uint32 `validate:"gt=0"` + // Validator is the subscription validator used to validate the subscriptions of peers, and determine if a peer is // authorized to subscribe to a topic. Validator p2p.SubscriptionValidator `validate:"required"` @@ -122,9 +144,15 @@ type GossipSubAppSpecificScoreRegistryConfig struct { // specific score of the peer is used even if it is expired. ScoreTTL time.Duration `validate:"required"` - // CacheFactory is a factory function that returns a new GossipSubSpamRecordCache. It is used to initialize the spamScoreCache. + // SpamRecordCacheFactory is a factory function that returns a new GossipSubSpamRecordCache. It is used to initialize the spamScoreCache. // The cache is used to store the application specific penalty of peers. - CacheFactory func() p2p.GossipSubSpamRecordCache `validate:"required"` + SpamRecordCacheFactory func() p2p.GossipSubSpamRecordCache `validate:"required"` + + // AppScoreCacheFactory is a factory function that returns a new GossipSubApplicationSpecificScoreCache. It is used to initialize the appScoreCache. + // The cache is used to store the application specific score of peers. + AppScoreCacheFactory func() p2p.GossipSubApplicationSpecificScoreCache `validate:"required"` + + HeroCacheMetricsFactory metrics.HeroCacheMetricsFactory `validate:"required"` } // NewGossipSubAppSpecificScoreRegistry returns a new GossipSubAppSpecificScoreRegistry. @@ -136,9 +164,13 @@ type GossipSubAppSpecificScoreRegistryConfig struct { // // a new GossipSubAppSpecificScoreRegistry. func NewGossipSubAppSpecificScoreRegistry(config *GossipSubAppSpecificScoreRegistryConfig) *GossipSubAppSpecificScoreRegistry { + lg := config.Logger.With().Str("module", "app_score_registry").Logger() + store := queue.NewHeroStore(config.AppSpecificScoreWorkerPoolSize, lg.With().Str("component", "app_specific_score_update").Logger(), config.HeroCacheMetricsFactory()) + reg := &GossipSubAppSpecificScoreRegistry{ logger: config.Logger.With().Str("module", "app_score_registry").Logger(), - spamScoreCache: config.CacheFactory(), + spamScoreCache: config.SpamRecordCacheFactory(), + appScoreCache: config.AppScoreCacheFactory(), penalty: config.Penalty, init: config.Init, validator: config.Validator, @@ -146,6 +178,10 @@ func NewGossipSubAppSpecificScoreRegistry(config *GossipSubAppSpecificScoreRegis scoreTTL: config.ScoreTTL, } + reg.appScoreUpdateWorkerPool = worker.NewWorkerPoolBuilder[peer.ID](lg.With().Str("component", "app_specific_score_update_worker_pool").Logger(), + store, + reg.processAppSpecificScoreUpdateWork).Build() + builder := component.NewComponentManagerBuilder() builder.AddWorker(func(ctx irrecoverable.SignalerContext, ready component.ReadyFunc) { reg.logger.Info().Msg("starting subscription validator") @@ -164,6 +200,11 @@ func NewGossipSubAppSpecificScoreRegistry(config *GossipSubAppSpecificScoreRegis <-reg.validator.Done() reg.logger.Info().Msg("subscription validator stopped") }) + + for i := 0; i < config.AppSpecificScoreNumWorkers; i++ { + builder.AddWorker(reg.appScoreUpdateWorkerPool.WorkerLogic()) + } + reg.Component = builder.Build() return reg @@ -181,23 +222,35 @@ var _ p2p.GossipSubInvCtrlMsgNotifConsumer = (*GossipSubAppSpecificScoreRegistry // Implementation must be thread-safe. func (r *GossipSubAppSpecificScoreRegistry) AppSpecificScoreFunc() func(peer.ID) float64 { return func(pid peer.ID) float64 { + lg := r.logger.With().Str("remote_peer_id", p2plogging.PeerId(pid)).Logger() appSpecificScore, lastUpdated, ok := r.appScoreCache.Get(pid) - if !ok || time.Since(lastUpdated) > r.scoreTTL { - appSpecificScore = r.computeAppSpecificScore(pid) - err := r.appScoreCache.Add(pid, appSpecificScore, time.Now()) - if err != nil { - // the error is considered fatal as it means the cache is not working properly. - r.logger.Fatal(). - Str("remote_peer_id", p2plogging.PeerId(pid)). - Err(err). - Msg("could not add application specific penalty for peer") - } + switch { + case !ok: + // record not found in the cache, or expired; submit a worker to update it. + submitted := r.appScoreUpdateWorkerPool.Submit(pid) + lg.Trace(). + Bool("worker_submitted", submitted). + Msg("application specific score not found in cache, submitting worker to update it") + + return 0 // in the mean time, return 0, which is a neutral score. + case time.Since(lastUpdated) > r.scoreTTL: + // record found in the cache, but expired; submit a worker to update it. + submitted := r.appScoreUpdateWorkerPool.Submit(pid) + lg.Trace(). + Bool("worker_submitted", submitted). + Float64("app_specific_score", appSpecificScore). + Dur("score_ttl", r.scoreTTL). + Msg("application specific score expired, submitting worker to update it") + + return appSpecificScore // in the mean time, return the expired score. + default: + // record found in the cache, check if it is expired. r.logger.Trace(). - Str("remote_peer_id", p2plogging.PeerId(pid)). Float64("app_specific_score", appSpecificScore). - Msg("application specific penalty computed and cache updated") + Msg("application specific score found in cache") + + return appSpecificScore } - return appSpecificScore } } @@ -262,6 +315,26 @@ func (r *GossipSubAppSpecificScoreRegistry) computeAppSpecificScore(pid peer.ID) return appSpecificScore } +// processMisbehaviorReport is the worker function that is called by the worker pool to update the application specific score of a peer. +// The function is called in a non-blocking way, and the worker pool is used to limit the number of concurrent executions of the function. +// Args: +// - pid: the peer ID of the peer in the GossipSub protocol. +// Returns: +// - error: an error if the update failed; any returned error is an irrecoverable error and indicates a bug or misconfiguration. +func (r *GossipSubAppSpecificScoreRegistry) processAppSpecificScoreUpdateWork(p peer.ID) error { + appSpecificScore := r.computeAppSpecificScore(p) + err := r.appScoreCache.Add(p, appSpecificScore, time.Now()) + if err != nil { + // the error is considered fatal as it means the cache is not working properly. + return fmt.Errorf("could not add application specific score %f for peer to cache: %w", appSpecificScore, err) + } + r.logger.Trace(). + Str("remote_peer_id", p2plogging.PeerId(p)). + Float64("app_specific_score", appSpecificScore). + Msg("application specific penalty computed and cache updated") + return nil +} + func (r *GossipSubAppSpecificScoreRegistry) stakingScore(pid peer.ID) (float64, flow.Identifier, flow.Role) { lg := r.logger.With().Str("peer_id", p2plogging.PeerId(pid)).Logger() diff --git a/network/p2p/scoring/registry_test.go b/network/p2p/scoring/registry_test.go index e89196601fc..841fb6a78be 100644 --- a/network/p2p/scoring/registry_test.go +++ b/network/p2p/scoring/registry_test.go @@ -450,7 +450,9 @@ func withUnknownIdentity(peer peer.ID) func(cfg *scoring.GossipSubAppSpecificSco // It is used for testing purposes and causes the given peer id to be penalized for subscribing to invalid topics. func withInvalidSubscriptions(peer peer.ID) func(cfg *scoring.GossipSubAppSpecificScoreRegistryConfig) { return func(cfg *scoring.GossipSubAppSpecificScoreRegistryConfig) { - cfg.Validator.(*mockp2p.SubscriptionValidator).On("CheckSubscribedToAllowedTopics", peer, testifymock.Anything).Return(fmt.Errorf("invalid subscriptions")).Maybe() + cfg.Validator.(*mockp2p.SubscriptionValidator).On("CheckSubscribedToAllowedTopics", + peer, + testifymock.Anything).Return(fmt.Errorf("invalid subscriptions")).Maybe() } } @@ -462,7 +464,8 @@ func withInitFunction(initFunction func() p2p.GossipSubSpamRecord) func(cfg *sco // newGossipSubAppSpecificScoreRegistry returns a new instance of GossipSubAppSpecificScoreRegistry with default values // for the testing purposes. -func newGossipSubAppSpecificScoreRegistry(t *testing.T, opts ...func(*scoring.GossipSubAppSpecificScoreRegistryConfig)) (*scoring.GossipSubAppSpecificScoreRegistry, *netcache.GossipSubSpamRecordCache) { +func newGossipSubAppSpecificScoreRegistry(t *testing.T, opts ...func(*scoring.GossipSubAppSpecificScoreRegistryConfig)) (*scoring.GossipSubAppSpecificScoreRegistry, + *netcache.GossipSubSpamRecordCache) { cache := netcache.NewGossipSubSpamRecordCache(100, unittest.Logger(), metrics.NewNoopCollector(), scoring.DefaultDecayFunction()) cfg := &scoring.GossipSubAppSpecificScoreRegistryConfig{ Logger: unittest.Logger(), @@ -470,7 +473,7 @@ func newGossipSubAppSpecificScoreRegistry(t *testing.T, opts ...func(*scoring.Go Penalty: penaltyValueFixtures(), IdProvider: mock.NewIdentityProvider(t), Validator: mockp2p.NewSubscriptionValidator(t), - CacheFactory: func() p2p.GossipSubSpamRecordCache { + SpamRecordCacheFactory: func() p2p.GossipSubSpamRecordCache { return cache }, } diff --git a/network/p2p/scoring/score_option.go b/network/p2p/scoring/score_option.go index b3585d108fb..3ea4e0f624a 100644 --- a/network/p2p/scoring/score_option.go +++ b/network/p2p/scoring/score_option.go @@ -398,7 +398,7 @@ func NewScoreOption(cfg *ScoreOptionConfig, provider p2p.SubscriptionProvider) * Validator: validator, Init: InitAppScoreRecordState, IdProvider: cfg.provider, - CacheFactory: func() p2p.GossipSubSpamRecordCache { + SpamRecordCacheFactory: func() p2p.GossipSubSpamRecordCache { return netcache.NewGossipSubSpamRecordCache(cfg.cacheSize, cfg.logger, cfg.cacheMetrics, DefaultDecayFunction()) }, }) From 3009bf0f3bf5e8133f18ddb9f6664e94ac55ae59 Mon Sep 17 00:00:00 2001 From: Yahya Hassanzadeh Date: Mon, 20 Nov 2023 10:05:23 -0800 Subject: [PATCH 11/67] adds herocache metrics collector --- module/metrics/herocache.go | 12 ++++++++++++ module/metrics/labels.go | 1 + network/p2p/scoring/registry.go | 4 +++- 3 files changed, 16 insertions(+), 1 deletion(-) diff --git a/module/metrics/herocache.go b/module/metrics/herocache.go index 59ddb0f2f36..e9d4017fb3b 100644 --- a/module/metrics/herocache.go +++ b/module/metrics/herocache.go @@ -193,6 +193,18 @@ func GossipSubRPCInspectorClusterPrefixedCacheMetricFactory(f HeroCacheMetricsFa return f(namespaceNetwork, r) } +// GossipSubAppSpecificScoreUpdateQueueMetricFactory is the factory method for creating a new HeroCacheCollector for the +// app-specific score update queue of the GossipSub peer scoring module. The app-specific score update queue is used to +// queue the update requests for the app-specific score of peers. The update requests are queued in a worker pool and +// processed asynchronously. +// Args: +// - f: the HeroCacheMetricsFactory to create the collector +// Returns: +// - a HeroCacheMetrics for the app-specific score update queue. +func GossipSubAppSpecificScoreUpdateQueueMetricFactory(f HeroCacheMetricsFactory) module.HeroCacheMetrics { + return f(namespaceNetwork, ResourceNetworkingAppSpecificScoreUpdateQueue) +} + func CollectionNodeTransactionsCacheMetrics(registrar prometheus.Registerer, epoch uint64) *HeroCacheCollector { return NewHeroCacheCollector(namespaceCollection, fmt.Sprintf("%s_%d", ResourceTransaction, epoch), registrar) } diff --git a/module/metrics/labels.go b/module/metrics/labels.go index e58610bec35..d19f43e3787 100644 --- a/module/metrics/labels.go +++ b/module/metrics/labels.go @@ -95,6 +95,7 @@ const ( ResourceNetworkingApplicationLayerSpamRecordCache = "application_layer_spam_record_cache" ResourceNetworkingApplicationLayerSpamReportQueue = "application_layer_spam_report_queue" ResourceNetworkingRpcClusterPrefixReceivedCache = "rpc_cluster_prefixed_received_cache" + ResourceNetworkingAppSpecificScoreUpdateQueue = "gossipsub_app_specific_score_update_queue" ResourceNetworkingDisallowListCache = "disallow_list_cache" ResourceNetworkingRPCSentTrackerCache = "gossipsub_rpc_sent_tracker_cache" ResourceNetworkingRPCSentTrackerQueue = "gossipsub_rpc_sent_tracker_queue" diff --git a/network/p2p/scoring/registry.go b/network/p2p/scoring/registry.go index 1cd0b65286d..20295186df9 100644 --- a/network/p2p/scoring/registry.go +++ b/network/p2p/scoring/registry.go @@ -165,7 +165,9 @@ type GossipSubAppSpecificScoreRegistryConfig struct { // a new GossipSubAppSpecificScoreRegistry. func NewGossipSubAppSpecificScoreRegistry(config *GossipSubAppSpecificScoreRegistryConfig) *GossipSubAppSpecificScoreRegistry { lg := config.Logger.With().Str("module", "app_score_registry").Logger() - store := queue.NewHeroStore(config.AppSpecificScoreWorkerPoolSize, lg.With().Str("component", "app_specific_score_update").Logger(), config.HeroCacheMetricsFactory()) + store := queue.NewHeroStore(config.AppSpecificScoreWorkerPoolSize, + lg.With().Str("component", "app_specific_score_update").Logger(), + metrics.GossipSubAppSpecificScoreUpdateQueueMetricFactory(config.HeroCacheMetricsFactory)) reg := &GossipSubAppSpecificScoreRegistry{ logger: config.Logger.With().Str("module", "app_score_registry").Logger(), From b3e842bac658e9b33192a0a072ed94fc2bd5a3cc Mon Sep 17 00:00:00 2001 From: Yahya Hassanzadeh Date: Mon, 20 Nov 2023 10:11:46 -0800 Subject: [PATCH 12/67] adds validation logic for app specific score registry --- network/p2p/p2pbuilder/gossipsub/gossipSubBuilder.go | 5 ++++- network/p2p/scoring/registry.go | 9 ++++++++- network/p2p/scoring/registry_test.go | 6 +++++- network/p2p/scoring/score_option.go | 11 ++++++++--- 4 files changed, 25 insertions(+), 6 deletions(-) diff --git a/network/p2p/p2pbuilder/gossipsub/gossipSubBuilder.go b/network/p2p/p2pbuilder/gossipsub/gossipSubBuilder.go index 251e578185a..91143caf0f9 100644 --- a/network/p2p/p2pbuilder/gossipsub/gossipSubBuilder.go +++ b/network/p2p/p2pbuilder/gossipsub/gossipSubBuilder.go @@ -347,7 +347,10 @@ func (g *Builder) Build(ctx irrecoverable.SignalerContext) (p2p.PubSubAdapter, e } g.scoreOptionConfig.SetRegisterNotificationConsumerFunc(inspectorSuite.AddInvalidControlMessageConsumer) - scoreOpt = scoring.NewScoreOption(g.scoreOptionConfig, subscriptionProvider) + scoreOpt, err = scoring.NewScoreOption(g.scoreOptionConfig, subscriptionProvider) + if err != nil { + return nil, fmt.Errorf("could not create gossipsub score option: %w", err) + } gossipSubConfigs.WithScoreOption(scoreOpt) if g.gossipSubScoreTracerInterval > 0 { diff --git a/network/p2p/scoring/registry.go b/network/p2p/scoring/registry.go index 20295186df9..07b30f21e4a 100644 --- a/network/p2p/scoring/registry.go +++ b/network/p2p/scoring/registry.go @@ -4,6 +4,7 @@ import ( "fmt" "time" + "github.com/go-playground/validator/v10" "github.com/libp2p/go-libp2p/core/peer" "github.com/rs/zerolog" @@ -163,7 +164,13 @@ type GossipSubAppSpecificScoreRegistryConfig struct { // Returns: // // a new GossipSubAppSpecificScoreRegistry. -func NewGossipSubAppSpecificScoreRegistry(config *GossipSubAppSpecificScoreRegistryConfig) *GossipSubAppSpecificScoreRegistry { +// +// error: if the configuration is invalid, an error is returned; any returned error is an irrecoverable error and indicates a bug or misconfiguration. +func NewGossipSubAppSpecificScoreRegistry(config *GossipSubAppSpecificScoreRegistryConfig) (*GossipSubAppSpecificScoreRegistry, error) { + if err := validator.New().Struct(config); err != nil { + return nil, fmt.Errorf("invalid config: %w", err) + } + lg := config.Logger.With().Str("module", "app_score_registry").Logger() store := queue.NewHeroStore(config.AppSpecificScoreWorkerPoolSize, lg.With().Str("component", "app_specific_score_update").Logger(), diff --git a/network/p2p/scoring/registry_test.go b/network/p2p/scoring/registry_test.go index 841fb6a78be..cb493cb2c72 100644 --- a/network/p2p/scoring/registry_test.go +++ b/network/p2p/scoring/registry_test.go @@ -480,7 +480,11 @@ func newGossipSubAppSpecificScoreRegistry(t *testing.T, opts ...func(*scoring.Go for _, opt := range opts { opt(cfg) } - return scoring.NewGossipSubAppSpecificScoreRegistry(cfg), cache + + reg, err := scoring.NewGossipSubAppSpecificScoreRegistry(cfg) + require.NoError(t, err, "failed to create GossipSubAppSpecificScoreRegistry") + + return reg, cache } // penaltyValueFixtures returns a set of penalty values for testing purposes. diff --git a/network/p2p/scoring/score_option.go b/network/p2p/scoring/score_option.go index 3ea4e0f624a..629ee91f8d8 100644 --- a/network/p2p/scoring/score_option.go +++ b/network/p2p/scoring/score_option.go @@ -382,7 +382,7 @@ func (c *ScoreOptionConfig) OverrideDecayInterval(interval time.Duration) { } // NewScoreOption creates a new penalty option with the given configuration. -func NewScoreOption(cfg *ScoreOptionConfig, provider p2p.SubscriptionProvider) *ScoreOption { +func NewScoreOption(cfg *ScoreOptionConfig, provider p2p.SubscriptionProvider) (*ScoreOption, error) { throttledSampler := logging.BurstSampler(MaxDebugLogs, time.Second) logger := cfg.logger.With(). Str("module", "pubsub_score_option"). @@ -392,7 +392,7 @@ func NewScoreOption(cfg *ScoreOptionConfig, provider p2p.SubscriptionProvider) * DebugSampler: throttledSampler, }) validator := NewSubscriptionValidator(cfg.logger, provider) - scoreRegistry := NewGossipSubAppSpecificScoreRegistry(&GossipSubAppSpecificScoreRegistryConfig{ + scoreRegistry, err := NewGossipSubAppSpecificScoreRegistry(&GossipSubAppSpecificScoreRegistryConfig{ Logger: logger, Penalty: DefaultGossipSubCtrlMsgPenaltyValue(), Validator: validator, @@ -402,6 +402,11 @@ func NewScoreOption(cfg *ScoreOptionConfig, provider p2p.SubscriptionProvider) * return netcache.NewGossipSubSpamRecordCache(cfg.cacheSize, cfg.logger, cfg.cacheMetrics, DefaultDecayFunction()) }, }) + + if err != nil { + return nil, fmt.Errorf("failed to create gossipsub app specific score registry: %w", err) + } + s := &ScoreOption{ logger: logger, validator: validator, @@ -459,7 +464,7 @@ func NewScoreOption(cfg *ScoreOptionConfig, provider p2p.SubscriptionProvider) * s.logger.Info().Msg("score registry stopped") }).Build() - return s + return s, nil } func (s *ScoreOption) BuildFlowPubSubScoreOption() (*pubsub.PeerScoreParams, *pubsub.PeerScoreThresholds) { From e19ba80e1517bdee9f5651a6bd7295a4a72ce9a5 Mon Sep 17 00:00:00 2001 From: Yahya Hassanzadeh Date: Mon, 20 Nov 2023 12:35:57 -0800 Subject: [PATCH 13/67] wip organizing parameters --- .../node_builder/access_node_builder.go | 2 +- cmd/observer/node_builder/observer_builder.go | 2 +- config/default-config.yml | 164 ++++++++++-------- follower/follower_builder.go | 2 +- module/metrics/herocache.go | 7 + module/metrics/labels.go | 1 + network/internal/p2pfixtures/fixtures.go | 2 +- network/netconf/config.go | 2 +- network/netconf/flags.go | 4 +- network/p2p/cache/gossipsub_spam_records.go | 9 +- .../control_message_validation_inspector.go | 16 +- ...ntrol_message_validation_inspector_test.go | 12 +- .../p2pbuilder/gossipsub/gossipSubBuilder.go | 4 +- network/p2p/p2pbuilder/libp2pNodeBuilder.go | 4 +- network/p2p/p2pconf/gossipsub.go | 70 ++++++-- .../p2p/p2pconf/gossipsub_rpc_inspectors.go | 56 +++--- network/p2p/scoring/registry.go | 40 +++-- network/p2p/scoring/registry_test.go | 2 +- network/p2p/scoring/score_option.go | 88 +++++----- network/p2p/scoring/subscription_provider.go | 4 +- .../p2p/scoring/subscription_provider_test.go | 8 +- .../scoring/subscription_validator_test.go | 2 +- network/p2p/test/fixtures.go | 2 +- 23 files changed, 283 insertions(+), 220 deletions(-) diff --git a/cmd/access/node_builder/access_node_builder.go b/cmd/access/node_builder/access_node_builder.go index d625a5ab663..d3fb48582a7 100644 --- a/cmd/access/node_builder/access_node_builder.go +++ b/cmd/access/node_builder/access_node_builder.go @@ -1563,7 +1563,7 @@ func (builder *FlowAccessNodeBuilder) initPublicLibp2pNode(networkKey crypto.Pri UpdateInterval: builder.FlowConfig.NetworkConfig.PeerUpdateInterval, ConnectorFactory: connection.DefaultLibp2pBackoffConnectorFactory(), }, - &builder.FlowConfig.NetworkConfig.GossipSubConfig.SubscriptionProviderConfig, + &builder.FlowConfig.NetworkConfig.GossipSubConfig.SubscriptionProvider, &p2p.DisallowListCacheConfig{ MaxSize: builder.FlowConfig.NetworkConfig.DisallowListNotificationCacheSize, Metrics: metrics.DisallowListCacheMetricsFactory(builder.HeroCacheMetricsFactory(), network.PublicNetwork), diff --git a/cmd/observer/node_builder/observer_builder.go b/cmd/observer/node_builder/observer_builder.go index 84e28f532f1..a42e0eb82a4 100644 --- a/cmd/observer/node_builder/observer_builder.go +++ b/cmd/observer/node_builder/observer_builder.go @@ -722,7 +722,7 @@ func (builder *ObserverServiceBuilder) initPublicLibp2pNode(networkKey crypto.Pr &builder.FlowConfig.NetworkConfig.ResourceManager, &builder.FlowConfig.NetworkConfig.GossipSubConfig.GossipSubRPCInspectorsConfig, p2pconfig.PeerManagerDisableConfig(), // disable peer manager for observer node. - &builder.FlowConfig.NetworkConfig.GossipSubConfig.SubscriptionProviderConfig, + &builder.FlowConfig.NetworkConfig.GossipSubConfig.SubscriptionProvider, &p2p.DisallowListCacheConfig{ MaxSize: builder.FlowConfig.NetworkConfig.DisallowListNotificationCacheSize, Metrics: metrics.DisallowListCacheMetricsFactory(builder.HeroCacheMetricsFactory(), network.PublicNetwork), diff --git a/config/default-config.yml b/config/default-config.yml index d4f244286fa..b4f7dff7a53 100644 --- a/config/default-config.yml +++ b/config/default-config.yml @@ -117,80 +117,96 @@ network-config: # The time to wait before start pruning connections libp2p-grace-period: 1m # Gossipsub config - # The default interval at which the mesh tracer logs the mesh topology. This is used for debugging and forensics purposes. - # Note that we purposefully choose this logging interval high enough to avoid spamming the logs. Moreover, the - # mesh updates will be logged individually and separately. The logging interval is only used to log the mesh - # topology as a whole specially when there are no updates to the mesh topology for a long time. - gossipsub-local-mesh-logging-interval: 1m - # The default interval at which the gossipsub score tracer logs the peer scores. This is used for debugging and forensics purposes. - # Note that we purposefully choose this logging interval high enough to avoid spamming the logs. - gossipsub-score-tracer-interval: 1m - # The default RPC sent tracker cache size. The RPC sent tracker is used to track RPC control messages sent from the local node. - # Note: this cache size must be large enough to keep a history of sent messages in a reasonable time window of past history. - gossipsub-rpc-sent-tracker-cache-size: 1_000_000 - # Cache size of the rpc sent tracker queue used for async tracking. - gossipsub-rpc-sent-tracker-queue-cache-size: 100_000 - # Number of workers for rpc sent tracker worker pool. - gossipsub-rpc-sent-tracker-workers: 5 - # Peer scoring is the default value for enabling peer scoring - gossipsub-peer-scoring-enabled: true - # The interval for updating the list of subscribed peers to all topics in gossipsub. This is used to keep track of subscriptions - # violations and penalize peers accordingly. Recommended value is in the order of a few minutes to avoid contentions; as the operation - # reads all topics and all peers subscribed to each topic. - gossipsub-subscription-provider-update-interval: 10m - # The size of cache for keeping the list of all peers subscribed to each topic (same as the local node). This cache is the local node's - # view of the network and is used to detect subscription violations and penalize peers accordingly. Recommended to be big enough to - # keep the entire network's size. Otherwise, the local node's view of the network will be incomplete due to cache eviction. - # Recommended size is 10x the number of peers in the network. - gossipsub-subscription-provider-cache-size: 10000 - - # Gossipsub rpc inspectors configs - # The size of the queue for notifications about invalid RPC messages - gossipsub-rpc-inspector-notification-cache-size: 10_000 - # RPC control message validation inspector configs - # Rpc validation inspector number of pool workers - gossipsub-rpc-validation-inspector-workers: 5 - # Max number of ihave messages in a sample to be inspected. If the number of ihave messages exceeds this configured value - # the control message ihaves will be truncated to the max sample size. This sample is randomly selected. - gossipsub-rpc-ihave-max-sample-size: 1000 - # Max number of ihave message ids in a sample to be inspected per ihave. Each ihave message includes a list of message ids - # each. If the size of the message ids list for a single ihave message exceeds the configured max message id sample size the list of message ids will be truncated. - gossipsub-rpc-ihave-max-message-id-sample-size: 1000 - # Max number of control messages in a sample to be inspected when inspecting GRAFT and PRUNE message types. If the total number of control messages (GRAFT or PRUNE) - # exceeds this max sample size then the respective message will be truncated before being processed. - gossipsub-rpc-graft-and-prune-message-max-sample-size: 1000 - # Max number of iwant messages in a sample to be inspected. If the total number of iWant control messages - # exceeds this max sample size then the respective message will be truncated before being processed. - gossipsub-rpc-iwant-max-sample-size: 1000 - # Max number of iwant message ids in a sample to be inspected per iwant. Each iwant message includes a list of message ids - # each, if the size of this list exceeds the configured max message id sample size the list of message ids will be truncated. - gossipsub-rpc-iwant-max-message-id-sample-size: 1000 - # The allowed threshold of iWant messages received without a corresponding tracked iHave message that was sent. If the cache miss threshold is exceeded an - # invalid control message notification is disseminated and the sender will be penalized. - gossipsub-rpc-iwant-cache-miss-threshold: .5 - # The iWants size at which message id cache misses will be checked. - gossipsub-rpc-iwant-cache-miss-check-size: 1000 - # The max allowed duplicate message IDs in a single iWant control message. If the duplicate message threshold is exceeded an invalid control message - # notification is disseminated and the sender will be penalized. - gossipsub-rpc-iwant-duplicate-message-id-threshold: .15 - # The size of the queue used by worker pool for the control message validation inspector - gossipsub-rpc-validation-inspector-queue-cache-size: 100 - # Cluster prefixed control message validation configs - # The size of the cache used to track the amount of cluster prefixed topics received by peers - gossipsub-cluster-prefix-tracker-cache-size: 100 - # The decay val used for the geometric decay of cache counters used to keep track of cluster prefixed topics received by peers - gossipsub-cluster-prefix-tracker-cache-decay: 0.99 - # The upper bound on the amount of cluster prefixed control messages that will be processed - gossipsub-rpc-cluster-prefixed-hard-threshold: 100 - # The max sample size used for RPC message validation. If the total number of RPC messages exceeds this value a sample will be taken but messages will not be truncated - gossipsub-rpc-message-max-sample-size: 1000 - # The threshold at which an error will be returned if the number of invalid RPC messages exceeds this value - gossipsub-rpc-message-error-threshold: 500 - # RPC metrics observer inspector configs - # The number of metrics inspector pool workers - gossipsub-rpc-metrics-inspector-workers: 1 - # The size of the queue used by worker pool for the control message metrics inspector - gossipsub-rpc-metrics-inspector-cache-size: 100 + gossipsub: + rpc-inspector: + # The size of the queue for notifications about invalid RPC messages + notification-cache-size: 10_000 + validation: # RPC control message validation inspector configs + # Rpc validation inspector number of pool workers + workers: 5 + # The size of the queue used by worker pool for the control message validation inspector + queue-size: 100 + # The max sample size used for RPC message validation. If the total number of RPC messages exceeds this value a sample will be taken but messages will not be truncated + message-max-sample-size: 1000 + # Max number of control messages in a sample to be inspected when inspecting GRAFT and PRUNE message types. If the total number of control messages (GRAFT or PRUNE) + # exceeds this max sample size then the respective message will be truncated before being processed. + graft-and-prune-message-max-sample-size: 1000 + # The threshold at which an error will be returned if the number of invalid RPC messages exceeds this value + error-threshold: 500 + ihave: # Max number of ihave messages in a sample to be inspected. If the number of ihave messages exceeds this configured value + # the control message ihaves will be truncated to the max sample size. This sample is randomly selected. + max-sample-size: 1000 + # Max number of ihave message ids in a sample to be inspected per ihave. Each ihave message includes a list of message ids + # each. If the size of the message ids list for a single ihave message exceeds the configured max message id sample size the list of message ids will be truncated. + max-message-id-sample-size: 1000 + iwant: + # Max number of iwant messages in a sample to be inspected. If the total number of iWant control messages + # exceeds this max sample size then the respective message will be truncated before being processed. + max-sample-size: 1000 + # Max number of iwant message ids in a sample to be inspected per iwant. Each iwant message includes a list of message ids + # each, if the size of this list exceeds the configured max message id sample size the list of message ids will be truncated. + max-message-id-sample-size: 1000 + # The allowed threshold of iWant messages received without a corresponding tracked iHave message that was sent. If the cache miss threshold is exceeded an + # invalid control message notification is disseminated and the sender will be penalized. + cache-miss-threshold: .5 + # The iWants size at which message id cache misses will be checked. + cache-miss-check-size: 1000 + # The max allowed duplicate message IDs in a single iWant control message. If the duplicate message threshold is exceeded an invalid control message + # notification is disseminated and the sender will be penalized. + duplicate-message-id-threshold: .15 + cluster-prefixed-message: + # Cluster prefixed control message validation configs + # The size of the cache used to track the amount of cluster prefixed topics received by peers + tracker-cache-size: 100 + # The decay val used for the geometric decay of cache counters used to keep track of cluster prefixed topics received by peers + tracker-cache-decay: 0.99 + # The upper bound on the amount of cluster prefixed control messages that will be processed + hard-threshold: 100 + metrics: + # RPC metrics observer inspector configs + # The number of metrics inspector pool workers + workers: 1 + # The size of the queue used by worker pool for the control message metrics inspector + cache-size: 100 + rpc-tracer: + # The default interval at which the mesh tracer logs the mesh topology. This is used for debugging and forensics purposes. + # Note that we purposefully choose this logging interval high enough to avoid spamming the logs. Moreover, the + # mesh updates will be logged individually and separately. The logging interval is only used to log the mesh + # topology as a whole specially when there are no updates to the mesh topology for a long time. + local-mesh-logging-interval: 1m + # The default interval at which the gossipsub score tracer logs the peer scores. This is used for debugging and forensics purposes. + # Note that we purposefully choose this logging interval high enough to avoid spamming the logs. + score-tracer-interval: 1m + # The default RPC sent tracker cache size. The RPC sent tracker is used to track RPC control messages sent from the local node. + # Note: this cache size must be large enough to keep a history of sent messages in a reasonable time window of past history. + rpc-sent-tracker-cache-size: 1_000_000 + # Cache size of the rpc sent tracker queue used for async tracking. + rpc-sent-tracker-queue-cache-size: 100_000 + # Number of workers for rpc sent tracker worker pool. + rpc-sent-tracker-workers: 5 + # Peer scoring is the default value for enabling peer scoring + peer-scoring-enabled: true + scoring-parameters: + app-specific-score: + # number of workers that asynchronously update the app specific score requests when they are expired. + score-update-worker-num: 5 + # size of the queue used by the worker pool for the app specific score update requests. The queue is used to buffer the app specific score update requests + # before they are processed by the worker pool. The queue size must be larger than total number of peers in the network. + # The queue is deduplicated based on the peer ids ensuring that there is only one app specific score update request per peer in the queue. + score-update-request-queue-size: 10_000 + # score ttl is the time to live for the app specific score. Once the score is expired; a new request will be sent to the app specific score provider to update the score. + # until the score is updated, the previous score will be used. + score-ttl: 1m + subscription-provider: + # The interval for updating the list of subscribed peers to all topics in gossipsub. This is used to keep track of subscriptions + # violations and penalize peers accordingly. Recommended value is in the order of a few minutes to avoid contentions; as the operation + # reads all topics and all peers subscribed to each topic. + update-interval: 10m + # The size of cache for keeping the list of all peers subscribed to each topic (same as the local node). This cache is the local node's + # view of the network and is used to detect subscription violations and penalize peers accordingly. Recommended to be big enough to + # keep the entire network's size. Otherwise, the local node's view of the network will be incomplete due to cache eviction. + # Recommended size is 10x the number of peers in the network. + cache-size: 10000 # Application layer spam prevention alsp-spam-record-cache-size: 1000 alsp-spam-report-queue-size: 10_000 diff --git a/follower/follower_builder.go b/follower/follower_builder.go index cf7b19c7214..58aa4eb5b74 100644 --- a/follower/follower_builder.go +++ b/follower/follower_builder.go @@ -600,7 +600,7 @@ func (builder *FollowerServiceBuilder) initPublicLibp2pNode(networkKey crypto.Pr &builder.FlowConfig.NetworkConfig.ResourceManager, &builder.FlowConfig.NetworkConfig.GossipSubConfig.GossipSubRPCInspectorsConfig, p2pconfig.PeerManagerDisableConfig(), // disable peer manager for follower - &builder.FlowConfig.NetworkConfig.GossipSubConfig.SubscriptionProviderConfig, + &builder.FlowConfig.NetworkConfig.GossipSubConfig.SubscriptionProvider, &p2p.DisallowListCacheConfig{ MaxSize: builder.FlowConfig.NetworkConfig.DisallowListNotificationCacheSize, Metrics: metrics.DisallowListCacheMetricsFactory(builder.HeroCacheMetricsFactory(), network.PublicNetwork), diff --git a/module/metrics/herocache.go b/module/metrics/herocache.go index e9d4017fb3b..56713683b6f 100644 --- a/module/metrics/herocache.go +++ b/module/metrics/herocache.go @@ -91,6 +91,13 @@ func DisallowListCacheMetricsFactory(f HeroCacheMetricsFactory, networkingType n return f(namespaceNetwork, r) } +// GossipSubSpamRecordCacheMetricsFactory is the factory method for creating a new HeroCacheCollector for the spam record cache. +// The spam record cache is used to keep track of peers that are spamming the network and the reasons for it. +// Currently, the spam record cache is only used for the private network. +func GossipSubSpamRecordCacheMetricsFactory(f HeroCacheMetricsFactory) module.HeroCacheMetrics { + return f(namespaceNetwork, ResourceNetworkingGossipSubSpamRecordCache) +} + func NetworkDnsTxtCacheMetricsFactory(registrar prometheus.Registerer) *HeroCacheCollector { return NewHeroCacheCollector(namespaceNetwork, ResourceNetworkingDnsTxtCache, registrar) } diff --git a/module/metrics/labels.go b/module/metrics/labels.go index d19f43e3787..bc1f0f3f853 100644 --- a/module/metrics/labels.go +++ b/module/metrics/labels.go @@ -96,6 +96,7 @@ const ( ResourceNetworkingApplicationLayerSpamReportQueue = "application_layer_spam_report_queue" ResourceNetworkingRpcClusterPrefixReceivedCache = "rpc_cluster_prefixed_received_cache" ResourceNetworkingAppSpecificScoreUpdateQueue = "gossipsub_app_specific_score_update_queue" + ResourceNetworkingGossipSubSpamRecordCache = "gossipsub_spam_record_cache" ResourceNetworkingDisallowListCache = "disallow_list_cache" ResourceNetworkingRPCSentTrackerCache = "gossipsub_rpc_sent_tracker_cache" ResourceNetworkingRPCSentTrackerQueue = "gossipsub_rpc_sent_tracker_queue" diff --git a/network/internal/p2pfixtures/fixtures.go b/network/internal/p2pfixtures/fixtures.go index 551870f2e4b..a26320c177e 100644 --- a/network/internal/p2pfixtures/fixtures.go +++ b/network/internal/p2pfixtures/fixtures.go @@ -126,7 +126,7 @@ func CreateNode(t *testing.T, networkKey crypto.PrivateKey, sporkID flow.Identif &defaultFlowConfig.NetworkConfig.ResourceManager, &defaultFlowConfig.NetworkConfig.GossipSubRPCInspectorsConfig, p2pconfig.PeerManagerDisableConfig(), - &defaultFlowConfig.NetworkConfig.GossipSubConfig.SubscriptionProviderConfig, + &defaultFlowConfig.NetworkConfig.GossipSubConfig.SubscriptionProvider, &p2p.DisallowListCacheConfig{ MaxSize: uint32(1000), Metrics: metrics.NewNoopCollector(), diff --git a/network/netconf/config.go b/network/netconf/config.go index b9df868d281..34d0da6d2b1 100644 --- a/network/netconf/config.go +++ b/network/netconf/config.go @@ -12,7 +12,7 @@ type Config struct { ResourceManager p2pconf.ResourceManagerConfig `mapstructure:"libp2p-resource-manager"` ConnectionManagerConfig `mapstructure:",squash"` // GossipSubConfig core gossipsub configuration. - p2pconf.GossipSubConfig `mapstructure:",squash"` + GossipSub p2pconf.GossipSubConfig `mapstructure:"gossipsub"` AlspConfig `mapstructure:",squash"` // NetworkConnectionPruning determines whether connections to nodes diff --git a/network/netconf/flags.go b/network/netconf/flags.go index f40d755fbfc..c37668e43e0 100644 --- a/network/netconf/flags.go +++ b/network/netconf/flags.go @@ -302,11 +302,11 @@ func InitializeNetworkFlags(flags *pflag.FlagSet, config *Config) { flags.Int(rpcMessageMaxSampleSize, config.GossipSubConfig.GossipSubRPCInspectorsConfig.GossipSubRPCValidationInspectorConfigs.RpcMessageMaxSampleSize, "the max sample size used for RPC message validation. If the total number of RPC messages exceeds this value a sample will be taken but messages will not be truncated") flags.Int(rpcMessageErrorThreshold, config.GossipSubConfig.GossipSubRPCInspectorsConfig.GossipSubRPCValidationInspectorConfigs.RpcMessageErrorThreshold, "the threshold at which an error will be returned if the number of invalid RPC messages exceeds this value") flags.Duration( - gossipSubSubscriptionProviderUpdateInterval, config.GossipSubConfig.SubscriptionProviderConfig.SubscriptionUpdateInterval, + gossipSubSubscriptionProviderUpdateInterval, config.GossipSubConfig.SubscriptionProvider.SubscriptionUpdateInterval, "interval for updating the list of subscribed topics for all peers in the gossipsub, recommended value is a few minutes") flags.Uint32( gossipSubSubscriptionProviderCacheSize, - config.GossipSubConfig.SubscriptionProviderConfig.CacheSize, + config.GossipSubConfig.SubscriptionProvider.CacheSize, "size of the cache that keeps the list of topics each peer has subscribed to, recommended size is 10x the number of authorized nodes") } diff --git a/network/p2p/cache/gossipsub_spam_records.go b/network/p2p/cache/gossipsub_spam_records.go index 265c2befbb7..2c34af6df92 100644 --- a/network/p2p/cache/gossipsub_spam_records.go +++ b/network/p2p/cache/gossipsub_spam_records.go @@ -8,10 +8,10 @@ import ( "github.com/rs/zerolog" "github.com/onflow/flow-go/model/flow" - "github.com/onflow/flow-go/module" herocache "github.com/onflow/flow-go/module/mempool/herocache/backdata" "github.com/onflow/flow-go/module/mempool/herocache/backdata/heropool" "github.com/onflow/flow-go/module/mempool/stdmap" + "github.com/onflow/flow-go/module/metrics" "github.com/onflow/flow-go/network/p2p" "github.com/onflow/flow-go/network/p2p/p2plogging" ) @@ -60,14 +60,17 @@ type PreprocessorFunc func(record p2p.GossipSubSpamRecord, lastUpdated time.Time // Returns: // // *GossipSubSpamRecordCache: the newly created cache with a HeroCache-based backend. -func NewGossipSubSpamRecordCache(sizeLimit uint32, logger zerolog.Logger, collector module.HeroCacheMetrics, prFns ...PreprocessorFunc) *GossipSubSpamRecordCache { +func NewGossipSubSpamRecordCache(sizeLimit uint32, + logger zerolog.Logger, + hcMetricsFactor metrics.HeroCacheMetricsFactory, + prFns ...PreprocessorFunc) *GossipSubSpamRecordCache { backData := herocache.NewCache(sizeLimit, herocache.DefaultOversizeFactor, // we should not evict any record from the cache, // eviction will open the node to spam attacks by malicious peers to erase their application specific penalty. heropool.NoEjection, logger.With().Str("mempool", "gossipsub-app-Penalty-cache").Logger(), - collector) + metrics.GossipSubSpamRecordCacheMetricsFactory(hcMetricsFactor)) return &GossipSubSpamRecordCache{ c: stdmap.NewBackend(stdmap.WithBackData(backData)), preprocessFns: prFns, diff --git a/network/p2p/inspector/validation/control_message_validation_inspector.go b/network/p2p/inspector/validation/control_message_validation_inspector.go index 66cb4339f9d..1e573508f06 100644 --- a/network/p2p/inspector/validation/control_message_validation_inspector.go +++ b/network/p2p/inspector/validation/control_message_validation_inspector.go @@ -41,7 +41,7 @@ type ControlMsgValidationInspector struct { sporkID flow.Identifier metrics module.GossipSubRpcValidationInspectorMetrics // config control message validation configurations. - config *p2pconf.GossipSubRPCValidationInspectorConfigs + config *p2pconf.RpcValidationInspector // distributor used to disseminate invalid RPC message notifications. distributor p2p.GossipSubInspectorNotifDistributor // workerPool queue that stores *InspectRPCRequest that will be processed by component workers. @@ -69,7 +69,7 @@ type InspectorParams struct { // SporkID the current spork ID. SporkID flow.Identifier `validate:"required"` // Config inspector configuration. - Config *p2pconf.GossipSubRPCValidationInspectorConfigs `validate:"required"` + Config *p2pconf.RpcValidationInspector `validate:"required"` // Distributor gossipsub inspector notification distributor. Distributor p2p.GossipSubInspectorNotifDistributor `validate:"required"` // HeroCacheMetricsFactory the metrics factory. @@ -116,10 +116,10 @@ func NewControlMsgValidationInspector(params *InspectorParams) (*ControlMsgValid return nil, fmt.Errorf("failed to create cluster prefix topics received tracker") } - if params.Config.RpcMessageMaxSampleSize < params.Config.RpcMessageErrorThreshold { + if params.Config.MessageMaxSampleSize < params.Config.MessageErrorThreshold { return nil, fmt.Errorf("rpc message max sample size must be greater than or equal to rpc message error threshold, got %d and %d respectively", - params.Config.RpcMessageMaxSampleSize, - params.Config.RpcMessageErrorThreshold) + params.Config.MessageMaxSampleSize, + params.Config.MessageErrorThreshold) } c := &ControlMsgValidationInspector{ @@ -135,7 +135,7 @@ func NewControlMsgValidationInspector(params *InspectorParams) (*ControlMsgValid topicOracle: params.TopicOracle, } - store := queue.NewHeroStore(params.Config.CacheSize, params.Logger, inspectMsgQueueCacheCollector) + store := queue.NewHeroStore(params.Config.QueueSize, params.Logger, inspectMsgQueueCacheCollector) pool := worker.NewWorkerPoolBuilder[*InspectRPCRequest](lg, store, c.processInspectRPCReq).Build() @@ -458,7 +458,7 @@ func (c *ControlMsgValidationInspector) inspectRpcPublishMessages(from peer.ID, if totalMessages == 0 { return nil, 0 } - sampleSize := c.config.RpcMessageMaxSampleSize + sampleSize := c.config.MessageMaxSampleSize if sampleSize > totalMessages { sampleSize = totalMessages } @@ -498,7 +498,7 @@ func (c *ControlMsgValidationInspector) inspectRpcPublishMessages(from peer.ID, } // return an error when we exceed the error threshold - if errs != nil && errs.Len() > c.config.RpcMessageErrorThreshold { + if errs != nil && errs.Len() > c.config.MessageErrorThreshold { return NewInvalidRpcPublishMessagesErr(errs.ErrorOrNil(), errs.Len()), uint64(errs.Len()) } diff --git a/network/p2p/inspector/validation/control_message_validation_inspector_test.go b/network/p2p/inspector/validation/control_message_validation_inspector_test.go index f076a2868f7..ae25192a92a 100644 --- a/network/p2p/inspector/validation/control_message_validation_inspector_test.go +++ b/network/p2p/inspector/validation/control_message_validation_inspector_test.go @@ -32,7 +32,7 @@ import ( type ControlMsgValidationInspectorSuite struct { suite.Suite sporkID flow.Identifier - config *p2pconf.GossipSubRPCValidationInspectorConfigs + config *p2pconf.RpcValidationInspector distributor *mockp2p.GossipSubInspectorNotificationDistributor params *validation.InspectorParams rpcTracker *mockp2p.RpcControlTracking @@ -587,12 +587,12 @@ func (suite *ControlMsgValidationInspectorSuite) TestControlMessageValidationIns time.Sleep(time.Second) }) - suite.T().Run("inspectRpcPublishMessages should disseminate invalid control message notification when invalid pubsub messages count greater than configured RpcMessageErrorThreshold", + suite.T().Run("inspectRpcPublishMessages should disseminate invalid control message notification when invalid pubsub messages count greater than configured MessageErrorThreshold", func(t *testing.T) { suite.SetupTest() defer suite.StopInspector() // 5 invalid pubsub messages will force notification dissemination - suite.config.RpcMessageErrorThreshold = 4 + suite.config.MessageErrorThreshold = 4 // create unknown topic unknownTopic := channels.Topic(fmt.Sprintf("%s/%s", unittest.IdentifierFixture(), suite.sporkID)).String() // create malformed topic @@ -632,7 +632,7 @@ func (suite *ControlMsgValidationInspectorSuite) TestControlMessageValidationIns suite.SetupTest() defer suite.StopInspector() // 5 invalid pubsub messages will force notification dissemination - suite.config.RpcMessageErrorThreshold = 4 + suite.config.MessageErrorThreshold = 4 pubsubMsgs := unittest.GossipSubMessageFixtures(5, fmt.Sprintf("%s/%s", channels.TestNetworkChannel, suite.sporkID)) from := unittest.PeerIdFixture(t) rpc := unittest.P2PRPCFixture(unittest.WithPubsubMessages(pubsubMsgs...)) @@ -648,7 +648,7 @@ func (suite *ControlMsgValidationInspectorSuite) TestControlMessageValidationIns suite.SetupTest() defer suite.StopInspector() // 5 invalid pubsub messages will force notification dissemination - suite.config.RpcMessageErrorThreshold = 4 + suite.config.MessageErrorThreshold = 4 pubsubMsgs := unittest.GossipSubMessageFixtures(10, "") rpc := unittest.P2PRPCFixture(unittest.WithPubsubMessages(pubsubMsgs...)) @@ -691,7 +691,7 @@ func (suite *ControlMsgValidationInspectorSuite) TestControlMessageValidationIns from := unittest.PeerIdFixture(t) topic := fmt.Sprintf("%s/%s", channels.TestNetworkChannel, suite.sporkID) suite.topicProviderOracle.UpdateTopics([]string{topic}) - // default RpcMessageErrorThreshold is 500, 501 messages should trigger a notification + // default MessageErrorThreshold is 500, 501 messages should trigger a notification pubsubMsgs := unittest.GossipSubMessageFixtures(501, topic, unittest.WithFrom(from)) suite.idProvider.On("ByPeerID", from).Return(nil, false).Times(501) rpc := unittest.P2PRPCFixture(unittest.WithPubsubMessages(pubsubMsgs...)) diff --git a/network/p2p/p2pbuilder/gossipsub/gossipSubBuilder.go b/network/p2p/p2pbuilder/gossipsub/gossipSubBuilder.go index 91143caf0f9..fa0c79f3417 100644 --- a/network/p2p/p2pbuilder/gossipsub/gossipSubBuilder.go +++ b/network/p2p/p2pbuilder/gossipsub/gossipSubBuilder.go @@ -200,7 +200,7 @@ func NewGossipSubBuilder( idProvider: idProvider, gossipSubFactory: defaultGossipSubFactory(), gossipSubConfigFunc: defaultGossipSubAdapterConfig(), - scoreOptionConfig: scoring.NewScoreOptionConfig(lg, idProvider), + scoreOptionConfig: scoring.NewScoreOptionConfig(lg, metricsCfg.HeroCacheFactory, idProvider), rpcInspectorConfig: rpcInspectorConfig, rpcInspectorSuiteFactory: defaultInspectorSuite(rpcTracker), subscriptionProviderParam: subscriptionProviderPrams, @@ -257,7 +257,7 @@ func defaultInspectorSuite(rpcTracker p2p.RpcControlTracking) p2p.GossipSubRpcIn }...) notificationDistributor := distributor.DefaultGossipSubInspectorNotificationDistributor( logger, []queue.HeroStoreConfigOption{ - queue.WithHeroStoreSizeLimit(inspectorCfg.GossipSubRPCInspectorNotificationCacheSize), + queue.WithHeroStoreSizeLimit(inspectorCfg.NotificationCacheSize), queue.WithHeroStoreCollector(metrics.RpcInspectorNotificationQueueMetricFactory(heroCacheMetricsFactory, networkType))}...) params := &validation.InspectorParams{ diff --git a/network/p2p/p2pbuilder/libp2pNodeBuilder.go b/network/p2p/p2pbuilder/libp2pNodeBuilder.go index 45baaa0a6d5..9c010696d7e 100644 --- a/network/p2p/p2pbuilder/libp2pNodeBuilder.go +++ b/network/p2p/p2pbuilder/libp2pNodeBuilder.go @@ -468,7 +468,7 @@ func DefaultNodeBuilder( sporkId, idProvider, rCfg, - rpcInspectorCfg, peerManagerCfg, &gossipCfg.SubscriptionProviderConfig, + rpcInspectorCfg, peerManagerCfg, &gossipCfg.SubscriptionProvider, disallowListCacheCfg, meshTracer, uniCfg). @@ -477,7 +477,7 @@ func DefaultNodeBuilder( SetConnectionGater(connGater). SetCreateNode(DefaultCreateNodeFunc) - if gossipCfg.PeerScoring { + if gossipCfg.ScoringParameters { // In production, we never override the default scoring config. builder.EnableGossipSubScoringWithOverride(p2p.PeerScoringConfigNoOverride) } diff --git a/network/p2p/p2pconf/gossipsub.go b/network/p2p/p2pconf/gossipsub.go index b8392c3268c..d019f10b2f5 100644 --- a/network/p2p/p2pconf/gossipsub.go +++ b/network/p2p/p2pconf/gossipsub.go @@ -2,6 +2,8 @@ package p2pconf import ( "time" + + "github.com/onflow/flow-go/network/p2p/scoring" ) // ResourceManagerConfig returns the resource manager configuration for the libp2p node. @@ -54,44 +56,80 @@ type ResourceManagerOverrideLimit struct { Memory int `validate:"gte=0" mapstructure:"memory-bytes"` } +// GossipSubConfig keys. +const ( + RpcInspectorKey = "rpc-inspector" + RpcTracerKey = "rpc-tracer" + PeerScoringKey = "peer-scoring-enabled" + ScoreParamsKey = "scoring-parameters" + SubscriptionProviderKey = "subscription-provider" +) + +var GossipSubConfigKeys = []string{RpcInspectorKey, RpcTracerKey, PeerScoringKey, ScoreParamsKey, SubscriptionProviderKey} + // GossipSubConfig is the configuration for the GossipSub pubsub implementation. type GossipSubConfig struct { // GossipSubRPCInspectorsConfig configuration for all gossipsub RPC control message inspectors. - GossipSubRPCInspectorsConfig `mapstructure:",squash"` + RpcInspector GossipSubRPCInspectorsConfig `mapstructure:"rpc-inspector"` + + // GossipSubTracerParameters is the configuration for the gossipsub tracer. GossipSub tracer is used to trace the local mesh events and peer scores. + RpcTracer GossipSubTracerParameters `mapstructure:"rpc-tracer"` - // GossipSubTracerConfig is the configuration for the gossipsub tracer. GossipSub tracer is used to trace the local mesh events and peer scores. - GossipSubTracerConfig `mapstructure:",squash"` + // ScoringParameters is whether to enable GossipSub peer scoring. + PeerScoringSwitch bool `mapstructure:"peer-scoring-enabled"` - // PeerScoring is whether to enable GossipSub peer scoring. - PeerScoring bool `mapstructure:"gossipsub-peer-scoring-enabled"` + SubscriptionProvider SubscriptionProviderParameters `mapstructure:"subscription-provider"` - SubscriptionProviderConfig SubscriptionProviderParameters `mapstructure:",squash"` + ScoringParameters scoring.Parameters `mapstructure:"scoring-parameters"` } +// SubscriptionProviderParameters keys. +const ( + UpdateIntervalKey = "update-interval" + CacheSizeKey = "cache-size" +) + +var SubscriptionProviderParametersKeys = []string{UpdateIntervalKey, CacheSizeKey} + type SubscriptionProviderParameters struct { - // SubscriptionUpdateInterval is the interval for updating the list of topics the node have subscribed to; as well as the list of all + // UpdateInterval is the interval for updating the list of topics the node have subscribed to; as well as the list of all // peers subscribed to each of those topics. This is used to penalize peers that have an invalid subscription based on their role. - SubscriptionUpdateInterval time.Duration `validate:"gt=0s" mapstructure:"gossipsub-subscription-provider-update-interval"` + UpdateInterval time.Duration `validate:"gt=0s" mapstructure:"update-interval"` // CacheSize is the size of the cache that keeps the list of peers subscribed to each topic as the local node. // This is the local view of the current node towards the subscription status of other nodes in the system. // The cache must be large enough to accommodate the maximum number of nodes in the system, otherwise the view of the local node will be incomplete // due to cache eviction. - CacheSize uint32 `validate:"gt=0" mapstructure:"gossipsub-subscription-provider-cache-size"` + CacheSize uint32 `validate:"gt=0" mapstructure:"cache-size"` } -// GossipSubTracerConfig is the config for the gossipsub tracer. GossipSub tracer is used to trace the local mesh events and peer scores. -type GossipSubTracerConfig struct { +// GossipSubTracerParameters keys. +const ( + LocalMeshLogIntervalKey = "local-mesh-logging-interval" + ScoreTracerIntervalKey = "score-tracer-interval" + RPCSentTrackerCacheSizeKey = "rpc-sent-tracker-cache-size" + RPCSentTrackerQueueCacheSizeKey = "rpc-sent-tracker-queue-cache-size" + RPCSentTrackerNumOfWorkersKey = "rpc-sent-tracker-workers" +) + +var TracerParametersKeys = []string{LocalMeshLogIntervalKey, + ScoreTracerIntervalKey, + RPCSentTrackerCacheSizeKey, + RPCSentTrackerQueueCacheSizeKey, + RPCSentTrackerNumOfWorkersKey} + +// GossipSubTracerParameters is the config for the gossipsub tracer. GossipSub tracer is used to trace the local mesh events and peer scores. +type GossipSubTracerParameters struct { // LocalMeshLogInterval is the interval at which the local mesh is logged. - LocalMeshLogInterval time.Duration `validate:"gt=0s" mapstructure:"gossipsub-local-mesh-logging-interval"` + LocalMeshLogInterval time.Duration `validate:"gt=0s" mapstructure:"local-mesh-logging-interval"` // ScoreTracerInterval is the interval at which the score tracer logs the peer scores. - ScoreTracerInterval time.Duration `validate:"gt=0s" mapstructure:"gossipsub-score-tracer-interval"` + ScoreTracerInterval time.Duration `validate:"gt=0s" mapstructure:"score-tracer-interval"` // RPCSentTrackerCacheSize cache size of the rpc sent tracker used by the gossipsub mesh tracer. - RPCSentTrackerCacheSize uint32 `validate:"gt=0" mapstructure:"gossipsub-rpc-sent-tracker-cache-size"` + RPCSentTrackerCacheSize uint32 `validate:"gt=0" mapstructure:"rpc-sent-tracker-cache-size"` // RPCSentTrackerQueueCacheSize cache size of the rpc sent tracker queue used for async tracking. - RPCSentTrackerQueueCacheSize uint32 `validate:"gt=0" mapstructure:"gossipsub-rpc-sent-tracker-queue-cache-size"` + RPCSentTrackerQueueCacheSize uint32 `validate:"gt=0" mapstructure:"rpc-sent-tracker-queue-cache-size"` // RpcSentTrackerNumOfWorkers number of workers for rpc sent tracker worker pool. - RpcSentTrackerNumOfWorkers int `validate:"gt=0" mapstructure:"gossipsub-rpc-sent-tracker-workers"` + RpcSentTrackerNumOfWorkers int `validate:"gt=0" mapstructure:"rpc-sent-tracker-workers"` } // ResourceScope is the scope of the resource, e.g., system, transient, protocol, peer, peer-protocol. diff --git a/network/p2p/p2pconf/gossipsub_rpc_inspectors.go b/network/p2p/p2pconf/gossipsub_rpc_inspectors.go index 3d3cea79b21..48f189c406c 100644 --- a/network/p2p/p2pconf/gossipsub_rpc_inspectors.go +++ b/network/p2p/p2pconf/gossipsub_rpc_inspectors.go @@ -2,58 +2,58 @@ package p2pconf // GossipSubRPCInspectorsConfig encompasses configuration related to gossipsub RPC message inspectors. type GossipSubRPCInspectorsConfig struct { - // GossipSubRPCValidationInspectorConfigs control message validation inspector validation configuration and limits. - GossipSubRPCValidationInspectorConfigs `mapstructure:",squash"` + // RpcValidationInspector control message validation inspector validation configuration and limits. + RpcValidation RpcValidationInspector `mapstructure:"validation"` // GossipSubRPCMetricsInspectorConfigs control message metrics inspector configuration. - GossipSubRPCMetricsInspectorConfigs `mapstructure:",squash"` - // GossipSubRPCInspectorNotificationCacheSize size of the queue for notifications about invalid RPC messages. - GossipSubRPCInspectorNotificationCacheSize uint32 `mapstructure:"gossipsub-rpc-inspector-notification-cache-size"` + RpcMetrics GossipSubRPCMetricsInspectorConfigs `mapstructure:"metrics"` + // NotificationCacheSize size of the queue for notifications about invalid RPC messages. + NotificationCacheSize uint32 `mapstructure:"notification-cache-size"` } -// GossipSubRPCValidationInspectorConfigs validation limits used for gossipsub RPC control message inspection. -type GossipSubRPCValidationInspectorConfigs struct { - ClusterPrefixedMessageConfig `mapstructure:",squash"` - IWantRPCInspectionConfig `mapstructure:",squash"` - IHaveRPCInspectionConfig `mapstructure:",squash"` +// RpcValidationInspector validation limits used for gossipsub RPC control message inspection. +type RpcValidationInspector struct { + ClusterPrefixedMessage ClusterPrefixedMessageConfig `mapstructure:"cluster-prefixed-messages"` + IWant IWantRPCInspectionConfig `mapstructure:"iwant"` + IHave IHaveRPCInspectionConfig `mapstructure:"ihave"` // NumberOfWorkers number of worker pool workers. - NumberOfWorkers int `validate:"gte=1" mapstructure:"gossipsub-rpc-validation-inspector-workers"` - // CacheSize size of the queue used by worker pool for the control message validation inspector. - CacheSize uint32 `validate:"gte=100" mapstructure:"gossipsub-rpc-validation-inspector-queue-cache-size"` + NumberOfWorkers int `validate:"gte=1" mapstructure:"workers"` + // QueueSize size of the queue used by worker pool for the control message validation inspector. + QueueSize uint32 `validate:"gte=100" mapstructure:"queue-size"` // GraftPruneMessageMaxSampleSize the max sample size used for control message validation of GRAFT and PRUNE. If the total number of control messages (GRAFT or PRUNE) // exceeds this max sample size then the respective message will be truncated to this value before being processed. - GraftPruneMessageMaxSampleSize int `validate:"gte=1000" mapstructure:"gossipsub-rpc-graft-and-prune-message-max-sample-size"` + GraftPruneMessageMaxSampleSize int `validate:"gte=1000" mapstructure:"graft-and-prune-message-max-sample-size"` // RPCMessageMaxSampleSize the max sample size used for RPC message validation. If the total number of RPC messages exceeds this value a sample will be taken but messages will not be truncated. - RpcMessageMaxSampleSize int `validate:"gte=1000" mapstructure:"gossipsub-rpc-message-max-sample-size"` + MessageMaxSampleSize int `validate:"gte=1000" mapstructure:"message-max-sample-size"` // RPCMessageErrorThreshold the threshold at which an error will be returned if the number of invalid RPC messages exceeds this value. - RpcMessageErrorThreshold int `validate:"gte=500" mapstructure:"gossipsub-rpc-message-error-threshold"` + MessageErrorThreshold int `validate:"gte=500" mapstructure:"message-error-threshold"` } // IWantRPCInspectionConfig validation configuration for iWANT RPC control messages. type IWantRPCInspectionConfig struct { // MaxSampleSize max inspection sample size to use. If the total number of iWant control messages // exceeds this max sample size then the respective message will be truncated before being processed. - MaxSampleSize uint `validate:"gt=0" mapstructure:"gossipsub-rpc-iwant-max-sample-size"` + MaxSampleSize uint `validate:"gt=0" mapstructure:"max-sample-size"` // MaxMessageIDSampleSize max inspection sample size to use for iWant message ids. Each iWant message includes a list of message ids // each, if the size of this list exceeds the configured max message id sample size the list of message ids will be truncated. - MaxMessageIDSampleSize int `validate:"gte=1000" mapstructure:"gossipsub-rpc-iwant-max-message-id-sample-size"` + MaxMessageIDSampleSize int `validate:"gte=1000" mapstructure:"max-message-id-sample-size"` // CacheMissThreshold the threshold of missing corresponding iHave messages for iWant messages received before an invalid control message notification is disseminated. // If the cache miss threshold is exceeded an invalid control message notification is disseminated and the sender will be penalized. - CacheMissThreshold float64 `validate:"gt=0" mapstructure:"gossipsub-rpc-iwant-cache-miss-threshold"` + CacheMissThreshold float64 `validate:"gt=0" mapstructure:"cache-miss-threshold"` // CacheMissCheckSize the iWants size at which message id cache misses will be checked. - CacheMissCheckSize int `validate:"gt=0" mapstructure:"gossipsub-rpc-iwant-cache-miss-check-size"` + CacheMissCheckSize int `validate:"gt=0" mapstructure:"cache-miss-check-size"` // DuplicateMsgIDThreshold maximum allowed duplicate message IDs in a single iWant control message. // If the duplicate message threshold is exceeded an invalid control message notification is disseminated and the sender will be penalized. - DuplicateMsgIDThreshold float64 `validate:"gt=0" mapstructure:"gossipsub-rpc-iwant-duplicate-message-id-threshold"` + DuplicateMsgIDThreshold float64 `validate:"gt=0" mapstructure:"duplicate-message-id-threshold"` } // IHaveRPCInspectionConfig validation configuration for iHave RPC control messages. type IHaveRPCInspectionConfig struct { // MaxSampleSize max inspection sample size to use. If the number of ihave messages exceeds this configured value // the control message ihaves will be truncated to the max sample size. This sample is randomly selected. - MaxSampleSize int `validate:"gte=1000" mapstructure:"gossipsub-rpc-ihave-max-sample-size"` + MaxSampleSize int `validate:"gte=1000" mapstructure:"max-sample-size"` // MaxMessageIDSampleSize max inspection sample size to use for iHave message ids. Each ihave message includes a list of message ids // each, if the size of this list exceeds the configured max message id sample size the list of message ids will be truncated. - MaxMessageIDSampleSize int `validate:"gte=1000" mapstructure:"gossipsub-rpc-ihave-max-message-id-sample-size"` + MaxMessageIDSampleSize int `validate:"gte=1000" mapstructure:"max-message-id-sample-size"` } // ClusterPrefixedMessageConfig configuration values for cluster prefixed control message validation. @@ -63,17 +63,17 @@ type ClusterPrefixedMessageConfig struct { // when the cluster ID's provider is set asynchronously. It also allows processing of some stale messages that may be sent by nodes // that fall behind in the protocol. After the amount of cluster prefixed control messages processed exceeds this threshold the node // will be pushed to the edge of the network mesh. - ClusterPrefixHardThreshold float64 `validate:"gte=0" mapstructure:"gossipsub-rpc-cluster-prefixed-hard-threshold"` + ClusterPrefixHardThreshold float64 `validate:"gte=0" mapstructure:"hard-threshold"` // ClusterPrefixedControlMsgsReceivedCacheSize size of the cache used to track the amount of cluster prefixed topics received by peers. - ClusterPrefixedControlMsgsReceivedCacheSize uint32 `validate:"gt=0" mapstructure:"gossipsub-cluster-prefix-tracker-cache-size"` + ClusterPrefixedControlMsgsReceivedCacheSize uint32 `validate:"gt=0" mapstructure:"tracker-cache-size"` // ClusterPrefixedControlMsgsReceivedCacheDecay decay val used for the geometric decay of cache counters used to keep track of cluster prefixed topics received by peers. - ClusterPrefixedControlMsgsReceivedCacheDecay float64 `validate:"gt=0" mapstructure:"gossipsub-cluster-prefix-tracker-cache-decay"` + ClusterPrefixedControlMsgsReceivedCacheDecay float64 `validate:"gt=0" mapstructure:"tracker-cache-decay"` } // GossipSubRPCMetricsInspectorConfigs rpc metrics observer inspector configuration. type GossipSubRPCMetricsInspectorConfigs struct { // NumberOfWorkers number of worker pool workers. - NumberOfWorkers int `validate:"gte=1" mapstructure:"gossipsub-rpc-metrics-inspector-workers"` + NumberOfWorkers int `validate:"gte=1" mapstructure:"workers"` // CacheSize size of the queue used by worker pool for the control message metrics inspector. - CacheSize uint32 `validate:"gt=0" mapstructure:"gossipsub-rpc-metrics-inspector-cache-size"` + CacheSize uint32 `validate:"gt=0" mapstructure:"cache-size"` } diff --git a/network/p2p/scoring/registry.go b/network/p2p/scoring/registry.go index 07b30f21e4a..ec00648f37f 100644 --- a/network/p2p/scoring/registry.go +++ b/network/p2p/scoring/registry.go @@ -113,16 +113,28 @@ type GossipSubAppSpecificScoreRegistry struct { appScoreUpdateWorkerPool *worker.Pool[peer.ID] } +// AppSpecificScoreRegistryParams is the parameters for the GossipSubAppSpecificScoreRegistry. +// Parameters are "numerical values" that are used to compute or build components that compute or maintain the application specific score of peers. +type AppSpecificScoreRegistryParams struct { + // ScoreUpdateWorkerNum is the number of workers in the worker pool for handling the application specific score update of peers in a non-blocking way. + ScoreUpdateWorkerNum int `validate:"gt=0" mapstructure:"score-update-worker-num"` + + // ScoreUpdateRequestQueueSize is the size of the worker pool for handling the application specific score update of peers in a non-blocking way. + ScoreUpdateRequestQueueSize uint32 `validate:"gt=0" mapstructure:"score-update-request-queue-size"` + + // ScoreTTL is the time to live of the application specific score of a peer; the registry keeps a cached copy of the + // application specific score of a peer for this duration. When the duration expires, the application specific score + // of the peer is updated asynchronously. As long as the update is in progress, the cached copy of the application + // specific score of the peer is used even if it is expired. + ScoreTTL time.Duration `validate:"required" mapstructure:"score-ttl"` +} + // GossipSubAppSpecificScoreRegistryConfig is the configuration for the GossipSubAppSpecificScoreRegistry. -// The configuration is used to initialize the registry. +// Configurations are the "union of parameters and other components" that are used to compute or build components that compute or maintain the application specific score of peers. type GossipSubAppSpecificScoreRegistryConfig struct { - Logger zerolog.Logger `validate:"required"` - - // AppSpecificScoreNumWorkers is the number of workers in the worker pool for handling the application specific score update of peers in a non-blocking way. - AppSpecificScoreNumWorkers int `validate:"gt=0"` + Parameters AppSpecificScoreRegistryParams `validate:"required"` - // AppSpecificScoreWorkerPoolSize is the size of the worker pool for handling the application specific score update of peers in a non-blocking way. - AppSpecificScoreWorkerPoolSize uint32 `validate:"gt=0"` + Logger zerolog.Logger `validate:"required"` // Validator is the subscription validator used to validate the subscriptions of peers, and determine if a peer is // authorized to subscribe to a topic. @@ -139,12 +151,6 @@ type GossipSubAppSpecificScoreRegistryConfig struct { // a peer when the peer is first observed by the local peer. Init func() p2p.GossipSubSpamRecord `validate:"required"` - // ScoreTTL is the time to live of the application specific score of a peer; the registry keeps a cached copy of the - // application specific score of a peer for this duration. When the duration expires, the application specific score - // of the peer is updated asynchronously. As long as the update is in progress, the cached copy of the application - // specific score of the peer is used even if it is expired. - ScoreTTL time.Duration `validate:"required"` - // SpamRecordCacheFactory is a factory function that returns a new GossipSubSpamRecordCache. It is used to initialize the spamScoreCache. // The cache is used to store the application specific penalty of peers. SpamRecordCacheFactory func() p2p.GossipSubSpamRecordCache `validate:"required"` @@ -172,7 +178,7 @@ func NewGossipSubAppSpecificScoreRegistry(config *GossipSubAppSpecificScoreRegis } lg := config.Logger.With().Str("module", "app_score_registry").Logger() - store := queue.NewHeroStore(config.AppSpecificScoreWorkerPoolSize, + store := queue.NewHeroStore(config.Parameters.ScoreUpdateRequestQueueSize, lg.With().Str("component", "app_specific_score_update").Logger(), metrics.GossipSubAppSpecificScoreUpdateQueueMetricFactory(config.HeroCacheMetricsFactory)) @@ -184,7 +190,7 @@ func NewGossipSubAppSpecificScoreRegistry(config *GossipSubAppSpecificScoreRegis init: config.Init, validator: config.Validator, idProvider: config.IdProvider, - scoreTTL: config.ScoreTTL, + scoreTTL: config.Parameters.ScoreTTL, } reg.appScoreUpdateWorkerPool = worker.NewWorkerPoolBuilder[peer.ID](lg.With().Str("component", "app_specific_score_update_worker_pool").Logger(), @@ -210,13 +216,13 @@ func NewGossipSubAppSpecificScoreRegistry(config *GossipSubAppSpecificScoreRegis reg.logger.Info().Msg("subscription validator stopped") }) - for i := 0; i < config.AppSpecificScoreNumWorkers; i++ { + for i := 0; i < config.Parameters.ScoreUpdateWorkerNum; i++ { builder.AddWorker(reg.appScoreUpdateWorkerPool.WorkerLogic()) } reg.Component = builder.Build() - return reg + return reg, nil } var _ p2p.GossipSubInvCtrlMsgNotifConsumer = (*GossipSubAppSpecificScoreRegistry)(nil) diff --git a/network/p2p/scoring/registry_test.go b/network/p2p/scoring/registry_test.go index cb493cb2c72..2c63ea0f112 100644 --- a/network/p2p/scoring/registry_test.go +++ b/network/p2p/scoring/registry_test.go @@ -466,7 +466,7 @@ func withInitFunction(initFunction func() p2p.GossipSubSpamRecord) func(cfg *sco // for the testing purposes. func newGossipSubAppSpecificScoreRegistry(t *testing.T, opts ...func(*scoring.GossipSubAppSpecificScoreRegistryConfig)) (*scoring.GossipSubAppSpecificScoreRegistry, *netcache.GossipSubSpamRecordCache) { - cache := netcache.NewGossipSubSpamRecordCache(100, unittest.Logger(), metrics.NewNoopCollector(), scoring.DefaultDecayFunction()) + cache := netcache.NewGossipSubSpamRecordCache(100, unittest.Logger(), metrics.NewNoopHeroCacheMetricsFactory(), scoring.DefaultDecayFunction()) cfg := &scoring.GossipSubAppSpecificScoreRegistryConfig{ Logger: unittest.Logger(), Init: scoring.InitAppScoreRecordState, diff --git a/network/p2p/scoring/score_option.go b/network/p2p/scoring/score_option.go index 629ee91f8d8..cf3c4c5f28d 100644 --- a/network/p2p/scoring/score_option.go +++ b/network/p2p/scoring/score_option.go @@ -305,42 +305,48 @@ type ScoreOption struct { appScoreFunc func(peer.ID) float64 } +// Parameters are the parameters for the score option. +// Parameters are "numerical values" that are used to compute or build components that compute the score of a peer in GossipSub system. +type Parameters struct { + AppSpecificScore AppSpecificScoreRegistryParams `validate:"required" mapstructure:"app-specific-score"` + // SpamRecordCacheSize is size of the cache used to store the spam records of peers. + // The spam records are used to penalize peers that send invalid messages. + SpamRecordCacheSize uint32 `validate:"gt=0" mapstructure:"spam-record-cache-size"` + + // DecayInterval is the interval at which the counters associated with a peer behavior in GossipSub system are decayed. + DecayInterval time.Duration `validate:"gt=0s" mapstructure:"decay-interval"` +} + type ScoreOptionConfig struct { logger zerolog.Logger + params Parameters provider module.IdentityProvider - cacheSize uint32 - cacheMetrics module.HeroCacheMetrics + heroCacheMetricsFactory metrics.HeroCacheMetricsFactory appScoreFunc func(peer.ID) float64 - decayInterval time.Duration // the decay interval, when is set to 0, the default value will be used. topicParams []func(map[string]*pubsub.TopicScoreParams) registerNotificationConsumerFunc func(p2p.GossipSubInvCtrlMsgNotifConsumer) } -func NewScoreOptionConfig(logger zerolog.Logger, idProvider module.IdentityProvider) *ScoreOptionConfig { +// NewScoreOptionConfig creates a new configuration for the GossipSub peer scoring option. +// Args: +// - logger: the logger to use. +// - hcMetricsFactory: HeroCache metrics factory to create metrics for the scoring-related caches. +// - idProvider: the identity provider to use. +// Returns: +// - a new configuration for the GossipSub peer scoring option. +func NewScoreOptionConfig(logger zerolog.Logger, + params Parameters, + hcMetricsFactory metrics.HeroCacheMetricsFactory, + idProvider module.IdentityProvider) *ScoreOptionConfig { return &ScoreOptionConfig{ - logger: logger, - provider: idProvider, - cacheSize: defaultScoreCacheSize, - cacheMetrics: metrics.NewNoopCollector(), // no metrics by default - topicParams: make([]func(map[string]*pubsub.TopicScoreParams), 0), + logger: logger.With().Str("module", "pubsub_score_option").Logger(), + provider: idProvider, + params: params, + heroCacheMetricsFactory: hcMetricsFactory, + topicParams: make([]func(map[string]*pubsub.TopicScoreParams), 0), } } -// SetCacheSize sets the size of the cache used to store the app specific penalty of peers. -// If the cache size is not set, the default value will be used. -// It is safe to call this method multiple times, the last call will be used. -func (c *ScoreOptionConfig) SetCacheSize(size uint32) { - c.cacheSize = size -} - -// SetCacheMetrics sets the cache metrics collector for the penalty option. -// It is used to collect metrics for the app specific penalty cache. If the cache metrics collector is not set, -// a no-op collector will be used. -// It is safe to call this method multiple times, the last call will be used. -func (c *ScoreOptionConfig) SetCacheMetrics(metrics module.HeroCacheMetrics) { - c.cacheMetrics = metrics -} - // OverrideAppSpecificScoreFunction sets the app specific penalty function for the penalty option. // It is used to calculate the app specific penalty of a peer. // If the app specific penalty function is not set, the default one is used. @@ -366,21 +372,6 @@ func (c *ScoreOptionConfig) SetRegisterNotificationConsumerFunc(f func(p2p.Gossi c.registerNotificationConsumerFunc = f } -// OverrideDecayInterval overrides the decay interval for the penalty option. It is used to override the default -// decay interval for the penalty option. The decay interval is the time interval that the decay values are applied and -// peer scores are updated. -// Note: It is always recommended to use the default value unless you know what you are doing. Hence, calling this method -// is not recommended in production. -// Args: -// -// interval: the decay interval. -// -// Returns: -// none -func (c *ScoreOptionConfig) OverrideDecayInterval(interval time.Duration) { - c.decayInterval = interval -} - // NewScoreOption creates a new penalty option with the given configuration. func NewScoreOption(cfg *ScoreOptionConfig, provider p2p.SubscriptionProvider) (*ScoreOption, error) { throttledSampler := logging.BurstSampler(MaxDebugLogs, time.Second) @@ -393,13 +384,14 @@ func NewScoreOption(cfg *ScoreOptionConfig, provider p2p.SubscriptionProvider) ( }) validator := NewSubscriptionValidator(cfg.logger, provider) scoreRegistry, err := NewGossipSubAppSpecificScoreRegistry(&GossipSubAppSpecificScoreRegistryConfig{ - Logger: logger, - Penalty: DefaultGossipSubCtrlMsgPenaltyValue(), - Validator: validator, - Init: InitAppScoreRecordState, - IdProvider: cfg.provider, + Logger: logger, + Penalty: DefaultGossipSubCtrlMsgPenaltyValue(), + Validator: validator, + Init: InitAppScoreRecordState, + IdProvider: cfg.provider, + HeroCacheMetricsFactory: cfg.heroCacheMetricsFactory, SpamRecordCacheFactory: func() p2p.GossipSubSpamRecordCache { - return netcache.NewGossipSubSpamRecordCache(cfg.cacheSize, cfg.logger, cfg.cacheMetrics, DefaultDecayFunction()) + return netcache.NewGossipSubSpamRecordCache(cfg.params.SpamRecordCacheSize, cfg.logger, cfg.heroCacheMetricsFactory, DefaultDecayFunction()) }, }) @@ -424,13 +416,13 @@ func NewScoreOption(cfg *ScoreOptionConfig, provider p2p.SubscriptionProvider) ( Msg("app specific score function is overridden, should never happen in production") } - if cfg.decayInterval > 0 { + if cfg.params.DecayInterval > 0 && cfg.params.DecayInterval != s.peerScoreParams.DecayInterval { // overrides the default decay interval if the decay interval is set. - s.peerScoreParams.DecayInterval = cfg.decayInterval + s.peerScoreParams.DecayInterval = cfg.params.DecayInterval s.logger. Warn(). Str(logging.KeyNetworkingSecurity, "true"). - Dur("decay_interval_ms", cfg.decayInterval). + Dur("decay_interval_ms", cfg.params.DecayInterval). Msg("decay interval is overridden, should never happen in production") } diff --git a/network/p2p/scoring/subscription_provider.go b/network/p2p/scoring/subscription_provider.go index 4f6918a81a0..b70a8cba949 100644 --- a/network/p2p/scoring/subscription_provider.go +++ b/network/p2p/scoring/subscription_provider.go @@ -59,7 +59,7 @@ func NewSubscriptionProvider(cfg *SubscriptionProviderConfig) (*SubscriptionProv p := &SubscriptionProvider{ logger: cfg.Logger.With().Str("module", "subscription_provider").Logger(), topicProviderOracle: cfg.TopicProviderOracle, - allTopicsUpdateInterval: cfg.Params.SubscriptionUpdateInterval, + allTopicsUpdateInterval: cfg.Params.UpdateInterval, idProvider: cfg.IdProvider, cache: cache, } @@ -69,7 +69,7 @@ func NewSubscriptionProvider(cfg *SubscriptionProviderConfig) (*SubscriptionProv func(ctx irrecoverable.SignalerContext, ready component.ReadyFunc) { ready() p.logger.Debug(). - Float64("update_interval_seconds", cfg.Params.SubscriptionUpdateInterval.Seconds()). + Float64("update_interval_seconds", cfg.Params.UpdateInterval.Seconds()). Msg("subscription provider started; starting update topics loop") p.updateTopicsLoop(ctx) diff --git a/network/p2p/scoring/subscription_provider_test.go b/network/p2p/scoring/subscription_provider_test.go index cb3b45ecbd1..ba2180af6cf 100644 --- a/network/p2p/scoring/subscription_provider_test.go +++ b/network/p2p/scoring/subscription_provider_test.go @@ -31,14 +31,14 @@ func TestSubscriptionProvider_GetSubscribedTopics(t *testing.T) { idProvider := mock.NewIdentityProvider(t) // set a low update interval to speed up the test - cfg.NetworkConfig.SubscriptionProviderConfig.SubscriptionUpdateInterval = 100 * time.Millisecond + cfg.NetworkConfig.SubscriptionProvider.SubscriptionUpdateInterval = 100 * time.Millisecond sp, err := scoring.NewSubscriptionProvider(&scoring.SubscriptionProviderConfig{ Logger: unittest.Logger(), TopicProviderOracle: func() p2p.TopicProvider { return tp }, - Params: &cfg.NetworkConfig.SubscriptionProviderConfig, + Params: &cfg.NetworkConfig.SubscriptionProvider, HeroCacheMetricsFactory: metrics.NewNoopHeroCacheMetricsFactory(), IdProvider: idProvider, }) @@ -92,14 +92,14 @@ func TestSubscriptionProvider_GetSubscribedTopics_SkippingUnknownPeers(t *testin idProvider := mock.NewIdentityProvider(t) // set a low update interval to speed up the test - cfg.NetworkConfig.SubscriptionProviderConfig.SubscriptionUpdateInterval = 100 * time.Millisecond + cfg.NetworkConfig.SubscriptionProvider.SubscriptionUpdateInterval = 100 * time.Millisecond sp, err := scoring.NewSubscriptionProvider(&scoring.SubscriptionProviderConfig{ Logger: unittest.Logger(), TopicProviderOracle: func() p2p.TopicProvider { return tp }, - Params: &cfg.NetworkConfig.SubscriptionProviderConfig, + Params: &cfg.NetworkConfig.SubscriptionProvider, HeroCacheMetricsFactory: metrics.NewNoopHeroCacheMetricsFactory(), IdProvider: idProvider, }) diff --git a/network/p2p/scoring/subscription_validator_test.go b/network/p2p/scoring/subscription_validator_test.go index 770f74cf146..0ac64f65fe8 100644 --- a/network/p2p/scoring/subscription_validator_test.go +++ b/network/p2p/scoring/subscription_validator_test.go @@ -170,7 +170,7 @@ func TestSubscriptionValidator_Integration(t *testing.T) { cfg, err := config.DefaultConfig() require.NoError(t, err) // set a low update interval to speed up the test - cfg.NetworkConfig.SubscriptionProviderConfig.SubscriptionUpdateInterval = 100 * time.Millisecond + cfg.NetworkConfig.SubscriptionProvider.SubscriptionUpdateInterval = 100 * time.Millisecond sporkId := unittest.IdentifierFixture() diff --git a/network/p2p/test/fixtures.go b/network/p2p/test/fixtures.go index 4a3c5686f1d..2acc998207c 100644 --- a/network/p2p/test/fixtures.go +++ b/network/p2p/test/fixtures.go @@ -149,7 +149,7 @@ func NodeFixture(t *testing.T, ¶meters.FlowConfig.NetworkConfig.ResourceManager, ¶meters.FlowConfig.NetworkConfig.GossipSubRPCInspectorsConfig, parameters.PeerManagerConfig, - ¶meters.FlowConfig.NetworkConfig.GossipSubConfig.SubscriptionProviderConfig, + ¶meters.FlowConfig.NetworkConfig.GossipSubConfig.SubscriptionProvider, &p2p.DisallowListCacheConfig{ MaxSize: uint32(1000), Metrics: metrics.NewNoopCollector(), From 26302f0f0c4518901fa21d52ee64b0d9ed654edc Mon Sep 17 00:00:00 2001 From: Yahya Hassanzadeh Date: Mon, 20 Nov 2023 13:24:48 -0800 Subject: [PATCH 14/67] wip organizing parameters --- network/p2p/builder.go | 2 +- ...ip_sub_rpc_inspector_suite_factory_func.go | 8 +- .../p2pbuilder/gossipsub/gossipSubBuilder.go | 6 +- network/p2p/p2pbuilder/libp2pNodeBuilder.go | 4 +- network/p2p/p2pconf/gossipsub.go | 4 +- .../p2p/p2pconf/gossipsub_rpc_inspectors.go | 87 +++++++++++++++---- network/p2p/scoring/scoring_test.go | 2 +- 7 files changed, 84 insertions(+), 29 deletions(-) diff --git a/network/p2p/builder.go b/network/p2p/builder.go index 31a7da024f5..f9eebad66dc 100644 --- a/network/p2p/builder.go +++ b/network/p2p/builder.go @@ -105,7 +105,7 @@ type GossipSubRpcInspectorSuiteFactoryFunc func( irrecoverable.SignalerContext, zerolog.Logger, flow.Identifier, - *p2pconf.GossipSubRPCInspectorsConfig, + *p2pconf.RpcInspectorParameters, module.GossipSubMetrics, metrics.HeroCacheMetricsFactory, flownet.NetworkingType, diff --git a/network/p2p/mock/gossip_sub_rpc_inspector_suite_factory_func.go b/network/p2p/mock/gossip_sub_rpc_inspector_suite_factory_func.go index 24c253b70d2..3f934bdea07 100644 --- a/network/p2p/mock/gossip_sub_rpc_inspector_suite_factory_func.go +++ b/network/p2p/mock/gossip_sub_rpc_inspector_suite_factory_func.go @@ -27,15 +27,15 @@ type GossipSubRpcInspectorSuiteFactoryFunc struct { } // Execute provides a mock function with given fields: _a0, _a1, _a2, _a3, _a4, _a5, _a6, _a7, _a8 -func (_m *GossipSubRpcInspectorSuiteFactoryFunc) Execute(_a0 irrecoverable.SignalerContext, _a1 zerolog.Logger, _a2 flow.Identifier, _a3 *p2pconf.GossipSubRPCInspectorsConfig, _a4 module.GossipSubMetrics, _a5 metrics.HeroCacheMetricsFactory, _a6 network.NetworkingType, _a7 module.IdentityProvider, _a8 func() p2p.TopicProvider) (p2p.GossipSubInspectorSuite, error) { +func (_m *GossipSubRpcInspectorSuiteFactoryFunc) Execute(_a0 irrecoverable.SignalerContext, _a1 zerolog.Logger, _a2 flow.Identifier, _a3 *p2pconf.RpcInspectorParameters, _a4 module.GossipSubMetrics, _a5 metrics.HeroCacheMetricsFactory, _a6 network.NetworkingType, _a7 module.IdentityProvider, _a8 func() p2p.TopicProvider) (p2p.GossipSubInspectorSuite, error) { ret := _m.Called(_a0, _a1, _a2, _a3, _a4, _a5, _a6, _a7, _a8) var r0 p2p.GossipSubInspectorSuite var r1 error - if rf, ok := ret.Get(0).(func(irrecoverable.SignalerContext, zerolog.Logger, flow.Identifier, *p2pconf.GossipSubRPCInspectorsConfig, module.GossipSubMetrics, metrics.HeroCacheMetricsFactory, network.NetworkingType, module.IdentityProvider, func() p2p.TopicProvider) (p2p.GossipSubInspectorSuite, error)); ok { + if rf, ok := ret.Get(0).(func(irrecoverable.SignalerContext, zerolog.Logger, flow.Identifier, *p2pconf.RpcInspectorParameters, module.GossipSubMetrics, metrics.HeroCacheMetricsFactory, network.NetworkingType, module.IdentityProvider, func() p2p.TopicProvider) (p2p.GossipSubInspectorSuite, error)); ok { return rf(_a0, _a1, _a2, _a3, _a4, _a5, _a6, _a7, _a8) } - if rf, ok := ret.Get(0).(func(irrecoverable.SignalerContext, zerolog.Logger, flow.Identifier, *p2pconf.GossipSubRPCInspectorsConfig, module.GossipSubMetrics, metrics.HeroCacheMetricsFactory, network.NetworkingType, module.IdentityProvider, func() p2p.TopicProvider) p2p.GossipSubInspectorSuite); ok { + if rf, ok := ret.Get(0).(func(irrecoverable.SignalerContext, zerolog.Logger, flow.Identifier, *p2pconf.RpcInspectorParameters, module.GossipSubMetrics, metrics.HeroCacheMetricsFactory, network.NetworkingType, module.IdentityProvider, func() p2p.TopicProvider) p2p.GossipSubInspectorSuite); ok { r0 = rf(_a0, _a1, _a2, _a3, _a4, _a5, _a6, _a7, _a8) } else { if ret.Get(0) != nil { @@ -43,7 +43,7 @@ func (_m *GossipSubRpcInspectorSuiteFactoryFunc) Execute(_a0 irrecoverable.Signa } } - if rf, ok := ret.Get(1).(func(irrecoverable.SignalerContext, zerolog.Logger, flow.Identifier, *p2pconf.GossipSubRPCInspectorsConfig, module.GossipSubMetrics, metrics.HeroCacheMetricsFactory, network.NetworkingType, module.IdentityProvider, func() p2p.TopicProvider) error); ok { + if rf, ok := ret.Get(1).(func(irrecoverable.SignalerContext, zerolog.Logger, flow.Identifier, *p2pconf.RpcInspectorParameters, module.GossipSubMetrics, metrics.HeroCacheMetricsFactory, network.NetworkingType, module.IdentityProvider, func() p2p.TopicProvider) error); ok { r1 = rf(_a0, _a1, _a2, _a3, _a4, _a5, _a6, _a7, _a8) } else { r1 = ret.Error(1) diff --git a/network/p2p/p2pbuilder/gossipsub/gossipSubBuilder.go b/network/p2p/p2pbuilder/gossipsub/gossipSubBuilder.go index fa0c79f3417..23b9b931e17 100644 --- a/network/p2p/p2pbuilder/gossipsub/gossipSubBuilder.go +++ b/network/p2p/p2pbuilder/gossipsub/gossipSubBuilder.go @@ -49,7 +49,7 @@ type Builder struct { subscriptionProviderParam *p2pconf.SubscriptionProviderParameters idProvider module.IdentityProvider routingSystem routing.Routing - rpcInspectorConfig *p2pconf.GossipSubRPCInspectorsConfig + rpcInspectorConfig *p2pconf.RpcInspectorParameters rpcInspectorSuiteFactory p2p.GossipSubRpcInspectorSuiteFactoryFunc } @@ -184,7 +184,7 @@ func NewGossipSubBuilder( networkType network.NetworkingType, sporkId flow.Identifier, idProvider module.IdentityProvider, - rpcInspectorConfig *p2pconf.GossipSubRPCInspectorsConfig, + rpcInspectorConfig *p2pconf.RpcInspectorParameters, subscriptionProviderPrams *p2pconf.SubscriptionProviderParameters, rpcTracker p2p.RpcControlTracking) *Builder { lg := logger.With(). @@ -238,7 +238,7 @@ func defaultInspectorSuite(rpcTracker p2p.RpcControlTracking) p2p.GossipSubRpcIn ctx irrecoverable.SignalerContext, logger zerolog.Logger, sporkId flow.Identifier, - inspectorCfg *p2pconf.GossipSubRPCInspectorsConfig, + inspectorCfg *p2pconf.RpcInspectorParameters, gossipSubMetrics module.GossipSubMetrics, heroCacheMetricsFactory metrics.HeroCacheMetricsFactory, networkType network.NetworkingType, diff --git a/network/p2p/p2pbuilder/libp2pNodeBuilder.go b/network/p2p/p2pbuilder/libp2pNodeBuilder.go index 9c010696d7e..24996c9b017 100644 --- a/network/p2p/p2pbuilder/libp2pNodeBuilder.go +++ b/network/p2p/p2pbuilder/libp2pNodeBuilder.go @@ -85,7 +85,7 @@ func NewNodeBuilder( sporkId flow.Identifier, idProvider module.IdentityProvider, rCfg *p2pconf.ResourceManagerConfig, - rpcInspectorCfg *p2pconf.GossipSubRPCInspectorsConfig, + rpcInspectorCfg *p2pconf.RpcInspectorParameters, peerManagerConfig *p2pconfig.PeerManagerConfig, subscriptionProviderParam *p2pconf.SubscriptionProviderParameters, disallowListCacheCfg *p2p.DisallowListCacheConfig, @@ -424,7 +424,7 @@ func DefaultNodeBuilder( connGaterCfg *p2pconfig.ConnectionGaterConfig, peerManagerCfg *p2pconfig.PeerManagerConfig, gossipCfg *p2pconf.GossipSubConfig, - rpcInspectorCfg *p2pconf.GossipSubRPCInspectorsConfig, + rpcInspectorCfg *p2pconf.RpcInspectorParameters, rCfg *p2pconf.ResourceManagerConfig, uniCfg *p2pconfig.UnicastConfig, connMgrConfig *netconf.ConnectionManagerConfig, diff --git a/network/p2p/p2pconf/gossipsub.go b/network/p2p/p2pconf/gossipsub.go index d019f10b2f5..2fd681e93ea 100644 --- a/network/p2p/p2pconf/gossipsub.go +++ b/network/p2p/p2pconf/gossipsub.go @@ -69,8 +69,8 @@ var GossipSubConfigKeys = []string{RpcInspectorKey, RpcTracerKey, PeerScoringKey // GossipSubConfig is the configuration for the GossipSub pubsub implementation. type GossipSubConfig struct { - // GossipSubRPCInspectorsConfig configuration for all gossipsub RPC control message inspectors. - RpcInspector GossipSubRPCInspectorsConfig `mapstructure:"rpc-inspector"` + // RpcInspectorParameters configuration for all gossipsub RPC control message inspectors. + RpcInspector RpcInspectorParameters `mapstructure:"rpc-inspector"` // GossipSubTracerParameters is the configuration for the gossipsub tracer. GossipSub tracer is used to trace the local mesh events and peer scores. RpcTracer GossipSubTracerParameters `mapstructure:"rpc-tracer"` diff --git a/network/p2p/p2pconf/gossipsub_rpc_inspectors.go b/network/p2p/p2pconf/gossipsub_rpc_inspectors.go index 48f189c406c..03d4a2f55a7 100644 --- a/network/p2p/p2pconf/gossipsub_rpc_inspectors.go +++ b/network/p2p/p2pconf/gossipsub_rpc_inspectors.go @@ -1,20 +1,49 @@ package p2pconf -// GossipSubRPCInspectorsConfig encompasses configuration related to gossipsub RPC message inspectors. -type GossipSubRPCInspectorsConfig struct { +// RpcInspectorParameters keys. +const ( + ValidationConfigKey = "validation" + MetricsConfigKey = "metrics" + NotificationCacheSizeKey = "notification-cache-size" +) + +var RpcInspectorParametersKeys = []string{ValidationConfigKey, MetricsConfigKey, NotificationCacheSizeKey} + +// RpcInspectorParameters contains the "numerical values" for the gossipsub RPC control message inspectors parameters. +type RpcInspectorParameters struct { // RpcValidationInspector control message validation inspector validation configuration and limits. - RpcValidation RpcValidationInspector `mapstructure:"validation"` - // GossipSubRPCMetricsInspectorConfigs control message metrics inspector configuration. - RpcMetrics GossipSubRPCMetricsInspectorConfigs `mapstructure:"metrics"` + Validation RpcValidationInspector `mapstructure:"validation"` + // RpcMetricsInspectorConfigs control message metrics inspector configuration. + Metrics RpcMetricsInspectorConfigs `mapstructure:"metrics"` // NotificationCacheSize size of the queue for notifications about invalid RPC messages. NotificationCacheSize uint32 `mapstructure:"notification-cache-size"` } +// RpcValidationInspectorParameters keys. +const ( + ClusterPrefixedMessageConfigKey = "cluster-prefixed-messages" + IWantConfigKey = "iwant" + IHaveConfigKey = "ihave" + QueueSizeKey = "queue-size" + GraftPruneMessageMaxSampleSizeKey = "graft-and-prune-message-max-sample-size" + MessageMaxSampleSizeKey = "message-max-sample-size" + MessageErrorThresholdKey = "message-error-threshold" +) + +var RpcValidationInspectorParamterKeys = []string{ClusterPrefixedMessageConfigKey, + IWantConfigKey, + IHaveConfigKey, + NumberOfWorkersKey, + QueueSizeKey, + GraftPruneMessageMaxSampleSizeKey, + MessageMaxSampleSizeKey, + MessageErrorThresholdKey} + // RpcValidationInspector validation limits used for gossipsub RPC control message inspection. type RpcValidationInspector struct { - ClusterPrefixedMessage ClusterPrefixedMessageConfig `mapstructure:"cluster-prefixed-messages"` - IWant IWantRPCInspectionConfig `mapstructure:"iwant"` - IHave IHaveRPCInspectionConfig `mapstructure:"ihave"` + ClusterPrefixedMessage ClusterPrefixedMessageInspectionParameters `mapstructure:"cluster-prefixed-messages"` + IWant IWantRPCInspectionParameters `mapstructure:"iwant"` + IHave IHaveRpcInspectionParameters `mapstructure:"ihave"` // NumberOfWorkers number of worker pool workers. NumberOfWorkers int `validate:"gte=1" mapstructure:"workers"` // QueueSize size of the queue used by worker pool for the control message validation inspector. @@ -28,8 +57,18 @@ type RpcValidationInspector struct { MessageErrorThreshold int `validate:"gte=500" mapstructure:"message-error-threshold"` } -// IWantRPCInspectionConfig validation configuration for iWANT RPC control messages. -type IWantRPCInspectionConfig struct { +const ( + MaxSampleSizeKey = "max-sample-size" + MaxMessageIDSampleSizeKey = "max-message-id-sample-size" + CacheMissThresholdKey = "cache-miss-threshold" + CacheMissCheckSizeKey = "cache-miss-check-size" + DuplicateMsgIDThresholdKey = "duplicate-message-id-threshold" +) + +var IWantRPCInspectionParametersKeys = []string{MaxSampleSizeKey, MaxMessageIDSampleSizeKey, CacheMissThresholdKey, CacheMissCheckSizeKey, DuplicateMsgIDThresholdKey} + +// IWantRPCInspectionParameters contains the "numerical values" for the iwant rpc control message inspection. +type IWantRPCInspectionParameters struct { // MaxSampleSize max inspection sample size to use. If the total number of iWant control messages // exceeds this max sample size then the respective message will be truncated before being processed. MaxSampleSize uint `validate:"gt=0" mapstructure:"max-sample-size"` @@ -46,8 +85,10 @@ type IWantRPCInspectionConfig struct { DuplicateMsgIDThreshold float64 `validate:"gt=0" mapstructure:"duplicate-message-id-threshold"` } -// IHaveRPCInspectionConfig validation configuration for iHave RPC control messages. -type IHaveRPCInspectionConfig struct { +var IHaveRPCInspectionConfigKeys = []string{MaxSampleSizeKey, MaxMessageIDSampleSizeKey} + +// IHaveRpcInspectionParameters contains the "numerical values" for ihave rpc control inspection. +type IHaveRpcInspectionParameters struct { // MaxSampleSize max inspection sample size to use. If the number of ihave messages exceeds this configured value // the control message ihaves will be truncated to the max sample size. This sample is randomly selected. MaxSampleSize int `validate:"gte=1000" mapstructure:"max-sample-size"` @@ -56,8 +97,16 @@ type IHaveRPCInspectionConfig struct { MaxMessageIDSampleSize int `validate:"gte=1000" mapstructure:"max-message-id-sample-size"` } -// ClusterPrefixedMessageConfig configuration values for cluster prefixed control message validation. -type ClusterPrefixedMessageConfig struct { +const ( + HardThresholdKey = "hard-threshold" + TrackerCacheSizeKey = "tracker-cache-size" + TrackerCacheDecayKey = "tracker-cache-decay" +) + +var ClusterPrefixedMessageParameterKeys = []string{HardThresholdKey, TrackerCacheSizeKey, TrackerCacheDecayKey} + +// ClusterPrefixedMessageInspectionParameters contains the "numerical values" for cluster prefixed control message inspection. +type ClusterPrefixedMessageInspectionParameters struct { // ClusterPrefixHardThreshold the upper bound on the amount of cluster prefixed control messages that will be processed // before a node starts to get penalized. This allows LN nodes to process some cluster prefixed control messages during startup // when the cluster ID's provider is set asynchronously. It also allows processing of some stale messages that may be sent by nodes @@ -70,8 +119,14 @@ type ClusterPrefixedMessageConfig struct { ClusterPrefixedControlMsgsReceivedCacheDecay float64 `validate:"gt=0" mapstructure:"tracker-cache-decay"` } -// GossipSubRPCMetricsInspectorConfigs rpc metrics observer inspector configuration. -type GossipSubRPCMetricsInspectorConfigs struct { +const ( + NumberOfWorkersKey = "workers" +) + +var RpcMetricsInspectorConfigsKeys = []string{NumberOfWorkersKey, CacheSizeKey} + +// RpcMetricsInspectorConfigs contains the "numerical values" for the gossipsub RPC control message metrics inspectors parameters. +type RpcMetricsInspectorConfigs struct { // NumberOfWorkers number of worker pool workers. NumberOfWorkers int `validate:"gte=1" mapstructure:"workers"` // CacheSize size of the queue used by worker pool for the control message metrics inspector. diff --git a/network/p2p/scoring/scoring_test.go b/network/p2p/scoring/scoring_test.go index 906cc0b2fc6..512418421ff 100644 --- a/network/p2p/scoring/scoring_test.go +++ b/network/p2p/scoring/scoring_test.go @@ -89,7 +89,7 @@ func TestInvalidCtrlMsgScoringIntegration(t *testing.T) { irrecoverable.SignalerContext, zerolog.Logger, flow.Identifier, - *p2pconf.GossipSubRPCInspectorsConfig, + *p2pconf.RpcInspectorParameters, module.GossipSubMetrics, metrics.HeroCacheMetricsFactory, flownet.NetworkingType, From a5efcfd2d704986093ce55c76e0b4fa87257a78a Mon Sep 17 00:00:00 2001 From: Yahya Hassanzadeh Date: Mon, 20 Nov 2023 16:18:08 -0800 Subject: [PATCH 15/67] wip organizing parameters --- config/default-config.yml | 2 +- network/netconf/config.go | 6 +- network/netconf/flags.go | 220 +++++++++--------- network/netconf/flags_test.go | 51 ++++ network/p2p/builder.go | 18 +- .../control_message_validation_inspector.go | 4 +- .../p2pbuilder/gossipsub/gossipSubBuilder.go | 14 +- network/p2p/p2pbuilder/libp2pNodeBuilder.go | 4 +- network/p2p/p2pconf/gossipsub.go | 54 +++-- .../p2p/p2pconf/gossipsub_rpc_inspectors.go | 35 +-- network/p2p/scoring/registry.go | 19 +- network/p2p/scoring/score_option.go | 17 +- network/p2p/scoring/scoring_test.go | 4 +- 13 files changed, 239 insertions(+), 209 deletions(-) create mode 100644 network/netconf/flags_test.go diff --git a/config/default-config.yml b/config/default-config.yml index b4f7dff7a53..93c184aa40e 100644 --- a/config/default-config.yml +++ b/config/default-config.yml @@ -154,7 +154,7 @@ network-config: # The max allowed duplicate message IDs in a single iWant control message. If the duplicate message threshold is exceeded an invalid control message # notification is disseminated and the sender will be penalized. duplicate-message-id-threshold: .15 - cluster-prefixed-message: + cluster-prefixed-messages: # Cluster prefixed control message validation configs # The size of the cache used to track the amount of cluster prefixed topics received by peers tracker-cache-size: 100 diff --git a/network/netconf/config.go b/network/netconf/config.go index 34d0da6d2b1..f1fd7a267cc 100644 --- a/network/netconf/config.go +++ b/network/netconf/config.go @@ -11,9 +11,9 @@ type Config struct { UnicastConfig `mapstructure:",squash"` ResourceManager p2pconf.ResourceManagerConfig `mapstructure:"libp2p-resource-manager"` ConnectionManagerConfig `mapstructure:",squash"` - // GossipSubConfig core gossipsub configuration. - GossipSub p2pconf.GossipSubConfig `mapstructure:"gossipsub"` - AlspConfig `mapstructure:",squash"` + // GossipSub core gossipsub configuration. + GossipSub p2pconf.GossipSubParameters `mapstructure:"gossipsub"` + AlspConfig `mapstructure:",squash"` // NetworkConnectionPruning determines whether connections to nodes // that are not part of protocol state should be trimmed diff --git a/network/netconf/flags.go b/network/netconf/flags.go index c37668e43e0..57bc0ac4f85 100644 --- a/network/netconf/flags.go +++ b/network/netconf/flags.go @@ -53,37 +53,36 @@ const ( gracePeriod = "libp2p-grace-period" silencePeriod = "libp2p-silence-period" // gossipsub - peerScoring = "gossipsub-peer-scoring-enabled" - localMeshLogInterval = "gossipsub-local-mesh-logging-interval" - rpcSentTrackerCacheSize = "gossipsub-rpc-sent-tracker-cache-size" - rpcSentTrackerQueueCacheSize = "gossipsub-rpc-sent-tracker-queue-cache-size" - rpcSentTrackerNumOfWorkers = "gossipsub-rpc-sent-tracker-workers" - scoreTracerInterval = "gossipsub-score-tracer-interval" + gossipSub = "gossipsub" + // rpcSentTrackerCacheSize = "gossipsub-rpc-sent-tracker-cache-size" + // rpcSentTrackerQueueCacheSize = "gossipsub-rpc-sent-tracker-queue-cache-size" + // rpcSentTrackerNumOfWorkers = "gossipsub-rpc-sent-tracker-workers" + // scoreTracerInterval = "gossipsub-score-tracer-interval" - gossipSubSubscriptionProviderUpdateInterval = "gossipsub-subscription-provider-update-interval" - gossipSubSubscriptionProviderCacheSize = "gossipsub-subscription-provider-cache-size" - - // gossipsub validation inspector - gossipSubRPCInspectorNotificationCacheSize = "gossipsub-rpc-inspector-notification-cache-size" - validationInspectorNumberOfWorkers = "gossipsub-rpc-validation-inspector-workers" - validationInspectorInspectMessageQueueCacheSize = "gossipsub-rpc-validation-inspector-queue-cache-size" - validationInspectorClusterPrefixedTopicsReceivedCacheSize = "gossipsub-cluster-prefix-tracker-cache-size" - validationInspectorClusterPrefixedTopicsReceivedCacheDecay = "gossipsub-cluster-prefix-tracker-cache-decay" - validationInspectorClusterPrefixHardThreshold = "gossipsub-rpc-cluster-prefixed-hard-threshold" - - ihaveMaxSampleSize = "gossipsub-rpc-ihave-max-sample-size" - ihaveMaxMessageIDSampleSize = "gossipsub-rpc-ihave-max-message-id-sample-size" - controlMessageMaxSampleSize = "gossipsub-rpc-graft-and-prune-message-max-sample-size" - iwantMaxSampleSize = "gossipsub-rpc-iwant-max-sample-size" - iwantMaxMessageIDSampleSize = "gossipsub-rpc-iwant-max-message-id-sample-size" - iwantCacheMissThreshold = "gossipsub-rpc-iwant-cache-miss-threshold" - iwantCacheMissCheckSize = "gossipsub-rpc-iwant-cache-miss-check-size" - iwantDuplicateMsgIDThreshold = "gossipsub-rpc-iwant-duplicate-message-id-threshold" - rpcMessageMaxSampleSize = "gossipsub-rpc-message-max-sample-size" - rpcMessageErrorThreshold = "gossipsub-rpc-message-error-threshold" + // gossipSubSubscriptionProviderUpdateInterval = "gossipsub-subscription-provider-update-interval" + // gossipSubSubscriptionProviderCacheSize = "gossipsub-subscription-provider-cache-size" + // + // // gossipsub validation inspector + // gossipSubRPCInspectorNotificationCacheSize = "gossipsub-rpc-inspector-notification-cache-size" + // validationInspectorNumberOfWorkers = "gossipsub-rpc-validation-inspector-workers" + // validationInspectorInspectMessageQueueCacheSize = "gossipsub-rpc-validation-inspector-queue-cache-size" + // validationInspectorClusterPrefixedTopicsReceivedCacheSize = "gossipsub-cluster-prefix-tracker-cache-size" + // validationInspectorClusterPrefixedTopicsReceivedCacheDecay = "gossipsub-cluster-prefix-tracker-cache-decay" + // validationInspectorClusterPrefixHardThreshold = "gossipsub-rpc-cluster-prefixed-hard-threshold" + // + // ihaveMaxSampleSize = "gossipsub-rpc-ihave-max-sample-size" + // ihaveMaxMessageIDSampleSize = "gossipsub-rpc-ihave-max-message-id-sample-size" + // controlMessageMaxSampleSize = "gossipsub-rpc-graft-and-prune-message-max-sample-size" + // iwantMaxSampleSize = "gossipsub-rpc-iwant-max-sample-size" + // iwantMaxMessageIDSampleSize = "gossipsub-rpc-iwant-max-message-id-sample-size" + // iwantCacheMissThreshold = "gossipsub-rpc-iwant-cache-miss-threshold" + // iwantCacheMissCheckSize = "gossipsub-rpc-iwant-cache-miss-check-size" + // iwantDuplicateMsgIDThreshold = "gossipsub-rpc-iwant-duplicate-message-id-threshold" + // rpcMessageMaxSampleSize = "gossipsub-rpc-message-max-sample-size" + // rpcMessageErrorThreshold = "gossipsub-rpc-message-error-threshold" // gossipsub metrics inspector - metricsInspectorNumberOfWorkers = "gossipsub-rpc-metrics-inspector-workers" - metricsInspectorCacheSize = "gossipsub-rpc-metrics-inspector-cache-size" + // metricsInspectorNumberOfWorkers = "gossipsub-rpc-metrics-inspector-workers" + // metricsInspectorCacheSize = "gossipsub-rpc-metrics-inspector-cache-size" alspDisabled = "alsp-disable-penalty" alspSpamRecordCacheSize = "alsp-spam-record-cache-size" @@ -119,21 +118,6 @@ func AllFlagNames() []string { lowWatermark, gracePeriod, silencePeriod, - peerScoring, - localMeshLogInterval, - rpcSentTrackerCacheSize, - rpcSentTrackerQueueCacheSize, - rpcSentTrackerNumOfWorkers, - scoreTracerInterval, - gossipSubRPCInspectorNotificationCacheSize, - validationInspectorNumberOfWorkers, - validationInspectorInspectMessageQueueCacheSize, - validationInspectorClusterPrefixedTopicsReceivedCacheSize, - validationInspectorClusterPrefixedTopicsReceivedCacheDecay, - validationInspectorClusterPrefixHardThreshold, - ihaveMaxSampleSize, - metricsInspectorNumberOfWorkers, - metricsInspectorCacheSize, alspDisabled, alspSpamRecordCacheSize, alspSpamRecordQueueSize, @@ -141,15 +125,32 @@ func AllFlagNames() []string { alspSyncEngineBatchRequestBaseProb, alspSyncEngineRangeRequestBaseProb, alspSyncEngineSyncRequestProb, - iwantMaxSampleSize, - iwantMaxMessageIDSampleSize, - ihaveMaxMessageIDSampleSize, - iwantCacheMissThreshold, - controlMessageMaxSampleSize, - iwantDuplicateMsgIDThreshold, - iwantCacheMissCheckSize, - rpcMessageMaxSampleSize, - rpcMessageErrorThreshold, + BuildFlagName(gossipSub, p2pconf.PeerScoringEnabledKey), + BuildFlagName(gossipSub, p2pconf.RpcTracerKey, p2pconf.LocalMeshLogIntervalKey), + BuildFlagName(gossipSub, p2pconf.RpcTracerKey, p2pconf.ScoreTracerIntervalKey), + BuildFlagName(gossipSub, p2pconf.RpcTracerKey, p2pconf.RPCSentTrackerCacheSizeKey), + BuildFlagName(gossipSub, p2pconf.RpcTracerKey, p2pconf.RPCSentTrackerQueueCacheSizeKey), + BuildFlagName(gossipSub, p2pconf.RpcTracerKey, p2pconf.RPCSentTrackerNumOfWorkersKey), + BuildFlagName(gossipSub, p2pconf.RpcInspectorKey, p2pconf.ValidationConfigKey, p2pconf.NumberOfWorkersKey), + BuildFlagName(gossipSub, p2pconf.RpcInspectorKey, p2pconf.ValidationConfigKey, p2pconf.QueueSizeKey), + BuildFlagName(gossipSub, p2pconf.RpcInspectorKey, p2pconf.ValidationConfigKey, p2pconf.ClusterPrefixedMessageConfigKey, p2pconf.TrackerCacheSizeKey), + BuildFlagName(gossipSub, p2pconf.RpcInspectorKey, p2pconf.ValidationConfigKey, p2pconf.ClusterPrefixedMessageConfigKey, p2pconf.TrackerCacheDecayKey), + BuildFlagName(gossipSub, p2pconf.RpcInspectorKey, p2pconf.ValidationConfigKey, p2pconf.ClusterPrefixedMessageConfigKey, p2pconf.HardThresholdKey), + BuildFlagName(gossipSub, p2pconf.RpcInspectorKey, p2pconf.MetricsConfigKey, p2pconf.NumberOfWorkersKey), + BuildFlagName(gossipSub, p2pconf.RpcInspectorKey, p2pconf.MetricsConfigKey, p2pconf.CacheSizeKey), + BuildFlagName(gossipSub, p2pconf.RpcInspectorKey, p2pconf.NotificationCacheSizeKey), + BuildFlagName(gossipSub, p2pconf.RpcInspectorKey, p2pconf.ValidationConfigKey, p2pconf.IHaveConfigKey, p2pconf.MaxSampleSizeKey), + BuildFlagName(gossipSub, p2pconf.RpcInspectorKey, p2pconf.ValidationConfigKey, p2pconf.IHaveConfigKey, p2pconf.MaxMessageIDSampleSizeKey), + BuildFlagName(gossipSub, p2pconf.RpcInspectorKey, p2pconf.ValidationConfigKey, p2pconf.GraftPruneMessageMaxSampleSizeKey), + BuildFlagName(gossipSub, p2pconf.RpcInspectorKey, p2pconf.ValidationConfigKey, p2pconf.IWantConfigKey, p2pconf.MaxSampleSizeKey), + BuildFlagName(gossipSub, p2pconf.RpcInspectorKey, p2pconf.ValidationConfigKey, p2pconf.IWantConfigKey, p2pconf.MaxMessageIDSampleSizeKey), + BuildFlagName(gossipSub, p2pconf.RpcInspectorKey, p2pconf.ValidationConfigKey, p2pconf.IWantConfigKey, p2pconf.CacheMissThresholdKey), + BuildFlagName(gossipSub, p2pconf.RpcInspectorKey, p2pconf.ValidationConfigKey, p2pconf.IWantConfigKey, p2pconf.CacheMissCheckSizeKey), + BuildFlagName(gossipSub, p2pconf.RpcInspectorKey, p2pconf.ValidationConfigKey, p2pconf.IWantConfigKey, p2pconf.DuplicateMsgIDThresholdKey), + BuildFlagName(gossipSub, p2pconf.RpcInspectorKey, p2pconf.ValidationConfigKey, p2pconf.MessageMaxSampleSizeKey), + BuildFlagName(gossipSub, p2pconf.RpcInspectorKey, p2pconf.ValidationConfigKey, p2pconf.MessageErrorThresholdKey), + BuildFlagName(gossipSub, p2pconf.SubscriptionProviderKey, p2pconf.UpdateIntervalKey), + BuildFlagName(gossipSub, p2pconf.SubscriptionProviderKey, p2pconf.CacheSizeKey), } for _, scope := range []string{systemScope, transientScope, protocolScope, peerScope, peerProtocolScope} { @@ -214,50 +215,42 @@ func InitializeNetworkFlags(flags *pflag.FlagSet, config *Config) { flags.Int(highWatermark, config.ConnectionManagerConfig.HighWatermark, "high watermarking for libp2p connection manager") flags.Duration(gracePeriod, config.ConnectionManagerConfig.GracePeriod, "grace period for libp2p connection manager") flags.Duration(silencePeriod, config.ConnectionManagerConfig.SilencePeriod, "silence period for libp2p connection manager") - flags.Bool(peerScoring, config.GossipSubConfig.PeerScoring, "enabling peer scoring on pubsub network") - flags.Duration(localMeshLogInterval, config.GossipSubConfig.LocalMeshLogInterval, "logging interval for local mesh in gossipsub") - flags.Duration( - scoreTracerInterval, - config.GossipSubConfig.ScoreTracerInterval, + flags.Bool(BuildFlagName(gossipSub, p2pconf.PeerScoringEnabledKey), config.GossipSub.PeerScoringEnabled, "enabling peer scoring on pubsub network") + flags.Duration(BuildFlagName(gossipSub, p2pconf.RpcTracerKey, p2pconf.LocalMeshLogIntervalKey), + config.GossipSub.RpcTracer.LocalMeshLogInterval, + "logging interval for local mesh in gossipsub tracer") + flags.Duration(BuildFlagName(gossipSub, p2pconf.RpcTracerKey, p2pconf.ScoreTracerIntervalKey), config.GossipSub.RpcTracer.ScoreTracerInterval, "logging interval for peer score tracer in gossipsub, set to 0 to disable") - flags.Uint32( - rpcSentTrackerCacheSize, - config.GossipSubConfig.RPCSentTrackerCacheSize, + flags.Uint32(BuildFlagName(gossipSub, p2pconf.RpcTracerKey, p2pconf.RPCSentTrackerCacheSizeKey), config.GossipSub.RpcTracer.RPCSentTrackerCacheSize, "cache size of the rpc sent tracker used by the gossipsub mesh tracer.") - flags.Uint32( - rpcSentTrackerQueueCacheSize, - config.GossipSubConfig.RPCSentTrackerQueueCacheSize, + flags.Uint32(BuildFlagName(gossipSub, p2pconf.RpcTracerKey, p2pconf.RPCSentTrackerQueueCacheSizeKey), config.GossipSub.RpcTracer.RPCSentTrackerQueueCacheSize, "cache size of the rpc sent tracker worker queue.") - flags.Int( - rpcSentTrackerNumOfWorkers, - config.GossipSubConfig.RpcSentTrackerNumOfWorkers, + flags.Int(BuildFlagName(gossipSub, p2pconf.RpcTracerKey, p2pconf.RPCSentTrackerNumOfWorkersKey), config.GossipSub.RpcTracer.RpcSentTrackerNumOfWorkers, "number of workers for the rpc sent tracker worker pool.") // gossipsub RPC control message validation limits used for validation configuration and rate limiting - flags.Int(validationInspectorNumberOfWorkers, - config.GossipSubConfig.GossipSubRPCInspectorsConfig.GossipSubRPCValidationInspectorConfigs.NumberOfWorkers, + flags.Int(BuildFlagName(gossipSub, p2pconf.RpcInspectorKey, p2pconf.ValidationConfigKey, p2pconf.NumberOfWorkersKey), + config.GossipSub.RpcInspector.Validation.NumberOfWorkers, "number of gossupsub RPC control message validation inspector component workers") - flags.Uint32(validationInspectorInspectMessageQueueCacheSize, - config.GossipSubConfig.GossipSubRPCInspectorsConfig.GossipSubRPCValidationInspectorConfigs.CacheSize, - "cache size for gossipsub RPC validation inspector events worker pool queue.") - flags.Uint32(validationInspectorClusterPrefixedTopicsReceivedCacheSize, - config.GossipSubConfig.GossipSubRPCInspectorsConfig.GossipSubRPCValidationInspectorConfigs.ClusterPrefixedControlMsgsReceivedCacheSize, + flags.Uint32(BuildFlagName(gossipSub, p2pconf.RpcInspectorKey, p2pconf.ValidationConfigKey, p2pconf.QueueSizeKey), + config.GossipSub.RpcInspector.Validation.QueueSize, + "queue size for gossipsub RPC validation inspector events worker pool queue.") + flags.Uint32(BuildFlagName(gossipSub, p2pconf.RpcInspectorKey, p2pconf.ValidationConfigKey, p2pconf.ClusterPrefixedMessageConfigKey, p2pconf.TrackerCacheSizeKey), + config.GossipSub.RpcInspector.Validation.ClusterPrefixedMessage.ControlMsgsReceivedCacheSize, "cache size for gossipsub RPC validation inspector cluster prefix received tracker.") - flags.Float64(validationInspectorClusterPrefixedTopicsReceivedCacheDecay, - config.GossipSubConfig.GossipSubRPCInspectorsConfig.GossipSubRPCValidationInspectorConfigs.ClusterPrefixedControlMsgsReceivedCacheDecay, + flags.Float64(BuildFlagName(gossipSub, p2pconf.RpcInspectorKey, p2pconf.ValidationConfigKey, p2pconf.ClusterPrefixedMessageConfigKey, p2pconf.TrackerCacheDecayKey), + config.GossipSub.RpcInspector.Validation.ClusterPrefixedMessage.ControlMsgsReceivedCacheDecay, "the decay value used to decay cluster prefix received topics received cached counters.") - flags.Float64(validationInspectorClusterPrefixHardThreshold, - config.GossipSubConfig.GossipSubRPCInspectorsConfig.GossipSubRPCValidationInspectorConfigs.ClusterPrefixHardThreshold, + flags.Float64(BuildFlagName(gossipSub, p2pconf.RpcInspectorKey, p2pconf.ValidationConfigKey, p2pconf.ClusterPrefixedMessageConfigKey, p2pconf.HardThresholdKey), + config.GossipSub.RpcInspector.Validation.ClusterPrefixedMessage.HardThreshold, "the maximum number of cluster-prefixed control messages allowed to be processed when the active cluster id is unset or a mismatch is detected, exceeding this threshold will result in node penalization by gossipsub.") // gossipsub RPC control message metrics observer inspector configuration - flags.Int(metricsInspectorNumberOfWorkers, - config.GossipSubConfig.GossipSubRPCInspectorsConfig.GossipSubRPCMetricsInspectorConfigs.NumberOfWorkers, - "cache size for gossipsub RPC metrics inspector events worker pool queue.") - flags.Uint32(metricsInspectorCacheSize, - config.GossipSubConfig.GossipSubRPCInspectorsConfig.GossipSubRPCMetricsInspectorConfigs.CacheSize, + flags.Int(BuildFlagName(gossipSub, p2pconf.RpcInspectorKey, p2pconf.MetricsConfigKey, p2pconf.NumberOfWorkersKey), + config.GossipSub.RpcInspector.Metrics.NumberOfWorkers, + "number of workers for gossipsub RPC metrics inspector queue.") + flags.Uint32(BuildFlagName(gossipSub, p2pconf.RpcInspectorKey, p2pconf.MetricsConfigKey, p2pconf.CacheSizeKey), config.GossipSub.RpcInspector.Metrics.CacheSize, "cache size for gossipsub RPC metrics inspector events worker pool.") // networking event notifications - flags.Uint32(gossipSubRPCInspectorNotificationCacheSize, - config.GossipSubConfig.GossipSubRPCInspectorsConfig.GossipSubRPCInspectorNotificationCacheSize, + flags.Uint32(BuildFlagName(gossipSub, p2pconf.RpcInspectorKey, p2pconf.NotificationCacheSizeKey), config.GossipSub.RpcInspector.NotificationCacheSize, "cache size for notification events from gossipsub rpc inspector") // application layer spam prevention (alsp) protocol flags.Bool(alspDisabled, config.AlspConfig.DisablePenalty, "disable the penalty mechanism of the alsp protocol. default value (recommended) is false") @@ -274,39 +267,41 @@ func InitializeNetworkFlags(flags *pflag.FlagSet, config *Config) { "base probability of creating a misbehavior report for a range request message") flags.Float32(alspSyncEngineSyncRequestProb, config.AlspConfig.SyncEngine.SyncRequestProb, "probability of creating a misbehavior report for a sync request message") - flags.Int(ihaveMaxSampleSize, - config.GossipSubConfig.GossipSubRPCInspectorsConfig.GossipSubRPCValidationInspectorConfigs.IHaveRPCInspectionConfig.MaxSampleSize, + flags.Int(BuildFlagName(gossipSub, p2pconf.RpcInspectorKey, p2pconf.ValidationConfigKey, p2pconf.IHaveConfigKey, p2pconf.MaxSampleSizeKey), + config.GossipSub.RpcInspector.Validation.IHave.MaxSampleSize, "max number of ihaves to sample when performing validation") - flags.Int(ihaveMaxMessageIDSampleSize, - config.GossipSubConfig.GossipSubRPCInspectorsConfig.GossipSubRPCValidationInspectorConfigs.IHaveRPCInspectionConfig.MaxMessageIDSampleSize, + flags.Int(BuildFlagName(gossipSub, p2pconf.RpcInspectorKey, p2pconf.ValidationConfigKey, p2pconf.IHaveConfigKey, p2pconf.MaxMessageIDSampleSizeKey), + config.GossipSub.RpcInspector.Validation.IHave.MaxMessageIDSampleSize, "max number of message ids to sample when performing validation per ihave") - flags.Int(controlMessageMaxSampleSize, - config.GossipSubConfig.GossipSubRPCInspectorsConfig.GossipSubRPCValidationInspectorConfigs.GraftPruneMessageMaxSampleSize, + flags.Int(BuildFlagName(gossipSub, p2pconf.RpcInspectorKey, p2pconf.ValidationConfigKey, p2pconf.GraftPruneMessageMaxSampleSizeKey), + config.GossipSub.RpcInspector.Validation.GraftPruneMessageMaxSampleSize, "max number of control messages to sample when performing validation on GRAFT and PRUNE message types") - flags.Uint(iwantMaxSampleSize, - config.GossipSubConfig.GossipSubRPCInspectorsConfig.GossipSubRPCValidationInspectorConfigs.IWantRPCInspectionConfig.MaxSampleSize, + flags.Uint(BuildFlagName(gossipSub, p2pconf.RpcInspectorKey, p2pconf.ValidationConfigKey, p2pconf.IWantConfigKey, p2pconf.MaxSampleSizeKey), + config.GossipSub.RpcInspector.Validation.IWant.MaxSampleSize, "max number of iwants to sample when performing validation") - flags.Int(iwantMaxMessageIDSampleSize, - config.GossipSubConfig.GossipSubRPCInspectorsConfig.GossipSubRPCValidationInspectorConfigs.IWantRPCInspectionConfig.MaxMessageIDSampleSize, + flags.Int(BuildFlagName(gossipSub, p2pconf.RpcInspectorKey, p2pconf.ValidationConfigKey, p2pconf.IWantConfigKey, p2pconf.MaxMessageIDSampleSizeKey), + config.GossipSub.RpcInspector.Validation.IWant.MaxMessageIDSampleSize, "max number of message ids to sample when performing validation per iwant") - flags.Float64(iwantCacheMissThreshold, - config.GossipSubConfig.GossipSubRPCInspectorsConfig.GossipSubRPCValidationInspectorConfigs.IWantRPCInspectionConfig.CacheMissThreshold, + flags.Float64(BuildFlagName(gossipSub, p2pconf.RpcInspectorKey, p2pconf.ValidationConfigKey, p2pconf.IWantConfigKey, p2pconf.CacheMissThresholdKey), + config.GossipSub.RpcInspector.Validation.IWant.CacheMissThreshold, "max number of iwants to sample when performing validation") - flags.Int(iwantCacheMissCheckSize, - config.GossipSubConfig.GossipSubRPCInspectorsConfig.GossipSubRPCValidationInspectorConfigs.IWantRPCInspectionConfig.CacheMissCheckSize, + flags.Int(BuildFlagName(gossipSub, p2pconf.RpcInspectorKey, p2pconf.ValidationConfigKey, p2pconf.IWantConfigKey, p2pconf.CacheMissCheckSizeKey), + config.GossipSub.RpcInspector.Validation.IWant.CacheMissCheckSize, "the iWants size at which message id cache misses will be checked") - flags.Float64(iwantDuplicateMsgIDThreshold, - config.GossipSubConfig.GossipSubRPCInspectorsConfig.GossipSubRPCValidationInspectorConfigs.IWantRPCInspectionConfig.DuplicateMsgIDThreshold, + flags.Float64(BuildFlagName(gossipSub, p2pconf.RpcInspectorKey, p2pconf.ValidationConfigKey, p2pconf.IWantConfigKey, p2pconf.DuplicateMsgIDThresholdKey), + config.GossipSub.RpcInspector.Validation.IWant.DuplicateMsgIDThreshold, "max allowed duplicate message IDs in a single iWant control message") - - flags.Int(rpcMessageMaxSampleSize, config.GossipSubConfig.GossipSubRPCInspectorsConfig.GossipSubRPCValidationInspectorConfigs.RpcMessageMaxSampleSize, "the max sample size used for RPC message validation. If the total number of RPC messages exceeds this value a sample will be taken but messages will not be truncated") - flags.Int(rpcMessageErrorThreshold, config.GossipSubConfig.GossipSubRPCInspectorsConfig.GossipSubRPCValidationInspectorConfigs.RpcMessageErrorThreshold, "the threshold at which an error will be returned if the number of invalid RPC messages exceeds this value") - flags.Duration( - gossipSubSubscriptionProviderUpdateInterval, config.GossipSubConfig.SubscriptionProvider.SubscriptionUpdateInterval, + flags.Int(BuildFlagName(gossipSub, p2pconf.RpcInspectorKey, p2pconf.ValidationConfigKey, p2pconf.MessageMaxSampleSizeKey), + config.GossipSub.RpcInspector.Validation.MessageMaxSampleSize, + "the max sample size used for RPC message validation. If the total number of RPC messages exceeds this value a sample will be taken but messages will not be truncated") + flags.Int(BuildFlagName(gossipSub, p2pconf.RpcInspectorKey, p2pconf.ValidationConfigKey, p2pconf.MessageErrorThresholdKey), + config.GossipSub.RpcInspector.Validation.MessageErrorThreshold, + "the threshold at which an error will be returned if the number of invalid RPC messages exceeds this value") + flags.Duration(BuildFlagName(gossipSub, p2pconf.SubscriptionProviderKey, p2pconf.UpdateIntervalKey), + config.GossipSub.SubscriptionProvider.UpdateInterval, "interval for updating the list of subscribed topics for all peers in the gossipsub, recommended value is a few minutes") - flags.Uint32( - gossipSubSubscriptionProviderCacheSize, - config.GossipSubConfig.SubscriptionProvider.CacheSize, + flags.Uint32(BuildFlagName(gossipSub, p2pconf.SubscriptionProviderKey, p2pconf.CacheSizeKey), + config.GossipSub.SubscriptionProvider.CacheSize, "size of the cache that keeps the list of topics each peer has subscribed to, recommended size is 10x the number of authorized nodes") } @@ -381,10 +376,13 @@ func SetAliases(conf *viper.Viper) error { for _, flagName := range AllFlagNames() { fullKey, ok := m[flagName] if !ok { - return fmt.Errorf( - "invalid network configuration missing configuration key flag name %s check config file and cli flags", flagName) + return fmt.Errorf("invalid network configuration missing configuration key flag name %s check config file and cli flags", flagName) } conf.RegisterAlias(fullKey, flagName) } return nil } + +func BuildFlagName(keys ...string) string { + return strings.Join(keys, "-") +} diff --git a/network/netconf/flags_test.go b/network/netconf/flags_test.go new file mode 100644 index 00000000000..984dc68936e --- /dev/null +++ b/network/netconf/flags_test.go @@ -0,0 +1,51 @@ +package netconf_test + +import ( + "testing" + + "github.com/onflow/flow-go/network/netconf" +) + +// TestBuildFlagName tests the BuildFlagName function for various cases +func TestBuildFlagName(t *testing.T) { + tests := []struct { + name string + keys []string + expected string + }{ + { + name: "Single key", + keys: []string{"key1"}, + expected: "key1", + }, + { + name: "Two keys", + keys: []string{"key1", "key2"}, + expected: "key1-key2", + }, + { + name: "Multiple keys", + keys: []string{"key1", "key2", "key3"}, + expected: "key1-key2-key3", + }, + { + name: "No keys", + keys: []string{}, + expected: "", + }, + { + name: "Key with spaces", + keys: []string{"key 1", "key 2"}, + expected: "key 1-key 2", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := netconf.BuildFlagName(tt.keys...) + if result != tt.expected { + t.Errorf("BuildFlagName(%v) = %v, want %v", tt.keys, result, tt.expected) + } + }) + } +} diff --git a/network/p2p/builder.go b/network/p2p/builder.go index f9eebad66dc..ab1e025d64f 100644 --- a/network/p2p/builder.go +++ b/network/p2p/builder.go @@ -26,7 +26,7 @@ type GossipSubFactoryFunc func(context.Context, zerolog.Logger, host.Host, PubSu type CreateNodeFunc func(zerolog.Logger, host.Host, ProtocolPeerCache, PeerManager, *DisallowListCacheConfig) LibP2PNode type GossipSubAdapterConfigFunc func(*BasePubSubAdapterConfig) PubSubAdapterConfig -// GossipSubBuilder provides a builder pattern for creating a GossipSub pubsub system. +// GossipSubBuilder provides a builder pattern for creating a GossipSubParameters pubsub system. type GossipSubBuilder interface { // SetHost sets the host of the builder. // If the host has already been set, a fatal error is logged. @@ -44,7 +44,7 @@ type GossipSubBuilder interface { // We expect the node to initialize with a default gossipsub config. Hence, this function overrides the default config. SetGossipSubConfigFunc(GossipSubAdapterConfigFunc) - // EnableGossipSubScoringWithOverride enables peer scoring for the GossipSub pubsub system with the given override. + // EnableGossipSubScoringWithOverride enables peer scoring for the GossipSubParameters pubsub system with the given override. // Any existing peer scoring config attribute that is set in the override will override the default peer scoring config. // Anything that is left to nil or zero value in the override will be ignored and the default value will be used. // Note: it is not recommended to override the default peer scoring config in production unless you know what you are doing. @@ -74,15 +74,15 @@ type GossipSubBuilder interface { // It is NOT recommended to override the default RPC inspector suite factory in production unless you know what you are doing. OverrideDefaultRpcInspectorSuiteFactory(GossipSubRpcInspectorSuiteFactoryFunc) - // Build creates a new GossipSub pubsub system. - // It returns the newly created GossipSub pubsub system and any errors encountered during its creation. + // Build creates a new GossipSubParameters pubsub system. + // It returns the newly created GossipSubParameters pubsub system and any errors encountered during its creation. // // Arguments: // - context.Context: the irrecoverable context of the node. // // Returns: - // - PubSubAdapter: a GossipSub pubsub system for the libp2p node. - // - error: if an error occurs during the creation of the GossipSub pubsub system, it is returned. Otherwise, nil is returned. + // - PubSubAdapter: a GossipSubParameters pubsub system for the libp2p node. + // - error: if an error occurs during the creation of the GossipSubParameters pubsub system, it is returned. Otherwise, nil is returned. // Note that on happy path, the returned error is nil. Any error returned is unexpected and should be handled as irrecoverable. Build(irrecoverable.SignalerContext) (PubSubAdapter, error) } @@ -122,7 +122,7 @@ type NodeBuilder interface { SetConnectionGater(ConnectionGater) NodeBuilder SetRoutingSystem(func(context.Context, host.Host) (routing.Routing, error)) NodeBuilder - // EnableGossipSubScoringWithOverride enables peer scoring for the GossipSub pubsub system with the given override. + // EnableGossipSubScoringWithOverride enables peer scoring for the GossipSubParameters pubsub system with the given override. // Any existing peer scoring config attribute that is set in the override will override the default peer scoring config. // Anything that is left to nil or zero value in the override will be ignored and the default value will be used. // Note: it is not recommended to override the default peer scoring config in production unless you know what you are doing. @@ -158,13 +158,13 @@ type PeerScoringConfigOverride struct { // DecayInterval is the interval over which we decay the effect of past behavior, so that // a good or bad behavior will not have a permanent effect on the penalty. It is also the interval - // that GossipSub uses to refresh the scores of all peers. + // that GossipSubParameters uses to refresh the scores of all peers. // Override criteria: if the value is not zero, it will override the default decay interval. // If the value is zero, the default decay interval is used. DecayInterval time.Duration } -// PeerScoringConfigNoOverride is a default peer scoring configuration for a GossipSub pubsub system. +// PeerScoringConfigNoOverride is a default peer scoring configuration for a GossipSubParameters pubsub system. // It is set to nil, which means that no override is done to the default peer scoring configuration. // It is the recommended way to use the default peer scoring configuration. var PeerScoringConfigNoOverride = (*PeerScoringConfigOverride)(nil) diff --git a/network/p2p/inspector/validation/control_message_validation_inspector.go b/network/p2p/inspector/validation/control_message_validation_inspector.go index 1e573508f06..f8b20ac638f 100644 --- a/network/p2p/inspector/validation/control_message_validation_inspector.go +++ b/network/p2p/inspector/validation/control_message_validation_inspector.go @@ -710,7 +710,7 @@ func (c *ControlMsgValidationInspector) validateTopic(from peer.ID, topic channe // - channels.UnknownClusterIDErr: if the topic contains a cluster ID prefix that is not in the active cluster IDs list. // // In the case where an ErrActiveClusterIdsNotSet or UnknownClusterIDErr is encountered and the cluster prefixed topic received -// tracker for the peer is less than or equal to the configured ClusterPrefixHardThreshold an error will only be logged and not returned. +// tracker for the peer is less than or equal to the configured HardThreshold an error will only be logged and not returned. // At the point where the hard threshold is crossed the error will be returned and the sender will start to be penalized. // Any errors encountered while incrementing or loading the cluster prefixed control message gauge for a peer will result in an irrecoverable error being thrown, these // errors are unexpected and irrecoverable indicating a bug. @@ -782,7 +782,7 @@ func (c *ControlMsgValidationInspector) getFlowIdentifier(peerID peer.ID) (flow. } // checkClusterPrefixHardThreshold returns true if the cluster prefix received tracker count is less than -// the configured ClusterPrefixHardThreshold, false otherwise. +// the configured HardThreshold, false otherwise. // If any error is encountered while loading from the tracker this func will throw an error on the signaler context, these errors // are unexpected and irrecoverable indicating a bug. func (c *ControlMsgValidationInspector) checkClusterPrefixHardThreshold(nodeID flow.Identifier) bool { diff --git a/network/p2p/p2pbuilder/gossipsub/gossipSubBuilder.go b/network/p2p/p2pbuilder/gossipsub/gossipSubBuilder.go index 23b9b931e17..cc1fa585f92 100644 --- a/network/p2p/p2pbuilder/gossipsub/gossipSubBuilder.go +++ b/network/p2p/p2pbuilder/gossipsub/gossipSubBuilder.go @@ -30,7 +30,7 @@ import ( "github.com/onflow/flow-go/utils/logging" ) -// The Builder struct is used to configure and create a new GossipSub pubsub system. +// The Builder struct is used to configure and create a new GossipSubParameters pubsub system. type Builder struct { networkType network.NetworkingType sporkId flow.Identifier @@ -92,7 +92,7 @@ func (g *Builder) SetGossipSubConfigFunc(gossipSubConfigFunc p2p.GossipSubAdapte g.gossipSubConfigFunc = gossipSubConfigFunc } -// EnableGossipSubScoringWithOverride enables peer scoring for the GossipSub pubsub system with the given override. +// EnableGossipSubScoringWithOverride enables peer scoring for the GossipSubParameters pubsub system with the given override. // Any existing peer scoring config attribute that is set in the override will override the default peer scoring config. // Anything that is left to nil or zero value in the override will be ignored and the default value will be used. // Note: it is not recommended to override the default peer scoring config in production unless you know what you are doing. @@ -280,15 +280,15 @@ func defaultInspectorSuite(rpcTracker p2p.RpcControlTracking) p2p.GossipSubRpcIn } } -// Build creates a new GossipSub pubsub system. -// It returns the newly created GossipSub pubsub system and any errors encountered during its creation. +// Build creates a new GossipSubParameters pubsub system. +// It returns the newly created GossipSubParameters pubsub system and any errors encountered during its creation. // Arguments: // - ctx: the irrecoverable context of the node. // // Returns: -// - p2p.PubSubAdapter: a GossipSub pubsub system for the libp2p node. -// - p2p.PeerScoreTracer: a peer score tracer for the GossipSub pubsub system (if enabled, otherwise nil). -// - error: if an error occurs during the creation of the GossipSub pubsub system, it is returned. Otherwise, nil is returned. +// - p2p.PubSubAdapter: a GossipSubParameters pubsub system for the libp2p node. +// - p2p.PeerScoreTracer: a peer score tracer for the GossipSubParameters pubsub system (if enabled, otherwise nil). +// - error: if an error occurs during the creation of the GossipSubParameters pubsub system, it is returned. Otherwise, nil is returned. // Note that on happy path, the returned error is nil. Any error returned is unexpected and should be handled as irrecoverable. func (g *Builder) Build(ctx irrecoverable.SignalerContext) (p2p.PubSubAdapter, error) { // placeholder for the gossipsub pubsub system that will be created (so that it can be passed around even diff --git a/network/p2p/p2pbuilder/libp2pNodeBuilder.go b/network/p2p/p2pbuilder/libp2pNodeBuilder.go index 24996c9b017..dc2a5bf1d4d 100644 --- a/network/p2p/p2pbuilder/libp2pNodeBuilder.go +++ b/network/p2p/p2pbuilder/libp2pNodeBuilder.go @@ -158,7 +158,7 @@ func (builder *LibP2PNodeBuilder) SetGossipSubFactory(gf p2p.GossipSubFactoryFun return builder } -// EnableGossipSubScoringWithOverride enables peer scoring for the GossipSub pubsub system with the given override. +// EnableGossipSubScoringWithOverride enables peer scoring for the GossipSubParameters pubsub system with the given override. // Any existing peer scoring config attribute that is set in the override will override the default peer scoring config. // Anything that is left to nil or zero value in the override will be ignored and the default value will be used. // Note: it is not recommended to override the default peer scoring config in production unless you know what you are doing. @@ -423,7 +423,7 @@ func DefaultNodeBuilder( role string, connGaterCfg *p2pconfig.ConnectionGaterConfig, peerManagerCfg *p2pconfig.PeerManagerConfig, - gossipCfg *p2pconf.GossipSubConfig, + gossipCfg *p2pconf.GossipSubParameters, rpcInspectorCfg *p2pconf.RpcInspectorParameters, rCfg *p2pconf.ResourceManagerConfig, uniCfg *p2pconfig.UnicastConfig, diff --git a/network/p2p/p2pconf/gossipsub.go b/network/p2p/p2pconf/gossipsub.go index 2fd681e93ea..33a30aeec99 100644 --- a/network/p2p/p2pconf/gossipsub.go +++ b/network/p2p/p2pconf/gossipsub.go @@ -2,8 +2,6 @@ package p2pconf import ( "time" - - "github.com/onflow/flow-go/network/p2p/scoring" ) // ResourceManagerConfig returns the resource manager configuration for the libp2p node. @@ -23,13 +21,13 @@ type ResourceManagerOverrideScope struct { // Transient is the limit for the resource at the transient scope. Transient limits are used for resources that have not fully established and are under negotiation. Transient ResourceManagerOverrideLimit `mapstructure:"transient"` - // Protocol is the limit for the resource at the protocol scope, e.g., DHT, GossipSub, etc. It dictates the maximum allowed resource across all peers for that protocol. + // Protocol is the limit for the resource at the protocol scope, e.g., DHT, GossipSubParameters, etc. It dictates the maximum allowed resource across all peers for that protocol. Protocol ResourceManagerOverrideLimit `mapstructure:"protocol"` // Peer is the limit for the resource at the peer scope. It dictates the maximum allowed resource for a specific peer. Peer ResourceManagerOverrideLimit `mapstructure:"peer"` - // Connection is the limit for the resource for a pair of (peer, protocol), e.g., (peer1, DHT), (peer1, GossipSub), etc. It dictates the maximum allowed resource for a protocol and a peer. + // Connection is the limit for the resource for a pair of (peer, protocol), e.g., (peer1, DHT), (peer1, GossipSubParameters), etc. It dictates the maximum allowed resource for a protocol and a peer. PeerProtocol ResourceManagerOverrideLimit `mapstructure:"peer-protocol"` } @@ -56,31 +54,59 @@ type ResourceManagerOverrideLimit struct { Memory int `validate:"gte=0" mapstructure:"memory-bytes"` } -// GossipSubConfig keys. +// GossipSubParameters keys. const ( RpcInspectorKey = "rpc-inspector" RpcTracerKey = "rpc-tracer" - PeerScoringKey = "peer-scoring-enabled" + PeerScoringEnabledKey = "peer-scoring-enabled" ScoreParamsKey = "scoring-parameters" SubscriptionProviderKey = "subscription-provider" ) -var GossipSubConfigKeys = []string{RpcInspectorKey, RpcTracerKey, PeerScoringKey, ScoreParamsKey, SubscriptionProviderKey} +var GossipSubConfigKeys = []string{RpcInspectorKey, RpcTracerKey, PeerScoringEnabledKey, ScoreParamsKey, SubscriptionProviderKey} -// GossipSubConfig is the configuration for the GossipSub pubsub implementation. -type GossipSubConfig struct { +// GossipSubParameters is the configuration for the GossipSubParameters pubsub implementation. +type GossipSubParameters struct { // RpcInspectorParameters configuration for all gossipsub RPC control message inspectors. RpcInspector RpcInspectorParameters `mapstructure:"rpc-inspector"` - // GossipSubTracerParameters is the configuration for the gossipsub tracer. GossipSub tracer is used to trace the local mesh events and peer scores. + // GossipSubTracerParameters is the configuration for the gossipsub tracer. GossipSubParameters tracer is used to trace the local mesh events and peer scores. RpcTracer GossipSubTracerParameters `mapstructure:"rpc-tracer"` - // ScoringParameters is whether to enable GossipSub peer scoring. - PeerScoringSwitch bool `mapstructure:"peer-scoring-enabled"` + // ScoringParameters is whether to enable GossipSubParameters peer scoring. + PeerScoringEnabled bool `mapstructure:"peer-scoring-enabled"` SubscriptionProvider SubscriptionProviderParameters `mapstructure:"subscription-provider"` - ScoringParameters scoring.Parameters `mapstructure:"scoring-parameters"` + ScoringParameters ScoringParameters `mapstructure:"scoring-parameters"` +} + +// Parameters are the parameters for the score option. +// Parameters are "numerical values" that are used to compute or build components that compute the score of a peer in GossipSub system. +type ScoringParameters struct { + AppSpecificScore AppSpecificScoreRegistryParams `validate:"required" mapstructure:"app-specific-score"` + // SpamRecordCacheSize is size of the cache used to store the spam records of peers. + // The spam records are used to penalize peers that send invalid messages. + SpamRecordCacheSize uint32 `validate:"gt=0" mapstructure:"spam-record-cache-size"` + + // DecayInterval is the interval at which the counters associated with a peer behavior in GossipSub system are decayed. + DecayInterval time.Duration `validate:"gt=0s" mapstructure:"decay-interval"` +} + +// AppSpecificScoreRegistryParams is the parameters for the GossipSubAppSpecificScoreRegistry. +// Parameters are "numerical values" that are used to compute or build components that compute or maintain the application specific score of peers. +type AppSpecificScoreRegistryParams struct { + // ScoreUpdateWorkerNum is the number of workers in the worker pool for handling the application specific score update of peers in a non-blocking way. + ScoreUpdateWorkerNum int `validate:"gt=0" mapstructure:"score-update-worker-num"` + + // ScoreUpdateRequestQueueSize is the size of the worker pool for handling the application specific score update of peers in a non-blocking way. + ScoreUpdateRequestQueueSize uint32 `validate:"gt=0" mapstructure:"score-update-request-queue-size"` + + // ScoreTTL is the time to live of the application specific score of a peer; the registry keeps a cached copy of the + // application specific score of a peer for this duration. When the duration expires, the application specific score + // of the peer is updated asynchronously. As long as the update is in progress, the cached copy of the application + // specific score of the peer is used even if it is expired. + ScoreTTL time.Duration `validate:"required" mapstructure:"score-ttl"` } // SubscriptionProviderParameters keys. @@ -118,7 +144,7 @@ var TracerParametersKeys = []string{LocalMeshLogIntervalKey, RPCSentTrackerQueueCacheSizeKey, RPCSentTrackerNumOfWorkersKey} -// GossipSubTracerParameters is the config for the gossipsub tracer. GossipSub tracer is used to trace the local mesh events and peer scores. +// GossipSubTracerParameters is the config for the gossipsub tracer. GossipSubParameters tracer is used to trace the local mesh events and peer scores. type GossipSubTracerParameters struct { // LocalMeshLogInterval is the interval at which the local mesh is logged. LocalMeshLogInterval time.Duration `validate:"gt=0s" mapstructure:"local-mesh-logging-interval"` diff --git a/network/p2p/p2pconf/gossipsub_rpc_inspectors.go b/network/p2p/p2pconf/gossipsub_rpc_inspectors.go index 03d4a2f55a7..ab5100dc8da 100644 --- a/network/p2p/p2pconf/gossipsub_rpc_inspectors.go +++ b/network/p2p/p2pconf/gossipsub_rpc_inspectors.go @@ -7,8 +7,6 @@ const ( NotificationCacheSizeKey = "notification-cache-size" ) -var RpcInspectorParametersKeys = []string{ValidationConfigKey, MetricsConfigKey, NotificationCacheSizeKey} - // RpcInspectorParameters contains the "numerical values" for the gossipsub RPC control message inspectors parameters. type RpcInspectorParameters struct { // RpcValidationInspector control message validation inspector validation configuration and limits. @@ -27,18 +25,9 @@ const ( QueueSizeKey = "queue-size" GraftPruneMessageMaxSampleSizeKey = "graft-and-prune-message-max-sample-size" MessageMaxSampleSizeKey = "message-max-sample-size" - MessageErrorThresholdKey = "message-error-threshold" + MessageErrorThresholdKey = "error-threshold" ) -var RpcValidationInspectorParamterKeys = []string{ClusterPrefixedMessageConfigKey, - IWantConfigKey, - IHaveConfigKey, - NumberOfWorkersKey, - QueueSizeKey, - GraftPruneMessageMaxSampleSizeKey, - MessageMaxSampleSizeKey, - MessageErrorThresholdKey} - // RpcValidationInspector validation limits used for gossipsub RPC control message inspection. type RpcValidationInspector struct { ClusterPrefixedMessage ClusterPrefixedMessageInspectionParameters `mapstructure:"cluster-prefixed-messages"` @@ -54,7 +43,7 @@ type RpcValidationInspector struct { // RPCMessageMaxSampleSize the max sample size used for RPC message validation. If the total number of RPC messages exceeds this value a sample will be taken but messages will not be truncated. MessageMaxSampleSize int `validate:"gte=1000" mapstructure:"message-max-sample-size"` // RPCMessageErrorThreshold the threshold at which an error will be returned if the number of invalid RPC messages exceeds this value. - MessageErrorThreshold int `validate:"gte=500" mapstructure:"message-error-threshold"` + MessageErrorThreshold int `validate:"gte=500" mapstructure:"error-threshold"` } const ( @@ -65,8 +54,6 @@ const ( DuplicateMsgIDThresholdKey = "duplicate-message-id-threshold" ) -var IWantRPCInspectionParametersKeys = []string{MaxSampleSizeKey, MaxMessageIDSampleSizeKey, CacheMissThresholdKey, CacheMissCheckSizeKey, DuplicateMsgIDThresholdKey} - // IWantRPCInspectionParameters contains the "numerical values" for the iwant rpc control message inspection. type IWantRPCInspectionParameters struct { // MaxSampleSize max inspection sample size to use. If the total number of iWant control messages @@ -85,8 +72,6 @@ type IWantRPCInspectionParameters struct { DuplicateMsgIDThreshold float64 `validate:"gt=0" mapstructure:"duplicate-message-id-threshold"` } -var IHaveRPCInspectionConfigKeys = []string{MaxSampleSizeKey, MaxMessageIDSampleSizeKey} - // IHaveRpcInspectionParameters contains the "numerical values" for ihave rpc control inspection. type IHaveRpcInspectionParameters struct { // MaxSampleSize max inspection sample size to use. If the number of ihave messages exceeds this configured value @@ -103,28 +88,24 @@ const ( TrackerCacheDecayKey = "tracker-cache-decay" ) -var ClusterPrefixedMessageParameterKeys = []string{HardThresholdKey, TrackerCacheSizeKey, TrackerCacheDecayKey} - // ClusterPrefixedMessageInspectionParameters contains the "numerical values" for cluster prefixed control message inspection. type ClusterPrefixedMessageInspectionParameters struct { - // ClusterPrefixHardThreshold the upper bound on the amount of cluster prefixed control messages that will be processed + // HardThreshold the upper bound on the amount of cluster prefixed control messages that will be processed // before a node starts to get penalized. This allows LN nodes to process some cluster prefixed control messages during startup // when the cluster ID's provider is set asynchronously. It also allows processing of some stale messages that may be sent by nodes // that fall behind in the protocol. After the amount of cluster prefixed control messages processed exceeds this threshold the node // will be pushed to the edge of the network mesh. - ClusterPrefixHardThreshold float64 `validate:"gte=0" mapstructure:"hard-threshold"` - // ClusterPrefixedControlMsgsReceivedCacheSize size of the cache used to track the amount of cluster prefixed topics received by peers. - ClusterPrefixedControlMsgsReceivedCacheSize uint32 `validate:"gt=0" mapstructure:"tracker-cache-size"` - // ClusterPrefixedControlMsgsReceivedCacheDecay decay val used for the geometric decay of cache counters used to keep track of cluster prefixed topics received by peers. - ClusterPrefixedControlMsgsReceivedCacheDecay float64 `validate:"gt=0" mapstructure:"tracker-cache-decay"` + HardThreshold float64 `validate:"gte=0" mapstructure:"hard-threshold"` + // ControlMsgsReceivedCacheSize size of the cache used to track the amount of cluster prefixed topics received by peers. + ControlMsgsReceivedCacheSize uint32 `validate:"gt=0" mapstructure:"tracker-cache-size"` + // ControlMsgsReceivedCacheDecay decay val used for the geometric decay of cache counters used to keep track of cluster prefixed topics received by peers. + ControlMsgsReceivedCacheDecay float64 `validate:"gt=0" mapstructure:"tracker-cache-decay"` } const ( NumberOfWorkersKey = "workers" ) -var RpcMetricsInspectorConfigsKeys = []string{NumberOfWorkersKey, CacheSizeKey} - // RpcMetricsInspectorConfigs contains the "numerical values" for the gossipsub RPC control message metrics inspectors parameters. type RpcMetricsInspectorConfigs struct { // NumberOfWorkers number of worker pool workers. diff --git a/network/p2p/scoring/registry.go b/network/p2p/scoring/registry.go index ec00648f37f..1f4d519df40 100644 --- a/network/p2p/scoring/registry.go +++ b/network/p2p/scoring/registry.go @@ -18,6 +18,7 @@ import ( "github.com/onflow/flow-go/network/p2p" netcache "github.com/onflow/flow-go/network/p2p/cache" p2pmsg "github.com/onflow/flow-go/network/p2p/message" + "github.com/onflow/flow-go/network/p2p/p2pconf" "github.com/onflow/flow-go/network/p2p/p2plogging" "github.com/onflow/flow-go/utils/logging" ) @@ -113,26 +114,10 @@ type GossipSubAppSpecificScoreRegistry struct { appScoreUpdateWorkerPool *worker.Pool[peer.ID] } -// AppSpecificScoreRegistryParams is the parameters for the GossipSubAppSpecificScoreRegistry. -// Parameters are "numerical values" that are used to compute or build components that compute or maintain the application specific score of peers. -type AppSpecificScoreRegistryParams struct { - // ScoreUpdateWorkerNum is the number of workers in the worker pool for handling the application specific score update of peers in a non-blocking way. - ScoreUpdateWorkerNum int `validate:"gt=0" mapstructure:"score-update-worker-num"` - - // ScoreUpdateRequestQueueSize is the size of the worker pool for handling the application specific score update of peers in a non-blocking way. - ScoreUpdateRequestQueueSize uint32 `validate:"gt=0" mapstructure:"score-update-request-queue-size"` - - // ScoreTTL is the time to live of the application specific score of a peer; the registry keeps a cached copy of the - // application specific score of a peer for this duration. When the duration expires, the application specific score - // of the peer is updated asynchronously. As long as the update is in progress, the cached copy of the application - // specific score of the peer is used even if it is expired. - ScoreTTL time.Duration `validate:"required" mapstructure:"score-ttl"` -} - // GossipSubAppSpecificScoreRegistryConfig is the configuration for the GossipSubAppSpecificScoreRegistry. // Configurations are the "union of parameters and other components" that are used to compute or build components that compute or maintain the application specific score of peers. type GossipSubAppSpecificScoreRegistryConfig struct { - Parameters AppSpecificScoreRegistryParams `validate:"required"` + Parameters p2pconf.AppSpecificScoreRegistryParams `validate:"required"` Logger zerolog.Logger `validate:"required"` diff --git a/network/p2p/scoring/score_option.go b/network/p2p/scoring/score_option.go index cf3c4c5f28d..79625e7df7e 100644 --- a/network/p2p/scoring/score_option.go +++ b/network/p2p/scoring/score_option.go @@ -15,6 +15,7 @@ import ( "github.com/onflow/flow-go/network/channels" "github.com/onflow/flow-go/network/p2p" netcache "github.com/onflow/flow-go/network/p2p/cache" + "github.com/onflow/flow-go/network/p2p/p2pconf" "github.com/onflow/flow-go/network/p2p/utils" "github.com/onflow/flow-go/utils/logging" ) @@ -305,21 +306,9 @@ type ScoreOption struct { appScoreFunc func(peer.ID) float64 } -// Parameters are the parameters for the score option. -// Parameters are "numerical values" that are used to compute or build components that compute the score of a peer in GossipSub system. -type Parameters struct { - AppSpecificScore AppSpecificScoreRegistryParams `validate:"required" mapstructure:"app-specific-score"` - // SpamRecordCacheSize is size of the cache used to store the spam records of peers. - // The spam records are used to penalize peers that send invalid messages. - SpamRecordCacheSize uint32 `validate:"gt=0" mapstructure:"spam-record-cache-size"` - - // DecayInterval is the interval at which the counters associated with a peer behavior in GossipSub system are decayed. - DecayInterval time.Duration `validate:"gt=0s" mapstructure:"decay-interval"` -} - type ScoreOptionConfig struct { logger zerolog.Logger - params Parameters + params p2pconf.ScoringParameters provider module.IdentityProvider heroCacheMetricsFactory metrics.HeroCacheMetricsFactory appScoreFunc func(peer.ID) float64 @@ -335,7 +324,7 @@ type ScoreOptionConfig struct { // Returns: // - a new configuration for the GossipSub peer scoring option. func NewScoreOptionConfig(logger zerolog.Logger, - params Parameters, + params p2pconf.ScoringParameters, hcMetricsFactory metrics.HeroCacheMetricsFactory, idProvider module.IdentityProvider) *ScoreOptionConfig { return &ScoreOptionConfig{ diff --git a/network/p2p/scoring/scoring_test.go b/network/p2p/scoring/scoring_test.go index 512418421ff..49ccac49b75 100644 --- a/network/p2p/scoring/scoring_test.go +++ b/network/p2p/scoring/scoring_test.go @@ -132,7 +132,7 @@ func TestInvalidCtrlMsgScoringIntegration(t *testing.T) { p2ptest.LetNodesDiscoverEachOther(t, ctx, nodes, ids) blockTopic := channels.TopicFromChannel(channels.PushBlocks, sporkId) - // checks end-to-end message delivery works on GossipSub + // checks end-to-end message delivery works on GossipSubParameters p2ptest.EnsurePubsubMessageExchange(t, ctx, nodes, blockTopic, 1, func() interface{} { return unittest.ProposalFixture() }) @@ -146,7 +146,7 @@ func TestInvalidCtrlMsgScoringIntegration(t *testing.T) { }) } - // checks no GossipSub message exchange should no longer happen between node1 and node2. + // checks no GossipSubParameters message exchange should no longer happen between node1 and node2. p2ptest.EnsureNoPubsubExchangeBetweenGroups( t, ctx, From 8217af4685d613324fef0cdf9d458a6b7bd100d5 Mon Sep 17 00:00:00 2001 From: Yahya Hassanzadeh Date: Mon, 20 Nov 2023 16:49:23 -0800 Subject: [PATCH 16/67] decouples cross referencing tests --- config/default-config.yml | 8 +++++--- network/netconf/config_test.go | 28 ++++++++++++++++++++++++++-- network/netconf/flags.go | 20 ++++++++++++++++++++ network/p2p/p2pconf/gossipsub.go | 28 +++++++++++++++------------- network/p2p/scoring/registry.go | 2 +- 5 files changed, 67 insertions(+), 19 deletions(-) diff --git a/config/default-config.yml b/config/default-config.yml index 93c184aa40e..e1225d62d59 100644 --- a/config/default-config.yml +++ b/config/default-config.yml @@ -197,6 +197,11 @@ network-config: # score ttl is the time to live for the app specific score. Once the score is expired; a new request will be sent to the app specific score provider to update the score. # until the score is updated, the previous score will be used. score-ttl: 1m + # size of cache used to track spam records at gossipsub. Each peer id is mapped to a spam record that keeps track of the spam score for that peer. + # cache should be big enough to keep track of the entire network's size. Otherwise, the local node's view of the network will be incomplete due to cache eviction. + spam-record-cache-size: 10_000 + # the intervals at which counters associated with a peer behavior in gossipsub system are decayed. + decay-interval: 1m subscription-provider: # The interval for updating the list of subscribed peers to all topics in gossipsub. This is used to keep track of subscriptions # violations and penalize peers accordingly. Recommended value is in the order of a few minutes to avoid contentions; as the operation @@ -212,7 +217,6 @@ network-config: alsp-spam-report-queue-size: 10_000 alsp-disable-penalty: false alsp-heart-beat-interval: 1s - # Base probability in [0,1] that's used in creating the final probability of creating a # misbehavior report for a BatchRequest message. This is why the word "base" is used in the name of this field, # since it's not the final probability and there are other factors that determine the final probability. @@ -229,7 +233,6 @@ network-config: # batchRequestBaseProb * (1000+1) / synccore.DefaultConfig().MaxSize # = 0.01 * 1001 / 64 = 0.15640625 = 15.640625% alsp-sync-engine-batch-request-base-prob: 0.01 - # Base probability in [0,1] that's used in creating the final probability of creating a # misbehavior report for a RangeRequest message. This is why the word "base" is used in the name of this field, # since it's not the final probability and there are other factors that determine the final probability. @@ -247,7 +250,6 @@ network-config: # rangeRequestBaseProb * (1000+1) / synccore.DefaultConfig().MaxSize # = 0.01 * 1001 / 64 = 0.15640625 = 15.640625% alsp-sync-engine-range-request-base-prob: 0.01 - # Probability in [0,1] of creating a misbehavior report for a SyncRequest message. # create misbehavior report for 1% of SyncRequest messages alsp-sync-engine-sync-request-prob: 0.01 diff --git a/network/netconf/config_test.go b/network/netconf/config_test.go index 3a0a21b10b9..d6a062cdb2f 100644 --- a/network/netconf/config_test.go +++ b/network/netconf/config_test.go @@ -42,10 +42,34 @@ func TestSetAliases(t *testing.T) { } } -// TestCrossReferenceFlagsAndConfigs ensures every network configuration in the config file has a corresponding CLI flag. -func TestCrossReferenceFlagsAndConfigs(t *testing.T) { +// TestCrossReferenceFlagsWithConfigs ensures that each flag is cross-referenced with the config file, i.e., that each +// flag has a corresponding config key. +func TestCrossReferenceFlagsWithConfigs(t *testing.T) { // reads the default config file c := config.RawViperConfig() err := netconf.SetAliases(c) require.NoError(t, err) } + +// TestCrossReferenceConfigsWithFlags ensures that each config is cross-referenced with the flags, i.e., that each config +// key has a corresponding flag. +func TestCrossReferenceConfigsWithFlags(t *testing.T) { + c := config.RawViperConfig() + // keeps all flag names + m := make(map[string]struct{}) + + // each flag name should correspond to exactly one key in our config store after it is loaded with the default config + for _, flagName := range netconf.AllFlagNames() { + m[flagName] = struct{}{} + } + + for _, key := range c.AllKeys() { + s := strings.Split(key, ".") + flag := strings.Join(s[1:], "-") + if len(flag) == 0 { + continue + } + _, ok := m[flag] + require.Truef(t, ok, "config key %s does not have a corresponding flag", flag) + } +} diff --git a/network/netconf/flags.go b/network/netconf/flags.go index 57bc0ac4f85..c2393f0bc06 100644 --- a/network/netconf/flags.go +++ b/network/netconf/flags.go @@ -151,6 +151,11 @@ func AllFlagNames() []string { BuildFlagName(gossipSub, p2pconf.RpcInspectorKey, p2pconf.ValidationConfigKey, p2pconf.MessageErrorThresholdKey), BuildFlagName(gossipSub, p2pconf.SubscriptionProviderKey, p2pconf.UpdateIntervalKey), BuildFlagName(gossipSub, p2pconf.SubscriptionProviderKey, p2pconf.CacheSizeKey), + BuildFlagName(gossipSub, p2pconf.ScoreParamsKey, p2pconf.AppSpecificScoreRegistryKey, p2pconf.ScoreUpdateWorkerNumKey), + BuildFlagName(gossipSub, p2pconf.ScoreParamsKey, p2pconf.AppSpecificScoreRegistryKey, p2pconf.ScoreUpdateRequestQueueSizeKey), + BuildFlagName(gossipSub, p2pconf.ScoreParamsKey, p2pconf.AppSpecificScoreRegistryKey, p2pconf.ScoreTTLKey), + BuildFlagName(gossipSub, p2pconf.ScoreParamsKey, p2pconf.SpamRecordCacheSizeKey), + BuildFlagName(gossipSub, p2pconf.ScoreParamsKey, p2pconf.DecayIntervalKey), } for _, scope := range []string{systemScope, transientScope, protocolScope, peerScope, peerProtocolScope} { @@ -303,6 +308,21 @@ func InitializeNetworkFlags(flags *pflag.FlagSet, config *Config) { flags.Uint32(BuildFlagName(gossipSub, p2pconf.SubscriptionProviderKey, p2pconf.CacheSizeKey), config.GossipSub.SubscriptionProvider.CacheSize, "size of the cache that keeps the list of topics each peer has subscribed to, recommended size is 10x the number of authorized nodes") + flags.Int(BuildFlagName(gossipSub, p2pconf.ScoreParamsKey, p2pconf.AppSpecificScoreRegistryKey, p2pconf.ScoreUpdateWorkerNumKey), + config.GossipSub.ScoringParameters.AppSpecificScore.ScoreUpdateWorkerNum, + "number of workers for the app specific score update worker pool") + flags.Uint32(BuildFlagName(gossipSub, p2pconf.ScoreParamsKey, p2pconf.AppSpecificScoreRegistryKey, p2pconf.ScoreUpdateRequestQueueSizeKey), + config.GossipSub.ScoringParameters.AppSpecificScore.ScoreUpdateRequestQueueSize, + "size of the app specific score update worker pool queue") + flags.Duration(BuildFlagName(gossipSub, p2pconf.ScoreParamsKey, p2pconf.AppSpecificScoreRegistryKey, p2pconf.ScoreTTLKey), + config.GossipSub.ScoringParameters.AppSpecificScore.ScoreTTL, + "time to live for app specific scores; when expired a new request will be sent to the score update worker pool; till then the expired score will be used") + flags.Uint32(BuildFlagName(gossipSub, p2pconf.ScoreParamsKey, p2pconf.SpamRecordCacheSizeKey), + config.GossipSub.ScoringParameters.SpamRecordCacheSize, + "size of the spam record cache, recommended size is 10x the number of authorized nodes") + flags.Duration(BuildFlagName(gossipSub, p2pconf.ScoreParamsKey, p2pconf.DecayIntervalKey), + config.GossipSub.ScoringParameters.DecayInterval, + "interval at which the counters associated with a peer behavior in GossipSub system are decayed, recommended value is one minute") } // LoadLibP2PResourceManagerFlags loads all CLI flags for the libp2p resource manager configuration on the provided pflag set. diff --git a/network/p2p/p2pconf/gossipsub.go b/network/p2p/p2pconf/gossipsub.go index 33a30aeec99..713b4207bdf 100644 --- a/network/p2p/p2pconf/gossipsub.go +++ b/network/p2p/p2pconf/gossipsub.go @@ -63,8 +63,6 @@ const ( SubscriptionProviderKey = "subscription-provider" ) -var GossipSubConfigKeys = []string{RpcInspectorKey, RpcTracerKey, PeerScoringEnabledKey, ScoreParamsKey, SubscriptionProviderKey} - // GossipSubParameters is the configuration for the GossipSubParameters pubsub implementation. type GossipSubParameters struct { // RpcInspectorParameters configuration for all gossipsub RPC control message inspectors. @@ -81,10 +79,16 @@ type GossipSubParameters struct { ScoringParameters ScoringParameters `mapstructure:"scoring-parameters"` } +const ( + AppSpecificScoreRegistryKey = "app-specific-score" + SpamRecordCacheSizeKey = "spam-record-cache-size" + DecayIntervalKey = "decay-interval" +) + // Parameters are the parameters for the score option. // Parameters are "numerical values" that are used to compute or build components that compute the score of a peer in GossipSub system. type ScoringParameters struct { - AppSpecificScore AppSpecificScoreRegistryParams `validate:"required" mapstructure:"app-specific-score"` + AppSpecificScore AppSpecificScoreParameters `validate:"required" mapstructure:"app-specific-score"` // SpamRecordCacheSize is size of the cache used to store the spam records of peers. // The spam records are used to penalize peers that send invalid messages. SpamRecordCacheSize uint32 `validate:"gt=0" mapstructure:"spam-record-cache-size"` @@ -93,9 +97,15 @@ type ScoringParameters struct { DecayInterval time.Duration `validate:"gt=0s" mapstructure:"decay-interval"` } -// AppSpecificScoreRegistryParams is the parameters for the GossipSubAppSpecificScoreRegistry. +const ( + ScoreUpdateWorkerNumKey = "score-update-worker-num" + ScoreUpdateRequestQueueSizeKey = "score-update-request-queue-size" + ScoreTTLKey = "score-ttl" +) + +// AppSpecificScoreParameters is the parameters for the GossipSubAppSpecificScoreRegistry. // Parameters are "numerical values" that are used to compute or build components that compute or maintain the application specific score of peers. -type AppSpecificScoreRegistryParams struct { +type AppSpecificScoreParameters struct { // ScoreUpdateWorkerNum is the number of workers in the worker pool for handling the application specific score update of peers in a non-blocking way. ScoreUpdateWorkerNum int `validate:"gt=0" mapstructure:"score-update-worker-num"` @@ -115,8 +125,6 @@ const ( CacheSizeKey = "cache-size" ) -var SubscriptionProviderParametersKeys = []string{UpdateIntervalKey, CacheSizeKey} - type SubscriptionProviderParameters struct { // UpdateInterval is the interval for updating the list of topics the node have subscribed to; as well as the list of all // peers subscribed to each of those topics. This is used to penalize peers that have an invalid subscription based on their role. @@ -138,12 +146,6 @@ const ( RPCSentTrackerNumOfWorkersKey = "rpc-sent-tracker-workers" ) -var TracerParametersKeys = []string{LocalMeshLogIntervalKey, - ScoreTracerIntervalKey, - RPCSentTrackerCacheSizeKey, - RPCSentTrackerQueueCacheSizeKey, - RPCSentTrackerNumOfWorkersKey} - // GossipSubTracerParameters is the config for the gossipsub tracer. GossipSubParameters tracer is used to trace the local mesh events and peer scores. type GossipSubTracerParameters struct { // LocalMeshLogInterval is the interval at which the local mesh is logged. diff --git a/network/p2p/scoring/registry.go b/network/p2p/scoring/registry.go index 1f4d519df40..493b5f4bac9 100644 --- a/network/p2p/scoring/registry.go +++ b/network/p2p/scoring/registry.go @@ -117,7 +117,7 @@ type GossipSubAppSpecificScoreRegistry struct { // GossipSubAppSpecificScoreRegistryConfig is the configuration for the GossipSubAppSpecificScoreRegistry. // Configurations are the "union of parameters and other components" that are used to compute or build components that compute or maintain the application specific score of peers. type GossipSubAppSpecificScoreRegistryConfig struct { - Parameters p2pconf.AppSpecificScoreRegistryParams `validate:"required"` + Parameters p2pconf.AppSpecificScoreParameters `validate:"required"` Logger zerolog.Logger `validate:"required"` From 6dd40920e0ea56c915525fe5e123efbf62e43315 Mon Sep 17 00:00:00 2001 From: Yahya Hassanzadeh Date: Tue, 21 Nov 2023 10:03:52 -0800 Subject: [PATCH 17/67] wip lint fix --- .../test/gossipsub/scoring/ihave_spam_test.go | 18 +++++++---- .../test/gossipsub/scoring/scoring_test.go | 29 +++++++++++++---- network/p2p/builder.go | 7 ---- .../control_message_validation_inspector.go | 32 +++++++++---------- ...ntrol_message_validation_inspector_test.go | 2 +- .../p2pbuilder/gossipsub/gossipSubBuilder.go | 13 ++------ network/p2p/scoring/score_option.go | 3 -- 7 files changed, 55 insertions(+), 49 deletions(-) diff --git a/insecure/integration/functional/test/gossipsub/scoring/ihave_spam_test.go b/insecure/integration/functional/test/gossipsub/scoring/ihave_spam_test.go index 2fb499b4c95..f450579259a 100644 --- a/insecure/integration/functional/test/gossipsub/scoring/ihave_spam_test.go +++ b/insecure/integration/functional/test/gossipsub/scoring/ihave_spam_test.go @@ -62,18 +62,23 @@ func TestGossipSubIHaveBrokenPromises_Below_Threshold(t *testing.T) { // the node would be penalized for invalid message delivery way sooner than it can mount an ihave broken-promises spam attack. blockTopicOverrideParams.InvalidMessageDeliveriesWeight = 0.0 blockTopicOverrideParams.InvalidMessageDeliveriesDecay = 0.0 + + conf, err := config.DefaultConfig() + require.NoError(t, err) + // we override the decay interval to 1 second so that the score is updated within 1 second intervals. + conf.NetworkConfig.GossipSub.ScoringParameters.DecayInterval = 1 * time.Second victimNode, victimIdentity := p2ptest.NodeFixture( t, sporkId, t.Name(), idProvider, p2ptest.WithRole(role), + p2ptest.OverrideFlowConfig(conf), p2ptest.WithPeerScoreTracerInterval(1*time.Second), p2ptest.EnablePeerScoringWithOverride(&p2p.PeerScoringConfigOverride{ TopicScoreParams: map[channels.Topic]*pubsub.TopicScoreParams{ blockTopic: blockTopicOverrideParams, }, - DecayInterval: 1 * time.Second, // we override the decay interval to 1 second so that the score is updated within 1 second intervals. }), ) @@ -187,10 +192,12 @@ func TestGossipSubIHaveBrokenPromises_Above_Threshold(t *testing.T) { conf, err := config.DefaultConfig() require.NoError(t, err) // overcompensate for RPC truncation - conf.NetworkConfig.GossipSubRPCInspectorsConfig.IHaveRPCInspectionConfig.MaxSampleSize = 10000 - conf.NetworkConfig.GossipSubRPCInspectorsConfig.IHaveRPCInspectionConfig.MaxMessageIDSampleSize = 10000 - conf.NetworkConfig.GossipSubRPCInspectorsConfig.IWantRPCInspectionConfig.MaxSampleSize = 10000 - conf.NetworkConfig.GossipSubRPCInspectorsConfig.IWantRPCInspectionConfig.MaxMessageIDSampleSize = 10000 + conf.NetworkConfig.GossipSub.RpcInspector.Validation.IHave.MaxSampleSize = 10000 + conf.NetworkConfig.GossipSub.RpcInspector.Validation.IHave.MaxMessageIDSampleSize = 10000 + conf.NetworkConfig.GossipSub.RpcInspector.Validation.IHave.MaxSampleSize = 10000 + conf.NetworkConfig.GossipSub.RpcInspector.Validation.IHave.MaxMessageIDSampleSize = 10000 + // we override the decay interval to 1 second so that the score is updated within 1 second intervals. + conf.NetworkConfig.GossipSub.ScoringParameters.DecayInterval = 1 * time.Second ctx, cancel := context.WithCancel(context.Background()) signalerCtx := irrecoverable.NewMockSignalerContext(t, ctx) @@ -213,7 +220,6 @@ func TestGossipSubIHaveBrokenPromises_Above_Threshold(t *testing.T) { TopicScoreParams: map[channels.Topic]*pubsub.TopicScoreParams{ blockTopic: blockTopicOverrideParams, }, - DecayInterval: 1 * time.Second, // we override the decay interval to 1 second so that the score is updated within 1 second intervals. }), ) diff --git a/insecure/integration/functional/test/gossipsub/scoring/scoring_test.go b/insecure/integration/functional/test/gossipsub/scoring/scoring_test.go index 152636ad9cf..3948de3264b 100644 --- a/insecure/integration/functional/test/gossipsub/scoring/scoring_test.go +++ b/insecure/integration/functional/test/gossipsub/scoring/scoring_test.go @@ -10,6 +10,7 @@ import ( "github.com/libp2p/go-libp2p/core/peer" "github.com/stretchr/testify/require" + "github.com/onflow/flow-go/config" "github.com/onflow/flow-go/insecure/corruptlibp2p" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/model/messages" @@ -171,7 +172,11 @@ func testGossipSubInvalidMessageDeliveryScoring(t *testing.T, spamMsgFactory fun require.True(t, ok) // ensure that the topic snapshot of the spammer contains a record of at least (60%) of the spam messages sent. The 60% is to account for the messages that were delivered before the score was updated, after the spammer is PRUNED, as well as to account for decay. - require.True(t, blkTopicSnapshot.InvalidMessageDeliveries > 0.6*float64(totalSpamMessages), "invalid message deliveries must be greater than %f. invalid message deliveries: %f", 0.9*float64(totalSpamMessages), blkTopicSnapshot.InvalidMessageDeliveries) + require.True(t, + blkTopicSnapshot.InvalidMessageDeliveries > 0.6*float64(totalSpamMessages), + "invalid message deliveries must be greater than %f. invalid message deliveries: %f", + 0.9*float64(totalSpamMessages), + blkTopicSnapshot.InvalidMessageDeliveries) p2ptest.EnsureNoPubsubExchangeBetweenGroups( t, @@ -203,7 +208,12 @@ func TestGossipSubMeshDeliveryScoring_UnderDelivery_SingleTopic(t *testing.T) { // we override some of the default scoring parameters in order to speed up the test in a time-efficient manner. blockTopicOverrideParams := scoring.DefaultTopicScoreParams() blockTopicOverrideParams.MeshMessageDeliveriesActivation = 1 * time.Second // we start observing the mesh message deliveries after 1 second of the node startup. - thisNode, thisId := p2ptest.NodeFixture( // this node is the one that will be penalizing the under-performer node. + + conf, err := config.DefaultConfig() + require.NoError(t, err) + // we override the decay interval to 1 second so that the score is updated within 1 second intervals. + conf.NetworkConfig.GossipSub.ScoringParameters.DecayInterval = 1 * time.Second + thisNode, thisId := p2ptest.NodeFixture( // this node is the one that will be penalizing the under-performer node. t, sporkId, t.Name(), @@ -215,7 +225,6 @@ func TestGossipSubMeshDeliveryScoring_UnderDelivery_SingleTopic(t *testing.T) { TopicScoreParams: map[channels.Topic]*pubsub.TopicScoreParams{ blockTopic: blockTopicOverrideParams, }, - DecayInterval: 1 * time.Second, // we override the decay interval to 1 second so that the score is updated within 1 second intervals. }), ) @@ -308,7 +317,12 @@ func TestGossipSubMeshDeliveryScoring_UnderDelivery_TwoTopics(t *testing.T) { blockTopicOverrideParams.MeshMessageDeliveriesActivation = 1 * time.Second // we start observing the mesh message deliveries after 1 second of the node startup. dkgTopicOverrideParams := scoring.DefaultTopicScoreParams() dkgTopicOverrideParams.MeshMessageDeliveriesActivation = 1 * time.Second // we start observing the mesh message deliveries after 1 second of the node startup. - thisNode, thisId := p2ptest.NodeFixture( // this node is the one that will be penalizing the under-performer node. + + conf, err := config.DefaultConfig() + require.NoError(t, err) + // we override the decay interval to 1 second so that the score is updated within 1 second intervals. + conf.NetworkConfig.GossipSub.ScoringParameters.DecayInterval = 1 * time.Second + thisNode, thisId := p2ptest.NodeFixture( // this node is the one that will be penalizing the under-performer node. t, sporkId, t.Name(), @@ -321,7 +335,6 @@ func TestGossipSubMeshDeliveryScoring_UnderDelivery_TwoTopics(t *testing.T) { blockTopic: blockTopicOverrideParams, dkgTopic: dkgTopicOverrideParams, }, - DecayInterval: 1 * time.Second, // we override the decay interval to 1 second so that the score is updated within 1 second intervals. }), ) @@ -416,6 +429,10 @@ func TestGossipSubMeshDeliveryScoring_Replay_Will_Not_Counted(t *testing.T) { blockTopic := channels.TopicFromChannel(channels.PushBlocks, sporkId) // we override some of the default scoring parameters in order to speed up the test in a time-efficient manner. + conf, err := config.DefaultConfig() + require.NoError(t, err) + // we override the decay interval to 1 second so that the score is updated within 1 second intervals. + conf.NetworkConfig.GossipSub.ScoringParameters.DecayInterval = 1 * time.Second blockTopicOverrideParams := scoring.DefaultTopicScoreParams() blockTopicOverrideParams.MeshMessageDeliveriesActivation = 1 * time.Second // we start observing the mesh message deliveries after 1 second of the node startup. thisNode, thisId := p2ptest.NodeFixture( // this node is the one that will be penalizing the under-performer node. @@ -424,13 +441,13 @@ func TestGossipSubMeshDeliveryScoring_Replay_Will_Not_Counted(t *testing.T) { t.Name(), idProvider, p2ptest.WithRole(role), + p2ptest.OverrideFlowConfig(conf), p2ptest.WithPeerScoreTracerInterval(1*time.Second), p2ptest.EnablePeerScoringWithOverride( &p2p.PeerScoringConfigOverride{ TopicScoreParams: map[channels.Topic]*pubsub.TopicScoreParams{ blockTopic: blockTopicOverrideParams, }, - DecayInterval: 1 * time.Second, // we override the decay interval to 1 second so that the score is updated within 1 second intervals. }), ) diff --git a/network/p2p/builder.go b/network/p2p/builder.go index ab1e025d64f..1665b838647 100644 --- a/network/p2p/builder.go +++ b/network/p2p/builder.go @@ -155,13 +155,6 @@ type PeerScoringConfigOverride struct { // Override criteria: if the function is not nil, it will override the default application specific score parameters. // If the function is nil, the default application specific score parameters are used. AppSpecificScoreParams func(peer.ID) float64 - - // DecayInterval is the interval over which we decay the effect of past behavior, so that - // a good or bad behavior will not have a permanent effect on the penalty. It is also the interval - // that GossipSubParameters uses to refresh the scores of all peers. - // Override criteria: if the value is not zero, it will override the default decay interval. - // If the value is zero, the default decay interval is used. - DecayInterval time.Duration } // PeerScoringConfigNoOverride is a default peer scoring configuration for a GossipSubParameters pubsub system. diff --git a/network/p2p/inspector/validation/control_message_validation_inspector.go b/network/p2p/inspector/validation/control_message_validation_inspector.go index f8b20ac638f..b4588b566b8 100644 --- a/network/p2p/inspector/validation/control_message_validation_inspector.go +++ b/network/p2p/inspector/validation/control_message_validation_inspector.go @@ -109,9 +109,9 @@ func NewControlMsgValidationInspector(params *InspectorParams) (*ControlMsgValid clusterPrefixedCacheCollector := metrics.GossipSubRPCInspectorClusterPrefixedCacheMetricFactory(params.HeroCacheMetricsFactory, params.NetworkingType) clusterPrefixedTracker, err := cache.NewClusterPrefixedMessagesReceivedTracker(params.Logger, - params.Config.ClusterPrefixedControlMsgsReceivedCacheSize, + params.Config.ClusterPrefixedMessage.ControlMsgsReceivedCacheSize, clusterPrefixedCacheCollector, - params.Config.ClusterPrefixedControlMsgsReceivedCacheDecay) + params.Config.ClusterPrefixedMessage.ControlMsgsReceivedCacheDecay) if err != nil { return nil, fmt.Errorf("failed to create cluster prefix topics received tracker") } @@ -341,7 +341,7 @@ func (c *ControlMsgValidationInspector) inspectIHaveMessages(from peer.ID, ihave lg := c.logger.With(). Str("peer_id", p2plogging.PeerId(from)). Int("sample_size", len(ihaves)). - Int("max_sample_size", c.config.IHaveRPCInspectionConfig.MaxSampleSize). + Int("max_sample_size", c.config.IHave.MaxSampleSize). Logger() duplicateTopicTracker := make(duplicateStrTracker) duplicateMessageIDTracker := make(duplicateStrTracker) @@ -389,16 +389,16 @@ func (c *ControlMsgValidationInspector) inspectIWantMessages(from peer.ID, iWant lastHighest := c.rpcTracker.LastHighestIHaveRPCSize() lg := c.logger.With(). Str("peer_id", p2plogging.PeerId(from)). - Uint("max_sample_size", c.config.IWantRPCInspectionConfig.MaxSampleSize). + Uint("max_sample_size", c.config.IWant.MaxSampleSize). Int64("last_highest_ihave_rpc_size", lastHighest). Logger() sampleSize := uint(len(iWants)) tracker := make(duplicateStrTracker) cacheMisses := 0 - allowedCacheMissesThreshold := float64(sampleSize) * c.config.IWantRPCInspectionConfig.CacheMissThreshold + allowedCacheMissesThreshold := float64(sampleSize) * c.config.IWant.CacheMissThreshold duplicates := 0 - allowedDuplicatesThreshold := float64(sampleSize) * c.config.IWantRPCInspectionConfig.DuplicateMsgIDThreshold - checkCacheMisses := len(iWants) >= c.config.IWantRPCInspectionConfig.CacheMissCheckSize + allowedDuplicatesThreshold := float64(sampleSize) * c.config.IWant.DuplicateMsgIDThreshold + checkCacheMisses := len(iWants) >= c.config.IWant.CacheMissCheckSize lg = lg.With(). Uint("iwant_sample_size", sampleSize). Float64("allowed_cache_misses_threshold", allowedCacheMissesThreshold). @@ -415,7 +415,7 @@ func (c *ControlMsgValidationInspector) inspectIWantMessages(from peer.ID, iWant if tracker.isDuplicate(messageID) { duplicates++ if float64(duplicates) > allowedDuplicatesThreshold { - return NewIWantDuplicateMsgIDThresholdErr(duplicates, messageIDCount, c.config.IWantRPCInspectionConfig.DuplicateMsgIDThreshold) + return NewIWantDuplicateMsgIDThresholdErr(duplicates, messageIDCount, c.config.IWant.DuplicateMsgIDThreshold) } } // check cache miss threshold @@ -423,7 +423,7 @@ func (c *ControlMsgValidationInspector) inspectIWantMessages(from peer.ID, iWant cacheMisses++ if checkCacheMisses { if float64(cacheMisses) > allowedCacheMissesThreshold { - return NewIWantCacheMissThresholdErr(cacheMisses, messageIDCount, c.config.IWantRPCInspectionConfig.CacheMissThreshold) + return NewIWantCacheMissThresholdErr(cacheMisses, messageIDCount, c.config.IWant.CacheMissThreshold) } } } @@ -577,7 +577,7 @@ func (c *ControlMsgValidationInspector) truncateIHaveMessages(rpc *pubsub.RPC) { if totalIHaves == 0 { return } - sampleSize := c.config.IHaveRPCInspectionConfig.MaxSampleSize + sampleSize := c.config.IHave.MaxSampleSize if sampleSize > totalIHaves { sampleSize = totalIHaves } @@ -600,7 +600,7 @@ func (c *ControlMsgValidationInspector) truncateIHaveMessageIds(rpc *pubsub.RPC) if totalMessageIDs == 0 { return } - sampleSize := c.config.IHaveRPCInspectionConfig.MaxMessageIDSampleSize + sampleSize := c.config.IHave.MaxMessageIDSampleSize if sampleSize > totalMessageIDs { sampleSize = totalMessageIDs } @@ -621,7 +621,7 @@ func (c *ControlMsgValidationInspector) truncateIWantMessages(from peer.ID, rpc if totalIWants == 0 { return } - sampleSize := c.config.IWantRPCInspectionConfig.MaxSampleSize + sampleSize := c.config.IWant.MaxSampleSize if sampleSize > totalIWants { sampleSize = totalIWants } @@ -640,15 +640,15 @@ func (c *ControlMsgValidationInspector) truncateIWantMessageIds(from peer.ID, rp lastHighest := c.rpcTracker.LastHighestIHaveRPCSize() lg := c.logger.With(). Str("peer_id", p2plogging.PeerId(from)). - Uint("max_sample_size", c.config.IWantRPCInspectionConfig.MaxSampleSize). + Uint("max_sample_size", c.config.IWant.MaxSampleSize). Int64("last_highest_ihave_rpc_size", lastHighest). Logger() sampleSize := int(10 * lastHighest) - if sampleSize == 0 || sampleSize > c.config.IWantRPCInspectionConfig.MaxMessageIDSampleSize { + if sampleSize == 0 || sampleSize > c.config.IWant.MaxMessageIDSampleSize { // invalid or 0 sample size is suspicious lg.Warn().Str(logging.KeySuspicious, "true").Msg("zero or invalid sample size, using default max sample size") - sampleSize = c.config.IWantRPCInspectionConfig.MaxMessageIDSampleSize + sampleSize = c.config.IWant.MaxMessageIDSampleSize } for _, iWant := range rpc.GetControl().GetIwant() { messageIDs := iWant.GetMessageIDs() @@ -791,7 +791,7 @@ func (c *ControlMsgValidationInspector) checkClusterPrefixHardThreshold(nodeID f // irrecoverable error encountered c.logAndThrowError(fmt.Errorf("cluster prefixed control message gauge during hard threshold check failed for node %s: %w", nodeID, err)) } - return gauge <= c.config.ClusterPrefixHardThreshold + return gauge <= c.config.ClusterPrefixedMessage.HardThreshold } // logAndDistributeErr logs the provided error and attempts to disseminate an invalid control message validation notification for the error. diff --git a/network/p2p/inspector/validation/control_message_validation_inspector_test.go b/network/p2p/inspector/validation/control_message_validation_inspector_test.go index ae25192a92a..b505e32622b 100644 --- a/network/p2p/inspector/validation/control_message_validation_inspector_test.go +++ b/network/p2p/inspector/validation/control_message_validation_inspector_test.go @@ -51,7 +51,7 @@ func (suite *ControlMsgValidationInspectorSuite) SetupTest() { suite.sporkID = unittest.IdentifierFixture() flowConfig, err := config.DefaultConfig() require.NoError(suite.T(), err, "failed to get default flow config") - suite.config = &flowConfig.NetworkConfig.GossipSubRPCValidationInspectorConfigs + suite.config = &flowConfig.NetworkConfig.GossipSub.RpcInspector.Validation distributor := mockp2p.NewGossipSubInspectorNotificationDistributor(suite.T()) p2ptest.MockInspectorNotificationDistributorReadyDoneAware(distributor) suite.distributor = distributor diff --git a/network/p2p/p2pbuilder/gossipsub/gossipSubBuilder.go b/network/p2p/p2pbuilder/gossipsub/gossipSubBuilder.go index cc1fa585f92..3f294b809a5 100644 --- a/network/p2p/p2pbuilder/gossipsub/gossipSubBuilder.go +++ b/network/p2p/p2pbuilder/gossipsub/gossipSubBuilder.go @@ -121,13 +121,6 @@ func (g *Builder) EnableGossipSubScoringWithOverride(override *p2p.PeerScoringCo g.scoreOptionConfig.OverrideTopicScoreParams(topic, params) } } - if override.DecayInterval > 0 { - g.logger.Warn(). - Str(logging.KeyNetworkingSecurity, "true"). - Dur("decay_interval", override.DecayInterval). - Msg("overriding decay interval for gossipsub") - g.scoreOptionConfig.OverrideDecayInterval(override.DecayInterval) - } } // SetGossipSubScoreTracerInterval sets the gossipsub score tracer interval of the builder. @@ -247,9 +240,9 @@ func defaultInspectorSuite(rpcTracker p2p.RpcControlTracking) p2p.GossipSubRpcIn metricsInspector := inspector.NewControlMsgMetricsInspector( logger, p2pnode.NewGossipSubControlMessageMetrics(gossipSubMetrics, logger), - inspectorCfg.GossipSubRPCMetricsInspectorConfigs.NumberOfWorkers, + inspectorCfg.Metrics.NumberOfWorkers, []queue.HeroStoreConfigOption{ - queue.WithHeroStoreSizeLimit(inspectorCfg.GossipSubRPCMetricsInspectorConfigs.CacheSize), + queue.WithHeroStoreSizeLimit(inspectorCfg.Metrics.CacheSize), queue.WithHeroStoreCollector( metrics.GossipSubRPCMetricsObserverInspectorQueueMetricFactory( heroCacheMetricsFactory, @@ -263,7 +256,7 @@ func defaultInspectorSuite(rpcTracker p2p.RpcControlTracking) p2p.GossipSubRpcIn params := &validation.InspectorParams{ Logger: logger, SporkID: sporkId, - Config: &inspectorCfg.GossipSubRPCValidationInspectorConfigs, + Config: &inspectorCfg.Validation, Distributor: notificationDistributor, HeroCacheMetricsFactory: heroCacheMetricsFactory, IdProvider: idProvider, diff --git a/network/p2p/scoring/score_option.go b/network/p2p/scoring/score_option.go index 79625e7df7e..86efa66d41c 100644 --- a/network/p2p/scoring/score_option.go +++ b/network/p2p/scoring/score_option.go @@ -106,9 +106,6 @@ const ( // this threshold are dropped. MaxDebugLogs = 50 - // defaultScoreCacheSize is the default size of the cache used to store the app specific penalty of peers. - defaultScoreCacheSize = 1000 - // defaultDecayInterval is the default decay interval for the overall score of a peer at the GossipSub scoring // system. We set it to 1 minute so that it is not too short so that a malicious node can recover from a penalty // and is not too long so that a well-behaved node can't recover from a penalty. From 44cc034bc3f502692142ba93c00f886b4e9e3a17 Mon Sep 17 00:00:00 2001 From: Yahya Hassanzadeh Date: Tue, 21 Nov 2023 10:42:24 -0800 Subject: [PATCH 18/67] wip lint fix --- .../node_builder/access_node_builder.go | 260 +++++++++++++----- cmd/observer/node_builder/observer_builder.go | 96 ++++--- network/p2p/builder.go | 11 - ...ntrol_message_validation_inspector_test.go | 40 +-- .../p2pbuilder/gossipsub/gossipSubBuilder.go | 86 +++--- network/p2p/p2pbuilder/libp2pNodeBuilder.go | 47 +--- 6 files changed, 316 insertions(+), 224 deletions(-) diff --git a/cmd/access/node_builder/access_node_builder.go b/cmd/access/node_builder/access_node_builder.go index d3fb48582a7..d955ab864d9 100644 --- a/cmd/access/node_builder/access_node_builder.go +++ b/cmd/access/node_builder/access_node_builder.go @@ -85,7 +85,6 @@ import ( p2pconfig "github.com/onflow/flow-go/network/p2p/p2pbuilder/config" "github.com/onflow/flow-go/network/p2p/p2pnet" "github.com/onflow/flow-go/network/p2p/subscription" - "github.com/onflow/flow-go/network/p2p/tracer" "github.com/onflow/flow-go/network/p2p/translator" "github.com/onflow/flow-go/network/p2p/unicast/protocols" relaynet "github.com/onflow/flow-go/network/relay" @@ -867,68 +866,211 @@ func (builder *FlowAccessNodeBuilder) extraFlags() { flags.UintVar(&builder.collectionGRPCPort, "collection-ingress-port", defaultConfig.collectionGRPCPort, "the grpc ingress port for all collection nodes") flags.UintVar(&builder.executionGRPCPort, "execution-ingress-port", defaultConfig.executionGRPCPort, "the grpc ingress port for all execution nodes") - flags.StringVarP(&builder.rpcConf.UnsecureGRPCListenAddr, "rpc-addr", "r", defaultConfig.rpcConf.UnsecureGRPCListenAddr, "the address the unsecured gRPC server listens on") - flags.StringVar(&builder.rpcConf.SecureGRPCListenAddr, "secure-rpc-addr", defaultConfig.rpcConf.SecureGRPCListenAddr, "the address the secure gRPC server listens on") - flags.StringVar(&builder.stateStreamConf.ListenAddr, "state-stream-addr", defaultConfig.stateStreamConf.ListenAddr, "the address the state stream server listens on (if empty the server will not be started)") + flags.StringVarP(&builder.rpcConf.UnsecureGRPCListenAddr, + "rpc-addr", + "r", + defaultConfig.rpcConf.UnsecureGRPCListenAddr, + "the address the unsecured gRPC server listens on") + flags.StringVar(&builder.rpcConf.SecureGRPCListenAddr, + "secure-rpc-addr", + defaultConfig.rpcConf.SecureGRPCListenAddr, + "the address the secure gRPC server listens on") + flags.StringVar(&builder.stateStreamConf.ListenAddr, + "state-stream-addr", + defaultConfig.stateStreamConf.ListenAddr, + "the address the state stream server listens on (if empty the server will not be started)") flags.StringVarP(&builder.rpcConf.HTTPListenAddr, "http-addr", "h", defaultConfig.rpcConf.HTTPListenAddr, "the address the http proxy server listens on") - flags.StringVar(&builder.rpcConf.RestConfig.ListenAddress, "rest-addr", defaultConfig.rpcConf.RestConfig.ListenAddress, "the address the REST server listens on (if empty the REST server will not be started)") - flags.DurationVar(&builder.rpcConf.RestConfig.WriteTimeout, "rest-write-timeout", defaultConfig.rpcConf.RestConfig.WriteTimeout, "timeout to use when writing REST response") - flags.DurationVar(&builder.rpcConf.RestConfig.ReadTimeout, "rest-read-timeout", defaultConfig.rpcConf.RestConfig.ReadTimeout, "timeout to use when reading REST request headers") + flags.StringVar(&builder.rpcConf.RestConfig.ListenAddress, + "rest-addr", + defaultConfig.rpcConf.RestConfig.ListenAddress, + "the address the REST server listens on (if empty the REST server will not be started)") + flags.DurationVar(&builder.rpcConf.RestConfig.WriteTimeout, + "rest-write-timeout", + defaultConfig.rpcConf.RestConfig.WriteTimeout, + "timeout to use when writing REST response") + flags.DurationVar(&builder.rpcConf.RestConfig.ReadTimeout, + "rest-read-timeout", + defaultConfig.rpcConf.RestConfig.ReadTimeout, + "timeout to use when reading REST request headers") flags.DurationVar(&builder.rpcConf.RestConfig.IdleTimeout, "rest-idle-timeout", defaultConfig.rpcConf.RestConfig.IdleTimeout, "idle timeout for REST connections") - flags.StringVarP(&builder.rpcConf.CollectionAddr, "static-collection-ingress-addr", "", defaultConfig.rpcConf.CollectionAddr, "the address (of the collection node) to send transactions to") - flags.StringVarP(&builder.ExecutionNodeAddress, "script-addr", "s", defaultConfig.ExecutionNodeAddress, "the address (of the execution node) forward the script to") - flags.StringVarP(&builder.rpcConf.HistoricalAccessAddrs, "historical-access-addr", "", defaultConfig.rpcConf.HistoricalAccessAddrs, "comma separated rpc addresses for historical access nodes") - flags.DurationVar(&builder.rpcConf.BackendConfig.CollectionClientTimeout, "collection-client-timeout", defaultConfig.rpcConf.BackendConfig.CollectionClientTimeout, "grpc client timeout for a collection node") - flags.DurationVar(&builder.rpcConf.BackendConfig.ExecutionClientTimeout, "execution-client-timeout", defaultConfig.rpcConf.BackendConfig.ExecutionClientTimeout, "grpc client timeout for an execution node") - flags.UintVar(&builder.rpcConf.BackendConfig.ConnectionPoolSize, "connection-pool-size", defaultConfig.rpcConf.BackendConfig.ConnectionPoolSize, "maximum number of connections allowed in the connection pool, size of 0 disables the connection pooling, and anything less than the default size will be overridden to use the default size") - flags.UintVar(&builder.rpcConf.MaxMsgSize, "rpc-max-message-size", grpcutils.DefaultMaxMsgSize, "the maximum message size in bytes for messages sent or received over grpc") - flags.UintVar(&builder.rpcConf.BackendConfig.MaxHeightRange, "rpc-max-height-range", defaultConfig.rpcConf.BackendConfig.MaxHeightRange, "maximum size for height range requests") - flags.StringSliceVar(&builder.rpcConf.BackendConfig.PreferredExecutionNodeIDs, "preferred-execution-node-ids", defaultConfig.rpcConf.BackendConfig.PreferredExecutionNodeIDs, "comma separated list of execution nodes ids to choose from when making an upstream call e.g. b4a4dbdcd443d...,fb386a6a... etc.") - flags.StringSliceVar(&builder.rpcConf.BackendConfig.FixedExecutionNodeIDs, "fixed-execution-node-ids", defaultConfig.rpcConf.BackendConfig.FixedExecutionNodeIDs, "comma separated list of execution nodes ids to choose from when making an upstream call if no matching preferred execution id is found e.g. b4a4dbdcd443d...,fb386a6a... etc.") - flags.StringVar(&builder.rpcConf.CompressorName, "grpc-compressor", defaultConfig.rpcConf.CompressorName, "name of grpc compressor that will be used for requests to other nodes. One of (gzip, snappy, deflate)") + flags.StringVarP(&builder.rpcConf.CollectionAddr, + "static-collection-ingress-addr", + "", + defaultConfig.rpcConf.CollectionAddr, + "the address (of the collection node) to send transactions to") + flags.StringVarP(&builder.ExecutionNodeAddress, + "script-addr", + "s", + defaultConfig.ExecutionNodeAddress, + "the address (of the execution node) forward the script to") + flags.StringVarP(&builder.rpcConf.HistoricalAccessAddrs, + "historical-access-addr", + "", + defaultConfig.rpcConf.HistoricalAccessAddrs, + "comma separated rpc addresses for historical access nodes") + flags.DurationVar(&builder.rpcConf.BackendConfig.CollectionClientTimeout, + "collection-client-timeout", + defaultConfig.rpcConf.BackendConfig.CollectionClientTimeout, + "grpc client timeout for a collection node") + flags.DurationVar(&builder.rpcConf.BackendConfig.ExecutionClientTimeout, + "execution-client-timeout", + defaultConfig.rpcConf.BackendConfig.ExecutionClientTimeout, + "grpc client timeout for an execution node") + flags.UintVar(&builder.rpcConf.BackendConfig.ConnectionPoolSize, + "connection-pool-size", + defaultConfig.rpcConf.BackendConfig.ConnectionPoolSize, + "maximum number of connections allowed in the connection pool, size of 0 disables the connection pooling, and anything less than the default size will be overridden to use the default size") + flags.UintVar(&builder.rpcConf.MaxMsgSize, + "rpc-max-message-size", + grpcutils.DefaultMaxMsgSize, + "the maximum message size in bytes for messages sent or received over grpc") + flags.UintVar(&builder.rpcConf.BackendConfig.MaxHeightRange, + "rpc-max-height-range", + defaultConfig.rpcConf.BackendConfig.MaxHeightRange, + "maximum size for height range requests") + flags.StringSliceVar(&builder.rpcConf.BackendConfig.PreferredExecutionNodeIDs, + "preferred-execution-node-ids", + defaultConfig.rpcConf.BackendConfig.PreferredExecutionNodeIDs, + "comma separated list of execution nodes ids to choose from when making an upstream call e.g. b4a4dbdcd443d...,fb386a6a... etc.") + flags.StringSliceVar(&builder.rpcConf.BackendConfig.FixedExecutionNodeIDs, + "fixed-execution-node-ids", + defaultConfig.rpcConf.BackendConfig.FixedExecutionNodeIDs, + "comma separated list of execution nodes ids to choose from when making an upstream call if no matching preferred execution id is found e.g. b4a4dbdcd443d...,fb386a6a... etc.") + flags.StringVar(&builder.rpcConf.CompressorName, + "grpc-compressor", + defaultConfig.rpcConf.CompressorName, + "name of grpc compressor that will be used for requests to other nodes. One of (gzip, snappy, deflate)") flags.BoolVar(&builder.logTxTimeToFinalized, "log-tx-time-to-finalized", defaultConfig.logTxTimeToFinalized, "log transaction time to finalized") flags.BoolVar(&builder.logTxTimeToExecuted, "log-tx-time-to-executed", defaultConfig.logTxTimeToExecuted, "log transaction time to executed") - flags.BoolVar(&builder.logTxTimeToFinalizedExecuted, "log-tx-time-to-finalized-executed", defaultConfig.logTxTimeToFinalizedExecuted, "log transaction time to finalized and executed") - flags.BoolVar(&builder.pingEnabled, "ping-enabled", defaultConfig.pingEnabled, "whether to enable the ping process that pings all other peers and report the connectivity to metrics") + flags.BoolVar(&builder.logTxTimeToFinalizedExecuted, + "log-tx-time-to-finalized-executed", + defaultConfig.logTxTimeToFinalizedExecuted, + "log transaction time to finalized and executed") + flags.BoolVar(&builder.pingEnabled, + "ping-enabled", + defaultConfig.pingEnabled, + "whether to enable the ping process that pings all other peers and report the connectivity to metrics") flags.BoolVar(&builder.retryEnabled, "retry-enabled", defaultConfig.retryEnabled, "whether to enable the retry mechanism at the access node level") flags.BoolVar(&builder.rpcMetricsEnabled, "rpc-metrics-enabled", defaultConfig.rpcMetricsEnabled, "whether to enable the rpc metrics") - flags.UintVar(&builder.TxResultCacheSize, "transaction-result-cache-size", defaultConfig.TxResultCacheSize, "transaction result cache size.(Disabled by default i.e 0)") - flags.StringVarP(&builder.nodeInfoFile, "node-info-file", "", defaultConfig.nodeInfoFile, "full path to a json file which provides more details about nodes when reporting its reachability metrics") - flags.StringToIntVar(&builder.apiRatelimits, "api-rate-limits", defaultConfig.apiRatelimits, "per second rate limits for Access API methods e.g. Ping=300,GetTransaction=500 etc.") - flags.StringToIntVar(&builder.apiBurstlimits, "api-burst-limits", defaultConfig.apiBurstlimits, "burst limits for Access API methods e.g. Ping=100,GetTransaction=100 etc.") - flags.BoolVar(&builder.supportsObserver, "supports-observer", defaultConfig.supportsObserver, "true if this staked access node supports observer or follower connections") - flags.StringVar(&builder.PublicNetworkConfig.BindAddress, "public-network-address", defaultConfig.PublicNetworkConfig.BindAddress, "staked access node's public network bind address") - flags.BoolVar(&builder.rpcConf.BackendConfig.CircuitBreakerConfig.Enabled, "circuit-breaker-enabled", defaultConfig.rpcConf.BackendConfig.CircuitBreakerConfig.Enabled, "specifies whether the circuit breaker is enabled for collection and execution API clients.") - flags.DurationVar(&builder.rpcConf.BackendConfig.CircuitBreakerConfig.RestoreTimeout, "circuit-breaker-restore-timeout", defaultConfig.rpcConf.BackendConfig.CircuitBreakerConfig.RestoreTimeout, "duration after which the circuit breaker will restore the connection to the client after closing it due to failures. Default value is 60s") - flags.Uint32Var(&builder.rpcConf.BackendConfig.CircuitBreakerConfig.MaxFailures, "circuit-breaker-max-failures", defaultConfig.rpcConf.BackendConfig.CircuitBreakerConfig.MaxFailures, "maximum number of failed calls to the client that will cause the circuit breaker to close the connection. Default value is 5") - flags.Uint32Var(&builder.rpcConf.BackendConfig.CircuitBreakerConfig.MaxRequests, "circuit-breaker-max-requests", defaultConfig.rpcConf.BackendConfig.CircuitBreakerConfig.MaxRequests, "maximum number of requests to check if connection restored after timeout. Default value is 1") + flags.UintVar(&builder.TxResultCacheSize, + "transaction-result-cache-size", + defaultConfig.TxResultCacheSize, + "transaction result cache size.(Disabled by default i.e 0)") + flags.StringVarP(&builder.nodeInfoFile, + "node-info-file", + "", + defaultConfig.nodeInfoFile, + "full path to a json file which provides more details about nodes when reporting its reachability metrics") + flags.StringToIntVar(&builder.apiRatelimits, + "api-rate-limits", + defaultConfig.apiRatelimits, + "per second rate limits for Access API methods e.g. Ping=300,GetTransaction=500 etc.") + flags.StringToIntVar(&builder.apiBurstlimits, + "api-burst-limits", + defaultConfig.apiBurstlimits, + "burst limits for Access API methods e.g. Ping=100,GetTransaction=100 etc.") + flags.BoolVar(&builder.supportsObserver, + "supports-observer", + defaultConfig.supportsObserver, + "true if this staked access node supports observer or follower connections") + flags.StringVar(&builder.PublicNetworkConfig.BindAddress, + "public-network-address", + defaultConfig.PublicNetworkConfig.BindAddress, + "staked access node's public network bind address") + flags.BoolVar(&builder.rpcConf.BackendConfig.CircuitBreakerConfig.Enabled, + "circuit-breaker-enabled", + defaultConfig.rpcConf.BackendConfig.CircuitBreakerConfig.Enabled, + "specifies whether the circuit breaker is enabled for collection and execution API clients.") + flags.DurationVar(&builder.rpcConf.BackendConfig.CircuitBreakerConfig.RestoreTimeout, + "circuit-breaker-restore-timeout", + defaultConfig.rpcConf.BackendConfig.CircuitBreakerConfig.RestoreTimeout, + "duration after which the circuit breaker will restore the connection to the client after closing it due to failures. Default value is 60s") + flags.Uint32Var(&builder.rpcConf.BackendConfig.CircuitBreakerConfig.MaxFailures, + "circuit-breaker-max-failures", + defaultConfig.rpcConf.BackendConfig.CircuitBreakerConfig.MaxFailures, + "maximum number of failed calls to the client that will cause the circuit breaker to close the connection. Default value is 5") + flags.Uint32Var(&builder.rpcConf.BackendConfig.CircuitBreakerConfig.MaxRequests, + "circuit-breaker-max-requests", + defaultConfig.rpcConf.BackendConfig.CircuitBreakerConfig.MaxRequests, + "maximum number of requests to check if connection restored after timeout. Default value is 1") // ExecutionDataRequester config - flags.BoolVar(&builder.executionDataSyncEnabled, "execution-data-sync-enabled", defaultConfig.executionDataSyncEnabled, "whether to enable the execution data sync protocol") + flags.BoolVar(&builder.executionDataSyncEnabled, + "execution-data-sync-enabled", + defaultConfig.executionDataSyncEnabled, + "whether to enable the execution data sync protocol") flags.StringVar(&builder.executionDataDir, "execution-data-dir", defaultConfig.executionDataDir, "directory to use for Execution Data database") - flags.Uint64Var(&builder.executionDataStartHeight, "execution-data-start-height", defaultConfig.executionDataStartHeight, "height of first block to sync execution data from when starting with an empty Execution Data database") - flags.Uint64Var(&builder.executionDataConfig.MaxSearchAhead, "execution-data-max-search-ahead", defaultConfig.executionDataConfig.MaxSearchAhead, "max number of heights to search ahead of the lowest outstanding execution data height") - flags.DurationVar(&builder.executionDataConfig.FetchTimeout, "execution-data-fetch-timeout", defaultConfig.executionDataConfig.FetchTimeout, "initial timeout to use when fetching execution data from the network. timeout increases using an incremental backoff until execution-data-max-fetch-timeout. e.g. 30s") - flags.DurationVar(&builder.executionDataConfig.MaxFetchTimeout, "execution-data-max-fetch-timeout", defaultConfig.executionDataConfig.MaxFetchTimeout, "maximum timeout to use when fetching execution data from the network e.g. 300s") - flags.DurationVar(&builder.executionDataConfig.RetryDelay, "execution-data-retry-delay", defaultConfig.executionDataConfig.RetryDelay, "initial delay for exponential backoff when fetching execution data fails e.g. 10s") - flags.DurationVar(&builder.executionDataConfig.MaxRetryDelay, "execution-data-max-retry-delay", defaultConfig.executionDataConfig.MaxRetryDelay, "maximum delay for exponential backoff when fetching execution data fails e.g. 5m") + flags.Uint64Var(&builder.executionDataStartHeight, + "execution-data-start-height", + defaultConfig.executionDataStartHeight, + "height of first block to sync execution data from when starting with an empty Execution Data database") + flags.Uint64Var(&builder.executionDataConfig.MaxSearchAhead, + "execution-data-max-search-ahead", + defaultConfig.executionDataConfig.MaxSearchAhead, + "max number of heights to search ahead of the lowest outstanding execution data height") + flags.DurationVar(&builder.executionDataConfig.FetchTimeout, + "execution-data-fetch-timeout", + defaultConfig.executionDataConfig.FetchTimeout, + "initial timeout to use when fetching execution data from the network. timeout increases using an incremental backoff until execution-data-max-fetch-timeout. e.g. 30s") + flags.DurationVar(&builder.executionDataConfig.MaxFetchTimeout, + "execution-data-max-fetch-timeout", + defaultConfig.executionDataConfig.MaxFetchTimeout, + "maximum timeout to use when fetching execution data from the network e.g. 300s") + flags.DurationVar(&builder.executionDataConfig.RetryDelay, + "execution-data-retry-delay", + defaultConfig.executionDataConfig.RetryDelay, + "initial delay for exponential backoff when fetching execution data fails e.g. 10s") + flags.DurationVar(&builder.executionDataConfig.MaxRetryDelay, + "execution-data-max-retry-delay", + defaultConfig.executionDataConfig.MaxRetryDelay, + "maximum delay for exponential backoff when fetching execution data fails e.g. 5m") // Execution State Streaming API - flags.Uint32Var(&builder.stateStreamConf.ExecutionDataCacheSize, "execution-data-cache-size", defaultConfig.stateStreamConf.ExecutionDataCacheSize, "block execution data cache size") - flags.Uint32Var(&builder.stateStreamConf.MaxGlobalStreams, "state-stream-global-max-streams", defaultConfig.stateStreamConf.MaxGlobalStreams, "global maximum number of concurrent streams") - flags.UintVar(&builder.stateStreamConf.MaxExecutionDataMsgSize, "state-stream-max-message-size", defaultConfig.stateStreamConf.MaxExecutionDataMsgSize, "maximum size for a gRPC message containing block execution data") - flags.StringToIntVar(&builder.stateStreamFilterConf, "state-stream-event-filter-limits", defaultConfig.stateStreamFilterConf, "event filter limits for ExecutionData SubscribeEvents API e.g. EventTypes=100,Addresses=100,Contracts=100 etc.") - flags.DurationVar(&builder.stateStreamConf.ClientSendTimeout, "state-stream-send-timeout", defaultConfig.stateStreamConf.ClientSendTimeout, "maximum wait before timing out while sending a response to a streaming client e.g. 30s") - flags.UintVar(&builder.stateStreamConf.ClientSendBufferSize, "state-stream-send-buffer-size", defaultConfig.stateStreamConf.ClientSendBufferSize, "maximum number of responses to buffer within a stream") - flags.Float64Var(&builder.stateStreamConf.ResponseLimit, "state-stream-response-limit", defaultConfig.stateStreamConf.ResponseLimit, "max number of responses per second to send over streaming endpoints. this helps manage resources consumed by each client querying data not in the cache e.g. 3 or 0.5. 0 means no limit") - flags.Uint64Var(&builder.stateStreamConf.HeartbeatInterval, "state-stream-heartbeat-interval", defaultConfig.stateStreamConf.HeartbeatInterval, "default interval in blocks at which heartbeat messages should be sent. applied when client did not specify a value.") + flags.Uint32Var(&builder.stateStreamConf.ExecutionDataCacheSize, + "execution-data-cache-size", + defaultConfig.stateStreamConf.ExecutionDataCacheSize, + "block execution data cache size") + flags.Uint32Var(&builder.stateStreamConf.MaxGlobalStreams, + "state-stream-global-max-streams", + defaultConfig.stateStreamConf.MaxGlobalStreams, + "global maximum number of concurrent streams") + flags.UintVar(&builder.stateStreamConf.MaxExecutionDataMsgSize, + "state-stream-max-message-size", + defaultConfig.stateStreamConf.MaxExecutionDataMsgSize, + "maximum size for a gRPC message containing block execution data") + flags.StringToIntVar(&builder.stateStreamFilterConf, + "state-stream-event-filter-limits", + defaultConfig.stateStreamFilterConf, + "event filter limits for ExecutionData SubscribeEvents API e.g. EventTypes=100,Addresses=100,Contracts=100 etc.") + flags.DurationVar(&builder.stateStreamConf.ClientSendTimeout, + "state-stream-send-timeout", + defaultConfig.stateStreamConf.ClientSendTimeout, + "maximum wait before timing out while sending a response to a streaming client e.g. 30s") + flags.UintVar(&builder.stateStreamConf.ClientSendBufferSize, + "state-stream-send-buffer-size", + defaultConfig.stateStreamConf.ClientSendBufferSize, + "maximum number of responses to buffer within a stream") + flags.Float64Var(&builder.stateStreamConf.ResponseLimit, + "state-stream-response-limit", + defaultConfig.stateStreamConf.ResponseLimit, + "max number of responses per second to send over streaming endpoints. this helps manage resources consumed by each client querying data not in the cache e.g. 3 or 0.5. 0 means no limit") + flags.Uint64Var(&builder.stateStreamConf.HeartbeatInterval, + "state-stream-heartbeat-interval", + defaultConfig.stateStreamConf.HeartbeatInterval, + "default interval in blocks at which heartbeat messages should be sent. applied when client did not specify a value.") // Execution Data Indexer - flags.BoolVar(&builder.executionDataIndexingEnabled, "execution-data-indexing-enabled", defaultConfig.executionDataIndexingEnabled, "whether to enable the execution data indexing") + flags.BoolVar(&builder.executionDataIndexingEnabled, + "execution-data-indexing-enabled", + defaultConfig.executionDataIndexingEnabled, + "whether to enable the execution data indexing") flags.StringVar(&builder.registersDBPath, "execution-state-dir", defaultConfig.registersDBPath, "directory to use for execution-state database") flags.StringVar(&builder.checkpointFile, "execution-state-checkpoint", defaultConfig.checkpointFile, "execution-state checkpoint file") // Script Execution - flags.StringVar(&builder.rpcConf.BackendConfig.ScriptExecutionMode, "script-execution-mode", defaultConfig.rpcConf.BackendConfig.ScriptExecutionMode, "mode to use when executing scripts. one of (local-only, execution-nodes-only, failover, compare)") + flags.StringVar(&builder.rpcConf.BackendConfig.ScriptExecutionMode, + "script-execution-mode", + defaultConfig.rpcConf.BackendConfig.ScriptExecutionMode, + "mode to use when executing scripts. one of (local-only, execution-nodes-only, failover, compare)") }).ValidateFlags(func() error { if builder.supportsObserver && (builder.PublicNetworkConfig.BindAddress == cmd.NotSet || builder.PublicNetworkConfig.BindAddress == "") { return errors.New("public-network-address must be set if supports-observer is true") @@ -1526,26 +1668,14 @@ func (builder *FlowAccessNodeBuilder) enqueuePublicNetworkInit() { // Returns: // - The libp2p node instance for the public network. // - Any error encountered during initialization. Any error should be considered fatal. -func (builder *FlowAccessNodeBuilder) initPublicLibp2pNode(networkKey crypto.PrivateKey, bindAddress string, networkMetrics module.LibP2PMetrics) (p2p.LibP2PNode, error) { +func (builder *FlowAccessNodeBuilder) initPublicLibp2pNode(networkKey crypto.PrivateKey, bindAddress string, networkMetrics module.LibP2PMetrics) (p2p.LibP2PNode, + error) { connManager, err := connection.NewConnManager(builder.Logger, networkMetrics, &builder.FlowConfig.NetworkConfig.ConnectionManagerConfig) if err != nil { return nil, fmt.Errorf("could not create connection manager: %w", err) } - meshTracerCfg := &tracer.GossipSubMeshTracerConfig{ - Logger: builder.Logger, - Metrics: networkMetrics, - IDProvider: builder.IdentityProvider, - LoggerInterval: builder.FlowConfig.NetworkConfig.GossipSubConfig.LocalMeshLogInterval, - RpcSentTrackerCacheSize: builder.FlowConfig.NetworkConfig.GossipSubConfig.RPCSentTrackerCacheSize, - RpcSentTrackerWorkerQueueCacheSize: builder.FlowConfig.NetworkConfig.GossipSubConfig.RPCSentTrackerQueueCacheSize, - RpcSentTrackerNumOfWorkers: builder.FlowConfig.NetworkConfig.GossipSubConfig.RpcSentTrackerNumOfWorkers, - HeroCacheMetricsFactory: builder.HeroCacheMetricsFactory(), - NetworkingType: network.PublicNetwork, - } - meshTracer := tracer.NewGossipSubMeshTracer(meshTracerCfg) - - libp2pNode, err := p2pbuilder.NewNodeBuilder(builder.Logger, &p2pconfig.MetricsConfig{ + libp2pNode, err := p2pbuilder.NewNodeBuilder(builder.Logger, &builder.FlowConfig.NetworkConfig.GossipSub, &p2pconfig.MetricsConfig{ HeroCacheFactory: builder.HeroCacheMetricsFactory(), Metrics: networkMetrics, }, @@ -1555,7 +1685,6 @@ func (builder *FlowAccessNodeBuilder) initPublicLibp2pNode(networkKey crypto.Pri builder.SporkID, builder.IdentityProvider, &builder.FlowConfig.NetworkConfig.ResourceManager, - &builder.FlowConfig.NetworkConfig.GossipSubConfig.GossipSubRPCInspectorsConfig, &p2pconfig.PeerManagerConfig{ // TODO: eventually, we need pruning enabled even on public network. However, it needs a modified version of // the peer manager that also operate on the public identities. @@ -1563,12 +1692,10 @@ func (builder *FlowAccessNodeBuilder) initPublicLibp2pNode(networkKey crypto.Pri UpdateInterval: builder.FlowConfig.NetworkConfig.PeerUpdateInterval, ConnectorFactory: connection.DefaultLibp2pBackoffConnectorFactory(), }, - &builder.FlowConfig.NetworkConfig.GossipSubConfig.SubscriptionProvider, &p2p.DisallowListCacheConfig{ MaxSize: builder.FlowConfig.NetworkConfig.DisallowListNotificationCacheSize, Metrics: metrics.DisallowListCacheMetricsFactory(builder.HeroCacheMetricsFactory(), network.PublicNetwork), }, - meshTracer, &p2pconfig.UnicastConfig{ UnicastConfig: builder.FlowConfig.NetworkConfig.UnicastConfig, }). @@ -1578,9 +1705,6 @@ func (builder *FlowAccessNodeBuilder) initPublicLibp2pNode(networkKey crypto.Pri SetRoutingSystem(func(ctx context.Context, h host.Host) (routing.Routing, error) { return dht.NewDHT(ctx, h, protocols.FlowPublicDHTProtocolID(builder.SporkID), builder.Logger, networkMetrics, dht.AsServer()) }). - // disable connection pruning for the access node which supports the observer - SetGossipSubTracer(meshTracer). - SetGossipSubScoreTracerInterval(builder.FlowConfig.NetworkConfig.GossipSubConfig.ScoreTracerInterval). Build() if err != nil { diff --git a/cmd/observer/node_builder/observer_builder.go b/cmd/observer/node_builder/observer_builder.go index a42e0eb82a4..2b508dec808 100644 --- a/cmd/observer/node_builder/observer_builder.go +++ b/cmd/observer/node_builder/observer_builder.go @@ -64,7 +64,6 @@ import ( "github.com/onflow/flow-go/network/p2p/p2plogging" "github.com/onflow/flow-go/network/p2p/p2pnet" "github.com/onflow/flow-go/network/p2p/subscription" - "github.com/onflow/flow-go/network/p2p/tracer" "github.com/onflow/flow-go/network/p2p/translator" "github.com/onflow/flow-go/network/p2p/unicast/protocols" "github.com/onflow/flow-go/network/p2p/utils" @@ -468,24 +467,70 @@ func (builder *ObserverServiceBuilder) extraFlags() { builder.ExtraFlags(func(flags *pflag.FlagSet) { defaultConfig := DefaultObserverServiceConfig() - flags.StringVarP(&builder.rpcConf.UnsecureGRPCListenAddr, "rpc-addr", "r", defaultConfig.rpcConf.UnsecureGRPCListenAddr, "the address the unsecured gRPC server listens on") - flags.StringVar(&builder.rpcConf.SecureGRPCListenAddr, "secure-rpc-addr", defaultConfig.rpcConf.SecureGRPCListenAddr, "the address the secure gRPC server listens on") + flags.StringVarP(&builder.rpcConf.UnsecureGRPCListenAddr, + "rpc-addr", + "r", + defaultConfig.rpcConf.UnsecureGRPCListenAddr, + "the address the unsecured gRPC server listens on") + flags.StringVar(&builder.rpcConf.SecureGRPCListenAddr, + "secure-rpc-addr", + defaultConfig.rpcConf.SecureGRPCListenAddr, + "the address the secure gRPC server listens on") flags.StringVarP(&builder.rpcConf.HTTPListenAddr, "http-addr", "h", defaultConfig.rpcConf.HTTPListenAddr, "the address the http proxy server listens on") - flags.StringVar(&builder.rpcConf.RestConfig.ListenAddress, "rest-addr", defaultConfig.rpcConf.RestConfig.ListenAddress, "the address the REST server listens on (if empty the REST server will not be started)") - flags.DurationVar(&builder.rpcConf.RestConfig.WriteTimeout, "rest-write-timeout", defaultConfig.rpcConf.RestConfig.WriteTimeout, "timeout to use when writing REST response") - flags.DurationVar(&builder.rpcConf.RestConfig.ReadTimeout, "rest-read-timeout", defaultConfig.rpcConf.RestConfig.ReadTimeout, "timeout to use when reading REST request headers") + flags.StringVar(&builder.rpcConf.RestConfig.ListenAddress, + "rest-addr", + defaultConfig.rpcConf.RestConfig.ListenAddress, + "the address the REST server listens on (if empty the REST server will not be started)") + flags.DurationVar(&builder.rpcConf.RestConfig.WriteTimeout, + "rest-write-timeout", + defaultConfig.rpcConf.RestConfig.WriteTimeout, + "timeout to use when writing REST response") + flags.DurationVar(&builder.rpcConf.RestConfig.ReadTimeout, + "rest-read-timeout", + defaultConfig.rpcConf.RestConfig.ReadTimeout, + "timeout to use when reading REST request headers") flags.DurationVar(&builder.rpcConf.RestConfig.IdleTimeout, "rest-idle-timeout", defaultConfig.rpcConf.RestConfig.IdleTimeout, "idle timeout for REST connections") - flags.UintVar(&builder.rpcConf.MaxMsgSize, "rpc-max-message-size", defaultConfig.rpcConf.MaxMsgSize, "the maximum message size in bytes for messages sent or received over grpc") - flags.UintVar(&builder.rpcConf.BackendConfig.ConnectionPoolSize, "connection-pool-size", defaultConfig.rpcConf.BackendConfig.ConnectionPoolSize, "maximum number of connections allowed in the connection pool, size of 0 disables the connection pooling, and anything less than the default size will be overridden to use the default size") - flags.UintVar(&builder.rpcConf.BackendConfig.MaxHeightRange, "rpc-max-height-range", defaultConfig.rpcConf.BackendConfig.MaxHeightRange, "maximum size for height range requests") - flags.StringToIntVar(&builder.apiRatelimits, "api-rate-limits", defaultConfig.apiRatelimits, "per second rate limits for Access API methods e.g. Ping=300,GetTransaction=500 etc.") - flags.StringToIntVar(&builder.apiBurstlimits, "api-burst-limits", defaultConfig.apiBurstlimits, "burst limits for Access API methods e.g. Ping=100,GetTransaction=100 etc.") - flags.StringVar(&builder.observerNetworkingKeyPath, "observer-networking-key-path", defaultConfig.observerNetworkingKeyPath, "path to the networking key for observer") - flags.StringSliceVar(&builder.bootstrapNodeAddresses, "bootstrap-node-addresses", defaultConfig.bootstrapNodeAddresses, "the network addresses of the bootstrap access node if this is an observer e.g. access-001.mainnet.flow.org:9653,access-002.mainnet.flow.org:9653") - flags.StringSliceVar(&builder.bootstrapNodePublicKeys, "bootstrap-node-public-keys", defaultConfig.bootstrapNodePublicKeys, "the networking public key of the bootstrap access node if this is an observer (in the same order as the bootstrap node addresses) e.g. \"d57a5e9c5.....\",\"44ded42d....\"") + flags.UintVar(&builder.rpcConf.MaxMsgSize, + "rpc-max-message-size", + defaultConfig.rpcConf.MaxMsgSize, + "the maximum message size in bytes for messages sent or received over grpc") + flags.UintVar(&builder.rpcConf.BackendConfig.ConnectionPoolSize, + "connection-pool-size", + defaultConfig.rpcConf.BackendConfig.ConnectionPoolSize, + "maximum number of connections allowed in the connection pool, size of 0 disables the connection pooling, and anything less than the default size will be overridden to use the default size") + flags.UintVar(&builder.rpcConf.BackendConfig.MaxHeightRange, + "rpc-max-height-range", + defaultConfig.rpcConf.BackendConfig.MaxHeightRange, + "maximum size for height range requests") + flags.StringToIntVar(&builder.apiRatelimits, + "api-rate-limits", + defaultConfig.apiRatelimits, + "per second rate limits for Access API methods e.g. Ping=300,GetTransaction=500 etc.") + flags.StringToIntVar(&builder.apiBurstlimits, + "api-burst-limits", + defaultConfig.apiBurstlimits, + "burst limits for Access API methods e.g. Ping=100,GetTransaction=100 etc.") + flags.StringVar(&builder.observerNetworkingKeyPath, + "observer-networking-key-path", + defaultConfig.observerNetworkingKeyPath, + "path to the networking key for observer") + flags.StringSliceVar(&builder.bootstrapNodeAddresses, + "bootstrap-node-addresses", + defaultConfig.bootstrapNodeAddresses, + "the network addresses of the bootstrap access node if this is an observer e.g. access-001.mainnet.flow.org:9653,access-002.mainnet.flow.org:9653") + flags.StringSliceVar(&builder.bootstrapNodePublicKeys, + "bootstrap-node-public-keys", + defaultConfig.bootstrapNodePublicKeys, + "the networking public key of the bootstrap access node if this is an observer (in the same order as the bootstrap node addresses) e.g. \"d57a5e9c5.....\",\"44ded42d....\"") flags.DurationVar(&builder.apiTimeout, "upstream-api-timeout", defaultConfig.apiTimeout, "tcp timeout for Flow API gRPC sockets to upstrem nodes") - flags.StringSliceVar(&builder.upstreamNodeAddresses, "upstream-node-addresses", defaultConfig.upstreamNodeAddresses, "the gRPC network addresses of the upstream access node. e.g. access-001.mainnet.flow.org:9000,access-002.mainnet.flow.org:9000") - flags.StringSliceVar(&builder.upstreamNodePublicKeys, "upstream-node-public-keys", defaultConfig.upstreamNodePublicKeys, "the networking public key of the upstream access node (in the same order as the upstream node addresses) e.g. \"d57a5e9c5.....\",\"44ded42d....\"") + flags.StringSliceVar(&builder.upstreamNodeAddresses, + "upstream-node-addresses", + defaultConfig.upstreamNodeAddresses, + "the gRPC network addresses of the upstream access node. e.g. access-001.mainnet.flow.org:9000,access-002.mainnet.flow.org:9000") + flags.StringSliceVar(&builder.upstreamNodePublicKeys, + "upstream-node-public-keys", + defaultConfig.upstreamNodePublicKeys, + "the networking public key of the upstream access node (in the same order as the upstream node addresses) e.g. \"d57a5e9c5.....\",\"44ded42d....\"") flags.BoolVar(&builder.rpcMetricsEnabled, "rpc-metrics-enabled", defaultConfig.rpcMetricsEnabled, "whether to enable the rpc metrics") }) } @@ -695,21 +740,9 @@ func (builder *ObserverServiceBuilder) initPublicLibp2pNode(networkKey crypto.Pr pis = append(pis, pi) } - meshTracerCfg := &tracer.GossipSubMeshTracerConfig{ - Logger: builder.Logger, - Metrics: builder.Metrics.Network, - IDProvider: builder.IdentityProvider, - LoggerInterval: builder.FlowConfig.NetworkConfig.GossipSubConfig.LocalMeshLogInterval, - RpcSentTrackerCacheSize: builder.FlowConfig.NetworkConfig.GossipSubConfig.RPCSentTrackerCacheSize, - RpcSentTrackerWorkerQueueCacheSize: builder.FlowConfig.NetworkConfig.GossipSubConfig.RPCSentTrackerQueueCacheSize, - RpcSentTrackerNumOfWorkers: builder.FlowConfig.NetworkConfig.GossipSubConfig.RpcSentTrackerNumOfWorkers, - HeroCacheMetricsFactory: builder.HeroCacheMetricsFactory(), - NetworkingType: network.PublicNetwork, - } - meshTracer := tracer.NewGossipSubMeshTracer(meshTracerCfg) - node, err := p2pbuilder.NewNodeBuilder( builder.Logger, + &builder.FlowConfig.NetworkConfig.GossipSub, &p2pconfig.MetricsConfig{ HeroCacheFactory: builder.HeroCacheMetricsFactory(), Metrics: builder.Metrics.Network, @@ -720,14 +753,11 @@ func (builder *ObserverServiceBuilder) initPublicLibp2pNode(networkKey crypto.Pr builder.SporkID, builder.IdentityProvider, &builder.FlowConfig.NetworkConfig.ResourceManager, - &builder.FlowConfig.NetworkConfig.GossipSubConfig.GossipSubRPCInspectorsConfig, p2pconfig.PeerManagerDisableConfig(), // disable peer manager for observer node. - &builder.FlowConfig.NetworkConfig.GossipSubConfig.SubscriptionProvider, &p2p.DisallowListCacheConfig{ MaxSize: builder.FlowConfig.NetworkConfig.DisallowListNotificationCacheSize, Metrics: metrics.DisallowListCacheMetricsFactory(builder.HeroCacheMetricsFactory(), network.PublicNetwork), }, - meshTracer, &p2pconfig.UnicastConfig{ UnicastConfig: builder.FlowConfig.NetworkConfig.UnicastConfig, }). @@ -744,8 +774,6 @@ func (builder *ObserverServiceBuilder) initPublicLibp2pNode(networkKey crypto.Pr dht.BootstrapPeers(pis...), ) }). - SetGossipSubTracer(meshTracer). - SetGossipSubScoreTracerInterval(builder.FlowConfig.NetworkConfig.GossipSubConfig.ScoreTracerInterval). Build() if err != nil { diff --git a/network/p2p/builder.go b/network/p2p/builder.go index 1665b838647..5a8733ec0e6 100644 --- a/network/p2p/builder.go +++ b/network/p2p/builder.go @@ -2,7 +2,6 @@ package p2p import ( "context" - "time" pubsub "github.com/libp2p/go-libp2p-pubsub" "github.com/libp2p/go-libp2p/core/connmgr" @@ -55,14 +54,6 @@ type GossipSubBuilder interface { // none EnableGossipSubScoringWithOverride(*PeerScoringConfigOverride) - // SetGossipSubScoreTracerInterval sets the gossipsub score tracer interval of the builder. - // If the gossipsub score tracer interval has already been set, a fatal error is logged. - SetGossipSubScoreTracerInterval(time.Duration) - - // SetGossipSubTracer sets the gossipsub tracer of the builder. - // If the gossipsub tracer has already been set, a fatal error is logged. - SetGossipSubTracer(PubSubTracer) - // SetRoutingSystem sets the routing system of the builder. // If the routing system has already been set, a fatal error is logged. SetRoutingSystem(routing.Routing) @@ -134,8 +125,6 @@ type NodeBuilder interface { EnableGossipSubScoringWithOverride(*PeerScoringConfigOverride) NodeBuilder SetCreateNode(CreateNodeFunc) NodeBuilder SetGossipSubFactory(GossipSubFactoryFunc, GossipSubAdapterConfigFunc) NodeBuilder - SetGossipSubTracer(PubSubTracer) NodeBuilder - SetGossipSubScoreTracerInterval(time.Duration) NodeBuilder OverrideDefaultRpcInspectorSuiteFactory(GossipSubRpcInspectorSuiteFactoryFunc) NodeBuilder Build() (LibP2PNode, error) } diff --git a/network/p2p/inspector/validation/control_message_validation_inspector_test.go b/network/p2p/inspector/validation/control_message_validation_inspector_test.go index b505e32622b..fd8e7ac8d5f 100644 --- a/network/p2p/inspector/validation/control_message_validation_inspector_test.go +++ b/network/p2p/inspector/validation/control_message_validation_inspector_test.go @@ -62,7 +62,7 @@ func (suite *ControlMsgValidationInspectorSuite) SetupTest() { params := &validation.InspectorParams{ Logger: unittest.Logger(), SporkID: suite.sporkID, - Config: &flowConfig.NetworkConfig.GossipSubRPCValidationInspectorConfigs, + Config: &flowConfig.NetworkConfig.GossipSub.RpcInspector.Validation, Distributor: distributor, IdProvider: suite.idProvider, HeroCacheMetricsFactory: metrics.NewNoopHeroCacheMetricsFactory(), @@ -98,7 +98,7 @@ func TestNewControlMsgValidationInspector(t *testing.T) { inspector, err := validation.NewControlMsgValidationInspector(&validation.InspectorParams{ Logger: unittest.Logger(), SporkID: sporkID, - Config: &flowConfig.NetworkConfig.GossipSubRPCValidationInspectorConfigs, + Config: &flowConfig.NetworkConfig.GossipSub.RpcInspector.Validation, Distributor: distributor, IdProvider: idProvider, HeroCacheMetricsFactory: metrics.NewNoopHeroCacheMetricsFactory(), @@ -203,22 +203,22 @@ func (suite *ControlMsgValidationInspectorSuite) TestControlMessageValidationIns suite.rpcTracker.On("LastHighestIHaveRPCSize").Return(int64(100)).Maybe() suite.rpcTracker.On("WasIHaveRPCSent", mock.AnythingOfType("string")).Return(true).Maybe() suite.distributor.On("Distribute", mock.AnythingOfType("*p2p.InvCtrlMsgNotif")).Return(nil).Twice() - suite.config.IHaveRPCInspectionConfig.MaxSampleSize = 100 + suite.config.IHave.MaxSampleSize = 100 suite.inspector.Start(suite.signalerCtx) // topic validation not performed so we can use random strings iHavesGreaterThanMaxSampleSize := unittest.P2PRPCFixture(unittest.WithIHaves(unittest.P2PRPCIHaveFixtures(200, unittest.IdentifierListFixture(200).Strings()...)...)) - require.Greater(t, len(iHavesGreaterThanMaxSampleSize.GetControl().GetIhave()), suite.config.IHaveRPCInspectionConfig.MaxSampleSize) + require.Greater(t, len(iHavesGreaterThanMaxSampleSize.GetControl().GetIhave()), suite.config.IHave.MaxSampleSize) iHavesLessThanMaxSampleSize := unittest.P2PRPCFixture(unittest.WithIHaves(unittest.P2PRPCIHaveFixtures(200, unittest.IdentifierListFixture(50).Strings()...)...)) - require.Less(t, len(iHavesLessThanMaxSampleSize.GetControl().GetIhave()), suite.config.IHaveRPCInspectionConfig.MaxSampleSize) + require.Less(t, len(iHavesLessThanMaxSampleSize.GetControl().GetIhave()), suite.config.IHave.MaxSampleSize) from := unittest.PeerIdFixture(t) require.NoError(t, suite.inspector.Inspect(from, iHavesGreaterThanMaxSampleSize)) require.NoError(t, suite.inspector.Inspect(from, iHavesLessThanMaxSampleSize)) require.Eventually(t, func() bool { // rpc with iHaves greater than configured max sample size should be truncated to MaxSampleSize - shouldBeTruncated := len(iHavesGreaterThanMaxSampleSize.GetControl().GetIhave()) == suite.config.IHaveRPCInspectionConfig.MaxSampleSize + shouldBeTruncated := len(iHavesGreaterThanMaxSampleSize.GetControl().GetIhave()) == suite.config.IHave.MaxSampleSize // rpc with iHaves less than MaxSampleSize should not be truncated shouldNotBeTruncated := len(iHavesLessThanMaxSampleSize.GetControl().GetIhave()) == 50 return shouldBeTruncated && shouldNotBeTruncated @@ -231,7 +231,7 @@ func (suite *ControlMsgValidationInspectorSuite) TestControlMessageValidationIns suite.rpcTracker.On("LastHighestIHaveRPCSize").Return(int64(100)).Maybe() suite.rpcTracker.On("WasIHaveRPCSent", mock.AnythingOfType("string")).Return(true).Maybe() suite.distributor.On("Distribute", mock.AnythingOfType("*p2p.InvCtrlMsgNotif")).Return(nil).Twice() - suite.config.IHaveRPCInspectionConfig.MaxMessageIDSampleSize = 100 + suite.config.IHave.MaxMessageIDSampleSize = 100 suite.inspector.Start(suite.signalerCtx) // topic validation not performed so we can use random strings @@ -245,7 +245,7 @@ func (suite *ControlMsgValidationInspectorSuite) TestControlMessageValidationIns require.Eventually(t, func() bool { for _, iHave := range iHavesGreaterThanMaxSampleSize.GetControl().GetIhave() { // rpc with iHaves message ids greater than configured max sample size should be truncated to MaxSampleSize - if len(iHave.GetMessageIDs()) != suite.config.IHaveRPCInspectionConfig.MaxMessageIDSampleSize { + if len(iHave.GetMessageIDs()) != suite.config.IHave.MaxMessageIDSampleSize { return false } } @@ -264,20 +264,20 @@ func (suite *ControlMsgValidationInspectorSuite) TestControlMessageValidationIns defer suite.StopInspector() suite.rpcTracker.On("LastHighestIHaveRPCSize").Return(int64(100)).Maybe() suite.rpcTracker.On("WasIHaveRPCSent", mock.AnythingOfType("string")).Return(true).Maybe() - suite.config.IWantRPCInspectionConfig.MaxSampleSize = 100 + suite.config.IWant.MaxSampleSize = 100 suite.inspector.Start(suite.signalerCtx) iWantsGreaterThanMaxSampleSize := unittest.P2PRPCFixture(unittest.WithIWants(unittest.P2PRPCIWantFixtures(200, 200)...)) - require.Greater(t, uint(len(iWantsGreaterThanMaxSampleSize.GetControl().GetIwant())), suite.config.IWantRPCInspectionConfig.MaxSampleSize) + require.Greater(t, uint(len(iWantsGreaterThanMaxSampleSize.GetControl().GetIwant())), suite.config.IWant.MaxSampleSize) iWantsLessThanMaxSampleSize := unittest.P2PRPCFixture(unittest.WithIWants(unittest.P2PRPCIWantFixtures(50, 200)...)) - require.Less(t, uint(len(iWantsLessThanMaxSampleSize.GetControl().GetIwant())), suite.config.IWantRPCInspectionConfig.MaxSampleSize) + require.Less(t, uint(len(iWantsLessThanMaxSampleSize.GetControl().GetIwant())), suite.config.IWant.MaxSampleSize) from := unittest.PeerIdFixture(t) require.NoError(t, suite.inspector.Inspect(from, iWantsGreaterThanMaxSampleSize)) require.NoError(t, suite.inspector.Inspect(from, iWantsLessThanMaxSampleSize)) require.Eventually(t, func() bool { // rpc with iWants greater than configured max sample size should be truncated to MaxSampleSize - shouldBeTruncated := len(iWantsGreaterThanMaxSampleSize.GetControl().GetIwant()) == int(suite.config.IWantRPCInspectionConfig.MaxSampleSize) + shouldBeTruncated := len(iWantsGreaterThanMaxSampleSize.GetControl().GetIwant()) == int(suite.config.IWant.MaxSampleSize) // rpc with iWants less than MaxSampleSize should not be truncated shouldNotBeTruncated := len(iWantsLessThanMaxSampleSize.GetControl().GetIwant()) == 50 return shouldBeTruncated && shouldNotBeTruncated @@ -290,7 +290,7 @@ func (suite *ControlMsgValidationInspectorSuite) TestControlMessageValidationIns suite.rpcTracker.On("LastHighestIHaveRPCSize").Return(int64(100)).Maybe() suite.rpcTracker.On("WasIHaveRPCSent", mock.AnythingOfType("string")).Return(true).Maybe() suite.distributor.On("Distribute", mock.AnythingOfType("*p2p.InvCtrlMsgNotif")).Return(nil).Maybe() - suite.config.IWantRPCInspectionConfig.MaxMessageIDSampleSize = 100 + suite.config.IWant.MaxMessageIDSampleSize = 100 suite.inspector.Start(suite.signalerCtx) iWantsGreaterThanMaxSampleSize := unittest.P2PRPCFixture(unittest.WithIWants(unittest.P2PRPCIWantFixtures(10, 200)...)) @@ -302,7 +302,7 @@ func (suite *ControlMsgValidationInspectorSuite) TestControlMessageValidationIns require.Eventually(t, func() bool { for _, iWant := range iWantsGreaterThanMaxSampleSize.GetControl().GetIwant() { // rpc with iWants message ids greater than configured max sample size should be truncated to MaxSampleSize - if len(iWant.GetMessageIDs()) != suite.config.IWantRPCInspectionConfig.MaxMessageIDSampleSize { + if len(iWant.GetMessageIDs()) != suite.config.IWant.MaxMessageIDSampleSize { return false } } @@ -535,9 +535,9 @@ func (suite *ControlMsgValidationInspectorSuite) TestControlMessageValidationIns suite.SetupTest() defer suite.StopInspector() // set cache miss check size to 0 forcing the inspector to check the cache misses with only a single iWant - suite.config.CacheMissCheckSize = 0 + suite.config.IWant.CacheMissCheckSize = 0 // set high cache miss threshold to ensure we only disseminate notification when it is exceeded - suite.config.IWantRPCInspectionConfig.CacheMissThreshold = .9 + suite.config.IWant.CacheMissThreshold = .9 msgIds := unittest.IdentifierListFixture(100).Strings() // oracle must be set even though iWant messages do not have topic IDs inspectMsgRpc := unittest.P2PRPCFixture(unittest.WithIWants(unittest.P2PRPCIWantFixture(msgIds...))) @@ -566,9 +566,9 @@ func (suite *ControlMsgValidationInspectorSuite) TestControlMessageValidationIns defer suite.StopInspector() defer suite.distributor.AssertNotCalled(t, "Distribute") // if size of iwants not greater than 10 cache misses will not be checked - suite.config.CacheMissCheckSize = 10 + suite.config.IWant.CacheMissCheckSize = 10 // set high cache miss threshold to ensure we only disseminate notification when it is exceeded - suite.config.IWantRPCInspectionConfig.CacheMissThreshold = .9 + suite.config.IWant.CacheMissThreshold = .9 msgIds := unittest.IdentifierListFixture(100).Strings() inspectMsgRpc := unittest.P2PRPCFixture(unittest.WithIWants(unittest.P2PRPCIWantFixture(msgIds...))) suite.rpcTracker.On("LastHighestIHaveRPCSize").Return(int64(100)).Maybe() @@ -756,7 +756,7 @@ func (suite *ControlMsgValidationInspectorSuite) TestNewControlMsgValidationInsp defer suite.StopInspector() defer suite.distributor.AssertNotCalled(t, "Distribute") // set hard threshold to small number , ensure that a single unknown cluster prefix id does not cause a notification to be disseminated - suite.config.ClusterPrefixHardThreshold = 2 + suite.config.ClusterPrefixedMessage.HardThreshold = 2 defer suite.distributor.AssertNotCalled(t, "Distribute") clusterID := flow.ChainID(unittest.IdentifierFixture().String()) clusterPrefixedTopic := channels.Topic(fmt.Sprintf("%s/%s", channels.SyncCluster(clusterID), suite.sporkID)).String() @@ -794,7 +794,7 @@ func (suite *ControlMsgValidationInspectorSuite) TestNewControlMsgValidationInsp clusterPrefixedTopic := channels.Topic(fmt.Sprintf("%s/%s", channels.SyncCluster(clusterID), suite.sporkID)).String() suite.topicProviderOracle.UpdateTopics([]string{clusterPrefixedTopic}) // the 11th unknown cluster ID error should cause an error - suite.config.ClusterPrefixHardThreshold = 10 + suite.config.ClusterPrefixedMessage.HardThreshold = 10 from := unittest.PeerIdFixture(t) identity := unittest.IdentityFixture() suite.idProvider.On("ByPeerID", from).Return(identity, true).Times(11) diff --git a/network/p2p/p2pbuilder/gossipsub/gossipSubBuilder.go b/network/p2p/p2pbuilder/gossipsub/gossipSubBuilder.go index 3f294b809a5..74094f1ce97 100644 --- a/network/p2p/p2pbuilder/gossipsub/gossipSubBuilder.go +++ b/network/p2p/p2pbuilder/gossipsub/gossipSubBuilder.go @@ -40,17 +40,15 @@ type Builder struct { subscriptionFilter pubsub.SubscriptionFilter gossipSubFactory p2p.GossipSubFactoryFunc gossipSubConfigFunc p2p.GossipSubAdapterConfigFunc - gossipSubPeerScoring bool // whether to enable gossipsub peer scoring gossipSubScoreTracerInterval time.Duration // the interval at which the gossipsub score tracer logs the peer scores. // gossipSubTracer is a callback interface that is called by the gossipsub implementation upon // certain events. Currently, we use it to log and observe the local mesh of the node. - gossipSubTracer p2p.PubSubTracer - scoreOptionConfig *scoring.ScoreOptionConfig - subscriptionProviderParam *p2pconf.SubscriptionProviderParameters - idProvider module.IdentityProvider - routingSystem routing.Routing - rpcInspectorConfig *p2pconf.RpcInspectorParameters - rpcInspectorSuiteFactory p2p.GossipSubRpcInspectorSuiteFactoryFunc + gossipSubTracer p2p.PubSubTracer + scoreOptionConfig *scoring.ScoreOptionConfig + idProvider module.IdentityProvider + routingSystem routing.Routing + rpcInspectorSuiteFactory p2p.GossipSubRpcInspectorSuiteFactoryFunc + gossipSubCfg *p2pconf.GossipSubParameters } var _ p2p.GossipSubBuilder = (*Builder)(nil) @@ -102,7 +100,7 @@ func (g *Builder) SetGossipSubConfigFunc(gossipSubConfigFunc p2p.GossipSubAdapte // Returns: // none func (g *Builder) EnableGossipSubScoringWithOverride(override *p2p.PeerScoringConfigOverride) { - g.gossipSubPeerScoring = true // TODO: we should enable peer scoring by default. + g.gossipSubCfg.PeerScoringEnabled = true // TODO: we should enable peer scoring by default. if override == nil { return } @@ -123,24 +121,10 @@ func (g *Builder) EnableGossipSubScoringWithOverride(override *p2p.PeerScoringCo } } -// SetGossipSubScoreTracerInterval sets the gossipsub score tracer interval of the builder. -// If the gossipsub score tracer interval has already been set, a fatal error is logged. -func (g *Builder) SetGossipSubScoreTracerInterval(gossipSubScoreTracerInterval time.Duration) { - if g.gossipSubScoreTracerInterval != time.Duration(0) { - g.logger.Fatal().Msg("gossipsub score tracer interval has already been set") - return - } - g.gossipSubScoreTracerInterval = gossipSubScoreTracerInterval -} - // SetGossipSubTracer sets the gossipsub tracer of the builder. // If the gossipsub tracer has already been set, a fatal error is logged. func (g *Builder) SetGossipSubTracer(gossipSubTracer p2p.PubSubTracer) { - if g.gossipSubTracer != nil { - g.logger.Fatal().Msg("gossipsub tracer has already been set") - return - } - g.gossipSubTracer = gossipSubTracer + } // SetRoutingSystem sets the routing system of the builder. @@ -171,32 +155,38 @@ func (g *Builder) OverrideDefaultRpcInspectorSuiteFactory(factory p2p.GossipSubR // Returns: // - a new gossipsub builder. // Note: the builder is not thread-safe. It should only be used in the main thread. -func NewGossipSubBuilder( - logger zerolog.Logger, - metricsCfg *p2pconfig.MetricsConfig, - networkType network.NetworkingType, - sporkId flow.Identifier, - idProvider module.IdentityProvider, - rpcInspectorConfig *p2pconf.RpcInspectorParameters, - subscriptionProviderPrams *p2pconf.SubscriptionProviderParameters, - rpcTracker p2p.RpcControlTracking) *Builder { +func NewGossipSubBuilder(logger zerolog.Logger, metricsCfg *p2pconfig.MetricsConfig, gossipSubCfg *p2pconf.GossipSubParameters, + networkType network.NetworkingType, sporkId flow.Identifier, idProvider module.IdentityProvider) *Builder { lg := logger.With(). Str("component", "gossipsub"). Str("network-type", networkType.String()). Logger() + meshTracerCfg := &tracer.GossipSubMeshTracerConfig{ + Logger: lg, + Metrics: metricsCfg.Metrics, + IDProvider: idProvider, + LoggerInterval: gossipSubCfg.RpcTracer.LocalMeshLogInterval, + RpcSentTrackerCacheSize: gossipSubCfg.RpcTracer.RPCSentTrackerCacheSize, + RpcSentTrackerWorkerQueueCacheSize: gossipSubCfg.RpcTracer.RPCSentTrackerQueueCacheSize, + RpcSentTrackerNumOfWorkers: gossipSubCfg.RpcTracer.RpcSentTrackerNumOfWorkers, + HeroCacheMetricsFactory: metricsCfg.HeroCacheFactory, + NetworkingType: networkType, + } + meshTracer := tracer.NewGossipSubMeshTracer(meshTracerCfg) + b := &Builder{ - logger: lg, - metricsCfg: metricsCfg, - sporkId: sporkId, - networkType: networkType, - idProvider: idProvider, - gossipSubFactory: defaultGossipSubFactory(), - gossipSubConfigFunc: defaultGossipSubAdapterConfig(), - scoreOptionConfig: scoring.NewScoreOptionConfig(lg, metricsCfg.HeroCacheFactory, idProvider), - rpcInspectorConfig: rpcInspectorConfig, - rpcInspectorSuiteFactory: defaultInspectorSuite(rpcTracker), - subscriptionProviderParam: subscriptionProviderPrams, + logger: lg, + metricsCfg: metricsCfg, + sporkId: sporkId, + networkType: networkType, + idProvider: idProvider, + gossipSubFactory: defaultGossipSubFactory(), + gossipSubConfigFunc: defaultGossipSubAdapterConfig(), + scoreOptionConfig: scoring.NewScoreOptionConfig(lg, gossipSubCfg.ScoringParameters, metricsCfg.HeroCacheFactory, idProvider), + rpcInspectorSuiteFactory: defaultInspectorSuite(meshTracer), + gossipSubTracer: meshTracer, + gossipSubCfg: gossipSubCfg, } return b @@ -304,9 +294,7 @@ func (g *Builder) Build(ctx irrecoverable.SignalerContext) (p2p.PubSubAdapter, e inspectorSuite, err := g.rpcInspectorSuiteFactory( ctx, - g.logger, - g.sporkId, - g.rpcInspectorConfig, + g.logger, g.sporkId, &g.gossipSubCfg.RpcInspector, g.metricsCfg.Metrics, g.metricsCfg.HeroCacheFactory, g.networkType, @@ -321,7 +309,7 @@ func (g *Builder) Build(ctx irrecoverable.SignalerContext) (p2p.PubSubAdapter, e var scoreOpt *scoring.ScoreOption var scoreTracer p2p.PeerScoreTracer - if g.gossipSubPeerScoring { + if g.gossipSubCfg.PeerScoringEnabled { // wires the gossipsub score option to the subscription provider. subscriptionProvider, err := scoring.NewSubscriptionProvider(&scoring.SubscriptionProviderConfig{ Logger: g.logger, @@ -332,7 +320,7 @@ func (g *Builder) Build(ctx irrecoverable.SignalerContext) (p2p.PubSubAdapter, e return gossipSub }, IdProvider: g.idProvider, - Params: g.subscriptionProviderParam, + Params: &g.gossipSubCfg.SubscriptionProvider, HeroCacheMetricsFactory: g.metricsCfg.HeroCacheFactory, }) if err != nil { diff --git a/network/p2p/p2pbuilder/libp2pNodeBuilder.go b/network/p2p/p2pbuilder/libp2pNodeBuilder.go index dc2a5bf1d4d..feef72f2ad2 100644 --- a/network/p2p/p2pbuilder/libp2pNodeBuilder.go +++ b/network/p2p/p2pbuilder/libp2pNodeBuilder.go @@ -5,7 +5,6 @@ import ( "errors" "fmt" "net" - "time" "github.com/libp2p/go-libp2p" pubsub "github.com/libp2p/go-libp2p-pubsub" @@ -39,7 +38,6 @@ import ( "github.com/onflow/flow-go/network/p2p/p2plogging" "github.com/onflow/flow-go/network/p2p/p2pnode" "github.com/onflow/flow-go/network/p2p/subscription" - "github.com/onflow/flow-go/network/p2p/tracer" "github.com/onflow/flow-go/network/p2p/unicast" unicastcache "github.com/onflow/flow-go/network/p2p/unicast/cache" "github.com/onflow/flow-go/network/p2p/unicast/protocols" @@ -77,7 +75,7 @@ type LibP2PNodeBuilder struct { } func NewNodeBuilder( - logger zerolog.Logger, + logger zerolog.Logger, gossipSubCfg *p2pconf.GossipSubParameters, metricsConfig *p2pconfig.MetricsConfig, networkingType flownet.NetworkingType, address string, @@ -85,11 +83,8 @@ func NewNodeBuilder( sporkId flow.Identifier, idProvider module.IdentityProvider, rCfg *p2pconf.ResourceManagerConfig, - rpcInspectorCfg *p2pconf.RpcInspectorParameters, peerManagerConfig *p2pconfig.PeerManagerConfig, - subscriptionProviderParam *p2pconf.SubscriptionProviderParameters, disallowListCacheCfg *p2p.DisallowListCacheConfig, - rpcTracker p2p.RpcControlTracking, unicastConfig *p2pconfig.UnicastConfig, ) *LibP2PNodeBuilder { return &LibP2PNodeBuilder{ @@ -103,12 +98,9 @@ func NewNodeBuilder( disallowListCacheCfg: disallowListCacheCfg, networkingType: networkingType, gossipSubBuilder: gossipsubbuilder.NewGossipSubBuilder(logger, - metricsConfig, + metricsConfig, gossipSubCfg, networkingType, - sporkId, - idProvider, - rpcInspectorCfg, subscriptionProviderParam, - rpcTracker), + sporkId, idProvider), peerManagerConfig: peerManagerConfig, unicastConfig: unicastConfig, } @@ -172,22 +164,11 @@ func (builder *LibP2PNodeBuilder) EnableGossipSubScoringWithOverride(config *p2p return builder } -func (builder *LibP2PNodeBuilder) SetGossipSubTracer(tracer p2p.PubSubTracer) p2p.NodeBuilder { - builder.gossipSubBuilder.SetGossipSubTracer(tracer) - builder.gossipSubTracer = tracer - return builder -} - func (builder *LibP2PNodeBuilder) SetCreateNode(f p2p.CreateNodeFunc) p2p.NodeBuilder { builder.createNode = f return builder } -func (builder *LibP2PNodeBuilder) SetGossipSubScoreTracerInterval(interval time.Duration) p2p.NodeBuilder { - builder.gossipSubBuilder.SetGossipSubScoreTracerInterval(interval) - return builder -} - func (builder *LibP2PNodeBuilder) OverrideDefaultRpcInspectorSuiteFactory(factory p2p.GossipSubRpcInspectorSuiteFactoryFunc) p2p.NodeBuilder { builder.gossipSubBuilder.OverrideDefaultRpcInspectorSuiteFactory(factory) return builder @@ -447,30 +428,15 @@ func DefaultNodeBuilder( connection.WithOnInterceptPeerDialFilters(append(peerFilters, connGaterCfg.InterceptPeerDialFilters...)), connection.WithOnInterceptSecuredFilters(append(peerFilters, connGaterCfg.InterceptSecuredFilters...))) - meshTracerCfg := &tracer.GossipSubMeshTracerConfig{ - Logger: logger, - Metrics: metricsCfg.Metrics, - IDProvider: idProvider, - LoggerInterval: gossipCfg.LocalMeshLogInterval, - RpcSentTrackerCacheSize: gossipCfg.RPCSentTrackerCacheSize, - RpcSentTrackerWorkerQueueCacheSize: gossipCfg.RPCSentTrackerQueueCacheSize, - RpcSentTrackerNumOfWorkers: gossipCfg.RpcSentTrackerNumOfWorkers, - HeroCacheMetricsFactory: metricsCfg.HeroCacheFactory, - NetworkingType: flownet.PrivateNetwork, - } - meshTracer := tracer.NewGossipSubMeshTracer(meshTracerCfg) - - builder := NewNodeBuilder(logger, + builder := NewNodeBuilder(logger, gossipCfg, metricsCfg, networkingType, address, flowKey, sporkId, idProvider, - rCfg, - rpcInspectorCfg, peerManagerCfg, &gossipCfg.SubscriptionProvider, + rCfg, peerManagerCfg, disallowListCacheCfg, - meshTracer, uniCfg). SetBasicResolver(resolver). SetConnectionManager(connManager). @@ -482,9 +448,6 @@ func DefaultNodeBuilder( builder.EnableGossipSubScoringWithOverride(p2p.PeerScoringConfigNoOverride) } - builder.SetGossipSubTracer(meshTracer) - builder.SetGossipSubScoreTracerInterval(gossipCfg.ScoreTracerInterval) - if role != "ghost" { r, err := flow.ParseRole(role) if err != nil { From 7ce586ff3e5f3129e8ddfbca0c2232ea8a9bb834 Mon Sep 17 00:00:00 2001 From: Yahya Hassanzadeh Date: Tue, 21 Nov 2023 10:51:31 -0800 Subject: [PATCH 19/67] wip lint fix --- network/internal/p2pfixtures/fixtures.go | 25 +++---------------- .../p2p/cache/gossipsub_spam_records_test.go | 24 +++++++++--------- 2 files changed, 16 insertions(+), 33 deletions(-) diff --git a/network/internal/p2pfixtures/fixtures.go b/network/internal/p2pfixtures/fixtures.go index a26320c177e..b2df52dbfd8 100644 --- a/network/internal/p2pfixtures/fixtures.go +++ b/network/internal/p2pfixtures/fixtures.go @@ -33,7 +33,6 @@ import ( p2pdht "github.com/onflow/flow-go/network/p2p/dht" "github.com/onflow/flow-go/network/p2p/p2pbuilder" p2pconfig "github.com/onflow/flow-go/network/p2p/p2pbuilder/config" - "github.com/onflow/flow-go/network/p2p/tracer" "github.com/onflow/flow-go/network/p2p/unicast/protocols" "github.com/onflow/flow-go/network/p2p/utils" "github.com/onflow/flow-go/utils/unittest" @@ -100,20 +99,9 @@ func CreateNode(t *testing.T, networkKey crypto.PrivateKey, sporkID flow.Identif defaultFlowConfig, err := config.DefaultConfig() require.NoError(t, err) - meshTracerCfg := &tracer.GossipSubMeshTracerConfig{ - Logger: logger, - Metrics: metrics.NewNoopCollector(), - IDProvider: idProvider, - LoggerInterval: defaultFlowConfig.NetworkConfig.GossipSubConfig.LocalMeshLogInterval, - RpcSentTrackerCacheSize: defaultFlowConfig.NetworkConfig.GossipSubConfig.RPCSentTrackerCacheSize, - RpcSentTrackerWorkerQueueCacheSize: defaultFlowConfig.NetworkConfig.GossipSubConfig.RPCSentTrackerQueueCacheSize, - RpcSentTrackerNumOfWorkers: defaultFlowConfig.NetworkConfig.GossipSubConfig.RpcSentTrackerNumOfWorkers, - HeroCacheMetricsFactory: metrics.NewNoopHeroCacheMetricsFactory(), - NetworkingType: flownet.PublicNetwork, - } - meshTracer := tracer.NewGossipSubMeshTracer(meshTracerCfg) - - builder := p2pbuilder.NewNodeBuilder(logger, + builder := p2pbuilder.NewNodeBuilder( + logger, + &defaultFlowConfig.NetworkConfig.GossipSub, &p2pconfig.MetricsConfig{ HeroCacheFactory: metrics.NewNoopHeroCacheMetricsFactory(), Metrics: metrics.NewNoopCollector(), @@ -124,23 +112,18 @@ func CreateNode(t *testing.T, networkKey crypto.PrivateKey, sporkID flow.Identif sporkID, idProvider, &defaultFlowConfig.NetworkConfig.ResourceManager, - &defaultFlowConfig.NetworkConfig.GossipSubRPCInspectorsConfig, p2pconfig.PeerManagerDisableConfig(), - &defaultFlowConfig.NetworkConfig.GossipSubConfig.SubscriptionProvider, &p2p.DisallowListCacheConfig{ MaxSize: uint32(1000), Metrics: metrics.NewNoopCollector(), }, - meshTracer, &p2pconfig.UnicastConfig{ UnicastConfig: defaultFlowConfig.NetworkConfig.UnicastConfig, }). SetRoutingSystem(func(c context.Context, h host.Host) (routing.Routing, error) { return p2pdht.NewDHT(c, h, protocols.FlowDHTProtocolID(sporkID), zerolog.Nop(), metrics.NewNoopCollector()) }). - SetResourceManager(&network.NullResourceManager{}). - SetGossipSubTracer(meshTracer). - SetGossipSubScoreTracerInterval(defaultFlowConfig.NetworkConfig.GossipSubConfig.ScoreTracerInterval) + SetResourceManager(&network.NullResourceManager{}) for _, opt := range opts { opt(builder) diff --git a/network/p2p/cache/gossipsub_spam_records_test.go b/network/p2p/cache/gossipsub_spam_records_test.go index 166776b93ba..5ddcf2a3484 100644 --- a/network/p2p/cache/gossipsub_spam_records_test.go +++ b/network/p2p/cache/gossipsub_spam_records_test.go @@ -21,7 +21,7 @@ import ( // adding a new record to the cache. func TestGossipSubSpamRecordCache_Add(t *testing.T) { // create a new instance of GossipSubSpamRecordCache. - cache := netcache.NewGossipSubSpamRecordCache(100, unittest.Logger(), metrics.NewNoopCollector()) + cache := netcache.NewGossipSubSpamRecordCache(100, unittest.Logger(), metrics.NewNoopHeroCacheMetricsFactory()) // tests adding a new record to the cache. require.True(t, cache.Add("peer0", p2p.GossipSubSpamRecord{ @@ -70,7 +70,7 @@ func TestGossipSubSpamRecordCache_Add(t *testing.T) { // It updates the cache with a number of records concurrently and then checks if the cache // can retrieve all records. func TestGossipSubSpamRecordCache_Concurrent_Add(t *testing.T) { - cache := netcache.NewGossipSubSpamRecordCache(200, unittest.Logger(), metrics.NewNoopCollector()) + cache := netcache.NewGossipSubSpamRecordCache(200, unittest.Logger(), metrics.NewNoopHeroCacheMetricsFactory()) // defines the number of records to update. numRecords := 100 @@ -113,7 +113,7 @@ func TestGossipSubSpamRecordCache_Concurrent_Add(t *testing.T) { // TestGossipSubSpamRecordCache_Update tests the Update method of the GossipSubSpamRecordCache. It tests if the cache can update // the penalty of an existing record and fail to update the penalty of a non-existing record. func TestGossipSubSpamRecordCache_Update(t *testing.T) { - cache := netcache.NewGossipSubSpamRecordCache(200, unittest.Logger(), metrics.NewNoopCollector()) + cache := netcache.NewGossipSubSpamRecordCache(200, unittest.Logger(), metrics.NewNoopHeroCacheMetricsFactory()) peerID := "peer1" @@ -141,7 +141,7 @@ func TestGossipSubSpamRecordCache_Update(t *testing.T) { // TestGossipSubSpamRecordCache_Concurrent_Update tests if the cache can be updated concurrently. It updates the cache // with a number of records concurrently and then checks if the cache can retrieve all records. func TestGossipSubSpamRecordCache_Concurrent_Update(t *testing.T) { - cache := netcache.NewGossipSubSpamRecordCache(200, unittest.Logger(), metrics.NewNoopCollector()) + cache := netcache.NewGossipSubSpamRecordCache(200, unittest.Logger(), metrics.NewNoopHeroCacheMetricsFactory()) // defines the number of records to update. numRecords := 100 @@ -199,7 +199,7 @@ func TestGossipSubSpamRecordCache_Concurrent_Update(t *testing.T) { func TestGossipSubSpamRecordCache_Update_With_Preprocess(t *testing.T) { cache := netcache.NewGossipSubSpamRecordCache(200, unittest.Logger(), - metrics.NewNoopCollector(), + metrics.NewNoopHeroCacheMetricsFactory(), func(record p2p.GossipSubSpamRecord, lastUpdated time.Time) (p2p.GossipSubSpamRecord, error) { record.Penalty += 1.5 return record, nil @@ -232,7 +232,7 @@ func TestGossipSubSpamRecordCache_Update_Preprocess_Error(t *testing.T) { secondPreprocessorCalled := false cache := netcache.NewGossipSubSpamRecordCache(200, unittest.Logger(), - metrics.NewNoopCollector(), + metrics.NewNoopHeroCacheMetricsFactory(), // the first preprocessor function does not return an error. func(record p2p.GossipSubSpamRecord, lastUpdated time.Time) (p2p.GossipSubSpamRecord, error) { return record, nil @@ -277,7 +277,7 @@ func TestGossipSubSpamRecordCache_Update_Preprocess_Error(t *testing.T) { // This is a desired behavior that is guaranteed by the underlying HeroCache library. // In other words, we don't desire the records to be externally mutable after they are added to the cache (unless by a subsequent call to Update). func TestGossipSubSpamRecordCache_ByValue(t *testing.T) { - cache := netcache.NewGossipSubSpamRecordCache(200, unittest.Logger(), metrics.NewNoopCollector()) + cache := netcache.NewGossipSubSpamRecordCache(200, unittest.Logger(), metrics.NewNoopHeroCacheMetricsFactory()) peerID := "peer1" added := cache.Add(peer.ID(peerID), p2p.GossipSubSpamRecord{ @@ -308,7 +308,7 @@ func TestGossipSubSpamRecordCache_ByValue(t *testing.T) { // TestGossipSubSpamRecordCache_Get_With_Preprocessors tests if the cache applies the preprocessors to the records // before returning them. func TestGossipSubSpamRecordCache_Get_With_Preprocessors(t *testing.T) { - cache := netcache.NewGossipSubSpamRecordCache(10, unittest.Logger(), metrics.NewNoopCollector(), + cache := netcache.NewGossipSubSpamRecordCache(10, unittest.Logger(), metrics.NewNoopHeroCacheMetricsFactory(), // first preprocessor: adds 1 to the penalty. func(record p2p.GossipSubSpamRecord, lastUpdated time.Time) (p2p.GossipSubSpamRecord, error) { record.Penalty++ @@ -348,7 +348,7 @@ func TestGossipSubSpamRecordCache_Get_Preprocessor_Error(t *testing.T) { secondPreprocessorCalledCount := 0 thirdPreprocessorCalledCount := 0 - cache := netcache.NewGossipSubSpamRecordCache(10, unittest.Logger(), metrics.NewNoopCollector(), + cache := netcache.NewGossipSubSpamRecordCache(10, unittest.Logger(), metrics.NewNoopHeroCacheMetricsFactory(), // first preprocessor: adds 1 to the penalty. func(record p2p.GossipSubSpamRecord, lastUpdated time.Time) (p2p.GossipSubSpamRecord, error) { record.Penalty++ @@ -402,7 +402,7 @@ func TestGossipSubSpamRecordCache_Get_Preprocessor_Error(t *testing.T) { // TestGossipSubSpamRecordCache_Get_Without_Preprocessors tests when no preprocessors are provided to the cache constructor // that the cache returns the original record without any modifications. func TestGossipSubSpamRecordCache_Get_Without_Preprocessors(t *testing.T) { - cache := netcache.NewGossipSubSpamRecordCache(10, unittest.Logger(), metrics.NewNoopCollector()) + cache := netcache.NewGossipSubSpamRecordCache(10, unittest.Logger(), metrics.NewNoopHeroCacheMetricsFactory()) record := p2p.GossipSubSpamRecord{ Decay: 0.5, @@ -423,7 +423,7 @@ func TestGossipSubSpamRecordCache_Get_Without_Preprocessors(t *testing.T) { // This test evaluates that the cache de-duplicates the records based on their peer id and not content, and hence // each peer id can only be added once to the cache. func TestGossipSubSpamRecordCache_Duplicate_Add_Sequential(t *testing.T) { - cache := netcache.NewGossipSubSpamRecordCache(10, unittest.Logger(), metrics.NewNoopCollector()) + cache := netcache.NewGossipSubSpamRecordCache(10, unittest.Logger(), metrics.NewNoopHeroCacheMetricsFactory()) record := p2p.GossipSubSpamRecord{ Decay: 0.5, @@ -445,7 +445,7 @@ func TestGossipSubSpamRecordCache_Duplicate_Add_Sequential(t *testing.T) { // TestGossipSubSpamRecordCache_Duplicate_Add_Concurrent tests if the cache returns false when a duplicate record is added to the cache. // Test is the concurrent version of TestAppScoreCache_DuplicateAdd_Sequential. func TestGossipSubSpamRecordCache_Duplicate_Add_Concurrent(t *testing.T) { - cache := netcache.NewGossipSubSpamRecordCache(10, unittest.Logger(), metrics.NewNoopCollector()) + cache := netcache.NewGossipSubSpamRecordCache(10, unittest.Logger(), metrics.NewNoopHeroCacheMetricsFactory()) successAdd := atomic.Int32{} successAdd.Store(0) From f390482fb03c9d09d2ee169c3c9b9aeab2726f9f Mon Sep 17 00:00:00 2001 From: Yahya Hassanzadeh Date: Tue, 21 Nov 2023 11:00:04 -0800 Subject: [PATCH 20/67] wip lint fix --- network/p2p/test/fixtures.go | 49 ++++++------------------------------ 1 file changed, 7 insertions(+), 42 deletions(-) diff --git a/network/p2p/test/fixtures.go b/network/p2p/test/fixtures.go index 2acc998207c..ecb7de77f70 100644 --- a/network/p2p/test/fixtures.go +++ b/network/p2p/test/fixtures.go @@ -38,7 +38,6 @@ import ( mockp2p "github.com/onflow/flow-go/network/p2p/mock" "github.com/onflow/flow-go/network/p2p/p2pbuilder" p2pconfig "github.com/onflow/flow-go/network/p2p/p2pbuilder/config" - "github.com/onflow/flow-go/network/p2p/tracer" "github.com/onflow/flow-go/network/p2p/unicast/protocols" "github.com/onflow/flow-go/network/p2p/utils" validator "github.com/onflow/flow-go/network/validator/pubsub" @@ -91,17 +90,6 @@ func NodeFixture(t *testing.T, }) require.NotNil(t, connectionGater) - meshTracerCfg := &tracer.GossipSubMeshTracerConfig{ - Logger: unittest.Logger(), - Metrics: metrics.NewNoopCollector(), - IDProvider: idProvider, - LoggerInterval: time.Second, - HeroCacheMetricsFactory: metrics.NewNoopHeroCacheMetricsFactory(), - RpcSentTrackerCacheSize: defaultFlowConfig.NetworkConfig.GossipSubConfig.RPCSentTrackerCacheSize, - RpcSentTrackerWorkerQueueCacheSize: defaultFlowConfig.NetworkConfig.GossipSubConfig.RPCSentTrackerQueueCacheSize, - RpcSentTrackerNumOfWorkers: defaultFlowConfig.NetworkConfig.GossipSubConfig.RpcSentTrackerNumOfWorkers, - } - parameters := &NodeFixtureParameters{ NetworkingType: flownet.PrivateNetwork, HandlerFunc: func(network.Stream) {}, @@ -115,12 +103,10 @@ func NodeFixture(t *testing.T, HeroCacheFactory: metrics.NewNoopHeroCacheMetricsFactory(), Metrics: metrics.NewNoopCollector(), }, - ResourceManager: &network.NullResourceManager{}, - GossipSubPeerScoreTracerInterval: 0, // disabled by default - ConnGater: connectionGater, - PeerManagerConfig: PeerManagerConfigFixture(), // disabled by default - FlowConfig: defaultFlowConfig, - PubSubTracer: tracer.NewGossipSubMeshTracer(meshTracerCfg), + ResourceManager: &network.NullResourceManager{}, + ConnGater: connectionGater, + PeerManagerConfig: PeerManagerConfigFixture(), // disabled by default + FlowConfig: defaultFlowConfig, UnicastConfig: &p2pconfig.UnicastConfig{ UnicastConfig: defaultFlowConfig.NetworkConfig.UnicastConfig, }, @@ -139,7 +125,9 @@ func NodeFixture(t *testing.T, connManager, err := connection.NewConnManager(logger, parameters.MetricsCfg.Metrics, ¶meters.FlowConfig.NetworkConfig.ConnectionManagerConfig) require.NoError(t, err) - builder := p2pbuilder.NewNodeBuilder(logger, + builder := p2pbuilder.NewNodeBuilder( + logger, + ¶meters.FlowConfig.NetworkConfig.GossipSub, parameters.MetricsCfg, parameters.NetworkingType, parameters.Address, @@ -147,14 +135,11 @@ func NodeFixture(t *testing.T, sporkID, parameters.IdProvider, ¶meters.FlowConfig.NetworkConfig.ResourceManager, - ¶meters.FlowConfig.NetworkConfig.GossipSubRPCInspectorsConfig, parameters.PeerManagerConfig, - ¶meters.FlowConfig.NetworkConfig.GossipSubConfig.SubscriptionProvider, &p2p.DisallowListCacheConfig{ MaxSize: uint32(1000), Metrics: metrics.NewNoopCollector(), }, - parameters.PubSubTracer, parameters.UnicastConfig). SetConnectionManager(connManager). SetCreateNode(p2pbuilder.DefaultCreateNodeFunc). @@ -202,12 +187,6 @@ func NodeFixture(t *testing.T, builder.SetConnectionManager(parameters.ConnManager) } - if parameters.PubSubTracer != nil { - builder.SetGossipSubTracer(parameters.PubSubTracer) - } - - builder.SetGossipSubScoreTracerInterval(parameters.GossipSubPeerScoreTracerInterval) - n, err := builder.Build() require.NoError(t, err) @@ -251,8 +230,6 @@ type NodeFixtureParameters struct { GossipSubConfig p2p.GossipSubAdapterConfigFunc MetricsCfg *p2pconfig.MetricsConfig ResourceManager network.ResourceManager - PubSubTracer p2p.PubSubTracer - GossipSubPeerScoreTracerInterval time.Duration // intervals at which the peer score is updated and logged. GossipSubRpcInspectorSuiteFactory p2p.GossipSubRpcInspectorSuiteFactoryFunc FlowConfig *config.FlowConfig } @@ -299,12 +276,6 @@ func EnablePeerScoringWithOverride(override *p2p.PeerScoringConfigOverride) Node } } -func WithGossipSubTracer(tracer p2p.PubSubTracer) NodeFixtureParameterOption { - return func(p *NodeFixtureParameters) { - p.PubSubTracer = tracer - } -} - func WithDefaultStreamHandler(handler network.StreamHandler) NodeFixtureParameterOption { return func(p *NodeFixtureParameters) { p.HandlerFunc = handler @@ -378,12 +349,6 @@ func WithMetricsCollector(metrics module.NetworkMetrics) NodeFixtureParameterOpt } } -func WithPeerScoreTracerInterval(interval time.Duration) NodeFixtureParameterOption { - return func(p *NodeFixtureParameters) { - p.GossipSubPeerScoreTracerInterval = interval - } -} - // WithDefaultResourceManager sets the resource manager to nil, which will cause the node to use the default resource manager. // Otherwise, it uses the resource manager provided by the test (the infinite resource manager). func WithDefaultResourceManager() NodeFixtureParameterOption { From 18774dbb2baf5d2b5dca26597574f6101ddc3f39 Mon Sep 17 00:00:00 2001 From: Yahya Hassanzadeh Date: Tue, 21 Nov 2023 11:04:28 -0800 Subject: [PATCH 21/67] wip lint fix --- cmd/scaffold.go | 3 +-- insecure/corruptlibp2p/libp2p_node_factory.go | 3 +-- network/p2p/p2pbuilder/libp2pNodeBuilder.go | 1 - network/p2p/tracer/internal/rpc_sent_tracker_test.go | 4 ++-- 4 files changed, 4 insertions(+), 7 deletions(-) diff --git a/cmd/scaffold.go b/cmd/scaffold.go index c9749dc691a..6b85e4cfba4 100644 --- a/cmd/scaffold.go +++ b/cmd/scaffold.go @@ -394,8 +394,7 @@ func (fnb *FlowNodeBuilder) EnqueueNetworkInit() { fnb.BaseConfig.NodeRole, connGaterCfg, peerManagerCfg, - &fnb.FlowConfig.NetworkConfig.GossipSubConfig, - &fnb.FlowConfig.NetworkConfig.GossipSubRPCInspectorsConfig, + &fnb.FlowConfig.NetworkConfig.GossipSub, &fnb.FlowConfig.NetworkConfig.ResourceManager, uniCfg, &fnb.FlowConfig.NetworkConfig.ConnectionManagerConfig, diff --git a/insecure/corruptlibp2p/libp2p_node_factory.go b/insecure/corruptlibp2p/libp2p_node_factory.go index 69c7b0cfba9..ec6392c882a 100644 --- a/insecure/corruptlibp2p/libp2p_node_factory.go +++ b/insecure/corruptlibp2p/libp2p_node_factory.go @@ -90,8 +90,7 @@ func InitCorruptLibp2pNode( role, connGaterCfg, peerManagerCfg, - &netConfig.GossipSubConfig, - &netConfig.GossipSubRPCInspectorsConfig, + &netConfig.GossipSub, &netConfig.ResourceManager, uniCfg, &netConfig.ConnectionManagerConfig, diff --git a/network/p2p/p2pbuilder/libp2pNodeBuilder.go b/network/p2p/p2pbuilder/libp2pNodeBuilder.go index feef72f2ad2..5a83c256c59 100644 --- a/network/p2p/p2pbuilder/libp2pNodeBuilder.go +++ b/network/p2p/p2pbuilder/libp2pNodeBuilder.go @@ -405,7 +405,6 @@ func DefaultNodeBuilder( connGaterCfg *p2pconfig.ConnectionGaterConfig, peerManagerCfg *p2pconfig.PeerManagerConfig, gossipCfg *p2pconf.GossipSubParameters, - rpcInspectorCfg *p2pconf.RpcInspectorParameters, rCfg *p2pconf.ResourceManagerConfig, uniCfg *p2pconfig.UnicastConfig, connMgrConfig *netconf.ConnectionManagerConfig, diff --git a/network/p2p/tracer/internal/rpc_sent_tracker_test.go b/network/p2p/tracer/internal/rpc_sent_tracker_test.go index 208bbc42940..938a998cf46 100644 --- a/network/p2p/tracer/internal/rpc_sent_tracker_test.go +++ b/network/p2p/tracer/internal/rpc_sent_tracker_test.go @@ -228,10 +228,10 @@ func mockTracker(t *testing.T, lastHighestIhavesSentResetInterval time.Duration) require.NoError(t, err) tracker := NewRPCSentTracker(&RPCSentTrackerConfig{ Logger: zerolog.Nop(), - RPCSentCacheSize: cfg.NetworkConfig.GossipSubConfig.RPCSentTrackerCacheSize, + RPCSentCacheSize: cfg.NetworkConfig.GossipSub.RpcTracer.RPCSentTrackerCacheSize, RPCSentCacheCollector: metrics.NewNoopCollector(), WorkerQueueCacheCollector: metrics.NewNoopCollector(), - WorkerQueueCacheSize: cfg.NetworkConfig.GossipSubConfig.RPCSentTrackerQueueCacheSize, + WorkerQueueCacheSize: cfg.NetworkConfig.GossipSub.RpcTracer.RPCSentTrackerQueueCacheSize, NumOfWorkers: 1, LastHighestIhavesSentResetInterval: lastHighestIhavesSentResetInterval, }) From 07bc1a17d0110f0c665d841a18ed06820d0f0cf2 Mon Sep 17 00:00:00 2001 From: Yahya Hassanzadeh Date: Tue, 21 Nov 2023 11:15:14 -0800 Subject: [PATCH 22/67] lint fix wip --- network/p2p/libp2pNode.go | 9 ++++++- network/p2p/p2pnode/gossipSubAdapter.go | 12 ++++++++++ network/p2p/p2pnode/libp2pNode.go | 9 +++++++ network/p2p/pubsub.go | 14 +++++++++++ network/p2p/scoring/app_score_test.go | 24 +++++-------------- network/p2p/tracer/gossipSubMeshTracer.go | 13 ++++++---- .../p2p/tracer/gossipSubMeshTracer_test.go | 8 +++---- 7 files changed, 62 insertions(+), 27 deletions(-) diff --git a/network/p2p/libp2pNode.go b/network/p2p/libp2pNode.go index 17295341bde..435f4ed5584 100644 --- a/network/p2p/libp2pNode.go +++ b/network/p2p/libp2pNode.go @@ -114,13 +114,20 @@ type UnicastManagement interface { type PubSub interface { // Subscribe subscribes the node to the given topic and returns the subscription Subscribe(topic channels.Topic, topicValidator TopicValidatorFunc) (Subscription, error) - // UnSubscribe cancels the subscriber and closes the topic. + // Unsubscribe cancels the subscriber and closes the topic. Unsubscribe(topic channels.Topic) error // Publish publishes the given payload on the topic. Publish(ctx context.Context, messageScope flownet.OutgoingMessageScope) error // SetPubSub sets the node's pubsub implementation. // SetPubSub may be called at most once. SetPubSub(ps PubSubAdapter) + + // GetLocalMeshPeers returns the list of peers in the local mesh for the given topic. + // Args: + // - topic: the topic. + // Returns: + // - []peer.ID: the list of peers in the local mesh for the given topic. + GetLocalMeshPeers(topic channels.Topic) []peer.ID } // LibP2PNode represents a Flow libp2p node. It provides the network layer with the necessary interface to diff --git a/network/p2p/p2pnode/gossipSubAdapter.go b/network/p2p/p2pnode/gossipSubAdapter.go index f2d1296b588..d1acf5af376 100644 --- a/network/p2p/p2pnode/gossipSubAdapter.go +++ b/network/p2p/p2pnode/gossipSubAdapter.go @@ -12,6 +12,7 @@ import ( "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module/component" "github.com/onflow/flow-go/module/irrecoverable" + "github.com/onflow/flow-go/network/channels" "github.com/onflow/flow-go/network/p2p" "github.com/onflow/flow-go/network/p2p/utils" "github.com/onflow/flow-go/utils/logging" @@ -31,6 +32,7 @@ type GossipSubAdapter struct { topicScoreParamFunc func(topic *pubsub.Topic) *pubsub.TopicScoreParams logger zerolog.Logger peerScoreExposer p2p.PeerScoreExposer + localMeshTracer p2p.PubSubTracer // clusterChangeConsumer is a callback that is invoked when the set of active clusters of collection nodes changes. // This callback is implemented by the rpc inspector suite of the GossipSubAdapter, and consumes the cluster changes // to update the rpc inspector state of the recent topics (i.e., channels). @@ -107,6 +109,7 @@ func NewGossipSubAdapter(ctx context.Context, <-tracer.Done() a.logger.Info().Msg("pubsub tracer stopped") }) + a.localMeshTracer = tracer } if inspectorSuite := gossipSubConfig.InspectorSuiteComponent(); inspectorSuite != nil { @@ -211,6 +214,15 @@ func (g *GossipSubAdapter) ListPeers(topic string) []peer.ID { return g.gossipSub.ListPeers(topic) } +// GetLocalMeshPeers returns the list of peers in the local mesh for the given topic. +// Args: +// - topic: the topic. +// Returns: +// - []peer.ID: the list of peers in the local mesh for the given topic. +func (g *GossipSubAdapter) GetLocalMeshPeers(topic channels.Topic) []peer.ID { + return g.localMeshTracer.GetLocalMeshPeers(topic) +} + // PeerScoreExposer returns the peer score exposer for the gossipsub adapter. The exposer is a read-only interface // for querying peer scores and returns the local scoring table of the underlying gossipsub node. // The exposer is only available if the gossipsub adapter was configured with a score tracer. diff --git a/network/p2p/p2pnode/libp2pNode.go b/network/p2p/p2pnode/libp2pNode.go index e22d3dbaa44..e8b10939b7f 100644 --- a/network/p2p/p2pnode/libp2pNode.go +++ b/network/p2p/p2pnode/libp2pNode.go @@ -558,6 +558,15 @@ func (n *Node) SetPubSub(ps p2p.PubSubAdapter) { n.pubSub = ps } +// GetLocalMeshPeers returns the list of peers in the local mesh for the given topic. +// Args: +// - topic: the topic. +// Returns: +// - []peer.ID: the list of peers in the local mesh for the given topic. +func (n *Node) GetLocalMeshPeers(topic channels.Topic) []peer.ID { + return n.pubSub.GetLocalMeshPeers(topic) +} + // SetComponentManager sets the component manager for the node. // SetComponentManager may be called at most once. func (n *Node) SetComponentManager(cm *component.ComponentManager) { diff --git a/network/p2p/pubsub.go b/network/p2p/pubsub.go index d0ceb33fe8c..02b4546553d 100644 --- a/network/p2p/pubsub.go +++ b/network/p2p/pubsub.go @@ -12,6 +12,7 @@ import ( "github.com/onflow/flow-go/engine/collection" "github.com/onflow/flow-go/module/component" + "github.com/onflow/flow-go/network/channels" ) type ValidationResult int @@ -54,6 +55,13 @@ type PubSubAdapter interface { // subscribed peers for topics A and B, and querying for topic C will return an empty list. ListPeers(topic string) []peer.ID + // GetLocalMeshPeers returns the list of peers in the local mesh for the given topic. + // Args: + // - topic: the topic. + // Returns: + // - []peer.ID: the list of peers in the local mesh for the given topic. + GetLocalMeshPeers(topic channels.Topic) []peer.ID + // PeerScoreExposer returns the peer score exposer for the gossipsub adapter. The exposer is a read-only interface // for querying peer scores and returns the local scoring table of the underlying gossipsub node. // The exposer is only available if the gossipsub adapter was configured with a score tracer. @@ -172,6 +180,12 @@ type PubSubTracer interface { component.Component pubsub.RawTracer RpcControlTracking + // GetLocalMeshPeers returns the list of peers in the mesh for the given topic. + // Args: + // - topic: the topic. + // Returns: + // - []peer.ID: the list of peers in the mesh for the given topic. + GetLocalMeshPeers(topic channels.Topic) []peer.ID } // RpcControlTracking is the abstraction of the underlying libp2p control message tracker used to track message ids advertised by the iHave control messages. diff --git a/network/p2p/scoring/app_score_test.go b/network/p2p/scoring/app_score_test.go index 39a0a405e8e..787010aac2b 100644 --- a/network/p2p/scoring/app_score_test.go +++ b/network/p2p/scoring/app_score_test.go @@ -13,7 +13,6 @@ import ( "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module/id" "github.com/onflow/flow-go/module/irrecoverable" - "github.com/onflow/flow-go/module/metrics" "github.com/onflow/flow-go/module/mock" "github.com/onflow/flow-go/network/channels" "github.com/onflow/flow-go/network/internal/p2pfixtures" @@ -21,7 +20,6 @@ import ( "github.com/onflow/flow-go/network/p2p" "github.com/onflow/flow-go/network/p2p/scoring" p2ptest "github.com/onflow/flow-go/network/p2p/test" - "github.com/onflow/flow-go/network/p2p/tracer" flowpubsub "github.com/onflow/flow-go/network/validator/pubsub" "github.com/onflow/flow-go/utils/unittest" ) @@ -137,21 +135,11 @@ func TestFullGossipSubConnectivityAmongHonestNodesWithMaliciousMajority(t *testi defaultConfig, err := config.DefaultConfig() require.NoError(t, err) - meshTracerCfg := &tracer.GossipSubMeshTracerConfig{ - Logger: unittest.Logger(), - Metrics: metrics.NewNoopCollector(), - IDProvider: idProvider, - LoggerInterval: time.Second, - HeroCacheMetricsFactory: metrics.NewNoopHeroCacheMetricsFactory(), - RpcSentTrackerCacheSize: defaultConfig.NetworkConfig.GossipSubConfig.RPCSentTrackerCacheSize, - RpcSentTrackerWorkerQueueCacheSize: defaultConfig.NetworkConfig.GossipSubConfig.RPCSentTrackerQueueCacheSize, - RpcSentTrackerNumOfWorkers: defaultConfig.NetworkConfig.GossipSubConfig.RpcSentTrackerNumOfWorkers, - } + // override the default config to make the mesh tracer log more frequently + defaultConfig.NetworkConfig.GossipSub.RpcTracer.LocalMeshLogInterval = time.Second - con1NodeTracer := tracer.NewGossipSubMeshTracer(meshTracerCfg) // mesh tracer for con1 - con2NodeTracer := tracer.NewGossipSubMeshTracer(meshTracerCfg) // mesh tracer for con2 - con1Node, con1Id := p2ptest.NodeFixture(t, sporkId, t.Name(), idProvider, append(opts, p2ptest.WithGossipSubTracer(con1NodeTracer))...) - con2Node, con2Id := p2ptest.NodeFixture(t, sporkId, t.Name(), idProvider, append(opts, p2ptest.WithGossipSubTracer(con2NodeTracer))...) + con1Node, con1Id := p2ptest.NodeFixture(t, sporkId, t.Name(), idProvider, append(opts, p2ptest.OverrideFlowConfig(defaultConfig))...) + con2Node, con2Id := p2ptest.NodeFixture(t, sporkId, t.Name(), idProvider, append(opts, p2ptest.OverrideFlowConfig(defaultConfig))...) // create > 2 * 12 malicious access nodes // 12 is the maximum size of default GossipSub mesh. @@ -214,7 +202,7 @@ func TestFullGossipSubConnectivityAmongHonestNodesWithMaliciousMajority(t *testi for { select { case <-ticker.C: - con1BlockTopicPeers := con1NodeTracer.GetMeshPeers(blockTopic.String()) + con1BlockTopicPeers := con1Node.GetLocalMeshPeers(blockTopic) for _, p := range con1BlockTopicPeers { if p == con2Node.ID() { con2HasCon1 = true @@ -222,7 +210,7 @@ func TestFullGossipSubConnectivityAmongHonestNodesWithMaliciousMajority(t *testi } } - con2BlockTopicPeers := con2NodeTracer.GetMeshPeers(blockTopic.String()) + con2BlockTopicPeers := con2Node.GetLocalMeshPeers(blockTopic) for _, p := range con2BlockTopicPeers { if p == con1Node.ID() { con1HasCon2 = true diff --git a/network/p2p/tracer/gossipSubMeshTracer.go b/network/p2p/tracer/gossipSubMeshTracer.go index d7d602c59fe..65166cfbd44 100644 --- a/network/p2p/tracer/gossipSubMeshTracer.go +++ b/network/p2p/tracer/gossipSubMeshTracer.go @@ -15,6 +15,7 @@ import ( "github.com/onflow/flow-go/module/irrecoverable" "github.com/onflow/flow-go/module/metrics" "github.com/onflow/flow-go/network" + "github.com/onflow/flow-go/network/channels" "github.com/onflow/flow-go/network/p2p" "github.com/onflow/flow-go/network/p2p/p2plogging" "github.com/onflow/flow-go/network/p2p/tracer/internal" @@ -115,13 +116,17 @@ func NewGossipSubMeshTracer(config *GossipSubMeshTracerConfig) *GossipSubMeshTra return g } -// GetMeshPeers returns the local mesh peers for the given topic. -func (t *GossipSubMeshTracer) GetMeshPeers(topic string) []peer.ID { +// GetLocalMeshPeers returns the local mesh peers for the given topic. +// Args: +// - topic: the topic. +// Returns: +// - []peer.ID: the local mesh peers for the given topic. +func (t *GossipSubMeshTracer) GetLocalMeshPeers(topic channels.Topic) []peer.ID { t.topicMeshMu.RLock() defer t.topicMeshMu.RUnlock() - peers := make([]peer.ID, 0, len(t.topicMeshMap[topic])) - for p := range t.topicMeshMap[topic] { + peers := make([]peer.ID, 0, len(t.topicMeshMap[topic.String()])) + for p := range t.topicMeshMap[topic.String()] { peers = append(peers, p) } return peers diff --git a/network/p2p/tracer/gossipSubMeshTracer_test.go b/network/p2p/tracer/gossipSubMeshTracer_test.go index 842fa3dfe72..5438f033416 100644 --- a/network/p2p/tracer/gossipSubMeshTracer_test.go +++ b/network/p2p/tracer/gossipSubMeshTracer_test.go @@ -152,14 +152,14 @@ func TestGossipSubMeshTracer(t *testing.T) { // eventually, the meshTracer should have the other nodes in its mesh. assert.Eventually(t, func() bool { topic1MeshSize := 0 - for _, peer := range meshTracer.GetMeshPeers(topic1.String()) { + for _, peer := range meshTracer.GetLocalMeshPeers(topic1.String()) { if peer == otherNode1.ID() || peer == otherNode2.ID() { topic1MeshSize++ } } topic2MeshSize := 0 - for _, peer := range meshTracer.GetMeshPeers(topic2.String()) { + for _, peer := range meshTracer.GetLocalMeshPeers(topic2.String()) { if peer == otherNode1.ID() { topic2MeshSize++ } @@ -184,14 +184,14 @@ func TestGossipSubMeshTracer(t *testing.T) { assert.Eventually(t, func() bool { // eventually, the tracerNode should not have the other node in its mesh for topic1. - for _, peer := range meshTracer.GetMeshPeers(topic1.String()) { + for _, peer := range meshTracer.GetLocalMeshPeers(topic1.String()) { if peer == otherNode1.ID() || peer == otherNode2.ID() || peer == unknownNode.ID() { return false } } // but the tracerNode should still have the otherNode1 in its mesh for topic2. - for _, peer := range meshTracer.GetMeshPeers(topic2.String()) { + for _, peer := range meshTracer.GetLocalMeshPeers(topic2.String()) { if peer != otherNode1.ID() { return false } From 6e1449d4aa21d43e85e62f352765b31b95066dae Mon Sep 17 00:00:00 2001 From: Yahya Hassanzadeh Date: Tue, 21 Nov 2023 11:17:37 -0800 Subject: [PATCH 23/67] lint fix --- network/p2p/scoring/subscription_provider_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/network/p2p/scoring/subscription_provider_test.go b/network/p2p/scoring/subscription_provider_test.go index ba2180af6cf..0c731f1b508 100644 --- a/network/p2p/scoring/subscription_provider_test.go +++ b/network/p2p/scoring/subscription_provider_test.go @@ -31,14 +31,14 @@ func TestSubscriptionProvider_GetSubscribedTopics(t *testing.T) { idProvider := mock.NewIdentityProvider(t) // set a low update interval to speed up the test - cfg.NetworkConfig.SubscriptionProvider.SubscriptionUpdateInterval = 100 * time.Millisecond + cfg.NetworkConfig.GossipSub.SubscriptionProvider.UpdateInterval = 100 * time.Millisecond sp, err := scoring.NewSubscriptionProvider(&scoring.SubscriptionProviderConfig{ Logger: unittest.Logger(), TopicProviderOracle: func() p2p.TopicProvider { return tp }, - Params: &cfg.NetworkConfig.SubscriptionProvider, + Params: &cfg.NetworkConfig.GossipSub.SubscriptionProvider, HeroCacheMetricsFactory: metrics.NewNoopHeroCacheMetricsFactory(), IdProvider: idProvider, }) @@ -92,14 +92,14 @@ func TestSubscriptionProvider_GetSubscribedTopics_SkippingUnknownPeers(t *testin idProvider := mock.NewIdentityProvider(t) // set a low update interval to speed up the test - cfg.NetworkConfig.SubscriptionProvider.SubscriptionUpdateInterval = 100 * time.Millisecond + cfg.NetworkConfig.GossipSub.SubscriptionProvider.UpdateInterval = 100 * time.Millisecond sp, err := scoring.NewSubscriptionProvider(&scoring.SubscriptionProviderConfig{ Logger: unittest.Logger(), TopicProviderOracle: func() p2p.TopicProvider { return tp }, - Params: &cfg.NetworkConfig.SubscriptionProvider, + Params: &cfg.NetworkConfig.GossipSub.SubscriptionProvider, HeroCacheMetricsFactory: metrics.NewNoopHeroCacheMetricsFactory(), IdProvider: idProvider, }) From 4fa3e56c6986add5e67b10519eaf0c671af24cd7 Mon Sep 17 00:00:00 2001 From: Yahya Hassanzadeh Date: Tue, 21 Nov 2023 12:21:23 -0800 Subject: [PATCH 24/67] lint fix --- module/irrecoverable/unittest.go | 6 ++-- network/p2p/p2pbuilder/libp2pNodeBuilder.go | 2 +- network/p2p/scoring/registry.go | 6 ++-- network/p2p/scoring/score_option.go | 2 +- .../scoring/subscription_validator_test.go | 2 +- .../p2p/tracer/gossipSubMeshTracer_test.go | 29 +++++++------------ .../p2p/tracer/gossipSubScoreTracer_test.go | 8 +++-- 7 files changed, 26 insertions(+), 29 deletions(-) diff --git a/module/irrecoverable/unittest.go b/module/irrecoverable/unittest.go index 814eaba53a4..1a31dd768db 100644 --- a/module/irrecoverable/unittest.go +++ b/module/irrecoverable/unittest.go @@ -4,7 +4,7 @@ import ( "context" "testing" - "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) // MockSignalerContext is a SignalerContext which will immediately fail a test if an error is thrown. @@ -20,10 +20,10 @@ func (m MockSignalerContext) sealed() {} func (m MockSignalerContext) Throw(err error) { if m.expectError != nil { - assert.EqualError(m.t, err, m.expectError.Error()) + require.EqualError(m.t, err, m.expectError.Error()) return } - m.t.Fatalf("mock signaler context received error: %v", err) + require.Failf(m.t, "mock signaler faced error", "mock signaler context received error: %v", err) } func NewMockSignalerContext(t *testing.T, ctx context.Context) *MockSignalerContext { diff --git a/network/p2p/p2pbuilder/libp2pNodeBuilder.go b/network/p2p/p2pbuilder/libp2pNodeBuilder.go index 5a83c256c59..bb4bb56fe91 100644 --- a/network/p2p/p2pbuilder/libp2pNodeBuilder.go +++ b/network/p2p/p2pbuilder/libp2pNodeBuilder.go @@ -442,7 +442,7 @@ func DefaultNodeBuilder( SetConnectionGater(connGater). SetCreateNode(DefaultCreateNodeFunc) - if gossipCfg.ScoringParameters { + if gossipCfg.PeerScoringEnabled { // In production, we never override the default scoring config. builder.EnableGossipSubScoringWithOverride(p2p.PeerScoringConfigNoOverride) } diff --git a/network/p2p/scoring/registry.go b/network/p2p/scoring/registry.go index 493b5f4bac9..7793374c911 100644 --- a/network/p2p/scoring/registry.go +++ b/network/p2p/scoring/registry.go @@ -117,7 +117,7 @@ type GossipSubAppSpecificScoreRegistry struct { // GossipSubAppSpecificScoreRegistryConfig is the configuration for the GossipSubAppSpecificScoreRegistry. // Configurations are the "union of parameters and other components" that are used to compute or build components that compute or maintain the application specific score of peers. type GossipSubAppSpecificScoreRegistryConfig struct { - Parameters p2pconf.AppSpecificScoreParameters `validate:"required"` + Parameters p2pconf.AppSpecificScoreParams `validate:"required"` Logger zerolog.Logger `validate:"required"` @@ -163,7 +163,7 @@ func NewGossipSubAppSpecificScoreRegistry(config *GossipSubAppSpecificScoreRegis } lg := config.Logger.With().Str("module", "app_score_registry").Logger() - store := queue.NewHeroStore(config.Parameters.ScoreUpdateRequestQueueSize, + store := queue.NewHeroStore(config.Parameters, lg.With().Str("component", "app_specific_score_update").Logger(), metrics.GossipSubAppSpecificScoreUpdateQueueMetricFactory(config.HeroCacheMetricsFactory)) @@ -175,7 +175,7 @@ func NewGossipSubAppSpecificScoreRegistry(config *GossipSubAppSpecificScoreRegis init: config.Init, validator: config.Validator, idProvider: config.IdProvider, - scoreTTL: config.Parameters.ScoreTTL, + scoreTTL: config.Parameters, } reg.appScoreUpdateWorkerPool = worker.NewWorkerPoolBuilder[peer.ID](lg.With().Str("component", "app_specific_score_update_worker_pool").Logger(), diff --git a/network/p2p/scoring/score_option.go b/network/p2p/scoring/score_option.go index 86efa66d41c..d9c12f592b2 100644 --- a/network/p2p/scoring/score_option.go +++ b/network/p2p/scoring/score_option.go @@ -379,8 +379,8 @@ func NewScoreOption(cfg *ScoreOptionConfig, provider p2p.SubscriptionProvider) ( SpamRecordCacheFactory: func() p2p.GossipSubSpamRecordCache { return netcache.NewGossipSubSpamRecordCache(cfg.params.SpamRecordCacheSize, cfg.logger, cfg.heroCacheMetricsFactory, DefaultDecayFunction()) }, + Parameters: cfg.params.AppSpecificScore, }) - if err != nil { return nil, fmt.Errorf("failed to create gossipsub app specific score registry: %w", err) } diff --git a/network/p2p/scoring/subscription_validator_test.go b/network/p2p/scoring/subscription_validator_test.go index 0ac64f65fe8..4c454c8d290 100644 --- a/network/p2p/scoring/subscription_validator_test.go +++ b/network/p2p/scoring/subscription_validator_test.go @@ -170,7 +170,7 @@ func TestSubscriptionValidator_Integration(t *testing.T) { cfg, err := config.DefaultConfig() require.NoError(t, err) // set a low update interval to speed up the test - cfg.NetworkConfig.SubscriptionProvider.SubscriptionUpdateInterval = 100 * time.Millisecond + cfg.NetworkConfig.GossipSub.SubscriptionProvider.UpdateInterval = 100 * time.Millisecond sporkId := unittest.IdentifierFixture() diff --git a/network/p2p/tracer/gossipSubMeshTracer_test.go b/network/p2p/tracer/gossipSubMeshTracer_test.go index 5438f033416..6496d89b649 100644 --- a/network/p2p/tracer/gossipSubMeshTracer_test.go +++ b/network/p2p/tracer/gossipSubMeshTracer_test.go @@ -14,7 +14,6 @@ import ( "github.com/onflow/flow-go/config" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module/irrecoverable" - "github.com/onflow/flow-go/module/metrics" mockmodule "github.com/onflow/flow-go/module/mock" "github.com/onflow/flow-go/network/channels" "github.com/onflow/flow-go/network/p2p" @@ -66,24 +65,18 @@ func TestGossipSubMeshTracer(t *testing.T) { // creates one node with a gossipsub mesh meshTracer, and the other nodes without a gossipsub mesh meshTracer. // we only need one node with a meshTracer to test the meshTracer. // meshTracer logs at 1 second intervals for sake of testing. - collector := mockmodule.NewGossipSubLocalMeshMetrics(t) - meshTracerCfg := &tracer.GossipSubMeshTracerConfig{ - Logger: logger, - Metrics: collector, - IDProvider: idProvider, - LoggerInterval: time.Second, - HeroCacheMetricsFactory: metrics.NewNoopHeroCacheMetricsFactory(), - RpcSentTrackerCacheSize: defaultConfig.NetworkConfig.GossipSubConfig.RPCSentTrackerCacheSize, - RpcSentTrackerWorkerQueueCacheSize: defaultConfig.NetworkConfig.GossipSubConfig.RPCSentTrackerQueueCacheSize, - RpcSentTrackerNumOfWorkers: defaultConfig.NetworkConfig.GossipSubConfig.RpcSentTrackerNumOfWorkers, - } - meshTracer := tracer.NewGossipSubMeshTracer(meshTracerCfg) + collector := mockmodule.NewNetworkMetrics(t) + // set the meshTracer to log at 1 second intervals for sake of testing. + defaultConfig.NetworkConfig.GossipSub.RpcTracer.LocalMeshLogInterval = time.Second + tracerNode, tracerId := p2ptest.NodeFixture( t, sporkId, t.Name(), idProvider, - p2ptest.WithGossipSubTracer(meshTracer), + p2ptest.WithLogger(logger), + p2ptest.OverrideFlowConfig(defaultConfig), + p2ptest.WithMetricsCollector(collector), p2ptest.WithRole(flow.RoleConsensus)) idProvider.On("ByPeerID", tracerNode.ID()).Return(&tracerId, true).Maybe() @@ -152,14 +145,14 @@ func TestGossipSubMeshTracer(t *testing.T) { // eventually, the meshTracer should have the other nodes in its mesh. assert.Eventually(t, func() bool { topic1MeshSize := 0 - for _, peer := range meshTracer.GetLocalMeshPeers(topic1.String()) { + for _, peer := range tracerNode.GetLocalMeshPeers(topic1) { if peer == otherNode1.ID() || peer == otherNode2.ID() { topic1MeshSize++ } } topic2MeshSize := 0 - for _, peer := range meshTracer.GetLocalMeshPeers(topic2.String()) { + for _, peer := range tracerNode.GetLocalMeshPeers(topic2) { if peer == otherNode1.ID() { topic2MeshSize++ } @@ -184,14 +177,14 @@ func TestGossipSubMeshTracer(t *testing.T) { assert.Eventually(t, func() bool { // eventually, the tracerNode should not have the other node in its mesh for topic1. - for _, peer := range meshTracer.GetLocalMeshPeers(topic1.String()) { + for _, peer := range tracerNode.GetLocalMeshPeers(topic1) { if peer == otherNode1.ID() || peer == otherNode2.ID() || peer == unknownNode.ID() { return false } } // but the tracerNode should still have the otherNode1 in its mesh for topic2. - for _, peer := range meshTracer.GetLocalMeshPeers(topic2.String()) { + for _, peer := range tracerNode.GetLocalMeshPeers(topic2) { if peer != otherNode1.ID() { return false } diff --git a/network/p2p/tracer/gossipSubScoreTracer_test.go b/network/p2p/tracer/gossipSubScoreTracer_test.go index 1592f182299..33a25888a2a 100644 --- a/network/p2p/tracer/gossipSubScoreTracer_test.go +++ b/network/p2p/tracer/gossipSubScoreTracer_test.go @@ -14,6 +14,7 @@ import ( "github.com/stretchr/testify/require" "go.uber.org/atomic" + "github.com/onflow/flow-go/config" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module" "github.com/onflow/flow-go/module/irrecoverable" @@ -72,6 +73,10 @@ func TestGossipSubScoreTracer(t *testing.T) { topic1 := channels.TopicFromChannel(channels.PushBlocks, sporkId) // 3. Creates three nodes with different roles and sets their roles as consensus, access, and tracer, respectively. + cfg, err := config.DefaultConfig() + require.NoError(t, err) + // set the peer score log interval to 1 second for sake of testing. + cfg.NetworkConfig.GossipSub.RpcTracer.LocalMeshLogInterval = 1 * time.Second tracerNode, tracerId := p2ptest.NodeFixture( t, sporkId, @@ -82,7 +87,6 @@ func TestGossipSubScoreTracer(t *testing.T) { c: scoreMetrics, }), p2ptest.WithLogger(logger), - p2ptest.WithPeerScoreTracerInterval(1*time.Second), // set the peer score log interval to 1 second for sake of testing. p2ptest.EnablePeerScoringWithOverride(&p2p.PeerScoringConfigOverride{ AppSpecificScoreParams: func(pid peer.ID) float64 { id, ok := idProvider.ByPeerID(pid) @@ -163,7 +167,7 @@ func TestGossipSubScoreTracer(t *testing.T) { scoreMetrics.On("SetWarningStateCount", uint(0)).Return() // 6. Subscribes the nodes to a common topic. - _, err := tracerNode.Subscribe( + _, err = tracerNode.Subscribe( topic1, validator.TopicValidator( unittest.Logger(), From 094a02756c8b3f35a89633fb5f29b6a1229153a4 Mon Sep 17 00:00:00 2001 From: Yahya Hassanzadeh Date: Tue, 21 Nov 2023 12:32:36 -0800 Subject: [PATCH 25/67] temp supress --- network/p2p/scoring/registry.go | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/network/p2p/scoring/registry.go b/network/p2p/scoring/registry.go index 7793374c911..d1626fce5a0 100644 --- a/network/p2p/scoring/registry.go +++ b/network/p2p/scoring/registry.go @@ -18,7 +18,6 @@ import ( "github.com/onflow/flow-go/network/p2p" netcache "github.com/onflow/flow-go/network/p2p/cache" p2pmsg "github.com/onflow/flow-go/network/p2p/message" - "github.com/onflow/flow-go/network/p2p/p2pconf" "github.com/onflow/flow-go/network/p2p/p2plogging" "github.com/onflow/flow-go/utils/logging" ) @@ -117,7 +116,7 @@ type GossipSubAppSpecificScoreRegistry struct { // GossipSubAppSpecificScoreRegistryConfig is the configuration for the GossipSubAppSpecificScoreRegistry. // Configurations are the "union of parameters and other components" that are used to compute or build components that compute or maintain the application specific score of peers. type GossipSubAppSpecificScoreRegistryConfig struct { - Parameters p2pconf.AppSpecificScoreParams `validate:"required"` + // Parameters p2pconf.AppSpecificScoreParams `validate:"required"` Logger zerolog.Logger `validate:"required"` @@ -163,7 +162,7 @@ func NewGossipSubAppSpecificScoreRegistry(config *GossipSubAppSpecificScoreRegis } lg := config.Logger.With().Str("module", "app_score_registry").Logger() - store := queue.NewHeroStore(config.Parameters, + store := queue.NewHeroStore(0, lg.With().Str("component", "app_specific_score_update").Logger(), metrics.GossipSubAppSpecificScoreUpdateQueueMetricFactory(config.HeroCacheMetricsFactory)) @@ -175,7 +174,7 @@ func NewGossipSubAppSpecificScoreRegistry(config *GossipSubAppSpecificScoreRegis init: config.Init, validator: config.Validator, idProvider: config.IdProvider, - scoreTTL: config.Parameters, + scoreTTL: 0, } reg.appScoreUpdateWorkerPool = worker.NewWorkerPoolBuilder[peer.ID](lg.With().Str("component", "app_specific_score_update_worker_pool").Logger(), @@ -201,7 +200,7 @@ func NewGossipSubAppSpecificScoreRegistry(config *GossipSubAppSpecificScoreRegis reg.logger.Info().Msg("subscription validator stopped") }) - for i := 0; i < config.Parameters.ScoreUpdateWorkerNum; i++ { + for i := 0; i < 0; i++ { builder.AddWorker(reg.appScoreUpdateWorkerPool.WorkerLogic()) } From bbcca9714266b155fcd7f322bf50d70372d17656 Mon Sep 17 00:00:00 2001 From: Yahya Hassanzadeh Date: Tue, 21 Nov 2023 12:35:52 -0800 Subject: [PATCH 26/67] temp supress --- network/p2p/scoring/score_option.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/network/p2p/scoring/score_option.go b/network/p2p/scoring/score_option.go index d9c12f592b2..b22617e1fd9 100644 --- a/network/p2p/scoring/score_option.go +++ b/network/p2p/scoring/score_option.go @@ -379,7 +379,7 @@ func NewScoreOption(cfg *ScoreOptionConfig, provider p2p.SubscriptionProvider) ( SpamRecordCacheFactory: func() p2p.GossipSubSpamRecordCache { return netcache.NewGossipSubSpamRecordCache(cfg.params.SpamRecordCacheSize, cfg.logger, cfg.heroCacheMetricsFactory, DefaultDecayFunction()) }, - Parameters: cfg.params.AppSpecificScore, + // Parameters: cfg.params.AppSpecificScore, }) if err != nil { return nil, fmt.Errorf("failed to create gossipsub app specific score registry: %w", err) From 8a4524ab34098565c2556141f6d453d59535a585 Mon Sep 17 00:00:00 2001 From: Yahya Hassanzadeh Date: Tue, 21 Nov 2023 12:36:39 -0800 Subject: [PATCH 27/67] lint fix --- network/p2p/p2pbuilder/libp2pNodeBuilder.go | 1 - 1 file changed, 1 deletion(-) diff --git a/network/p2p/p2pbuilder/libp2pNodeBuilder.go b/network/p2p/p2pbuilder/libp2pNodeBuilder.go index bb4bb56fe91..fe149f7f47b 100644 --- a/network/p2p/p2pbuilder/libp2pNodeBuilder.go +++ b/network/p2p/p2pbuilder/libp2pNodeBuilder.go @@ -68,7 +68,6 @@ type LibP2PNodeBuilder struct { routingFactory func(context.Context, host.Host) (routing.Routing, error) peerManagerConfig *p2pconfig.PeerManagerConfig createNode p2p.CreateNodeFunc - gossipSubTracer p2p.PubSubTracer disallowListCacheCfg *p2p.DisallowListCacheConfig unicastConfig *p2pconfig.UnicastConfig networkingType flownet.NetworkingType // whether the node is running in private (staked) or public (unstaked) network From fc649164e7c0a02f8d4135e94b659ace9d67dfb2 Mon Sep 17 00:00:00 2001 From: Yahya Hassanzadeh Date: Tue, 21 Nov 2023 12:42:26 -0800 Subject: [PATCH 28/67] lint fix --- network/p2p/scoring/registry.go | 9 +++++---- network/p2p/scoring/score_option.go | 2 +- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/network/p2p/scoring/registry.go b/network/p2p/scoring/registry.go index d1626fce5a0..493b5f4bac9 100644 --- a/network/p2p/scoring/registry.go +++ b/network/p2p/scoring/registry.go @@ -18,6 +18,7 @@ import ( "github.com/onflow/flow-go/network/p2p" netcache "github.com/onflow/flow-go/network/p2p/cache" p2pmsg "github.com/onflow/flow-go/network/p2p/message" + "github.com/onflow/flow-go/network/p2p/p2pconf" "github.com/onflow/flow-go/network/p2p/p2plogging" "github.com/onflow/flow-go/utils/logging" ) @@ -116,7 +117,7 @@ type GossipSubAppSpecificScoreRegistry struct { // GossipSubAppSpecificScoreRegistryConfig is the configuration for the GossipSubAppSpecificScoreRegistry. // Configurations are the "union of parameters and other components" that are used to compute or build components that compute or maintain the application specific score of peers. type GossipSubAppSpecificScoreRegistryConfig struct { - // Parameters p2pconf.AppSpecificScoreParams `validate:"required"` + Parameters p2pconf.AppSpecificScoreParameters `validate:"required"` Logger zerolog.Logger `validate:"required"` @@ -162,7 +163,7 @@ func NewGossipSubAppSpecificScoreRegistry(config *GossipSubAppSpecificScoreRegis } lg := config.Logger.With().Str("module", "app_score_registry").Logger() - store := queue.NewHeroStore(0, + store := queue.NewHeroStore(config.Parameters.ScoreUpdateRequestQueueSize, lg.With().Str("component", "app_specific_score_update").Logger(), metrics.GossipSubAppSpecificScoreUpdateQueueMetricFactory(config.HeroCacheMetricsFactory)) @@ -174,7 +175,7 @@ func NewGossipSubAppSpecificScoreRegistry(config *GossipSubAppSpecificScoreRegis init: config.Init, validator: config.Validator, idProvider: config.IdProvider, - scoreTTL: 0, + scoreTTL: config.Parameters.ScoreTTL, } reg.appScoreUpdateWorkerPool = worker.NewWorkerPoolBuilder[peer.ID](lg.With().Str("component", "app_specific_score_update_worker_pool").Logger(), @@ -200,7 +201,7 @@ func NewGossipSubAppSpecificScoreRegistry(config *GossipSubAppSpecificScoreRegis reg.logger.Info().Msg("subscription validator stopped") }) - for i := 0; i < 0; i++ { + for i := 0; i < config.Parameters.ScoreUpdateWorkerNum; i++ { builder.AddWorker(reg.appScoreUpdateWorkerPool.WorkerLogic()) } diff --git a/network/p2p/scoring/score_option.go b/network/p2p/scoring/score_option.go index b22617e1fd9..d9c12f592b2 100644 --- a/network/p2p/scoring/score_option.go +++ b/network/p2p/scoring/score_option.go @@ -379,7 +379,7 @@ func NewScoreOption(cfg *ScoreOptionConfig, provider p2p.SubscriptionProvider) ( SpamRecordCacheFactory: func() p2p.GossipSubSpamRecordCache { return netcache.NewGossipSubSpamRecordCache(cfg.params.SpamRecordCacheSize, cfg.logger, cfg.heroCacheMetricsFactory, DefaultDecayFunction()) }, - // Parameters: cfg.params.AppSpecificScore, + Parameters: cfg.params.AppSpecificScore, }) if err != nil { return nil, fmt.Errorf("failed to create gossipsub app specific score registry: %w", err) From c20ce6648e0864a6ad1d3fd8d8c83940bc046a3f Mon Sep 17 00:00:00 2001 From: Yahya Hassanzadeh Date: Tue, 21 Nov 2023 14:06:07 -0800 Subject: [PATCH 29/67] fixes tests --- module/metrics/herocache.go | 11 ++ module/metrics/labels.go | 111 +++++++++--------- .../scoring/internal/appSpecificScoreCache.go | 8 +- network/p2p/scoring/score_option.go | 4 + network/p2p/test/fixtures.go | 9 +- .../p2p/tracer/gossipSubMeshTracer_test.go | 50 ++++++-- 6 files changed, 120 insertions(+), 73 deletions(-) diff --git a/module/metrics/herocache.go b/module/metrics/herocache.go index 56713683b6f..755e68e2b38 100644 --- a/module/metrics/herocache.go +++ b/module/metrics/herocache.go @@ -76,6 +76,17 @@ func NewSubscriptionRecordCacheMetricsFactory(f HeroCacheMetricsFactory) module. return f(namespaceNetwork, ResourceNetworkingSubscriptionRecordsCache) } +// NewGossipSubApplicationSpecificScoreCacheMetrics is the factory method for creating a new HeroCacheCollector for the +// application specific score cache of the GossipSub peer scoring module. The application specific score cache is used +// to keep track of the application specific score of peers in GossipSub. +// Args: +// - f: the HeroCacheMetricsFactory to create the collector +// Returns: +// - a HeroCacheMetrics for the application specific score cache +func NewGossipSubApplicationSpecificScoreCacheMetrics(f HeroCacheMetricsFactory) module.HeroCacheMetrics { + return f(namespaceNetwork, ResourceNetworkingGossipSubApplicationSpecificScoreCache) +} + // DisallowListCacheMetricsFactory is the factory method for creating a new HeroCacheCollector for the disallow list cache. // The disallow-list cache is used to keep track of peers that are disallow-listed and the reasons for it. // Args: diff --git a/module/metrics/labels.go b/module/metrics/labels.go index bc1f0f3f853..7648c0805f9 100644 --- a/module/metrics/labels.go +++ b/module/metrics/labels.go @@ -46,61 +46,62 @@ const ( ) const ( - ResourceUndefined = "undefined" - ResourceProposal = "proposal" - ResourceHeader = "header" - ResourceFinalizedHeight = "finalized_height" - ResourceIndex = "index" - ResourceIdentity = "identity" - ResourceGuarantee = "guarantee" - ResourceResult = "result" - ResourceResultApprovals = "result_approvals" - ResourceReceipt = "receipt" - ResourceQC = "qc" - ResourceMyReceipt = "my_receipt" - ResourceCollection = "collection" - ResourceApproval = "approval" - ResourceSeal = "seal" - ResourcePendingIncorporatedSeal = "pending_incorporated_seal" - ResourceCommit = "commit" - ResourceTransaction = "transaction" - ResourceClusterPayload = "cluster_payload" - ResourceClusterProposal = "cluster_proposal" - ResourceProcessedResultID = "processed_result_id" // verification node, finder engine // TODO: remove finder engine labels - ResourceDiscardedResultID = "discarded_result_id" // verification node, finder engine - ResourcePendingReceipt = "pending_receipt" // verification node, finder engine - ResourceReceiptIDsByResult = "receipt_ids_by_result" // verification node, finder engine - ResourcePendingReceiptIDsByBlock = "pending_receipt_ids_by_block" // verification node, finder engine - ResourcePendingResult = "pending_result" // verification node, match engine - ResourceChunkIDsByResult = "chunk_ids_by_result" // verification node, match engine - ResourcePendingChunk = "pending_chunk" // verification node, match engine - ResourcePendingBlock = "pending_block" // verification node, match engine - ResourceCachedReceipt = "cached_receipt" // verification node, finder engine - ResourceCachedBlockID = "cached_block_id" // verification node, finder engine - ResourceChunkStatus = "chunk_status" // verification node, fetcher engine - ResourceChunkRequest = "chunk_request" // verification node, requester engine - ResourceChunkConsumer = "chunk_consumer_jobs" // verification node - ResourceBlockConsumer = "block_consumer_jobs" // verification node - ResourceEpochSetup = "epoch_setup" - ResourceEpochCommit = "epoch_commit" - ResourceEpochStatus = "epoch_status" - ResourceNetworkingReceiveCache = "networking_received_message" // networking layer - ResourceNetworkingSubscriptionRecordsCache = "subscription_records_cache" // networking layer - ResourceNetworkingDnsIpCache = "networking_dns_ip_cache" // networking layer - ResourceNetworkingDnsTxtCache = "networking_dns_txt_cache" // networking layer - ResourceNetworkingDisallowListNotificationQueue = "networking_disallow_list_notification_queue" - ResourceNetworkingRpcInspectorNotificationQueue = "networking_rpc_inspector_notification_queue" - ResourceNetworkingRpcValidationInspectorQueue = "networking_rpc_validation_inspector_queue" - ResourceNetworkingRpcMetricsObserverInspectorQueue = "networking_rpc_metrics_observer_inspector_queue" - ResourceNetworkingApplicationLayerSpamRecordCache = "application_layer_spam_record_cache" - ResourceNetworkingApplicationLayerSpamReportQueue = "application_layer_spam_report_queue" - ResourceNetworkingRpcClusterPrefixReceivedCache = "rpc_cluster_prefixed_received_cache" - ResourceNetworkingAppSpecificScoreUpdateQueue = "gossipsub_app_specific_score_update_queue" - ResourceNetworkingGossipSubSpamRecordCache = "gossipsub_spam_record_cache" - ResourceNetworkingDisallowListCache = "disallow_list_cache" - ResourceNetworkingRPCSentTrackerCache = "gossipsub_rpc_sent_tracker_cache" - ResourceNetworkingRPCSentTrackerQueue = "gossipsub_rpc_sent_tracker_queue" - ResourceNetworkingUnicastDialConfigCache = "unicast_dial_config_cache" + ResourceUndefined = "undefined" + ResourceProposal = "proposal" + ResourceHeader = "header" + ResourceFinalizedHeight = "finalized_height" + ResourceIndex = "index" + ResourceIdentity = "identity" + ResourceGuarantee = "guarantee" + ResourceResult = "result" + ResourceResultApprovals = "result_approvals" + ResourceReceipt = "receipt" + ResourceQC = "qc" + ResourceMyReceipt = "my_receipt" + ResourceCollection = "collection" + ResourceApproval = "approval" + ResourceSeal = "seal" + ResourcePendingIncorporatedSeal = "pending_incorporated_seal" + ResourceCommit = "commit" + ResourceTransaction = "transaction" + ResourceClusterPayload = "cluster_payload" + ResourceClusterProposal = "cluster_proposal" + ResourceProcessedResultID = "processed_result_id" // verification node, finder engine // TODO: remove finder engine labels + ResourceDiscardedResultID = "discarded_result_id" // verification node, finder engine + ResourcePendingReceipt = "pending_receipt" // verification node, finder engine + ResourceReceiptIDsByResult = "receipt_ids_by_result" // verification node, finder engine + ResourcePendingReceiptIDsByBlock = "pending_receipt_ids_by_block" // verification node, finder engine + ResourcePendingResult = "pending_result" // verification node, match engine + ResourceChunkIDsByResult = "chunk_ids_by_result" // verification node, match engine + ResourcePendingChunk = "pending_chunk" // verification node, match engine + ResourcePendingBlock = "pending_block" // verification node, match engine + ResourceCachedReceipt = "cached_receipt" // verification node, finder engine + ResourceCachedBlockID = "cached_block_id" // verification node, finder engine + ResourceChunkStatus = "chunk_status" // verification node, fetcher engine + ResourceChunkRequest = "chunk_request" // verification node, requester engine + ResourceChunkConsumer = "chunk_consumer_jobs" // verification node + ResourceBlockConsumer = "block_consumer_jobs" // verification node + ResourceEpochSetup = "epoch_setup" + ResourceEpochCommit = "epoch_commit" + ResourceEpochStatus = "epoch_status" + ResourceNetworkingReceiveCache = "networking_received_message" // networking layer + ResourceNetworkingSubscriptionRecordsCache = "subscription_records_cache" // networking layer + ResourceNetworkingDnsIpCache = "networking_dns_ip_cache" // networking layer + ResourceNetworkingDnsTxtCache = "networking_dns_txt_cache" // networking layer + ResourceNetworkingDisallowListNotificationQueue = "networking_disallow_list_notification_queue" + ResourceNetworkingRpcInspectorNotificationQueue = "networking_rpc_inspector_notification_queue" + ResourceNetworkingRpcValidationInspectorQueue = "networking_rpc_validation_inspector_queue" + ResourceNetworkingRpcMetricsObserverInspectorQueue = "networking_rpc_metrics_observer_inspector_queue" + ResourceNetworkingApplicationLayerSpamRecordCache = "application_layer_spam_record_cache" + ResourceNetworkingApplicationLayerSpamReportQueue = "application_layer_spam_report_queue" + ResourceNetworkingRpcClusterPrefixReceivedCache = "rpc_cluster_prefixed_received_cache" + ResourceNetworkingAppSpecificScoreUpdateQueue = "gossipsub_app_specific_score_update_queue" + ResourceNetworkingGossipSubApplicationSpecificScoreCache = "gossipsub_application_specific_score_cache" + ResourceNetworkingGossipSubSpamRecordCache = "gossipsub_spam_record_cache" + ResourceNetworkingDisallowListCache = "disallow_list_cache" + ResourceNetworkingRPCSentTrackerCache = "gossipsub_rpc_sent_tracker_cache" + ResourceNetworkingRPCSentTrackerQueue = "gossipsub_rpc_sent_tracker_queue" + ResourceNetworkingUnicastDialConfigCache = "unicast_dial_config_cache" ResourceFollowerPendingBlocksCache = "follower_pending_block_cache" // follower engine ResourceFollowerLoopCertifiedBlocksChannel = "follower_loop_certified_blocks_channel" // follower loop, certified blocks buffered channel diff --git a/network/p2p/scoring/internal/appSpecificScoreCache.go b/network/p2p/scoring/internal/appSpecificScoreCache.go index 5513db07df8..0fa8eb2b8a1 100644 --- a/network/p2p/scoring/internal/appSpecificScoreCache.go +++ b/network/p2p/scoring/internal/appSpecificScoreCache.go @@ -8,10 +8,10 @@ import ( "github.com/rs/zerolog" "github.com/onflow/flow-go/model/flow" - "github.com/onflow/flow-go/module" herocache "github.com/onflow/flow-go/module/mempool/herocache/backdata" "github.com/onflow/flow-go/module/mempool/herocache/backdata/heropool" "github.com/onflow/flow-go/module/mempool/stdmap" + "github.com/onflow/flow-go/module/metrics" "github.com/onflow/flow-go/network/p2p" ) @@ -33,12 +33,12 @@ var _ p2p.GossipSubApplicationSpecificScoreCache = (*AppSpecificScoreCache)(nil) // - collector: the metrics collector to use for collecting metrics. // Returns: // - *AppSpecificScoreCache: the created cache. -func NewAppSpecificScoreCache(sizeLimit uint32, logger zerolog.Logger, collector module.HeroCacheMetrics) *AppSpecificScoreCache { +func NewAppSpecificScoreCache(sizeLimit uint32, logger zerolog.Logger, hcMetricsFactory metrics.HeroCacheMetricsFactory) *AppSpecificScoreCache { backData := herocache.NewCache(sizeLimit, herocache.DefaultOversizeFactor, heropool.LRUEjection, - logger.With().Str("mempool", "subscription-records").Logger(), - collector) + logger.With().Str("mempool", "gossipsub-app-specific-score-cache").Logger(), + metrics.NewGossipSubApplicationSpecificScoreCacheMetrics(hcMetricsFactory)) return &AppSpecificScoreCache{ c: stdmap.NewBackend(stdmap.WithBackData(backData)), diff --git a/network/p2p/scoring/score_option.go b/network/p2p/scoring/score_option.go index d9c12f592b2..7aa2d3093d9 100644 --- a/network/p2p/scoring/score_option.go +++ b/network/p2p/scoring/score_option.go @@ -16,6 +16,7 @@ import ( "github.com/onflow/flow-go/network/p2p" netcache "github.com/onflow/flow-go/network/p2p/cache" "github.com/onflow/flow-go/network/p2p/p2pconf" + "github.com/onflow/flow-go/network/p2p/scoring/internal" "github.com/onflow/flow-go/network/p2p/utils" "github.com/onflow/flow-go/utils/logging" ) @@ -376,6 +377,9 @@ func NewScoreOption(cfg *ScoreOptionConfig, provider p2p.SubscriptionProvider) ( Init: InitAppScoreRecordState, IdProvider: cfg.provider, HeroCacheMetricsFactory: cfg.heroCacheMetricsFactory, + AppScoreCacheFactory: func() p2p.GossipSubApplicationSpecificScoreCache { + return internal.NewAppSpecificScoreCache(cfg.params.SpamRecordCacheSize, cfg.logger, cfg.heroCacheMetricsFactory) + }, SpamRecordCacheFactory: func() p2p.GossipSubSpamRecordCache { return netcache.NewGossipSubSpamRecordCache(cfg.params.SpamRecordCacheSize, cfg.logger, cfg.heroCacheMetricsFactory, DefaultDecayFunction()) }, diff --git a/network/p2p/test/fixtures.go b/network/p2p/test/fixtures.go index ecb7de77f70..4ef30ea1675 100644 --- a/network/p2p/test/fixtures.go +++ b/network/p2p/test/fixtures.go @@ -3,6 +3,7 @@ package p2ptest import ( "bufio" "context" + "fmt" crand "math/rand" "sync" "testing" @@ -80,10 +81,9 @@ func NodeFixture(t *testing.T, dhtPrefix string, idProvider module.IdentityProvider, opts ...NodeFixtureParameterOption) (p2p.LibP2PNode, flow.Identity) { + defaultFlowConfig, err := config.DefaultConfig() require.NoError(t, err) - - logger := unittest.Logger().Level(zerolog.WarnLevel) require.NotNil(t, idProvider) connectionGater := NewConnectionGater(idProvider, func(p peer.ID) error { return nil @@ -96,7 +96,7 @@ func NodeFixture(t *testing.T, Unicasts: nil, Key: NetworkingKeyFixtures(t), Address: unittest.DefaultAddress, - Logger: logger, + Logger: unittest.Logger().Level(zerolog.WarnLevel), Role: flow.RoleCollection, IdProvider: idProvider, MetricsCfg: &p2pconfig.MetricsConfig{ @@ -120,7 +120,7 @@ func NodeFixture(t *testing.T, unittest.WithAddress(parameters.Address), unittest.WithRole(parameters.Role)) - logger = parameters.Logger.With().Hex("node_id", logging.ID(identity.NodeID)).Logger() + logger := parameters.Logger.With().Hex("node_id", logging.ID(identity.NodeID)).Logger() connManager, err := connection.NewConnManager(logger, parameters.MetricsCfg.Metrics, ¶meters.FlowConfig.NetworkConfig.ConnectionManagerConfig) require.NoError(t, err) @@ -340,6 +340,7 @@ func WithPeerScoreParamsOption(cfg *p2p.PeerScoringConfigOverride) NodeFixturePa func WithLogger(logger zerolog.Logger) NodeFixtureParameterOption { return func(p *NodeFixtureParameters) { p.Logger = logger + fmt.Println("logger set") } } diff --git a/network/p2p/tracer/gossipSubMeshTracer_test.go b/network/p2p/tracer/gossipSubMeshTracer_test.go index 6496d89b649..be8f9b0afa7 100644 --- a/network/p2p/tracer/gossipSubMeshTracer_test.go +++ b/network/p2p/tracer/gossipSubMeshTracer_test.go @@ -2,10 +2,12 @@ package tracer_test import ( "context" + "fmt" "os" "testing" "time" + "github.com/libp2p/go-libp2p/core/peer" "github.com/rs/zerolog" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -14,6 +16,7 @@ import ( "github.com/onflow/flow-go/config" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module/irrecoverable" + "github.com/onflow/flow-go/module/metrics" mockmodule "github.com/onflow/flow-go/module/mock" "github.com/onflow/flow-go/network/channels" "github.com/onflow/flow-go/network/p2p" @@ -65,9 +68,10 @@ func TestGossipSubMeshTracer(t *testing.T) { // creates one node with a gossipsub mesh meshTracer, and the other nodes without a gossipsub mesh meshTracer. // we only need one node with a meshTracer to test the meshTracer. // meshTracer logs at 1 second intervals for sake of testing. - collector := mockmodule.NewNetworkMetrics(t) + collector := newLocalMeshTracerMetricsCollector(t) // set the meshTracer to log at 1 second intervals for sake of testing. - defaultConfig.NetworkConfig.GossipSub.RpcTracer.LocalMeshLogInterval = time.Second + defaultConfig.NetworkConfig.GossipSub.RpcTracer.LocalMeshLogInterval = 1 * time.Second + defaultConfig.NetworkConfig.GossipSub.PeerScoringEnabled = false tracerNode, tracerId := p2ptest.NodeFixture( t, @@ -106,6 +110,13 @@ func TestGossipSubMeshTracer(t *testing.T) { p2ptest.WithRole(flow.RoleConsensus)) idProvider.On("ByPeerID", unknownNode.ID()).Return(nil, false).Maybe() + peerProvider := func() peer.IDSlice { + return peer.IDSlice{tracerNode.ID(), otherNode1.ID(), otherNode2.ID(), unknownNode.ID()} + } + tracerNode.WithPeersProvider(peerProvider) + otherNode1.WithPeersProvider(peerProvider) + otherNode2.WithPeersProvider(peerProvider) + nodes := []p2p.LibP2PNode{tracerNode, otherNode1, otherNode2, unknownNode} ids := flow.IdentityList{&tracerId, &otherId1, &otherId2, &unknownId} @@ -145,15 +156,15 @@ func TestGossipSubMeshTracer(t *testing.T) { // eventually, the meshTracer should have the other nodes in its mesh. assert.Eventually(t, func() bool { topic1MeshSize := 0 - for _, peer := range tracerNode.GetLocalMeshPeers(topic1) { - if peer == otherNode1.ID() || peer == otherNode2.ID() { + for _, peerId := range tracerNode.GetLocalMeshPeers(topic1) { + if peerId == otherNode1.ID() || peerId == otherNode2.ID() { topic1MeshSize++ } } topic2MeshSize := 0 - for _, peer := range tracerNode.GetLocalMeshPeers(topic2) { - if peer == otherNode1.ID() { + for _, peerId := range tracerNode.GetLocalMeshPeers(topic2) { + if peerId == otherNode1.ID() { topic2MeshSize++ } } @@ -177,18 +188,37 @@ func TestGossipSubMeshTracer(t *testing.T) { assert.Eventually(t, func() bool { // eventually, the tracerNode should not have the other node in its mesh for topic1. - for _, peer := range tracerNode.GetLocalMeshPeers(topic1) { - if peer == otherNode1.ID() || peer == otherNode2.ID() || peer == unknownNode.ID() { + for _, peerId := range tracerNode.GetLocalMeshPeers(topic1) { + if peerId == otherNode1.ID() || peerId == otherNode2.ID() || peerId == unknownNode.ID() { return false } } // but the tracerNode should still have the otherNode1 in its mesh for topic2. - for _, peer := range tracerNode.GetLocalMeshPeers(topic2) { - if peer != otherNode1.ID() { + for _, peerId := range tracerNode.GetLocalMeshPeers(topic2) { + if peerId != otherNode1.ID() { return false } } return true }, 2*time.Second, 10*time.Millisecond) } + +// localMeshTracerMetricsCollector is a mock metrics that can be mocked for GossipSubLocalMeshMetrics while acting as a NoopCollector for other metrics. +type localMeshTracerMetricsCollector struct { + *metrics.NoopCollector + *mockmodule.GossipSubLocalMeshMetrics +} + +func newLocalMeshTracerMetricsCollector(t *testing.T) *localMeshTracerMetricsCollector { + return &localMeshTracerMetricsCollector{ + GossipSubLocalMeshMetrics: mockmodule.NewGossipSubLocalMeshMetrics(t), + NoopCollector: metrics.NewNoopCollector(), + } +} + +func (c *localMeshTracerMetricsCollector) OnLocalMeshSizeUpdated(topic string, size int) { + // calls the mock method to assert the metrics. + fmt.Println("OnLocalMeshSizeUpdated", topic, size) + c.GossipSubLocalMeshMetrics.OnLocalMeshSizeUpdated(topic, size) +} From 53dc21c488fd8684141ff64ac7338fa8fa8a7f28 Mon Sep 17 00:00:00 2001 From: Yahya Hassanzadeh Date: Tue, 21 Nov 2023 14:06:44 -0800 Subject: [PATCH 30/67] adds a comment --- network/p2p/tracer/gossipSubMeshTracer_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/network/p2p/tracer/gossipSubMeshTracer_test.go b/network/p2p/tracer/gossipSubMeshTracer_test.go index be8f9b0afa7..fe111594535 100644 --- a/network/p2p/tracer/gossipSubMeshTracer_test.go +++ b/network/p2p/tracer/gossipSubMeshTracer_test.go @@ -71,6 +71,7 @@ func TestGossipSubMeshTracer(t *testing.T) { collector := newLocalMeshTracerMetricsCollector(t) // set the meshTracer to log at 1 second intervals for sake of testing. defaultConfig.NetworkConfig.GossipSub.RpcTracer.LocalMeshLogInterval = 1 * time.Second + // disables peer scoring for sake of testing; so that unknown peers are not penalized and could be detected by the meshTracer. defaultConfig.NetworkConfig.GossipSub.PeerScoringEnabled = false tracerNode, tracerId := p2ptest.NodeFixture( From a7985971d6f4186bccfd0e12775db8d5b2abf9b1 Mon Sep 17 00:00:00 2001 From: Yahya Hassanzadeh Date: Tue, 21 Nov 2023 14:41:43 -0800 Subject: [PATCH 31/67] fixes TestNoPenaltyRecord --- .../internal/appSpecificScoreCache_test.go | 6 +- network/p2p/scoring/registry_test.go | 88 +++++++++++++++---- network/p2p/test/fixtures.go | 2 - 3 files changed, 72 insertions(+), 24 deletions(-) diff --git a/network/p2p/scoring/internal/appSpecificScoreCache_test.go b/network/p2p/scoring/internal/appSpecificScoreCache_test.go index 415218f3c30..ef4b89eec50 100644 --- a/network/p2p/scoring/internal/appSpecificScoreCache_test.go +++ b/network/p2p/scoring/internal/appSpecificScoreCache_test.go @@ -16,7 +16,7 @@ import ( // specifically, it tests the Add and Get methods. // It does not test the eviction policy of the cache. func TestAppSpecificScoreCache(t *testing.T) { - cache := internal.NewAppSpecificScoreCache(10, unittest.Logger(), metrics.NewNoopCollector()) + cache := internal.NewAppSpecificScoreCache(10, unittest.Logger(), metrics.NewNoopHeroCacheMetricsFactory()) require.NotNil(t, cache, "failed to create AppSpecificScoreCache") peerID := unittest.PeerIdFixture(t) @@ -47,7 +47,7 @@ func TestAppSpecificScoreCache(t *testing.T) { // TestAppSpecificScoreCache_Concurrent_Add_Get_Update tests the concurrent functionality of AppSpecificScoreCache; // specifically, it tests the Add and Get methods under concurrent access. func TestAppSpecificScoreCache_Concurrent_Add_Get_Update(t *testing.T) { - cache := internal.NewAppSpecificScoreCache(10, unittest.Logger(), metrics.NewNoopCollector()) + cache := internal.NewAppSpecificScoreCache(10, unittest.Logger(), metrics.NewNoopHeroCacheMetricsFactory()) require.NotNil(t, cache, "failed to create AppSpecificScoreCache") peerId1 := unittest.PeerIdFixture(t) @@ -140,7 +140,7 @@ func TestAppSpecificScoreCache_Concurrent_Add_Get_Update(t *testing.T) { // TestAppSpecificScoreCache_Eviction tests the eviction policy of AppSpecificScoreCache; // specifically, it tests that the cache evicts the least recently used record when the cache is full. func TestAppSpecificScoreCache_Eviction(t *testing.T) { - cache := internal.NewAppSpecificScoreCache(10, unittest.Logger(), metrics.NewNoopCollector()) + cache := internal.NewAppSpecificScoreCache(10, unittest.Logger(), metrics.NewNoopHeroCacheMetricsFactory()) require.NotNil(t, cache, "failed to create AppSpecificScoreCache") peerIds := unittest.PeerIdFixtures(t, 11) diff --git a/network/p2p/scoring/registry_test.go b/network/p2p/scoring/registry_test.go index 2c63ea0f112..55c4710987b 100644 --- a/network/p2p/scoring/registry_test.go +++ b/network/p2p/scoring/registry_test.go @@ -1,6 +1,7 @@ package scoring_test import ( + "context" "fmt" "math" "testing" @@ -11,6 +12,8 @@ import ( testifymock "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" + "github.com/onflow/flow-go/config" + "github.com/onflow/flow-go/module/irrecoverable" "github.com/onflow/flow-go/module/metrics" "github.com/onflow/flow-go/module/mock" "github.com/onflow/flow-go/network/p2p" @@ -18,6 +21,7 @@ import ( p2pmsg "github.com/onflow/flow-go/network/p2p/message" mockp2p "github.com/onflow/flow-go/network/p2p/mock" "github.com/onflow/flow-go/network/p2p/scoring" + "github.com/onflow/flow-go/network/p2p/scoring/internal" "github.com/onflow/flow-go/utils/unittest" ) @@ -26,21 +30,48 @@ import ( // penalized. func TestNoPenaltyRecord(t *testing.T) { peerID := peer.ID("peer-1") - reg, spamRecords := newGossipSubAppSpecificScoreRegistry( - t, + ctx, cancel := context.WithCancel(context.Background()) + + reg, spamRecords, appScoreCache := newGossipSubAppSpecificScoreRegistry(t, withStakedIdentity(peerID), withValidSubscriptions(peerID)) - // initially, the spamRecords should not have the peer id. - assert.False(t, spamRecords.Has(peerID)) + // starts the registry. + signalerCtx := irrecoverable.NewMockSignalerContext(t, ctx) + reg.Start(signalerCtx) + unittest.RequireCloseBefore(t, reg.Ready(), 1*time.Second, "failed to start GossipSubAppSpecificScoreRegistry") - score := reg.AppSpecificScoreFunc()(peerID) - // since the peer id does not have a spam record, the app specific score should be the max app specific reward, which - // is the default reward for a staked peer that has valid subscriptions. - assert.Equal(t, scoring.MaxAppSpecificReward, score) + // initially, the spamRecords should not have the peer id, and there should be no app-specific score in the cache. + require.False(t, spamRecords.Has(peerID)) + score, updated, exists := appScoreCache.Get(peerID) // get the score from the cache. + require.False(t, exists) + require.Equal(t, time.Time{}, updated) + require.Equal(t, float64(0), score) + + queryTime := time.Now() + require.Eventually(t, func() bool { + // calling the app specific score function when there is no app specific score in the cache should eventually update the cache. + score := reg.AppSpecificScoreFunc()(peerID) + // since the peer id does not have a spam record, the app specific score should be the max app specific reward, which + // is the default reward for a staked peer that has valid subscriptions. + if score == scoring.MaxAppSpecificReward { + return true + } + return false + }, 5*time.Second, 100*time.Millisecond) // still the spamRecords should not have the peer id (as there is no spam record for the peer id). - assert.False(t, spamRecords.Has(peerID)) + require.False(t, spamRecords.Has(peerID)) + + // however, the app specific score should be updated in the cache. + score, updated, exists = appScoreCache.Get(peerID) // get the score from the cache. + require.True(t, exists) + require.True(t, updated.After(queryTime)) + require.Equal(t, scoring.MaxAppSpecificReward, score) + + // stop the registry. + cancel() + unittest.RequireCloseBefore(t, reg.Done(), 1*time.Second, "failed to stop GossipSubAppSpecificScoreRegistry") } // TestPeerWithSpamRecord tests the app specific penalty computation of the node when there is a spam record for the peer id. @@ -67,7 +98,7 @@ func TestPeerWithSpamRecord(t *testing.T) { func testPeerWithSpamRecord(t *testing.T, messageType p2pmsg.ControlMessageType, expectedPenalty float64) { peerID := peer.ID("peer-1") - reg, spamRecords := newGossipSubAppSpecificScoreRegistry( + reg, spamRecords, _ := newGossipSubAppSpecificScoreRegistry( t, withStakedIdentity(peerID), withValidSubscriptions(peerID)) @@ -121,7 +152,7 @@ func TestSpamRecord_With_UnknownIdentity(t *testing.T) { // the peer id has an unknown identity. func testSpamRecordWithUnknownIdentity(t *testing.T, messageType p2pmsg.ControlMessageType, expectedPenalty float64) { peerID := peer.ID("peer-1") - reg, spamRecords := newGossipSubAppSpecificScoreRegistry( + reg, spamRecords, _ := newGossipSubAppSpecificScoreRegistry( t, withUnknownIdentity(peerID), withValidSubscriptions(peerID)) @@ -174,7 +205,7 @@ func TestSpamRecord_With_SubscriptionPenalty(t *testing.T) { // the peer id has an invalid subscription as well. func testSpamRecordWithSubscriptionPenalty(t *testing.T, messageType p2pmsg.ControlMessageType, expectedPenalty float64) { peerID := peer.ID("peer-1") - reg, spamRecords := newGossipSubAppSpecificScoreRegistry( + reg, spamRecords, _ := newGossipSubAppSpecificScoreRegistry( t, withStakedIdentity(peerID), withInvalidSubscriptions(peerID)) @@ -208,7 +239,7 @@ func testSpamRecordWithSubscriptionPenalty(t *testing.T, messageType p2pmsg.Cont // TestSpamPenaltyDecaysInCache tests that the spam penalty records decay over time in the cache. func TestSpamPenaltyDecaysInCache(t *testing.T) { peerID := peer.ID("peer-1") - reg, _ := newGossipSubAppSpecificScoreRegistry(t, + reg, _, _ := newGossipSubAppSpecificScoreRegistry(t, withStakedIdentity(peerID), withValidSubscriptions(peerID)) @@ -271,7 +302,7 @@ func TestSpamPenaltyDecaysInCache(t *testing.T) { // a peer is set back to zero, its app specific penalty is also reset to the initial state. func TestSpamPenaltyDecayToZero(t *testing.T) { peerID := peer.ID("peer-1") - reg, spamRecords := newGossipSubAppSpecificScoreRegistry( + reg, spamRecords, _ := newGossipSubAppSpecificScoreRegistry( t, withStakedIdentity(peerID), withValidSubscriptions(peerID), @@ -317,7 +348,7 @@ func TestSpamPenaltyDecayToZero(t *testing.T) { // is persisted. This is because the unknown identity penalty is not decayed. func TestPersistingUnknownIdentityPenalty(t *testing.T) { peerID := peer.ID("peer-1") - reg, spamRecords := newGossipSubAppSpecificScoreRegistry( + reg, spamRecords, _ := newGossipSubAppSpecificScoreRegistry( t, withUnknownIdentity(peerID), // the peer id has an unknown identity. withValidSubscriptions(peerID), @@ -374,7 +405,7 @@ func TestPersistingUnknownIdentityPenalty(t *testing.T) { // is persisted. This is because the invalid subscription penalty is not decayed. func TestPersistingInvalidSubscriptionPenalty(t *testing.T) { peerID := peer.ID("peer-1") - reg, spamRecords := newGossipSubAppSpecificScoreRegistry( + reg, spamRecords, _ := newGossipSubAppSpecificScoreRegistry( t, withStakedIdentity(peerID), withInvalidSubscriptions(peerID), // the peer id has an invalid subscription. @@ -465,17 +496,36 @@ func withInitFunction(initFunction func() p2p.GossipSubSpamRecord) func(cfg *sco // newGossipSubAppSpecificScoreRegistry returns a new instance of GossipSubAppSpecificScoreRegistry with default values // for the testing purposes. func newGossipSubAppSpecificScoreRegistry(t *testing.T, opts ...func(*scoring.GossipSubAppSpecificScoreRegistryConfig)) (*scoring.GossipSubAppSpecificScoreRegistry, - *netcache.GossipSubSpamRecordCache) { + *netcache.GossipSubSpamRecordCache, + *internal.AppSpecificScoreCache) { cache := netcache.NewGossipSubSpamRecordCache(100, unittest.Logger(), metrics.NewNoopHeroCacheMetricsFactory(), scoring.DefaultDecayFunction()) + appSpecificScoreCache := internal.NewAppSpecificScoreCache(100, unittest.Logger(), metrics.NewNoopHeroCacheMetricsFactory()) + flowCfg, err := config.DefaultConfig() + require.NoError(t, err) + + validator := mockp2p.NewSubscriptionValidator(t) + validator.On("Start", testifymock.Anything).Return().Maybe() + done := make(chan struct{}) + close(done) + f := func() <-chan struct{} { + return done + } + validator.On("Ready").Return(f()).Maybe() + validator.On("Done").Return(f()).Maybe() cfg := &scoring.GossipSubAppSpecificScoreRegistryConfig{ Logger: unittest.Logger(), Init: scoring.InitAppScoreRecordState, Penalty: penaltyValueFixtures(), IdProvider: mock.NewIdentityProvider(t), - Validator: mockp2p.NewSubscriptionValidator(t), + Validator: validator, + AppScoreCacheFactory: func() p2p.GossipSubApplicationSpecificScoreCache { + return appSpecificScoreCache + }, SpamRecordCacheFactory: func() p2p.GossipSubSpamRecordCache { return cache }, + Parameters: flowCfg.NetworkConfig.GossipSub.ScoringParameters.AppSpecificScore, + HeroCacheMetricsFactory: metrics.NewNoopHeroCacheMetricsFactory(), } for _, opt := range opts { opt(cfg) @@ -484,7 +534,7 @@ func newGossipSubAppSpecificScoreRegistry(t *testing.T, opts ...func(*scoring.Go reg, err := scoring.NewGossipSubAppSpecificScoreRegistry(cfg) require.NoError(t, err, "failed to create GossipSubAppSpecificScoreRegistry") - return reg, cache + return reg, cache, appSpecificScoreCache } // penaltyValueFixtures returns a set of penalty values for testing purposes. diff --git a/network/p2p/test/fixtures.go b/network/p2p/test/fixtures.go index 4ef30ea1675..11132bd6949 100644 --- a/network/p2p/test/fixtures.go +++ b/network/p2p/test/fixtures.go @@ -3,7 +3,6 @@ package p2ptest import ( "bufio" "context" - "fmt" crand "math/rand" "sync" "testing" @@ -340,7 +339,6 @@ func WithPeerScoreParamsOption(cfg *p2p.PeerScoringConfigOverride) NodeFixturePa func WithLogger(logger zerolog.Logger) NodeFixtureParameterOption { return func(p *NodeFixtureParameters) { p.Logger = logger - fmt.Println("logger set") } } From f7c5e11ad10420e0418c2f9a593f92ca9122a471 Mon Sep 17 00:00:00 2001 From: Yahya Hassanzadeh Date: Tue, 21 Nov 2023 15:02:52 -0800 Subject: [PATCH 32/67] fixes testPeerWithSpamRecord --- network/p2p/scoring/registry_test.go | 63 ++++++++++++++++++++++------ 1 file changed, 51 insertions(+), 12 deletions(-) diff --git a/network/p2p/scoring/registry_test.go b/network/p2p/scoring/registry_test.go index 55c4710987b..6decd1ed0d1 100644 --- a/network/p2p/scoring/registry_test.go +++ b/network/p2p/scoring/registry_test.go @@ -30,13 +30,13 @@ import ( // penalized. func TestNoPenaltyRecord(t *testing.T) { peerID := peer.ID("peer-1") - ctx, cancel := context.WithCancel(context.Background()) reg, spamRecords, appScoreCache := newGossipSubAppSpecificScoreRegistry(t, withStakedIdentity(peerID), withValidSubscriptions(peerID)) // starts the registry. + ctx, cancel := context.WithCancel(context.Background()) signalerCtx := irrecoverable.NewMockSignalerContext(t, ctx) reg.Start(signalerCtx) unittest.RequireCloseBefore(t, reg.Ready(), 1*time.Second, "failed to start GossipSubAppSpecificScoreRegistry") @@ -98,18 +98,36 @@ func TestPeerWithSpamRecord(t *testing.T) { func testPeerWithSpamRecord(t *testing.T, messageType p2pmsg.ControlMessageType, expectedPenalty float64) { peerID := peer.ID("peer-1") - reg, spamRecords, _ := newGossipSubAppSpecificScoreRegistry( + + reg, spamRecords, appScoreCache := newGossipSubAppSpecificScoreRegistry( t, withStakedIdentity(peerID), withValidSubscriptions(peerID)) - // initially, the spamRecords should not have the peer id. - assert.False(t, spamRecords.Has(peerID)) + // starts the registry. + ctx, cancel := context.WithCancel(context.Background()) + signalerCtx := irrecoverable.NewMockSignalerContext(t, ctx) + reg.Start(signalerCtx) + unittest.RequireCloseBefore(t, reg.Ready(), 1*time.Second, "failed to start GossipSubAppSpecificScoreRegistry") - // since the peer id does not have a spam record, the app specific score should be the max app specific reward, which - // is the default reward for a staked peer that has valid subscriptions. - score := reg.AppSpecificScoreFunc()(peerID) - assert.Equal(t, scoring.MaxAppSpecificReward, score) + // initially, the spamRecords should not have the peer id; also the app specific score record should not be in the cache. + require.False(t, spamRecords.Has(peerID)) + score, updated, exists := appScoreCache.Get(peerID) // get the score from the cache. + require.False(t, exists) + require.Equal(t, time.Time{}, updated) + require.Equal(t, float64(0), score) + + // eventually, the app specific score should be updated in the cache. + require.Eventually(t, func() bool { + // calling the app specific score function when there is no app specific score in the cache should eventually update the cache. + score := reg.AppSpecificScoreFunc()(peerID) + // since the peer id does not have a spam record, the app specific score should be the max app specific reward, which + // is the default reward for a staked peer that has valid subscriptions. + if scoring.MaxAppSpecificReward == score { + return true + } + return false + }, 5*time.Second, 100*time.Millisecond) // report a misbehavior for the peer id. reg.OnInvalidControlMessageNotification(&p2p.InvCtrlMsgNotif{ @@ -124,10 +142,29 @@ func testPeerWithSpamRecord(t *testing.T, messageType p2pmsg.ControlMessageType, assert.Less(t, math.Abs(expectedPenalty-record.Penalty), 10e-3) // penalty should be updated to -10. assert.Equal(t, scoring.InitAppScoreRecordState().Decay, record.Decay) // decay should be initialized to the initial state. - // this peer has a spam record, with no subscription penalty. Hence, the app specific score should only be the spam penalty, - // and the peer should be deprived of the default reward for its valid staked role. - score = reg.AppSpecificScoreFunc()(peerID) - assert.Less(t, math.Abs(expectedPenalty-score), 10e-3) + queryTime := time.Now() + // eventually, the app specific score should be updated in the cache. + require.Eventually(t, func() bool { + // calling the app specific score function when there is no app specific score in the cache should eventually update the cache. + score := reg.AppSpecificScoreFunc()(peerID) + // this peer has a spam record, with no subscription penalty. Hence, the app specific score should only be the spam penalty, + // and the peer should be deprived of the default reward for its valid staked role. + // As the app specific score in the cache and spam penalty in the spamRecords are updated at different times, we account for 0.1% error. + if math.Abs(expectedPenalty-score)/math.Max(expectedPenalty, score) < 0.001 { + return true + } + return false + }, 5*time.Second, 100*time.Millisecond) + + // the app specific score should now be updated in the cache. + score, updated, exists = appScoreCache.Get(peerID) // get the score from the cache. + require.True(t, exists) + require.True(t, updated.After(queryTime)) + require.True(t, math.Abs(expectedPenalty-score)/math.Max(expectedPenalty, score) < 0.001) + + // stop the registry. + cancel() + unittest.RequireCloseBefore(t, reg.Done(), 1*time.Second, "failed to stop GossipSubAppSpecificScoreRegistry") } func TestSpamRecord_With_UnknownIdentity(t *testing.T) { @@ -502,6 +539,8 @@ func newGossipSubAppSpecificScoreRegistry(t *testing.T, opts ...func(*scoring.Go appSpecificScoreCache := internal.NewAppSpecificScoreCache(100, unittest.Logger(), metrics.NewNoopHeroCacheMetricsFactory()) flowCfg, err := config.DefaultConfig() require.NoError(t, err) + // overrides the default values for testing purposes. + flowCfg.NetworkConfig.GossipSub.ScoringParameters.AppSpecificScore.ScoreTTL = 1 * time.Second validator := mockp2p.NewSubscriptionValidator(t) validator.On("Start", testifymock.Anything).Return().Maybe() From 65e5ef16438869bcab7d661c02eb79ffb215669e Mon Sep 17 00:00:00 2001 From: Yahya Hassanzadeh Date: Wed, 22 Nov 2023 09:46:42 -0800 Subject: [PATCH 33/67] lint fix --- network/p2p/scoring/registry_test.go | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/network/p2p/scoring/registry_test.go b/network/p2p/scoring/registry_test.go index 6decd1ed0d1..3bec1bde15d 100644 --- a/network/p2p/scoring/registry_test.go +++ b/network/p2p/scoring/registry_test.go @@ -123,10 +123,7 @@ func testPeerWithSpamRecord(t *testing.T, messageType p2pmsg.ControlMessageType, score := reg.AppSpecificScoreFunc()(peerID) // since the peer id does not have a spam record, the app specific score should be the max app specific reward, which // is the default reward for a staked peer that has valid subscriptions. - if scoring.MaxAppSpecificReward == score { - return true - } - return false + return scoring.MaxAppSpecificReward == score }, 5*time.Second, 100*time.Millisecond) // report a misbehavior for the peer id. @@ -150,10 +147,7 @@ func testPeerWithSpamRecord(t *testing.T, messageType p2pmsg.ControlMessageType, // this peer has a spam record, with no subscription penalty. Hence, the app specific score should only be the spam penalty, // and the peer should be deprived of the default reward for its valid staked role. // As the app specific score in the cache and spam penalty in the spamRecords are updated at different times, we account for 0.1% error. - if math.Abs(expectedPenalty-score)/math.Max(expectedPenalty, score) < 0.001 { - return true - } - return false + return math.Abs(expectedPenalty-score)/math.Max(expectedPenalty, score) < 0.001 }, 5*time.Second, 100*time.Millisecond) // the app specific score should now be updated in the cache. From 616a10f9122e23d85da5d70a75ee0c359659cf9c Mon Sep 17 00:00:00 2001 From: Yahya Hassanzadeh Date: Wed, 22 Nov 2023 09:47:29 -0800 Subject: [PATCH 34/67] lint fix --- network/p2p/scoring/registry_test.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/network/p2p/scoring/registry_test.go b/network/p2p/scoring/registry_test.go index 3bec1bde15d..6c4b0a183fc 100644 --- a/network/p2p/scoring/registry_test.go +++ b/network/p2p/scoring/registry_test.go @@ -54,10 +54,7 @@ func TestNoPenaltyRecord(t *testing.T) { score := reg.AppSpecificScoreFunc()(peerID) // since the peer id does not have a spam record, the app specific score should be the max app specific reward, which // is the default reward for a staked peer that has valid subscriptions. - if score == scoring.MaxAppSpecificReward { - return true - } - return false + return score == scoring.MaxAppSpecificReward }, 5*time.Second, 100*time.Millisecond) // still the spamRecords should not have the peer id (as there is no spam record for the peer id). From 17baf1ef0f50393a127d5945acd095e4e894e7dd Mon Sep 17 00:00:00 2001 From: Yahya Hassanzadeh Date: Wed, 22 Nov 2023 10:14:27 -0800 Subject: [PATCH 35/67] fixes insecure package tests --- insecure/corruptlibp2p/pubsub_adapter.go | 12 ++++- .../test/gossipsub/rpc_inspector/utils.go | 6 +-- .../validation_inspector_test.go | 51 +++++++++---------- 3 files changed, 38 insertions(+), 31 deletions(-) diff --git a/insecure/corruptlibp2p/pubsub_adapter.go b/insecure/corruptlibp2p/pubsub_adapter.go index 64975a18e3c..debac4da43b 100644 --- a/insecure/corruptlibp2p/pubsub_adapter.go +++ b/insecure/corruptlibp2p/pubsub_adapter.go @@ -14,6 +14,7 @@ import ( "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module/component" "github.com/onflow/flow-go/module/irrecoverable" + "github.com/onflow/flow-go/network/channels" "github.com/onflow/flow-go/network/p2p" "github.com/onflow/flow-go/network/p2p/p2plogging" "github.com/onflow/flow-go/utils/logging" @@ -108,6 +109,11 @@ func (c *CorruptGossipSubAdapter) ListPeers(topic string) []peer.ID { return c.gossipSub.ListPeers(topic) } +func (c *CorruptGossipSubAdapter) GetLocalMeshPeers(topic channels.Topic) []peer.ID { + // this method is a no-op in the corrupt gossipsub; as the corrupt gossipsub is solely used for testing, it does not come with a mesh tracer. + return []peer.ID{} +} + func (c *CorruptGossipSubAdapter) ActiveClustersChanged(lst flow.ChainIDList) { c.clusterChangeConsumer.ActiveClustersChanged(lst) } @@ -127,7 +133,11 @@ func (c *CorruptGossipSubAdapter) PeerScoreExposer() p2p.PeerScoreExposer { return c.peerScoreExposer } -func NewCorruptGossipSubAdapter(ctx context.Context, logger zerolog.Logger, h host.Host, cfg p2p.PubSubAdapterConfig, clusterChangeConsumer p2p.CollectionClusterChangesConsumer) (p2p.PubSubAdapter, *corrupt.GossipSubRouter, error) { +func NewCorruptGossipSubAdapter(ctx context.Context, + logger zerolog.Logger, + h host.Host, + cfg p2p.PubSubAdapterConfig, + clusterChangeConsumer p2p.CollectionClusterChangesConsumer) (p2p.PubSubAdapter, *corrupt.GossipSubRouter, error) { gossipSubConfig, ok := cfg.(*CorruptPubSubAdapterConfig) if !ok { return nil, nil, fmt.Errorf("invalid gossipsub config type: %T", cfg) diff --git a/insecure/integration/functional/test/gossipsub/rpc_inspector/utils.go b/insecure/integration/functional/test/gossipsub/rpc_inspector/utils.go index 555a06a6bba..fdbba188f45 100644 --- a/insecure/integration/functional/test/gossipsub/rpc_inspector/utils.go +++ b/insecure/integration/functional/test/gossipsub/rpc_inspector/utils.go @@ -65,9 +65,9 @@ func meshTracerFixture(flowConfig *config.FlowConfig, idProvider module.Identity IDProvider: idProvider, LoggerInterval: time.Second, HeroCacheMetricsFactory: metrics.NewNoopHeroCacheMetricsFactory(), - RpcSentTrackerCacheSize: flowConfig.NetworkConfig.GossipSubConfig.RPCSentTrackerCacheSize, - RpcSentTrackerWorkerQueueCacheSize: flowConfig.NetworkConfig.GossipSubConfig.RPCSentTrackerQueueCacheSize, - RpcSentTrackerNumOfWorkers: flowConfig.NetworkConfig.GossipSubConfig.RpcSentTrackerNumOfWorkers, + RpcSentTrackerCacheSize: flowConfig.NetworkConfig.GossipSub.RpcTracer.RPCSentTrackerCacheSize, + RpcSentTrackerWorkerQueueCacheSize: flowConfig.NetworkConfig.GossipSub.RpcTracer.RPCSentTrackerQueueCacheSize, + RpcSentTrackerNumOfWorkers: flowConfig.NetworkConfig.GossipSub.RpcTracer.RpcSentTrackerNumOfWorkers, } return tracer.NewGossipSubMeshTracer(meshTracerCfg) } diff --git a/insecure/integration/functional/test/gossipsub/rpc_inspector/validation_inspector_test.go b/insecure/integration/functional/test/gossipsub/rpc_inspector/validation_inspector_test.go index fcaca9d2ca5..f18a8aba6d8 100644 --- a/insecure/integration/functional/test/gossipsub/rpc_inspector/validation_inspector_test.go +++ b/insecure/integration/functional/test/gossipsub/rpc_inspector/validation_inspector_test.go @@ -44,7 +44,7 @@ func TestValidationInspector_InvalidTopicId_Detection(t *testing.T) { sporkID := unittest.IdentifierFixture() flowConfig, err := config.DefaultConfig() require.NoError(t, err) - inspectorConfig := flowConfig.NetworkConfig.GossipSubConfig.GossipSubRPCInspectorsConfig.GossipSubRPCValidationInspectorConfigs + inspectorConfig := flowConfig.NetworkConfig.GossipSub.RpcInspector.Validation messageCount := 100 inspectorConfig.NumberOfWorkers = 1 @@ -89,6 +89,7 @@ func TestValidationInspector_InvalidTopicId_Detection(t *testing.T) { distributor := mockp2p.NewGossipSubInspectorNotificationDistributor(t) p2ptest.MockInspectorNotificationDistributorReadyDoneAware(distributor) withExpectedNotificationDissemination(expectedNumOfTotalNotif, inspectDisseminatedNotifyFunc)(distributor, spammer) + meshTracer := meshTracerFixture(flowConfig, idProvider) topicProvider := newMockUpdatableTopicProvider() validationInspector, err := validation.NewControlMsgValidationInspector(&validation.InspectorParams{ @@ -112,7 +113,6 @@ func TestValidationInspector_InvalidTopicId_Detection(t *testing.T) { t.Name(), idProvider, p2ptest.WithRole(role), - p2ptest.WithGossipSubTracer(meshTracer), internal.WithCorruptGossipSub(corruptlibp2p.CorruptGossipSubFactory(), corruptlibp2p.CorruptGossipSubConfigFactoryWithInspector(corruptInspectorFunc))) idProvider.On("ByPeerID", victimNode.ID()).Return(&victimIdentity, true).Maybe() idProvider.On("ByPeerID", spammer.SpammerNode.ID()).Return(&spammer.SpammerId, true).Maybe() @@ -178,7 +178,7 @@ func TestValidationInspector_DuplicateTopicId_Detection(t *testing.T) { sporkID := unittest.IdentifierFixture() flowConfig, err := config.DefaultConfig() require.NoError(t, err) - inspectorConfig := flowConfig.NetworkConfig.GossipSubConfig.GossipSubRPCInspectorsConfig.GossipSubRPCValidationInspectorConfigs + inspectorConfig := flowConfig.NetworkConfig.GossipSub.RpcInspector.Validation inspectorConfig.NumberOfWorkers = 1 @@ -248,7 +248,6 @@ func TestValidationInspector_DuplicateTopicId_Detection(t *testing.T) { t.Name(), idProvider, p2ptest.WithRole(role), - p2ptest.WithGossipSubTracer(meshTracer), internal.WithCorruptGossipSub(corruptlibp2p.CorruptGossipSubFactory(), corruptlibp2p.CorruptGossipSubConfigFactoryWithInspector(corruptInspectorFunc))) idProvider.On("ByPeerID", victimNode.ID()).Return(&victimIdentity, true).Maybe() idProvider.On("ByPeerID", spammer.SpammerNode.ID()).Return(&spammer.SpammerId, true).Maybe() @@ -289,7 +288,7 @@ func TestValidationInspector_IHaveDuplicateMessageId_Detection(t *testing.T) { sporkID := unittest.IdentifierFixture() flowConfig, err := config.DefaultConfig() require.NoError(t, err) - inspectorConfig := flowConfig.NetworkConfig.GossipSubConfig.GossipSubRPCInspectorsConfig.GossipSubRPCValidationInspectorConfigs + inspectorConfig := flowConfig.NetworkConfig.GossipSub.RpcInspector.Validation inspectorConfig.NumberOfWorkers = 1 @@ -349,7 +348,6 @@ func TestValidationInspector_IHaveDuplicateMessageId_Detection(t *testing.T) { t.Name(), idProvider, p2ptest.WithRole(role), - p2ptest.WithGossipSubTracer(meshTracer), internal.WithCorruptGossipSub(corruptlibp2p.CorruptGossipSubFactory(), corruptlibp2p.CorruptGossipSubConfigFactoryWithInspector(corruptInspectorFunc))) idProvider.On("ByPeerID", victimNode.ID()).Return(&victimIdentity, true).Maybe() idProvider.On("ByPeerID", spammer.SpammerNode.ID()).Return(&spammer.SpammerId, true).Maybe() @@ -392,10 +390,10 @@ func TestValidationInspector_UnknownClusterId_Detection(t *testing.T) { sporkID := unittest.IdentifierFixture() flowConfig, err := config.DefaultConfig() require.NoError(t, err) - inspectorConfig := flowConfig.NetworkConfig.GossipSubConfig.GossipSubRPCInspectorsConfig.GossipSubRPCValidationInspectorConfigs + inspectorConfig := flowConfig.NetworkConfig.GossipSub.RpcInspector.Validation // set hard threshold to 0 so that in the case of invalid cluster ID // we force the inspector to return an error - inspectorConfig.ClusterPrefixHardThreshold = 0 + inspectorConfig.ClusterPrefixedMessage.HardThreshold = 0 inspectorConfig.NumberOfWorkers = 1 // SafetyThreshold < messageCount < HardThreshold ensures that the RPC message will be further inspected and topic IDs will be checked @@ -462,7 +460,6 @@ func TestValidationInspector_UnknownClusterId_Detection(t *testing.T) { t.Name(), idProvider, p2ptest.WithRole(role), - p2ptest.WithGossipSubTracer(meshTracer), internal.WithCorruptGossipSub(corruptlibp2p.CorruptGossipSubFactory(), corruptlibp2p.CorruptGossipSubConfigFactoryWithInspector(corruptInspectorFunc))) idProvider.On("ByPeerID", victimNode.ID()).Return(&victimIdentity, true).Maybe() idProvider.On("ByPeerID", spammer.SpammerNode.ID()).Return(&spammer.SpammerId, true).Times(4) @@ -504,8 +501,8 @@ func TestValidationInspector_ActiveClusterIdsNotSet_Graft_Detection(t *testing.T sporkID := unittest.IdentifierFixture() flowConfig, err := config.DefaultConfig() require.NoError(t, err) - inspectorConfig := flowConfig.NetworkConfig.GossipSubConfig.GossipSubRPCInspectorsConfig.GossipSubRPCValidationInspectorConfigs - inspectorConfig.ClusterPrefixHardThreshold = 5 + inspectorConfig := flowConfig.NetworkConfig.GossipSub.RpcInspector.Validation + inspectorConfig.ClusterPrefixedMessage.HardThreshold = 5 inspectorConfig.NumberOfWorkers = 1 controlMessageCount := int64(10) @@ -558,7 +555,6 @@ func TestValidationInspector_ActiveClusterIdsNotSet_Graft_Detection(t *testing.T t.Name(), idProvider, p2ptest.WithRole(role), - p2ptest.WithGossipSubTracer(meshTracer), internal.WithCorruptGossipSub(corruptlibp2p.CorruptGossipSubFactory(), corruptlibp2p.CorruptGossipSubConfigFactoryWithInspector(corruptInspectorFunc))) idProvider.On("ByPeerID", victimNode.ID()).Return(&victimIdentity, true).Maybe() idProvider.On("ByPeerID", spammer.SpammerNode.ID()).Return(&spammer.SpammerId, true).Maybe() @@ -594,8 +590,8 @@ func TestValidationInspector_ActiveClusterIdsNotSet_Prune_Detection(t *testing.T sporkID := unittest.IdentifierFixture() flowConfig, err := config.DefaultConfig() require.NoError(t, err) - inspectorConfig := flowConfig.NetworkConfig.GossipSubConfig.GossipSubRPCInspectorsConfig.GossipSubRPCValidationInspectorConfigs - inspectorConfig.ClusterPrefixHardThreshold = 5 + inspectorConfig := flowConfig.NetworkConfig.GossipSub.RpcInspector.Validation + inspectorConfig.ClusterPrefixedMessage.HardThreshold = 5 inspectorConfig.NumberOfWorkers = 1 controlMessageCount := int64(10) @@ -645,7 +641,6 @@ func TestValidationInspector_ActiveClusterIdsNotSet_Prune_Detection(t *testing.T t.Name(), idProvider, p2ptest.WithRole(role), - p2ptest.WithGossipSubTracer(meshTracer), internal.WithCorruptGossipSub(corruptlibp2p.CorruptGossipSubFactory(), corruptlibp2p.CorruptGossipSubConfigFactoryWithInspector(corruptInspectorFunc))) idProvider.On("ByPeerID", victimNode.ID()).Return(&victimIdentity, true).Maybe() idProvider.On("ByPeerID", spammer.SpammerNode.ID()).Return(&spammer.SpammerId, true).Maybe() @@ -672,18 +667,18 @@ func TestValidationInspector_ActiveClusterIdsNotSet_Prune_Detection(t *testing.T unittest.RequireCloseBefore(t, done, 5*time.Second, "failed to inspect RPC messages on time") } -// TestValidationInspector_UnstakedNode_Detection ensures that RPC control message inspector disseminates an invalid control message notification when an unstaked peer +// TestValidationInspector_Unstaked_Node_Detection ensures that RPC control message inspector disseminates an invalid control message notification when an unstaked peer // sends a control message for a cluster prefixed topic. -func TestValidationInspector_UnstakedNode_Detection(t *testing.T) { +func TestValidationInspector_Unstaked_Node_Detection(t *testing.T) { t.Parallel() role := flow.RoleConsensus sporkID := unittest.IdentifierFixture() flowConfig, err := config.DefaultConfig() require.NoError(t, err) - inspectorConfig := flowConfig.NetworkConfig.GossipSubConfig.GossipSubRPCInspectorsConfig.GossipSubRPCValidationInspectorConfigs + inspectorConfig := flowConfig.NetworkConfig.GossipSub.RpcInspector.Validation // set hard threshold to 0 so that in the case of invalid cluster ID // we force the inspector to return an error - inspectorConfig.ClusterPrefixHardThreshold = 0 + inspectorConfig.ClusterPrefixedMessage.HardThreshold = 0 inspectorConfig.NumberOfWorkers = 1 // SafetyThreshold < messageCount < HardThreshold ensures that the RPC message will be further inspected and topic IDs will be checked @@ -738,7 +733,6 @@ func TestValidationInspector_UnstakedNode_Detection(t *testing.T) { t.Name(), idProvider, p2ptest.WithRole(role), - p2ptest.WithGossipSubTracer(meshTracer), internal.WithCorruptGossipSub(corruptlibp2p.CorruptGossipSubFactory(), corruptlibp2p.CorruptGossipSubConfigFactoryWithInspector(corruptInspectorFunc))) idProvider.On("ByPeerID", victimNode.ID()).Return(&victimIdentity, true).Maybe() idProvider.On("ByPeerID", spammer.SpammerNode.ID()).Return(&spammer.SpammerId, true).Maybe() @@ -780,11 +774,11 @@ func TestValidationInspector_InspectIWants_CacheMissThreshold(t *testing.T) { // create our RPC validation inspector flowConfig, err := config.DefaultConfig() require.NoError(t, err) - inspectorConfig := flowConfig.NetworkConfig.GossipSubConfig.GossipSubRPCInspectorsConfig.GossipSubRPCValidationInspectorConfigs + inspectorConfig := flowConfig.NetworkConfig.GossipSub.RpcInspector.Validation // force all cache miss checks - inspectorConfig.IWantRPCInspectionConfig.CacheMissCheckSize = 1 + inspectorConfig.IWant.CacheMissCheckSize = 1 inspectorConfig.NumberOfWorkers = 1 - inspectorConfig.IWantRPCInspectionConfig.CacheMissThreshold = .5 // set cache miss threshold to 50% + inspectorConfig.IWant.CacheMissThreshold = .5 // set cache miss threshold to 50% messageCount := 1 controlMessageCount := int64(1) cacheMissThresholdNotifCount := atomic.NewUint64(0) @@ -840,7 +834,6 @@ func TestValidationInspector_InspectIWants_CacheMissThreshold(t *testing.T) { t.Name(), idProvider, p2ptest.WithRole(role), - p2ptest.WithGossipSubTracer(meshTracer), internal.WithCorruptGossipSub(corruptlibp2p.CorruptGossipSubFactory(), corruptlibp2p.CorruptGossipSubConfigFactoryWithInspector(corruptInspectorFunc))) idProvider.On("ByPeerID", victimNode.ID()).Return(&victimIdentity, true).Maybe() idProvider.On("ByPeerID", spammer.SpammerNode.ID()).Return(&spammer.SpammerId, true).Maybe() @@ -885,7 +878,7 @@ func TestValidationInspector_InspectRpcPublishMessages(t *testing.T) { // create our RPC validation inspector flowConfig, err := config.DefaultConfig() require.NoError(t, err) - inspectorConfig := flowConfig.NetworkConfig.GossipSubConfig.GossipSubRPCInspectorsConfig.GossipSubRPCValidationInspectorConfigs + inspectorConfig := flowConfig.NetworkConfig.GossipSub.RpcInspector.Validation inspectorConfig.NumberOfWorkers = 1 idProvider := mock.NewIdentityProvider(t) @@ -978,7 +971,7 @@ func TestValidationInspector_InspectRpcPublishMessages(t *testing.T) { topicProvider.UpdateTopics(topics) // after 7 errors encountered disseminate a notification - inspectorConfig.RpcMessageErrorThreshold = 6 + inspectorConfig.MessageErrorThreshold = 6 require.NoError(t, err) corruptInspectorFunc := corruptlibp2p.CorruptInspectorFunc(validationInspector) @@ -987,7 +980,6 @@ func TestValidationInspector_InspectRpcPublishMessages(t *testing.T) { t.Name(), idProvider, p2ptest.WithRole(role), - p2ptest.WithGossipSubTracer(meshTracer), internal.WithCorruptGossipSub(corruptlibp2p.CorruptGossipSubFactory(), corruptlibp2p.CorruptGossipSubConfigFactoryWithInspector(corruptInspectorFunc))) idProvider.On("ByPeerID", victimNode.ID()).Return(&victimIdentity, true).Maybe() idProvider.On("ByPeerID", spammer.SpammerNode.ID()).Return(&spammer.SpammerId, true).Maybe() @@ -1031,11 +1023,16 @@ func TestGossipSubSpamMitigationIntegration(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) signalerCtx := irrecoverable.NewMockSignalerContext(t, ctx) + cfg, err := config.DefaultConfig() + require.NoError(t, err) + // set the scoring parameters to be more aggressive to speed up the test + cfg.NetworkConfig.GossipSub.ScoringParameters.AppSpecificScore.ScoreTTL = 100 * time.Millisecond victimNode, victimId := p2ptest.NodeFixture(t, sporkID, t.Name(), idProvider, p2ptest.WithRole(flow.RoleConsensus), + p2ptest.OverrideFlowConfig(cfg), p2ptest.EnablePeerScoringWithOverride(p2p.PeerScoringConfigNoOverride)) ids := flow.IdentityList{&victimId, &spammer.SpammerId} From 42b664c660ee29327c1cd755c8d266a9abc23be2 Mon Sep 17 00:00:00 2001 From: Yahya Hassanzadeh Date: Wed, 22 Nov 2023 10:24:45 -0800 Subject: [PATCH 36/67] fixes build issues --- follower/follower_builder.go | 23 ++----------------- .../p2pbuilder/gossipsub/gossipSubBuilder.go | 6 ----- 2 files changed, 2 insertions(+), 27 deletions(-) diff --git a/follower/follower_builder.go b/follower/follower_builder.go index 58aa4eb5b74..4c894d3bbf1 100644 --- a/follower/follower_builder.go +++ b/follower/follower_builder.go @@ -53,7 +53,6 @@ import ( "github.com/onflow/flow-go/network/p2p/p2plogging" "github.com/onflow/flow-go/network/p2p/p2pnet" "github.com/onflow/flow-go/network/p2p/subscription" - "github.com/onflow/flow-go/network/p2p/tracer" "github.com/onflow/flow-go/network/p2p/translator" "github.com/onflow/flow-go/network/p2p/unicast/protocols" "github.com/onflow/flow-go/network/p2p/utils" @@ -573,21 +572,9 @@ func (builder *FollowerServiceBuilder) initPublicLibp2pNode(networkKey crypto.Pr pis = append(pis, pi) } - meshTracerCfg := &tracer.GossipSubMeshTracerConfig{ - Logger: builder.Logger, - Metrics: builder.Metrics.Network, - IDProvider: builder.IdentityProvider, - LoggerInterval: builder.FlowConfig.NetworkConfig.GossipSubConfig.LocalMeshLogInterval, - RpcSentTrackerCacheSize: builder.FlowConfig.NetworkConfig.GossipSubConfig.RPCSentTrackerCacheSize, - RpcSentTrackerWorkerQueueCacheSize: builder.FlowConfig.NetworkConfig.GossipSubConfig.RPCSentTrackerQueueCacheSize, - RpcSentTrackerNumOfWorkers: builder.FlowConfig.NetworkConfig.GossipSubConfig.RpcSentTrackerNumOfWorkers, - HeroCacheMetricsFactory: builder.HeroCacheMetricsFactory(), - NetworkingType: network.PublicNetwork, - } - meshTracer := tracer.NewGossipSubMeshTracer(meshTracerCfg) - node, err := p2pbuilder.NewNodeBuilder( builder.Logger, + &builder.FlowConfig.NetworkConfig.GossipSub, &p2pconfig.MetricsConfig{ HeroCacheFactory: builder.HeroCacheMetricsFactory(), Metrics: builder.Metrics.Network, @@ -598,14 +585,11 @@ func (builder *FollowerServiceBuilder) initPublicLibp2pNode(networkKey crypto.Pr builder.SporkID, builder.IdentityProvider, &builder.FlowConfig.NetworkConfig.ResourceManager, - &builder.FlowConfig.NetworkConfig.GossipSubConfig.GossipSubRPCInspectorsConfig, p2pconfig.PeerManagerDisableConfig(), // disable peer manager for follower - &builder.FlowConfig.NetworkConfig.GossipSubConfig.SubscriptionProvider, &p2p.DisallowListCacheConfig{ MaxSize: builder.FlowConfig.NetworkConfig.DisallowListNotificationCacheSize, Metrics: metrics.DisallowListCacheMetricsFactory(builder.HeroCacheMetricsFactory(), network.PublicNetwork), }, - meshTracer, &p2pconfig.UnicastConfig{ UnicastConfig: builder.FlowConfig.NetworkConfig.UnicastConfig, }). @@ -621,10 +605,7 @@ func (builder *FollowerServiceBuilder) initPublicLibp2pNode(networkKey crypto.Pr p2pdht.AsClient(), dht.BootstrapPeers(pis...), ) - }). - SetGossipSubTracer(meshTracer). - SetGossipSubScoreTracerInterval(builder.FlowConfig.NetworkConfig.GossipSubConfig.ScoreTracerInterval). - Build() + }).Build() if err != nil { return nil, fmt.Errorf("could not build public libp2p node: %w", err) } diff --git a/network/p2p/p2pbuilder/gossipsub/gossipSubBuilder.go b/network/p2p/p2pbuilder/gossipsub/gossipSubBuilder.go index 74094f1ce97..8afafa96110 100644 --- a/network/p2p/p2pbuilder/gossipsub/gossipSubBuilder.go +++ b/network/p2p/p2pbuilder/gossipsub/gossipSubBuilder.go @@ -121,12 +121,6 @@ func (g *Builder) EnableGossipSubScoringWithOverride(override *p2p.PeerScoringCo } } -// SetGossipSubTracer sets the gossipsub tracer of the builder. -// If the gossipsub tracer has already been set, a fatal error is logged. -func (g *Builder) SetGossipSubTracer(gossipSubTracer p2p.PubSubTracer) { - -} - // SetRoutingSystem sets the routing system of the builder. // If the routing system has already been set, a fatal error is logged. func (g *Builder) SetRoutingSystem(routingSystem routing.Routing) { From e2e3696d44d3fb77ef02004aeedcb08b6dd8aa36 Mon Sep 17 00:00:00 2001 From: Yahya Hassanzadeh Date: Wed, 22 Nov 2023 13:50:57 -0800 Subject: [PATCH 37/67] fixes tests with insecure package --- .../test/gossipsub/scoring/ihave_spam_test.go | 4 +- .../test/gossipsub/scoring/scoring_test.go | 14 ++- .../p2pbuilder/gossipsub/gossipSubBuilder.go | 22 +++-- network/p2p/scoring/registry_test.go | 90 +++++++++++++------ 4 files changed, 83 insertions(+), 47 deletions(-) diff --git a/insecure/integration/functional/test/gossipsub/scoring/ihave_spam_test.go b/insecure/integration/functional/test/gossipsub/scoring/ihave_spam_test.go index f450579259a..62dd6549952 100644 --- a/insecure/integration/functional/test/gossipsub/scoring/ihave_spam_test.go +++ b/insecure/integration/functional/test/gossipsub/scoring/ihave_spam_test.go @@ -67,6 +67,7 @@ func TestGossipSubIHaveBrokenPromises_Below_Threshold(t *testing.T) { require.NoError(t, err) // we override the decay interval to 1 second so that the score is updated within 1 second intervals. conf.NetworkConfig.GossipSub.ScoringParameters.DecayInterval = 1 * time.Second + conf.NetworkConfig.GossipSub.RpcTracer.ScoreTracerInterval = 1 * time.Second victimNode, victimIdentity := p2ptest.NodeFixture( t, sporkId, @@ -74,7 +75,6 @@ func TestGossipSubIHaveBrokenPromises_Below_Threshold(t *testing.T) { idProvider, p2ptest.WithRole(role), p2ptest.OverrideFlowConfig(conf), - p2ptest.WithPeerScoreTracerInterval(1*time.Second), p2ptest.EnablePeerScoringWithOverride(&p2p.PeerScoringConfigOverride{ TopicScoreParams: map[channels.Topic]*pubsub.TopicScoreParams{ blockTopic: blockTopicOverrideParams, @@ -198,6 +198,7 @@ func TestGossipSubIHaveBrokenPromises_Above_Threshold(t *testing.T) { conf.NetworkConfig.GossipSub.RpcInspector.Validation.IHave.MaxMessageIDSampleSize = 10000 // we override the decay interval to 1 second so that the score is updated within 1 second intervals. conf.NetworkConfig.GossipSub.ScoringParameters.DecayInterval = 1 * time.Second + conf.NetworkConfig.GossipSub.RpcTracer.ScoreTracerInterval = 1 * time.Second ctx, cancel := context.WithCancel(context.Background()) signalerCtx := irrecoverable.NewMockSignalerContext(t, ctx) @@ -215,7 +216,6 @@ func TestGossipSubIHaveBrokenPromises_Above_Threshold(t *testing.T) { idProvider, p2ptest.OverrideFlowConfig(conf), p2ptest.WithRole(role), - p2ptest.WithPeerScoreTracerInterval(1*time.Second), p2ptest.EnablePeerScoringWithOverride(&p2p.PeerScoringConfigOverride{ TopicScoreParams: map[channels.Topic]*pubsub.TopicScoreParams{ blockTopic: blockTopicOverrideParams, diff --git a/insecure/integration/functional/test/gossipsub/scoring/scoring_test.go b/insecure/integration/functional/test/gossipsub/scoring/scoring_test.go index 3948de3264b..b5e3e526fc7 100644 --- a/insecure/integration/functional/test/gossipsub/scoring/scoring_test.go +++ b/insecure/integration/functional/test/gossipsub/scoring/scoring_test.go @@ -107,13 +107,17 @@ func testGossipSubInvalidMessageDeliveryScoring(t *testing.T, spamMsgFactory fun ctx, cancel := context.WithCancel(context.Background()) signalerCtx := irrecoverable.NewMockSignalerContext(t, ctx) + cfg, err := config.DefaultConfig() + require.NoError(t, err) + // we override the decay interval to 1 second so that the score is updated within 1 second intervals. + cfg.NetworkConfig.GossipSub.RpcTracer.ScoreTracerInterval = 1 * time.Second victimNode, victimIdentity := p2ptest.NodeFixture( t, sporkId, t.Name(), idProvider, p2ptest.WithRole(role), - p2ptest.WithPeerScoreTracerInterval(1*time.Second), + p2ptest.OverrideFlowConfig(cfg), p2ptest.EnablePeerScoringWithOverride(p2p.PeerScoringConfigNoOverride), ) @@ -213,13 +217,14 @@ func TestGossipSubMeshDeliveryScoring_UnderDelivery_SingleTopic(t *testing.T) { require.NoError(t, err) // we override the decay interval to 1 second so that the score is updated within 1 second intervals. conf.NetworkConfig.GossipSub.ScoringParameters.DecayInterval = 1 * time.Second + conf.NetworkConfig.GossipSub.RpcTracer.ScoreTracerInterval = 1 * time.Second thisNode, thisId := p2ptest.NodeFixture( // this node is the one that will be penalizing the under-performer node. t, sporkId, t.Name(), idProvider, p2ptest.WithRole(role), - p2ptest.WithPeerScoreTracerInterval(1*time.Second), + p2ptest.OverrideFlowConfig(conf), p2ptest.EnablePeerScoringWithOverride( &p2p.PeerScoringConfigOverride{ TopicScoreParams: map[channels.Topic]*pubsub.TopicScoreParams{ @@ -322,13 +327,14 @@ func TestGossipSubMeshDeliveryScoring_UnderDelivery_TwoTopics(t *testing.T) { require.NoError(t, err) // we override the decay interval to 1 second so that the score is updated within 1 second intervals. conf.NetworkConfig.GossipSub.ScoringParameters.DecayInterval = 1 * time.Second + conf.NetworkConfig.GossipSub.RpcTracer.ScoreTracerInterval = 1 * time.Second thisNode, thisId := p2ptest.NodeFixture( // this node is the one that will be penalizing the under-performer node. t, sporkId, t.Name(), idProvider, p2ptest.WithRole(role), - p2ptest.WithPeerScoreTracerInterval(1*time.Second), + p2ptest.OverrideFlowConfig(conf), p2ptest.EnablePeerScoringWithOverride( &p2p.PeerScoringConfigOverride{ TopicScoreParams: map[channels.Topic]*pubsub.TopicScoreParams{ @@ -433,6 +439,7 @@ func TestGossipSubMeshDeliveryScoring_Replay_Will_Not_Counted(t *testing.T) { require.NoError(t, err) // we override the decay interval to 1 second so that the score is updated within 1 second intervals. conf.NetworkConfig.GossipSub.ScoringParameters.DecayInterval = 1 * time.Second + conf.NetworkConfig.GossipSub.RpcTracer.ScoreTracerInterval = 1 * time.Second blockTopicOverrideParams := scoring.DefaultTopicScoreParams() blockTopicOverrideParams.MeshMessageDeliveriesActivation = 1 * time.Second // we start observing the mesh message deliveries after 1 second of the node startup. thisNode, thisId := p2ptest.NodeFixture( // this node is the one that will be penalizing the under-performer node. @@ -442,7 +449,6 @@ func TestGossipSubMeshDeliveryScoring_Replay_Will_Not_Counted(t *testing.T) { idProvider, p2ptest.WithRole(role), p2ptest.OverrideFlowConfig(conf), - p2ptest.WithPeerScoreTracerInterval(1*time.Second), p2ptest.EnablePeerScoringWithOverride( &p2p.PeerScoringConfigOverride{ TopicScoreParams: map[channels.Topic]*pubsub.TopicScoreParams{ diff --git a/network/p2p/p2pbuilder/gossipsub/gossipSubBuilder.go b/network/p2p/p2pbuilder/gossipsub/gossipSubBuilder.go index 8afafa96110..56f3bc24919 100644 --- a/network/p2p/p2pbuilder/gossipsub/gossipSubBuilder.go +++ b/network/p2p/p2pbuilder/gossipsub/gossipSubBuilder.go @@ -3,7 +3,6 @@ package gossipsubbuilder import ( "context" "fmt" - "time" pubsub "github.com/libp2p/go-libp2p-pubsub" "github.com/libp2p/go-libp2p/core/host" @@ -32,15 +31,14 @@ import ( // The Builder struct is used to configure and create a new GossipSubParameters pubsub system. type Builder struct { - networkType network.NetworkingType - sporkId flow.Identifier - logger zerolog.Logger - metricsCfg *p2pconfig.MetricsConfig - h host.Host - subscriptionFilter pubsub.SubscriptionFilter - gossipSubFactory p2p.GossipSubFactoryFunc - gossipSubConfigFunc p2p.GossipSubAdapterConfigFunc - gossipSubScoreTracerInterval time.Duration // the interval at which the gossipsub score tracer logs the peer scores. + networkType network.NetworkingType + sporkId flow.Identifier + logger zerolog.Logger + metricsCfg *p2pconfig.MetricsConfig + h host.Host + subscriptionFilter pubsub.SubscriptionFilter + gossipSubFactory p2p.GossipSubFactoryFunc + gossipSubConfigFunc p2p.GossipSubAdapterConfigFunc // gossipSubTracer is a callback interface that is called by the gossipsub implementation upon // certain events. Currently, we use it to log and observe the local mesh of the node. gossipSubTracer p2p.PubSubTracer @@ -328,8 +326,8 @@ func (g *Builder) Build(ctx irrecoverable.SignalerContext) (p2p.PubSubAdapter, e } gossipSubConfigs.WithScoreOption(scoreOpt) - if g.gossipSubScoreTracerInterval > 0 { - scoreTracer = tracer.NewGossipSubScoreTracer(g.logger, g.idProvider, g.metricsCfg.Metrics, g.gossipSubScoreTracerInterval) + if g.gossipSubCfg.RpcTracer.ScoreTracerInterval > 0 { + scoreTracer = tracer.NewGossipSubScoreTracer(g.logger, g.idProvider, g.metricsCfg.Metrics, g.gossipSubCfg.RpcTracer.ScoreTracerInterval) gossipSubConfigs.WithScoreTracer(scoreTracer) } } else { diff --git a/network/p2p/scoring/registry_test.go b/network/p2p/scoring/registry_test.go index 6c4b0a183fc..45cea8669bf 100644 --- a/network/p2p/scoring/registry_test.go +++ b/network/p2p/scoring/registry_test.go @@ -25,10 +25,11 @@ import ( "github.com/onflow/flow-go/utils/unittest" ) -// TestNoPenaltyRecord tests that if there is no penalty record for a peer id, the app specific score should be the max -// app specific reward. This is the default reward for a staked peer that has valid subscriptions and has not been -// penalized. -func TestNoPenaltyRecord(t *testing.T) { +// TestScoreRegistry_FreshStart tests the app specific score computation of the node when there is no spam record for the peer id upon fresh start of the registry. +// It tests the state that a staked peer with a valid role and valid subscriptions has no spam records; hence it should "eventually" be rewarded with the default reward +// for its GossipSub app specific score. The "eventually" comes from the fact that the app specific score is updated asynchronously in the cache, and the cache is +// updated when the app specific score function is called by GossipSub. +func TestScoreRegistry_FreshStart(t *testing.T) { peerID := peer.ID("peer-1") reg, spamRecords, appScoreCache := newGossipSubAppSpecificScoreRegistry(t, @@ -71,29 +72,30 @@ func TestNoPenaltyRecord(t *testing.T) { unittest.RequireCloseBefore(t, reg.Done(), 1*time.Second, "failed to stop GossipSubAppSpecificScoreRegistry") } -// TestPeerWithSpamRecord tests the app specific penalty computation of the node when there is a spam record for the peer id. +// TestScoreRegistry_PeerWithSpamRecord tests the app specific penalty computation of the node when there is a spam record for the peer id. // It tests the state that a staked peer with a valid role and valid subscriptions has spam records. -// Since the peer has spam records, it should be deprived of the default reward for its staked role, and only have the -// penalty value as the app specific score. -func TestPeerWithSpamRecord(t *testing.T) { +// Since the peer has spam records, it should "eventually" be deprived of the default reward for its staked role, and only have the +// penalty value as the app specific score. The "eventually" comes from the fact that the app specific score is updated asynchronously +// in the cache, and the cache is updated when the app specific score function is called by GossipSub. +func TestScoreRegistry_PeerWithSpamRecord(t *testing.T) { t.Run("graft", func(t *testing.T) { - testPeerWithSpamRecord(t, p2pmsg.CtrlMsgGraft, penaltyValueFixtures().Graft) + testScoreRegistryPeerWithSpamRecord(t, p2pmsg.CtrlMsgGraft, penaltyValueFixtures().Graft) }) t.Run("prune", func(t *testing.T) { - testPeerWithSpamRecord(t, p2pmsg.CtrlMsgPrune, penaltyValueFixtures().Prune) + testScoreRegistryPeerWithSpamRecord(t, p2pmsg.CtrlMsgPrune, penaltyValueFixtures().Prune) }) t.Run("ihave", func(t *testing.T) { - testPeerWithSpamRecord(t, p2pmsg.CtrlMsgIHave, penaltyValueFixtures().IHave) + testScoreRegistryPeerWithSpamRecord(t, p2pmsg.CtrlMsgIHave, penaltyValueFixtures().IHave) }) t.Run("iwant", func(t *testing.T) { - testPeerWithSpamRecord(t, p2pmsg.CtrlMsgIWant, penaltyValueFixtures().IWant) + testScoreRegistryPeerWithSpamRecord(t, p2pmsg.CtrlMsgIWant, penaltyValueFixtures().IWant) }) t.Run("RpcPublishMessage", func(t *testing.T) { - testPeerWithSpamRecord(t, p2pmsg.RpcPublishMessage, penaltyValueFixtures().RpcPublishMessage) + testScoreRegistryPeerWithSpamRecord(t, p2pmsg.RpcPublishMessage, penaltyValueFixtures().RpcPublishMessage) }) } -func testPeerWithSpamRecord(t *testing.T, messageType p2pmsg.ControlMessageType, expectedPenalty float64) { +func testScoreRegistryPeerWithSpamRecord(t *testing.T, messageType p2pmsg.ControlMessageType, expectedPenalty float64) { peerID := peer.ID("peer-1") reg, spamRecords, appScoreCache := newGossipSubAppSpecificScoreRegistry( @@ -180,18 +182,33 @@ func TestSpamRecord_With_UnknownIdentity(t *testing.T) { // the peer id has an unknown identity. func testSpamRecordWithUnknownIdentity(t *testing.T, messageType p2pmsg.ControlMessageType, expectedPenalty float64) { peerID := peer.ID("peer-1") - reg, spamRecords, _ := newGossipSubAppSpecificScoreRegistry( + reg, spamRecords, appScoreCache := newGossipSubAppSpecificScoreRegistry( t, withUnknownIdentity(peerID), withValidSubscriptions(peerID)) - // initially, the spamRecords should not have the peer id. - assert.False(t, spamRecords.Has(peerID)) + // starts the registry. + ctx, cancel := context.WithCancel(context.Background()) + signalerCtx := irrecoverable.NewMockSignalerContext(t, ctx) + reg.Start(signalerCtx) + unittest.RequireCloseBefore(t, reg.Ready(), 1*time.Second, "failed to start GossipSubAppSpecificScoreRegistry") - // peer does not have spam record, but has an unknown identity. Hence, the app specific score should be the staking penalty. - score := reg.AppSpecificScoreFunc()(peerID) - require.Equal(t, scoring.DefaultUnknownIdentityPenalty, score) + // initially, the spamRecords should not have the peer id; ; also the app specific score record should not be in the cache. + require.False(t, spamRecords.Has(peerID)) + score, updated, exists := appScoreCache.Get(peerID) // get the score from the cache. + require.False(t, exists) + require.Equal(t, time.Time{}, updated) + require.Equal(t, float64(0), score) + + // eventually the app specific score should be updated in the cache to the penalty value for unknown identity. + require.Eventually(t, func() bool { + // calling the app specific score function when there is no app specific score in the cache should eventually update the cache. + score := reg.AppSpecificScoreFunc()(peerID) + // peer does not have spam record, but has an unknown identity. Hence, the app specific score should be the staking penalty. + return scoring.DefaultUnknownIdentityPenalty == score + }, 5*time.Second, 100*time.Millisecond) + // queryTime := time.Now() // report a misbehavior for the peer id. reg.OnInvalidControlMessageNotification(&p2p.InvCtrlMsgNotif{ PeerID: peerID, @@ -200,15 +217,30 @@ func testSpamRecordWithUnknownIdentity(t *testing.T, messageType p2pmsg.ControlM // the penalty should now be updated. record, err, ok := spamRecords.Get(peerID) // get the record from the spamRecords. - assert.True(t, ok) - assert.NoError(t, err) - assert.Less(t, math.Abs(expectedPenalty-record.Penalty), 10e-3) // penalty should be updated to -10, we account for decay. - assert.Equal(t, scoring.InitAppScoreRecordState().Decay, record.Decay) // decay should be initialized to the initial state. + require.True(t, ok) + require.NoError(t, err) + require.Less(t, math.Abs(expectedPenalty-record.Penalty), 10e-3) // penalty should be updated to -10, we account for decay. + require.Equal(t, scoring.InitAppScoreRecordState().Decay, record.Decay) // decay should be initialized to the initial state. - // the peer has spam record as well as an unknown identity. Hence, the app specific score should be the spam penalty - // and the staking penalty. - score = reg.AppSpecificScoreFunc()(peerID) - assert.Less(t, math.Abs(expectedPenalty+scoring.DefaultUnknownIdentityPenalty-score), 10e-3) + // eventually, the app specific score should be updated in the cache. + require.Eventually(t, func() bool { + // calling the app specific score function when there is no app specific score in the cache should eventually update the cache. + score := reg.AppSpecificScoreFunc()(peerID) + // the peer has spam record as well as an unknown identity. Hence, the app specific score should be the spam penalty + // and the staking penalty. + // As the app specific score in the cache and spam penalty in the spamRecords are updated at different times, we account for 0.1% error. + return math.Abs(expectedPenalty+scoring.DefaultUnknownIdentityPenalty-score)/math.Max(expectedPenalty+scoring.DefaultUnknownIdentityPenalty, score) < 0.001 + }, 5*time.Second, 100*time.Millisecond) + + // the app specific score should now be updated in the cache. + score, updated, exists = appScoreCache.Get(peerID) // get the score from the cache. + require.True(t, exists) + // require.True(t, updated.After(queryTime)) + require.True(t, math.Abs(expectedPenalty+scoring.DefaultUnknownIdentityPenalty-score)/math.Max(expectedPenalty+scoring.DefaultUnknownIdentityPenalty, score) < 0.001) + + // stop the registry. + cancel() + unittest.RequireCloseBefore(t, reg.Done(), 1*time.Second, "failed to stop GossipSubAppSpecificScoreRegistry") } func TestSpamRecord_With_SubscriptionPenalty(t *testing.T) { @@ -531,7 +563,7 @@ func newGossipSubAppSpecificScoreRegistry(t *testing.T, opts ...func(*scoring.Go flowCfg, err := config.DefaultConfig() require.NoError(t, err) // overrides the default values for testing purposes. - flowCfg.NetworkConfig.GossipSub.ScoringParameters.AppSpecificScore.ScoreTTL = 1 * time.Second + flowCfg.NetworkConfig.GossipSub.ScoringParameters.AppSpecificScore.ScoreTTL = 100 * time.Second validator := mockp2p.NewSubscriptionValidator(t) validator.On("Start", testifymock.Anything).Return().Maybe() From a91198292926bd80d46cab8ddb07f9b293b669dd Mon Sep 17 00:00:00 2001 From: Yahya Hassanzadeh Date: Wed, 22 Nov 2023 14:39:24 -0800 Subject: [PATCH 38/67] fixes TestScoreRegistry_SpamRecordWithSubscriptionPenalty --- .../test/gossipsub/scoring/scoring_test.go | 3 +- ...ip_sub_application_specific_score_cache.go | 76 +++++++++ network/p2p/mock/gossip_sub_builder.go | 12 -- network/p2p/mock/lib_p2_p_node.go | 16 ++ network/p2p/mock/node_builder.go | 34 ---- network/p2p/mock/pub_sub.go | 18 +++ network/p2p/mock/pub_sub_adapter.go | 18 +++ network/p2p/mock/pub_sub_tracer.go | 18 +++ network/p2p/scoring/registry_test.go | 153 +++++++++++++----- 9 files changed, 264 insertions(+), 84 deletions(-) create mode 100644 network/p2p/mock/gossip_sub_application_specific_score_cache.go diff --git a/insecure/integration/functional/test/gossipsub/scoring/scoring_test.go b/insecure/integration/functional/test/gossipsub/scoring/scoring_test.go index b5e3e526fc7..0c347986ec1 100644 --- a/insecure/integration/functional/test/gossipsub/scoring/scoring_test.go +++ b/insecure/integration/functional/test/gossipsub/scoring/scoring_test.go @@ -175,7 +175,8 @@ func testGossipSubInvalidMessageDeliveryScoring(t *testing.T, spamMsgFactory fun blkTopicSnapshot, ok := topicsSnapshot[blockTopic.String()] require.True(t, ok) - // ensure that the topic snapshot of the spammer contains a record of at least (60%) of the spam messages sent. The 60% is to account for the messages that were delivered before the score was updated, after the spammer is PRUNED, as well as to account for decay. + // ensure that the topic snapshot of the spammer contains a record of at least (60%) of the spam messages sent. The 60% is to account for the messages that were + // delivered before the score was updated, after the spammer is PRUNED, as well as to account for decay. require.True(t, blkTopicSnapshot.InvalidMessageDeliveries > 0.6*float64(totalSpamMessages), "invalid message deliveries must be greater than %f. invalid message deliveries: %f", diff --git a/network/p2p/mock/gossip_sub_application_specific_score_cache.go b/network/p2p/mock/gossip_sub_application_specific_score_cache.go new file mode 100644 index 00000000000..68260079f1a --- /dev/null +++ b/network/p2p/mock/gossip_sub_application_specific_score_cache.go @@ -0,0 +1,76 @@ +// Code generated by mockery v2.21.4. DO NOT EDIT. + +package mockp2p + +import ( + mock "github.com/stretchr/testify/mock" + + peer "github.com/libp2p/go-libp2p/core/peer" + + time "time" +) + +// GossipSubApplicationSpecificScoreCache is an autogenerated mock type for the GossipSubApplicationSpecificScoreCache type +type GossipSubApplicationSpecificScoreCache struct { + mock.Mock +} + +// Add provides a mock function with given fields: peerID, score, _a2 +func (_m *GossipSubApplicationSpecificScoreCache) Add(peerID peer.ID, score float64, _a2 time.Time) error { + ret := _m.Called(peerID, score, _a2) + + var r0 error + if rf, ok := ret.Get(0).(func(peer.ID, float64, time.Time) error); ok { + r0 = rf(peerID, score, _a2) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// Get provides a mock function with given fields: peerID +func (_m *GossipSubApplicationSpecificScoreCache) Get(peerID peer.ID) (float64, time.Time, bool) { + ret := _m.Called(peerID) + + var r0 float64 + var r1 time.Time + var r2 bool + if rf, ok := ret.Get(0).(func(peer.ID) (float64, time.Time, bool)); ok { + return rf(peerID) + } + if rf, ok := ret.Get(0).(func(peer.ID) float64); ok { + r0 = rf(peerID) + } else { + r0 = ret.Get(0).(float64) + } + + if rf, ok := ret.Get(1).(func(peer.ID) time.Time); ok { + r1 = rf(peerID) + } else { + r1 = ret.Get(1).(time.Time) + } + + if rf, ok := ret.Get(2).(func(peer.ID) bool); ok { + r2 = rf(peerID) + } else { + r2 = ret.Get(2).(bool) + } + + return r0, r1, r2 +} + +type mockConstructorTestingTNewGossipSubApplicationSpecificScoreCache interface { + mock.TestingT + Cleanup(func()) +} + +// NewGossipSubApplicationSpecificScoreCache creates a new instance of GossipSubApplicationSpecificScoreCache. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewGossipSubApplicationSpecificScoreCache(t mockConstructorTestingTNewGossipSubApplicationSpecificScoreCache) *GossipSubApplicationSpecificScoreCache { + mock := &GossipSubApplicationSpecificScoreCache{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/network/p2p/mock/gossip_sub_builder.go b/network/p2p/mock/gossip_sub_builder.go index 08d82bd03c6..13342243e3b 100644 --- a/network/p2p/mock/gossip_sub_builder.go +++ b/network/p2p/mock/gossip_sub_builder.go @@ -13,8 +13,6 @@ import ( pubsub "github.com/libp2p/go-libp2p-pubsub" routing "github.com/libp2p/go-libp2p/core/routing" - - time "time" ) // GossipSubBuilder is an autogenerated mock type for the GossipSubBuilder type @@ -68,16 +66,6 @@ func (_m *GossipSubBuilder) SetGossipSubFactory(_a0 p2p.GossipSubFactoryFunc) { _m.Called(_a0) } -// SetGossipSubScoreTracerInterval provides a mock function with given fields: _a0 -func (_m *GossipSubBuilder) SetGossipSubScoreTracerInterval(_a0 time.Duration) { - _m.Called(_a0) -} - -// SetGossipSubTracer provides a mock function with given fields: _a0 -func (_m *GossipSubBuilder) SetGossipSubTracer(_a0 p2p.PubSubTracer) { - _m.Called(_a0) -} - // SetHost provides a mock function with given fields: _a0 func (_m *GossipSubBuilder) SetHost(_a0 host.Host) { _m.Called(_a0) diff --git a/network/p2p/mock/lib_p2_p_node.go b/network/p2p/mock/lib_p2_p_node.go index bbcd4c226cb..df7a01e79a8 100644 --- a/network/p2p/mock/lib_p2_p_node.go +++ b/network/p2p/mock/lib_p2_p_node.go @@ -104,6 +104,22 @@ func (_m *LibP2PNode) GetIPPort() (string, string, error) { return r0, r1, r2 } +// GetLocalMeshPeers provides a mock function with given fields: topic +func (_m *LibP2PNode) GetLocalMeshPeers(topic channels.Topic) []peer.ID { + ret := _m.Called(topic) + + var r0 []peer.ID + if rf, ok := ret.Get(0).(func(channels.Topic) []peer.ID); ok { + r0 = rf(topic) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]peer.ID) + } + } + + return r0 +} + // GetPeersForProtocol provides a mock function with given fields: pid func (_m *LibP2PNode) GetPeersForProtocol(pid protocol.ID) peer.IDSlice { ret := _m.Called(pid) diff --git a/network/p2p/mock/node_builder.go b/network/p2p/mock/node_builder.go index 331290f90c6..2a9f1ecaef8 100644 --- a/network/p2p/mock/node_builder.go +++ b/network/p2p/mock/node_builder.go @@ -20,8 +20,6 @@ import ( pubsub "github.com/libp2p/go-libp2p-pubsub" routing "github.com/libp2p/go-libp2p/core/routing" - - time "time" ) // NodeBuilder is an autogenerated mock type for the NodeBuilder type @@ -167,38 +165,6 @@ func (_m *NodeBuilder) SetGossipSubFactory(_a0 p2p.GossipSubFactoryFunc, _a1 p2p return r0 } -// SetGossipSubScoreTracerInterval provides a mock function with given fields: _a0 -func (_m *NodeBuilder) SetGossipSubScoreTracerInterval(_a0 time.Duration) p2p.NodeBuilder { - ret := _m.Called(_a0) - - var r0 p2p.NodeBuilder - if rf, ok := ret.Get(0).(func(time.Duration) p2p.NodeBuilder); ok { - r0 = rf(_a0) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(p2p.NodeBuilder) - } - } - - return r0 -} - -// SetGossipSubTracer provides a mock function with given fields: _a0 -func (_m *NodeBuilder) SetGossipSubTracer(_a0 p2p.PubSubTracer) p2p.NodeBuilder { - ret := _m.Called(_a0) - - var r0 p2p.NodeBuilder - if rf, ok := ret.Get(0).(func(p2p.PubSubTracer) p2p.NodeBuilder); ok { - r0 = rf(_a0) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(p2p.NodeBuilder) - } - } - - return r0 -} - // SetResourceManager provides a mock function with given fields: _a0 func (_m *NodeBuilder) SetResourceManager(_a0 network.ResourceManager) p2p.NodeBuilder { ret := _m.Called(_a0) diff --git a/network/p2p/mock/pub_sub.go b/network/p2p/mock/pub_sub.go index b035607ef57..e80a4009598 100644 --- a/network/p2p/mock/pub_sub.go +++ b/network/p2p/mock/pub_sub.go @@ -12,6 +12,8 @@ import ( network "github.com/onflow/flow-go/network" p2p "github.com/onflow/flow-go/network/p2p" + + peer "github.com/libp2p/go-libp2p/core/peer" ) // PubSub is an autogenerated mock type for the PubSub type @@ -19,6 +21,22 @@ type PubSub struct { mock.Mock } +// GetLocalMeshPeers provides a mock function with given fields: topic +func (_m *PubSub) GetLocalMeshPeers(topic channels.Topic) []peer.ID { + ret := _m.Called(topic) + + var r0 []peer.ID + if rf, ok := ret.Get(0).(func(channels.Topic) []peer.ID); ok { + r0 = rf(topic) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]peer.ID) + } + } + + return r0 +} + // Publish provides a mock function with given fields: ctx, messageScope func (_m *PubSub) Publish(ctx context.Context, messageScope network.OutgoingMessageScope) error { ret := _m.Called(ctx, messageScope) diff --git a/network/p2p/mock/pub_sub_adapter.go b/network/p2p/mock/pub_sub_adapter.go index ec05980dea1..159b149f183 100644 --- a/network/p2p/mock/pub_sub_adapter.go +++ b/network/p2p/mock/pub_sub_adapter.go @@ -4,6 +4,8 @@ package mockp2p import ( flow "github.com/onflow/flow-go/model/flow" + channels "github.com/onflow/flow-go/network/channels" + irrecoverable "github.com/onflow/flow-go/module/irrecoverable" mock "github.com/stretchr/testify/mock" @@ -39,6 +41,22 @@ func (_m *PubSubAdapter) Done() <-chan struct{} { return r0 } +// GetLocalMeshPeers provides a mock function with given fields: topic +func (_m *PubSubAdapter) GetLocalMeshPeers(topic channels.Topic) []peer.ID { + ret := _m.Called(topic) + + var r0 []peer.ID + if rf, ok := ret.Get(0).(func(channels.Topic) []peer.ID); ok { + r0 = rf(topic) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]peer.ID) + } + } + + return r0 +} + // GetTopics provides a mock function with given fields: func (_m *PubSubAdapter) GetTopics() []string { ret := _m.Called() diff --git a/network/p2p/mock/pub_sub_tracer.go b/network/p2p/mock/pub_sub_tracer.go index 26e37611bc2..9dc380aed65 100644 --- a/network/p2p/mock/pub_sub_tracer.go +++ b/network/p2p/mock/pub_sub_tracer.go @@ -4,6 +4,8 @@ package mockp2p import ( irrecoverable "github.com/onflow/flow-go/module/irrecoverable" + channels "github.com/onflow/flow-go/network/channels" + mock "github.com/stretchr/testify/mock" peer "github.com/libp2p/go-libp2p/core/peer" @@ -54,6 +56,22 @@ func (_m *PubSubTracer) DuplicateMessage(msg *pubsub.Message) { _m.Called(msg) } +// GetLocalMeshPeers provides a mock function with given fields: topic +func (_m *PubSubTracer) GetLocalMeshPeers(topic channels.Topic) []peer.ID { + ret := _m.Called(topic) + + var r0 []peer.ID + if rf, ok := ret.Get(0).(func(channels.Topic) []peer.ID); ok { + r0 = rf(topic) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]peer.ID) + } + } + + return r0 +} + // Graft provides a mock function with given fields: p, topic func (_m *PubSubTracer) Graft(p peer.ID, topic string) { _m.Called(p, topic) diff --git a/network/p2p/scoring/registry_test.go b/network/p2p/scoring/registry_test.go index 45cea8669bf..b7b3ee2c148 100644 --- a/network/p2p/scoring/registry_test.go +++ b/network/p2p/scoring/registry_test.go @@ -72,11 +72,12 @@ func TestScoreRegistry_FreshStart(t *testing.T) { unittest.RequireCloseBefore(t, reg.Done(), 1*time.Second, "failed to stop GossipSubAppSpecificScoreRegistry") } -// TestScoreRegistry_PeerWithSpamRecord tests the app specific penalty computation of the node when there is a spam record for the peer id. -// It tests the state that a staked peer with a valid role and valid subscriptions has spam records. -// Since the peer has spam records, it should "eventually" be deprived of the default reward for its staked role, and only have the -// penalty value as the app specific score. The "eventually" comes from the fact that the app specific score is updated asynchronously -// in the cache, and the cache is updated when the app specific score function is called by GossipSub. +// TestScoreRegistry_PeerWithSpamRecord is a test suite designed to assess the app-specific penalty computation +// in a scenario where a peer with a staked identity and valid subscriptions has a spam record. The suite runs multiple +// sub-tests, each targeting a specific type of control message (graft, prune, ihave, iwant, RpcPublishMessage). The focus +// is on the impact of spam records on the app-specific score, specifically how such records negate the default reward +// a staked peer would otherwise receive, leaving only the penalty as the app-specific score. This testing reflects the +// asynchronous nature of app-specific score updates in GossipSub's cache. func TestScoreRegistry_PeerWithSpamRecord(t *testing.T) { t.Run("graft", func(t *testing.T) { testScoreRegistryPeerWithSpamRecord(t, p2pmsg.CtrlMsgGraft, penaltyValueFixtures().Graft) @@ -95,6 +96,17 @@ func TestScoreRegistry_PeerWithSpamRecord(t *testing.T) { }) } +// testScoreRegistryPeerWithSpamRecord conducts an individual test within the TestScoreRegistry_PeerWithSpamRecord suite. +// It evaluates the ScoreRegistry's handling of a staked peer with valid subscriptions when a spam record is present for +// the peer ID. The function simulates the process of starting the registry, recording a misbehavior, and then verifying the +// updates to the spam records and app-specific score cache based on the type of control message received. +// Parameters: +// - t *testing.T: The test context. +// - messageType p2pmsg.ControlMessageType: The type of control message being tested. +// - expectedPenalty float64: The expected penalty value for the given control message type. +// This function specifically tests how the ScoreRegistry updates a peer's app-specific score in response to spam records, +// emphasizing the removal of the default reward for staked peers with valid roles and focusing on the asynchronous update +// mechanism of the app-specific score in the cache. func testScoreRegistryPeerWithSpamRecord(t *testing.T, messageType p2pmsg.ControlMessageType, expectedPenalty float64) { peerID := peer.ID("peer-1") @@ -160,27 +172,39 @@ func testScoreRegistryPeerWithSpamRecord(t *testing.T, messageType p2pmsg.Contro unittest.RequireCloseBefore(t, reg.Done(), 1*time.Second, "failed to stop GossipSubAppSpecificScoreRegistry") } -func TestSpamRecord_With_UnknownIdentity(t *testing.T) { +// TestScoreRegistry_SpamRecordWithUnknownIdentity is a test suite for verifying the behavior of the ScoreRegistry +// when handling spam records associated with unknown identities. It tests various scenarios based on different control +// message types, including graft, prune, ihave, iwant, and RpcPublishMessage. Each sub-test validates the app-specific +// penalty computation and updates to the score registry when a peer with an unknown identity sends these control messages. +func TestScoreRegistry_SpamRecordWithUnknownIdentity(t *testing.T) { t.Run("graft", func(t *testing.T) { - testSpamRecordWithUnknownIdentity(t, p2pmsg.CtrlMsgGraft, penaltyValueFixtures().Graft) + testScoreRegistrySpamRecordWithUnknownIdentity(t, p2pmsg.CtrlMsgGraft, penaltyValueFixtures().Graft) }) t.Run("prune", func(t *testing.T) { - testSpamRecordWithUnknownIdentity(t, p2pmsg.CtrlMsgPrune, penaltyValueFixtures().Prune) + testScoreRegistrySpamRecordWithUnknownIdentity(t, p2pmsg.CtrlMsgPrune, penaltyValueFixtures().Prune) }) t.Run("ihave", func(t *testing.T) { - testSpamRecordWithUnknownIdentity(t, p2pmsg.CtrlMsgIHave, penaltyValueFixtures().IHave) + testScoreRegistrySpamRecordWithUnknownIdentity(t, p2pmsg.CtrlMsgIHave, penaltyValueFixtures().IHave) }) t.Run("iwant", func(t *testing.T) { - testSpamRecordWithUnknownIdentity(t, p2pmsg.CtrlMsgIWant, penaltyValueFixtures().IWant) + testScoreRegistrySpamRecordWithUnknownIdentity(t, p2pmsg.CtrlMsgIWant, penaltyValueFixtures().IWant) }) t.Run("RpcPublishMessage", func(t *testing.T) { - testSpamRecordWithUnknownIdentity(t, p2pmsg.RpcPublishMessage, penaltyValueFixtures().RpcPublishMessage) + testScoreRegistrySpamRecordWithUnknownIdentity(t, p2pmsg.RpcPublishMessage, penaltyValueFixtures().RpcPublishMessage) }) } -// testSpamRecordWithUnknownIdentity tests the app specific penalty computation of the node when there is a spam record for the peer id and -// the peer id has an unknown identity. -func testSpamRecordWithUnknownIdentity(t *testing.T, messageType p2pmsg.ControlMessageType, expectedPenalty float64) { +// testScoreRegistrySpamRecordWithUnknownIdentity tests the app-specific penalty computation of the node when there +// is a spam record for a peer ID with an unknown identity. It examines the functionality of the GossipSubAppSpecificScoreRegistry +// under various conditions, including the initialization state, spam record creation, and the impact of different control message types. +// Parameters: +// - t *testing.T: The testing context. +// - messageType p2pmsg.ControlMessageType: The type of control message being tested. +// - expectedPenalty float64: The expected penalty value for the given control message type. +// The function simulates the process of starting the registry, reporting a misbehavior for the peer ID, and verifying the +// updates to the spam records and app-specific score cache. It ensures that the penalties are correctly computed and applied +// based on the given control message type and the state of the peer ID (unknown identity and spam record presence). +func testScoreRegistrySpamRecordWithUnknownIdentity(t *testing.T, messageType p2pmsg.ControlMessageType, expectedPenalty float64) { peerID := peer.ID("peer-1") reg, spamRecords, appScoreCache := newGossipSubAppSpecificScoreRegistry( t, @@ -193,7 +217,7 @@ func testSpamRecordWithUnknownIdentity(t *testing.T, messageType p2pmsg.ControlM reg.Start(signalerCtx) unittest.RequireCloseBefore(t, reg.Ready(), 1*time.Second, "failed to start GossipSubAppSpecificScoreRegistry") - // initially, the spamRecords should not have the peer id; ; also the app specific score record should not be in the cache. + // initially, the spamRecords should not have the peer id; also the app specific score record should not be in the cache. require.False(t, spamRecords.Has(peerID)) score, updated, exists := appScoreCache.Get(peerID) // get the score from the cache. require.False(t, exists) @@ -222,6 +246,7 @@ func testSpamRecordWithUnknownIdentity(t *testing.T, messageType p2pmsg.ControlM require.Less(t, math.Abs(expectedPenalty-record.Penalty), 10e-3) // penalty should be updated to -10, we account for decay. require.Equal(t, scoring.InitAppScoreRecordState().Decay, record.Decay) // decay should be initialized to the initial state. + queryTime := time.Now() // eventually, the app specific score should be updated in the cache. require.Eventually(t, func() bool { // calling the app specific score function when there is no app specific score in the cache should eventually update the cache. @@ -229,53 +254,86 @@ func testSpamRecordWithUnknownIdentity(t *testing.T, messageType p2pmsg.ControlM // the peer has spam record as well as an unknown identity. Hence, the app specific score should be the spam penalty // and the staking penalty. // As the app specific score in the cache and spam penalty in the spamRecords are updated at different times, we account for 0.1% error. - return math.Abs(expectedPenalty+scoring.DefaultUnknownIdentityPenalty-score)/math.Max(expectedPenalty+scoring.DefaultUnknownIdentityPenalty, score) < 0.001 + nominator := math.Abs(expectedPenalty + scoring.DefaultUnknownIdentityPenalty - score) + denominator := math.Max(expectedPenalty+scoring.DefaultUnknownIdentityPenalty, score) + return math.Abs(nominator/denominator) < 0.001 }, 5*time.Second, 100*time.Millisecond) // the app specific score should now be updated in the cache. score, updated, exists = appScoreCache.Get(peerID) // get the score from the cache. require.True(t, exists) - // require.True(t, updated.After(queryTime)) - require.True(t, math.Abs(expectedPenalty+scoring.DefaultUnknownIdentityPenalty-score)/math.Max(expectedPenalty+scoring.DefaultUnknownIdentityPenalty, score) < 0.001) + require.True(t, updated.After(queryTime)) + + nominator := math.Abs(expectedPenalty + scoring.DefaultUnknownIdentityPenalty - score) + denominator := math.Max(expectedPenalty+scoring.DefaultUnknownIdentityPenalty, score) + require.True(t, math.Abs(nominator/denominator) < 0.001) // stop the registry. cancel() unittest.RequireCloseBefore(t, reg.Done(), 1*time.Second, "failed to stop GossipSubAppSpecificScoreRegistry") } -func TestSpamRecord_With_SubscriptionPenalty(t *testing.T) { +// TestScoreRegistry_SpamRecordWithSubscriptionPenalty is a test suite for verifying the behavior of the ScoreRegistry +// in handling spam records associated with invalid subscriptions. It encompasses a series of sub-tests, each focusing on +// a different control message type: graft, prune, ihave, iwant, and RpcPublishMessage. These sub-tests are designed to +// validate the appropriate application of penalties in the ScoreRegistry when a peer with an invalid subscription is involved +// in spam activities, as indicated by these control messages. +func TestScoreRegistry_SpamRecordWithSubscriptionPenalty(t *testing.T) { t.Run("graft", func(t *testing.T) { - testSpamRecordWithSubscriptionPenalty(t, p2pmsg.CtrlMsgGraft, penaltyValueFixtures().Graft) + testScoreRegistrySpamRecordWithSubscriptionPenalty(t, p2pmsg.CtrlMsgGraft, penaltyValueFixtures().Graft) }) t.Run("prune", func(t *testing.T) { - testSpamRecordWithSubscriptionPenalty(t, p2pmsg.CtrlMsgPrune, penaltyValueFixtures().Prune) + testScoreRegistrySpamRecordWithSubscriptionPenalty(t, p2pmsg.CtrlMsgPrune, penaltyValueFixtures().Prune) }) t.Run("ihave", func(t *testing.T) { - testSpamRecordWithSubscriptionPenalty(t, p2pmsg.CtrlMsgIHave, penaltyValueFixtures().IHave) + testScoreRegistrySpamRecordWithSubscriptionPenalty(t, p2pmsg.CtrlMsgIHave, penaltyValueFixtures().IHave) }) t.Run("iwant", func(t *testing.T) { - testSpamRecordWithSubscriptionPenalty(t, p2pmsg.CtrlMsgIWant, penaltyValueFixtures().IWant) + testScoreRegistrySpamRecordWithSubscriptionPenalty(t, p2pmsg.CtrlMsgIWant, penaltyValueFixtures().IWant) }) t.Run("RpcPublishMessage", func(t *testing.T) { - testSpamRecordWithSubscriptionPenalty(t, p2pmsg.RpcPublishMessage, penaltyValueFixtures().RpcPublishMessage) + testScoreRegistrySpamRecordWithSubscriptionPenalty(t, p2pmsg.RpcPublishMessage, penaltyValueFixtures().RpcPublishMessage) }) } -// testSpamRecordWithUnknownIdentity tests the app specific penalty computation of the node when there is a spam record for the peer id and -// the peer id has an invalid subscription as well. -func testSpamRecordWithSubscriptionPenalty(t *testing.T, messageType p2pmsg.ControlMessageType, expectedPenalty float64) { +// testScoreRegistrySpamRecordWithSubscriptionPenalty tests the application-specific penalty computation in the ScoreRegistry +// when a spam record exists for a peer ID that also has an invalid subscription. The function simulates the process of +// initializing the registry, handling spam records, and updating penalties based on various control message types. +// Parameters: +// - t *testing.T: The testing context. +// - messageType p2pmsg.ControlMessageType: The type of control message being tested. +// - expectedPenalty float64: The expected penalty value for the given control message type. +// The function focuses on evaluating the registry's response to spam activities (as represented by control messages) from a +// peer with invalid subscriptions. It verifies that penalties are accurately computed and applied, taking into account both +// the spam record and the invalid subscription status of the peer. +func testScoreRegistrySpamRecordWithSubscriptionPenalty(t *testing.T, messageType p2pmsg.ControlMessageType, expectedPenalty float64) { peerID := peer.ID("peer-1") - reg, spamRecords, _ := newGossipSubAppSpecificScoreRegistry( + reg, spamRecords, appScoreCache := newGossipSubAppSpecificScoreRegistry( t, withStakedIdentity(peerID), withInvalidSubscriptions(peerID)) - // initially, the spamRecords should not have the peer id. - assert.False(t, spamRecords.Has(peerID)) + // starts the registry. + ctx, cancel := context.WithCancel(context.Background()) + signalerCtx := irrecoverable.NewMockSignalerContext(t, ctx) + reg.Start(signalerCtx) + unittest.RequireCloseBefore(t, reg.Ready(), 1*time.Second, "failed to start GossipSubAppSpecificScoreRegistry") + + // initially, the spamRecords should not have the peer id; also the app specific score record should not be in the cache. + require.False(t, spamRecords.Has(peerID)) + score, updated, exists := appScoreCache.Get(peerID) // get the score from the cache. + require.False(t, exists) + require.Equal(t, time.Time{}, updated) + require.Equal(t, float64(0), score) // peer does not have spam record, but has invalid subscription. Hence, the app specific score should be subscription penalty. - score := reg.AppSpecificScoreFunc()(peerID) - require.Equal(t, scoring.DefaultInvalidSubscriptionPenalty, score) + // eventually the app specific score should be updated in the cache to the penalty value for subscription penalty. + require.Eventually(t, func() bool { + // calling the app specific score function when there is no app specific score in the cache should eventually update the cache. + score := reg.AppSpecificScoreFunc()(peerID) + // peer does not have spam record, but has an invalid subscription penalty. + return scoring.DefaultInvalidSubscriptionPenalty == score + }, 5*time.Second, 100*time.Millisecond) // report a misbehavior for the peer id. reg.OnInvalidControlMessageNotification(&p2p.InvCtrlMsgNotif{ @@ -290,10 +348,31 @@ func testSpamRecordWithSubscriptionPenalty(t *testing.T, messageType p2pmsg.Cont assert.Less(t, math.Abs(expectedPenalty-record.Penalty), 10e-3) assert.Equal(t, scoring.InitAppScoreRecordState().Decay, record.Decay) // decay should be initialized to the initial state. - // the peer has spam record as well as an unknown identity. Hence, the app specific score should be the spam penalty - // and the staking penalty. - score = reg.AppSpecificScoreFunc()(peerID) - assert.Less(t, math.Abs(expectedPenalty+scoring.DefaultInvalidSubscriptionPenalty-score), 10e-3) + queryTime := time.Now() + // eventually, the app specific score should be updated in the cache. + require.Eventually(t, func() bool { + // calling the app specific score function when there is no app specific score in the cache should eventually update the cache. + score := reg.AppSpecificScoreFunc()(peerID) + // the peer has spam record as well as an unknown identity. Hence, the app specific score should be the spam penalty + // and the staking penalty. + // As the app specific score in the cache and spam penalty in the spamRecords are updated at different times, we account for 0.1% error. + nominator := math.Abs(expectedPenalty + scoring.DefaultInvalidSubscriptionPenalty - score) + denominator := math.Max(expectedPenalty+scoring.DefaultInvalidSubscriptionPenalty, score) + return math.Abs(nominator/denominator) < 0.001 + }, 5*time.Second, 100*time.Millisecond) + + // the app specific score should now be updated in the cache. + score, updated, exists = appScoreCache.Get(peerID) // get the score from the cache. + require.True(t, exists) + require.True(t, updated.After(queryTime)) + + nominator := math.Abs(expectedPenalty + scoring.DefaultInvalidSubscriptionPenalty - score) + denominator := math.Max(expectedPenalty+scoring.DefaultInvalidSubscriptionPenalty, score) + require.True(t, math.Abs(nominator/denominator) < 0.001) + + // stop the registry. + cancel() + unittest.RequireCloseBefore(t, reg.Done(), 1*time.Second, "failed to stop GossipSubAppSpecificScoreRegistry") } // TestSpamPenaltyDecaysInCache tests that the spam penalty records decay over time in the cache. @@ -563,7 +642,7 @@ func newGossipSubAppSpecificScoreRegistry(t *testing.T, opts ...func(*scoring.Go flowCfg, err := config.DefaultConfig() require.NoError(t, err) // overrides the default values for testing purposes. - flowCfg.NetworkConfig.GossipSub.ScoringParameters.AppSpecificScore.ScoreTTL = 100 * time.Second + flowCfg.NetworkConfig.GossipSub.ScoringParameters.AppSpecificScore.ScoreTTL = 100 * time.Millisecond validator := mockp2p.NewSubscriptionValidator(t) validator.On("Start", testifymock.Anything).Return().Maybe() From 9917ed6ce40367e3d209ddcf4aed7fd066798d9c Mon Sep 17 00:00:00 2001 From: Yahya Hassanzadeh Date: Thu, 23 Nov 2023 11:11:33 -0800 Subject: [PATCH 39/67] wip --- network/p2p/scoring/registry.go | 6 +- network/p2p/scoring/registry_test.go | 157 ++++++++++++++++++++------- network/p2p/scoring/scoring_test.go | 12 ++ 3 files changed, 132 insertions(+), 43 deletions(-) diff --git a/network/p2p/scoring/registry.go b/network/p2p/scoring/registry.go index 493b5f4bac9..67a7941219f 100644 --- a/network/p2p/scoring/registry.go +++ b/network/p2p/scoring/registry.go @@ -310,7 +310,7 @@ func (r *GossipSubAppSpecificScoreRegistry) computeAppSpecificScore(pid peer.ID) lg.Trace(). Float64("total_app_specific_score", appSpecificScore). - Msg("application specific penalty computed") + Msg("application specific score computed") return appSpecificScore } @@ -331,7 +331,7 @@ func (r *GossipSubAppSpecificScoreRegistry) processAppSpecificScoreUpdateWork(p r.logger.Trace(). Str("remote_peer_id", p2plogging.PeerId(p)). Float64("app_specific_score", appSpecificScore). - Msg("application specific penalty computed and cache updated") + Msg("application specific score computed and cache updated") return nil } @@ -424,7 +424,7 @@ func (r *GossipSubAppSpecificScoreRegistry) OnInvalidControlMessageNotification( } lg.Debug(). - Float64("app_specific_score", record.Penalty). + Float64("spam_record_penalty", record.Penalty). Msg("applied misbehaviour penalty and updated application specific penalty") } diff --git a/network/p2p/scoring/registry_test.go b/network/p2p/scoring/registry_test.go index b7b3ee2c148..6ed3d40a88e 100644 --- a/network/p2p/scoring/registry_test.go +++ b/network/p2p/scoring/registry_test.go @@ -33,6 +33,7 @@ func TestScoreRegistry_FreshStart(t *testing.T) { peerID := peer.ID("peer-1") reg, spamRecords, appScoreCache := newGossipSubAppSpecificScoreRegistry(t, + withScoreTTL(100*time.Millisecond), // refresh cached app-specific score every 100 milliseconds to speed up the test. withStakedIdentity(peerID), withValidSubscriptions(peerID)) @@ -112,6 +113,7 @@ func testScoreRegistryPeerWithSpamRecord(t *testing.T, messageType p2pmsg.Contro reg, spamRecords, appScoreCache := newGossipSubAppSpecificScoreRegistry( t, + withScoreTTL(100*time.Millisecond), // refresh cached app-specific score every 100 milliseconds to speed up the test. withStakedIdentity(peerID), withValidSubscriptions(peerID)) @@ -208,6 +210,7 @@ func testScoreRegistrySpamRecordWithUnknownIdentity(t *testing.T, messageType p2 peerID := peer.ID("peer-1") reg, spamRecords, appScoreCache := newGossipSubAppSpecificScoreRegistry( t, + withScoreTTL(100*time.Millisecond), // refresh cached app-specific score every 100 milliseconds to speed up the test. withUnknownIdentity(peerID), withValidSubscriptions(peerID)) @@ -310,6 +313,7 @@ func testScoreRegistrySpamRecordWithSubscriptionPenalty(t *testing.T, messageTyp peerID := peer.ID("peer-1") reg, spamRecords, appScoreCache := newGossipSubAppSpecificScoreRegistry( t, + withScoreTTL(100*time.Millisecond), // refresh cached app-specific score every 100 milliseconds to speed up the test. withStakedIdentity(peerID), withInvalidSubscriptions(peerID)) @@ -376,12 +380,19 @@ func testScoreRegistrySpamRecordWithSubscriptionPenalty(t *testing.T, messageTyp } // TestSpamPenaltyDecaysInCache tests that the spam penalty records decay over time in the cache. -func TestSpamPenaltyDecaysInCache(t *testing.T) { +func TestScoreRegistry_SpamPenaltyDecaysInCache(t *testing.T) { peerID := peer.ID("peer-1") reg, _, _ := newGossipSubAppSpecificScoreRegistry(t, + withScoreTTL(100*time.Millisecond), // refresh cached app-specific score every 100 milliseconds to speed up the test. withStakedIdentity(peerID), withValidSubscriptions(peerID)) + // starts the registry. + ctx, cancel := context.WithCancel(context.Background()) + signalerCtx := irrecoverable.NewMockSignalerContext(t, ctx) + reg.Start(signalerCtx) + unittest.RequireCloseBefore(t, reg.Ready(), 1*time.Second, "failed to start GossipSubAppSpecificScoreRegistry") + // report a misbehavior for the peer id. reg.OnInvalidControlMessageNotification(&p2p.InvCtrlMsgNotif{ PeerID: peerID, @@ -418,10 +429,6 @@ func TestSpamPenaltyDecaysInCache(t *testing.T) { time.Sleep(1 * time.Second) // wait for the penalty to decay. - // when the app specific penalty function is called for the first time, the decay functionality should be kicked in - // the cache, and the penalty should be updated. Note that since the penalty values are negative, the default staked identity - // reward is not applied. Hence, the penalty is only comprised of the penalties. - score := reg.AppSpecificScoreFunc()(peerID) // the upper bound is the sum of the penalties without decay. scoreUpperBound := penaltyValueFixtures().Prune + penaltyValueFixtures().Graft + @@ -432,17 +439,28 @@ func TestSpamPenaltyDecaysInCache(t *testing.T) { // in reality, the decay is applied 4 times to the first penalty, then 3 times to the second penalty, and so on. scoreLowerBound := scoreUpperBound * math.Pow(scoring.InitAppScoreRecordState().Decay, 4) - // with decay, the penalty should be between the upper and lower bounds. - assert.Greater(t, score, scoreUpperBound) - assert.Less(t, score, scoreLowerBound) + // eventually, the app specific score should be updated in the cache. + require.Eventually(t, func() bool { + // when the app specific penalty function is called for the first time, the decay functionality should be kicked in + // the cache, and the penalty should be updated. Note that since the penalty values are negative, the default staked identity + // reward is not applied. Hence, the penalty is only comprised of the penalties. + score := reg.AppSpecificScoreFunc()(peerID) + // with decay, the penalty should be between the upper and lower bounds. + return score > scoreUpperBound && score < scoreLowerBound + }, 5*time.Second, 100*time.Millisecond) + + // stop the registry. + cancel() + unittest.RequireCloseBefore(t, reg.Done(), 1*time.Second, "failed to stop GossipSubAppSpecificScoreRegistry") } // TestSpamPenaltyDecayToZero tests that the spam penalty decays to zero over time, and when the spam penalty of // a peer is set back to zero, its app specific penalty is also reset to the initial state. -func TestSpamPenaltyDecayToZero(t *testing.T) { +func TestScoreRegistry_SpamPenaltyDecayToZero(t *testing.T) { peerID := peer.ID("peer-1") reg, spamRecords, _ := newGossipSubAppSpecificScoreRegistry( t, + withScoreTTL(100*time.Millisecond), // refresh cached app-specific score every 100 milliseconds to speed up the test. withStakedIdentity(peerID), withValidSubscriptions(peerID), withInitFunction(func() p2p.GossipSubSpamRecord { @@ -452,6 +470,12 @@ func TestSpamPenaltyDecayToZero(t *testing.T) { } })) + // starts the registry. + ctx, cancel := context.WithCancel(context.Background()) + signalerCtx := irrecoverable.NewMockSignalerContext(t, ctx) + reg.Start(signalerCtx) + unittest.RequireCloseBefore(t, reg.Ready(), 1*time.Second, "failed to start GossipSubAppSpecificScoreRegistry") + // report a misbehavior for the peer id. reg.OnInvalidControlMessageNotification(&p2p.InvCtrlMsgNotif{ PeerID: peerID, @@ -461,9 +485,11 @@ func TestSpamPenaltyDecayToZero(t *testing.T) { // decays happen every second, so we wait for 1 second to make sure the penalty is updated. time.Sleep(1 * time.Second) // the penalty should now be updated, it should be still negative but greater than the penalty value (due to decay). - score := reg.AppSpecificScoreFunc()(peerID) - require.Less(t, score, float64(0)) // the penalty should be less than zero. - require.Greater(t, score, penaltyValueFixtures().Graft) // the penalty should be less than the penalty value due to decay. + require.Eventually(t, func() bool { + score := reg.AppSpecificScoreFunc()(peerID) + // the penalty should be less than zero and greater than the penalty value (due to decay). + return score < 0 && score > penaltyValueFixtures().Graft + }, 5*time.Second, 100*time.Millisecond) require.Eventually(t, func() bool { // the spam penalty should eventually decay to zero. @@ -481,15 +507,20 @@ func TestSpamPenaltyDecayToZero(t *testing.T) { assert.True(t, ok) assert.NoError(t, err) assert.Equal(t, 0.0, record.Penalty) // penalty should be zero. + + // stop the registry. + cancel() + unittest.RequireCloseBefore(t, reg.Done(), 1*time.Second, "failed to stop GossipSubAppSpecificScoreRegistry") } // TestPersistingUnknownIdentityPenalty tests that even though the spam penalty is decayed to zero, the unknown identity penalty // is persisted. This is because the unknown identity penalty is not decayed. -func TestPersistingUnknownIdentityPenalty(t *testing.T) { +func TestScoreRegistry_PersistingUnknownIdentityPenalty(t *testing.T) { peerID := peer.ID("peer-1") reg, spamRecords, _ := newGossipSubAppSpecificScoreRegistry( t, - withUnknownIdentity(peerID), // the peer id has an unknown identity. + withScoreTTL(100*time.Millisecond), // refresh cached app-specific score every 100 milliseconds to speed up the test. + withUnknownIdentity(peerID), // the peer id has an unknown identity. withValidSubscriptions(peerID), withInitFunction(func() p2p.GossipSubSpamRecord { return p2p.GossipSubSpamRecord{ @@ -498,8 +529,17 @@ func TestPersistingUnknownIdentityPenalty(t *testing.T) { } })) + // starts the registry. + ctx, cancel := context.WithCancel(context.Background()) + signalerCtx := irrecoverable.NewMockSignalerContext(t, ctx) + reg.Start(signalerCtx) + unittest.RequireCloseBefore(t, reg.Ready(), 1*time.Second, "failed to start GossipSubAppSpecificScoreRegistry") + // initially, the app specific score should be the default unknown identity penalty. - require.Equal(t, scoring.DefaultUnknownIdentityPenalty, reg.AppSpecificScoreFunc()(peerID)) + require.Eventually(t, func() bool { + score := reg.AppSpecificScoreFunc()(peerID) + return score == scoring.DefaultUnknownIdentityPenalty + }, 5*time.Second, 100*time.Millisecond) // report a misbehavior for the peer id. reg.OnInvalidControlMessageNotification(&p2p.InvCtrlMsgNotif{ @@ -507,20 +547,17 @@ func TestPersistingUnknownIdentityPenalty(t *testing.T) { MsgType: p2pmsg.CtrlMsgGraft, }) - // with reported spam, the app specific score should be the default unknown identity + the spam penalty. - diff := math.Abs(scoring.DefaultUnknownIdentityPenalty + penaltyValueFixtures().Graft - reg.AppSpecificScoreFunc()(peerID)) - normalizedDiff := diff / (scoring.DefaultUnknownIdentityPenalty + penaltyValueFixtures().Graft) - require.NotZero(t, normalizedDiff, "difference between the expected and actual app specific score should not be zero") - require.Less(t, - normalizedDiff, - 0.01, "normalized difference between the expected and actual app specific score should be less than 1%") - // decays happen every second, so we wait for 1 second to make sure the penalty is updated. time.Sleep(1 * time.Second) + // the penalty should now be updated, it should be still negative but greater than the penalty value (due to decay). - score := reg.AppSpecificScoreFunc()(peerID) - require.Less(t, score, float64(0)) // the penalty should be less than zero. - require.Greater(t, score, penaltyValueFixtures().Graft+scoring.DefaultUnknownIdentityPenalty) // the penalty should be less than the penalty value due to decay. + require.Eventually(t, func() bool { + score := reg.AppSpecificScoreFunc()(peerID) + // Ideally, the score should be the sum of the default invalid subscription penalty and the graft penalty, however, + // due to exponential decay of the spam penalty and asynchronous update the app specific score; score should be in the range of [scoring. + // (scoring.DefaultUnknownIdentityPenalty+penaltyValueFixtures().Graft, scoring.DefaultUnknownIdentityPenalty). + return score < scoring.DefaultUnknownIdentityPenalty && score > scoring.DefaultUnknownIdentityPenalty+penaltyValueFixtures().Graft + }, 5*time.Second, 100*time.Millisecond) require.Eventually(t, func() bool { // the spam penalty should eventually decay to zero. @@ -538,14 +575,19 @@ func TestPersistingUnknownIdentityPenalty(t *testing.T) { assert.True(t, ok) assert.NoError(t, err) assert.Equal(t, 0.0, record.Penalty) // penalty should be zero. + + // stop the registry. + cancel() + unittest.RequireCloseBefore(t, reg.Done(), 1*time.Second, "failed to stop GossipSubAppSpecificScoreRegistry") } // TestPersistingInvalidSubscriptionPenalty tests that even though the spam penalty is decayed to zero, the invalid subscription penalty // is persisted. This is because the invalid subscription penalty is not decayed. -func TestPersistingInvalidSubscriptionPenalty(t *testing.T) { +func TestScoreRegistry_PersistingInvalidSubscriptionPenalty(t *testing.T) { peerID := peer.ID("peer-1") reg, spamRecords, _ := newGossipSubAppSpecificScoreRegistry( t, + withScoreTTL(100*time.Millisecond), // refresh cached app-specific score every 100 milliseconds to speed up the test. withStakedIdentity(peerID), withInvalidSubscriptions(peerID), // the peer id has an invalid subscription. withInitFunction(func() p2p.GossipSubSpamRecord { @@ -555,8 +597,17 @@ func TestPersistingInvalidSubscriptionPenalty(t *testing.T) { } })) + // starts the registry. + ctx, cancel := context.WithCancel(context.Background()) + signalerCtx := irrecoverable.NewMockSignalerContext(t, ctx) + reg.Start(signalerCtx) + unittest.RequireCloseBefore(t, reg.Ready(), 1*time.Second, "failed to start GossipSubAppSpecificScoreRegistry") + // initially, the app specific score should be the default invalid subscription penalty. - require.Equal(t, scoring.DefaultUnknownIdentityPenalty, reg.AppSpecificScoreFunc()(peerID)) + require.Eventually(t, func() bool { + score := reg.AppSpecificScoreFunc()(peerID) + return score == scoring.DefaultInvalidSubscriptionPenalty + }, 5*time.Second, 100*time.Millisecond) // report a misbehavior for the peer id. reg.OnInvalidControlMessageNotification(&p2p.InvCtrlMsgNotif{ @@ -565,14 +616,13 @@ func TestPersistingInvalidSubscriptionPenalty(t *testing.T) { }) // with reported spam, the app specific score should be the default invalid subscription penalty + the spam penalty. - require.Less(t, math.Abs(scoring.DefaultInvalidSubscriptionPenalty+penaltyValueFixtures().Graft-reg.AppSpecificScoreFunc()(peerID)), 10e-3) - - // decays happen every second, so we wait for 1 second to make sure the penalty is updated. - time.Sleep(1 * time.Second) - // the penalty should now be updated, it should be still negative but greater than the penalty value (due to decay). - score := reg.AppSpecificScoreFunc()(peerID) - require.Less(t, score, float64(0)) // the penalty should be less than zero. - require.Greater(t, score, penaltyValueFixtures().Graft+scoring.DefaultInvalidSubscriptionPenalty) // the penalty should be less than the penalty value due to decay. + require.Eventually(t, func() bool { + score := reg.AppSpecificScoreFunc()(peerID) + // Ideally, the score should be the sum of the default invalid subscription penalty and the graft penalty, however, + // due to exponential decay of the spam penalty and asynchronous update the app specific score; score should be in the range of [scoring. + // (DefaultInvalidSubscriptionPenalty+penaltyValueFixtures().Graft, scoring.DefaultInvalidSubscriptionPenalty). + return score < scoring.DefaultInvalidSubscriptionPenalty && score > scoring.DefaultInvalidSubscriptionPenalty+penaltyValueFixtures().Graft + }, 5*time.Second, 100*time.Millisecond) require.Eventually(t, func() bool { // the spam penalty should eventually decay to zero. @@ -590,6 +640,10 @@ func TestPersistingInvalidSubscriptionPenalty(t *testing.T) { assert.True(t, ok) assert.NoError(t, err) assert.Equal(t, 0.0, record.Penalty) // penalty should be zero. + + // stop the registry. + cancel() + unittest.RequireCloseBefore(t, reg.Done(), 1*time.Second, "failed to stop GossipSubAppSpecificScoreRegistry") } // withStakedIdentity returns a function that sets the identity provider to return an staked identity for the given peer id. @@ -632,8 +686,33 @@ func withInitFunction(initFunction func() p2p.GossipSubSpamRecord) func(cfg *sco } } -// newGossipSubAppSpecificScoreRegistry returns a new instance of GossipSubAppSpecificScoreRegistry with default values -// for the testing purposes. +func withScoreTTL(scoreTTL time.Duration) func(cfg *scoring.GossipSubAppSpecificScoreRegistryConfig) { + return func(cfg *scoring.GossipSubAppSpecificScoreRegistryConfig) { + cfg.Parameters.ScoreTTL = scoreTTL + } +} + +// newGossipSubAppSpecificScoreRegistry creates a new instance of GossipSubAppSpecificScoreRegistry along with its associated +// GossipSubSpamRecordCache and AppSpecificScoreCache. This function is primarily used in testing scenarios to set up a controlled +// environment for evaluating the behavior of the GossipSub scoring mechanism. +// +// The function accepts a variable number of options to configure the GossipSubAppSpecificScoreRegistryConfig, allowing for +// customization of the registry's behavior in tests. These options can modify various aspects of the configuration, such as +// penalty values, identity providers, validators, and caching mechanisms. +// +// Parameters: +// - t *testing.T: The test context, used for asserting the absence of errors during the setup. +// - opts ...func(*scoring.GossipSubAppSpecificScoreRegistryConfig): A variadic set of functions that modify the registry's configuration. +// +// Returns: +// - *scoring.GossipSubAppSpecificScoreRegistry: The configured GossipSub application-specific score registry. +// - *netcache.GossipSubSpamRecordCache: The cache used for storing spam records. +// - *internal.AppSpecificScoreCache: The cache for storing application-specific scores. +// +// This function initializes and configures the scoring registry with default and test-specific settings. It sets up a spam record cache +// and an application-specific score cache with predefined sizes and functionalities. The function also configures the scoring parameters +// with test-specific values, particularly modifying the ScoreTTL value for the purpose of the tests. The creation and configuration of +// the GossipSubAppSpecificScoreRegistry are validated to ensure no errors occur during the process. func newGossipSubAppSpecificScoreRegistry(t *testing.T, opts ...func(*scoring.GossipSubAppSpecificScoreRegistryConfig)) (*scoring.GossipSubAppSpecificScoreRegistry, *netcache.GossipSubSpamRecordCache, *internal.AppSpecificScoreCache) { @@ -641,8 +720,6 @@ func newGossipSubAppSpecificScoreRegistry(t *testing.T, opts ...func(*scoring.Go appSpecificScoreCache := internal.NewAppSpecificScoreCache(100, unittest.Logger(), metrics.NewNoopHeroCacheMetricsFactory()) flowCfg, err := config.DefaultConfig() require.NoError(t, err) - // overrides the default values for testing purposes. - flowCfg.NetworkConfig.GossipSub.ScoringParameters.AppSpecificScore.ScoreTTL = 100 * time.Millisecond validator := mockp2p.NewSubscriptionValidator(t) validator.On("Start", testifymock.Anything).Return().Maybe() diff --git a/network/p2p/scoring/scoring_test.go b/network/p2p/scoring/scoring_test.go index 49ccac49b75..55b005be9f2 100644 --- a/network/p2p/scoring/scoring_test.go +++ b/network/p2p/scoring/scoring_test.go @@ -5,6 +5,7 @@ import ( "fmt" "math/rand" "testing" + "time" pubsub "github.com/libp2p/go-libp2p-pubsub" "github.com/libp2p/go-libp2p/core/peer" @@ -12,6 +13,7 @@ import ( mocktestify "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" + "github.com/onflow/flow-go/config" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module" "github.com/onflow/flow-go/module/component" @@ -98,12 +100,19 @@ func TestInvalidCtrlMsgScoringIntegration(t *testing.T) { // override the gossipsub rpc inspector suite factory to return the mock inspector suite return inspectorSuite1, nil } + + cfg, err := config.DefaultConfig() + require.NoError(t, err) + + cfg.NetworkConfig.GossipSub.ScoringParameters.AppSpecificScore.ScoreTTL = 10 * time.Millisecond // speed up the test + node1, id1 := p2ptest.NodeFixture( t, sporkId, t.Name(), idProvider, p2ptest.WithRole(flow.RoleConsensus), + p2ptest.OverrideFlowConfig(cfg), p2ptest.EnablePeerScoringWithOverride(p2p.PeerScoringConfigNoOverride), p2ptest.OverrideGossipSubRpcInspectorSuiteFactory(factory)) @@ -113,6 +122,7 @@ func TestInvalidCtrlMsgScoringIntegration(t *testing.T) { t.Name(), idProvider, p2ptest.WithRole(flow.RoleConsensus), + p2ptest.OverrideFlowConfig(cfg), p2ptest.EnablePeerScoringWithOverride(p2p.PeerScoringConfigNoOverride)) ids := flow.IdentityList{&id1, &id2} @@ -146,6 +156,8 @@ func TestInvalidCtrlMsgScoringIntegration(t *testing.T) { }) } + time.Sleep(1 * time.Second) // wait for app-specific score to be updated in the cache (remember that we need at least 100 ms for the score to be updated (ScoreTTL)) + // checks no GossipSubParameters message exchange should no longer happen between node1 and node2. p2ptest.EnsureNoPubsubExchangeBetweenGroups( t, From 83e83164d0866b06285574cd6d7249b81e851873 Mon Sep 17 00:00:00 2001 From: Yahya Hassanzadeh Date: Wed, 29 Nov 2023 18:08:33 -0800 Subject: [PATCH 40/67] fixes errors in flags --- network/netconf/flags.go | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/network/netconf/flags.go b/network/netconf/flags.go index e2f3f03f0a9..01818244ecb 100644 --- a/network/netconf/flags.go +++ b/network/netconf/flags.go @@ -160,17 +160,8 @@ func AllFlagNames() []string { BuildFlagName(gossipSub, p2pconf.ScoreParamsKey, p2pconf.AppSpecificScoreRegistryKey, p2pconf.ScoreTTLKey), BuildFlagName(gossipSub, p2pconf.ScoreParamsKey, p2pconf.SpamRecordCacheSizeKey), BuildFlagName(gossipSub, p2pconf.ScoreParamsKey, p2pconf.DecayIntervalKey), - iwantMaxSampleSize, - iwantMaxMessageIDSampleSize, - ihaveMaxMessageIDSampleSize, - iwantCacheMissThreshold, - controlMessageMaxSampleSize, - iwantDuplicateMsgIDThreshold, - iwantCacheMissCheckSize, scoringRegistrySlowerDecayThreshold, scoringRegistryDecayRateDecrement, - rpcMessageMaxSampleSize, - rpcMessageErrorThreshold, scoringRegistryDecayAdjustInterval, } @@ -289,13 +280,13 @@ func InitializeNetworkFlags(flags *pflag.FlagSet, config *Config) { flags.Float32(alspSyncEngineSyncRequestProb, config.AlspConfig.SyncEngine.SyncRequestProb, "probability of creating a misbehavior report for a sync request message") flags.Float64(scoringRegistrySlowerDecayThreshold, - config.GossipSubConfig.GossipSubScoringRegistryConfig.PenaltyDecaySlowdownThreshold, + config.GossipSub.GossipSubScoringRegistryConfig.PenaltyDecaySlowdownThreshold, "the penalty level at which the decay rate is reduced by --gossipsub-app-specific-penalty-decay-rate-reduction-factor") flags.Float64(scoringRegistryDecayRateDecrement, - config.GossipSubConfig.GossipSubScoringRegistryConfig.DecayRateReductionFactor, + config.GossipSub.GossipSubScoringRegistryConfig.DecayRateReductionFactor, "defines the value by which the decay rate is decreased every time the penalty is below the --gossipsub-app-specific-penalty-decay-slowdown-threshold.") flags.Duration(scoringRegistryDecayAdjustInterval, - config.GossipSubConfig.GossipSubScoringRegistryConfig.PenaltyDecayEvaluationPeriod, + config.GossipSub.GossipSubScoringRegistryConfig.PenaltyDecayEvaluationPeriod, "defines the period at which the decay for a spam record is okay to be adjusted.") flags.Int(BuildFlagName(gossipSub, p2pconf.RpcInspectorKey, p2pconf.ValidationConfigKey, p2pconf.IHaveConfigKey, p2pconf.MaxSampleSizeKey), config.GossipSub.RpcInspector.Validation.IHave.MaxSampleSize, From f9db0d0185116d6ad124e3fff3eed630520af7b7 Mon Sep 17 00:00:00 2001 From: Yahya Hassanzadeh Date: Wed, 29 Nov 2023 18:28:52 -0800 Subject: [PATCH 41/67] fixes flags names --- network/netconf/flags.go | 84 +++++++++-------------------- network/p2p/p2pconf/gossipsub.go | 46 +++++++++------- network/p2p/scoring/score_option.go | 20 ++++--- 3 files changed, 63 insertions(+), 87 deletions(-) diff --git a/network/netconf/flags.go b/network/netconf/flags.go index 01818244ecb..aa1d742b114 100644 --- a/network/netconf/flags.go +++ b/network/netconf/flags.go @@ -48,51 +48,15 @@ const ( memoryLimitBytes = "memory-bytes" // connection manager - highWatermark = "libp2p-high-watermark" - lowWatermark = "libp2p-low-watermark" - gracePeriod = "libp2p-grace-period" - silencePeriod = "libp2p-silence-period" - // gossipsub - gossipSub = "gossipsub" - // rpcSentTrackerCacheSize = "gossipsub-rpc-sent-tracker-cache-size" - // rpcSentTrackerQueueCacheSize = "gossipsub-rpc-sent-tracker-queue-cache-size" - // rpcSentTrackerNumOfWorkers = "gossipsub-rpc-sent-tracker-workers" - // scoreTracerInterval = "gossipsub-score-tracer-interval" - - // gossipSubSubscriptionProviderUpdateInterval = "gossipsub-subscription-provider-update-interval" - // gossipSubSubscriptionProviderCacheSize = "gossipsub-subscription-provider-cache-size" - // - // // gossipsub validation inspector - // gossipSubRPCInspectorNotificationCacheSize = "gossipsub-rpc-inspector-notification-cache-size" - // validationInspectorNumberOfWorkers = "gossipsub-rpc-validation-inspector-workers" - // validationInspectorInspectMessageQueueCacheSize = "gossipsub-rpc-validation-inspector-queue-cache-size" - // validationInspectorClusterPrefixedTopicsReceivedCacheSize = "gossipsub-cluster-prefix-tracker-cache-size" - // validationInspectorClusterPrefixedTopicsReceivedCacheDecay = "gossipsub-cluster-prefix-tracker-cache-decay" - // validationInspectorClusterPrefixHardThreshold = "gossipsub-rpc-cluster-prefixed-hard-threshold" - // - // ihaveMaxSampleSize = "gossipsub-rpc-ihave-max-sample-size" - // ihaveMaxMessageIDSampleSize = "gossipsub-rpc-ihave-max-message-id-sample-size" - // controlMessageMaxSampleSize = "gossipsub-rpc-graft-and-prune-message-max-sample-size" - // iwantMaxSampleSize = "gossipsub-rpc-iwant-max-sample-size" - // iwantMaxMessageIDSampleSize = "gossipsub-rpc-iwant-max-message-id-sample-size" - // iwantCacheMissThreshold = "gossipsub-rpc-iwant-cache-miss-threshold" - // iwantCacheMissCheckSize = "gossipsub-rpc-iwant-cache-miss-check-size" - // iwantDuplicateMsgIDThreshold = "gossipsub-rpc-iwant-duplicate-message-id-threshold" - // rpcMessageMaxSampleSize = "gossipsub-rpc-message-max-sample-size" - // rpcMessageErrorThreshold = "gossipsub-rpc-message-error-threshold" - // gossipsub metrics inspector - // metricsInspectorNumberOfWorkers = "gossipsub-rpc-metrics-inspector-workers" - // metricsInspectorCacheSize = "gossipsub-rpc-metrics-inspector-cache-size" - - // gossipsub scoring registry - scoringRegistrySlowerDecayThreshold = "gossipsub-app-specific-penalty-decay-slowdown-threshold" - scoringRegistryDecayRateDecrement = "gossipsub-app-specific-penalty-decay-rate-reduction-factor" - scoringRegistryDecayAdjustInterval = "gossipsub-app-specific-penalty-decay-evaluation-period" - alspDisabled = "alsp-disable-penalty" - alspSpamRecordCacheSize = "alsp-spam-record-cache-size" - alspSpamRecordQueueSize = "alsp-spam-report-queue-size" - alspHearBeatInterval = "alsp-heart-beat-interval" - + highWatermark = "libp2p-high-watermark" + lowWatermark = "libp2p-low-watermark" + gracePeriod = "libp2p-grace-period" + silencePeriod = "libp2p-silence-period" + gossipSub = "gossipsub" + alspDisabled = "alsp-disable-penalty" + alspSpamRecordCacheSize = "alsp-spam-record-cache-size" + alspSpamRecordQueueSize = "alsp-spam-report-queue-size" + alspHearBeatInterval = "alsp-heart-beat-interval" alspSyncEngineBatchRequestBaseProb = "alsp-sync-engine-batch-request-base-prob" alspSyncEngineRangeRequestBaseProb = "alsp-sync-engine-range-request-base-prob" alspSyncEngineSyncRequestProb = "alsp-sync-engine-sync-request-prob" @@ -158,11 +122,11 @@ func AllFlagNames() []string { BuildFlagName(gossipSub, p2pconf.ScoreParamsKey, p2pconf.AppSpecificScoreRegistryKey, p2pconf.ScoreUpdateWorkerNumKey), BuildFlagName(gossipSub, p2pconf.ScoreParamsKey, p2pconf.AppSpecificScoreRegistryKey, p2pconf.ScoreUpdateRequestQueueSizeKey), BuildFlagName(gossipSub, p2pconf.ScoreParamsKey, p2pconf.AppSpecificScoreRegistryKey, p2pconf.ScoreTTLKey), - BuildFlagName(gossipSub, p2pconf.ScoreParamsKey, p2pconf.SpamRecordCacheSizeKey), + BuildFlagName(gossipSub, p2pconf.ScoreParamsKey, p2pconf.SpamRecordCacheKey, p2pconf.CacheSizeKey), + BuildFlagName(gossipSub, p2pconf.ScoreParamsKey, p2pconf.SpamRecordCacheKey, p2pconf.PenaltyDecaySlowdownThresholdKey), + BuildFlagName(gossipSub, p2pconf.ScoreParamsKey, p2pconf.SpamRecordCacheKey, p2pconf.DecayRateReductionFactorKey), + BuildFlagName(gossipSub, p2pconf.ScoreParamsKey, p2pconf.SpamRecordCacheKey, p2pconf.PenaltyDecayEvaluationPeriodKey), BuildFlagName(gossipSub, p2pconf.ScoreParamsKey, p2pconf.DecayIntervalKey), - scoringRegistrySlowerDecayThreshold, - scoringRegistryDecayRateDecrement, - scoringRegistryDecayAdjustInterval, } for _, scope := range []string{systemScope, transientScope, protocolScope, peerScope, peerProtocolScope} { @@ -279,14 +243,16 @@ func InitializeNetworkFlags(flags *pflag.FlagSet, config *Config) { "base probability of creating a misbehavior report for a range request message") flags.Float32(alspSyncEngineSyncRequestProb, config.AlspConfig.SyncEngine.SyncRequestProb, "probability of creating a misbehavior report for a sync request message") - flags.Float64(scoringRegistrySlowerDecayThreshold, - config.GossipSub.GossipSubScoringRegistryConfig.PenaltyDecaySlowdownThreshold, - "the penalty level at which the decay rate is reduced by --gossipsub-app-specific-penalty-decay-rate-reduction-factor") - flags.Float64(scoringRegistryDecayRateDecrement, - config.GossipSub.GossipSubScoringRegistryConfig.DecayRateReductionFactor, - "defines the value by which the decay rate is decreased every time the penalty is below the --gossipsub-app-specific-penalty-decay-slowdown-threshold.") - flags.Duration(scoringRegistryDecayAdjustInterval, - config.GossipSub.GossipSubScoringRegistryConfig.PenaltyDecayEvaluationPeriod, + flags.Float64(BuildFlagName(gossipSub, p2pconf.ScoreParamsKey, p2pconf.SpamRecordCacheKey, p2pconf.PenaltyDecaySlowdownThresholdKey), + config.GossipSub.ScoringParameters.SpamRecordCache.PenaltyDecaySlowdownThreshold, + fmt.Sprintf("the penalty level at which the decay rate is reduced by --%s", + BuildFlagName(gossipSub, p2pconf.ScoreParamsKey, p2pconf.SpamRecordCacheKey, p2pconf.DecayRateReductionFactorKey))) + flags.Float64(BuildFlagName(gossipSub, p2pconf.ScoreParamsKey, p2pconf.SpamRecordCacheKey, p2pconf.DecayRateReductionFactorKey), + config.GossipSub.ScoringParameters.SpamRecordCache.DecayRateReductionFactor, + fmt.Sprintf("defines the value by which the decay rate is decreased every time the penalty is below the --%s", + BuildFlagName(gossipSub, p2pconf.ScoreParamsKey, p2pconf.SpamRecordCacheKey, p2pconf.PenaltyDecaySlowdownThresholdKey))) + flags.Duration(BuildFlagName(gossipSub, p2pconf.ScoreParamsKey, p2pconf.SpamRecordCacheKey, p2pconf.PenaltyDecayEvaluationPeriodKey), + config.GossipSub.ScoringParameters.SpamRecordCache.PenaltyDecayEvaluationPeriod, "defines the period at which the decay for a spam record is okay to be adjusted.") flags.Int(BuildFlagName(gossipSub, p2pconf.RpcInspectorKey, p2pconf.ValidationConfigKey, p2pconf.IHaveConfigKey, p2pconf.MaxSampleSizeKey), config.GossipSub.RpcInspector.Validation.IHave.MaxSampleSize, @@ -333,8 +299,8 @@ func InitializeNetworkFlags(flags *pflag.FlagSet, config *Config) { flags.Duration(BuildFlagName(gossipSub, p2pconf.ScoreParamsKey, p2pconf.AppSpecificScoreRegistryKey, p2pconf.ScoreTTLKey), config.GossipSub.ScoringParameters.AppSpecificScore.ScoreTTL, "time to live for app specific scores; when expired a new request will be sent to the score update worker pool; till then the expired score will be used") - flags.Uint32(BuildFlagName(gossipSub, p2pconf.ScoreParamsKey, p2pconf.SpamRecordCacheSizeKey), - config.GossipSub.ScoringParameters.SpamRecordCacheSize, + flags.Uint32(BuildFlagName(gossipSub, p2pconf.ScoreParamsKey, p2pconf.SpamRecordCacheKey, p2pconf.CacheSizeKey), + config.GossipSub.ScoringParameters.SpamRecordCache.CacheSize, "size of the spam record cache, recommended size is 10x the number of authorized nodes") flags.Duration(BuildFlagName(gossipSub, p2pconf.ScoreParamsKey, p2pconf.DecayIntervalKey), config.GossipSub.ScoringParameters.DecayInterval, diff --git a/network/p2p/p2pconf/gossipsub.go b/network/p2p/p2pconf/gossipsub.go index 61b7a978359..d250d60e369 100644 --- a/network/p2p/p2pconf/gossipsub.go +++ b/network/p2p/p2pconf/gossipsub.go @@ -68,22 +68,19 @@ type GossipSubParameters struct { // RpcInspectorParameters configuration for all gossipsub RPC control message inspectors. RpcInspector RpcInspectorParameters `mapstructure:"rpc-inspector"` - GossipSubScoringRegistryConfig `mapstructure:",squash"` // GossipSubScoringRegistryConfig is the configuration for the GossipSub score registry. // GossipSubTracerParameters is the configuration for the gossipsub tracer. GossipSubParameters tracer is used to trace the local mesh events and peer scores. RpcTracer GossipSubTracerParameters `mapstructure:"rpc-tracer"` // ScoringParameters is whether to enable GossipSubParameters peer scoring. - PeerScoringEnabled bool `mapstructure:"peer-scoring-enabled"` - + PeerScoringEnabled bool `mapstructure:"peer-scoring-enabled"` SubscriptionProvider SubscriptionProviderParameters `mapstructure:"subscription-provider"` - - ScoringParameters ScoringParameters `mapstructure:"scoring-parameters"` + ScoringParameters ScoringParameters `mapstructure:"scoring-parameters"` } const ( AppSpecificScoreRegistryKey = "app-specific-score" - SpamRecordCacheSizeKey = "spam-record-cache-size" + SpamRecordCacheKey = "spam-record-cache" DecayIntervalKey = "decay-interval" ) @@ -91,10 +88,7 @@ const ( // Parameters are "numerical values" that are used to compute or build components that compute the score of a peer in GossipSub system. type ScoringParameters struct { AppSpecificScore AppSpecificScoreParameters `validate:"required" mapstructure:"app-specific-score"` - // SpamRecordCacheSize is size of the cache used to store the spam records of peers. - // The spam records are used to penalize peers that send invalid messages. - SpamRecordCacheSize uint32 `validate:"gt=0" mapstructure:"spam-record-cache-size"` - + SpamRecordCache SpamRecordCacheParameters `validate:"required" mapstructure:"spam-record-cache"` // DecayInterval is the interval at which the counters associated with a peer behavior in GossipSub system are decayed. DecayInterval time.Duration `validate:"gt=0s" mapstructure:"decay-interval"` } @@ -121,6 +115,28 @@ type AppSpecificScoreParameters struct { ScoreTTL time.Duration `validate:"required" mapstructure:"score-ttl"` } +const ( + PenaltyDecaySlowdownThresholdKey = "penalty-decay-slowdown-threshold" + DecayRateReductionFactorKey = "decay-rate-reduction-factor" + PenaltyDecayEvaluationPeriodKey = "penalty-decay-evaluation-period" +) + +type SpamRecordCacheParameters struct { + // CacheSize is size of the cache used to store the spam records of peers. + // The spam records are used to penalize peers that send invalid messages. + CacheSize uint32 `validate:"gt=0" mapstructure:"cache-size"` + + // PenaltyDecaySlowdownThreshold defines the penalty level which the decay rate is reduced by `DecayRateReductionFactor` every time the penalty of a node falls below the threshold, thereby slowing down the decay process. + // This mechanism ensures that malicious nodes experience longer decay periods, while honest nodes benefit from quicker decay. + PenaltyDecaySlowdownThreshold float64 `validate:"lt=0" mapstructure:"penalty-decay-slowdown-threshold"` + + // DecayRateReductionFactor defines the value by which the decay rate is decreased every time the penalty is below the PenaltyDecaySlowdownThreshold. A reduced decay rate extends the time it takes for penalties to diminish. + DecayRateReductionFactor float64 `validate:"gt=0,lt=1" mapstructure:"penalty-decay-rate-reduction-factor"` + + // PenaltyDecayEvaluationPeriod defines the interval at which the decay for a spam record is okay to be adjusted. + PenaltyDecayEvaluationPeriod time.Duration `validate:"gt=0" mapstructure:"penalty-decay-evaluation-period"` +} + // SubscriptionProviderParameters keys. const ( UpdateIntervalKey = "update-interval" @@ -139,16 +155,6 @@ type SubscriptionProviderParameters struct { CacheSize uint32 `validate:"gt=0" mapstructure:"cache-size"` } -// GossipSubScoringRegistryConfig is the configuration for the GossipSub score registry. -type GossipSubScoringRegistryConfig struct { - // PenaltyDecaySlowdownThreshold defines the penalty level which the decay rate is reduced by `DecayRateReductionFactor` every time the penalty of a node falls below the threshold, thereby slowing down the decay process. - // This mechanism ensures that malicious nodes experience longer decay periods, while honest nodes benefit from quicker decay. - PenaltyDecaySlowdownThreshold float64 `validate:"lt=0" mapstructure:"gossipsub-app-specific-penalty-decay-slowdown-threshold"` - // DecayRateReductionFactor defines the value by which the decay rate is decreased every time the penalty is below the PenaltyDecaySlowdownThreshold. A reduced decay rate extends the time it takes for penalties to diminish. - DecayRateReductionFactor float64 `validate:"gt=0,lt=1" mapstructure:"gossipsub-app-specific-penalty-decay-rate-reduction-factor"` - // PenaltyDecayEvaluationPeriod defines the interval at which the decay for a spam record is okay to be adjusted. - PenaltyDecayEvaluationPeriod time.Duration `validate:"gt=0" mapstructure:"gossipsub-app-specific-penalty-decay-evaluation-period"` -} // GossipSubTracerParameters keys. const ( LocalMeshLogIntervalKey = "local-mesh-logging-interval" diff --git a/network/p2p/scoring/score_option.go b/network/p2p/scoring/score_option.go index 0589a92a0ea..2173d2dd4db 100644 --- a/network/p2p/scoring/score_option.go +++ b/network/p2p/scoring/score_option.go @@ -362,7 +362,7 @@ func (c *ScoreOptionConfig) SetRegisterNotificationConsumerFunc(f func(p2p.Gossi // NewScoreOption creates a new penalty option with the given configuration. func NewScoreOption(cfg *ScoreOptionConfig, provider p2p.SubscriptionProvider) (*ScoreOption, error) { throttledSampler := logging.BurstSampler(MaxDebugLogs, time.Second) - logger := scoreOptionConfig.logger.With(). + logger := cfg.logger.With(). Str("module", "pubsub_score_option"). Logger(). Sample(zerolog.LevelSampler{ @@ -378,10 +378,14 @@ func NewScoreOption(cfg *ScoreOptionConfig, provider p2p.SubscriptionProvider) ( IdProvider: cfg.provider, HeroCacheMetricsFactory: cfg.heroCacheMetricsFactory, AppScoreCacheFactory: func() p2p.GossipSubApplicationSpecificScoreCache { - return internal.NewAppSpecificScoreCache(cfg.params.SpamRecordCacheSize, cfg.logger, cfg.heroCacheMetricsFactory) + return internal.NewAppSpecificScoreCache(cfg.params.SpamRecordCache.CacheSize, cfg.logger, cfg.heroCacheMetricsFactory) }, SpamRecordCacheFactory: func() p2p.GossipSubSpamRecordCache { - return netcache.NewGossipSubSpamRecordCache(cfg.params.SpamRecordCacheSize, cfg.logger, cfg.heroCacheMetricsFactory, DefaultDecayFunction()) + return netcache.NewGossipSubSpamRecordCache(cfg.params.SpamRecordCache.CacheSize, cfg.logger, cfg.heroCacheMetricsFactory, + DefaultDecayFunction( + cfg.params.SpamRecordCache.PenaltyDecaySlowdownThreshold, + cfg.params.SpamRecordCache.DecayRateReductionFactor, + cfg.params.SpamRecordCache.PenaltyDecayEvaluationPeriod)) }, Parameters: cfg.params.AppSpecificScore, }) @@ -398,8 +402,8 @@ func NewScoreOption(cfg *ScoreOptionConfig, provider p2p.SubscriptionProvider) ( // set the app specific penalty function for the penalty option // if the app specific penalty function is not set, use the default one - if scoreOptionConfig.appScoreFunc != nil { - s.appScoreFunc = scoreOptionConfig.appScoreFunc + if cfg.appScoreFunc != nil { + s.appScoreFunc = cfg.appScoreFunc s.logger. Warn(). Str(logging.KeyNetworkingSecurity, "true"). @@ -417,14 +421,14 @@ func NewScoreOption(cfg *ScoreOptionConfig, provider p2p.SubscriptionProvider) ( } // registers the score registry as the consumer of the invalid control message notifications - if scoreOptionConfig.registerNotificationConsumerFunc != nil { - scoreOptionConfig.registerNotificationConsumerFunc(scoreRegistry) + if cfg.registerNotificationConsumerFunc != nil { + cfg.registerNotificationConsumerFunc(scoreRegistry) } s.peerScoreParams.AppSpecificScore = s.appScoreFunc // apply the topic penalty parameters if any. - for _, topicParams := range scoreOptionConfig.topicParams { + for _, topicParams := range cfg.topicParams { topicParams(s.peerScoreParams.Topics) } From aa0325a8c9b86a765d95ff842db24eb2e0e427da Mon Sep 17 00:00:00 2001 From: Yahya Hassanzadeh Date: Wed, 29 Nov 2023 18:34:58 -0800 Subject: [PATCH 42/67] adds config for spam record decay --- config/default-config.yml | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/config/default-config.yml b/config/default-config.yml index e1225d62d59..b76dd0ad379 100644 --- a/config/default-config.yml +++ b/config/default-config.yml @@ -197,9 +197,26 @@ network-config: # score ttl is the time to live for the app specific score. Once the score is expired; a new request will be sent to the app specific score provider to update the score. # until the score is updated, the previous score will be used. score-ttl: 1m - # size of cache used to track spam records at gossipsub. Each peer id is mapped to a spam record that keeps track of the spam score for that peer. - # cache should be big enough to keep track of the entire network's size. Otherwise, the local node's view of the network will be incomplete due to cache eviction. - spam-record-cache-size: 10_000 + spam-record-cache: + # size of cache used to track spam records at gossipsub. Each peer id is mapped to a spam record that keeps track of the spam score for that peer. + # cache should be big enough to keep track of the entire network's size. Otherwise, the local node's view of the network will be incomplete due to cache eviction. + cache-size: 10_000 + # Threshold level for spam record penalty. + # At each evaluation period, when a node's penalty is below this value, the decay rate slows down, ensuring longer decay periods for malicious nodes and quicker decay for honest ones. + penalty-decay-slowdown-threshold: -99 + # This setting adjusts the decay rate when a node's penalty falls below the threshold. + # The decay rate, ranging between 0 and 1, dictates how quickly penalties decrease: a higher rate results in slower decay. + # The decay calculation is multiplicative (newPenalty = decayRate * oldPenalty). + # The reduction factor increases the decay rate, thus decelerating the penalty reduction. For instance, with a 0.01 reduction factor, + # the decay rate increases by 0.01 at each evaluation interval when the penalty is below the threshold. + # Consequently, a decay rate of `x` diminishes the penalty to zero more rapidly than a rate of `x+0.01`. + penalty-decay-rate-reduction-factor: 0.01 + # Defines the frequency for evaluating and potentially adjusting the decay process of a spam record. + # At each interval, the system assesses the current penalty of a node. + # If this penalty is below the defined threshold, the decay rate is modified according to the reduction factor, slowing down the penalty reduction process. + # This reassessment at regular intervals ensures that the decay rate is dynamically adjusted to reflect the node's ongoing behavior, + # maintaining a balance between penalizing malicious activity and allowing recovery for honest nodes. + penalty-decay-evaluation-period: 10m # the intervals at which counters associated with a peer behavior in gossipsub system are decayed. decay-interval: 1m subscription-provider: From 9ee1532028bff0e5dee15f82597984c8ccd1c778 Mon Sep 17 00:00:00 2001 From: Yahya Hassanzadeh Date: Wed, 29 Nov 2023 18:37:00 -0800 Subject: [PATCH 43/67] lint fix --- network/internal/p2pfixtures/fixtures.go | 1 - network/p2p/p2pbuilder/libp2pNodeBuilder.go | 1 - 2 files changed, 2 deletions(-) diff --git a/network/internal/p2pfixtures/fixtures.go b/network/internal/p2pfixtures/fixtures.go index b25c18356be..b2df52dbfd8 100644 --- a/network/internal/p2pfixtures/fixtures.go +++ b/network/internal/p2pfixtures/fixtures.go @@ -111,7 +111,6 @@ func CreateNode(t *testing.T, networkKey crypto.PrivateKey, sporkID flow.Identif networkKey, sporkID, idProvider, - defaultFlowConfig.NetworkConfig.GossipSubConfig.GossipSubScoringRegistryConfig, &defaultFlowConfig.NetworkConfig.ResourceManager, p2pconfig.PeerManagerDisableConfig(), &p2p.DisallowListCacheConfig{ diff --git a/network/p2p/p2pbuilder/libp2pNodeBuilder.go b/network/p2p/p2pbuilder/libp2pNodeBuilder.go index 18e456b2902..fe149f7f47b 100644 --- a/network/p2p/p2pbuilder/libp2pNodeBuilder.go +++ b/network/p2p/p2pbuilder/libp2pNodeBuilder.go @@ -81,7 +81,6 @@ func NewNodeBuilder( networkKey fcrypto.PrivateKey, sporkId flow.Identifier, idProvider module.IdentityProvider, - scoringRegistryConfig p2pconf.GossipSubScoringRegistryConfig, rCfg *p2pconf.ResourceManagerConfig, peerManagerConfig *p2pconfig.PeerManagerConfig, disallowListCacheCfg *p2p.DisallowListCacheConfig, From 12935316b51b925d0da5cfcc4b8219098edb665a Mon Sep 17 00:00:00 2001 From: Yahya Hassanzadeh Date: Wed, 29 Nov 2023 18:40:09 -0800 Subject: [PATCH 44/67] lint fix --- .../node_builder/access_node_builder.go | 61 +++++++++++++++---- 1 file changed, 48 insertions(+), 13 deletions(-) diff --git a/cmd/access/node_builder/access_node_builder.go b/cmd/access/node_builder/access_node_builder.go index 4a6c964a64e..bb9679e4511 100644 --- a/cmd/access/node_builder/access_node_builder.go +++ b/cmd/access/node_builder/access_node_builder.go @@ -1038,13 +1038,34 @@ func (builder *FlowAccessNodeBuilder) extraFlags() { // Execution State Streaming API flags.Uint32Var(&builder.stateStreamConf.ExecutionDataCacheSize, "execution-data-cache-size", defaultConfig.stateStreamConf.ExecutionDataCacheSize, "block execution data cache size") flags.Uint32Var(&builder.stateStreamConf.MaxGlobalStreams, "state-stream-global-max-streams", defaultConfig.stateStreamConf.MaxGlobalStreams, "global maximum number of concurrent streams") - flags.UintVar(&builder.stateStreamConf.MaxExecutionDataMsgSize, "state-stream-max-message-size", defaultConfig.stateStreamConf.MaxExecutionDataMsgSize, "maximum size for a gRPC message containing block execution data") - flags.StringToIntVar(&builder.stateStreamFilterConf, "state-stream-event-filter-limits", defaultConfig.stateStreamFilterConf, "event filter limits for ExecutionData SubscribeEvents API e.g. EventTypes=100,Addresses=100,Contracts=100 etc.") - flags.DurationVar(&builder.stateStreamConf.ClientSendTimeout, "state-stream-send-timeout", defaultConfig.stateStreamConf.ClientSendTimeout, "maximum wait before timing out while sending a response to a streaming client e.g. 30s") - flags.UintVar(&builder.stateStreamConf.ClientSendBufferSize, "state-stream-send-buffer-size", defaultConfig.stateStreamConf.ClientSendBufferSize, "maximum number of responses to buffer within a stream") - flags.Float64Var(&builder.stateStreamConf.ResponseLimit, "state-stream-response-limit", defaultConfig.stateStreamConf.ResponseLimit, "max number of responses per second to send over streaming endpoints. this helps manage resources consumed by each client querying data not in the cache e.g. 3 or 0.5. 0 means no limit") - flags.Uint64Var(&builder.stateStreamConf.HeartbeatInterval, "state-stream-heartbeat-interval", defaultConfig.stateStreamConf.HeartbeatInterval, "default interval in blocks at which heartbeat messages should be sent. applied when client did not specify a value.") - flags.Uint32Var(&builder.stateStreamConf.RegisterIDsRequestLimit, "state-stream-max-register-values", defaultConfig.stateStreamConf.RegisterIDsRequestLimit, "maximum number of register ids to include in a single request to the GetRegisters endpoint") + flags.UintVar(&builder.stateStreamConf.MaxExecutionDataMsgSize, + "state-stream-max-message-size", + defaultConfig.stateStreamConf.MaxExecutionDataMsgSize, + "maximum size for a gRPC message containing block execution data") + flags.StringToIntVar(&builder.stateStreamFilterConf, + "state-stream-event-filter-limits", + defaultConfig.stateStreamFilterConf, + "event filter limits for ExecutionData SubscribeEvents API e.g. EventTypes=100,Addresses=100,Contracts=100 etc.") + flags.DurationVar(&builder.stateStreamConf.ClientSendTimeout, + "state-stream-send-timeout", + defaultConfig.stateStreamConf.ClientSendTimeout, + "maximum wait before timing out while sending a response to a streaming client e.g. 30s") + flags.UintVar(&builder.stateStreamConf.ClientSendBufferSize, + "state-stream-send-buffer-size", + defaultConfig.stateStreamConf.ClientSendBufferSize, + "maximum number of responses to buffer within a stream") + flags.Float64Var(&builder.stateStreamConf.ResponseLimit, + "state-stream-response-limit", + defaultConfig.stateStreamConf.ResponseLimit, + "max number of responses per second to send over streaming endpoints. this helps manage resources consumed by each client querying data not in the cache e.g. 3 or 0.5. 0 means no limit") + flags.Uint64Var(&builder.stateStreamConf.HeartbeatInterval, + "state-stream-heartbeat-interval", + defaultConfig.stateStreamConf.HeartbeatInterval, + "default interval in blocks at which heartbeat messages should be sent. applied when client did not specify a value.") + flags.Uint32Var(&builder.stateStreamConf.RegisterIDsRequestLimit, + "state-stream-max-register-values", + defaultConfig.stateStreamConf.RegisterIDsRequestLimit, + "maximum number of register ids to include in a single request to the GetRegisters endpoint") // Execution Data Indexer flags.BoolVar(&builder.executionDataIndexingEnabled, @@ -1055,11 +1076,26 @@ func (builder *FlowAccessNodeBuilder) extraFlags() { flags.StringVar(&builder.checkpointFile, "execution-state-checkpoint", defaultConfig.checkpointFile, "execution-state checkpoint file") // Script Execution - flags.StringVar(&builder.rpcConf.BackendConfig.ScriptExecutionMode, "script-execution-mode", defaultConfig.rpcConf.BackendConfig.ScriptExecutionMode, "mode to use when executing scripts. one of (local-only, execution-nodes-only, failover, compare)") - flags.Uint64Var(&builder.scriptExecutorConfig.ComputationLimit, "script-execution-computation-limit", defaultConfig.scriptExecutorConfig.ComputationLimit, "maximum number of computation units a locally executed script can use. default: 100000") - flags.IntVar(&builder.scriptExecutorConfig.MaxErrorMessageSize, "script-execution-max-error-length", defaultConfig.scriptExecutorConfig.MaxErrorMessageSize, "maximum number characters to include in error message strings. additional characters are truncated. default: 1000") - flags.DurationVar(&builder.scriptExecutorConfig.LogTimeThreshold, "script-execution-log-time-threshold", defaultConfig.scriptExecutorConfig.LogTimeThreshold, "emit a log for any scripts that take over this threshold. default: 1s") - flags.DurationVar(&builder.scriptExecutorConfig.ExecutionTimeLimit, "script-execution-timeout", defaultConfig.scriptExecutorConfig.ExecutionTimeLimit, "timeout value for locally executed scripts. default: 10s") + flags.StringVar(&builder.rpcConf.BackendConfig.ScriptExecutionMode, + "script-execution-mode", + defaultConfig.rpcConf.BackendConfig.ScriptExecutionMode, + "mode to use when executing scripts. one of (local-only, execution-nodes-only, failover, compare)") + flags.Uint64Var(&builder.scriptExecutorConfig.ComputationLimit, + "script-execution-computation-limit", + defaultConfig.scriptExecutorConfig.ComputationLimit, + "maximum number of computation units a locally executed script can use. default: 100000") + flags.IntVar(&builder.scriptExecutorConfig.MaxErrorMessageSize, + "script-execution-max-error-length", + defaultConfig.scriptExecutorConfig.MaxErrorMessageSize, + "maximum number characters to include in error message strings. additional characters are truncated. default: 1000") + flags.DurationVar(&builder.scriptExecutorConfig.LogTimeThreshold, + "script-execution-log-time-threshold", + defaultConfig.scriptExecutorConfig.LogTimeThreshold, + "emit a log for any scripts that take over this threshold. default: 1s") + flags.DurationVar(&builder.scriptExecutorConfig.ExecutionTimeLimit, + "script-execution-timeout", + defaultConfig.scriptExecutorConfig.ExecutionTimeLimit, + "timeout value for locally executed scripts. default: 10s") }).ValidateFlags(func() error { if builder.supportsObserver && (builder.PublicNetworkConfig.BindAddress == cmd.NotSet || builder.PublicNetworkConfig.BindAddress == "") { @@ -1681,7 +1717,6 @@ func (builder *FlowAccessNodeBuilder) initPublicLibp2pNode(networkKey crypto.Pri networkKey, builder.SporkID, builder.IdentityProvider, - builder.FlowConfig.NetworkConfig.GossipSubConfig.GossipSubScoringRegistryConfig, &builder.FlowConfig.NetworkConfig.ResourceManager, &p2pconfig.PeerManagerConfig{ // TODO: eventually, we need pruning enabled even on public network. However, it needs a modified version of From e0cc367a3e556e9dfdc5be0536d4dcc7896d592f Mon Sep 17 00:00:00 2001 From: Yahya Hassanzadeh Date: Mon, 4 Dec 2023 09:58:18 -0800 Subject: [PATCH 45/67] lint fix --- cmd/observer/node_builder/observer_builder.go | 1 - ...ntrol_message_validation_inspector_test.go | 94 +++++++++++-------- network/p2p/test/fixtures.go | 1 - 3 files changed, 53 insertions(+), 43 deletions(-) diff --git a/cmd/observer/node_builder/observer_builder.go b/cmd/observer/node_builder/observer_builder.go index a63e867c0b8..e184177c778 100644 --- a/cmd/observer/node_builder/observer_builder.go +++ b/cmd/observer/node_builder/observer_builder.go @@ -754,7 +754,6 @@ func (builder *ObserverServiceBuilder) initPublicLibp2pNode(networkKey crypto.Pr networkKey, builder.SporkID, builder.IdentityProvider, - builder.FlowConfig.NetworkConfig.GossipSubConfig.GossipSubScoringRegistryConfig, &builder.FlowConfig.NetworkConfig.ResourceManager, p2pconfig.PeerManagerDisableConfig(), // disable peer manager for observer node. &p2p.DisallowListCacheConfig{ diff --git a/network/p2p/inspector/validation/control_message_validation_inspector_test.go b/network/p2p/inspector/validation/control_message_validation_inspector_test.go index 798d4f64fdd..966bcb927b4 100644 --- a/network/p2p/inspector/validation/control_message_validation_inspector_test.go +++ b/network/p2p/inspector/validation/control_message_validation_inspector_test.go @@ -123,7 +123,7 @@ func TestControlMessageValidationInspector_truncateRPC(t *testing.T) { distributor.On("Distribute", mock.AnythingOfType("*p2p.InvCtrlMsgNotif")).Return(nil).Twice() inspector.Start(signalerCtx) - //unittest.RequireCloseBefore(t, inspector.Ready(), 100*time.Millisecond, "inspector did not start") + // unittest.RequireCloseBefore(t, inspector.Ready(), 100*time.Millisecond, "inspector did not start") // topic validation not performed, so we can use random strings prunesGreaterThanMaxSampleSize := unittest.P2PRPCFixture(unittest.WithPrunes(unittest.P2PRPCPruneFixtures(unittest.IdentifierListFixture(2000).Strings()...)...)) require.Greater(t, len(prunesGreaterThanMaxSampleSize.GetControl().GetPrune()), graftPruneMessageMaxSampleSize) @@ -145,7 +145,7 @@ func TestControlMessageValidationInspector_truncateRPC(t *testing.T) { t.Run("truncateIHaveMessages should truncate iHave messages as expected", func(t *testing.T) { maxSampleSize := 1000 inspector, signalerCtx, cancel, distributor, rpcTracker, _, _, _ := inspectorFixture(t, func(params *validation.InspectorParams) { - params.Config.IHaveRPCInspectionConfig.MaxSampleSize = maxSampleSize + params.Config.IHave.MaxSampleSize = maxSampleSize }) // topic validation is ignored set any topic oracle rpcTracker.On("LastHighestIHaveRPCSize").Return(int64(100)).Maybe() @@ -175,7 +175,7 @@ func TestControlMessageValidationInspector_truncateRPC(t *testing.T) { t.Run("truncateIHaveMessageIds should truncate iHave message ids as expected", func(t *testing.T) { maxMessageIDSampleSize := 1000 inspector, signalerCtx, cancel, distributor, rpcTracker, _, _, _ := inspectorFixture(t, func(params *validation.InspectorParams) { - params.Config.IHaveRPCInspectionConfig.MaxMessageIDSampleSize = maxMessageIDSampleSize + params.Config.IHave.MaxMessageIDSampleSize = maxMessageIDSampleSize }) // topic validation is ignored set any topic oracle rpcTracker.On("LastHighestIHaveRPCSize").Return(int64(100)).Maybe() @@ -211,7 +211,7 @@ func TestControlMessageValidationInspector_truncateRPC(t *testing.T) { t.Run("truncateIWantMessages should truncate iWant messages as expected", func(t *testing.T) { maxSampleSize := uint(100) inspector, signalerCtx, cancel, distributor, rpcTracker, _, _, _ := inspectorFixture(t, func(params *validation.InspectorParams) { - params.Config.IWantRPCInspectionConfig.MaxSampleSize = maxSampleSize + params.Config.IWant.MaxSampleSize = maxSampleSize }) // topic validation is ignored set any topic oracle rpcTracker.On("LastHighestIHaveRPCSize").Return(int64(100)).Maybe() @@ -239,7 +239,7 @@ func TestControlMessageValidationInspector_truncateRPC(t *testing.T) { t.Run("truncateIWantMessageIds should truncate iWant message ids as expected", func(t *testing.T) { maxMessageIDSampleSize := 1000 inspector, signalerCtx, cancel, distributor, rpcTracker, _, _, _ := inspectorFixture(t, func(params *validation.InspectorParams) { - params.Config.IWantRPCInspectionConfig.MaxMessageIDSampleSize = maxMessageIDSampleSize + params.Config.IWant.MaxMessageIDSampleSize = maxMessageIDSampleSize }) // topic validation is ignored set any topic oracle rpcTracker.On("LastHighestIHaveRPCSize").Return(int64(100)).Maybe() @@ -486,9 +486,9 @@ func TestControlMessageValidationInspector_processInspectRPCReq(t *testing.T) { t.Run("inspectIWantMessages should disseminate invalid control message notification for iWant messages when cache misses exceeds allowed threshold", func(t *testing.T) { cacheMissCheckSize := 1000 inspector, signalerCtx, cancel, distributor, rpcTracker, _, _, _ := inspectorFixture(t, func(params *validation.InspectorParams) { - params.Config.CacheMissCheckSize = cacheMissCheckSize + params.Config.IWant.CacheMissCheckSize = cacheMissCheckSize // set high cache miss threshold to ensure we only disseminate notification when it is exceeded - params.Config.IWantRPCInspectionConfig.CacheMissThreshold = .9 + params.Config.IWant.CacheMissThreshold = .9 }) // oracle must be set even though iWant messages do not have topic IDs inspectMsgRpc := unittest.P2PRPCFixture(unittest.WithIWants(unittest.P2PRPCIWantFixtures(cacheMissCheckSize+1, 100)...)) @@ -520,39 +520,40 @@ func TestControlMessageValidationInspector_processInspectRPCReq(t *testing.T) { stopInspector(t, cancel, inspector) }) - t.Run("inspectIWantMessages should not disseminate invalid control message notification for iWant messages when cache misses exceeds allowed threshold if cache miss check size not exceeded", func(t *testing.T) { - inspector, signalerCtx, cancel, distributor, rpcTracker, _, _, _ := inspectorFixture(t, func(params *validation.InspectorParams) { - // if size of iwants not greater than 10 cache misses will not be checked - params.Config.CacheMissCheckSize = 10 - // set high cache miss threshold to ensure we only disseminate notification when it is exceeded - params.Config.IWantRPCInspectionConfig.CacheMissThreshold = .9 - }) - // oracle must be set even though iWant messages do not have topic IDs - defer distributor.AssertNotCalled(t, "Distribute") + t.Run("inspectIWantMessages should not disseminate invalid control message notification for iWant messages when cache misses exceeds allowed threshold if cache miss check size not exceeded", + func(t *testing.T) { + inspector, signalerCtx, cancel, distributor, rpcTracker, _, _, _ := inspectorFixture(t, func(params *validation.InspectorParams) { + // if size of iwants not greater than 10 cache misses will not be checked + params.Config.IWant.CacheMissCheckSize = 10 + // set high cache miss threshold to ensure we only disseminate notification when it is exceeded + params.Config.IWant.CacheMissThreshold = .9 + }) + // oracle must be set even though iWant messages do not have topic IDs + defer distributor.AssertNotCalled(t, "Distribute") + + msgIds := unittest.IdentifierListFixture(100).Strings() + inspectMsgRpc := unittest.P2PRPCFixture(unittest.WithIWants(unittest.P2PRPCIWantFixture(msgIds...))) + rpcTracker.On("LastHighestIHaveRPCSize").Return(int64(100)).Maybe() + // return false each time to eventually force a notification to be disseminated when the cache miss count finally exceeds the 90% threshold + rpcTracker.On("WasIHaveRPCSent", mock.AnythingOfType("string")).Return(false).Run(func(args mock.Arguments) { + id, ok := args[0].(string) + require.True(t, ok) + require.Contains(t, msgIds, id) + }) + + from := unittest.PeerIdFixture(t) + inspector.Start(signalerCtx) - msgIds := unittest.IdentifierListFixture(100).Strings() - inspectMsgRpc := unittest.P2PRPCFixture(unittest.WithIWants(unittest.P2PRPCIWantFixture(msgIds...))) - rpcTracker.On("LastHighestIHaveRPCSize").Return(int64(100)).Maybe() - // return false each time to eventually force a notification to be disseminated when the cache miss count finally exceeds the 90% threshold - rpcTracker.On("WasIHaveRPCSent", mock.AnythingOfType("string")).Return(false).Run(func(args mock.Arguments) { - id, ok := args[0].(string) - require.True(t, ok) - require.Contains(t, msgIds, id) + require.NoError(t, inspector.Inspect(from, inspectMsgRpc)) + // sleep for 1 second to ensure rpc's is processed + time.Sleep(time.Second) + stopInspector(t, cancel, inspector) }) - from := unittest.PeerIdFixture(t) - inspector.Start(signalerCtx) - - require.NoError(t, inspector.Inspect(from, inspectMsgRpc)) - // sleep for 1 second to ensure rpc's is processed - time.Sleep(time.Second) - stopInspector(t, cancel, inspector) - }) - t.Run("inspectRpcPublishMessages should disseminate invalid control message notification when invalid pubsub messages count greater than configured RpcMessageErrorThreshold", func(t *testing.T) { errThreshold := 500 inspector, signalerCtx, cancel, distributor, _, sporkID, _, topicProviderOracle := inspectorFixture(t, func(params *validation.InspectorParams) { - params.Config.RpcMessageErrorThreshold = errThreshold + params.Config.MessageErrorThreshold = errThreshold }) // create unknown topic unknownTopic := channels.Topic(fmt.Sprintf("%s/%s", unittest.IdentifierFixture(), sporkID)).String() @@ -592,7 +593,7 @@ func TestControlMessageValidationInspector_processInspectRPCReq(t *testing.T) { t.Run("inspectRpcPublishMessages should disseminate invalid control message notification when subscription missing for topic", func(t *testing.T) { errThreshold := 500 inspector, signalerCtx, cancel, distributor, _, sporkID, _, _ := inspectorFixture(t, func(params *validation.InspectorParams) { - params.Config.RpcMessageErrorThreshold = errThreshold + params.Config.MessageErrorThreshold = errThreshold }) pubsubMsgs := unittest.GossipSubMessageFixtures(errThreshold+1, fmt.Sprintf("%s/%s", channels.TestNetworkChannel, sporkID)) from := unittest.PeerIdFixture(t) @@ -610,7 +611,7 @@ func TestControlMessageValidationInspector_processInspectRPCReq(t *testing.T) { errThreshold := 500 inspector, signalerCtx, cancel, distributor, _, _, _, _ := inspectorFixture(t, func(params *validation.InspectorParams) { // 5 invalid pubsub messages will force notification dissemination - params.Config.RpcMessageErrorThreshold = errThreshold + params.Config.MessageErrorThreshold = errThreshold }) pubsubMsgs := unittest.GossipSubMessageFixtures(errThreshold+1, "") rpc := unittest.P2PRPCFixture(unittest.WithPubsubMessages(pubsubMsgs...)) @@ -707,7 +708,7 @@ func TestNewControlMsgValidationInspector_validateClusterPrefixedTopic(t *testin t.Run("validateClusterPrefixedTopic should not return error if cluster prefixed hard threshold not exceeded for unknown cluster ids", func(t *testing.T) { inspector, signalerCtx, cancel, distributor, _, sporkID, idProvider, _ := inspectorFixture(t, func(params *validation.InspectorParams) { // set hard threshold to small number , ensure that a single unknown cluster prefix id does not cause a notification to be disseminated - params.Config.ClusterPrefixHardThreshold = 2 + params.Config.ClusterPrefixedMessage.HardThreshold = 2 }) defer distributor.AssertNotCalled(t, "Distribute") clusterID := flow.ChainID(unittest.IdentifierFixture().String()) @@ -744,7 +745,7 @@ func TestNewControlMsgValidationInspector_validateClusterPrefixedTopic(t *testin t.Run("validateClusterPrefixedTopic should return error if cluster prefixed hard threshold exceeded for unknown cluster ids", func(t *testing.T) { inspector, signalerCtx, cancel, distributor, _, sporkID, idProvider, topicProviderOracle := inspectorFixture(t, func(params *validation.InspectorParams) { // the 11th unknown cluster ID error should cause an error - params.Config.ClusterPrefixHardThreshold = 10 + params.Config.ClusterPrefixedMessage.HardThreshold = 10 }) clusterID := flow.ChainID(unittest.IdentifierFixture().String()) clusterPrefixedTopic := channels.Topic(fmt.Sprintf("%s/%s", channels.SyncCluster(clusterID), sporkID)).String() @@ -804,7 +805,11 @@ func invalidTopics(t *testing.T, sporkID flow.Identifier) (string, string, strin } // checkNotificationFunc returns util func used to ensure invalid control message notification disseminated contains expected information. -func checkNotificationFunc(t *testing.T, expectedPeerID peer.ID, expectedMsgType p2pmsg.ControlMessageType, isExpectedErr func(err error) bool, topicType p2p.CtrlMsgTopicType) func(args mock.Arguments) { +func checkNotificationFunc(t *testing.T, + expectedPeerID peer.ID, + expectedMsgType p2pmsg.ControlMessageType, + isExpectedErr func(err error) bool, + topicType p2p.CtrlMsgTopicType) func(args mock.Arguments) { return func(args mock.Arguments) { notification, ok := args[0].(*p2p.InvCtrlMsgNotif) require.True(t, ok) @@ -815,7 +820,14 @@ func checkNotificationFunc(t *testing.T, expectedPeerID peer.ID, expectedMsgType } } -func inspectorFixture(t *testing.T, opts ...func(params *validation.InspectorParams)) (*validation.ControlMsgValidationInspector, *irrecoverable.MockSignalerContext, context.CancelFunc, *mockp2p.GossipSubInspectorNotificationDistributor, *mockp2p.RpcControlTracking, flow.Identifier, *mockmodule.IdentityProvider, *internal.MockUpdatableTopicProvider) { +func inspectorFixture(t *testing.T, opts ...func(params *validation.InspectorParams)) (*validation.ControlMsgValidationInspector, + *irrecoverable.MockSignalerContext, + context.CancelFunc, + *mockp2p.GossipSubInspectorNotificationDistributor, + *mockp2p.RpcControlTracking, + flow.Identifier, + *mockmodule.IdentityProvider, + *internal.MockUpdatableTopicProvider) { sporkID := unittest.IdentifierFixture() flowConfig, err := config.DefaultConfig() require.NoError(t, err) @@ -827,7 +839,7 @@ func inspectorFixture(t *testing.T, opts ...func(params *validation.InspectorPar params := &validation.InspectorParams{ Logger: unittest.Logger(), SporkID: sporkID, - Config: &flowConfig.NetworkConfig.GossipSubRPCValidationInspectorConfigs, + Config: &flowConfig.NetworkConfig.GossipSub.RpcInspector.Validation, Distributor: distributor, IdProvider: idProvider, HeroCacheMetricsFactory: metrics.NewNoopHeroCacheMetricsFactory(), diff --git a/network/p2p/test/fixtures.go b/network/p2p/test/fixtures.go index da947967e7f..11132bd6949 100644 --- a/network/p2p/test/fixtures.go +++ b/network/p2p/test/fixtures.go @@ -133,7 +133,6 @@ func NodeFixture(t *testing.T, parameters.Key, sporkID, parameters.IdProvider, - defaultFlowConfig.NetworkConfig.GossipSubConfig.GossipSubScoringRegistryConfig, ¶meters.FlowConfig.NetworkConfig.ResourceManager, parameters.PeerManagerConfig, &p2p.DisallowListCacheConfig{ From 94fafeee45d84604c591b0d3b0f9e3af728ee223 Mon Sep 17 00:00:00 2001 From: Yahya Hassanzadeh Date: Mon, 4 Dec 2023 15:01:45 -0800 Subject: [PATCH 46/67] adds test utils --- utils/unittest/math.go | 61 +++++++++++++++++++++++++++++++++++++ utils/unittest/math_test.go | 36 ++++++++++++++++++++++ 2 files changed, 97 insertions(+) create mode 100644 utils/unittest/math.go create mode 100644 utils/unittest/math_test.go diff --git a/utils/unittest/math.go b/utils/unittest/math.go new file mode 100644 index 00000000000..eb44762247d --- /dev/null +++ b/utils/unittest/math.go @@ -0,0 +1,61 @@ +package unittest + +import ( + "math" + "testing" + + "github.com/stretchr/testify/require" +) + +// RequireNumericallyClose is a wrapper around require.Equal that allows for a small epsilon difference between +// two floats. This is useful when comparing floats that are the result of a computation. For example, when comparing +// the result of a computation with a constant. +// The epsilon is calculated as: +// +// epsilon = max(|a|, |b|) * epsilon +// +// Example: +// +// RequireNumericallyClose(t, 1.0, 1.1, 0.1) // passes since 1.0 * 0.1 = 0.1 < 0.1 +// RequireNumericallyClose(t, 1.0, 1.1, 0.01) // fails since 1.0 * 0.01 = 0.01 < 0.1 +// RequireNumericallyClose(t, 1.0, 1.1, 0.11) // fails since 1.1 * 0.11 = 0.121 > 0.1 +// +// Args: +// +// t: the testing.TB instance +// a: the first float +// b: the second float +func RequireNumericallyClose(t testing.TB, a, b float64, epsilon float64, msgAndArgs ...interface{}) { + require.True(t, AreNumericallyClose(a, b, epsilon), msgAndArgs...) +} + +// AreNumericallyClose returns true if the two floats are within epsilon of each other. +// The epsilon is calculated as: +// +// epsilon = max(|a|, |b|) * epsilon +// +// Example: +// +// AreNumericallyClose(1.0, 1.1, 0.1) // true since 1.0 * 0.1 = 0.1 < 0.1 +// AreNumericallyClose(1.0, 1.1, 0.01) // false since 1.0 * 0.01 = 0.01 < 0.1 +// AreNumericallyClose(1.0, 1.1, 0.11) // false since 1.1 * 0.11 = 0.121 > 0.1 +// +// Args: +// a: the first float +// b: the second float +// epsilon: the epsilon value +// Returns: +// true if the two floats are within epsilon of each other +// false otherwise +func AreNumericallyClose(a, b float64, epsilon float64) bool { + if a == float64(0) { + return math.Abs(b) <= epsilon + } + if b == float64(0) { + return math.Abs(a) <= epsilon + } + + nominator := math.Abs(a - b) + denominator := math.Max(math.Abs(a), math.Abs(b)) + return nominator/denominator <= epsilon +} diff --git a/utils/unittest/math_test.go b/utils/unittest/math_test.go new file mode 100644 index 00000000000..bfbc4547d91 --- /dev/null +++ b/utils/unittest/math_test.go @@ -0,0 +1,36 @@ +package unittest_test + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/onflow/flow-go/utils/unittest" +) + +func TestAreNumericallyClose(t *testing.T) { + tests := []struct { + name string + a float64 + b float64 + epsilon float64 + expected bool + }{ + {"close enough under epsilon", 1.0, 1.1, 0.1, true}, + {"not close under epsilon", 1.0, 1.1, 0.01, false}, + {"equal values", 2.0, 2.0, 0.1, true}, + {"zero epsilon with equal values", 2.0, 2.0, 0.0, true}, + {"zero epsilon with different values", 2.0, 2.1, 0.0, false}, + {"first value zero", 0, 0.1, 0.1, true}, + {"both values zero", 0, 0, 0.1, true}, + {"negative values close enough", -1.0, -1.1, 0.1, true}, + {"negative values not close enough", -1.0, -1.2, 0.1, false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + actual := unittest.AreNumericallyClose(tt.a, tt.b, tt.epsilon) + require.Equal(t, tt.expected, actual, "test Failed: %s", tt.name) + }) + } +} From b9e887b863c521eb9c5c8ec33cf83dc6264e2510 Mon Sep 17 00:00:00 2001 From: Yahya Hassanzadeh Date: Mon, 4 Dec 2023 15:01:59 -0800 Subject: [PATCH 47/67] fixes registry tests --- network/p2p/p2pconf/gossipsub.go | 2 +- network/p2p/scoring/decay_test.go | 4 +- network/p2p/scoring/registry_test.go | 240 +++++++++++++++++---------- 3 files changed, 151 insertions(+), 95 deletions(-) diff --git a/network/p2p/p2pconf/gossipsub.go b/network/p2p/p2pconf/gossipsub.go index d250d60e369..5767418aa61 100644 --- a/network/p2p/p2pconf/gossipsub.go +++ b/network/p2p/p2pconf/gossipsub.go @@ -84,7 +84,7 @@ const ( DecayIntervalKey = "decay-interval" ) -// Parameters are the parameters for the score option. +// ScoringParameters are the parameters for the score option. // Parameters are "numerical values" that are used to compute or build components that compute the score of a peer in GossipSub system. type ScoringParameters struct { AppSpecificScore AppSpecificScoreParameters `validate:"required" mapstructure:"app-specific-score"` diff --git a/network/p2p/scoring/decay_test.go b/network/p2p/scoring/decay_test.go index 281ed194f15..2a15b60b754 100644 --- a/network/p2p/scoring/decay_test.go +++ b/network/p2p/scoring/decay_test.go @@ -287,7 +287,7 @@ func TestDefaultDecayFunction(t *testing.T) { record: p2p.GossipSubSpamRecord{ Penalty: -100, Decay: 0.8, - LastDecayAdjustment: time.Now().Add(-flowConfig.NetworkConfig.GossipSubConfig.GossipSubScoringRegistryConfig.PenaltyDecayEvaluationPeriod), + LastDecayAdjustment: time.Now().Add(-flowConfig.NetworkConfig.GossipSub.ScoringParameters.SpamRecordCache.PenaltyDecayEvaluationPeriod), }, lastUpdated: time.Now(), }, @@ -299,7 +299,7 @@ func TestDefaultDecayFunction(t *testing.T) { }, }, } - scoringRegistryConfig := flowConfig.NetworkConfig.GossipSubConfig.GossipSubScoringRegistryConfig + scoringRegistryConfig := flowConfig.NetworkConfig.GossipSub.ScoringParameters.SpamRecordCache decayFunc := scoring.DefaultDecayFunction(scoringRegistryConfig.PenaltyDecaySlowdownThreshold, scoringRegistryConfig.DecayRateReductionFactor, scoringRegistryConfig.PenaltyDecayEvaluationPeriod) for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { diff --git a/network/p2p/scoring/registry_test.go b/network/p2p/scoring/registry_test.go index 68f2e2c35d6..5968c922f93 100644 --- a/network/p2p/scoring/registry_test.go +++ b/network/p2p/scoring/registry_test.go @@ -14,6 +14,7 @@ import ( "github.com/stretchr/testify/require" "github.com/onflow/flow-go/config" + "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module/irrecoverable" "github.com/onflow/flow-go/module/metrics" "github.com/onflow/flow-go/module/mock" @@ -34,9 +35,12 @@ import ( func TestScoreRegistry_FreshStart(t *testing.T) { peerID := peer.ID("peer-1") - reg, spamRecords, appScoreCache := newGossipSubAppSpecificScoreRegistry(t, - withScoreTTL(100*time.Millisecond), // refresh cached app-specific score every 100 milliseconds to speed up the test. - withStakedIdentity(peerID), + cfg, err := config.DefaultConfig() + require.NoError(t, err) + // refresh cached app-specific score every 100 milliseconds to speed up the test. + cfg.NetworkConfig.GossipSub.ScoringParameters.AppSpecificScore.ScoreTTL = 100 * time.Millisecond + + reg, spamRecords, appScoreCache := newGossipSubAppSpecificScoreRegistry(t, cfg.NetworkConfig.GossipSub.ScoringParameters, withStakedIdentities(peerID), withValidSubscriptions(peerID)) // starts the registry. @@ -113,10 +117,12 @@ func TestScoreRegistry_PeerWithSpamRecord(t *testing.T) { func testScoreRegistryPeerWithSpamRecord(t *testing.T, messageType p2pmsg.ControlMessageType, expectedPenalty float64) { peerID := peer.ID("peer-1") - reg, spamRecords, appScoreCache := newGossipSubAppSpecificScoreRegistry( - t, - withScoreTTL(100*time.Millisecond), // refresh cached app-specific score every 100 milliseconds to speed up the test. - withStakedIdentity(peerID), + cfg, err := config.DefaultConfig() + require.NoError(t, err) + // refresh cached app-specific score every 100 milliseconds to speed up the test. + cfg.NetworkConfig.GossipSub.ScoringParameters.AppSpecificScore.ScoreTTL = 10 * time.Millisecond + + reg, spamRecords, appScoreCache := newGossipSubAppSpecificScoreRegistry(t, cfg.NetworkConfig.GossipSub.ScoringParameters, withStakedIdentities(peerID), withValidSubscriptions(peerID)) // starts the registry. @@ -210,9 +216,12 @@ func TestScoreRegistry_SpamRecordWithUnknownIdentity(t *testing.T) { // based on the given control message type and the state of the peer ID (unknown identity and spam record presence). func testScoreRegistrySpamRecordWithUnknownIdentity(t *testing.T, messageType p2pmsg.ControlMessageType, expectedPenalty float64) { peerID := peer.ID("peer-1") - reg, spamRecords, appScoreCache := newGossipSubAppSpecificScoreRegistry( - t, - withScoreTTL(100*time.Millisecond), // refresh cached app-specific score every 100 milliseconds to speed up the test. + cfg, err := config.DefaultConfig() + require.NoError(t, err) + // refresh cached app-specific score every 100 milliseconds to speed up the test. + cfg.NetworkConfig.GossipSub.ScoringParameters.AppSpecificScore.ScoreTTL = 100 * time.Millisecond + + reg, spamRecords, appScoreCache := newGossipSubAppSpecificScoreRegistry(t, cfg.NetworkConfig.GossipSub.ScoringParameters, withUnknownIdentity(peerID), withValidSubscriptions(peerID)) @@ -259,31 +268,20 @@ func testScoreRegistrySpamRecordWithUnknownIdentity(t *testing.T, messageType p2 // the peer has spam record as well as an unknown identity. Hence, the app specific score should be the spam penalty // and the staking penalty. // As the app specific score in the cache and spam penalty in the spamRecords are updated at different times, we account for 0.1% error. - nominator := math.Abs(expectedPenalty + scoring.DefaultUnknownIdentityPenalty - score) - denominator := math.Max(expectedPenalty+scoring.DefaultUnknownIdentityPenalty, score) - return math.Abs(nominator/denominator) < 0.001 - }, 5*time.Second, 100*time.Millisecond) + return unittest.AreNumericallyClose(expectedPenalty+scoring.DefaultUnknownIdentityPenalty, score, 0.01) + }, 5*time.Second, 10*time.Millisecond) // the app specific score should now be updated in the cache. score, updated, exists = appScoreCache.Get(peerID) // get the score from the cache. require.True(t, exists) require.True(t, updated.After(queryTime)) - nominator := math.Abs(expectedPenalty + scoring.DefaultUnknownIdentityPenalty - score) - denominator := math.Max(expectedPenalty+scoring.DefaultUnknownIdentityPenalty, score) - require.True(t, math.Abs(nominator/denominator) < 0.001) + unittest.RequireNumericallyClose(t, expectedPenalty+scoring.DefaultUnknownIdentityPenalty, score, 0.01) + assert.Equal(t, scoring.InitAppScoreRecordState().Decay, record.Decay) // decay should be initialized to the initial state. // stop the registry. cancel() unittest.RequireCloseBefore(t, reg.Done(), 1*time.Second, "failed to stop GossipSubAppSpecificScoreRegistry") - assert.True(t, ok) - assert.NoError(t, err) - assert.Less(t, math.Abs(expectedPenalty-record.Penalty), 10e-3) // penalty should be updated to -10, we account for decay. - assert.Equal(t, scoring.InitAppScoreRecordState().Decay, record.Decay) // decay should be initialized to the initial state. - // the peer has spam record as well as an unknown identity. Hence, the app specific score should be the spam penalty - // and the staking penalty. - score = reg.AppSpecificScoreFunc()(peerID) - assert.Less(t, math.Abs(expectedPenalty+scoring.DefaultUnknownIdentityPenalty-score), 10e-3) } // TestScoreRegistry_SpamRecordWithSubscriptionPenalty is a test suite for verifying the behavior of the ScoreRegistry @@ -321,10 +319,12 @@ func TestScoreRegistry_SpamRecordWithSubscriptionPenalty(t *testing.T) { // the spam record and the invalid subscription status of the peer. func testScoreRegistrySpamRecordWithSubscriptionPenalty(t *testing.T, messageType p2pmsg.ControlMessageType, expectedPenalty float64) { peerID := peer.ID("peer-1") - reg, spamRecords, appScoreCache := newGossipSubAppSpecificScoreRegistry( - t, - withScoreTTL(100*time.Millisecond), // refresh cached app-specific score every 100 milliseconds to speed up the test. - withStakedIdentity(peerID), + cfg, err := config.DefaultConfig() + require.NoError(t, err) + // refresh cached app-specific score every 100 milliseconds to speed up the test. + cfg.NetworkConfig.GossipSub.ScoringParameters.AppSpecificScore.ScoreTTL = 100 * time.Millisecond + + reg, spamRecords, appScoreCache := newGossipSubAppSpecificScoreRegistry(t, cfg.NetworkConfig.GossipSub.ScoringParameters, withStakedIdentities(peerID), withInvalidSubscriptions(peerID)) // starts the registry. @@ -371,9 +371,9 @@ func testScoreRegistrySpamRecordWithSubscriptionPenalty(t *testing.T, messageTyp // and the staking penalty. // As the app specific score in the cache and spam penalty in the spamRecords are updated at different times, we account for 0.1% error. nominator := math.Abs(expectedPenalty + scoring.DefaultInvalidSubscriptionPenalty - score) - denominator := math.Max(expectedPenalty+scoring.DefaultInvalidSubscriptionPenalty, score) - return math.Abs(nominator/denominator) < 0.001 - }, 5*time.Second, 100*time.Millisecond) + denominator := math.Max(math.Abs(expectedPenalty+scoring.DefaultInvalidSubscriptionPenalty), math.Abs(score)) + return math.Abs(nominator/denominator) < 0.01 + }, 5*time.Second, 10*time.Millisecond) // the app specific score should now be updated in the cache. score, updated, exists = appScoreCache.Get(peerID) // get the score from the cache. @@ -382,23 +382,22 @@ func testScoreRegistrySpamRecordWithSubscriptionPenalty(t *testing.T, messageTyp nominator := math.Abs(expectedPenalty + scoring.DefaultInvalidSubscriptionPenalty - score) denominator := math.Max(expectedPenalty+scoring.DefaultInvalidSubscriptionPenalty, score) - require.True(t, math.Abs(nominator/denominator) < 0.001) + require.True(t, math.Abs(nominator/denominator) < 0.01) // stop the registry. cancel() unittest.RequireCloseBefore(t, reg.Done(), 1*time.Second, "failed to stop GossipSubAppSpecificScoreRegistry") - // the peer has spam record as well as an unknown identity. Hence, the app specific score should be the spam penalty - // and the staking penalty. - score = reg.AppSpecificScoreFunc()(peerID) - assert.Less(t, math.Abs(expectedPenalty+scoring.DefaultInvalidSubscriptionPenalty-score), 10e-3) } // TestSpamPenaltyDecaysInCache tests that the spam penalty records decay over time in the cache. func TestScoreRegistry_SpamPenaltyDecaysInCache(t *testing.T) { peerID := peer.ID("peer-1") - reg, _, _ := newGossipSubAppSpecificScoreRegistry(t, - withScoreTTL(100*time.Millisecond), // refresh cached app-specific score every 100 milliseconds to speed up the test. - withStakedIdentity(peerID), + cfg, err := config.DefaultConfig() + require.NoError(t, err) + // refresh cached app-specific score every 100 milliseconds to speed up the test. + cfg.NetworkConfig.GossipSub.ScoringParameters.AppSpecificScore.ScoreTTL = 100 * time.Millisecond + + reg, _, _ := newGossipSubAppSpecificScoreRegistry(t, cfg.NetworkConfig.GossipSub.ScoringParameters, withStakedIdentities(peerID), withValidSubscriptions(peerID)) // starts the registry. @@ -473,10 +472,12 @@ func TestScoreRegistry_SpamPenaltyDecaysInCache(t *testing.T) { // a peer is set back to zero, its app specific penalty is also reset to the initial state. func TestScoreRegistry_SpamPenaltyDecayToZero(t *testing.T) { peerID := peer.ID("peer-1") - reg, spamRecords, _ := newGossipSubAppSpecificScoreRegistry( - t, - withScoreTTL(100*time.Millisecond), // refresh cached app-specific score every 100 milliseconds to speed up the test. - withStakedIdentity(peerID), + cfg, err := config.DefaultConfig() + require.NoError(t, err) + // refresh cached app-specific score every 100 milliseconds to speed up the test. + cfg.NetworkConfig.GossipSub.ScoringParameters.AppSpecificScore.ScoreTTL = 100 * time.Millisecond + + reg, spamRecords, _ := newGossipSubAppSpecificScoreRegistry(t, cfg.NetworkConfig.GossipSub.ScoringParameters, withStakedIdentities(peerID), withValidSubscriptions(peerID), withInitFunction(func() p2p.GossipSubSpamRecord { return p2p.GossipSubSpamRecord{ @@ -532,10 +533,13 @@ func TestScoreRegistry_SpamPenaltyDecayToZero(t *testing.T) { // is persisted. This is because the unknown identity penalty is not decayed. func TestScoreRegistry_PersistingUnknownIdentityPenalty(t *testing.T) { peerID := peer.ID("peer-1") - reg, spamRecords, _ := newGossipSubAppSpecificScoreRegistry( - t, - withScoreTTL(100*time.Millisecond), // refresh cached app-specific score every 100 milliseconds to speed up the test. - withUnknownIdentity(peerID), // the peer id has an unknown identity. + + cfg, err := config.DefaultConfig() + require.NoError(t, err) + // refresh cached app-specific score every 100 milliseconds to speed up the test. + cfg.NetworkConfig.GossipSub.ScoringParameters.AppSpecificScore.ScoreTTL = 100 * time.Millisecond + + reg, spamRecords, _ := newGossipSubAppSpecificScoreRegistry(t, cfg.NetworkConfig.GossipSub.ScoringParameters, withUnknownIdentity(peerID), // the peer id has an unknown identity. withValidSubscriptions(peerID), withInitFunction(func() p2p.GossipSubSpamRecord { return p2p.GossipSubSpamRecord{ @@ -600,10 +604,13 @@ func TestScoreRegistry_PersistingUnknownIdentityPenalty(t *testing.T) { // is persisted. This is because the invalid subscription penalty is not decayed. func TestScoreRegistry_PersistingInvalidSubscriptionPenalty(t *testing.T) { peerID := peer.ID("peer-1") - reg, spamRecords, _ := newGossipSubAppSpecificScoreRegistry( - t, - withScoreTTL(100*time.Millisecond), // refresh cached app-specific score every 100 milliseconds to speed up the test. - withStakedIdentity(peerID), + + cfg, err := config.DefaultConfig() + require.NoError(t, err) + // refresh cached app-specific score every 100 milliseconds to speed up the test. + cfg.NetworkConfig.GossipSub.ScoringParameters.AppSpecificScore.ScoreTTL = 100 * time.Millisecond + + reg, spamRecords, _ := newGossipSubAppSpecificScoreRegistry(t, cfg.NetworkConfig.GossipSub.ScoringParameters, withStakedIdentities(peerID), withInvalidSubscriptions(peerID), // the peer id has an invalid subscription. withInitFunction(func() p2p.GossipSubSpamRecord { return p2p.GossipSubSpamRecord{ @@ -661,33 +668,38 @@ func TestScoreRegistry_PersistingInvalidSubscriptionPenalty(t *testing.T) { unittest.RequireCloseBefore(t, reg.Done(), 1*time.Second, "failed to stop GossipSubAppSpecificScoreRegistry") } -// TestSpamRecordDecayAdjustment ensures that spam record decay is increased each time a peers score reaches the scoring.IncreaseDecayThreshold eventually +// TestScoreRegistry_TestSpamRecordDecayAdjustment ensures that spam record decay is increased each time a peers score reaches the scoring.IncreaseDecayThreshold eventually // sustained misbehavior will result in the spam record decay reaching the minimum decay speed .99, and the decay speed is reset to the max decay speed .8. -func TestSpamRecordDecayAdjustment(t *testing.T) { - flowConfig, err := config.DefaultConfig() +func TestScoreRegistry_TestSpamRecordDecayAdjustment(t *testing.T) { + cfg, err := config.DefaultConfig() require.NoError(t, err) - scoringRegistryConfig := flowConfig.NetworkConfig.GossipSubConfig.GossipSubScoringRegistryConfig + // refresh cached app-specific score every 100 milliseconds to speed up the test. + cfg.NetworkConfig.GossipSub.ScoringParameters.AppSpecificScore.ScoreTTL = 100 * time.Millisecond // increase configured DecayRateReductionFactor so that the decay time is increased faster - scoringRegistryConfig.DecayRateReductionFactor = .1 - scoringRegistryConfig.PenaltyDecayEvaluationPeriod = time.Second + cfg.NetworkConfig.GossipSub.ScoringParameters.SpamRecordCache.DecayRateReductionFactor = .1 + cfg.NetworkConfig.GossipSub.ScoringParameters.SpamRecordCache.PenaltyDecayEvaluationPeriod = time.Second peer1 := unittest.PeerIdFixture(t) peer2 := unittest.PeerIdFixture(t) - reg, spamRecords := newScoringRegistry( - t, - scoringRegistryConfig, - withStakedIdentity(peer1), - withValidSubscriptions(peer1), - withStakedIdentity(peer2), - withValidSubscriptions(peer2)) + reg, spamRecords, _ := newGossipSubAppSpecificScoreRegistry(t, + cfg.NetworkConfig.GossipSub.ScoringParameters, + withStakedIdentities(peer1, peer2), + withValidSubscriptions(peer1, peer2)) + + ctx, cancel := context.WithCancel(context.Background()) + signalerCtx := irrecoverable.NewMockSignalerContext(t, ctx) + reg.Start(signalerCtx) + unittest.RequireCloseBefore(t, reg.Ready(), 1*time.Second, "failed to start GossipSubAppSpecificScoreRegistry") // initially, the spamRecords should not have the peer ids. assert.False(t, spamRecords.Has(peer1)) assert.False(t, spamRecords.Has(peer2)) // since the both peers do not have a spam record, their app specific score should be the max app specific reward, which // is the default reward for a staked peer that has valid subscriptions. - assert.Equal(t, scoring.MaxAppSpecificReward, reg.AppSpecificScoreFunc()(peer1)) - assert.Equal(t, scoring.MaxAppSpecificReward, reg.AppSpecificScoreFunc()(peer2)) + require.Eventually(t, func() bool { + // when the spam penalty is decayed to zero, the app specific penalty of the node should only contain the unknown identity penalty. + return scoring.MaxAppSpecificReward == reg.AppSpecificScoreFunc()(peer1) && scoring.MaxAppSpecificReward == reg.AppSpecificScoreFunc()(peer2) + }, 5*time.Second, 100*time.Millisecond) // simulate sustained malicious activity from peer1, eventually the decay speed // for a spam record should be reduced to the MinimumSpamPenaltyDecayFactor @@ -744,6 +756,10 @@ func TestSpamRecordDecayAdjustment(t *testing.T) { require.True(t, ok) return record.Decay == scoring.MinimumSpamPenaltyDecayFactor }, 5*time.Second, 500*time.Millisecond) + + // stop the registry. + cancel() + unittest.RequireCloseBefore(t, reg.Done(), 1*time.Second, "failed to stop GossipSubAppSpecificScoreRegistry") } // TestPeerSpamPenaltyClusterPrefixed evaluates the application-specific penalty calculation for a node when a spam record is present @@ -753,19 +769,36 @@ func TestSpamRecordDecayAdjustment(t *testing.T) { func TestPeerSpamPenaltyClusterPrefixed(t *testing.T) { ctlMsgTypes := p2pmsg.ControlMessageTypes() peerIds := unittest.PeerIdFixtures(t, len(ctlMsgTypes)) - opts := make([]scoringRegistryParamsOpt, 0) - for _, peerID := range peerIds { - opts = append(opts, withStakedIdentity(peerID), withValidSubscriptions(peerID)) - } - reg, spamRecords := newGossipSubAppSpecificScoreRegistry(t, opts...) + + cfg, err := config.DefaultConfig() + require.NoError(t, err) + // refresh cached app-specific score every 100 milliseconds to speed up the test. + cfg.NetworkConfig.GossipSub.ScoringParameters.AppSpecificScore.ScoreTTL = 100 * time.Millisecond + + reg, spamRecords, _ := newGossipSubAppSpecificScoreRegistry(t, + cfg.NetworkConfig.GossipSub.ScoringParameters, + withStakedIdentities(peerIds...), + withValidSubscriptions(peerIds...)) + + // starts the registry. + ctx, cancel := context.WithCancel(context.Background()) + signalerCtx := irrecoverable.NewMockSignalerContext(t, ctx) + reg.Start(signalerCtx) + unittest.RequireCloseBefore(t, reg.Ready(), 1*time.Second, "failed to start GossipSubAppSpecificScoreRegistry") for _, peerID := range peerIds { // initially, the spamRecords should not have the peer id. assert.False(t, spamRecords.Has(peerID)) - // since the peer id does not have a spam record, the app specific score should be the max app specific reward, which + // since the peer id does not have a spam record, the app specific score should (eventually, due to caching) be the max app specific reward, which // is the default reward for a staked peer that has valid subscriptions. - score := reg.AppSpecificScoreFunc()(peerID) - assert.Equal(t, scoring.MaxAppSpecificReward, score) + require.Eventually(t, func() bool { + // calling the app specific score function when there is no app specific score in the cache should eventually update the cache. + score := reg.AppSpecificScoreFunc()(peerID) + // since the peer id does not have a spam record, the app specific score should be the max app specific reward, which + // is the default reward for a staked peer that has valid subscriptions. + return score == scoring.MaxAppSpecificReward + }, 5*time.Second, 100*time.Millisecond) + } // Report consecutive misbehavior's for the specified peer ID. Two misbehavior's are reported concurrently: @@ -812,21 +845,49 @@ func TestPeerSpamPenaltyClusterPrefixed(t *testing.T) { assert.Less(t, math.Abs(expectedPenalty-score)/expectedPenalty, tolerance) } } + + // stop the registry. + cancel() + unittest.RequireCloseBefore(t, reg.Done(), 1*time.Second, "failed to stop GossipSubAppSpecificScoreRegistry") } -// withStakedIdentity returns a function that sets the identity provider to return an staked identity for the given peer id. +// withStakedIdentities returns a function that sets the identity provider to return staked identities for the given peer ids. // It is used for testing purposes, and causes the given peer id to benefit from the staked identity reward in GossipSub. -func withStakedIdentity(peerId peer.ID) func(cfg *scoring.GossipSubAppSpecificScoreRegistryConfig) { +func withStakedIdentities(peerIds ...peer.ID) func(cfg *scoring.GossipSubAppSpecificScoreRegistryConfig) { return func(cfg *scoring.GossipSubAppSpecificScoreRegistryConfig) { - cfg.IdProvider.(*mock.IdentityProvider).On("ByPeerID", peerId).Return(unittest.IdentityFixture(), true).Maybe() + cfg.IdProvider.(*mock.IdentityProvider).On("ByPeerID", testifymock.AnythingOfType("peer.ID")). + Return(func(pid peer.ID) *flow.Identity { + for _, peerID := range peerIds { + if peerID == pid { + return unittest.IdentityFixture() + } + } + return nil + }, func(pid peer.ID) bool { + for _, peerID := range peerIds { + if peerID == pid { + return true + } + } + return false + }).Maybe() } } -// withValidSubscriptions returns a function that sets the subscription validator to return nil for the given peer id. +// withValidSubscriptions returns a function that sets the subscription validator to return nil for the given peer ids. // It is used for testing purposes and causes the given peer id to never be penalized for subscribing to invalid topics. -func withValidSubscriptions(peer peer.ID) func(cfg *scoring.GossipSubAppSpecificScoreRegistryConfig) { +func withValidSubscriptions(peerIds ...peer.ID) func(cfg *scoring.GossipSubAppSpecificScoreRegistryConfig) { return func(cfg *scoring.GossipSubAppSpecificScoreRegistryConfig) { - cfg.Validator.(*mockp2p.SubscriptionValidator).On("CheckSubscribedToAllowedTopics", peer, testifymock.Anything).Return(nil).Maybe() + cfg.Validator.(*mockp2p.SubscriptionValidator). + On("CheckSubscribedToAllowedTopics", testifymock.AnythingOfType("peer.ID"), testifymock.Anything). + Return(func(pid peer.ID, _ flow.Role) error { + for _, peerID := range peerIds { + if peerID == pid { + return nil + } + } + return fmt.Errorf("invalid subscriptions") + }).Maybe() } } @@ -854,12 +915,6 @@ func withInitFunction(initFunction scoring.SpamRecordInitFunc) func(cfg *scoring } } -func withScoreTTL(scoreTTL time.Duration) func(cfg *scoring.GossipSubAppSpecificScoreRegistryConfig) { - return func(cfg *scoring.GossipSubAppSpecificScoreRegistryConfig) { - cfg.Parameters.ScoreTTL = scoreTTL - } -} - // newGossipSubAppSpecificScoreRegistry creates a new instance of GossipSubAppSpecificScoreRegistry along with its associated // GossipSubSpamRecordCache and AppSpecificScoreCache. This function is primarily used in testing scenarios to set up a controlled // environment for evaluating the behavior of the GossipSub scoring mechanism. @@ -881,13 +936,14 @@ func withScoreTTL(scoreTTL time.Duration) func(cfg *scoring.GossipSubAppSpecific // and an application-specific score cache with predefined sizes and functionalities. The function also configures the scoring parameters // with test-specific values, particularly modifying the ScoreTTL value for the purpose of the tests. The creation and configuration of // the GossipSubAppSpecificScoreRegistry are validated to ensure no errors occur during the process. -func newGossipSubAppSpecificScoreRegistry(t *testing.T, opts ...func(*scoring.GossipSubAppSpecificScoreRegistryConfig)) (*scoring.GossipSubAppSpecificScoreRegistry, +func newGossipSubAppSpecificScoreRegistry(t *testing.T, params p2pconf.ScoringParameters, opts ...func(*scoring.GossipSubAppSpecificScoreRegistryConfig)) (*scoring.GossipSubAppSpecificScoreRegistry, *netcache.GossipSubSpamRecordCache, *internal.AppSpecificScoreCache) { - cache := netcache.NewGossipSubSpamRecordCache(100, unittest.Logger(), metrics.NewNoopHeroCacheMetricsFactory(), scoring.DefaultDecayFunction()) + cache := netcache.NewGossipSubSpamRecordCache(100, + unittest.Logger(), + metrics.NewNoopHeroCacheMetricsFactory(), + scoring.DefaultDecayFunction(params.SpamRecordCache.PenaltyDecaySlowdownThreshold, params.SpamRecordCache.DecayRateReductionFactor, params.SpamRecordCache.PenaltyDecayEvaluationPeriod)) appSpecificScoreCache := internal.NewAppSpecificScoreCache(100, unittest.Logger(), metrics.NewNoopHeroCacheMetricsFactory()) - flowCfg, err := config.DefaultConfig() - require.NoError(t, err) validator := mockp2p.NewSubscriptionValidator(t) validator.On("Start", testifymock.Anything).Return().Maybe() @@ -910,7 +966,7 @@ func newGossipSubAppSpecificScoreRegistry(t *testing.T, opts ...func(*scoring.Go SpamRecordCacheFactory: func() p2p.GossipSubSpamRecordCache { return cache }, - Parameters: flowCfg.NetworkConfig.GossipSub.ScoringParameters.AppSpecificScore, + Parameters: params.AppSpecificScore, HeroCacheMetricsFactory: metrics.NewNoopHeroCacheMetricsFactory(), } for _, opt := range opts { From bba63ae6fc58fcbbfc454d6b3333f99a0b155bab Mon Sep 17 00:00:00 2001 From: Yahya Hassanzadeh Date: Mon, 4 Dec 2023 15:56:32 -0800 Subject: [PATCH 48/67] fixes tests --- network/p2p/scoring/registry.go | 3 ++- network/p2p/scoring/registry_test.go | 9 ++------- network/p2p/scoring/subscription_validator_test.go | 11 ++++++++++- utils/unittest/logging.go | 2 +- 4 files changed, 15 insertions(+), 10 deletions(-) diff --git a/network/p2p/scoring/registry.go b/network/p2p/scoring/registry.go index 9953168635d..5d23e1f6a4a 100644 --- a/network/p2p/scoring/registry.go +++ b/network/p2p/scoring/registry.go @@ -386,7 +386,8 @@ func (r *GossipSubAppSpecificScoreRegistry) stakingScore(pid peer.ID) (float64, func (r *GossipSubAppSpecificScoreRegistry) subscriptionPenalty(pid peer.ID, flowId flow.Identifier, role flow.Role) float64 { // checks if peer has any subscription violation. if err := r.validator.CheckSubscribedToAllowedTopics(pid, role); err != nil { - r.logger.Err(err). + r.logger.Warn(). + Err(err). Str("peer_id", p2plogging.PeerId(pid)). Hex("flow_id", logging.ID(flowId)). Bool(logging.KeySuspicious, true). diff --git a/network/p2p/scoring/registry_test.go b/network/p2p/scoring/registry_test.go index 5968c922f93..57ca8b6d8c6 100644 --- a/network/p2p/scoring/registry_test.go +++ b/network/p2p/scoring/registry_test.go @@ -370,19 +370,14 @@ func testScoreRegistrySpamRecordWithSubscriptionPenalty(t *testing.T, messageTyp // the peer has spam record as well as an unknown identity. Hence, the app specific score should be the spam penalty // and the staking penalty. // As the app specific score in the cache and spam penalty in the spamRecords are updated at different times, we account for 0.1% error. - nominator := math.Abs(expectedPenalty + scoring.DefaultInvalidSubscriptionPenalty - score) - denominator := math.Max(math.Abs(expectedPenalty+scoring.DefaultInvalidSubscriptionPenalty), math.Abs(score)) - return math.Abs(nominator/denominator) < 0.01 + return unittest.AreNumericallyClose(expectedPenalty+scoring.DefaultInvalidSubscriptionPenalty, score, 0.01) }, 5*time.Second, 10*time.Millisecond) // the app specific score should now be updated in the cache. score, updated, exists = appScoreCache.Get(peerID) // get the score from the cache. require.True(t, exists) require.True(t, updated.After(queryTime)) - - nominator := math.Abs(expectedPenalty + scoring.DefaultInvalidSubscriptionPenalty - score) - denominator := math.Max(expectedPenalty+scoring.DefaultInvalidSubscriptionPenalty, score) - require.True(t, math.Abs(nominator/denominator) < 0.01) + unittest.RequireNumericallyClose(t, expectedPenalty+scoring.DefaultInvalidSubscriptionPenalty, score, 0.01) // stop the registry. cancel() diff --git a/network/p2p/scoring/subscription_validator_test.go b/network/p2p/scoring/subscription_validator_test.go index 4c454c8d290..15f93104539 100644 --- a/network/p2p/scoring/subscription_validator_test.go +++ b/network/p2p/scoring/subscription_validator_test.go @@ -170,7 +170,8 @@ func TestSubscriptionValidator_Integration(t *testing.T) { cfg, err := config.DefaultConfig() require.NoError(t, err) // set a low update interval to speed up the test - cfg.NetworkConfig.GossipSub.SubscriptionProvider.UpdateInterval = 100 * time.Millisecond + cfg.NetworkConfig.GossipSub.SubscriptionProvider.UpdateInterval = 10 * time.Millisecond + cfg.NetworkConfig.GossipSub.ScoringParameters.AppSpecificScore.ScoreTTL = 10 * time.Millisecond sporkId := unittest.IdentifierFixture() @@ -198,6 +199,14 @@ func TestSubscriptionValidator_Integration(t *testing.T) { p2ptest.EnablePeerScoringWithOverride(p2p.PeerScoringConfigNoOverride), p2ptest.WithRole(flow.RoleVerification)) + // suppress peer provider error + peerProvider := func() peer.IDSlice { + return []peer.ID{conNode.ID(), verNode1.ID(), verNode2.ID()} + } + verNode1.WithPeersProvider(peerProvider) + verNode2.WithPeersProvider(peerProvider) + conNode.WithPeersProvider(peerProvider) + ids := flow.IdentityList{&conId, &verId1, &verId2} nodes := []p2p.LibP2PNode{conNode, verNode1, verNode2} diff --git a/utils/unittest/logging.go b/utils/unittest/logging.go index a200a61525e..c1b993cfab7 100644 --- a/utils/unittest/logging.go +++ b/utils/unittest/logging.go @@ -30,7 +30,7 @@ func Logger() zerolog.Logger { writer = os.Stderr } - return LoggerWithWriterAndLevel(writer, zerolog.TraceLevel) + return LoggerWithWriterAndLevel(writer, zerolog.InfoLevel) } func LoggerWithWriterAndLevel(writer io.Writer, level zerolog.Level) zerolog.Logger { From 27d674a6948ca5b44106754587375fa8c0054a8e Mon Sep 17 00:00:00 2001 From: Yahya Hassanzadeh Date: Mon, 4 Dec 2023 16:04:30 -0800 Subject: [PATCH 49/67] removes dead code --- .../validation/control_message_validation_inspector_test.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/network/p2p/inspector/validation/control_message_validation_inspector_test.go b/network/p2p/inspector/validation/control_message_validation_inspector_test.go index 966bcb927b4..cc4aecc8325 100644 --- a/network/p2p/inspector/validation/control_message_validation_inspector_test.go +++ b/network/p2p/inspector/validation/control_message_validation_inspector_test.go @@ -864,7 +864,3 @@ func stopInspector(t *testing.T, cancel context.CancelFunc, inspector *validatio cancel() unittest.RequireCloseBefore(t, inspector.Done(), 500*time.Millisecond, "inspector did not stop") } - -func defaultTopicOracle() []string { - return []string{} -} From 7984689ec1ab0bf87f534691f0f7f6b8a549df0b Mon Sep 17 00:00:00 2001 From: Yahya Hassanzadeh Date: Mon, 4 Dec 2023 16:21:30 -0800 Subject: [PATCH 50/67] fixes networking test --- follower/follower_builder.go | 1 - network/p2p/p2pconf/gossipsub.go | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/follower/follower_builder.go b/follower/follower_builder.go index 7781bea0b71..4c894d3bbf1 100644 --- a/follower/follower_builder.go +++ b/follower/follower_builder.go @@ -584,7 +584,6 @@ func (builder *FollowerServiceBuilder) initPublicLibp2pNode(networkKey crypto.Pr networkKey, builder.SporkID, builder.IdentityProvider, - builder.FlowConfig.NetworkConfig.GossipSubConfig.GossipSubScoringRegistryConfig, &builder.FlowConfig.NetworkConfig.ResourceManager, p2pconfig.PeerManagerDisableConfig(), // disable peer manager for follower &p2p.DisallowListCacheConfig{ diff --git a/network/p2p/p2pconf/gossipsub.go b/network/p2p/p2pconf/gossipsub.go index 5767418aa61..1566796656b 100644 --- a/network/p2p/p2pconf/gossipsub.go +++ b/network/p2p/p2pconf/gossipsub.go @@ -117,7 +117,7 @@ type AppSpecificScoreParameters struct { const ( PenaltyDecaySlowdownThresholdKey = "penalty-decay-slowdown-threshold" - DecayRateReductionFactorKey = "decay-rate-reduction-factor" + DecayRateReductionFactorKey = "penalty-decay-rate-reduction-factor" PenaltyDecayEvaluationPeriodKey = "penalty-decay-evaluation-period" ) From aef7292a4b47bd0671acc870186d2eefaf55dffc Mon Sep 17 00:00:00 2001 From: Yahya Hassanzadeh Date: Mon, 4 Dec 2023 16:44:08 -0800 Subject: [PATCH 51/67] fixing gossipsub score tracer test --- network/p2p/tracer/gossipSubScoreTracer_test.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/network/p2p/tracer/gossipSubScoreTracer_test.go b/network/p2p/tracer/gossipSubScoreTracer_test.go index 33a25888a2a..d12de3e634a 100644 --- a/network/p2p/tracer/gossipSubScoreTracer_test.go +++ b/network/p2p/tracer/gossipSubScoreTracer_test.go @@ -75,8 +75,12 @@ func TestGossipSubScoreTracer(t *testing.T) { // 3. Creates three nodes with different roles and sets their roles as consensus, access, and tracer, respectively. cfg, err := config.DefaultConfig() require.NoError(t, err) - // set the peer score log interval to 1 second for sake of testing. + // tracer will update the score and local mesh every 1 second (for testing purposes) cfg.NetworkConfig.GossipSub.RpcTracer.LocalMeshLogInterval = 1 * time.Second + cfg.NetworkConfig.GossipSub.RpcTracer.ScoreTracerInterval = 1 * time.Second + // the libp2p node updates the subscription list as well as the app-specific score every 10 milliseconds (for testing purposes) + cfg.NetworkConfig.GossipSub.SubscriptionProvider.UpdateInterval = 10 * time.Millisecond + cfg.NetworkConfig.GossipSub.ScoringParameters.AppSpecificScore.ScoreTTL = 10 * time.Millisecond tracerNode, tracerId := p2ptest.NodeFixture( t, sporkId, @@ -87,6 +91,7 @@ func TestGossipSubScoreTracer(t *testing.T) { c: scoreMetrics, }), p2ptest.WithLogger(logger), + p2ptest.OverrideFlowConfig(cfg), p2ptest.EnablePeerScoringWithOverride(&p2p.PeerScoringConfigOverride{ AppSpecificScoreParams: func(pid peer.ID) float64 { id, ok := idProvider.ByPeerID(pid) @@ -190,7 +195,7 @@ func TestGossipSubScoreTracer(t *testing.T) { // 7. Expects the tracer node to have the correct app scores, a non-zero score, an existing behaviour score, an existing // IP score, and an existing mesh score. - assert.Eventually(t, func() bool { + require.Eventually(t, func() bool { // we expect the tracerNode to have the consensusNodes and accessNodes with the correct app scores. exposer := tracerNode.PeerScoreExposer() score, ok := exposer.GetAppScore(consensusNode.ID()) From d21f2ea30dace91cf7451750b99641fb4ccdadc0 Mon Sep 17 00:00:00 2001 From: Yahya Hassanzadeh Date: Mon, 4 Dec 2023 16:45:20 -0800 Subject: [PATCH 52/67] fixes insecure package test --- .../test/gossipsub/rpc_inspector/validation_inspector_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/insecure/integration/functional/test/gossipsub/rpc_inspector/validation_inspector_test.go b/insecure/integration/functional/test/gossipsub/rpc_inspector/validation_inspector_test.go index 53fb3ae8d8b..75287750db4 100644 --- a/insecure/integration/functional/test/gossipsub/rpc_inspector/validation_inspector_test.go +++ b/insecure/integration/functional/test/gossipsub/rpc_inspector/validation_inspector_test.go @@ -1034,12 +1034,12 @@ func TestGossipSubSpamMitigationIntegration(t *testing.T) { cfg, err := config.DefaultConfig() require.NoError(t, err) // set the scoring parameters to be more aggressive to speed up the test + cfg.NetworkConfig.GossipSub.RpcTracer.ScoreTracerInterval = 100 * time.Millisecond cfg.NetworkConfig.GossipSub.ScoringParameters.AppSpecificScore.ScoreTTL = 100 * time.Millisecond victimNode, victimId := p2ptest.NodeFixture(t, sporkID, t.Name(), idProvider, - p2ptest.WithPeerScoreTracerInterval(100*time.Millisecond), p2ptest.WithRole(flow.RoleConsensus), p2ptest.OverrideFlowConfig(cfg), p2ptest.EnablePeerScoringWithOverride(p2p.PeerScoringConfigNoOverride)) From 24c098e6b26ced034daf15a49e66ebcd17d48e7d Mon Sep 17 00:00:00 2001 From: Yahya Hassanzadeh Date: Mon, 4 Dec 2023 16:59:56 -0800 Subject: [PATCH 53/67] fixes cross talk prevention test --- network/p2p/test/sporking_test.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/network/p2p/test/sporking_test.go b/network/p2p/test/sporking_test.go index c49773d7214..aa6703da7ee 100644 --- a/network/p2p/test/sporking_test.go +++ b/network/p2p/test/sporking_test.go @@ -11,6 +11,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/onflow/flow-go/config" "github.com/onflow/flow-go/model/flow" libp2pmessage "github.com/onflow/flow-go/model/libp2p/message" "github.com/onflow/flow-go/network/message" @@ -204,15 +205,23 @@ func TestOneToKCrosstalkPrevention(t *testing.T) { previousSporkId := unittest.IdentifierFixture() // create and start node 1 on localhost and random port + cfg, err := config.DefaultConfig() + require.NoError(t, err) + // cross-talk prevention is intrinsically tied to how we encode topics, peer scoring adds another layer of protection by preventing unknown identifiers + // from joining the mesh. As this test simulates the scenario where a node is moved from the old chain to the new chain, we disable peer scoring + // to allow the node to join the mesh on the new chain, otherwise the node will be disconnected from the mesh due to peer scoring penalty for unknown identifiers. + cfg.NetworkConfig.GossipSub.PeerScoringEnabled = false node1, id1 := p2ptest.NodeFixture(t, previousSporkId, "test_one_to_k_crosstalk_prevention", idProvider, + p2ptest.OverrideFlowConfig(cfg), ) p2ptest.StartNode(t, signalerCtx1, node1) defer p2ptest.StopNode(t, node1, cancel1) idProvider.SetIdentities(flow.IdentityList{&id1}) + // create and start node 2 on localhost and random port with the same root block ID node2, id2 := p2ptest.NodeFixture(t, previousSporkId, From 1171225bf447cf39937befab0744512153915c19 Mon Sep 17 00:00:00 2001 From: Yahya Hassanzadeh Date: Tue, 5 Dec 2023 10:14:24 -0800 Subject: [PATCH 54/67] adds networking type for herocache of subscription provider --- module/metrics/herocache.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/module/metrics/herocache.go b/module/metrics/herocache.go index 755e68e2b38..89839c6ac5a 100644 --- a/module/metrics/herocache.go +++ b/module/metrics/herocache.go @@ -72,8 +72,12 @@ func NetworkReceiveCacheMetricsFactory(f HeroCacheMetricsFactory, networkType ne return f(namespaceNetwork, r) } -func NewSubscriptionRecordCacheMetricsFactory(f HeroCacheMetricsFactory) module.HeroCacheMetrics { - return f(namespaceNetwork, ResourceNetworkingSubscriptionRecordsCache) +func NewSubscriptionRecordCacheMetricsFactory(f HeroCacheMetricsFactory, networkType network.NetworkingType) module.HeroCacheMetrics { + r := ResourceNetworkingSubscriptionRecordsCache + if networkType == network.PublicNetwork { + r = PrependPublicPrefix(r) + } + return f(namespaceNetwork, r) } // NewGossipSubApplicationSpecificScoreCacheMetrics is the factory method for creating a new HeroCacheCollector for the From ae83a8e87b46e4d0e8cf5bab7422deed844bf663 Mon Sep 17 00:00:00 2001 From: Yahya Hassanzadeh Date: Tue, 5 Dec 2023 10:24:13 -0800 Subject: [PATCH 55/67] adds networking type to subscription provider --- .../p2pbuilder/gossipsub/gossipSubBuilder.go | 1 + .../internal/subscriptionCache_test.go | 19 ++++++++++--------- network/p2p/scoring/subscription_provider.go | 4 +++- .../p2p/scoring/subscription_provider_test.go | 3 +++ 4 files changed, 17 insertions(+), 10 deletions(-) diff --git a/network/p2p/p2pbuilder/gossipsub/gossipSubBuilder.go b/network/p2p/p2pbuilder/gossipsub/gossipSubBuilder.go index 56f3bc24919..3129cb2a004 100644 --- a/network/p2p/p2pbuilder/gossipsub/gossipSubBuilder.go +++ b/network/p2p/p2pbuilder/gossipsub/gossipSubBuilder.go @@ -314,6 +314,7 @@ func (g *Builder) Build(ctx irrecoverable.SignalerContext) (p2p.PubSubAdapter, e IdProvider: g.idProvider, Params: &g.gossipSubCfg.SubscriptionProvider, HeroCacheMetricsFactory: g.metricsCfg.HeroCacheFactory, + NetworkingType: g.networkType, }) if err != nil { return nil, fmt.Errorf("could not create subscription provider: %w", err) diff --git a/network/p2p/scoring/internal/subscriptionCache_test.go b/network/p2p/scoring/internal/subscriptionCache_test.go index a333c18bdd8..355bdab9523 100644 --- a/network/p2p/scoring/internal/subscriptionCache_test.go +++ b/network/p2p/scoring/internal/subscriptionCache_test.go @@ -8,6 +8,7 @@ import ( "github.com/stretchr/testify/require" "github.com/onflow/flow-go/module/metrics" + "github.com/onflow/flow-go/network" "github.com/onflow/flow-go/network/p2p/scoring/internal" "github.com/onflow/flow-go/utils/unittest" ) @@ -19,7 +20,7 @@ func TestNewSubscriptionRecordCache(t *testing.T) { cache := internal.NewSubscriptionRecordCache( sizeLimit, unittest.Logger(), - metrics.NewSubscriptionRecordCacheMetricsFactory(metrics.NewNoopHeroCacheMetricsFactory())) + metrics.NewSubscriptionRecordCacheMetricsFactory(metrics.NewNoopHeroCacheMetricsFactory(), network.PrivateNetwork)) require.NotNil(t, cache, "cache should not be nil") require.IsType(t, &internal.SubscriptionRecordCache{}, cache, "cache should be of type *SubscriptionRecordCache") @@ -31,7 +32,7 @@ func TestSubscriptionCache_GetSubscribedTopics(t *testing.T) { cache := internal.NewSubscriptionRecordCache( sizeLimit, unittest.Logger(), - metrics.NewSubscriptionRecordCacheMetricsFactory(metrics.NewNoopHeroCacheMetricsFactory())) + metrics.NewSubscriptionRecordCacheMetricsFactory(metrics.NewNoopHeroCacheMetricsFactory(), network.PrivateNetwork)) // create a dummy peer ID peerID := unittest.PeerIdFixture(t) @@ -63,7 +64,7 @@ func TestSubscriptionCache_MoveToNextUpdateCycle(t *testing.T) { cache := internal.NewSubscriptionRecordCache( sizeLimit, unittest.Logger(), - metrics.NewSubscriptionRecordCacheMetricsFactory(metrics.NewNoopHeroCacheMetricsFactory())) + metrics.NewSubscriptionRecordCacheMetricsFactory(metrics.NewNoopHeroCacheMetricsFactory(), network.PrivateNetwork)) // initial cycle should be 0, so first increment sets it to 1 firstCycle := cache.MoveToNextUpdateCycle() @@ -80,7 +81,7 @@ func TestSubscriptionCache_TestAddTopicForPeer(t *testing.T) { cache := internal.NewSubscriptionRecordCache( sizeLimit, unittest.Logger(), - metrics.NewSubscriptionRecordCacheMetricsFactory(metrics.NewNoopHeroCacheMetricsFactory())) + metrics.NewSubscriptionRecordCacheMetricsFactory(metrics.NewNoopHeroCacheMetricsFactory(), network.PrivateNetwork)) // case when adding a topic to an existing peer existingPeerID := unittest.PeerIdFixture(t) @@ -117,7 +118,7 @@ func TestSubscriptionCache_DuplicateTopics(t *testing.T) { cache := internal.NewSubscriptionRecordCache( sizeLimit, unittest.Logger(), - metrics.NewSubscriptionRecordCacheMetricsFactory(metrics.NewNoopHeroCacheMetricsFactory())) + metrics.NewSubscriptionRecordCacheMetricsFactory(metrics.NewNoopHeroCacheMetricsFactory(), network.PrivateNetwork)) peerID := unittest.PeerIdFixture(t) topic := "topic1" @@ -141,7 +142,7 @@ func TestSubscriptionCache_MoveUpdateCycle(t *testing.T) { cache := internal.NewSubscriptionRecordCache( sizeLimit, unittest.Logger(), - metrics.NewSubscriptionRecordCacheMetricsFactory(metrics.NewNoopHeroCacheMetricsFactory())) + metrics.NewSubscriptionRecordCacheMetricsFactory(metrics.NewNoopHeroCacheMetricsFactory(), network.PrivateNetwork)) peerID := unittest.PeerIdFixture(t) topic1 := "topic1" @@ -188,7 +189,7 @@ func TestSubscriptionCache_MoveUpdateCycleWithDifferentPeers(t *testing.T) { cache := internal.NewSubscriptionRecordCache( sizeLimit, unittest.Logger(), - metrics.NewSubscriptionRecordCacheMetricsFactory(metrics.NewNoopHeroCacheMetricsFactory())) + metrics.NewSubscriptionRecordCacheMetricsFactory(metrics.NewNoopHeroCacheMetricsFactory(), network.PrivateNetwork)) peer1 := unittest.PeerIdFixture(t) peer2 := unittest.PeerIdFixture(t) @@ -234,7 +235,7 @@ func TestSubscriptionCache_ConcurrentUpdate(t *testing.T) { cache := internal.NewSubscriptionRecordCache( sizeLimit, unittest.Logger(), - metrics.NewSubscriptionRecordCacheMetricsFactory(metrics.NewNoopHeroCacheMetricsFactory())) + metrics.NewSubscriptionRecordCacheMetricsFactory(metrics.NewNoopHeroCacheMetricsFactory(), network.PrivateNetwork)) peerIds := unittest.PeerIdFixtures(t, 100) topics := []string{"topic1", "topic2", "topic3"} @@ -277,7 +278,7 @@ func TestSubscriptionCache_TestSizeLimit(t *testing.T) { cache := internal.NewSubscriptionRecordCache( sizeLimit, unittest.Logger(), - metrics.NewSubscriptionRecordCacheMetricsFactory(metrics.NewNoopHeroCacheMetricsFactory())) + metrics.NewSubscriptionRecordCacheMetricsFactory(metrics.NewNoopHeroCacheMetricsFactory(), network.PrivateNetwork)) peerIds := unittest.PeerIdFixtures(t, 100) topics := []string{"topic1", "topic2", "topic3"} diff --git a/network/p2p/scoring/subscription_provider.go b/network/p2p/scoring/subscription_provider.go index b70a8cba949..272778f15a8 100644 --- a/network/p2p/scoring/subscription_provider.go +++ b/network/p2p/scoring/subscription_provider.go @@ -13,6 +13,7 @@ import ( "github.com/onflow/flow-go/module/component" "github.com/onflow/flow-go/module/irrecoverable" "github.com/onflow/flow-go/module/metrics" + "github.com/onflow/flow-go/network" "github.com/onflow/flow-go/network/p2p" "github.com/onflow/flow-go/network/p2p/p2pconf" "github.com/onflow/flow-go/network/p2p/p2plogging" @@ -44,6 +45,7 @@ type SubscriptionProviderConfig struct { IdProvider module.IdentityProvider `validate:"required"` HeroCacheMetricsFactory metrics.HeroCacheMetricsFactory `validate:"required"` Params *p2pconf.SubscriptionProviderParameters `validate:"required"` + NetworkingType network.NetworkingType `validate:"required"` } var _ p2p.SubscriptionProvider = (*SubscriptionProvider)(nil) @@ -53,7 +55,7 @@ func NewSubscriptionProvider(cfg *SubscriptionProviderConfig) (*SubscriptionProv return nil, fmt.Errorf("invalid subscription provider config: %w", err) } - cacheMetrics := metrics.NewSubscriptionRecordCacheMetricsFactory(cfg.HeroCacheMetricsFactory) + cacheMetrics := metrics.NewSubscriptionRecordCacheMetricsFactory(cfg.HeroCacheMetricsFactory, cfg.NetworkingType) cache := internal.NewSubscriptionRecordCache(cfg.Params.CacheSize, cfg.Logger, cacheMetrics) p := &SubscriptionProvider{ diff --git a/network/p2p/scoring/subscription_provider_test.go b/network/p2p/scoring/subscription_provider_test.go index 0c731f1b508..84f5aeb6896 100644 --- a/network/p2p/scoring/subscription_provider_test.go +++ b/network/p2p/scoring/subscription_provider_test.go @@ -15,6 +15,7 @@ import ( "github.com/onflow/flow-go/module/irrecoverable" "github.com/onflow/flow-go/module/metrics" "github.com/onflow/flow-go/module/mock" + "github.com/onflow/flow-go/network" "github.com/onflow/flow-go/network/p2p" mockp2p "github.com/onflow/flow-go/network/p2p/mock" "github.com/onflow/flow-go/network/p2p/scoring" @@ -41,6 +42,7 @@ func TestSubscriptionProvider_GetSubscribedTopics(t *testing.T) { Params: &cfg.NetworkConfig.GossipSub.SubscriptionProvider, HeroCacheMetricsFactory: metrics.NewNoopHeroCacheMetricsFactory(), IdProvider: idProvider, + NetworkingType: network.PrivateNetwork, }) require.NoError(t, err) @@ -102,6 +104,7 @@ func TestSubscriptionProvider_GetSubscribedTopics_SkippingUnknownPeers(t *testin Params: &cfg.NetworkConfig.GossipSub.SubscriptionProvider, HeroCacheMetricsFactory: metrics.NewNoopHeroCacheMetricsFactory(), IdProvider: idProvider, + NetworkingType: network.PrivateNetwork, }) require.NoError(t, err) From 0806d9794671d7bf4ba3d34079f90c55e668a097 Mon Sep 17 00:00:00 2001 From: Yahya Hassanzadeh Date: Tue, 5 Dec 2023 10:54:32 -0800 Subject: [PATCH 56/67] fixes duplicate metrics registry for app specific queue --- module/metrics/herocache.go | 8 ++++++-- network/p2p/p2pbuilder/gossipsub/gossipSubBuilder.go | 2 +- network/p2p/scoring/registry.go | 5 ++++- network/p2p/scoring/registry_test.go | 2 ++ network/p2p/scoring/score_option.go | 10 ++++++++-- 5 files changed, 21 insertions(+), 6 deletions(-) diff --git a/module/metrics/herocache.go b/module/metrics/herocache.go index 89839c6ac5a..84e30a3f0c1 100644 --- a/module/metrics/herocache.go +++ b/module/metrics/herocache.go @@ -223,8 +223,12 @@ func GossipSubRPCInspectorClusterPrefixedCacheMetricFactory(f HeroCacheMetricsFa // - f: the HeroCacheMetricsFactory to create the collector // Returns: // - a HeroCacheMetrics for the app-specific score update queue. -func GossipSubAppSpecificScoreUpdateQueueMetricFactory(f HeroCacheMetricsFactory) module.HeroCacheMetrics { - return f(namespaceNetwork, ResourceNetworkingAppSpecificScoreUpdateQueue) +func GossipSubAppSpecificScoreUpdateQueueMetricFactory(f HeroCacheMetricsFactory, networkingType network.NetworkingType) module.HeroCacheMetrics { + r := ResourceNetworkingAppSpecificScoreUpdateQueue + if networkingType == network.PublicNetwork { + r = PrependPublicPrefix(r) + } + return f(namespaceNetwork, r) } func CollectionNodeTransactionsCacheMetrics(registrar prometheus.Registerer, epoch uint64) *HeroCacheCollector { diff --git a/network/p2p/p2pbuilder/gossipsub/gossipSubBuilder.go b/network/p2p/p2pbuilder/gossipsub/gossipSubBuilder.go index 3129cb2a004..d44030f392f 100644 --- a/network/p2p/p2pbuilder/gossipsub/gossipSubBuilder.go +++ b/network/p2p/p2pbuilder/gossipsub/gossipSubBuilder.go @@ -175,7 +175,7 @@ func NewGossipSubBuilder(logger zerolog.Logger, metricsCfg *p2pconfig.MetricsCon idProvider: idProvider, gossipSubFactory: defaultGossipSubFactory(), gossipSubConfigFunc: defaultGossipSubAdapterConfig(), - scoreOptionConfig: scoring.NewScoreOptionConfig(lg, gossipSubCfg.ScoringParameters, metricsCfg.HeroCacheFactory, idProvider), + scoreOptionConfig: scoring.NewScoreOptionConfig(lg, gossipSubCfg.ScoringParameters, metricsCfg.HeroCacheFactory, idProvider, networkType), rpcInspectorSuiteFactory: defaultInspectorSuite(meshTracer), gossipSubTracer: meshTracer, gossipSubCfg: gossipSubCfg, diff --git a/network/p2p/scoring/registry.go b/network/p2p/scoring/registry.go index 5d23e1f6a4a..8fdbd0b2067 100644 --- a/network/p2p/scoring/registry.go +++ b/network/p2p/scoring/registry.go @@ -16,6 +16,7 @@ import ( "github.com/onflow/flow-go/module/irrecoverable" "github.com/onflow/flow-go/module/mempool/queue" "github.com/onflow/flow-go/module/metrics" + "github.com/onflow/flow-go/network" "github.com/onflow/flow-go/network/p2p" netcache "github.com/onflow/flow-go/network/p2p/cache" p2pmsg "github.com/onflow/flow-go/network/p2p/message" @@ -161,6 +162,8 @@ type GossipSubAppSpecificScoreRegistryConfig struct { AppScoreCacheFactory func() p2p.GossipSubApplicationSpecificScoreCache `validate:"required"` HeroCacheMetricsFactory metrics.HeroCacheMetricsFactory `validate:"required"` + + NetworkingType network.NetworkingType } // NewGossipSubAppSpecificScoreRegistry returns a new GossipSubAppSpecificScoreRegistry. @@ -181,7 +184,7 @@ func NewGossipSubAppSpecificScoreRegistry(config *GossipSubAppSpecificScoreRegis lg := config.Logger.With().Str("module", "app_score_registry").Logger() store := queue.NewHeroStore(config.Parameters.ScoreUpdateRequestQueueSize, lg.With().Str("component", "app_specific_score_update").Logger(), - metrics.GossipSubAppSpecificScoreUpdateQueueMetricFactory(config.HeroCacheMetricsFactory)) + metrics.GossipSubAppSpecificScoreUpdateQueueMetricFactory(config.HeroCacheMetricsFactory, config.NetworkingType)) reg := &GossipSubAppSpecificScoreRegistry{ logger: config.Logger.With().Str("module", "app_score_registry").Logger(), diff --git a/network/p2p/scoring/registry_test.go b/network/p2p/scoring/registry_test.go index 57ca8b6d8c6..4ab01f07282 100644 --- a/network/p2p/scoring/registry_test.go +++ b/network/p2p/scoring/registry_test.go @@ -18,6 +18,7 @@ import ( "github.com/onflow/flow-go/module/irrecoverable" "github.com/onflow/flow-go/module/metrics" "github.com/onflow/flow-go/module/mock" + "github.com/onflow/flow-go/network" "github.com/onflow/flow-go/network/p2p" netcache "github.com/onflow/flow-go/network/p2p/cache" p2pmsg "github.com/onflow/flow-go/network/p2p/message" @@ -963,6 +964,7 @@ func newGossipSubAppSpecificScoreRegistry(t *testing.T, params p2pconf.ScoringPa }, Parameters: params.AppSpecificScore, HeroCacheMetricsFactory: metrics.NewNoopHeroCacheMetricsFactory(), + NetworkingType: network.PrivateNetwork, } for _, opt := range opts { opt(cfg) diff --git a/network/p2p/scoring/score_option.go b/network/p2p/scoring/score_option.go index 2173d2dd4db..87d8a94c447 100644 --- a/network/p2p/scoring/score_option.go +++ b/network/p2p/scoring/score_option.go @@ -12,6 +12,7 @@ import ( "github.com/onflow/flow-go/module/component" "github.com/onflow/flow-go/module/irrecoverable" "github.com/onflow/flow-go/module/metrics" + "github.com/onflow/flow-go/network" "github.com/onflow/flow-go/network/channels" "github.com/onflow/flow-go/network/p2p" netcache "github.com/onflow/flow-go/network/p2p/cache" @@ -312,6 +313,7 @@ type ScoreOptionConfig struct { appScoreFunc func(peer.ID) float64 topicParams []func(map[string]*pubsub.TopicScoreParams) registerNotificationConsumerFunc func(p2p.GossipSubInvCtrlMsgNotifConsumer) + networkingType network.NetworkingType } // NewScoreOptionConfig creates a new configuration for the GossipSub peer scoring option. @@ -319,18 +321,21 @@ type ScoreOptionConfig struct { // - logger: the logger to use. // - hcMetricsFactory: HeroCache metrics factory to create metrics for the scoring-related caches. // - idProvider: the identity provider to use. +// - networkingType: the networking type to use, public or private. // Returns: // - a new configuration for the GossipSub peer scoring option. func NewScoreOptionConfig(logger zerolog.Logger, params p2pconf.ScoringParameters, hcMetricsFactory metrics.HeroCacheMetricsFactory, - idProvider module.IdentityProvider) *ScoreOptionConfig { + idProvider module.IdentityProvider, + networkingType network.NetworkingType) *ScoreOptionConfig { return &ScoreOptionConfig{ logger: logger.With().Str("module", "pubsub_score_option").Logger(), provider: idProvider, params: params, heroCacheMetricsFactory: hcMetricsFactory, topicParams: make([]func(map[string]*pubsub.TopicScoreParams), 0), + networkingType: networkingType, } } @@ -387,7 +392,8 @@ func NewScoreOption(cfg *ScoreOptionConfig, provider p2p.SubscriptionProvider) ( cfg.params.SpamRecordCache.DecayRateReductionFactor, cfg.params.SpamRecordCache.PenaltyDecayEvaluationPeriod)) }, - Parameters: cfg.params.AppSpecificScore, + Parameters: cfg.params.AppSpecificScore, + NetworkingType: cfg.networkingType, }) if err != nil { return nil, fmt.Errorf("failed to create gossipsub app specific score registry: %w", err) From e69343ee1cb39fddde8b5782fed91119d7e72a96 Mon Sep 17 00:00:00 2001 From: Yahya Hassanzadeh Date: Tue, 5 Dec 2023 11:35:22 -0800 Subject: [PATCH 57/67] fixes duplicate metrics registry for spam records cache --- module/metrics/herocache.go | 17 +++++++++---- network/p2p/cache/gossipsub_spam_records.go | 6 ++--- .../p2p/cache/gossipsub_spam_records_test.go | 24 +++++++++---------- .../scoring/internal/appSpecificScoreCache.go | 6 ++--- .../internal/appSpecificScoreCache_test.go | 6 ++--- network/p2p/scoring/registry_test.go | 4 ++-- network/p2p/scoring/score_option.go | 6 +++-- 7 files changed, 39 insertions(+), 30 deletions(-) diff --git a/module/metrics/herocache.go b/module/metrics/herocache.go index 84e30a3f0c1..9813c8b2b2f 100644 --- a/module/metrics/herocache.go +++ b/module/metrics/herocache.go @@ -87,8 +87,12 @@ func NewSubscriptionRecordCacheMetricsFactory(f HeroCacheMetricsFactory, network // - f: the HeroCacheMetricsFactory to create the collector // Returns: // - a HeroCacheMetrics for the application specific score cache -func NewGossipSubApplicationSpecificScoreCacheMetrics(f HeroCacheMetricsFactory) module.HeroCacheMetrics { - return f(namespaceNetwork, ResourceNetworkingGossipSubApplicationSpecificScoreCache) +func NewGossipSubApplicationSpecificScoreCacheMetrics(f HeroCacheMetricsFactory, networkingType network.NetworkingType) module.HeroCacheMetrics { + r := ResourceNetworkingGossipSubApplicationSpecificScoreCache + if networkingType == network.PublicNetwork { + r = PrependPublicPrefix(r) + } + return f(namespaceNetwork, r) } // DisallowListCacheMetricsFactory is the factory method for creating a new HeroCacheCollector for the disallow list cache. @@ -108,9 +112,12 @@ func DisallowListCacheMetricsFactory(f HeroCacheMetricsFactory, networkingType n // GossipSubSpamRecordCacheMetricsFactory is the factory method for creating a new HeroCacheCollector for the spam record cache. // The spam record cache is used to keep track of peers that are spamming the network and the reasons for it. -// Currently, the spam record cache is only used for the private network. -func GossipSubSpamRecordCacheMetricsFactory(f HeroCacheMetricsFactory) module.HeroCacheMetrics { - return f(namespaceNetwork, ResourceNetworkingGossipSubSpamRecordCache) +func GossipSubSpamRecordCacheMetricsFactory(f HeroCacheMetricsFactory, networkingType network.NetworkingType) module.HeroCacheMetrics { + r := ResourceNetworkingGossipSubSpamRecordCache + if networkingType == network.PublicNetwork { + r = PrependPublicPrefix(r) + } + return f(namespaceNetwork, r) } func NetworkDnsTxtCacheMetricsFactory(registrar prometheus.Registerer) *HeroCacheCollector { diff --git a/network/p2p/cache/gossipsub_spam_records.go b/network/p2p/cache/gossipsub_spam_records.go index 2c34af6df92..55dce60a98c 100644 --- a/network/p2p/cache/gossipsub_spam_records.go +++ b/network/p2p/cache/gossipsub_spam_records.go @@ -8,10 +8,10 @@ import ( "github.com/rs/zerolog" "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/module" herocache "github.com/onflow/flow-go/module/mempool/herocache/backdata" "github.com/onflow/flow-go/module/mempool/herocache/backdata/heropool" "github.com/onflow/flow-go/module/mempool/stdmap" - "github.com/onflow/flow-go/module/metrics" "github.com/onflow/flow-go/network/p2p" "github.com/onflow/flow-go/network/p2p/p2plogging" ) @@ -62,7 +62,7 @@ type PreprocessorFunc func(record p2p.GossipSubSpamRecord, lastUpdated time.Time // *GossipSubSpamRecordCache: the newly created cache with a HeroCache-based backend. func NewGossipSubSpamRecordCache(sizeLimit uint32, logger zerolog.Logger, - hcMetricsFactor metrics.HeroCacheMetricsFactory, + collector module.HeroCacheMetrics, prFns ...PreprocessorFunc) *GossipSubSpamRecordCache { backData := herocache.NewCache(sizeLimit, herocache.DefaultOversizeFactor, @@ -70,7 +70,7 @@ func NewGossipSubSpamRecordCache(sizeLimit uint32, // eviction will open the node to spam attacks by malicious peers to erase their application specific penalty. heropool.NoEjection, logger.With().Str("mempool", "gossipsub-app-Penalty-cache").Logger(), - metrics.GossipSubSpamRecordCacheMetricsFactory(hcMetricsFactor)) + collector) return &GossipSubSpamRecordCache{ c: stdmap.NewBackend(stdmap.WithBackData(backData)), preprocessFns: prFns, diff --git a/network/p2p/cache/gossipsub_spam_records_test.go b/network/p2p/cache/gossipsub_spam_records_test.go index 5ddcf2a3484..166776b93ba 100644 --- a/network/p2p/cache/gossipsub_spam_records_test.go +++ b/network/p2p/cache/gossipsub_spam_records_test.go @@ -21,7 +21,7 @@ import ( // adding a new record to the cache. func TestGossipSubSpamRecordCache_Add(t *testing.T) { // create a new instance of GossipSubSpamRecordCache. - cache := netcache.NewGossipSubSpamRecordCache(100, unittest.Logger(), metrics.NewNoopHeroCacheMetricsFactory()) + cache := netcache.NewGossipSubSpamRecordCache(100, unittest.Logger(), metrics.NewNoopCollector()) // tests adding a new record to the cache. require.True(t, cache.Add("peer0", p2p.GossipSubSpamRecord{ @@ -70,7 +70,7 @@ func TestGossipSubSpamRecordCache_Add(t *testing.T) { // It updates the cache with a number of records concurrently and then checks if the cache // can retrieve all records. func TestGossipSubSpamRecordCache_Concurrent_Add(t *testing.T) { - cache := netcache.NewGossipSubSpamRecordCache(200, unittest.Logger(), metrics.NewNoopHeroCacheMetricsFactory()) + cache := netcache.NewGossipSubSpamRecordCache(200, unittest.Logger(), metrics.NewNoopCollector()) // defines the number of records to update. numRecords := 100 @@ -113,7 +113,7 @@ func TestGossipSubSpamRecordCache_Concurrent_Add(t *testing.T) { // TestGossipSubSpamRecordCache_Update tests the Update method of the GossipSubSpamRecordCache. It tests if the cache can update // the penalty of an existing record and fail to update the penalty of a non-existing record. func TestGossipSubSpamRecordCache_Update(t *testing.T) { - cache := netcache.NewGossipSubSpamRecordCache(200, unittest.Logger(), metrics.NewNoopHeroCacheMetricsFactory()) + cache := netcache.NewGossipSubSpamRecordCache(200, unittest.Logger(), metrics.NewNoopCollector()) peerID := "peer1" @@ -141,7 +141,7 @@ func TestGossipSubSpamRecordCache_Update(t *testing.T) { // TestGossipSubSpamRecordCache_Concurrent_Update tests if the cache can be updated concurrently. It updates the cache // with a number of records concurrently and then checks if the cache can retrieve all records. func TestGossipSubSpamRecordCache_Concurrent_Update(t *testing.T) { - cache := netcache.NewGossipSubSpamRecordCache(200, unittest.Logger(), metrics.NewNoopHeroCacheMetricsFactory()) + cache := netcache.NewGossipSubSpamRecordCache(200, unittest.Logger(), metrics.NewNoopCollector()) // defines the number of records to update. numRecords := 100 @@ -199,7 +199,7 @@ func TestGossipSubSpamRecordCache_Concurrent_Update(t *testing.T) { func TestGossipSubSpamRecordCache_Update_With_Preprocess(t *testing.T) { cache := netcache.NewGossipSubSpamRecordCache(200, unittest.Logger(), - metrics.NewNoopHeroCacheMetricsFactory(), + metrics.NewNoopCollector(), func(record p2p.GossipSubSpamRecord, lastUpdated time.Time) (p2p.GossipSubSpamRecord, error) { record.Penalty += 1.5 return record, nil @@ -232,7 +232,7 @@ func TestGossipSubSpamRecordCache_Update_Preprocess_Error(t *testing.T) { secondPreprocessorCalled := false cache := netcache.NewGossipSubSpamRecordCache(200, unittest.Logger(), - metrics.NewNoopHeroCacheMetricsFactory(), + metrics.NewNoopCollector(), // the first preprocessor function does not return an error. func(record p2p.GossipSubSpamRecord, lastUpdated time.Time) (p2p.GossipSubSpamRecord, error) { return record, nil @@ -277,7 +277,7 @@ func TestGossipSubSpamRecordCache_Update_Preprocess_Error(t *testing.T) { // This is a desired behavior that is guaranteed by the underlying HeroCache library. // In other words, we don't desire the records to be externally mutable after they are added to the cache (unless by a subsequent call to Update). func TestGossipSubSpamRecordCache_ByValue(t *testing.T) { - cache := netcache.NewGossipSubSpamRecordCache(200, unittest.Logger(), metrics.NewNoopHeroCacheMetricsFactory()) + cache := netcache.NewGossipSubSpamRecordCache(200, unittest.Logger(), metrics.NewNoopCollector()) peerID := "peer1" added := cache.Add(peer.ID(peerID), p2p.GossipSubSpamRecord{ @@ -308,7 +308,7 @@ func TestGossipSubSpamRecordCache_ByValue(t *testing.T) { // TestGossipSubSpamRecordCache_Get_With_Preprocessors tests if the cache applies the preprocessors to the records // before returning them. func TestGossipSubSpamRecordCache_Get_With_Preprocessors(t *testing.T) { - cache := netcache.NewGossipSubSpamRecordCache(10, unittest.Logger(), metrics.NewNoopHeroCacheMetricsFactory(), + cache := netcache.NewGossipSubSpamRecordCache(10, unittest.Logger(), metrics.NewNoopCollector(), // first preprocessor: adds 1 to the penalty. func(record p2p.GossipSubSpamRecord, lastUpdated time.Time) (p2p.GossipSubSpamRecord, error) { record.Penalty++ @@ -348,7 +348,7 @@ func TestGossipSubSpamRecordCache_Get_Preprocessor_Error(t *testing.T) { secondPreprocessorCalledCount := 0 thirdPreprocessorCalledCount := 0 - cache := netcache.NewGossipSubSpamRecordCache(10, unittest.Logger(), metrics.NewNoopHeroCacheMetricsFactory(), + cache := netcache.NewGossipSubSpamRecordCache(10, unittest.Logger(), metrics.NewNoopCollector(), // first preprocessor: adds 1 to the penalty. func(record p2p.GossipSubSpamRecord, lastUpdated time.Time) (p2p.GossipSubSpamRecord, error) { record.Penalty++ @@ -402,7 +402,7 @@ func TestGossipSubSpamRecordCache_Get_Preprocessor_Error(t *testing.T) { // TestGossipSubSpamRecordCache_Get_Without_Preprocessors tests when no preprocessors are provided to the cache constructor // that the cache returns the original record without any modifications. func TestGossipSubSpamRecordCache_Get_Without_Preprocessors(t *testing.T) { - cache := netcache.NewGossipSubSpamRecordCache(10, unittest.Logger(), metrics.NewNoopHeroCacheMetricsFactory()) + cache := netcache.NewGossipSubSpamRecordCache(10, unittest.Logger(), metrics.NewNoopCollector()) record := p2p.GossipSubSpamRecord{ Decay: 0.5, @@ -423,7 +423,7 @@ func TestGossipSubSpamRecordCache_Get_Without_Preprocessors(t *testing.T) { // This test evaluates that the cache de-duplicates the records based on their peer id and not content, and hence // each peer id can only be added once to the cache. func TestGossipSubSpamRecordCache_Duplicate_Add_Sequential(t *testing.T) { - cache := netcache.NewGossipSubSpamRecordCache(10, unittest.Logger(), metrics.NewNoopHeroCacheMetricsFactory()) + cache := netcache.NewGossipSubSpamRecordCache(10, unittest.Logger(), metrics.NewNoopCollector()) record := p2p.GossipSubSpamRecord{ Decay: 0.5, @@ -445,7 +445,7 @@ func TestGossipSubSpamRecordCache_Duplicate_Add_Sequential(t *testing.T) { // TestGossipSubSpamRecordCache_Duplicate_Add_Concurrent tests if the cache returns false when a duplicate record is added to the cache. // Test is the concurrent version of TestAppScoreCache_DuplicateAdd_Sequential. func TestGossipSubSpamRecordCache_Duplicate_Add_Concurrent(t *testing.T) { - cache := netcache.NewGossipSubSpamRecordCache(10, unittest.Logger(), metrics.NewNoopHeroCacheMetricsFactory()) + cache := netcache.NewGossipSubSpamRecordCache(10, unittest.Logger(), metrics.NewNoopCollector()) successAdd := atomic.Int32{} successAdd.Store(0) diff --git a/network/p2p/scoring/internal/appSpecificScoreCache.go b/network/p2p/scoring/internal/appSpecificScoreCache.go index 0fa8eb2b8a1..7e75996132f 100644 --- a/network/p2p/scoring/internal/appSpecificScoreCache.go +++ b/network/p2p/scoring/internal/appSpecificScoreCache.go @@ -8,10 +8,10 @@ import ( "github.com/rs/zerolog" "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/module" herocache "github.com/onflow/flow-go/module/mempool/herocache/backdata" "github.com/onflow/flow-go/module/mempool/herocache/backdata/heropool" "github.com/onflow/flow-go/module/mempool/stdmap" - "github.com/onflow/flow-go/module/metrics" "github.com/onflow/flow-go/network/p2p" ) @@ -33,12 +33,12 @@ var _ p2p.GossipSubApplicationSpecificScoreCache = (*AppSpecificScoreCache)(nil) // - collector: the metrics collector to use for collecting metrics. // Returns: // - *AppSpecificScoreCache: the created cache. -func NewAppSpecificScoreCache(sizeLimit uint32, logger zerolog.Logger, hcMetricsFactory metrics.HeroCacheMetricsFactory) *AppSpecificScoreCache { +func NewAppSpecificScoreCache(sizeLimit uint32, logger zerolog.Logger, collector module.HeroCacheMetrics) *AppSpecificScoreCache { backData := herocache.NewCache(sizeLimit, herocache.DefaultOversizeFactor, heropool.LRUEjection, logger.With().Str("mempool", "gossipsub-app-specific-score-cache").Logger(), - metrics.NewGossipSubApplicationSpecificScoreCacheMetrics(hcMetricsFactory)) + collector) return &AppSpecificScoreCache{ c: stdmap.NewBackend(stdmap.WithBackData(backData)), diff --git a/network/p2p/scoring/internal/appSpecificScoreCache_test.go b/network/p2p/scoring/internal/appSpecificScoreCache_test.go index ef4b89eec50..415218f3c30 100644 --- a/network/p2p/scoring/internal/appSpecificScoreCache_test.go +++ b/network/p2p/scoring/internal/appSpecificScoreCache_test.go @@ -16,7 +16,7 @@ import ( // specifically, it tests the Add and Get methods. // It does not test the eviction policy of the cache. func TestAppSpecificScoreCache(t *testing.T) { - cache := internal.NewAppSpecificScoreCache(10, unittest.Logger(), metrics.NewNoopHeroCacheMetricsFactory()) + cache := internal.NewAppSpecificScoreCache(10, unittest.Logger(), metrics.NewNoopCollector()) require.NotNil(t, cache, "failed to create AppSpecificScoreCache") peerID := unittest.PeerIdFixture(t) @@ -47,7 +47,7 @@ func TestAppSpecificScoreCache(t *testing.T) { // TestAppSpecificScoreCache_Concurrent_Add_Get_Update tests the concurrent functionality of AppSpecificScoreCache; // specifically, it tests the Add and Get methods under concurrent access. func TestAppSpecificScoreCache_Concurrent_Add_Get_Update(t *testing.T) { - cache := internal.NewAppSpecificScoreCache(10, unittest.Logger(), metrics.NewNoopHeroCacheMetricsFactory()) + cache := internal.NewAppSpecificScoreCache(10, unittest.Logger(), metrics.NewNoopCollector()) require.NotNil(t, cache, "failed to create AppSpecificScoreCache") peerId1 := unittest.PeerIdFixture(t) @@ -140,7 +140,7 @@ func TestAppSpecificScoreCache_Concurrent_Add_Get_Update(t *testing.T) { // TestAppSpecificScoreCache_Eviction tests the eviction policy of AppSpecificScoreCache; // specifically, it tests that the cache evicts the least recently used record when the cache is full. func TestAppSpecificScoreCache_Eviction(t *testing.T) { - cache := internal.NewAppSpecificScoreCache(10, unittest.Logger(), metrics.NewNoopHeroCacheMetricsFactory()) + cache := internal.NewAppSpecificScoreCache(10, unittest.Logger(), metrics.NewNoopCollector()) require.NotNil(t, cache, "failed to create AppSpecificScoreCache") peerIds := unittest.PeerIdFixtures(t, 11) diff --git a/network/p2p/scoring/registry_test.go b/network/p2p/scoring/registry_test.go index 4ab01f07282..fc7045f0b29 100644 --- a/network/p2p/scoring/registry_test.go +++ b/network/p2p/scoring/registry_test.go @@ -937,9 +937,9 @@ func newGossipSubAppSpecificScoreRegistry(t *testing.T, params p2pconf.ScoringPa *internal.AppSpecificScoreCache) { cache := netcache.NewGossipSubSpamRecordCache(100, unittest.Logger(), - metrics.NewNoopHeroCacheMetricsFactory(), + metrics.NewNoopCollector(), scoring.DefaultDecayFunction(params.SpamRecordCache.PenaltyDecaySlowdownThreshold, params.SpamRecordCache.DecayRateReductionFactor, params.SpamRecordCache.PenaltyDecayEvaluationPeriod)) - appSpecificScoreCache := internal.NewAppSpecificScoreCache(100, unittest.Logger(), metrics.NewNoopHeroCacheMetricsFactory()) + appSpecificScoreCache := internal.NewAppSpecificScoreCache(100, unittest.Logger(), metrics.NewNoopCollector()) validator := mockp2p.NewSubscriptionValidator(t) validator.On("Start", testifymock.Anything).Return().Maybe() diff --git a/network/p2p/scoring/score_option.go b/network/p2p/scoring/score_option.go index 87d8a94c447..934488852bb 100644 --- a/network/p2p/scoring/score_option.go +++ b/network/p2p/scoring/score_option.go @@ -383,10 +383,12 @@ func NewScoreOption(cfg *ScoreOptionConfig, provider p2p.SubscriptionProvider) ( IdProvider: cfg.provider, HeroCacheMetricsFactory: cfg.heroCacheMetricsFactory, AppScoreCacheFactory: func() p2p.GossipSubApplicationSpecificScoreCache { - return internal.NewAppSpecificScoreCache(cfg.params.SpamRecordCache.CacheSize, cfg.logger, cfg.heroCacheMetricsFactory) + collector := metrics.NewGossipSubApplicationSpecificScoreCacheMetrics(cfg.heroCacheMetricsFactory, cfg.networkingType) + return internal.NewAppSpecificScoreCache(cfg.params.SpamRecordCache.CacheSize, cfg.logger, collector) }, SpamRecordCacheFactory: func() p2p.GossipSubSpamRecordCache { - return netcache.NewGossipSubSpamRecordCache(cfg.params.SpamRecordCache.CacheSize, cfg.logger, cfg.heroCacheMetricsFactory, + collector := metrics.GossipSubSpamRecordCacheMetricsFactory(cfg.heroCacheMetricsFactory, cfg.networkingType) + return netcache.NewGossipSubSpamRecordCache(cfg.params.SpamRecordCache.CacheSize, cfg.logger, collector, DefaultDecayFunction( cfg.params.SpamRecordCache.PenaltyDecaySlowdownThreshold, cfg.params.SpamRecordCache.DecayRateReductionFactor, From ff01e27da7abc3b456c46ab93e47f36134995b4d Mon Sep 17 00:00:00 2001 From: Yahya Hassanzadeh Date: Tue, 5 Dec 2023 14:57:21 -0800 Subject: [PATCH 58/67] adds godoc --- network/p2p/scoring/README.md | 32 ++++++++++++++++++ .../p2p/scoring/app-specific-score-cache.png | Bin 0 -> 286449 bytes 2 files changed, 32 insertions(+) create mode 100644 network/p2p/scoring/app-specific-score-cache.png diff --git a/network/p2p/scoring/README.md b/network/p2p/scoring/README.md index 38a758db439..e08bc0c6787 100644 --- a/network/p2p/scoring/README.md +++ b/network/p2p/scoring/README.md @@ -258,4 +258,36 @@ This option is passed to the GossipSub at the time of initialization. ```go flowPubSubOption := scoreOption.BuildFlowPubSubScoreOption() gossipSubOption := scoreOption.BuildGossipSubScoreOption() +``` + +# Caching Application Specific Score +![app-specific-score-cache.png](app-specific-score-cache.png) +As the figure above illustrates, GossipSub's peer scoring mechanism invokes the application-specific scoring function on a peer id upon receiving a gossip message from that peer. +This means that the application-specific score of a peer is computed every time a gossip message is received from that peer. +This can be computationally expensive, especially when the network is large and the number of gossip messages is high. +As shown by the figure above, each time the application-specific score of a peer is computed, the score is computed from scratch by +computing the spam penalty, staking score and subscription penalty. Each of these computations involves a cache lookup and a computation. +Hence, a single computation of the application-specific score of a peer involves 3 cache lookups and 3 computations. +As the application-specific score of a peer is not expected to change frequently, we can cache the score of a peer and reuse it for a certain period of time. +This can significantly reduce the computational overhead of the scoring mechanism. +By caching the application-specific score of a peer, we can reduce the number of cache lookups and computations from 3 to 1 per computation of the application-specific score of a peer, which +results in a 66% reduction in the computational overhead of the scoring mechanism. +The caching mechanism is implemented in the `GossipSubAppSpecificScoreRegistry` struct. Each time the application-specific score of a peer is requested by the GossipSub protocol, the registry +checks if the score of the peer is cached. If the score is cached, the cached score is returned. Otherwise, a score of zero is returned, and a request for the application-specific score of the peer is +queued to the `appScoreUpdateWorkerPool` to be computed asynchronously. Once the score is computed, it is cached and the score is updated in the `appScoreCache`. +Each score record in the cache is associated with a TTL (time-to-live) value, which is the duration for which the score is valid. +When the retrieved score is expired, the expired score is still returned to the GossipSub protocol, but the score is updated asynchronously in the background by submitting a request to the `appScoreUpdateWorkerPool`. +The app-specific score configuration values are configurable through the following parameters in the `default-config.yaml` file: +```yaml + scoring-parameters: + app-specific-score: + # number of workers that asynchronously update the app specific score requests when they are expired. + score-update-worker-num: 5 + # size of the queue used by the worker pool for the app specific score update requests. The queue is used to buffer the app specific score update requests + # before they are processed by the worker pool. The queue size must be larger than total number of peers in the network. + # The queue is deduplicated based on the peer ids ensuring that there is only one app specific score update request per peer in the queue. + score-update-request-queue-size: 10_000 + # score ttl is the time to live for the app specific score. Once the score is expired; a new request will be sent to the app specific score provider to update the score. + # until the score is updated, the previous score will be used. + score-ttl: 1m ``` \ No newline at end of file diff --git a/network/p2p/scoring/app-specific-score-cache.png b/network/p2p/scoring/app-specific-score-cache.png new file mode 100644 index 0000000000000000000000000000000000000000..fe5b24dcbe435ccbf4a737e36a2acd4eba7e59e6 GIT binary patch literal 286449 zcmd43cU)81_CBnL1A>YaK>-1gUPT0?i-Jg3dI=yU(tEEF5fKm&>AfhucS4H|kQ#c2 zC=eoq5NZO1!0+J9+&gpMd;QLzuYCA$JZG1)*IIk6XFY2t5sx&LFHkU2oH}*tg33ch z?Ng`7;HOTJ29o~<+@UQUqdRr#w4UAl`;S!a-)DQ|?()>m5q#>DuzZB-7l4Tc!?wFjT5}k?mfI~D0DboF`4G!fv44cTz> zmWci|Jmq-Bctfc#!F}DfuhC%Gpnn3kpNr~Q<}2x=iZdUJ=9>)s6fsxgyxG1{cU}41 zgunoPuWvy;mT9if_!d{-4Vhy4gtCoyRl=g*P-IQIvyWlURTVy{y3vw8-XQ_rdoLnr z>B3ZcVj)}Ko#h@}&6GXg@Kq-#GW^vYDJAz$YPLmto9D>Ry}G9lhY`>LNGXB#kGADl zaLo6uOHWx2vhbu_7r!lN8k|QK>Y2=#O`Sg*uom-tvHP}($2Xbhn*FyC=z=N9MB?yJ zJ?`8?9GK8e7?M{&bSEqqrM&S32gZOR|>dv0()Ap`8qi|dr0}pvj1!$1zexp7G`Jr*~HU9mfcX}5!-zi zcQBi{&`qJ6>~a)rY-}>_)=#Ch6(9W39r&LtyRE0EtCX;?kB^U#kEoD~yN$4jq@<+q z&0E5^ZV3V{1U>wmJuQ3%ojq>+KFFWrD1tq#-0fUF?OdGMPR6ycbn)_(WoJK`=wJVS ze^0Ql-M?pY_V{C2zygI&o(PKw-4y=U*g#jAleAIf~bdr!Ss5b@!BtqJpmP>9x$Wvvhj+E~??{Y`o-r?7y)$JYg_= z;^RZs&>lHAXIfOb%lArXEUqnL>6m!6#>Wjl46{O&1|UtNY6t3N3=p7l<<_t` zI*gp(VJqz4V%&p}-2biY0_87c%5yqoFacFEBZqDjR=C_v^6yvsp!BU_44@1F#6;a}RF)xhZqRsjm<) zC~x&+G0dCtmdR>j!1Rl1#CXB-i2nbUwb2TvJmC)YfHF_o{?da9UPy>WzLLPyLWYag@QxJ_Wx| zY*`@1&*}fWeS{33I930GCGwxo#h!iWa%r=5ks-JcoR`9JsD+LJ`ms`T`GPvr3GwyX zQm2N`X5%>)H*8wdX!f4I?(&G8O|sPtAK;(TBxBCLL$&>6dg+|_QV1GDoCu#rs0}xU z6`NFMmA9yvqhZIJS?scX&2J=rVH%_AB;Wbq#}%oAwwHf48wHVg*c@9@&AV2p_+@Fl z*FAP#Ql3Yxq*&r7A#=74yYqY|H6?9w;7^kY&Bu(jijCy+yEiN;wp%LhbQ~RQ!xOgR zwLCHLjphwnh-*)i5rmrZL8s`Y-h6pnqY3J$YB*Un<`?nu68mYK3>CGI(NAY`*Nb{@ z50kvM7TS1GH$%dCx-YHK5#yKuYDNT&*$p^|ak?VK#x?9z3lsI#B&imc3R$}*gC;T( z^X^pY6vos$PYgAeO^>7rk8tH18s#6$*nw29aQ|ZZQz0Po7Y=?|Y8PIe`CWxYr%13D zm>hk~;{nIB^OUuESWRoGf2HWo>>EhsfeZ%{l0jR`6$4+>Bt0zPt zCHrlAL8fEvcF1^WU3r6XBpVD;_{){wxn>h|V`j}fxqh*9Lo6GENOz|!bV-5FlPG!l)J>m#A)|n>ky7)03IMJq{D9z9|g9=I}qApb{5DwRIKSgDMrbyy#7JTV+wXfRe{854L?un6&~nS7NkP6hBOZhhatZMF>!e^UojPj|e}+CD`;a6DIcOkfGmb`~_QS1X=)oy|QhN;T&g zb!=#xY2`Gz|ADp*t-}@Q;v$DHiauJUC5Y)`lgekN_+-VaoE<7$99-(6#q63rRpq4i zod(JhswAs}kd&7(nMcF%J^{P-!(~v6M;}Zb5~>LQQUq7_jXW5$N?J2P*}BR5wNCH9 ziauDlpT3PgZ&J`-(_jz3;Fu^TQC08ngoL(H@x}ixp8sxUJi@2(ul@bdiYveGnvk#L zG6bv%1`9i`zG~(&g0buwFw0GV-B}(dMo0_ST4>3?NrKUpyD=cu1obAoCHJgncHf*1 z>A|8{+vc*adcG5kd;f;K~TXYf~;4!>Q%eR}6J`IZyHxtfR5+&f|lz@#e7A0chb zkA2mLX64*W$&A(C2v}K0;#YLIj9X%QxyewTG6%h}G`rf*1rIQ2j1_Bw@7{LoG!-Ix zN9cIxvyV|C%6=x^$J|a>)~%7aJCPmXIHoC>G$?DHJHzNbUVj`Tc*i!qY2frP+^O!x zkbJ&EzERTO_s1o{)4Y<(K}QOzxdfH~=N`Xe&G$QP8!1Pc9Z=YLUd#L|u%$gyEB*QH z!(`cxCc66I4yb%yLVq`fgK4sM+hE<%VQuDdwQKvF>-u-x?2X2Pu^W=*WuL~zuj+7f zl9`O;n;4>Ty_}!w3J)(Au3>lwX+ti%yN-N#ryfP~_A=_I#IE`SC->U2ALEAkBJ(dc zZI9Ne-t8^K8T;Q+7MOiw(Z{@ir?9@uFC5wOQahcjLVCtyhy@OEo^Slom|Z>_bPXN` zoo$5dcDLYm3$b6`O*TC?pNtYwdkR(7N>X-i<8hlVa?_Fi_Jz`k*x%Ur+XW`S^%?oV zZNGq^ADEfMC@q8mVK+f;r+z7K7WsxlpRO(iO+dc5f)?5;&!>QlZDs| zwXkG?%?dR)x!cR@#G=|twJ#G2h`Ec^-0RKk)Z!kiTHXWKO-zo7v5momYYrEf#8g21 zs1Q1Wn^7rVzrhVL=kyT^n@VPF182wj3VxphqUL6`_q?@ohM$Iv6yz7sCg{&~Zs zAodf#viXz1)Uz|RZ4Dzmc)!4ocIFg~rjrj7da+T^aKugOhUMQ(z9f9?nu4siP1lab zn7?7#kZWPdaNRAJ?Lel<3I#B`y>zM{Ioi}QaZ3~f+)D3A@PSByOl5qNzWTHx*IDLK zm%+gp8MN_?^~@Y?apM${4c@VDdM9%WpthKc?}IZ#J7HVBiq&;Ko45QM{!a0 zBlYkAnTx+#J+R#3W!s@-t%yH1e+~7X6RwgMOvVdGvJey60%nG!y{QhFM_ev2c`o|# z?ssTqHKBXXe;<+?%{Rv4zQ&tu~c{i-lR*heQK1?g6hHA_CsJj%XO*3ex%H zb<1U4AfARe8ss0~ynYfW1T}R}+oM(tn;K- zL?QC|tog=+Aa9j(!$&CcUwI#TAel+@k}taOvytM%9E;DLy!rhZNvoZ*!PQE=;`)uw zxJ=EY!(AA$WpQQ4AOOVO`_g2$QA!OQOvD+w)Y2u-k8o=}v>wXSlXig3Qbc(0TEL|( zrBD1T0Cb)!HlcG;>M9Q1&!Gn;w^_CvcBlC`)6H$(8;^Gh_YecKCy!7PGMnu{ID%?& z9>8h4rp|WZw5CktvgFoa*Q5%C>^G7d2OiA4CnZWPq7pnb|I7>ji-vLzNpHdDlVQQ5 ze?Xs0j3G}2lT^A*dJK|nh0$o1fG(jne108#%hbS!Q{{ynV$<5L#SyingQ`wImrr{= z1dW&>A;(0Yg~L~qN8h^xQsHSLV=T>XIhjo}&A360Gb41kh!@qNmcv{qosva*5eEoV zCbqY-y21J_fbk~*nC&7VyBl}Sy(eF!akm)%h?0E0Iq(?~>g!hE$POzAqvXY8#B9lI zrQZ$t-?*~o(p|r-vzL;8dYYVN8ry31l5|}6B8apq#LNwQu*LD2iYevXksb#>@$ zjaN&;c24}k1x8MxlSl-3OV)_(b+n_S4TxAJW?rDdu1>ZMIqo-Oy7hJ7&J&M-jRS~Z zfJ33;t4zB1@oa`V$g);1V_|$GJEV;&|dG)s2EDs^@i$i{iS@#x@ zECjPxf@V@5e)v^bd!fmui-f*j+WYX#=J%f*?d*_5eRJ-Pu!B};t>e~<>ixEsz)E|F zN&Mx4$OGlRThpvvURfC!ZMCs_|6Qhn_vou0KEZSEuv6__Sq|khUg|N<5Md)rAoqD? zD)PhyPBN1X5wVM>d@I;0?~=)__9$b?(`)r`+e@)xC%zIa?L1r-mgzo4Y;ebW#Yfe% zDnk;Jv{YYAsuZ+(M5HrwCwmJejt&4(1}q@2GxxTVb0Wm*eAO8>9=MjoMioLKBh2;|qAOQ)fi=jC#eiPb(d-5|8mQSOiTdiO}r01BO6 zCgL+5;Z4c4(dEiQ@P)ZA#9!V897>7ut2z ztPAD8-d~MRSfg;VQRT|`1xa*Q5c~_R>x3{hiEYjrH%M-KARcmpoR8vA71EWIa(OE$BGpc+qJ})0D=dn|>|g!!ux; z{$_1=pQUwvCQA0l0ip7uCUgmv=q~cP`vn%J37rOy)oTwV+l|N}4+~iz6lQQ|pND)G zVUl0&UCJrzy3g9o`%a7ibW`bmSB`}~gnm8u3wN3jcB*H)=1JkB-?Ilg5_eS4k79!q z+~wFPP;hpDuX(^W6w(#?{FsXuY8sbz+rinvxk_naI6I1A>Yd2MhFa2LBDtE}75QcU z_$_fE0GCv^w_Kx<7lQ_6i~9vG%zC|g2kq_ zCb-CWo?gpdv8*5dvUlfE`m__nHHO-lBR=YiP}tGINS5D^2kE!m9N7c&$BrJeGsKI> zS)3-550n@dlJ=vDY7sK8a}DVxIWE|~*eK5+thv;*f01Aq1)k;_;ErWotZqj>f7jM= zzVc%52P=95e+Q8dRfX8bsF)evj^?1S#EGgCe-;;eak_IBY$9gT_&BHsUp*(TDPo>u~hX=l# zz4)uhRz|v-Sno*@{yPn=OCCbhYWiT@;Or90+9aejB)J+zrJfCXuTqWC4?$b^j6V~|)wBifYOls-N9Xm%=aQL^J5ooEw;1%ih0Cm-rgaaC*Tj-Hs=w*$7 zyoquXxK6<_5NZ_<0XIBZFoQ|GBh2N*9#2BW70m=r&T2{*vmKp$FN0SLeHmSb2<|E% zm9QIU_m=CX)bWRoB!4xzJLkWvvmWnr9hWUY)M{U1DP?HM5>T!EJg+=MoXAQgKU$BX z!Ys*-M+s`$GSyzm&dmF&rP#B5my%yd+K$R$w5eDSv=LxzZh~;*#k4>tZP?xi##eCF?gw49VDi4sN1XD3tbsM0K8OB5s?|Y<@-b;a zYU5L5%@Lk0vW{*24YAo=M+5s_4rwZn85UhpRiN!n3^d8=qZgnaIDB z{Pw6dJsU@_^ks3A;TM@0UpNq$rV@3y?MZ%THh7p$LviijaJk}#P-RTKk69%2+tFlk zk=|Lb*nCi%h7&R6o_DC=L26lHs3LtAS%z<1dD~;wfP;32wL(_qLL)A<$L)a=V@^@Q zgfvh}i}ge(nPKg6uWx`yFQ!FpJR-4r#-h;Z1cHEsxXN$Y&+f1)wF^L*2y_JAFye7H z*Ub8NBV(%OP0&dDx9;IGKQ$At^7;9!Z9$Xc#WXni$-?m^#Do)2y3p+G#hRifD7JM4ldM^#^UqDi4j)zwh`%wc&9XgDSQAW!g zTTrbG8cIH_{{j%9NN5BeCF!#6Xwqt%+r1?-b?*p*VYZ1nGJKM06?I+8b3JV$Xg4{2 zCGaas)bZ=HdQqkCjK2tYOApy}d$%9eZAps1w*4dH)4o*1I`b|7DpXuCytq%ln*}Vn zF@_hhtCAGB-23dwLgm1!lZ2*C_VqoKnL#9p;MWiK zI1zyhD<&PYt!P>Xh+Kd2z72KGaoX_hXrqBH&i!UsW@LPzY}QNI{|3=)kBmqsw_9u0 z$?E^C6#Q38^k1kXB8UES`2n{pN4wbE2kz>U=+&3>W{HY@H^VL0J~2Y2uk72jP2Z%@FEk7l<0*|oS|6T-C5OPE#%JDd%EA|JG#5{6Y|Q$hGI2x7tf}D+`q<+zLZ`l|3-l&jzsQxzKf)}faDP%619fJL4s{E#g_XKcjiFl%HHc3B2LXW|Ua z$22X*v}UW_Yu`{;9kRJ7^&=;P4pow$6?(9X=H>%MZqcUd{Wwj`yXT^QS+`VtC-$Cm z=)ou|8{#b7w(}O>o+|E?wjvsI;5mZ*^6~-In`T#nFp z)s%J+cDNOVf1F|B(l!=A#_T<5;XQ6-W7Oh}$ygyjpL`dvBt*R&ac3lhzZi9Q{FNJT z!=bNS;z4ro<6@rzsa3rZf^xgh_rmPpn$n8qo|osZE(m5_R6Xuuh4>;|cCQ(1t<}Q) zVb_z*{O#a`vEsrkdnU&!g-5^%ue&*1GA*wWoW7U7vU8Jwb+UYBw)!&o9&-G&r&lo_ z@GhoIrWOgLzY9z2F1>?+nP0xC&zu*RLEwJR^Rtv|?S%DOG8W$t7iBi?x_oN)+g{#( zdjF7w{44)6wWVgC&4rE%wHJvg9{`KElJf3gJYGxA4tTY|C(J@G?mufkJ>g2Om|dEM z48l!y%?IJJ`?c+LDjL}k z%Y)am95DQRfL)@QO{Q0Cxe?$v_~Bk#k^B|5;mq!@+77pRvuvINK9_TtPA8Z>oP<`Ua?J2Uke=9?q(29OeQej(gw>BS0J=5?TKpSNg#M4JPzx zar`!8Nd65ret8WNFbzgS#(DM*;4B>qY(e}Yrw<3Vr0=9jDB);*`4B zyiCPk5C}0#bk~FxB}hLe?iNK5Z)CT)MhJ5dIp_dBE|nU}E>9_&SByh6n__#SFNchKdN}(#a{PC=F z)A!$8Tw1J^)VJK+{bSU$?h@>33%ysMPkT38uxA$EKu#(h#*H_hf6q7TkjNThAK-lz=I7q4^bGk9$*u+c zo888-90JgaG^8pBP&*FTW(8riCn?Vdr6)&+`?zU`%6wD%(V{r~fEVuoF%B$J zygB z1pSyPwKH0Vc^o?~8{8HwW0CeQ>lSOAH=qfo^}k|IguF)7A5ru0KhyPiQ==dD>2LM2 z5LHe#pMb(F|B_n&8pD&2SPorI8$hR$1njONjdBryWdf9`43HqdYzf`$PSzOlVXcKCyI|AIA82xgN}X{*%!;xRzvBOE%JLW_C@9~58qGF3A*tKW&zZk)+BhaP0EK3=VH&@mT6Npl{Z zZ`&S_#sP7wOH1d7#s5d{JbpCc1H0yQql*i7hU-0Q$@koXyW7cT_xvBGz-?3f3LD>V zywP^GIUIP-?RC2bEP|4l3nwBgVwgSNK-smh!S?( z+xR}O7=!Cc2e4IthUAE+M$g^}+VS7{@jVDPX%+j{5Z<3I0XGAHEL>`DL`z;Hf%oh? z9QBwSB;BbY6;t*3V#7*7x^XdAN&RtlSzMsRTV=qX|4pNQ>e>N_9D`2?~H1{P?bBCk+%WU*tzfW)(%n7mS zSjX1Fh`4mw6JU`aJL|5Qac2mK?OJefK^8h%eEREBj0aG+f!7-fn@RQ#l;*X64Z~e9 z7pGrNp&q}p2h8YeW!Eam=yS-gXr{2G%zUK5v!gh}GE)yhSX3zbT|6yEw4}YErE5{QOpR z4V9)i9$<{+!)(ied+4y4<@h9lD&$2&_)7sM0GUmEIcn*+N%0Pbnzmtq9J=X1UiIcE zgf4_fz)G@95v$UDSpn-SLD>%#kB&n@z4*fI?C_7B<_`+C-FKpL4ir1`XgZaHSX%&P zI&OUXC~onI)TGWf4nL<(>5NA}jx!ow@F<&KZ1y?YxObpj-=?$R)V@~JyAaL`*_Gz_ zAz`Wdgpq@idNR9%Iq2VJHD6%nQK$A=y!O|H3(^>bUj!Y)7{t^v}|+u4r8 z3rE0>g0tgW!#PgvJ0>P&3m!CSQEwW_`@`A7Xp3kA&V^Z;Hxc^HCxsm9RZ6F(x>#1i zya0e(=Ro>13%KidF7SA~-t25eJGy;K|Hbh2Qz04TGScn^7p_)$Yq~{{hFFr@lK6*S z5O;lg{VXoY#CfGvfAs@^Po@B%YU4BSS2*G?Q6$6bge2Jj&>x9D}LnV||I86NC`7k4)w@S`A+(;t^PlTV`HkdfvdTfG*y+KyK~d9*XcQ z`B&5c9^PMmIet-Xg_GeR^-H2Ag*$HuQfLn_m>rOWkXh>;9r5|LuZ;tn=fzG3061?3 z@SOJAhxGhuU%a3?!s3Xf9m^2_R*iUM2%xe8Dtuxu7j&H0UfNS~o+GMyK{9eYG_)P6 zRceZa(gx#^kqNv8Ah)o>9~v=>QuFqu<*iE8P@z#Co2e2=!4cBlT&8dLHELL5VdXQk z=qF|}`TeE%FZW`6O(^zDBR+k;C@mmpR}EUcYG2)V)h5FNNYb=rcnV?uEMf)AixlH} zsG_cx3`YEMT-~z|V%4QeV1I8=6ysJm;&S6Gan`)Bao)dI6Tmlh_;h)*Ehs(NimWh} zbze1wIJbMa-^Gq`$wH?vkBVcg4hQd3<(m%VAJ!?VoVKV@;p0MENFId2v>#U&C2T&9 z(yzNbk|F7F$U}qDE>er5<9Lim=J=C_F$~Lt?D+OG^BN!kP?;aUCv?%D%3;Wr^I-x! zxV7z|+yh^ydE-(U7Vft7$P=&9U~E{M8Jc>I7g?(yqY_yj>aU5E|i zmW9?s*nyBUWL#umneo;TdaXSr7=-H1w^I<;1(^{ngP`+x9!J0E;Sm8Faa)1x1Z&`f zC$MrH5E3A6p9SMxT3-8gc=k^O8cQmVlNw}6sT1^N_jC`Z(94RO2F774P+0&awc6Y? z?|=^XZrQY88n$ILmTpz>MVAIsMA^6MT-%~HdSDk9V_x^<0p-_EZ&X)MX^+O2VD$Sx z%uqM9nR{zHB@lt}TDK@`VWsQ}aC3LgP|!42SGTu-%MGFK{6GV#u&{ovB9MWvd0+( z8N$N#{E9N5up*L;pmq*TDk(t*a#Np{ zVho(?ky&L|hDf)8j3+hT(g682$O}8)Hd5xjT^uTKbGKXBnp42QA?;zW@ipbO1bYGE+jC*g#VdK=gC$OA+x)j2CpTh^%wmw{Bi{pQ?yZ79@QYRYtNjJS6OX zm8UT<|LsX`nzeAa7%M*@t)}6Pt?3mqJN_!|CZHhMEA+m7|NS|SzE_sa35goK?{)V% zCX{;UjA|W1-o!*v`=w60bzN&KKTpxkU9ws6rkOXop~STtpog%6U)@dly$nZ*ApN!) z_e7txdrJCITQ%>L_@EC{h_cCBi*Au##FD@?F$a2h1qXybOvBx(XH58~e%_TOAiJ-p z5q|IzM@sk9rm6J3CdHs2%YHV;?tu4X#D*ehw+^p z*)@IhATJkrY>salU9a-(&7!teV&0bZV;+1|% z(mptl%2Ri90f`sNtU-11ewrv?lND0XU*TO7xe@91$n0#SUZ6wzYqK{0%MV5Fq|Umw z%&Kx14vLmWC>bU9GOEKp-h6!*S=K=jSJc^ZZ?)hyBW38)TQ#9g`gdpj$jDWbU3>w3 z!NX-dn-rtoS{LWzpU-BjHzI;V0-o9D4p(=vq>n}3TsvB~SAJwTQDIuHk-p<38oJQ( zNn*xP)WB82BJm3s^fdAZ1K@?_AG10)xP29jLTBdC_n1J!#>|jWzPbV-$k{WUFf4;fxt>5(O zL(%*$@(L{K1C6S-Q~6R_@?b_+q=?J2wD_-eV(@U*;2%N9l=<#rcXaTtwJ$i|)c89= z?#|5?bv3!8kxOGKPnu#=>t>N7#)z=D7#mPA3NDtl{(7keUD4wBV=yCO0#$T984wG<}91PFg z1^Mx9+M}zI!&$EtdUSbNcMnngfK?!xmucmHw~7F;gInaf!{OpID`>rZae@e`4|gg~ zMEQ?9w_wCkm^U?Mgp^oj$zdiYF`R?Uo3NO9^VEZe`KzB_Uhr5>*Lreg(NqpeSnOeZ zG!sl3A%Hb70$+wl0wnpq!cZhdY)v*mTlRXRGJT}b{08~%5PoFrJmoY65-^wTtXO~t zHR(+f92vH&j&Jbq=CHuy)xm^HJGW;I-yg0=)x6q?qFd{BTj)|P4YSFAt7;iPTL8dM z9z)%Oy&{nVfNS081V=Yc^F3hksY7@G^u=43xWsqDg}OQSH2GFCw?2}w-f3&|1QK`S zdV4*6OJpF<6jr+vWF$1T8=wfAiIG3@ZTa?`vK1k;6<(gfC}1CyLFvMR_51 z)zmy@v*i)j<9vsqys*aOgF3sBRzb{;=0rwxJDQYtXqRY+4dn0vPDgmyD5?LoS}-a_ zlIH0hA&|NAPLLuF+^DaoW+!COq8({Zq8Zr*BqWlhY{+Q3zM}|9r7TgKm}nN+=~T@S z?J|IC=^sF!5a2-e6TpQmsH|9ud7U%E$-k}_6!QUtp7^6mw=T*mo4(5z#lJV zFHI;yLA^cEdA%Kt6i|R}ZX46=)b$_r+<*zDK5A-e6-ms z3=6b$(-#3hRV}sQC=+=AS9(KsdTs{TQQ{jdkw6JCs2uf0*#GJC_G85%1aY_9y8>&~ z9W<0bI_we0%pF{Y^#BaUn%2bx@Q)PqiE?s6Eo(LN_jCmx#3^42GH3zp5(CV#L%+fa z2^Us2kCA6e+-V%1=T$IMRb461yL1E?OZ|`WdX2<+CI!%r^y8!h;~#|xLE|;dTfpHY z8uV1%MC?9}MdUBhJTObMc~dAQIH-z~d51*-p$w5(fo)^7i6Ua?9|Z78Y5}J*1AU5s z64<}1)jjHW=?#U90Ez+hG53Ya$Gv_Dgs=V!_n`8Ke2`+~nes!Q2prYLw*)KjY5Qz}SMQQ; zq6MhV1WRswI3wFgJla^=UW}0z0-JB{i5b->UvFkGcKyx(y?F60+XNJP&NkmlOnh5bov%2-PMTbp+DfUV zddXXUf}O{&XQLOOFtGK#VtyIS1^{I@Ma*;4FnQL*5qY%E*U0#ZFH|c-P zDLa)XL?yfvMrl{_Eb1;uE6waQ0l-6ID`^+tE(0h<>aU1SXf+5{_Ux}YY=+~E@oE75 zd8lBAGG;41JE<^l@u^R`{1)eVa)N1P$pgvY_vAWhzJy z&0AHNF&M&A$U0<{Ri_a&dI$MY~I8WrUaYzQ5}o(aZPga;2_Ceo-yG zN|Pc#5?ua08QAwTtO?!~5q!y#sF8P4)LP!XqP4z$ z!(m4~O7750dhc7PZ0xlo*G#fGd4pMB!<0qS*4LKDEUq6;&wskc+gTft(6J?#+il1j z4;C{iQyhZV*S|2oz1*7!smSvwa%jCU-OTyJFJepjUWHSXl$lsnM2WfL$2{#WWm%tt z_4v{S&}Q#YvYvDP!~Axgr+q2k6_*g7XRh+n^Yj`L>Ej!#0_C=<^3U=pXb;#xzJCfE zum^A1FROneKQHQ{?8bR6a>10f(pGoTr?pj;z<-9sNpGUesJ2XR#2ZurrR~SqDp0;< z0?s=_-Pq~>y0PpzL)dz^L3GZEr_}X*DuR&D6HwdB{T_VWNPq2hT;3(46gi;y8A3?h zM_;o3$h>4fl-*8;vY0R;@bvn32=hrWxC!37!a6e}N#UlEqn3&Q3Z$<)$mR?Tg0G6Y zgA#6O=u7YKECLm-vAFoD33CYMJQan2&6I){A6F|)=WEAZxDRt*lDX~#gRq(7^vI~N zJA}_+D-ikWMvN<0+^(M-4bC@N%e1Cr>>e{zJy{~kQGc>WZLzGipsZ!RdEl(tR8N$7 zY+km*<6(g5xh#BZvk`K9CXDHdD+=Olu$Yg1jqB+_Vb;ac>+-{qs?RgRD-BqC3#2lp|pINu?#1JIXY zsXi56U%Acbu>O#sPb@GZQm+Tv5E4I82Lc#sDJ;M~Z+=dI8NZHRdH|e`aL#yj_GhK< z_tZ4RSmnu3p3;CfLC`KUj*p|t12teL=EiGzSw%GgK-K($M}T)qsvYw)UXVGnv^rOB zZudys-3mpcG5>%wzr^DWrK=H5Gslnpoi2@e1JPhXgVI#r!+S?PnD@P(OvfdrI3D@| z^|)9DS;>h)t?UG1f9;skL}&sko?B~x(~n(KvR6|#?*&=n&8JJl8G)T2L0eJmF|8U+ zWmWHgoSzPCJUlMg;v;ogQ>mE^-v!^_%T#p2=? zxD!~iy2pdPtYE@ID(iv#(W~IatL)1O(*eOEzI@$v_O2eWY`g~su+hMKWH4@*@Nsv* zcOimX27Qo&xHZtZNAkIE#z0p^^PTTR3m@b19)_=y<`YmqXBx3Q3IwE3ft3LK(X8kt zXnsGVTFlB}c3H%$KR&I){f)InQVWv(k(4WO<9tF;RSrr0^Sf`r_HVYJG`UT5g$ldp zjU%pEpSzTxmGE6b^_wM^%clxRyVp2JqS^{lTfJpY5GZ7PeL))}a&b@P-pQPr`tdLXy>q(NNSiwLBM-t_eVX?*8% z?b+3!W$iM1Y}d`~#lWr>M-0zSGpMv3DBaAPr5c})@24^CiF+3#)${(+t9x4*R85hO z3Q);*Tpca;3%V^t0C&ITT*g1r79`Fgl6VG66E&{<3;o)>b--mY>Vj?)Mk&lBJv41 z?}AEz16WYIj64)K4it0jAA~kW%Y@M~ntbrV31N zKMLS|N;3yZMt4;uqJU~xq0S0NZ%BjF%+nJ0g*1>LVWuofy9bY%k)RfiJTrG@YzD@2 zBl27d;Kiezpk1$Tc7Akpidk*$^iKY&VVgS)PVQ+FEHiceGaiEGL&^B3q zo!gOaKpDq(QH9Ip7ElOy?o{1D?-x+|P)Ia+F_5s|mnxb$1r*bU1~%@IaBF7RmX_71 zvfDuk0eS3~eIH>QRi{o;ZJkbK*&e`*FHeTKS7;|-6WEl3DtfkoV$Q@#*>X&qII5#n z4@&F@1a2OHsmB3Kc?w#(3Kt=d%J2ZlUZHJ)pu%4ddo z@Kzcx*p>~tNH{@~1T2KEXb2T@F#}bZ%2{Yynk&}!4B*g1Hpp)Vhj%4qoK%~1uZHBv zVs7VxE~d%`G{|m!y-|=z1S&{b;Is|l_rW~~&TG=%o@WizQ+n+yEcqB7#Ra0vWY-TP zm%e`vPpA9QbHj<%}ZGo6J2z%bkW^H`#{wN^wG}Z0^{wfGZ*i$3C^;z zaRvZdQ1!*tH-TWv0KIHOInAZ^efmI=)#y~VTjy1?K;GM;fW6!BN%T1Ex1zU;4R|}T zfjy-!n=CN~SuONIy7x_b9(TDWqL8-K;J<(G0q2cN)_y`~)3{om&+aFdk*Hi%xysvN zRWW{1Ma4tzeA*ieOBtINxk}7;Tyq*-Dj`%~fQ>&t51bol*%?+VcbKTiqA{R#J*j4{ zOvlO}rRa0>4gi&~#kvjc-U&ZNx1huqmmTqryE~b5z$c8GvEo!V;0S0BonO% z)US}_H67LR=p17gafX^M4N_kak0OVrT%I{O3O5B55CQIMu~on1L*N0F-??}-xBQW{ zlQ4gSbwmw}JK5wymsp%e?!Qg5;-#G4Ij>pZL`$nu@ey`Z`w0XnOf&$zv$9#gsy`0- z_UPnHmqO@oQ(hCMHddXlyT2|1iHVpKlbG+ux3U>w0Bm%sK5!ixL#Yun+XpwLpD!yL zzd#3ZR7&v-c(hzdB`Pi@k^KkWW>md9?t5~aq(L*FP3bx-a2q3%s%Bkg#(5y8FSWP zKJJnJnVNkO3|flj3EI6uGhLcC*+IEDN&BPs~6vypzekZTD6b zo+GH55$E9h?c@xmIfUqC%nt)dv$`s6vL#>(A7lxze|G=Os>_9-wm-rb1|BHU!BsRE zSwnrprjpfUy~magm+pb|mxKMjg^;ZQ)vmh#$JkqkMZLBE!it5UB2oeZqI9S93?SVh zJv4}vl$4Z8NvD*+(B0iANGn4L(kM0WbTluO@NptkyL3CpUESS-`@z7tAIeE*Qdvw^5slF*7JMc zSM&{o3zoRSBP@N0kWSpcd!Rc~)NUg$u;01g^k{^U;P2`#4GE)RMLy?I7jQ(uP_q1` zpO|(r^fWjv=9tq<=CRoE%0|gLq{eZ`i$)$%$F1mm@%MB;Tpj#<^5*f$W<1O>$a!su z)^sf?_UBUS{zq|#5?P!rYf68K^sIA}`za4EU{P&xZ&ATS44*C8(y}V-8z*|SQ zV?xwp4PjJJ2x~!+#F3KNg>)JaqbLy?>%o>MLDhREhD(zi*2~Qie35GLiHYv5qD72y z>&Fk0;IV;{$DFhHLQE29Aowg;myKmZ@-pMB+>V7@w>RC{kw8#l6QlUd;R#TsYjMBr zb)*Zv5s>Jrl9+u5aRMGu#Q|yS;&S~l4xG=<;M`H;{HMvVfYa}+pS;1duPyyGEkUtk zYIb<>k9Z+f`~ibSwHFOb7&#~ilx zNShjjWFNWLMU}$#7*Jj6%-_h~fMMif-CV?Hv%D+WVL#;^qs4gtO+wO#9SQhH{0Vnu zsZTicQcE_(M$IFH_dhbf9W4Xc>p;up1**!e`R$S8acR}ZYML+0N*ao;iZr*V8J9?V zTtjsLXP(mG1D;X;uG)2}!C84#gyWLx@c7a&=ByOc+2we+?06Vu#p)EoTEo#yukW}R z#y0`V>~;g8t~Rd-ibC+s?&-&L!zCq;sXB*?xnxB9ccC#kiM2H=LTOU|jFzwf~62W+JliX%5^slCiMZk#n9{;IS7SR;o5ZQ|W zEihVXp*#r~>#gtHh$`mNrlD|=W9>=DK2}8ndDW0UGT1ip$)p`r`TcB`Y)sRom5+4& zk2gVu!dgvBWq@$!`VkF9Hf-?A7$@#rR}koPsV@9g#?Tp>*Y(wm?yU=&%e3>3j|#&| zA8T#0x-km0?gZlSC1$=UdtYwZNR0$;M^@}}pg~P8-uzXb$vs8%P!c*e%F5=7ViIE6 zIAAzy46)$#`}yV>5(5B_WbKtd@~S60dqMki&+zgrzEV(^r2yS{pur@_-N#urk%|Tw z>o`~$J2v{|v$<{GCv}VsjlhU&7F+1hoAG&DTE3C6Y!(1f_3y7NL3AT*9Q^mdnveq4AAW1Y*|jceS7W!p~Hl~L#1b16Uwy0G^I#XC8mxJ@-`ZJnP0?bcJIudG ziNQD0Jp5~&CmQ-o{X#o7fG|(3 zynCkfxb~MCixl%OeABIY7(yS-GOSnH-9C%CXg19OtQl(*|wD1o$1`t1#5n)aPvE(tBkH+QMo6YQ+kK!!bL{sk^ zhW6Ncyx3pOj1UZ&09bbNJo5TB;6jv`VXXIivZ1GaYKA~oT53KSEjmLBj#cbFjys+} zXjtvCH6c)Hjsckx*ELqF%eq1xLwy~HOPaqpUydL$A~Dmn&UbM5dWC5<2T4$s#dmZU z`lI4Pn`OQ?>far$sIRTO^;sA3ny}1Gq#Mo_y{GFoZxurP)KWDJ)^jEq!T<}l@@ZNE zI}OR%^(Y2rwr~nW3G8cG&QBJ9iTy&0)E8L^l}J2WUMEp4%adI+1tWO%QK{po7LK(} z%QL%~DV?ytj07=#?edTv3Uj*bv<>to%j)H|KIEp)?$=w-fLeQ{kt?sCjW$OZRRqQ@ zMp=NxW~V(!^+RIRJ*>i*L!R0Ycj}ru5gucGb&7WWGN2d_ek&S)0W(x5$My->Ey+js|ihiUY1&6F;-A z$i@K~7%e^6F6g)=Yc)?2W=E>L6$Gu=Yb4AerhOzZHupH#Rnf(Y>kR-d zDj(NIi^WX$m|v%euiQ)%BqF0cPMC+o!p6A2J8&2TOl3Am1nZ0$#3%8vWN!j->}Z3t zwM86>fm?ctecNr);yGSAYhn8>-{PPRpga*gDCrc{x$KZcZvZLvUZ*J9dFzNWf*C;s z#ihzl^pY}s1mocIyuk2e{CuH*XK(W7+sJJ>C6FDjwwQ21oZ>B5Xw2o9*THP`Z%z{Zx`??8}<+5!d@LW{(Xa~G$V z+U?hdE&Qz9d@U*e4i7(8d3bfJkd8Km1s8~$SH4TgIq9^lJzdT&BG#+1u{_%=?D0TH z_AQxATt0uXJ(cZwsO;Srt%8#c=-+-bfxHG;cSe%6q_U{VafM}m=F7|X6ZCqThg$m1 z8oE3`pWDU^wQ7|b7*J%r8p~=tp2K{&;i0U!(JkHXB|{??2VC6oFkx#rHLE0D3RJJA~%9TcE) z)J$V&@TDke-IML<=9Wya-#d*vkeE-t@@Dmqt!GR?10Yei^O^WlDE6K;)RYhKy0iQO zWl53Xm(?W(v<_=@+_?mtNxa$DkG+;tb>%BMPhL%8)!U7YQY9^oBov91Ejvrh&>s#u zZK~}v+l3vF+g3&0v-Yba_OiShN~e7^D5MiIdN|mqLs0gXz3;|q0;ee!=|`^t@8b7C z(QJO>2wrdN&7mh$mqLcKx7mQ=YuH54OIZcAc;_V|b<2x=&@~8y&J@)x71QWco%SD{ z2Tsq%Y^d0Fed^AE{cY~7xzMN0`6)No2yyEQ)RaU4CHJEPTTQQ}%k z)`nV9V~ttxN_wmA95D@`?-p_N(KQ1mSDP&kDa_*aT{u}eR;TJKds6Br@@5}_6J7jT z%3{1C-}DI8Qgb}vL#Y`{|4;mnL;q55r4Lq4w)*&BEhdlEuO19o=fiK*&# zV>ULnTJ-7gOQCqlMPNWu9K~WA>~$QQ87-;P6@Eu3ElThz$#cjonvO2`lfz=Cd$0*( zI4pz)rw7`7)=0=Uxp)-ysSftCx9zFUx4$rxv*(U(kglZfBN-oH63zpo*R!J?0??rQ0?}UCxZ@=tEfg|XQfnO0S3AL zd0o#1J_D5&jM7rcox$sz^atbBC}FIx@@`lDc$fauO1_900X^p~Ast%?i}YdghoxxU z?J9Jy6YXeAO=8hCUNP$0A+g{C@!0qZGoR&8T1`Fa$z=EM%Wg$mn`gOU0pmu4g_yRd z(&f@)YXBAM6|IiPR2LhrgW10R_#lwvZOMunt`uy+k#r|Gm!n?#6R#6>M$lu%b=*7Eg1)_43Kd zafkLtRAEaS4zrtw%QM@F5g*14cc9mi!`8VHr^e-e=XN8=exXoZ(fIT1O)wKn9?Ee9 zotYDxyOXhBLsV6)P`)*SS^=%CFG*1#6>TGjwh4C9`2jun&OH;Sc5yQxv9`5HEjbWf z!oG6K3i)i;Qon*My07WjQ(AwW6^F?mXt#w%v*qJIR17uBgj_LGj98KejFETK$Gd-H zjILsT!9tZlM?;~mC zzy16cMi;QOP;`F{nD;DFTEN+eBs!T#_PSYh2739N?OS&DC)x&O3whB zPL5e|$%-|QVJr8t9;x7v)Ss|QE%|@3yIXQzGwq^>SV&7~LRSYjlVVtDhQLKL~kubw^A?n_^y(lQ60Q zqviOQx66)Vh*DxrbG1}vIM^I)M@38gpE`4r|5o{yT3+-x$MA94qS8AuC-!p*hccz{ z8o`4WL74A*Ml1I2ZT8tV*O)U%d|Mm0fLEu@1{nX=m+`914qDJR?BCPd8nctFmv*g> z#9PQ@-CjV!Z*}zWc>=>h1(q(QHcw;TS&}K)sE^fu)v#3IoUELZ&2ijV z5VJ3jVhgDyl$=mkWy|3;PDk5;!mfqT6-C4!zZ-VBwtsz&q#kv$M8fz_ubX+RX^#kR_eYI ztcLzs2oug=gmpu+PX5Z5d@eSFA3R?m_zE^`^03+IkWXvpmWapkVGYf|6V_((im*i{ zX10z|fmCJ4qQ04`)N>pm;BlHBsO)wyfb%4 z3XCzw4uK+nUH5e|f@e6jzGuyBI>$92b_bvwF}3;%`$uVu9H_=#mP4cyYpT8fjcR5} zgI7p7s%M|wD4fuG{TNc%ijqqIgk!yBv*3A68BpOw-+lYC` zHG()x?rLh`YGXxHh(3Tx4Jdn8fRp3L+KE|HXdc`PJvxw~zr{LTs8yQWqAp`uvNL66 zNGHFLt)+R}H)n%7=sl}P< zqHDl%*Nd5D5{W}MN`KQ(dY>8(yG|-(I4%pwb>2&A7sYw1LjPM%T)mYxe%kJ$&1HR1 zkagB4#Xgo~)V7a%xCH+X-?>b?i`KiW{NA{K?9r!(F9$@kj!$;a$69|yUe&JpeLX}b z3LkLBP+`d>F!I`*9j>JcIiqDpzNKsCZe&t9c}to?h(?qjZsivS;d*(W?@1%jCEg$9 z5-Y}Ee$FnwH+ofvvL>1UK**NF_Lg3drLPEB|8xt)G1*s*AM z@!XWNYyyTqcju5U_9Z*U_2dmeZ=ox)2StXL)te*fOL?#|S283i zG(MWqkg>|sN5HsjFFp$4{tNv?T|dJ_m37gSPIJ%NgxK6LJpS{R9W^>LlIP-W>_N&x zX~@Po=!(bQnU;FRrxrjtGNYY!BbUsF_qexF7J@qgGp}z0-jc=Tb-t-vRGxF_d+K^e zMFnAMDqYZe7L$;b_;bGRO$T20nAiR)kuw8kS4(!(&Sn_fuj;tUJFrQy4bCi`%@Dx} z99VxVDdirok&f;NqdRevgpdHE62?7oyGW8AfQ1lSuL`+S@0>5awSS~nN8U&Z()Azw zMIe89`ffk@)>W*Zm_!NZX?82ztJs86q8^=nKOHM%REs*WkV-g-OChN0&x{t1ZnVGZtDLDIZ=th-YqGWb&aW6l6wrR-Krn`;P#*6%73)V zx30Pq?}o6XA7~^qgj%+o$>|ZiKOY-=h%x1q=nw`Y0egm78J`MxOaxc>vKaVu+17>rjeHPPr@>FGI zC&5p3Vw=y$80}G|)OR*T5We5mVC(*Jg7;5qRa(>a7M~5>QY@`0Z##EpaS)3vs&w7p z^o)d@%V;Zk;lb^{Ge>`0T$mrZ2tIad>HQY8|Nj~A|5!y6sEhl8;s3PU{&OMyXa9pI z_W;|kVa1*Q-aPYFa%arR1K2lG8tY%>U1a`Zxu=n>TcR|Efg)j}IZr z3QB)d+_Zht|6RQP>r2+ayZJwr{(s&sxa9>+I=kUj(IWS;|>`xc;~&kA`B<;@@VRHhzM?#4k2c(z!Xsz zaG-7k7QtXdnVjR?cz!gwMqhzWXcG!@$_}^EzgAbw>e^C_L@sP0SX%Nf2qi1`bluuDF*+r9R_CTCgZr!3Ts= zbAVQ?!SF}&=Kx3@Uye6?9Y)5>#8Qc^rlIHcR8);qatgygfa3i2h*a3kK573PxdK;a z=>N=X{SxG`q{$9m{QhjG3;0eJy|ZA;S9eRXPtq%1@7AxRepX4gtLUMt`FfMwO>+gf zxl4KdfwC8f@~ERp!pdG`3?^s<_@v>P0v=n+KyFM;%pW|s{u+3 zs2JcP;1g?3rqV-07!d6(E2=gA6&ciDxC!pAn8@b`pkCZ{W;upLQeu6Q09jO@-tgu9 z;{p>Vzjv|5GRp+7rW>3yqScbDxtd`WU=2=yY?D>R-?x6IN3WB==$as#$x-|)G5i@w z3;4sSnt;Y|?>RuuP@8~$I%F{gZQ4xkGppAV&RgjZ<{;<3eDR!X@L$=`Brfw{FM#bC zbOO#{ml!_fwitE{{35@=x=gA5;y4DizTxzd!+HG$;%F26eR2-}No%Q8=)?JDD29x9 zJQp^u#Q;6?m)z?)sGEn6JVcR#ET2-6JAzvR(YV;`iBJ@Bh8N#tEl%s+Hh zD?cjy>!TmK?qUqBng;IX2-Ds&=1vIhq2))n`Y2dW>FO_uKhW3JFF& zV8Su_E>~b@L~Fg~4_JYT^}^C^#;?onZR|b7nV7t;%L;ROc-*^pD%kc^7hjY55)PF& zhp1>z5^Zb7JJ&~xKWkAhmw;duX9jgoe5L-3qjr^l~;^S+B*eC_K zSyYEQY^g@TeRj)@-zSs443r8f!&msUntEO#udFm^e^P#*h=9m|I<&1Hjm6>4^05mM z2ce8;od-+9ua6Htbs5oI`L4F0m+Vwc?0%M5q>8vW$9T#4? ze=jemFx?cDR8ggo6ucf4{a|qCoUa6REV47@fvDeeD<9oCnaNMNY(_vQndq9(w;1w(I6dV1iz{0P19b-dnLnqM1qGt5v)A;Mm+Qb;KIA&~zvG z#w4ooDz`wTQ@e~c5mn!7D&SnAc9b5I|l8-WaRIC;(AzVK*MO1zuoWq;0$`E;^;W>WMgERBRDG zp*P4zJ-{*2abxXl?9C-$${o(_uL*$|0kNoZ&^3Ai3B_JmYT(UVTIXp`-j_7yxeDhr z4d}bq=QPVoRWjwp=FlrIu3c^#C6gG>Zn#azR0r7}E!Hzj^f{fs^_6!{*SdI_kf-i? zXEy&bpvGfElvB_Z4Dc+ysnv{M55bbEq}v+G4ms$(uNhfxu(df}sSmZ_!a2EO9q0pQ z0JTOJTUYPG;+kks_RUz^K(f*A^>?_d;7?868XHCoCywf?9<`mS$xYt%C68OyrQ0SL zN)x$Xg4*g6>}PQ1>cGPGCEyw}%(RuE=M za=0c4*Of)!Wxz z1Xyf=a+9206?YfcGWBHKubkqB$FuZH$DIS4Ib_&E%1Rq%zDl)DT%g-JacH+_oW-eE zF6+;vz)xi^;Kv(_@OaFE8MJ_mxJ!iDT}h&+u(X801Zz}NgeK0^Bac$);PUk$dEW_b97=$g>7 zs@UHzA(^@3hPR?0dK#Jy{1Da?6;SQnq2E9yR7qa=>Hy%|elbcyB_B^>tL|L#&l${% z_*;;nxyL+af{B=?oUY;NHP!B4OhDX}X(aY36>hoc%DBeztW$AA!Oe4T3mvg4S-(F@pqy{4lI7%y#~0x zT7`i{2V{T{T)r9g@?MRZ!DVlH>?w+iOGR%7a%lvB26~uH1Z2K37Q!GkDL^vS|HDQ% zZSG2;70b7_?P1(7s{Sr0oic=YcQ7)-&R9&f=WV6u6+B!J`$p-Zpdv^ui{yB31c)(N z`M8R-I~wD_usz+SC(^HB0J(!+X7|B|Y)kjRFAnM50n?a_(!D$b=J-!mPYLyb+X-9% zax(3Gzczw*33D{*YGQp;b`8IBu{&1hVw-~K`Ly}T>l|2>(073;t zt^p1P$<>HD=kNzp6qF$GWc#`6sUYzjp78FwM{fqi=H<8fWhbEkO=R>&2p@zt8kUoF zv<;F#%#d4sBELYQPEhbp*K<{XRmKbT4ed5}NqiVq5>h-n*`X5PY*VJUB)`uGZweX2 zL-OO1)7jFNFm;y9!-Z7swb>nf+xu&kC?;`US#k(rA7DEkhraqlg z$ApJ`=6D129y!w?3MF!abjY>9lY_Nem{7YZaFc+gY&xN42M2+r3Y~;3#IyjeWbQtS z3F-L2b=UaO_w4(-8Ah67)3K*Ic?`ZMbE&b zn#85awf7-NJ&|zU+h3Jtt%6M9Bj!Mh?t2YKNyq;-VoX&)=o02O3C*cNS*jR!?g72% z%Tu+^MH(1y6b*tSEXEGn!&_6vqXd)%m5>b_FFfO+YY6cfPRH9@jzCkig7Z6dS@7k9 z3y?Sy3xum}%S}3a)YvHSE_Is^@+j1BouR8@ci;}Q`3RvM;dz18K7f|^Ly72e4lWX9Per;C z{5dX$pc!;t+*ieXKAU-znD=(|lfPHE-uwBX8II*j7-r-AV;8yKH;NM=?msKSq#Do-A}t-bY9&r~S+w?ByigYV6|j#OEUx8&};EVoW# zkW+Cn{u(W-@?pKp4D#rpT=fPm17N(=UY#pg9t;YzPeb%V4LX{`sJ^k;Uiv za%vE5dem0_+UX<%$t)~_iQUV8`O0|&Qd`p+bw2`+Rh#hsvzUQg&yM=h_+18P(9TN~ zR4Y@VV1i2~x@F$={3?!J(I@-N5RGqhUwxl09&oyVR4hft{_=1wF|qf2*Vc)Mm*4w8 z#KMhL(UzsgxYnp4G~!ga%2xIy zj^}+A!t#Rs5=2*TkjAK;3qB}`)0o1&xz9XoBeK_l!VD!myNRyu;{9rx_!Ob%3y$~-j{A(d%nhZkT=Q;B{ zfj=+Usq^;CY+DQ`?~cL@_TgJM`DQl@)sFg0gX`9Ga=QuxCwiBHJA7ci)n4+5504>2 zr?zE@8#-*q^R0_S$TEcDo$2|N$MtBsm8#5BT);2Sb19`>A6RNZ)9v9sD=7hoaz9hO zb;jA35@V~vy4lxIV${n|TNTubT!Q7`K@x6)?FWg$7jYRr>(?2Y<1ES9GO0@btmew* zjaNEO&Fu3UiXs8?=08qWGcC)Oy;Xwxl-nmqvCWO0R)~e=ATz6Da<|R zV?&qOwBg+Dvw;k3I7iF&23DWQghZ4V40PP(={j5S8R%9S8L@=u7NA^UqCkTLTN3oA;I{Tl9fR4XO}Z-bV5ro z?e*X$g_6?zYnlmD*G6tj;=K+;+K1RH!$3;vvSUD?H%#bgL|uO}J&oS3=*!6Z4j59E z4*DaNC*%8s`M)-c`=qn}yzvC*x~*P~XIdX^-A@c=tj|Gr#a;nq|GzskItn=T8ce}P z_vPN*?`hOVN49)lx^_lxN_2un6%(Zv=0I(h|D2F(R|){0_r9|3fYT?G?_L{rDRzq^ z3lV3R5@bJ_tvR9Q{X zB~7gkwF_0nCNSHzSNmvRikYt2Zl5un5STK(9!%VI`FevSE85}-{&tB5sPI|yyl~j0 z=QhC%^9$ywtW7d0uw~dT$f%Vdh1+q~=)Ebp!dTDx z*8_jlzz;>Pk1euneBe-xy?rZ`NLYF|Aj}mY z`*%nN&dZh;e}c%WBssn!gnFlhzg=9rNBi7cI&adX8 zvbSF&qpiX{TFGRywlqb*YgE$LA|D$s%1w}mxiH{g)={vTcsI$nL zRiVFb&huLB6JD?aOXiER1vN?q;`z*KyBqz|CWNWjR6X{jrP0ch)>FHhlVb?I$vE3b z^Uu?92<|83iNCF4jXSI>{r1#PDR{7K-WFmT$+Kf}{Jhr(v<`ueTh`Z2>vIWM)KNcFmC0HW8O@InTfX6Iz5 za*v3A&C6FDu$5{Ty}LynlUP)h_;D}OBam%Qf#naf#ZK}3AD`5-3m3jnNzRn{HgVq) zP#_1d-izZ?8^#5y$WH9U1dMq)+8%%@0qeKEI2?az55SKN=9+aDUK44{UufE!6LYZO{gC?xpSMIJoEKFm z_9#2~^W$1Z0v%-NYd8aT!xLoNki`4mi3p)E!xEF)TzE|@e21IyJChc9lm%qx7BUV| zYFdL9pX^a4@3Ta2eaJ|5HJrgCwR7k}GzRMGqW1R3**j1c(XLjVES>mX2I{wjU#>Q) zJnDP1uh$a>tQ4sS zu{xt7KO)3c`QUg1GB*i<%0 zzGPkNSf9K`k3t_ns`8EQX8*NNP3#iwcL6HfV5UPyZKZQ9HrY*T0L89O8eJm>CfiD? zg_jM6t=`K8>b;yLu$Ml`q7G>vo=DMuSgrT0MTS~*JhZ5T*XKGOC3WvH;7JpI#$q?(in`;MM&DlI4YjqVXlIonzN1m ziar#PXy#QR_L0R3SACW*e)onwr0w)wMn61MOz@)dL5yzu_URthx8^{<*)XkT4>`4# zSgP>EZ-#W#Fl8uEzntx%(OEk@-cxE-AXS_5_`p~r^u*UBc3L?z(q(_xc-C+xXf|wQ zjKrJ$RQ@J%EP6A30nH6}U8gbcCyZ+;Y5laTn@uUmBW&GV_z1}V8?5!7st{sT2S*3f zn*5m*|90T*7QGVkj={g$4q051FN;MITo)%{-`CslMKz;WoKW@K9<*P|9}nJsSD1>q ze4>Mww^x4ZH+9{wewz7+B(i&apTAdh5gFFt+4*;lojck(AoDW_?jG{@KQn|l429z? z#*^UPNHha}vsjFKRsWc^8oirXbdT#uV5t4>+t{=;(I8PO2q1)0Za}voI(iAFqvJm# z1XcG00;}&5^Q*Q=+^Oa`L51z0=VmW2YSp>?7b}i80mL8GGC&1eDvcK$i9gu z+@@g|4`EGA&8&K!>`KupEL(qeVwh~9osRy-Ze89Pr*O{I95~>kQC6}!)7(fYNV3R+ z=lf=3l+_v1b8qqah{s36R0Z>#%Z#eAY$(sH1~N4|zsTF`e3n@EWlhsY<(u0(m!Txu z6{HfN+_qF3&vUhH(($*6WKhZ4(OU=mSM2&np^()tUmw?ndSb66bZY z6A)QShj!vd7TF1~D$xAd>w(^s*$)>pCIYg_DLr?^1VEDH^yxQ=yRc&e#~xb8w%hdl z*)Y!HI;Sv;LaXE;69VUcyG*qs-#kxwhms6CkYI=}jb$%@n=!dE_FKHg``UQLX04S)`dgMz+?nVg z3RsJ#i*=OT6Y8`SR9$z+hrSbbt{CXdsB`UQdpB`ce7jb%(Ov?t{0Z%^wn4rOfiI=d z=?T7NRo7c{u5oBPoeyb-hiI64LK!rLINjAG zKvTH{zkxt9L`7BGZ*@~Y4=()t1&i8oc3Dp@a2 zWfT+wn-N$T0pE9I+~rjf%T{GFh|+kaf6p7ZS+KA49l}*$+{thBQ8ik`frk6L{clTk<1)kaRso+{QRU$a{hVtv&6>)_^kR|@AVt{ zQa4YI+(JzR9(RTSA-j#5 zwyN-Rh#-J_mw~^=bt={UH|dH)2dsufZggSg378mRewz*d{?3TxzG(TGiupEUK#SRYv_gr=zY}kz>)RE`%_rK4zbot+c;WWQ zwyr9n*F5-BF&Umg4`!{dofLFL0v<{4G~}s5#g`kbJ`5~nGa9aNi)4p@$}0*BNmHc} zvvzao8$lV7UCkaa*@Ik&MGvdJ(}~Av&>%09)Ov; z!DS#Tu)PeGFMVCW&qYd~m~6}icIw;Y9-Un%)(Z*fN4+>Te`a&&x1V*oQj5CDyhSmg zA-Wc=q3boerT(>oaT!f`jkrpfgC$-(?Y2V>ibm@kM0sFu4<{aeV6dLCPy37e+qN4jeFvlRE(L=#NjVl!5;#a0$H zyHw9-$y%(SWyO>l2^a3vAz#u?u zSf=5qOAY{Ut>p*qxk|b%HuFeUzx_w^e8Se9cw{`?6z#l|YVv@P;l^^muPqY6Pu%f0 z(DzR(SK-3Kustl|kr-jl6j=!Jr7$?|#(tI>m0Lt;C3h*FfD1oD?yNVaAB~mVxy1$w zZnHucu#U>;5Amw0(?4;+S51$lM0P^9{mdK*ZXQX&oyf8@!-W@P=IAG9 z#`&FfHpy*c_L*VKMi9}0IWg*d8VUpsg++91X(VUs;7=6oQPK2FN~Bs2j-&fhm50f zT)zCV85I^E&+QN*Z3PQGjS8Sq>qm-dtsW4U}}n3wi_WUJrw$s*HSpfWg8Q2 z;UCVO*o*Jn@@EaaNJKmN$IdF#4|_ABfYBY;D&?50g~^hE56%*iIMQKsE2N!42TdUSw3bf~w0Sc@Bi=AFM<7oqu|kuhe593Ufbxtbs0-1*f9ZC!n)q2s`fA!X*NEeY|F|`s|xUv6njKXqB z_W=-zxQ_3I`|>iR>T1KAf%14;|Cj3tENZzoP75rTp5sm94~u}ih%YXnt&clQ$)WL= ziF}n0mmQz@X!83kMYCX^s4@i;XHV!5hl8tJXI`-hJX)zZ>YFLzsVHQVy1KV5$#Ei~ z;m1cFo8@eRt;Sx;_!mPO{#kYoVWJ7|7!mCfo4@|G``>FQUAaLH-HpO{tBY680f5pN zaYuXCB;6O+1@fSf`55NE4b*>3-C&$ppEATG%A+H@uwtm@jcLu8#9!SiY-m27kjLDD zv2x}Bm{$3E-)~%bV8}{;xEAQ&sjSR;_Az3jC@471&D>TRB5d6A&BiMngM=VKUW$H# zIPx^kj)DF3RfMCq{${;G#knb~oIVMOIXCf5hLdvQjAo3?qz4n%B{yGCSmH6U%#2=4h62gWv(Enu20c@6$ehHzSp5-2!e5nElrqA84l~N%UjE^%UNl%JBF_)>A_yA^|O$G>imO+5tz}&V#bk?j^9LjHnYx|XIFn83oKCey%;Zl3VzRf_8 zFZgoa_UhpgozU-aj&ix*%dYPZAT0^(v^+%mQqWd4Xw3Pu-W6Bf{9di^E^U<(5tJg2G|#9^nFs>h1Ik11 z#eC2q%Zyd)c7m=6@gD6?`lG^U#m>V`$Ww(Q3uhP*Uh*z&c0j$8KigQpFzNg>GFG%W z3!kOw-^UH9w2|v7IeXVqppXaqgQKqOn=9}hlOK5P5H>Q^^#bSH&22sL;5Hp z70(X&E(rGr-~@l5kz=YR`wmp`S&3N#r-?&%;kgh`dmv&*PoHIZrtMm+S-UyE*J7Ze zyOTB`YJXA?Y#6K3@BuO-l&~csk5Zhba8#IYf)o(^vm@VylSi6k_h!s65IOQ#)O~e z`3i!BuJ4qM8g1PGS0-PIag{v8tgT)~D_kKK(t6hnx<{PlPpSG+vKDpH7ZqMibEPo9 zQ7HK69htwG%wBaqJ~WD1Q`$ckDpFQ9ZEL3ip>`-nSe|&D*{XL5H+T#aRpDN2bjKS| zhEs5=056zKW@M>G;ikI9kT;yiv+hn}I4E@5+uWS*PT;`wG>n$a{;`z5%Y7r{5(JG; zi*T1MnK`Dh1Mf?%bv#TCS2&$R?5er+uV#*9$L5Vwz_S5vq$m<_BhDII2dQsBGqX7C z|B`j9=vm_Z_tIE|MxVqIuBa2usQ2?P-r<}7|wKPW;wO)JaZkZ|8XC(4|* z6s#W{zAUMa6RD&5KuwTsOd(5kszL?7VNvbhNYz6DB8QOS-N-q&i}>Im1FO!l9T+z zruyJ?n~MBCVQ->D>Dnk>pgFc?m$nRwxSs50A${z;Hcb~1EPWr}pf6{yj6sUDq*r#z z?BxTNe8$g+D{Mfe_F&X#v0eOG4}FW_3}%8zAk4&9{qmFa%}iXJ8I2r zVkPs@vK5Fgt$)lQXyCp~bi!FQ)iHPdsEw#_CoaW)1YeTQc^?&%^PY>8@^BB#!>el} z#I*QvhT0u|rS!HQaA`s)fZ<+HU;OGJhS#($Lo^Y_eV*GC5JJZtm3SLK$$@2;k4hgc zCp3lSxQH6}JR~BXAc&vPt`6w>X#Xp35dgK~outKvIo_-BoZAUYU3(|EYqzo%;|FI? z4!UcoYR?@KUR^XC9k5D5M-EBHv|X^Ys@%5bbT*i4Z+D3VFF#!V!YuBVOmjz0vPs=2 zh$0*j=-T#uIBb-YB{q1hLG2>F_n+#|(sJ7dPDOe&{{Vf3h3;h;^Dt@kTf z+Rv8123m(Ub9e()iQ0w;YMDN?1@+}&1iQBQ8G?>ZAl)rp#0%GT;blw~OqO|#JCq-J zOt8QdVzadxXbOF#IsCDqQ{Y=f=DccJ8$k!#iEl%0s<3;79vg;$weO=WQV{bqzv5}f z{T5Fa14|(6 zug3`q(|B>JBQtv=cbuSbSF8OM(swI&4DGM&lwJi7`ohd~ zTW3^=j}t5VNty2MyNhPRAgVVTS`{k+ihZ-%pm)ErFB{u#36wX+`FLfuH z{RSCX3; z`=$jKil%QtrzZFDY2>2!_l6&DuMTGFH`dtALblqm=NIDQ%e3`kh_{A;)1k=AZ#VIw zgQ9tVvOo!R!Td2*I=dR(**)7CAdYnmwpOeK0=5!py5DfMwd%dW2;`i?6pi?}j%!Md z#9nxI_GLMvx~{_{gd94Qz-O1*$j7yu$l7A~>=#fi4|!=X`#wAIc_2Gand_mo4?9&-tA!DJ-fc-g9sEy2eGi@JEd1$pC9`9|0CF3?=FL3n)h6^n z^J8pK`GXGk5t@>`w*>Lo5I4IyfvgWi+}0V=%^rKnVr$tr&t?|gyg8-zN@B8#d7#sD z^92sSI=upr4$;Rk{3lU4@cbY3sar{%c&&jP?x8(c*fHWd3yscfIPx z&xn_Kz&vHVktz~n?F4lfk_)!4*e2@PfrLAHgUnwEDKbvIoA(e#!U}{ zv@claD^@(RL7b~uwaE`l)C*l+$zOPZuDiEQjRH-vXRk+@mnjIpE8<<4>4{yr24~2- zDLT%AMw~nr3yOkulhZzs97&P@r|{UAgDTY?1JFS&mgDE^o0+JIl9qVS;354%V#ipJ z={2B{Vnl=BT#tDa7#Uv?mzEU!`!y{<%U`m~mmEm`N7eGb9x38!W9U0gY4)IB%fqA+ z%d7?HD)3cs!mV@ivqOZ=I(-l#KuN7$%2GWAZno-uOs5VJ>|jUmc~WRJswr`0v$0kG zwcq)xKQY0E$soU?eLkz_#wxv?6qwXRH}SWyBo(tor?85|P>T zKni+6&4OJ;eh@@USfDg(|eYK0>}GVca}J4=9=5(%Olvmzn7?y5;7rC!Yfl*ebF%zxb(PR=Ce2E`Lkdw_1q&va8v`fD9Q$ysJywXNOy zD2P2p;pUS)4uSl>7c6bYAj(rNVHy-y5pCo&^+CUHd-lB=AXl9l8w38SR--^L_&9xs z(T>4|oG_*X#lj8G zjr{+opBlP`Rw<>~IU}7=lV$bCMWK0Ou;9}DQ>*R&^W*+dFutUrF^EOPR2V`v{ui(Q z&+JvOL#;4&P!_sBb&&t7AJ5H1ms+c8j2x+*ASxIkDUi%@z5_WJR%24{By7H5m5#e$ z6gsa^iM=SxfIIPhmP^}nzmv7edyr`r;ts1&w9>3OR!<--A>F<3OyIsi?6(YWynUaB zOJ0sRh1FW!NGRVSu009*g}fMA(?yt%_^Xls<<~Q&0d@j5;PdCF0@(*Kps%3`q`wKA z4SM`Q!UO@(zOr^A?bo2qc;J{dqc&_W;ae!@6qKKuT<`!SKYtZoP?uqGKq%?1eXh?Y zYWrtFfEr!xOyCnW{`?1lm62lOqdQ+>alBW}=xQC3&n^q#oP3H8d_4wcgkR`18;*#+ z-P+D!i6s_Fp|$aTp-jZ0m%`~OTVEq~WMYrJ4&W#5ci7du=6|#!$4H|V!{oC~Z!5>J zC3#r7fXq!NWmQ*XA%lHqxsh^bm2%_nSyeE*bYr`4ny_->p|82{Y}g-@3QAj9Y7TC- zf+9K)nm@1IteVtu7Wr`7u4jkajmv661%&Z2P1U=^0U`7Yxo9c;?6TN1AOMbJP|h}n zr~vQBIM_%k88TYvDq&P4Grmgu>8Zs=>-4zn}Qw9l2-L*VUW)@#r}#eS|1 z7pRw(m+b$$w)VSQ^^)2brj-&W%0j6bLE>W{*0RPQH&6Du07Gpxd1554{O&qc=T>P` zl1Jw#yED>{WB~UX-NuaM>E^f~UC3eKTDMrI6}MvQEmgOShD&&$Qp5f);b-|EXgQAr zh4Zeo;rsi10|$I53pc$rnD*&IESXEg(({9ffhi{_jC81&g|y0s`pjxn%iW1VIo3$k zv%@fXX>nccLtnfBXY)?Pd%m1u+o4Ph`>C&AVI!w{K2xXmEvXYdmy^}86Bba`=Azt` z>{|I|PEC_rQ8!mO7T#C(2S$AP($R7EZ#Nl9_pV?IAaAZ} zJdJhw-##PsB|awD&|u;*8QpWe(r(xvRry84s>M2J$`V^Gl=>J!IVp0uHpu8y=yCR; z%&lbP=;vn#C4@OluCda*`;?{EHL4dUY(bZWWcSPdpT%=%U|r>8;1pUhCsEA}_sk;*xWmW51<5s;>_HG!sS0 z!lJu|kpK9)K7K##nx0)bqqy5{;!6*rHWq6)79Cz+v^sNlVQ1q%CdTlG7GQCr()ZjA z@C0=ScHqzel*&BI+NIdCi^Iv@wE~Iao0Y)e_7JrDZc_^C0A~uH*=SL+WyLRXImOrK z^x*!v7z2cJ89*cQ#lfXxnLmTVn}Xo+?2T3{CIpJsp=nh)8?cPqnt0ZHby0i{B!@6S zqil_=mi_X=oyY17)Jl7SK_g2(Rj>?%eD%Vzq?dr}Ef&}i#LY*GHv4S==?zTEQ@#vZ zB)EF32b z`CU}ZXw5fo0O&6RdzyM)@qjxLs36?iz!Xqaf@D^g8pNw_JXhMB_xKMl4E=PU{ z9JgYqoxA}Ah|F{XcmU9F*Cs(p587jVO6l}fKR1+&hrK=Oa;(}$e>BB?Y6iB9;-H40 zxRKR|+!w8E)s%WiIA%DqUo)y)xn`RdymgyAYu5uya%MhktLxAYvT0YE(*Po*Q7zTI zRKYu36V*nY;Y)3{@;^H@za^h>3|aL%V6oMcOWs5ERodT<`wc;^(>x z@(^9N&9nFSzzkMBSaR%z+^RJdBY)-E|Y@ZR;?`Q zc&^ct)hfLkX3bkYwoz!+J4UoD(Vu-i_Tq$+NIWrr6q++VFJxWkj8?{7Wwsd6)Adej zmSL?!SeT9E7M*VnkiMSj)8bZNWAdx>x|s6hx!Nj*{F-0#R%PEB)pc0%zjo(mzS%gg ztw5Zkwol;%5N0P3rC#W+%ArT&t4oA`kqJMWnO;`Cr05$#NJ6Jr7FH3|Y*!>AV& z1-+Ol5A)D&@=11%3-9giu^tLutmJ+Tw~tC`Gvl&`YXI8gbyTZIDGoG->V{m!>Yr7K zVhvzy`p_3E!l)$oLZtY%c%q9ztv-^8Z2(>0^$^8wL^*j+bEdonjXur{XeQ2sph3H% z;?!f13_IuA_66%T97pEmZh+}SY^Sj^d%%e91?N#sYyP39aGwg4FMU6H@^tA9$Y$KQ4V1XM|| zXICB|0~IX^A1s^!oD@>pzdVD%$?zIlHh*#Yg@JWi=ssi>xlc)=W6>xlP<#Hph4-;S zVqlg?$t9PMhvk&i;Z}K(j^_+!!ZQy$ZjH5?2WF+qt^@rLp+nwk3+mD2veJsTI){%T zVZ-WCeSVx%lh5^~JLJNT6m4NY9D$7p>&5!Sa|OApXhxsNfA>C$JTj0mD@mgH9=< zm}KbuH71VxW2qo#ol#kEc?L-1^NgaiJUu(L?z}pANfs>z0GRrWNzL!uZZAEJjmHcY zP*qW&jXuc?oM-kV#~BCFz&{w&O5TLuyN^b{Q>rFfVxU69Nwk4#hbW7tS%%(Ykv`TM za)XW;rnv%&L`zg3UNfc#_TCs%gH);^PF+(sBb^D;&qUAk@dxw+lG%1}z z%p|3Ust}B*$Mgh#?E=^%D4|~)bs*>o^Q?$HLnA1{hLW6}4=x-Y*RjyS0XKx3{B&Qqb|3!&lWOs^h=6iGBv! z{i%q^stCVhZ4cVA@N-b|}B_!y(_xz)tS$ zX*br~<<|=or7KL$y0<3ph>iHdv$$@JQ4{RiruI@SB)6I7b43NT0U5lxqSP0`eZnE0 zu7UC*O~rR{U~O}`X3n03Th^^+Z#DW~ek;e?ei6zi)5G!XuUp=yXumEA!=x+e@Dd-f z;1K694R|=4G2-ghKr-j~Z|7K~>Cjnf+f3OC$&|vRQU_UJSKD7Q8TvWpy;Tf%$<)2$ z%SAO1lf*C+^5e`hiPRDMy;Xr;hW@Xx@!BI^Shf22#Ge0nt}=n!@q8f_AwtpYoS~$~ zXECxc6H<=EMtFijCALI**8w5q5u=C8WZ|?QY8Wh(=_vT$(K8BZZe|*` znpr;;%~C-!K#*P=n`sr}33j-jN58CSSpZ?0E3u|!i#FD1jC?Wjot_=9XG@!^K5AO% zK01B5BHXaGtJ4{-vp_`t>aj0X#)G!H!ksohiY0Wv0*&y40d)#xHr1qsHHrdD=;(OH z;s$9AwS>a)G|1nu*1s}5DA@X;D4Ov}So0l>oT=drmefsJ)u(QL`*wZM9*z62LCmug zV*w~xvsOk}Y?k~~(;%d(Arb^|>~F7N;ouzX+_?!JnKH?yRQ_fDoWY;z({;@pq9r%J zhAI;;X#6r-U6FolHJ77|e}+OODN7ma7!4M$B4pmS$a)0|!Q&lll0nu{9#wjhT5AtK zbAHJpzZED3Zf9e(21iMfb%E#CCfp$hqs#8y{GAZpE%lz@$mm0r*2Gw(>AmQ(ebNq+ zEvzjXsO@LZ&aatX%=(AX&r=+J0o$K{tu z5dFI8FYz(=&^GAi75y!O8rQoHY_LOMkqXswKBDm=lXTb5x)(GyNB)v-IBgaYzxy76 ze{tvT$EMVk@=p#=jcbW%Z?^i^xTZgA=W&$z+tKwADDNeX;iBm-j0hxV4||>u#gyCV z=3p{sUb(5LBdWA-%w=Erd5Cwl65ov#VWnV>6XI5~%_>wX-XgJ|_fFn_TG>8Pr(u?L z@4_fY^BcSL;@kqFmt$KtM$wnlEi~}**or7Ta`$-EB_~VwTg+Ks&Ovmq{KS~%Y>w^^ zTLqg!E#%RA*uz=9%ceNmu38r_mbfdA%!d8l(Ais732U`;6OevN1EB3G7Eoc6H*ro@>AZ%6Z@>p(cx#%WLB9|J2C8IMCshQOC(Seb^+?KCsJCda=k-Dbunf0Wn z3s@uDWNlxT_GR9@Pu7kHpDEvB?IC5pSfMOY4j}4za9We^6iwq1mvk-xeQjWUA=30`nC7HT7C7Zvx8WaAEM$cL(lKR*VbSyF4A1K>oN8}ro<1EgeDL&gUiF=qe@(eb zpZ%HS78CK~ULl>ff~V=%ZRqzoI~oMNPBHb^K{JGUUj)6^lW>|v1ydd_qi6d($?9+Q zd2=F^#t(YvTsdWPR<*?3)10c7y$H9Md{Ne9D%O9xLZ^FH>v_C{>lbtQV_bT%aKNO? zPW{ej9-N;0D;i2J*dx7%S@C8ktEXo!0#}XaMlA{tA3*zLdCo2`$kpY9WQr~%^e*%| zFf4)sAvt>MT~Tp1EOD1kMyn5?x+${Z)8+4H^>hNTZqexrsZgA3AF;b0Te?|v-F~>U z@|x$oOy2nW(VlJz&tA{*W~Ho!=D*BHqBK^e2ce4jb864(k))6mO3# zQqDe$qe87^=F|@?H`!#F8X5%^Oi~>t)sf3NgG&J}IbAb04`}tnzQhq(7F&(T@-3%D za3vUBW-QrDT^X`_Zs=$>EUJ&rSbMlbZ@UY5`F%SN=v^9)$nI`?)bP8Z^SckBK!TRf zO?QBDl$lYEa!N$2)>STP`7%D33c@}PoPRy4B#{siAN=%M-L5OwWbH>>{QltmWcQk6I_2z5!(ly3P0YUFJ^CN$6I&R z&xn3=dGKaa`7V5_AVXtNb?IJb&9*8-Cd>0MZf)k}1WUYwPJe&y45D~{F)ZC??+g21 z8of!9xQy=G_-eYV3*ICFnZes-x&r|*h|5yOmz{ZU~R0jylAZRzHRX!bz>I>P!?hepR<2ptX zMq#M10w~fDHKu^6_Fq~Xpu#^(K>r18Ben9vTr$l%Sqo;9de{DY|0ZnhXgba}MA~L* z8!X$O>MpenPCP}@8gv^Rmu}S#1sEW5TJldnkJ5D~1UkzlPczy&33~VV4cR>#l%iSN zTKl3Q2IQ_I;_7af?^l=LD-N6EFCY5Y&aii&+6zY+jSc2XRY1q_ zubvw=bqh&4V?K0vA(usMf?gJ%S=Vlrg2wav@_Q~d&l(4bZo<1E2YFiRXJ#ed`BJnv zOF;U{8^WtQsq*bi`~DrpRf3yhDO&O-$|T3Dgcg+2`jzJacPy5~gBX()`tfdMYuM+9 zOjw?KS#;-CI3)LW-0hWMhT*4@_sFV#pRd!6sfoSKHWz85EwyC~g&e~HSxmNp{jOv*u0=b9`HF_NUy702K39{nn7@ps%J}_) z-E(H+FNZV4aB}D+7{W*$qlYnx)_5hm)XRt?Q>D4*ggS6fNn>|wcuX4>9$4(;dQ`?6 zy&l>N-@XPHDrtLvgXr9Fce_|-K$Ny`wj;?DbAY_z{nG@AfFR8T>UrhL+SY6$FFpvB+++>o#qY zT!x$3F2j_>*13?w5?P*&~XoEC9B_75mEN`3Zp>pYz96ifyUsx$WUSQA1ux9SB`1#A);xE!-@ zhQXV5<|?9*N!K>DheK}X(1r9#4?;X_W)lUW+@tFnpV;n1uhg_jByzDj?DpIBW<$I_ zEKpL>lG%Ds`#Iw5iCe@mz-z7l)1|tf`mQ*0SIFPAryDmS#^#0M=mhmcBt#fW0oFo_T!R&Z)p6wUgmSFbSbkLO~~CrC4c^pS>go27yi~zv&ya z^~kT^J1dKdazbL#>Ig?&2}=*?tXCFskzlV98BI!4YSWlZUG(9gH*s@%@GRVwzFVcF zNdjNuf&oYRS;@e}4yIZr5R1ntr#Q$wh(^=|st4$l(g`Y1(5gI5Hw0;3a&Gx*&C!qJ zzC(n0duUc&Md39kZz;a-j-Ty@@bsJXM4W#`&vG70JR%_yb75Vo)hA94P2Y8}GMhvJ z6lg&GYrsxqS@Xpi?YfJm8q26gyiKuSy3fElci#+W3^R;suh1 zd_Xy9ajg5Gq1QRtyNjK%<~=T3&gUnSNpTmYNr%kY$2tpej>SG06{;AB624pW8w%f* zZq@mhJ*^Ai7{8P$v6*PHu>5t=_05-TiAJcFPua)TU20WA1`5@Sp2aZi;5x+h5VrEd z>BCgDs$Z)czpz%&KDC7Du*FS&(B-VM(j#h)H;d`z=H6W)jq<~#Z@<9ru&F~#ComV| zrm)^Rw4w{#hQH_|&0KT-0jz(NVN-+k>3iK>!@$}(hk#r5Sl^M=bmIypypTGW;b=! zMM3A8_bpe9^yD~rSP3D0+GLL>JuRbFsYOIfp3A4z3^#8qnqxFyk87<)et4o*2n~=P zFM%5iOv^}>1~b}Jd19M|SgyvNY5}rZ^LD(Y?=Xk;)vozFd+aRktulP(U#0dP+BaCE z6jRDsYm4^>Iw2mcL=&kB*G@M-GwB_>>(GARtMwo3b|T}V6p9FbV0v-RK78HgQB7DU zngFxLVHveJ>+a-72%jTXSgx!80pXl~Yrf9tu8Cz@#IsuQxEtd@Z$70plc<+|V#X_3 z1v*krde1cwXTat5IYRKXONgzI# zhHke`dw1r(C3Ct9n#J+TH67i)2+2~2C-bLDmei3U0ptWexM&f|-3DN2b(`3s8O?aI2sEnBx!gATx{==j-KBb>slw`W5&Cql$dlEpYT1iNnnzw1vv0Bq zW6W5W>{XI#5Vp!!siSmV<-Sy2shg8sSZ5V+H+^C7IERQe+L_Q)wmsrYs-|`U!xA!i zs=`r8gdoTJgUVD|f?FC6P3@Gi@UZn_hJ&?{V8>GV1|=l2m$33mx#cw=I(#~2cj$j} zLbDQOUVDWQs*FKQe_<558O5h!YjFiI|0DDZ=SRxKYmi9*77UxAd1o&9+eVZB==%L~ z&zBMfP&v9bNq1M2`F|GjPR(O3atR6R`ISOk|i22B(aBemVXx?o;Rz7)S znJ_7dG=m$+j;AAdS$myCVi)#^W9>dFXYcUNI*73LaAYa=I(%#PY_}p8q_;=bXfOzA@$%*yhgv9)n}w*^cM5UE_0@Aq1{lIxx`OtAI_0;D z?YA~7`oqh>yW~n(1(D27RBt+QBlxU!CdIKVsMjZ8$$T#1*h)x;nu6r#W=h6_Ac;$O z%~Rt#G}t@(d(*)eaRnw2d}rfC^;oKvo=J7&SS%<21S1kN#Es24W`rxoo1fCZQr%q% z6nl)q=v}GIdyPSq#G8)iL-O|9Hhwpl9z40$qdp=!;%y7)XB}GAFJfFvzB(->X}vxh z#hGHuWWny^GxOA!%(3W_$ePnW4IHIAnkxFrKgzl0akRCmvk^s6G>Olz2nRZQcK7`r znwCSLhpafX^?h@B22^`gA|~~?PXN|~$cR69uoD}y9mnze=q*+2vxCJnzw_XnXKk$+x@G(pe8qvS{KvTT!S!+Ak%CFWPDfqCPV2J(w3NC}|im z)uzC_ClTbJ50M^fHg%+uui8-@-3djaT*{_*!n;bQ5tXz&Q(rOMNg}h9oM~4HD$xhODY)+%0E*7V|=EbG%JNQfj z ztJ1fbI+&1+nP#8br|xK%ms(}?Q=m{N;qq{ac76L)_)A*+DY!-FgF#0O#9mK+Y=MTQ z6Kk_1-g?thzJ5wz6SLH*^(yEn5zE7gUPvvYen4<1m?fY>4EUaz1bmVqaQp}o&U~Hw z(QBx9W4?j!H=(j#hIbq)#U<>DdK`q37zcs^rL73{cZ(47)}?5TAZ~lm&%x)!OK22{ zVfOioo4&*Keoh2okKiim_^AjDcIrQ(0H;cwBGbs+ZVCT`7zDq*O1HGoZ6?={&-&R& zvEkmpi6ph5X7 zKpg)kkY*wBk9(Mk9iiPJoDQXGrn2Tb4^i<5_TTX(Y7VqHzW{Kc7%E`h<)gON2wMkZ zJU{m(@bhuLeKiSGQoPR5A&1#=zIy*cXizWt9RWV zAgZ{J=7%Ul+GL7)jcB+12w_B$NQURk7R2B|(vL*>5v#I=^j+ry+DLw(R5~cDlC0w| z&mz$9srIg=SO;D6qg=Mk!nVfnYRvn}jP!XIPu&H(4}lff*1B$WKqG4!q|!0O!D{@s zM~n3%7nKZg$Rp+a|WWc!gAW3`DPk(jo!Wcl{H2*NQtyJ*)h~JN*|IruHq^dHA6@fc<8)f zuSO5lwK8)s`t9+;n;}Qp$Q>pomK1ie_389d=v=K;>iM^MnO95)ZaU=)`Q)t!=R7W+ zERFS?G2w;l(!I1!5KyLLH(RE9D0E=zwpX6eMt$=?=i9V5m}%chxNXzY6qMB~nkq|T z+tgF`d=qiE>2E81$`GAnW3HL>nh&B&Vw&Em4CXbtV;arX1XJG=e-&U)%}k%8xPgz` zD-d1Lw4<1E@Nw9x*eA>@`W*_w$ron-G2#OABo9dzKoETWQe!>tp!FmqvZRbcEQzrg z6NSvpVHh3NvS7(UH}IPD45$)OynZ7(>iop-;H;Z2bt6@3ljGMrr}ANvPdJKFr7Hf+ zd_YDj&oHhK`7?YXwnF4tGy<%Lu^DlT_m0*;u*U*F+EdN8>f58YYjYGSPDUEt!>lyi z`UURkRg4~iCQD8sA1 zW7M(5INdh1a;Ej%B{T9m5jcVmvQ>S#_720HfqKg;hQ;Re2DVitwhy4ZgAXn_S0~=C zAYcE=ak)!OL<>6^pz>ikIN3wgJLh=G{;joRe$&^rDc#~GP66bl!AO!L&f4Y+D^`{- zg)zimz_?Mr{zg0>Ncx5}9kP%w)T6_AIi+ye&F6((`1f+~s>8lW7yRU1F~fiV{)6wZ zdje8G;F6Y{wA+1s1Xn_mPtY-r{pgL(%UT`6EQwCXxQ!=tKjJ)6Na+yMmofYGNM)Xj zY(x9Og|AEPxY>#G4U8{JV#fbiCG+{O>tGw1g?Z@chG;VdED^Deee1Ds$N_alaDJOs$`o-8@>*>Ztc}huyY; z@`5qZB?lRopd|-R!EngOQN}kUY@$@8wf1WXXRBfoQRk1 z^eT6`it^f>YNT)cZj9ud+n4U~oSu!(e}!gJvcEv_j3bs0AAVgi)v%M$rNi^e2f2)S zH8O`gXA*0|CCab8$Jgc%>1Q(DZ_a`9&^JZO>LIQ^upS~-pEw~Sl07@|SSY)V&6xy> zCvB$%xrAimSasp9EndqwPTxpz26)K7I&O^GO7@762{mNnBbty4ClI}B@u-5a=m>5$-|6rsufXfR%qAI=T!LKlXv$H8~L22#8SA8O355f=<*h*`M8pQNms3? zE%RoHfO5l=a5VBFbytf(?wXbk~+~V_v2Cu>G$6m(bTVSb*p9Es&Qkg-8kOLD>Eqd z`c9MDgPVfk+0-!qJ%pfxZNH`5z1Kun&HEf2rgY_WYquU4$_eI{lQmAd=gk`BFh{$5 z=!?bT@Q%&uRYTw0{la|bl88l@wH(13L#TOb$cR7L{RQ~_Zr0SqP21Z{n`wxmG`-<1 zR~_fn0~iCez-gq2t|9TIGFtN}=0Ml3K4twqd+aYLeh3XS_I!yFn~w7cnyIeieL6)c zqqc!$zELrp`m^B*db(I7{q^&RZ5oUTlJc~z0z4c zA@WIrVpDGiEUm;J;+Tclk{e1k?MtdE1$^_5SQrvpFgTE={Mu-HlA6b6?mAUQ#Ch}p zl3}Ts(XgVdfk8*eoCDyRj>KWz6oF*+AM3XzKzKp}%dApj)eqWGhUj1FWhOWSm6;2l zQ}wC@mw|y)|L}XCX_9QBex$c(LR%!YV9}`CO$D|NAAWWJ&L!JZG6ISa{rl+&9jCdj zl1OPKTp_lO>su_bn`x;?c-M3MmWWPo*fVN%p&a>dL8Uq|jdT39{a(qd-YI|_@6oXX zkOg)C5-~Sm+;n*i12&WEQb_Bp5n&<*0QAg_J~e-ZTf@t;m8!y=bsmjwjQm&&|h_a`qM3Eu?W%8OKuw1 zcRKyN#J+i9W7>WoWC8Lc6NCK(hh3}-!n?~g=926W!A|9N_B<& zMJ1^5DSeTfLVr#0?fISbT;Fzz=yZn;SPNcUs6XLnKkw#&S+Gs(_RIu}WT$?I-W7Lh~X zoWOJ*5a7Wcwa7@k`!+-;^7rU*OPdYWt8%|qZW^jQYI%0_x8WZ_ZnBWh(Pu}I4N0oa(=7yt1KX9W6_SPi#Ic) zXrI1pcpLCsqo7K<%DY71Sl((%zsfWAvP_t@;cUHyy$)KmjiI(faA?g&&hN4=z{eFk z+0VEgt}<9esj*(*DPUdX$v-bK?);UVI3#&uDDKjy_tMvWqyd6lStsC3V646{-;lSz zVj@`l?!~*v%xBSHG<8*dgReW?cj;1c^p4ZW3eOX*YkwlbdR95@;2tQKrjg_-Sl3z0ETF2Ty!@BKh9(s=OiMkQzR$-7Zn^LM;UI#0aoC0+y zXazAVqcW`p^i6h_8Ii+TO98>XTC`Mh`U18(8-JO^Qu@LN8mXzAJ4~%p?qsd@7d|2; znHY`x)A&drK7sW^_>K*ZHw<)l28>@B#OSF8RL;1}l&Na%()H^w*VvfyIrPI?tj?rt zZh5DnOgfc0#;F>!fz=nr=xs*T^L(kD5CdHhE%frwn`dVUE&6KjV6v|RL=SW62oah)AToStN zJ6~vg_}<=VMK2ald2eCFE+edXaRi5;E_Ez7ZVozS743;ZA&v^wbM>r+d(%;{#Wx4Go?Q^=kH z)h+m6_qLz>-xZA~Q-=qzB((sSlu{5G2w%(dn#`+3-_V2BTG?=N{ky69=ayJS9IfDj z;xz-d^Up&4ZdTndvC^l(9CG#u-lGfkS_|ilr)cyKNJsRQy>SxnOE12uwk+>(LZdPD zG8;0G)x1yt62S%xJtl0i?b1EkUl93{O=fdyjs1xQx3*yFnYDofG5V3~^hVmxiZz$De1+=OCA7B5yWq4R z=JUp2Q*fcP-7I*?GH9unNv?t0u)@;!8=BdDjrcu7%ef5w#Zf}0z+_tA^(*4ynHO2N zDG8yQ-pGU0-tB3vox~5;Y=VYQ##uCubeHr&jZD7zL=o3Y`;o8R_8nUW9sAx!jBMjy zK$VhzLDDeq%vD^@04QzGo;wj6t!0D_OS zQ$2xKIX_sngq_&81$bW;-{T5mdM`j9^KDi{p-drSuZJW5uYOkr?(60JY)OV$l7?bw zpOWegG`o8^=t>%viyuVVXlZ<5lS=dG&w*+@Bkqk%ki0qL$;FlW1{zAR60cZAr&Qxn z`{&i5msyOjE@`khdBL)lbsyn>u~WdKNr>>*yr1~J#MTbkO`2)+pDyH(3>v5d$o4^9 zY4V)5ryB!=q7r4(G|NrwXMJytSYoPq{B@oA-CY9LCK_F;ZrLphT1wah?08Zl24W@f zcWm00W1{B6*gu9iUZ1rD`{3GmInMhKL&N^Z1?gXJ_n+&5jk2@`+^aYS(|J*L_RI_i zBLa(==bhz9Rw>-6p9pz=Hn=#sjlfb{a$1KzUPrgyub!!>-DvhAMO5i?cpmkRFO7ta zTKq*X@aNF}`%?JtPnHCpG=3}=9ALhGsffjV1_|KbJ-pB3UVWgn>a1Pto8%)?J!F@i zjMaL&eS?BLX0|j3iUr?;*BDGWe13mc6~kNUP)xn6J)2J+ zvd5P~P8TAwo=RMA8JqG~)8U`X_P>AEf3I;b`yQhdiGw3*%1;!`u;2o3llV_3M8};9 z)L`|}1e(pN0%jV`RYQf1TC;Ut0Ww#1c`~WxqS^ggR5rND3>-c>N@nYCF#ZWb{)emi z&u^8Rh%&-mHJY=RDuH#l1RIlCt3gN4)g|~$8)IBM_A%NE@4$Cs>Hmx0U11Yz@$j?G zCcXy7--KWPKQ~D{EgNVB++??}tNHw2hU0(6&p+P>PdP=K?w|jiMvWdWbm@x9SXoMP z=x2~TZI^Kbfquxi=5w*CX?Kg&Uta?^Kvfk9e1Sw9AtW&)sJxe3P}-&^44> zrwu8%<(vQ9Ta{+Mqter&5lCr4zFmD6a9=R;hsa<*G4O&u^(Lz4M&D>G<0fN?f!#X> zPPt4#3A1mxeya;be4O~fe9(Y3<{>=2!Y&S7X}SV&ay{>^-!M+~I{XacpL*4_t!RKj zqv-syo+T-<^fO~?KxC1d9@d(v*gwD3BV084a1aWW%xwZ<2Qxu;l$8P-7*`NI!0D9< z$aq08cVOR`!u`=6Lfm_Y3;rlmn-N9hgMfb}_7*Jw{y3$HDy#Kq@U6Y;Zjy6PFDSd@IihI<*^eATt*QhNzq*DcXf%?<$ z5+*YWSODOjW@0A#o5-G#&V1NjL*3!g+Rr)rgC`!v(7S~?9BVs36Wj}~=mulG=ZQVf zYG#zVczb~c%cem3@g23#UX&O-QPgflXY3%(Y|h2LK&|k6uu2~_vl{Mkejxk@*-?cE z?ME<3MU$JKPXc0RdmyT(4kr$`1+i+VY@{_gz=zDu?AgO%t$CEiJ>xuS1qorPvQw{!u7qAm|8d2!SN{BD-@RNdVCMCl&`J39WQm>{u~4xzL+AzrTefW!KBL0Z)J zE_OMgvZI~{k61pUUYGU(&lSYsCI}u6oHy&UgB?$u>MB!>R?VU2;*9foO=5;430aA^1KV4EE3d$5^@$; zpYxa%6N^D8WuDRGGqi%p7GTn#Beg8?bzB>goCfzzRr5ZoJevjNBxOg_9>p;G@7)b$ zag}9pY30uG34CgWCC~Kx5?9Rkg*SA$`ZKxSMioA`tVKuhel zFA~Mt%`MRLg?vHq4vlIza3<)3JQY^Nu3=9!!@SVCvae+h43BcH@ z3!M=z#I~=>cyOiN0uShBo6fw{7w3>Nsp$BHC3Qh;%TP@JE=G*FXKV6EoZ)=Sqx$r zGg|c@!)}Ng4b0r89fb5kN0@S84H?r?9TdvdpBR}%G-9N_TNt!YHVJqnEkYGuf2>~` z80d^^wI$g3CYow^zWMw7)t7p&lBpsr%TQC}$BkUE-(5i!BeYW;gUF^Ufh-dB+6k3H zFm`A1@6ZxgC5U4y5C+_k`dKfnx|hJpS_9$0Z|seYsg zXxrL+{NiZxmh0`$G@S<{$tZU(1l2-%(*R;L``HDf?!o0zxsvQ4n!I`>KIICy?3A~P zwL!?5fXAU@+;5}s8pY9=23(>W#D+n}N(_-GuuPeO?@WeRNJSbjpETy;oTB;_g$y%= z+Faw7<$bWW)$0~bY#Jpmf%TdIwYa7@YmdBM2s{P`;Vm5(TykC&J@-|bYG)6yaxH>1 zK3NA73ee)7~Kz}=^s*K zmxcJHRz|KX}kRcp0Le7Ad|C}zg8`LRH%#?s%|`JaFTLeP( z(1SO@4B42#>uw08)X@J%&}*s_$dM}mHc~Xz<=Jq~Q5^#MC;iGFdReM<2052MZUe#K z9iQWGlx-kIkfQh6=wZ{yI9sZK!s%H2LYYW23%R*svuatEgpeK2sVnt%DkXy zl0XBkfJqs)6RL;40!Xx&Yc3O?u?9?vCc7ui3kr#76O1EWdHO<+eNKAyJDlGayj<%6 z1zUn|s7x;3I)6kEkU9vUF3;yx@OYXm+=?LQdr?K%hDX$jgdhepmA2;UWG&;td>GgCN9nYu-Tkrl z4>(L@Y5)fVZFGlG#3xpr2$UmS(x4}T2KOkP{11flI`=J-a&QMuSxY<-Rv`@oy+cdv zo;@QW;s;YglwMC}L>h{ZZzcdv;Eip@wUyf(KW5+4o)x;P(|qO}W0gtPE zDSO_C6UPeu&=&Ls@65_}p6#E5saORAGZ#-b$MEahlkVtkw!Gm{p~uI&j*sU{@qc)G z%cv;3zHeA1qy%YD0Yyq_Q948grAuJw5>RQ77+?qi5l~PXNlB%8oFUeB}E{q0$6J`5}-IF7ydzkUI%jIr{+%E_e;r@`a-EAq=Y)*QqTBfr;L&F*z&8K~lvKU@9y~o{inl zkpHa6GjO2?=uu>ZdOi=R26R4$c_u&m-~eapO6z-)ET^RL?-dk?*j zcYfrb4MzBk1QFnQ0|jTVdIFKJ7X{8sq7%_^>mw`yz2Wr_kt<@?_eB z%+z$7zUf%*Q-FVMzQWauNhO1z`LknyBdE=)035pDFuqzYWf|OoQQ?WF?8z;scFOr7 zCH^g{50{lvLaeR)%z|85Yg=HjS>#JG$23KV_FyRBy z%;n`$qWQ#?gx}7`M?V%#qD!D@J8IYZI7fG6+3gzQMO%AtDx zL$jZWzNrV;vq8YYQrtK>yKd@b|Y<)U+l!p75gMHZ*1AO$M3A zgZUL;LAFW<%edIIGgMCC3z3@89r%KTS$z%{M|T;yC~m6Z$Vp{-@GdgI{07>9{UldS35SFEnh zua?j-w3kN72LCP-oXT)TUVH;XI1i8IM|MeU7C%)EmHy>Jaa^8|)*HwY=*~=?UM~QR zCr&9>;Ydl3mh(2zI5a{JE1=_kQdl+}u4gW%*71YxzsYz%GDEvbZq6uYI}RG01&_68 zWKTFAd~sX=M?c71N2{5L+`#pRg{+#qxiT@ltn@tN=CjB6|9IR^yo4Ztki~2E({u*@ zdYjb3N%(w^O$zDWggXio$`y+^U!Jt5Y59Peii;kuo3ra4Y~3e(*brZgH8DySaBy&VyzcazH7prf#2cAlOlZiqb`C z{4yi34|VCW>-uu1n~kXFx}2@fj56wOkxxecUZ~4RzrU|d2ISoW!6j08Y8vgU zw;cVLuSqjFL&!okD^%yA^0uBy#odj+nj~9s_C!-A;3_s=EP)jaw|BpEeITz%>-i7!BvNIe`wbLlz2Fy~xoxtYhOWcHc*)=<|cMPVWA& zrG~dtdnXF$u>yKJ7F-&b7$mo~6S)?V_d*AE zPb)5O=cg8b=^n^Gu^$R5C+}+{4Uf%f|H3D@(DkFlx=F;%>Ogg~7Q4g?haGeagj^fB zd!ieBjZOHVc9X5vF-=X#Rx>+zRc7k3|NEZPE3@;_Hk9|U4TkJnIwNhf$FV^sF{nq| zdX}9P`TKg74!h%svDDIKE2md>8khF^!sW_CU=+K3)}bzSv9mh&OcLuA49~nK7k+8p z%yXfxkIW>XP(GUaa`c5=d-hOocf6+b@cU*}N>*E?t^V0;<6iRI{z##);j)py}w_?itNGVY$_z_0D&yMpVtiN0!Nqx`sM|CjeW?SC%yGy;6}*AH%o4n@KU zIeD8yi9ay-&eNRhrvU1ffuOq)t{h>vT|4lC%{$F&KIoIO1 zK;1TYO)!dd?!yA3C80COrLFb-te4}gh2!R@BdD0<2=N*qcEZ#>BpU+w%RGwQhNq?M zCe^o|t3R;5SF_OJbnZCT-`oyRNVHP7=6Y_$trHctmt@LK)bI6hKMxW)41Z_$`#QIl z*2~VsU$}s{dv6`zMC4rNU@*P>bmE(rOZEA2r%dHkt=;Y|KUbdwafe$WpR7HB9c^o} z`z+)BM1vR2}N5{k_ijCc|TZD&@0((uKY0o@H^EtSB`Hn`yy- zkWg`V^Zfud9X0j$^UT*nBz;E~24=KRWb;J_c0-bT(h4`mAGM1g6MMU(99Q~7ln7#S{?7q5lt zE~M5h0r6tJTw~!7n)CS)+-tOQbd_;aKgQDxQ<}xIL-f=rhgp;EK$qh{1d28|y<<2y z95FO}oiW{lBB<&LP{LiClnIFYJGanm< z?sNv;+y85`j9zS(g1!0_hXmXMyC&P3NARo6O8Je&wxmczb;++p0>s(p%(o|dDzA~> zz44%OVr#nwzncQeqETmtvut_y(WmvO0!@}4pAEf9?V(aHyQ#QF>MpIisf!Fw@bAsx zbM?3TzE@Z5sab33dGad@yn<8Q#HM@7vdYu~ex@XUzv3Oae+Gm5RL$#`gZcp}i!#Rj z5795&cZJ!x8lx&+MJ~c#%IKxp0uhpMfN?AN33uC;^Y5Bm<%&~QZ+8xtahSIn8#Zvc z(*+%>s6cd6hkBHq$xlq%u0X}NA9Qk2sh6fdw!U@%l6)cZY>WA@1!~zlpZ8i(ctK<& zdVwrcTv1^kb-k~1#yac()#88?uKXJzy5bq1*atYcOvF0VuD_=eQ&aB}Sp7^uM<-GJ z6hoId4df9H$+;PN@Z8VyWkb?TF_s?*GIpQ*9KgS9h!SCh+98;GA3hvEpj9jdBHRx12w6C_wR`0g zWNlvl!v0fgTJN=jOT2|-Wo_&d8e6&Z)^yhvuOpu3r0#6BE$mE*+dWKnUsukkAA4gU zJ+OuoKWy!k^HpeSUE)po?0I1h{(RsrUB4G`u}fu#5D8|O{w~>LBw#!!AY1Dw;N`VL z)LYrA4KHMnpw>uAdg7#-YLCV@WLkv25Ah`qVX~fQ(iaKz?Al%;%P-C}Vw}G5z9f1=^3k-BLZsE`l=Mo{s1!uT~ zLx};D0(P0`1=Kh^?n#&*(Z~OIM6>y6v)fl|xOob)bD-28(o#l$JHn%hE7IEtq7TWT zH7kx3-YwJ=>!h%_`Q^#I2|ih|?n!sy*W1H_nDHcd8Iko)FFt9w)%!?dCFQ_}dQn2W zy(!jq7@iiY=^iq6BD$bA-y744pO}7%lu%M%5F&0P?aT)2h!{?rFV~R9C9oTdvMPW- zqLEs|iVleH0k3(+I*t0hnf{UjO%&tp2WERgDpVyd1>@=X(@sfsJ3Qa}SBUvKRV0NZ z#%v98_>S=7MG3466;~og+|K3? z_40XrY1=8t%W*lb(o$ZM*q!LK-J0KIGNgY>dEC ze)dcB#?#E4)|y~2))`2s+7+1+c$EDrwW#DGQTJ&{kuAut2x)NcrLme0n}ShGvMqH3&lG09nkCSt8CWP$Ozv!@ z%DU_g+yb2Om#_A013@b>eu=F9VV!Tp+mcRZ zvFT0CyrKc|whfUjbxC&CzN*adl?K%*)&^G1Y~#$ueOy7)FHKIaXRGKvj;{ijNl5E> zWbZh{hCngaGjXnk_C|$moZK>f{a;N^DKiJX;xQb^dyvwhxC}C-rZRZt)5%|;gO^g* zOFC;EZ>ybyxxhmoZM_p$ZADrTli zml0etJVNHJRW`9pu%aAh|(Bvj*3#N$J{?6$5N`3d^-EbnHbf#pu zF=9X(r%?3*l0qQTRGsnMY*umiqK=teiR}EYq329N&EuH7d6jp6;q54NV1hd|vBM;5 zGcS9Thrh2{xUOl?wdgtqeT!q}cd8bKG0C!(1SU#--J@z;lXj_(Uucl4K*t0Quj3Cc zwae3E&Kn7{a+-9FA5Q(Q-ohl!EI&lILMLn!HZ<0b(63l5H5e$K*~+$^Ep5gA5%$z| zDrzYDfv%Qge)w8lCwiuTx#B6ubXLI&T%R({f@V#$UeO~sd z#@tit#?nKmjck);gG|*K&hssg;H;YQ64daIf(dLA)|7RP@rs>xN<`|_2_cH~m;X9L z^q)V*xo<%3bNs9Cg`jSB_HUs_d;-(@;RW%L%!`><=V5{#YAl~#KlM~skgxprlffB5 zd+!s>q4vbcN<$HncNp3 zX*ofScHQi8%Bocv$Vz97>o1_OhNRW*k%l%4ZLw|x`joNKL9F|XbYra7_>GZWgH&CeV43r?;&IS>IcO0^B z|H@*sk+lk`vT9Z_%Ol{+xBfV~chPeM^1*Y}%tE1dJcj^%Ik?W2Vfp1=zaIT5AdsLJ z$fMrv8*b8fD|4@#J2>9kvt1J)Q%6@JeRdy`FdObr0CJC>S3K!~_Ezx?Sp^5pk(ft1 zX4*ZCiu$otjg=k1W+fv$bmEEFi-}^o((a1^K-|l5FnZa1nL1IKZf7Z_JnH$CRu@ro z4s>0WW_|O`jCH(a43~XUjlr$8mzMTjgPk)!!gQ_)xY;JDKVGgmttghORL&P){3H~L z|Jh7`!T~kPIFPaFE*DeE|7VBxN+b(M-kpPaZOK2GQM{)hE3g}ANIon!f*DSrg7S2) zRPJMw%R^&tn(edU`i!k6zUo95-D`SN#}m@(bqJ)BKTj+hP|HGnz(AlA#?yadXV$X} zTK%Fa=n!Aaji<>D4W(~JVP|d?-_uWWqkKj$4tI2FUJP1*r3e|UW^#;M+t0)m5h)*! z79RZ{8NoggPvyAzV0z2++Jm&^u|4}S>Vys1>C!TJkqta^rcVTtI(iUt)HUzP+rlU1 zvaG2Cz*XQki_YZvibzw!bm!?r0r1G-I;TVk^xlws^>{6WGo?$}zF!PQM%H*&P3rLF zOA>3s&7dfsv>iZ%?e1$S^2Kc|O@i|xb+gJd*p=Uj>GR%bV6CP3a`P_eNZRFa+reOM zJu~apgIzrQBf&g*Av%Z-E5mQgHG0O0GssPGSLF0;hVAmekgXg47_TieT;3^d=@As* za3U*4w+s=LZzpsWxVdqSj^x}`K)6@HieP}foReqX^doC>V?*e|bWh~Rjmkku)dE`s z>CYgg>1;UzI-jx9v=J8#;U!6Gx>EcjpBYV->@J+NcLwghYYlm7?O`h_&Vf9(q??LI z(kU#0j9F5Y`$a@D_urYIBs?8vCGq{_PyEKZPWITUSB>qYw70~DS>35<>UnxT@W4c5 zirShhI-5_EqkU$Xpqdq1BKNE6%S9s=2Jn|%VI)>-^H&{D3K z{Te>EMWhEOIrCujG|!T<@fab66>8LPJf@vbzmO&$gKK`woD z3-)$TV9_Y7cdPh_Z*d4(a(KY-tQA*JZ+O$b@nq%fWfl$y<=(BKz#Qeff8*36`IBEv z7HFpMevWPuaiy{t5;M2bZ^$0ga&18di?XZ6bx9?y3gO_}WYJ(h}nLeB;{`cDKbkMc&A z#X*xL>}U43`@3vy5*XlJi9RaK?AMG_ade-F()>FGe0dM+=4@ygqaC?gQhlLT9cAHm z%ptxUtwjR68;djMOLS8kZF4(xvP`8vJKE|$vy3yM`{RDbnd(g?Jnh)_CFj#h3hPCm zh0-Ag1 ziu%i*WMWu<=zmn8pc*}py+Zb_%uS%jGCx=SiD0z{usse0a161P_?837Ftm;|m7+Pm z(iE6S^9V4Mwo@rRBjqO?heUj8Vc}uKUAl?!uxtSm{xmaI84r;_4zTU__osE$WE|eK zdZ3wUVK_~i1IA{OhJf%l*oOmrD%upU?HbYQvXyT(zaClWnbdvU1Y+I_0)}CW={Mx2 zL5K3D4#X_zch}p*n;NTvkPmaYUn>wGoNbH#2#&c!hifN(>f+0US9QXoBT)QUeX^MF zLBI$^>I)|!4jhYNZ1jNhuAId6HR(sqQ%}g%(pj+-GWAtKBBf-3mM5LblZulC?w!{Q(<2Z%`qv<1X67`Ix!O-`z*%$`~j#b`){91k#Oyy?GVrLLg~Vu z{Rhx!g;JtWW12RIA|Mst*H@qNoP`EyWp2gs{&hTB=gZOQu@Qck{+{_%>^ZVI^0D$I z9GLPsXS|j>8NztEW7vMVgI%MVkf{`3t}ZQg&vGNc_i8i_!JVjMME+M6q|FH%sOc4X zI|FKECY|VO^PX6YbwWLPdoeXddoV)izgjBqI4)0Tn?c)d3;Mok>0`0a@p_{EHU&sC zhtwh8eS-v=*?rs#eH7F{%SEyZ)C`2k!yn`^XMKyiP_ruZ)zKkdUo>TbJ4P=-5+68z z*yB_yowlf<$+fK9`Vs;FoUg|D*BjAXTLM^%5_|4)A`QMFijL-46g?1!E!(kiL?6r6z84Z}oYecoTX}YVfHX?C z*Y0&PScI_yPfaE}Z#V)`yp0uwHq_VCh`rDdqUso7A8BRJrWm`2OjR<4la=1L{?*Fs zV0#cNDBJRC+ha={O+wRDzAHKJGi!vXI&nPU2XB%nqYZ7c1@&|e?csZWTmZQ^myhm_ zN;`IZwww9Nw)2HxvV9LXOEweM^>pjLYj(f8+(fCvRsOIj3?A}M{-LeX^^qc|z0O))jtsSP#GFQE{f^AioZx})s5JP^16NgZA(&3rM%Z(y&tcI@(p zgH_EVW3K8#bbFJ(Qz63xBjCV%X3pK?e!7+u5fz56L8Y**mydBopL)j~hy0SNnH6K? zHegxz-iT4*zN~AJ!({hE*)2EQ1Z8B9vpoh4j+r+%^V-McEXz*WUMZCvSV=Eqy?y0B zANQ_QPqayix6WA!%?ER~UV^5KKbq1Pt&6kpz1Be$u&kvfRY9Oqsl-?Fs(LxrVBxlg zb7^;Y?5-{2-&z0>Q)uiI5C54sUL+(U!}itn_UZi+LRR}_y$DwT*i?8}edjQChRa4> z)11jlXB__*Nkqco1+TnxyF4rPnXr9l|B2+u1`zf|0R+xxonf$`%aTwj z&BS5Q+Hxa{$?i8PW^SgL)c!S2-#3=CHFLNe3#m&+Ddwn2Sfs_&zCC=HCs}f%l~h*T z*J}5fxb}uQE#Hx$-V`fzS$Z+H^OFF0)1;@W0EzHpyj1j8>a{@|!X!H?CGA)`6GW_M z2<9qBd8U-(Z{J0u%NuVer8ILk;n{MYzu5Xd*>GHZq1o-_>fT3nNaW&Ty7FoU~z^>Wy+)+IA25(MuU`r`{frxvmFA(4(mjFpbD8=Akil3Jt>* zVw@oSRpw0Tu!vC0N9X-reMz0l6umMV%ME8E>4G2h%Cl`@_)3p5c_k7gJG{t9E!W@a z-w-S<{xwkP*4R}ECNLHKFL8xj21k;7#Bvm)ot7v7z&pT_sEk(Urcf#B;7o}LxPX2XPN779^?aD&Bfrp>uJgyGZR|< zAfqwWbnOnu(O~fSD|r4N#61)EFaXn9*6q5pQA;-LBz8&4IQb09(SP=p{PM?_<1KWDrP@Tk#phrUSrw|BEclGR;dv@#R&ipzO1D}~xo90k4nQ{cWAC2)HR4Qj zC;oHN_*D=uILXrnRR;`g@ z_F=opged0l7TKxLj(^$`80l=fbCTS5Ne&$iDxha}_=A*5VZlU5a>}Hr6dPLGjq?cW zK})eW>;OP>=Et;kgEP%_NY)9<%;)XCx+4I5A;^EAv2e<=1yK>?`uw2kt9mS8{yTes z*{L}MQCQ_)pO3vUO4@2spqdbA?>RTIV574_M;`Wkoe|HAYhcKAq;;b5Ma{$9{r0EH zAFj@;zo)o?5*9~`@fb6{A-myc#+qos#n> z^K!`?NaK*y#vEIt{8sklY#IC5F`a03r{>7|jJb5?I|m}l+5S$o`8*MQd)D*c>=Y7R zSKlO(JrTa&ByOoD9h#{fXOmGoz52~5eRfy6^%NN>>vd{9DS~5E3@SgCPHjxBFo(b3 zo_wt{TehcOX=m9nKM+$u4^iI&y(&Qx4t=}I39G^N3yq`(!G-L4{9R;GyP|9I=he9gVc zH0+;C^?D+7Fix2oM^KF%ffkYoS!Ke#D9<;+^=f~-IB>W6CVfVom2ws%fJtHCjrB8(-tn z-&E|?7>s@hJ+geWqCIr61N1Rrjcr2%anhB-@-iAr?(P6+Cbupxt|rbW4<`BMNi>QV z%bXDZ^1Xqp&jdH_U@5^ftcKY-kkq_nL-=juDIp94TpD_*NG z0n-&O`0kC?B|;u}S+rLEo1P(JA~*60eRXc$T4VFOs+T@z)UvXRWdW zcm57Ozuk$Hr~C8EJF^>-#dN0s6_T#2pBH%k9iKiEA0Ki6%ZR|&9sZ`;nal zc+-$m%D&<2fIj=~d-1B@IYhrxftWbeTaa@^*dIFG40FAO@CC~D2IdMXzw}bIS^xgb zf4%c>U*d9Y=&WzaTK9i-BnTh)gAm78$OtN7ip~nB;A=POoaWLg#{#N3ADXG?Aa4Kt zpG#e-X6U>ov3)QghDti_jJ#9bDF3}58g&8U(odNxsr=6n3!dX5ngem@uLsZm>-Vg` zjj!a(!a#!b0k{M4>vhJ{=w(dPpz__Tffr3+C$kq%@vppR|2m7%l;aoh7ogljqhsVc z%I1<^*P|boE!yeQb%A5x18PicpWMDtcRBkb{}k@tS)Vp|cr9qd3;*RdtVlWf?!Oyt z3?K1dL8Y%N84d9K^TMbnHoqHIp;rX}_SF;^Fn^f`?Gl?~05QvW00V};$JsxXzIFy zYQe>A1rTiJm1K8gcbz(1!s8k7;na!MiT}9|p22?6sD?J9pZ{ImQD5QeqBpQDIUZAE zR*3j(72!oTgXS6Canwsr0nh;Z0Io~9Gd_1=wfVA_tA#by4+CrQ{hu*5fA<&t8{!#z>Qqc=3;sDn`Ddkkv3Vb-vfqa9OB+1wiT~gJ^nYIbSdH?_UVY9BZl=Gl=zsQY z|F6&erIoY7|L@j0=qi&U>?{au3pE1c;j7%wyh0jzqNMXD?# zy+14d0Uy_sH>N>f_q4IRXcAH^s`B0bxoH;7;0hQT>BS!AWr!YgizlI0#%IzU>rM z2Hmdsr)=Lf1^{gVQ0WFjR}wgrToHQ7eq7Ul&|wC~Pi6%b2$J$U3P{%9$VRn`8dU(m zu7iP}7O&*dxCM-~()sR{^Sh31*F*w=InxJlPSg%Pqi7&~8BG)Ifd6nNGC? z0I}N+5jm(e3;fB^6@$b03#?MQrp3jM26D4r@o#f~w$8N#WV9}Bn4EjIy^=ovv%hmN ztaf$VQh>seDgg0UJr8i~ZGqHo6#>nfD}ZA_0i1md<1NC)S^?gHp3dmq;DAHOEG6BIrn^+H*meM&?1X7bXNEjj+zDzhx)XKe&IRPNbL(1uyMeAvA|Wf zdd&yXw~93Gem-U{alyf`05lN@Y;m|hF-wX2wDpQEekv(~fc{h~xNEaWmZquw!*?uB zpcV=~2W6!oi-1E_d3wRn*hK;)nCE*soZ7k&#y`7l%Y8Z~o9(_*g6bGBkcXw^uaWXzA4XZghM-dm|3|J*^3^{!YaImuTaA`BMofX-Aw zG)nAJD(JlkydC-BL7Nv1n!Uox;EWaf?y8S}%72-VKSFY^%Mo;~U)5epc@?RU9_i&z zK-U%M2$%_hUwR{e;3{dJblIQ8Yzz$G8i;P`NxyP#Y%>DK$vlZV5~!w>jiVyqm?gSttuqY08O-x49c}f z3;kaF;1ttfl*5fdTaOL{)cSv7Nj^TlcSWf0p%zu+FkxuqCrmSWtTP8!Um1;?n~?$K zY7xG%FYfS%w=AQT4Jw`I`8^95F#-_?kIV=czfZpE*&#-U0@OB>^6MyO1k)qg-D&_l zdTyglSanf(|L|Kn1$;vK?fQLH(r>oi^dufMC}7QFmoBP!<#olFU{X7p+oP}I7S4VZIFHDbQ)l+ObJMNUR6 zCmBIi4e;+&X9o5)mNs_>LJIT@n1G831DMG|z*l!MU;Q(D;qc>K1lvzUk~N+5If$S8 z>mTF84!t}9&F!OFNX{pl!kX*@YK&XMk)c=5uqtz=PVpSB`I%CzvScwDNxktiGNTuw%Y7 zf#faBNFkVdwcI_Xg1Wc}{A`vU!m++X#gz)%r)#Mqpf)LPv}PpWQA?QH1E9(F7i*-& zJkjMTe!5q_ITs)PeD}FT&me2MM*N1=x1uNe1Q<}$+Aa!Xdh#$(U%I9^-3@40YT%oi)+z7In4|OryKd7EX&R62#08ETS`Kx`-Ej2UrTYc zVwE7`g?zE@h$HW{thfa+Y_OMF{uCvlan*#?&ALpxe_gP9;Vs*|S0w5g z6fIeTM5$jPAQqqbFlHgXfNsJgWyKLHD(d4_KDFy7dm zLVBD1I*Eb;VPDpu*N^-U>KaFPbweV~f*VO9gfU28-#?Ab+^RW5NXtKmSIzWp)1W*~ z7MS)jSa3f&Pm9i)`Eg{P^qC>6CxH zAO6g`3`l()Weww@^60Ox_#E}|*H$!4b^6dO`9q#G%`3npvVI4HU)s5#5@o0!mFu;f{&6iw#jar?L{)=+STjaWJSE>^eI2mfZedAc$*jT+-rZjZg`6KX7XOY$RD^snA zY8)9r4I$wy7*m}&d<>2=3hVv&zY*@L(kry z_N+CI!idA3C18x0IcJv-gkDUpPfv0qT_Ha;`G;FEM2EJyKeI#m5X6bGV6rUWgZwT~ zwb%bPPdO=_c`v-wQmUBkkgxAV-sWJ&k3a-Q29rxbtDVM|BBQ)T_RexrR_?KZ>UHKg6?#e_b&2)4k z^$Rls)B$f*9Uu$79MpYrs0I<}o{tu9#fau^VpopVp+tebL?*@jRHRRsv{Q_>PDqm! zY_?}}!$|!DqM4L?`J|cKMXKZG1Z$L>-9a#~uWgPtuoUQH`x#@$eZ5pBr#=ncM^)_%FJKBA^ zd6fSCFitN85YI*hjq5ODO=qsopD%k&-#K5PZz_g&)6pkGyiWaZ)IpZTN*r-56h7B! zOXwo(G)_xDtOLU<`<%#v0sese>h~Suqi`@Zy9tllvl4It6fUyfF3kHdBbs#|JDozU z6tX#i7d?I-=z9We0A!2eg1tp%gV#=1sBt3jLMh>VUAL)puEqdSFa?<8E=5xWshq8g z&Nm5y;y?rtyKkT|a^ZUFU)_+F88mN|Xt$!qiG>7zVi07l9a2?2bFc;hj-1};okl$H zo~$!8zhn=33tDt9*s)%{S!4>4E$*H-9bOy8` z)m%5X)4eYSAxK^mu2hC&A^PNUtsX|R&yIwxWi)TedIS}&nth>Ve|+~l-!S?%0Cm&E zQJ%6#fz)%@Ez!Q<+PC+8_QjqSitqX_#;Il?k1t`V1u1LYs6<^AtpP`eH`)ssrfAN> z!nZJRF_pgW(zGrS+@=0)Q8iY<_|1286Bq{K1vGy|Qzy*H`C23X7+O11qPe2(yT*&l zv-lbK5PlkNT}lPH5U(v*&}&Nmo+?uxp6fI4SS33~xi#gjm&4$94Ng-IZp@B6Kl=g( z5fVE-PkN#$NV=2VCht`h3|;-LYt5_mn;QN8_%Zw0t|X%H#O<+@{>vW6yZtGJW0`st z5nNawwgSl#>BJ2LCuIDBrJ(=Lxz}9@QN?73K-{o;aGRR6LEQ(kzL(sIuS%owLdPJ> z!D=CzzA<4_s{I61hYj#dYEq>?^}u5+CRQGq8d$I=)r}46&LoQRxF8xp^b#c=ZZ)9=FAAVcqW_cOm6d7`Bj|#E%Zvnd|QQ z$!=Y2H1!WGadb;n;#=jRpf(7aY_37&-K)YwB^+k{S87H;Y)aXBTAwZYoUy~!{bm;q zYz*2MHm=_drV1Y(z-h+0oQkVEo=oCfUMUPMdzJB7m;^L|Y}0L_BMB>TLC?LV+#l9N$w{c9VIIFw|N112rlf_6<^!C+ z$-5jf(1WLAY8fai?h(H*EyJy2!L`6JBHU5a)c=?~R&=DUT%g5YhR3Jo>IMbpWVm>e4F^BPl8S0ze zme0Dv*8?8ud`8NY`M0vNewJ$;0QP)_T6iy;{rEPykmowJ3tHgEbGBQ<9Cga~VVBPL>$QKmb{f!0D< zl545huWvn%H`%R$l;`Y$LG&jME|n9)T-MLklqPCpHbeQt&!eHpuR3=HErreU`73se zp&J`9Mx76s0CS6ne)xSS5q37yq=(8T&-<={Oag7V3)C*4JKWgG9>;`vczR~rfdQn& z^7^IZ0WU&dhcfBb*4Q=lhBx+$@q~jb{4P!<8Oa8{71Gp$c{kHIeg2`1^rg^G%_0L0 z{={YO2%CxhOsxYWqae95)ax8Zy0Evwq=RhA3?>2Sjd;c( z=Pk#{4%}rKb?^l9qfvMzBZs~JSmEklF7?=Hz2CgxXQ|u`Wi;eZ_XaFF1qD1zAdZy(DWRqwnU(Npe=KlKogZ5Ms#|fJYSjyPa5~29GF7E=9Tqb`ow!f~=Fm`AEw%>#K+vPZ zaYVe;<4yw=r@Qm>K?37le(XA&wr?b=+2!JY&vWXi=k#5n7TF5SI-_Et%O|sfy7yVO zbw2CFG7gf|VcA+HFE!F~?sW>w-gdfE0-mJ!Pmz_?6(2W!re}m0 zLLShv%LEh%biAb`eN}WDWn54H<@uhn@6)=>R4eG^K#!eCzpAVFMb|yLUgwz3Si*#; ziGM6sTYtD0MgL7<)SaA+;ayev?uXX8AXE-=zDDEmncw*I_OO~(b~T)x{FDs=Uy)MR0i@vEN?d;Z zyvm0VMUP7YX74Yfjx!tvBITE#CYKQFdHAGB?b#lQByYg!DsN#kxwuhsi-Tx{WKveil65cvty zYq*ra2h-^Uo)FaM+y}pr1ll#%GO++C_advnn)W{%s{T~?>I(v2!qSM$9a)y&69>4E z&D|yYon9^2!Wz>eK;zp>Ba+2Ertu?Q^ZjK38b@OW$Ro2f8Unnv^uXqICihgx-q#$S zJ5teTuHotkmbK^1&{tFs9UP^J)y;za;h-lktv{6UqP><-F5w<6RGufM9u!axAs^mf z?GVJ~TuRX+vxmaF4;=gaDAzKs zW44nR4fhskA1YrLl#?)HxQ@EDcH4@w_bhKQnv<-Ym8-i!$$gz_0Buwmcg>{2=t1=> z5~uY5miS=C+3VM%A1hbxY&F~uV=!y+t?uijCZK-2$_WulsOT%O7w``bAtHiS{TZ|t zJl-U*5he_xT))3Os?5+XY{p7yM@wD~<*+gZv@3#$>+Rk6*Qkf8B(H}n+`8yrCjEZD zN8>aJBDiNkodot?y#&wEYpk|^CgmZw~fk)g9r=8gRW zc%0G0Y6GBe?UYWw{n*Sr_FQjm1d3|z;2_gD+TomzPqPsaI2ICfijD=kkO43GCIs91^ zmgY&s)T*MLCU;3U^7p>0PPmic9&Gt3usWJMU#4p+e^d4(C$&oa?HblD4@n@soev{C zgGOkvsP7G1*{PTrvJG>@OKLqY1tG~{WCN5-6BwhrR$RfGMsWNrp0L+NJE{h$28?<) z4y4vX(scOmP1U2d>r#2`-V2*XnR}?|)tTBvsp6zIbEY1yJ;I*ZE@NJa%X|)pa_;jS zZ7D(|N5@^PVxH=FTVk=fNa;+YBLk^qWGS+K4SSO*E~HKyR#PLvX|jfVqzC2ID)P!R zwpy-Rt$Bc6f4~!(jVyHZb{HDi{XF2n5KEKpkh`C(lf(3&`&ZyDk&pXML-R8|jw^yS zHK7+_iPj2liy4eE^+PtG|Da%|-M;K|J8`LRUH*>wd$~kt_8`J6;en6Qp>q9F5bPhGlj!dxBYqq zdRP)KHJ1|7sh#>r@ZC(o=9Vjsf1|anCHpw6_HbueJ(ysZj*`$h;gc!J)w-*4G~3~n z(e2p+0S%+De5t-X|CvtD7LFUOfUmPXqA>nl0fmp^ zo$Vw%9lhEjbkkgI6W``y;}cf>({Hg$y54L_#8^3_O{{NEc=>mFb|^_O=Y5LShM$^ z6=v&7M$uQ4iUwo51)))p_)S?EQzgxZp`2)a=BnA;G1t?N_tABdd_Jwz%PH*D>%s~c z17)(qCyb}HUHYp>eVu_siCxR+H={li!ZYjWZXK!T2_Crz80|(_k567ss(-w@E?tBKO(ZnS7xcm>C+(&*&P(?p6odk z!Rc4y${;ce=C`<&Smls)bTHm{0-(eyo5}>y$EMZL-zUByB zi-oDSPF<3SeFsHA$rc3GE|Ky=gcxFIg}Ix!W;8x6Fy9JN^PG8`(#Nf;lJL!EMZQzJ zCm1EzqB#!iUdYGlm*@qn_IK0TW%gd_%$*6Fxarjsky6Oq?iMqwwXpl7PRJ!x*3OZb z);`DF6JA@lmYpb{r8QPG4iLk?3mhNry8{CX14zA>R!5FtC z-FF{rIrT2Di8F7hW?QOpCT35jyE_DwR_R7iQR(g$5D*sK-JOfB-&}j2bI);~ect!p=iv|f8#Bh3V|+r)pU4wy z3IdpQULvG=IKL$oa0)HhiX88B)RiACNv5|NxX8^gV`Z5q1T8h3H$LuV%0FtW-fGZb zzjW;t$-~}9E-E{+vS*XQ=FNGHgxQ0Z=v9>B_Nsh4ftHc4*5c66m#I-Z_npv|aqq0mSvTA>!ec9oem8~GV4;^i1rvLQ_w zeYqOrH225ig{R`!S1FDz?_rcUculqso(vjD*o@E9_!gF*l*c~n|B+Zw0;Y|DK~Iy* zbz`Bg$GK;Ig+(UmXO>>;bMbX^p#xk;08Z=Ze#pR}yzudp;L=;_HXv4nJMIW6L$e`t zm~%52zp4LjJq?k|8(sIi;P&R7@_7&4MCg@P7tLyFmQIEWKWgU{=CB4eVeEn%x#>@{ zoJL_T6_e`W>tk^zb@mi0r+!IY=zDZ)7acKnau1oYw6fB|)#Pp4KFrhU<6y~23HceK z_Pt>}y-;nEl=~Q%#hnIl!zWQRFk0@rupoQco|$OtAgj`N>s=WYhPG_PYIONg<@8 z`fMB`y{iZ@orHChW?3npKM#InF916anIN(32@FkQM`O5`D+ABFxRxWpB-7^YVWqju zd0@jjK`f>Vit9O7uaKZ6Ic+4uEEvkE6U;7xdrH#+kWJ*GRF1>fd3|lCqub~#70Ry1 zQW?er_i1yqMzk|ar%t!0rZ46uF5O=fxM!|rt=bLLbYsr#QYYGB^p}63AL}U?j}!t; zY$H%u#w0TEbuymqyx?2};@?ABppBh-lw5;j`D|dO^LbZ8 zCT~wzVm>tVTDZt{y7Hupp5mdSHI|Vq5ZrA-$-KJLjf=?Hv_1+?sca?wKaW>{? zMz@OVW3pM7?JS=C&8x%={IMw51bVzD}O zbuLn=e#<=%R9laVs+{fD$C?Fu{PWc8-nh+^Mtg4P4ZMvh5{HrYsD5Md0C57J#)DGgOAKqT zwTvFD`|t>?45LCMUd>FDL)p*4bAVdbfJ)bBp#?c${`8#374?COka)@`c2w6W59-q43ddoW*EtLJ3CyD!7 z7sqBgRl;0BhA@?9M9RX4pzKn6hUM%F-ZgCc z6|*l2w;~eGv$8ZWHXkmvexPkQv(3SouR`fL&W#lj*fViboVSo2&yfrGk>;}jOn0Bm z$2l%dM_N>%Y6-`m(4>^z_>DSH3~_8i=V=PJdWDhaW1?P;+Rb`v@yx1^uFv^vX!zZh zovA3QH4lHUYKQvd+Gdi5p1FBuH=>g`4OadaV&4K2{M^uc9=_hUTD{KL>dq1gH4KaL zFl%H}VNJntgDSJE;Wu4N?@pY{5hEFNn(aVWOG{i}FI{V9F)_ofO^XNzNz=Ie^eJIV zp=x|3d_N3Hc2E&01X}TfyY?BLFeI4eq4{0|alxtVqU%g_8z{VM4t^|oUg9g&8VMlW z_#G5xAJo(Fsz#aLa4 z{5#X=R}|@}lxy~6M2CQ1ZsR;TA4_`8d}x-G7vK4W7@Ly8j@T9|hqU5##GY?~ud3CP zTL7wWCI=IU5-4#+^bAP$OREha10RSWwtN zm}XI%_OCxSJM;;a%QZu8&@XS@e{Vl zV{vq+Zdi;D=`bBPc<=UBSU1Y&X+dl3{Z2RWy)<}D7MHheu0YQ5z3Mq#2XQ&T1aP|y zVcqb}>zbKSHlz2Hk$heJEJX2@ws`%w%_4{nv0;^VFV~J`pKe8%RJmfQb1SqprkgH}`H5hoAP{x~YMmXXT94%Pv@vBA}nZLC^W@;*MVb=%1qny>MW`EGM@8~8> ze{N(FhwYmT_>Uswsz`HqVFtl%Zy?io5Y9*qq{L#txUv^tL}L08&r0~SI6Sf~O@>Ne zA4uD^Xg8zC`_1pP-vAU=3}H70C+9arqAt3oTTdq4?QxY`Pmgq%9jC5CntzB4rNobT z`H&7WFYgb?aV1qV&b>2VuU!)S)|HXvOYDy#r21cW!m6IG;|_}vw%a3%lDp9rjN3vT zIH0-Pbncr2xI`=J-xl9`Oht4n^{ay7k!2&Y5 zr=d`!>g?QcNrm^@yktNOp$XFheD=7rX9E3_T@4yIQ=&JH`JV~qaZ5PU;9wIGJW2uWya4 zKiS_hXO38@t!^8+P|Tmg+H^RC4_3a@eB6WH9O;zd&}(*X(|3vt5))!lz$X{$aT!JY z3p_y{z6OI#z}2#$^WGhRAdnYw5kLANyAJwSk+1In%*Nah(`Y6lmAUi>ZWSp(qb3#S zMzE?Cklc$^&zA`t=~n_oxD|{dcBec5IN(JtkUO0ZxJ$;?o%P;+vl50i#dwuYpYHhV z$h9Ch^2%V$+Wd>~>q>@ky9Gl4;rc@}0=Fs7x~F zDOyv16u@$Jc;u`KDyjifu;dqKFQTsh@)$nRiV(URtFYFWCz0nR_wN=z_`(1^46&Q^ z$^sXc*vrfk)IBcc=B1xX;PZ-^s4<#$fjLVz%a2G>A7;~^*sC&W5)TngQ{dP%>(ZFq z|``vA#ns*C+RZ&kuD};a_@97X8gjqp|dZ4Rxj-Q;D zfvMosPq=-la#l5Ow_4EQhtlmil6bN-0xzA6;i9ByWLey8&kBGnhT|3Q)Kpi=8@5Ie zf98{gqw{5_dO#~0hi+BOVD^Bn67=7xSRmH_Q~lyHgwdUFX&^qRaE`ru^n^|^x6VC` zd8WK*MnliCUP%DTXVJ=2I<}ALYm;^HT@n&GbG5 z+INXkH)DSVc$E?br8?(o+Iq7qpLrs#3s=fpIx7i-J^GaM(>Q$j*JK83VDj5H+P?E` zmvh^4N$_yPA&lmf59r`cy(>TP{BW?I?|ZBXXhs6_R&G;^phuo*`o_%myl6e6+gZ-{ zj`v6?ZYH^WdHYDrhiHXz&`QeCxVl*H_XPoxwP%5ovkcv(r;=2Q1gd)QF@gtsv|35T zq3GqmRPK58%I-bCcvC5r$j;EI4XL_imsPoK$WDf`-d@*vDsNshkr}!DIaj#%{|g!y)<( z&65{8Fdj2Y?|pf2c~Av8wgYLwASv9g6VNQIl5>j8i94Acj$>w420aqH*Xzd7Mmke1 zK6oYPeQM2QM3d@;?KQy?GKQJz${1d>3|?P$8YQw<-&95C1j8ANwssbAY3f!MX+M&v za+~shf3AZQ<~(os8q>J(Sx`Sscts0I%u`|}zs9`SZ=~{{KHgabBL`?bn_kHniwVbV zR4MlA{QgIYmQX1G%YjCIPdqvKL*9;g3MO*~j32miT5$$Xho++L4{tDf(!bfedvga) z4fJuED*l()VmO~ld^|E@Luwk$C8 z8X>lP;n=(zD$Zg4?l^aX*>+ceFq6Tc#Gh6V4~gx8ztDO(&_C1X3z%D4)xB(z9B0M~ zh{cAS0GnV{jQeEvIm3y{7*F?rZ1G^(5G1F%;T-DMv?HJ+=%52kJqsEXhK!hRp?BRhC3&xm& zrl|2@s@X{tEpG&m3qr3S##fav^H5u-($DfLo;k_fpz>i||B0oYkcxsq^zb?m9v^~S zp{DbO)W)kku*z56$mnd%u5iBPJAVJBKx`XA_0Q&;U5EOSX;A9?rAj!Z z#FL8H;^`x>H95Me<-V>vo+k2#!%c6dWW%q5THnni14ePaNQ1}p5! z0(zL2^?8tI>BY6Mi0^EbsL1Kwou9Y={2S=SPxR-W_)flg1Bt`C2t^|jnXlmjre10@ zXEbnMd!eVr)V(T?xfJ}S^{CH$r{14C~q3aNMJz-f1^;orD zrfcO~90@SUYyO6kM&HDW{li0?oG61r$uQXIHIB(NoXN%}BqU)ytx0Euepl4L^K4R8s35GLqV`ECR*Y zY1(QMUum%EJuYEK8z)CHxe;h!KrW}HYu{4e6AC(+iK#?}R43{TTKP}xNhg|bCZ{~L zjCOV33r(sbYPrRDS|T%0ew&V-f;&ln8W_rLs}(SX|2C1Lnf4mEb1xmC)P!|d*9;nJ zO&)ww`=XbFAIP-31|%mUwvn!`%l3OCI8FSvnHhT@Ghy|s7&yZ9XXi2gaCE>l(KTK= zs(ZW8G%940?b>nUrT3AnQK0`4IrOR5$E*q=!f?_Kp7c`VPY~Ou=i$#D5zW7-E-|1H0r~NUw7}-6-!0sO@^h(O;U%T`X|MbAN588?%pqcBswqe|k=`LYnvq zqDrV`Hc+5cA#`PzAOKqO6k?)~75XifX9lvO&xtKCPjNG^7`WnPUw5*^y&LD{^-h!= z-;HPnn?+L!4;Fi9RRzwYq|! zXR0ZoY|qNQ2(3D^U_9~eiQkKavy9J4bH+DOxDQ*=QoHTJmPY3mTA9+QI@Ij`?8`*Hxh_q|oBt_q%CUIUn!3?<=`~Zq zMHTBnwg=flX6bLR8@$cD0#$3%&l)Q_jnY{kPWo&%dFq-6dFn-(_Uioj=hfwdb2H0V zN%JDO^T$Q{1x|*#+C|W8=pInU8_n^$jb7@m>5*}9>=_Ufx+@7*OKhpDU#c|?Z&f{S zdhcby3Dr zRV`yVLQ>0f)lBS}w(+>gmU~oza%wJDQ8sfxfr9C{8%UkcS`PL|AiWy0FXeI=>TW}P z$j~uGVTPZ$swFEuldvpH(wgBq=vIbqk)r_TbZj~HbA-2m6uEBUG}SE^hmL6BgC-|C zDs;U{lwIzX(k^N`0K6KW?GXITjO<-oxqWeEGwndc)-kPxxUu{zUy{oM$V8d;6r#A&f_vr# za&JEt1WWKj_KE%JVfbVfC&f&d06$~D;Kv61@s@vT0UUVq#d;G#*$=Q}iyC`e9<}A& z382LMSsF%26sy}Gb2>XqNq#G1@D%}}DsJdC;8SDZ!xyyM+Dw;so|YK3=hREj94;bd ziI>3i=9UcaU4vYkn8DH;%6|sr0e;+PiBOv%U8+lT`{bn23~D;`tU_8`k9v<4-vJ>Z zz9)Cx+d%wdXhhhXUg+hl5a-V*CbOx;LKg#G#o75<#7mp${TW=9ejS;64o}u**I*1s z*V(zAbAlukC;`0NrLz#WFkoyQssR6iDUP^T7f)Ap!pySdubVQ4)dBZRnd~R^fc~nN z{&xYk-|3TokO&n}N~-n&yNg6mg0sY^O(3;EG-4sx2}qKLBlyTMmxkOQtO`YZI%Sz_ z`LS{9dp=!hSrLl36Q;d z{O-X(J*D_ZGO6rFo6GA4a*@@mUx|vbpo&!y; zWr3eS7JF69*H`%X9<}mkwk~SSo@2$!xlS{|hyWoP77)A#T}JOQSqt0Czj-LXcQOQv)7-qVF>4Jj%7qi);qCM)ZgeBz%i2RelTcm6@;pV=exZdH7fU z9wm&)%5UhZ{lEST!-WU$j(q>GR}s@07otdu`>Ob-f(+CSeyjP){er>cH(1`^5nK!D z0R?o7OD*dE_c#CN7sz0&aV#cwGmZcCQ)Qn3No)t&n=djJ+iB2hrB2qxOL2ugU^S^k$+#iASFm_*)moU#AIR~IzCU_8;i z6Cp-~{~5vGK4E_c1f50R$?8D<=-vJ6XAe+e2z$5#gmnQiZV@P6W5RZ;WZSat%NKK?_d*CGk0%q=+VA5@VV=sTS3UiJPWA}KnQ{cj_b@OBq7Bd?IU^v^v&}U5QsghM_8(tn z8vvED1_PpEa>d_fy7u!0Fr4TxX~FY$irWV;vnTd;46YCuv8iZKcrj#_OdIqRbQWRa zi9Pc$R=hmmGaA9TV6b9Zvip=3srh$XZILn!13#WGSD+{v29uXsa{>j!1Mv@j@xyn3 zdDG4dn1Qgm9?kkBIMw^WLk@Nw0tBBFcMpTsPO9cNwnpfX(I0F*MSioSt081q4cYUk zo1Pk(xd%MyTww4F`gVH2u8o15;$j`yD!(&~;QAbkpQiT9e`9b5o#I#)A6+-^_SP8x zS+=*NhC7+J3(5ZqTZ-AoDB0XFKb${!UiK*lmLEgg0PU1KQQP_(Ojkr7omw&qg?GmB z`8iO`tPhoP^8JUkP2Dt)V!TYR)B$L5yn$H-0c0TYt?P^^D2leJNRf@&>A5gNKyCHr zt0~AY*6U~9T)7ruQiurc09(VLlH%rD?<96$?25%Z=V=lst_}01O3pI4+0xjQn~h-u z>bxa@B1^X(O-oN6aDBMa;Qsd`D~32Fmt7w&?KAvx=U)^4kBLk6VgWEP^3h#`ZKpnsHIf|dXlATyY>a0;=Q4IiC3 zl=nT=!(UGnZG)Nl{%JlkXBMzCQK}bn1D7}t%!90R_|DW%18$GA+JUnTf$~hASzd-a z+84eo`?}4??G0|Dd8yXlS*v0465Sl?+aO;3{XUmT1&|#ONA)MaJjU7yMP3= zfn1tkb_@>+h=5X{F340j6Iy=>li|y3CllIE6a~m?4g<~AG_X&Jyw~&IJxMW7WrVSz zIUZs-`LO*4Q2Dnf2Bb?y$TUQg@wZ(-ObWLK7%eV)&zky&(CRw}9%(*HU_6Mw^5SZ8`%&o@=_*G)S5VX#M5l(DMfEDZvBi@Grkw^}5 zu5b5xuVv^`XvlL7>6@5S0O{|;a6u#M|M|fVSldDU9I~L7IE?^?&({cWxk}qNGG`0E z#KqlBr+|E`4Q!cYu_a$mRe|!_9gG$>rsE(S@I)S%MGY3#WxV)ipbp6O4nN%bh9MO| z&5cIo^RnM+g2o3MSZA{gf)xRU3Fu||fc>IQ_Zaz;P4G$mpJF5bjCKE7MA3Zemw;CHVE+!N4ouT#)>)-U_$Enr zk$G;PJ=ufgfua8qS{DnX`U{TjodI+s6qwjeV70>-rehgs^0M=1%^mg}$BJAs z_-S?hHp~Alf3Tw*P+z^6cxtMt@@FKd$O^pNKjl(KgZ;$-Dq_R4X{N}u)te7;tf(+T zjdZ=agR8-41xZJDWwGC`jm*0zHktIidG4CqvovJ;@2H+HP3@Wty`UOkX#AC%r z=Zn|@4{vYw1G;pYL4?)BpRF?RJ(O@}6&|Fl&Hk@jY9cNm&Nh7JS2O*~%?ryefUQP; z=n71@xEo2q0s`p`H1=37usJgU;H19>K+K-nq$*Ol8 z8V)5VKqj-C1z7_d{27Sl zFdt~V>iMF^JvWEq0p@e3sZ+22e&bmfEX-bm*cmx?1`8iR1JL-U`QU$MH~9r+vV1^~ z+_)cDMeZDcSt5E~32|!n-};sp2J4RNx{azW0#OY4%p1(^J=$xx1<*s zG3nXwnIKnvxd1^19|)Axw3HhOi-7LfANGzQ4`3b)_0z9{(Vh+MkJdp`YXg4njUSL5 zZUeM9vXVVf;Gk2dcV<-icc)U=%u`Mmi)DgDS}6S0oW?t@1tynC2qyDJ9a-oexE3Bv z1)`({9vv}$nPDUgcx0sH1Qrgk0w}TfdLG^c6H|c|T5JI#W)^>Q&g=@T!O30!JuZ(+@aKlG;J~86-K7TTuHWJSJ30ziAJ3PWCMwZx zmtN}`MD2zIeOai73Lpt`>>r^EC>;`j>@4p9 z{38zFvpi&g1ww|SbNcF&d8TnjqW`+CY*6GUQZ4Z#f1loB0W^WVPWX*B1*MYJm31)v zWk;A!1n}Wx^#YStRQQqWfVdIJ*p&K9%LksJhXNMAFwl-*1>DJTE5@`$B@>{~DJ?}5 zd*els8R@>$u1N#Rst@ZSRq0&`H$Z!WgkW7Bg}7S5m1;gT;%%D_uhw;G~hUv}l`p3vwo7U)(Z z6-!Pw2Zn$h>?2ZVy9{s3SHM$a{-Hq!1h3j)@&pCFHz0qMG&_WavhjP6x+E8gVPnig z;C#ql#F$Am4ssaMapbP8S#+=Kh1b|ijRh2YppAfj2EyG7T3Gf$L6_;-@RDte+5-1p zFu^TImTg+=ZB@Kssm9?9Ouhx?epzz|0!k?Oq2QCnNpRN7*V|hT%yQJs;_@;YlA+&q z*Q5VesQW$meaErPpFXX!T+J8G+rtI6o7vv;CSMcY{qyySVdIIlMm=tLYgpS?H+Suo zVwn@(29o|2C9^#u1sfnT^Rr<-3zqNR7?mlJ_q7S}Uo>Zc4iq!t`Wt3^*m*Y0}Y zGXn#qgcv?4gQ_h9VN1;YS0y+%fiFrWUf`BU;^k`QNbW>WphrA*|I8Ti_{n2wl~-bj zi1#;#W*jnGxqi*opEddPPG>Eim6g}OyyFtA_vlaOY8vA&f~0UsImm5~6=sO;9Nxd^ zOXYE?hJ`JrM_r-h!rnF8O~~kksZZ^;405s%r33%&gAyCjvPu~HK?)-3yL-QV1P1Qi zVcmiBgc;=h50Y&)kfRSl6;5-`|H{tin?xjGu&A4L8QBz+NZ<3L{@VM$`|seA*(fOZ z-`Biw;eSJKj{_I8PJJXU8j}@xZl*eC5LNh zlW8Tg+`s+0gnYJQhJZ8J_j^H-VLDcz#py67q*d=;NbrT%VNSQCi$7$?XuRA)NvqMj zk|0xR7W>UQ|8H;oU++lF9~HmoK&aWw%=)kQqil^LcCQJjO;0t~gcboS@|kQ5slU5r z@Eq?f;JK$WEwbff{>uRU=d{R%;$$B*Kr4ymO&)e>qRSA76b7}Yv1uz&u_5C95_c{Ep zxB2hi7h6Mog4k(B9?b#P1XjJFJWU8H_9J4i3?iBTd>McD9a&bmJuiLd=`h;=!_z(n zFNg@o(){1%%U{R!FRxYE8CBWyzg;HCSm6|Jb_SbWH%QlPy`a0( z8Uh}B%mB;(q}o^03AP{@WDwC@bXk8@Z5#*iY!Y7mhNR`Od+XVh|r9P*Gsn94kx%PG+t5vp7t`xUGhMpbbmKJYwukVoRcx zOZ*AS>RW9CFa1%V`|}1sCg$Zd1x&`|=%v$GJfbW^kZ#wiE5xCVcdsbaQCV#={W1~&V?p}^8PMaQ{P6Dj_MK!k#^3!p8L^aQOqi5h-5Pbmp# zyiJiPrE<&IgFGFmz}fZSz^Nx7_7K8}44dT(YmxqA`(fAxTWe!QHD#~TZ?ig7-f(v9 z#Q7o~dG#$v0GqO7Oiq;6QK&0ffC5!gM0cFn=3{pMYtKoL4SxdL?9u78LsC&2k&_h> zt1iRKFY=|ed*Fb51d34IpcKSN;Rk&W&SxwVx+LZrBZx~vOm5p_ZXLk`4bMW+_R`65 z8Nv~P1p-xH8gCMy5`Ls1ES~87VNo}c7I>c$PAsVQ34yj$xDE<+>YS1*0< z=@?r2e_y;okEt9NSL#;}YxnCKNHpffk#^nc9w~b$S?T13+MfI6`c`|tkGecGAwHd* zK5DP~TsDR>vk8`06|QE9VURD_5>MM^YF(=5vf090fTs9H5t-M@d)05Mv>u8|lk^YP zwX8)7VI&US+vB6V+T-Unx|J6Dk-DMNe#(E>{J=tN7$A+>PZZnFUX7H(Gb}l+B;k%Z+y; z_t;NAbvOGpatux2>6>l(Dn%+`LTj0v5C zgC8v>3HLkq-KojuS69{+ac4m9ip?P4rcxe=<|QDckGcv@mAuSTRQ)VV;6XvYb0^?+ zWC{f1917M>fORjGUcGyQd}wzXuS24e=*_DVHfWASq$zkkW3C55$mEu1I7&aEv2l(R z&+a^d`jws5)@VLG&`GG{Gfwn zz`uefRK~*yByvYUl~D&v8odE1c$|gY!eRD-f6B{9T!D9q*%`_2vL+2~byqW}Bqb3c z++ujXJ+!S{zTGqw5iMrOmh^3P|zc71qXw;I{rM~DjW z)c>wNd#QNH)vhVzwDdwS3}!&^v>`y0;$~6_+doRpQ-nP+x5yF?nhSH)aAuZDx1Hji zu?NX!`7*ZW=vS5Yn2e3Irf)u$B*qv9Af55+6mZ{R09t`~5${$yqll^Hl2|L3cL4U= zHtP)-uhFtDzOX~0{aCQCJ!LEkph~L??V=ljvkMn65$d|akw^Ew!yi4TKn!9b?bM56 zaU2jDKcraqq<_oFVdmKw5hGp$4$gRdtmY{ad$ZWPiGycWwlj`pIm zR@X;K&5FZ#J}=w5t^91g0k(HF;-Pt>%Qot_C3&GN&teAS_l3e$yuA# z@4qB%ddQ#OMEGZIO@px!Oq-|I<|UWO{)v*@`W~col<4qpPD#mL+?c(`&;3`h6APy~ z*b0t?_d0!eb~dMEV6NBS*PJ%lCOWirJ9i=;i*4_??(KJZw6Aw*LKJ|r!+lini5*2} zanj^9Q}aVY+nI*$&@G1q|8 z7J`Yt0K4cZ?rl^F)k8iHf$i;z6=V*N&V>H_Hq3%W^o>IG*b^dnfCO}yN^@59hw!=R*PWXbW> zgNJGNaZs(&f!-LZ3XI#*g2IH?W=TF<=Y}3Nz)j`*J+AqBmF>NirMCIwL5a~S_Fzqq zzBAL|u3yH4X`&OP!-xLPV-CMU>W@-nAfK*k*gX#ni8@j6soOu*z4f<1Ok`~u6ZGY7 zTlz$k31m83P;#l4NrJ2#!D6TFVB84i>|fL3LZ=Fbi8eX^dhWM1UQCfJGa2&O^s>P6 zBC2KNvkOqQ6+{B$h5_CTAGg#EHIK`}^pS812KUHJ3 zcr8?Vvm^97+ zAho_?P%1|^?WvvaL+TwFG-E>P`{(ZB!oPxfJPfXoI#-oME@+N!;`VL|4sR^9@y30= zbza*}s|{mQ6q8-JQLZx6HnBbuK3JX?wQs6#c3t&O z>L=G|-Y>e!EG8;;H^Qnrq`pf$j?K7mDH)#cs&({Fd6Mj`$8YRj;`9>AwSbH2bzVpU zZ>(5OmS53qetbKkYC4@)-Yqy>`dug>-}-7H*^2a{S(p{5=!-IL2Y{a5%tZcvn0#~e zJLRYI9Jg1M*$#KMm&pc`wgFj|P2c7a{K=tlVj?KgBDh8*G|J?tFHrlPqW|rb)q@t4 zW+Aj6(xuml(eq8a$x%CWb#Zx|iD*JdlK7)vhefQ5*K@RZ|Q&S-%Z}g znDsohPd~Dn8_g7@(rkE|$*zo!BAzCyg(XgnsC`=r+66CPBTM25$WR(_KaE=w(+n$V zI;Iz;G#wUdYtzhCBZsxIP}>R}AIeXJWMnlBhPiK&VnRYH-od2MNMkZz&b~`lOkY5O ziyeblItq#wiKB#vFO(p#>1)rXw}?9`KeSr`<|n#1?*61wEQ&Q7Igz^f z%?%r(lCL@0wOdgA2--fpLTA){*7;h3M*Hk5HKM>)1;q=6Y_qFt=-%w~D#hsS*Xl%c zKkT?JBjiq$*SD6;93c*QLR~LDS8*&+ljTnC9S!?BLHQ|LZ*Q*XL!Qq`%84$LEEzdH zqf;cdI4-z|v{oi(_GKsJ!o9p7;?QXw%B1=Q5|IemM*#EgQ@%$MtDf#uQLqp8Lzloo1lFf;|q?O}}^Xr7reh@eCa; zZ`0)Cd(pixq5;GAiKT1rl-bf(RCxnzkLE?cKCkfyjbR7XISJw0=Os&As$;i@z&^-n zl{--qo>^$%%5yAMVC!qP({krt>Y%a5V)@}i7Re;S5U7zw1vZ}M?J~t33UVp>uZk#8 zYAvlV_Cq)uxFwRbr^r@hhj_&qJa@~zpr&1X0W1!vA997IeQrQ^`mi*z?Ty`xez0}+!zIGTUz$<}$pdNyBt z-|uRDsM+PhFXuwCk@}Iew=`ue$b}))@^orV;P9p2G?i9K`{x-V=tz1YJ-nLF=wv=m zC><}DVbOJa9D<$QC*riJ(V9`Nk2Yp|-De%Qe`sxxv|mdxOHwdhF}Lam+0iIi<42rS z%MzqYuD+!Az)szm?2LXwfq3e0&Bu6dHymcTf=z)`8Rp@#T6fz^uiJFr3$*PCc!My+ zcVX&qCbyH;fWp=a$9<2ZHEMgyw@}(dRw&S%recHL%3eMj={r@V=YzoW{;ca3Ka@?X z7lDZ%IVi{-e1m1r5$2O|s^mXzZlHI5G$L~MzXhg*26jT2$3O6t_>A!AVxqdAHJZgv zu3&!Yyv`ZD9_4(EQ?v*f-;>fL;1gqCv9W#LOOcxC!~=12Mr-phdCb>Z21jW9RT&e{ z#t4uYqX)6~fZOH0;0#z#_Z)Z`uwYx#SZ7ei^O~=bv7s17@q2sRfuzgzOx5c^HxRYp zv1MJ!M0UaT=|I)nRI%e}zPRQAnp2AZK);L#Z_0_^_+P_=Y6d!|E*cbn=OBxP3M*0ZA3o`dn@ zZXBrHJsjR*Ws7=lE{Li@7eFOR2SLLfIlAz>mpW3%jw;mVM|xPo{%m_?eZTydRDh)Z z!|IPI`4<@g0SA|f$yf4_)AViS)IgJ;j}PYE#p=&53JH;G;S%!Fc<7q;wH*!PeLk&s zuMO`&tGXOQ%#4)G!ZG6~-}Xk2&ku?|I`@WBjOZODwh7ocy^Gw?V-+5#1Xa$&teih8*No%|Eoq&5(dj4^D#J&^q~K>13= z-F#BGA6&W3dw+oZjd?ZOtANYp7dEv1*S=_wj|y1PZhcoXQq34qDIOZvKArvO*5rI>=oM-LyT95mZk&b5QehEwiIGco^d# z;iser;BNN19al1Xbn}w_&3#Zox@&)hligTnv+1q}yh-!)sSYE8@K(d`#hO4RDbpcG zzIf&VDDBN~^DxQ&qb9MM*y8V?zZ23C`)l$$L7`X-i6GyE*+}jC0E+;=7q2HxH2I=E z<;G~!5GkH^flTzH-rQO=I{XjxdM=Tg!O&W>PWAG#+8>xn#qEMOuL-vCDrN5uKXbTU zN=+)J96%gPRWq-GLqMaGk?|4Y68m@oJe%d#rYR|%$S?8=87D(k?fWlKw$rXpKnR<_ zO6%wG*{p@)jPF#Z#UmpgX(OTF`>p52a*9Ph-tm1|*JSQe9=qa2=kDq8x1szU3~xFxwM8t&*S9E>WmYeto6AS4Mm1q4 z#5qDRiC(CB8x9?9WrGUImj`s)#(}f$(=1f>b$BS@+_R((4IG?bt0X+(S&~G;EEITt zWPjU7uQYtD-TO((;2`2zn|#gO>vx<*ui14g6>@)ZVn1YW`Mixj_P(iE3~rYY+zLNR z(XxP9lz`b~fjE=swXJoNh+O$umA-oy9|R5F=AzB7g?H8;oE7U`M_&t7W?Xj&hYn_O zRN3f*qO^kMkDS%k9Q=UoFx}}_wUh?7H;ZQ`D^{od?q+Q(o}OuF10F{+`4XhznL7w&Fz1lqUgak?mk)ui#4Rd$JJlh zFp*IeYds?Hj6Le?RNMSS{Y668+eI`NjzSnO`qJVQ%Cim^h**EIIN1B$;-WniSgf|r zA;{G2da~Q=8OG^H=)5~zyAo}$GN9SvFxt{o?uH>@5bpfVU&-)ek^>nE`ejLp3X5yY z0A5(m#?fm3@>ff*YE5Z14nq?PC8?`IbTS*F7t* zSFe{I1p-)|^d5JX>#dR;_2WD@t^dw*-Ro?;{Cei1&8?X)Kp-~dCy~h*1c}2u zvSa5i9e>7>Fk(;=pSqy@>=*e8Yt*37gIv3<{!tUUqR$qPOz$V=Lmd}>v4f6;xn4U4 zW$Th9M#U-N4jgT7J_-GLkri2P*b2Xn?GqKK97RdztMzN8zTO=1sy_=-m(oqkk*aDO zlELcd36rV!u;Z+!A0^5^`?VaK;{W&CfWSOy1A&=ow{~&$_sqznrfq8A=avz zcf`*%-6Z)~%sh&}~kRn5OaAtaaAA z`YE+rc<+qnCq=?+z6C{vBuNcAqe(4_iQ$5hSx5_-OD@YrZ}ouix-ngs9f+moM&Zd4 zc&Dl&)A+mQh|4+4jcZmFt0PPR=AYqrSA*VZQow_9iKkn4%JHSTV@GGdlbN*j`pxO+ zSxia;K6O-e^^q1Y4XiOAX+Wg=T(55l#k25u9*)-US}Led9a9I*{+3sr7jgbU=;AMU z{-lD*>bZ+W4)i_T{=u-@8f)%RiCR!{Qj1y*4msfKHvc6pOS2a0TWM3bQ?yr?|ET^f z(V`7|{A?g&W)m8!)9L8(@tsgszuuy1)2HKpbNNZms`|*z;oO_Cv!B&+;}_MBZ?4q| z4!gE|W7H;v&@Z-zCs)WO#h(#f6sAt^qn59RUjurs+pB0bMt-32-C>zMZDT%|btUN> zWbAo#K^6y!0=iMQSDGUJSs}e`F1QRB@H+kpM3O!7alcpXvtQv^s^o?DgEl%@9 ze%NxpwbwH{?V*@CiEegCuVA`aMW~_szu1g$MC)DcG0?ng< z{&fDm50mBN7wzP^dX$~awT)4}Iq4kOX|BZ2xKTEu)Jw!#%((UA+RovH2)1bNa}r%U z-ujiVS7s>ll7yqSCHao^CAY-l_Y=Ep7cq41)E>8+a0|^|05ZXkWG;{OEe`FQEdmtZ zT-~Q$QlIgf!b0zL>ec}H2Xte3fyh!cLrC;#1fFE*@(=zuloLDoteq954;AD&*3$5{ zityZ3tUd~<^e1}|<~mtAP$}fDQV__y!oky>Jm@0Al92p)Xk6rnfvy|H@2-P}lJF|I z2%Z*2x3p#j6z7O?iJx#;^v5T%=x7s?blOH!cK)bWK!AK5B#bRc;8 zCZ#9+q3GF#?aPX^!G6V_ehTW`Ih?nFrLzVj@8jRr26kwuY*I550+jTY@ zKDEi4%ZvTM4;ikDnGqJtn+@)>-98R8@(sv%>q)H7pCt8H6V}Jel-}N5oRt=j+7?Pe}8=O zD8I~$@MsjA(RHqr>!_R6T<(P7jQE?dOW9GEi<|OG4mjgFlKcf%j|J&QtDCbZu0eRv z{nt7G-8Lk(FMqUj{-+&a7AV#`rLY>eT5hXgUYMNqDpzOvmUeNf2h-;BsoY?2RY;(f zVv|ne#O%$WrN=T`M!spq#ONxn#;~+UfeNGd5x3qW1IHz$jjrg|1UKIvaTAS*QznK- z4&fg!6S>Y~ldirABu|AVV{ope6_%ipjEcmzeNSVW5q#>W1Wk#!-PHy|C^Wpw&O5R$oLu z>`hRhc)bb_H6VDH(l}K(zcX!vj$Np3sq@Se3t1dDWF3!A-f?QVNvwMZnc1?J;mv(q z$Di2Gyg~MkOT|zR%Jlc2VN?iDkzw7SkB@W|GPP}c26?9lSAX_8UQ3l9>k7UUqzj9l zya|zD(y~Yte-Xu#oE&yaxFzU&?nun%n61z{{^D*p>vl!;L~E^F^lG!y3;BF;_{0ET zX!>b1S5p`2@`8jX`^TdGom`tn2^C(`6*i&6c{u-DTDNQ7FE<^@Jp|?B4SpWaYB1rd zILcYgvz&+lctxEge2UQfM>4|hF))i#Dk(ul@6NW`(3xmCJ_Jxnt!UjSlafcl7ILBM z14r04w>FpGV(v#1e_Q%ouRpPy%DO+rQ>B>y+}!?@oXvId!b7ea4H9te*0=!15dUKe zIo3W`6qh1Z69xUO=I3SnrT9t*wGo-^q|JJs+46;QmTx9%Af)w4kWTi@3oq(M5$%42Y)pDbrS4^qh^O>2bXSz#Ye$jN1CdaEI zE0oFW>2i%Xl~4`~p)oaBbBLql0;Bkbh+KPre#0eg1-z?ORnfM~Hl9ZP|3} z>jKh^DBU2Sq=a+}h?0VIOr%Rf8tGC}$w^2{cZalugmiaz{~+ccw@Y7<2?dUW3RyT#}|> z2333zF%*IGBJvh#q+w8IVElqL9Y^nT9A-dOV38A5@vCQ^eY>*O3jI>#gxV+2uz=qv z=1<#wAG1n|uMlOwqkZ8AQAc@+zG8)wAQDE;#z2$Ifi8ABC{=VwF=6P2;BAOT>;7bK zq}E9~QTnMq#2%5AF<0u!=nGm=%It-68 zl3SKv&U3YHp$R5xGt6IK#lF-bcI=cm3SRomFQEsud{ez{zmAAw5??<=B#Ynb#dNUH z(xI^S(dp+$q>l$;hegfx|0zxU>^;8bc1(}?uPDLtd^6sS!!6xwF+7~f>`Kq^LRPwd1M z{FHWy+^qGjA$I&^BfsxpvR18-W{+!Q|1hQCP>;nOM6v_V2Lbc-TaY_eE&Ht@LmyRlIi=+V!o5TkqAn?#@#S>?w zmE?VA_)Phmh<$Rrr3(-W2RUga32yxLW^h|)vOAAPK!*L7^UeH>(s%u$C$B5Czad$v z*1bj{UXQOz;K;XDX%6bstTfq4OS$6xtU8lJt*UEkA$+YqC#Z->*p<)owWHxp3OKeX zT}q`#K^(=_(Cl&}AR?hGxZiRoJYeoXXr zrQ`@+MpJV%oxjb8s3*O)-M;N{*H`P3i_Vxa2VFriIzDPXQ0y8uTRP?Y=5GXg1|;Y@ z<#!ZrY(;?Wx8&hsejFDM!j0#YYR8*!r`h1nw@oP7lTu{UyLJE$uH)8v$MImr`(o3m zNg2KEJ+DcaNDmlldgutD$1XxtmY0I5iFS8zTmT{E5&EaS4y0RuMyAS{C5RTn&AsUq z5^fqu=WT&ut3cc33Kq}nvQ*a?|7UsoIa^jbvVr|wRbV1JYD4JburhFPhqAU5+`p=Y zk+XAPfqODCxasHbJP!EyU+}+!_FzG#zJ-o*jIaWA#uRU;9c3rjI(l)%K^sbH=%cP+7CPh^$dQR7Cy0E$gis==yamA zJ*Zv5ZD1y{DRp+8KU=p=?~tB^WLkF_peq>0KBhW7IMG zm7MSZP4YKj$`0xW+i6$*h(aPdMltV%r-NbCrKP}2+=lg}6ZZXj2`f!X*uwzQ4K>B8 zo!4^%r|w`@ruP6nDU(8;^jC_A(3bkQm$8L)PK}5*^M1*gMg+KRfJU;3;%m!}OD4<7 zhK<%*xUjzDXqeK#so;2Zv6dOx`n^{Bz0aa<;KuFy?6&u-MOPN>2Uzp=R9#ysgXp1+ z;cukKJlDZ}SCP1z-O#YNE*44nbH!s+A2M9aeEntV-`=a;M|mmY6_d?Yvq z62-~QBAuGJW}wA@ozbR9@w2LhOapxoEjr(5%%pe0V)3)t$@QW#NxNMcTU)de7 zY=mxHdV;v&?0UNr-IScjF5}Mm#RG`yrfy8^)f60{8-a^Nemv$_nPK?VbAYH{JwU@A zz>JdMB~x2%Q^Wq;SNWma+cJboTOr3y2yYSrJ8^Ia4Ybf-Y9Z+H?o*@A$xCtihc^$` z8_0PrSNmKyq70I2FSd6}#Jyf0!ecC&39xnA}t#be)MWbiQ1>GX=KZvl^# z>-qYxuU1SR={A9TNW*N3pH2#YJQkWpORTUPEL1Ql*Bf~fc01eX;Ue^l22a@AT!WR$ zM_r(kD%t3$Bj&McB?-)JB06d~TRY{DU;wVSvhGS4qo(<%7XY{XxdBj$V$4V^UYrFs zc1#(BzQ0D}Yc?<<=mo+TYhp4Gdr6PWGJm#NGkI_gLI{Xh3lsS5Gr-i`+upEJX$2`j z=bX`}12gPo&~o6%(%jxbbZD)s*l3~F$fFgG2g9M86OTF`zG;y3X`0|AT(#xsi(`@b z!1nds-SfEO`3zqD0!!(-+L3V-T$D0_4#G+)Wv^Spk+p2Trda=Xmzu>Myj6=7TPdU2 zz7%C$L|Dw9*D@*MMZ{E7Qoq0G9METM?>{OT`1Lv4e<3<l@JKfH6oQz` zn&bs&b+uub_%FQ^kfmjm*=&AR>DDPuvwC;A$d8c)C{|Gvf zJ=s64>?7g}+`Ta2j>&OKJYHG*TrE zz!uWP#xHuL%0#D5nR8#2iMZ<$@KFccs`o+gqK>4t_T6Eb*Ucs3W$6J>QWDKvZJVr#JvSO$+S+;#jH%*L`*yfBg2DY8*Py<6V=+v(#%kNS7PAlXBcZO+-0Wy zi2>iQquVwIOhZ4yIolX(%J1BH)+=t_BG*+~NazAK6ov*7eYaA#hEm^ZHI5OH>DD8c z(Nh(e;{ieiGeeXUyHhClGg8{Lf>f=Rl^z@=`-nSJEOZTa8yAXJIX#J+AFOIGNP@$@ z#Ru^8(-9~)Lf(%eAML45)@XWX&U+zmU)B^a7aDRp2SB*;b!c+VC674kIOHKA>rRCKElhrTC z#}Yn?8>T!N|3Q9!p|n`riPo*rCC2Ix+kPB8J|nxWHlH7x_0;N3K06cNKFt8dhrqD? z?{CQOpRqu_$DX`<%B$D-+MR97b9?vgNquR@r*f%S8(udTfYF=W?YYD^%os3uyqox; z7tAzMYu}?$Cuw5V*5v+;%|nFAIAs~xW3;tTkmqL`SvzMJ4x5>Il4Pg^C68i`;@2mD z?ASD8t9BsKOJQnfI8C~){iW%kR5oAGTuD}-vAD~QgdX2M{&k=Pf_a~WRSRCEV3qB{ z{P^RSLTq4|t5jrA@318zLYE@AyN6LdIL__&qH#B35J5smyn^wA*u?kn!IzNzaTm^! z$*iAoUf67&l@~ywe0Bxk;h#VZZhAHfTgf?wQ!c6_)7U-)S6;#PSfZJfoV8aBdr`EN zeuTc=%n61C3s{>EUXo>#p-%G*>b{O{aTl2z8W=gdvabwNIeWin)Q#qephK72KHpN` z)u7Ma<{;s%;R2Lx`Fxr5DJIsw2*J)lq}G}loP-tX{uD-C`W|~G2km6O35%6pHwSS# z@5;tmN~ND`hCaxS9m+|Y3ajL{-y*RR>@kRRr!?X5Lp%Gv?WCzL?wKx)N4)PrRtz4M zrR5kro43eOiG?jO4PyFr8q+LIb(ZJ6((ni}Mn6|xd^$LsnqinEpVBnj97G=oW0daDi5y+^-&0cV|j ztPJ-?RM>O{b(;s$f?(zQ<3NftwGdbP6VRBAW!30&Y`TtXnbpyd3@Gw1A-mw>$)gnT zQY6Ej7hrg|rKv1He1 z(w=zie0THhj8^+Om zp0T}%n_+xFYWYhg0vvxvTi-=>@;`6zVDk-j)yuAUcV~1>Ro>Dqvc(CtOa@NYxL?BzxG7)U8Xne=+>T^6fsh{;&w5}&GFLdqA=|{-s1&@`^>*$`kz``j%}Q++meayfV5+O; zgDkr6iA&@@Gu|Ic24h|(VLRhhCw+#0;!!ld2F+Szv<{7l&}iDQ7;K4lQ(Fz|7xxAEjz(SK4tf${qpuvGh6&rfmckwY3lF#H#M<(R=iP(u{D?< zZ%Pfl8STMF+Mm5(Ty#;OSCy@HWAs#m5d(Y|+c~6b!H-wuw}XB%?XHB?Zy~Gl&jE@%25j`+jGty}GB=ELq1HZA zFBih&IG3f&Klfn`hkFnIvmYU{vLRx|kZ76wpay=YCgzL#tHHf-`xMy=Kv|%}64P}X zVrfZ9@kK+8nl%?sdM&u}^8n~|Xf|~KL;9Dy!K~Vi(nFT)754b@;gmw-urWZ#WkBEK6=5wMx~Z_|neKCE0dPP04e1_QIt6mVR6T!K z)oor1ZEC3A9w!iMA=*zi(NesoQm6beuFaz2gNplU-7 z)3DbAlPR6huS20Mhzhg0fr*|dZyql{mukHLSTgq|orwx<=oD>e$V#(TE7hd_R z<*(Q-AOaq}TDrS)Nzcc@5Igk#OUO2f4Xy<4WmvO$_W+0D!Ye*gkWRgyVt{(iZ82e| zDVTdPr}I`6fdrA>u2{_W0mIG53q<&=5Y<0_l_ucO5(TK-Iw1m07L;p(3oJ^M$o z*fk307)HEwAI-ceVJLE$9uG%rwz<|*zo}N)hBwtuzTH;KbiPtAj-LvhfF2mr$8fiu z>g`N?P8_SoJc!05UO{SpLr7~PsNS(vvXv6}W0reoMO9i}MC5&-cJW zL5}8+AKq0p>a)fWuf<3gL}u&Ut$-FW&bH-Rj~ENzXJ$b@FVa@mj{5zFE@qkjv)7jP zQEaHdU$g`NQYwijFqHyNhABZwtDn2}77E)^G^l7rv=wOI&$D9z%B$%SdG`eYBgOz} zavO04nWqD^Gcv7i7kik6|I(C+)57^4_-R~n{}3#|-}z5pAEgkoXFN4YK)XH1k*%T)pcu zQfk>-?TJhlA~0(RbAHtdSNNpgx}pmK2la_%1B z3TrxWL5=Z%4>97dV*+V%;0EY|qJZjJ9u=EJ`2&jKF9u~ybi2YQDB(ZoqQI3B$6#m? z8F&TBEPe6KAK^@xgL3c3YxrOM^J_I8LH?DI?HltR=dIxm%kFhPv}_rqwfL6hj!?Lk zpxkCI}cl2+eP% z@c!$QJ3IonPBAiR^Xz>%spgpoi{gL6cK>-bKohNuiZN*IL29k0{^>7x@Sm^df4-*S z2ErDz&d~J#r9ch*eh&>I=0Gnbv;4oU$bU5QfBItr(ESN>>tieYFFwY9{Tlc%fa-@z z9{YywPj}Uy@38;=x;qR&S_wyQ&Hr*I{`a5z_ji1`UVyucYcjX+{=ZlDUrYQyzYsTt zvk}9HrT+48iPzB$*4P?s&DK?5lkqTTx|@-6WXh93|8^yg;qNTmX8iguE2%t=4n*Xy zJ0qz%;N~^2$1cwfI4FgjV|?$SG0wdr=6Gy}(R2t`XnkTzVb-g6{4yYR_Z>MtM*2T$ z=6`+0?pX0PT|&r*zg`pHV_67bEpa(+D(f7{MKk^k=bLAqv93@^xMvUN4F%o<+T72u zy_xtr!E(vJ{1^YUtU5p%n=In{*Uv~}gap@#+0*cpD5v`galA@#|GixYc43wFi!Iu} zSxjaRxH=3r9mD&V<@S|i8q8IpcLVw%ll|E`ht2fhmq7M|DKfC!y$CG+zJMV)X22%r zg5rPU>ilz;aE9MU&NFemEFH+Ezmgxrl{^%Hc<=XV%Nl4RJelCaB>(`uI9cznPzXAL zKOr>qNB|s(0>YeEYdzO}-RKneqQxIG3^sb4z1H0B3z^N5BW#`Dcl^8SVv>qrR@Li8 zUJ>*ytOf8+)<7tB9JTkg%6{{mR~^XmC@*dWI1`;z5cU@!f{~vG4n~C|k8KHna}R|Z zumf8KlgU^xY^wbJMy$Mv+qCcVJzV~ldARSBF<2g#C9j*`8apMbIS~!&vne#_#sHZf z2Gj+vy1RGwM+#j|pcF#d>jRQa-8fb)y4WXgqMAIfdVoy?!=tju2gFkAuaHc_5t=xN zfbx$`yVCC9lbX{XdMf~kIU7>rkUC(6q28AACx>9sc&p0Z{cJ_?`-)m$PRR%)#T7@b?XRuxCUl=iU~xF1;DP&65L4!s{p8c zfc~K!cN&79q6Ac3irtq;19p>Y8^B7c@gajZP(V0ykr0jpxl}Z}UVS(4BVd9jgy-je zg9$Ru@!3opPsL5YT&-KQ8ZRV37g=1?I0MYsZV-5HtJDP)N?!$?hIKmJ11=Un5i<3_ zwL~aX9okT?7&pOUU`3TV3xuD=)o_UzP%LV`w4}C2`4$ZBJ0_&_ai2j~aAXmo`>NIf z)O^H?4`QEoK(iVoPO;)N`FnGyJ$tUiK#V1sTG+J|LJ%n!CP!ig4lx>f*@#N7#+Ver1KyF+11FhK4Mk7MH7lQ zk^kU*Iwb1=3?48mXVTO4s3aLWR!0mTUIC?^4HN~7C<`ZP zAPKJf8x!9JD&?^!*f}_RBDEQ;s z6)?s*szswT9a2E;2nwwa{w3MLa;O`)zjJ@+PFB$U}txu3N7L>mk z%mht>DVjcRK+67#^I-6aY9V(lGx zoZ$`#u#X3VRo#9avXc?Als%`DMv+sh=raV)AhQG}bO=T-pC$ZJf6@k1WVE{dyJ`LF z`NK?yG%CzNH`o`1yo|5hpVt%1BAbZ3Hd}3CdZX@fZ_z=_qjl`I&V82*OFQkh4ze#$YOXf@fVjP*_)AH z+v|OIE$cbok=f)O-_or3D&fv7KNO0`cm!2VpGV8~&|iuuE95uX9vt~2UB*r5U{ zxL3haLV9Q9!Ew(hkcaX0+xzMc(td{9KH4qA&57f_=4=vsbLj{6ZfE+z-1zvw;<%EvrP6F zEE+RZI4Eukc>6t!FdZl(L6|lh97N(kKZg%oe5cf%pkcwNB*VU+4oW@1j!`5x2Fzcj z6L~D|H`xrN6Yv2tkq$Ku0%74`qJMi9Aa8w0cCZjsku4?Xdzajo#HUo+C5()bODz8@ zXl}C1XpgEw^TAMtG*&~ih#fS>MLN=JRE&f}%)g$>~8+><0edK|$2h?_s-B&jd+Y z^N}cygX~cE0Voxnr<$XUI!Va4tNiZtX@GAvH3=MVsRfJ2O;ln0R$@YLc$;H5?6|8S*0vPWv zHVI2V_=+Z%h-R#zQ?DF8PnN$lV)R6|$ZSm83o(0Wgc0DjnKSr2uoRmyc}#-)p%HP4 zfb{?(EnUi(-NMKrfPO5Pe583?CxV;xIAIwFD~*vGgzG(-0UNH=t>5b~e|b_Zoke(; zfV$urZpwz^jBD1b1tpH%;AbN9kL*ayGGR7du0FP(;UwZFTToUpXfk{%LzGQ6YIe4c z9s!kv9npV|oc~p-GFu{to8dP+(S{b@ps2c)3ZulWP^7Mh!}&>{(tXY9N$M^f(>uOOQYap96b>zg|O zwMXN8kq)0nKlG#163=(<@4$xHx>mktjq5~M28)d|(eaC$Z|c+V{lNL4*n+_*p0_=JyB$Qxz!)ktoBraGpNN z;1=j!)t2#V+X{|%#_rl?^tn|QPl#=#JdPso!Vu^=B}2O%!<->LFxb~|k!aEFuCtqj zQ-UHXMw`&@^I&|~y@Bqzxm{4A=o(jP4Ht2ed&fMA02+I|6z^L%93u0Cm;!=IS%*=c zFdu$puiEu9N(Qo;$XcWT;e^9Jh7$QJ^pJ1XoWDqEwULM`He2E-1|D?Mt*uap!)zc# z6oHoAs`r^mk^A1V%-HHwS;nfRSsz$PlB}F+afTkBmWyP+5~5wCX6j`403rH}Vrr;s z2M%>q8IPH7;byG6%h}%G;i_}&dn*_pHr3Xl*L4x79<5uM#D%JZ`Tq9Dq%MX1F?31xIKO>!Ei3AVN^K zAWyC#E?HuSjQNv#JYkq#kUAUeEuY_$0tGA_rQNQG~! zZv%sPhfn4}lV`~r+B%QkTa^8@GALadyJ^sm+fP;OrFFC0*Qv^MJuEaET}=sbvET%K z^F=W4G8@+xXZ(#ye3-y#=(k5cXc~|$OtBpy`sJ5|6#ibIpj7V}I3VMIfyb0(XJ+cJ z1n#bKae?%Qn+@`pxzqz^W+Sp(7Bj#Z@kJQ^gr!{@nV**HTQ|cU6!UN+r6o{2hH4ov zhoFQK!{;0#U7~9cB%2qNSe3`{dTb^~7kS{G46r4OtI|jCvfF&seUB`=GzcMEg*{sI z{zK}C0YAu1lZ%^-Vlt_9b%g|2bdt+aY=GnB*2N-f0rLm_nOKOv2O-X+J+TjWOvK`mk|LGyBlVxrv1R7o+%bJfZ{`xTz{OH*ca z%wgeXU^!^bZ}dx;vu)H-4pqi{yT=f=E>Aw*Y$0Du79VMry$$T0L%+ROao9FaIsL^Fu??z+mlpm>I4sY2V*cYt4-%cxa-?F5gf?BX zKB(b{xFCtthxka*<(!SHmE*oFG1C!QWk!>GsHG4H(eb%7PT1h=Q$O{>1I}vmg@%(~ zLN_`<+x&^}&!h7$MKNeLqyG>I_sS zVEUX9RqByT9-XsKbsP*6qTneR%OhffQSc$XZ)63?N%$PooF4OArQ$V9VBCRCB|K?S zX>~--V@}hV+#vE>VAGQ#of1#Ma969i7?|8S57;Irb&$FXEPeFcfs;szC05?>el5c* zc#62cZ?f#WPC>!b4qCHjrso|^rGrhpNVpy%@$O6vx-Fjwh8r3!%FuXW2{1iiJ22~2 z1<;y9TLCfGj;%~~$?j+c>Q6J(gK?>V7twzNchzm9nlW7pgF9MQoTX9b4b&01H{7fY zDNcIpP*x5M5vTQQk$@*S>d9TL@XC5@r0DA+ZC*I+f}(xh72l_M?4{-YXG?@i!nCgYhq!F4HNd^F zKI^E+gedgejVcQ@d(;u+gDCL5ER=qz`NVB1OHv8s$`8u`M{0I?dJ=jlSGK#(p$=@`ELp^1I}G1-WDY0+NRZ-v@LU4fAeLd zROW-0buIfbzq21P=Km`s_^-~$w-5bCyt#24I~?IOe$g_)>Sy;T^_LPJ;rA^H7_)l( z%U-R6E7>`vIpWc)v!w;YV9LTnis(&yN?1+M$%VKS5m^P`Syn#$_}2EF`IzL@F0hWY zyTa<~r#DzSRw@2MFWUp!N~S@d{^0heUOW6`DH^~J9WGYM3?f854;38lz*ta4glJwp zqfi_;(X&;U5VK6MV~eH90B{96=B0(E>;60Ulw8s6ImN^E%t&*;>A2}?uN%k>?bneF zxj@j?TYb$5O)Bxwp0D*}c(MS$r@t+I)~NPG*3ME<9O;}isW$kC`L@JjCJVcT96E}TRw}4A z|1Ba#%9ea`^Gzq*Zf-;fSSc6MTn))mKdhj3XZ&gM5C)fM>tsQedf~L{{?5z@Q#{93 zArvCEQj{J5T+uVnfYQg}MB<{4Y75qg_kQ1Iy0UBufUov7W0*~dWY4TOdwv5&`5Oq( zs%)hKBbI$Ck-r0SFKEFa#*3fp;r;Fe9}Z!>B%NQp_F^Jpskx4{cK$G8D^t>~v32PBDDBgtsQkG8b(@#p!pGZ`gAK%@p(kf0`+y$noUMcg-i~g$EqxYI$s{Xw* zQTKjBlPtLjm3cbf^9trg6u3Q*75PBzv!82cf3F$zK-90&upeRIH!wHe@XP;k#h{#x z-ja{gBn_Y*|vj&Cp&Tzrn$?`s#pJzYy8KOHOciZ8RXAKaA@|1+BDD*kwV;`<8t z`-JRea2B-f(kDJk83EJGj-vbRUqJnohBcJ0QIevT;&?W?+y+SNhQzhNk&B&Hv^upG zBvqiSP!zrm(5tWH-36TLN5HPJTwVK=EP_p^x{_?6UQ@bvIrvkwXqy%Pc_zuCUI@wu zGzvTMEZ7I|lZBtDbrf>_Q83MuWwitlbAVx+f}-FS1J1ihSxToEkv4}OfT~5=MKb+w z9vg&!m?!wByd5_Iw5tJ}ukJ=GtGz{7(WO5;Op&tF|GX~HcD6S&VJX&*XUNa!Z^-Bg zW~b_$xF?x8Den1m8-%7WmelZZ`nOghgcRXq+7hW=OvV>*DBlBDwFo{V`E7PWjodcK zCGDmrL`ISJs6-LPgkK}N7(Z8v9LGM*FjQ^_(5wwIe+cH&Q282?d0iZ@siULX49oXE zj7{Gsj;v?Zs~<)6^&!ok1rFBjVCBi=-x)9@b+8Dx*P zB1-hiaJG}~zGii+=n}ryl@8-MY1S*9b_r5#haeV6F-vc0#l`+^rN)3`>^YaAh zi}Sk7{OU<7@_nC9kQr;UD-4K{`4+JxO-YGPnx%RoRooRcpst$^&qv-j*Qye+bZ2?!gomkY|d-U39Y%CEd zRYgzts7H%*CdSZowQJr{LQx(eZinzV1I>OwMmV*Y=#qdm)$RMYL=Clv+S!~M^jQvW zbRrAiR)}jN&$sdb-P!)6Cp$7tK5x?8&7KE~fP$$KrL!3BCCE8rQoGYRTR|P@^*0sf zf0dfPbPVGC@3}n`WRV^PBJdc%({Ds_+;-#HVolZSIR{PsoQWxT+ExUiBtfQ!KgzXiHi|wXRaTu$W|$3^(pPyj z*HB4 zvwnti!=Xoc^ZnY6Y(mU*1^_xbyDZo?@WI`PDB+^t(-!nG8{@)oRA)m!j(6>W5#Z0A z_6DFugng&lpzCvdTVlcwRo(}Nd&%Y&?GS23=&L|wf~(zeFxs>^SL35)?z|la5D74z zbQn+)W0!x@&a(q|XzXvinZ4oM4oh7mON<=WB;bLLXUI;;_~s4C4pSgk|5u@jL_6sw(GlyOwHzEV z*u2c#XCmZ&W`74C@mYiC_F%}<4a{0-@k=9sum3non`$LtK?v(b>xzL0HYN73Me z-@aN}KaRA{OrA|^tEmHJzrr>Lls2WwfZxZD%xd# zj>~(O#6;NUse_d_M)iW0GvLvrUO4O8qo1lQO2z9zd4I|o8~IGyphL87i=uDR$OBc_ zYH{n}B-C}QmX7^6Y`U3`QY?$SCX&W_vSTgpi0dZ{Xx$^i*Y~Jfv9OXpqrs%crDeTH z{$EQJ2bvA(-lV!%TBJZ)VKgfALYk~++0}a~X?1k|r~=45@Z)~Yn|tDvq}iAN9ZPd1 zhW({06%$qa12$jl>1y*=-Lrq`)c*;+{rj6~)&yx&u=25#LIqM9PEd8}U_tx!%Z=qg z69bH4xu=S^)?;Ys|3anto4W9`I4PVKMQWp|UT}JUbFQI=_g;iZiEKDotg3WrDldh+ z_m0=Lz@zT&zZ@8UcaSK{V5qTY{(f8aB5m}3i~g8x(?zy?s;D0D8ilHgz>i;xp^QLy z+JdvvQSyDG)%$X3z;6YxETpzTnYepwc(4N$15=`{o^eJ-EXO`ml{R09qe%bE5&IwN zTX9l4z$@!jHY$H2{VJ^#Cd@Id&B^zF-6#Lmm?Pq+O*zBv{V#tQ7-Tyr!I_5(DCqxv zVgB|zZSP_Ke#8IQTm653hqOY%$x4gpzOO+?goh z;AFk`p9>*w0Vii#m_LdGR1q2Yr0}>Db#NERaR<;~3v|th; z<9nw|$0)VJb@_Y30L43@78uLD1iHy@KwX{i+-MTt7t}I(iF~%~Zwl07LD9kB$agaG z?NxjaC~{KWesJMu3Qsg*^H3Ln#|NJd^m=05@a7usX7cvv&mLm4_{m3d800 zZa2RtMuJxQhvgQ*bhJXU1kSbrgNGE=N0a zYAoO6=GN0?#FeKAbnvo=Dcz%M(R$f#8$uuC$X}i4HAhA1OB{Us5H=;PrGegE=7SSj za2Npe`!GdkbhHei>JJ-~dYeB916YeIy(iuJs@M1r| zoW?tlttAIqDq}G9iUs*9TsD*C&K2O&7FIplkwRMg+5&ou)!|nhRf!>u?!TjeoK3$J zvIl3j!TTMMu<-=&`#!jJ{kV)*fqTT)+;{>4sRQ`ueuDptx{Oti<(lLo2vVz-}z!XS{%QO;@n^Ru_N-lm*CsFfX9!|7O@8bPVE6wDykS zu`;w&iaczrEgc#0@_H7Kc#E$jP3Pz_olma-ko1t1&bR$-EXdsv|0Qy!dquHKn)PS8 zA+tL?fj|UUMasZ)3J^L4d-qD zg=GxfwgkBK7U%4LM6u(sKyvfa3uFq`ecUfPie*ny%*OEgZ5>T%I_%8D;bc3#z4gd+sjRrZ2lvWnV zOTeGh3{c(FhW)I~Q+E7j?m$A*{)`x{KM&KRuX>3YC+Zjs%7ainq)=<(9P>KJ#UAkQ zslif#{A~3W<-v~p^L7%b3`RROU*#K-mTFl~OMnun2GFWF<#(_XWD05b*HM%0_hkXUpePM%!RJxGVLD>Kf*mt;HSAb#30$aIcTU}( zH1G?>^)f*dQ~3RM1R$CJK7R2hHn4IS-0xoIye|f{ce7mm(KyopGGv|NiZxzsBPmLQd&f(QR=ye7I2bRs_`-u+gna9Q z>cE`J=B6!<`1BTztkm$ez-^(p?x(~%I%~^Py~6VyerDS4ho=jK=g8w!f1#^~q=KTV zV*!;F5o0wRAs}UPWe3PsD=j2d#m~NiZ7Y_-t_6|31mG7h;Y3PuqG%cbiq*_*;C7Yz zW4P{Wgv-UwH+u8}aFnJ!7_Y{3Ed7xeV7dfn*1k6{OqU-N44N9pAY_DBriT4JQv6s& z$h*ON4PYO~kRQm@o!nuuzO>t$uJUsfyI3WJ$HDNjxdJQjD>>uUfdG`_uYg=N_AwZS z(!*)mM^d0SS8bysAy7UxE^|fONXl)B&chD_fvPN7=o8ej_wGS8p1bac4&Q-0 zX4fkeZoI+viUb+P1z4+RHVy69`*3L(uk71V@uvNKpK2zX}g zKV?$rg)gb63GGXnB`Jwy({MrBTFRa*{9yb5IEOG8AW710YN+5cGL^+M^q}_>#Wa9+ zei#sDvKB#>2l1{DZi=SzFaY#GDv6Y_w=}i55TMiE#;pNWv9(xW-gf}U>sp_y*ULmJ_hEYbo>s z9W_qmsE=M5aubUk=$-Tu6(IhWvKc@B-eHsa}Onm=yH=;Fk#is<)yWsUt{#)eGf ze&`uo)&K~+WUNacpHdthdihJ!eIIyQynGYkqA_~gfq`*t?{`K=vZOZzphx6As}Dkb z$pWhQs4EuW$jbs`3mBe>{KjFp57*Wi_9P(Qv~Fc&eGfjOI3PX@8+MhfX-ip~ zoDsOb`4n)}bdP%!W1$8d82l4>B%^O+~_K6e1h^G$!(+atX{ z(q4JBB!0gP!M8_j_BaIL2V7J5xp92?=HFpNCH+nm+SRAyczw;ow_uN6 za&QM+y0M?@L%G|az8jlh4)E$6H7q1`k559hD=^pLU&QPfDY?3xLPQmD}wh95(d5@LgI? z`uQ`Ubcv8#z^Hv51kU;rq}97>E7E_&(sx}73@}fmZi9ZB4DdMF7{Cj|=XgvBAV}yZ z3y(Q;Il|>E!Jk2a;~Svf!3h)R&7%NQ5j;_V^{Q#f=<^}fwxC0zhb3LR?b# zY^k=y26U_@Mx8#KjOX$w{bh0=hzonb&8u_232^TSkQ08}I8CYTt|W z&0kghVR43%(|L+v;w>G#V!3*p+>eb)GMf`Hu3N5aotJnb0R??JP$TjJkqk;*C}&8} z1x+1=z|3LmIDL`J-K8@86j16jj3s!0SUcI?&ore}QN*5Xlhz^mQnXDDQ#wvgKc4^( z)3=Sw2YjsvIt2W}}ILT|3$CFysgFTo6k;&)~ubXy7 z2|TMG|yxm~`#di0w!}))?9PW zHLu@wSQ6oW-BecEmqfNeJu0htCy9?Jpm9F_HsvLCLZ?v5^F}X`hJ~h^0e>ak^H6mg zqw4nxC#2!tvoytoeW>NaBU+xS(x%D@TH>UI)wMqLOD)QDXe*mE>C<*!4>3D@8|Qlu z2ZJx(wX!-nK;DtSBV#`5xH>Q>_iK*3zoYLP)b{lm6?uEm$nb3c=&Sy*!?zh}cTILr znHzr>6Hn3EvrSF&w`A;ZybX2@KS{gw?X|ig-rfZoDcb>yiA{tJ$hbqsNWxrd zj2rC`3P7PfpJjSAHG0Yz5y6^aWxrTDB9apf?_Jg8W$-@7_@Scea1T{()(|)ot0>b? zbbl$8Kb%wIn_nA^yg&I>Fe;oM2X{O={Nr>#731P_-38m+x8ehtF?J{YH%5%{Y=b)p z2V`JE>a2G2qRs|dy7h>pvSKdK+uMfXY7B311V6qg`4Y$Eus!iKKm3a7YkA39I@QjN z1cZP8uezTfu8bxZ{4HLbO4&Zjv6@jb1k<@sofM_Vm?^&Sa*RjbV1z^4lKtT=e!LK9 zBRgL5pCcbatv(?h3U7ay{ zkg!OU7h;L1K2~Hq~Cxw<2m;uY)i($$zp~^4B4Br-}sVn4Dv08a} zVt+mmL$$qn1o+bxF0(8<8C3(oFTMjpyn?wh{Y~SGdY>=zc0d?h4_a#;)FM7VFK`)Bw?IX7` z=eimxE<)rBYkSJmda$%0P3;%_qD_l`r8amG9Z4|)HG}=Uk)`U4K7XmXtwa1x@FJ~u zFVRxj!&yIIW~N2V9?ZbRmiwbY0L}kVw9}yPy({+-z}|L7VPf54(U>AUq&G z|9Lc~%B(?%WbW57(#3z>_p|D5Zd8nTmz?Nk#C>(X+vl|oF`9MdCM5%DGEwFXQ=AXG zSY~%1PSewwi;$AYJ&s&+BRR0|nJ@SP;pmH#s>|26F8HdshYGtbhF{cSburqqV)%U1 zoE8^W>}LEX2#e@=KbOpiU8B;j0GC47Dt;yE^Fbysn9<1O&n zS(>8G=+osL-@vnmSP%8QKlZ%s_ieMO(R0SGUMcuSN-6P-ltdx_}vI`aCcQX zNA@3FIAlMakFGzJFmJyTwmCikzEN>n!e#+2){9#VGN%TJW&+8fX68|bz4oWIACppC za$;LnXya-lP+>+~Fm4lXk7oj(D56zd8-Ftt)W7-K?Y${XN6;N6S`&DASrfVN4CL43 zQaqC=dV@mZv_sWd*mHkP)-cHX_MPOox_(4J!!3yDb7U^oF_S8mn((qbT!|K^_xa7t zlFq>&l;q-RTwuknFsReJ)O2S68dmMqYC>P^r1T%v6!e!KXvJ2%ttu$J7y0P+d@OSEDP`PlXyIpmG-P${F$e5 zVXD2*9!t?UxmAoP(htSMkFV3Rv(6K+wr3U@dy1PmQ#v|K?H)kSr~H+}b`NJ=;;XWK zP_{TOb7F7wgANXHZUAfm)TX!YgScw{qE_9vMrRj_0O=9fG9aH}cY)sha*t2wV^fCX zEXER_)r6rDP4kVIqe{R}U6N}rY)`VvVy*k*ljEV%ZGRdx(7JGW-)CJ})VTP&AQQD{ z`vscHpUr%FPYu zsuikBu-Xz8#(wzyF+JpV`3kO|?#2shinvcUcr!V%{X`HYz$2jHEoN+&P=HdbgkXyU zydQu|>W3U3H0+c(wpuygWJqQtiq+qWtvA%x#4jYsaIkqj@?$!IjN)G@iXq)GP)+?)|v4s~zgrtQXPNbsr}B8rI;G zM@V7qWN9P8yy9)80gOXOBEMBQ_#bXhSaoVA^4`J)L(P#bP059wr`v?^o{hBR?0*cX zBeLsxg(U2FXtT+j0{NZW2SCEANpPqM_{jGN=(#(s%*%pxtHSpj327h(a~&=!e)n|H0(xZdX3z%+{jobq6#bd`PLjaD=jrY!d?6sr>c9Na5`bx zn0JMdy*xVJPcm~{bsv0#vmjb@AZ8!sGM?)I%57x(qbLno4xNbwzY%&?(Hrm13;%~b zT%`@rfx}o?%7}TnvM)+7rLCtAS;0RNwk%7ksKglTBK(c7;;w&p%X-5t`ku3GILk6UO}DL!;AP2aP43hV`WNAh;p zvLUn@ET&?&WRE|uH(#z%<-N>!F@M;+=0i^`6i;X-$-rLJ>or7_?DX*^(-w2kaht7v zk|gRrJP~_^2k}CtU8}!8sTYQ6pQGYT$3@GcH<3!Y3yIwc{S@^+aOm>)RWWxqDyy(p_>bPxAt@}=&@WusEJe68RktWVDbGzYSRxCa zFFU7*?01*LTt^AN%6gQZet8!hJ4DG+%e~u4GgI2RpEt!5VI^LJclZjN^Cbz2~!cx!|*|g zm?YAw_uf~G8_Siq?D|(oes{#|Hi;>lFw(h~-!~rx48!O>ork6zf4ZYFO20!4*N*gf%IP*!~opL|f75o>9ArQy;#@L3*xZKKxh z^yvb7^xn-bXUuj)sjUCXG#(Sm=bcW(5AT))tskTXf8XPT;~9-aTD8z%gup6Pp|ham zw94d$c)*t21n?0cFCd0@w;vPGy_WNBf4jIYiX?T5O5eh47SYd&%8dXAnVzAcYcw)N zFU5mc-o!y}udfaHmyUUqZ^3Gn%z+E~!Z?1&wj3dv{3wWru2-tt=qpk{C7o+#d4rhA z%dpFJUpUGkR->}2ko6$ejK}xQ2qt?Afjh6?yneIv%V0Aj4AxzNG_E*j1#7TFVi0NM z^YX2rm>VYZ`$Yq*Q{pvIq0ezVtk#R8v-*>(BzQfVuNnG2AJEyETgA;|vlSkbDGs=k z5SZOd(Pzf3lc&G)U6gAVKjP;JHV!>GRbD6YEy|!Kw4Y+ma+OwDtecPy* zY-xJCEKL3zh{*o>oVb6ov|Cm4Qa?fOu zU2f&dYmx3iTiER5xyuPq1tb4FEIMAa`p6sK*@Ne`*!4gW?C{*Z^aa17@foYUSE*vq zwwjF_(_X7)>Eo0YYK7|y?bhW1 zd@qzdB)E4{+pv$WX=wA;wZHe%;g^*Ac%TLSLP8ALR$s{(seCiU$gtSC=37Aon~R~v z!f%6K)auKq=SJJP4FUujGaHCwDb4$7Odisrmd7PG^iE4-|v75cAn=%%g z`sD%gI$0uLJ<-jx19WnpeVi9a|3%(F;F1*{xKyS58rvC`^?HQ0AhX1r_3I;Nw}QM# zbUxsLS399IltbApu0N|fbz!45?-kDEH{XKk9l-<$j+jkhn5Wu@O~p)b()7%ACxVu& zZxJcbK2s*`YtyrbVoC&+NMqerIrX^M=}@%O zo6U@h@9hLtVguE+I>kK=_Nfjo3J4H$th zFe#J1Z(wuBo@^zUpZbC0?;=eu^*-sPJsVW>E73lVu4q>jUT?g(sES3`p8ed+PNKNX z|HLwLvk`T_cw31UCC&+1apJ56J?es~^@E@vH9K)t0Q-G;iPg4IpA)5*Zt?i>%*&|+ z%wus83TqmCZ_ot;q77yTYnEpHO!!nt2-)dqq8x^6Vl^V4c$_xuzXx0PyuI= z1YFqRrep6|s5%(Y$0L~xPs$;*!ZuaIrfXe9(%Nd(PYoka1m(024UBUp8*B9J^laQJwp$U=zYt;qEct~Y zDTeyivxYejb?4_#bIufI^A8orf8epD)E%%rkauNdjtE!svyB@HQujB1qsB_V)jVt@ zz7Qv^WEgj8f05AK-WZXfSDbFGb!d}?`y;??w;&%WiSIn3Ic}+eef=CCT8BIgLea=! z?w4Um_KVemkDu^Uk}smNZDO`$D&00L->DiDG%D1|>>N7~uuQKsk`(QFvDRe?y}6-Y zne6gvXMcL}=l;$r$FW?{W*&5p6EmYRhM#p6XQzRXeZAqxuZu*-y%>2+$hIXw8ra)g zKiGwV(I+zo(K6o0Lt*YZp(mt#-8ukiGhzf5 z=G!wIO~MxO>uFU51-`6MFQ1hc&g0~`OuM!Iw~8w>b0yNXs^={ZWG2NWZw;7|rx))#B%=g* zcsQeG`^*-Ps(5(+rQ`+?cWGQQKu8k{)(ux}RID(!^!h6I z3ghaCsgMwFDtQjR-4^bk>ka-!=*|VbqO#nR?W0b{j@ZCzj(ff@y0J4`iZ$z}stnyn zgQ%(s;NxC4yjlvtYB;H*zgfh%4;+E|w+4mXgm@&m|8z*&ZRBE0^C?L?el_89=OA@) zcYQNe;rSxJo=;=rSx2eSj7X3R{3+p??*`qZX@nW>*|>RQ`-O+ylg@Mrke^yEid<$y zkdSIWKEy<%WzX<)z37;h=4P2*qQJ&~5Wv4^2{?4kJfbI&8e?!y`%;Rh(Eq?XOY_ro z`OeC<)E!)H%{3AbEjKRu3>_`jw#`Rznh5Q=>dbhDIzV^XM!VOagEf{#2pzf+J@S5M zbl?({wQIQT7HVZXIlD^qC?G$V0`N^Y=EQ%BlGb$~Z-QPqIj#gkpL%F! z;(8&Q<4OF<7wr&y4<{ImcY7Br`AHh{cf5T4Z(7xYX6CR~5A>efvwh_c&Ou7Nv7fRV zTrpLH_YQe;Q>vi-kR9Fm2|*rRyG-H=eh4L*!4x=N#03V#V)`#ui2mwGANL`m30iV_ zl?eA{Wf}qQWYbp!CPIy2dDkTz*3t+2hxvCOn&CFor3+MAK|I|1<-PHpbq}nyl%&QHUtE+#+pjc!$$al-R zlTXevF!w!DaBxL#7on38AFpmMAa%VU>)`AVbBnrwPp!hX;0Z39U~M_>z@w5oDJr>F z9W(E^Lx__Ap4F=}O7n|o_lIP?&xqojM&)U}c&rVBIxJM#PgR=uoWXr9gcot>rq#&9 zg_WqzAKiQAi{e+RdMVY2GgW8u13H@(58S&=Ff#6NVDEZSg<4%!c0td5b=RHw2%Gi8 z!E#@VMvh5NZ$cvdUNi3}pBb}E>H48fhkd1qs}HgrS&+<*0~Fi}Ct`NKWwN+EsM}Y& z3&}O%zAAQC^v!#PSQlE8+jFK#myaQ5M#5H3g;iY_32FySvO;o1ff4coUbLTreRfY| zubYDxxrC>)BeOYFo&c^6&Ah}Nw*8&3)pbiu?(QSWKZ`1iAee*=j67N$EF4)en92{? zhtFtrxY1run?jIGC-X))bSOJ>;QZUw9pC6OU@?5-Fjr%Y)6O z%*+sg+rRqwzECh3@fEcdkM_>A0|F6$NY+rQ^s1(w_4?q|Ls3(5nQ`Lw`}q1y$=?lzP589j#8mvmzsiq$%0n8xN!M6<+Go12;VNWDeeQC6pUxSZwxPyxb3{;H zz|JL=>HEqR+G!%W_J`^onmj>8dQMeVRV7X)8YL=#xST~(%Wp?X9vN_hh4&VkA}^vC z=itzFv>DLG4`VM*Y};msKU^GqD?4X>rDR^XRR7q**=7pedhb4Xu{v`DP_cLQyCvJh z1=|KVkQ4ys=7q&_Se}^Yw+e|ZtqV8Mc0k0n^X(wC(J@)hwfcs1fI{#kmX&3|+29pl zp)nY)8kC^KrSm)zFqOlP&x@fdIw1Hp1HW9EjJ>rHVL`_76RH}#@a%kVx~DF=@|Rr*fur>;7Zz+s&EdKo|<0 zA@CeC?s%kMZU%>cP0#}CWN?;cdEd;)e{~ZjN=)TGAgX+w#*i;u{mbXBso(8Ik_TnM zphe)K&H8SqSt1xMQ9!!SfunVwS+wMil#h-(H&~0S0+w-KOpp5an7c>0pJD{I-&MvN z&%qv`@Me>msT0AsX(JJt;P4iWe2(SKw7hkQx?6lb<5K2uY)1CaI9woXTf{o#&L_=-2RJOMY_yv)oGgOnn)UT<%xfzp5dIsb zHuEP)th+>sd6q`%-?dXd=5tCXTaq!Hutp7yZKDzTZ`yX_NqrBHfQyWT@wPp`J)3u; zUp5}14D3EZwYquDTIbWDhlH>bAcJ_-VM-AIfJY+$$6#1Q!uzR#(3ySxML}#=ZNdoX0%iNlHxvBP}^rE zokhWKoj-m9yUwQ8Lnz}JB+U$)Okv5Flx6V9(7ex4y~RQ&V_WeX@3l?% zbDO(-TXiQK*Hap0R~P+D@VBc=@CBUXFU;C8NXd==y(t=(A~(a@?}geZX~*#% zrDr=DgWi91$lxdLJ^-Jr7319%z#?-R;f(U%V52ihY3nCV3yTT z=LkdZ`<=UU;)F^gXo2VB&i6`5o(Kke#EHJg!-z0*{ld{Lu#XPhF|2_;16 z?w=12YV_8W2v2?eZ1a~b-!_ilnvuS9;W!IAn2gdvcn3oy|5V#HS;*A0tW7DgxG;nV z=`%d4x%sAahpuiJB*>p4AfAY&Td=!^f?|@Gl@?5c=A!!_uYMk%+Y{%E)>(S3q02ZRQ#wK|0ghHrFARe#&)x#XQGep+S5QW zyjYAM@#}WAH`)dM`zA8>8pe)UXz4pA`qNCtz-(ETYD~-Qs#vDaM~`i7rm{CVq7i_M z-7b!`4SCY9EWDEn5vhl|tRo<+v^}A0psrWyah*MnG(R5A>u#Gfe~Bjz1T1s}$sKt_ zxJDzN%t0~|qG6KF8?ur=o+YDp4l&jib>uk9SxuGg0lJ-Cbnn6B-0K^Y*nKkxgS1DKBEtOS>D)y6 z!{mx}{x=#7`AY8u@PtYtS?NSlb|6hROG6vF2Il@6Ef_ef_ZRu1cZ{wF5*aNaBt?Xo zFK)v;zl?|XsO)sNMISU8^a{ZhDN(ypB)gEDh{0QNH>H~T;~^Y;eRFlayn0=)Q1|J6 zLi2$^Go7U$L50Io(&M46F-BC59`#24`fa(prbF?|=G&^}2EsnW8=JOQubRg4A1|WS z#H+C?HH?&b&v<8y*zvf1xKPm}d9;XgX%m?G7G>Xxw|um3jhRa6ht1UO9#HlWk|kdA zqQPw|&m51xekfieHu&L{)sXqv+avc^??q*Eej7T{{>7zo2Mr6u=+U0GUby3Ur%?_D z#Z(!>i?k&QIF(1(H@~CMwbVC2OSijk^P~*bp}QkBAIK_l^VAln5)(6{mA+S6O;i^T z+(iqic-;<*8S7 zJj@m)P^ZC71toV!pFN_6P7|ryB=+GEsJyO^iYCqVt1=e%#hRp<+)0$Nt`rkc^oy2* zfn2srm;tqnlQrDbF9>F;u^sl#nmC zEg58>^fxemiGdek7i2p*z$+GZN%uvpduB9`#o)H;_PEXY>-utL^l#V*Yf{koEU)kD zfk8y-%5IaHHwi=NE-ZG5Y%q;MhRb8=SR!(+saA#pAZuH!xmAkth@DrhonHWT)ku<3 z;fhgz#s2q;Hh8SAo@xR8=entqziijfPF(!V?rF^_lL(NwsgmuMRC;8`e2(;beQEj4 zVvwvK(M~F`H?NJhtS9Q}r_y7D$X-6zHtPD+O++MZX}HY&~x!n*vP`lhb6M!*1LDBzsa zyEdf|w;RkHH#IpV-5<-37U~jXqYK2xQH1L-qV_v;eVH9-5taGnfPmt{RbBq(&5R?$ zhFxi~b-AB=JUsk~svqr3XU2D>^}jzYU4@{A*M-bF-J5V1GWw5dcfxWc>%oNs_o@Xy zLGBVawvw3G;W%3>b>@>KH?1|Qg+9SKaOJR#uS9u3&Z~+&#qpRsjO+Vo`$dqPv+j%Q zH#cs&3)SV_H`I8O*uuC!Z}6fxq>PQ$Ccx`POZ&j2vP>6xf-`WaD_f95dEF(!~FEfH-iXI_#u`P$)Yfj;#dq z#cwA&p8W7@D%R?(?}Z6k%E*h7a(MkJqKb}!n;*xbJE_Ri-BQQeSJMV_-fuXyamO2$ z2UM{*9d@t7YTD;es_ZwE%;LKUI4O(1b-Y$hRX2YRA+*9Gr|F*uNX_iSx7O)n?I{rL zGn|+)9vw$bHVvG>l_fFcDTlWhMByRlA|TyzeA~X-vGeK6egmfL@0Tf=Nl1MG6Rl>wsO;(v+8(|spI#9#h zTDf>u!o)sHUX768j)O^%W2>aFg4>-jAO4Lv#anVhefF2V5*w_h*$q3sR?75A2yWRT zg4i)0K&0UKb=P@A!`}P{-W)-{(euY^;|+^>&L;zW8+qCCZ(Ug3MdikPoO8sLksT$@ z#0-Yn-ooOorEsT7LoQMU=44_B?(KY3*HdnkmWZZuE(o35lg9k|sW81O8(M6Yr-a{c zj9+u-%S-fkSWo2enkIZODwyI!C?gXvRyqs^_W9HIh*Qpo{P{Lwk{VjJxb4ojhJzwEyJ<{Z^O^}PY%Cd9f?||`>gK^78LNWF zz0W1+ch;QXRFw^QLq=tYcP}hw@9%CJxmJj`A8b^;&lGXGdTkAgXB`CXKB)(#dr*Ak z>Y7htVb#z?T#negTpzA>G7I?1^O^aZvf78{5`trKgYkZ8>=y~6?QZc32D2N$S1(O{ z-A8#!9Lqpmg;-p8c0?|g?L&iUt$O*agfk?l*+Yh#Zd*bmw3H&y|8_)VDK9R}fuXV? zd2l%;2fj(5NpEv4l#1fLQ->Kg;f-KBd&owiGMdvetLVEUwiL}_tvl|rQrs$uAJdgn z9n14L2PyVyAI!il4tGSJiJPv@;ZMaUJR_oSzbZlp(>*NI`;}ZW+xSoyVbT*N$E zFJt4qKmjh-n5`J(ExA7a4U+M77v;KIxxncHE>kAk>#PH>OFi#>wlRKY?SaTCak^&; zf3w7Hj2Jb6_e$P1LrW0L2kWC-CYt+3KK`AxqBN$|xs%Ng&jwMRSLQ5Qb#I$q<*};J z-c22Uu!)PuweRzKq2+G-^^{n(Xc4dRby`v8<%Owd(z1@l)Jl!(RGZ5;Hbx6M+L@J9 zlr5leqnDdSzZ+*KyvAHmINU3kJQg^sBGB&@k~~e#$0Kcm*S#k43Bxsj`x!J6wqT!h zGN>`5b&PX+htOhg-&%S6bvej`T@|s$!>HUIhEzxpIw@2Jn%0reptDwnbPj@Ybala} zPCuF;HNcg?ZnJ^p%bJs$#6`=83%VxwrxY!cO7i6|_?a6Qu~@xnFAy^$luc`C>i_a1 z4aII#l=??_S;5=j*CLcE_D|!YzvjeaCB@5!pqXIT#BZ|9qz%4g!)G~Yz{mkNb*dL7 zHXMin25-aS2^W;(`>>q5U|5j?JjK&*+wC88%kOeo+VH};)+p%AzM+EM8t9GW!n9qm}QImM%5}%=LY_O!~skwpw6Xfnj zpCL(~v*^1fJBmk-hi0absK)yyu}Efmvxu(~8mxa7B8s=4N`7<|gF8!x+5eb_6mHMn zfsj^*Dqr9Zs!9Kwg!$Ss4JUzXAgs;x`&z*o6{b%Tzt*3>zNOnwqXzS+`ceaG>AX$Qu>= zqM1?Wcibd#=bh7VBcqgn8uyO1w2t*!F#fTsTt`c6%ctCV$QOwVKZk7%29>7D%Q*k> zG37%F+;yDiHF8OHM)_{}(O=Xzi4HWk3wx`n<6j5S<#ctkcVU{KmmiLm->Yr91OO;P z7UZD;Ah2&~ByKL1HLxVJ4--SIM<_Yv=&vzUi`Ww7W{zKu`mhdd^2!OYDuXW8KV}WW z&r(gNiUubKM?`{t;Jz=OCTQiW(o_2(?-q4CE~K}aLawDuL|Gt{)wGgest!IEJTOcY ztA%|9d+ z+ggh+YH8y8v?IjP{J#bWH$m?N9It&(AxQ}v>k|d;rqT%;6LIM3ptmmv6Q`GhOMYmN zIfQ(<)GbDZw8ZcO#TZV`sRP5*^zeJp0cWx-7)&a7lEqe?zRt(;{$18U4J2lB6si7) zG~IIh(@>oYZD9rv6ODazI+ylQWKAuC!5YYtnK93&Xwgg2chhCUwrtz`zjR`Ib;Kl4 zEdnMIv1%YyoulM&=CNo+r7Re(*P*zTIRO&<${dgTMtv&=2o>LT8uJ&evx1wEj5#G; zUk%)-3F`vNfj#iAriPKab6=7#;rAQ|o0p;Xy$8NONDB~fi_%b}h>07ErgQk&AnR6= zKLUA&Z7s-xvT0{YSlBZcze`3pk&RAP8WHXN>0|U0kdI0QAoS+()BUl8UcXL9Vi%Ol z`*ZzVI=SUf%i%Y4g;y>`n8gM@UjcW|aaE+F?5Ctm`G(!cT^=7wyZmza!rDkCuLW5k zl>`qIHDIPGU%mxfh(_&0ka=uw^z{YDoPq))FXs>3yAe7BA|c@myG15$lOk<{qNeFD ztSa?)Q$u5mD!R%M>Vwb$HQZxRn*1Zuo<;7c9}*cGa@eVK2%$EaWwB=$9h&AT6e6}h zQyi9Of<)y+w?}BEqZ)qyJ4LJU~GG7=OM2^zx>8r zH87Eyh&6+3Xt%33)EDid;A}$4Md7>deL9bw@zd_%7HIqZ!!mfoI6kpo`}+92FmxSi z3luZ5k8;h^eyXT1+T|AKaDe_>2XkTi_D4}7TkT&GvDU|y{*fZXjNl85sPAr)EX>TB z>`pMtH%NDWfYflH_(up2%>?Zmv+g!Ay`-Idb;}IjQOos}Ym2JH1}7kY7@MqbxRy~$LcM^| z2t~gZ*eBe?ecP>5@W%h7dm7o#dr@u-EZEdMqg}rEFNS%?Qo5G|s~16wUO1}VnM;k2 z{4b~syr0Tdb(uK%$89=jv)x)ZXx25bDw7eyqL(vw!`a83tInC(W2qyWFErz9;o&$l z?_JaE*MB8ry_#itnPhpro=C6s5czwHlAbl6l78UX%u0fzZAIO5;>ibwN!xIPO3&5n zcqTFFqzAh_(Kn+sBdEFr!qcX#Z}hB4^{PEFOYmpQH^{w*f-+Lqs|c?fbze3vCMmBt z4*fP=m+e5hpg_q`w~x(bvp=#c8EsKxFi4ZBABWG}T$8^oVGHMxO}2tmSFlQlUui^ z#XnpAksm?JKeBv6XG0MM;+F9Uh&X@nbK;>&p%w&%6ilS6!$_BkAHn6-m-JIKcdb4_ z7-!8C25#Hs)Z_l@;XO!8z)QwYP=-p`&2R(l2OJa-B}PsKLra23n;hdG#tlJ9 zQiGvo<&6inK#Y(DDr*D!x~0H`+&5fgsN+0vki>6<%zXArNy*_FP1{_0HvVjV-ayp( z9RR^}roOfx{}s%^3mTdjZ#18dK;jQj;<^}&rZJJ?)@&`iANXD_RaS}3JWON*0f95$ z1`^GLQe&k9gs^7cCEKSkTE5J6Pzi5D8wK#lf%+?N~IHk30K9Ux_^} zL;36bk%=%Y%}~too5`Y-cYw52G3U_Fqf1c44<(J!(3aUPOZbtx6scqcwV9uR)VIyU zj({WOMKcwL*R>PWL=@oa>9vu*0&>wHTC?Fx-=Kefrm#2+a$ur3Hk8O0VFeZ9Mllp* z@z2NcKjIX?0}#Qd{V!h>M1y}) z8eQDO^}m8?`~lf`1)Y&rwCCak`)-Er5Oi$&{SndMHWEd$JwS~sz#!$TnW{IDCHs+i z9ni-E-Ld=ZN&P&DZ!0tY`rQBJi^x!DKqx5J(ZI3%j$r6>=e#xXEr|uFM@PRXkwSfP zn!rhqQQs%MgS7q|XgJrP^mmfM*3d9yARSBq-kCpuivCtX`oafb=zhJ)Pk#l(*-mIq z==#dUENr#E9w!UAlq|7F>VvlUXtpFW*uJ*d)<-MdT!1qIhN2B#?M|1syZuuH@-IGU zXo!)d2E9eeH*%8!2qc`a3`kH3r61fKkQF-77{lHHE)|b16ZEVtD(@jt9!bIL0OX}7 zMxIAOKR@v(08g0|?oYypZSY(`@%Lt?s_+pRKS!vRGO_>nxgUXT>MGCIU~l~<^spcxKDodXSQE}wf( znI&L{o(R>$1JWZs@Kmw>oOisHpV`ph16mdngD{$X*oVR0t#V3{{D+^ra+ce3cF_nGXtehWy3(~OruEpXPI_7a zQWKuHT!aQrAAvFiom$~=8wRd_P0Pq@g4Q}*qd)&kpYfG_s){kdwAxV+m&J>>dfy$x z=L*A5-2&ek>^eJ2z{ex=xjkZ3H@5p5+(&2|wJsjo#`LY|32>&-+tjB;hICvUEE)x4QSR&4 zs=FDYD)szT+y3jlWd(g>E7lI^5kfcmZ1vo1;1EQ66(C&R zSwV(%Mz&eGa{_2(YAAP5a~fDbIT0+TTGymRz$N{H@R|;60nUj$C?H?8`NCEJ(%+37 z!5rnY*B`!Z00<);pa?h;tOHY-I_6Hg)OSyy+C=jKo~cLOA+zq|@KfdIt}j7Xn*mTI zn#@AD%M;H8cZzT^G>Qm&NlIK*;Zdddpk^ zj0X_nvpLMX$w!T!vG*5=G{*pz4NrjaM=}7NFyXxe77YGXs^{Q9GIR$ltchS%`c_+h0_ zKsZVBKgx|$js-ToKLG!O#^sbs~r4g#vSLX18 z^_N3ZWz-C|)D{Y^xCq2mwrko;v(BD_GXGwIja{prV$;9;0WMzsH$cTmV1gD-doH@s zFeVh4t#8+kSTZ!j!#r#BaINJA>NRe<()$agW-N3>6h+Dsl}N;l|ucg~O{j-kN!{|3DVUK`~^jg z-kxMWoW?OX_yF6i1}JNxN&UM@D(^r7mK!-hEhwXs!_)}$e#d_<7{ky<-Z}S9 z#9K7xB40@hH>`D5I>b5P+_XPrz;Ji~TwGkVuqoyB9WOAGj>{xoT*dYQ0tpGi8UEMy zAMrxqlZaPOdpeUP9suPhp{Y#+#zp!%H{GK)a{>JXt$dnwedQFBGpuzUIzi)=;ClTK z`T&p?s=tULV@%R_2s?=#&XJC2dfR~ur@vp?blm4Q6`SEZ1+O0S01)PmhB(+4(Bs*Q zZ+pH?g)rPhkxQq}E?dk^`7im`zl`Hle!&u1ZzKy*cEq6vSsn@$@8uTmSd@f`NQZ!? zvslbC*q$JyUnEK4zDZphdEW@0Gkmu|NxPBf_0hG}Uy}4}1oG2BcdO;H zBS`KVOJ2Rsmaki2Wv}s|#m2yKS8_{Nap^6^ zAS?{rlM$Ag2kmh*?--u0fz|zfcKbRh*l~udQ+Kh?XTfa)5!|2Vy&3x={tzhSqF5G+ z7QYoOd@L&+?wIQa{-^rf?7ChYc)G(LUn{`otyC}9v$ACxGCMk?`L~It*mGL3{uB2Z zI~HD${kX%Gl|`b(r_gEF3M()5kSG2E0{SGtG7>;{7S(rBvS$$2(zE7Cd1es>6<4sz*~!(S{L1 zFYXi-Q{MKjzZ-}p*ZlQ6V@|~aa9;O0lnZx5zB!3vY*d-o95So|eLR!4%UlvbjUtB> z5L*O9{s#*Hpx2&*k!P_&gFYS~1_?j-9dzxIEc5=vY{)<=p9?ORv4)ZC@Ivwj*raPP zaP{AFnE&(@O9pV9+cL2Ja0kbEJPl0V`RIa+2mbCz<*D9B}vN739(yts~TT zKLyEE29a8$;B7z-XYIK$>SFky$t$?o3;O1Y%h1G(ZxsRd7i5J_Q2W}u<$VcVP0z#z zh!g}A>&?JTFM>d*B*WgT&hiN&o30TUUihsn?DjXMe z*<)DcrB>O@&%EFHNzkV+8%L*P+HZ^4?HtDvpV5MU3qm3*_!JvxjV34Q@Y`Y^LZ3!4 zBqQn9O$B$^E9<>Rk3KfC@5cA@c(C+Z`rQgxUL#{EA_1`<8}jdSQQ53) z?1E7eo_LaS3TV`d@oxj#MdH9<((U4!87BT3^0W}KBc1Pp^Bt2t1c@^-37WccWv`v7 zb<7hq{1xmdDV(cqCcDJ5aD-Zpg}1$%q{gTFEzz~lYprKBqeQMZr#Y(KxT3&5xVgh% z8{-3+w{9WMH5{GrrOn!%fBZIjDEiTjFJLH`YdhdLlgp_2#iW>wz4AZfVa z_aH*U&6?^n+du20dXRLQxBDX=L%~-7yh_f%r6RB%pCcVGaRSTxoq#~Ft*dC&*VK+P{w0XiQ)pjok?(KZz{2n1W*0FRqq z{AG}qWI~U9>O9dIxH$f{0>Sm~PotfFbmHHEj_k%Yj#a zvx=a?cKN&2iwVtMF!?YeUI0IkX9#8Ir{Fj{;0ie7nG2SrMwi|I?4G%+)o4+}_tPdJ zpBNz@^^H-X3fYG0ie-0?pUIqT+Q(I-%w zSMW6SOB%MBj!r&-c0}7X#Y$%H5hxkUKVF?|G+KZ1krPN#{|&MneL<&9Ds5Y`K9?fW zmUbl4?_gYeZyPn~Y{EJLGwBFaL!Tv&fk6t|wHi3N%t1x+K5=uh#Qvw}VCuUHIqY0* zJp{kH{}|Z)pH`Z4GIWjA*=uuyHTDXjj6k5P#+#+`1Dg~126WoS4G^HxkLy7g9>j44 zo_vF7?(@!52>>nk0}6PcC)L&`l%MJ|)f25q2UeLnOnir6HYY^iaexQf1+d_Z;s%D$ ztWe_N2a4yqP#?EfW@j}`yKwQl&>|*z)z>IF)eZ-T}Gy1yJ++3m$TSd^u6V6_d#lAZY_JYyNy_?fbbHwa<503Eu}(a$ni zy)g7Opf9u}%<^?0z;}YDPTQ0o!-6KE=4UdV@7dVHX}+i&AV?x4?r-{l=;}b*N#h5d zb$Qt*!6NBripfI3CVkTZY6+<0p-Z2Ke#nGSA5z4nRcSLpzQo;J`moRMV;@i=U|ge@ z;J-zW=weT((AOWoH-LKm9Jl3t6v1 z)gSGwpxDFg-v<1$**yPQ2Ywk`1>M&XAu=3^$jN~AdIASyiwu(QA_L5tZfzt#v|YTN z{80bJAy9QK9UEC!Px!w)O!e>ZN-GMI=s*vAe0&gv8a?k63Nf`y_(+>lATxdA5FJ992%418bFUpt|AlzKChm!$P9i{ zPVost55mzB8o5u4SeQ+?(}QQWSKVH4;}Mh-NdXLlV~(4&iIlgiPKei_mfk(?NHuY6X*==CRtgin6sRCy`?tWSXKn5p>XfKMKXm$c0e%vdd!*XLvDy%gz!zbg+Ryso_}#lgqdK>c=@VhO zzd2R^jD!#u)M0Cnd272@A3cM=KrT3 z+AvLiY?Z5r(T(zG)*~4>jEuIQVU=b((}gnsZ8N5M$deQQvLCbPk z;y6UY_TQ%6kN9vRg6RKu7YVv3!q=nwDY*)@dnv}Rp8QXT_J9BSSqeDD&+5;#-~UYn z_~)ke`{No9^#z^8|8B#8Y!#9Tu0;$p^`g!HBozPKXz-7>`W^^=vj6*K{_n~B-!l7O z?3Vve_Dn%9g17!LfCX2Jd0$;6x$C17q=V`?8{2XL!qfq6nf&`dp`IekR#yjaT>7vg z62Eix*Tg)ytbQELyOEcYgmVsV>$r+oU$DxEiC#=&sr)jOFjp$@X0$4-s72XA;QF@t z)r1Z6WJ*N^6$oRI5guj~vSNew8mjngka6Zw1iv0ONYJrqf}+ zw10;A$7nyxgzsCuCML(0bb2!iPoNy{LZS>uApUDDj3Szkv3Ux2Ww$^uxP3X)dk)QR%NTZc(Qe%r|rF zt$m}cNwEo<2MkK7y!!dI&VWf#>%FHhcXE@O-TJZro=d7Fl~eSkHp6hfVj7ce)D=XR zwN^3mA>&ug|HIgqz(d)-`sOPv5(0*A*mGEvkcj?8|#d9 zCP|hF*=8{IZ7|H(hr#$y=YQVw{?7Tm?|aVY<1-$^JkRaE?rZseuj{_)=N|uV*Z+Ly(XBA~%tMVIY6lI$ z8#^3(dj&7d+PhO+g~l<2t5z#bVeZZQ_p1D&Uzgn%C{W?&p#*>R&mU6iDv$Ey;Wfke zyocdC)~tSXh#4U9;?B7NvThR(T8+Z;J}Ca)PeJLdSk76)NpWK*ZkQrS_27&-r{HdhQc}r zPoq7mbh7xXbSC+?byIBZsvaR5M@&uqM$e`|XG40~m1~UZT(4U6L5DPPUiq2I-=t0* zh+mSRQW?oA5%D2yl6-S7R+&p+M(V^^o&8{)Q(ANfb#uad%Lz-A;60O3*s!URzlWKi z48T$A^E5@j^WE8jf8w#jlr*M3+!E+eXRSkio@vT22aX5sIoj);P9a7BUMTEXYATKZ z4^1>pir&g56ojg2zP44lnx_y&2)?1%u(K!*9F4-A@F*&PoHN~sI&IU-`a_-SVf*&h zv;~^ip1ls*+aJC(z8GhBvt)CX4H})nkeYpb;phxTCqxI z^2OGFDpJs3VSll=4xYno3Y?xleSH;vLf-pbfox#;42OXNd6rF`Y3yuk|6cxMPxMbu zr0;t2!J7SzP)+5$@unSxP25c{pBjzPTF+p=S zW-;;Ao{C>6rxPA`$@XdAy3|RZpdUKKWd4xn$F=XpCc+5gOl;@#r`i7mFs9_`n7PE0 zgJ1zyU!5qAISLMIcskh-w5YTB$>Kcv@$`6f&pb*;9>*Qx2*d(O%LQ?|(R;Dv)nRp?saB_()7-jc;`jTIbd`79~Y z^GhZ}1+tzg^C_RHhcE$Hn@lY$)3Y;v$htqb3Ybgpl+N4aHc9v>Zfi4Y3Kl5S0|DI~ zvvE5=Ff*lqzjr4Z%~#i^wha{BT*bCT$4wKNqo5Z{6c9JzK$^dBmwv)t|7_Q8er8`E zJ}~xN<4IbBUDS4-jsZy3#vMwUH>n>ZeVwRsPk-T(5X=O)l@p7C0R5B69L9b@SJb{0Wy7bO_*y05JyI~nRv*}2TX6%>EHtordJTfv7 zpbnL^3z4#UZjdF+r;P9|@2HGXzz7qoc69rk1KR)aR$oHpTFW+@7x&wWt!+)Q(zZHs zqaHRR!BOgUpqS08%Wr*Vax$(l#?tD%CC-%|A@SbI_i?P~$oMGceQm}M30cVpbITRD zbGUlKqd^sN2hT>QvRz#h4&*N3=2@5am1X}m&X3LFfn_)|UZTnd0fiA~&=*RZ55C&~ z>V0Zmo8PYlJyUOie^ik_VA)&j?!2n_SZ}BNtpxFDw<2*GD@)`^mERA-#x{87ZI5|@ zMn02AdsaiTRSq}VZBqp&d&`O;Feo;qHfiV(tA44@?)64+tol)3EN5euWBTO?@d0$z zhiY*%-j2602r(347p(FSdEk&n;IY|4z(3k66 zpkQM{FRL(r|3PXI8v5khrA!|4z-vmCC$Ro%h2URD16zGEy)_~f9M94Bpz>vyiiNBW zlQ6Myhly4})FsL7IKI6Gv>C2hrB+(*qDnnBr;|=5H0&roEy(HRbD+~V7~9UV;uq?y z?z&LQPtDzE;+Nx&{-YN2FTCh4g;VbvRs%ZcjWsEa?qdZ}ev>ai%0K<0YoBKy1XCu= zHu|BkF3wVKmX@!FFLNm!U+pQJ#-ZwRTLA@mufP3B+4JR80yn`__A*t1j$D0uKCADi zjEB~HXM2SkjyNQ!?<8&y@l^mmomrddm`r>jfRfenS9EZ@6pA(vIN0IxlNhm!_*@9! z_}HtqR&11$;oW*JAUO`&7-@4FLLsI|`7VR|U6SS_{NjmUA66zYd&7C^iu{IT!i_gQiDeD!ySIAuw`qAqx_ z9^}T#B)5DY!=w+MWp2&)Bq#LpEzP7TudEM9-lQeFo_IEs+v<0OvsSWEhvTs;?57P^ z$mAc@0v$%=Cf?JYi~to1j?@>E{rRq)cyeZBEl_d3rwS-064llwR}yqn=+W>Y{X6e5 zJT+((N+dh)J>6g)^-GCBBwuLos~GUucx-woa$lxJzeLW2{l?iQP@|*0z2neiwli{e zBTnC&zx&jo2vPo}SIKgSz=ZW>oqgv~r%}BI!5`A(dG)9`r?EXK>zHnGf4+1-e4i1f z-|uKsOq~DZ9ELP}aF6wpFzldwSi5eOjLXUU)@&XK+O{0pdevZgCpr3`5|Z3kj^9v( zf-23(hXzDEpCf`fpA@Nm05GKuh(UMZT*QwqnpIookmw^kdawDI$0S3$^|s=)SgCh& zd|ls`9v{HsAlI6VXmlI|_Rc4@#xj?zMaLqwo`*F0*{9u#SoPG4 zU)jY5VKyq1{d4Or6a z=`#NNr6PAXKiKJpo$D32*x1}y=`0BLFd_j<|XCwlBwWA>g(`^b+o zW99NL{dbcMVRS+kJHmR>ot)%+oj4{Qv9gI<09Vubi2QAk)0t?K47{V|^jn`ym+ll@ zYEaAEsZwEqB*9f=3uaui&-QKV%UQ;+(&H#{>xmvwP_RuP>cC|u0ils*eCdBFzqv|R zA^tiE%5<`LmgBU4bNuQry?Y&I%U{OnJth$2_C`xPQ6~9i4L93m&5J_TT$8@7Y_cJf zg^9sB@G+dA5JLUiFy;*l*ZkeZsY;*0`XGks$`d7xjD8OT+0CV? zn=c9ZM^AD0k2Dc+oDwN;z1oJzZv*tik?gs9hr)&E8L;iH!kWj;*%HMb`>*GYhFin& zz)53lm$NdvxlH=k3r99)@6VUMWo>C#WTe(YO0AruXFaQ(6G{h4V&r;?Gr@L>x=*gV zh0k+LoqP5nv!p@pCE`?FK&G-=7&D#cvDnG=@D;G(i}S1;K(LGDqviA(yUH^60`jrC z5WAWwMiY->Fnz6IEs|b-P>11=<+R=@A^N?9a0=hBvP1p5$tQ+IHDeaF!2EfQkm@GJ z{e!7{6IqKDJwI$5hic9IGVdZYx5^p;R=YIopP%9oy`k#WYShD57fBin2(Snl@A1Fr zUM)JA8_|9UuW2EPv<|e~c3N3h;TQikFaJYrt3gF}W&Vlb1>)>Jv?ig!&r>wLSUW++ zut&O#G?88u^=2}YkJNmiGqugI$Eu>VvYtQNwfao;tDp<`#p$pDjt65ezxwcbj3~qU zI{CDZn#nW5RSu7bRO!f}FWx!7xkD>ubvMDh{d;w$?GFCL_gKG-*Zysa*7fYOwu4?WhV$i=739GPs}sE8RF*NHrg37_`tpn$&cM_m2K>tXdu4GQkAdPEI1ox zNbgn?T4Gv_+-QogyM5Ae2c*(xX-3#{^QhQljZ)hSKQd02e{qt6{>XK3>PkW z|AE!x@kJcTar!{(%KYBT-2H`05z~{_Vi=+o30^5_BPDd74ucO8K-qh(ke)~-0;tjh z>XT=8Hi3IQcgoBe|DHmj<&*IApI>a{e;iUwtaUomWT`(!oe75suX?>5DrSyo3C9p=DutI1|icwBh5%7dO8QGRO|JcZCaC7j8_q<_- z)r5It1Af76G~0-ySw+d(pz)O3Y;36A054u{9O;y4vX|UwY>{*ALE@#J3kkY7!8HH; zB@;6qGfPRxH%(vYF`wa)Ao|l9S+@!EDJE`PlNYdo)9nJ^%2Uqy&wa!n+}OZFR@n_R za9TI&dmHv{?2$u+(ScTFjsXK8;8{#;zZK`Q$%jSjT`k$jJ~B9Z=Kqn(x~%#=8G1^8Szp~o zdyMOarH)wb`h)1?O5;W{M&}k)P!6K%t1h8}dJ^Cf&Lp?%R`^?Qffks5aKoQ%+06^P z%?A+~)5<%<&2yhf9&noMvZyUI(hMRI{mTCCz#zPeU!Mrn4v_)fYzHfIIH1gv_$Ud-H5qg)hFpWcl}wc z>K6H_s9PJbR;mG#TH4isWiuMzFD!svj$J@R;sg1x71V$XjY>N3S0Sq{X0a}Bez(v2 z#O2C+k{4+WI7h$tibZb~=Y9|q;j;b=DUwWnC^w*UrmYkaq?H&at!hSc=9y#&B7ZPT zksOZ}YcsX*Wk5Gz;1)@Z{r{m#v}#yxdiJCp6}T&%mdu0XLyR|ch#Kg#!X`lZJ~zT>3*v-v3VT@TjMZ($XGDw2=Xg(@ z#&OwoE@nl|^ILzmyLI4)$gClq$9;Qv^eoxh@B7hQi9s~=tic22{vyM7yjd9XJwtu2zc1-vbDANi^NbDScnV7(n9N342~D0a&jX4{Xk*Azbtieej0M7y zLauaHsbA>I9o+L&%esOtL~ki{Rt2wKP56aOKL>%ezLK#!z^6fw!=XTxfrE`iS1@y9 z_sfUwSm3PCX!An7JFQ6#M%lBYzv+ru4CtRWu7pu^pVY(4?r3GnIhrmGvwn6!txOL% zOgFXpiw?8aFUFa>xiGH*?|Ne@eU%%Hy`sZ5cN+9p21yXFr_YP>EhST!6MeN0><#KS zUg5?R>0kQHl#AU{M0{-qk3XU;aZiaAMh0aVpO=?x*6nSfoF&#m_V&0^2Eo!?91qXG zvoFQ7rl5LRAez{CJ~6LJ54w~V1eI__k+g%#vqfO~3u7b_RYFzq-)|&--pX3G=&{ey zx)RB2jj&(!OIXjHi~};hz98c-u_^~lPt;Wh2FecoXzQ9cFu#5H3%izL94t1vsQTHa zKu#y2Xvk~mG0|rf-tTjF*2$YKM(#`~u1ye4HNKHW!y<9MC;4*8{dVJ)cylt9#+R`) z8U8eM+=l4aYf3iZ?^@`lTv@x0-jJXjr9Rc0tkv?p9J%iNUJK&S~5A*_vy< zcJG1M5YcqH8zjQ1l3RI(d9Kz;sN*&SCu<&mf*&{`a8$&Q!k=D##as+|MQ^%Dh@~D= z%JcWU3#}r>n+|B4y@ammls;_zrDhHDU+q&Q!6HR5(VahwCL<*ggo`9@HIDmi$*zy{ z$98Ges|=VnAyqjmP5s7&N1&(3Qm7BEt=Jv1Lp7Tx?c0iht#_PnJ|Zy|3zlc;C+~Or z@!2NW4chphZ{Z#^x7)4K)@LqDkE!!Ir1mfiT-W(>uT{p)wkZ=(LN~@`-mmU;XjBOi zKvL^zXO<_}cDSl@+?>9x2s9L5^64*mD<^4OUMN**K0Z^v$L^csQv0EmOfZi(FQ`b%?gu*yRJv zU5^9pQC2K!w@&pVw|XNSzKJD_Z7`VQc-<;vgYsOZbla0~ovwt^46|}U;pG`=2}>ep zSu&^i;gj&U4D7ahKUwF?@It4p@#*fhgnMw@6NHJU0t@_I>BWDXbags&7>DTu{D+MA zr-Fy6B4XgxL<`3>zyzw31m;9pSTdpFfu%jue!42<27G(==k}nh#_R^S4Y%Qih zQSW;p$ojhXK(eaNi^&fmgGr%9yT8GeupQ{5H8thR#>l=ML(5}w_uwc@@~s_h2M~SN zF73$C*W#VD&N@N|*U!}`V)UX+LOEwrWP$15=k0U?hem!xb}`7WB1@~i%`Eb*sz4nn zm^zwsG33A3%t+(0$I@cimnxQWf z>D091vxcDq922rgfwez(>$2C(Ld-dKefOMOPVi)lIpjMy8QbhN+2oWV@MSR6Cg@9B z5|~EnB&YC~q+QwlC_ASQYDEa2|i85LjZ4 zambexMNV)D7%(sUlOWpYB`iNDuUW&1Xpr4Pw|4(|=KBxJ3Ug@%Q3=rDs0|y= zlp;{|p^3avP+a;#UoC`;zI;*qAd`2eikUs*Af$i>Y->anZJWGwI_@ANEH8h#@Id?} z+Q%+{)r^e1yMARdjiZiWQ&p(za#~N%PA$;B>ZinX)dZUWB6hpK`cbA+pHS#t)qQ{t z&whZ=2*9*lpOU`mc*4YI8etl9cl^~pYL&LMixgx{h=#PlM6r02KchyXYQ;cjv&M^u zymZBF%UVkaqA&!tRDe_(&3K?Z0nAq|lG*f}#zH&aTwZ;qylUnyMuSkO(&oP9w(#o> zre>Kgk*6EW1K%%I7+R&OhGa6+x%sNFPnrB63s!k&fgDp!*WCuVbQ{$&3pSe6;N0kc zqLgl&T6ys`S!(y2c6h@bCYF>*9$=$~S9v^CzvNXCty=mU)ZHH;Mfp|SnPCM_iLK8c9f`)22WZET&7)@iPsgxUWxd+0{P>v6 zp?r%6@MToos@$Rktxak}fQ91U$L3*_2Tus&^2szVStMbiq-vPoe`vNp;KK#M;zu_M zGNGnSV$&16@{gzZk;B|@gGutc-4nYdepa`GIq2JRtR?VtyGDy#V!s6;8rD%NI zMGLn>$$p=D$7d0DlYTt5t&Dr7AG*FSpIz;?Wq#eSDcGTGuPCeglnt3JlZX1cp^2>k zZx@S$`2A9_aElnC$HaO|*_M3^S-%kYD6{F-PV1Bu=7!oqMwrLp%#=eIG;%cMpf)1T zKx94u1B?d+G87l{U;PF=51e&w((M`wjg+GUPr_N(-3$yq9vY0)i%0(|oZl21y|emgMZW3O$^y|cD<;h7>tNL-EAr1)Rc0E3 z5^A>F76;Z_Gq-HqNRCC9o;QWbb0&Bu=ow8LEfX3Er;;S~5e$5TQVzuI^h|5`xUZ8$q` z+>v=Oc1y_4I^Y|y%By9j37v)9jBMKZ`D#jKxqiNfhrZo&>DB4Ud}U1Ihw@Ec`j{2> zdi3jTY1rauz4=koC*yu0? z=ndKRvCtLQfh~4_(kx3q<&s@*TtGD{g_h9}3CKEq+RQ;X+Q|~PgO2psr3`p# zwp;L*{>qOiZJzdI=ln=~WhTUw86ot7y4UNeJb9MpYeiTp{Y%-I95kLJJVcc%B#HMI z)}8&m8U8Q7KHHvO><_s<^7KfQK~^+Ow%8N3^|<~3py{v$1vK0y?kz9#yaSl8-d^`= zc`6@;8Id&J3o#fIsd>Jq7u$SIJ`)5BmJWFSIWJP`I3BRKdAY#6qQ?I*xDlvq%tBNz zlpN_D-RIszRg=)wF|P^O-*l-=uf!I&s#$Fw4U*ly>bUZxY#bfK!&}-;92+Xm-`;i_ z<1^p6Is)_?-;m3^I-B0>q=0!6*?Fe7P)qGoYCq4rgaQ&DfQxTW-QwWTvm71ADh|W$ zYML~AvnEeKv5T*4gq_hP21r$NC*D;GjWS2`rF=Lu`u!VihiD+bviVQG-B`WQctqCG z{qnlBo47?mTwM70{=#5^Fz9$^n@?_1pMgUB798l?HBv@`7bGs zJXRJ5!20<++!d^zn1iEgFqfAP$H<9xyRq@{4|V6w-h8B|qZPC$&?A&Hj=a4Xd5KQ2d3OsUJWZan20u;P>Es4yJ>b)`vks9h`20Fd|MY0JwKHH|O9v7R% zJ^%hJ=8|ZS|J(e@^(I8+mL|*Wp0vVf(D`1xNSd18rC*Sr%@3Y`EGQJ?w z^`~~i_+$$uXY{}Yt+{o1!-j9a_Eg}bMl~crlcDSrga1un5NF+6>pP>TjSDO6x>6>E znV9YPc&I-f{a((_ z>0TM`Nrq_`@SXxfe&GA)d2}WGgR?qla9Ucv84$3E_x5R`AQI&5_?LV^+F9Kbk2eDk z@PJU#3U-9fXDM+s^JR_dquBb{`j(zVoPT=g@~!6bmCFL^s$Jtv!97a#q;NbmgpZ=A$eCIwFUq{J?*#4da~*v>y5arEOyS0_X1qH zof~ZrCKmdA`stgugFJArZPZ%SLs~SmT&&Xk3dL$4P{N0peCbrr`Z1dK4 zuJq4i@UniPDNAURi5yhB8>M+`emrX^`BFgU5b*{!L5QyMJ#Xy%uuT0$Zmna->CbBX z_m>VoItrstcPwSj38AcLd9C>v%S)D~!-LOJ<7>NZ{hRN0)UN%it%~`CDRMtNfd}?@ zu54Y0+@5^d1m3FQM}^FKSlkSWWDj0D*#nxar8SWrS~mHmRq)-X6nL$_*Z;YtH)+v^ z4@9){o145w8oxoU|F9jp$b5{drG10K$X%;1!v3picBJLD?+^m@xu|Aw^ealyiTkPN zRFfuZ!Kd%xxZxl({I)NX-DXASV`8f0q%upRs=m|N7rQpT(1rX{R?i<=Ugf@9rFB9~ zDbhzUALuz5x16Rvelnb{mL0A7796yWY@eXE%_;$;tM6?!BBOhohwz?QOtX%WWUuu# zCIXm3-^0u$c7qcC3Q|vq6r0b$$R5foUmH2h!gtl8bqM?)3{D%Wss!9(V5zbRcUFe2kPt!-kD(hQ)eu7l3 z2#f&LE$?XIBVm0P`UNGgl$Nze6B(O~SGg91pP;9^E=u`k9ql0kVRI~ajCW1l5O!#K z<-!;yE_ilz(f1zIDLd+=-tGS`zTeW6dij!ii(1L%DYw_-)ajn!7SwjYa{NT?sLA2A zx|6LkmvP0~nptsQs3|!uHDXkZcA9^&@HOgwwYz1rL|cagOQKnkk-n1MprO(Zn@TMq zQXT}l_ua+A1Nu$nk;`ah=9!D5QdAU+KP+_lqg0I3E`V!>UIDWy4%n%Lvtxd-8ae4-U_-8FWI$OV5fpeSNQVwzsent zSlzkI@_lme;DQvT*=|0?(@DX|9v8wr1skb|-wcnK(1Ws0Bx_((C*O-Bw8&tdVN2If zO_)}@i^6GcIIAtizP>q%RKyWS3*(MpgNZD_I|_-s%?oyV!w_=&j;|}ss-dk2bkM;h z+}^bCj@9^Cz3{pRV1hi1L%pyrP?J94souLsrBa60V=DF6Np@TN*>|?3`kVAZHpOe0 zF1in-2ooWjN(rRkchYV9`s^>sfcb9c9byVRt~{ki|J+NmT2waj=yV#sn=|Cg#sV|% zm(z%5=Y2j>5D!qjRI4fUq+8JR=!h?__l26l=<}!0+Noc+3iiUEE7tZe9Nd$dfo6?= zU0cr;4rYC;eRL10+!;{$U9MF92!Lu#u>q-VM_zeYRIwJDB(;+(LS(*M{gY(P7YB3# z-tSh#OuE30H2GKX2ey3b?r9lHNe5C1MC|1HPFvw1#U7`(TD_l-f7kPDT7t4Es+`2T zAdxpPZM$|I)}6qt{GJ)1w0*Y2kj@IhnIwSpyx1pe^9y{-2(B%7&+*@EgvPP3`}BT! zHTp&6xAnW(O0EscUM$Yd(=!w2*D+E9CESA%6T02tin*`Eu%g0_qA6!C`fV!ys?NDK zD}&k#Yku1!GwìWnDd(^j_sX5l>9INU#^(KK$4aIaZ=mZn|DfAun10%AcWCMYP z#r;@~_nBJWY8tg;88((ZZpRmSQKv6%o7-!!s7H3!5c6WmxHXJ&PQS77{7&cVc`L!# zRNF@Vfnt$+nnF-~4Ts3%Urt+1K&yg4EMBu<9Q!#zA`q&?ZBZ~&{_NgaHw~HL4y_Q@ ztr4YwnMCOjgua`=!@ifX3*(qeXJkTNk}tg>|27*Fgb(xl1&*kukU$MYn?dd01UgYoh7}j;-=1K&G$b;h*Ot8o}-spGcI~QQU}(GD57Ww9I5}* z_Q{!XeD@|)%P2?Hbn{}6?+zbiNa`?E7&v-~h)~Qw{*pb2keVmV>PXL$>+3+!Zl9Ga<1+b^Fy6ptgGM;lw&T z8%+r@?wzxW$$PbbSWra&O=KF52a2VI64uv|h`yaxTfI{Ch}_C*{7&eP4V9($m*S|b zZ?-<&rxb|Xcb~F`J$4c}wKMZkwgXT8_9zr3GTsV`$;;Xqqh=}1G+Q3N>q$9MYvlKU zxF2d{8ENvbSe}Pp@X2%vPru709it(*hrUKTc(wdv=3hive@0#wdS9EE z=Zs7NV~f>9RF1UZ0ku^HT;202juTK)FWhmAS@`GT!P4wPgKA&gQCYViWu)xphDzhO zce%V!nAd+uLFhT_y??j%e>(6Pp-0)4mxM#Y_(ICVydPHfZO3B_OCCE4H%FF}ZWXa{ zQjvE;cemt)FGQ_sq-TB*)YeCxY8p?~Qgccl6iTD}k^F^^pp$)7#OY?`e)Y0&%Vuey zQ+>G%`>%8J>cIlR4T7q-a(f2eXT3i6KZ1sT{W0tfZBdK|-PgXfM+6YPA=EmJEj>Lo z>FfJ6!!|cqWJC;YeAIkf>(YjXdSjZSdcpgQC12o(cb7lIz)5CSm`&K`j+0%)yXdG_zgcWKO8r!R-hKR`>D5~eC}tG(P#$t{>e$K_QqWI z=#?QGrhSKz6;SzYT9}rn6V*Tr^tbF71uH3!>y*pdFx>f%CpJs~OfDS%qZhzw{n!5% zIMYO`F-(`=#g$-q;T<~ed#+)HSE$Hixbtwk;VmHjOo+kExNTFpQtpnJEmm9klmlm2 zwCC#6d!NA{C);~du0zU_Uu`Qxi@IX@6t6_STS)1@pnHGmU0c3&l3~s@*f#i{{|({T zth0xTpt)bSL@cX5`k`t*IQML>nEPbNT_0mc&D93@JdN_6HZ9<%v#^}n_r~A96HeM% zrNzIVsOuJ#hI)m+u-N}Pc~bFTKP??Pd{^y$Ml|gCWcB$teN~ah=W`2Ta7QD4NuGcG z+dr4rA6H=yu4~Ya2^*JGN{^#=C;H-qeV|rMYX7t*XgBN#rlaqbo>a|kzjXhra{pLk z25keX*&p}(yC+^g6E^mAFq?*9=+CXis9_!d8qNHvXQv$W!~EDX)m0oU7ZP)@r|2)- z5Bvhz7jYNqs)nvU%o((c=04!yqW9&#D;MdG`FJ690F_{&<`=X-;3$ z@1*=62GMUulLjTwSMGmNGuo}f&fK6)+KCB{hFAanf&X*He*g9(lCFM$?3cmvS3(YZ zf9>XaiMqqqK1z3fbbf9c?D*G&WK{Sy`-C>{ zn{cfSCzp>u58kWN6e1ZZZ_WGrIne*m1OEo=a@l*dxxYo{qwp8gsqumiFtPVmkf?t3 zu$&|=(FkSV`V6q+2Z$|{C2-pG#!=<{m@EPO$~&t{&J;g$e~%IN*aayQ9&I*(6npI$ z?mIz~eC9FF^xSl^G9?N4=c z7+9v2soSfHSk>p1ermMkJV4%I6;Qc4v}~4mvdmRCRrfkwrh#t|09tzwBR(pq{UDU2 z^srcWmUealrj`dZx)}*6O=J;+ceR8>r4m48m;bDvTtC&YgM_J zYz+4J7KkUXpb<*@Zt`WK^Tml(50zHM)0;6P5QX(Z<02bdSJ|SgNv_Et!V9h)qRM4o zAwUw%+s)6tO)x}Sy0{EAXFDI0Gk;rS0O#vIM6QvOs$8c}ikKF^8FV$a=y5SZ6ZO}N zgyO9A-kt1me>7VgEsrJw_w&5g!wq3tAQy*w4)~7LYi(DQy7$g+^193Qw!EDc&bzZ+ z)nA2Eq=l2&93Xkvh$JD|J^4ckT!(caksIr5uK2ljB6#!(RuBVt5!EAvEvUO&*zt z;h~d@(oG?o{33>>pA+nxZxhl)ic5wJH{FS&4cOsKKV*uZL-6*=3r2%OEO9kJ9tEr3 zPpax!e@L6wP?^5hi2a%Re~I`{yP)yuV}|zZ66!>DM$u4Ont^8j;m!>hc@sex?qL z%MZ|dsvnT=D+VrpYJA9r4DroTkzZC5v2A$M>)&`@Cu`*vw|MCM>~u~`py$#Px=#56 z`r8^6zBXEXd7pOOu374~%!B|l zUMZ*9yHUxo&ipvL_1IPEP|F)MWUJWHtF2yV(Pb(kg5Pl$r*Rxg*K%t4`0}kQ?jJAc zr0KdsRtcwvA6?t10Ly#VvoesDrXJ>FXnbixrNG0utJhB}-nLiO(3Z!2t&cfa4b*W{ z1Z~u8ZhWPY%JaX%T5m_-r*}G+hEtD%cfOH(46o0cRzJOtSK0UHOQZo4B`gfF(?3F17mRtgv1UcQUJNXg9qb#g-7DBDLu+ce?3l-YR4|^`;`J&uf5gJMy zuF@8QkW+Y6Ae&(T=zd93$xzAVZA*-ot%ms)UKc-c=xqD#my=^h#Nk1sV1Yg~{%LOv zhz>u*W9)Np-|eX8JE6fSW6(2VXL=8(=5r4c<*R_!B6dC3e>0U0O}XA zsrnGmX%Nq4l4@SWnT7&HwM{{OD8ACu@Vza9YXrX>^SohMw}%5utNN%3jq4(i zAy+m-t%PRPGOvz5_-L(t*X`Fpi~@?BRMK5D9tow4pM?9J%T(Jz4-V=$1d4J=gcfpm+z=bB2t+j?euW9h)6+tMJa-rC>E0Bm-|)0mcWBBqW| zpr08Jv*ja|rWBdq*~*Wcag4t=PC(LfCl{c_w&JaH$cBh&g96`4WB{VtU0a#(ZlP%>nT8dW!|s#d<|(jUD*^dOCBP(c zvj>)+tS2pHK6>cM(6nm%s0LeTv_>!{z~^oO*oIRW{L&XBvFDa71JhHuX@mvVH?0rP z#ciZ4ONA}8F-dUK^9Oqh$a{GpY(l{RCzO6SF+`s>5!3-VIDsQ<^Mx&fY~7NqInD;% zyId<3WN4@xao#cd9rDzS1m#)#t$+D{Si$4>GNW?u^8H2h8+PW}qoU{a9ARp!V~S;NU4haUwvkXq=n_H} zQm3z6mLFP@7Eeq6IE_%XPr$gn>>R!Xvzhwc-DG9|HQ_d@Qgc+{F1H5*Du&9q)KuHB z?bLVE_X7JK_fBYJdtjIPfFnp3L}#0=scVLj&S@^d4MsA5{E-^vgWn?VL65+oe(~To zU!O;8^mesv^s-3W4dA^1xAg_**ab^Nnc(M|2|J*>n@mhlRX0;?`2;_&Es=b@qyw;HM< zQlB>^496|u4D#Y5@e6>UgN#l+$HAb4y9Is*Wp_vY3wP*jjP6yi4ms^y`Ts~r1H(x}jI|_y zS=4`Xs{etA9WEY+>MHlWpMdAKM1X$A{M^wW=PNA-H!c)oa_W;d=gEAa;Ql!UHpJo^ zTngzDQNPxYMT19yPVK2jFSJc?t72U4ekbr)HJ@w)NiV+C zbj!6dRJT-;?&z2Nu+CiD@dus!9)0H)(;zn5_6B^?R@-EaxlJq;f?j#ECHc?;9wADY0_?fyiTU(`{?dgkSbcK~x zVuBoe^Y*cdK8uB^;YTm4>)n$Ksf`ik1#KWkRSRDK#PUaD0moZnzOxO@8Q2{V>3g>K zR|N#vKwoMR+gOQ__^U7Yb#QR#eKqNHkRbM+&41daj$OKG;ik@A>>aj0T*JTma;XXp z8kEkMD3fp5R7^;rUs)<73)kahN#M^_!*e(6Hn6T)HcBkyu_AN^@Pb=1h@1qo9JRoF zZa$J3S32C1k*>m^oG1LOEd5m5C6%(Mz+tGIIgCIJqimQYMAQsGJ~^?yT{J2uU7)m#ny&RaMEc`Jd1Eh@i)8|2Yv-Q8Y&7r+3!k zUo5AOOAHk20UbYA9|;n55|MMw7Q|6;atWT^grVi+bwdw#08WtTa+W;*gyNp6BS;LN zsw8ZJociFBtX@XkccG65KO*e9`dZsFit%h71pfX?2(y#L=4aIQn%T1xRnZTQQB%_= zz#DHHlX}HpY&sh7siOI6o1ail2jFVX6>^a0k~7x_?&zp^4cE)Dz*D4hUs{1Y7kTAI z9-!3v9z6E&O()zdE7|+m9;z@>lzX)Ibl7vF52gLC8&HPxcOSXNV=6eGCy8h3ARsla z8)wlCu{l}EV9yuYgp5ot@l2+u|L9Fg5RU(1jy<=(!*ELV$H{J4R8ZLulZ@wngGYj! z?-v(?Si*auRmS|N83|$aX4Az0$Do4D2~rA?Unt!a=!*eylb3|nsN*DmHb}%gtXt} z0ok0dC`3QqvgiJG(k2ES!lgo*?du-TfT4-YBIxI^q=~_=xfH)I$RWb~S*pOu!u$t(s#frF+?AucnTI0gf~bHAeT8<{2#R6L z$9+w8JK|ea=Jh%c17DYh;X)gr%UF4g_@#gVM}IUR}DbY=>V!~R)@ZK zO2YtEW4Qf*q9^@gi#6n({F6s9_xj;SmDA-;LIf|^+ra+>iysx~>5kmgeYc>tSO3Lg z|KaT%q{2K-SfOXL#zeQ%%)p!RG?ca=N&<&)1uPR9Gpy%ZB+CL)PWsGG0?u|bdK66c z*xk(9s#~$%!x6CeV|W#VS(`~oSNgN1}3%IcHVhBPG01ARL6YbqsYDo1e+ch3}*qruVE|w>RXu8OEoF6qAV?GgZD;Ehr z>6@ecJ`x0YrhM-=a}LcPsc|olu{WavCuP0L0p}N2=roT&saAF{3zTlMUW36Xa)ON| z2cZay`;D#*ck`cTe`ud=sD(bkpN8AS_MKk|66lepe!OLgf(&`@x&bz~_o7n}>eenupNW%^?FU({P%_Y|gq>FnjP{An?tj z%TcMUOMZ3j)*GWL^%bpkUL}mB&Z@Go;p%x-=e+R zCshC4AF}?f%_DjmCqCo43*DgKth8<1yrhi-<|n3ej0)V=4v1VZzk4uGi|SVTT;FtS ztYb#sbao83s7W3+;n8FCmBrO9L=|?Y*o1M(>^TiEdnDY^gs4fNh3$G-oD(@&`t&4-bos7NSy`#18r9k7B7B2GDd+wht z^r!%4Q}PBT@HdaD0h}*zzh+r4r3VXXl&OX33S!T%y^x*>O~2#_@$$?mOtKul?ERV2 zu)3ISU)AbYkSu@k(28g)alH9M6Ui z-6aq3BYe@ePC^VJv>ogtIbp?bdlo9X^9+8aScndZ5`I=a>9zRUlo2nTLPB4ihVv0* z+g$n9p0y5lOjfpPgH%O8L_n$1L8Le79TcP+Kze{sL_|?~laiqH-g^f{IspO%5;}xX zLkY#u@@>y=X5KUN`p(REUHgw@XYZY-tYp^vkk?0MrsjS5q)L}4{SBCDi{+e8DGaJ2mp#WzN5dog$4iY=^Q2ebpFUTJ z;(2KybV>hCeWL81OT2^_7v>+3`$MLqex|gt_xtZ1=Q(a@-oMTiB;d@W}cdv?#HaNbNGRN&)#fG=o_uo7QGe$UKIV; z|7ZJ1c})_&Ao{uQH|$bHxNKLzfI{oNe}czkO)dctHl;r57tY^t6M%2S;`uBL+?KmJ zBo3Y9XmT|hloIconCve1@c12U#jz#%V+PK89`fHQ_ zF?MkG$EMTLcoP5RR&B@MspbD+ml-9odT4J>3Ajsb&!&`<7>NpGBd2Ne`YXQM%NZ%0DJQP+P_%>CZOi zn_U0oJif9(D^(81Q8ZaGWj@p36qhEHIpmJ%=g{w0yM47lI5vBozW(?>4PvQmA_Z)H zWigCQW{ZMZJmnrz1}%Xgn+vDq<4f{xlsl?v?jViE*CB1_MK+DNtaGT7b_5l>xg&&Z zzh>wf%lKnH>Qp^{jV-vJIyY~mjDtuIR z9a&&xXwkGN*ptfRViYEWC(OP)>QH$xHt;E7&}DkIX+yVr^{C~$xq$VK-l4z&7cLH5 z7?70cDIta)bP^7lt1tXU^1rQ~zqyYl6f3fIZ)|1l7Uo0TPL_76BxynH(O*d`^tPM4 z8D||Lgzt@geN9c+Db;46kV+ej=hfD`B#wnt*y7#tO|!qo^BQY+fVx0;--K=iMBVuU zgyWZv-`!}d_j^WX-uWMipG6#YQDd+E*w+@@OHu|lU z&%L7Da|x^$DnyL0{&?Hk5|^x&o4(2SI)GFlR95k9hFia?%KxbQY0nF?dpK09c?mej zF6Uu(+@Pg!z*p{};W(CK6nkU{Yb8i5sc1)!Fu6FNrV~@qw|d%CSR81gz3^ z8s9y4kkU$fd|Ro%?d!EbKF#7)^;#MBz^$~$;Eu~_JOw%Q*}q853ia(KT?BsG<>l5@ z$y^h6Zo`TvIla^tg>F*$ru8v?40vRJw4F@gGml$#rUeOw+#wCKccH0!JynPiwA1AS zFfp+7Q9Z0s0$B_0dKWaNO3T0HY}pLXZ`?4yl*{aL>Y7-uG=DGK!fuELS~({3uIbJu z^yFycJ_jkBm+BwX&tG@{$aDD|h%-6m^}R2{ zW^#%fx{gTgTvZ78U^xmJ{3LjdMYwXKM9YXHj^5wpj7nUSib?!h-YxWqtgK}V_~Zx~ zL68PJEg>rSUzbXhIbmB!v7Z1LPG}^P=1cW1NkX=sdf(c6!+E{?4-#WepMZeA^Cc@4 z5%}zYgdCi2l5I04RfV;2odNRItCLLH+<&o=3qpcza6hdP-aSc*N@2Bvch6^r`LEd} zB*-h>upzwbl;h{Jgf&Um&r!3?u$DPHZjwsgONf}t)C-z`7RcuQXr2-mOyYva&pAt0 zKD0UW5%rlV^Hs|?YN&WMS>MxoYyF=#g};r(VYXSTzm zVPi2aI0Ju|CKKM^brPab3wL?3>`Mrl=evRD9G0*XP7Wt_tkbjFue)0>y!%ExDMR$( zyJ&VWi&f`}(+Iy%ak5aOyS(7NeL^XV2%_30ZkOZ!M>4oyP0KUq=56gbE74Y=f*IAE zl=w`%!QiL;DyAsecQ?ClwBv8lI;O5Ruh+G0Mfu6$n)hZ0_m&%Wn&>X*T!o32j6HEQ zDB|LZWQ|jC4{+V3Sl8P&PuUm9Om`BOxn=+|zTY>_ADzIMM0y%Z8|8IBUIi9zcHAoS z$n}}g;p*P<83zkw+okg5u#T;ezkI%Sd5fMP%Q5w4!kB4dPs3ZPZg?V*6-)d`}BxPHy1im!W*-;8^KXpDFQ{1JPcR$m$>uscUE6JD5D zc)R-kSQ1G<>&w(pN8V7%o1Q>%|Ajhr)9p&Z0x=_r8TUzy>+v7!QWeee?49zhq1qp)~V(}(%Wg%S7SHRBb-E%Qhw}YB~xLMU>YF4{YGvo zA zGfMFC8A+*)JDXz=4aDm8OD@ehKdMbQ%mZ=$6}IaF#`O*#9+(Kiu@ldEVy5Ja*P#`V z!SLA-VIILG^*HOUD1Z=z=Ob!f7&1J zyL;FN^GY>t^sFZw9}9R-H%41Pc42}ueV1)h0#A=y&!kKch~?hoE-8ibHylecqJei0 zTOLh(zfF?s>B(a^9I6ZpGrHVMS2`!6ouSMCUe2MOnuK|hy*%Ab49_mOCVBX#EG?U~ z;^}si6P|J^wF0tbOEYJ2w{?)29l6)kHEEIWAxu+M%(|EK$)F*l>zOEjpD)c2$gDEL zZ1|K8|4lpzU2{5L7Jya-Z-kiIz=O?k(v8=myu3DcuKLu7f+M*GnXI*9*6bvY=A=|w zy^?w&PFbRx>UVub8~gCc6K?*`+S`6q8(y_wdZDnMXp7n&4(3w#zzN?&>P=$V$u`z- z_pB~S0dGGckcHjENii6o=n|e!@eG#qHCXUZr&`(<)~BOdYOhR>%&N0iXuqjmEM|0I zwQg2An2LcRS#s0OQx9V$$WL@BmUvdCH~7*;_=WrhF%W{)f}eCrO$RM@df|xWVSSfh zGI#rCs!h!xU>z;R?YN-yv4Z2+_T2JIhR!1JuMq8==fOzKx}`M5LRDCG3E z>0q>OVJ(%;yIZ_eehxbO2rwtE!p(7wfPU6q)Nad7l=bI#35fE*Z#u3#!RPKiJ`p4UzyJ%#+fgi-QHyM#81-URI4Cm zn=-!plx}E+sEb$AoSJil(Y>jMXIi~C`R%CPW!$m7y!_LT)XBUWr01wji}Mc4EE{<+ zO$OW-7M$f_p{|PcPW!r?+l-nYa-w$Xl!ns`OS!@b4O_chOTlt*;E@vh{WoPo8mJQJ zU5@pH+87HY?NPhbX-*L6i#Ag$2ul;lCLVK4F14;Mx(Lz!ffj{0EW^~&x1HURGMmGy zc{s@7S}<|$V=A(Mo$oOFu9jl2VRhqpp^Ta7w%-|PvNbo~->qKy!X?oI7wocC%k?p@ z(x7f2^k2!}aTHDT0=D^#X4oQERA2hX+-~bFYGFc6964W29zrgh?miu;l8HX2tw%Ta zC91>n6OmT{JX05>bT$2FgQV0(015WI=fvj(Qi0QMcW6eF-f#s@Dp`C>$qu|;GEiZK zxZ_=Ai_D?#R$8=3py7O$U{Wq7DwMa6Pt&e=zHgbKYjr3D9{WhEO7YnV7Iwx=)>C%b zg&f3(@=8o*Vp*>(7GZzoBnDb%G>p0mR=JhFKRs{z%J*8>g&(=ygP2@dAB+xGfuMF1xOkCZ$sqWfJ*$ zxB>k{-594Qj!(Bg`XkEk<^Z_551Inq%DPjA{6ft8!W?eO>WD{UTkK{W#qSiR4 z^UnsROijktHy%%b)Q|&*1V$T2NJZDGouqEi5B94$wJtL@8VcgzGoSjr(`(-Dog@0{ zzGw)p!Th~Qa~`odYTvgYzL-x#U3I9y#_92+ET-kHy%a8gNXLBwWVqDr;dghZ(#~Us z9{jTmWfd-^CYDkM<)k9hIZ&^iSgXoaR5qB^?%7QHw_{9_@#0hr5vAGc6_VxB*shZ$ zJC{T&#FF_4s*QNp7Zn&G@cJ{m^k(H7S%d4{d+qL9?{_04#*au71EHehS7A;=jroLZ z3G~t+RW3kg<-VKc`G|z|cuO8CB?EO4n_SVZqHNQR12(&u?}zed6;V0Nr(NIrW+m^# zr*}X2@ol=VADO4|U&TSr@aT$;rCZdwPZ8u-CoE^N(zS$yRJuO)t=YR_Q7m3Hpqbmt ztP<2c<`<}yx5ksi(Y`eePl2LiEw5%P2aNf3%JQzPzHUVS0Ej8f@S%%Ht56N_?er|d zY~pp9nT_I+3#w&sn#J+O*fJ)=lCZn9e;G!snaeHPDh0`)q<{EEWk{8_Jah$>W_ghM zUM276uj7)bH&0B_yI7+R6&*f#a;5-l!tru{Zhc<~b_I+%gk`4uS?qX{!kN^DGd}i* z2RiaQI7szN;+1o=`S^ExV-)7JTm`RmynepAU8OwCyION;Hg{SGCl`|8Fe ztjNZl%)sLjDetPoXN>oBABY9^Yr^x~YG1d8LXTrxayOP^Ik8vT-3!~f{0 z>&U}N9@dlXH<$GST|~2we#vCHp1xRZfMM>yg|Ne(=v-e?^~C~Oj{{4n!$czBJSJ6cz|2gl3278u5? znhHQ~9eVi{wY#|_*UZ_ax0yxJoi;ay@Hw$b@e>m4C|p;TU)(B zL>KNf#0)J1YVKdYA`fBP7rHArVt=KF(>+CYkV`QBA=SMmR%&0|Jsot;vu7dBW zANBr3#OQkqmMR5DbOgTCMFkd^J>>7(#-tW+Pq8rxFD9xo-BWiJ|IkLg(nGE5| zskJ_l{x}JX#?kNYv1Fd7Ej}5N+k;~jAh>PqV8>Vw)>E`1NS5w;_b5BuRCHxYP{1?< zGgIM5zOY27 zIkp1XK{{^Lv59LyzT5z@-cQJ16Uog?r7so} zB$=H~sVZ|KYGTRlCl^%M9r}{gsp<^x=}#6HyKrk*MxV96uDu{fXSHcvb$AbjHxNOz zOt|%*gv@s_WNtWy!&NG~da7-2B`|Rw*3)f?o$~M-N$CDaF+TSu!AFw19Ys|mWf!?e z*b#cMOw?(*mkAk}yg1HL?5)_5G=)MA13`o4dHs$tyOCVAg>`q?_HMgR1`w%~vacl6#Nz58%^E8$4A zA%!#B(NYlV;UpMJ7+!2aJO$)tSSn`rNnTZAfWCZ*Vv6Q|V4REq0tENWF<>3_%tIeM zjw^9*wA&u|#GyL{nSYR_Kmlx=&)*^JiV?UpXJ`FpE=HR1hV>m~(RC49Aj$zIS&wfB z#E!L66D7SrB{O(8Y={VM(Q)gU@!9*;tN5W2VhLvvXKNPTy)>+|n^K-b7(mFr**;v^ zZzeAPIPX!j+}i)%ji~eEd&Mn1(6aLxc+lS4n4S~xV@4i~3JAh*2`*fLj$}FE@sfgH ziadrXI6)gX>haZmHp*~QoAtp6=}OsSq3;wV#E zJhEP*rkikHo&PWZ$x=TLLV=G>OUL) z^k#Fop!JFX!o?4sulrK+;ZGSiA{trG-qw5ySaG!ddO*oW{wSiUE0jdxY`!nXYWyp-_1?1Ck379jZY2>F(F>Uow+R&WruT`+%Ix|j_b6% zOt|T}@op?NX`(YIeaDA|L|8K5hp4kAE%9_--3}M1F0|gB6M?Kb*bpUL9*l1S<0loc zf+#;Dwm`QzGsY3Wc#}!*p~EP%GZ&K_FUjM-!tZ~VwF;PQ5#ZOTzO-E6i(}bk@&3`z z&B?Lmf)9wrI-h>eTIkCDg>n;;5s*y}GYehNwfEFrIS|rKr7drBz#wD_z{S@h(UBi~9a1a~522~ZI zdsh86!+ZF{CY&e_7*_EQ?Jyqhut-gJ5Tgh6z3Q#H>WVr1oU{*J*dwsRPJv)pN_w;jY zclUesMe8(pg*R7|oVvG_s5&U$4DwEY`_inj_#IlDx$Quhw0IODnnoR*g%j zPS1H)ViSMxgeG|R!@5jj2Jf$#h8`Ekz)7#d$P3bE(2+3ZBPHW6ctt;It+YMp8*WP& zNpQ;HWRA>iQ+8SMyYxTx!@n}YK^nY-iD<*L>(dt3HHi6^2)fgIF8+t{0QTS3D4~%k ze2EZ6?GiU;YVfnFRIoXca4J||)Tc3cU=^w=#;r|X0$VKF5{s5>%X&FHlEG~XY8w@) z`sU*Gwz`e$jjGiiLNfwFf+K6aUg zN`gC1!`{JKbHGmAKy`&YX&%~B8Z<1P(1Ol4I(8RW`3 zivrlf<)NhEA30r;h(9o&c8$j*Xp*qE2WuaP`CMC^@-i9#YqIv>A>XqihK6xo;->#F61!g8+fQ>UTv_Nsb}l1R`@>qln%6{}&k#Xx7m zvl*6Cy>4yfaE^Y}=}tgS;`oTEXd1>Ik^c_iRp;51NmxuK5kd)l4nC6F7*4+c@&eu? z4XInXc@*G zFru83{gL-7u#^UJcwYSt%=-cVk zEbV;*P0xW3G1t{h4jzq4g&#DD>-)vsDrW8#P=L6*lJD|j@+)nSN3!}ID7q-@?IN0N zogRr!-xqP3bFUHszf&{NeTcLb#0M4V)FIAn=Qs{I0qC~KmL&E!`7r{pAhIP756`Mq znMF^9uOshErWYi9$0P*6IX%NofO*4hm=TLeXh>k}DfIT|P#cHGybcWDMPP{!xHJia zD}R7YI%Sb5CMJEdU2jozy|-gW%f1fApjxU_&##>_76-CE$!^~(WpsqVgPC`S-x@9% zw%%-Vh_Mi2L1rRSY8{4DJeDVP6ne$$JKR&-UB-IHFMB4xMtkr5e3UG(<+tjN!)T@s zIM%xBoZAp9>oeZ6C(A@-tX|IHJ?Sjj{cw&1$ak~ZZgNX5$Y_-WB|rAd>NgP?D9QQR zpUjhVvRRRl>_1I61X#b;0a#?kTfh%~6q`;lfyCtwv)}W_xp|#GT1T zDU|fx4b+lICb-`-<)k?PoHR`#M|Z=^<@Mp?Rw1iDcD6)rVneGotn+^DZ#0tp!YeX2 z0jHRgv6h@FT(o#vwFv=nnjzS$Ny5VH;hR#6S<}Eop~@$$dF~sd!`D_!ROvCZ|*ScoRVWCYl7r*b2qu_;CK!{wLqo zYbML2Ykd{P*MS#Kps8cEezYbpOq_u6YUwo>Y0i3a)1rfX1 zabVbMnt4VOy)wdwY((!Ho434(X=GV_0!*DUoFT8}aG-P9QzEMyk`;_sdMC+WeP8&J zhmTpU0x*L#jNU~tyLP+CBwmY0V*v?|_{%2T0E2b1MdGKLL=DS7^I1SXli?L+Hk;n# z!LlUOqd(2zijQsIRg&>aBH{BbXf| zIKt!O*L|n4dn&nE3Olz6*#Hdiw}7PC(O%Oi_9kc+Z> zzWY886hW~MUFO626oVNZqCR~VknH5I+~wd=1t`0_F1QnclU!m$F42AX9d-Lb6_FJR14SbdPm`L?Yp#H|JmCwnz__~H zY=(tIea7*#e9P)&)2+z!QT6S3d&K>Q2xxZbvsJyt$?VDm5+N9JZq`6ig}>m`akeg1 zYtH{5++la;r=f-!hnSQc^!`3{@4^`}#$a7s4e!~E^Grc9VN=*=By1YV_Jn7i%AE@y zB1z`+u>j(S(nUQ3phWo5t~;`#i~!t-zH%C+tS3DMIN=GJ zfZfGHTh+KF?vYKeExWsrp~X?m>6T@@o%2zfgQ8O8>=1S36+!crrZ(!G)SnVEq01r@N8H zyV~Us6cAlLQm6!dQAupSd?Y%L*j2GSclqe#E@+gp&watFs(s7pBVIgD6 zE&C1k{3flSs;~B@uY8v8%)ujz!+RrmlT-)Fz(<*Tvu-0~6QCor6{%nNBfPRe z%!=jb)>!^v^mFv(eE>JTt#>@XfvyhB^_L>ZrxzSGZ(L1tnXZ-#NZW^ML90wY9I%v3 zRgLDV6*GDq7$26{hhN^hsoy6`mTuGG$JF;b!rs_aR#&a2Bwh0O<(I;%31@=O!k@Y@ z4reKEOyni0eDJNUWv~-7989zQV)Bz)8+75;)AYygS?0p4Nmb9^+L!>K(W_pOKCh_b z0mEuj5uwN-$lSoIH1O$iFbVc*As6`sXj)mC9m2Ag9ZPUJ^U!*&ABBca#7;j1m$(%D zaK-{LC;k@&%(t72+L!Xp>58DMc(3HG>}<1R@PR!xsc>4i=$T663SP_&mAebqa;Arl ze>e3Y{ka=Kx%Ab$G)|>bmvx5)BoWmjI(m7~>6*E_{xSWuNZ`N~!f{|&H_xo(d@I@x zu&*W*TLckRv-_xQ0Xv(KZb;+waebG1r!?UH?*dNSzOz^2KK&0$ODWD$$e|B~Rb|ETl`|Ralb{_wG)P?kaClK^qrg~tgU^q?smnJB1pPbH14?f_AwcS9 zrvjmmSlDS6fzyVc&-Hw;XatnU_F=+S}C>7I#CQnTktk~M^diWT3SJKQh4P0-rv zc49x{_M`D8c<}FZ2Mp^`9JYWH9+=C_PJ7>NbDwXv*sAawN)G*IdyCOD(g&v0$cyr; zbD#ABsQGe`VuO%Iz%hK<$M&oIB7RE4*CpAY3-kOpY)^L$iO4CI2HDKPKXnjO; zBIPIt6oskXl&zo4>aXgQIko$8^TVyB@eJ-8#I3cYEKmsUV7p9)+zTj!`N#JA_9?{Tq2& zYce{Z1~jAr@QlkvboZZq&q^7v^UgkU&<#R`ix+NqLU8eEc8VP9UK;0*h7yP$oG708 zwN(AAT2%m{Q=Gk?Clz+eL^a`%9+e>tmi&f?O#=o|>sSM=6J>Ilqt>+78e3@0atrkj zk3oA_*auE3y1We`@C1EvGNU~`8nA=4*bo}%vk_!eU!%39>zg@pYL3&`y~dm`g&)WJ znQYf1C*1^^!V8MSr&@%cg_GCw3$kV z-B(k(nZ%sEKI!A+foR!&C>e%*q7#y)^C$~1x5@&d`|e1Nb3sq zXYJia>W%p6V@Sy78SBkmEy1q$0kr*=-Jfoh+$P{@V^7JwB{C0%&oSivqFlMn%HG!_ zbC=q_#viVI@1YS``Jkzpdh4yss~j$BJsqbm78(`SMR3#|vC_@LfRc&2(NFhV+X-QE z#)6yPojUscE5xUh(Pf-@p@%sS0Dw7AF(Pm)uqk*yn{l}Zb|F8o6&z*vTj+lM@-OJ#Bu9Bv7Q`wzB59;L3c!>L z?z)Z!_Rn+f@!u_8^K>JFKA3&16$yKMf55RO!6YG8=@S4b_d&UZ6RX0dxwO6|C?oP6 z4aZ$fF=x|d+7Sl3J>O({f!H2fQ+9?OG}rD9b!phx;xCJ(Xi&kJJ2Q)Z49AZyll84_ z8EuZGcV_0p8xpy1WjX5a3X-b!WW?wW`8006;ROT z4fe=u>g+p2d-m4VGBV1FBhdmG8)tK50MXN%LYV6ArTbS z**dF^&QJ%+I9;te#@_~`c6*VCRA=S9uIkJoR;|y#WmjcD;oo^0x#0H-xNq$D(P_lw z3AUlq7QOt+;Iyf9D&{mxV^)awzUylnC8be5BUgruvC|J44_uPo*f>5$ujx6j0JYr{ z4wAK(Oa1Y6-f{LOoq&U{hO~uDZ>EUowKid1pL&7rHJVV(BW1 z?TfQq(^pZIW4z~CVEU--jmCN3(E6@=w`DiSiJM=}#)L1fxLZzb39<(TLO+!`BR*P8 z0E?(6mr!-J)9Quj2y(-mF2e+R-6`I&o3 zxE2h+V-uWdaBg?5pGB1b4%mKCV7h9voj+6x*mQv;pNR_hxVV7!gF3Z1X|LhajftQB z-o|!@pZjH8&8X@AH=-20!FMfcG_z zH*OTu^x&UIG0H3kdJPlIDm{~vhuLc6GNi9oypg4&?|$L7^6nJB8jD=+31!wfv-7eacfgEi`naBlnAxY#RtCzp-p#}2YG%4 zb7q99Jscv6a=F=2s z5g0P_6gjnnQ8kDi>FgDE5cL$XhMwwuTy8*Ock?diITf6Dxm?TYtNs`s0%{DXR@@x{ zrBCXEw~nzd_gpL+)+Zf;TfOaa8+iq6ePtM4YgK;c7YDX97CbHDPYO(qbh+sB{1Q)r zMN0`HQv~bb6@ne>C)Sq*yF(WM=XdbE4-YE6bX?DMT|K(fhw<|1-fEB@B=DN=` z%ulQUX}$TiLlXyz*#DJW_(#CxwP#=45f-+$2q(A}%jHPVKOvB9(n~PEw2pz)1+lwx zR!ykO6F#3A1)s2XY}_CF-zw*?eazIy(ZZ28<^~&Gf4V=rk&gR1sajWZpaz*~nPcONFk|M@yUT%;6 zZ#nqK&BH+AOq61T^?%WXig+L)=7Fa~{U86Xr~j}0!S=vJD`o!pZ<+v%!yGuV0cv*T zH_P*1rbJc<80<{##s8)W4}O&pRGZf1Bm1}V@n?hCZUJL9ocH>_X+q)+po&1YGZ%fx zzclVQU#~>~V`e}4_`hj_+GQZ^NvAgM<8Pttf0m2)iolo&IWqkhO^D70rot{C-P8Gx zN&mm);wd{YX19bR|7{a0-nR{ZU(B$)3CICkq8N_igSZZ}Dim7gOT zcm=E`c5g}znr=r=5%e_j&6=&kA+LV*v7+G|JG}CUdHLwCUHE6o0Irh>W{ATcfmAH6 zR`b`>sR6SsJ5B@0>^_d$@ZXjQB&8kF#-@tiC7mukuA;gKESqlUUpaqhW~}e;LInQS z3N};9w}KUtY2uT&Vm>MXedhkP@MaA7DE{5TQ|69F<+=NjX2Od}0lXi`tpUU|w{f8Q z-=xUN*3i9w`4oKDvr_mzJs@z#zm`E_eS+Dgf2TUGnXyf2V{0xdrvF(Fg{=|Ld2l}O zl=Z(=1^itrgJ1Dp8&KZ0kL(%mHj^Z|^wKeS%C7LRQYk^kG0B@Y- z`DvivRo8qDDkB|+AAtV33&u~_Mzpa98JjEk-Z37OhX1dhjSc3CF2V&r+P9F9W2%;7 zt-~l^Ch{TR9yjugyuodPAY*O*IWLiae$rnTu4@gc6@y$!mqAP99AUD-JB|K4>4+BF zbO<)tL8u#uj>i-pDyhs{DP$4Z6w9+_t&_zD_l{ag$l-zrL6}(9pS3-%nSH?oy`7Vo z1Qf}nVvuoh#X#}$lfTY2n`*GV5)_SibTp;94e*7==0R!L+qmsrg+d#p4rk*=|K^R! zdj6*xUVp2y@_RMFpv#sBj6wLq@H@~)tzU{GlLDw}ivL70KU*5drPa$CXIr0&Jt>#< z-P>9!srTJ>cjw&xc{PU(_1FMx^45-ui5-_g@jDaUldI04TgNc z&16#=s4cXA`_o+b$NhgQ!V?)kyYsA_)uB7`mmX!jCA}e8*C-#8p`gmmcX4eruT|%; zW1T0yVT8^^)ggw2_t_LjojlvM2T$JKdwPtkPnAfne(6o~bx)8TQN(odx7YvnWOgd>khtTdv^3Q)FXMxUt*@m*B3T~WjF9C1_WIwR%!dAY zh(w*=qIB<#+o0ASgmslV4M`HMP#(3vD$J;IxP6V8wOys;;#ZD z(Fu`sKVed^Y;i;~H~ZiyI-mZS93xAR`|rkK_mV&Uh;>gkSJWOSp@yeWTN!R8o~e}^ zmT2258kF~!)phYCDkN;nrK$ZqaX#8}b8~y|4wbMX_hQO9J3Mg9;zs@FX=?*11$DL! z?RVYVcuoqHc^#U;|3h2B#k;_`^?r3@C3p4a^xL$wxnr<|)F1gONNz5PFr6EvNk45` ziF4i_%i9+nCvrrZ_x3M{LRvbdW;86HasM$_Ll2L_$tZld+(R{8fatJp#zQxI&`8AB z1AEfntfYUD$6CtZ`0qkTxB(6ido*_@2NzeCj^5J;o5tSpYeM;oM8a->+rI0g0(S{0 z(@a3hAhb^1Gj;5XEp%;KQ;mPe2{XX*>hdEM1p(w6pk7Us;oUgCMd;M{0@zlOb91I? zcHXqVF_UUTQeBG7>vbKq5b1xEhu{iLHerHzv)DXk_(#=vwPK4>ifY{O=Qr!5&#Ybv zFyDBy^I^zU-1|hpT#`m%-(>W`x^hv8%V~zWhmX=9)ti%~3-$V{waXpbN?Tb7@;iC@ zQHlxz!T;Pm0_Ocaio7)~<7DG_XtGAK@nk$SLBWAr^q!y~BF8lF@QnR9^CnpzW7cOSTnOgSP;y#T=k!T`m7 zJ7)V2asK9!Oa;>n$J@brxelHLXH<~X1V1z)=U0_^xYuB5z^g-JtF1#v&%$Te^Ss?- z)7R)=QuE#?km!X>8XNO5#aZnBsE2WGMv{c*4Um}9VTo`UCavl8zA8UK5wsrd9=dyE z>*0ZA2>(>4dfN)-nQA(ec{ZiQAS#d#D&<_FU(uLX;@4gx6lQ1?ZE%UK-2z#HWmy{Q z^uHWHQW*AG12$s!M(sob5}NaDzp*WV^Qsy2>9~}<-VN7VCt*yaAW|=9aJFx>#klZ( zFHnhaN=Nt$f;T|W^(J{|IER=rf+FV*{I!10=oKumBl~RXvawSGg2j>6t`T?pB9<*x$p84X7RSMurq6h zzW!k{VdsQUKd(nZTt4B+yeqV6l;q<)rL?KouGgy1TU_kBY~tml96ZLOvbFb3LKERR zUuLQ{3Yy}3OHx2^*atNu$A9QDPvYwznzPDuDS1P`mdRgVTHN9oA=S&}WL_&MaX}xV zO(bg%OB(AM*jnqGhn}(iCzU=|!+EU%_ufPO<)si#@7jB5rWfL4U@DY$r5O<)YpS>ze~zRBz@ zyRHq@*58j^c!0!jyT#Q{U|X8wjqZ&zejG>GMsSHnH7R!n2Gqt3GPTG;$!mmY;FH?; zFz=)uzNx1HhtBe-p$Zofx;vEJ zQSd+HndmGTN?i`B2p(>;V)}PyiAS#~_n8sNKcu{qwCh(t6mPO?D&BCnvc8_E@UnT+ z1oGKOomi0AiQiji*dqkejm%u_+8bDEh91q_b7UVceW3RqHQQPlKY|d@i}5EeR#pqS zPYN#ah|t_yJXz}B`3b8mP;TKT2G9t1Q?nyjBsJjZdHv17 z`rQQgM^=K0=0|=r281cs-t)~qC9rx%t4xLO`O2u;ow{M8rt*atZ$+I(iPT;<)2-_2 z9VUX=3eqt9oebn)fkZW2OE|^Ou-qXi^U1>y^cWj#w#1udo$NoWLo2pxwAXVZI$-Sw z6L$P31($oP-`aV~-`1&o?(1RrdC%UG{jD2a23OPLi6S)Z6DAw?Lyt^@4))Jd>HO~$ zwD&q6`)-}NtgI!Ff}sjSsvUR{?ozmc8ku`Pk{Ve_Z59Lpn1(_?Cx#N#s%3 zlt*i7r@Vjo<-7aAW8z-u#S!`RE{E}ama#m`gbcp#!(WSYSK^;TEMDn6zi!SjYdMqF z#)7S8G>m)FU{$(i_Dfh7`1(`C~f2 zzqGM$L}d`+V5u}aXs|3S*B#W)Uj zH0!faw)={S?5xm`uIDNxW56uCs7e~m{Bc4T0a3{n9=!;wa4z*j>zXYI7u=OZCzRJp zAnh1;d@QdM=BPYXq{%-@8_S>R_se%+Ut)(=98MkD-a4B=OsUsCTszeY7xpnMxR<(N z9{5Q*(BbRNMbVS;Fx)Zx8aSP@0nnQ~;cx{n11gOaX}!MLK?h0;mi-S)4H~Zs$__S6 zQ+18+w?ZJez7g~9CjWVt{POn3`+6W__eXunW_iRD`SLWSBWceXx!YEG3N^i3i*qx9oNz1q;zVuxHEF1O zFLj#zgSVCBNABsG6`!@B?r4cSX??=|^;$H8U}vTRC>~a6?5l~nC=qb;z`b2Q0^;?B zGYu~mD1u%B-t3%lqkg4E_&v6Ev$(6{Kby*P%QVmCzm?|FZD5MhHAX{?z$l4$Tz- zk!71Ur&QLE-xzxT;+(otUM4j-q!G)p=C)WWTV@(fD@4;Z`c6j2HR;g06pl$LIIpgA zi@uC>EllvBpE#D3DSURDxSoh|yXT9gz}Qf&j@}LN0p(k0$D235+FodN<9kOJ(1dbK zdLY1K9ktFgZ|Hzsli!s-*~SSRfd(DdWJ(GzSaEYV1bGt6ix) zL@d4m_FJU}bIrlGediz8H^-iM*LhI_Ljrm2D)ljutWHDj#pHh;iE|_eG?$9z`EP|@ z$1=5gaHc#qtKl^$X_9N}@9nQKkurU&YNy+gJ{B&zL*jDjxt6Plp||0;;4?c|WHG%k%OQuk;nX`%?@@<9p(&SVu3%Q*GHY6lAF}Ja&4kLo-B)79_{-!FtNz%&a!3~ znD~2VNeOb_@9UYq)RxL;rtsNcqSc&=tilie`*r{-JN@nEH!ZgFA}L75Hzl#fv=nBz2#4wY7X}i> z5vuW>_X6juUmT<=5-i4}N~@e<_7)UIgwn;FVNqUL$TuVzcU+P@L-*G_X6ME?i@ck; zY-y$;m+`_jn>blEYr|IACornxVh}Sk)@1y0L&h=Pv5CHI=4v@+vjMahgDmUaOobXV zinldq7WJ!0FGuIu9CCZ85PQ#OR21uhoJNa9eioH6^?U!lUwrgcCXnxrVeA8e8;DKxLENU|nYY2clEg-R z0u_4gqRL_qA2qQ~1rs5=Dz6O_O@?M~HM^>Cg*W9ZedysJ6U1s%E0J~H*mJBJb5no0 z{QU^4%xbSgh2vh6Cy+Q@_)&(B!}`hk?8VyImJT?HZU(G7BbL$X&{yHF!$@u^{W9Hw6)?fE7aFx%JZ{Em$w@0BvY9_HQVXk} zM5G}HdOMI6>32IH{NXdoV1$0*st~fWNbkXKkU(+0vg&KHx8?H~_pdlo(Q4_} z3d?aM-{eu2Jb86&z6wC2Sg;xF2Unu9^IStV35Pgq172p z`~Djq?BJbq^@QK=d4VZDhQ79@ zMqn&MFUEKnstzy_Wa*3_rUz6(GF(j&r>olYFHsywLM#BhSv_^|Al3Cf#iWE`ThG&# z>#oou_vEK0A(FxypV{tj(TZ$oLq+(ASm&+xO_?MbL!}V2yEAEyfA%!aqjq_wD%Q`J z%Jpx0^mM?Acj0Xt%VgwT^7)xyolgTvHk_{Ik*SRTPt9^P6~knYX7qxSP}JeA47c#p>C%&U|6xV&hm* z!qUxgRc+sSyHK+On6TOqx|fo*x^~Obt$VO`nq)v?A`6&uFcOnyruo>)q;!`rIdOo( zc#hC(wB|*yRC(=ZwjH_t7Dm{fiKMvxaUfp-G~)R?Zh--&^cgUBZ(LQL$mYj@fxFKq z=)YZm-IuvvvSr$T(yb}zGv+NY!n>B*=jW@+9r&i=e-Obr znqR_X6w%DntLb}{6Ba!Cow;AwoS)GzUB+h{C+mq=6XeO7i%$Uw9O1t&d8%UBAyTuE zQ^r_sbJ}kd{GH3%Do2$*^TEI6VFCL0BJFU26O<3!lUIFIgeG?mZ-u~6^2dbxna3U? zIe}!2DgtkKoqLNqLOa`KTIV=A&6-EFtD?}>?GZseePx-2dd|Zm<2I2+Wjf?sV9=#xTN#5t_&rRr_9IdCYTwA4OQ5DKX7z~6_`WmITE!K=C|pB zm- zyCL2$>4T^bn)Jo2qK_a9&075fX=_sF5x6=I{l*mxm~G?Ao$x(T8w69M_s<)?%H{$U z^wc;L6B?!YNd|nvtY92EmTS`J{3J46q;0!#Eze3Vqmw!FHsl-}>b{B%dv~QSfbizM zKZKR)1`qOnT@3>hj~elU42^Q!qPsNLkyb{*K^`b4R(IRwbIZaKa@VqwNaP?gGXPys z@d~P_bdd}3UI2nzoC+1=R24y|eE*q3bV>Vy%fI`!C_h%xZ6wbYIJIPffn=NT2M>VKIla8YjesJgCFFs8>P6L zgGUN)`y9aF8eA=3j$U`oJ)qm7+R22A6#ekemBlT~R>KwW_v|gE#X&xVGc|kHo{W z$RYqKTBNnLSbTosDde}SJDw1Zz9I4hnp{-A$4%GdKDwswtwM~%KB$3F!E zj+(pShY$zFM^0j>6}y5J7MCE%saGOiM6Pcojz>1G#qJLBWn7T2D!(0ad$7Gdr!lXH zP0Ws}^ERf#WO+OAgYAW{Ndt*5NNTS7)amI!or0%hZJnRj7h4hB*aEGopp;;+VJ7qt zbFnS2-RwZ|ZZSEs`yIcx-H!fBWc@?_-u8s*K`PHVhMk7V*h1w|AI6{zAHbq#qr zhSnTf-VqsRpX`ZNIJR}evPksy`n7`NcyHm!36I;oyXT~KY^T%{%LDgb9&pBzLAFz9 z@@RP3uga|*dx}AZxrkO&S0MM|yGsWcxi)v(Ce7%>@m+vc(w6kALxX?0p$r*OFUg>U zdW+3vtunKvPbUv|f^2%Kb%WEnq~HXRB=iBYi?~hiX6I`uLbO+js6s7PWpo!?bq#?Xd+ogK0 zh2%MODMf$$!h&$oHXL5P!=E2{)o}Br?Z3FKRaLDK07{E@fxLE0kW_@P2z^R( z%bmj0o`elyH<-!Nss1x~ylvm&@)X&;Zk2nPJvDqBjtysRMGG)q`j)%xT9h#SMpe2i zR-|hxzD3-V`+nz&;H%v5dP)3AM4Vje{lvksb^hww=dP*J@1kazT;DnVyieG%6X&*V zM*{bnn2({ik|&D1w;4Nzd04UhYBgbTP8aSk685n(iqC@1_Wk}PXnjl1FV8n6s*mJJ zDm1ub`@8(Iu+*bNl@4Xf+w);PU6LOO~f!Oc!ANG0gTfW|9iapmu4;3D~ehPDA@D+$UCipQ!dEhez zAQN18niRVA;6;1==Fg9l&X~zBLq`{b(b=9YwpStzU~P7nGV95(PLqSiYw_o+!Cw*f zPIgsuUUZ+YMj{0pxQtu#^{W{WHr=axD4$MpOsB6Og!dKn9 z86{?h!(t6@$P(lsjJTlFO_z(_J~{t-zo?^LUi%@Pugqe*P+G^15m7XgObjL%bS^I1 zkfRPtO@LhIl997M$8yF^Dy-qsYMfV%blF=!uXXJgw~C5Dn^MwM3RcOBL1o_&&BuK~ z9XL%Uvq*NVw{Ix*{BF)n=@&)(a1lI!cVf@Ol~EVuRw%VPU*BU<1@oEzr76fVI-M5C5kjHJkTcNu%mO%=DmWSW+%?>l!i($Lls z2+Ra}+FpBknc7Bdp$*4|(UxLOF1v;MuVnbgsaelgE>xc_={x^Ep%(MluIyj_Evb1b z1Q$>B49S}p`%Y9A3{7}Qn)yd%<>?agr}H_9udr7?G@@LNYJF*gBWXHNj#`JTQHHjr z#QvaO@5_fp!$f4qhs=M~;Z2e0=-2g625Fx2XHO2?R_xcEz^ z^E~2smBmzA6LBh*PShV%g#irJFi@o8{Kl;xYpIEv+ZtIYSEvNB(zrOFHqU-jf-e{R zmwUH;cFLxPbW-VA{l}yqhV7BYSsNEk?yK zD`n8y!Flv&4g#a!r?xE+yRfF-B{aZ{hR#YS#dddf{-`fGoEI~C)#_|){oCC4i-<+> zl|^I!^1Q!~on7l-XoAN7)0IzI%o+L$p`S<5Yps^~-J)F~D8Sz?4iy3`QMY^=- zWmwa6uvxDcDdy7Q$XNCxHI+{1F%;3r<~9Md7k^7sl{b(;`|=3=f6FMk{V8mTEM>-y z7pE&VGD23FY~iT2fHKo_mOM2Euo8>MG&`HgCe+R17jMNnZG=~~tnTXNYN+RkL!-Qi3}3B#rwxR?!WznM-Y zu~6l_`9q5}Y2H$_&*nkqW$>o|y&8f7EJ{Yu$V0aosh~{xOaG#5?q@Rju(+u5v*Q~< zXvo+qyL?K)GiSnHH|QM;ne}Dn+;G)J7Q26JrWnMlO0?+#%uDO`^~dkuF3ZT(r%*Dy zr`~}};)+x!2hB38)st0?aeumD^QR$o(D(}x-ubLBWGYyC`9*F&au~Qx^zg_zftF`7 zONNh2!vWXxv@xWbb$X=E_BSoX6Y|YQ+ zm2p>i0stcT*$du<^8&vx>+yp0;;SiwR^@1j^kOIhVJAOzliA{W>67|n;pZGkD}MmD zt2VN?bakEDPuiuj;7=a%IHjDNoz#Hm z3M9E=#U`Tl0{ORvQ^VHuZ~ZK?FCVuJ$+4?48a&AREFMRKMaP<8T0VSZ1KDT|dGm~u zEdzs(sWPvdb|L16G5Uu#E7IDuWO zZ&UIc-1mFudM*A+1T}MtpFUl$oMO8=ZM`%&?XcWr{bXlFDq-R%J|+It+rU4DswJgm za#MgH_+N5}Zh|OLljJHbr@G6DBhmQ)bMc{=zxT0xJ@^alQ9~W56VJIimd~hVJZ3il zS+{Q1NGyS;pd!A$jTRa~ReT+KSupsBU zz6X*K#r%E(3Mv%k0C2b7X*V8bN$scy-^CJvc=f!X^xOZ8>*p(eDN6kgDpxLZ{L8vz ztzynfc(Dly+F5T&_-wn_BmM>CXuVLrkP^Qe$FQEOLHLv9I6WO8F|Zvh2k(v1kCJWq zYG!k*GaXH-T%i&udn2i7uO{U%msX*rwWo^+8T+{4*{iYoXzRs!al&8S@T+2w+#2tI z%Ij^ygy6Z5>f9K$qY> zelr-{Ldkq+3*NP=6$;_+^=QVpO#Z;t&-E)u@`yt`N z2=3=G+{>!Ba@)mCN024ak%F`Qz!z#>j(fJ0;MFdkfT2!9jxkcD5pq(srglx-RG$X91msywGZ4;FkbvJyN#p;y^8F%SRNq6V0s z|1G<4V`h?l$?|r)U4PgzgyaO6j46Y_oUXxy8baB<^~sQLA_66lu*CWEbp}8` z;Dj1Y96yXee@GQH3C zuHK+KECw1!Cro=WiOoae%xyK3=-3emO>wBDh^axg?G! z`H-o(7x55-P1saM>$W{%)Ms3J7T)sSGu|t`iL@hmL&ki^TeRX$WXI2{1S$WkQ++F9 zw`ToiLnC;IA0J?J|M$%h@Tb5O1ps>uFh}@3YrBWFO;Ji+RF6G*k;J|3na`9h1mW-? zIZY}_YLPr1AAobMes*7z@UECPpe4dC>(_p-wOZTKy@DTL`8~gJ zoQSXCuMj#!pzzbvF1O32Y+YHcpH9W}m8sJ9WsBRX->L0ngGeOgicg5&WaR40b5w*6 z5}P_?*>ryhLCz%=paZkL!`I)bH>}*=dom>O3w?r33 ze%hZDontjRk8Im~)Ig9blNK2ari|nEriQucte7?ZQ7G%Lt3;m`NnkfuQJu!^7gdAN zOfjue@qV5j)R!Vw!f3_`olll~rK>J_DY!oPo~7C#h6w|M&*2RN4~Wq%x27nba4cxCDYJV2!u! z7zB8TLFl;&WV|)jP`b$Yd9@Y%6Q2@YIXP>)5%29^A0jH=7JZQebN|weEAXJI!*2&S z21>Y{7_a)3Aqpwo$i#n^P=`3_b(z40;-%zL5$0=VqFOQ~EsV>5%F5VvJAo zAlhtB=u`Jk8<@NBy;dtcN0k1^nw{CUSCktKtrUXn-2yX#3H@C^ALQ zsi9`w8Y8a`w$S2NTeP4`g%xZO(U`qvu`ev3@&k~oTc6ddxvR*JB88KJjzlw8`;sjA z6QM-a4vVj+>oz|1yR{r`p@084%WIKQPN8DVSGqfXHV(-VGSwr1+!{5gC-wkrKIG$P z+N+&tpZo(r@c7S)py)T*bXzwe#b*fM|0wYuh9TJ&d(nhfr*1gB=iWg)wmi?D-dKxR z7PU?F=(jqcTiqHH=)G+atp58{mZ1u%Up5Z@w5KoRZO=x$T0ZbhV_?2}Lhn&mgX@RV`WiyElt(|-Z*52Sywfk+P za`*OihLlDGD1w(d1kY4#hQy@FBqI6gPS(MxH-C}z(WTCKWL9ZP3pn4p_Qp5W7vkTe zK9=kDM(o;c9O0_7qHeV%Se_H}XV-8_%g)7QC477)M`);e74Cb+>{Sf!Y`NSEvYd!$ z$N+3|m3`(AnC-h_kh#;zCPT-x#ztKGU! zPX!N}7n8aOPiCI`U(WL8$wPyHDv^Ee7?QlSK1d z=RMko|M5fwtj3Q3tFg-9K%2IM4A$5ujyx#?box^GZ-6%02*JdYMvR;IQP^x(-Qj%p z$-|0Y<1!+cLn|U$WQEmgZ}g)SyxzVHBKg&WHRxo3c`wzdByM5XKv5m~e2vU9Sy8=tdGZr6{^8MFgeglO2; zbvuPb+c?Km0{yqkgm<}H5RbRWj}==5(sIQ9Fb&K2ngzy8@PRFV9n^{2Wm7$kMq`>v zo?_B+vf;D?DL8zoQL$PDnnM2k>&b*4KY6tL8cFCVbv!YTK6+!tbkep)c62rgw}=Ao zzJl&S`k;_iU*uuOjM@klKktehOI4m@)@S$B$-cg*(#XPN+6jepu-`#+da`2E)W=9D zbZ5y+8_P9lye8kKyW!A4<8nc2k1+!23+RE5L9?BoLxGz@bsQcq3YQJ$8`~ z5QLRPA!tPX=2!Bw)2UUTRXe|_bVtag0_6?@vS2tBf`1zh&v|fIUpoDC$k4}fDshM% zew_*GyYTqkMBTSh^3qh}WJrbFi2WHE_O3gTWaI%J$Sr2>-C}aqgJ+3FHN~K$7Gkr_O zoyp3;3}cwSY_u4~%LG_g;sB&*e^mFmAOU>u)zx${5!EPQQJV*5U=P*w)nBesu^Ax( zCJxg+qr{S)5Ja!M_9sjl#8li^TSZ6wznI$^CIcNVq#iVhzd z+({KO5NlQo@|6k|eUi-vG?jM3_hXI9Ou`eEE^8@pR_Fk=4c82m{ASYCxkT3EzP;fB z-3#fV=qt>Mn&wle3>f?Ju(kEiOV4s{KIQxNfNf{C?JGHyxpN5};jZ!nsxKj~Kfm?f z4uSo8SQ#7QrN5Ee_HO=do7s`L#52I!do(=>U&$qgt-@&OCw;gfL!Xd1RFerT6qDy? z-Iy!#Ocee=N7}g!^YG*9?eX$9z7Xlr2um4|DJ;%^<9s7%^^kfIvQ4|dii$0=;%#)jM7c262WnPEqHA`fg;xb-0_+9QQtL z#;7pswPOrRP(e6yJ+!4pHvE{q{!8MKqZRfWZ9-BbNdfH zi?Yn^tCsI+{hDIfJooP^licpJeTA3&HY{1W{(&&UAX78*w}vUE}b5o=j&BTdiO5&687diJg*^Eq;`Xb5>G^ z&GC6Djm?Ikp&oM+#`t|cuq7r)w#j=J{?ODGsFuNe6DPG&yO&LI6A2>r00HPn z2!esL@vY;PO;uN)w^5sTi@q3A6^Gx~SH}6oq4p5w2UIAWRAu$@(-f;Ah3Ppf>%tF~fFTfC8U(V5Un#sKq-jnNF;RLI1 zomw`D>`cM0ta5q!W3*cJnJ)^oNH@8gUTR6p@{@m8hMUE(5uSUAsxN8csaX{y>Vs(z zyvcXxg4W@+RZEBPN0V>hHC;*-xRjS|SnM=q2-{Hb(v|0lh5WWUxG1=e{I=ElfG=HN z)C#|l;;jItj(53&gP!~OxgHfM)Ay796jq>Ks4MV$5C{qIFChgCLx7%~j7+{S}{Go_<3i!|@_VyDe1A!vyB>RkH)} zQA`5jK!vQfr_VlwT9ac%k7@#7FnXl}ef#dGOj%ycmTdD=a{n#rf`iL*VvE<=^*FJH z?tK3Vetxn)7gyDbe^KllT6e=YA-jUu*b$tRSh)yz;WP1%x#8`7q#ojjeE08uU@G@H ze+vvGkK@%rm6$u-?ytk=`*;g?_7xVLZW{LRdW181!>)BqDDsD0bT6lGcNbQu$JJe+lO873q9U!LJ?xSmTR8$M&mB6AM>16wQuEsxB2xJ+Z%*7x-=;Xh>?c{8 zY*i0f*;+q5xtiTa#+qVNV@eQ%v8KK^kJ)hxwa6k;_@8W4H)K&pHYw1$4e#Qvfejxl zsIR9IoM+?C!Jb!pu+FsC7=Cm+aQ#kM+1$F0v80a3@a=Yue#7#KZLOb1m8eF4xGd42zg{9R4aB+W`WgeA6wgY)BA zP`kkXa8i@fH^5z%O_0aD`UY(InWC_) zfgz%Q4{!d5pUb#~rNKO~3cohy%xZMG`={QKKMt#=g*syKChp;@tqY8uaqPmM2H;ss zq14*wjW3-A+rdpEvV+K#P8d@~MWvJ{g=?m*`Y`-E`!-g{~r&njZ@s_oEb)WRMx8UCakvDJEibT9MNN((-W z1WqBC+)h1TKb*2IZ3$zAT)LJ7A-) z|KV!A2Wq`mY1J>)QGb2?I!#4CX-Zo)+nUhEAW{t9{+$s%kaG(7pG1N1M3??0O61*Q z>PYsXuo|O>g#QwL1-qw_cIKCs^Ft3}8?-7hmYsI&B`bUO@WG&yS3Q8H-&F-L(Oa6w z_~T(`d8${Q`E=A$uEB$p7fafy1xgDkW%)7W%Q*e;I>NYOROuuv)&nnvdJa+pt^p zA6$-ox88y|k4vmKWBv;YKs^~x3IHcN=L27kUL_ywV+485=Oeo}rpMkN67I)liW49| zXqnKzmCmeM|dhoVoV7=PweWvNo(wcIyL^onUqg>$P%>+nfpCzNN{9c%;U6Z!^KV zY)sGqlN|iCgzwYtw2zV%k?HrnT$wIuNp&*VsX-2x^k}Z4xqf7G4_f5yb5Wti)Uvwm zaoHkDS~pcm_mzw$u-E2jYw!o}=blq2w%Zm7F!>RHq*%2{Xf!Bz*Ax^mcNe&8R8;Wk zzXaXDSd{>5ghumj-m|nW&EP|dem3uoO)rBMU+I$R=1(qHmlNA(J0^pbCXzI>UQdPd z=Rb0tYYVg=lr2!WpFX^-*%F<0eM?dC&tJt#MKzGHBTl(9 zgj*983ssbrzD=0(b=ujZaR_Hd$DzH;y}$ep)0$fWvO~MoTinF6I#1nT*sZ~b??q93 zlF7-Q=1q09quR~(2VdnF+YyZf;2B$7(=ntEbe3QNc~N6Pa0ZYbs{xIPQ+tIOS-vaUcjJWmF$<)^!Y+^@PY-<5 z99qDXu#(`LW6Y~=*k0ZiwT(7Q2GJOvZ8mTB#7Mcy8}6Tx#Iwer^?@F!dX>aZKgZ)- zR&`~A2iRxWv|fioeYJ#*xWq@omi7+FkFtoS(OZ5dw~wxZ5H=~KyVe3Y6^O;+b=y#K$!bZ8pMw>2veoakV zldA#)o#)oBdpJ1(NiOsZBjIDA4Bz@cp@L~!a{Z6_ver+O^dmtd;@Rg)qz_sxl-Tuk ze@*Q(YK#%jhqdS*$VfbBiQe6ij-eU*kb{Tx7P)|)&asOQSK7;r#8jQ56C8Rf9t>a{ zK#xR@v!6+|vU=PO0^s2PQ)Ndttrtb0;{by_f*oI1T~6U@57&^{B=fN_X6% zoep`JD{CsidyaCbNkBp+xzba->x%GPX$E8c&fgO{`*|Eo!~35U&osue3-2USwU~@` zOX5z-0s910txolieUkA9_c43!b9sKjWO?*QT3Yn+#!}svPvDk^r1Y<^FVPk;5of*) z=?=IEi_vuvsRK+Y&!_p9)4f81BR1oEWrO1#Th)Tne@;b1h(7Jal%uhZ}^Pb zWJ!oI^}>37*ei~Y%zvAaH$hG0TNVAXe6v<|`N!&BSPyj(;KPLuQl&wBF_8>lbvn&kx$1_~aqTXu_ZJ&An+%GiY-`1E)II|5{DpVvQ zL>P=zH$GaGiFjy4e05)8c6qPZX_~3icvU{B$m;07=*7QAI~!_?cl^?3VtqUAo8|&h zqvvznnt16$*Huoz=LSaf!C-p3m&_ff}po3+v`BE2P99}s|+Y` zcvD$X;YUloG)ssY8QKpzv(=5JU)@0Jgx&LCPz{oZ7Eor&U|etU8?lxRA#F$bNS91i zeQ|g{J80Un(M>YoDDm$8{}agqes%hWM>p*rz9xksi1evs)G0Fzem=pNbrX(Iya`8a z))jL`pGR)InhDN_nT-Jy>21*2E-(pz~s`}T*&#|14GgwJ;Oh;1(CTl)*Y zG^5OTKY4s22~7!c&8TQ*{Bg*9{ZlY*S)5toi^=J(hO$zWRzfuv4UeR$IH=x zw!w(DWBd-K0s9eDxua3F_Myx9A`|}*qLFgAj@wE@*F9dNZae1kce!XNIMh(^N^L50 z05SZr29a2^dIp?hnFp_0V&s7<45*Owkqu{ho&ncCH-)k)f$=26cY;hGB^>g-SlwJv zKjAx5_nc*a++D1-4O(_YOT9-3Gf!0&zudm*{{Ht*cuX5$@(DG4y_)v^#DyUlG^8j!q?V!(c~7Tc zZyk07FO%fBSZDQip5!?sIAR09VtDlHY`|eq=>S9-f*-(Bs>@X91VNnA!9ctwYMg(X zH&v@Q%m?|>9r6(@vu%G9&e@sqOeMm*EiEaOvg?B(z)$Uy98rvhXF(d(DmYTXhOkS}Pn6i^2<%ap3w4!`-{pIAa-(-^-ppg~!}N*BPhhu5gqKivHL2rf#Fr?7j&G6@G%b2t zdn>^~M<=PI%;9p?JTuN8c$@df6rhKOl1|;4NUHobJ_H=JVq57a~N3iU4 zfQI1m|3Ah8WSvPoms-sWG-f1RIAG_-uYHgY09xPVY}2nQ6#>Kvl(Z0{o~d9EWwkgl z&|L6jg87bs;UqRA?(Q%!zH{l@-JqiO!i0%26yTxuoa+^;R8m^O$RUZM;-c&Fli9Y1?3 z4POVr@8R%IZ5RBMR4--L|2A%?mn-ngk!wnPuKdwp2~L48Mt3yZ!nfkv_(g(EibR!q z!N>h_U54Rp`o`ThmuU}M$ximJuX+lm4Migyo8$fES_#+Y0u|R}M{C4t`s+RFD{D)9 z#FJI-b5|CwC)*55<*S#|MBjcaoW9d(-F%w8Wu07iMj*#RAVa@{TsdW7&O6mF)X6E` zu6cfnf7JVFirm{p=(2iEOM=R)tKGUwA*kU?_ScLT24){LpYkFO2#o7#nA(T*Vt&-x z>DfOjJNG=ZMK{!Ks`yUdAAt!~3E6bQOxH()SzJ7vgEj}O)w>0lj~a@KwM;+hG0wV? z^G;nu1nv#bFHS=|vVc%n7Fc+?lLm>?!J4M3H56Sq(B@{n}$tccK_*%cI z!&_oFPMGjj5W5oakd}w?`RNrgu9(V6t_6)yMbV_qcx#fwW3#qP24cX~dos;w$LZ5t z?QIheXc@ZfbQR2nZ`va%8)ZBtsU$_# zy%8p!W&ONWkyG$EQx%>g;w(u_hkj~$*?mt^@wwX#yyZB(8~Y9a^_1qJULf`Vz3@1& z=)5mwo^wPuyF@-%j^)gEzXe!$3i>Sf=+8vh{c`dw)PdadncfO6hLUNJTSp*t`t06z z$<^wbUNEz6lo6;u>jF*jW1*4ALAA7$;%I;)+s4g%E1~(ooU- zGt$weIlo5`%-VG}4Eu6c$HOz2My;89a~2)|W}}+qg($R2UfW2x?`Hfc{S(=|+QxO} zwCSUnc@jXxvN2JEctFrU-XG)lvsFLBBsVGgB0sY8vd(c-uH}vVrNtbTtHH{3?Ryi3 z`KOD3Yg#42oMnC0k@0~^hk(IUFMWiyb)CRd6VFN0aB=>=DRXblYa4^Km93XY2Dp0y%?tokzOWI7N_qUDC7n;ge4o5kZoFkmzrPm1c?YVI z#|*^?Bwy_Wxq{|M9JUX2GRctFa%%G_ukK7oe?^PQpF7gFL!2I81n92DPTo3LA52&kS0rAatj4*lN9*l)YwWMU#Hubl zxUK9iU9zzmqE)@F^5&Fs9OM17$2d3dC@O%3qer^o1#*97aFh*WQx~5E<~9{oJcNl4 z5eSMBgP$4JnEhzTOn6qUYjNfQ(=UY^!X}K~Sc`7Nb_{?g-R!!S@7_J{Nyw1^*rsZW zdZtWcAXt{&=1{=UE8Av*<#j;1Z69{_&wk{Q`j@-kLt%@vAfM8(<}#iU3RsJ=Km>q` zvjEWLZQO`abK__)!C9fTO~vk;X!U=pCI+{s=96lhA7{rkQZ~k(Re7eqpC-o%={2dIHvgVtD zY8eGQk?z*OEd!${w_1gi7J&ya!W{W-I3wjLtW_^hduYHgs(|8+*ophpoTv=35|`lcl=y!xenM$Pec8nSUyXC{eOJsQXH zYiv*`V=fII(#R3LoAW&}K128bj=radt?)|=o#PMhg^|SEf;~_a5Tmk4HIOweQ;O*cygO^Xs#)? zph=}m6{r0pbY45r!#7F7qzXff-{_;6M$^0}Wl49hq4MRfz3*!KF&QbvYc(IK*cN^9 z4kaEPbEk(WwLISJM=^Vbj?b>fYP5*VTQAEGQfngEaK`Qsp4R*nlf%uibLST-i1BDJ zLnxDWNji=M`Smv$e9r1XKDP4*ynyYdoXcKJffr!SFEZ6V;54q zL9985X@3`v2)!@bu(D^*W3R>0 z&CX=%mjwUzpPvz@pW`OrqLVUezoQ=Pyy`$~ZB>5t4kfS==9hHHMQoMkYfjsY&Y>|4L@j(p|Ii?0<8akMMXW>FWXcB zL^yWRf!TMZE{f;8B|Don#hlAK(E#1rvJhI>QOl1XZ$}iQ@je&gs z?2%GTI8y$C0Juttdq>)VB(3Na8T=%g3ahT9i^Ls`=heKEh*vm+rc&vbuyE zg&oMu_QY&+pLv&b?q-R#?L1r9?!ld|cZRA)2vZH%+gfU|0&SqT{bBbgz>jJ8!U*X3 zyj3skg-NeyNRqN zKAUczw$s5eOROvc>{RkIeV4Dkw?`_sT!1HF*Op>b|AhH000Nm?nfd|w8*C82+vIzg zS@k_t-;C}5T?tNGxUy2MZrcJbob2kHZDrrfZjIpFcs32?9&Nh(G8qunda75nJKU5I z1EXVg5~VwAhV{Q`@k5_-Blps zeuK1jlFXmEMSQe+WnGC%=h;K`SK0YzLzsf7DLF$&@3MpZ6Ygo&r1Lp;SxNZb zBW3n?;Q4KuK(9rF>P$`XM3fQNPutE^!#Rx+NF4j(*c)7n8G5#W61{|qk)9^ng-|~I4F2EqPEojjCXSqsuHA*Rs%F{~#$z=S( zZsG{XfzQDePQP4WHo#Un>B=h?jHwl-9WoDh4--#qaMG5n!z{RbNS|oX5@#343p;jy zb{$R$)*BG96$Pf-j+S($y|Bn-Z=NB_CiTs@WCeJQUpSN&NacI zCPvu;ME_tN{x?tf{q>EP4e8ra{CAQ$;}ywIip+YfJdAfKgSsJE&=~ssp41CsLT`<< z=_5Z!4Cm2gc>>UolH@kuG#M+Q69s!PXXQ>%8)mKXkZ< zAjC%YoYbI(m@!Q)1h$*`CMLQ`VU9?9tgOL(AUz0BVg`-r6-R;-oeGzzcsk$so_D7B zk3tPVwRun=IAl$M>J#T4&vL z?#X9A@!3!8vp1Z1tu4SlZAX**^vxj=*IWb~dUYt^ zcKvzB$o?n}>^*1yW#KZhx3jyqEwDs4H0$nYvUEl=osOK`r_*Z>Gj@ntujyBJOfMOJ zT)5pZbjFx`Z$pn+)l#IIS z6tvJ%x53wpMsgN2n2xFbn!I5c*o)r~-0)Nm_0qS{l>WbKTLwoy&+_{`7|Rr}GL|F)L<+$;3sL z9um@WZeUBQ)zRkhfb%2jmp%{u-}p zP_yf+fk{?3y$SX+yGi7OIFtZgkfz~$cdCrWq=sq+{mc;#5pQL34o0G^%BA`1!-+c= zeakopharjNozA*pw#9zkzMn1|FZ$aY#5b;5ldp8&C|AZ(85C~2UWpIx;OZ`<4|NB9 zDwtSJg*SWG?&8k-f|JFLJ-S{KX)_DE$-gKHjD#Zjypn!nw6$kGd?rPOUN5O zm9syZZN_%?JxZGuf~$JVa)^YvN_V8q_Z)N<^bMUa^ux^tgBcEXBk=QW`^QX(k04ZJ zr?gw#&5^2`>aX+BG&(X4pb6&`hdVBqj&C5*fUgh@cK)bvIYJ-_BuUVUF?58)2e!vc zaN8@DA04@=o*7#D)gg}$D^+I_aF1zba6Y3=JL8ugW)5Bb>!<4a2uv~)cd30miJ9wMzaF*c2jPs1TxvxrthL|5ldVj0FA|Y4(qRmpb`YlF4 z+HV9WVOObnDxHlf*(SL1f5s^Eh$G%}Qy8JNQG1-!&aqOXc@=UWu{0N-k<)6gAi@T? zoNEKMM5XZkxC6(>rq40GDH{uUy20!2{xHxEaH&2mO6v zp#>zP%UO%T6VVmb7hspGSmM<`BI0~)6YOfwFcMD0aFxtk8fxv36bk#PbL;-sQ#i$_J%<_^L?}KqbJ;`=|^QO8~`}!?fm7UOzM{ zkMp}GkdNV&8CQ>ioyR>^yf-J%CMt>R`DawpvLz#U35Fl#_>A6Qs^BE4&kkBohbR4O z++Gl74&|^Wu5xk924&k}VT)NN!k?WUgco5=h_DIEQ?~h~_&nA!A>NFtAy-tpvlnj1 zy$I4p*P0gNuG$MK3u+3F zpw8~weBm0(gHe2&mdhQ-lw1tmjL${8hf91F9{!4=735`F=D_;yRsIYR|A~^%c!3mAfN2o%-;(D4 z@>hxrxF!4IuEC$l22hxwqLePiLrU#@*aulTajp~ zlB_Jdad7-6Rp2Sy{8N8Q_piRm?|}Z-5-h3wBZS8qj|0p~su>ErsxWN-$)X>R=;Pn^ zw*;Z#)BfT!pdVfwBMNv#-$!%*3xEE0Y7?0L9KxubhyHUv;3x8x$GeeZs57oP!v7{j zfRklFao)bcCWGZqt^QA+_XGKVIU=Y*aci%UepaOj^)D*7&ESb-QM@0D7%kV6JeVi6AtfJ3+*X_X zWmlV=gU%|g;z zo4Q#qqGnQ46H`wIXJ>5ExnL2uQ2%Wc?85r!Ld>Gn9v1cQ7yfWI3-@*i%uFwmw+ zA04=bj8$dx*}qcr!%HFrH6MI*NmDhNWc=T6`TyM*qbCSlm9Q8fl#vdz;9}`d;F8Dg zdW~3NT6No&mqHNX7h8lTB2X`3xL;87wZC7J|AnfBMvA6PWh%PJ$Jf3FEbFn8QIyhn zTyC_gXH8I=J0HS?$$a~SeHL6-63xS65G8#6b!(ABVAyboIC~Jh;Ck@w(km2pSol0L zhi!`1d}))#x!+;3#=0_{PvUyv6_=f4ccw8#ql88`QJk7=XKKu4x%<-B^3{W#mq4o9 z?!uTMK@sUxl=TvL`&kWc6(EDUaH&Jr2oG8Dw9U2Xpmtxp5xF&4_}$w9Z$DPm?yt#!J4+^FI-guQL~* z$i5ek8w5*DI*RCYYb;pdeYvh|L5?=oA63nxYD6VMNQdfA0oVa!&Q^%g_NXFb#BZVh zDp4OqzjuP4Jw%rRbayl^Cvlo&oHn6vh>&{+CgG+=I~h8P8Air4*`DPMe%a<2mtARW z$&Ge{xfrdkkkjD`8oTP0hDX%PA!<=l^K3=jz(~ECBi0SsV4?Z3uV^qP#vOs@5W8tF zou*_&fg|Pm*IXemCsGKtFDa$q9z@_PtX*!Katjv|ZjaPTrAp580QSm9U7e;bPWylCnrZG1kNacro~ zmwN)XXF(R>Qxwu13=x%T45@ShYLOxHD%t9vqcHAs}Ck^3`dThAJF zm#Gw+MJQ) zdiz4L&Vfw_WZ=Tq9n1G}d?wfbdt;$Msb4j7qaW3M^yIL_WENk04Y4wczic7 zT{RuM6W5ac%;Hk)JmzY`$|a8ggN(U$dXYAiAl;@bxyq#qJ)K(wHkv0dPP~1V=~TQW z-{oX;EkkW|@;=;npl7W{d#6F8(kRx~Qy>P9jYmB|eBcSWcOQWz(ec)S|48`jjBsk) zCzNfuo+hbn13Dc1Z&Jg#h$-hqdwSYtuBa!K3GJe}pKoS3;}@mV2zz@gF>2SpmgTZ& z06LOOZaxOr)_tC(1|FAr7d>D}s!xRcX%t7FEDGz1jE+KRwhQX|1{3*m54J994=n7{ zrSVt79Y~(B#)ZX`PV2Qb+&jXy66~x29oRcK zRT`=@lWVW86V_6TK{}CR1y}6XS8q~cI=M^Qne>`1Pdbln>C%$WnV&D_hEG-=UR`ns z&~a#|;v12QHKw|I2sNg)VF&bdJ)OC`-j1V$cbQZ~WCFM(i1@xmpGS#~-u(gsB}gl= zd`6GVkI`-EQr_PAGV)}3-Hc*SZNXD0`lG)zg-eX*vF{GwJ=ilfQaLi$?IY*vQPwa| z8M&FOq_;N2?&O*)B}Z~TAia%t{laZZzZ zX0zpt(9HS3DG*z3_N>0DV0e~naQMZM#pLoZwRrEA<&HB=$|pv=Cl2PWAjkxTEt!uQ zru~8T4rnQxpQIjlK!II4j(#Y2=IMU-;-?t4X7lpts?&-$y&9aITmG9M*BsP?G1D>iNghk8Y=U0>$t1dBfd&FI8x zZRYe)o(D(=ZWF$A_xZXgk-V^m zAWX?PVW2JCq|!t%J9)*_4_d1rp;oOxmm68ec|TgDg1}qWi;rp)*XKvP9MU z7c<@I4L&7)k0uL2Osb4?__WRkxG}39;iJmaZaPK>dU{;7B!jt-RDrQel|pn8$g5~* zB;PllERlH|*VPsLY|T3Eu~X(sa7-Jup^^20RVs+h?rn%!vovi?QBn9|=DX(f?vP@* z@ra&Yfs>P_qt2Ft<)}1ytGRk^SPD0V($%>J2Jvj6&SsmVYrEp;!KvTrJ&onE;hGoX zb>-iRRf!>8aw_mTOX%J10__boIxX>fcCeD&N>_HTvI^r^L-jTT4lKNSy>{BS`mFV^ zl45(|v6*~!wsxb-3w%qh9WgVVhEFP%-kVlz1B0=lCwz*e)-UR+W%L~N9L8X1H2Bxf zv%6&;T=P1GyiWRwdb)eu#fYZ@D<2f&_Q*x=JsdXPD3fM2DL-E@B<-7r* zKmZ(nRJAK-SD-NW4QP91!h%V+`B`)<+A)r4SoeEM?;!ieZATupo{z|O(u;2_9b;*; zqy0UG-xXShvwISjSo_;sGdJl}FQ+)`wy%3_ z66)A&m$*JgejBC3I?O|;eRDQ@kc8h>V)oH#t;h9(vgNDrzR-9k&f2^{eZD1-+H#w? zcT`zQU9r`XK0dPG+t3^vBA(_Bxr}gq)rrEc?qo9yDz}k zksmmH;4CKaX@_hWuSWe5fYXB-8ax;lQEFZ;5(7G%x$sPkP>bM;bY9PMv$dfEjMVXb zNx4(UZA@%;-ftCzi9!BW&Y#2GvF+a)(T^?;q{r^$dKAD<05^{GIm%Z{_t$+m?vlR< z10IL9>WkzrqllJIz=lUw7?ABYt#f1{7ZXYR^+Tlo#h+Wlk>G7bz zy54@hfMxCtofmHy?Y*n(8p_w`XICt+(Xf98?mwS>!O20*E?%OxHz1xM+)bb=Xrm*y z7HCF7g)^YgtQ$+ESYh+Hkk(*^U>?$=%(H0XO}Bf?2T_4H(;>>ZX~5ykk?Sh6LK{k1 zRu9ElT)hj;e}*XuxVQb;n?SUzSJ;5xQ2!oaKO4rMQ-#b=;gzPU+T|? zmJo)}rLTL>da)nv6$kQj>9X(WFbv0C(T0v+K8=q(-x`sy$(|`P(W1PFi_SciUOgF^ ztb|3}ql_x5Y0{wv8nhcyQF`B7{ZODoi!FU{y8RVQs#A`}dqGWUASHMY+`sqv>3Ls? zGNXloTIZBiFZ##>&((c44fbrMXPAN^H|L^4?R zhf+zpLiM1z43*OOh5D(X{Pj>KrrX(;yoSm;rl2#`wl~NIN5QP(RnE7T;%OBJyry1x ztBsfh8lL-lVmvUsItJlQ$?m{h;oe)bc7aRY`%j{q=a?NhcR{Y9LOrHMGl z=>9d&bZN9%IP1NPf3ZqN4>QjjOgoslD%v%fr^B z;+1q7+c2D;BblQW{KhZbutQ{oWxLYbYZgp-$RVL;dHLe*kG)WX`BuWc40;p(@ zUvj$*eY@k+4YJ0uh=T>i6HCzQe6hXluS0q~GP{g8K6aM2zDtN2A-MVgWwRUGZP6$k zFqMA~f&3x_jrtf}S3>!R>P{`~%s0FgGv#YXrPa@yJ)z_KSA&B*UwUfpxd02w^W`>r zLFMyFzCki576M6>3@jl5m5N3BG`;&X84;mxRi^Q!WGL5R-LY!vTaK5)QbD@8qDJ`b z!wefsiHmk8M(<J!l^46H<0Vb-r5;$tb#-iPtz=oBm-7MKFJYEO z^4cuZYY1?Vvqe97QcGc0HT<>Rr;o{QmyKlHvPlJcW zjjtp5D$I_bfG&!9h@JGaE#*4iK$>3*c1AO)Ylo%13>ma4U2G&V#Hn}?3gJ?1X!5wE zm7y>V%TnJ7DkYW(;c{y!xI1_kNHeIGkA!QvLdb4ocR>&f(W0*ZnbW1 z;zk!)k;ytYGOQd>Qr`x84#&bG&)k>6A!GAM#xvVeoTWO++r5iI1*kZPmMHxt(fKNgs8JX`WB4eOtEz z;OR5bIMkY`gZfKvjVD2|p|?|J7==cWX@8W;;hL^@K_xw4uy?5id+pV5Jla*L;Qwx~ zz*)uUmca1)@o@Zs?g?;UcUWFAh72fwyqjCL)=on5? zM3p;`7SmVvukY~Z0!3Vm>mZ>Zf*=1Q_gzr!ln0@ zeV|$TX|45|xm0|{N=r{pO_!DRk|nF{BAKVNgt~j`n=tOXO8^OwrMXB?JK1&zu+3V^ zP3ovqjwJGqIC-qfwKs;<29Po$KX&wz9iM4*u70V>DF&o3DdHEH)*3e_E2LpLpf2H= zYtuuHc_}}O>4q01Oe%3F)b&KMLOndUhPKA}NIPXo;%Gv>MmtYN%QWp>tIh?tG`%(j z^yub!Y%$`jbxE-*dmk2l?45;VhNm>yXo?L!q3WpMQwvMkw&l}?LZW(dxEb2+@Pa++>L*B zlSt}!H7xxFkdiK;iujMm@Yd~1MXy^T#q~`emBymQGfBJ+!c(k!83im)$w$Y)JdWWBpk)sEM)RQ%%?QYU&kxeCED)-?_rn$kwyT zij}WhEtyY3k+ids>w!WL`xQg_#eR}WQbdG4ZecbAvPjK8AEoJnl}&#*DD4utAH}0a zT!`-Eb!MK2yGRuJ(Chpd*)>ZmbD6K?@-;{;OYU4e#s{ocH_gP*juuxjDi@Vkj`9kK z#k%N@J>H~>Yad!MY{zXW5Ky>}Z2}1#2GrifX>QEpJ#gRYXYM0s z80M$QR&FmyAv^D<=uhDfl~vEaDHdDk(CCrdzG$sOvjy^JNd5a1Seb~cHGCq}VKcwT zBKyiy+Z-qtu_2<&&uGp*1%W(UUT_My9_4u)O%tecRkGTClQU7XwqHhxUSD2E{W4vP zX6i7jZTz9xb>~STp}(T~Z*5W#6d1Mgxo-{|Ph4+p9aMeP-lUi2PApc6jt)$dDhOKE z=NvyZvPKbp_$c`@Uvrs3Xap!_fqv1!tVQQaE}ku4xYLzPi}y@X)j1=R?Vt=kS9f`>22WgTBD1x3g7DyRu{cg;?6-`@2FDP7F~N1)=fbh|~X zVVq6J&GQ^TzQJbefn)~lvSf$Zv0Mv{%AP{cKy_8vH}T`{wqjSOpf7| zF9{G|CwcJC-w#$O*-ypszc}46=^9Qy9@?8o1?K2_)SNfd;038@;wzlqV7I;x-=6wd|F**!u$FR3xve&LlPp{kLaP|CRcx15+6a|d*5i~}G zsk;T<#rqyzcoZMJ=IfWv|0=*|-Qiu~WxNnN#GkZb{f6c+nW`?hJbjV>4|hZG?7^c6 z=g4ODMJ>ajoy1|40;Tlf6g$TP;Pl>wwa*PE$#~93YZJ}|ot*4>1^K7z5o!gEy@5r5;y2*{R)IVd> zUL7k;5>#rnkIFh|q&?Nb=*EdtV|+G`XVtZZ8n=16zod346)|ipxlq}Je)Pbc2oHv3 zwRL%<3Y~VfC*piJ7@zCI&s%8FoSZNR6o1OKBNmrf^$y~6kU<-E17PB{y-dhZL22w| zi#P{mC=o{CPE-ABOwO^SW$E1RL!&f-yeLy8QJ}Ngfn@eqsIJ`GFfcB&)i_`HQ;c()d@jMMAG5nH!>(&!ZnWUlmzQHhfE`Rn>S82Z~@u znawIK{b8kqGs@$$_qAO)ZPe0EZyeI-8;6vulo0nnQa8GXU~>36=Mg>f{}Hn>qNH#} z8xeg_Up{vBRP7jevZDV2^LUeCqbmNqp4YUx?0il&kYx;Y`=&%INJRzIU2x?117-ir zAY@2=bf~suZ1A#icZ@&nqxh%8==)wuf4ccTx2%i$0K_c_RJ~2DlYAJ-a6V2eeHb03 zSLep0R221)PJT$}Rw0}u$J>JFCXEPy4QV7$)tg2AuWLpo?qmBw*s4ddsZ9^vjv);` zE8#d|5%CXTBJDyY#W?@?O|Dn#GEj1Wu+G7{ycmcjj0fe~QzNW%y-1UfiX2_j5^tUy zk$6*|>#H0r|3!W77OFq5FOYwd0%LMZ1$O9UTh0ss6#!$v1oXl;3@$`*1$d<{2_vc- z;=1~>abmO2P9sQIk-ryR&0YXSQcZc|xbQ3Ci^ncg;&H=YYnQ>l_Z zrdyl9)D5|5s}NA^gv+P#9i{%G<^MovZLC^CZ;jnHk?#_lwluf8A^+=F1M(@XuSCjV!1f3HbLFy9Cv0p6n!`8ON;zyF>9Jgq}h zS1ZxBjqIiWpPK}zBOnX0RE@A-)i(cZaEmDr*n5h#`~9LBKNQ#>#{#-5U~-dh_lhz3 zb!F}k`hDfe03%a23Ay}; z`9Bj9|KrKP^L?BV#5e~NsT<~y$+?jnb|C70e^+r9Y@k5>*o>VE?`MSH2BQ!Ly^mwM z`{Anrmk2x)7rsvN^y7wo0s2o`(m@ThWO0yLT;lIk-iR4!i9bB}6V75XkS=`qHYkjj zBIbn#)j!_q`(6gnr;aqeu&}?=dZR!sNVnx$>u6+e&qU1q!-Zl&1{#xSn8qdbkH-Sy z@GV62crb9E{@&_4M9|+gFJkY;VovP)9{hBWi#DY`{s0_zFnclC{q3FGM8=V;tuq7+ zX+X*GN9!i_{d!YNcYuZQ{r$}M3wPi8RNAD0T@nVX72*);2W!>;fwn?Xf=3aXY&&w4IV_~UD$&x$Eef_0|83$n%GZ4e zQX(DW{y!Dy<|3>lL`7jvj;Fau^<;*h2Ccnh!%KD+J6V)??90up+DL9 zCE(p^gQvj5&Q|`L{zp&>5bJRu>1ExyaH#Q5BEh+&2NKfr5cCj)AKNaNhwU@!I@0{8 z@O(m?_`9wfsoo?bH~Q=$|K;nSI}tBXvjT0bzA6ZEP}EZXp%W;g2qpCWI?DkI6BDN2 zo8}K#t6*L^#@~7T-5rp}mXMo=-1_KaC1E4|hxD#U))1(k!J^Xqf z^*>4l2*NMbho{hf*jh8402>0Bh!Oh@JEq^;H~z_m|H5^|XB5P@HdYaee`SXWLbH0@ zxgu3UXIJx$E(wwXeIc^Y2gjcZ+Rsn$H&obN6eI^>KdnEk%vYacu74eJkT?eW-G~!| z0habXp9aR-{Ie-WqzIp%r4Tg!ttlUT1n`!xV(!=2RtS98e-r>VeB}*+Angy){kh|7 zE=8IUV1MV`UwK-FwL1#X5&vI00&eJF878Fu868*%8Kfx$BY&gASCBMr$#(?xU3mrO z|3HUO3u6j?afBMFEV8>0$v3quV_?3U^wH0=#z*0QKqcuN?A`Ho2heXae{S zysUjJoj(KR$38c3#ANcNIPAJiTwAUA1l9Uo}i6%dp?3JB7K;7fGo1TWJ zn()UE=nv&FB1FibPf6DJtL{e3A|h0Cb6jTqL5vvCLqOWmO(W?T?Gpddm))?yh5^M0 zaUNpx=`ThQk%p+)X2;7mB{9eh*ZiGTzF+sO@w!&ke|K$v0Eio9BF#d5NysLQvH4-l z&}SMhZ#TM)(fHtN_%>gm;YV@N4vE1ljt05`k3R)2!R$M|v$0Q6c}^Y3y0MWxYR?(^O#K?$r{ z4;tMuhABoZwJpiBv8%R<2P+v^(IE5lA%FfND zwN;p0;c?k6IrI>~D+AgTIuEPE33=yDW$>tdtH^v{d}-$|DdvI&#qb9DYiU0IZl-^+ z2|*jIkJCqU3CKr&h$WAGk-STym`P+~5k&X9VcwTaGiSDYt(1??F4Tj;giK7{p5PIS zCy`d;vO8$)sr;o(iPYZeZhG8#CC)Z-MU68%S~IyX7Sm#)mP78UTtgi1hVq1orN{WS z?@NZIG&ix6p)`6-o7&AoL|s?=(-grnj?Vq2XCCx(i6{L66)tB_xEwc&jQW#QrjA}y z95*>h=a@!o-fH$<#MC%CzH2s9?aRA+W|==qKfM=yYt}iP_b{3;oR9MI?oq@Mh~uy_ zAZx+5Z|$O+&iG6YElacQ-hR(9CM8=3AGU-kf;+qT8kp~RC+I=roqM>wOoQIDay=?^ zu>*Yb%YLbXCRhXggpY0$F+x5daqH9)aNj`MUwL(gy(@zx0p;I&1L)hD$gRw|%{}RZ z(I?e~iJJuk@!K)&!((0BGRR9y<*ZR0-c~d!&3W3;%KYaI<+2s~S4K-csY-7+LL&~Z zw;}?RRL?2Xc4l(4pU{@y7xI_Xh%pU=asH(t*mg@3#L^>9Rw5qgj38TMA2Fjltn`Ur(u76|mpe?jbNlyN#p1H%P^hV%nV~ z`b1Z)ErZA#3i9ovnT2Bwf=Vap0*^=zU2o&iFXt8vnyJrf?+>XRqn-`MoFTH8PB5^F zG3?gOTTE6+!I~{EO5k7Sc#yo`-uJXBO5t@A@HIv&CPY$cXl1a&ZQRv>i82jCxJ& zT;9y`vga`hPF{v2!Zk0BvLj^*pd0ws&KeHw0tCz9uPU>TSyQtYQyFyP_SkUrcCUC| z)iHV*?%+ky$;s*!YAzdK65`b6iThQ*5|hpUO!w$i&`79f74BR-F?i~k*!;ZcRr=`x zah=0_1>RAKinognwgeqQrMHZBk!tY^BpPMlTi?V37wCe)gX&(Xy`g~R-Or0|-=>J< zoT|>|%&58dgIW|Hp@wc(oSA-Mdt;^+dX^z_($-A6MIH`7Qw?B)(6RiD>)v%2#AP zj5)tYsy1atvlSkoq9l+;!!4rYHFO$1KHkY%53}Kz;7Ml@v^~y}@p?@`!vM#5V&l4W z9zDkV8Q#NYBMwjW9LqPm_j2|Unh!iS?^@r0Hbt@m{Tv5n3$ zh?s@AP%%deijm*?>~`F7$4#Ptw3XRiD*U*(PM5V>9%BrK68#mvN8pfvXdk8pMd>@W z%-1~`uhl75#vLSZ&ZC^PH?m*-66Sbrbh>$;8#9!M)GgT`{B6CeXu+B2g3p;jY14P{ zY@f0RH&QHwED8?&$Q=!4Vesu^Oi2#T{YwY+hv%~<2gELCtT0?j(q||w%coNT(++w` zwD_9?R_fN<-l-!Niju9Cae}dLV!!ECUGGo6wDhboRZt!jnv=xUzQsm$ ztMnGfk=guLG?XPVUhq{7RaB`yaUh%%B}=wKY>0Vr>D%zx8>CRXNM&vV%tUDgPUjTb zd#ho*E4H{CISkfK1#-6B^UB}ltaHnqWeSgg4D8l6jfy)&qj4;*_no^sjKelr;9 zk>|0QX?oR-F$v<+)QVXdAfPfXj2Rj4N~8;H%A2(=D4HTm6Q{+`vkARl&P9o8aF&3X z`$jFRW2VO5J!l>Sze=G;#q1U3#AhL*@~#h}_>cs~^d<$2&G+R6C5d2_IYr!jsmpzC zReJ68D-OfrlJOiZ;jJF8Iy(YPj;;HprTO&zbiJpXF&flC-C@nk=+Xl%THe5*g^fHh z#8&ZUp>8m5pu01j4RqHz1>?}LLIH@xIX%@;vAqsuSkW)in z(foVuS(1TM#ISZXT?~9hLe8>MiZiQ9=yVZQ6g47S2!#V zujdm&30T=xufx-Bv3>iK?Cb(N&}It8Hdhh-@KfXKutzis zH9^F}wF@iO`+WVY!gdCrpvTWrTz0an@9Z%P(mF9OpOEI|@r;F{l9XAt=bBTW0-K1u)6?@d2AlLtkkD59>73SFI+qIrq;;x; zx}&STKf*0aI*YsNMT0ZM%AWO~J|aCS>Q=yep+BCD~m=f(z{ z!>;$mB{~GW7(u+Bh+~Tf9$L4?hJaos#<%!(y-wtGdv`Y?(IDlVWzp&OW$25iS*GCR zY>Rdqy@Cq#zd8{&F^g22)v{H!WB(GahDX$NPf zei~jbaN5AdqAvk8id)ATlWC1VrNJG1H5$lNPPV$zjc<}PrseUO$8A?xgT+2om>+`_ zTY&3Cmf=9yi12C*F_0x%`C4Y(>H5&_9EanLuj{7xa1|T>mlg$*m*oq|BoLg!cKa_O z0SV#Cbyga_^2Sl6m%fhM6XCfVZ*oVjEPFoG+9%z3W}7?rlhPjymIvKl+VwF>nOl^Ddatwg@B74H+#bbHp#wVc)MqYshL~5_XN344Y%5-L(NgaSjo-q zm(ISs8!2-v$KPHwQ3x?;mgQ;yOW!VTdB~Z20=b^wP(9{!$H(2rN9<=iHhGdZS?{V1 zQN=1O9N;?6ZNUTfX&xIMBTsifs6Nr}rtdUur9Y&1Uk(g>j(l)CgYuI*-P&q>f1!>q{Ae}%uFrx(;^*|^WH|%KCf_N4D+zApigM*nZ@lc#S7>`!7V#9IQpxoTG&e<(DqFCFB8({q>Nd2OkLj;mfaOlF9yuO6)O zG%3G+EG~T|xUHoJDERr^9r9>kR+_x@_7#^}gZt99dYz{00sCX=u$==1G^6Dd5lr+) zta#*eI`90Eum`=y)OO%%Kpt`)a-KqP`_cen+|=cRQ15I)|Kg7{#2+0T&5bigd7vJT zD3suHN=`yfRut95hJ2~TJ>Jo|z(CTT5T9Jlk1C3k|IJ3RYv0#hSIC5ivWmHeM_M-;+Ei#%+r^lW|^w>VQB-=#0 z9Np0J1h?&)vhBTBU1rWL$bFG!q6i#__QUSU4i?v3v-8$^9wA{$EU_Tt``RYzL%Hg@CNB>9L$D3CJt z>)mpaL(W%^r*`X~C_ZnDI1=(1e6e|D$a3=S0gKB*!Rv#ohxyR@dR>2C_%?RY2XD6>InO8Q>382? zROBP^mtl#aZ@S{dX)V-&zZ=7_z|)I)ue|a4q?$P-Z{7_r<>O0tFkf+cc#a2P3{}yS zpuiCrjvpiT_EWtm&~dTZ9b$rrxq*iKeRBJYuQZ2g(-}KTeNdjobmcT`LcR|zpAfNU z>+&OaA78HSk;JM6xHNIL471Lp!&29#zgXkom(FV-Zfr-t`8Imj5o*BPHC}o}`?iCR z#@I6S)@75m=OsOx;w*XV2vmCax!*A^-uY)kR7KtDXJ~_+f*qzEkKbr{r-NNO7IsQg zH;&EZgk5Hn!Kb^7hCL~W&r;6$cOHKAG;NRi-PQT?H$o9ef+Xkpgsy^g;UlS zzG}(dSXpjBinGLmInlhp(_>jglT(l6K{Mo-nHCG}TXYJ5qs>Z|M?{9(o%83L@J_Y%(6*E6)1 z?$23rtIn64cdm7{UbKm_X)6ij&>xZNS~TzQ=#&ilFm@NM6yF_@74q;qc>GnP{=m|3 z(5Yd^+FNs0S?}r*yH==MnbX?@vmrOiuaC|5R0{})N`+0=>ijq_r-A~Dk+35}oE7M3 z%scV|&OQo8Htu^1@V}|Ue0NLW;ItfXFzZRl@x?2!!6sE1{(E6 zlk?VAq9^@?>%ojJLi}-QzN>isVI+oPlY(f|&PDGGjE=1%2#IO#PB@5GeR*1oNFs?KS&P4nJ#stYLzoDbW9 z52w3-*1j`sry||ms|U5N!yq|cb4cX#%CA8P&|2?N6SmLl;GHf_PCV{E$3wqY81S?t z3cRD`PM|U>RBp;&*O{b5zj=C84LxX$t^+g0H#0d`;)C#d2?ac)&}IiRXbJC~yI^^9 zy*L98wQYR2gUcb65zo`ACr{Hr5@q_t>^6%q%C`?Gw{G zVjEyK@;9-u)G2tHNZkB0r&#C9J8*nmlwEi~9r&IQL|@?}D;P_-WjiP0R)GEWZAHQ3 z^vC9Z$&dV=zOA^c7<`kx6-xIBcHZum;Wog&iHzwL$6CrB`q_jjOSh}M%g(X)v)QCi z`ShMIht|NKn|e4xxI;w{a z2y4ubuGM%piV2f6YMAX2^F;Os`>D^^_wHCT=P=z1k(2u zfO?yX54mdX!<*{$KdV1nL|ho1!YPU+bILWT-#AUCk3`yP#jcCA3k@@JW+v`ZcX+Dn zHFMc26K4~asg5!vTrT8S7F=7OhRushI?Flq!BZ3XSTt1d8^MsyX5~dyGwB>yl14j_hYBxAgVSOG%3JdkQh;k2VR5ocKhQ-uCTJB%SORow-2HP^I77$zK8Iv0*S^hllEcW#?6qcJ<(jzIbkZ?H$MV2F z1bSTj%Aczp-a#x*6w2La>ecU8oRe40piQpO5(s+e(kt;{X)O_Bkd-QN5Hrlvb9|NA zYjzNOKmosGs-L`~x`z3#9rW(Lh7gH%wqOUWAXR@*`UA|mEIjg3S_{Exo>y;A8$ltV zn*%`~$wVRu2Q%*FpSbvkw~#OG69>!hjV_J^K0E(SZzY-On~r1S@Jw-xIlRJ7wXTl8 z5aQN01F|IwV_HgA8H2IKMr@~GSre)*QHP~O)+)g$q}s39*MrDx`$P@MxDy;KM)llXf5f#;1q#!r4tBFRTg+GnZ- zDq^8brm|nUIZCgniet6T!=#ky)QW*&-33}tAyWhlIYsu7o6J*p9s@qE^Z7t|uR@9H zaF43(7w}=sreCzDZBV|F21C~_+iBa?SKlvn?Fe`%W!q7yTG1HQ+IVjXoYlM2mG=CZ zcbxX_?QRm^ob2o_{4@U1mx=xmvU|pcIR<=K7rKqh|ATxA(zSV5z0CG z6c|(51w6H0tupZy`FR)d)*FCZei7D(yEIHkdecxrYW2{53X4y z=6G0|JYy^wIeFUVE(diF?eZJ1I_!9Jw$O+TYfFPsX2y9;S~BlOJF{&@4Q&XnO|&&W z738697e0?H*_2=w+Qc|MP_S;Y1Tr6x&F=5>V2$eqHY(?7B5o67fphS17(B=$L&GW?nlZC z2KSt|2rD%d@n~1H&<&j}PupfuDv(qzEabXlgdeLPHb}WzlCKAU%8d?&ZEvp$ZtJLx zA5#yi=qVb$FFCulVJ}A`jxh+6(Lq+5(Op4vyzul&+H-Hff9Vae+`IUadCQ+s;}}-KDa}KYWRhml-?h837Fh<5c@1DX;Ikya}ql10n=CKmt|oeE~T^(9u6plGa(&} z7d^9Z{a&_w2j|L{CI<+#L zn1OVvC?eUk4~NVB0?{8C)}aL&TT|jnKN>6QG?eQB+ClG^IwwblS{u3WhZ5d%IZijxZf%2hdsVUm9{r+;E+Mw*ghy zn}a?&eVwF{p+#g3f7aliZN`NP4QG#^8OyGYI1WK$2MHY)u!xt z)G9Wt{Z<{}heAh%+FTAA(AmXF5<;D`r{#3Iof(&y_t|_&Q6gOmIFHxL1$+5{`3w}6 zYX+Tenk(_1-G6JG)^y1FaA3MPXKz6C8M<}%@cEdQP6`Ja*;iRc+_=^>tPMz3Orx_5 z!IhP#=d7#wIB(m*qc-eoFBCvnTZ4*kJ`4gMi!Yz*dtd2Fvp3eb1A>~VL_I)9IuPtG z|Hq&Z7pjP?G8e|qN&u@LzZWDn&Oq9mp3DX@wfUPn_m^NI~@ewX) zJY~;eTHrd1i9)=6*}>1S1SHLo#wN}2EuzJ?-mFuF#?xfZ#KIKHr%D^L%c1e*TEgc# zlcE8FXLkzg&x#ApH_mWr2Tm=+oyrXCL5hsiGD6c-qsn_AI@+GgpeAk^)=!!Wj#@Hginw`RLl!mt7QaX!--<112Xl)AC3gWWDh9FLd zM_M=1B(OAFc)r5KpGHZw;asonq>_%2vEw5y79yYElSl}mBIXd$agKYNQ{+j|)ns0p zu6b_8)iJXAY37or=#`9ziJB$y{>8zeu8kv~-oBr4n&+B%S-dUQpr!4y=NhQ+xR~iM z^C{+r0aECSbd60LkU~1{k9Sfke9ttV`NJH1CD z2$@)PcgtIV=L?>_?a+cK%jqqCi5zCRf@}DlbR`6?q~gGBhls&Wd+)338^Pc?Dgvt= zbN2kKLUOATPz+w9qT&Ce>pi2IdbhS;L6D|Y0a3d2Dp=@6dN0zuG(kFq79a!^1OWl* zNQofQdv75W5dk66OXw~179fNY-ngH=|Ia?>yl0FxM%EbllCg5%_nPyXzd0{~$_tsJ zEQuifu)r9~gQOB%HB>X?q@Wo#P}RFHyeYEY-}1Mz1iG zhm0H+^a>pQu+t51_uM-b6GZT4nd~+?Z9U;~-SC_M{(Srq>Utsa;Xktg!k`qrZlJn! z^(v4qTxu*z+ofjV&FOw4cZ7s#-O~00clZuYX#vY=&QyNzI9xx#xB2PBVp6>OXn51} zi7Z$#(*DgBcO~ zz7xJ$0T`C*qu^{wsPJmg9d^8u5YpqQde&a~p|bYvjL@J$(^!sRgFG?+wy@1Zr}b~F zz7&gW)11>d7EGRw zBQYM+mK>jqLZw`u&a9@J0;09zZXf+Fiq9w-T$+j-AzSYY;3|2r+E*@3{Q9zy`VMr*&GL%sM=-S9vPQkVueeYG7 zE$4(YTXg=APsfJ@)Lxy|Ax%+djaRmd%V%#|`7CR|M=S72P@M9UZ$!GEn=ZlpZDP!N zLRXq<^JzwYW^>$s(^WF>kFkbJ-gdv9e)Dq+zTdo8^T=1i222=qa539awLa<-*KyWm z&~iM{{k@j7B8&k32ykOsN$<8KYxWcz+wMK}gaKE!0?8&Ki>lD?I^=J`@H3s-b#wQ{ zO|3f|qvIYWi8tO7%2_UbEfQ~L>CklXQ~ow^aTX63jjDf;l~UQzwJuQxw14Xe!q7t# ze8frvAOK1KQ)n!q>jr4WlB**A@B3HQvimlJm!bB912PxatvB0kqKQXlIWQHe^Q=4I9f_MIt7Rd=OVpGVEE9so2^B>w0MgB^ zzAq;KuR$)q&i}Sz{12KZL^5qfthyJM>T-|Ybp5pv;ovwp)T$JG_+4%#KUXHz;S>4i zi_ok`2j<%Rn&Ir26xINh*M3sSCP(w~55-O^@fP&edyLb5za(qX{R4MjBU>#V4Gy$q z)$@?QT#o0K+76s&x~xxufib$VvBHazN^P^lCQlOltvr9;!Gl>&yt*iTKtb?CfX?*v z;Eh=k9sUpgk>j)|#17l-6pT!4L!U@`m<>rT8%hSjR&@^}lF@eum6v!zq(H^D3c z5Y-{`1&2Jhcuhh^g)aoNr8R$LCK3Ry&Z+b<{kTCnPOokC4b^kJVMs|_4aI_!bDf#s>48DfzwN_ie zZ(0BSQM^oQ@4%U>;fsoCU&k&1Zo!Q6k;;?5uR=_z;h_sBx;$wT+>{5CTMfQP_0I+@77SMshzBDw;K zXZm6lbUhXfE+u?sP~pTd&IYutT zolBIMSybld{lBYA%4d*C*0TAL7{+%w7mMoT|8HobfC4Xsh3tLtq}8MI`yTi2_^N)> z)p*&!BO7%=p7D`Dn(FI%3%8`SDn3KTtq#K|z36(M%5R9RR>4xW#q?GeDv4O1mN4PS zPsGkEF);MbQ@f^T!zMflcOUp9&}EHql`pbpD?WK}K7adFz2Qp-{`Bqmgq&2si>;6! zaY6J(8fU8yOg~#WS4J-H}M*?ioqE|^!-~s&wktQJS z9=oJn-ju$5Q_EKZ=C*U6c7{yTx4TRFVQY=Qj5QmC1Jm(RrV8 z=GMO5{du$C30Fhu_;EU!k_&!enFgxa8CiN3Xz-B-!TXX0;-%(B=4u<`aYb(SCdP3E zuih`;*IVER6R{M&%EGW$-J0+s+&K1mvGn5rfx_IPJp-0`&gX{GfgHrhW|r6LVuuK# z0)Ny1xsf)^$MRE&tByiFFmw|-pdFe|7C=00$}{@a{T>i-vO0OaE6r@M=yvURv+D#} z-@v*Piuiqw^NNShuCknb<~yaX*cdeC4K$ljSI^s*&2-aQASmE8y$Bx9n@(r{4B~h)I5C%rEL?WDSaGqysbm-! z3(E*xxjriR9^Bx!YS#Xzwa+i1z76saNXG(MVbw6YdJE84hHhkN>qiIV_VClvw?9r~ z-{vr%&uSFCZ1gB|dDzux^x5a)+>A_>^}xdL1t18G5+hU6lBW30{?!ubqtTYDy^ayS zuz&~yUbYh*#;KMf_(r56C6FGj3oRF@*TKl3%eXJHvvR6f*@HhtAEu^K+H9IZRGwr7xtwlLJ>__ulSPj4?Mc#NLWTxr#yR}oO>)T@^*4#~#MI|k z7>MIucg3uR|F4tua^ZJ-is6s8?Fan0Eqmzrt){zVnxWYsmTn1bM!4fsB{q(b7Dli{ z7xh&lk=Uqiz?Hnk(EM~!O)6g()H`|=oijiNFCBkgk-;nM>)kL}-|qhC#^1~>_;?T56KE0YboE2hZi zW1EG`+UXv=yv4qS8*9SpiJcqnwPL;;v<4VoM>P@6!A!$egqCIH82z*kTX-v!`~`}= z(nA=kpp`ImB~<84|0d?bNy+3AXd^3C7~&F7H92;ucB?ED8KL})ejFvB+wmkFu7y0_ zc(5w{0BE|%5>3CmKeN7r8mo`3XbSE^WUpXCFm~;#^a~xsPjfW`Wo>PB&5QoRg20bX z3xR(eu?@eKEH9h(uT=u_aJf@`^WGc_c%kWnsfcbj14-7bT6RfB#x_tKDR0WL%GY(9 zP{Ch5j!AOIF$$eocFYBzmzo-L8pqW%(veKk!K^1_KTtR!*y4uC+Y4=bdNm-lOdHe2 z^lj^DW5+^MVSE!jiMI%0d{x6LXc1o?L`9p+z=aW&HJC)u?bacGzlrQWGG<2JE?L!V z+aEYTy)vS^L=G1Rl$syfI&DMrPcOz>^bRdC_h zYxh656e}pWWsnu(@F<>e-q^POA9%cAl|N!HqZe-Nz91>M6aI#2$N7k{=666Pg4^g3 zM~jEp-fAomwcg&3YK&BFo#$g(_j5Z@daqP=@sM*)(cz`bWKCu2!C<^p-q?6Ai$U&> zN73^O<&|0%)k5zR<5-Yu8i!;BadOu$FDV!1K6(yMsot<%toB*pKh>RUpXkHL1M_m5 zf`=eA=c7}0-^ogp*>YQ!20@-$)xSD{d3Z`EyAmPvl-G_`68yrXhXqT5FV*<-=<3ds z{;nY=oN^m%wHs_6xH5JAi~tYEE$9%Y+E?fLmHNP_si;Zf`HJjlO+| zHy3`v;O;s8c5w>ltaVp6_G$K)p=VKni?3hK20YPy&-1|0b7~c{#3{cl9k_9aZd1*7nOPdpo%@f7 zAISB@b=MAaiZW}Z#KIBznlgfvJ%OaTiZ@vu#!*F)n+r{WD?h7pL0;6|dtz^6_AHXW9r%mWQ+T}^39%0yNk zqYB8Ti=<#xU*o^=`z3R#P2T{12Gp|QOsQoPMAF}w^uGGBT@o{B7AEu4XJ*0dH1gwL zV(nEj?i?O3$dq)~m)B*AcZ>WF#>Tow(pk2$#!cx3F`>;+f#Oe7ib zlcFIZ(75&CZe<(;zI(!{=!?HSF2SJ^lGJ}U{ki1)ghti21up)C%@qc_O-ChriL||} zNK$OwT6iHl{+jCFSTL~_zI&@!O8+8!La2LEXqsc!k{po%B+|CrRJ#{L8o1@@a3T$_`s4i;Tz#2gCvqsheV#c=kbO#Gku zELX8PQ7NigR9CR3S9`TVJC=ciG4npJr`Ss7+hbBH4K4j~NMlyo9(BYkeuI{RdPBMc zuDC^g+B+y1gX>@Fv9-KpwQ}gfy_C%IT~90U8jLXdRf?L`w8Wt={nDx)mbC`?Xy!EH z3HX5DpoADN8P@W3G>P}AOzu1+ZfH^jKKdZiYfI~K z-`dYRVanNZq_xjabwW5oqZ1k*4AXA#{rHf%z4;Y`%^ell9|iC%l^@Vbf=j3tg-4uMhYxRq->w;8yw) zB$RZsOHOr{?cZ{cgFn7ke|&_OllJv_?)gfjrYN>9^Y|cwwx3d=GlHmDDz|xr?P{SM zLQiJr`8Nk*{3t1;`KAI;Eam!}9f-qNd^2lUv}r7RIHL0?U=vTx&h^JOfSmICMt0BR z#&<~;!EzTm!f$;PM+26?A-Evzv~_Ejwa{<_4z0|Ge%}W;mThp8J6|<_J@;HA0w+M-KXnpvS7@p_)N2J z(eOKLSnqwcrz*nZC{mGWq1-IYPTu~<)hONmZr(CKAd1ZPAQ!XdQSwT-A0VW(D{Os* z8=ArRtk=@xDCbTG!Yh3#GvJ3*a(VCL!Z{Y3p)?9^IV9!70PI@ob|{3CvUiUJ5>Di8 zGfSuU8m}*)>ZX!jaeB}WrqsfIy!qAo;*t$O-Yv5qFh~DdeEU7Gxp*JueXyP2(Htar zu%4;ZA_kc!3D^}QDWJv+$xL`4x|Kuuo*vBj@~EQ|a{4V15@6!$)_ zZ=_TKKjg)m&PuI>F4{m(VmSwliQ@cI_ z5O>sS5rV!DTi=$!+t~@KD0>UXqg=+p@BBDC)@OK|~S3T~_Y`ZavOkZDW4`;{0JVf`8MhjNs4#Dn6k**r9Vzhb&wgh()&FnVLv+ z)3E7MpfpW|XI0Np!)@KM%l2{_3JUM`O%#n9odenSB5$pJoCj;&k&{%oqZ{%fGn`>N zj#=7>3`y&nHlomKma=>RUH6#FOVa)SZ9NTVWyQUkiMhJqLVEIOng_A}v~v}Z;YCky z(HeMs2WfjXc$8-%<|mad?(N!=9C%${Z;gE;*l$8=o0kawS^hXA@scs*?KCJ!^&_0J znP?uJLnp_kqVOcd$BSjJ_Tg!%n0cO3FR!_tRawN)M$WdRSW!Y55B;t#*e79KC3rns zThwhSP*7#~+P>v)Mwouu?r0Zxj&bXBUZ)O?QX1#u*A4l@ZPs8DHB24-%M1P&4&iK~ zp8M>vJFj4-k&xU#)-+K68TB1X?P9AAb6ocTbySc(pV}$W31Wf0&ZLN6Nc#I9ODVM3 z>150Nq2;rdXDwH}|8s@7`7j_gltI*>&9zU$rhLPy0dMdu9oAv9)XMyJu4v5h?EL$z zqwE~Krx#md?^Is~&OW$tp?K>DxV<|tbd4B@otK-KTp)~DLrG2r8y4fm5pB)D_ZtMp z5sM3kxK5Nr_V1+I09+4^Xy~RElwxYfR9>$*Fmz|G{xjKHQY%>d;(8EP?-ehyg#EAs zOiuf4$t~9|@iHB9GH(ey{Dk<;>l(Xr`Y(3I?{34+0o3vNgM6uv9SjQfC|nf#PayTb zH@Nu`)nrIz{x#ai1BEsef#2mxUROSGQXAZ;(wB=D;=S#je)>;zt;VW{UqcjcW)zzu z@Om#8fiWQ&QKRfQ2sz`@6c+UXELW|RmYkEr)n0Ps?G9iSx-#sPFZB)_o z47VTFma}U*W%7UcnUJ{47Iinwtm^g{NePsEwmijshHv!lN|jo5eZvY;G}J`^7<=+z zI5eWBY`&#Wh0{mO{#_feE%t&jCW<>?wUzCZ|NE)7)U-mmAKmb08<_TUH(!PF0230A z4W|T~d|g7mY?F=xBJ*v`MfDCoLz0#}qkLpgiM z*Zs>8J9><1D_s6Tt3YNCkD}9hB zPTxfiXVbUId(Qr?yB}`^_?Fx&vraC3UjEwul+Mlv+YKb}*nsCsBE5zgziLNghD(gc zh+oDXC#w!S$G)cVoNKYvNTA?7n&U8?XOZ$9W!x5=!_C^}`p%!47*tU@5Z@3?X#B@> z;ksdNyy{b9Y1AVhn^zh;ZLN38iz)cMZz5J`i%eBZG7wx=t%(k{?#GkA7unBWzMD9k z7oKy|!8G(}OvvA-zCEO#*RCJfPDP8tOlnDuc86FBK|Ej)?@nG(M$F=lXI z4&6}74nb{uj<&kq-YWhPgiH)RyKJibBCOjVO7Uz+nb+2 ziOHSGCk4vlvXX>}Dh|sL!s<4shZQ zZG9NbZde6`*}lggw0FmHTkdq(>iCp@wLvTl*Q`Yrs)ydE_iQZxxtDj3v@8&rn`GrI z{7Q6WuML1{ZzRO$j?dNM%I(n%2av8U_;@@XkH39cA2rJ}WAncqh5yw;8b+)qPC;y; zXObVrJU-Sl{O?9W`9e0mT1$P`^3Pcm=9dfpwGNv1&xLGa0xZUfOKd&t9NH=}2Dm8| zACh=910p5Ne7RkwOMH~0+qt^jp;UX?xicGg2ig2AjK-q8n6SU30@N<0zVyot>|siQ z7QW>u^~E8oW-{>A^+Ms7L3fRvEEOt1=&vx5Gz2huB-QW=V5-cLAj5m3V`>}2g?Qm? zp(te2D)6_rjWF3ned2<)K)Z>9{;Hp^;K^$V8oQ65<4FI1+2dgZqY0fq^q;PEE;xf;HeVK&ot@Z2 zCQW?k{(NJ%FL1i)oU<(SvzhtdIX*u*ZiP2_eZ<#bmEkO-chjGQtA01caJ-zKFV;-q z&0m#jixmXP?!TiG(}zW9e+`(jHs4Tut*mhOXjh_k3$$8^{oOAP+C8WuSXY^opf%Cb zDxGOgGBot)0%;rVq?_aefV5MW1$6J12SB*qp;0G!C&Bx{u6>Y;%8L&B>rhScgBuNs zw^MZ`e(FDt&Ny;=UR`{C8SpC$1e-*UWONyPS7Gj?&YPkq@1TSSUPYTM3#Wxgj%-^kAJXQeP;8-F*Oe3_2sol*T1XeTxVYRUQRZ7 zDsPco<*U?=$qE!oWz9QA+i7*fnM|n6aN>hu$@!G$h?F^t!cTe#E;}C2?>OhWves6%%UTyzyk@8P2 zM@%~qBNSD*920_r(j=>Vl}L6h^YTCLH=z&WYA!w|M_0(~*JT<@AamSItoji4dRd}p z%0#hF<eh~y1T#y+SP#1uz|F$Et$r=~xt$9?4G`gwBpof|xx9KZM^hoIzZ*$09 zXeW>tnuG!q7cO_ygDzsdIc#@PsJrkgmRjPk=mw_Zw^6O78E;th`2+NS#LQ%&4B5z( zIh>%T6WQpW4pj-YZ6J|3}9V9i$(6kBfM*;PK$cKUTaTg(hG8Y zA(}9+eNoXWC?0<6ln#?5`M*7NIN!-vQSB9*=XgC&v$_(?H}SlQF}{HsMgE|(4MCJ% zKf`YIpPWG#N523OhArVuz6KWkkFi~klQ+-9$F@xsjH(>My=UpkCcM8j>W{YrKn*tI za!U2r=SPsYGmuAWz>d1ZtC^?CyfV)fpUW&23aQY$h*g%JDYNEWVDJZ8GnAA1t`R1M zV@V;{c+(OMpdTcV;$Xom!=bCI22?cOQtgsjtO(n10rN)Pvgx#)_VWNpWOgr%BpC!x zHfb6$P@OF{yXM;o?#1@Q)*GgU``#=Y#}936Ut><^v)izXnU4(R3M$K+X+v9%y*Vp&z1W3z%O2B(L@;iG>pvqP8*D(vAC8_7=#SY2$9E%~Z6g0xM z;W_o$9l`E*GEpggxmsVpNuU@qX~Oef1dmL?k67s`u6nORqMcf=pmwO*`IFHhNSWo) z=f{CFP(-rWIbeyT^dijh<@$+e3lb9rxi?O}x6pRdBI zn)XMgog2B)*HxbBya$^wuz6mDM-I%Yy~$v>h!=06na!z4IsF4)M2o;f%C1Sd&IA<5 zY2?R*aB@Nd#rz>8yxwWmHp%M{7WyQ;Fy7`Xffv=8ayGq))$wl|kO_;1?x5j~hg|g@ z&-K-U8@Uv-%13W?zLS<`Q;uHz)CVrA|5`s#v%U+)J5Xt|wtko8p8UlrzA9%Dr0}ZG zum8-pO9zLm_-~akMZ}bK;AUQas?`OUK}L1@WfKFmRu$`?^+d<;_1{p7bPY&09BE zFP3|4-{>cbS=8B!U6Zf5vU@DLeG-6NCU;rTD-9gNIZb9*LH_QR=*^sIw%rLEmVspS z>@SJjLdKw45s;j4cD@J2}GiDxZx9Jk>%`92jTZzlM_kZovY%d=AEG~)6 zznvqS+TNd?Zw&&6f`n~K7-cS&D>$qt?U2LfN>>jVx3M!-2OMU4tj3dK=oYVM=>gD? z9IB2hh>daS2N{2%c;I_=H2e8ci%_$nPZ93)`q%W< ziHVTYM-wxYQ;uD~sN#R{QQg{pR-uys_Q{&4_;LAh+HP_G#gb!iO?_qc(TeLAjPbq!iL6|W#Y496{~J?KK9 z<%gap;uX=Kt{lidZRXxY1V1J~$O`8|-}$=@JW8uMsAe9o1?_LMA}+n|;xDLuy*|f` zANQ@7?zlo8s${NT^(WluStl#zkDEdlAK*ZZ%5S%?UhY;-n1aIYNFRS|1u+MI%~_$i zN^&Fo)Fg@QWcXFq`(p#-$DOkx9LCv2`mpJ#eM;}63Yb}GK8qU*BKZ}&piO*s%T)3J z!4Z4d$omC-EAyM_*R%cnEVe+BXP17okpSyl8motY!k67qbe#;8Uh+$0Ca{L!_0Q@1 zwZY8XIoz!LDK>4Zu72+?>=hL?#U+rgtVxvKmoHQe9$}-PS0mjP+So%MU9^cI`e5Zi z;W**d1L711{5Q+eFRXLOwzC|bWZ==Dmbeh8-V4Fc$ zug=jTvGP^fedhc|jBOg_ipVajF!SZ~w24WcKO$G0LZ%we19A#(Lz7hFRf0=M2mChFR`(t2ff9xHl zX1e4?(NRG%ym68&7WUy7R_jT46iVNu#@y{|Y|IHgJg=3;4qiq`Xlukj!;~IfRy0#1 zgs~)5-}mAzB6PGe3AHde5H3Do&( zZIv4}?hVBtgf*p2QkRH`#a}K-gGG|O_Nqr;zBbW`FC7&zkE15gpuoeZ7PP;{S^F(0 z-s)~a|2FbIF{IOt4y%;;!}gV)$gdvnKL&$eNutq}(fz}?Sr@aXU#0}An~7If-tPS> zU{>V7;`PuBf&6MG$)e^! zmmOlfvw2Ps2N91GDQaSD!NmxL*)A-MG_CG>ZAh-xJ zCbCl&`>lx%&RNh_f$@6qxs}Xc??Lb<CO<;B9rI0G#f!+?rqFpeapO{@V&-f1$Buu}S|YVdQT z%f@S-O;BnQ4PQT#aijb545*n*pZ8S)DIhc$z_KP|lD^0?G0DV1vcm#FNdPAMg=0nw zXoqw;jp-~re} z?cy%*&2W8Xg*5i*ZpgB59HwFW7CC87Sb3Vw^MF`$iG}x%#deYKJijaJ&#Aq&D^Of7 zf=X7>f!1*@(>f;o(&c%v#;L^wJ4twf%lxMfPZsGpbf*n^|Hc3_@>)+0$wZ z#M1$KVs<3Fn(Wl_^ak5-ZU63M>n>Ps=I4kHhk$i}!3n9;cKYRz z|FM@q@F2@FqkWN~cpq2U|a8=D^>4-9~dR>KPPpGX3{xjfpM8D-lT_7nUJ&nMjsR{Q4$uBMt#GESOGX6SFDmk|9mi-m|E9o=wlSSXbl*89;=VZw z{^GQWfk6+lBN;f+{Kw1{XTVwkTp}ir?^I%Kb7P>ZlQ1Q^p;|EhO(#-0MF!TR9gehn zL4}&tvQ*%=W>#0deD<9$`$jQ-h|4asu`|MfGdGraE?e0{Dm*qS#k;wZQL`l@^o2$y?rx}o^|pan>|#&R9E9b zm!P!ia(QZJrRDDGReV|qtNIQL1xEKuUGiUMGgY;${CxU?2d`WpmTs2uRMQ9Ev}7r` zCu>to0J5^~-i_^N$3L2Vk)WI|;>lkt|FwnShy^nQVDHYShFdug&Uj0Wc_>?}vFcO& zi1*HAF-yt&9f>9A<$Sc7eDn+D?7G)sDbN1ZsdlLr(-FE|TjsWIwtcpt#UJ`g-=5q= zAiscrW8f(_Jf!Ld+eM~KUH+wb(B_Z1dhW`>!8bC-Q4Ot&(XAd>i9pl?Tw|S$P=13h z#UPCk^eftk6#2?0HPsz)VS2&6O7(I^Aa_UhS8(f|No-)(pgxFf!mE1!D+V6Sh*C-Q$S`psYckvirc5fyc` z@SHtec5vFw)dJ%L7PtT#uK%MR;Bt{p_Eu1}kIL3O7CAgwwUf9PVqJfV=)2ML_iywz zKfk+v!d5vQ6sijvWSeyZ^y)If!WJ+n@3IM05(PF3Ck+T$KX7P-by#V90@~)SiI3&s za}r>uu3A_;QOs$V>@G%ym*OxpC|!#+UF5Y}#%0uXlJe5|=LcV8-&Q_MSrbR6>FtS~ zJyLC=2pM+~-JWvRI5vj@NOGiuEv;M;#ZRH)8}@XS0BGz0A%VdJ`STK|!7g7!)-~^Q znN;THjkgaMPL0bhwhi5tj<+X*ctB6lGGAIkYQ3R;P$8F_Pi{#&Us{as$scb|w#&&3!HNvV05^MfbDOqOZ}6-XTI1V6JQL-~QaThnLan#|6MFp1 zVOd#YY52xk_233cMEe3f|88$LqI$kmhNLKTVzcrY<;aVQ=0F7A4}fQu$O%0`%tv{K z;hWK;RiK7=Iv2ctfEIt<{DwO;FJ{+Wh|8|YlKxn8TU-#l+tNny8{AZeVZKj!69+_Tx+lbZg#Sa5`?o5h_fw z7*&7ub}-s%%o_N|rOJ=$Hlp@V%~`)~Y1~t}davA03HL#-{Ri%q_3OjUObum-ruo)U z$n?f@!%AN&(q4qA(!LvZC0{$@$1g!T5o4M60*TSh+k6&OiKpVC3iw+N$UEC0y+{Q>iE^z;sWv0b?B%F*)$Y;OMe%F?9k#%&%Y?v&@;bq(t zm-DGNvjOhG6P8hMtUmmMK=ka8&5|RY9;da^G6=hpSqwcH4C22^Q5o;jg-Z zOxAdU3ax9C)KZb95)F2> zZf4)mxNfz0g6xK}pzwVCvYqYTfzMMy!j#8U zqljml4#Y_MQG-qNcN(7k5P|0v^B+F=cYF>RIR9g%nEUm)RQl-Fd4@Df=m6Pk@kYIq zUo!c1%R&RtQ&BuY7L+J6F2`AN*_zvA(DxHs>@o4v*BvhM47eO2eF|J55&Wu2?rY|T zhc{2-L<0K9{=NAXf#mUGBwuY^gc8;J)DV1Jz4UmmBw<9ynphYtrY0V2eJ!Jcm9bW4 zUp-#DzXl8gis68~1sU+B4Do%waO1@rP5Rp*C5&XkR2JVPQd&6;+oIh!^tL`t8<>+J zcUYLTr22ftF4z>-I-|gP?+gKSbW?IKGN6*GfM1)ppS?yUl+L9EU#{-}-x_YL=#U2I zT+1AJ&~{iY^4T@bytr~=-D{p-QRaHL{#@*n343+~WNM{_7_#Cn+_-!CNKMJHMn)h% zw#d^*=a-A$=!PokO_Hxu`|ydT^cVtEBj3Q2m+7P(6gG-UxU!?>3;OeZk-w{aF^G?o*p#L-A$TR60&uRSkIVJlN_F zmpZ;kF#6mbNSTH-vVt`N>?jkMQa10)^ONfrYXC`AlXUtG}{yELv<{ z-FDk>3WT5LlrQWMReGpSeY#KrRVsT(*hM2(MufQXE~aEel=cQGY#$#`x$n2rpFpRr z&G2X1tKCn&y>~v$U^ZsLe4_9ESp9ezYIAc@$vD@>p z&O71vYBk#u+=@KkfD6vnRj1NGC45nD>9bn4xxXAv5xLle{iN~>)JSnUpwlM?qY9!UE>eSAMC>_D)@I4<)4ZBD@4R$xBh*tv9Z1#UnAs~ zA>c@F7I>FVU+zGC$eY>IU5rc6=h#?*g`8`3fukzX-QR9)9`S`++zUrv{Kd>a@)sR_8axihe4s# zr;!024>qfYjb?fvk0oM-*CN(JVXp^zEay;Vl94Qt3SY^PElsq;y|Ea1oq_kRfq4;Y zgi~BMe;$QoIUZ$20OSDOwXbz-0{mU8tW?phV-TV2cgo#GvFft^b?Tu3HGJ(~wT^4r z3}6vr=^?9j)W+JlhbUluI{_S$wJYg3i?AU*Kq^PSq0#l0A(w!89|`%Hbq`Ur3pWsE60H1%#aED&u#3a?&Ctw>wXI78qN41`Ij7 zadv!{8)sh9i%(n_#8ChU=WNG`7;U9qgEJDgUZ1)ntCKxpS`u^51K!pS z=jRw^0>-{nw5y+cMP3hE03H)3y$f`XRd06BGzi$H7^W)x&Rj>#-)HBv7xA*mvj1`{)u|Fc*d`{&!$381ci19p zT}w?zLLz40NzSl2aMk3E>`J6f?l(T?5u>TM2cUIwek+KS#gXI`261|Q0pjmV3hFZB z2aAR)NL` z1J*P@C7h6x&EWwKBkz@j5|s&FX49De%tJ0n7t+i>ELDD9dkZ=HNcKDEu~e}#ZTFO0 zuMBLXu?7qzxP!n5$PKNuyg1@&8cG+G=xxYZUfi45QQ1^1YC4C`>M)N;C-I`LS zna-L!#+I6KYE1B~ptlHi4u)Dwq6b<=1kRY6tiL{X70En+!<>}etkkd;;xPoZ0(M6semMb)#?VQb*>B5wD_F2XXwX(!iYRM>960{bJ?%n zR@uDK*X$lad?*NLk~*8FHL}={opqfGnQuSv0F&2pzagTF0tq|l2@bRI+qJAk!SBs1 zs61cQ!7V9PTy77L*w}r_Em5P)O3PoYtuFF)PobG}u0?jz;wWQs1K#q(edu2YMdQMt z44vm&cSnH{^l9FFMf9kQM0pLll}A$|8TTGk&VF`^jl^D<%tYyl&CN6`c}}i&L%Mms ziCPL7fPezAfy+BUp8Uf`^D9NPKkc|d3k&`z&~-6BSqZhaqHJ5%O?%Rpo7xz1RIuIN zSkJf@%H+h10#L?97R06*u8`1~Jg;))!n@R@dm9>Z@%KQ{Ar? zo%#qm;Lv*&0mzecpV8%mooC=t7z`as-e00uQ*4VXWUja~@)#0dI1P~GD69X^EPx`3 z`nL5;GvmflK=#d3jc{p?Tf-dCc* zBf;Ocl<6t|K)uiYIQ>)2{MSwRJ7Rfh;b_Xr_wu|PiB!}-S~@%nc2q+Pe3oB0hKF2E zTXWdvD{Bp&p^U8CS0>tAed(hjd@H@82pKI8ZNsq|S%K8LSx4f}H`@Pvaw<1j7+9H^ z$&%@>4ZOk$aDrz-UlfBio8M4v@0!seqh)5NI*Mwb$7jQ|P8MggO)^LiFvkHf>WD1G zR48v&S$c|XF4@KSMLun_!42o)+~M_^LM|dc@2kC2t_0^B&2u~JT{b({(6l|zqrfE5 z;S*v0&Xy{I{px`#h%!hpOh%qa<`7!5spI~{xq#TVy=of8k?Qjy_?Jrn1 zeR?_=m<+u``TW}bleyNb56ZWIa?!74k(a?n7et;}HZrnqdo+iui$t>?Ar6#;Cd8vr z#ungE4|9K42fs=l^0_@-BAwVnR9oE^yQ;CN7`TlZJ~h-77_32lvMo_UI#;?_zrnhb zZy!En#!Rhnjms5rQ!nCZ$OYz1SJIN5Rg>55d@^WOF&vWhL8pU zX{2i?>Fx$WKyqk=p=-#YyWWfY-~Fw-`#$jSTwv}w=brP;&o}N439?kQZAmc8d`;xa zc*AQ{BEEGT3!OqdRXX+H61ZSeOY$Los({TtCkp{~5q}d)w&tyMjN9PzwBd2Z@!slC zk7ngKWl@wf?Kv0p9{dPrkAdNn?fMyB0;?o*F8PKm_vQ_LWJjAMrX#%H1eI_b9#XxR zqQh#*W4(zLZ>@HT&x=0ZMo5&5JB=k-kIlYY*4`ngm(K^;`jHfsjui?V&mIIeo@884 zR)KVa{TP;-tu0L!<-Rj(KY28`RO5(GNL_n&QZA6cv1^&P+p+s>+3rrKYcacTS;2PHh{x|N4#FO+5G zW@53@GS0-ui*h=;Uq7`i6yZcg5pbwrT@r~sP+{f_eqoz7cg2`54nL&Kn1W1V^(i+U z+>CbBXD7LqJ|mFjgAWGee>R*g{-#Le7=sX;PO)oh6}_b^QbZ6(@^nru7^bA(?5Z6m zhMCA{f>*WB69qS*BM%sW=S@y)|1)48-x+aNR7V6GO%`xsD%=z?v* z*9OGKZ$t*%p;yz-s0LRbqj7m)v9DguI@yn)uf);0>?%=szI3h?`qq?J>U7#wWw*hs z1sN7CS$>NFRohWc-R+`)R@W=Ht`JW?cWoQ!wx0`Lz`t-dVjx|Q_VOHJn%$Uhfv|rm7X!){QIcK8AfL=T%QZjV^SV z9GHDzHwKY&v4q~>vg<()28qF?x}9_rW*7I3TIqm(RE2b1Ov`|BurED1!`t1`yB)aB z$`{O8;uPW}uUFIRY!QK-e~& zTyN(_vfkHEk;l2@sZT+U7#jnO_tt%E@`w+b^6b(cb;;T*dzM*LD}N1=Dw|0U?$46( zBs(jUFf$Ej%^XO*C`|IKwbppQZ!&q@<2+3ehxTaAVGu4|cQgmY8*@lD3$zFK+i9(E zu1ImAixzH< zD-4@jCTh;pnL2KP2C)@;>h1joIA};+i3nr-a2mMHiaoat1UdxAj#(r`lS(Q;G z)fY#0bglWGBcCLF(Esi4M%dkTTQa&Z|LNM4==|&SAav?+w66&%LT8_U;P#x0cRE+l z#@5`(-}I9@zClmQ_dZ`Z>krRUw8v~gY6ETb~v%$p)c(nst%CjM3(RhJ5Z$G z8uUmrox9THmagcZ()%kEh?`>k+&Jk1vmccNt2nF*p0L-tH1%n`C=X0X$gN z$w9lni8m@z;PNv`pnNu8sKne!$_zVyGuBk>C#nNn$F&AkXpcnbcF>hp5{UddxR>r` zPsfm5T+^nlQ|e&V@A7RPkEj$>z>rhTRiS9oER!qs*bw&~E9SHwQFxk}i?~foD_GI= zH6Og}Mc!a$leK3nXL3an(qL{u$~=>oj-2jXs_MFI$Z37TN9P}xO52e=Y-*{tdw1bp zm?o^LbTI802O3_`i4%L;Ii!%L&Q!Ez(hHVJPrlzIwxfYO*4_{_^YCW6DC8;F65e0R zf6Dd13w&STOyJnXxIvNDxrE<`^~4(92KF(OcEy=N@_dxwkeo~XwIF4tvPHL`cbzO& z#(l_-><6%5PYxcYWyL!pr>L-`OM$4~m_!=OOd3i^w z5;F7$0SMUe767y!JUT+wyLz>;)4Cp8u0KpMxT}{+z*W6HtuRizoOXTvN(vSvY-o2i zcexiO`J%OfSRdr&HN|ZVFS9QB%xqBJbgH90S0p>MB8z*=J6Uc@ULNfk}Y*4NV! zshpn^fy`bYp2zHG6m83BwH%XiJn9~NGkO&kH!mIiE^?=p@hU}*R%5QO;2k2+!#CwEx}>RU zKId`c2tkBwZlaBK_WTsz&1(5_zCILZEZw9M$zngP4~Pt<@u02Gv4Pg<|A#8w!Art= zNNCWs_}~sU_& z0vm$nTk`8E(B+*=ARtYC0Z64pPS8FDaavXH;IiYfm3{#)D11Zi4k{*Gy{C%if;0FP z3#WhJ`T0ahU)Hb?k8ZG{y1>a37>K3NUwtPwaYs9@A1S^*V(r?u%aVDk_okscaA?TA zg<>PZ-zqNNrRsBTCH}=I_ll=M0-qJ$c%3mn{73@+#T3RJ!aUulHJI2uxT*no0$lTx z;ES7i+)h1vg+6I*w@!Q9$OJAFAHbt;IOvz0bH?%Ms3q(3xJ_hOzGf8aD+LjbrU!bF zOExRSqp-S8C9MR+queh^;49L}Z)NQx-JN^u(efh`@w~sFw8^GVV~!$dcE=@y}tOSlf>(ZC^l9gLwKe0_J=Lb zDAg3^#b)Eh)9ZYNdj9dXy!#Taue-Wv8qFA3uzQ#0*(PU3)tf_ot+(q&$;`z?qWeU= zy?C5H*OWNp=K!WU8tJ6O6`#IHq5J~MTOb@uAbI0}Q^T^T>6l~eQwTp4t+>3x@_TAlqH|dhWJuGecBAAxcTcB8J^Tu?{Su7wx-u! zY{LoI^*|Ri4{6woq`Yo!(40I$H3uK?<%~5E6uU&;@$1hEG=!{jgTW}(EVT`z6RY9c z(;TM*{G|ZfY(9ZPLWC_GN_6&-TYfBkU-T(vB#;rB&DTHBVDa2F**sd*sb+ms@Wph) zv5~iGv)(fA_K6UK4d{?tMwX}%&_D6q5B8c$;o0G2kgWR-oD2p!>W*nBvHl2nD)x|bc(B{4wDs%DEr$pCoSxa2B*!P-A=H znk^Gkw$99UpLdA_i27+2-X!gUs&;#4tljoraDc;1J}5--tT;&Fvyu=FMMoE?XV&Pm zaCXH!#VZkadyl`Meh|gtZZI&0rdlE?&tiUE%iDIn2B+j5cNlQ zt>#n2*lL)>_m--;&Q7~Z$@((}Zs8fBRWiJu3soO4y0*qBJc~3pDgu%5RSl{q`^E|*!ddUDdo_Y9$ zY`%+hzH)tRxKgSOzr0Zv@`?TA9Or2Duyt54JWhGyyLLo((+*cbWv`LvJLpT*2%z}} zRcxrIBkTAQ+A9BkzGIcRw9Gha1XcrH;eSm8r73cGGU#2j1^l&nwUU<2R*z}I$O^r!{$6Y8kIwwWe-xi z(Xsa|kgjmZNfRkExqEvUGDFGSMT{%f&E#dCyv#AIqG2@9GGnFbBbz*bB!42~Ul{ph zU#ba5Q96*6Gd>o3Cl;&I-s>a%aLm>ySTs+HY``}*2Jcbt1N@>lGL@#UTBK*laHRX=aF}XkDUyLi6f%fYJU+_io^KB>NP;!Nx$$W`f@3JdQixUb0g(Sv^H+o%@ci9pkuum!! z8GW67x*yg`ALR$oZMMad$qg1C39V?PXyKBZBagwz=byD1OwmL%yDG-G-OpTBS-)@$ zSD=7|c&d>&4NRTKLJY|8HNMU*RmkgKPgpBQG&#o49?~4YCt|r-(7?cLJJgNrnY?kq zI!?uQ*9#DSRH!BC%E#9r-wR%aRO8-+MzPLQT=4<#CfJc$KUBH##8wd$UVT1!npg{L z5j>W7z5;RwCZk`9x={1fwC=zY@GT3UKVoQ?`){04|Fl*V?L zUt|HVV1E?^bH*0ouf2zo`;ntTv|M4tqjBb+}W zqBaIJ_I~t9=JP(9d7eK%|2YYG4|C@8(2H3b1jJcg`?^Pee*BLa7o;BVHAjbKS^W>U zq?1GrKQ?=_|B#Bl#Qb3cHmOqdr%&a7gfd;o5z%0z-1O_wKzZFzCpBt=OL_ z8L%UN|0b0}SgH?{d(Z&Xo&52i1rOw!x!K7}-Z0WXcEW`Q7)+1H>TAU)_K@^ymC?}O z2OwXlM#iq0wampLK}G!E>JWZi(}f|Er^I(@p2A;J`8R?fWNwfCoaWbLUxN|8AC9m? zpECR|hya7xTm7!qZG2R{fbS7vvx>aX{oilZ69pDNDTBw0@!N=ZOIrv=UK;#9`c*pq za}$80@iV?iaOB>mw0Zi_TbKeT(o`)?_hkkW_odZeO3Id-I}I`UE7IxbDRbk@O~ zkI$b_&Hsg%y=k$%VfD2ANEd%>_TPUlj3W=GNv897F=qDYL(=^kBXA37dHo$qJ=up` z`H-u;nd29HtWmM${xPV|OC*f~YXXnqKxieDQO@@tQzD?ba3C^est2Pn;mSLr!AgX% z?l6QFt3VzPBWE@I^BATrmGkRv(8#HS&{uZ}e#zoZr_AQ=no-kHJKq9CypfL*@CcO(Fb)J_dX}1|HN1aUNVpwWfYhc@{a6xCIE8eV*z?G>%{)Lka~g$5(6F&E=0pn1If)Eb5O$z-Q)=Gy*N_@EZi!SQ_4b1&23izk#(J`Q!$ zwicRK)JnD^RPOhD;IjQAJ>({cH0$C1$BR}W1(}&1eq-BATXbPz7M--bhyPfI$K5*K z#V16*Iq>~EDd-p=8QbGJB%9Uutrgd|lZYPP9ZRH;kMA2`U(OFZ#-pVXklcDwwL^a= z{H9><2reglkGm!RW}WOL#c?z}7+mSQ$3Ip9$M$xy4%E7Y7w4HLi#wMr;Qnp(fDHKA zWYW0#2`F+ziN!qfXU=#8-2XM9ZP7!5p8jPgJyK0({rPho^jsid7f=vB75mme@!Jn~ z>-+xKkLAg!Rw+BhwCAL;2c@qZjl9Vh_L0*_D%+PYgRp#s4Y1LQi41Frnoze5ze%#t z?43XanyO&yC$Z!}^c3q@DwZMoR~5})D+WZ4Tk%`S_ER@N1t`2gr+!Y2N0$bpO-jy4 zL~+Eyp-4~Oc-Kk|NMpTdQvxHPRuCYPz4u7yxdra_r$bzKztTAaG#Ns|LxlVf`T_pM zgq(smFu+fqo^1ZEl^q)xh6vb0WW-&Xp_^&tIjHLq@g0Fn@%V&!sWGi0vwj2t!8lML z5ghSfgCY2QM607pN-p&Cw$Aanj+Q}aca==j4gH-(nj!$5yPM2*^s6%auVcNtXcI-( zDKWDK8n3Ne##=2-n#7_KQOpcD@*4?p)A=`4uYbv915ywvT)!RAzQZ5nJGGVuY?Hp* z`#S$*4j;hg(}PiGZ#U}Iu>E+WjOyq|I%B*n2F@RAI+F$89y9saIK zw#F0jq#n?Xu!=U)zmdkB-h%CCQsnbf^m~XlyuNAx9=pTju%OF8uXWEYyGrh)G)< z<3!l*N}p`|fZ#U;AcX*=kkY(+{1W-EJN~@z8Ic48{6-4M<|&8&?^fp@57!n4aE3M| z_x*o!79tBJFm1^5!Wc(9VA@2p2Ty*Vmi%s7TcwxOf6n&T7(OFnche@>$dfYpJIJSz z9R1FZ!s+C|>0?s*|A(DGzJP*Ep6>pQI(Cm)gqvmYN$BF=b(*&@Js`K5aAjV7Li>IH z0M(|ziw0;&lEdxbEPq>gaHX%{Z?__#)!iAlh2IrOe|wTY&X+#{0r{v8oMK_X@m8dFt9>djq*<+_uLNBmvoU7R;n=(+WM6I{0`2v?y90?PExuY5VC&># zTQIh~AWtD6v#bJwd4D$x6^;Pysdp4QjP$#p_%$moA26T9oS;fI13=~}WxnXhXUX`4 zIt?m&55Ly=137)BvMkedoo`tk|8Z)>Q~Q6W4;KJ@Fr<)q?a zgH_%O80&l$=phQ~edX>PfqWfdS6<@Eh#InKVM&7MPxco$W(g#P|5H8OF$W9z zf)$dRH&{r(W%kffwb8P8-NYE^Z!onxRHWKkc<9F$p=A7K+w*Aj*@i+tBF0MDR%z&^ zFXao>tguTpy8WvzL6Bl#mjFw?Glmf~S-3VGF^!N4?n+^xTKJPfxSMAA3wMeX4s+><*e#_5fFyAsN3nrh9p{7{#JU zDCpkj?H+5(?0b>zC0I#h6!t=~U^{6x#C$7*V>2me`;pFJiYku;suuS=UUMRlnekj7 ze?+u5U?`;4BisMbKU?F%Xo`z^vJ)r=5|s{V*QqhMom~*DH;{tz zmJ=DgGVjrz6nI>V@=xmDZYa9#bknY%qo0np%d(Ast$Em}sL^Sw%Q(Md%BGi*+e$29 zG*9$BW9re+5OWqc?ne|Uh)hwp@*xYNc;QWf5{gY zIz)MbAu1j&*L1JQ%=qXuk>=?KDy7b0jV~58nN=PYS`DS{??P}rF4B2PXZG)}YLX_q zeD+Mdja3xiy=r#GD4wgAAsdr(iRx>VvdW85|Q1i;=It# z(4ooFcyGp0O@=otY*kA(C43paqjqVYxWSkyWDs-wodj4rH^cfL*v(%87g)RS;9+L^ zXWsviHEn+AW{4ga+4a528sCQj&w_M#Ifytzlb5Xl((HVRyKagzxw_O@6mqmPbGklm zS8sSuCjDH=Lfj(FU3rRc=0on1d=eH*;>pe~X^21yOr&~9Ycs`P@3u90ZoolT4%wE} zxqL6BoJ62no@9$Gk$~;Eq}z%j#eF6wUG#qn#64TDX*mIAQ{l!a_y|_vfA7V-`c3Dd&ed0o72lZC10DfCWIkKS9PYm6?|oN`)M{y)AFp;^hZVfqKi_@#DB-Ud`A- z>0ho4s&6lX0S(t8sKrAde9 zeld;MKm0gXjfE?^QNnbgxaAfnmiF>0b@@Q=A?lDptbvjrsD8oOy5|>CR08V`1f05-WGlR|n|1nfBmLd6Qy37HT&f-pZi?r27T@^lV0soYAn7HbiZ@f`6_b9?x@P5DFFTE_N z8e$>{-oHG|ihX^f33)pn&oiU?@zsUQ4 z7m?C!%E*rUzUPZz`SpFpQpMsRH}?JfDaW-L$&=9i44?*FC2(HGQzRHRdDIZw9G;Zt z@WiB9b|1EEMhO%0GCAVI)hmB7%e&`!VR9l-@3+#dR~(oZf4(yIb$&#QOTc=qG+C0$ zJ@jjVLG^n0>xkhn4>$$72mKD}1#0=3*AHw6flZRAQHfiSgY>7N<__>zJJGZ%2E&9) zwV6xo!H{I`={u{#CzA#nhe#4Ec)%Bctm^nPl5x{*Z~9ir!i0!vj=S5Se#9|DQiUO~ zz1|i(V%F`7^|aPiU1cXthHxe6w=f`~xi1@iz<$8dijYkH(nIc;^fC^z52OqM3?(Kfd)oNT}8 zV9iD`_uh4*6RFiyYl{k7$AUBqB6x()G4z)|{}25t+$JN8?AT+I3UprS+?L7?jhVSx zUb7c#Znf=TbTjQ8#qHB$J?I|#F+?QA4e1@s9=KE=zFFo|CAeSJsAk2rtt(gP%&2S!eYL@blnG%f} z5+4CJhA(MX2hLh8R-}OWp-}Jrt^iL@9Ub;K=X&?pD-U?L{1wh=Vre)1Yu)h9QCfPA zC{NuhJd(AP4a1XZ1%l6Kf&JA<4r%Ty+$@Ga^M3ggpM$;Mn7P6y9u@f?Vwy`D1wHK?~Vu9-c! zhFwUj8sv01fGQ8q+S1?5EHTyTSBRgx+MqvxDa4y^Ozf;*4${yLL0bh2+q(lR1HmhA z1;EjH?FO5#+}yd>5N2-|-xfU83GPJ=$Rkdr9KNkte}=ay5ay7FUmI*o+U>w}f3c}< zBkPxoPsfXNY{{uMSJx0w95^uoZROAMl-O`kj5;=RY;BBniA%wd*B$6|c0YAje>P9} z(ZxU?X41S)aWWCl&NdAvJz*CR^b8gJUW1Jmx+xH+HMa9kbu^|^fAs317g5GRCmT@3 zJ>Qm(?&xVJG~ZOeS^R=N02SbDdFUgn>@BxBaaoJnp30}MBY(6HSG&gn(2Zx#4k}ly zh}6?pXc}m^x2>M9|I4!!Zp%V<`RvUcsWcKj-bc!czR%2}{p2O@xX)0k2YoWi)^bg) z1_>17rzT?+UjgJ})|dyc;AG|O#{p;k1DRgwR=)zrawQ0QkTo}J_sxs~vs@S|4wqa;;pJ|Rg zUWsQqS60nR!Gab?eG_TNr-M8+i=BqYi^)m23hjSPIyo@JhC;kQk!Sc@Ec z4r`%l>xZ!n{&v*fYbv#FPd*zu!&p+X{Af?lgn417c||>byY&^lHT#jp=9=x% zbHa6~b>0$ummn|UrrE?IjrY5#zJ!q3k7azp#;n>AtP@W`Qui?f_ltI>d+=HeDnZvC zbyt4}T{W9{`wLRnDK-#Tz(E&&z%_}Z_A*6l!bYBHDoZaDk7Q{~;4)&@kXgIGa~rNd z$Oc#$iJw3uvKYbQ%>OSEmPCim`4Ba;(+ zwuJftrqLgRu;K*Cmf^o?tlpNuoVYc9p|}ShxZdRuF_5Ty3$4(jZK$93P>DOtxBT+4 z$8b3JbQE)bMjl<#?jpL}>hZWQobV(DgTN}KYKo9+h4`+mgI1M6_yAsJa%lr(z1y5r zeW&2qd?vm7&_9jTVchTNupMH4&>l=px%wqbA9l$L%irBTA3f?leN=AiN)F*rm&-QM za^KRg>S-pmeCM^np8E|u}@Hu(R~47D)*Tv(|)D3 zxkB*Qddr-l?^9%J!M#*-KOgB|zLLDLb$_1G;HA$As1#NTNVC#hzaae%|HDo0SC2-e zx38}|=9j}Z{kS~_N4Z+XQj?Ef4)zSH?v$u@w=!P%f+@NWRLFDt2!9%6+m*f%OQ>`! zu-on(TorU5nONQ9I(4#zMLFIE!BnOxJX(NUM2pjTE3eURuO^v|CWX#4T{s)N^}5lR zP%cyct%%m+Sj-16lX`mxm3Fu8Kk(mm5+j!pmaf`@dTr?)t zBMPYA4{lze8mJ|@RDJPggzxnBEvSj(hk@JEtTI26o@!m%iYPk{2EH$YKlg7uZ)#5S z+z%1OdEEJ;3*E1byoQXxvm)g%Ic>asrY&hQe)ix^RZBysxsT)?gmcIEy#1tChZAz@ z@&1=aTsIKxqJh&{`g^&CKGTbkDsYbcR9 zInb+gz}exi2nI^FZB`#LE#_9!=)FzvOg0^!a>!jzg4Zg(`D(GL1ejh81rJqFRl&V7bOou;*0tz*p8&@i?3T>q&F5vR_BxT`It*AUKoMg9(p z9G1r{Rj2FD2aC4nBc}#UZEfIg%I6v5fuPo?9F|K>646o$9tZD=iqkN#6gBs;kxt;M0~>&GmP2@OXQ z_?y8m)X~(f)bka>z#+OvfF(AP;B%_nKV`)1CHT~@4GS#>;f@a4s14EngANqN0Q8)I zV==Rotlf2(O&s&?GYgW)Zy2xlPGK9Vm!4$@Pj`Q2bA=={$GV*MwX>L!!Jy6)-#gBZ z!K7KKfer_34xSrcT(`2j0()PEyB6fQhE`CXNKO-c;a~SENqGZ{UN&bAoU0iqZ8>hc z?{+*$ZZV&i`(i%bO}Tntw>#hn7YoO$cKEb)QYNbwe)%a$th%~kXCZj@=!)x&djMnL_Zo z9=gtO*WRAVxmhpG#w4W`45kLrOna={t_w}l4ljgfjq#wX-awavx4&8M3qGP(_vuP# zs*+C$%YgyiV_cUYPmc%nj`_mY?Vil`Qfx)QR@@0}Hp>0%39zZZrsb-8xvY1b~uww3>M>&bYbMhMJwFm)_mXIc+l31jw>NR?UsxPQCs z`qR^_QW5YSeCxdYJE1yFJA7QbR{UQA_8)9~(qQ=21L4v6`9rp*%MJ!jN_L&M(i!Mp z9`9nl0mrX~RBy*3#ef+4>_4DM;Wl~zD3bR4P!8mrNPU)d*c8NVI6!`1@_z7p0?IH-g$b3^LM zT{}!mE+h%EKCg=8o2ROEm2byT+Kd}EGdK$5uG4LT)kua9Z` z;R_F4ouJpk0#8e`d~^rLVl!qQ0&UmA3JO)KTr!@~5^sk-yF#AjS&?k`dV6mycYqaO za+hr!w}TKU5C!lbpDiYGBqu@AW_~LOVoFKIylGfqcx?0jC64@-D&!$#4zT$|f5vUK z+Lzc?+A_o5;HK6aV$7bk>Q&ia8uEyxCY>lKB8(y4o_TB_1401|fJ)C`e79<3O1AG% zqB##MgDdCD8B+QL0DysdKYjA#@M`~)Z3(1!x;%#99lteW$q;M+N~D_EA4uFQ2rBm; zN1?@HhD|xdw>Den!%i+y0bxSVWwY}jdL3=L-M;DDY8`7Dg>LE2iJ{tk31RNHk9!vO z=)27yf7tB+eT&?WjkjA6eWh8s?zNgo2@Bx_ z(jqpAOkZhGUz_fLyUKS4m%=Jz)td!vtd+r|P^-YnQN|NT(0iPW`VgG=uiFlGFt_n_ zYu!<-roodAx&&AUk0u%|k_SSlKKCBDpY${8EZUTo9?THoJd;787U{tG;zEKSeJO+A z#rEE!*MbY^r691kc+7q>@cL-phODCD(>-ju9U$whxk?D}snCC5i*;|?h}(5TDu7aU z<_X`W&euSmfSrQ)vtQEj9GUuKwF~WShhE7wpN?!64}YyDcX*zPUs}rs6JV$CJzz0| zDjGvq4mtHB!g<~cRuw=$zHS!VTqvr%5p^j4cuF&0BfuA*venEm zSP7Jhl*cd7fhTq0xc98#;H=L=638{s?gS9`l%9 ztS0c?874c&qr(mi#!m(v9^4WioNpaMW2&}wM=*i7MqqsenQVCd`?Gykb<7-ngJEIP9LUIC*bys2Sfo@&Tmq< zl8bFgR>~pH=K6pC@H`Xga3TT!LX4z@2ZGwpK3HLTALT=45st1S%bkz7)G3TKC} zr*crxqgiHkLs$d83B?n`tT`N`QM@VxDkU1TQEj{bVCKSWnO3u4?Dy{a2K=HmPD?=Kyc$`DPLV0AO^7vQoX(2J zZ>rjC#<}Qq<$VLXF(0hQU9H|WFB@6tmR73L_Z$*K+Tnn#Nl;y@r}xiU7>O`>RhTAN&NM8V)rt#M4=o( z`|AvT(SV^xDu_s5!|swUIhUXQe^7a}^#HWW$O2#6yWWcr@3<=l35iYp%7P{>FsAPz z8N7E-l)+yUFM`9B4`ewtDaAcsXM^H#d)n>zb47BdOzp|VSq8nDlgq)k4S!{J(;Zy= zf$n_kPoGpWtcx#(M`RB}t-HU&+~mQnw&8+jJ2cBlm3+BN5y~OFBr5wa@azwR2^?Gf zEo@Y4vtHoqQE)wI%DQZqF_!BR<9ngD;OcO5bKQ#lXD@)g-uNi_8^ zXE3$BF9Tq11LUr|VdQCyxlGtw>Zt=@sAckP-8beb{}rpfqkkZcZLS`qt*T+49^!)8 z_zK9+(IP>cK7qN%t=}S z1Dcd)1kE2zCwl4M4;l$H^~tJHvAWAklO6(?U~vhbgYAo(Y(*K7@<7!GT;YdM9^=Pw z8XM>0pqahhn$v3QA)hNAa3g0=F}$RcChetlED8!>%7+>qhQE0ployjsHv3Wb1hTi} zzdm{yHj395Fyeh{x+}PlO*bMK9v}T>C8ZKMiViI=FE41ZpKfhCk#)v(lMwy@>e3AKZv9978XbJg%Qdo_#q&nk$eCBha?8Ju>502R5e=Lw0n8m zj}eiv>DrKBr0)3}_GS@vrnVmfzjVai);OB{JY9%Svyx|5PkBYo-6zX{J$)K<$@m;rzxq2otb9l69%N>wh&F&Xv1kpgZW1E*PR;{cBW@l&V7D2Np=lZ^(2KU z4l-I>u1u5F$&F|68igjjUOk)V23y4Q=VN8N7xt0Y^&2VfvR|eHO*Ga_%|4#13q4x_ z_NVV+#=@7MwiCp?-W_xbc&+7}Z;{vgPHsHp_YI>waU}c{41*w;5YiN(FBu4{L93 zIZ2XpX*E7_+68G=3pP(8v|49^M?F*DPuy?Vp48Zrs1>59a_zK_?H7m_v~~mWQ?av5 zd|%<2Pu9nm!D5;?I{+Pf_3JoX?3|EV7%JwXo zGi_A5?S9=PD{#c8tym`QHN^W#>_bW4mH-nwsBjqh2i2mjgd!NF!$K{anjdpV?lEwyNE@n5y-P&@8u3v^sAd9V;1Wi=kKLtO|dwcYB5YWIKoVgTX={ zeU1A;i|%D&XHBf#K(EsW>`Iw;`7cGgn@LYR2A2qP^E$yvh6v`Mmt)S_kHU9z)Q%#w!r(kyiMw8gDQ$H77?+E1%Y-+faz zXTE$gVYSBC-3fZ^di=RsWb0WLC$ww3n(4><)m7_S2__jP6W?h)d((*F984ETrAq(c+%63NW%C@Q+fd?@!qVSL8 z-rqtZh|Ne_HUlC{YtEHVLuMly`sXV_AdUD#Pp~o8%ook(Q>Ek6;~$%*&NHH##rL38 z3b-g{u6nqoys*%}3liXjgt0J&kOWZesGMaN_>ufee;u0KyV063leKnw1ENkWW`nT< z?O9`q`-811yzLLJF9ynvmpyoX)V#CSs5IBubAq%8td*bc*67ycg49bqLQ@aQ6RXX% z%s?2Fh1G|_sSpg5LBHZcQ3ibP;O5Jdq+w;j2jeOwx>~srw}P`Zf?mg!-Ju^oDpq(m zv1i z8Ix$T+gw})ubse{U-`{lN~|nBm=BEb>X^B`I5t^rpfTth%k+Rp6Rz2?=$PDFHGkc= zfN}YO;S6=IZYuN)E<1T4P$yHI+M3KWJ#+TGe(lt_a+Y#j`t`;;#gGsBzN4OLf=(&3 zHD7C38n;`rVpAL0<|o73c%dLGzM(X}>ABGJk0WJe%Wx=xwSeP{_F4`tqnRc&%+Xx^9l%{}}aSKzg7p5^|AfkMBK*`u1&WGmlq z-D`vKC9J2d&R?c#*qG2M_~6Y&XQ<7-qfxYF9dkx4+Ye^8JoVIeDO$+{afHcV`uJ@N zMVeo`P?z0k%HO<~+ugTAvaG=Prv^nMf7d?}z|ddDHL8~#U#i!ul9%S9o_f}ak9L0$ z(vV%6Kbhye9jobk$1RanZK(udAgCj+k?{z0t^48FQk}`_bdKS9Z;4wW7bBM?XOaD^ zdw1b(Yl-VzjS75CO>ysPH3GH%W#h9k3&R3ZrP&X&eD12bFAz|`t!&o|=4a2Rlar>PjXmzzujn(U?Q|*WO$2z#2x)m;x zlLdN+#!jiEh3nkBb0?|LcC7%^$@s*RTC46jnba#gyrWYe=t7I{s49aRms7zwSLe=t z?N3sq38?qM#lXgL66$0aXK@2p(bUfvX4SzDa}9IVNAlnGvr;GXl%QkzTJv-i@5-wq zOjjaO^P2UA4k|Y(#%27pWCy0h7}$=a6iQ@Jl(!fBiaT=Z&@rMe$GBm@ES16c8m(+S znw#S%0^wOdoUTt*b_H6SZ)LiK6!~QXgqdE(GNyF%f*G0U{ZBx5wxo78o5^9M9T@>ydM9a({5|Cp;)W6 z{hGScF7u@Pq~7{a3##?eYOE6jH&@9K{JlZlaEAr3;OxD6&S))iEh~*L*M5k0UZst@ z)2)ST0(Oa7jL=kL3KEi>ipA`%PcU0ttCg*};h6ccFp<7=gib~&3tTb$V>>w zua8y8zdM{bdijIP3L5FbXCnVgouI8K`z)PT>8W7;YdskG|BtyZkB4%7<1QsdT1b-6 zsZhieA-l>_mh9Wuv+u%;b%+*8D6%v5br?Gt%TTg2_Q4pV>|-}}#=MVyXX~8ve%^oI ze|#P@^W67!-`9TK-|Kl^oJ~3FL(*ivpyTq{F5UI@x3>qVmI@4tuq%@Z_X{|gsNiA7|5G%UXcxw+ z6Y9N|BHZpB-Vz{j;i@M*X#2|Le~=ZBc01+BZs(SfEm0_6<%r}D`IP{-;rrL^|MAG!a>BROoKkdk0@#zO~fIn`{ap}Lt^@s9% zDkIJe)`| zntbiA$^UmH(@WKGWcQtZLdTM$Pvn@w1DwpvY!_tD7z>j$DrYX^eGL2iqEd=Z0&efN zZUpI{D^2-ua{CDCsWm>$@wM0sWPMeadZzK$kvH{K>Eq1562lD;Lwc3;^54Wr1f-pl z;dWtPK4bb*jsH~p4||6TAS7P@_dQfpuSH)x9hV4TeWCq}hz1y51o+xMe&x)sJOOr- zfm!WH?fhM~(G#yT>9$PA4QJ$a7lR3B^Mn7K@o;-4GmbQn68kL){}bQ8ap{9)0J7Qr z_PwL)Ex<}c|H)VB<0s)W;qx+lzYugt#?>$xvy&-|_ru)+^k4lt*|9@S8^5MT{V&G0 z7^Yl3WURFxO#j#~i)pb3Zt{Xr-~UBve-e7*_Bj?ALW!027u@u~yS(%!d)=N(GwuU$ zUERQmU$=1;$eo(E%x?X{rk@)i7+`t?HXm=qB~8<c`xI42G}6boyzk}zs6P%0U3C7 zLHTcN_+5q{Ft&srb*Gx4UgS@!`#S+IWa5uBxGf$3*N)!orrNm$NKK6lK~anS?_2ur zv&`x>zk|eoZ|DW8qtxBnTww`T70BVKEM6_pqLkY_a{y^HV*37}&Muu&W>M$eI@9#= zceY=}0O2-#Cguh0ziEb^oV26^XdZq}Jxue<)_ka>DwDt%_=ko4A|HoC4Bz~~%y%dV z6Vw*Ehf_+OzqAd-_-Zib1pw2ZhLJQ&9lq0g)T8L_;z~i{oLgtQ4v7iyy#^RbU)SGq zPW&>};r>~K%Q#iY03`%T?EBi^>696xy=_o-?ZSOsm1sU=+h~46cxBX&*~1yx0Rwc^O*yS^43~v zB@J6DQ5JxfE~7s@{%g6bB?DE`AaJtJDIl^-xp^3ND*|sy`c*W56!PthzV1!ms% zxp{;txJ+y-`@y^$sDYd6%JNtpwvjHyK5kI`Hy8G% z7Uti#3Me!MR{-NY{(q12;lb69N5le1ryV+ecNzf?uYFMd`&O6%mfASBkbmY6w*N)U zUFsjyfW$Xjm>>OJrY^VySQsSj^=1CfM%q$w0)l(la6C{33T zhOcy9?GM8Y^sRnS1V~x>gDYlC04aKfw`X4exe@EEbcqmKg4z)y^F&@3q}PQ% zrc1eVDr1B=%G$jB>w964u`pMdxbz_d{>FN;ABCeZRcN(|Mior zyaipg$NrG4OVsa}n<9TC{z2#8P1IkXeWC8EyWx!yD~>oJ^ossa&~GXN#^XN~a2fmQ^04cZxj5Mzq-obZ4dn9X}@Qm;sh-Nay3fRs?1EKHS;}-PjBee znTC2Trk=N2hQp>4S)blf>dXHj|V!_49D^bD72Am8&Xtt!}-jVG+^CH9CA-s%`^D0tx1qR7YKj(xIJ zzkthTI0;{sZ(Pyfl@l_kV-&xw{Y5IK)5KA$nR9b@2-kx*0A>#H4)285)VDkW5fJ?n zjZpkvy1vC=rL?Y=&I7l+7plqiJu9eVH})aV;Ao>Rkb#|)0&y4=?| zm^o**A?UEy8Kh}gi9*HflNXmmT$$QFPhQwqzw@2Mf;u=e0=giUH}F z`*7&lMDJJh_UGt+E0apg&1KtO?kQy6SDxJMqFCnY7rTU(_Ey;II^83{Bsv$fEan)p zl$I+RjNyqXJYtqNqm#Of?Ch|JM`?bV{+pryu zM@FXi@AW;sIqpZCQb-U)O7iE=Z*idaztyabyOi4dY;PxWNJx%I2>j5|Y&~&D^H#OT z;@y`gYNi4)4ZyQRP95PMdiCOs31Yrc(e9CD%bzNVCah4JxJPFy$o>=N-GVS%n?N#!L^}F~z zb|-kP+uJz%V`&Y&CFa)!$BXG4dGmWyhm-4e3g|{W?q@~O!9hEaOw!o`Q|>qDJ6!7< z{q5m*txX8x>yoYy8`iQHcW#%N8!AMnV*G5qx905@p9UL#`tFlT+CShnm=M4!G4%Fvs^=_8lX~Q@&Lb5`I(l zUGjd>?ZG6vUb~)C20D+9tg5?ie8qm9Pznl}zU_f+1)f(rhWqx8xm{QsZ2}|)M*HB;<#Kt|T_IN6)YSF?A-4xIa#S;=_gQs0+8{Aw>K-A_Xw`jg7w?^H|H zT0PL@vKHkMjH+T;AHiP|-N>uxv`e1h06v1B%PgaRUL{tQF4rf%4SHy=|MkqeC&I=AvjNThQ z6Cwb-Kr8Fn0Bx2n&YS-ze+lxyCk@+H$gvaGA*3vPUZ;7j)i-3(O-Fw(e`J`8mI%jo zgAfZY57`!1K;CejC!E9Gj-KPXB^Qe$M7KsVjOZUk4P^A0;!ErSZv%Z&!trP1@u!Kt zN^?B00K@cFt7tXhqbB}iKLfr}EZN$mgU7UhIbLX9S4eUp#E$w@wy#io&g*rB!Z1N5 z2PH6dq4xT-E}E`^G<|&dGGQx-&YdW?zR4%9aRzSh+b3EvRTk*OZ8U^0f2{Li&0vqp z5F}rU*nZ*cg>$WGKjql@MmxdLqX^AHt;nS1@3KO_Dd*!E;sb$DX6Xy7Cy$k@A#I!A zVcTJjqytTpSWwWu>0rUAMVV^(d(Eo`pc|IkQy)%Em+|_1%l8Z2)qdZUOeir+h_f`& zLCj{NjERUc;^+{pX;%n;A`T)!eL;*3*ETCx$8fv}XZn(=Fr7x#g>QTSC2IVn-(;&z z(;wDczBza>0=1uRvNoymOp0@26#67n)P2Gcg2|O2V?7=n#lNZ0_Ecik()sba!*Uad z4cx+*@z3)Nq_>sYwc6M;nricP>D=%fJSZJvlWxGw(k3Si2wplzm{>bvx42O3=h>_0 z=wmukjgPqI+0>{cB`=+&ZnwuB>C6vCCul1TR{89d(sm7SBQ7NO zW(vZUkTIL-Vbk{vQfW~ZNN0t;-s&>Ps6w5^gh59L-_!uDab_a@`V4oq%}kpFz~u34 z=jqQ!3Jx5xgeGs2J!*VsDaohfQ*r62o8mRa|Jo^|P6v>< z_ra>1KQ_7A=k1<&-sA%g3Eh?L*FJ=^^H0Z|)6o-HDHo*evtzk++c$VQ3AsUVkpoGA zd+gh5)QE`_80!)Ab=A?`=y^GLN#yLXGJ7jmsAcAv^aX+mRy^Es1KqOuDt+dQsXB%S zE9Hrt^Yu;WUSOib##gfMw7{Ib^(6aJo=AQ4cUlis<*uG9sZQHFvoJKX5amh`)>L45eUno zz?iykJA5zsD+-+2s$pzCP-f!j*x7|nRjQxUYv?mb{ybD(7SRJ^ zmNGX$7KjjVNUI5I43i7GtHM(^BvC)6cAS_#xUi9cb?u8I+nH7S#l$eM^fQdaOmvBU z8GYV#mH9}w(m)4rvaqXqN$C28`_eOS7&~souq~dQL;dEzj80C-6XLD%@*Z9^c(^52 zT3R*P)%OU$JRi?fUqCwW-nQ5((J_iuxJb>p%?U1x~1~$*y?Dc)2QjZ$Fg4I}#l7%!b89QCG$t379#OLIBzH&9d#2cyxx@Gr# zEJquE8gdGWu%+)=?Ppv5T)*?1c3T228P!kSRepOQqu63T#e7F#VW$l4@~JT^Ra$u1 zqA7V%l(%(UKfz+9UR)&Bdy_@L_=)fwzpF{Ha!JIYSpUF<`~AqvvYzIG+OXrr6}2=m zCl%jlmIQ%vjd{_qN$VQ>Schx*n+3+_#=I6ZJz`rdq>?{h(ht%?bN>`*9xVR=io2Sv zfn@PbkK<2s{%2^^Vi~~LwLA^p$O~!Zkk1U|AcB){I(qlnBAZM>P(MnvG$IELKFGCk=!*m`u~ZmP?{B0LVhW{VC%KsV zr76*oj{@8}`)f!}QmxciE)1#JKPrdPUgUHbUALhW)mKIDJ+GM;&OLweK2~7kbPB(_ zkkf7ri;}~Np_?0W9ZjB88#h$wfg#M!!)q4(D|FwGkqI~;F>Nh>gC%G7Ruev^)>qZ- zHPTVzuPZ@~OwfdVi<$XGpjLMEYCr*NWpyKmiic!zl9XX@a$p5fTlR@@!>b#W58s=x z0dHiag(C(K5ho2Gwm$~7JmwW0NSp$m$YmZyS39U4OoV)(Y5U0WX3I9#nxD>Iy}-Q2 zouZcRR3C)CGic~Vh;vF92a)+^MaFOR&7`fM;~F6M3T8^0oxR#QHRhbR32^ka{wZG_ z#$<+qM;1xzMrTBZUp6~4)Q|!kX>CRjg?#ZS9WLBB; zxq1NwR7JWMlb_>_G%0T#g^l^+H+t_JmGXoS*fZIVqnlB0TDiX1`6t&z58_k=T7B!- zTYa;h9a>93xZ=dDE%7f)OYY+SS?fHkPfXUlK&>vhsfTw5Tr)^2ECJcV)aOmg@0~ix zp>s0X51DR)UXdk$_o}K1f`uc+n1KyT=qD&Xp&CDHIZGxd3lV(pbN27@if=A&saHPf z1o)V|Y+yD2tp@y7E_%A+wMWs-h7`MkvR45oggudiHVr?T-}ci%AR-B@au95}7S#)s z0KUYmT6_U+DW#_{Fi z`e%KEVcM((atX#+h#~gLuS@#=6elT{v7SwqQGcki!Wi>NwlnZ#IfyobLD}?vjWwMJ zXd97BziX{i;tC_zks^}{8G%ypixHiOzA#I>hft8w(+%jJ%NVd+ZTsxSTR^B<(YfM(WH1fmx;8tV{gdoMQvo_WAln-ZStm@28F%C zekJ`2$}qG%Jih~EQv4ogXk_k0sOSqjpR_4CCKVEM`=ZgCExnl@jhq&+vM z!KKnXWAZ07f#I4YrD%S$d+ZBil6wxAa_AM;=UF#?stUE&u_&L7JVgm zWPt|c9`xwquVH7A zgCwce?m}$yne<3~77K$!M2^q8X^0!z$l^z2^0Ks`(q*NLGkzuws+?C&1ie~qcMrYT zsqa+%oIpgC>IPcnM=19ixp2hAZNGXN$D68Ido0P+2bxQ`R&3@R*sN$V!u{kpw!msx ziHSNJ)=Dg2+Y8=*52b&(l^wzT@`0~oq0?@F>*fhcBu&$(vZV8*_xVq(Xdrd#v+-NSWglCxOVAue8 zYMi6v0_(ZK&qH5LE}2II<&cB@LXv=%>V%Dz?y!GKc@>x^s>H~d`BPebUQ?u2w+U7a zv^sD(Pd4|*v;i(O2dR3EQ{A5s!TWJ=frd$m&X4!)6~sa*emL(_g%O?`d?8jx9pNlJ z=vn4C@Y?4K=bjTM?%KgwERS(Bzjl@un2E~kh@F0%O)GMKK0M|+)Nckj;yFTfi#Ff* znL+nK;-DSazkPpb#?NDlCmPb=R({+>%vrntyW&3Tpp@KGV7Ow)=g4$l>xkcBedk&E zCEfPH*QGI4#Rw*eEu)J0s^%^E3C5g7w^Y2 zFJ8`yIGGS-QKtt95t3#Kk&P5JXA=a?WK*OOHTl%d@_z;pe;xQg)=%-q^Q1Z&K;oC&!VrOjwS4TdmuT+o_)VuiE7ft>fyZf13_kjh-T(x-({+qa#n4T ztvaYm4y(IkrOV*jtCuqe_v3bSON~-_;u

ws3NL`DV*^sl_gQeyelu;v!HN98J5t zfsi-gBue0`?NqcYd~MTL8%s4lS)`OefHyv`<}b86SGRH9-ab-=3e)bnv|5+Lsk2f& z-TV%3J+-?O9$^@f1U>NGKCCX4@T`2zVM#So-v42po~$Pqu;Jz`c6abc_x`3rc^QVA zrs@?}xsT5yqYdBL#It2cCTj@djs4F(6bw^C$(oJMy>HBGiiLw)VE(8Y3) zr9ia5jf=CvCo0aK z9=tw0H(H}}i}OoXYzMRJ+l_h#@NckdKl#=d9^s$0wn>qk&hb$Bm9U}9b!%B1#JUh)%~Cqg)uMFrF@83N+dbb8s^pi$cWTZj|WqCo>m zG2r4lonuU@%1BlvzmEOA&?xr8j>P*IXpa~nba)Y~dCp_xbJ=t8`FqGE>ltFUe?!j8 zJl3w3=i27lR^LaboO)7|dYjK;Peb(UydHUbBo$YL3rR<#14n7$6oKy|%=J1Y4~l?_ z)gabEuk&U@S_N5a#dk1c#|k)|XrhTA?@qi?6+WQ$6(|5+<4d-CkBN))^Jv&oJmvn| zayZM5ey$yOi=;GN;0{?0VBow{BRE^yG9c*7u{nB4{8Ph@*(qj%zW`sEF`i3k)V<)u zz(W#O_|CVo68tHe_HJwRX+L_EY*$za+a-akLrDmN8FXX^wsvT>EVRe>Fuf=TEP-7b z4xN! z>xq#pVjdE8hNh`_&^^@K7Vm2B{aWR?CiB|IZ`^Nht@7*4ueUf}L-U^W9ZU`v;gAp< zp?|Wr`;{Z!o6T|i9u+6vx ztYY&TXOHaAxZn;CZsO5O^(GUqFkG?3Oy{xpFQf-7&(gzylnyPW^~~OJp+ly%WAKqixO8c|;#Qpu_vy zX;a#ygW`bvU|GfMnRKnRts=Xk6x|a*UmMrEU_I<{g^Q5LSrYI z+j!Jy2FrfgOo~GAaMGVF)It`j-cgsa-khXfLeF#39I$pit!_cBu>qOfW4Un;oKZE$ zvq_-76_>!i$g$B(A@>kiIcB>SJTJG&T6XonwjzEWJ}f|6EtNh;pi+wmf+R%S7K;5f z??FBG;Cy&jvc{K>Ai@Y7dM%`ag|ez8QvH!EA`Qlle8e^Gj&1d(JJ@fRnX7L5zF{F9 zG*e(a2z9ak9_%NH4OKD-j4SMLFC3{(U4JV0b~TGTZxT0Y8EOWWc0`zI3Ax-djN3 z{zZICS?BnoV@zJ;f$I!31Uagf+!*iYkowV|})-uey7z)uO8pG*vGLCcU*y5xRFR_(N$M>*sLX% zP|^5ua}S%C%(jeBmO`iJ0|hAWV``P#TgEG8w+vFthHLE$>3r>vanc*E$7nvnaS5oS z<@nqlqOW%{FZ*%iy#zmg`Fk4LVQqPP`91np`V1 z#`$sE=LyK6(s7JMG5>A(*$q?*M>%+lZw`_S_c8aZ{F05c7j5Bkx;abv0po}T3Vgjr z+-!v!8?^EhEqkav^GsJ5;NG0-F7Kw#cTuE;Q)MZR#7iKu3%d2GxQW8g%6$IG?bXf= zXPazugXYmFIJG?c*M!+&jZU1-C6C}kVrvW6L4c5(<9@k zcMR}0T}6(sHBTolr=qglGM4eIpm^n?FldYtuEiUiS>A*Ji0 zq5g#jYwMQkk+VPIZl-Et#Iue4-ou=fm5Sa+b+aHM04*qRyq0`$asKm1$@#_+g~@%u zdrO^%t_{V>2N$_^3zn$RfNT!cq zOT@ezQ+b`;Xckg*E{#1^B}_aUL8pvKC~nL*Oq2>MN69UUq=eIT>_lx)b;OwRGE0ND zbD2$PyTi*1@-ImAU&`R&RElu{b$8VW@4qgiYNG{S;$;y?-?;P~WHE5f2!w3cy={J2 zhsn~D6-ldmtw_P`q8ta(#wy)=l~%6rU2)>yZ@;zlI=9f^`Ce}qGXNz=Vi;&p&0)gw z-oX*JxV9d2W52MUH0L#9wq*>#?TyzavVs5k15nGxn$FQtXiFlYn5XYe8J_8u(C zCzmka(IwCKWz2i>I-^9m&7R^QCHet#Tq*`O2exYOM$78{2&8K49LX%N7%X;wN}GnT zH>9qpbbe9|x*+RpZ$L3}Z_Z7P_OEkVeZMcpJPIwrJ4<50zKd(2{2#wm-qtu*`}Jf?r3YbgL9?4GgcN+J_9My-4HLB{w zJgD#0T@mQ1bszMhXmQ)5DHfs+4KrlXwRDp!((>%0=K$G>5iN?TN-ROBg~ zB_~LHJ3$9Jz~!bEx26{qdTqoHxF2<<*4DmoZa~7MA5tASQ#VV%w7A0A9$FoAWG?N< z7PvCK*FAfm_;4)7+_6ZN(qvt3gBCe0LD-+qiy1BL6!bVFOxY)Fc*SpD;Qg*PGjGt= zEWQkcu4#+h(N6?UCG~6~ymGntn?{nSXP^;apirNet~6bW5%Hc5R-TUZu(+070UTG| z(GRhR5M``_w%%AHUMU5igj;KWSo|+%z_5*Zi|arpRjUlIWu~*Q9+r+FXOm|F9p-hN z1XvyBFHj332JPBj{G_iwr^cuBKr;2pIU=zN<=DOs;sdB2=2IheV zGIchlBw$&vP_!$*qySerqBGCL6XOP13Fse3(aO(Trh;p4SMtRxjp)?X8K@O(bqt|# z%*5}7MM#Q=bZOrSUZ&gHq6ec$q)=Jl0>rVs#PkKOz|?=vH<{HVT(spt_2_2H8rB-* zws+frKYj5$_Z6#`m7PXcg=0Ub#QdPWA9Kru$JN8hVNl@>!ESJ5wV`r2X3b9hMv$l6 zQT5s(-5)QSke^`qEFr@bEo}DulOW)oD1z*~uF0+22uKQ8+rC4e8YxmpAU1i0UJ>cz zcvOJdNPk5e%Hs2QYAp+#r0Dnj4aQTxA<8m8*$C%RZE+@P?>FH)5eu=UC}|GAWiIBE z`n3+Vx!AHSrP1!YunAv8iP(DKbkK>?M_IiVRbu`&UZ$wISR_x|^C4Xn%V2>Kv z$fakC*W(X`wl1tkYJ!u$DKDH5PT6CNGmX)Y08aw-kdXb3g>A63@wNOV_2G?EH@?`h za!QSQUyW16gh`#!z$_@`NjW5Zj-7bX6Sw`obse(>f1^W#65B8x{QV!X-zJ z5qb3jn^qM&*t|DYaV`FMBtC6X;R)E@AtK6KU45CK_#3y*ux)iiSd{NfWpHH}vCMh- zAP4gFp6%_i0~lKx%=PJR31boB(*HARwD$XMOKK4Akg0=X=|2iE1 z;S1$-WkAJJ-jKv&vmS07R5Nm0I^!Mxvpy|aDauuifzy5pt;ahVhvf@`LSi1zM*;86 zo7d#?dMSx-1gnx5pPYDG83t}_=23jA#k?68<++D(z>T}o-%b0?TYxu?O_gt8z&yn7 z8Suqht2C#kjmW4UL`xqH4QIuVoW`)Sq~)HQuHqEQ9{VY5`2Ks3{T7He#|AhfSvJHw zrB_uT(>Jq(#XCu3O(y?16?$2guh$!rW2by2ll*T(ix^4~7wmm{&qYAG|hZ14SU%Bjjs zgl?g0oqNPr9qv!OEH&d;p!(&ff7QMIeicB)C$mx`V9{@?5PYem3GG8+JhEq(M6 z_G`BN_n-W^_ZMscKsNR6n!4A&u^IoY#RV(?5OO%%yM9 z|7?)|bLzjSvj_svz6)v=9REq;RnusYo+DC{cyz1Ou2oDn;#@udsoOe#BJy1yA9g4; zloC%*9i9vDkDm z%d;Vmzx>Y`{%$I-?onI$ZXxvV_{7yF z_k$}LhXA2=k=Q3D`wkbojGYck%75y8!*K7S(s>gDR2&~u@HLL_IZoW0ax>N zflx_sz?WU0f?b%TKLux=L@40@OTtRzOgnrSJKit*3nqD&KjrbPb6o|K8U)jg*bWOq zM3KTMz6~?(|Ly7p%YdJ7v(4_4rRF%~oj3FdkpwSk(B0%Z(J@0m3`V)BS8HjexpC1`zH$fBQHC@(dA6=JkPOk~_pkn2Gl2#yyQ&RxwluJrit+Gl z=$-#zpbAh2LzJL77iHeQEc-sS%=L%fU7x6}R!5nT-*m}V<;s1~4Hcm3_HbVI>1~su zCLnZ8)pszli{UdFahIwnx9vwE?K^ARD!P_z4BT}Kw0j)0$f{-dB6i=EhUuI>vDhD+ zUrp5ck}BIe;1UF_Vxv-M$zJenk~O~&ofV&8`*fl3?+8ZWAgJ9!7oV#UjIt9%=m?B! z@7UE&P~K6rRA~7?QFQ;^`QnA0!UX%?TLp>S0l-i6e7GuD%#P#kv(iaylsvv$r%-da-x1t5 zSgqdS9VA@A*v_0UEXaM})TU=(gl-W4fRa}2ZycwKt|ugb=@GB_#%lZ!>U4E4^F0bn z#l$_(4+S<9r`gx`&iWBiWG4f#cL;Y%ZP z#-gi%OIgZQ8mVOYryU2+>-2Z`c-%ZO(OL4fJez(-7Rwlu&aeV;m9LNHV|9`eVgV4T zg41*z_r!(K2YEQ^t%gIygxaU$>mv4zO6K@mF@<`AquaC8S(ZWaX2-3Kd}>&BM`O%L z+lM7&ix`=0ieM|U_P?NF0Qj1-sP{w=0l+@;yPz;(EW1r;1yp?%Pi}qwSvYU^{+jfg z_|Eyt*8`N4muCcky72(tIhsqisF&Gp zs8%{-YkRa$&Q0et%EppSH=x5*{pxlk7X7?o({#l!phbkn&%z*H3s2mlYx2hRIdHEF?ilE3cjl3WXQ|Jqr2X6B-PYn_sY8h&S zzMSzJ522zQf})12t&^4B&DutCxvO&ephLZVYI?TU0esxoj`;Oz`+(W-?BnlRj)a(G zVbm#mU5?d#ToJcRf`8Mji)S)i@U_71Yzv!_A|Y$6*{CD}{IG#3~P zR{9NC#fu~(ea%!?QD)E04V`%i zN53}0sodAt-o&b_v>;!)vH^g1VfG!1d>-Va*n5aK*#bzw#bV)DpaH-gmltqE`_f}6_U$uCJS zS@DKiPX$rmG?JcS-=XlEF)pFL)Z^^kvo&rnbFF=-)t(I88dw~CuyfdKb;0Z?q!pPq zW`G)c;V1wAVlz!OV!dB^?S^?6sr7KUP4b!~_#NAgU4oqz!HkJEc9dgi8`9toCnTV$ z^Gr<`RZex@Sgo9T>o*x+)W1J@-Fjn%Ga(}jZfES};(0TmAsI`f8V<$iMLC2^uPUbD zvVUZl?(}Wyt38SPXgOPI5mH?q z)Qrew0KwE5z7A0Cw_kbw#mrBqL)Aagf^j`U8|sn?Vc}F+)Cce)=zZe@P?od zT=5dVR*<~E-}=6&HTf7jx~1RQxE<5B0`yp9Td$qkH0CqI!}ou5ovuBayM&>TB`cSe z6fbal!XLyPHcb92!1UrO@+bK132Yytgy3e7-{FWJKm7fth}}wsV>1vz;N(MmI@@_B zHr#~OI!qFwl!Mq%HQsAlocIG|N8AMC>!hFKa4g1kZ`kexgJ|!~D@?8Vbe){vG$`of zK<&#hpVJi?Bi0$n8KDt5_S-$*%*kHa*MO=PJ`N1+C8B3k{^YFNvHkI z@~hVB1Ru3u<6ggJgr7zrydgqv2ma|S1*;8(P z6eAmVLY^|aVBb%yl;@qzmrtf6&`HzN*~Cvo=u*o5VGGl{ocbmqgAOEHHGbHmOcH&Z zu|eQ^t730N#25D3?~oX^p$rvC&Ay7a}lQTb@{qIa1~g8{v6 zdgB*Rkg^Mg>EnhHmpZR2uA`1SzCOkwA|B4-J8oF;*=KmBmb`c15@5->-;8m1mQ(n0 zX_&IVqa72QGqcH6!F`-liRZl5*R!|cw4cE1a%R|p*_A`1qYCw!!2BLxZ{~g@8-)8> zEcxx&p}pT=WS6Rp#EGc!D!6_2`y^Z6a7{mJMg__Omym;IP`zNT2|QWK)(f7DJOTNS zwA1y0jFAuU1R0Z{*AHmR@LVqIyeH-PPCkNJW}??ez5?_q3yjT zE&u?PJRqsCc}C`h0BlWKp+3Lg6<6vHFDESRo4tDqioXi6Z~nyhNi4e5ah9NFb(F3V zI5WYGTKQn9c1W88jrT1?%nlFjEX}c*pWvzlXQ{*=;O2Qw^P2IhLF~{vEgX|j*9Z{- zFUjz3#RWvqfK@@Nuoc5oG(<}oIUK@5p*9Ww$Z00Pgf?*2GQa5=fWXRjMN>l;BSf6M z%lt=Uwi`BH_IV%mosWzeGNw`tvO*xBwQ4$*WNDaSGc`sNq6n#0vy45=%--d^po**0 zD&=d4yrr)K*0{5l=bz~4Jp^~u>L*u#TkYxsLk_AATVwctTd1Q;zI)j0MZn44=bQq- zUR-u^IN`AxS;p-M6%9=ge_i{&!3m=)-a2BN3;g`|jWTt)II6p#sJf+V*G(dv9miF5tlAYY}VE80fp$KC5d|`_)fwhj8N!`vyVRQ z;Lc=-4~m3^UAY~V$MNif9+s4cgyr7|kX;gmtQNB8t&e6O?`av|xq-8+Rz&cJYI^(8 z4T@)HpMCH0sV~Tk@?WNIM=m9t4J1~QU((=aa&IXO_3g}oG6jk>SunmhAA+*pnp%PL z?OV*2=F;0)>#7%Nf7`stp#w5$?qsc~p7Q><7{cMK4su&wrsUIU?Ok;AOG?~79S@a4 zLi!&8fGVa?vnq)Bh^VYlPuF^Z`}#E#d?~`F#+y1<2=1kdQ^|g}oF#d`puVU{QM)zF zTdq22wCfo9#PG!#ltRCxchNU53pR#Y0#&uJg&U#f=pcc~!0xt0B3EcZRj4Q}SPvjqo(MfDk;X!NbZG%~Vay{TD3 zZ=gjsEiCj2jD;B`FoCB)W&a)wB+2uY;KDh(v)c6%lhuGC-uL&K?(ME1M$~36#E4w2=Sk4|P?$ z3Kq#h#AMN66l&T0^IZuMUa;|RhM9K3=*g#3{ewe;OFXm}&sN~un|bP-z(^h8_L;)x z69!1bn!y?%aTc6xRiDl{kaVb=QFv|qu4zP9XC{c|XjpFF<0nJ3f$ z3*mzZkK9EK!1BNsIum1s$EuoamwkPKFXo=%dX4c=_Gbo^9O~$IXKU=m#EDFc?7hJA zvuZ0iH755qwU#Q!>mo({T&g|$q#mbD9_JLyIP@8-#HOFmue*8j)6FRMl96JF?w9OD zo8E{wpH^G#KKeTk7B8Pk7%tFCPJoF92WN&R)3rO^?+ zEtxdVI8w0k=}-yzvc~RlF{$5f+{bJUr3?~!-Yw}$YC4w%2jUjJ_)0MYb&VyYcE%m8 z-Ugtj;f$Z{Cfx+_KbY}P0`MXqF)u?h(dP(2Dg|4Guim*VJ{ltcJ&o~{ zFRkVL>JkS zx31x(e3g?-a-Dfm)pDr)qlTbQOO<*wZCN`9;Fc<_xrkfhQ_;@CNy62Kj%_VeCe&N| zAw)k$mUk#o5zJ#OusD2OE4N;Dm0=E}5-=S1R35O^C)@?`4+42JA~JNc3~#wjTxX88 zUcS5l&hQs^&FfKZKNG43DhW;sq?TZ{$}{Lu%Mxq3aPeyE#$|s z5yOd3C(!mdJ+oEQad#XSESq;<^y*xe z_oge8FS67&&Wns^sOe7oZIDl$Adz<)FfGhcLd48U8*s6Wqs^7Iabey?U#8mn4SXI= zjKWjel2P&!086@S}y&`>9o%sc!f@AOh?O`j#dsQ!f6wUFNVyq?v4nlN&= zP8+)qPD@hd2~N#TOa@O&OO5m4$a~K!s~FT-ajS%Z_WBbzZwew+I;#X1i@Vvnfq#%3 zCs)CJL(*VztXRB10rL@dHoVHn4-7o3LE4C(_#eo=$&WGxC+0x;=)LvqSAjF-m$m2y zplh*lEKM3$b{Aq&ry7D42ik(__{)r9gl!YXz=r16{7BzFb-PNdBg}-hdYz{;2#&gu z28LTxe)?(2#AoHnj@p!NCJ;)ce=7T|9O?_s7RS z-c6!8*LBWy&biKcU9YQxyT8rmGMOkOTY@On3uH0VwKA(_mEgPs09U^>`bhV|yPHCk zlkOQz!57Mg=Oy)@pcs5$L3T13Oij=udELOdcP43rKo`nsLn_k4uWzBWB=!VMe@2a( z*+NCE0V%QV>FK)fI5&|Wxx1wWbmpapWi&c8^4WNxk-3`GwcV$Q0C;afH&J_}9sBA> z#UqD)mJoaTyJ*4a(=3?qP>wNvd@S&RAT1A2?a)%iojvBl6{maN=F|n34wdjYqjz>e zPmhCFgS}xM_^{Am>JRC_DMCaUskP~j)kad^Vo9B~x>?mK`e*{yL3d=BJSdVCq?WU$ z@b%nod1FV5Msim>7vI~|uL(nrZmllt;x%iCp1KBQvz4Ld$QQUUQUX#3!2QQxqU;QF zKaz)ckQUVOi7HQe^0PCZYj%@Ecn_CkhWuTUBXL=?uT*`>%Y+3*K-#Nv$eE_f^_L1r znP3z5xDt{rj9(bIfjn#*09Z&XL$Vq~sJR%?z(m>h>1tFr_%sMH+uTFbG0o3oyqXP) zS~+}ReIdrNP6-WXI^?=(V~8v3xi!^K8tsNp=J}hB3M7=YhFI--?v_vFsyr%yz{Mm!Jn{qVn_(3v|1O0Eh#UNOVqxn%qLu zjKiiFN7nhod;+fye4M>zQw=MSr!T%*F?k0_3R0OC>?=jU`K^f+o!)G=p7hIpBsUqO z3J*yNfEs+(69!Og<-|y7Mw-o2@67uo*}M4U9xD|H+pEudYplZp3(1j|i=!ot(oXmx z%i%X)Eph802kD;dC3h_66>ER4gt4|oF}-YEld5)O3 zH;$oXuivc1eU99VQDAAXys@NEOEn(F zzS9q!=!rh^?x%x*aF8wOqzvqj)|| z_;^i)>X;RZ9O2p(TA`9C2hPn;Hu}a;1h1d|)_58FUF52#-B6lzSK7m}kZIL;giWoB zx54cfrVU$5+U%XDedvZ$hK-UDNJl>4}JF*zSZ_x_Jq3M!(YPP%yl}yd;Kl z{pOr`Nx>HRF8nA38ZZa__Q(=fIxH>D0v8(|n7+)8V>z$lntt z4wR#ZwP9ynwaE)9)IvArbW1gwa}^aJX7^dd8WTJgzh~W^b`~uE7Ugpc zj2_;APEA|gz@>lbB*?}Br~UY@uT}WBgFBq^YEH{CwV66x4cV=CL4gwOE`HZljf$JA zjFb4zmmP$|a*WQ-iy95Gf$q-<>uhHK#8*>-iG{)Mm+ZkjWzm&15lw@+P;;8}X zNRq1G-^8J_&VJwO17EdY>$klEP%F&V&({$~p3>lw%gzl}+8;0yNYd@q&Txvan8^r) zmJxPio(H33ojxADZ95EhiX7?Jn1TD4*CZ@}L-QbP+(IrFlNem9!~XzH8L9*KC^^s6148kftOEA_TW* z`p&zzbz-RYOjqT7Z~x-JAd(sf^?*AN7*A@dCYU!*$QZw@a1gRY+6{5^yub}=0Ogu^ z4O2?PVquRHTGnI3mpFQ-T_3-vq0MhKSRMg zCRLJUV&0bm?Zub`ro~}KO%kWvRRPYpoG2CT)OM+&gVp8{y%>HF0eXR%(sh%mN(h9= zSnRTjQVr16p6JSktPE?(>5Rx~amffB_jweApTI@SQm&V<7(i-aV|ALnD$}*CcRRQS zzI5tpWV*&=EtwANb*?Vs_KvROG-d8^tt7BWJ6Mtu%uv92rPrKYn2*>zt^75L=?-+0 z7%}bvL*MBAnd$RnlOh#5#nWL}X684tT*;RJz23fWd%!K~CcM2J{CKH?_5gjR8n9t9 z`6N2%#w-OL|6pvAkLYW`#M`=!sPyrbMc?egV5Mn8iQ1$Eyl4D$sjH##5HtFabHZslX9$yNjM+p*Y-R{8@mwZ>u&V1Y;% zHrWF#CI(bGw2+Q1up`jYhDz;PUuPOVfjDDlBt*l00hVtEDr!5_&^M?GPxb2BfTNC8 zH1)X#69<4cb643Q{ZzJgB=_>#`$|ySbEAp5OwSJq;JPC-XQeP@_Qg{O&@;nlk^@jq zXNG!XTz^qz8AgY^zQoYSH$YYs=x1CO)T!28)(uaRtu0-boM~;h-1PD!L(!iKcOhP=`uKJIS zljXIioNz9#GL+V$C+7GIEle`FHI~lCS%Pj=cry?J6QT884rNhYqTjW@2DHG;rju+u z2WoIog!XgOYp2v)i5*^D;Y$JRW!2C_x;Gx1f5(mxIGjC-2|)F+w1fY8@e`ze>K9Eo zyR@8{PaCNoE8;FY%@-vC)Wkchz9Yi~FDcX=&CSW?3ijLmj3!jFF$XYe5VS`KTye%# zo`Me;O5U@}h}&Dgdfq#4f_wJPe#?WnJk_ZU0uAym zd=8*;z2GVk68T@T!A+S>|fX1m^hdIxhGV7@psvJ$PulnMO^*AMv)>Z-+vvr zZJf?#by~T11($xs&szDFWOO*aw>%b!)OfYoE8|vtP_`xsqkNM`d!(^)0=ghNPi>A2ZT& zF-~DL3esGAg+pLB`1U$R6~paM(GOa524pW}pVnHUOEfFC|?IOeP zXd0jb@GlGs_ho9^kC`7cA&bNI#-Kid%T{eRGX_+_!N&iFD{Ynp{NV+;m#N2K;(duP zphsrAE7km@*a581Qn=QmpOG>t4-e-I^#Z+Xf%8$B(XUMZjaT~D^iM?|cx&|b>p9Wv z&u+4BLG}H1B2ra;EIMcf|JPSdu^S&r?@+1$D;r!+N$Gb{{|*+!eUSQ!EvMV5qwzYA zq>{fN9xs$^ne=k3f2RKM@_NV7PYP)Pe^&UpwHB{xn+61A82w zzJYfc{|@l;7m1zqQ{3B^lF<7@tTe@Gm#VD2CZw;o=QSPCMd)3cmjH(5n?ZlcxyWjBTsTgNn8k(t% zzs_**$pLw2IbV9Qs)c|Mi;RMS$ov z=}Gaz`Twkz{r@5*Wsg{z+aLh?ck}{(F{r8cz+f+?TQvTpt-ose05z=w07ko2uHE|! zY?Qw>?ok<@7m)h9Q?GxKotqm_($VIqw4`4ULgFg0KpL$FuJ5OuKMHd4w@v{IL=$nA zxbFhlpsX*^NC#f_8Q&1P6s&pYUh2zEPlm&X57Xx~UgFM4Nl6>deTc4M>~OZKg+;G^ zNNo1IOn)vQ;J16oRBY#^BUj|DFI~#`M&(6u{&QVG&^b1*3BZhqb3B88bB_EE zLI3qME(#c^Lwe@pUl2h9nCc}S83n0dSlYvY3Wx2Bb^n7>{2y8$=K)MZCNtZ5FJ=GP zBLC|(-yvYiDubf~enA9jz?93g*Um2PYZ-so=(BiW%1~X}zhH5`*}&2xMJA;s{+DEb znBlFXO%KKywOxcK{YW**WkaQW)gbp;o?H4@7{~7NO zkNkhe`@^pOzZtKbh7I>WEVXB#6r;@f)21BS?dBP(pTlSnX7|3{rq-GuW-9i@g*&fY zi+<&D_c`_R4CRq?Gp!*T0?wj*MQ^k(-cH%xgJV;??;hQM;V-*Wu@eub_I>8$zjE8z zxsY?S|B^(k+vmyT7N~E4U+w8}_TO@CvZsbO!I+8LL-Kl8??tJ}MO-zMFnrZ+)=Wb> z=4Apc!8_(%T+i~O3*m3NXN zc)=(BH8yc8pTE#q)MFH;Qw@u${7xSgr@V^Va2EW-DgE{7Q_(^7WR(~hsdd%-b+wG= z>=d!`N>2?sF){?LKG50>omZT~EDH#-6L#=>7eRQrMfGn3b<9 zyKC40@0S{E6s_BNy^)DWjHf0@ubGUOv=ci0tfV%@rxst-csCf7OyYg`PnofsDNDFM zaA1eHh6SI;SVazF2vp#ud|LLeUj`-v`AM^#F;f9{x?7kK!ihnE8~5xy-!Y zSI{*jCUzM~9b5gtXT09K+aAiYVcZ{wn7#&OnyGy2a?f6^X;GK(z`|mza9M0ZpW<+A zW3QzD>^uLtqQDI2&fi^vLGxhIMn zhLFaxEak@N*MqmWsTaJoF38FGfcxbfjdf#J+zJfQ(JNcC8f_YNM!_w;RbbwJn$ zbd;?7p1lou`oP>fF^*Uq3bA*}OoX(~%6s`|jkKkNU}*A8!EsJ*mF#)06U?pOIwvghY~>0NZ3!}HmECS**z z%#iTuk>hA>u7afD&dC!x;E=c*UI1rhp4LnYr}&3(7snorEw_u-&wU)qMD$E=`+b-% z$*^!y(s|;3&wc=GZ4hf~hW1W9vOcdeC{}%coomr45Bd0^f{hU>*qdK6?X8icv~EbE z%E!>Tz3YfFRO<6)`m;vzuLzSvKen%z(%>x`3csEnXW}MaX3nWezcn)%nc^F;($pDa z(VOoREo`geb(;EcqzRB0er+q;_pryOz=Ma=Dv`qRcT0--l0PnhW2H$Po<3^B^qVU& zp}l8kL>8Wy@1CdrC>1?eyLv)RFlJ6QOI*f1aJE#t_R-B6*@0?hrr4SprEF(=7Dy>@ zwW>`Eg_~#d!E=5ixklJ9d$hNJV@PbqZuI92qi1hSFooItCZr7mf|YmW@>0z0C^%F_NOgEZI>vl+>@}^HfE^L7v9h zmXXI6WbZR$94T(eM@YW0#-?T-py#)b2inaJ(c;O4l@+4lLT2C z8?+sMrq_DFrY9vLmcHsavZhe{O8k05bFmp$)0}YbOx*07*EiPkYVKuQ$}C8}USrP1 zoUg$w;D6iL7Ln`6K}#3;65~kkuDTj2*ME9A5MQkG?VDJ05T{vQSf|y3b36nYRgl2n zMX(=w9TN*5MQINyZr*ajKM0~PWIC1vu^w)d6tIKvqgI!oTnjAX=0+Gtz8A={J2H2C z`Ny{MTTM6N-WWXwZ@)fDJvv57gRK-V@LpLq=}JvO{3S8a_2z^C%_17{tZ?W9teZm@ zNHW*vJ4TQU3IUf#bixs5O%NmIFQO{uMy`a0u%}W_ zI?b5Gzh<5K_y`$bDm8SyFZY5#3~GT7>&A9@vfBNwTLX@64Q}OlftACL>Q>s0OLw~4 zoxI+gn;~kI@1gm;EPOUGS6z*BB^|@Ru8zhC&A_Y^E3O#-Fcj~}T@YQ%>@AI*+(Wh*E>RWE-uaI`}ht9|&%_`rn*<#HMY`BEIJzbU% zp&ReVJ>EHc%{Jhs0!n=f0Sk90O3r}h)91GvUy`TByAFbPgSI|ASxxFoQ%a=UqEjxK z)wlIQd&6NQ7Zuqg>zZV!J(%5fp-O_owt3|M<95gKLaxj&sz5LSH-HFtN9_|ugCjdu zA+U$CXFIvvt~gxBsm`Yg6>im(kydWkg1x=<-F6%4wrH66q8d0l<>Z$xti3epv3a8x zJTMQrOBo>%oQqnGWDR8#zE^BMsLwqqb=bWO*cdPn*JbKD?w%!%_=GMpkm9=y9p|Pnoa=V+6B+9AZPidqwfA zaR=)u?M8DVH?cn6QuR2a^iqF@|B-@u`OUsWy4>Febo0y5CDb^n)1vuZ^-l9~OILjn zG^wG-Sh*lUm5PPnLW`l@VB0B3-a$G&t@MffmLZZ4j}DV}TGrfdp{B9R^u(o8&SS?r zRD7B1^LsEmcAV-X=}Gh}jW-t`DRN^C@Fw{2ayB*c0xMy_kFrtcX+E+vZ8=4>5P&KZ zRZ_3IllwajELv+0Nd>22UG2mY+bu0}x>n*dKQcdJ%fSR;h#_d=@g7TWSnkSjv1iHM zmzXHOk=OBaUmO>OjUG=%-y8NDewZr=$<5F~1*X2u)b5s_ly4@UW z-dI|pC)lto^Xgt@qM@nMm@4@cHE(=9XCbor5Y9n+xmtdVA1n2lD2KDC$cKq z$d9LI-(HC@%f$#{ZSFaWFv|!%gGm$Lv>IJkXgAMNO-OBb95NoUvXD)R zU)jRWmnmFzN31R~LrPxYfKYy&1FIIhc(m6DH{tA&56iMYuJ-{-s!iOAya~s4?|yo0 zF-#3^1rc86R((PmrIt)@YHj+ha>nDD&XyeWQ_uYVAx~%tUEI!ug69XpBc11SzD8pv zXZVx`{8lfDkiQhR*+#$CQSQFicCZWah)IV8m3bzO!SpFR z{i87kOS{O~Fp}i6p~@#dBAMIw!HJW|h`Gk;w+xCZ+Y>wM6IwKEr8GENrT30QDAOe7 zT_GKPjlP`~T<8!M)Et!W?d;ugYW>Y>J&RRKenj$3Q^^D5Z4l)n&1i_dak*EM%fgiGHeG;aiwmcGpkF^Lr6x+l^ZPr1!rP=3&>5 zK9!YBKH01Dktd`j54FR)^H&rP;dhvAjaHuX=+SB=?@qBvJwtS8m~Ce!XxqKVpUuQ5 zAkXR!OtJcS7i|aw?}aAcm%212u$v6Zut8(oU|$99^hx#py6(Bb~WO$+95-9zCVDD#vM)BSY{yrs2o@S$YNFL9nZ3z+YwS2d66ZT#m z6OE)~IXE)1yjTdv|pwEDNZ^7hnH$K!b5oecYvjI|T z(=)?gpmT{Uy@!ow5yz4xH@47Q_dcsYa}V&%F`VOX$#NaKXCd;bZK0TI!Kz7H9BSZpY6j>#o=MWlozx4=MZ|Nw{B##iZc!RkW1y& z#6HOxVxKPdk*%(q#awT3Ht3bc=-}u&Rg|(s>=h;TsYCbSlalArP8}*Dk9U~C_d?y( zbkc`dd-O%9D<2M)NnvEq>i0=(3Dp%t?vV z=ZI6bAlF2<+}uH(9*qGaW9x?AvW{Qn;LI}OCER(VY*l#nj9sZviBSIHwagONXFe+q zhA}A7O4n9{V0T26L8*#@jKKB~hQ~7>`yIoRrnrQWb$WY)b1ve|I}@b0vJ^cr-PAzZ z6fQLOTcKf?K)#N!iji2*^`10r&P86v?d`irJVHl{pb{39^PhXJ$p^U{6N8dC#=*&ziA@UP~`LW>~{N0f%P|FG*ZV{opPi$tCakt1*`jMp z!kK7+ZFzr7BR6+=r|s+g(2=}7FgmH5^M2%o9`U|d@&&zl~mbti^+|C z{cTblD0Cb<=RU8+WZNW=CRgbGc!RIIq+UcDxP~DT|8e}%GJBJ;n_!b1_zXZ3w_Zf2 zg^R$eWpO%R#`LwY&yG?3b9EagM%9>7Uc;PfoTRPm?k(cID&^J*fqc_9L@G`u-Io%y zpKrW}n?b7#54W}+BQqN)T_Wo(qma4d5$s+lh8c_YT#;BS@KZwsZIjKgB45?Z4BhN8t2hl;I&V+Tjf&{ELo zgN;0cVhg4gF@5-ShKg;`qA@MFCqhZi*k&=r;wl~i`CJiNq9&WfZPlKoywj9rnX&t% zuHh>$5a6ns*hGl`vqmr>Nz&e@ZwbvQhUcTC*_-!nu84F8$m z3en7+p)q6gr&u-~ODz_(5n&~op#LMuj#L?Uv5jj@Op-DpK4y%>rIh!rtG87S`K=oGWvaLBU6Qdy%L%AVw1T#!EF#|9rk z>&fX%WkB2M#?bWz4340l?8#4$vprDF8+>++F4s6mN7clu3}7H~1ls50{nuOfp7bQ? zb!)+e*)K42`pi2=luuPb*-=YF)@Icf9g;*r^Vl$4Vw`E}N_qM&(PhbB`$U}cUb><< zW#KaJHH-^wl49_(knQZWXUC9`H$0YdWl^W`@43r8{2)&YB z-4kQ}8CAqSIQZ$QWtY9~#psLTpSkp#`A_y;gS7SDFt|(mL%nR5o_%}p-Ok3CVf@G~ za6a7iqM~FYYyVSL(Fh)hb$9)!5N|q7bm%efTcg|ahKyI*kLyBg`cyC9o`3&Mxv}q! zd81aAM^sR?L&23iUzcG&_{`P)!b^^O+yhKP_v>kIpbtv(SUX%`)uwXbJ6`_n1W-TC zeI65gD=pL)AcX|Ygjr&EC2ENaJwzEm z+;AhGjz})wR`nQvjk~`~aqy}&sZ>>?VPiBkbEtY8Ih(ST^hNxPG{v&ZYGn%U_`Wcm zZWnYr|5eLtQ2v!pxqF9*=LHv8T!nFKbO%ns#L(Ommr5+! z)-%JQGZ`;)iD)S?!X=p~e6K=s+#C)r$<;yrbkL8uO0>m}-5=ZS$;)+KS-YOAC#>SI z;2LwG3(c-!Rdsx9L)(pbh}pPeFYo)3l0UDd|G1urjFST%D-C$qH| zx^|$0-|@Xht1OU4mGp|X=}~-Aq{^*!jy^bjh(tl|sMrj&i(N`rvuBB2r&dRXkqqQ0 zGk0F+EWZY7-A7h7he^GSjC6N>bi0a8S>{HTj2PD(xFP7cF`WL0-DGs&S?l@NBb)d{ z<7r^a{(ii?1c|cy7A?6HP&s-eJAP`Bf8eG@mJr!QjZ5xxRZr;HN}Tj^dt0GJ$JPyx zv#!W)HooA-!7@hY@nwfxrqO%0uP_lF>-ct)9C!Ps2}~Gp2fi zPuuw_Wr1LJH8SZ=PxJ@5hW6OVjI3uDjTL5VYDh#qr5>N)os~`_oX;-Aw2KAG>D?kc zD9Tq7FvBjM6W-b@X$DnY^~osJ8%5n-g3hb<&f;7_(W-QeSFlw0R5U?UPG178W}x%5 zhTJBnB6=D+q-|L3x>xXPynhDP*vIg8m*;)nK?AU4u{Xt%(R{n#;RPEb(ue*kkkK0V ztSD_UNjol9c5|tUwnni)haIZclxRpNyqYVjn&3&GhqbTSr{>jPv~31Pp?C)%*4Z+N z-fG0u`no$wV%b?j@mKP7m$&O;g<1mBWku5mfy;^nI>t8rTyxQV1Vt^LTeJg2$#0ma zbQP_yNAiy{f>3v8ds+1dCGs)ou|f!1M*@OAqb}yuzq#g*%Jk4Rw#kC69Xkv(p9Fz| zq2W~q|M`{#fna}_l5E0efHZ{cIWrb{s3flCOr5DvZ*-JxFQuyA@Kp%-0F?&QGI5PA zWoKV)9gftOr)%>x?NL~Ax6W~y?Kaxpk#=m2Wt$*xXVCQxcF-l9UweZ$$d-1{XiNeN z4$W1KzLhI9$ZDl@1}T=R&!oL7e${M~tN8SN{+fh%iYE*;X zwG^P-R?lLG@Y}9HUGS>O6gAnM2@p9UX~}pDvOaHPsRky$Y@R)Uuc;wt8~IB^m|a-| zyw_`JE!gzA=GAO(!~nmA>Ok6FTCfDd`uGd}F!mo~g(#tl$I6i#a}Yvd+WB zwIajVm4?WVKwYoHm`6kgVoS82uk*X^$PnFMRv#XVvy18)LMuA~jXNe*28n4V`&Kh9 z85s_}QmF!n1LJUZ>&3C})5bH5^LHTCXB5z4Ti)KAfsIdNfc^2z*t>4`vwAhe1!#Ok z#1qoMt5Hy+oeKUV3}@>xJ`!&}IocPlHxBCbUak#tTG@W-zqmf@X(<)i(%ino0)Beb zbc&ma%8M~WA)3ln#xnroIZL2oPZISQU7Bhg`8@1_LlP`uz z^|`kam8ZW+cK@h6iN{8AaGl8abq=sGc;}O{v7vvB$|66NU3w+)?kf>Z9R%SW zd9>OP9t+Y^6E-!V{p5UQqDeNpqylaSEuMRV{J#A;!{59<`jxE_Zcw{K+m*5Rex!~K z7d-7%X@&ixOQ*C2#J?Pz3=d^Jiw+^a3P&=lv1z53&X#-*&1Tc-&=r5T=JOg)c~D+A$Cro+DCt+!W-&R%;##_4Ec7OphjzE*4I(n%&#*bK^FodmTB%p zgM7_=w@AG5r&vkueQkCp4A&<8QGV;0XzN6R1CkJwll-4yyv~|=s8{vpX&63q61TegeDXM4BO!AFaSL-@{Ki&HYQ ztw;A^X<5b#3=`fFq_pz!&P;Y0G(ZyeF_dRbgvrjV7AiC?<&;XgiW@F}JYuppH5l;R zDI6k15`}OYu9%;24~CU~N&g<=AJAsT{lRoWAwfPRVeh_lE7sfKeFr?uH;#*`v@G=` z++O^*HypR2F7Q~gkx}&AADr-eFJG|y07r+A1P|lB&C1_2t7$EfalEd!m>8O`l_n0e zqZx@e>^9AHU^UGVf^d18t*P!(-@zXu9!^?4u|H^PneNO6%*o+o)s@i191H>Gvt`Fw z{<4tsB)3{}WC}FkhYV%EeJWlQ{7viNepQD*a=cHKv;blaLRoRyKM?#bjUa%Nf>6D^ z{0ko)1=s z3P7qr@@a>E;iHhfh!}a)Q$;Y1q zeJQ4Q+<$Yd=&vIk9|1;&uJrMc`~{33(;mLg_wK1*_$UFmg??#dr={o5OqlxsILdwl z6ij<1;IFd%jZ)lEz{m(oW=SW0!5(P=1^=Jv{-5dYt*QSr-M!WI{|m1xM}SHHuk}l! zFW8jrMvJAm{oBuMs2x1-u^WF4eCD4WkKYRar>51FVX5Nd`+10zVfv~Sk5K0!QRe?# zPtVEpaO!^onEr@2pIY{C*@pfre@oBtXDJ{1J8beOkT&cq>y(s`QC} zjxq9Ifk+!O9K^S8wD{wVs{26Ph;ii(eX)=C{gc<#bf&5aH@)#>&Ntlj_AhYeQXOnA z@$K-4YIZ$pbMc?WK0xho?%|-D7>$n1KYv($b8qf;$#2f>p9}pSAj%AZLB{*#vhAny zKlJx2VA0=|G|Ue~rnXPIKjS<8qG;ea>F%CRD6(mZp%r1OP}?26IVL`{3t7^-$mSh`27C?HGGv2 literal 0 HcmV?d00001 From 3ef0aef9954f001535049b830df3e9fb32f576e1 Mon Sep 17 00:00:00 2001 From: Yahya Hassanzadeh Date: Tue, 5 Dec 2023 14:58:57 -0800 Subject: [PATCH 59/67] extends documentation --- network/p2p/scoring/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/network/p2p/scoring/README.md b/network/p2p/scoring/README.md index e08bc0c6787..dda1cd7cb0b 100644 --- a/network/p2p/scoring/README.md +++ b/network/p2p/scoring/README.md @@ -262,6 +262,8 @@ gossipSubOption := scoreOption.BuildGossipSubScoreOption() # Caching Application Specific Score ![app-specific-score-cache.png](app-specific-score-cache.png) +The application-specific score of a peer is part of its overall score in the GossipSub protocol. In contrast to the rest of the GossipSub score of the peer that is computed +internally by the GossipSub protocol, the application-specific score of a peer is computed externally by the application, i.e., the Flow protocol-level semantics. As the figure above illustrates, GossipSub's peer scoring mechanism invokes the application-specific scoring function on a peer id upon receiving a gossip message from that peer. This means that the application-specific score of a peer is computed every time a gossip message is received from that peer. This can be computationally expensive, especially when the network is large and the number of gossip messages is high. From 17580a2fb96b0c0370e2b27e4b651581bf02499e Mon Sep 17 00:00:00 2001 From: Yahya Hassanzadeh Date: Tue, 5 Dec 2023 17:05:01 -0800 Subject: [PATCH 60/67] re-organizes the code --- .../p2pbuilder/gossipsub/gossipSubBuilder.go | 47 +++++----- network/p2p/p2pbuilder/libp2pNodeBuilder.go | 86 +++++++++---------- 2 files changed, 63 insertions(+), 70 deletions(-) diff --git a/network/p2p/p2pbuilder/gossipsub/gossipSubBuilder.go b/network/p2p/p2pbuilder/gossipsub/gossipSubBuilder.go index d44030f392f..1068fc0d66a 100644 --- a/network/p2p/p2pbuilder/gossipsub/gossipSubBuilder.go +++ b/network/p2p/p2pbuilder/gossipsub/gossipSubBuilder.go @@ -147,8 +147,12 @@ func (g *Builder) OverrideDefaultRpcInspectorSuiteFactory(factory p2p.GossipSubR // Returns: // - a new gossipsub builder. // Note: the builder is not thread-safe. It should only be used in the main thread. -func NewGossipSubBuilder(logger zerolog.Logger, metricsCfg *p2pconfig.MetricsConfig, gossipSubCfg *p2pconf.GossipSubParameters, - networkType network.NetworkingType, sporkId flow.Identifier, idProvider module.IdentityProvider) *Builder { +func NewGossipSubBuilder(logger zerolog.Logger, + metricsCfg *p2pconfig.MetricsConfig, + gossipSubCfg *p2pconf.GossipSubParameters, + networkType network.NetworkingType, + sporkId flow.Identifier, + idProvider module.IdentityProvider) *Builder { lg := logger.With(). Str("component", "gossipsub"). Str("network-type", networkType.String()). @@ -187,12 +191,7 @@ func NewGossipSubBuilder(logger zerolog.Logger, metricsCfg *p2pconfig.MetricsCon // defaultGossipSubFactory returns the default gossipsub factory function. It is used to create the default gossipsub factory. // Note: always use the default gossipsub factory function to create the gossipsub factory (unless you know what you are doing). func defaultGossipSubFactory() p2p.GossipSubFactoryFunc { - return func( - ctx context.Context, - logger zerolog.Logger, - h host.Host, - cfg p2p.PubSubAdapterConfig, - clusterChangeConsumer p2p.CollectionClusterChangesConsumer) (p2p.PubSubAdapter, error) { + return func(ctx context.Context, logger zerolog.Logger, h host.Host, cfg p2p.PubSubAdapterConfig, clusterChangeConsumer p2p.CollectionClusterChangesConsumer) (p2p.PubSubAdapter, error) { return p2pnode.NewGossipSubAdapter(ctx, logger, h, cfg, clusterChangeConsumer) } } @@ -209,8 +208,7 @@ func defaultGossipSubAdapterConfig() p2p.GossipSubAdapterConfigFunc { // Inspector suite is utilized to inspect the incoming gossipsub rpc messages from different perspectives. // Note: always use the default inspector suite factory function to create the inspector suite (unless you know what you are doing). func defaultInspectorSuite(rpcTracker p2p.RpcControlTracking) p2p.GossipSubRpcInspectorSuiteFactoryFunc { - return func( - ctx irrecoverable.SignalerContext, + return func(ctx irrecoverable.SignalerContext, logger zerolog.Logger, sporkId flow.Identifier, inspectorCfg *p2pconf.RpcInspectorParameters, @@ -219,21 +217,16 @@ func defaultInspectorSuite(rpcTracker p2p.RpcControlTracking) p2p.GossipSubRpcIn networkType network.NetworkingType, idProvider module.IdentityProvider, topicProvider func() p2p.TopicProvider) (p2p.GossipSubInspectorSuite, error) { - metricsInspector := inspector.NewControlMsgMetricsInspector( - logger, + metricsInspector := inspector.NewControlMsgMetricsInspector(logger, p2pnode.NewGossipSubControlMessageMetrics(gossipSubMetrics, logger), inspectorCfg.Metrics.NumberOfWorkers, []queue.HeroStoreConfigOption{ queue.WithHeroStoreSizeLimit(inspectorCfg.Metrics.CacheSize), - queue.WithHeroStoreCollector( - metrics.GossipSubRPCMetricsObserverInspectorQueueMetricFactory( - heroCacheMetricsFactory, - networkType)), + queue.WithHeroStoreCollector(metrics.GossipSubRPCMetricsObserverInspectorQueueMetricFactory(heroCacheMetricsFactory, networkType)), }...) - notificationDistributor := distributor.DefaultGossipSubInspectorNotificationDistributor( - logger, []queue.HeroStoreConfigOption{ - queue.WithHeroStoreSizeLimit(inspectorCfg.NotificationCacheSize), - queue.WithHeroStoreCollector(metrics.RpcInspectorNotificationQueueMetricFactory(heroCacheMetricsFactory, networkType))}...) + notificationDistributor := distributor.DefaultGossipSubInspectorNotificationDistributor(logger, []queue.HeroStoreConfigOption{ + queue.WithHeroStoreSizeLimit(inspectorCfg.NotificationCacheSize), + queue.WithHeroStoreCollector(metrics.RpcInspectorNotificationQueueMetricFactory(heroCacheMetricsFactory, networkType))}...) params := &validation.InspectorParams{ Logger: logger, @@ -270,10 +263,9 @@ func (g *Builder) Build(ctx irrecoverable.SignalerContext) (p2p.PubSubAdapter, e // before it is created). var gossipSub p2p.PubSubAdapter - gossipSubConfigs := g.gossipSubConfigFunc( - &p2p.BasePubSubAdapterConfig{ - MaxMessageSize: p2pnode.DefaultMaxPubSubMsgSize, - }) + gossipSubConfigs := g.gossipSubConfigFunc(&p2p.BasePubSubAdapterConfig{ + MaxMessageSize: p2pnode.DefaultMaxPubSubMsgSize, + }) gossipSubConfigs.WithMessageIdFunction(utils.MessageID) if g.routingSystem != nil { @@ -284,9 +276,10 @@ func (g *Builder) Build(ctx irrecoverable.SignalerContext) (p2p.PubSubAdapter, e gossipSubConfigs.WithSubscriptionFilter(g.subscriptionFilter) } - inspectorSuite, err := g.rpcInspectorSuiteFactory( - ctx, - g.logger, g.sporkId, &g.gossipSubCfg.RpcInspector, + inspectorSuite, err := g.rpcInspectorSuiteFactory(ctx, + g.logger, + g.sporkId, + &g.gossipSubCfg.RpcInspector, g.metricsCfg.Metrics, g.metricsCfg.HeroCacheFactory, g.networkType, diff --git a/network/p2p/p2pbuilder/libp2pNodeBuilder.go b/network/p2p/p2pbuilder/libp2pNodeBuilder.go index fe149f7f47b..4ba5f047a96 100644 --- a/network/p2p/p2pbuilder/libp2pNodeBuilder.go +++ b/network/p2p/p2pbuilder/libp2pNodeBuilder.go @@ -74,7 +74,8 @@ type LibP2PNodeBuilder struct { } func NewNodeBuilder( - logger zerolog.Logger, gossipSubCfg *p2pconf.GossipSubParameters, + logger zerolog.Logger, + gossipSubCfg *p2pconf.GossipSubParameters, metricsConfig *p2pconfig.MetricsConfig, networkingType flownet.NetworkingType, address string, @@ -97,9 +98,11 @@ func NewNodeBuilder( disallowListCacheCfg: disallowListCacheCfg, networkingType: networkingType, gossipSubBuilder: gossipsubbuilder.NewGossipSubBuilder(logger, - metricsConfig, gossipSubCfg, + metricsConfig, + gossipSubCfg, networkingType, - sporkId, idProvider), + sporkId, + idProvider), peerManagerConfig: peerManagerConfig, unicastConfig: unicastConfig, } @@ -277,45 +280,43 @@ func (builder *LibP2PNodeBuilder) Build() (p2p.LibP2PNode, error) { node.SetUnicastManager(unicastManager) cm := component.NewComponentManagerBuilder(). - AddWorker( - func(ctx irrecoverable.SignalerContext, ready component.ReadyFunc) { - if builder.routingFactory != nil { - routingSystem, err := builder.routingFactory(ctx, h) - if err != nil { - ctx.Throw(fmt.Errorf("could not create routing system: %w", err)) - } - if err := node.SetRouting(routingSystem); err != nil { - ctx.Throw(fmt.Errorf("could not set routing system: %w", err)) - } - builder.gossipSubBuilder.SetRoutingSystem(routingSystem) - lg.Debug().Msg("routing system created") - } - // gossipsub is created here, because it needs to be created during the node startup. - gossipSub, err := builder.gossipSubBuilder.Build(ctx) + AddWorker(func(ctx irrecoverable.SignalerContext, ready component.ReadyFunc) { + if builder.routingFactory != nil { + routingSystem, err := builder.routingFactory(ctx, h) if err != nil { - ctx.Throw(fmt.Errorf("could not create gossipsub: %w", err)) + ctx.Throw(fmt.Errorf("could not create routing system: %w", err)) } - node.SetPubSub(gossipSub) - gossipSub.Start(ctx) - ready() - - <-gossipSub.Done() - }). - AddWorker( - func(ctx irrecoverable.SignalerContext, ready component.ReadyFunc) { - // encapsulates shutdown logic for the libp2p node. - ready() - <-ctx.Done() - // we wait till the context is done, and then we stop the libp2p node. - - err = node.Stop() - if err != nil { - // ignore context cancellation errors - if !errors.Is(err, context.Canceled) && !errors.Is(err, context.DeadlineExceeded) { - ctx.Throw(fmt.Errorf("could not stop libp2p node: %w", err)) - } + if err := node.SetRouting(routingSystem); err != nil { + ctx.Throw(fmt.Errorf("could not set routing system: %w", err)) } - }) + builder.gossipSubBuilder.SetRoutingSystem(routingSystem) + lg.Debug().Msg("routing system created") + } + // gossipsub is created here, because it needs to be created during the node startup. + gossipSub, err := builder.gossipSubBuilder.Build(ctx) + if err != nil { + ctx.Throw(fmt.Errorf("could not create gossipsub: %w", err)) + } + node.SetPubSub(gossipSub) + gossipSub.Start(ctx) + ready() + + <-gossipSub.Done() + }). + AddWorker(func(ctx irrecoverable.SignalerContext, ready component.ReadyFunc) { + // encapsulates shutdown logic for the libp2p node. + ready() + <-ctx.Done() + // we wait till the context is done, and then we stop the libp2p node. + + err = node.Stop() + if err != nil { + // ignore context cancellation errors + if !errors.Is(err, context.Canceled) && !errors.Is(err, context.DeadlineExceeded) { + ctx.Throw(fmt.Errorf("could not stop libp2p node: %w", err)) + } + } + }) node.SetComponentManager(cm.Build()) @@ -364,10 +365,9 @@ func defaultLibP2POptions(address string, key fcrypto.PrivateKey) ([]config.Opti // While this sounds great, it intermittently causes a 'broken pipe' error // as the 1-k discovery process and the 1-1 messaging both sometimes attempt to open connection to the same target // As of now there is no requirement of client sockets to be a well-known port, so disabling port reuse all together. - t := libp2p.Transport( - func(u transport.Upgrader) (*tcp.TcpTransport, error) { - return tcp.NewTCPTransport(u, nil, tcp.DisableReuseport()) - }) + t := libp2p.Transport(func(u transport.Upgrader) (*tcp.TcpTransport, error) { + return tcp.NewTCPTransport(u, nil, tcp.DisableReuseport()) + }) // gather all the options for the libp2p node options := []config.Option{ From 2b8ec645b42d7911dc639f3d9f86b989b8211087 Mon Sep 17 00:00:00 2001 From: Yahya Hassanzadeh Date: Tue, 5 Dec 2023 17:29:41 -0800 Subject: [PATCH 61/67] removes manual enable of gossipsub scoring though builder --- .../rpc_inspector/validation_inspector_test.go | 3 +-- .../test/gossipsub/scoring/scoring_test.go | 4 +--- network/p2p/builder.go | 13 +++++-------- network/p2p/mock/node_builder.go | 12 ++++++------ network/p2p/p2pbuilder/libp2pNodeBuilder.go | 10 +++------- network/p2p/scoring/app_score_test.go | 17 +++++------------ network/p2p/scoring/scoring_test.go | 4 +--- .../p2p/scoring/subscription_validator_test.go | 3 --- network/p2p/test/fixtures.go | 2 +- 9 files changed, 23 insertions(+), 45 deletions(-) diff --git a/insecure/integration/functional/test/gossipsub/rpc_inspector/validation_inspector_test.go b/insecure/integration/functional/test/gossipsub/rpc_inspector/validation_inspector_test.go index 75287750db4..54f94f6e721 100644 --- a/insecure/integration/functional/test/gossipsub/rpc_inspector/validation_inspector_test.go +++ b/insecure/integration/functional/test/gossipsub/rpc_inspector/validation_inspector_test.go @@ -1041,8 +1041,7 @@ func TestGossipSubSpamMitigationIntegration(t *testing.T) { t.Name(), idProvider, p2ptest.WithRole(flow.RoleConsensus), - p2ptest.OverrideFlowConfig(cfg), - p2ptest.EnablePeerScoringWithOverride(p2p.PeerScoringConfigNoOverride)) + p2ptest.OverrideFlowConfig(cfg)) ids := flow.IdentityList{&victimId, &spammer.SpammerId} idProvider.On("ByPeerID", mockery.Anything).Return(func(peerId peer.ID) *flow.Identity { diff --git a/insecure/integration/functional/test/gossipsub/scoring/scoring_test.go b/insecure/integration/functional/test/gossipsub/scoring/scoring_test.go index 0c347986ec1..0197f219960 100644 --- a/insecure/integration/functional/test/gossipsub/scoring/scoring_test.go +++ b/insecure/integration/functional/test/gossipsub/scoring/scoring_test.go @@ -117,9 +117,7 @@ func testGossipSubInvalidMessageDeliveryScoring(t *testing.T, spamMsgFactory fun t.Name(), idProvider, p2ptest.WithRole(role), - p2ptest.OverrideFlowConfig(cfg), - p2ptest.EnablePeerScoringWithOverride(p2p.PeerScoringConfigNoOverride), - ) + p2ptest.OverrideFlowConfig(cfg)) idProvider.On("ByPeerID", victimNode.ID()).Return(&victimIdentity, true).Maybe() idProvider.On("ByPeerID", spammer.SpammerNode.ID()).Return(&spammer.SpammerId, true).Maybe() diff --git a/network/p2p/builder.go b/network/p2p/builder.go index 5a8733ec0e6..7e7d38504f7 100644 --- a/network/p2p/builder.go +++ b/network/p2p/builder.go @@ -113,16 +113,18 @@ type NodeBuilder interface { SetConnectionGater(ConnectionGater) NodeBuilder SetRoutingSystem(func(context.Context, host.Host) (routing.Routing, error)) NodeBuilder - // EnableGossipSubScoringWithOverride enables peer scoring for the GossipSubParameters pubsub system with the given override. + // OverrideGossipSubScoringConfig overrides the default peer scoring config for the GossipSub protocol. + // Note that it does not enable peer scoring. The peer scoring is enabled directly by setting the `peer-scoring-enabled` flag to true in `default-config.yaml`, or + // by setting the `gossipsub-peer-scoring-enabled` runtime flag to true. This function only overrides the default peer scoring config which takes effect + // only if the peer scoring is enabled (mostly for testing purposes). // Any existing peer scoring config attribute that is set in the override will override the default peer scoring config. // Anything that is left to nil or zero value in the override will be ignored and the default value will be used. // Note: it is not recommended to override the default peer scoring config in production unless you know what you are doing. - // Production Tip: use PeerScoringConfigNoOverride as the argument to this function to enable peer scoring without any override. // Args: // - PeerScoringConfigOverride: override for the peer scoring config- Recommended to use PeerScoringConfigNoOverride for production. // Returns: // none - EnableGossipSubScoringWithOverride(*PeerScoringConfigOverride) NodeBuilder + OverrideGossipSubScoringConfig(*PeerScoringConfigOverride) NodeBuilder SetCreateNode(CreateNodeFunc) NodeBuilder SetGossipSubFactory(GossipSubFactoryFunc, GossipSubAdapterConfigFunc) NodeBuilder OverrideDefaultRpcInspectorSuiteFactory(GossipSubRpcInspectorSuiteFactoryFunc) NodeBuilder @@ -145,8 +147,3 @@ type PeerScoringConfigOverride struct { // If the function is nil, the default application specific score parameters are used. AppSpecificScoreParams func(peer.ID) float64 } - -// PeerScoringConfigNoOverride is a default peer scoring configuration for a GossipSubParameters pubsub system. -// It is set to nil, which means that no override is done to the default peer scoring configuration. -// It is the recommended way to use the default peer scoring configuration. -var PeerScoringConfigNoOverride = (*PeerScoringConfigOverride)(nil) diff --git a/network/p2p/mock/node_builder.go b/network/p2p/mock/node_builder.go index 2a9f1ecaef8..c6eec64d026 100644 --- a/network/p2p/mock/node_builder.go +++ b/network/p2p/mock/node_builder.go @@ -53,12 +53,12 @@ func (_m *NodeBuilder) Build() (p2p.LibP2PNode, error) { return r0, r1 } -// EnableGossipSubScoringWithOverride provides a mock function with given fields: _a0 -func (_m *NodeBuilder) EnableGossipSubScoringWithOverride(_a0 *p2p.PeerScoringConfigOverride) p2p.NodeBuilder { +// OverrideDefaultRpcInspectorSuiteFactory provides a mock function with given fields: _a0 +func (_m *NodeBuilder) OverrideDefaultRpcInspectorSuiteFactory(_a0 p2p.GossipSubRpcInspectorSuiteFactoryFunc) p2p.NodeBuilder { ret := _m.Called(_a0) var r0 p2p.NodeBuilder - if rf, ok := ret.Get(0).(func(*p2p.PeerScoringConfigOverride) p2p.NodeBuilder); ok { + if rf, ok := ret.Get(0).(func(p2p.GossipSubRpcInspectorSuiteFactoryFunc) p2p.NodeBuilder); ok { r0 = rf(_a0) } else { if ret.Get(0) != nil { @@ -69,12 +69,12 @@ func (_m *NodeBuilder) EnableGossipSubScoringWithOverride(_a0 *p2p.PeerScoringCo return r0 } -// OverrideDefaultRpcInspectorSuiteFactory provides a mock function with given fields: _a0 -func (_m *NodeBuilder) OverrideDefaultRpcInspectorSuiteFactory(_a0 p2p.GossipSubRpcInspectorSuiteFactoryFunc) p2p.NodeBuilder { +// OverrideGossipSubScoringConfig provides a mock function with given fields: _a0 +func (_m *NodeBuilder) OverrideGossipSubScoringConfig(_a0 *p2p.PeerScoringConfigOverride) p2p.NodeBuilder { ret := _m.Called(_a0) var r0 p2p.NodeBuilder - if rf, ok := ret.Get(0).(func(p2p.GossipSubRpcInspectorSuiteFactoryFunc) p2p.NodeBuilder); ok { + if rf, ok := ret.Get(0).(func(*p2p.PeerScoringConfigOverride) p2p.NodeBuilder); ok { r0 = rf(_a0) } else { if ret.Get(0) != nil { diff --git a/network/p2p/p2pbuilder/libp2pNodeBuilder.go b/network/p2p/p2pbuilder/libp2pNodeBuilder.go index 4ba5f047a96..24525fef43a 100644 --- a/network/p2p/p2pbuilder/libp2pNodeBuilder.go +++ b/network/p2p/p2pbuilder/libp2pNodeBuilder.go @@ -161,7 +161,7 @@ func (builder *LibP2PNodeBuilder) SetGossipSubFactory(gf p2p.GossipSubFactoryFun // - PeerScoringConfigOverride: override for the peer scoring config- Recommended to use PeerScoringConfigNoOverride for production. // Returns: // none -func (builder *LibP2PNodeBuilder) EnableGossipSubScoringWithOverride(config *p2p.PeerScoringConfigOverride) p2p.NodeBuilder { +func (builder *LibP2PNodeBuilder) OverrideGossipSubScoringConfig(config *p2p.PeerScoringConfigOverride) p2p.NodeBuilder { builder.gossipSubBuilder.EnableGossipSubScoringWithOverride(config) return builder } @@ -426,7 +426,8 @@ func DefaultNodeBuilder( connection.WithOnInterceptPeerDialFilters(append(peerFilters, connGaterCfg.InterceptPeerDialFilters...)), connection.WithOnInterceptSecuredFilters(append(peerFilters, connGaterCfg.InterceptSecuredFilters...))) - builder := NewNodeBuilder(logger, gossipCfg, + builder := NewNodeBuilder(logger, + gossipCfg, metricsCfg, networkingType, address, @@ -441,11 +442,6 @@ func DefaultNodeBuilder( SetConnectionGater(connGater). SetCreateNode(DefaultCreateNodeFunc) - if gossipCfg.PeerScoringEnabled { - // In production, we never override the default scoring config. - builder.EnableGossipSubScoringWithOverride(p2p.PeerScoringConfigNoOverride) - } - if role != "ghost" { r, err := flow.ParseRole(role) if err != nil { diff --git a/network/p2p/scoring/app_score_test.go b/network/p2p/scoring/app_score_test.go index 787010aac2b..d693b77207c 100644 --- a/network/p2p/scoring/app_score_test.go +++ b/network/p2p/scoring/app_score_test.go @@ -36,16 +36,13 @@ func TestFullGossipSubConnectivity(t *testing.T) { // two groups of non-access nodes and one group of access nodes. groupOneNodes, groupOneIds := p2ptest.NodesFixture(t, sporkId, t.Name(), 5, idProvider, - p2ptest.WithRole(flow.RoleConsensus), - p2ptest.EnablePeerScoringWithOverride(p2p.PeerScoringConfigNoOverride)) + p2ptest.WithRole(flow.RoleConsensus)) groupTwoNodes, groupTwoIds := p2ptest.NodesFixture(t, sporkId, t.Name(), 5, idProvider, - p2ptest.WithRole(flow.RoleCollection), - p2ptest.EnablePeerScoringWithOverride(p2p.PeerScoringConfigNoOverride)) + p2ptest.WithRole(flow.RoleCollection)) accessNodeGroup, accessNodeIds := p2ptest.NodesFixture(t, sporkId, t.Name(), 5, idProvider, - p2ptest.WithRole(flow.RoleAccess), - p2ptest.EnablePeerScoringWithOverride(p2p.PeerScoringConfigNoOverride)) + p2ptest.WithRole(flow.RoleAccess)) ids := append(append(groupOneIds, groupTwoIds...), accessNodeIds...) nodes := append(append(groupOneNodes, groupTwoNodes...), accessNodeGroup...) @@ -129,17 +126,13 @@ func TestFullGossipSubConnectivityAmongHonestNodesWithMaliciousMajority(t *testi sporkId := unittest.IdentifierFixture() idProvider := mock.NewIdentityProvider(t) - // two (honest) consensus nodes - opts := []p2ptest.NodeFixtureParameterOption{p2ptest.WithRole(flow.RoleConsensus)} - opts = append(opts, p2ptest.EnablePeerScoringWithOverride(p2p.PeerScoringConfigNoOverride)) - defaultConfig, err := config.DefaultConfig() require.NoError(t, err) // override the default config to make the mesh tracer log more frequently defaultConfig.NetworkConfig.GossipSub.RpcTracer.LocalMeshLogInterval = time.Second - con1Node, con1Id := p2ptest.NodeFixture(t, sporkId, t.Name(), idProvider, append(opts, p2ptest.OverrideFlowConfig(defaultConfig))...) - con2Node, con2Id := p2ptest.NodeFixture(t, sporkId, t.Name(), idProvider, append(opts, p2ptest.OverrideFlowConfig(defaultConfig))...) + con1Node, con1Id := p2ptest.NodeFixture(t, sporkId, t.Name(), idProvider, p2ptest.WithRole(flow.RoleConsensus), p2ptest.OverrideFlowConfig(defaultConfig)) + con2Node, con2Id := p2ptest.NodeFixture(t, sporkId, t.Name(), idProvider, p2ptest.WithRole(flow.RoleConsensus), p2ptest.OverrideFlowConfig(defaultConfig)) // create > 2 * 12 malicious access nodes // 12 is the maximum size of default GossipSub mesh. diff --git a/network/p2p/scoring/scoring_test.go b/network/p2p/scoring/scoring_test.go index cb8c7eccbc1..50503882d6c 100644 --- a/network/p2p/scoring/scoring_test.go +++ b/network/p2p/scoring/scoring_test.go @@ -113,7 +113,6 @@ func TestInvalidCtrlMsgScoringIntegration(t *testing.T) { idProvider, p2ptest.WithRole(flow.RoleConsensus), p2ptest.OverrideFlowConfig(cfg), - p2ptest.EnablePeerScoringWithOverride(p2p.PeerScoringConfigNoOverride), p2ptest.OverrideGossipSubRpcInspectorSuiteFactory(factory)) node2, id2 := p2ptest.NodeFixture( @@ -122,8 +121,7 @@ func TestInvalidCtrlMsgScoringIntegration(t *testing.T) { t.Name(), idProvider, p2ptest.WithRole(flow.RoleConsensus), - p2ptest.OverrideFlowConfig(cfg), - p2ptest.EnablePeerScoringWithOverride(p2p.PeerScoringConfigNoOverride)) + p2ptest.OverrideFlowConfig(cfg)) ids := flow.IdentityList{&id1, &id2} nodes := []p2p.LibP2PNode{node1, node2} diff --git a/network/p2p/scoring/subscription_validator_test.go b/network/p2p/scoring/subscription_validator_test.go index 15f93104539..b4edad1f04c 100644 --- a/network/p2p/scoring/subscription_validator_test.go +++ b/network/p2p/scoring/subscription_validator_test.go @@ -181,7 +181,6 @@ func TestSubscriptionValidator_Integration(t *testing.T) { idProvider, p2ptest.WithLogger(unittest.Logger()), p2ptest.OverrideFlowConfig(cfg), - p2ptest.EnablePeerScoringWithOverride(p2p.PeerScoringConfigNoOverride), p2ptest.WithRole(flow.RoleConsensus)) // two verification node. @@ -189,14 +188,12 @@ func TestSubscriptionValidator_Integration(t *testing.T) { idProvider, p2ptest.WithLogger(unittest.Logger()), p2ptest.OverrideFlowConfig(cfg), - p2ptest.EnablePeerScoringWithOverride(p2p.PeerScoringConfigNoOverride), p2ptest.WithRole(flow.RoleVerification)) verNode2, verId2 := p2ptest.NodeFixture(t, sporkId, t.Name(), idProvider, p2ptest.WithLogger(unittest.Logger()), p2ptest.OverrideFlowConfig(cfg), - p2ptest.EnablePeerScoringWithOverride(p2p.PeerScoringConfigNoOverride), p2ptest.WithRole(flow.RoleVerification)) // suppress peer provider error diff --git a/network/p2p/test/fixtures.go b/network/p2p/test/fixtures.go index 11132bd6949..2001b9653e4 100644 --- a/network/p2p/test/fixtures.go +++ b/network/p2p/test/fixtures.go @@ -175,7 +175,7 @@ func NodeFixture(t *testing.T, } if parameters.PeerScoringEnabled { - builder.EnableGossipSubScoringWithOverride(parameters.PeerScoringConfigOverride) + builder.OverrideGossipSubScoringConfig(parameters.PeerScoringConfigOverride) } if parameters.GossipSubFactory != nil && parameters.GossipSubConfig != nil { From 63c3454db34e39a4c03b898b73577daf3cd3672d Mon Sep 17 00:00:00 2001 From: Yahya Hassanzadeh Date: Tue, 5 Dec 2023 17:34:15 -0800 Subject: [PATCH 62/67] adds validate tag --- network/p2p/scoring/registry.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/network/p2p/scoring/registry.go b/network/p2p/scoring/registry.go index 8fdbd0b2067..824b66ac579 100644 --- a/network/p2p/scoring/registry.go +++ b/network/p2p/scoring/registry.go @@ -163,7 +163,7 @@ type GossipSubAppSpecificScoreRegistryConfig struct { HeroCacheMetricsFactory metrics.HeroCacheMetricsFactory `validate:"required"` - NetworkingType network.NetworkingType + NetworkingType network.NetworkingType `validate:"required"` } // NewGossipSubAppSpecificScoreRegistry returns a new GossipSubAppSpecificScoreRegistry. From eb7e444cdd5909330dec22866c20e997ff24b366 Mon Sep 17 00:00:00 2001 From: Yahya Hassanzadeh Date: Tue, 5 Dec 2023 17:40:15 -0800 Subject: [PATCH 63/67] reverts unittest logger --- utils/unittest/logging.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/unittest/logging.go b/utils/unittest/logging.go index c1b993cfab7..a200a61525e 100644 --- a/utils/unittest/logging.go +++ b/utils/unittest/logging.go @@ -30,7 +30,7 @@ func Logger() zerolog.Logger { writer = os.Stderr } - return LoggerWithWriterAndLevel(writer, zerolog.InfoLevel) + return LoggerWithWriterAndLevel(writer, zerolog.TraceLevel) } func LoggerWithWriterAndLevel(writer io.Writer, level zerolog.Level) zerolog.Logger { From a47ee19999dcdfe2511096b4567a5c4f5c4150e6 Mon Sep 17 00:00:00 2001 From: "Yahya Hassanzadeh, Ph.D" Date: Mon, 11 Dec 2023 15:28:08 -0800 Subject: [PATCH 64/67] Update network/p2p/scoring/registry.go Co-authored-by: Khalil Claybon --- network/p2p/scoring/registry.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/network/p2p/scoring/registry.go b/network/p2p/scoring/registry.go index 824b66ac579..eb5acb18194 100644 --- a/network/p2p/scoring/registry.go +++ b/network/p2p/scoring/registry.go @@ -263,7 +263,7 @@ func (r *GossipSubAppSpecificScoreRegistry) AppSpecificScoreFunc() func(peer.ID) return appSpecificScore // in the mean time, return the expired score. default: - // record found in the cache, check if it is expired. + // record found in the cache. r.logger.Trace(). Float64("app_specific_score", appSpecificScore). Msg("application specific score found in cache") From 9c96d156dc4214fa73be2cfb88158ac26b2b23ef Mon Sep 17 00:00:00 2001 From: Yahya Hassanzadeh Date: Mon, 11 Dec 2023 15:34:41 -0800 Subject: [PATCH 65/67] fixes gossipsub parameters instances --- network/p2p/builder.go | 12 ++++++------ .../p2p/p2pbuilder/gossipsub/gossipSubBuilder.go | 14 +++++++------- network/p2p/p2pbuilder/libp2pNodeBuilder.go | 2 +- network/p2p/p2pconf/gossipsub.go | 12 ++++++------ network/p2p/scoring/scoring_test.go | 4 ++-- 5 files changed, 22 insertions(+), 22 deletions(-) diff --git a/network/p2p/builder.go b/network/p2p/builder.go index 7e7d38504f7..ce563d61858 100644 --- a/network/p2p/builder.go +++ b/network/p2p/builder.go @@ -25,7 +25,7 @@ type GossipSubFactoryFunc func(context.Context, zerolog.Logger, host.Host, PubSu type CreateNodeFunc func(zerolog.Logger, host.Host, ProtocolPeerCache, PeerManager, *DisallowListCacheConfig) LibP2PNode type GossipSubAdapterConfigFunc func(*BasePubSubAdapterConfig) PubSubAdapterConfig -// GossipSubBuilder provides a builder pattern for creating a GossipSubParameters pubsub system. +// GossipSubBuilder provides a builder pattern for creating a GossipSub pubsub system. type GossipSubBuilder interface { // SetHost sets the host of the builder. // If the host has already been set, a fatal error is logged. @@ -43,7 +43,7 @@ type GossipSubBuilder interface { // We expect the node to initialize with a default gossipsub config. Hence, this function overrides the default config. SetGossipSubConfigFunc(GossipSubAdapterConfigFunc) - // EnableGossipSubScoringWithOverride enables peer scoring for the GossipSubParameters pubsub system with the given override. + // EnableGossipSubScoringWithOverride enables peer scoring for the GossipSub pubsub system with the given override. // Any existing peer scoring config attribute that is set in the override will override the default peer scoring config. // Anything that is left to nil or zero value in the override will be ignored and the default value will be used. // Note: it is not recommended to override the default peer scoring config in production unless you know what you are doing. @@ -65,15 +65,15 @@ type GossipSubBuilder interface { // It is NOT recommended to override the default RPC inspector suite factory in production unless you know what you are doing. OverrideDefaultRpcInspectorSuiteFactory(GossipSubRpcInspectorSuiteFactoryFunc) - // Build creates a new GossipSubParameters pubsub system. - // It returns the newly created GossipSubParameters pubsub system and any errors encountered during its creation. + // Build creates a new GossipSub pubsub system. + // It returns the newly created GossipSub pubsub system and any errors encountered during its creation. // // Arguments: // - context.Context: the irrecoverable context of the node. // // Returns: - // - PubSubAdapter: a GossipSubParameters pubsub system for the libp2p node. - // - error: if an error occurs during the creation of the GossipSubParameters pubsub system, it is returned. Otherwise, nil is returned. + // - PubSubAdapter: a GossipSub pubsub system for the libp2p node. + // - error: if an error occurs during the creation of the GossipSub pubsub system, it is returned. Otherwise, nil is returned. // Note that on happy path, the returned error is nil. Any error returned is unexpected and should be handled as irrecoverable. Build(irrecoverable.SignalerContext) (PubSubAdapter, error) } diff --git a/network/p2p/p2pbuilder/gossipsub/gossipSubBuilder.go b/network/p2p/p2pbuilder/gossipsub/gossipSubBuilder.go index 1068fc0d66a..9dc7d819195 100644 --- a/network/p2p/p2pbuilder/gossipsub/gossipSubBuilder.go +++ b/network/p2p/p2pbuilder/gossipsub/gossipSubBuilder.go @@ -29,7 +29,7 @@ import ( "github.com/onflow/flow-go/utils/logging" ) -// The Builder struct is used to configure and create a new GossipSubParameters pubsub system. +// The Builder struct is used to configure and create a new GossipSub pubsub system. type Builder struct { networkType network.NetworkingType sporkId flow.Identifier @@ -88,7 +88,7 @@ func (g *Builder) SetGossipSubConfigFunc(gossipSubConfigFunc p2p.GossipSubAdapte g.gossipSubConfigFunc = gossipSubConfigFunc } -// EnableGossipSubScoringWithOverride enables peer scoring for the GossipSubParameters pubsub system with the given override. +// EnableGossipSubScoringWithOverride enables peer scoring for the GossipSub pubsub system with the given override. // Any existing peer scoring config attribute that is set in the override will override the default peer scoring config. // Anything that is left to nil or zero value in the override will be ignored and the default value will be used. // Note: it is not recommended to override the default peer scoring config in production unless you know what you are doing. @@ -248,15 +248,15 @@ func defaultInspectorSuite(rpcTracker p2p.RpcControlTracking) p2p.GossipSubRpcIn } } -// Build creates a new GossipSubParameters pubsub system. -// It returns the newly created GossipSubParameters pubsub system and any errors encountered during its creation. +// Build creates a new GossipSub pubsub system. +// It returns the newly created GossipSub pubsub system and any errors encountered during its creation. // Arguments: // - ctx: the irrecoverable context of the node. // // Returns: -// - p2p.PubSubAdapter: a GossipSubParameters pubsub system for the libp2p node. -// - p2p.PeerScoreTracer: a peer score tracer for the GossipSubParameters pubsub system (if enabled, otherwise nil). -// - error: if an error occurs during the creation of the GossipSubParameters pubsub system, it is returned. Otherwise, nil is returned. +// - p2p.PubSubAdapter: a GossipSub pubsub system for the libp2p node. +// - p2p.PeerScoreTracer: a peer score tracer for the GossipSub pubsub system (if enabled, otherwise nil). +// - error: if an error occurs during the creation of the GossipSub pubsub system, it is returned. Otherwise, nil is returned. // Note that on happy path, the returned error is nil. Any error returned is unexpected and should be handled as irrecoverable. func (g *Builder) Build(ctx irrecoverable.SignalerContext) (p2p.PubSubAdapter, error) { // placeholder for the gossipsub pubsub system that will be created (so that it can be passed around even diff --git a/network/p2p/p2pbuilder/libp2pNodeBuilder.go b/network/p2p/p2pbuilder/libp2pNodeBuilder.go index 24525fef43a..06a8e97d26d 100644 --- a/network/p2p/p2pbuilder/libp2pNodeBuilder.go +++ b/network/p2p/p2pbuilder/libp2pNodeBuilder.go @@ -152,7 +152,7 @@ func (builder *LibP2PNodeBuilder) SetGossipSubFactory(gf p2p.GossipSubFactoryFun return builder } -// EnableGossipSubScoringWithOverride enables peer scoring for the GossipSubParameters pubsub system with the given override. +// OverrideGossipSubScoringConfig enables peer scoring for the GossipSub pubsub system with the given override. // Any existing peer scoring config attribute that is set in the override will override the default peer scoring config. // Anything that is left to nil or zero value in the override will be ignored and the default value will be used. // Note: it is not recommended to override the default peer scoring config in production unless you know what you are doing. diff --git a/network/p2p/p2pconf/gossipsub.go b/network/p2p/p2pconf/gossipsub.go index 1566796656b..946cef57cb1 100644 --- a/network/p2p/p2pconf/gossipsub.go +++ b/network/p2p/p2pconf/gossipsub.go @@ -21,13 +21,13 @@ type ResourceManagerOverrideScope struct { // Transient is the limit for the resource at the transient scope. Transient limits are used for resources that have not fully established and are under negotiation. Transient ResourceManagerOverrideLimit `mapstructure:"transient"` - // Protocol is the limit for the resource at the protocol scope, e.g., DHT, GossipSubParameters, etc. It dictates the maximum allowed resource across all peers for that protocol. + // Protocol is the limit for the resource at the protocol scope, e.g., DHT, GossipSub, etc. It dictates the maximum allowed resource across all peers for that protocol. Protocol ResourceManagerOverrideLimit `mapstructure:"protocol"` // Peer is the limit for the resource at the peer scope. It dictates the maximum allowed resource for a specific peer. Peer ResourceManagerOverrideLimit `mapstructure:"peer"` - // Connection is the limit for the resource for a pair of (peer, protocol), e.g., (peer1, DHT), (peer1, GossipSubParameters), etc. It dictates the maximum allowed resource for a protocol and a peer. + // Connection is the limit for the resource for a pair of (peer, protocol), e.g., (peer1, DHT), (peer1, GossipSub), etc. It dictates the maximum allowed resource for a protocol and a peer. PeerProtocol ResourceManagerOverrideLimit `mapstructure:"peer-protocol"` } @@ -63,16 +63,16 @@ const ( SubscriptionProviderKey = "subscription-provider" ) -// GossipSubParameters is the configuration for the GossipSubParameters pubsub implementation. +// GossipSubParameters is the configuration for the GossipSub pubsub implementation. type GossipSubParameters struct { // RpcInspectorParameters configuration for all gossipsub RPC control message inspectors. RpcInspector RpcInspectorParameters `mapstructure:"rpc-inspector"` // GossipSubScoringRegistryConfig is the configuration for the GossipSub score registry. - // GossipSubTracerParameters is the configuration for the gossipsub tracer. GossipSubParameters tracer is used to trace the local mesh events and peer scores. + // GossipSubTracerParameters is the configuration for the gossipsub tracer. GossipSub tracer is used to trace the local mesh events and peer scores. RpcTracer GossipSubTracerParameters `mapstructure:"rpc-tracer"` - // ScoringParameters is whether to enable GossipSubParameters peer scoring. + // ScoringParameters is whether to enable GossipSub peer scoring. PeerScoringEnabled bool `mapstructure:"peer-scoring-enabled"` SubscriptionProvider SubscriptionProviderParameters `mapstructure:"subscription-provider"` ScoringParameters ScoringParameters `mapstructure:"scoring-parameters"` @@ -164,7 +164,7 @@ const ( RPCSentTrackerNumOfWorkersKey = "rpc-sent-tracker-workers" ) -// GossipSubTracerParameters is the config for the gossipsub tracer. GossipSubParameters tracer is used to trace the local mesh events and peer scores. +// GossipSubTracerParameters is the config for the gossipsub tracer. GossipSub tracer is used to trace the local mesh events and peer scores. type GossipSubTracerParameters struct { // LocalMeshLogInterval is the interval at which the local mesh is logged. LocalMeshLogInterval time.Duration `validate:"gt=0s" mapstructure:"local-mesh-logging-interval"` diff --git a/network/p2p/scoring/scoring_test.go b/network/p2p/scoring/scoring_test.go index 50503882d6c..7ab83fc11c8 100644 --- a/network/p2p/scoring/scoring_test.go +++ b/network/p2p/scoring/scoring_test.go @@ -140,7 +140,7 @@ func TestInvalidCtrlMsgScoringIntegration(t *testing.T) { p2ptest.LetNodesDiscoverEachOther(t, ctx, nodes, ids) blockTopic := channels.TopicFromChannel(channels.PushBlocks, sporkId) - // checks end-to-end message delivery works on GossipSubParameters + // checks end-to-end message delivery works on GossipSub. p2ptest.EnsurePubsubMessageExchange(t, ctx, nodes, blockTopic, 1, func() interface{} { return unittest.ProposalFixture() }) @@ -157,7 +157,7 @@ func TestInvalidCtrlMsgScoringIntegration(t *testing.T) { time.Sleep(1 * time.Second) // wait for app-specific score to be updated in the cache (remember that we need at least 100 ms for the score to be updated (ScoreTTL)) - // checks no GossipSubParameters message exchange should no longer happen between node1 and node2. + // checks no GossipSub message exchange should no longer happen between node1 and node2. p2ptest.EnsureNoPubsubExchangeBetweenGroups( t, ctx, From 669af4ef3cc59525090d186ba862ac8f4bb9eb33 Mon Sep 17 00:00:00 2001 From: Yahya Hassanzadeh Date: Thu, 14 Dec 2023 10:00:02 -0800 Subject: [PATCH 66/67] removes defensive checks on add --- network/p2p/scoring/internal/appSpecificScoreCache.go | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/network/p2p/scoring/internal/appSpecificScoreCache.go b/network/p2p/scoring/internal/appSpecificScoreCache.go index 7e75996132f..10e2957e03e 100644 --- a/network/p2p/scoring/internal/appSpecificScoreCache.go +++ b/network/p2p/scoring/internal/appSpecificScoreCache.go @@ -79,7 +79,7 @@ func (a *AppSpecificScoreCache) Add(peerID peer.ID, score float64, time time.Tim LastUpdated: time, }) if !added { - updated, ok := a.c.Adjust(entityId, func(entity flow.Entity) flow.Entity { + _, ok := a.c.Adjust(entityId, func(entity flow.Entity) flow.Entity { r := entity.(appSpecificScoreRecordEntity) r.Score = score r.LastUpdated = time @@ -89,14 +89,6 @@ func (a *AppSpecificScoreCache) Add(peerID peer.ID, score float64, time time.Tim if !ok { return fmt.Errorf("failed to add app specific score record for peer %s", peerID) } - - u := updated.(appSpecificScoreRecordEntity) - if u.Score != score { - return fmt.Errorf("incorrect update of app specific score record for peer %s, expected score %f, got score %f", peerID, score, u.Score) - } - if u.LastUpdated != time { - return fmt.Errorf("incorrect update of app specific score record for peer %s, expected last updated %s, got last updated %s", peerID, time, u.LastUpdated) - } } return nil From d2ece43bb8ad29e0cbde66fc4cd21b69a6fb8da0 Mon Sep 17 00:00:00 2001 From: Yahya Hassanzadeh Date: Thu, 14 Dec 2023 15:49:21 -0800 Subject: [PATCH 67/67] disables peer scoring on public network --- network/p2p/p2pbuilder/gossipsub/gossipSubBuilder.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/network/p2p/p2pbuilder/gossipsub/gossipSubBuilder.go b/network/p2p/p2pbuilder/gossipsub/gossipSubBuilder.go index 9dc7d819195..1954bb22430 100644 --- a/network/p2p/p2pbuilder/gossipsub/gossipSubBuilder.go +++ b/network/p2p/p2pbuilder/gossipsub/gossipSubBuilder.go @@ -294,7 +294,8 @@ func (g *Builder) Build(ctx irrecoverable.SignalerContext) (p2p.PubSubAdapter, e var scoreOpt *scoring.ScoreOption var scoreTracer p2p.PeerScoreTracer - if g.gossipSubCfg.PeerScoringEnabled { + // currently, peer scoring is not supported for public networks. + if g.gossipSubCfg.PeerScoringEnabled && g.networkType != network.PublicNetwork { // wires the gossipsub score option to the subscription provider. subscriptionProvider, err := scoring.NewSubscriptionProvider(&scoring.SubscriptionProviderConfig{ Logger: g.logger,