Skip to content

Commit

Permalink
Fix flaky test due to sorting issue
Browse files Browse the repository at this point in the history 
Signed-off-by: Emmanuel Lodovice <[email protected]>
  • Loading branch information
@emanlodovice
emanlodovice committed Apr 12, 2024
1 parent daf1f83 commit d8955ab
Show file tree
Hide file tree
Showing 2 changed files with 48 additions and 45 deletions.
1 change: 1 addition & 0 deletions pkg/ring/ring_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -987,6 +987,7 @@ func TestRing_GetAllInstanceDescs(t *testing.T) {
require.EqualValues(t, []InstanceDesc{
{Addr: "127.0.0.1", Tokens: []uint32{1}, State: ACTIVE, Timestamp: now},
}, healthyInstanceDescs)
sort.Slice(unhealthyInstanceDescs, func(i, j int) bool { return unhealthyInstanceDescs[i].Addr < unhealthyInstanceDescs[j].Addr })
require.EqualValues(t, []InstanceDesc{
{Addr: "127.0.0.2", Tokens: []uint32{2}, State: LEAVING, Timestamp: now},
{Addr: "127.0.0.3", Tokens: []uint32{3}, State: ACTIVE, Timestamp: twoMinutesAgo},
Expand Down
92 changes: 47 additions & 45 deletions pkg/ruler/ruler_ring_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,8 @@ import (
func TestGetReplicationSetForListRule(t *testing.T) {
now := time.Now()

g := ring.NewRandomTokenGenerator()

tests := map[string]struct {
ringInstances map[string]ring.InstanceDesc
ringHeartbeatTimeout time.Duration
Expand All @@ -38,31 +40,31 @@ func TestGetReplicationSetForListRule(t *testing.T) {
},
"should succeed on all healthy instances and RF=1": {
ringInstances: map[string]ring.InstanceDesc{
"instance-1": {Addr: "127.0.0.1", State: ring.ACTIVE, Timestamp: now.Unix(), Tokens: ring.GenerateTokens(128, nil)},
"instance-2": {Addr: "127.0.0.2", State: ring.ACTIVE, Timestamp: now.Add(-10 * time.Second).Unix(), Tokens: ring.GenerateTokens(128, nil)},
"instance-3": {Addr: "127.0.0.3", State: ring.ACTIVE, Timestamp: now.Add(-20 * time.Second).Unix(), Tokens: ring.GenerateTokens(128, nil)},
"instance-1": {Addr: "127.0.0.1", State: ring.ACTIVE, Timestamp: now.Unix(), Tokens: g.GenerateTokens(ring.NewDesc(), "instance-1", "", 128, true)},
"instance-2": {Addr: "127.0.0.2", State: ring.ACTIVE, Timestamp: now.Add(-10 * time.Second).Unix(), Tokens: g.GenerateTokens(ring.NewDesc(), "instance-2", "", 128, true)},
"instance-3": {Addr: "127.0.0.3", State: ring.ACTIVE, Timestamp: now.Add(-20 * time.Second).Unix(), Tokens: g.GenerateTokens(ring.NewDesc(), "instance-3", "", 128, true)},
},
ringHeartbeatTimeout: time.Minute,
ringReplicationFactor: 1,
expectedSet: []string{"127.0.0.1", "127.0.0.2", "127.0.0.3"},
},
"should fail on 1 unhealthy instance and RF=1": {
ringInstances: map[string]ring.InstanceDesc{
"instance-1": {Addr: "127.0.0.1", State: ring.ACTIVE, Timestamp: now.Unix(), Tokens: ring.GenerateTokens(128, nil)},
"instance-2": {Addr: "127.0.0.2", State: ring.ACTIVE, Timestamp: now.Add(-10 * time.Second).Unix(), Tokens: ring.GenerateTokens(128, nil)},
"instance-3": {Addr: "127.0.0.3", State: ring.ACTIVE, Timestamp: now.Add(-20 * time.Second).Unix(), Tokens: ring.GenerateTokens(128, nil)},
"instance-4": {Addr: "127.0.0.4", State: ring.ACTIVE, Timestamp: now.Add(-2 * time.Minute).Unix(), Tokens: ring.GenerateTokens(128, nil)},
"instance-1": {Addr: "127.0.0.1", State: ring.ACTIVE, Timestamp: now.Unix(), Tokens: g.GenerateTokens(ring.NewDesc(), "instance-1", "", 128, true)},
"instance-2": {Addr: "127.0.0.2", State: ring.ACTIVE, Timestamp: now.Add(-10 * time.Second).Unix(), Tokens: g.GenerateTokens(ring.NewDesc(), "instance-2", "", 128, true)},
"instance-3": {Addr: "127.0.0.3", State: ring.ACTIVE, Timestamp: now.Add(-20 * time.Second).Unix(), Tokens: g.GenerateTokens(ring.NewDesc(), "instance-3", "", 128, true)},
"instance-4": {Addr: "127.0.0.4", State: ring.ACTIVE, Timestamp: now.Add(-2 * time.Minute).Unix(), Tokens: g.GenerateTokens(ring.NewDesc(), "instance-4", "", 128, true)},
},
ringHeartbeatTimeout: time.Minute,
ringReplicationFactor: 1,
expectedErr: ring.ErrTooManyUnhealthyInstances,
},
"should succeed on 1 unhealthy instances and RF=3": {
ringInstances: map[string]ring.InstanceDesc{
"instance-1": {Addr: "127.0.0.1", State: ring.ACTIVE, Timestamp: now.Unix(), Tokens: ring.GenerateTokens(128, nil)},
"instance-2": {Addr: "127.0.0.2", State: ring.ACTIVE, Timestamp: now.Add(-10 * time.Second).Unix(), Tokens: ring.GenerateTokens(128, nil)},
"instance-3": {Addr: "127.0.0.3", State: ring.ACTIVE, Timestamp: now.Add(-20 * time.Second).Unix(), Tokens: ring.GenerateTokens(128, nil)},
"instance-4": {Addr: "127.0.0.4", State: ring.ACTIVE, Timestamp: now.Add(-2 * time.Minute).Unix(), Tokens: ring.GenerateTokens(128, nil)},
"instance-1": {Addr: "127.0.0.1", State: ring.ACTIVE, Timestamp: now.Unix(), Tokens: g.GenerateTokens(ring.NewDesc(), "instance-1", "", 128, true)},
"instance-2": {Addr: "127.0.0.2", State: ring.ACTIVE, Timestamp: now.Add(-10 * time.Second).Unix(), Tokens: g.GenerateTokens(ring.NewDesc(), "instance-2", "", 128, true)},
"instance-3": {Addr: "127.0.0.3", State: ring.ACTIVE, Timestamp: now.Add(-20 * time.Second).Unix(), Tokens: g.GenerateTokens(ring.NewDesc(), "instance-3", "", 128, true)},
"instance-4": {Addr: "127.0.0.4", State: ring.ACTIVE, Timestamp: now.Add(-2 * time.Minute).Unix(), Tokens: g.GenerateTokens(ring.NewDesc(), "instance-4", "", 128, true)},
},
ringHeartbeatTimeout: time.Minute,
ringReplicationFactor: 3,
Expand All @@ -71,11 +73,11 @@ func TestGetReplicationSetForListRule(t *testing.T) {
},
"should succeed on 2 unhealthy instances and RF=3": {
ringInstances: map[string]ring.InstanceDesc{
"instance-1": {Addr: "127.0.0.1", State: ring.ACTIVE, Timestamp: now.Unix(), Tokens: ring.GenerateTokens(128, nil)},
"instance-2": {Addr: "127.0.0.2", State: ring.ACTIVE, Timestamp: now.Add(-10 * time.Second).Unix(), Tokens: ring.GenerateTokens(128, nil)},
"instance-3": {Addr: "127.0.0.3", State: ring.ACTIVE, Timestamp: now.Add(-20 * time.Second).Unix(), Tokens: ring.GenerateTokens(128, nil)},
"instance-4": {Addr: "127.0.0.4", State: ring.ACTIVE, Timestamp: now.Add(-2 * time.Minute).Unix(), Tokens: ring.GenerateTokens(128, nil)},
"instance-5": {Addr: "127.0.0.5", State: ring.ACTIVE, Timestamp: now.Add(-2 * time.Minute).Unix(), Tokens: ring.GenerateTokens(128, nil)},
"instance-1": {Addr: "127.0.0.1", State: ring.ACTIVE, Timestamp: now.Unix(), Tokens: g.GenerateTokens(ring.NewDesc(), "instance-1", "", 128, true)},
"instance-2": {Addr: "127.0.0.2", State: ring.ACTIVE, Timestamp: now.Add(-10 * time.Second).Unix(), Tokens: g.GenerateTokens(ring.NewDesc(), "instance-2", "", 128, true)},
"instance-3": {Addr: "127.0.0.3", State: ring.ACTIVE, Timestamp: now.Add(-20 * time.Second).Unix(), Tokens: g.GenerateTokens(ring.NewDesc(), "instance-3", "", 128, true)},
"instance-4": {Addr: "127.0.0.4", State: ring.ACTIVE, Timestamp: now.Add(-2 * time.Minute).Unix(), Tokens: g.GenerateTokens(ring.NewDesc(), "instance-4", "", 128, true)},
"instance-5": {Addr: "127.0.0.5", State: ring.ACTIVE, Timestamp: now.Add(-2 * time.Minute).Unix(), Tokens: g.GenerateTokens(ring.NewDesc(), "instance-5", "", 128, true)},
},
ringHeartbeatTimeout: time.Minute,
ringReplicationFactor: 3,
Expand All @@ -84,22 +86,22 @@ func TestGetReplicationSetForListRule(t *testing.T) {
},
"should fail on 3 unhealthy instances and RF=3": {
ringInstances: map[string]ring.InstanceDesc{
"instance-1": {Addr: "127.0.0.1", State: ring.ACTIVE, Timestamp: now.Unix(), Tokens: ring.GenerateTokens(128, nil)},
"instance-2": {Addr: "127.0.0.2", State: ring.ACTIVE, Timestamp: now.Add(-10 * time.Second).Unix(), Tokens: ring.GenerateTokens(128, nil)},
"instance-3": {Addr: "127.0.0.3", State: ring.ACTIVE, Timestamp: now.Add(-2 * time.Minute).Unix(), Tokens: ring.GenerateTokens(128, nil)},
"instance-4": {Addr: "127.0.0.4", State: ring.ACTIVE, Timestamp: now.Add(-2 * time.Minute).Unix(), Tokens: ring.GenerateTokens(128, nil)},
"instance-5": {Addr: "127.0.0.5", State: ring.ACTIVE, Timestamp: now.Add(-2 * time.Minute).Unix(), Tokens: ring.GenerateTokens(128, nil)},
"instance-1": {Addr: "127.0.0.1", State: ring.ACTIVE, Timestamp: now.Unix(), Tokens: g.GenerateTokens(ring.NewDesc(), "instance-1", "", 128, true)},
"instance-2": {Addr: "127.0.0.2", State: ring.ACTIVE, Timestamp: now.Add(-10 * time.Second).Unix(), Tokens: g.GenerateTokens(ring.NewDesc(), "instance-2", "", 128, true)},
"instance-3": {Addr: "127.0.0.3", State: ring.ACTIVE, Timestamp: now.Add(-2 * time.Minute).Unix(), Tokens: g.GenerateTokens(ring.NewDesc(), "instance-3", "", 128, true)},
"instance-4": {Addr: "127.0.0.4", State: ring.ACTIVE, Timestamp: now.Add(-2 * time.Minute).Unix(), Tokens: g.GenerateTokens(ring.NewDesc(), "instance-4", "", 128, true)},
"instance-5": {Addr: "127.0.0.5", State: ring.ACTIVE, Timestamp: now.Add(-2 * time.Minute).Unix(), Tokens: g.GenerateTokens(ring.NewDesc(), "instance-5", "", 128, true)},
},
ringHeartbeatTimeout: time.Minute,
ringReplicationFactor: 3,
expectedErr: ring.ErrTooManyUnhealthyInstances,
},
"should succeed on 0 unhealthy instances and RF=3 zone replication enabled": {
ringInstances: map[string]ring.InstanceDesc{
"instance-1": {Addr: "127.0.0.1", State: ring.ACTIVE, Timestamp: now.Unix(), Tokens: ring.GenerateTokens(128, nil), Zone: "z1"},
"instance-2": {Addr: "127.0.0.2", State: ring.ACTIVE, Timestamp: now.Add(-10 * time.Second).Unix(), Tokens: ring.GenerateTokens(128, nil), Zone: "z2"},
"instance-3": {Addr: "127.0.0.3", State: ring.ACTIVE, Timestamp: now.Add(-20 * time.Second).Unix(), Tokens: ring.GenerateTokens(128, nil), Zone: "z3"},
"instance-4": {Addr: "127.0.0.4", State: ring.ACTIVE, Timestamp: now.Add(-30 * time.Second).Unix(), Tokens: ring.GenerateTokens(128, nil), Zone: "z1"},
"instance-1": {Addr: "127.0.0.1", State: ring.ACTIVE, Timestamp: now.Unix(), Tokens: g.GenerateTokens(ring.NewDesc(), "instance-1", "z1", 128, true), Zone: "z1"},
"instance-2": {Addr: "127.0.0.2", State: ring.ACTIVE, Timestamp: now.Add(-10 * time.Second).Unix(), Tokens: g.GenerateTokens(ring.NewDesc(), "instance-2", "z2", 128, true), Zone: "z2"},
"instance-3": {Addr: "127.0.0.3", State: ring.ACTIVE, Timestamp: now.Add(-20 * time.Second).Unix(), Tokens: g.GenerateTokens(ring.NewDesc(), "instance-3", "z3", 128, true), Zone: "z3"},
"instance-4": {Addr: "127.0.0.4", State: ring.ACTIVE, Timestamp: now.Add(-30 * time.Second).Unix(), Tokens: g.GenerateTokens(ring.NewDesc(), "instance-4", "z1", 128, true), Zone: "z1"},
},
ringHeartbeatTimeout: time.Minute,
ringReplicationFactor: 3,
Expand All @@ -110,11 +112,11 @@ func TestGetReplicationSetForListRule(t *testing.T) {
},
"should succeed on 3 unhealthy instances in 2 zones and RF=3 zone replication enabled": {
ringInstances: map[string]ring.InstanceDesc{
"instance-1": {Addr: "127.0.0.1", State: ring.PENDING, Timestamp: now.Unix(), Tokens: ring.GenerateTokens(128, nil), Zone: "z1"},
"instance-2": {Addr: "127.0.0.2", State: ring.ACTIVE, Timestamp: now.Add(-10 * time.Second).Unix(), Tokens: ring.GenerateTokens(128, nil), Zone: "z2"},
"instance-3": {Addr: "127.0.0.3", State: ring.ACTIVE, Timestamp: now.Add(-20 * time.Second).Unix(), Tokens: ring.GenerateTokens(128, nil), Zone: "z3"},
"instance-4": {Addr: "127.0.0.4", State: ring.ACTIVE, Timestamp: now.Add(-2 * time.Minute).Unix(), Tokens: ring.GenerateTokens(128, nil), Zone: "z1"},
"instance-5": {Addr: "127.0.0.5", State: ring.PENDING, Timestamp: now.Add(-10 * time.Second).Unix(), Tokens: ring.GenerateTokens(128, nil), Zone: "z2"},
"instance-1": {Addr: "127.0.0.1", State: ring.PENDING, Timestamp: now.Unix(), Tokens: g.GenerateTokens(ring.NewDesc(), "instance-1", "z1", 128, true), Zone: "z1"},
"instance-2": {Addr: "127.0.0.2", State: ring.ACTIVE, Timestamp: now.Add(-10 * time.Second).Unix(), Tokens: g.GenerateTokens(ring.NewDesc(), "instance-2", "z2", 128, true), Zone: "z2"},
"instance-3": {Addr: "127.0.0.3", State: ring.ACTIVE, Timestamp: now.Add(-20 * time.Second).Unix(), Tokens: g.GenerateTokens(ring.NewDesc(), "instance-3", "z3", 128, true), Zone: "z3"},
"instance-4": {Addr: "127.0.0.4", State: ring.ACTIVE, Timestamp: now.Add(-2 * time.Minute).Unix(), Tokens: g.GenerateTokens(ring.NewDesc(), "instance-4", "z1", 128, true), Zone: "z1"},
"instance-5": {Addr: "127.0.0.5", State: ring.PENDING, Timestamp: now.Add(-10 * time.Second).Unix(), Tokens: g.GenerateTokens(ring.NewDesc(), "instance-5", "z2", 128, true), Zone: "z2"},
},
ringHeartbeatTimeout: time.Minute,
ringReplicationFactor: 3,
Expand All @@ -128,10 +130,10 @@ func TestGetReplicationSetForListRule(t *testing.T) {
},
"should succeed on 1 unhealthy instances in 1 zone and RF=3 zone replication enabled": {
ringInstances: map[string]ring.InstanceDesc{
"instance-1": {Addr: "127.0.0.1", State: ring.ACTIVE, Timestamp: now.Unix(), Tokens: ring.GenerateTokens(128, nil), Zone: "z1"},
"instance-2": {Addr: "127.0.0.2", State: ring.ACTIVE, Timestamp: now.Add(-10 * time.Second).Unix(), Tokens: ring.GenerateTokens(128, nil), Zone: "z2"},
"instance-3": {Addr: "127.0.0.3", State: ring.ACTIVE, Timestamp: now.Add(-20 * time.Second).Unix(), Tokens: ring.GenerateTokens(128, nil), Zone: "z3"},
"instance-4": {Addr: "127.0.0.4", State: ring.PENDING, Timestamp: now.Add(-30 * time.Second).Unix(), Tokens: ring.GenerateTokens(128, nil), Zone: "z1"},
"instance-1": {Addr: "127.0.0.1", State: ring.ACTIVE, Timestamp: now.Unix(), Tokens: g.GenerateTokens(ring.NewDesc(), "instance-1", "z1", 128, true), Zone: "z1"},
"instance-2": {Addr: "127.0.0.2", State: ring.ACTIVE, Timestamp: now.Add(-10 * time.Second).Unix(), Tokens: g.GenerateTokens(ring.NewDesc(), "instance-2", "z2", 128, true), Zone: "z2"},
"instance-3": {Addr: "127.0.0.3", State: ring.ACTIVE, Timestamp: now.Add(-20 * time.Second).Unix(), Tokens: g.GenerateTokens(ring.NewDesc(), "instance-3", "z3", 128, true), Zone: "z3"},
"instance-4": {Addr: "127.0.0.4", State: ring.PENDING, Timestamp: now.Add(-30 * time.Second).Unix(), Tokens: g.GenerateTokens(ring.NewDesc(), "instance-4", "z1", 128, true), Zone: "z1"},
},
ringHeartbeatTimeout: time.Minute,
ringReplicationFactor: 3,
Expand All @@ -144,12 +146,12 @@ func TestGetReplicationSetForListRule(t *testing.T) {
},
"should fail on 3 unhealthy instances in 3 zonez and RF=3 zone replication enabled": {
ringInstances: map[string]ring.InstanceDesc{
"instance-1": {Addr: "127.0.0.1", State: ring.ACTIVE, Timestamp: now.Unix(), Tokens: ring.GenerateTokens(128, nil), Zone: "z1"},
"instance-2": {Addr: "127.0.0.2", State: ring.ACTIVE, Timestamp: now.Add(-10 * time.Second).Unix(), Tokens: ring.GenerateTokens(128, nil), Zone: "z2"},
"instance-3": {Addr: "127.0.0.3", State: ring.ACTIVE, Timestamp: now.Add(-20 * time.Second).Unix(), Tokens: ring.GenerateTokens(128, nil), Zone: "z3"},
"instance-4": {Addr: "127.0.0.4", State: ring.PENDING, Timestamp: now.Add(-30 * time.Second).Unix(), Tokens: ring.GenerateTokens(128, nil), Zone: "z1"},
"instance-5": {Addr: "127.0.0.5", State: ring.ACTIVE, Timestamp: now.Add(-2 * time.Minute).Unix(), Tokens: ring.GenerateTokens(128, nil), Zone: "z2"},
"instance-6": {Addr: "127.0.0.6", State: ring.ACTIVE, Timestamp: now.Add(-2 * time.Minute).Unix(), Tokens: ring.GenerateTokens(128, nil), Zone: "z3"},
"instance-1": {Addr: "127.0.0.1", State: ring.ACTIVE, Timestamp: now.Unix(), Tokens: g.GenerateTokens(ring.NewDesc(), "instance-1", "z1", 128, true), Zone: "z1"},
"instance-2": {Addr: "127.0.0.2", State: ring.ACTIVE, Timestamp: now.Add(-10 * time.Second).Unix(), Tokens: g.GenerateTokens(ring.NewDesc(), "instance-2", "z2", 128, true), Zone: "z2"},
"instance-3": {Addr: "127.0.0.3", State: ring.ACTIVE, Timestamp: now.Add(-20 * time.Second).Unix(), Tokens: g.GenerateTokens(ring.NewDesc(), "instance-3", "z3", 128, true), Zone: "z3"},
"instance-4": {Addr: "127.0.0.4", State: ring.PENDING, Timestamp: now.Add(-30 * time.Second).Unix(), Tokens: g.GenerateTokens(ring.NewDesc(), "instance-4", "z1", 128, true), Zone: "z1"},
"instance-5": {Addr: "127.0.0.5", State: ring.ACTIVE, Timestamp: now.Add(-2 * time.Minute).Unix(), Tokens: g.GenerateTokens(ring.NewDesc(), "instance-5", "z2", 128, true), Zone: "z2"},
"instance-6": {Addr: "127.0.0.6", State: ring.ACTIVE, Timestamp: now.Add(-2 * time.Minute).Unix(), Tokens: g.GenerateTokens(ring.NewDesc(), "instance-6", "z3", 128, true), Zone: "z3"},
},
ringHeartbeatTimeout: time.Minute,
ringReplicationFactor: 3,
Expand All @@ -163,11 +165,11 @@ func TestGetReplicationSetForListRule(t *testing.T) {
},
"should fail on 2 unhealthy instances in 2 zones and RF=2 zone replication enabled": {
ringInstances: map[string]ring.InstanceDesc{
"instance-1": {Addr: "127.0.0.1", State: ring.ACTIVE, Timestamp: now.Unix(), Tokens: ring.GenerateTokens(128, nil), Zone: "z1"},
"instance-2": {Addr: "127.0.0.2", State: ring.ACTIVE, Timestamp: now.Add(-10 * time.Second).Unix(), Tokens: ring.GenerateTokens(128, nil), Zone: "z2"},
"instance-3": {Addr: "127.0.0.3", State: ring.ACTIVE, Timestamp: now.Add(-20 * time.Second).Unix(), Tokens: ring.GenerateTokens(128, nil), Zone: "z3"},
"instance-4": {Addr: "127.0.0.4", State: ring.PENDING, Timestamp: now.Add(-30 * time.Second).Unix(), Tokens: ring.GenerateTokens(128, nil), Zone: "z1"},
"instance-5": {Addr: "127.0.0.5", State: ring.ACTIVE, Timestamp: now.Add(-2 * time.Minute).Unix(), Tokens: ring.GenerateTokens(128, nil), Zone: "z2"},
"instance-1": {Addr: "127.0.0.1", State: ring.ACTIVE, Timestamp: now.Unix(), Tokens: g.GenerateTokens(ring.NewDesc(), "instance-1", "z1", 128, true), Zone: "z1"},
"instance-2": {Addr: "127.0.0.2", State: ring.ACTIVE, Timestamp: now.Add(-10 * time.Second).Unix(), Tokens: g.GenerateTokens(ring.NewDesc(), "instance-2", "z2", 128, true), Zone: "z2"},
"instance-3": {Addr: "127.0.0.3", State: ring.ACTIVE, Timestamp: now.Add(-20 * time.Second).Unix(), Tokens: g.GenerateTokens(ring.NewDesc(), "instance-3", "z3", 128, true), Zone: "z3"},
"instance-4": {Addr: "127.0.0.4", State: ring.PENDING, Timestamp: now.Add(-30 * time.Second).Unix(), Tokens: g.GenerateTokens(ring.NewDesc(), "instance-4", "z1", 128, true), Zone: "z1"},
"instance-5": {Addr: "127.0.0.5", State: ring.ACTIVE, Timestamp: now.Add(-2 * time.Minute).Unix(), Tokens: g.GenerateTokens(ring.NewDesc(), "instance-5", "z2", 128, true), Zone: "z2"},
},
ringHeartbeatTimeout: time.Minute,
ringReplicationFactor: 2,
Expand Down

0 comments on commit d8955ab

Please sign in to comment.