From 02864726260798c31f8db307c5e65687c2bb1c54 Mon Sep 17 00:00:00 2001 From: Arul Ajmani Date: Tue, 8 Oct 2024 11:56:19 -0400 Subject: [PATCH] raft: move StateType from raft into raftpb We'll need this to avoid a circular dependency when we use StateType in the FortificationTracker. Epic: none Release note: None --- pkg/kv/kvclient/kvcoord/BUILD.bazel | 2 +- .../kvcoord/dist_sender_server_test.go | 4 +- .../allocator/allocatorimpl/allocator_test.go | 8 +- pkg/kv/kvserver/allocator/plan/replicate.go | 3 +- pkg/kv/kvserver/allocator_impl_test.go | 2 +- pkg/kv/kvserver/asim/state/impl.go | 2 +- pkg/kv/kvserver/client_lease_test.go | 3 +- pkg/kv/kvserver/client_raft_test.go | 66 +-- pkg/kv/kvserver/leases/BUILD.bazel | 1 + pkg/kv/kvserver/leases/build.go | 5 +- pkg/kv/kvserver/leases/build_test.go | 4 +- pkg/kv/kvserver/leases/status_test.go | 18 +- pkg/kv/kvserver/leases/verify.go | 5 +- pkg/kv/kvserver/leases/verify_test.go | 16 +- pkg/kv/kvserver/raft_log_queue.go | 2 +- pkg/kv/kvserver/raft_log_queue_test.go | 2 +- pkg/kv/kvserver/raftutil/util.go | 4 +- pkg/kv/kvserver/raftutil/util_test.go | 34 +- pkg/kv/kvserver/replica_application_result.go | 4 +- pkg/kv/kvserver/replica_command.go | 2 +- pkg/kv/kvserver/replica_gc_queue.go | 8 +- pkg/kv/kvserver/replica_metrics.go | 2 +- pkg/kv/kvserver/replica_proposal_buf.go | 6 +- pkg/kv/kvserver/replica_proposal_buf_test.go | 44 +- pkg/kv/kvserver/replica_raft.go | 14 +- pkg/kv/kvserver/replica_raft_quiesce.go | 6 +- pkg/kv/kvserver/replica_rangefeed_test.go | 3 +- pkg/kv/kvserver/replica_test.go | 46 +- pkg/kv/kvserver/split_delay_helper.go | 5 +- pkg/kv/kvserver/split_delay_helper_test.go | 14 +- pkg/kv/kvserver/store_rebalancer_test.go | 4 +- pkg/raft/node.go | 2 +- pkg/raft/node_test.go | 2 +- pkg/raft/raft.go | 73 +-- pkg/raft/raft_paper_test.go | 84 ++-- pkg/raft/raft_test.go | 464 +++++++++--------- pkg/raft/raftpb/raft.go | 33 +- pkg/raft/rawnode.go | 2 +- pkg/raft/rawnode_test.go | 8 +- pkg/raft/status.go | 6 +- pkg/raft/util.go | 4 - pkg/testutils/testcluster/BUILD.bazel | 2 +- pkg/testutils/testcluster/testcluster.go | 4 +- 43 files changed, 515 insertions(+), 508 deletions(-) diff --git a/pkg/kv/kvclient/kvcoord/BUILD.bazel b/pkg/kv/kvclient/kvcoord/BUILD.bazel index aea076f63278..54ec02522033 100644 --- a/pkg/kv/kvclient/kvcoord/BUILD.bazel +++ b/pkg/kv/kvclient/kvcoord/BUILD.bazel @@ -186,7 +186,7 @@ go_test( "//pkg/kv/kvserver/tscache", "//pkg/kv/kvserver/txnwait", "//pkg/multitenant/tenantcapabilities/tenantcapabilitiesauthorizer", - "//pkg/raft", + "//pkg/raft/raftpb", "//pkg/roachpb", "//pkg/rpc", "//pkg/rpc/nodedialer", diff --git a/pkg/kv/kvclient/kvcoord/dist_sender_server_test.go b/pkg/kv/kvclient/kvcoord/dist_sender_server_test.go index bb3ae5b3480c..8a0d38151122 100644 --- a/pkg/kv/kvclient/kvcoord/dist_sender_server_test.go +++ b/pkg/kv/kvclient/kvcoord/dist_sender_server_test.go @@ -27,7 +27,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/isolation" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/kvserverbase" - "github.com/cockroachdb/cockroach/pkg/raft" + "github.com/cockroachdb/cockroach/pkg/raft/raftpb" "github.com/cockroachdb/cockroach/pkg/roachpb" "github.com/cockroachdb/cockroach/pkg/rpc" "github.com/cockroachdb/cockroach/pkg/rpc/nodedialer" @@ -4771,7 +4771,7 @@ func TestPartialPartition(t *testing.T) { store, err := sl.GetStores().(*kvserver.Stores).GetStore(sl.GetFirstStoreID()) require.NoError(t, err) status := store.LookupReplica(roachpb.RKey(scratchKey)).RaftStatus() - if status == nil || status.RaftState != raft.StateLeader { + if status == nil || status.RaftState != raftpb.StateLeader { return errors.Newf("Leader leaseholder split %v", status) } return nil diff --git a/pkg/kv/kvserver/allocator/allocatorimpl/allocator_test.go b/pkg/kv/kvserver/allocator/allocatorimpl/allocator_test.go index 86d4440bb913..a7491a59c836 100644 --- a/pkg/kv/kvserver/allocator/allocatorimpl/allocator_test.go +++ b/pkg/kv/kvserver/allocator/allocatorimpl/allocator_test.go @@ -1917,7 +1917,7 @@ func (r *mockRepl) RaftStatus() *raft.Status { raftStatus := &raft.Status{ Progress: make(map[raftpb.PeerID]tracker.Progress), } - raftStatus.RaftState = raft.StateLeader + raftStatus.RaftState = raftpb.StateLeader for i := int32(1); i <= r.replicationFactor; i++ { state := tracker.StateReplicate if _, ok := r.replsInNeedOfSnapshot[roachpb.ReplicaID(i)]; ok { @@ -8290,7 +8290,7 @@ func TestFilterBehindReplicas(t *testing.T) { Progress: make(map[raftpb.PeerID]tracker.Progress), } status.Lead = c.leader - status.RaftState = raft.StateLeader + status.RaftState = raftpb.StateLeader status.Commit = c.commit var replicas []roachpb.ReplicaDescriptor for j, v := range c.progress { @@ -8363,7 +8363,7 @@ func TestFilterUnremovableReplicas(t *testing.T) { // Use an invalid replica ID for the leader. TestFilterBehindReplicas covers // valid replica IDs. status.Lead = 99 - status.RaftState = raft.StateLeader + status.RaftState = raftpb.StateLeader status.Commit = c.commit var replicas []roachpb.ReplicaDescriptor for j, v := range c.progress { @@ -8421,7 +8421,7 @@ func TestSimulateFilterUnremovableReplicas(t *testing.T) { // Use an invalid replica ID for the leader. TestFilterBehindReplicas covers // valid replica IDs. status.Lead = 99 - status.RaftState = raft.StateLeader + status.RaftState = raftpb.StateLeader status.Commit = c.commit var replicas []roachpb.ReplicaDescriptor for j, v := range c.progress { diff --git a/pkg/kv/kvserver/allocator/plan/replicate.go b/pkg/kv/kvserver/allocator/plan/replicate.go index e6044af355bd..26fc66b12e63 100644 --- a/pkg/kv/kvserver/allocator/plan/replicate.go +++ b/pkg/kv/kvserver/allocator/plan/replicate.go @@ -18,6 +18,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/kv/kvserver/kvflowcontrol/rac2" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/kvserverpb" "github.com/cockroachdb/cockroach/pkg/raft" + "github.com/cockroachdb/cockroach/pkg/raft/raftpb" "github.com/cockroachdb/cockroach/pkg/roachpb" "github.com/cockroachdb/cockroach/pkg/util/hlc" "github.com/cockroachdb/cockroach/pkg/util/log" @@ -534,7 +535,7 @@ func (rp ReplicaPlanner) findRemoveVoter( lastReplAdded = 0 } raftStatus := repl.RaftStatus() - if raftStatus == nil || raftStatus.RaftState != raft.StateLeader { + if raftStatus == nil || raftStatus.RaftState != raftpb.StateLeader { // If requested, assume all replicas are up-to-date. if rp.knobs.AllowVoterRemovalWhenNotLeader { candidates = allocatorimpl.FilterUnremovableReplicasWithoutRaftStatus( diff --git a/pkg/kv/kvserver/allocator_impl_test.go b/pkg/kv/kvserver/allocator_impl_test.go index 40d5ded9c848..a14772114525 100644 --- a/pkg/kv/kvserver/allocator_impl_test.go +++ b/pkg/kv/kvserver/allocator_impl_test.go @@ -303,7 +303,7 @@ func TestAllocatorRebalanceTarget(t *testing.T) { Progress: make(map[raftpb.PeerID]tracker.Progress), } status.Lead = 1 - status.RaftState = raft.StateLeader + status.RaftState = raftpb.StateLeader status.Commit = 10 for _, replica := range replicas { status.Progress[raftpb.PeerID(replica.ReplicaID)] = tracker.Progress{ diff --git a/pkg/kv/kvserver/asim/state/impl.go b/pkg/kv/kvserver/asim/state/impl.go index 743ae91b2dd9..afc19a16f999 100644 --- a/pkg/kv/kvserver/asim/state/impl.go +++ b/pkg/kv/kvserver/asim/state/impl.go @@ -1203,7 +1203,7 @@ func (s *state) RaftStatus(rangeID RangeID, storeID StoreID) *raft.Status { // TODO(kvoli): The raft leader will always be the current leaseholder // here. This should change to enable testing this scenario. status.Lead = raftpb.PeerID(leader.ReplicaID()) - status.RaftState = raft.StateLeader + status.RaftState = raftpb.StateLeader status.Commit = 2 // TODO(kvoli): A replica is never behind on their raft log, this should // change to enable testing this scenario where replicas fall behind. e.g. diff --git a/pkg/kv/kvserver/client_lease_test.go b/pkg/kv/kvserver/client_lease_test.go index affb53eed7d8..e660666ee3c7 100644 --- a/pkg/kv/kvserver/client_lease_test.go +++ b/pkg/kv/kvserver/client_lease_test.go @@ -27,7 +27,6 @@ import ( "github.com/cockroachdb/cockroach/pkg/kv/kvserver/kvserverpb" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/liveness" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/liveness/livenesspb" - "github.com/cockroachdb/cockroach/pkg/raft" "github.com/cockroachdb/cockroach/pkg/raft/raftpb" "github.com/cockroachdb/cockroach/pkg/raft/tracker" "github.com/cockroachdb/cockroach/pkg/roachpb" @@ -431,7 +430,7 @@ func TestTransferLeaseDuringJointConfigWithDeadIncomingVoter(t *testing.T) { require.NoError(t, repl0.RaftReportUnreachable(4)) // Check the Raft progress. s := repl0.RaftStatus() - require.Equal(t, raft.StateLeader, s.RaftState) + require.Equal(t, raftpb.StateLeader, s.RaftState) p := s.Progress require.Len(t, p, 4) require.Contains(t, p, raftpb.PeerID(4)) diff --git a/pkg/kv/kvserver/client_raft_test.go b/pkg/kv/kvserver/client_raft_test.go index 23541b162788..cea2ed35277a 100644 --- a/pkg/kv/kvserver/client_raft_test.go +++ b/pkg/kv/kvserver/client_raft_test.go @@ -780,7 +780,7 @@ func TestSnapshotAfterTruncation(t *testing.T) { if status == nil { return errors.New("raft status not initialized") } - if status.RaftState == raft.StateLeader { + if status.RaftState == raftpb.StateLeader { hasLeader = true } if term == 0 { @@ -2010,7 +2010,7 @@ func TestLogGrowthWhenRefreshingPendingCommands(t *testing.T) { case <-ticker: // Verify that the leader is node 0. status := leaderRepl.RaftStatus() - if status == nil || status.RaftState != raft.StateLeader { + if status == nil || status.RaftState != raftpb.StateLeader { t.Fatalf("raft leader should be node 0, but got status %+v", status) } @@ -2822,7 +2822,7 @@ func TestRaftHeartbeats(t *testing.T) { } status := leaderRepl.RaftStatus() - if status.SoftState.RaftState != raft.StateLeader { + if status.SoftState.RaftState != raftpb.StateLeader { t.Errorf("expected node %d to be leader after sleeping but was %s", leaderRepl.NodeID(), status.SoftState.RaftState) } if status.Term != initialTerm { @@ -2879,7 +2879,7 @@ func TestReportUnreachableHeartbeats(t *testing.T) { // Ensure that the leadership has not changed, to confirm that heartbeats // are sent to the store with a functioning transport. status := leaderRepl.RaftStatus() - if status.SoftState.RaftState != raft.StateLeader { + if status.SoftState.RaftState != raftpb.StateLeader { t.Errorf("expected node %d to be leader after sleeping but was %s", leaderRepl.NodeID(), status.SoftState.RaftState) } if status.Term != initialTerm { @@ -2915,7 +2915,7 @@ func TestReportUnreachableRemoveRace(t *testing.T) { for idx := range tc.Servers { repl := tc.GetFirstStoreFromServer(t, idx).LookupReplica(roachpb.RKey(key)) require.NotNil(t, repl) - if repl.RaftStatus().SoftState.RaftState == raft.StateLeader { + if repl.RaftStatus().SoftState.RaftState == raftpb.StateLeader { leaderIdx = idx leaderRepl = repl return nil @@ -4335,7 +4335,7 @@ func TestRangeQuiescence(t *testing.T) { waitForQuiescence(roachpb.RKey(key)) // The leadership should not have changed. - if state := leader.RaftStatus().SoftState.RaftState; state != raft.StateLeader { + if state := leader.RaftStatus().SoftState.RaftState; state != raftpb.StateLeader { t.Fatalf("%s should be the leader: %s", leader, state) } } @@ -4472,7 +4472,7 @@ func TestFailedConfChange(t *testing.T) { i, l, repl.Desc()) } status := repl.RaftStatus() - if status.RaftState != raft.StateLeader { + if status.RaftState != raftpb.StateLeader { return errors.Errorf("store %d: expected StateLeader, was %s", i, status.RaftState) } // In issue #13506, the Progress map would be updated as if the @@ -4496,7 +4496,7 @@ func TestFailedConfChange(t *testing.T) { return err } status := repl.RaftStatus() - if status.RaftState != raft.StateLeader { + if status.RaftState != raftpb.StateLeader { return errors.Errorf("store %d: expected StateLeader, was %s", 1, status.RaftState) } return nil @@ -6038,7 +6038,7 @@ func TestRaftCampaignPreVoteCheckQuorum(t *testing.T) { // Make sure n1 is leader. initialStatus := repl1.RaftStatus() - require.Equal(t, raft.StateLeader, initialStatus.RaftState) + require.Equal(t, raftpb.StateLeader, initialStatus.RaftState) logStatus(initialStatus) t.Logf("n1 is leader") @@ -6050,7 +6050,7 @@ func TestRaftCampaignPreVoteCheckQuorum(t *testing.T) { require.Eventually(t, func() bool { status := repl3.RaftStatus() logStatus(status) - return status.RaftState == raft.StateFollower + return status.RaftState == raftpb.StateFollower }, 10*time.Second, 500*time.Millisecond) t.Logf("n3 reverted to follower") @@ -6059,9 +6059,9 @@ func TestRaftCampaignPreVoteCheckQuorum(t *testing.T) { st := repl.RaftStatus() logStatus(st) if st.ID == 1 { - require.Equal(t, raft.StateLeader, st.RaftState) + require.Equal(t, raftpb.StateLeader, st.RaftState) } else { - require.Equal(t, raft.StateFollower, st.RaftState) + require.Equal(t, raftpb.StateFollower, st.RaftState) } require.Equal(t, initialStatus.Term, st.Term) } @@ -6119,7 +6119,7 @@ func TestRaftForceCampaignPreVoteCheckQuorum(t *testing.T) { // Make sure n1 is leader. initialStatus := repl1.RaftStatus() - require.Equal(t, raft.StateLeader, initialStatus.RaftState) + require.Equal(t, raftpb.StateLeader, initialStatus.RaftState) logStatus(initialStatus) t.Logf("n1 is leader in term %d", initialStatus.Term) @@ -6136,7 +6136,7 @@ func TestRaftForceCampaignPreVoteCheckQuorum(t *testing.T) { if st.Term <= initialStatus.Term { return false } - if st.RaftState == raft.StateLeader { + if st.RaftState == raftpb.StateLeader { leaderStatus = st } } @@ -6284,7 +6284,7 @@ func TestRaftPreVote(t *testing.T) { var lastChanged time.Time require.Eventually(t, func() bool { status := repl1.RaftStatus() - require.Equal(t, raft.StateLeader, status.RaftState) + require.Equal(t, raftpb.StateLeader, status.RaftState) if i := status.Progress[1].Match; i > lastIndex { t.Logf("n1 last index changed: %d -> %d", lastIndex, i) lastIndex, lastChanged = i, time.Now() @@ -6313,7 +6313,7 @@ func TestRaftPreVote(t *testing.T) { // Fetch the leader's initial status. initialStatus := repl1.RaftStatus() - require.Equal(t, raft.StateLeader, initialStatus.RaftState) + require.Equal(t, raftpb.StateLeader, initialStatus.RaftState) logStatus(initialStatus) // Unquiesce n3 if necessary. @@ -6329,7 +6329,7 @@ func TestRaftPreVote(t *testing.T) { require.Eventually(t, func() bool { status := repl3.RaftStatus() logStatus(status) - return status.RaftState == raft.StatePreCandidate + return status.RaftState == raftpb.StatePreCandidate }, 10*time.Second, 500*time.Millisecond) t.Logf("n3 became pre-candidate") @@ -6339,7 +6339,7 @@ func TestRaftPreVote(t *testing.T) { require.Never(t, func() bool { status := repl3.RaftStatus() logStatus(status) - return status.RaftState != raft.StatePreCandidate + return status.RaftState != raftpb.StatePreCandidate }, 3*time.Second, time.Second) t.Logf("n3 is still pre-candidate") @@ -6349,7 +6349,7 @@ func TestRaftPreVote(t *testing.T) { // due to the prevote received by n2 which will wake the leader. leaderStatus := repl1.RaftStatus() logStatus(leaderStatus) - require.Equal(t, raft.StateLeader, leaderStatus.RaftState) + require.Equal(t, raftpb.StateLeader, leaderStatus.RaftState) require.Equal(t, initialStatus.Term, leaderStatus.Term) if !quiesce { require.Equal(t, initialStatus.Commit, leaderStatus.Commit) @@ -6367,14 +6367,14 @@ func TestRaftPreVote(t *testing.T) { require.Eventually(t, func() bool { status := repl3.RaftStatus() logStatus(status) - return status.RaftState == raft.StateFollower + return status.RaftState == raftpb.StateFollower }, 10*time.Second, 500*time.Millisecond) t.Logf("n3 became follower") // Make sure the leader and term are still the same. leaderStatus = repl1.RaftStatus() logStatus(leaderStatus) - require.Equal(t, raft.StateLeader, leaderStatus.RaftState) + require.Equal(t, raftpb.StateLeader, leaderStatus.RaftState) require.Equal(t, initialStatus.Term, leaderStatus.Term) t.Logf("n1 is still leader") @@ -6387,7 +6387,7 @@ func TestRaftPreVote(t *testing.T) { // Make sure the leader and term are still the same. leaderStatus = repl1.RaftStatus() logStatus(leaderStatus) - require.Equal(t, raft.StateLeader, leaderStatus.RaftState) + require.Equal(t, raftpb.StateLeader, leaderStatus.RaftState) require.Equal(t, initialStatus.Term, leaderStatus.Term) t.Logf("n1 is still leader") }) @@ -6501,7 +6501,7 @@ func TestRaftCheckQuorum(t *testing.T) { // Fetch the leader's initial status. initialStatus := repl1.RaftStatus() - require.Equal(t, raft.StateLeader, initialStatus.RaftState) + require.Equal(t, raftpb.StateLeader, initialStatus.RaftState) logStatus(initialStatus) // Unquiesce the leader if necessary. We have to do so by submitting an @@ -6520,7 +6520,7 @@ func TestRaftCheckQuorum(t *testing.T) { require.Eventually(t, func() bool { status := repl1.RaftStatus() logStatus(status) - return status.RaftState == raft.StatePreCandidate + return status.RaftState == raftpb.StatePreCandidate }, 10*time.Second, 500*time.Millisecond) t.Logf("n1 became pre-candidate") @@ -6536,7 +6536,7 @@ func TestRaftCheckQuorum(t *testing.T) { require.Eventually(t, func() bool { for _, status := range []*raft.Status{repl2.RaftStatus(), repl3.RaftStatus()} { logStatus(status) - if status.RaftState == raft.StateLeader { + if status.RaftState == raftpb.StateLeader { leaderStatus = status return true } @@ -6550,7 +6550,7 @@ func TestRaftCheckQuorum(t *testing.T) { require.Never(t, func() bool { status := repl1.RaftStatus() logStatus(status) - return status.RaftState != raft.StatePreCandidate + return status.RaftState != raftpb.StatePreCandidate }, 3*time.Second, 500*time.Millisecond) t.Logf("n1 remains pre-candidate") @@ -6558,7 +6558,7 @@ func TestRaftCheckQuorum(t *testing.T) { var finalStatus *raft.Status for _, status := range []*raft.Status{repl2.RaftStatus(), repl3.RaftStatus()} { logStatus(status) - if status.RaftState == raft.StateLeader { + if status.RaftState == raftpb.StateLeader { finalStatus = status break } @@ -6655,7 +6655,7 @@ func TestRaftLeaderRemovesItself(t *testing.T) { // Make sure n1 is still leader. st := repl1.RaftStatus() - require.Equal(t, raft.StateLeader, st.RaftState) + require.Equal(t, raftpb.StateLeader, st.RaftState) logStatus(st) // Remove n1 and wait for n3 to become leader. @@ -6668,7 +6668,7 @@ func TestRaftLeaderRemovesItself(t *testing.T) { require.Eventually(t, func() bool { logStatus(repl2.RaftStatus()) logStatus(repl3.RaftStatus()) - if repl3.RaftStatus().RaftState == raft.StateLeader { + if repl3.RaftStatus().RaftState == raftpb.StateLeader { t.Logf("n3 is leader") return true } @@ -6764,7 +6764,7 @@ func TestRaftUnquiesceLeaderNoProposal(t *testing.T) { // Make sure n1 is still leader. initialStatus := repl1.RaftStatus() - require.Equal(t, raft.StateLeader, initialStatus.RaftState) + require.Equal(t, raftpb.StateLeader, initialStatus.RaftState) logStatus(initialStatus) t.Logf("n1 leader") @@ -6778,7 +6778,7 @@ func TestRaftUnquiesceLeaderNoProposal(t *testing.T) { status := repl1.RaftStatus() logStatus(status) - require.Equal(t, raft.StateLeader, status.RaftState) + require.Equal(t, raftpb.StateLeader, status.RaftState) require.Equal(t, initialStatus.Term, status.Term) require.Equal(t, initialStatus.Progress[1].Match, status.Progress[1].Match) t.Logf("n1 still leader with no new proposals at log index %d", status.Progress[1].Match) @@ -6900,7 +6900,7 @@ func TestRaftPreVoteUnquiesceDeadLeader(t *testing.T) { // Fetch the leader's initial status. initialStatus := repl1.RaftStatus() - require.Equal(t, raft.StateLeader, initialStatus.RaftState) + require.Equal(t, raftpb.StateLeader, initialStatus.RaftState) logStatus(initialStatus) // Unquiesce n2. This should cause it to see n1 as dead and immediately @@ -6917,7 +6917,7 @@ func TestRaftPreVoteUnquiesceDeadLeader(t *testing.T) { require.Eventually(t, func() bool { status := repl2.RaftStatus() logStatus(status) - return status.RaftState == raft.StateLeader + return status.RaftState == raftpb.StateLeader }, 5*time.Second, 500*time.Millisecond) t.Logf("n2 is leader") } diff --git a/pkg/kv/kvserver/leases/BUILD.bazel b/pkg/kv/kvserver/leases/BUILD.bazel index 7c3e153d92d5..3854c9085bef 100644 --- a/pkg/kv/kvserver/leases/BUILD.bazel +++ b/pkg/kv/kvserver/leases/BUILD.bazel @@ -17,6 +17,7 @@ go_library( "//pkg/kv/kvserver/liveness/livenesspb", "//pkg/kv/kvserver/raftutil", "//pkg/raft", + "//pkg/raft/raftpb", "//pkg/roachpb", "//pkg/util/hlc", "//pkg/util/log", diff --git a/pkg/kv/kvserver/leases/build.go b/pkg/kv/kvserver/leases/build.go index 5ac454f9ca28..5cf1511473ae 100644 --- a/pkg/kv/kvserver/leases/build.go +++ b/pkg/kv/kvserver/leases/build.go @@ -14,6 +14,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/kv/kvserver/liveness" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/liveness/livenesspb" "github.com/cockroachdb/cockroach/pkg/raft" + "github.com/cockroachdb/cockroach/pkg/raft/raftpb" "github.com/cockroachdb/cockroach/pkg/roachpb" "github.com/cockroachdb/cockroach/pkg/util/hlc" "github.com/cockroachdb/errors" @@ -383,7 +384,7 @@ func leaseType(st Settings, i BuildInput) roachpb.LeaseType { // construct an epoch-based lease. return roachpb.LeaseEpoch } - if i.RaftStatus.RaftState != raft.StateLeader || i.RaftStatus.LeadTransferee != raft.None { + if i.RaftStatus.RaftState != raftpb.StateLeader || i.RaftStatus.LeadTransferee != raft.None { // If this range wants to use a leader lease, but the local replica is not // currently the raft leader, we construct an expiration-based lease. It is // highly likely that the lease acquisition will be rejected before being @@ -541,7 +542,7 @@ func leaseTerm(i BuildInput, nextType roachpb.LeaseType) uint64 { if nextType != roachpb.LeaseLeader { panic("leaseTerm called for non-leader lease") } - if i.RaftStatus.RaftState != raft.StateLeader { + if i.RaftStatus.RaftState != raftpb.StateLeader { panic("leaseTerm called when not leader") } if i.RaftStatus.Term == 0 { diff --git a/pkg/kv/kvserver/leases/build_test.go b/pkg/kv/kvserver/leases/build_test.go index d1425dc35958..67e0a8476fe7 100644 --- a/pkg/kv/kvserver/leases/build_test.go +++ b/pkg/kv/kvserver/leases/build_test.go @@ -206,13 +206,13 @@ func raftStatusFollower(replicaID roachpb.ReplicaID) *raft.Status { s := &raft.Status{} s.ID = raftpb.PeerID(replicaID) s.Term = 5 - s.RaftState = raft.StateFollower + s.RaftState = raftpb.StateFollower return s } func raftStatusLeader(replicaID roachpb.ReplicaID) *raft.Status { s := raftStatusFollower(replicaID) - s.RaftState = raft.StateLeader + s.RaftState = raftpb.StateLeader s.LeadSupportUntil = ts30 return s } diff --git a/pkg/kv/kvserver/leases/status_test.go b/pkg/kv/kvserver/leases/status_test.go index 1275a81ea4a4..41f2be2532d9 100644 --- a/pkg/kv/kvserver/leases/status_test.go +++ b/pkg/kv/kvserver/leases/status_test.go @@ -69,12 +69,12 @@ func TestStatus(t *testing.T) { leaderLeaseRemote := leaderLease leaderLeaseRemote.Replica = repl2 - raftStatus := func(state raft.StateType, term uint64, leadSupport hlc.Timestamp) raft.LeadSupportStatus { + raftStatus := func(state raftpb.StateType, term uint64, leadSupport hlc.Timestamp) raft.LeadSupportStatus { var s raft.LeadSupportStatus s.ID = raftpb.PeerID(repl1.ReplicaID) s.RaftState = state s.Term = term - if state == raft.StateLeader { + if state == raftpb.StateLeader { s.Lead = raftpb.PeerID(repl1.ReplicaID) } else { s.Lead = raftpb.PeerID(repl2.ReplicaID) @@ -83,15 +83,15 @@ func TestStatus(t *testing.T) { return s } // Basic case. - leaderStatus := raftStatus(raft.StateLeader, 5, ts[4].ToTimestamp()) + leaderStatus := raftStatus(raftpb.StateLeader, 5, ts[4].ToTimestamp()) // Advanced cases. - followerStatus := raftStatus(raft.StateFollower, 5, hlc.Timestamp{}) - unfortifiedLeaderStatus := raftStatus(raft.StateLeader, 5, hlc.Timestamp{}) - steppingDownFollowerStatus := raftStatus(raft.StateFollower, 5, ts[4].ToTimestamp()) - followerNewTermUnknownLeadStatus := raftStatus(raft.StateFollower, 6, hlc.Timestamp{}) + followerStatus := raftStatus(raftpb.StateFollower, 5, hlc.Timestamp{}) + unfortifiedLeaderStatus := raftStatus(raftpb.StateLeader, 5, hlc.Timestamp{}) + steppingDownFollowerStatus := raftStatus(raftpb.StateFollower, 5, ts[4].ToTimestamp()) + followerNewTermUnknownLeadStatus := raftStatus(raftpb.StateFollower, 6, hlc.Timestamp{}) followerNewTermUnknownLeadStatus.Lead = raft.None - followerNewTermKnownLeadStatus := raftStatus(raft.StateFollower, 6, hlc.Timestamp{}) - leaderNewTermKnownLeadStatus := raftStatus(raft.StateLeader, 6, ts[4].ToTimestamp()) + followerNewTermKnownLeadStatus := raftStatus(raftpb.StateFollower, 6, hlc.Timestamp{}) + leaderNewTermKnownLeadStatus := raftStatus(raftpb.StateLeader, 6, ts[4].ToTimestamp()) for _, tc := range []struct { lease roachpb.Lease diff --git a/pkg/kv/kvserver/leases/verify.go b/pkg/kv/kvserver/leases/verify.go index ce9c0143a13d..79bf25fb613e 100644 --- a/pkg/kv/kvserver/leases/verify.go +++ b/pkg/kv/kvserver/leases/verify.go @@ -11,6 +11,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/kv/kvpb" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/raftutil" "github.com/cockroachdb/cockroach/pkg/raft" + "github.com/cockroachdb/cockroach/pkg/raft/raftpb" "github.com/cockroachdb/cockroach/pkg/roachpb" "github.com/cockroachdb/cockroach/pkg/util/log" "github.com/cockroachdb/errors" @@ -117,7 +118,7 @@ func verifyAcquisition(ctx context.Context, st Settings, i VerifyInput) error { // the leader. leader := roachpb.ReplicaID(i.RaftStatus.Lead) leaderKnown := leader != roachpb.ReplicaID(raft.None) - iAmTheLeader := i.RaftStatus.RaftState == raft.StateLeader + iAmTheLeader := i.RaftStatus.RaftState == raftpb.StateLeader if (leader == i.LocalReplicaID) != iAmTheLeader { log.Fatalf(ctx, "inconsistent Raft state: %s", i.RaftStatus) } @@ -213,7 +214,7 @@ func verifyExtension(ctx context.Context, st Settings, i VerifyInput) error { if i.PrevLeaseExpired { return verifyAcquisition(ctx, st, i) } - iAmTheLeader := i.RaftStatus.RaftState == raft.StateLeader + iAmTheLeader := i.RaftStatus.RaftState == raftpb.StateLeader if !iAmTheLeader { log.VEventf(ctx, 2, "proposing lease extension even though we're not the leader; we hold the current lease") } diff --git a/pkg/kv/kvserver/leases/verify_test.go b/pkg/kv/kvserver/leases/verify_test.go index 7cee5175ec3e..5dfea2b684ac 100644 --- a/pkg/kv/kvserver/leases/verify_test.go +++ b/pkg/kv/kvserver/leases/verify_test.go @@ -52,7 +52,7 @@ func TestVerify(t *testing.T) { }) } } - makeRaftStatus := func(lead roachpb.ReplicaID, state raft.StateType) *raft.Status { + makeRaftStatus := func(lead roachpb.ReplicaID, state raftpb.StateType) *raft.Status { var status raft.Status status.ID = raftpb.PeerID(repl1.ReplicaID) status.Lead = raftpb.PeerID(lead) @@ -68,7 +68,7 @@ func TestVerify(t *testing.T) { LocalStoreID: repl1.StoreID, LocalReplicaID: repl1.ReplicaID, Desc: &descCpy, - RaftStatus: makeRaftStatus(repl1.ReplicaID, raft.StateLeader), + RaftStatus: makeRaftStatus(repl1.ReplicaID, raftpb.StateLeader), RaftFirstIndex: 5, PrevLease: roachpb.Lease{ Replica: repl2, @@ -82,7 +82,7 @@ func TestVerify(t *testing.T) { } defaultFollowerInput := func() VerifyInput { in := defaultLeaderInput() - in.RaftStatus = makeRaftStatus(repl2.ReplicaID, raft.StateFollower) + in.RaftStatus = makeRaftStatus(repl2.ReplicaID, raftpb.StateFollower) return in } @@ -220,7 +220,7 @@ func TestVerify(t *testing.T) { BypassSafetyChecks: false, } } - defaultNonLeaderInput := func(state raft.StateType) VerifyInput { + defaultNonLeaderInput := func(state raftpb.StateType) VerifyInput { in := defaultInput() in.RaftStatus = makeRaftStatus(repl2.ReplicaID, state) return in @@ -228,7 +228,7 @@ func TestVerify(t *testing.T) { const noTargetState rafttracker.StateType = math.MaxUint64 defaultLeaderInput := func(targetState rafttracker.StateType, targetMatch kvpb.RaftIndex) VerifyInput { in := defaultInput() - in.RaftStatus = makeRaftStatus(repl1.ReplicaID, raft.StateLeader) + in.RaftStatus = makeRaftStatus(repl1.ReplicaID, raftpb.StateLeader) in.RaftStatus.Progress = map[raftpb.PeerID]rafttracker.Progress{ raftpb.PeerID(repl1.ReplicaID): {State: rafttracker.StateReplicate, Match: uint64(in.RaftFirstIndex)}, } @@ -244,17 +244,17 @@ func TestVerify(t *testing.T) { runTests(t, []testCase{ { name: "follower", - input: defaultNonLeaderInput(raft.StateFollower), + input: defaultNonLeaderInput(raftpb.StateFollower), expErr: leaseTransferErrPrefix + raftutil.LocalReplicaNotLeader.String(), }, { name: "candidate", - input: defaultNonLeaderInput(raft.StateCandidate), + input: defaultNonLeaderInput(raftpb.StateCandidate), expErr: leaseTransferErrPrefix + raftutil.LocalReplicaNotLeader.String(), }, { name: "pre-candidate", - input: defaultNonLeaderInput(raft.StatePreCandidate), + input: defaultNonLeaderInput(raftpb.StatePreCandidate), expErr: leaseTransferErrPrefix + raftutil.LocalReplicaNotLeader.String(), }, { diff --git a/pkg/kv/kvserver/raft_log_queue.go b/pkg/kv/kvserver/raft_log_queue.go index e76e44cbe677..819e881297a3 100644 --- a/pkg/kv/kvserver/raft_log_queue.go +++ b/pkg/kv/kvserver/raft_log_queue.go @@ -288,7 +288,7 @@ func newTruncateDecision(ctx context.Context, r *Replica) (truncateDecision, err // Is this the raft leader? We only propose log truncation on the raft // leader which has the up to date info on followers. - if raftStatus.RaftState != raft.StateLeader { + if raftStatus.RaftState != raftpb.StateLeader { return truncateDecision{}, nil } diff --git a/pkg/kv/kvserver/raft_log_queue_test.go b/pkg/kv/kvserver/raft_log_queue_test.go index 32adc62eba75..e6ece138b244 100644 --- a/pkg/kv/kvserver/raft_log_queue_test.go +++ b/pkg/kv/kvserver/raft_log_queue_test.go @@ -205,7 +205,7 @@ func TestComputeTruncateDecision(t *testing.T) { assert.False(t, recompute) assert.Equal(t, decision.ShouldTruncate(), prio != 0) input.LogSizeTrusted = false - input.RaftStatus.RaftState = raft.StateLeader + input.RaftStatus.RaftState = raftpb.StateLeader if input.LastIndex <= input.FirstIndex { input.LastIndex = input.FirstIndex + 1 } diff --git a/pkg/kv/kvserver/raftutil/util.go b/pkg/kv/kvserver/raftutil/util.go index 0ad2b796bd66..17f1e6150921 100644 --- a/pkg/kv/kvserver/raftutil/util.go +++ b/pkg/kv/kvserver/raftutil/util.go @@ -22,7 +22,7 @@ func ReplicaIsBehind(st *raft.Status, replicaID roachpb.ReplicaID) bool { // Testing only. return true } - if st.RaftState != raft.StateLeader { + if st.RaftState != raftpb.StateLeader { // If we aren't the Raft leader, we aren't tracking the replica's progress, // so we can't be sure it's not behind. return true @@ -131,7 +131,7 @@ func ReplicaMayNeedSnapshot( // Testing only. return NoRaftStatusAvailable } - if st.RaftState != raft.StateLeader { + if st.RaftState != raftpb.StateLeader { // If we aren't the Raft leader, we aren't tracking the replica's progress, // so we can't be sure it does not need a snapshot. return LocalReplicaNotLeader diff --git a/pkg/kv/kvserver/raftutil/util_test.go b/pkg/kv/kvserver/raftutil/util_test.go index cdaa12388a9a..269b40564ef4 100644 --- a/pkg/kv/kvserver/raftutil/util_test.go +++ b/pkg/kv/kvserver/raftutil/util_test.go @@ -32,28 +32,28 @@ func TestReplicaIsBehind(t *testing.T) { { name: "local follower", st: makeStatus(func(st *raft.Status) { - st.RaftState = raft.StateFollower + st.RaftState = raftpb.StateFollower }), expect: true, }, { name: "local candidate", st: makeStatus(func(st *raft.Status) { - st.RaftState = raft.StateCandidate + st.RaftState = raftpb.StateCandidate }), expect: true, }, { name: "local leader, no progress for peer", st: makeStatus(func(st *raft.Status) { - st.RaftState = raft.StateLeader + st.RaftState = raftpb.StateLeader }), expect: true, }, { name: "local leader, peer leader", st: makeStatus(func(st *raft.Status) { - st.RaftState = raft.StateLeader + st.RaftState = raftpb.StateLeader st.Progress[replicaID] = tracker.Progress{State: tracker.StateReplicate} st.Lead = replicaID }), @@ -62,7 +62,7 @@ func TestReplicaIsBehind(t *testing.T) { { name: "local leader, peer state probe", st: makeStatus(func(st *raft.Status) { - st.RaftState = raft.StateLeader + st.RaftState = raftpb.StateLeader st.Progress[replicaID] = tracker.Progress{State: tracker.StateProbe} }), expect: true, @@ -70,7 +70,7 @@ func TestReplicaIsBehind(t *testing.T) { { name: "local leader, peer state snapshot", st: makeStatus(func(st *raft.Status) { - st.RaftState = raft.StateLeader + st.RaftState = raftpb.StateLeader st.Progress[replicaID] = tracker.Progress{State: tracker.StateSnapshot} }), expect: true, @@ -78,7 +78,7 @@ func TestReplicaIsBehind(t *testing.T) { { name: "local leader, peer state replicate, match < commit", st: makeStatus(func(st *raft.Status) { - st.RaftState = raft.StateLeader + st.RaftState = raftpb.StateLeader st.Progress[replicaID] = tracker.Progress{State: tracker.StateReplicate, Match: 9} }), expect: true, @@ -86,7 +86,7 @@ func TestReplicaIsBehind(t *testing.T) { { name: "local leader, peer state replicate, match == commit", st: makeStatus(func(st *raft.Status) { - st.RaftState = raft.StateLeader + st.RaftState = raftpb.StateLeader st.Progress[replicaID] = tracker.Progress{State: tracker.StateReplicate, Match: 10} }), expect: false, @@ -94,7 +94,7 @@ func TestReplicaIsBehind(t *testing.T) { { name: "local leader, peer state replicate, match > commit", st: makeStatus(func(st *raft.Status) { - st.RaftState = raft.StateLeader + st.RaftState = raftpb.StateLeader st.Progress[replicaID] = tracker.Progress{State: tracker.StateReplicate, Match: 11} }), expect: false, @@ -131,28 +131,28 @@ func TestReplicaMayNeedSnapshot(t *testing.T) { { name: "local follower", st: makeStatus(func(st *raft.Status) { - st.RaftState = raft.StateFollower + st.RaftState = raftpb.StateFollower }), expect: LocalReplicaNotLeader, }, { name: "local candidate", st: makeStatus(func(st *raft.Status) { - st.RaftState = raft.StateCandidate + st.RaftState = raftpb.StateCandidate }), expect: LocalReplicaNotLeader, }, { name: "local leader, no progress for peer", st: makeStatus(func(st *raft.Status) { - st.RaftState = raft.StateLeader + st.RaftState = raftpb.StateLeader }), expect: ReplicaUnknown, }, { name: "local leader, peer state probe", st: makeStatus(func(st *raft.Status) { - st.RaftState = raft.StateLeader + st.RaftState = raftpb.StateLeader st.Progress[replicaID] = tracker.Progress{State: tracker.StateProbe} }), expect: ReplicaStateProbe, @@ -160,7 +160,7 @@ func TestReplicaMayNeedSnapshot(t *testing.T) { { name: "local leader, peer state snapshot", st: makeStatus(func(st *raft.Status) { - st.RaftState = raft.StateLeader + st.RaftState = raftpb.StateLeader st.Progress[replicaID] = tracker.Progress{State: tracker.StateSnapshot} }), expect: ReplicaStateSnapshot, @@ -168,7 +168,7 @@ func TestReplicaMayNeedSnapshot(t *testing.T) { { name: "local leader, peer state replicate, match+1 < firstIndex", st: makeStatus(func(st *raft.Status) { - st.RaftState = raft.StateLeader + st.RaftState = raftpb.StateLeader st.Progress[replicaID] = tracker.Progress{State: tracker.StateReplicate, Match: 8} }), expect: ReplicaMatchBelowLeadersFirstIndex, @@ -176,7 +176,7 @@ func TestReplicaMayNeedSnapshot(t *testing.T) { { name: "local leader, peer state replicate, match+1 == firstIndex", st: makeStatus(func(st *raft.Status) { - st.RaftState = raft.StateLeader + st.RaftState = raftpb.StateLeader st.Progress[replicaID] = tracker.Progress{State: tracker.StateReplicate, Match: 9} }), expect: NoSnapshotNeeded, @@ -184,7 +184,7 @@ func TestReplicaMayNeedSnapshot(t *testing.T) { { name: "local leader, peer state replicate, match+1 == firstIndex", st: makeStatus(func(st *raft.Status) { - st.RaftState = raft.StateLeader + st.RaftState = raftpb.StateLeader st.Progress[replicaID] = tracker.Progress{State: tracker.StateReplicate, Match: 10} }), expect: NoSnapshotNeeded, diff --git a/pkg/kv/kvserver/replica_application_result.go b/pkg/kv/kvserver/replica_application_result.go index 73c7c472fe8f..4f85db924d3c 100644 --- a/pkg/kv/kvserver/replica_application_result.go +++ b/pkg/kv/kvserver/replica_application_result.go @@ -13,7 +13,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/kv/kvserver/kvserverpb" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/raftlog" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/readsummary/rspb" - "github.com/cockroachdb/cockroach/pkg/raft" + "github.com/cockroachdb/cockroach/pkg/raft/raftpb" "github.com/cockroachdb/cockroach/pkg/roachpb" "github.com/cockroachdb/cockroach/pkg/storage/enginepb" "github.com/cockroachdb/cockroach/pkg/util/hlc" @@ -582,7 +582,7 @@ func (r *Replica) handleChangeReplicasResult( // This is currently executed before the conf change is applied to the Raft // node, so we still see ourselves as the leader. - if r.raftBasicStatusRLocked().RaftState == raft.StateLeader { + if r.raftBasicStatusRLocked().RaftState == raftpb.StateLeader { r.store.metrics.RangeRaftLeaderRemovals.Inc(1) } diff --git a/pkg/kv/kvserver/replica_command.go b/pkg/kv/kvserver/replica_command.go index 4c85550664fe..8f540d7a588a 100644 --- a/pkg/kv/kvserver/replica_command.go +++ b/pkg/kv/kvserver/replica_command.go @@ -136,7 +136,7 @@ func maybeDescriptorChangedError( func splitSnapshotWarningStr(rangeID roachpb.RangeID, status *raft.Status) redact.RedactableString { var s redact.RedactableString - if status != nil && status.RaftState == raft.StateLeader { + if status != nil && status.RaftState == raftpb.StateLeader { for replicaID, pr := range status.Progress { if replicaID == status.Lead { // TODO(tschottdorf): remove this line once we have picked up diff --git a/pkg/kv/kvserver/replica_gc_queue.go b/pkg/kv/kvserver/replica_gc_queue.go index 967f8e99c11f..2874c355ad9a 100644 --- a/pkg/kv/kvserver/replica_gc_queue.go +++ b/pkg/kv/kvserver/replica_gc_queue.go @@ -12,7 +12,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/kv" "github.com/cockroachdb/cockroach/pkg/kv/kvpb" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/kvserverbase" - "github.com/cockroachdb/cockroach/pkg/raft" + "github.com/cockroachdb/cockroach/pkg/raft/raftpb" "github.com/cockroachdb/cockroach/pkg/roachpb" "github.com/cockroachdb/cockroach/pkg/spanconfig" "github.com/cockroachdb/cockroach/pkg/util/hlc" @@ -168,7 +168,7 @@ func replicaIsSuspect(repl *Replica) bool { switch raftStatus.SoftState.RaftState { // If a replica is a candidate, then by definition it has lost contact with // its leader and possibly the rest of the Raft group, so consider it suspect. - case raft.StateCandidate, raft.StatePreCandidate: + case raftpb.StateCandidate, raftpb.StatePreCandidate: return true // If the replica is a follower, check that the leader is in our range @@ -177,7 +177,7 @@ func replicaIsSuspect(repl *Replica) bool { // a quiesced follower which was partitioned away from the Raft group during // its own removal from the range -- this case is vulnerable to race // conditions, but if it fails it will be GCed within 12 hours anyway. - case raft.StateFollower: + case raftpb.StateFollower: leadDesc, ok := repl.Desc().GetReplicaDescriptorByID(roachpb.ReplicaID(raftStatus.Lead)) if !ok || !livenessMap[leadDesc.NodeID].IsLive { return true @@ -186,7 +186,7 @@ func replicaIsSuspect(repl *Replica) bool { // If the replica is a leader, check that it has a quorum. This handles e.g. // a stuck leader with a lost quorum being replaced via Node.ResetQuorum, // which must cause the stale leader to relinquish its lease and GC itself. - case raft.StateLeader: + case raftpb.StateLeader: if !repl.Desc().Replicas().CanMakeProgress(func(d roachpb.ReplicaDescriptor) bool { return livenessMap[d.NodeID].IsLive }) { diff --git a/pkg/kv/kvserver/replica_metrics.go b/pkg/kv/kvserver/replica_metrics.go index cbe2d2b3be5a..4a167625199a 100644 --- a/pkg/kv/kvserver/replica_metrics.go +++ b/pkg/kv/kvserver/replica_metrics.go @@ -174,7 +174,7 @@ func calcReplicaMetrics(d calcReplicaMetricsInput) ReplicaMetrics { // The raft leader computes the number of raft entries that replicas are // behind. - leader := d.raftStatus != nil && d.raftStatus.RaftState == raft.StateLeader + leader := d.raftStatus != nil && d.raftStatus.RaftState == raftpb.StateLeader var leaderBehindCount, leaderPausedFollowerCount int64 if leader { leaderBehindCount = calcBehindCount(d.raftStatus, d.desc, d.vitalityMap) diff --git a/pkg/kv/kvserver/replica_proposal_buf.go b/pkg/kv/kvserver/replica_proposal_buf.go index 96358f99bade..633568fa7b5e 100644 --- a/pkg/kv/kvserver/replica_proposal_buf.go +++ b/pkg/kv/kvserver/replica_proposal_buf.go @@ -125,7 +125,7 @@ type admitEntHandle struct { type singleBatchProposer interface { getReplicaID() roachpb.ReplicaID flowControlHandle(ctx context.Context) kvflowcontrol.Handle - onErrProposalDropped([]raftpb.Entry, []*ProposalData, raft.StateType) + onErrProposalDropped([]raftpb.Entry, []*ProposalData, raftpb.StateType) } // A proposer is an object that uses a propBuf to coordinate Raft proposals. @@ -1185,11 +1185,11 @@ func (rp *replicaProposer) withGroupLocked(fn func(raftGroup proposerRaft) error } func (rp *replicaProposer) onErrProposalDropped( - ents []raftpb.Entry, _ []*ProposalData, stateType raft.StateType, + ents []raftpb.Entry, _ []*ProposalData, stateType raftpb.StateType, ) { n := int64(len(ents)) rp.store.metrics.RaftProposalsDropped.Inc(n) - if stateType == raft.StateLeader { + if stateType == raftpb.StateLeader { rp.store.metrics.RaftProposalsDroppedLeader.Inc(n) } } diff --git a/pkg/kv/kvserver/replica_proposal_buf_test.go b/pkg/kv/kvserver/replica_proposal_buf_test.go index 7f30a9a74bfd..bdb47cb3a7eb 100644 --- a/pkg/kv/kvserver/replica_proposal_buf_test.go +++ b/pkg/kv/kvserver/replica_proposal_buf_test.go @@ -60,7 +60,7 @@ type testProposer struct { onRejectProposalWithErrLocked func(err error) // If not nil, this is called by onErrProposalDropped. onProposalsDropped func( - ents []raftpb.Entry, proposalData []*ProposalData, stateType raft.StateType, + ents []raftpb.Entry, proposalData []*ProposalData, stateType raftpb.StateType, ) // validLease is returned by ownsValidLease. validLease bool @@ -189,7 +189,7 @@ func (t *testProposer) withGroupLocked(fn func(proposerRaft) error) error { } func (rp *testProposer) onErrProposalDropped( - ents []raftpb.Entry, props []*ProposalData, typ raft.StateType, + ents []raftpb.Entry, props []*ProposalData, typ raftpb.StateType, ) { if rp.onProposalsDropped == nil { return @@ -516,7 +516,7 @@ func TestProposalBufferRejectLeaseAcqOnFollower(t *testing.T) { // scenario. Some proposals should be allowed, some should be rejected. for _, tc := range []struct { name string - state raft.StateType + state raftpb.StateType // raft.None means there's no leader, or the leader is unknown. leader raftpb.PeerID // Empty means VOTER_FULL. @@ -535,14 +535,14 @@ func TestProposalBufferRejectLeaseAcqOnFollower(t *testing.T) { }{ { name: "leader", - state: raft.StateLeader, + state: raftpb.StateLeader, leader: self, // No rejection. The leader can request a lease. expRejection: false, }, { name: "follower, known eligible leader", - state: raft.StateFollower, + state: raftpb.StateFollower, // Someone else is leader. leader: self + 1, // Rejection - a follower can't request a lease. @@ -550,7 +550,7 @@ func TestProposalBufferRejectLeaseAcqOnFollower(t *testing.T) { }, { name: "follower, lease extension despite known eligible leader", - state: raft.StateFollower, + state: raftpb.StateFollower, // Someone else is leader, but we're the leaseholder. leader: self + 1, ownsValidLease: true, @@ -559,7 +559,7 @@ func TestProposalBufferRejectLeaseAcqOnFollower(t *testing.T) { }, { name: "follower, known ineligible leader", - state: raft.StateFollower, + state: raftpb.StateFollower, // Someone else is leader. leader: self + 1, // The leader type makes it ineligible to get the lease. Thus, the local @@ -571,7 +571,7 @@ func TestProposalBufferRejectLeaseAcqOnFollower(t *testing.T) { // Here we simulate the leader being known by Raft, but the local replica // is so far behind that it doesn't contain the leader replica. name: "follower, known leader not in range descriptor", - state: raft.StateFollower, + state: raftpb.StateFollower, // Someone else is leader. leader: self + 1, leaderNotInRngDesc: true, @@ -580,7 +580,7 @@ func TestProposalBufferRejectLeaseAcqOnFollower(t *testing.T) { }, { name: "follower, unknown leader", - state: raft.StateFollower, + state: raftpb.StateFollower, // Unknown leader. leader: raft.None, // No rejection if the leader is unknown. See comments in @@ -589,7 +589,7 @@ func TestProposalBufferRejectLeaseAcqOnFollower(t *testing.T) { }, { name: "follower, known eligible non-live leader", - state: raft.StateFollower, + state: raftpb.StateFollower, // Someone else is leader. leader: self + 1, leaderNotLive: true, @@ -686,7 +686,7 @@ func TestProposalBufferRejectUnsafeLeaseTransfer(t *testing.T) { // scenario. Some proposals should be allowed, some should be rejected. for _, tc := range []struct { name string - proposerState raft.StateType + proposerState raftpb.StateType // math.MaxUint64 if the target is not in the raft group. targetState rafttracker.StateType targetMatch kvpb.RaftIndex @@ -696,40 +696,40 @@ func TestProposalBufferRejectUnsafeLeaseTransfer(t *testing.T) { }{ { name: "follower", - proposerState: raft.StateFollower, + proposerState: raftpb.StateFollower, expRejection: true, expRejectionReason: raftutil.LocalReplicaNotLeader, }, { name: "candidate", - proposerState: raft.StateCandidate, + proposerState: raftpb.StateCandidate, expRejection: true, expRejectionReason: raftutil.LocalReplicaNotLeader, }, { name: "leader, no progress for target", - proposerState: raft.StateLeader, + proposerState: raftpb.StateLeader, targetState: math.MaxUint64, expRejection: true, expRejectionReason: raftutil.ReplicaUnknown, }, { name: "leader, target state probe", - proposerState: raft.StateLeader, + proposerState: raftpb.StateLeader, targetState: rafttracker.StateProbe, expRejection: true, expRejectionReason: raftutil.ReplicaStateProbe, }, { name: "leader, target state snapshot", - proposerState: raft.StateLeader, + proposerState: raftpb.StateLeader, targetState: rafttracker.StateSnapshot, expRejection: true, expRejectionReason: raftutil.ReplicaStateSnapshot, }, { name: "leader, target state replicate, match+1 < firstIndex", - proposerState: raft.StateLeader, + proposerState: raftpb.StateLeader, targetState: rafttracker.StateReplicate, targetMatch: proposerFirstIndex - 2, expRejection: true, @@ -737,14 +737,14 @@ func TestProposalBufferRejectUnsafeLeaseTransfer(t *testing.T) { }, { name: "leader, target state replicate, match+1 == firstIndex", - proposerState: raft.StateLeader, + proposerState: raftpb.StateLeader, targetState: rafttracker.StateReplicate, targetMatch: proposerFirstIndex - 1, expRejection: false, }, { name: "leader, target state replicate, match+1 > firstIndex", - proposerState: raft.StateLeader, + proposerState: raftpb.StateLeader, targetState: rafttracker.StateReplicate, targetMatch: proposerFirstIndex, expRejection: false, @@ -773,7 +773,7 @@ func TestProposalBufferRejectUnsafeLeaseTransfer(t *testing.T) { var raftStatus raft.Status raftStatus.ID = proposer raftStatus.RaftState = tc.proposerState - if tc.proposerState == raft.StateLeader { + if tc.proposerState == raftpb.StateLeader { raftStatus.Lead = proposer raftStatus.Progress = map[raftpb.PeerID]rafttracker.Progress{ proposer: {State: rafttracker.StateReplicate, Match: uint64(proposerFirstIndex)}, @@ -831,7 +831,7 @@ func TestProposalBufferLinesUpEntriesAndProposals(t *testing.T) { var matchingDroppedProposalsSeen int p := testProposer{ - onProposalsDropped: func(ents []raftpb.Entry, props []*ProposalData, _ raft.StateType) { + onProposalsDropped: func(ents []raftpb.Entry, props []*ProposalData, _ raftpb.StateType) { require.Equal(t, len(ents), len(props)) for i := range ents { if ents[i].Type == raftpb.EntryNormal { @@ -1097,7 +1097,7 @@ func TestProposalBufferClosedTimestamp(t *testing.T) { t.Run(tc.name, func(t *testing.T) { r := &testProposerRaft{} r.status.Lead = 1 - r.status.RaftState = raft.StateLeader + r.status.RaftState = raftpb.StateLeader r.status.Progress = map[raftpb.PeerID]rafttracker.Progress{ 1: {State: rafttracker.StateReplicate}, } diff --git a/pkg/kv/kvserver/replica_raft.go b/pkg/kv/kvserver/replica_raft.go index ff7fea50843d..e5b91ebe7985 100644 --- a/pkg/kv/kvserver/replica_raft.go +++ b/pkg/kv/kvserver/replica_raft.go @@ -631,7 +631,7 @@ func (r *Replica) stepRaftGroupRaftMuLocked(req *kvserverpb.RaftMessageRequest) // mass unquiescence due to the continuous prevotes. if r.mu.quiescent { st := r.raftBasicStatusRLocked() - hasLeader := st.RaftState == raft.StateFollower && st.Lead != 0 + hasLeader := st.RaftState == raftpb.StateFollower && st.Lead != 0 fromLeader := raftpb.PeerID(req.FromReplica.ReplicaID) == st.Lead wakeLeader := hasLeader && !fromLeader r.maybeUnquiesceLocked(wakeLeader, false /* mayCampaign */) @@ -1487,7 +1487,7 @@ func (r *Replica) tick( r.mu.internalRaftGroup.Tick() postTickState := r.mu.internalRaftGroup.BasicStatus().RaftState if preTickState != postTickState { - if postTickState == raft.StatePreCandidate { + if postTickState == raftpb.StatePreCandidate { r.store.Metrics().RaftTimeoutCampaign.Inc(1) if k := r.store.TestingKnobs(); k != nil && k.OnRaftTimeoutCampaign != nil { k.OnRaftTimeoutCampaign(r.RangeID) @@ -2317,7 +2317,7 @@ func shouldCampaignOnWake( return false } // If we're already campaigning don't start a new term. - if raftStatus.RaftState != raft.StateFollower { + if raftStatus.RaftState != raftpb.StateFollower { return false } // If we don't know who the leader is, then campaign. @@ -2428,7 +2428,7 @@ func shouldForgetLeaderOnVoteRequest( now hlc.Timestamp, ) bool { // If we're not a follower with a leader, there's noone to forget. - if raftStatus.RaftState != raft.StateFollower || raftStatus.Lead == raft.None { + if raftStatus.RaftState != raftpb.StateFollower || raftStatus.Lead == raft.None { return false } @@ -2479,7 +2479,7 @@ func shouldCampaignOnLeaseRequestRedirect( now hlc.Timestamp, ) bool { // If we're already campaigning don't start a new term. - if raftStatus.RaftState != raft.StateFollower { + if raftStatus.RaftState != raftpb.StateFollower { return false } // If we don't know who the leader is, then campaign. @@ -2657,7 +2657,7 @@ func shouldTransferRaftLeadershipToLeaseholderLocked( draining bool, ) bool { // If we're not the leader, there's nothing to do. - if raftStatus.RaftState != raft.StateLeader { + if raftStatus.RaftState != raftpb.StateLeader { return false } @@ -3014,7 +3014,7 @@ func shouldCampaignAfterConfChange( // throwing spurious elections. return false } - if raftStatus.RaftState == raft.StateLeader { + if raftStatus.RaftState == raftpb.StateLeader { // We're already the leader, no point in campaigning. return false } diff --git a/pkg/kv/kvserver/replica_raft_quiesce.go b/pkg/kv/kvserver/replica_raft_quiesce.go index d4c3fbb5b18b..fc7364da2393 100644 --- a/pkg/kv/kvserver/replica_raft_quiesce.go +++ b/pkg/kv/kvserver/replica_raft_quiesce.go @@ -87,11 +87,11 @@ func (r *Replica) maybeUnquiesceLocked(wakeLeader, mayCampaign bool) bool { r.store.unquiescedReplicas.Unlock() st := r.raftSparseStatusRLocked() - if st.RaftState == raft.StateLeader { + if st.RaftState == raftpb.StateLeader { r.mu.lastUpdateTimes.updateOnUnquiesce( r.shMu.state.Desc.Replicas().Descriptors(), st.Progress, r.Clock().PhysicalTime()) - } else if st.RaftState == raft.StateFollower && st.Lead != raft.None && wakeLeader { + } else if st.RaftState == raftpb.StateFollower && st.Lead != raft.None && wakeLeader { // Propose an empty command which will wake the leader. if log.V(3) { log.Infof(ctx, "waking r%d leader", r.RangeID) @@ -371,7 +371,7 @@ func shouldReplicaQuiesceRaftMuLockedReplicaMuLocked( } return nil, nil, false } - if status.SoftState.RaftState != raft.StateLeader { + if status.SoftState.RaftState != raftpb.StateLeader { if log.V(4) { log.Infof(ctx, "not quiescing: not leader") } diff --git a/pkg/kv/kvserver/replica_rangefeed_test.go b/pkg/kv/kvserver/replica_rangefeed_test.go index dbbec57d033d..013fdd9a049d 100644 --- a/pkg/kv/kvserver/replica_rangefeed_test.go +++ b/pkg/kv/kvserver/replica_rangefeed_test.go @@ -23,7 +23,6 @@ import ( "github.com/cockroachdb/cockroach/pkg/kv/kvserver/kvserverpb" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/liveness" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed" - "github.com/cockroachdb/cockroach/pkg/raft" "github.com/cockroachdb/cockroach/pkg/raft/raftpb" "github.com/cockroachdb/cockroach/pkg/roachpb" "github.com/cockroachdb/cockroach/pkg/server" @@ -1092,7 +1091,7 @@ func TestReplicaRangefeedErrors(t *testing.T) { return err } raftStatus := repl.RaftStatus() - if raftStatus != nil && raftStatus.RaftState == raft.StateFollower { + if raftStatus != nil && raftStatus.RaftState == raftpb.StateFollower { return nil } err = repl.AdminTransferLease(ctx, roachpb.StoreID(1), false /* bypassSafetyChecks */) diff --git a/pkg/kv/kvserver/replica_test.go b/pkg/kv/kvserver/replica_test.go index 5a052a9762a0..99d2d78f4115 100644 --- a/pkg/kv/kvserver/replica_test.go +++ b/pkg/kv/kvserver/replica_test.go @@ -133,7 +133,7 @@ func upToDateRaftStatus(repls []roachpb.ReplicaDescriptor) *raft.Status { return &raft.Status{ BasicStatus: raft.BasicStatus{ HardState: raftpb.HardState{Commit: 100, Lead: 1}, - SoftState: raft.SoftState{RaftState: raft.StateLeader}, + SoftState: raft.SoftState{RaftState: raftpb.StateLeader}, }, Progress: prs, } @@ -8995,9 +8995,9 @@ func TestReplicaMetrics(t *testing.T) { // and 2 is ok. status.HardState.Commit = 12 if lead == 1 { - status.SoftState.RaftState = raft.StateLeader + status.SoftState.RaftState = raftpb.StateLeader } else { - status.SoftState.RaftState = raft.StateFollower + status.SoftState.RaftState = raftpb.StateFollower } status.HardState.Lead = lead return status @@ -9886,7 +9886,7 @@ func (q *testQuiescer) descRLocked() *roachpb.RangeDescriptor { } func (q *testQuiescer) isRaftLeaderRLocked() bool { - return q.status != nil && q.status.RaftState == raft.StateLeader + return q.status != nil && q.status.RaftState == raftpb.StateLeader } func (q *testQuiescer) raftSparseStatusRLocked() *raft.SparseStatus { @@ -9960,7 +9960,7 @@ func TestShouldReplicaQuiesce(t *testing.T) { Commit: logIndex, }, SoftState: raft.SoftState{ - RaftState: raft.StateLeader, + RaftState: raftpb.StateLeader, }, Applied: logIndex, LeadTransferee: 0, @@ -10034,8 +10034,8 @@ func TestShouldReplicaQuiesce(t *testing.T) { test(false, func(q *testQuiescer) { q.mergeInProgress = true }) test(false, func(q *testQuiescer) { q.isDestroyed = true }) test(false, func(q *testQuiescer) { q.status = nil }) - test(false, func(q *testQuiescer) { q.status.RaftState = raft.StateFollower }) - test(false, func(q *testQuiescer) { q.status.RaftState = raft.StateCandidate }) + test(false, func(q *testQuiescer) { q.status.RaftState = raftpb.StateFollower }) + test(false, func(q *testQuiescer) { q.status.RaftState = raftpb.StateCandidate }) test(false, func(q *testQuiescer) { q.status.LeadTransferee = 1 }) test(false, func(q *testQuiescer) { q.status.Commit = invalidIndex }) test(false, func(q *testQuiescer) { q.status.Applied = invalidIndex }) @@ -11507,7 +11507,7 @@ func TestReplicaShouldCampaignOnWake(t *testing.T) { }, raftStatus: raft.BasicStatus{ SoftState: raft.SoftState{ - RaftState: raft.StateFollower, + RaftState: raftpb.StateFollower, }, HardState: raftpb.HardState{ Lead: 2, @@ -11533,15 +11533,15 @@ func TestReplicaShouldCampaignOnWake(t *testing.T) { p.leaseStatus.Lease.Replica.StoreID = 1 }}, "pre-candidate": {false, func(p *params) { - p.raftStatus.SoftState.RaftState = raft.StatePreCandidate + p.raftStatus.SoftState.RaftState = raftpb.StatePreCandidate p.raftStatus.Lead = raft.None }}, "candidate": {false, func(p *params) { - p.raftStatus.SoftState.RaftState = raft.StateCandidate + p.raftStatus.SoftState.RaftState = raftpb.StateCandidate p.raftStatus.Lead = raft.None }}, "leader": {false, func(p *params) { - p.raftStatus.SoftState.RaftState = raft.StateLeader + p.raftStatus.SoftState.RaftState = raftpb.StateLeader p.raftStatus.Lead = 1 }}, "unknown leader": {true, func(p *params) { @@ -11627,7 +11627,7 @@ func TestReplicaShouldCampaignOnLeaseRequestRedirect(t *testing.T) { }, raftStatus: raft.BasicStatus{ SoftState: raft.SoftState{ - RaftState: raft.StateFollower, + RaftState: raftpb.StateFollower, }, HardState: raftpb.HardState{ Lead: 2, @@ -11648,15 +11648,15 @@ func TestReplicaShouldCampaignOnLeaseRequestRedirect(t *testing.T) { }{ "dead leader": {true, func(p *params) {}}, "pre-candidate": {false, func(p *params) { - p.raftStatus.SoftState.RaftState = raft.StatePreCandidate + p.raftStatus.SoftState.RaftState = raftpb.StatePreCandidate p.raftStatus.Lead = raft.None }}, "candidate": {false, func(p *params) { - p.raftStatus.SoftState.RaftState = raft.StateCandidate + p.raftStatus.SoftState.RaftState = raftpb.StateCandidate p.raftStatus.Lead = raft.None }}, "leader": {false, func(p *params) { - p.raftStatus.SoftState.RaftState = raft.StateLeader + p.raftStatus.SoftState.RaftState = raftpb.StateLeader p.raftStatus.Lead = 1 }}, "unknown leader": {true, func(p *params) { @@ -11744,7 +11744,7 @@ func TestReplicaShouldForgetLeaderOnVoteRequest(t *testing.T) { }, raftStatus: raft.BasicStatus{ SoftState: raft.SoftState{ - RaftState: raft.StateFollower, + RaftState: raftpb.StateFollower, }, HardState: raftpb.HardState{ Lead: 2, @@ -11764,15 +11764,15 @@ func TestReplicaShouldForgetLeaderOnVoteRequest(t *testing.T) { }{ "dead leader": {true, func(p *params) {}}, "pre-candidate": {false, func(p *params) { - p.raftStatus.SoftState.RaftState = raft.StatePreCandidate + p.raftStatus.SoftState.RaftState = raftpb.StatePreCandidate p.raftStatus.Lead = raft.None }}, "candidate": {false, func(p *params) { - p.raftStatus.SoftState.RaftState = raft.StateCandidate + p.raftStatus.SoftState.RaftState = raftpb.StateCandidate p.raftStatus.Lead = raft.None }}, "leader": {false, func(p *params) { - p.raftStatus.SoftState.RaftState = raft.StateLeader + p.raftStatus.SoftState.RaftState = raftpb.StateLeader p.raftStatus.Lead = 1 }}, "unknown leader": {false, func(p *params) { @@ -11848,7 +11848,7 @@ func TestReplicaShouldTransferRaftLeadershipToLeaseholder(t *testing.T) { raftStatus: raft.SparseStatus{ BasicStatus: raft.BasicStatus{ SoftState: raft.SoftState{ - RaftState: raft.StateLeader, + RaftState: raftpb.StateLeader, }, HardState: raftpb.HardState{ Lead: localID, @@ -11878,15 +11878,15 @@ func TestReplicaShouldTransferRaftLeadershipToLeaseholder(t *testing.T) { true, func(p *params) {}, }, "follower": {false, func(p *params) { - p.raftStatus.SoftState.RaftState = raft.StateFollower + p.raftStatus.SoftState.RaftState = raftpb.StateFollower p.raftStatus.Lead = remoteID }}, "pre-candidate": {false, func(p *params) { - p.raftStatus.SoftState.RaftState = raft.StatePreCandidate + p.raftStatus.SoftState.RaftState = raftpb.StatePreCandidate p.raftStatus.Lead = raft.None }}, "candidate": {false, func(p *params) { - p.raftStatus.SoftState.RaftState = raft.StateCandidate + p.raftStatus.SoftState.RaftState = raftpb.StateCandidate p.raftStatus.Lead = raft.None }}, "invalid lease": {false, func(p *params) { diff --git a/pkg/kv/kvserver/split_delay_helper.go b/pkg/kv/kvserver/split_delay_helper.go index 237b95afa342..63bd46fb9084 100644 --- a/pkg/kv/kvserver/split_delay_helper.go +++ b/pkg/kv/kvserver/split_delay_helper.go @@ -11,6 +11,7 @@ import ( "time" "github.com/cockroachdb/cockroach/pkg/raft" + "github.com/cockroachdb/cockroach/pkg/raft/raftpb" "github.com/cockroachdb/cockroach/pkg/raft/tracker" "github.com/cockroachdb/cockroach/pkg/roachpb" "github.com/cockroachdb/cockroach/pkg/util/timeutil" @@ -86,7 +87,7 @@ func maybeDelaySplitToAvoidSnapshot( problems = problems[:0] rangeID, raftStatus := sdh.RaftStatus(ctx) - if raftStatus == nil || raftStatus.RaftState == raft.StateFollower { + if raftStatus == nil || raftStatus.RaftState == raftpb.StateFollower { // Don't delay on followers (we don't have information about the // peers in that state and thus can't determine when it is safe // to continue). This case is hit rarely enough to not matter, @@ -145,7 +146,7 @@ func maybeDelaySplitToAvoidSnapshot( // // See TestSplitBurstWithSlowFollower for end-to-end verification of this // mechanism. - if raftStatus.RaftState != raft.StateLeader { + if raftStatus.RaftState != raftpb.StateLeader { problems = append(problems, redact.Sprintf("not leader (%s)", redact.Safe(raftStatus.RaftState))) } diff --git a/pkg/kv/kvserver/split_delay_helper_test.go b/pkg/kv/kvserver/split_delay_helper_test.go index 8d80d2b3e4fd..2ad5ad9ca5da 100644 --- a/pkg/kv/kvserver/split_delay_helper_test.go +++ b/pkg/kv/kvserver/split_delay_helper_test.go @@ -68,7 +68,7 @@ func TestSplitDelayToAvoidSnapshot(t *testing.T) { assert.EqualValues(t, 0, h.slept) }) - statusWithState := func(status raft.StateType) *raft.Status { + statusWithState := func(status raftpb.StateType) *raft.Status { return &raft.Status{ BasicStatus: raft.BasicStatus{ SoftState: raft.SoftState{ @@ -95,14 +95,14 @@ func TestSplitDelayToAvoidSnapshot(t *testing.T) { h := &testSplitDelayHelper{ numAttempts: 5, rangeID: 1, - raftStatus: statusWithState(raft.StateFollower), + raftStatus: statusWithState(raftpb.StateFollower), } s := maybeDelaySplitToAvoidSnapshot(ctx, h) assert.EqualValues(t, "; delayed by 0.0s to resolve: replica is raft follower (without success)", s) assert.EqualValues(t, 0, h.slept) }) - for _, state := range []raft.StateType{raft.StatePreCandidate, raft.StateCandidate} { + for _, state := range []raftpb.StateType{raftpb.StatePreCandidate, raftpb.StateCandidate} { t.Run(state.String(), func(t *testing.T) { h := &testSplitDelayHelper{ numAttempts: 5, @@ -115,7 +115,7 @@ func TestSplitDelayToAvoidSnapshot(t *testing.T) { } t.Run("inactive", func(t *testing.T) { - st := statusWithState(raft.StateLeader) + st := statusWithState(raftpb.StateLeader) st.Progress = map[raftpb.PeerID]tracker.Progress{ 2: {State: tracker.StateProbe}, } @@ -132,7 +132,7 @@ func TestSplitDelayToAvoidSnapshot(t *testing.T) { for _, state := range []tracker.StateType{tracker.StateProbe, tracker.StateSnapshot} { t.Run(state.String(), func(t *testing.T) { - st := statusWithState(raft.StateLeader) + st := statusWithState(raftpb.StateLeader) st.Progress = map[raftpb.PeerID]tracker.Progress{ 2: { State: state, @@ -155,7 +155,7 @@ func TestSplitDelayToAvoidSnapshot(t *testing.T) { } t.Run("immediately-replicating", func(t *testing.T) { - st := statusWithState(raft.StateLeader) + st := statusWithState(raftpb.StateLeader) st.Progress = map[raftpb.PeerID]tracker.Progress{ 2: {State: tracker.StateReplicate}, // intentionally not recently active } @@ -170,7 +170,7 @@ func TestSplitDelayToAvoidSnapshot(t *testing.T) { }) t.Run("becomes-replicating", func(t *testing.T) { - st := statusWithState(raft.StateLeader) + st := statusWithState(raftpb.StateLeader) st.Progress = map[raftpb.PeerID]tracker.Progress{ 2: {State: tracker.StateProbe, RecentActive: true, Inflights: &tracker.Inflights{}}, } diff --git a/pkg/kv/kvserver/store_rebalancer_test.go b/pkg/kv/kvserver/store_rebalancer_test.go index ffe874021c0e..a3da9e7ee11c 100644 --- a/pkg/kv/kvserver/store_rebalancer_test.go +++ b/pkg/kv/kvserver/store_rebalancer_test.go @@ -1562,7 +1562,7 @@ func TestNoLeaseTransferToBehindReplicas(t *testing.T) { require.True(t, ok, "Could not find replica descriptor for replica on store with id %d", storeID) status.Lead = raftpb.PeerID(replDesc.ReplicaID) - status.RaftState = raft.StateLeader + status.RaftState = raftpb.StateLeader status.Commit = 2 for _, replica := range desc.InternalReplicas { match := uint64(2) @@ -1850,7 +1850,7 @@ func TestingRaftStatusFn(desc *roachpb.RangeDescriptor, storeID roachpb.StoreID) } status.Lead = raftpb.PeerID(replDesc.ReplicaID) - status.RaftState = raft.StateLeader + status.RaftState = raftpb.StateLeader status.Commit = 2 for _, replica := range desc.InternalReplicas { status.Progress[raftpb.PeerID(replica.ReplicaID)] = tracker.Progress{ diff --git a/pkg/raft/node.go b/pkg/raft/node.go index 3c46a5cffb25..e0585c219e46 100644 --- a/pkg/raft/node.go +++ b/pkg/raft/node.go @@ -31,7 +31,7 @@ var emptyState = pb.HardState{} // SoftState provides state that is useful for logging and debugging. // The state is volatile and does not need to be persisted to the WAL. type SoftState struct { - RaftState StateType + RaftState pb.StateType } func (a *SoftState) equal(b *SoftState) bool { diff --git a/pkg/raft/node_test.go b/pkg/raft/node_test.go index f2737a1f7193..6760383145bb 100644 --- a/pkg/raft/node_test.go +++ b/pkg/raft/node_test.go @@ -462,7 +462,7 @@ func TestSoftStateEqual(t *testing.T) { we bool }{ {&SoftState{}, true}, - {&SoftState{RaftState: StateLeader}, false}, + {&SoftState{RaftState: raftpb.StateLeader}, false}, } for i, tt := range tests { assert.Equal(t, tt.we, tt.st.equal(&SoftState{}), "#%d", i) diff --git a/pkg/raft/raft.go b/pkg/raft/raft.go index 61bd86c82878..40c8e0e22dab 100644 --- a/pkg/raft/raft.go +++ b/pkg/raft/raft.go @@ -54,15 +54,6 @@ const ( LocalApplyThread pb.PeerID = math.MaxUint64 - 1 ) -// Possible values for StateType. -const ( - StateFollower StateType = iota - StateCandidate - StateLeader - StatePreCandidate - numStates -) - // Possible values for CampaignType const ( // campaignPreElection represents the first phase of a normal election when @@ -102,20 +93,6 @@ var globalRand = &lockedRand{} // is because it's simpler to compare and fill in raft entries type CampaignType string -// StateType represents the role of a node in a cluster. -type StateType uint64 - -var stmap = [...]string{ - "StateFollower", - "StateCandidate", - "StateLeader", - "StatePreCandidate", -} - -func (st StateType) String() string { - return stmap[st] -} - // Config contains the parameters to start a raft. type Config struct { // ID is the identity of the local raft. ID cannot be 0. @@ -345,7 +322,7 @@ type raft struct { fortificationTracker tracker.FortificationTracker lazyReplication bool - state StateType + state pb.StateType // isLearner is true if the local raft node is a learner. isLearner bool @@ -659,7 +636,7 @@ func (r *raft) prepareMsgApp(to pb.PeerID, pr *tracker.Progress, ls logSlice) pb // Returns false if the current state of the node does not permit this MsgApp // send, e.g. the log slice is misaligned with the replication flow status. func (r *raft) maybePrepareMsgApp(to pb.PeerID, ls logSlice) (pb.Message, bool) { - if r.state != StateLeader || r.Term != ls.term { + if r.state != pb.StateLeader || r.Term != ls.term { return pb.Message{}, false } pr := r.trk.Progress(to) @@ -716,7 +693,7 @@ func (r *raft) maybeSendAppend(to pb.PeerID) bool { } func (r *raft) sendPing(to pb.PeerID) bool { - if r.state != StateLeader { + if r.state != pb.StateLeader { return false } pr := r.trk.Progress(to) @@ -887,7 +864,7 @@ func (r *raft) bcastHeartbeat() { // (including the leader itself) whose stores are currently providing store // liveness support to the leader's store but who have not fortified the leader. func (r *raft) bcastFortify() { - assertTrue(r.state == StateLeader, "only leaders can fortify") + assertTrue(r.state == pb.StateLeader, "only leaders can fortify") r.trk.Visit(func(id pb.PeerID, _ *tracker.Progress) { r.maybeSendFortify(id) }) @@ -930,7 +907,7 @@ func (r *raft) appliedTo(index uint64, size entryEncodingSize) { newApplied := max(index, oldApplied) r.raftLog.appliedTo(newApplied, size) - if r.config.AutoLeave && newApplied >= r.pendingConfIndex && r.state == StateLeader { + if r.config.AutoLeave && newApplied >= r.pendingConfIndex && r.state == pb.StateLeader { // If the current (and most recent, at least for this leader's term) // configuration should be auto-left, initiate that now. We use a // nil Data which unmarshals into an empty ConfChangeV2 and has the @@ -1070,7 +1047,7 @@ func (r *raft) appendEntry(es ...pb.Entry) (accepted bool) { // tickElection is run by followers and candidates after r.electionTimeout. func (r *raft) tickElection() { - assertTrue(r.state != StateLeader, "tickElection called by leader") + assertTrue(r.state != pb.StateLeader, "tickElection called by leader") if r.leadEpoch != 0 { if r.supportingFortifiedLeader() { @@ -1113,7 +1090,7 @@ func (r *raft) tickElection() { // tickHeartbeat is run by leaders to send a MsgBeat after r.heartbeatTimeout. func (r *raft) tickHeartbeat() { - assertTrue(r.state == StateLeader, "tickHeartbeat called by non-leader") + assertTrue(r.state == pb.StateLeader, "tickHeartbeat called by non-leader") r.heartbeatElapsed++ r.electionElapsed++ @@ -1123,7 +1100,7 @@ func (r *raft) tickHeartbeat() { if r.checkQuorum { if err := r.Step(pb.Message{From: r.id, Type: pb.MsgCheckQuorum}); err != nil { r.logger.Debugf("error occurred during checking sending heartbeat: %v", err) - } else if r.state != StateLeader { + } else if r.state != pb.StateLeader { return // stepped down } } @@ -1168,26 +1145,26 @@ func (r *raft) becomeFollower(term uint64, lead pb.PeerID) { r.reset(term) r.tick = r.tickElection r.lead = lead - r.state = StateFollower + r.state = pb.StateFollower r.logger.Infof("%x became follower at term %d", r.id, r.Term) } func (r *raft) becomeCandidate() { // TODO(xiangli) remove the panic when the raft implementation is stable - if r.state == StateLeader { + if r.state == pb.StateLeader { panic("invalid transition [leader -> candidate]") } r.step = stepCandidate r.reset(r.Term + 1) r.tick = r.tickElection r.Vote = r.id - r.state = StateCandidate + r.state = pb.StateCandidate r.logger.Infof("%x became candidate at term %d", r.id, r.Term) } func (r *raft) becomePreCandidate() { // TODO(xiangli) remove the panic when the raft implementation is stable - if r.state == StateLeader { + if r.state == pb.StateLeader { panic("invalid transition [leader -> pre-candidate]") } assertTrue(!r.supportingFortifiedLeader() || r.lead == r.id, @@ -1205,13 +1182,13 @@ func (r *raft) becomePreCandidate() { // a bit weird from the perspective of raft though. See if we can avoid this. r.lead = None r.leadEpoch = 0 - r.state = StatePreCandidate + r.state = pb.StatePreCandidate r.logger.Infof("%x became pre-candidate at term %d", r.id, r.Term) } func (r *raft) becomeLeader() { // TODO(xiangli) remove the panic when the raft implementation is stable - if r.state == StateFollower { + if r.state == pb.StateFollower { panic("invalid transition [follower -> leader]") } r.step = stepLeader @@ -1223,7 +1200,7 @@ func (r *raft) becomeLeader() { r.fortificationTracker.Reset(r.Term) r.tick = r.tickHeartbeat r.lead = r.id - r.state = StateLeader + r.state = pb.StateLeader // Followers enter replicate mode when they've been successfully probed // (perhaps after having received a snapshot as a result). The leader is // trivially in this state. Note that r.reset() has initialized this @@ -1254,7 +1231,7 @@ func (r *raft) becomeLeader() { } func (r *raft) hup(t CampaignType) { - if r.state == StateLeader { + if r.state == pb.StateLeader { r.logger.Debugf("%x ignoring MsgHup because already leader", r.id) return } @@ -1971,7 +1948,7 @@ func stepCandidate(r *raft, m pb.Message) error { // StateCandidate, we may get stale MsgPreVoteResp messages in this term from // our pre-candidate state). var myVoteRespType pb.MessageType - if r.state == StatePreCandidate { + if r.state == pb.StatePreCandidate { myVoteRespType = pb.MsgPreVoteResp } else { myVoteRespType = pb.MsgVoteResp @@ -1997,7 +1974,7 @@ func stepCandidate(r *raft, m pb.Message) error { r.logger.Infof("%x has received %d %s votes and %d vote rejections", r.id, gr, m.Type, rj) switch res { case quorum.VoteWon: - if r.state == StatePreCandidate { + if r.state == pb.StatePreCandidate { r.campaign(campaignElection) } else { r.becomeLeader() @@ -2252,7 +2229,7 @@ func (r *raft) handleSnapshot(m pb.Message) { } func (r *raft) handleFortify(m pb.Message) { - assertTrue(r.state == StateFollower, "leaders should locally fortify without sending a message") + assertTrue(r.state == pb.StateFollower, "leaders should locally fortify without sending a message") assertTrue(r.lead == m.From, "only the leader should send fortification requests") epoch, live := r.storeLiveness.SupportFor(r.lead) @@ -2275,7 +2252,7 @@ func (r *raft) handleFortify(m pb.Message) { } func (r *raft) handleFortifyResp(m pb.Message) { - assertTrue(r.state == StateLeader, "only leaders should be handling fortification responses") + assertTrue(r.state == pb.StateLeader, "only leaders should be handling fortification responses") if m.Reject { // Couldn't successfully fortify the follower. Typically, this happens when // the follower isn't supporting the leader's store in StoreLiveness or the @@ -2287,7 +2264,7 @@ func (r *raft) handleFortifyResp(m pb.Message) { } func (r *raft) handleDeFortify(m pb.Message) { - assertTrue(r.state != StateLeader, "leaders should locally de-fortify without sending a message") + assertTrue(r.state != pb.StateLeader, "leaders should locally de-fortify without sending a message") assertTrue(r.lead == m.From, "only the leader should send de-fortification requests") if r.leadEpoch == 0 { @@ -2322,7 +2299,7 @@ func (r *raft) restore(s snapshot) bool { if id.index <= r.raftLog.committed { return false } - if r.state != StateFollower { + if r.state != pb.StateFollower { // This is defense-in-depth: if the leader somehow ended up applying a // snapshot, it could move into a new term without moving into a // follower state. This should never fire, but if it does, we panic. @@ -2453,7 +2430,7 @@ func (r *raft) switchToConfig(cfg quorum.Config, progressMap tracker.ProgressMap // node is removed. r.isLearner = pr != nil && pr.IsLearner - if (pr == nil || r.isLearner) && r.state == StateLeader { + if (pr == nil || r.isLearner) && r.state == pb.StateLeader { // This node is leader and was removed or demoted, step down. // // We prevent demotions at the time writing but hypothetically we handle @@ -2471,7 +2448,7 @@ func (r *raft) switchToConfig(cfg quorum.Config, progressMap tracker.ProgressMap // The remaining steps only make sense if this node is the leader and there // are other nodes. - if r.state != StateLeader || len(cs.Voters) == 0 { + if r.state != pb.StateLeader || len(cs.Voters) == 0 { return cs } @@ -2518,7 +2495,7 @@ func (r *raft) resetRandomizedElectionTimeout() { } func (r *raft) transferLeader(to pb.PeerID) { - assertTrue(r.state == StateLeader, "only the leader can transfer leadership") + assertTrue(r.state == pb.StateLeader, "only the leader can transfer leadership") r.send(pb.Message{To: to, Type: pb.MsgTimeoutNow}) r.becomeFollower(r.Term, r.lead) } diff --git a/pkg/raft/raft_paper_test.go b/pkg/raft/raft_paper_test.go index 095c69cc128f..067eb94fad94 100644 --- a/pkg/raft/raft_paper_test.go +++ b/pkg/raft/raft_paper_test.go @@ -43,13 +43,13 @@ import ( ) func TestFollowerUpdateTermFromMessage(t *testing.T) { - testUpdateTermFromMessage(t, StateFollower) + testUpdateTermFromMessage(t, pb.StateFollower) } func TestCandidateUpdateTermFromMessage(t *testing.T) { - testUpdateTermFromMessage(t, StateCandidate) + testUpdateTermFromMessage(t, pb.StateCandidate) } func TestLeaderUpdateTermFromMessage(t *testing.T) { - testUpdateTermFromMessage(t, StateLeader) + testUpdateTermFromMessage(t, pb.StateLeader) } // testUpdateTermFromMessage tests that if one server’s current term is @@ -57,14 +57,14 @@ func TestLeaderUpdateTermFromMessage(t *testing.T) { // value. If a candidate or leader discovers that its term is out of date, // it immediately reverts to follower state. // Reference: section 5.1 -func testUpdateTermFromMessage(t *testing.T, state StateType) { +func testUpdateTermFromMessage(t *testing.T, state pb.StateType) { r := newTestRaft(1, 10, 1, newTestMemoryStorage(withPeers(1, 2, 3))) switch state { - case StateFollower: + case pb.StateFollower: r.becomeFollower(1, 2) - case StateCandidate: + case pb.StateCandidate: r.becomeCandidate() - case StateLeader: + case pb.StateLeader: r.becomeCandidate() r.becomeLeader() } @@ -72,7 +72,7 @@ func testUpdateTermFromMessage(t *testing.T, state StateType) { r.Step(pb.Message{Type: pb.MsgApp, Term: 2}) assert.Equal(t, uint64(2), r.Term) - assert.Equal(t, StateFollower, r.state) + assert.Equal(t, pb.StateFollower, r.state) } // TestRejectStaleTermMessage tests that if a server receives a request with @@ -98,7 +98,7 @@ func TestRejectStaleTermMessage(t *testing.T) { // Reference: section 5.2 func TestStartAsFollower(t *testing.T) { r := newTestRaft(1, 10, 1, newTestMemoryStorage(withPeers(1, 2, 3))) - assert.Equal(t, StateFollower, r.state) + assert.Equal(t, pb.StateFollower, r.state) } // TestLeaderBcastBeat tests that if the leader receives a heartbeat tick, @@ -152,10 +152,10 @@ func TestLeaderBcastBeat(t *testing.T) { } func TestFollowerStartElection(t *testing.T) { - testNonleaderStartElection(t, StateFollower) + testNonleaderStartElection(t, pb.StateFollower) } func TestCandidateStartNewElection(t *testing.T) { - testNonleaderStartElection(t, StateCandidate) + testNonleaderStartElection(t, pb.StateCandidate) } // testNonleaderStartElection tests that if a follower receives no communication @@ -168,14 +168,14 @@ func TestCandidateStartNewElection(t *testing.T) { // start a new election by incrementing its term and initiating another // round of RequestVote RPCs. // Reference: section 5.2 -func testNonleaderStartElection(t *testing.T, state StateType) { +func testNonleaderStartElection(t *testing.T, state pb.StateType) { // election timeout et := 10 r := newTestRaft(1, et, 1, newTestMemoryStorage(withPeers(1, 2, 3))) switch state { - case StateFollower: + case pb.StateFollower: r.becomeFollower(1, 2) - case StateCandidate: + case pb.StateCandidate: r.becomeCandidate() } @@ -185,7 +185,7 @@ func testNonleaderStartElection(t *testing.T, state StateType) { r.advanceMessagesAfterAppend() assert.Equal(t, uint64(2), r.Term) - assert.Equal(t, StateCandidate, r.state) + assert.Equal(t, pb.StateCandidate, r.state) assert.True(t, r.electionTracker.TestingGetVotes()[r.id]) msgs := r.readMessages() @@ -206,26 +206,26 @@ func TestLeaderElectionInOneRoundRPC(t *testing.T) { tests := []struct { size int votes map[pb.PeerID]bool - state StateType + state pb.StateType }{ // win the election when receiving votes from a majority of the servers - {1, map[pb.PeerID]bool{}, StateLeader}, - {3, map[pb.PeerID]bool{2: true, 3: true}, StateLeader}, - {3, map[pb.PeerID]bool{2: true}, StateLeader}, - {5, map[pb.PeerID]bool{2: true, 3: true, 4: true, 5: true}, StateLeader}, - {5, map[pb.PeerID]bool{2: true, 3: true, 4: true}, StateLeader}, - {5, map[pb.PeerID]bool{2: true, 3: true}, StateLeader}, + {1, map[pb.PeerID]bool{}, pb.StateLeader}, + {3, map[pb.PeerID]bool{2: true, 3: true}, pb.StateLeader}, + {3, map[pb.PeerID]bool{2: true}, pb.StateLeader}, + {5, map[pb.PeerID]bool{2: true, 3: true, 4: true, 5: true}, pb.StateLeader}, + {5, map[pb.PeerID]bool{2: true, 3: true, 4: true}, pb.StateLeader}, + {5, map[pb.PeerID]bool{2: true, 3: true}, pb.StateLeader}, // return to follower state if it receives vote denial from a majority - {3, map[pb.PeerID]bool{2: false, 3: false}, StateFollower}, - {5, map[pb.PeerID]bool{2: false, 3: false, 4: false, 5: false}, StateFollower}, - {5, map[pb.PeerID]bool{2: true, 3: false, 4: false, 5: false}, StateFollower}, + {3, map[pb.PeerID]bool{2: false, 3: false}, pb.StateFollower}, + {5, map[pb.PeerID]bool{2: false, 3: false, 4: false, 5: false}, pb.StateFollower}, + {5, map[pb.PeerID]bool{2: true, 3: false, 4: false, 5: false}, pb.StateFollower}, // stay in candidate if it does not obtain the majority - {3, map[pb.PeerID]bool{}, StateCandidate}, - {5, map[pb.PeerID]bool{2: true}, StateCandidate}, - {5, map[pb.PeerID]bool{2: false, 3: false}, StateCandidate}, - {5, map[pb.PeerID]bool{}, StateCandidate}, + {3, map[pb.PeerID]bool{}, pb.StateCandidate}, + {5, map[pb.PeerID]bool{2: true}, pb.StateCandidate}, + {5, map[pb.PeerID]bool{2: false, 3: false}, pb.StateCandidate}, + {5, map[pb.PeerID]bool{}, pb.StateCandidate}, } for i, tt := range tests { r := newTestRaft(1, 10, 1, newTestMemoryStorage(withPeers(idsBySize(tt.size)...))) @@ -282,11 +282,11 @@ func TestCandidateFallback(t *testing.T) { for i, tt := range tests { r := newTestRaft(1, 10, 1, newTestMemoryStorage(withPeers(1, 2, 3))) r.Step(pb.Message{From: 1, To: 1, Type: pb.MsgHup}) - require.Equal(t, StateCandidate, r.state, "#%d", i) + require.Equal(t, pb.StateCandidate, r.state, "#%d", i) r.Step(tt) - assert.Equal(t, StateFollower, r.state, "#%d", i) + assert.Equal(t, pb.StateFollower, r.state, "#%d", i) assert.Equal(t, tt.Term, r.Term, "#%d", i) } } @@ -294,26 +294,26 @@ func TestCandidateFallback(t *testing.T) { func TestFollowerElectionTimeoutRandomized(t *testing.T) { raftlogger.SetLogger(raftlogger.DiscardLogger) defer raftlogger.SetLogger(raftlogger.DefaultRaftLogger) - testNonleaderElectionTimeoutRandomized(t, StateFollower) + testNonleaderElectionTimeoutRandomized(t, pb.StateFollower) } func TestCandidateElectionTimeoutRandomized(t *testing.T) { raftlogger.SetLogger(raftlogger.DiscardLogger) defer raftlogger.SetLogger(raftlogger.DefaultRaftLogger) - testNonleaderElectionTimeoutRandomized(t, StateCandidate) + testNonleaderElectionTimeoutRandomized(t, pb.StateCandidate) } // testNonleaderElectionTimeoutRandomized tests that election timeout for // follower or candidate is randomized. // Reference: section 5.2 -func testNonleaderElectionTimeoutRandomized(t *testing.T, state StateType) { +func testNonleaderElectionTimeoutRandomized(t *testing.T, state pb.StateType) { et := 10 r := newTestRaft(1, et, 1, newTestMemoryStorage(withPeers(1, 2, 3))) timeouts := make(map[int]bool) for round := 0; round < 50*et; round++ { switch state { - case StateFollower: + case pb.StateFollower: r.becomeFollower(r.Term+1, 2) - case StateCandidate: + case pb.StateCandidate: r.becomeCandidate() } @@ -333,19 +333,19 @@ func testNonleaderElectionTimeoutRandomized(t *testing.T, state StateType) { func TestFollowersElectionTimeoutNonconflict(t *testing.T) { raftlogger.SetLogger(raftlogger.DiscardLogger) defer raftlogger.SetLogger(raftlogger.DefaultRaftLogger) - testNonleadersElectionTimeoutNonconflict(t, StateFollower) + testNonleadersElectionTimeoutNonconflict(t, pb.StateFollower) } func TestCandidatesElectionTimeoutNonconflict(t *testing.T) { raftlogger.SetLogger(raftlogger.DiscardLogger) defer raftlogger.SetLogger(raftlogger.DefaultRaftLogger) - testNonleadersElectionTimeoutNonconflict(t, StateCandidate) + testNonleadersElectionTimeoutNonconflict(t, pb.StateCandidate) } // testNonleadersElectionTimeoutNonconflict tests that in most cases only a // single server(follower or candidate) will time out, which reduces the // likelihood of split vote in the new election. // Reference: section 5.2 -func testNonleadersElectionTimeoutNonconflict(t *testing.T, state StateType) { +func testNonleadersElectionTimeoutNonconflict(t *testing.T, state pb.StateType) { et := 10 size := 5 rs := make([]*raft, size) @@ -357,9 +357,9 @@ func testNonleadersElectionTimeoutNonconflict(t *testing.T, state StateType) { for round := 0; round < 1000; round++ { for _, r := range rs { switch state { - case StateFollower: + case pb.StateFollower: r.becomeFollower(r.Term+1, None) - case StateCandidate: + case pb.StateCandidate: r.becomeCandidate() } } @@ -807,7 +807,7 @@ func cmpMessages(a, b pb.Message) int { } func commitNoopEntry(r *raft, s *MemoryStorage) { - if r.state != StateLeader { + if r.state != pb.StateLeader { panic("it should only be used when it is the leader") } r.bcastAppend() diff --git a/pkg/raft/raft_test.go b/pkg/raft/raft_test.go index aeb2404bbf16..dc0e0b0cc8a8 100644 --- a/pkg/raft/raft_test.go +++ b/pkg/raft/raft_test.go @@ -299,32 +299,32 @@ func TestLeaderElectionPreVote(t *testing.T) { func testLeaderElection(t *testing.T, preVote bool) { var cfg func(*Config) - candState := StateCandidate + candState := pb.StateCandidate candTerm := uint64(1) if preVote { cfg = preVoteConfig // In pre-vote mode, an election that fails to complete // leaves the node in pre-candidate state without advancing // the term. - candState = StatePreCandidate + candState = pb.StatePreCandidate candTerm = 0 } tests := []struct { *network - state StateType + state pb.StateType expTerm uint64 }{ - {newNetworkWithConfig(cfg, nil, nil, nil), StateLeader, 1}, - {newNetworkWithConfig(cfg, nil, nil, nopStepper), StateLeader, 1}, + {newNetworkWithConfig(cfg, nil, nil, nil), pb.StateLeader, 1}, + {newNetworkWithConfig(cfg, nil, nil, nopStepper), pb.StateLeader, 1}, {newNetworkWithConfig(cfg, nil, nopStepper, nopStepper), candState, candTerm}, {newNetworkWithConfig(cfg, nil, nopStepper, nopStepper, nil), candState, candTerm}, - {newNetworkWithConfig(cfg, nil, nopStepper, nopStepper, nil, nil), StateLeader, 1}, + {newNetworkWithConfig(cfg, nil, nopStepper, nopStepper, nil, nil), pb.StateLeader, 1}, // three logs further along than 0, but in the same term so rejections // are returned instead of the votes being ignored. {newNetworkWithConfig(cfg, nil, entsWithConfig(cfg, 1), entsWithConfig(cfg, 1), entsWithConfig(cfg, 1, 1), nil), - StateFollower, 1}, + pb.StateFollower, 1}, } for i, tt := range tests { @@ -350,7 +350,7 @@ func TestLearnerElectionTimeout(t *testing.T) { n2.tick() } - assert.Equal(t, StateFollower, n2.state) + assert.Equal(t, pb.StateFollower, n2.state) } // TestLearnerPromotion verifies that the learner should not election until @@ -365,7 +365,7 @@ func TestLearnerPromotion(t *testing.T) { nt := newNetwork(n1, n2) - assert.NotEqual(t, StateLeader, n1.state) + assert.NotEqual(t, pb.StateLeader, n1.state) // n1 should become leader setRandomizedElectionTimeout(n1, n1.electionTimeout) @@ -374,8 +374,8 @@ func TestLearnerPromotion(t *testing.T) { } n1.advanceMessagesAfterAppend() - assert.Equal(t, StateLeader, n1.state) - assert.Equal(t, StateFollower, n2.state) + assert.Equal(t, pb.StateLeader, n1.state) + assert.Equal(t, pb.StateFollower, n2.state) nt.send(pb.Message{From: 1, To: 1, Type: pb.MsgBeat}) @@ -392,8 +392,8 @@ func TestLearnerPromotion(t *testing.T) { nt.send(pb.Message{From: 2, To: 2, Type: pb.MsgBeat}) - assert.Equal(t, StateFollower, n1.state) - assert.Equal(t, StateLeader, n2.state) + assert.Equal(t, pb.StateFollower, n1.state) + assert.Equal(t, pb.StateLeader, n2.state) } // TestLearnerCanVote checks that a learner can vote when it receives a valid Vote request. @@ -435,9 +435,9 @@ func testLeaderCycle(t *testing.T, preVote bool) { for _, peer := range n.peers { sm := peer.(*raft) if sm.id == campaignerID { - assert.Equal(t, StateLeader, sm.state, "preVote=%v: campaigning node %d", preVote, sm.id) + assert.Equal(t, pb.StateLeader, sm.state, "preVote=%v: campaigning node %d", preVote, sm.id) } else { - assert.Equal(t, StateFollower, sm.state, "preVote=%v: campaigning node %d, current node %d", preVote, campaignerID, sm.id) + assert.Equal(t, pb.StateFollower, sm.state, "preVote=%v: campaigning node %d, current node %d", preVote, campaignerID, sm.id) } } } @@ -486,12 +486,12 @@ func testLeaderElectionOverwriteNewerLogs(t *testing.T, preVote bool) { // term is pushed ahead to 2. n.send(pb.Message{From: 1, To: 1, Type: pb.MsgHup}) sm1 := n.peers[1].(*raft) - assert.Equal(t, StateFollower, sm1.state) + assert.Equal(t, pb.StateFollower, sm1.state) assert.Equal(t, uint64(2), sm1.Term) // Node 1 campaigns again with a higher term. This time it succeeds. n.send(pb.Message{From: 1, To: 1, Type: pb.MsgHup}) - assert.Equal(t, StateLeader, sm1.state) + assert.Equal(t, pb.StateLeader, sm1.state) assert.Equal(t, uint64(3), sm1.Term) // Now all nodes agree on a log entry with term 1 at index 1 (and @@ -514,18 +514,18 @@ func TestPreVoteFromAnyState(t *testing.T) { } func testVoteFromAnyState(t *testing.T, vt pb.MessageType) { - for st := StateType(0); st < numStates; st++ { + for st := pb.StateType(0); st < pb.NumStates; st++ { r := newTestRaft(1, 10, 1, newTestMemoryStorage(withPeers(1, 2, 3))) r.Term = 1 switch st { - case StateFollower: + case pb.StateFollower: r.becomeFollower(r.Term, 3) - case StatePreCandidate: + case pb.StatePreCandidate: r.becomePreCandidate() - case StateCandidate: + case pb.StateCandidate: r.becomeCandidate() - case StateLeader: + case pb.StateLeader: r.becomeCandidate() r.becomeLeader() } @@ -553,7 +553,7 @@ func testVoteFromAnyState(t *testing.T, vt pb.MessageType) { // If this was a real vote, we reset our state and term. if vt == pb.MsgVote { - assert.Equal(t, StateFollower, r.state, "%s,%s", vt, st) + assert.Equal(t, pb.StateFollower, r.state, "%s,%s", vt, st) assert.Equal(t, newTerm, r.Term, "%s,%s", vt, st) assert.Equal(t, pb.PeerID(2), r.Vote, "%s,%s", vt, st) } else { @@ -643,7 +643,7 @@ func TestLearnerLogReplication(t *testing.T) { nt.send(pb.Message{From: 1, To: 1, Type: pb.MsgBeat}) // n1 is leader and n2 is learner - assert.Equal(t, StateLeader, n1.state) + assert.Equal(t, pb.StateLeader, n1.state) assert.True(t, n2.isLearner) nextCommitted := uint64(2) @@ -754,11 +754,11 @@ func TestDuelingCandidates(t *testing.T) { // 1 becomes leader since it receives votes from 1 and 2 sm := nt.peers[1].(*raft) - assert.Equal(t, StateLeader, sm.state) + assert.Equal(t, pb.StateLeader, sm.state) // 3 stays as candidate since it receives a vote from 3 and a rejection from 2 sm = nt.peers[3].(*raft) - assert.Equal(t, StateCandidate, sm.state) + assert.Equal(t, pb.StateCandidate, sm.state) nt.recover() @@ -766,17 +766,17 @@ func TestDuelingCandidates(t *testing.T) { // we expect it to disrupt the leader 1 since it has a higher term // 3 will be follower again since both 1 and 2 rejects its vote request since 3 does not have a long enough log nt.send(pb.Message{From: 3, To: 3, Type: pb.MsgHup}) - assert.Equal(t, StateFollower, sm.state) + assert.Equal(t, pb.StateFollower, sm.state) tests := []struct { sm *raft - state StateType + state pb.StateType term uint64 lastIndex uint64 }{ - {a, StateFollower, 2, 1}, - {b, StateFollower, 2, 1}, - {c, StateFollower, 2, 0}, + {a, pb.StateFollower, 2, 1}, + {b, pb.StateFollower, 2, 1}, + {c, pb.StateFollower, 2, 0}, } for i, tt := range tests { @@ -806,11 +806,11 @@ func TestDuelingPreCandidates(t *testing.T) { // 1 becomes leader since it receives votes from 1 and 2 sm := nt.peers[1].(*raft) - assert.Equal(t, StateLeader, sm.state) + assert.Equal(t, pb.StateLeader, sm.state) // 3 campaigns then reverts to follower when its PreVote is rejected sm = nt.peers[3].(*raft) - assert.Equal(t, StateFollower, sm.state) + assert.Equal(t, pb.StateFollower, sm.state) nt.recover() @@ -820,13 +820,13 @@ func TestDuelingPreCandidates(t *testing.T) { tests := []struct { sm *raft - state StateType + state pb.StateType term uint64 lastIndex uint64 }{ - {a, StateLeader, 1, 1}, - {b, StateFollower, 1, 1}, - {c, StateFollower, 1, 0}, + {a, pb.StateLeader, 1, 1}, + {b, pb.StateFollower, 1, 1}, + {c, pb.StateFollower, 1, 0}, } for i, tt := range tests { @@ -860,7 +860,7 @@ func TestCandidateConcede(t *testing.T) { } a := tt.peers[1].(*raft) - assert.Equal(t, StateFollower, a.state) + assert.Equal(t, pb.StateFollower, a.state) assert.Equal(t, uint64(1), a.Term) wantLog := ltoa(newLog(&MemoryStorage{ @@ -881,7 +881,7 @@ func TestSingleNodeCandidate(t *testing.T) { tt.send(pb.Message{From: 1, To: 1, Type: pb.MsgHup}) sm := tt.peers[1].(*raft) - assert.Equal(t, StateLeader, sm.state) + assert.Equal(t, pb.StateLeader, sm.state) } func TestSingleNodePreCandidate(t *testing.T) { @@ -889,7 +889,7 @@ func TestSingleNodePreCandidate(t *testing.T) { tt.send(pb.Message{From: 1, To: 1, Type: pb.MsgHup}) sm := tt.peers[1].(*raft) - assert.Equal(t, StateLeader, sm.state) + assert.Equal(t, pb.StateLeader, sm.state) } func TestOldMessages(t *testing.T) { @@ -1325,48 +1325,48 @@ func TestRecvMsgPreVote(t *testing.T) { func testRecvMsgVote(t *testing.T, msgType pb.MessageType) { tests := []struct { - state StateType + state pb.StateType index, logTerm uint64 voteFor pb.PeerID wreject bool }{ - {StateFollower, 0, 0, None, true}, - {StateFollower, 0, 1, None, true}, - {StateFollower, 0, 2, None, true}, - {StateFollower, 0, 3, None, false}, + {pb.StateFollower, 0, 0, None, true}, + {pb.StateFollower, 0, 1, None, true}, + {pb.StateFollower, 0, 2, None, true}, + {pb.StateFollower, 0, 3, None, false}, - {StateFollower, 1, 0, None, true}, - {StateFollower, 1, 1, None, true}, - {StateFollower, 1, 2, None, true}, - {StateFollower, 1, 3, None, false}, + {pb.StateFollower, 1, 0, None, true}, + {pb.StateFollower, 1, 1, None, true}, + {pb.StateFollower, 1, 2, None, true}, + {pb.StateFollower, 1, 3, None, false}, - {StateFollower, 2, 0, None, true}, - {StateFollower, 2, 1, None, true}, - {StateFollower, 2, 2, None, false}, - {StateFollower, 2, 3, None, false}, + {pb.StateFollower, 2, 0, None, true}, + {pb.StateFollower, 2, 1, None, true}, + {pb.StateFollower, 2, 2, None, false}, + {pb.StateFollower, 2, 3, None, false}, - {StateFollower, 3, 0, None, true}, - {StateFollower, 3, 1, None, true}, - {StateFollower, 3, 2, None, false}, - {StateFollower, 3, 3, None, false}, + {pb.StateFollower, 3, 0, None, true}, + {pb.StateFollower, 3, 1, None, true}, + {pb.StateFollower, 3, 2, None, false}, + {pb.StateFollower, 3, 3, None, false}, - {StateFollower, 3, 2, 2, false}, - {StateFollower, 3, 2, 1, true}, + {pb.StateFollower, 3, 2, 2, false}, + {pb.StateFollower, 3, 2, 1, true}, - {StateLeader, 3, 3, 1, true}, - {StatePreCandidate, 3, 3, 1, true}, - {StateCandidate, 3, 3, 1, true}, + {pb.StateLeader, 3, 3, 1, true}, + {pb.StatePreCandidate, 3, 3, 1, true}, + {pb.StateCandidate, 3, 3, 1, true}, } for i, tt := range tests { sm := newTestRaft(1, 10, 1, newTestMemoryStorage(withPeers(1))) sm.state = tt.state switch tt.state { - case StateFollower: + case pb.StateFollower: sm.step = stepFollower - case StateCandidate, StatePreCandidate: + case pb.StateCandidate, pb.StatePreCandidate: sm.step = stepCandidate - case StateLeader: + case pb.StateLeader: sm.step = stepLeader } sm.Vote = tt.voteFor @@ -1393,31 +1393,31 @@ func testRecvMsgVote(t *testing.T, msgType pb.MessageType) { func TestStateTransition(t *testing.T) { tests := []struct { - from StateType - to StateType + from pb.StateType + to pb.StateType wallow bool wterm uint64 wlead pb.PeerID }{ - {StateFollower, StateFollower, true, 1, None}, - {StateFollower, StatePreCandidate, true, 0, None}, - {StateFollower, StateCandidate, true, 1, None}, - {StateFollower, StateLeader, false, 0, None}, + {pb.StateFollower, pb.StateFollower, true, 1, None}, + {pb.StateFollower, pb.StatePreCandidate, true, 0, None}, + {pb.StateFollower, pb.StateCandidate, true, 1, None}, + {pb.StateFollower, pb.StateLeader, false, 0, None}, - {StatePreCandidate, StateFollower, true, 0, None}, - {StatePreCandidate, StatePreCandidate, true, 0, None}, - {StatePreCandidate, StateCandidate, true, 1, None}, - {StatePreCandidate, StateLeader, true, 0, 1}, + {pb.StatePreCandidate, pb.StateFollower, true, 0, None}, + {pb.StatePreCandidate, pb.StatePreCandidate, true, 0, None}, + {pb.StatePreCandidate, pb.StateCandidate, true, 1, None}, + {pb.StatePreCandidate, pb.StateLeader, true, 0, 1}, - {StateCandidate, StateFollower, true, 0, None}, - {StateCandidate, StatePreCandidate, true, 0, None}, - {StateCandidate, StateCandidate, true, 1, None}, - {StateCandidate, StateLeader, true, 0, 1}, + {pb.StateCandidate, pb.StateFollower, true, 0, None}, + {pb.StateCandidate, pb.StatePreCandidate, true, 0, None}, + {pb.StateCandidate, pb.StateCandidate, true, 1, None}, + {pb.StateCandidate, pb.StateLeader, true, 0, 1}, - {StateLeader, StateFollower, true, 1, None}, - {StateLeader, StatePreCandidate, false, 0, None}, - {StateLeader, StateCandidate, false, 1, None}, - {StateLeader, StateLeader, true, 0, 1}, + {pb.StateLeader, pb.StateFollower, true, 1, None}, + {pb.StateLeader, pb.StatePreCandidate, false, 0, None}, + {pb.StateLeader, pb.StateCandidate, false, 1, None}, + {pb.StateLeader, pb.StateLeader, true, 0, 1}, } for i, tt := range tests { @@ -1432,13 +1432,13 @@ func TestStateTransition(t *testing.T) { sm.state = tt.from switch tt.to { - case StateFollower: + case pb.StateFollower: sm.becomeFollower(tt.wterm, tt.wlead) - case StatePreCandidate: + case pb.StatePreCandidate: sm.becomePreCandidate() - case StateCandidate: + case pb.StateCandidate: sm.becomeCandidate() - case StateLeader: + case pb.StateLeader: sm.becomeLeader() } @@ -1450,16 +1450,16 @@ func TestStateTransition(t *testing.T) { func TestAllServerStepdown(t *testing.T) { tests := []struct { - state StateType + state pb.StateType - wstate StateType + wstate pb.StateType wterm uint64 windex uint64 }{ - {StateFollower, StateFollower, 3, 0}, - {StatePreCandidate, StateFollower, 3, 0}, - {StateCandidate, StateFollower, 3, 0}, - {StateLeader, StateFollower, 3, 1}, + {pb.StateFollower, pb.StateFollower, 3, 0}, + {pb.StatePreCandidate, pb.StateFollower, 3, 0}, + {pb.StateCandidate, pb.StateFollower, 3, 0}, + {pb.StateLeader, pb.StateFollower, 3, 1}, } tmsgTypes := [...]pb.MessageType{pb.MsgVote, pb.MsgApp} @@ -1468,13 +1468,13 @@ func TestAllServerStepdown(t *testing.T) { for i, tt := range tests { sm := newTestRaft(1, 10, 1, newTestMemoryStorage(withPeers(1, 2, 3))) switch tt.state { - case StateFollower: + case pb.StateFollower: sm.becomeFollower(1, None) - case StatePreCandidate: + case pb.StatePreCandidate: sm.becomePreCandidate() - case StateCandidate: + case pb.StateCandidate: sm.becomeCandidate() - case StateLeader: + case pb.StateLeader: sm.becomeCandidate() sm.becomeLeader() } @@ -1522,9 +1522,9 @@ func testCandidateResetTerm(t *testing.T, mt pb.MessageType) { nt.send(pb.Message{From: 1, To: 1, Type: pb.MsgHup}) - assert.Equal(t, StateLeader, a.state) - assert.Equal(t, StateFollower, b.state) - assert.Equal(t, StateFollower, c.state) + assert.Equal(t, pb.StateLeader, a.state) + assert.Equal(t, pb.StateFollower, b.state) + assert.Equal(t, pb.StateFollower, c.state) // isolate 3 and increase term in rest nt.isolate(3) @@ -1532,8 +1532,8 @@ func testCandidateResetTerm(t *testing.T, mt pb.MessageType) { nt.send(pb.Message{From: 2, To: 2, Type: pb.MsgHup}) nt.send(pb.Message{From: 1, To: 1, Type: pb.MsgHup}) - assert.Equal(t, StateLeader, a.state) - assert.Equal(t, StateFollower, b.state) + assert.Equal(t, pb.StateLeader, a.state) + assert.Equal(t, pb.StateFollower, b.state) // trigger campaign in isolated c c.resetRandomizedElectionTimeout() @@ -1542,7 +1542,7 @@ func testCandidateResetTerm(t *testing.T, mt pb.MessageType) { } c.advanceMessagesAfterAppend() - assert.Equal(t, StateCandidate, c.state) + assert.Equal(t, pb.StateCandidate, c.state) nt.recover() @@ -1550,7 +1550,7 @@ func testCandidateResetTerm(t *testing.T, mt pb.MessageType) { // and expects candidate to revert to follower nt.send(pb.Message{From: 1, To: 3, Term: a.Term, Type: mt}) - assert.Equal(t, StateFollower, c.state) + assert.Equal(t, pb.StateFollower, c.state) // follower c term is reset with leader's assert.Equal(t, a.Term, c.Term) @@ -1580,11 +1580,11 @@ func testCandidateSelfVoteAfterLostElection(t *testing.T, preVote bool) { // change to sync its vote to disk and account for its self-vote. // Becomes a follower. sm.Step(pb.Message{From: 2, To: 1, Term: sm.Term, Type: pb.MsgHeartbeat}) - assert.Equal(t, StateFollower, sm.state) + assert.Equal(t, pb.StateFollower, sm.state) // n1 remains a follower even after its self-vote is delivered. sm.stepOrSend(steps) - assert.Equal(t, StateFollower, sm.state) + assert.Equal(t, pb.StateFollower, sm.state) // Its self-vote does not make its way to its ProgressTracker. granted, _, _ := sm.electionTracker.TallyVotes() @@ -1598,7 +1598,7 @@ func TestCandidateDeliversPreCandidateSelfVoteAfterBecomingCandidate(t *testing. // n1 calls an election. sm.Step(pb.Message{From: 1, To: 1, Type: pb.MsgHup}) - assert.Equal(t, StatePreCandidate, sm.state) + assert.Equal(t, pb.StatePreCandidate, sm.state) steps := sm.takeMessagesAfterAppend() // n1 receives pre-candidate votes from both other peers before @@ -1606,12 +1606,12 @@ func TestCandidateDeliversPreCandidateSelfVoteAfterBecomingCandidate(t *testing. // NB: pre-vote messages carry the local term + 1. sm.Step(pb.Message{From: 2, To: 1, Term: sm.Term + 1, Type: pb.MsgPreVoteResp}) sm.Step(pb.Message{From: 3, To: 1, Term: sm.Term + 1, Type: pb.MsgPreVoteResp}) - assert.Equal(t, StateCandidate, sm.state) + assert.Equal(t, pb.StateCandidate, sm.state) // n1 remains a candidate even after its delayed pre-vote self-vote is // delivered. sm.stepOrSend(steps) - assert.Equal(t, StateCandidate, sm.state) + assert.Equal(t, pb.StateCandidate, sm.state) steps = sm.takeMessagesAfterAppend() @@ -1621,12 +1621,12 @@ func TestCandidateDeliversPreCandidateSelfVoteAfterBecomingCandidate(t *testing. // A single vote from n2 does not move n1 to the leader. sm.Step(pb.Message{From: 2, To: 1, Term: sm.Term, Type: pb.MsgVoteResp}) - assert.Equal(t, StateCandidate, sm.state) + assert.Equal(t, pb.StateCandidate, sm.state) // n1 becomes the leader once its self-vote is received because now // quorum is reached. sm.stepOrSend(steps) - assert.Equal(t, StateLeader, sm.state) + assert.Equal(t, pb.StateLeader, sm.state) } func TestLeaderMsgAppSelfAckAfterTermChange(t *testing.T) { @@ -1640,12 +1640,12 @@ func TestLeaderMsgAppSelfAckAfterTermChange(t *testing.T) { // n1 hears that n2 is the new leader. sm.Step(pb.Message{From: 2, To: 1, Term: sm.Term + 1, Type: pb.MsgHeartbeat}) - assert.Equal(t, StateFollower, sm.state) + assert.Equal(t, pb.StateFollower, sm.state) // n1 advances, ignoring its earlier self-ack of its MsgApp. The // corresponding MsgAppResp is ignored because it carries an earlier term. sm.stepOrSend(steps) - assert.Equal(t, StateFollower, sm.state) + assert.Equal(t, pb.StateFollower, sm.state) } func TestLeaderStepdownWhenQuorumActive(t *testing.T) { @@ -1661,7 +1661,7 @@ func TestLeaderStepdownWhenQuorumActive(t *testing.T) { sm.tick() } - assert.Equal(t, StateLeader, sm.state) + assert.Equal(t, pb.StateLeader, sm.state) } func TestLeaderStepdownWhenQuorumLost(t *testing.T) { @@ -1676,7 +1676,7 @@ func TestLeaderStepdownWhenQuorumLost(t *testing.T) { sm.tick() } - assert.Equal(t, StateFollower, sm.state) + assert.Equal(t, pb.StateFollower, sm.state) } func TestLeaderSupersedingWithCheckQuorum(t *testing.T) { @@ -1701,13 +1701,13 @@ func TestLeaderSupersedingWithCheckQuorum(t *testing.T) { } nt.send(pb.Message{From: 1, To: 1, Type: pb.MsgHup}) - assert.Equal(t, StateLeader, a.state) - assert.Equal(t, StateFollower, c.state) + assert.Equal(t, pb.StateLeader, a.state) + assert.Equal(t, pb.StateFollower, c.state) nt.send(pb.Message{From: 3, To: 3, Type: pb.MsgHup}) // Peer b rejected c's vote since its electionElapsed had not reached to electionTimeout - assert.Equal(t, StateCandidate, c.state) + assert.Equal(t, pb.StateCandidate, c.state) // Letting b's electionElapsed reach to electionTimeout for i := 0; i < b.electionTimeout; i++ { @@ -1715,7 +1715,7 @@ func TestLeaderSupersedingWithCheckQuorum(t *testing.T) { } nt.send(pb.Message{From: 3, To: 3, Type: pb.MsgHup}) - assert.Equal(t, StateLeader, c.state) + assert.Equal(t, pb.StateLeader, c.state) } func TestLeaderElectionWithCheckQuorum(t *testing.T) { @@ -1741,8 +1741,8 @@ func TestLeaderElectionWithCheckQuorum(t *testing.T) { // election timeout. nt.send(pb.Message{From: 1, To: 1, Type: pb.MsgHup}) - assert.Equal(t, StateLeader, a.state) - assert.Equal(t, StateFollower, c.state) + assert.Equal(t, pb.StateLeader, a.state) + assert.Equal(t, pb.StateFollower, c.state) // need to reset randomizedElectionTimeout larger than electionTimeout again, // because the value might be reset to electionTimeout since the last state changes @@ -1756,8 +1756,8 @@ func TestLeaderElectionWithCheckQuorum(t *testing.T) { } nt.send(pb.Message{From: 3, To: 3, Type: pb.MsgHup}) - assert.Equal(t, StateFollower, a.state) - assert.Equal(t, StateLeader, c.state) + assert.Equal(t, pb.StateFollower, a.state) + assert.Equal(t, pb.StateLeader, c.state) } // TestFreeStuckCandidateWithCheckQuorum ensures that a candidate with a higher term @@ -1789,28 +1789,28 @@ func TestFreeStuckCandidateWithCheckQuorum(t *testing.T) { nt.isolate(1) nt.send(pb.Message{From: 3, To: 3, Type: pb.MsgHup}) - assert.Equal(t, StateFollower, b.state) - assert.Equal(t, StateCandidate, c.state) + assert.Equal(t, pb.StateFollower, b.state) + assert.Equal(t, pb.StateCandidate, c.state) assert.Equal(t, b.Term+1, c.Term) // Vote again for safety nt.send(pb.Message{From: 3, To: 3, Type: pb.MsgHup}) - assert.Equal(t, StateFollower, b.state) - assert.Equal(t, StateCandidate, c.state) + assert.Equal(t, pb.StateFollower, b.state) + assert.Equal(t, pb.StateCandidate, c.state) assert.Equal(t, b.Term+2, c.Term) nt.recover() nt.send(pb.Message{From: 1, To: 3, Type: pb.MsgHeartbeat, Term: a.Term}) // Disrupt the leader so that the stuck peer is freed - assert.Equal(t, StateFollower, a.state) + assert.Equal(t, pb.StateFollower, a.state) assert.Equal(t, a.Term, c.Term) // Vote again, should become leader this time nt.send(pb.Message{From: 3, To: 3, Type: pb.MsgHup}) - assert.Equal(t, StateLeader, c.state) + assert.Equal(t, pb.StateLeader, c.state) } func TestNonPromotableVoterWithCheckQuorum(t *testing.T) { @@ -1832,8 +1832,8 @@ func TestNonPromotableVoterWithCheckQuorum(t *testing.T) { } nt.send(pb.Message{From: 1, To: 1, Type: pb.MsgHup}) - assert.Equal(t, StateLeader, a.state) - assert.Equal(t, StateFollower, b.state) + assert.Equal(t, pb.StateLeader, a.state) + assert.Equal(t, pb.StateFollower, b.state) assert.Equal(t, pb.PeerID(1), b.lead) } @@ -1866,9 +1866,9 @@ func TestDisruptiveFollower(t *testing.T) { nt.send(pb.Message{From: 1, To: 1, Type: pb.MsgHup}) // check state - require.Equal(t, StateLeader, n1.state) - require.Equal(t, StateFollower, n2.state) - require.Equal(t, StateFollower, n3.state) + require.Equal(t, pb.StateLeader, n1.state) + require.Equal(t, pb.StateFollower, n2.state) + require.Equal(t, pb.StateFollower, n3.state) // etcd server "advanceTicksForElection" on restart; // this is to expedite campaign trigger when given larger @@ -1890,9 +1890,9 @@ func TestDisruptiveFollower(t *testing.T) { // while its heartbeat to candidate n3 is being delayed // check state - require.Equal(t, StateLeader, n1.state) - require.Equal(t, StateFollower, n2.state) - require.Equal(t, StateCandidate, n3.state) + require.Equal(t, pb.StateLeader, n1.state) + require.Equal(t, pb.StateFollower, n2.state) + require.Equal(t, pb.StateCandidate, n3.state) // check term require.Equal(t, uint64(2), n1.Term) @@ -1911,9 +1911,9 @@ func TestDisruptiveFollower(t *testing.T) { // with higher term can be freed with following election // check state - require.Equal(t, StateFollower, n1.state) - require.Equal(t, StateFollower, n2.state) - require.Equal(t, StateCandidate, n3.state) + require.Equal(t, pb.StateFollower, n1.state) + require.Equal(t, pb.StateFollower, n2.state) + require.Equal(t, pb.StateCandidate, n3.state) // check term require.Equal(t, uint64(3), n1.Term) @@ -1944,9 +1944,9 @@ func TestDisruptiveFollowerPreVote(t *testing.T) { nt.send(pb.Message{From: 1, To: 1, Type: pb.MsgHup}) // check state - require.Equal(t, StateLeader, n1.state) - require.Equal(t, StateFollower, n2.state) - require.Equal(t, StateFollower, n3.state) + require.Equal(t, pb.StateLeader, n1.state) + require.Equal(t, pb.StateFollower, n2.state) + require.Equal(t, pb.StateFollower, n3.state) nt.isolate(3) nt.send(pb.Message{From: 1, To: 1, Type: pb.MsgProp, Entries: []pb.Entry{{Data: []byte("somedata")}}}) @@ -1959,9 +1959,9 @@ func TestDisruptiveFollowerPreVote(t *testing.T) { nt.send(pb.Message{From: 3, To: 3, Type: pb.MsgHup}) // check state - require.Equal(t, StateLeader, n1.state) - require.Equal(t, StateFollower, n2.state) - require.Equal(t, StatePreCandidate, n3.state) + require.Equal(t, pb.StateLeader, n1.state) + require.Equal(t, pb.StateFollower, n2.state) + require.Equal(t, pb.StatePreCandidate, n3.state) // check term require.Equal(t, uint64(2), n1.Term) @@ -1970,7 +1970,7 @@ func TestDisruptiveFollowerPreVote(t *testing.T) { // delayed leader heartbeat does not force current leader to step down nt.send(pb.Message{From: 1, To: 3, Term: n1.Term, Type: pb.MsgHeartbeat}) - require.Equal(t, StateLeader, n1.state) + require.Equal(t, pb.StateLeader, n1.state) } func TestLeaderAppResp(t *testing.T) { @@ -2114,13 +2114,13 @@ func TestBcastBeat(t *testing.T) { // TestRecvMsgBeat tests the output of the state machine when receiving MsgBeat func TestRecvMsgBeat(t *testing.T) { tests := []struct { - state StateType + state pb.StateType wMsg int }{ - {StateLeader, 2}, + {pb.StateLeader, 2}, // candidate and follower should ignore MsgBeat - {StateCandidate, 0}, - {StateFollower, 0}, + {pb.StateCandidate, 0}, + {pb.StateFollower, 0}, } for i, tt := range tests { @@ -2129,11 +2129,11 @@ func TestRecvMsgBeat(t *testing.T) { sm.Term = 1 sm.state = tt.state switch tt.state { - case StateFollower: + case pb.StateFollower: sm.step = stepFollower - case StateCandidate: + case pb.StateCandidate: sm.step = stepCandidate - case StateLeader: + case pb.StateLeader: sm.step = stepLeader } sm.Step(pb.Message{From: 1, To: 1, Type: pb.MsgBeat}) @@ -2358,7 +2358,7 @@ func TestRestore(t *testing.T) { for i := 0; i < sm.randomizedElectionTimeout; i++ { sm.tick() } - assert.Equal(t, StateFollower, sm.state) + assert.Equal(t, pb.StateFollower, sm.state) } // TestRestoreWithLearner restores a snapshot which contains learners. @@ -2420,7 +2420,7 @@ func TestRestoreWithVotersOutgoing(t *testing.T) { for i := 0; i < sm.randomizedElectionTimeout; i++ { sm.tick() } - assert.Equal(t, StateFollower, sm.state) + assert.Equal(t, pb.StateFollower, sm.state) } // TestRestoreVoterToLearner verifies that a normal peer can be downgraded to a @@ -2742,7 +2742,7 @@ func TestAddNodeCheckQuorum(t *testing.T) { r.tick() // Node 1 should still be the leader after a single tick. - assert.Equal(t, StateLeader, r.state) + assert.Equal(t, pb.StateLeader, r.state) // After another electionTimeout ticks without hearing from node 2, // node 1 should step down. @@ -2750,7 +2750,7 @@ func TestAddNodeCheckQuorum(t *testing.T) { r.tick() } - assert.Equal(t, StateFollower, r.state) + assert.Equal(t, pb.StateFollower, r.state) } // TestRemoveNode tests that removeNode could update nodes and @@ -2835,17 +2835,17 @@ func testCampaignWhileLeader(t *testing.T, preVote bool) { cfg := newTestConfig(1, 5, 1, newTestMemoryStorage(withPeers(1))) cfg.PreVote = preVote r := newRaft(cfg) - assert.Equal(t, StateFollower, r.state) + assert.Equal(t, pb.StateFollower, r.state) // We don't call campaign() directly because it comes after the check // for our current state. r.Step(pb.Message{From: 1, To: 1, Type: pb.MsgHup}) r.advanceMessagesAfterAppend() - assert.Equal(t, StateLeader, r.state) + assert.Equal(t, pb.StateLeader, r.state) term := r.Term r.Step(pb.Message{From: 1, To: 1, Type: pb.MsgHup}) r.advanceMessagesAfterAppend() - assert.Equal(t, StateLeader, r.state) + assert.Equal(t, pb.StateLeader, r.state) assert.Equal(t, term, r.Term) } @@ -2919,14 +2919,14 @@ func TestLeaderTransferToUpToDateNode(t *testing.T) { // Transfer leadership to 2. nt.send(pb.Message{From: 2, To: 1, Type: pb.MsgTransferLeader}) - checkLeaderTransferState(t, lead, StateFollower, 2) + checkLeaderTransferState(t, lead, pb.StateFollower, 2) // After some log replication, transfer leadership back to 1. nt.send(pb.Message{From: 1, To: 1, Type: pb.MsgProp, Entries: []pb.Entry{{}}}) nt.send(pb.Message{From: 1, To: 2, Type: pb.MsgTransferLeader}) - checkLeaderTransferState(t, lead, StateLeader, 1) + checkLeaderTransferState(t, lead, pb.StateLeader, 1) } // TestLeaderTransferToUpToDateNodeFromFollower verifies transferring should @@ -2945,14 +2945,14 @@ func TestLeaderTransferToUpToDateNodeFromFollower(t *testing.T) { // Transfer leadership to 2. nt.send(pb.Message{From: 2, To: 2, Type: pb.MsgTransferLeader}) - checkLeaderTransferState(t, lead, StateFollower, 2) + checkLeaderTransferState(t, lead, pb.StateFollower, 2) // After some log replication, transfer leadership back to 1. nt.send(pb.Message{From: 1, To: 1, Type: pb.MsgProp, Entries: []pb.Entry{{}}}) nt.send(pb.Message{From: 1, To: 1, Type: pb.MsgTransferLeader}) - checkLeaderTransferState(t, lead, StateLeader, 1) + checkLeaderTransferState(t, lead, pb.StateLeader, 1) } // TestLeaderTransferLeaderStepsDownImmediately verifies that the outgoing @@ -2977,7 +2977,7 @@ func TestLeaderTransferLeaderStepsDownImmediately(t *testing.T) { nt.send(pb.Message{From: 3, To: 1, Type: pb.MsgTransferLeader}) require.Equal(t, uint64(1), lead.Term) - checkLeaderTransferState(t, lead, StateFollower, 1) + checkLeaderTransferState(t, lead, pb.StateFollower, 1) // TODO(arul): a leader that steps down will currently never campaign due to // the fortification promise that it made to itself. We'll need to fix this. @@ -2991,7 +2991,7 @@ func TestLeaderTransferLeaderStepsDownImmediately(t *testing.T) { nt.send(lead.readMessages()...) require.Equal(t, uint64(2), lead.Term) - checkLeaderTransferState(t, lead, StateLeader, 1) + checkLeaderTransferState(t, lead, pb.StateLeader, 1) } // TestLeaderTransferWithCheckQuorum ensures transferring leader still works @@ -3014,19 +3014,19 @@ func TestLeaderTransferWithCheckQuorum(t *testing.T) { lead := nt.peers[1].(*raft) - require.Equal(t, StateLeader, lead.state) + require.Equal(t, pb.StateLeader, lead.state) // Transfer leadership to 2. nt.send(pb.Message{From: 2, To: 1, Type: pb.MsgTransferLeader}) - checkLeaderTransferState(t, lead, StateFollower, 2) + checkLeaderTransferState(t, lead, pb.StateFollower, 2) // After some log replication, transfer leadership back to 1. nt.send(pb.Message{From: 1, To: 1, Type: pb.MsgProp, Entries: []pb.Entry{{}}}) nt.send(pb.Message{From: 1, To: 2, Type: pb.MsgTransferLeader}) - checkLeaderTransferState(t, lead, StateLeader, 1) + checkLeaderTransferState(t, lead, pb.StateLeader, 1) } func TestLeaderTransferToSlowFollower(t *testing.T) { @@ -3049,7 +3049,7 @@ func TestLeaderTransferToSlowFollower(t *testing.T) { nt.recover() nt.send(pb.Message{From: 3, To: 1, Type: pb.MsgTransferLeader}) - checkLeaderTransferState(t, lead, StateFollower, 3) + checkLeaderTransferState(t, lead, pb.StateFollower, 3) } func TestLeaderTransferAfterSnapshot(t *testing.T) { @@ -3078,7 +3078,7 @@ func TestLeaderTransferAfterSnapshot(t *testing.T) { } // Transfer leadership to 3 when node 3 is missing a snapshot. nt.send(pb.Message{From: 3, To: 1, Type: pb.MsgTransferLeader}) - require.Equal(t, StateLeader, lead.state) + require.Equal(t, pb.StateLeader, lead.state) require.NotEqual(t, pb.Message{}, filtered) // Apply snapshot and resume progress. @@ -3089,7 +3089,7 @@ func TestLeaderTransferAfterSnapshot(t *testing.T) { nt.msgHook = nil nt.send(filtered) - checkLeaderTransferState(t, lead, StateFollower, 3) + checkLeaderTransferState(t, lead, pb.StateFollower, 3) } func TestLeaderTransferToSelf(t *testing.T) { @@ -3100,7 +3100,7 @@ func TestLeaderTransferToSelf(t *testing.T) { // Transfer leadership to self, there will be noop. nt.send(pb.Message{From: 1, To: 1, Type: pb.MsgTransferLeader}) - checkLeaderTransferState(t, lead, StateLeader, 1) + checkLeaderTransferState(t, lead, pb.StateLeader, 1) } func TestLeaderTransferToNonExistingNode(t *testing.T) { @@ -3110,7 +3110,7 @@ func TestLeaderTransferToNonExistingNode(t *testing.T) { lead := nt.peers[1].(*raft) // Transfer leadership to non-existing node, there will be noop. nt.send(pb.Message{From: 4, To: 1, Type: pb.MsgTransferLeader}) - checkLeaderTransferState(t, lead, StateLeader, 1) + checkLeaderTransferState(t, lead, pb.StateLeader, 1) } func TestLeaderTransferTimeout(t *testing.T) { @@ -3138,7 +3138,7 @@ func TestLeaderTransferTimeout(t *testing.T) { lead.tick() } - checkLeaderTransferState(t, lead, StateLeader, 1) + checkLeaderTransferState(t, lead, pb.StateLeader, 1) } func TestLeaderTransferIgnoreProposal(t *testing.T) { @@ -3186,7 +3186,7 @@ func TestLeaderTransferReceiveHigherTermVote(t *testing.T) { nt.send(pb.Message{From: 2, To: 2, Type: pb.MsgHup, Index: 1, Term: 2}) - checkLeaderTransferState(t, lead, StateFollower, 2) + checkLeaderTransferState(t, lead, pb.StateFollower, 2) } func TestLeaderTransferRemoveNode(t *testing.T) { @@ -3207,7 +3207,7 @@ func TestLeaderTransferRemoveNode(t *testing.T) { lead.applyConfChange(pb.ConfChange{NodeID: 3, Type: pb.ConfChangeRemoveNode}.AsV2()) - checkLeaderTransferState(t, lead, StateLeader, 1) + checkLeaderTransferState(t, lead, pb.StateLeader, 1) } func TestLeaderTransferDemoteNode(t *testing.T) { @@ -3241,7 +3241,7 @@ func TestLeaderTransferDemoteNode(t *testing.T) { // Make the Raft group commit the LeaveJoint entry. lead.applyConfChange(pb.ConfChangeV2{}) - checkLeaderTransferState(t, lead, StateLeader, 1) + checkLeaderTransferState(t, lead, pb.StateLeader, 1) } // TestLeaderTransferBack verifies leadership can transfer back to self when @@ -3264,7 +3264,7 @@ func TestLeaderTransferBack(t *testing.T) { // Transfer leadership back to self. nt.send(pb.Message{From: 1, To: 1, Type: pb.MsgTransferLeader}) - checkLeaderTransferState(t, lead, StateLeader, 1) + checkLeaderTransferState(t, lead, pb.StateLeader, 1) } // TestLeaderTransferSecondTransferToAnotherNode verifies leader can transfer to @@ -3288,7 +3288,7 @@ func TestLeaderTransferSecondTransferToAnotherNode(t *testing.T) { // Transfer leadership to another node. nt.send(pb.Message{From: 2, To: 1, Type: pb.MsgTransferLeader}) - checkLeaderTransferState(t, lead, StateFollower, 2) + checkLeaderTransferState(t, lead, pb.StateFollower, 2) } // TestLeaderTransferSecondTransferToSameNode verifies second transfer leader @@ -3319,10 +3319,10 @@ func TestLeaderTransferSecondTransferToSameNode(t *testing.T) { lead.tick() } - checkLeaderTransferState(t, lead, StateLeader, 1) + checkLeaderTransferState(t, lead, pb.StateLeader, 1) } -func checkLeaderTransferState(t *testing.T, r *raft, state StateType, lead pb.PeerID) { +func checkLeaderTransferState(t *testing.T, r *raft, state pb.StateType, lead pb.PeerID) { require.Equal(t, state, r.state) require.Equal(t, lead, r.lead) require.Equal(t, None, r.leadTransferee) @@ -3337,7 +3337,7 @@ func TestLeaderTransferNonMember(t *testing.T) { r.Step(pb.Message{From: 2, To: 1, Type: pb.MsgVoteResp}) r.Step(pb.Message{From: 3, To: 1, Type: pb.MsgVoteResp}) - require.Equal(t, StateFollower, r.state) + require.Equal(t, pb.StateFollower, r.state) } // TestLeaderTransferDifferentTerms verifies that a MsgTimeoutNow will only be @@ -3351,9 +3351,9 @@ func TestLeaderTransferDifferentTerms(t *testing.T) { nt.send(pb.Message{From: 3, To: 2, Type: pb.MsgTransferLeader}) for i, p := range nt.peers { r := p.(*raft) - expState := StateFollower + expState := pb.StateFollower if i == 3 { - expState = StateLeader + expState = pb.StateLeader } require.Equal(t, expState, r.state) require.Equal(t, uint64(3), r.Term) @@ -3366,13 +3366,13 @@ func TestLeaderTransferDifferentTerms(t *testing.T) { // permission to overthrow a newer leader. nt.send(pb.Message{From: 2, To: 1, Term: 2, Type: pb.MsgTimeoutNow}) n1 := nt.peers[1].(*raft) - require.Equal(t, StateFollower, n1.state) + require.Equal(t, pb.StateFollower, n1.state) require.Equal(t, uint64(3), n1.Term) // Send a MsgTimeoutNow to node 1 from the current term. This should cause it // to call an election for the _next_ term, which it will win. nt.send(pb.Message{From: 3, To: 1, Term: 3, Type: pb.MsgTimeoutNow}) - require.Equal(t, StateLeader, n1.state) + require.Equal(t, pb.StateLeader, n1.state) require.Equal(t, uint64(4), n1.Term) // Send a MsgTimeoutNow to node 2 from a new term. This should advance the @@ -3380,7 +3380,7 @@ func TestLeaderTransferDifferentTerms(t *testing.T) { // it will win. nt.send(pb.Message{From: 1, To: 2, Term: 5, Type: pb.MsgTimeoutNow}) n2 := nt.peers[2].(*raft) - require.Equal(t, StateLeader, n2.state) + require.Equal(t, pb.StateLeader, n2.state) require.Equal(t, uint64(6), n2.Term) } @@ -3401,7 +3401,7 @@ func TestLeaderTransferStaleFollower(t *testing.T) { nt.isolate(3) nt.send(pb.Message{From: 3, To: 1, Type: pb.MsgTransferLeader}) for _, n := range nodes { - require.Equal(t, StateFollower, n.state) + require.Equal(t, pb.StateFollower, n.state) require.Equal(t, uint64(1), n.Term) } @@ -3417,9 +3417,9 @@ func TestLeaderTransferStaleFollower(t *testing.T) { } nt.send(nt.filter(n1.readMessages())...) for _, n := range nodes { - expState := StateFollower + expState := pb.StateFollower if n == n1 { - expState = StateLeader + expState = pb.StateLeader } expTerm := uint64(2) if n == n3 { @@ -3435,9 +3435,9 @@ func TestLeaderTransferStaleFollower(t *testing.T) { nt.recover() nt.send(pb.Message{From: 1, To: 3, Term: 1, Type: pb.MsgTimeoutNow}) for _, n := range nodes { - expState := StateFollower + expState := pb.StateFollower if n == n1 { - expState = StateLeader + expState = pb.StateLeader } require.Equal(t, expState, n.state) require.Equal(t, uint64(2), n.Term) @@ -3470,14 +3470,14 @@ func TestNodeWithSmallerTermCanCompleteElection(t *testing.T) { nt.send(pb.Message{From: 1, To: 1, Type: pb.MsgHup}) sm := nt.peers[1].(*raft) - assert.Equal(t, StateLeader, sm.state) + assert.Equal(t, pb.StateLeader, sm.state) sm = nt.peers[2].(*raft) - assert.Equal(t, StateFollower, sm.state) + assert.Equal(t, pb.StateFollower, sm.state) nt.send(pb.Message{From: 3, To: 3, Type: pb.MsgHup}) sm = nt.peers[3].(*raft) - assert.Equal(t, StatePreCandidate, sm.state) + assert.Equal(t, pb.StatePreCandidate, sm.state) nt.send(pb.Message{From: 2, To: 2, Type: pb.MsgHup}) @@ -3491,11 +3491,11 @@ func TestNodeWithSmallerTermCanCompleteElection(t *testing.T) { // check state sm = nt.peers[1].(*raft) - assert.Equal(t, StateFollower, sm.state) + assert.Equal(t, pb.StateFollower, sm.state) sm = nt.peers[2].(*raft) - assert.Equal(t, StateLeader, sm.state) + assert.Equal(t, pb.StateLeader, sm.state) sm = nt.peers[3].(*raft) - assert.Equal(t, StatePreCandidate, sm.state) + assert.Equal(t, pb.StatePreCandidate, sm.state) sm.logger.Infof("going to bring back peer 3 and kill peer 2") // recover the network then immediately isolate b which is currently @@ -3511,7 +3511,7 @@ func TestNodeWithSmallerTermCanCompleteElection(t *testing.T) { // do we have a leader? sma := nt.peers[1].(*raft) smb := nt.peers[3].(*raft) - assert.True(t, sma.state == StateLeader || smb.state == StateLeader) + assert.True(t, sma.state == pb.StateLeader || smb.state == pb.StateLeader) } // TestPreVoteWithSplitVote verifies that after split vote, cluster can complete @@ -3547,9 +3547,9 @@ func TestPreVoteWithSplitVote(t *testing.T) { // check state sm = nt.peers[2].(*raft) - assert.Equal(t, StateCandidate, sm.state) + assert.Equal(t, pb.StateCandidate, sm.state) sm = nt.peers[3].(*raft) - assert.Equal(t, StateCandidate, sm.state) + assert.Equal(t, pb.StateCandidate, sm.state) // node 2 election timeout first nt.send(pb.Message{From: 2, To: 2, Type: pb.MsgHup}) @@ -3562,9 +3562,9 @@ func TestPreVoteWithSplitVote(t *testing.T) { // check state sm = nt.peers[2].(*raft) - assert.Equal(t, StateLeader, sm.state) + assert.Equal(t, pb.StateLeader, sm.state) sm = nt.peers[3].(*raft) - assert.Equal(t, StateFollower, sm.state) + assert.Equal(t, pb.StateFollower, sm.state) } // TestPreVoteWithCheckQuorum ensures that after a node become pre-candidate, @@ -3594,20 +3594,20 @@ func TestPreVoteWithCheckQuorum(t *testing.T) { // check state sm := nt.peers[1].(*raft) - require.Equal(t, StateLeader, sm.state) + require.Equal(t, pb.StateLeader, sm.state) sm = nt.peers[2].(*raft) - require.Equal(t, StateFollower, sm.state) + require.Equal(t, pb.StateFollower, sm.state) sm = nt.peers[3].(*raft) - require.Equal(t, StateFollower, sm.state) + require.Equal(t, pb.StateFollower, sm.state) // node 2 will ignore node 3's PreVote nt.send(pb.Message{From: 3, To: 3, Type: pb.MsgHup}) nt.send(pb.Message{From: 2, To: 2, Type: pb.MsgHup}) // Do we have a leader? - assert.True(t, n2.state == StateLeader || n3.state == StateFollower) + assert.True(t, n2.state == pb.StateLeader || n3.state == pb.StateFollower) } // TestLearnerCampaign verifies that a learner won't campaign even if it receives @@ -3622,17 +3622,17 @@ func TestLearnerCampaign(t *testing.T) { require.True(t, n2.isLearner) - require.Equal(t, StateFollower, n2.state) + require.Equal(t, pb.StateFollower, n2.state) nt.send(pb.Message{From: 1, To: 1, Type: pb.MsgHup}) - require.True(t, n1.state == StateLeader && n1.lead == 1) + require.True(t, n1.state == pb.StateLeader && n1.lead == 1) // NB: TransferLeader already checks that the recipient is not a learner, but // the check could have happened by the time the recipient becomes a learner, // in which case it will receive MsgTimeoutNow as in this test case and we // verify that it's ignored. nt.send(pb.Message{From: 1, To: 2, Type: pb.MsgTimeoutNow}) - require.Equal(t, StateFollower, n2.state) + require.Equal(t, pb.StateFollower, n2.state) } // simulate rolling update a cluster for Pre-Vote. cluster has 3 nodes [n1, n2, n3]. @@ -3664,9 +3664,9 @@ func newPreVoteMigrationCluster(t *testing.T) *network { nt.send(pb.Message{From: 3, To: 3, Type: pb.MsgHup}) // check state - require.Equal(t, StateLeader, n1.state) - require.Equal(t, StateFollower, n2.state) - require.Equal(t, StateCandidate, n3.state) + require.Equal(t, pb.StateLeader, n1.state) + require.Equal(t, pb.StateFollower, n2.state) + require.Equal(t, pb.StateCandidate, n3.state) // check term require.Equal(t, uint64(2), n1.Term) @@ -3697,14 +3697,14 @@ func TestPreVoteMigrationCanCompleteElection(t *testing.T) { nt.send(pb.Message{From: 2, To: 2, Type: pb.MsgHup}) // check state - assert.Equal(t, StateFollower, n2.state) - assert.Equal(t, StatePreCandidate, n3.state) + assert.Equal(t, pb.StateFollower, n2.state) + assert.Equal(t, pb.StatePreCandidate, n3.state) nt.send(pb.Message{From: 3, To: 3, Type: pb.MsgHup}) nt.send(pb.Message{From: 2, To: 2, Type: pb.MsgHup}) // Do we have a leader? - assert.True(t, n2.state == StateLeader || n3.state == StateFollower) + assert.True(t, n2.state == pb.StateLeader || n3.state == pb.StateFollower) } func TestPreVoteMigrationWithFreeStuckPreCandidate(t *testing.T) { @@ -3719,21 +3719,21 @@ func TestPreVoteMigrationWithFreeStuckPreCandidate(t *testing.T) { nt.send(pb.Message{From: 3, To: 3, Type: pb.MsgHup}) - assert.Equal(t, StateLeader, n1.state) - assert.Equal(t, StateFollower, n2.state) - assert.Equal(t, StatePreCandidate, n3.state) + assert.Equal(t, pb.StateLeader, n1.state) + assert.Equal(t, pb.StateFollower, n2.state) + assert.Equal(t, pb.StatePreCandidate, n3.state) // Pre-Vote again for safety nt.send(pb.Message{From: 3, To: 3, Type: pb.MsgHup}) - assert.Equal(t, StateLeader, n1.state) - assert.Equal(t, StateFollower, n2.state) - assert.Equal(t, StatePreCandidate, n3.state) + assert.Equal(t, pb.StateLeader, n1.state) + assert.Equal(t, pb.StateFollower, n2.state) + assert.Equal(t, pb.StatePreCandidate, n3.state) nt.send(pb.Message{From: 1, To: 3, Type: pb.MsgHeartbeat, Term: n1.Term}) // Disrupt the leader so that the stuck peer is freed - assert.Equal(t, StateFollower, n1.state) + assert.Equal(t, pb.StateFollower, n1.state) assert.Equal(t, n1.Term, n3.Term) } @@ -3743,7 +3743,7 @@ func testConfChangeCheckBeforeCampaign(t *testing.T, v2 bool) { n1 := nt.peers[1].(*raft) n2 := nt.peers[2].(*raft) nt.send(pb.Message{From: 1, To: 1, Type: pb.MsgHup}) - assert.Equal(t, StateLeader, n1.state) + assert.Equal(t, pb.StateLeader, n1.state) // Begin to remove the third node. cc := pb.ConfChange{ @@ -3776,15 +3776,15 @@ func testConfChangeCheckBeforeCampaign(t *testing.T, v2 bool) { n2.tick() } // It's still follower because committed conf change is not applied. - assert.Equal(t, StateFollower, n2.state) + assert.Equal(t, pb.StateFollower, n2.state) // Transfer leadership to peer 2. nt.send(pb.Message{From: 2, To: 1, Type: pb.MsgTransferLeader}) // The outgoing leader steps down immediately. - assert.Equal(t, StateFollower, n1.state) + assert.Equal(t, pb.StateFollower, n1.state) // The transfer target does not campaign immediately because the committed // conf change is not applied. - assert.Equal(t, StateFollower, n2.state) + assert.Equal(t, pb.StateFollower, n2.state) // TODO(arul): a leader that steps down will currently never campaign due to // the fortification promise that it made to itself. We'll need to fix this. @@ -3796,7 +3796,7 @@ func testConfChangeCheckBeforeCampaign(t *testing.T, v2 bool) { n1.tick() } nt.send(n1.readMessages()...) - assert.Equal(t, StateLeader, n1.state) + assert.Equal(t, pb.StateLeader, n1.state) // Advance apply on node 2. nextEnts(n2, nt.storage[2]) @@ -3804,10 +3804,10 @@ func testConfChangeCheckBeforeCampaign(t *testing.T, v2 bool) { // Transfer leadership to peer 2 again. nt.send(pb.Message{From: 2, To: 1, Type: pb.MsgTransferLeader}) // The outgoing leader steps down immediately. - assert.Equal(t, StateFollower, n1.state) + assert.Equal(t, pb.StateFollower, n1.state) // The transfer target campaigns immediately now that the committed conf // change is applied. - assert.Equal(t, StateLeader, n2.state) + assert.Equal(t, pb.StateLeader, n2.state) } // TestConfChangeCheckBeforeCampaign tests if unapplied ConfChange is checked before campaign. diff --git a/pkg/raft/raftpb/raft.go b/pkg/raft/raftpb/raft.go index 71457619e4f1..b2073df26232 100644 --- a/pkg/raft/raftpb/raft.go +++ b/pkg/raft/raftpb/raft.go @@ -5,7 +5,11 @@ package raftpb -import "github.com/cockroachdb/redact" +import ( + "fmt" + + "github.com/cockroachdb/redact" +) // PeerID is a custom type for peer IDs in a raft group. type PeerID uint64 @@ -76,3 +80,30 @@ func (p Priority) SafeFormat(w redact.SafePrinter, _ rune) { panic("invalid raft priority") } } + +// StateType represents the role of a node in a cluster. +type StateType uint64 + +// Possible values for StateType. +const ( + StateFollower StateType = iota + StateCandidate + StateLeader + StatePreCandidate + NumStates +) + +var stmap = [...]string{ + "StateFollower", + "StateCandidate", + "StateLeader", + "StatePreCandidate", +} + +func (st StateType) String() string { + return stmap[st] +} + +func (st StateType) MarshalJSON() ([]byte, error) { + return []byte(fmt.Sprintf("%q", st.String())), nil +} diff --git a/pkg/raft/rawnode.go b/pkg/raft/rawnode.go index 179f415b748a..7e93ace6f64d 100644 --- a/pkg/raft/rawnode.go +++ b/pkg/raft/rawnode.go @@ -169,7 +169,7 @@ func (rn *RawNode) SetLazyReplication(lazy bool) { } // The lazy mode was disabled. We need to check whether any replication flows // are unblocked and can be saturated. - if r.state == StateLeader { + if r.state == pb.StateLeader { // TODO(pav-kv): this sends at most one MsgApp message per peer. It may not // completely saturate the flow. Consider looping while maybeSendAppend() // returns true. diff --git a/pkg/raft/rawnode_test.go b/pkg/raft/rawnode_test.go index 617cd853544e..40dc2981b9fc 100644 --- a/pkg/raft/rawnode_test.go +++ b/pkg/raft/rawnode_test.go @@ -473,7 +473,7 @@ func TestRawNodeStart(t *testing.T) { {Term: 1, Index: 3, Data: []byte("foo")}, // non-empty entry } want := Ready{ - SoftState: &SoftState{RaftState: StateLeader}, + SoftState: &SoftState{RaftState: pb.StateLeader}, HardState: pb.HardState{Term: 1, Commit: 3, Vote: 1, Lead: 1, LeadEpoch: 1}, Entries: nil, // emitted & checked in intermediate Ready cycle CommittedEntries: entries, @@ -582,7 +582,7 @@ func TestRawNodeRestart(t *testing.T) { assert.Equal(t, uint64(1), rawNode.raft.Term) assert.Equal(t, uint64(1), rawNode.raft.raftLog.committed) assert.Equal(t, pb.PeerID(1), rawNode.raft.lead) - assert.True(t, rawNode.raft.state == StateFollower) + assert.True(t, rawNode.raft.state == pb.StateFollower) assert.Equal(t, pb.Epoch(1), rawNode.raft.leadEpoch) // Ensure we campaign after the election timeout has elapsed. @@ -592,7 +592,7 @@ func TestRawNodeRestart(t *testing.T) { rawNode.raft.leadEpoch = 0 rawNode.raft.tick() } - assert.Equal(t, StateCandidate, rawNode.raft.state) + assert.Equal(t, pb.StateCandidate, rawNode.raft.state) assert.Equal(t, uint64(2), rawNode.raft.Term) // this should in-turn bump the term } @@ -643,7 +643,7 @@ func TestRawNodeStatus(t *testing.T) { rn.Advance(rd) status := rn.Status() require.Equal(t, pb.PeerID(1), status.Lead) - require.Equal(t, StateLeader, status.RaftState) + require.Equal(t, pb.StateLeader, status.RaftState) require.Equal(t, *rn.raft.trk.Progress(1), status.Progress[1]) expCfg := quorum.Config{Voters: quorum.JointConfig{ diff --git a/pkg/raft/status.go b/pkg/raft/status.go index 32c42c332bbd..e4811f1f8962 100644 --- a/pkg/raft/status.go +++ b/pkg/raft/status.go @@ -101,7 +101,7 @@ func getBasicStatus(r *raft) BasicStatus { s.HardState = r.hardState() s.SoftState = r.softState() s.Applied = r.raftLog.applied - if s.RaftState == StateFollower && s.Lead == r.id { + if s.RaftState == pb.StateFollower && s.Lead == r.id { // A raft leader's term ends when it is shut down. It'll rejoin its peers as // a follower when it comes back up, but its Lead and Term field may still // correspond to its pre-restart leadership term. We expect this to quickly @@ -125,7 +125,7 @@ func getBasicStatus(r *raft) BasicStatus { func getStatus(r *raft) Status { var s Status s.BasicStatus = getBasicStatus(r) - if s.RaftState == StateLeader { + if s.RaftState == pb.StateLeader { s.Progress = getProgressCopy(r) } s.Config = r.config.Clone() @@ -142,7 +142,7 @@ func getStatus(r *raft) Status { func getSparseStatus(r *raft) SparseStatus { var s SparseStatus s.BasicStatus = getBasicStatus(r) - if s.RaftState == StateLeader { + if s.RaftState == pb.StateLeader { s.Progress = make(map[pb.PeerID]tracker.Progress, r.trk.Len()) withProgress(r, func(id pb.PeerID, _ ProgressType, pr tracker.Progress) { s.Progress[id] = pr diff --git a/pkg/raft/util.go b/pkg/raft/util.go index 6967c395dd63..dfad989b062b 100644 --- a/pkg/raft/util.go +++ b/pkg/raft/util.go @@ -26,10 +26,6 @@ import ( pb "github.com/cockroachdb/cockroach/pkg/raft/raftpb" ) -func (st StateType) MarshalJSON() ([]byte, error) { - return []byte(fmt.Sprintf("%q", st.String())), nil -} - var isLocalMsg = [...]bool{ pb.MsgHup: true, pb.MsgBeat: true, diff --git a/pkg/testutils/testcluster/BUILD.bazel b/pkg/testutils/testcluster/BUILD.bazel index ba3a160c26e0..768147be5c46 100644 --- a/pkg/testutils/testcluster/BUILD.bazel +++ b/pkg/testutils/testcluster/BUILD.bazel @@ -14,7 +14,7 @@ go_library( "//pkg/kv/kvserver", "//pkg/kv/kvserver/liveness/livenesspb", "//pkg/multitenant/tenantcapabilities", - "//pkg/raft", + "//pkg/raft/raftpb", "//pkg/roachpb", "//pkg/rpc", "//pkg/rpc/nodedialer", diff --git a/pkg/testutils/testcluster/testcluster.go b/pkg/testutils/testcluster/testcluster.go index 8f7dcc336a03..ac49505dcfc9 100644 --- a/pkg/testutils/testcluster/testcluster.go +++ b/pkg/testutils/testcluster/testcluster.go @@ -23,7 +23,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/kv/kvserver" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/liveness/livenesspb" "github.com/cockroachdb/cockroach/pkg/multitenant/tenantcapabilities" - "github.com/cockroachdb/cockroach/pkg/raft" + "github.com/cockroachdb/cockroach/pkg/raft/raftpb" "github.com/cockroachdb/cockroach/pkg/roachpb" "github.com/cockroachdb/cockroach/pkg/rpc" "github.com/cockroachdb/cockroach/pkg/rpc/nodedialer" @@ -1855,7 +1855,7 @@ func (tc *TestCluster) GetRaftLeader( // invalid. raftLeaderRepl = nil latestTerm = raftStatus.Term - if raftStatus.RaftState == raft.StateLeader { + if raftStatus.RaftState == raftpb.StateLeader { raftLeaderRepl = repl } }