Forbid removed node from being a Candidate (#476)

* modify `TestRaft_RemoveFollower` to check removed node state

* add test to verify that a removed node is not able to vote

* do not transition to `Candidate` state if not part of stable configuration

* add some comments to the tests

* do not return if not transitioning state

* add wait loop to test

* remove test related to removed node voting

* check `inConfig` instead of `hasVote`

* updating warn log message to be more accurate

configuration.go

@@ -173,6 +173,17 @@ func hasVote(configuration Configuration, id ServerID) bool {
 	return false
 }
 
+// hasVote returns true if the server identified by 'id' is a Voter in the
+// provided Configuration.
+func inConfig(configuration Configuration, id ServerID) bool {
+	for _, server := range configuration.Servers {
+		if server.ID == id {
+			return true
+		}
+	}
+	return false
+}
+
 // checkConfiguration tests a cluster membership configuration for common
 // errors.
 func checkConfiguration(configuration Configuration) error {

raft.go

@@ -213,10 +213,17 @@ func (r *Raft) runFollower() {
 					didWarn = true
 				}
 			} else {
-				r.logger.Warn("heartbeat timeout reached, starting election", "last-leader", lastLeader)
 				metrics.IncrCounter([]string{"raft", "transition", "heartbeat_timeout"}, 1)
-				r.setState(Candidate)
-				return
+				if inConfig(r.configurations.latest, r.localID) {
+					r.logger.Warn("heartbeat timeout reached, starting election", "last-leader", lastLeader)
+					r.setState(Candidate)
+					return
+				} else {
+					if !didWarn {
+						r.logger.Warn("heartbeat timeout reached, not part of stable configuration, not triggering a leader election")
+						didWarn = true
+					}
+				}
 			}
 
 		case <-r.shutdownCh:

raft_test.go

@@ -697,6 +697,9 @@ func TestRaft_RemoveFollower(t *testing.T) {
 	if configuration := c.getConfiguration(followers[1]); len(configuration.Servers) != 2 {
 		t.Fatalf("too many peers")
 	}
+
+	// The removed node should remain in a follower state
+	require.Equal(t, Follower, follower.getState())
 }
 
 func TestRaft_RemoveLeader(t *testing.T) {