Skip to content

Commit

Permalink
Ignore services with 1 replica if needed
Browse files Browse the repository at this point in the history 
Signed-off-by: Mostafa Saber <[email protected]>
  • Loading branch information
@MustafaSaber
MustafaSaber committed May 19, 2023
1 parent 57909ce commit 301a1d3
Show file tree
Hide file tree
Showing 3 changed files with 78 additions and 26 deletions.
13 changes: 10 additions & 3 deletions chaoskube/chaoskube.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,9 @@ type Chaoskube struct {
Notifier notifier.Notifier
// namespace scope for the Kubernetes client
ClientNamespaceScope string

// ignoreSingleReplicas will make chaoskube ignore services with one replica when needed.
IgnoreSingleReplicas bool
}

var (
Expand All @@ -97,7 +100,7 @@ var (
// * a logger implementing logrus.FieldLogger to send log output to
// * what specific terminator to use to imbue chaos on victim pods
// * whether to enable/disable dry-run mode
func New(client kubernetes.Interface, labels, annotations, kinds, namespaces, namespaceLabels labels.Selector, includedPodNames, excludedPodNames *regexp.Regexp, excludedWeekdays []time.Weekday, excludedTimesOfDay []util.TimePeriod, excludedDaysOfYear []time.Time, timezone *time.Location, minimumAge time.Duration, logger log.FieldLogger, dryRun bool, terminator terminator.Terminator, maxKill int, notifier notifier.Notifier, clientNamespaceScope string) *Chaoskube {
func New(client kubernetes.Interface, labels, annotations, kinds, namespaces, namespaceLabels labels.Selector, includedPodNames, excludedPodNames *regexp.Regexp, excludedWeekdays []time.Weekday, excludedTimesOfDay []util.TimePeriod, excludedDaysOfYear []time.Time, timezone *time.Location, minimumAge time.Duration, logger log.FieldLogger, dryRun bool, terminator terminator.Terminator, maxKill int, notifier notifier.Notifier, clientNamespaceScope string, ignoreSingleReplicas bool) *Chaoskube {
broadcaster := record.NewBroadcaster()
broadcaster.StartRecordingToSink(&typedcorev1.EventSinkImpl{Interface: client.CoreV1().Events(clientNamespaceScope)})
recorder := broadcaster.NewRecorder(scheme.Scheme, v1.EventSource{Component: "chaoskube"})
Expand All @@ -124,6 +127,7 @@ func New(client kubernetes.Interface, labels, annotations, kinds, namespaces, na
MaxKill: maxKill,
Notifier: notifier,
ClientNamespaceScope: clientNamespaceScope,
IgnoreSingleReplicas: ignoreSingleReplicas,
}
}

Expand Down Expand Up @@ -239,7 +243,7 @@ func (c *Chaoskube) Candidates(ctx context.Context) ([]v1.Pod, error) {
pods = filterTerminatingPods(pods)
pods = filterByMinimumAge(pods, c.MinimumAge, c.Now())
pods = filterByPodName(pods, c.IncludedPodNames, c.ExcludedPodNames)
pods = filterByOwnerReference(pods)
pods = filterByOwnerReference(pods, c.IgnoreSingleReplicas)

return pods, nil
}
Expand Down Expand Up @@ -513,7 +517,7 @@ func filterByPodName(pods []v1.Pod, includedPodNames, excludedPodNames *regexp.R
return filteredList
}

func filterByOwnerReference(pods []v1.Pod) []v1.Pod {
func filterByOwnerReference(pods []v1.Pod, ignoreSingleReplicas bool) []v1.Pod {
owners := make(map[types.UID][]v1.Pod)
filteredList := []v1.Pod{}
for _, pod := range pods {
Expand All @@ -531,6 +535,9 @@ func filterByOwnerReference(pods []v1.Pod) []v1.Pod {

// For each owner reference select a random pod from its group
for _, pods := range owners {
if ignoreSingleReplicas && len(pods) <= 1 {
continue
}
filteredList = append(filteredList, util.RandomPodSubSlice(pods, 1)...)
}

Expand Down
87 changes: 64 additions & 23 deletions chaoskube/chaoskube_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,22 +48,23 @@ func (suite *Suite) SetupTest() {
// TestNew tests that arguments are passed to the new instance correctly
func (suite *Suite) TestNew() {
var (
client = fake.NewSimpleClientset()
labelSelector, _ = labels.Parse("foo=bar")
annotations, _ = labels.Parse("baz=waldo")
kinds, _ = labels.Parse("job")
namespaces, _ = labels.Parse("qux")
namespaceLabels, _ = labels.Parse("taz=wubble")
includedPodNames = regexp.MustCompile("foo")
excludedPodNames = regexp.MustCompile("bar")
excludedWeekdays = []time.Weekday{time.Friday}
excludedTimesOfDay = []util.TimePeriod{util.TimePeriod{}}
excludedDaysOfYear = []time.Time{time.Now()}
minimumAge = time.Duration(42)
dryRun = true
terminator = terminator.NewDeletePodTerminator(client, logger, 10*time.Second)
maxKill = 1
notifier = testNotifier
client = fake.NewSimpleClientset()
labelSelector, _ = labels.Parse("foo=bar")
annotations, _ = labels.Parse("baz=waldo")
kinds, _ = labels.Parse("job")
namespaces, _ = labels.Parse("qux")
namespaceLabels, _ = labels.Parse("taz=wubble")
includedPodNames = regexp.MustCompile("foo")
excludedPodNames = regexp.MustCompile("bar")
excludedWeekdays = []time.Weekday{time.Friday}
excludedTimesOfDay = []util.TimePeriod{{}}
excludedDaysOfYear = []time.Time{time.Now()}
minimumAge = time.Duration(42)
dryRun = true
terminator = terminator.NewDeletePodTerminator(client, logger, 10*time.Second)
maxKill = 1
notifier = testNotifier
ignoreSingleReplicas = false
)

chaoskube := New(
Expand All @@ -86,6 +87,7 @@ func (suite *Suite) TestNew() {
maxKill,
notifier,
v1.NamespaceAll,
ignoreSingleReplicas,
)
suite.Require().NotNil(chaoskube)

Expand All @@ -105,6 +107,7 @@ func (suite *Suite) TestNew() {
suite.Equal(logger, chaoskube.Logger)
suite.Equal(dryRun, chaoskube.DryRun)
suite.Equal(terminator, chaoskube.Terminator)
suite.Equal(ignoreSingleReplicas, chaoskube.IgnoreSingleReplicas)
}

// TestRunContextCanceled tests that a canceled context will exit the Run function.
Expand All @@ -126,6 +129,7 @@ func (suite *Suite) TestRunContextCanceled() {
10,
1,
v1.NamespaceAll,
false,
)

ctx, cancel := context.WithCancel(context.Background())
Expand Down Expand Up @@ -182,6 +186,7 @@ func (suite *Suite) TestCandidates() {
false,
10,
v1.NamespaceAll,
false,
)

suite.assertCandidates(chaoskube, tt.pods)
Expand Down Expand Up @@ -228,6 +233,7 @@ func (suite *Suite) TestCandidatesNamespaceLabels() {
false,
10,
v1.NamespaceAll,
false,
)

suite.assertCandidates(chaoskube, tt.pods)
Expand Down Expand Up @@ -262,6 +268,7 @@ func (suite *Suite) TestCandidatesClientNamespaceScope() {
false,
10,
tt.clientNamespaceScope,
false,
)

suite.assertCandidates(chaoskube, tt.pods)
Expand Down Expand Up @@ -306,6 +313,7 @@ func (suite *Suite) TestCandidatesPodNameRegexp() {
false,
10,
v1.NamespaceAll,
false,
)

suite.assertCandidates(chaoskube, tt.pods)
Expand Down Expand Up @@ -347,6 +355,7 @@ func (suite *Suite) TestVictim() {
false,
10,
v1.NamespaceAll,
false,
)

suite.assertVictim(chaoskube, tt.victim)
Expand Down Expand Up @@ -402,6 +411,7 @@ func (suite *Suite) TestVictims() {
10,
tt.maxKill,
v1.NamespaceAll,
false,
)
suite.createPods(chaoskube.Client, podsInfo)

Expand All @@ -428,6 +438,7 @@ func (suite *Suite) TestNoVictimReturnsError() {
10,
1,
v1.NamespaceAll,
false,
)

_, err := chaoskube.Victims(context.Background())
Expand Down Expand Up @@ -463,6 +474,7 @@ func (suite *Suite) TestDeletePod() {
tt.dryRun,
10,
v1.NamespaceAll,
false,
)

victim := util.NewPod("default", "foo", v1.PodRunning)
Expand Down Expand Up @@ -494,6 +506,7 @@ func (suite *Suite) TestDeletePodNotFound() {
10,
1,
v1.NamespaceAll,
false,
)

victim := util.NewPod("default", "foo", v1.PodRunning)
Expand Down Expand Up @@ -726,6 +739,7 @@ func (suite *Suite) TestTerminateVictim() {
false,
10,
v1.NamespaceAll,
false,
)
chaoskube.Now = tt.now

Expand Down Expand Up @@ -758,6 +772,7 @@ func (suite *Suite) TestTerminateNoVictimLogsInfo() {
10,
1,
v1.NamespaceAll,
false,
)

err := chaoskube.TerminateVictims(context.Background())
Expand Down Expand Up @@ -792,7 +807,7 @@ func (suite *Suite) assertNotified(notifier *notifier.Noop) {
suite.Assert().Greater(notifier.Calls, 0)
}

func (suite *Suite) setupWithPods(labelSelector labels.Selector, annotations labels.Selector, kinds labels.Selector, namespaces labels.Selector, namespaceLabels labels.Selector, includedPodNames *regexp.Regexp, excludedPodNames *regexp.Regexp, excludedWeekdays []time.Weekday, excludedTimesOfDay []util.TimePeriod, excludedDaysOfYear []time.Time, timezone *time.Location, minimumAge time.Duration, dryRun bool, gracePeriod time.Duration, clientNamespaceScope string) *Chaoskube {
func (suite *Suite) setupWithPods(labelSelector labels.Selector, annotations labels.Selector, kinds labels.Selector, namespaces labels.Selector, namespaceLabels labels.Selector, includedPodNames *regexp.Regexp, excludedPodNames *regexp.Regexp, excludedWeekdays []time.Weekday, excludedTimesOfDay []util.TimePeriod, excludedDaysOfYear []time.Time, timezone *time.Location, minimumAge time.Duration, dryRun bool, gracePeriod time.Duration, clientNamespaceScope string, ignoreSingleReplicas bool) *Chaoskube {
chaoskube := suite.setup(
labelSelector,
annotations,
Expand All @@ -810,6 +825,7 @@ func (suite *Suite) setupWithPods(labelSelector labels.Selector, annotations lab
gracePeriod,
1,
clientNamespaceScope,
ignoreSingleReplicas,
)

for _, namespace := range []v1.Namespace{
Expand Down Expand Up @@ -845,7 +861,7 @@ func (suite *Suite) createPods(client kubernetes.Interface, podsInfo []podInfo)
}
}

func (suite *Suite) setup(labelSelector labels.Selector, annotations labels.Selector, kinds labels.Selector, namespaces labels.Selector, namespaceLabels labels.Selector, includedPodNames *regexp.Regexp, excludedPodNames *regexp.Regexp, excludedWeekdays []time.Weekday, excludedTimesOfDay []util.TimePeriod, excludedDaysOfYear []time.Time, timezone *time.Location, minimumAge time.Duration, dryRun bool, gracePeriod time.Duration, maxKill int, clientNamespaceScope string) *Chaoskube {
func (suite *Suite) setup(labelSelector labels.Selector, annotations labels.Selector, kinds labels.Selector, namespaces labels.Selector, namespaceLabels labels.Selector, includedPodNames *regexp.Regexp, excludedPodNames *regexp.Regexp, excludedWeekdays []time.Weekday, excludedTimesOfDay []util.TimePeriod, excludedDaysOfYear []time.Time, timezone *time.Location, minimumAge time.Duration, dryRun bool, gracePeriod time.Duration, maxKill int, clientNamespaceScope string, ignoreSingleReplicas bool) *Chaoskube {
logOutput.Reset()

client := fake.NewSimpleClientset()
Expand All @@ -871,6 +887,7 @@ func (suite *Suite) setup(labelSelector labels.Selector, annotations labels.Sele
maxKill,
testNotifier,
clientNamespaceScope,
ignoreSingleReplicas,
)
}

Expand Down Expand Up @@ -977,6 +994,7 @@ func (suite *Suite) TestMinimumAge() {
10,
1,
v1.NamespaceAll,
false,
)
chaoskube.Now = tt.now

Expand Down Expand Up @@ -1091,10 +1109,11 @@ func (suite *Suite) TestFilterByOwnerReference() {
baz1 := util.NewPod("default", "baz-1", v1.PodRunning)

for _, tt := range []struct {
seed int64
name string
pods []v1.Pod
expected []v1.Pod
seed int64
name string
pods []v1.Pod
expected []v1.Pod
ignoreSingleReplicas bool
}{
{
seed: 1000,
Expand Down Expand Up @@ -1126,10 +1145,31 @@ func (suite *Suite) TestFilterByOwnerReference() {
pods: []v1.Pod{baz, baz1},
expected: []v1.Pod{baz, baz1},
},
{
seed: 1000,
name: "1 pod, with parent, don't pick",
pods: []v1.Pod{foo},
expected: []v1.Pod{},
ignoreSingleReplicas: true,
},
{
seed: 1000,
name: "3 pods, 2 with same parent one without, pick first from first parent and ignore second",
pods: []v1.Pod{foo, foo1, bar},
expected: []v1.Pod{foo},
ignoreSingleReplicas: true,
},
{
seed: 1000,
name: "2 pods, different parents, don't pick",
pods: []v1.Pod{foo, bar},
expected: []v1.Pod{},
ignoreSingleReplicas: true,
},
} {
rand.Seed(tt.seed)

results := filterByOwnerReference(tt.pods)
results := filterByOwnerReference(tt.pods, tt.ignoreSingleReplicas)
suite.Require().Len(results, len(tt.expected))

// ensure returned pods are ordered by name
Expand Down Expand Up @@ -1160,6 +1200,7 @@ func (suite *Suite) TestNotifierCall() {
false,
10,
v1.NamespaceAll,
false,
)

victim := util.NewPod("default", "foo", v1.PodRunning)
Expand Down
4 changes: 4 additions & 0 deletions main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,7 @@ var (
logCaller bool
slackWebhook string
clientNamespaceScope string
ignoreSingleReplicas bool
)

func cliEnvVar(name string) string {
Expand Down Expand Up @@ -97,6 +98,7 @@ func init() {
kingpin.Flag("log-caller", "Include the calling function name and location in the log messages.").Envar(cliEnvVar("LOG_CALLER")).BoolVar(&logCaller)
kingpin.Flag("slack-webhook", "The address of the slack webhook for notifications").Envar(cliEnvVar("SLACK_WEBHOOK")).StringVar(&slackWebhook)
kingpin.Flag("client-namespace-scope", "Scope Kubernetes API calls to the given namespace. Defaults to v1.NamespaceAll which requires global read permission.").Envar(cliEnvVar("CLIENT_NAMESPACE_SCOPE")).Default(v1.NamespaceAll).StringVar(&clientNamespaceScope)
kingpin.Flag("ignore-single-replicas", "Ignore services with one replica when needed.").Envar(cliEnvVar("IGNORE_SINGLE_REPLICAS")).BoolVar(&ignoreSingleReplicas)
}

func main() {
Expand Down Expand Up @@ -141,6 +143,7 @@ func main() {
"logFormat": logFormat,
"slackWebhook": slackWebhook,
"clientNamespaceScope": clientNamespaceScope,
"ignoreSingleReplicas": ignoreSingleReplicas,
}).Debug("reading config")

log.WithFields(log.Fields{
Expand Down Expand Up @@ -234,6 +237,7 @@ func main() {
maxKill,
notifiers,
clientNamespaceScope,
ignoreSingleReplicas,
)

if metricsAddress != "" {
Expand Down

0 comments on commit 301a1d3

Please sign in to comment.