From 301a1d35a196717d4b6249a8fae54faab087e9b9 Mon Sep 17 00:00:00 2001 From: Mostafa Saber Date: Fri, 19 May 2023 21:26:04 +0200 Subject: [PATCH] Ignore services with 1 replica if needed Signed-off-by: Mostafa Saber --- chaoskube/chaoskube.go | 13 ++++-- chaoskube/chaoskube_test.go | 87 +++++++++++++++++++++++++++---------- main.go | 4 ++ 3 files changed, 78 insertions(+), 26 deletions(-) diff --git a/chaoskube/chaoskube.go b/chaoskube/chaoskube.go index e2295cf2..3e48f520 100644 --- a/chaoskube/chaoskube.go +++ b/chaoskube/chaoskube.go @@ -74,6 +74,9 @@ type Chaoskube struct { Notifier notifier.Notifier // namespace scope for the Kubernetes client ClientNamespaceScope string + + // ignoreSingleReplicas will make chaoskube ignore services with one replica when needed. + IgnoreSingleReplicas bool } var ( @@ -97,7 +100,7 @@ var ( // * a logger implementing logrus.FieldLogger to send log output to // * what specific terminator to use to imbue chaos on victim pods // * whether to enable/disable dry-run mode -func New(client kubernetes.Interface, labels, annotations, kinds, namespaces, namespaceLabels labels.Selector, includedPodNames, excludedPodNames *regexp.Regexp, excludedWeekdays []time.Weekday, excludedTimesOfDay []util.TimePeriod, excludedDaysOfYear []time.Time, timezone *time.Location, minimumAge time.Duration, logger log.FieldLogger, dryRun bool, terminator terminator.Terminator, maxKill int, notifier notifier.Notifier, clientNamespaceScope string) *Chaoskube { +func New(client kubernetes.Interface, labels, annotations, kinds, namespaces, namespaceLabels labels.Selector, includedPodNames, excludedPodNames *regexp.Regexp, excludedWeekdays []time.Weekday, excludedTimesOfDay []util.TimePeriod, excludedDaysOfYear []time.Time, timezone *time.Location, minimumAge time.Duration, logger log.FieldLogger, dryRun bool, terminator terminator.Terminator, maxKill int, notifier notifier.Notifier, clientNamespaceScope string, ignoreSingleReplicas bool) *Chaoskube { broadcaster := record.NewBroadcaster() broadcaster.StartRecordingToSink(&typedcorev1.EventSinkImpl{Interface: client.CoreV1().Events(clientNamespaceScope)}) recorder := broadcaster.NewRecorder(scheme.Scheme, v1.EventSource{Component: "chaoskube"}) @@ -124,6 +127,7 @@ func New(client kubernetes.Interface, labels, annotations, kinds, namespaces, na MaxKill: maxKill, Notifier: notifier, ClientNamespaceScope: clientNamespaceScope, + IgnoreSingleReplicas: ignoreSingleReplicas, } } @@ -239,7 +243,7 @@ func (c *Chaoskube) Candidates(ctx context.Context) ([]v1.Pod, error) { pods = filterTerminatingPods(pods) pods = filterByMinimumAge(pods, c.MinimumAge, c.Now()) pods = filterByPodName(pods, c.IncludedPodNames, c.ExcludedPodNames) - pods = filterByOwnerReference(pods) + pods = filterByOwnerReference(pods, c.IgnoreSingleReplicas) return pods, nil } @@ -513,7 +517,7 @@ func filterByPodName(pods []v1.Pod, includedPodNames, excludedPodNames *regexp.R return filteredList } -func filterByOwnerReference(pods []v1.Pod) []v1.Pod { +func filterByOwnerReference(pods []v1.Pod, ignoreSingleReplicas bool) []v1.Pod { owners := make(map[types.UID][]v1.Pod) filteredList := []v1.Pod{} for _, pod := range pods { @@ -531,6 +535,9 @@ func filterByOwnerReference(pods []v1.Pod) []v1.Pod { // For each owner reference select a random pod from its group for _, pods := range owners { + if ignoreSingleReplicas && len(pods) <= 1 { + continue + } filteredList = append(filteredList, util.RandomPodSubSlice(pods, 1)...) } diff --git a/chaoskube/chaoskube_test.go b/chaoskube/chaoskube_test.go index e9392a5d..bd213420 100644 --- a/chaoskube/chaoskube_test.go +++ b/chaoskube/chaoskube_test.go @@ -48,22 +48,23 @@ func (suite *Suite) SetupTest() { // TestNew tests that arguments are passed to the new instance correctly func (suite *Suite) TestNew() { var ( - client = fake.NewSimpleClientset() - labelSelector, _ = labels.Parse("foo=bar") - annotations, _ = labels.Parse("baz=waldo") - kinds, _ = labels.Parse("job") - namespaces, _ = labels.Parse("qux") - namespaceLabels, _ = labels.Parse("taz=wubble") - includedPodNames = regexp.MustCompile("foo") - excludedPodNames = regexp.MustCompile("bar") - excludedWeekdays = []time.Weekday{time.Friday} - excludedTimesOfDay = []util.TimePeriod{util.TimePeriod{}} - excludedDaysOfYear = []time.Time{time.Now()} - minimumAge = time.Duration(42) - dryRun = true - terminator = terminator.NewDeletePodTerminator(client, logger, 10*time.Second) - maxKill = 1 - notifier = testNotifier + client = fake.NewSimpleClientset() + labelSelector, _ = labels.Parse("foo=bar") + annotations, _ = labels.Parse("baz=waldo") + kinds, _ = labels.Parse("job") + namespaces, _ = labels.Parse("qux") + namespaceLabels, _ = labels.Parse("taz=wubble") + includedPodNames = regexp.MustCompile("foo") + excludedPodNames = regexp.MustCompile("bar") + excludedWeekdays = []time.Weekday{time.Friday} + excludedTimesOfDay = []util.TimePeriod{{}} + excludedDaysOfYear = []time.Time{time.Now()} + minimumAge = time.Duration(42) + dryRun = true + terminator = terminator.NewDeletePodTerminator(client, logger, 10*time.Second) + maxKill = 1 + notifier = testNotifier + ignoreSingleReplicas = false ) chaoskube := New( @@ -86,6 +87,7 @@ func (suite *Suite) TestNew() { maxKill, notifier, v1.NamespaceAll, + ignoreSingleReplicas, ) suite.Require().NotNil(chaoskube) @@ -105,6 +107,7 @@ func (suite *Suite) TestNew() { suite.Equal(logger, chaoskube.Logger) suite.Equal(dryRun, chaoskube.DryRun) suite.Equal(terminator, chaoskube.Terminator) + suite.Equal(ignoreSingleReplicas, chaoskube.IgnoreSingleReplicas) } // TestRunContextCanceled tests that a canceled context will exit the Run function. @@ -126,6 +129,7 @@ func (suite *Suite) TestRunContextCanceled() { 10, 1, v1.NamespaceAll, + false, ) ctx, cancel := context.WithCancel(context.Background()) @@ -182,6 +186,7 @@ func (suite *Suite) TestCandidates() { false, 10, v1.NamespaceAll, + false, ) suite.assertCandidates(chaoskube, tt.pods) @@ -228,6 +233,7 @@ func (suite *Suite) TestCandidatesNamespaceLabels() { false, 10, v1.NamespaceAll, + false, ) suite.assertCandidates(chaoskube, tt.pods) @@ -262,6 +268,7 @@ func (suite *Suite) TestCandidatesClientNamespaceScope() { false, 10, tt.clientNamespaceScope, + false, ) suite.assertCandidates(chaoskube, tt.pods) @@ -306,6 +313,7 @@ func (suite *Suite) TestCandidatesPodNameRegexp() { false, 10, v1.NamespaceAll, + false, ) suite.assertCandidates(chaoskube, tt.pods) @@ -347,6 +355,7 @@ func (suite *Suite) TestVictim() { false, 10, v1.NamespaceAll, + false, ) suite.assertVictim(chaoskube, tt.victim) @@ -402,6 +411,7 @@ func (suite *Suite) TestVictims() { 10, tt.maxKill, v1.NamespaceAll, + false, ) suite.createPods(chaoskube.Client, podsInfo) @@ -428,6 +438,7 @@ func (suite *Suite) TestNoVictimReturnsError() { 10, 1, v1.NamespaceAll, + false, ) _, err := chaoskube.Victims(context.Background()) @@ -463,6 +474,7 @@ func (suite *Suite) TestDeletePod() { tt.dryRun, 10, v1.NamespaceAll, + false, ) victim := util.NewPod("default", "foo", v1.PodRunning) @@ -494,6 +506,7 @@ func (suite *Suite) TestDeletePodNotFound() { 10, 1, v1.NamespaceAll, + false, ) victim := util.NewPod("default", "foo", v1.PodRunning) @@ -726,6 +739,7 @@ func (suite *Suite) TestTerminateVictim() { false, 10, v1.NamespaceAll, + false, ) chaoskube.Now = tt.now @@ -758,6 +772,7 @@ func (suite *Suite) TestTerminateNoVictimLogsInfo() { 10, 1, v1.NamespaceAll, + false, ) err := chaoskube.TerminateVictims(context.Background()) @@ -792,7 +807,7 @@ func (suite *Suite) assertNotified(notifier *notifier.Noop) { suite.Assert().Greater(notifier.Calls, 0) } -func (suite *Suite) setupWithPods(labelSelector labels.Selector, annotations labels.Selector, kinds labels.Selector, namespaces labels.Selector, namespaceLabels labels.Selector, includedPodNames *regexp.Regexp, excludedPodNames *regexp.Regexp, excludedWeekdays []time.Weekday, excludedTimesOfDay []util.TimePeriod, excludedDaysOfYear []time.Time, timezone *time.Location, minimumAge time.Duration, dryRun bool, gracePeriod time.Duration, clientNamespaceScope string) *Chaoskube { +func (suite *Suite) setupWithPods(labelSelector labels.Selector, annotations labels.Selector, kinds labels.Selector, namespaces labels.Selector, namespaceLabels labels.Selector, includedPodNames *regexp.Regexp, excludedPodNames *regexp.Regexp, excludedWeekdays []time.Weekday, excludedTimesOfDay []util.TimePeriod, excludedDaysOfYear []time.Time, timezone *time.Location, minimumAge time.Duration, dryRun bool, gracePeriod time.Duration, clientNamespaceScope string, ignoreSingleReplicas bool) *Chaoskube { chaoskube := suite.setup( labelSelector, annotations, @@ -810,6 +825,7 @@ func (suite *Suite) setupWithPods(labelSelector labels.Selector, annotations lab gracePeriod, 1, clientNamespaceScope, + ignoreSingleReplicas, ) for _, namespace := range []v1.Namespace{ @@ -845,7 +861,7 @@ func (suite *Suite) createPods(client kubernetes.Interface, podsInfo []podInfo) } } -func (suite *Suite) setup(labelSelector labels.Selector, annotations labels.Selector, kinds labels.Selector, namespaces labels.Selector, namespaceLabels labels.Selector, includedPodNames *regexp.Regexp, excludedPodNames *regexp.Regexp, excludedWeekdays []time.Weekday, excludedTimesOfDay []util.TimePeriod, excludedDaysOfYear []time.Time, timezone *time.Location, minimumAge time.Duration, dryRun bool, gracePeriod time.Duration, maxKill int, clientNamespaceScope string) *Chaoskube { +func (suite *Suite) setup(labelSelector labels.Selector, annotations labels.Selector, kinds labels.Selector, namespaces labels.Selector, namespaceLabels labels.Selector, includedPodNames *regexp.Regexp, excludedPodNames *regexp.Regexp, excludedWeekdays []time.Weekday, excludedTimesOfDay []util.TimePeriod, excludedDaysOfYear []time.Time, timezone *time.Location, minimumAge time.Duration, dryRun bool, gracePeriod time.Duration, maxKill int, clientNamespaceScope string, ignoreSingleReplicas bool) *Chaoskube { logOutput.Reset() client := fake.NewSimpleClientset() @@ -871,6 +887,7 @@ func (suite *Suite) setup(labelSelector labels.Selector, annotations labels.Sele maxKill, testNotifier, clientNamespaceScope, + ignoreSingleReplicas, ) } @@ -977,6 +994,7 @@ func (suite *Suite) TestMinimumAge() { 10, 1, v1.NamespaceAll, + false, ) chaoskube.Now = tt.now @@ -1091,10 +1109,11 @@ func (suite *Suite) TestFilterByOwnerReference() { baz1 := util.NewPod("default", "baz-1", v1.PodRunning) for _, tt := range []struct { - seed int64 - name string - pods []v1.Pod - expected []v1.Pod + seed int64 + name string + pods []v1.Pod + expected []v1.Pod + ignoreSingleReplicas bool }{ { seed: 1000, @@ -1126,10 +1145,31 @@ func (suite *Suite) TestFilterByOwnerReference() { pods: []v1.Pod{baz, baz1}, expected: []v1.Pod{baz, baz1}, }, + { + seed: 1000, + name: "1 pod, with parent, don't pick", + pods: []v1.Pod{foo}, + expected: []v1.Pod{}, + ignoreSingleReplicas: true, + }, + { + seed: 1000, + name: "3 pods, 2 with same parent one without, pick first from first parent and ignore second", + pods: []v1.Pod{foo, foo1, bar}, + expected: []v1.Pod{foo}, + ignoreSingleReplicas: true, + }, + { + seed: 1000, + name: "2 pods, different parents, don't pick", + pods: []v1.Pod{foo, bar}, + expected: []v1.Pod{}, + ignoreSingleReplicas: true, + }, } { rand.Seed(tt.seed) - results := filterByOwnerReference(tt.pods) + results := filterByOwnerReference(tt.pods, tt.ignoreSingleReplicas) suite.Require().Len(results, len(tt.expected)) // ensure returned pods are ordered by name @@ -1160,6 +1200,7 @@ func (suite *Suite) TestNotifierCall() { false, 10, v1.NamespaceAll, + false, ) victim := util.NewPod("default", "foo", v1.PodRunning) diff --git a/main.go b/main.go index 731bb025..69879a5c 100644 --- a/main.go +++ b/main.go @@ -62,6 +62,7 @@ var ( logCaller bool slackWebhook string clientNamespaceScope string + ignoreSingleReplicas bool ) func cliEnvVar(name string) string { @@ -97,6 +98,7 @@ func init() { kingpin.Flag("log-caller", "Include the calling function name and location in the log messages.").Envar(cliEnvVar("LOG_CALLER")).BoolVar(&logCaller) kingpin.Flag("slack-webhook", "The address of the slack webhook for notifications").Envar(cliEnvVar("SLACK_WEBHOOK")).StringVar(&slackWebhook) kingpin.Flag("client-namespace-scope", "Scope Kubernetes API calls to the given namespace. Defaults to v1.NamespaceAll which requires global read permission.").Envar(cliEnvVar("CLIENT_NAMESPACE_SCOPE")).Default(v1.NamespaceAll).StringVar(&clientNamespaceScope) + kingpin.Flag("ignore-single-replicas", "Ignore services with one replica when needed.").Envar(cliEnvVar("IGNORE_SINGLE_REPLICAS")).BoolVar(&ignoreSingleReplicas) } func main() { @@ -141,6 +143,7 @@ func main() { "logFormat": logFormat, "slackWebhook": slackWebhook, "clientNamespaceScope": clientNamespaceScope, + "ignoreSingleReplicas": ignoreSingleReplicas, }).Debug("reading config") log.WithFields(log.Fields{ @@ -234,6 +237,7 @@ func main() { maxKill, notifiers, clientNamespaceScope, + ignoreSingleReplicas, ) if metricsAddress != "" {