Merge pull request #2 from jakub-dzon/suport-for-node-placement

Support for node placement

go.mod

@@ -13,13 +13,14 @@ require (
 	github.com/openshift/custom-resource-status v0.0.0-20200602122900-c002fd1547ca
 	github.com/prometheus/client_golang v1.1.0 // indirect
 	go.uber.org/multierr v1.3.0 // indirect
-	golang.org/x/tools v0.0.0-20200115044656-831fdb1e1868 // indirect
+	golang.org/x/tools v0.0.0-20200616195046-dc31b401abb5
 	k8s.io/api v0.18.6
 	k8s.io/apiextensions-apiserver v0.18.6
 	k8s.io/apimachinery v0.18.6
 	k8s.io/client-go v8.0.0+incompatible
 	k8s.io/kubernetes v1.14.0
 	sigs.k8s.io/controller-runtime v0.6.2
+	sigs.k8s.io/controller-tools v0.4.0
 )
 
 replace (

pkg/sdk/api/types.go

@@ -2,6 +2,7 @@ package api
 
 import (
 	conditions "github.com/openshift/custom-resource-status/conditions/v1"
+	corev1 "k8s.io/api/core/v1"
 )
 
 // Phase is the current phase of the deployment
@@ -43,6 +44,30 @@ type Status struct {
 	ObservedVersion string `json:"observedVersion,omitempty" optional:"true"`
 }
 
+// NodePlacement describes node scheduling configuration.
+type NodePlacement struct {
+	// nodeSelector is the node selector applied to the relevant kind of pods
+	// It specifies a map of key-value pairs: for the pod to be eligible to run on a node,
+	// the node must have each of the indicated key-value pairs as labels
+	// (it can have additional labels as well).
+	// See https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector
+	// +optional
+	NodeSelector map[string]string `json:"nodeSelector,omitempty"`
+
+	// affinity enables pod affinity/anti-affinity placement expanding the types of constraints
+	// that can be expressed with nodeSelector.
+	// affinity is going to be applied to the relevant kind of pods in parallel with nodeSelector
+	// See https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity
+	// +optional
+	Affinity corev1.Affinity `json:"affinity,omitempty"`
+
+	// tolerations is a list of tolerations applied to the relevant kind of pods
+	// See https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ for more info.
+	// These are additional tolerations other than default ones.
+	// +optional
+	Tolerations []corev1.Toleration `json:"tolerations,omitempty"`
+}
+
 // DeepCopyInto is copying the receiver, writing into out. in must be non-nil.
 func (in *Status) DeepCopyInto(out *Status) {
 	*out = *in
@@ -54,3 +79,43 @@ func (in *Status) DeepCopyInto(out *Status) {
 		}
 	}
 }
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *NodePlacement) DeepCopyInto(out *NodePlacement) {
+	*out = *in
+	if in.NodeSelector != nil {
+		in, out := &in.NodeSelector, &out.NodeSelector
+		*out = make(map[string]string, len(*in))
+		for key, val := range *in {
+			(*out)[key] = val
+		}
+	}
+	in.Affinity.DeepCopyInto(&out.Affinity)
+	if in.Tolerations != nil {
+		in, out := &in.Tolerations, &out.Tolerations
+		*out = make([]corev1.Toleration, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NodePlacement.
+func (in *NodePlacement) DeepCopy() *NodePlacement {
+	if in == nil {
+		return nil
+	}
+	out := new(NodePlacement)
+	in.DeepCopyInto(out)
+	return out
+}
+
+func (NodePlacement) SwaggerDoc() map[string]string {
+	return map[string]string{
+		"":             "NodePlacement describes  node scheduling configuration.",
+		"nodeSelector": "nodeSelector is the node selector applied to the relevant kind of pods\nIt specifies a map of key-value pairs: for the pod to be eligible to run on a node,\nthe node must have each of the indicated key-value pairs as labels\n(it can have additional labels as well).\nSee https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector\n+optional",
+		"affinity":     "affinity enables pod affinity/anti-affinity placement expanding the types of constraints\nthat can be expressed with nodeSelector.\naffinity is going to be applied to the relevant kind of pods in parallel with nodeSelector\nSee https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity\n+optional",
+		"tolerations":  "tolerations is a list of tolerations applied to the relevant kind of pods\nSee https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ for more info.\nThese are additional tolerations other than default ones.\n+optional",
+	}
+}

pkg/sdk/reconciler/reconciler.go

@@ -115,7 +115,7 @@ type Reconciler struct {
 func (r *Reconciler) Reconcile(request reconcile.Request, operatorVersion string, reqLogger logr.Logger) (reconcile.Result, error) {
 	// Fetch the CR instance
 	// check at cluster level
-	cr, err := r.getCr(request.NamespacedName)
+	cr, err := r.GetCr(request.NamespacedName)
 	if err != nil {
 		if errors.IsNotFound(err) {
 			// Request object not found, could have been deleted after reconcile request.
@@ -739,7 +739,7 @@ func (r *Reconciler) CrInit(cr controllerutil.Object, operatorVersion string) er
 	return r.CrUpdate(sdkapi.PhaseDeploying, cr)
 }
 
-func (r *Reconciler) getCr(name types.NamespacedName) (controllerutil.Object, error) {
+func (r *Reconciler) GetCr(name types.NamespacedName) (controllerutil.Object, error) {
 	cr := r.crManager.Create()
 	crKey := client.ObjectKey{Namespace: "", Name: name.Name}
 	err := r.client.Get(context.TODO(), crKey, cr)

pkg/sdk/reconciler/reconciler_test.go

@@ -521,7 +521,7 @@ var _ = Describe("Reconciler", func() {
 	},
 		Entry("verify - unused deployment deleted",
 			func() (runtime.Object, error) {
-				deployment := testcr.ResourceBuilder.CreateDeployment("fake-deployment", testcr.Namespace, "match-key", "match-value", "", int32(1), corev1.PodSpec{})
+				deployment := testcr.ResourceBuilder.CreateDeployment("fake-deployment", testcr.Namespace, "match-key", "match-value", "", int32(1), corev1.PodSpec{}, nil)
 				return deployment, nil
 			}),
 

pkg/sdk/resources/openapi/crdgenerator/generator.go

@@ -0,0 +1,75 @@
+package crdgenerator
+
+import (
+	"github.com/kubevirt/controller-lifecycle-operator-sdk/pkg/sdk"
+	"golang.org/x/tools/go/packages"
+	extv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"sigs.k8s.io/controller-tools/pkg/loader"
+	"sigs.k8s.io/controller-tools/pkg/markers"
+
+	crdgen "sigs.k8s.io/controller-tools/pkg/crd"
+	crdmarkers "sigs.k8s.io/controller-tools/pkg/crd/markers"
+)
+
+// CreateCRD creates CRD with given Group Kind and marks its version with given name based on sources find in the working directory
+func CreateCRD(groupKind schema.GroupKind, crVersionName string) *extv1.CustomResourceDefinition {
+	pkgs, err := loader.LoadRoots(sdk.GetOperatorToplevel() + "/...")
+	if err != nil {
+		panic(err)
+	}
+	reg := &markers.Registry{}
+	crdmarkers.Register(reg)
+
+	parser := &crdgen.Parser{
+		Collector: &markers.Collector{Registry: reg},
+		Checker:   &loader.TypeChecker{},
+	}
+	crdgen.AddKnownTypes(parser)
+	if len(pkgs) == 0 {
+		panic("Failed identifying packages")
+	}
+	for _, p := range pkgs {
+		parser.NeedPackage(p)
+	}
+	parser.NeedCRDFor(groupKind, nil)
+	for _, p := range pkgs {
+		err = PackageErrors(p, packages.TypeError)
+		if err != nil {
+			panic(err)
+		}
+	}
+	c := parser.CustomResourceDefinitions[groupKind]
+	// enforce validation of CR name to prevent multiple CRs
+
+	for _, v := range c.Spec.Versions {
+		v.Schema.OpenAPIV3Schema.Properties["metadata"] = extv1.JSONSchemaProps{
+			Type: "object",
+			Properties: map[string]extv1.JSONSchemaProps{
+				"name": {
+					Type:    "string",
+					Pattern: crVersionName,
+				},
+			},
+		}
+	}
+
+	return &c
+}
+
+func PackageErrors(pkg *loader.Package, filterKinds ...packages.ErrorKind) error {
+	toSkip := make(map[packages.ErrorKind]struct{})
+	for _, errKind := range filterKinds {
+		toSkip[errKind] = struct{}{}
+	}
+	var outErr error
+	packages.Visit([]*packages.Package{pkg.Package}, nil, func(pkgRaw *packages.Package) {
+		for _, err := range pkgRaw.Errors {
+			if _, skip := toSkip[err.Kind]; skip {
+				continue
+			}
+			outErr = err
+		}
+	})
+	return outErr
+}

pkg/sdk/resources/resources.go

@@ -3,6 +3,8 @@ package resources
 import (
 	"fmt"
 
+	sdkapi "github.com/kubevirt/controller-lifecycle-operator-sdk/pkg/sdk/api"
+
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -36,14 +38,14 @@ func (b *ResourceBuilder) WithOperatorLabels(labels map[string]string) map[strin
 // CreateOperatorDeployment creates deployment
 func (b *ResourceBuilder) CreateOperatorDeployment(name, namespace, matchKey, matchValue, serviceAccount string, numReplicas int32, podSpec corev1.PodSpec) *appsv1.Deployment {
 	labels := WithLabels(map[string]string{matchKey: matchValue}, b.operatorLabels)
-	return CreateDeployment(name, namespace, labels, labels, numReplicas, podSpec, serviceAccount)
+	return CreateDeployment(name, namespace, labels, labels, numReplicas, podSpec, serviceAccount, nil)
 }
 
 // CreateDeployment creates deployment
-func (b *ResourceBuilder) CreateDeployment(name, namespace, matchKey, matchValue, serviceAccount string, numReplicas int32, podSpec corev1.PodSpec) *appsv1.Deployment {
+func (b *ResourceBuilder) CreateDeployment(name, namespace, matchKey, matchValue, serviceAccount string, numReplicas int32, podSpec corev1.PodSpec, infraNodePlacement *sdkapi.NodePlacement) *appsv1.Deployment {
 	selectorMatchMap := map[string]string{matchKey: matchValue}
 	finalLabels := WithLabels(map[string]string{matchKey: matchValue}, b.commonLabels)
-	return CreateDeployment(name, namespace, selectorMatchMap, finalLabels, numReplicas, podSpec, serviceAccount)
+	return CreateDeployment(name, namespace, selectorMatchMap, finalLabels, numReplicas, podSpec, serviceAccount, infraNodePlacement)
 }
 
 // CreateContainer creates container
@@ -98,7 +100,7 @@ func CreateConfigMap(name string, labels map[string]string) *corev1.ConfigMap {
 }
 
 // CreateDeployment creates deployment
-func CreateDeployment(name string, namespace string, selectorMatchMap map[string]string, labels map[string]string, numReplicas int32, podSpec corev1.PodSpec, serviceAccount string) *appsv1.Deployment {
+func CreateDeployment(name string, namespace string, selectorMatchMap map[string]string, labels map[string]string, numReplicas int32, podSpec corev1.PodSpec, serviceAccount string, infraNodePlacement *sdkapi.NodePlacement) *appsv1.Deployment {
 	deployment := &appsv1.Deployment{
 		TypeMeta: metav1.TypeMeta{
 			APIVersion: "apps/v1",
@@ -122,6 +124,11 @@ func CreateDeployment(name string, namespace string, selectorMatchMap map[string
 			},
 		},
 	}
+	if infraNodePlacement != nil {
+		deployment.Spec.Template.Spec.NodeSelector = infraNodePlacement.NodeSelector
+		deployment.Spec.Template.Spec.Tolerations = infraNodePlacement.Tolerations
+		deployment.Spec.Template.Spec.Affinity = &infraNodePlacement.Affinity
+	}
 	if serviceAccount != "" {
 		deployment.Spec.Template.Spec.ServiceAccountName = serviceAccount
 	}

pkg/sdk/util.go

@@ -220,3 +220,15 @@ func SetLastAppliedConfiguration(obj metav1.Object, lastAppliedConfigAnnotation
 
 	return nil
 }
+
+// GetOperatorToplevel returns the top level source directory of the operator.
+// Can be overridden using the environment variable "OPERATOR_DIR".
+func GetOperatorToplevel() string {
+	// When running unit tests, we pass the OPERATOR_DIR environment variable, because
+	// the tests run in their own directory and module.
+	cwd := os.Getenv("OPERATOR_DIR")
+	if cwd == "" {
+		cwd, _ = os.Getwd()
+	}
+	return cwd
+}