RFC Implementation Supporting ODCR

aws · Sep 30, 2024 · 8a3d6bf · 8a3d6bf
1 parent 79fbbac
commit 8a3d6bf
Show file tree

Hide file tree

Showing 29 changed files with 1,247 additions and 176 deletions.
diff --git a/charts/karpenter-crd/templates/karpenter.k8s.aws_ec2nodeclasses.yaml b/charts/karpenter-crd/templates/karpenter.k8s.aws_ec2nodeclasses.yaml
@@ -236,6 +236,40 @@ spec:
                   x-kubernetes-validations:
                     - message: must have only one blockDeviceMappings with rootVolume
                       rule: self.filter(x, has(x.rootVolume)?x.rootVolume==true:false).size() <= 1
+                capacityReservationSelectorTerms:
+                  description: CapacityReservationSelectorTerms is a list of or Capacity Reservation selector terms. The terms are ORed.
+                  items:
+                    description: |-
+                      CapacityReservationSelectorTerms specify selectors which are ORed together to generate
+                      a list of filters against the EC2 DescribeCapacityReservation API
+                      ID cannot be specified with any other field within a single selector
+                      All other fields are not mutually exclusive and can be combined
+                    properties:
+                      id:
+                        description: |-
+                          The id for the Capacity Reservation
+                          Specifying '*' for this field selects all ids
+                        type: string
+                      ownerId:
+                        description: |-
+                          The id of the AWS account that owns the Capacity Reservation
+                          If no ownerID is specified, only ODCRs owned by the current account will be used
+                          Specifying '*' for this field selects all ownerIDs
+                        type: string
+                      tags:
+                        additionalProperties:
+                          type: string
+                        description: |-
+                          Tags is a map of key/value tags used to select capacity reservations
+                          Specifying '*' for a value selects all values for a given tag key.
+                        maxProperties: 20
+                        type: object
+                        x-kubernetes-validations:
+                          - message: empty tag keys or values aren't supported
+                            rule: self.all(k, k != '' && self[k] != '')
+                    type: object
+                  maxItems: 30
+                  type: array
                 context:
                   description: |-
                     Context is a Reserved field in EC2 APIs
@@ -634,6 +668,64 @@ spec:
                       - requirements
                     type: object
                   type: array
+                capacityReservations:
+                  description: |-
+                    CapacityReservations contains the current Capacity Reservations values that are available to the
+                    cluster under the CapacityReservations selectors.
+                  items:
+                    description: CapacityReservation contains resolved Capacity Reservation selector values utilized for node launch
+                    properties:
+                      availabilityZone:
+                        description: AvailabilityZone of the Capacity Reservation
+                        type: string
+                      availableInstanceCount:
+                        description: Available Instance Count of the Capacity Reservation
+                        type: integer
+                      endDate:
+                        description: |-
+                          The date and time at which the Capacity Reservation expires. When a Capacity
+                          Reservation expires, the reserved capacity is released and you can no longer
+                          launch instances into it. The Capacity Reservation's state changes to expired
+                          when it reaches its end date and time.
+                        type: string
+                      id:
+                        description: ID of the Capacity Reservation
+                        type: string
+                      instanceMatchCriteria:
+                        description: |-
+                          Indicates the type of instance launches that the Capacity Reservation accepts. The options include:
+                            - open:
+                                The Capacity Reservation accepts all instances that have
+                                matching attributes (instance type, platform, and Availability
+                                Zone). Instances that have matching attributes launch into the
+                                Capacity Reservation automatically without specifying any
+                                additional parameters.
+                            - targeted:
+                                The Capacity Reservation only accepts instances that
+                                have matching attributes (instance type, platform, and
+                                Availability Zone), and explicitly target the Capacity
+                                Reservation. This ensures that only permitted instances can use
+                                the reserved capacity.
+                        type: string
+                      instanceType:
+                        description: Instance Type of the Capacity Reservation
+                        type: string
+                      ownerId:
+                        description: The id of the AWS account that owns the Capacity Reservation
+                        type: string
+                      totalInstanceCount:
+                        description: Total Instance Count of the Capacity Reservation
+                        type: integer
+                    required:
+                      - availabilityZone
+                      - availableInstanceCount
+                      - id
+                      - instanceMatchCriteria
+                      - instanceType
+                      - ownerId
+                      - totalInstanceCount
+                    type: object
+                  type: array
                 conditions:
                   description: Conditions contains signals for health and readiness
                   items:

diff --git a/cmd/controller/main.go b/cmd/controller/main.go
@@ -65,6 +65,7 @@ func main() {
 			op.AMIProvider,
 			op.LaunchTemplateProvider,
 			op.InstanceTypesProvider,
+			op.CapacityReservationProvider,
 		)...).
 		Start(ctx)
 }
diff --git a/go.mod b/go.mod
@@ -100,3 +100,5 @@ require (
 	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
 	sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect
 )
+
+replace sigs.k8s.io/karpenter v1.0.1-0.20240921204958-04a921c00ad8 => github.com/tvonhacht-apple/karpenter v0.0.0-20240930181435-5767eb96c2b1
diff --git a/go.sum b/go.sum
@@ -157,6 +157,8 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO
 github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
 github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
 github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+github.com/tvonhacht-apple/karpenter v0.0.0-20240930181435-5767eb96c2b1 h1:PyjMAjguMsms3zghDm5+VE6nelfQQp0z9t24gfMcvAs=
+github.com/tvonhacht-apple/karpenter v0.0.0-20240930181435-5767eb96c2b1/go.mod h1:odwr/cPdG0y6oTMHsItGMazdAMGUJzz8Kd1+SfrqjKs=
 github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM=
 github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
@@ -276,8 +278,6 @@ sigs.k8s.io/controller-runtime v0.19.0 h1:nWVM7aq+Il2ABxwiCizrVDSlmDcshi9llbaFbC
 sigs.k8s.io/controller-runtime v0.19.0/go.mod h1:iRmWllt8IlaLjvTTDLhRBXIEtkCK6hwVBJJsYS9Ajf4=
 sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo=
 sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0=
-sigs.k8s.io/karpenter v1.0.1-0.20240921204958-04a921c00ad8 h1:F0eGjPSRLd6emk614DNjxgNc97AkVqRSvpnSxxxm0hM=
-sigs.k8s.io/karpenter v1.0.1-0.20240921204958-04a921c00ad8/go.mod h1:U70Yuu2wiH1nBC5/nDYYpqYNadYnMgetmVQ5s2NouyU=
 sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4=
 sigs.k8s.io/structured-merge-diff/v4 v4.4.1/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08=
 sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E=

diff --git a/pkg/apis/crds/karpenter.k8s.aws_ec2nodeclasses.yaml b/pkg/apis/crds/karpenter.k8s.aws_ec2nodeclasses.yaml
@@ -236,6 +236,40 @@ spec:
                   x-kubernetes-validations:
                     - message: must have only one blockDeviceMappings with rootVolume
                       rule: self.filter(x, has(x.rootVolume)?x.rootVolume==true:false).size() <= 1
+                capacityReservationSelectorTerms:
+                  description: CapacityReservationSelectorTerms is a list of or Capacity Reservation selector terms. The terms are ORed.
+                  items:
+                    description: |-
+                      CapacityReservationSelectorTerms specify selectors which are ORed together to generate
+                      a list of filters against the EC2 DescribeCapacityReservation API
+                      ID cannot be specified with any other field within a single selector
+                      All other fields are not mutually exclusive and can be combined
+                    properties:
+                      id:
+                        description: |-
+                          The id for the Capacity Reservation
+                          Specifying '*' for this field selects all ids
+                        type: string
+                      ownerId:
+                        description: |-
+                          The id of the AWS account that owns the Capacity Reservation
+                          If no ownerID is specified, only ODCRs owned by the current account will be used
+                          Specifying '*' for this field selects all ownerIDs
+                        type: string
+                      tags:
+                        additionalProperties:
+                          type: string
+                        description: |-
+                          Tags is a map of key/value tags used to select capacity reservations
+                          Specifying '*' for a value selects all values for a given tag key.
+                        maxProperties: 20
+                        type: object
+                        x-kubernetes-validations:
+                          - message: empty tag keys or values aren't supported
+                            rule: self.all(k, k != '' && self[k] != '')
+                    type: object
+                  maxItems: 30
+                  type: array
                 context:
                   description: |-
                     Context is a Reserved field in EC2 APIs
@@ -634,6 +668,64 @@ spec:
                       - requirements
                     type: object
                   type: array
+                capacityReservations:
+                  description: |-
+                    CapacityReservations contains the current Capacity Reservations values that are available to the
+                    cluster under the CapacityReservations selectors.
+                  items:
+                    description: CapacityReservation contains resolved Capacity Reservation selector values utilized for node launch
+                    properties:
+                      availabilityZone:
+                        description: AvailabilityZone of the Capacity Reservation
+                        type: string
+                      availableInstanceCount:
+                        description: Available Instance Count of the Capacity Reservation
+                        type: integer
+                      endDate:
+                        description: |-
+                          The date and time at which the Capacity Reservation expires. When a Capacity
+                          Reservation expires, the reserved capacity is released and you can no longer
+                          launch instances into it. The Capacity Reservation's state changes to expired
+                          when it reaches its end date and time.
+                        type: string
+                      id:
+                        description: ID of the Capacity Reservation
+                        type: string
+                      instanceMatchCriteria:
+                        description: |-
+                          Indicates the type of instance launches that the Capacity Reservation accepts. The options include:
+                            - open:
+                                The Capacity Reservation accepts all instances that have
+                                matching attributes (instance type, platform, and Availability
+                                Zone). Instances that have matching attributes launch into the
+                                Capacity Reservation automatically without specifying any
+                                additional parameters.
+                            - targeted:
+                                The Capacity Reservation only accepts instances that
+                                have matching attributes (instance type, platform, and
+                                Availability Zone), and explicitly target the Capacity
+                                Reservation. This ensures that only permitted instances can use
+                                the reserved capacity.
+                        type: string
+                      instanceType:
+                        description: Instance Type of the Capacity Reservation
+                        type: string
+                      ownerId:
+                        description: The id of the AWS account that owns the Capacity Reservation
+                        type: string
+                      totalInstanceCount:
+                        description: Total Instance Count of the Capacity Reservation
+                        type: integer
+                    required:
+                      - availabilityZone
+                      - availableInstanceCount
+                      - id
+                      - instanceMatchCriteria
+                      - instanceType
+                      - ownerId
+                      - totalInstanceCount
+                    type: object
+                  type: array
                 conditions:
                   description: Conditions contains signals for health and readiness
                   items:

diff --git a/pkg/apis/v1/ec2nodeclass.go b/pkg/apis/v1/ec2nodeclass.go
@@ -64,6 +64,10 @@ type EC2NodeClassSpec struct {
 	// +kubebuilder:validation:Enum:={AL2,AL2023,Bottlerocket,Custom,Windows2019,Windows2022}
 	// +optional
 	AMIFamily *string `json:"amiFamily,omitempty" hash:"ignore"`
+	// CapacityReservationSelectorTerms is a list of or Capacity Reservation selector terms. The terms are ORed.
+	// +kubebuilder:validation:MaxItems:=30
+	// +optional
+	CapacityReservationSelectorTerms []CapacityReservationSelectorTerm `json:"capacityReservationSelectorTerms,omitempty" hash:"ignore"`
 	// UserData to be applied to the provisioned nodes.
 	// It must be in the appropriate format based on the AMIFamily in use. Karpenter will merge certain fields into
 	// this UserData to ensure nodes are being provisioned with the correct configuration.
@@ -273,6 +277,28 @@ type KubeletConfiguration struct {
 	CPUCFSQuota *bool `json:"cpuCFSQuota,omitempty"`
 }
 
+// CapacityReservationSelectorTerms specify selectors which are ORed together to generate
+// a list of filters against the EC2 DescribeCapacityReservation API
+// ID cannot be specified with any other field within a single selector
+// All other fields are not mutually exclusive and can be combined
+type CapacityReservationSelectorTerm struct {
+	// The id for the Capacity Reservation
+	// Specifying '*' for this field selects all ids
+	// +optional
+	ID string `json:"id,omitempty"`
+	// Tags is a map of key/value tags used to select capacity reservations
+	// Specifying '*' for a value selects all values for a given tag key.
+	// +kubebuilder:validation:XValidation:message="empty tag keys or values aren't supported",rule="self.all(k, k != '' && self[k] != '')"
+	// +kubebuilder:validation:MaxProperties:=20
+	// +optional
+	Tags map[string]string `json:"tags,omitempty"`
+	// The id of the AWS account that owns the Capacity Reservation
+	// If no ownerID is specified, only ODCRs owned by the current account will be used
+	// Specifying '*' for this field selects all ownerIDs
+	// +optional
+	OwnerID string `json:"ownerId,omitempty"`
+}
+
 // MetadataOptions contains parameters for specifying the exposure of the
 // Instance Metadata Service to provisioned EC2 nodes.
 type MetadataOptions struct {

diff --git a/pkg/apis/v1/ec2nodeclass_status.go b/pkg/apis/v1/ec2nodeclass_status.go
@@ -62,8 +62,55 @@ type AMI struct {
 	Requirements []corev1.NodeSelectorRequirement `json:"requirements"`
 }
 
+// CapacityReservation contains resolved Capacity Reservation selector values utilized for node launch
+type CapacityReservation struct {
+	// ID of the Capacity Reservation
+	// +required
+	ID string `json:"id"`
+	// AvailabilityZone of the Capacity Reservation
+	// +required
+	AvailabilityZone string `json:"availabilityZone"`
+	// Available Instance Count of the Capacity Reservation
+	// +required
+	AvailableInstanceCount int `json:"availableInstanceCount"`
+	// The date and time at which the Capacity Reservation expires. When a Capacity
+	// Reservation expires, the reserved capacity is released and you can no longer
+	// launch instances into it. The Capacity Reservation's state changes to expired
+	// when it reaches its end date and time.
+	// +optional
+	EndDate *string `json:"endDate,omitempty"`
+	// Indicates the type of instance launches that the Capacity Reservation accepts. The options include:
+	//   - open:
+	//       The Capacity Reservation accepts all instances that have
+	//       matching attributes (instance type, platform, and Availability
+	//       Zone). Instances that have matching attributes launch into the
+	//       Capacity Reservation automatically without specifying any
+	//       additional parameters.
+	//   - targeted:
+	//       The Capacity Reservation only accepts instances that
+	//       have matching attributes (instance type, platform, and
+	//       Availability Zone), and explicitly target the Capacity
+	//       Reservation. This ensures that only permitted instances can use
+	//       the reserved capacity.
+	// +required
+	InstanceMatchCriteria string `json:"instanceMatchCriteria"`
+	// Instance Type of the Capacity Reservation
+	// +required
+	InstanceType string `json:"instanceType"`
+	// The id of the AWS account that owns the Capacity Reservation
+	// +required
+	OwnerID string `json:"ownerId"`
+	// Total Instance Count of the Capacity Reservation
+	// +required
+	TotalInstanceCount int `json:"totalInstanceCount"`
+}
+
 // EC2NodeClassStatus contains the resolved state of the EC2NodeClass
 type EC2NodeClassStatus struct {
+	// CapacityReservations contains the current Capacity Reservations values that are available to the
+	// cluster under the CapacityReservations selectors.
+	// +optional
+	CapacityReservations []CapacityReservation `json:"capacityReservations,omitempty"`
 	// Subnets contains the current Subnet values that are available to the
 	// cluster under the subnet selectors.
 	// +optional

diff --git a/pkg/apis/v1/labels.go b/pkg/apis/v1/labels.go
@@ -29,6 +29,7 @@ import (
 func init() {
 	karpv1.RestrictedLabelDomains = karpv1.RestrictedLabelDomains.Insert(RestrictedLabelDomains...)
 	karpv1.WellKnownLabels = karpv1.WellKnownLabels.Insert(
+		LabelCapactiyReservationID,
 		LabelInstanceHypervisor,
 		LabelInstanceEncryptionInTransitSupported,
 		LabelInstanceCategory,
@@ -124,6 +125,7 @@ var (
 	AnnotationClusterNameTaggedCompatability  = apis.CompatibilityGroup + "/cluster-name-tagged"
 	AnnotationEC2NodeClassHashVersion         = apis.Group + "/ec2nodeclass-hash-version"
 	AnnotationInstanceTagged                  = apis.Group + "/tagged"
+	LabelCapactiyReservationID                = apis.Group + "/capacity-reservation-id"
 
 	TagNodeClaim             = coreapis.Group + "/nodeclaim"
 	TagManagedLaunchTemplate = apis.Group + "/cluster"