Add initial implementation to add capacityReservationSelectorTerms an…

…d update status when found

cmd/controller/main.go

@@ -62,6 +62,7 @@ func main() {
 			cloudProvider,
 			op.SubnetProvider,
 			op.SecurityGroupProvider,
+			op.CapacityReservationProvider,
 			op.InstanceProfileProvider,
 			op.InstanceProvider,
 			op.PricingProvider,

pkg/apis/crds/karpenter.k8s.aws_ec2nodeclasses.yaml

@@ -216,6 +216,54 @@ spec:
                 - message: must have only one blockDeviceMappings with rootVolume
                   rule: self.filter(x, has(x.rootVolume)?x.rootVolume==true:false).size()
                     <= 1
+              capacityReservationSelectorTerms:
+                description: CapacityReservationSelectorTerms is a list of or Capacity
+                  Reservation selector terms. The terms are ORed.
+                items:
+                  description: |-
+                    CapacityReservationSelectorTerm defines selection logic for a Capacity Reservation used by Karpenter to launch nodes.
+                    If multiple fields are used for selection, the requirements are ANDed.
+                  properties:
+                    availabilityZone:
+                      description: The Availability Zone of the Capacity Reservation
+                      type: string
+                    id:
+                      description: The platform of operating system for which the
+                        Capacity Reservation reserves capacity
+                      type: string
+                    instancePlatform:
+                      description: |-
+                        Indicates the tenancy of the Capacity Reservation.
+                        A Capacity Reservation can have one of the following tenancy 'default' or 'dedicated'
+                        default - The Capacity Reservation is created on hardware that is shared with other Amazon Web Services accounts.
+                        dedicated - The Capacity Reservation is created on single-tenant hardware that is dedicated to a single Amazon Web Services account.
+                      type: string
+                    instanceType:
+                      description: The type of operating system for which the Capacity
+                        Reservation reserves capacity
+                      type: string
+                    ownerId:
+                      description: The ID of the Amazon Web Services account that
+                        owns the Capacity Reservation
+                      type: string
+                    tags:
+                      additionalProperties:
+                        type: string
+                      description: |-
+                        Tags is a map of key/value tags used to select subnets
+                        Specifying '*' for a value selects all values for a given tag key.
+                      maxProperties: 20
+                      type: object
+                      x-kubernetes-validations:
+                      - message: empty tag keys or values aren't supported
+                        rule: self.all(k, k != '' && self[k] != '')
+                    tenancy:
+                      description: ID is the Capacity Reservation id in EC2
+                      pattern: cr-[0-9a-z]+
+                      type: string
+                  type: object
+                maxItems: 30
+                type: array
               context:
                 description: |-
                   Context is a Reserved field in EC2 APIs
@@ -516,6 +564,49 @@ spec:
                   - requirements
                   type: object
                 type: array
+              capacityReservations:
+                description: |-
+                  CapacityReservations contains the current Capacity Reservations values that are available to the
+                  cluster under the CapacityReservations selectors.
+                items:
+                  description: CapacityReservation contains resolved Capacity Reservation
+                    selector values utilized for node launch
+                  properties:
+                    availabilityZone:
+                      description: AvailabilityZone of the Capacity Reservation
+                      type: string
+                    availableInstanceCount:
+                      description: Available Instance Count of the Capacity Reservation
+                      type: integer
+                    id:
+                      description: ID of the Capacity Reservation
+                      type: string
+                    instanceMatchCriteria:
+                      description: Instance Match Criteria of the Capacity Reservation
+                      type: string
+                    instancePlatform:
+                      description: Instance Platform of the Capacity Reservation
+                      type: string
+                    instanceType:
+                      description: Instance Type of the Capacity Reservation
+                      type: string
+                    ownerId:
+                      description: Owner Id of the Capacity Reservation
+                      type: string
+                    totalInstanceCount:
+                      description: Total Instance Count of the Capacity Reservation
+                      type: integer
+                  required:
+                  - availabilityZone
+                  - availableInstanceCount
+                  - id
+                  - instanceMatchCriteria
+                  - instancePlatform
+                  - instanceType
+                  - ownerId
+                  - totalInstanceCount
+                  type: object
+                type: array
               instanceProfile:
                 description: InstanceProfile contains the resolved instance profile
                   for the role

pkg/apis/v1beta1/ec2nodeclass.go

@@ -55,6 +55,10 @@ type EC2NodeClassSpec struct {
 	// +kubebuilder:validation:Enum:={AL2,AL2023,Bottlerocket,Ubuntu,Custom,Windows2019,Windows2022}
 	// +required
 	AMIFamily *string `json:"amiFamily"`
+	// CapacityReservationSelectorTerms is a list of or Capacity Reservation selector terms. The terms are ORed.
+	// +kubebuilder:validation:MaxItems:=30
+	// +required
+	CapacityReservationSelectorTerms []CapacityReservationSelectorTerm `json:"capacityReservationSelectorTerms,omitempty" hash:"ignore"`
 	// UserData to be applied to the provisioned nodes.
 	// It must be in the appropriate format based on the AMIFamily in use. Karpenter will merge certain fields into
 	// this UserData to ensure nodes are being provisioned with the correct configuration.
@@ -175,6 +179,39 @@ type AMISelectorTerm struct {
 	Owner string `json:"owner,omitempty"`
 }
 
+// CapacityReservationSelectorTerm defines selection logic for a Capacity Reservation used by Karpenter to launch nodes.
+// If multiple fields are used for selection, the requirements are ANDed.
+type CapacityReservationSelectorTerm struct {
+	// The Availability Zone of the Capacity Reservation
+	// +optional
+	AvailabilityZone string `json:"availabilityZone,omitempty"`
+	// The platform of operating system for which the Capacity Reservation reserves capacity
+	// +optional
+	ID string `json:"id,omitempty"`
+	// Tags is a map of key/value tags used to select subnets
+	// Specifying '*' for a value selects all values for a given tag key.
+	// +kubebuilder:validation:XValidation:message="empty tag keys or values aren't supported",rule="self.all(k, k != '' && self[k] != '')"
+	// +kubebuilder:validation:MaxProperties:=20
+	// +optional
+	Tags map[string]string `json:"tags,omitempty"`
+	// ID is the Capacity Reservation id in EC2
+	// +kubebuilder:validation:Pattern:="cr-[0-9a-z]+"
+	// +optional
+	Tenancy string `json:"tenancy,omitempty"`
+	// Indicates the tenancy of the Capacity Reservation.
+	// A Capacity Reservation can have one of the following tenancy 'default' or 'dedicated'
+	// default - The Capacity Reservation is created on hardware that is shared with other Amazon Web Services accounts.
+	// dedicated - The Capacity Reservation is created on single-tenant hardware that is dedicated to a single Amazon Web Services account.
+	// +optional
+	InstancePlatform string `json:"instancePlatform,omitempty"`
+	// The type of operating system for which the Capacity Reservation reserves capacity
+	// +optional
+	InstanceType string `json:"instanceType,omitempty"`
+	// The ID of the Amazon Web Services account that owns the Capacity Reservation
+	// +optional
+	OwnerId string `json:"ownerId,omitempty"`
+}
+
 // MetadataOptions contains parameters for specifying the exposure of the
 // Instance Metadata Service to provisioned EC2 nodes.
 type MetadataOptions struct {

pkg/apis/v1beta1/ec2nodeclass_status.go

@@ -38,6 +38,34 @@ type SecurityGroup struct {
 	Name string `json:"name,omitempty"`
 }
 
+// CapacityReservation contains resolved Capacity Reservation selector values utilized for node launch
+type CapacityReservation struct {
+	// ID of the Capacity Reservation
+	// +required
+	ID string `json:"id"`
+	// AvailabilityZone of the Capacity Reservation
+	// +required
+	AvailabilityZone string `json:"availabilityZone"`
+	// Available Instance Count of the Capacity Reservation
+	// +required
+	AvailableInstanceCount int `json:"availableInstanceCount"`
+	// Instance Match Criteria of the Capacity Reservation
+	// +required
+	InstanceMatchCriteria string `json:"instanceMatchCriteria"`
+	// Instance Platform of the Capacity Reservation
+	// +required
+	InstancePlatform string `json:"instancePlatform"`
+	// Instance Type of the Capacity Reservation
+	// +required
+	InstanceType string `json:"instanceType"`
+	// Owner Id of the Capacity Reservation
+	// +required
+	OwnerID string `json:"ownerId"`
+	// Total Instance Count of the Capacity Reservation
+	// +required
+	TotalInstanceCount int `json:"totalInstanceCount"`
+}
+
 // AMI contains resolved AMI selector values utilized for node launch
 type AMI struct {
 	// ID of the AMI
@@ -53,6 +81,10 @@ type AMI struct {
 
 // EC2NodeClassStatus contains the resolved state of the EC2NodeClass
 type EC2NodeClassStatus struct {
+	// CapacityReservations contains the current Capacity Reservations values that are available to the
+	// cluster under the CapacityReservations selectors.
+	// +optional
+	CapacityReservations []CapacityReservation `json:"capacityReservations,omitempty"`
 	// Subnets contains the current Subnet values that are available to the
 	// cluster under the subnet selectors.
 	// +optional

pkg/cloudprovider/cloudprovider.go

@@ -96,6 +96,9 @@ func (c *CloudProvider) Create(ctx context.Context, nodeClaim *corev1beta1.NodeC
 	}
 	instance, err := c.instanceProvider.Create(ctx, nodeClass, nodeClaim, instanceTypes)
 	if err != nil {
+		if cloudprovider.IsInsufficientCapacityError(err) {
+			return nil, cloudprovider.NewInsufficientCapacityError(fmt.Errorf("creating instance, %w", err))
+		}
 		return nil, fmt.Errorf("creating instance, %w", err)
 	}
 	instanceType, _ := lo.Find(instanceTypes, func(i *cloudprovider.InstanceType) bool {

pkg/controllers/controllers.go

@@ -24,6 +24,7 @@ import (
 	nodeclasstermination "github.com/aws/karpenter-provider-aws/pkg/controllers/nodeclass/termination"
 	controllersinstancetype "github.com/aws/karpenter-provider-aws/pkg/controllers/providers/instancetype"
 	controllerspricing "github.com/aws/karpenter-provider-aws/pkg/controllers/providers/pricing"
+	"github.com/aws/karpenter-provider-aws/pkg/providers/capacityreservation"
 	"github.com/aws/karpenter-provider-aws/pkg/providers/launchtemplate"
 
 	"github.com/aws/aws-sdk-go/aws/session"
@@ -52,12 +53,12 @@ import (
 
 func NewControllers(ctx context.Context, sess *session.Session, clk clock.Clock, kubeClient client.Client, recorder events.Recorder,
 	unavailableOfferings *cache.UnavailableOfferings, cloudProvider cloudprovider.CloudProvider, subnetProvider subnet.Provider,
-	securityGroupProvider securitygroup.Provider, instanceProfileProvider instanceprofile.Provider, instanceProvider instance.Provider,
+	securityGroupProvider securitygroup.Provider, capacityReservationProvider capacityreservation.Provider, instanceProfileProvider instanceprofile.Provider, instanceProvider instance.Provider,
 	pricingProvider pricing.Provider, amiProvider amifamily.Provider, launchTemplateProvider launchtemplate.Provider, instanceTypeProvider instancetype.Provider) []controller.Controller {
 
 	controllers := []controller.Controller{
 		nodeclasshash.NewController(kubeClient),
-		nodeclassstatus.NewController(kubeClient, subnetProvider, securityGroupProvider, amiProvider, instanceProfileProvider, launchTemplateProvider),
+		nodeclassstatus.NewController(kubeClient, subnetProvider, securityGroupProvider, capacityReservationProvider, amiProvider, instanceProfileProvider, launchTemplateProvider),
 		nodeclasstermination.NewController(kubeClient, recorder, instanceProfileProvider, launchTemplateProvider),
 		nodeclaimgarbagecollection.NewController(kubeClient, cloudProvider),
 		nodeclaimtagging.NewController(kubeClient, instanceProvider),

pkg/controllers/nodeclass/status/controller.go

@@ -34,6 +34,7 @@ import (
 
 	"github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1"
 	"github.com/aws/karpenter-provider-aws/pkg/providers/amifamily"
+	"github.com/aws/karpenter-provider-aws/pkg/providers/capacityreservation"
 	"github.com/aws/karpenter-provider-aws/pkg/providers/instanceprofile"
 	"github.com/aws/karpenter-provider-aws/pkg/providers/launchtemplate"
 	"github.com/aws/karpenter-provider-aws/pkg/providers/securitygroup"
@@ -49,23 +50,25 @@ type nodeClassStatusReconciler interface {
 type Controller struct {
 	kubeClient client.Client
 
-	ami             *AMI
-	instanceprofile *InstanceProfile
-	subnet          *Subnet
-	securitygroup   *SecurityGroup
-	launchtemplate  *LaunchTemplate
+	ami                 *AMI
+	instanceprofile     *InstanceProfile
+	subnet              *Subnet
+	securitygroup       *SecurityGroup
+	capacityreservation *CapacityReservation
+	launchtemplate      *LaunchTemplate
 }
 
-func NewController(kubeClient client.Client, subnetProvider subnet.Provider, securityGroupProvider securitygroup.Provider,
+func NewController(kubeClient client.Client, subnetProvider subnet.Provider, securityGroupProvider securitygroup.Provider, capacityReservationProvider capacityreservation.Provider,
 	amiProvider amifamily.Provider, instanceProfileProvider instanceprofile.Provider, launchTemplateProvider launchtemplate.Provider) corecontroller.Controller {
 	return corecontroller.Typed[*v1beta1.EC2NodeClass](kubeClient, &Controller{
 		kubeClient: kubeClient,
 
-		ami:             &AMI{amiProvider: amiProvider},
-		subnet:          &Subnet{subnetProvider: subnetProvider},
-		securitygroup:   &SecurityGroup{securityGroupProvider: securityGroupProvider},
-		instanceprofile: &InstanceProfile{instanceProfileProvider: instanceProfileProvider},
-		launchtemplate:  &LaunchTemplate{launchTemplateProvider: launchTemplateProvider},
+		ami:                 &AMI{amiProvider: amiProvider},
+		subnet:              &Subnet{subnetProvider: subnetProvider},
+		securitygroup:       &SecurityGroup{securityGroupProvider: securityGroupProvider},
+		capacityreservation: &CapacityReservation{capacityReservationProvider: capacityReservationProvider},
+		instanceprofile:     &InstanceProfile{instanceProfileProvider: instanceProfileProvider},
+		launchtemplate:      &LaunchTemplate{launchTemplateProvider: launchTemplateProvider},
 	})
 }
 
@@ -87,6 +90,7 @@ func (c *Controller) Reconcile(ctx context.Context, nodeClass *v1beta1.EC2NodeCl
 		c.securitygroup,
 		c.instanceprofile,
 		c.launchtemplate,
+		c.capacityreservation,
 	} {
 		res, err := reconciler.Reconcile(ctx, nodeClass)
 		errs = multierr.Append(errs, err)

pkg/controllers/nodeclass/status/suite_test.go

@@ -59,6 +59,7 @@ var _ = BeforeSuite(func() {
 		env.Client,
 		awsEnv.SubnetProvider,
 		awsEnv.SecurityGroupProvider,
+		awsEnv.CapacityReservationProvider,
 		awsEnv.AMIProvider,
 		awsEnv.InstanceProfileProvider,
 		awsEnv.LaunchTemplateProvider,

pkg/errors/errors.go

@@ -48,6 +48,7 @@ var (
 		"UnfulfillableCapacity",
 		"Unsupported",
 		"InsufficientFreeAddressesInSubnet",
+		"ReservationCapacityExceeded",
 	)
 )