From 814a2f4393712ac2bdb6f7faafb909a31d4a93b8 Mon Sep 17 00:00:00 2001 From: Chris McKee Date: Wed, 22 Mar 2023 21:29:08 +0000 Subject: [PATCH] Windows node support (#139) * Add initial changes for windows managed node support Add taint example to windows doc * Update iam.tf Co-authored-by: Andriy Knysh * Update Changes to allow windows autotaint label, fixes * Add the windows taint by default to windows nodes * Correct ami_kind filter * Alter userdata nt to match current userscript * Guard userdata against errors from userscript stopping node-join. * Remove disk_size * Add windnows_coredns_service_address because EKS Windows networking isn't fab. * Remove extraneous statement, fix output * Add drive mapping to windows userdata; put bootstrap extra args in correct place and remove extraneous params (based on eksctl code base and aws generated script) * Convert ami.tf maps to map format ("":"" to key = "value") * Revert min provider; remove enclosure on ipv6 policy * Uncomment userdata/label versions in example * Remove leading whitespace in markdown * Change `ami_release_version` regex to support windows ami format Tested here: https://regex101.com/r/xb7q2f/2 * Auto Format * Update versions.tf Ref version change related to change log https://github.com/hashicorp/terraform-provider-aws/blob/main/CHANGELOG.md#4480-december-19-2022 * Revert renaming of linux worker node group in example/complete * Auto Format --------- Co-authored-by: Andriy Knysh Co-authored-by: cloudpossebot <11232728+cloudpossebot@users.noreply.github.com> --- .gitignore | 4 ++ README.md | 46 +++++++++++++++++-- README.yaml | 1 + ami.tf | 49 ++++++++++++++------- docs/terraform.md | 9 ++-- docs/windows.md | 37 ++++++++++++++++ examples/complete/fixtures.us-east-2.tfvars | 2 - examples/complete/main.tf | 8 ++-- examples/complete/outputs.tf | 1 - examples/complete/variables.tf | 11 ++--- iam.tf | 2 +- launch-template.tf | 1 - main.tf | 18 +++++--- outputs.tf | 5 +++ userdata.tf | 2 +- userdata.tpl | 1 + userdata_nt.tpl | 37 ++++++++++++++++ variables.tf | 12 ++--- versions.tf | 5 ++- 19 files changed, 195 insertions(+), 56 deletions(-) create mode 100644 docs/windows.md create mode 100644 userdata_nt.tpl diff --git a/.gitignore b/.gitignore index 0c6f3c5..fbbeaaa 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,7 @@ # vim editor *.swp + +/.terraform.lock.hcl +/.vscode +/examples/complete/.terraform.lock.hcl diff --git a/README.md b/README.md index 15f484f..69203dd 100644 --- a/README.md +++ b/README.md @@ -243,19 +243,56 @@ Available targets: ``` +## Windows Managed Node groups + +Windows managed node-groups have a few pre-requisites. + +* Your cluster must contain at least one linux based worker node +* Your EKS Cluster must have the `AmazonEKSVPCResourceController` and `AmazonEKSClusterPolicy` policies attached +* Your cluster must have a config-map called amazon-vpc-cni with the following content + +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: +name: amazon-vpc-cni +namespace: kube-system +data: +enable-windows-ipam: "true" +``` + +* Windows nodes will automatically be tainted + +```yaml +kubernetes_taints = [{ + key = "WINDOWS" + value = "true" + effect = "NO_SCHEDULE" +}] +``` +* Any pods that target Windows will need to have the following attributes set in their manifest + +```yaml + nodeSelector: + kubernetes.io/os: windows + kubernetes.io/arch: amd64 +``` + +https://docs.aws.amazon.com/eks/latest/userguide/windows-support.html + ## Requirements | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 0.14.11 | -| [aws](#requirement\_aws) | >= 3.56 | +| [aws](#requirement\_aws) | >= 4.48 | | [random](#requirement\_random) | >= 2.0 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 3.56 | +| [aws](#provider\_aws) | >= 4.48 | | [random](#provider\_random) | >= 2.0 | ## Modules @@ -294,8 +331,8 @@ Available targets: | [additional\_tag\_map](#input\_additional\_tag\_map) | Additional key-value pairs to add to each map in `tags_as_list_of_maps`. Not added to `tags` or `id`.
This is for some rare cases where resources want additional configuration of tags
and therefore take a list of maps with tag key, value, and additional configuration. | `map(string)` | `{}` | no | | [after\_cluster\_joining\_userdata](#input\_after\_cluster\_joining\_userdata) | Additional `bash` commands to execute on each worker node after joining the EKS cluster (after executing the `bootstrap.sh` script). For more info, see https://kubedex.com/90-days-of-aws-eks-in-production | `list(string)` | `[]` | no | | [ami\_image\_id](#input\_ami\_image\_id) | AMI to use. Ignored if `launch_template_id` is supplied. | `list(string)` | `[]` | no | -| [ami\_release\_version](#input\_ami\_release\_version) | EKS AMI version to use, e.g. For AL2 "1.16.13-20200821" or for bottlerocket "1.2.0-ccf1b754" (no "v"). For AL2 and bottlerocket, it defaults to latest version for Kubernetes version. | `list(string)` | `[]` | no | -| [ami\_type](#input\_ami\_type) | Type of Amazon Machine Image (AMI) associated with the EKS Node Group.
Defaults to `AL2_x86_64`. Valid values: `AL2_x86_64`, `AL2_x86_64_GPU`, `AL2_ARM_64`, `BOTTLEROCKET_x86_64`, and `BOTTLEROCKET_ARM_64`. | `string` | `"AL2_x86_64"` | no | +| [ami\_release\_version](#input\_ami\_release\_version) | EKS AMI version to use, e.g. For AL2 "1.16.13-20200821" or for bottlerocket "1.2.0-ccf1b754" (no "v") or for Windows "2023.02.14". For AL2, bottlerocket and Windows, it defaults to latest version for Kubernetes version. | `list(string)` | `[]` | no | +| [ami\_type](#input\_ami\_type) | Type of Amazon Machine Image (AMI) associated with the EKS Node Group.
Defaults to `AL2_x86_64`. Valid values: `AL2_x86_64, AL2_x86_64_GPU, AL2_ARM_64, CUSTOM, BOTTLEROCKET_ARM_64, BOTTLEROCKET_x86_64, BOTTLEROCKET_ARM_64_NVIDIA, BOTTLEROCKET_x86_64_NVIDIA, WINDOWS_CORE_2019_x86_64, WINDOWS_FULL_2019_x86_64, WINDOWS_CORE_2022_x86_64, WINDOWS_FULL_2022_x86_64`. | `string` | `"AL2_x86_64"` | no | | [associate\_cluster\_security\_group](#input\_associate\_cluster\_security\_group) | When true, associate the default cluster security group to the nodes. If disabled the EKS managed security group will not
be associated to the nodes, therefore the communications between pods and nodes will not work. Be aware that if no `associated_security_group_ids`
nor `ssh_access_security_group_ids` are provided then the nodes will have no inbound or outbound rules. | `bool` | `true` | no | | [associated\_security\_group\_ids](#input\_associated\_security\_group\_ids) | A list of IDs of Security Groups to associate the node group with, in addition to the EKS' created security group.
These security groups will not be modified. | `list(string)` | `[]` | no | | [attributes](#input\_attributes) | ID element. Additional attributes (e.g. `workers` or `cluster`) to add to `id`,
in the order they appear in the list. New attributes are appended to the
end of the list. The elements of the list are joined by the `delimiter`
and treated as a single ID element. | `list(string)` | `[]` | no | @@ -367,6 +404,7 @@ Available targets: | [eks\_node\_group\_role\_name](#output\_eks\_node\_group\_role\_name) | Name of the worker nodes IAM role | | [eks\_node\_group\_status](#output\_eks\_node\_group\_status) | Status of the EKS Node Group | | [eks\_node\_group\_tags\_all](#output\_eks\_node\_group\_tags\_all) | A map of tags assigned to the resource, including those inherited from the provider default\_tags configuration block. | +| [eks\_node\_group\_windows\_note](#output\_eks\_node\_group\_windows\_note) | Instructions on changes a user needs to follow or script for a windows node group in the event of a custom ami | diff --git a/README.yaml b/README.yaml index 5ffe1e4..7fb2255 100644 --- a/README.yaml +++ b/README.yaml @@ -191,6 +191,7 @@ usage: |2- include: - "docs/targets.md" + - "docs/windows.md" - "docs/terraform.md" # Contributors to this project diff --git a/ami.tf b/ami.tf index e349760..1560b81 100644 --- a/ami.tf +++ b/ami.tf @@ -2,22 +2,34 @@ locals { # "amazon-eks-gpu-node-", arch_label_map = { - "AL2_x86_64" : "", - "AL2_x86_64_GPU" : "-gpu", - "AL2_ARM_64" : "-arm64", - "BOTTLEROCKET_x86_64" : "x86_64", - "BOTTLEROCKET_ARM_64" : "aarch64" + AL2_x86_64 = "", + AL2_x86_64_GPU = "-gpu", + AL2_ARM_64 = "-arm64", + BOTTLEROCKET_x86_64 = "x86_64", + BOTTLEROCKET_ARM_64 = "aarch64" + BOTTLEROCKET_ARM_64_NVIDIA = "-gpu" + BOTTLEROCKET_x86_64_NVIDIA = "-gpu" + WINDOWS_CORE_2019_x86_64 = "" + WINDOWS_FULL_2019_x86_64 = "" + WINDOWS_CORE_2022_x86_64 = "" + WINDOWS_FULL_2022_x86_64 = "" } - ami_kind = split("_", var.ami_type)[0] + ami_kind = split("_", var.ami_type)[0] != "WINDOWS" ? split("_", var.ami_type)[0] : format("WINDOWS_%s_%s", split("_", var.ami_type)[1], split("_", var.ami_type)[2]) ami_format = { # amazon-eks{arch_label}-node-{ami_kubernetes_version}-v{ami_version} # e.g. amazon-eks-arm64-node-1.21-v20211013 - "AL2" : "amazon-eks%s-node-%s" + AL2 = "amazon-eks%s-node-%s" # bottlerocket-aws-k8s-{ami_kubernetes_version}-{arch_label}-v{ami_version} # e.g. bottlerocket-aws-k8s-1.21-x86_64-v1.2.0-ccf1b754 - "BOTTLEROCKET" : "bottlerocket-aws-k8s-%s-%s-%s" + BOTTLEROCKET = "bottlerocket-aws-k8s-%s-%s-%s" + # Windows_Server-2019-English-Core-EKS_Optimized-{ami_kubernetes_version}-{ami_version} + # e.g. Windows_Server-2019-English-Core-EKS_Optimized-1.23-2022.11.08 + WINDOWS_CORE_2019 = "Windows_Server-2019-English-Core-EKS_Optimized-%s-%s" + WINDOWS_FULL_2019 = "Windows_Server-2019-English-Full-EKS_Optimized-%s-%s" + WINDOWS_CORE_2022 = "Windows_Server-2022-English-Core-EKS_Optimized-%s-%s" + WINDOWS_FULL_2022 = "Windows_Server-2022-English-Full-EKS_Optimized-%s-%s" } # Kubernetes version priority (first one to be set wins) @@ -37,19 +49,24 @@ locals { # if ami_release_version = "1.21-20211013" # insert the letter v prior to the ami_version so it becomes 1.21-v20211013 # if not, use the kubernetes version - "AL2" : (length(var.ami_release_version) == 1 ? - replace(var.ami_release_version[0], "/^(\\d+\\.\\d+)\\.\\d+-(\\d+)$/", "$1-v$2") : - "${local.ami_kubernetes_version}-*"), + AL2 = (length(var.ami_release_version) == 1 ? replace(var.ami_release_version[0], "/^(\\d+\\.\\d+)\\.\\d+-(\\d+)$/", "$1-v$2") : "${local.ami_kubernetes_version}-*"), # if ami_release_version = "1.2.0-ccf1b754" - # prefex the ami release version with the letter v + # prefix the ami release version with the letter v # if not, use an asterisk - "BOTTLEROCKET" : (length(var.ami_release_version) == 1 ? - format("v%s", var.ami_release_version[0]) : "*"), + BOTTLEROCKET = (length(var.ami_release_version) == 1 ? format("v%s", var.ami_release_version[0]) : "*"), + WINDOWS_CORE_2019 = (length(var.ami_release_version) == 1 ? format("%s", var.ami_release_version[0]) : "*"), + WINDOWS_FULL_2019 = (length(var.ami_release_version) == 1 ? format("%s", var.ami_release_version[0]) : "*"), + WINDOWS_CORE_2022 = (length(var.ami_release_version) == 1 ? format("%s", var.ami_release_version[0]) : "*"), + WINDOWS_FULL_2022 = (length(var.ami_release_version) == 1 ? format("%s", var.ami_release_version[0]) : "*"), } : {} ami_regex = local.need_ami_id ? { - "AL2" : format(local.ami_format["AL2"], local.arch_label_map[var.ami_type], local.ami_version_regex[local.ami_kind]), - "BOTTLEROCKET" : format(local.ami_format["BOTTLEROCKET"], local.ami_kubernetes_version, local.arch_label_map[var.ami_type], local.ami_version_regex[local.ami_kind]), + AL2 = format(local.ami_format["AL2"], local.arch_label_map[var.ami_type], local.ami_version_regex[local.ami_kind]), + BOTTLEROCKET = format(local.ami_format["BOTTLEROCKET"], local.ami_kubernetes_version, local.arch_label_map[var.ami_type], local.ami_version_regex[local.ami_kind]), + WINDOWS_CORE_2019 = format(local.ami_format["WINDOWS_CORE_2019"], local.ami_kubernetes_version, local.ami_version_regex[local.ami_kind]), + WINDOWS_FULL_2019 = format(local.ami_format["WINDOWS_FULL_2019"], local.ami_kubernetes_version, local.ami_version_regex[local.ami_kind]), + WINDOWS_CORE_2022 = format(local.ami_format["WINDOWS_CORE_2022"], local.ami_kubernetes_version, local.ami_version_regex[local.ami_kind]), + WINDOWS_FULL_2022 = format(local.ami_format["WINDOWS_FULL_2022"], local.ami_kubernetes_version, local.ami_version_regex[local.ami_kind]), } : {} } diff --git a/docs/terraform.md b/docs/terraform.md index ab2059c..dac4ed3 100644 --- a/docs/terraform.md +++ b/docs/terraform.md @@ -4,14 +4,14 @@ | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 0.14.11 | -| [aws](#requirement\_aws) | >= 3.56 | +| [aws](#requirement\_aws) | >= 4.48 | | [random](#requirement\_random) | >= 2.0 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 3.56 | +| [aws](#provider\_aws) | >= 4.48 | | [random](#provider\_random) | >= 2.0 | ## Modules @@ -50,8 +50,8 @@ | [additional\_tag\_map](#input\_additional\_tag\_map) | Additional key-value pairs to add to each map in `tags_as_list_of_maps`. Not added to `tags` or `id`.
This is for some rare cases where resources want additional configuration of tags
and therefore take a list of maps with tag key, value, and additional configuration. | `map(string)` | `{}` | no | | [after\_cluster\_joining\_userdata](#input\_after\_cluster\_joining\_userdata) | Additional `bash` commands to execute on each worker node after joining the EKS cluster (after executing the `bootstrap.sh` script). For more info, see https://kubedex.com/90-days-of-aws-eks-in-production | `list(string)` | `[]` | no | | [ami\_image\_id](#input\_ami\_image\_id) | AMI to use. Ignored if `launch_template_id` is supplied. | `list(string)` | `[]` | no | -| [ami\_release\_version](#input\_ami\_release\_version) | EKS AMI version to use, e.g. For AL2 "1.16.13-20200821" or for bottlerocket "1.2.0-ccf1b754" (no "v"). For AL2 and bottlerocket, it defaults to latest version for Kubernetes version. | `list(string)` | `[]` | no | -| [ami\_type](#input\_ami\_type) | Type of Amazon Machine Image (AMI) associated with the EKS Node Group.
Defaults to `AL2_x86_64`. Valid values: `AL2_x86_64`, `AL2_x86_64_GPU`, `AL2_ARM_64`, `BOTTLEROCKET_x86_64`, and `BOTTLEROCKET_ARM_64`. | `string` | `"AL2_x86_64"` | no | +| [ami\_release\_version](#input\_ami\_release\_version) | EKS AMI version to use, e.g. For AL2 "1.16.13-20200821" or for bottlerocket "1.2.0-ccf1b754" (no "v") or for Windows "2023.02.14". For AL2, bottlerocket and Windows, it defaults to latest version for Kubernetes version. | `list(string)` | `[]` | no | +| [ami\_type](#input\_ami\_type) | Type of Amazon Machine Image (AMI) associated with the EKS Node Group.
Defaults to `AL2_x86_64`. Valid values: `AL2_x86_64, AL2_x86_64_GPU, AL2_ARM_64, CUSTOM, BOTTLEROCKET_ARM_64, BOTTLEROCKET_x86_64, BOTTLEROCKET_ARM_64_NVIDIA, BOTTLEROCKET_x86_64_NVIDIA, WINDOWS_CORE_2019_x86_64, WINDOWS_FULL_2019_x86_64, WINDOWS_CORE_2022_x86_64, WINDOWS_FULL_2022_x86_64`. | `string` | `"AL2_x86_64"` | no | | [associate\_cluster\_security\_group](#input\_associate\_cluster\_security\_group) | When true, associate the default cluster security group to the nodes. If disabled the EKS managed security group will not
be associated to the nodes, therefore the communications between pods and nodes will not work. Be aware that if no `associated_security_group_ids`
nor `ssh_access_security_group_ids` are provided then the nodes will have no inbound or outbound rules. | `bool` | `true` | no | | [associated\_security\_group\_ids](#input\_associated\_security\_group\_ids) | A list of IDs of Security Groups to associate the node group with, in addition to the EKS' created security group.
These security groups will not be modified. | `list(string)` | `[]` | no | | [attributes](#input\_attributes) | ID element. Additional attributes (e.g. `workers` or `cluster`) to add to `id`,
in the order they appear in the list. New attributes are appended to the
end of the list. The elements of the list are joined by the `delimiter`
and treated as a single ID element. | `list(string)` | `[]` | no | @@ -123,4 +123,5 @@ | [eks\_node\_group\_role\_name](#output\_eks\_node\_group\_role\_name) | Name of the worker nodes IAM role | | [eks\_node\_group\_status](#output\_eks\_node\_group\_status) | Status of the EKS Node Group | | [eks\_node\_group\_tags\_all](#output\_eks\_node\_group\_tags\_all) | A map of tags assigned to the resource, including those inherited from the provider default\_tags configuration block. | +| [eks\_node\_group\_windows\_note](#output\_eks\_node\_group\_windows\_note) | Instructions on changes a user needs to follow or script for a windows node group in the event of a custom ami | diff --git a/docs/windows.md b/docs/windows.md new file mode 100644 index 0000000..1dd4a22 --- /dev/null +++ b/docs/windows.md @@ -0,0 +1,37 @@ + +## Windows Managed Node groups + +Windows managed node-groups have a few pre-requisites. + +* Your cluster must contain at least one linux based worker node +* Your EKS Cluster must have the `AmazonEKSVPCResourceController` and `AmazonEKSClusterPolicy` policies attached +* Your cluster must have a config-map called amazon-vpc-cni with the following content + +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: +name: amazon-vpc-cni +namespace: kube-system +data: +enable-windows-ipam: "true" +``` + +* Windows nodes will automatically be tainted + +```yaml +kubernetes_taints = [{ + key = "WINDOWS" + value = "true" + effect = "NO_SCHEDULE" +}] +``` +* Any pods that target Windows will need to have the following attributes set in their manifest + +```yaml + nodeSelector: + kubernetes.io/os: windows + kubernetes.io/arch: amd64 +``` + +https://docs.aws.amazon.com/eks/latest/userguide/windows-support.html diff --git a/examples/complete/fixtures.us-east-2.tfvars b/examples/complete/fixtures.us-east-2.tfvars index 8f20c7f..e4f02dc 100644 --- a/examples/complete/fixtures.us-east-2.tfvars +++ b/examples/complete/fixtures.us-east-2.tfvars @@ -26,8 +26,6 @@ max_size = 3 min_size = 2 -disk_size = 20 - kubernetes_labels = { terratest = "true" } diff --git a/examples/complete/main.tf b/examples/complete/main.tf index 286f0ff..77d22c0 100644 --- a/examples/complete/main.tf +++ b/examples/complete/main.tf @@ -114,11 +114,9 @@ module "https_sg" { context = module.label.context } - module "eks_cluster" { - source = "cloudposse/eks-cluster/aws" - version = "2.4.0" - + source = "cloudposse/eks-cluster/aws" + version = "2.4.0" region = var.region vpc_id = module.vpc.vpc_id subnet_ids = module.subnets.public_subnet_ids @@ -147,7 +145,7 @@ module "eks_node_group" { kubernetes_version = [var.kubernetes_version] kubernetes_labels = merge(var.kubernetes_labels, { attributes = coalesce(join(module.this.delimiter, module.this.attributes), "none") }) kubernetes_taints = var.kubernetes_taints - # disk_size = var.disk_size + ec2_ssh_key_name = var.ec2_ssh_key_name ssh_access_security_group_ids = [module.ssh_source_access.id] associated_security_group_ids = [module.ssh_source_access.id, module.https_sg.id] diff --git a/examples/complete/outputs.tf b/examples/complete/outputs.tf index cbff747..022b0db 100644 --- a/examples/complete/outputs.tf +++ b/examples/complete/outputs.tf @@ -93,7 +93,6 @@ output "eks_node_group_cbd_pet_name" { value = module.eks_node_group.eks_node_group_cbd_pet_name } - output "eks_node_group_launch_template_id" { value = module.eks_node_group.eks_node_group_launch_template_id } diff --git a/examples/complete/variables.tf b/examples/complete/variables.tf index eb2a6d8..208f203 100644 --- a/examples/complete/variables.tf +++ b/examples/complete/variables.tf @@ -73,11 +73,6 @@ variable "local_exec_interpreter" { description = "shell to use for local_exec" } -variable "disk_size" { - type = number - description = "Disk size in GiB for worker nodes. Defaults to 20. Terraform will only perform drift detection if a configuration value is provided" -} - variable "instance_types" { type = list(string) description = "Set of instance types associated with the EKS Node Group. Defaults to [\"t3.medium\"]. Terraform will only perform drift detection if a configuration value is provided" @@ -164,14 +159,14 @@ variable "ami_type" { type = string description = <<-EOT Type of Amazon Machine Image (AMI) associated with the EKS Node Group. - Defaults to `AL2_x86_64`. Valid values: `AL2_x86_64`, `AL2_x86_64_GPU`, `AL2_ARM_64`, `BOTTLEROCKET_x86_64`, and `BOTTLEROCKET_ARM_64`. + Defaults to `AL2_x86_64`. Valid values: `AL2_x86_64, AL2_x86_64_GPU, AL2_ARM_64, CUSTOM, BOTTLEROCKET_ARM_64, BOTTLEROCKET_x86_64, BOTTLEROCKET_ARM_64_NVIDIA, BOTTLEROCKET_x86_64_NVIDIA, WINDOWS_CORE_2019_x86_64, WINDOWS_FULL_2019_x86_64, WINDOWS_CORE_2022_x86_64, WINDOWS_FULL_2022_x86_64`. EOT default = "AL2_x86_64" validation { condition = ( - contains(["AL2_x86_64", "AL2_x86_64_GPU", "AL2_ARM_64", "BOTTLEROCKET_x86_64", "BOTTLEROCKET_ARM_64"], var.ami_type) + contains(["AL2_x86_64", "AL2_x86_64_GPU", "AL2_ARM_64", "CUSTOM", "BOTTLEROCKET_ARM_64", "BOTTLEROCKET_x86_64", "BOTTLEROCKET_ARM_64_NVIDIA", "BOTTLEROCKET_x86_64_NVIDIA", "WINDOWS_CORE_2019_x86_64", "WINDOWS_FULL_2019_x86_64", "WINDOWS_CORE_2022_x86_64", "WINDOWS_FULL_2022_x86_64"], var.ami_type) ) - error_message = "Var ami_type must be one of \"AL2_x86_64\", \"AL2_x86_64_GPU\", \"AL2_ARM_64\", \"BOTTLEROCKET_x86_64\", and \"BOTTLEROCKET_ARM_64\"." + error_message = "Var ami_type must be one of \"AL2_x86_64\",\"AL2_x86_64_GPU\",\"AL2_ARM_64\",\"BOTTLEROCKET_ARM_64\",\"BOTTLEROCKET_x86_64\",\"BOTTLEROCKET_ARM_64_NVIDIA\",\"BOTTLEROCKET_x86_64_NVIDIA\",\"WINDOWS_CORE_2019_x86_64\",\"WINDOWS_FULL_2019_x86_64\",\"WINDOWS_CORE_2022_x86_64\",\"WINDOWS_FULL_2022_x86_64\", or \"CUSTOM\"." } } diff --git a/iam.tf b/iam.tf index 01addf8..cd3241f 100644 --- a/iam.tf +++ b/iam.tf @@ -51,7 +51,7 @@ resource "aws_iam_role_policy_attachment" "existing_policies_for_eks_workers_rol # Create a CNI policy that is a merger of AmazonEKS_CNI_Policy and required IPv6 permissions # https://github.com/SummitRoute/aws_managed_policies/blob/master/policies/AmazonEKS_CNI_Policy # https://docs.aws.amazon.com/eks/latest/userguide/cni-iam-role.html#cni-iam-role-create-ipv6-policy - +# https://docs.aws.amazon.com/eks/latest/userguide/windows-support.html data "aws_iam_policy_document" "ipv6_eks_cni_policy" { count = local.create_role && var.node_role_cni_policy_enabled ? 1 : 0 diff --git a/launch-template.tf b/launch-template.tf index 01e1798..14cbcfd 100644 --- a/launch-template.tf +++ b/launch-template.tf @@ -60,7 +60,6 @@ resource "aws_launch_template" "default" { device_name = block_device_mappings.value.device_name ebs { - delete_on_termination = lookup(block_device_mappings.value, "delete_on_termination", null) encrypted = lookup(block_device_mappings.value, "encrypted", null) iops = lookup(block_device_mappings.value, "iops", null) diff --git a/main.tf b/main.tf index 36dd6fd..ed5d18f 100644 --- a/main.tf +++ b/main.tf @@ -44,6 +44,12 @@ locals { } ) node_group_tags = merge(local.node_tags, local.autoscaler_enabled ? local.autoscaler_tags : null) + + windows_taint = [{ + key = "OS" + value = "Windows" + effect = "NO_SCHEDULE" + }] } module "label" { @@ -62,6 +68,7 @@ data "aws_eks_cluster" "this" { # Support keeping 2 node groups in sync by extracting common variable settings locals { + is_windows = can(regex("WINDOWS", var.ami_type)) ng = { cluster_name = var.cluster_name node_role_arn = local.create_role ? join("", aws_iam_role.default.*.arn) : try(var.node_role_arn[0], null) @@ -71,11 +78,12 @@ locals { # because node group supports up to 20 types but launch template does not. # See https://docs.aws.amazon.com/eks/latest/APIReference/API_CreateNodegroup.html#API_CreateNodegroup_RequestSyntax # Keep sorted so that change in order does not trigger replacement via random_pet - instance_types = sort(var.instance_types) - ami_type = local.launch_template_ami == "" ? var.ami_type : null - capacity_type = var.capacity_type - labels = var.kubernetes_labels == null ? {} : var.kubernetes_labels - taints = var.kubernetes_taints + instance_types = sort(var.instance_types) + ami_type = local.launch_template_ami == "" ? var.ami_type : null + capacity_type = var.capacity_type + labels = var.kubernetes_labels == null ? {} : var.kubernetes_labels + + taints = local.is_windows ? concat(local.windows_taint, var.kubernetes_taints) : var.kubernetes_taints release_version = local.launch_template_ami == "" ? try(var.ami_release_version[0], null) : null version = length(compact(concat([local.launch_template_ami], var.ami_release_version))) == 0 ? try(var.kubernetes_version[0], null) : null diff --git a/outputs.tf b/outputs.tf index 8883079..d9701bf 100644 --- a/outputs.tf +++ b/outputs.tf @@ -52,3 +52,8 @@ output "eks_node_group_tags_all" { description = "A map of tags assigned to the resource, including those inherited from the provider default_tags configuration block." value = local.enabled ? (var.create_before_destroy ? aws_eks_node_group.cbd[0].tags_all : aws_eks_node_group.default[0].tags_all) : {} } + +output "eks_node_group_windows_note" { + description = "Instructions on changes a user needs to follow or script for a windows node group in the event of a custom ami" + value = local.enabled && local.is_windows && local.need_bootstrap ? "When specifying a custom AMI ID for Windows managed node groups, add eks:kube-proxy-windows to your AWS IAM Authenticator configuration map. For more information, see Limits and conditions when specifying an AMI ID. https://docs.aws.amazon.com/eks/latest/userguide/windows-support.html" : "" +} diff --git a/userdata.tf b/userdata.tf index 2296a7a..0a3a076 100644 --- a/userdata.tf +++ b/userdata.tf @@ -41,7 +41,7 @@ locals { (length(var.before_cluster_joining_userdata) > 0) || local.need_bootstrap) : false userdata = local.need_userdata ? ( - base64encode(templatefile("${path.module}/userdata.tpl", merge(local.userdata_vars, local.cluster_data)))) : ( + base64encode(templatefile(local.is_windows ? "${path.module}/userdata_nt.tpl" : "${path.module}/userdata.tpl", merge(local.userdata_vars, local.cluster_data)))) : ( try(var.userdata_override_base64[0], null) ) } diff --git a/userdata.tpl b/userdata.tpl index 181eabb..f11a015 100644 --- a/userdata.tpl +++ b/userdata.tpl @@ -4,6 +4,7 @@ Content-Type: multipart/mixed; boundary="/:/+++" --/:/+++ Content-Type: text/x-shellscript; charset="us-ascii" #!/bin/bash +set -ex # In multipart MIME format to support EKS appending to it diff --git a/userdata_nt.tpl b/userdata_nt.tpl new file mode 100644 index 0000000..5507816 --- /dev/null +++ b/userdata_nt.tpl @@ -0,0 +1,37 @@ + +try{ +${before_cluster_joining_userdata} +}catch{ + Write-Host "An error occurred in pre-script" -ForegroundColor Red + Write-Host $_.ScriptStackTrace +} +Write-Host -Foreground Red -Background Black ($formatstring -f $fields) + +# Deal with extra new disks +$disks_to_adjust = Get-Disk | Select-Object Number,Size,PartitionStyle | Where-Object PartitionStyle -Match RAW +if ($disks_to_adjust -ne $null) { + [int64] $partition_mbr_max_size = 2199023255552 + $partition_style = "MBR" + foreach ($disk in $disks_to_adjust) { + if ($disk.Size -gt $partition_mbr_max_size) { + $partition_style = "GPT" + } + + Initialize-Disk -Number $disk.Number -PartitionStyle $partition_style + New-Partition -DiskNumber $disk.Number -UseMaximumSize -AssignDriveLetter | Format-Volume -FileSystem NTFS + } +} + +[string]$EKSBinDir = "$env:ProgramFiles\Amazon\EKS" +[string]$EKSBootstrapScriptName = 'Start-EKSBootstrap.ps1' +[string]$EKSBootstrapScriptFile = "$EKSBinDir\$EKSBootstrapScriptName" + +& $EKSBootstrapScriptFile -EKSClusterName "${cluster_name}" -APIServerEndpoint "${cluster_endpoint}" -Base64ClusterCA "${certificate_authority_data}" ${bootstrap_extra_args} -KubeletExtraArgs "${kubelet_extra_args}" 3>&1 4>&1 5>&1 6>&1 + +try{ +${after_cluster_joining_userdata} +}catch{ + Write-Host "An error occurred in post-script" -ForegroundColor Red + Write-Host $_.ScriptStackTrace +} + diff --git a/variables.tf b/variables.tf index 5e3178c..521e0db 100644 --- a/variables.tf +++ b/variables.tf @@ -121,14 +121,14 @@ variable "ami_type" { type = string description = <<-EOT Type of Amazon Machine Image (AMI) associated with the EKS Node Group. - Defaults to `AL2_x86_64`. Valid values: `AL2_x86_64`, `AL2_x86_64_GPU`, `AL2_ARM_64`, `BOTTLEROCKET_x86_64`, and `BOTTLEROCKET_ARM_64`. + Defaults to `AL2_x86_64`. Valid values: `AL2_x86_64, AL2_x86_64_GPU, AL2_ARM_64, CUSTOM, BOTTLEROCKET_ARM_64, BOTTLEROCKET_x86_64, BOTTLEROCKET_ARM_64_NVIDIA, BOTTLEROCKET_x86_64_NVIDIA, WINDOWS_CORE_2019_x86_64, WINDOWS_FULL_2019_x86_64, WINDOWS_CORE_2022_x86_64, WINDOWS_FULL_2022_x86_64`. EOT default = "AL2_x86_64" validation { condition = ( - contains(["AL2_x86_64", "AL2_x86_64_GPU", "AL2_ARM_64", "BOTTLEROCKET_x86_64", "BOTTLEROCKET_ARM_64", "CUSTOM"], var.ami_type) + contains(["AL2_x86_64", "AL2_x86_64_GPU", "AL2_ARM_64", "CUSTOM", "BOTTLEROCKET_ARM_64", "BOTTLEROCKET_x86_64", "BOTTLEROCKET_ARM_64_NVIDIA", "BOTTLEROCKET_x86_64_NVIDIA", "WINDOWS_CORE_2019_x86_64", "WINDOWS_FULL_2019_x86_64", "WINDOWS_CORE_2022_x86_64", "WINDOWS_FULL_2022_x86_64"], var.ami_type) ) - error_message = "Var ami_type must be one of \"AL2_x86_64\", \"AL2_x86_64_GPU\", \"AL2_ARM_64\", \"BOTTLEROCKET_x86_64\", \"BOTTLEROCKET_ARM_64\", or \"CUSTOM\"." + error_message = "Var ami_type must be one of \"AL2_x86_64\",\"AL2_x86_64_GPU\",\"AL2_ARM_64\",\"BOTTLEROCKET_ARM_64\",\"BOTTLEROCKET_x86_64\",\"BOTTLEROCKET_ARM_64_NVIDIA\",\"BOTTLEROCKET_x86_64_NVIDIA\",\"WINDOWS_CORE_2019_x86_64\",\"WINDOWS_FULL_2019_x86_64\",\"WINDOWS_CORE_2022_x86_64\",\"WINDOWS_FULL_2022_x86_64\", or \"CUSTOM\"." } } @@ -241,12 +241,12 @@ variable "ami_image_id" { variable "ami_release_version" { type = list(string) default = [] - description = "EKS AMI version to use, e.g. For AL2 \"1.16.13-20200821\" or for bottlerocket \"1.2.0-ccf1b754\" (no \"v\"). For AL2 and bottlerocket, it defaults to latest version for Kubernetes version." + description = "EKS AMI version to use, e.g. For AL2 \"1.16.13-20200821\" or for bottlerocket \"1.2.0-ccf1b754\" (no \"v\") or for Windows \"2023.02.14\". For AL2, bottlerocket and Windows, it defaults to latest version for Kubernetes version." validation { condition = ( - length(var.ami_release_version) == 0 ? true : length(regexall("^\\d+\\.\\d+\\.\\d+-[\\da-z]+$", var.ami_release_version[0])) == 1 + length(var.ami_release_version) == 0 ? true : length(regexall("(^\\d+\\.\\d+\\.\\d+-[\\da-z]+$)|(^\\d+\\.\\d+\\.\\d+$)", var.ami_release_version[0])) == 1 ) - error_message = "Var ami_release_version, if supplied, must be like for AL2 \"1.16.13-20200821\" or for bottlerocket \"1.2.0-ccf1b754\" (no \"v\")." + error_message = "Var ami_release_version, if supplied, must be like for AL2 \"1.16.13-20200821\" or for bottlerocket \"1.2.0-ccf1b754\" (no \"v\") or for Windows \"2023.02.14\"." } } diff --git a/versions.tf b/versions.tf index 6995caf..41afb1a 100644 --- a/versions.tf +++ b/versions.tf @@ -3,10 +3,11 @@ terraform { required_providers { aws = { - source = "hashicorp/aws" # retrieve launch template by ID starts at 3.21.0 # update_config starts at 3.56 - version = ">= 3.56" + # Windows support starts at 4.48 https://github.com/hashicorp/terraform-provider-aws/blob/main/CHANGELOG.md#4480-december-19-2022 + source = "hashicorp/aws" + version = ">= 4.48" } random = { source = "hashicorp/random"