diff --git a/README.md b/README.md index b564792..ee12173 100644 --- a/README.md +++ b/README.md @@ -47,10 +47,10 @@ The file `variables.tf` declares the Terraform variables required to run this st ### Example (minimal for development env, creates a VPC and all resources in us-east-1) ``` module "etcd3-terraform" { - source = "github.com/cvlc/etcd3-terraform" + source = "github.com/ondat/etcd3-terraform" key_pair_public_key = "ssh-rsa..." ssh_cidrs = ["10.2.3.4/32"] # ssh jumpbox - dns = { "domain_name": "mycompany.local" } + dns = { "domain_name": "mycompany.int" } vpc_id = module.vpc.vpc_id subnet_ids = module.vpc.private_subnets @@ -86,14 +86,14 @@ module "vpc" { ### Example ('airgapped' environment, eu-west-2) -Though 'airgapped' in terms of inbound/outbound internet access, this will still rely on access to the AWS metadata and API services from the instance in order to attach the volumes. +Though 'airgapped' in terms of inbound/outbound internet access, this will still rely on access to the AWS metadata and API services from the instance in order to attach the volumes. This example will use Debian 10 as an alternative to Ubuntu. ``` module "etcd3-terraform" { - source = "github.com/cvlc/etcd3-terraform" + source = "github.com/ondat/etcd3-terraform" key_pair_public_key = "ssh-rsa..." ssh_cidrs = ["10.2.3.4/32"] # ssh jumpbox - dns = { "domain_name": "mycompany.local" } + dns = { "domain_name": "mycompany.int" } client_cirs = ["10.3.0.0/16"] # k8s cluster @@ -109,7 +109,7 @@ module "etcd3-terraform" { allow_download_from_cidrs = ["10.2.3.5/32"] # HTTPS server for file (certificate must be valid and verifiable) create_s3_bucket = "false" - etcd3_bootstrap_binary_url = "https://10.2.3.5/etcd3_bootstrap" + ebs_bootstrap_binary_url = "https://10.2.3.5/ebs_bootstrap" etcd_url = "https://10.2.3.5/etcd-v3.5.1.tgz" ami = "ami-031283ff8a43b021c" # Debian 10 } @@ -142,7 +142,7 @@ terraform apply ``` This module requires a private DNS zone. If you use custom DNS servers on your VPC, there are two options - either delegate the etcd subdomain to your VPC from your subdomain or implement [Route53 Resolver](https://docs.aws.amazon.com/Route53/latest/DeveloperGuide/resolver.html) to intercept queries for the internal domain and forward requests for it to your VPC DNS servers. -This module requires outbound internet access in it's default configuration to retrieve etcd binaries and the etcd3-bootstrap utility from S3. The sources for these tools are configurable via variables outlined in the examples - we can provide an https endpoint to retrieve them from in an internal environment. +This module requires outbound internet access in it's default configuration to retrieve etcd binaries and the ebs-bootstrap utility from S3. The sources for these tools are configurable via variables outlined in the examples - we can provide an https endpoint to retrieve them from in an internal environment. ### Maintenance etcd is configured with a 100GB data disk per node on Amazon EBS SSDs by default (configurable via `ssd_size` variable), a `revision` auto compaction mode and a retention of `20000`. An automatic cronjob runs on each node to ensure defragmentation happens at least once every month, this briefly blocks reads/writes on a single node at a time from 3:05am on a different day of the month for each node. It's configured with a backend space quota of `8589934592` bytes. @@ -160,9 +160,9 @@ You're now ready to test etcdctl functionality - replace `$insert_nlb_address` w ``` $ ETCDCTL_API=3 ETCDCTL_CERT=client.pem ETCDCTL_KEY=client.key ETCDCTL_ENDPOINTS="https://$insert_nlb_address:2379" etcdctl member list -25f97d08c726ed1, started, peer-2, https://peer-2.etcd.eu-west-2.i.development.mycompany.local:2380, https://peer-2.ondat.eu-west-2.i.development.mycompany.local:2379, false -326a6d27c048c8ea, started, peer-1, https://peer-1.etcd.eu-west-2.i.development.mycompany.local:2380, https://peer-1.ondat.eu-west-2.i.development.mycompany.local:2379, false -38308ae09ffc8b32, started, peer-0, https://peer-0.etcd.eu-west-2.i.development.mycompany.local:2380, https://peer-0.ondat.eu-west-2.i.development.mycompany.local:2379, false +25f97d08c726ed1, started, peer-2, https://peer-2.etcd.eu-west-2.i.development.mycompany.int:2380, https://peer-2.ondat.eu-west-2.i.development.mycompany.int:2379, false +326a6d27c048c8ea, started, peer-1, https://peer-1.etcd.eu-west-2.i.development.mycompany.int:2380, https://peer-1.ondat.eu-west-2.i.development.mycompany.int:2379, false +38308ae09ffc8b32, started, peer-0, https://peer-0.etcd.eu-west-2.i.development.mycompany.int:2380, https://peer-0.ondat.eu-west-2.i.development.mycompany.int:2379, false ``` ## How to (synthetically) [benchmark](https://etcd.io/docs/v3.5/op-guide/performance/) etcd in your environment 📊 @@ -462,12 +462,13 @@ No requirements. |------|---------| | [archive](#provider\_archive) | 2.2.0 | | [aws](#provider\_aws) | 3.71.0 | -| [local](#provider\_local) | 2.1.0 | | [tls](#provider\_tls) | 3.1.0 | ### Modules -No modules. +| Name | Source | Version | +|------|--------|---------| +| [attached-ebs](#module\_attached-ebs) | github.com/ondat/etcd3-bootstrap/terraform/modules/attached_ebs | n/a | ### Resources @@ -478,15 +479,11 @@ No modules. | [aws_cloudwatch_event_rule.ec2](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_event_rule) | resource | | [aws_cloudwatch_event_target.lambda-cloudwatch-dns-service-autoscaling](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_event_target) | resource | | [aws_cloudwatch_event_target.lambda-cloudwatch-dns-service-ec2](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_event_target) | resource | -| [aws_dlm_lifecycle_policy.automatic_snapshots](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/dlm_lifecycle_policy) | resource | -| [aws_ebs_volume.ssd](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/ebs_volume) | resource | | [aws_iam_instance_profile.default](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_instance_profile) | resource | | [aws_iam_role.default](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource | -| [aws_iam_role.dlm_lifecycle_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource | | [aws_iam_role.lambda-cloudwatch-dns-service](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource | -| [aws_iam_role_policy.default](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) | resource | -| [aws_iam_role_policy.dlm_lifecycle](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) | resource | | [aws_iam_role_policy.lambda-cloudwatch-dns-service](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) | resource | +| [aws_iam_role_policy_attachment.default](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | | [aws_iam_role_policy_attachment.lambda-cloudwatch-dns-service-logs](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | | [aws_iam_role_policy_attachment.lambda-cloudwatch-dns-service-xray](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | | [aws_key_pair.default](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/key_pair) | resource | @@ -502,13 +499,7 @@ No modules. | [aws_route53_record.nlb](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/route53_record) | resource | | [aws_route53_record.peers](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/route53_record) | resource | | [aws_route53_zone.default](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/route53_zone) | resource | -| [aws_s3_bucket.files](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket) | resource | -| [aws_s3_bucket_object.etcd3-bootstrap-linux-amd64](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket_object) | resource | -| [aws_s3_bucket_public_access_block.example](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket_public_access_block) | resource | | [aws_security_group.default](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group) | resource | -| [local_file.ca-cert](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | -| [local_file.client-cert](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | -| [local_file.client-key](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | | [tls_cert_request.client](https://registry.terraform.io/providers/hashicorp/tls/latest/docs/resources/cert_request) | resource | | [tls_cert_request.peer](https://registry.terraform.io/providers/hashicorp/tls/latest/docs/resources/cert_request) | resource | | [tls_cert_request.server](https://registry.terraform.io/providers/hashicorp/tls/latest/docs/resources/cert_request) | resource | @@ -538,10 +529,9 @@ No modules. | [associate\_public\_ips](#input\_associate\_public\_ips) | Whether to associate public IPs with etcd instances (suggest false for security) | `string` | `"false"` | no | | [client\_cidrs](#input\_client\_cidrs) | CIDRs to allow client access to etcd | `list` |
[| no | | [cluster\_size](#input\_cluster\_size) | Number of etcd nodes to launch | `number` | `3` | no | -| [create\_s3\_bucket](#input\_create\_s3\_bucket) | Whether to create the S3 bucket used by default for instances to obtain the etcd3-bootstrap binary through cloud-init | `string` | `"true"` | no | -| [dns](#input\_dns) | Domain to install etcd | `map(string)` |
"10.0.0.0/8"
]
{| no | +| [dns](#input\_dns) | Domain to install etcd | `map(string)` |
"domain_name": "mycompany.local"
}
{| no | | [environment](#input\_environment) | Target environment, used to apply tags | `string` | `"development"` | no | -| [etcd3\_bootstrap\_binary\_url](#input\_etcd3\_bootstrap\_binary\_url) | Custom URL from which to download the etcd3-bootstrap binary | `any` | `null` | no | +| [ebs\_bootstrap\_binary\_url](#input\_ebs\_bootstrap\_binary\_url) | Custom URL from which to download the ebs-bootstrap binary | `any` | `null` | no | | [etcd\_url](#input\_etcd\_url) | Custom URL from which to download the etcd tgz | `any` | `null` | no | | [etcd\_version](#input\_etcd\_version) | etcd version to install | `string` | `"3.5.1"` | no | | [instance\_type](#input\_instance\_type) | AWS instance type, at least c5a.large is recommended. etcd suggest m4.large. | `string` | `"c5a.large"` | no | diff --git a/asg.tf b/asg.tf index a4d3f96..b9a5543 100644 --- a/asg.tf +++ b/asg.tf @@ -4,12 +4,12 @@ resource "aws_launch_configuration" "default" { image_id = var.ami != "" ? var.ami : data.aws_ami.ami.id instance_type = var.instance_type ebs_optimized = true - iam_instance_profile = aws_iam_instance_profile.default.id + iam_instance_profile = aws_iam_instance_profile.default[count.index].id key_name = aws_key_pair.default.key_name enable_monitoring = false associate_public_ip_address = var.associate_public_ips security_groups = [aws_security_group.default.id] - user_data = join("\n", [module.asg-attached-ebs[count.index].userdata_snippet, templatefile("${path.module}/cloudinit/userdata-template.sh", { + user_data = join("\n", ["#!/bin/bash", module.attached-ebs.userdata_snippets_by_az[data.aws_subnet.target[count.index].availability_zone], templatefile("${path.module}/cloudinit/userdata-template.sh", { environment = var.environment, role = var.role, region = data.aws_region.current.name, @@ -21,6 +21,7 @@ resource "aws_launch_configuration" "default" { discovery_domain_name = "${var.role}.${data.aws_region.current.name}.i.${var.environment}.${var.dns["domain_name"]}" cluster_name = var.role }), + etcd_endpoint = "peer-${count.index}.${var.role}.${data.aws_region.current.name}.i.${var.environment}.${var.dns["domain_name"]}" ca_file = tls_self_signed_cert.ca.cert_pem, peer_cert_file = tls_locally_signed_cert.peer[count.index].cert_pem, peer_key_file = tls_private_key.peer[count.index].private_key_pem, @@ -54,7 +55,11 @@ resource "aws_autoscaling_group" "default" { value = "peer-${count.index}.${var.role}.${data.aws_region.current.name}.i.${var.environment}.${var.dns["domain_name"]}" propagate_at_launch = true } - + tag { + key = "Group" + value = "peer.${var.role}.${data.aws_region.current.name}.i.${var.environment}.${var.dns["domain_name"]}" + propagate_at_launch = true + } tag { key = "environment" value = var.environment @@ -93,23 +98,23 @@ data "aws_subnet" "target" { id = element(var.subnet_ids, count.index) } -module "asg-attached-ebs" { - count = var.cluster_size - source = "./modules/asg_attached_ebs" - asg_name = "peer-${count.index}.${var.role}.${data.aws_region.current.name}.i.${var.environment}.${var.dns["domain_name"]}" - availability_zone = element(data.aws_subnet.target.*.availability_zone, count.index) +module "attached-ebs" { + source = "github.com/ondat/etcd3-bootstrap/terraform/modules/attached_ebs" + group = "peer.${var.role}.${data.aws_region.current.name}.i.${var.environment}.${var.dns["domain_name"]}" + ebs_bootstrap_binary_url = var.ebs_bootstrap_binary_url attached_ebs = { - "data-${count.index}.${var.role}.${data.aws_region.current.name}.i.${var.environment}.${var.dns["domain_name"]}" = { + for i in range(var.cluster_size) : + "data-${i}.${var.role}.${data.aws_region.current.name}.i.${var.environment}.${var.dns["domain_name"]}" => { size = var.ssd_size + availability_zone = element(data.aws_subnet.target, i)["availability_zone"] encrypted = true volume_type = "gp3" - block_device_aws = "/dev/xvda1" - block_device_os = "/dev/nvme0n1" + block_device_aws = "/dev/xvdf" + block_device_os = "/dev/nvme1n1" block_device_mount_path = "/var/lib/etcd" tags = { - Name = "peer-${count.index}-ssd.${var.role}.${data.aws_region.current.name}.i.${var.environment}.${var.dns["domain_name"]}" environment = var.environment - role = "peer-${count.index}-ssd.${var.role}" + role = "peer-${i}-ssd.${var.role}" cluster = var.role } } diff --git a/cloudinit/userdata-template.sh b/cloudinit/userdata-template.sh index e7bc933..b9e1b12 100644 --- a/cloudinit/userdata-template.sh +++ b/cloudinit/userdata-template.sh @@ -1,5 +1,3 @@ -#!/usr/bin/env bash - ## Install ETCD useradd -U -M -s /dev/null etcd @@ -11,6 +9,8 @@ tar xvf /tmp/etcd-v${etcd_version}-linux-amd64.tar.gz -C /tmp mv /tmp/etcd-v${etcd_version}-linux-amd64/{etcd,etcdctl,etcdutl} /usr/local/bin mkdir -p /var/lib/etcd/ +chown -R etcd:etcd /var/lib/etcd +chmod -R 700 /var/lib/etcd mkdir -p /etc/etcd cat << EOT > /etc/systemd/system/etcd-member.service @@ -45,7 +45,7 @@ sed -e "s/~private_ipv4/$local_ipv4/g" -i /etc/systemd/system/etcd-member.servic ## Create a cronjob to defrag etcd data, but be careful to spread out the time across all nodes to maintain service availability cat <
"domain_name": "mycompany.int"
}