From 27c84443eb5940b7b80f13d0f9511b0f49b00a27 Mon Sep 17 00:00:00 2001 From: ZacharyWills <60660437+ZacharyWills@users.noreply.github.com> Date: Fri, 17 May 2024 09:49:13 -0600 Subject: [PATCH 1/6] Add s3 buckets, and infra for making more --- terraform/ioos-s3.tf | 112 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 112 insertions(+) create mode 100644 terraform/ioos-s3.tf diff --git a/terraform/ioos-s3.tf b/terraform/ioos-s3.tf new file mode 100644 index 0000000..42e0d08 --- /dev/null +++ b/terraform/ioos-s3.tf @@ -0,0 +1,112 @@ +resource "aws_iam_user" "noaa-ioos-users" { + count = length(var.noaa-ioos-users) + name = element(var.noaa-ioos-users, count.index) + path = "/external-users/" +} + +variable "noaa-ioos-users" { + type = list(string) + default = ["OWP", "GLERL", "NOS", "CORA", "LiveOcean"] +} + +resource "aws_iam_policy" "input-policy" { + name = "noaa-ioos_bucket_input_policy" + path = "/" + description = "policy allowing noaa-ioos sources to put data into their bucket locations" + + # Terraform's "jsonencode" function converts a + # Terraform expression result to valid JSON syntax. + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Action = [ + "S3:*", + ] + Effect = "Allow" + Resource = "arn:aws:s3:::noaa-ioos/$${aws:username}" + }, + ] + }) +} + + resource "aws_s3_bucket" "noaa-ioos" { + bucket = "noaa-ioos" + tags = { + Name = "noaa-ioos" + } + } + +#data "aws_s3_bucket" "noaa-ioos" { +# bucket = noaa-ioos +# # bucket = "lynker-noaa-ioos" +#} + +resource "aws_iam_group" "noaa-ioos" { + name = "noaa-ioos" + path = "/" +} + +resource "aws_iam_group_policy_attachment" "noaa-ioos-attach" { + group = aws_iam_group.noaa-ioos.name + policy_arn = aws_iam_policy.input-policy.arn +} + +resource "aws_iam_user_group_membership" "Users" { +# user = aws_iam_user.NERFC.name + count = length(var.noaa-ioos-users) + user = element(var.noaa-ioos-users, count.index) + groups = [ + aws_iam_group.noaa-ioos.name, + ] +} + + +# data "aws_iam_policy" "input-policy" { +# name = "noaa-ioos_bucket_input_policy" +# path = "/" +# description = "policy allowing noaa-ioos sources to put data into their bucket locations" + +# # Terraform's "jsonencode" function converts a +# # Terraform expression result to valid JSON syntax. +# policy = jsonencode({ +# Version = "2012-10-17" +# Statement = [ +# { +# Action = [ +# "S3:*", +# ] +# Effect = "Allow" +# Resource = "arn:aws:s3:::noaa-ioos/$${aws:username}" +# }, +# ] +# }) +# } + +# data "aws_s3_bucket" "noaa-ioos" { +# bucket = var.mount_bucket_name +# # bucket = "noaa-ioos" + +# tags = { +# Name = "noaa-ioos" +# } +# } + +# data "aws_iam_group" "noaa-ioos" { +# name = "noaa-ioos" +# path = "/" +# } + +#data "aws_iam_group_policy_attachment" "noaa-ioos-attach" { +# group = aws_iam_group.noaa-ioos.name +# policy_arn = aws_iam_policy.input-policy.arn +# } + +# data "aws_iam_user_group_membership" "NERFC" { +# # user = aws_iam_user.NERFC.name +# count = length(var.noaa-ioos-users) +# user = element(var.noaa-ioos-users, count.index) +# groups = [ +# aws_iam_group.noaa-ioos.name, +# ] +# } \ No newline at end of file From 42948b297767ce375fc7f35ab20ff6b34833e464 Mon Sep 17 00:00:00 2001 From: ZacharyWills <60660437+ZacharyWills@users.noreply.github.com> Date: Mon, 20 May 2024 14:58:25 -0600 Subject: [PATCH 2/6] Update getaws.sh to add SSM Agent --- scripts/getaws.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/scripts/getaws.sh b/scripts/getaws.sh index 19e91e1..c7a8949 100755 --- a/scripts/getaws.sh +++ b/scripts/getaws.sh @@ -3,3 +3,8 @@ curl "https://s3.amazonaws.com/aws-cli/awscli-bundle.zip" -o "awscli-bundle.zip" unzip awscli-bundle.zip sudo ./awscli-bundle/install -i /usr/local/aws -b /usr/local/bin/aws + +#Add SSM agent for AWS +sudo dnf install -y https://s3.amazonaws.com/ec2-downloads-windows/SSMAgent/latest/linux_amd64/amazon-ssm-agent.rpm +sudo systemctl enable amazon-ssm-agent +sudo systemctl start amazon-ssm-agent From 97506d0cc1750b4dc44a8093bc160858fe80e4d5 Mon Sep 17 00:00:00 2001 From: ZacharyWills <60660437+ZacharyWills@users.noreply.github.com> Date: Thu, 23 May 2024 23:37:34 -0600 Subject: [PATCH 3/6] Update ioos-s3.tf --- terraform/ioos-s3.tf | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/terraform/ioos-s3.tf b/terraform/ioos-s3.tf index 42e0d08..26f0be6 100644 --- a/terraform/ioos-s3.tf +++ b/terraform/ioos-s3.tf @@ -30,17 +30,20 @@ resource "aws_iam_policy" "input-policy" { }) } - resource "aws_s3_bucket" "noaa-ioos" { - bucket = "noaa-ioos" +# resource "aws_s3_bucket" "noaa-ioos" { +# bucket = "noaa-ioos" +# tags = { +# Name = "noaa-ioos" +# } +# } + +data "aws_s3_bucket" "noaa-ioos" { + bucket = ioos-coastalsb-inputs tags = { - Name = "noaa-ioos" + Name = "ioos-coastalsb-inputs" } - } +} -#data "aws_s3_bucket" "noaa-ioos" { -# bucket = noaa-ioos -# # bucket = "lynker-noaa-ioos" -#} resource "aws_iam_group" "noaa-ioos" { name = "noaa-ioos" @@ -109,4 +112,4 @@ resource "aws_iam_user_group_membership" "Users" { # groups = [ # aws_iam_group.noaa-ioos.name, # ] -# } \ No newline at end of file +# } From 38589dc7e8f7a0d5e39cf0276ce511e336f71b90 Mon Sep 17 00:00:00 2001 From: ZacharyWills <60660437+ZacharyWills@users.noreply.github.com> Date: Tue, 2 Jul 2024 11:49:04 -0600 Subject: [PATCH 4/6] Update init_template.tpl to mount buckets --- terraform/init_template.tpl | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/terraform/init_template.tpl b/terraform/init_template.tpl index f333067..e8f9b1b 100644 --- a/terraform/init_template.tpl +++ b/terraform/init_template.tpl @@ -16,6 +16,11 @@ BRANCH=main mkdir -p /mnt/efs/fs1 sudo yum -y -q install git +#Install mountpoint-s3 and mount buckets: +sudo yum -y install https://s3.amazonaws.com/mountpoint-s3-release/latest/x86_64/mount-s3.rpm +sudo mount-s3 --allow-delete --allow-overwrite --allow-other ioos-coastalsb-inputs /inputs_bucket +sudo mount-s3 --allow-delete --allow-overwrite --allow-other ioos-coastalsb-outputs /outputs_bucket + sudo yum -y install amazon-efs-utils if [ $? -ne 0 ]; then From 8951d8ea1c99147d550b4263679c73741d512f24 Mon Sep 17 00:00:00 2001 From: ZacharyWills <60660437+ZacharyWills@users.noreply.github.com> Date: Tue, 2 Jul 2024 11:50:23 -0600 Subject: [PATCH 5/6] Update init_template.tpl to add SSM agent --- terraform/init_template.tpl | 3 +++ 1 file changed, 3 insertions(+) diff --git a/terraform/init_template.tpl b/terraform/init_template.tpl index e8f9b1b..3612121 100644 --- a/terraform/init_template.tpl +++ b/terraform/init_template.tpl @@ -21,6 +21,9 @@ sudo yum -y install https://s3.amazonaws.com/mountpoint-s3-release/latest/x86_64 sudo mount-s3 --allow-delete --allow-overwrite --allow-other ioos-coastalsb-inputs /inputs_bucket sudo mount-s3 --allow-delete --allow-overwrite --allow-other ioos-coastalsb-outputs /outputs_bucket +#Install AWS Utils +sudo yum -y install https://s3.amazonaws.com/ec2-downloads-windows/SSMAgent/latest/linux_amd64/amazon-ssm-agent.rpm + sudo yum -y install amazon-efs-utils if [ $? -ne 0 ]; then From c024b13b1e7ad7fa193970e1e47f6bf06e231787 Mon Sep 17 00:00:00 2001 From: Mykel Alvis Date: Wed, 30 Oct 2024 09:16:39 -0500 Subject: [PATCH 6/6] Move inputs slightly --- terraform/init_template.tpl | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/terraform/init_template.tpl b/terraform/init_template.tpl index 3612121..90fb8d7 100644 --- a/terraform/init_template.tpl +++ b/terraform/init_template.tpl @@ -17,8 +17,14 @@ mkdir -p /mnt/efs/fs1 sudo yum -y -q install git #Install mountpoint-s3 and mount buckets: + +for a in ioos-coastalsb-inputs ioos-transfers ioos-und-transfer +do + sudo mkdir -p /inputs/$a + sudo mount-s3 --read-only --allow-other $a /inputs/$a +done +sudo mkdir -p /outputs_bucket sudo yum -y install https://s3.amazonaws.com/mountpoint-s3-release/latest/x86_64/mount-s3.rpm -sudo mount-s3 --allow-delete --allow-overwrite --allow-other ioos-coastalsb-inputs /inputs_bucket sudo mount-s3 --allow-delete --allow-overwrite --allow-other ioos-coastalsb-outputs /outputs_bucket #Install AWS Utils