From 198e97ac22d8844fce427fbaef8c5553a14a9fb7 Mon Sep 17 00:00:00 2001 From: Jhon Lucas Date: Mon, 11 Mar 2024 10:11:00 -0300 Subject: [PATCH] chore: organizing the project to be open-source. --- devops/ecs/Dockerfile | 6 -- devops/ecs/infrastructure/.terraform.lock.hcl | 71 ------------------- .../build_and_push_ecr_image.sh | 8 +-- devops/ecs/infrastructure/main.tf | 6 +- devops/ecs/infrastructure/variables.tf | 4 +- devops/ecs/infrastructure/versions.tf | 2 +- 6 files changed, 9 insertions(+), 88 deletions(-) delete mode 100644 devops/ecs/infrastructure/.terraform.lock.hcl diff --git a/devops/ecs/Dockerfile b/devops/ecs/Dockerfile index 2b1824a..7dfc807 100644 --- a/devops/ecs/Dockerfile +++ b/devops/ecs/Dockerfile @@ -29,12 +29,6 @@ COPY ./src/ /project/src/ COPY ./config/ /project/config/ # 👇 Install pytnon packages -#RUN pip install -r requirements.txt -RUN poetry source add spectral-data-repository --secondary https://finance-362197681756.d.codeartifact.us-east-2.amazonaws.com/pypi/spectral-data-repository/simple/ \ - && poetry config repositories.spectral-data-repository https://finance-362197681756.d.codeartifact.us-east-2.amazonaws.com/pypi/spectral-data-repository/ \ - && poetry config http-basic.spectral-data-repository aws ${CODEARTIFACT_TOKEN} - -# Installing project's module dependencies RUN poetry install --no-interaction --no-ansi --no-dev # 👇 Fixing Critical vulnerabilites - GHSA-xvch-5gv4-984h and CVE-2021-44906 diff --git a/devops/ecs/infrastructure/.terraform.lock.hcl b/devops/ecs/infrastructure/.terraform.lock.hcl deleted file mode 100644 index 380ecc6..0000000 --- a/devops/ecs/infrastructure/.terraform.lock.hcl +++ /dev/null @@ -1,71 +0,0 @@ -# This file is maintained automatically by "terraform init". -# Manual edits may be lost in future updates. - -provider "registry.terraform.io/hashicorp/aws" { - version = "4.67.0" - constraints = ">= 3.0.0, < 5.0.0" - hashes = [ - "h1:5Zfo3GfRSWBaXs4TGQNOflr1XaYj6pRnVJLX5VAjFX4=", - "h1:L5c1etFqHmzTzxe8SDT78dIuGfuETeXDtctyUtiBpRs=", - "h1:dCRc4GqsyfqHEMjgtlM1EympBcgTmcTkWaJmtd91+KA=", - "zh:0843017ecc24385f2b45f2c5fce79dc25b258e50d516877b3affee3bef34f060", - "zh:19876066cfa60de91834ec569a6448dab8c2518b8a71b5ca870b2444febddac6", - "zh:24995686b2ad88c1ffaa242e36eee791fc6070e6144f418048c4ce24d0ba5183", - "zh:4a002990b9f4d6d225d82cb2fb8805789ffef791999ee5d9cb1fef579aeff8f1", - "zh:559a2b5ace06b878c6de3ecf19b94fbae3512562f7a51e930674b16c2f606e29", - "zh:6a07da13b86b9753b95d4d8218f6dae874cf34699bca1470d6effbb4dee7f4b7", - "zh:768b3bfd126c3b77dc975c7c0e5db3207e4f9997cf41aa3385c63206242ba043", - "zh:7be5177e698d4b547083cc738b977742d70ed68487ce6f49ecd0c94dbf9d1362", - "zh:8b562a818915fb0d85959257095251a05c76f3467caa3ba95c583ba5fe043f9b", - "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425", - "zh:9c385d03a958b54e2afd5279cd8c7cbdd2d6ca5c7d6a333e61092331f38af7cf", - "zh:b3ca45f2821a89af417787df8289cb4314b273d29555ad3b2a5ab98bb4816b3b", - "zh:da3c317f1db2469615ab40aa6baba63b5643bae7110ff855277a1fb9d8eb4f2c", - "zh:dc6430622a8dc5cdab359a8704aec81d3825ea1d305bbb3bbd032b1c6adfae0c", - "zh:fac0d2ddeadf9ec53da87922f666e1e73a603a611c57bcbc4b86ac2821619b1d", - ] -} - -provider "registry.terraform.io/hashicorp/null" { - version = "3.2.2" - hashes = [ - "h1:+0yGEKrNZZ6b9F1AqXYoaS2SPtAcCMvwLZynYaylMX8=", - "h1:IMVAUHKoydFrlPrl9OzasDnw/8ntZFerCC9iXw1rXQY=", - "h1:zT1ZbegaAYHwQa+QwIFugArWikRJI9dqohj8xb0GY88=", - "zh:3248aae6a2198f3ec8394218d05bd5e42be59f43a3a7c0b71c66ec0df08b69e7", - "zh:32b1aaa1c3013d33c245493f4a65465eab9436b454d250102729321a44c8ab9a", - "zh:38eff7e470acb48f66380a73a5c7cdd76cc9b9c9ba9a7249c7991488abe22fe3", - "zh:4c2f1faee67af104f5f9e711c4574ff4d298afaa8a420680b0cb55d7bbc65606", - "zh:544b33b757c0b954dbb87db83a5ad921edd61f02f1dc86c6186a5ea86465b546", - "zh:696cf785090e1e8cf1587499516b0494f47413b43cb99877ad97f5d0de3dc539", - "zh:6e301f34757b5d265ae44467d95306d61bef5e41930be1365f5a8dcf80f59452", - "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", - "zh:913a929070c819e59e94bb37a2a253c228f83921136ff4a7aa1a178c7cce5422", - "zh:aa9015926cd152425dbf86d1abdbc74bfe0e1ba3d26b3db35051d7b9ca9f72ae", - "zh:bb04798b016e1e1d49bcc76d62c53b56c88c63d6f2dfe38821afef17c416a0e1", - "zh:c23084e1b23577de22603cff752e59128d83cfecc2e6819edadd8cf7a10af11e", - ] -} - -provider "registry.terraform.io/kreuzwerker/docker" { - version = "2.15.0" - constraints = "2.15.0" - hashes = [ - "h1:NEBNtKNukqr6qk5vWu8Xx/nHBVNNdlty0hrFG76K7AE=", - "h1:V4SqgWd0+DqHYadZ/27lh9MeVStxzQGUBW1W+C4MVP0=", - "h1:yHmQdQ1lo2jycnUt0dTvixBi0f9KhfUTUEBPTqomYTQ=", - "zh:0241e5c7b66c14aa54e367dfe380fbde8388d3254cbe8a70717c12f71897e82b", - "zh:0f162f0a01ffe9eec32e78dfc2a5b05a373230c41f041439efa3f4b68903fdcb", - "zh:1c222c1425fbb0367154bcb8e4d87b19d6eae133fbb341f73102fa3b300f34bf", - "zh:679206433e31d8fa69d95167af6d2cd762218e89811b735ee20bd8da19f97854", - "zh:a16baab045bc7a709a9767816304cc5548aa2ee0b72c0eee49e3826e6a46a3fd", - "zh:a29c4e304a6a7faf5b651a61a91a1aa2e837591cff049fbe1c747b6319e43956", - "zh:bed7a69dbb9a024aecfac840ca2ac2f1527062f3d9c8c718e4e81d464b5ad329", - "zh:c22aa10806de216b6aa0b36a2c1114a9fdaa5b47925aaad3333de3ce24cc52c9", - "zh:d034295663d8a39035fd6fdf0488b72544b13f48acaee797af967343248676f8", - "zh:d9001dfeac0db1799d8ab9d04be090522529baae0dba7f7e82b71f2168f05582", - "zh:d9f3eb7ef8f256eb9148d72bd4a00e34f3be5570484598e26b603cbdc5eed923", - "zh:ef573f1e0f000127fa9f569c8ee320f03ba1d302a6fbf9aac8300efa6fc6f4b2", - "zh:ef7e246b4205202161b4119c10a1282f74243e029508184132731f9f6f6d9f4c", - ] -} diff --git a/devops/ecs/infrastructure/build_and_push_ecr_image.sh b/devops/ecs/infrastructure/build_and_push_ecr_image.sh index 0cc1d2f..84189d8 100644 --- a/devops/ecs/infrastructure/build_and_push_ecr_image.sh +++ b/devops/ecs/infrastructure/build_and_push_ecr_image.sh @@ -4,12 +4,12 @@ set -e IMAGE_NAME=$1 ENVIRONMENT=$2 DOMAIN_NAME=finance -DOMAIN_OWNER=362197681756 +DOMAIN_OWNER=${DOMAIN_OWNER} cd ../../../; # go to root of project # login to ECR -aws ecr get-login-password --region us-east-2 | docker login --username AWS --password-stdin 362197681756.dkr.ecr.us-east-2.amazonaws.com; +aws ecr get-login-password --region us-east-2 | docker login --username AWS --password-stdin ${DOMAIN_OWNER}.dkr.ecr.us-east-2.amazonaws.com; # Build the docker image docker build --platform=linux/amd64 --build-arg CODEARTIFACT_TOKEN=`aws codeartifact get-authorization-token --domain ${DOMAIN_NAME} \ @@ -17,7 +17,7 @@ docker build --platform=linux/amd64 --build-arg CODEARTIFACT_TOKEN=`aws codearti -t data-lakehouse/${IMAGE_NAME}-${ENVIRONMENT} -f devops/ecs/Dockerfile .; # Tag the image -docker tag data-lakehouse/${IMAGE_NAME}-${ENVIRONMENT}:latest 362197681756.dkr.ecr.us-east-2.amazonaws.com/data-lakehouse/${IMAGE_NAME}-${ENVIRONMENT}:latest; +docker tag data-lakehouse/${IMAGE_NAME}-${ENVIRONMENT}:latest ${DOMAIN_OWNER}.dkr.ecr.us-east-2.amazonaws.com/data-lakehouse/${IMAGE_NAME}-${ENVIRONMENT}:latest; # Push the image to ECR -docker push 362197681756.dkr.ecr.us-east-2.amazonaws.com/data-lakehouse/${IMAGE_NAME}-${ENVIRONMENT}:latest; +docker push ${DOMAIN_OWNER}.dkr.ecr.us-east-2.amazonaws.com/data-lakehouse/${IMAGE_NAME}-${ENVIRONMENT}:latest; diff --git a/devops/ecs/infrastructure/main.tf b/devops/ecs/infrastructure/main.tf index 1b7d051..482221d 100644 --- a/devops/ecs/infrastructure/main.tf +++ b/devops/ecs/infrastructure/main.tf @@ -171,14 +171,12 @@ data "aws_iam_policy_document" "ecs_task_execution_role" { ] effect = "Allow" resources = [ - "arn:aws:s3:::spectral-athena-results", - "arn:aws:s3:::spectral-athena-results/*", "arn:aws:s3:::data-lakehouse-dev", "arn:aws:s3:::data-lakehouse-dev/*", "arn:aws:s3:::data-lakehouse-prod", "arn:aws:s3:::data-lakehouse-prod/*", - "arn:aws:s3:::aws-athena-query-results-362197681756-us-east-2", - "arn:aws:s3:::aws-athena-query-results-362197681756-us-east-2/*", + "arn:aws:s3:::aws-athena-query-results", + "arn:aws:s3:::aws-athena-query-results/*", ] } } diff --git a/devops/ecs/infrastructure/variables.tf b/devops/ecs/infrastructure/variables.tf index e388480..4b07028 100644 --- a/devops/ecs/infrastructure/variables.tf +++ b/devops/ecs/infrastructure/variables.tf @@ -4,13 +4,13 @@ locals { env = terraform.workspace == "default" ? "dev" : terraform.workspace project = "defi-features-data-pipeline" team = "data-engineering" - account_id = "362197681756" #TODO: Remove this hardcode and use data.aws_caller_identity.current.account_id + account_id = "account_id" #TODO: Remove this hardcode and use data.aws_caller_identity.current.account_id } variable "ecr_repo_url" { type = string description = "URI of the ECR repository" - default = "362197681756.dkr.ecr.us-east-2.amazonaws.com/data-lakehouse/defi-features-data-pipeline" + default = "account_id.dkr.ecr.us-east-2.amazonaws.com/data-lakehouse/defi-features-data-pipeline" } variable "ecr_repo_image_tag" { diff --git a/devops/ecs/infrastructure/versions.tf b/devops/ecs/infrastructure/versions.tf index 09a4398..73fe16f 100644 --- a/devops/ecs/infrastructure/versions.tf +++ b/devops/ecs/infrastructure/versions.tf @@ -2,7 +2,7 @@ terraform { required_version = ">= 0.13" backend "s3" { - bucket = "spectral-terraform-state" + bucket = "bucket-terraform-state" key = "tf-worskpaces/defi-features-data-pipeline/terraform.tfstate" region = "us-east-2" }