From 5e9bfe74424b5d093a8d8c32c393ff7cfc9165b6 Mon Sep 17 00:00:00 2001 From: Khai Do Date: Tue, 10 Sep 2024 10:07:44 -0700 Subject: [PATCH 1/2] [IT-3512] Refactor and cleanup * remove docker folder because this repo doesn't use it * remove LoadBalancedHttpServiceStack because it's not used * Get namespace, subdomain and vpc cidr from config file instead of hard coding it in code. --- app.py | 24 ++++++++----- cdk.json | 3 +- docker/README.md | 63 --------------------------------- docker/service-one/dockerfile | 13 ------- openchallenges/service_stack.py | 49 +------------------------ 5 files changed, 19 insertions(+), 133 deletions(-) delete mode 100644 docker/README.md delete mode 100644 docker/service-one/dockerfile diff --git a/app.py b/app.py index 3d6cc5b..7e1f50f 100644 --- a/app.py +++ b/app.py @@ -4,22 +4,30 @@ from openchallenges.network_stack import NetworkStack from openchallenges.ecs_stack import EcsStack from openchallenges.service_stack import ServiceStack -from openchallenges.service_stack import LoadBalancedHttpsServiceStack +from openchallenges.service_stack import LoadBalancedServiceStack from openchallenges.load_balancer_stack import LoadBalancerStack from openchallenges.service_props import ServiceProps import openchallenges.utils as utils app = cdk.App() +# get the environment environment = utils.get_environment() + +# get VARS from cdk.json env_vars = app.node.try_get_context(environment) +dns_namespace = env_vars["DNS_NAMESPACE"] +dns_sub_domain = env_vars["DNS_SUBDOMAIN"] +vpc_cidr = env_vars["VPC_CIDR"] + +# get secrets from cdk.json or aws parameter store secrets = utils.get_secrets(app) bucket_stack = BucketStack(app, "openchallenges-buckets") -network_stack = NetworkStack(app, "openchallenges-network", env_vars["VPC_CIDR"]) -ecs_stack = EcsStack( - app, "openchallenges-ecs", network_stack.vpc, env_vars["DNS_NAMESPACE"] -) + +network_stack = NetworkStack(app, "openchallenges-network", vpc_cidr) + +ecs_stack = EcsStack(app, "openchallenges-ecs", network_stack.vpc, dns_namespace) mariadb_props = ServiceProps( "openchallenges-mariadb", @@ -275,9 +283,9 @@ 1024, "ghcr.io/sage-bionetworks/openchallenges-app:edge", { - "API_DOCS_URL": "https://newinfra.openchallenges.io/api-docs", + "API_DOCS_URL": f"https://{dns_sub_domain}.{dns_namespace}/api-docs", "APP_VERSION": "1.0.0-alpha", - "CSR_API_URL": "https://newinfra.openchallenges.io/api/v1", + "CSR_API_URL": f"https://{dns_sub_domain}.{dns_namespace}/api/v1", "DATA_UPDATED_ON": "2023-09-26", "ENVIRONMENT": "production", "GOOGLE_TAG_MANAGER_ID": "", @@ -335,7 +343,7 @@ }, ) -apex_service_stack = LoadBalancedHttpsServiceStack( +apex_service_stack = LoadBalancedServiceStack( app, "openchallenges-apex", network_stack.vpc, diff --git a/cdk.json b/cdk.json index 6172212..318d450 100644 --- a/cdk.json +++ b/cdk.json @@ -65,7 +65,8 @@ "@aws-cdk/aws-eks:nodegroupNameAttribute": true, "dev": { "VPC_CIDR": "10.255.92.0/24", - "DNS_NAMESPACE": "openchallenges.io" + "DNS_NAMESPACE": "openchallenges.io", + "DNS_SUBDOMAIN": "newinfra" }, "secrets": { "MARIADB_PASSWORD": "/openchallenges/MARIADB_PASSWORD", diff --git a/docker/README.md b/docker/README.md deleted file mode 100644 index e010df2..0000000 --- a/docker/README.md +++ /dev/null @@ -1,63 +0,0 @@ -# Build and push to docker - -This folder contains docker containers that can be built and integrated with the infrastructure -deployment. - -## Build and Tag docker image -``` -cd service-one -docker build -t service-one . -``` - -Tag for ECR -``` -docker tag service-two:latest 804034162148.dkr.ecr.us-east-1.amazonaws.com/service-one -``` - -check builds with `docker images` - -## Tag and Push to AWS ECR - -### login to AWS and ECR - -login to AWS: -``` -aws sso login --profile itsandbox-admin` -``` - -login to ECR: -``` -aws ecr get-login-password --profile itsandbox-admin --region us-east-1 | docker login --username AWS --password-stdin 804034162148.dkr.ecr.us-east-1.amazonaws.com -``` - -if get-login-password doesn't work it might be due to -[issues with the ecr credentials helper](https://github.com/awslabs/amazon-ecr-credential-helper/issues/229). -try to install the latest version of the ecr credentials helper tools -``` -go install github.com/awslabs/amazon-ecr-credential-helper/ecr-login/cli/docker-credential-ecr-login@latest -``` - - -### create a repository and push container - -create repository: -``` -AWS_PROFILE=itsandbox-admin AWS_DEFAULT_REGION=us-east-1 aws ecr create-repository --repository-name service-one --region us-east-1 -``` - -push container to ECR: -``` - AWS_PROFILE=itsandbox-admin AWS_DEFAULT_REGION=us-east-1 docker push 804034162148.dkr.ecr.us-east-1.amazonaws.com/service-one -``` - - -### Reference container with CDK - -Use the [ECS from_asset method](https://docs.aws.amazon.com/cdk/api/v2/python/aws_cdk.aws_ecs/ContainerImage.html#aws_cdk.aws_ecs.ContainerImage.from_asset) -to build and use the container image when running `cdk synth` - -```python -from aws_cdk import aws_ecs as ecs - -ecs.ContainerImage.from_asset("docker/service-one") -``` diff --git a/docker/service-one/dockerfile b/docker/service-one/dockerfile deleted file mode 100644 index 16ef563..0000000 --- a/docker/service-one/dockerfile +++ /dev/null @@ -1,13 +0,0 @@ -FROM public.ecr.aws/docker/library/busybox - -# Create a non-root user to own the files and run our server -RUN adduser -D static -USER static -WORKDIR /home/static - -RUN echo "Healthy" > index.html && \ - mkdir service-one && \ - echo "Hello from service one" > service-one/index.html - -# Run BusyBox httpd -CMD ["busybox", "httpd", "-f", "-v", "-p", "8080"] diff --git a/openchallenges/service_stack.py b/openchallenges/service_stack.py index 47a8664..9c1a683 100644 --- a/openchallenges/service_stack.py +++ b/openchallenges/service_stack.py @@ -117,54 +117,7 @@ def __init__( ) -class LoadBalancedHttpServiceStack(ServiceStack): - """ - A special stack to create an ECS service fronted by a load balancer. This allows us to split up - the ECS services and the load balancer into separate stacks. It makes maintaining the stacks - easier. Unfortunately, due to the way AWS works, setting up a load balancer and ECS service - in different stacks may cause cyclic references. - https://docs.aws.amazon.com/cdk/api/v2/python/aws_cdk.aws_ecs/README.html#using-a-load-balancer-from-a-different-stack - - To work around this problem we use the "Split at listener" option from - https://github.com/aws-samples/aws-cdk-examples - """ - - def __init__( - self, - scope: Construct, - construct_id: str, - vpc: ec2.Vpc, - cluster: ecs.Cluster, - props: ServiceProps, - load_balancer: elbv2.ApplicationLoadBalancer, - listener_port: int, - health_check_path: str = "/", - health_check_interval: int = 1, # max is 5 - **kwargs, - ) -> None: - super().__init__(scope, construct_id, vpc, cluster, props, **kwargs) - - http_listener = elbv2.ApplicationListener( - self, - "HttpListener", - load_balancer=load_balancer, - port=listener_port, - open=True, - protocol=elbv2.ApplicationProtocol.HTTP, - ) - - http_listener.add_targets( - "HttpTarget", - port=props.container_port, - protocol=elbv2.ApplicationProtocol.HTTP, - targets=[self.service], - health_check=elbv2.HealthCheck( - path=health_check_path, interval=duration.minutes(health_check_interval) - ), - ) - - -class LoadBalancedHttpsServiceStack(ServiceStack): +class LoadBalancedServiceStack(ServiceStack): """ A special stack to create an ECS service fronted by a load balancer. This allows us to split up the ECS services and the load balancer into separate stacks. It makes maintaining the stacks From 23d9187b269993e860fcdb875b6944daedaf0f28 Mon Sep 17 00:00:00 2001 From: Khai Do Date: Thu, 12 Sep 2024 09:25:18 -0700 Subject: [PATCH 2/2] read certificate arn from config file --- README.md | 13 +++++++++++++ app.py | 2 ++ cdk.json | 3 ++- openchallenges/service_stack.py | 5 ++--- 4 files changed, 19 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index e9ab755..a800626 100644 --- a/README.md +++ b/README.md @@ -123,6 +123,19 @@ For example, using the `prod` environment: ENV=prod cdk synth ``` +# Certificates + +Certificates to set up HTTPS connections should be created manually in AWS certificate manager. +This is not automated due to AWS requiring manual verification of the domain ownership. +Once created take the ARN of the certificate and add it to a context in cdk.json. + +```json + "context": { + "dev": { + "CERTIFICATE_ARN": "arn:aws:acm:us-east-1:804034162148:certificate/76ed5a71-4aa8-4cc1-9db6-aa7a322ec077" + } + } +``` # Secrets diff --git a/app.py b/app.py index 7e1f50f..9b60048 100644 --- a/app.py +++ b/app.py @@ -19,6 +19,7 @@ dns_namespace = env_vars["DNS_NAMESPACE"] dns_sub_domain = env_vars["DNS_SUBDOMAIN"] vpc_cidr = env_vars["VPC_CIDR"] +certificate_arn = env_vars["CERTIFICATE_ARN"] # get secrets from cdk.json or aws parameter store secrets = utils.get_secrets(app) @@ -351,6 +352,7 @@ apex_service_props, load_balancer_stack.alb, 80, + certificate_arn, "/health", health_check_interval=5, ) diff --git a/cdk.json b/cdk.json index 318d450..1085491 100644 --- a/cdk.json +++ b/cdk.json @@ -66,7 +66,8 @@ "dev": { "VPC_CIDR": "10.255.92.0/24", "DNS_NAMESPACE": "openchallenges.io", - "DNS_SUBDOMAIN": "newinfra" + "DNS_SUBDOMAIN": "newinfra", + "CERTIFICATE_ARN": "arn:aws:acm:us-east-1:804034162148:certificate/76ed5a71-4aa8-4cc1-9db6-aa7a322ec077" }, "secrets": { "MARIADB_PASSWORD": "/openchallenges/MARIADB_PASSWORD", diff --git a/openchallenges/service_stack.py b/openchallenges/service_stack.py index 9c1a683..3ab568d 100644 --- a/openchallenges/service_stack.py +++ b/openchallenges/service_stack.py @@ -15,8 +15,6 @@ ALB_HTTP_LISTENER_PORT = 80 ALB_HTTPS_LISTENER_PORT = 443 -# manually created cert -CERTIFICATE_ARN = "arn:aws:acm:us-east-1:804034162148:certificate/76ed5a71-4aa8-4cc1-9db6-aa7a322ec077" class ServiceStack(cdk.Stack): @@ -138,6 +136,7 @@ def __init__( props: ServiceProps, load_balancer: elbv2.ApplicationLoadBalancer, listener_port: int, + certificate_arn: str, health_check_path: str = "/", health_check_interval: int = 1, # max is 5 **kwargs, @@ -148,7 +147,7 @@ def __init__( # ACM Certificate for HTTPS # ------------------- self.cert = acm.Certificate.from_certificate_arn( - self, "Cert", certificate_arn=CERTIFICATE_ARN + self, "Cert", certificate_arn=certificate_arn ) # -------------------------------