From 734fc9dbcd148588b045244fad768c5c3d5fdb6a Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 24 Sep 2024 09:38:23 -0700 Subject: [PATCH 01/26] [pre-commit.ci] pre-commit autoupdate (#48) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/PyCQA/flake8: 7.1.0 → 7.1.1](https://github.com/PyCQA/flake8/compare/7.1.0...7.1.1) - [github.com/awslabs/cfn-python-lint: v1.9.0 → v1.15.0](https://github.com/awslabs/cfn-python-lint/compare/v1.9.0...v1.15.0) - [github.com/psf/black: 24.4.2 → 24.8.0](https://github.com/psf/black/compare/24.4.2...24.8.0) - [github.com/sirosen/check-jsonschema: 0.29.1 → 0.29.2](https://github.com/sirosen/check-jsonschema/compare/0.29.1...0.29.2) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9a6e254..9f7206e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -9,7 +9,7 @@ repos: - id: mixed-line-ending - id: trailing-whitespace - repo: https://github.com/PyCQA/flake8 - rev: 7.1.0 + rev: 7.1.1 hooks: - id: flake8 - repo: https://github.com/adrienverge/yamllint @@ -17,7 +17,7 @@ repos: hooks: - id: yamllint - repo: https://github.com/awslabs/cfn-python-lint - rev: v1.9.0 + rev: v1.15.0 hooks: - id: cfn-python-lint args: @@ -32,11 +32,11 @@ repos: ^.pre-commit-config.yaml ) - repo: https://github.com/psf/black - rev: 24.4.2 + rev: 24.8.0 hooks: - id: black - repo: https://github.com/sirosen/check-jsonschema - rev: 0.29.1 + rev: 0.29.2 hooks: - id: check-github-workflows - id: check-github-actions From 2f1d1b5de94a43809be64f80c274562f634f07a5 Mon Sep 17 00:00:00 2001 From: Thomas Schaffter Date: Thu, 26 Sep 2024 10:26:15 -0700 Subject: [PATCH 02/26] [IT-3918] Fix the image URLs returned by the image service (#51) * update the config of the image service * use `{fully_qualified_domain_name}` * use an f-string --- README.md | 2 +- app.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 1421fb3..fce5714 100644 --- a/README.md +++ b/README.md @@ -233,7 +233,7 @@ OpenChallenges application. * Add the Certificate ARN to the cdk.json * Update references to the OC docker images in [app.py](app.py) (i.e. `ghcr.io/sage-bionetworks/openchallenges-xxx:`) -* (Optional) Update the ServiceProps objects in [app.py](app.py) with parameters specific to +* (Optional) Update the `ServiceProps` objects in [app.py](app.py) with parameters specific to each container. ## Login with the AWS CLI diff --git a/app.py b/app.py index 4d5f43c..116e95b 100644 --- a/app.py +++ b/app.py @@ -180,6 +180,8 @@ "SERVER_PORT": "8086", "SPRING_CLOUD_CONFIG_URI": "http://openchallenges-config-server:8090", "SERVICE_REGISTRY_URL": "http://openchallenges-service-registry:8081/eureka", + "OPENCHALLENGES_IMAGE_SERVICE_THUMBOR_HOST": f"https://{fully_qualified_domain_name}/img/", + "OPENCHALLENGES_IMAGE_SERVICE_THUMBOR_SECURITY_KEY": secrets["SECURITY_KEY"], "OPENCHALLENGES_IMAGE_SERVICE_IS_DEPLOYED_ON_AWS": "true", }, ) From 63303d1acbdad547bb8560c939119068263c1029 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 30 Sep 2024 13:11:20 -0700 Subject: [PATCH 03/26] [pre-commit.ci] pre-commit autoupdate (#52) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/awslabs/cfn-python-lint: v1.15.0 → v1.15.2](https://github.com/awslabs/cfn-python-lint/compare/v1.15.0...v1.15.2) - [github.com/sirosen/check-jsonschema: 0.29.2 → 0.29.3](https://github.com/sirosen/check-jsonschema/compare/0.29.2...0.29.3) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9f7206e..6699ecb 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -17,7 +17,7 @@ repos: hooks: - id: yamllint - repo: https://github.com/awslabs/cfn-python-lint - rev: v1.15.0 + rev: v1.15.2 hooks: - id: cfn-python-lint args: @@ -36,7 +36,7 @@ repos: hooks: - id: black - repo: https://github.com/sirosen/check-jsonschema - rev: 0.29.2 + rev: 0.29.3 hooks: - id: check-github-workflows - id: check-github-actions From 81459a47ac718bc9ccf1ba523965187f639564b6 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 7 Oct 2024 15:10:50 -0700 Subject: [PATCH 04/26] [pre-commit.ci] pre-commit autoupdate (#53) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/pre-commit/pre-commit-hooks: v4.6.0 → v5.0.0](https://github.com/pre-commit/pre-commit-hooks/compare/v4.6.0...v5.0.0) - [github.com/awslabs/cfn-python-lint: v1.15.2 → v1.16.0](https://github.com/awslabs/cfn-python-lint/compare/v1.15.2...v1.16.0) - [github.com/psf/black: 24.8.0 → 24.10.0](https://github.com/psf/black/compare/24.8.0...24.10.0) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 6699ecb..b3f17c8 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -3,7 +3,7 @@ default_language_version: repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.6.0 + rev: v5.0.0 hooks: - id: end-of-file-fixer - id: mixed-line-ending @@ -17,7 +17,7 @@ repos: hooks: - id: yamllint - repo: https://github.com/awslabs/cfn-python-lint - rev: v1.15.2 + rev: v1.16.0 hooks: - id: cfn-python-lint args: @@ -32,7 +32,7 @@ repos: ^.pre-commit-config.yaml ) - repo: https://github.com/psf/black - rev: 24.8.0 + rev: 24.10.0 hooks: - id: black - repo: https://github.com/sirosen/check-jsonschema From e42765109dd858a36920aec0ee6b772d5bfcdff6 Mon Sep 17 00:00:00 2001 From: Thomas Schaffter Date: Fri, 11 Oct 2024 14:45:08 -0700 Subject: [PATCH 05/26] Parametrize stack version and update app config (#54) * parametrize stack version * update data update date * set Google tag manager ID * rename `stack_version` to `image_version` --- app.py | 31 ++++++++++++++++--------------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/app.py b/app.py index 116e95b..bfcea3a 100644 --- a/app.py +++ b/app.py @@ -14,6 +14,7 @@ # get the environment environment = utils.get_environment() stack_name_prefix = f"openchallenges-{environment}" +image_version = "0.0.11" # get VARS from cdk.json env_vars = app.node.try_get_context(environment) @@ -38,7 +39,7 @@ "openchallenges-mariadb", 3306, 512, - "ghcr.io/sage-bionetworks/openchallenges-mariadb:edge", + f"ghcr.io/sage-bionetworks/openchallenges-mariadb:{image_version}", { "MARIADB_USER": "maria", "MARIADB_PASSWORD": secrets["MARIADB_PASSWORD"], @@ -58,7 +59,7 @@ "openchallenges-elasticsearch", 9200, 2048, - "ghcr.io/sage-bionetworks/openchallenges-elasticsearch:edge", + f"ghcr.io/sage-bionetworks/openchallenges-elasticsearch:{image_version}", { "bootstrap.memory_lock": "true", "discovery.type": "single-node", # https://stackoverflow.com/a/68253868 @@ -78,7 +79,7 @@ "openchallenges-thumbor", 8889, 512, - "ghcr.io/sage-bionetworks/openchallenges-thumbor:edge", + f"ghcr.io/sage-bionetworks/openchallenges-thumbor:{image_version}", { "LOG_LEVEL": "info", "PORT": "8889", @@ -115,7 +116,7 @@ "openchallenges-config-server", 8090, 1024, - "ghcr.io/sage-bionetworks/openchallenges-config-server:edge", + f"ghcr.io/sage-bionetworks/openchallenges-config-server:{image_version}", { "GIT_DEFAULT_LABEL": "test-2", "GIT_HOST_KEY_ALGORITHM": "ssh-ed25519", @@ -138,7 +139,7 @@ "openchallenges-service-registry", 8081, 1024, - "ghcr.io/sage-bionetworks/openchallenges-service-registry:edge", + f"ghcr.io/sage-bionetworks/openchallenges-service-registry:{image_version}", { "SERVER_PORT": "8081", "DEFAULT_ZONE": "http://localhost:8081/eureka", @@ -159,7 +160,7 @@ "openchallenges-zipkin", 9411, 512, - "ghcr.io/sage-bionetworks/openchallenges-zipkin:edge", + f"ghcr.io/sage-bionetworks/openchallenges-zipkin:{image_version}", {}, ) @@ -175,7 +176,7 @@ "openchallenges-image-service", 8086, 1024, - "ghcr.io/sage-bionetworks/openchallenges-image-service:edge", + f"ghcr.io/sage-bionetworks/openchallenges-image-service:{image_version}", { "SERVER_PORT": "8086", "SPRING_CLOUD_CONFIG_URI": "http://openchallenges-config-server:8090", @@ -201,7 +202,7 @@ "openchallenges-challenge-service", 8085, 1024, - "ghcr.io/sage-bionetworks/openchallenges-challenge-service:edge", + f"ghcr.io/sage-bionetworks/openchallenges-challenge-service:{image_version}", { "SERVER_PORT": "8085", "SPRING_CLOUD_CONFIG_URI": "http://openchallenges-config-server:8090", @@ -239,7 +240,7 @@ "openchallenges-organization-service", 8084, 1024, - "ghcr.io/sage-bionetworks/openchallenges-organization-service:edge", + f"ghcr.io/sage-bionetworks/openchallenges-organization-service:{image_version}", { "SERVER_PORT": "8084", "SPRING_CLOUD_CONFIG_URI": "http://openchallenges-config-server:8090", @@ -270,7 +271,7 @@ "openchallenges-api-gateway", 8082, 1024, - "ghcr.io/sage-bionetworks/openchallenges-api-gateway:edge", + f"ghcr.io/sage-bionetworks/openchallenges-api-gateway:{image_version}", { "SERVER_PORT": "8082", "SPRING_CLOUD_CONFIG_URI": "http://openchallenges-config-server:8090", @@ -293,14 +294,14 @@ "openchallenges-app", 4200, 1024, - "ghcr.io/sage-bionetworks/openchallenges-app:edge", + f"ghcr.io/sage-bionetworks/openchallenges-app:{image_version}", { "API_DOCS_URL": f"https://{fully_qualified_domain_name}/api-docs", "APP_VERSION": "1.0.0-alpha", "CSR_API_URL": f"https://{fully_qualified_domain_name}/api/v1", - "DATA_UPDATED_ON": "2023-09-26", + "DATA_UPDATED_ON": "2024-10-11", "ENVIRONMENT": "production", - "GOOGLE_TAG_MANAGER_ID": "", + "GOOGLE_TAG_MANAGER_ID": "GTM-NBR5XD8C", "SSR_API_URL": "http://openchallenges-api-gateway:8082/api/v1", }, ) @@ -325,7 +326,7 @@ "openchallenges-api-docs", 8010, 256, - "ghcr.io/sage-bionetworks/openchallenges-api-docs:edge", + f"ghcr.io/sage-bionetworks/openchallenges-api-docs:{image_version}", {"PORT": "8010"}, ) api_docs_stack = ServiceStack( @@ -340,7 +341,7 @@ "openchallenges-apex", 8000, 200, - "ghcr.io/sage-bionetworks/openchallenges-apex:edge", + f"ghcr.io/sage-bionetworks/openchallenges-apex:{image_version}", { "API_DOCS_HOST": "openchallenges-api-docs", "API_DOCS_PORT": "8010", From e1e23ea00b68d74ecf4d978280afc76286ea05de Mon Sep 17 00:00:00 2001 From: Khai Do <3697686+zaro0508@users.noreply.github.com> Date: Mon, 14 Oct 2024 11:39:26 -0700 Subject: [PATCH 06/26] Increase GH workflow timeout (#55) A change[1] was made to update all containers at the same time which takes longer to deploy so we need to increase the deployment timeout. [1] https://github.com/Sage-Bionetworks-IT/openchallenges/pull/54 --- .github/workflows/aws-deploy.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/aws-deploy.yaml b/.github/workflows/aws-deploy.yaml index db9a712..2052d8f 100644 --- a/.github/workflows/aws-deploy.yaml +++ b/.github/workflows/aws-deploy.yaml @@ -20,7 +20,7 @@ on: type: string role-duration-seconds: type: number - default: 1200 + default: 3600 environment: required: true type: string From a29c10f88043c2744a9c2117fa64c5a333cf6cd1 Mon Sep 17 00:00:00 2001 From: Thomas Schaffter Date: Mon, 14 Oct 2024 18:28:20 -0700 Subject: [PATCH 07/26] Update stage environment (#57) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * [pre-commit.ci] pre-commit autoupdate (#48) updates: - [github.com/PyCQA/flake8: 7.1.0 → 7.1.1](https://github.com/PyCQA/flake8/compare/7.1.0...7.1.1) - [github.com/awslabs/cfn-python-lint: v1.9.0 → v1.15.0](https://github.com/awslabs/cfn-python-lint/compare/v1.9.0...v1.15.0) - [github.com/psf/black: 24.4.2 → 24.8.0](https://github.com/psf/black/compare/24.4.2...24.8.0) - [github.com/sirosen/check-jsonschema: 0.29.1 → 0.29.2](https://github.com/sirosen/check-jsonschema/compare/0.29.1...0.29.2) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> * [IT-3918] Fix the image URLs returned by the image service (#51) * update the config of the image service * use `{fully_qualified_domain_name}` * use an f-string * [pre-commit.ci] pre-commit autoupdate (#52) updates: - [github.com/awslabs/cfn-python-lint: v1.15.0 → v1.15.2](https://github.com/awslabs/cfn-python-lint/compare/v1.15.0...v1.15.2) - [github.com/sirosen/check-jsonschema: 0.29.2 → 0.29.3](https://github.com/sirosen/check-jsonschema/compare/0.29.2...0.29.3) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> * [pre-commit.ci] pre-commit autoupdate (#53) updates: - [github.com/pre-commit/pre-commit-hooks: v4.6.0 → v5.0.0](https://github.com/pre-commit/pre-commit-hooks/compare/v4.6.0...v5.0.0) - [github.com/awslabs/cfn-python-lint: v1.15.2 → v1.16.0](https://github.com/awslabs/cfn-python-lint/compare/v1.15.2...v1.16.0) - [github.com/psf/black: 24.8.0 → 24.10.0](https://github.com/psf/black/compare/24.8.0...24.10.0) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> * Parametrize stack version and update app config (#54) * parametrize stack version * update data update date * set Google tag manager ID * rename `stack_version` to `image_version` * Increase GH workflow timeout (#55) A change[1] was made to update all containers at the same time which takes longer to deploy so we need to increase the deployment timeout. [1] https://github.com/Sage-Bionetworks-IT/openchallenges/pull/54 --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Khai Do <3697686+zaro0508@users.noreply.github.com> --- .github/workflows/aws-deploy.yaml | 2 +- .pre-commit-config.yaml | 10 +++++----- README.md | 2 +- app.py | 33 +++++++++++++++++-------------- 4 files changed, 25 insertions(+), 22 deletions(-) diff --git a/.github/workflows/aws-deploy.yaml b/.github/workflows/aws-deploy.yaml index db9a712..2052d8f 100644 --- a/.github/workflows/aws-deploy.yaml +++ b/.github/workflows/aws-deploy.yaml @@ -20,7 +20,7 @@ on: type: string role-duration-seconds: type: number - default: 1200 + default: 3600 environment: required: true type: string diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9a6e254..b3f17c8 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -3,13 +3,13 @@ default_language_version: repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.6.0 + rev: v5.0.0 hooks: - id: end-of-file-fixer - id: mixed-line-ending - id: trailing-whitespace - repo: https://github.com/PyCQA/flake8 - rev: 7.1.0 + rev: 7.1.1 hooks: - id: flake8 - repo: https://github.com/adrienverge/yamllint @@ -17,7 +17,7 @@ repos: hooks: - id: yamllint - repo: https://github.com/awslabs/cfn-python-lint - rev: v1.9.0 + rev: v1.16.0 hooks: - id: cfn-python-lint args: @@ -32,11 +32,11 @@ repos: ^.pre-commit-config.yaml ) - repo: https://github.com/psf/black - rev: 24.4.2 + rev: 24.10.0 hooks: - id: black - repo: https://github.com/sirosen/check-jsonschema - rev: 0.29.1 + rev: 0.29.3 hooks: - id: check-github-workflows - id: check-github-actions diff --git a/README.md b/README.md index 1421fb3..fce5714 100644 --- a/README.md +++ b/README.md @@ -233,7 +233,7 @@ OpenChallenges application. * Add the Certificate ARN to the cdk.json * Update references to the OC docker images in [app.py](app.py) (i.e. `ghcr.io/sage-bionetworks/openchallenges-xxx:`) -* (Optional) Update the ServiceProps objects in [app.py](app.py) with parameters specific to +* (Optional) Update the `ServiceProps` objects in [app.py](app.py) with parameters specific to each container. ## Login with the AWS CLI diff --git a/app.py b/app.py index 4d5f43c..bfcea3a 100644 --- a/app.py +++ b/app.py @@ -14,6 +14,7 @@ # get the environment environment = utils.get_environment() stack_name_prefix = f"openchallenges-{environment}" +image_version = "0.0.11" # get VARS from cdk.json env_vars = app.node.try_get_context(environment) @@ -38,7 +39,7 @@ "openchallenges-mariadb", 3306, 512, - "ghcr.io/sage-bionetworks/openchallenges-mariadb:edge", + f"ghcr.io/sage-bionetworks/openchallenges-mariadb:{image_version}", { "MARIADB_USER": "maria", "MARIADB_PASSWORD": secrets["MARIADB_PASSWORD"], @@ -58,7 +59,7 @@ "openchallenges-elasticsearch", 9200, 2048, - "ghcr.io/sage-bionetworks/openchallenges-elasticsearch:edge", + f"ghcr.io/sage-bionetworks/openchallenges-elasticsearch:{image_version}", { "bootstrap.memory_lock": "true", "discovery.type": "single-node", # https://stackoverflow.com/a/68253868 @@ -78,7 +79,7 @@ "openchallenges-thumbor", 8889, 512, - "ghcr.io/sage-bionetworks/openchallenges-thumbor:edge", + f"ghcr.io/sage-bionetworks/openchallenges-thumbor:{image_version}", { "LOG_LEVEL": "info", "PORT": "8889", @@ -115,7 +116,7 @@ "openchallenges-config-server", 8090, 1024, - "ghcr.io/sage-bionetworks/openchallenges-config-server:edge", + f"ghcr.io/sage-bionetworks/openchallenges-config-server:{image_version}", { "GIT_DEFAULT_LABEL": "test-2", "GIT_HOST_KEY_ALGORITHM": "ssh-ed25519", @@ -138,7 +139,7 @@ "openchallenges-service-registry", 8081, 1024, - "ghcr.io/sage-bionetworks/openchallenges-service-registry:edge", + f"ghcr.io/sage-bionetworks/openchallenges-service-registry:{image_version}", { "SERVER_PORT": "8081", "DEFAULT_ZONE": "http://localhost:8081/eureka", @@ -159,7 +160,7 @@ "openchallenges-zipkin", 9411, 512, - "ghcr.io/sage-bionetworks/openchallenges-zipkin:edge", + f"ghcr.io/sage-bionetworks/openchallenges-zipkin:{image_version}", {}, ) @@ -175,11 +176,13 @@ "openchallenges-image-service", 8086, 1024, - "ghcr.io/sage-bionetworks/openchallenges-image-service:edge", + f"ghcr.io/sage-bionetworks/openchallenges-image-service:{image_version}", { "SERVER_PORT": "8086", "SPRING_CLOUD_CONFIG_URI": "http://openchallenges-config-server:8090", "SERVICE_REGISTRY_URL": "http://openchallenges-service-registry:8081/eureka", + "OPENCHALLENGES_IMAGE_SERVICE_THUMBOR_HOST": f"https://{fully_qualified_domain_name}/img/", + "OPENCHALLENGES_IMAGE_SERVICE_THUMBOR_SECURITY_KEY": secrets["SECURITY_KEY"], "OPENCHALLENGES_IMAGE_SERVICE_IS_DEPLOYED_ON_AWS": "true", }, ) @@ -199,7 +202,7 @@ "openchallenges-challenge-service", 8085, 1024, - "ghcr.io/sage-bionetworks/openchallenges-challenge-service:edge", + f"ghcr.io/sage-bionetworks/openchallenges-challenge-service:{image_version}", { "SERVER_PORT": "8085", "SPRING_CLOUD_CONFIG_URI": "http://openchallenges-config-server:8090", @@ -237,7 +240,7 @@ "openchallenges-organization-service", 8084, 1024, - "ghcr.io/sage-bionetworks/openchallenges-organization-service:edge", + f"ghcr.io/sage-bionetworks/openchallenges-organization-service:{image_version}", { "SERVER_PORT": "8084", "SPRING_CLOUD_CONFIG_URI": "http://openchallenges-config-server:8090", @@ -268,7 +271,7 @@ "openchallenges-api-gateway", 8082, 1024, - "ghcr.io/sage-bionetworks/openchallenges-api-gateway:edge", + f"ghcr.io/sage-bionetworks/openchallenges-api-gateway:{image_version}", { "SERVER_PORT": "8082", "SPRING_CLOUD_CONFIG_URI": "http://openchallenges-config-server:8090", @@ -291,14 +294,14 @@ "openchallenges-app", 4200, 1024, - "ghcr.io/sage-bionetworks/openchallenges-app:edge", + f"ghcr.io/sage-bionetworks/openchallenges-app:{image_version}", { "API_DOCS_URL": f"https://{fully_qualified_domain_name}/api-docs", "APP_VERSION": "1.0.0-alpha", "CSR_API_URL": f"https://{fully_qualified_domain_name}/api/v1", - "DATA_UPDATED_ON": "2023-09-26", + "DATA_UPDATED_ON": "2024-10-11", "ENVIRONMENT": "production", - "GOOGLE_TAG_MANAGER_ID": "", + "GOOGLE_TAG_MANAGER_ID": "GTM-NBR5XD8C", "SSR_API_URL": "http://openchallenges-api-gateway:8082/api/v1", }, ) @@ -323,7 +326,7 @@ "openchallenges-api-docs", 8010, 256, - "ghcr.io/sage-bionetworks/openchallenges-api-docs:edge", + f"ghcr.io/sage-bionetworks/openchallenges-api-docs:{image_version}", {"PORT": "8010"}, ) api_docs_stack = ServiceStack( @@ -338,7 +341,7 @@ "openchallenges-apex", 8000, 200, - "ghcr.io/sage-bionetworks/openchallenges-apex:edge", + f"ghcr.io/sage-bionetworks/openchallenges-apex:{image_version}", { "API_DOCS_HOST": "openchallenges-api-docs", "API_DOCS_PORT": "8010", From 757f4b41ab84f330fd42d28e4f0a3dc8a4d3717f Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 16 Oct 2024 09:30:41 -0700 Subject: [PATCH 08/26] [pre-commit.ci] pre-commit autoupdate (#56) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/awslabs/cfn-python-lint: v1.16.0 → v1.16.1](https://github.com/awslabs/cfn-python-lint/compare/v1.16.0...v1.16.1) - [github.com/sirosen/check-jsonschema: 0.29.3 → 0.29.4](https://github.com/sirosen/check-jsonschema/compare/0.29.3...0.29.4) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b3f17c8..3cbc0a7 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -17,7 +17,7 @@ repos: hooks: - id: yamllint - repo: https://github.com/awslabs/cfn-python-lint - rev: v1.16.0 + rev: v1.16.1 hooks: - id: cfn-python-lint args: @@ -36,7 +36,7 @@ repos: hooks: - id: black - repo: https://github.com/sirosen/check-jsonschema - rev: 0.29.3 + rev: 0.29.4 hooks: - id: check-github-workflows - id: check-github-actions From 95e10f5c2ef5a5617d7c4ad4ea46d330aa32a13b Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 22 Oct 2024 15:57:27 -0700 Subject: [PATCH 09/26] [pre-commit.ci] pre-commit autoupdate (#59) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/awslabs/cfn-python-lint: v1.16.1 → v1.18.1](https://github.com/awslabs/cfn-python-lint/compare/v1.16.1...v1.18.1) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3cbc0a7..b598e3c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -17,7 +17,7 @@ repos: hooks: - id: yamllint - repo: https://github.com/awslabs/cfn-python-lint - rev: v1.16.1 + rev: v1.18.1 hooks: - id: cfn-python-lint args: From 9ce15a373b60fdbe74664f4375f6970148ebc929 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 30 Oct 2024 09:48:08 -0700 Subject: [PATCH 10/26] [pre-commit.ci] pre-commit autoupdate (#60) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/awslabs/cfn-python-lint: v1.18.1 → v1.18.2](https://github.com/awslabs/cfn-python-lint/compare/v1.18.1...v1.18.2) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b598e3c..f5045d7 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -17,7 +17,7 @@ repos: hooks: - id: yamllint - repo: https://github.com/awslabs/cfn-python-lint - rev: v1.18.1 + rev: v1.18.2 hooks: - id: cfn-python-lint args: From dd6af19db4ba4eb9411aba18d44d724482fc48d8 Mon Sep 17 00:00:00 2001 From: Thomas Schaffter Date: Thu, 31 Oct 2024 10:30:07 -0700 Subject: [PATCH 11/26] Set concurrency to 5 (best results) (#61) --- .github/workflows/aws-deploy.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/aws-deploy.yaml b/.github/workflows/aws-deploy.yaml index 2052d8f..6b6486e 100644 --- a/.github/workflows/aws-deploy.yaml +++ b/.github/workflows/aws-deploy.yaml @@ -52,7 +52,7 @@ jobs: role-session-name: ${{ inputs.role-session-name }} role-duration-seconds: ${{ inputs.role-duration-seconds }} - name: CDK deploy - run: cdk deploy --all --require-approval never + run: cdk deploy --all --concurrency 5 --require-approval never env: ENV: ${{ inputs.environment }} SECRETS: ${{ inputs.secrets-location }} From 156e6c76cd51533eb7d98e052f916d13618a1437 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 11 Nov 2024 10:40:52 -0800 Subject: [PATCH 12/26] [pre-commit.ci] pre-commit autoupdate (#62) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/awslabs/cfn-python-lint: v1.18.2 → v1.18.4](https://github.com/awslabs/cfn-python-lint/compare/v1.18.2...v1.18.4) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f5045d7..14cfab8 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -17,7 +17,7 @@ repos: hooks: - id: yamllint - repo: https://github.com/awslabs/cfn-python-lint - rev: v1.18.2 + rev: v1.18.4 hooks: - id: cfn-python-lint args: From 652ba4ce2e11e4b353dd333eaae770e78ee5dc0b Mon Sep 17 00:00:00 2001 From: Thomas Schaffter Date: Wed, 13 Nov 2024 14:13:54 -0800 Subject: [PATCH 13/26] update image tag, app version and data release date (#63) --- app.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app.py b/app.py index bfcea3a..fcaa4cc 100644 --- a/app.py +++ b/app.py @@ -14,7 +14,7 @@ # get the environment environment = utils.get_environment() stack_name_prefix = f"openchallenges-{environment}" -image_version = "0.0.11" +image_version = "0.0.12" # get VARS from cdk.json env_vars = app.node.try_get_context(environment) @@ -297,9 +297,9 @@ f"ghcr.io/sage-bionetworks/openchallenges-app:{image_version}", { "API_DOCS_URL": f"https://{fully_qualified_domain_name}/api-docs", - "APP_VERSION": "1.0.0-alpha", + "APP_VERSION": "1.0.12-beta", "CSR_API_URL": f"https://{fully_qualified_domain_name}/api/v1", - "DATA_UPDATED_ON": "2024-10-11", + "DATA_UPDATED_ON": "2024-11-13", "ENVIRONMENT": "production", "GOOGLE_TAG_MANAGER_ID": "GTM-NBR5XD8C", "SSR_API_URL": "http://openchallenges-api-gateway:8082/api/v1", From 4d28a5530b2736eb7db31e82c25815a921bc6814 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 19 Nov 2024 08:12:13 -0800 Subject: [PATCH 14/26] [pre-commit.ci] pre-commit autoupdate (#66) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/awslabs/cfn-python-lint: v1.18.4 → v1.19.0](https://github.com/awslabs/cfn-python-lint/compare/v1.18.4...v1.19.0) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 14cfab8..32b7a9f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -17,7 +17,7 @@ repos: hooks: - id: yamllint - repo: https://github.com/awslabs/cfn-python-lint - rev: v1.18.4 + rev: v1.19.0 hooks: - id: cfn-python-lint args: From a3ead0cea7c31a2961a6d88fe014eafa5ba47994 Mon Sep 17 00:00:00 2001 From: Khai Do <3697686+zaro0508@users.noreply.github.com> Date: Tue, 19 Nov 2024 11:01:21 -0800 Subject: [PATCH 15/26] Refactor mounting volumes (#67) The current implementation to mount volumes was very specific to one container. We are replacing it with an implementation that is much more generic to make it easy to mount volumes in other containers. --- app.py | 8 +++++++- openchallenges/service_props.py | 25 +++++++++++++++++++++++++ openchallenges/service_stack.py | 19 +++++++++---------- 3 files changed, 41 insertions(+), 11 deletions(-) diff --git a/app.py b/app.py index fcaa4cc..998b536 100644 --- a/app.py +++ b/app.py @@ -6,7 +6,7 @@ from openchallenges.service_stack import ServiceStack from openchallenges.service_stack import LoadBalancedServiceStack from openchallenges.load_balancer_stack import LoadBalancerStack -from openchallenges.service_props import ServiceProps +from openchallenges.service_props import ServiceProps, ContainerVolume import openchallenges.utils as utils app = cdk.App() @@ -45,6 +45,12 @@ "MARIADB_PASSWORD": secrets["MARIADB_PASSWORD"], "MARIADB_ROOT_PASSWORD": secrets["MARIADB_ROOT_PASSWORD"], }, + container_volumes=[ + ContainerVolume( + path="/data/db", + size=30, + ) + ], ) mariadb_stack = ServiceStack( diff --git a/openchallenges/service_props.py b/openchallenges/service_props.py index efa7316..25b16c9 100644 --- a/openchallenges/service_props.py +++ b/openchallenges/service_props.py @@ -1,6 +1,25 @@ +from dataclasses import dataclass +from typing import List + CONTAINER_LOCATION_PATH_ID = "path://" +@dataclass +class ContainerVolume: + """ + Holds onto configuration for a volume used in the container. + + Attributes: + path: The path on the container to mount the host volume at. + size: The size of the volume in GiB. + read_only: Container has read-only access to the volume, set to `false` for write access. + """ + + path: str + size: int = 15 + read_only: bool = False + + class ServiceProps: """ ECS service properties @@ -13,6 +32,7 @@ class ServiceProps: supports docker registry references (i.e. ghcr.io/sage-bionetworks/openchallenges-thumbor:latest) container_env_vars: a json dictionary of environment variables to pass into the container i.e. {"EnvA": "EnvValueA", "EnvB": "EnvValueB"} + container_volumes: List of `ContainerVolume` resources to mount into the container """ def __init__( @@ -22,6 +42,7 @@ def __init__( container_memory: int, container_location: str, container_env_vars: dict, + container_volumes: List[ContainerVolume] = None, ) -> None: self.container_name = container_name self.container_port = container_port @@ -32,3 +53,7 @@ def __init__( ) self.container_location = container_location self.container_env_vars = container_env_vars + if container_volumes is None: + self.container_volumes = [] + else: + self.container_volumes = container_volumes diff --git a/openchallenges/service_stack.py b/openchallenges/service_stack.py index 3911a7e..4dd351b 100644 --- a/openchallenges/service_stack.py +++ b/openchallenges/service_stack.py @@ -118,14 +118,14 @@ def __init__( ), ) - # mount volume for DB - if "mariadb" in construct_id: - self.volume = ecs.ServiceManagedVolume( + # mount volumes + for container_volume in props.container_volumes: + service_volume = ecs.ServiceManagedVolume( self, - "ServiceVolume", + "ContainerVolume", name=props.container_name, managed_ebs_volume=ecs.ServiceManagedEBSVolumeConfiguration( - size=size.gibibytes(30), + size=size.gibibytes(container_volume.size), volume_type=ec2.EbsDeviceVolumeType.GP3, ), ) @@ -133,13 +133,12 @@ def __init__( self.task_definition.add_volume( name=props.container_name, configured_at_launch=True ) - self.service.add_volume(self.volume) + self.service.add_volume(service_volume) - self.volume.mount_in( - # should be mounted at openchallenges-mariadb:/data/db + service_volume.mount_in( self.container, - container_path="/data/db", - read_only=False, + container_path=container_volume.path, + read_only=container_volume.read_only, ) From 23dd10ce7d0b273b2a96347ee2edb54b416d5b32 Mon Sep 17 00:00:00 2001 From: Joni Harker Date: Thu, 21 Nov 2024 10:33:38 -0800 Subject: [PATCH 16/26] [IT-4003] Auto-update pre-commit hook versions monthly Change the frequency that PRs to update pre-commit hook versions are auto-generated from weekly (the default) to monthly. --- .pre-commit-config.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 32b7a9f..1612c2e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,3 +1,6 @@ +ci: + autoupdate_schedule: monthly + default_language_version: python: python3 From e622e7bed4d832d31605cdfb56e942e28c9443cd Mon Sep 17 00:00:00 2001 From: Thomas Schaffter Date: Wed, 27 Nov 2024 11:28:13 -0800 Subject: [PATCH 17/26] Update to OC v1.1.1 (#69) * Update to v1.1.1 * update data updated on --- app.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app.py b/app.py index 998b536..54c6929 100644 --- a/app.py +++ b/app.py @@ -14,7 +14,7 @@ # get the environment environment = utils.get_environment() stack_name_prefix = f"openchallenges-{environment}" -image_version = "0.0.12" +image_version = "1.1.1" # get VARS from cdk.json env_vars = app.node.try_get_context(environment) @@ -303,9 +303,9 @@ f"ghcr.io/sage-bionetworks/openchallenges-app:{image_version}", { "API_DOCS_URL": f"https://{fully_qualified_domain_name}/api-docs", - "APP_VERSION": "1.0.12-beta", + "APP_VERSION": image_version, "CSR_API_URL": f"https://{fully_qualified_domain_name}/api/v1", - "DATA_UPDATED_ON": "2024-11-13", + "DATA_UPDATED_ON": "2024-11-27", "ENVIRONMENT": "production", "GOOGLE_TAG_MANAGER_ID": "GTM-NBR5XD8C", "SSR_API_URL": "http://openchallenges-api-gateway:8082/api/v1", From 1d9e1306aa3fe8f39bab09b9af3ed80b3fb323d4 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 3 Dec 2024 12:33:44 -0800 Subject: [PATCH 18/26] [pre-commit.ci] pre-commit autoupdate (#70) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/awslabs/cfn-python-lint: v1.19.0 → v1.20.1](https://github.com/awslabs/cfn-python-lint/compare/v1.19.0...v1.20.1) - [github.com/sirosen/check-jsonschema: 0.29.4 → 0.30.0](https://github.com/sirosen/check-jsonschema/compare/0.29.4...0.30.0) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 1612c2e..d843665 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -20,7 +20,7 @@ repos: hooks: - id: yamllint - repo: https://github.com/awslabs/cfn-python-lint - rev: v1.19.0 + rev: v1.20.1 hooks: - id: cfn-python-lint args: @@ -39,7 +39,7 @@ repos: hooks: - id: black - repo: https://github.com/sirosen/check-jsonschema - rev: 0.29.4 + rev: 0.30.0 hooks: - id: check-github-workflows - id: check-github-actions From 9965cd434791aebcb27550d5b8bf5cdeaeda6fba Mon Sep 17 00:00:00 2001 From: Khai Do <3697686+zaro0508@users.noreply.github.com> Date: Tue, 3 Dec 2024 14:20:28 -0800 Subject: [PATCH 19/26] [IT-3951] Fix guardduty container (#71) We enable guardduty security monitoring for ECS in every account. For that to work we need to give Fragate tasks access to do ECS stuff with the service-role/AmazonECSTaskExecutionRolePolicy[1]. [1] https://docs.aws.amazon.com/guardduty/latest/ug/prereq-runtime-monitoring-ecs-support.html#before-enable-runtime-monitoring-ecs --- openchallenges/service_stack.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/openchallenges/service_stack.py b/openchallenges/service_stack.py index 4dd351b..e1be848 100644 --- a/openchallenges/service_stack.py +++ b/openchallenges/service_stack.py @@ -60,6 +60,28 @@ def __init__( ) ) + # default ECS execution policy plus Guardduty access + execution_role = iam.Role( + self, + "ExecutionRole", + assumed_by=iam.ServicePrincipal("ecs-tasks.amazonaws.com"), + managed_policies=[ + iam.ManagedPolicy.from_aws_managed_policy_name( + "service-role/AmazonECSTaskExecutionRolePolicy" + ), + ], + ) + execution_role.add_to_policy( + iam.PolicyStatement( + actions=[ + "logs:CreateLogStream", + "logs:PutLogEvents", + ], + resources=["*"], + effect=iam.Effect.ALLOW, + ) + ) + # ECS task with fargate self.task_definition = ecs.FargateTaskDefinition( self, @@ -67,6 +89,7 @@ def __init__( cpu=1024, memory_limit_mib=4096, task_role=task_role, + execution_role=execution_role, ) image = ecs.ContainerImage.from_registry(props.container_location) From d70c9a677bbdcdea36a5ac2431dc56d21f7e1154 Mon Sep 17 00:00:00 2001 From: Thomas Schaffter Date: Wed, 4 Dec 2024 07:57:22 -0800 Subject: [PATCH 20/26] remove source.bat (#74) --- source.bat | 13 ------------- 1 file changed, 13 deletions(-) delete mode 100644 source.bat diff --git a/source.bat b/source.bat deleted file mode 100644 index 9e1a834..0000000 --- a/source.bat +++ /dev/null @@ -1,13 +0,0 @@ -@echo off - -rem The sole purpose of this script is to make the command -rem -rem source .venv/bin/activate -rem -rem (which activates a Python virtualenv on Linux or Mac OS X) work on Windows. -rem On Windows, this command just runs this batch file (the argument is ignored). -rem -rem Now we don't need to document a Windows command for activating a virtualenv. - -echo Executing .venv\Scripts\activate.bat for you -.venv\Scripts\activate.bat From 26fb4b6c57056966c7dc4e272c072cf038d7f1b2 Mon Sep 17 00:00:00 2001 From: Thomas Schaffter Date: Wed, 4 Dec 2024 15:31:16 -0800 Subject: [PATCH 21/26] Add Docker in Docker to the dev container (#73) * Add Docker in Docker to the devcontainer * add docs about docker * forward local environment variables to the devcontainer * remove containerEnv --- .devcontainer/devcontainer.json | 6 +++++- README.md | 5 +++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 3746fd1..fe17278 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -8,7 +8,11 @@ "ghcr.io/devcontainers/features/python:1.6.3": { "version": "3.12.0" }, - "ghcr.io/devcontainers/features/aws-cli:1": {} + "ghcr.io/devcontainers/features/aws-cli:1": {}, + "ghcr.io/devcontainers/features/docker-in-docker:2.12.0": { + "version": "27.0.3", + "moby": false + } }, "postCreateCommand": "./tools/setup.sh", "shutdownAction": "stopContainer" diff --git a/README.md b/README.md index fce5714..aa71846 100644 --- a/README.md +++ b/README.md @@ -44,6 +44,11 @@ tools can be installed by running: ./tools/setup.sh ``` +When developing outside the dev container, the following tools must be installed +manually. + +- [Docker](https://docs.docker.com/engine/install/) >= v27 + Development requires the activation of the Python virtual environment: ``` From 25c5008466b31f50143cabcf01272eeba447c4c5 Mon Sep 17 00:00:00 2001 From: Thomas Schaffter Date: Fri, 6 Dec 2024 13:30:24 -0800 Subject: [PATCH 22/26] Add AWS Lambda for upcoming data integration (ARCH-356) (#72) * update docs on setup tools * define lambda role and function * update path to Dockerfile * update README * trigger the lambda every 5 minutes * use plural form of the unit * Remove lambda fct architecture * Migrate data integration code to L2 constructs * Add @dataclass to DataIntegrationProps * Add docstrings * Replace `_lambda` by `lambda_` * Add docstrings * Add docstrings --- README.md | 2 +- app.py | 16 +++++ cdk_docker/data-integration-lambda/Dockerfile | 1 + openchallenges/data_integration_lambda.py | 62 +++++++++++++++++ openchallenges/data_integration_props.py | 15 +++++ openchallenges/data_integration_stack.py | 66 +++++++++++++++++++ requirements.txt | 2 + 7 files changed, 163 insertions(+), 1 deletion(-) create mode 100644 cdk_docker/data-integration-lambda/Dockerfile create mode 100644 openchallenges/data_integration_lambda.py create mode 100644 openchallenges/data_integration_props.py create mode 100644 openchallenges/data_integration_stack.py diff --git a/README.md b/README.md index aa71846..872fb23 100644 --- a/README.md +++ b/README.md @@ -38,7 +38,7 @@ also include a Python virtual environment where all the Python packages needed are already installed. If you decide the develop outside of the dev container, some of the development -tools can be installed by running: +tools can be installed manually by running: ```console ./tools/setup.sh diff --git a/app.py b/app.py index 54c6929..d207739 100644 --- a/app.py +++ b/app.py @@ -1,4 +1,5 @@ import aws_cdk as cdk +from aws_cdk.aws_scheduler_alpha import ScheduleExpression from openchallenges.bucket_stack import BucketStack from openchallenges.network_stack import NetworkStack @@ -7,6 +8,8 @@ from openchallenges.service_stack import LoadBalancedServiceStack from openchallenges.load_balancer_stack import LoadBalancerStack from openchallenges.service_props import ServiceProps, ContainerVolume +from openchallenges.data_integration_stack import DataIntegrationStack +from openchallenges.data_integration_props import DataIntegrationProps import openchallenges.utils as utils app = cdk.App() @@ -328,6 +331,19 @@ app, f"{stack_name_prefix}-load-balancer", network_stack.vpc ) +data_integration_props = DataIntegrationProps( + schedule=ScheduleExpression.cron( + minute="*/5", + hour="*", + day="*", + month="*", + time_zone=cdk.TimeZone.AMERICA_LOS_ANGELES, + ) +) +data_integration_stack = DataIntegrationStack( + app, f"{stack_name_prefix}-data-integration", data_integration_props +) + api_docs_props = ServiceProps( "openchallenges-api-docs", 8010, diff --git a/cdk_docker/data-integration-lambda/Dockerfile b/cdk_docker/data-integration-lambda/Dockerfile new file mode 100644 index 0000000..6665871 --- /dev/null +++ b/cdk_docker/data-integration-lambda/Dockerfile @@ -0,0 +1 @@ +FROM ghcr.io/sage-bionetworks/sandbox-lambda-python:sha-b38dc22 diff --git a/openchallenges/data_integration_lambda.py b/openchallenges/data_integration_lambda.py new file mode 100644 index 0000000..5a03836 --- /dev/null +++ b/openchallenges/data_integration_lambda.py @@ -0,0 +1,62 @@ +from aws_cdk import aws_iam as iam +from aws_cdk import aws_lambda as lambda_ +from constructs import Construct + + +class DataIntegrationLambda(Construct): + """ + A CDK construct to define an AWS Lambda function for data integration. + + This construct creates an IAM role with the necessary permissions and a Docker-based + Lambda function for handling data integration tasks. + """ + + def __init__(self, scope: Construct, id: str) -> None: + """ + Builds the IAM role for the Lambda function. + + This role allows the Lambda function to execute basic AWS operations. + + Returns: + iam.Role: The IAM role for the Lambda function. + """ + super().__init__(scope, id) + + self.lambda_role = self._build_lambda_role() + self.lambda_function = self._build_lambda_function(self.lambda_role) + + def _build_lambda_role(self) -> iam.Role: + return iam.Role( + self, + "LambdaRole", + assumed_by=iam.ServicePrincipal("lambda.amazonaws.com"), + managed_policies=[ + iam.ManagedPolicy.from_aws_managed_policy_name( + managed_policy_name=("service-role/AWSLambdaBasicExecutionRole") + ) + ], + ) + + def _build_lambda_function(self, role: iam.Role) -> lambda_.Function: + """ + Builds the Docker-based AWS Lambda function. + + The Lambda function uses a Docker image built from a local directory. + + Args: + role (iam.Role): The IAM role to associate with the Lambda function. + + Returns: + _lambda.Function: The Docker-based AWS Lambda function. + """ + return lambda_.DockerImageFunction( + self, + "LambdaFunction", + code=lambda_.DockerImageCode.from_image_asset( + # Directory relative to where you execute cdk deploy contains a + # Dockerfile with build instructions. + directory="cdk_docker/data-integration-lambda" + ), + role=role, + memory_size=128, + ) diff --git a/openchallenges/data_integration_props.py b/openchallenges/data_integration_props.py new file mode 100644 index 0000000..6d8dd13 --- /dev/null +++ b/openchallenges/data_integration_props.py @@ -0,0 +1,15 @@ +from dataclasses import dataclass +from aws_cdk.aws_scheduler_alpha import ScheduleExpression + + +@dataclass +class DataIntegrationProps: + """ + Data integration properties. + + Attributes: + schedule (ScheduleExpression): The schedule for triggering the data integration. + """ + + schedule: ScheduleExpression + """The schedule for triggering the data integration.""" diff --git a/openchallenges/data_integration_stack.py b/openchallenges/data_integration_stack.py new file mode 100644 index 0000000..ca651e3 --- /dev/null +++ b/openchallenges/data_integration_stack.py @@ -0,0 +1,66 @@ +import aws_cdk as cdk +from aws_cdk import ( + aws_scheduler_alpha as scheduler_alpha, + aws_scheduler_targets_alpha as scheduler_targets, +) +from openchallenges.data_integration_lambda import DataIntegrationLambda +from openchallenges.data_integration_props import DataIntegrationProps +from constructs import Construct + + +class DataIntegrationStack(cdk.Stack): + """ + Defines an AWS CDK stack for data integration. + + This stack sets up the resources required for scheduling and executing + data integration tasks using AWS Lambda and EventBridge Scheduler. + + The stack includes: + - A Lambda function for data integration. + - An EventBridge Scheduler schedule to trigger the Lambda function. + - An EventBridge Scheduler group for organizing schedules. + + Attributes: + scope (Construct): The parent construct. + id (str): The unique identifier for this stack. + props (DataIntegrationProps): The properties for the data integration, including the schedule. + """ + + def __init__( + self, scope: Construct, id: str, props: DataIntegrationProps, **kwargs + ) -> None: + """ + Initializes the DataIntegrationStack. + + Arguments: + scope (Construct): The parent construct for this stack. + id (str): The unique identifier for this stack. + props (DataIntegrationProps): The properties required for data integration, + including the schedule. + **kwargs: Additional arguments passed to the base `cdk.Stack` class. + """ + super().__init__(scope, id, **kwargs) + + data_integration_lambda = DataIntegrationLambda(self, "data-integration-lambda") + + target = scheduler_targets.LambdaInvoke( + data_integration_lambda.lambda_function, + input=scheduler_alpha.ScheduleTargetInput.from_object({}), + ) + + # Create a group for the schedule (maybe we want to add more schedules + # to this group the future) + schedule_group = scheduler_alpha.Group( + self, + "group", + group_name="schedule-group", + ) + + scheduler_alpha.Schedule( + self, + "schedule", + schedule=props.schedule, + target=target, + group=schedule_group, + description="This is a cron-based schedule that will run every 5 minutes", + ) diff --git a/requirements.txt b/requirements.txt index 92c10ed..b1c51a6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,5 @@ aws-cdk-lib==2.139.0 +aws-cdk.aws-scheduler-alpha==2.139.0a0 +aws-cdk.aws-scheduler-targets-alpha==2.139.0a0 constructs>=10.0.0,<11.0.0 boto3>=1.34.1 From 26202879856d58b08f11b8ffdd5869c1b73f533c Mon Sep 17 00:00:00 2001 From: Thomas Schaffter Date: Sat, 7 Dec 2024 10:30:47 -0800 Subject: [PATCH 23/26] Externalize the description of the schedule (#75) * define lambda role and function * update path to Dockerfile * Externalize the description of the schedule --- app.py | 3 ++- openchallenges/data_integration_props.py | 4 ++++ openchallenges/data_integration_stack.py | 2 +- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/app.py b/app.py index d207739..8fe9231 100644 --- a/app.py +++ b/app.py @@ -338,7 +338,8 @@ day="*", month="*", time_zone=cdk.TimeZone.AMERICA_LOS_ANGELES, - ) + ), + schedule_description="This is a cron-based schedule that will run every 5 minutes", ) data_integration_stack = DataIntegrationStack( app, f"{stack_name_prefix}-data-integration", data_integration_props diff --git a/openchallenges/data_integration_props.py b/openchallenges/data_integration_props.py index 6d8dd13..59416e4 100644 --- a/openchallenges/data_integration_props.py +++ b/openchallenges/data_integration_props.py @@ -9,7 +9,11 @@ class DataIntegrationProps: Attributes: schedule (ScheduleExpression): The schedule for triggering the data integration. + schedule_description (str): The description of the schedule. """ schedule: ScheduleExpression """The schedule for triggering the data integration.""" + + schedule_description: str + """The description of the schedule.""" diff --git a/openchallenges/data_integration_stack.py b/openchallenges/data_integration_stack.py index ca651e3..5b87a76 100644 --- a/openchallenges/data_integration_stack.py +++ b/openchallenges/data_integration_stack.py @@ -62,5 +62,5 @@ def __init__( schedule=props.schedule, target=target, group=schedule_group, - description="This is a cron-based schedule that will run every 5 minutes", + description=props.schedule_description, ) From be639e28523d1fe1f70db9b744f0c12af45a68f9 Mon Sep 17 00:00:00 2001 From: Thomas Schaffter Date: Tue, 10 Dec 2024 13:37:09 -0800 Subject: [PATCH 24/26] Merge dev to stage (#76) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * [pre-commit.ci] pre-commit autoupdate (#48) updates: - [github.com/PyCQA/flake8: 7.1.0 → 7.1.1](https://github.com/PyCQA/flake8/compare/7.1.0...7.1.1) - [github.com/awslabs/cfn-python-lint: v1.9.0 → v1.15.0](https://github.com/awslabs/cfn-python-lint/compare/v1.9.0...v1.15.0) - [github.com/psf/black: 24.4.2 → 24.8.0](https://github.com/psf/black/compare/24.4.2...24.8.0) - [github.com/sirosen/check-jsonschema: 0.29.1 → 0.29.2](https://github.com/sirosen/check-jsonschema/compare/0.29.1...0.29.2) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> * [IT-3918] Fix the image URLs returned by the image service (#51) * update the config of the image service * use `{fully_qualified_domain_name}` * use an f-string * [pre-commit.ci] pre-commit autoupdate (#52) updates: - [github.com/awslabs/cfn-python-lint: v1.15.0 → v1.15.2](https://github.com/awslabs/cfn-python-lint/compare/v1.15.0...v1.15.2) - [github.com/sirosen/check-jsonschema: 0.29.2 → 0.29.3](https://github.com/sirosen/check-jsonschema/compare/0.29.2...0.29.3) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> * [pre-commit.ci] pre-commit autoupdate (#53) updates: - [github.com/pre-commit/pre-commit-hooks: v4.6.0 → v5.0.0](https://github.com/pre-commit/pre-commit-hooks/compare/v4.6.0...v5.0.0) - [github.com/awslabs/cfn-python-lint: v1.15.2 → v1.16.0](https://github.com/awslabs/cfn-python-lint/compare/v1.15.2...v1.16.0) - [github.com/psf/black: 24.8.0 → 24.10.0](https://github.com/psf/black/compare/24.8.0...24.10.0) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> * Parametrize stack version and update app config (#54) * parametrize stack version * update data update date * set Google tag manager ID * rename `stack_version` to `image_version` * Increase GH workflow timeout (#55) A change[1] was made to update all containers at the same time which takes longer to deploy so we need to increase the deployment timeout. [1] https://github.com/Sage-Bionetworks-IT/openchallenges/pull/54 * [pre-commit.ci] pre-commit autoupdate (#56) updates: - [github.com/awslabs/cfn-python-lint: v1.16.0 → v1.16.1](https://github.com/awslabs/cfn-python-lint/compare/v1.16.0...v1.16.1) - [github.com/sirosen/check-jsonschema: 0.29.3 → 0.29.4](https://github.com/sirosen/check-jsonschema/compare/0.29.3...0.29.4) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> * [pre-commit.ci] pre-commit autoupdate (#59) updates: - [github.com/awslabs/cfn-python-lint: v1.16.1 → v1.18.1](https://github.com/awslabs/cfn-python-lint/compare/v1.16.1...v1.18.1) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> * [pre-commit.ci] pre-commit autoupdate (#60) updates: - [github.com/awslabs/cfn-python-lint: v1.18.1 → v1.18.2](https://github.com/awslabs/cfn-python-lint/compare/v1.18.1...v1.18.2) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> * Set concurrency to 5 (best results) (#61) * [pre-commit.ci] pre-commit autoupdate (#62) updates: - [github.com/awslabs/cfn-python-lint: v1.18.2 → v1.18.4](https://github.com/awslabs/cfn-python-lint/compare/v1.18.2...v1.18.4) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> * update image tag, app version and data release date (#63) * [pre-commit.ci] pre-commit autoupdate (#66) updates: - [github.com/awslabs/cfn-python-lint: v1.18.4 → v1.19.0](https://github.com/awslabs/cfn-python-lint/compare/v1.18.4...v1.19.0) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> * Refactor mounting volumes (#67) The current implementation to mount volumes was very specific to one container. We are replacing it with an implementation that is much more generic to make it easy to mount volumes in other containers. * [IT-4003] Auto-update pre-commit hook versions monthly Change the frequency that PRs to update pre-commit hook versions are auto-generated from weekly (the default) to monthly. * Update to OC v1.1.1 (#69) * Update to v1.1.1 * update data updated on * [pre-commit.ci] pre-commit autoupdate (#70) updates: - [github.com/awslabs/cfn-python-lint: v1.19.0 → v1.20.1](https://github.com/awslabs/cfn-python-lint/compare/v1.19.0...v1.20.1) - [github.com/sirosen/check-jsonschema: 0.29.4 → 0.30.0](https://github.com/sirosen/check-jsonschema/compare/0.29.4...0.30.0) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> * [IT-3951] Fix guardduty container (#71) We enable guardduty security monitoring for ECS in every account. For that to work we need to give Fragate tasks access to do ECS stuff with the service-role/AmazonECSTaskExecutionRolePolicy[1]. [1] https://docs.aws.amazon.com/guardduty/latest/ug/prereq-runtime-monitoring-ecs-support.html#before-enable-runtime-monitoring-ecs * remove source.bat (#74) * Add Docker in Docker to the dev container (#73) * Add Docker in Docker to the devcontainer * add docs about docker * forward local environment variables to the devcontainer * remove containerEnv * Add AWS Lambda for upcoming data integration (ARCH-356) (#72) * update docs on setup tools * define lambda role and function * update path to Dockerfile * update README * trigger the lambda every 5 minutes * use plural form of the unit * Remove lambda fct architecture * Migrate data integration code to L2 constructs * Add @dataclass to DataIntegrationProps * Add docstrings * Replace `_lambda` by `lambda_` * Add docstrings * Add docstrings * Externalize the description of the schedule (#75) * define lambda role and function * update path to Dockerfile * Externalize the description of the schedule --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Khai Do <3697686+zaro0508@users.noreply.github.com> Co-authored-by: Joni Harker Co-authored-by: Joni Harker <506966+ConsoleCatzirl@users.noreply.github.com> --- .devcontainer/devcontainer.json | 6 +- .github/workflows/aws-deploy.yaml | 2 +- .pre-commit-config.yaml | 7 +- README.md | 7 +- app.py | 31 +++++++-- cdk_docker/data-integration-lambda/Dockerfile | 1 + openchallenges/data_integration_lambda.py | 62 +++++++++++++++++ openchallenges/data_integration_props.py | 19 ++++++ openchallenges/data_integration_stack.py | 66 +++++++++++++++++++ openchallenges/service_props.py | 25 +++++++ openchallenges/service_stack.py | 42 +++++++++--- requirements.txt | 2 + source.bat | 13 ---- 13 files changed, 251 insertions(+), 32 deletions(-) create mode 100644 cdk_docker/data-integration-lambda/Dockerfile create mode 100644 openchallenges/data_integration_lambda.py create mode 100644 openchallenges/data_integration_props.py create mode 100644 openchallenges/data_integration_stack.py delete mode 100644 source.bat diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 3746fd1..fe17278 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -8,7 +8,11 @@ "ghcr.io/devcontainers/features/python:1.6.3": { "version": "3.12.0" }, - "ghcr.io/devcontainers/features/aws-cli:1": {} + "ghcr.io/devcontainers/features/aws-cli:1": {}, + "ghcr.io/devcontainers/features/docker-in-docker:2.12.0": { + "version": "27.0.3", + "moby": false + } }, "postCreateCommand": "./tools/setup.sh", "shutdownAction": "stopContainer" diff --git a/.github/workflows/aws-deploy.yaml b/.github/workflows/aws-deploy.yaml index 2052d8f..6b6486e 100644 --- a/.github/workflows/aws-deploy.yaml +++ b/.github/workflows/aws-deploy.yaml @@ -52,7 +52,7 @@ jobs: role-session-name: ${{ inputs.role-session-name }} role-duration-seconds: ${{ inputs.role-duration-seconds }} - name: CDK deploy - run: cdk deploy --all --require-approval never + run: cdk deploy --all --concurrency 5 --require-approval never env: ENV: ${{ inputs.environment }} SECRETS: ${{ inputs.secrets-location }} diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b3f17c8..d843665 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,3 +1,6 @@ +ci: + autoupdate_schedule: monthly + default_language_version: python: python3 @@ -17,7 +20,7 @@ repos: hooks: - id: yamllint - repo: https://github.com/awslabs/cfn-python-lint - rev: v1.16.0 + rev: v1.20.1 hooks: - id: cfn-python-lint args: @@ -36,7 +39,7 @@ repos: hooks: - id: black - repo: https://github.com/sirosen/check-jsonschema - rev: 0.29.3 + rev: 0.30.0 hooks: - id: check-github-workflows - id: check-github-actions diff --git a/README.md b/README.md index fce5714..872fb23 100644 --- a/README.md +++ b/README.md @@ -38,12 +38,17 @@ also include a Python virtual environment where all the Python packages needed are already installed. If you decide the develop outside of the dev container, some of the development -tools can be installed by running: +tools can be installed manually by running: ```console ./tools/setup.sh ``` +When developing outside the dev container, the following tools must be installed +manually. + +- [Docker](https://docs.docker.com/engine/install/) >= v27 + Development requires the activation of the Python virtual environment: ``` diff --git a/app.py b/app.py index bfcea3a..8fe9231 100644 --- a/app.py +++ b/app.py @@ -1,4 +1,5 @@ import aws_cdk as cdk +from aws_cdk.aws_scheduler_alpha import ScheduleExpression from openchallenges.bucket_stack import BucketStack from openchallenges.network_stack import NetworkStack @@ -6,7 +7,9 @@ from openchallenges.service_stack import ServiceStack from openchallenges.service_stack import LoadBalancedServiceStack from openchallenges.load_balancer_stack import LoadBalancerStack -from openchallenges.service_props import ServiceProps +from openchallenges.service_props import ServiceProps, ContainerVolume +from openchallenges.data_integration_stack import DataIntegrationStack +from openchallenges.data_integration_props import DataIntegrationProps import openchallenges.utils as utils app = cdk.App() @@ -14,7 +17,7 @@ # get the environment environment = utils.get_environment() stack_name_prefix = f"openchallenges-{environment}" -image_version = "0.0.11" +image_version = "1.1.1" # get VARS from cdk.json env_vars = app.node.try_get_context(environment) @@ -45,6 +48,12 @@ "MARIADB_PASSWORD": secrets["MARIADB_PASSWORD"], "MARIADB_ROOT_PASSWORD": secrets["MARIADB_ROOT_PASSWORD"], }, + container_volumes=[ + ContainerVolume( + path="/data/db", + size=30, + ) + ], ) mariadb_stack = ServiceStack( @@ -297,9 +306,9 @@ f"ghcr.io/sage-bionetworks/openchallenges-app:{image_version}", { "API_DOCS_URL": f"https://{fully_qualified_domain_name}/api-docs", - "APP_VERSION": "1.0.0-alpha", + "APP_VERSION": image_version, "CSR_API_URL": f"https://{fully_qualified_domain_name}/api/v1", - "DATA_UPDATED_ON": "2024-10-11", + "DATA_UPDATED_ON": "2024-11-27", "ENVIRONMENT": "production", "GOOGLE_TAG_MANAGER_ID": "GTM-NBR5XD8C", "SSR_API_URL": "http://openchallenges-api-gateway:8082/api/v1", @@ -322,6 +331,20 @@ app, f"{stack_name_prefix}-load-balancer", network_stack.vpc ) +data_integration_props = DataIntegrationProps( + schedule=ScheduleExpression.cron( + minute="*/5", + hour="*", + day="*", + month="*", + time_zone=cdk.TimeZone.AMERICA_LOS_ANGELES, + ), + schedule_description="This is a cron-based schedule that will run every 5 minutes", +) +data_integration_stack = DataIntegrationStack( + app, f"{stack_name_prefix}-data-integration", data_integration_props +) + api_docs_props = ServiceProps( "openchallenges-api-docs", 8010, diff --git a/cdk_docker/data-integration-lambda/Dockerfile b/cdk_docker/data-integration-lambda/Dockerfile new file mode 100644 index 0000000..6665871 --- /dev/null +++ b/cdk_docker/data-integration-lambda/Dockerfile @@ -0,0 +1 @@ +FROM ghcr.io/sage-bionetworks/sandbox-lambda-python:sha-b38dc22 diff --git a/openchallenges/data_integration_lambda.py b/openchallenges/data_integration_lambda.py new file mode 100644 index 0000000..5a03836 --- /dev/null +++ b/openchallenges/data_integration_lambda.py @@ -0,0 +1,62 @@ +from aws_cdk import aws_iam as iam +from aws_cdk import aws_lambda as lambda_ +from constructs import Construct + + +class DataIntegrationLambda(Construct): + """ + A CDK construct to define an AWS Lambda function for data integration. + + This construct creates an IAM role with the necessary permissions and a Docker-based + Lambda function for handling data integration tasks. + """ + + def __init__(self, scope: Construct, id: str) -> None: + """ + Builds the IAM role for the Lambda function. + + This role allows the Lambda function to execute basic AWS operations. + + Returns: + iam.Role: The IAM role for the Lambda function. + """ + super().__init__(scope, id) + + self.lambda_role = self._build_lambda_role() + self.lambda_function = self._build_lambda_function(self.lambda_role) + + def _build_lambda_role(self) -> iam.Role: + return iam.Role( + self, + "LambdaRole", + assumed_by=iam.ServicePrincipal("lambda.amazonaws.com"), + managed_policies=[ + iam.ManagedPolicy.from_aws_managed_policy_name( + managed_policy_name=("service-role/AWSLambdaBasicExecutionRole") + ) + ], + ) + + def _build_lambda_function(self, role: iam.Role) -> lambda_.Function: + """ + Builds the Docker-based AWS Lambda function. + + The Lambda function uses a Docker image built from a local directory. + + Args: + role (iam.Role): The IAM role to associate with the Lambda function. + + Returns: + _lambda.Function: The Docker-based AWS Lambda function. + """ + return lambda_.DockerImageFunction( + self, + "LambdaFunction", + code=lambda_.DockerImageCode.from_image_asset( + # Directory relative to where you execute cdk deploy contains a + # Dockerfile with build instructions. + directory="cdk_docker/data-integration-lambda" + ), + role=role, + memory_size=128, + ) diff --git a/openchallenges/data_integration_props.py b/openchallenges/data_integration_props.py new file mode 100644 index 0000000..59416e4 --- /dev/null +++ b/openchallenges/data_integration_props.py @@ -0,0 +1,19 @@ +from dataclasses import dataclass +from aws_cdk.aws_scheduler_alpha import ScheduleExpression + + +@dataclass +class DataIntegrationProps: + """ + Data integration properties. + + Attributes: + schedule (ScheduleExpression): The schedule for triggering the data integration. + schedule_description (str): The description of the schedule. + """ + + schedule: ScheduleExpression + """The schedule for triggering the data integration.""" + + schedule_description: str + """The description of the schedule.""" diff --git a/openchallenges/data_integration_stack.py b/openchallenges/data_integration_stack.py new file mode 100644 index 0000000..5b87a76 --- /dev/null +++ b/openchallenges/data_integration_stack.py @@ -0,0 +1,66 @@ +import aws_cdk as cdk +from aws_cdk import ( + aws_scheduler_alpha as scheduler_alpha, + aws_scheduler_targets_alpha as scheduler_targets, +) +from openchallenges.data_integration_lambda import DataIntegrationLambda +from openchallenges.data_integration_props import DataIntegrationProps +from constructs import Construct + + +class DataIntegrationStack(cdk.Stack): + """ + Defines an AWS CDK stack for data integration. + + This stack sets up the resources required for scheduling and executing + data integration tasks using AWS Lambda and EventBridge Scheduler. + + The stack includes: + - A Lambda function for data integration. + - An EventBridge Scheduler schedule to trigger the Lambda function. + - An EventBridge Scheduler group for organizing schedules. + + Attributes: + scope (Construct): The parent construct. + id (str): The unique identifier for this stack. + props (DataIntegrationProps): The properties for the data integration, including the schedule. + """ + + def __init__( + self, scope: Construct, id: str, props: DataIntegrationProps, **kwargs + ) -> None: + """ + Initializes the DataIntegrationStack. + + Arguments: + scope (Construct): The parent construct for this stack. + id (str): The unique identifier for this stack. + props (DataIntegrationProps): The properties required for data integration, + including the schedule. + **kwargs: Additional arguments passed to the base `cdk.Stack` class. + """ + super().__init__(scope, id, **kwargs) + + data_integration_lambda = DataIntegrationLambda(self, "data-integration-lambda") + + target = scheduler_targets.LambdaInvoke( + data_integration_lambda.lambda_function, + input=scheduler_alpha.ScheduleTargetInput.from_object({}), + ) + + # Create a group for the schedule (maybe we want to add more schedules + # to this group the future) + schedule_group = scheduler_alpha.Group( + self, + "group", + group_name="schedule-group", + ) + + scheduler_alpha.Schedule( + self, + "schedule", + schedule=props.schedule, + target=target, + group=schedule_group, + description=props.schedule_description, + ) diff --git a/openchallenges/service_props.py b/openchallenges/service_props.py index efa7316..25b16c9 100644 --- a/openchallenges/service_props.py +++ b/openchallenges/service_props.py @@ -1,6 +1,25 @@ +from dataclasses import dataclass +from typing import List + CONTAINER_LOCATION_PATH_ID = "path://" +@dataclass +class ContainerVolume: + """ + Holds onto configuration for a volume used in the container. + + Attributes: + path: The path on the container to mount the host volume at. + size: The size of the volume in GiB. + read_only: Container has read-only access to the volume, set to `false` for write access. + """ + + path: str + size: int = 15 + read_only: bool = False + + class ServiceProps: """ ECS service properties @@ -13,6 +32,7 @@ class ServiceProps: supports docker registry references (i.e. ghcr.io/sage-bionetworks/openchallenges-thumbor:latest) container_env_vars: a json dictionary of environment variables to pass into the container i.e. {"EnvA": "EnvValueA", "EnvB": "EnvValueB"} + container_volumes: List of `ContainerVolume` resources to mount into the container """ def __init__( @@ -22,6 +42,7 @@ def __init__( container_memory: int, container_location: str, container_env_vars: dict, + container_volumes: List[ContainerVolume] = None, ) -> None: self.container_name = container_name self.container_port = container_port @@ -32,3 +53,7 @@ def __init__( ) self.container_location = container_location self.container_env_vars = container_env_vars + if container_volumes is None: + self.container_volumes = [] + else: + self.container_volumes = container_volumes diff --git a/openchallenges/service_stack.py b/openchallenges/service_stack.py index 3911a7e..e1be848 100644 --- a/openchallenges/service_stack.py +++ b/openchallenges/service_stack.py @@ -60,6 +60,28 @@ def __init__( ) ) + # default ECS execution policy plus Guardduty access + execution_role = iam.Role( + self, + "ExecutionRole", + assumed_by=iam.ServicePrincipal("ecs-tasks.amazonaws.com"), + managed_policies=[ + iam.ManagedPolicy.from_aws_managed_policy_name( + "service-role/AmazonECSTaskExecutionRolePolicy" + ), + ], + ) + execution_role.add_to_policy( + iam.PolicyStatement( + actions=[ + "logs:CreateLogStream", + "logs:PutLogEvents", + ], + resources=["*"], + effect=iam.Effect.ALLOW, + ) + ) + # ECS task with fargate self.task_definition = ecs.FargateTaskDefinition( self, @@ -67,6 +89,7 @@ def __init__( cpu=1024, memory_limit_mib=4096, task_role=task_role, + execution_role=execution_role, ) image = ecs.ContainerImage.from_registry(props.container_location) @@ -118,14 +141,14 @@ def __init__( ), ) - # mount volume for DB - if "mariadb" in construct_id: - self.volume = ecs.ServiceManagedVolume( + # mount volumes + for container_volume in props.container_volumes: + service_volume = ecs.ServiceManagedVolume( self, - "ServiceVolume", + "ContainerVolume", name=props.container_name, managed_ebs_volume=ecs.ServiceManagedEBSVolumeConfiguration( - size=size.gibibytes(30), + size=size.gibibytes(container_volume.size), volume_type=ec2.EbsDeviceVolumeType.GP3, ), ) @@ -133,13 +156,12 @@ def __init__( self.task_definition.add_volume( name=props.container_name, configured_at_launch=True ) - self.service.add_volume(self.volume) + self.service.add_volume(service_volume) - self.volume.mount_in( - # should be mounted at openchallenges-mariadb:/data/db + service_volume.mount_in( self.container, - container_path="/data/db", - read_only=False, + container_path=container_volume.path, + read_only=container_volume.read_only, ) diff --git a/requirements.txt b/requirements.txt index 92c10ed..b1c51a6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,5 @@ aws-cdk-lib==2.139.0 +aws-cdk.aws-scheduler-alpha==2.139.0a0 +aws-cdk.aws-scheduler-targets-alpha==2.139.0a0 constructs>=10.0.0,<11.0.0 boto3>=1.34.1 diff --git a/source.bat b/source.bat deleted file mode 100644 index 9e1a834..0000000 --- a/source.bat +++ /dev/null @@ -1,13 +0,0 @@ -@echo off - -rem The sole purpose of this script is to make the command -rem -rem source .venv/bin/activate -rem -rem (which activates a Python virtualenv on Linux or Mac OS X) work on Windows. -rem On Windows, this command just runs this batch file (the argument is ignored). -rem -rem Now we don't need to document a Windows command for activating a virtualenv. - -echo Executing .venv\Scripts\activate.bat for you -.venv\Scripts\activate.bat From e8d67ee650646a8a0fecb08c3ab9ad78d4119ec8 Mon Sep 17 00:00:00 2001 From: Thomas Schaffter Date: Thu, 12 Dec 2024 09:32:56 -0800 Subject: [PATCH 25/26] update dev (#78) --- app.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app.py b/app.py index 8fe9231..b87bddc 100644 --- a/app.py +++ b/app.py @@ -17,7 +17,7 @@ # get the environment environment = utils.get_environment() stack_name_prefix = f"openchallenges-{environment}" -image_version = "1.1.1" +image_version = "1.1.2" # get VARS from cdk.json env_vars = app.node.try_get_context(environment) @@ -308,7 +308,7 @@ "API_DOCS_URL": f"https://{fully_qualified_domain_name}/api-docs", "APP_VERSION": image_version, "CSR_API_URL": f"https://{fully_qualified_domain_name}/api/v1", - "DATA_UPDATED_ON": "2024-11-27", + "DATA_UPDATED_ON": "2024-12-12", "ENVIRONMENT": "production", "GOOGLE_TAG_MANAGER_ID": "GTM-NBR5XD8C", "SSR_API_URL": "http://openchallenges-api-gateway:8082/api/v1", From de6880695c7b638a0bc10c2f669d2b71841a9393 Mon Sep 17 00:00:00 2001 From: Thomas Schaffter Date: Thu, 12 Dec 2024 10:40:40 -0800 Subject: [PATCH 26/26] Increase role duration to 90 minutes (#79) --- .github/workflows/aws-deploy.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/aws-deploy.yaml b/.github/workflows/aws-deploy.yaml index 6b6486e..a636b92 100644 --- a/.github/workflows/aws-deploy.yaml +++ b/.github/workflows/aws-deploy.yaml @@ -20,7 +20,7 @@ on: type: string role-duration-seconds: type: number - default: 3600 + default: 5400 environment: required: true type: string