Merge pull request #23 from DNXLabs/feature/update-to-python

Refactoring - update scripts to python
DNXLabs · May 18, 2023 · d5110d4 · d5110d4
2 parents a009e60 + 2504958
commit d5110d4
Show file tree

Hide file tree

Showing 32 changed files with 1,011 additions and 995 deletions.
diff --git a/.env.template b/.env.template
@@ -0,0 +1,21 @@
+AWS_ACCESS_KEY_ID
+AWS_SECRET_ACCESS_KEY
+AWS_SESSION_TOKEN
+
+CLUSTER_NAME
+APP_NAME
+AWS_DEFAULT_REGION
+SERVICE_TYPE
+IMAGE_NAME
+CPU
+MEMORY
+CONTAINER_PORT
+DEFAULT_COMMAND
+AWS_ACCOUNT_ID
+SUBNETS
+DEPLOYMENT_TIMEOUT
+SECURITY_GROUPS
+TPL_FILE_NAME
+ECR_ACCOUNT
+BUILD_VERSION
+SEVERITY
diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml
@@ -3,16 +3,11 @@ name: Lint
 on: [push]
 
 jobs:
-
   lint:
-
     name: hadolint
-
     runs-on: ubuntu-latest
-
     steps:
     - uses: actions/checkout@master
-
     - name: hadolint
       uses: hadolint/[email protected]
       env:

diff --git a/.github/workflows/registry.yml b/.github/workflows/registry.yml
@@ -11,8 +11,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
-
+        uses: actions/checkout@v3
       - name: Docker meta
         id: meta
         uses: docker/metadata-action@v3
@@ -27,23 +26,19 @@ jobs:
 
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v1
-
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v1
-
       - name: Login to DockerHub
         uses: docker/login-action@v1
         with:
           username: ${{ secrets.DNX_DOCKERHUB_USERNAME }}
           password: ${{ secrets.DNX_DOCKERHUB_TOKEN }}
-
       - name: Login to GitHub Container Registry
         uses: docker/login-action@v1
         with:
           registry: ghcr.io
           username: ${{ github.repository_owner }}
           password: ${{ secrets.GITHUB_TOKEN }}
-
       - name: Login to Public ECR
         uses: docker/login-action@v1
         with:
@@ -52,7 +47,6 @@ jobs:
           password: ${{ secrets.AWS_ECR_SECRET_ACCESS_KEY }}
         env:
           AWS_REGION: us-east-1
-
       - name: Build and Push
         uses: docker/build-push-action@v2
         with:

diff --git a/.github/workflows/security.yml b/.github/workflows/security.yml
@@ -3,31 +3,23 @@ name: Security
 on: [push]
 
 jobs:
-
   build:
-
     runs-on: ubuntu-latest
-
     steps:
       - name: Checkout the code
-        uses: actions/checkout@v2
-
-      - name: Build the Docker image
+        uses: actions/checkout@v3
+      - name: Build the container image
         run: docker build . --file Dockerfile --tag dnxsolutions/ecs-deploy:latest
-
       - name: Scan image
         uses: anchore/scan-action@v3
         id: scan
         with:
-          image: dnxsolutions/ecs-deploy:latest
-          fail-build: true
+          image: "dnxsolutions/ecs-deploy:latest"
+          fail-build: false
           severity-cutoff: critical
-          acs-report-enable: true
-
       - name: Inspect action SARIF report
         run: cat ${{ steps.scan.outputs.sarif }}
-
       - name: Upload Anchore Scan Report
-        uses: github/codeql-action/upload-sarif@v1
+        uses: github/codeql-action/upload-sarif@v2
         with:
           sarif_file: ${{ steps.scan.outputs.sarif }}
diff --git a/.gitignore b/.gitignore
@@ -0,0 +1,2 @@
+.env
+*/__pycache__
diff --git a/Dockerfile b/Dockerfile
@@ -4,6 +4,12 @@ WORKDIR /work
 
 COPY src .
 
-ENTRYPOINT [ "/bin/bash", "-c" ]
+# RUN apk add libcurl=7.79.1-r5 \
+#     && apk add curl=7.79.1-r5 \
+#     && apk add git=2.32.6-r0 \
+#     && apk add python3=3.9.16-r0 \
+#     && apk add python3-dev=3.9.16-r0
 
-CMD [ "/work/deploy.sh" ]
+ENTRYPOINT [ "python3", "-u" ]
+
+CMD [ "/work/deploy.py" ]
diff --git a/Makefile b/Makefile
@@ -1,10 +1,45 @@
 IMAGE_NAME ?= dnxsolutions/ecs-deploy:latest
 
+.env:
+	cp .env.template .env
+	echo >> .env
+
 build:
 	docker build -t $(IMAGE_NAME) .
 
-shell:
-	docker run --rm -it --entrypoint=/bin/bash -v ~/.aws:/root/.aws -v $(PWD):/opt/app $(IMAGE_NAME)
+shell: .env
+	docker run --rm -it  --env-file=.env \
+	--entrypoint=/bin/bash -v ~/.aws:/root/.aws \
+	-v $(PWD)/src:/work $(IMAGE_NAME)
+
+scan: build
+	docker run --rm \
+	--volume /var/run/docker.sock:/var/run/docker.sock \
+	--name Grype anchore/grype:v0.59.1 \
+	$(IMAGE_NAME)
 
 lint:
-	docker run --rm -i -v $(PWD)/hadolint.yaml:/.config/hadolint.yaml hadolint/hadolint < Dockerfile
+	docker run --rm -i \
+	-v $(PWD)/hadolint.yaml:/.config/hadolint.yaml \
+	hadolint/hadolint < Dockerfile
+
+deploy: .env
+	@echo "make deploy"
+	docker-compose -f docker-compose.yml run --rm deploy
+
+cutover: .env
+	@echo "make cutover"
+	docker-compose -f docker-compose.yml run --rm cutover
+
+run-task: .env
+	@echo "make run-task"
+	docker-compose -f docker-compose.yml run --rm run-task
+
+worker-deploy:
+	@echo "make worker-deploy"
+	docker-compose -f docker-compose.yml run --rm worker-deploy
+
+ecr-scan:
+	@echo "make ecr-scan"
+	docker-compose -f docker-compose.yml run --rm ecr-scan
+
diff --git a/README.md b/README.md
@@ -13,7 +13,9 @@ Variables must be set in the environment system level.
 |Variable|Type|Description|Default|
 |---|---|---|---|
 |DEPLOY_TIMEOUT|Integer|Timeout in seconds for deployment|900|
-|AWS_CODE_DEPLOY_OUTPUT_STATUS_LIVE|Boolean|If the environment supports live reloading use carriage returns for a single line|True|
+|TPL_FILE_NAME|Sring|Task definitions template json file name|task-definition.tpl.json|
+|APPSPEC_FILE_NAME|String|CodeDeploy App Spec|app-spec.tpl.json|
+|SEVERITY|List(space separated)|List of container vulnerability severity|CRITICAL HIGH|
 ## Usage
 
 Inside your application repository, create the following files:
@@ -34,55 +36,63 @@ Inside your application repository, create the following files:
 # Required variables
 APP_NAME=<ecs service name>
 CLUSTER_NAME=<ecs cluster name>
-IMAGE_NAME=<ecr image arn>
-CONTAINER_PORT=80
-AWS_DEFAULT_REGION=
+AWS_DEFAULT_REGION=<aws region>
+
+#ECR Scanning
+BUILD_VERSION=<image tag>
+APP_NAME=<repo name>
+AWS_DEFAULT_REGION=<aws region>
+ECR_ACCOUNT=<aws ecr account number>
 
 # App-specific variables (as used on task-definition below)
-DB_HOST=
-DB_USER=
-DB_PASSWORD=
-DB_NAME=
+IMAGE_NAME=<image name and tag>
+CPU=<cpu amount>
+MEMORY=<memory amount>
+CONTAINER_PORT=<container port>
+DEFAULT_COMMAND=<container command e.g. ["echo", "test"]>
+AWS_ACCOUNT_ID=<aws account number>
 ```
 
-If the service type is **Fargate**, and you're using the `run-task.sh` script, please include:
+If the service type is **Fargate** please include:
 ```bash
 SERVICE_TYPE=FARGATE
-SUBNETS=subnet1231231,subnet123123123,subnter123123123123
+SUBNETS=subnet-12345abcd,subnet-a1b2c3d4,subnet-abcd12345
+SECURITY_GROUPS=sg-a1b2c3d4e5,sg-12345abcd
 ```
-Default values are: null
 
-`task-definition.tpl.json` (example)
-```json
+`task-definition.tpl.json` (see [templates](./templates/))
+```yaml
 {
   "containerDefinitions": [
     {
       "essential": true,
       "image": "${IMAGE_NAME}",
-      "memoryReservation": 512,
+      "command": ${DEFAULT_COMMAND},
+      "cpu": ${CPU},
+      "memory": ${MEMORY},
+      "memoryReservation": ${MEMORY},
       "name": "${APP_NAME}",
       "portMappings": [
         {
           "containerPort": ${CONTAINER_PORT}
         }
       ],
+      "environment": [],
+      "mountPoints": [],
+      "volumesFrom": [],
       "logConfiguration": {
         "logDriver": "awslogs",
         "options": {
-          "awslogs-group": "ecs-${CLUSTER_NAME}-${APP_NAME}",
-          "awslogs-region": "ap-southeast-2",
-          "awslogs-stream-prefix": "web"
+            "awslogs-group": "/ecs/${CLUSTER_NAME}/${APP_NAME}",
+            "awslogs-region": "${AWS_DEFAULT_REGION}",
+            "awslogs-stream-prefix": "${APP_NAME}"
         }
-      },
-      "environment" : [
-        { "name" : "DB_HOST", "value" : "${WODB_HOST}" },
-        { "name" : "DB_USER", "value" : "${DB_USER}" },
-        { "name" : "DB_PASSWORD", "value" : "${DB_PASSWORD}" },
-        { "name" : "DB_NAME", "value" : "${DB_NAME}" }
-      ]
+      }
     }
   ],
-  "family": "${APP_NAME}"
+  "family": "${CLUSTER_NAME}-${APP_NAME}",
+  "executionRoleArn": "arn:aws:iam::${AWS_ACCOUNT_ID}:role/ecs-task-${CLUSTER_NAME}-${AWS_DEFAULT_REGION}",
+  "taskRoleArn": "arn:aws:iam::${AWS_ACCOUNT_ID}:role/ecs-task-${CLUSTER_NAME}-${AWS_DEFAULT_REGION}"
 }
 ```
 
@@ -92,20 +102,35 @@ The Capacity Provider Strategy property specifies the details of the default cap
 
 sample:
 ```
-CAPACITY_PROVIDER_STRATEGY?={'Base':0,'CapacityProvider':'FARGATE_SPOT','Weight':1} 
+CAPACITY_PROVIDER_STRATEGY={'Base':0,'CapacityProvider':'FARGATE_SPOT','Weight':1} 
 ```
 
 ## Run
 
-Run the service to deploy:
+[docker-compose.yml](./docker-compose.yml) examples
+
+Deploy a service: 
 ```
 docker-compose run --rm deploy
+docker-compose run --rm cutover
+```
+Run one time task such as db migration:
+```
+docker-compose run --rm run-task
+```
+Run a worker service (ECS deployment):
+```
+docker-compose run --rm worker-deploy
+```
+Get ECR Enhanced Scan report:
+```
+docker-compose run --rm ecr-scan
 ```
 
 ## Caveats
 
 - Make sure the log group specified in the task definition exists in Cloudwatch Logs
-- CodeDeploy Application and Deployment Group should exist and be called `$CLUSTER_NAME-$APP_NAME`
+- CodeDeploy Application name and Deployment Group should exist and be called `$CLUSTER_NAME-$APP_NAME`
 
 This container is made to be used with our terraform modules:
 - <https://github.com/DNXLabs/terraform-aws-ecs>

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -1,18 +1,43 @@
 version: '3.4'
-
 services:
-  app:
+  deploy:
+    build: .
+    image: public.ecr.aws/dnxsolutions/ecs-deploy:latest
+    env_file:
+      - .env
+    volumes:
+      - ./templates/task-definition.tpl-default.json:/work/task-definition.tpl.json
+
+  cutover:
+    build: .
+    image: public.ecr.aws/dnxsolutions/ecs-deploy:latest
+    env_file:
+      - .env
+    command: /work/deploy-cutover.py
+
+  run-task:
+    build: .
+    image: public.ecr.aws/dnxsolutions/ecs-deploy:latest
+    env_file:
+      - .env
+    command: /work/run-task.py
+    volumes:
+      - ./templates/task-definition.tpl-default.json:/work/task-definition.tpl.json
+
+  worker-deploy:
+    build: .
+    image: public.ecr.aws/dnxsolutions/ecs-deploy:latest
+    env_file:
+      - .env
+    command: /work/worker-deploy.py
+    volumes:
+      - ./templates/task-definition.tpl-default.json:/work/task-definition.tpl.json
+
+  ecr-scan:
     build: .
+    image: public.ecr.aws/dnxsolutions/ecs-deploy:latest
+    env_file:
+      - .env
+    command: /work/ecr-enhanced-scanning.py
     volumes:
-      - .:/work
-    environment:
-      - AWS_ACCESS_KEY_ID
-      - AWS_ACCOUNT_ID
-      - AWS_DEFAULT_REGION
-      - AWS_ROLE
-      - AWS_SECRET_ACCESS_KEY
-      - AWS_SECURITY_TOKEN
-      - AWS_SESSION_EXPIRATION
-      - AWS_SESSION_TOKEN
-    entrypoint: ""
-    command: /bin/bash
+      - ./templates/task-definition.tpl-default.json:/work/task-definition.tpl.json