Merge branch 'main' into christina/move-frontend-off-aws

.github/workflows/aws-cd.yml

@@ -35,7 +35,7 @@ jobs:
           # Build a docker container and
           # push it to ECR so that it can
           # be deployed to ECS.
-          docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG -f infrastructure/prod/Dockerfile.server .
+          docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG --build-arg="COMMIT=$IMAGE_TAG" --build-arg="BUILD_TIMESTAMP=$(date +%s)" --build-arg="COMMIT_MESSAGE=$(git --no-pager show -s --format=%s)" -f infrastructure/prod/Dockerfile.server .
           docker push $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG
           echo "image=$ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG" >> $GITHUB_OUTPUT
 
@@ -49,15 +49,15 @@ jobs:
           # Build a docker container and
           # push it to ECR so that it can
           # be deployed to ECS.
-          docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG -f infrastructure/prod/Dockerfile.app .
+          docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG --build-arg="COMMIT=$IMAGE_TAG" --build-arg="BUILD_TIMESTAMP=$(date +%s)" --build-arg="COMMIT_MESSAGE=$(git --no-pager show -s --format=%s)" -f infrastructure/prod/Dockerfile.app .
           docker push $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG
           echo "image=$ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG" >> $GITHUB_OUTPUT
 
       - name: Update & Deploy ECS task definitions
         run: |
           # - create new revision for task definition with latest image.
           # - redeploy ECS services with the latest revision.
-          ./infrastructure/prod/redeploy.sh staging
+          ./infrastructure/prod/redeploy.sh staging local-head both
 
       - name: Logout of Amazon ECR
         if: always()

.github/workflows/main.yml

@@ -14,9 +14,9 @@ jobs:
           node-version: 16
       - uses: bahmutov/npm-install@v1
       - run: yarn lint
-  run-api-v2-tests:
+  run-api-tests:
     runs-on: ubuntu-latest
-    name: API V2 Tests
+    name: API Tests
     steps:
       - name: Checkout code
         uses: actions/checkout@v3
@@ -26,7 +26,7 @@ jobs:
           node-version: "16"
       - run: yarn install --immutable
       - name: Run Jest Tests
-        run: yarn packages/api-v2 test
+        run: yarn packages/api test
   build-frontend-image:
     runs-on: ubuntu-latest
     steps:

README.md

@@ -6,17 +6,17 @@ GraduateNU aims to empower Northeastern students to customize their plan of stud
 
 1. Make sure you run the dev postgres database using `yarn dev:db:up`. Run `yarn dev:db:down` when you're done.
 
-2. Make a copy of the packages/api-v2/.env.development and name it .env.development.local. Fill in placeholder secrets.
+2. Make a copy of the packages/api/.env.development and name it .env.development.local. Fill in placeholder secrets.
 
-3. Make a copy of the packages/api-v2/.env.testing and name it .env.testing.local. for running BE tests
+3. Make a copy of the packages/api/.env.testing and name it .env.testing.local. for running BE tests
 
-4. If you haven't run migrations in a while or this is a fresh repo, then run the migrations for the the api using `yarn packages/api-v2 dev:migration:run`. See packages/API-V2/README for more info.
+4. If you haven't run migrations in a while or this is a fresh repo, then run the migrations for the the api using `yarn packages/api dev:migration:run`. See packages/API/README for more info.
 
-5. Then run the new version of the application by running `yarn dev:v2` at the root of the project. This starts up a NestJS server + a NextJS frontend + a Proxy. The proxy listens on port [3002](http://localhost:3002/), forwards /api requests to the NestJS server running on port 3001, and all other requests to the frontend running on port 3000.
+5. Then run the new version of the application by running `yarn dev` at the root of the project. This starts up a NestJS server + a NextJS frontend + a Proxy. The proxy listens on port [3002](http://localhost:3002/), forwards /api requests to the NestJS server running on port 3001, and all other requests to the frontend running on port 3000.
 
 6. Visit [http://localhost:3002](http://localhost:3002/) to view the app.
 
-To run the two separately, visit the frontend and api-v2 packages(sub directories of the monorepo).
+To run the two separately, visit the frontend and api packages(sub directories of the monorepo).
 
 ## Running the prod builds in prod docker configurations locally
 
@@ -43,9 +43,9 @@ Example: `yarn packages/frontend lint`
 
 The workspaces within this monorepo are:
 
-1. **frontend-v2**: A Next.js web UI. It is what users see when they visit our application.
+1. **frontend**: A Next.js web UI. It is what users see when they visit our application.
 
-2. **api-v2**: A Nest.js API reponsible for storing and managing Graduate's data. Our frontend leans on our api for data related services.
+2. **api**: A Nest.js API reponsible for storing and managing Graduate's data. Our frontend leans on our api for data related services.
 
 3. **api-client:** A typescript client responsible for providing a streamlined and typed interface to interact with our API. The frontend uses this client to send request to our API.
 

infrastructure/develop/docker-compose.api.yml

@@ -25,7 +25,7 @@ services:
     ports:
       - 4001:3001
     env_file:
-      - "../../packages/api-v2/.env.development"
+      - "../../packages/api/.env.development"
     # env vars here override the env vars in the env_file above
     environment:
       # we are no longer connecting to the db on localhost, the host is the name of the container

infrastructure/prod/Dockerfile.app

@@ -10,21 +10,29 @@ WORKDIR /app
 # package.json of root and of needed packages
 COPY package.json yarn.lock babel.config.js .yarnrc.yml ./
 COPY .yarn .yarn
-COPY packages/frontend-v2/package.json packages/frontend-v2/package.json
+COPY packages/frontend/package.json packages/frontend/package.json
 COPY packages/api-client/package.json packages/api-client/package.json
 COPY packages/common/package.json packages/common/package.json
 
 # Install at root level
 RUN yarn install > /dev/null
 
 # Get src files
-COPY packages/frontend-v2 packages/frontend-v2
+COPY packages/frontend packages/frontend
 COPY packages/api-client packages/api-client
 COPY packages/common packages/common
 
+ARG COMMIT
+ARG BUILD_TIMESTAMP
+ARG COMMIT_MESSAGE
+
+ENV NEXT_PUBLIC_COMMIT_HASH $COMMIT
+ENV NEXT_PUBLIC_COMMIT_MESSAGE $COMMIT_MESSAGE
+ENV NEXT_PUBLIC_BUILD_TIMESTAMP $BUILD_TIMESTAMP
+
 RUN yarn packages/api-client build
 RUN yarn packages/common build
-RUN yarn packages/frontend-v2 build
+RUN yarn packages/frontend build
 
 # Production image 
 FROM node:16-alpine AS runner
@@ -36,9 +44,9 @@ RUN addgroup --system --gid 1001 nodejs
 RUN adduser --system --uid 1001 nextjs
 
 COPY --from=builder --chown=nextjs:nodejs /app/node_modules ./node_modules
-COPY --from=builder --chown=nextjs:nodejs /app/packages/frontend-v2/public ./public
-COPY --from=builder --chown=nextjs:nodejs /app/packages/frontend-v2/.next/standalone ./
-COPY --from=builder --chown=nextjs:nodejs /app/packages/frontend-v2/.next/static ./.next/static
+COPY --from=builder --chown=nextjs:nodejs /app/packages/frontend/public ./public
+COPY --from=builder --chown=nextjs:nodejs /app/packages/frontend/.next/standalone ./
+COPY --from=builder --chown=nextjs:nodejs /app/packages/frontend/.next/static ./.next/static
 
 USER nextjs
 

infrastructure/prod/Dockerfile.server

@@ -5,24 +5,31 @@ WORKDIR /server
 # package.json of root and of needed packages
 COPY package.json yarn.lock babel.config.js .yarnrc.yml ./
 COPY .yarn .yarn
-COPY packages/api-v2/package.json packages/api-v2/package.json
+COPY packages/api/package.json packages/api/package.json
 COPY packages/common/package.json packages/common/package.json
 
 # Install at root level
 RUN yarn install > /dev/null
 
 # Get src files
-COPY packages/api-v2 packages/api-v2
+COPY packages/api packages/api
 COPY packages/common packages/common
 
 # Build server and common dependency
 RUN yarn packages/common build
-RUN yarn packages/api-v2 build
+RUN yarn packages/api build
 
 FROM node:16-alpine AS runner
 WORKDIR /server
 
+ARG COMMIT
+ARG COMMIT_MESSAGE
+ARG BUILD_TIMESTAMP
+
 ENV NODE_ENV production
+ENV COMMIT_HASH $COMMIT
+ENV COMMIT_MESSAGE $COMMIT_MESSAGE
+ENV BUILD_TIMESTAMP $BUILD_TIMESTAMP
 
 RUN addgroup --system --gid 1001 nodejs
 RUN adduser --system --uid 1001 nestjs

infrastructure/prod/README.md

@@ -0,0 +1,36 @@
+# How to do a Manual Deploy (to staging or prod).
+
+Note: All of the `yarn` scripts in this guide are in the root `package.json`.
+
+**This will deploy the current files as they are in your local repo**. They will be tagged with the local HEAD commit.
+
+If you don't want this, you'll need to run modified calls rather than the package.json scripts directly.
+
+## Prereqs:
+
+- Install Docker
+- Install the AWS CLI
+- Have an AWS Access Key ID and AWS Secret Access Key pair.
+- Run `aws configure`, and input those keys along with us-east-1 as the region. You can set the Default Output Format to json.
+
+## Steps:
+
+1. Run `yarn frontend:build` and/or `yarn backend:build` depending on what you want to build.
+2. Run `yarn frontend:run` and/or `yarn backend:run` to test that the image runs (the backend won't be able to connect to the database which is expected, and the frontend will not connect to the backend).
+3. You may have to run `yarn ecr:docker:auth` to reauthenticate Docker with AWS ECR. (There is no harm in running this command unnecessarily, so if you're unsure you can just run it).
+4. If the images run successfully, run `yarn frontend:push` and/or `yarn backend:push` to push the built images to AWS ECR.
+5. Run `./infrastructure/prod/redeploy.sh <environment> local-head <service>`
+
+   `<environment>` is either `prod` or `staging`.  
+   `<service>` is one of `frontend`, `backend`, or `both`.
+
+# How to Manually Redeploy a Previously Deployed Commit
+
+If you know that a commit already has an image pushed to ECR (ex. the commit is in main, or you already pushed the image using the above steps) and you want to deploy that particular image to staging or prod, you can use the redeploy.sh script.
+
+1. Find the commit's hash (looks like this: 1445f9533c4ec9a7324c721f09fc5ccec1542d8d).
+2. Run `./infrastructure/prod/redeploy.sh <environment> <commit_hash> <service>`
+
+   `<environment>` is either `prod` or `staging`.  
+   `<commit_hash>` is the hash of the commit you just found.  
+   `<service>` is one of `frontend`, `backend`, or `both`.

infrastructure/prod/entrypoint.server.sh

@@ -1,6 +1,7 @@
 #!/bin/sh
 
-cd packages/api-v2
+cd packages/api
+echo "Running on commit: $COMMIT_HASH"
 yarn typeorm migration:run 
 exec "$@"
 

infrastructure/prod/get-ecr-image-name.sh

@@ -0,0 +1,34 @@
+if [[ $# != 2 ]]; then
+  echo "Usage: get-ecr-image-name.sh <COMMIT_TAG | latest-main | local-head> <frontend | backend>"
+  exit 1
+fi
+
+# Accept a few different formats for the commit hash.
+if [[ ${#1} = 40 ]]; then
+  ECR_IMAGE_COMMIT_HASH=$1
+elif [[ $1 = "latest-main" ]]; then
+  ECR_IMAGE_COMMIT_HASH=$(git ls-remote https://github.com/sandboxnu/graduatenu.git main | awk '{ print $1 }')
+elif [[ $1 = "local-head" ]]; then
+  ECR_IMAGE_COMMIT_HASH=$(git rev-parse HEAD)
+else
+  echo "ERROR: Invalid commit '$1'"
+  exit 1
+fi
+
+if [[ $2 = "frontend" ]]; then
+  REPO="graduatenu-node"
+elif [[ $2 = "backend" ]]; then
+  REPO="graduatenu-rails"
+else
+  echo "Please choose a service to create an image name for: 'frontend' or 'backend'"
+  exit 1
+fi
+
+AWS_ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text)
+AWS_DEFAULT_REGION="us-east-1"
+
+ECR_REGISTRY="${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_DEFAULT_REGION}.amazonaws.com"
+
+ECR_IMAGE_NAME="${ECR_REGISTRY}/${REPO}:${ECR_IMAGE_COMMIT_HASH}"
+
+echo $ECR_IMAGE_NAME

infrastructure/prod/redeploy.sh

@@ -1,35 +1,65 @@
 #!/bin/bash
 
+if [[ $# != 3 ]]; then
+  echo "Usage: redeploy.sh <prod | staging> <COMMIT_TAG | latest-main | local-head> <frontend | backend | both>"
+  exit 1
+fi
+
 if [[ ! " prod staging " =~ " $1 " ]]; then
   echo "Please provide environment to use: prod or staging"
   exit 1
 fi
 
+# Accept a few different formats for the commit hash.
+if [[ ${#2} = 40 ]]; then
+  ECR_IMAGE_COMMIT_HASH=$2
+elif [[ $2 = "latest-main" ]]; then
+  ECR_IMAGE_COMMIT_HASH=$(git ls-remote https://github.com/sandboxnu/graduatenu.git main | awk '{ print $1 }')
+elif [[ $2 = "local-head" ]]; then
+  ECR_IMAGE_COMMIT_HASH=$(git rev-parse HEAD)
+else
+  echo "ERROR: Invalid commit '$2'"
+  exit 1
+fi
+
+# Only deploy the service specified.
+if [[ $3 = "frontend" ]]; then
+  DEPLOY_INDEXES=(1)
+elif [[ $3 = "backend" ]]; then
+  DEPLOY_INDEXES=(0)
+elif [[ $3 = "both" ]]; then
+  DEPLOY_INDEXES=(0 1)
+else
+  echo "Please choose a service to deploy: 'frontend', 'backend', or 'both'"
+  exit 1
+fi
+
+echo "Deploying $3 repo(s) to $1 with commit "$2" ($ECR_IMAGE_COMMIT_HASH)"
+
 AWS_ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text)
 AWS_DEFAULT_REGION="us-east-1"
 REPOS=( "graduatenu-rails" "graduatenu-node" )
-LATEST_HASH=$(git ls-remote https://github.com/sandboxnu/graduatenu.git main | awk '{ print $1 }')
 ECS_CLUSTER="$1-graduatenu"
 TASK_FAMILIES=( "${ECS_CLUSTER}-api" "${ECS_CLUSTER}-web" )
 SERVICES=( "${ECS_CLUSTER}-api" "${ECS_CLUSTER}-web" )
 
+
 # Disable aws from sending stdout to less
 export AWS_PAGER=""
 
-echo "Redeploying services for cluster: ${ECS_CLUSTER} with last pushed image"
-
-
-for i in "${!REPOS[@]}"; do
+for service_index in "${!DEPLOY_INDEXES[@]}"; do
+    echo "Deploying ${REPOS[DEPLOY_INDEXES[service_index]]}..."
+    i=${DEPLOY_INDEXES[service_index]}
     # Last pushed image should always be tagged with the latest commit hash on main 
-    ECR_IMAGE="${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_DEFAULT_REGION}.amazonaws.com/${REPOS[$i]}:${LATEST_HASH}"
+    ECR_IMAGE="${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_DEFAULT_REGION}.amazonaws.com/${REPOS[$i]}:${ECR_IMAGE_COMMIT_HASH}"
     TASK_FAMILY="${TASK_FAMILIES[$i]}"
     SERVICE="${SERVICES[$i]}"
     # fetch template for task definition 
     TASK_DEFINITION=$(aws ecs describe-task-definition --task-definition "$TASK_FAMILY" --region "$AWS_DEFAULT_REGION")
     # update the template's image to use the latest ECR_IMAGE
-    NEW_TASK_DEFINTIION=$(echo $TASK_DEFINITION | jq --arg IMAGE "$ECR_IMAGE" '.taskDefinition | .containerDefinitions[0].image = $IMAGE | del(.taskDefinitionArn) | del(.revision) | del(.status) | del(.requiresAttributes) | del(.compatibilities) | del(.registeredAt) | del(.registeredBy)')
+    NEW_TASK_DEFINITION=$(echo $TASK_DEFINITION | jq --arg IMAGE "$ECR_IMAGE" '.taskDefinition | .containerDefinitions[0].image = $IMAGE | del(.taskDefinitionArn) | del(.revision) | del(.status) | del(.requiresAttributes) | del(.compatibilities) | del(.registeredAt) | del(.registeredBy)')
     # register the new revision for the task definition 
-    NEW_TASK_INFO=$(aws ecs register-task-definition --region "$AWS_DEFAULT_REGION" --cli-input-json "$NEW_TASK_DEFINTIION")
+    NEW_TASK_INFO=$(aws ecs register-task-definition --region "$AWS_DEFAULT_REGION" --cli-input-json "$NEW_TASK_DEFINITION")
     NEW_REVISION=$(echo $NEW_TASK_INFO | jq '.taskDefinition.revision')
     # update the service to replace tasks with the latest revision using the latest image
     aws ecs update-service --cluster ${ECS_CLUSTER} \

package.json

@@ -11,8 +11,8 @@
     ]
   },
   "scripts": {
-    "dev:v2": "concurrently \"yarn foreach '@graduate/(common|api-v2|frontend-v2|api-client)' run dev\" \"yarn dev:proxy\"",
-    "prod:v2": "concurrently \"yarn foreach '@graduate/(common|api-v2|frontend-v2|api-client)' run prod\" \"yarn dev:proxy\"",
+    "dev": "concurrently \"yarn foreach '@graduate/(common|api|frontend|api-client)' run dev\" \"yarn dev:proxy\"",
+    "prod": "concurrently \"yarn foreach '@graduate/(common|api|frontend|api-client)' run prod\" \"yarn dev:proxy\"",
     "foreach": "yarn workspaces foreach --parallel --verbose --interlaced --include",
     "dev:proxy": "node infrastructure/develop/dev-proxy.js",
     "dev:db:up": "docker compose -f infrastructure/develop/docker-compose.db.yml up -d",
@@ -22,6 +22,13 @@
     "backend:docker:build": "docker compose -f infrastructure/develop/docker-compose.api.yml build",
     "backend:docker:run": "docker compose -f infrastructure/develop/docker-compose.api.yml up -d",
     "backend:docker:down": "docker compose -f infrastructure/develop/docker-compose.api.yml down",
+    "frontend:build": "docker build --platform linux/amd64 --build-arg=\"COMMIT=$(git rev-parse HEAD)\" --build-arg=\"BUILD_TIMESTAMP=$(date +%s)\" --build-arg=\"COMMIT_MESSAGE=$(git --no-pager show -s --format=%s)\" -t $(sh ./infrastructure/prod/get-ecr-image-name.sh local-head frontend) -f ./infrastructure/prod/Dockerfile.app .",
+    "frontend:run": "docker run -p 4000:3000 $(sh ./infrastructure/prod/get-ecr-image-name.sh local-head frontend)",
+    "frontend:push": "docker push $(sh ./infrastructure/prod/get-ecr-image-name.sh local-head frontend)",
+    "backend:build": "docker build --platform linux/amd64 --build-arg=\"COMMIT=$(git rev-parse HEAD)\" --build-arg=\"BUILD_TIMESTAMP=$(date +%s)\" --build-arg=\"COMMIT_MESSAGE=$(git --no-pager show -s --format=%s)\" -t $(sh ./infrastructure/prod/get-ecr-image-name.sh local-head backend) -f ./infrastructure/prod/Dockerfile.server .",
+    "backend:run": "docker run -p 4001:3001 $(sh ./infrastructure/prod/get-ecr-image-name.sh local-head backend)",
+    "backend:push": "docker push $(sh ./infrastructure/prod/get-ecr-image-name.sh local-head backend)",
+    "ecr:docker:auth": "aws ecr get-login-password --region us-east-1 | docker login --username AWS --password-stdin $(aws sts get-caller-identity --query Account --output text).dkr.ecr.us-east-1.amazonaws.com",
     "lint": "eslint packages/ --ext .ts,.tsx .",
     "tsc": "yarn workspaces foreach -v --exclude . run tsc",
     "g:babel": "cd $INIT_CWD && babel",
@@ -31,6 +38,7 @@
   "dependencies": {
     "@babel/plugin-proposal-class-properties": "^7.18.6",
     "@babel/plugin-proposal-decorators": "^7.17.8",
+    "@nestjs/throttler": "^5.0.1",
     "cross-env": "^7.0.3",
     "nodemailer": "^6.9.1"
   },