Skip to content

Commit

Permalink
DH-5581 modularized terraform resources (Dataherald#490)
Browse files Browse the repository at this point in the history 
* DH-5581 modularized terraform resources
  • Loading branch information
@DishenWang2023
DishenWang2023 committed May 7, 2024
1 parent 680022c commit ec601c2
Show file tree
Hide file tree
Showing 8 changed files with 236 additions and 125 deletions.
70 changes: 51 additions & 19 deletions .github/workflows/ai-deploy-branch-env.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,8 @@ env:
SLACK_ICON: https://files.dataherald.com/random/github-actions.png
SLACK_USERNAME: GitHub Actions
SLACK_CHANNEL: eng-notifications
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
PINECONE_API_KEY: ${{ secrets.PINECONE_API_KEY }}

jobs:
extract-branch-names:
Expand Down Expand Up @@ -78,13 +80,13 @@ jobs:
target_env: branch
image_tag: ${{ needs.extract-branch-names.outputs.branch_name}}-${{ github.sha }}

deploy-backend-dbs-infra:
name: Deploy Backend and DBs infrastructure
deploy-backend:
name: Deploy Backend infrastructure
needs: [extract-branch-names, build-engine, build-server]
runs-on: ubuntu-latest
defaults:
run:
working-directory: ./apps/ai/server/terraform
working-directory: ./apps/ai/server/terraform/branch
outputs:
api_url: ${{ steps.get-api-url.outputs.api_url }}
steps:
Expand All @@ -105,9 +107,12 @@ jobs:
terraform_wrapper: false
- name: Apply Terraform
env:
PINECONE_API_KEY: ${{ secrets.PINECONE_API_KEY }}
TF_VAR_sha: ${{ github.sha }}
TF_VAR_branch_name: ${{ needs.extract-branch-names.outputs.branch_name }}
TF_VAR_vpc_id: ${{ vars.VPC_ID }}
TF_VAR_subnet_1_id: ${{ vars.PRIVATE_SUBNET_1_ID }}
TF_VAR_subnet_2_id: ${{ vars.PRIVATE_SUBNET_2_ID }}
TF_VAR_ecs_security_group_id: ${{ vars.BRANCH_SECURITY_GROUP_ID }}
TF_VAR_pinecone_index_name: ${{ needs.extract-branch-names.outputs.db_name }}
TF_VAR_mongodb_uri: ${{ vars.EPHEMERAL_MONGODB_URI }}
TF_VAR_mongodb_name: ${{ needs.extract-branch-names.outputs.db_name }}
Expand All @@ -116,8 +121,8 @@ jobs:

run: |
terraform init -upgrade -backend-config="key=${{ needs.extract-branch-names.outputs.branch_name }}"
terraform plan
terraform apply -auto-approve
terraform plan -target=module.aws_ecs
terraform apply -target=module.aws_ecs -auto-approve
- name: Get API URL
id: get-api-url
run: |
Expand All @@ -135,7 +140,36 @@ jobs:
if [ -n "$UNTAGGED_SERVER_IMAGES" ] && [ "$UNTAGGED_SERVER_IMAGES" != "[]" ]; then
aws ecr batch-delete-image --repository-name $ECR_SERVER_REPOSITORY --image-ids "$UNTAGGED_SERVER_IMAGES" --region us-east-1
fi
create-vector-db:
name: Create PineconeDB
needs: [extract-branch-names]
runs-on: ubuntu-latest
defaults:
run:
working-directory: ./apps/ai/server/terraform/branch
steps:
- name: Check out code
uses: actions/checkout@v4
with:
token: ${{ secrets.GH_ACCESS_TOKEN }}
submodules: true
- name: Configure AWS credentials
uses: aws-actions/configure-aws-credentials@v4
with:
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
aws-region: ${{ vars.AWS_REGION }}
- name: Setup Terraform
uses: hashicorp/setup-terraform@v3
with:
terraform_wrapper: false
- name: Apply Terraform
env:
TF_VAR_pinecone_index_name: ${{ needs.extract-branch-names.outputs.db_name }}
run: |
terraform init -upgrade -backend-config="key=${{ needs.extract-branch-names.outputs.branch_name }}"
terraform plan -target=module.pinecone_index
terraform apply -target=module.pinecone_index -auto-approve
populate-db:
name: Populate MongoDB
needs: [extract-branch-names, start-self-hosted-runner]
Expand Down Expand Up @@ -201,7 +235,7 @@ jobs:
extract-branch-names,
start-self-hosted-runner,
populate-db,
deploy-backend-dbs-infra,
create-vector-db,
]
runs-on: ${{ needs.start-self-hosted-runner.outputs.label }}
defaults:
Expand All @@ -215,8 +249,6 @@ jobs:
submodules: true
- name: Populate PineconeDB
env:
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
PINECONE_API_KEY: ${{ secrets.PINECONE_API_KEY }}
PINECONE_INDEX_NAME: ${{ needs.extract-branch-names.outputs.db_name }}
MONGODB_URI: ${{ vars.EPHEMERAL_MONGODB_URI }}
MONGODB_USERNAME: ${{ secrets.MONGODB_USERNAME }}
Expand All @@ -231,7 +263,7 @@ jobs:
deploy-frontend:
name: Deploy Frontend
needs: [extract-branch-names, deploy-backend-dbs-infra]
needs: [extract-branch-names, deploy-backend]
runs-on: ubuntu-latest
outputs:
url: ${{ steps.console_dns.outputs.url }}
Expand All @@ -252,10 +284,10 @@ jobs:
- name: Deploy to Vercel and set custom domain alias
run: |
echo Deploying to Vercel with the following environment variables:
echo "NEXT_PUBLIC_API_URL=${{ needs.deploy-backend-dbs-infra.outputs.api_url }}"
echo "NEXT_PUBLIC_API_URL=${{ needs.deploy-backend.outputs.api_url }}"
echo "AUTH0_BASE_URL=${{ steps.console_dns.outputs.url }}"
vercel deploy --yes --build-env NEXT_PUBLIC_API_URL=${{ needs.deploy-backend-dbs-infra.outputs.api_url }} --env AUTH0_BASE_URL=${{ steps.console_dns.outputs.url }} --token=${{ secrets.VERCEL_ACCESS_TOKEN }} >deployment-url.txt 2>error.txt
vercel deploy --yes --build-env NEXT_PUBLIC_API_URL=${{ needs.deploy-backend.outputs.api_url }} --env AUTH0_BASE_URL=${{ steps.console_dns.outputs.url }} --token=${{ secrets.VERCEL_ACCESS_TOKEN }} >deployment-url.txt 2>error.txt
# check the exit code
code=$?
Expand All @@ -274,7 +306,7 @@ jobs:
needs:
[
extract-branch-names,
deploy-backend-dbs-infra,
deploy-backend,
populate-db,
populate-vector-db,
deploy-frontend,
Expand All @@ -299,12 +331,12 @@ jobs:
needs:
[
extract-branch-names,
deploy-backend-dbs-infra,
deploy-backend,
populate-db,
populate-vector-db,
deploy-frontend,
]
if: needs.deploy-backend-dbs-infra.result == 'failure' || needs.populate-db.result == 'failure' || needs.populate-vector-db.result == 'failure' || needs.deploy-frontend.result == 'failure'
if: needs.deploy-backend.result == 'failure' || needs.populate-db.result == 'failure' || needs.populate-vector-db.result == 'failure' || needs.deploy-frontend.result == 'failure'
runs-on: ubuntu-latest
steps:
- name: Send failed notification
Expand All @@ -324,12 +356,12 @@ jobs:
needs:
[
extract-branch-names,
deploy-backend-dbs-infra,
deploy-backend,
populate-db,
populate-vector-db,
deploy-frontend,
]
if: needs.deploy-backend-dbs-infra.result == 'success' && needs.populate-db.result == 'success' && needs.populate-vector-db.result == 'success' && needs.deploy-frontend.result == 'success'
if: needs.deploy-backend.result == 'success' && needs.populate-db.result == 'success' && needs.populate-vector-db.result == 'success' && needs.deploy-frontend.result == 'success'
runs-on: ubuntu-latest
steps:
- name: Send success notification
Expand All @@ -341,7 +373,7 @@ jobs:
The branch environment deployment succeeded. Here are the details:
*Branch*: `${{ needs.extract-branch-names.outputs.branch }}`
*API URL*: ${{ needs.deploy-backend-dbs-infra.outputs.api_url }}
*API URL*: ${{ needs.deploy-backend.outputs.api_url }}
*Console URL*: ${{ needs.deploy-frontend.outputs.url }}
SLACK_COLOR: success

Expand Down
3 changes: 1 addition & 2 deletions .github/workflows/ai-destroy-branch-env.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ jobs:
PINECONE_API_KEY: ${{ secrets.PINECONE_API_KEY }}
defaults:
run:
working-directory: ./apps/ai/server
working-directory: ./apps/ai/server/terraform/branch

steps:
- name: Check out code
Expand All @@ -75,7 +75,6 @@ jobs:
TF_VAR_mongodb_password: ${{ secrets.MONGODB_PASSWORD }}
timeout-minutes: 3
run: |
cd terraform
terraform init -upgrade -backend-config="key=${{ needs.extract-branch-name.outputs.branch_name }}"
terraform destroy -auto-approve
- name: Delete images
Expand Down
130 changes: 26 additions & 104 deletions apps/ai/server/terraform/main.tf → ...i/server/terraform/branch/aws_ecs/main.tf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,97 +4,19 @@ terraform {
source = "aws"
version = "4.1.0"
}

pinecone = {
source = "skyscrapr/pinecone"
version = "0.5.1"
}
}
backend "s3" {
bucket = "terraform-states2"
region = "us-east-1"
}
}


provider "aws" {
region = "us-east-1"
}
provider "pinecone" {}

variable "branch_name" { type= string }

variable "sha" { type= string }

variable "pinecone_index_name" {
description = "pinecone_index_name"
type = string
}

variable "mongodb_uri" {
description = "mongodb_uri"
type = string
}

variable "mongodb_name" {
description = "mongodb_name"
type = string
}

variable "mongodb_username" {
description = "mongodb_username"
type = string
}

variable "mongodb_password" {
description = "mongodb_password"
type = string
}

variable "subnet_1_id" {
description = "subnet_1_id"
type = string
default = "subnet-076afb4a159204349"
}

variable "subnet_2_id" {
description = "subnet_2_id"
type = string
default = "subnet-0b6b9dbf631131b09"
}

variable "ecs_security_group_id" {
description = "security_group_id"
type = string
default = "sg-07fac199a96aa3b65"
}

resource "pinecone_index" "my_index" {
name = var.pinecone_index_name
dimension = 1536
metric = "cosine"
spec = {
serverless = {
cloud = "aws"
region = "us-west-2"
}
}
}

locals {
srv_connection_string = replace(var.mongodb_uri, "mongodb+srv://", "mongodb+srv://${var.mongodb_username}:${var.mongodb_password}@")
}

resource "aws_ecs_task_definition" "my_task_definition" {
family = "ai-backend-branch-${var.branch_name}"
task_role_arn = "arn:aws:iam::422486916789:role/ecsk2TaskExecutionRole"
execution_role_arn = "arn:aws:iam::422486916789:role/ecsk2TaskExecutionRole"
network_mode = "awsvpc"
family = "ai-backend-branch-${var.branch_name}"
task_role_arn = "arn:aws:iam::422486916789:role/ecsk2TaskExecutionRole"
execution_role_arn = "arn:aws:iam::422486916789:role/ecsk2TaskExecutionRole"
network_mode = "awsvpc"
requires_compatibilities = ["FARGATE"]
cpu = "2048"
memory = "4096"
# container_definitions = file("task_definition.json")
container_definitions =<<DEFINITION
cpu = "2048"
memory = "4096"
# container_definitions = file("task_definition.json")
container_definitions = <<DEFINITION
[
{
"name": "ai-engine-branch",
Expand Down Expand Up @@ -128,11 +50,11 @@ resource "aws_ecs_task_definition" "my_task_definition" {
},
{
"name": "MONGODB_URI",
"value": "${local.srv_connection_string}"
"value": "${var.mongodb_uri}"
},
{
"name": "GOLDEN_SQL_COLLECTION",
"value": "${var.pinecone_index_name}"
"value": "${var.index_name}"
}
],
"command": ["sh", "-c", "uvicorn dataherald.app:app --host 0.0.0.0 --port $CORE_PORT --log-config log_config.yml --log-level debug --reload"],
Expand Down Expand Up @@ -180,7 +102,7 @@ resource "aws_ecs_task_definition" "my_task_definition" {
},
{
"name": "MONGODB_URI",
"value": "${local.srv_connection_string}"
"value": "${var.mongodb_uri}"
}
],
"mountPoints": [],
Expand All @@ -200,19 +122,19 @@ resource "aws_ecs_task_definition" "my_task_definition" {
}

resource "aws_lb" "my_load_balancer" {
name = "${var.branch_name}"
name = var.branch_name
internal = false
idle_timeout = 300
load_balancer_type = "application"
security_groups = ["sg-07fac199a96aa3b65"] # Replace with your security group ID
subnets = ["subnet-076afb4a159204349", "subnet-0b6b9dbf631131b09"] # Replace with your subnet IDs
security_groups = [var.ecs_security_group_id]
subnets = [var.subnet_1_id, var.subnet_2_id]
}

resource "aws_lb_target_group" "ecs_target_group" {
name = "${var.branch_name}"
name = var.branch_name
port = 80
protocol = "HTTP"
vpc_id = "vpc-09c492a49b76fdf80"
vpc_id = var.vpc_id
target_type = "ip"

health_check {
Expand Down Expand Up @@ -240,7 +162,7 @@ resource "aws_lb_listener" "https_listener" {
load_balancer_arn = aws_lb.my_load_balancer.arn
port = 443
protocol = "HTTPS"
ssl_policy = "ELBSecurityPolicy-TLS13-1-2-2021-06" # Choose an appropriate SSL policy for your application
ssl_policy = "ELBSecurityPolicy-TLS13-1-2-2021-06" # Choose an appropriate SSL policy for your application

default_action {
type = "forward"
Expand All @@ -250,11 +172,11 @@ resource "aws_lb_listener" "https_listener" {
}

resource "aws_ecs_service" "my_service" {
name = "ai-backend-branch-${var.branch_name}"
cluster = "arn:aws:ecs:us-east-1:422486916789:cluster/ai"
task_definition = aws_ecs_task_definition.my_task_definition.arn
desired_count = 1
launch_type = "FARGATE"
name = "ai-backend-branch-${var.branch_name}"
cluster = "arn:aws:ecs:us-east-1:422486916789:cluster/ai"
task_definition = aws_ecs_task_definition.my_task_definition.arn
desired_count = 1
launch_type = "FARGATE"
enable_ecs_managed_tags = true
wait_for_steady_state = true

Expand All @@ -265,15 +187,15 @@ resource "aws_ecs_service" "my_service" {
}

network_configuration {
subnets = [var.subnet_1_id, var.subnet_2_id]
security_groups = [var.ecs_security_group_id]
subnets = [var.subnet_1_id, var.subnet_2_id]
security_groups = [var.ecs_security_group_id]
assign_public_ip = true
}
}

resource "aws_route53_record" "my_load_balancer_record" {
zone_id = "Z07539241TW7P7NHVR11T" # Replace with your Route 53 hosted zone ID
name = "${var.branch_name}.api.dataherald.ai" # Replace with the desired domain name
zone_id = "Z07539241TW7P7NHVR11T" # Replace with your Route 53 hosted zone ID
name = "${var.branch_name}.api.dataherald.ai" # Replace with the desired domain name
type = "A"
alias {
name = aws_lb.my_load_balancer.dns_name
Expand Down
Loading

0 comments on commit ec601c2

Please sign in to comment.