Skip to content

Commit

Permalink
feat: reworking ecs tasks to use aws (#724)
Browse files Browse the repository at this point in the history 
* feat: reworking ecs tasks to use aws

* feat(infra): consistent naming of updated services

* feat(infra): remove remaining awsx reference

* chore: maintain existing db version on review
  • Loading branch information
@tim-schultz
tim-schultz authored Nov 13, 2024
1 parent 130420f commit 0b144c6
Show file tree
Hide file tree
Showing 5 changed files with 213 additions and 1,040 deletions.
36 changes: 18 additions & 18 deletions infra/aws/index.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -219,12 +219,12 @@ type EcsTaskConfigurationType = {
desiredCount: number;
};

type EcsServiceNameType = "scorer-api-default" | "scorer-api-reg";
type EcsServiceNameType = "scorer-api-default-1" | "scorer-api-reg-1";
const ecsTaskConfigurations: Record<
EcsServiceNameType,
Record<StackType, EcsTaskConfigurationType>
> = {
"scorer-api-default": {
"scorer-api-default-1": {
review: {
memory: 1024,
cpu: 512,
Expand All @@ -241,7 +241,7 @@ const ecsTaskConfigurations: Record<
desiredCount: 2,
},
},
"scorer-api-reg": {
"scorer-api-reg-1": {
review: {
memory: 1024,
cpu: 512,
Expand All @@ -263,10 +263,10 @@ const ecsTaskConfigurations: Record<
if (PROVISION_STAGING_FOR_LOADTEST) {
// If we are provisioning for staging we want to have the same sizing as for production
// So we copy over the production values to the staging values in ecsTaskConfigurations
ecsTaskConfigurations["scorer-api-default"]["staging"] =
ecsTaskConfigurations["scorer-api-default"]["production"];
ecsTaskConfigurations["scorer-api-reg"]["staging"] =
ecsTaskConfigurations["scorer-api-reg"]["production"];
ecsTaskConfigurations["scorer-api-default-1"]["staging"] =
ecsTaskConfigurations["scorer-api-default-1"]["production"];
ecsTaskConfigurations["scorer-api-reg-1"]["staging"] =
ecsTaskConfigurations["scorer-api-reg-1"]["production"];
}

// This matches the default security group that awsx previously created when creating the Cluster.
Expand Down Expand Up @@ -844,22 +844,22 @@ const baseScorerServiceConfig: ScorerService = {
};

const scorerServiceDefault = createScorerECSService({
name: "scorer-api-default",
name: "scorer-api-default-1",
config: {
...baseScorerServiceConfig,
targetGroup: targetGroupDefault,
memory: ecsTaskConfigurations["scorer-api-default"][stack].memory,
cpu: ecsTaskConfigurations["scorer-api-default"][stack].cpu,
memory: ecsTaskConfigurations["scorer-api-default-1"][stack].memory,
cpu: ecsTaskConfigurations["scorer-api-default-1"][stack].cpu,
desiredCount:
ecsTaskConfigurations["scorer-api-default"][stack].desiredCount,
ecsTaskConfigurations["scorer-api-default-1"][stack].desiredCount,
},
environment: apiEnvironment,
secrets: apiSecrets,
loadBalancerAlarmThresholds: alarmConfigurations,
});

const scorerServiceRegistry = createScorerECSService({
name: "scorer-api-reg",
name: "scorer-api-reg-1",
config: {
...baseScorerServiceConfig,
listenerRulePriority: 3000,
Expand All @@ -872,9 +872,9 @@ const scorerServiceRegistry = createScorerECSService({
],

targetGroup: targetGroupRegistry,
memory: ecsTaskConfigurations["scorer-api-reg"][stack].memory,
cpu: ecsTaskConfigurations["scorer-api-reg"][stack].cpu,
desiredCount: ecsTaskConfigurations["scorer-api-reg"][stack].desiredCount,
memory: ecsTaskConfigurations["scorer-api-reg-1"][stack].memory,
cpu: ecsTaskConfigurations["scorer-api-reg-1"][stack].cpu,
desiredCount: ecsTaskConfigurations["scorer-api-reg-1"][stack].desiredCount,
},
environment: apiEnvironment,
secrets: apiSecrets,
Expand Down Expand Up @@ -1003,7 +1003,7 @@ const redashDb = new aws.rds.Instance(
allocatedStorage: 20,
maxAllocatedStorage: 30, // maxAllocatedStorage needs to be bigger than allocatedStorage
engine: "postgres",
engineVersion: "13.15",
engineVersion: stack === "review" ? "16.3" : "13.15",
instanceClass: "db.t3.micro",
dbName: redashDbName,
password: redashDbPassword,
Expand Down Expand Up @@ -1671,7 +1671,7 @@ createIndexerService(
alertTopic: pagerdutyTopic,
secretReferences: indexerSecrets,
environment: indexerEnvironment,
dockerGtcStakingIndexerImage,
indexerImage: dockerGtcStakingIndexerImage,
},
alarmConfigurations
);
Expand Down Expand Up @@ -2015,7 +2015,7 @@ const createdTask = createTask({
export const s3TriggeredECSTask = createS3InitiatedECSTask(
`bulk-score-requests-${stack}`,
cluster.arn,
createdTask.task.taskDefinition.arn,
createdTask.task.arn,
vpcPrivateSubnetIds,
[secgrp.id],
createdTask.eventsStsAssumeRole.arn
Expand Down
196 changes: 120 additions & 76 deletions infra/lib/scorer/new_service.ts
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
import { LogGroup } from "@pulumi/aws/cloudwatch/logGroup";
import { Role } from "@pulumi/aws/iam/role";
import * as awsx from "@pulumi/awsx";
import { Input, Output, interpolate } from "@pulumi/pulumi";
import { TargetGroup, ListenerRule } from "@pulumi/aws/lb";
import * as aws from "@pulumi/aws";
Expand Down Expand Up @@ -71,7 +70,7 @@ export function createScorerECSService({
environment: secretsManager.EnvironmentVar[];
secrets: pulumi.Output<secretsManager.SecretRef[]>;
loadBalancerAlarmThresholds: AlarmConfigurations;
}): awsx.ecs.FargateService | undefined {
}): aws.ecs.Service | undefined {
//////////////////////////////////////////////////////////////
// Create target group and load balancer rules
//////////////////////////////////////////////////////////////
Expand All @@ -95,65 +94,85 @@ export function createScorerECSService({
// Create the task definition and the service
//////////////////////////////////////////////////////////////

const containers: Record<
string,
awsx.types.input.ecs.TaskDefinitionContainerDefinitionArgs
> = {
scorer: {
name: "scorer",
image: config.dockerImageScorer,
memory: config.memory ? config.memory : 4096,
cpu: config.cpu ? config.cpu : 4096,
portMappings: [{ containerPort: 80, hostPort: 80, protocol: "tcp" }],
command: [
"gunicorn",
"-w",
"4",
"-k",
"uvicorn.workers.UvicornWorker",
"scorer.asgi:application",
"-b",
"0.0.0.0:80",
],
links: [],
linuxParameters: {
initProcessEnabled: true,
},
environment,
const containerDefinitions = pulumi
.all([
config.dockerImageScorer,
secrets,
},
};
environment,
config.logGroup.name,
aws.config.region,
])
.apply(
([scorerImage, secrets, environment, logGroupName, logGroupRegion]) => {
return JSON.stringify([
{
name: "scorer",
image: scorerImage,
memory: config.memory ? config.memory : 4096,
cpu: config.cpu ? config.cpu : 4096,
portMappings: [
{ containerPort: 80, hostPort: 80, protocol: "tcp" },
],
command: [
"gunicorn",
"-w",
"4",
"-k",
"uvicorn.workers.UvicornWorker",
"scorer.asgi:application",
"-b",
"0.0.0.0:80",
],
linuxParameters: {
initProcessEnabled: true,
},
environment,
secrets,
logConfiguration: {
logDriver: "awslogs",
options: {
"awslogs-group": logGroupName,
"awslogs-region": logGroupRegion,
"awslogs-stream-prefix": name,
},
},
},
]);
}
);

const service = new awsx.ecs.FargateService(name, {
name: name,
propagateTags: "TASK_DEFINITION",
const taskDefinition = new aws.ecs.TaskDefinition(name, {
family: name,
cpu: String(config.cpu ? config.cpu : 4096),
memory: String(config.memory ? config.memory : 4096),
networkMode: "awsvpc",
requiresCompatibilities: ["FARGATE"],
executionRoleArn: config.executionRole.arn,
taskRoleArn: config.taskRole.arn,
containerDefinitions,
tags: { ...defaultTags, Name: name },
});

const service = new aws.ecs.Service(name, {
name: name,
cluster: config.cluster.arn,
taskDefinition: taskDefinition.arn,
desiredCount: config.desiredCount ? config.desiredCount : 1,
launchType: "FARGATE",
propagateTags: "TASK_DEFINITION",
networkConfiguration: {
subnets: config.subnets,
securityGroups: [config.securityGroup.id],
assignPublicIp: false,
},
loadBalancers: [
{
targetGroupArn: config.targetGroup.arn,
containerName: "scorer",
containerPort: 80,
targetGroupArn: config.targetGroup.arn,
},
],
taskDefinitionArgs: {
tags: { ...defaultTags, Name: name },
logGroup: {
existing: config.logGroup,
},
executionRole: {
roleArn: config.executionRole.arn,
},
taskRole: {
roleArn: config.taskRole.arn,
},
containers,
},
tags: { ...defaultTags, Name: name },
});

function getAutoScaleMinCapacity() {
Expand All @@ -170,7 +189,7 @@ export function createScorerECSService({
tags: { ...defaultTags, Name: name },
maxCapacity: getAutoScaleMaxCapacity(),
minCapacity: getAutoScaleMinCapacity(),
resourceId: interpolate`service/${config.cluster.name}/${service.service.name}`,
resourceId: interpolate`service/${config.cluster.name}/${service.name}`,
scalableDimension: "ecs:service:DesiredCount",
serviceNamespace: "ecs",
}
Expand Down Expand Up @@ -206,7 +225,7 @@ export function createScorerECSService({
datapointsToAlarm: 1,
dimensions: {
ClusterName: config.cluster.name,
ServiceName: service.service.name,
ServiceName: service.name,
},
evaluationPeriods: 1,
metricName: "RunningTaskCount",
Expand All @@ -230,7 +249,7 @@ export function createScorerECSService({
datapointsToAlarm: 1,
dimensions: {
ClusterName: config.cluster.name,
ServiceName: service.service.name,
ServiceName: service.name,
},
evaluationPeriods: 1,
metricName: "MemoryUtilization",
Expand Down Expand Up @@ -576,7 +595,7 @@ type IndexerServiceParams = {
alertTopic: aws.sns.Topic;
secretReferences: pulumi.Output<secretsManager.SecretRef[]>;
environment: secretsManager.EnvironmentVar[];
dockerGtcStakingIndexerImage: Input<string>;
indexerImage: Input<string>;
};

export function createIndexerService(
Expand All @@ -588,7 +607,7 @@ export function createIndexerService(
alertTopic,
secretReferences,
environment,
dockerGtcStakingIndexerImage,
indexerImage,
}: IndexerServiceParams,
alarmThresholds: AlarmConfigurations
) {
Expand All @@ -597,36 +616,61 @@ export function createIndexerService(
tags: { ...defaultTags, Name: "scorer-indexer" },
});

new awsx.ecs.FargateService("scorer-staking-indexer", {
propagateTags: "TASK_DEFINITION",
cluster: cluster.arn,
desiredCount: 1,
networkConfiguration: {
subnets: privateSubnetIds,
securityGroups: [privateSubnetSecurityGroup.id],
},
taskDefinitionArgs: {
logGroup: {
existing: indexerLogGroup,
},
executionRole: {
roleArn: workerRole.arn,
},
containers: {
worker1: {
name: "indexer-process",
const containerDefinitions = pulumi
.all([
indexerImage,
secretReferences,
environment,
indexerLogGroup.name,
aws.config.region,
])
.apply(([indexerImage, secrets, environment, logGroupName, region]) => {
return JSON.stringify([
{
name: "indexer-process-1",
memory: 1024,
cpu: 512,
image: dockerGtcStakingIndexerImage,
// command: ["cargo", "run"],
image: indexerImage,
portMappings: [],
secrets: secretReferences,
secrets,
environment,
dependsOn: [],
links: [],
logConfiguration: {
logDriver: "awslogs",
options: {
"awslogs-group": logGroupName,
"awslogs-region": region,
"awslogs-stream-prefix": "scorer-staking-indexer-1",
},
},
},
},
tags: { ...defaultTags, Name: "scorer-staking-indexer" },
]);
});

const taskDefinition = new aws.ecs.TaskDefinition(
"scorer-staking-indexer-1",
{
family: "scorer-staking-indexer-1",
cpu: "512",
memory: "1024",
networkMode: "awsvpc",
requiresCompatibilities: ["FARGATE"],
executionRoleArn: workerRole.arn,
containerDefinitions,
tags: { ...defaultTags, Name: "scorer-staking-indexer-1" },
}
);

const service = new aws.ecs.Service("scorer-staking-indexer-1", {
name: "scorer-staking-indexer-1",
cluster: cluster.arn,
taskDefinition: taskDefinition.arn,
desiredCount: 1,
launchType: "FARGATE",
propagateTags: "TASK_DEFINITION",
networkConfiguration: {
subnets: privateSubnetIds,
securityGroups: [privateSubnetSecurityGroup.id],
assignPublicIp: false,
},
tags: { ...defaultTags, Name: "scorer-staking-indexer" },
});
Expand Down
Loading

0 comments on commit 0b144c6

Please sign in to comment.