From e75a4b7824116b1d0a700b0a9eb209935a0fe048 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emanuel=20Tesa=C5=99?= Date: Sun, 19 Nov 2023 15:10:59 +0100 Subject: [PATCH 1/6] Fix docker scripts --- packages/api/package.json | 2 +- packages/pusher/package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/api/package.json b/packages/api/package.json index 2d4cfe41..296230ea 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -11,7 +11,7 @@ "clean": "rm -rf coverage dist", "dev": "nodemon --ext ts,js,json,env --exec \"pnpm ts-node src/index.ts\"", "docker:build": "docker build --target api --tag api3/signed-api:latest ../../", - "docker:start": "docker run --publish 8090:8090 -it --init --volume $(pwd)/config:/app/config --env-file .env --rm api3/signed-api:latest", + "docker:run": "docker run --publish 8090:8090 -it --init --volume $(pwd)/config:/app/config --env-file .env --rm api3/signed-api:latest", "eslint:check": "eslint . --ext .js,.ts --max-warnings 0", "eslint:fix": "eslint . --ext .js,.ts --fix", "prettier:check": "prettier --check \"./**/*.{js,ts,md,json}\"", diff --git a/packages/pusher/package.json b/packages/pusher/package.json index 3322ad45..ed7c360d 100644 --- a/packages/pusher/package.json +++ b/packages/pusher/package.json @@ -10,7 +10,7 @@ "clean": "rm -rf coverage dist", "dev": "nodemon --ext ts,js,json,env --exec \"pnpm ts-node src/index.ts\"", "docker:build": "docker build --target pusher --tag api3/pusher:latest ../../", - "docker:start": "docker run -it --init --volume $(pwd)/config:/app/config --network host --env-file .env --rm api3/pusher:latest", + "docker:run": "docker run -it --init --volume $(pwd)/config:/app/config --network host --env-file .env --rm api3/pusher:latest", "eslint:check": "eslint . --ext .js,.ts --max-warnings 0", "eslint:fix": "eslint . --ext .js,.ts --fix", "prettier:check": "prettier --check \"./**/*.{js,ts,md,yml,json}\"", From ad135a393519040e085d81754df63cf4141d4a89 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emanuel=20Tesa=C5=99?= Date: Sun, 19 Nov 2023 15:18:58 +0100 Subject: [PATCH 2/6] Implement signed API CF template --- packages/api/README.md | 2 +- packages/api/deployment/nodary.pusher.json | 100 +++++++ .../api/deployment/pusher-cf-template.json | 220 ++++++++++++++ .../deployment/signed-api-cf-template.json | 272 ++++++++++++++++++ packages/e2e/src/signed-api/signed-api.json | 10 +- packages/pusher/README.md | 3 +- 6 files changed, 600 insertions(+), 7 deletions(-) create mode 100644 packages/api/deployment/nodary.pusher.json create mode 100644 packages/api/deployment/pusher-cf-template.json create mode 100644 packages/api/deployment/signed-api-cf-template.json diff --git a/packages/api/README.md b/packages/api/README.md index 49be0e2b..0a434781 100644 --- a/packages/api/README.md +++ b/packages/api/README.md @@ -177,7 +177,7 @@ The API provides the following endpoints: ## Deployment -TODO: Write example how to deploy on AWS. +To deploy signed API on AWS you can use a CloudFormation template in the `deployment` folder. TODO: What to change To deploy on premise you can use the Docker instructions below. diff --git a/packages/api/deployment/nodary.pusher.json b/packages/api/deployment/nodary.pusher.json new file mode 100644 index 00000000..d8a2692a --- /dev/null +++ b/packages/api/deployment/nodary.pusher.json @@ -0,0 +1,100 @@ +{ + "templates": { + "0xcc35bd1800c06c12856a87311dd95bfcbb3add875844021d59a929d79f3c99bd": { + "endpointId": "0x3528e42b017a5fbf9d2993a2df04efc3ed474357575065a111b054ddf9de2acc", + "parameters": [{ "type": "string32", "name": "name", "value": "WTI/USD" }] + }, + "0x086130c54864b2129f8ac6d8d7ab819fa8181bbe676e35047b1bca4c31d51c66": { + "endpointId": "0x3528e42b017a5fbf9d2993a2df04efc3ed474357575065a111b054ddf9de2acc", + "parameters": [{ "type": "string32", "name": "name", "value": "XAG/USD" }] + }, + "0x1d65c1f1e127a41cebd2339f823d0290322c63f3044380cbac105db8e522ebb9": { + "endpointId": "0x3528e42b017a5fbf9d2993a2df04efc3ed474357575065a111b054ddf9de2acc", + "parameters": [{ "type": "string32", "name": "name", "value": "XAU/USD" }] + } + }, + "endpoints": { + "0x3528e42b017a5fbf9d2993a2df04efc3ed474357575065a111b054ddf9de2acc": { + "endpointName": "feed", + "oisTitle": "Nodary" + } + }, + "triggers": { + "signedApiUpdates": [ + { + "signedApiName": "aws-signed-api", + "templateIds": [ + "0xcc35bd1800c06c12856a87311dd95bfcbb3add875844021d59a929d79f3c99bd", + "0x086130c54864b2129f8ac6d8d7ab819fa8181bbe676e35047b1bca4c31d51c66", + "0x1d65c1f1e127a41cebd2339f823d0290322c63f3044380cbac105db8e522ebb9" + ], + "fetchInterval": 5, + "updateDelay": 30 + } + ] + }, + "signedApis": [ + { + "name": "aws-signed-api", + "url": "http://signed-api-elb-256792608.us-east-1.elb.amazonaws.com/" + } + ], + "ois": [ + { + "oisFormat": "2.2.0", + "title": "Nodary", + "version": "0.2.0", + "apiSpecifications": { + "components": { + "securitySchemes": { + "NodarySecurityScheme1ApiKey": { "in": "header", "name": "x-nodary-api-key", "type": "apiKey" } + } + }, + "paths": { + "/feed/latest": { "get": { "parameters": [{ "in": "query", "name": "name" }] } }, + "/feed/latestV2": { "get": { "parameters": [{ "in": "query", "name": "names" }] } } + }, + "servers": [{ "url": "https://api.nodary.io" }], + "security": { "NodarySecurityScheme1ApiKey": [] } + }, + "endpoints": [ + { + "fixedOperationParameters": [], + "name": "feed", + "operation": { "method": "get", "path": "/feed/latestV2" }, + "parameters": [{ "name": "name", "operationParameter": { "in": "query", "name": "names" } }], + "reservedParameters": [ + { "name": "_type", "fixed": "int256" }, + { "name": "_times", "fixed": "1000000000000000000" } + ], + "preProcessingSpecifications": [ + { + "environment": "Node", + "value": "const output = {};", + "timeoutMs": 5000 + } + ], + "postProcessingSpecifications": [ + { + "environment": "Node", + "value": "const output = input[endpointParameters.name].value;", + "timeoutMs": 5000 + } + ] + } + ] + } + ], + "apiCredentials": [ + { + "oisTitle": "Nodary", + "securitySchemeName": "NodarySecurityScheme1ApiKey", + "securitySchemeValue": "${NODARY_API_KEY}" + } + ], + "nodeSettings": { + "nodeVersion": "0.1.0", + "airnodeWalletMnemonic": "${WALLET_MNEMONIC}", + "stage": "${STAGE}" + } +} diff --git a/packages/api/deployment/pusher-cf-template.json b/packages/api/deployment/pusher-cf-template.json new file mode 100644 index 00000000..ef95a5fa --- /dev/null +++ b/packages/api/deployment/pusher-cf-template.json @@ -0,0 +1,220 @@ +{ + "AWSTemplateFormatVersion": "2010-09-09", + "Description": "A CloudFormation template for deploying the Pusher on AWS.", + "Parameters": { + "airnodeMnemonic": { + "Type": "String", + "MinLength": 10, + "Description": "Mnemonic phrase of your Airnode address." + }, + "nodaryApiKey": { + "Type": "String", + "MinLength": 10, + "Description": "Nodary API key." + } + }, + "Resources": { + "CloudWatchLogsGroup": { + "Type": "AWS::Logs::LogGroup", + "Properties": { + "LogGroupName": "PusherLogGroup", + "RetentionInDays": 7 + } + }, + "AppDefinition": { + "Type": "AWS::ECS::TaskDefinition", + "Properties": { + "NetworkMode": "awsvpc", + "Cpu": 256, + "Memory": 512, + "ExecutionRoleArn": { + "Ref": "ECSTaskRole" + }, + "RequiresCompatibilities": ["FARGATE"], + "ContainerDefinitions": [ + { + "Name": "Pusher-8feb122755282d6450e5529205e5ad8565d35f33", + "Image": "siegrift/api3-pusher:8feb122755282d6450e5529205e5ad8565d35f33", + "Environment": [ + { + "Name": "SECRETS_ENV", + "Value": { + "Fn::Join": [ + "", + [ + "WALLET_MNEMONIC", + "=", + { + "Ref": "airnodeMnemonic" + }, + "\\n", + "NODARY_API_KEY", + "=", + { + "Ref": "nodaryApiKey" + }, + "\\n", + "STAGE", + "=", + "aws-1700482577" + ] + ] + } + }, + { + "Name": "LOG_LEVEL", + "Value": "debug" + } + ], + "EntryPoint": [ + "/bin/sh", + "-c", + "echo -e $SECRETS_ENV >> ./config/secrets.env && wget -O - https://raw.githubusercontent.com/api3dao/signed-api/deployment-template/packages/api/deployment/nodary-pusher.json >> ./config/pusher.json && node dist/src/index.js" + ], + "LogConfiguration": { + "LogDriver": "awslogs", + "Options": { + "awslogs-group": { + "Ref": "CloudWatchLogsGroup" + }, + "awslogs-region": { + "Ref": "AWS::Region" + }, + "awslogs-stream-prefix": "siegrift-pusher-1700482577" + } + } + } + ] + } + }, + "AppCluster": { + "Type": "AWS::ECS::Cluster", + "Properties": { + "ClusterName": "PusherCluster-1700482577" + } + }, + "AppService": { + "Type": "AWS::ECS::Service", + "Properties": { + "Cluster": { + "Ref": "AppCluster" + }, + "ServiceName": "PusherService", + "DesiredCount": 1, + "LaunchType": "FARGATE", + "TaskDefinition": { + "Ref": "AppDefinition" + }, + "NetworkConfiguration": { + "AwsvpcConfiguration": { + "AssignPublicIp": "ENABLED", + "Subnets": [ + { + "Ref": "InstanceSubnet" + } + ] + } + }, + "DeploymentConfiguration": { + "MinimumHealthyPercent": 100, + "MaximumPercent": 200 + } + } + }, + "InstanceVPC": { + "Type": "AWS::EC2::VPC", + "Properties": { + "CidrBlock": "10.0.0.0/16", + "EnableDnsSupport": true, + "EnableDnsHostnames": true + } + }, + "InstanceInternetGateway": { + "Type": "AWS::EC2::InternetGateway" + }, + "InstanceVPCGatewayAttachment": { + "Type": "AWS::EC2::VPCGatewayAttachment", + "Properties": { + "VpcId": { + "Ref": "InstanceVPC" + }, + "InternetGatewayId": { + "Ref": "InstanceInternetGateway" + } + } + }, + "InstancePublicRouteTable": { + "Type": "AWS::EC2::RouteTable", + "Properties": { + "VpcId": { + "Ref": "InstanceVPC" + } + } + }, + "InstancePublicRoute": { + "Type": "AWS::EC2::Route", + "DependsOn": "InstanceVPCGatewayAttachment", + "Properties": { + "RouteTableId": { + "Ref": "InstancePublicRouteTable" + }, + "DestinationCidrBlock": "0.0.0.0/0", + "GatewayId": { + "Ref": "InstanceInternetGateway" + } + } + }, + "InstanceSubnet": { + "Type": "AWS::EC2::Subnet", + "Properties": { + "CidrBlock": "10.0.0.0/24", + "VpcId": { + "Ref": "InstanceVPC" + }, + "MapPublicIpOnLaunch": true + } + }, + "InstancePublicSubnet1RouteTableAssociation": { + "Type": "AWS::EC2::SubnetRouteTableAssociation", + "Properties": { + "RouteTableId": { + "Ref": "InstancePublicRouteTable" + }, + "SubnetId": { + "Ref": "InstanceSubnet" + } + } + }, + "ECSTaskRole": { + "Type": "AWS::IAM::Role", + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": { + "Effect": "Allow", + "Principal": { + "Service": ["ecs-tasks.amazonaws.com"] + }, + "Action": ["sts:AssumeRole"] + } + }, + "Policies": [ + { + "PolicyName": "ECSTaskExecutionRolePolicy", + "PolicyDocument": { + "Statement": { + "Effect": "Allow", + "Action": [ + "logs:CreateLogGroup", + "logs:CreateLogStream", + "logs:DescribeLogStreams", + "logs:PutLogEvents" + ], + "Resource": "*" + } + } + } + ] + } + } + } +} diff --git a/packages/api/deployment/signed-api-cf-template.json b/packages/api/deployment/signed-api-cf-template.json new file mode 100644 index 00000000..9907dd3b --- /dev/null +++ b/packages/api/deployment/signed-api-cf-template.json @@ -0,0 +1,272 @@ +{ + "AWSTemplateFormatVersion": "2010-09-09", + "Description": "CloudFormation template for deploying Signed API with public access", + "Outputs": { + "LoadBalancerDNS": { + "Description": "The DNS name of the load balancer", + "Value": { "Fn::GetAtt": ["ELB", "DNSName"] }, + "Export": { + "Name": { "Fn::Sub": "${AWS::StackName}-LoadBalancerDNS" } + } + } + }, + "Resources": { + "SignedApiLogsGroup": { + "Type": "AWS::Logs::LogGroup", + "Properties": { + "LogGroupName": "/ecs/signedApi", + "RetentionInDays": 7 + } + }, + "SignedApiTaskDefinition": { + "Type": "AWS::ECS::TaskDefinition", + "Properties": { + "Family": "signed-api-task", + "Cpu": "256", + "Memory": "512", + "NetworkMode": "awsvpc", + "RequiresCompatibilities": ["FARGATE"], + "ExecutionRoleArn": { "Ref": "ECSTaskRole" }, + "ContainerDefinitions": [ + { + "Name": "signed-api-container", + "Image": "siegrift/api3-signed-api:8feb122755282d6450e5529205e5ad8565d35f33", + "Environment": [ + { + "Name": "CONFIG_SOURCE", + "Value": "local" + }, + { + "Name": "LOG_LEVEL", + "Value": "debug" + } + ], + "EntryPoint": [ + "/bin/sh", + "-c", + "wget -O - https://raw.githubusercontent.com/api3dao/signed-api/deployment-template/packages/e2e/src/signed-api/signed-api.json >> ./config/signed-api.json && node dist/index.js" + ], + "PortMappings": [ + { + "ContainerPort": 80, + "HostPort": 80 + } + ], + "LogConfiguration": { + "LogDriver": "awslogs", + "Options": { + "awslogs-group": { "Ref": "SignedApiLogsGroup" }, + "awslogs-region": { "Ref": "AWS::Region" }, + "awslogs-stream-prefix": "ecs" + } + } + } + ] + } + }, + "SignedApiService": { + "Type": "AWS::ECS::Service", + "DependsOn": "SignedApiListener", + "Properties": { + "Cluster": { "Ref": "ECSCluster" }, + "LaunchType": "FARGATE", + "TaskDefinition": { "Ref": "SignedApiTaskDefinition" }, + "DesiredCount": 1, + "NetworkConfiguration": { + "AwsvpcConfiguration": { + "Subnets": [{ "Ref": "PublicSubnet1" }, { "Ref": "PublicSubnet2" }], + "SecurityGroups": [{ "Ref": "ECSSecurityGroup" }], + "AssignPublicIp": "ENABLED" + } + }, + "LoadBalancers": [ + { + "ContainerName": "signed-api-container", + "ContainerPort": 80, + "TargetGroupArn": { "Ref": "SignedApiTargetGroup" } + } + ] + } + }, + "ECSCluster": { + "Type": "AWS::ECS::Cluster", + "Properties": { + "ClusterName": "signed-api-cluster" + } + }, + "ELB": { + "Type": "AWS::ElasticLoadBalancingV2::LoadBalancer", + "Properties": { + "Name": "signed-api-elb", + "Subnets": [{ "Ref": "PublicSubnet1" }, { "Ref": "PublicSubnet2" }], + "SecurityGroups": [{ "Ref": "ELBSecurityGroup" }], + "Scheme": "internet-facing" + } + }, + "SignedApiListener": { + "Type": "AWS::ElasticLoadBalancingV2::Listener", + "Properties": { + "DefaultActions": [ + { + "Type": "forward", + "TargetGroupArn": { "Ref": "SignedApiTargetGroup" } + } + ], + "LoadBalancerArn": { "Ref": "ELB" }, + "Port": 80, + "Protocol": "HTTP" + } + }, + "SignedApiTargetGroup": { + "Type": "AWS::ElasticLoadBalancingV2::TargetGroup", + "Properties": { + "Port": 80, + "Protocol": "HTTP", + "VpcId": { "Ref": "VPC" }, + "TargetType": "ip" + } + }, + "VPC": { + "Type": "AWS::EC2::VPC", + "Properties": { + "CidrBlock": "10.0.0.0/16", + "EnableDnsSupport": "true", + "EnableDnsHostnames": "true" + } + }, + "PublicSubnet1": { + "Type": "AWS::EC2::Subnet", + "Properties": { + "VpcId": { "Ref": "VPC" }, + "CidrBlock": "10.0.1.0/24", + "AvailabilityZone": { "Fn::Select": [0, { "Fn::GetAZs": "" }] }, + "MapPublicIpOnLaunch": "true" + } + }, + "PublicSubnet2": { + "Type": "AWS::EC2::Subnet", + "Properties": { + "VpcId": { "Ref": "VPC" }, + "CidrBlock": "10.0.2.0/24", + "AvailabilityZone": { "Fn::Select": [1, { "Fn::GetAZs": "" }] }, + "MapPublicIpOnLaunch": "true" + } + }, + "InternetGateway": { + "Type": "AWS::EC2::InternetGateway" + }, + "GatewayAttachment": { + "Type": "AWS::EC2::VPCGatewayAttachment", + "Properties": { + "VpcId": { "Ref": "VPC" }, + "InternetGatewayId": { "Ref": "InternetGateway" } + } + }, + "RouteTable": { + "Type": "AWS::EC2::RouteTable", + "Properties": { + "VpcId": { "Ref": "VPC" } + } + }, + "Route": { + "Type": "AWS::EC2::Route", + "DependsOn": "GatewayAttachment", + "Properties": { + "RouteTableId": { "Ref": "RouteTable" }, + "DestinationCidrBlock": "0.0.0.0/0", + "GatewayId": { "Ref": "InternetGateway" } + } + }, + "Subnet1RouteTableAssociation": { + "Type": "AWS::EC2::SubnetRouteTableAssociation", + "Properties": { + "SubnetId": { "Ref": "PublicSubnet1" }, + "RouteTableId": { "Ref": "RouteTable" } + } + }, + "Subnet2RouteTableAssociation": { + "Type": "AWS::EC2::SubnetRouteTableAssociation", + "Properties": { + "SubnetId": { "Ref": "PublicSubnet2" }, + "RouteTableId": { "Ref": "RouteTable" } + } + }, + "ECSSecurityGroup": { + "Type": "AWS::EC2::SecurityGroup", + "Properties": { + "GroupDescription": "Security Group for ECS Tasks", + "VpcId": { "Ref": "VPC" }, + "SecurityGroupIngress": [ + { + "IpProtocol": "tcp", + "FromPort": 80, + "ToPort": 80, + "SourceSecurityGroupId": { "Ref": "ELBSecurityGroup" } + } + ] + } + }, + "ELBSecurityGroup": { + "Type": "AWS::EC2::SecurityGroup", + "Properties": { + "GroupDescription": "Security Group for ELB", + "VpcId": { "Ref": "VPC" }, + "SecurityGroupIngress": [ + { + "IpProtocol": "tcp", + "FromPort": 80, + "ToPort": 80, + "CidrIp": "0.0.0.0/0" + } + ] + } + }, + "ECSTaskRole": { + "Type": "AWS::IAM::Role", + "Properties": { + "AssumeRolePolicyDocument": { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": { + "Service": ["ecs-tasks.amazonaws.com"] + }, + "Action": ["sts:AssumeRole"] + } + ] + }, + "Path": "/", + "Policies": [ + { + "PolicyName": "ecs-service", + "PolicyDocument": { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "ec2:AuthorizeSecurityGroupIngress", + "ec2:Describe*", + "elasticloadbalancing:DeregisterInstancesFromLoadBalancer", + "elasticloadbalancing:DeregisterTargets", + "elasticloadbalancing:Describe*", + "elasticloadbalancing:RegisterInstancesWithLoadBalancer", + "elasticloadbalancing:RegisterTargets", + "ec2:CreateSecurityGroup", + "ec2:DeleteSecurityGroup", + "logs:CreateLogGroup", + "logs:CreateLogStream", + "logs:DescribeLogStreams", + "logs:PutLogEvents" + ], + "Resource": "*" + } + ] + } + } + ] + } + } + } +} diff --git a/packages/e2e/src/signed-api/signed-api.json b/packages/e2e/src/signed-api/signed-api.json index f6751e1d..1a916a25 100644 --- a/packages/e2e/src/signed-api/signed-api.json +++ b/packages/e2e/src/signed-api/signed-api.json @@ -6,13 +6,13 @@ }, { "urlPath": "/delayed", - "delaySeconds": 10 + "delaySeconds": 15 } ], + "allowedAirnodes": "*", "maxBatchSize": 10, - "port": 8090, + "port": 80, "cache": { - "maxAgeSeconds": 0 - }, - "allowedAirnodes": ["0xbF3137b0a7574563a23a8fC8badC6537F98197CC"] + "maxAgeSeconds": 300 + } } diff --git a/packages/pusher/README.md b/packages/pusher/README.md index 91b221b9..cf7b84b8 100644 --- a/packages/pusher/README.md +++ b/packages/pusher/README.md @@ -333,7 +333,8 @@ alphanumeric characters and hyphens. ## Deployment -TODO: Write example how to deploy on AWS +To deploy Pusher on AWS you can use the Cloud Formation template created by the API integrations team. The template can +be found [here](https://github.com/api3dao/api-integrations/blob/main/data/cloudformation-template.json). To deploy on premise you can use the Docker instructions below. From 19c55c67fc0278bc34c5218639b7e9df7c5ce83b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emanuel=20Tesa=C5=99?= Date: Tue, 21 Nov 2023 05:05:50 +0100 Subject: [PATCH 3/6] Generalize the template --- packages/api/README.md | 7 +- packages/api/deployment/nodary.pusher.json | 100 -------- .../api/deployment/pusher-cf-template.json | 220 ------------------ ...> signed-api-cloudformation-template.json} | 4 +- 4 files changed, 8 insertions(+), 323 deletions(-) delete mode 100644 packages/api/deployment/nodary.pusher.json delete mode 100644 packages/api/deployment/pusher-cf-template.json rename packages/api/deployment/{signed-api-cf-template.json => signed-api-cloudformation-template.json} (96%) diff --git a/packages/api/README.md b/packages/api/README.md index 0a434781..320fb53c 100644 --- a/packages/api/README.md +++ b/packages/api/README.md @@ -177,7 +177,12 @@ The API provides the following endpoints: ## Deployment -To deploy signed API on AWS you can use a CloudFormation template in the `deployment` folder. TODO: What to change +To deploy signed API on AWS you can use a CloudFormation template in the `deployment` folder. You need to specify the +docker image of the signed API and the URL of the signed API configuration which will be download when the service is +started. + +The template will create all necessary AWS resources and assign a domain name to access the the API. You can get the URL +from the output parameters of the CloudFormation stack or by checking the DNS record of the load balancer. To deploy on premise you can use the Docker instructions below. diff --git a/packages/api/deployment/nodary.pusher.json b/packages/api/deployment/nodary.pusher.json deleted file mode 100644 index d8a2692a..00000000 --- a/packages/api/deployment/nodary.pusher.json +++ /dev/null @@ -1,100 +0,0 @@ -{ - "templates": { - "0xcc35bd1800c06c12856a87311dd95bfcbb3add875844021d59a929d79f3c99bd": { - "endpointId": "0x3528e42b017a5fbf9d2993a2df04efc3ed474357575065a111b054ddf9de2acc", - "parameters": [{ "type": "string32", "name": "name", "value": "WTI/USD" }] - }, - "0x086130c54864b2129f8ac6d8d7ab819fa8181bbe676e35047b1bca4c31d51c66": { - "endpointId": "0x3528e42b017a5fbf9d2993a2df04efc3ed474357575065a111b054ddf9de2acc", - "parameters": [{ "type": "string32", "name": "name", "value": "XAG/USD" }] - }, - "0x1d65c1f1e127a41cebd2339f823d0290322c63f3044380cbac105db8e522ebb9": { - "endpointId": "0x3528e42b017a5fbf9d2993a2df04efc3ed474357575065a111b054ddf9de2acc", - "parameters": [{ "type": "string32", "name": "name", "value": "XAU/USD" }] - } - }, - "endpoints": { - "0x3528e42b017a5fbf9d2993a2df04efc3ed474357575065a111b054ddf9de2acc": { - "endpointName": "feed", - "oisTitle": "Nodary" - } - }, - "triggers": { - "signedApiUpdates": [ - { - "signedApiName": "aws-signed-api", - "templateIds": [ - "0xcc35bd1800c06c12856a87311dd95bfcbb3add875844021d59a929d79f3c99bd", - "0x086130c54864b2129f8ac6d8d7ab819fa8181bbe676e35047b1bca4c31d51c66", - "0x1d65c1f1e127a41cebd2339f823d0290322c63f3044380cbac105db8e522ebb9" - ], - "fetchInterval": 5, - "updateDelay": 30 - } - ] - }, - "signedApis": [ - { - "name": "aws-signed-api", - "url": "http://signed-api-elb-256792608.us-east-1.elb.amazonaws.com/" - } - ], - "ois": [ - { - "oisFormat": "2.2.0", - "title": "Nodary", - "version": "0.2.0", - "apiSpecifications": { - "components": { - "securitySchemes": { - "NodarySecurityScheme1ApiKey": { "in": "header", "name": "x-nodary-api-key", "type": "apiKey" } - } - }, - "paths": { - "/feed/latest": { "get": { "parameters": [{ "in": "query", "name": "name" }] } }, - "/feed/latestV2": { "get": { "parameters": [{ "in": "query", "name": "names" }] } } - }, - "servers": [{ "url": "https://api.nodary.io" }], - "security": { "NodarySecurityScheme1ApiKey": [] } - }, - "endpoints": [ - { - "fixedOperationParameters": [], - "name": "feed", - "operation": { "method": "get", "path": "/feed/latestV2" }, - "parameters": [{ "name": "name", "operationParameter": { "in": "query", "name": "names" } }], - "reservedParameters": [ - { "name": "_type", "fixed": "int256" }, - { "name": "_times", "fixed": "1000000000000000000" } - ], - "preProcessingSpecifications": [ - { - "environment": "Node", - "value": "const output = {};", - "timeoutMs": 5000 - } - ], - "postProcessingSpecifications": [ - { - "environment": "Node", - "value": "const output = input[endpointParameters.name].value;", - "timeoutMs": 5000 - } - ] - } - ] - } - ], - "apiCredentials": [ - { - "oisTitle": "Nodary", - "securitySchemeName": "NodarySecurityScheme1ApiKey", - "securitySchemeValue": "${NODARY_API_KEY}" - } - ], - "nodeSettings": { - "nodeVersion": "0.1.0", - "airnodeWalletMnemonic": "${WALLET_MNEMONIC}", - "stage": "${STAGE}" - } -} diff --git a/packages/api/deployment/pusher-cf-template.json b/packages/api/deployment/pusher-cf-template.json deleted file mode 100644 index ef95a5fa..00000000 --- a/packages/api/deployment/pusher-cf-template.json +++ /dev/null @@ -1,220 +0,0 @@ -{ - "AWSTemplateFormatVersion": "2010-09-09", - "Description": "A CloudFormation template for deploying the Pusher on AWS.", - "Parameters": { - "airnodeMnemonic": { - "Type": "String", - "MinLength": 10, - "Description": "Mnemonic phrase of your Airnode address." - }, - "nodaryApiKey": { - "Type": "String", - "MinLength": 10, - "Description": "Nodary API key." - } - }, - "Resources": { - "CloudWatchLogsGroup": { - "Type": "AWS::Logs::LogGroup", - "Properties": { - "LogGroupName": "PusherLogGroup", - "RetentionInDays": 7 - } - }, - "AppDefinition": { - "Type": "AWS::ECS::TaskDefinition", - "Properties": { - "NetworkMode": "awsvpc", - "Cpu": 256, - "Memory": 512, - "ExecutionRoleArn": { - "Ref": "ECSTaskRole" - }, - "RequiresCompatibilities": ["FARGATE"], - "ContainerDefinitions": [ - { - "Name": "Pusher-8feb122755282d6450e5529205e5ad8565d35f33", - "Image": "siegrift/api3-pusher:8feb122755282d6450e5529205e5ad8565d35f33", - "Environment": [ - { - "Name": "SECRETS_ENV", - "Value": { - "Fn::Join": [ - "", - [ - "WALLET_MNEMONIC", - "=", - { - "Ref": "airnodeMnemonic" - }, - "\\n", - "NODARY_API_KEY", - "=", - { - "Ref": "nodaryApiKey" - }, - "\\n", - "STAGE", - "=", - "aws-1700482577" - ] - ] - } - }, - { - "Name": "LOG_LEVEL", - "Value": "debug" - } - ], - "EntryPoint": [ - "/bin/sh", - "-c", - "echo -e $SECRETS_ENV >> ./config/secrets.env && wget -O - https://raw.githubusercontent.com/api3dao/signed-api/deployment-template/packages/api/deployment/nodary-pusher.json >> ./config/pusher.json && node dist/src/index.js" - ], - "LogConfiguration": { - "LogDriver": "awslogs", - "Options": { - "awslogs-group": { - "Ref": "CloudWatchLogsGroup" - }, - "awslogs-region": { - "Ref": "AWS::Region" - }, - "awslogs-stream-prefix": "siegrift-pusher-1700482577" - } - } - } - ] - } - }, - "AppCluster": { - "Type": "AWS::ECS::Cluster", - "Properties": { - "ClusterName": "PusherCluster-1700482577" - } - }, - "AppService": { - "Type": "AWS::ECS::Service", - "Properties": { - "Cluster": { - "Ref": "AppCluster" - }, - "ServiceName": "PusherService", - "DesiredCount": 1, - "LaunchType": "FARGATE", - "TaskDefinition": { - "Ref": "AppDefinition" - }, - "NetworkConfiguration": { - "AwsvpcConfiguration": { - "AssignPublicIp": "ENABLED", - "Subnets": [ - { - "Ref": "InstanceSubnet" - } - ] - } - }, - "DeploymentConfiguration": { - "MinimumHealthyPercent": 100, - "MaximumPercent": 200 - } - } - }, - "InstanceVPC": { - "Type": "AWS::EC2::VPC", - "Properties": { - "CidrBlock": "10.0.0.0/16", - "EnableDnsSupport": true, - "EnableDnsHostnames": true - } - }, - "InstanceInternetGateway": { - "Type": "AWS::EC2::InternetGateway" - }, - "InstanceVPCGatewayAttachment": { - "Type": "AWS::EC2::VPCGatewayAttachment", - "Properties": { - "VpcId": { - "Ref": "InstanceVPC" - }, - "InternetGatewayId": { - "Ref": "InstanceInternetGateway" - } - } - }, - "InstancePublicRouteTable": { - "Type": "AWS::EC2::RouteTable", - "Properties": { - "VpcId": { - "Ref": "InstanceVPC" - } - } - }, - "InstancePublicRoute": { - "Type": "AWS::EC2::Route", - "DependsOn": "InstanceVPCGatewayAttachment", - "Properties": { - "RouteTableId": { - "Ref": "InstancePublicRouteTable" - }, - "DestinationCidrBlock": "0.0.0.0/0", - "GatewayId": { - "Ref": "InstanceInternetGateway" - } - } - }, - "InstanceSubnet": { - "Type": "AWS::EC2::Subnet", - "Properties": { - "CidrBlock": "10.0.0.0/24", - "VpcId": { - "Ref": "InstanceVPC" - }, - "MapPublicIpOnLaunch": true - } - }, - "InstancePublicSubnet1RouteTableAssociation": { - "Type": "AWS::EC2::SubnetRouteTableAssociation", - "Properties": { - "RouteTableId": { - "Ref": "InstancePublicRouteTable" - }, - "SubnetId": { - "Ref": "InstanceSubnet" - } - } - }, - "ECSTaskRole": { - "Type": "AWS::IAM::Role", - "Properties": { - "AssumeRolePolicyDocument": { - "Statement": { - "Effect": "Allow", - "Principal": { - "Service": ["ecs-tasks.amazonaws.com"] - }, - "Action": ["sts:AssumeRole"] - } - }, - "Policies": [ - { - "PolicyName": "ECSTaskExecutionRolePolicy", - "PolicyDocument": { - "Statement": { - "Effect": "Allow", - "Action": [ - "logs:CreateLogGroup", - "logs:CreateLogStream", - "logs:DescribeLogStreams", - "logs:PutLogEvents" - ], - "Resource": "*" - } - } - } - ] - } - } - } -} diff --git a/packages/api/deployment/signed-api-cf-template.json b/packages/api/deployment/signed-api-cloudformation-template.json similarity index 96% rename from packages/api/deployment/signed-api-cf-template.json rename to packages/api/deployment/signed-api-cloudformation-template.json index 9907dd3b..c6701ec1 100644 --- a/packages/api/deployment/signed-api-cf-template.json +++ b/packages/api/deployment/signed-api-cloudformation-template.json @@ -30,7 +30,7 @@ "ContainerDefinitions": [ { "Name": "signed-api-container", - "Image": "siegrift/api3-signed-api:8feb122755282d6450e5529205e5ad8565d35f33", + "Image": "", "Environment": [ { "Name": "CONFIG_SOURCE", @@ -44,7 +44,7 @@ "EntryPoint": [ "/bin/sh", "-c", - "wget -O - https://raw.githubusercontent.com/api3dao/signed-api/deployment-template/packages/e2e/src/signed-api/signed-api.json >> ./config/signed-api.json && node dist/index.js" + "wget -O - >> ./config/signed-api.json && node dist/index.js" ], "PortMappings": [ { From 959f65bb49ddacace7982586f985efacd80de72a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emanuel=20Tesa=C5=99?= Date: Tue, 21 Nov 2023 05:35:51 +0100 Subject: [PATCH 4/6] Remove port configuration from signed API --- packages/api/README.md | 14 +++++----- packages/api/config/signed-api.example.json | 1 - packages/api/package.json | 6 ++--- packages/api/src/dev-server.ts | 30 +++++++++++++++++++++ packages/api/src/index.ts | 4 +-- packages/api/src/schema.ts | 1 - packages/api/src/server.ts | 11 +++++--- packages/e2e/package.json | 4 +-- packages/e2e/src/signed-api/signed-api.json | 9 +++---- 9 files changed, 55 insertions(+), 25 deletions(-) create mode 100644 packages/api/src/dev-server.ts diff --git a/packages/api/README.md b/packages/api/README.md index 320fb53c..072ac407 100644 --- a/packages/api/README.md +++ b/packages/api/README.md @@ -10,7 +10,8 @@ the data in memory and provides endpoints to push and retrieve beacon data. 1. `cp config/signed-api.example.json config/signed-api.json` - To create a config file from the example one. Optionally change the defaults. 2. `cp .env.example .env` - To copy the example environment variables. Optionally change the defaults. -3. `pnpm run dev` - To start the API server. The port number can be configured in the configuration file. +3. `pnpm run dev` - To start the API server. The port number can be configured by `DEV_SERVER_PORT` environment + variable. ### Testing @@ -134,10 +135,6 @@ The delay in seconds for the endpoint. The endpoint will only serve data that is The maximum number of signed data entries that can be inserted in one batch. This is a safety measure to prevent spamming theAPI with large payloads. The batch is rejected if it contains more entries than this value. -#### `port` - -The port on which the API is served. - #### `cache.maxAgeSeconds` The maximum age of the cache header in seconds. @@ -190,7 +187,8 @@ To deploy on premise you can use the Docker instructions below. The API is also dockerized. To run the dockerized APi, you need to: -1. Publish the port of the API to the host machine using the `--publish` flag. +1. Publish the port of the API to the host machine. The port number of signed API in the container is set to `80`. So + the command should look like `--publish :80`. 2. Mount config folder to `/app/config`. The folder should contain the `signed-api.json` file. 3. Pass the `-it --init` flags to the docker run command. This is needed to ensure the docker is stopped gracefully. See [this](https://github.com/nodejs/docker-node/blob/main/docs/BestPractices.md#handling-kernel-signals) for details. @@ -200,8 +198,8 @@ The API is also dockerized. To run the dockerized APi, you need to: For example: ```sh -# Assuming the current folder contains the "config" folder and ".env" file and the API port is 8090. -docker run --publish 8090:8090 -it --init --volume $(pwd)/config:/app/config --env-file .env --rm api3/signed-api:latest +# Assuming the current folder contains the "config" folder and ".env" file and the intended host port is 8090. +docker run --publish 8090:80 -it --init --volume $(pwd)/config:/app/config --env-file .env --rm api3/signed-api:latest ``` As of now, the docker image is not published anywhere. You need to build it locally. To build the image run: diff --git a/packages/api/config/signed-api.example.json b/packages/api/config/signed-api.example.json index dd0a91b9..7b8075b4 100644 --- a/packages/api/config/signed-api.example.json +++ b/packages/api/config/signed-api.example.json @@ -11,7 +11,6 @@ ], "allowedAirnodes": ["0x8A791620dd6260079BF849Dc5567aDC3F2FdC318"], "maxBatchSize": 10, - "port": 8090, "cache": { "maxAgeSeconds": 300 } diff --git a/packages/api/package.json b/packages/api/package.json index 296230ea..c12b981b 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -9,14 +9,14 @@ "scripts": { "build": "tsc --project tsconfig.build.json", "clean": "rm -rf coverage dist", - "dev": "nodemon --ext ts,js,json,env --exec \"pnpm ts-node src/index.ts\"", + "dev": "nodemon --ext ts,js,json,env --exec \"pnpm ts-node src/dev-server.ts\"", "docker:build": "docker build --target api --tag api3/signed-api:latest ../../", - "docker:run": "docker run --publish 8090:8090 -it --init --volume $(pwd)/config:/app/config --env-file .env --rm api3/signed-api:latest", + "docker:run": "docker run --publish 8090:80 -it --init --volume $(pwd)/config:/app/config --env-file .env --rm api3/signed-api:latest", "eslint:check": "eslint . --ext .js,.ts --max-warnings 0", "eslint:fix": "eslint . --ext .js,.ts --fix", "prettier:check": "prettier --check \"./**/*.{js,ts,md,json}\"", "prettier:fix": "prettier --write \"./**/*.{js,ts,md,json}\"", - "start-prod": "node dist/index.js", + "start-prod": "node dist/dev-server.js", "test": "jest", "tsc": "tsc --project ." }, diff --git a/packages/api/src/dev-server.ts b/packages/api/src/dev-server.ts new file mode 100644 index 00000000..1249bcd4 --- /dev/null +++ b/packages/api/src/dev-server.ts @@ -0,0 +1,30 @@ +import z from 'zod'; + +import { fetchAndCacheConfig } from './config'; +import { logger } from './logger'; +import { DEFAULT_PORT, startServer } from './server'; + +const portSchema = z.number().int().positive(); + +const startDevServer = async () => { + const config = await fetchAndCacheConfig(); + logger.info('Using configuration', config); + + const parsedPort = portSchema.safeParse(process.env.DEV_SERVER_PORT); + let port: number; + if (parsedPort.success) { + port = parsedPort.data; + logger.debug('Using DEV_SERVER_PORT environment variable as port number.', { + port, + }); + } else { + port = DEFAULT_PORT; + logger.debug('DEV_SERVER_PORT environment variable not set or invalid. Using default port.', { + port, + }); + } + + startServer(config, port); +}; + +void startDevServer(); diff --git a/packages/api/src/index.ts b/packages/api/src/index.ts index 269d9732..ab61a472 100644 --- a/packages/api/src/index.ts +++ b/packages/api/src/index.ts @@ -1,12 +1,12 @@ import { fetchAndCacheConfig } from './config'; import { logger } from './logger'; -import { startServer } from './server'; +import { DEFAULT_PORT, startServer } from './server'; const main = async () => { const config = await fetchAndCacheConfig(); logger.info('Using configuration', config); - startServer(config); + startServer(config, DEFAULT_PORT); }; void main(); diff --git a/packages/api/src/schema.ts b/packages/api/src/schema.ts index e30ea815..eb092e22 100644 --- a/packages/api/src/schema.ts +++ b/packages/api/src/schema.ts @@ -27,7 +27,6 @@ export const allowedAirnodesSchema = z.union([z.literal('*'), z.array(evmAddress export const configSchema = z.strictObject({ endpoints: endpointsSchema, maxBatchSize: z.number().nonnegative().int(), - port: z.number().nonnegative().int(), cache: z.strictObject({ maxAgeSeconds: z.number().nonnegative().int(), }), diff --git a/packages/api/src/server.ts b/packages/api/src/server.ts index 9ddfe93a..5e064944 100644 --- a/packages/api/src/server.ts +++ b/packages/api/src/server.ts @@ -4,7 +4,12 @@ import { getData, listAirnodeAddresses, batchInsertData } from './handlers'; import { logger } from './logger'; import type { Config } from './schema'; -export const startServer = (config: Config) => { +// The port number is defaulted to 80 because the service is dockerized and users can re-publish the container to any +// port they want. The CloudFormation template needs to know what is the container port so we hardcode it to 80. We +// still want the port to be configurable when running the signed API in development (by running it with node). +export const DEFAULT_PORT = 80; + +export const startServer = (config: Config, port: number) => { const app = express(); app.use(express.json()); @@ -41,7 +46,7 @@ export const startServer = (config: Config) => { }); } - app.listen(config.port, () => { - logger.info(`Server listening at http://localhost:${config.port}`); + app.listen(port, () => { + logger.info(`Server listening at http://localhost:${port}`); }); }; diff --git a/packages/e2e/package.json b/packages/e2e/package.json index 7be82a93..dc42aeb8 100644 --- a/packages/e2e/package.json +++ b/packages/e2e/package.json @@ -14,9 +14,9 @@ "prettier:fix": "prettier --write \"./**/*.{js,ts,md,yml,json}\"", "start:data-provider-api": "ts-node src/data-provider-api.ts", "start:pusher": "docker run -it --init --volume $(pwd)/src/pusher:/app/config --network host --env-file ./src/pusher/.env --rm --memory=256m api3/pusher:latest", - "start:signed-api": "docker run --publish 8090:8090 -it --init --volume $(pwd)/src/signed-api:/app/config --env-file ./src/signed-api/.env --rm --memory=256m api3/signed-api:latest", + "start:signed-api": "docker run --publish 8090:80 -it --init --volume $(pwd)/src/signed-api:/app/config --env-file ./src/signed-api/.env --rm --memory=256m api3/signed-api:latest", "start:ci:pusher": "docker run --init --volume $(pwd)/src/pusher:/app/config --network host --env-file ./src/pusher/.env --rm --memory=256m api3/pusher:latest", - "start:ci:signed-api": "docker run --publish 8090:8090 --init --volume $(pwd)/src/signed-api:/app/config --env-file ./src/signed-api/.env --rm --memory=256m api3/signed-api:latest", + "start:ci:signed-api": "docker run --publish 8090:80 --init --volume $(pwd)/src/signed-api:/app/config --env-file ./src/signed-api/.env --rm --memory=256m api3/signed-api:latest", "start:user": "ts-node src/user.ts", "test:e2e": "jest", "tsc": "tsc --project ." diff --git a/packages/e2e/src/signed-api/signed-api.json b/packages/e2e/src/signed-api/signed-api.json index 1a916a25..ff6fc5e9 100644 --- a/packages/e2e/src/signed-api/signed-api.json +++ b/packages/e2e/src/signed-api/signed-api.json @@ -6,13 +6,12 @@ }, { "urlPath": "/delayed", - "delaySeconds": 15 + "delaySeconds": 10 } ], - "allowedAirnodes": "*", "maxBatchSize": 10, - "port": 80, "cache": { - "maxAgeSeconds": 300 - } + "maxAgeSeconds": 0 + }, + "allowedAirnodes": ["0xbF3137b0a7574563a23a8fC8badC6537F98197CC"] } From 8d2389cfb6042ed28c399385df326a7a7bb18bec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emanuel=20Tesa=C5=99?= Date: Tue, 21 Nov 2023 05:56:35 +0100 Subject: [PATCH 5/6] Fix link to a private repository --- packages/pusher/README.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/packages/pusher/README.md b/packages/pusher/README.md index cf7b84b8..6a674dd2 100644 --- a/packages/pusher/README.md +++ b/packages/pusher/README.md @@ -333,8 +333,13 @@ alphanumeric characters and hyphens. ## Deployment + + To deploy Pusher on AWS you can use the Cloud Formation template created by the API integrations team. The template can -be found [here](https://github.com/api3dao/api-integrations/blob/main/data/cloudformation-template.json). +be found in the private api-integrations repository +[here](https://github.com/api3dao/api-integrations/blob/main/data/cloudformation-template.json). + + To deploy on premise you can use the Docker instructions below. From bf6d0f321a4e48b3e132e535fde12e11acfc9100 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emanuel=20Tesa=C5=99?= Date: Wed, 22 Nov 2023 19:01:12 +0100 Subject: [PATCH 6/6] Rename cloudformation template --- ...-cloudformation-template.json => cloudformation-template.json} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename packages/api/deployment/{signed-api-cloudformation-template.json => cloudformation-template.json} (100%) diff --git a/packages/api/deployment/signed-api-cloudformation-template.json b/packages/api/deployment/cloudformation-template.json similarity index 100% rename from packages/api/deployment/signed-api-cloudformation-template.json rename to packages/api/deployment/cloudformation-template.json