[IT-3471] Create cloudwatch metrics (#25)

Send beanstalk metrics to cloudwatch so that we can monitor beanstalk request errors.  This feature requires enhanced Beanstalk health checks[1] so we change the template always enable it and then set a ConfigDocument[2][3] to enabled the metric.

[1] https://docs.aws.amazon.com/elasticbeanstalk/latest/dg/command-options-general.html#command-options-general-elasticbeanstalkhealthreporting
[2] https://docs.aws.amazon.com/elasticbeanstalk/latest/dg/health-enhanced-rules.html#health-enhanced-rules.configdocument
[3] https://docs.aws.amazon.com/elasticbeanstalk/latest/dg/health-enhanced-cloudwatch.html#health-enhanced-cloudwatch-configdocument

config/EnhancedHealthRulesConfigDocument.json

@@ -0,0 +1,48 @@
+{
+  "Rules": {
+    "Environment": {
+      "Application": {
+        "ApplicationRequests4xx": {
+          "Enabled": false
+        },
+        "ApplicationRequests5xx": {
+          "Enabled": false
+        }
+      },
+      "ELB": {
+        "ELBRequests4xx": {
+          "Enabled": true
+        },
+        "ELBRequests5xx": {
+          "Enabled": true
+        }
+      }
+    }
+  },
+  "CloudWatchMetrics": {
+    "Environment": {
+      "ApplicationLatencyP99.9": 60,
+      "InstancesSevere": 60,
+      "ApplicationLatencyP90": 60,
+      "ApplicationLatencyP99": 60,
+      "ApplicationLatencyP95": 60,
+      "InstancesUnknown": 60,
+      "ApplicationLatencyP85": 60,
+      "InstancesInfo": 60,
+      "ApplicationRequests2xx": 60,
+      "InstancesDegraded": 60,
+      "InstancesWarning": 60,
+      "ApplicationLatencyP50": 60,
+      "ApplicationRequestsTotal": 60,
+      "InstancesNoData": 60,
+      "InstancesPending": 60,
+      "ApplicationLatencyP10": 60,
+      "ApplicationRequests5xx": 60,
+      "ApplicationLatencyP75": 60,
+      "InstancesOk": 60,
+      "ApplicationRequests3xx": 60,
+      "ApplicationRequests4xx": 60
+    }
+  },
+  "Version": 1
+}

config/agoradev/develop/agora.yaml

@@ -13,7 +13,6 @@ parameters:
   AppHealthcheckUrl: '/about'
   AutoScalingMaxSize: '4'
   AutoScalingMinSize: '2'
-  EbHealthReportingSystem: 'enhanced'
   EbDeploymentPolicy: 'RollingWithAdditionalBatch'
   SnsBounceNotificationEndpoint: '[email protected]'
   SnsNotificationEndpoint: '[email protected]'
@@ -24,3 +23,4 @@ parameters:
   MongodbAccessSecurityGroup: !stack_output_external agora-docdb-develop::DocumentDbAccessSecurityGroup
   VpcName: 'agoravpc'
   CertificateArn: !stack_output_external agora-certificate::SSLCertificate
+  EnhancedHealthRulesConfigDocument: !file_contents config/EnhancedHealthRulesConfigDocument.json

config/agoraprod/prod/agora.yaml

@@ -13,7 +13,6 @@ parameters:
   AppHealthcheckUrl: '/about'
   AutoScalingMaxSize: '6'
   AutoScalingMinSize: '3'
-  EbHealthReportingSystem: 'enhanced'
   EbDeploymentPolicy: 'RollingWithAdditionalBatch'
   SnsBounceNotificationEndpoint: '[email protected]'
   SnsNotificationEndpoint: '[email protected]'
@@ -24,3 +23,4 @@ parameters:
   MongodbAccessSecurityGroup: !stack_output_external agora-docdb-prod::DocumentDbAccessSecurityGroup
   VpcName: 'agoravpc-prod'
   CertificateArn: !stack_output_external agora-certificate::SSLCertificate
+  EnhancedHealthRulesConfigDocument: !file_contents config/EnhancedHealthRulesConfigDocument.json

config/agoraprod/staging/agora.yaml

@@ -13,7 +13,6 @@ parameters:
   AppHealthcheckUrl: '/about'
   AutoScalingMaxSize: '4'
   AutoScalingMinSize: '2'
-  EbHealthReportingSystem: 'enhanced'
   EbDeploymentPolicy: 'RollingWithAdditionalBatch'
   SnsBounceNotificationEndpoint: '[email protected]'
   SnsNotificationEndpoint: '[email protected]'
@@ -24,3 +23,4 @@ parameters:
   MongodbAccessSecurityGroup: !stack_output_external agora-docdb-staging::DocumentDbAccessSecurityGroup
   VpcName: 'agoravpc-staging'
   CertificateArn: !stack_output_external agora-certificate::SSLCertificate
+  EnhancedHealthRulesConfigDocument: !file_contents config/EnhancedHealthRulesConfigDocument.json

templates/beanstalk.yaml

@@ -24,13 +24,6 @@ Parameters:
       - true
       - false
     ConstraintDescription: 'Valid values are true or false'
-  EbHealthReportingSystem:
-    Type: String
-    Default: basic
-    AllowedValues:
-      - basic
-      - enhanced
-    ConstraintDescription: must be either basic or enhanced
   SnsBounceNotificationEndpoint:
     Type: String
     Description: Email address for SNS bounce notifications
@@ -86,6 +79,9 @@ Parameters:
   CertificateArn:
     Description: ARN of the SSL Certificate
     Type: String
+  EnhancedHealthRulesConfigDocument:
+    Description: Configuration enhanced health rules (https://docs.aws.amazon.com/elasticbeanstalk/latest/dg/health-enhanced-rules.html#health-enhanced-rules.configdocument)
+    Type: String
 Resources:
   S3AppDeployBucket:
     Type: 'AWS::S3::Bucket'
@@ -285,7 +281,10 @@ Resources:
           Value: 'HTTP'
         - Namespace: 'aws:elasticbeanstalk:healthreporting:system'
           OptionName: SystemType
-          Value: !Ref EbHealthReportingSystem
+          Value: 'enhanced'
+        - Namespace: 'aws:elasticbeanstalk:healthreporting:system'
+          OptionName: ConfigDocument
+          Value: !Ref EnhancedHealthRulesConfigDocument
         - Namespace: 'aws:elasticbeanstalk:hostmanager'
           OptionName: LogPublicationControl
           Value: 'true'