diff --git a/cdk.json b/cdk.json index 30aab4a3..0a8d6dc1 100644 --- a/cdk.json +++ b/cdk.json @@ -1,3 +1,32 @@ { - "app": "npx ts-node dist/lib/common/default-main.js" -} \ No newline at end of file + "app": "npx ts-node dist/lib/common/default-main.js", + "context": { + "conformitron.amp.endpoint": "https://aps-workspaces..amazonaws.com/workspaces//", + "conformitron.amp.arn":"arn:aws:aps:::workspace/", + "conformitron.amg.endpoint": "", + "conformitron.version": ["1.28","1.29","1.30"] + }, + "fluxRepository": { + "name": "grafana-dashboards", + "namespace": "grafana-operator", + "repository": { + "repoUrl": "https://github.com/aws-observability/aws-observability-accelerator", + "name": "grafana-dashboards", + "targetRevision": "main", + "path": "./artifacts/grafana-operator-manifests/eks/infrastructure" + }, + "values": { + "GRAFANA_CLUSTER_DASH_URL" : "https://raw.githubusercontent.com/aws-observability/aws-observability-accelerator/main/artifacts/grafana-dashboards/eks/infrastructure/cluster.json", + "GRAFANA_KUBELET_DASH_URL" : "https://raw.githubusercontent.com/aws-observability/aws-observability-accelerator/main/artifacts/grafana-dashboards/eks/infrastructure/kubelet.json", + "GRAFANA_NSWRKLDS_DASH_URL" : "https://raw.githubusercontent.com/aws-observability/aws-observability-accelerator/main/artifacts/grafana-dashboards/eks/infrastructure/namespace-workloads.json", + "GRAFANA_NODEEXP_DASH_URL" : "https://raw.githubusercontent.com/aws-observability/aws-observability-accelerator/main/artifacts/grafana-dashboards/eks/infrastructure/nodeexporter-nodes.json", + "GRAFANA_NODES_DASH_URL" : "https://raw.githubusercontent.com/aws-observability/aws-observability-accelerator/main/artifacts/grafana-dashboards/eks/infrastructure/nodes.json", + "GRAFANA_WORKLOADS_DASH_URL" : "https://raw.githubusercontent.com/aws-observability/aws-observability-accelerator/main/artifacts/grafana-dashboards/eks/infrastructure/workloads.json" + }, + "kustomizations": [ + { + "kustomizationPath": "./artifacts/grafana-operator-manifests/eks/infrastructure" + } + ] + } +} diff --git a/docs/patterns/multi-cluster-conformitron.md b/docs/patterns/multi-cluster-conformitron.md index 790dffd2..92315e33 100644 --- a/docs/patterns/multi-cluster-conformitron.md +++ b/docs/patterns/multi-cluster-conformitron.md @@ -8,6 +8,8 @@ 1. Enable all the available [EKS Anywhere Addons](https://github.com/aws-samples/eks-anywhere-addons), on each of the clusters, essentially testing their compatibility across all the potential architecture/version available today on AWS. +To learn more about our EKS Addon validation checkout our [blog](https://aws.amazon.com/blogs/containers/conformitron-validate-third-party-software-with-amazon-eks-and-amazon-eks-anywhere/) + ### GitOps confguration GitOps is a branch of DevOps that focuses on using Git code repositories to manage infrastructure and application code deployments. @@ -56,10 +58,56 @@ AWS_REGION=$(aws configure get region) Amazon Elastic Compute Cloud (Amazon EC2) | EC2-VPC Elastic IPs | 30 ``` -1. Amazon Managed Grafana Workspace: To visualize metrics collected, you need an Amazon Managed Grafana workspace. If you have an existing workspace, create an environment variable `AMG_ENDPOINT_URL` as described below. To create a new workspace, visit [our supporting example for Grafana](https://aws-observability.github.io/terraform-aws-observability-accelerator/helpers/managed-grafana/) +1. Amazon Managed Grafana Workspace: To visualize metrics collected, you need an Amazon Managed Grafana workspace. If you have an existing workspace, create environment variables `AMG_ENDPOINT_URL` as described below. To create a new workspace, visit [our supporting example for Grafana](https://aws-observability.github.io/terraform-aws-observability-accelerator/helpers/managed-grafana/) ```bash export AMG_ENDPOINT_URL=https://g-xxx.grafana-workspace.region.amazonaws.com +export AMG_WORKSPACE_ID=g-xxx + +``` + +1. Grafana API Key: Amazon Managed Grafana provides a control plane API for generating Grafana API keys or Service Account Tokens. This allows programatic provisioning of Grafana dashboards using the EKS grafana operator. + +=== "v10.4 & v9.4 workspaces" + + ```bash + # IMPORTANT NOTE: skip this command if you already have a service token + GRAFANA_SA_ID=$(aws grafana create-workspace-service-account \ + --workspace-id $AMG_WORKSPACE_ID \ + --grafana-role ADMIN \ + --name cdk-accelerator-eks \ + --query 'id' \ + --output text) + + # creates a new token + export AMG_API_KEY=$(aws grafana create-workspace-service-account-token \ + --workspace-id $AMG_WORKSPACE_ID \ + -name "grafana-operator-key" \ + --seconds-to-live 432000 \ + --service-account-id $GRAFANA_SA_ID \ + --query 'serviceAccountToken.key' \ + --output text) + ``` + +=== "v8.4 workspaces" + + ```bash + export AMG_API_KEY=$(aws grafana create-workspace-api-key \ + --key-name "grafana-operator-key" \ + --key-role "ADMIN" \ + --seconds-to-live 432000 \ + --workspace-id $AMG_WORKSPACE_ID \ + --query key \ + --output text) + ``` + +1. AWS SSM Parameter Store for GRAFANA API KEY: Update the Grafana API key secret in AWS SSM Parameter Store using the above new Grafana API key. This will be referenced by Grafana Operator deployment of our solution to access Amazon Managed Grafana from Amazon EKS Cluster + +```bash +aws ssm put-parameter --name "/grafana-api-key" \ + --type "SecureString" \ + --value $AMG_API_KEY \ + --region $AWS_REGION ``` 1. Amazon Managed Prometheus Workspace: To store observability metrics from all clusters we will use Amazon Managed Prometheus due to it's ease of setup and easy integration with other AWS services. We recommend setting up a new seperate Prometheus workspace using the commands below. @@ -97,7 +145,30 @@ cat << EOF > cdk.json "conformitron.amp.arn":"arn:aws:aps:${AWS_REGION}:${ACCOUNT_ID}:workspace/${AMP_WS_ID}", "conformitron.amg.endpoint": "${AMG_ENDPOINT_URL}", "conformitron.version": ["1.28","1.29","1.30"] - } + }, + "fluxRepository": { + "name": "grafana-dashboards", + "namespace": "grafana-operator", + "repository": { + "repoUrl": "https://github.com/aws-observability/aws-observability-accelerator", + "name": "grafana-dashboards", + "targetRevision": "main", + "path": "./artifacts/grafana-operator-manifests/eks/infrastructure" + }, + "values": { + "GRAFANA_CLUSTER_DASH_URL" : "https://raw.githubusercontent.com/aws-observability/aws-observability-accelerator/main/artifacts/grafana-dashboards/eks/infrastructure/cluster.json", + "GRAFANA_KUBELET_DASH_URL" : "https://raw.githubusercontent.com/aws-observability/aws-observability-accelerator/main/artifacts/grafana-dashboards/eks/infrastructure/kubelet.json", + "GRAFANA_NSWRKLDS_DASH_URL" : "https://raw.githubusercontent.com/aws-observability/aws-observability-accelerator/main/artifacts/grafana-dashboards/eks/infrastructure/namespace-workloads.json", + "GRAFANA_NODEEXP_DASH_URL" : "https://raw.githubusercontent.com/aws-observability/aws-observability-accelerator/main/artifacts/grafana-dashboards/eks/infrastructure/nodeexporter-nodes.json", + "GRAFANA_NODES_DASH_URL" : "https://raw.githubusercontent.com/aws-observability/aws-observability-accelerator/main/artifacts/grafana-dashboards/eks/infrastructure/nodes.json", + "GRAFANA_WORKLOADS_DASH_URL" : "https://raw.githubusercontent.com/aws-observability/aws-observability-accelerator/main/artifacts/grafana-dashboards/eks/infrastructure/workloads.json" + }, + "kustomizations": [ + { + "kustomizationPath": "./artifacts/grafana-operator-manifests/eks/infrastructure" + } + ] + } } EOF ``` @@ -114,7 +185,7 @@ Now you can go to [AWS CodePipeline console](https://eu-west-1.console.aws.amazo # SSM Cost Optimizations for conformitron clusters -Running all the clusters for 24 hours results in a daily spend of $300+ +Running all the clusters by default for 24 hours results in a daily spend of $300+ To minimize these costs we have written a systems manager automation which automatically scales down autoscaling group to 0 desired nodes during off-business hours. @@ -124,4 +195,4 @@ On weekends clusters stay scaled to 0. These optimizations bring down the weekly cost to less than 1000$ essentially for a more than 60% cost savings. -Please find the SSM Automation documents `lib/multi-cluster-construct/CostOptimizationSSM/ScaleDownEKStoZero.yml` and `lib/multi-cluster-construct/CostOptimizationSSM/ScaleUpEKStoOne.yml`in this directory. They are triggered by event bridge on the con schedule specified above. \ No newline at end of file +Please find the SSM Automation documents `lib/multi-cluster-construct/resources/cost-optimization/scaleDownEksToZero.yml` and `lib/multi-cluster-construct/resources/cost-optimization/scaleUpEksToOne.yml`. They are triggered by event bridge on the cron schedule specified above. \ No newline at end of file diff --git a/lib/multi-cluster-construct/grafana-monitor-builder.ts b/lib/multi-cluster-construct/grafana-monitor-builder.ts index 0a366c56..506c3fc4 100644 --- a/lib/multi-cluster-construct/grafana-monitor-builder.ts +++ b/lib/multi-cluster-construct/grafana-monitor-builder.ts @@ -32,13 +32,13 @@ export class GrafanaMonitoringConstruct { ampRules: { ampWorkspaceArn: ampWorkspaceArn, ruleFilePaths: [ - __dirname + '/../common/resources/amp-config/alerting-rules.yml', - __dirname + '/../common/resources/amp-config/recording-rules.yml' + __dirname + '/resources/amp-config/alerting-rules.yml', + __dirname + '/resources/amp-config/recording-rules.yml' ] } }; - let doc = blueprints.utils.readYamlDocument(__dirname + '/../common/resources/otel-collector-config.yml'); + let doc = blueprints.utils.readYamlDocument(__dirname + '/resources/otel-collector-config.yml'); doc = blueprints.utils.changeTextBetweenTokens( doc, "{{ start enableJavaMonJob }}", @@ -89,10 +89,10 @@ export class GrafanaMonitoringConstruct { true ); - fs.writeFileSync(__dirname + '/../common/resources/otel-collector-config-new.yml', doc); + fs.writeFileSync(__dirname + '/resources/otel-collector-config-new.yml', doc); ampAddOnProps.openTelemetryCollector = { - manifestPath: __dirname + '/../common/resources/otel-collector-config-new.yml', + manifestPath: __dirname + '/resources/otel-collector-config-new.yml', manifestParameterMap: { logGroupName: `/aws/eks/conformitron/workspace`, logStreamName: `$NODE_NAME`, diff --git a/lib/multi-cluster-construct/grafana-operator-secret-addon.ts b/lib/multi-cluster-construct/grafana-operator-secret-addon.ts index a06d358e..28ce3fe9 100644 --- a/lib/multi-cluster-construct/grafana-operator-secret-addon.ts +++ b/lib/multi-cluster-construct/grafana-operator-secret-addon.ts @@ -61,7 +61,7 @@ export class GrafanaOperatorSecretAddon implements blueprints.ClusterAddOn { { secretKey: "GF_SECURITY_ADMIN_APIKEY", remoteRef: { - key: "/cdk-accelerator/grafana-api-key" + key: "/grafana-api-key" }, }, ], diff --git a/lib/multi-cluster-construct/multi-cluster-builder.ts b/lib/multi-cluster-construct/multi-cluster-builder.ts index 4bcbd663..479ce49e 100644 --- a/lib/multi-cluster-construct/multi-cluster-builder.ts +++ b/lib/multi-cluster-construct/multi-cluster-builder.ts @@ -29,7 +29,7 @@ export default class MultiClusterBuilderConstruct { }; ampAddOnProps.openTelemetryCollector = { - manifestPath: __dirname + '/../common/resources/otel-collector-config-new.yml', + manifestPath: __dirname + '/resources/otel-collector-config-new.yml', manifestParameterMap: { logGroupName: `/aws/eks/conformitron/cluster`, logStreamName: `$NODE_NAME`, diff --git a/lib/multi-cluster-construct/pipeline.ts b/lib/multi-cluster-construct/pipeline.ts index d30eff48..8746fe29 100644 --- a/lib/multi-cluster-construct/pipeline.ts +++ b/lib/multi-cluster-construct/pipeline.ts @@ -1,5 +1,6 @@ import * as blueprints from '@aws-quickstart/eks-blueprints'; import * as eks from 'aws-cdk-lib/aws-eks'; +import * as ec2 from 'aws-cdk-lib/aws-ec2'; import { Construct } from 'constructs'; import MultiClusterBuilderConstruct from './multi-cluster-builder'; import { GrafanaMonitoringConstruct } from './grafana-monitor-builder'; @@ -36,14 +37,16 @@ export class PipelineMultiCluster { Similar to approach in multi-region-construct pattern */ - const clusterProps = this.getClusterProps(); + let clusterProps; for(const version of CLUSTER_VERSIONS) { const blueprintBuilderX86 = new MultiClusterBuilderConstruct().create(scope, accountID, region); - // let clusterProps = this.getClusterProps() - clusterProps.amiType = clusterMappings[ClusterName.X86]!.amiType; - clusterProps.instanceTypes = [clusterMappings[ClusterName.X86]!.instanceType]; + clusterProps = this.buildClusterProps( + clusterMappings[ClusterName.X86]!.amiType, + clusterMappings[ClusterName.X86]!.instanceType + ); + const blueprintX86 = blueprintBuilderX86 .version(version) .clusterProvider(new blueprints.MngClusterProvider(clusterProps)) @@ -54,10 +57,11 @@ export class PipelineMultiCluster { stackBuilder : blueprintX86.clone(region) }); - // clusterProps = this.getClusterProps() const blueprintBuilderArm = new MultiClusterBuilderConstruct().create(scope, accountID, region); - clusterProps.amiType = clusterMappings[ClusterName.ARM]!.amiType; - clusterProps.instanceTypes = [clusterMappings[ClusterName.ARM]!.instanceType]; + clusterProps = this.buildClusterProps( + clusterMappings[ClusterName.ARM]!.amiType, + clusterMappings[ClusterName.ARM]!.instanceType + ); const blueprintARM = blueprintBuilderArm .version(version) .clusterProvider(new blueprints.MngClusterProvider(clusterProps)) @@ -74,9 +78,11 @@ export class PipelineMultiCluster { const blueprintBuilderBrX86= new MultiClusterBuilderConstruct().create(scope, accountID, region); - // let clusterProps = this.getClusterProps() - clusterProps.amiType = clusterMappings[ClusterName.BR_X86]!.amiType; - clusterProps.instanceTypes = [clusterMappings[ClusterName.BR_X86]!.instanceType]; + clusterProps = this.buildClusterProps( + clusterMappings[ClusterName.BR_X86]!.amiType, + clusterMappings[ClusterName.BR_X86]!.instanceType + ); + const blueprintBrX86 = blueprintBuilderBrX86 .version(LATEST_VERSION) .clusterProvider(new blueprints.MngClusterProvider(clusterProps)) @@ -89,9 +95,11 @@ export class PipelineMultiCluster { const blueprintBuilderBrArm = new MultiClusterBuilderConstruct().create(scope, accountID, region); - // clusterProps = this.getClusterProps() - clusterProps.amiType = clusterMappings[ClusterName.BR_ARM]!.amiType; - clusterProps.instanceTypes = [clusterMappings[ClusterName.BR_ARM]!.instanceType]; + clusterProps = this.buildClusterProps( + clusterMappings[ClusterName.BR_ARM]!.amiType, + clusterMappings[ClusterName.BR_ARM]!.instanceType + ); + const blueprintBottleRocketArm = blueprintBuilderBrArm .version(LATEST_VERSION) .clusterProvider(new blueprints.MngClusterProvider(clusterProps)) @@ -127,13 +135,14 @@ export class PipelineMultiCluster { } }); } - - getClusterProps() { + buildClusterProps(amiType:eks.NodegroupAmiType,instanceType:ec2.InstanceType) : blueprints.MngClusterProviderProps{ let clusterProps : blueprints.MngClusterProviderProps = { maxSize : 2, minSize : 1, desiredSize: 1, - diskSize: 100 + diskSize: 100, + amiType: amiType, + instanceTypes:[instanceType] }; return clusterProps; } diff --git a/lib/common/resources/amp-config/alerting-rules.yml b/lib/multi-cluster-construct/resources/amp-config/alerting-rules.yml similarity index 100% rename from lib/common/resources/amp-config/alerting-rules.yml rename to lib/multi-cluster-construct/resources/amp-config/alerting-rules.yml diff --git a/lib/common/resources/amp-config/apiserver/recording-rules.yml b/lib/multi-cluster-construct/resources/amp-config/apiserver/recording-rules.yml similarity index 100% rename from lib/common/resources/amp-config/apiserver/recording-rules.yml rename to lib/multi-cluster-construct/resources/amp-config/apiserver/recording-rules.yml diff --git a/lib/common/resources/amp-config/istio/alerting-rules.yml b/lib/multi-cluster-construct/resources/amp-config/istio/alerting-rules.yml similarity index 100% rename from lib/common/resources/amp-config/istio/alerting-rules.yml rename to lib/multi-cluster-construct/resources/amp-config/istio/alerting-rules.yml diff --git a/lib/common/resources/amp-config/istio/recording-rules.yml b/lib/multi-cluster-construct/resources/amp-config/istio/recording-rules.yml similarity index 100% rename from lib/common/resources/amp-config/istio/recording-rules.yml rename to lib/multi-cluster-construct/resources/amp-config/istio/recording-rules.yml diff --git a/lib/common/resources/amp-config/java/alerting-rules.yml b/lib/multi-cluster-construct/resources/amp-config/java/alerting-rules.yml similarity index 100% rename from lib/common/resources/amp-config/java/alerting-rules.yml rename to lib/multi-cluster-construct/resources/amp-config/java/alerting-rules.yml diff --git a/lib/common/resources/amp-config/java/recording-rules.yml b/lib/multi-cluster-construct/resources/amp-config/java/recording-rules.yml similarity index 100% rename from lib/common/resources/amp-config/java/recording-rules.yml rename to lib/multi-cluster-construct/resources/amp-config/java/recording-rules.yml diff --git a/lib/common/resources/amp-config/nginx/alerting-rules.yml b/lib/multi-cluster-construct/resources/amp-config/nginx/alerting-rules.yml similarity index 100% rename from lib/common/resources/amp-config/nginx/alerting-rules.yml rename to lib/multi-cluster-construct/resources/amp-config/nginx/alerting-rules.yml diff --git a/lib/common/resources/amp-config/recording-rules.yml b/lib/multi-cluster-construct/resources/amp-config/recording-rules.yml similarity index 100% rename from lib/common/resources/amp-config/recording-rules.yml rename to lib/multi-cluster-construct/resources/amp-config/recording-rules.yml diff --git a/lib/multi-cluster-construct/CostOptimizationSSM/ScaleDownEKStoZero.yml b/lib/multi-cluster-construct/resources/cost-optimization/scaleDownEksToZero.yml similarity index 100% rename from lib/multi-cluster-construct/CostOptimizationSSM/ScaleDownEKStoZero.yml rename to lib/multi-cluster-construct/resources/cost-optimization/scaleDownEksToZero.yml diff --git a/lib/multi-cluster-construct/CostOptimizationSSM/ScaleUpEKStoOne.yml b/lib/multi-cluster-construct/resources/cost-optimization/scaleUpEksToOne.yml similarity index 100% rename from lib/multi-cluster-construct/CostOptimizationSSM/ScaleUpEKStoOne.yml rename to lib/multi-cluster-construct/resources/cost-optimization/scaleUpEksToOne.yml diff --git a/lib/common/resources/otel-collector-config-new.yml b/lib/multi-cluster-construct/resources/otel-collector-config-new.yml similarity index 100% rename from lib/common/resources/otel-collector-config-new.yml rename to lib/multi-cluster-construct/resources/otel-collector-config-new.yml diff --git a/lib/common/resources/otel-collector-config.yml b/lib/multi-cluster-construct/resources/otel-collector-config.yml similarity index 100% rename from lib/common/resources/otel-collector-config.yml rename to lib/multi-cluster-construct/resources/otel-collector-config.yml