Merge pull request #125 from appvia/develop

[RELEASE] Release v0.1.7

.github/workflows/ci.yaml

@@ -19,7 +19,7 @@ jobs:
         with:
           go-version: '1.18.1'
       - name: Unit Tests
-        run: make check
+        run: make test
 
   dependencies:
     runs-on: ubuntu-latest
@@ -33,14 +33,18 @@ jobs:
   linting:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/setup-go@v3
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Setup Go
+        uses: actions/setup-go@v2
         with:
           go-version: '1.18.1'
-      - uses: actions/checkout@v3
-      - name: golangci-lint
-        uses: golangci/golangci-lint-action@v3
-        with:
-          args: --timeout 2m
+      - name: Linting
+        run: |
+          make check-gofmt
+          make shfmt
+          make spelling
+          make golangci-lint
 
   helm:
     runs-on: ubuntu-latest

Makefile

@@ -113,9 +113,8 @@ step: golang
 
 test:
 	@echo "--> Running the tests"
-	@rm -rf cover.out
-	@mkdir -p ./test/results
-	@go run ./vendor/gotest.tools/gotestsum/main.go --format pkgname -- -coverprofile=cover.out `go list ./... | egrep -v /test/crdtests/`
+	@rm -f cover.out || true
+	@go run ./vendor/gotest.tools/gotestsum/main.go --format pkgname -- -coverprofile=cover.out ./...
 	@echo "--> Coverage: $(shell go tool cover -func=cover.out | grep total | grep -Eo '[0-9]+\.[0-9]+')" || true
 
 ### IMAGES ###
@@ -228,12 +227,6 @@ check: test
 	@$(MAKE) spelling
 	@$(MAKE) golangci-lint
 
-verify-circleci:
-	@echo "--> Verifying the circleci config"
-	@docker run -ti --rm -v ${PWD}:/workspace \
-		-w /workspace circleci/circleci-cli \
-		circleci config validate
-
 ### UTILITIES ###
 
 clean:

charts/terraform-controller/Chart.yaml

@@ -3,7 +3,7 @@ apiVersion: v2
 name: terraform-controller
 description: Controller used to provision a terraform workflow within kubernetes
 type: application
-version: v0.2.3
+version: v0.2.4
 appVersion: v0.1.6
 sources:
   - https://github.com/appvia/terraform-controller

charts/terraform-controller/crds/terraform.appvia.io_configurations.yaml

@@ -21,12 +21,12 @@ spec:
         - jsonPath: .spec.writeConnectionSecretToRef.name
           name: Secret
           type: string
-        - jsonPath: .status.resources
-          name: Resources
-          type: string
         - jsonPath: .status.costs.monthly
           name: Estimated
           type: string
+        - jsonPath: .status.resourceStatus
+          name: Synchronized
+          type: string
         - jsonPath: .metadata.creationTimestamp
           name: Age
           type: date
@@ -47,7 +47,7 @@ spec:
               description: ConfigurationSpec defines the desired state of a terraform
               properties:
                 auth:
-                  description: SCMAuth is used to configure any options required when the source of the terraform module is private or requires credentials to retrieve. This could be SSH keys or git user/pass or AWS credentials for an s3 bucket.
+                  description: Auth is used to configure any options required when the source of the terraform module is private or requires credentials to retrieve. This could be SSH keys or git user/pass or AWS credentials for an s3 bucket.
                   properties:
                     name:
                       description: name is unique within a namespace to reference a secret resource.
@@ -59,6 +59,9 @@ spec:
                 enableAutoApproval:
                   description: EnableAutoApproval when enabled indicates the configuration does not need to be manually approved. On a change to the configuration, the controller will automatically approve the configuration. Note it still needs to adhere to any checks or policies.
                   type: boolean
+                enableDriftDetection:
+                  description: EnableDriftDetection when enabled run periodic reconciliation configurations looking for any drift between the expected and current state. If any drift is detected the status is changed and a kubernetes event raised.
+                  type: boolean
                 module:
                   description: Module is the URL to the source of the terraform module. The format of the URL is a direct implementation of terraform's module reference. Please see the following repository for more details https://github.com/hashicorp/go-getter
                   type: string
@@ -73,7 +76,6 @@ spec:
                       type: string
                   required:
                     - name
-                    - namespace
                   type: object
                 terraformVersion:
                   description: TerraformVersion provides the ability to override the default terraform version. Before changing this field its best to consult with platform administrator. As the value of this field is used to change the tag of the terraform container image.
@@ -188,6 +190,9 @@ spec:
                       description: Monthly is the monthly estimated cost of the configuration
                       type: string
                   type: object
+                driftTimestamp:
+                  description: DriftTimestamp is the timestamp of the last drift detection
+                  type: string
                 lastReconcile:
                   description: LastReconcile describes the generation and time of the last reconciliation
                   properties:
@@ -212,9 +217,15 @@ spec:
                       format: date-time
                       type: string
                   type: object
+                resourceStatus:
+                  description: ResourceStatus indicates the status of the resources and if the resources are insync with the configuration
+                  type: string
                 resources:
                   description: Resources is the number of managed cloud resources which are currently under management. This field is taken from the terraform state itself.
                   type: integer
+                terraformVersion:
+                  description: TerraformVersion is the version of terraform which was last used to run this configuration
+                  type: string
               type: object
           type: object
       served: true

charts/terraform-controller/crds/terraform.appvia.io_providers.yaml

@@ -14,7 +14,7 @@ spec:
     listKind: ProviderList
     plural: providers
     singular: provider
-  scope: Namespaced
+  scope: Cluster
   versions:
     - additionalPrinterColumns:
         - jsonPath: .spec.source

charts/terraform-controller/templates/deployment.yaml

@@ -56,6 +56,9 @@ spec:
             {{- if .Values.controller.costs.secret }}
             - --cost-secret={{ .Values.controller.costs.secret }}
             {{- end }}
+            - --drift-controller-interval={{ .Values.controller.driftControllerInterval }}
+            - --drift-interval={{ .Values.controller.driftInterval }}
+            - --drift-threshold={{ .Values.controller.driftThreshold }}
             - --enable-terraform-versions={{ .Values.controller.enableTerraformVersions }}
             - --enable-watchers={{ .Values.controller.enableWatchers }}
             - --enable-webhook={{ .Values.controller.webhooks.enabled }}

charts/terraform-controller/values.yaml

@@ -32,6 +32,17 @@ controller:
     # The terraform image used when running jobs
     executor: quay.io/appvia/terraform-executor:v0.1.6
 
+  # driftInterval is the minimum time to check for drift
+  driftInterval: 5h
+  # driftThreshold is the percentage of configurations which are permitted
+  # to run a drift detection at any one time
+  driftThreshold: 0.10
+  # driftControllerInterval is the interval the controller will use to requeue. On every iteration
+  # the configurations are checked and anyone who's last plan occurred longer than the driftInterval
+  # is up for a drift trigger. Its fine to have this low, it's the driftInterval and threshold which
+  # ultimately effective jobs running to check drift.
+  driftControllerInterval: 5m
+
   # Allows you to overload the templates
   templates:
     # is the name of config map holding a override to the job template

cmd/controller/main.go

@@ -56,7 +56,10 @@ func main() {
 	flags.BoolVar(&config.EnableTerraformVersions, "enable-terraform-versions", true, "Indicates the terraform version can be overridden by configurations")
 	flags.BoolVar(&config.EnableWatchers, "enable-watchers", true, "Indicates we create watcher jobs in the configuration namespaces")
 	flags.BoolVar(&config.EnableWebhook, "enable-webhook", true, "Indicates we should register the webhooks")
-	flags.DurationVar(&config.ResyncPeriod, "resync-period", 1*time.Hour, "The resync period for the controller")
+	flags.DurationVar(&config.ResyncPeriod, "resync-period", 5*time.Hour, "The resync period for the controller")
+	flags.DurationVar(&config.DriftControllerInterval, "drift-controller-interval", 5*time.Minute, "Is the check interval for the controller to search for configurations which should be checked for drift")
+	flags.DurationVar(&config.DriftInterval, "drift-interval", 3*time.Hour, "The minimum duration the controller will wait before triggering a drift check")
+	flags.Float64Var(&config.DriftThreshold, "drift-threshold", 0.10, "The maximum percentage of configurations that can be run drift detection at any one time")
 	flags.IntVar(&config.APIServerPort, "apiserver-port", 10080, "The port the apiserver should be listening on")
 	flags.IntVar(&config.MetricsPort, "metrics-port", 9090, "The port the metric endpoint binds to")
 	flags.IntVar(&config.WebhookPort, "webhooks-port", 10081, "The port the webhook endpoint binds to")

examples/configuration.yaml

@@ -8,7 +8,6 @@ spec:
   module: https://github.com/terraform-aws-modules/terraform-aws-s3-bucket.git?ref=v3.1.0
 
   providerRef:
-    namespace: terraform-system
     name: aws
 
   writeConnectionSecretToRef:

examples/database.yaml

@@ -7,7 +7,6 @@ spec:
   module: github.com/terraform-aws-modules/terraform-aws-rds-aurora
 
   providerRef:
-    namespace: default
     name: aws
 
   writeConnectionSecretToRef: