Stop relying on lastAppliedConfig annotation fix #94

Currently, the provider relies on the lastAppliedConfig
annotation. The read method reads the annotation and stores
its value as the manifest in the Terraform state. This means,
Terraform only detects drift for Kubernetes resources managed
by the Kustomization provider, if there is a diff between
what's in the lastAppliedConfig annotation and what's in the
Terraform files.

Not all `kubectl` commands however update the annotation,
e.g. scale doesn't, so such drift is never corrected, even
if replicas was specified in the Terraform files.

Additionally, there are a number of issues (e.g. #136) that
although I have a hard time reproducing them reliably, I
strongly suspect to be a result of the current implementation
here too.

This change, stop relying on the annotation and instead uses
similar but reverse patching logic to the diff and upate
method to determine which attributes of the configuration in
the Terraform files / from YAML are different on the API
server. This is then stored in the state. And now drift is
determined between the values of all attributes set in
TF/YAML and what the API last returned for them.

go.mod

@@ -14,4 +14,7 @@ require (
 	sigs.k8s.io/kustomize/kyaml v0.12.0
 )
 
-require golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6 // indirect
+require (
+	github.com/evanphx/json-patch v4.11.0+incompatible
+	golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6 // indirect
+)

go.sum

@@ -143,12 +143,10 @@ github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d/go.mod h1:ZZM
 github.com/fatih/camelcase v1.0.0/go.mod h1:yN2Sb0lFhZJUdVvtELVWefmrXpuZESvPmqwoZc+/fpc=
 github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys=
 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
-github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568 h1:BHsljHzVlRcyQhjrss6TZTdY2VfCqZPbv5k3iBFa2ZQ=
 github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc=
 github.com/form3tech-oss/jwt-go v3.2.2+incompatible h1:TcekIExNqud5crz4xD2pavyTgWiPvpYe4Xau31I0PRk=
 github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
-github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
 github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
 github.com/fvbommel/sortorder v1.0.1/go.mod h1:uk88iVf1ovNn1iLfgUVU2F9o5eO30ui720w+kxuqRs0=
 github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=

kustomize/resource_kustomization.go

@@ -79,8 +79,6 @@ func kustomizationResourceCreate(d *schema.ResourceData, m interface{}) error {
 
 	namespace := u.GetNamespace()
 
-	setLastAppliedConfig(u, srcJSON)
-
 	if namespace != "" {
 		// wait for the namespace to exist
 		nsGvk := k8sschema.GroupVersionKind{
@@ -136,8 +134,6 @@ func kustomizationResourceCreate(d *schema.ResourceData, m interface{}) error {
 	id := string(resp.GetUID())
 	d.SetId(id)
 
-	d.Set("manifest", getLastAppliedConfig(resp))
-
 	return kustomizationResourceRead(d, m)
 }
 
@@ -173,7 +169,22 @@ func kustomizationResourceRead(d *schema.ResourceData, m interface{}) error {
 	id := string(resp.GetUID())
 	d.SetId(id)
 
-	d.Set("manifest", getLastAppliedConfig(resp))
+	respJSON, err := resp.MarshalJSON()
+	if err != nil {
+		return logErrorForResource(
+			u,
+			fmt.Errorf("JSON error: %s", err),
+		)
+	}
+
+	manifest, err := flattenApiResponse(u.GroupVersionKind(), respJSON, []byte(srcJSON))
+	if err != nil {
+		return logErrorForResource(
+			u,
+			fmt.Errorf("manifest flatten error: %s", err),
+		)
+	}
+	d.Set("manifest", manifest)
 
 	return nil
 }
@@ -382,8 +393,6 @@ func kustomizationResourceUpdate(d *schema.ResourceData, m interface{}) error {
 	id := string(patchResp.GetUID())
 	d.SetId(id)
 
-	d.Set("manifest", getLastAppliedConfig(patchResp))
-
 	return kustomizationResourceRead(d, m)
 }
 
@@ -494,14 +503,16 @@ func kustomizationResourceImport(d *schema.ResourceData, m interface{}) ([]*sche
 	id := string(resp.GetUID())
 	d.SetId(id)
 
-	lac := getLastAppliedConfig(resp)
-	if lac == "" {
-		return nil, logError(
-			fmt.Errorf("group: %q, kind: %q, namespace: %q, name: %q: can not import resources without %q annotation", gk.Group, gk.Kind, k.Namespace, k.Name, lastAppliedConfig),
-		)
+	respJSON, err := resp.MarshalJSON()
+	if err != nil {
+		return nil, logError(fmt.Errorf("JSON error: %s", err))
 	}
 
-	d.Set("manifest", lac)
+	manifest, err := flattenApiResponse(resp.GroupVersionKind(), respJSON, respJSON)
+	if err != nil {
+		return nil, logError(fmt.Errorf("manifest flatten error: %s", err))
+	}
+	d.Set("manifest", manifest)
 
 	return []*schema.ResourceData{d}, nil
 }

kustomize/structures_kustomization.go

@@ -1,7 +1,14 @@
 package kustomize
 
 import (
+	"fmt"
+	jsonpatch "github.com/evanphx/json-patch"
+	"k8s.io/apimachinery/pkg/util/strategicpatch"
 	"sigs.k8s.io/kustomize/api/resmap"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apimachinery/pkg/types"
+
+	"k8s.io/kubectl/pkg/scheme"
 )
 
 func flattenKustomizationIDs(rm resmap.ResMap, legacy bool) (ids []string, idsPrio [][]string, err error) {
@@ -52,3 +59,34 @@ func flattenKustomizationResources(rm resmap.ResMap, legacy bool) (res map[strin
 	}
 	return res, nil
 }
+
+func flattenApiResponse(gvk schema.GroupVersionKind, re []byte, ma []byte) (json string, err error) {
+	p, pt, err := getPatch(gvk, re, ma, ma)
+	if err != nil {
+		return json, fmt.Errorf("determining patch failed: %s", err)
+	}
+
+	var out []byte
+	switch pt {
+	case types.MergePatchType:
+		out, err = jsonpatch.MergePatch(re, p)
+		if err != nil {
+			return json, fmt.Errorf("merge patch failed: %s", err)
+		}
+	case types.StrategicMergePatchType:
+		versionedObject, err := scheme.Scheme.New(gvk)
+		if err != nil {
+			// if getPatch returns this patchType it already handled all error cases
+			return json, fmt.Errorf("unexpected error creating versionedObject: %s", err)
+		}
+
+		out, err = strategicpatch.StrategicMergePatch(re, p, versionedObject)
+		if err != nil {
+			return json, fmt.Errorf("strategic merge patch failed: %s", err)
+		}
+	}
+
+	json = string(out)
+
+	return json, nil
+}

kustomize/test_kustomizations/_example_app/ingress.yaml

@@ -9,6 +9,7 @@ spec:
   - http:
       paths:
       - path: /testpath
+        pathType: ImplementationSpecific
         backend:
           serviceName: test
           servicePort: 80

kustomize/util.go

@@ -4,9 +4,7 @@ import (
 	"context"
 	"fmt"
 	"runtime"
-	"strings"
 
-	k8scorev1 "k8s.io/api/core/v1"
 	k8serrors "k8s.io/apimachinery/pkg/api/errors"
 	k8smetav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	k8sunstructured "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
@@ -20,21 +18,6 @@ import (
 	"k8s.io/kubectl/pkg/scheme"
 )
 
-const lastAppliedConfig = k8scorev1.LastAppliedConfigAnnotation
-
-func setLastAppliedConfig(u *k8sunstructured.Unstructured, srcJSON string) {
-	annotations := u.GetAnnotations()
-	if len(annotations) == 0 {
-		annotations = make(map[string]string)
-	}
-	annotations[lastAppliedConfig] = srcJSON
-	u.SetAnnotations(annotations)
-}
-
-func getLastAppliedConfig(u *k8sunstructured.Unstructured) string {
-	return strings.TrimRight(u.GetAnnotations()[lastAppliedConfig], "\r\n")
-}
-
 func getOriginalModifiedCurrent(originalJSON string, modifiedJSON string, currentAllowNotFound bool, m interface{}) (original []byte, modified []byte, current []byte, err error) {
 	client := m.(*Config).Client
 	mapper := m.(*Config).Mapper
@@ -48,9 +31,6 @@ func getOriginalModifiedCurrent(originalJSON string, modifiedJSON string, curren
 		return nil, nil, nil, err
 	}
 
-	setLastAppliedConfig(o, originalJSON)
-	setLastAppliedConfig(n, modifiedJSON)
-
 	mapping, err := mapper.RESTMapping(n.GroupVersionKind().GroupKind(), n.GroupVersionKind().Version)
 	if err != nil {
 		return nil, nil, nil, err

kustomize/util_test.go

@@ -7,26 +7,6 @@ import (
 	"k8s.io/apimachinery/pkg/types"
 )
 
-func TestLastAppliedConfig(t *testing.T) {
-	srcJSON := "{\"apiVersion\": \"v1\", \"kind\": \"Namespace\", \"metadata\": {\"name\": \"test-unit\"}}"
-	u, err := parseJSON(srcJSON)
-	if err != nil {
-		t.Errorf("Error: %s", err)
-	}
-	setLastAppliedConfig(u, srcJSON)
-
-	annotations := u.GetAnnotations()
-	count := len(annotations)
-	if count != 1 {
-		t.Errorf("TestLastAppliedConfig: incorrect number of annotations, got: %d, want: %d.", count, 1)
-	}
-
-	lac := getLastAppliedConfig(u)
-	if lac != srcJSON {
-		t.Errorf("TestLastAppliedConfig: incorrect annotation value, got: %s, want: %s.", srcJSON, lac)
-	}
-}
-
 func TestGetPatchStrategicMergePatch1(t *testing.T) {
 	o, _ := parseJSON(testGetPatchStrategicMergePatch1OriginalJSON)
 	m, _ := parseJSON(testGetPatchStrategicMergePatch1ModifiedJSON)