feat: Generate secrets from templating

README.md

@@ -271,6 +271,99 @@ secretFrom:
 EOF
 ```
 
+## Generate secret from templates
+
+`KSOPS` can generate a Kubernetes Secret by templating via [golang text/template](https://pkg.go.dev/text/template).
+
+This is useful if you want advanced templating to generate a config file,
+or you want to partially encrypt a file format not supported by sops.
+This could also come in handy when you do not want to specify every unencrypted fields
+in `.sops.yaml` for each sops secret you have.
+
+Values inside `secretFromTemplate.template.stringData` and `secretFromTemplate.template.data` are considered template strings.
+Each template string is templated using variables read from `secretFromTemplate.vars`.
+
+#### Create a Kubernetes Secret from a template
+
+Let's say you have the following config file, and you want to mask only the `password` field in it.
+
+```bash
+cat <<EOF > config.yaml
+cleartext_field: "foo"
+# You want to mask only the password field
+password: "super-secret-password"
+EOF
+```
+
+This is possible with the following configuration.
+
+```bash
+cat <<EOF > vars.env
+password=super-secret-password
+EOF
+sops -e vars.env > vars.enc.env
+
+cat <<EOF > secret-generator.yaml
+apiVersion: viaduct.ai/v1
+kind: ksops
+metadata:
+  name: example-secret-generator
+  annotations:
+    config.kubernetes.io/function: |
+        exec:
+          path: ksops
+secretFromTemplate:
+- template:
+    metadata:
+      name: secret-name
+    type: Opaque
+    stringData:
+      config.yaml: |
+        cleartext_field: "foo"
+        password: "{{ .password }}"
+  vars:
+    envs:
+    - ./vars.enc.env
+EOF
+```
+
+#### Create a Kubernetes Secret from a file using template
+
+You can store stringData inside a file.
+
+```bash
+cat <<EOF > vars.env
+password=super-secret-password
+EOF
+sops -e vars.env > vars.enc.env
+
+cat <<EOF >config-template.yaml
+cleartext_field: "foo"
+password: "{{ .password }}"
+EOF
+
+cat <<EOF > secret-generator.yaml
+apiVersion: viaduct.ai/v1
+kind: ksops
+metadata:
+  name: example-secret-generator
+  annotations:
+    config.kubernetes.io/function: |
+        exec:
+          path: ksops
+secretFromTemplate:
+- template:
+    metadata:
+      name: secret-name
+    files:
+    - ./config-template.yaml
+    - config.yaml=./config-template.yaml
+  vars:
+    envs:
+    - ./secret.enc.env
+EOF
+```
+
 ## Generator Options
 
 `KSOPS` supports kustomize annotation based generator options. At the time of writing, the supported annotations are:

ksops.go

@@ -12,16 +12,16 @@ import (
 	"bytes"
 	"encoding/base64"
 	"fmt"
-	"os"
-	"path/filepath"
-	"strings"
-
 	"github.com/GoogleContainerTools/kpt-functions-sdk/go/fn"
 	"github.com/getsops/sops/v3/cmd/sops/formats"
 	"github.com/getsops/sops/v3/decrypt"
 	"github.com/joho/godotenv"
+	"os"
+	"path/filepath"
 	"sigs.k8s.io/kustomize/api/types"
 	"sigs.k8s.io/yaml"
+	"strings"
+	"text/template"
 )
 
 type kubernetesSecret struct {
@@ -41,9 +41,28 @@ type secretFrom struct {
 	Type        string           `json:"type,omitempty" yaml:"type,omitempty"`
 }
 
+type templateDef struct {
+	Metadata   types.ObjectMeta  `json:"metadata,omitempty" yaml:"metadata,omitempty"`
+	Type       string            `json:"type,omitempty" yaml:"type,omitempty"`
+	StringData map[string]string `json:"stringData,omitempty" yaml:"stringData,omitempty"`
+	Data       map[string]string `json:"data,omitempty" yaml:"data,omitempty"`
+	Files      []string          `json:"files,omitempty" yaml:"files,omitempty"`
+}
+
+type templateVars struct {
+	Files []string `json:"files,omitempty" yaml:"files,omitempty"`
+	Envs  []string `json:"envs,omitempty" yaml:"envs,omitempty"`
+}
+
+type secretFromTemplate struct {
+	Template templateDef  `json:"template,omitempty" yaml:"template,omitempty"`
+	Vars     templateVars `json:"vars,omitempty" yaml:"vars,omitempty"`
+}
+
 type ksops struct {
-	Files      []string     `json:"files,omitempty" yaml:"files,omitempty"`
-	SecretFrom []secretFrom `json:"secretFrom,omitempty" yaml:"secretFrom,omitempty"`
+	Files              []string             `json:"files,omitempty" yaml:"files,omitempty"`
+	SecretFrom         []secretFrom         `json:"secretFrom,omitempty" yaml:"secretFrom,omitempty"`
+	SecretFromTemplate []secretFromTemplate `json:"secretFromTemplate,omitempty" yaml:"secretFromTemplate,omitempty"`
 }
 
 func help() {
@@ -138,26 +157,22 @@ func generate(raw []byte) (string, error) {
 		return "", fmt.Errorf("error unmarshalling manifest content: %q \n%s", err, raw)
 	}
 
-	if manifest.Files == nil && manifest.SecretFrom == nil {
-		return "", fmt.Errorf("missing the required 'files' or 'secretFrom' key in the ksops manifests: %s", raw)
+	if manifest.Files == nil && manifest.SecretFrom == nil && manifest.SecretFromTemplate == nil {
+		return "", fmt.Errorf("missing the required 'files', 'secretFrom', or 'secretFromTemplate' key in the ksops manifests: %s", raw)
 	}
 
-	var output bytes.Buffer
+	var documents [][]byte
 
-	for i, file := range manifest.Files {
+	for _, file := range manifest.Files {
 		data, err := decryptFile(file)
 		if err != nil {
 			return "", fmt.Errorf("error decrypting file %q from manifest.Files: %w", file, err)
 		}
 
-		output.Write(data)
-		// KRM treats will try parse (and fail) empty documents if there is a trailing separator
-		if i < (len(manifest.Files)+len(manifest.SecretFrom))-1 {
-			output.WriteString("\n---\n")
-		}
+		documents = append(documents, data)
 	}
 
-	for i, secretFrom := range manifest.SecretFrom {
+	for _, secretFrom := range manifest.SecretFrom {
 		stringData := make(map[string]string)
 		binaryData := make(map[string]string)
 
@@ -208,16 +223,106 @@ func generate(raw []byte) (string, error) {
 		if err != nil {
 			return "", fmt.Errorf("error marshalling manifest: %w", err)
 		}
-		output.WriteString(string(d))
-		// KRM treats will try parse (and fail) empty documents if there is a trailing separator
-		if i < len(manifest.SecretFrom)-1 {
-			output.WriteString("---\n")
+		documents = append(documents, d)
+	}
+
+	for _, secretFrom := range manifest.SecretFromTemplate {
+		// Read template
+		if secretFrom.Template.StringData == nil {
+			secretFrom.Template.StringData = make(map[string]string)
+		}
+		for _, file := range secretFrom.Template.Files {
+			key, path := fileKeyPath(file)
+			data, err := os.ReadFile(path)
+			if err != nil {
+				return "", fmt.Errorf("error reading file %q from secretFrom.template.files: %w", path, err)
+			}
+			secretFrom.Template.StringData[key] = string(data)
+		}
+
+		// Read variables
+		vars := make(map[string]string)
+		for _, file := range secretFrom.Vars.Files {
+			key, path := fileKeyPath(file)
+			data, err := decryptFile(path)
+			if err != nil {
+				return "", fmt.Errorf("error decrypting file %q from secretFromTemplate.vars.files: %w", path, err)
+			}
+
+			vars[key] = string(data)
+		}
+		for _, file := range secretFrom.Vars.Envs {
+			data, err := decryptFile(file)
+			if err != nil {
+				return "", fmt.Errorf("error decrypting file %q from secretFromTemplate.vars.envs: %w", file, err)
+			}
+
+			env, err := godotenv.Unmarshal(string(data))
+			if err != nil {
+				return "", fmt.Errorf("error unmarshalling .env file %q: %w", file, err)
+			}
+			for k, v := range env {
+				vars[k] = v
+			}
+		}
+
+		// Template stringData and data fields
+		stringData := make(map[string]string, len(secretFrom.Template.StringData))
+		binaryData := make(map[string]string, len(secretFrom.Template.Data))
+		for k, v := range secretFrom.Template.StringData {
+			stringData[k], err = execTemplate(v, vars)
+			if err != nil {
+				return "", fmt.Errorf("error templating from secretFromTemplate.template.stringData.%v: %w", k, err)
+			}
+		}
+		for k, v := range secretFrom.Template.Data {
+			binaryData[k], err = execTemplate(v, vars)
+			if err != nil {
+				return "", fmt.Errorf("error executing template from secretFrom.template.data.%v: %w", k, err)
+			}
+		}
+
+		// Construct secret document
+		s := kubernetesSecret{
+			APIVersion: "v1",
+			Kind:       "Secret",
+			Metadata:   secretFrom.Template.Metadata,
+			Type:       secretFrom.Template.Type,
+			StringData: stringData,
+			Data:       binaryData,
+		}
+		d, err := yaml.Marshal(&s)
+		if err != nil {
+			return "", fmt.Errorf("error marshalling manifest: %w", err)
+		}
+		documents = append(documents, d)
+	}
+
+	var output bytes.Buffer
+	for i, doc := range documents {
+		output.Write(doc)
+		// Note that KRM treats will try parse (and fail) empty documents if there is a trailing separator
+		if i != len(documents)-1 {
+			output.WriteString("\n---\n")
 		}
 	}
 
 	return output.String(), nil
 }
 
+func execTemplate(tmpl string, vars map[string]string) (string, error) {
+	t, err := template.New("tmpl").Parse(tmpl)
+	if err != nil {
+		return "", fmt.Errorf("error parsing template: %w", err)
+	}
+	var buf bytes.Buffer
+	err = t.Execute(&buf, vars)
+	if err != nil {
+		return "", fmt.Errorf("error executing template: %w", err)
+	}
+	return buf.String(), nil
+}
+
 func decryptFile(file string) ([]byte, error) {
 	b, err := os.ReadFile(file)
 	if err != nil {

ksops_test.go

@@ -115,6 +115,14 @@ func TestKSOPSPluginInstallation(t *testing.T) {
 			name: "KRM Secret Metadata",
 			dir:  "test/krm/metadata",
 		},
+		{
+			name: "KRM Secret from Template",
+			dir:  "test/krm/template",
+		},
+		{
+			name: "KRM Secret from Template using File",
+			dir:  "test/krm/template-file",
+		},
 	}
 
 	// run kustomize version to validate installation

test/krm/template-file/config-template.yaml

@@ -0,0 +1,4 @@
+cleartext_field: "foo"
+user: "{{ .username }}"
+pass: "{{ .password }}"
+role: "{{ .role }}"

test/krm/template-file/generate-resources.yaml

@@ -0,0 +1,23 @@
+apiVersion: viaduct.ai/v1
+kind: ksops
+metadata:
+  name: ksops-secret-from-generator
+  annotations:
+    config.kubernetes.io/function: |
+        exec:
+          # if the binary is your PATH, you can do 
+          path: ksops
+          # otherwise, path should be relative to manifest files, like
+          # path: ../../../ksops
+secretFromTemplate:
+- template:
+    metadata:
+      name: mysecret
+    type: Opaque
+    files:
+    - config.yaml=./config-template.yaml
+  vars:
+    files:
+    - role=./secret.enc.txt
+    envs:
+    - ./secret.enc.env

test/krm/template-file/kustomization.yaml

@@ -0,0 +1,2 @@
+generators:
+  - ./generate-resources.yaml

test/krm/template-file/secret.enc.env

@@ -0,0 +1,9 @@
+password=ENC[AES256_GCM,data:POoodkvLJlYkuFwZ,iv:zAqcWK57VQr59EkJ3Bx6utY1wexj1+zDZasl76fzMho=,tag:1z4RJr+7BTeYMs3P7WFKGw==,type:str]
+username=ENC[AES256_GCM,data:Cpxi7uQ=,iv:qEmlgTHiOsEF83VR7sranL4uuvS/EsF9Udlt9ykcGd0=,tag:80Rg/s0jsvefbSwjgIcUTQ==,type:str]
+sops_version=3.7.2
+sops_lastmodified=2022-12-14T18:15:04Z
+sops_unencrypted_regex=^(apiVersion|metadata|kind|type)$
+sops_mac=ENC[AES256_GCM,data:KHJWDwHp9AjlQhXmOOIIOs3zsiIA4UkE5O9ue81qffkUIEvG0ts8RqSI0JetdxvqFN8jMAwgxKZlslsViMxTU9dD9h3WCXCteh6wcBxSSNLbuAnxAlZke2tR10ZCWizGA3oBcTcd/maN+hZq5fNqBMzdUpqyhmJZ9/OV9w6CNiU=,iv:wpUpAzsyEcUByWU9Km2gfiTyCE3RQjvkbW5EV/7OZ80=,tag:I4PMdJiPJ63l0KDmWFZmwg==,type:str]
+sops_pgp__list_0__map_enc=-----BEGIN PGP MESSAGE-----\n\nhQEMAyUpShfNkFB/AQgAkk0vfCLQKkP8tDXe5Bmu4s1bndkH5YyyNlUgOTDeCzDx\nkYGv8lXwCpGBlc2RQxzB4Ygr8k80M76IGZHDWrDLuNkNuKriPZQPP4OIpbOSGnHs\nQmLvJpICisZjUbo6gQpHi5iRZ3GiGFyX386bASDJbDM6FKft6MBarXApt1LYfL9z\no83MK4ZYVR0Nh4ttUXwDbug2de9hIGmnnXCWOQ7XLAlK+TT6/qdocAksTb+7Te1l\n8lQz/Slq1vyctOChZG/m2Vc6w6Ux0c01BOUB7uCcMnygndbHkgbsMBw6pru3Dv8V\n+Kpb40+3rY+u1dXediYV62vX48SUv6XPChUlAK2k1NJeAdd0s0ukU6thdloWjKZa\n7KH2RGDS7D7khRP5dyIQYf1DiLEUNfG/+J6zgU7DJep05mOvoapRp/vBHGTssjtj\nHjjSa4/kV89Brm6mPRbCnOj4HHWrP/0lKfecmEsBPg==\n=h8SP\n-----END PGP MESSAGE-----\n
+sops_pgp__list_0__map_fp=FBC7B9E2A4F9289AC0C1D4843D16CEE4A27381B4
+sops_pgp__list_0__map_created_at=2022-12-14T18:14:54Z

test/krm/template-file/secret.enc.txt

@@ -0,0 +1,21 @@
+{
+	"data": "ENC[AES256_GCM,data:KFazMWP2kQ==,iv:NW0McgCBG0SyUFrhbOJMA2JuEZMNjnvhebc4rG99Fwc=,tag:jCMY9g28qXgxgtlBS2/MiQ==,type:str]",
+	"sops": {
+		"kms": null,
+		"gcp_kms": null,
+		"azure_kv": null,
+		"hc_vault": null,
+		"age": null,
+		"lastmodified": "2024-10-20T05:30:27Z",
+		"mac": "ENC[AES256_GCM,data:ot7nHiAZQ4m9t6UXo7xVjDYQaXOtnwG7DVpdyevP5GijOyVhJlE4+DBMzgSI19CEXkZZk5v6KvRdjO0LcyAfVLaf/9+jadc5EB9CNwZ8mjs0gahXzW00VOKbxiWMjmVfJuKjJKqvTk+N/C6Ps0wqtDW5NsS+MLym5Zd2vHw3aU4=,iv:w3ctBqJvwSnLZkl6K3MrdfGSh/pRz/gT6MlgT+j6ljo=,tag:r1oUCDA9fFraMoBK270niA==,type:str]",
+		"pgp": [
+			{
+				"created_at": "2024-10-20T05:30:26Z",
+				"enc": "-----BEGIN PGP MESSAGE-----\n\nhQEMAyUpShfNkFB/AQgAgLTzmwsYMyrKZjZAx/g3XiMMAbbz5oAg5M6ovUVc/0jA\nckXInXVz683Ej0437DINxeOyT455MuRqTrJfUd1BdZErnu2O/5ApimUl14XMP9K8\nEmNd9+C2lbH2UmGuMezsGNTtdurMjYTzuTPBnufuc2J53eE9xs/xZ0StAuLig+X0\na2q/aul0iTkasr0VX5wrmYEsaHQD+VZvurSoGaCjv4qq3IGMitT5PtijCKLYroEX\nptYMMgchJTy+hI75snMrMhfnucHCjJtnDEJweGAV9CXxbtDuJq1r4zIAsNMODJ9H\nw975TMu3xBJN3vXNUPUe4zBxsX5GlCaqqHBqF5awKdJeAYwC7RCe1gMw5jmYvyqW\nvnooICVWf7gu6IVYSSEY007jsRolM+QfdZ8mWu5YKRtAjMwLP8RGSRggB++YrBMp\nPpE3RWpvIN06OIoEfUFukd0wwNjcRWynWKtQhlEFVA==\n=drlB\n-----END PGP MESSAGE-----\n",
+				"fp": "FBC7B9E2A4F9289AC0C1D4843D16CEE4A27381B4"
+			}
+		],
+		"unencrypted_regex": "^(apiVersion|metadata|kind|type)$",
+		"version": "3.7.3"
+	}
+}

test/krm/template-file/want.yaml

@@ -0,0 +1,11 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: mysecret
+stringData:
+  config.yaml: |
+    cleartext_field: "foo"
+    user: "admin"
+    pass: "1f2d1e2e67df"
+    role: "foo-bar"
+type: Opaque