diff --git a/.github/workflows/docker-ci.yaml b/.github/workflows/docker-ci.yaml index 5148fe8..61c1355 100644 --- a/.github/workflows/docker-ci.yaml +++ b/.github/workflows/docker-ci.yaml @@ -47,6 +47,7 @@ jobs: - name: Login to Docker Hub uses: docker/login-action@v3 + if: ${{ github.ref_name == 'main' }} with: username: ${{ vars.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} @@ -68,4 +69,4 @@ jobs: with: username: ${{ vars.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - repository: ${{ vars.DOCKERHUB_USERNAME }}/swarm-cd \ No newline at end of file + repository: ${{ vars.DOCKERHUB_USERNAME }}/swarm-cd diff --git a/README.md b/README.md index a5e3fd5..f648395 100644 --- a/README.md +++ b/README.md @@ -57,7 +57,6 @@ docker stack deploy --compose-file docker-compose.yaml swarm-cd This will start SwarmCD, it will periodically check the stack repo for new changes, pulling them and updating the stack. - ## Manage Encrypted Secrets Using SOPS You can use [sops](https://github.com/getsops/sops) to encrypt secrets in git repos and @@ -113,6 +112,19 @@ secrets: This way, SwarmCD will decrypt the files each time before it updates the stack. +### Automatic SOPS secrets detection + +Instead of specifying the paths of every single secrets you need to decrypt, +you can use the `sops_secrets_discovery: true` option: + +- in the `config.yaml` file to enable it globally +- in the `stacks.yaml` file for the individual stacks. + +Please note that: + +- if the global setting is set to `true`, it ignores individual stacks overrides. +- if the stack-level setting is set to `true`, it ignores the `sops_files` setting altogether. + ## Connect SwarmCD to a remote docker socket You can use the `DOCKER_HOST` environment variable to point SwarmCD to a remote docker socket, diff --git a/docs/README.md b/docs/README.md index 5ece8f8..66753d5 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1,6 +1,7 @@ # Documentation -Here you can find configuration file references for +Here you can find configuration file references for: + - [repos.yaml](repos.yaml) - [stacks.yaml](stacks.yaml) -- [config.yaml](config.yaml) \ No newline at end of file +- [config.yaml](config.yaml) diff --git a/docs/config.yaml b/docs/config.yaml index abe2ec7..6c70361 100644 --- a/docs/config.yaml +++ b/docs/config.yaml @@ -7,6 +7,9 @@ update_interval: 120 # The path where SwarmCD will checkout repos repos_path: repos/ +# Automatically detect secrets to decrypt with SOPS +sops_secrets_discovery: true + # Automatically rotate configs and secrets # when the change. Adds a hash to config # and secret names @@ -19,4 +22,3 @@ repos: # You can define stacks here instead of # defining a separate stacks.yaml file stacks: - diff --git a/docs/stacks.yaml b/docs/stacks.yaml index 524d2f0..720ea53 100644 --- a/docs/stacks.yaml +++ b/docs/stacks.yaml @@ -22,4 +22,6 @@ stack-name: # before updating stack sops_files: - path/to/sops/encrypted/file - + # Enable the automatic secret discovery + # alternative to sops_files + sops_secrets_discovery: false diff --git a/swarmcd/init.go b/swarmcd/init.go index ba48ca0..e17940d 100644 --- a/swarmcd/init.go +++ b/swarmcd/init.go @@ -19,7 +19,6 @@ type StackStatus struct { RepoURL string } - var config *util.Config = &util.Configs var logger *slog.Logger = util.Logger @@ -44,7 +43,7 @@ func Init() (err error) { return } -func initRepos() (error) { +func initRepos() error { for repoName, repoConfig := range config.RepoConfigs { repoPath := path.Join(config.ReposPath, repoName) auth, err := createHTTPBasicAuth(repoName) @@ -98,7 +97,8 @@ func initStacks() error { if !ok { return fmt.Errorf("error initializing %s stack, no such repo: %s", stack, stackConfig.Repo) } - swarmStack := newSwarmStack(stack, stackRepo, stackConfig.Branch, stackConfig.ComposeFile, stackConfig.SopsFiles, stackConfig.ValuesFile) + discoverSecrets := config.SopsSecretsDiscovery || stackConfig.SopsSecretsDiscovery + swarmStack := newSwarmStack(stack, stackRepo, stackConfig.Branch, stackConfig.ComposeFile, stackConfig.SopsFiles, stackConfig.ValuesFile, discoverSecrets) stacks = append(stacks, swarmStack) stackStatus[stack] = &StackStatus{} stackStatus[stack].RepoURL = stackRepo.url diff --git a/swarmcd/stack.go b/swarmcd/stack.go index b1d71c4..2b11584 100644 --- a/swarmcd/stack.go +++ b/swarmcd/stack.go @@ -1,6 +1,7 @@ package swarmcd import ( + "bytes" "crypto/md5" "fmt" "log/slog" @@ -14,22 +15,24 @@ import ( ) type swarmStack struct { - name string - repo *stackRepo - branch string - composePath string - sopsFiles []string - valuesFile string + name string + repo *stackRepo + branch string + composePath string + sopsFiles []string + valuesFile string + discoverSecrets bool } -func newSwarmStack(name string, repo *stackRepo, branch string, composePath string, sopsFiles []string, valuesFile string) *swarmStack { +func newSwarmStack(name string, repo *stackRepo, branch string, composePath string, sopsFiles []string, valuesFile string, discoverSecrets bool) *swarmStack { return &swarmStack{ - name: name, - repo: repo, - branch: branch, - composePath: composePath, - sopsFiles: sopsFiles, - valuesFile: valuesFile, + name: name, + repo: repo, + branch: branch, + composePath: composePath, + sopsFiles: sopsFiles, + valuesFile: valuesFile, + discoverSecrets: discoverSecrets, } } @@ -46,92 +49,138 @@ func (swarmStack *swarmStack) updateStack() (revision string, err error) { } log.Debug("changes pulled", "revision", revision) - log.Debug("decrypting sops files...") - err = swarmStack.decryptSopsFiles() + log.Debug("reading stack file...") + stackBytes, err := swarmStack.readStack() if err != nil { - return "", fmt.Errorf("failed to decrypt one or more sops files for %s stack: %w", swarmStack.name, err) + return } if swarmStack.valuesFile != "" { log.Debug("rendering template...") - err = swarmStack.renderComposeTemplate() - if err != nil { - return - } + stackBytes, err = swarmStack.renderComposeTemplate(stackBytes) + } + if err != nil { + return } - log.Debug("rotating configs and secrets...") - err = swarmStack.rotateConfigsAndSecrets() + log.Debug("parsing stack content...") + stackContents, err := swarmStack.parseStackString([]byte(stackBytes)) if err != nil { return } - log.Debug("deploying stack...") - err = swarmStack.deployStack() + log.Debug("decrypting secrets...") + err = swarmStack.decryptSopsFiles(stackContents) + if err != nil { + return "", fmt.Errorf("failed to decrypt one or more sops files for %s stack: %w", swarmStack.name, err) + } + + log.Debug("rotating configs and secrets...") + err = swarmStack.rotateConfigsAndSecrets(stackContents) if err != nil { return } - return -} -func (swarmStack *swarmStack) decryptSopsFiles() (err error) { - for _, sopsFile := range swarmStack.sopsFiles { - err = util.DecryptFile(path.Join(swarmStack.repo.path, sopsFile)) - if err != nil { - return - } + log.Debug("writing stack to file...") + err = swarmStack.writeStack(stackContents) + if err != nil { + return } + + log.Debug("deploying stack...") + err = swarmStack.deployStack() return } -func (swarmStack *swarmStack) deployStack() error { - cmd := stack.NewStackCommand(dockerCli) - cmd.SetArgs([]string{ - "deploy", "--detach", "--with-registry-auth", "-c", - path.Join(swarmStack.repo.path, swarmStack.composePath), - swarmStack.name, - }) - // To stop printing errors and usage message to stdout - cmd.SilenceErrors = true - cmd.SilenceUsage = true - err := cmd.Execute() +func (swarmStack *swarmStack) readStack() ([]byte, error) { + composeFile := path.Join(swarmStack.repo.path, swarmStack.composePath) + composeFileBytes, err := os.ReadFile(composeFile) if err != nil { - return fmt.Errorf("could not deploy stack %s: %s", swarmStack.name, err) + return nil, fmt.Errorf("could not read compose file %s: %w", composeFile, err) } - return nil + return composeFileBytes, nil } -func (swarmStack *swarmStack) rotateConfigsAndSecrets() error { - composeFile := path.Join(swarmStack.repo.path, swarmStack.composePath) - composeFileBytes, err := os.ReadFile(composeFile) +func (swarmStack *swarmStack) renderComposeTemplate(templateContents []byte) ([]byte, error) { + valuesFile := path.Join(config.ReposPath, swarmStack.repo.path, swarmStack.valuesFile) + valuesBytes, err := os.ReadFile(valuesFile) + if err != nil { + return nil, fmt.Errorf("could not read %s stack values file: %w", swarmStack.name, err) + } + var valuesMap map[string]any + yaml.Unmarshal(valuesBytes, &valuesMap) + templ, err := template.New(swarmStack.name).Parse(string(templateContents[:])) + if err != nil { + return nil, fmt.Errorf("could not parse %s stack compose file as a Go template: %w", swarmStack.name, err) + } + var stackContents bytes.Buffer + err = templ.Execute(&stackContents, map[string]map[string]any{"Values": valuesMap}) if err != nil { - return fmt.Errorf("could not read compose file %s: %w", composeFile, err) + return nil, fmt.Errorf("error rending %s stack compose template: %w", swarmStack.name, err) } + return stackContents.Bytes(), nil +} + +func (swarmStack *swarmStack) parseStackString(stackContent []byte) (map[string]any, error) { var composeMap map[string]any - err = yaml.Unmarshal(composeFileBytes, &composeMap) + err := yaml.Unmarshal(stackContent, &composeMap) if err != nil { - return fmt.Errorf("could not parse yaml file %s: %w", composeFile, err) + return nil, fmt.Errorf("could not parse stack yaml: %w", err) } + return composeMap, nil +} +func (swarmStack *swarmStack) decryptSopsFiles(composeMap map[string]any) (err error) { + var sopsFiles []string + if !swarmStack.discoverSecrets { + sopsFiles = swarmStack.sopsFiles + } else { + sopsFiles, err = discoverSecrets(composeMap, swarmStack.composePath) + if err != nil { + return + } + } + for _, sopsFile := range sopsFiles { + err = util.DecryptFile(path.Join(swarmStack.repo.path, sopsFile)) + if err != nil { + return + } + } + return +} + +func discoverSecrets(composeMap map[string]any, composePath string) ([]string, error) { + var sopsFiles []string + if secrets, ok := composeMap["secrets"].(map[string]any); ok { + for secretName, secret := range secrets { + secretMap, ok := secret.(map[string]any) + if !ok { + return nil, fmt.Errorf("invalid compose file: %s secret must be a map", secretName) + } + secretFile, ok := secretMap["file"].(string) + if !ok { + return nil, fmt.Errorf("invalid compose file: %s file field must be a string", secretName) + } + objectDir := path.Join(path.Dir(composePath), secretFile) + sopsFiles = append(sopsFiles, objectDir) + } + } + return sopsFiles, nil +} + +func (swarmStack *swarmStack) rotateConfigsAndSecrets(composeMap map[string]any) error { if configs, ok := composeMap["configs"].(map[string]any); ok { - err = swarmStack.rotateObjects(configs) + err := swarmStack.rotateObjects(configs) if err != nil { return fmt.Errorf("could not rotate one or more config files of stack %s: %w", swarmStack.name, err) } } if secrets, ok := composeMap["secrets"].(map[string]any); ok { - err = swarmStack.rotateObjects(secrets) + err := swarmStack.rotateObjects(secrets) if err != nil { return fmt.Errorf("could not rotate one or more secret files of stack %s: %w", swarmStack.name, err) } } - - composeFileBytes, err = yaml.Marshal(composeMap) - if err != nil { - return fmt.Errorf("could not store comopse file as yaml after calculating hashes for stack %s", swarmStack.name) - } - fileInfo, _ := os.Stat(composeFile) - os.WriteFile(composeFile, composeFileBytes, fileInfo.Mode()) return nil } @@ -157,26 +206,31 @@ func (swarmStack *swarmStack) rotateObjects(objects map[string]any) error { return nil } -func (swarmStack *swarmStack) renderComposeTemplate() error { - composeFile := path.Join(config.ReposPath, swarmStack.repo.path, swarmStack.composePath) - valuesFile := path.Join(config.ReposPath, swarmStack.repo.path, swarmStack.valuesFile) - valuesBytes, err := os.ReadFile(valuesFile) +func (swarmStack *swarmStack) writeStack(composeMap map[string]any) error { + composeFileBytes, err := yaml.Marshal(composeMap) if err != nil { - return fmt.Errorf("could not read %s stack values file: %w", swarmStack.name, err) + return fmt.Errorf("could not store compose file as yaml after calculating hashes for stack %s", swarmStack.name) } - var valuesMap map[string]any - yaml.Unmarshal(valuesBytes, &valuesMap) - templ, err := template.New(path.Base(composeFile)).ParseFiles(composeFile) - if err != nil { - return fmt.Errorf("could not parse %s stack compose file as a Go template: %w", swarmStack.name, err) - } - composeFileWriter, err := os.Create(composeFile) - if err != nil { - return fmt.Errorf("could not open %s stack compose file: %w", swarmStack.name, err) - } - err = templ.Execute(composeFileWriter, map[string]map[string]any{"Values": valuesMap}) + composeFile := path.Join(swarmStack.repo.path, swarmStack.composePath) + fileInfo, _ := os.Stat(composeFile) + os.WriteFile(composeFile, composeFileBytes, fileInfo.Mode()) + return nil +} + +func (swarmStack *swarmStack) deployStack() error { + cmd := stack.NewStackCommand(dockerCli) + cmd.SetArgs([]string{ + "deploy", "--detach", "--with-registry-auth", "-c", + path.Join(swarmStack.repo.path, swarmStack.composePath), + swarmStack.name, + }) + // To stop printing errors and + // usage message to stdout + cmd.SilenceErrors = true + cmd.SilenceUsage = true + err := cmd.Execute() if err != nil { - return fmt.Errorf("error rending %s stack compose template: %w", swarmStack.name, err) + return fmt.Errorf("could not deploy stack %s: %s", swarmStack.name, err) } return nil } diff --git a/ui/package-lock.json b/ui/package-lock.json index df31ebb..2ff892d 100644 --- a/ui/package-lock.json +++ b/ui/package-lock.json @@ -28,7 +28,7 @@ "eslint-plugin-react-hooks": "^4.6.2", "eslint-plugin-react-refresh": "^0.4.12", "prettier": "^3.3.3", - "typescript": "^5.2.2", + "typescript": "^5.6.2", "vite": "^5.4.7" } }, @@ -6267,9 +6267,9 @@ } }, "node_modules/typescript": { - "version": "5.5.4", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.5.4.tgz", - "integrity": "sha512-Mtq29sKDAEYP7aljRgtPOpTvOfbwRWlS6dPRzwjdE+C0R4brX/GUyhHSecbHMFLNBLcJIPt9nl9yG5TZ1weH+Q==", + "version": "5.6.2", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.6.2.tgz", + "integrity": "sha512-NW8ByodCSNCwZeghjN3o+JX5OFH0Ojg6sadjEKY4huZ52TqbJTJnDo5+Tw98lSy63NZvi4n+ez5m2u5d4PkZyw==", "dev": true, "bin": { "tsc": "bin/tsc", diff --git a/ui/package.json b/ui/package.json index 3612b82..ea70ca7 100644 --- a/ui/package.json +++ b/ui/package.json @@ -31,7 +31,7 @@ "eslint-plugin-react-hooks": "^4.6.2", "eslint-plugin-react-refresh": "^0.4.12", "prettier": "^3.3.3", - "typescript": "^5.2.2", + "typescript": "^5.6.2", "vite": "^5.4.7" } } diff --git a/util/config.go b/util/config.go index 76ad5b8..ef17535 100644 --- a/util/config.go +++ b/util/config.go @@ -8,11 +8,12 @@ import ( ) type StackConfig struct { - Repo string - Branch string - ComposeFile string `mapstructure:"compose_file"` - ValuesFile string `mapstructure:"values_file"` - SopsFiles []string `mapstructure:"sops_files"` + Repo string + Branch string + ComposeFile string `mapstructure:"compose_file"` + ValuesFile string `mapstructure:"values_file"` + SopsFiles []string `mapstructure:"sops_files"` + SopsSecretsDiscovery bool `mapstructure:"sops_secrets_discovery"` } type RepoConfig struct { @@ -23,11 +24,12 @@ type RepoConfig struct { } type Config struct { - ReposPath string `mapstructure:"repos_path"` - UpdateInterval int `mapstructure:"update_interval"` - AutoRotate bool `mapstructure:"auto_rotate"` - StackConfigs map[string]*StackConfig `mapstructure:"stacks"` - RepoConfigs map[string]*RepoConfig `mapstructure:"repos"` + ReposPath string `mapstructure:"repos_path"` + UpdateInterval int `mapstructure:"update_interval"` + AutoRotate bool `mapstructure:"auto_rotate"` + StackConfigs map[string]*StackConfig `mapstructure:"stacks"` + RepoConfigs map[string]*RepoConfig `mapstructure:"repos"` + SopsSecretsDiscovery bool `mapstructure:"sops_secrets_discovery"` } var Configs Config @@ -59,6 +61,7 @@ func readConfig() (err error) { configViper.SetDefault("update_interval", 120) configViper.SetDefault("repos_path", "repos") configViper.SetDefault("auto_rotate", true) + configViper.SetDefault("sops_secrets_discovery", false) err = configViper.ReadInConfig() if err != nil && !errors.As(err, &viper.ConfigFileNotFoundError{}) { return