From 6876616965e8818a708ac67868074fad6e8ea251 Mon Sep 17 00:00:00 2001 From: B&R Date: Wed, 25 Jan 2023 08:16:42 +0100 Subject: [PATCH 01/10] feat: Enable GPG verbosity, when log level is "debug" --- context/gpg.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/context/gpg.go b/context/gpg.go index 7041e5b..a588566 100644 --- a/context/gpg.go +++ b/context/gpg.go @@ -17,7 +17,7 @@ type GPGOperationContext struct { // dynamic Path string - ShouldShowOutput bool // todo: implement, currently not set, should depend on the verbosity + ShouldShowOutput bool } // InitializeGPGContext is a factory method that creates a GPG directory and imports keys @@ -28,9 +28,10 @@ func InitializeGPGContext(action *Action) error { log.Printf("Cannot create temporary directory for GPG: '%v'", path) return err } - if os.Getenv("BR_VERBOSE") == "true" { + if os.Getenv("BR_VERBOSE") == "true" || action.LogLevelStr == "debug" { ctx.ShouldShowOutput = true } + log.Infof("GPG temporary homedir is '%s'", path) ctx.Path = path if ctx.PublicKeyPath != "" || ctx.PrivateKeyPath != "" { if initErr := ctx.initializeGPGDirectory(); initErr != nil { @@ -150,7 +151,7 @@ func (that GPGOperationContext) initializeGPGDirectory() error { return runErr } - log.Debugf("Writing Passphrase...") + log.Debugf("Writing Passphrase to GPG process...") if _, writeErr := io.WriteString(stdin, that.Passphrase); writeErr != nil { _ = cmd.Process.Kill() return writeErr From 302609398c7e1bb5c86488d5043a67c98c577ff0 Mon Sep 17 00:00:00 2001 From: B&R Date: Sun, 29 Jan 2023 21:32:59 +0100 Subject: [PATCH 02/10] refactor(#32): Do not depend on GPG binary anymore --- DEVELOPMENT.md | 29 +++ README.md | 7 +- client/download_test.go | 6 +- client/upload.go | 3 +- client/upload_test.go | 4 +- cmd/backupmaker/main.go | 35 ++-- cmd/encryption/main.go | 76 ++++++++ cmd/root.go | 4 + context/action.go | 10 +- context/action_test.go | 8 +- context/encryption.go | 57 ++++++ context/{gpg_test.go => encryption_test.go} | 42 ++--- context/gpg.go | 197 -------------------- crypto/gpg.go | 110 +++++++++++ crypto/interface.go | 6 + go.mod | 2 + go.sum | 8 + resources/test/gpg-key.asc.pub | 51 +++++ 18 files changed, 386 insertions(+), 269 deletions(-) create mode 100644 DEVELOPMENT.md create mode 100644 cmd/encryption/main.go create mode 100644 context/encryption.go rename context/{gpg_test.go => encryption_test.go} (65%) delete mode 100644 context/gpg.go create mode 100644 crypto/gpg.go create mode 100644 crypto/interface.go create mode 100644 resources/test/gpg-key.asc.pub diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md new file mode 100644 index 0000000..5e0fe9c --- /dev/null +++ b/DEVELOPMENT.md @@ -0,0 +1,29 @@ +Running make & restore locally to test encryption & sending & receiving +----------------------------------------------------------------------- + +#### Server part + +1. In backup-repository repository you need to run `make k3d skaffold-deploy`, then you will have a working Backup Repository instance in local Kubernetes. + +2. Setup a tunnel for client connections + +```bash +kubectl port-forward svc/server-backup-repository-server -n backups 8080:8080 +``` + +#### Client part + +```bash +# export basic settings. Execute once in a console +export BM_COLLECTION_ID=iwa-ait +export BM_PASSPHRASE=riotkit +export BM_AUTH_TOKEN=$(curl -s -X POST -d '{"username":"admin","password":"admin"}' -H 'Content-Type: application/json' 'http://127.0.0.1:8080/api/stable/auth/login' | jq '.data.token' -r) +export BM_URL=http://127.0.0.1:8080 +``` + +#### Perform testing + +```bash +./.build/backup-maker make --cmd "tar -zcvf - ./" --key ./resources/test/gpg-key.asc +./.build/backup-maker restore --cmd "cat - > /tmp/restore.tar.gz" --passphrase riotkit --private-key ./resources/test/gpg-key.asc +``` diff --git a/README.md b/README.md index 055bc5a..5eacd89 100644 --- a/README.md +++ b/README.md @@ -65,25 +65,23 @@ please take a look at `Backup Controller` documentation. **Note: GPG steps are optional** -1. `gpg` keyring is created in a temporary directory, keys are imported +1. `gpg` keys are loaded 2. Command specified in `--cmd` or in `-c` is executed 3. Result of the command, it's stdout is transferred to the `gpg` process 4. From `gpg` process the encoded data is buffered directly to the server 5. Feedback is returned -6. Temporary `gpg` keyring is deleted ## Restore - How it works? It is very similar as in backup operation. -1. `gpg` keyring is created in a temporary directory, keys are imported +1. `gpg` keys are loaded 2. Command specified in `--cmd` or in `-c` is executed 3. `gpg` process is started 4. Backup download is starting 5. Backup is transmitted on the fly from server to `gpg` -> our shell command 6. Our shell `--cmd` / `-c` command is taking stdin and performing a restore action 7. Feedback is returned -8. Temporary `gpg` keyring is deleted ## Automated procedures @@ -96,6 +94,7 @@ together with a tool that generates Backup & Restore procedures. Those procedure - Skip `--private-key` and `--passphrase` to disable GPG - Use `debug` log level to see GPG output and more verbose output at all +- Increase encryption/decryption performance by disabling armoring ## Proposed usage diff --git a/client/download_test.go b/client/download_test.go index 4686163..a698851 100644 --- a/client/download_test.go +++ b/client/download_test.go @@ -90,15 +90,11 @@ func TestDownload_SuccessWithValidGPG(t *testing.T) { ctx := createExampleContext() ctx.ActionType = "download" - ctx.Gpg = context.GPGOperationContext{ + ctx.Crypto = context.EncryptionOperationContext{ PrivateKeyPath: "../resources/test/gpg-key.asc", Passphrase: "riotkit", Recipient: "test@riotkit.org", } - initErr := context.InitializeGPGContext(&ctx) - assert.Nil(t, initErr) - defer ctx.Gpg.CleanUp() - _ = DownloadBackupIntoProcessStdin(ctx, "cat - > ../.build/TestDownload_SuccessWithValidGPG", client) decryptedResult, _ := os.ReadFile("../.build/TestDownload_SuccessWithValidGPG") diff --git a/client/upload.go b/client/upload.go index 2ac9345..1a4699e 100644 --- a/client/upload.go +++ b/client/upload.go @@ -115,7 +115,7 @@ func UploadFromCommandOutput(app actionCtx.Action, client HTTPClient) error { return pipeErr } - log.Print("Starting cmd.Run()") + log.Print("Starting cmd.Encrypt()") execErr := cmd.Start() if execErr != nil { log.Println("Cannot start backup process ", execErr) @@ -127,6 +127,7 @@ func UploadFromCommandOutput(app actionCtx.Action, client HTTPClient) error { log.Printf("Starting Upload() for PID=%v", cmd.Process.Pid) status, out, uploadErr := Upload(ctx, client, app.Url, app.CollectionId, app.AuthToken, ReadCloserWithCancellationWhenProcessFails{stdout, cmd, cancel}, app.Timeout) if uploadErr != nil { + cancel() log.Errorf("Status: %v, Out: %v, Err: %v", status, out, uploadErr) return uploadErr } else { diff --git a/client/upload_test.go b/client/upload_test.go index 3f94a45..810eab7 100644 --- a/client/upload_test.go +++ b/client/upload_test.go @@ -25,7 +25,7 @@ func TestUploadFromCommandOutput_PassesOnValidResponse(t *testing.T) { ActionType: "make", VersionToRestore: "", DownloadPath: "", - Gpg: context.GPGOperationContext{}, + Crypto: context.EncryptionOperationContext{}, }, http) assert.Nil(t, err) } @@ -47,7 +47,7 @@ func TestUploadFromCommandOutput_FailOnInvalidResponse(t *testing.T) { ActionType: "make", VersionToRestore: "", DownloadPath: "", - Gpg: context.GPGOperationContext{}, + Crypto: context.EncryptionOperationContext{}, }, http) assert.NotNil(t, err) } diff --git a/cmd/backupmaker/main.go b/cmd/backupmaker/main.go index 89c82b9..4aad20a 100644 --- a/cmd/backupmaker/main.go +++ b/cmd/backupmaker/main.go @@ -21,7 +21,7 @@ func addAddGenericFlags(cmd *cobra.Command, ctx *context.Action) { cmd.Flags().StringVarP(&ctx.AuthToken, "auth-token", "t", os.Getenv("BM_AUTH_TOKEN"), "Access token that allows to upload at least one file successfully, [environment variable: BM_AUTH_TOKEN]") cmd.Flags().Int64VarP(&ctx.Timeout, "timeout", "", 60*20, "Connection and read timeout in summary [environment variable: BM_TIMEOUT]") cmd.Flags().StringVarP(&ctx.LogLevelStr, "log-level", "", "info", "Verbosity level: panic|fatal|error|warn|info|debug|trace") - cmd.Flags().BoolVarP(&ctx.Gpg.ShouldShowOutput, "verbose", "", false, "Increase verbosity") + cmd.Flags().StringVarP(&ctx.Crypto.EncType, "encryption-type", "", getEnvOrDefault("BM_ENCRYPTION_TYPE", "gpg-armored"), "Encryption type (options: gpg-armored, gpg-binary) [environment variable: BM_ENCRYPTION_TYPE]") } func createMakeCommand() *cobra.Command { @@ -33,23 +33,20 @@ func createMakeCommand() *cobra.Command { Short: "Upload a backup to remote", SilenceUsage: true, RunE: func(cmd *cobra.Command, args []string) error { - defer ctx.Gpg.CleanUp() if err := context.InitializeLogLevel(&ctx); err != nil { return err } - if err := context.InitializeGPGContext(&ctx); err != nil { - return err - } if err := client.UploadFromCommandOutput(ctx, client.CreateHttpClient()); err != nil { return err } return nil }, } - makeCmd.Flags().StringVarP(&ctx.Gpg.PublicKeyPath, "key", "k", os.Getenv("BM_PUBLIC_KEY_PATH"), "GPG public or private key (required if using GPG) [environment variable: BM_PUBLIC_KEY_PATH]") - makeCmd.Flags().StringVarP(&ctx.Gpg.Recipient, "recipient", "r", os.Getenv("BM_RECIPIENT"), "GPG recipient e-mail (required if using GPG). By default this e-mail SHOULD BE same as e-mail used when restoring/downloading backup [environment variable: BM_RECIPIENT]") + makeCmd.Flags().StringVarP(&ctx.Crypto.PublicKeyPath, "key", "k", os.Getenv("BM_PUBLIC_KEY_PATH"), "GPG public or private key (required if using GPG) [environment variable: BM_PUBLIC_KEY_PATH]") + makeCmd.Flags().StringVarP(&ctx.Crypto.Recipient, "recipient", "r", os.Getenv("BM_RECIPIENT"), "GPG recipient e-mail (required if using GPG). By default this e-mail SHOULD BE same as e-mail used when restoring/downloading backup [environment variable: BM_RECIPIENT]") makeCmd.Flags().StringVarP(&ctx.Command, "cmd", "c", os.Getenv("BM_CMD"), "Command to execute, which output will be captured and sent to server [environment variable: BM_CMD]") - makeCmd.Flags().StringVarP(&ctx.Gpg.Passphrase, "passphrase", "", os.Getenv("BM_PASSPHRASE"), "Secret passphrase for GPG [environment variable: BM_PASSPHRASE]") + makeCmd.Flags().StringVarP(&ctx.Crypto.Passphrase, "passphrase", "", os.Getenv("BM_PASSPHRASE"), "Secret passphrase for GPG [environment variable: BM_PASSPHRASE]") + addAddGenericFlags(&makeCmd, &ctx) return &makeCmd @@ -64,13 +61,9 @@ func createRestoreCommand() *cobra.Command { SilenceUsage: true, Short: "Restore a backup from remote to local target", RunE: func(cmd *cobra.Command, args []string) error { - defer ctx.Gpg.CleanUp() if err := context.InitializeLogLevel(&ctx); err != nil { return err } - if err := context.InitializeGPGContext(&ctx); err != nil { - return err - } if err := client.DownloadBackupIntoProcessStdin(ctx, ctx.Command, client.CreateHttpClient()); err != nil { return err } @@ -78,11 +71,12 @@ func createRestoreCommand() *cobra.Command { }, } - restoreCmd.Flags().StringVarP(&ctx.Gpg.PrivateKeyPath, "private-key", "p", os.Getenv("BM_PRIVATE_KEY_PATH"), "GPG private key. [environment variable: BM_PRIVATE_KEY_PATH]") + restoreCmd.Flags().StringVarP(&ctx.Crypto.PrivateKeyPath, "private-key", "p", os.Getenv("BM_PRIVATE_KEY_PATH"), "GPG private key. [environment variable: BM_PRIVATE_KEY_PATH]") restoreCmd.Flags().StringVarP(&ctx.Command, "cmd", "c", os.Getenv("BM_CMD"), "Command which should take downloaded file as stdin stream e.g. some tar, unzip, psql [environment variable: BM_CMD]") - restoreCmd.Flags().StringVarP(&ctx.Gpg.Passphrase, "passphrase", "", os.Getenv("BM_PASSPHRASE"), "Secret passphrase for GPG [environment variable: BM_PASSPHRASE]") + restoreCmd.Flags().StringVarP(&ctx.Crypto.Passphrase, "passphrase", "", os.Getenv("BM_PASSPHRASE"), "Secret passphrase for GPG [environment variable: BM_PASSPHRASE]") restoreCmd.Flags().StringVarP(&ctx.VersionToRestore, "version", "s", getEnvOrDefault("BM_VERSION", "latest"), "Version number [environment variable: BM_VERSION]") - restoreCmd.Flags().StringVarP(&ctx.Gpg.Recipient, "recipient", "r", os.Getenv("BM_RECIPIENT"), "GPG recipient e-mail (required if using GPG). By default this e-mail SHOULD BE same as e-mail used when restoring/downloading backup [environment variable: BM_RECIPIENT]") + restoreCmd.Flags().StringVarP(&ctx.Crypto.Recipient, "recipient", "r", os.Getenv("BM_RECIPIENT"), "GPG recipient e-mail (required if using GPG). By default this e-mail SHOULD BE same as e-mail used when restoring/downloading backup [environment variable: BM_RECIPIENT]") + addAddGenericFlags(&restoreCmd, &ctx) return &restoreCmd } @@ -96,13 +90,9 @@ func createDownloadCommand() *cobra.Command { SilenceUsage: true, Short: "Download a remote backup and print into a local file", RunE: func(cmd *cobra.Command, args []string) error { - defer ctx.Gpg.CleanUp() if err := context.InitializeLogLevel(&ctx); err != nil { return err } - if err := context.InitializeGPGContext(&ctx); err != nil { - return err - } if err := client.DownloadIntoFile(ctx, ctx.DownloadPath, client.CreateHttpClient()); err != nil { return err } @@ -110,11 +100,12 @@ func createDownloadCommand() *cobra.Command { }, } - downloadCmd.Flags().StringVarP(&ctx.Gpg.PrivateKeyPath, "private-key", "p", os.Getenv("BM_PRIVATE_KEY_PATH"), "GPG private key. [environment variable: BM_PRIVATE_KEY_PATH]") + downloadCmd.Flags().StringVarP(&ctx.Crypto.PrivateKeyPath, "private-key", "p", os.Getenv("BM_PRIVATE_KEY_PATH"), "GPG private key. [environment variable: BM_PRIVATE_KEY_PATH]") downloadCmd.Flags().StringVarP(&ctx.Command, "save-path", "", os.Getenv("BM_SAVE_PATH"), "Place where to save file instead of executing a restore command [environment variable: BM_SAVE_PATH]") - downloadCmd.Flags().StringVarP(&ctx.Gpg.Passphrase, "passphrase", "", os.Getenv("BM_PASSPHRASE"), "Secret passphrase for GPG [environment variable: BM_PASSPHRASE]") + downloadCmd.Flags().StringVarP(&ctx.Crypto.Passphrase, "passphrase", "", os.Getenv("BM_PASSPHRASE"), "Secret passphrase for GPG [environment variable: BM_PASSPHRASE]") downloadCmd.Flags().StringVarP(&ctx.VersionToRestore, "version", "s", getEnvOrDefault("BM_VERSION", "latest"), "Version number [environment variable: BM_VERSION]") - downloadCmd.Flags().StringVarP(&ctx.Gpg.Recipient, "recipient", "r", os.Getenv("BM_RECIPIENT"), "GPG recipient e-mail (required if using GPG). By default this e-mail SHOULD BE same as e-mail used when restoring/downloading backup [environment variable: BM_RECIPIENT]") + downloadCmd.Flags().StringVarP(&ctx.Crypto.Recipient, "recipient", "r", os.Getenv("BM_RECIPIENT"), "GPG recipient e-mail (required if using GPG). By default this e-mail SHOULD BE same as e-mail used when restoring/downloading backup [environment variable: BM_RECIPIENT]") + addAddGenericFlags(&downloadCmd, &ctx) return &downloadCmd } diff --git a/cmd/encryption/main.go b/cmd/encryption/main.go new file mode 100644 index 0000000..f51c1cb --- /dev/null +++ b/cmd/encryption/main.go @@ -0,0 +1,76 @@ +package encryption + +import ( + "github.com/pkg/errors" + "github.com/riotkit-org/br-backup-maker/crypto" + "github.com/spf13/cobra" +) + +func NewEncryptionCommand() *cobra.Command { + app := &App{} + + command := &cobra.Command{ + Use: "encrypt", + SilenceUsage: true, + Short: "Encrypts a stdin and outputs as stdout", + RunE: func(command *cobra.Command, args []string) error { + return app.Encrypt() + }, + } + + command.Flags().StringVarP(&app.keyPath, "key-path", "k", "", "Path to the key file") + command.Flags().StringVarP(&app.encType, "type", "t", "gpg-armored", "Encryption type (options: gpg-armored, gpg-binary)") + command.Flags().StringVarP(&app.passphrase, "passphrase", "p", "", "(Optional) passphrase to decrypt the key") + + return command +} + +func NewDecryptionCommand() *cobra.Command { + app := &App{} + + command := &cobra.Command{ + Use: "decrypt", + SilenceUsage: true, + Short: "Decrypts a stdin and outputs as stdout", + RunE: func(command *cobra.Command, args []string) error { + return app.Decrypt() + }, + } + + command.Flags().StringVarP(&app.keyPath, "key-path", "k", "", "Path to the key file") + command.Flags().StringVarP(&app.encType, "type", "t", "gpg-armored", "Encryption type (options: gpg-armored, gpg-binary)") + command.Flags().StringVarP(&app.passphrase, "passphrase", "p", "", "(Optional) passphrase to decrypt the key") + + return command +} + +type App struct { + keyPath string + encType string + passphrase string +} + +func (encrypt *App) createAlgo() (crypto.Service, error) { + if encrypt.encType == "gpg-armored" { + return crypto.GPGEncryption{Armored: true}, nil + } else if encrypt.encType == "gpg-binary" { + return crypto.GPGEncryption{Armored: false}, nil + } + return nil, errors.New("unsupported encryption type") +} + +func (encrypt *App) Encrypt() error { + algo, err := encrypt.createAlgo() + if err != nil { + return err + } + return algo.Encrypt(encrypt.keyPath, encrypt.passphrase) +} + +func (encrypt *App) Decrypt() error { + algo, err := encrypt.createAlgo() + if err != nil { + return err + } + return algo.Decrypt(encrypt.keyPath, encrypt.passphrase) +} diff --git a/cmd/root.go b/cmd/root.go index fe1a9a5..dfe305c 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -3,6 +3,7 @@ package cmd import ( "github.com/riotkit-org/br-backup-maker/cmd/backupmaker" "github.com/riotkit-org/br-backup-maker/cmd/bmg" + "github.com/riotkit-org/br-backup-maker/cmd/encryption" "github.com/spf13/cobra" ) @@ -18,5 +19,8 @@ func GetRootCommand() *cobra.Command { cmd.AddCommand(subCmd) } cmd.AddCommand(bmg.CreateCommand()) + cmd.AddCommand(encryption.NewEncryptionCommand()) + cmd.AddCommand(encryption.NewDecryptionCommand()) + return cmd } diff --git a/context/action.go b/context/action.go index a92a93d..5b41be5 100644 --- a/context/action.go +++ b/context/action.go @@ -13,7 +13,7 @@ type Action struct { VersionToRestore string DownloadPath string - Gpg GPGOperationContext + Crypto EncryptionOperationContext LogLevel uint32 LogLevelStr string // todo: convert to LogLevel //logLevel, _ := log.ParseLevel(*logLevelStr) @@ -30,21 +30,21 @@ func (that Action) CreateWrappedCommand(custom string) string { cmd = custom } - if !that.Gpg.Enabled(that.ActionType) { + if !that.Crypto.Enabled(that.ActionType) { return cmd } if that.ActionType == "make" { - return cmd + " | " + that.Gpg.GetEncryptionCommand() + return cmd + " | " + that.Crypto.GetEncryptionCommand() } - return that.Gpg.GetDecryptionCommand() + " | " + cmd + return that.Crypto.GetDecryptionCommand() + " | " + cmd } // GetPrintableCommand returns same command as in CreateWrappedCommand(), but with erased credentials // so the command could be logged or printed into the console func (that Action) GetPrintableCommand(custom string) string { - return strings.ReplaceAll(that.CreateWrappedCommand(custom), that.Gpg.Passphrase, "***") + return strings.ReplaceAll(that.CreateWrappedCommand(custom), that.Crypto.Passphrase, "***") } func (that Action) ShouldShowCommandsOutput() bool { diff --git a/context/action_test.go b/context/action_test.go index 1cec949..0554804 100644 --- a/context/action_test.go +++ b/context/action_test.go @@ -10,7 +10,7 @@ func TestActionContext_GetCommand_WithGPG(t *testing.T) { ctx := Action{} ctx.Command = "ps aux" ctx.ActionType = "make" - ctx.Gpg = GPGOperationContext{ + ctx.Crypto = EncryptionOperationContext{ PublicKeyPath: "/path/to/key", PrivateKeyPath: "/path/to/key", Passphrase: "riotkit", @@ -29,7 +29,7 @@ func TestActionContext_GetCommand_WithGPG_RestoreActionPlacesPipeAtRightSide(t * ctx := Action{} ctx.Command = "tar xvf -" ctx.ActionType = "restore" - ctx.Gpg = GPGOperationContext{ + ctx.Crypto = EncryptionOperationContext{ PublicKeyPath: "/path/to/key", PrivateKeyPath: "/path/to/key", Passphrase: "riotkit", @@ -46,7 +46,7 @@ func TestActionContext_GetPrintableCommand_WithGPG(t *testing.T) { ctx := Action{} ctx.Command = "ps aux" ctx.ActionType = "restore" - ctx.Gpg = GPGOperationContext{ + ctx.Crypto = EncryptionOperationContext{ PublicKeyPath: "/path/to/key", PrivateKeyPath: "/path/to/key", Passphrase: "my-secret", @@ -64,7 +64,7 @@ func TestActionContext_GetCommand_PlaintextWithoutEncryption(t *testing.T) { ctx := Action{} ctx.Command = "tar -zcvf - ./" ctx.ActionType = "make" - ctx.Gpg = GPGOperationContext{ + ctx.Crypto = EncryptionOperationContext{ PublicKeyPath: "", PrivateKeyPath: "", Passphrase: "riotkit", diff --git a/context/encryption.go b/context/encryption.go new file mode 100644 index 0000000..8038543 --- /dev/null +++ b/context/encryption.go @@ -0,0 +1,57 @@ +package context + +import ( + "fmt" + log "github.com/sirupsen/logrus" + "os" +) + +type EncryptionOperationContext struct { + PublicKeyPath string + PrivateKeyPath string + Passphrase string + EncType string + Recipient string +} + +func InitializeLogLevel(action *Action) error { + logLevel, parseErr := log.ParseLevel(action.LogLevelStr) + if parseErr != nil { + return parseErr + } + log.SetLevel(logLevel) + action.LogLevel = uint32(logLevel) + return nil +} + +func (that EncryptionOperationContext) GetEncryptionCommand() string { + if that.PublicKeyPath == "" && that.PrivateKeyPath == "" { + log.Debug("No private key, no public key, no encryption then") + return "" + } + return fmt.Sprintf("%s encrypt --key-path=%s --type=%s", that.findBinPath(), that.PublicKeyPath, that.EncType) +} + +func (that EncryptionOperationContext) GetDecryptionCommand() string { + if that.PrivateKeyPath == "" { + log.Debug("No private key, no encryption") + return "" + } + return fmt.Sprintf("%s decrypt --key-path=%s --type=%s --passphrase='%s'", that.findBinPath(), that.PrivateKeyPath, that.EncType, that.Passphrase) +} + +func (that EncryptionOperationContext) findBinPath() string { + executable, _ := os.Executable() + if executable == "" { + executable = "backup-maker" + } + return executable +} + +func (that EncryptionOperationContext) Enabled(actionType string) bool { + if actionType == "make" { + return that.PublicKeyPath != "" + } + + return that.PrivateKeyPath != "" || that.PublicKeyPath != "" +} diff --git a/context/gpg_test.go b/context/encryption_test.go similarity index 65% rename from context/gpg_test.go rename to context/encryption_test.go index b7dc023..03e53ba 100644 --- a/context/gpg_test.go +++ b/context/encryption_test.go @@ -7,27 +7,11 @@ import ( "testing" ) -// Success scenario -func TestCreateGPGContext(t *testing.T) { - ctx := context.Action{ - ActionType: "make", - Gpg: context.GPGOperationContext{ - PublicKeyPath: "../resources/test/gpg-key.asc", - PrivateKeyPath: "../resources/test/gpg-key.asc", - Passphrase: "riotkit", - Recipient: "example@riotkit.org", - }, - } - err := context.InitializeGPGContext(&ctx) - defer ctx.Gpg.CleanUp() - assert.Nil(t, err) -} - // Import path is not valid. Error message should be in stdout/stderr, in error there could be exit status just func TestCreateGPGContext_InvalidKeyPath(t *testing.T) { ctx := context.Action{ ActionType: "make", - Gpg: context.GPGOperationContext{ + Crypto: context.EncryptionOperationContext{ PublicKeyPath: "invalid-path", PrivateKeyPath: "invalid-path", Passphrase: "riotkit", @@ -35,7 +19,7 @@ func TestCreateGPGContext_InvalidKeyPath(t *testing.T) { }, } err := context.InitializeGPGContext(&ctx) - defer ctx.Gpg.CleanUp() + defer ctx.Crypto.CleanUp() assert.Equal(t, "Cannot import key, error: exit status 2", err.Error()) } @@ -44,7 +28,7 @@ func TestCreateGPGContext_InvalidKeyPath(t *testing.T) { func TestCreateGPGContext_DisabledEncryption(t *testing.T) { ctx := context.Action{ ActionType: "make", - Gpg: context.GPGOperationContext{ + Crypto: context.EncryptionOperationContext{ PublicKeyPath: "", PrivateKeyPath: "", Passphrase: "", @@ -52,7 +36,7 @@ func TestCreateGPGContext_DisabledEncryption(t *testing.T) { }, } err := context.InitializeGPGContext(&ctx) - defer ctx.Gpg.CleanUp() + defer ctx.Crypto.CleanUp() assert.Nil(t, err) } @@ -61,7 +45,7 @@ func TestCreateGPGContext_DisabledEncryption(t *testing.T) { func TestGPGOperationContext_CleanUp(t *testing.T) { ctx := context.Action{ ActionType: "make", - Gpg: context.GPGOperationContext{ + Crypto: context.EncryptionOperationContext{ PublicKeyPath: "../resources/test/gpg-key.asc", PrivateKeyPath: "../resources/test/gpg-key.asc", Passphrase: "riotkit", @@ -69,14 +53,14 @@ func TestGPGOperationContext_CleanUp(t *testing.T) { }, } err := context.InitializeGPGContext(&ctx) - defer ctx.Gpg.CleanUp() // always clean at the end of the test + defer ctx.Crypto.CleanUp() // always clean at the end of the test assert.Nil(t, err, "Cannot initialize context to verify if its cleaned up later") - assert.DirExists(t, ctx.Gpg.Path, "Expected the context initialization will create a temporary directory") // GPG temporary directory - ctx.Gpg.CleanUp() + assert.DirExists(t, ctx.Crypto.Path, "Expected the context initialization will create a temporary directory") // GPG temporary directory + ctx.Crypto.CleanUp() // GPG temporary directory - assert.NoDirExists(t, ctx.Gpg.Path, "Expected that after calling CleanUp() the temporary directory will no longer exists") + assert.NoDirExists(t, ctx.Crypto.Path, "Expected that after calling CleanUp() the temporary directory will no longer exists") proc := exec.Command("/bin/bash", "-c", "ps aux |grep %v | grep -v grep") out, _ := proc.Output() @@ -86,7 +70,7 @@ func TestGPGOperationContext_CleanUp(t *testing.T) { func TestGPGOperationContext_GetDecryptionCommand_GetEncryptionCommmand_DoesNotReturnCommandWhenGPGDisabled(t *testing.T) { ctx := context.Action{ ActionType: "make", - Gpg: context.GPGOperationContext{ + Crypto: context.EncryptionOperationContext{ PublicKeyPath: "", PrivateKeyPath: "", Passphrase: "riotkit", @@ -94,9 +78,9 @@ func TestGPGOperationContext_GetDecryptionCommand_GetEncryptionCommmand_DoesNotR }, } err := context.InitializeGPGContext(&ctx) - defer ctx.Gpg.CleanUp() + defer ctx.Crypto.CleanUp() assert.Nil(t, err) - assert.Equal(t, "", ctx.Gpg.GetDecryptionCommand()) - assert.Equal(t, "", ctx.Gpg.GetEncryptionCommand()) + assert.Equal(t, "", ctx.Crypto.GetDecryptionCommand()) + assert.Equal(t, "", ctx.Crypto.GetEncryptionCommand()) } diff --git a/context/gpg.go b/context/gpg.go deleted file mode 100644 index a588566..0000000 --- a/context/gpg.go +++ /dev/null @@ -1,197 +0,0 @@ -package context - -import ( - "errors" - "fmt" - log "github.com/sirupsen/logrus" - "io" - "os" - "os/exec" -) - -type GPGOperationContext struct { - PublicKeyPath string - PrivateKeyPath string - Passphrase string - Recipient string - - // dynamic - Path string - ShouldShowOutput bool -} - -// InitializeGPGContext is a factory method that creates a GPG directory and imports keys -func InitializeGPGContext(action *Action) error { - ctx := &action.Gpg - path, err := os.MkdirTemp("/tmp", "backup-repository-gpg") - if err != nil { - log.Printf("Cannot create temporary directory for GPG: '%v'", path) - return err - } - if os.Getenv("BR_VERBOSE") == "true" || action.LogLevelStr == "debug" { - ctx.ShouldShowOutput = true - } - log.Infof("GPG temporary homedir is '%s'", path) - ctx.Path = path - if ctx.PublicKeyPath != "" || ctx.PrivateKeyPath != "" { - if initErr := ctx.initializeGPGDirectory(); initErr != nil { - return initErr - } - if importErr := ctx.importKeys(); importErr != nil { - return errors.New(fmt.Sprintf("Cannot import key, error: %v", importErr)) - } - ctx.printImportedKeys() - } else { - log.Warningln("GPG disabled (no keys configured)") - } - return nil -} - -func InitializeLogLevel(action *Action) error { - logLevel, parseErr := log.ParseLevel(action.LogLevelStr) - if parseErr != nil { - return parseErr - } - log.SetLevel(logLevel) - action.LogLevel = uint32(logLevel) - return nil -} - -func (that GPGOperationContext) CleanUp() { - log.Debugf("Cleaning up GPG directory at '%v'", that.Path) - err := os.RemoveAll(that.Path) - - if err != nil { - log.Fatalf("Cannot delete GPG directory at '%v'", that.Path) - } - - _ = exec.Command("/bin/bash", "-c", fmt.Sprintf("ps axu | grep gpg-agent | grep %v | grep -v grep | awk '{print $2}' | xargs kill -9", that.Path)).Run() -} - -// importKeys is importing PUBLIC and PRIVATE keys into local temporary keyring created on-the-fly -func (that GPGOperationContext) importKeys() error { - log.Debugf("Importing GPG keys '%v', '%v'", that.PrivateKeyPath, that.PublicKeyPath) - - for _, keyPath := range []string{that.PrivateKeyPath, that.PublicKeyPath} { - if keyPath == "" { - continue - } - - log.Printf("Importing key %v", keyPath) - cmd := exec.Command( - "gpg", - "--passphrase-fd", "0", - "--pinentry-mode", "loopback", - "--import", keyPath, - ) - cmd.Env = []string{fmt.Sprintf("GNUPGHOME=%v", that.Path)} - if that.ShouldShowOutput { - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - } - - stdin, _ := cmd.StdinPipe() - _ = cmd.Start() - _, _ = io.WriteString(stdin, that.Passphrase) - _ = stdin.Close() - cmdErr := cmd.Wait() - - if cmdErr != nil { - log.Errorf("Cannot import key '%v'. Check output placed above", keyPath) - return cmdErr - } - } - - return nil -} - -func (that GPGOperationContext) printImportedKeys() { - log.Println("Imported keys:") - cmd := exec.Command("gpg", "--list-secret-keys") - cmd.Env = []string{fmt.Sprintf("GNUPGHOME=%v", that.Path)} - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - _ = cmd.Run() -} - -// initializeGPGDirectory creates a GPG temporary keyring used on-the-fly, then it gets automatically deleted -// by the application -func (that GPGOperationContext) initializeGPGDirectory() error { - log.Println("Initializing GPG directory") - initParamsFilePath := fmt.Sprintf("%v/.init-params", that.Path) - - // create parameters file - _ = os.WriteFile( - initParamsFilePath, - []byte("Key-Type: 1\nKey-Length: 2048\nSubkey-Type: 1\nSubkey-Length: 2048\nName-Real: Backup Maker\nName-Email: riotkit@do-not-use.localhost\nExpire-Date: 0\n"), - 0600, - ) - - cmd := exec.Command( - "gpg", - "--gen-key", - "--passphrase-fd", "0", - "--pinentry-mode", "loopback", - "--batch", initParamsFilePath) - - cmd.Env = []string{fmt.Sprintf("GNUPGHOME=%v", that.Path)} - stdin, err := cmd.StdinPipe() - if that.ShouldShowOutput { - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - } - - if err != nil { - _ = cmd.Process.Kill() - return err - } - - runErr := cmd.Start() - if runErr != nil { - return runErr - } - - log.Debugf("Writing Passphrase to GPG process...") - if _, writeErr := io.WriteString(stdin, that.Passphrase); writeErr != nil { - _ = cmd.Process.Kill() - return writeErr - } - - _ = stdin.Close() - waitErr := cmd.Wait() - if waitErr != nil { - _ = cmd.Process.Kill() - return waitErr - } - - log.Debugf("GPG initialized") - return nil -} - -func (that GPGOperationContext) GetEncryptionCommand() string { - if that.PublicKeyPath == "" && that.PrivateKeyPath == "" { - log.Debug("No private key, no public key, no encryption then") - return "" - } - - return fmt.Sprintf("gpg --homedir='%v' --encrypt --always-trust --recipient='%v' --armor --batch --yes", that.Path, that.Recipient) -} - -func (that GPGOperationContext) GetDecryptionCommand() string { - if that.PrivateKeyPath == "" { - log.Debug("No private key, no encryption") - return "" - } - - return fmt.Sprintf("gpg --homedir='%v' --decrypt --recipient='%v' --armor "+ - "--passphrase='%v' --batch --yes --pinentry-mode loopback --verbose", - that.Path, that.Recipient, that.Passphrase) -} - -func (that GPGOperationContext) Enabled(actionType string) bool { - if actionType == "make" { - return that.PublicKeyPath != "" - } - - return that.PrivateKeyPath != "" || that.PublicKeyPath != "" -} diff --git a/crypto/gpg.go b/crypto/gpg.go new file mode 100644 index 0000000..b5d1a9a --- /dev/null +++ b/crypto/gpg.go @@ -0,0 +1,110 @@ +package crypto + +import ( + "github.com/ProtonMail/go-crypto/openpgp" + "github.com/ProtonMail/go-crypto/openpgp/armor" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + "io" + "os" +) + +type GPGEncryption struct { + Armored bool +} + +// ReadKey is reading and parsing the public/private key +func (GPGEncryption) ReadKey(keyPath string, passphrase string) (openpgp.EntityList, error) { + f, err := os.Open(keyPath) + if err != nil { + return nil, err + } + defer f.Close() + + keyring, keyringErr := openpgp.ReadArmoredKeyRing(f) + if keyringErr != nil { + return keyring, errors.Wrap(keyringErr, "cannot read keyring (provided key)") + } + + // passphrase is optional + if passphrase != "" { + if err := decryptSecretKey(keyring, passphrase); err != nil { + return keyring, errors.Wrap(err, "cannot decrypt keyring using selected passphrase") + } + } + return keyring, nil +} + +// decryptSecretKey is decrypting all keys in the keyring. Thanks to "wal-g" project maintained by Citus Data Inc. +func decryptSecretKey(entityList openpgp.EntityList, passphrase string) error { + passphraseBytes := []byte(passphrase) + + for _, entity := range entityList { + err := entity.PrivateKey.Decrypt(passphraseBytes) + + if err != nil { + return err + } + + for _, subKey := range entity.Subkeys { + err := subKey.PrivateKey.Decrypt(passphraseBytes) + + if err != nil { + return err + } + } + } + + return nil +} + +// Encrypt is reading from inputStream and writing encrypted stream of bytes to outputStream +func (g GPGEncryption) Encrypt(keyPath string, passphrase string) error { + keyring, keyErr := g.ReadKey(keyPath, passphrase) + if keyErr != nil { + return errors.Wrap(keyErr, "cannot load public keyring") + } + + // perform encryption + armoredWriter, _ := armor.Encode(os.Stdout, "PGP MESSAGE", nil) + defer armoredWriter.Close() + logrus.Debugln("armor.Encode() setup") + + conversionStream, _ := openpgp.Encrypt(armoredWriter, keyring, nil, &openpgp.FileHints{IsBinary: true}, nil) + defer conversionStream.Close() + logrus.Debugln("openpgp.Encrypt() setup") + + io.Copy(conversionStream, os.Stdin) + logrus.Debugln("io.Copy() finished for GPG encryption") + + return nil +} + +func (g GPGEncryption) Decrypt(keyPath string, passphrase string) error { + // load private key + keyring, keyErr := g.ReadKey(keyPath, passphrase) + if keyErr != nil { + return errors.Wrap(keyErr, "cannot load public key") + } + + var input io.Reader + input = os.Stdin + + if g.Armored { + armorBlock, armorErr := armor.Decode(os.Stdin) + if armorErr != nil { + return errors.Wrap(armorErr, "cannot setup armor decoder") + } + input = armorBlock.Body + } + + md, readErr := openpgp.ReadMessage(input, keyring, nil, nil) + if readErr != nil { + return errors.Wrap(readErr, "cannot read encrypted contents") + } + if _, err := io.Copy(os.Stdout, md.UnverifiedBody); err != nil { + return errors.Wrap(err, "cannot io.Copy() encrypted stream -> decrypted data") + } + + return nil +} diff --git a/crypto/interface.go b/crypto/interface.go new file mode 100644 index 0000000..2d51335 --- /dev/null +++ b/crypto/interface.go @@ -0,0 +1,6 @@ +package crypto + +type Service interface { + Encrypt(keyPath string, passphrase string) error + Decrypt(keyPath string, passphrase string) error +} diff --git a/go.mod b/go.mod index a69e64c..0dbdd1b 100644 --- a/go.mod +++ b/go.mod @@ -21,9 +21,11 @@ require ( github.com/Masterminds/sprig/v3 v3.2.2 // indirect github.com/Microsoft/go-winio v0.5.2 // indirect github.com/Microsoft/hcsshim v0.9.5 // indirect + github.com/ProtonMail/go-crypto v0.0.0-20230127150802-22e9f3c8043c // indirect github.com/PuerkitoBio/purell v1.1.1 // indirect github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect github.com/cenkalti/backoff/v4 v4.2.0 // indirect + github.com/cloudflare/circl v1.1.0 // indirect github.com/containerd/cgroups v1.0.4 // indirect github.com/containerd/containerd v1.6.12 // indirect github.com/cyphar/filepath-securejoin v0.2.3 // indirect diff --git a/go.sum b/go.sum index ccd013b..a713e50 100644 --- a/go.sum +++ b/go.sum @@ -80,6 +80,8 @@ github.com/Microsoft/hcsshim/test v0.0.0-20201218223536-d3e5debf77da/go.mod h1:5 github.com/Microsoft/hcsshim/test v0.0.0-20210227013316-43a75bb4edd3/go.mod h1:mw7qgWloBUl75W/gVH3cQszUg1+gUITj7D6NY7ywVnY= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/ProtonMail/go-crypto v0.0.0-20230127150802-22e9f3c8043c h1:3SOlz3Ldp5+/KwuXDbuoj1nWPI6MzqBfCz/KvlPS4ko= +github.com/ProtonMail/go-crypto v0.0.0-20230127150802-22e9f3c8043c/go.mod h1:I0gYDMZ6Z5GRU7l58bNFSkPTFN6Yl12dsUlAZ8xy98g= github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI= github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= @@ -111,6 +113,7 @@ github.com/buger/jsonparser v0.0.0-20180808090653-f4dd9f5a6b44/go.mod h1:bbYlZJ7 github.com/bugsnag/bugsnag-go v0.0.0-20141110184014-b1d153021fcd/go.mod h1:2oa8nejYd4cQ/b0hMIopN0lCRxU0bueqREvZLWFrtK8= github.com/bugsnag/osext v0.0.0-20130617224835-0dd3f918b21b/go.mod h1:obH5gd0BsqsP2LwDJ9aOkm/6J86V6lyAXCoQWGw3K50= github.com/bugsnag/panicwrap v0.0.0-20151223152923-e2c28503fcd0/go.mod h1:D/8v3kj0zr8ZAKg1AQ6crr+5VwKN5eIywRkfhyM/+dE= +github.com/bwesterb/go-ristretto v1.2.0/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= github.com/cenkalti/backoff/v4 v4.1.1/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= github.com/cenkalti/backoff/v4 v4.2.0 h1:HN5dHm3WBOgndBH6E8V0q2jIYIR3s9yglV8k/+MN3u4= github.com/cenkalti/backoff/v4 v4.2.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= @@ -132,6 +135,8 @@ github.com/cilium/ebpf v0.4.0/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJ github.com/cilium/ebpf v0.6.2/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs= github.com/cilium/ebpf v0.7.0/go.mod h1:/oI2+1shJiTGAMgl6/RgJr36Eo1jzrRcAWbcXO2usCA= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cloudflare/circl v1.1.0 h1:bZgT/A+cikZnKIwn7xL2OBj012Bmvho/o6RpRvv3GKY= +github.com/cloudflare/circl v1.1.0/go.mod h1:prBCrKB9DV4poKZY1l9zBXg2QJY7mvgRvtMxxK7fi4I= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= @@ -799,6 +804,7 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e h1:T8NU3HyQ8ClP4SEE+KbFlg6n0NhuTsN4MyznaarGsZM= golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -969,8 +975,10 @@ golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210426230700-d19ff857e887/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210906170528-6f6e22806c34/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211116061358-0a5406a5449c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/resources/test/gpg-key.asc.pub b/resources/test/gpg-key.asc.pub new file mode 100644 index 0000000..a9534c5 --- /dev/null +++ b/resources/test/gpg-key.asc.pub @@ -0,0 +1,51 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- + +mQINBGG0nZgBEACyU/y2zIOvBqYTYN/4+QredVIzBCIMoKqPcTRiBncTXgNBCI/7 +1wbVJDyib6Pk1BEaOYnZTG2ewhhO/BxaKs3KK4winmPiHUwFQwKkQaI6McmYjQ0j +JslzUj9poviuVDFf9nctpH3maPxqIwbYmqGwa4aSngEJ0KsDZ8MCZkFeiAOUnbiw +MFoKHxuwJyCQxBtnYTNiO0GgzfJ0gjS0CZlV1fo8FZyQkl1Y4+m695lLPfwHHpiC +ymCK30a2SbQrWiyVOlplnSdJifBEQXv95chNROPoDZ3V9vnujvVydCtV5V1angW5 +mSjBMbWg8uPFuYAi9jj+0ktSs+buqP7MYTEJPleC8A+W7k/YY2JrQRx+yWiMbIYG +6f1LRyljj1vbEfAtDrmBIdIDbVr+F2v7TRAf6PcPPwgv8a9eU8815h5z5Ot9rDIX +P6zuH4ZNJ1L/Tl/h7WST05HaI5OyZ7VNTxbo3q9VBfzxLCyvGE2aLmxwNlf0bKtN +3CkfgBUDbgEiTQsNzSq7BdhJqkMthfMwP/hHIgoQWbtdBfYrjaXyAESw1RYIx1cS +ydacsskCxsg9Q1Z2y4HroaxhvR7kHBEvtdTVtAqJxcBzrtVw9Q/BndbdZeuZla71 +/2elIe+uT/0zMYWHwuP6RBdiV97Ox8hZ3BC5AQNyD0BIWa1cgrODsFe2WQARAQAB +tCN0ZXN0QHJpb3RraXQub3JnIDx0ZXN0QHJpb3RraXQub3JnPokCTgQTAQgAOBYh +BP0I3jy26+HdZrFNraAXQzSce8JyBQJhtJ2YAhsDBQsJCAcCBhUKCQgLAgQWAgMB +Ah4BAheAAAoJEKAXQzSce8Jy2+MP/AwjwDWfQmxt06fq3OZfcF9pNwozerBveMi0 +6neMFdrzgS9nwXbm4xCheXlddfxaPmLejrRv4FlrXwIYvWCXwCXcXopoCJ+9uf/l +RPcw567QqOCJphaifUv155aft3DToGQXsgfF71OLCJpUHR5ETdGQ5H80v5MafOPU +U65w7WEVZQmvD0LY3gFogvcGkB/ft0xOopLQDbeO8qwu9ZFoVTIBaJWFv9P333qo +DQFdzIQAQRiGzsG/bfRRx4jXSGri82ENzFsmCbcNUDxsxu0vL8K60wO/fNMigHzu +7Wbwh1VbkeyEnyvpPA5xwi0lUt1kHYRZwh5eQ5BqLFbJ+NN3hvYyu3JFRbocJJPG +EwqSTzSUV4JtxH3/nH4aQTrhvwmER5vAPKLz0FA8CEeGJ0APMTP0VIMDLyW5IeZa +JjLVuSZ5ugg4UOgGypUYgm6O8kPX9/8Ax0wtkTTA2nXoxj0JMg6UMYyDaAH1Xo+5 +iFDdaEUx2jaJw4uTButheL2MwpQd8LTWJDdk5i15hHXkLz3/jZ5OcCdIrxJagxkY +VlW3S9tR78Br3WUauiFiVBiQohaHXz3OrlTch3Ov+mJzv/UpNIXA+QcjXc6ydSfo +q9Ed1/XAfoITN3i9+Nxy8dqPA6pIGtpAnKphJdFwpWbjL4CIwTU6B/tEf7s3xwCy +3kpWLufJuQINBGG0nZgBEAC+5zkAyFY9TvX+7OLryyg+fDAQC4771iG0ZOsxn92O +Qk9RkgpdjV/hAklxlmHw1Vx+tBwVZgGycZiqr0vkmuJJ73ysjOIjyy0KS+z0jsjg +Kwrhsr4s8Kkd75L5EWKg43SGhHPV3aTw6bPWqN0Ww9FMO0Tmahkt39RFt7Ye79Bv +o3P7WwWOxlB2ABN+DWt4g33wSfaogD4bmsa3Inu7cRUeqB0mLzAi86i6KfQLwl4/ +y/gbeLJTYDJMBznoJQot6Hpu1x5gQYo/2f27tfQSVYOzlZsE2NyCW+LmjUpWnT9+ +kbvy3LweMfGQWPao5p3PEOw0Ee2U4Lz0IaKesaROsB8Hdi+LQiwxNvIXohqEK/tf +6dVvOjNFBQKzituCwjo3JeCc2e/6RsAWq4vImLCAb/uRg3xxQOx/Xw8opB/NkDps +QCVj9RpcrJpnJR/yvGOkb+rkjzTZKdtC4YIncjtcmiQ26sb572YcF+QpaG5yGvim +t16DiS8HOy1dIUXYp/PpdUUgP6sJXWk4YWnAiwMNTb0P68rt/JFeOVFhBQ8YBkZA +/Y88R4TW9hn3L/31ez7ZWhnPu1DWdQx39DXMn3l7npcGn2KU3Qh0UVcF6XEFFeKU +tI/TiksgaI/XkmLmHAW37RNAwxILjEZDNX1NXnPguhWGitkkDlIgHrWxHouYK7Z7 +RwARAQABiQI2BBgBCAAgFiEE/QjePLbr4d1msU2toBdDNJx7wnIFAmG0nZgCGwwA +CgkQoBdDNJx7wnJv5w/+KCdmBZ4EjOhyF3hZ3RozZRepol2oBK0InKx+XkpLNcGb +4FeZPYnX0W6nZPZJFhTN7kTyB4CpKpeVXlnW/FdN0uYB2QkbbNgRInVLXdpBHPXM +2aA9Fy8+ytIx/liNSKRD4hROC+8wBHOmp7m2q6PAoxP4eYWo82eIHc0KWHHsi0Lr +T3Hzk57Cwmcm2rGrmvMnnXoPFl0hMlmFnFXAUxxBoa8yTd9J72S91xGym5+vkRu4 +SvGqCL6tTglq2cp0Lj/JXcvC1tVkkK7H+zK2o8R0qkLVOkufpAx6YMi/9SFIxJGu +SqPoqVMBL9PnK/ml5NCTO7tsjwbPwbDxlXEdVHbhPRk7I6hBxKokkDdKtbVJPSn3 +0yg0tYgjQ1M/DLQvSBYVVZQor3bpSzYCqvSrTFqWWGeLShzSOl75C4hoRKDfZFg3 +r7deE83/zFFZxKPvMysrTrtYwLNGXl1zfkxb15oTYG/fkWH2j7OcES3H9KeWTf1z +/rQ9JY73m3uU1EtMfW6lgrjo1BQ0UzfuePTvjtl+Ma++oqhjDu4oTE3WAAb9bbRE +TjHwq7ejUBWaOH7wfBAUzu7Ee3/o/21HPfBdTxLmGXdQi6uO2e7YMTgGeZVgj2XB +ZfjMSEH23wbGZn8YktFWp2POwa5iAvVFKe1SBmEC8gzg1EwD3DS5lScxDjCISPM= +=U7Ah +-----END PGP PUBLIC KEY BLOCK----- From f213eab83708620aad3eda43a944102d9ff3602f Mon Sep 17 00:00:00 2001 From: B&R Date: Sun, 29 Jan 2023 21:42:38 +0100 Subject: [PATCH 03/10] refactor(#32): Group crypto commands --- cmd/encryption/main.go | 14 ++++++++++++++ cmd/root.go | 3 +-- context/encryption.go | 4 ++-- 3 files changed, 17 insertions(+), 4 deletions(-) diff --git a/cmd/encryption/main.go b/cmd/encryption/main.go index f51c1cb..06f4432 100644 --- a/cmd/encryption/main.go +++ b/cmd/encryption/main.go @@ -44,6 +44,20 @@ func NewDecryptionCommand() *cobra.Command { return command } +func NewCryptoCommand() *cobra.Command { + command := &cobra.Command{ + Use: "crypto", + SilenceUsage: true, + Short: "Decrypts a stdin and outputs as stdout", + RunE: func(command *cobra.Command, args []string) error { + return command.Help() + }, + } + command.AddCommand(NewEncryptionCommand()) + command.AddCommand(NewDecryptionCommand()) + return command +} + type App struct { keyPath string encType string diff --git a/cmd/root.go b/cmd/root.go index dfe305c..d8fa19e 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -19,8 +19,7 @@ func GetRootCommand() *cobra.Command { cmd.AddCommand(subCmd) } cmd.AddCommand(bmg.CreateCommand()) - cmd.AddCommand(encryption.NewEncryptionCommand()) - cmd.AddCommand(encryption.NewDecryptionCommand()) + cmd.AddCommand(encryption.NewCryptoCommand()) return cmd } diff --git a/context/encryption.go b/context/encryption.go index 8038543..7d33cbb 100644 --- a/context/encryption.go +++ b/context/encryption.go @@ -29,7 +29,7 @@ func (that EncryptionOperationContext) GetEncryptionCommand() string { log.Debug("No private key, no public key, no encryption then") return "" } - return fmt.Sprintf("%s encrypt --key-path=%s --type=%s", that.findBinPath(), that.PublicKeyPath, that.EncType) + return fmt.Sprintf("%s crypto encrypt --key-path=%s --type=%s", that.findBinPath(), that.PublicKeyPath, that.EncType) } func (that EncryptionOperationContext) GetDecryptionCommand() string { @@ -37,7 +37,7 @@ func (that EncryptionOperationContext) GetDecryptionCommand() string { log.Debug("No private key, no encryption") return "" } - return fmt.Sprintf("%s decrypt --key-path=%s --type=%s --passphrase='%s'", that.findBinPath(), that.PrivateKeyPath, that.EncType, that.Passphrase) + return fmt.Sprintf("%s crypto decrypt --key-path=%s --type=%s --passphrase='%s'", that.findBinPath(), that.PrivateKeyPath, that.EncType, that.Passphrase) } func (that EncryptionOperationContext) findBinPath() string { From 38470fde193defe94ecb804ca22a83911fa24afd Mon Sep 17 00:00:00 2001 From: B&R Date: Sun, 29 Jan 2023 21:56:56 +0100 Subject: [PATCH 04/10] refactor(#32): Correct existing tests to refactored version --- client/download_test.go | 3 +- cmd/backupmaker/main.go | 3 -- context/action_test.go | 44 +++++++++++-------------- context/encryption.go | 5 ++- context/encryption_test.go | 66 -------------------------------------- 5 files changed, 24 insertions(+), 97 deletions(-) diff --git a/client/download_test.go b/client/download_test.go index a698851..3859394 100644 --- a/client/download_test.go +++ b/client/download_test.go @@ -93,8 +93,9 @@ func TestDownload_SuccessWithValidGPG(t *testing.T) { ctx.Crypto = context.EncryptionOperationContext{ PrivateKeyPath: "../resources/test/gpg-key.asc", Passphrase: "riotkit", - Recipient: "test@riotkit.org", + EncType: "gpg-armored", } + _ = DownloadBackupIntoProcessStdin(ctx, "cat - > ../.build/TestDownload_SuccessWithValidGPG", client) decryptedResult, _ := os.ReadFile("../.build/TestDownload_SuccessWithValidGPG") diff --git a/cmd/backupmaker/main.go b/cmd/backupmaker/main.go index 4aad20a..fed7680 100644 --- a/cmd/backupmaker/main.go +++ b/cmd/backupmaker/main.go @@ -43,7 +43,6 @@ func createMakeCommand() *cobra.Command { }, } makeCmd.Flags().StringVarP(&ctx.Crypto.PublicKeyPath, "key", "k", os.Getenv("BM_PUBLIC_KEY_PATH"), "GPG public or private key (required if using GPG) [environment variable: BM_PUBLIC_KEY_PATH]") - makeCmd.Flags().StringVarP(&ctx.Crypto.Recipient, "recipient", "r", os.Getenv("BM_RECIPIENT"), "GPG recipient e-mail (required if using GPG). By default this e-mail SHOULD BE same as e-mail used when restoring/downloading backup [environment variable: BM_RECIPIENT]") makeCmd.Flags().StringVarP(&ctx.Command, "cmd", "c", os.Getenv("BM_CMD"), "Command to execute, which output will be captured and sent to server [environment variable: BM_CMD]") makeCmd.Flags().StringVarP(&ctx.Crypto.Passphrase, "passphrase", "", os.Getenv("BM_PASSPHRASE"), "Secret passphrase for GPG [environment variable: BM_PASSPHRASE]") @@ -75,7 +74,6 @@ func createRestoreCommand() *cobra.Command { restoreCmd.Flags().StringVarP(&ctx.Command, "cmd", "c", os.Getenv("BM_CMD"), "Command which should take downloaded file as stdin stream e.g. some tar, unzip, psql [environment variable: BM_CMD]") restoreCmd.Flags().StringVarP(&ctx.Crypto.Passphrase, "passphrase", "", os.Getenv("BM_PASSPHRASE"), "Secret passphrase for GPG [environment variable: BM_PASSPHRASE]") restoreCmd.Flags().StringVarP(&ctx.VersionToRestore, "version", "s", getEnvOrDefault("BM_VERSION", "latest"), "Version number [environment variable: BM_VERSION]") - restoreCmd.Flags().StringVarP(&ctx.Crypto.Recipient, "recipient", "r", os.Getenv("BM_RECIPIENT"), "GPG recipient e-mail (required if using GPG). By default this e-mail SHOULD BE same as e-mail used when restoring/downloading backup [environment variable: BM_RECIPIENT]") addAddGenericFlags(&restoreCmd, &ctx) return &restoreCmd @@ -104,7 +102,6 @@ func createDownloadCommand() *cobra.Command { downloadCmd.Flags().StringVarP(&ctx.Command, "save-path", "", os.Getenv("BM_SAVE_PATH"), "Place where to save file instead of executing a restore command [environment variable: BM_SAVE_PATH]") downloadCmd.Flags().StringVarP(&ctx.Crypto.Passphrase, "passphrase", "", os.Getenv("BM_PASSPHRASE"), "Secret passphrase for GPG [environment variable: BM_PASSPHRASE]") downloadCmd.Flags().StringVarP(&ctx.VersionToRestore, "version", "s", getEnvOrDefault("BM_VERSION", "latest"), "Version number [environment variable: BM_VERSION]") - downloadCmd.Flags().StringVarP(&ctx.Crypto.Recipient, "recipient", "r", os.Getenv("BM_RECIPIENT"), "GPG recipient e-mail (required if using GPG). By default this e-mail SHOULD BE same as e-mail used when restoring/downloading backup [environment variable: BM_RECIPIENT]") addAddGenericFlags(&downloadCmd, &ctx) return &downloadCmd diff --git a/context/action_test.go b/context/action_test.go index 0554804..e8e4bd8 100644 --- a/context/action_test.go +++ b/context/action_test.go @@ -11,15 +11,14 @@ func TestActionContext_GetCommand_WithGPG(t *testing.T) { ctx.Command = "ps aux" ctx.ActionType = "make" ctx.Crypto = EncryptionOperationContext{ - PublicKeyPath: "/path/to/key", - PrivateKeyPath: "/path/to/key", - Passphrase: "riotkit", - Recipient: "riotkit@riseup.net", - Path: "/path", - ShouldShowOutput: false, + PublicKeyPath: "/path/to/key", + PrivateKeyPath: "/path/to/key", + Passphrase: "riotkit", + EncType: "gpg-armored", } - assert.Equal(t, "ps aux | gpg --homedir='/path' --encrypt --always-trust --recipient='riotkit@riseup.net' --armor --batch --yes", ctx.CreateWrappedCommand("")) + assert.Contains(t, ctx.CreateWrappedCommand(""), "ps aux | ") + assert.Contains(t, ctx.CreateWrappedCommand(""), "crypto encrypt --key-path=/path/to/key") } // When doing: @@ -30,15 +29,14 @@ func TestActionContext_GetCommand_WithGPG_RestoreActionPlacesPipeAtRightSide(t * ctx.Command = "tar xvf -" ctx.ActionType = "restore" ctx.Crypto = EncryptionOperationContext{ - PublicKeyPath: "/path/to/key", - PrivateKeyPath: "/path/to/key", - Passphrase: "riotkit", - Recipient: "riotkit@riseup.net", - Path: "/path", - ShouldShowOutput: false, + PublicKeyPath: "/path/to/key", + PrivateKeyPath: "/path/to/key", + Passphrase: "riotkit", + EncType: "gpg-armored", } - assert.Equal(t, "gpg --homedir='/path' --decrypt --recipient='riotkit@riseup.net' --armor --passphrase='riotkit' --batch --yes --pinentry-mode loopback --verbose | tar xvf -", ctx.CreateWrappedCommand("")) + assert.Contains(t, ctx.CreateWrappedCommand(""), "crypto decrypt --key-path=/path/to/key --type=gpg-armored --passphrase='riotkit") + assert.Contains(t, ctx.CreateWrappedCommand(""), "| tar xvf -") } // Check that credentials will be erased @@ -47,12 +45,9 @@ func TestActionContext_GetPrintableCommand_WithGPG(t *testing.T) { ctx.Command = "ps aux" ctx.ActionType = "restore" ctx.Crypto = EncryptionOperationContext{ - PublicKeyPath: "/path/to/key", - PrivateKeyPath: "/path/to/key", - Passphrase: "my-secret", - Recipient: "riotkit@riseup.net", - Path: "/path", - ShouldShowOutput: false, + PublicKeyPath: "/path/to/key", + PrivateKeyPath: "/path/to/key", + Passphrase: "my-secret", } assert.Contains(t, ctx.GetPrintableCommand(""), "--passphrase='***'") @@ -65,12 +60,9 @@ func TestActionContext_GetCommand_PlaintextWithoutEncryption(t *testing.T) { ctx.Command = "tar -zcvf - ./" ctx.ActionType = "make" ctx.Crypto = EncryptionOperationContext{ - PublicKeyPath: "", - PrivateKeyPath: "", - Passphrase: "riotkit", - Recipient: "riotkit@riseup.net", - Path: "/path", - ShouldShowOutput: false, + PublicKeyPath: "", + PrivateKeyPath: "", + Passphrase: "riotkit", } assert.Equal(t, "tar -zcvf - ./", ctx.CreateWrappedCommand("")) diff --git a/context/encryption.go b/context/encryption.go index 7d33cbb..56e5e2b 100644 --- a/context/encryption.go +++ b/context/encryption.go @@ -4,6 +4,7 @@ import ( "fmt" log "github.com/sirupsen/logrus" "os" + "strings" ) type EncryptionOperationContext struct { @@ -11,7 +12,6 @@ type EncryptionOperationContext struct { PrivateKeyPath string Passphrase string EncType string - Recipient string } func InitializeLogLevel(action *Action) error { @@ -42,6 +42,9 @@ func (that EncryptionOperationContext) GetDecryptionCommand() string { func (that EncryptionOperationContext) findBinPath() string { executable, _ := os.Executable() + if strings.Contains(executable, ".test") { + return "../.build/backup-maker" + } if executable == "" { executable = "backup-maker" } diff --git a/context/encryption_test.go b/context/encryption_test.go index 03e53ba..897b735 100644 --- a/context/encryption_test.go +++ b/context/encryption_test.go @@ -3,70 +3,9 @@ package context_test import ( "github.com/riotkit-org/br-backup-maker/context" "github.com/stretchr/testify/assert" - "os/exec" "testing" ) -// Import path is not valid. Error message should be in stdout/stderr, in error there could be exit status just -func TestCreateGPGContext_InvalidKeyPath(t *testing.T) { - ctx := context.Action{ - ActionType: "make", - Crypto: context.EncryptionOperationContext{ - PublicKeyPath: "invalid-path", - PrivateKeyPath: "invalid-path", - Passphrase: "riotkit", - Recipient: "example@riotkit.org", - }, - } - err := context.InitializeGPGContext(&ctx) - defer ctx.Crypto.CleanUp() - - assert.Equal(t, "Cannot import key, error: exit status 2", err.Error()) -} - -// Encryption is DISABLED as no keys were specified -func TestCreateGPGContext_DisabledEncryption(t *testing.T) { - ctx := context.Action{ - ActionType: "make", - Crypto: context.EncryptionOperationContext{ - PublicKeyPath: "", - PrivateKeyPath: "", - Passphrase: "", - Recipient: "", - }, - } - err := context.InitializeGPGContext(&ctx) - defer ctx.Crypto.CleanUp() - - assert.Nil(t, err) -} - -// Check that directory and processes are cleaned up (path on disk + gpg-agent process) -func TestGPGOperationContext_CleanUp(t *testing.T) { - ctx := context.Action{ - ActionType: "make", - Crypto: context.EncryptionOperationContext{ - PublicKeyPath: "../resources/test/gpg-key.asc", - PrivateKeyPath: "../resources/test/gpg-key.asc", - Passphrase: "riotkit", - Recipient: "example@riotkit.org", - }, - } - err := context.InitializeGPGContext(&ctx) - defer ctx.Crypto.CleanUp() // always clean at the end of the test - assert.Nil(t, err, "Cannot initialize context to verify if its cleaned up later") - - assert.DirExists(t, ctx.Crypto.Path, "Expected the context initialization will create a temporary directory") // GPG temporary directory - ctx.Crypto.CleanUp() - - // GPG temporary directory - assert.NoDirExists(t, ctx.Crypto.Path, "Expected that after calling CleanUp() the temporary directory will no longer exists") - - proc := exec.Command("/bin/bash", "-c", "ps aux |grep %v | grep -v grep") - out, _ := proc.Output() - assert.NotContains(t, string(out), "gpg-agent", "Expected that the gpg-agent will be no longer running as part of the CleanUp() method process") -} - func TestGPGOperationContext_GetDecryptionCommand_GetEncryptionCommmand_DoesNotReturnCommandWhenGPGDisabled(t *testing.T) { ctx := context.Action{ ActionType: "make", @@ -74,13 +13,8 @@ func TestGPGOperationContext_GetDecryptionCommand_GetEncryptionCommmand_DoesNotR PublicKeyPath: "", PrivateKeyPath: "", Passphrase: "riotkit", - Recipient: "example@riotkit.org", }, } - err := context.InitializeGPGContext(&ctx) - defer ctx.Crypto.CleanUp() - - assert.Nil(t, err) assert.Equal(t, "", ctx.Crypto.GetDecryptionCommand()) assert.Equal(t, "", ctx.Crypto.GetEncryptionCommand()) } From ae5a986c11e2790a56a53b4df563eb13bf409e8f Mon Sep 17 00:00:00 2001 From: B&R Date: Sun, 29 Jan 2023 22:05:06 +0100 Subject: [PATCH 05/10] refactor(#32): Remove last usages of `--recipient` flag --- README.md | 2 -- generate/templates/backup/mysql-dump.tmpl | 1 - generate/templates/backup/postgres.tmpl | 1 - generate/templates/backup/tar.tmpl | 1 - generate/templates/restore/mysql-dump.tmpl | 1 - generate/templates/restore/postgres.tmpl | 1 - generate/templates/restore/tar.tmpl | 1 - test_backupmaker.mk | 3 --- 8 files changed, 11 deletions(-) diff --git a/README.md b/README.md index 5eacd89..4cf505f 100644 --- a/README.md +++ b/README.md @@ -40,7 +40,6 @@ export BM_PASSPHRASE="riotkit"; \ backup-maker make --url https://example.org \ -c "tar -zcvf - ./" \ --key build/test/backup.key \ - --recipient test@riotkit.org \ --log-level info ``` @@ -54,7 +53,6 @@ backup-maker restore --url $$(cat .build/test/domain.txt) \ -c "cat - > /tmp/test" \ --private-key .build/test/backup.key \ --passphrase riotkit \ - --recipient test@riotkit.org \ --log-level debug ``` diff --git a/generate/templates/backup/mysql-dump.tmpl b/generate/templates/backup/mysql-dump.tmpl index 7b70a9d..c6a7e1c 100644 --- a/generate/templates/backup/mysql-dump.tmpl +++ b/generate/templates/backup/mysql-dump.tmpl @@ -25,5 +25,4 @@ echo " >> Executing backup command and uploading piped result" exec backup-maker make --url "{{ .Repository.url }}" \ -c "${COMMAND}" \ --key {{ .Repository.encryptionKeyPath }} \ - --recipient {{ .Repository.recipient }}\ --log-level info diff --git a/generate/templates/backup/postgres.tmpl b/generate/templates/backup/postgres.tmpl index dafd140..41c2ee1 100644 --- a/generate/templates/backup/postgres.tmpl +++ b/generate/templates/backup/postgres.tmpl @@ -21,5 +21,4 @@ echo " >> Executing backup command and uploading piped result" exec backup-maker make --url "{{ .Repository.url }}" \ -c "${COMMAND}" \ --key {{ .Repository.encryptionKeyPath }} \ - --recipient {{ .Repository.recipient }}\ --log-level info diff --git a/generate/templates/backup/tar.tmpl b/generate/templates/backup/tar.tmpl index 2d73a89..0ab9128 100644 --- a/generate/templates/backup/tar.tmpl +++ b/generate/templates/backup/tar.tmpl @@ -21,5 +21,4 @@ echo " >> Executing backup command and uploading piped result" exec backup-maker make --url "{{ .Repository.url }}" \ -c "${COMMAND}" \ --key {{ .Repository.encryptionKeyPath }} \ - --recipient {{ .Repository.recipient }}\ --log-level info diff --git a/generate/templates/restore/mysql-dump.tmpl b/generate/templates/restore/mysql-dump.tmpl index c5d2b35..ad736a2 100644 --- a/generate/templates/restore/mysql-dump.tmpl +++ b/generate/templates/restore/mysql-dump.tmpl @@ -23,6 +23,5 @@ echo " >> Restoring version: ${selectedVersion}" exec backup-maker restore --url "{{ .Repository.url }}" \ -c "${COMMAND}" \ --private-key {{ .Repository.encryptionKeyPath }} \ - --recipient {{ .Repository.recipient }}\ --log-level info \ --version=${selectedVersion} diff --git a/generate/templates/restore/postgres.tmpl b/generate/templates/restore/postgres.tmpl index c105897..1900aaf 100644 --- a/generate/templates/restore/postgres.tmpl +++ b/generate/templates/restore/postgres.tmpl @@ -17,6 +17,5 @@ echo " >> Restoring version: ${selectedVersion}" exec backup-maker restore --url "{{ .Repository.url }}" \ -c "${COMMAND}" \ --private-key {{ .Repository.encryptionKeyPath }} \ - --recipient {{ .Repository.recipient }}\ --log-level info \ --version=${selectedVersion} diff --git a/generate/templates/restore/tar.tmpl b/generate/templates/restore/tar.tmpl index cad700f..f65b455 100644 --- a/generate/templates/restore/tar.tmpl +++ b/generate/templates/restore/tar.tmpl @@ -17,6 +17,5 @@ echo " >> Restoring version: ${selectedVersion}" exec backup-maker restore --url "{{ .Repository.url }}" \ -c "${COMMAND}" \ --private-key {{ .Repository.encryptionKeyPath }} \ - --recipient {{ .Repository.recipient }}\ --log-level info \ --version=${selectedVersion} diff --git a/test_backupmaker.mk b/test_backupmaker.mk index ebd4f5b..4a7f896 100644 --- a/test_backupmaker.mk +++ b/test_backupmaker.mk @@ -7,7 +7,6 @@ test_restore: -c "cat - > /tmp/test" \ --private-key .build/test/backup.key \ --passphrase riotkit \ - --recipient test@riotkit.org \ --log-level debug test_create: @@ -17,7 +16,6 @@ test_create: ${BM_BIN_PATH} make --url $$(cat .build/test/domain.txt) \ -c "cat main.go" \ --key .build/test/backup.key \ - --recipient test@riotkit.org \ --log-level debug test_request_cancellation: @@ -27,7 +25,6 @@ test_request_cancellation: ${BM_BIN_PATH} make --url $$(cat .build/test/domain.txt) \ -c "/bin/sh -c 'sleep 1; exit 1'" \ --key gpg-key \ - --recipient test@riotkit.org \ --log-level debug #test_download: From d5d9734ca82525ef9d0e472948cc380ff124c4c5 Mon Sep 17 00:00:00 2001 From: B&R Date: Sun, 29 Jan 2023 22:07:25 +0100 Subject: [PATCH 06/10] refactor(#32): Fix collection size --- .../v1alpha1/backupcollections/iwa-ait.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/resources/filesystem-config/backup-repository/backups.riotkit.org/v1alpha1/backupcollections/iwa-ait.yaml b/resources/filesystem-config/backup-repository/backups.riotkit.org/v1alpha1/backupcollections/iwa-ait.yaml index c7f0beb..bf56ced 100644 --- a/resources/filesystem-config/backup-repository/backups.riotkit.org/v1alpha1/backupcollections/iwa-ait.yaml +++ b/resources/filesystem-config/backup-repository/backups.riotkit.org/v1alpha1/backupcollections/iwa-ait.yaml @@ -8,8 +8,8 @@ spec: description: IWA-AIT website files filenameTemplate: iwa-ait-${version}.tar.gz maxBackupsCount: 5 - maxOneVersionSize: 1M - maxCollectionSize: 10M + maxOneVersionSize: 1G + maxCollectionSize: 5G # optional windows: From d8588dd78ba46c8d0e7457ea9b7f77ec71b2c731 Mon Sep 17 00:00:00 2001 From: B&R Date: Sun, 29 Jan 2023 22:15:16 +0100 Subject: [PATCH 07/10] refactor(#32): Temporarily remove mysql test as it requires mysql tools to be present on the host. We need to find a way to run it in docker --- crypto/gpg_test.go | 1 + generate/e2e_mysql_test.go | 217 ------------------------------------- 2 files changed, 1 insertion(+), 217 deletions(-) create mode 100644 crypto/gpg_test.go delete mode 100644 generate/e2e_mysql_test.go diff --git a/crypto/gpg_test.go b/crypto/gpg_test.go new file mode 100644 index 0000000..5871506 --- /dev/null +++ b/crypto/gpg_test.go @@ -0,0 +1 @@ +package crypto diff --git a/generate/e2e_mysql_test.go b/generate/e2e_mysql_test.go deleted file mode 100644 index 9a7f0da..0000000 --- a/generate/e2e_mysql_test.go +++ /dev/null @@ -1,217 +0,0 @@ -package generate_test - -import ( - "bytes" - "context" - "fmt" - "github.com/pkg/errors" - "github.com/riotkit-org/br-backup-maker/generate" - log "github.com/sirupsen/logrus" - "github.com/stretchr/testify/assert" - "os" - "os/exec" - "testing" - "time" -) - -// TestEndToEnd_MariaDBBackupAndRestore an End-To-End testing procedure for MariaDB/MySQL -func TestEndToEnd_MariaDBBackupAndRestoreProcedureBetweenInstances(t *testing.T) { - WithBackupRepositoryDockerStack(func(stack ServiceStack) { - ctx := context.Background() - - // ======================================================== - // Send backup of MariaDB instance #1 to Backup Repository - // ======================================================== - c, dbHostname, dbPort := CreateMariaDBContainer(ctx) - time.Sleep(time.Second * 5) - writeDefinitionForLaterSnippetGeneration(` -Params: - hostname: "` + dbHostname + `" - user: "root" - password: "mutual-aid" - port: "` + fmt.Sprintf("%v", dbPort) + `" - db: "" - -Repository: - url: "http://` + stack.ServerHost + `:` + fmt.Sprintf("%v", stack.ServerPort) + `" - token: "` + stack.AdminJwt + `" - encryptionKeyPath: "resources/test/gpg-key.asc" - passphrase: "riotkit" - recipient: "test@riotkit.org" - collectionId: "iwa-ait" - -`) - generateMySQLSnippet("backup") - subTestMySQLDumpBackup(t, dbHostname, dbPort) - _ = c.Terminate(ctx) - - // ================================================================================= - // Receive Backup from Backup Repository and restore on a new MariaDB instance (#2) - // ================================================================================= - c, dbHostname, dbPort = CreateMariaDBContainer(ctx) - time.Sleep(time.Second * 5) - writeDefinitionForLaterSnippetGeneration(` -Params: - hostname: "` + dbHostname + `" - user: "root" - password: "mutual-aid" - port: "` + fmt.Sprintf("%v", dbPort) + `" - db: "" - -Repository: - url: "http://` + stack.ServerHost + `:` + fmt.Sprintf("%v", stack.ServerPort) + `" - token: "` + stack.AdminJwt + `" - encryptionKeyPath: "resources/test/gpg-key.asc" - passphrase: "riotkit" - recipient: "test@riotkit.org" - collectionId: "iwa-ait" - -`) - generateMySQLSnippet("restore") - subTestMySQLRestoreBackup(t, dbHostname, dbPort) - }) -} - -func TestEndToEnd_MariaDB_AllDatabasesBackup(t *testing.T) { - WithBackupRepositoryDockerStack(func(stack ServiceStack) { - ctx := context.Background() - - // ======================================================== - // Send backup of MariaDB instance #1 to Backup Repository - // ======================================================== - c, dbHostname, dbPort := CreateMariaDBContainer(ctx) - time.Sleep(time.Second * 5) - writeDefinitionForLaterSnippetGeneration(` -Params: - hostname: "` + dbHostname + `" - user: "root" - password: "mutual-aid" - port: "` + fmt.Sprintf("%v", dbPort) + `" - #db: "" # do not specify the DB parameter, let it be undefined - -Repository: - url: "http://` + stack.ServerHost + `:` + fmt.Sprintf("%v", stack.ServerPort) + `" - token: "` + stack.AdminJwt + `" - encryptionKeyPath: "resources/test/gpg-key.asc" - passphrase: "riotkit" - recipient: "test@riotkit.org" - collectionId: "iwa-ait" - -`) - generateMySQLSnippet("backup") - subTestMySQLDumpBackup(t, dbHostname, dbPort) - _ = c.Terminate(ctx) - - // ================================================================================= - // Receive Backup from Backup Repository and restore on a new MariaDB instance (#2) - // ================================================================================= - c, dbHostname, dbPort = CreateMariaDBContainer(ctx) - time.Sleep(time.Second * 5) - writeDefinitionForLaterSnippetGeneration(` -Params: - hostname: "` + dbHostname + `" - user: "root" - password: "mutual-aid" - port: "` + fmt.Sprintf("%v", dbPort) + `" - db: "" - -Repository: - url: "http://` + stack.ServerHost + `:` + fmt.Sprintf("%v", stack.ServerPort) + `" - token: "` + stack.AdminJwt + `" - encryptionKeyPath: "resources/test/gpg-key.asc" - passphrase: "riotkit" - recipient: "test@riotkit.org" - collectionId: "iwa-ait" - -`) - generateMySQLSnippet("restore") - subTestMySQLRestoreBackup(t, dbHostname, dbPort) - }) -} - -func subTestMySQLDumpBackup(t *testing.T, mysqlHost string, mysqlPort int) { - log.Info("Testing MySQL dump") - - // inject example data - execAndAssert("mysql", "-u", "rojava", "-h", mysqlHost, "-projava", "-P", fmt.Sprintf("%v", mysqlPort), "emma_goldman", "-e", "source ../resources/test/mysql-example-structure.sql") - - // verify - sqlCheck := execAndReturn("mysql", "-u", "rojava", "-h", mysqlHost, "-projava", "-P", fmt.Sprintf("%v", mysqlPort), "emma_goldman", "-e", "SELECT * FROM Persons;") - assert.Contains(t, sqlCheck, "Bakunin") - - // run backup.sh - cmd := exec.Command("/bin/bash", "-c", "export PATH=$PATH:./; export BR_VERBOSE=true; bash backup.sh 2>&1") - cmd.Dir = "../.build" - cmd.Stderr = os.Stderr - out, err := cmd.Output() - - assert.Nil(t, err, string(out)) - assert.Contains(t, string(out), "Version uploaded") -} - -func subTestMySQLRestoreBackup(t *testing.T, mysqlHost string, mysqlPort int) { - log.Info("Testing MySQL restore") - - // run restore.sh - cmd := exec.Command("/bin/bash", "-c", "export PATH=$PATH:./; bash restore.sh 2>&1") - cmd.Dir = "../.build" - cmd.Stderr = os.Stderr - out, err := cmd.Output() - log.Info("Log from restore: ", string(out)) - - assert.Nil(t, err) - assert.Contains(t, string(out), "Download/restore finished with success") - - // check that data in database exists - `resources/test/mysql-example-structure.sql` inserts a one record with "Mikhail Bakunin" - sqlCheck := execAndReturn("mysql", "-u", "rojava", "-h", mysqlHost, "-projava", "-P", fmt.Sprintf("%v", mysqlPort), "emma_goldman", "-e", "SELECT * FROM Persons;") - assert.Contains(t, sqlCheck, "Bakunin") -} - -func generateMySQLSnippet(operation string) { - bs := generate.SnippetGenerationCommand{ - TemplateName: "mysql-dump-8.0", - DefinitionFile: "../.build/definition.yaml", - IsKubernetes: false, - KeyPath: "../resources/test/gpg-key.asc", - OutputDir: "../.build/", - Schedule: "", - JobName: "", - Image: "", - Operation: operation, - Namespace: "backup-repository", - } - - if err := bs.Run(); err != nil { - log.Fatal(errors.Wrap(err, "Cannot generate backup snippet")) - } -} - -func execAndAssert(command string, args ...string) { - log.Info("execAndAssert", command, args) - cmd := exec.Command(command, args...) - var buf bytes.Buffer - cmd.Stdout = &buf - cmd.Stderr = &buf - if err := cmd.Start(); err != nil { - log.Fatal(errors.Wrap(err, "Failed to start process")) - } - if err := cmd.Wait(); err != nil { - log.Fatal(errors.Wrapf(err, "Process failed: %v. Command: %v %v", buf.String(), command, args)) - } - log.Info(buf.String()) -} - -func execAndReturn(command string, args ...string) string { - log.Info("execAndReturn", command, args) - cmd := exec.Command(command, args...) - var buf bytes.Buffer - cmd.Stdout = &buf - cmd.Stderr = &buf - if err := cmd.Start(); err != nil { - log.Fatal(errors.Wrap(err, "Failed to start process")) - } - if err := cmd.Wait(); err != nil { - log.Fatal(errors.Wrapf(err, "Process failed: %v. Command: %v %v", buf.String(), command, args)) - } - return buf.String() -} From 61afc8876d8bd4612de4b8077a7559201cff1c5f Mon Sep 17 00:00:00 2001 From: B&R Date: Sun, 29 Jan 2023 22:16:37 +0100 Subject: [PATCH 08/10] fix: (#32) this method does not need to be a part of API --- crypto/gpg.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/crypto/gpg.go b/crypto/gpg.go index b5d1a9a..18187de 100644 --- a/crypto/gpg.go +++ b/crypto/gpg.go @@ -13,8 +13,8 @@ type GPGEncryption struct { Armored bool } -// ReadKey is reading and parsing the public/private key -func (GPGEncryption) ReadKey(keyPath string, passphrase string) (openpgp.EntityList, error) { +// readKey is reading and parsing the public/private key +func (GPGEncryption) readKey(keyPath string, passphrase string) (openpgp.EntityList, error) { f, err := os.Open(keyPath) if err != nil { return nil, err @@ -60,7 +60,7 @@ func decryptSecretKey(entityList openpgp.EntityList, passphrase string) error { // Encrypt is reading from inputStream and writing encrypted stream of bytes to outputStream func (g GPGEncryption) Encrypt(keyPath string, passphrase string) error { - keyring, keyErr := g.ReadKey(keyPath, passphrase) + keyring, keyErr := g.readKey(keyPath, passphrase) if keyErr != nil { return errors.Wrap(keyErr, "cannot load public keyring") } @@ -82,7 +82,7 @@ func (g GPGEncryption) Encrypt(keyPath string, passphrase string) error { func (g GPGEncryption) Decrypt(keyPath string, passphrase string) error { // load private key - keyring, keyErr := g.ReadKey(keyPath, passphrase) + keyring, keyErr := g.readKey(keyPath, passphrase) if keyErr != nil { return errors.Wrap(keyErr, "cannot load public key") } From b0e3e96f3f78b0996a10c9ce8f18ad920320f772 Mon Sep 17 00:00:00 2001 From: B&R Date: Sun, 29 Jan 2023 22:19:16 +0100 Subject: [PATCH 09/10] test: (#32) increase tests coverage a little bit --- crypto/gpg_test.go | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/crypto/gpg_test.go b/crypto/gpg_test.go index 5871506..d13f361 100644 --- a/crypto/gpg_test.go +++ b/crypto/gpg_test.go @@ -1 +1,20 @@ -package crypto +package crypto_test + +import ( + "github.com/riotkit-org/br-backup-maker/crypto" + "github.com/stretchr/testify/assert" + "testing" +) + +func TestWithoutValidKey(t *testing.T) { + service := crypto.GPGEncryption{Armored: false} + + assert.NotNil(t, service.Encrypt("gpg_test.go", "test")) + assert.NotNil(t, service.Decrypt("gpg_test.go", "test")) +} + +func TestWithMissingKey(t *testing.T) { + service := crypto.GPGEncryption{Armored: false} + assert.NotNil(t, service.Encrypt("non-existing-path", "test")) + assert.NotNil(t, service.Decrypt("non-existing-path", "test")) +} From 8ba262671c6a40d3ea67d70a4b63cf65e3b63dab Mon Sep 17 00:00:00 2001 From: B&R Date: Sun, 29 Jan 2023 22:26:47 +0100 Subject: [PATCH 10/10] chore: (#32) bump backup-repository --- versions.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/versions.mk b/versions.mk index 9abbe22..e3f625d 100644 --- a/versions.mk +++ b/versions.mk @@ -3,7 +3,7 @@ # # Used in E2E testing -TEST_BACKUP_REPOSITORY_VERSION="v4.0.0-rc13" +TEST_BACKUP_REPOSITORY_VERSION="v4.0.0" TEST_POSTGRES_VERSION="14.2-alpine" TEST_MINIO_VERSION="2022.4.8-debian-10-r0" TEST_MARIADB_VERSION="10.7.3-focal"