From becde9bff9def51c7ba1ccd00e324b1324efe2b4 Mon Sep 17 00:00:00 2001 From: Victor Date: Fri, 18 Oct 2024 08:45:49 +0000 Subject: [PATCH 1/4] Fixed the removal of bloat files after vacuum full --- pkg/database/database.go | 13 +++++++------ pkg/proc/delete_handler.go | 14 ++------------ 2 files changed, 9 insertions(+), 18 deletions(-) diff --git a/pkg/database/database.go b/pkg/database/database.go index 6b9ebe2..45c4eb7 100644 --- a/pkg/database/database.go +++ b/pkg/database/database.go @@ -81,12 +81,7 @@ func (database *DatabaseHandler) GetVirtualExpireIndexes(port uint64) (map[strin if err := rows2.Scan(&xpath); err != nil { return nil, nil, fmt.Errorf("unable to parse query output %v", err) } - p1 := strings.Split(xpath, "/") - p2 := p1[len(p1)-1] - p3 := strings.Split(p2, "_") - if len(p3) >= 4 { - p2 = fmt.Sprintf("%s_%s_%s_%s_", p3[0], p3[1], p3[2], p3[3]) - } + p2 := ReworkFileName(xpath) c2[p2] = true ylogger.Zero.Debug().Str("file", p2).Msg("added") } @@ -175,3 +170,9 @@ func connectToDatabase(port uint64, database string) (*pgx.Conn, error) { } return conn, nil } + +func ReworkFileName(str string) string { + p1 := strings.Split(str, "/") + p2 := strings.Join(p1, "_") + return p2 +} diff --git a/pkg/proc/delete_handler.go b/pkg/proc/delete_handler.go index 9115ed7..45fbffb 100644 --- a/pkg/proc/delete_handler.go +++ b/pkg/proc/delete_handler.go @@ -93,7 +93,7 @@ func (dh *BasicDeleteHandler) HandleDeleteGarbage(msg message.DeleteMessage) err func (dh *BasicDeleteHandler) HandleDeleteFile(msg message.DeleteMessage) error { err := dh.StorageInterractor.DeleteObject(msg.Name) if err != nil { - ylogger.Zero.Error().AnErr("err", err).Msg("failed to delete file") + ylogger.Zero.Error().AnErr("err", err).Msg("failed to delete file " + msg.Name) return errors.Wrap(err, "failed to delete file") } return nil @@ -134,7 +134,7 @@ func (dh *BasicDeleteHandler) ListGarbageFiles(msg message.DeleteMessage) ([]str filesToDelete := make([]string, 0) for i := 0; i < len(objectMetas); i++ { - reworkedName := ReworkFileName(objectMetas[i].Path) + reworkedName := database.ReworkFileName(objectMetas[i].Path) ylogger.Zero.Debug().Str("reworked name", reworkedName).Msg("lookup chunk") if vi[reworkedName] { @@ -156,13 +156,3 @@ func (dh *BasicDeleteHandler) ListGarbageFiles(msg message.DeleteMessage) ([]str return filesToDelete, nil } - -func ReworkFileName(str string) string { - p1 := strings.Split(str, "/") - p2 := p1[len(p1)-1] - p3 := strings.Split(p2, "_") - if len(p3) >= 4 { - p2 = fmt.Sprintf("%s_%s_%s_%s_", p3[0], p3[1], p3[2], p3[3]) - } - return p2 -} From b37e08eedfcd1a375a3b0e724daf428cd22f4086 Mon Sep 17 00:00:00 2001 From: Victor Date: Fri, 18 Oct 2024 09:07:30 +0000 Subject: [PATCH 2/4] Fix import in test --- pkg/proc/delete_handler_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/proc/delete_handler_test.go b/pkg/proc/delete_handler_test.go index 5e535df..dd9f0a6 100644 --- a/pkg/proc/delete_handler_test.go +++ b/pkg/proc/delete_handler_test.go @@ -5,6 +5,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/yezzey-gp/yproxy/config" + "github.com/yezzey-gp/yproxy/pkg/database" "github.com/yezzey-gp/yproxy/pkg/message" mock "github.com/yezzey-gp/yproxy/pkg/mock" "github.com/yezzey-gp/yproxy/pkg/object" @@ -46,7 +47,7 @@ func TestReworkingName(t *testing.T) { } for _, testCase := range testCases { - ans := proc.ReworkFileName(testCase.input) + ans := database.ReworkFileName(testCase.input) assert.Equal(t, testCase.expected, ans) } } From 67294751bde29c035cfb8c95ca6d42372b4803a0 Mon Sep 17 00:00:00 2001 From: Victor Date: Tue, 22 Oct 2024 11:08:45 +0000 Subject: [PATCH 3/4] simplifying logic (removed the ReworkFileName function) --- pkg/database/database.go | 9 +------- pkg/proc/delete_handler.go | 2 +- pkg/proc/delete_handler_test.go | 40 --------------------------------- 3 files changed, 2 insertions(+), 49 deletions(-) diff --git a/pkg/database/database.go b/pkg/database/database.go index 45c4eb7..fb215ec 100644 --- a/pkg/database/database.go +++ b/pkg/database/database.go @@ -2,7 +2,6 @@ package database import ( "fmt" - "strings" "github.com/jackc/pgx" "github.com/jackc/pgx/pgtype" @@ -81,7 +80,7 @@ func (database *DatabaseHandler) GetVirtualExpireIndexes(port uint64) (map[strin if err := rows2.Scan(&xpath); err != nil { return nil, nil, fmt.Errorf("unable to parse query output %v", err) } - p2 := ReworkFileName(xpath) + p2 := xpath c2[p2] = true ylogger.Zero.Debug().Str("file", p2).Msg("added") } @@ -170,9 +169,3 @@ func connectToDatabase(port uint64, database string) (*pgx.Conn, error) { } return conn, nil } - -func ReworkFileName(str string) string { - p1 := strings.Split(str, "/") - p2 := strings.Join(p1, "_") - return p2 -} diff --git a/pkg/proc/delete_handler.go b/pkg/proc/delete_handler.go index 45fbffb..93ace46 100644 --- a/pkg/proc/delete_handler.go +++ b/pkg/proc/delete_handler.go @@ -134,7 +134,7 @@ func (dh *BasicDeleteHandler) ListGarbageFiles(msg message.DeleteMessage) ([]str filesToDelete := make([]string, 0) for i := 0; i < len(objectMetas); i++ { - reworkedName := database.ReworkFileName(objectMetas[i].Path) + reworkedName := objectMetas[i].Path ylogger.Zero.Debug().Str("reworked name", reworkedName).Msg("lookup chunk") if vi[reworkedName] { diff --git a/pkg/proc/delete_handler_test.go b/pkg/proc/delete_handler_test.go index dd9f0a6..a4a535f 100644 --- a/pkg/proc/delete_handler_test.go +++ b/pkg/proc/delete_handler_test.go @@ -5,7 +5,6 @@ import ( "github.com/stretchr/testify/assert" "github.com/yezzey-gp/yproxy/config" - "github.com/yezzey-gp/yproxy/pkg/database" "github.com/yezzey-gp/yproxy/pkg/message" mock "github.com/yezzey-gp/yproxy/pkg/mock" "github.com/yezzey-gp/yproxy/pkg/object" @@ -13,45 +12,6 @@ import ( "go.uber.org/mock/gomock" ) -func TestReworkingName(t *testing.T) { - type TestCase struct { - input string - expected string - } - - testCases := []TestCase{ - { - input: "/segments_005/seg1/basebackups_005/yezzey/1663_16530_a4c5ad8305b83f07200b020694c36563_17660_1__DY_1_xlog_19649822496", - expected: "1663_16530_a4c5ad8305b83f07200b020694c36563_17660_", - }, - { - input: "1663_16530_a4c5ad8305b83f07200b020694c36563_17660_1__DY_1_xlog_19649822496", - expected: "1663_16530_a4c5ad8305b83f07200b020694c36563_17660_", - }, - { - input: "seg1/basebackups_005/yezzey/1663_16530_a4c5ad8305b83f07200b020694c36563_17660_1__DY_1_xlog_19649822496", - expected: "1663_16530_a4c5ad8305b83f07200b020694c36563_17660_", - }, - { - input: "1663_16530_a4c5ad8305b83f07200b020694c36563", - expected: "1663_16530_a4c5ad8305b83f07200b020694c36563", - }, - { - input: "1663___a4c5ad8305b83f07200b020694c36563___", - expected: "1663___a4c5ad8305b83f07200b020694c36563_", - }, - { - input: "file", - expected: "file", - }, - } - - for _, testCase := range testCases { - ans := database.ReworkFileName(testCase.input) - assert.Equal(t, testCase.expected, ans) - } -} - func TestFilesToDeletion(t *testing.T) { ctrl := gomock.NewController(t) From 0f8873d13c052ae2e15ec16c38c124839425f4a7 Mon Sep 17 00:00:00 2001 From: Victor Date: Tue, 22 Oct 2024 12:12:07 +0000 Subject: [PATCH 4/4] remove ReworkFileName --- pkg/proc/interaction.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/proc/interaction.go b/pkg/proc/interaction.go index 8b3f491..909373b 100644 --- a/pkg/proc/interaction.go +++ b/pkg/proc/interaction.go @@ -248,7 +248,7 @@ func ProcessCopyExtended(msg message.CopyMessage, s storage.StorageInteractor, c retryCount++ for i := 0; i < len(objectMetas); i++ { path := strings.TrimPrefix(objectMetas[i].Path, instanceCnf.StorageCnf.StoragePrefix) - reworked := ReworkFileName(path) + reworked := path if _, ok := vi[reworked]; !ok { ylogger.Zero.Debug().Str("object path", objectMetas[i].Path).Msg("not in virtual index, skipping...") continue