Feature - deleting bloat files

pkg/database/database.go

@@ -2,7 +2,6 @@ package database
 
 import (
 	"fmt"
-	"strings"
 
 	"github.com/jackc/pgx"
 	"github.com/jackc/pgx/pgtype"
@@ -81,12 +80,7 @@ func (database *DatabaseHandler) GetVirtualExpireIndexes(port uint64) (map[strin
 		if err := rows2.Scan(&xpath); err != nil {
 			return nil, nil, fmt.Errorf("unable to parse query output %v", err)
 		}
-		p1 := strings.Split(xpath, "/")
-		p2 := p1[len(p1)-1]
-		p3 := strings.Split(p2, "_")
-		if len(p3) >= 4 {
-			p2 = fmt.Sprintf("%s_%s_%s_%s_", p3[0], p3[1], p3[2], p3[3])
-		}
+		p2 := xpath
 		c2[p2] = true
 		ylogger.Zero.Debug().Str("file", p2).Msg("added")
 	}

pkg/proc/delete_handler.go

@@ -93,7 +93,7 @@ func (dh *BasicDeleteHandler) HandleDeleteGarbage(msg message.DeleteMessage) err
 func (dh *BasicDeleteHandler) HandleDeleteFile(msg message.DeleteMessage) error {
 	err := dh.StorageInterractor.DeleteObject(msg.Name)
 	if err != nil {
-		ylogger.Zero.Error().AnErr("err", err).Msg("failed to delete file")
+		ylogger.Zero.Error().AnErr("err", err).Msg("failed to delete file " + msg.Name)
 		return errors.Wrap(err, "failed to delete file")
 	}
 	return nil
@@ -134,7 +134,7 @@ func (dh *BasicDeleteHandler) ListGarbageFiles(msg message.DeleteMessage) ([]str
 
 	filesToDelete := make([]string, 0)
 	for i := 0; i < len(objectMetas); i++ {
-		reworkedName := ReworkFileName(objectMetas[i].Path)
+		reworkedName := objectMetas[i].Path
 		ylogger.Zero.Debug().Str("reworked name", reworkedName).Msg("lookup chunk")
 
 		if vi[reworkedName] {
@@ -156,13 +156,3 @@ func (dh *BasicDeleteHandler) ListGarbageFiles(msg message.DeleteMessage) ([]str
 
 	return filesToDelete, nil
 }
-
-func ReworkFileName(str string) string {
-	p1 := strings.Split(str, "/")
-	p2 := p1[len(p1)-1]
-	p3 := strings.Split(p2, "_")
-	if len(p3) >= 4 {
-		p2 = fmt.Sprintf("%s_%s_%s_%s_", p3[0], p3[1], p3[2], p3[3])
-	}
-	return p2
-}

pkg/proc/delete_handler_test.go

@@ -12,45 +12,6 @@ import (
 	"go.uber.org/mock/gomock"
 )
 
-func TestReworkingName(t *testing.T) {
-	type TestCase struct {
-		input    string
-		expected string
-	}
-
-	testCases := []TestCase{
-		{
-			input:    "/segments_005/seg1/basebackups_005/yezzey/1663_16530_a4c5ad8305b83f07200b020694c36563_17660_1__DY_1_xlog_19649822496",
-			expected: "1663_16530_a4c5ad8305b83f07200b020694c36563_17660_",
-		},
-		{
-			input:    "1663_16530_a4c5ad8305b83f07200b020694c36563_17660_1__DY_1_xlog_19649822496",
-			expected: "1663_16530_a4c5ad8305b83f07200b020694c36563_17660_",
-		},
-		{
-			input:    "seg1/basebackups_005/yezzey/1663_16530_a4c5ad8305b83f07200b020694c36563_17660_1__DY_1_xlog_19649822496",
-			expected: "1663_16530_a4c5ad8305b83f07200b020694c36563_17660_",
-		},
-		{
-			input:    "1663_16530_a4c5ad8305b83f07200b020694c36563",
-			expected: "1663_16530_a4c5ad8305b83f07200b020694c36563",
-		},
-		{
-			input:    "1663___a4c5ad8305b83f07200b020694c36563___",
-			expected: "1663___a4c5ad8305b83f07200b020694c36563_",
-		},
-		{
-			input:    "file",
-			expected: "file",
-		},
-	}
-
-	for _, testCase := range testCases {
-		ans := proc.ReworkFileName(testCase.input)
-		assert.Equal(t, testCase.expected, ans)
-	}
-}
-
 func TestFilesToDeletion(t *testing.T) {
 	ctrl := gomock.NewController(t)
 

pkg/proc/interaction.go

@@ -248,7 +248,7 @@ func ProcessCopyExtended(msg message.CopyMessage, s storage.StorageInteractor, c
 		retryCount++
 		for i := 0; i < len(objectMetas); i++ {
 			path := strings.TrimPrefix(objectMetas[i].Path, instanceCnf.StorageCnf.StoragePrefix)
-			reworked := ReworkFileName(path)
+			reworked := path
 			if _, ok := vi[reworked]; !ok {
 				ylogger.Zero.Debug().Str("object path", objectMetas[i].Path).Msg("not in virtual index, skipping...")
 				continue

pkg/storage/filestorage.go

@@ -31,7 +31,7 @@ func (s *FileStorageInteractor) CatFileFromStorage(name string, offset int64, _
 }
 func (s *FileStorageInteractor) ListPath(prefix string) ([]*object.ObjectInfo, error) {
 	var data []*object.ObjectInfo
-	err := filepath.WalkDir(s.cnf.StoragePrefix+prefix, func(path string, d fs.DirEntry, err error) error {
+	err := filepath.WalkDir(s.cnf.StoragePrefix, func(path string, d fs.DirEntry, err error) error {
 		if err != nil {
 			return err
 		}
@@ -46,6 +46,12 @@ func (s *FileStorageInteractor) ListPath(prefix string) ([]*object.ObjectInfo, e
 		if err != nil {
 			return err
 		}
+
+		cuttedPrefix, _ := strings.CutPrefix(prefix, "/")
+		neededPrefix := s.cnf.StoragePrefix + cuttedPrefix
+		if !strings.HasPrefix(path, neededPrefix) {
+			return nil
+		}
 		cPath, ok := strings.CutPrefix(path, s.cnf.StoragePrefix)
 
 		if !ok {
@@ -89,3 +95,11 @@ func (s *FileStorageInteractor) DeleteObject(key string) error {
 func (s *FileStorageInteractor) AbortMultipartUploads() error {
 	return nil
 }
+
+func (s *FileStorageInteractor) AbortMultipartUpload(key, uploadId string) error {
+	return nil
+}
+
+func (s *FileStorageInteractor) ListFailedMultipartUploads() (map[string]string, error) {
+	return nil, nil
+}