From 65f097af9d49a9146ba3d300ccf356ce81e8f6cd Mon Sep 17 00:00:00 2001 From: Evgenii Baidakov Date: Wed, 29 Jan 2025 13:38:40 +0400 Subject: [PATCH] *: Remove unused tree service methods Signed-off-by: Evgenii Baidakov --- api/layer/system_object.go | 11 -- api/layer/tagging.go | 64 -------- api/layer/tree_mock.go | 130 ----------------- api/layer/tree_service.go | 9 -- internal/neofs/tree.go | 290 ------------------------------------- 5 files changed, 504 deletions(-) diff --git a/api/layer/system_object.go b/api/layer/system_object.go index 2e56bfd0..6a4b4ad6 100644 --- a/api/layer/system_object.go +++ b/api/layer/system_object.go @@ -179,17 +179,6 @@ func (n *layer) getLockDataFromObjects(ctx context.Context, bkt *data.BucketInfo return &lock, nil } -func (n *layer) getNodeVersionFromCacheOrNeofs(ctx context.Context, objVersion *ObjectVersion) (nodeVersion *data.NodeVersion, err error) { - // check cache if node version is stored inside extendedObjectVersion - nodeVersion = n.getNodeVersionFromCache(n.Owner(ctx), objVersion) - if nodeVersion == nil { - // else get node version from tree service - return n.getNodeVersion(ctx, objVersion) - } - - return nodeVersion, nil -} - func (n *layer) putLockObject(ctx context.Context, bktInfo *data.BucketInfo, objID oid.ID, lock *data.ObjectLock, copiesNumber uint32, objectName, objectVersion string) (oid.ID, error) { prm := PrmObjectCreate{ Container: bktInfo.CID, diff --git a/api/layer/tagging.go b/api/layer/tagging.go index 15410f18..41a40eb1 100644 --- a/api/layer/tagging.go +++ b/api/layer/tagging.go @@ -8,14 +8,11 @@ import ( "fmt" "slices" - "github.com/nspcc-dev/neofs-s3-gw/api" "github.com/nspcc-dev/neofs-s3-gw/api/data" "github.com/nspcc-dev/neofs-s3-gw/api/s3errors" apistatus "github.com/nspcc-dev/neofs-sdk-go/client/status" cid "github.com/nspcc-dev/neofs-sdk-go/container/id" "github.com/nspcc-dev/neofs-sdk-go/object" - oid "github.com/nspcc-dev/neofs-sdk-go/object/id" - "github.com/nspcc-dev/neofs-sdk-go/user" "go.uber.org/zap" "golang.org/x/exp/maps" ) @@ -249,64 +246,3 @@ func objectTaggingCacheKey(p *ObjectVersion) string { func bucketTaggingCacheKey(cnrID cid.ID) string { return ".tagset." + cnrID.EncodeToString() } - -func (n *layer) getNodeVersion(ctx context.Context, objVersion *ObjectVersion) (*data.NodeVersion, error) { - var err error - var version *data.NodeVersion - - if objVersion.VersionID == data.UnversionedObjectVersionID { - version, err = n.treeService.GetUnversioned(ctx, objVersion.BktInfo, objVersion.ObjectName) - } else if len(objVersion.VersionID) == 0 { - version, err = n.treeService.GetLatestVersion(ctx, objVersion.BktInfo, objVersion.ObjectName) - } else { - versions, err2 := n.treeService.GetVersions(ctx, objVersion.BktInfo, objVersion.ObjectName) - if err2 != nil { - return nil, err2 - } - for _, v := range versions { - if v.OID.EncodeToString() == objVersion.VersionID { - version = v - break - } - } - if version == nil { - err = s3errors.GetAPIError(s3errors.ErrNoSuchVersion) - } - } - - if err == nil && version.IsDeleteMarker() && !objVersion.NoErrorOnDeleteMarker || errorsStd.Is(err, ErrNodeNotFound) { - return nil, s3errors.GetAPIError(s3errors.ErrNoSuchKey) - } - - if err == nil && version != nil && !version.IsDeleteMarker() { - reqInfo := api.GetReqInfo(ctx) - n.log.Debug("target details", - zap.String("reqId", reqInfo.RequestID), - zap.String("bucket", objVersion.BktInfo.Name), zap.Stringer("cid", objVersion.BktInfo.CID), - zap.String("object", objVersion.ObjectName), zap.Stringer("oid", version.OID)) - } - - return version, err -} - -func (n *layer) getNodeVersionFromCache(owner user.ID, o *ObjectVersion) *data.NodeVersion { - if len(o.VersionID) == 0 || o.VersionID == data.UnversionedObjectVersionID { - return nil - } - - var objID oid.ID - if objID.DecodeString(o.VersionID) != nil { - return nil - } - - var addr oid.Address - addr.SetContainer(o.BktInfo.CID) - addr.SetObject(objID) - - extObjectInfo := n.cache.GetObject(owner, addr) - if extObjectInfo == nil { - return nil - } - - return extObjectInfo.NodeVersion -} diff --git a/api/layer/tree_mock.go b/api/layer/tree_mock.go index f5eb3ec5..136d89a5 100644 --- a/api/layer/tree_mock.go +++ b/api/layer/tree_mock.go @@ -1,7 +1,6 @@ package layer import ( - "cmp" "context" "fmt" "slices" @@ -20,39 +19,6 @@ type TreeServiceMock struct { parts map[string]map[int]*data.PartInfo } -func (t *TreeServiceMock) GetObjectTagging(_ context.Context, bktInfo *data.BucketInfo, nodeVersion *data.NodeVersion) (map[string]string, error) { - cnrTagsMap, ok := t.tags[bktInfo.CID.EncodeToString()] - if !ok { - return nil, nil - } - - return cnrTagsMap[nodeVersion.ID], nil -} - -func (t *TreeServiceMock) PutObjectTagging(_ context.Context, bktInfo *data.BucketInfo, nodeVersion *data.NodeVersion, tagSet map[string]string) error { - cnrTagsMap, ok := t.tags[bktInfo.CID.EncodeToString()] - if !ok { - t.tags[bktInfo.CID.EncodeToString()] = map[uint64]map[string]string{ - nodeVersion.ID: tagSet, - } - return nil - } - - cnrTagsMap[nodeVersion.ID] = tagSet - - return nil -} - -func (t *TreeServiceMock) DeleteObjectTagging(_ context.Context, bktInfo *data.BucketInfo, objVersion *data.NodeVersion) error { - cnrTagsMap, ok := t.tags[bktInfo.CID.EncodeToString()] - if !ok { - return nil - } - - delete(cnrTagsMap, objVersion.ID) - return nil -} - func (t *TreeServiceMock) GetBucketTagging(_ context.Context, _ *data.BucketInfo) (map[string]string, error) { // TODO implement me panic("implement me") @@ -114,102 +80,6 @@ func (t *TreeServiceMock) DeleteBucketCORS(_ context.Context, _ *data.BucketInfo panic("implement me") } -func (t *TreeServiceMock) GetVersions(_ context.Context, bktInfo *data.BucketInfo, objectName string) ([]*data.NodeVersion, error) { - cnrVersionsMap, ok := t.versions[bktInfo.CID.EncodeToString()] - if !ok { - return nil, ErrNodeNotFound - } - - versions, ok := cnrVersionsMap[objectName] - if !ok { - return nil, ErrNodeNotFound - } - - return versions, nil -} - -func (t *TreeServiceMock) GetLatestVersion(_ context.Context, bktInfo *data.BucketInfo, objectName string) (*data.NodeVersion, error) { - cnrVersionsMap, ok := t.versions[bktInfo.CID.EncodeToString()] - if !ok { - return nil, ErrNodeNotFound - } - - versions, ok := cnrVersionsMap[objectName] - if !ok { - return nil, ErrNodeNotFound - } - - slices.SortFunc(versions, func(a, b *data.NodeVersion) int { - return cmp.Compare(a.ID, b.ID) - }) - - if len(versions) != 0 { - return versions[len(versions)-1], nil - } - - return nil, ErrNodeNotFound -} - -func (t *TreeServiceMock) GetUnversioned(_ context.Context, bktInfo *data.BucketInfo, objectName string) (*data.NodeVersion, error) { - cnrVersionsMap, ok := t.versions[bktInfo.CID.EncodeToString()] - if !ok { - return nil, ErrNodeNotFound - } - - versions, ok := cnrVersionsMap[objectName] - if !ok { - return nil, ErrNodeNotFound - } - - for _, version := range versions { - if version.IsUnversioned { - return version, nil - } - } - - return nil, ErrNodeNotFound -} - -func (t *TreeServiceMock) AddVersion(_ context.Context, bktInfo *data.BucketInfo, newVersion *data.NodeVersion) (uint64, error) { - cnrVersionsMap, ok := t.versions[bktInfo.CID.EncodeToString()] - if !ok { - t.versions[bktInfo.CID.EncodeToString()] = map[string][]*data.NodeVersion{ - newVersion.FilePath: {newVersion}, - } - return newVersion.ID, nil - } - - versions, ok := cnrVersionsMap[newVersion.FilePath] - if !ok { - cnrVersionsMap[newVersion.FilePath] = []*data.NodeVersion{newVersion} - return newVersion.ID, nil - } - - slices.SortFunc(versions, func(a, b *data.NodeVersion) int { - return cmp.Compare(a.ID, b.ID) - }) - - if len(versions) != 0 { - newVersion.ID = versions[len(versions)-1].ID + 1 - newVersion.Timestamp = versions[len(versions)-1].Timestamp + 1 - } - - result := versions - - if newVersion.IsUnversioned { - result = make([]*data.NodeVersion, 0, len(versions)) - for _, node := range versions { - if !node.IsUnversioned { - result = append(result, node) - } - } - } - - cnrVersionsMap[newVersion.FilePath] = append(result, newVersion) - - return newVersion.ID, nil -} - func (t *TreeServiceMock) CreateMultipartUpload(_ context.Context, bktInfo *data.BucketInfo, info *data.MultipartInfo) (uint64, error) { cnrMultipartsMap, ok := t.multiparts[bktInfo.CID.EncodeToString()] if !ok { diff --git a/api/layer/tree_service.go b/api/layer/tree_service.go index ad4861f0..1220d5e1 100644 --- a/api/layer/tree_service.go +++ b/api/layer/tree_service.go @@ -44,19 +44,10 @@ type TreeService interface { // If object id to remove is not found returns ErrNoNodeToRemove error. DeleteBucketCORS(ctx context.Context, bktInfo *data.BucketInfo) (oid.ID, error) - GetObjectTagging(ctx context.Context, bktInfo *data.BucketInfo, objVersion *data.NodeVersion) (map[string]string, error) - PutObjectTagging(ctx context.Context, bktInfo *data.BucketInfo, objVersion *data.NodeVersion, tagSet map[string]string) error - DeleteObjectTagging(ctx context.Context, bktInfo *data.BucketInfo, objVersion *data.NodeVersion) error - GetBucketTagging(ctx context.Context, bktInfo *data.BucketInfo) (map[string]string, error) PutBucketTagging(ctx context.Context, bktInfo *data.BucketInfo, tagSet map[string]string) error DeleteBucketTagging(ctx context.Context, bktInfo *data.BucketInfo) error - GetVersions(ctx context.Context, bktInfo *data.BucketInfo, objectName string) ([]*data.NodeVersion, error) - GetLatestVersion(ctx context.Context, bktInfo *data.BucketInfo, objectName string) (*data.NodeVersion, error) - GetUnversioned(ctx context.Context, bktInfo *data.BucketInfo, objectName string) (*data.NodeVersion, error) - AddVersion(ctx context.Context, bktInfo *data.BucketInfo, newVersion *data.NodeVersion) (uint64, error) - CreateMultipartUpload(ctx context.Context, bktInfo *data.BucketInfo, info *data.MultipartInfo) (uint64, error) DeleteMultipartUpload(ctx context.Context, bktInfo *data.BucketInfo, multipartNodeID uint64) error GetMultipartUploadsByPrefix(ctx context.Context, bktInfo *data.BucketInfo, prefix string) ([]*data.MultipartInfo, error) diff --git a/internal/neofs/tree.go b/internal/neofs/tree.go index d7c8d192..3bc51260 100644 --- a/internal/neofs/tree.go +++ b/internal/neofs/tree.go @@ -18,7 +18,6 @@ import ( "github.com/nspcc-dev/neofs-s3-gw/creds/accessbox" "github.com/nspcc-dev/neofs-s3-gw/internal/neofs/services/tree" oid "github.com/nspcc-dev/neofs-sdk-go/object/id" - "github.com/nspcc-dev/neofs-sdk-go/user" "google.golang.org/grpc" "google.golang.org/grpc/credentials/insecure" ) @@ -76,9 +75,6 @@ const ( corsFilename = "bucket-cors" bucketTaggingFilename = "bucket-tagging" - // versionTree -- ID of a tree with object versions. - versionTree = "version" - // systemTree -- ID of a tree with system objects // i.e. bucket settings with versioning and lock configuration, cors, notifications. systemTree = "system" @@ -154,54 +150,6 @@ func (n *TreeNode) FileName() (string, bool) { return value, ok } -func newNodeVersion(filePath string, node NodeResponse) (*data.NodeVersion, error) { - treeNode, err := newTreeNode(node) - if err != nil { - return nil, fmt.Errorf("invalid tree node: %w", err) - } - - return newNodeVersionFromTreeNode(filePath, treeNode), nil -} - -func newNodeVersionFromTreeNode(filePath string, treeNode *TreeNode) *data.NodeVersion { - _, isUnversioned := treeNode.Get(isUnversionedKV) - _, isDeleteMarker := treeNode.Get(isDeleteMarkerKV) - eTag, _ := treeNode.Get(etagKV) - - version := &data.NodeVersion{ - BaseNodeVersion: data.BaseNodeVersion{ - ID: treeNode.ID, - ParenID: treeNode.ParentID, - OID: treeNode.ObjID, - Timestamp: treeNode.TimeStamp, - ETag: eTag, - Size: treeNode.Size, - FilePath: filePath, - }, - IsUnversioned: isUnversioned, - } - - if isDeleteMarker { - var created time.Time - if createdStr, ok := treeNode.Get(createdKV); ok { - if utcMilli, err := strconv.ParseInt(createdStr, 10, 64); err == nil { - created = time.UnixMilli(utcMilli) - } - } - - var owner user.ID - if ownerStr, ok := treeNode.Get(ownerKV); ok { - _ = owner.DecodeString(ownerStr) - } - - version.DeleteMarker = &data.DeleteMarkerInfo{ - Created: created, - Owner: owner, - } - } - return version -} - func newMultipartInfo(node NodeResponse) (*data.MultipartInfo, error) { multipartInfo := &data.MultipartInfo{ ID: node.GetNodeId(), @@ -410,66 +358,6 @@ func (c *TreeClient) DeleteBucketCORS(ctx context.Context, bktInfo *data.BucketI return oid.ID{}, layer.ErrNoNodeToRemove } -func (c *TreeClient) GetObjectTagging(ctx context.Context, bktInfo *data.BucketInfo, objVersion *data.NodeVersion) (map[string]string, error) { - tagNode, err := c.getTreeNode(ctx, bktInfo, objVersion.ID, isTagKV) - if err != nil { - return nil, err - } - - return getObjectTagging(tagNode), nil -} - -func getObjectTagging(tagNode *TreeNode) map[string]string { - if tagNode == nil { - return nil - } - - meta := make(map[string]string) - - for key, val := range tagNode.Meta { - if strings.HasPrefix(key, userDefinedTagPrefix) { - meta[strings.TrimPrefix(key, userDefinedTagPrefix)] = val - } - } - - return meta -} - -func (c *TreeClient) PutObjectTagging(ctx context.Context, bktInfo *data.BucketInfo, objVersion *data.NodeVersion, tagSet map[string]string) error { - tagNode, err := c.getTreeNode(ctx, bktInfo, objVersion.ID, isTagKV) - if err != nil { - return err - } - - treeTagSet := make(map[string]string) - treeTagSet[isTagKV] = "true" - - for key, val := range tagSet { - treeTagSet[userDefinedTagPrefix+key] = val - } - - if tagNode == nil { - _, err = c.addNode(ctx, bktInfo, versionTree, objVersion.ID, treeTagSet) - } else { - err = c.moveNode(ctx, bktInfo, versionTree, tagNode.ID, objVersion.ID, treeTagSet) - } - - return err -} - -func (c *TreeClient) DeleteObjectTagging(ctx context.Context, bktInfo *data.BucketInfo, objVersion *data.NodeVersion) error { - tagNode, err := c.getTreeNode(ctx, bktInfo, objVersion.ID, isTagKV) - if err != nil { - return err - } - - if tagNode == nil { - return nil - } - - return c.removeNode(ctx, bktInfo, versionTree, tagNode.ID) -} - func (c *TreeClient) GetBucketTagging(ctx context.Context, bktInfo *data.BucketInfo) (map[string]string, error) { node, err := c.getSystemNodeWithAllAttributes(ctx, bktInfo, []string{bucketTaggingFilename}) if err != nil { @@ -523,71 +411,6 @@ func (c *TreeClient) DeleteBucketTagging(ctx context.Context, bktInfo *data.Buck return nil } -func (c *TreeClient) getTreeNode(ctx context.Context, bktInfo *data.BucketInfo, nodeID uint64, key string) (*TreeNode, error) { - nodes, err := c.getTreeNodes(ctx, bktInfo, nodeID, key) - if err != nil { - return nil, err - } - // if there will be many allocations, consider having separate - // implementations of 'getTreeNode' and 'getTreeNodes' - return nodes[key], nil -} - -func (c *TreeClient) getTreeNodes(ctx context.Context, bktInfo *data.BucketInfo, nodeID uint64, keys ...string) (map[string]*TreeNode, error) { - subtree, err := c.getSubTree(ctx, bktInfo, versionTree, nodeID, 2) - if err != nil { - return nil, err - } - - treeNodes := make(map[string]*TreeNode, len(keys)) - - for _, s := range subtree { - node, err := newTreeNode(s) - if err != nil { - return nil, err - } - for _, key := range keys { - if _, ok := node.Get(key); ok { - treeNodes[key] = node - break - } - } - if len(treeNodes) == len(keys) { - break - } - } - - return treeNodes, nil -} - -func (c *TreeClient) GetVersions(ctx context.Context, bktInfo *data.BucketInfo, filepath string) ([]*data.NodeVersion, error) { - return c.getVersions(ctx, bktInfo, versionTree, filepath, false) -} - -func (c *TreeClient) GetLatestVersion(ctx context.Context, bktInfo *data.BucketInfo, objectName string) (*data.NodeVersion, error) { - meta := []string{oidKV, isUnversionedKV, isDeleteMarkerKV, etagKV, sizeKV} - path := pathFromName(objectName) - - p := &getNodesParams{ - BktInfo: bktInfo, - TreeID: versionTree, - Path: path, - Meta: meta, - LatestOnly: true, - AllAttrs: false, - } - nodes, err := c.getNodes(ctx, p) - if err != nil { - return nil, err - } - - if len(nodes) == 0 { - return nil, layer.ErrNodeNotFound - } - - return newNodeVersion(objectName, nodes[0]) -} - // pathFromName splits name by '/'. func pathFromName(objectName string) []string { return strings.Split(objectName, separator) @@ -711,31 +534,6 @@ func isIntermediate(node NodeResponse) bool { return node.GetMeta()[0].GetKey() == fileNameKV } -func (c *TreeClient) GetUnversioned(ctx context.Context, bktInfo *data.BucketInfo, filepath string) (*data.NodeVersion, error) { - return c.getUnversioned(ctx, bktInfo, versionTree, filepath) -} - -func (c *TreeClient) getUnversioned(ctx context.Context, bktInfo *data.BucketInfo, treeID, filepath string) (*data.NodeVersion, error) { - nodes, err := c.getVersions(ctx, bktInfo, treeID, filepath, true) - if err != nil { - return nil, err - } - - if len(nodes) > 1 { - return nil, fmt.Errorf("found more than one unversioned node") - } - - if len(nodes) != 1 { - return nil, layer.ErrNodeNotFound - } - - return nodes[0], nil -} - -func (c *TreeClient) AddVersion(ctx context.Context, bktInfo *data.BucketInfo, version *data.NodeVersion) (uint64, error) { - return c.addVersion(ctx, bktInfo, versionTree, version) -} - func (c *TreeClient) CreateMultipartUpload(ctx context.Context, bktInfo *data.BucketInfo, info *data.MultipartInfo) (uint64, error) { path := pathFromName(info.Key) meta := metaFromMultipart(info, path[len(path)-1]) @@ -969,94 +767,6 @@ func (c *TreeClient) Close() error { return nil } -func (c *TreeClient) addVersion(ctx context.Context, bktInfo *data.BucketInfo, treeID string, version *data.NodeVersion) (uint64, error) { - path := pathFromName(version.FilePath) - meta := map[string]string{ - oidKV: version.OID.EncodeToString(), - fileNameKV: path[len(path)-1], - } - - if version.Size > 0 { - meta[sizeKV] = strconv.FormatInt(version.Size, 10) - } - if len(version.ETag) > 0 { - meta[etagKV] = version.ETag - } - - if version.IsDeleteMarker() { - meta[isDeleteMarkerKV] = "true" - meta[ownerKV] = version.DeleteMarker.Owner.EncodeToString() - meta[createdKV] = strconv.FormatInt(version.DeleteMarker.Created.UTC().UnixMilli(), 10) - } - - if version.IsUnversioned { - meta[isUnversionedKV] = "true" - - node, err := c.getUnversioned(ctx, bktInfo, treeID, version.FilePath) - if err == nil { - if err = c.moveNode(ctx, bktInfo, treeID, node.ID, node.ParenID, meta); err != nil { - return 0, err - } - - return node.ID, c.clearOutdatedVersionInfo(ctx, bktInfo, treeID, node.ID) - } - - if !errors.Is(err, layer.ErrNodeNotFound) { - return 0, err - } - } - - return c.addNodeByPath(ctx, bktInfo, treeID, path[:len(path)-1], meta) -} - -func (c *TreeClient) clearOutdatedVersionInfo(ctx context.Context, bktInfo *data.BucketInfo, treeID string, nodeID uint64) error { - taggingNode, err := c.getTreeNode(ctx, bktInfo, nodeID, isTagKV) - if err != nil { - return err - } - if taggingNode != nil { - return c.removeNode(ctx, bktInfo, treeID, taggingNode.ID) - } - - return nil -} - -func (c *TreeClient) getVersions(ctx context.Context, bktInfo *data.BucketInfo, treeID, filepath string, onlyUnversioned bool) ([]*data.NodeVersion, error) { - keysToReturn := []string{oidKV, isUnversionedKV, isDeleteMarkerKV, etagKV, sizeKV} - path := pathFromName(filepath) - p := &getNodesParams{ - BktInfo: bktInfo, - TreeID: treeID, - Path: path, - Meta: keysToReturn, - LatestOnly: false, - AllAttrs: false, - } - nodes, err := c.getNodes(ctx, p) - if err != nil { - if errors.Is(err, layer.ErrNodeNotFound) { - return nil, nil - } - return nil, err - } - - result := make([]*data.NodeVersion, 0, len(nodes)) - for _, node := range nodes { - nodeVersion, err := newNodeVersion(filepath, node) - if err != nil { - return nil, err - } - - if onlyUnversioned && !nodeVersion.IsUnversioned { - continue - } - - result = append(result, nodeVersion) - } - - return result, nil -} - func (c *TreeClient) getSubTree(ctx context.Context, bktInfo *data.BucketInfo, treeID string, rootID uint64, depth uint32) ([]*tree.GetSubTreeResponse_Body, error) { request := &tree.GetSubTreeRequest{ Body: &tree.GetSubTreeRequest_Body{