diff --git a/ais/proxy.go b/ais/proxy.go index e515591724a..2682400fb46 100644 --- a/ais/proxy.go +++ b/ais/proxy.go @@ -58,7 +58,6 @@ type ( authn *authManager metasyncer *metasyncer ic ic - qm lsobjMem rproxy reverseProxy notifs notifs lstca lstca @@ -200,7 +199,6 @@ func (p *proxy) Run() error { p.notifs.init(p) p.ic.init(p) - p.qm.init() // // REST API: register proxy handlers and start listening @@ -1474,9 +1472,6 @@ func (p *proxy) _bckpost(w http.ResponseWriter, r *http.Request, msg *apc.ActMsg p.writeErr(w, r, err) return } - case apc.ActInvalListCache: - p.qm.c.invalidate(bck.Bucket()) - return case apc.ActMakeNCopies: if xid, err = p.makeNCopies(msg, bck); err != nil { p.writeErr(w, r, err) @@ -2299,41 +2294,14 @@ func (p *proxy) lsObjsA(bck *meta.Bck, lsmsg *apc.LsoMsg) (allEntries *cmn.LsoRe var ( actMsgExt *actMsgExt args *bcastArgs - entries cmn.LsoEntries results sliceResults smap = p.owner.smap.get() - cacheID = cacheReqID{bck: bck.Bucket(), prefix: lsmsg.Prefix} - token = lsmsg.ContinuationToken - props = lsmsg.PropsSet() - hasEnough bool - flags uint32 ) if lsmsg.PageSize == 0 { lsmsg.PageSize = apc.MaxPageSizeAIS } pageSize := lsmsg.PageSize - // TODO: Before checking cache and buffer we should check if there is another - // request in-flight that asks for the same page - if true wait for the cache - // to get populated. - - if lsmsg.IsFlagSet(apc.UseListObjsCache) { - entries, hasEnough = p.qm.c.get(cacheID, token, pageSize) - if hasEnough { - goto end - } - } - entries, hasEnough = p.qm.b.get(lsmsg.UUID, token, pageSize) - if hasEnough { - // We have enough in the buffer to fulfill the request. - goto endWithCache - } - - // User requested some page but we don't have enough (but we may have part - // of the full page). Therefore, we must ask targets for page starting from - // what we have locally, so we don't re-request the objects. - lsmsg.ContinuationToken = p.qm.b.last(lsmsg.UUID, token) - actMsgExt = p.newAmsgActVal(apc.ActList, &lsmsg) args = allocBcArgs() args.req = cmn.HreqArgs{ @@ -2349,6 +2317,7 @@ func (p *proxy) lsObjsA(bck *meta.Bck, lsmsg *apc.LsoMsg) (allEntries *cmn.LsoRe // Combine the results. results = p.bcastGroup(args) freeBcArgs(args) + lsoResList := make([]*cmn.LsoRes, 0, len(results)) for _, res := range results { if res.err != nil { if res.details == "" || res.details == dfltDetail { @@ -2359,46 +2328,13 @@ func (p *proxy) lsObjsA(bck *meta.Bck, lsmsg *apc.LsoMsg) (allEntries *cmn.LsoRe return nil, err } lst := res.v.(*cmn.LsoRes) - flags |= lst.Flags - p.qm.b.set(lsmsg.UUID, res.si.ID(), lst.Entries, pageSize) - } - freeBcastRes(results) - entries, hasEnough = p.qm.b.get(lsmsg.UUID, token, pageSize) - debug.Assert(hasEnough) - -endWithCache: - if lsmsg.IsFlagSet(apc.UseListObjsCache) { - p.qm.c.set(cacheID, token, entries, pageSize) - } -end: - if lsmsg.IsFlagSet(apc.UseListObjsCache) && !props.All(apc.GetPropsAll...) { - // Since cache keeps entries with whole subset props we must create copy - // of the entries with smaller subset of props (if we would change the - // props of the `entries` it would also affect entries inside cache). - propsEntries := make(cmn.LsoEntries, len(entries)) - for idx := range entries { - propsEntries[idx] = entries[idx].CopyWithProps(props) + if len(lst.Entries) > 0 { + lsoResList = append(lsoResList, lst) } - entries = propsEntries - } - - allEntries = &cmn.LsoRes{ - UUID: lsmsg.UUID, - Entries: entries, - Flags: flags, - } - if len(entries) >= int(pageSize) { - allEntries.ContinuationToken = entries[len(entries)-1].Name - } - - // when recursion is disabled (i.e., lsmsg.IsFlagSet(apc.LsNoRecursion)) - // the (`cmn.LsoRes`) result _may_ include duplicated names of the virtual subdirectories - // - that's why: - if lsmsg.IsFlagSet(apc.LsNoRecursion) { - allEntries.Entries = cmn.DedupLso(allEntries.Entries, len(entries), false /*no-dirs*/) } + freeBcastRes(results) - return allEntries, nil + return cmn.ConcatLso(lsoResList, lsmsg, int(pageSize)), nil } func (p *proxy) lsObjsR(bck *meta.Bck, lsmsg *apc.LsoMsg, hdr http.Header, smap *smapX, tsi *meta.Snode, config *cmn.Config, diff --git a/ais/prxlso.go b/ais/prxlso.go deleted file mode 100644 index 96fad11a041..00000000000 --- a/ais/prxlso.go +++ /dev/null @@ -1,535 +0,0 @@ -// Package ais provides core functionality for the AIStore object storage. -/* - * Copyright (c) 2018-2024, NVIDIA CORPORATION. All rights reserved. - */ -package ais - -import ( - "sort" - "strings" - "sync" - "time" - - "github.com/NVIDIA/aistore/cmn" - "github.com/NVIDIA/aistore/cmn/atomic" - "github.com/NVIDIA/aistore/cmn/debug" - "github.com/NVIDIA/aistore/cmn/mono" - "github.com/NVIDIA/aistore/hk" -) - -// Brief theory of operation ================================================ -// -// * BUFFER - container for a single request that keeps entries so they won't -// be re-requested. Thanks to buffering, we eliminate the case when a given -// object is requested more than once. -// * CACHE - container shared by multiple requests which are identified with -// the same id. Thanks to caching, we reuse previously calculated requests. -// -// Buffering is designed to work for a single request and is identified by -// list-objects uuid. Each buffer consists of: -// - a *main buffer* that in turn contains entries ready to be returned to the -// client (user), and -// - *leftovers* - per target structures consisting of entries that couldn't -// be included into the *main buffer* yet. -// When a buffer doesn't contain enough entries, the new entries -// are loaded and added to *leftovers*. After this, they are merged and put -// into the *main buffer* so they can be returned to the client. -// -// Caching is thread safe and is used across multiple requests (clients). -// Each request is identified by its `cacheReqID`. List-objects requests -// that share the same ID will also share a common cache. -// -// Cache consists of contiguous intervals of `cmn.LsoEnt`. -// Cached response (to a request) is valid if and only if the request can be -// fulfilled by a single cache interval (otherwise, cache cannot be trusted -// as we don't know how many objects can fit in the requested interval). - -// internal timers (rough estimates) -const ( - cacheIntervalTTL = 10 * time.Minute // *cache interval's* time to live - lsobjBufferTTL = 10 * time.Minute // *lsobj buffer* time to live - qmTimeHk = 10 * time.Minute // housekeeping timer - qmTimeHkMax = time.Hour // max HK time (when no activity whatsoever) -) - -type ( - // Request buffer per target. - lsobjBufferTarget struct { - // Leftovers entries which we keep locally so they will not be requested - // again by the proxy. Out of these `currentBuff` is extended. - entries cmn.LsoEntries - // Determines if the target is done with listing. - done bool - } - - // Request buffer that corresponds to a single `uuid`. - lsobjBuffer struct { - // Buffers for each target that are finally merged and the entries are - // appended to the `currentBuff`. - leftovers map[string]*lsobjBufferTarget // targetID (string) -> target buffer - // Contains the last entry that was returned to the user. - nextToken string - // Currently maintained buffer that keeps the entries sorted - // and ready to be dispatched to the client. - currentBuff cmn.LsoEntries - // Timestamp of the last access to this buffer. Idle buffers get removed - // after `lsobjBufferTTL`. - lastAccess atomic.Int64 - } - - // Contains all lsobj buffers. - lsobjBuffers struct { - buffers sync.Map // request uuid (string) -> buffer (*lsobjBuffer) - } - - // Cache request ID. This identifies and splits requests into - // multiple caches that these requests can use. - cacheReqID struct { - bck *cmn.Bck - prefix string - } - - // Single (contiguous) interval of `cmn.LsoEnt`. - cacheInterval struct { - // Contains the previous entry (`ContinuationToken`) that was requested - // to get this interval. Thanks to this we can match and merge two - // adjacent intervals. - token string - // Entries that are contained in this interval. They are sorted and ready - // to be dispatched to the client. - entries cmn.LsoEntries - // Contains the timestamp of the last access to this interval. Idle interval - // gets removed after `cacheIntervalTTL`. - lastAccess int64 - // Determines if this is the last page/interval (no more objects after - // the last entry). - last bool - } - - // Contains additional parameters to interval request. - reqParams struct { - prefix string - } - - // Single cache that corresponds to single `cacheReqID`. - lsobjCache struct { - intervals []*cacheInterval - mtx sync.RWMutex - } - - // Contains all lsobj caches. - lsobjCaches struct { - caches sync.Map // cache id (cacheReqID) -> cache (*lsobjCache) - } - - lsobjMem struct { - b *lsobjBuffers - c *lsobjCaches - d time.Duration - } -) - -func (qm *lsobjMem) init() { - qm.b = &lsobjBuffers{} - qm.c = &lsobjCaches{} - qm.d = qmTimeHk - hk.Reg("lsobj-buffer-cache"+hk.NameSuffix, qm.housekeep, qmTimeHk) -} - -func (qm *lsobjMem) housekeep(now int64) time.Duration { - num := qm.b.housekeep(now) - num += qm.c.housekeep(now) - if num == 0 { - qm.d = min(qm.d+qmTimeHk, qmTimeHkMax) - } else { - qm.d = qmTimeHk - } - return qm.d -} - -///////////////// -// lsobjBuffer // -///////////////// - -// mergeTargetBuffers merges `b.leftovers` buffers into `b.currentBuff`. -// It returns `filled` equal to `true` if there was anything to merge, otherwise `false`. -func (b *lsobjBuffer) mergeTargetBuffers() (filled bool) { - var ( - totalCnt int - allDone = true - ) - // If `b.leftovers` is empty then there was no initial `set`. - if len(b.leftovers) == 0 { - return false - } - for _, list := range b.leftovers { - totalCnt += len(list.entries) - allDone = allDone && list.done - } - // If there are no entries and some targets are not yet done then there wasn't `set`. - if totalCnt == 0 && !allDone { - return false - } - - var ( - minObj string - entries = make(cmn.LsoEntries, 0, totalCnt) - ) - for _, list := range b.leftovers { - for i := range list.entries { - if list.entries[i] == nil { - list.entries = list.entries[:i] - break - } - } - entries = append(entries, list.entries...) - - if list.done || len(list.entries) == 0 { - continue - } - if minObj == "" || list.entries[len(list.entries)-1].Name < minObj { - minObj = list.entries[len(list.entries)-1].Name - } - } - - cmn.SortLso(entries) - - if minObj != "" { - idx := sort.Search(len(entries), func(i int) bool { - return entries[i].Name > minObj - }) - entries = entries[:idx] - } - for id := range b.leftovers { - b.leftovers[id].entries = nil - } - b.currentBuff = append(b.currentBuff, entries...) - return true -} - -func (b *lsobjBuffer) get(token string, size int64) (entries cmn.LsoEntries, hasEnough bool) { - b.lastAccess.Store(mono.NanoTime()) - - // If user requested something before what we have currently in the buffer - // then we just need to forget it. - if token < b.nextToken { - b.leftovers = nil - b.currentBuff = nil - b.nextToken = token - return nil, false - } - - filled := b.mergeTargetBuffers() - - // Move to first object after token. - idx := sort.Search(len(b.currentBuff), func(i int) bool { - return b.currentBuff[i].Name > token - }) - entries = b.currentBuff[idx:] - - if size > int64(len(entries)) { - // In case we don't have enough entries and we haven't filled anything then - // we must request more (if filled then we don't have enough because it's end). - if !filled { - return nil, false - } - size = int64(len(entries)) - } - - // Move buffer after returned entries. - b.currentBuff = entries[size:] - // Select only the entries that need to be returned to user. - entries = entries[:size] - if len(entries) > 0 { - b.nextToken = entries[len(entries)-1].Name - } - return entries, true -} - -func (b *lsobjBuffer) set(id string, entries cmn.LsoEntries, size int64) { - if b.leftovers == nil { - b.leftovers = make(map[string]*lsobjBufferTarget, 5) - } - b.leftovers[id] = &lsobjBufferTarget{ - entries: entries, - done: len(entries) < int(size), - } - b.lastAccess.Store(mono.NanoTime()) -} - -func (b *lsobjBuffers) last(id, token string) string { - v, ok := b.buffers.LoadOrStore(id, &lsobjBuffer{}) - if !ok { - return token - } - buffer := v.(*lsobjBuffer) - if len(buffer.currentBuff) == 0 { - return token - } - last := buffer.currentBuff[len(buffer.currentBuff)-1].Name - if cmn.TokenGreaterEQ(token, last) { - return token - } - return last -} - -func (b *lsobjBuffers) get(id, token string, size int64) (entries cmn.LsoEntries, hasEnough bool) { - v, _ := b.buffers.LoadOrStore(id, &lsobjBuffer{}) - return v.(*lsobjBuffer).get(token, size) -} - -func (b *lsobjBuffers) set(id, targetID string, entries cmn.LsoEntries, size int64) { - v, _ := b.buffers.LoadOrStore(id, &lsobjBuffer{}) - v.(*lsobjBuffer).set(targetID, entries, size) -} - -func (b *lsobjBuffers) housekeep(now int64) (num int) { - b.buffers.Range(func(key, value any) bool { - buffer := value.(*lsobjBuffer) - num++ - // mono.Since(lastAccess) - if now-buffer.lastAccess.Load() > int64(lsobjBufferTTL) { - b.buffers.Delete(key) - } - return true - }) - return -} - -/////////////////// -// cacheInterval // -/////////////////// - -func (ci *cacheInterval) contains(token string) bool { - if ci.token == token { - return true - } - if len(ci.entries) > 0 { - return ci.entries[0].Name <= token && token <= ci.entries[len(ci.entries)-1].Name - } - return false -} - -func (ci *cacheInterval) get(token string, objCnt int64, params reqParams) (entries cmn.LsoEntries, hasEnough bool) { - ci.lastAccess = mono.NanoTime() - entries = ci.entries - - start := ci.find(token) - if params.prefix != "" { - // Move `start` to first entry that starts with `params.prefix`. - for ; start < uint(len(entries)); start++ { - if strings.HasPrefix(entries[start].Name, params.prefix) { - break - } - if entries[start].Name > params.prefix { - // Prefix is fully contained in the interval (but there are no entries), examples: - // * interval = ["a", "z"], token = "", objCnt = 1, prefix = "b" - // * interval = ["a", "z"], token = "a", objCnt = 1, prefix = "b" - return cmn.LsoEntries{}, true - } - } - if !ci.last && start == uint(len(entries)) { - // Prefix is out of the interval (right boundary), examples: - // * interval = ["b", "y"], token = "", objCnt = 1, prefix = "z" - // * interval = ["b", "y"], token = "", objCnt = 1, prefix = "ya" - return nil, false - } - } - entries = entries[start:] - - end := min(len(entries), int(objCnt)) - if params.prefix != "" { - // Move `end-1` to last entry that starts with `params.prefix`. - for ; end > 0; end-- { - if strings.HasPrefix(entries[end-1].Name, params.prefix) { - break - } - } - if !ci.last && end < len(entries) { - // We filtered out entries that start with `params.prefix` and - // the entries are fully contained in the interval, examples: - // * interval = ["a", "ma", "mb", "z"], token = "", objCnt = 4, prefix = "m" - // * interval = ["a", "z"], token = "", objCnt = 2, prefix = "a" - return entries[:end], true - } - } - entries = entries[:end] - - if ci.last || len(entries) >= int(objCnt) { - return entries, true - } - return nil, false -} - -func (ci *cacheInterval) find(token string) (idx uint) { - if ci.token == token { - return 0 - } - return uint(sort.Search(len(ci.entries), func(i int) bool { - return ci.entries[i].Name > token - })) -} - -func (ci *cacheInterval) append(objs *cacheInterval) { - idx := ci.find(objs.token) - ci.entries = append(ci.entries[:idx], objs.entries...) - ci.last = objs.last - ci.lastAccess = mono.NanoTime() -} - -func (ci *cacheInterval) prepend(objs *cacheInterval) { - debug.Assert(!objs.last) - objs.append(ci) - *ci = *objs -} - -//////////////// -// lsobjCache // -//////////////// - -// PRECONDITION: `c.mtx` must be at least rlocked. -func (c *lsobjCache) findInterval(token string) *cacheInterval { - // TODO: finding intervals should be faster than just walking. - for _, interval := range c.intervals { - if interval.contains(token) { - return interval - } - } - return nil -} - -// PRECONDITION: `c.mtx` must be locked. -func (c *lsobjCache) merge(start, end, cur *cacheInterval) { - debug.AssertRWMutexLocked(&c.mtx) - switch { - case start == nil && end == nil: - c.intervals = append(c.intervals, cur) - case start != nil && end == nil: - start.append(cur) - case start == nil && end != nil: - end.prepend(cur) - default: - debug.Assert(start != nil && end != nil) - if start == end { - // `cur` is part of some interval. - return - } - start.append(cur) - start.append(end) - c.removeInterval(end) - } -} - -// PRECONDITION: `c.mtx` must be locked. -func (c *lsobjCache) removeInterval(ci *cacheInterval) { - debug.AssertRWMutexLocked(&c.mtx) - - // TODO: this should be faster - for idx := range c.intervals { - if c.intervals[idx] == ci { - ci.entries = nil - c.intervals = append(c.intervals[:idx], c.intervals[idx+1:]...) - return - } - } -} - -func (c *lsobjCache) get(token string, objCnt int64, params reqParams) (entries cmn.LsoEntries, hasEnough bool) { - c.mtx.RLock() - if interval := c.findInterval(token); interval != nil { - entries, hasEnough = interval.get(token, objCnt, params) - } - c.mtx.RUnlock() - return -} - -func (c *lsobjCache) set(token string, entries cmn.LsoEntries, size int64) { - var ( - end *cacheInterval - cur = &cacheInterval{ - token: token, - entries: entries, - last: len(entries) < int(size), - lastAccess: mono.NanoTime(), - } - ) - c.mtx.Lock() - start := c.findInterval(token) - if len(cur.entries) > 0 { - end = c.findInterval(entries[len(entries)-1].Name) - } - c.merge(start, end, cur) - c.mtx.Unlock() -} - -func (c *lsobjCache) invalidate() { - c.mtx.Lock() - c.intervals = nil - c.mtx.Unlock() -} - -///////////////// -// lsobjCaches // -///////////////// - -func (c *lsobjCaches) get(reqID cacheReqID, token string, objCnt int64) (entries cmn.LsoEntries, hasEnough bool) { - if v, ok := c.caches.Load(reqID); ok { - if entries, hasEnough = v.(*lsobjCache).get(token, objCnt, reqParams{}); hasEnough { - return - } - } - - // When `prefix` is requested we must also check if there is enough entries - // in the "main" (whole bucket) cache with given prefix. - if reqID.prefix != "" { - // We must adjust parameters and cache id. - params := reqParams{prefix: reqID.prefix} - reqID = cacheReqID{bck: reqID.bck} - - if v, ok := c.caches.Load(reqID); ok { - return v.(*lsobjCache).get(token, objCnt, params) - } - } - return nil, false -} - -func (c *lsobjCaches) set(reqID cacheReqID, token string, entries cmn.LsoEntries, size int64) { - v, _ := c.caches.LoadOrStore(reqID, &lsobjCache{}) - v.(*lsobjCache).set(token, entries, size) -} - -func (c *lsobjCaches) invalidate(bck *cmn.Bck) { - c.caches.Range(func(key, value any) bool { - id := key.(cacheReqID) - if id.bck.Equal(bck) { - value.(*lsobjCache).invalidate() - } - return true - }) -} - -// TODO: factor-in memory pressure. -func (c *lsobjCaches) housekeep(now int64) (num int) { - var toRemove []*cacheInterval - c.caches.Range(func(key, value any) bool { - cache := value.(*lsobjCache) - cache.mtx.Lock() - for _, interval := range cache.intervals { - num++ - // mono.Since(lastAccess) - if now-interval.lastAccess > int64(cacheIntervalTTL) { - toRemove = append(toRemove, interval) - } - } - for _, interval := range toRemove { - cache.removeInterval(interval) - } - if len(cache.intervals) == 0 { - c.caches.Delete(key) - } - cache.mtx.Unlock() - toRemove = toRemove[:0] - return true - }) - return -} diff --git a/ais/prxlsobj_internal_test.go b/ais/prxlsobj_internal_test.go deleted file mode 100644 index 14f9576d2f0..00000000000 --- a/ais/prxlsobj_internal_test.go +++ /dev/null @@ -1,455 +0,0 @@ -// Package ais provides core functionality for the AIStore object storage. -/* - * Copyright (c) 2018-2024, NVIDIA CORPORATION. All rights reserved. - */ -package ais - -import ( - "github.com/NVIDIA/aistore/cmn" - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" -) - -var _ = Describe("ListObjectsCache+ListObjectsBuffer", func() { - makeEntries := func(xs ...string) (entries cmn.LsoEntries) { - for _, x := range xs { - entries = append(entries, &cmn.LsoEnt{ - Name: x, - }) - } - return - } - - extractNames := func(entries cmn.LsoEntries) (xs []string) { - for _, entry := range entries { - xs = append(xs, entry.Name) - } - return - } - - Describe("ListObjectsCache", func() { - var ( - id = cacheReqID{bck: &cmn.Bck{Name: "some_bck"}} - cache *lsobjCaches - ) - - BeforeEach(func() { - cache = &lsobjCaches{} - }) - - It("should correctly add entries to cache", func() { - cache.set(id, "", makeEntries("a", "b", "c"), 3) - entries, hasEnough := cache.get(id, "", 3) - Expect(hasEnough).To(BeTrue()) - Expect(extractNames(entries)).To(Equal([]string{"a", "b", "c"})) - }) - - It("should correctly add streaming intervals", func() { - cache.set(id, "", makeEntries("a", "b", "c"), 3) - cache.set(id, "c", makeEntries("d", "e", "f"), 3) - cache.set(id, "f", makeEntries("g", "h", "i"), 3) - - entries, hasEnough := cache.get(id, "", 9) - Expect(hasEnough).To(BeTrue()) - Expect(extractNames(entries)).To(Equal([]string{"a", "b", "c", "d", "e", "f", "g", "h", "i"})) - }) - - It("should correctly handle last page", func() { - cache.set(id, "", makeEntries("a", "b", "c"), 3) - cache.set(id, "c", makeEntries("d", "e", "f"), 3) - cache.set(id, "f", makeEntries("g", "h", "i"), 4) - - entries, hasEnough := cache.get(id, "", 10) - Expect(hasEnough).To(BeTrue()) - Expect(extractNames(entries)).To(Equal([]string{"a", "b", "c", "d", "e", "f", "g", "h", "i"})) - }) - - It("should correctly handle empty last page", func() { - cache.set(id, "", makeEntries("a", "b", "c"), 3) - cache.set(id, "c", makeEntries("d", "e", "f"), 3) - cache.set(id, "f", cmn.LsoEntries{}, 4) - - entries, hasEnough := cache.get(id, "", 10) - Expect(hasEnough).To(BeTrue()) - Expect(extractNames(entries)).To(Equal([]string{"a", "b", "c", "d", "e", "f"})) - }) - - It("should correctly handle overlapping entries", func() { - cache.set(id, "", makeEntries("a", "b", "c"), 3) - cache.set(id, "a", makeEntries("d", "e", "f"), 3) - - entries, hasEnough := cache.get(id, "", 4) - Expect(hasEnough).To(BeTrue()) - Expect(extractNames(entries)).To(Equal([]string{"a", "d", "e", "f"})) - }) - - It("should correctly merge intervals", func() { - cache.set(id, "", makeEntries("a", "b", "c"), 3) - cache.set(id, "g", makeEntries("h", "i", "j"), 3) - - _, hasEnough := cache.get(id, "", 3) - Expect(hasEnough).To(BeTrue()) - _, hasEnough = cache.get(id, "", 4) - Expect(hasEnough).To(BeFalse()) - - _, hasEnough = cache.get(id, "g", 2) - Expect(hasEnough).To(BeTrue()) - - // Add interval in the middle. - cache.set(id, "c", makeEntries("d", "e", "f", "g"), 4) - - // Check that now intervals are connected. - entries, hasEnough := cache.get(id, "", 4) - Expect(hasEnough).To(BeTrue()) - Expect(extractNames(entries)).To(Equal([]string{"a", "b", "c", "d"})) - entries, hasEnough = cache.get(id, "", 10) - Expect(hasEnough).To(BeTrue()) - Expect(extractNames(entries)).To(Equal([]string{"a", "b", "c", "d", "e", "f", "g", "h", "i", "j"})) - }) - - It("should correctly merge overlapping intervals", func() { - cache.set(id, "", makeEntries("a", "b", "c"), 3) - cache.set(id, "g", makeEntries("h", "i", "j"), 3) - cache.set(id, "a", makeEntries("b", "c", "d", "e", "f", "g", "h", "i"), 8) - - entries, hasEnough := cache.get(id, "", 10) - Expect(hasEnough).To(BeTrue()) - Expect(extractNames(entries)).To(Equal([]string{"a", "b", "c", "d", "e", "f", "g", "h", "i", "j"})) - }) - - Describe("prepend", func() { - It("should correctly prepend interval", func() { - cache.set(id, "c", makeEntries("d", "e", "f"), 3) - cache.set(id, "", makeEntries("a", "b", "c"), 3) - - entries, hasEnough := cache.get(id, "", 6) - Expect(hasEnough).To(BeTrue()) - Expect(extractNames(entries)).To(Equal([]string{"a", "b", "c", "d", "e", "f"})) - }) - - It("should correctly prepend overlapping intervals", func() { - cache.set(id, "c", makeEntries("d", "e", "f"), 3) - cache.set(id, "", makeEntries("a", "b", "c", "d", "e"), 5) - - entries, hasEnough := cache.get(id, "", 6) - Expect(hasEnough).To(BeTrue()) - Expect(extractNames(entries)).To(Equal([]string{"a", "b", "c", "d", "e", "f"})) - }) - }) - - It("should discard interval if already exists", func() { - cache.set(id, "", makeEntries("a", "b", "c", "d", "e"), 5) - cache.set(id, "a", makeEntries("b", "c", "d"), 3) - - entries, hasEnough := cache.get(id, "", 5) - Expect(hasEnough).To(BeTrue()) - Expect(extractNames(entries)).To(Equal([]string{"a", "b", "c", "d", "e"})) - }) - - It("should discard interval if already exists", func() { - cache.set(id, "", makeEntries("a", "b", "c", "d", "e"), 5) - cache.set(id, "", makeEntries("a", "b"), 2) - - entries, hasEnough := cache.get(id, "", 5) - Expect(hasEnough).To(BeTrue()) - Expect(extractNames(entries)).To(Equal([]string{"a", "b", "c", "d", "e"})) - }) - - It("should return empty response if cache is empty", func() { - entries, hasEnough := cache.get(id, "", 0) - Expect(hasEnough).To(BeFalse()) - Expect(entries).To(BeNil()) - - entries, hasEnough = cache.get(id, "", 1) - Expect(hasEnough).To(BeFalse()) - Expect(entries).To(BeNil()) - - entries, hasEnough = cache.get(id, "a", 1) - Expect(hasEnough).To(BeFalse()) - Expect(entries).To(BeNil()) - }) - - It("should correctly distinguish between different caches", func() { - otherID := cacheReqID{bck: &cmn.Bck{Name: "something"}} - - cache.set(id, "", makeEntries("a", "b", "c"), 3) - entries, hasEnough := cache.get(id, "", 3) - Expect(hasEnough).To(BeTrue()) - Expect(extractNames(entries)).To(Equal([]string{"a", "b", "c"})) - - // Check if `otherID` cache is empty. - entries, hasEnough = cache.get(otherID, "", 3) - Expect(hasEnough).To(BeFalse()) - Expect(entries).To(BeNil()) - - cache.set(otherID, "", makeEntries("d", "e", "f"), 3) - entries, hasEnough = cache.get(otherID, "", 3) - Expect(hasEnough).To(BeTrue()) - Expect(extractNames(entries)).To(Equal([]string{"d", "e", "f"})) - }) - - Describe("prefix", func() { - It("should get prefixed entries from `id='bck'` cache", func() { - prefixID := cacheReqID{bck: id.bck, prefix: "p-"} - - cache.set(id, "", makeEntries("a", "p-b", "p-c", "p-d", "z"), 5) - entries, hasEnough := cache.get(id, "", 3) - Expect(hasEnough).To(BeTrue()) - Expect(extractNames(entries)).To(Equal([]string{"a", "p-b", "p-c"})) - - // Now check that getting for prefix works. - entries, hasEnough = cache.get(prefixID, "", 3) - Expect(hasEnough).To(BeTrue()) - Expect(extractNames(entries)).To(Equal([]string{"p-b", "p-c", "p-d"})) - - entries, hasEnough = cache.get(prefixID, "", 2) - Expect(hasEnough).To(BeTrue()) - Expect(extractNames(entries)).To(Equal([]string{"p-b", "p-c"})) - - // User requests more than we have but also all the prefixes are - // fully contained in the interval so we are sure that there isn't - // more of them. Therefore, we should return what we have. - entries, hasEnough = cache.get(prefixID, "", 4) - Expect(hasEnough).To(BeTrue()) - Expect(extractNames(entries)).To(Equal([]string{"p-b", "p-c", "p-d"})) - }) - - It("should get prefixed entries from `id='bck' cache (boundaries)", func() { - cache.set(id, "", makeEntries("b", "d", "y"), 3) - entries, hasEnough := cache.get(id, "", 3) - Expect(hasEnough).To(BeTrue()) - Expect(extractNames(entries)).To(Equal([]string{"b", "d", "y"})) - - // Get entries with prefix `y`. - entries, hasEnough = cache.get(cacheReqID{bck: id.bck, prefix: "y"}, "", 1) - Expect(hasEnough).To(BeTrue()) - Expect(extractNames(entries)).To(Equal([]string{"y"})) - - _, hasEnough = cache.get(cacheReqID{bck: id.bck, prefix: "y"}, "", 2) - Expect(hasEnough).To(BeFalse()) - - // Get entries with prefix `b`. - entries, hasEnough = cache.get(cacheReqID{bck: id.bck, prefix: "b"}, "", 1) - Expect(hasEnough).To(BeTrue()) - Expect(extractNames(entries)).To(Equal([]string{"b"})) - - entries, hasEnough = cache.get(cacheReqID{bck: id.bck, prefix: "b"}, "", 2) - Expect(hasEnough).To(BeTrue()) - Expect(extractNames(entries)).To(Equal([]string{"b"})) - - // Get entries with prefix `a`. - entries, hasEnough = cache.get(cacheReqID{bck: id.bck, prefix: "a"}, "", 1) - Expect(hasEnough).To(BeTrue()) - Expect(entries).To(Equal(cmn.LsoEntries{})) - - entries, hasEnough = cache.get(cacheReqID{bck: id.bck, prefix: "a"}, "", 2) - Expect(hasEnough).To(BeTrue()) - Expect(entries).To(Equal(cmn.LsoEntries{})) - - // Make interval "last". - cache.set(id, "y", makeEntries(), 1) - - // Get entries with prefix `y`. - entries, hasEnough = cache.get(cacheReqID{bck: id.bck, prefix: "y"}, "", 1) - Expect(hasEnough).To(BeTrue()) - Expect(extractNames(entries)).To(Equal([]string{"y"})) - - entries, hasEnough = cache.get(cacheReqID{bck: id.bck, prefix: "y"}, "", 2) - Expect(hasEnough).To(BeTrue()) - Expect(extractNames(entries)).To(Equal([]string{"y"})) - - // Get entries with prefix `ya`. - entries, hasEnough = cache.get(cacheReqID{bck: id.bck, prefix: "ya"}, "", 1) - Expect(hasEnough).To(BeTrue()) - Expect(entries).To(Equal(cmn.LsoEntries{})) - }) - - It("should correctly behave in `id='bck'` cache if prefix is contained in interval but there aren't matching entries", func() { - prefixID := cacheReqID{bck: id.bck, prefix: "b-"} - - cache.set(id, "", makeEntries("a", "p-b", "p-c", "p-d", "z"), 5) - entries, hasEnough := cache.get(id, "", 3) - Expect(hasEnough).To(BeTrue()) - Expect(extractNames(entries)).To(Equal([]string{"a", "p-b", "p-c"})) - - // It should correctly return no entries when prefixes are - // contained in the interval but there is no such entries. - entries, hasEnough = cache.get(prefixID, "", 2) - Expect(hasEnough).To(BeTrue()) - Expect(entries).To(Equal(cmn.LsoEntries{})) - - entries, hasEnough = cache.get(prefixID, "a", 2) - Expect(hasEnough).To(BeTrue()) - Expect(entries).To(Equal(cmn.LsoEntries{})) - }) - - It("should correctly behave in `id='bck'` cache if prefix is out of the interval", func() { - cache.set(id, "", makeEntries("b", "m", "p", "y"), 4) - entries, hasEnough := cache.get(id, "", 3) - Expect(hasEnough).To(BeTrue()) - Expect(extractNames(entries)).To(Equal([]string{"b", "m", "p"})) - - _, hasEnough = cache.get(cacheReqID{bck: id.bck, prefix: "z"}, "", 1) - Expect(hasEnough).To(BeFalse()) - }) - - It("should get prefixed entries from `id='bck+prefix'` cache", func() { - prefixID := cacheReqID{bck: id.bck, prefix: "p-"} - - cache.set(id, "", makeEntries("p-a", "p-b", "p-c", "p-d", "p-e"), 5) - entries, hasEnough := cache.get(prefixID, "", 3) - Expect(hasEnough).To(BeTrue()) - Expect(extractNames(entries)).To(Equal([]string{"p-a", "p-b", "p-c"})) - - // Now check that getting for prefix works. - entries, hasEnough = cache.get(prefixID, "p-b", 3) - Expect(hasEnough).To(BeTrue()) - Expect(extractNames(entries)).To(Equal([]string{"p-c", "p-d", "p-e"})) - - _, hasEnough = cache.get(prefixID, "p-b", 4) - Expect(hasEnough).To(BeFalse()) - }) - - It("should fallback to `id='bck'` cache when there is not enough entries in `id='bck+prefix'` cache", func() { - prefixID := cacheReqID{bck: id.bck, prefix: "p-"} - - cache.set(id, "", makeEntries("a", "p-b", "p-c", "p-d"), 4) - cache.set(prefixID, "", makeEntries("p-b", "p-c"), 2) - - entries, hasEnough := cache.get(prefixID, "", 3) - Expect(hasEnough).To(BeTrue()) - Expect(extractNames(entries)).To(Equal([]string{"p-b", "p-c", "p-d"})) - - // Insert more into `id="bck+prefix"` cache end check that we can get from it. - cache.set(prefixID, "p-c", makeEntries("p-d", "p-f", "p-g"), 3) - entries, hasEnough = cache.get(prefixID, "", 5) - Expect(hasEnough).To(BeTrue()) - Expect(extractNames(entries)).To(Equal([]string{"p-b", "p-c", "p-d", "p-f", "p-g"})) - }) - }) - }) - - Describe("ListObjectsBuffer", func() { - var ( - id = "some_id" - buffer *lsobjBuffers - ) - - BeforeEach(func() { - buffer = &lsobjBuffers{} - }) - - It("should correctly create single buffer", func() { - buffer.set(id, "target1", makeEntries("a", "d", "g"), 3) - buffer.set(id, "target2", makeEntries("b", "c", "h"), 3) - buffer.set(id, "target3", makeEntries("e", "f"), 2) - - entries, hasEnough := buffer.get(id, "", 6) - Expect(hasEnough).To(BeTrue()) - Expect(extractNames(entries)).To(Equal([]string{"a", "b", "c", "d", "e", "f"})) - - // Since `f` is the smallest of the last elements we cannot join: - // `g` and `h` because there might be still some elements after `f` - // in `target3`. Therefore, we should answer "not enough" to next calls. - entries, hasEnough = buffer.get(id, "f", 1) - Expect(hasEnough).To(BeFalse()) - Expect(entries).To(BeNil()) - }) - - It("should correctly append new entries", func() { - buffer.set(id, "target1", makeEntries("a", "d", "g"), 3) - buffer.set(id, "target2", makeEntries("b", "c", "h"), 3) - buffer.set(id, "target3", makeEntries("e", "f"), 2) - - entries, hasEnough := buffer.get(id, "", 3) - Expect(hasEnough).To(BeTrue()) - Expect(extractNames(entries)).To(Equal([]string{"a", "b", "c"})) - entries, hasEnough = buffer.get(id, "c", 4) - Expect(hasEnough).To(BeFalse()) - Expect(entries).To(BeNil()) - - last := buffer.last(id, "c") - // `f` is the smallest of the last elements of all targets, so it - // should be next continuation token. - Expect(last).To(Equal("f")) - - // Now we simulate receiving entries after `f` token. - buffer.set(id, "target1", makeEntries("g", "k", "m"), 3) - buffer.set(id, "target2", makeEntries("h", "i", "n"), 3) - buffer.set(id, "target3", makeEntries("j", "l"), 2) - - entries, hasEnough = buffer.get(id, "c", 9) - Expect(hasEnough).To(BeTrue()) - Expect(extractNames(entries)).To(Equal([]string{"d", "e", "f", "g", "h", "i", "j", "k", "l"})) - entries, hasEnough = buffer.get(id, "l", 1) - Expect(hasEnough).To(BeFalse()) - Expect(entries).To(BeNil()) - }) - - It("should correctly identify no objects", func() { - entries, hasEnough := buffer.get("id", "a", 10) - Expect(hasEnough).To(BeFalse()) - Expect(entries).To(BeNil()) - entries, hasEnough = buffer.get("id", "a", 10) - Expect(hasEnough).To(BeFalse()) - Expect(entries).To(BeNil()) - }) - - It("should correctly handle end of entries", func() { - buffer.set(id, "target1", makeEntries("a", "d", "e"), 4) - buffer.set(id, "target2", makeEntries("b", "c", "f"), 4) - - entries, hasEnough := buffer.get(id, "", 7) - Expect(hasEnough).To(BeTrue()) - Expect(extractNames(entries)).To(Equal([]string{"a", "b", "c", "d", "e", "f"})) - - entries, hasEnough = buffer.get(id, "f", 1) - Expect(hasEnough).To(BeTrue()) - Expect(entries).To(HaveLen(0)) - }) - - It("should correctly handle getting 0 entries", func() { - buffer.set(id, "target1", makeEntries(), 2) - buffer.set(id, "target2", makeEntries(), 2) - - entries, hasEnough := buffer.get(id, "", 7) - Expect(hasEnough).To(BeTrue()) - Expect(entries).To(HaveLen(0)) - - entries, hasEnough = buffer.get(id, "f", 1) - Expect(hasEnough).To(BeTrue()) - Expect(entries).To(HaveLen(0)) - }) - - It("should correctly handle rerequesting the page", func() { - buffer.set(id, "target1", makeEntries("a", "d", "g"), 3) - buffer.set(id, "target2", makeEntries("b", "c", "h"), 3) - buffer.set(id, "target3", makeEntries("e", "f"), 2) - - entries, hasEnough := buffer.get(id, "", 2) - Expect(hasEnough).To(BeTrue()) - Expect(extractNames(entries)).To(Equal([]string{"a", "b"})) - entries, hasEnough = buffer.get(id, "b", 3) - Expect(hasEnough).To(BeTrue()) - Expect(extractNames(entries)).To(Equal([]string{"c", "d", "e"})) - - // Rerequest the page with token `b`. - _, hasEnough = buffer.get(id, "b", 3) - Expect(hasEnough).To(BeFalse()) - - // Simulate targets resending data. - buffer.set(id, "target1", makeEntries("d", "g"), 2) - buffer.set(id, "target2", makeEntries("c", "h"), 2) - buffer.set(id, "target3", makeEntries("e", "f"), 2) - - entries, hasEnough = buffer.get(id, "b", 3) - Expect(hasEnough).To(BeTrue()) - Expect(extractNames(entries)).To(Equal([]string{"c", "d", "e"})) - entries, hasEnough = buffer.get(id, "e", 1) - Expect(hasEnough).To(BeTrue()) - Expect(extractNames(entries)).To(Equal([]string{"f"})) - _, hasEnough = buffer.get(id, "f", 1) - Expect(hasEnough).To(BeFalse()) - }) - }) -}) diff --git a/ais/test/bucket_test.go b/ais/test/bucket_test.go index 00a37a7ca63..3860ea9f847 100644 --- a/ais/test/bucket_test.go +++ b/ais/test/bucket_test.go @@ -931,11 +931,8 @@ func TestListObjectsProps(t *testing.T) { } tlog.Logf("%s: versioning is %s\n", m.bck.Cname(""), s) } - checkProps := func(useCache bool, props []string, f func(en *cmn.LsoEnt)) { + checkProps := func(props []string, f func(en *cmn.LsoEnt)) { msg := &apc.LsoMsg{PageSize: 100} - if useCache { - msg.SetFlag(apc.UseListObjsCache) - } msg.AddProps(props...) lst, err := api.ListObjects(baseParams, m.bck, msg, api.ListArgs{}) tassert.CheckFatal(t, err) @@ -949,74 +946,72 @@ func TestListObjectsProps(t *testing.T) { } } - for _, useCache := range []bool{false, true} { - tlog.Logf("[cache=%t] trying empty (minimal) subset of props...\n", useCache) - checkProps(useCache, []string{}, func(en *cmn.LsoEnt) { - tassert.Errorf(t, en.Name != "", "name is not set") - tassert.Errorf(t, en.Size != 0, "size is not set") + tlog.Logf("trying empty (minimal) subset of props...\n") + checkProps([]string{}, func(en *cmn.LsoEnt) { + tassert.Errorf(t, en.Name != "", "name is not set") + tassert.Errorf(t, en.Size != 0, "size is not set") - tassert.Errorf(t, en.Atime == "", "atime is set") - tassert.Errorf(t, en.Location == "", "target location is set %q", en.Location) - tassert.Errorf(t, en.Copies == 0, "copies is set") - }) + tassert.Errorf(t, en.Atime == "", "atime is set") + tassert.Errorf(t, en.Location == "", "target location is set %q", en.Location) + tassert.Errorf(t, en.Copies == 0, "copies is set") + }) - tlog.Logf("[cache=%t] trying ais-default subset of props...\n", useCache) - checkProps(useCache, apc.GetPropsDefaultAIS, func(en *cmn.LsoEnt) { - tassert.Errorf(t, en.Size != 0, "size is not set") - tassert.Errorf(t, en.Checksum != "", "checksum is not set") - tassert.Errorf(t, en.Atime != "", "atime is not set") + tlog.Logf("trying ais-default subset of props...\n") + checkProps(apc.GetPropsDefaultAIS, func(en *cmn.LsoEnt) { + tassert.Errorf(t, en.Size != 0, "size is not set") + tassert.Errorf(t, en.Checksum != "", "checksum is not set") + tassert.Errorf(t, en.Atime != "", "atime is not set") - tassert.Errorf(t, en.Location == "", "target location is set %q", en.Location) - tassert.Errorf(t, en.Copies == 0, "copies is set") - }) + tassert.Errorf(t, en.Location == "", "target location is set %q", en.Location) + tassert.Errorf(t, en.Copies == 0, "copies is set") + }) - tlog.Logf("[cache=%t] trying cloud-default subset of props...\n", useCache) - checkProps(useCache, apc.GetPropsDefaultCloud, func(en *cmn.LsoEnt) { - tassert.Errorf(t, en.Size != 0, "size is not set") + tlog.Logf("trying cloud-default subset of props...\n") + checkProps(apc.GetPropsDefaultCloud, func(en *cmn.LsoEnt) { + tassert.Errorf(t, en.Size != 0, "size is not set") + tassert.Errorf(t, en.Checksum != "", "checksum is not set") + if bck.IsAIS() || remoteVersioning { + tassert.Errorf(t, en.Version != "", "version is not set") + } + tassert.Errorf(t, !m.bck.IsCloud() || en.Custom != "", "custom is not set") + + tassert.Errorf(t, en.Atime == "", "atime is set") + tassert.Errorf(t, en.Copies == 0, "copies is set") + }) + + tlog.Logf("trying specific subset of props...\n") + checkProps( + []string{apc.GetPropsChecksum, apc.GetPropsVersion, apc.GetPropsCopies}, func(en *cmn.LsoEnt) { tassert.Errorf(t, en.Checksum != "", "checksum is not set") if bck.IsAIS() || remoteVersioning { - tassert.Errorf(t, en.Version != "", "version is not set") + tassert.Error(t, en.Version != "", "version is not set: "+m.bck.Cname(en.Name)) } - tassert.Errorf(t, !m.bck.IsCloud() || en.Custom != "", "custom is not set") + tassert.Error(t, en.Copies > 0, "copies is not set") - tassert.Errorf(t, en.Atime == "", "atime is set") - tassert.Errorf(t, en.Copies == 0, "copies is set") + tassert.Error(t, en.Atime == "", "atime is set") + tassert.Errorf(t, en.Location == "", "target location is set %q", en.Location) }) - tlog.Logf("[cache=%t] trying specific subset of props...\n", useCache) - checkProps(useCache, - []string{apc.GetPropsChecksum, apc.GetPropsVersion, apc.GetPropsCopies}, func(en *cmn.LsoEnt) { - tassert.Errorf(t, en.Checksum != "", "checksum is not set") - if bck.IsAIS() || remoteVersioning { - tassert.Error(t, en.Version != "", "version is not set: "+m.bck.Cname(en.Name)) - } - tassert.Error(t, en.Copies > 0, "copies is not set") + tlog.Logf("trying small subset of props...\n") + checkProps([]string{apc.GetPropsSize}, func(en *cmn.LsoEnt) { + tassert.Errorf(t, en.Size != 0, "size is not set") - tassert.Error(t, en.Atime == "", "atime is set") - tassert.Errorf(t, en.Location == "", "target location is set %q", en.Location) - }) - - tlog.Logf("[cache=%t] trying small subset of props...\n", useCache) - checkProps(useCache, []string{apc.GetPropsSize}, func(en *cmn.LsoEnt) { - tassert.Errorf(t, en.Size != 0, "size is not set") - - tassert.Errorf(t, en.Atime == "", "atime is set") - tassert.Errorf(t, en.Location == "", "target location is set %q", en.Location) - tassert.Errorf(t, en.Copies == 0, "copies is set") - }) + tassert.Errorf(t, en.Atime == "", "atime is set") + tassert.Errorf(t, en.Location == "", "target location is set %q", en.Location) + tassert.Errorf(t, en.Copies == 0, "copies is set") + }) - tlog.Logf("[cache=%t] trying all props...\n", useCache) - checkProps(useCache, apc.GetPropsAll, func(en *cmn.LsoEnt) { - tassert.Errorf(t, en.Size != 0, "size is not set") - if bck.IsAIS() || remoteVersioning { - tassert.Error(t, en.Version != "", "version is not set: "+m.bck.Cname(en.Name)) - } - tassert.Errorf(t, en.Checksum != "", "checksum is not set") - tassert.Errorf(t, en.Atime != "", "atime is not set") - tassert.Errorf(t, en.Location != "", "target location is not set [%#v]", en) - tassert.Errorf(t, en.Copies != 0, "copies is not set") - }) - } + tlog.Logf("trying all props...\n") + checkProps(apc.GetPropsAll, func(en *cmn.LsoEnt) { + tassert.Errorf(t, en.Size != 0, "size is not set") + if bck.IsAIS() || remoteVersioning { + tassert.Error(t, en.Version != "", "version is not set: "+m.bck.Cname(en.Name)) + } + tassert.Errorf(t, en.Checksum != "", "checksum is not set") + tassert.Errorf(t, en.Atime != "", "atime is not set") + tassert.Errorf(t, en.Location != "", "target location is not set [%#v]", en) + tassert.Errorf(t, en.Copies != 0, "copies is not set") + }) }) } @@ -1248,7 +1243,6 @@ func TestListObjects(t *testing.T) { // Confirm PUTs by listing objects. msg := &apc.LsoMsg{PageSize: test.pageSize} msg.AddProps(apc.GetPropsChecksum, apc.GetPropsAtime, apc.GetPropsVersion, apc.GetPropsCopies, apc.GetPropsSize) - tassert.CheckError(t, api.ListObjectsInvalidateCache(baseParams, bck)) lst, err := api.ListObjects(baseParams, bck, msg, api.ListArgs{}) tassert.CheckFatal(t, err) @@ -1324,8 +1318,6 @@ func TestListObjects(t *testing.T) { return true }) } - - tassert.CheckError(t, api.ListObjectsInvalidateCache(baseParams, bck)) }) } } @@ -1486,36 +1478,24 @@ func TestListObjectsCache(t *testing.T) { tools.CreateBucket(t, m.proxyURL, m.bck, nil, true /*cleanup*/) m.puts() - for _, useCache := range []bool{true, false} { - t.Run(fmt.Sprintf("cache=%t", useCache), func(t *testing.T) { - // Do it N times - first: fill the cache; next calls: use it. - for iter := range totalIters { - var ( - started = time.Now() - msg = &apc.LsoMsg{PageSize: rand.Int64N(20) + 4} - ) - if useCache { - msg.SetFlag(apc.UseListObjsCache) - } - lst, err := api.ListObjects(baseParams, m.bck, msg, api.ListArgs{}) - tassert.CheckFatal(t, err) - - tlog.Logf( - "[iter: %d] cache: %5t, page_size: %d, time: %s\n", - iter, useCache, msg.PageSize, time.Since(started), - ) + // Do it N times - first: fill the cache; next calls: use it. + for iter := range totalIters { + var ( + started = time.Now() + msg = &apc.LsoMsg{PageSize: rand.Int64N(20) + 4} + ) + lst, err := api.ListObjects(baseParams, m.bck, msg, api.ListArgs{}) + tassert.CheckFatal(t, err) - tassert.Errorf( - t, len(lst.Entries) == m.num, - "unexpected number of entries (got: %d, expected: %d)", len(lst.Entries), m.num, - ) - } + tlog.Logf( + "[iter: %d] page_size: %d, time: %s\n", + iter, msg.PageSize, time.Since(started), + ) - if useCache { - err := api.ListObjectsInvalidateCache(baseParams, m.bck) - tassert.CheckError(t, err) - } - }) + tassert.Errorf( + t, len(lst.Entries) == m.num, + "unexpected number of entries (got: %d, expected: %d)", len(lst.Entries), m.num, + ) } } diff --git a/api/apc/actmsg.go b/api/apc/actmsg.go index 3d2df03aa40..6ce60ca4be0 100644 --- a/api/apc/actmsg.go +++ b/api/apc/actmsg.go @@ -50,7 +50,6 @@ const ( ActStoreCleanup = "cleanup-store" ActEvictRemoteBck = "evict-remote-bck" // evict remote bucket's data - ActInvalListCache = "inval-listobj-cache" ActList = "list" ActLoadLomCache = "load-lom-cache" ActNewPrimary = "new-primary" diff --git a/api/apc/lsmsg.go b/api/apc/lsmsg.go index ddad3bc5ff3..523a87a720d 100644 --- a/api/apc/lsmsg.go +++ b/api/apc/lsmsg.go @@ -56,8 +56,8 @@ const ( // * `QparamDontAddRemote` (this package) LsDontAddRemote - // cache list-objects results and use this cache to speed-up - UseListObjsCache + // Deprecated + _useListObjsCache //nolint:unused // Kept for backward compatibility. // For remote buckets - list only remote props (aka `wantOnlyRemote`). When false, // the default that's being used is: `WantOnlyRemoteProps` - see below. diff --git a/api/ls.go b/api/ls.go index 87c9f4eae03..822dff79803 100644 --- a/api/ls.go +++ b/api/ls.go @@ -240,26 +240,6 @@ func ListObjectsPage(bp BaseParams, bck cmn.Bck, lsmsg *apc.LsoMsg, args ListArg return page, nil } -// TODO: obsolete this function after introducing mechanism to detect remote bucket changes. -func ListObjectsInvalidateCache(bp BaseParams, bck cmn.Bck) error { - var ( - path = apc.URLPathBuckets.Join(bck.Name) - q = url.Values{} - ) - bp.Method = http.MethodPost - reqParams := AllocRp() - { - reqParams.Query = bck.AddToQuery(q) - reqParams.BaseParams = bp - reqParams.Path = path - reqParams.Body = cos.MustMarshal(apc.ActMsg{Action: apc.ActInvalListCache}) - reqParams.Header = http.Header{cos.HdrContentType: []string{cos.ContentJSON}} - } - err := reqParams.DoRequest() - FreeRp(reqParams) - return err -} - //////////////// // LsoCounter // //////////////// diff --git a/bench/microbenchmarks/apitests/listobj_test.go b/bench/microbenchmarks/apitests/listobj_test.go index 313d3a4a264..f47e223370e 100644 --- a/bench/microbenchmarks/apitests/listobj_test.go +++ b/bench/microbenchmarks/apitests/listobj_test.go @@ -21,13 +21,12 @@ import ( type testConfig struct { objectCnt int pageSize int - useCache bool } func (tc testConfig) name() string { return fmt.Sprintf( - "objs:%d/use_cache:%t/page_size:%d", - tc.objectCnt, tc.useCache, tc.pageSize, + "objs:%d/page_size:%d", + tc.objectCnt, tc.pageSize, ) } @@ -60,18 +59,12 @@ func BenchmarkListObject(b *testing.B) { tools.CheckSkip(b, &tools.SkipTestArgs{Long: true}) u := "http://127.0.0.1:8080" tests := []testConfig{ - {objectCnt: 1_000, pageSize: 10, useCache: false}, - {objectCnt: 1_000, pageSize: 10, useCache: true}, - - {objectCnt: 10_000, pageSize: 100, useCache: false}, - {objectCnt: 10_000, pageSize: 100, useCache: true}, - - {objectCnt: 10_000, pageSize: 10_000, useCache: false}, - {objectCnt: 10_000, pageSize: 10_000, useCache: true}, - + {objectCnt: 1_000, pageSize: 10}, + {objectCnt: 10_000, pageSize: 100}, + {objectCnt: 10_000, pageSize: 10_000}, // Hardcore cases, use only when needed. - // {objectCnt: 100_000, pageSize: 10_000, useCache: true}, - // {objectCnt: 1_000_000, pageSize: 10_000, useCache: true}, + // {objectCnt: 100_000, pageSize: 10_000}, + // {objectCnt: 1_000_000, pageSize: 10_000}, } for _, test := range tests { b.Run(test.name(), func(b *testing.B) { @@ -83,9 +76,6 @@ func BenchmarkListObject(b *testing.B) { b.ResetTimer() for range b.N { msg := &apc.LsoMsg{PageSize: int64(test.pageSize)} - if test.useCache { - msg.SetFlag(apc.UseListObjsCache) - } objs, err := api.ListObjects(baseParams, bck, msg, api.ListArgs{}) tassert.CheckFatal(b, err) tassert.Errorf( diff --git a/cmn/objlist_utils.go b/cmn/objlist_utils.go index 8fd4bad6e41..ce141820e9f 100644 --- a/cmn/objlist_utils.go +++ b/cmn/objlist_utils.go @@ -125,7 +125,7 @@ func (be *LsoEnt) CopyWithProps(propsSet cos.StrSet) (ne *LsoEnt) { func SortLso(entries LsoEntries) { sort.Slice(entries, entries.cmp) } -func DedupLso(entries LsoEntries, maxSize int, noDirs bool) []*LsoEnt { +func dedupLso(entries LsoEntries, maxSize int, noDirs bool) []*LsoEnt { var j int for _, en := range entries { if j > 0 && entries[j-1].Name == en.Name { @@ -157,7 +157,7 @@ func MergeLso(lists []*LsoRes, lsmsg *apc.LsoMsg, maxSize int) *LsoRes { token := resList.ContinuationToken if len(lists) == 1 { SortLso(resList.Entries) - resList.Entries = DedupLso(resList.Entries, maxSize, noDirs) + resList.Entries = dedupLso(resList.Entries, maxSize, noDirs) resList.ContinuationToken = token return resList } @@ -212,6 +212,56 @@ func MergeLso(lists []*LsoRes, lsmsg *apc.LsoMsg, maxSize int) *LsoRes { return resList } +// ConcatLso takes a slice of object lists and concatenates them: all lists +// are appended to the first one. +// If maxSize is greater than 0, the resulting list is sorted and truncated. Zero +// or negative maxSize means returning all objects. +func ConcatLso(lists []*LsoRes, lsmsg *apc.LsoMsg, maxSize int) (objs *LsoRes) { + objs = &LsoRes{ + UUID: lsmsg.UUID, + } + + if len(lists) == 0 { + return objs + } + + entryCount := 0 + for _, l := range lists { + objs.Flags |= l.Flags + entryCount += len(l.Entries) + } + if entryCount == 0 { + return objs + } + + objs.Entries = make(LsoEntries, 0, entryCount) + for _, l := range lists { + objs.Entries = append(objs.Entries, l.Entries...) + clear(l.Entries) + } + + // For corner case: we have objects with replicas on page threshold + // we have to sort taking status into account. Otherwise wrong + // one(Status=moved) may get into the response + SortLso(objs.Entries) + + // Remove duplicates + // when recursion is disabled (i.e., lsmsg.IsFlagSet(apc.LsNoRecursion)) + // the (`cmn.LsoRes`) result _may_ include duplicated names of the virtual subdirectories + // - that's why: + if lsmsg.IsFlagSet(apc.LsNoRecursion) { + objs.Entries = dedupLso(objs.Entries, maxSize, false /*no-dirs*/) + } + + if maxSize > 0 && len(objs.Entries) >= maxSize { + objs.Entries = objs.Entries[:maxSize] + clear(objs.Entries[maxSize:]) + objs.ContinuationToken = objs.Entries[len(objs.Entries)-1].Name + } + + return +} + // Returns true if the continuation token >= object's name (in other words, the object is // already listed and must be skipped). Note that string `>=` is lexicographic. func TokenGreaterEQ(token, objName string) bool { return token >= objName } diff --git a/xact/api.go b/xact/api.go index 502c01ad1a3..d021c6a4790 100644 --- a/xact/api.go +++ b/xact/api.go @@ -262,8 +262,7 @@ var Table = map[string]Descriptor{ apc.ActList: {Scope: ScopeB, Access: apc.AceObjLIST, Startable: false, Metasync: false, Idles: true}, // cache management, internal usage - apc.ActLoadLomCache: {DisplayName: "warm-up-metadata", Scope: ScopeB, Startable: true}, - apc.ActInvalListCache: {Scope: ScopeB, Access: apc.AceObjLIST, Startable: false}, + apc.ActLoadLomCache: {DisplayName: "warm-up-metadata", Scope: ScopeB, Startable: true}, } func GetDescriptor(kindOrName string) (string, Descriptor, error) { diff --git a/xact/xs/utils_test.go b/xact/xs/utils_test.go deleted file mode 100644 index ce0f91e31de..00000000000 --- a/xact/xs/utils_test.go +++ /dev/null @@ -1,110 +0,0 @@ -// Package xs_test - basic list-concatenation unit tests. -/* - * Copyright (c) 2018-2024, NVIDIA CORPORATION. All rights reserved. - */ -package xs_test - -import ( - "testing" - - "github.com/NVIDIA/aistore/cmn" - "github.com/NVIDIA/aistore/tools/tassert" - "github.com/NVIDIA/aistore/tools/trand" -) - -func TestConcatObjLists(t *testing.T) { - if testing.Short() { - t.Skipf("skipping %s in short mode", t.Name()) - } - tests := []struct { - name string - objCounts []int - maxSize int - token bool - }{ - // * `st` stands for "single target" - {name: "st/all", objCounts: []int{10}, maxSize: 0, token: false}, - {name: "st/half", objCounts: []int{10}, maxSize: 5, token: true}, - {name: "st/all_with_marker", objCounts: []int{10}, maxSize: 10, token: true}, - {name: "st/more_than_all", objCounts: []int{10}, maxSize: 11, token: false}, - - // * `mt` stands for "multiple targets" - // * `one` stands for "one target has objects" - {name: "mt/one/all", objCounts: []int{0, 0, 10}, maxSize: 0, token: false}, - {name: "mt/one/half", objCounts: []int{0, 0, 10}, maxSize: 5, token: true}, - {name: "mt/one/all_with_marker", objCounts: []int{0, 0, 10}, maxSize: 10, token: true}, - {name: "mt/one/more_than_all", objCounts: []int{0, 0, 10}, maxSize: 11, token: false}, - - // * `mt` stands for "multiple targets" - // * `more` stands for "more than one target has objects" - {name: "mt/more/all", objCounts: []int{5, 1, 4, 10}, maxSize: 0, token: false}, - {name: "mt/more/half", objCounts: []int{5, 1, 4, 10}, maxSize: 10, token: true}, - {name: "mt/more/all_with_marker", objCounts: []int{5, 1, 4, 10}, maxSize: 20, token: true}, - {name: "mt/more/more_than_all", objCounts: []int{5, 1, 4, 10}, maxSize: 21, token: false}, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - var ( - lists = make([]*cmn.LsoRes, 0, len(test.objCounts)) - expectedObjCnt = 0 - ) - for _, objCount := range test.objCounts { - list := &cmn.LsoRes{} - for range objCount { - list.Entries = append(list.Entries, &cmn.LsoEnt{ - Name: trand.String(5), - }) - } - lists = append(lists, list) - expectedObjCnt += len(list.Entries) - } - expectedObjCnt = min(expectedObjCnt, test.maxSize) - - objs := concatLso(lists, test.maxSize) - tassert.Errorf( - t, test.maxSize == 0 || len(objs.Entries) == expectedObjCnt, - "number of objects (%d) is different from expected (%d)", len(objs.Entries), expectedObjCnt, - ) - tassert.Errorf( - t, (objs.ContinuationToken != "") == test.token, - "continuation token expected to be set=%t", test.token, - ) - }) - } -} - -// concatLso takes a slice of object lists and concatenates them: all lists -// are appended to the first one. -// If maxSize is greater than 0, the resulting list is sorted and truncated. Zero -// or negative maxSize means returning all objects. -func concatLso(lists []*cmn.LsoRes, maxSize int) (objs *cmn.LsoRes) { - if len(lists) == 0 { - return &cmn.LsoRes{} - } - - objs = &cmn.LsoRes{} - objs.Entries = make(cmn.LsoEntries, 0) - - for _, l := range lists { - objs.Flags |= l.Flags - objs.Entries = append(objs.Entries, l.Entries...) - } - - if len(objs.Entries) == 0 { - return objs - } - - // For corner case: we have objects with replicas on page threshold - // we have to sort taking status into account. Otherwise wrong - // one(Status=moved) may get into the response - cmn.SortLso(objs.Entries) - - // Remove duplicates - objs.Entries = cmn.DedupLso(objs.Entries, maxSize, false /*no-dirs*/) - l := len(objs.Entries) - if maxSize > 0 && l >= maxSize { - objs.ContinuationToken = objs.Entries[l-1].Name - } - return -}