From 5eac4e3e024054f6252498e2546342887e067019 Mon Sep 17 00:00:00 2001 From: egor Date: Tue, 3 Sep 2024 15:01:58 +0300 Subject: [PATCH] RavenDB-16815 - add new flag to attachment tombstone, keep retireAt date time after retirement as well --- .../RetiredAttachmentsConfiguration.cs | 5 + .../Documents/AttachmentsStorage.cs | 57 +++++-- .../Documents/DocumentsStorage.cs | 19 ++- .../Incoming/IncomingReplicationHandler.cs | 22 ++- .../AttachmentTombstoneFlags.cs | 12 ++ .../AttachmentTombstoneReplicationItem.cs | 12 +- .../TombstoneReplicationItem.cs | 54 +++++-- .../AzureRetiredAttachmentsSlowTests.cs | 2 +- ...lder.cs => RetiredAttachmentsAzureBase.cs} | 6 +- .../Attachments/RetiredAttachmentsHolder.cs | 19 ++- .../RetiredAttachmentsHolderBase.cs | 2 +- ...sHolder.cs => RetiredAttachmentsS3Base.cs} | 6 +- .../S3RetiredAttachmentsSlowTests.cs | 147 +++++++++++++++++- 13 files changed, 317 insertions(+), 46 deletions(-) create mode 100644 src/Raven.Server/Documents/Replication/ReplicationItems/AttachmentTombstoneFlags.cs rename test/SlowTests/Server/Documents/Attachments/{AzureRetiredAttachmentsHolder.cs => RetiredAttachmentsAzureBase.cs} (93%) rename test/SlowTests/Server/Documents/Attachments/{S3RetiredAttachmentsHolder.cs => RetiredAttachmentsS3Base.cs} (93%) diff --git a/src/Raven.Client/Documents/Attachments/RetiredAttachmentsConfiguration.cs b/src/Raven.Client/Documents/Attachments/RetiredAttachmentsConfiguration.cs index 126ff40e3c73..9cf6367eb29b 100644 --- a/src/Raven.Client/Documents/Attachments/RetiredAttachmentsConfiguration.cs +++ b/src/Raven.Client/Documents/Attachments/RetiredAttachmentsConfiguration.cs @@ -15,6 +15,11 @@ public sealed class RetiredAttachmentsConfiguration : IDynamicJson public Dictionary RetirePeriods { get; set; } public long? RetireFrequencyInSec { get; set; } public long? MaxItemsToProcess { get; set; } + + /// + /// Purge the retired attachments when the document is deleted. + /// Default: false + /// public bool PurgeOnDelete { get; set; } public override int GetHashCode() diff --git a/src/Raven.Server/Documents/AttachmentsStorage.cs b/src/Raven.Server/Documents/AttachmentsStorage.cs index 856b370a06f5..9662aee3135b 100644 --- a/src/Raven.Server/Documents/AttachmentsStorage.cs +++ b/src/Raven.Server/Documents/AttachmentsStorage.cs @@ -207,11 +207,14 @@ public void RetireAttachment(DocumentsOperationContext context, AttachmentDetail var attachmentEtag = _documentsStorage.GenerateNextEtag(); var changeVector = _documentsStorage.GetNewChangeVector(context, attachmentEtag); var size = TableValueToLong((int)AttachmentsTable.Size, ref attachmentTvr); + var retireAt = TableValueToLong((int)AttachmentsTable.RetireAt, ref attachmentTvr); + using (TableValueToSlice(context, (int)AttachmentsTable.Name, ref attachmentTvr, out var nameSlice)) using (TableValueToSlice(context, (int)AttachmentsTable.ContentType, ref attachmentTvr, out var contentTypeSlice)) using (TableValueToSlice(context, (int)AttachmentsTable.Hash, ref attachmentTvr, out var hashSlice)) using (Slice.From(context.Allocator, changeVector, out var changeVectorSlice)) using (TableValueToSlice(context, (int)AttachmentsTable.Collection, ref attachmentTvr, out var collectionSlice)) + using (table.Allocate(out TableValueBuilder tvb)) { // add Retired flag @@ -224,7 +227,7 @@ public void RetireAttachment(DocumentsOperationContext context, AttachmentDetail tvb.Add(changeVectorSlice); tvb.Add(size); tvb.Add(Bits.SwapBytes((int)AttachmentFlags.Retired)); - tvb.Add(-1L); + tvb.Add(retireAt); tvb.Add(collectionSlice); table.Update(attachmentTvr.Id, tvb); @@ -311,7 +314,8 @@ void SetTableValue(TableValueBuilder tvb, Slice cv) size = TableValueToLong((int)AttachmentsTable.Size, ref oldValue); retireAt = TableValueToLong((int)AttachmentsTable.RetireAt, ref oldValue); - if (retireAt == -1L) + var existingFlags = TableValueToAttachmentFlags((int)AttachmentsTable.Flags, ref oldValue); + if (existingFlags.HasFlag(AttachmentFlags.Retired) == false && retireAt == -1L) { var dbRecord = _documentDatabase.ReadDatabaseRecord(); Debug.Assert(collectionName != null, "collectionName != null"); @@ -372,10 +376,37 @@ void SetTableValue(TableValueBuilder tvb, Slice cv) { var existingEtag = TableValueToEtag((int)AttachmentsTable.Etag, ref partialTvr); var lastModifiedTicks = _documentDatabase.Time.GetUtcNow().Ticks; - //TODO: egor if we update retired attachment (can this even happen?) we might delete the old one from cloud storage... + //TODO: egor if we update retired attachment (can this even happen?) we might delete the old one from cloud storage... I think need to check if it is retired && if it has purgeOn delete ? var existingAttachmentFlags = TableValueToAttachmentFlags((int)AttachmentsTable.Flags, ref partialTvr); var existingRetireAtTicks = TableValueToLong((int)AttachmentsTable.RetireAt, ref partialTvr); - DeleteInternal(context, existingKey, existingEtag, existingHash, changeVector, lastModifiedTicks, flags: DocumentFlags.None, existingAttachmentFlags, existingRetireAtTicks, collectionName.Name); + + if (existingAttachmentFlags.Contain(AttachmentFlags.Retired)) + { + var dbRecord2 = _documentDatabase.ReadDatabaseRecord(); + if (dbRecord2.RetiredAttachments is { Disabled: false }) + { + if (dbRecord2.RetiredAttachments.PurgeOnDelete == false) + { + // we cannot delete from cloud since PurgeOnDelete is false + DeleteInternal(context, existingKey, existingEtag, existingHash, changeVector, lastModifiedTicks, flags: DocumentFlags.None, existingAttachmentFlags, existingRetireAtTicks, collectionName.Name, storageOnly: true); + + } + else + { + DeleteInternal(context, existingKey, existingEtag, existingHash, changeVector, lastModifiedTicks, flags: DocumentFlags.None, existingAttachmentFlags, existingRetireAtTicks, collectionName.Name, storageOnly: false); + } + } + else + { + DeleteInternal(context, existingKey, existingEtag, existingHash, changeVector, lastModifiedTicks, flags: DocumentFlags.None, existingAttachmentFlags, existingRetireAtTicks, collectionName.Name, storageOnly: false); + } + } + else + { + // we cannot delete from retired since there is no configuration + DeleteInternal(context, existingKey, existingEtag, existingHash, changeVector, lastModifiedTicks, flags: DocumentFlags.None, existingAttachmentFlags, existingRetireAtTicks, collectionName.Name, storageOnly: false); + } + } } } @@ -403,7 +434,7 @@ void SetTableValue(TableValueBuilder tvb, Slice cv) { if (fromEtl && flags.Contain(AttachmentFlags.Retired)) { - retireAt = -1L; + retireAt = retireAtDt.HasValue == false ? -1L : retireAtDt.Value.Ticks; } else { @@ -1535,7 +1566,7 @@ public void DeleteAttachmentDirect(DocumentsOperationContext context, Slice key, attachmentEtag = _documentsStorage.GenerateNextEtagForReplicatedTombstoneMissingDocument(context); } - CreateTombstone(context, key, attachmentEtag, changeVector, lastModifiedTicks, flags: DocumentFlags.None); + CreateTombstone(context, key, attachmentEtag, changeVector, lastModifiedTicks, (int)DocumentFlags.None); return; } @@ -1569,7 +1600,6 @@ public void DeleteAttachmentDirect(DocumentsOperationContext context, Slice key, private void DeleteInternal(DocumentsOperationContext context, Slice key, long etag, Slice hash, string changeVector, long lastModifiedTicks, DocumentFlags flags, AttachmentFlags attachmentFlags, long retireAtTicks, string collection, bool storageOnly = false) { - CreateTombstone(context, key, etag, changeVector, lastModifiedTicks, flags); if (attachmentFlags.HasFlag(AttachmentFlags.Retired)) { if (storageOnly == false) @@ -1577,9 +1607,18 @@ private void DeleteInternal(DocumentsOperationContext context, Slice key, long e // populate retired tree TryDeleteRetiredAttachment(context, key, collection); } + else + { + RetiredAttachmentsStorage.RemoveRetirePutValue(context, key, retireAtTicks); + + // we create attachment tombstone with special flag to mark that we don't want to delete the attachment from cloud + CreateTombstone(context, key, etag, changeVector, lastModifiedTicks, (int)AttachmentTombstoneFlags.FromStorageOnly); + } } else { + CreateTombstone(context, key, etag, changeVector, lastModifiedTicks, (int)flags); + if (retireAtTicks != -1) { RetiredAttachmentsStorage.RemoveRetirePutValue(context, key, retireAtTicks); @@ -1598,7 +1637,7 @@ private void DeleteTombstoneIfNeeded(DocumentsOperationContext context, Slice ke } private void CreateTombstone(DocumentsOperationContext context, Slice keySlice, long attachmentEtag, - string changeVector, long lastModifiedTicks, DocumentFlags flags) + string changeVector, long lastModifiedTicks, int flags) { var newEtag = _documentsStorage.GenerateNextEtag(); @@ -1616,7 +1655,7 @@ private void CreateTombstone(DocumentsOperationContext context, Slice keySlice, tvb.Add(context.GetTransactionMarker()); tvb.Add((byte)Tombstone.TombstoneType.Attachment); tvb.Add(null, 0); - tvb.Add((int)flags); + tvb.Add(flags); tvb.Add(cv.Content.Ptr, cv.Size); tvb.Add(lastModifiedTicks); table.Insert(tvb); diff --git a/src/Raven.Server/Documents/DocumentsStorage.cs b/src/Raven.Server/Documents/DocumentsStorage.cs index 1f067edfe967..54837dc549f8 100644 --- a/src/Raven.Server/Documents/DocumentsStorage.cs +++ b/src/Raven.Server/Documents/DocumentsStorage.cs @@ -6,6 +6,7 @@ using System.Linq; using System.Runtime.CompilerServices; using System.Threading; +using Mono.Unix.Native; using Raven.Client; using Raven.Client.Documents.Attachments; using Raven.Client.Documents.Changes; @@ -1257,7 +1258,7 @@ public IEnumerable GetTombstonesFrom(DocumentsOperationCon // ReSharper disable once LoopCanBeConvertedToQuery foreach (var result in table.SeekForwardFrom(TombstonesSchema.FixedSizeIndexes[AllTombstonesEtagsSlice], etag, 0)) { - var tombstoneItem = TombstoneReplicationItem.From(context, TableValueToTombstone(context, ref result.Reader)); + var tombstoneItem = TombstoneReplicationItem.From(context, ref result.Reader); if (revisionTombstonesWithId == false && tombstoneItem is RevisionTombstoneReplicationItem revisionTombstone) revisionTombstone.StripDocumentIdFromKeyIfNeeded(context); @@ -1579,7 +1580,6 @@ public static Tombstone TableValueToTombstone(JsonOperationContext context, ref TransactionMarker = *(short*)tvr.Read((int)TombstoneTable.TransactionMarker, out int _), ChangeVector = TableValueToChangeVector(context, (int)TombstoneTable.ChangeVector, ref tvr), LastModified = TableValueToDateTime((int)TombstoneTable.LastModified, ref tvr), - Flags = TableValueToFlags((int)TombstoneTable.Flags, ref tvr) }; switch (result.Type) @@ -1587,9 +1587,18 @@ public static Tombstone TableValueToTombstone(JsonOperationContext context, ref case Tombstone.TombstoneType.Document: result.Collection = TableValueToId(context, (int)TombstoneTable.Collection, ref tvr); result.LowerId = UnwrapLowerIdIfNeeded(context, result.LowerId); + result.Flags = TableValueToFlags((int)TombstoneTable.Flags, ref tvr); + break; case Tombstone.TombstoneType.Revision: result.Collection = TableValueToId(context, (int)TombstoneTable.Collection, ref tvr); + result.Flags = TableValueToFlags((int)TombstoneTable.Flags, ref tvr); + break; + case Tombstone.TombstoneType.Attachment: + result.Flags = TableValueToFlags((int)TombstoneTable.Flags, ref tvr); + break; + case Tombstone.TombstoneType.Counter: + result.Flags = TableValueToFlags((int)TombstoneTable.Flags, ref tvr); break; } @@ -2789,6 +2798,12 @@ public static DocumentFlags TableValueToFlags(int index, ref TableValueReader tv return *(DocumentFlags*)tvr.Read(index, out _); } + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static int TableValueToInt(int index, ref TableValueReader tvr) + { + return *(int*)tvr.Read(index, out _); + } + [MethodImpl(MethodImplOptions.AggressiveInlining)] public static AttachmentFlags TableValueToAttachmentFlags(int index, ref TableValueReader tvr) { diff --git a/src/Raven.Server/Documents/Replication/Incoming/IncomingReplicationHandler.cs b/src/Raven.Server/Documents/Replication/Incoming/IncomingReplicationHandler.cs index c68d71e061de..985ac2605fd2 100644 --- a/src/Raven.Server/Documents/Replication/Incoming/IncomingReplicationHandler.cs +++ b/src/Raven.Server/Documents/Replication/Incoming/IncomingReplicationHandler.cs @@ -437,10 +437,24 @@ protected override long ExecuteCmd(DocumentsOperationContext context) try { - - database.DocumentsStorage.AttachmentsStorage.DeleteAttachmentDirect(context, attachmentTombstone.Key, false, "$fromReplication", null, - newChangeVector, - attachmentTombstone.LastModifiedTicks); + if (attachmentTombstone.TombstoneFlags.HasFlag(AttachmentTombstoneFlags.FromStorageOnly)) + { + + + + database.DocumentsStorage.AttachmentsStorage.DeleteAttachmentDirect(context, attachmentTombstone.Key, false, "$fromReplication", null, + newChangeVector, + attachmentTombstone.LastModifiedTicks, storageOnly: true); + } + else + { + + database.DocumentsStorage.AttachmentsStorage.DeleteAttachmentDirect(context, attachmentTombstone.Key, false, "$fromReplication", null, + newChangeVector, + attachmentTombstone.LastModifiedTicks, storageOnly: false); + + } + } catch (Exception e) { diff --git a/src/Raven.Server/Documents/Replication/ReplicationItems/AttachmentTombstoneFlags.cs b/src/Raven.Server/Documents/Replication/ReplicationItems/AttachmentTombstoneFlags.cs new file mode 100644 index 000000000000..8c9f88fa4b99 --- /dev/null +++ b/src/Raven.Server/Documents/Replication/ReplicationItems/AttachmentTombstoneFlags.cs @@ -0,0 +1,12 @@ +using System; + +namespace Raven.Server.Documents.Replication.ReplicationItems; + +[Flags] +public enum AttachmentTombstoneFlags +{ + FromStorageOnly = 0x7FFFFFFF, + + + None = 0 +} diff --git a/src/Raven.Server/Documents/Replication/ReplicationItems/AttachmentTombstoneReplicationItem.cs b/src/Raven.Server/Documents/Replication/ReplicationItems/AttachmentTombstoneReplicationItem.cs index 0bf0ff4ef562..3b07628f74d1 100644 --- a/src/Raven.Server/Documents/Replication/ReplicationItems/AttachmentTombstoneReplicationItem.cs +++ b/src/Raven.Server/Documents/Replication/ReplicationItems/AttachmentTombstoneReplicationItem.cs @@ -13,16 +13,20 @@ public sealed class AttachmentTombstoneReplicationItem : ReplicationBatchItem { public Slice Key; public DocumentFlags Flags; + public AttachmentTombstoneFlags TombstoneFlags; public override long Size => base.Size + // common sizeof(long) + // Last modified ticks sizeof(int) + // size of key - Key.Size; + Key.Size + + sizeof(AttachmentTombstoneFlags); public override DynamicJsonValue ToDebugJson() { var djv = base.ToDebugJson(); djv[nameof(Key)] = CompoundKeyHelper.ExtractDocumentId(Key); + djv[nameof(Flags)] = Flags; + djv[nameof(TombstoneFlags)] = TombstoneFlags; return djv; } @@ -46,6 +50,9 @@ public override unsafe void Write(Slice changeVector, Stream stream, byte[] temp Memory.Copy(pTemp + tempBufferPos, Key.Content.Ptr, Key.Size); tempBufferPos += Key.Size; + *(AttachmentTombstoneFlags*)(pTemp + tempBufferPos) = TombstoneFlags; + tempBufferPos += sizeof(AttachmentTombstoneFlags); + stream.Write(tempBuffer, 0, tempBufferPos); stats.RecordAttachmentTombstoneOutput(Size); @@ -61,6 +68,7 @@ public override unsafe void Read(JsonOperationContext context, ByteStringContext var size = *(int*)Reader.ReadExactly(sizeof(int)); ToDispose(Slice.From(allocator, Reader.ReadExactly(size), size, ByteStringType.Immutable, out Key)); + TombstoneFlags = *(AttachmentTombstoneFlags*)Reader.ReadExactly(sizeof(AttachmentTombstoneFlags)) | AttachmentTombstoneFlags.None; stats.RecordAttachmentTombstoneRead(Size); } } @@ -69,7 +77,7 @@ protected override ReplicationBatchItem CloneInternal(JsonOperationContext conte { var item = new AttachmentTombstoneReplicationItem(); item.Key = Key.Clone(allocator); - + item.TombstoneFlags = TombstoneFlags; item.ToDispose(new DisposableAction(() => { item.Key.Release(allocator); diff --git a/src/Raven.Server/Documents/Replication/ReplicationItems/TombstoneReplicationItem.cs b/src/Raven.Server/Documents/Replication/ReplicationItems/TombstoneReplicationItem.cs index a207979b7f0e..c18bca4a1f72 100644 --- a/src/Raven.Server/Documents/Replication/ReplicationItems/TombstoneReplicationItem.cs +++ b/src/Raven.Server/Documents/Replication/ReplicationItems/TombstoneReplicationItem.cs @@ -2,11 +2,38 @@ using Raven.Server.ServerWide.Context; using Sparrow.Server; using Voron; +using Voron.Data.Tables; +using static Raven.Server.Documents.Schemas.Tombstones; namespace Raven.Server.Documents.Replication.ReplicationItems { public sealed class TombstoneReplicationItem { + public static ReplicationBatchItem From(DocumentsOperationContext context, ref TableValueReader tvr) + { + var doc = DocumentsStorage.TableValueToTombstone(context, ref tvr); + switch (doc.Type) + { + case Tombstone.TombstoneType.Attachment: + AttachmentTombstoneReplicationItem item = AttachmentTombstoneReplicationItemInternal(context, doc); + + var enumVal = DocumentsStorage.TableValueToInt((int)TombstoneTable.Flags, ref tvr); + if (enumVal == (int)AttachmentTombstoneFlags.FromStorageOnly) + { + item.TombstoneFlags = AttachmentTombstoneFlags.FromStorageOnly; + } + else + { + item.Flags = DocumentsStorage.TableValueToFlags((int)TombstoneTable.Flags, ref tvr); + item.TombstoneFlags = AttachmentTombstoneFlags.None; + } + + return item; + default: + return From(context, doc); + } + } + public static unsafe ReplicationBatchItem From(DocumentsOperationContext context, Tombstone doc) { switch (doc.Type) @@ -25,17 +52,9 @@ public static unsafe ReplicationBatchItem From(DocumentsOperationContext context }; case Tombstone.TombstoneType.Attachment: - var item = new AttachmentTombstoneReplicationItem - { - Type = ReplicationBatchItem.ReplicationItemType.AttachmentTombstone, - Etag = doc.Etag, - TransactionMarker = doc.TransactionMarker, - ChangeVector = doc.ChangeVector, - Flags = doc.Flags, - LastModifiedTicks = doc.LastModified.Ticks, - }; + AttachmentTombstoneReplicationItem item = AttachmentTombstoneReplicationItemInternal(context, doc); + item.Flags = doc.Flags; - item.ToDispose(Slice.From(context.Allocator, doc.LowerId.Buffer, doc.LowerId.Size, ByteStringType.Immutable, out item.Key)); return item; case Tombstone.TombstoneType.Revision: @@ -53,5 +72,20 @@ public static unsafe ReplicationBatchItem From(DocumentsOperationContext context throw new ArgumentOutOfRangeException(nameof(doc.Type)); } } + + private static unsafe AttachmentTombstoneReplicationItem AttachmentTombstoneReplicationItemInternal(DocumentsOperationContext context, Tombstone doc) + { + var item = new AttachmentTombstoneReplicationItem + { + Type = ReplicationBatchItem.ReplicationItemType.AttachmentTombstone, + Etag = doc.Etag, + TransactionMarker = doc.TransactionMarker, + ChangeVector = doc.ChangeVector, + LastModifiedTicks = doc.LastModified.Ticks, + }; + + item.ToDispose(Slice.From(context.Allocator, doc.LowerId.Buffer, doc.LowerId.Size, ByteStringType.Immutable, out item.Key)); + return item; + } } } diff --git a/test/SlowTests/Server/Documents/Attachments/AzureRetiredAttachmentsSlowTests.cs b/test/SlowTests/Server/Documents/Attachments/AzureRetiredAttachmentsSlowTests.cs index d249cdfc2932..d64875eadcfc 100644 --- a/test/SlowTests/Server/Documents/Attachments/AzureRetiredAttachmentsSlowTests.cs +++ b/test/SlowTests/Server/Documents/Attachments/AzureRetiredAttachmentsSlowTests.cs @@ -7,7 +7,7 @@ namespace SlowTests.Server.Documents.Attachments { - public class AzureRetiredAttachmentsSlowTests : AzureRetiredAttachmentsHolder + public class AzureRetiredAttachmentsSlowTests : RetiredAttachmentsAzureBase { //TODO: egor test CanUploadRetiredAttachmentToAzureIfItAlreadyExists - will rewrite the retired attachment, even if it is the same - is it the behaviour we want? //TODO: egor do big attachments tests diff --git a/test/SlowTests/Server/Documents/Attachments/AzureRetiredAttachmentsHolder.cs b/test/SlowTests/Server/Documents/Attachments/RetiredAttachmentsAzureBase.cs similarity index 93% rename from test/SlowTests/Server/Documents/Attachments/AzureRetiredAttachmentsHolder.cs rename to test/SlowTests/Server/Documents/Attachments/RetiredAttachmentsAzureBase.cs index 2deb8870a0b0..833f8e5d58c0 100644 --- a/test/SlowTests/Server/Documents/Attachments/AzureRetiredAttachmentsHolder.cs +++ b/test/SlowTests/Server/Documents/Attachments/RetiredAttachmentsAzureBase.cs @@ -18,11 +18,11 @@ namespace SlowTests.Server.Documents.Attachments; -public abstract class AzureRetiredAttachmentsHolder : RetiredAttachmentsHolder +public abstract class RetiredAttachmentsAzureBase : RetiredAttachmentsHolder { - protected AzureRetiredAttachmentsHolder RetiredAttachments; + protected RetiredAttachmentsAzureBase RetiredAttachments; - protected AzureRetiredAttachmentsHolder(ITestOutputHelper output) : base(output) + protected RetiredAttachmentsAzureBase(ITestOutputHelper output) : base(output) { RetiredAttachments = this; } diff --git a/test/SlowTests/Server/Documents/Attachments/RetiredAttachmentsHolder.cs b/test/SlowTests/Server/Documents/Attachments/RetiredAttachmentsHolder.cs index 92f25fe0fbb1..e5c7dea13496 100644 --- a/test/SlowTests/Server/Documents/Attachments/RetiredAttachmentsHolder.cs +++ b/test/SlowTests/Server/Documents/Attachments/RetiredAttachmentsHolder.cs @@ -1081,9 +1081,14 @@ protected async Task CanIndexWithRetiredAttachmentInternal(int attachmentsCount, await Indexes.WaitForIndexingAsync(store); using (var session = store.OpenSession()) { - var res = session.Advanced.RawQuery("from index 'MultipleAttachmentsIndex' as o where o.AttachmentRetiredAt != null").WaitForNonStaleResults().ToList(); - + var res = session.Advanced.RawQuery("from index 'MultipleAttachmentsIndex' as o where o.AttachmentFlags != null").WaitForNonStaleResults().ToList(); Assert.Equal(docsCount, res.Count); + + var res2 = session.Advanced.RawQuery("from index 'MultipleAttachmentsIndex' as o where o.AttachmentFlags != 'Retired'").WaitForNonStaleResults().ToList(); + Assert.Equal(docsCount, res2.Count); + + var res3 = session.Advanced.RawQuery("from index 'MultipleAttachmentsIndex' as o where o.AttachmentFlags == 'None'").WaitForNonStaleResults().ToList(); + Assert.Equal(docsCount, res3.Count); } // move in time & start retire @@ -1096,9 +1101,17 @@ protected async Task CanIndexWithRetiredAttachmentInternal(int attachmentsCount, await Indexes.WaitForIndexingAsync(store); using (var session = store.OpenSession()) { - var res = session.Advanced.RawQuery("from index 'MultipleAttachmentsIndex' as o where o.AttachmentRetiredAt == null").WaitForNonStaleResults().ToList(); + var res = session.Advanced.RawQuery("from index 'MultipleAttachmentsIndex' as o where o.AttachmentRetiredAt != null").WaitForNonStaleResults().ToList(); + Assert.Equal(docsCount, res.Count); + } + using (var session = store.OpenSession()) + { + var res = session.Advanced.RawQuery("from index 'MultipleAttachmentsIndex' as o where o.AttachmentFlags == 'Retired'").WaitForNonStaleResults().ToList(); Assert.Equal(docsCount, res.Count); + + var res2 = session.Advanced.RawQuery("from index 'MultipleAttachmentsIndex' as o where o.AttachmentFlags != 'Retired'").WaitForNonStaleResults().ToList(); + Assert.Equal(0, res2.Count); } } } diff --git a/test/SlowTests/Server/Documents/Attachments/RetiredAttachmentsHolderBase.cs b/test/SlowTests/Server/Documents/Attachments/RetiredAttachmentsHolderBase.cs index b3ad7225ea87..e97b3736da6c 100644 --- a/test/SlowTests/Server/Documents/Attachments/RetiredAttachmentsHolderBase.cs +++ b/test/SlowTests/Server/Documents/Attachments/RetiredAttachmentsHolderBase.cs @@ -79,7 +79,7 @@ public static async Task GetAndCompareRetiredAttachment(IDocumentStore store, st Assert.Equal(attachmentName, retired.Details.Name); Assert.Equal(streamSize, retired.Details.Size); Assert.Equal(AttachmentFlags.Retired, retired.Details.Flags); - Assert.Null(retired.Details.RetireAt); + Assert.NotNull(retired.Details.RetireAt); using var retiredStream = new MemoryStream(); await retired.Stream.CopyToAsync(retiredStream); stream.Position = 0; diff --git a/test/SlowTests/Server/Documents/Attachments/S3RetiredAttachmentsHolder.cs b/test/SlowTests/Server/Documents/Attachments/RetiredAttachmentsS3Base.cs similarity index 93% rename from test/SlowTests/Server/Documents/Attachments/S3RetiredAttachmentsHolder.cs rename to test/SlowTests/Server/Documents/Attachments/RetiredAttachmentsS3Base.cs index 2f6c6af08724..3debfbc1ba0a 100644 --- a/test/SlowTests/Server/Documents/Attachments/S3RetiredAttachmentsHolder.cs +++ b/test/SlowTests/Server/Documents/Attachments/RetiredAttachmentsS3Base.cs @@ -19,10 +19,10 @@ namespace SlowTests.Server.Documents.Attachments; -public class S3RetiredAttachmentsHolder : RetiredAttachmentsHolder +public abstract class RetiredAttachmentsS3Base : RetiredAttachmentsHolder { - protected S3RetiredAttachmentsHolder RetiredAttachments; - public S3RetiredAttachmentsHolder(ITestOutputHelper output) : base(output) + protected RetiredAttachmentsS3Base RetiredAttachments; + protected RetiredAttachmentsS3Base(ITestOutputHelper output) : base(output) { RetiredAttachments = this; } diff --git a/test/SlowTests/Server/Documents/Attachments/S3RetiredAttachmentsSlowTests.cs b/test/SlowTests/Server/Documents/Attachments/S3RetiredAttachmentsSlowTests.cs index e63855b26fcd..c34ff024a7d2 100644 --- a/test/SlowTests/Server/Documents/Attachments/S3RetiredAttachmentsSlowTests.cs +++ b/test/SlowTests/Server/Documents/Attachments/S3RetiredAttachmentsSlowTests.cs @@ -1,17 +1,15 @@ using System; using System.Collections.Generic; -using System.Diagnostics; -using System.IO; using System.Linq; using System.Threading; using System.Threading.Tasks; -using FastTests.Client; using Orders; using Raven.Client.Documents; -using Raven.Client.Documents.Attachments; +using Raven.Client.Documents.Conventions; using Raven.Client.Documents.Operations; using Raven.Client.Documents.Operations.Attachments; using Raven.Client.ServerWide; +using Raven.Client.ServerWide.Operations; using Raven.Server.Documents; using Raven.Server.ServerWide.Context; using Tests.Infrastructure; @@ -20,7 +18,7 @@ namespace SlowTests.Server.Documents.Attachments { - public class S3RetiredAttachmentsSlowTests : S3RetiredAttachmentsHolder + public class S3RetiredAttachmentsSlowTests : RetiredAttachmentsS3Base { //TODO: egor test CanUploadRetiredAttachmentToS3IfItAlreadyExists - will rewrite the retired attachment, even if it is the same - is it the behaviour we want? //TODO: egor do big attachments tests @@ -59,9 +57,6 @@ public async Task DeletingDocumentWithRetiredAttachmentShouldKeepRetiredAttachme " }, patchIfMissing: null); var res = await store.Operations.SendAsync(operation); - - - Console.WriteLine(); } using (var s = store.OpenAsyncSession()) @@ -89,6 +84,133 @@ public async Task DeletingDocumentWithRetiredAttachmentShouldKeepRetiredAttachme } } + [RavenTheory(RavenTestCategory.Attachments)] + [InlineData(true)] + [InlineData(false)] + public async Task DeletingDocumentWithRetiredAttachmentShouldKeepRetiredAttachmentByDefaultInCluster(bool purgeOnDelete) + { + var attachmentsCount = 1; + var size = 3; + var srcDb = GetDatabaseName(); + var srcRaft = await CreateRaftCluster(3); + var leader = srcRaft.Leader; + var srcNodes = await CreateDatabaseInCluster(srcDb, 3, leader.WebUrl); + var mentorNode = srcNodes.Servers.First(s => s != leader); + var mentorTag = mentorNode.ServerStore.NodeTag; + using (DocumentStore store = (DocumentStore)new DocumentStore { Urls = srcNodes.Servers.Select(s => s.WebUrl).ToArray(), Database = srcDb, }.Initialize()) + { + await using (var holder = CreateCloudSettings()) + { + int docsCount = GetDocsAndAttachmentCount(attachmentsCount, out int attachmentsPerDoc); + var ids = new List<(string Id, string Collection)>(); + + RetiredAttachments.ModifyRetiredAttachmentsConfig = config => + { + config.PurgeOnDelete = purgeOnDelete; + }; + + //TODO: egor /////////////////////// DUPLICATE CODE WITH CanUploadRetiredAttachmentToCloudInClusterAndDeleteInternal + await PutRetireAttachmentsConfiguration(store, Settings); + await CreateDocs(store, docsCount, ids); + await PopulateDocsWithRandomAttachments(store, size, ids, attachmentsPerDoc); + Assert.Equal(true, await WaitForChangeVectorInClusterAsync(srcNodes.Servers, srcDb)); + + int count = 0; + DocumentDatabase database = null; + var retired = await WaitForValueAsync(async () => + { + var record = await store.Maintenance.Server.SendAsync(new GetDatabaseRecordOperation(store.Database)).ConfigureAwait(false); + var f = record.Topology.AllNodes.FirstOrDefault(); + var srv = srcRaft.Nodes.FirstOrDefault(x => x.ServerStore.NodeTag == f); + database = await Databases.GetDocumentDatabaseInstanceFor(srv, store); + + GetStorageAttachmentsMetadataFromAllAttachments(database); + Assert.Equal(attachmentsCount, Attachments.Count); + + database.Time.UtcDateTime = () => DateTime.UtcNow.AddMinutes(10); + + count += await database.RetireAttachmentsSender.RetireAttachments(int.MaxValue, int.MaxValue); + + return count; + }, attachmentsCount, interval: 1000); + + + Assert.Equal(attachmentsCount, retired); + + var cloudObjects = await GetBlobsFromCloudAndAssertForCount(Settings, attachmentsCount, 15_000); + await AssertAllRetiredAttachments(store, cloudObjects, size); + + //TODO: egor ////////////////////// DUPLICATE CODE WITH CanUploadRetiredAttachmentToCloudInClusterAndDeleteInternal + var stores = srcNodes.Servers.Select(s => new DocumentStore { Urls = new string[1] { $"{s.WebUrl}" }, Database = srcDb, Conventions = new DocumentConventions { DisableTopologyUpdates = true } }.Initialize()).ToList(); + try + { + var l = Attachments.Select(x => x.DocumentId).ToList().Distinct().ToList(); + for (int i = 0; i < l.Count; i++) + { + var docId = l[i]; + + PatchOperation operation = new PatchOperation(id: docId, changeVector: null, patch: new PatchRequest + { + Script = @$" + del('{docId}'); + " + }, patchIfMissing: null); + var index = i % stores.Count; + var s = stores[++index]; + var res = await store.Operations.SendAsync(operation); + } + Assert.Equal(true, await WaitForChangeVectorInClusterAsync(srcNodes.Servers, srcDb)); + using (var s = store.OpenAsyncSession()) + { + var q = await s.Query().ToListAsync(); + + Assert.Equal(0, q.Count); + + } + // Assert.Equal(true, await WaitForChangeVectorInClusterAsync(srcNodes.Servers, srcDb)); + if (purgeOnDelete) + { + + var record = await store.Maintenance.Server.SendAsync(new GetDatabaseRecordOperation(store.Database)).ConfigureAwait(false); + var f = record.Topology.AllNodes.FirstOrDefault(); + + foreach (var node in srcRaft.Nodes) + { + database = await Databases.GetDocumentDatabaseInstanceFor(node, store); + + GetToRetireAttachmentsCount(database, 1); + database.Time.UtcDateTime = () => DateTime.UtcNow.AddMinutes(10); + await database.RetireAttachmentsSender.RetireAttachments(int.MaxValue, int.MaxValue); + } + await GetBlobsFromCloudAndAssertForCount(Settings, 0, 15_000); + } + else + { + foreach (var node in srcRaft.Nodes) + { + database = await Databases.GetDocumentDatabaseInstanceFor(node, store); + //var database = await Databases.GetDocumentDatabaseInstanceFor(Server, store); + GetToRetireAttachmentsCount(database, 0); + + // nothing should happen + database.Time.UtcDateTime = () => DateTime.UtcNow.AddMinutes(10); + await database.RetireAttachmentsSender.RetireAttachments(int.MaxValue, int.MaxValue); + } + + await GetBlobsFromCloudAndAssertForCount(Settings, 1, 15_000); + } + } + finally + { + foreach (var s in stores) + { + s.Dispose(); + } + } + } + } + } + [RavenFact(RavenTestCategory.Attachments)] public async Task DeletingAttachmentShouldRemoveFromRetireTree() { @@ -138,7 +260,16 @@ private static void GetToRetireAttachmentsCount(DocumentDatabase database, int e { var expired = database.DocumentsStorage.AttachmentsStorage.RetiredAttachmentsStorage.GetDocuments(options, ref totalCount, out _, CancellationToken.None); + Assert.Equal(expected, totalCount); + + //if (expected == 0) + //{ + // if (totalCount == 1) + // { + // //TODO: egor I ahve delete retired attachment.... + // } + //} if (expected == 0) { Assert.Null(expired);