Skip to content

Commit

Permalink
RavenDB-16815 - add new flag to attachment tombstone, keep retireAt d…
Browse files Browse the repository at this point in the history 
…ate time after retirement as well
  • Loading branch information
@garayx
garayx committed Sep 3, 2024
1 parent d8ccb87 commit 5eac4e3
Show file tree
Hide file tree
Showing 13 changed files with 317 additions and 46 deletions.
5 changes: 5 additions & 0 deletions src/Raven.Client/Documents/Attachments/RetiredAttachmentsConfiguration.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,11 @@ public sealed class RetiredAttachmentsConfiguration : IDynamicJson
public Dictionary<string, TimeSpan> RetirePeriods { get; set; }
public long? RetireFrequencyInSec { get; set; }
public long? MaxItemsToProcess { get; set; }

/// <summary>
/// Purge the retired attachments when the document is deleted.
/// Default: false
/// </summary>
public bool PurgeOnDelete { get; set; }

public override int GetHashCode()
Expand Down
57 changes: 48 additions & 9 deletions src/Raven.Server/Documents/AttachmentsStorage.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -207,11 +207,14 @@ public void RetireAttachment(DocumentsOperationContext context, AttachmentDetail
var attachmentEtag = _documentsStorage.GenerateNextEtag();
var changeVector = _documentsStorage.GetNewChangeVector(context, attachmentEtag);
var size = TableValueToLong((int)AttachmentsTable.Size, ref attachmentTvr);
var retireAt = TableValueToLong((int)AttachmentsTable.RetireAt, ref attachmentTvr);

using (TableValueToSlice(context, (int)AttachmentsTable.Name, ref attachmentTvr, out var nameSlice))
using (TableValueToSlice(context, (int)AttachmentsTable.ContentType, ref attachmentTvr, out var contentTypeSlice))
using (TableValueToSlice(context, (int)AttachmentsTable.Hash, ref attachmentTvr, out var hashSlice))
using (Slice.From(context.Allocator, changeVector, out var changeVectorSlice))
using (TableValueToSlice(context, (int)AttachmentsTable.Collection, ref attachmentTvr, out var collectionSlice))

using (table.Allocate(out TableValueBuilder tvb))
{
// add Retired flag
Expand All @@ -224,7 +227,7 @@ public void RetireAttachment(DocumentsOperationContext context, AttachmentDetail
tvb.Add(changeVectorSlice);
tvb.Add(size);
tvb.Add(Bits.SwapBytes((int)AttachmentFlags.Retired));
tvb.Add(-1L);
tvb.Add(retireAt);
tvb.Add(collectionSlice);
table.Update(attachmentTvr.Id, tvb);

Expand Down Expand Up @@ -311,7 +314,8 @@ void SetTableValue(TableValueBuilder tvb, Slice cv)
size = TableValueToLong((int)AttachmentsTable.Size, ref oldValue);

retireAt = TableValueToLong((int)AttachmentsTable.RetireAt, ref oldValue);
if (retireAt == -1L)
var existingFlags = TableValueToAttachmentFlags((int)AttachmentsTable.Flags, ref oldValue);
if (existingFlags.HasFlag(AttachmentFlags.Retired) == false && retireAt == -1L)
{
var dbRecord = _documentDatabase.ReadDatabaseRecord();
Debug.Assert(collectionName != null, "collectionName != null");
Expand Down Expand Up @@ -372,10 +376,37 @@ void SetTableValue(TableValueBuilder tvb, Slice cv)
{
var existingEtag = TableValueToEtag((int)AttachmentsTable.Etag, ref partialTvr);
var lastModifiedTicks = _documentDatabase.Time.GetUtcNow().Ticks;
//TODO: egor if we update retired attachment (can this even happen?) we might delete the old one from cloud storage...
//TODO: egor if we update retired attachment (can this even happen?) we might delete the old one from cloud storage... I think need to check if it is retired && if it has purgeOn delete ?
var existingAttachmentFlags = TableValueToAttachmentFlags((int)AttachmentsTable.Flags, ref partialTvr);
var existingRetireAtTicks = TableValueToLong((int)AttachmentsTable.RetireAt, ref partialTvr);
DeleteInternal(context, existingKey, existingEtag, existingHash, changeVector, lastModifiedTicks, flags: DocumentFlags.None, existingAttachmentFlags, existingRetireAtTicks, collectionName.Name);

if (existingAttachmentFlags.Contain(AttachmentFlags.Retired))
{
var dbRecord2 = _documentDatabase.ReadDatabaseRecord();
if (dbRecord2.RetiredAttachments is { Disabled: false })
{
if (dbRecord2.RetiredAttachments.PurgeOnDelete == false)
{
// we cannot delete from cloud since PurgeOnDelete is false
DeleteInternal(context, existingKey, existingEtag, existingHash, changeVector, lastModifiedTicks, flags: DocumentFlags.None, existingAttachmentFlags, existingRetireAtTicks, collectionName.Name, storageOnly: true);

}
else
{
DeleteInternal(context, existingKey, existingEtag, existingHash, changeVector, lastModifiedTicks, flags: DocumentFlags.None, existingAttachmentFlags, existingRetireAtTicks, collectionName.Name, storageOnly: false);
}
}
else
{
DeleteInternal(context, existingKey, existingEtag, existingHash, changeVector, lastModifiedTicks, flags: DocumentFlags.None, existingAttachmentFlags, existingRetireAtTicks, collectionName.Name, storageOnly: false);
}
}
else
{
// we cannot delete from retired since there is no configuration
DeleteInternal(context, existingKey, existingEtag, existingHash, changeVector, lastModifiedTicks, flags: DocumentFlags.None, existingAttachmentFlags, existingRetireAtTicks, collectionName.Name, storageOnly: false);
}

}
}
}
Expand Down Expand Up @@ -403,7 +434,7 @@ void SetTableValue(TableValueBuilder tvb, Slice cv)
{
if (fromEtl && flags.Contain(AttachmentFlags.Retired))
{
retireAt = -1L;
retireAt = retireAtDt.HasValue == false ? -1L : retireAtDt.Value.Ticks;
}
else
{
Expand Down Expand Up @@ -1535,7 +1566,7 @@ public void DeleteAttachmentDirect(DocumentsOperationContext context, Slice key,
attachmentEtag = _documentsStorage.GenerateNextEtagForReplicatedTombstoneMissingDocument(context);
}

CreateTombstone(context, key, attachmentEtag, changeVector, lastModifiedTicks, flags: DocumentFlags.None);
CreateTombstone(context, key, attachmentEtag, changeVector, lastModifiedTicks, (int)DocumentFlags.None);
return;
}

Expand Down Expand Up @@ -1569,17 +1600,25 @@ public void DeleteAttachmentDirect(DocumentsOperationContext context, Slice key,
private void DeleteInternal(DocumentsOperationContext context, Slice key, long etag, Slice hash,
string changeVector, long lastModifiedTicks, DocumentFlags flags, AttachmentFlags attachmentFlags, long retireAtTicks, string collection, bool storageOnly = false)
{
CreateTombstone(context, key, etag, changeVector, lastModifiedTicks, flags);
if (attachmentFlags.HasFlag(AttachmentFlags.Retired))
{
if (storageOnly == false)
{
// populate retired tree
TryDeleteRetiredAttachment(context, key, collection);
}
else
{
RetiredAttachmentsStorage.RemoveRetirePutValue(context, key, retireAtTicks);

// we create attachment tombstone with special flag to mark that we don't want to delete the attachment from cloud
CreateTombstone(context, key, etag, changeVector, lastModifiedTicks, (int)AttachmentTombstoneFlags.FromStorageOnly);
}
}
else
{
CreateTombstone(context, key, etag, changeVector, lastModifiedTicks, (int)flags);

if (retireAtTicks != -1)
{
RetiredAttachmentsStorage.RemoveRetirePutValue(context, key, retireAtTicks);
Expand All @@ -1598,7 +1637,7 @@ private void DeleteTombstoneIfNeeded(DocumentsOperationContext context, Slice ke
}

private void CreateTombstone(DocumentsOperationContext context, Slice keySlice, long attachmentEtag,
string changeVector, long lastModifiedTicks, DocumentFlags flags)
string changeVector, long lastModifiedTicks, int flags)
{
var newEtag = _documentsStorage.GenerateNextEtag();

Expand All @@ -1616,7 +1655,7 @@ private void CreateTombstone(DocumentsOperationContext context, Slice keySlice,
tvb.Add(context.GetTransactionMarker());
tvb.Add((byte)Tombstone.TombstoneType.Attachment);
tvb.Add(null, 0);
tvb.Add((int)flags);
tvb.Add(flags);
tvb.Add(cv.Content.Ptr, cv.Size);
tvb.Add(lastModifiedTicks);
table.Insert(tvb);
Expand Down
19 changes: 17 additions & 2 deletions src/Raven.Server/Documents/DocumentsStorage.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
using System.Linq;
using System.Runtime.CompilerServices;
using System.Threading;
using Mono.Unix.Native;
using Raven.Client;
using Raven.Client.Documents.Attachments;
using Raven.Client.Documents.Changes;
Expand Down Expand Up @@ -1257,7 +1258,7 @@ public IEnumerable<ReplicationBatchItem> GetTombstonesFrom(DocumentsOperationCon
// ReSharper disable once LoopCanBeConvertedToQuery
foreach (var result in table.SeekForwardFrom(TombstonesSchema.FixedSizeIndexes[AllTombstonesEtagsSlice], etag, 0))
{
var tombstoneItem = TombstoneReplicationItem.From(context, TableValueToTombstone(context, ref result.Reader));
var tombstoneItem = TombstoneReplicationItem.From(context, ref result.Reader);

if (revisionTombstonesWithId == false && tombstoneItem is RevisionTombstoneReplicationItem revisionTombstone)
revisionTombstone.StripDocumentIdFromKeyIfNeeded(context);
Expand Down Expand Up @@ -1579,17 +1580,25 @@ public static Tombstone TableValueToTombstone(JsonOperationContext context, ref
TransactionMarker = *(short*)tvr.Read((int)TombstoneTable.TransactionMarker, out int _),
ChangeVector = TableValueToChangeVector(context, (int)TombstoneTable.ChangeVector, ref tvr),
LastModified = TableValueToDateTime((int)TombstoneTable.LastModified, ref tvr),
Flags = TableValueToFlags((int)TombstoneTable.Flags, ref tvr)
};

switch (result.Type)
{
case Tombstone.TombstoneType.Document:
result.Collection = TableValueToId(context, (int)TombstoneTable.Collection, ref tvr);
result.LowerId = UnwrapLowerIdIfNeeded(context, result.LowerId);
result.Flags = TableValueToFlags((int)TombstoneTable.Flags, ref tvr);

break;
case Tombstone.TombstoneType.Revision:
result.Collection = TableValueToId(context, (int)TombstoneTable.Collection, ref tvr);
result.Flags = TableValueToFlags((int)TombstoneTable.Flags, ref tvr);
break;
case Tombstone.TombstoneType.Attachment:
result.Flags = TableValueToFlags((int)TombstoneTable.Flags, ref tvr);
break;
case Tombstone.TombstoneType.Counter:
result.Flags = TableValueToFlags((int)TombstoneTable.Flags, ref tvr);
break;
}

Expand Down Expand Up @@ -2789,6 +2798,12 @@ public static DocumentFlags TableValueToFlags(int index, ref TableValueReader tv
return *(DocumentFlags*)tvr.Read(index, out _);
}

[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static int TableValueToInt(int index, ref TableValueReader tvr)
{
return *(int*)tvr.Read(index, out _);
}

[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static AttachmentFlags TableValueToAttachmentFlags(int index, ref TableValueReader tvr)
{
Expand Down
22 changes: 18 additions & 4 deletions src/Raven.Server/Documents/Replication/Incoming/IncomingReplicationHandler.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -437,10 +437,24 @@ protected override long ExecuteCmd(DocumentsOperationContext context)

try
{

database.DocumentsStorage.AttachmentsStorage.DeleteAttachmentDirect(context, attachmentTombstone.Key, false, "$fromReplication", null,
newChangeVector,
attachmentTombstone.LastModifiedTicks);
if (attachmentTombstone.TombstoneFlags.HasFlag(AttachmentTombstoneFlags.FromStorageOnly))
{



database.DocumentsStorage.AttachmentsStorage.DeleteAttachmentDirect(context, attachmentTombstone.Key, false, "$fromReplication", null,
newChangeVector,
attachmentTombstone.LastModifiedTicks, storageOnly: true);
}
else
{

database.DocumentsStorage.AttachmentsStorage.DeleteAttachmentDirect(context, attachmentTombstone.Key, false, "$fromReplication", null,
newChangeVector,
attachmentTombstone.LastModifiedTicks, storageOnly: false);

}

}
catch (Exception e)
{
Expand Down
12 changes: 12 additions & 0 deletions src/Raven.Server/Documents/Replication/ReplicationItems/AttachmentTombstoneFlags.cs
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
using System;

namespace Raven.Server.Documents.Replication.ReplicationItems;

[Flags]
public enum AttachmentTombstoneFlags
{
FromStorageOnly = 0x7FFFFFFF,


None = 0
}
12 changes: 10 additions & 2 deletions ...Raven.Server/Documents/Replication/ReplicationItems/AttachmentTombstoneReplicationItem.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,16 +13,20 @@ public sealed class AttachmentTombstoneReplicationItem : ReplicationBatchItem
{
public Slice Key;
public DocumentFlags Flags;
public AttachmentTombstoneFlags TombstoneFlags;

public override long Size => base.Size + // common
sizeof(long) + // Last modified ticks
sizeof(int) + // size of key
Key.Size;
Key.Size +
sizeof(AttachmentTombstoneFlags);

public override DynamicJsonValue ToDebugJson()
{
var djv = base.ToDebugJson();
djv[nameof(Key)] = CompoundKeyHelper.ExtractDocumentId(Key);
djv[nameof(Flags)] = Flags;
djv[nameof(TombstoneFlags)] = TombstoneFlags;
return djv;
}

Expand All @@ -46,6 +50,9 @@ public override unsafe void Write(Slice changeVector, Stream stream, byte[] temp
Memory.Copy(pTemp + tempBufferPos, Key.Content.Ptr, Key.Size);
tempBufferPos += Key.Size;

*(AttachmentTombstoneFlags*)(pTemp + tempBufferPos) = TombstoneFlags;
tempBufferPos += sizeof(AttachmentTombstoneFlags);

stream.Write(tempBuffer, 0, tempBufferPos);

stats.RecordAttachmentTombstoneOutput(Size);
Expand All @@ -61,6 +68,7 @@ public override unsafe void Read(JsonOperationContext context, ByteStringContext
var size = *(int*)Reader.ReadExactly(sizeof(int));
ToDispose(Slice.From(allocator, Reader.ReadExactly(size), size, ByteStringType.Immutable, out Key));

TombstoneFlags = *(AttachmentTombstoneFlags*)Reader.ReadExactly(sizeof(AttachmentTombstoneFlags)) | AttachmentTombstoneFlags.None;
stats.RecordAttachmentTombstoneRead(Size);
}
}
Expand All @@ -69,7 +77,7 @@ protected override ReplicationBatchItem CloneInternal(JsonOperationContext conte
{
var item = new AttachmentTombstoneReplicationItem();
item.Key = Key.Clone(allocator);

item.TombstoneFlags = TombstoneFlags;
item.ToDispose(new DisposableAction(() =>
{
item.Key.Release(allocator);
Expand Down
Loading

0 comments on commit 5eac4e3

Please sign in to comment.