RavenDB-16815 - persist document collection in attachment metadata

src/Raven.Client/Constants.cs

@@ -88,6 +88,7 @@ private Headers()
             public const string AttachmentSize = "Attachment-Size";
             public const string AttachmentRetireAt = "Attachment-Retire-At";
             public const string AttachmentFlags = "Attachment-Flags";
+   //         public const string AttachmentCollection = "Attachment-Collection";
 
             internal const string DatabaseMissing = "Database-Missing";
 

src/Raven.Client/Documents/Operations/Attachments/AttachmentDetails.cs

@@ -35,6 +35,8 @@ public class AttachmentName
         public long Size;
         public AttachmentFlags Flags;
         public DateTime? RetireAt;
+        public string Collection;
+
         internal virtual DynamicJsonValue ToJson()
         {
             var json = new DynamicJsonValue
@@ -46,7 +48,7 @@ internal virtual DynamicJsonValue ToJson()
             };
             json[nameof(Flags)] = Flags.ToString();
             json[nameof(RetireAt)] = RetireAt;
-
+            json[nameof(Collection)] = Collection;
             return json;
         }
     }

src/Raven.Client/Documents/Operations/Attachments/GetAttachmentOperation.cs

@@ -128,7 +128,7 @@ internal static async Task<AttachmentResult> AttachmentResult(HttpResponseMessag
                 int flags = 0;
                 if (response.Headers.TryGetValues(Constants.Headers.AttachmentFlags, out IEnumerable<string> flagsVal))
                     int.TryParse(flagsVal.First(), out flags);
-
+         //       var collection = response.Headers.TryGetValues(Constants.Headers.AttachmentCollection, out IEnumerable<string> collectionVal) ? collectionVal.First() : null;
                 var attachmentDetails = new AttachmentDetails
                 {
                     ContentType = contentType,
@@ -138,7 +138,8 @@ internal static async Task<AttachmentResult> AttachmentResult(HttpResponseMessag
                     ChangeVector = changeVector,
                     DocumentId = documentId,
                     RetireAt = attachmentRetireAt,
-                    Flags = (AttachmentFlags)flags
+                    Flags = (AttachmentFlags)flags,
+            //        Collection = collection
                 };
 
                 var responseStream = await response.Content.ReadAsStreamWithZstdSupportAsync().ConfigureAwait(false);

src/Raven.Server/Documents/Attachment.cs

@@ -20,5 +20,6 @@ public sealed class Attachment
         public long Size;
         public AttachmentFlags Flags;
         public DateTime? RetiredAt;
+        public LazyStringValue Collection;
     }
 }

src/Raven.Server/Documents/DocumentsStorage.cs

@@ -1779,7 +1779,7 @@ public static Tombstone TableValueToTombstone(JsonOperationContext context, ref
                     }
 
                     if (flags.Contain(DocumentFlags.HasAttachments))
-                        AttachmentsStorage.DeleteAttachmentsOfDocument(context, lowerId, changeVector, modifiedTicks, collectionName.Name, newFlags);
+                        AttachmentsStorage.DeleteAttachmentsOfDocument(context, lowerId, changeVector, modifiedTicks, newFlags);
 
                     if (flags.Contain(DocumentFlags.HasCounters))
                         CountersStorage.DeleteCountersForDocument(context, id, collectionName);

src/Raven.Server/Documents/Handlers/Processors/Attachments/AttachmentHandlerProcessorForGetAttachment.cs

@@ -67,7 +67,7 @@ protected override async ValueTask GetAttachmentAsync(DocumentsOperationContext
                 HttpContext.Response.Headers[Constants.Headers.Etag] = $"\"{attachment.ChangeVector}\"";
                 HttpContext.Response.Headers[Constants.Headers.AttachmentRetireAt] = attachment.RetiredAt?.GetDefaultRavenFormat();
                 HttpContext.Response.Headers[Constants.Headers.AttachmentFlags] = ((int)attachment.Flags).ToString();
-
+         //       HttpContext.Response.Headers[Constants.Headers.AttachmentCollection] = attachment.Collection.ToString();
                 DisposeReadTransactionIfNeeded(tx);
 
                 await WriteResponseStream(context, attachment, collection, token);

src/Raven.Server/Documents/Replication/Incoming/IncomingReplicationHandler.cs

@@ -3,8 +3,10 @@
 using System.Diagnostics;
 using System.IO;
 using System.Linq;
+using System.Threading;
 using System.Threading.Tasks;
 using Microsoft.AspNetCore.Http;
+using NetTopologySuite.Utilities;
 using Raven.Client.Documents.Attachments;
 using Raven.Client.Documents.Operations.Attachments;
 using Raven.Client.Documents.Replication.Messages;
@@ -392,6 +394,8 @@ protected override long ExecuteCmd(DocumentsOperationContext context)
 
                                 toDispose.Add(DocumentIdWorker.GetLowerIdSliceAndStorageKey(context, attachment.Name, out _, out Slice attachmentName));
                                 toDispose.Add(DocumentIdWorker.GetLowerIdSliceAndStorageKey(context, attachment.ContentType, out _, out Slice contentType));
+                                toDispose.Add(DocumentIdWorker.GetLowerIdSliceAndStorageKey(context, attachment.Collection, out _, out Slice collectionSlice));
+
 
                                 var local = context.GetChangeVector(result.ChangeVector);
                                 var newChangeVector = ChangeVectorUtils.GetConflictStatus(incomingChangeVector, local) switch
@@ -406,7 +410,7 @@ protected override long ExecuteCmd(DocumentsOperationContext context)
                                 if (newChangeVector != null)
                                 {
                                     database.DocumentsStorage.AttachmentsStorage.PutDirect(context, attachment.Key, attachmentName,
-                                        contentType, attachment.Base64Hash, attachment.RetiredAtUtc, attachment.Flags, attachment.AttachmentSize, isRevision, newChangeVector);
+                                        contentType, attachment.Base64Hash, attachment.RetiredAtUtc, collectionSlice, attachment.Flags, attachment.AttachmentSize,isRevision, newChangeVector);
                                 }
 
                                 break;
@@ -433,15 +437,10 @@ protected override long ExecuteCmd(DocumentsOperationContext context)
 
                                 try
                                 {
-                                    string collection;
-                                    using (var doc1 = context.DocumentDatabase.DocumentsStorage.Get(context, documentId, DocumentFields.Data, throwOnConflict: false))
-                                    {
-                                        doc1.TryGetCollection(out collection);
-                                    }
-
+
                                     database.DocumentsStorage.AttachmentsStorage.DeleteAttachmentDirect(context, attachmentTombstone.Key, false, "$fromReplication", null,
                                         newChangeVector,
-                                        attachmentTombstone.LastModifiedTicks, collection);
+                                        attachmentTombstone.LastModifiedTicks);
                                 }
                                 catch (Exception e)
                                 {

src/Raven.Server/Documents/Replication/ReplicationItems/AttachmentReplicationItem.cs

@@ -2,6 +2,7 @@
 using System.Globalization;
 using System.IO;
 using Raven.Client.Documents.Attachments;
+using Raven.Client.Extensions;
 using Raven.Client.Util;
 using Raven.Server.Documents.Replication.Stats;
 using Raven.Server.ServerWide.Context;
@@ -23,6 +24,7 @@ public sealed class AttachmentReplicationItem : ReplicationBatchItem
         public long AttachmentSize;
         public AttachmentFlags Flags;
         public DateTime? RetiredAtUtc;
+        public LazyStringValue Collection;
 
         public override long Size => base.Size + // common
 
@@ -36,10 +38,14 @@ public sealed class AttachmentReplicationItem : ReplicationBatchItem
                                      ContentType.Size +
 
                                      sizeof(byte) + // size of Base64Hash
-                                     Base64Hash.Size 
+                                     Base64Hash.Size
+
                                      + sizeof(long)
                                      + sizeof(int)
-                                     + (RetiredAtUtc == null ? 0 : sizeof(long));
+                                     + (RetiredAtUtc == null ? 0 : sizeof(long))
+                                     + sizeof(int)
+                                     + Collection.Size
+                              ;
 
 
         public long StreamSize => sizeof(byte) + // type
@@ -57,6 +63,9 @@ public override DynamicJsonValue ToDebugJson()
             djv[nameof(ContentType)] = ContentType.ToString(CultureInfo.InvariantCulture);
             djv[nameof(Base64Hash)] = Base64Hash.ToString();
             djv[nameof(Key)] = CompoundKeyHelper.ExtractDocumentId(Key);
+            djv[nameof(Flags)] = Flags.ToString();
+            djv[nameof(RetiredAtUtc)] = RetiredAtUtc;
+            djv[nameof(Collection)] = Collection.ToString();
             return djv;
         }
 
@@ -74,7 +83,8 @@ public static unsafe AttachmentReplicationItem From(DocumentsOperationContext co
                 TransactionMarker = attachment.TransactionMarker,
                 AttachmentSize = attachment.Size,
                 Flags = attachment.Flags,
-                RetiredAtUtc = attachment.RetiredAt
+                RetiredAtUtc = attachment.RetiredAt,
+                Collection = attachment.Collection
             };
 
             // although the key is LSV but is treated as slice and doesn't respect escaping
@@ -128,6 +138,14 @@ public override unsafe void Write(Slice changeVector, Stream stream, byte[] temp
                 *(AttachmentFlags*)(pTemp + tempBufferPos) = Flags;
                 tempBufferPos += sizeof(AttachmentFlags);
 
+
+                *(int*)(pTemp + tempBufferPos) = Collection.Size;
+                tempBufferPos += sizeof(int);
+                Memory.Copy(pTemp + tempBufferPos, Collection.Buffer, Collection.Size);
+                tempBufferPos += Collection.Size;
+
+
+
                 stream.Write(tempBuffer, 0, tempBufferPos);
                 stats.RecordAttachmentOutput(Size);
             }
@@ -152,6 +170,8 @@ public override unsafe void Read(JsonOperationContext context, ByteStringContext
                     RetiredAtUtc = new DateTime(ticks, DateTimeKind.Utc);
 
                 Flags = *(AttachmentFlags*)Reader.ReadExactly(sizeof(AttachmentFlags)) | AttachmentFlags.None;
+                SetLazyStringValueFromString(context, out Collection);
+
 
                 stats.RecordAttachmentRead(Size);
             }
@@ -182,7 +202,7 @@ protected override ReplicationBatchItem CloneInternal(JsonOperationContext conte
             item.AttachmentSize = AttachmentSize;
             item.RetiredAtUtc = RetiredAtUtc;
             item.Flags = Flags;
-
+            item.Collection = Collection.Clone(context);
             item.ToDispose(new DisposableAction(() =>
             {
                 item.Base64Hash.Release(allocator);
@@ -252,6 +272,7 @@ protected override void InnerDispose()
             Name?.Dispose();
             ContentType?.Dispose();
             Stream?.Dispose();
+            Collection?.Dispose();
         }
     }
 }

src/Raven.Server/Documents/Replication/ResolveConflictOnReplicationConfigurationChange.cs

@@ -509,7 +509,8 @@ private void ResolveAttachmentsConflicts(DocumentsOperationContext context, Docu
                             x.ContentType == attachment.ContentType &&
                             x.Flags == attachment.Flags &&
                             x.Size == attachment.Size &&
-                            x.RetireAt == attachment.RetireAt))
+                            x.RetireAt == attachment.RetireAt &&
+                            x.Collection == attachment.Collection))
                     {
                         found = true;
                        // we have to generate a _new_ change vector for the attachment, since it is resolved, to ensure

src/Raven.Server/Documents/Revisions/RevisionsStorage.cs

@@ -923,7 +923,7 @@ internal void DeleteRevisionFromTable(DocumentsOperationContext context, Table t
 
                 if (revision.Flags.Contain(DocumentFlags.HasAttachments))
                 {
-                    _documentsStorage.AttachmentsStorage.DeleteRevisionAttachments(context, revision, changeVector, lastModifiedTicks, collectionName.Name, flags);
+                    _documentsStorage.AttachmentsStorage.DeleteRevisionAttachments(context, revision, changeVector, lastModifiedTicks, flags);
                 }
 
                 Table writeTable = null;

src/Raven.Server/Documents/Schemas/Attachments.cs

@@ -33,7 +33,8 @@ e.g. fitz(record-separator)profile.png and fitz0(record-separator)profile.png, w
             ChangeVector = 6,
             Size = 7,
             Flags = 8,
-            RetireAt = 9
+            RetireAt = 9,
+            Collection = 10
         }
 
         static Attachments()

src/Raven.Server/Smuggler/Documents/DatabaseDestination.cs

@@ -645,12 +645,7 @@ protected override long ExecuteCmd(DocumentsOperationContext context)
                                         throw new InvalidOperationException("Cannot find a document ID inside the attachment key");
                                     var attachmentId = key.Content.Substring(idEnd);
                                     idsOfDocumentsToUpdateAfterAttachmentDeletion.Add(attachmentId);
-                                    string collection;
-                                    using (var doc1 = context.DocumentDatabase.DocumentsStorage.Get(context, attachmentId, DocumentFields.Default, throwOnConflict: false))
-                                    {
-                                        doc1.TryGetCollection(out collection);
-                                    }
-                                    _database.DocumentsStorage.AttachmentsStorage.DeleteAttachmentDirect(context, key, false, "$fromReplication", null, tombstone.ChangeVector, tombstone.LastModified.Ticks, collection);
+                                    _database.DocumentsStorage.AttachmentsStorage.DeleteAttachmentDirect(context, key, false, "$fromReplication", null, tombstone.ChangeVector, tombstone.LastModified.Ticks);
                                     break;
 
                                 case Tombstone.TombstoneType.Revision:
@@ -845,7 +840,8 @@ private unsafe void PutAttachments(DocumentsOperationContext context, Document d
                         attachment.TryGet(nameof(AttachmentName.Hash), out LazyStringValue hash) == false ||
                         attachment.TryGet(nameof(AttachmentName.Flags), out AttachmentFlags flags) == false ||
                         attachment.TryGet(nameof(AttachmentName.Size), out long size) == false ||
-                        attachment.TryGet(nameof(AttachmentName.RetireAt), out DateTime? retireAt) == false)
+                        attachment.TryGet(nameof(AttachmentName.RetireAt), out DateTime? retireAt) == false ||
+                        attachment.TryGet(nameof(AttachmentName.Collection), out LazyStringValue collection) == false) 
 
                         throw new ArgumentException($"The attachment info is missing a mandatory value: {attachment}");
 
@@ -862,11 +858,12 @@ private unsafe void PutAttachments(DocumentsOperationContext context, Document d
                     using (DocumentIdWorker.GetLowerIdSliceAndStorageKey(_context, name, out Slice lowerName, out Slice nameSlice))
                     using (DocumentIdWorker.GetLowerIdSliceAndStorageKey(_context, contentType, out Slice lowerContentType, out Slice contentTypeSlice))
                     using (Slice.External(_context.Allocator, hash, out Slice base64Hash))
+                    using (DocumentIdWorker.GetLowerIdSliceAndStorageKey(_context, collection, out Slice _, out Slice collectionSlice))
                     using (Slice.From(_context.Allocator, document.ChangeVector, out Slice cv))
                     using (attachmentsStorage.GetAttachmentKey(_context, lowerDocumentId.Content.Ptr, lowerDocumentId.Size, lowerName.Content.Ptr, lowerName.Size,
                                base64Hash, lowerContentType.Content.Ptr, lowerContentType.Size, type, cv, out Slice keySlice))
                     {
-                        attachmentsStorage.PutDirect(context, keySlice, nameSlice, contentTypeSlice, base64Hash, retireAt, flags, size, isRevision: true);
+                        attachmentsStorage.PutDirect(context, keySlice, nameSlice, contentTypeSlice, base64Hash, retireAt, collectionSlice, flags, size, isRevision: true);
                     }
                 }
             }

src/Raven.Server/Smuggler/Documents/StreamSource.cs

@@ -1,4 +1,5 @@
 using System;
+using System.Collections;
 using System.Collections.Generic;
 using System.Diagnostics;
 using System.IO;
@@ -1551,7 +1552,8 @@ public static BlittableJsonReaderObject WriteDummyDocumentForAttachment(JsonOper
                 ["ContentType"] = string.Empty,
                 ["Size"] = details.Size,
                 [nameof(AttachmentName.Flags)] = AttachmentFlags.None,
-                [nameof(AttachmentName.RetireAt)] = null
+                [nameof(AttachmentName.RetireAt)] = null,
+                [nameof(AttachmentName.Collection)] = "@files"
             };
             var attachments = new DynamicJsonArray();
             attachments.Add(attachment);
@@ -1617,10 +1619,10 @@ private async IAsyncEnumerable<DocumentItem> ReadDocumentsAsync(List<string> col
 
                     var data = builder.CreateReader();
                     builder.Reset();
-
+                    string collectionName = null;
                     if (data.TryGet(Constants.Documents.Metadata.Key, out BlittableJsonReaderObject metadata))
                     {
-                        if (metadata.TryGet(Constants.Documents.Metadata.Collection, out string collectionName))
+                        if (metadata.TryGet(Constants.Documents.Metadata.Collection, out  collectionName))
                         {
                             if (collectionsHashSet.Count > 0 && collectionsHashSet.Contains(collectionName) == false)
                             {
@@ -1741,6 +1743,24 @@ private async IAsyncEnumerable<DocumentItem> ReadDocumentsAsync(List<string> col
                                 }
                             }
 
+                            if (attachmentInMetadata.TryGet(nameof(AttachmentName.Collection), out string _) == false)
+                            {
+                                if (attachmentInMetadata.Modifications == null)
+                                {
+
+                                    Debug.Assert(collectionName != null);
+                                    attachmentInMetadata.Modifications = new DynamicJsonValue(attachmentInMetadata)
+                                    {
+                                        [nameof(AttachmentName.Collection)] = collectionName
+                                    };
+                                }
+                                else
+                                {
+                                    attachmentInMetadata.Modifications[nameof(AttachmentName.Collection)] = collectionName;
+                                }
+                            }
+
+
                             if (attachmentInMetadata.Modifications != null)
                             {
                                 didWork = true;

test/FastTests/Client/RavenCommandTest.cs

@@ -71,7 +71,7 @@ public void WhenCommandCanBeCheckedForFastestNode_ItCanRunInParallel()
                 "PutDatabaseClientConfigurationCommand", "PutDatabaseSettingsCommand", "PutDatabaseStudioConfigurationCommand", "GetTcpInfoForReplicationCommand",
                 "AddQueueSinkCommand", "UpdateQueueSinkCommand", "ConfigureDataArchivalCommand",
                 "AdoptOrphanedRevisionsCommand", "ConfigureAttachmentsRetireCommand",
-                "DeleteRetiredAttachmentCommand", "GetRetiredAttachmentCommand", "GetRetiredAttachmentsCommand", "DeleteRetiredAttachmentsCommand"
+                "DeleteRetiredAttachmentCommand", "GetRetiredAttachmentCommand", "GetRetiredAttachmentsCommand", "DeleteRetiredAttachmentsCommand", "GetRetireAttachmentsConfigurationCommand"
             }.OrderBy(t => t);
 
             var commandBaseType = typeof(RavenCommand<>);