RavenDB-23100 - remove the need to lower the key for indexes with the…

… new version

src/Raven.Server/Documents/Indexes/MapReduce/Static/MapReduceIndex.cs

@@ -1,6 +1,7 @@
 using System;
 using System.Collections;
 using System.Collections.Generic;
+using System.Diagnostics;
 using System.Linq;
 using Raven.Client;
 using Raven.Client.Documents.Changes;
@@ -332,11 +333,25 @@ protected override IIndexingWork[] CreateIndexWorkExecutors()
 
             workers.Add(new CleanupDocumentsForMapReduce(this, DocumentDatabase.DocumentsStorage, _indexStorage, Configuration, MapReduceWorkContext));
 
+            Debug.Assert(SourceType == IndexSourceType.Documents);
+
             if (_compiled.CollectionsWithCompareExchangeReferences.Count > 0)
-                workers.Add(_handleCompareExchangeReferences = new HandleCompareExchangeReferences(this, _compiled.CollectionsWithCompareExchangeReferences, DocumentDatabase.DocumentsStorage, _indexStorage, Configuration));
+            {
+                _handleCompareExchangeReferences = Definition.Version >= IndexDefinitionBaseServerSide.IndexVersion.LowerCasedReferences
+                    ? new HandleCompareExchangeReferences(this, _compiled.CollectionsWithCompareExchangeReferences, DocumentDatabase.DocumentsStorage, _indexStorage, Configuration)
+                    : new HandleNotNormalizedCompareExchangeReferences(this, _compiled.CollectionsWithCompareExchangeReferences, DocumentDatabase.DocumentsStorage, _indexStorage, Configuration);
+
+                workers.Add(_handleCompareExchangeReferences);
+            }
 
             if (_referencedCollections.Count > 0)
-                workers.Add(_handleReferences = new HandleDocumentReferences(this, _compiled.ReferencedCollections, DocumentDatabase.DocumentsStorage, _indexStorage, Configuration));
+            {
+                _handleReferences = Definition.Version >= IndexDefinitionBaseServerSide.IndexVersion.LowerCasedReferences
+                    ? new HandleDocumentReferences(this, _compiled.ReferencedCollections, DocumentDatabase.DocumentsStorage, _indexStorage, Configuration)
+                    : new HandleNotNormalizedDocumentReferences(this, _compiled.ReferencedCollections, DocumentDatabase.DocumentsStorage, _indexStorage, Configuration);
+
+                workers.Add(_handleReferences);
+            }
 
             workers.Add(new MapDocuments(this, DocumentDatabase.DocumentsStorage, _indexStorage, MapReduceWorkContext, Configuration));
             workers.Add(new ReduceMapResultsOfStaticIndex(this, _compiled.Reduce, Definition, _indexStorage, DocumentDatabase.Metrics, MapReduceWorkContext));

src/Raven.Server/Documents/Indexes/Static/CurrentIndexingScope.cs

@@ -36,6 +36,8 @@ public class CurrentIndexingScope : IDisposable
         /// [collection: [key: [referenceKeys]]]
         public Dictionary<string, Dictionary<Slice, HashSet<Slice>>> ReferencesByCollectionForCompareExchange;
 
+        public HashSet<Slice> ReferencesToDelete;
+
         public MismatchedReferencesWarningHandler MismatchedReferencesWarningHandler;
 
         public readonly bool UseNormalizedIds;

src/Raven.Server/Documents/Indexes/Static/MapIndex.cs

@@ -1,5 +1,6 @@
 using System;
 using System.Collections.Generic;
+using System.Diagnostics;
 using System.Linq;
 using Raven.Client;
 using Raven.Client.Documents.Changes;
@@ -66,11 +67,25 @@ protected override IIndexingWork[] CreateIndexWorkExecutors()
                 new CleanupDocuments(this, DocumentDatabase.DocumentsStorage, _indexStorage, Configuration, null)
             };
 
+            Debug.Assert(SourceType == IndexSourceType.Documents);
+
             if (_compiled.CollectionsWithCompareExchangeReferences.Count > 0)
-                workers.Add(_handleCompareExchangeReferences = new HandleCompareExchangeReferences(this, _compiled.CollectionsWithCompareExchangeReferences, DocumentDatabase.DocumentsStorage, _indexStorage, Configuration));
+            {
+                _handleCompareExchangeReferences = Definition.Version >= IndexDefinitionBaseServerSide.IndexVersion.LowerCasedReferences
+                    ? new HandleCompareExchangeReferences(this, _compiled.CollectionsWithCompareExchangeReferences, DocumentDatabase.DocumentsStorage, _indexStorage, Configuration)
+                    : new HandleNotNormalizedCompareExchangeReferences(this, _compiled.CollectionsWithCompareExchangeReferences, DocumentDatabase.DocumentsStorage, _indexStorage, Configuration);
+
+                workers.Add(_handleCompareExchangeReferences);
+            }
 
             if (_referencedCollections.Count > 0)
-                workers.Add(_handleReferences = new HandleDocumentReferences(this, _compiled.ReferencedCollections, DocumentDatabase.DocumentsStorage, _indexStorage, Configuration));
+            {
+                _handleReferences = Definition.Version >= IndexDefinitionBaseServerSide.IndexVersion.LowerCasedReferences
+                    ? new HandleDocumentReferences(this, _compiled.ReferencedCollections, DocumentDatabase.DocumentsStorage, _indexStorage, Configuration)
+                    : new HandleNotNormalizedDocumentReferences(this, _compiled.ReferencedCollections, DocumentDatabase.DocumentsStorage, _indexStorage, Configuration);
+
+                workers.Add(_handleReferences);
+            }
 
             workers.Add(new MapDocuments(this, DocumentDatabase.DocumentsStorage, _indexStorage, null, Configuration));
 

src/Raven.Server/Documents/Indexes/Workers/HandleNotNormalizedCompareExchangeReferences.cs

@@ -0,0 +1,23 @@
+using System.Collections.Generic;
+using Raven.Server.Config.Categories;
+using Raven.Server.ServerWide.Context;
+using Voron;
+
+namespace Raven.Server.Documents.Indexes.Workers;
+
+public class HandleNotNormalizedCompareExchangeReferences : HandleCompareExchangeReferences
+{
+    public HandleNotNormalizedCompareExchangeReferences(Index index, HashSet<string> collectionsWithCompareExchangeReferences, DocumentsStorage documentsStorage, IndexStorage indexStorage, IndexingConfiguration configuration) : base(index, collectionsWithCompareExchangeReferences, documentsStorage, indexStorage, configuration)
+    {
+    }
+
+    protected override IndexItem GetItem(DocumentsOperationContext databaseContext, Slice key)
+    {
+        return HandleNotNormalizedDocumentReferences.GetNonNormalizedDocumentItem(databaseContext, key);
+    }
+
+    protected override void AfterGetItemsFromCollectionThatReference(string collection, IndexingStatsScope stats, DocumentsOperationContext databaseContext, TransactionOperationContext indexContext)
+    {
+        HandleNotNormalizedDocumentReferences.AfterGetItemsFromCollectionThatReference(collection, stats, _referencesStorage, databaseContext, indexContext);
+    }
+}

src/Raven.Server/Documents/Indexes/Workers/HandleNotNormalizedDocumentReferences.cs

@@ -0,0 +1,63 @@
+using System.Collections.Generic;
+using Raven.Client.Documents.Indexes;
+using Raven.Server.Config.Categories;
+using Raven.Server.Documents.Indexes.Static;
+using Raven.Server.ServerWide.Context;
+using Voron;
+
+namespace Raven.Server.Documents.Indexes.Workers;
+
+public class HandleNotNormalizedDocumentReferences : HandleDocumentReferences
+{
+    public HandleNotNormalizedDocumentReferences(Index index, Dictionary<string, HashSet<CollectionName>> referencedCollections, DocumentsStorage documentsStorage, IndexStorage indexStorage, IndexingConfiguration configuration) : base(index, referencedCollections, documentsStorage, indexStorage, configuration)
+    {
+    }
+
+    protected HandleNotNormalizedDocumentReferences(Index index, Dictionary<string, HashSet<CollectionName>> referencedCollections, DocumentsStorage documentsStorage, IndexStorage indexStorage, IndexStorage.ReferencesBase referencesStorage, IndexingConfiguration configuration) : base(index, referencedCollections, documentsStorage, indexStorage, referencesStorage, configuration)
+    {
+    }
+
+    protected override IndexItem GetItem(DocumentsOperationContext databaseContext, Slice key)
+    {
+        return GetNonNormalizedDocumentItem(databaseContext, key);
+    }
+
+    public static unsafe IndexItem GetNonNormalizedDocumentItem(DocumentsOperationContext databaseContext, Slice key)
+    {
+        using (DocumentIdWorker.GetLower(databaseContext.Allocator, key.Content.Ptr, key.Size, out var loweredKey))
+        {
+            var documentItem = GetDocumentItem(databaseContext, loweredKey);
+            if (documentItem == null)
+            {
+                // this isn't required for new indexes as CleanupDocuments will handle it.
+                // however, for older indexes, we need to clean up any leftovers.
+                CurrentIndexingScope.Current.ReferencesToDelete ??= new HashSet<Slice>();
+                CurrentIndexingScope.Current.ReferencesToDelete.Add(key.Clone(databaseContext.Allocator));
+            }
+
+            return documentItem;
+        }
+    }
+
+    protected override void AfterGetItemsFromCollectionThatReference(string collection, IndexingStatsScope stats, DocumentsOperationContext databaseContext, TransactionOperationContext indexContext)
+    {
+        AfterGetItemsFromCollectionThatReference(collection, stats, _referencesStorage, databaseContext, indexContext);
+    }
+
+    public static void AfterGetItemsFromCollectionThatReference(string collection, IndexingStatsScope stats, IndexStorage.ReferencesBase referencesStorage, DocumentsOperationContext databaseContext, TransactionOperationContext indexContext)
+    {
+        if (CurrentIndexingScope.Current.ReferencesToDelete == null)
+            return;
+
+        using (stats.For(IndexingOperation.Map.DocumentRead, start: false).For(IndexingOperation.Storage.UpdateReferences))
+        {
+            foreach (var keyToRemove in CurrentIndexingScope.Current.ReferencesToDelete)
+            {
+                referencesStorage.RemoveReferences(keyToRemove, collection, null, indexContext.Transaction);
+                keyToRemove.Release(databaseContext.Allocator);
+            }
+        }
+
+        CurrentIndexingScope.Current.ReferencesToDelete.Clear();
+    }
+}

src/Raven.Server/Documents/Indexes/Workers/HandleReferences.cs

@@ -28,17 +28,19 @@ protected HandleDocumentReferences(Index index, Dictionary<string, HashSet<Colle
         {
         }
 
-        protected override unsafe IndexItem GetItem(DocumentsOperationContext databaseContext, Slice key)
+        protected override IndexItem GetItem(DocumentsOperationContext databaseContext, Slice key)
         {
-            using (DocumentIdWorker.GetLower(databaseContext.Allocator, key.Content.Ptr, key.Size, out var loweredKey))
-            {
-                // when there is conflict, we need to apply same behavior as if the document would not exist
-                var doc = _documentsStorage.Get(databaseContext, loweredKey, throwOnConflict: false);
-                if (doc == null)
-                    return default;
+            return GetDocumentItem(databaseContext, key);
+        }
 
-                return new DocumentIndexItem(doc.Id, doc.LowerId, doc.Etag, doc.LastModified, doc.Data.Size, doc);
-            }
+        public static IndexItem GetDocumentItem(DocumentsOperationContext databaseContext, Slice key)
+        {
+            // when there is conflict, we need to apply same behavior as if the document would not exist
+            var doc = databaseContext.DocumentDatabase.DocumentsStorage.Get(databaseContext, key, throwOnConflict: false);
+            if (doc == null)
+                return default;
+
+            return new DocumentIndexItem(doc.Id, doc.LowerId, doc.Etag, doc.LastModified, doc.Data.Size, doc);
         }
 
         public override void HandleDelete(Tombstone tombstone, string collection, Lazy<IndexWriteOperationBase> writer, TransactionOperationContext indexContext, IndexingStatsScope stats)
@@ -393,22 +395,12 @@ private IEnumerable<IndexItem> GetItemsFromCollectionThatReference(QueryOperatio
             string collection, Reference referencedItem, long lastIndexedEtag, HashSet<string> indexed, ReferencesState.ReferenceState referenceState, IndexingStatsScope stats)
         {
             var lastProcessedItemId = referenceState?.GetLastProcessedItemId(referencedItem);
-            List<Slice> keysToRemove = null;
 
             foreach (var key in _referencesStorage.GetItemKeysFromCollectionThatReference(collection, referencedItem.Key, indexContext.Transaction, lastProcessedItemId))
             {
                 var item = GetItem(queryContext.Documents, key);
                 if (item == null)
-                {
-                    if (CurrentIndexingScope.Current.UseNormalizedIds == false)
-                    {
-                        // this isn't required for new indexes as CleanupDocuments will handle it.
-                        // however, for older indexes, we need to clean up any leftovers.
-                        keysToRemove ??= new List<Slice>();
-                        keysToRemove.Add(key.Clone(queryContext.Documents.Allocator));
-                    }
                     continue;
-                }
 
                 if (indexed.Add(item.Id) == false)
                 {
@@ -440,17 +432,7 @@ void DisposeItem()
                 }
             }
 
-            if (keysToRemove != null)
-            {
-                using (stats.For(IndexingOperation.Map.DocumentRead, start: false).For(IndexingOperation.Storage.UpdateReferences))
-                {
-                    foreach (var keyToRemove in keysToRemove)
-                    {
-                        _referencesStorage.RemoveReferences(keyToRemove, collection, null, indexContext.Transaction);
-                        keyToRemove.Release(queryContext.Documents.Allocator);
-                    }
-                }
-            }
+            AfterGetItemsFromCollectionThatReference(collection, stats, queryContext.Documents, indexContext);
         }
 
         protected virtual IEnumerable<Reference> GetItemReferences(QueryOperationContext queryContext, CollectionName referencedCollection, long lastEtag, long pageSize)
@@ -486,6 +468,10 @@ public InMemoryReferencesInfo GetReferencesInfo(string collection)
 
         protected abstract IndexItem GetItem(DocumentsOperationContext databaseContext, Slice key);
 
+        protected virtual void AfterGetItemsFromCollectionThatReference(string collection, IndexingStatsScope stats, DocumentsOperationContext databaseContext, TransactionOperationContext indexContext)
+        {
+        }
+
         public abstract void HandleDelete(Tombstone tombstone, string collection, Lazy<IndexWriteOperationBase> writer, TransactionOperationContext indexContext, IndexingStatsScope stats);
 
         public class Reference : IDisposable