Skip to content

Commit

Permalink
RavenDB-23100 - add MapReduce tests
Browse files Browse the repository at this point in the history
  • Loading branch information
@grisha-kotler
grisha-kotler committed Nov 19, 2024
1 parent 7ac4c09 commit 5fa58c2
Show file tree
Hide file tree
Showing 6 changed files with 299 additions and 14 deletions.
Binary file removed test/SlowTests/Data/RavenDB_23100/2024-11-18-18-07-42-3204449.ravendb-snapshot
View file
Binary file not shown.
Binary file added test/SlowTests/Data/RavenDB_23100/2024-11-19-08-30-34-7519596.ravendb-snapshot
View file
Binary file not shown.
Binary file added test/SlowTests/Data/RavenDB_23100/2024-11-19-10-39-25-6242058.ravendb-snapshot
View file
Binary file not shown.
Binary file added test/SlowTests/Data/RavenDB_23100/2024-11-19-10-43-34-8245899.ravendb-snapshot
View file
Binary file not shown.
305 changes: 293 additions & 12 deletions test/SlowTests/Issues/RavenDB-23100.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,12 +30,14 @@ public RavenDB_23100(ITestOutputHelper output) : base(output)
private static readonly Employee _employee1 = new Employee { Id = EmployeeId1, CompanyId = _company.Id };
private static readonly Employee _employee2 = new Employee { Id = EmployeeId2, CompanyId = _company.Id };

[RavenFact(RavenTestCategory.Indexes)]
public async Task Can_Delete_Document_References()
[RavenTheory(RavenTestCategory.Indexes)]
[InlineData(typeof(DocumentsIndex))]
[InlineData(typeof(DocumentsIndexMapReduce))]
public async Task Can_Delete_Document_References(Type type)
{
using (var store = GetDocumentStore())
{
var index = new DocumentsIndex();
var index = (AbstractIndexCreationTask)Activator.CreateInstance(type);
await store.ExecuteIndexAsync(index);

using (var session = store.OpenAsyncSession())
Expand Down Expand Up @@ -197,11 +199,109 @@ public async Task Can_Delete_Document_References_For_Legacy_Index()
}

[RavenFact(RavenTestCategory.Indexes)]
public async Task Can_Delete_CompareExchange_References()
public async Task Can_Delete_Document_References_Map_Reduce_For_Legacy_Index()
{
var backupPath = NewDataPath(forceCreateDir: true);
var fullBackupPath = Path.Combine(backupPath, "2024-11-19-08-30-34-7519596.ravendb-snapshot");

await using (var file = File.Create(fullBackupPath))
{
await using (var stream = typeof(RavenDB_23100).Assembly.GetManifestResourceStream("SlowTests.Data.RavenDB_23100.2024-11-19-08-30-34-7519596.ravendb-snapshot"))
{
await stream.CopyToAsync(file);
}
}

using (var store = GetDocumentStore(new Options
{
CreateDatabase = false
}))
{
using (Backup.RestoreDatabase(store, new RestoreBackupConfiguration
{
BackupLocation = backupPath,
DatabaseName = store.Database
}))
{
var database = await GetDatabase(store.Database);
var indexInstance = database.IndexStore.GetIndex(new DocumentsIndexMapReduce().IndexName);

using (indexInstance._contextPool.AllocateOperationContext(out TransactionOperationContext context))
using (var tx = context.OpenReadTransaction())
{
var counts = indexInstance._indexStorage.ReferencesForDocuments.GetReferenceTablesCount("Employees", tx);

Assert.Equal(2, counts.ReferenceTableCount);
Assert.Equal(1, counts.CollectionTableCount);
}

using (var session = store.OpenAsyncSession())
{
var company = await session.LoadAsync<Company>(CompanyId);
company.Name += " LTD";
await session.SaveChangesAsync();
}

Indexes.WaitForIndexing(store);

using (var session = store.OpenAsyncSession())
{
var results = await session.Query<DocumentsIndexMapReduce.Result, DocumentsIndexMapReduce>()
.ProjectInto<DocumentsIndex.Result>().ToListAsync();

Assert.Equal(1, results.Count);
Assert.Equal("RavenDB LTD", results[0].CompanyName);
}

using (var session = store.OpenAsyncSession())
{
session.Delete(EmployeeId1);
session.Delete(EmployeeId2);
await session.SaveChangesAsync();
// deleting the documents won't change the internal references tree like in new indexes
}

Indexes.WaitForIndexing(store);

using (indexInstance._contextPool.AllocateOperationContext(out TransactionOperationContext context))
using (var tx = context.OpenReadTransaction())
{
var counts = indexInstance._indexStorage.ReferencesForDocuments.GetReferenceTablesCount("Employees", tx);

Assert.Equal(2, counts.ReferenceTableCount);
Assert.Equal(1, counts.CollectionTableCount);
}

using (var session = store.OpenAsyncSession())
{
var company = await session.LoadAsync<Company>(CompanyId);
company.Name += " HR";
await session.SaveChangesAsync();
// when we update the references, this will clean the leftovers
}

Indexes.WaitForIndexing(store);

using (indexInstance._contextPool.AllocateOperationContext(out TransactionOperationContext context))
using (var tx = context.OpenReadTransaction())
{
var counts = indexInstance._indexStorage.ReferencesForDocuments.GetReferenceTablesCount("Employees", tx);

Assert.Equal(0, counts.ReferenceTableCount);
Assert.Equal(0, counts.CollectionTableCount);
}
}
}
}

[RavenTheory(RavenTestCategory.Indexes)]
[InlineData(typeof(CompareExchangeIndex))]
[InlineData(typeof(CompareExchangeMapReduceIndex))]
public async Task Can_Delete_CompareExchange_References(Type type)
{
using (var store = GetDocumentStore())
{
var index = new DocumentsWithCompareExchangeIndex();
var index = (AbstractIndexCreationTask)Activator.CreateInstance(type);

await store.ExecuteIndexAsync(index);

Expand Down Expand Up @@ -273,11 +373,11 @@ public async Task Can_Delete_CompareExchange_References()
public async Task Can_Delete_CompareExchange_References_For_Legacy_Index()
{
var backupPath = NewDataPath(forceCreateDir: true);
var fullBackupPath = Path.Combine(backupPath, "2024-11-18-18-07-42-3204449.ravendb-snapshot");
var fullBackupPath = Path.Combine(backupPath, "2024-11-19-10-43-34-8245899.ravendb-snapshot");

await using (var file = File.Create(fullBackupPath))
{
await using (var stream = typeof(RavenDB_23100).Assembly.GetManifestResourceStream("SlowTests.Data.RavenDB_23100.2024-11-18-18-07-42-3204449.ravendb-snapshot"))
await using (var stream = typeof(RavenDB_23100).Assembly.GetManifestResourceStream("SlowTests.Data.RavenDB_23100.2024-11-19-10-43-34-8245899.ravendb-snapshot"))
{
await stream.CopyToAsync(file);
}
Expand All @@ -295,7 +395,7 @@ public async Task Can_Delete_CompareExchange_References_For_Legacy_Index()
}))
{
var database = await GetDatabase(store.Database);
var indexInstance = database.IndexStore.GetIndex(new DocumentsWithCompareExchangeIndex().IndexName);
var indexInstance = database.IndexStore.GetIndex(new CompareExchangeIndex().IndexName);

using (indexInstance._contextPool.AllocateOperationContext(out TransactionOperationContext context))
using (var tx = context.OpenReadTransaction())
Expand All @@ -317,8 +417,8 @@ public async Task Can_Delete_CompareExchange_References_For_Legacy_Index()

using (var session = store.OpenAsyncSession())
{
var results = await session.Query<DocumentsWithCompareExchangeIndex.Result, DocumentsWithCompareExchangeIndex>()
.ProjectInto<DocumentsWithCompareExchangeIndex.Result>().ToListAsync();
var results = await session.Query<CompareExchangeIndex.Result, CompareExchangeIndex>()
.ProjectInto<CompareExchangeIndex.Result>().ToListAsync();

Assert.Equal(2, results.Count);

Expand Down Expand Up @@ -369,6 +469,100 @@ public async Task Can_Delete_CompareExchange_References_For_Legacy_Index()
}
}

[RavenFact(RavenTestCategory.Indexes, Skip = "https://issues.hibernatingrhinos.com/issue/RavenDB-23166/Snapshot-restore-of-with-compare-exchange-references")]
public async Task Can_Delete_CompareExchange_References_For_Legacy_Index_Map_Reduce()
{
var backupPath = NewDataPath(forceCreateDir: true);
var fullBackupPath = Path.Combine(backupPath, "2024-11-19-10-39-25-6242058.ravendb-snapshot");

await using (var file = File.Create(fullBackupPath))
{
await using (var stream = typeof(RavenDB_23100).Assembly.GetManifestResourceStream("SlowTests.Data.RavenDB_23100.2024-11-19-10-39-25-6242058.ravendb-snapshot"))
{
await stream.CopyToAsync(file);
}
}

using (var store = GetDocumentStore(new Options
{
CreateDatabase = false
}))
{
using (Backup.RestoreDatabase(store, new RestoreBackupConfiguration
{
BackupLocation = backupPath,
DatabaseName = store.Database
}))
{
var database = await GetDatabase(store.Database);
var indexInstance = database.IndexStore.GetIndex(new CompareExchangeMapReduceIndex().IndexName);

using (indexInstance._contextPool.AllocateOperationContext(out TransactionOperationContext context))
using (var tx = context.OpenReadTransaction())
{
var counts = indexInstance._indexStorage.ReferencesForCompareExchange.GetReferenceTablesCount("Employees", tx);

Assert.Equal(2, counts.ReferenceTableCount);
Assert.Equal(1, counts.CollectionTableCount);
}

using (var session = store.OpenAsyncSession(new SessionOptions { TransactionMode = TransactionMode.ClusterWide }))
{
var company = await session.Advanced.ClusterTransaction.GetCompareExchangeValueAsync<Company>(CompanyId);
company.Value.Name += " LTD";
await session.SaveChangesAsync();
}

Indexes.WaitForIndexing(store);

using (var session = store.OpenAsyncSession())
{
var results = await session.Query<CompareExchangeMapReduceIndex.Result, CompareExchangeMapReduceIndex>().ToListAsync();
Assert.Equal(1, results.Count);
Assert.Equal("RavenDB LTD", results[0].CompanyName);
}

using (var session = store.OpenAsyncSession())
{
session.Delete(EmployeeId1);
session.Delete(EmployeeId2);
await session.SaveChangesAsync();
// deleting the documents won't change the internal references tree like in new indexes
}

Indexes.WaitForIndexing(store);

using (indexInstance._contextPool.AllocateOperationContext(out TransactionOperationContext context))
using (var tx = context.OpenReadTransaction())
{
var counts = indexInstance._indexStorage.ReferencesForCompareExchange.GetReferenceTablesCount("Employees", tx);

Assert.Equal(2, counts.ReferenceTableCount);
Assert.Equal(1, counts.CollectionTableCount);
}

using (var session = store.OpenAsyncSession(new SessionOptions { TransactionMode = TransactionMode.ClusterWide }))
{
var company = await session.Advanced.ClusterTransaction.GetCompareExchangeValueAsync<Company>(CompanyId);
company.Value.Name += " HR";
await session.SaveChangesAsync();
// when we update the references, this will clean the leftovers
}

Indexes.WaitForIndexing(store);

using (indexInstance._contextPool.AllocateOperationContext(out TransactionOperationContext context))
using (var tx = context.OpenReadTransaction())
{
var counts = indexInstance._indexStorage.ReferencesForCompareExchange.GetReferenceTablesCount("Employees", tx);

Assert.Equal(0, counts.ReferenceTableCount);
Assert.Equal(0, counts.CollectionTableCount);
}
}
}
}

[RavenFact(RavenTestCategory.Indexes)]
public async Task Can_Delete_Counters_References()
{
Expand Down Expand Up @@ -563,14 +757,41 @@ public DocumentsIndex()
}
}

private class DocumentsWithCompareExchangeIndex : AbstractIndexCreationTask<Employee>
private class DocumentsIndexMapReduce : AbstractIndexCreationTask<Employee, DocumentsIndexMapReduce.Result>
{
public class Result
{
public string CompanyName { get; set; }
public int Count { get; set; }
}

public DocumentsIndexMapReduce()
{
Map = employees => from employee in employees
select new Result
{
CompanyName = LoadDocument<Company>(employee.CompanyId).Name,
Count = 1
};

Reduce = results => from result in results
group result by result.CompanyName into g
select new Result
{
CompanyName = g.Key,
Count = g.Sum(x => x.Count)
};
}
}

private class CompareExchangeIndex : AbstractIndexCreationTask<Employee, CompareExchangeIndex.Result>
{
public class Result
{
public string CompanyName { get; set; }
}

public DocumentsWithCompareExchangeIndex()
public CompareExchangeIndex()
{
Map = employees =>
from employee in employees
Expand All @@ -583,6 +804,66 @@ from employee in employees
}
}

private class CompareExchangeMapReduceIndex : AbstractIndexCreationTask<Employee, CompareExchangeMapReduceIndex.Result>
{
public class Result
{
public string CompanyName { get; set; }

public int Count { get; set; }
}

public CompareExchangeMapReduceIndex()
{
Map = employees =>
from employee in employees
select new Result
{
CompanyName = LoadCompareExchangeValue<Company>(employee.CompanyId).Name,
Count = 1
};

Reduce = results => from result in results
group result by result.CompanyName into g
select new Result
{
CompanyName = g.Key,
Count = g.Sum(x => x.Count)
};

StoreAllFields(FieldStorage.Yes);
}
}

private class DocumentsWithCompareExchangeIndexMapReduce : AbstractIndexCreationTask<Employee, DocumentsWithCompareExchangeIndexMapReduce.Result>
{
public class Result
{
public string CompanyName { get; set; }
public int Count { get; set; }
}

public DocumentsWithCompareExchangeIndexMapReduce()
{
Map = employees =>
from employee in employees
select new Result
{
CompanyName = LoadCompareExchangeValue<Company>(employee.CompanyId).Name,
Count = 1
};

Reduce = results => from result in results
group result by result.CompanyName into g
select new Result
{
CompanyName = g.Key,
Count = g.Sum(x => x.Count)
};

}
}

private class CountersIndex : AbstractCountersIndexCreationTask<Employee>
{
public class Result
Expand Down
8 changes: 6 additions & 2 deletions test/SlowTests/SlowTests.csproj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,9 @@
<None Remove="Data\RavenDB_16378\RavenDB-16401-NRE.ravendb-snapshot" />
<None Remove="Data\RavenDB_18381\2022-03-27-12-38-05-8912792.ravendb-snapshot" />
<None Remove="Data\RavenDB_23100\2024-11-18-17-16-52-7843589.ravendb-snapshot" />
<None Remove="Data\RavenDB_23100\2024-11-18-18-07-42-3204449.ravendb-snapshot" />
<None Remove="Data\RavenDB_23100\2024-11-19-08-30-34-7519596.ravendb-snapshot" />
<None Remove="Data\RavenDB_23100\2024-11-19-10-39-25-6242058.ravendb-snapshot" />
<None Remove="Data\RavenDB_23100\2024-11-19-10-43-34-8245899.ravendb-snapshot" />
<None Remove="Smuggler\Data\DocumentWithRevisions.ravendbdump" />
<None Remove="Smuggler\Data\RevisionsWithoutADocument.ravendbdump" />
</ItemGroup>
Expand Down Expand Up @@ -125,7 +127,9 @@
<EmbeddedResource Include="Data\RavenDB_17070\Precorax\Orders_ByOrderBy.zip" />
<EmbeddedResource Include="Data\RavenDB_18381\2022-03-27-12-38-05-8912792.ravendb-snapshot" />
<EmbeddedResource Include="Data\RavenDB_23100\2024-11-18-17-16-52-7843589.ravendb-snapshot" />
<EmbeddedResource Include="Data\RavenDB_23100\2024-11-18-18-07-42-3204449.ravendb-snapshot" />
<EmbeddedResource Include="Data\RavenDB_23100\2024-11-19-08-30-34-7519596.ravendb-snapshot" />
<EmbeddedResource Include="Data\RavenDB_23100\2024-11-19-10-39-25-6242058.ravendb-snapshot" />
<EmbeddedResource Include="Data\RavenDB_23100\2024-11-19-10-43-34-8245899.ravendb-snapshot" />
<EmbeddedResource Include="Data\RavenDB_8355\MySorter.cs" />
<EmbeddedResource Include="Data\RavenDB_8355\MySorterWithDiagnostics.cs" />
<EmbeddedResource Include="Data\RavenDB_9912\1.ravendbdump" />
Expand Down

0 comments on commit 5fa58c2

Please sign in to comment.