From 526ec536a3f230da0fef2bd226ae53be47df8610 Mon Sep 17 00:00:00 2001 From: Alexander Rutkovsky Date: Wed, 24 Jan 2024 16:20:24 +0300 Subject: [PATCH] DS proxy mirror-3-dc restoration strategy test (#1238) --- .../blobstorage/dsproxy/dsproxy_blackboard.h | 8 +- .../dsproxy/ut_strategy/strategy_ut.cpp | 174 ++++++++++++++++++ .../blobstorage/dsproxy/ut_strategy/ya.make | 2 + 3 files changed, 180 insertions(+), 4 deletions(-) diff --git a/ydb/core/blobstorage/dsproxy/dsproxy_blackboard.h b/ydb/core/blobstorage/dsproxy/dsproxy_blackboard.h index 41b18eaf0869..8083e17fd415 100644 --- a/ydb/core/blobstorage/dsproxy/dsproxy_blackboard.h +++ b/ydb/core/blobstorage/dsproxy/dsproxy_blackboard.h @@ -108,9 +108,9 @@ struct TBlobState { }; struct TDiskGetRequest { - const TLogoBlobID Id; - const ui32 Shift; - const ui32 Size; + TLogoBlobID Id; + ui32 Shift; + ui32 Size; ssize_t PartMapIndex = -1; TDiskGetRequest(const TLogoBlobID &id, const ui32 shift, const ui32 size) @@ -127,7 +127,7 @@ struct TDiskPutRequest { ReasonInitial, ReasonAccelerate }; - const TLogoBlobID Id; + TLogoBlobID Id; TRope Buffer; EPutReason Reason; bool IsHandoff; diff --git a/ydb/core/blobstorage/dsproxy/ut_strategy/strategy_ut.cpp b/ydb/core/blobstorage/dsproxy/ut_strategy/strategy_ut.cpp index 9d7a548dfcdc..17ba3f7a16b7 100644 --- a/ydb/core/blobstorage/dsproxy/ut_strategy/strategy_ut.cpp +++ b/ydb/core/blobstorage/dsproxy/ut_strategy/strategy_ut.cpp @@ -1,7 +1,9 @@ #include #include +#include #include #include +#include using namespace NActors; using namespace NKikimr; @@ -193,10 +195,182 @@ void RunStrategyTest(TBlobStorageGroupType type) { } } +struct TGetQuery { + ui32 OrderNumber; + TLogoBlobID Id; + ui32 Shift; + ui32 Size; + + auto AsTuple() const { return std::make_tuple(OrderNumber, Id, Shift, Size); } + friend bool operator ==(const TGetQuery& x, const TGetQuery& y) { return x.AsTuple() == y.AsTuple(); } + friend bool operator <(const TGetQuery& x, const TGetQuery& y) { return x.AsTuple() < y.AsTuple(); } +}; + +struct TPutQuery { + ui32 OrderNumber; + TLogoBlobID Id; + + auto AsTuple() const { return std::make_tuple(OrderNumber, Id); } + friend bool operator ==(const TPutQuery& x, const TPutQuery& y) { return x.AsTuple() == y.AsTuple(); } + friend bool operator <(const TPutQuery& x, const TPutQuery& y) { return x.AsTuple() < y.AsTuple(); } +}; + +using TOperation = std::variant; + +void RunTestLevel(const TBlobStorageGroupInfo& info, TBlackboard& blackboard, + const std::function& runStrategies, const TLogoBlobID& id, + std::vector& stock, TSubgroupPartLayout presenceMask, bool nonWorkingDomain, + std::set& context, ui32& terminals) { + // see which operations we can add to the stock + const size_t stockSizeOnEntry = stock.size(); + auto& requests = blackboard.GroupDiskRequests.DiskRequestsForOrderNumber; + for (ui32 i = 0; i < info.GetTotalVDisksNum(); ++i) { + for (auto& j = requests[i].FirstUnsentRequestIdx; j < requests[i].GetsToSend.size(); ++j) { + auto& get = requests[i].GetsToSend[j]; + stock.push_back(TGetQuery{i, get.Id, get.Shift, get.Size}); + const bool inserted = context.insert(stock.back()).second; + UNIT_ASSERT(inserted); + } + for (auto& j = requests[i].FirstUnsentPutIdx; j < requests[i].PutsToSend.size(); ++j) { + auto& put = requests[i].PutsToSend[j]; + stock.push_back(TPutQuery{i, put.Id}); + const bool inserted = context.insert(stock.back()).second; + UNIT_ASSERT(inserted); + } + } + UNIT_ASSERT(!stock.empty()); + + bool canIssuePuts = true; + for (size_t i = 0; i < stock.size(); ++i) { + if (std::holds_alternative(stock[i])) { + canIssuePuts = false; + break; + } + } + + // try every single operation in stock + for (size_t i = 0; i < stock.size(); ++i) { + if (!canIssuePuts && std::holds_alternative(stock[i])) { + continue; + } + if (auto *get = std::get_if(&stock[i]); get && context.contains(TPutQuery{get->OrderNumber, get->Id})) { + continue; + } + + std::swap(stock[i], stock.back()); + TOperation operation = std::move(stock.back()); + stock.pop_back(); + + TBlackboard branch(blackboard); + TSubgroupPartLayout myPresenceMask(presenceMask); + + std::visit(TOverloaded{ + [&](const TGetQuery& op) { + const ui32 idxInSubgroup = info.GetTopology().GetIdxInSubgroup(info.GetVDiskId(op.OrderNumber), id.Hash()); + if (nonWorkingDomain && idxInSubgroup % 3 == 2) { + branch.AddErrorResponse(op.Id, op.OrderNumber); + } else if (myPresenceMask.GetDisksWithPart(op.Id.PartId() - 1) >> idxInSubgroup & 1) { + const ui32 blobSize = op.Id.BlobSize(); + const ui32 shift = Min(op.Shift, blobSize); + const ui32 size = Min(op.Size ? op.Size : Max(), blobSize - shift); + branch.AddResponseData(op.Id, op.OrderNumber, shift, TRope(TString(size, 'X'))); + } else { + branch.AddNoDataResponse(op.Id, op.OrderNumber); + } + }, + [&](const TPutQuery& op) { + const ui32 idxInSubgroup = info.GetTopology().GetIdxInSubgroup(info.GetVDiskId(op.OrderNumber), id.Hash()); + if (nonWorkingDomain && idxInSubgroup % 3 == 2) { + branch.AddErrorResponse(op.Id, op.OrderNumber); + } else { + myPresenceMask.AddItem(idxInSubgroup, op.Id.PartId() - 1, info.Type); + branch.AddPutOkResponse(op.Id, op.OrderNumber); + } + } + }, operation); + + auto outcome = runStrategies(branch); + UNIT_ASSERT(outcome != EStrategyOutcome::ERROR); + if (outcome == EStrategyOutcome::DONE) { + TBlobStorageGroupInfo::TOrderNums nums; + info.GetTopology().PickSubgroup(id.Hash(), nums); + UNIT_ASSERT(info.GetQuorumChecker().GetBlobState(myPresenceMask, {&info.GetTopology()}) == TBlobStorageGroupInfo::EBS_FULL); + ++terminals; + } else { + RunTestLevel(info, branch, runStrategies, id, stock, myPresenceMask, nonWorkingDomain, context, terminals); + } + + stock.push_back(std::move(operation)); + std::swap(stock[i], stock.back()); + } + + // revert stock + for (size_t i = stockSizeOnEntry; i < stock.size(); ++i) { + const size_t n = context.erase(stock[i]); + UNIT_ASSERT(n); + } + stock.resize(stockSizeOnEntry); +} + Y_UNIT_TEST_SUITE(DSProxyStrategyTest) { Y_UNIT_TEST(Restore_block42) { RunStrategyTest(TBlobStorageGroupType::Erasure4Plus2Block); } + Y_UNIT_TEST(Restore_mirror3dc) { + THPTimer timer; + const TBlobStorageGroupType type(TBlobStorageGroupType::ErasureMirror3dc); + + TBlobStorageGroupInfo info(type, 1, 3, 3); + info.Ref(); + TGroupQueues groupQueues(info.GetTopology()); + groupQueues.Ref(); + + std::vector stock; + + TLogContext logCtx(NKikimrServices::BS_PROXY, false); + logCtx.SuppressLog = true; + + auto runStrategies = [&](TBlackboard& blackboard) { + return blackboard.RunStrategy(logCtx, TMirror3dcGetWithRestoreStrategy()); + }; + + const ui32 base = RandomNumber(512u); + for (ui32 i = 0; i < 512; ++i) { + const ui32 diskMask = (base + i) % 512; + for (bool nonWorkingDomain : {false, true}) { + TBlackboard blackboard(&info, &groupQueues, NKikimrBlobStorage::UserData, NKikimrBlobStorage::FastRead); + + const TLogoBlobID id(1'000'000'000, 1, 1, 0, 1000, 0); + TSubgroupPartLayout presenceMask; + blackboard.AddNeeded(id, 0, id.BlobSize()); + bool partsAvailable = false; + for (ui32 idxInSubgroup = 0; idxInSubgroup < 9; ++idxInSubgroup) { + if (diskMask >> idxInSubgroup & 1 && (!nonWorkingDomain || idxInSubgroup % 3 != 2)) { + presenceMask.AddItem(idxInSubgroup, idxInSubgroup % 3, info.Type); + partsAvailable = true; + } + } + if (!partsAvailable) { + continue; + } + + Cerr << "diskMask# " << diskMask << " nonWorkingDomain# " << nonWorkingDomain; + + auto outcome = runStrategies(blackboard); + UNIT_ASSERT(outcome == EStrategyOutcome::IN_PROGRESS); + + std::set context; + ui32 terminals = 0; + RunTestLevel(info, blackboard, runStrategies, id, stock, presenceMask, nonWorkingDomain, context, terminals); + Cerr << " " << terminals << Endl; + + if (TDuration::Seconds(timer.Passed()) >= TDuration::Minutes(5)) { + return; + } + } + } + } + } diff --git a/ydb/core/blobstorage/dsproxy/ut_strategy/ya.make b/ydb/core/blobstorage/dsproxy/ut_strategy/ya.make index a307788274d6..97c82483c9e9 100644 --- a/ydb/core/blobstorage/dsproxy/ut_strategy/ya.make +++ b/ydb/core/blobstorage/dsproxy/ut_strategy/ya.make @@ -1,5 +1,7 @@ UNITTEST() +FORK_SUBTESTS() + TIMEOUT(600) SIZE(MEDIUM)