From ad9481c5d49e37ea320e1257058a5c26a4d4e6cb Mon Sep 17 00:00:00 2001 From: Ilia Shakhov Date: Tue, 10 Dec 2024 16:01:49 +0300 Subject: [PATCH] 24-3: Add scale recommender for autoscaling (#12425) --- ydb/core/base/hive.h | 23 ++ .../cms/console/console__alter_tenant.cpp | 52 ++++ .../cms/console/console__create_tenant.cpp | 43 +++ ydb/core/cms/console/console__scheme.h | 3 +- .../cms/console/console_tenants_manager.cpp | 271 ++++++++++++++++++ .../cms/console/console_tenants_manager.h | 8 + ydb/core/cms/console/console_ut_tenants.cpp | 160 +++++++++++ .../rpc_get_scale_recommendation.cpp | 146 ++++++++++ ydb/core/grpc_services/service_cms.h | 2 + ydb/core/grpc_services/ya.make | 1 + ydb/core/mind/hive/domain_info.cpp | 128 +++++++++ ydb/core/mind/hive/domain_info.h | 34 +++ ydb/core/mind/hive/hive_events.h | 3 + ydb/core/mind/hive/hive_impl.cpp | 132 +++++++++ ydb/core/mind/hive/hive_impl.h | 12 + ydb/core/mind/hive/hive_schema.h | 4 +- ydb/core/mind/hive/hive_ut.cpp | 150 ++++++++++ ydb/core/mind/hive/monitoring.cpp | 10 + ydb/core/mind/hive/node_info.cpp | 3 + ydb/core/mind/hive/node_info.h | 1 + .../mind/hive/scale_recommender_policy_ut.cpp | 263 +++++++++++++++++ .../hive/tx__configure_scale_recommender.cpp | 60 ++++ ydb/core/mind/hive/tx__load_everything.cpp | 11 +- ydb/core/mind/hive/ut/ya.make | 1 + ydb/core/mind/hive/ya.make | 1 + ydb/core/mind/local.cpp | 21 +- ydb/core/protos/config.proto | 5 + ydb/core/protos/counters_hive.proto | 4 + ydb/core/protos/feature_flags.proto | 1 + ydb/core/protos/hive.proto | 37 ++- ydb/core/testlib/tenant_helpers.h | 49 ++++ ydb/core/testlib/tenant_runtime.cpp | 1 + ydb/core/util/metrics.h | 11 +- ydb/public/api/grpc/ydb_cms_v1.proto | 3 + ydb/public/api/protos/ydb_cms.proto | 44 +++ ydb/services/cms/grpc_service.cpp | 17 +- .../flat_hive.schema | 6 + 37 files changed, 1698 insertions(+), 23 deletions(-) create mode 100644 ydb/core/grpc_services/rpc_get_scale_recommendation.cpp create mode 100644 ydb/core/mind/hive/scale_recommender_policy_ut.cpp create mode 100644 ydb/core/mind/hive/tx__configure_scale_recommender.cpp diff --git a/ydb/core/base/hive.h b/ydb/core/base/hive.h index f7bf6071771e..2e0e9a737683 100644 --- a/ydb/core/base/hive.h +++ b/ydb/core/base/hive.h @@ -49,6 +49,8 @@ namespace NKikimr { EvUpdateTabletsObject, EvUpdateDomain, EvRequestTabletDistribution, + EvRequestScaleRecommendation, + EvConfigureScaleRecommender, // replies EvBootTabletReply = EvBootTablet + 512, @@ -84,6 +86,8 @@ namespace NKikimr { EvUpdateTabletsObjectReply, EvUpdateDomainReply, EvResponseTabletDistribution, + EvResponseScaleRecommendation, + EvConfigureScaleRecommenderReply, EvEnd }; @@ -876,6 +880,25 @@ namespace NKikimr { struct TEvResponseTabletDistribution : TEventPB {}; + + struct TEvRequestScaleRecommendation : TEventPB + { + TEvRequestScaleRecommendation() = default; + + TEvRequestScaleRecommendation(TSubDomainKey domainKey) { + Record.MutableDomainKey()->CopyFrom(domainKey); + } + }; + + struct TEvResponseScaleRecommendation : TEventPB {}; + + struct TEvConfigureScaleRecommender : TEventPB {}; + + struct TEvConfigureScaleRecommenderReply : TEventPB {}; }; IActor* CreateDefaultHive(const TActorId &tablet, TTabletStorageInfo *info); diff --git a/ydb/core/cms/console/console__alter_tenant.cpp b/ydb/core/cms/console/console__alter_tenant.cpp index f3f00d4eb5d8..8f1a7b8f8a70 100644 --- a/ydb/core/cms/console/console__alter_tenant.cpp +++ b/ydb/core/cms/console/console__alter_tenant.cpp @@ -201,6 +201,48 @@ class TTenantsManager::TTxAlterTenant : public TTransactionBase return Error(Ydb::StatusIds::BAD_REQUEST, "Data size soft quota cannot be larger than hard quota", ctx); } } + + // Check scale recommender policies. + if (rec.has_scale_recommender_policies()) { + if (!Self->FeatureFlags.GetEnableScaleRecommender()) { + return Error(Ydb::StatusIds::UNSUPPORTED, "Feature flag EnableScaleRecommender is off", ctx); + } + + const auto& policies = rec.scale_recommender_policies(); + if (policies.policies().size() > 1) { + return Error(Ydb::StatusIds::BAD_REQUEST, "Currently, no more than one policy is supported at a time", ctx); + } + + if (!policies.policies().empty()) { + using enum Ydb::Cms::ScaleRecommenderPolicies_ScaleRecommenderPolicy_TargetTrackingPolicy::TargetCase; + using enum Ydb::Cms::ScaleRecommenderPolicies_ScaleRecommenderPolicy::PolicyCase; + + const auto& policy = policies.policies()[0]; + switch (policy.GetPolicyCase()) { + case kTargetTrackingPolicy: { + const auto& targetTracking = policy.target_tracking_policy(); + switch (targetTracking.GetTargetCase()) { + case kAverageCpuUtilizationPercent: { + auto cpuUtilization = targetTracking.average_cpu_utilization_percent(); + if (cpuUtilization < 10 || cpuUtilization > 90) { + return Error(Ydb::StatusIds::BAD_REQUEST, "Average CPU utilization target must be from 10% to 90%", ctx); + } + break; + } + case TARGET_NOT_SET: + return Error(Ydb::StatusIds::BAD_REQUEST, "Target type for target tracking policy is not set", ctx); + default: + return Error(Ydb::StatusIds::BAD_REQUEST, "Unsupported target type for target tracking policy", ctx); + } + break; + } + case POLICY_NOT_SET: + return Error(Ydb::StatusIds::BAD_REQUEST, "Policy type is not set", ctx); + default: + return Error(Ydb::StatusIds::BAD_REQUEST, "Unsupported policy type", ctx); + } + } + } // Check attributes. THashSet attrNames; @@ -274,6 +316,11 @@ class TTenantsManager::TTxAlterTenant : public TTransactionBase updateSubdomainVersion = true; } + if (rec.has_scale_recommender_policies()) { + ScaleRecommenderPolicies.ConstructInPlace(rec.scale_recommender_policies()); + Self->DbUpdateScaleRecommenderPolicies(Tenant, *ScaleRecommenderPolicies, txc, ctx); + } + if (rec.idempotency_key() || Tenant->AlterIdempotencyKey) { Tenant->AlterIdempotencyKey = rec.idempotency_key(); Self->DbUpdateTenantAlterIdempotencyKey(Tenant, Tenant->AlterIdempotencyKey, txc, ctx); @@ -367,6 +414,10 @@ class TTenantsManager::TTxAlterTenant : public TTransactionBase if (DatabaseQuotas) { Tenant->DatabaseQuotas.ConstructInPlace(*DatabaseQuotas); } + if (ScaleRecommenderPolicies) { + Tenant->ScaleRecommenderPolicies.ConstructInPlace(*ScaleRecommenderPolicies); + Tenant->ScaleRecommenderPoliciesConfirmed = false; + } if (SubdomainVersion) { Tenant->SubdomainVersion = *SubdomainVersion; } @@ -389,6 +440,7 @@ class TTenantsManager::TTxAlterTenant : public TTransactionBase THashMap PoolsToAdd; TMaybe SchemaOperationQuotas; TMaybe DatabaseQuotas; + TMaybe ScaleRecommenderPolicies; TMaybe SubdomainVersion; bool ComputationalUnitsModified; TTenant::TPtr Tenant; diff --git a/ydb/core/cms/console/console__create_tenant.cpp b/ydb/core/cms/console/console__create_tenant.cpp index 2d6997a27f00..add969983151 100644 --- a/ydb/core/cms/console/console__create_tenant.cpp +++ b/ydb/core/cms/console/console__create_tenant.cpp @@ -300,6 +300,49 @@ class TTenantsManager::TTxCreateTenant : public TTransactionBaseDatabaseQuotas.ConstructInPlace(quotas); } + if (rec.has_scale_recommender_policies()) { + if (!Self->FeatureFlags.GetEnableScaleRecommender()) { + return Error(Ydb::StatusIds::UNSUPPORTED, "Feature flag EnableScaleRecommender is off", ctx); + } + + const auto& policies = rec.scale_recommender_policies(); + if (policies.policies().size() > 1) { + return Error(Ydb::StatusIds::BAD_REQUEST, "Currently, no more than one policy is supported at a time", ctx); + } + + if (!policies.policies().empty()) { + using enum Ydb::Cms::ScaleRecommenderPolicies_ScaleRecommenderPolicy_TargetTrackingPolicy::TargetCase; + using enum Ydb::Cms::ScaleRecommenderPolicies_ScaleRecommenderPolicy::PolicyCase; + + const auto& policy = policies.policies()[0]; + switch (policy.GetPolicyCase()) { + case kTargetTrackingPolicy: { + const auto& targetTracking = policy.target_tracking_policy(); + switch (targetTracking.GetTargetCase()) { + case kAverageCpuUtilizationPercent: { + auto cpuUtilization = targetTracking.average_cpu_utilization_percent(); + if (cpuUtilization < 10 || cpuUtilization > 90) { + return Error(Ydb::StatusIds::BAD_REQUEST, "Average CPU utilization target must be from 10% to 90%", ctx); + } + break; + } + case TARGET_NOT_SET: + return Error(Ydb::StatusIds::BAD_REQUEST, "Target type for target tracking policy is not set", ctx); + default: + return Error(Ydb::StatusIds::BAD_REQUEST, "Unsupported target type for target tracking policy", ctx); + } + break; + } + case POLICY_NOT_SET: + return Error(Ydb::StatusIds::BAD_REQUEST, "Policy type is not set", ctx); + default: + return Error(Ydb::StatusIds::BAD_REQUEST, "Unsupported policy type", ctx); + } + } + Tenant->ScaleRecommenderPolicies.ConstructInPlace(policies); + Tenant->ScaleRecommenderPoliciesConfirmed = false; + } + if (rec.idempotency_key()) { Tenant->CreateIdempotencyKey = rec.idempotency_key(); } diff --git a/ydb/core/cms/console/console__scheme.h b/ydb/core/cms/console/console__scheme.h index ca43cad79bf6..f02ff4709483 100644 --- a/ydb/core/cms/console/console__scheme.h +++ b/ydb/core/cms/console/console__scheme.h @@ -49,6 +49,7 @@ struct Schema : NIceDb::Schema { struct DatabaseQuotas : Column<27, NScheme::NTypeIds::String> {}; struct IsExternalStatisticsAggregator : Column<28, NScheme::NTypeIds::Bool> {}; struct IsExternalBackupController : Column<29, NScheme::NTypeIds::Bool> {}; + struct ScaleRecommenderPolicies : Column<30, NScheme::NTypeIds::String> {}; using TKey = TableKey; using TColumns = TableColumns; + IsExternalBackupController, ScaleRecommenderPolicies>; }; struct TenantPools : Table<3> { diff --git a/ydb/core/cms/console/console_tenants_manager.cpp b/ydb/core/cms/console/console_tenants_manager.cpp index 88b5a6190ee7..10b5402b7147 100644 --- a/ydb/core/cms/console/console_tenants_manager.cpp +++ b/ydb/core/cms/console/console_tenants_manager.cpp @@ -3,6 +3,7 @@ #include "http.h" #include "util.h" +#include #include #include #include @@ -921,6 +922,227 @@ class TSubDomainManip : public TActorBootstrapped { } }; +class TScaleRecommenderManip : public TActorBootstrapped { +private: + TTenantsManager::TTenant::TPtr Tenant; + ui64 HiveId; + TActorId HivePipe; + +public: + static constexpr NKikimrServices::TActivity::EType ActorActivityType() + { + return NKikimrServices::TActivity::CMS_TENANTS_MANAGER; + } + + TScaleRecommenderManip(TTenantsManager::TTenant::TPtr tenant) + : Tenant(tenant) + , HiveId(0) + {} + + void Bootstrap(const TActorContext &ctx) { + BLOG_D("TScaleRecommenderManip(" << Tenant->Path << ")::Bootstrap"); + + Become(&TThis::StateResolveHive); + ResolveHive(ctx); + } + + void ResolveHive(const TActorContext &ctx) const { + auto request = MakeHolder(); + request->DatabaseName = Tenant->Path; + + auto& entry = request->ResultSet.emplace_back(); + entry.Operation = NSchemeCache::TSchemeCacheNavigate::OpPath; + entry.Path = NKikimr::SplitPath(Tenant->Path); + + ctx.Send(MakeSchemeCacheID(), new TEvTxProxySchemeCache::TEvNavigateKeySet(request.Release())); + } + + STFUNC(StateResolveHive) { + switch (ev->GetTypeRewrite()) { + HFunc(TEvTxProxySchemeCache::TEvNavigateKeySetResult, Handle); + default: + Y_ABORT("unexpected event type: %" PRIx32 " event: %s", + ev->GetTypeRewrite(), ev->ToString().data()); + break; + } + } + + void Handle(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev, const TActorContext &ctx) { + const auto& request = ev->Get()->Request; + + if (request->ResultSet.empty()) { + LOG_ERROR_S(ctx, NKikimrServices::CMS_TENANTS, + "TScaleRecommenderManip got empty results during resolving " + << Tenant->Path); + Finish(); + return; + } + + const auto& entry = request->ResultSet.front(); + + if (request->ErrorCount > 0) { + switch (entry.Status) { + case NSchemeCache::TSchemeCacheNavigate::EStatus::Ok: + break; + case NSchemeCache::TSchemeCacheNavigate::EStatus::AccessDenied: + case NSchemeCache::TSchemeCacheNavigate::EStatus::RootUnknown: + case NSchemeCache::TSchemeCacheNavigate::EStatus::PathErrorUnknown: + case NSchemeCache::TSchemeCacheNavigate::EStatus::LookupError: + case NSchemeCache::TSchemeCacheNavigate::EStatus::RedirectLookupError: + default: + LOG_ERROR_S(ctx, NKikimrServices::CMS_TENANTS, + "TScaleRecommenderManip got entry with error during resolving " + << Tenant->Path + << ", entry# " << entry.ToString()); + Finish(); + return; + } + } + + + auto domainInfo = entry.DomainInfo; + if (!domainInfo || !domainInfo->Params.HasHive()) { + LOG_ERROR_S(ctx, NKikimrServices::CMS_TENANTS, + "TScaleRecommenderManip resolved tenant " + << Tenant->Path + << " that has no hive" + << ", entry# " << entry.ToString()); + Finish(); + return; + } + + HiveId = domainInfo->Params.GetHive(); + Become(&TThis::StateWork); + ConfigureScaleRecommender(ctx); + } + + void ConfigureScaleRecommender(const TActorContext &ctx) { + OpenHivePipe(ctx); + + auto request = std::make_unique(); + auto& record = request->Record; + record.MutableDomainKey()->SetSchemeShard(Tenant->DomainId.OwnerId); + record.MutableDomainKey()->SetPathId(Tenant->DomainId.LocalPathId); + for (const auto& p : Tenant->ScaleRecommenderPolicies->policies()) { + switch (p.GetPolicyCase()) { + case Ydb::Cms::ScaleRecommenderPolicies_ScaleRecommenderPolicy::kTargetTrackingPolicy: { + auto* hivePolicy = record.MutablePolicies()->AddPolicies()->MutableTargetTrackingPolicy(); + switch (p.target_tracking_policy().GetTargetCase()) { + case Ydb::Cms:: + ScaleRecommenderPolicies_ScaleRecommenderPolicy_TargetTrackingPolicy::kAverageCpuUtilizationPercent: + hivePolicy->SetAverageCpuUtilizationPercent(p.target_tracking_policy().average_cpu_utilization_percent()); + break; + default: + LOG_ERROR_S(ctx, NKikimrServices::CMS_TENANTS, + "TScaleRecommenderManip got unknown taget for target tracking policy for " + << Tenant->Path + << ", policy# " << p.target_tracking_policy().ShortDebugString()); + Finish(); + break; + } + break; + } + default: + LOG_ERROR_S(ctx, NKikimrServices::CMS_TENANTS, + "TScaleRecommenderManip got unknown scale policy for " + << Tenant->Path + << ", policies# " << Tenant->ScaleRecommenderPolicies->ShortDebugString()); + Finish(); + return; + } + } + + LOG_TRACE_S(ctx, NKikimrServices::CMS_TENANTS, + "Send TEvHive::TEvConfigureScaleRecommender: " + << request->Record.ShortDebugString()); + + NTabletPipe::SendData(ctx, HivePipe, request.release()); + } + + STFUNC(StateWork) { + switch (ev->GetTypeRewrite()) { + HFunc(TEvHive::TEvConfigureScaleRecommenderReply, Handle); + HFunc(TEvTabletPipe::TEvClientConnected, Handle); + HFunc(TEvTabletPipe::TEvClientDestroyed, Handle); + + default: + Y_ABORT("unexpected event type: %" PRIx32 " event: %s", + ev->GetTypeRewrite(), ev->ToString().data()); + break; + } + } + + void Handle(TEvHive::TEvConfigureScaleRecommenderReply::TPtr& ev, const TActorContext& ctx) { + switch (ev->Get()->Record.GetStatus()) { + case NKikimrProto::OK: + Tenant->ScaleRecommenderPoliciesConfirmed = true; + Finish(); + break; + case NKikimrProto::ERROR: + case NKikimrProto::ALREADY: + case NKikimrProto::TIMEOUT: + case NKikimrProto::RACE: + case NKikimrProto::NODATA: + case NKikimrProto::BLOCKED: + case NKikimrProto::NOTREADY: + case NKikimrProto::OVERRUN: + case NKikimrProto::TRYLATER: + case NKikimrProto::TRYLATER_TIME: + case NKikimrProto::TRYLATER_SIZE: + case NKikimrProto::DEADLINE: + case NKikimrProto::CORRUPTED: + case NKikimrProto::SCHEDULED: + case NKikimrProto::OUT_OF_SPACE: + case NKikimrProto::VDISK_ERROR_STATE: + case NKikimrProto::INVALID_OWNER: + case NKikimrProto::INVALID_ROUND: + case NKikimrProto::RESTART: + case NKikimrProto::NOT_YET: + case NKikimrProto::NO_GROUP: + case NKikimrProto::UNKNOWN: + LOG_ERROR_S(ctx, NKikimrServices::CMS_TENANTS, + "TScaleRecommenderManip got error reply during configuring hive for " + << Tenant->Path + << ", reply# " << ev->Get()->Record.ShortDebugString()); + Finish(); + break; + } + } + + void Handle(TEvTabletPipe::TEvClientConnected::TPtr& ev, const TActorContext& ctx) { + if (ev->Get()->Status != NKikimrProto::OK) { + OnPipeDestroyed(ctx); + } + } + + void Handle(TEvTabletPipe::TEvClientDestroyed::TPtr&, const TActorContext& ctx) { + OnPipeDestroyed(ctx); + } + + void OnPipeDestroyed(const TActorContext &ctx) { + if (HivePipe) { + NTabletPipe::CloseClient(ctx, HivePipe); + HivePipe = TActorId(); + } + ConfigureScaleRecommender(ctx); + } + + void OpenHivePipe(const TActorContext &ctx) { + Y_ABORT_UNLESS(HiveId); + NTabletPipe::TClientConfig pipeConfig; + pipeConfig.RetryPolicy = FastConnectRetryPolicy(); + auto pipe = NTabletPipe::CreateClient(ctx.SelfID, HiveId, pipeConfig); + HivePipe = ctx.ExecutorThread.RegisterActor(pipe); + } + + void Finish() { + if (Tenant->ScaleRecommenderPoliciesWorker == this->SelfId()) { + Tenant->ScaleRecommenderPoliciesWorker = TActorId(); + } + PassAway(); + } +}; + THashMap TSubDomainManip::IssuesMap; } // anonymous namespace @@ -1214,6 +1436,7 @@ TTenantsManager::TTenant::TTenant(const TString &path, , IsExternalStatisticsAggregator(false) , IsExternalBackupController(false) , AreResourcesShared(false) + , ScaleRecommenderPoliciesConfirmed(false) { } @@ -1820,6 +2043,19 @@ void TTenantsManager::DeleteTenantPools(TTenant::TPtr tenant, const TActorContex } } +void TTenantsManager::CongifureScaleRecommender(TTenant::TPtr tenant, const TActorContext &ctx) +{ + Y_ABORT_UNLESS(tenant->IsConfiguring() || tenant->IsRunning()); + + if (tenant->ScaleRecommenderPoliciesConfirmed) { + return; + } + + if (tenant->ScaleRecommenderPolicies && tenant->IsExternalHive && !tenant->ScaleRecommenderPoliciesWorker) { + tenant->ScaleRecommenderPoliciesWorker = ctx.RegisterWithSameMailbox(new TScaleRecommenderManip(tenant)); + } +} + void TTenantsManager::RequestTenantResources(TTenant::TPtr tenant, const TActorContext &ctx) { if (!TenantSlotBrokerPipe) @@ -1944,6 +2180,10 @@ void TTenantsManager::FillTenantStatus(TTenant::TPtr tenant, Ydb::Cms::GetDataba if (tenant->DatabaseQuotas) { status.mutable_database_quotas()->CopyFrom(*tenant->DatabaseQuotas); } + + if (tenant->ScaleRecommenderPolicies && !tenant->ScaleRecommenderPolicies->policies().empty()) { + status.mutable_scale_recommender_policies()->CopyFrom(*tenant->ScaleRecommenderPolicies); + } } void TTenantsManager::FillTenantAllocatedSlots(TTenant::TPtr tenant, Ydb::Cms::GetDatabaseStatusResult &status, @@ -2039,6 +2279,8 @@ void TTenantsManager::ProcessTenantActions(TTenant::TPtr tenant, const TActorCon // Process slots allocation. if (!tenant->SlotsAllocationConfirmed) RequestTenantResources(tenant, ctx); + // Deliver scale recommender policies. + CongifureScaleRecommender(tenant, ctx); } else if (tenant->State == TTenant::REMOVING_UNITS) { RequestTenantResources(tenant, ctx); } else if (tenant->State == TTenant::REMOVING_SUBDOMAIN) { @@ -2336,6 +2578,13 @@ void TTenantsManager::DbAddTenant(TTenant::TPtr tenant, .Update(NIceDb::TUpdate(serialized)); } + if (tenant->ScaleRecommenderPolicies) { + TString serialized; + Y_ABORT_UNLESS(tenant->ScaleRecommenderPolicies->SerializeToString(&serialized)); + db.Table().Key(tenant->Path) + .Update(NIceDb::TUpdate(serialized)); + } + for (auto &pr : tenant->StoragePools) { auto &pool = *pr.second; @@ -2450,6 +2699,11 @@ bool TTenantsManager::DbLoadState(TTransactionContext &txc, const TActorContext Y_ABORT_UNLESS(ParseFromStringNoSizeLimit(deserialized, tenantRowset.GetValue())); } + if (tenantRowset.HaveValue()) { + auto& deserialized = tenant->ScaleRecommenderPolicies.ConstructInPlace(); + Y_ABORT_UNLESS(ParseFromStringNoSizeLimit(deserialized, tenantRowset.GetValue())); + } + if (tenantRowset.HaveValue()) { tenant->CreateIdempotencyKey = tenantRowset.GetValue(); } @@ -2935,6 +3189,23 @@ void TTenantsManager::DbUpdateDatabaseQuotas(TTenant::TPtr tenant, .Update(NIceDb::TUpdate(serialized)); } +void TTenantsManager::DbUpdateScaleRecommenderPolicies(TTenant::TPtr tenant, + const Ydb::Cms::ScaleRecommenderPolicies &policies, + TTransactionContext &txc, + const TActorContext &ctx) +{ + LOG_TRACE_S(ctx, NKikimrServices::CMS_TENANTS, + "Update scale recommender policies for " << tenant->Path + << " policies = " << policies.DebugString()); + + TString serialized; + Y_ABORT_UNLESS(policies.SerializeToString(&serialized)); + + NIceDb::TNiceDb db(txc.DB); + db.Table().Key(tenant->Path) + .Update(NIceDb::TUpdate(serialized)); +} + void TTenantsManager::Handle(TEvConsole::TEvAlterTenantRequest::TPtr &ev, const TActorContext &ctx) { Counters.Inc(COUNTER_ALTER_REQUESTS); diff --git a/ydb/core/cms/console/console_tenants_manager.h b/ydb/core/cms/console/console_tenants_manager.h index ad95ea342093..dfe439202887 100644 --- a/ydb/core/cms/console/console_tenants_manager.h +++ b/ydb/core/cms/console/console_tenants_manager.h @@ -536,6 +536,9 @@ class TTenantsManager : public TActorBootstrapped { TMaybe SchemaOperationQuotas; TMaybe DatabaseQuotas; + TMaybe ScaleRecommenderPolicies; + bool ScaleRecommenderPoliciesConfirmed; + TActorId ScaleRecommenderPoliciesWorker; TString CreateIdempotencyKey; TString AlterIdempotencyKey; }; @@ -798,6 +801,7 @@ class TTenantsManager : public TActorBootstrapped { void RequestTenantSlotsState(TTenant::TPtr tenant, const TActorContext &ctx); void RequestTenantSlotsStats(const TActorContext &ctx); void RetryResourcesRequests(const TActorContext &ctx); + void CongifureScaleRecommender(TTenant::TPtr tenant, const TActorContext &ctx); void FillTenantStatus(TTenant::TPtr tenant, Ydb::Cms::GetDatabaseStatusResult &status); void FillTenantAllocatedSlots(TTenant::TPtr tenant, Ydb::Cms::GetDatabaseStatusResult &status, @@ -913,6 +917,10 @@ class TTenantsManager : public TActorBootstrapped { const Ydb::Cms::DatabaseQuotas "as, TTransactionContext &txc, const TActorContext &ctx); + void DbUpdateScaleRecommenderPolicies(TTenant::TPtr tenant, + const Ydb::Cms::ScaleRecommenderPolicies &policies, + TTransactionContext &txc, + const TActorContext &ctx); void Handle(TEvConsole::TEvAlterTenantRequest::TPtr &ev, const TActorContext &ctx); void Handle(TEvConsole::TEvCreateTenantRequest::TPtr &ev, const TActorContext &ctx); diff --git a/ydb/core/cms/console/console_ut_tenants.cpp b/ydb/core/cms/console/console_ut_tenants.cpp index ca80ae11efa7..8913b7ed9875 100644 --- a/ydb/core/cms/console/console_ut_tenants.cpp +++ b/ydb/core/cms/console/console_ut_tenants.cpp @@ -2093,6 +2093,166 @@ Y_UNIT_TEST_SUITE(TConsoleTests) { ) ); } + + Y_UNIT_TEST(TestScaleRecommenderPolicies) { + TTenantTestRuntime runtime(DefaultConsoleTestConfig()); + + // Create tenant with scale recommender policies + CheckCreateTenant(runtime, Ydb::StatusIds::SUCCESS, + TCreateTenantRequest(TENANT1_1_NAME, TCreateTenantRequest::EType::Common) + .WithPools({{"hdd", 1}}) + .WithScaleRecommenderPolicies(R"( + policies { + target_tracking_policy { + average_cpu_utilization_percent: 60 + } + } + )" + ) + ); + RestartTenantPool(runtime); + + // Check that tenant was successfully created + CheckTenantStatus(runtime, TENANT1_1_NAME, false, Ydb::StatusIds::SUCCESS, + Ydb::Cms::GetDatabaseStatusResult::RUNNING, + {{"hdd", 1, 1}}, {}); + CheckTenantScaleRecommenderPolicies(runtime, TENANT1_1_NAME, R"( + policies { + target_tracking_policy { + average_cpu_utilization_percent: 60 + } + } + )" + ); + + // Check persistence after creation + RestartConsole(runtime); + CheckTenantScaleRecommenderPolicies(runtime, TENANT1_1_NAME, R"( + policies { + target_tracking_policy { + average_cpu_utilization_percent: 60 + } + } + )" + ); + + // Alter tenant scale recommender policies + AlterScaleRecommenderPolicies(runtime, TENANT1_1_NAME, Ydb::StatusIds::SUCCESS, R"( + policies { + target_tracking_policy { + average_cpu_utilization_percent: 70 + } + } + )" + ); + + // Check that tenant was successfully altered + CheckTenantStatus(runtime, TENANT1_1_NAME, false, Ydb::StatusIds::SUCCESS, + Ydb::Cms::GetDatabaseStatusResult::RUNNING, + {{"hdd", 1, 1}}, {}); + CheckTenantScaleRecommenderPolicies(runtime, TENANT1_1_NAME, R"( + policies { + target_tracking_policy { + average_cpu_utilization_percent: 70 + } + } + )" + ); + + // Check persistence after altering + RestartConsole(runtime); + CheckTenantScaleRecommenderPolicies(runtime, TENANT1_1_NAME, R"( + policies { + target_tracking_policy { + average_cpu_utilization_percent: 70 + } + } + )" + ); + + // Reset tenant scale recommender policies + AlterScaleRecommenderPolicies(runtime, TENANT1_1_NAME, Ydb::StatusIds::SUCCESS, ""); + + // Check that tenant's scale recommender policies was successfully reset + CheckTenantStatus(runtime, TENANT1_1_NAME, false, Ydb::StatusIds::SUCCESS, + Ydb::Cms::GetDatabaseStatusResult::RUNNING, + {{"hdd", 1, 1}}, {}); + CheckTenantScaleRecommenderPolicies(runtime, TENANT1_1_NAME, ""); + + // Check persistence after resetting + RestartConsole(runtime); + CheckTenantScaleRecommenderPolicies(runtime, TENANT1_1_NAME, ""); + } + + Y_UNIT_TEST(TestScaleRecommenderPoliciesValidation) { + TTenantTestRuntime runtime(DefaultConsoleTestConfig()); + + CheckCreateTenant(runtime, Ydb::StatusIds::BAD_REQUEST, + TCreateTenantRequest(TENANT1_1_NAME, TCreateTenantRequest::EType::Common) + .WithPools({{"hdd", 1}}) + .WithScaleRecommenderPolicies(R"( + policies { + target_tracking_policy { + average_cpu_utilization_percent: 60 + } + } + policies { + target_tracking_policy { + average_cpu_utilization_percent: 50 + } + } + )" + ) + ); + + CheckCreateTenant(runtime, Ydb::StatusIds::BAD_REQUEST, + TCreateTenantRequest(TENANT1_1_NAME, TCreateTenantRequest::EType::Common) + .WithPools({{"hdd", 1}}) + .WithScaleRecommenderPolicies(R"( + policies { + target_tracking_policy { + average_cpu_utilization_percent: 100 + } + } + )" + ) + ); + + CheckCreateTenant(runtime, Ydb::StatusIds::BAD_REQUEST, + TCreateTenantRequest(TENANT1_1_NAME, TCreateTenantRequest::EType::Common) + .WithPools({{"hdd", 1}}) + .WithScaleRecommenderPolicies(R"( + policies { + target_tracking_policy { + average_cpu_utilization_percent: 0 + } + } + )" + ) + ); + + CheckCreateTenant(runtime, Ydb::StatusIds::BAD_REQUEST, + TCreateTenantRequest(TENANT1_1_NAME, TCreateTenantRequest::EType::Common) + .WithPools({{"hdd", 1}}) + .WithScaleRecommenderPolicies(R"( + policies { + target_tracking_policy { + } + } + )" + ) + ); + + CheckCreateTenant(runtime, Ydb::StatusIds::BAD_REQUEST, + TCreateTenantRequest(TENANT1_1_NAME, TCreateTenantRequest::EType::Common) + .WithPools({{"hdd", 1}}) + .WithScaleRecommenderPolicies(R"( + policies { + } + )" + ) + ); + } } } // namespace NKikimr diff --git a/ydb/core/grpc_services/rpc_get_scale_recommendation.cpp b/ydb/core/grpc_services/rpc_get_scale_recommendation.cpp new file mode 100644 index 000000000000..5876e17fc308 --- /dev/null +++ b/ydb/core/grpc_services/rpc_get_scale_recommendation.cpp @@ -0,0 +1,146 @@ +#include "service_cms.h" + +#include +#include +#include +#include +#include +#include + +#include + +namespace NKikimr::NGRpcService { + +using TEvGetScaleRecommendationRequest = TGrpcRequestNoOperationCall; + +class TGetScaleRecommendationRPC : public TRpcRequestActor { +public: + using TRpcRequestActor::TRpcRequestActor; + + void Bootstrap(const TActorContext&) { + ResolveDatabase(GetProtoRequest()->path()); + this->Become(&TThis::StateWork); + } + + STATEFN(StateWork) { + switch (ev->GetTypeRewrite()) { + hFunc(TEvTxProxySchemeCache::TEvNavigateKeySetResult, Handle); + hFunc(TEvHive::TEvResponseScaleRecommendation, Handle); + + hFunc(TEvTabletPipe::TEvClientConnected, Handle); + hFunc(TEvTabletPipe::TEvClientDestroyed, Handle); + } + } + + void ResolveDatabase(const TString& databaseName) { + auto request = MakeHolder(); + request->DatabaseName = databaseName; + + auto& entry = request->ResultSet.emplace_back(); + entry.Operation = NSchemeCache::TSchemeCacheNavigate::OpPath; + entry.Path = NKikimr::SplitPath(databaseName); + + this->Send(MakeSchemeCacheID(), new TEvTxProxySchemeCache::TEvNavigateKeySet(request.Release())); + } + + void Handle(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev) { + const auto& request = ev->Get()->Request; + + if (request->ResultSet.empty()) { + return this->Reply(Ydb::StatusIds::SCHEME_ERROR, NKikimrIssues::TIssuesIds::GENERIC_RESOLVE_ERROR); + } + + const auto& entry = request->ResultSet.front(); + + if (request->ErrorCount > 0) { + switch (entry.Status) { + case NSchemeCache::TSchemeCacheNavigate::EStatus::Ok: + break; + case NSchemeCache::TSchemeCacheNavigate::EStatus::AccessDenied: + return this->Reply(Ydb::StatusIds::UNAUTHORIZED, NKikimrIssues::TIssuesIds::ACCESS_DENIED); + case NSchemeCache::TSchemeCacheNavigate::EStatus::RootUnknown: + case NSchemeCache::TSchemeCacheNavigate::EStatus::PathErrorUnknown: + return this->Reply(Ydb::StatusIds::SCHEME_ERROR, NKikimrIssues::TIssuesIds::PATH_NOT_EXIST); + case NSchemeCache::TSchemeCacheNavigate::EStatus::LookupError: + case NSchemeCache::TSchemeCacheNavigate::EStatus::RedirectLookupError: + return this->Reply(Ydb::StatusIds::UNAVAILABLE, NKikimrIssues::TIssuesIds::RESOLVE_LOOKUP_ERROR); + default: + return this->Reply(Ydb::StatusIds::SCHEME_ERROR, NKikimrIssues::TIssuesIds::GENERIC_RESOLVE_ERROR); + } + } + + if (!this->CheckAccess(CanonizePath(entry.Path), entry.SecurityObject, NACLib::GenericList)) { + return; + } + + auto domainInfo = entry.DomainInfo; + if (!domainInfo || !domainInfo->Params.HasHive()) { + return this->Reply(Ydb::StatusIds::INTERNAL_ERROR, NKikimrIssues::TIssuesIds::GENERIC_RESOLVE_ERROR); + } + + SendRequest(domainInfo->DomainKey, domainInfo->Params.GetHive()); + } + + void SendRequest(TPathId domainKey, ui64 hiveId) { + if (!PipeClient) { + NTabletPipe::TClientConfig config; + config.RetryPolicy = {.RetryLimitCount = 3}; + PipeClient = this->RegisterWithSameMailbox(NTabletPipe::CreateClient(this->SelfId(), hiveId, config)); + } + + auto ev = std::make_unique(); + ev->Record.MutableDomainKey()->SetSchemeShard(domainKey.OwnerId); + ev->Record.MutableDomainKey()->SetPathId(domainKey.LocalPathId); + NTabletPipe::SendData(this->SelfId(), PipeClient, ev.release()); + } + + void Handle(TEvHive::TEvResponseScaleRecommendation::TPtr& ev) { + TResponse response; + + switch (ev->Get()->Record.GetStatus()) { + case NKikimrProto::OK: { + ui32 recommendedNodes = ev->Get()->Record.GetRecommendedNodes(); + response.mutable_recommended_resources()->add_computational_units()->set_count(recommendedNodes); + response.set_status(Ydb::StatusIds::SUCCESS); + break; + } + case NKikimrProto::NOTREADY: + response.set_status(Ydb::StatusIds::UNAVAILABLE); + break; + default: + response.set_status(Ydb::StatusIds::INTERNAL_ERROR); + break; + } + + return this->Reply(response); + } + + void Handle(TEvTabletPipe::TEvClientConnected::TPtr& ev) { + if (ev->Get()->Status != NKikimrProto::OK) { + DeliveryProblem(); + } + } + + void Handle(TEvTabletPipe::TEvClientDestroyed::TPtr&) { + DeliveryProblem(); + } + + void DeliveryProblem() { + this->Reply(Ydb::StatusIds::UNAVAILABLE); + } + + void PassAway() override { + NTabletPipe::CloseAndForgetClient(this->SelfId(), PipeClient); + IActor::PassAway(); + } + +private: + TActorId PipeClient; + +}; + +void DoGetScaleRecommendationRequest(std::unique_ptr p, const IFacilityProvider& f) { + f.RegisterActor(new TGetScaleRecommendationRPC(p.release())); +} + +} // namespace NKikimr::NGRpcService diff --git a/ydb/core/grpc_services/service_cms.h b/ydb/core/grpc_services/service_cms.h index 90bf661069e1..8c92cc94968c 100644 --- a/ydb/core/grpc_services/service_cms.h +++ b/ydb/core/grpc_services/service_cms.h @@ -6,6 +6,7 @@ namespace NKikimr { namespace NGRpcService { class IRequestOpCtx; +class IRequestNoOpCtx; class IFacilityProvider; void DoCreateTenantRequest(std::unique_ptr p, const IFacilityProvider& f); @@ -14,6 +15,7 @@ void DoGetTenantStatusRequest(std::unique_ptr p, const IFacilityP void DoListTenantsRequest(std::unique_ptr p, const IFacilityProvider& f); void DoRemoveTenantRequest(std::unique_ptr p, const IFacilityProvider& f); void DoDescribeTenantOptionsRequest(std::unique_ptr p, const IFacilityProvider& f); +void DoGetScaleRecommendationRequest(std::unique_ptr p, const IFacilityProvider& f); } } diff --git a/ydb/core/grpc_services/ya.make b/ydb/core/grpc_services/ya.make index 3c4583b87001..de2c3a434246 100644 --- a/ydb/core/grpc_services/ya.make +++ b/ydb/core/grpc_services/ya.make @@ -44,6 +44,7 @@ SRCS( rpc_fq_internal.cpp rpc_fq.cpp rpc_get_operation.cpp + rpc_get_scale_recommendation.cpp rpc_get_shard_locations.cpp rpc_import.cpp rpc_import_data.cpp diff --git a/ydb/core/mind/hive/domain_info.cpp b/ydb/core/mind/hive/domain_info.cpp index 5c47ee219cb7..552140ba31fa 100644 --- a/ydb/core/mind/hive/domain_info.cpp +++ b/ydb/core/mind/hive/domain_info.cpp @@ -1,8 +1,29 @@ #include "domain_info.h" +#include "hive_log.h" namespace NKikimr { namespace NHive { +namespace { + +constexpr double EPS = 0.001; +constexpr double PERCENT_EPS = 0.01; // 1% + +template +ui32 CalculateRecommendedNodes(TIt windowBegin, TIt windowEnd, size_t readyNodes, double target) { + double maxOnWindow = *std::max_element(windowBegin, windowEnd); + double ratio = maxOnWindow / target; + + double recommendedNodes = readyNodes * ratio; + if (recommendedNodes - std::floor(recommendedNodes) < EPS) { + return std::floor(recommendedNodes); + } + + return std::ceil(recommendedNodes); +} + +} // anonymous + ENodeSelectionPolicy TDomainInfo::GetNodeSelectionPolicy() const { if (ServerlessComputeResourcesMode.Empty()) { return ENodeSelectionPolicy::Default; @@ -18,5 +39,112 @@ ENodeSelectionPolicy TDomainInfo::GetNodeSelectionPolicy() const { } } +TScaleRecommenderPolicy::TScaleRecommenderPolicy(ui64 hiveId, bool dryRun) + : HiveId(hiveId) + , DryRun(dryRun) +{} + +TString TScaleRecommenderPolicy::GetLogPrefix() const { + TStringBuilder logPrefix = TStringBuilder() << "HIVE#" << HiveId << " "; + if (DryRun) { + logPrefix << "[DryRun] "; + } + return logPrefix; +} + +TTargetTrackingPolicy::TTargetTrackingPolicy(double target, const std::deque& usageHistory, ui64 hiveId, bool dryRun) + : TScaleRecommenderPolicy(hiveId, dryRun) + , TargetUsage(target) + , UsageHistory(usageHistory) +{} + +TString TTargetTrackingPolicy::GetLogPrefix() const { + return TStringBuilder() << TScaleRecommenderPolicy::GetLogPrefix() << "[TargetTracking] "; +} + +ui32 TTargetTrackingPolicy::MakeScaleRecommendation(ui32 readyNodesCount, const NKikimrConfig::THiveConfig& config) const { + ui32 recommendedNodes = readyNodesCount; + + if (UsageHistory.size() >= config.GetScaleInWindowSize()) { + auto scaleInWindowBegin = UsageHistory.end() - config.GetScaleInWindowSize(); + auto scaleInWindowEnd = UsageHistory.end(); + double usageBottomThreshold = TargetUsage - config.GetTargetTrackingCPUMargin(); + + BLOG_TRACE("[MSR] Scale in window: [" << JoinRange(", ", scaleInWindowBegin, scaleInWindowEnd) + << "], bottom threshold: " << usageBottomThreshold); + bool needScaleIn = std::all_of( + scaleInWindowBegin, + scaleInWindowEnd, + [usageBottomThreshold](double value){ return value <= usageBottomThreshold - PERCENT_EPS; } + ); + + if (needScaleIn) { + recommendedNodes = CalculateRecommendedNodes( + scaleInWindowBegin, + scaleInWindowEnd, + readyNodesCount, + TargetUsage + ); + BLOG_TRACE("[MSR] Need scale in, rounded recommended nodes: " << recommendedNodes); + } + } else { + BLOG_TRACE("[MSR] Not enough history for scale in"); + } + + if (UsageHistory.size() >= config.GetScaleOutWindowSize()) { + auto scaleOutWindowBegin = UsageHistory.end() - config.GetScaleOutWindowSize(); + auto scaleOutWindowEnd = UsageHistory.end(); + + BLOG_TRACE("[MSR] Scale out window: [" << JoinRange(", ", scaleOutWindowBegin, scaleOutWindowEnd) + << "], target: " << TargetUsage); + bool needScaleOut = std::all_of( + scaleOutWindowBegin, + scaleOutWindowEnd, + [this](double value){ return value >= TargetUsage + PERCENT_EPS; } + ); + + if (needScaleOut) { + recommendedNodes = CalculateRecommendedNodes( + scaleOutWindowBegin, + scaleOutWindowEnd, + readyNodesCount, + TargetUsage + ); + BLOG_TRACE("[MSR] Need scale out, rounded recommended nodes: " << recommendedNodes); + } + } else { + BLOG_TRACE("[MSR] Not enough history for scale out"); + } + + return std::max(recommendedNodes, 1u); +} + +void TDomainInfo::SetScaleRecommenderPolicies(const NKikimrHive::TScaleRecommenderPolicies& policies) { + using enum NKikimrHive::TScaleRecommenderPolicies_TScaleRecommenderPolicy::PolicyCase; + using enum NKikimrHive::TScaleRecommenderPolicies_TScaleRecommenderPolicy_TTargetTrackingPolicy::TargetCase; + + ScaleRecommenderPolicies.clear(); + for (const auto& policy : policies.GetPolicies()) { + switch (policy.GetPolicyCase()) { + case kTargetTrackingPolicy: { + const auto& targetTracking = policy.GetTargetTrackingPolicy(); + switch (targetTracking.GetTargetCase()) { + case kAverageCpuUtilizationPercent: { + ui32 target = targetTracking.GetAverageCpuUtilizationPercent(); + auto convertedPolicy = std::make_shared(target / 100., AvgCpuUsageHistory, HiveId); + ScaleRecommenderPolicies.push_back(convertedPolicy); + break; + } + case TARGET_NOT_SET: + break; + } + break; + } + case POLICY_NOT_SET: + break; + } + } +} + } // NHive } // NKikimr diff --git a/ydb/core/mind/hive/domain_info.h b/ydb/core/mind/hive/domain_info.h index 2d5b3608fcca..1d4c4f75b828 100644 --- a/ydb/core/mind/hive/domain_info.h +++ b/ydb/core/mind/hive/domain_info.h @@ -5,11 +5,39 @@ namespace NKikimr { namespace NHive { +struct TScaleRecommendation { + ui64 Nodes = 0; + TInstant Timestamp; +}; + enum class ENodeSelectionPolicy : ui32 { Default, PreferObjectDomain, }; +class TScaleRecommenderPolicy { +public: + TScaleRecommenderPolicy(ui64 hiveId, bool dryRun); + virtual ~TScaleRecommenderPolicy() = default; + virtual ui32 MakeScaleRecommendation(ui32 readyNodes, const NKikimrConfig::THiveConfig& config) const = 0; + + virtual TString GetLogPrefix() const; +private: + ui64 HiveId; + bool DryRun; +}; + +class TTargetTrackingPolicy : public TScaleRecommenderPolicy { +public: + TTargetTrackingPolicy(double target, const std::deque& usageHistory, ui64 hiveId = 0, bool dryRun = false); + ui32 MakeScaleRecommendation(ui32 readyNodesCount, const NKikimrConfig::THiveConfig& config) const override; + + virtual TString GetLogPrefix() const override; +private: + double TargetUsage; + const std::deque& UsageHistory; +}; + struct TDomainInfo { TString Path; TTabletId HiveId = 0; @@ -18,6 +46,12 @@ struct TDomainInfo { ui64 TabletsTotal = 0; ui64 TabletsAlive = 0; ui64 TabletsAliveInObjectDomain = 0; + + std::deque AvgCpuUsageHistory; + TMaybeFail LastScaleRecommendation; + TVector> ScaleRecommenderPolicies; + + void SetScaleRecommenderPolicies(const NKikimrHive::TScaleRecommenderPolicies& policies); ENodeSelectionPolicy GetNodeSelectionPolicy() const; }; diff --git a/ydb/core/mind/hive/hive_events.h b/ydb/core/mind/hive/hive_events.h index f42d64510656..25dee5fef2f0 100644 --- a/ydb/core/mind/hive/hive_events.h +++ b/ydb/core/mind/hive/hive_events.h @@ -33,6 +33,7 @@ struct TEvPrivate { EvStorageBalancerOut, EvDeleteNode, EvCanMoveTablets, + EvRefreshScaleRecommendation, EvEnd }; @@ -120,6 +121,8 @@ struct TEvPrivate { }; struct TEvCanMoveTablets : TEventLocal {}; + + struct TEvRefreshScaleRecommendation : TEventLocal {}; }; } // NHive diff --git a/ydb/core/mind/hive/hive_impl.cpp b/ydb/core/mind/hive/hive_impl.cpp index 55dada0352af..6a750967168e 100644 --- a/ydb/core/mind/hive/hive_impl.cpp +++ b/ydb/core/mind/hive/hive_impl.cpp @@ -579,6 +579,8 @@ void THive::Handle(TEvPrivate::TEvBootTablets::TPtr&) { } SendToRootHivePipe(request.Release()); } + UpdateCounterNodesConnected(+1); // self node + Schedule(GetScaleRecommendationRefreshFrequency(), new TEvPrivate::TEvRefreshScaleRecommendation()); ProcessPendingOperations(); } @@ -3030,6 +3032,9 @@ void THive::ProcessEvent(std::unique_ptr event) { hFunc(TEvHive::TEvUpdateDomain, Handle); hFunc(TEvPrivate::TEvDeleteNode, Handle); hFunc(TEvHive::TEvRequestTabletDistribution, Handle); + hFunc(TEvHive::TEvRequestScaleRecommendation, Handle); + hFunc(TEvPrivate::TEvRefreshScaleRecommendation, Handle); + hFunc(TEvHive::TEvConfigureScaleRecommender, Handle); } } @@ -3132,6 +3137,9 @@ STFUNC(THive::StateWork) { fFunc(TEvPrivate::TEvProcessStorageBalancer::EventType, EnqueueIncomingEvent); fFunc(TEvPrivate::TEvDeleteNode::EventType, EnqueueIncomingEvent); fFunc(TEvHive::TEvRequestTabletDistribution::EventType, EnqueueIncomingEvent); + fFunc(TEvHive::TEvRequestScaleRecommendation::EventType, EnqueueIncomingEvent); + fFunc(TEvPrivate::TEvRefreshScaleRecommendation::EventType, EnqueueIncomingEvent); + fFunc(TEvHive::TEvConfigureScaleRecommender::EventType, EnqueueIncomingEvent); hFunc(TEvPrivate::TEvProcessIncomingEvent, Handle); default: if (!HandleDefaultEvents(ev, SelfId())) { @@ -3432,6 +3440,130 @@ void THive::Handle(TEvHive::TEvRequestTabletDistribution::TPtr& ev) { Send(ev->Sender, response.release()); } +void THive::MakeScaleRecommendation() { + BLOG_D("[MSR] Started"); + + if (AreWeRootHive()) { + return; + } + + auto subdomainKey = GetMySubDomainKey(); + auto it = Domains.find(subdomainKey); + if (it == Domains.end()) { + BLOG_ERROR("[MSR] Can't find domain " << subdomainKey); + Schedule(GetScaleRecommendationRefreshFrequency(), new TEvPrivate::TEvRefreshScaleRecommendation()); + return; + } + auto& domain = it->second; + + if (domain.ScaleRecommenderPolicies.empty() && CurrentConfig.GetDryRunTargetTrackingCPU() == 0) { + BLOG_TRACE("[MSR] No scaling policies configured, rescheduled"); + Schedule(GetScaleRecommendationRefreshFrequency(), new TEvPrivate::TEvRefreshScaleRecommendation()); + return; + } + + double cpuUsageSum = 0; + ui32 readyNodesCount = 0; + for (auto& [id, node] : Nodes) { + if (!node.IsAlive()) { + BLOG_TRACE("[MSR] Skip node " << id << ", not alive"); + continue; + } + + if (!node.AveragedNodeTotalCpuUsage.IsValueReady()) { + BLOG_TRACE("[MSR] Skip node " << id << ", no CPU usage value"); + continue; + } + + if (node.GetServicedDomain() != subdomainKey) { + BLOG_TRACE("[MSR] Skip node " << id << ", serviced domain doesn't match"); + continue; + } + + double avgCpuUsage = node.AveragedNodeTotalCpuUsage.GetValue(); + BLOG_TRACE("[MSR] Node " << id << " is ready, avg CPU usage: " << avgCpuUsage); + ++readyNodesCount; + + cpuUsageSum += avgCpuUsage; + node.AveragedNodeTotalCpuUsage.Clear(); + } + + double avgCpuUsage = readyNodesCount != 0 ? cpuUsageSum / readyNodesCount : 0; + BLOG_TRACE("[MSR] Total avg CPU usage: " << avgCpuUsage << ", ready nodes: " << readyNodesCount); + TabletCounters->Simple()[NHive::COUNTER_AVG_CPU_UTILIZATION].Set(avgCpuUsage * 100); + + auto& avgCpuUsageHistory = domain.AvgCpuUsageHistory; + avgCpuUsageHistory.push_back(avgCpuUsage); + size_t maxHistorySize = std::max(CurrentConfig.GetScaleInWindowSize(), CurrentConfig.GetScaleOutWindowSize()); + while (avgCpuUsageHistory.size() > maxHistorySize) { + avgCpuUsageHistory.pop_front(); + } + BLOG_TRACE("[MSR] Avg CPU usage history: " << '[' << JoinSeq(", ", avgCpuUsageHistory) << ']'); + + if (!domain.ScaleRecommenderPolicies.empty()) { + ui32 recommendedNodes = 1; + for (auto& policy : domain.ScaleRecommenderPolicies) { + recommendedNodes = std::max(recommendedNodes, policy->MakeScaleRecommendation(readyNodesCount, CurrentConfig)); + } + + domain.LastScaleRecommendation = TScaleRecommendation{ + .Nodes = recommendedNodes, + .Timestamp = TActivationContext::Now() + }; + TabletCounters->Simple()[NHive::COUNTER_NODES_RECOMMENDED].Set(recommendedNodes); + BLOG_TRACE("[MSR] Recommended nodes: " << recommendedNodes << ", current nodes: " << readyNodesCount); + } + + if (CurrentConfig.GetDryRunTargetTrackingCPU() != 0) { + ui32 dryRunRecommendedNodes = 1; + TTargetTrackingPolicy dryRunPolicy(CurrentConfig.GetDryRunTargetTrackingCPU(), avgCpuUsageHistory, TabletID(), true); + dryRunRecommendedNodes = std::max(dryRunRecommendedNodes, dryRunPolicy.MakeScaleRecommendation(readyNodesCount, CurrentConfig)); + TabletCounters->Simple()[NHive::COUNTER_NODES_RECOMMENDED_DRY_RUN].Set(dryRunRecommendedNodes); + BLOG_TRACE("[MSR] Dry run recommended nodes: " << dryRunRecommendedNodes << ", current nodes: " << readyNodesCount); + } + + Schedule(GetScaleRecommendationRefreshFrequency(), new TEvPrivate::TEvRefreshScaleRecommendation()); +} + +void THive::Handle(TEvPrivate::TEvRefreshScaleRecommendation::TPtr&) { + MakeScaleRecommendation(); +} + +void THive::Handle(TEvHive::TEvRequestScaleRecommendation::TPtr& ev) { + BLOG_D("Handle TEvHive::TEvRequestScaleRecommendation(" << ev->Get()->Record.ShortDebugString() << ")"); + auto response = std::make_unique(); + + const auto& record = ev->Get()->Record; + if (!record.HasDomainKey()) { + response->Record.SetStatus(NKikimrProto::ERROR); + Send(ev->Sender, response.release()); + return; + } + + const TSubDomainKey domainKey(ev->Get()->Record.GetDomainKey()); + if (!Domains.contains(domainKey)) { + response->Record.SetStatus(NKikimrProto::ERROR); + Send(ev->Sender, response.release()); + return; + } + + const TDomainInfo& domainInfo = Domains[domainKey]; + if (domainInfo.LastScaleRecommendation.Empty()) { + response->Record.SetStatus(NKikimrProto::NOTREADY); + Send(ev->Sender, response.release()); + return; + } + + response->Record.SetStatus(NKikimrProto::OK); + response->Record.SetRecommendedNodes(domainInfo.LastScaleRecommendation->Nodes); + Send(ev->Sender, response.release()); +} + +void THive::Handle(TEvHive::TEvConfigureScaleRecommender::TPtr& ev) { + BLOG_D("Handle TEvHive::TEvConfigureScaleRecommender(" << ev->Get()->Record.ShortDebugString() << ")"); + Execute(CreateConfigureScaleRecommender(ev)); +} + TVector THive::GetNodesForWhiteboardBroadcast(size_t maxNodesToReturn) { TVector nodes; TNodeId selfNodeId = SelfId().NodeId(); diff --git a/ydb/core/mind/hive/hive_impl.h b/ydb/core/mind/hive/hive_impl.h index 4cadc764ac56..14d0c986f9a7 100644 --- a/ydb/core/mind/hive/hive_impl.h +++ b/ydb/core/mind/hive/hive_impl.h @@ -302,6 +302,7 @@ class THive : public TActor, public TTabletExecutedFlat, public THiveShar ITransaction* CreateUpdateTabletsObject(TEvHive::TEvUpdateTabletsObject::TPtr event); ITransaction* CreateUpdateDomain(TSubDomainKey subdomainKey, TEvHive::TEvUpdateDomain::TPtr event = {}); ITransaction* CreateDeleteNode(TNodeId nodeId); + ITransaction* CreateConfigureScaleRecommender(TEvHive::TEvConfigureScaleRecommender::TPtr event); public: TDomainsView DomainsView; @@ -575,6 +576,9 @@ class THive : public TActor, public TTabletExecutedFlat, public THiveShar void Handle(TEvHive::TEvUpdateDomain::TPtr& ev); void Handle(TEvPrivate::TEvDeleteNode::TPtr& ev); void Handle(TEvHive::TEvRequestTabletDistribution::TPtr& ev); + void Handle(TEvHive::TEvRequestScaleRecommendation::TPtr& ev); + void Handle(TEvPrivate::TEvRefreshScaleRecommendation::TPtr& ev); + void Handle(TEvHive::TEvConfigureScaleRecommender::TPtr& ev); protected: void RestartPipeTx(ui64 tabletId); @@ -939,6 +943,10 @@ TTabletInfo* FindTabletEvenInDeleting(TTabletId tabletId, TFollowerId followerId return TDuration::MilliSeconds(CurrentConfig.GetStorageInfoRefreshFrequency()); } + TDuration GetScaleRecommendationRefreshFrequency() const { + return TDuration::MilliSeconds(CurrentConfig.GetScaleRecommendationRefreshFrequency()); + } + double GetMinStorageScatterToBalance() const { return CurrentConfig.GetMinStorageScatterToBalance(); } @@ -1026,6 +1034,10 @@ TTabletInfo* FindTabletEvenInDeleting(TTabletId tabletId, TFollowerId followerId void ResolveDomain(TSubDomainKey domain); TString GetDomainName(TSubDomainKey domain); TSubDomainKey GetMySubDomainKey() const; + + template + static ui32 CalculateRecommendedNodes(TIt windowBegin, TIt windowEnd, size_t readyNodes, double target); + void MakeScaleRecommendation(); }; } // NHive diff --git a/ydb/core/mind/hive/hive_schema.h b/ydb/core/mind/hive/hive_schema.h index 014532e805ee..c47897fc30fb 100644 --- a/ydb/core/mind/hive/hive_schema.h +++ b/ydb/core/mind/hive/hive_schema.h @@ -270,9 +270,11 @@ struct Schema : NIceDb::Schema { struct Primary : Column<4, NScheme::NTypeIds::Bool> {}; struct HiveId : Column<5, NScheme::NTypeIds::Uint64> {}; struct ServerlessComputeResourcesMode : Column<6, NScheme::NTypeIds::Uint32> { using Type = NKikimrSubDomains::EServerlessComputeResourcesMode; }; + struct ScaleRecommenderPolicies : Column<7, NScheme::NTypeIds::String> { using Type = NKikimrHive::TScaleRecommenderPolicies; }; using TKey = TableKey; - using TColumns = TableColumns; + using TColumns = TableColumns; }; struct BlockedOwner : Table<18> { diff --git a/ydb/core/mind/hive/hive_ut.cpp b/ydb/core/mind/hive/hive_ut.cpp index 4bfa4031dee3..d2efc3567c3d 100644 --- a/ydb/core/mind/hive/hive_ut.cpp +++ b/ydb/core/mind/hive/hive_ut.cpp @@ -265,6 +265,8 @@ namespace { app.HiveConfig.SetMinCounterScatterToBalance(0.02); app.HiveConfig.SetMinScatterToBalance(0.5); app.HiveConfig.SetObjectImbalanceToBalance(0.02); + app.HiveConfig.SetScaleInWindowSize(1); + app.HiveConfig.SetScaleOutWindowSize(1); if (appConfigSetup) { appConfigSetup(app); } @@ -7054,4 +7056,152 @@ Y_UNIT_TEST_SUITE(TStorageBalanceTest) { UNIT_ASSERT_LE(bsc.GetOccupancyStDev("def1"), 0.1); } } + +Y_UNIT_TEST_SUITE(TScaleRecommenderTest) { + using namespace NTestSuiteTHiveTest; + + void ConfigureScaleRecommender(TTestBasicRuntime& runtime, ui64 hiveId, TSubDomainKey subdomainKey, + ui32 targetCPUUtilization) + { + const auto sender = runtime.AllocateEdgeActor(); + + auto request = std::make_unique(); + request->Record.MutableDomainKey()->SetSchemeShard(subdomainKey.GetSchemeShard()); + request->Record.MutableDomainKey()->SetPathId(subdomainKey.GetPathId()); + auto* policy = request->Record.MutablePolicies()->AddPolicies()->MutableTargetTrackingPolicy(); + policy->SetAverageCpuUtilizationPercent(targetCPUUtilization); + + runtime.SendToPipe(hiveId, sender, request.release()); + + TAutoPtr handle; + const auto* response = runtime.GrabEdgeEventRethrow(handle); + UNIT_ASSERT_VALUES_EQUAL(response->Record.GetStatus(), NKikimrProto::OK); + } + + void AssertScaleRecommencation(TTestBasicRuntime& runtime, ui64 hiveId, TSubDomainKey subdomainKey, + NKikimrProto::EReplyStatus expectedStatus, ui32 expectedNodes = 0) + { + const auto sender = runtime.AllocateEdgeActor(); + runtime.SendToPipe(hiveId, sender, new TEvHive::TEvRequestScaleRecommendation(subdomainKey)); + + TAutoPtr handle; + const auto* response = runtime.GrabEdgeEventRethrow(handle); + UNIT_ASSERT_VALUES_EQUAL(response->Record.GetStatus(), expectedStatus); + if (expectedNodes) { + UNIT_ASSERT_VALUES_EQUAL(response->Record.GetRecommendedNodes(), expectedNodes); + } + } + + void RefreshScaleRecommendation(TTestBasicRuntime& runtime, ui64 hiveId) { + const auto sender = runtime.AllocateEdgeActor(); + runtime.SendToPipe(hiveId, sender, new NHive::TEvPrivate::TEvRefreshScaleRecommendation()); + + TDispatchOptions options; + options.FinalEvents.emplace_back(NHive::TEvPrivate::EvRefreshScaleRecommendation); + runtime.DispatchEvents(options); + } + + void SendUsage(TTestBasicRuntime& runtime, ui64 hiveId, ui64 nodeIdx, double cpuUsage) { + const auto sender = runtime.AllocateEdgeActor(nodeIdx); + + auto ev = std::make_unique(); + ev->Record.SetTotalNodeCpuUsage(cpuUsage); + runtime.SendToPipe(hiveId, sender, ev.release(), nodeIdx, GetPipeConfigWithRetries()); + + TAutoPtr handle; + runtime.GrabEdgeEvent(handle); + } + + constexpr double LOW_CPU_USAGE = 0.2; + constexpr double HIGH_CPU_USAGE = 0.95; + + Y_UNIT_TEST(BasicTest) { + // Setup test runtime + TTestBasicRuntime runtime(1, false); + Setup(runtime, true); + + // Setup hive + const ui64 hiveTablet = MakeDefaultHiveID(); + const ui64 testerTablet = MakeTabletID(false, 1); + const TActorId hiveActor = CreateTestBootstrapper(runtime, CreateTestTabletInfo(hiveTablet, TTabletTypes::Hive), &CreateDefaultHive); + runtime.EnableScheduleForActor(hiveActor); + CreateTestBootstrapper(runtime, CreateTestTabletInfo(TTestTxConfig::SchemeShard, TTabletTypes::SchemeShard), &CreateFlatTxSchemeShard); + MakeSureTabletIsUp(runtime, hiveTablet, 0); // root hive good + MakeSureTabletIsUp(runtime, TTestTxConfig::SchemeShard, 0); // root ss good + + TActorId sender = runtime.AllocateEdgeActor(0); + InitSchemeRoot(runtime, sender); + + TSubDomainKey subdomainKey; + + // Create subdomain + do { + auto x = MakeHolder(); + auto* tran = x->Record.AddTransaction(); + tran->SetWorkingDir("/dc-1"); + tran->SetOperationType(NKikimrSchemeOp::ESchemeOpCreateSubDomain); + auto* subd = tran->MutableSubDomain(); + subd->SetName("tenant1"); + runtime.SendToPipe(TTestTxConfig::SchemeShard, sender, x.Release()); + TAutoPtr handle; + auto reply = runtime.GrabEdgeEventRethrow(handle, TDuration::MilliSeconds(100)); + if (reply) { + subdomainKey = TSubDomainKey(reply->Record.GetSchemeshardId(), reply->Record.GetPathId()); + UNIT_ASSERT_VALUES_EQUAL(reply->Record.GetStatus(), NKikimrScheme::EStatus::StatusAccepted); + break; + } + } while (true); + + THolder createHive = MakeHolder(testerTablet, 0, TTabletTypes::Hive, BINDED_CHANNELS); + createHive->Record.AddAllowedDomains(); + createHive->Record.MutableAllowedDomains(0)->SetSchemeShard(subdomainKey.first); + createHive->Record.MutableAllowedDomains(0)->SetPathId(subdomainKey.second); + ui64 subHiveTablet = SendCreateTestTablet(runtime, hiveTablet, testerTablet, std::move(createHive), 0, false); + + TTestActorRuntime::TEventObserver prevObserverFunc; + prevObserverFunc = runtime.SetObserverFunc([&](TAutoPtr& event) { + if (event->GetTypeRewrite() == NSchemeShard::TEvSchemeShard::EvDescribeSchemeResult) { + event->Get()->MutableRecord()-> + MutablePathDescription()->MutableDomainDescription()->MutableProcessingParams()->SetHive(subHiveTablet); + } + return prevObserverFunc(event); + }); + + SendKillLocal(runtime, 0); + CreateLocalForTenant(runtime, 0, "/dc-1/tenant1"); + MakeSureTabletIsUp(runtime, subHiveTablet, 0); // sub hive good + + THolder createTablet = MakeHolder(testerTablet, 1, TTabletTypes::Dummy, BINDED_CHANNELS); + createTablet->Record.AddAllowedDomains(); + createTablet->Record.MutableAllowedDomains(0)->SetSchemeShard(subdomainKey.first); + createTablet->Record.MutableAllowedDomains(0)->SetPathId(subdomainKey.second); + ui64 tabletId = SendCreateTestTablet(runtime, subHiveTablet, testerTablet, std::move(createTablet), 0, true); + MakeSureTabletIsUp(runtime, tabletId, 0); // dummy from sub hive also good + + // Configure target CPU usage + ConfigureScaleRecommender(runtime, subHiveTablet, subdomainKey, 60); + + // No data yet + AssertScaleRecommencation(runtime, subHiveTablet, subdomainKey, NKikimrProto::NOTREADY); + + // Set low CPU usage on Node + SendUsage(runtime, subHiveTablet, 0, LOW_CPU_USAGE); + + // Refresh to calculate new scale recommendation + RefreshScaleRecommendation(runtime, subHiveTablet); + + // Check scale recommendation for low CPU usage + AssertScaleRecommencation(runtime, subHiveTablet, subdomainKey, NKikimrProto::OK, 1); + + // Set high CPU usage on Node + SendUsage(runtime, subHiveTablet, 0, HIGH_CPU_USAGE); + + // Refresh to calculate new scale recommendation + RefreshScaleRecommendation(runtime, subHiveTablet); + + // Check scale recommendation for high CPU usage + AssertScaleRecommencation(runtime, subHiveTablet, subdomainKey, NKikimrProto::OK, 2); + } +} + } diff --git a/ydb/core/mind/hive/monitoring.cpp b/ydb/core/mind/hive/monitoring.cpp index f442726a910f..4a694952341c 100644 --- a/ydb/core/mind/hive/monitoring.cpp +++ b/ydb/core/mind/hive/monitoring.cpp @@ -837,6 +837,11 @@ class TTxMonEvent_Settings : public TTransactionBase, public TLoggedMonTr UpdateConfig(db, "MinGroupUsageToBalance", configUpdates); UpdateConfig(db, "StorageBalancerInflight", configUpdates); UpdateConfig(db, "LessSystemTabletsMoves", configUpdates); + UpdateConfig(db, "ScaleRecommendationRefreshFrequency", configUpdates); + UpdateConfig(db, "ScaleOutWindowSize", configUpdates); + UpdateConfig(db, "ScaleInWindowSize", configUpdates); + UpdateConfig(db, "TargetTrackingCPUMargin", configUpdates); + UpdateConfig(db, "DryRunTargetTrackingCPU", configUpdates); if (params.contains("BalancerIgnoreTabletTypes")) { auto value = params.Get("BalancerIgnoreTabletTypes"); @@ -1185,6 +1190,11 @@ class TTxMonEvent_Settings : public TTransactionBase, public TLoggedMonTr ShowConfig(out, "StorageBalancerInflight"); ShowConfig(out, "LessSystemTabletsMoves"); ShowConfigForBalancerIgnoreTabletTypes(out); + ShowConfig(out, "ScaleRecommendationRefreshFrequency"); + ShowConfig(out, "ScaleOutWindowSize"); + ShowConfig(out, "ScaleInWindowSize"); + ShowConfig(out, "TargetTrackingCPUMargin"); + ShowConfig(out, "DryRunTargetTrackingCPU"); out << "
"; out << "
"; diff --git a/ydb/core/mind/hive/node_info.cpp b/ydb/core/mind/hive/node_info.cpp index 341e309b94ef..de446e908a57 100644 --- a/ydb/core/mind/hive/node_info.cpp +++ b/ydb/core/mind/hive/node_info.cpp @@ -490,6 +490,9 @@ void TNodeInfo::UpdateResourceTotalUsage(const NKikimrHive::TEvTabletMetrics& me AveragedNodeTotalUsage.Push(metrics.GetTotalNodeUsage()); NodeTotalUsage = AveragedNodeTotalUsage.GetValue(); } + if (metrics.HasTotalNodeCpuUsage()) { + AveragedNodeTotalCpuUsage.Push(metrics.GetTotalNodeCpuUsage()); + } } TResourceRawValues TNodeInfo::GetResourceCurrentValues() const { diff --git a/ydb/core/mind/hive/node_info.h b/ydb/core/mind/hive/node_info.h index 957ff626abf6..4c4483317b0e 100644 --- a/ydb/core/mind/hive/node_info.h +++ b/ydb/core/mind/hive/node_info.h @@ -76,6 +76,7 @@ struct TNodeInfo { NMetrics::TAverageValue AveragedResourceTotalValues; double NodeTotalUsage = 0; NMetrics::TFastRiseAverageValue AveragedNodeTotalUsage; + NMetrics::TAverageValue AveragedNodeTotalCpuUsage; TResourceRawValues ResourceMaximumValues; TInstant StartTime; TNodeLocation Location; diff --git a/ydb/core/mind/hive/scale_recommender_policy_ut.cpp b/ydb/core/mind/hive/scale_recommender_policy_ut.cpp new file mode 100644 index 000000000000..309dbf73b8ad --- /dev/null +++ b/ydb/core/mind/hive/scale_recommender_policy_ut.cpp @@ -0,0 +1,263 @@ + +#include "domain_info.h" + +#include + +#include + +using namespace NKikimr; +using namespace NHive; + +Y_UNIT_TEST_SUITE(TargetTrackingScaleRecommenderPolicy) { + Y_UNIT_TEST(ScaleOut) { + TActorSystemStub stub; + NKikimrConfig::THiveConfig config; + config.SetScaleOutWindowSize(3); + + std::deque history; + + history = {}; + UNIT_ASSERT_VALUES_EQUAL(TTargetTrackingPolicy(0.6, history).MakeScaleRecommendation(3, config), 3); + + history = { 0.8 }; + UNIT_ASSERT_VALUES_EQUAL(TTargetTrackingPolicy(0.6, history).MakeScaleRecommendation(3, config), 3); + + history = { 0.8, 0.8 }; + UNIT_ASSERT_VALUES_EQUAL(TTargetTrackingPolicy(0.6, history).MakeScaleRecommendation(3, config), 3); + + history = { 0.8, 0.8, 0.8 }; + UNIT_ASSERT_VALUES_EQUAL(TTargetTrackingPolicy(0.6, history).MakeScaleRecommendation(3, config), 4); + } + + Y_UNIT_TEST(ScaleIn) { + TActorSystemStub stub; + NKikimrConfig::THiveConfig config; + config.SetScaleInWindowSize(3); + + std::deque history; + + history = {}; + UNIT_ASSERT_VALUES_EQUAL(TTargetTrackingPolicy(0.6, history).MakeScaleRecommendation(3, config), 3); + + history = { 0.3 }; + UNIT_ASSERT_VALUES_EQUAL(TTargetTrackingPolicy(0.6, history).MakeScaleRecommendation(3, config), 3); + + history = { 0.3, 0.3 }; + UNIT_ASSERT_VALUES_EQUAL(TTargetTrackingPolicy(0.6, history).MakeScaleRecommendation(3, config), 3); + + history = { 0.3, 0.3, 0.3 }; + UNIT_ASSERT_VALUES_EQUAL(TTargetTrackingPolicy(0.6, history).MakeScaleRecommendation(3, config), 2); + } + + Y_UNIT_TEST(BigNumbersScaleOut) { + TActorSystemStub stub; + NKikimrConfig::THiveConfig config; + config.SetScaleOutWindowSize(3); + + std::deque history; + + history = {}; + UNIT_ASSERT_VALUES_EQUAL(TTargetTrackingPolicy(0.6, history).MakeScaleRecommendation(1000, config), 1000); + + history = { 0.8 }; + UNIT_ASSERT_VALUES_EQUAL(TTargetTrackingPolicy(0.6, history).MakeScaleRecommendation(1000, config), 1000); + + history = { 0.8, 0.8 }; + UNIT_ASSERT_VALUES_EQUAL(TTargetTrackingPolicy(0.6, history).MakeScaleRecommendation(1000, config), 1000); + + history = { 0.8, 0.8, 0.8 }; + UNIT_ASSERT_VALUES_EQUAL(TTargetTrackingPolicy(0.6, history).MakeScaleRecommendation(1000, config), 1334); + } + + Y_UNIT_TEST(BigNumbersScaleIn) { + TActorSystemStub stub; + NKikimrConfig::THiveConfig config; + config.SetScaleInWindowSize(3); + + std::deque history; + + history = {}; + UNIT_ASSERT_VALUES_EQUAL(TTargetTrackingPolicy(0.6, history).MakeScaleRecommendation(1000, config), 1000); + + history = { 0.3 }; + UNIT_ASSERT_VALUES_EQUAL(TTargetTrackingPolicy(0.6, history).MakeScaleRecommendation(1000, config), 1000); + + history = { 0.3, 0.3 }; + UNIT_ASSERT_VALUES_EQUAL(TTargetTrackingPolicy(0.6, history).MakeScaleRecommendation(1000, config), 1000); + + history = { 0.3, 0.3, 0.3 }; + UNIT_ASSERT_VALUES_EQUAL(TTargetTrackingPolicy(0.6, history).MakeScaleRecommendation(1000, config), 500); + } + + Y_UNIT_TEST(SpikeResistance) { + TActorSystemStub stub; + NKikimrConfig::THiveConfig config; + config.SetScaleOutWindowSize(3); + + std::deque history; + + history = {}; + UNIT_ASSERT_VALUES_EQUAL(TTargetTrackingPolicy(0.6, history).MakeScaleRecommendation(3, config), 3); + + history = { 0.3, }; + UNIT_ASSERT_VALUES_EQUAL(TTargetTrackingPolicy(0.6, history).MakeScaleRecommendation(3, config), 3); + + history = { 0.3, 0.9 }; + UNIT_ASSERT_VALUES_EQUAL(TTargetTrackingPolicy(0.6, history).MakeScaleRecommendation(3, config), 3); + + history = { 0.3, 0.9, 0.3 }; + UNIT_ASSERT_VALUES_EQUAL(TTargetTrackingPolicy(0.6, history).MakeScaleRecommendation(3, config), 3); + } + + Y_UNIT_TEST(NearTarget) { + TActorSystemStub stub; + NKikimrConfig::THiveConfig config; + config.SetScaleOutWindowSize(3); + config.SetScaleInWindowSize(3); + + std::deque history; + + history = {}; + UNIT_ASSERT_VALUES_EQUAL(TTargetTrackingPolicy(0.6, history).MakeScaleRecommendation(3, config), 3); + + history = { 0.55, }; + UNIT_ASSERT_VALUES_EQUAL(TTargetTrackingPolicy(0.6, history).MakeScaleRecommendation(3, config), 3); + + history = { 0.55, 0.55 }; + UNIT_ASSERT_VALUES_EQUAL(TTargetTrackingPolicy(0.6, history).MakeScaleRecommendation(3, config), 3); + + history = { 0.55, 0.55, 0.55 }; + UNIT_ASSERT_VALUES_EQUAL(TTargetTrackingPolicy(0.6, history).MakeScaleRecommendation(3, config), 3); + } + + Y_UNIT_TEST(AtTarget) { + TActorSystemStub stub; + NKikimrConfig::THiveConfig config; + config.SetScaleOutWindowSize(3); + config.SetScaleInWindowSize(3); + + std::deque history; + + history = {}; + UNIT_ASSERT_VALUES_EQUAL(TTargetTrackingPolicy(0.6, history).MakeScaleRecommendation(3, config), 3); + + history = { 0.6, }; + UNIT_ASSERT_VALUES_EQUAL(TTargetTrackingPolicy(0.6, history).MakeScaleRecommendation(3, config), 3); + + history = { 0.6, 0.6 }; + UNIT_ASSERT_VALUES_EQUAL(TTargetTrackingPolicy(0.6, history).MakeScaleRecommendation(3, config), 3); + + history = { 0.6, 0.6, 0.6 }; + UNIT_ASSERT_VALUES_EQUAL(TTargetTrackingPolicy(0.6, history).MakeScaleRecommendation(3, config), 3); + } + + void TestFluctuations(ui32 initialNodes) { + NKikimrConfig::THiveConfig config; + config.SetScaleOutWindowSize(1); + config.SetScaleInWindowSize(1); + + for (double avgInitialLoad = 0; avgInitialLoad < 1.1; avgInitialLoad += 0.1) { + const double totalLoad = avgInitialLoad * initialNodes; + + for (double avgTargetLoad = 0.1; avgTargetLoad < 1.0; avgTargetLoad += 0.1) { + ui32 currentNodes = initialNodes; + std::set uniqueCurrentNodes = { currentNodes }; + std::vector currentNodesHistory = { currentNodes }; + + std::deque history; + for (size_t i = 0; i < 10; ++i) { + history.push_back(totalLoad / currentNodes); + currentNodes = TTargetTrackingPolicy(avgTargetLoad, history).MakeScaleRecommendation(currentNodes, config); + uniqueCurrentNodes.insert(currentNodes); + currentNodesHistory.push_back(currentNodes); + } + + UNIT_ASSERT_C(uniqueCurrentNodes.size() <= 2, + TStringBuilder() << "Fluctuations detected: target=" << avgTargetLoad + << ", currentNodesHistory=" << "[" << JoinSeq(',', currentNodesHistory) << "]" + << ", history=" << "[" << JoinSeq(',', history) << "]"); + } + } + } + + Y_UNIT_TEST(Fluctuations) { + TActorSystemStub stub; + + TestFluctuations(1); + TestFluctuations(2); + TestFluctuations(3); + TestFluctuations(4); + TestFluctuations(5); + TestFluctuations(6); + TestFluctuations(7); + TestFluctuations(8); + TestFluctuations(9); + TestFluctuations(10); + } + + Y_UNIT_TEST(FluctuationsBigNumbers) { + TActorSystemStub stub; + + TestFluctuations(1001); + TestFluctuations(1002); + TestFluctuations(1003); + TestFluctuations(1004); + TestFluctuations(1005); + TestFluctuations(1006); + TestFluctuations(1007); + TestFluctuations(1008); + TestFluctuations(1009); + TestFluctuations(1010); + + TestFluctuations(1000); + TestFluctuations(2000); + TestFluctuations(3000); + TestFluctuations(4000); + TestFluctuations(5000); + TestFluctuations(6000); + TestFluctuations(7000); + TestFluctuations(8000); + TestFluctuations(9000); + TestFluctuations(10000); + } + + Y_UNIT_TEST(ScaleInToMaxSeen) { + TActorSystemStub stub; + NKikimrConfig::THiveConfig config; + config.SetScaleInWindowSize(3); + + std::deque history; + + history = {}; + UNIT_ASSERT_VALUES_EQUAL(TTargetTrackingPolicy(0.6, history).MakeScaleRecommendation(3, config), 3); + + history = { 0.3, }; + UNIT_ASSERT_VALUES_EQUAL(TTargetTrackingPolicy(0.6, history).MakeScaleRecommendation(3, config), 3); + + history = { 0.3, 0.1 }; + UNIT_ASSERT_VALUES_EQUAL(TTargetTrackingPolicy(0.6, history).MakeScaleRecommendation(3, config), 3); + + history = { 0.3, 0.1, 0.1 }; + UNIT_ASSERT_VALUES_EQUAL(TTargetTrackingPolicy(0.6, history).MakeScaleRecommendation(3, config), 2); + } + + Y_UNIT_TEST(Idle) { + TActorSystemStub stub; + NKikimrConfig::THiveConfig config; + config.SetScaleInWindowSize(3); + + std::deque history; + + history = {}; + UNIT_ASSERT_VALUES_EQUAL(TTargetTrackingPolicy(0.6, history).MakeScaleRecommendation(3, config), 3); + + history = { 0.01, }; + UNIT_ASSERT_VALUES_EQUAL(TTargetTrackingPolicy(0.6, history).MakeScaleRecommendation(3, config), 3); + + history = { 0.01, 0.01 }; + UNIT_ASSERT_VALUES_EQUAL(TTargetTrackingPolicy(0.6, history).MakeScaleRecommendation(3, config), 3); + + history = { 0.01, 0.01, 0.01 }; + UNIT_ASSERT_VALUES_EQUAL(TTargetTrackingPolicy(0.6, history).MakeScaleRecommendation(3, config), 1); + } +} diff --git a/ydb/core/mind/hive/tx__configure_scale_recommender.cpp b/ydb/core/mind/hive/tx__configure_scale_recommender.cpp new file mode 100644 index 000000000000..c5169d5c74fa --- /dev/null +++ b/ydb/core/mind/hive/tx__configure_scale_recommender.cpp @@ -0,0 +1,60 @@ +#include "hive_impl.h" +#include "hive_log.h" + +namespace NKikimr::NHive { + +class TTxConfigureScaleRecommender : public TTransactionBase { + const TEvHive::TEvConfigureScaleRecommender::TPtr Request; + TSideEffects SideEffects; + +public: + TTxConfigureScaleRecommender(TEvHive::TEvConfigureScaleRecommender::TPtr request, THive* hive) + : TBase(hive) + , Request(std::move(request)) + {} + + TTxType GetTxType() const override { return NHive::TXTYPE_CONFIGURE_SCALE_RECOMMENDER; } + + bool Execute(TTransactionContext& txc, const TActorContext&) override { + BLOG_D("THive::TTxConfigureScaleRecommender::Execute"); + SideEffects.Reset(Self->SelfId()); + + auto response = MakeHolder(); + + const auto& record = Request->Get()->Record; + if (!record.HasDomainKey()) { + response->Record.SetStatus(NKikimrProto::ERROR); + SideEffects.Send(Request->Sender, response.Release(), 0, Request->Cookie); + return true; + } + + TSubDomainKey domainKey(record.GetDomainKey()); + TDomainInfo* domain = Self->FindDomain(domainKey); + if (domain == nullptr) { + response->Record.SetStatus(NKikimrProto::ERROR); + SideEffects.Send(Request->Sender, response.Release(), 0, Request->Cookie); + return true; + } + + NIceDb::TNiceDb db(txc.DB); + db.Table() + .Key(domainKey.first, domainKey.second) + .Update(record.policies()); + domain->SetScaleRecommenderPolicies(record.policies()); + + response->Record.SetStatus(NKikimrProto::OK); + SideEffects.Send(Request->Sender, response.Release(), 0, Request->Cookie); + return true; + } + + void Complete(const TActorContext& ctx) override { + BLOG_D("THive::TTxConfigureScaleRecommender::Complete"); + SideEffects.Complete(ctx); + } +}; + +ITransaction* THive::CreateConfigureScaleRecommender(TEvHive::TEvConfigureScaleRecommender::TPtr event) { + return new TTxConfigureScaleRecommender(std::move(event), this); +}; + +} // namespace NKikimr::NHive diff --git a/ydb/core/mind/hive/tx__load_everything.cpp b/ydb/core/mind/hive/tx__load_everything.cpp index 770dea99cad3..03c9d086131d 100644 --- a/ydb/core/mind/hive/tx__load_everything.cpp +++ b/ydb/core/mind/hive/tx__load_everything.cpp @@ -279,7 +279,10 @@ class TTxLoadEverything : public TTransactionBase { if (domainRowset.HaveValue()) { domain.ServerlessComputeResourcesMode = domainRowset.GetValue(); } - + if (domainRowset.HaveValue()) { + domain.SetScaleRecommenderPolicies(domainRowset.GetValue()); + } + if (!domainRowset.Next()) return false; } @@ -644,9 +647,9 @@ class TTxLoadEverything : public TTransactionBase { TFollowerId followerId = metricsRowset.GetValue(); auto* leaderOrFollower = tablet->FindTablet(followerId); if (leaderOrFollower) { - leaderOrFollower->MutableResourceMetricsAggregates().MaximumCPU.InitiaizeFrom(metricsRowset.GetValueOrDefault()); - leaderOrFollower->MutableResourceMetricsAggregates().MaximumMemory.InitiaizeFrom(metricsRowset.GetValueOrDefault()); - leaderOrFollower->MutableResourceMetricsAggregates().MaximumNetwork.InitiaizeFrom(metricsRowset.GetValueOrDefault()); + leaderOrFollower->MutableResourceMetricsAggregates().MaximumCPU.InitializeFrom(metricsRowset.GetValueOrDefault()); + leaderOrFollower->MutableResourceMetricsAggregates().MaximumMemory.InitializeFrom(metricsRowset.GetValueOrDefault()); + leaderOrFollower->MutableResourceMetricsAggregates().MaximumNetwork.InitializeFrom(metricsRowset.GetValueOrDefault()); // do not reorder leaderOrFollower->UpdateResourceUsage(metricsRowset.GetValueOrDefault()); } diff --git a/ydb/core/mind/hive/ut/ya.make b/ydb/core/mind/hive/ut/ya.make index 73104898c4bf..24fc73172f84 100644 --- a/ydb/core/mind/hive/ut/ya.make +++ b/ydb/core/mind/hive/ut/ya.make @@ -20,6 +20,7 @@ YQL_LAST_ABI_VERSION() SRCS( object_distribution_ut.cpp + scale_recommender_policy_ut.cpp sequencer_ut.cpp storage_pool_info_ut.cpp hive_ut.cpp diff --git a/ydb/core/mind/hive/ya.make b/ydb/core/mind/hive/ya.make index 4666bb93cc34..6b116b8c0579 100644 --- a/ydb/core/mind/hive/ya.make +++ b/ydb/core/mind/hive/ya.make @@ -44,6 +44,7 @@ SRCS( tablet_move_info.cpp tx__adopt_tablet.cpp tx__block_storage_result.cpp + tx__configure_scale_recommender.cpp tx__configure_subdomain.cpp tx__create_tablet.cpp tx__cut_tablet_history.cpp diff --git a/ydb/core/mind/local.cpp b/ydb/core/mind/local.cpp index ebd564891b0a..8dd1e47faed4 100644 --- a/ydb/core/mind/local.cpp +++ b/ydb/core/mind/local.cpp @@ -107,10 +107,11 @@ class TLocalNodeRegistrar : public TActorBootstrapped { constexpr static TDuration UPDATE_SYSTEM_USAGE_INTERVAL = TDuration::MilliSeconds(1000); constexpr static TDuration DRAIN_NODE_TIMEOUT = TDuration::MilliSeconds(15000); ui64 UserPoolUsage = 0; // (usage uS x threads) / sec + ui64 UserPoolLimit = 0; // PotentialMaxThreadCount of UserPool ui64 MemUsage = 0; ui64 MemLimit = 0; - ui64 CpuLimit = 0; // PotentialMaxThreadCount of UserPool double NodeUsage = 0; + double CpuUsage = 0; // Sum of CPU usage in all pools / Number of CPUs bool SentDrainNode = false; bool DrainResultReceived = false; @@ -276,8 +277,8 @@ class TLocalNodeRegistrar : public TActorBootstrapped { void FillResourceMaximum(NKikimrTabletBase::TMetrics* record) { record->CopyFrom(ResourceLimit); if (!record->HasCPU()) { - if (CpuLimit != 0) { - record->SetCPU(CpuLimit); + if (UserPoolLimit != 0) { + record->SetCPU(UserPoolLimit); } } if (!record->HasMemory()) { @@ -588,6 +589,7 @@ class TLocalNodeRegistrar : public TActorBootstrapped { record.MutableTotalResourceUsage()->SetMemory(MemUsage); } record.SetTotalNodeUsage(NodeUsage); + record.SetTotalNodeCpuUsage(CpuUsage); FillResourceMaximum(record.MutableResourceMaximum()); NTabletPipe::SendData(ctx, HivePipeClient, event.Release()); SendTabletMetricsTime = ctx.Now(); @@ -649,10 +651,19 @@ class TLocalNodeRegistrar : public TActorBootstrapped { const NKikimrWhiteboard::TEvSystemStateResponse& record = ev->Get()->Record; if (!record.GetSystemStateInfo().empty()) { const NKikimrWhiteboard::TSystemStateInfo& info = record.GetSystemStateInfo(0); + + if (info.HasNumberOfCpus()) { + double cpuUsageSum = 0; + for (const auto& poolInfo : info.poolstats()) { + cpuUsageSum += poolInfo.usage() * poolInfo.limit(); + } + CpuUsage = cpuUsageSum / info.GetNumberOfCpus(); + } + if (static_cast(info.PoolStatsSize()) > AppData()->UserPoolId) { const auto& poolStats(info.GetPoolStats(AppData()->UserPoolId)); - CpuLimit = poolStats.limit() * 1'000'000; // microseconds - UserPoolUsage = poolStats.usage() * CpuLimit; // microseconds + UserPoolLimit = poolStats.limit() * 1'000'000; // microseconds + UserPoolUsage = poolStats.usage() * UserPoolLimit; // microseconds } // Note: we use allocated memory because MemoryUsed(AnonRSS) has lag diff --git a/ydb/core/protos/config.proto b/ydb/core/protos/config.proto index 63d3c2dfcca9..a2384c961cc4 100644 --- a/ydb/core/protos/config.proto +++ b/ydb/core/protos/config.proto @@ -1557,6 +1557,11 @@ message THiveConfig { optional double NodeUsageRangeToKick = 75 [default = 0.2]; optional bool LessSystemTabletsMoves = 77 [default = true]; optional uint64 MaxPingsInFlight = 78 [default = 1000]; + optional uint64 ScaleRecommendationRefreshFrequency = 80 [default = 60000]; // calculate scale recommendation every x milliseconds + optional uint64 ScaleOutWindowSize = 81 [default = 15]; // buckets + optional uint64 ScaleInWindowSize = 82 [default = 5]; // buckets + optional double TargetTrackingCPUMargin = 83 [default = 0.1]; // percent + optional double DryRunTargetTrackingCPU = 84; // percent } message TBlobCacheConfig { diff --git a/ydb/core/protos/counters_hive.proto b/ydb/core/protos/counters_hive.proto index 7a39f2b3f730..75a8ecc7a5d7 100644 --- a/ydb/core/protos/counters_hive.proto +++ b/ydb/core/protos/counters_hive.proto @@ -31,6 +31,9 @@ enum ESimpleCounters { COUNTER_STORAGE_SCATTER = 21 [(CounterOpts) = {Name: "StorageScatter"}]; COUNTER_TABLETS_STARTING = 22 [(CounterOpts) = {Name: "TabletsStarting"}]; COUNTER_PINGQUEUE_SIZE = 23 [(CounterOpts) = {Name: "PingQueueSize"}]; + COUNTER_NODES_RECOMMENDED = 24 [(CounterOpts) = {Name: "NodesRecommended"}]; + COUNTER_NODES_RECOMMENDED_DRY_RUN = 25 [(CounterOpts) = {Name: "NodesRecommendedDryRun"}]; + COUNTER_AVG_CPU_UTILIZATION = 26 [(CounterOpts) = {Name: "AvgCPUUtilization"}]; } enum ECumulativeCounters { @@ -161,4 +164,5 @@ enum ETxTypes { TXTYPE_MON_OBJECT_STATS = 63 [(TxTypeOpts) = {Name: "TxMonObjectStats"}]; TXTYPE_MON_SUBACTORS = 64 [(TxTypeOpts) = {Name: "TxMonSubactors"}]; TXTYPE_MON_TABLET_AVAILABILITY = 65 [(TxTypeOpts) = {Name: "TxMonTabletAvailability"}]; + TXTYPE_CONFIGURE_SCALE_RECOMMENDER = 66 [(TxTypeOpts) = {Name: "TxConfigureScaleRecommender"}]; } diff --git a/ydb/core/protos/feature_flags.proto b/ydb/core/protos/feature_flags.proto index 3c03775ba299..bc1358e31d34 100644 --- a/ydb/core/protos/feature_flags.proto +++ b/ydb/core/protos/feature_flags.proto @@ -160,4 +160,5 @@ message TFeatureFlags { optional bool EnableImmediateWritingOnBulkUpsert = 146 [default = false]; optional bool EnableInsertWriteIdSpecialColumnCompatibility = 147 [default = false]; optional bool EnableDriveSerialsDiscovery = 152 [default = false]; + optional bool EnableScaleRecommender = 157 [default = false]; } diff --git a/ydb/core/protos/hive.proto b/ydb/core/protos/hive.proto index 578809b86ffe..0b14258fa228 100644 --- a/ydb/core/protos/hive.proto +++ b/ydb/core/protos/hive.proto @@ -245,8 +245,9 @@ message TTabletMetrics { message TEvTabletMetrics { repeated TTabletMetrics TabletMetrics = 1; optional NKikimrTabletBase.TMetrics TotalResourceUsage = 2; - optional double TotalNodeUsage = 3; optional NKikimrTabletBase.TMetrics ResourceMaximum = 4; + optional double TotalNodeUsage = 3; + optional double TotalNodeCpuUsage = 5; } message TEvReassignTablet { @@ -589,3 +590,37 @@ message TEvResponseTabletDistribution { } repeated TNode Nodes = 1; } + +message TEvRequestScaleRecommendation { + optional NKikimrSubDomains.TDomainKey DomainKey = 1; +} + +message TEvResponseScaleRecommendation { + optional NKikimrProto.EReplyStatus Status = 1; + optional uint32 RecommendedNodes = 2; +} + +message TScaleRecommenderPolicies { + message TScaleRecommenderPolicy { + message TTargetTrackingPolicy { + oneof Target { + uint32 AverageCpuUtilizationPercent = 1; + } + } + + oneof Policy { + TTargetTrackingPolicy TargetTrackingPolicy = 1; + } + } + + repeated TScaleRecommenderPolicy Policies = 1; +} + +message TEvConfigureScaleRecommender { + optional NKikimrSubDomains.TDomainKey DomainKey = 1; + optional TScaleRecommenderPolicies Policies = 2; +} + +message TEvConfigureScaleRecommenderReply { + optional NKikimrProto.EReplyStatus Status = 1; +} diff --git a/ydb/core/testlib/tenant_helpers.h b/ydb/core/testlib/tenant_helpers.h index 474f735ca378..d69b780715e4 100644 --- a/ydb/core/testlib/tenant_helpers.h +++ b/ydb/core/testlib/tenant_helpers.h @@ -5,6 +5,7 @@ #include #include +#include #include namespace NKikimr { @@ -97,6 +98,7 @@ struct TCreateTenantRequest { EType Type; TAttrsCont Attrs; Ydb::Cms::DatabaseQuotas DatabaseQuotas; + Ydb::Cms::ScaleRecommenderPolicies ScaleRecommenderPolicies; // Common & Shared TPoolsCont Pools; TSlotsCont Slots; @@ -129,6 +131,13 @@ struct TCreateTenantRequest { return *this; } + TSelf& WithScaleRecommenderPolicies(const TString& policies) { + Ydb::Cms::ScaleRecommenderPolicies parsedPolicies; + UNIT_ASSERT_C(NProtoBuf::TextFormat::ParseFromString(policies, &parsedPolicies), policies); + ScaleRecommenderPolicies = std::move(parsedPolicies); + return *this; + } + TSelf& WithPools(const TPoolsCont& pools) { if (Type == EType::Unspecified) { Type = EType::Common; @@ -311,6 +320,45 @@ inline void CheckTenantStatus(TTenantTestRuntime &runtime, const TString &path, CheckTenantStatus(runtime, path, false, code, state, poolTypes, unitRegistrations, args...); } +inline void CheckTenantScaleRecommenderPolicies(TTenantTestRuntime &runtime, const TString &path, + const TString &policies) +{ + auto *event = new NConsole::TEvConsole::TEvGetTenantStatusRequest; + event->Record.MutableRequest()->set_path(path); + + TAutoPtr handle; + runtime.SendToConsole(event); + auto reply = runtime.GrabEdgeEventRethrow(handle); + auto &operation = reply->Record.GetResponse().operation(); + UNIT_ASSERT_VALUES_EQUAL(operation.status(), Ydb::StatusIds::SUCCESS); + + Ydb::Cms::GetDatabaseStatusResult status; + UNIT_ASSERT(operation.result().UnpackTo(&status)); + + if (!policies.empty()) { + Ydb::Cms::ScaleRecommenderPolicies expectedPolicies; + UNIT_ASSERT_C(NProtoBuf::TextFormat::ParseFromString(policies, &expectedPolicies), policies); + UNIT_ASSERT_C(NProtoBuf::util::MessageDifferencer::Equals(status.scale_recommender_policies(), expectedPolicies), + TStringBuilder() << "Expected: " << policies << ", got: " << status.scale_recommender_policies().ShortDebugString()); + } else { + UNIT_ASSERT(!status.has_scale_recommender_policies()); + } +} + +inline void AlterScaleRecommenderPolicies(TTenantTestRuntime &runtime, const TString &path, + Ydb::StatusIds::StatusCode code, const TString &policies) +{ + auto *event = new NConsole::TEvConsole::TEvAlterTenantRequest; + event->Record.MutableRequest()->set_path(path); + + auto *requestPolicies = event->Record.MutableRequest()->mutable_scale_recommender_policies(); + UNIT_ASSERT_C(NProtoBuf::TextFormat::ParseFromString(policies, requestPolicies), policies); + + TAutoPtr handle; + runtime.SendToConsole(event); + auto reply = runtime.GrabEdgeEventRethrow(handle); + UNIT_ASSERT_VALUES_EQUAL(reply->Record.GetResponse().operation().status(), code); +} inline void CheckCreateTenant(TTenantTestRuntime &runtime, const TString &token, @@ -355,6 +403,7 @@ inline void CheckCreateTenant(TTenantTestRuntime &runtime, } event->Record.MutableRequest()->mutable_database_quotas()->CopyFrom(request.DatabaseQuotas); + event->Record.MutableRequest()->mutable_scale_recommender_policies()->CopyFrom(request.ScaleRecommenderPolicies); TAutoPtr handle; runtime.SendToConsole(event); diff --git a/ydb/core/testlib/tenant_runtime.cpp b/ydb/core/testlib/tenant_runtime.cpp index 994928e4cfc4..29c2b709ecb6 100644 --- a/ydb/core/testlib/tenant_runtime.cpp +++ b/ydb/core/testlib/tenant_runtime.cpp @@ -1153,6 +1153,7 @@ TTenantTestRuntime::TTenantTestRuntime(const TTenantTestConfig &config, , Extension(extension) { Extension.MutableFeatureFlags()->SetEnableExternalHive(false); + Extension.MutableFeatureFlags()->SetEnableScaleRecommender(true); Setup(createTenantPools); } diff --git a/ydb/core/util/metrics.h b/ydb/core/util/metrics.h index d726e40555da..80ceb5728c0e 100644 --- a/ydb/core/util/metrics.h +++ b/ydb/core/util/metrics.h @@ -294,6 +294,11 @@ class TAverageValue { return AccumulatorCount >= MaxCount / 2; } + void Clear() { + AccumulatorValue = ValueType(); + AccumulatorCount = 0; + } + protected: ValueType AccumulatorValue; size_t AccumulatorCount; @@ -378,7 +383,7 @@ class TMaximumValueUI64 : public NKikimrMetricsProto::TMaximumValueUI64 { return MaximumValue; } - void InitiaizeFrom(const TProto& proto) { + void InitializeFrom(const TProto& proto) { TProto::CopyFrom(proto); if (TProto::ValuesSize() > 0) { MaximumValue = *std::max_element(TProto::GetValues().begin(), TProto::GetValues().end()); @@ -440,7 +445,7 @@ class TMaximumValueVariableWindowUI64 : public NKikimrMetricsProto::TMaximumValu } void AdvanceTime(TInstant now) { - // Nothing changed, last value is stiil relevant + // Nothing changed, last value is still relevant TType lastValue = {}; if (!TProto::GetValues().empty()) { lastValue = *std::prev(TProto::MutableValues()->end()); @@ -452,7 +457,7 @@ class TMaximumValueVariableWindowUI64 : public NKikimrMetricsProto::TMaximumValu return MaximumValue; } - void InitiaizeFrom(const TProto& proto) { + void InitializeFrom(const TProto& proto) { TProto::CopyFrom(proto); if (TProto::ValuesSize() > 0) { MaximumValue = *std::max_element(TProto::GetValues().begin(), TProto::GetValues().end()); diff --git a/ydb/public/api/grpc/ydb_cms_v1.proto b/ydb/public/api/grpc/ydb_cms_v1.proto index 11e4517381f4..58fb067f2498 100644 --- a/ydb/public/api/grpc/ydb_cms_v1.proto +++ b/ydb/public/api/grpc/ydb_cms_v1.proto @@ -27,4 +27,7 @@ service CmsService { // Describe supported database options. rpc DescribeDatabaseOptions(Cms.DescribeDatabaseOptionsRequest) returns (Cms.DescribeDatabaseOptionsResponse); + + // Get resources scale recommendation for database. + rpc GetScaleRecommendation(Cms.GetScaleRecommendationRequest) returns (Cms.GetScaleRecommendationResponse); } diff --git a/ydb/public/api/protos/ydb_cms.proto b/ydb/public/api/protos/ydb_cms.proto index aa2a7df79478..9a8c2cdb047d 100644 --- a/ydb/public/api/protos/ydb_cms.proto +++ b/ydb/public/api/protos/ydb_cms.proto @@ -4,7 +4,10 @@ option cc_enable_arenas = true; package Ydb.Cms; option java_package = "com.yandex.ydb.cms"; +import "ydb/public/api/protos/annotations/validation.proto"; +import "ydb/public/api/protos/ydb_issue_message.proto"; import "ydb/public/api/protos/ydb_operation.proto"; +import "ydb/public/api/protos/ydb_status_codes.proto"; // A set of uniform storage units. // Single storage unit can be thought of as a reserved part of a RAID. @@ -104,6 +107,28 @@ message DatabaseQuotas { repeated StorageQuotas storage_quotas = 6; } +message ScaleRecommenderPolicies { + // A policy that is used for resource scale recommendation. If multiple are used, + // recommender combines them to recommend the largest scale. + message ScaleRecommenderPolicy { + // Policy that tracks metric and reactively recommend to adjust resources scale + // to keep metric close to the specified target value. + message TargetTrackingPolicy { + oneof target { + // A percentage of compute resources' average CPU utilization. + uint32 average_cpu_utilization_percent = 1 [(Ydb.value) = "[10; 90]"]; + } + } + + oneof policy { + TargetTrackingPolicy target_tracking_policy = 1; + } + } + + repeated ScaleRecommenderPolicy policies = 1; +} + + // Request to create a new database. For successfull creation // specified database shouldn't exist. At least one storage // unit should be requested for the database. @@ -129,6 +154,8 @@ message CreateDatabaseRequest { string idempotency_key = 9; // Optional quotas for the database DatabaseQuotas database_quotas = 10; + // Optional scale recommender policies + ScaleRecommenderPolicies scale_recommender_policies = 11; } message CreateDatabaseResponse { @@ -179,6 +206,8 @@ message GetDatabaseStatusResult { SchemaOperationQuotas schema_operation_quotas = 9; // Current quotas for the database DatabaseQuotas database_quotas = 10; + // Current scale recommender policies + ScaleRecommenderPolicies scale_recommender_policies = 11; } // Change resources allocated for database. @@ -207,6 +236,8 @@ message AlterDatabaseRequest { DatabaseQuotas database_quotas = 11; // Alter attributes. Leave the value blank to drop an attribute. map alter_attributes = 12; + // Change scale recommender policies. + ScaleRecommenderPolicies scale_recommender_policies = 13; } message AlterDatabaseResponse { @@ -271,3 +302,16 @@ message DescribeDatabaseOptionsResult { repeated AvailabilityZoneDescription availability_zones = 2; repeated ComputationalUnitDescription computational_units = 3; } + +// Get resources scale recommendation for database. +message GetScaleRecommendationRequest { + // Required. Full path to database's home dir. + string path = 1; +} + +message GetScaleRecommendationResponse { + StatusIds.StatusCode status = 1; + repeated Ydb.Issue.IssueMessage issues = 2; + // Recommended resources scale to be allocated for database. + Resources recommended_resources = 3; +} diff --git a/ydb/services/cms/grpc_service.cpp b/ydb/services/cms/grpc_service.cpp index f78a907023c0..c46f4acab132 100644 --- a/ydb/services/cms/grpc_service.cpp +++ b/ydb/services/cms/grpc_service.cpp @@ -14,23 +14,24 @@ void TGRpcCmsService::SetupIncomingRequests(NYdbGrpc::TLoggerPtr logger) { #ifdef ADD_REQUEST #error ADD_REQUEST macro already defined #endif -#define ADD_REQUEST(NAME, CB) \ +#define ADD_REQUEST(NAME, CB, TCALL) \ MakeIntrusive> \ (this, &Service_, CQ_, \ [this](NYdbGrpc::IRequestContextBase *ctx) { \ NGRpcService::ReportGrpcReqToMon(*ActorSystem_, ctx->GetPeer()); \ ActorSystem_->Send(GRpcRequestProxyId_, \ - new TGrpcRequestOperationCall \ + new TCALL \ (ctx, &CB, TRequestAuxSettings{RLSWITCH(TRateLimiterMode::Rps), nullptr})); \ }, &Cms::V1::CmsService::AsyncService::Request ## NAME, \ #NAME, logger, getCounterBlock("cms", #NAME))->Run(); - ADD_REQUEST(CreateDatabase, DoCreateTenantRequest) - ADD_REQUEST(AlterDatabase, DoAlterTenantRequest) - ADD_REQUEST(GetDatabaseStatus, DoGetTenantStatusRequest) - ADD_REQUEST(ListDatabases, DoListTenantsRequest) - ADD_REQUEST(RemoveDatabase, DoRemoveTenantRequest) - ADD_REQUEST(DescribeDatabaseOptions, DoDescribeTenantOptionsRequest) + ADD_REQUEST(CreateDatabase, DoCreateTenantRequest, TGrpcRequestOperationCall) + ADD_REQUEST(AlterDatabase, DoAlterTenantRequest, TGrpcRequestOperationCall) + ADD_REQUEST(GetDatabaseStatus, DoGetTenantStatusRequest, TGrpcRequestOperationCall) + ADD_REQUEST(ListDatabases, DoListTenantsRequest, TGrpcRequestOperationCall) + ADD_REQUEST(RemoveDatabase, DoRemoveTenantRequest, TGrpcRequestOperationCall) + ADD_REQUEST(DescribeDatabaseOptions, DoDescribeTenantOptionsRequest, TGrpcRequestOperationCall) + ADD_REQUEST(GetScaleRecommendation, DoGetScaleRecommendationRequest, TGrpcRequestNoOperationCall) #undef ADD_REQUEST } diff --git a/ydb/tests/functional/scheme_tests/canondata/tablet_scheme_tests.TestTabletSchemes.test_tablet_schemes_flat_hive_/flat_hive.schema b/ydb/tests/functional/scheme_tests/canondata/tablet_scheme_tests.TestTabletSchemes.test_tablet_schemes_flat_hive_/flat_hive.schema index 8592ac5c5345..21d0402f1add 100644 --- a/ydb/tests/functional/scheme_tests/canondata/tablet_scheme_tests.TestTabletSchemes.test_tablet_schemes_flat_hive_/flat_hive.schema +++ b/ydb/tests/functional/scheme_tests/canondata/tablet_scheme_tests.TestTabletSchemes.test_tablet_schemes_flat_hive_/flat_hive.schema @@ -7,6 +7,11 @@ 2 ], "ColumnsAdded": [ + { + "ColumnId": 7, + "ColumnName": "ScaleRecommenderPolicies", + "ColumnType": "String" + }, { "ColumnId": 1, "ColumnName": "SchemeshardId", @@ -42,6 +47,7 @@ "ColumnFamilies": { "0": { "Columns": [ + 7, 1, 2, 3,