new tiered_ttl mode in public TtlSettings (ydb-platform#12405)

GrigoriyPA · Dec 18, 2024 · de64fff · de64fff
1 parent de58538
commit de64fff
Show file tree

Hide file tree

Showing 16 changed files with 421 additions and 373 deletions.
diff --git a/ydb/core/kqp/host/kqp_gateway_proxy.cpp b/ydb/core/kqp/host/kqp_gateway_proxy.cpp
@@ -514,7 +514,7 @@ bool FillCreateColumnTableDesc(NYql::TKikimrTableMetadataPtr metadata,
             auto* tierProto = resultSettings.MutableEnabled()->AddTiers();
             tierProto->SetApplyAfterSeconds(tier.ApplyAfter.Seconds());
             if (tier.StorageName) {
-                tierProto->MutableEvictToExternalStorage()->SetStorageName(*tier.StorageName);
+                tierProto->MutableEvictToExternalStorage()->SetStorage(*tier.StorageName);
             } else {
                 tierProto->MutableDelete();
             }

diff --git a/ydb/core/kqp/provider/yql_kikimr_gateway.cpp b/ydb/core/kqp/provider/yql_kikimr_gateway.cpp
@@ -308,21 +308,22 @@ bool ConvertReadReplicasSettingsToProto(const TString settings, Ydb::Table::Read
 }
 
 void ConvertTtlSettingsToProto(const NYql::TTtlSettings& settings, Ydb::Table::TtlSettings& proto) {
-    if (!settings.ColumnUnit) {
-        auto& opts = *proto.mutable_date_type_column_v1();
-        opts.set_column_name(settings.ColumnName);
-    } else {
-        auto& opts = *proto.mutable_value_since_unix_epoch_v1();
-        opts.set_column_name(settings.ColumnName);
-        opts.set_column_unit(static_cast<Ydb::Table::ValueSinceUnixEpochModeSettings::Unit>(*settings.ColumnUnit));
-    }
     for (const auto& tier : settings.Tiers) {
-        auto* tierProto = proto.add_tiers();
-        tierProto->set_apply_after_seconds(tier.ApplyAfter.Seconds());
+        auto* outTier = proto.mutable_tiered_ttl()->add_tiers();
+        if (!settings.ColumnUnit) {
+            auto& expr = *outTier->mutable_date_type_column();
+            expr.set_column_name(settings.ColumnName);
+            expr.set_expire_after_seconds(tier.ApplyAfter.Seconds());
+        } else {
+            auto& expr = *outTier->mutable_value_since_unix_epoch();
+            expr.set_column_name(settings.ColumnName);
+            expr.set_column_unit(static_cast<Ydb::Table::ValueSinceUnixEpochModeSettings::Unit>(*settings.ColumnUnit));
+            expr.set_expire_after_seconds(tier.ApplyAfter.Seconds());
+        }
         if (tier.StorageName) {
-            tierProto->mutable_evict_to_external_storage()->set_storage_name(*tier.StorageName);
+            outTier->mutable_evict_to_external_storage()->set_storage(*tier.StorageName);
         } else {
-            tierProto->mutable_delete_();
+            outTier->mutable_delete_();
         }
     }
 }

diff --git a/ydb/core/kqp/ut/scheme/kqp_scheme_ut.cpp b/ydb/core/kqp/ut/scheme/kqp_scheme_ut.cpp
@@ -5479,8 +5479,8 @@ Y_UNIT_TEST_SUITE(KqpScheme) {
             UNIT_ASSERT(desc.GetTableDescription().GetTtlSettings());
             auto ttl = desc.GetTableDescription().GetTtlSettings();
             UNIT_ASSERT_VALUES_EQUAL(ttl->GetTiers().size(), 1);
-            UNIT_ASSERT_VALUES_EQUAL(std::get<TTtlEvictToExternalStorageAction>(ttl->GetTiers()[0].GetAction()).StorageName, "tier1");
-            UNIT_ASSERT_VALUES_EQUAL(ttl->GetTiers()[0].GetApplyAfter(), TDuration::Seconds(10));
+            UNIT_ASSERT_VALUES_EQUAL(std::get<TTtlEvictToExternalStorageAction>(ttl->GetTiers()[0].GetAction()).GetStorage(), "tier1");
+            UNIT_ASSERT_VALUES_EQUAL(std::get<TDateTypeColumnModeSettings>(ttl->GetTiers()[0].GetExpression()).GetExpireAfter(), TDuration::Seconds(10));
         }
         auto query2 = TStringBuilder() << R"(
             --!syntax_v1
@@ -5495,8 +5495,8 @@ Y_UNIT_TEST_SUITE(KqpScheme) {
             UNIT_ASSERT(desc.GetTableDescription().GetTtlSettings());
             auto ttl = desc.GetTableDescription().GetTtlSettings();
             UNIT_ASSERT_VALUES_EQUAL(ttl->GetTiers().size(), 1);
-            UNIT_ASSERT_VALUES_EQUAL(std::get<TTtlEvictToExternalStorageAction>(ttl->GetTiers()[0].GetAction()).StorageName, "tier2");
-            UNIT_ASSERT_VALUES_EQUAL(ttl->GetTiers()[0].GetApplyAfter(), TDuration::Seconds(10));
+            UNIT_ASSERT_VALUES_EQUAL(std::get<TTtlEvictToExternalStorageAction>(ttl->GetTiers()[0].GetAction()).GetStorage(), "tier2");
+            UNIT_ASSERT_VALUES_EQUAL(std::get<TDateTypeColumnModeSettings>(ttl->GetTiers()[0].GetExpression()).GetExpireAfter(), TDuration::Seconds(10));
         }
 
         auto query3 = TStringBuilder() << R"(
@@ -5526,8 +5526,8 @@ Y_UNIT_TEST_SUITE(KqpScheme) {
             UNIT_ASSERT(desc.GetTableDescription().GetTtlSettings());
             auto ttl = desc.GetTableDescription().GetTtlSettings();
             UNIT_ASSERT_VALUES_EQUAL(ttl->GetTiers().size(), 1);
-            UNIT_ASSERT_VALUES_EQUAL(std::get<TTtlEvictToExternalStorageAction>(ttl->GetTiers()[0].GetAction()).StorageName, "tier1");
-            UNIT_ASSERT_VALUES_EQUAL(ttl->GetTiers()[0].GetApplyAfter(), TDuration::Seconds(10));
+            UNIT_ASSERT_VALUES_EQUAL(std::get<TTtlEvictToExternalStorageAction>(ttl->GetTiers()[0].GetAction()).GetStorage(), "tier1");
+            UNIT_ASSERT_VALUES_EQUAL(std::get<TDateTypeColumnModeSettings>(ttl->GetTiers()[0].GetExpression()).GetExpireAfter(), TDuration::Seconds(10));
         }
 
         auto query5 = TStringBuilder() << R"(
@@ -8524,11 +8524,11 @@ Y_UNIT_TEST_SUITE(KqpOlapScheme) {
             UNIT_ASSERT_VALUES_EQUAL(ttl->GetTiers().size(), 2);
             auto evictTier = ttl->GetTiers()[0];
             UNIT_ASSERT(std::holds_alternative<TTtlEvictToExternalStorageAction>(evictTier.GetAction()));
-            UNIT_ASSERT_VALUES_EQUAL(std::get<TTtlEvictToExternalStorageAction>(evictTier.GetAction()).StorageName, "tier1");
-            UNIT_ASSERT_VALUES_EQUAL(evictTier.GetApplyAfter(), TDuration::Seconds(10));
+            UNIT_ASSERT_VALUES_EQUAL(std::get<TTtlEvictToExternalStorageAction>(evictTier.GetAction()).GetStorage(), "tier1");
+            UNIT_ASSERT_VALUES_EQUAL(std::get<TDateTypeColumnModeSettings>(evictTier.GetExpression()).GetExpireAfter(), TDuration::Seconds(10));
             auto deleteTier = ttl->GetTiers()[1];
             UNIT_ASSERT(std::holds_alternative<TTtlDeleteAction>(deleteTier.GetAction()));
-            UNIT_ASSERT_VALUES_EQUAL(deleteTier.GetApplyAfter(), TDuration::Hours(1));
+            UNIT_ASSERT_VALUES_EQUAL(std::get<TDateTypeColumnModeSettings>(deleteTier.GetExpression()).GetExpireAfter(), TDuration::Hours(1));
         }
         {
             auto alterQuery = TStringBuilder() << "ALTER TABLE `" << testTable.GetName() <<  R"(` RESET (TTL);)";

diff --git a/ydb/core/protos/flat_scheme_op.proto b/ydb/core/protos/flat_scheme_op.proto
@@ -223,7 +223,7 @@ message TTTLSettings {
     }
 
     message TEvictionToExternalStorageSettings {
-        optional string StorageName = 1;
+        optional string Storage = 1;
     }
 
     message TTier {

diff --git a/ydb/core/tx/columnshard/engines/scheme/tiering/tier_info.h b/ydb/core/tx/columnshard/engines/scheme/tiering/tier_info.h
@@ -242,7 +242,7 @@ class TTiering {
                     tierInfo = TTierInfo::MakeTtl(TDuration::Seconds(tier.GetApplyAfterSeconds()), ttlColumnName, unitsInSecond);
                     break;
                 case NKikimrSchemeOp::TTTLSettings_TTier::kEvictToExternalStorage:
-                    tierInfo = std::make_shared<TTierInfo>(tier.GetEvictToExternalStorage().GetStorageName(),
+                    tierInfo = std::make_shared<TTierInfo>(tier.GetEvictToExternalStorage().GetStorage(),
                         TDuration::Seconds(tier.GetApplyAfterSeconds()), ttlColumnName, unitsInSecond);
                     break;
                 case NKikimrSchemeOp::TTTLSettings_TTier::ACTION_NOT_SET:
@@ -273,7 +273,7 @@ class TTiering {
         for (const auto& tier : ttlSettings.GetTiers()) {
             switch (tier.GetActionCase()) {
                 case NKikimrSchemeOp::TTTLSettings_TTier::kEvictToExternalStorage:
-                    usedTiers.emplace(tier.GetEvictToExternalStorage().GetStorageName());
+                    usedTiers.emplace(tier.GetEvictToExternalStorage().GetStorage());
                     break;
                 case NKikimrSchemeOp::TTTLSettings_TTier::kDelete:
                 case NKikimrSchemeOp::TTTLSettings_TTier::ACTION_NOT_SET:

diff --git a/ydb/core/tx/columnshard/test_helper/columnshard_ut_common.h b/ydb/core/tx/columnshard/test_helper/columnshard_ut_common.h
@@ -257,7 +257,7 @@ struct TTestSchema {
             UNIT_ASSERT(tier.EvictAfter);
             UNIT_ASSERT_EQUAL(specials.TtlColumn, tier.TtlColumn);
             auto* tierSettings = ttlSettings->MutableEnabled()->AddTiers();
-            tierSettings->MutableEvictToExternalStorage()->SetStorageName(tier.Name);
+            tierSettings->MutableEvictToExternalStorage()->SetStorage(tier.Name);
             tierSettings->SetApplyAfterSeconds(tier.EvictAfter->Seconds());
         }
         if (specials.HasTtl()) {

diff --git a/ydb/core/tx/schemeshard/olap/manager/manager.cpp b/ydb/core/tx/schemeshard/olap/manager/manager.cpp
@@ -7,7 +7,7 @@ void TTablesStorage::OnAddObject(const TPathId& pathId, TColumnTableInfo::TPtr o
         std::optional<TString> usedExternalStorage;
         switch (tier.GetActionCase()) {
             case NKikimrSchemeOp::TTTLSettings_TTier::kEvictToExternalStorage:
-                usedExternalStorage = tier.GetEvictToExternalStorage().GetStorageName();
+                usedExternalStorage = tier.GetEvictToExternalStorage().GetStorage();
                 break;
             case NKikimrSchemeOp::TTTLSettings_TTier::kDelete:
             case NKikimrSchemeOp::TTTLSettings_TTier::ACTION_NOT_SET:
@@ -27,7 +27,7 @@ void TTablesStorage::OnRemoveObject(const TPathId& pathId, TColumnTableInfo::TPt
         std::optional<TString> usedExternalStorage;
         switch (tier.GetActionCase()) {
             case NKikimrSchemeOp::TTTLSettings_TTier::kEvictToExternalStorage:
-                usedExternalStorage = tier.GetEvictToExternalStorage().GetStorageName();
+                usedExternalStorage = tier.GetEvictToExternalStorage().GetStorage();
                 break;
             case NKikimrSchemeOp::TTTLSettings_TTier::kDelete:
             case NKikimrSchemeOp::TTTLSettings_TTier::ACTION_NOT_SET:

diff --git a/ydb/core/tx/schemeshard/ut_helpers/ls_checks.cpp b/ydb/core/tx/schemeshard/ut_helpers/ls_checks.cpp
@@ -1230,7 +1230,7 @@ TCheckFunc HasColumnTableTtlSettingsTier(const TString& columnName, const TDurat
         UNIT_ASSERT_VALUES_EQUAL(tier.GetApplyAfterSeconds(), evictAfter.Seconds());
         if (storageName) {
             UNIT_ASSERT(tier.HasEvictToExternalStorage());
-            UNIT_ASSERT_VALUES_EQUAL(tier.GetEvictToExternalStorage().GetStorageName(), storageName);
+            UNIT_ASSERT_VALUES_EQUAL(tier.GetEvictToExternalStorage().GetStorage(), storageName);
         } else {
             UNIT_ASSERT(tier.HasDelete());
         }

diff --git a/ydb/core/tx/schemeshard/ut_olap/ut_olap.cpp b/ydb/core/tx/schemeshard/ut_olap/ut_olap.cpp
@@ -553,7 +553,7 @@ Y_UNIT_TEST_SUITE(TOlap) {
                     Tiers: {
                         ApplyAfterSeconds: 360
                         EvictToExternalStorage {
-                            StorageName: "Tier1"
+                            Storage: "Tier1"
                         }
                     }
                 }
@@ -579,7 +579,7 @@ Y_UNIT_TEST_SUITE(TOlap) {
                     Tiers: {
                         ApplyAfterSeconds: 3600000000
                         EvictToExternalStorage {
-                            StorageName: "Tier1"
+                            Storage: "Tier1"
                         }
                     }
                 }
@@ -732,7 +732,7 @@ Y_UNIT_TEST_SUITE(TOlap) {
                     Tiers: {
                         ApplyAfterSeconds: 3600000000
                         EvictToExternalStorage {
-                            StorageName: "Tier1"
+                            Storage: "Tier1"
                         }
                     }
                 }

diff --git a/ydb/core/tx/schemeshard/ut_ttl/ut_ttl.cpp b/ydb/core/tx/schemeshard/ut_ttl/ut_ttl.cpp
@@ -1230,7 +1230,7 @@ Y_UNIT_TEST_SUITE(TSchemeShardTTLTests) {
                 ColumnName: "modified_at"
                 Tiers {
                     EvictToExternalStorage {
-                        StorageName: "/Root/abc"
+                        Storage: "/Root/abc"
                     }
                     ApplyAfterSeconds: 3600
                 }

diff --git a/ydb/core/tx/schemeshard/ut_ttl/ut_ttl_utility.cpp b/ydb/core/tx/schemeshard/ut_ttl/ut_ttl_utility.cpp
@@ -33,7 +33,7 @@ Y_UNIT_TEST_SUITE(TSchemeShardTTLUtility) {
         };
         auto makeEvictTier = [](const ui32 seconds) {
             NKikimrSchemeOp::TTTLSettings::TTier tier;
-            tier.MutableEvictToExternalStorage()->SetStorageName("/Root/abc");
+            tier.MutableEvictToExternalStorage()->SetStorage("/Root/abc");
             tier.SetApplyAfterSeconds(seconds);
             return tier;
         };
@@ -60,7 +60,7 @@ Y_UNIT_TEST_SUITE(TSchemeShardTTLUtility) {
 
     Y_UNIT_TEST(GetExpireAfter) {
         NKikimrSchemeOp::TTTLSettings::TTier evictTier;
-        evictTier.MutableEvictToExternalStorage()->SetStorageName("/Root/abc");
+        evictTier.MutableEvictToExternalStorage()->SetStorage("/Root/abc");
         evictTier.SetApplyAfterSeconds(1800);
         NKikimrSchemeOp::TTTLSettings::TTier deleteTier;
         deleteTier.MutableDelete();