add translation function

velox/common/config/GlobalConfig.h

@@ -22,7 +22,7 @@ namespace facebook::velox::config {
 
 struct GlobalConfiguration {
   /// Number of shared leaf memory pools per process.
-  int32_t memoryNumSharedLeafPoolsConfig{32};
+  int32_t memoryNumSharedLeafPools{32};
   /// If true, check fails on any memory leaks in memory pool and memory
   /// manager.
   bool memoryLeakCheckEnabled{false};
@@ -47,4 +47,6 @@ struct GlobalConfiguration {
 
 extern GlobalConfiguration globalConfig;
 
+void translateFlagsToGlobalConfig();
+
 } // namespace facebook::velox::config

velox/common/memory/Memory.cpp

@@ -78,7 +78,7 @@ std::vector<std::shared_ptr<MemoryPool>> createSharedLeafMemoryPools(
   VELOX_CHECK_EQ(sysPool.name(), kSysRootName);
   std::vector<std::shared_ptr<MemoryPool>> leafPools;
   const size_t numSharedPools =
-      std::max(1, config::globalConfig.memoryNumSharedLeafPoolsConfig);
+      std::max(1, config::globalConfig.memoryNumSharedLeafPools);
   leafPools.reserve(numSharedPools);
   for (size_t i = 0; i < numSharedPools; ++i) {
     leafPools.emplace_back(
@@ -128,7 +128,7 @@ MemoryManager::MemoryManager(const MemoryManagerOptions& options)
       sysRoot_->name());
   VELOX_CHECK_EQ(
       sharedLeafPools_.size(),
-      std::max(1, config::globalConfig.memoryNumSharedLeafPoolsConfig));
+      std::max(1, config::globalConfig.memoryNumSharedLeafPools));
 }
 
 MemoryManager::~MemoryManager() {

velox/common/memory/tests/MemoryAllocatorTest.cpp

@@ -16,7 +16,6 @@
 #include "velox/common/memory/MemoryAllocator.h"
 #include <thread>
 #include "velox/common/base/tests/GTestUtils.h"
-#include "velox/common/config/GlobalConfig.h"
 #include "velox/common/memory/AllocationPool.h"
 #include "velox/common/memory/MallocAllocator.h"
 #include "velox/common/memory/MmapAllocator.h"
@@ -35,6 +34,9 @@
 #include <fstream>
 #endif // linux
 
+DECLARE_bool(velox_memory_leak_check_enabled);
+DECLARE_bool(velox_time_allocations);
+
 using namespace facebook::velox::common::testutil;
 
 namespace facebook::velox::memory {
@@ -54,7 +56,8 @@ class MemoryAllocatorTest : public testing::TestWithParam<int> {
  protected:
   static void SetUpTestCase() {
     TestValue::enable();
-    config::globalConfig.memoryLeakCheckEnabled = true;
+    FLAGS_velox_memory_leak_check_enabled = true;
+    config::translateFlagsToGlobalConfig();
   }
 
   void SetUp() override {
@@ -637,6 +640,8 @@ TEST_P(MemoryAllocatorTest, allocationClass2) {
 TEST_P(MemoryAllocatorTest, stats) {
   const std::vector<MachinePageCount>& sizes = instance_->sizeClasses();
   MachinePageCount capacity = kCapacityPages;
+  FLAGS_velox_time_allocations = false;
+  config::translateFlagsToGlobalConfig();
   for (auto i = 0; i < sizes.size(); ++i) {
     std::unique_ptr<Allocation> allocation = std::make_unique<Allocation>();
     auto size = sizes[i];
@@ -649,8 +654,8 @@ TEST_P(MemoryAllocatorTest, stats) {
     ASSERT_EQ(stats.sizes[i].numAllocations, 0);
   }
 
-  gflags::FlagSaver flagSaver;
-  config::globalConfig.timeAllocations = true;
+  FLAGS_velox_time_allocations = true;
+  config::translateFlagsToGlobalConfig();
   for (auto i = 0; i < sizes.size(); ++i) {
     std::unique_ptr<Allocation> allocation = std::make_unique<Allocation>();
     auto size = sizes[i];
@@ -668,8 +673,9 @@ TEST_P(MemoryAllocatorTest, singleAllocation) {
   if (!useMmap_ && enableReservation_) {
     return;
   }
-  gflags::FlagSaver flagSaver;
-  config::globalConfig.timeAllocations = true;
+
+  FLAGS_velox_time_allocations = true;
+  config::translateFlagsToGlobalConfig();
   const std::vector<MachinePageCount>& sizes = instance_->sizeClasses();
   MachinePageCount capacity = kCapacityPages;
   for (auto i = 0; i < sizes.size(); ++i) {
@@ -716,6 +722,8 @@ TEST_P(MemoryAllocatorTest, singleAllocation) {
     }
     ASSERT_TRUE(instance_->checkConsistency());
   }
+  FLAGS_velox_time_allocations = false;
+  config::translateFlagsToGlobalConfig();
 }
 
 TEST_P(MemoryAllocatorTest, increasingSize) {
@@ -1613,6 +1621,11 @@ TEST_P(MemoryAllocatorTest, allocatorCapacityWithThreads) {
   EXPECT_EQ(instance_->numAllocated(), 0);
 }
 
+VELOX_INSTANTIATE_TEST_SUITE_P(
+    MemoryAllocatorTestSuite,
+    MemoryAllocatorTest,
+    testing::ValuesIn({0, 1, 2}));
+
 class MmapArenaTest : public testing::Test {
  public:
   // 32 MB arena space
@@ -1942,11 +1955,6 @@ TEST_P(MemoryAllocatorTest, unmap) {
   }
 }
 
-VELOX_INSTANTIATE_TEST_SUITE_P(
-    MemoryAllocatorTestSuite,
-    MemoryAllocatorTest,
-    testing::ValuesIn({0, 1, 2}));
-
 class MmapConfigTest : public testing::Test {
  public:
  protected:
@@ -1993,5 +2001,4 @@ TEST_F(MmapConfigTest, sizeClasses) {
     runPages = runPages / 2;
   }
 }
-
 } // namespace facebook::velox::memory

velox/common/memory/tests/MemoryCapExceededTest.cpp

@@ -34,11 +34,13 @@ class MemoryCapExceededTest : public OperatorTestBase,
     // NOTE: if 'GetParam()' is true, then suppress the verbose error message in
     // memory capacity exceeded exception.
     FLAGS_velox_suppress_memory_capacity_exceeding_error_message = GetParam();
+    config::translateFlagsToGlobalConfig();
   }
 
   void TearDown() override {
     OperatorTestBase::TearDown();
     FLAGS_velox_suppress_memory_capacity_exceeding_error_message = false;
+    config::translateFlagsToGlobalConfig();
   }
 };
 

velox/common/memory/tests/MemoryManagerTest.cpp

@@ -44,6 +44,7 @@ class MemoryManagerTest : public testing::Test {
  protected:
   static void SetUpTestCase() {
     SharedArbitrator::registerFactory();
+    config::translateFlagsToGlobalConfig();
   }
 
   inline static const std::string arbitratorKind_{"SHARED"};

velox/dwio/dwrf/test/TestIntegerDictionaryEncoder.cpp

@@ -29,6 +29,7 @@ class TestIntegerDictionaryEncoder : public ::testing::Test {
  protected:
   static void SetUpTestCase() {
     FLAGS_velox_enable_memory_usage_track_in_default_memory_pool = true;
+    config::translateFlagsToGlobalConfig();
     memory::MemoryManager::testingSetInstance({});
   }
 };

velox/dwio/dwrf/test/TestStringDictionaryEncoder.cpp

@@ -28,6 +28,7 @@ class TestStringDictionaryEncoder : public ::testing::Test {
  protected:
   static void SetUpTestCase() {
     FLAGS_velox_enable_memory_usage_track_in_default_memory_pool = true;
+    config::translateFlagsToGlobalConfig();
     memory::MemoryManager::testingSetInstance({});
   }
 };

velox/exec/tests/utils/OperatorTestBase.cpp

@@ -63,6 +63,7 @@ OperatorTestBase::~OperatorTestBase() {
 void OperatorTestBase::SetUpTestCase() {
   FLAGS_velox_enable_memory_usage_track_in_default_memory_pool = true;
   FLAGS_velox_memory_leak_check_enabled = true;
+  config::translateFlagsToGlobalConfig();
   memory::SharedArbitrator::registerFactory();
   resetMemory();
   functions::prestosql::registerAllScalarFunctions();

velox/flag_definitions/flags.cpp

@@ -16,6 +16,8 @@
 
 #include <gflags/gflags.h>
 
+#include "velox/common/config/GlobalConfig.h"
+
 // Used in velox/common/memory/Memory.cpp
 
 DEFINE_int32(
@@ -100,3 +102,28 @@ DEFINE_bool(
     "exception. This is only used by test to control the test error output size");
 
 DEFINE_bool(velox_memory_use_hugepages, true, "Use explicit huge pages");
+
+DEFINE_bool(
+    velox_memory_pool_capacity_transfer_across_tasks,
+    false,
+    "Whether allow to memory capacity transfer between memory pools from "
+    "different tasks, which might happen in use case like Spark-Gluten");
+
+namespace facebook::velox::config {
+void translateFlagsToGlobalConfig() {
+  config::globalConfig.memoryNumSharedLeafPools =
+      FLAGS_velox_memory_num_shared_leaf_pools;
+  config::globalConfig.memoryLeakCheckEnabled =
+      FLAGS_velox_memory_leak_check_enabled;
+  config::globalConfig.memoryPoolDebugEnabled =
+      FLAGS_velox_memory_pool_debug_enabled;
+  config::globalConfig.enableMemoryUsageTrackInDefaultMemoryPool =
+      FLAGS_velox_enable_memory_usage_track_in_default_memory_pool;
+  config::globalConfig.timeAllocations = FLAGS_velox_time_allocations;
+  config::globalConfig.memoryUseHugepages = FLAGS_velox_memory_use_hugepages;
+  config::globalConfig.suppressMemoryCapacityExceedingErrorMessage =
+      FLAGS_velox_suppress_memory_capacity_exceeding_error_message;
+  config::globalConfig.memoryPoolCapacityTransferAcrossTasks =
+      FLAGS_velox_memory_pool_capacity_transfer_across_tasks;
+}
+} // namespace facebook::velox::config