Upgrade folly to v2023.12.04.00 (from v2022.11.14.00)

CMake/resolve_dependency_modules/folly/CMakeLists.txt

@@ -14,11 +14,11 @@
 project(Folly)
 cmake_minimum_required(VERSION 3.14)
 
-set(VELOX_FOLLY_BUILD_VERSION v2022.11.14.00)
+set(VELOX_FOLLY_BUILD_VERSION v2023.12.04.00)
 set(VELOX_FOLLY_BUILD_SHA256_CHECKSUM
-    b249436cb61b6dfd5288093565438d8da642b07ae021191a4042b221bc1bdc0e)
+    ed66d435eacb8065ade2fac310c452ef288998e16fd73a95a7db4bdbbb4eea5d)
 set(VELOX_FOLLY_SOURCE_URL
-    "https://github.com/facebook/folly/archive/${VELOX_FOLLY_BUILD_VERSION}.tar.gz"
+    "https://github.com/facebook/folly/releases/download/${VELOX_FOLLY_BUILD_VERSION}/folly-${VELOX_FOLLY_BUILD_VERSION}.tar.gz"
 )
 
 resolve_dependency_url(FOLLY)
@@ -52,11 +52,17 @@ add_library(Folly::folly ALIAS folly)
 # The folly target does not contain any include directories, they are propagated
 # from folly_base. This marks them as system headers which should suppress
 # warnigs generated by them when they are included else where.
-get_target_property(_inc folly_base INTERFACE_INCLUDE_DIRECTORIES)
-target_include_directories(folly_base SYSTEM INTERFACE ${_inc})
+set_target_properties(
+  folly_deps
+  PROPERTIES INTERFACE_SYSTEM_INCLUDE_DIRECTORIES
+             $<TARGET_PROPERTY:folly_deps,INTERFACE_INCLUDE_DIRECTORIES>)
+set_target_properties(
+  folly_base
+  PROPERTIES INTERFACE_SYSTEM_INCLUDE_DIRECTORIES
+             $<TARGET_PROPERTY:folly_base,INTERFACE_INCLUDE_DIRECTORIES>)
 
 if(${gflags_SOURCE} STREQUAL "BUNDLED")
-  add_dependencies(folly glog gflags)
+  add_dependencies(folly glog gflags_static fmt::fmt)
 endif()
 
 set(FOLLY_BENCHMARK_STATIC_LIB

CMake/resolve_dependency_modules/glog.cmake

@@ -37,7 +37,7 @@ set(BUILD_TESTING OFF)
 FetchContent_MakeAvailable(glog)
 unset(BUILD_TESTING)
 unset(BUILD_SHARED_LIBS)
-add_dependencies(glog gflags)
+add_dependencies(glog gflags_static)
 
 list(PREPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_LIST_DIR}/glog)
 set(glog_INCLUDE_DIR ${glog_BINARY_DIR})

CMakeLists.txt

@@ -180,6 +180,10 @@ if(${VELOX_BUILD_PYTHON_PACKAGE})
   set(VELOX_ENABLE_BENCHMARKS OFF)
 endif()
 
+# We look for OpenSSL here to cache the result enforce the version across our
+# dependencies.
+find_package(OpenSSL REQUIRED)
+
 if(VELOX_ENABLE_CCACHE
    AND NOT CMAKE_C_COMPILER_LAUNCHER
    AND NOT CMAKE_CXX_COMPILER_LAUNCHER)

scripts/setup-circleci.sh

@@ -118,9 +118,6 @@ cmake_install glog -DBUILD_SHARED_LIBS=ON -DCMAKE_INSTALL_PREFIX:PATH=/usr
 cmake_install snappy -DSNAPPY_BUILD_TESTS=OFF
 cmake_install fmt -DFMT_TEST=OFF
 cmake_install folly -DFOLLY_HAVE_INT128_T=ON
-# remove in #7700
-sed -i 's/\[\[deprecated.*\]\]//g' /usr/local/include/folly/init/Init.h
-
 
 cmake_install fizz/fizz -DBUILD_TESTS=OFF
 cmake_install wangle/wangle -DBUILD_TESTS=OFF

scripts/setup-macos.sh

@@ -93,7 +93,7 @@ function install_fmt {
 }
 
 function install_folly {
-  github_checkout facebook/folly "v2022.11.14.00"
+  github_checkout facebook/folly "v2023.12.04.00"
   OPENSSL_ROOT_DIR=$(brew --prefix [email protected]) \
   cmake_install -DBUILD_TESTS=OFF -DFOLLY_HAVE_INT128_T=ON
 }

scripts/setup-ubuntu.sh

@@ -23,7 +23,7 @@ source $SCRIPTDIR/setup-helper-functions.sh
 CPU_TARGET="${CPU_TARGET:-avx}"
 COMPILER_FLAGS=$(get_cxx_flags "$CPU_TARGET")
 export COMPILER_FLAGS
-FB_OS_VERSION=v2022.11.14.00
+FB_OS_VERSION=v2023.12.04.00
 NPROC=$(getconf _NPROCESSORS_ONLN)
 DEPENDENCY_DIR=${DEPENDENCY_DIR:-$(pwd)}
 export CMAKE_BUILD_TYPE=Release
@@ -102,6 +102,11 @@ function install_wangle {
   cmake_install -DBUILD_TESTS=OFF -S wangle
 }
 
+function install_mvfst {
+  github_checkout facebook/mvfst "${FB_OS_VERSION}"
+  cmake_install -DBUILD_TESTS=OFF
+}
+
 function install_fbthrift {
   github_checkout facebook/fbthrift "${FB_OS_VERSION}"
   cmake_install -DBUILD_TESTS=OFF
@@ -118,6 +123,7 @@ function install_velox_deps {
   run_and_time install_folly
   run_and_time install_fizz
   run_and_time install_wangle
+  run_and_time install_mvfst
   run_and_time install_fbthrift
   run_and_time install_conda
 }

scripts/setup-velox-torcharrow.sh

@@ -79,7 +79,7 @@ function wget_and_untar {
 wget_and_untar https://github.com/gflags/gflags/archive/refs/tags/v2.2.2.tar.gz gflags
 wget_and_untar https://ftp.openssl.org/source/openssl-1.1.1k.tar.gz openssl &
 wget_and_untar https://boostorg.jfrog.io/artifactory/main/release/1.69.0/source/boost_1_69_0.tar.gz boost &
-wget_and_untar https://github.com/facebook/folly/archive/v2022.11.14.00.tar.gz folly &
+wget_and_untar https://github.com/facebook/folly/archive/v2023.12.04.00.tar.gz folly &
 wget_and_untar https://github.com/fmtlib/fmt/archive/refs/tags/10.1.1.tar.gz fmt &
 
 wait

velox/benchmarks/basic/ComparisonConjunct.cpp

@@ -175,7 +175,7 @@ BENCHMARK(conjunctsNested) {
 } // namespace
 
 int main(int argc, char* argv[]) {
-  folly::init(&argc, &argv);
+  folly::Init init{&argc, &argv};
   gflags::ParseCommandLineFlags(&argc, &argv, true);
   memory::MemoryManager::initialize({});
   benchmark = std::make_unique<ComparisonBenchmark>(1'000);

velox/benchmarks/basic/DecodedVector.cpp

@@ -188,7 +188,7 @@ BENCHMARK(decodeDictionary5Nested) {
 } // namespace
 
 int main(int argc, char* argv[]) {
-  folly::init(&argc, &argv);
+  folly::Init init{&argc, &argv};
   gflags::ParseCommandLineFlags(&argc, &argv, true);
   memory::MemoryManager::initialize({});
   benchmark = std::make_unique<DecodedVectorBenchmark>(10'000);

velox/benchmarks/basic/FeatureNormalization.cpp

@@ -110,7 +110,7 @@ BENCHMARK(normalizeConstant) {
 } // namespace
 
 int main(int argc, char* argv[]) {
-  folly::init(&argc, &argv);
+  folly::Init init{&argc, &argv};
   gflags::ParseCommandLineFlags(&argc, &argv, true);
   memory::MemoryManager::initialize({});
   benchmark = std::make_unique<FeatureNormailzationBenchmark>();

velox/benchmarks/basic/LikeTpchBenchmark.cpp

@@ -242,7 +242,7 @@ BENCHMARK(tpchQuery20) {
 } // namespace
 
 int main(int argc, char* argv[]) {
-  folly::init(&argc, &argv, true);
+  folly::Init init{&argc, &argv, true};
   memory::MemoryManager::initialize({});
   benchmark = std::make_unique<LikeFunctionsBenchmark>();
   folly::runBenchmarks();

velox/benchmarks/basic/Preproc.cpp

@@ -444,7 +444,7 @@ BENCHMARK(allFusedWithNulls) {
 } // namespace
 
 int main(int argc, char** argv) {
-  folly::init(&argc, &argv);
+  folly::Init init{&argc, &argv};
   memory::MemoryManager::initialize({});
   benchmark = std::make_unique<PreprocBenchmark>();
   // Verify that benchmark calculations are correct.

velox/benchmarks/basic/SelectivityVector.cpp

@@ -164,7 +164,7 @@ BENCHMARK(sumSelectivity1PerCent) {
 } // namespace
 
 int main(int argc, char* argv[]) {
-  folly::init(&argc, &argv);
+  folly::Init init{&argc, &argv};
   gflags::ParseCommandLineFlags(&argc, &argv, true);
   memory::MemoryManager::initialize({});
   benchmark = std::make_unique<SelectivityVectorBenchmark>(10'000);

velox/benchmarks/basic/SimpleArithmetic.cpp

@@ -343,7 +343,7 @@ BENCHMARK(plusCheckedLarge) {
 } // namespace
 
 int main(int argc, char* argv[]) {
-  folly::init(&argc, &argv);
+  folly::Init init{&argc, &argv};
   gflags::ParseCommandLineFlags(&argc, &argv, true);
   memory::MemoryManager::initialize({});
   benchmark = std::make_unique<SimpleArithmeticBenchmark>();

velox/benchmarks/basic/SimpleCastExpr.cpp

@@ -250,7 +250,7 @@ BENCHMARK(castStructManyFieldsNestedCastMedium) {
 } // namespace
 
 int main(int argc, char* argv[]) {
-  folly::init(&argc, &argv);
+  folly::Init init{&argc, &argv};
   gflags::ParseCommandLineFlags(&argc, &argv, true);
   memory::MemoryManager::initialize({});
   benchmark = std::make_unique<SimpleCastBenchmark>();

velox/benchmarks/basic/VectorCompare.cpp

@@ -114,7 +114,7 @@ BENCHMARK_DRAW_LINE();
 } // namespace
 
 int main(int argc, char* argv[]) {
-  folly::init(&argc, &argv);
+  folly::Init init{&argc, &argv};
   gflags::ParseCommandLineFlags(&argc, &argv, true);
   memory::MemoryManager::initialize({});
   benchmark = std::make_unique<VectorCompareBenchmark>(1000);

velox/benchmarks/basic/VectorFuzzer.cpp

@@ -131,7 +131,7 @@ BENCHMARK_RELATIVE_MULTI(flatMapArrayNested, n) {
 } // namespace
 
 int main(int argc, char* argv[]) {
-  folly::init(&argc, &argv);
+  folly::Init init{&argc, &argv};
   gflags::ParseCommandLineFlags(&argc, &argv, true);
   memory::MemoryManager::initialize({});
   folly::runBenchmarks();

velox/benchmarks/basic/VectorSlice.cpp

@@ -110,7 +110,7 @@ DEFINE_BENCHMARKS(row)
 } // namespace facebook::velox
 
 int main(int argc, char* argv[]) {
-  folly::init(&argc, &argv);
+  folly::Init init{&argc, &argv};
   using namespace facebook::velox;
   gflags::ParseCommandLineFlags(&argc, &argv, true);
   VELOX_CHECK_LE(FLAGS_slice_size, kVectorSize);

velox/benchmarks/tpch/CMakeLists.txt

@@ -33,8 +33,8 @@ target_link_libraries(
   velox_type_fbhive
   velox_caching
   velox_vector_test_lib
-  Folly::folly
   ${FOLLY_BENCHMARK}
+  Folly::folly
   fmt::fmt)
 
 add_executable(velox_tpch_benchmark TpchBenchmarkMain.cpp)

velox/benchmarks/tpch/TpchBenchmarkMain.cpp

@@ -23,6 +23,6 @@ int main(int argc, char** argv) {
   std::string kUsage(
       "This program benchmarks TPC-H queries. Run 'velox_tpch_benchmark -helpon=TpchBenchmark' for available options.\n");
   gflags::SetUsageMessage(kUsage);
-  folly::init(&argc, &argv, false);
+  folly::Init init{&argc, &argv, false};
   tpchBenchmarkMain();
 }

velox/benchmarks/unstable/MemoryAllocationBenchmark.cpp

@@ -435,7 +435,7 @@ BENCHMARK_RELATIVE_MULTI(MmapReallocateMix64) {
 } // namespace
 
 int main(int argc, char* argv[]) {
-  folly::init(&argc, &argv);
+  folly::Init init{&argc, &argv};
   // TODO: add to run benchmark as a standalone program with multithreading as
   // well as actual memory access to trigger minor page faults in OS which traps
   // into kernel context to setup physical pages for the lazy-mapped virtual

velox/common/base/benchmarks/BitUtilBenchmark.cpp

@@ -191,7 +191,7 @@ BENCHMARK_RELATIVE_MULTI(forEachBitFirstBitFalse) {
 } // namespace facebook
 
 int main(int argc, char** argv) {
-  folly::init(&argc, &argv);
+  folly::Init init{&argc, &argv};
   folly::runBenchmarks();
   return 0;
 }

velox/common/base/tests/Memcpy.cpp

@@ -58,7 +58,7 @@ struct CopyCallable {
 
 int main(int argc, char** argv) {
   constexpr int32_t kAlignment = folly::hardware_destructive_interference_size;
-  folly::init(&argc, &argv);
+  folly::Init init{&argc, &argv};
   auto chunk = bits::roundUp(
       std::max<int64_t>(FLAGS_bytes / FLAGS_threads, kAlignment), kAlignment);
   int64_t bytes = chunk * FLAGS_threads;

velox/common/base/tests/StatsReporterTest.cpp

@@ -134,7 +134,7 @@ folly::Singleton<BaseStatsReporter> reporter([]() {
 
 int main(int argc, char** argv) {
   testing::InitGoogleTest(&argc, argv);
-  folly::init(&argc, &argv, false);
+  folly::Init init{&argc, &argv, false};
   facebook::velox::BaseStatsReporter::registered = true;
   return RUN_ALL_TESTS();
 }

velox/common/file/benchmark/ReadBenchmarkMain.cpp

@@ -24,7 +24,7 @@ using namespace facebook::velox;
 // and the IO throughput is 100 MBps, then it takes 10 seconds to just read the
 // data.
 int main(int argc, char** argv) {
-  folly::init(&argc, &argv, false);
+  folly::Init init{&argc, &argv, false};
   ReadBenchmark bm;
   bm.initialize();
   bm.run();

velox/common/memory/tests/MemoryPoolBenchmark.cpp

@@ -239,7 +239,7 @@ BENCHMARK(FlatSticks, iters) {
 }
 
 int main(int argc, char* argv[]) {
-  folly::init(&argc, &argv);
+  folly::Init init{&argc, &argv};
   folly::runBenchmarks();
   return 0;
 }

velox/common/process/ThreadDebugInfo.h

@@ -47,9 +47,9 @@ const ThreadDebugInfo* GetThreadDebugInfo();
 
 // Install a signal handler to dump thread local debug information. This should
 // be called before calling folly::symbolizer::installFatalSignalCallbacks()
-// which is usually called at startup via folly::Init(). This is just a default
-// implementation but you can install your own signal handler. Make sure to
-// install one at the start of your program.
+// which is usually called at startup via folly::Init init{}. This is just a
+// default implementation but you can install your own signal handler. Make sure
+// to install one at the start of your program.
 void addDefaultFatalSignalHandler();
 
 } // namespace facebook::velox::process

velox/connectors/hive/benchmarks/HivePartitionFunctionBenchmark.cpp

@@ -329,7 +329,7 @@ BENCHMARK_DRAW_LINE();
 } // namespace
 
 int main(int argc, char** argv) {
-  folly::init(&argc, &argv);
+  folly::Init init{&argc, &argv};
   gflags::ParseCommandLineFlags(&argc, &argv, true);
   memory::MemoryManager::initialize({});
   benchmarkFew = std::make_unique<HivePartitionFunctionBenchmark>(1'000);

velox/connectors/hive/storage_adapters/gcs/benchmark/GCSReadBenchmarkMain.cpp

@@ -24,7 +24,7 @@ using namespace facebook::velox;
 // and the IO throughput is 100 MBps, then it takes 10 seconds to just read the
 // data.
 int main(int argc, char** argv) {
-  folly::init(&argc, &argv, false);
+  folly::Init init{&argc, &argv, false};
   GCSReadBenchmark bm;
   bm.initialize();
   bm.run();

velox/connectors/hive/storage_adapters/s3fs/benchmark/S3ReadBenchmarkMain.cpp

@@ -24,7 +24,7 @@ using namespace facebook::velox;
 // and the IO throughput is 100 MBps, then it takes 10 seconds to just read the
 // data.
 int main(int argc, char** argv) {
-  folly::init(&argc, &argv, false);
+  folly::Init init{&argc, &argv, false};
   S3ReadBenchmark bm;
   bm.initialize();
   bm.run();

velox/connectors/hive/storage_adapters/s3fs/tests/S3InsertTest.cpp

@@ -119,6 +119,6 @@ TEST_F(S3InsertTest, s3InsertTest) {
 
 int main(int argc, char** argv) {
   testing::InitGoogleTest(&argc, argv);
-  folly::init(&argc, &argv, false);
+  folly::Init init{&argc, &argv, false};
   return RUN_ALL_TESTS();
 }

velox/connectors/hive/storage_adapters/s3fs/tests/S3MultipleEndpointsTest.cpp

@@ -174,6 +174,6 @@ TEST_F(S3MultipleEndpoints, s3Join) {
 
 int main(int argc, char** argv) {
   testing::InitGoogleTest(&argc, argv);
-  folly::init(&argc, &argv, false);
+  folly::Init init{&argc, &argv, false};
   return RUN_ALL_TESTS();
 }

velox/connectors/hive/storage_adapters/s3fs/tests/S3ReadTest.cpp

@@ -90,6 +90,6 @@ TEST_F(S3ReadTest, s3ReadTest) {
 
 int main(int argc, char** argv) {
   testing::InitGoogleTest(&argc, argv);
-  folly::init(&argc, &argv, false);
+  folly::Init init{&argc, &argv, false};
   return RUN_ALL_TESTS();
 }

velox/connectors/hive/tests/HiveDataSinkTest.cpp

@@ -906,6 +906,6 @@ int main(int argc, char** argv) {
   testing::InitGoogleTest(&argc, argv);
   // Signal handler required for ThreadDebugInfoTest
   facebook::velox::process::addDefaultFatalSignalHandler();
-  folly::init(&argc, &argv, false);
+  folly::Init init{&argc, &argv, false};
   return RUN_ALL_TESTS();
 }

velox/connectors/tpch/tests/SpeedTest.cpp

@@ -180,7 +180,7 @@ class TpchSpeedTest {
 } // namespace
 
 int main(int argc, char** argv) {
-  folly::init(&argc, &argv, false);
+  folly::Init init{&argc, &argv, false};
 
   TpchSpeedTest speedTest;
   speedTest.run(

velox/connectors/tpch/tests/TpchConnectorTest.cpp

@@ -290,6 +290,6 @@ TEST_F(TpchConnectorTest, orderDateCount) {
 
 int main(int argc, char** argv) {
   testing::InitGoogleTest(&argc, argv);
-  folly::init(&argc, &argv, false);
+  folly::Init init{&argc, &argv, false};
   return RUN_ALL_TESTS();
 }

velox/dwio/common/tests/BitPackDecoderBenchmark.cpp

@@ -593,7 +593,7 @@ void naiveDecodeBitsLE(
 }
 
 int32_t main(int32_t argc, char* argv[]) {
-  folly::init(&argc, &argv);
+  folly::Init init{&argc, &argv};
 
   // Populate uint32 buffer
 

velox/dwio/common/tests/DataBufferBenchmark.cpp

@@ -53,7 +53,7 @@ BENCHMARK(ChainedBufferOps, iters) {
 }
 
 int main(int argc, char* argv[]) {
-  folly::init(&argc, &argv);
+  folly::Init init{&argc, &argv};
   folly::runBenchmarks();
   facebook::velox::memory::MemoryManager::initialize({});
   return 0;

velox/dwio/common/tests/IntDecoderBenchmark.cpp

@@ -939,7 +939,7 @@ BENCHMARK_RELATIVE(decodeNew_64) {
 }
 
 int32_t main(int32_t argc, char* argv[]) {
-  folly::init(&argc, &argv);
+  folly::Init init{&argc, &argv};
 
   // Populate uint16 buffer
   buffer_u16.resize(kNumElements);