Add support for M_PURGE_ALL.

This is a new mallopt option that will force purge absolutely
everything no matter how long it takes to purge.

Wrote a unit test for the new mallopt, and added a test to help
verify that new mallopt parameters do not conflict with each other.

Modified some benchmarks to use this new parameter so that we can
get better RSS data.

Added a new M_PURGE_ALL benchmark.

Bug: 243851006

Test: All unit tests pass.
Test: Ran changed benchmarks.
Change-Id: I1b46a5e6253538108e052d11ee46fd513568adec

benchmarks/malloc_benchmark.cpp

@@ -36,11 +36,11 @@
 
 #if defined(__BIONIC__)
 
-static void BM_mallopt_purge(benchmark::State& state) {
+static void RunMalloptPurge(benchmark::State& state, int purge_value) {
   static size_t sizes[] = {8, 16, 32, 64, 128, 1024, 4096, 16384, 65536, 131072, 1048576};
   static int pagesize = getpagesize();
   mallopt(M_DECAY_TIME, 1);
-  mallopt(M_PURGE, 0);
+  mallopt(M_PURGE_ALL, 0);
   for (auto _ : state) {
     state.PauseTiming();
     std::vector<void*> ptrs;
@@ -63,10 +63,19 @@ static void BM_mallopt_purge(benchmark::State& state) {
     ptrs.clear();
     state.ResumeTiming();
 
-    mallopt(M_PURGE, 0);
+    mallopt(purge_value, 0);
   }
   mallopt(M_DECAY_TIME, 0);
 }
+
+static void BM_mallopt_purge(benchmark::State& state) {
+  RunMalloptPurge(state, M_PURGE);
+}
 BIONIC_BENCHMARK(BM_mallopt_purge);
 
+static void BM_mallopt_purge_all(benchmark::State& state) {
+  RunMalloptPurge(state, M_PURGE_ALL);
+}
+BIONIC_BENCHMARK(BM_mallopt_purge_all);
+
 #endif

benchmarks/malloc_map_benchmark.cpp

@@ -69,7 +69,7 @@ static void MapBenchmark(benchmark::State& state, size_t num_elements) {
   for (auto _ : state) {
 #if defined(__BIONIC__)
     state.PauseTiming();
-    mallopt(M_PURGE, 0);
+    mallopt(M_PURGE_ALL, 0);
     uint64_t rss_bytes_before = 0;
     Gather(&rss_bytes_before);
     state.ResumeTiming();
@@ -80,7 +80,7 @@ static void MapBenchmark(benchmark::State& state, size_t num_elements) {
     }
 #if defined(__BIONIC__)
     state.PauseTiming();
-    mallopt(M_PURGE, 0);
+    mallopt(M_PURGE_ALL, 0);
     Gather(&rss_bytes);
     // Try and record only the memory used in the map.
     rss_bytes -= rss_bytes_before;

benchmarks/malloc_rss_benchmark.cpp

@@ -112,7 +112,7 @@ void StressSizeClass(size_t numThreads, size_t allocSize) {
 
   // Do an explicit purge to ensure we will be more likely to get the actual
   // in-use memory.
-  mallopt(M_PURGE, 0);
+  mallopt(M_PURGE_ALL, 0);
 
   android::meminfo::ProcMemInfo proc_mem(getpid());
   const std::vector<android::meminfo::Vma>& maps = proc_mem.MapsWithoutUsageStats();

libc/bionic/jemalloc_wrapper.cpp

@@ -102,7 +102,7 @@ int je_mallopt(int param, int value) {
       }
     }
     return 1;
-  } else if (param == M_PURGE) {
+  } else if (param == M_PURGE || param == M_PURGE_ALL) {
     // Only clear the current thread cache since there is no easy way to
     // clear the caches of other threads.
     // This must be done first so that cleared allocations get purged

libc/include/malloc.h

@@ -183,7 +183,15 @@ int malloc_info(int __must_be_zero, FILE* _Nonnull __fp) __INTRODUCED_IN(23);
  * Available since API level 28.
  */
 #define M_PURGE (-101)
-
+/**
+ * mallopt() option to immediately purge all possible memory back to
+ * the kernel. This call can take longer than a normal purge since it
+ * examines everything. In some cases, it can take more than twice the
+ * time of a M_PURGE call. The value is ignored.
+ *
+ * Available since API level 34.
+ */
+#define M_PURGE_ALL (-104)
 
 /**
  * mallopt() option to tune the allocator's choice of memory tags to

tests/malloc_test.cpp

@@ -36,7 +36,10 @@
 #include <algorithm>
 #include <atomic>
 #include <functional>
+#include <string>
 #include <thread>
+#include <unordered_map>
+#include <utility>
 #include <vector>
 
 #include <tinyxml2.h>
@@ -695,6 +698,44 @@ TEST(malloc, mallopt_purge) {
 #endif
 }
 
+TEST(malloc, mallopt_purge_all) {
+#if defined(__BIONIC__)
+  SKIP_WITH_HWASAN << "hwasan does not implement mallopt";
+  errno = 0;
+  ASSERT_EQ(1, mallopt(M_PURGE_ALL, 0));
+#else
+  GTEST_SKIP() << "bionic-only test";
+#endif
+}
+
+// Verify that all of the mallopt values are unique.
+TEST(malloc, mallopt_unique_params) {
+#if defined(__BIONIC__)
+  std::vector<std::pair<int, std::string>> params{
+      std::make_pair(M_DECAY_TIME, "M_DECAY_TIME"),
+      std::make_pair(M_PURGE, "M_PURGE"),
+      std::make_pair(M_PURGE_ALL, "M_PURGE_ALL"),
+      std::make_pair(M_MEMTAG_TUNING, "M_MEMTAG_TUNING"),
+      std::make_pair(M_THREAD_DISABLE_MEM_INIT, "M_THREAD_DISABLE_MEM_INIT"),
+      std::make_pair(M_CACHE_COUNT_MAX, "M_CACHE_COUNT_MAX"),
+      std::make_pair(M_CACHE_SIZE_MAX, "M_CACHE_SIZE_MAX"),
+      std::make_pair(M_TSDS_COUNT_MAX, "M_TSDS_COUNT_MAX"),
+      std::make_pair(M_BIONIC_ZERO_INIT, "M_BIONIC_ZERO_INIT"),
+      std::make_pair(M_BIONIC_SET_HEAP_TAGGING_LEVEL, "M_BIONIC_SET_HEAP_TAGGING_LEVEL"),
+  };
+
+  std::unordered_map<int, std::string> all_params;
+  for (const auto& param : params) {
+    EXPECT_TRUE(all_params.count(param.first) == 0)
+        << "mallopt params " << all_params[param.first] << " and " << param.second
+        << " have the same value " << param.first;
+    all_params.insert(param);
+  }
+#else
+  GTEST_SKIP() << "bionic-only test";
+#endif
+}
+
 #if defined(__BIONIC__)
 static void GetAllocatorVersion(bool* allocator_scudo) {
   TemporaryFile tf;