MDEV-32575 MSAN / Valgrind errors in test_if_cheaper_ordering upon re…

mysql-test/main/opt_tvc.result

@@ -780,3 +780,15 @@ deallocate prepare stmt;
 drop table t1;
 set in_predicate_conversion_threshold=default;
 # End of 10.3 tests
+#
+# MDEV-32575 MSAN / Valgrind errors in test_if_cheaper_ordering
+#            upon reaching in_predicate_conversion_threshold
+#
+create table t1 (a int, b int, c int, primary key (a, b));
+insert into t1 (a, b) values (1,1),(1,2),(1,3),(1,4),(1,5),(1,6),(1,7),(1,8);
+set in_predicate_conversion_threshold = 2;
+select * from t1 where a in (1, 2) and b = 2 order by a, b;
+a	b	c
+1	2	NULL
+drop table t1;
+# End of 11.4 tests

mysql-test/main/opt_tvc.test

@@ -474,3 +474,18 @@ drop table t1;
 set in_predicate_conversion_threshold=default;
 
 --echo # End of 10.3 tests
+
+--echo #
+--echo # MDEV-32575 MSAN / Valgrind errors in test_if_cheaper_ordering
+--echo #            upon reaching in_predicate_conversion_threshold
+--echo #
+
+create table t1 (a int, b int, c int, primary key (a, b));
+insert into t1 (a, b) values (1,1),(1,2),(1,3),(1,4),(1,5),(1,6),(1,7),(1,8);
+
+set in_predicate_conversion_threshold = 2;
+select * from t1 where a in (1, 2) and b = 2 order by a, b;
+
+drop table t1;
+
+--echo # End of 11.4 tests

storage/heap/ha_heap.cc

@@ -144,8 +144,6 @@ int ha_heap::open(const char *name, int mode, uint test_if_locked)
   }
 
   ref_length= sizeof(HEAP_PTR);
-  /* Initialize variables for the opened table */
-  set_keys_for_scanning();
   /*
     We cannot run update_key_stats() here because we do not have a
     lock on the table. The 'records' count might just be changed
@@ -186,32 +184,34 @@ handler *ha_heap::clone(const char *name, MEM_ROOT *mem_root)
 
 
 /*
-  Compute which keys to use for scanning
+  Return set of keys usable for scanning
 
   SYNOPSIS
-    set_keys_for_scanning()
-    no parameter
+    keys_to_use_for_scanning()
+    (no parameters)
 
   DESCRIPTION
-    Set the bitmap btree_keys, which is used when the upper layers ask
-    which keys to use for scanning. For each btree index the
-    corresponding bit is set.
+    This function populates the bitmap `btree_keys`, where each bit represents
+    a key that can be used for scanning the table. The bitmap is dynamically
+    updated on every call, ensuring it reflects the current state of the
+    table's keys. Caching is avoided because the set of usable keys for
+    MEMORY tables may change during optimization or execution.
 
   RETURN
-    void
+    Pointer to the updated bitmap of keys (`btree_keys`)
 */
 
-void ha_heap::set_keys_for_scanning(void)
+const key_map *ha_heap::keys_to_use_for_scanning()
 {
   btree_keys.clear_all();
   for (uint i= 0 ; i < table->s->keys ; i++)
   {
     if (table->key_info[i].algorithm == HA_KEY_ALG_BTREE)
       btree_keys.set_bit(i);
   }
+  return &btree_keys;
 }
 
-
 int ha_heap::can_continue_handler_scan()
 {
   int error= 0;
@@ -515,8 +515,7 @@ int ha_heap::disable_indexes(key_map map, bool persist)
   if (!persist)
   {
     DBUG_ASSERT(map.is_clear_all());
-    if (!(error= heap_disable_indexes(file)))
-      set_keys_for_scanning();
+    error= heap_disable_indexes(file);
   }
   else
   {
@@ -534,8 +533,7 @@ int ha_heap::disable_indexes(key_map map, bool persist)
     enable_indexes()
 
   DESCRIPTION
-    Enable indexes and set keys to use for scanning.
-    The indexes might have been disabled by disable_index() before.
+    Enable indexes taht might have been disabled by disable_index() before.
     The function works only if both data and indexes are empty,
     since the heap storage engine cannot repair the indexes.
     To be sure, call handler::delete_all_rows() before.
@@ -555,8 +553,7 @@ int ha_heap::enable_indexes(key_map map, bool persist)
   if (!persist)
   {
     DBUG_ASSERT(map.is_prefix(table->s->keys));
-    if (!(error= heap_enable_indexes(file)))
-      set_keys_for_scanning();
+    error= heap_enable_indexes(file);
   }
   else
   {

storage/heap/ha_heap.h

@@ -59,7 +59,7 @@ class ha_heap final : public handler
             HA_READ_NEXT | HA_READ_PREV | HA_READ_ORDER | HA_READ_RANGE :
             HA_ONLY_WHOLE_INDEX | HA_KEY_SCAN_NOT_ROR);
   }
-  const key_map *keys_to_use_for_scanning() override { return &btree_keys; }
+  const key_map *keys_to_use_for_scanning() override;
   uint max_supported_keys()          const override { return MAX_KEY; }
   uint max_supported_key_part_length() const override { return MAX_KEY_LENGTH; }
   IO_AND_CPU_COST scan_time() override;
@@ -121,5 +121,4 @@ class ha_heap final : public handler
   int find_unique_row(uchar *record, uint unique_idx) override;
 private:
   void update_key_stats();
-  void set_keys_for_scanning(void);
 };