MDEV-35253: xa_prepare_unlock_unmodified fails: shift exponent 32 is too large

The code in best_access_path() uses PREV_BITS(uint, N) to
compute a bitmap of all keyparts: {keypart0, ... keypart{N-1}).

The problem is that PREV_BITS($type, N) macro code can't handle the case
when N=<number of bits in $type).
Also, why use PREV_BITS(uint, ...) for key part map computations when
we could have used PREV_BITS(key_part_map) ?

Fixed both:
- Change PREV_BITS(type, N) to handle any N in [0; n_bits(type)].
- Change PREV_BITS() to use key_part_map when computing key_part_map bitmaps.

include/my_global.h

@@ -819,7 +819,6 @@ typedef long long	my_ptrdiff_t;
 #define ALIGN_PTR(A, t) ((t*) MY_ALIGN((A), sizeof(double)))
 #define ADD_TO_PTR(ptr,size,type) (type) ((uchar*) (ptr)+size)
 #define PTR_BYTE_DIFF(A,B) (my_ptrdiff_t) ((uchar*) (A) - (uchar*) (B))
-#define PREV_BITS(type,A)	((type) (((type) 1 << (A)) -1))
 
 /*
   Custom version of standard offsetof() macro which can be used to get

mysql-test/main/join_optimizer.result

@@ -35,3 +35,25 @@ id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
 1	SIMPLE	a	ALL	NULL	NULL	NULL	NULL	2	Using where
 1	SIMPLE	g	ref	groups_dt	groups_dt	70	const,test.a.type	13	Using index condition
 drop table t0,t1,t2,t3;
+#
+# MDEV-35253: xa_prepare_unlock_unmodified fails: shift exponent 32 is too large
+#
+set @create=
+concat("create table t1(",
+(select group_concat(concat("col",seq, " int")) from seq_1_to_32),
+",\n index idx1(",
+(select group_concat(concat("col",seq)) from seq_1_to_32),
+")\n)"
+  );
+$create_tbl;
+insert into t1()  values (),(),();
+analyze table t1;
+Table	Op	Msg_type	Msg_text
+test.t1	analyze	status	Engine-independent statistics collected
+test.t1	analyze	status	OK
+# Must not produce a "shift exponent 32 is too large" runtime ubsan error
+explain select * from t1 where col1=1;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	SIMPLE	t1	ref	idx1	idx1	5	const	1	Using index
+drop table t1;
+End of 10.5 tests

mysql-test/main/join_optimizer.test

@@ -45,3 +45,27 @@ SELECT STRAIGHT_JOIN g.id FROM t2 a, t3 g USE INDEX(groups_dt)
 WHERE g.domain = 'queue' AND g.type = a.type;
 
 drop table t0,t1,t2,t3;
+
+--echo #
+--echo # MDEV-35253: xa_prepare_unlock_unmodified fails: shift exponent 32 is too large
+--echo #
+--source include/have_sequence.inc
+set @create=
+ concat("create table t1(",
+   (select group_concat(concat("col",seq, " int")) from seq_1_to_32),
+   ",\n index idx1(",
+   (select group_concat(concat("col",seq)) from seq_1_to_32),
+   ")\n)"
+  );
+
+let $create_tbl=`select @create`;
+evalp $create_tbl;
+insert into t1()  values (),(),();
+analyze table t1;
+
+--echo # Must not produce a "shift exponent 32 is too large" runtime ubsan error
+explain select * from t1 where col1=1;
+
+drop table t1;
+
+--echo End of 10.5 tests

sql/opt_subselect.cc

@@ -2752,7 +2752,8 @@ bool find_eq_ref_candidate(TABLE *table, table_map sj_inner_tables)
           keyuse++;
         } while (keyuse->key == key && keyuse->table == table);
 
-        if (bound_parts == PREV_BITS(uint, keyinfo->user_defined_key_parts))
+        if (bound_parts == PREV_BITS(key_part_map,
+                                     keyinfo->user_defined_key_parts))
           return TRUE;
       }
       else

sql/sql_select.cc

@@ -7797,7 +7797,7 @@ best_access_path(JOIN      *join,
         loose_scan_opt.check_ref_access_part1(s, key, start_key, found_part);
 
         /* Check if we found full key */
-        const key_part_map all_key_parts= PREV_BITS(uint, key_parts);
+        const key_part_map all_key_parts= PREV_BITS(key_part_map, key_parts);
         if (found_part == all_key_parts && !ref_or_null_part)
         {                                         /* use eq key */
           max_key_part= (uint) ~0;
@@ -7935,7 +7935,8 @@ best_access_path(JOIN      *join,
           */
           if ((found_part & 1) &&
               (!(table->file->index_flags(key, 0, 0) & HA_ONLY_WHOLE_INDEX) ||
-               found_part == PREV_BITS(uint,keyinfo->user_defined_key_parts)))
+               found_part == PREV_BITS(key_part_map,
+                                       keyinfo->user_defined_key_parts)))
           {
             max_key_part= max_part_bit(found_part);
             /*
@@ -23636,7 +23637,7 @@ static int test_if_order_by_key(JOIN *join,
     if (have_pk_suffix &&
         reverse == 0 && // all were =const so far
         key_parts == table->key_info[idx].ext_key_parts && 
-        table->const_key_parts[pk] == PREV_BITS(uint, 
+        table->const_key_parts[pk] == PREV_BITS(key_part_map,
                                                 table->key_info[pk].
                                                 user_defined_key_parts))
     {

sql/sql_select.h

@@ -2598,4 +2598,15 @@ void propagate_new_equalities(THD *thd, Item *cond,
                               COND_EQUAL *inherited,
                               bool *is_simplifiable_cond);
 
+template<typename T> T prev_bits(T n_bits)
+{
+  if (!n_bits)
+    return 0;
+  T tmp= ((T)1 << (n_bits - 1));
+  return (tmp - 1) | tmp;
+}
+// A wrapper for the above function:
+#define PREV_BITS(type, A) prev_bits<type>(A)
+
+
 #endif /* SQL_SELECT_INCLUDED */