From a0e7bd735b5bc1fd458766c2005d5ac349682d9f Mon Sep 17 00:00:00 2001
From: Sergei Petrunia <sergey@mariadb.com>
Date: Thu, 1 Jun 2023 14:06:06 +0300
Subject: [PATCH] MDEV-31380: Assertion `s->table->opt_range_condition_rows <=
 s->found_records' failed

LooseScan code set opt_range_condition_rows to be the

  MIN(loose_scan_plan->records, table->records)

totally ignoring possible quick range selects.  If there was a quick
select $QUICK on another index with

  $QUICK->records < loose_scan_plan->records

this would create a situation where

   opt_range_condition_rows > $QUICK->records

which causes an assert in 10.6+ and potentially wrong query plan
choice in 10.5.

Fixed by making opt_range_condition_rows to be the minimum #rows
of any quick select.

Approved-by: Monty <monty@mariadb.org>
---
 mysql-test/main/group_min_max.result | 24 ++++++++++++++++++++++++
 mysql-test/main/group_min_max.test   | 25 +++++++++++++++++++++++++
 sql/opt_range.cc                     |  4 ++--
 3 files changed, 51 insertions(+), 2 deletions(-)

diff --git a/mysql-test/main/group_min_max.result b/mysql-test/main/group_min_max.result
index 712466c8afb..0df9abc7a25 100644
--- a/mysql-test/main/group_min_max.result
+++ b/mysql-test/main/group_min_max.result
@@ -4204,6 +4204,30 @@ a	b	s1
 2	2	t2:t2a-null;min_t3_b:t3b-null
 3	3	t2:1;min_t3_b:3
 drop table t1,t2,t3;
+# 
+# MDEV-31380: Assertion `s->table->opt_range_condition_rows <= s->found_records' failed
+#  (assertion in 10.6+, DBL_MAX costs in 10.5)
+# 
+CREATE TABLE t1 (a INT, b INT, PRIMARY KEY(a), KEY(b)) ENGINE=InnoDB;
+INSERT INTO t1 SELECT seq, seq FROM seq_1_to_100;
+SET
+@tmp=@@optimizer_use_condition_selectivity,
+optimizer_use_condition_selectivity = 1,
+@tmp2=@@optimizer_trace,
+optimizer_trace=1;
+SELECT DISTINCT * FROM t1 WHERE a IN (1, 2);
+a	b
+1	1
+2	2
+select 
+CAST(json_value(json_extract(trace, '$**.chosen_access_method.cost'), '$[0]')
+as DOUBLE) < 1.0e100
+from information_schema.optimizer_trace;
+CAST(json_value(json_extract(trace, '$**.chosen_access_method.cost'), '$[0]')
+as DOUBLE) < 1.0e100
+1
+set optimizer_use_condition_selectivity = @tmp, optimizer_trace=@tmp2;
+drop table t1;
 #
 # End of 10.5 tests
 #
diff --git a/mysql-test/main/group_min_max.test b/mysql-test/main/group_min_max.test
index 1fc2be6231a..482235571db 100644
--- a/mysql-test/main/group_min_max.test
+++ b/mysql-test/main/group_min_max.test
@@ -6,6 +6,7 @@
 --source include/no_valgrind_without_big.inc
 --source include/default_optimizer_switch.inc
 --source include/have_innodb.inc
+--source include/have_sequence.inc
 #
 # TODO:
 # Add queries with:
@@ -1858,6 +1859,30 @@ from t1;
 
 drop table t1,t2,t3;
 
+--echo # 
+--echo # MDEV-31380: Assertion `s->table->opt_range_condition_rows <= s->found_records' failed
+--echo #  (assertion in 10.6+, DBL_MAX costs in 10.5)
+--echo # 
+
+CREATE TABLE t1 (a INT, b INT, PRIMARY KEY(a), KEY(b)) ENGINE=InnoDB;
+INSERT INTO t1 SELECT seq, seq FROM seq_1_to_100;
+
+SET
+  @tmp=@@optimizer_use_condition_selectivity,
+  optimizer_use_condition_selectivity = 1,
+  @tmp2=@@optimizer_trace,
+  optimizer_trace=1;
+
+SELECT DISTINCT * FROM t1 WHERE a IN (1, 2);
+
+select 
+  CAST(json_value(json_extract(trace, '$**.chosen_access_method.cost'), '$[0]')
+       as DOUBLE) < 1.0e100
+from information_schema.optimizer_trace;
+
+set optimizer_use_condition_selectivity = @tmp, optimizer_trace=@tmp2;
+drop table t1;
+
 --echo #
 --echo # End of 10.5 tests
 --echo #
diff --git a/sql/opt_range.cc b/sql/opt_range.cc
index 7b6f373eea4..905d5aa5ef9 100644
--- a/sql/opt_range.cc
+++ b/sql/opt_range.cc
@@ -3030,8 +3030,8 @@ int SQL_SELECT::test_quick_select(THD *thd, key_map keys_to_use,
         restore_nonrange_trees(&param, tree, backup_keys);
       if ((group_trp= get_best_group_min_max(&param, tree, read_time)))
       {
-        param.table->opt_range_condition_rows= MY_MIN(group_trp->records,
-                                                  head->stat_records());
+        set_if_smaller(param.table->opt_range_condition_rows,
+                       group_trp->records);
         Json_writer_object grp_summary(thd, "best_group_range_summary");
 
         if (unlikely(thd->trace_started()))