diff --git a/mysql-test/r/subselect_mat.result b/mysql-test/r/subselect_mat.result index 4a38ccf75b9..c0b250aa07f 100644 --- a/mysql-test/r/subselect_mat.result +++ b/mysql-test/r/subselect_mat.result @@ -1657,6 +1657,24 @@ a 0 DROP TABLE t2,t3,t4,t5; set optimizer_switch=@subselect_sj_mat_tmp; +# +# BUG#860300: Second crash with get_fanout_with_deps() with semijoin + materialization +# +set @tmp_860300=@@optimizer_switch; +set optimizer_switch='semijoin=on,materialization=on,loosescan=off,firstmatch=off'; +CREATE TABLE t1 (f2 int); +INSERT INTO t1 VALUES (9),(6); +CREATE TABLE t3 (f4 int); +CREATE TABLE t4 (f6 varchar(1)); +SELECT * +FROM t3 +WHERE 'h' IN (SELECT f6 +FROM t4 +WHERE 5 IN (SELECT f2 FROM t1) +GROUP BY t4.f6); +f4 +DROP TABLE t1,t3,t4; +set optimizer_switch=@tmp_860300; set @subselect_mat_test_optimizer_switch_value=null; set @@optimizer_switch='materialization=on,in_to_exists=off,semijoin=off'; set optimizer_switch='mrr=on,mrr_sort_keys=on,index_condition_pushdown=on'; diff --git a/mysql-test/r/subselect_sj_mat.result b/mysql-test/r/subselect_sj_mat.result index 2ce54609fa4..d406e103ed0 100644 --- a/mysql-test/r/subselect_sj_mat.result +++ b/mysql-test/r/subselect_sj_mat.result @@ -1697,3 +1697,21 @@ a 0 DROP TABLE t2,t3,t4,t5; set optimizer_switch=@subselect_sj_mat_tmp; +# +# BUG#860300: Second crash with get_fanout_with_deps() with semijoin + materialization +# +set @tmp_860300=@@optimizer_switch; +set optimizer_switch='semijoin=on,materialization=on,loosescan=off,firstmatch=off'; +CREATE TABLE t1 (f2 int); +INSERT INTO t1 VALUES (9),(6); +CREATE TABLE t3 (f4 int); +CREATE TABLE t4 (f6 varchar(1)); +SELECT * +FROM t3 +WHERE 'h' IN (SELECT f6 +FROM t4 +WHERE 5 IN (SELECT f2 FROM t1) +GROUP BY t4.f6); +f4 +DROP TABLE t1,t3,t4; +set optimizer_switch=@tmp_860300; diff --git a/mysql-test/t/subselect_sj_mat.test b/mysql-test/t/subselect_sj_mat.test index 95ef6a25df3..86dbeab8555 100644 --- a/mysql-test/t/subselect_sj_mat.test +++ b/mysql-test/t/subselect_sj_mat.test @@ -1351,3 +1351,22 @@ DROP TABLE t2,t3,t4,t5; set optimizer_switch=@subselect_sj_mat_tmp; +--echo # +--echo # BUG#860300: Second crash with get_fanout_with_deps() with semijoin + materialization +--echo # +set @tmp_860300=@@optimizer_switch; +set optimizer_switch='semijoin=on,materialization=on,loosescan=off,firstmatch=off'; +CREATE TABLE t1 (f2 int); +INSERT INTO t1 VALUES (9),(6); +CREATE TABLE t3 (f4 int); +CREATE TABLE t4 (f6 varchar(1)); + +SELECT * +FROM t3 +WHERE 'h' IN (SELECT f6 + FROM t4 + WHERE 5 IN (SELECT f2 FROM t1) + GROUP BY t4.f6); +DROP TABLE t1,t3,t4; +set optimizer_switch=@tmp_860300; + diff --git a/sql/item_subselect.cc b/sql/item_subselect.cc index 43df9fe32a9..a79d035160c 100644 --- a/sql/item_subselect.cc +++ b/sql/item_subselect.cc @@ -4394,7 +4394,13 @@ double get_fanout_with_deps(JOIN *join, table_map tset) for (JOIN_TAB *tab= first_top_level_tab(join, WITHOUT_CONST_TABLES); tab; tab= next_top_level_tab(join, tab)) { - if ((tab->table->map & checked_deps) && !tab->emb_sj_nest && + /* + Ignore SJM nests. They have tab->table==NULL. There is no point to walk + inside them, because GROUP BY clause cannot refer to tables from within + subquery. + */ + if (!tab->is_sjm_nest() && (tab->table->map & checked_deps) && + !tab->emb_sj_nest && tab->records_read != 0) { fanout *= rows2double(tab->records_read);