From 54a995cd2206995f6dd675cabdce12a4b7ff7540 Mon Sep 17 00:00:00 2001 From: Igor Babaev <igor@askmonty.org> Date: Thu, 20 Apr 2017 13:09:31 -0700 Subject: [PATCH] Fixed the bug mdev-12519. This patch fixed some problems that occurred with subqueries that contained directly or indirectly recursive references to recursive CTEs. 1. A [NOT] IN predicate with a constant left operand and a non-correlated subquery as the right operand used in the specification of a recursive CTE was considered as a constant predicate and was evaluated only once. Now such a predicate is re-evaluated after every iteration of the process that produces the records of the recursive CTE. 2. The Exists-To-IN transformation could be applied to [NOT] IN predicates with recursive references. This opened a possibility of materialization for the subqueries used as right operands. Yet, materialization is prohibited for the subqueries if they contain a recursive reference. Now the Exists-To-IN transformation cannot be applied for subquery predicates with recursive references. The function st_select_lex::check_subqueries_with_recursive_references() is called now only for the first execution of the SELECT. --- mysql-test/r/cte_recursive.result | 210 ++++++++++++++++++++++++++++++ mysql-test/t/cte_recursive.test | 173 ++++++++++++++++++++++++ sql/item_subselect.cc | 34 ++++- sql/item_subselect.h | 8 ++ sql/sql_cte.cc | 1 + sql/sql_cte.h | 3 + sql/sql_select.cc | 3 +- sql/sql_union.cc | 9 +- 8 files changed, 433 insertions(+), 8 deletions(-) diff --git a/mysql-test/r/cte_recursive.result b/mysql-test/r/cte_recursive.result index 7ec2b7a7072..368d7e571ed 100644 --- a/mysql-test/r/cte_recursive.result +++ b/mysql-test/r/cte_recursive.result @@ -2549,3 +2549,213 @@ m m1 m2 drop table value_nodes, module_nodes, module_arguments, module_results; +# +# mdev-12519: recursive references in subqueries +# +create table t1 (lp char(4) not null, rp char(4) not null); +insert into t1 values +('p1','p2'), ('p2','p3'), ('p3','p4'), ('p4','p5'), +('p2','p7'), ('p7','p8'), ('p8','p3'), ('p8','p4'); +set standard_compliant_cte=0; +with recursive +reachables(p) as +( +select lp from t1 where lp = 'p1' + union +select t1.rp from reachables, t1 +where t1.lp = reachables.p +) +select * from reachables; +p +p1 +p2 +p3 +p7 +p4 +p8 +p5 +with recursive +reachables(p) as +( +select lp from t1 where lp = 'p1' + union +select t1.rp from reachables, t1 +where 'p3' not in (select * from reachables) and +t1.lp = reachables.p +) +select * from reachables; +p +p1 +p2 +p3 +p7 +with recursive +reachables(p) as +( +select lp from t1 where lp = 'p1' + union +select t1.rp from reachables, t1 +where 'p3' not in (select p from reachables where p <= 'p5' + union +select p from reachables where p > 'p5') and +t1.lp = reachables.p +) +select * from reachables; +p +p1 +p2 +p3 +p7 +prepare stmt from " +with recursive +reachables(p) as +( + select lp from t1 where lp = 'p1' + union + select t1.rp from reachables, t1 + where 'p3' not in (select p from reachables where p <= 'p5' + union + select p from reachables where p > 'p5') and + t1.lp = reachables.p +) +select * from reachables; +"; +execute stmt; +p +p1 +p2 +p3 +p7 +execute stmt; +p +p1 +p2 +p3 +p7 +deallocate prepare stmt; +drop table t1; +create table objects(v char(4) not null); +insert into objects values +('v1'), ('v2'), ('v3'), ('v4'), ('v5'), +('v6'), ('v7'), ('v8'), ('v9'), ('v10'); +create table modules(m char(4) not null); +insert into modules values +('m1'), ('m2'), ('m3'), ('m4'); +create table module_arguments(m char(4) not null, v char(4) not null); +insert into module_arguments values +('m1','v3'), ('m1','v9'), +('m2','v4'), ('m2','v7'), +('m3','v6'), ('m4','v2'); +create table module_results(m char(4) not null, v char(4) not null); +insert into module_results values +('m1','v4'), +('m2','v1'), ('m2','v6'), +('m3','v10'), ('m4','v7'); +set standard_compliant_cte=0; +with recursive +reached_objects as +( +select v, 'init' as m from objects where v in ('v3','v7','v9') +union +select module_results.v, module_results.m from module_results, applied_modules +where module_results.m = applied_modules.m +), +applied_modules as +( +select * from modules where 1=0 +union +select modules.m +from +modules +where +not exists (select * from module_arguments +where module_arguments.m = modules.m and +module_arguments.v not in +(select v from reached_objects)) +) +select * from reached_objects; +v m +v3 init +v7 init +v9 init +v4 m1 +v1 m2 +v6 m2 +v10 m3 +with recursive +reached_objects as +( +select v, 'init' as m from objects where v in ('v3','v7','v9') +union +select module_results.v, module_results.m from module_results, applied_modules +where module_results.m = applied_modules.m +), +applied_modules as +( +select * from modules where 1=0 +union +select modules.m +from +modules +where +'v6' not in (select v from reached_objects) and +not exists (select * from module_arguments +where module_arguments.m = modules.m and +module_arguments.v not in +(select v from reached_objects)) +) +select * from reached_objects; +v m +v3 init +v7 init +v9 init +v4 m1 +v1 m2 +v6 m2 +prepare stmt from " +with recursive +reached_objects as +( + select v, 'init' as m from objects where v in ('v3','v7','v9') + union + select module_results.v, module_results.m from module_results, applied_modules + where module_results.m = applied_modules.m +), +applied_modules as +( + select * from modules where 1=0 + union + select modules.m + from + modules + where + 'v6' not in (select v from reached_objects) and + not exists (select * from module_arguments + where module_arguments.m = modules.m and + module_arguments.v not in + (select v from reached_objects)) +) +select * from reached_objects; +"; +execute stmt; +v m +v3 init +v7 init +v9 init +v4 m1 +v1 m2 +v6 m2 +execute stmt; +v m +v3 init +v7 init +v9 init +v4 m1 +v1 m2 +v6 m2 +deallocate prepare stmt; +drop table objects, modules, module_arguments, module_results; +set standard_compliant_cte=default; +select @@standard_compliant_cte; +@@standard_compliant_cte +1 diff --git a/mysql-test/t/cte_recursive.test b/mysql-test/t/cte_recursive.test index dd49582e719..26dbe2bccc0 100644 --- a/mysql-test/t/cte_recursive.test +++ b/mysql-test/t/cte_recursive.test @@ -1661,3 +1661,176 @@ select * from applied_modules; drop table value_nodes, module_nodes, module_arguments, module_results; +--echo # +--echo # mdev-12519: recursive references in subqueries +--echo # + +create table t1 (lp char(4) not null, rp char(4) not null); +insert into t1 values + ('p1','p2'), ('p2','p3'), ('p3','p4'), ('p4','p5'), + ('p2','p7'), ('p7','p8'), ('p8','p3'), ('p8','p4'); + +set standard_compliant_cte=0; + +with recursive +reachables(p) as +( + select lp from t1 where lp = 'p1' + union + select t1.rp from reachables, t1 + where t1.lp = reachables.p +) +select * from reachables; + +with recursive +reachables(p) as +( + select lp from t1 where lp = 'p1' + union + select t1.rp from reachables, t1 + where 'p3' not in (select * from reachables) and + t1.lp = reachables.p +) +select * from reachables; + +with recursive +reachables(p) as +( + select lp from t1 where lp = 'p1' + union + select t1.rp from reachables, t1 + where 'p3' not in (select p from reachables where p <= 'p5' + union + select p from reachables where p > 'p5') and + t1.lp = reachables.p +) +select * from reachables; + +prepare stmt from " +with recursive +reachables(p) as +( + select lp from t1 where lp = 'p1' + union + select t1.rp from reachables, t1 + where 'p3' not in (select p from reachables where p <= 'p5' + union + select p from reachables where p > 'p5') and + t1.lp = reachables.p +) +select * from reachables; +"; + +execute stmt; +execute stmt; + +deallocate prepare stmt; + +drop table t1; + +create table objects(v char(4) not null); +insert into objects values + ('v1'), ('v2'), ('v3'), ('v4'), ('v5'), + ('v6'), ('v7'), ('v8'), ('v9'), ('v10'); + +create table modules(m char(4) not null); +insert into modules values + ('m1'), ('m2'), ('m3'), ('m4'); + +create table module_arguments(m char(4) not null, v char(4) not null); +insert into module_arguments values + ('m1','v3'), ('m1','v9'), + ('m2','v4'), ('m2','v7'), + ('m3','v6'), ('m4','v2'); + +create table module_results(m char(4) not null, v char(4) not null); +insert into module_results values + ('m1','v4'), + ('m2','v1'), ('m2','v6'), + ('m3','v10'), ('m4','v7'); + +set standard_compliant_cte=0; + +with recursive +reached_objects as +( + select v, 'init' as m from objects where v in ('v3','v7','v9') + union + select module_results.v, module_results.m from module_results, applied_modules + where module_results.m = applied_modules.m +), +applied_modules as +( + select * from modules where 1=0 + union + select modules.m + from + modules + where + not exists (select * from module_arguments + where module_arguments.m = modules.m and + module_arguments.v not in + (select v from reached_objects)) +) +select * from reached_objects; + +with recursive +reached_objects as +( + select v, 'init' as m from objects where v in ('v3','v7','v9') + union + select module_results.v, module_results.m from module_results, applied_modules + where module_results.m = applied_modules.m +), +applied_modules as +( + select * from modules where 1=0 + union + select modules.m + from + modules + where + 'v6' not in (select v from reached_objects) and + not exists (select * from module_arguments + where module_arguments.m = modules.m and + module_arguments.v not in + (select v from reached_objects)) +) +select * from reached_objects; + +prepare stmt from " +with recursive +reached_objects as +( + select v, 'init' as m from objects where v in ('v3','v7','v9') + union + select module_results.v, module_results.m from module_results, applied_modules + where module_results.m = applied_modules.m +), +applied_modules as +( + select * from modules where 1=0 + union + select modules.m + from + modules + where + 'v6' not in (select v from reached_objects) and + not exists (select * from module_arguments + where module_arguments.m = modules.m and + module_arguments.v not in + (select v from reached_objects)) +) +select * from reached_objects; +"; + +execute stmt; +execute stmt; + +deallocate prepare stmt; + +drop table objects, modules, module_arguments, module_results; + +set standard_compliant_cte=default; +select @@standard_compliant_cte; + diff --git a/sql/item_subselect.cc b/sql/item_subselect.cc index 76c754d5627..d144dddf764 100644 --- a/sql/item_subselect.cc +++ b/sql/item_subselect.cc @@ -40,6 +40,7 @@ #include "set_var.h" #include "sql_select.h" #include "sql_parse.h" // check_stack_overrun +#include "sql_cte.h" #include "sql_test.h" double get_post_group_estimate(JOIN* join, double join_op_rows); @@ -312,7 +313,8 @@ bool Item_subselect::fix_fields(THD *thd_param, Item **ref) else goto end; - if ((uncacheable= engine->uncacheable() & ~UNCACHEABLE_EXPLAIN)) + if ((uncacheable= engine->uncacheable() & ~UNCACHEABLE_EXPLAIN) || + with_recursive_reference) { const_item_cache= 0; if (uncacheable & UNCACHEABLE_RAND) @@ -917,7 +919,7 @@ table_map Item_subselect::used_tables() const bool Item_subselect::const_item() const { DBUG_ASSERT(thd); - return (thd->lex->context_analysis_only ? + return (thd->lex->context_analysis_only || with_recursive_reference ? FALSE : forced_const || const_item_cache); } @@ -937,7 +939,8 @@ void Item_subselect::update_used_tables() if (!(engine->uncacheable() & ~UNCACHEABLE_EXPLAIN)) { // did all used tables become static? - if (!(used_tables_cache & ~engine->upper_select_const_tables())) + if (!(used_tables_cache & ~engine->upper_select_const_tables()) && + ! with_recursive_reference) const_item_cache= 1; } } @@ -1735,7 +1738,7 @@ bool Item_in_subselect::val_bool() if (forced_const) return value; DBUG_ASSERT((engine->uncacheable() & ~UNCACHEABLE_EXPLAIN) || - ! engine->is_executed()); + ! engine->is_executed() || with_recursive_reference); null_value= was_null= FALSE; if (exec()) { @@ -2828,7 +2831,8 @@ bool Item_exists_subselect::exists2in_processor(void *opt_arg) join->having || first_select->with_sum_func || !first_select->leaf_tables.elements|| - !join->conds) + !join->conds || + with_recursive_reference) DBUG_RETURN(FALSE); DBUG_ASSERT(first_select->order_list.elements == 0 && @@ -3480,6 +3484,11 @@ int subselect_single_select_engine::get_identifier() return select_lex->select_number; } +void subselect_single_select_engine::force_reexecution() +{ + executed= false; +} + void subselect_single_select_engine::cleanup() { DBUG_ENTER("subselect_single_select_engine::cleanup"); @@ -3508,6 +3517,11 @@ bool subselect_union_engine::is_executed() const return unit->executed; } +void subselect_union_engine::force_reexecution() +{ + unit->executed= false; +} + /* Check if last execution of the subquery engine produced any rows @@ -3827,7 +3841,8 @@ int subselect_single_select_engine::exec() tab->read_record.read_record= tab->save_read_record; } executed= 1; - if (!(uncacheable() & ~UNCACHEABLE_EXPLAIN)) + if (!(uncacheable() & ~UNCACHEABLE_EXPLAIN) && + !item->with_recursive_reference) item->make_const(); thd->where= save_where; thd->lex->current_select= save_select; @@ -6674,6 +6689,13 @@ void subselect_table_scan_engine::cleanup() } +void Item_subselect::register_as_with_rec_ref(With_element *with_elem) +{ + with_elem->sq_with_rec_ref.link_in_list(this, &this->next_with_rec_ref); + with_recursive_reference= true; +} + + /* Create an execution tracker for the expression cache we're using for this subselect; add the tracker to the query plan. diff --git a/sql/item_subselect.h b/sql/item_subselect.h index aa01f571a3d..83bd9ed0bdf 100644 --- a/sql/item_subselect.h +++ b/sql/item_subselect.h @@ -32,6 +32,7 @@ class subselect_engine; class subselect_hash_sj_engine; class Item_bool_func2; class Comp_creator; +class With_element; typedef class st_select_lex SELECT_LEX; @@ -135,6 +136,9 @@ public: */ bool with_recursive_reference; + /* To link Item_subselects containing references to the same recursive CTE */ + Item_subselect *next_with_rec_ref; + enum subs_type {UNKNOWN_SUBS, SINGLEROW_SUBS, EXISTS_SUBS, IN_SUBS, ALL_SUBS, ANY_SUBS}; @@ -256,6 +260,7 @@ public: return TRUE; } + void register_as_with_rec_ref(With_element *with_elem); void init_expr_cache_tracker(THD *thd); Item* build_clone(THD *thd, MEM_ROOT *mem_root) { return 0; } @@ -836,6 +841,7 @@ public: virtual bool no_rows() = 0; virtual enum_engine_type engine_type() { return ABSTRACT_ENGINE; } virtual int get_identifier() { DBUG_ASSERT(0); return 0; } + virtual void force_reexecution() {} protected: void set_row(List<Item> &item_list, Item_cache **row); }; @@ -869,6 +875,7 @@ public: bool no_rows(); virtual enum_engine_type engine_type() { return SINGLE_SELECT_ENGINE; } int get_identifier(); + void force_reexecution(); friend class subselect_hash_sj_engine; friend class Item_in_subselect; @@ -899,6 +906,7 @@ public: bool temp= FALSE); bool no_tables(); bool is_executed() const; + void force_reexecution(); bool no_rows(); virtual enum_engine_type engine_type() { return UNION_ENGINE; } }; diff --git a/sql/sql_cte.cc b/sql/sql_cte.cc index d76ee13a010..c9976baeabd 100644 --- a/sql/sql_cte.cc +++ b/sql/sql_cte.cc @@ -1229,6 +1229,7 @@ bool st_select_lex::check_subqueries_with_recursive_references() continue; Item_subselect *subq= (Item_subselect *) sl_master->item; subq->with_recursive_reference= true; + subq->register_as_with_rec_ref(tbl->with); } } return false; diff --git a/sql/sql_cte.h b/sql/sql_cte.h index 27d5923ad07..1cb77af8099 100644 --- a/sql/sql_cte.h +++ b/sql/sql_cte.h @@ -118,6 +118,9 @@ public: stage and is used at the execution stage. */ select_union_recursive *rec_result; + + /* List of Item_subselects containing recursive references to this CTE */ + SQL_I_List<Item_subselect> sq_with_rec_ref; With_element(LEX_STRING *name, List <LEX_STRING> list, diff --git a/sql/sql_select.cc b/sql/sql_select.cc index ecbac041dc4..fbddab20908 100644 --- a/sql/sql_select.cc +++ b/sql/sql_select.cc @@ -868,7 +868,8 @@ JOIN::prepare(TABLE_LIST *tables_init, select_lex->check_unrestricted_recursive( thd->variables.only_standard_compliant_cte)) DBUG_RETURN(-1); - select_lex->check_subqueries_with_recursive_references(); + if (select_lex->first_execution) + select_lex->check_subqueries_with_recursive_references(); int res= check_and_do_in_subquery_rewrites(this); diff --git a/sql/sql_union.cc b/sql/sql_union.cc index f8fe8b0be00..58601a178b7 100644 --- a/sql/sql_union.cc +++ b/sql/sql_union.cc @@ -930,7 +930,8 @@ bool st_select_lex_unit::exec() if (executed && !uncacheable && !describe) DBUG_RETURN(FALSE); executed= 1; - if (!(uncacheable & ~UNCACHEABLE_EXPLAIN) && item) + if (!(uncacheable & ~UNCACHEABLE_EXPLAIN) && item && + !item->with_recursive_reference) item->make_const(); saved_error= optimize(); @@ -1263,6 +1264,12 @@ bool st_select_lex_unit::exec_recursive() if (with_element->level == 1) rec_table->reginfo.join_tab->preread_init_done= true; } + for (Item_subselect *sq= with_element->sq_with_rec_ref.first; + sq; + sq= sq->next_with_rec_ref) + { + sq->engine->force_reexecution(); + } thd->lex->current_select= lex_select_save; err: