From 58e2ca68bb86e6d1e086ab978f4b8d434ffef184 Mon Sep 17 00:00:00 2001 From: Balasubramanian Kandasamy Date: Mon, 29 May 2017 14:52:50 +0530 Subject: [PATCH 001/199] Raise version number after cloning 5.5.57 --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 68358acf754..2905c37cc09 100644 --- a/VERSION +++ b/VERSION @@ -1,4 +1,4 @@ MYSQL_VERSION_MAJOR=5 MYSQL_VERSION_MINOR=5 -MYSQL_VERSION_PATCH=57 +MYSQL_VERSION_PATCH=58 MYSQL_VERSION_EXTRA= From def68691ce5a97f6a717fc6d33a4f1d26c063ac6 Mon Sep 17 00:00:00 2001 From: Piotr Obrzut Date: Thu, 1 Jun 2017 15:25:04 +0200 Subject: [PATCH 002/199] Bug#26181622 MSI BUILD FAIL DUE TO DUPLICATED FILE ID Fixed generated mysql_server.wxs not to contain duplicates, or too long ids --- packaging/WiX/create_msi.cmake.in | 29 ++++++++++++++++++++++------- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/packaging/WiX/create_msi.cmake.in b/packaging/WiX/create_msi.cmake.in index 95cbbb3d437..b306c0c70d3 100644 --- a/packaging/WiX/create_msi.cmake.in +++ b/packaging/WiX/create_msi.cmake.in @@ -218,22 +218,37 @@ MACRO(GENERATE_GUID VarName) OUTPUT_STRIP_TRAILING_WHITESPACE) ENDMACRO() -SET(INC_VAR 0) +# Make sure that WIX identifier created from a path matches all the rules: +# - it is shorter than 72 characters +# - doesn't contain reserver characters ('+', '-' and '/') +# ID_SET contains a global list of all identifiers which are too long. +# Every time we use an identifier which is too long we use its index in +# ID_SET to shorten the name. +SET_PROPERTY(GLOBAL PROPERTY ID_SET) MACRO(MAKE_WIX_IDENTIFIER str varname) STRING(REPLACE "/" "." ${varname} "${str}") + STRING(REPLACE "+" "p" ${varname} "${str}") + STRING(REPLACE "-" "m" ${varname} "${str}") STRING(REGEX REPLACE "[^a-zA-Z_0-9.]" "_" ${varname} "${${varname}}") STRING(LENGTH "${${varname}}" len) + # FIXME: the prefix length has to be controlled better # Identifier should be smaller than 72 character - # We have to cut down the length to 70 chars, since we add 2 char prefix + # We have to cut down the length to 40 chars, since we add prefixes # pretty often - IF(len GREATER 70) - STRING(SUBSTRING "${${varname}}" 0 67 shortstr) - MATH(EXPR INC_VAR ${INC_VAR}+1) - SET(${varname} "${shortstr}${INC_VAR}") + IF(len GREATER 40) + STRING(SUBSTRING "${${varname}}" 0 37 shortstr) + GET_PROPERTY(LOCAL_LIST GLOBAL PROPERTY ID_SET) + LIST(FIND LOCAL_LIST "${${varname}}" STRING_ID) + IF(${STRING_ID} EQUAL -1) + LIST(APPEND LOCAL_LIST "${${varname}}") + SET_PROPERTY(GLOBAL PROPERTY ID_SET "${LOCAL_LIST}") + LIST(LENGTH LOCAL_LIST STRING_ID) + MATH(EXPR STRING_ID "${STRING_ID}-1" ) + ENDIF() + SET(${varname} "${shortstr}${STRING_ID}") ENDIF() ENDMACRO() - FUNCTION(TRAVERSE_FILES dir topdir file file_comp dir_root) FILE(RELATIVE_PATH dir_rel ${topdir} ${dir}) IF(dir_rel) From 1bb43334fc8e349624c2dd168dffb789e1550633 Mon Sep 17 00:00:00 2001 From: Piotr Obrzut Date: Fri, 2 Jun 2017 19:17:30 +0200 Subject: [PATCH 003/199] Bug#26171638 MYSQL 5.5.57 - MSI COMMUNITY PACKAGES NOT GETTING INSTALLED Temporary revert of the VS2008 redist check. --- packaging/WiX/create_msi.cmake.in | 31 ------------------------------- 1 file changed, 31 deletions(-) diff --git a/packaging/WiX/create_msi.cmake.in b/packaging/WiX/create_msi.cmake.in index b306c0c70d3..ae6a5773dce 100644 --- a/packaging/WiX/create_msi.cmake.in +++ b/packaging/WiX/create_msi.cmake.in @@ -218,37 +218,6 @@ MACRO(GENERATE_GUID VarName) OUTPUT_STRIP_TRAILING_WHITESPACE) ENDMACRO() -# Make sure that WIX identifier created from a path matches all the rules: -# - it is shorter than 72 characters -# - doesn't contain reserver characters ('+', '-' and '/') -# ID_SET contains a global list of all identifiers which are too long. -# Every time we use an identifier which is too long we use its index in -# ID_SET to shorten the name. -SET_PROPERTY(GLOBAL PROPERTY ID_SET) -MACRO(MAKE_WIX_IDENTIFIER str varname) - STRING(REPLACE "/" "." ${varname} "${str}") - STRING(REPLACE "+" "p" ${varname} "${str}") - STRING(REPLACE "-" "m" ${varname} "${str}") - STRING(REGEX REPLACE "[^a-zA-Z_0-9.]" "_" ${varname} "${${varname}}") - STRING(LENGTH "${${varname}}" len) - # FIXME: the prefix length has to be controlled better - # Identifier should be smaller than 72 character - # We have to cut down the length to 40 chars, since we add prefixes - # pretty often - IF(len GREATER 40) - STRING(SUBSTRING "${${varname}}" 0 37 shortstr) - GET_PROPERTY(LOCAL_LIST GLOBAL PROPERTY ID_SET) - LIST(FIND LOCAL_LIST "${${varname}}" STRING_ID) - IF(${STRING_ID} EQUAL -1) - LIST(APPEND LOCAL_LIST "${${varname}}") - SET_PROPERTY(GLOBAL PROPERTY ID_SET "${LOCAL_LIST}") - LIST(LENGTH LOCAL_LIST STRING_ID) - MATH(EXPR STRING_ID "${STRING_ID}-1" ) - ENDIF() - SET(${varname} "${shortstr}${STRING_ID}") - ENDIF() -ENDMACRO() - FUNCTION(TRAVERSE_FILES dir topdir file file_comp dir_root) FILE(RELATIVE_PATH dir_rel ${topdir} ${dir}) IF(dir_rel) From b5258c7134398b7b0c7b915c2c6c22e8decba453 Mon Sep 17 00:00:00 2001 From: Piotr Obrzut Date: Mon, 5 Jun 2017 08:09:07 +0200 Subject: [PATCH 004/199] Bug#26171638 MYSQL 5.5.57 - MSI COMMUNITY PACKAGES NOT GETTING INSTALLED Corrected the revert. --- packaging/WiX/create_msi.cmake.in | 31 +++++++++++++++++++++++++++++++ packaging/WiX/mysql_server.wxs.in | 26 -------------------------- 2 files changed, 31 insertions(+), 26 deletions(-) diff --git a/packaging/WiX/create_msi.cmake.in b/packaging/WiX/create_msi.cmake.in index ae6a5773dce..b306c0c70d3 100644 --- a/packaging/WiX/create_msi.cmake.in +++ b/packaging/WiX/create_msi.cmake.in @@ -218,6 +218,37 @@ MACRO(GENERATE_GUID VarName) OUTPUT_STRIP_TRAILING_WHITESPACE) ENDMACRO() +# Make sure that WIX identifier created from a path matches all the rules: +# - it is shorter than 72 characters +# - doesn't contain reserver characters ('+', '-' and '/') +# ID_SET contains a global list of all identifiers which are too long. +# Every time we use an identifier which is too long we use its index in +# ID_SET to shorten the name. +SET_PROPERTY(GLOBAL PROPERTY ID_SET) +MACRO(MAKE_WIX_IDENTIFIER str varname) + STRING(REPLACE "/" "." ${varname} "${str}") + STRING(REPLACE "+" "p" ${varname} "${str}") + STRING(REPLACE "-" "m" ${varname} "${str}") + STRING(REGEX REPLACE "[^a-zA-Z_0-9.]" "_" ${varname} "${${varname}}") + STRING(LENGTH "${${varname}}" len) + # FIXME: the prefix length has to be controlled better + # Identifier should be smaller than 72 character + # We have to cut down the length to 40 chars, since we add prefixes + # pretty often + IF(len GREATER 40) + STRING(SUBSTRING "${${varname}}" 0 37 shortstr) + GET_PROPERTY(LOCAL_LIST GLOBAL PROPERTY ID_SET) + LIST(FIND LOCAL_LIST "${${varname}}" STRING_ID) + IF(${STRING_ID} EQUAL -1) + LIST(APPEND LOCAL_LIST "${${varname}}") + SET_PROPERTY(GLOBAL PROPERTY ID_SET "${LOCAL_LIST}") + LIST(LENGTH LOCAL_LIST STRING_ID) + MATH(EXPR STRING_ID "${STRING_ID}-1" ) + ENDIF() + SET(${varname} "${shortstr}${STRING_ID}") + ENDIF() +ENDMACRO() + FUNCTION(TRAVERSE_FILES dir topdir file file_comp dir_root) FILE(RELATIVE_PATH dir_rel ${topdir} ${dir}) IF(dir_rel) diff --git a/packaging/WiX/mysql_server.wxs.in b/packaging/WiX/mysql_server.wxs.in index 2092d0ebe3d..3eb81ec532c 100644 --- a/packaging/WiX/mysql_server.wxs.in +++ b/packaging/WiX/mysql_server.wxs.in @@ -61,32 +61,6 @@ - - - - - - - - Installed OR VS08REDISTX64 - - - - - - - Installed OR VS08REDISTX86 - - - From 790770c26438520d3dd139009ce51e3873d5fd9b Mon Sep 17 00:00:00 2001 From: Balasubramanian Kandasamy Date: Fri, 7 Jul 2017 17:43:40 +0530 Subject: [PATCH 005/199] Bug#26400146 - 5.5 AND 5.6 DOCKER PACKAGES MISSING MYSQLCHECK UPGRADE NOT POSSIBLE - Add mysqlcheck tool to docker rpms for upgrade --- packaging/rpm-docker/mysql.spec.in | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packaging/rpm-docker/mysql.spec.in b/packaging/rpm-docker/mysql.spec.in index 497ae395577..cc4e7bc541e 100644 --- a/packaging/rpm-docker/mysql.spec.in +++ b/packaging/rpm-docker/mysql.spec.in @@ -178,7 +178,7 @@ for f in msql2mysql mysqlaccess mysqlaccess.conf mysqlbug mysql_convert_table_fo done for f in innochecksum myisamchk myisam_ftdump myisamlog myisampack \ - mysqlbinlog mysqlcheck mysql_client_test mysql_config_editor \ + mysqlbinlog mysql_client_test mysql_config_editor \ mysqld_multi mysqld_safe mysqldumpslow mysql_embedded mysqlimport \ mysql_plugin mysql_secure_installation mysqlshow mysqlslap mysqltest \ perror replace resolveip resolve_stack_dump; do @@ -232,6 +232,7 @@ rm -r $(readlink var) var %attr(755, root, root) %{_sbindir}/mysqld %attr(755, root, root) %{_bindir}/mysql %attr(755, root, root) %{_bindir}/mysqladmin +%attr(755, root, root) %{_bindir}/mysqlcheck %attr(755, root, root) %{_bindir}/mysqldump %attr(755, root, root) %{_bindir}/mysql_config %attr(755, root, root) %{_bindir}/mysql_install_db From 6a6d5bc98abdd25b32590fea7ef6572ecc355d7e Mon Sep 17 00:00:00 2001 From: Deepa Dixit Date: Tue, 25 Jul 2017 11:49:51 +0530 Subject: [PATCH 006/199] Bug#26161247: MTR: --NOREORDER IS SEARCHING FOR TEST SCRIPT ONLY IN MAIN SUITE Issue: ------ Running MTR with the --no-reorder option by specifying test cases on the command line, without prefixing the suite name results in an error saying the test case was not found in the main suite. This is because MTR looks for the test case only in the main suite, and no other suites. Fix: ---- The fix involves searching for the test in every suite if only the test name is specified. This back-ports two bug fixes: Bug#24967869 and Bug#24365783 Reviewed-by: Pavan Naik RB: 16812 --- mysql-test/lib/mtr_cases.pm | 66 ++++++++++++++++++++++++++++++------- 1 file changed, 54 insertions(+), 12 deletions(-) diff --git a/mysql-test/lib/mtr_cases.pm b/mysql-test/lib/mtr_cases.pm index e8d24eb399f..ce7d4e690c6 100644 --- a/mysql-test/lib/mtr_cases.pm +++ b/mysql-test/lib/mtr_cases.pm @@ -1,5 +1,5 @@ # -*- cperl -*- -# Copyright (c) 2005, 2015, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2005, 2017, Oracle and/or its affiliates. All rights reserved. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -143,7 +143,13 @@ sub collect_test_cases ($$$$) { if ( @$opt_cases ) { - # A list of tests was specified on the command line + # A list of tests was specified on the command line. + # Among those, the tests which are not already collected will be + # collected and stored temporarily in an array of hashes pointed + # by the below reference. This array is eventually appeneded to + # the one having all collected test cases. + my $cmdline_cases; + # Check that the tests specified was found # in at least one suite foreach my $test_name_spec ( @$opt_cases ) @@ -162,20 +168,56 @@ sub collect_test_cases ($$$$) { } if ( not $found ) { - $sname= "main" if !$opt_reorder and !$sname; - mtr_error("Could not find '$tname' in '$suites' suite(s)") unless $sname; - # If suite was part of name, find it there, may come with combinations - my @this_case = collect_one_suite($sname, [ $tname ]); - if (@this_case) + if ( $sname ) { - push (@$cases, @this_case); - } - else - { - mtr_error("Could not find '$tname' in '$sname' suite"); + # If suite was part of name, find it there, may come with combinations + my @this_case = collect_one_suite($sname, [ $tname ]); + + # If a test is specified multiple times on the command line, all + # instances of the test need to be picked. Hence, such tests are + # stored in the temporary array instead of adding them to $cases + # directly so that repeated tests are not run only once + if (@this_case) + { + push (@$cmdline_cases, @this_case); + } + else + { + mtr_error("Could not find '$tname' in '$sname' suite"); + } + } + else + { + if ( !$opt_reorder ) + { + # If --no-reorder is passed and if suite was not part of name, + # search in all the suites + foreach my $suite (split(",", $suites)) + { + my @this_case = collect_one_suite($suite, [ $tname ]); + if ( @this_case ) + { + push (@$cmdline_cases, @this_case); + $found= 1; + } + @this_case= collect_one_suite("i_".$suite, [ $tname ]); + if ( @this_case ) + { + push (@$cmdline_cases, @this_case); + $found= 1; + } + } + } + if ( !$found ) + { + mtr_error("Could not find '$tname' in '$suites' suite(s)"); + } } } } + # Add test cases collected in the temporary array to the one + # containing all previously collected test cases + push (@$cases, @$cmdline_cases) if $cmdline_cases; } if ( $opt_reorder && !$quick_collect) From d75f8a174218fd0ada64222d2a538f7aace897bc Mon Sep 17 00:00:00 2001 From: Venkatesh Duggirala Date: Wed, 23 Aug 2017 09:16:12 +0530 Subject: [PATCH 007/199] Bug#24763131 LOCAL-INFILE DEFAULT SHOULD BE DISABLED Problem & Analysis: Slave's Receiver thread, Applier thread and worker threads are created with LOCAL-INFILE option enabled. As the document says https://dev.mysql.com/doc/refman/5.7/en/load-data-local.html, there are some issues if a thread enables local infile. This flag should be enabled with care. But for the above mentioned internal threads, server is enabling it at the time of creation. Fix: Further analysis on the code shows that none of threads really need this flag to be enabled at any time as Slave never executes "LOAD DATA LOCAL INFILE" after reading it from Relay log. Applier thread removes "LOCAL" before start executing the query. --- sql/slave.cc | 1 - 1 file changed, 1 deletion(-) diff --git a/sql/slave.cc b/sql/slave.cc index 1e641ac6d7e..e533bc09b3d 100644 --- a/sql/slave.cc +++ b/sql/slave.cc @@ -2109,7 +2109,6 @@ static int init_slave_thread(THD* thd, SLAVE_THD_TYPE thd_type) thd->slave_thread = 1; thd->enable_slow_log= opt_log_slow_slave_statements; set_slave_thread_options(thd); - thd->client_capabilities = CLIENT_LOCAL_FILES; mysql_mutex_lock(&LOCK_thread_count); thd->thread_id= thd->variables.pseudo_thread_id= thread_id++; mysql_mutex_unlock(&LOCK_thread_count); From ebd96c314953f5c0073ff5846484fd5e438fe0ad Mon Sep 17 00:00:00 2001 From: Tor Didriksen Date: Tue, 15 Aug 2017 13:15:19 +0200 Subject: [PATCH 008/199] Bug#19875294 ASSERTION `SRC' FAILED IN MY_STRNXFRM_UNICODE (SIG 6 -STRINGS/CTYPE-UTF8.C:5151) Backport from 5.7 to 5.5 Field_set::val_str() should return String("", 0, cs) rather than String(NULL, 0, cs) --- mysql-test/r/case.result | 16 ++++++++++++++++ mysql-test/t/case.test | 12 ++++++++++++ sql/field.cc | 12 ++++++------ 3 files changed, 34 insertions(+), 6 deletions(-) diff --git a/mysql-test/r/case.result b/mysql-test/r/case.result index 40d900a0389..be712114a29 100644 --- a/mysql-test/r/case.result +++ b/mysql-test/r/case.result @@ -220,3 +220,19 @@ a d 3 11120436154190595086 drop table t1, t2; End of 5.0 tests +# +# Bug#19875294 ASSERTION `SRC' FAILED IN MY_STRNXFRM_UNICODE +# (SIG 6 -STRINGS/CTYPE-UTF8.C:5151) +# +set @@sql_mode=''; +CREATE TABLE t1(c1 SET('','')CHARACTER SET ucs2) engine=innodb; +Warnings: +Note 1291 Column 'c1' has duplicated value '' in SET +INSERT INTO t1 VALUES(990101.102); +Warnings: +Warning 1265 Data truncated for column 'c1' at row 1 +SELECT COALESCE(c1)FROM t1 ORDER BY 1; +COALESCE(c1) + +DROP TABLE t1; +set @@sql_mode=default; diff --git a/mysql-test/t/case.test b/mysql-test/t/case.test index e1c807fe32b..c2d8f44e953 100644 --- a/mysql-test/t/case.test +++ b/mysql-test/t/case.test @@ -173,3 +173,15 @@ select t1.a, (case t1.a when 0 then 0 else t1.b end) d from t1 drop table t1, t2; --echo End of 5.0 tests + +--echo # +--echo # Bug#19875294 ASSERTION `SRC' FAILED IN MY_STRNXFRM_UNICODE +--echo # (SIG 6 -STRINGS/CTYPE-UTF8.C:5151) +--echo # + +set @@sql_mode=''; +CREATE TABLE t1(c1 SET('','')CHARACTER SET ucs2) engine=innodb; +INSERT INTO t1 VALUES(990101.102); +SELECT COALESCE(c1)FROM t1 ORDER BY 1; +DROP TABLE t1; +set @@sql_mode=default; diff --git a/sql/field.cc b/sql/field.cc index 15571afefb8..e98c17fabae 100644 --- a/sql/field.cc +++ b/sql/field.cc @@ -1,5 +1,5 @@ /* - Copyright (c) 2000, 2016, Oracle and/or its affiliates. All rights reserved. + Copyright (c) 2000, 2017, Oracle and/or its affiliates. All rights reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -8257,13 +8257,13 @@ String *Field_set::val_str(String *val_buffer, ulonglong tmp=(ulonglong) Field_enum::val_int(); uint bitnr=0; + /* + Some callers expect *val_buffer to contain the result, + so we assign to it, rather than doing 'return &empty_set_string. + */ + *val_buffer= empty_set_string; if (tmp == 0) { - /* - Some callers expect *val_buffer to contain the result, - so we assign to it, rather than doing 'return &empty_set_string. - */ - *val_buffer= empty_set_string; return val_buffer; } From be901b60ae59c93848c829d1b0b2cb523ab8692e Mon Sep 17 00:00:00 2001 From: Nisha Gopalakrishnan Date: Wed, 16 Aug 2017 13:58:25 +0530 Subject: [PATCH 009/199] Bug#26390632: CREATE TABLE CAN CAUSE MYSQL TO EXIT. Analysis ======== CREATE TABLE of InnoDB table with a partition name which exceeds the path limit can cause the server to exit. During the preparation of the partition name, there was no check to identify whether the complete path name for partition exceeds the max supported path length, causing the server to exit during subsequent processing. Fix === During the preparation of partition name, check and report an error if the partition path name exceeds the maximum path name limit. This is a 5.5 patch. --- sql/ha_partition.cc | 150 +++++++++++++++++++++++--------------- sql/sql_partition.cc | 169 +++++++++++++++++++++++++++---------------- sql/sql_partition.h | 6 +- 3 files changed, 200 insertions(+), 125 deletions(-) diff --git a/sql/ha_partition.cc b/sql/ha_partition.cc index aadac36e2ee..414f9d52536 100644 --- a/sql/ha_partition.cc +++ b/sql/ha_partition.cc @@ -1,5 +1,5 @@ /* - Copyright (c) 2005, 2013, Oracle and/or its affiliates. All rights reserved. + Copyright (c) 2005, 2017, Oracle and/or its affiliates. All rights reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -624,7 +624,7 @@ int ha_partition::create(const char *name, TABLE *table_arg, int ha_partition::drop_partitions(const char *path) { List_iterator part_it(m_part_info->partitions); - char part_name_buff[FN_REFLEN]; + char part_name_buff[FN_REFLEN + 1]; uint num_parts= m_part_info->partitions.elements; uint num_subparts= m_part_info->num_subparts; uint i= 0; @@ -657,9 +657,12 @@ int ha_partition::drop_partitions(const char *path) { partition_element *sub_elem= sub_it++; part= i * num_subparts + j; - create_subpartition_name(part_name_buff, path, - part_elem->partition_name, - sub_elem->partition_name, name_variant); + if ((ret_error= create_subpartition_name(part_name_buff, path, + part_elem->partition_name, + sub_elem->partition_name, + name_variant))) + error= ret_error; + file= m_file[part]; DBUG_PRINT("info", ("Drop subpartition %s", part_name_buff)); if ((ret_error= file->ha_delete_table(part_name_buff))) @@ -670,9 +673,11 @@ int ha_partition::drop_partitions(const char *path) } else { - create_partition_name(part_name_buff, path, - part_elem->partition_name, name_variant, - TRUE); + if ((ret_error= create_partition_name(part_name_buff, path, + part_elem->partition_name, + name_variant, TRUE))) + error= ret_error; + file= m_file[i]; DBUG_PRINT("info", ("Drop partition %s", part_name_buff)); if ((ret_error= file->ha_delete_table(part_name_buff))) @@ -714,8 +719,8 @@ int ha_partition::rename_partitions(const char *path) { List_iterator part_it(m_part_info->partitions); List_iterator temp_it(m_part_info->temp_partitions); - char part_name_buff[FN_REFLEN]; - char norm_name_buff[FN_REFLEN]; + char part_name_buff[FN_REFLEN + 1]; + char norm_name_buff[FN_REFLEN + 1]; uint num_parts= m_part_info->partitions.elements; uint part_count= 0; uint num_subparts= m_part_info->num_subparts; @@ -757,10 +762,11 @@ int ha_partition::rename_partitions(const char *path) { sub_elem= sub_it++; file= m_reorged_file[part_count++]; - create_subpartition_name(norm_name_buff, path, - part_elem->partition_name, - sub_elem->partition_name, - NORMAL_PART_NAME); + if ((ret_error= create_subpartition_name(norm_name_buff, path, + part_elem->partition_name, + sub_elem->partition_name, + NORMAL_PART_NAME))) + error= ret_error; DBUG_PRINT("info", ("Delete subpartition %s", norm_name_buff)); if ((ret_error= file->ha_delete_table(norm_name_buff))) error= ret_error; @@ -773,9 +779,11 @@ int ha_partition::rename_partitions(const char *path) else { file= m_reorged_file[part_count++]; - create_partition_name(norm_name_buff, path, - part_elem->partition_name, NORMAL_PART_NAME, - TRUE); + if ((ret_error= create_partition_name(norm_name_buff, path, + part_elem->partition_name, + NORMAL_PART_NAME, TRUE))) + error= ret_error; + DBUG_PRINT("info", ("Delete partition %s", norm_name_buff)); if ((ret_error= file->ha_delete_table(norm_name_buff))) error= ret_error; @@ -825,10 +833,12 @@ int ha_partition::rename_partitions(const char *path) { sub_elem= sub_it++; part= i * num_subparts + j; - create_subpartition_name(norm_name_buff, path, - part_elem->partition_name, - sub_elem->partition_name, - NORMAL_PART_NAME); + if ((ret_error= create_subpartition_name(norm_name_buff, path, + part_elem->partition_name, + sub_elem->partition_name, + NORMAL_PART_NAME))) + error= ret_error; + if (part_elem->part_state == PART_IS_CHANGED) { file= m_reorged_file[part_count++]; @@ -840,10 +850,12 @@ int ha_partition::rename_partitions(const char *path) (void) sync_ddl_log(); } file= m_new_file[part]; - create_subpartition_name(part_name_buff, path, - part_elem->partition_name, - sub_elem->partition_name, - TEMP_PART_NAME); + if ((ret_error= create_subpartition_name(part_name_buff, path, + part_elem->partition_name, + sub_elem->partition_name, + TEMP_PART_NAME))) + error= ret_error; + DBUG_PRINT("info", ("Rename subpartition from %s to %s", part_name_buff, norm_name_buff)); if ((ret_error= file->ha_rename_table(part_name_buff, @@ -857,9 +869,11 @@ int ha_partition::rename_partitions(const char *path) } else { - create_partition_name(norm_name_buff, path, - part_elem->partition_name, NORMAL_PART_NAME, - TRUE); + if ((ret_error= create_partition_name(norm_name_buff, path, + part_elem->partition_name, + NORMAL_PART_NAME, TRUE))) + error= ret_error; + if (part_elem->part_state == PART_IS_CHANGED) { file= m_reorged_file[part_count++]; @@ -871,9 +885,11 @@ int ha_partition::rename_partitions(const char *path) (void) sync_ddl_log(); } file= m_new_file[i]; - create_partition_name(part_name_buff, path, - part_elem->partition_name, TEMP_PART_NAME, - TRUE); + if ((ret_error= create_partition_name(part_name_buff, path, + part_elem->partition_name, + TEMP_PART_NAME, TRUE))) + error= ret_error; + DBUG_PRINT("info", ("Rename partition from %s to %s", part_name_buff, norm_name_buff)); if ((ret_error= file->ha_rename_table(part_name_buff, @@ -1477,7 +1493,7 @@ int ha_partition::change_partitions(HA_CREATE_INFO *create_info, { List_iterator part_it(m_part_info->partitions); List_iterator t_it(m_part_info->temp_partitions); - char part_name_buff[FN_REFLEN]; + char part_name_buff[FN_REFLEN + 1]; uint num_parts= m_part_info->partitions.elements; uint num_subparts= m_part_info->num_subparts; uint i= 0; @@ -1687,10 +1703,15 @@ int ha_partition::change_partitions(HA_CREATE_INFO *create_info, do { partition_element *sub_elem= sub_it++; - create_subpartition_name(part_name_buff, path, - part_elem->partition_name, - sub_elem->partition_name, - name_variant); + if ((error= create_subpartition_name(part_name_buff, path, + part_elem->partition_name, + sub_elem->partition_name, + name_variant))) + { + cleanup_new_partition(part_count); + DBUG_RETURN(error); + } + part= i * num_subparts + j; DBUG_PRINT("info", ("Add subpartition %s", part_name_buff)); if ((error= prepare_new_partition(table, create_info, @@ -1708,9 +1729,14 @@ int ha_partition::change_partitions(HA_CREATE_INFO *create_info, } else { - create_partition_name(part_name_buff, path, - part_elem->partition_name, name_variant, - TRUE); + if ((error= create_partition_name(part_name_buff, path, + part_elem->partition_name, + name_variant, TRUE))) + { + cleanup_new_partition(part_count); + DBUG_RETURN(error); + } + DBUG_PRINT("info", ("Add partition %s", part_name_buff)); if ((error= prepare_new_partition(table, create_info, new_file_array[i], @@ -1967,8 +1993,8 @@ int ha_partition::del_ren_cre_table(const char *from, { int save_error= 0; int error= HA_ERR_INTERNAL_ERROR; - char from_buff[FN_REFLEN], to_buff[FN_REFLEN], from_lc_buff[FN_REFLEN], - to_lc_buff[FN_REFLEN], buff[FN_REFLEN]; + char from_buff[FN_REFLEN + 1], to_buff[FN_REFLEN + 1], from_lc_buff[FN_REFLEN + 1], + to_lc_buff[FN_REFLEN + 1], buff[FN_REFLEN + 1]; char *name_buffer_ptr; const char *from_path; const char *to_path= NULL; @@ -2015,13 +2041,16 @@ int ha_partition::del_ren_cre_table(const char *from, i= 0; do { - create_partition_name(from_buff, from_path, name_buffer_ptr, - NORMAL_PART_NAME, FALSE); + if ((error= create_partition_name(from_buff, from_path, name_buffer_ptr, + NORMAL_PART_NAME, FALSE))) + DBUG_RETURN(error); if (to != NULL) { // Rename branch - create_partition_name(to_buff, to_path, name_buffer_ptr, - NORMAL_PART_NAME, FALSE); + if ((error= create_partition_name(to_buff, to_path, name_buffer_ptr, + NORMAL_PART_NAME, FALSE))) + DBUG_RETURN(error); + error= (*file)->ha_rename_table(from_buff, to_buff); if (error) goto rename_error; @@ -2066,9 +2095,9 @@ create_error: name_buffer_ptr= m_name_buffer_ptr; for (abort_file= file, file= m_file; file < abort_file; file++) { - create_partition_name(from_buff, from_path, name_buffer_ptr, NORMAL_PART_NAME, - FALSE); - (void) (*file)->ha_delete_table((const char*) from_buff); + if (!create_partition_name(from_buff, from_path, name_buffer_ptr, NORMAL_PART_NAME, + FALSE)) + (void) (*file)->ha_delete_table((const char*) from_buff); name_buffer_ptr= strend(name_buffer_ptr) + 1; } DBUG_RETURN(error); @@ -2077,12 +2106,12 @@ rename_error: for (abort_file= file, file= m_file; file < abort_file; file++) { /* Revert the rename, back from 'to' to the original 'from' */ - create_partition_name(from_buff, from_path, name_buffer_ptr, - NORMAL_PART_NAME, FALSE); - create_partition_name(to_buff, to_path, name_buffer_ptr, - NORMAL_PART_NAME, FALSE); - /* Ignore error here */ - (void) (*file)->ha_rename_table(to_buff, from_buff); + if (!create_partition_name(from_buff, from_path, name_buffer_ptr, + NORMAL_PART_NAME, FALSE)) + if (!create_partition_name(to_buff, to_path, name_buffer_ptr, + NORMAL_PART_NAME, FALSE)) + /* Ignore error here */ + (void) (*file)->ha_rename_table(to_buff, from_buff); name_buffer_ptr= strend(name_buffer_ptr) + 1; } DBUG_RETURN(error); @@ -2707,7 +2736,7 @@ int ha_partition::open(const char *name, int mode, uint test_if_locked) char *name_buffer_ptr; int error= HA_ERR_INITIALIZATION; handler **file; - char name_buff[FN_REFLEN]; + char name_buff[FN_REFLEN + 1]; bool is_not_tmp_table= (table_share->tmp_table == NO_TMP_TABLE); ulonglong check_table_flags; DBUG_ENTER("ha_partition::open"); @@ -2777,8 +2806,10 @@ int ha_partition::open(const char *name, int mode, uint test_if_locked) file= m_is_clone_of->m_file; for (i= 0; i < m_tot_parts; i++) { - create_partition_name(name_buff, name, name_buffer_ptr, NORMAL_PART_NAME, - FALSE); + if ((error= create_partition_name(name_buff, name, name_buffer_ptr, + NORMAL_PART_NAME, FALSE))) + goto err_handler; + if (!(m_file[i]= file[i]->clone(name_buff, m_clone_mem_root))) { error= HA_ERR_INITIALIZATION; @@ -2793,8 +2824,9 @@ int ha_partition::open(const char *name, int mode, uint test_if_locked) file= m_file; do { - create_partition_name(name_buff, name, name_buffer_ptr, NORMAL_PART_NAME, - FALSE); + if ((error= create_partition_name(name_buff, name, name_buffer_ptr, + NORMAL_PART_NAME, FALSE))) + goto err_handler; if ((error= (*file)->ha_open(table, name_buff, mode, test_if_locked))) goto err_handler; m_num_locks+= (*file)->lock_count(); diff --git a/sql/sql_partition.cc b/sql/sql_partition.cc index 5358535e9f9..65d4da0f2f6 100644 --- a/sql/sql_partition.cc +++ b/sql/sql_partition.cc @@ -1,4 +1,4 @@ -/* Copyright (c) 2005, 2015, Oracle and/or its affiliates. All rights reserved. +/* Copyright (c) 2005, 2017, Oracle and/or its affiliates. All rights reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -5958,8 +5958,8 @@ static bool write_log_changed_partitions(ALTER_PARTITION_PARAM_TYPE *lpt, DDL_LOG_ENTRY ddl_log_entry; partition_info *part_info= lpt->part_info; DDL_LOG_MEMORY_ENTRY *log_entry; - char tmp_path[FN_REFLEN]; - char normal_path[FN_REFLEN]; + char tmp_path[FN_REFLEN + 1]; + char normal_path[FN_REFLEN + 1]; List_iterator part_it(part_info->partitions); uint temp_partitions= part_info->temp_partitions.elements; uint num_elements= part_info->partitions.elements; @@ -5983,14 +5983,18 @@ static bool write_log_changed_partitions(ALTER_PARTITION_PARAM_TYPE *lpt, ddl_log_entry.next_entry= *next_entry; ddl_log_entry.handler_name= ha_resolve_storage_engine_name(sub_elem->engine_type); - create_subpartition_name(tmp_path, path, - part_elem->partition_name, - sub_elem->partition_name, - TEMP_PART_NAME); - create_subpartition_name(normal_path, path, - part_elem->partition_name, - sub_elem->partition_name, - NORMAL_PART_NAME); + if (create_subpartition_name(tmp_path, path, + part_elem->partition_name, + sub_elem->partition_name, + TEMP_PART_NAME)) + DBUG_RETURN(TRUE); + + if (create_subpartition_name(normal_path, path, + part_elem->partition_name, + sub_elem->partition_name, + NORMAL_PART_NAME)) + DBUG_RETURN(TRUE); + ddl_log_entry.name= normal_path; ddl_log_entry.from_name= tmp_path; if (part_elem->part_state == PART_IS_CHANGED) @@ -6011,12 +6015,13 @@ static bool write_log_changed_partitions(ALTER_PARTITION_PARAM_TYPE *lpt, ddl_log_entry.next_entry= *next_entry; ddl_log_entry.handler_name= ha_resolve_storage_engine_name(part_elem->engine_type); - create_partition_name(tmp_path, path, - part_elem->partition_name, - TEMP_PART_NAME, TRUE); - create_partition_name(normal_path, path, - part_elem->partition_name, - NORMAL_PART_NAME, TRUE); + if ((create_partition_name(tmp_path, path, part_elem->partition_name, + TEMP_PART_NAME, TRUE)) || + (create_partition_name(normal_path, path, + part_elem->partition_name, + NORMAL_PART_NAME, TRUE))) + DBUG_RETURN(TRUE); + ddl_log_entry.name= normal_path; ddl_log_entry.from_name= tmp_path; if (part_elem->part_state == PART_IS_CHANGED) @@ -6055,7 +6060,7 @@ static bool write_log_dropped_partitions(ALTER_PARTITION_PARAM_TYPE *lpt, DDL_LOG_ENTRY ddl_log_entry; partition_info *part_info= lpt->part_info; DDL_LOG_MEMORY_ENTRY *log_entry; - char tmp_path[FN_LEN]; + char tmp_path[FN_REFLEN + 1]; List_iterator part_it(part_info->partitions); List_iterator temp_it(part_info->temp_partitions); uint num_temp_partitions= part_info->temp_partitions.elements; @@ -6094,10 +6099,12 @@ static bool write_log_dropped_partitions(ALTER_PARTITION_PARAM_TYPE *lpt, ddl_log_entry.next_entry= *next_entry; ddl_log_entry.handler_name= ha_resolve_storage_engine_name(sub_elem->engine_type); - create_subpartition_name(tmp_path, path, - part_elem->partition_name, - sub_elem->partition_name, - name_variant); + if (create_subpartition_name(tmp_path, path, + part_elem->partition_name, + sub_elem->partition_name, + name_variant)) + DBUG_RETURN(TRUE); + ddl_log_entry.name= tmp_path; if (write_ddl_log_entry(&ddl_log_entry, &log_entry)) { @@ -6113,9 +6120,10 @@ static bool write_log_dropped_partitions(ALTER_PARTITION_PARAM_TYPE *lpt, ddl_log_entry.next_entry= *next_entry; ddl_log_entry.handler_name= ha_resolve_storage_engine_name(part_elem->engine_type); - create_partition_name(tmp_path, path, - part_elem->partition_name, - name_variant, TRUE); + if ((create_partition_name(tmp_path, path, part_elem->partition_name, + name_variant, TRUE))) + DBUG_RETURN(TRUE); + ddl_log_entry.name= tmp_path; if (write_ddl_log_entry(&ddl_log_entry, &log_entry)) { @@ -8194,29 +8202,28 @@ static uint32 get_next_subpartition_via_walking(PARTITION_ITERATOR *part_iter) } -/* - Create partition names +/** + Create partition names. This method is used to calculate the + partition name, service routine to the del_ren_cre_table method. + The output buffer size should be FN_REFLEN + 1(terminating '\0'). - SYNOPSIS - create_partition_name() - out:out Created partition name string - in1 First part - in2 Second part - name_variant Normal, temporary or renamed partition name + @param [out] out Created partition name string + @param in1 First part + @param in2 Second part + @param in3 Third part + @param name_variant Normal, temporary or renamed partition name + @param translate Flag to determine whether to convert a table name + to it its corresponding filename. - RETURN VALUE - NONE - - DESCRIPTION - This method is used to calculate the partition name, service routine to - the del_ren_cre_table method. + @retval true Error. + @retval false Success. */ -void create_partition_name(char *out, const char *in1, +bool create_partition_name(char *out, const char *in1, const char *in2, uint name_variant, bool translate) { - char transl_part_name[FN_REFLEN]; + char transl_part_name[FN_REFLEN + 1]; const char *transl_part; if (translate) @@ -8226,35 +8233,50 @@ void create_partition_name(char *out, const char *in1, } else transl_part= in2; + + // Check if the path name for partition exceeds maximum path length. if (name_variant == NORMAL_PART_NAME) - strxmov(out, in1, "#P#", transl_part, NullS); + { + if ((strlen(in1) + strlen(transl_part) + 3) > FN_REFLEN) + { + my_error(ER_PATH_LENGTH, MYF(0), in2); + return true; + } + } + else + if ((strlen(in1) + strlen(transl_part) + 8) > FN_REFLEN) + { + my_error(ER_PATH_LENGTH, MYF(0), in2); + return true; + } + + if (name_variant == NORMAL_PART_NAME) + strxnmov(out, FN_REFLEN, in1, "#P#", transl_part, NullS); else if (name_variant == TEMP_PART_NAME) - strxmov(out, in1, "#P#", transl_part, "#TMP#", NullS); + strxnmov(out, FN_REFLEN, in1, "#P#", transl_part, "#TMP#", NullS); else if (name_variant == RENAMED_PART_NAME) - strxmov(out, in1, "#P#", transl_part, "#REN#", NullS); + strxnmov(out, FN_REFLEN, in1, "#P#", transl_part, "#REN#", NullS); + + return false; } -/* - Create subpartition name +/** + Create subpartition name. This method is used to calculate the + subpartition name, service routine to the del_ren_cre_table method. + The output buffer size should be FN_REFLEN + 1(terminating '\0'). - SYNOPSIS - create_subpartition_name() - out:out Created partition name string - in1 First part - in2 Second part - in3 Third part - name_variant Normal, temporary or renamed partition name + @param [out] out Created partition name string + @param in1 First part + @param in2 Second part + @param in3 Third part + @param name_variant Normal, temporary or renamed partition name - RETURN VALUE - NONE - - DESCRIPTION - This method is used to calculate the subpartition name, service routine to - the del_ren_cre_table method. + @retval true Error. + @retval false Success. */ -void create_subpartition_name(char *out, const char *in1, +bool create_subpartition_name(char *out, const char *in1, const char *in2, const char *in3, uint name_variant) { @@ -8262,15 +8284,36 @@ void create_subpartition_name(char *out, const char *in1, tablename_to_filename(in2, transl_part_name, FN_REFLEN); tablename_to_filename(in3, transl_subpart_name, FN_REFLEN); + + // Check if the path name for subpartition exceeds maximum path length. if (name_variant == NORMAL_PART_NAME) - strxmov(out, in1, "#P#", transl_part_name, + { + if ((strlen(in1) + strlen(transl_part_name) + + strlen(transl_subpart_name) + 7) > FN_REFLEN) + { + my_error(ER_PATH_LENGTH, MYF(0), in3); + return true; + } + } + else + if ((strlen(in1) + strlen(transl_part_name) + + strlen(transl_subpart_name) + 12) > FN_REFLEN) + { + my_error(ER_PATH_LENGTH, MYF(0), in3); + return true; + } + + if (name_variant == NORMAL_PART_NAME) + strxnmov(out, FN_REFLEN, in1, "#P#", transl_part_name, "#SP#", transl_subpart_name, NullS); else if (name_variant == TEMP_PART_NAME) - strxmov(out, in1, "#P#", transl_part_name, + strxnmov(out, FN_REFLEN, in1, "#P#", transl_part_name, "#SP#", transl_subpart_name, "#TMP#", NullS); else if (name_variant == RENAMED_PART_NAME) - strxmov(out, in1, "#P#", transl_part_name, + strxnmov(out, FN_REFLEN, in1, "#P#", transl_part_name, "#SP#", transl_subpart_name, "#REN#", NullS); + + return false; } uint get_partition_field_store_length(Field *field) diff --git a/sql/sql_partition.h b/sql/sql_partition.h index f232eaa0629..cfaab903f04 100644 --- a/sql/sql_partition.h +++ b/sql/sql_partition.h @@ -1,7 +1,7 @@ #ifndef SQL_PARTITION_INCLUDED #define SQL_PARTITION_INCLUDED -/* Copyright (c) 2006, 2013, Oracle and/or its affiliates. All rights reserved. +/* Copyright (c) 2006, 2017, Oracle and/or its affiliates. All rights reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -272,10 +272,10 @@ char *generate_partition_syntax(partition_info *part_info, const char *current_comment_start); #endif -void create_partition_name(char *out, const char *in1, +bool create_partition_name(char *out, const char *in1, const char *in2, uint name_variant, bool translate); -void create_subpartition_name(char *out, const char *in1, +bool create_subpartition_name(char *out, const char *in1, const char *in2, const char *in3, uint name_variant); From f2f6025a445d9a799ccce27bc9124c3a63c28764 Mon Sep 17 00:00:00 2001 From: Arun Kuruvila Date: Thu, 24 Aug 2017 14:19:38 +0530 Subject: [PATCH 010/199] Bug#26482173: TLS CIPHER NEGOTIATION INCORRECTLY MATCHES ON LAST BYTE ONLY (YASSL) Description:- TLS cipher negociation happens incorrectly leading to the use of a different Analysis:- YaSSL based MySQL server will compare only the last byte of each cipher sent in the Client Hello message. This can cause TLS connections to fail, due to the server picking a cipher which the client doesn't actually support. Fix:- A fix for detecting cipher suites with non leading zeros is included as YaSSL only supports cipher suites with leading zeros. --- extra/yassl/README | 8 ++ extra/yassl/certs/ca-cert.pem | 96 ++++++++++++------------ extra/yassl/certs/client-cert.pem | 99 ++++++++++++------------ extra/yassl/certs/server-cert.pem | 120 +++++++++++++++--------------- extra/yassl/include/openssl/ssl.h | 4 +- extra/yassl/src/yassl_imp.cpp | 6 +- extra/yassl/src/yassl_int.cpp | 13 +++- 7 files changed, 182 insertions(+), 164 deletions(-) diff --git a/extra/yassl/README b/extra/yassl/README index a3d4f60f561..de1bf5132aa 100644 --- a/extra/yassl/README +++ b/extra/yassl/README @@ -12,6 +12,14 @@ before calling SSL_new(); *** end Note *** +yaSSL Release notes, version 2.4.4 (8/8/2017) + This release of yaSSL fixes an interop issue. A fix for detecting cipher + suites with non leading zeros is included as yaSSL only supports cipher + suites with leading zeros. Thanks for the report from Security Innovation + and Oracle. + + Users interoping with other SSL stacks should update. + yaSSL Release notes, version 2.4.2 (9/22/2016) This release of yaSSL fixes a medium security vulnerability. A fix for potential AES side channel leaks is included that a local user monitoring diff --git a/extra/yassl/certs/ca-cert.pem b/extra/yassl/certs/ca-cert.pem index 7e64eb47961..8b34ea43dd2 100644 --- a/extra/yassl/certs/ca-cert.pem +++ b/extra/yassl/certs/ca-cert.pem @@ -1,40 +1,13 @@ ------BEGIN CERTIFICATE----- -MIIEqjCCA5KgAwIBAgIJAJpBR82hFGKMMA0GCSqGSIb3DQEBBQUAMIGUMQswCQYD -VQQGEwJVUzEQMA4GA1UECAwHTW9udGFuYTEQMA4GA1UEBwwHQm96ZW1hbjERMA8G -A1UECgwIU2F3dG9vdGgxEzARBgNVBAsMCkNvbnN1bHRpbmcxGDAWBgNVBAMMD3d3 -dy53b2xmc3NsLmNvbTEfMB0GCSqGSIb3DQEJARYQaW5mb0B3b2xmc3NsLmNvbTAe -Fw0xNDA3MTEwMzIwMDhaFw0xNzA0MDYwMzIwMDhaMIGUMQswCQYDVQQGEwJVUzEQ -MA4GA1UECAwHTW9udGFuYTEQMA4GA1UEBwwHQm96ZW1hbjERMA8GA1UECgwIU2F3 -dG9vdGgxEzARBgNVBAsMCkNvbnN1bHRpbmcxGDAWBgNVBAMMD3d3dy53b2xmc3Ns -LmNvbTEfMB0GCSqGSIb3DQEJARYQaW5mb0B3b2xmc3NsLmNvbTCCASIwDQYJKoZI -hvcNAQEBBQADggEPADCCAQoCggEBAL8Myi0Ush6EQlvNOB9K8k11EPG2NZ/fyn0D -mNOs3gNm7irx2LB9bgdUCxCYIU2AyxIg58xP3kV9yXJ3MurKkLtpUhADL6jzlcXx -i2JWG+9nb6QQQZWtCpvjpcCw0nB2UDBbqOgILHztp6J6jTgpHKzH7fJ8lbCVgn1J -XDjNdyXvvYB1U5Q8PcpjW58VtdMdEy8Z0TzbdjrMuH3J5cLX2kBv2CHccxtCLVOc -/hr8fat6Nj+Y3oR8BWfOahQ4h6nxjLVoy2h/cSAr9aBj9VYvoybSt2+xWhfXOJkI -/pNYb/7DE0kIFgunTWcAUjFnI06Y7VFFHbkE2Qvs2CizS73tNnkCAwEAAaOB/DCB -+TAdBgNVHQ4EFgQUJ45nEXTDJh0/7TNjs6TYHTDl6NUwgckGA1UdIwSBwTCBvoAU -J45nEXTDJh0/7TNjs6TYHTDl6NWhgZqkgZcwgZQxCzAJBgNVBAYTAlVTMRAwDgYD -VQQIDAdNb250YW5hMRAwDgYDVQQHDAdCb3plbWFuMREwDwYDVQQKDAhTYXd0b290 -aDETMBEGA1UECwwKQ29uc3VsdGluZzEYMBYGA1UEAwwPd3d3LndvbGZzc2wuY29t -MR8wHQYJKoZIhvcNAQkBFhBpbmZvQHdvbGZzc2wuY29tggkAmkFHzaEUYowwDAYD -VR0TBAUwAwEB/zANBgkqhkiG9w0BAQUFAAOCAQEAeXgMbXmIkfw6FZz5J2IW8CEf -+n0/oqgyHvfyEal0FnRe3BjK8AAq1QMGJjDxR4P9Mm787apPfQxjYDEvfAy/mWaH -7ScIhi3EM+iYIxz+o9uaSU78WkLvccM/rdxKqNKjHQmsMwR7hvNtAFmjyNvRPHP2 -DpDWXkngvzZjCHulsI81O1aMETVJBBzQ57pWxQ0KkY3Wt2IZNBJSTNJtfMU9DxiB -VMv2POWE0tZxFewaNAvwoCF0Q8ijsN/ZZ9rirZNI+KCHvXkU4GIK3/cxLjF70TIq -Cv5dFO/ZZFDkg5G8cA3XiI3ZvIQOxRqzv2QCTlGRpKKFFYOv8FubKElfsrMD2A== ------END CERTIFICATE----- Certificate: Data: Version: 3 (0x2) Serial Number: - 9a:41:47:cd:a1:14:62:8c - Signature Algorithm: sha1WithRSAEncryption + b7:b6:90:33:66:1b:6b:23 + Signature Algorithm: sha256WithRSAEncryption Issuer: C=US, ST=Montana, L=Bozeman, O=Sawtooth, OU=Consulting, CN=www.wolfssl.com/emailAddress=info@wolfssl.com Validity - Not Before: Jul 11 03:20:08 2014 GMT - Not After : Apr 6 03:20:08 2017 GMT + Not Before: Aug 11 20:07:37 2016 GMT + Not After : May 8 20:07:37 2019 GMT Subject: C=US, ST=Montana, L=Bozeman, O=Sawtooth, OU=Consulting, CN=www.wolfssl.com/emailAddress=info@wolfssl.com Subject Public Key Info: Public Key Algorithm: rsaEncryption @@ -65,23 +38,50 @@ Certificate: X509v3 Authority Key Identifier: keyid:27:8E:67:11:74:C3:26:1D:3F:ED:33:63:B3:A4:D8:1D:30:E5:E8:D5 DirName:/C=US/ST=Montana/L=Bozeman/O=Sawtooth/OU=Consulting/CN=www.wolfssl.com/emailAddress=info@wolfssl.com - serial:9A:41:47:CD:A1:14:62:8C + serial:B7:B6:90:33:66:1B:6B:23 X509v3 Basic Constraints: CA:TRUE - Signature Algorithm: sha1WithRSAEncryption - 79:78:0c:6d:79:88:91:fc:3a:15:9c:f9:27:62:16:f0:21:1f: - fa:7d:3f:a2:a8:32:1e:f7:f2:11:a9:74:16:74:5e:dc:18:ca: - f0:00:2a:d5:03:06:26:30:f1:47:83:fd:32:6e:fc:ed:aa:4f: - 7d:0c:63:60:31:2f:7c:0c:bf:99:66:87:ed:27:08:86:2d:c4: - 33:e8:98:23:1c:fe:a3:db:9a:49:4e:fc:5a:42:ef:71:c3:3f: - ad:dc:4a:a8:d2:a3:1d:09:ac:33:04:7b:86:f3:6d:00:59:a3: - c8:db:d1:3c:73:f6:0e:90:d6:5e:49:e0:bf:36:63:08:7b:a5: - b0:8f:35:3b:56:8c:11:35:49:04:1c:d0:e7:ba:56:c5:0d:0a: - 91:8d:d6:b7:62:19:34:12:52:4c:d2:6d:7c:c5:3d:0f:18:81: - 54:cb:f6:3c:e5:84:d2:d6:71:15:ec:1a:34:0b:f0:a0:21:74: - 43:c8:a3:b0:df:d9:67:da:e2:ad:93:48:f8:a0:87:bd:79:14: - e0:62:0a:df:f7:31:2e:31:7b:d1:32:2a:0a:fe:5d:14:ef:d9: - 64:50:e4:83:91:bc:70:0d:d7:88:8d:d9:bc:84:0e:c5:1a:b3: - bf:64:02:4e:51:91:a4:a2:85:15:83:af:f0:5b:9b:28:49:5f: - b2:b3:03:d8 + Signature Algorithm: sha256WithRSAEncryption + 0e:93:48:44:4a:72:96:60:71:25:82:a9:2c:ca:60:5b:f2:88: + 3e:cf:11:74:5a:11:4a:dc:d9:d8:f6:58:2c:05:d3:56:d9:e9: + 8f:37:ef:8e:3e:3b:ff:22:36:00:ca:d8:e2:96:3f:a7:d1:ed: + 1f:de:7a:b0:d7:8f:36:bd:41:55:1e:d4:b9:86:3b:87:25:69: + 35:60:48:d6:e4:5a:94:ce:a2:fa:70:38:36:c4:85:b4:4b:23: + fe:71:9e:2f:db:06:c7:b5:9c:21:f0:3e:7c:eb:91:f8:5c:09: + fd:84:43:a4:b3:4e:04:0c:22:31:71:6a:48:c8:ab:bb:e8:ce: + fa:67:15:1a:3a:82:98:43:33:b5:0e:1f:1e:89:f8:37:de:1b: + e6:b5:a0:f4:a2:8b:b7:1c:90:ba:98:6d:94:21:08:80:5d:f3: + bf:66:ad:c9:72:28:7a:6a:48:ee:cf:63:69:31:8c:c5:8e:66: + da:4b:78:65:e8:03:3a:4b:f8:cc:42:54:d3:52:5c:2d:04:ae: + 26:87:e1:7e:40:cb:45:41:16:4b:6e:a3:2e:4a:76:bd:29:7f: + 1c:53:37:06:ad:e9:5b:6a:d6:b7:4e:94:a2:7c:e8:ac:4e:a6: + 50:3e:2b:32:9e:68:42:1b:e4:59:67:61:ea:c7:9a:51:9c:1c: + 55:a3:77:76 +-----BEGIN CERTIFICATE----- +MIIEqjCCA5KgAwIBAgIJALe2kDNmG2sjMA0GCSqGSIb3DQEBCwUAMIGUMQswCQYD +VQQGEwJVUzEQMA4GA1UECAwHTW9udGFuYTEQMA4GA1UEBwwHQm96ZW1hbjERMA8G +A1UECgwIU2F3dG9vdGgxEzARBgNVBAsMCkNvbnN1bHRpbmcxGDAWBgNVBAMMD3d3 +dy53b2xmc3NsLmNvbTEfMB0GCSqGSIb3DQEJARYQaW5mb0B3b2xmc3NsLmNvbTAe +Fw0xNjA4MTEyMDA3MzdaFw0xOTA1MDgyMDA3MzdaMIGUMQswCQYDVQQGEwJVUzEQ +MA4GA1UECAwHTW9udGFuYTEQMA4GA1UEBwwHQm96ZW1hbjERMA8GA1UECgwIU2F3 +dG9vdGgxEzARBgNVBAsMCkNvbnN1bHRpbmcxGDAWBgNVBAMMD3d3dy53b2xmc3Ns +LmNvbTEfMB0GCSqGSIb3DQEJARYQaW5mb0B3b2xmc3NsLmNvbTCCASIwDQYJKoZI +hvcNAQEBBQADggEPADCCAQoCggEBAL8Myi0Ush6EQlvNOB9K8k11EPG2NZ/fyn0D +mNOs3gNm7irx2LB9bgdUCxCYIU2AyxIg58xP3kV9yXJ3MurKkLtpUhADL6jzlcXx +i2JWG+9nb6QQQZWtCpvjpcCw0nB2UDBbqOgILHztp6J6jTgpHKzH7fJ8lbCVgn1J +XDjNdyXvvYB1U5Q8PcpjW58VtdMdEy8Z0TzbdjrMuH3J5cLX2kBv2CHccxtCLVOc +/hr8fat6Nj+Y3oR8BWfOahQ4h6nxjLVoy2h/cSAr9aBj9VYvoybSt2+xWhfXOJkI +/pNYb/7DE0kIFgunTWcAUjFnI06Y7VFFHbkE2Qvs2CizS73tNnkCAwEAAaOB/DCB ++TAdBgNVHQ4EFgQUJ45nEXTDJh0/7TNjs6TYHTDl6NUwgckGA1UdIwSBwTCBvoAU +J45nEXTDJh0/7TNjs6TYHTDl6NWhgZqkgZcwgZQxCzAJBgNVBAYTAlVTMRAwDgYD +VQQIDAdNb250YW5hMRAwDgYDVQQHDAdCb3plbWFuMREwDwYDVQQKDAhTYXd0b290 +aDETMBEGA1UECwwKQ29uc3VsdGluZzEYMBYGA1UEAwwPd3d3LndvbGZzc2wuY29t +MR8wHQYJKoZIhvcNAQkBFhBpbmZvQHdvbGZzc2wuY29tggkAt7aQM2YbayMwDAYD +VR0TBAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAQEADpNIREpylmBxJYKpLMpgW/KI +Ps8RdFoRStzZ2PZYLAXTVtnpjzfvjj47/yI2AMrY4pY/p9HtH956sNePNr1BVR7U +uYY7hyVpNWBI1uRalM6i+nA4NsSFtEsj/nGeL9sGx7WcIfA+fOuR+FwJ/YRDpLNO +BAwiMXFqSMiru+jO+mcVGjqCmEMztQ4fHon4N94b5rWg9KKLtxyQuphtlCEIgF3z +v2atyXIoempI7s9jaTGMxY5m2kt4ZegDOkv4zEJU01JcLQSuJofhfkDLRUEWS26j +Lkp2vSl/HFM3Bq3pW2rWt06UonzorE6mUD4rMp5oQhvkWWdh6seaUZwcVaN3dg== +-----END CERTIFICATE----- diff --git a/extra/yassl/certs/client-cert.pem b/extra/yassl/certs/client-cert.pem index 38330d5380e..9262ad60991 100644 --- a/extra/yassl/certs/client-cert.pem +++ b/extra/yassl/certs/client-cert.pem @@ -2,13 +2,13 @@ Certificate: Data: Version: 3 (0x2) Serial Number: - b6:63:af:8f:5d:62:57:a0 - Signature Algorithm: sha1WithRSAEncryption - Issuer: C=US, ST=Montana, L=Bozeman, O=wolfSSL, OU=Programming, CN=www.wolfssl.com/emailAddress=info@wolfssl.com + b9:bc:90:ed:ad:aa:0a:8c + Signature Algorithm: sha256WithRSAEncryption + Issuer: C=US, ST=Montana, L=Bozeman, O=wolfSSL_2048, OU=Programming-2048, CN=www.wolfssl.com/emailAddress=info@wolfssl.com Validity - Not Before: Jul 11 17:39:44 2014 GMT - Not After : Apr 6 17:39:44 2017 GMT - Subject: C=US, ST=Montana, L=Bozeman, O=wolfSSL, OU=Programming, CN=www.wolfssl.com/emailAddress=info@wolfssl.com + Not Before: Aug 11 20:07:37 2016 GMT + Not After : May 8 20:07:37 2019 GMT + Subject: C=US, ST=Montana, L=Bozeman, O=wolfSSL_2048, OU=Programming-2048, CN=www.wolfssl.com/emailAddress=info@wolfssl.com Subject Public Key Info: Public Key Algorithm: rsaEncryption Public-Key: (2048 bit) @@ -37,51 +37,52 @@ Certificate: 33:D8:45:66:D7:68:87:18:7E:54:0D:70:27:91:C7:26:D7:85:65:C0 X509v3 Authority Key Identifier: keyid:33:D8:45:66:D7:68:87:18:7E:54:0D:70:27:91:C7:26:D7:85:65:C0 - DirName:/C=US/ST=Montana/L=Bozeman/O=wolfSSL/OU=Programming/CN=www.wolfssl.com/emailAddress=info@wolfssl.com - serial:B6:63:AF:8F:5D:62:57:A0 + DirName:/C=US/ST=Montana/L=Bozeman/O=wolfSSL_2048/OU=Programming-2048/CN=www.wolfssl.com/emailAddress=info@wolfssl.com + serial:B9:BC:90:ED:AD:AA:0A:8C X509v3 Basic Constraints: CA:TRUE - Signature Algorithm: sha1WithRSAEncryption - 85:10:90:c5:5d:de:25:8c:f2:57:7b:2d:14:1c:05:f9:71:63: - 40:b0:e3:c1:c1:2e:13:2a:7a:b7:d6:24:58:87:eb:03:fb:0d: - af:e0:f4:d0:c8:bc:51:36:10:4f:79:cc:4f:66:7d:af:99:cb: - 7b:ce:68:94:c6:36:aa:42:6e:8c:78:5b:b2:85:ca:d1:e1:a8: - 31:d1:81:d9:f9:c1:a3:9e:34:43:ef:0a:79:7d:3e:83:61:fc: - 14:5c:d1:dd:bc:0e:d7:51:b7:71:6e:41:7e:8b:2c:5a:9a:cb: - 77:4b:6a:f5:06:ff:02:af:1e:e6:63:4f:bc:44:d9:3f:56:9e: - 09:9c:43:f9:55:21:32:46:82:09:86:a9:7b:74:1c:9e:5a:2a: - bf:03:79:91:cb:f2:29:7f:c9:15:82:89:b9:53:cd:7e:07:90: - a9:5d:76:e1:19:5e:0d:58:b8:59:d5:0d:df:23:ab:6b:63:76: - 19:9e:9c:df:b0:57:49:6c:d0:86:97:c3:6c:3c:fa:e0:56:c2: - 1b:e3:a1:42:1a:58:62:85:9d:74:19:83:08:af:59:90:f8:99: - bd:67:d3:4a:ea:0e:c9:ca:61:8a:0d:8a:42:cc:90:e9:2e:c2: - 54:73:7f:5e:af:8d:e2:32:cb:45:20:d6:19:4d:5b:77:31:cc: - 0f:2d:c0:7e + Signature Algorithm: sha256WithRSAEncryption + 33:85:08:b4:58:0e:a2:00:03:74:de:77:fb:d1:2b:76:9c:97: + 90:20:21:a2:e8:2e:22:50:26:04:76:ba:5b:47:79:e5:52:f7: + c4:0d:79:ff:62:3f:05:7c:c3:08:6c:e0:b7:81:d0:ce:c6:c9: + 46:b9:8e:4b:5f:56:79:4b:13:b6:d1:6b:66:4b:ce:00:0d:e3: + 76:5e:fb:cb:b5:5d:12:31:05:f1:bb:39:f6:86:90:ca:92:56: + a4:a0:75:21:b6:1d:4c:96:c3:45:eb:5a:91:94:32:d3:59:b8: + c9:73:1f:03:a9:81:63:e0:43:c0:1e:c8:65:be:3b:a7:53:c3: + 44:ff:b3:fb:47:84:a8:b6:9d:00:d5:6b:ae:87:f8:bb:35:b2: + 6c:66:0b:11:ee:6f:fe:12:ed:59:79:f1:3e:f2:d3:61:27:8b: + 95:7e:99:75:8d:a4:9f:34:85:f1:25:4d:48:1e:9b:6b:70:f6: + 66:cc:56:b1:a3:02:52:8a:7c:aa:af:07:da:97:c6:0c:a5:8f: + ed:cb:f5:d8:04:5d:97:0a:5d:5a:2b:49:f5:bd:93:e5:23:9b: + 99:b5:0c:ff:0c:7e:38:82:b2:6e:ab:8a:c9:a7:45:ab:d6:d7: + 93:35:70:07:7e:c8:3d:a5:fe:33:8f:d9:85:c0:c7:5a:02:e4: + 7c:d6:35:9e -----BEGIN CERTIFICATE----- -MIIEqjCCA5KgAwIBAgIJALZjr49dYlegMA0GCSqGSIb3DQEBBQUAMIGUMQswCQYD -VQQGEwJVUzEQMA4GA1UECAwHTW9udGFuYTEQMA4GA1UEBwwHQm96ZW1hbjEQMA4G -A1UECgwHd29sZlNTTDEUMBIGA1UECwwLUHJvZ3JhbW1pbmcxGDAWBgNVBAMMD3d3 -dy53b2xmc3NsLmNvbTEfMB0GCSqGSIb3DQEJARYQaW5mb0B3b2xmc3NsLmNvbTAe -Fw0xNDA3MTExNzM5NDRaFw0xNzA0MDYxNzM5NDRaMIGUMQswCQYDVQQGEwJVUzEQ -MA4GA1UECAwHTW9udGFuYTEQMA4GA1UEBwwHQm96ZW1hbjEQMA4GA1UECgwHd29s -ZlNTTDEUMBIGA1UECwwLUHJvZ3JhbW1pbmcxGDAWBgNVBAMMD3d3dy53b2xmc3Ns -LmNvbTEfMB0GCSqGSIb3DQEJARYQaW5mb0B3b2xmc3NsLmNvbTCCASIwDQYJKoZI -hvcNAQEBBQADggEPADCCAQoCggEBAMMD0Sv+OaQyRTtTyIQrKnx0mr2qKlIHR9am -NrIHMo7Quml7xsNEntSBSP0taKKLZ7uhdcg2LErSG/eLus8N+e/s8YEee5sDR5q/ -Zcx/ZSRppugUiVvkNPfFsBST9Wd7Onp44QFWVpGmE0KN0jxAnEzv0YbfN1EbDKE7 -9fGjSjXk4c6W3xt+v06X0BDoqAgwga8gC0MUxXRntDKCb42GwohAmTaDuh5AciIX -11JlJHOwzu8Zza7/eGx7wBID1E5yDVBtO6M7o5lencjZDIWz2YrZVCbbbfqsu/8l -TMTRefRx04ZAGBOwY7VyTjDEl4SGLVYv1xX3f8Cu9fxb5fuhutMCAwEAAaOB/DCB -+TAdBgNVHQ4EFgQUM9hFZtdohxh+VA1wJ5HHJteFZcAwgckGA1UdIwSBwTCBvoAU -M9hFZtdohxh+VA1wJ5HHJteFZcChgZqkgZcwgZQxCzAJBgNVBAYTAlVTMRAwDgYD -VQQIDAdNb250YW5hMRAwDgYDVQQHDAdCb3plbWFuMRAwDgYDVQQKDAd3b2xmU1NM -MRQwEgYDVQQLDAtQcm9ncmFtbWluZzEYMBYGA1UEAwwPd3d3LndvbGZzc2wuY29t -MR8wHQYJKoZIhvcNAQkBFhBpbmZvQHdvbGZzc2wuY29tggkAtmOvj11iV6AwDAYD -VR0TBAUwAwEB/zANBgkqhkiG9w0BAQUFAAOCAQEAhRCQxV3eJYzyV3stFBwF+XFj -QLDjwcEuEyp6t9YkWIfrA/sNr+D00Mi8UTYQT3nMT2Z9r5nLe85olMY2qkJujHhb -soXK0eGoMdGB2fnBo540Q+8KeX0+g2H8FFzR3bwO11G3cW5BfossWprLd0tq9Qb/ -Aq8e5mNPvETZP1aeCZxD+VUhMkaCCYape3QcnloqvwN5kcvyKX/JFYKJuVPNfgeQ -qV124RleDVi4WdUN3yOra2N2GZ6c37BXSWzQhpfDbDz64FbCG+OhQhpYYoWddBmD -CK9ZkPiZvWfTSuoOycphig2KQsyQ6S7CVHN/Xq+N4jLLRSDWGU1bdzHMDy3Afg== +MIIEyjCCA7KgAwIBAgIJALm8kO2tqgqMMA0GCSqGSIb3DQEBCwUAMIGeMQswCQYD +VQQGEwJVUzEQMA4GA1UECAwHTW9udGFuYTEQMA4GA1UEBwwHQm96ZW1hbjEVMBMG +A1UECgwMd29sZlNTTF8yMDQ4MRkwFwYDVQQLDBBQcm9ncmFtbWluZy0yMDQ4MRgw +FgYDVQQDDA93d3cud29sZnNzbC5jb20xHzAdBgkqhkiG9w0BCQEWEGluZm9Ad29s +ZnNzbC5jb20wHhcNMTYwODExMjAwNzM3WhcNMTkwNTA4MjAwNzM3WjCBnjELMAkG +A1UEBhMCVVMxEDAOBgNVBAgMB01vbnRhbmExEDAOBgNVBAcMB0JvemVtYW4xFTAT +BgNVBAoMDHdvbGZTU0xfMjA0ODEZMBcGA1UECwwQUHJvZ3JhbW1pbmctMjA0ODEY +MBYGA1UEAwwPd3d3LndvbGZzc2wuY29tMR8wHQYJKoZIhvcNAQkBFhBpbmZvQHdv +bGZzc2wuY29tMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAwwPRK/45 +pDJFO1PIhCsqfHSavaoqUgdH1qY2sgcyjtC6aXvGw0Se1IFI/S1oootnu6F1yDYs +StIb94u6zw357+zxgR57mwNHmr9lzH9lJGmm6BSJW+Q098WwFJP1Z3s6enjhAVZW +kaYTQo3SPECcTO/Rht83URsMoTv18aNKNeThzpbfG36/TpfQEOioCDCBryALQxTF +dGe0MoJvjYbCiECZNoO6HkByIhfXUmUkc7DO7xnNrv94bHvAEgPUTnINUG07ozuj +mV6dyNkMhbPZitlUJttt+qy7/yVMxNF59HHThkAYE7BjtXJOMMSXhIYtVi/XFfd/ +wK71/Fvl+6G60wIDAQABo4IBBzCCAQMwHQYDVR0OBBYEFDPYRWbXaIcYflQNcCeR +xybXhWXAMIHTBgNVHSMEgcswgciAFDPYRWbXaIcYflQNcCeRxybXhWXAoYGkpIGh +MIGeMQswCQYDVQQGEwJVUzEQMA4GA1UECAwHTW9udGFuYTEQMA4GA1UEBwwHQm96 +ZW1hbjEVMBMGA1UECgwMd29sZlNTTF8yMDQ4MRkwFwYDVQQLDBBQcm9ncmFtbWlu +Zy0yMDQ4MRgwFgYDVQQDDA93d3cud29sZnNzbC5jb20xHzAdBgkqhkiG9w0BCQEW +EGluZm9Ad29sZnNzbC5jb22CCQC5vJDtraoKjDAMBgNVHRMEBTADAQH/MA0GCSqG +SIb3DQEBCwUAA4IBAQAzhQi0WA6iAAN03nf70St2nJeQICGi6C4iUCYEdrpbR3nl +UvfEDXn/Yj8FfMMIbOC3gdDOxslGuY5LX1Z5SxO20WtmS84ADeN2XvvLtV0SMQXx +uzn2hpDKklakoHUhth1MlsNF61qRlDLTWbjJcx8DqYFj4EPAHshlvjunU8NE/7P7 +R4Sotp0A1Wuuh/i7NbJsZgsR7m/+Eu1ZefE+8tNhJ4uVfpl1jaSfNIXxJU1IHptr +cPZmzFaxowJSinyqrwfal8YMpY/ty/XYBF2XCl1aK0n1vZPlI5uZtQz/DH44grJu +q4rJp0Wr1teTNXAHfsg9pf4zj9mFwMdaAuR81jWe -----END CERTIFICATE----- diff --git a/extra/yassl/certs/server-cert.pem b/extra/yassl/certs/server-cert.pem index f56cba9de70..5504c822fae 100644 --- a/extra/yassl/certs/server-cert.pem +++ b/extra/yassl/certs/server-cert.pem @@ -2,11 +2,11 @@ Certificate: Data: Version: 3 (0x2) Serial Number: 1 (0x1) - Signature Algorithm: sha1WithRSAEncryption + Signature Algorithm: sha256WithRSAEncryption Issuer: C=US, ST=Montana, L=Bozeman, O=Sawtooth, OU=Consulting, CN=www.wolfssl.com/emailAddress=info@wolfssl.com Validity - Not Before: Jul 11 17:20:14 2014 GMT - Not After : Apr 6 17:20:14 2017 GMT + Not Before: Aug 11 20:07:37 2016 GMT + Not After : May 8 20:07:37 2019 GMT Subject: C=US, ST=Montana, L=Bozeman, O=wolfSSL, OU=Support, CN=www.wolfssl.com/emailAddress=info@wolfssl.com Subject Public Key Info: Public Key Algorithm: rsaEncryption @@ -37,32 +37,32 @@ Certificate: X509v3 Authority Key Identifier: keyid:27:8E:67:11:74:C3:26:1D:3F:ED:33:63:B3:A4:D8:1D:30:E5:E8:D5 DirName:/C=US/ST=Montana/L=Bozeman/O=Sawtooth/OU=Consulting/CN=www.wolfssl.com/emailAddress=info@wolfssl.com - serial:9A:41:47:CD:A1:14:62:8C + serial:B7:B6:90:33:66:1B:6B:23 X509v3 Basic Constraints: CA:TRUE - Signature Algorithm: sha1WithRSAEncryption - 3d:8c:70:05:5b:62:4b:bf:6c:b6:48:61:01:10:1d:5e:05:ba: - 55:94:2c:ae:59:6f:97:80:5d:6c:86:ec:9a:eb:15:45:44:e4: - 56:f8:75:ca:8a:45:32:f4:c7:e1:fa:f2:98:1c:91:d3:3f:e8: - 0e:c9:1b:fa:e1:79:99:67:0e:0d:6b:8a:ec:1a:2c:59:c4:34: - 04:8d:39:77:cd:b5:e9:60:5b:82:bf:34:ce:ed:c6:4f:3f:b4: - 5c:4d:8a:b4:f4:0a:04:12:a0:56:c1:e1:33:37:a1:54:87:48: - e9:81:c2:0f:8f:6f:d3:52:4c:4c:32:4c:6b:9f:3a:04:8f:77: - 5d:ad:dc:3d:2b:f2:c9:df:3c:60:5d:d8:fc:86:72:7c:3d:d0: - 84:4b:8c:df:26:43:fe:c0:cc:5b:e1:36:b3:3d:32:28:a3:ef: - 0c:20:d6:b1:50:39:d6:67:a9:8b:84:bc:92:34:eb:19:23:e8: - 10:8f:ea:bd:18:8c:93:27:3c:74:75:8e:58:04:fa:2a:74:44: - 7d:fc:4d:39:df:54:17:ba:78:e1:5d:6a:70:d3:7c:a2:80:81: - e6:19:51:91:c3:44:51:ec:bb:88:a9:53:e1:d7:a9:8c:28:f4: - 21:1c:42:51:09:b4:12:6d:a0:d6:25:09:85:c6:2a:0c:af:a7: - 58:e6:52:8b + Signature Algorithm: sha256WithRSAEncryption + 51:fe:2a:df:07:7e:43:ca:66:8d:15:c4:2b:db:57:b2:06:6d: + 0d:90:66:ff:a5:24:9c:14:ef:81:f2:a4:ab:99:a9:6a:49:20: + a5:d2:71:e7:1c:3c:99:07:c7:47:fc:e8:96:b4:f5:42:30:ce: + 39:01:4b:d1:c2:e8:bc:95:84:87:ce:55:5d:97:9f:cf:78:f3: + 56:9b:a5:08:6d:ac:f6:a5:5c:c4:ef:3e:2a:39:a6:48:26:29: + 7b:2d:e0:cd:a6:8c:57:48:0b:bb:31:32:c2:bf:d9:43:4c:47: + 25:18:81:a8:c9:33:82:41:9b:ba:61:86:d7:84:93:17:24:25: + 36:ca:4d:63:6b:4f:95:79:d8:60:e0:1e:f5:ac:c1:8a:a1:b1: + 7e:85:8e:87:20:2f:08:31:ad:5e:c6:4a:c8:61:f4:9e:07:1e: + a2:22:ed:73:7c:85:ee:fa:62:dc:50:36:aa:fd:c7:9d:aa:18: + 04:fb:ea:cc:2c:68:9b:b3:a9:c2:96:d8:c1:cc:5a:7e:f7:0d: + 9e:08:e0:9d:29:8b:84:46:8f:d3:91:6a:b5:b8:7a:5c:cc:4f: + 55:01:b8:9a:48:a0:94:43:ca:25:47:52:0a:f7:f4:be:b0:d1: + 71:6d:a5:52:4a:65:50:b2:ad:4e:1d:e0:6c:01:d8:fb:43:80: + e6:e4:0c:37 -----BEGIN CERTIFICATE----- -MIIEnjCCA4agAwIBAgIBATANBgkqhkiG9w0BAQUFADCBlDELMAkGA1UEBhMCVVMx +MIIEnjCCA4agAwIBAgIBATANBgkqhkiG9w0BAQsFADCBlDELMAkGA1UEBhMCVVMx EDAOBgNVBAgMB01vbnRhbmExEDAOBgNVBAcMB0JvemVtYW4xETAPBgNVBAoMCFNh d3Rvb3RoMRMwEQYDVQQLDApDb25zdWx0aW5nMRgwFgYDVQQDDA93d3cud29sZnNz -bC5jb20xHzAdBgkqhkiG9w0BCQEWEGluZm9Ad29sZnNzbC5jb20wHhcNMTQwNzEx -MTcyMDE0WhcNMTcwNDA2MTcyMDE0WjCBkDELMAkGA1UEBhMCVVMxEDAOBgNVBAgM +bC5jb20xHzAdBgkqhkiG9w0BCQEWEGluZm9Ad29sZnNzbC5jb20wHhcNMTYwODEx +MjAwNzM3WhcNMTkwNTA4MjAwNzM3WjCBkDELMAkGA1UEBhMCVVMxEDAOBgNVBAgM B01vbnRhbmExEDAOBgNVBAcMB0JvemVtYW4xEDAOBgNVBAoMB3dvbGZTU0wxEDAO BgNVBAsMB1N1cHBvcnQxGDAWBgNVBAMMD3d3dy53b2xmc3NsLmNvbTEfMB0GCSqG SIb3DQEJARYQaW5mb0B3b2xmc3NsLmNvbTCCASIwDQYJKoZIhvcNAQEBBQADggEP @@ -76,24 +76,24 @@ sxEyyZKYhOLJ+NA7bgNCyh8OjjwwgckGA1UdIwSBwTCBvoAUJ45nEXTDJh0/7TNj s6TYHTDl6NWhgZqkgZcwgZQxCzAJBgNVBAYTAlVTMRAwDgYDVQQIDAdNb250YW5h MRAwDgYDVQQHDAdCb3plbWFuMREwDwYDVQQKDAhTYXd0b290aDETMBEGA1UECwwK Q29uc3VsdGluZzEYMBYGA1UEAwwPd3d3LndvbGZzc2wuY29tMR8wHQYJKoZIhvcN -AQkBFhBpbmZvQHdvbGZzc2wuY29tggkAmkFHzaEUYowwDAYDVR0TBAUwAwEB/zAN -BgkqhkiG9w0BAQUFAAOCAQEAPYxwBVtiS79stkhhARAdXgW6VZQsrllvl4BdbIbs -musVRUTkVvh1yopFMvTH4frymByR0z/oDskb+uF5mWcODWuK7BosWcQ0BI05d821 -6WBbgr80zu3GTz+0XE2KtPQKBBKgVsHhMzehVIdI6YHCD49v01JMTDJMa586BI93 -Xa3cPSvyyd88YF3Y/IZyfD3QhEuM3yZD/sDMW+E2sz0yKKPvDCDWsVA51mepi4S8 -kjTrGSPoEI/qvRiMkyc8dHWOWAT6KnREffxNOd9UF7p44V1qcNN8ooCB5hlRkcNE -Uey7iKlT4depjCj0IRxCUQm0Em2g1iUJhcYqDK+nWOZSiw== +AQkBFhBpbmZvQHdvbGZzc2wuY29tggkAt7aQM2YbayMwDAYDVR0TBAUwAwEB/zAN +BgkqhkiG9w0BAQsFAAOCAQEAUf4q3wd+Q8pmjRXEK9tXsgZtDZBm/6UknBTvgfKk +q5mpakkgpdJx5xw8mQfHR/zolrT1QjDOOQFL0cLovJWEh85VXZefz3jzVpulCG2s +9qVcxO8+KjmmSCYpey3gzaaMV0gLuzEywr/ZQ0xHJRiBqMkzgkGbumGG14STFyQl +NspNY2tPlXnYYOAe9azBiqGxfoWOhyAvCDGtXsZKyGH0ngceoiLtc3yF7vpi3FA2 +qv3HnaoYBPvqzCxom7OpwpbYwcxafvcNngjgnSmLhEaP05Fqtbh6XMxPVQG4mkig +lEPKJUdSCvf0vrDRcW2lUkplULKtTh3gbAHY+0OA5uQMNw== -----END CERTIFICATE----- Certificate: Data: Version: 3 (0x2) Serial Number: - 9a:41:47:cd:a1:14:62:8c - Signature Algorithm: sha1WithRSAEncryption + b7:b6:90:33:66:1b:6b:23 + Signature Algorithm: sha256WithRSAEncryption Issuer: C=US, ST=Montana, L=Bozeman, O=Sawtooth, OU=Consulting, CN=www.wolfssl.com/emailAddress=info@wolfssl.com Validity - Not Before: Jul 11 03:20:08 2014 GMT - Not After : Apr 6 03:20:08 2017 GMT + Not Before: Aug 11 20:07:37 2016 GMT + Not After : May 8 20:07:37 2019 GMT Subject: C=US, ST=Montana, L=Bozeman, O=Sawtooth, OU=Consulting, CN=www.wolfssl.com/emailAddress=info@wolfssl.com Subject Public Key Info: Public Key Algorithm: rsaEncryption @@ -124,32 +124,32 @@ Certificate: X509v3 Authority Key Identifier: keyid:27:8E:67:11:74:C3:26:1D:3F:ED:33:63:B3:A4:D8:1D:30:E5:E8:D5 DirName:/C=US/ST=Montana/L=Bozeman/O=Sawtooth/OU=Consulting/CN=www.wolfssl.com/emailAddress=info@wolfssl.com - serial:9A:41:47:CD:A1:14:62:8C + serial:B7:B6:90:33:66:1B:6B:23 X509v3 Basic Constraints: CA:TRUE - Signature Algorithm: sha1WithRSAEncryption - 79:78:0c:6d:79:88:91:fc:3a:15:9c:f9:27:62:16:f0:21:1f: - fa:7d:3f:a2:a8:32:1e:f7:f2:11:a9:74:16:74:5e:dc:18:ca: - f0:00:2a:d5:03:06:26:30:f1:47:83:fd:32:6e:fc:ed:aa:4f: - 7d:0c:63:60:31:2f:7c:0c:bf:99:66:87:ed:27:08:86:2d:c4: - 33:e8:98:23:1c:fe:a3:db:9a:49:4e:fc:5a:42:ef:71:c3:3f: - ad:dc:4a:a8:d2:a3:1d:09:ac:33:04:7b:86:f3:6d:00:59:a3: - c8:db:d1:3c:73:f6:0e:90:d6:5e:49:e0:bf:36:63:08:7b:a5: - b0:8f:35:3b:56:8c:11:35:49:04:1c:d0:e7:ba:56:c5:0d:0a: - 91:8d:d6:b7:62:19:34:12:52:4c:d2:6d:7c:c5:3d:0f:18:81: - 54:cb:f6:3c:e5:84:d2:d6:71:15:ec:1a:34:0b:f0:a0:21:74: - 43:c8:a3:b0:df:d9:67:da:e2:ad:93:48:f8:a0:87:bd:79:14: - e0:62:0a:df:f7:31:2e:31:7b:d1:32:2a:0a:fe:5d:14:ef:d9: - 64:50:e4:83:91:bc:70:0d:d7:88:8d:d9:bc:84:0e:c5:1a:b3: - bf:64:02:4e:51:91:a4:a2:85:15:83:af:f0:5b:9b:28:49:5f: - b2:b3:03:d8 + Signature Algorithm: sha256WithRSAEncryption + 0e:93:48:44:4a:72:96:60:71:25:82:a9:2c:ca:60:5b:f2:88: + 3e:cf:11:74:5a:11:4a:dc:d9:d8:f6:58:2c:05:d3:56:d9:e9: + 8f:37:ef:8e:3e:3b:ff:22:36:00:ca:d8:e2:96:3f:a7:d1:ed: + 1f:de:7a:b0:d7:8f:36:bd:41:55:1e:d4:b9:86:3b:87:25:69: + 35:60:48:d6:e4:5a:94:ce:a2:fa:70:38:36:c4:85:b4:4b:23: + fe:71:9e:2f:db:06:c7:b5:9c:21:f0:3e:7c:eb:91:f8:5c:09: + fd:84:43:a4:b3:4e:04:0c:22:31:71:6a:48:c8:ab:bb:e8:ce: + fa:67:15:1a:3a:82:98:43:33:b5:0e:1f:1e:89:f8:37:de:1b: + e6:b5:a0:f4:a2:8b:b7:1c:90:ba:98:6d:94:21:08:80:5d:f3: + bf:66:ad:c9:72:28:7a:6a:48:ee:cf:63:69:31:8c:c5:8e:66: + da:4b:78:65:e8:03:3a:4b:f8:cc:42:54:d3:52:5c:2d:04:ae: + 26:87:e1:7e:40:cb:45:41:16:4b:6e:a3:2e:4a:76:bd:29:7f: + 1c:53:37:06:ad:e9:5b:6a:d6:b7:4e:94:a2:7c:e8:ac:4e:a6: + 50:3e:2b:32:9e:68:42:1b:e4:59:67:61:ea:c7:9a:51:9c:1c: + 55:a3:77:76 -----BEGIN CERTIFICATE----- -MIIEqjCCA5KgAwIBAgIJAJpBR82hFGKMMA0GCSqGSIb3DQEBBQUAMIGUMQswCQYD +MIIEqjCCA5KgAwIBAgIJALe2kDNmG2sjMA0GCSqGSIb3DQEBCwUAMIGUMQswCQYD VQQGEwJVUzEQMA4GA1UECAwHTW9udGFuYTEQMA4GA1UEBwwHQm96ZW1hbjERMA8G A1UECgwIU2F3dG9vdGgxEzARBgNVBAsMCkNvbnN1bHRpbmcxGDAWBgNVBAMMD3d3 dy53b2xmc3NsLmNvbTEfMB0GCSqGSIb3DQEJARYQaW5mb0B3b2xmc3NsLmNvbTAe -Fw0xNDA3MTEwMzIwMDhaFw0xNzA0MDYwMzIwMDhaMIGUMQswCQYDVQQGEwJVUzEQ +Fw0xNjA4MTEyMDA3MzdaFw0xOTA1MDgyMDA3MzdaMIGUMQswCQYDVQQGEwJVUzEQ MA4GA1UECAwHTW9udGFuYTEQMA4GA1UEBwwHQm96ZW1hbjERMA8GA1UECgwIU2F3 dG9vdGgxEzARBgNVBAsMCkNvbnN1bHRpbmcxGDAWBgNVBAMMD3d3dy53b2xmc3Ns LmNvbTEfMB0GCSqGSIb3DQEJARYQaW5mb0B3b2xmc3NsLmNvbTCCASIwDQYJKoZI @@ -163,11 +163,11 @@ XDjNdyXvvYB1U5Q8PcpjW58VtdMdEy8Z0TzbdjrMuH3J5cLX2kBv2CHccxtCLVOc J45nEXTDJh0/7TNjs6TYHTDl6NWhgZqkgZcwgZQxCzAJBgNVBAYTAlVTMRAwDgYD VQQIDAdNb250YW5hMRAwDgYDVQQHDAdCb3plbWFuMREwDwYDVQQKDAhTYXd0b290 aDETMBEGA1UECwwKQ29uc3VsdGluZzEYMBYGA1UEAwwPd3d3LndvbGZzc2wuY29t -MR8wHQYJKoZIhvcNAQkBFhBpbmZvQHdvbGZzc2wuY29tggkAmkFHzaEUYowwDAYD -VR0TBAUwAwEB/zANBgkqhkiG9w0BAQUFAAOCAQEAeXgMbXmIkfw6FZz5J2IW8CEf -+n0/oqgyHvfyEal0FnRe3BjK8AAq1QMGJjDxR4P9Mm787apPfQxjYDEvfAy/mWaH -7ScIhi3EM+iYIxz+o9uaSU78WkLvccM/rdxKqNKjHQmsMwR7hvNtAFmjyNvRPHP2 -DpDWXkngvzZjCHulsI81O1aMETVJBBzQ57pWxQ0KkY3Wt2IZNBJSTNJtfMU9DxiB -VMv2POWE0tZxFewaNAvwoCF0Q8ijsN/ZZ9rirZNI+KCHvXkU4GIK3/cxLjF70TIq -Cv5dFO/ZZFDkg5G8cA3XiI3ZvIQOxRqzv2QCTlGRpKKFFYOv8FubKElfsrMD2A== +MR8wHQYJKoZIhvcNAQkBFhBpbmZvQHdvbGZzc2wuY29tggkAt7aQM2YbayMwDAYD +VR0TBAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAQEADpNIREpylmBxJYKpLMpgW/KI +Ps8RdFoRStzZ2PZYLAXTVtnpjzfvjj47/yI2AMrY4pY/p9HtH956sNePNr1BVR7U +uYY7hyVpNWBI1uRalM6i+nA4NsSFtEsj/nGeL9sGx7WcIfA+fOuR+FwJ/YRDpLNO +BAwiMXFqSMiru+jO+mcVGjqCmEMztQ4fHon4N94b5rWg9KKLtxyQuphtlCEIgF3z +v2atyXIoempI7s9jaTGMxY5m2kt4ZegDOkv4zEJU01JcLQSuJofhfkDLRUEWS26j +Lkp2vSl/HFM3Bq3pW2rWt06UonzorE6mUD4rMp5oQhvkWWdh6seaUZwcVaN3dg== -----END CERTIFICATE----- diff --git a/extra/yassl/include/openssl/ssl.h b/extra/yassl/include/openssl/ssl.h index 0609dfc0592..0cce783de35 100644 --- a/extra/yassl/include/openssl/ssl.h +++ b/extra/yassl/include/openssl/ssl.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005, 2014, Oracle and/or its affiliates. All rights reserved. + Copyright (c) 2005, 2017, Oracle and/or its affiliates. All rights reserved. Use is subject to license terms. This program is free software; you can redistribute it and/or modify @@ -35,7 +35,7 @@ #include "rsa.h" -#define YASSL_VERSION "2.4.2" +#define YASSL_VERSION "2.4.4" #if defined(__cplusplus) diff --git a/extra/yassl/src/yassl_imp.cpp b/extra/yassl/src/yassl_imp.cpp index f190761604d..c5892388eaf 100644 --- a/extra/yassl/src/yassl_imp.cpp +++ b/extra/yassl/src/yassl_imp.cpp @@ -1,5 +1,5 @@ /* - Copyright (c) 2005, 2014, Oracle and/or its affiliates. All rights reserved. + Copyright (c) 2005, 2017, Oracle and/or its affiliates. All rights reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -1578,6 +1578,10 @@ void ServerHello::Process(input_buffer& input, SSL& ssl) ssl.SetError(badVersion_error); return; } + if (cipher_suite_[0] != 0x00) { + ssl.SetError(unknown_cipher); + return; + } ssl.set_pending(cipher_suite_[1]); ssl.set_random(random_, server_end); if (id_len_) diff --git a/extra/yassl/src/yassl_int.cpp b/extra/yassl/src/yassl_int.cpp index f041850f85f..34a1c3b73a2 100644 --- a/extra/yassl/src/yassl_int.cpp +++ b/extra/yassl/src/yassl_int.cpp @@ -1,5 +1,5 @@ /* - Copyright (c) 2005, 2014, Oracle and/or its affiliates. All rights reserved. + Copyright (c) 2005, 2017, Oracle and/or its affiliates. All rights reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -1399,12 +1399,17 @@ void SSL::matchSuite(const opaque* peer, uint length) // start with best, if a match we are good, Ciphers are at odd index // since all SSL and TLS ciphers have 0x00 first byte for (uint i = 1; i < secure_.get_parms().suites_size_; i += 2) - for (uint j = 1; j < length; j+= 2) - if (secure_.use_parms().suites_[i] == peer[j]) { + for (uint j = 0; (j + 1) < length; j+= 2) { + if (peer[j] != 0x00) { + continue; // only 0x00 first byte supported + } + + if (secure_.use_parms().suites_[i] == peer[j + 1]) { secure_.use_parms().suite_[0] = 0x00; - secure_.use_parms().suite_[1] = peer[j]; + secure_.use_parms().suite_[1] = peer[j + 1]; return; } + } SetError(match_error); } From f7316aa0c9a3909fc7498e7b95d5d3af044a7e21 Mon Sep 17 00:00:00 2001 From: Ajo Robert Date: Thu, 24 Aug 2017 17:03:21 +0530 Subject: [PATCH 011/199] Bug#26361149 MYSQL SERVER CRASHES AT: COL IN(IFNULL(CONST, COL), NAME_CONST('NAME', NULL)) Backport of Bug#19143243 fix. NAME_CONST item can return NULL_ITEM type in case of incorrect arguments. NULL_ITEM has special processing in Item_func_in function. In Item_func_in::fix_length_and_dec an array of possible comparators is created. Since NAME_CONST function has NULL_ITEM type, corresponding array element is empty. Then NAME_CONST is wrapped to ITEM_CACHE. ITEM_CACHE can not return proper type(NULL_ITEM) in Item_func_in::val_int(), so the NULL_ITEM is attempted compared with an empty comparator. The fix is to disable the caching of Item_name_const item. --- sql/item.h | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/sql/item.h b/sql/item.h index 8caa2bc5f9f..9f4e1d24424 100644 --- a/sql/item.h +++ b/sql/item.h @@ -1,7 +1,7 @@ #ifndef ITEM_INCLUDED #define ITEM_INCLUDED -/* Copyright (c) 2000, 2016, Oracle and/or its affiliates. All rights reserved. +/* Copyright (c) 2000, 2017, Oracle and/or its affiliates. All rights reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -1552,6 +1552,12 @@ public: return TRUE; } + virtual bool cache_const_expr_analyzer(uchar **arg) + { + // Item_name_const always wraps a literal, so there is no need to cache it. + return false; + } + int save_in_field(Field *field, bool no_conversions) { return value_item->save_in_field(field, no_conversions); From 765519694d2c1515c56f7f7aadaf8004b5ad8a45 Mon Sep 17 00:00:00 2001 From: Sergei Petrunia Date: Thu, 17 Aug 2017 22:59:41 +0300 Subject: [PATCH 012/199] Enable basic XA between MyRocks and the binlog This allows basic master crash-safety - Un-comment and update relevant parts of the code - Make rocksdb_rpl suite work like other MyRocks testsuites (load the MyRocks plugin, don't start if it is not compiled in, etc) - For now, disable all tests in the rocksdb_rpl suite. - MariaDB-fication of rpl_rocksdb_2p_crash_recover test. --- storage/rocksdb/ha_rocksdb.cc | 23 ++++++++--------- .../rocksdb_rpl/include/have_rocksdb.inc | 10 ++++++++ .../rocksdb_rpl/include/have_rocksdb.opt | 12 +++++++++ storage/rocksdb/mysql-test/rocksdb_rpl/my.cnf | 17 +++++++++++++ .../r/rpl_rocksdb_2pc_crash_recover.result | 10 ++++---- .../rocksdb/mysql-test/rocksdb_rpl/suite.opt | 2 ++ .../rocksdb/mysql-test/rocksdb_rpl/suite.pm | 25 +++++++++++++++++++ .../mysql-test/rocksdb_rpl/t/disabled.def | 18 +++++++++++++ .../rpl_rocksdb_2pc_crash_recover-master.opt | 2 +- .../t/rpl_rocksdb_2pc_crash_recover-slave.opt | 2 +- .../t/rpl_rocksdb_2pc_crash_recover.test | 10 ++++---- storage/rocksdb/rdb_datadic.cc | 11 +++++--- storage/rocksdb/rdb_datadic.h | 1 - storage/rocksdb/rdb_mariadb_port.h | 3 +++ 14 files changed, 116 insertions(+), 30 deletions(-) create mode 100644 storage/rocksdb/mysql-test/rocksdb_rpl/include/have_rocksdb.inc create mode 100644 storage/rocksdb/mysql-test/rocksdb_rpl/include/have_rocksdb.opt create mode 100644 storage/rocksdb/mysql-test/rocksdb_rpl/my.cnf create mode 100644 storage/rocksdb/mysql-test/rocksdb_rpl/suite.opt create mode 100644 storage/rocksdb/mysql-test/rocksdb_rpl/suite.pm create mode 100644 storage/rocksdb/mysql-test/rocksdb_rpl/t/disabled.def diff --git a/storage/rocksdb/ha_rocksdb.cc b/storage/rocksdb/ha_rocksdb.cc index 293d995126a..bfc7ffd921c 100644 --- a/storage/rocksdb/ha_rocksdb.cc +++ b/storage/rocksdb/ha_rocksdb.cc @@ -170,7 +170,6 @@ static std::shared_ptr properties_collector_factory; Rdb_dict_manager dict_manager; Rdb_cf_manager cf_manager; Rdb_ddl_manager ddl_manager; -const char *m_mysql_gtid; Rdb_binlog_manager binlog_manager; #ifndef _WIN32 @@ -1668,8 +1667,11 @@ protected: public: const char *m_mysql_log_file_name; my_off_t m_mysql_log_offset; +#ifdef MARIAROCKS_NOT_YET + // TODO: MariaDB probably doesn't need these at all: const char *m_mysql_gtid; const char *m_mysql_max_gtid; +#endif String m_detailed_error; int64_t m_snapshot_timestamp = 0; bool m_ddl_transaction; @@ -1856,13 +1858,10 @@ public: rollback(); return true; } else { -#ifdef MARIAROCKS_NOT_YET - my_core::thd_binlog_pos(m_thd, &m_mysql_log_file_name, - &m_mysql_log_offset, &m_mysql_gtid, - &m_mysql_max_gtid); + mysql_bin_log_commit_pos(m_thd, &m_mysql_log_offset, + &m_mysql_log_file_name); binlog_manager.update(m_mysql_log_file_name, m_mysql_log_offset, - m_mysql_max_gtid, get_write_batch()); -#endif + get_write_batch()); return commit_no_binlog(); } } @@ -2683,26 +2682,25 @@ static bool rocksdb_flush_wal(handlerton* hton __attribute__((__unused__))) */ static int rocksdb_prepare(handlerton* hton, THD* thd, bool prepare_tx) { -#ifdef MARIAROCKS_NOT_YET // This is "ASYNC_COMMIT" feature which is only in webscalesql bool async=false; -#endif Rdb_transaction *&tx = get_tx_from_thd(thd); if (!tx->can_prepare()) { return HA_EXIT_FAILURE; } -#ifdef MARIAROCKS_NOT_YET // disable prepare/commit if (prepare_tx || (!my_core::thd_test_options(thd, OPTION_NOT_AUTOCOMMIT | OPTION_BEGIN))) { /* We were instructed to prepare the whole transaction, or this is an SQL statement end and autocommit is on */ +#ifdef MARIAROCKS_NOT_YET // disable prepare/commit std::vector slave_gtid_info; my_core::thd_slave_gtid_info(thd, &slave_gtid_info); for (const auto &it : slave_gtid_info) { rocksdb::WriteBatchBase *const write_batch = tx->get_blind_write_batch(); binlog_manager.update_slave_gtid_info(it.id, it.db, it.gtid, write_batch); } +#endif if (tx->is_two_phase()) { if (thd->durability_property == HA_IGNORE_DURABILITY || async) { @@ -2713,10 +2711,10 @@ static int rocksdb_prepare(handlerton* hton, THD* thd, bool prepare_tx) if (!tx->prepare(rdb_xid_to_string(xid))) { return HA_EXIT_FAILURE; } - if (thd->durability_property == HA_IGNORE_DURABILITY + if (thd->durability_property == HA_IGNORE_DURABILITY ) #ifdef MARIAROCKS_NOT_YET && - THDVAR(thd, flush_log_at_trx_commit)) { + THDVAR(thd, flush_log_at_trx_commit)) #endif { #ifdef MARIAROCKS_NOT_YET @@ -2733,7 +2731,6 @@ static int rocksdb_prepare(handlerton* hton, THD* thd, bool prepare_tx) DEBUG_SYNC(thd, "rocksdb.prepared"); } -#endif return HA_EXIT_SUCCESS; } diff --git a/storage/rocksdb/mysql-test/rocksdb_rpl/include/have_rocksdb.inc b/storage/rocksdb/mysql-test/rocksdb_rpl/include/have_rocksdb.inc new file mode 100644 index 00000000000..1f762d38c64 --- /dev/null +++ b/storage/rocksdb/mysql-test/rocksdb_rpl/include/have_rocksdb.inc @@ -0,0 +1,10 @@ +if (`SELECT COUNT(*) = 0 FROM INFORMATION_SCHEMA.ENGINES WHERE engine = 'rocksdb' AND support IN ('YES', 'DEFAULT', 'ENABLED')`) +{ + --skip Test requires engine RocksDB. +} + +--disable_query_log +# Table statistics can vary depending on when the memtables are flushed, so +# flush them at the beginning of the test to ensure the test runs consistently. +set global rocksdb_force_flush_memtable_now = true; +--enable_query_log diff --git a/storage/rocksdb/mysql-test/rocksdb_rpl/include/have_rocksdb.opt b/storage/rocksdb/mysql-test/rocksdb_rpl/include/have_rocksdb.opt new file mode 100644 index 00000000000..36d7dda1609 --- /dev/null +++ b/storage/rocksdb/mysql-test/rocksdb_rpl/include/have_rocksdb.opt @@ -0,0 +1,12 @@ +--loose-enable-rocksdb +--loose-enable-rocksdb_global_info +--loose-enable-rocksdb_ddl +--loose-enable-rocksdb_cf_options +--loose-enable_rocksdb_perf_context +--loose-enable_rocksdb_perf_context_global +--loose-enable-rocksdb_index_file_map +--loose-enable-rocksdb_dbstats +--loose-enable-rocksdb_cfstats +--loose-enable-rocksdb_lock_info +--loose-enable-rocksdb_trx +--loose-enable-rocksdb_locks diff --git a/storage/rocksdb/mysql-test/rocksdb_rpl/my.cnf b/storage/rocksdb/mysql-test/rocksdb_rpl/my.cnf new file mode 100644 index 00000000000..2beaf514cee --- /dev/null +++ b/storage/rocksdb/mysql-test/rocksdb_rpl/my.cnf @@ -0,0 +1,17 @@ +!include include/default_my.cnf + +[server] +skip-innodb +default-storage-engine=rocksdb + + +sql-mode=NO_ENGINE_SUBSTITUTION +explicit-defaults-for-timestamp=1 +loose-rocksdb_lock_wait_timeout=1 +loose-rocksdb_strict_collation_check=0 + +loose-rocksdb-flush-log-at-trx-commit=0 + +# The following is to get rid of the harmless +# "Deadlock found when trying to get lock" errors, see MDEV-12285. +log-warnings=1 diff --git a/storage/rocksdb/mysql-test/rocksdb_rpl/r/rpl_rocksdb_2pc_crash_recover.result b/storage/rocksdb/mysql-test/rocksdb_rpl/r/rpl_rocksdb_2pc_crash_recover.result index 59d1a231327..cf3fe03a305 100644 --- a/storage/rocksdb/mysql-test/rocksdb_rpl/r/rpl_rocksdb_2pc_crash_recover.result +++ b/storage/rocksdb/mysql-test/rocksdb_rpl/r/rpl_rocksdb_2pc_crash_recover.result @@ -1,19 +1,19 @@ DROP TABLE IF EXISTS t1; create table t1 (a int primary key, msg varchar(255)) engine=rocksdb; SET GLOBAL ROCKSDB_ENABLE_2PC = ON; -SET SESSION debug="d,crash_commit_after_prepare"; +SET SESSION debug_dbug="d,crash_commit_after_prepare"; insert into t1 values (1, 'dogz'); select * from t1; a msg SET GLOBAL ROCKSDB_ENABLE_2PC = ON; -SET SESSION debug="d,crash_commit_after_log"; +SET SESSION debug_dbug="d,crash_commit_after_log"; insert into t1 values (2, 'catz'), (3, 'men'); select * from t1; a msg 2 catz 3 men SET GLOBAL ROCKSDB_ENABLE_2PC = ON; -SET SESSION debug="d,crash_commit_after"; +SET SESSION debug_dbug="d,crash_commit_after"; insert into t1 values (4, 'cars'), (5, 'foo'); select * from t1; a msg @@ -22,7 +22,7 @@ a msg 4 cars 5 foo SET GLOBAL ROCKSDB_ENABLE_2PC = OFF; -SET SESSION debug="d,crash_commit_after_log"; +SET SESSION debug_dbug="d,crash_commit_after_log"; insert into t1 values (6, 'shipz'), (7, 'tankz'); select * from t1; a msg @@ -31,7 +31,7 @@ a msg 4 cars 5 foo SET GLOBAL ROCKSDB_ENABLE_2PC = OFF; -SET SESSION debug="d,crash_commit_after"; +SET SESSION debug_dbug="d,crash_commit_after"; insert into t1 values (8, 'space'), (9, 'time'); select * from t1; a msg diff --git a/storage/rocksdb/mysql-test/rocksdb_rpl/suite.opt b/storage/rocksdb/mysql-test/rocksdb_rpl/suite.opt new file mode 100644 index 00000000000..f5dc0ce891c --- /dev/null +++ b/storage/rocksdb/mysql-test/rocksdb_rpl/suite.opt @@ -0,0 +1,2 @@ +--ignore-db-dirs=.rocksdb --plugin-load=$HA_ROCKSDB_SO --default-storage-engine=rocksdb + diff --git a/storage/rocksdb/mysql-test/rocksdb_rpl/suite.pm b/storage/rocksdb/mysql-test/rocksdb_rpl/suite.pm new file mode 100644 index 00000000000..79c630f87f1 --- /dev/null +++ b/storage/rocksdb/mysql-test/rocksdb_rpl/suite.pm @@ -0,0 +1,25 @@ +package My::Suite::Rocksdb; + +# +# Note: ../rocksdb_sys_vars/suite.pm file has a similar +# function. If you modify this file, consider modifying that one, too. +# +@ISA = qw(My::Suite); +use My::Find; +use File::Basename; +use strict; + +sub is_default { not $::opt_embedded_server } + +my $sst_dump= +::mtr_exe_maybe_exists( + "$::bindir/storage/rocksdb$::opt_vs_config/sst_dump", + "$::path_client_bindir/sst_dump"); +return "RocksDB is not compiled, no sst_dump" unless $sst_dump; +$ENV{MARIAROCKS_SST_DUMP}="$sst_dump"; + +# Temporarily disable testing under valgrind, due to MDEV-12439 +return "RocksDB tests disabled under valgrind" if ($::opt_valgrind); + +bless { }; + diff --git a/storage/rocksdb/mysql-test/rocksdb_rpl/t/disabled.def b/storage/rocksdb/mysql-test/rocksdb_rpl/t/disabled.def new file mode 100644 index 00000000000..c45236fbd37 --- /dev/null +++ b/storage/rocksdb/mysql-test/rocksdb_rpl/t/disabled.def @@ -0,0 +1,18 @@ + +# rpl_rocksdb_2pc_crash_recover + +consistent_snapshot_mixed_engines : Didn't try with MariaDB, yet +multiclient_2pc : Didn't try with MariaDB, yet +rpl_crash_safe_wal_corrupt : Didn't try with MariaDB, yet +rpl_gtid_crash_safe : Didn't try with MariaDB, yet +rpl_gtid_crash_safe_wal_corrupt : Didn't try with MariaDB, yet +rpl_gtid_rocksdb_sys_header : Didn't try with MariaDB, yet +rpl_no_unique_check_on_lag : Didn't try with MariaDB, yet +rpl_no_unique_check_on_lag_mts : Didn't try with MariaDB, yet +rpl_rocksdb_snapshot : Didn't try with MariaDB, yet +rpl_rocksdb_snapshot_without_gtid : Didn't try with MariaDB, yet +rpl_rocksdb_stress_crash : Didn't try with MariaDB, yet +rpl_skip_trx_api_binlog_format : Didn't try with MariaDB, yet +singledelete_idempotent_recovery : Didn't try with MariaDB, yet +singledelete_idempotent_table : Didn't try with MariaDB, yet + diff --git a/storage/rocksdb/mysql-test/rocksdb_rpl/t/rpl_rocksdb_2pc_crash_recover-master.opt b/storage/rocksdb/mysql-test/rocksdb_rpl/t/rpl_rocksdb_2pc_crash_recover-master.opt index 74c2de37100..f8f297c567c 100644 --- a/storage/rocksdb/mysql-test/rocksdb_rpl/t/rpl_rocksdb_2pc_crash_recover-master.opt +++ b/storage/rocksdb/mysql-test/rocksdb_rpl/t/rpl_rocksdb_2pc_crash_recover-master.opt @@ -1 +1 @@ ---gtid_mode=ON --enforce_gtid_consistency --log_bin --log_slave_updates --rocksdb_flush_log_at_trx_commit=1 --rocksdb_write_disable_wal=OFF +--log_bin --log_slave_updates --rocksdb_flush_log_at_trx_commit=1 diff --git a/storage/rocksdb/mysql-test/rocksdb_rpl/t/rpl_rocksdb_2pc_crash_recover-slave.opt b/storage/rocksdb/mysql-test/rocksdb_rpl/t/rpl_rocksdb_2pc_crash_recover-slave.opt index c747adc94d5..7a3f630fda2 100644 --- a/storage/rocksdb/mysql-test/rocksdb_rpl/t/rpl_rocksdb_2pc_crash_recover-slave.opt +++ b/storage/rocksdb/mysql-test/rocksdb_rpl/t/rpl_rocksdb_2pc_crash_recover-slave.opt @@ -1 +1 @@ ---gtid_mode=ON --enforce_gtid_consistency --log_bin --log_slave_updates +--log_bin --log_slave_updates diff --git a/storage/rocksdb/mysql-test/rocksdb_rpl/t/rpl_rocksdb_2pc_crash_recover.test b/storage/rocksdb/mysql-test/rocksdb_rpl/t/rpl_rocksdb_2pc_crash_recover.test index ea1fe3e34d6..3aa006c84be 100644 --- a/storage/rocksdb/mysql-test/rocksdb_rpl/t/rpl_rocksdb_2pc_crash_recover.test +++ b/storage/rocksdb/mysql-test/rocksdb_rpl/t/rpl_rocksdb_2pc_crash_recover.test @@ -10,7 +10,7 @@ create table t1 (a int primary key, msg varchar(255)) engine=rocksdb; SET GLOBAL ROCKSDB_ENABLE_2PC = ON; --exec echo "restart" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect -SET SESSION debug="d,crash_commit_after_prepare"; +SET SESSION debug_dbug="d,crash_commit_after_prepare"; --error 0,2013 insert into t1 values (1, 'dogz'); --enable_reconnect @@ -19,7 +19,7 @@ select * from t1; SET GLOBAL ROCKSDB_ENABLE_2PC = ON; --exec echo "restart" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect -SET SESSION debug="d,crash_commit_after_log"; +SET SESSION debug_dbug="d,crash_commit_after_log"; --error 0,2013 insert into t1 values (2, 'catz'), (3, 'men'); --enable_reconnect @@ -28,7 +28,7 @@ select * from t1; SET GLOBAL ROCKSDB_ENABLE_2PC = ON; --exec echo "restart" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect -SET SESSION debug="d,crash_commit_after"; +SET SESSION debug_dbug="d,crash_commit_after"; --error 0,2013 insert into t1 values (4, 'cars'), (5, 'foo'); --enable_reconnect @@ -37,7 +37,7 @@ select * from t1; SET GLOBAL ROCKSDB_ENABLE_2PC = OFF; --exec echo "restart" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect -SET SESSION debug="d,crash_commit_after_log"; +SET SESSION debug_dbug="d,crash_commit_after_log"; --error 0,2013 insert into t1 values (6, 'shipz'), (7, 'tankz'); --enable_reconnect @@ -46,7 +46,7 @@ select * from t1; SET GLOBAL ROCKSDB_ENABLE_2PC = OFF; --exec echo "restart" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect -SET SESSION debug="d,crash_commit_after"; +SET SESSION debug_dbug="d,crash_commit_after"; --error 0,2013 insert into t1 values (8, 'space'), (9, 'time'); --enable_reconnect diff --git a/storage/rocksdb/rdb_datadic.cc b/storage/rocksdb/rdb_datadic.cc index d2404c2f8b1..4b3dedfedfe 100644 --- a/storage/rocksdb/rdb_datadic.cc +++ b/storage/rocksdb/rdb_datadic.cc @@ -3961,12 +3961,10 @@ void Rdb_binlog_manager::cleanup() {} write succeeded or not is not possible here. @param binlog_name Binlog name @param binlog_pos Binlog pos - @param binlog_gtid Binlog max GTID @param batch WriteBatch */ void Rdb_binlog_manager::update(const char *const binlog_name, const my_off_t binlog_pos, - const char *const binlog_max_gtid, rocksdb::WriteBatchBase *const batch) { if (binlog_name && binlog_pos) { // max binlog length (512) + binlog pos (4) + binlog gtid (57) < 1024 @@ -3974,7 +3972,7 @@ void Rdb_binlog_manager::update(const char *const binlog_name, uchar value_buf[RDB_MAX_BINLOG_INFO_LEN]; m_dict->put_key( batch, m_key_slice, - pack_value(value_buf, binlog_name, binlog_pos, binlog_max_gtid)); + pack_value(value_buf, binlog_name, binlog_pos, NULL)); } } @@ -4009,7 +4007,6 @@ bool Rdb_binlog_manager::read(char *const binlog_name, @param buf Preallocated buffer to set binlog info. @param binlog_name Binlog name @param binlog_pos Binlog pos - @param binlog_gtid Binlog GTID @return rocksdb::Slice converted from buf and its length */ rocksdb::Slice @@ -4038,15 +4035,21 @@ Rdb_binlog_manager::pack_value(uchar *const buf, const char *const binlog_name, // store binlog gtid length. // If gtid was not set, store 0 instead +#ifdef MARIAROCKS_NOT_YET const uint16_t binlog_gtid_len = binlog_gtid ? (uint16_t)strlen(binlog_gtid) : 0; rdb_netbuf_store_uint16(buf + pack_len, binlog_gtid_len); +#endif pack_len += sizeof(uint16); + // MariaDB: + rdb_netbuf_store_uint16(buf + pack_len, 0); +#ifdef MARIAROCKS_NOT_YET if (binlog_gtid_len > 0) { // store binlog gtid memcpy(buf + pack_len, binlog_gtid, binlog_gtid_len); pack_len += binlog_gtid_len; } +#endif return rocksdb::Slice((char *)buf, pack_len); } diff --git a/storage/rocksdb/rdb_datadic.h b/storage/rocksdb/rdb_datadic.h index 9c7cd956eb3..2711ace9e47 100644 --- a/storage/rocksdb/rdb_datadic.h +++ b/storage/rocksdb/rdb_datadic.h @@ -1082,7 +1082,6 @@ public: bool init(Rdb_dict_manager *const dict); void cleanup(); void update(const char *const binlog_name, const my_off_t binlog_pos, - const char *const binlog_max_gtid, rocksdb::WriteBatchBase *const batch); bool read(char *const binlog_name, my_off_t *const binlog_pos, char *const binlog_gtid) const; diff --git a/storage/rocksdb/rdb_mariadb_port.h b/storage/rocksdb/rdb_mariadb_port.h index 1661338030f..627674905cc 100644 --- a/storage/rocksdb/rdb_mariadb_port.h +++ b/storage/rocksdb/rdb_mariadb_port.h @@ -49,4 +49,7 @@ typedef struct my_io_perf_struct my_io_perf_t; std::vector split_into_vector(const std::string& input, char delimiter); +void +mysql_bin_log_commit_pos(THD *thd, ulonglong *out_pos, const char **out_file); + #endif From fba53e777564704016bbe3a105fa5419c82979a8 Mon Sep 17 00:00:00 2001 From: Sergei Petrunia Date: Fri, 18 Aug 2017 15:42:58 +0300 Subject: [PATCH 013/199] Make MyRocks test stable: Data_length may vary slightly as well --- .../r/rocksdb_flush_memtable_on_analyze_basic.result | 4 ++-- .../t/rocksdb_flush_memtable_on_analyze_basic.test | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/storage/rocksdb/mysql-test/rocksdb_sys_vars/r/rocksdb_flush_memtable_on_analyze_basic.result b/storage/rocksdb/mysql-test/rocksdb_sys_vars/r/rocksdb_flush_memtable_on_analyze_basic.result index 165f3811f84..905feec9b1a 100644 --- a/storage/rocksdb/mysql-test/rocksdb_sys_vars/r/rocksdb_flush_memtable_on_analyze_basic.result +++ b/storage/rocksdb/mysql-test/rocksdb_sys_vars/r/rocksdb_flush_memtable_on_analyze_basic.result @@ -48,11 +48,11 @@ a b 3 3 SHOW TABLE STATUS LIKE 't1'; Name Engine Version Row_format Rows Avg_row_length Data_length Max_data_length Index_length Data_free Auto_increment Create_time Update_time Check_time Collation Checksum Create_options Comment -t1 ROCKSDB 10 Fixed # # 69 0 0 0 4 NULL NULL NULL latin1_swedish_ci NULL +t1 ROCKSDB 10 Fixed # # # 0 0 0 4 NULL NULL NULL latin1_swedish_ci NULL ANALYZE TABLE t1; Table Op Msg_type Msg_text test.t1 analyze status OK SHOW TABLE STATUS LIKE 't1'; Name Engine Version Row_format Rows Avg_row_length Data_length Max_data_length Index_length Data_free Auto_increment Create_time Update_time Check_time Collation Checksum Create_options Comment -t1 ROCKSDB 10 Fixed # # 24 0 0 0 4 NULL NULL NULL latin1_swedish_ci NULL +t1 ROCKSDB 10 Fixed # # # 0 0 0 4 NULL NULL NULL latin1_swedish_ci NULL DROP TABLE t1; diff --git a/storage/rocksdb/mysql-test/rocksdb_sys_vars/t/rocksdb_flush_memtable_on_analyze_basic.test b/storage/rocksdb/mysql-test/rocksdb_sys_vars/t/rocksdb_flush_memtable_on_analyze_basic.test index c7e04f89498..574375cd1ea 100644 --- a/storage/rocksdb/mysql-test/rocksdb_sys_vars/t/rocksdb_flush_memtable_on_analyze_basic.test +++ b/storage/rocksdb/mysql-test/rocksdb_sys_vars/t/rocksdb_flush_memtable_on_analyze_basic.test @@ -37,10 +37,10 @@ INSERT INTO t1 (b) VALUES (3); --sorted_result SELECT * FROM t1; ---replace_column 5 # 6 # +--replace_column 5 # 6 # 7 # SHOW TABLE STATUS LIKE 't1'; ANALYZE TABLE t1; ---replace_column 5 # 6 # +--replace_column 5 # 6 # 7 # SHOW TABLE STATUS LIKE 't1'; DROP TABLE t1; From 575afea7864f0118464e55fd4435ddcb317ac0e8 Mon Sep 17 00:00:00 2001 From: Sergei Petrunia Date: Fri, 18 Aug 2017 16:22:05 +0300 Subject: [PATCH 014/199] MyRocks: Remove todo-#ifdef in Rdb_trx_info_aggregator::process_tran --- storage/rocksdb/ha_rocksdb.cc | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/storage/rocksdb/ha_rocksdb.cc b/storage/rocksdb/ha_rocksdb.cc index bfc7ffd921c..31bece23cd4 100644 --- a/storage/rocksdb/ha_rocksdb.cc +++ b/storage/rocksdb/ha_rocksdb.cc @@ -3143,11 +3143,7 @@ public: const auto state_it = state_map.find(rdb_trx->GetState()); DBUG_ASSERT(state_it != state_map.end()); -#ifdef MARIAROCKS_NOT_YET - const int is_replication = (thd->rli_slave != nullptr); -#else - const int is_replication= false; -#endif + const int is_replication = (thd->rgi_slave != nullptr); uint32_t waiting_cf_id; std::string waiting_key; rdb_trx->GetWaitingTxns(&waiting_cf_id, &waiting_key), From c52ffbeba9e90fb5df04de450406734f2abc89be Mon Sep 17 00:00:00 2001 From: Sergei Petrunia Date: Fri, 18 Aug 2017 16:26:12 +0300 Subject: [PATCH 015/199] Remove garbage code --- storage/rocksdb/ha_rocksdb.cc | 2 -- 1 file changed, 2 deletions(-) diff --git a/storage/rocksdb/ha_rocksdb.cc b/storage/rocksdb/ha_rocksdb.cc index 31bece23cd4..bf124584759 100644 --- a/storage/rocksdb/ha_rocksdb.cc +++ b/storage/rocksdb/ha_rocksdb.cc @@ -3838,8 +3838,6 @@ static int rocksdb_init_func(void *const p) { rocksdb::Options main_opts(*rocksdb_db_options, cf_options_map->get_defaults()); -#ifdef MARIAROCKS_NOT_YET -#endif rocksdb::TransactionDBOptions tx_db_options; tx_db_options.transaction_lock_timeout = 2; // 2 seconds tx_db_options.custom_mutex_factory = std::make_shared(); From 5ac61b2af0cf37eeed5050a91819d6d273f037a5 Mon Sep 17 00:00:00 2001 From: Anushree Prakash B Date: Fri, 8 Sep 2017 18:29:07 +0530 Subject: [PATCH 016/199] Bug#26372491 - RCE THROUGH THE MISHANDLE OF BACKSLASH DESCRIPTION: =========== The bug is related to incorrect parsing of SQL queries when typed in on the CLI. The incorrect parsing can result in unexpected results. ANALYSIS: ======== The scenarios mainly happens for identifier names with a typical combination of backslashes and backticks. The incorrect parsing can either result in executing additional queries or can result in query truncation. This can impact mysqldump as well. FIX: === The fix makes sure that such identifier names are correctly parsed and a proper query is sent to the server for execution. (cherry picked from commit 31a372aa1c2b93dc75267d1f05a7f7fca6080dc0) --- client/mysql.cc | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/client/mysql.cc b/client/mysql.cc index d09499c120a..715d74f18b2 100644 --- a/client/mysql.cc +++ b/client/mysql.cc @@ -2119,7 +2119,10 @@ static bool add_line(String &buffer,char *line,char *in_string, if (*in_string || inchar == 'N') // \N is short for NULL { // Don't allow commands in string *out++='\\'; - *out++= (char) inchar; + if ((inchar == '`') && (*in_string == inchar)) + pos--; + else + *out++= (char) inchar; continue; } if ((com=find_command(NullS,(char) inchar))) From 3fae64b196cfb94ac4084d02c5745285589c6b48 Mon Sep 17 00:00:00 2001 From: Sergei Petrunia Date: Fri, 15 Sep 2017 10:04:49 +0000 Subject: [PATCH 018/199] Copy of commit 184a4a2d82f4f6f3cbcb1015bcdb32bebe73315c Author: Abhinav Sharma Date: Thu Sep 14 11:40:08 2017 -0700 Bump rocksdb submodule Summary: Bump rocksdb to include the fix for rocksdb.trx_info_rpl The bug was introduced in: https://github.com/facebook/rocksdb/pull/2850 Fixed in: https://github.com/facebook/rocksdb/pull/2881 update-submodule: rocksdb Reviewed By: lth Differential Revision: D5834658 fbshipit-source-id: d1551bf --- storage/rocksdb/CMakeLists.txt | 6 + storage/rocksdb/ha_rocksdb.cc | 819 +++++++++++-- storage/rocksdb/ha_rocksdb.h | 72 +- storage/rocksdb/ha_rocksdb_proto.h | 1 + .../include/prefix_index_only_query_check.inc | 21 + storage/rocksdb/mysql-test/rocksdb/my.cnf | 1 + .../rocksdb/r/2pc_group_commit.result | 14 +- .../rocksdb/r/add_index_inplace.result | 13 +- .../rocksdb/r/bulk_load_errors.result | 26 + .../rocksdb/r/bulk_load_unsorted.result | 101 ++ .../r/bulk_load_unsorted_errors.result | 4 + .../mysql-test/rocksdb/r/bytes_written.result | 10 + .../r/cons_snapshot_read_committed.result | 12 +- .../r/cons_snapshot_repeatable_read.result | 2 +- .../r/covered_unpack_info_format.result | 73 ++ .../rocksdb/r/ddl_high_priority.result | 1009 +++++++++++++++++ .../rocksdb/r/deadlock_tracking.result | 490 ++++++++ .../mysql-test/rocksdb/r/drop_table.result | 3 + .../mysql-test/rocksdb/r/drop_table2.result | 2 + .../mysql-test/rocksdb/r/drop_table3.result | 2 + .../rocksdb/r/dup_key_update.result | 4 + .../rocksdb/r/fast_prefix_index_fetch.result | 80 ++ .../mysql-test/rocksdb/r/i_s_ddl.result | 6 +- .../rocksdb/mysql-test/rocksdb/r/index.result | 27 + .../mysql-test/rocksdb/r/index_primary.result | 23 + .../rocksdb/r/index_type_btree.result | 27 + .../rocksdb/r/index_type_hash.result | 27 + .../r/issue243_transactionStatus.result | 18 +- .../rocksdb/r/multi_varchar_sk_lookup.result | 37 + .../mysql-test/rocksdb/r/rocksdb.result | 48 +- .../mysql-test/rocksdb/r/show_engine.result | 33 +- .../rocksdb/r/skip_validate_tmp_table.result | 4 + .../rocksdb/r/truncate_table3.result | 2 + .../mysql-test/rocksdb/r/ttl_primary.result | 4 +- .../mysql-test/rocksdb/r/ttl_secondary.result | 709 ++++++++++++ .../r/ttl_secondary_read_filtering.result | 494 ++++++++ ...ndary_read_filtering_multiple_index.result | 82 ++ .../r/ttl_secondary_with_partitions.result | 389 +++++++ .../mysql-test/rocksdb/r/write_sync.result | 25 +- .../rocksdb/t/2pc_group_commit.test | 6 +- .../rocksdb/t/add_index_inplace.test | 8 +- .../rocksdb/t/bulk_load_errors.test | 26 + .../rocksdb/t/bulk_load_unsorted.test | 136 +++ .../rocksdb/t/bulk_load_unsorted_errors.test | 8 + .../mysql-test/rocksdb/t/bytes_written.test | 22 + .../rocksdb/t/covered_unpack_info_format.test | 79 ++ .../rocksdb/t/ddl_high_priority.test | 18 + .../rocksdb/t/deadlock_tracking.test | 153 +++ .../rocksdb/t/drop_table-master.opt | 1 - .../mysql-test/rocksdb/t/drop_table.test | 5 +- .../mysql-test/rocksdb/t/drop_table2.test | 4 +- .../mysql-test/rocksdb/t/drop_table3.inc | 4 +- .../mysql-test/rocksdb/t/dup_key_update.test | 4 + .../rocksdb/t/fast_prefix_index_fetch.test | 120 ++ .../rocksdb/mysql-test/rocksdb/t/index.inc | 34 + .../mysql-test/rocksdb/t/index_primary.test | 32 + .../rocksdb/t/multi_varchar_sk_lookup.test | 49 + .../rocksdb/mysql-test/rocksdb/t/rocksdb.test | 2 + .../mysql-test/rocksdb/t/show_engine.test | 13 +- .../rocksdb/t/skip_validate_tmp_table.test | 39 + .../mysql-test/rocksdb/t/ttl_primary.test | 14 +- .../rocksdb/t/ttl_secondary-master.opt | 2 + .../mysql-test/rocksdb/t/ttl_secondary.test | 780 +++++++++++++ .../t/ttl_secondary_read_filtering-master.opt | 1 + .../t/ttl_secondary_read_filtering.test | 500 ++++++++ ...condary_read_filtering_multiple_index.test | 87 ++ .../ttl_secondary_with_partitions-master.opt | 1 + .../t/ttl_secondary_with_partitions.test | 300 +++++ .../mysql-test/rocksdb/t/write_sync.test | 27 +- .../rocksdb_rpl/r/multiclient_2pc.result | 9 +- .../r/rpl_ddl_high_priority.result | 39 + .../rocksdb_rpl/t/multiclient_2pc.test | 29 +- .../rocksdb_rpl/t/rpl_ddl_high_priority.test | 2 + .../mysql-test/rocksdb_sys_vars/my.cnf | 4 + ...ksdb_bulk_load_allow_unsorted_basic.result | 100 ++ .../r/rocksdb_concurrent_prepare_basic.result | 14 + ...rocksdb_deadlock_detect_depth_basic.result | 79 ++ .../rocksdb_debug_ttl_ignore_pk_basic.result | 64 ++ ..._force_compute_memtable_stats_basic.result | 4 +- ...pute_memtable_stats_cachetime_basic.result | 68 ++ .../r/rocksdb_large_prefix_basic.result | 64 ++ .../r/rocksdb_manual_wal_flush_basic.result | 14 + .../rocksdb_max_latest_deadlocks_basic.result | 53 + ...rge_tmp_file_removal_delay_ms_basic.result | 93 ++ .../r/rocksdb_sim_cache_size_basic.result | 7 + .../r/rocksdb_use_clock_cache_basic.result | 19 + ...ocksdb_bulk_load_allow_unsorted_basic.test | 18 + .../t/rocksdb_concurrent_prepare_basic.test | 16 + .../rocksdb_deadlock_detect_depth_basic.test | 17 + .../t/rocksdb_debug_ttl_ignore_pk_basic.test | 18 + ...db_force_compute_memtable_stats_basic.test | 4 +- ...ompute_memtable_stats_cachetime_basic.test | 18 + .../t/rocksdb_large_prefix_basic.test | 18 + .../t/rocksdb_manual_wal_flush_basic.test | 16 + .../t/rocksdb_max_latest_deadlocks_basic.test | 17 + ...merge_tmp_file_removal_delay_ms_basic.test | 49 + .../t/rocksdb_sim_cache_size_basic.test | 6 + .../t/rocksdb_use_clock_cache_basic.test | 21 + storage/rocksdb/rdb_buff.h | 16 + storage/rocksdb/rdb_compact_filter.h | 8 + storage/rocksdb/rdb_datadic.cc | 357 +++++- storage/rocksdb/rdb_datadic.h | 66 +- storage/rocksdb/rdb_i_s.cc | 15 +- storage/rocksdb/rdb_index_merge.cc | 186 +-- storage/rocksdb/rdb_index_merge.h | 70 +- storage/rocksdb/rdb_perf_context.cc | 22 + storage/rocksdb/rdb_perf_context.h | 16 +- 107 files changed, 8419 insertions(+), 412 deletions(-) create mode 100644 storage/rocksdb/mysql-test/rocksdb/include/prefix_index_only_query_check.inc create mode 100644 storage/rocksdb/mysql-test/rocksdb/r/bulk_load_unsorted.result create mode 100644 storage/rocksdb/mysql-test/rocksdb/r/bulk_load_unsorted_errors.result create mode 100644 storage/rocksdb/mysql-test/rocksdb/r/bytes_written.result create mode 100644 storage/rocksdb/mysql-test/rocksdb/r/covered_unpack_info_format.result create mode 100644 storage/rocksdb/mysql-test/rocksdb/r/ddl_high_priority.result create mode 100644 storage/rocksdb/mysql-test/rocksdb/r/deadlock_tracking.result create mode 100644 storage/rocksdb/mysql-test/rocksdb/r/fast_prefix_index_fetch.result create mode 100644 storage/rocksdb/mysql-test/rocksdb/r/multi_varchar_sk_lookup.result create mode 100644 storage/rocksdb/mysql-test/rocksdb/r/skip_validate_tmp_table.result create mode 100644 storage/rocksdb/mysql-test/rocksdb/r/ttl_secondary.result create mode 100644 storage/rocksdb/mysql-test/rocksdb/r/ttl_secondary_read_filtering.result create mode 100644 storage/rocksdb/mysql-test/rocksdb/r/ttl_secondary_read_filtering_multiple_index.result create mode 100644 storage/rocksdb/mysql-test/rocksdb/r/ttl_secondary_with_partitions.result create mode 100644 storage/rocksdb/mysql-test/rocksdb/t/bulk_load_unsorted.test create mode 100644 storage/rocksdb/mysql-test/rocksdb/t/bulk_load_unsorted_errors.test create mode 100644 storage/rocksdb/mysql-test/rocksdb/t/bytes_written.test create mode 100644 storage/rocksdb/mysql-test/rocksdb/t/covered_unpack_info_format.test create mode 100644 storage/rocksdb/mysql-test/rocksdb/t/ddl_high_priority.test create mode 100644 storage/rocksdb/mysql-test/rocksdb/t/deadlock_tracking.test create mode 100644 storage/rocksdb/mysql-test/rocksdb/t/fast_prefix_index_fetch.test create mode 100644 storage/rocksdb/mysql-test/rocksdb/t/multi_varchar_sk_lookup.test create mode 100644 storage/rocksdb/mysql-test/rocksdb/t/skip_validate_tmp_table.test create mode 100644 storage/rocksdb/mysql-test/rocksdb/t/ttl_secondary-master.opt create mode 100644 storage/rocksdb/mysql-test/rocksdb/t/ttl_secondary.test create mode 100644 storage/rocksdb/mysql-test/rocksdb/t/ttl_secondary_read_filtering-master.opt create mode 100644 storage/rocksdb/mysql-test/rocksdb/t/ttl_secondary_read_filtering.test create mode 100644 storage/rocksdb/mysql-test/rocksdb/t/ttl_secondary_read_filtering_multiple_index.test create mode 100644 storage/rocksdb/mysql-test/rocksdb/t/ttl_secondary_with_partitions-master.opt create mode 100644 storage/rocksdb/mysql-test/rocksdb/t/ttl_secondary_with_partitions.test create mode 100644 storage/rocksdb/mysql-test/rocksdb_rpl/r/rpl_ddl_high_priority.result create mode 100644 storage/rocksdb/mysql-test/rocksdb_rpl/t/rpl_ddl_high_priority.test create mode 100644 storage/rocksdb/mysql-test/rocksdb_sys_vars/my.cnf create mode 100644 storage/rocksdb/mysql-test/rocksdb_sys_vars/r/rocksdb_bulk_load_allow_unsorted_basic.result create mode 100644 storage/rocksdb/mysql-test/rocksdb_sys_vars/r/rocksdb_concurrent_prepare_basic.result create mode 100644 storage/rocksdb/mysql-test/rocksdb_sys_vars/r/rocksdb_deadlock_detect_depth_basic.result create mode 100644 storage/rocksdb/mysql-test/rocksdb_sys_vars/r/rocksdb_debug_ttl_ignore_pk_basic.result create mode 100644 storage/rocksdb/mysql-test/rocksdb_sys_vars/r/rocksdb_force_compute_memtable_stats_cachetime_basic.result create mode 100644 storage/rocksdb/mysql-test/rocksdb_sys_vars/r/rocksdb_large_prefix_basic.result create mode 100644 storage/rocksdb/mysql-test/rocksdb_sys_vars/r/rocksdb_manual_wal_flush_basic.result create mode 100644 storage/rocksdb/mysql-test/rocksdb_sys_vars/r/rocksdb_max_latest_deadlocks_basic.result create mode 100644 storage/rocksdb/mysql-test/rocksdb_sys_vars/r/rocksdb_merge_tmp_file_removal_delay_ms_basic.result create mode 100644 storage/rocksdb/mysql-test/rocksdb_sys_vars/r/rocksdb_sim_cache_size_basic.result create mode 100644 storage/rocksdb/mysql-test/rocksdb_sys_vars/r/rocksdb_use_clock_cache_basic.result create mode 100644 storage/rocksdb/mysql-test/rocksdb_sys_vars/t/rocksdb_bulk_load_allow_unsorted_basic.test create mode 100644 storage/rocksdb/mysql-test/rocksdb_sys_vars/t/rocksdb_concurrent_prepare_basic.test create mode 100644 storage/rocksdb/mysql-test/rocksdb_sys_vars/t/rocksdb_deadlock_detect_depth_basic.test create mode 100644 storage/rocksdb/mysql-test/rocksdb_sys_vars/t/rocksdb_debug_ttl_ignore_pk_basic.test create mode 100644 storage/rocksdb/mysql-test/rocksdb_sys_vars/t/rocksdb_force_compute_memtable_stats_cachetime_basic.test create mode 100644 storage/rocksdb/mysql-test/rocksdb_sys_vars/t/rocksdb_large_prefix_basic.test create mode 100644 storage/rocksdb/mysql-test/rocksdb_sys_vars/t/rocksdb_manual_wal_flush_basic.test create mode 100644 storage/rocksdb/mysql-test/rocksdb_sys_vars/t/rocksdb_max_latest_deadlocks_basic.test create mode 100644 storage/rocksdb/mysql-test/rocksdb_sys_vars/t/rocksdb_merge_tmp_file_removal_delay_ms_basic.test create mode 100644 storage/rocksdb/mysql-test/rocksdb_sys_vars/t/rocksdb_sim_cache_size_basic.test create mode 100644 storage/rocksdb/mysql-test/rocksdb_sys_vars/t/rocksdb_use_clock_cache_basic.test diff --git a/storage/rocksdb/CMakeLists.txt b/storage/rocksdb/CMakeLists.txt index ec9705891e0..9e6c8160e32 100644 --- a/storage/rocksdb/CMakeLists.txt +++ b/storage/rocksdb/CMakeLists.txt @@ -106,6 +106,12 @@ IF (NOT "$ENV{WITH_ZSTD}" STREQUAL "") ADD_DEFINITIONS(-DZSTD) ENDIF() +IF (NOT "$ENV{WITH_TBB}" STREQUAL "") + SET(rocksdb_static_libs ${rocksdb_static_libs} + $ENV{WITH_TBB}/libtbb${PIC_EXT}.a) + ADD_DEFINITIONS(-DTBB) +ENDIF() + CHECK_INCLUDE_FILES(zlib.h HAVE_ZLIB_H) IF (HAVE_ZLIB_H) diff --git a/storage/rocksdb/ha_rocksdb.cc b/storage/rocksdb/ha_rocksdb.cc index 0df153b4853..cc52b343dd1 100644 --- a/storage/rocksdb/ha_rocksdb.cc +++ b/storage/rocksdb/ha_rocksdb.cc @@ -36,6 +36,7 @@ #include "./debug_sync.h" #include "./my_bit.h" #include "./my_stacktrace.h" +#include "./my_sys.h" #include "./sql_audit.h" #include "./sql_table.h" #include @@ -57,6 +58,7 @@ #include "rocksdb/utilities/checkpoint.h" #include "rocksdb/utilities/convenience.h" #include "rocksdb/utilities/memory_util.h" +#include "rocksdb/utilities/sim_cache.h" #include "util/stop_watch.h" /* MyRocks includes */ @@ -101,6 +103,7 @@ namespace myrocks { static st_global_stats global_stats; static st_export_stats export_stats; static st_memory_stats memory_stats; +static st_io_stall_stats io_stall_stats; const std::string DEFAULT_CF_NAME("default"); const std::string DEFAULT_SYSTEM_CF_NAME("__system__"); @@ -127,6 +130,14 @@ bool can_use_bloom_filter(THD *thd, const Rdb_key_def &kd, const rocksdb::Slice &eq_cond, const bool use_all_keys); +static rocksdb::CompactRangeOptions getCompactRangeOptions() { + rocksdb::CompactRangeOptions compact_range_options; + compact_range_options.bottommost_level_compaction = + rocksdb::BottommostLevelCompaction::kForce; + compact_range_options.exclusive_manual_compaction = false; + return compact_range_options; +} + /////////////////////////////////////////////////////////// // Parameters and settings /////////////////////////////////////////////////////////// @@ -149,7 +160,6 @@ static std::shared_ptr properties_collector_factory; Rdb_dict_manager dict_manager; Rdb_cf_manager cf_manager; Rdb_ddl_manager ddl_manager; -const char *m_mysql_gtid; Rdb_binlog_manager binlog_manager; Rdb_io_watchdog *io_watchdog = nullptr; @@ -165,7 +175,7 @@ static Rdb_background_thread rdb_bg_thread; // collation check requirement. Regex_list_handler *rdb_collation_exceptions; -static const char **rdb_get_error_messages(); +static const char *rdb_get_error_message(int nr); static void rocksdb_flush_all_memtables() { const Rdb_cf_manager &cf_manager = rdb_get_cf_manager(); @@ -192,7 +202,7 @@ static int rocksdb_compact_column_family(THD *const thd, if (cfh != nullptr && rdb != nullptr) { sql_print_information("RocksDB: Manual compaction of column family: %s\n", cf); - rdb->CompactRange(rocksdb::CompactRangeOptions(), cfh, nullptr, nullptr); + rdb->CompactRange(getCompactRangeOptions(), cfh, nullptr, nullptr); } } return HA_EXIT_SUCCESS; @@ -388,6 +398,10 @@ static void rocksdb_set_delayed_write_rate(THD *thd, struct st_mysql_sys_var *var, void *var_ptr, const void *save); +static void rocksdb_set_max_latest_deadlocks(THD *thd, + struct st_mysql_sys_var *var, + void *var_ptr, const void *save); + static void rdb_set_collation_exception_list(const char *exception_list); static void rocksdb_set_collation_exception_list(THD *thd, struct st_mysql_sys_var *var, @@ -404,6 +418,10 @@ rocksdb_set_bulk_load(THD *thd, struct st_mysql_sys_var *var MY_ATTRIBUTE((__unused__)), void *var_ptr, const void *save); +static void rocksdb_set_bulk_load_allow_unsorted( + THD *thd, struct st_mysql_sys_var *var MY_ATTRIBUTE((__unused__)), + void *var_ptr, const void *save); + static void rocksdb_set_max_background_jobs(THD *thd, struct st_mysql_sys_var *const var, void *const var_ptr, @@ -412,12 +430,15 @@ static void rocksdb_set_max_background_jobs(THD *thd, // Options definitions ////////////////////////////////////////////////////////////////////////////// static long long rocksdb_block_cache_size; +static long long rocksdb_sim_cache_size; +static my_bool rocksdb_use_clock_cache; /* Use unsigned long long instead of uint64_t because of MySQL compatibility */ static unsigned long long // NOLINT(runtime/int) rocksdb_rate_limiter_bytes_per_sec; static unsigned long long // NOLINT(runtime/int) rocksdb_sst_mgr_rate_bytes_per_sec; static unsigned long long rocksdb_delayed_write_rate; +static uint32_t rocksdb_max_latest_deadlocks; static unsigned long // NOLINT(runtime/int) rocksdb_persistent_cache_size_mb; static uint64_t rocksdb_info_log_level; @@ -427,6 +448,7 @@ static uint64_t rocksdb_index_type; static uint32_t rocksdb_flush_log_at_trx_commit; static uint32_t rocksdb_debug_optimizer_n_rows; static my_bool rocksdb_force_compute_memtable_stats; +static uint32_t rocksdb_force_compute_memtable_stats_cachetime; static my_bool rocksdb_debug_optimizer_no_zero_cardinality; static uint32_t rocksdb_wal_recovery_mode; static uint32_t rocksdb_access_hint_on_compaction_start; @@ -444,6 +466,7 @@ static my_bool rocksdb_enable_ttl_read_filtering = 1; static int rocksdb_debug_ttl_rec_ts = 0; static int rocksdb_debug_ttl_snapshot_ts = 0; static int rocksdb_debug_ttl_read_filter_ts = 0; +static my_bool rocksdb_debug_ttl_ignore_pk = 0; static my_bool rocksdb_reset_stats = 0; static uint32_t rocksdb_io_write_timeout_secs = 0; static uint64_t rocksdb_number_stat_computes = 0; @@ -456,6 +479,7 @@ static char *rocksdb_datadir; static uint32_t rocksdb_table_stats_sampling_pct; static my_bool rocksdb_enable_bulk_load_api = 1; static my_bool rocksdb_print_snapshot_conflict_queries = 0; +static my_bool rocksdb_large_prefix = 0; std::atomic rocksdb_snapshot_conflict_errors(0); std::atomic rocksdb_wal_group_syncs(0); @@ -468,6 +492,8 @@ static std::unique_ptr rdb_init_rocksdb_db_options(void) { o->info_log_level = rocksdb::InfoLogLevel::INFO_LEVEL; o->max_subcompactions = DEFAULT_SUBCOMPACTIONS; + o->concurrent_prepare = true; + o->manual_wal_flush = true; return o; } @@ -559,9 +585,12 @@ const size_t RDB_DEFAULT_MERGE_BUF_SIZE = 64 * 1024 * 1024; const size_t RDB_MIN_MERGE_BUF_SIZE = 100; const size_t RDB_DEFAULT_MERGE_COMBINE_READ_SIZE = 1024 * 1024 * 1024; const size_t RDB_MIN_MERGE_COMBINE_READ_SIZE = 100; +const size_t RDB_DEFAULT_MERGE_TMP_FILE_REMOVAL_DELAY = 0; +const size_t RDB_MIN_MERGE_TMP_FILE_REMOVAL_DELAY = 0; const int64 RDB_DEFAULT_BLOCK_CACHE_SIZE = 512 * 1024 * 1024; const int64 RDB_MIN_BLOCK_CACHE_SIZE = 1024; const int RDB_MAX_CHECKSUMS_PCT = 100; +const ulong RDB_DEADLOCK_DETECT_DEPTH = 50; // TODO: 0 means don't wait at all, and we don't support it yet? static MYSQL_THDVAR_ULONG(lock_wait_timeout, PLUGIN_VAR_RQCMDARG, @@ -572,6 +601,14 @@ static MYSQL_THDVAR_ULONG(lock_wait_timeout, PLUGIN_VAR_RQCMDARG, static MYSQL_THDVAR_BOOL(deadlock_detect, PLUGIN_VAR_RQCMDARG, "Enables deadlock detection", nullptr, nullptr, FALSE); +static MYSQL_THDVAR_ULONG(deadlock_detect_depth, PLUGIN_VAR_RQCMDARG, + "Number of transactions deadlock detection will " + "traverse through before assuming deadlock", + nullptr, nullptr, + /*default*/ RDB_DEADLOCK_DETECT_DEPTH, + /*min*/ 2, + /*max*/ ULONG_MAX, 0); + static MYSQL_THDVAR_BOOL( trace_sst_api, PLUGIN_VAR_RQCMDARG, "Generate trace output in the log for each call to the SstFileWriter", @@ -583,6 +620,11 @@ static MYSQL_THDVAR_BOOL( "unique_checks and enables rocksdb_commit_in_the_middle.", nullptr, rocksdb_set_bulk_load, FALSE); +static MYSQL_THDVAR_BOOL(bulk_load_allow_unsorted, PLUGIN_VAR_RQCMDARG, + "Allow unsorted input during bulk-load. " + "Can be changed only when bulk load is disabled.", + nullptr, rocksdb_set_bulk_load_allow_unsorted, FALSE); + static MYSQL_SYSVAR_BOOL(enable_bulk_load_api, rocksdb_enable_bulk_load_api, PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, "Enables using SstFileWriter for bulk loading", @@ -662,6 +704,18 @@ static MYSQL_THDVAR_ULONGLONG( /* min (100B) */ RDB_MIN_MERGE_COMBINE_READ_SIZE, /* max */ SIZE_T_MAX, 1); +static MYSQL_THDVAR_ULONGLONG( + merge_tmp_file_removal_delay_ms, PLUGIN_VAR_RQCMDARG, + "Fast index creation creates a large tmp file on disk during index " + "creation. Removing this large file all at once when index creation is " + "complete can cause trim stalls on Flash. This variable specifies a " + "duration to sleep (in milliseconds) between calling chsize() to truncate " + "the file in chunks. The chunk size is the same as merge_buf_size.", + nullptr, nullptr, + /* default (0ms) */ RDB_DEFAULT_MERGE_TMP_FILE_REMOVAL_DELAY, + /* min (0ms) */ RDB_MIN_MERGE_TMP_FILE_REMOVAL_DELAY, + /* max */ SIZE_T_MAX, 1); + static MYSQL_SYSVAR_BOOL( create_if_missing, *reinterpret_cast(&rocksdb_db_options->create_if_missing), @@ -669,6 +723,20 @@ static MYSQL_SYSVAR_BOOL( "DBOptions::create_if_missing for RocksDB", nullptr, nullptr, rocksdb_db_options->create_if_missing); +static MYSQL_SYSVAR_BOOL( + concurrent_prepare, + *reinterpret_cast(&rocksdb_db_options->concurrent_prepare), + PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, + "DBOptions::concurrent_prepare for RocksDB", nullptr, nullptr, + rocksdb_db_options->concurrent_prepare); + +static MYSQL_SYSVAR_BOOL( + manual_wal_flush, + *reinterpret_cast(&rocksdb_db_options->manual_wal_flush), + PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, + "DBOptions::manual_wal_flush for RocksDB", nullptr, nullptr, + rocksdb_db_options->manual_wal_flush); + static MYSQL_SYSVAR_BOOL( create_missing_column_families, *reinterpret_cast( @@ -712,6 +780,13 @@ static MYSQL_SYSVAR_ULONGLONG(delayed_write_rate, rocksdb_delayed_write_rate, rocksdb_db_options->delayed_write_rate, 0, UINT64_MAX, 0); +static MYSQL_SYSVAR_UINT(max_latest_deadlocks, rocksdb_max_latest_deadlocks, + PLUGIN_VAR_RQCMDARG, + "Maximum number of recent " + "deadlocks to store", + nullptr, rocksdb_set_max_latest_deadlocks, + rocksdb::kInitialMaxDeadlocks, 0, UINT32_MAX, 0); + static MYSQL_SYSVAR_ENUM( info_log_level, rocksdb_info_log_level, PLUGIN_VAR_RQCMDARG, "Filter level for info logs to be written mysqld error log. " @@ -988,6 +1063,22 @@ static MYSQL_SYSVAR_LONGLONG(block_cache_size, rocksdb_block_cache_size, /* max */ LONGLONG_MAX, /* Block size */ RDB_MIN_BLOCK_CACHE_SIZE); +static MYSQL_SYSVAR_LONGLONG(sim_cache_size, rocksdb_sim_cache_size, + PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, + "Simulated cache size for RocksDB", nullptr, + nullptr, + /* default */ 0, + /* min */ 0, + /* max */ LONGLONG_MAX, + /* Block size */ 0); + +static MYSQL_SYSVAR_BOOL( + use_clock_cache, + rocksdb_use_clock_cache, + PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, + "Use ClockCache instead of default LRUCache for RocksDB", + nullptr, nullptr, false); + static MYSQL_SYSVAR_BOOL( cache_index_and_filter_blocks, *reinterpret_cast( @@ -1076,12 +1167,21 @@ static MYSQL_SYSVAR_STR(update_cf_options, rocksdb_update_cf_options, "Option updates per column family for RocksDB", nullptr, rocksdb_set_update_cf_options, nullptr); +enum rocksdb_flush_log_at_trx_commit_type : unsigned int { + FLUSH_LOG_NEVER = 0, + FLUSH_LOG_SYNC, + FLUSH_LOG_BACKGROUND, + FLUSH_LOG_MAX /* must be last */ +}; + static MYSQL_SYSVAR_UINT(flush_log_at_trx_commit, rocksdb_flush_log_at_trx_commit, PLUGIN_VAR_RQCMDARG, "Sync on transaction commit. Similar to " "innodb_flush_log_at_trx_commit. 1: sync on commit, " "0,2: not sync on commit", - nullptr, nullptr, 1, 0, 2, 0); + nullptr, nullptr, /* default */ FLUSH_LOG_SYNC, + /* min */ FLUSH_LOG_NEVER, + /* max */ FLUSH_LOG_BACKGROUND, 0); static MYSQL_THDVAR_BOOL(write_disable_wal, PLUGIN_VAR_RQCMDARG, "WriteOptions::disableWAL for RocksDB", nullptr, @@ -1125,6 +1225,13 @@ static MYSQL_SYSVAR_BOOL(force_compute_memtable_stats, "Force to always compute memtable stats", nullptr, nullptr, TRUE); +static MYSQL_SYSVAR_UINT(force_compute_memtable_stats_cachetime, + rocksdb_force_compute_memtable_stats_cachetime, + PLUGIN_VAR_RQCMDARG, + "Time in usecs to cache memtable estimates", nullptr, + nullptr, /* default */ 60 * 1000 * 1000, + /* min */ 0, /* max */ INT_MAX, 0); + static MYSQL_SYSVAR_BOOL( debug_optimizer_no_zero_cardinality, rocksdb_debug_optimizer_no_zero_cardinality, PLUGIN_VAR_RQCMDARG, @@ -1191,6 +1298,12 @@ static MYSQL_SYSVAR_INT( "is not set. This variable is a no-op in non-debug builds.", nullptr, nullptr, 0, /* min */ -3600, /* max */ 3600, 0); +static MYSQL_SYSVAR_BOOL( + debug_ttl_ignore_pk, rocksdb_debug_ttl_ignore_pk, PLUGIN_VAR_RQCMDARG, + "For debugging purposes only. If true, compaction filtering will not occur " + "on PK TTL data. This variable is a no-op in non-debug builds.", + nullptr, nullptr, FALSE); + static MYSQL_SYSVAR_BOOL( reset_stats, rocksdb_reset_stats, PLUGIN_VAR_RQCMDARG, "Reset the RocksDB internal statistics without restarting the DB.", nullptr, @@ -1346,15 +1459,23 @@ static MYSQL_SYSVAR_UINT( RDB_DEFAULT_TBL_STATS_SAMPLE_PCT, /* everything */ 0, /* max */ RDB_TBL_STATS_SAMPLE_PCT_MAX, 0); +static MYSQL_SYSVAR_BOOL( + large_prefix, rocksdb_large_prefix, PLUGIN_VAR_RQCMDARG, + "Support large index prefix length of 3072 bytes. If off, the maximum " + "index prefix length is 767.", + nullptr, nullptr, FALSE); + static const int ROCKSDB_ASSUMED_KEY_VALUE_DISK_SIZE = 100; static struct st_mysql_sys_var *rocksdb_system_variables[] = { MYSQL_SYSVAR(lock_wait_timeout), MYSQL_SYSVAR(deadlock_detect), + MYSQL_SYSVAR(deadlock_detect_depth), MYSQL_SYSVAR(max_row_locks), MYSQL_SYSVAR(write_batch_max_bytes), MYSQL_SYSVAR(lock_scanned_rows), MYSQL_SYSVAR(bulk_load), + MYSQL_SYSVAR(bulk_load_allow_unsorted), MYSQL_SYSVAR(skip_unique_check_tables), MYSQL_SYSVAR(trace_sst_api), MYSQL_SYSVAR(commit_in_the_middle), @@ -1365,15 +1486,19 @@ static struct st_mysql_sys_var *rocksdb_system_variables[] = { MYSQL_SYSVAR(enable_bulk_load_api), MYSQL_SYSVAR(tmpdir), MYSQL_SYSVAR(merge_combine_read_size), + MYSQL_SYSVAR(merge_tmp_file_removal_delay_ms), MYSQL_SYSVAR(skip_bloom_filter_on_read), MYSQL_SYSVAR(create_if_missing), + MYSQL_SYSVAR(concurrent_prepare), + MYSQL_SYSVAR(manual_wal_flush), MYSQL_SYSVAR(create_missing_column_families), MYSQL_SYSVAR(error_if_exists), MYSQL_SYSVAR(paranoid_checks), MYSQL_SYSVAR(rate_limiter_bytes_per_sec), MYSQL_SYSVAR(sst_mgr_rate_bytes_per_sec), MYSQL_SYSVAR(delayed_write_rate), + MYSQL_SYSVAR(max_latest_deadlocks), MYSQL_SYSVAR(info_log_level), MYSQL_SYSVAR(max_open_files), MYSQL_SYSVAR(max_total_wal_size), @@ -1413,6 +1538,8 @@ static struct st_mysql_sys_var *rocksdb_system_variables[] = { MYSQL_SYSVAR(enable_write_thread_adaptive_yield), MYSQL_SYSVAR(block_cache_size), + MYSQL_SYSVAR(sim_cache_size), + MYSQL_SYSVAR(use_clock_cache), MYSQL_SYSVAR(cache_index_and_filter_blocks), MYSQL_SYSVAR(pin_l0_filter_and_index_blocks_in_cache), MYSQL_SYSVAR(index_type), @@ -1438,6 +1565,7 @@ static struct st_mysql_sys_var *rocksdb_system_variables[] = { MYSQL_SYSVAR(force_index_records_in_range), MYSQL_SYSVAR(debug_optimizer_n_rows), MYSQL_SYSVAR(force_compute_memtable_stats), + MYSQL_SYSVAR(force_compute_memtable_stats_cachetime), MYSQL_SYSVAR(debug_optimizer_no_zero_cardinality), MYSQL_SYSVAR(compact_cf), @@ -1454,6 +1582,7 @@ static struct st_mysql_sys_var *rocksdb_system_variables[] = { MYSQL_SYSVAR(debug_ttl_rec_ts), MYSQL_SYSVAR(debug_ttl_snapshot_ts), MYSQL_SYSVAR(debug_ttl_read_filter_ts), + MYSQL_SYSVAR(debug_ttl_ignore_pk), MYSQL_SYSVAR(reset_stats), MYSQL_SYSVAR(io_write_timeout), MYSQL_SYSVAR(flush_memtable_on_analyze), @@ -1475,13 +1604,15 @@ static struct st_mysql_sys_var *rocksdb_system_variables[] = { MYSQL_SYSVAR(validate_tables), MYSQL_SYSVAR(table_stats_sampling_pct), + + MYSQL_SYSVAR(large_prefix), nullptr}; static rocksdb::WriteOptions rdb_get_rocksdb_write_options(my_core::THD *const thd) { rocksdb::WriteOptions opt; - opt.sync = (rocksdb_flush_log_at_trx_commit == 1); + opt.sync = (rocksdb_flush_log_at_trx_commit == FLUSH_LOG_SYNC); opt.disableWAL = THDVAR(thd, write_disable_wal); opt.ignore_missing_column_families = THDVAR(thd, write_ignore_missing_column_families); @@ -1743,6 +1874,13 @@ public: } } + void update_bytes_written(ulonglong bytes_written) { + if (m_tbl_io_perf != nullptr) { + m_tbl_io_perf->update_bytes_written(rocksdb_perf_context_level(m_thd), + bytes_written); + } + } + void set_params(int timeout_sec_arg, int max_row_locks_arg) { m_timeout_sec = timeout_sec_arg; m_max_row_locks = max_row_locks_arg; @@ -2238,9 +2376,10 @@ public: tx_opts.set_snapshot = false; tx_opts.lock_timeout = rdb_convert_sec_to_ms(m_timeout_sec); tx_opts.deadlock_detect = THDVAR(m_thd, deadlock_detect); + tx_opts.deadlock_detect_depth = THDVAR(m_thd, deadlock_detect_depth); tx_opts.max_write_batch_size = THDVAR(m_thd, write_batch_max_bytes); - write_opts.sync = (rocksdb_flush_log_at_trx_commit == 1); + write_opts.sync = (rocksdb_flush_log_at_trx_commit == FLUSH_LOG_SYNC); write_opts.disableWAL = THDVAR(m_thd, write_disable_wal); write_opts.ignore_missing_column_families = THDVAR(m_thd, write_ignore_missing_column_families); @@ -2459,7 +2598,7 @@ public: void start_tx() override { reset(); - write_opts.sync = (rocksdb_flush_log_at_trx_commit == 1); + write_opts.sync = (rocksdb_flush_log_at_trx_commit == FLUSH_LOG_SYNC); write_opts.disableWAL = THDVAR(m_thd, write_disable_wal); write_opts.ignore_missing_column_families = THDVAR(m_thd, write_ignore_missing_column_families); @@ -2503,14 +2642,21 @@ namespace { class Rdb_perf_context_guard { Rdb_io_perf m_io_perf; - THD *m_thd; + Rdb_io_perf *m_io_perf_ptr; + Rdb_transaction *m_tx; + uint m_level; -public: + public: Rdb_perf_context_guard(const Rdb_perf_context_guard &) = delete; Rdb_perf_context_guard &operator=(const Rdb_perf_context_guard &) = delete; - explicit Rdb_perf_context_guard(THD *const thd) : m_thd(thd) { - Rdb_transaction *&tx = get_tx_from_thd(m_thd); + explicit Rdb_perf_context_guard(Rdb_io_perf *io_perf, uint level) + : m_io_perf_ptr(io_perf), m_tx(nullptr), m_level(level) { + m_io_perf_ptr->start(m_level); + } + + explicit Rdb_perf_context_guard(Rdb_transaction *tx, uint level) + : m_io_perf_ptr(nullptr), m_tx(tx), m_level(level) { /* if perf_context information is already being recorded, this becomes a no-op @@ -2521,9 +2667,10 @@ public: } ~Rdb_perf_context_guard() { - Rdb_transaction *&tx = get_tx_from_thd(m_thd); - if (tx != nullptr) { - tx->io_perf_end_and_record(); + if (m_tx != nullptr) { + m_tx->io_perf_end_and_record(); + } else if (m_io_perf_ptr != nullptr) { + m_io_perf_ptr->end_and_record(m_level); } } }; @@ -2611,8 +2758,17 @@ static std::string rdb_xid_to_string(const XID &src) { static bool rocksdb_flush_wal(handlerton *const hton MY_ATTRIBUTE((__unused__)), ulonglong target_lsn MY_ATTRIBUTE((__unused__))) { DBUG_ASSERT(rdb != nullptr); - rocksdb_wal_group_syncs++; - const rocksdb::Status s = rdb->SyncWAL(); + + rocksdb::Status s; + /* + target_lsn is set to 0 when MySQL wants to sync the wal files + */ + if (target_lsn == 0 || rocksdb_flush_log_at_trx_commit != FLUSH_LOG_NEVER) { + rocksdb_wal_group_syncs++; + s = rdb->FlushWAL(target_lsn == 0 || + rocksdb_flush_log_at_trx_commit == FLUSH_LOG_SYNC); + } + if (!s.ok()) { rdb_log_status_error(s); return HA_EXIT_FAILURE; @@ -2651,7 +2807,7 @@ static int rocksdb_prepare(handlerton *const hton, THD *const thd, return HA_EXIT_FAILURE; } if (thd->durability_property == HA_IGNORE_DURABILITY && - (rocksdb_flush_log_at_trx_commit == 1)) { + (rocksdb_flush_log_at_trx_commit != FLUSH_LOG_NEVER)) { /** we set the log sequence as '1' just to trigger hton->flush_logs */ @@ -2812,12 +2968,12 @@ static int rocksdb_commit(handlerton *const hton, THD *const thd, rocksdb::StopWatchNano timer(rocksdb::Env::Default(), true); - /* this will trigger saving of perf_context information */ - Rdb_perf_context_guard guard(thd); - /* note: h->external_lock(F_UNLCK) is called after this function is called) */ Rdb_transaction *&tx = get_tx_from_thd(thd); + /* this will trigger saving of perf_context information */ + Rdb_perf_context_guard guard(tx, rocksdb_perf_context_level(thd)); + if (tx != nullptr) { if (commit_tx || (!my_core::thd_test_options(thd, OPTION_NOT_AUTOCOMMIT | OPTION_BEGIN))) { @@ -2854,8 +3010,8 @@ static int rocksdb_commit(handlerton *const hton, THD *const thd, static int rocksdb_rollback(handlerton *const hton, THD *const thd, bool rollback_tx) { - Rdb_perf_context_guard guard(thd); Rdb_transaction *&tx = get_tx_from_thd(thd); + Rdb_perf_context_guard guard(tx, rocksdb_perf_context_level(thd)); if (tx != nullptr) { if (rollback_tx) { @@ -2978,7 +3134,82 @@ private: "=========================================\n"; } -public: + static std::string get_dlock_txn_info(const rocksdb::DeadlockInfo &txn, + const GL_INDEX_ID &gl_index_id, + bool is_last_path = false) { + std::string txn_data; + + /* extract table name and index names using the index id */ + std::string table_name = ddl_manager.safe_get_table_name(gl_index_id); + if (table_name.empty()) { + table_name = + "NOT FOUND; INDEX_ID: " + std::to_string(gl_index_id.index_id); + } + auto kd = ddl_manager.safe_find(gl_index_id); + std::string idx_name = + (kd) ? kd->get_name() + : "NOT FOUND; INDEX_ID: " + std::to_string(gl_index_id.index_id); + + /* get the name of the column family */ + rocksdb::ColumnFamilyHandle *cfh = cf_manager.get_cf(txn.m_cf_id); + std::string cf_name = cfh->GetName(); + + txn_data += format_string( + "TRANSACTIONID: %u\n" + "COLUMN FAMILY NAME: %s\n" + "WAITING KEY: %s\n" + "LOCK TYPE: %s\n" + "INDEX NAME: %s\n" + "TABLE NAME: %s\n", + txn.m_txn_id, cf_name.c_str(), + rdb_hexdump(txn.m_waiting_key.c_str(), txn.m_waiting_key.length()) + .c_str(), + txn.m_exclusive ? "EXCLUSIVE" : "SHARED", idx_name.c_str(), + table_name.c_str()); + if (!is_last_path) { + txn_data += "---------------WAITING FOR---------------\n"; + } + return txn_data; + } + + static std::string + get_dlock_path_info(const rocksdb::DeadlockPath &path_entry) { + std::string path_data; + if (path_entry.limit_exceeded) { + path_data += "\n-------DEADLOCK EXCEEDED MAX DEPTH-------\n"; + } else { + path_data += "\n*** DEADLOCK PATH\n" + "=========================================\n"; + for (auto it = path_entry.path.begin(); it != path_entry.path.end(); + it++) { + auto txn = *it; + const GL_INDEX_ID gl_index_id = { + txn.m_cf_id, rdb_netbuf_to_uint32(reinterpret_cast( + txn.m_waiting_key.c_str()))}; + path_data += get_dlock_txn_info(txn, gl_index_id); + } + + DBUG_ASSERT_IFF(path_entry.limit_exceeded, path_entry.path.empty()); + /* print the first txn in the path to display the full deadlock cycle */ + if (!path_entry.path.empty() && !path_entry.limit_exceeded) { + auto txn = path_entry.path[0]; + const GL_INDEX_ID gl_index_id = { + txn.m_cf_id, rdb_netbuf_to_uint32(reinterpret_cast( + txn.m_waiting_key.c_str()))}; + path_data += get_dlock_txn_info(txn, gl_index_id, true); + + /* prints the txn id of the transaction that caused the deadlock */ + auto deadlocking_txn = *(path_entry.path.end() - 1); + path_data += + format_string("\n--------TRANSACTIONID: %u GOT DEADLOCK---------\n", + deadlocking_txn.m_txn_id); + } + } + + return path_data; + } + + public: Rdb_snapshot_status() : m_data(get_header()) {} std::string getResult() { return m_data + get_footer(); } @@ -3007,6 +3238,15 @@ public: tx->get_delete_count()); } } + + void populate_deadlock_buffer() { + auto dlock_buffer = rdb->GetDeadlockInfoBuffer(); + m_data += "----------LATEST DETECTED DEADLOCKS----------\n"; + + for (auto path_entry : dlock_buffer) { + m_data += get_dlock_path_info(path_entry); + } + } }; /** @@ -3103,10 +3343,10 @@ static bool rocksdb_show_snapshot_status(handlerton *const hton, THD *const thd, Rdb_snapshot_status showStatus; Rdb_transaction::walk_tx_list(&showStatus); + showStatus.populate_deadlock_buffer(); /* Send the result data back to MySQL */ - return print_stats(thd, "SNAPSHOTS", "rocksdb", showStatus.getResult(), - stat_print); + return print_stats(thd, "rocksdb", "", showStatus.getResult(), stat_print); } /* @@ -3332,8 +3572,6 @@ static int rocksdb_start_tx_and_assign_read_view( char **gtid_executed, /* out: Gtids logged until last commit */ int *const gtid_executed_length) /*out: Length of gtid_executed string */ { - Rdb_perf_context_guard guard(thd); - ulong const tx_isolation = my_core::thd_tx_isolation(thd); if (tx_isolation != ISO_REPEATABLE_READ) { @@ -3349,6 +3587,8 @@ static int rocksdb_start_tx_and_assign_read_view( } Rdb_transaction *const tx = get_or_create_tx(thd); + Rdb_perf_context_guard guard(tx, rocksdb_perf_context_level(thd)); + DBUG_ASSERT(!tx->has_snapshot()); tx->set_tx_read_only(true); rocksdb_register_tx(hton, thd, tx); @@ -3395,6 +3635,7 @@ static void rocksdb_update_table_stats( int n_lock_wait, int n_lock_wait_timeout, int n_lock_deadlock, const char *engine)) { my_io_perf_t io_perf_read; + my_io_perf_t io_perf_write; my_io_perf_t io_perf; page_stats_t page_stats; comp_stats_t comp_stats; @@ -3409,6 +3650,7 @@ static void rocksdb_update_table_stats( memset(&io_perf, 0, sizeof(io_perf)); memset(&page_stats, 0, sizeof(page_stats)); memset(&comp_stats, 0, sizeof(comp_stats)); + memset(&io_perf_write, 0, sizeof(io_perf_write)); tablenames = rdb_open_tables.get_table_names(); @@ -3441,6 +3683,8 @@ static void rocksdb_update_table_stats( io_perf_read.bytes = table_handler->m_io_perf_read.bytes.load(); io_perf_read.requests = table_handler->m_io_perf_read.requests.load(); + io_perf_write.bytes = table_handler->m_io_perf_write.bytes.load(); + io_perf_write.requests = table_handler->m_io_perf_write.requests.load(); lock_wait_timeout_stats = table_handler->m_lock_wait_timeout_counter.load(); deadlock_stats = table_handler->m_deadlock_counter.load(); @@ -3468,9 +3712,10 @@ static void rocksdb_update_table_stats( sizeof(dbname_sys)); my_core::filename_to_tablename(tablename.c_str(), tablename_sys, sizeof(tablename_sys)); - (*cb)(dbname_sys, tablename_sys, is_partition, &io_perf_read, &io_perf, - &io_perf, &io_perf, &io_perf, &page_stats, &comp_stats, 0, - lock_wait_timeout_stats, deadlock_stats, rocksdb_hton_name); + (*cb)(dbname_sys, tablename_sys, is_partition, &io_perf_read, + &io_perf_write, &io_perf, &io_perf, &io_perf, &page_stats, + &comp_stats, 0, lock_wait_timeout_stats, deadlock_stats, + rocksdb_hton_name); } } @@ -3680,8 +3925,18 @@ static int rocksdb_init_func(void *const p) { (rocksdb::BlockBasedTableOptions::IndexType)rocksdb_index_type; if (!rocksdb_tbl_options->no_block_cache) { - rocksdb_tbl_options->block_cache = - rocksdb::NewLRUCache(rocksdb_block_cache_size); + std::shared_ptr block_cache = rocksdb_use_clock_cache + ? rocksdb::NewClockCache(rocksdb_block_cache_size) + : rocksdb::NewLRUCache(rocksdb_block_cache_size); + if (rocksdb_sim_cache_size > 0) { + // Simulated cache enabled + // Wrap block cache inside a simulated cache and pass it to RocksDB + rocksdb_tbl_options->block_cache = + rocksdb::NewSimCache(block_cache, rocksdb_sim_cache_size, 6); + } else { + // Pass block cache to RocksDB + rocksdb_tbl_options->block_cache = block_cache; + } } // Using newer BlockBasedTable format version for better compression // and better memory allocation. @@ -3870,7 +4125,7 @@ static int rocksdb_init_func(void *const p) { } #endif - err = my_error_register(rdb_get_error_messages, HA_ERR_ROCKSDB_FIRST, + err = my_error_register(rdb_get_error_message, HA_ERR_ROCKSDB_FIRST, HA_ERR_ROCKSDB_LAST); if (err != 0) { // NO_LINT_DEBUG @@ -4064,6 +4319,7 @@ Rdb_open_tables_map::get_table_handler(const char *const table_name) { thr_lock_init(&table_handler->m_thr_lock); table_handler->m_io_perf_read.init(); + table_handler->m_io_perf_write.init(); } DBUG_ASSERT(table_handler->m_ref_count >= 0); table_handler->m_ref_count++; @@ -4330,11 +4586,11 @@ bool ha_rocksdb::init_with_fields() { rows within a transaction, etc, because the compaction filter ignores snapshots when filtering keys. */ -bool ha_rocksdb::should_hide_ttl_rec(const rocksdb::Slice &ttl_rec_val, +bool ha_rocksdb::should_hide_ttl_rec(const Rdb_key_def &kd, + const rocksdb::Slice &ttl_rec_val, const int64_t curr_ts) { - DBUG_ASSERT(m_pk_descr != nullptr); - DBUG_ASSERT(m_pk_descr->has_ttl()); - DBUG_ASSERT(m_pk_descr->m_ttl_rec_offset != UINT_MAX); + DBUG_ASSERT(kd.has_ttl()); + DBUG_ASSERT(kd.m_ttl_rec_offset != UINT_MAX); /* Curr_ts can only be 0 if there are no snapshots open. @@ -4360,7 +4616,7 @@ bool ha_rocksdb::should_hide_ttl_rec(const rocksdb::Slice &ttl_rec_val, Find where the 8-byte ttl is for each record in this index. */ uint64 ts; - if (!reader.read(m_pk_descr->m_ttl_rec_offset) || reader.read_uint64(&ts)) { + if (!reader.read(kd.m_ttl_rec_offset) || reader.read_uint64(&ts)) { /* This condition should never be reached since all TTL records have an 8 byte ttl field in front. Don't filter the record out, and log an error. @@ -4368,7 +4624,7 @@ bool ha_rocksdb::should_hide_ttl_rec(const rocksdb::Slice &ttl_rec_val, std::string buf; buf = rdb_hexdump(ttl_rec_val.data(), ttl_rec_val.size(), RDB_MAX_HEXDUMP_LEN); - const GL_INDEX_ID gl_index_id = m_pk_descr->get_gl_index_id(); + const GL_INDEX_ID gl_index_id = kd.get_gl_index_id(); // NO_LINT_DEBUG sql_print_error("Decoding ttl from PK value failed, " "for index (%u,%u), val: %s", @@ -4382,10 +4638,23 @@ bool ha_rocksdb::should_hide_ttl_rec(const rocksdb::Slice &ttl_rec_val, #ifndef NDEBUG read_filter_ts += rdb_dbug_set_ttl_read_filter_ts(); #endif - return ts + m_pk_descr->m_ttl_duration + read_filter_ts <= + return ts + kd.m_ttl_duration + read_filter_ts <= static_cast(curr_ts); } +void ha_rocksdb::rocksdb_skip_expired_records(const Rdb_key_def &kd, + rocksdb::Iterator *const iter, + bool seek_backward) { + if (kd.has_ttl()) { + while (iter->Valid() && + should_hide_ttl_rec( + kd, iter->value(), + get_or_create_tx(table->in_use)->m_snapshot_timestamp)) { + rocksdb_smart_next(seek_backward, iter); + } + } +} + /** Convert record from table->record[0] form into a form that can be written into rocksdb. @@ -4411,6 +4680,7 @@ int ha_rocksdb::convert_record_to_storage_format( if (has_ttl) { /* If it's a TTL record, reserve space for 8 byte TTL value in front. */ m_storage_record.fill(ROCKSDB_SIZEOF_TTL_RECORD + m_null_bytes_in_rec, 0); + m_ttl_bytes_updated = false; /* If the TTL is contained within the key, we use the offset to find the @@ -4441,6 +4711,8 @@ int ha_rocksdb::convert_record_to_storage_format( rdb_netbuf_to_uint64(reinterpret_cast(data)) + rdb_dbug_set_ttl_rec_ts()); #endif + // Also store in m_ttl_bytes to propagate to update_sk + memcpy(m_ttl_bytes, data, ROCKSDB_SIZEOF_TTL_RECORD); } else if (!has_ttl_column) { /* For implicitly generated TTL records we need to copy over the old @@ -4459,6 +4731,8 @@ int ha_rocksdb::convert_record_to_storage_format( #endif char *const data = const_cast(m_storage_record.ptr()); rdb_netbuf_store_uint64(reinterpret_cast(data), ts); + // Also store in m_ttl_bytes to propagate to update_sk + memcpy(m_ttl_bytes, data, ROCKSDB_SIZEOF_TTL_RECORD); } } } else { @@ -4540,6 +4814,15 @@ int ha_rocksdb::convert_record_to_storage_format( ts += rdb_dbug_set_ttl_rec_ts(); #endif rdb_netbuf_store_uint64(reinterpret_cast(data), ts); + + // If this is an update and the timestamp has been updated, take note + // so we can avoid updating SKs unnecessarily. + if (!row_info.old_pk_slice.empty()) { + m_ttl_bytes_updated = + memcmp(m_ttl_bytes, data, ROCKSDB_SIZEOF_TTL_RECORD); + } + // Store timestamp in m_ttl_bytes to propagate to update_sk + memcpy(m_ttl_bytes, data, ROCKSDB_SIZEOF_TTL_RECORD); } } } @@ -4818,8 +5101,9 @@ int ha_rocksdb::convert_record_from_storage_format( } if (m_maybe_unpack_info) { - unpack_info = reader.read(RDB_UNPACK_HEADER_SIZE); - if (!unpack_info || unpack_info[0] != RDB_UNPACK_DATA_TAG) { + unpack_info = reader.get_current_ptr(); + if (!unpack_info || !Rdb_key_def::is_unpack_data_tag(unpack_info[0]) || + !reader.read(Rdb_key_def::get_unpack_header_size(unpack_info[0]))) { return HA_ERR_ROCKSDB_CORRUPT_DATA; } @@ -4827,7 +5111,8 @@ int ha_rocksdb::convert_record_from_storage_format( rdb_netbuf_to_uint16(reinterpret_cast(unpack_info + 1)); unpack_slice = rocksdb::Slice(unpack_info, unpack_info_len); - reader.read(unpack_info_len - RDB_UNPACK_HEADER_SIZE); + reader.read(unpack_info_len - + Rdb_key_def::get_unpack_header_size(unpack_info[0])); } int err = m_pk_descr->unpack_record(table, buf, &rowkey_slice, @@ -4983,8 +5268,7 @@ void ha_rocksdb::setup_field_converters() { If hidden pk exists, we skip this check since the field will never be part of the hidden pk. */ - if (!has_hidden_pk(table) && - field->part_of_key.is_set(table->s->primary_key)) { + if (!has_hidden_pk(table)) { KEY *const pk_info = &table->key_info[table->s->primary_key]; for (uint kp = 0; kp < pk_info->user_defined_key_parts; kp++) { /* key_part->fieldnr is counted from 1 */ @@ -5185,7 +5469,10 @@ int ha_rocksdb::open(const char *const name, int mode, uint test_if_locked) { my_core::thr_lock_data_init(&m_table_handler->m_thr_lock, &m_db_lock, nullptr); m_io_perf.init(&m_table_handler->m_table_perf_context, - &m_table_handler->m_io_perf_read, &stats); + &m_table_handler->m_io_perf_read, + &m_table_handler->m_io_perf_write, &stats); + Rdb_perf_context_guard guard(&m_io_perf, + rocksdb_perf_context_level(ha_thd())); std::string fullname; err = rdb_normalize_tablename(name, &fullname); @@ -5282,6 +5569,13 @@ int ha_rocksdb::close(void) { m_table_handler = nullptr; } + // These are needed to suppress valgrind errors in rocksdb.partition + m_storage_record.free(); + m_last_rowkey.free(); + m_sk_tails.free(); + m_sk_tails_old.free(); + m_pk_unpack_info.free(); + DBUG_RETURN(HA_EXIT_SUCCESS); } @@ -5317,7 +5611,9 @@ static_assert((sizeof(rdb_error_messages) / sizeof(rdb_error_messages[0])) == ((HA_ERR_ROCKSDB_LAST - HA_ERR_ROCKSDB_FIRST) + 1), "Number of error messages doesn't match number of error codes"); -static const char **rdb_get_error_messages() { return rdb_error_messages; } +static const char *rdb_get_error_message(int nr) { + return rdb_error_messages[nr - HA_ERR_ROCKSDB_FIRST]; +} bool ha_rocksdb::get_error_message(const int error, String *const buf) { DBUG_ENTER_FUNC(); @@ -5865,9 +6161,8 @@ int ha_rocksdb::create_key_def(const TABLE *const table_arg, const uint &i, DBUG_RETURN(err); } - /* We don't currently support TTL on tables with secondary keys. */ - if (ttl_duration > 0 && - (table_arg->s->keys > 1 || is_hidden_pk(i, table_arg, tbl_def_arg))) { + /* We don't currently support TTL on tables with hidden primary keys. */ + if (ttl_duration > 0 && is_hidden_pk(i, table_arg, tbl_def_arg)) { my_error(ER_RDB_TTL_UNSUPPORTED, MYF(0)); DBUG_RETURN(HA_EXIT_FAILURE); } @@ -5906,6 +6201,12 @@ int ha_rocksdb::create_key_def(const TABLE *const table_arg, const uint &i, kv_version = Rdb_key_def::PRIMARY_FORMAT_VERSION_UPDATE1; }); + DBUG_EXECUTE_IF("MYROCKS_NO_COVERED_BITMAP_FORMAT", { + if (index_type == Rdb_key_def::INDEX_TYPE_SECONDARY) { + kv_version = Rdb_key_def::SECONDARY_FORMAT_VERSION_UPDATE2; + } + }); + uint32 index_flags = (ttl_duration > 0 ? Rdb_key_def::TTL_FLAG : 0); uint32 ttl_rec_offset = @@ -6263,7 +6564,7 @@ int ha_rocksdb::read_key_exact(const Rdb_key_def &kd, from the POV of the current transaction. If it has, try going to the next key. */ - if (kd.has_ttl() && should_hide_ttl_rec(iter->value(), ttl_filter_ts)) { + if (kd.has_ttl() && should_hide_ttl_rec(kd, iter->value(), ttl_filter_ts)) { rocksdb_smart_next(kd.m_is_reverse_cf, iter); continue; } @@ -6299,7 +6600,7 @@ int ha_rocksdb::read_before_key(const Rdb_key_def &kd, if ((full_key_match && kd.value_matches_prefix(m_scan_it->key(), key_slice)) || (kd.has_ttl() && - should_hide_ttl_rec(m_scan_it->value(), ttl_filter_ts))) { + should_hide_ttl_rec(kd, m_scan_it->value(), ttl_filter_ts))) { rocksdb_smart_next(!kd.m_is_reverse_cf, m_scan_it); continue; } @@ -6329,7 +6630,7 @@ int ha_rocksdb::read_after_key(const Rdb_key_def &kd, key. */ while (m_scan_it->Valid() && kd.has_ttl() && - should_hide_ttl_rec(m_scan_it->value(), ttl_filter_ts)) { + should_hide_ttl_rec(kd, m_scan_it->value(), ttl_filter_ts)) { rocksdb_smart_next(kd.m_is_reverse_cf, m_scan_it); } @@ -6487,17 +6788,21 @@ int ha_rocksdb::read_row_from_secondary_key(uchar *const buf, int rc = 0; uint pk_size; - if (m_keyread_only && m_lock_rows == RDB_LOCK_NONE && !has_hidden_pk(table)) { - /* Get the key columns and primary key value */ - const rocksdb::Slice &rkey = m_scan_it->key(); + /* Get the key columns and primary key value */ + const rocksdb::Slice &rkey = m_scan_it->key(); + const rocksdb::Slice &value = m_scan_it->value(); + + bool covered_lookup = + m_keyread_only || kd.covers_lookup(table, &value, &m_lookup_bitmap); + if (covered_lookup && m_lock_rows == RDB_LOCK_NONE && !has_hidden_pk(table)) { pk_size = kd.get_primary_key_tuple(table, *m_pk_descr, &rkey, m_pk_packed_tuple); - const rocksdb::Slice &value = m_scan_it->value(); if (pk_size == RDB_INVALID_KEY_LEN) { rc = HA_ERR_ROCKSDB_CORRUPT_DATA; } else { rc = kd.unpack_record(table, buf, &rkey, &value, m_verify_row_debug_checksums); + global_stats.covered_secondary_key_lookups.inc(); } } else { if (kd.m_is_reverse_cf) @@ -6607,11 +6912,15 @@ int ha_rocksdb::secondary_index_read(const int keyno, uchar *const buf) { m_last_rowkey.copy((const char *)m_pk_packed_tuple, size, &my_charset_bin); - if (m_keyread_only && m_lock_rows == RDB_LOCK_NONE && + rocksdb::Slice value = m_scan_it->value(); + bool covered_lookup = + m_keyread_only || m_key_descr_arr[keyno]->covers_lookup( + table, &value, &m_lookup_bitmap); + if (covered_lookup && m_lock_rows == RDB_LOCK_NONE && !has_hidden_pk(table)) { - rocksdb::Slice value = m_scan_it->value(); rc = m_key_descr_arr[keyno]->unpack_record( table, buf, &key, &value, m_verify_row_debug_checksums); + global_stats.covered_secondary_key_lookups.inc(); } else { rc = get_row_by_rowid(buf, m_pk_packed_tuple, size); } @@ -6898,6 +7207,8 @@ int ha_rocksdb::find_icp_matching_index_rec(const bool &move_forward, const Rdb_key_def &kd = *m_key_descr_arr[active_index]; while (1) { + rocksdb_skip_expired_records(kd, m_scan_it, !move_forward); + if (!m_scan_it->Valid()) { table->status = STATUS_NOT_FOUND; return HA_ERR_END_OF_FILE; @@ -6927,10 +7238,7 @@ int ha_rocksdb::find_icp_matching_index_rec(const bool &move_forward, const enum icp_result icp_status = check_index_cond(); if (icp_status == ICP_NO_MATCH) { - if (move_forward) - m_scan_it->Next(); - else - m_scan_it->Prev(); + rocksdb_smart_next(!move_forward, m_scan_it); continue; /* Get the next (or prev) index tuple */ } else if (icp_status == ICP_OUT_OF_RANGE) { /* We have walked out of range we are scanning */ @@ -7263,7 +7571,8 @@ int ha_rocksdb::get_row_by_rowid(uchar *const buf, const char *const rowid, if (found) { /* If we found the record, but it's expired, pretend we didn't find it. */ if (!skip_ttl_check && m_pk_descr->has_ttl() && - should_hide_ttl_rec(rocksdb::Slice(&m_retrieved_record.front(), + should_hide_ttl_rec(*m_pk_descr, + rocksdb::Slice(&m_retrieved_record.front(), m_retrieved_record.size()), tx->m_snapshot_timestamp)) { DBUG_RETURN(HA_ERR_KEY_NOT_FOUND); @@ -7344,6 +7653,8 @@ int ha_rocksdb::index_next_with_direction(uchar *const buf, bool move_forward) { else m_scan_it->Prev(); } + rocksdb_skip_expired_records(*m_key_descr_arr[active_index], m_scan_it, + !move_forward); rc = find_icp_matching_index_rec(move_forward, buf); if (!rc) rc = secondary_index_read(active_index, buf); @@ -7696,6 +8007,12 @@ bool ha_rocksdb::is_pk(const uint index, const TABLE *const table_arg, is_hidden_pk(index, table_arg, tbl_def_arg); } +uint ha_rocksdb::max_supported_key_part_length() const { + DBUG_ENTER_FUNC(); + DBUG_RETURN(rocksdb_large_prefix ? MAX_INDEX_COL_LEN_LARGE + : MAX_INDEX_COL_LEN_SMALL); +} + const char *ha_rocksdb::get_key_name(const uint index, const TABLE *const table_arg, const Rdb_tbl_def *const tbl_def_arg) { @@ -8075,7 +8392,8 @@ int ha_rocksdb::check_uniqueness_and_lock( m_retrieved_record by check_and_lock_unique_pk(). */ if (is_pk(key_id, table, m_tbl_def) && found && m_pk_descr->has_ttl() && - should_hide_ttl_rec(rocksdb::Slice(&m_retrieved_record.front(), + should_hide_ttl_rec(*m_pk_descr, + rocksdb::Slice(&m_retrieved_record.front(), m_retrieved_record.size()), (row_info.tx->m_snapshot_timestamp ? row_info.tx->m_snapshot_timestamp @@ -8123,48 +8441,112 @@ int ha_rocksdb::check_duplicate_sk(const TABLE *table_arg, int ha_rocksdb::bulk_load_key(Rdb_transaction *const tx, const Rdb_key_def &kd, const rocksdb::Slice &key, - const rocksdb::Slice &value) { - rocksdb::ColumnFamilyHandle *const cf = kd.get_cf(); + const rocksdb::Slice &value, bool sort) { + DBUG_ENTER_FUNC(); + + rocksdb::ColumnFamilyHandle *cf = kd.get_cf(); DBUG_ASSERT(cf != nullptr); - if (m_sst_info == nullptr) { - m_sst_info = std::make_shared( - rdb, m_table_handler->m_table_name, kd.get_name(), cf, - *rocksdb_db_options, THDVAR(ha_thd(), trace_sst_api)); - tx->start_bulk_load(this); - m_bulk_load_tx = tx; + int res = HA_EXIT_SUCCESS; + + if (sort) { + GL_INDEX_ID kd_gl_id = kd.get_gl_index_id(); + auto it = m_key_merge.find(kd_gl_id); + if (it == m_key_merge.end()) { + m_key_merge.emplace( + std::piecewise_construct, std::make_tuple(kd_gl_id), + std::make_tuple( + thd_rocksdb_tmpdir(), THDVAR(ha_thd(), merge_buf_size), + THDVAR(ha_thd(), merge_combine_read_size), + THDVAR(ha_thd(), merge_tmp_file_removal_delay_ms), cf)); + it = m_key_merge.find(kd_gl_id); + if ((res = it->second.init()) != 0) { + DBUG_RETURN(res); + } + + if (m_bulk_load_tx == nullptr) { + tx->start_bulk_load(this); + m_bulk_load_tx = tx; + } + } + res = it->second.add(key, value); + } else { + if (!m_sst_info) { + m_sst_info.reset(new Rdb_sst_info(rdb, m_table_handler->m_table_name, + kd.get_name(), cf, *rocksdb_db_options, + THDVAR(ha_thd(), trace_sst_api))); + tx->start_bulk_load(this); + m_bulk_load_tx = tx; + } + + DBUG_ASSERT(m_sst_info); + + res = m_sst_info->put(key, value); } - DBUG_ASSERT(m_sst_info != nullptr); - - return m_sst_info->put(key, value); + DBUG_RETURN(res); } int ha_rocksdb::finalize_bulk_load() { - int rc = 0; + DBUG_ENTER_FUNC(); + + DBUG_ASSERT_IMP(!m_key_merge.empty() || m_sst_info, + m_bulk_load_tx != nullptr); /* Skip if there are no possible ongoing bulk loads */ - if (m_sst_info == nullptr && m_bulk_load_tx == nullptr) { - return rc; + if (m_key_merge.empty() && !m_sst_info && m_bulk_load_tx == nullptr) { + DBUG_RETURN(HA_EXIT_SUCCESS); } + int res = HA_EXIT_SUCCESS; + RDB_MUTEX_LOCK_CHECK(m_bulk_load_mutex); - /* - We need this check because it's possible that m_sst_info has been - flushed and cleared by another thread by the time the mutex has been - acquired. - */ - if (m_sst_info != nullptr) { - rc = m_sst_info->commit(); - m_sst_info = nullptr; + if (m_sst_info) { + res = m_sst_info->commit(); + m_sst_info.reset(); + } + + if (!m_key_merge.empty()) { + rocksdb::Slice merge_key; + rocksdb::Slice merge_val; + for (auto it = m_key_merge.begin(); it != m_key_merge.end(); it++) { + const std::string &index_name = + ddl_manager.safe_find(it->first)->get_name(); + Rdb_index_merge &rdb_merge = it->second; + Rdb_sst_info sst_info(rdb, m_table_handler->m_table_name, index_name, + rdb_merge.get_cf(), *rocksdb_db_options, + THDVAR(ha_thd(), trace_sst_api)); + + while ((res = rdb_merge.next(&merge_key, &merge_val)) == 0) { + if ((res = sst_info.put(merge_key, merge_val)) != 0) { + break; + } + } + // res == -1 => finished ok; res > 0 => error + if (res <= 0) { + if ((res = sst_info.commit()) != 0) { + break; + } + } + } + m_key_merge.clear(); + + /* + Explicitly tell jemalloc to clean up any unused dirty pages at this point. + See https://reviews.facebook.net/D63723 for more details. + */ + purge_all_jemalloc_arenas(); + } + + if (m_bulk_load_tx != nullptr) { m_bulk_load_tx->end_bulk_load(this); m_bulk_load_tx = nullptr; } RDB_MUTEX_UNLOCK_CHECK(m_bulk_load_mutex); - return rc; + DBUG_RETURN(res); } int ha_rocksdb::update_pk(const Rdb_key_def &kd, @@ -8172,6 +8554,7 @@ int ha_rocksdb::update_pk(const Rdb_key_def &kd, const bool &pk_changed) { const uint key_id = kd.get_keyno(); const bool hidden_pk = is_hidden_pk(key_id, table, m_tbl_def); + ulonglong bytes_written = 0; /* If the PK has changed, or if this PK uses single deletes and this is an @@ -8186,6 +8569,8 @@ int ha_rocksdb::update_pk(const Rdb_key_def &kd, if (!s.ok()) { return row_info.tx->set_status_error(table->in_use, s, kd, m_tbl_def, m_table_handler); + } else { + bytes_written = row_info.old_pk_slice.size(); } } @@ -8206,7 +8591,8 @@ int ha_rocksdb::update_pk(const Rdb_key_def &kd, /* Write the primary key directly to an SST file using an SstFileWriter */ - rc = bulk_load_key(row_info.tx, kd, row_info.new_pk_slice, value_slice); + rc = bulk_load_key(row_info.tx, kd, row_info.new_pk_slice, value_slice, + THDVAR(table->in_use, bulk_load_allow_unsorted)); } else if (row_info.skip_unique_check || row_info.tx->m_ddl_transaction) { /* It is responsibility of the user to make sure that the data being @@ -8228,6 +8614,10 @@ int ha_rocksdb::update_pk(const Rdb_key_def &kd, } } + if (rc == HA_EXIT_SUCCESS) { + row_info.tx->update_bytes_written( + bytes_written + row_info.new_pk_slice.size() + value_slice.size()); + } return rc; } @@ -8241,24 +8631,31 @@ int ha_rocksdb::update_sk(const TABLE *const table_arg, const Rdb_key_def &kd, rocksdb::Slice old_key_slice; const uint key_id = kd.get_keyno(); + + ulonglong bytes_written = 0; + /* - Can skip updating this key if none of the key fields have changed. + Can skip updating this key if none of the key fields have changed and, if + this table has TTL, the TTL timestamp has not changed. */ - if (row_info.old_data != nullptr && !m_update_scope.is_set(key_id)) { + if (row_info.old_data != nullptr && !m_update_scope.is_set(key_id) && + (!kd.has_ttl() || !m_ttl_bytes_updated)) { return HA_EXIT_SUCCESS; } const bool store_row_debug_checksums = should_store_row_debug_checksums(); - new_packed_size = kd.pack_record( - table_arg, m_pack_buffer, row_info.new_data, m_sk_packed_tuple, - &m_sk_tails, store_row_debug_checksums, row_info.hidden_pk_id); + new_packed_size = + kd.pack_record(table_arg, m_pack_buffer, row_info.new_data, + m_sk_packed_tuple, &m_sk_tails, store_row_debug_checksums, + row_info.hidden_pk_id, 0, nullptr, nullptr, m_ttl_bytes); if (row_info.old_data != nullptr) { // The old value old_packed_size = kd.pack_record( table_arg, m_pack_buffer, row_info.old_data, m_sk_packed_tuple_old, - &m_sk_tails_old, store_row_debug_checksums, row_info.hidden_pk_id); + &m_sk_tails_old, store_row_debug_checksums, row_info.hidden_pk_id, 0, + nullptr, nullptr, m_ttl_bytes); /* Check if we are going to write the same value. This can happen when @@ -8276,6 +8673,7 @@ int ha_rocksdb::update_sk(const TABLE *const table_arg, const Rdb_key_def &kd, */ if (old_packed_size == new_packed_size && m_sk_tails_old.get_current_pos() == m_sk_tails.get_current_pos() && + !(kd.has_ttl() && m_ttl_bytes_updated) && memcmp(m_sk_packed_tuple_old, m_sk_packed_tuple, old_packed_size) == 0 && memcmp(m_sk_tails_old.ptr(), m_sk_tails.ptr(), @@ -8294,6 +8692,8 @@ int ha_rocksdb::update_sk(const TABLE *const table_arg, const Rdb_key_def &kd, row_info.tx->get_indexed_write_batch()->SingleDelete(kd.get_cf(), old_key_slice); + + bytes_written = old_key_slice.size(); } new_key_slice = rocksdb::Slice( @@ -8305,6 +8705,9 @@ int ha_rocksdb::update_sk(const TABLE *const table_arg, const Rdb_key_def &kd, row_info.tx->get_indexed_write_batch()->Put(kd.get_cf(), new_key_slice, new_value_slice); + row_info.tx->update_bytes_written(bytes_written + new_key_slice.size() + + new_value_slice.size()); + return HA_EXIT_SUCCESS; } @@ -8312,14 +8715,19 @@ int ha_rocksdb::update_indexes(const struct update_row_info &row_info, const bool &pk_changed) { int rc; + // The PK must be updated first to pull out the TTL value. + rc = update_pk(*m_pk_descr, row_info, pk_changed); + if (rc != HA_EXIT_SUCCESS) { + return rc; + } + + // Update the remaining indexes. for (uint key_id = 0; key_id < m_tbl_def->m_key_count; key_id++) { - const Rdb_key_def &kd = *m_key_descr_arr[key_id]; if (is_pk(key_id, table, m_tbl_def)) { - rc = update_pk(kd, row_info, pk_changed); - } else { - rc = update_sk(table, kd, row_info); + continue; } + rc = update_sk(table, *m_key_descr_arr[key_id], row_info); if (rc != HA_EXIT_SUCCESS) { return rc; } @@ -8597,7 +9005,8 @@ int ha_rocksdb::rnd_next_with_direction(uchar *const buf, bool move_forward) { DEBUG_SYNC(ha_thd(), "rocksdb_concurrent_delete"); if (m_pk_descr->has_ttl() && - should_hide_ttl_rec(m_scan_it->value(), tx->m_snapshot_timestamp)) { + should_hide_ttl_rec(*m_pk_descr, m_scan_it->value(), + tx->m_snapshot_timestamp)) { continue; } @@ -8628,7 +9037,8 @@ int ha_rocksdb::rnd_next_with_direction(uchar *const buf, bool move_forward) { if (m_pk_descr->has_ttl() && should_hide_ttl_rec( - value, get_or_create_tx(table->in_use)->m_snapshot_timestamp)) { + *m_pk_descr, value, + get_or_create_tx(table->in_use)->m_snapshot_timestamp)) { continue; } @@ -8670,6 +9080,10 @@ int ha_rocksdb::index_init(uint idx, bool sorted) { setup_read_decoders(); + if (!m_keyread_only) { + m_key_descr_arr[idx]->get_lookup_bitmap(table, &m_lookup_bitmap); + } + // If m_lock_rows is not RDB_LOCK_NONE then we will be doing a get_for_update // when accessing the index, so don't acquire the snapshot right away. // Otherwise acquire the snapshot immediately. @@ -8689,6 +9103,8 @@ int ha_rocksdb::index_end() { release_scan_iterator(); + bitmap_free(&m_lookup_bitmap); + active_index = MAX_KEY; in_range_check_pushed_down = FALSE; @@ -8733,6 +9149,7 @@ int ha_rocksdb::delete_row(const uchar *const buf) { rocksdb::Slice key_slice(m_last_rowkey.ptr(), m_last_rowkey.length()); Rdb_transaction *const tx = get_or_create_tx(table->in_use); + ulonglong bytes_written = 0; const uint index = pk_index(table, m_tbl_def); rocksdb::Status s = @@ -8740,6 +9157,8 @@ int ha_rocksdb::delete_row(const uchar *const buf) { if (!s.ok()) { DBUG_RETURN(tx->set_status_error(table->in_use, s, *m_pk_descr, m_tbl_def, m_table_handler)); + } else { + bytes_written = key_slice.size(); } longlong hidden_pk_id = 0; @@ -8762,6 +9181,7 @@ int ha_rocksdb::delete_row(const uchar *const buf) { /* Deleting on secondary key doesn't need any locks: */ tx->get_indexed_write_batch()->SingleDelete(kd.get_cf(), secondary_key_slice); + bytes_written += secondary_key_slice.size(); } } @@ -8772,6 +9192,7 @@ int ha_rocksdb::delete_row(const uchar *const buf) { } stats.rows_deleted++; update_row_stats(ROWS_DELETED); + tx->update_bytes_written(bytes_written); DBUG_RETURN(HA_EXIT_SUCCESS); } @@ -8860,13 +9281,37 @@ int ha_rocksdb::info(uint flag) { stats.data_file_length+= sz; } - // Second, compute memtable stats - uint64_t memtableCount; - uint64_t memtableSize; - rdb->GetApproximateMemTableStats(m_pk_descr->get_cf(), r, - &memtableCount, &memtableSize); - stats.records += memtableCount; - stats.data_file_length += memtableSize; + // Second, compute memtable stats. This call is expensive, so cache + // values computed for some time. + uint64_t cachetime = rocksdb_force_compute_memtable_stats_cachetime; + uint64_t time = (cachetime == 0) ? 0 : my_micro_time(); + if (cachetime == 0 || + time > m_table_handler->m_mtcache_last_update + cachetime) { + uint64_t memtableCount; + uint64_t memtableSize; + + rdb->GetApproximateMemTableStats(m_pk_descr->get_cf(), r, + &memtableCount, &memtableSize); + + // Atomically update all of these fields at the same time + if (cachetime > 0) { + if (m_table_handler->m_mtcache_lock.fetch_add( + 1, std::memory_order_acquire) == 0) { + m_table_handler->m_mtcache_count = memtableCount; + m_table_handler->m_mtcache_size = memtableSize; + m_table_handler->m_mtcache_last_update = time; + } + m_table_handler->m_mtcache_lock.fetch_sub(1, + std::memory_order_release); + } + + stats.records += memtableCount; + stats.data_file_length += memtableSize; + } else { + // Cached data is still valid, so use it instead + stats.records += m_table_handler->m_mtcache_count; + stats.data_file_length += m_table_handler->m_mtcache_size; + } if (rocksdb_debug_optimizer_n_rows > 0) stats.records = rocksdb_debug_optimizer_n_rows; @@ -9386,10 +9831,6 @@ void Rdb_drop_index_thread::run() { uchar buf[Rdb_key_def::INDEX_NUMBER_SIZE * 2]; rocksdb::Range range = get_range(d.index_id, buf, is_reverse_cf ? 1 : 0, is_reverse_cf ? 0 : 1); - rocksdb::CompactRangeOptions compact_range_options; - compact_range_options.bottommost_level_compaction = - rocksdb::BottommostLevelCompaction::kForce; - compact_range_options.exclusive_manual_compaction = false; rocksdb::Status status = DeleteFilesInRange(rdb->GetBaseDB(), cfh, &range.start, &range.limit); if (!status.ok()) { @@ -9398,7 +9839,7 @@ void Rdb_drop_index_thread::run() { } rdb_handle_io_error(status, RDB_IO_ERROR_BG_THREAD); } - status = rdb->CompactRange(compact_range_options, cfh, &range.start, + status = rdb->CompactRange(getCompactRangeOptions(), cfh, &range.start, &range.limit); if (!status.ok()) { if (status.IsShutdownInProgress()) { @@ -9485,6 +9926,7 @@ int ha_rocksdb::remove_rows(Rdb_tbl_def *const tbl) { char key_buf[MAX_KEY_LENGTH]; uint key_len; + ulonglong bytes_written = 0; /* Remove all records in each index. (This is is not crash-safe, but it doesn't matter, because bulk row @@ -9516,11 +9958,13 @@ int ha_rocksdb::remove_rows(Rdb_tbl_def *const tbl) { return tx->set_status_error(table->in_use, s, *m_pk_descr, m_tbl_def, m_table_handler); } - + bytes_written += key.size(); it->Next(); } } + tx->update_bytes_written(bytes_written); + return HA_EXIT_SUCCESS; } @@ -9569,6 +10013,8 @@ int ha_rocksdb::rename_table(const char *const from, const char *const to) { DBUG_RETURN(-1); } + DBUG_EXECUTE_IF("gen_sql_table_name", to_str = to_str + "#sql-test";); + const std::unique_ptr wb = dict_manager.begin(); rocksdb::WriteBatch *const batch = wb.get(); dict_manager.lock(); @@ -9773,7 +10219,7 @@ int ha_rocksdb::optimize(THD *const thd, HA_CHECK_OPT *const check_opt) { for (uint i = 0; i < table->s->keys; i++) { uchar buf[Rdb_key_def::INDEX_NUMBER_SIZE * 2]; auto range = get_range(i, buf); - const rocksdb::Status s = rdb->CompactRange(rocksdb::CompactRangeOptions(), + const rocksdb::Status s = rdb->CompactRange(getCompactRangeOptions(), m_key_descr_arr[i]->get_cf(), &range.start, &range.limit); if (!s.ok()) { @@ -10311,7 +10757,6 @@ int ha_rocksdb::inplace_populate_sk( const std::unordered_set> &indexes) { DBUG_ENTER_FUNC(); int res = HA_EXIT_SUCCESS; - const std::unique_ptr wb = dict_manager.begin(); rocksdb::WriteBatch *const batch = wb.get(); @@ -10367,7 +10812,7 @@ int ha_rocksdb::inplace_populate_sk( open handlers have been closed at this point, and the one we're on is the only one left. */ - if (m_sst_info != nullptr) { + if (m_sst_info) { if ((res = finalize_bulk_load())) { DBUG_RETURN(res); } @@ -10377,14 +10822,16 @@ int ha_rocksdb::inplace_populate_sk( const ulonglong rdb_merge_buf_size = THDVAR(ha_thd(), merge_buf_size); const ulonglong rdb_merge_combine_read_size = THDVAR(ha_thd(), merge_combine_read_size); + const ulonglong rdb_merge_tmp_file_removal_delay = + THDVAR(ha_thd(), merge_tmp_file_removal_delay_ms); for (const auto &index : indexes) { - const rocksdb::Comparator *index_comp = index->get_cf()->GetComparator(); bool is_unique_index = new_table_arg->key_info[index->get_keyno()].flags & HA_NOSAME; - Rdb_index_merge rdb_merge(thd_rocksdb_tmpdir(), rdb_merge_buf_size, - rdb_merge_combine_read_size, index_comp); + Rdb_index_merge rdb_merge( + thd_rocksdb_tmpdir(), rdb_merge_buf_size, rdb_merge_combine_read_size, + rdb_merge_tmp_file_removal_delay, index->get_cf()); if ((res = rdb_merge.init())) { DBUG_RETURN(res); @@ -10413,7 +10860,8 @@ int ha_rocksdb::inplace_populate_sk( /* Create new secondary index entry */ const int new_packed_size = index->pack_record( new_table_arg, m_pack_buffer, table->record[0], m_sk_packed_tuple, - &m_sk_tails, should_store_row_debug_checksums(), hidden_pk_id); + &m_sk_tails, should_store_row_debug_checksums(), hidden_pk_id, 0, + nullptr, nullptr, m_ttl_bytes); const rocksdb::Slice key = rocksdb::Slice( reinterpret_cast(m_sk_packed_tuple), new_packed_size); @@ -10478,7 +10926,7 @@ int ha_rocksdb::inplace_populate_sk( /* Insert key and slice to SST via SSTFileWriter API. */ - if ((res = bulk_load_key(tx, *index, merge_key, merge_val))) { + if ((res = bulk_load_key(tx, *index, merge_key, merge_val, false))) { break; } } @@ -10831,6 +11279,9 @@ static void myrocks_update_status() { export_stats.queries_point = global_stats.queries[QUERIES_POINT]; export_stats.queries_range = global_stats.queries[QUERIES_RANGE]; + + export_stats.covered_secondary_key_lookups = + global_stats.covered_secondary_key_lookups; } static void myrocks_update_memory_status() { @@ -10874,6 +11325,9 @@ static SHOW_VAR myrocks_status_variables[] = { SHOW_LONGLONG), DEF_STATUS_VAR_FUNC("queries_range", &export_stats.queries_range, SHOW_LONGLONG), + DEF_STATUS_VAR_FUNC("covered_secondary_key_lookups", + &export_stats.covered_secondary_key_lookups, + SHOW_LONGLONG), {NullS, NullS, SHOW_LONG}}; @@ -10884,6 +11338,91 @@ static void show_myrocks_vars(THD *thd, SHOW_VAR *var, char *buff) { var->value = reinterpret_cast(&myrocks_status_variables); } +static ulonglong +io_stall_prop_value(const std::map &props, + const std::string &key) { + std::map::const_iterator iter = + props.find("io_stalls." + key); + if (iter != props.end()) { + return std::stoull(iter->second); + } else { + DBUG_PRINT("warning", + ("RocksDB GetMapPropery hasn't returned key=%s", key.c_str())); + DBUG_ASSERT(0); + return 0; + } +} + +static void update_rocksdb_stall_status() { + st_io_stall_stats local_io_stall_stats; + for (const auto &cf_name : cf_manager.get_cf_names()) { + rocksdb::ColumnFamilyHandle *cfh = cf_manager.get_cf(cf_name); + if (cfh == nullptr) { + continue; + } + + std::map props; + if (!rdb->GetMapProperty(cfh, "rocksdb.cfstats", &props)) { + continue; + } + + local_io_stall_stats.level0_slowdown += + io_stall_prop_value(props, "level0_slowdown"); + local_io_stall_stats.level0_slowdown_with_compaction += + io_stall_prop_value(props, "level0_slowdown_with_compaction"); + local_io_stall_stats.level0_numfiles += + io_stall_prop_value(props, "level0_numfiles"); + local_io_stall_stats.level0_numfiles_with_compaction += + io_stall_prop_value(props, "level0_numfiles_with_compaction"); + local_io_stall_stats.stop_for_pending_compaction_bytes += + io_stall_prop_value(props, "stop_for_pending_compaction_bytes"); + local_io_stall_stats.slowdown_for_pending_compaction_bytes += + io_stall_prop_value(props, "slowdown_for_pending_compaction_bytes"); + local_io_stall_stats.memtable_compaction += + io_stall_prop_value(props, "memtable_compaction"); + local_io_stall_stats.memtable_slowdown += + io_stall_prop_value(props, "memtable_slowdown"); + local_io_stall_stats.total_stop += io_stall_prop_value(props, "total_stop"); + local_io_stall_stats.total_slowdown += + io_stall_prop_value(props, "total_slowdown"); + } + io_stall_stats = local_io_stall_stats; +} + +static SHOW_VAR rocksdb_stall_status_variables[] = { + DEF_STATUS_VAR_FUNC("l0_file_count_limit_slowdowns", + &io_stall_stats.level0_slowdown, SHOW_LONGLONG), + DEF_STATUS_VAR_FUNC("locked_l0_file_count_limit_slowdowns", + &io_stall_stats.level0_slowdown_with_compaction, + SHOW_LONGLONG), + DEF_STATUS_VAR_FUNC("l0_file_count_limit_stops", + &io_stall_stats.level0_numfiles, SHOW_LONGLONG), + DEF_STATUS_VAR_FUNC("locked_l0_file_count_limit_stops", + &io_stall_stats.level0_numfiles_with_compaction, + SHOW_LONGLONG), + DEF_STATUS_VAR_FUNC("pending_compaction_limit_stops", + &io_stall_stats.stop_for_pending_compaction_bytes, + SHOW_LONGLONG), + DEF_STATUS_VAR_FUNC("pending_compaction_limit_slowdowns", + &io_stall_stats.slowdown_for_pending_compaction_bytes, + SHOW_LONGLONG), + DEF_STATUS_VAR_FUNC("memtable_limit_stops", + &io_stall_stats.memtable_compaction, SHOW_LONGLONG), + DEF_STATUS_VAR_FUNC("memtable_limit_slowdowns", + &io_stall_stats.memtable_slowdown, SHOW_LONGLONG), + DEF_STATUS_VAR_FUNC("total_stops", &io_stall_stats.total_stop, + SHOW_LONGLONG), + DEF_STATUS_VAR_FUNC("total_slowdowns", &io_stall_stats.total_slowdown, + SHOW_LONGLONG), + // end of the array marker + {NullS, NullS, SHOW_LONG}}; + +static void show_rocksdb_stall_vars(THD *thd, SHOW_VAR *var, char *buff) { + update_rocksdb_stall_status(); + var->type = SHOW_ARRAY; + var->value = reinterpret_cast(&rocksdb_stall_status_variables); +} + static SHOW_VAR rocksdb_status_vars[] = { DEF_STATUS_VAR(block_cache_miss), DEF_STATUS_VAR(block_cache_hit), @@ -10951,7 +11490,12 @@ static SHOW_VAR rocksdb_status_vars[] = { SHOW_LONGLONG), DEF_STATUS_VAR_PTR("number_sst_entry_other", &rocksdb_num_sst_entry_other, SHOW_LONGLONG), + // the variables generated by SHOW_FUNC are sorted only by prefix (first + // arg in the tuple below), so make sure it is unique to make sorting + // deterministic as quick sort is not stable {"rocksdb", reinterpret_cast(&show_myrocks_vars), SHOW_FUNC}, + {"rocksdb_stall", reinterpret_cast(&show_rocksdb_stall_vars), + SHOW_FUNC}, {NullS, NullS, SHOW_LONG}}; /* @@ -10998,10 +11542,13 @@ void Rdb_background_thread::run() { timespec ts; clock_gettime(CLOCK_REALTIME, &ts); - // Flush the WAL. - if (rdb && (rocksdb_flush_log_at_trx_commit == 2)) { + // Flush the WAL. Sync it for both background and never modes to copy + // InnoDB's behavior. For mode never, the wal file isn't even written, + // whereas background writes to the wal file, but issues the syncs in a + // background thread. + if (rdb && (rocksdb_flush_log_at_trx_commit != FLUSH_LOG_SYNC)) { DBUG_ASSERT(!rocksdb_db_options->allow_mmap_writes); - const rocksdb::Status s = rdb->SyncWAL(); + const rocksdb::Status s = rdb->FlushWAL(true); if (!s.ok()) { rdb_handle_io_error(s, RDB_IO_ERROR_BG_THREAD); } @@ -11101,6 +11648,7 @@ int rdb_dbug_set_ttl_snapshot_ts() { return rocksdb_debug_ttl_snapshot_ts; } int rdb_dbug_set_ttl_read_filter_ts() { return rocksdb_debug_ttl_read_filter_ts; } +bool rdb_dbug_set_ttl_ignore_pk() { return rocksdb_debug_ttl_ignore_pk; } #endif void rdb_update_global_stats(const operation_type &type, uint count, @@ -11322,6 +11870,17 @@ void rocksdb_set_delayed_write_rate(THD *thd, struct st_mysql_sys_var *var, RDB_MUTEX_UNLOCK_CHECK(rdb_sysvars_mutex); } +void rocksdb_set_max_latest_deadlocks(THD *thd, struct st_mysql_sys_var *var, + void *var_ptr, const void *save) { + RDB_MUTEX_LOCK_CHECK(rdb_sysvars_mutex); + const uint32_t new_val = *static_cast(save); + if (rocksdb_max_latest_deadlocks != new_val) { + rocksdb_max_latest_deadlocks = new_val; + rdb->SetDeadlockInfoBufferSize(rocksdb_max_latest_deadlocks); + } + RDB_MUTEX_UNLOCK_CHECK(rdb_sysvars_mutex); +} + void rdb_set_collation_exception_list(const char *const exception_list) { DBUG_ASSERT(rdb_collation_exceptions != nullptr); @@ -11361,6 +11920,18 @@ void rocksdb_set_bulk_load(THD *const thd, struct st_mysql_sys_var *const var *static_cast(var_ptr) = *static_cast(save); } +void rocksdb_set_bulk_load_allow_unsorted( + THD *const thd, + struct st_mysql_sys_var *const var MY_ATTRIBUTE((__unused__)), + void *const var_ptr, const void *const save) { + if (THDVAR(thd, bulk_load)) { + my_error(ER_ERROR_WHEN_EXECUTING_COMMAND, MYF(0), "SET", + "Cannot change this setting while bulk load is enabled"); + } else { + *static_cast(var_ptr) = *static_cast(save); + } +} + static void rocksdb_set_max_background_jobs(THD *thd, struct st_mysql_sys_var *const var, void *const var_ptr, diff --git a/storage/rocksdb/ha_rocksdb.h b/storage/rocksdb/ha_rocksdb.h index e6d4c2e6485..3b72173f94c 100644 --- a/storage/rocksdb/ha_rocksdb.h +++ b/storage/rocksdb/ha_rocksdb.h @@ -22,6 +22,7 @@ /* C++ standard header files */ #include #include +#include #include #include @@ -235,6 +236,12 @@ const char *const RDB_TTL_COL_QUALIFIER = "ttl_col"; */ #define ROCKSDB_SIZEOF_TTL_RECORD sizeof(longlong) +/* + Maximum index prefix length in bytes. +*/ +#define MAX_INDEX_COL_LEN_LARGE 3072 +#define MAX_INDEX_COL_LEN_SMALL 767 + /* MyRocks specific error codes. NB! Please make sure that you will update HA_ERR_ROCKSDB_LAST when adding new ones. Also update the strings in @@ -293,7 +300,14 @@ struct Rdb_table_handler { /* Stores cumulative table statistics */ my_io_perf_atomic_t m_io_perf_read; + my_io_perf_atomic_t m_io_perf_write; Rdb_atomic_perf_counters m_table_perf_context; + + /* Stores cached memtable estimate statistics */ + std::atomic_uint m_mtcache_lock; + uint64_t m_mtcache_count; + uint64_t m_mtcache_size; + uint64_t m_mtcache_last_update; }; class Rdb_key_def; @@ -360,6 +374,8 @@ struct st_global_stats { ib_counter_t system_rows[ROWS_MAX]; ib_counter_t queries[QUERIES_MAX]; + + ib_counter_t covered_secondary_key_lookups; }; /* Struct used for exporting status to MySQL */ @@ -379,6 +395,8 @@ struct st_export_stats { ulonglong queries_point; ulonglong queries_range; + + ulonglong covered_secondary_key_lookups; }; /* Struct used for exporting RocksDB memory status */ @@ -387,6 +405,27 @@ struct st_memory_stats { ulonglong memtable_unflushed; }; +/* Struct used for exporting RocksDB IO stalls stats */ +struct st_io_stall_stats { + ulonglong level0_slowdown; + ulonglong level0_slowdown_with_compaction; + ulonglong level0_numfiles; + ulonglong level0_numfiles_with_compaction; + ulonglong stop_for_pending_compaction_bytes; + ulonglong slowdown_for_pending_compaction_bytes; + ulonglong memtable_compaction; + ulonglong memtable_slowdown; + ulonglong total_stop; + ulonglong total_slowdown; + + st_io_stall_stats() + : level0_slowdown(0), level0_slowdown_with_compaction(0), + level0_numfiles(0), level0_numfiles_with_compaction(0), + stop_for_pending_compaction_bytes(0), + slowdown_for_pending_compaction_bytes(0), memtable_compaction(0), + memtable_slowdown(0), total_stop(0), total_slowdown(0) {} +}; + } // namespace myrocks #include "./rdb_buff.h" @@ -495,6 +534,12 @@ class ha_rocksdb : public my_core::handler { Pointer to the original TTL timestamp value (8 bytes) during UPDATE. */ char m_ttl_bytes[ROCKSDB_SIZEOF_TTL_RECORD]; + /* + The TTL timestamp value can change if the explicit TTL column is + updated. If we detect this when updating the PK, we indicate it here so + we know we must always update any SK's. + */ + bool m_ttl_bytes_updated; /* rowkey of the last record we've read, in StorageFormat. */ String m_last_rowkey; @@ -541,7 +586,9 @@ class ha_rocksdb : public my_core::handler { bool m_update_scope_is_valid; /* SST information used for bulk loading the primary key */ - std::shared_ptr m_sst_info; + std::unique_ptr m_sst_info; + /* External merge sorts for bulk load: key ID -> merge sort instance */ + std::unordered_map m_key_merge; Rdb_transaction *m_bulk_load_tx; /* Mutex to protect finalizing bulk load */ mysql_mutex_t m_bulk_load_mutex; @@ -636,6 +683,13 @@ class ha_rocksdb : public my_core::handler { /* Setup field_decoders based on type of scan and table->read_set */ void setup_read_decoders(); + /* + For the active index, indicates which columns must be covered for the + current lookup to be covered. If the bitmap field is null, that means this + index does not cover the current lookup for any record. + */ + MY_BITMAP m_lookup_bitmap = {nullptr, 0, 0, nullptr, nullptr}; + /* Number of bytes in on-disk (storage) record format that are used for storing SQL NULL flags. @@ -854,11 +908,7 @@ public: DBUG_RETURN(MAX_REF_PARTS); } - uint max_supported_key_part_length() const override { - DBUG_ENTER_FUNC(); - - DBUG_RETURN(2048); - } + uint max_supported_key_part_length() const override; /** @brief unireg.cc will call this to make sure that the storage engine can handle @@ -1040,9 +1090,13 @@ private: rocksdb::Slice *const packed_rec) MY_ATTRIBUTE((__nonnull__)); - bool should_hide_ttl_rec(const rocksdb::Slice &ttl_rec_val, + bool should_hide_ttl_rec(const Rdb_key_def &kd, + const rocksdb::Slice &ttl_rec_val, const int64_t curr_ts) MY_ATTRIBUTE((__warn_unused_result__)); + void rocksdb_skip_expired_records(const Rdb_key_def &kd, + rocksdb::Iterator *const iter, + bool seek_backward); int index_first_intern(uchar *buf) MY_ATTRIBUTE((__nonnull__, __warn_unused_result__)); @@ -1076,8 +1130,10 @@ private: struct unique_sk_buf_info *sk_info) MY_ATTRIBUTE((__nonnull__, __warn_unused_result__)); int bulk_load_key(Rdb_transaction *const tx, const Rdb_key_def &kd, - const rocksdb::Slice &key, const rocksdb::Slice &value) + const rocksdb::Slice &key, const rocksdb::Slice &value, + bool sort) MY_ATTRIBUTE((__nonnull__, __warn_unused_result__)); + void update_bytes_written(ulonglong bytes_written); int update_pk(const Rdb_key_def &kd, const struct update_row_info &row_info, const bool &pk_changed) MY_ATTRIBUTE((__warn_unused_result__)); int update_sk(const TABLE *const table_arg, const Rdb_key_def &kd, diff --git a/storage/rocksdb/ha_rocksdb_proto.h b/storage/rocksdb/ha_rocksdb_proto.h index 1beb6b80622..71ec6957456 100644 --- a/storage/rocksdb/ha_rocksdb_proto.h +++ b/storage/rocksdb/ha_rocksdb_proto.h @@ -76,6 +76,7 @@ bool rdb_is_ttl_read_filtering_enabled(); int rdb_dbug_set_ttl_rec_ts(); int rdb_dbug_set_ttl_snapshot_ts(); int rdb_dbug_set_ttl_read_filter_ts(); +bool rdb_dbug_set_ttl_ignore_pk(); #endif enum operation_type : int; diff --git a/storage/rocksdb/mysql-test/rocksdb/include/prefix_index_only_query_check.inc b/storage/rocksdb/mysql-test/rocksdb/include/prefix_index_only_query_check.inc new file mode 100644 index 00000000000..e96eb573c1f --- /dev/null +++ b/storage/rocksdb/mysql-test/rocksdb/include/prefix_index_only_query_check.inc @@ -0,0 +1,21 @@ +# +# A helper include file for prefix index index-only query tests +# +# Parameters: +# $prefix_index_check_title - title of the test +# $prefix_index_check_query - test query +# $prefix_index_check_read_avoided_delta - expected change of +# 'rocksdb_covered_secondary_key_lookups' status variable +# value after running the query + +--let $show_count_statement = show status like 'rocksdb_covered_secondary_key_lookups' + +--echo # $prefix_index_check_title +--let $base_count = query_get_value($show_count_statement, Value, 1) + +--eval $prefix_index_check_query + +--let $count = query_get_value($show_count_statement, Value, 1) +--let $assert_text= $prefix_index_check_title: $prefix_index_check_read_avoided_delta rocksdb_covered_secondary_key_lookups +--let $assert_cond= $count - $base_count = $prefix_index_check_read_avoided_delta +--source include/assert.inc diff --git a/storage/rocksdb/mysql-test/rocksdb/my.cnf b/storage/rocksdb/mysql-test/rocksdb/my.cnf index 2ed68088259..faeae84da5f 100644 --- a/storage/rocksdb/mysql-test/rocksdb/my.cnf +++ b/storage/rocksdb/mysql-test/rocksdb/my.cnf @@ -5,3 +5,4 @@ sql-mode=NO_ENGINE_SUBSTITUTION explicit-defaults-for-timestamp=1 rocksdb_lock_wait_timeout=1 rocksdb_strict_collation_check=0 +rocksdb_force_compute_memtable_stats_cachetime=0 diff --git a/storage/rocksdb/mysql-test/rocksdb/r/2pc_group_commit.result b/storage/rocksdb/mysql-test/rocksdb/r/2pc_group_commit.result index 06452a5437f..7b15ed47d44 100644 --- a/storage/rocksdb/mysql-test/rocksdb/r/2pc_group_commit.result +++ b/storage/rocksdb/mysql-test/rocksdb/r/2pc_group_commit.result @@ -4,20 +4,20 @@ CREATE DATABASE mysqlslap; USE mysqlslap; CREATE TABLE t1(id BIGINT AUTO_INCREMENT, value BIGINT, PRIMARY KEY(id)) ENGINE=rocksdb; # 2PC enabled, MyRocks durability enabled -SET GLOBAL rocksdb_enable_2pc=0; +SET GLOBAL rocksdb_enable_2pc=1; SET GLOBAL rocksdb_flush_log_at_trx_commit=1; ## 2PC + durability + single thread select variable_value into @c from information_schema.global_status where variable_name='rocksdb_wal_group_syncs'; select case when variable_value-@c = 1000 then 'true' else 'false' end from information_schema.global_status where variable_name='rocksdb_wal_group_syncs'; case when variable_value-@c = 1000 then 'true' else 'false' end -false +true ## 2PC + durability + group commit select variable_value into @c from information_schema.global_status where variable_name='rocksdb_wal_group_syncs'; select case when variable_value-@c > 0 and variable_value-@c < 10000 then 'true' else 'false' end from information_schema.global_status where variable_name='rocksdb_wal_group_syncs'; case when variable_value-@c > 0 and variable_value-@c < 10000 then 'true' else 'false' end -false +true # 2PC enabled, MyRocks durability disabled -SET GLOBAL rocksdb_enable_2pc=0; +SET GLOBAL rocksdb_enable_2pc=1; SET GLOBAL rocksdb_flush_log_at_trx_commit=0; select variable_value into @c from information_schema.global_status where variable_name='rocksdb_wal_group_syncs'; select case when variable_value-@c = 0 then 'true' else 'false' end from information_schema.global_status where variable_name='rocksdb_wal_group_syncs'; @@ -28,16 +28,16 @@ select case when variable_value-@c = 0 then 'true' else 'false' end from informa case when variable_value-@c = 0 then 'true' else 'false' end true # 2PC disabled, MyRocks durability enabled -SET GLOBAL rocksdb_enable_2pc=1; +SET GLOBAL rocksdb_enable_2pc=0; SET GLOBAL rocksdb_flush_log_at_trx_commit=1; select variable_value into @c from information_schema.global_status where variable_name='rocksdb_wal_group_syncs'; select case when variable_value-@c = 0 then 'true' else 'false' end from information_schema.global_status where variable_name='rocksdb_wal_group_syncs'; case when variable_value-@c = 0 then 'true' else 'false' end -false +true select variable_value into @c from information_schema.global_status where variable_name='rocksdb_wal_group_syncs'; select case when variable_value-@c = 0 then 'true' else 'false' end from information_schema.global_status where variable_name='rocksdb_wal_group_syncs'; case when variable_value-@c = 0 then 'true' else 'false' end -false +true SET GLOBAL rocksdb_enable_2pc=1; SET GLOBAL rocksdb_flush_log_at_trx_commit=1; DROP TABLE t1; diff --git a/storage/rocksdb/mysql-test/rocksdb/r/add_index_inplace.result b/storage/rocksdb/mysql-test/rocksdb/r/add_index_inplace.result index f5a2b28ee42..b7b60f26816 100644 --- a/storage/rocksdb/mysql-test/rocksdb/r/add_index_inplace.result +++ b/storage/rocksdb/mysql-test/rocksdb/r/add_index_inplace.result @@ -284,12 +284,16 @@ DROP TABLE t1; set global rocksdb_bulk_load=1; # Establish connection con1 (user=root) # Switch to connection con1 -show global variables like 'rocksdb_bulk_load'; +show global variables like 'rocksdb_bulk_load%'; Variable_name Value rocksdb_bulk_load ON -show session variables like 'rocksdb_bulk_load'; +rocksdb_bulk_load_allow_unsorted OFF +rocksdb_bulk_load_size 1000 +show session variables like 'rocksdb_bulk_load%'; Variable_name Value rocksdb_bulk_load ON +rocksdb_bulk_load_allow_unsorted OFF +rocksdb_bulk_load_size 1000 CREATE TABLE t1 (i INT, j INT, PRIMARY KEY (i)) ENGINE = ROCKSDB; INSERT INTO t1 VALUES (1,1); # Disconnecting on con1 @@ -327,10 +331,11 @@ SET @prior_rocksdb_strict_collation_check= @@rocksdb_strict_collation_check; SET @prior_rocksdb_merge_buf_size = @@rocksdb_merge_buf_size; SET global rocksdb_strict_collation_check = off; SET session rocksdb_merge_combine_read_size = 566; -SET session rocksdb_merge_buf_size = 336; -show variables like '%rocksdb_bulk_load%'; +SET session rocksdb_merge_buf_size = 340; +show variables like 'rocksdb_bulk_load%'; Variable_name Value rocksdb_bulk_load OFF +rocksdb_bulk_load_allow_unsorted OFF rocksdb_bulk_load_size 1000 CREATE TABLE t1 (a VARCHAR(80)) ENGINE=RocksDB; INSERT INTO t1 (a) VALUES (REPEAT("a", 80)); diff --git a/storage/rocksdb/mysql-test/rocksdb/r/bulk_load_errors.result b/storage/rocksdb/mysql-test/rocksdb/r/bulk_load_errors.result index 31562d1da10..be198d02aaf 100644 --- a/storage/rocksdb/mysql-test/rocksdb/r/bulk_load_errors.result +++ b/storage/rocksdb/mysql-test/rocksdb/r/bulk_load_errors.result @@ -16,4 +16,30 @@ INSERT INTO t1 VALUES(20); INSERT INTO t1 VALUES(21); SET rocksdb_bulk_load=0; ERROR HY000: Lost connection to MySQL server during query +TRUNCATE TABLE t1; +SET rocksdb_bulk_load_allow_unsorted=1; +SET rocksdb_bulk_load=1; +INSERT INTO t1 VALUES(100); +INSERT INTO t1 VALUES(101); +INSERT INTO t1 VALUES(99); +SET rocksdb_bulk_load=0; +SELECT * FROM t1; +pk +99 +100 +101 +TRUNCATE TABLE t1; +SET rocksdb_bulk_load=1; +INSERT INTO t1 VALUES(201); +INSERT INTO t1 VALUES(200); +INSERT INTO t1 VALUES(202); +INSERT INTO t1 VALUES(201); +ERROR 23000: Failed to insert the record: the key already exists +SET rocksdb_bulk_load=0; +SELECT * FROM t1; +pk +200 +201 +202 +SET rocksdb_bulk_load_allow_unsorted=DEFAULT; DROP TABLE t1; diff --git a/storage/rocksdb/mysql-test/rocksdb/r/bulk_load_unsorted.result b/storage/rocksdb/mysql-test/rocksdb/r/bulk_load_unsorted.result new file mode 100644 index 00000000000..31509227279 --- /dev/null +++ b/storage/rocksdb/mysql-test/rocksdb/r/bulk_load_unsorted.result @@ -0,0 +1,101 @@ +DROP TABLE IF EXISTS t1; +SET rocksdb_bulk_load_size=3; +SET rocksdb_bulk_load_allow_unsorted=1; +CREATE TABLE t1(a INT, b INT, PRIMARY KEY(a) COMMENT "cf1"); +SET rocksdb_bulk_load=1; +SELECT * FROM t1; +a b +SET rocksdb_bulk_load=0; +SELECT * FROM t1; +a b +-3 5 +-1 3 +2 0 +4 -2 +6 -4 +DROP TABLE t1; +CREATE TABLE t1(a INT, b INT, PRIMARY KEY(a) COMMENT "cf1", KEY(b)); +SET rocksdb_bulk_load=1; +SELECT * FROM t1; +a b +6 -4 +4 -2 +2 0 +-1 3 +-3 5 +SET rocksdb_bulk_load=0; +DROP TABLE t1; +CREATE TABLE t1(a INT, b INT, PRIMARY KEY(a) COMMENT "cf1"); +CREATE TABLE t2(a INT, b INT, PRIMARY KEY(a) COMMENT "cf1"); +SET rocksdb_bulk_load=1; +INSERT INTO t1 VALUES (1,1); +INSERT INTO t2 VALUES (1,1); +SELECT * FROM t1; +a b +1 1 +INSERT INTO t1 VALUES (2,2); +SELECT * FROM t2; +a b +1 1 +SELECT * FROM t1; +a b +1 1 +SET rocksdb_bulk_load=0; +SELECT * FROM t1; +a b +1 1 +2 2 +DROP TABLE t1, t2; +CREATE TABLE t1(a INT, b INT, PRIMARY KEY(a) COMMENT "cf1"); +CREATE TABLE t2(a INT, b INT, PRIMARY KEY(b) COMMENT "cf1"); +CREATE TABLE t3(a INT, b INT, PRIMARY KEY(a) COMMENT "cf1") +PARTITION BY KEY() PARTITIONS 4; +set session transaction isolation level repeatable read; +select * from information_schema.rocksdb_dbstats where stat_type='DB_NUM_SNAPSHOTS'; +STAT_TYPE VALUE +DB_NUM_SNAPSHOTS 0 +start transaction with consistent snapshot; +select * from information_schema.rocksdb_dbstats where stat_type='DB_NUM_SNAPSHOTS'; +STAT_TYPE VALUE +DB_NUM_SNAPSHOTS 1 +set rocksdb_bulk_load=1; +set rocksdb_bulk_load_size=100000; +LOAD DATA INFILE INTO TABLE t1; +LOAD DATA INFILE INTO TABLE t2; +LOAD DATA INFILE INTO TABLE t3; +set rocksdb_bulk_load=0; +SHOW TABLE STATUS WHERE name LIKE 't%'; +Name Engine Version Row_format Rows Avg_row_length Data_length Max_data_length Index_length Data_free Auto_increment Create_time Update_time Check_time Collation Checksum Create_options Comment +t1 ROCKSDB 10 Fixed 5000000 # # # # 0 NULL NULL NULL NULL latin1_swedish_ci NULL +t2 ROCKSDB 10 Fixed 5000000 # # # # 0 NULL NULL NULL NULL latin1_swedish_ci NULL +t3 ROCKSDB 10 Fixed 5000000 # # # # 0 NULL NULL NULL NULL latin1_swedish_ci NULL partitioned +ANALYZE TABLE t1, t2, t3; +Table Op Msg_type Msg_text +test.t1 analyze status OK +test.t2 analyze status OK +test.t3 analyze status OK +SHOW TABLE STATUS WHERE name LIKE 't%'; +Name Engine Version Row_format Rows Avg_row_length Data_length Max_data_length Index_length Data_free Auto_increment Create_time Update_time Check_time Collation Checksum Create_options Comment +t1 ROCKSDB 10 Fixed 5000000 # # # # 0 NULL NULL NULL NULL latin1_swedish_ci NULL +t2 ROCKSDB 10 Fixed 5000000 # # # # 0 NULL NULL NULL NULL latin1_swedish_ci NULL +t3 ROCKSDB 10 Fixed 5000000 # # # # 0 NULL NULL NULL NULL latin1_swedish_ci NULL partitioned +select count(a) from t1; +count(a) +5000000 +select count(b) from t1; +count(b) +5000000 +select count(a) from t2; +count(a) +5000000 +select count(b) from t2; +count(b) +5000000 +select count(a) from t3; +count(a) +5000000 +select count(b) from t3; +count(b) +5000000 +DROP TABLE t1, t2, t3; +SET rocksdb_bulk_load_allow_unsorted=0; diff --git a/storage/rocksdb/mysql-test/rocksdb/r/bulk_load_unsorted_errors.result b/storage/rocksdb/mysql-test/rocksdb/r/bulk_load_unsorted_errors.result new file mode 100644 index 00000000000..d8e5b92e897 --- /dev/null +++ b/storage/rocksdb/mysql-test/rocksdb/r/bulk_load_unsorted_errors.result @@ -0,0 +1,4 @@ +SET rocksdb_bulk_load=1; +SET rocksdb_bulk_load_allow_unsorted=1; +ERROR HY000: Error when executing command SET: Cannot change this setting while bulk load is enabled +SET rocksdb_bulk_load=0; diff --git a/storage/rocksdb/mysql-test/rocksdb/r/bytes_written.result b/storage/rocksdb/mysql-test/rocksdb/r/bytes_written.result new file mode 100644 index 00000000000..d9d29e6ac69 --- /dev/null +++ b/storage/rocksdb/mysql-test/rocksdb/r/bytes_written.result @@ -0,0 +1,10 @@ +DROP TABLE IF EXISTS stats_test_table; +CREATE TABLE stats_test_table (a INT, b INT, PRIMARY KEY (a)) ENGINE=ROCKSDB; +SET GLOBAL rocksdb_perf_context_level=3; +INSERT INTO stats_test_table VALUES (7,1); +INSERT INTO stats_test_table VALUES (2,2); +SELECT io_write_bytes > 0 FROM INFORMATION_SCHEMA.TABLE_STATISTICS WHERE TABLE_NAME = "stats_test_table"; +io_write_bytes > 0 +1 +DROP TABLE stats_test_table; +SET GLOBAL rocksdb_perf_context_level=DEFAULT; diff --git a/storage/rocksdb/mysql-test/rocksdb/r/cons_snapshot_read_committed.result b/storage/rocksdb/mysql-test/rocksdb/r/cons_snapshot_read_committed.result index 521edec0c83..7f1e3d8e53f 100644 --- a/storage/rocksdb/mysql-test/rocksdb/r/cons_snapshot_read_committed.result +++ b/storage/rocksdb/mysql-test/rocksdb/r/cons_snapshot_read_committed.result @@ -5,7 +5,7 @@ connection con1; CREATE TABLE t1 (a INT, pk INT AUTO_INCREMENT PRIMARY KEY) ENGINE=ROCKSDB; SET SESSION TRANSACTION ISOLATION LEVEL READ COMMITTED; START TRANSACTION WITH CONSISTENT SNAPSHOT; -ERROR: 1938 +ERROR: 12048 connection con2; select * from information_schema.rocksdb_dbstats where stat_type='DB_NUM_SNAPSHOTS'; STAT_TYPE VALUE @@ -18,7 +18,7 @@ STAT_TYPE VALUE DB_NUM_SNAPSHOTS 0 connection con1; START TRANSACTION WITH CONSISTENT SNAPSHOT; -ERROR: 1938 +ERROR: 12048 connection con2; INSERT INTO t1 (a) VALUES (1); connection con1; @@ -69,7 +69,7 @@ id value value2 5 5 5 6 6 6 START TRANSACTION WITH CONSISTENT SNAPSHOT; -ERROR: 1938 +ERROR: 12048 connection con2; INSERT INTO r1 values (7,7,7); connection con1; @@ -107,12 +107,12 @@ id value value2 7 7 7 8 8 8 START TRANSACTION WITH CONSISTENT SNAPSHOT; -ERROR: 1938 +ERROR: 12048 connection con2; INSERT INTO r1 values (9,9,9); connection con1; START TRANSACTION WITH CONSISTENT SNAPSHOT; -ERROR: 1938 +ERROR: 12048 connection con2; INSERT INTO r1 values (10,10,10); connection con1; @@ -129,7 +129,7 @@ id value value2 9 9 9 10 10 10 START TRANSACTION WITH CONSISTENT SNAPSHOT; -ERROR: 1938 +ERROR: 12048 INSERT INTO r1 values (11,11,11); ERROR: 0 SELECT * FROM r1; diff --git a/storage/rocksdb/mysql-test/rocksdb/r/cons_snapshot_repeatable_read.result b/storage/rocksdb/mysql-test/rocksdb/r/cons_snapshot_repeatable_read.result index 805a0aaa0fd..90723ff762c 100644 --- a/storage/rocksdb/mysql-test/rocksdb/r/cons_snapshot_repeatable_read.result +++ b/storage/rocksdb/mysql-test/rocksdb/r/cons_snapshot_repeatable_read.result @@ -125,7 +125,7 @@ id value value2 START TRANSACTION WITH CONSISTENT SNAPSHOT; ERROR: 0 INSERT INTO r1 values (11,11,11); -ERROR: 1935 +ERROR: 12045 SELECT * FROM r1; id value value2 1 1 1 diff --git a/storage/rocksdb/mysql-test/rocksdb/r/covered_unpack_info_format.result b/storage/rocksdb/mysql-test/rocksdb/r/covered_unpack_info_format.result new file mode 100644 index 00000000000..195215331b8 --- /dev/null +++ b/storage/rocksdb/mysql-test/rocksdb/r/covered_unpack_info_format.result @@ -0,0 +1,73 @@ +DROP TABLE IF EXISTS t1; +CREATE TABLE t1 ( +id INT, +fake_id INT, +bigfield VARCHAR(4096), +PRIMARY KEY (id), +KEY bf (bigfield(32)), +KEY fid (fake_id, bigfield(32)) +) ENGINE=rocksdb; +INSERT INTO t1 VALUES (1, 1001, REPEAT('a', 1)), +(8, 1008, REPEAT('b', 8)), +(24, 1024, REPEAT('c', 24)), +(31, 1031, REPEAT('d', 31)), +(32, 1032, REPEAT('x', 32)), +(33, 1033, REPEAT('y', 33)), +(128, 1128, REPEAT('z', 128)); +SELECT * FROM t1; +id fake_id bigfield +1 1001 a +8 1008 bbbbbbbb +24 1024 cccccccccccccccccccccccc +31 1031 ddddddddddddddddddddddddddddddd +32 1032 xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +33 1033 yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy +128 1128 zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz +# Eligible for optimization, access via fake_id only +SELECT id, bigfield FROM t1 FORCE INDEX(fid) WHERE fake_id = 1031; +id bigfield +31 ddddddddddddddddddddddddddddddd +include/assert.inc [Eligible for optimization, access via fake_id only: 2 rocksdb_covered_secondary_key_lookups] +# Not eligible for optimization, access via fake_id of big row. +SELECT id, bigfield FROM t1 FORCE INDEX(fid) WHERE fake_id = 1033; +id bigfield +33 yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy +include/assert.inc [Not eligible for optimization, access via fake_id of big row.: 0 rocksdb_covered_secondary_key_lookups] +DROP TABLE t1; +set session debug= '+d,MYROCKS_NO_COVERED_BITMAP_FORMAT'; +CREATE TABLE t1 ( +id INT, +fake_id INT, +bigfield VARCHAR(4096), +PRIMARY KEY (id), +KEY bf (bigfield(32)), +KEY fid (fake_id, bigfield(32)) +) ENGINE=rocksdb; +set session debug= '-d,MYROCKS_NO_COVERED_BITMAP_FORMAT'; +INSERT INTO t1 VALUES (1, 1001, REPEAT('a', 1)), +(8, 1008, REPEAT('b', 8)), +(24, 1024, REPEAT('c', 24)), +(31, 1031, REPEAT('d', 31)), +(32, 1032, REPEAT('x', 32)), +(33, 1033, REPEAT('y', 33)), +(128, 1128, REPEAT('z', 128)); +SELECT * FROM t1; +id fake_id bigfield +1 1001 a +8 1008 bbbbbbbb +24 1024 cccccccccccccccccccccccc +31 1031 ddddddddddddddddddddddddddddddd +32 1032 xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +33 1033 yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy +128 1128 zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz +# No longer eligible for optimization since no covered bitmap was stored. +SELECT id, bigfield FROM t1 FORCE INDEX(fid) WHERE fake_id = 1031; +id bigfield +31 ddddddddddddddddddddddddddddddd +include/assert.inc [No longer eligible for optimization since no covered bitmap was stored.: 0 rocksdb_covered_secondary_key_lookups] +# Not eligible for optimization. +SELECT id, bigfield FROM t1 FORCE INDEX(fid) WHERE fake_id = 1033; +id bigfield +33 yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy +include/assert.inc [Not eligible for optimization.: 0 rocksdb_covered_secondary_key_lookups] +DROP TABLE t1; diff --git a/storage/rocksdb/mysql-test/rocksdb/r/ddl_high_priority.result b/storage/rocksdb/mysql-test/rocksdb/r/ddl_high_priority.result new file mode 100644 index 00000000000..39130475349 --- /dev/null +++ b/storage/rocksdb/mysql-test/rocksdb/r/ddl_high_priority.result @@ -0,0 +1,1009 @@ +## +## Using the system variable high_priority_ddl" +## +create user test_user1@localhost; +grant all on test to test_user1@localhost; +create user test_user2@localhost; +grant all on test to test_user2@localhost; + +## Test parameters: +## use_sys_var = 1; +## con_block = con1 +## con_kill = default +## cmd = alter table t1 modify i bigint; +## high_priority_cmd = alter high_priority table t1 modify i bigint; +## should_kill = 1 +## recreate_table = 1 +## throw_error = 1 + +drop table if exists t1; +create table t1 (i int); +show create table t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `i` int(11) DEFAULT NULL +) ENGINE=ROCKSDB DEFAULT CHARSET=latin1 +insert into t1 values (1), (2), (3); +connection: con1 +lock tables t1 read;; +connection: default +set lock_wait_timeout = 0.02; +set high_priority_lock_wait_timeout = 0.02; +describe t1; +Field Type Null Key Default Extra +i int(11) YES NULL +connection: default (for show processlist) +# both con1 and default exist +show processlist; +Id User Host db Command Time State Info Rows examined Rows sent Tid Srv_Id + root test