From d29e7f747209f428458a97c128678a21abf2680d Mon Sep 17 00:00:00 2001 From: Michael Widenius Date: Sat, 28 Jun 2008 15:45:15 +0300 Subject: [PATCH] Fix for Bug #37007 Maria: different checksum for MyISAM table depending on CHECKSUM=0|1 This also adds a check that MyISAM tables with incompatible checksums are detected by CHECK TABLE ... [FOR UPGRADE] and thus also by mysql_upgrade. The tables that are incomatible are MyISAM tables with ROW_FORMAT=fixed and has VARCHAR fields and have CHECKSUM enabled. Before these tables gave different checksum if you used CHECK TABLE with or without EXTENDED mysql-test/r/old-mode.result: Now we get same results with and without EXTENDED mysql-test/r/row-checksum-old.result: Initial results mysql-test/r/row-checksum.result: Initial results mysql-test/t/old-mode.test: Added test with QUICK to show that the live checksum is not used when running with --old mysql-test/t/row-checksum-old-master.opt: Start mysqld with --old mode to enable old checksum code mysql-test/t/row-checksum-old.test: Run row-checksum test under mysqld --old mysql-test/t/row-checksum.test: Verify that checksum are calculated the same way with and without EXTENDED We run this with several storage engines to ensure results are the same over storage engines sql/ha_partition.cc: Use new HA_HAS_xxx_CHECKSUM flags sql/handler.cc: Use new HA_HAS_xxx_CHECKSUM flags sql/handler.h: Split HA_HAS_CHECKSUM into HA_HAS_NEW_CHECKSUM and HA_HAS_OLD_CHECKSUM flags. This is a safe API change as only MyISAM and Maria should use these handler flags. sql/sql_show.cc: Use new HA_HAS_xxx_CHECKSUM flags sql/sql_table.cc: Use file->checksum() for live checksums if the life checksum method corresponds to the mysqld --old flag storage/maria/ha_maria.cc: Use new HA_HAS_xxx_CHECKSUM flags storage/myisam/ha_myisam.cc: Set HA_HAS_OLD_CHECKSUM and/or HA_HAS_NEW_CHECKSUM flags depending on if this is a new myisam or old myisam file Add method check_for_upgrade() to detect if the table is of old version with a checksum that is incompatible with CHECK TABLE ... EXTENDED storage/myisam/ha_myisam.h: Added check_for_upgrade() storage/myisam/mi_open.c: Removed not neede cast Initialize share->has_null_fields and share->has_varchar_fields variables storage/myisam/myisamdef.h: Added share->has_null_fields and share->has_varchar_fields --- mysql-test/r/old-mode.result | 6 +- mysql-test/r/row-checksum-old.result | 85 ++++++++++++++++++++++++ mysql-test/r/row-checksum.result | 85 ++++++++++++++++++++++++ mysql-test/t/old-mode.test | 1 + mysql-test/t/row-checksum-old-master.opt | 1 + mysql-test/t/row-checksum-old.test | 4 ++ mysql-test/t/row-checksum.test | 62 +++++++++++++++++ sql/ha_partition.cc | 2 +- sql/handler.cc | 2 +- sql/handler.h | 5 +- sql/sql_show.cc | 4 +- sql/sql_table.cc | 11 +-- storage/maria/ha_maria.cc | 2 +- storage/myisam/ha_myisam.cc | 35 +++++++++- storage/myisam/ha_myisam.h | 1 + storage/myisam/mi_open.c | 10 ++- storage/myisam/myisamdef.h | 3 + 17 files changed, 306 insertions(+), 13 deletions(-) create mode 100644 mysql-test/r/row-checksum-old.result create mode 100644 mysql-test/r/row-checksum.result create mode 100644 mysql-test/t/row-checksum-old-master.opt create mode 100644 mysql-test/t/row-checksum-old.test create mode 100644 mysql-test/t/row-checksum.test diff --git a/mysql-test/r/old-mode.result b/mysql-test/r/old-mode.result index df2c0b6fee0..a9815d7dab2 100644 --- a/mysql-test/r/old-mode.result +++ b/mysql-test/r/old-mode.result @@ -5,8 +5,12 @@ insert t1 values (1, "aaa", "bbb"), (NULL, "", "ccccc"), (0, NULL, ""); insert t2 select * from t1; checksum table t1, t2; Table Checksum -test.t1 3442722830 +test.t1 2948697075 test.t2 2948697075 +checksum table t1, t2 quick; +Table Checksum +test.t1 NULL +test.t2 NULL checksum table t1, t2 extended; Table Checksum test.t1 2948697075 diff --git a/mysql-test/r/row-checksum-old.result b/mysql-test/r/row-checksum-old.result new file mode 100644 index 00000000000..3cf5a7104b9 --- /dev/null +++ b/mysql-test/r/row-checksum-old.result @@ -0,0 +1,85 @@ +drop table if exists t1; +create table t1 (a int null, v varchar(100)) engine=myisam checksum=0; +insert into t1 values(null, null), (1, "hello"); +checksum table t1; +Table Checksum +test.t1 452555338 +checksum table t1 quick; +Table Checksum +test.t1 NULL +checksum table t1 extended; +Table Checksum +test.t1 452555338 +drop table if exists t1; +create table t1 (a int null, v varchar(100)) engine=myisam checksum=1; +insert into t1 values(null, null), (1, "hello"); +checksum table t1; +Table Checksum +test.t1 452555338 +checksum table t1 quick; +Table Checksum +test.t1 NULL +checksum table t1 extended; +Table Checksum +test.t1 452555338 +drop table if exists t1; +create table t1 (a int null, v varchar(100)) engine=innodb checksum=0; +insert into t1 values(null, null), (1, "hello"); +checksum table t1; +Table Checksum +test.t1 452555338 +checksum table t1 quick; +Table Checksum +test.t1 NULL +checksum table t1 extended; +Table Checksum +test.t1 452555338 +drop table t1; +create table t1 (a int null, v varchar(100)) engine=maria checksum=0; +insert into t1 values(null, null), (1, "hello"); +checksum table t1; +Table Checksum +test.t1 452555338 +checksum table t1 quick; +Table Checksum +test.t1 NULL +checksum table t1 extended; +Table Checksum +test.t1 452555338 +drop table t1; +create table t1 (a int null, v varchar(100)) engine=maria checksum=1; +insert into t1 values(null, null), (1, "hello"); +checksum table t1; +Table Checksum +test.t1 452555338 +checksum table t1 quick; +Table Checksum +test.t1 NULL +checksum table t1 extended; +Table Checksum +test.t1 452555338 +drop table t1; +create table t1 (a int null, v varchar(100)) engine=myisam checksum=1 row_format=fixed; +insert into t1 values(null, null), (1, "hello"); +checksum table t1; +Table Checksum +test.t1 4108368782 +checksum table t1 quick; +Table Checksum +test.t1 NULL +checksum table t1 extended; +Table Checksum +test.t1 4108368782 +drop table if exists t1; +create table t1 (a int null, v varchar(100)) engine=innodb checksum=0 row_format=fixed; +insert into t1 values(null, null), (1, "hello"); +checksum table t1; +Table Checksum +test.t1 4108368782 +checksum table t1 quick; +Table Checksum +test.t1 NULL +checksum table t1 extended; +Table Checksum +test.t1 4108368782 +drop table t1; diff --git a/mysql-test/r/row-checksum.result b/mysql-test/r/row-checksum.result new file mode 100644 index 00000000000..31ae094859b --- /dev/null +++ b/mysql-test/r/row-checksum.result @@ -0,0 +1,85 @@ +drop table if exists t1; +create table t1 (a int null, v varchar(100)) engine=myisam checksum=0; +insert into t1 values(null, null), (1, "hello"); +checksum table t1; +Table Checksum +test.t1 229851577 +checksum table t1 quick; +Table Checksum +test.t1 NULL +checksum table t1 extended; +Table Checksum +test.t1 229851577 +drop table if exists t1; +create table t1 (a int null, v varchar(100)) engine=myisam checksum=1; +insert into t1 values(null, null), (1, "hello"); +checksum table t1; +Table Checksum +test.t1 229851577 +checksum table t1 quick; +Table Checksum +test.t1 229851577 +checksum table t1 extended; +Table Checksum +test.t1 229851577 +drop table if exists t1; +create table t1 (a int null, v varchar(100)) engine=innodb checksum=0; +insert into t1 values(null, null), (1, "hello"); +checksum table t1; +Table Checksum +test.t1 229851577 +checksum table t1 quick; +Table Checksum +test.t1 NULL +checksum table t1 extended; +Table Checksum +test.t1 229851577 +drop table t1; +create table t1 (a int null, v varchar(100)) engine=maria checksum=0; +insert into t1 values(null, null), (1, "hello"); +checksum table t1; +Table Checksum +test.t1 229851577 +checksum table t1 quick; +Table Checksum +test.t1 NULL +checksum table t1 extended; +Table Checksum +test.t1 229851577 +drop table t1; +create table t1 (a int null, v varchar(100)) engine=maria checksum=1; +insert into t1 values(null, null), (1, "hello"); +checksum table t1; +Table Checksum +test.t1 229851577 +checksum table t1 quick; +Table Checksum +test.t1 229851577 +checksum table t1 extended; +Table Checksum +test.t1 229851577 +drop table t1; +create table t1 (a int null, v varchar(100)) engine=myisam checksum=1 row_format=fixed; +insert into t1 values(null, null), (1, "hello"); +checksum table t1; +Table Checksum +test.t1 3885665021 +checksum table t1 quick; +Table Checksum +test.t1 3885665021 +checksum table t1 extended; +Table Checksum +test.t1 3885665021 +drop table if exists t1; +create table t1 (a int null, v varchar(100)) engine=innodb checksum=0 row_format=fixed; +insert into t1 values(null, null), (1, "hello"); +checksum table t1; +Table Checksum +test.t1 3885665021 +checksum table t1 quick; +Table Checksum +test.t1 NULL +checksum table t1 extended; +Table Checksum +test.t1 3885665021 +drop table t1; diff --git a/mysql-test/t/old-mode.test b/mysql-test/t/old-mode.test index 4fa21f761ca..6d0fe64bbb8 100644 --- a/mysql-test/t/old-mode.test +++ b/mysql-test/t/old-mode.test @@ -12,5 +12,6 @@ create table t2 (a int, b varchar(200), c text not null) checksum=0; insert t1 values (1, "aaa", "bbb"), (NULL, "", "ccccc"), (0, NULL, ""); insert t2 select * from t1; checksum table t1, t2; +checksum table t1, t2 quick; checksum table t1, t2 extended; drop table t1,t2; diff --git a/mysql-test/t/row-checksum-old-master.opt b/mysql-test/t/row-checksum-old-master.opt new file mode 100644 index 00000000000..8e7b7f9e36f --- /dev/null +++ b/mysql-test/t/row-checksum-old-master.opt @@ -0,0 +1 @@ +--old diff --git a/mysql-test/t/row-checksum-old.test b/mysql-test/t/row-checksum-old.test new file mode 100644 index 00000000000..71188ddf432 --- /dev/null +++ b/mysql-test/t/row-checksum-old.test @@ -0,0 +1,4 @@ +# +# Run row-checksum.test with old mode +# +--source t/row-checksum.test diff --git a/mysql-test/t/row-checksum.test b/mysql-test/t/row-checksum.test new file mode 100644 index 00000000000..920a2384aa8 --- /dev/null +++ b/mysql-test/t/row-checksum.test @@ -0,0 +1,62 @@ +# +# Test checksum +# + +-- source include/have_innodb.inc +-- source include/have_maria.inc + +--disable_warnings +drop table if exists t1; +--enable_warnings + +create table t1 (a int null, v varchar(100)) engine=myisam checksum=0; +insert into t1 values(null, null), (1, "hello"); +checksum table t1; +checksum table t1 quick; +checksum table t1 extended; +drop table if exists t1; +create table t1 (a int null, v varchar(100)) engine=myisam checksum=1; +insert into t1 values(null, null), (1, "hello"); +checksum table t1; +checksum table t1 quick; +checksum table t1 extended; +drop table if exists t1; + +create table t1 (a int null, v varchar(100)) engine=innodb checksum=0; +insert into t1 values(null, null), (1, "hello"); +checksum table t1; +checksum table t1 quick; +checksum table t1 extended; +drop table t1; + +create table t1 (a int null, v varchar(100)) engine=maria checksum=0; +insert into t1 values(null, null), (1, "hello"); +checksum table t1; +checksum table t1 quick; +checksum table t1 extended; +drop table t1; +create table t1 (a int null, v varchar(100)) engine=maria checksum=1; +insert into t1 values(null, null), (1, "hello"); +checksum table t1; +checksum table t1 quick; +checksum table t1 extended; +drop table t1; + + +# +# These checksums will be different prefixes fixed sizes rows with one extra +# flag byte +# +create table t1 (a int null, v varchar(100)) engine=myisam checksum=1 row_format=fixed; +insert into t1 values(null, null), (1, "hello"); +checksum table t1; +checksum table t1 quick; +checksum table t1 extended; +drop table if exists t1; + +create table t1 (a int null, v varchar(100)) engine=innodb checksum=0 row_format=fixed; +insert into t1 values(null, null), (1, "hello"); +checksum table t1; +checksum table t1 quick; +checksum table t1 extended; +drop table t1; diff --git a/sql/ha_partition.cc b/sql/ha_partition.cc index c90f82d24a1..b239253fbc7 100644 --- a/sql/ha_partition.cc +++ b/sql/ha_partition.cc @@ -4615,7 +4615,7 @@ void ha_partition::get_dynamic_partition_info(PARTITION_INFO *stat_info, stat_info->update_time= file->stats.update_time; stat_info->check_time= file->stats.check_time; stat_info->check_sum= 0; - if (file->ha_table_flags() & HA_HAS_CHECKSUM) + if (file->ha_table_flags() & (HA_HAS_OLD_CHECKSUM | HA_HAS_NEW_CHECKSUM)) stat_info->check_sum= file->checksum(); return; } diff --git a/sql/handler.cc b/sql/handler.cc index f637ebfb02e..68921eb2a3f 100644 --- a/sql/handler.cc +++ b/sql/handler.cc @@ -3435,7 +3435,7 @@ void handler::get_dynamic_partition_info(PARTITION_INFO *stat_info, stat_info->update_time= stats.update_time; stat_info->check_time= stats.check_time; stat_info->check_sum= 0; - if (table_flags() & (ulong) HA_HAS_CHECKSUM) + if (table_flags() & (HA_HAS_OLD_CHECKSUM | HA_HAS_OLD_CHECKSUM)) stat_info->check_sum= checksum(); return; } diff --git a/sql/handler.h b/sql/handler.h index b2234dc4b75..12ba5daf1e1 100644 --- a/sql/handler.h +++ b/sql/handler.h @@ -107,7 +107,8 @@ #define HA_CAN_FULLTEXT (1 << 21) #define HA_CAN_SQL_HANDLER (1 << 22) #define HA_NO_AUTO_INCREMENT (1 << 23) -#define HA_HAS_CHECKSUM (1 << 24) +/* Has automatic checksums and uses the old checksum format */ +#define HA_HAS_OLD_CHECKSUM (1 << 24) /* Table data are stored in separate files (for lower_case_table_names) */ #define HA_FILE_BASED (1 << 26) #define HA_NO_VARCHAR (1 << 27) @@ -124,6 +125,8 @@ */ #define HA_BINLOG_ROW_CAPABLE (LL(1) << 34) #define HA_BINLOG_STMT_CAPABLE (LL(1) << 35) +/* Has automatic checksums and uses the new checksum format */ +#define HA_HAS_NEW_CHECKSUM (LL(1) << 36) /* Set of all binlog flags. Currently only contain the capabilities diff --git a/sql/sql_show.cc b/sql/sql_show.cc index e0baa986272..bde1274631a 100644 --- a/sql/sql_show.cc +++ b/sql/sql_show.cc @@ -3613,7 +3613,7 @@ static int get_schema_tables_record(THD *thd, TABLE_LIST *tables, table->field[16]->store_time(&time, MYSQL_TIMESTAMP_DATETIME); table->field[16]->set_notnull(); } - if (file->ha_table_flags() & (ulong) HA_HAS_CHECKSUM) + if (file->ha_table_flags() & (HA_HAS_OLD_CHECKSUM | HA_HAS_NEW_CHECKSUM)) { table->field[18]->store((longlong) file->checksum(), TRUE); table->field[18]->set_notnull(); @@ -4670,7 +4670,7 @@ static void store_schema_partitions_record(THD *thd, TABLE *schema_table, table->field[20]->store_time(&time, MYSQL_TIMESTAMP_DATETIME); table->field[20]->set_notnull(); } - if (file->ha_table_flags() & (ulong) HA_HAS_CHECKSUM) + if (file->ha_table_flags() & (HA_HAS_OLD_CHECKSUM | HA_HAS_NEW_CHECKSUM)) { table->field[21]->store((longlong) stat_info.check_sum, TRUE); table->field[21]->set_notnull(); diff --git a/sql/sql_table.cc b/sql/sql_table.cc index 5ee2ccb5a64..d64b621a7db 100644 --- a/sql/sql_table.cc +++ b/sql/sql_table.cc @@ -7287,11 +7287,14 @@ bool mysql_checksum_table(THD *thd, TABLE_LIST *tables, } else { - if (t->file->ha_table_flags() & HA_HAS_CHECKSUM && - !(check_opt->flags & T_EXTEND)) + /* Call ->checksum() if the table checksum matches 'old_mode' settings */ + if (!(check_opt->flags & T_EXTEND) && + (((t->file->ha_table_flags() & HA_HAS_OLD_CHECKSUM) && + thd->variables.old_mode) || + ((t->file->ha_table_flags() & HA_HAS_NEW_CHECKSUM) && + !thd->variables.old_mode))) protocol->store((ulonglong)t->file->checksum()); - else if (!(t->file->ha_table_flags() & HA_HAS_CHECKSUM) && - (check_opt->flags & T_QUICK)) + else if (check_opt->flags & T_QUICK) protocol->store_null(); else { diff --git a/storage/maria/ha_maria.cc b/storage/maria/ha_maria.cc index 420cc953529..34199fd085e 100644 --- a/storage/maria/ha_maria.cc +++ b/storage/maria/ha_maria.cc @@ -923,7 +923,7 @@ int ha_maria::open(const char *name, int mode, uint test_if_locked) int_table_flags|= HA_CAN_INSERT_DELAYED; } if (file->s->options & (HA_OPTION_CHECKSUM | HA_OPTION_COMPRESS_RECORD)) - int_table_flags |= HA_HAS_CHECKSUM; + int_table_flags |= HA_HAS_NEW_CHECKSUM; for (i= 0; i < table->s->keys; i++) { diff --git a/storage/myisam/ha_myisam.cc b/storage/myisam/ha_myisam.cc index ee453d34b75..8e755567a5b 100644 --- a/storage/myisam/ha_myisam.cc +++ b/storage/myisam/ha_myisam.cc @@ -690,7 +690,19 @@ int ha_myisam::open(const char *name, int mode, uint test_if_locked) if (!table->s->db_record_offset) int_table_flags|=HA_REC_NOT_IN_SEQ; if (file->s->options & (HA_OPTION_CHECKSUM | HA_OPTION_COMPRESS_RECORD)) - int_table_flags|=HA_HAS_CHECKSUM; + { + /* + Set which type of automatic checksum we have + The old checksum and new checksum are identical if there is no + null fields. + Files with new checksum has the HA_OPTION_NULL_FIELDS bit set. + */ + if ((file->s->options & HA_OPTION_NULL_FIELDS) || + !file->s->has_null_fields) + int_table_flags|= HA_HAS_NEW_CHECKSUM; + if (!(file->s->options & HA_OPTION_NULL_FIELDS)) + int_table_flags|= HA_HAS_OLD_CHECKSUM; + } for (i= 0; i < table->s->keys; i++) { @@ -2042,6 +2054,27 @@ bool ha_myisam::check_if_incompatible_data(HA_CREATE_INFO *info, return COMPATIBLE_DATA_YES; } + +/** + Check if a table is incompatible with the current version. + + The cases are: + - Table has checksum, varchars and are not of dynamic record type +*/ + +int ha_myisam::check_for_upgrade(HA_CHECK_OPT *check_opt) +{ + if (!(file->s->options & HA_OPTION_NULL_FIELDS) && + !(file->s->options & HA_OPTION_PACK_RECORD) && + file->s->has_varchar_fields) + { + /* We need alter there to get the HA_OPTION_NULL_FIELDS flag to be set */ + return HA_ADMIN_NEEDS_ALTER; + } + return HA_ADMIN_OK; +} + + extern int mi_panic(enum ha_panic_function flag); int myisam_panic(handlerton *hton, ha_panic_function flag) { diff --git a/storage/myisam/ha_myisam.h b/storage/myisam/ha_myisam.h index 606d5ca354f..62792a10a20 100644 --- a/storage/myisam/ha_myisam.h +++ b/storage/myisam/ha_myisam.h @@ -119,6 +119,7 @@ class ha_myisam: public handler ulonglong *nb_reserved_values); int rename_table(const char * from, const char * to); int delete_table(const char *name); + int check_for_upgrade(HA_CHECK_OPT *check_opt); int check(THD* thd, HA_CHECK_OPT* check_opt); int analyze(THD* thd,HA_CHECK_OPT* check_opt); int repair(THD* thd, HA_CHECK_OPT* check_opt); diff --git a/storage/myisam/mi_open.c b/storage/myisam/mi_open.c index ec8fb7ed1d9..1537633bd9d 100644 --- a/storage/myisam/mi_open.c +++ b/storage/myisam/mi_open.c @@ -19,6 +19,7 @@ #include "sp_defs.h" #include "rt_index.h" #include +#include #if defined(MSDOS) || defined(__WIN__) #ifdef __WIN__ @@ -453,13 +454,20 @@ MI_INFO *mi_open(const char *name, int mode, uint open_flags) share->rec[i].pack_type=0; share->rec[i].huff_tree=0; share->rec[i].offset=offset; - if (share->rec[i].type == (int) FIELD_BLOB) + if (share->rec[i].type == FIELD_BLOB) { share->blobs[j].pack_length= share->rec[i].length - portable_sizeof_char_ptr; share->blobs[j].offset=offset; j++; } +#if MYSQL_VERSION_ID <= 60100 + /* This is to detect old checksum option */ + if (share->rec[i].null_bit) + share->has_null_fields= 1; + if (share->rec[i].type == FIELD_VARCHAR) + share->has_varchar_fields= 1; +#endif offset+=share->rec[i].length; } share->rec[i].type=(int) FIELD_LAST; /* End marker */ diff --git a/storage/myisam/myisamdef.h b/storage/myisam/myisamdef.h index db6344e14f8..e432708dba2 100644 --- a/storage/myisam/myisamdef.h +++ b/storage/myisam/myisamdef.h @@ -210,6 +210,9 @@ typedef struct st_mi_isam_share enum data_file_type data_file_type; /* Below flag is needed to make log tables work with concurrent insert */ my_bool is_log_table; + /* This is 1 if they table checksum is of old type */ + my_bool has_null_fields; + my_bool has_varchar_fields; my_bool changed, /* If changed since lock */ global_changed, /* If changed since open */