MDEV-15563: Instant ROW_FORMAT=REDUNDANT column extension

This was developed by Aleksey Midenkov based on my design. In the original InnoDB storage format (that was retroactively named ROW_FORMAT=REDUNDANT in MySQL 5.0.3), the length of each index field is stored explicitly. Because of this, we can and now will allow instant conversion from VARCHAR to CHAR or VARBINARY to BINARY of equal or greater size, as well as instant conversion of TINYINT to SMALLINT to MEDIUMINT to INT to BIGINT (while not changing between signed and unsigned). Theoretically, we could allow changing from an unsigned integer to a bigger unsigned integer, as well as changing CHAR to VARCHAR, but that would require additional metadata and conversions whenever reading old records. Field_str::is_equal(), Field_varstring::is_equal(), Field_num::is_equal(): Return the new result IS_EQUAL_PACK_LENGTH_EXT if the table advertises HA_EXTENDED_TYPES_CONVERSION capability and we are considering the above-mentioned conversions. ALTER_COLUMN_EQUAL_PACK_LENGTH_EXT: A new ALTER TABLE flag, similar to ALTER_COLUMN_EQUAL_PACK_LENGTH but requiring conversions when reading the data. The Field::is_equal() result IS_EQUAL_PACK_LENGTH_EXT will map to this flag. dtype_get_fixed_size_low(): For BINARY, CHAR and integer columns in ROW_FORMAT=REDUNDANT, return 0 (variable length) from now on. dtype_get_sql_null_size(): Keep returning the current size for BINARY, CHAR and integer columns, so that in ROW_FORMAT=REDUNDANT it will remain possible to update in place between NULL and NOT NULL values. btr_index_rec_validate(): Relax a CHECK TABLE length check for ROW_FORMAT=REDUNDANT tables. btr_cur_instant_init_low(): No longer trust fixed_len for ROW_FORMAT=REDUNDANT tables. We cannot rely on fixed_len anymore because the record can have shorter length from before instant extension. Note that importing such tablespace into earlier MariaDB versions produces ER_TABLE_SCHEMA_MISMATCH when using a .cfg file.
2025-03-23 23:48:41 +01:00 · 2019-02-13 17:39:05 +02:00 · 2019-02-13 17:39:05 +02:00 · 22feb179ae
commit 22feb179ae
parent 0ae3ea7919
19 changed files with 697 additions and 71 deletions
--- a/mysql-test/suite/innodb/r/instant_alter_extend,utf8.rdiff
+++ b/mysql-test/suite/innodb/r/instant_alter_extend,utf8.rdiff
@ -0,0 +1,29 @@
+--- instant_alter_convert.result
+++ instant_alter_convert,utf8.result
+@@ -37,7 +37,7 @@
+ test.t	check	status	OK
+ call check_table('t');
+ name	mtype	prtype	len
+-a	2	800FE	200
+a	13	2100FE	600
+ # CHAR enlargement
+ alter table t modify a char(220), algorithm=instant;
+ select count(a) from t where a = @bigval;
+@@ -51,7 +51,7 @@
+ test.t	check	status	OK
+ call check_table('t');
+ name	mtype	prtype	len
+-a	2	800FE	200
+a	13	2100FE	600
+ # Convert from VARCHAR to a bigger CHAR
+ alter table t modify a varchar(200), algorithm=instant;
+ ERROR 0A000: ALGORITHM=INSTANT is not supported. Reason: Cannot change column type INPLACE. Try ALGORITHM=COPY
+@@ -72,7 +72,7 @@
+ test.t	check	status	OK
+ call check_table('t');
+ name	mtype	prtype	len
+-a	2	800FE	255
+a	13	2100FE	765
+ # BINARY/VARBINARY test
+ create or replace table t (a varbinary(300));
+ alter table t modify a binary(255), algorithm=instant;
--- a/mysql-test/suite/innodb/r/instant_alter_extend.result
+++ b/mysql-test/suite/innodb/r/instant_alter_extend.result
@ -0,0 +1,235 @@
+#
+# MDEV-15563: Instant ROW_FORMAT=REDUNDANT column type change&extension
+#
+create or replace database test;
+use test;
+set default_storage_engine=innodb;
+set @save_format= @@GLOBAL.innodb_default_row_format;
+SET GLOBAL innodb_default_row_format=redundant;
+set @bigval= repeat('0123456789', 30);
+create or replace procedure check_table(table_name varchar(255))
+begin
+select table_id into @table_id
+from information_schema.innodb_sys_tables
+where name = concat('test/', table_name);
+select name, mtype, hex(prtype) as prtype, len
+from information_schema.innodb_sys_columns
+where table_id = @table_id;
+end~~
+# VARCHAR -> CHAR, VARBINARY -> BINARY conversion
+set @bigval= repeat('0123456789', 20);
+create or replace table t (a varchar(300));
+alter table t modify a char(255), algorithm=instant;
+ERROR 0A000: ALGORITHM=INSTANT is not supported. Reason: Cannot change column type INPLACE. Try ALGORITHM=COPY
+alter table t modify a char(255), algorithm=copy;
+create or replace table t (a varchar(200));
+insert into t values (@bigval);
+insert into t values ('z');
+alter table t modify a char(200), algorithm=instant;
+select count(a) from t where a = @bigval;
+count(a)
+1
+select a, length(a) from t where a = 'z';
+a	length(a)
+z	1
+check table t extended;
+Table	Op	Msg_type	Msg_text
+test.t	check	status	OK
+call check_table('t');
+name	mtype	prtype	len
+a	2	800FE	200
+# CHAR enlargement
+alter table t modify a char(220), algorithm=instant;
+select count(a) from t where a = @bigval;
+count(a)
+1
+select a, length(a) from t where a = 'z';
+a	length(a)
+z	1
+check table t extended;
+Table	Op	Msg_type	Msg_text
+test.t	check	status	OK
+call check_table('t');
+name	mtype	prtype	len
+a	2	800FE	200
+# Convert from VARCHAR to a bigger CHAR
+alter table t modify a varchar(200), algorithm=instant;
+ERROR 0A000: ALGORITHM=INSTANT is not supported. Reason: Cannot change column type INPLACE. Try ALGORITHM=COPY
+alter table t modify a varchar(200), algorithm=copy;
+alter table t modify a char(255), algorithm=instant;
+select count(a) from t where a = @bigval;
+count(a)
+1
+select a, length(a) from t where a = 'z';
+a	length(a)
+z	1
+select * from t;
+a
+01234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789
+z
+check table t extended;
+Table	Op	Msg_type	Msg_text
+test.t	check	status	OK
+call check_table('t');
+name	mtype	prtype	len
+a	2	800FE	255
+# BINARY/VARBINARY test
+create or replace table t (a varbinary(300));
+alter table t modify a binary(255), algorithm=instant;
+ERROR 0A000: ALGORITHM=INSTANT is not supported. Reason: Cannot change column type INPLACE. Try ALGORITHM=COPY
+alter table t modify a binary(255), algorithm=copy;
+create or replace table t (a varbinary(200));
+insert into t values (@bigval);
+insert into t values ('z');
+alter table t modify a binary(200), algorithm=instant;
+select count(a) from t where a = @bigval;
+count(a)
+1
+select length(a) from t where left(a, 1) = 'z';
+length(a)
+200
+check table t extended;
+Table	Op	Msg_type	Msg_text
+test.t	check	status	OK
+call check_table('t');
+name	mtype	prtype	len
+a	3	3F04FE	200
+# BINARY enlargement
+alter table t modify a binary(220), algorithm=instant;
+check table t extended;
+Table	Op	Msg_type	Msg_text
+test.t	check	status	OK
+call check_table('t');
+name	mtype	prtype	len
+a	3	3F04FE	200
+# Convert from VARBINARY to a bigger BINARY
+alter table t modify a varbinary(220), algorithm=instant;
+ERROR 0A000: ALGORITHM=INSTANT is not supported. Reason: Cannot change column type INPLACE. Try ALGORITHM=COPY
+alter table t modify a varbinary(220), algorithm=copy;
+alter table t modify a binary(255), algorithm=instant;
+select count(a) from t where a = @bigval;
+count(a)
+0
+select a, length(a) from t where a = 'z';
+a	length(a)
+select * from t;
+a
+01234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789                                                       
+z                                                                                                                                                                                                                                                              
+check table t extended;
+Table	Op	Msg_type	Msg_text
+test.t	check	status	OK
+call check_table('t');
+name	mtype	prtype	len
+a	3	3F04FE	255
+# Integer conversions
+create or replace table t (x tinyint);
+insert into t values (127);
+alter table t modify x smallint, algorithm=instant;
+select * from t;
+x
+127
+check table t extended;
+Table	Op	Msg_type	Msg_text
+test.t	check	status	OK
+call check_table('t');
+name	mtype	prtype	len
+x	6	402	2
+update t set x= 32767;
+alter table t modify x mediumint, algorithm=instant;
+select * from t;
+x
+32767
+check table t extended;
+Table	Op	Msg_type	Msg_text
+test.t	check	status	OK
+call check_table('t');
+name	mtype	prtype	len
+x	6	409	3
+update t set x= 8388607;
+alter table t modify x int, algorithm=instant;
+select * from t;
+x
+8388607
+check table t extended;
+Table	Op	Msg_type	Msg_text
+test.t	check	status	OK
+call check_table('t');
+name	mtype	prtype	len
+x	6	403	4
+update t set x= 2147483647;
+alter table t modify x bigint, algorithm=instant;
+select * from t;
+x
+2147483647
+check table t extended;
+Table	Op	Msg_type	Msg_text
+test.t	check	status	OK
+call check_table('t');
+name	mtype	prtype	len
+x	6	408	8
+# Check IMPORT TABLESPACE
+create or replace table t2 (x int);
+alter table t2 discard tablespace;
+create or replace table t1 (x tinyint);
+insert into t1 set x= 42;
+alter table t1 modify x int;
+flush tables t1 for export;
+unlock tables;
+alter table t2 import tablespace;
+select * from t2;
+x
+42
+check table t2 extended;
+Table	Op	Msg_type	Msg_text
+test.t2	check	status	OK
+call check_table('t2');
+name	mtype	prtype	len
+x	6	403	4
+# Check innobase_col_to_mysql() len < flen
+create or replace table t1 (x mediumint);
+insert into t1 values (1);
+insert into t1 values (1);
+alter table t1 add column y int first, modify x int, algorithm instant;
+alter table t1 add column z int first, add primary key (x);
+ERROR 23000: Duplicate entry '1' for key 'PRIMARY'
+# Check assertion in wrong instant operation
+create or replace table t1 (a varchar(26) not null) default character set utf8mb4;
+alter table t1 modify a varchar(25) not null;
+# Check row_mysql_store_col_in_innobase_format()
+create or replace table t1(x int primary key, a varchar(20));
+insert into t1 (x) values (1);
+update t1 set a= 'foo' where x = 2;
+#
+# MDEV-18124 PK on inplace-enlarged type fails
+#
+create or replace table t1 (x int, y int);
+insert into t1 (x, y) values (11, 22);
+alter table t1 modify x bigint, algorithm instant;
+alter table t1 add primary key (x), algorithm inplace;
+check table t1;
+Table	Op	Msg_type	Msg_text
+test.t1	check	status	OK
+create or replace table t1 (a varchar(10), y int);
+insert into t1 (a, y) values ("0123456789", 33);
+alter table t1 modify a char(15), algorithm instant;
+alter table t1 add primary key (a), algorithm inplace;
+check table t1;
+Table	Op	Msg_type	Msg_text
+test.t1	check	status	OK
+create or replace table t1 (x int primary key, y int);
+insert into t1 (x, y) values (44, 55);
+alter table t1 modify x bigint, algorithm inplace;
+check table t1;
+Table	Op	Msg_type	Msg_text
+test.t1	check	status	OK
+create or replace table t1 (x int primary key, y int);
+insert into t1 values (66, 77);
+alter table t1 add column z int, algorithm instant;
+alter table t1 drop column y, algorithm instant;
+create or replace table t1 (x integer, a varchar(20));
+alter table t1 add index idx3 (a);
+insert into t1 (x, a) values (73, 'a');
+alter table t1 modify a char(20);
+create or replace database test charset latin1;
+SET GLOBAL innodb_default_row_format=@save_format;
--- a/mysql-test/suite/innodb/t/instant_alter_extend.combinations
+++ b/mysql-test/suite/innodb/t/instant_alter_extend.combinations
@ -0,0 +1,5 @@
+[latin1]
+character-set-server=latin1
+
+[utf8]
+character-set-server=utf8
--- a/mysql-test/suite/innodb/t/instant_alter_extend.test
+++ b/mysql-test/suite/innodb/t/instant_alter_extend.test
@ -0,0 +1,226 @@
+--source include/have_innodb.inc
+--source include/maybe_debug.inc
+
+-- echo #
+-- echo # MDEV-15563: Instant ROW_FORMAT=REDUNDANT column type change&extension
+-- echo #
+
+# Use character-set-server in test db
+create or replace database test;
+use test;
+
+set default_storage_engine=innodb;
+set @save_format= @@GLOBAL.innodb_default_row_format;
+SET GLOBAL innodb_default_row_format=redundant;
+set @bigval= repeat('0123456789', 30);
+
+delimiter ~~;
+create or replace procedure check_table(table_name varchar(255))
+begin
+  select table_id into @table_id
+  from information_schema.innodb_sys_tables
+  where name = concat('test/', table_name);
+  select name, mtype, hex(prtype) as prtype, len
+  from information_schema.innodb_sys_columns
+  where table_id = @table_id;
+end~~
+delimiter ;~~
+
+
+--echo # VARCHAR -> CHAR, VARBINARY -> BINARY conversion
+set @bigval= repeat('0123456789', 20);
+
+create or replace table t (a varchar(300));
+--error ER_ALTER_OPERATION_NOT_SUPPORTED_REASON
+alter table t modify a char(255), algorithm=instant;
+alter table t modify a char(255), algorithm=copy;
+
+create or replace table t (a varchar(200));
+if ($have_debug) {
+--disable_query_log
+--disable_result_log
+set debug_dbug= '+d,ib_instant_error';
+--error ER_RECORD_FILE_FULL
+alter table t modify a char(200);
+set debug_dbug= default;
+--enable_query_log
+--enable_result_log
+}
+insert into t values (@bigval);
+insert into t values ('z');
+alter table t modify a char(200), algorithm=instant;
+select count(a) from t where a = @bigval;
+select a, length(a) from t where a = 'z';
+
+check table t extended;
+call check_table('t');
+
+--echo # CHAR enlargement
+alter table t modify a char(220), algorithm=instant;
+select count(a) from t where a = @bigval;
+select a, length(a) from t where a = 'z';
+
+check table t extended;
+call check_table('t');
+
+--echo # Convert from VARCHAR to a bigger CHAR
+--error ER_ALTER_OPERATION_NOT_SUPPORTED_REASON
+alter table t modify a varchar(200), algorithm=instant;
+alter table t modify a varchar(200), algorithm=copy;
+alter table t modify a char(255), algorithm=instant;
+select count(a) from t where a = @bigval;
+select a, length(a) from t where a = 'z';
+
+select * from t;
+check table t extended;
+call check_table('t');
+
+--echo # BINARY/VARBINARY test
+create or replace table t (a varbinary(300));
+--error ER_ALTER_OPERATION_NOT_SUPPORTED_REASON
+alter table t modify a binary(255), algorithm=instant;
+alter table t modify a binary(255), algorithm=copy;
+
+create or replace table t (a varbinary(200));
+if ($have_debug) {
+--disable_query_log
+--disable_result_log
+set debug_dbug= '+d,ib_instant_error';
+--error ER_RECORD_FILE_FULL
+alter table t modify a binary(200);
+set debug_dbug= default;
+--enable_query_log
+--enable_result_log
+}
+insert into t values (@bigval);
+insert into t values ('z');
+alter table t modify a binary(200), algorithm=instant;
+select count(a) from t where a = @bigval;
+select length(a) from t where left(a, 1) = 'z';
+
+check table t extended;
+call check_table('t');
+
+--echo # BINARY enlargement
+alter table t modify a binary(220), algorithm=instant;
+
+check table t extended;
+call check_table('t');
+
+--echo # Convert from VARBINARY to a bigger BINARY
+--error ER_ALTER_OPERATION_NOT_SUPPORTED_REASON
+alter table t modify a varbinary(220), algorithm=instant;
+alter table t modify a varbinary(220), algorithm=copy;
+alter table t modify a binary(255), algorithm=instant;
+select count(a) from t where a = @bigval;
+select a, length(a) from t where a = 'z';
+
+select * from t;
+check table t extended;
+call check_table('t');
+
+
+--echo # Integer conversions
+create or replace table t (x tinyint);
+if ($have_debug) {
+--disable_query_log
+--disable_result_log
+set debug_dbug= '+d,ib_instant_error';
+--error ER_RECORD_FILE_FULL
+alter table t modify x smallint;
+set debug_dbug= default;
+--enable_query_log
+--enable_result_log
+}
+insert into t values (127);
+alter table t modify x smallint, algorithm=instant;
+select * from t;
+check table t extended;
+call check_table('t');
+
+update t set x= 32767;
+alter table t modify x mediumint, algorithm=instant;
+select * from t;
+check table t extended;
+call check_table('t');
+
+update t set x= 8388607;
+alter table t modify x int, algorithm=instant;
+select * from t;
+check table t extended;
+call check_table('t');
+
+update t set x= 2147483647;
+alter table t modify x bigint, algorithm=instant;
+select * from t;
+check table t extended;
+call check_table('t');
+
+--echo # Check IMPORT TABLESPACE
+--let $MYSQLD_DATADIR= `select @@datadir`
+create or replace table t2 (x int);
+alter table t2 discard tablespace;
+
+create or replace table t1 (x tinyint);
+insert into t1 set x= 42;
+alter table t1 modify x int;
+flush tables t1 for export;
+--move_file $MYSQLD_DATADIR/test/t1.cfg $MYSQLD_DATADIR/test/t2.cfg
+--copy_file $MYSQLD_DATADIR/test/t1.ibd $MYSQLD_DATADIR/test/t2.ibd
+unlock tables;
+
+alter table t2 import tablespace;
+
+select * from t2;
+check table t2 extended;
+call check_table('t2');
+
+--echo # Check innobase_col_to_mysql() len < flen
+create or replace table t1 (x mediumint);
+insert into t1 values (1);
+insert into t1 values (1);
+alter table t1 add column y int first, modify x int, algorithm instant;
+--error ER_DUP_ENTRY
+alter table t1 add column z int first, add primary key (x);
+
+--echo # Check assertion in wrong instant operation
+create or replace table t1 (a varchar(26) not null) default character set utf8mb4;
+alter table t1 modify a varchar(25) not null;
+
+--echo # Check row_mysql_store_col_in_innobase_format()
+create or replace table t1(x int primary key, a varchar(20));
+insert into t1 (x) values (1);
+update t1 set a= 'foo' where x = 2;
+
+--echo #
+--echo # MDEV-18124 PK on inplace-enlarged type fails
+--echo #
+create or replace table t1 (x int, y int);
+insert into t1 (x, y) values (11, 22);
+alter table t1 modify x bigint, algorithm instant;
+alter table t1 add primary key (x), algorithm inplace;
+check table t1;
+
+create or replace table t1 (a varchar(10), y int);
+insert into t1 (a, y) values ("0123456789", 33);
+alter table t1 modify a char(15), algorithm instant;
+alter table t1 add primary key (a), algorithm inplace;
+check table t1;
+
+create or replace table t1 (x int primary key, y int);
+insert into t1 (x, y) values (44, 55);
+alter table t1 modify x bigint, algorithm inplace;
+check table t1;
+
+create or replace table t1 (x int primary key, y int);
+insert into t1 values (66, 77);
+alter table t1 add column z int, algorithm instant;
+alter table t1 drop column y, algorithm instant;
+
+create or replace table t1 (x integer, a varchar(20));
+alter table t1 add index idx3 (a);
+insert into t1 (x, a) values (73, 'a');
+alter table t1 modify a char(20);
+
+create or replace database test charset latin1;
+SET GLOBAL innodb_default_row_format=@save_format;
--- a/sql/field.cc
+++ b/sql/field.cc
@ -7063,9 +7063,16 @@ uint Field::is_equal(Create_field *new_field)

 uint Field_str::is_equal(Create_field *new_field)
 {
-  return new_field->type_handler() == type_handler() &&
-         new_field->charset == field_charset &&
-         new_field->length == max_display_length();
+  if (new_field->type_handler() == type_handler() &&
+      new_field->charset == field_charset)
+  {
+    if (new_field->length == max_display_length())
+      return IS_EQUAL_YES;
+    if (new_field->length > max_display_length() &&
+        (table->file->ha_table_flags() & HA_EXTENDED_TYPES_CONVERSION))
+      return IS_EQUAL_PACK_LENGTH_EXT;
+  }
+  return IS_EQUAL_NO;
 }


@ -7904,19 +7911,26 @@ uint Field_varstring::is_equal(Create_field *new_field)
  if (new_field->length < field_length)
    return IS_EQUAL_NO;

-  if (new_field->type_handler() == type_handler() &&
-      new_field->charset == field_charset &&
+  if (new_field->charset == field_charset &&
      !new_field->compression_method() == !compression_method())
  {
-    if (new_field->length == field_length)
-      return IS_EQUAL_YES;
-    if (field_length <= 127 ||
-        new_field->length <= 255 ||
-        field_length > 255 ||
-	(table->file->ha_table_flags() & HA_EXTENDED_TYPES_CONVERSION))
-      return IS_EQUAL_PACK_LENGTH; // VARCHAR, longer length
+    const Type_handler *new_type_handler= new_field->type_handler();
+    if (new_type_handler == type_handler())
+    {
+      if (new_field->length == field_length)
+        return IS_EQUAL_YES;
+      if (field_length <= 127 ||
+          new_field->length <= 255 ||
+          field_length > 255 ||
+	  (table->file->ha_table_flags() & HA_EXTENDED_TYPES_CONVERSION))
+        return IS_EQUAL_PACK_LENGTH; // VARCHAR, longer length
+    }
+    else if (new_type_handler == &type_handler_string) // converting to CHAR
+    {
+      if (table->file->ha_table_flags() & HA_EXTENDED_TYPES_CONVERSION)
+        return IS_EQUAL_PACK_LENGTH_EXT;
+    }
  }
-
  return IS_EQUAL_NO;
 }

@ -9485,12 +9499,22 @@ bool Field_num::eq_def(const Field *field) const

 uint Field_num::is_equal(Create_field *new_field)
 {
-  return ((new_field->type_handler() == type_handler()) &&
-          ((new_field->flags & UNSIGNED_FLAG) == 
-           (uint) (flags & UNSIGNED_FLAG)) &&
-	  ((new_field->flags & AUTO_INCREMENT_FLAG) ==
-	   (uint) (flags & AUTO_INCREMENT_FLAG)) &&
-          (new_field->pack_length == pack_length()));
+  if ((new_field->flags ^ flags) & (UNSIGNED_FLAG | AUTO_INCREMENT_FLAG))
+    return IS_EQUAL_NO;
+
+  const Type_handler *th= type_handler(), *new_th = new_field->type_handler();
+
+  if (th == new_th && new_field->pack_length == pack_length())
+    return IS_EQUAL_YES;
+
+  if (table->file->ha_table_flags() & HA_EXTENDED_TYPES_CONVERSION)
+  {
+    if (th->result_type() == new_th->result_type() &&
+        new_field->pack_length >= pack_length())
+      return IS_EQUAL_PACK_LENGTH_EXT;
+  }
+
+  return IS_EQUAL_NO;
 }


--- a/sql/field.h
+++ b/sql/field.h
@ -4564,7 +4564,7 @@ public:
   :Type_handler_hybrid_field_type(&type_handler_null),
    compression_method_ptr(0),
    comment(null_clex_str),
-    on_update(NULL), invisible(VISIBLE), decimals(0),
+    on_update(NULL), invisible(VISIBLE), char_length(0), decimals(0),
    flags(0), pack_length(0), key_length(0),
    option_list(NULL),
    vcol_info(0), default_value(0), check_constraint(0),
--- a/sql/handler.h
+++ b/sql/handler.h
@ -746,6 +746,14 @@ typedef ulonglong alter_table_operations;
 */
 #define ALTER_COLUMN_INDEX_LENGTH            (1ULL << 60)

+/**
+  Change the column length or type such that no rebuild is needed.
+  Only set if ALTER_COLUMN_EQUAL_PACK_LENGTH does not apply, and
+  if HA_EXTENDED_TYPES_CONVERSION holds.
+  @see IS_EQUAL_PACK_LENGTH_EXT
+*/
+#define ALTER_COLUMN_EQUAL_PACK_LENGTH_EXT   (1ULL << 61)
+
 /*
  Flags set in partition_flags when altering partitions
 */
--- a/sql/sql_priv.h
+++ b/sql/sql_priv.h
@ -1,5 +1,5 @@
 /* Copyright (c) 2000, 2018, Oracle and/or its affiliates.
-   Copyright (c) 2010, 2018, Monty Program Ab.
+   Copyright (c) 2010, 2019, MariaDB

   This program is free software; you can redistribute it and/or modify
   it under the terms of the GNU General Public License as published by
@ -345,11 +345,21 @@
 #ifndef MYSQL_CLIENT

 /*
-  Some defines for exit codes for ::is_equal class functions.
+  Field::is_equal() return codes.
 */
 #define IS_EQUAL_NO 0
 #define IS_EQUAL_YES 1
+/**
+  new_field has compatible packed representation with old type,
+  so it is theoretically possible to perform change by only updating
+  data dictionary without changing table rows
+*/
 #define IS_EQUAL_PACK_LENGTH 2
+/**
+  new_field has a representation that is compatible with the old type
+  when the storage engine advertises HA_EXTENDED_TYPES_CONVERSION
+*/
+#define IS_EQUAL_PACK_LENGTH_EXT 3

 enum enum_parsing_place
 {
--- a/sql/sql_table.cc
+++ b/sql/sql_table.cc
@ -6615,6 +6615,9 @@ static bool fill_alter_inplace_info(THD *thd,
        */
        ha_alter_info->handler_flags|= ALTER_COLUMN_EQUAL_PACK_LENGTH;
        break;
+      case IS_EQUAL_PACK_LENGTH_EXT:
+        ha_alter_info->handler_flags|= ALTER_COLUMN_EQUAL_PACK_LENGTH_EXT;
+        break;
      default:
        DBUG_ASSERT(0);
        /* Safety. */
--- a/storage/innobase/btr/btr0btr.cc
+++ b/storage/innobase/btr/btr0btr.cc
@ -4839,7 +4839,9 @@ n_field_mismatch:
 		if (len_is_stored(len)
 		    && (field->prefix_len
 			? len > field->prefix_len
-			: (fixed_size && len != fixed_size))) {
+			: (fixed_size && (page_is_comp(page)
+					  ? len != fixed_size
+					  : len > fixed_size)))) {
 len_mismatch:
 			btr_index_rec_validate_report(page, rec, index);
 			ib::error	error;
--- a/storage/innobase/btr/btr0cur.cc
+++ b/storage/innobase/btr/btr0cur.cc
@ -486,8 +486,14 @@ incompatible:
 			For the metadata record, variable-length columns are
 			always written with zero length. The DB_TRX_ID will
 			start right after any fixed-length columns. */
-			for (uint i = index->n_uniq; i--; ) {
-				trx_id_offset += index->fields[i].fixed_len;
+			if (index->table->not_redundant()) {
+				for (uint i = index->n_uniq; i--; ) {
+					trx_id_offset += index->fields[i]
+						.fixed_len;
+				}
+			} else {
+				trx_id_offset = rec_get_field_start_offs(
+					rec, index->n_uniq);
 			}
 		}

--- a/storage/innobase/dict/dict0dict.cc
+++ b/storage/innobase/dict/dict0dict.cc
@ -2203,6 +2203,14 @@ dict_index_too_big_for_tree(
 		}

 		field_max_size = dict_col_get_max_size(col);
+		if (!comp && (col->mtype == DATA_INT
+			      || col->mtype == DATA_CHAR
+			      || col->mtype == DATA_FIXBINARY)) {
+			/* DATA_INT, DATA_FIXBINARY and DATA_CHAR are variable-
+			length (enlarged instantly), but are stored locally. */
+			field_ext_max_size = 0;
+			goto add_field_size;
+		}
 		field_ext_max_size = field_max_size < 256 ? 1 : 2;

 		if (field->prefix_len) {
--- a/storage/innobase/handler/ha_innodb.cc
+++ b/storage/innobase/handler/ha_innodb.cc
@ -9967,7 +9967,7 @@ innobase_fts_create_doc_id_key(
 	/* The unique Doc ID field should be an eight-bytes integer */
 	dict_field_t*	field = dict_index_get_nth_field(index, 0);
        ut_a(field->col->mtype == DATA_INT);
-	ut_ad(sizeof(*doc_id) == field->fixed_len);
+	ut_ad(sizeof(*doc_id) == field->col->len);
 	ut_ad(!strcmp(index->name, FTS_DOC_ID_INDEX_NAME));
 #endif /* UNIV_DEBUG */

--- a/storage/innobase/handler/handler0alter.cc
+++ b/storage/innobase/handler/handler0alter.cc
@ -84,6 +84,7 @@ static const alter_table_operations INNOBASE_ALTER_REBUILD
 	| ALTER_OPTIONS
 	/* ALTER_OPTIONS needs to check alter_options_need_rebuild() */
 	| ALTER_COLUMN_NULLABLE
+	| ALTER_COLUMN_EQUAL_PACK_LENGTH_EXT
 	| INNOBASE_DEFAULTS
 	| ALTER_STORED_COLUMN_ORDER
 	| ALTER_DROP_STORED_COLUMN
@ -268,7 +269,7 @@ inline void dict_table_t::prepare_instant(const dict_table_t& old,
 			ut_ad(!not_redundant());
 			for (unsigned i = index.n_fields; i--; ) {
 				ut_ad(index.fields[i].col->same_format(
-					      *oindex.fields[i].col));
+					      *oindex.fields[i].col, true));
 			}
 		}
 #endif
@ -456,9 +457,13 @@ inline void dict_index_t::instant_add_field(const dict_index_t& instant)
 	as this index. Fields for any added columns are appended at the end. */
 #ifndef DBUG_OFF
 	for (unsigned i = 0; i < n_fields; i++) {
-		DBUG_ASSERT(fields[i].same(instant.fields[i]));
-		DBUG_ASSERT(instant.fields[i].col->same_format(*fields[i]
-							       .col));
+		DBUG_ASSERT(fields[i].prefix_len
+			    == instant.fields[i].prefix_len);
+		DBUG_ASSERT(fields[i].fixed_len
+			    == instant.fields[i].fixed_len
+			    || !table->not_redundant());
+		DBUG_ASSERT(instant.fields[i].col->same_format(
+				    *fields[i].col, !table->not_redundant()));
 		/* Instant conversion from NULL to NOT NULL is not allowed. */
 		DBUG_ASSERT(!fields[i].col->is_nullable()
 			    || instant.fields[i].col->is_nullable());
@ -534,10 +539,7 @@ inline bool dict_table_t::instant_column(const dict_table_t& table,

 		if (const dict_col_t* o = find(old_cols, col_map, n_cols, i)) {
 			c.def_val = o->def_val;
-			DBUG_ASSERT(!((c.prtype ^ o->prtype)
-				      & ~(DATA_NOT_NULL | DATA_VERSIONED)));
-			DBUG_ASSERT(c.mtype == o->mtype);
-			DBUG_ASSERT(c.len >= o->len);
+			ut_ad(c.same_format(*o, !not_redundant()));

 			if (o->vers_sys_start()) {
 				ut_ad(o->ind == vers_start);
@ -1505,7 +1507,8 @@ instant_alter_column_possible(
 		= ALTER_ADD_STORED_BASE_COLUMN
 		| ALTER_DROP_STORED_COLUMN
 		| ALTER_STORED_COLUMN_ORDER
-		| ALTER_COLUMN_NULLABLE;
+		| ALTER_COLUMN_NULLABLE
+		| ALTER_COLUMN_EQUAL_PACK_LENGTH_EXT;

 	if (!(ha_alter_info->handler_flags & avoid_rebuild)) {
 		alter_table_operations flags = ha_alter_info->handler_flags
@ -1548,6 +1551,7 @@ instant_alter_column_possible(
 	       & ~ALTER_STORED_COLUMN_ORDER
 	       & ~ALTER_ADD_STORED_BASE_COLUMN
 	       & ~ALTER_COLUMN_NULLABLE
+	       & ~ALTER_COLUMN_EQUAL_PACK_LENGTH_EXT
 	       & ~ALTER_OPTIONS)) {
 		return false;
 	}
@ -1748,6 +1752,10 @@ ha_innobase::check_if_supported_inplace_alter(
 {
 	DBUG_ENTER("check_if_supported_inplace_alter");

+	DBUG_ASSERT(!(ALTER_COLUMN_EQUAL_PACK_LENGTH_EXT
+		      & ha_alter_info->handler_flags)
+		    || !m_prebuilt->table->not_redundant());
+
 	if ((ha_alter_info->handler_flags
 	     & INNOBASE_ALTER_VERSIONED_REBUILD)
 	    && altered_table->versioned(VERS_TIMESTAMP)) {
@ -2943,7 +2951,7 @@ innobase_col_to_mysql(

 	switch (col->mtype) {
 	case DATA_INT:
-		ut_ad(len == flen);
+		ut_ad(len <= flen);

 		/* Convert integer data from Innobase to little-endian
 		format, sign bit restored to normal */
@ -5583,7 +5591,8 @@ static bool innobase_instant_try(

 		bool update = old && (!ctx->first_alter_pos
 				      || i < ctx->first_alter_pos - 1);
-		DBUG_ASSERT(!old || col->same_format(*old));
+		ut_ad(!old || col->same_format(
+			      *old, !user_table->not_redundant()));
 		if (update
 		    && old->prtype == d->type.prtype) {
 			/* The record is already present in SYS_COLUMNS. */
@ -5672,6 +5681,8 @@ add_all_virtual:
 		NULL, trx, ctx->heap, NULL);

 	dberr_t err = DB_SUCCESS;
+	DBUG_EXECUTE_IF("ib_instant_error",
+			err = DB_OUT_OF_FILE_SPACE; goto func_exit;);
 	if (rec_is_metadata(rec, *index)) {
 		ut_ad(page_rec_is_user_rec(rec));
 		if (!page_has_next(block->frame)
@ -9050,12 +9061,16 @@ innobase_enlarge_column_try(
 #ifdef UNIV_DEBUG
 	ut_ad(col->len < new_len);
 	switch (col->mtype) {
+	case DATA_FIXBINARY:
+	case DATA_CHAR:
 	case DATA_MYSQL:
 		/* NOTE: we could allow this when !(prtype & DATA_BINARY_TYPE)
 		and ROW_FORMAT is not REDUNDANT and mbminlen<mbmaxlen.
 		That is, we treat a UTF-8 CHAR(n) column somewhat like
 		a VARCHAR. */
-		ut_error;
+		if (user_table->not_redundant()) {
+			ut_error;
+		}
 	case DATA_BINARY:
 	case DATA_VARCHAR:
 	case DATA_VARMYSQL:
@ -9188,14 +9203,16 @@ innobase_rename_or_enlarge_columns_cache(
 			ulint	col_n = is_virtual ? num_v : i - num_v;

 			if ((*fp)->is_equal(cf) == IS_EQUAL_PACK_LENGTH) {
-				if (is_virtual) {
-					dict_table_get_nth_v_col(
-						user_table, col_n)->m_col.len
-					= cf->length;
-				} else {
-					dict_table_get_nth_col(
-						user_table, col_n)->len
-					= cf->length;
+				dict_col_t *col = is_virtual ?
+					&dict_table_get_nth_v_col(
+						user_table, col_n)->m_col
+					: dict_table_get_nth_col(
+						user_table, col_n);
+				col->len = cf->length;
+				if (col->len > 255
+				    && (col->prtype & DATA_MYSQL_TRUE_VARCHAR)
+				    == DATA_MYSQL_TRUE_VARCHAR) {
+					col->prtype |= DATA_LONG_TRUE_VARCHAR;
 				}
 			}

--- a/storage/innobase/include/data0type.ic
+++ b/storage/innobase/include/data0type.ic
@ -1,7 +1,7 @@
 /*****************************************************************************

 Copyright (c) 1996, 2014, Oracle and/or its affiliates. All Rights Reserved.
-Copyright (c) 2017, 2018, MariaDB Corporation.
+Copyright (c) 2017, 2019, MariaDB Corporation.

 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@ -472,12 +472,18 @@ dtype_get_fixed_size_low(
 		}
 #endif /* UNIV_DEBUG */
 		/* fall through */
-	case DATA_CHAR:
-	case DATA_FIXBINARY:
-	case DATA_INT:
 	case DATA_FLOAT:
 	case DATA_DOUBLE:
 		return(len);
+	case DATA_FIXBINARY:
+	case DATA_CHAR:
+	case DATA_INT:
+		/* Treat these types as variable length for redundant
+		row format. We can't rely on fixed_len anymore because record
+		can have shorter length from before instant enlargement
+		[MDEV-15563]. Note, that importing such tablespace to
+		earlier MariaDB versions produces ER_TABLE_SCHEMA_MISMATCH. */
+		return comp ? len : 0;
 	case DATA_MYSQL:
 		if (prtype & DATA_BINARY_TYPE) {
 			return(len);
@ -625,6 +631,14 @@ dtype_get_sql_null_size(
 	const dtype_t*	type,	/*!< in: type */
 	ulint		comp)	/*!< in: nonzero=ROW_FORMAT=COMPACT  */
 {
+	switch (type->mtype) {
+	case DATA_INT:
+	case DATA_CHAR:
+	case DATA_FIXBINARY:
+		return(type->len);
+	default:
+		break;
+	}
 	return(dtype_get_fixed_size_low(type->mtype, type->prtype, type->len,
 					type->mbminlen, type->mbmaxlen, comp));
 }
--- a/storage/innobase/include/dict0mem.h
+++ b/storage/innobase/include/dict0mem.h
@ -2,7 +2,7 @@

 Copyright (c) 1996, 2017, Oracle and/or its affiliates. All Rights Reserved.
 Copyright (c) 2012, Facebook Inc.
-Copyright (c) 2013, 2018, MariaDB Corporation.
+Copyright (c) 2013, 2019, MariaDB Corporation.

 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@ -685,15 +685,52 @@ public:
 	/** Determine if the columns have the same format
 	except for is_nullable() and is_versioned().
 	@param[in]	other	column to compare to
+	@param[in]	redundant table is redundant row format
 	@return	whether the columns have the same format */
-	bool same_format(const dict_col_t& other) const
+	bool same_format(const dict_col_t& other, bool redundant = false) const
 	{
-		return mtype == other.mtype
-			&& len >= other.len
-			&& mbminlen == other.mbminlen
-			&& mbmaxlen == other.mbmaxlen
-			&& !((prtype ^ other.prtype)
-			     & ~(DATA_NOT_NULL | DATA_VERSIONED));
+		if (len < other.len
+		    || mbminlen != other.mbminlen
+		    || mbmaxlen != other.mbmaxlen) {
+			return false;
+		}
+		if (dtype_is_non_binary_string_type(mtype, prtype)
+		    && dtype_is_non_binary_string_type(other.mtype,
+						       other.prtype)) {
+			uint csn1 = (uint) dtype_get_charset_coll(prtype);
+			uint csn2 = (uint) dtype_get_charset_coll(other.prtype);
+			CHARSET_INFO* cs1 = get_charset(csn1, MYF(MY_WME));
+			CHARSET_INFO* cs2 = get_charset(csn2, MYF(MY_WME));
+			if (!my_charset_same(cs1, cs2)) {
+				return false;
+			}
+		}
+
+		if (!((prtype ^ other.prtype)
+		      & ~(DATA_NOT_NULL | DATA_VERSIONED))) {
+			return mtype == other.mtype;
+		}
+
+		if (redundant) {
+			switch (other.mtype) {
+			case DATA_CHAR:
+			case DATA_MYSQL:
+			case DATA_VARCHAR:
+			case DATA_VARMYSQL:
+				return mtype == DATA_CHAR
+					|| mtype == DATA_MYSQL
+					|| mtype == DATA_VARCHAR
+					|| mtype == DATA_VARMYSQL;
+			case DATA_FIXBINARY:
+			case DATA_BINARY:
+				return mtype == DATA_FIXBINARY
+					|| mtype == DATA_BINARY;
+			case DATA_INT:
+				return mtype == DATA_INT;
+			}
+		}
+
+		return false;
 	}
 };

--- a/storage/innobase/row/row0merge.cc
+++ b/storage/innobase/row/row0merge.cc
@ -706,11 +706,6 @@ row_merge_buf_add(
 						row_field, field, col->len,
 						old_table->space->zip_size(),
 						conv_heap);
-				} else {
-					/* Field length mismatch should not
-					happen when rebuilding redundant row
-					format table. */
-					ut_ad(dict_table_is_comp(index->table));
 				}
 			}
 		}
--- a/storage/innobase/row/row0mysql.cc
+++ b/storage/innobase/row/row0mysql.cc
@ -887,10 +887,15 @@ row_create_prebuilt(
 						== MAX_REF_PARTS););
 		uint temp_len = 0;
 		for (uint i = 0; i < temp_index->n_uniq; i++) {
-			ulint type = temp_index->fields[i].col->mtype;
-			if (type == DATA_INT) {
-				temp_len +=
-					temp_index->fields[i].fixed_len;
+			const dict_field_t& f = temp_index->fields[i];
+			if (f.col->mtype == DATA_INT) {
+				ut_ad(f.col->len >= f.fixed_len);
+				/* dtype_get_fixed_size_low() returns 0
+				for ROW_FORMAT=REDUNDANT */
+				ut_ad(table->not_redundant()
+				      ? f.col->len == f.fixed_len
+				      : f.fixed_len == 0);
+				temp_len += f.col->len;
 			}
 		}
 		srch_key_len = std::max(srch_key_len,temp_len);
--- a/storage/innobase/row/row0sel.cc
+++ b/storage/innobase/row/row0sel.cc
@ -2742,12 +2742,15 @@ row_sel_field_store_in_mysql_format_func(
 			dest[len - 1] = (byte) (dest[len - 1] ^ 128);
 		}

-		ut_ad(templ->mysql_col_len == len);
+		ut_ad(templ->mysql_col_len == len
+		      || !index->table->not_redundant());
 		break;

 	case DATA_VARCHAR:
 	case DATA_VARMYSQL:
 	case DATA_BINARY:
+	case DATA_CHAR:
+	case DATA_FIXBINARY:
 		field_end = dest + templ->mysql_col_len;

 		if (templ->mysql_type == DATA_MYSQL_TRUE_VARCHAR) {
@ -2835,7 +2838,8 @@ row_sel_field_store_in_mysql_format_func(
 		ut_ad(len * templ->mbmaxlen >= templ->mysql_col_len
 		      || (field_no == templ->icp_rec_field_no
 			  && field->prefix_len > 0)
-		      || templ->rec_field_is_prefix);
+		      || templ->rec_field_is_prefix
+		      || !index->table->not_redundant());

 		ut_ad(templ->is_virtual
 		      || !(field->prefix_len % templ->mbmaxlen));
@ -2857,8 +2861,6 @@ row_sel_field_store_in_mysql_format_func(
 		ut_ad(0);
 		/* fall through */

-	case DATA_CHAR:
-	case DATA_FIXBINARY:
 	case DATA_FLOAT:
 	case DATA_DOUBLE:
 	case DATA_DECIMAL: