Merge 10.2 into bb-10.2-ext

BUILD/FINISH.sh

@@ -14,7 +14,7 @@
 # License along with this library; if not, write to the Free
 # Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston,
 # MA 02110-1301, USA
-set -x -v
+
 cflags="$c_warnings $extra_flags $EXTRA_FLAGS $EXTRA_CFLAGS"
 cxxflags="$cxx_warnings $base_cxxflags $extra_flags $EXTRA_FLAGS $EXTRA_CXXFLAGS"
 extra_configs="$extra_configs $local_infile_configs $EXTRA_CONFIGS"

BUILD/check-cpu

@@ -40,6 +40,12 @@ check_compiler_cpu_flags () {
     cc_major=$1
     cc_minor=$2
     cc_patch=$3
+    if test -z "$cc_minor"; then
+      cc_minor="0";
+    fi
+    if test -z "$cc_patch"; then
+      cc_minor="0";
+    fi
     cc_comp=`expr $cc_major '*' 100 '+' $cc_minor`
   fi
 

storage/innobase/fil/fil0fil.cc

@@ -1,7 +1,7 @@
 /*****************************************************************************
 
 Copyright (c) 1995, 2017, Oracle and/or its affiliates. All Rights Reserved.
-Copyright (c) 2014, 2017, MariaDB Corporation.
+Copyright (c) 2014, 2018, MariaDB Corporation.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -566,7 +566,6 @@ bool
 fil_node_open_file(
 	fil_node_t*	node)
 {
-	os_offset_t	size_bytes;
 	bool		success;
 	bool		read_only_mode;
 	fil_space_t*	space = node->space;
@@ -611,7 +610,7 @@ retry:
 			return(false);
 		}
 
-		size_bytes = os_file_get_size(node->handle);
+		os_offset_t size_bytes = os_file_get_size(node->handle);
 		ut_a(size_bytes != (os_offset_t) -1);
 
 		ut_a(space->purpose != FIL_TYPE_LOG);
@@ -694,20 +693,17 @@ retry:
 		space->free_len = free_len;
 
 		if (first_time_open) {
-			ulint	extent_size;
-
-			extent_size = psize * FSP_EXTENT_SIZE;
-
-			/* After apply-incremental, tablespaces are not extended
-			to a whole megabyte. Do not cut off valid data. */
-
 			/* Truncate the size to a multiple of extent size. */
-			if (size_bytes >= extent_size) {
-				size_bytes = ut_2pow_round(size_bytes,
-							   extent_size);
+			ulint	mask = psize * FSP_EXTENT_SIZE - 1;
+
+			if (size_bytes <= mask) {
+				/* .ibd files start smaller than an
+				extent size. Do not truncate valid data. */
+			} else {
+				size_bytes &= ~os_offset_t(mask);
 			}
 
-			node->size = static_cast<ulint>(size_bytes / psize);
+			node->size = ulint(size_bytes / psize);
 			space->size += node->size;
 		}
 	}

storage/rocksdb/ha_rocksdb.h

@@ -1412,6 +1412,6 @@ private:
   Rdb_inplace_alter_ctx &operator=(const Rdb_inplace_alter_ctx &);
 };
 
-const int MYROCKS_MARIADB_PLUGIN_MATURITY_LEVEL= MariaDB_PLUGIN_MATURITY_ALPHA;
+const int MYROCKS_MARIADB_PLUGIN_MATURITY_LEVEL= MariaDB_PLUGIN_MATURITY_BETA;
 
 } // namespace myrocks

storage/rocksdb/mysql-test/rocksdb/r/information_schema.result

@@ -1,6 +1,22 @@
 DROP TABLE IF EXISTS t1;
 DROP TABLE IF EXISTS t2;
 DROP TABLE IF EXISTS t3;
+SET GLOBAL ROCKSDB_PAUSE_BACKGROUND_WORK=1;
+create table t1 (a int) engine=rocksdb;
+drop table t1;
+select * from INFORMATION_SCHEMA.ROCKSDB_GLOBAL_INFO;
+TYPE	NAME	VALUE
+MAX_INDEX_ID	MAX_INDEX_ID	max_index_id
+CF_FLAGS	0	default [0]
+CF_FLAGS	1	__system__ [0]
+DDL_DROP_INDEX_ONGOING	cf_id:0,index_id:max_index_id	
+select count(*) from INFORMATION_SCHEMA.ROCKSDB_GLOBAL_INFO;
+count(*)
+4
+SET GLOBAL ROCKSDB_PAUSE_BACKGROUND_WORK=0;
+select VALUE into @keysIn from INFORMATION_SCHEMA.ROCKSDB_COMPACTION_STATS where CF_NAME = 'default' and LEVEL = 'Sum' and TYPE = 'KeyIn';
+CREATE TABLE t1 (i1 INT, i2 INT, PRIMARY KEY (i1)) ENGINE = ROCKSDB;
+INSERT INTO t1 VALUES (1, 1), (2, 2), (3, 3);
 select * from INFORMATION_SCHEMA.ROCKSDB_GLOBAL_INFO;
 TYPE	NAME	VALUE
 MAX_INDEX_ID	MAX_INDEX_ID	max_index_id
@@ -9,20 +25,6 @@ CF_FLAGS	1	__system__ [0]
 select count(*) from INFORMATION_SCHEMA.ROCKSDB_GLOBAL_INFO;
 count(*)
 3
-select VALUE into @keysIn from INFORMATION_SCHEMA.ROCKSDB_COMPACTION_STATS where CF_NAME = 'default' and LEVEL = 'Sum' and TYPE = 'KeyIn';
-CREATE TABLE t1 (i1 INT, i2 INT, PRIMARY KEY (i1)) ENGINE = ROCKSDB;
-INSERT INTO t1 VALUES (1, 1), (2, 2), (3, 3);
-select * from INFORMATION_SCHEMA.ROCKSDB_GLOBAL_INFO;
-TYPE	NAME	VALUE
-BINLOG	FILE	master-bin.000001
-BINLOG	POS	1066
-BINLOG	GTID	uuid:5
-MAX_INDEX_ID	MAX_INDEX_ID	max_index_id
-CF_FLAGS	0	default [0]
-CF_FLAGS	1	__system__ [0]
-select count(*) from INFORMATION_SCHEMA.ROCKSDB_GLOBAL_INFO;
-count(*)
-6
 set global rocksdb_force_flush_memtable_now = true;
 set global rocksdb_compact_cf='default';
 select case when VALUE-@keysIn >= 3 then 'true' else 'false' end from INFORMATION_SCHEMA.ROCKSDB_COMPACTION_STATS where CF_NAME = 'default' and LEVEL = 'Sum' and TYPE = 'KeyIn';
@@ -66,7 +68,7 @@ SHOW GLOBAL VARIABLES LIKE 'ROCKSDB_PAUSE_BACKGROUND_WORK';
 Variable_name	Value
 rocksdb_pause_background_work	ON
 DROP TABLE t3;
-cf_id:0,index_id:267
+cf_id:0,index_id:264
 SET GLOBAL ROCKSDB_PAUSE_BACKGROUND_WORK=0;
 SHOW GLOBAL VARIABLES LIKE 'ROCKSDB_PAUSE_BACKGROUND_WORK';
 Variable_name	Value

storage/rocksdb/mysql-test/rocksdb/r/mariadb_port_fixes.result

@@ -69,15 +69,15 @@ set global rocksdb_strict_collation_check=@tmp_rscc;
 #
 select plugin_name, plugin_maturity from information_schema.plugins where plugin_name like '%rocksdb%';
 plugin_name	plugin_maturity
-ROCKSDB	Alpha
-ROCKSDB_CFSTATS	Alpha
-ROCKSDB_DBSTATS	Alpha
-ROCKSDB_PERF_CONTEXT	Alpha
-ROCKSDB_PERF_CONTEXT_GLOBAL	Alpha
-ROCKSDB_CF_OPTIONS	Alpha
-ROCKSDB_COMPACTION_STATS	Alpha
-ROCKSDB_GLOBAL_INFO	Alpha
-ROCKSDB_DDL	Alpha
-ROCKSDB_INDEX_FILE_MAP	Alpha
-ROCKSDB_LOCKS	Alpha
-ROCKSDB_TRX	Alpha
+ROCKSDB	Beta
+ROCKSDB_CFSTATS	Beta
+ROCKSDB_DBSTATS	Beta
+ROCKSDB_PERF_CONTEXT	Beta
+ROCKSDB_PERF_CONTEXT_GLOBAL	Beta
+ROCKSDB_CF_OPTIONS	Beta
+ROCKSDB_COMPACTION_STATS	Beta
+ROCKSDB_GLOBAL_INFO	Beta
+ROCKSDB_DDL	Beta
+ROCKSDB_INDEX_FILE_MAP	Beta
+ROCKSDB_LOCKS	Beta
+ROCKSDB_TRX	Beta

storage/rocksdb/mysql-test/rocksdb/t/disabled.def

@@ -69,11 +69,12 @@ compact_deletes: MDEV-12663 : rocksdb.compact_deletes times out and causes other
 blind_delete_without_tx_api: MDEV-12286: rocksdb.blind_delete_without_tx_api test fails
 unique_check: wrong error number
 autoinc_vars_thread: debug sync point wait timed out
+information_schema: MDEV-14372: unstable testcase
 
 ##
 ## Tests that fail for some other reason
 ##
 
-information_schema : MariaRocks: requires GTIDs
 mysqlbinlog_gtid_skip_empty_trans_rocksdb : MariaRocks: requires GTIDs
 rpl_row_triggers : MariaRocks: requires GTIDs
+

storage/rocksdb/mysql-test/rocksdb/t/information_schema-master.opt

@@ -1 +1 @@
---binlog_format=row --gtid_mode=ON --enforce_gtid_consistency --log_slave_updates
+--binlog_format=row --log_slave_updates

storage/rocksdb/mysql-test/rocksdb/t/information_schema.test

@@ -9,19 +9,29 @@ DROP TABLE IF EXISTS t2;
 DROP TABLE IF EXISTS t3;
 --enable_warnings
 
+# MariaDB: the following is for handling the case where the tests
+#   is started on a totally empty datadir, where no MyRocks table has
+#   ever been created). In that case, there is no MAX_INDEX_ID.
+#   Create/drop a table so that we do have MAX_INDEX_ID.
+SET GLOBAL ROCKSDB_PAUSE_BACKGROUND_WORK=1;
+create table t1 (a int) engine=rocksdb;
+drop table t1;
+
 --let $max_index_id = query_get_value(SELECT * from INFORMATION_SCHEMA.ROCKSDB_GLOBAL_INFO where type = 'MAX_INDEX_ID', VALUE, 1)
 --replace_result $max_index_id max_index_id
 select * from INFORMATION_SCHEMA.ROCKSDB_GLOBAL_INFO;
 select count(*) from INFORMATION_SCHEMA.ROCKSDB_GLOBAL_INFO;
+SET GLOBAL ROCKSDB_PAUSE_BACKGROUND_WORK=0;
 
 select VALUE into @keysIn from INFORMATION_SCHEMA.ROCKSDB_COMPACTION_STATS where CF_NAME = 'default' and LEVEL = 'Sum' and TYPE = 'KeyIn';
 
 CREATE TABLE t1 (i1 INT, i2 INT, PRIMARY KEY (i1)) ENGINE = ROCKSDB;
 INSERT INTO t1 VALUES (1, 1), (2, 2), (3, 3);
 
---let $MASTER_UUID = query_get_value(SELECT @@SERVER_UUID, @@SERVER_UUID, 1)
+# No binlog coordinates in MariaDB: --let $MASTER_UUID = query_get_value(SELECT @@SERVER_UUID, @@SERVER_UUID, 1)
 --let $max_index_id = query_get_value(SELECT * from INFORMATION_SCHEMA.ROCKSDB_GLOBAL_INFO where type = 'MAX_INDEX_ID', VALUE, 1)
---replace_result $MASTER_UUID uuid $max_index_id max_index_id
+# No binlog coordinates in MariaDB: --replace_result $MASTER_UUID uuid $max_index_id max_index_id
+--replace_result $max_index_id max_index_id
 select * from INFORMATION_SCHEMA.ROCKSDB_GLOBAL_INFO;
 select count(*) from INFORMATION_SCHEMA.ROCKSDB_GLOBAL_INFO;
 

storage/rocksdb/mysql-test/rocksdb_sys_vars/include/rocksdb_sys_var.inc

@@ -10,6 +10,7 @@
 --eval SET @start_global_value = @@global.$sys_var
 if (!$suppress_default_value)
 {
+  --replace_regex /[a-f0-9]{40}/#/
   SELECT @start_global_value;
   if ($session)
   {

storage/rocksdb/mysql-test/rocksdb_sys_vars/r/rocksdb_git_hash_basic.result

@@ -0,0 +1,7 @@
+SET @start_global_value = @@global.ROCKSDB_GIT_HASH;
+SELECT @start_global_value;
+@start_global_value
+#
+"Trying to set variable @@global.ROCKSDB_GIT_HASH to 444. It should fail because it is readonly."
+SET @@global.ROCKSDB_GIT_HASH   = 444;
+ERROR HY000: Variable 'rocksdb_git_hash' is a read only variable

storage/rocksdb/rdb_datadic.cc

@@ -4261,7 +4261,7 @@ bool Rdb_binlog_manager::read(char *const binlog_name,
     std::string value;
     rocksdb::Status status = m_dict->get_value(m_key_slice, &value);
     if (status.ok()) {
-      if (!unpack_value((const uchar *)value.c_str(), binlog_name, binlog_pos,
+      if (!unpack_value((const uchar *)value.c_str(), value.size(), binlog_name, binlog_pos,
                         binlog_gtid))
         ret = true;
     }
@@ -4331,35 +4331,60 @@ Rdb_binlog_manager::pack_value(uchar *const buf, const char *const binlog_name,
   @return     true on error
 */
 bool Rdb_binlog_manager::unpack_value(const uchar *const value,
+                                      size_t value_size_arg,
                                       char *const binlog_name,
                                       my_off_t *const binlog_pos,
                                       char *const binlog_gtid) const {
   uint pack_len = 0;
+  intmax_t value_size= value_size_arg;
 
   DBUG_ASSERT(binlog_pos != nullptr);
 
+  if ((value_size -= Rdb_key_def::VERSION_SIZE) < 0)
+    return true;
   // read version
   const uint16_t version = rdb_netbuf_to_uint16(value);
+
   pack_len += Rdb_key_def::VERSION_SIZE;
   if (version != Rdb_key_def::BINLOG_INFO_INDEX_NUMBER_VERSION)
     return true;
 
+  if ((value_size -= sizeof(uint16)) < 0)
+    return true;
+
   // read binlog file name length
   const uint16_t binlog_name_len = rdb_netbuf_to_uint16(value + pack_len);
   pack_len += sizeof(uint16);
+
+  if (binlog_name_len >= (FN_REFLEN+1))
+    return true;
+
+  if ((value_size -= binlog_name_len) < 0)
+    return true;
+
   if (binlog_name_len) {
     // read and set binlog name
     memcpy(binlog_name, value + pack_len, binlog_name_len);
     binlog_name[binlog_name_len] = '\0';
     pack_len += binlog_name_len;
 
+    if ((value_size -= sizeof(uint32)) < 0)
+      return true;
     // read and set binlog pos
     *binlog_pos = rdb_netbuf_to_uint32(value + pack_len);
     pack_len += sizeof(uint32);
 
+    if ((value_size -= sizeof(uint16)) < 0)
+      return true;
     // read gtid length
     const uint16_t binlog_gtid_len = rdb_netbuf_to_uint16(value + pack_len);
     pack_len += sizeof(uint16);
+
+    if (binlog_gtid_len >= GTID_BUF_LEN)
+      return true;
+    if ((value_size -= binlog_gtid_len) < 0)
+      return true;
+
     if (binlog_gtid && binlog_gtid_len > 0) {
       // read and set gtid
       memcpy(binlog_gtid, value + pack_len, binlog_gtid_len);

storage/rocksdb/rdb_datadic.h

@@ -46,6 +46,8 @@ class Rdb_field_packing;
 class Rdb_cf_manager;
 class Rdb_ddl_manager;
 
+const uint32_t GTID_BUF_LEN = 60;
+
 /*
   @brief
   Field packing context.
@@ -1154,7 +1156,8 @@ private:
   rocksdb::Slice pack_value(uchar *const buf, const char *const binlog_name,
                             const my_off_t &binlog_pos,
                             const char *const binlog_gtid) const;
-  bool unpack_value(const uchar *const value, char *const binlog_name,
+  bool unpack_value(const uchar *const value, size_t value_size,
+                    char *const binlog_name,
                     my_off_t *const binlog_pos, char *const binlog_gtid) const;
 
   std::atomic<Rdb_tbl_def *> m_slave_gtid_info_tbl;

storage/rocksdb/rdb_i_s.cc

@@ -729,7 +729,6 @@ static int rdb_i_s_global_info_fill_table(
   DBUG_ASSERT(tables != nullptr);
 
   static const uint32_t INT_BUF_LEN = 21;
-  static const uint32_t GTID_BUF_LEN = 60;
   static const uint32_t CF_ID_INDEX_BUF_LEN = 60;
 
   int ret = 0;