diff --git a/CMakeLists.txt b/CMakeLists.txt index 1128566e1cb..8ad049446d1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -160,6 +160,7 @@ INCLUDE(plugin) INCLUDE(install_macros) INCLUDE(systemd) INCLUDE(mysql_add_executable) +INCLUDE(compile_flags) INCLUDE(crc32-vpmsum) # Handle options diff --git a/mysql-test/lib/My/SafeProcess/Base.pm b/mysql-test/lib/My/SafeProcess/Base.pm index 1ac0120a735..c5ac2ab51c2 100644 --- a/mysql-test/lib/My/SafeProcess/Base.pm +++ b/mysql-test/lib/My/SafeProcess/Base.pm @@ -186,8 +186,10 @@ sub create_process { # it and any childs(that hasn't changed group themself) setpgrp(0,0) if $opts{setpgrp}; - if ( $output and !open(STDOUT, $open_mode, $output) ) { - croak("can't redirect STDOUT to '$output': $!"); + if ( $output ) { + close STDOUT; + open(STDOUT, $open_mode, $output) + or croak "can't redirect STDOUT to '$output': $!"; } if ( $error ) { @@ -196,8 +198,10 @@ sub create_process { croak("can't dup STDOUT: $!"); } } - elsif ( ! open(STDERR, $open_mode, $error) ) { - croak("can't redirect STDERR to '$error': $!"); + else { + close STDERR; + open(STDERR, $open_mode, $error) + or croak "can't redirect STDERR to '$error': $!"; } } diff --git a/mysql-test/lib/My/Tee.pm b/mysql-test/lib/My/Tee.pm new file mode 100644 index 00000000000..ee82e6f45ae --- /dev/null +++ b/mysql-test/lib/My/Tee.pm @@ -0,0 +1,23 @@ +package My::Tee; + +# see PerlIO::via + +our $copyfh; + +sub PUSHED +{ + open($copyfh, '>', "$::opt_vardir/log/stdout.log") + or die "open(>$::opt_vardir/log/stdout.log): $!" + unless $copyfh; + bless { }, shift; +} + +sub WRITE +{ + my ($obj, $buf, $fh) = @_; + print $fh $buf; + print $copyfh $buf; + return length($buf); +} + +1; diff --git a/mysql-test/mysql-test-run.pl b/mysql-test/mysql-test-run.pl index d3f82e281e5..5e6eeb8ede2 100755 --- a/mysql-test/mysql-test-run.pl +++ b/mysql-test/mysql-test-run.pl @@ -91,6 +91,7 @@ use My::Platform; use My::SafeProcess; use My::ConfigFactory; use My::Options; +use My::Tee; use My::Find; use My::SysInfo; use My::CoreDump; @@ -384,6 +385,11 @@ sub main { initialize_servers(); init_timers(); + unless (IS_WINDOWS) { + binmode(STDOUT,":via(My::Tee)") or die "binmode(STDOUT, :via(My::Tee)):$!"; + binmode(STDERR,":via(My::Tee)") or die "binmode(STDERR, :via(My::Tee)):$!"; + } + mtr_report("Checking supported features..."); executable_setup(); @@ -6253,7 +6259,8 @@ sub xterm_stat { my $done = $num_tests - $left; my $spent = time - $^T; - printf "\e];mtr: spent %s on %d tests. %s (%d tests) left\a", + syswrite STDOUT, sprintf + "\e];mtr: spent %s on %d tests. %s (%d tests) left\a", time_format($spent), $done, time_format($spent/$done * $left), $left; } diff --git a/mysql-test/suite/encryption/r/innodb_encryption-page-compression.result b/mysql-test/suite/encryption/r/innodb_encryption-page-compression.result index da5b5b49558..712690d2925 100644 --- a/mysql-test/suite/encryption/r/innodb_encryption-page-compression.result +++ b/mysql-test/suite/encryption/r/innodb_encryption-page-compression.result @@ -134,6 +134,13 @@ count(*) select count(*) from innodb_page_compressed9 where c1 < 500000; count(*) 2000 +flush tables innodb_page_compressed1, innodb_page_compressed2, +innodb_page_compressed3, innodb_page_compressed4, +innodb_page_compressed5, innodb_page_compressed6, +innodb_page_compressed7, innodb_page_compressed8, +innodb_page_compressed9 for export; +unlock tables; +# Wait until dirty pages are compressed and encrypted SELECT variable_value > 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_encrypted'; variable_value > 0 1 @@ -151,9 +158,14 @@ update innodb_page_compressed6 set c1 = c1 + 1; update innodb_page_compressed7 set c1 = c1 + 1; update innodb_page_compressed8 set c1 = c1 + 1; update innodb_page_compressed9 set c1 = c1 + 1; -SELECT variable_value > 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_encrypted'; -variable_value > 0 -1 +flush tables innodb_page_compressed1, innodb_page_compressed2, +innodb_page_compressed3, innodb_page_compressed4, +innodb_page_compressed5, innodb_page_compressed6, +innodb_page_compressed7, innodb_page_compressed8, +innodb_page_compressed9 for export; +unlock tables; +# Wait until dirty pages are compressed and encrypted 2 +unlock tables; SELECT variable_value > 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_decrypted'; variable_value > 0 1 diff --git a/mysql-test/suite/encryption/t/innodb_encryption-page-compression.test b/mysql-test/suite/encryption/t/innodb_encryption-page-compression.test index e52b37dee89..4f8a3373fb2 100644 --- a/mysql-test/suite/encryption/t/innodb_encryption-page-compression.test +++ b/mysql-test/suite/encryption/t/innodb_encryption-page-compression.test @@ -73,6 +73,15 @@ select count(*) from innodb_page_compressed7 where c1 < 500000; select count(*) from innodb_page_compressed8 where c1 < 500000; select count(*) from innodb_page_compressed9 where c1 < 500000; +flush tables innodb_page_compressed1, innodb_page_compressed2, +innodb_page_compressed3, innodb_page_compressed4, +innodb_page_compressed5, innodb_page_compressed6, +innodb_page_compressed7, innodb_page_compressed8, +innodb_page_compressed9 for export; + +unlock tables; + +--echo # Wait until dirty pages are compressed and encrypted let $wait_condition= select variable_value > 0 from information_schema.global_status where variable_name = 'INNODB_NUM_PAGES_PAGE_COMPRESSED'; --source include/wait_condition.inc let $wait_condition= select variable_value > 0 from information_schema.global_status where variable_name = 'INNODB_NUM_PAGES_ENCRYPTED'; @@ -96,12 +105,20 @@ update innodb_page_compressed7 set c1 = c1 + 1; update innodb_page_compressed8 set c1 = c1 + 1; update innodb_page_compressed9 set c1 = c1 + 1; -let $wait_condition= select variable_value > 0 from information_schema.global_status where variable_name = 'INNODB_NUM_PAGES_ENCRYPTED'; ---source include/wait_condition.inc +flush tables innodb_page_compressed1, innodb_page_compressed2, +innodb_page_compressed3, innodb_page_compressed4, +innodb_page_compressed5, innodb_page_compressed6, +innodb_page_compressed7, innodb_page_compressed8, +innodb_page_compressed9 for export; +unlock tables; + +--echo # Wait until dirty pages are compressed and encrypted 2 let $wait_condition= select variable_value > 0 from information_schema.global_status where variable_name = 'INNODB_NUM_PAGES_PAGE_COMPRESSED'; --source include/wait_condition.inc +unlock tables; +let $wait_condition= select variable_value > 0 from information_schema.global_status where variable_name = 'INNODB_NUM_PAGES_DECRYPTED'; +--source include/wait_condition.inc -SELECT variable_value > 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_encrypted'; SELECT variable_value > 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_decrypted'; SELECT variable_value > 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_page_compressed'; SELECT variable_value > 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_page_decompressed'; diff --git a/mysql-test/suite/galera/disabled.def b/mysql-test/suite/galera/disabled.def index bcf4a107724..463f002901a 100644 --- a/mysql-test/suite/galera/disabled.def +++ b/mysql-test/suite/galera/disabled.def @@ -61,3 +61,6 @@ MW-328B: MDEV-13549 Galera test failures 10.1 MW-328: MDEV-13549 Galera test failures 10.1 galera_suspend_slave: MDEV-13549 Galera test failures 10.1 galera_ist_progress: MDEV-15236 galera_ist_progress fails when trying to read transfer status +galera_gtid : MDEV-13549 Galera test failures 10.1 +galera_gtid_slave : MDEV-13549 Galera test failures 10.1 +galera_unicode_identifiers : MDEV-13549 Galera test failures 10.1 diff --git a/mysql-test/suite/parts/t/partition_alter_myisam.test b/mysql-test/suite/parts/t/partition_alter_myisam.test index d3abb8842e1..326cf52b018 100644 --- a/mysql-test/suite/parts/t/partition_alter_myisam.test +++ b/mysql-test/suite/parts/t/partition_alter_myisam.test @@ -1,4 +1,5 @@ --source include/have_partition.inc +--source include/have_symlink.inc --let $engine=MyISAM --source inc/part_alter_values.inc diff --git a/scripts/wsrep_sst_xtrabackup-v2.sh b/scripts/wsrep_sst_xtrabackup-v2.sh index 64dd182e2f2..9104daf19bc 100644 --- a/scripts/wsrep_sst_xtrabackup-v2.sh +++ b/scripts/wsrep_sst_xtrabackup-v2.sh @@ -644,7 +644,7 @@ wait_for_listen() for i in {1..300} do - LSOF_OUT=$(lsof -sTCP:LISTEN -i TCP:${PORT} -a -c nc -c socat -F c) + LSOF_OUT=$(lsof -sTCP:LISTEN -i TCP:${PORT} -a -c nc -c socat -F c 2> /dev/null || :) [ -n "${LSOF_OUT}" ] && break sleep 0.2 done diff --git a/sql/mysqld.cc b/sql/mysqld.cc index ab360a5bf6c..1cf15abebdc 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -6025,12 +6025,14 @@ int mysqld_main(int argc, char **argv) mysqld_port, MYSQL_COMPILATION_COMMENT); } +#ifndef _WIN32 // try to keep fd=0 busy - if (!freopen(IF_WIN("NUL","/dev/null"), "r", stdin)) + if (!freopen("/dev/null", "r", stdin)) { // fall back on failure fclose(stdin); } +#endif #if defined(_WIN32) && !defined(EMBEDDED_LIBRARY) Service.SetRunning(); diff --git a/sql/partition_info.cc b/sql/partition_info.cc index d66f011e5d7..bddc66189e4 100644 --- a/sql/partition_info.cc +++ b/sql/partition_info.cc @@ -2383,23 +2383,6 @@ end: } -bool partition_info::error_if_requires_values() const -{ - switch (part_type) { - case NOT_A_PARTITION: - case HASH_PARTITION: - break; - case RANGE_PARTITION: - my_error(ER_PARTITION_REQUIRES_VALUES_ERROR, MYF(0), "RANGE", "LESS THAN"); - return true; - case LIST_PARTITION: - my_error(ER_PARTITION_REQUIRES_VALUES_ERROR, MYF(0), "LIST", "IN"); - return true; - } - return false; -} - - /** Fix partition data from parser. @@ -2893,3 +2876,19 @@ bool check_partition_dirs(partition_info *part_info) } #endif /* WITH_PARTITION_STORAGE_ENGINE */ + +bool partition_info::error_if_requires_values() const +{ + switch (part_type) { + case NOT_A_PARTITION: + case HASH_PARTITION: + break; + case RANGE_PARTITION: + my_error(ER_PARTITION_REQUIRES_VALUES_ERROR, MYF(0), "RANGE", "LESS THAN"); + return true; + case LIST_PARTITION: + my_error(ER_PARTITION_REQUIRES_VALUES_ERROR, MYF(0), "LIST", "IN"); + return true; + } + return false; +} diff --git a/storage/connect/ha_connect.cc b/storage/connect/ha_connect.cc index ae7335d2904..51b249c4ab0 100644 --- a/storage/connect/ha_connect.cc +++ b/storage/connect/ha_connect.cc @@ -1754,11 +1754,13 @@ bool ha_connect::CheckVirtualIndex(TABLE_SHARE *s) bool ha_connect::IsPartitioned(void) { +#ifdef WITH_PARTITION_STORAGE_ENGINE if (tshp) return tshp->partition_info_str_len > 0; else if (table && table->part_info) return true; else +#endif return false; } // end of IsPartitioned @@ -6165,8 +6167,10 @@ int ha_connect::create(const char *name, TABLE *table_arg, TABLE *st= table; // Probably unuseful THD *thd= ha_thd(); LEX_STRING cnc = table_arg->s->connect_string; -#if defined(WITH_PARTITION_STORAGE_ENGINE) +#ifdef WITH_PARTITION_STORAGE_ENGINE partition_info *part_info= table_arg->part_info; +#else +#define part_info 0 #endif // WITH_PARTITION_STORAGE_ENGINE xp= GetUser(thd, xp); PGLOBAL g= xp->g; @@ -6268,9 +6272,7 @@ int ha_connect::create(const char *name, TABLE *table_arg, // fall through case TAB_MYSQL: -#if defined(WITH_PARTITION_STORAGE_ENGINE) if (!part_info) -#endif // WITH_PARTITION_STORAGE_ENGINE {const char *src= options->srcdef; PCSZ host, db, tab= options->tabname; int port; @@ -6534,7 +6536,6 @@ int ha_connect::create(const char *name, TABLE *table_arg, } else lwt[i]= tolower(options->type[i]); -#if defined(WITH_PARTITION_STORAGE_ENGINE) if (part_info) { char *p; @@ -6544,7 +6545,6 @@ int ha_connect::create(const char *name, TABLE *table_arg, strcat(strcat(strcpy(buf, p), "."), lwt); *p= 0; } else { -#endif // WITH_PARTITION_STORAGE_ENGINE strcat(strcat(strcpy(buf, GetTableName()), "."), lwt); sprintf(g->Message, "No file name. Table will use %s", buf); @@ -6552,9 +6552,7 @@ int ha_connect::create(const char *name, TABLE *table_arg, push_warning(thd, Sql_condition::WARN_LEVEL_WARN, 0, g->Message); strcat(strcat(strcpy(dbpath, "./"), table->s->db.str), "/"); -#if defined(WITH_PARTITION_STORAGE_ENGINE) } // endif part_info -#endif // WITH_PARTITION_STORAGE_ENGINE PlugSetPath(fn, buf, dbpath); @@ -6619,11 +6617,9 @@ int ha_connect::create(const char *name, TABLE *table_arg, push_warning(thd, Sql_condition::WARN_LEVEL_WARN, 0, "Unexpected command in create, please contact CONNECT team"); -#if defined(WITH_PARTITION_STORAGE_ENGINE) if (part_info && !inward) strncpy(partname, decode(g, strrchr(name, '#') + 1), sizeof(partname) - 1); // strcpy(partname, part_info->curr_part_elem->partition_name); -#endif // WITH_PARTITION_STORAGE_ENGINE if (g->Alchecked == 0 && (!IsFileType(type) || FileExists(options->filename, false))) { @@ -6659,12 +6655,10 @@ int ha_connect::create(const char *name, TABLE *table_arg, my_message(ER_UNKNOWN_ERROR, g->Message, MYF(0)); rc = HA_ERR_INTERNAL_ERROR; } else if (cat) { -#if defined(WITH_PARTITION_STORAGE_ENGINE) if (part_info) strncpy(partname, decode(g, strrchr(name, (inward ? slash : '#')) + 1), sizeof(partname) - 1); -#endif // WITH_PARTITION_STORAGE_ENGINE if ((rc= optimize(table->in_use, NULL))) { htrc("Create rc=%d %s\n", rc, g->Message); diff --git a/storage/maria/ma_bitmap.c b/storage/maria/ma_bitmap.c index d50df294b8a..0857f25352d 100644 --- a/storage/maria/ma_bitmap.c +++ b/storage/maria/ma_bitmap.c @@ -145,6 +145,11 @@ static my_bool _ma_bitmap_create_missing(MARIA_HA *info, MARIA_FILE_BITMAP *bitmap, pgcache_page_no_t page); static void _ma_bitmap_unpin_all(MARIA_SHARE *share); +#ifndef DBUG_OFF +static void _ma_check_bitmap(MARIA_FILE_BITMAP *bitmap); +#else +#define _ma_check_bitmap(A) do { } while(0) +#endif /* Write bitmap page to key cache */ @@ -267,6 +272,13 @@ my_bool _ma_bitmap_init(MARIA_SHARE *share, File file, bitmap->sizes[6]= max_page_size - max_page_size * 80 / 100; bitmap->sizes[7]= 0; + /* + If a record size will fit into the smallest empty page, return first + found page in find_head() + */ + if (bitmap->sizes[3] >= share->base.max_pack_length) + bitmap->return_first_match= 1; + mysql_mutex_init(key_SHARE_BITMAP_lock, &share->bitmap.bitmap_lock, MY_MUTEX_INIT_SLOW); mysql_cond_init(key_SHARE_BITMAP_cond, @@ -677,7 +689,8 @@ void _ma_bitmap_delete_all(MARIA_SHARE *share) bzero(bitmap->map, bitmap->block_size); bitmap->changed= 1; bitmap->page= 0; - bitmap->used_size= bitmap->total_size= bitmap->max_total_size; + bitmap->used_size= bitmap->full_tail_size= bitmap->full_head_size= 0; + bitmap->total_size= bitmap->max_total_size; } DBUG_VOID_RETURN; } @@ -715,6 +728,7 @@ void _ma_bitmap_reset_cache(MARIA_SHARE *share) */ bitmap->page= ((pgcache_page_no_t) 0) - bitmap->pages_covered; bitmap->used_size= bitmap->total_size= bitmap->max_total_size; + bitmap->full_head_size= bitmap->full_tail_size= bitmap->max_total_size; bfill(bitmap->map, share->block_size, 255); #ifndef DBUG_OFF memcpy(bitmap->map + bitmap->block_size, bitmap->map, bitmap->block_size); @@ -1016,9 +1030,6 @@ static void adjust_total_size(MARIA_HA *info, pgcache_page_no_t page) bitmap Bitmap handler page Page to read - TODO - Update 'bitmap->used_size' to real size of used bitmap - NOTE We don't always have share->bitmap.bitmap_lock here (when called from_ma_check_bitmap_data() for example). @@ -1035,6 +1046,9 @@ static my_bool _ma_read_bitmap_page(MARIA_HA *info, MARIA_SHARE *share= info->s; my_bool res; DBUG_ENTER("_ma_read_bitmap_page"); + DBUG_PRINT("enter", ("page: %lld data_file_length: %lld", + (longlong) page, + (longlong) share->state.state.data_file_length)); DBUG_ASSERT(page % bitmap->pages_covered == 0); DBUG_ASSERT(!bitmap->changed); @@ -1049,13 +1063,22 @@ static my_bool _ma_read_bitmap_page(MARIA_HA *info, } adjust_total_size(info, page); - bitmap->used_size= bitmap->total_size; + bitmap->full_head_size= bitmap->full_tail_size= 0; DBUG_ASSERT(share->pagecache->block_size == bitmap->block_size); res= pagecache_read(share->pagecache, &bitmap->file, page, 0, bitmap->map, PAGECACHE_PLAIN_PAGE, PAGECACHE_LOCK_LEFT_UNLOCKED, 0) == NULL; + if (!res) + { + /* Calculate used_size */ + const uchar *data, *end= bitmap->map; + for (data= bitmap->map + bitmap->total_size; --data >= end && *data == 0; ) + {} + bitmap->used_size= (uint) ((data + 1) - end); + DBUG_ASSERT(bitmap->used_size <= bitmap->total_size); + } /* We can't check maria_bitmap_marker here as if the bitmap page previously had a true checksum and the user switched mode to not checksum @@ -1067,7 +1090,10 @@ static my_bool _ma_read_bitmap_page(MARIA_HA *info, #ifndef DBUG_OFF if (!res) + { memcpy(bitmap->map + bitmap->block_size, bitmap->map, bitmap->block_size); + _ma_check_bitmap(bitmap); + } #endif DBUG_RETURN(res); } @@ -1097,6 +1123,8 @@ static my_bool _ma_change_bitmap_page(MARIA_HA *info, { DBUG_ENTER("_ma_change_bitmap_page"); + _ma_check_bitmap(bitmap); + /* We have to mark the file changed here, as otherwise the following read/write to pagecache may force a page out from this file, which would @@ -1228,6 +1256,9 @@ static void fill_block(MARIA_FILE_BITMAP *bitmap, This is defined as the first page of the set of pages with the smallest free space that can hold 'size'. + NOTES + Updates bitmap->full_head_size while scanning data + RETURN 0 ok (block is updated) 1 error (no space in bitmap; block is not touched) @@ -1238,10 +1269,11 @@ static my_bool allocate_head(MARIA_FILE_BITMAP *bitmap, uint size, MARIA_BITMAP_BLOCK *block) { uint min_bits= size_to_head_pattern(bitmap, size); - uchar *data= bitmap->map, *end= data + bitmap->used_size; + uchar *data, *end; uchar *best_data= 0; uint best_bits= (uint) -1, UNINIT_VAR(best_pos); - uint first_pattern= 0; /* if doing insert_order */ + my_bool first_pattern= 0; /* if doing insert_order */ + my_bool first_found= 1; MARIA_SHARE *share= bitmap->share; my_bool insert_order= MY_TEST(share->base.extra_options & MA_EXTRA_OPTIONS_INSERT_ORDER); @@ -1249,16 +1281,19 @@ static my_bool allocate_head(MARIA_FILE_BITMAP *bitmap, uint size, DBUG_ASSERT(size <= FULL_PAGE_SIZE(share)); + end= bitmap->map + bitmap->used_size; if (insert_order && bitmap->page == share->last_insert_bitmap) { uint last_insert_page= share->last_insert_page; uint byte= 6 * (last_insert_page / 16); first_pattern= last_insert_page % 16; - DBUG_ASSERT(data + byte < end); - data+= byte; + data= bitmap->map+byte; + DBUG_ASSERT(data <= end); } + else + data= bitmap->map + (bitmap->full_head_size/6)*6; - for (; data < end; data+= 6) + for (; data < end; data+= 6, first_pattern= 0) { ulonglong bits= uint6korr(data); /* 6 bytes = 6*8/3= 16 patterns */ uint i; @@ -1271,17 +1306,24 @@ static my_bool allocate_head(MARIA_FILE_BITMAP *bitmap, uint size, */ if ((!bits && best_data) || ((bits & 04444444444444444LL) == 04444444444444444LL)) - { - first_pattern= 0; // always restart from 0 when moving to new 6-byte continue; - } + for (i= first_pattern, bits >>= (3 * first_pattern); i < 16 ; i++, bits >>= 3) { uint pattern= (uint) (bits & 7); + + if (pattern <= 3) /* Room for more data */ + { + if (first_found) + { + first_found= 0; + bitmap->full_head_size= (uint)(data - bitmap->map); + } + } if (pattern <= min_bits) { - /* There is enough space here */ + /* There is enough space here, check if we have found better */ if ((int) pattern > (int) best_bits) { /* @@ -1292,23 +1334,32 @@ static my_bool allocate_head(MARIA_FILE_BITMAP *bitmap, uint size, best_bits= pattern; best_data= data; best_pos= i; - if (pattern == min_bits) + if (pattern == min_bits || bitmap->return_first_match) goto found; /* Best possible match */ } } } - first_pattern= 0; // always restart from 0 when moving to new 6-byte } if (!best_data) /* Found no place */ { if (data >= bitmap->map + bitmap->total_size) DBUG_RETURN(1); /* No space in bitmap */ + DBUG_ASSERT(uint6korr(data) == 0); /* Allocate data at end of bitmap */ - bitmap->used_size+= 6; - set_if_smaller(bitmap->used_size, bitmap->total_size); + bitmap->used_size= (uint) (data - bitmap->map) + 6; best_data= data; best_pos= best_bits= 0; } + else + { + /* + This is not stricly needed as used_size should be alligned on 6, + but for easier debugging lets try to keep it more accurate + */ + uint position= (uint) (best_data - bitmap->map) + 6; + set_if_bigger(bitmap->used_size, position); + } + DBUG_ASSERT(bitmap->used_size <= bitmap->total_size); found: if (insert_order) @@ -1341,12 +1392,15 @@ static my_bool allocate_tail(MARIA_FILE_BITMAP *bitmap, uint size, MARIA_BITMAP_BLOCK *block) { uint min_bits= size_to_tail_pattern(bitmap, size); - uchar *data= bitmap->map, *end= data + bitmap->used_size; - uchar *best_data= 0; + uchar *data, *end, *best_data= 0; + my_bool first_found= 1; uint best_bits= (uint) -1, UNINIT_VAR(best_pos); DBUG_ENTER("allocate_tail"); DBUG_PRINT("enter", ("size: %u", size)); + data= bitmap->map + (bitmap->full_tail_size/6)*6; + end= bitmap->map + bitmap->used_size; + /* We have to add DIR_ENTRY_SIZE here as this is not part of the data size See call to allocate_tail() in find_tail(). @@ -1375,7 +1429,19 @@ static my_bool allocate_tail(MARIA_FILE_BITMAP *bitmap, uint size, for (i= 0; i < 16; i++, bits >>= 3) { uint pattern= (uint) (bits & 7); - if (pattern <= min_bits && (!pattern || pattern >= 5)) + + if (pattern == 0 || + (pattern > FULL_HEAD_PAGE && pattern < FULL_TAIL_PAGE)) + { + /* There is room for tail data */ + if (first_found) + { + first_found= 0; + bitmap->full_tail_size= (uint)(data - bitmap->map); + } + } + + if (pattern <= min_bits && (!pattern || pattern > FULL_HEAD_PAGE)) { if ((int) pattern > (int) best_bits) { @@ -1392,10 +1458,11 @@ static my_bool allocate_tail(MARIA_FILE_BITMAP *bitmap, uint size, { if (data >= bitmap->map + bitmap->total_size) DBUG_RETURN(1); + DBUG_ASSERT(uint6korr(data) == 0); /* Allocate data at end of bitmap */ best_data= data; - bitmap->used_size+= 6; - set_if_smaller(bitmap->used_size, bitmap->total_size); + bitmap->used_size= (uint) (data - bitmap->map) + 6; + DBUG_ASSERT(bitmap->used_size <= bitmap->total_size); best_pos= best_bits= 0; } @@ -1434,8 +1501,7 @@ static ulong allocate_full_pages(MARIA_FILE_BITMAP *bitmap, ulong pages_needed, MARIA_BITMAP_BLOCK *block, my_bool full_page) { - uchar *data= bitmap->map, *data_end= data + bitmap->used_size; - uchar *page_end= data + bitmap->total_size; + uchar *data, *data_end, *page_end; uchar *best_data= 0; uint min_size; uint best_area_size, UNINIT_VAR(best_prefix_area_size); @@ -1449,6 +1515,10 @@ static ulong allocate_full_pages(MARIA_FILE_BITMAP *bitmap, min_size= BLOB_SEGMENT_MIN_SIZE; best_area_size= ~(uint) 0; + data= bitmap->map + (bitmap->full_head_size/6)*6; + data_end= bitmap->map + bitmap->used_size; + page_end= bitmap->map + bitmap->total_size; + for (; data < page_end; data+= 6) { ulonglong bits= uint6korr(data); /* 6 bytes = 6*8/3= 16 patterns */ @@ -1466,6 +1536,12 @@ static ulong allocate_full_pages(MARIA_FILE_BITMAP *bitmap, if ((bits= uint6korr(data))) break; } + /* + Check if we are end of bitmap. In this case we know that + the rest of the bitmap is usable + */ + if (data >= data_end) + data= page_end; area_size= (uint) (data - data_start) / 6 * 16; if (area_size >= best_area_size) continue; @@ -1823,7 +1899,7 @@ static my_bool allocate_blobs(MARIA_HA *info, MARIA_ROW *row) /* - Store in the bitmap the new size for a head page + Reserve the current head page SYNOPSIS use_head() @@ -2225,7 +2301,7 @@ static my_bool set_page_bits(MARIA_HA *info, MARIA_FILE_BITMAP *bitmap, pgcache_page_no_t page, uint fill_pattern) { pgcache_page_no_t bitmap_page; - uint offset_page, offset, tmp, org_tmp; + uint offset_page, offset, tmp, org_tmp, used_offset; uchar *data; DBUG_ENTER("set_page_bits"); DBUG_ASSERT(fill_pattern <= 7); @@ -2237,6 +2313,7 @@ static my_bool set_page_bits(MARIA_HA *info, MARIA_FILE_BITMAP *bitmap, /* Find page number from start of bitmap */ offset_page= (uint) (page - bitmap->page - 1); + /* Mark place used by reading/writing 2 bytes at a time to handle bitmaps in overlapping bytes @@ -2248,11 +2325,37 @@ static my_bool set_page_bits(MARIA_HA *info, MARIA_FILE_BITMAP *bitmap, tmp= (tmp & ~(7 << offset)) | (fill_pattern << offset); if (tmp == org_tmp) DBUG_RETURN(0); /* No changes */ - int2store(data, tmp); + /* + Take care to not write bytes outside of bitmap. + fill_pattern is 3 bits, so we need to write two bytes + if bit position we write to is > (8-3) + */ + if (offset > 5) + int2store(data, tmp); + else + data[0]= tmp; + + /* + Reset full_head_size or full_tail_size if we are releasing data before + it. Increase used_size if we are allocating data. + */ + used_offset= (uint) (data - bitmap->map); + if (fill_pattern < 4) + set_if_smaller(bitmap->full_head_size, used_offset); + if (fill_pattern == 0 || (fill_pattern > 4 && fill_pattern < 7)) + set_if_smaller(bitmap->full_tail_size, used_offset); + if (fill_pattern != 0) + { + /* Calulcate which was the last changed byte */ + used_offset+= offset > 5 ? 2 : 1; + set_if_bigger(bitmap->used_size, used_offset); + } + + _ma_check_bitmap(bitmap); bitmap->changed= 1; DBUG_EXECUTE("bitmap", _ma_print_bitmap_changes(bitmap);); - if (fill_pattern != 3 && fill_pattern != 7) + if (fill_pattern != FULL_HEAD_PAGE && fill_pattern != FULL_TAIL_PAGE) set_if_smaller(info->s->state.first_bitmap_with_space, bitmap_page); /* Note that if the condition above is false (page is full), and all pages of @@ -2345,7 +2448,7 @@ my_bool _ma_bitmap_reset_full_page_bits(MARIA_HA *info, uint page_count) { ulonglong bitmap_page; - uint offset, bit_start, bit_count, tmp; + uint offset, bit_start, bit_count, tmp, byte_offset; uchar *data; DBUG_ENTER("_ma_bitmap_reset_full_page_bits"); DBUG_PRINT("enter", ("page: %lu page_count: %u", (ulong) page, page_count)); @@ -2365,7 +2468,8 @@ my_bool _ma_bitmap_reset_full_page_bits(MARIA_HA *info, bit_start= offset * 3; bit_count= page_count * 3; - data= bitmap->map + bit_start / 8; + byte_offset= bit_start/8; + data= bitmap->map + byte_offset; offset= bit_start & 7; tmp= (255 << offset); /* Bits to keep */ @@ -2376,6 +2480,9 @@ my_bool _ma_bitmap_reset_full_page_bits(MARIA_HA *info, } *data&= ~tmp; + set_if_smaller(bitmap->full_head_size, byte_offset); + set_if_smaller(bitmap->full_tail_size, byte_offset); + if ((int) (bit_count-= (8 - offset)) > 0) { uint fill; @@ -2477,6 +2584,8 @@ my_bool _ma_bitmap_set_full_page_bits(MARIA_HA *info, tmp= (1 << bit_count) - 1; *data|= tmp; } + set_if_bigger(bitmap->used_size, (uint) (data - bitmap->map) + 1); + _ma_check_bitmap(bitmap); bitmap->changed= 1; DBUG_EXECUTE("bitmap", _ma_print_bitmap_changes(bitmap);); DBUG_RETURN(0); @@ -2835,6 +2944,72 @@ my_bool _ma_check_bitmap_data(MARIA_HA *info, enum en_page_type page_type, return (bitmap_pattern != bits); } +/** + Check that bitmap looks correct + + - All data before full_head_size and full_tail_size are allocated + - There is no allocated data after used_size + All of the above need to be correct only according to 6 byte + alignment as all loops reads 6 bytes at a time and we check both + start and end position according to the current 6 byte position. +*/ + +#ifndef DBUG_OFF +static void _ma_check_bitmap(MARIA_FILE_BITMAP *bitmap) +{ + uchar *data= bitmap->map; + uchar *end= bitmap->map + bitmap->total_size; + uchar *full_head_end=0, *full_tail_end=0, *first_empty= bitmap->map; + + for (; data < end; data+= 6) + { + ulonglong bits= uint6korr(data); /* 6 bytes = 6*8/3= 16 patterns */ + uint i; + + if (bits == 04444444444444444LL || bits == 0xffffffffffffLL) + { + first_empty= data + 6; + continue; /* block fully used */ + } + if (bits == 0) + { + if (!full_head_end) + full_head_end= data; + if (!full_tail_end) + full_tail_end= data; + continue; + } + + first_empty= data + 6; + if (!full_head_end || !full_tail_end) + { + for (i= 0, bits >>= 0; i < 16 ; i++, bits >>= 3) + { + uint pattern= (uint) (bits & 7); + if (pattern == FULL_HEAD_PAGE || pattern == FULL_TAIL_PAGE) + continue; + + if (pattern < 4 && !full_head_end) + full_head_end= data; + if ((pattern == 0 || (pattern > 4 && pattern < 7)) && !full_tail_end) + full_tail_end= data; + } + } + } + if (!full_head_end) + full_head_end= data; + if (!full_tail_end) + full_tail_end= data; + + /* used_size must point after the last byte that had some data) */ + DBUG_ASSERT(bitmap->used_size <= bitmap->total_size); + DBUG_ASSERT((bitmap->map + (bitmap->used_size+5)/6*6) >= first_empty); + /* full_xxxx_size can't point after the first block that has free data */ + DBUG_ASSERT((bitmap->map + (bitmap->full_head_size/6*6)) <= full_head_end); + DBUG_ASSERT((bitmap->map + (bitmap->full_tail_size/6*6)) <= full_tail_end); +} +#endif + /* Check if the page type matches the one that we have in the bitmap @@ -3072,6 +3247,7 @@ static my_bool _ma_bitmap_create_missing(MARIA_HA *info, pgcache_page_no_t from, to; my_off_t data_file_length= share->state.state.data_file_length; DBUG_ENTER("_ma_bitmap_create_missing"); + DBUG_PRINT("enter", ("page: %lld", (longlong) page)); /* First (in offset order) bitmap page to create */ if (data_file_length < block_size) @@ -3124,7 +3300,8 @@ static my_bool _ma_bitmap_create_missing(MARIA_HA *info, only later as we are going to modify it very soon. */ bzero(bitmap->map, bitmap->block_size); - bitmap->used_size= 0; + bitmap->used_size= bitmap->full_head_size= bitmap->full_tail_size= 0; + bitmap->changed=1; #ifndef DBUG_OFF /* Make a copy of the page to be able to print out bitmap changes during diff --git a/storage/maria/maria_def.h b/storage/maria/maria_def.h index a35403fff39..99ef3e18cc1 100644 --- a/storage/maria/maria_def.h +++ b/storage/maria/maria_def.h @@ -331,7 +331,10 @@ typedef struct st_maria_file_bitmap pgcache_page_no_t last_bitmap_page; /* Last possible bitmap page */ my_bool changed; /* 1 if page needs to be written */ my_bool changed_not_flushed; /* 1 if some bitmap is not flushed */ + my_bool return_first_match; /* Shortcut find_head() */ uint used_size; /* Size of bitmap head that is not 0 */ + uint full_head_size; /* Where to start search for head */ + uint full_tail_size; /* Where to start search for tail */ uint flush_all_requested; /**< If _ma_bitmap_flush_all waiting */ uint waiting_for_flush_all_requested; /* If someone is waiting for above */ uint non_flushable; /**< 0 if bitmap and log are in sync */ diff --git a/storage/tokudb/CMakeLists.txt b/storage/tokudb/CMakeLists.txt index 702b2eaa2dd..aa5949b4860 100644 --- a/storage/tokudb/CMakeLists.txt +++ b/storage/tokudb/CMakeLists.txt @@ -4,6 +4,8 @@ IF(CMAKE_VERSION VERSION_LESS "2.8.9") MESSAGE(STATUS "CMake 2.8.9 or higher is required by TokuDB") ELSEIF(NOT HAVE_DLOPEN) MESSAGE(STATUS "dlopen is required by TokuDB") +ELSEIF(NOT TARGET perfschema) + MESSAGE(STATUS "Performance Schema is required by TokuDB") ELSEIF(CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" OR CMAKE_SYSTEM_PROCESSOR STREQUAL "amd64") # tokudb requires F_NOCACHE or O_DIRECT, and designated initializers