mariadb/storage/innobase/srv/srv0start.cc

3366 lines
86 KiB
C++
Raw Normal View History

/*****************************************************************************
2017-05-15 17:17:16 +03:00
Copyright (c) 1996, 2017, Oracle and/or its affiliates. All rights reserved.
Copyright (c) 2008, Google Inc.
Copyright (c) 2009, Percona Inc.
Copyright (c) 2013, 2017, MariaDB Corporation.
Portions of this file contain modifications contributed and copyrighted by
Google, Inc. Those modifications are gratefully acknowledged and are described
briefly in the InnoDB documentation. The contributions by Google are
incorporated with their permission, and subject to the conditions contained in
the file COPYING.Google.
Portions of this file contain modifications contributed and copyrighted
by Percona Inc.. Those modifications are
gratefully acknowledged and are described briefly in the InnoDB
documentation. The contributions by Percona Inc. are incorporated with
their permission, and subject to the conditions contained in the file
COPYING.Percona.
This program is free software; you can redistribute it and/or modify it under
the terms of the GNU General Public License as published by the Free Software
Foundation; version 2 of the License.
This program is distributed in the hope that it will be useful, but WITHOUT
ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
You should have received a copy of the GNU General Public License along with
this program; if not, write to the Free Software Foundation, Inc.,
51 Franklin Street, Suite 500, Boston, MA 02110-1335 USA
*****************************************************************************/
/********************************************************************//**
@file srv/srv0start.cc
Starts the InnoDB database server
Created 2/16/1996 Heikki Tuuri
*************************************************************************/
2014-02-26 19:23:04 +01:00
#include "mysqld.h"
#include "pars0pars.h"
#include "row0ftsort.h"
#include "ut0mem.h"
#include "ut0timer.h"
#include "mem0mem.h"
#include "data0data.h"
#include "data0type.h"
#include "dict0dict.h"
#include "buf0buf.h"
#include "buf0dump.h"
#include "os0file.h"
#include "os0thread.h"
#include "fil0fil.h"
#include "fil0crypt.h"
#include "fsp0fsp.h"
#include "rem0rec.h"
#include "mtr0mtr.h"
#include "log0log.h"
#include "log0recv.h"
#include "page0page.h"
#include "page0cur.h"
#include "trx0trx.h"
#include "trx0sys.h"
#include "btr0btr.h"
#include "btr0cur.h"
#include "rem0rec.h"
#include "ibuf0ibuf.h"
#include "srv0start.h"
#include "srv0srv.h"
#include "btr0defragment.h"
#include "mysql/service_wsrep.h" /* wsrep_recovery */
2015-07-14 16:05:29 -04:00
#ifndef UNIV_HOTBACKUP
# include "trx0rseg.h"
# include "os0proc.h"
# include "sync0sync.h"
# include "buf0flu.h"
# include "buf0rea.h"
# include "buf0mtflu.h"
# include "dict0boot.h"
# include "dict0load.h"
# include "dict0stats_bg.h"
# include "que0que.h"
# include "usr0sess.h"
# include "lock0lock.h"
# include "trx0roll.h"
# include "trx0purge.h"
# include "lock0lock.h"
# include "pars0pars.h"
# include "btr0sea.h"
# include "rem0cmp.h"
# include "dict0crea.h"
# include "row0ins.h"
# include "row0sel.h"
# include "row0upd.h"
# include "row0row.h"
# include "row0mysql.h"
# include "btr0pcur.h"
# include "os0sync.h"
# include "zlib.h"
# include "ut0crc32.h"
2014-12-22 16:53:17 +02:00
# include "btr0scrub.h"
/** Log sequence number immediately after startup */
UNIV_INTERN lsn_t srv_start_lsn;
/** Log sequence number at shutdown */
UNIV_INTERN lsn_t srv_shutdown_lsn;
#ifdef HAVE_DARWIN_THREADS
# include <sys/utsname.h>
/** TRUE if the F_FULLFSYNC option is available */
UNIV_INTERN ibool srv_have_fullfsync = FALSE;
#endif
/** TRUE if a raw partition is in use */
UNIV_INTERN ibool srv_start_raw_disk_in_use = FALSE;
2017-05-15 17:17:16 +03:00
/** UNDO tablespaces starts with space id. */
ulint srv_undo_space_id_start;
/** TRUE if the server is being started, before rolling back any
incomplete transactions */
UNIV_INTERN ibool srv_startup_is_before_trx_rollback_phase = FALSE;
/** TRUE if the server is being started */
UNIV_INTERN ibool srv_is_being_started = FALSE;
/** TRUE if the server was successfully started */
UNIV_INTERN ibool srv_was_started = FALSE;
/** TRUE if innobase_start_or_create_for_mysql() has been called */
2017-06-12 14:26:32 +03:00
static ibool srv_start_has_been_called;
MDEV-13039 innodb_fast_shutdown=0 may fail to purge all undo log When a slow shutdown is performed soon after spawning some work for background threads that can create or commit transactions, it is possible that new transactions are started or committed after the purge has finished. This is violating the specification of innodb_fast_shutdown=0, namely that the purge must be completed. (None of the history of the recent transactions would be purged.) Also, it is possible that the purge threads would exit in slow shutdown while there exist active transactions, such as recovered incomplete transactions that are being rolled back. Thus, the slow shutdown could fail to purge some undo log that becomes purgeable after the transaction commit or rollback. srv_undo_sources: A flag that indicates if undo log can be generated or the persistent, whether by background threads or by user SQL. Even when this flag is clear, active transactions that already exist in the system may be committed or rolled back. innodb_shutdown(): Renamed from innobase_shutdown_for_mysql(). Do not return an error code; the operation never fails. Clear the srv_undo_sources flag, and also ensure that the background DROP TABLE queue is empty. srv_purge_should_exit(): Do not allow the purge to exit if srv_undo_sources are active or the background DROP TABLE queue is not empty, or in slow shutdown, if any active transactions exist (and are being rolled back). srv_purge_coordinator_thread(): Remove some previous workarounds for this bug. innobase_start_or_create_for_mysql(): Set buf_page_cleaner_is_active and srv_dict_stats_thread_active directly. Set srv_undo_sources before starting the purge subsystem, to prevent immediate shutdown of the purge. Create dict_stats_thread and fts_optimize_thread immediately after setting srv_undo_sources, so that shutdown can use this flag to determine if these subsystems were started. dict_stats_shutdown(): Shut down dict_stats_thread. Backported from 10.2. srv_shutdown_table_bg_threads(): Remove (unused).
2017-06-08 15:43:06 +03:00
/** Whether any undo log records can be generated */
2017-06-12 14:26:32 +03:00
UNIV_INTERN bool srv_undo_sources;
MDEV-11556 InnoDB redo log apply fails to adjust data file sizes fil_space_t::recv_size: New member: recovered tablespace size in pages; 0 if no size change was read from the redo log, or if the size change was implemented. fil_space_set_recv_size(): New function for setting space->recv_size. innodb_data_file_size_debug: A debug parameter for setting the system tablespace size in recovery even when the redo log does not contain any size changes. It is hard to write a small test case that would cause the system tablespace to be extended at the critical moment. recv_parse_log_rec(): Note those tablespaces whose size is being changed by the redo log, by invoking fil_space_set_recv_size(). innobase_init(): Correct an error message, and do not require a larger innodb_buffer_pool_size when starting up with a smaller innodb_page_size. innobase_start_or_create_for_mysql(): Allow startup with any initial size of the ibdata1 file if the autoextend attribute is set. Require the minimum size of fixed-size system tablespaces to be 640 pages, not 10 megabytes. Implement innodb_data_file_size_debug. open_or_create_data_files(): Round the system tablespace size down to pages, not to full megabytes, (Our test truncates the system tablespace to more than 800 pages with innodb_page_size=4k. InnoDB should not imagine that it was truncated to 768 pages and then overwrite good pages in the tablespace.) fil_flush_low(): Refactored from fil_flush(). fil_space_extend_must_retry(): Refactored from fil_extend_space_to_desired_size(). fil_mutex_enter_and_prepare_for_io(): Extend the tablespace if fil_space_set_recv_size() was called. The test case has been successfully run with all the innodb_page_size values 4k, 8k, 16k, 32k, 64k.
2016-12-28 12:05:43 +02:00
#ifdef UNIV_DEBUG
/** InnoDB system tablespace to set during recovery */
UNIV_INTERN uint srv_sys_space_size_debug;
MDEV-11556 InnoDB redo log apply fails to adjust data file sizes fil_space_t::recv_size: New member: recovered tablespace size in pages; 0 if no size change was read from the redo log, or if the size change was implemented. fil_space_set_recv_size(): New function for setting space->recv_size. innodb_data_file_size_debug: A debug parameter for setting the system tablespace size in recovery even when the redo log does not contain any size changes. It is hard to write a small test case that would cause the system tablespace to be extended at the critical moment. recv_parse_log_rec(): Note those tablespaces whose size is being changed by the redo log, by invoking fil_space_set_recv_size(). innobase_init(): Correct an error message, and do not require a larger innodb_buffer_pool_size when starting up with a smaller innodb_page_size. innobase_start_or_create_for_mysql(): Allow startup with any initial size of the ibdata1 file if the autoextend attribute is set. Require the minimum size of fixed-size system tablespaces to be 640 pages, not 10 megabytes. Implement innodb_data_file_size_debug. open_or_create_data_files(): Round the system tablespace size down to pages, not to full megabytes, (Our test truncates the system tablespace to more than 800 pages with innodb_page_size=4k. InnoDB should not imagine that it was truncated to 768 pages and then overwrite good pages in the tablespace.) fil_flush_low(): Refactored from fil_flush(). fil_space_extend_must_retry(): Refactored from fil_extend_space_to_desired_size(). fil_mutex_enter_and_prepare_for_io(): Extend the tablespace if fil_space_set_recv_size() was called. The test case has been successfully run with all the innodb_page_size values 4k, 8k, 16k, 32k, 64k.
2016-12-28 12:05:43 +02:00
#endif /* UNIV_DEBUG */
/** At a shutdown this value climbs from SRV_SHUTDOWN_NONE to
SRV_SHUTDOWN_CLEANUP and then to SRV_SHUTDOWN_LAST_PHASE, and so on */
UNIV_INTERN enum srv_shutdown_state srv_shutdown_state = SRV_SHUTDOWN_NONE;
/** Files comprising the system tablespace */
2017-05-15 17:17:16 +03:00
static pfs_os_file_t files[1000];
/** io_handler_thread parameters for thread identification */
static ulint n[SRV_MAX_N_IO_THREADS + 6];
/** io_handler_thread identifiers, 32 is the maximum number of purge threads */
/** 6 is the ? */
#define START_OLD_THREAD_CNT (SRV_MAX_N_IO_THREADS + 6 + 32)
static os_thread_id_t thread_ids[SRV_MAX_N_IO_THREADS + 6 + 32 + MTFLUSH_MAX_WORKER];
/* Thread contex data for multi-threaded flush */
void *mtflush_ctx=NULL;
/** Thead handles */
static os_thread_t thread_handles[SRV_MAX_N_IO_THREADS + 6 + 32];
static os_thread_t buf_flush_page_cleaner_thread_handle;
static os_thread_t buf_dump_thread_handle;
static os_thread_t dict_stats_thread_handle;
/** Status variables, is thread started ?*/
static bool thread_started[SRV_MAX_N_IO_THREADS + 6 + 32] = {false};
static bool buf_flush_page_cleaner_thread_started = false;
static bool buf_dump_thread_started = false;
static bool dict_stats_thread_started = false;
/** We use this mutex to test the return value of pthread_mutex_trylock
on successful locking. HP-UX does NOT return 0, though Linux et al do. */
static os_fast_mutex_t srv_os_test_mutex;
/** Name of srv_monitor_file */
static char* srv_monitor_file_name;
#endif /* !UNIV_HOTBACKUP */
/** Default undo tablespace size in UNIV_PAGEs count (10MB). */
static const ulint SRV_UNDO_TABLESPACE_SIZE_IN_PAGES =
((1024 * 1024) * 10) / UNIV_PAGE_SIZE_DEF;
/** */
#define SRV_N_PENDING_IOS_PER_THREAD OS_AIO_N_PENDING_IOS_PER_THREAD
#define SRV_MAX_N_PENDING_SYNC_IOS 100
#ifdef UNIV_PFS_THREAD
/* Keys to register InnoDB threads with performance schema */
UNIV_INTERN mysql_pfs_key_t io_handler_thread_key;
UNIV_INTERN mysql_pfs_key_t srv_lock_timeout_thread_key;
UNIV_INTERN mysql_pfs_key_t srv_error_monitor_thread_key;
UNIV_INTERN mysql_pfs_key_t srv_monitor_thread_key;
UNIV_INTERN mysql_pfs_key_t srv_master_thread_key;
UNIV_INTERN mysql_pfs_key_t srv_purge_thread_key;
#endif /* UNIV_PFS_THREAD */
/** Innobase start-up aborted. Perform cleanup actions.
@param[in] create_new_db TRUE if new db is being created
@param[in] file File name
@param[in] line Line number
@param[in] err Reason for aborting InnoDB startup
@return DB_SUCCESS or error code. */
static
dberr_t
srv_init_abort(
bool create_new_db,
const char* file,
ulint line,
dberr_t err)
{
if (create_new_db) {
ib_logf(IB_LOG_LEVEL_ERROR,
"Database creation was aborted"
" at %s [" ULINTPF "]"
" with error %s. You may need"
" to delete the ibdata1 file before trying to start"
" up again.",
file, line, ut_strerr(err));
} else {
ib_logf(IB_LOG_LEVEL_ERROR,
"Plugin initialization aborted"
" at %s [" ULINTPF "]"
" with error %s.",
file, line, ut_strerr(err));
}
return(err);
}
/*********************************************************************//**
2014-05-06 21:13:16 +02:00
Convert a numeric string that optionally ends in G or M or K, to a number
containing megabytes.
@return next character in string */
static
char*
srv_parse_megabytes(
/*================*/
char* str, /*!< in: string containing a quantity in bytes */
ulint* megs) /*!< out: the number in megabytes */
{
char* endp;
ulint size;
size = strtoul(str, &endp, 10);
str = endp;
switch (*str) {
case 'G': case 'g':
size *= 1024;
/* fall through */
case 'M': case 'm':
str++;
break;
2014-05-06 21:13:16 +02:00
case 'K': case 'k':
size /= 1024;
str++;
break;
default:
size /= 1024 * 1024;
break;
}
*megs = size;
return(str);
}
/*********************************************************************//**
Check if a file can be opened in read-write mode.
@return true if it doesn't exist or can be opened in rw mode. */
static
bool
srv_file_check_mode(
/*================*/
const char* name) /*!< in: filename to check */
{
os_file_stat_t stat;
memset(&stat, 0x0, sizeof(stat));
dberr_t err = os_file_get_status(name, &stat, true);
if (err == DB_FAIL) {
ib_logf(IB_LOG_LEVEL_ERROR,
"os_file_get_status() failed on '%s'. Can't determine "
"file permissions", name);
return(false);
} else if (err == DB_SUCCESS) {
/* Note: stat.rw_perm is only valid of files */
2015-06-16 10:57:05 +02:00
if (stat.type == OS_FILE_TYPE_FILE) {
if (!stat.rw_perm) {
ib_logf(IB_LOG_LEVEL_ERROR,
"%s can't be opened in %s mode",
name,
srv_read_only_mode
? "read" : "read-write");
return(false);
}
} else {
/* Not a regular file, bail out. */
ib_logf(IB_LOG_LEVEL_ERROR,
"'%s' not a regular file.", name);
return(false);
}
} else {
/* This is OK. If the file create fails on RO media, there
is nothing we can do. */
ut_a(err == DB_NOT_FOUND);
}
return(true);
}
/*********************************************************************//**
Reads the data files and their sizes from a character string given in
the .cnf file.
@return TRUE if ok, FALSE on parse error */
UNIV_INTERN
ibool
srv_parse_data_file_paths_and_sizes(
/*================================*/
char* str) /*!< in/out: the data file path string */
{
char* input_str;
char* path;
ulint size;
ulint i = 0;
srv_auto_extend_last_data_file = FALSE;
srv_last_file_size_max = 0;
srv_data_file_names = NULL;
srv_data_file_sizes = NULL;
srv_data_file_is_raw_partition = NULL;
input_str = str;
/* First calculate the number of data files and check syntax:
path:size[M | G];path:size[M | G]... . Note that a Windows path may
contain a drive name and a ':'. */
while (*str != '\0') {
path = str;
while ((*str != ':' && *str != '\0')
|| (*str == ':'
&& (*(str + 1) == '\\' || *(str + 1) == '/'
|| *(str + 1) == ':'))) {
str++;
}
if (*str == '\0') {
return(FALSE);
}
str++;
str = srv_parse_megabytes(str, &size);
if (0 == strncmp(str, ":autoextend",
(sizeof ":autoextend") - 1)) {
str += (sizeof ":autoextend") - 1;
if (0 == strncmp(str, ":max:",
(sizeof ":max:") - 1)) {
str += (sizeof ":max:") - 1;
str = srv_parse_megabytes(str, &size);
}
if (*str != '\0') {
return(FALSE);
}
}
if (strlen(str) >= 6
&& *str == 'n'
&& *(str + 1) == 'e'
&& *(str + 2) == 'w') {
str += 3;
}
if (*str == 'r' && *(str + 1) == 'a' && *(str + 2) == 'w') {
str += 3;
}
if (size == 0) {
return(FALSE);
}
i++;
if (*str == ';') {
str++;
} else if (*str != '\0') {
return(FALSE);
}
}
if (i == 0) {
/* If innodb_data_file_path was defined it must contain
at least one data file definition */
return(FALSE);
}
srv_data_file_names = static_cast<char**>(
malloc(i * sizeof *srv_data_file_names));
srv_data_file_sizes = static_cast<ulint*>(
malloc(i * sizeof *srv_data_file_sizes));
srv_data_file_is_raw_partition = static_cast<ulint*>(
malloc(i * sizeof *srv_data_file_is_raw_partition));
srv_n_data_files = i;
/* Then store the actual values to our arrays */
str = input_str;
i = 0;
while (*str != '\0') {
path = str;
/* Note that we must step over the ':' in a Windows path;
a Windows path normally looks like C:\ibdata\ibdata1:1G, but
a Windows raw partition may have a specification like
\\.\C::1Gnewraw or \\.\PHYSICALDRIVE2:1Gnewraw */
while ((*str != ':' && *str != '\0')
|| (*str == ':'
&& (*(str + 1) == '\\' || *(str + 1) == '/'
|| *(str + 1) == ':'))) {
str++;
}
if (*str == ':') {
/* Make path a null-terminated string */
*str = '\0';
str++;
}
str = srv_parse_megabytes(str, &size);
srv_data_file_names[i] = path;
srv_data_file_sizes[i] = size;
if (0 == strncmp(str, ":autoextend",
(sizeof ":autoextend") - 1)) {
srv_auto_extend_last_data_file = TRUE;
str += (sizeof ":autoextend") - 1;
if (0 == strncmp(str, ":max:",
(sizeof ":max:") - 1)) {
str += (sizeof ":max:") - 1;
str = srv_parse_megabytes(
str, &srv_last_file_size_max);
}
if (*str != '\0') {
return(FALSE);
}
}
(srv_data_file_is_raw_partition)[i] = 0;
if (strlen(str) >= 6
&& *str == 'n'
&& *(str + 1) == 'e'
&& *(str + 2) == 'w') {
str += 3;
2015-06-16 10:57:05 +02:00
/* Initialize new raw device only during bootstrap */
(srv_data_file_is_raw_partition)[i] = SRV_NEW_RAW;
}
if (*str == 'r' && *(str + 1) == 'a' && *(str + 2) == 'w') {
str += 3;
2015-06-16 10:57:05 +02:00
/* Initialize new raw device only during bootstrap */
if ((srv_data_file_is_raw_partition)[i] == 0) {
(srv_data_file_is_raw_partition)[i] = SRV_NEW_RAW;
}
}
i++;
if (*str == ';') {
str++;
}
}
return(TRUE);
}
/*********************************************************************//**
Frees the memory allocated by srv_parse_data_file_paths_and_sizes()
and srv_parse_log_group_home_dirs(). */
UNIV_INTERN
void
srv_free_paths_and_sizes(void)
/*==========================*/
{
free(srv_data_file_names);
srv_data_file_names = NULL;
free(srv_data_file_sizes);
srv_data_file_sizes = NULL;
free(srv_data_file_is_raw_partition);
srv_data_file_is_raw_partition = NULL;
}
#ifndef UNIV_HOTBACKUP
/********************************************************************//**
I/o-handler thread function.
@return OS_THREAD_DUMMY_RETURN */
extern "C" UNIV_INTERN
os_thread_ret_t
DECLARE_THREAD(io_handler_thread)(
/*==============================*/
void* arg) /*!< in: pointer to the number of the segment in
the aio array */
{
ulint segment;
segment = *((ulint*) arg);
#ifdef UNIV_DEBUG_THREAD_CREATION
ib_logf(IB_LOG_LEVEL_INFO,
"Io handler thread %lu starts, id %lu\n", segment,
os_thread_pf(os_thread_get_curr_id()));
#endif
#ifdef UNIV_PFS_THREAD
pfs_register_thread(io_handler_thread_key);
#endif /* UNIV_PFS_THREAD */
while (srv_shutdown_state != SRV_SHUTDOWN_EXIT_THREADS) {
fil_aio_wait(segment);
}
/* We count the number of threads in os_thread_exit(). A created
thread should always use that to exit and not use return() to exit.
The thread actually never comes here because it is exited in an
os_event_wait(). */
os_thread_exit(NULL);
OS_THREAD_DUMMY_RETURN;
}
#endif /* !UNIV_HOTBACKUP */
/*********************************************************************//**
Normalizes a directory path for Windows: converts slashes to backslashes. */
UNIV_INTERN
void
srv_normalize_path_for_win(
/*=======================*/
2016-06-21 14:21:03 +02:00
char* str MY_ATTRIBUTE((unused))) /*!< in/out: null-terminated
character string */
{
#ifdef __WIN__
for (; *str; str++) {
if (*str == '/') {
*str = '\\';
}
}
#endif
}
#ifndef UNIV_HOTBACKUP
/*********************************************************************//**
Creates a log file.
@return DB_SUCCESS or error code */
2016-06-21 14:21:03 +02:00
static MY_ATTRIBUTE((nonnull, warn_unused_result))
dberr_t
create_log_file(
/*============*/
2017-05-15 17:17:16 +03:00
pfs_os_file_t* file, /*!< out: file handle */
const char* name) /*!< in: log file name */
{
ibool ret;
*file = os_file_create(
innodb_file_log_key, name,
OS_FILE_CREATE|OS_FILE_ON_ERROR_NO_EXIT, OS_FILE_NORMAL,
2014-03-03 14:27:56 +02:00
OS_LOG_FILE, &ret, FALSE);
if (!ret) {
ib_logf(IB_LOG_LEVEL_ERROR, "Cannot create %s", name);
return(DB_ERROR);
}
ib_logf(IB_LOG_LEVEL_INFO,
"Setting log file %s size to %lu MB",
name, (ulong) srv_log_file_size
>> (20 - UNIV_PAGE_SIZE_SHIFT));
ret = os_file_set_size(name, *file,
(os_offset_t) srv_log_file_size
<< UNIV_PAGE_SIZE_SHIFT);
if (!ret) {
ib_logf(IB_LOG_LEVEL_ERROR, "Cannot set log file"
" %s to size %lu MB", name, (ulong) srv_log_file_size
>> (20 - UNIV_PAGE_SIZE_SHIFT));
return(DB_ERROR);
}
ret = os_file_close(*file);
ut_a(ret);
return(DB_SUCCESS);
}
/** Initial number of the first redo log file */
#define INIT_LOG_FILE0 (SRV_N_LOG_FILES_MAX + 1)
/*********************************************************************//**
Creates all log files.
@return DB_SUCCESS or error code */
static
dberr_t
create_log_files(
/*=============*/
bool create_new_db, /*!< in: TRUE if new database is being
created */
char* logfilename, /*!< in/out: buffer for log file name */
size_t dirnamelen, /*!< in: length of the directory path */
lsn_t lsn, /*!< in: FIL_PAGE_FILE_FLUSH_LSN value */
char*& logfile0) /*!< out: name of the first log file */
{
if (srv_read_only_mode) {
ib_logf(IB_LOG_LEVEL_ERROR,
"Cannot create log files in read-only mode");
return(DB_READ_ONLY);
}
/* We prevent system tablespace creation with existing files in
data directory. So we do not delete log files when creating new system
tablespace */
if (!create_new_db) {
/* Remove any old log files. */
for (unsigned i = 0; i <= INIT_LOG_FILE0; i++) {
sprintf(logfilename + dirnamelen, "ib_logfile%u", i);
/* Ignore errors about non-existent files or files
that cannot be removed. The create_log_file() will
return an error when the file exists. */
#ifdef __WIN__
DeleteFile((LPCTSTR) logfilename);
#else
unlink(logfilename);
#endif
/* Crashing after deleting the first
file should be recoverable. The buffer
pool was clean, and we can simply create
all log files from the scratch. */
DBUG_EXECUTE_IF("innodb_log_abort_6",
return(DB_ERROR););
}
}
ut_ad(!buf_pool_check_no_pending_io());
DBUG_EXECUTE_IF("innodb_log_abort_7", return(DB_ERROR););
for (unsigned i = 0; i < srv_n_log_files; i++) {
sprintf(logfilename + dirnamelen,
"ib_logfile%u", i ? i : INIT_LOG_FILE0);
dberr_t err = create_log_file(&files[i], logfilename);
if (err != DB_SUCCESS) {
return(err);
}
}
DBUG_EXECUTE_IF("innodb_log_abort_8", return(DB_ERROR););
/* We did not create the first log file initially as
ib_logfile0, so that crash recovery cannot find it until it
has been completed and renamed. */
sprintf(logfilename + dirnamelen, "ib_logfile%u", INIT_LOG_FILE0);
fil_space_create(
MDEV-11623 MariaDB 10.1 fails to start datadir created with MariaDB 10.0/MySQL 5.6 using innodb-page-size!=16K The storage format of FSP_SPACE_FLAGS was accidentally broken already in MariaDB 10.1.0. This fix is bringing the format in line with other MySQL and MariaDB release series. Please refer to the comments that were added to fsp0fsp.h for details. This is an INCOMPATIBLE CHANGE that affects users of page_compression and non-default innodb_page_size. Upgrading to this release will correct the flags in the data files. If you want to downgrade to earlier MariaDB 10.1.x, please refer to the test innodb.101_compatibility how to reset the FSP_SPACE_FLAGS in the files. NOTE: MariaDB 10.1.0 to 10.1.20 can misinterpret uncompressed data files with innodb_page_size=4k or 64k as compressed innodb_page_size=16k files, and then probably fail when trying to access the pages. See the comments in the function fsp_flags_convert_from_101() for detailed analysis. Move PAGE_COMPRESSION to FSP_SPACE_FLAGS bit position 16. In this way, compressed innodb_page_size=16k tablespaces will not be mistaken for uncompressed ones by MariaDB 10.1.0 to 10.1.20. Derive PAGE_COMPRESSION_LEVEL, ATOMIC_WRITES and DATA_DIR from the dict_table_t::flags when the table is available, in fil_space_for_table_exists_in_mem() or fil_open_single_table_tablespace(). During crash recovery, fil_load_single_table_tablespace() will use innodb_compression_level for the PAGE_COMPRESSION_LEVEL. FSP_FLAGS_MEM_MASK: A bitmap of the memory-only fil_space_t::flags that are not to be written to FSP_SPACE_FLAGS. Currently, these will include PAGE_COMPRESSION_LEVEL, ATOMIC_WRITES and DATA_DIR. Introduce the macro FSP_FLAGS_PAGE_SSIZE(). We only support one innodb_page_size for the whole instance. When creating a dummy tablespace for the redo log, use fil_space_t::flags=0. The flags are never written to the redo log files. Remove many FSP_FLAGS_SET_ macros. dict_tf_verify_flags(): Remove. This is basically only duplicating the logic of dict_tf_to_fsp_flags(), used in a debug assertion. fil_space_t::mark: Remove. This flag was not used for anything. fil_space_for_table_exists_in_mem(): Remove the unnecessary parameter mark_space, and add a parameter for table flags. Check that fil_space_t::flags match the table flags, and adjust the (memory-only) flags based on the table flags. fil_node_open_file(): Remove some redundant or unreachable conditions, do not use stderr for output, and avoid unnecessary server aborts. fil_user_tablespace_restore_page(): Convert the flags, so that the correct page_size will be used when restoring a page from the doublewrite buffer. fil_space_get_page_compressed(), fsp_flags_is_page_compressed(): Remove. It suffices to have fil_space_is_page_compressed(). FSP_FLAGS_WIDTH_DATA_DIR, FSP_FLAGS_WIDTH_PAGE_COMPRESSION_LEVEL, FSP_FLAGS_WIDTH_ATOMIC_WRITES: Remove, because these flags do not exist in the FSP_SPACE_FLAGS but only in memory. fsp_flags_try_adjust(): New function, to adjust the FSP_SPACE_FLAGS in page 0. Called by fil_open_single_table_tablespace(), fil_space_for_table_exists_in_mem(), innobase_start_or_create_for_mysql() except if --innodb-read-only is active. fsp_flags_is_valid(ulint): Reimplement from the scratch, with accurate comments. Do not display any details of detected inconsistencies, because the output could be confusing when dealing with MariaDB 10.1.x data files. fsp_flags_convert_from_101(ulint): Convert flags from buggy MariaDB 10.1.x format, or return ULINT_UNDEFINED if the flags cannot be in MariaDB 10.1.x format. fsp_flags_match(): Check the flags when probing files. Implemented based on fsp_flags_is_valid() and fsp_flags_convert_from_101(). dict_check_tablespaces_and_store_max_id(): Do not access the page after committing the mini-transaction. IMPORT TABLESPACE fixes: AbstractCallback::init(): Convert the flags. FetchIndexRootPages::operator(): Check that the tablespace flags match the table flags. Do not attempt to convert tablespace flags to table flags, because the conversion would necessarily be lossy. PageConverter::update_header(): Write back the correct flags. This takes care of the flags in IMPORT TABLESPACE.
2017-01-14 00:13:16 +02:00
logfilename, SRV_LOG_SPACE_FIRST_ID, 0,
2014-12-22 16:53:17 +02:00
FIL_LOG,
NULL /* no encryption yet */,
true /* this is create */);
MDEV-11738: Mariadb uses 100% of several of my 8 cpus doing nothing MDEV-11581: Mariadb starts InnoDB encryption threads when key has not changed or data scrubbing turned off Background: Key rotation is based on background threads (innodb-encryption-threads) periodically going through all tablespaces on fil_system. For each tablespace current used key version is compared to max key age (innodb-encryption-rotate-key-age). This process naturally takes CPU. Similarly, in same time need for scrubbing is investigated. Currently, key rotation is fully supported on Amazon AWS key management plugin only but InnoDB does not have knowledge what key management plugin is used. This patch re-purposes innodb-encryption-rotate-key-age=0 to disable key rotation and background data scrubbing. All new tables are added to special list for key rotation and key rotation is based on sending a event to background encryption threads instead of using periodic checking (i.e. timeout). fil0fil.cc: Added functions fil_space_acquire_low() to acquire a tablespace when it could be dropped concurrently. This function is used from fil_space_acquire() or fil_space_acquire_silent() that will not print any messages if we try to acquire space that does not exist. fil_space_release() to release a acquired tablespace. fil_space_next() to iterate tablespaces in fil_system using fil_space_acquire() and fil_space_release(). Similarly, fil_space_keyrotation_next() to iterate new list fil_system->rotation_list where new tables. are added if key rotation is disabled. Removed unnecessary functions fil_get_first_space_safe() fil_get_next_space_safe() fil_node_open_file(): After page 0 is read read also crypt_info if it is not yet read. btr_scrub_lock_dict_func() buf_page_check_corrupt() buf_page_encrypt_before_write() buf_merge_or_delete_for_page() lock_print_info_all_transactions() row_fts_psort_info_init() row_truncate_table_for_mysql() row_drop_table_for_mysql() Use fil_space_acquire()/release() to access fil_space_t. buf_page_decrypt_after_read(): Use fil_space_get_crypt_data() because at this point we might not yet have read page 0. fil0crypt.cc/fil0fil.h: Lot of changes. Pass fil_space_t* directly to functions needing it and store fil_space_t* to rotation state. Use fil_space_acquire()/release() when iterating tablespaces and removed unnecessary is_closing from fil_crypt_t. Use fil_space_t::is_stopping() to detect when access to tablespace should be stopped. Removed unnecessary fil_space_get_crypt_data(). fil_space_create(): Inform key rotation that there could be something to do if key rotation is disabled and new table with encryption enabled is created. Remove unnecessary functions fil_get_first_space_safe() and fil_get_next_space_safe(). fil_space_acquire() and fil_space_release() are used instead. Moved fil_space_get_crypt_data() and fil_space_set_crypt_data() to fil0crypt.cc. fsp_header_init(): Acquire fil_space_t*, write crypt_data and release space. check_table_options() Renamed FIL_SPACE_ENCRYPTION_* TO FIL_ENCRYPTION_* i_s.cc: Added ROTATING_OR_FLUSHING field to information_schema.innodb_tablespace_encryption to show current status of key rotation.
2017-03-14 12:56:01 +02:00
ut_a(fil_validate());
logfile0 = fil_node_create(
logfilename, (ulint) srv_log_file_size,
SRV_LOG_SPACE_FIRST_ID, FALSE);
ut_a(logfile0);
for (unsigned i = 1; i < srv_n_log_files; i++) {
sprintf(logfilename + dirnamelen, "ib_logfile%u", i);
if (!fil_node_create(
logfilename,
(ulint) srv_log_file_size,
SRV_LOG_SPACE_FIRST_ID, FALSE)) {
ut_error;
}
}
log_group_init(0, srv_n_log_files,
srv_log_file_size * UNIV_PAGE_SIZE,
SRV_LOG_SPACE_FIRST_ID,
SRV_LOG_SPACE_FIRST_ID + 1);
fil_open_log_and_system_tablespace_files();
/* Create a log checkpoint. */
mutex_enter(&log_sys->mutex);
ut_d(recv_no_log_write = FALSE);
recv_reset_logs(lsn);
mutex_exit(&log_sys->mutex);
return(DB_SUCCESS);
}
/** Rename the first redo log file.
@param[in,out] logfilename buffer for the log file name
@param[in] dirnamelen length of the directory path
@param[in] lsn FIL_PAGE_FILE_FLUSH_LSN value
@param[in,out] logfile0 name of the first log file
@return error code
@retval DB_SUCCESS on successful operation */
MY_ATTRIBUTE((warn_unused_result, nonnull))
static
dberr_t
create_log_files_rename(
/*====================*/
char* logfilename, /*!< in/out: buffer for log file name */
size_t dirnamelen, /*!< in: length of the directory path */
lsn_t lsn, /*!< in: FIL_PAGE_FILE_FLUSH_LSN value */
char* logfile0) /*!< in/out: name of the first log file */
{
/* If innodb_flush_method=O_DSYNC,
we need to explicitly flush the log buffers. */
fil_flush(SRV_LOG_SPACE_FIRST_ID);
DBUG_EXECUTE_IF("innodb_log_abort_9", return(DB_ERROR););
/* Close the log files, so that we can rename
the first one. */
fil_close_log_files(false);
/* Rename the first log file, now that a log
checkpoint has been created. */
sprintf(logfilename + dirnamelen, "ib_logfile%u", 0);
ib_logf(IB_LOG_LEVEL_INFO,
"Renaming log file %s to %s", logfile0, logfilename);
mutex_enter(&log_sys->mutex);
ut_ad(strlen(logfile0) == 2 + strlen(logfilename));
dberr_t err = os_file_rename(
innodb_file_log_key, logfile0, logfilename)
? DB_SUCCESS : DB_ERROR;
/* Replace the first file with ib_logfile0. */
strcpy(logfile0, logfilename);
mutex_exit(&log_sys->mutex);
DBUG_EXECUTE_IF("innodb_log_abort_10", err = DB_ERROR;);
if (err == DB_SUCCESS) {
fil_open_log_and_system_tablespace_files();
ib_logf(IB_LOG_LEVEL_WARN,
"New log files created, LSN=" LSN_PF, lsn);
}
return(err);
}
/*********************************************************************//**
Opens a log file.
@return DB_SUCCESS or error code */
2016-06-21 14:21:03 +02:00
static MY_ATTRIBUTE((nonnull, warn_unused_result))
dberr_t
open_log_file(
/*==========*/
2017-05-15 17:17:16 +03:00
pfs_os_file_t* file, /*!< out: file handle */
const char* name, /*!< in: log file name */
os_offset_t* size) /*!< out: file size */
{
ibool ret;
*file = os_file_create(innodb_file_log_key, name,
OS_FILE_OPEN, OS_FILE_AIO,
OS_LOG_FILE, &ret, FALSE);
if (!ret) {
ib_logf(IB_LOG_LEVEL_ERROR, "Unable to open '%s'", name);
return(DB_ERROR);
}
*size = os_file_get_size(*file);
ret = os_file_close(*file);
ut_a(ret);
return(DB_SUCCESS);
}
MDEV-12113: install_db shows corruption for rest encryption with innodb_data_file_path=ibdata1:3M; Problem was that FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION field that for encrypted pages even in system datafiles should contain key_version except very first page (0:0) is after encryption overwritten with flush lsn. Ported WL#7990 Repurpose FIL_PAGE_FLUSH_LSN to 10.1 The field FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION is consulted during InnoDB startup. At startup, InnoDB reads the FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION from the first page of each file in the InnoDB system tablespace. If there are multiple files, the minimum and maximum LSN can differ. These numbers are passed to InnoDB startup. Having the number in other files than the first file of the InnoDB system tablespace is not providing much additional value. It is conflicting with other use of the field, such as on InnoDB R-tree index pages and encryption key_version. This worklog will stop writing FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION to other files than the first file of the InnoDB system tablespace (page number 0:0) when system tablespace is encrypted. If tablespace is not encrypted we continue writing FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION to all first pages of system tablespace to avoid unnecessary warnings on downgrade. open_or_create_data_files(): pass only one flushed_lsn parameter xb_load_tablespaces(): pass only one flushed_lsn parameter. buf_page_create(): Improve comment about where FIL_PAGE_FIL_FLUSH_LSN_OR_KEY_VERSION is set. fil_write_flushed_lsn(): A new function, merged from fil_write_lsn_and_arch_no_to_file() and fil_write_flushed_lsn_to_data_files(). Only write to the first page of the system tablespace (page 0:0) if tablespace is encrypted, or write all first pages of system tablespace and invoke fil_flush_file_spaces(FIL_TYPE_TABLESPACE) afterwards. fil_read_first_page(): read flush_lsn and crypt_data only from first datafile. fil_open_single_table_tablespace(): Remove output of LSN, because it was only valid for the system tablespace and the undo tablespaces, not user tablespaces. fil_validate_single_table_tablespace(): Remove output of LSN. checkpoint_now_set(): Use fil_write_flushed_lsn and output a error if operation fails. Remove lsn variable from fsp_open_info. recv_recovery_from_checkpoint_start(): Remove unnecessary second flush_lsn parameter. log_empty_and_mark_files_at_shutdown(): Use fil_writte_flushed_lsn and output error if it fails. open_or_create_data_files(): Pass only one flushed_lsn variable.
2017-05-30 11:30:43 +03:00
/** Creates or opens database data files and closes them.
@param[out] create_new_db true = create new database
@param[out] min_arch_log_no min of archived log numbers in
data files
@param[out] max_arch_log_no max of archived log numbers in
data files
@param[out] flushed_lsn flushed lsn in fist datafile
@param[out] sum_of_new_sizes sum of sizes of the new files
added
@return DB_SUCCESS or error code */
2016-06-21 14:21:03 +02:00
static MY_ATTRIBUTE((nonnull, warn_unused_result))
dberr_t
open_or_create_data_files(
MDEV-12113: install_db shows corruption for rest encryption with innodb_data_file_path=ibdata1:3M; Problem was that FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION field that for encrypted pages even in system datafiles should contain key_version except very first page (0:0) is after encryption overwritten with flush lsn. Ported WL#7990 Repurpose FIL_PAGE_FLUSH_LSN to 10.1 The field FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION is consulted during InnoDB startup. At startup, InnoDB reads the FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION from the first page of each file in the InnoDB system tablespace. If there are multiple files, the minimum and maximum LSN can differ. These numbers are passed to InnoDB startup. Having the number in other files than the first file of the InnoDB system tablespace is not providing much additional value. It is conflicting with other use of the field, such as on InnoDB R-tree index pages and encryption key_version. This worklog will stop writing FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION to other files than the first file of the InnoDB system tablespace (page number 0:0) when system tablespace is encrypted. If tablespace is not encrypted we continue writing FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION to all first pages of system tablespace to avoid unnecessary warnings on downgrade. open_or_create_data_files(): pass only one flushed_lsn parameter xb_load_tablespaces(): pass only one flushed_lsn parameter. buf_page_create(): Improve comment about where FIL_PAGE_FIL_FLUSH_LSN_OR_KEY_VERSION is set. fil_write_flushed_lsn(): A new function, merged from fil_write_lsn_and_arch_no_to_file() and fil_write_flushed_lsn_to_data_files(). Only write to the first page of the system tablespace (page 0:0) if tablespace is encrypted, or write all first pages of system tablespace and invoke fil_flush_file_spaces(FIL_TYPE_TABLESPACE) afterwards. fil_read_first_page(): read flush_lsn and crypt_data only from first datafile. fil_open_single_table_tablespace(): Remove output of LSN, because it was only valid for the system tablespace and the undo tablespaces, not user tablespaces. fil_validate_single_table_tablespace(): Remove output of LSN. checkpoint_now_set(): Use fil_write_flushed_lsn and output a error if operation fails. Remove lsn variable from fsp_open_info. recv_recovery_from_checkpoint_start(): Remove unnecessary second flush_lsn parameter. log_empty_and_mark_files_at_shutdown(): Use fil_writte_flushed_lsn and output error if it fails. open_or_create_data_files(): Pass only one flushed_lsn variable.
2017-05-30 11:30:43 +03:00
bool* create_new_db,
#ifdef UNIV_LOG_ARCHIVE
MDEV-12113: install_db shows corruption for rest encryption with innodb_data_file_path=ibdata1:3M; Problem was that FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION field that for encrypted pages even in system datafiles should contain key_version except very first page (0:0) is after encryption overwritten with flush lsn. Ported WL#7990 Repurpose FIL_PAGE_FLUSH_LSN to 10.1 The field FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION is consulted during InnoDB startup. At startup, InnoDB reads the FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION from the first page of each file in the InnoDB system tablespace. If there are multiple files, the minimum and maximum LSN can differ. These numbers are passed to InnoDB startup. Having the number in other files than the first file of the InnoDB system tablespace is not providing much additional value. It is conflicting with other use of the field, such as on InnoDB R-tree index pages and encryption key_version. This worklog will stop writing FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION to other files than the first file of the InnoDB system tablespace (page number 0:0) when system tablespace is encrypted. If tablespace is not encrypted we continue writing FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION to all first pages of system tablespace to avoid unnecessary warnings on downgrade. open_or_create_data_files(): pass only one flushed_lsn parameter xb_load_tablespaces(): pass only one flushed_lsn parameter. buf_page_create(): Improve comment about where FIL_PAGE_FIL_FLUSH_LSN_OR_KEY_VERSION is set. fil_write_flushed_lsn(): A new function, merged from fil_write_lsn_and_arch_no_to_file() and fil_write_flushed_lsn_to_data_files(). Only write to the first page of the system tablespace (page 0:0) if tablespace is encrypted, or write all first pages of system tablespace and invoke fil_flush_file_spaces(FIL_TYPE_TABLESPACE) afterwards. fil_read_first_page(): read flush_lsn and crypt_data only from first datafile. fil_open_single_table_tablespace(): Remove output of LSN, because it was only valid for the system tablespace and the undo tablespaces, not user tablespaces. fil_validate_single_table_tablespace(): Remove output of LSN. checkpoint_now_set(): Use fil_write_flushed_lsn and output a error if operation fails. Remove lsn variable from fsp_open_info. recv_recovery_from_checkpoint_start(): Remove unnecessary second flush_lsn parameter. log_empty_and_mark_files_at_shutdown(): Use fil_writte_flushed_lsn and output error if it fails. open_or_create_data_files(): Pass only one flushed_lsn variable.
2017-05-30 11:30:43 +03:00
ulint* min_arch_log_no,
ulint* max_arch_log_no,
#endif /* UNIV_LOG_ARCHIVE */
MDEV-12113: install_db shows corruption for rest encryption with innodb_data_file_path=ibdata1:3M; Problem was that FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION field that for encrypted pages even in system datafiles should contain key_version except very first page (0:0) is after encryption overwritten with flush lsn. Ported WL#7990 Repurpose FIL_PAGE_FLUSH_LSN to 10.1 The field FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION is consulted during InnoDB startup. At startup, InnoDB reads the FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION from the first page of each file in the InnoDB system tablespace. If there are multiple files, the minimum and maximum LSN can differ. These numbers are passed to InnoDB startup. Having the number in other files than the first file of the InnoDB system tablespace is not providing much additional value. It is conflicting with other use of the field, such as on InnoDB R-tree index pages and encryption key_version. This worklog will stop writing FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION to other files than the first file of the InnoDB system tablespace (page number 0:0) when system tablespace is encrypted. If tablespace is not encrypted we continue writing FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION to all first pages of system tablespace to avoid unnecessary warnings on downgrade. open_or_create_data_files(): pass only one flushed_lsn parameter xb_load_tablespaces(): pass only one flushed_lsn parameter. buf_page_create(): Improve comment about where FIL_PAGE_FIL_FLUSH_LSN_OR_KEY_VERSION is set. fil_write_flushed_lsn(): A new function, merged from fil_write_lsn_and_arch_no_to_file() and fil_write_flushed_lsn_to_data_files(). Only write to the first page of the system tablespace (page 0:0) if tablespace is encrypted, or write all first pages of system tablespace and invoke fil_flush_file_spaces(FIL_TYPE_TABLESPACE) afterwards. fil_read_first_page(): read flush_lsn and crypt_data only from first datafile. fil_open_single_table_tablespace(): Remove output of LSN, because it was only valid for the system tablespace and the undo tablespaces, not user tablespaces. fil_validate_single_table_tablespace(): Remove output of LSN. checkpoint_now_set(): Use fil_write_flushed_lsn and output a error if operation fails. Remove lsn variable from fsp_open_info. recv_recovery_from_checkpoint_start(): Remove unnecessary second flush_lsn parameter. log_empty_and_mark_files_at_shutdown(): Use fil_writte_flushed_lsn and output error if it fails. open_or_create_data_files(): Pass only one flushed_lsn variable.
2017-05-30 11:30:43 +03:00
lsn_t* flushed_lsn,
ulint* sum_of_new_sizes)
{
ibool ret;
ulint i;
ibool one_opened = FALSE;
ibool one_created = FALSE;
os_offset_t size;
ulint flags;
ulint space=0;
ulint rounded_size_pages;
char name[10000];
fil_space_crypt_t* crypt_data=NULL;
if (srv_n_data_files >= 1000) {
ib_logf(IB_LOG_LEVEL_ERROR,
"Can only have < 1000 data files, you have "
"defined %lu", (ulong) srv_n_data_files);
return(DB_ERROR);
}
*sum_of_new_sizes = 0;
MDEV-12113: install_db shows corruption for rest encryption with innodb_data_file_path=ibdata1:3M; Problem was that FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION field that for encrypted pages even in system datafiles should contain key_version except very first page (0:0) is after encryption overwritten with flush lsn. Ported WL#7990 Repurpose FIL_PAGE_FLUSH_LSN to 10.1 The field FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION is consulted during InnoDB startup. At startup, InnoDB reads the FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION from the first page of each file in the InnoDB system tablespace. If there are multiple files, the minimum and maximum LSN can differ. These numbers are passed to InnoDB startup. Having the number in other files than the first file of the InnoDB system tablespace is not providing much additional value. It is conflicting with other use of the field, such as on InnoDB R-tree index pages and encryption key_version. This worklog will stop writing FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION to other files than the first file of the InnoDB system tablespace (page number 0:0) when system tablespace is encrypted. If tablespace is not encrypted we continue writing FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION to all first pages of system tablespace to avoid unnecessary warnings on downgrade. open_or_create_data_files(): pass only one flushed_lsn parameter xb_load_tablespaces(): pass only one flushed_lsn parameter. buf_page_create(): Improve comment about where FIL_PAGE_FIL_FLUSH_LSN_OR_KEY_VERSION is set. fil_write_flushed_lsn(): A new function, merged from fil_write_lsn_and_arch_no_to_file() and fil_write_flushed_lsn_to_data_files(). Only write to the first page of the system tablespace (page 0:0) if tablespace is encrypted, or write all first pages of system tablespace and invoke fil_flush_file_spaces(FIL_TYPE_TABLESPACE) afterwards. fil_read_first_page(): read flush_lsn and crypt_data only from first datafile. fil_open_single_table_tablespace(): Remove output of LSN, because it was only valid for the system tablespace and the undo tablespaces, not user tablespaces. fil_validate_single_table_tablespace(): Remove output of LSN. checkpoint_now_set(): Use fil_write_flushed_lsn and output a error if operation fails. Remove lsn variable from fsp_open_info. recv_recovery_from_checkpoint_start(): Remove unnecessary second flush_lsn parameter. log_empty_and_mark_files_at_shutdown(): Use fil_writte_flushed_lsn and output error if it fails. open_or_create_data_files(): Pass only one flushed_lsn variable.
2017-05-30 11:30:43 +03:00
*create_new_db = false;
srv_normalize_path_for_win(srv_data_home);
for (i = 0; i < srv_n_data_files; i++) {
ulint dirnamelen;
srv_normalize_path_for_win(srv_data_file_names[i]);
dirnamelen = strlen(srv_data_home);
ut_a(dirnamelen + strlen(srv_data_file_names[i])
< (sizeof name) - 1);
memcpy(name, srv_data_home, dirnamelen);
/* Add a path separator if needed. */
if (dirnamelen && name[dirnamelen - 1] != SRV_PATH_SEPARATOR) {
name[dirnamelen++] = SRV_PATH_SEPARATOR;
}
strcpy(name + dirnamelen, srv_data_file_names[i]);
/* Note: It will return true if the file doesn' exist. */
if (!srv_file_check_mode(name)) {
return(DB_FAIL);
} else if (srv_data_file_is_raw_partition[i] == 0) {
/* First we try to create the file: if it already
exists, ret will get value FALSE */
files[i] = os_file_create(
innodb_file_data_key, name, OS_FILE_CREATE,
OS_FILE_NORMAL, OS_DATA_FILE, &ret, FALSE);
if (srv_read_only_mode) {
if (ret) {
goto size_check;
}
ib_logf(IB_LOG_LEVEL_ERROR,
"Opening %s failed!", name);
return(DB_ERROR);
} else if (!ret
&& os_file_get_last_error(false)
!= OS_FILE_ALREADY_EXISTS
#ifdef UNIV_AIX
2017-05-15 17:17:16 +03:00
/* AIX 5.1 after security patch ML7 may have
errno set to 0 here, which causes our
function to return 100; work around that
AIX problem */
&& os_file_get_last_error(false) != 100
#endif /* UNIV_AIX */
) {
ib_logf(IB_LOG_LEVEL_ERROR,
"Creating or opening %s failed!",
name);
return(DB_ERROR);
}
} else if (srv_data_file_is_raw_partition[i] == SRV_NEW_RAW) {
ut_a(!srv_read_only_mode);
/* The partition is opened, not created; then it is
written over */
srv_start_raw_disk_in_use = TRUE;
srv_created_new_raw = TRUE;
files[i] = os_file_create(
innodb_file_data_key, name, OS_FILE_OPEN_RAW,
OS_FILE_NORMAL, OS_DATA_FILE, &ret, FALSE);
if (!ret) {
ib_logf(IB_LOG_LEVEL_ERROR,
"Error in opening %s", name);
return(DB_ERROR);
}
2015-06-16 10:57:05 +02:00
const char* check_msg;
MDEV-12113: install_db shows corruption for rest encryption with innodb_data_file_path=ibdata1:3M; Problem was that FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION field that for encrypted pages even in system datafiles should contain key_version except very first page (0:0) is after encryption overwritten with flush lsn. Ported WL#7990 Repurpose FIL_PAGE_FLUSH_LSN to 10.1 The field FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION is consulted during InnoDB startup. At startup, InnoDB reads the FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION from the first page of each file in the InnoDB system tablespace. If there are multiple files, the minimum and maximum LSN can differ. These numbers are passed to InnoDB startup. Having the number in other files than the first file of the InnoDB system tablespace is not providing much additional value. It is conflicting with other use of the field, such as on InnoDB R-tree index pages and encryption key_version. This worklog will stop writing FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION to other files than the first file of the InnoDB system tablespace (page number 0:0) when system tablespace is encrypted. If tablespace is not encrypted we continue writing FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION to all first pages of system tablespace to avoid unnecessary warnings on downgrade. open_or_create_data_files(): pass only one flushed_lsn parameter xb_load_tablespaces(): pass only one flushed_lsn parameter. buf_page_create(): Improve comment about where FIL_PAGE_FIL_FLUSH_LSN_OR_KEY_VERSION is set. fil_write_flushed_lsn(): A new function, merged from fil_write_lsn_and_arch_no_to_file() and fil_write_flushed_lsn_to_data_files(). Only write to the first page of the system tablespace (page 0:0) if tablespace is encrypted, or write all first pages of system tablespace and invoke fil_flush_file_spaces(FIL_TYPE_TABLESPACE) afterwards. fil_read_first_page(): read flush_lsn and crypt_data only from first datafile. fil_open_single_table_tablespace(): Remove output of LSN, because it was only valid for the system tablespace and the undo tablespaces, not user tablespaces. fil_validate_single_table_tablespace(): Remove output of LSN. checkpoint_now_set(): Use fil_write_flushed_lsn and output a error if operation fails. Remove lsn variable from fsp_open_info. recv_recovery_from_checkpoint_start(): Remove unnecessary second flush_lsn parameter. log_empty_and_mark_files_at_shutdown(): Use fil_writte_flushed_lsn and output error if it fails. open_or_create_data_files(): Pass only one flushed_lsn variable.
2017-05-30 11:30:43 +03:00
2015-06-16 10:57:05 +02:00
check_msg = fil_read_first_page(
files[i], FALSE, &flags, &space,
#ifdef UNIV_LOG_ARCHIVE
min_arch_log_no, max_arch_log_no,
#endif /* UNIV_LOG_ARCHIVE */
MDEV-12113: install_db shows corruption for rest encryption with innodb_data_file_path=ibdata1:3M; Problem was that FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION field that for encrypted pages even in system datafiles should contain key_version except very first page (0:0) is after encryption overwritten with flush lsn. Ported WL#7990 Repurpose FIL_PAGE_FLUSH_LSN to 10.1 The field FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION is consulted during InnoDB startup. At startup, InnoDB reads the FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION from the first page of each file in the InnoDB system tablespace. If there are multiple files, the minimum and maximum LSN can differ. These numbers are passed to InnoDB startup. Having the number in other files than the first file of the InnoDB system tablespace is not providing much additional value. It is conflicting with other use of the field, such as on InnoDB R-tree index pages and encryption key_version. This worklog will stop writing FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION to other files than the first file of the InnoDB system tablespace (page number 0:0) when system tablespace is encrypted. If tablespace is not encrypted we continue writing FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION to all first pages of system tablespace to avoid unnecessary warnings on downgrade. open_or_create_data_files(): pass only one flushed_lsn parameter xb_load_tablespaces(): pass only one flushed_lsn parameter. buf_page_create(): Improve comment about where FIL_PAGE_FIL_FLUSH_LSN_OR_KEY_VERSION is set. fil_write_flushed_lsn(): A new function, merged from fil_write_lsn_and_arch_no_to_file() and fil_write_flushed_lsn_to_data_files(). Only write to the first page of the system tablespace (page 0:0) if tablespace is encrypted, or write all first pages of system tablespace and invoke fil_flush_file_spaces(FIL_TYPE_TABLESPACE) afterwards. fil_read_first_page(): read flush_lsn and crypt_data only from first datafile. fil_open_single_table_tablespace(): Remove output of LSN, because it was only valid for the system tablespace and the undo tablespaces, not user tablespaces. fil_validate_single_table_tablespace(): Remove output of LSN. checkpoint_now_set(): Use fil_write_flushed_lsn and output a error if operation fails. Remove lsn variable from fsp_open_info. recv_recovery_from_checkpoint_start(): Remove unnecessary second flush_lsn parameter. log_empty_and_mark_files_at_shutdown(): Use fil_writte_flushed_lsn and output error if it fails. open_or_create_data_files(): Pass only one flushed_lsn variable.
2017-05-30 11:30:43 +03:00
flushed_lsn, NULL);
2015-06-16 10:57:05 +02:00
/* If first page is valid, don't overwrite DB.
It prevents overwriting DB when mysql_install_db
starts mysqld multiple times during bootstrap. */
if (check_msg == NULL) {
srv_created_new_raw = FALSE;
ret = FALSE;
}
} else if (srv_data_file_is_raw_partition[i] == SRV_OLD_RAW) {
srv_start_raw_disk_in_use = TRUE;
ret = FALSE;
} else {
ut_a(0);
}
if (ret == FALSE) {
const char* check_msg;
/* We open the data file */
if (one_created) {
ib_logf(IB_LOG_LEVEL_ERROR,
"Data files can only be added at "
"the end of a tablespace, but "
"data file %s existed beforehand.",
name);
return(DB_ERROR);
}
MDEV-12113: install_db shows corruption for rest encryption with innodb_data_file_path=ibdata1:3M; Problem was that FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION field that for encrypted pages even in system datafiles should contain key_version except very first page (0:0) is after encryption overwritten with flush lsn. Ported WL#7990 Repurpose FIL_PAGE_FLUSH_LSN to 10.1 The field FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION is consulted during InnoDB startup. At startup, InnoDB reads the FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION from the first page of each file in the InnoDB system tablespace. If there are multiple files, the minimum and maximum LSN can differ. These numbers are passed to InnoDB startup. Having the number in other files than the first file of the InnoDB system tablespace is not providing much additional value. It is conflicting with other use of the field, such as on InnoDB R-tree index pages and encryption key_version. This worklog will stop writing FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION to other files than the first file of the InnoDB system tablespace (page number 0:0) when system tablespace is encrypted. If tablespace is not encrypted we continue writing FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION to all first pages of system tablespace to avoid unnecessary warnings on downgrade. open_or_create_data_files(): pass only one flushed_lsn parameter xb_load_tablespaces(): pass only one flushed_lsn parameter. buf_page_create(): Improve comment about where FIL_PAGE_FIL_FLUSH_LSN_OR_KEY_VERSION is set. fil_write_flushed_lsn(): A new function, merged from fil_write_lsn_and_arch_no_to_file() and fil_write_flushed_lsn_to_data_files(). Only write to the first page of the system tablespace (page 0:0) if tablespace is encrypted, or write all first pages of system tablespace and invoke fil_flush_file_spaces(FIL_TYPE_TABLESPACE) afterwards. fil_read_first_page(): read flush_lsn and crypt_data only from first datafile. fil_open_single_table_tablespace(): Remove output of LSN, because it was only valid for the system tablespace and the undo tablespaces, not user tablespaces. fil_validate_single_table_tablespace(): Remove output of LSN. checkpoint_now_set(): Use fil_write_flushed_lsn and output a error if operation fails. Remove lsn variable from fsp_open_info. recv_recovery_from_checkpoint_start(): Remove unnecessary second flush_lsn parameter. log_empty_and_mark_files_at_shutdown(): Use fil_writte_flushed_lsn and output error if it fails. open_or_create_data_files(): Pass only one flushed_lsn variable.
2017-05-30 11:30:43 +03:00
if (srv_data_file_is_raw_partition[i] == SRV_OLD_RAW) {
ut_a(!srv_read_only_mode);
files[i] = os_file_create(
innodb_file_data_key,
name, OS_FILE_OPEN_RAW,
OS_FILE_NORMAL, OS_DATA_FILE, &ret, FALSE);
} else if (i == 0) {
files[i] = os_file_create(
innodb_file_data_key,
name, OS_FILE_OPEN_RETRY,
OS_FILE_NORMAL, OS_DATA_FILE, &ret, FALSE);
} else {
files[i] = os_file_create(
innodb_file_data_key,
name, OS_FILE_OPEN, OS_FILE_NORMAL,
OS_DATA_FILE, &ret, FALSE);
}
if (!ret) {
os_file_get_last_error(true);
ib_logf(IB_LOG_LEVEL_ERROR,
"Can't open '%s'", name);
return(DB_ERROR);
}
if (srv_data_file_is_raw_partition[i] == SRV_OLD_RAW) {
goto skip_size_check;
}
size_check:
size = os_file_get_size(files[i]);
ut_a(size != (os_offset_t) -1);
MDEV-11556 InnoDB redo log apply fails to adjust data file sizes fil_space_t::recv_size: New member: recovered tablespace size in pages; 0 if no size change was read from the redo log, or if the size change was implemented. fil_space_set_recv_size(): New function for setting space->recv_size. innodb_data_file_size_debug: A debug parameter for setting the system tablespace size in recovery even when the redo log does not contain any size changes. It is hard to write a small test case that would cause the system tablespace to be extended at the critical moment. recv_parse_log_rec(): Note those tablespaces whose size is being changed by the redo log, by invoking fil_space_set_recv_size(). innobase_init(): Correct an error message, and do not require a larger innodb_buffer_pool_size when starting up with a smaller innodb_page_size. innobase_start_or_create_for_mysql(): Allow startup with any initial size of the ibdata1 file if the autoextend attribute is set. Require the minimum size of fixed-size system tablespaces to be 640 pages, not 10 megabytes. Implement innodb_data_file_size_debug. open_or_create_data_files(): Round the system tablespace size down to pages, not to full megabytes, (Our test truncates the system tablespace to more than 800 pages with innodb_page_size=4k. InnoDB should not imagine that it was truncated to 768 pages and then overwrite good pages in the tablespace.) fil_flush_low(): Refactored from fil_flush(). fil_space_extend_must_retry(): Refactored from fil_extend_space_to_desired_size(). fil_mutex_enter_and_prepare_for_io(): Extend the tablespace if fil_space_set_recv_size() was called. The test case has been successfully run with all the innodb_page_size values 4k, 8k, 16k, 32k, 64k.
2016-12-28 12:05:43 +02:00
/* If InnoDB encountered an error or was killed
while extending the data file, the last page
could be incomplete. */
2015-12-13 10:11:49 +01:00
MDEV-11556 InnoDB redo log apply fails to adjust data file sizes fil_space_t::recv_size: New member: recovered tablespace size in pages; 0 if no size change was read from the redo log, or if the size change was implemented. fil_space_set_recv_size(): New function for setting space->recv_size. innodb_data_file_size_debug: A debug parameter for setting the system tablespace size in recovery even when the redo log does not contain any size changes. It is hard to write a small test case that would cause the system tablespace to be extended at the critical moment. recv_parse_log_rec(): Note those tablespaces whose size is being changed by the redo log, by invoking fil_space_set_recv_size(). innobase_init(): Correct an error message, and do not require a larger innodb_buffer_pool_size when starting up with a smaller innodb_page_size. innobase_start_or_create_for_mysql(): Allow startup with any initial size of the ibdata1 file if the autoextend attribute is set. Require the minimum size of fixed-size system tablespaces to be 640 pages, not 10 megabytes. Implement innodb_data_file_size_debug. open_or_create_data_files(): Round the system tablespace size down to pages, not to full megabytes, (Our test truncates the system tablespace to more than 800 pages with innodb_page_size=4k. InnoDB should not imagine that it was truncated to 768 pages and then overwrite good pages in the tablespace.) fil_flush_low(): Refactored from fil_flush(). fil_space_extend_must_retry(): Refactored from fil_extend_space_to_desired_size(). fil_mutex_enter_and_prepare_for_io(): Extend the tablespace if fil_space_set_recv_size() was called. The test case has been successfully run with all the innodb_page_size values 4k, 8k, 16k, 32k, 64k.
2016-12-28 12:05:43 +02:00
rounded_size_pages = static_cast<ulint>(
size >> UNIV_PAGE_SIZE_SHIFT);
if (i == srv_n_data_files - 1
&& srv_auto_extend_last_data_file) {
if (srv_data_file_sizes[i] > rounded_size_pages
|| (srv_last_file_size_max > 0
&& srv_last_file_size_max
< rounded_size_pages)) {
ib_logf(IB_LOG_LEVEL_ERROR,
"auto-extending "
"data file %s is "
"of a different size "
MDEV-12113: install_db shows corruption for rest encryption with innodb_data_file_path=ibdata1:3M; Problem was that FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION field that for encrypted pages even in system datafiles should contain key_version except very first page (0:0) is after encryption overwritten with flush lsn. Ported WL#7990 Repurpose FIL_PAGE_FLUSH_LSN to 10.1 The field FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION is consulted during InnoDB startup. At startup, InnoDB reads the FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION from the first page of each file in the InnoDB system tablespace. If there are multiple files, the minimum and maximum LSN can differ. These numbers are passed to InnoDB startup. Having the number in other files than the first file of the InnoDB system tablespace is not providing much additional value. It is conflicting with other use of the field, such as on InnoDB R-tree index pages and encryption key_version. This worklog will stop writing FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION to other files than the first file of the InnoDB system tablespace (page number 0:0) when system tablespace is encrypted. If tablespace is not encrypted we continue writing FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION to all first pages of system tablespace to avoid unnecessary warnings on downgrade. open_or_create_data_files(): pass only one flushed_lsn parameter xb_load_tablespaces(): pass only one flushed_lsn parameter. buf_page_create(): Improve comment about where FIL_PAGE_FIL_FLUSH_LSN_OR_KEY_VERSION is set. fil_write_flushed_lsn(): A new function, merged from fil_write_lsn_and_arch_no_to_file() and fil_write_flushed_lsn_to_data_files(). Only write to the first page of the system tablespace (page 0:0) if tablespace is encrypted, or write all first pages of system tablespace and invoke fil_flush_file_spaces(FIL_TYPE_TABLESPACE) afterwards. fil_read_first_page(): read flush_lsn and crypt_data only from first datafile. fil_open_single_table_tablespace(): Remove output of LSN, because it was only valid for the system tablespace and the undo tablespaces, not user tablespaces. fil_validate_single_table_tablespace(): Remove output of LSN. checkpoint_now_set(): Use fil_write_flushed_lsn and output a error if operation fails. Remove lsn variable from fsp_open_info. recv_recovery_from_checkpoint_start(): Remove unnecessary second flush_lsn parameter. log_empty_and_mark_files_at_shutdown(): Use fil_writte_flushed_lsn and output error if it fails. open_or_create_data_files(): Pass only one flushed_lsn variable.
2017-05-30 11:30:43 +03:00
ULINTPF " pages (rounded "
"down to MB) than specified "
"in the .cnf file: "
MDEV-12113: install_db shows corruption for rest encryption with innodb_data_file_path=ibdata1:3M; Problem was that FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION field that for encrypted pages even in system datafiles should contain key_version except very first page (0:0) is after encryption overwritten with flush lsn. Ported WL#7990 Repurpose FIL_PAGE_FLUSH_LSN to 10.1 The field FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION is consulted during InnoDB startup. At startup, InnoDB reads the FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION from the first page of each file in the InnoDB system tablespace. If there are multiple files, the minimum and maximum LSN can differ. These numbers are passed to InnoDB startup. Having the number in other files than the first file of the InnoDB system tablespace is not providing much additional value. It is conflicting with other use of the field, such as on InnoDB R-tree index pages and encryption key_version. This worklog will stop writing FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION to other files than the first file of the InnoDB system tablespace (page number 0:0) when system tablespace is encrypted. If tablespace is not encrypted we continue writing FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION to all first pages of system tablespace to avoid unnecessary warnings on downgrade. open_or_create_data_files(): pass only one flushed_lsn parameter xb_load_tablespaces(): pass only one flushed_lsn parameter. buf_page_create(): Improve comment about where FIL_PAGE_FIL_FLUSH_LSN_OR_KEY_VERSION is set. fil_write_flushed_lsn(): A new function, merged from fil_write_lsn_and_arch_no_to_file() and fil_write_flushed_lsn_to_data_files(). Only write to the first page of the system tablespace (page 0:0) if tablespace is encrypted, or write all first pages of system tablespace and invoke fil_flush_file_spaces(FIL_TYPE_TABLESPACE) afterwards. fil_read_first_page(): read flush_lsn and crypt_data only from first datafile. fil_open_single_table_tablespace(): Remove output of LSN, because it was only valid for the system tablespace and the undo tablespaces, not user tablespaces. fil_validate_single_table_tablespace(): Remove output of LSN. checkpoint_now_set(): Use fil_write_flushed_lsn and output a error if operation fails. Remove lsn variable from fsp_open_info. recv_recovery_from_checkpoint_start(): Remove unnecessary second flush_lsn parameter. log_empty_and_mark_files_at_shutdown(): Use fil_writte_flushed_lsn and output error if it fails. open_or_create_data_files(): Pass only one flushed_lsn variable.
2017-05-30 11:30:43 +03:00
"initial " ULINTPF " pages, "
"max " ULINTPF " (relevant if "
"non-zero) pages!",
name,
MDEV-12113: install_db shows corruption for rest encryption with innodb_data_file_path=ibdata1:3M; Problem was that FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION field that for encrypted pages even in system datafiles should contain key_version except very first page (0:0) is after encryption overwritten with flush lsn. Ported WL#7990 Repurpose FIL_PAGE_FLUSH_LSN to 10.1 The field FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION is consulted during InnoDB startup. At startup, InnoDB reads the FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION from the first page of each file in the InnoDB system tablespace. If there are multiple files, the minimum and maximum LSN can differ. These numbers are passed to InnoDB startup. Having the number in other files than the first file of the InnoDB system tablespace is not providing much additional value. It is conflicting with other use of the field, such as on InnoDB R-tree index pages and encryption key_version. This worklog will stop writing FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION to other files than the first file of the InnoDB system tablespace (page number 0:0) when system tablespace is encrypted. If tablespace is not encrypted we continue writing FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION to all first pages of system tablespace to avoid unnecessary warnings on downgrade. open_or_create_data_files(): pass only one flushed_lsn parameter xb_load_tablespaces(): pass only one flushed_lsn parameter. buf_page_create(): Improve comment about where FIL_PAGE_FIL_FLUSH_LSN_OR_KEY_VERSION is set. fil_write_flushed_lsn(): A new function, merged from fil_write_lsn_and_arch_no_to_file() and fil_write_flushed_lsn_to_data_files(). Only write to the first page of the system tablespace (page 0:0) if tablespace is encrypted, or write all first pages of system tablespace and invoke fil_flush_file_spaces(FIL_TYPE_TABLESPACE) afterwards. fil_read_first_page(): read flush_lsn and crypt_data only from first datafile. fil_open_single_table_tablespace(): Remove output of LSN, because it was only valid for the system tablespace and the undo tablespaces, not user tablespaces. fil_validate_single_table_tablespace(): Remove output of LSN. checkpoint_now_set(): Use fil_write_flushed_lsn and output a error if operation fails. Remove lsn variable from fsp_open_info. recv_recovery_from_checkpoint_start(): Remove unnecessary second flush_lsn parameter. log_empty_and_mark_files_at_shutdown(): Use fil_writte_flushed_lsn and output error if it fails. open_or_create_data_files(): Pass only one flushed_lsn variable.
2017-05-30 11:30:43 +03:00
rounded_size_pages,
srv_data_file_sizes[i],
srv_last_file_size_max);
return(DB_ERROR);
}
srv_data_file_sizes[i] = rounded_size_pages;
}
if (rounded_size_pages != srv_data_file_sizes[i]) {
ib_logf(IB_LOG_LEVEL_ERROR,
"Data file %s is of a different "
MDEV-12113: install_db shows corruption for rest encryption with innodb_data_file_path=ibdata1:3M; Problem was that FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION field that for encrypted pages even in system datafiles should contain key_version except very first page (0:0) is after encryption overwritten with flush lsn. Ported WL#7990 Repurpose FIL_PAGE_FLUSH_LSN to 10.1 The field FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION is consulted during InnoDB startup. At startup, InnoDB reads the FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION from the first page of each file in the InnoDB system tablespace. If there are multiple files, the minimum and maximum LSN can differ. These numbers are passed to InnoDB startup. Having the number in other files than the first file of the InnoDB system tablespace is not providing much additional value. It is conflicting with other use of the field, such as on InnoDB R-tree index pages and encryption key_version. This worklog will stop writing FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION to other files than the first file of the InnoDB system tablespace (page number 0:0) when system tablespace is encrypted. If tablespace is not encrypted we continue writing FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION to all first pages of system tablespace to avoid unnecessary warnings on downgrade. open_or_create_data_files(): pass only one flushed_lsn parameter xb_load_tablespaces(): pass only one flushed_lsn parameter. buf_page_create(): Improve comment about where FIL_PAGE_FIL_FLUSH_LSN_OR_KEY_VERSION is set. fil_write_flushed_lsn(): A new function, merged from fil_write_lsn_and_arch_no_to_file() and fil_write_flushed_lsn_to_data_files(). Only write to the first page of the system tablespace (page 0:0) if tablespace is encrypted, or write all first pages of system tablespace and invoke fil_flush_file_spaces(FIL_TYPE_TABLESPACE) afterwards. fil_read_first_page(): read flush_lsn and crypt_data only from first datafile. fil_open_single_table_tablespace(): Remove output of LSN, because it was only valid for the system tablespace and the undo tablespaces, not user tablespaces. fil_validate_single_table_tablespace(): Remove output of LSN. checkpoint_now_set(): Use fil_write_flushed_lsn and output a error if operation fails. Remove lsn variable from fsp_open_info. recv_recovery_from_checkpoint_start(): Remove unnecessary second flush_lsn parameter. log_empty_and_mark_files_at_shutdown(): Use fil_writte_flushed_lsn and output error if it fails. open_or_create_data_files(): Pass only one flushed_lsn variable.
2017-05-30 11:30:43 +03:00
"size " ULINTPF " pages (rounded down to MB) "
"than specified in the .cnf file "
MDEV-12113: install_db shows corruption for rest encryption with innodb_data_file_path=ibdata1:3M; Problem was that FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION field that for encrypted pages even in system datafiles should contain key_version except very first page (0:0) is after encryption overwritten with flush lsn. Ported WL#7990 Repurpose FIL_PAGE_FLUSH_LSN to 10.1 The field FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION is consulted during InnoDB startup. At startup, InnoDB reads the FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION from the first page of each file in the InnoDB system tablespace. If there are multiple files, the minimum and maximum LSN can differ. These numbers are passed to InnoDB startup. Having the number in other files than the first file of the InnoDB system tablespace is not providing much additional value. It is conflicting with other use of the field, such as on InnoDB R-tree index pages and encryption key_version. This worklog will stop writing FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION to other files than the first file of the InnoDB system tablespace (page number 0:0) when system tablespace is encrypted. If tablespace is not encrypted we continue writing FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION to all first pages of system tablespace to avoid unnecessary warnings on downgrade. open_or_create_data_files(): pass only one flushed_lsn parameter xb_load_tablespaces(): pass only one flushed_lsn parameter. buf_page_create(): Improve comment about where FIL_PAGE_FIL_FLUSH_LSN_OR_KEY_VERSION is set. fil_write_flushed_lsn(): A new function, merged from fil_write_lsn_and_arch_no_to_file() and fil_write_flushed_lsn_to_data_files(). Only write to the first page of the system tablespace (page 0:0) if tablespace is encrypted, or write all first pages of system tablespace and invoke fil_flush_file_spaces(FIL_TYPE_TABLESPACE) afterwards. fil_read_first_page(): read flush_lsn and crypt_data only from first datafile. fil_open_single_table_tablespace(): Remove output of LSN, because it was only valid for the system tablespace and the undo tablespaces, not user tablespaces. fil_validate_single_table_tablespace(): Remove output of LSN. checkpoint_now_set(): Use fil_write_flushed_lsn and output a error if operation fails. Remove lsn variable from fsp_open_info. recv_recovery_from_checkpoint_start(): Remove unnecessary second flush_lsn parameter. log_empty_and_mark_files_at_shutdown(): Use fil_writte_flushed_lsn and output error if it fails. open_or_create_data_files(): Pass only one flushed_lsn variable.
2017-05-30 11:30:43 +03:00
ULINTPF " pages!",
name,
MDEV-12113: install_db shows corruption for rest encryption with innodb_data_file_path=ibdata1:3M; Problem was that FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION field that for encrypted pages even in system datafiles should contain key_version except very first page (0:0) is after encryption overwritten with flush lsn. Ported WL#7990 Repurpose FIL_PAGE_FLUSH_LSN to 10.1 The field FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION is consulted during InnoDB startup. At startup, InnoDB reads the FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION from the first page of each file in the InnoDB system tablespace. If there are multiple files, the minimum and maximum LSN can differ. These numbers are passed to InnoDB startup. Having the number in other files than the first file of the InnoDB system tablespace is not providing much additional value. It is conflicting with other use of the field, such as on InnoDB R-tree index pages and encryption key_version. This worklog will stop writing FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION to other files than the first file of the InnoDB system tablespace (page number 0:0) when system tablespace is encrypted. If tablespace is not encrypted we continue writing FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION to all first pages of system tablespace to avoid unnecessary warnings on downgrade. open_or_create_data_files(): pass only one flushed_lsn parameter xb_load_tablespaces(): pass only one flushed_lsn parameter. buf_page_create(): Improve comment about where FIL_PAGE_FIL_FLUSH_LSN_OR_KEY_VERSION is set. fil_write_flushed_lsn(): A new function, merged from fil_write_lsn_and_arch_no_to_file() and fil_write_flushed_lsn_to_data_files(). Only write to the first page of the system tablespace (page 0:0) if tablespace is encrypted, or write all first pages of system tablespace and invoke fil_flush_file_spaces(FIL_TYPE_TABLESPACE) afterwards. fil_read_first_page(): read flush_lsn and crypt_data only from first datafile. fil_open_single_table_tablespace(): Remove output of LSN, because it was only valid for the system tablespace and the undo tablespaces, not user tablespaces. fil_validate_single_table_tablespace(): Remove output of LSN. checkpoint_now_set(): Use fil_write_flushed_lsn and output a error if operation fails. Remove lsn variable from fsp_open_info. recv_recovery_from_checkpoint_start(): Remove unnecessary second flush_lsn parameter. log_empty_and_mark_files_at_shutdown(): Use fil_writte_flushed_lsn and output error if it fails. open_or_create_data_files(): Pass only one flushed_lsn variable.
2017-05-30 11:30:43 +03:00
rounded_size_pages,
srv_data_file_sizes[i]);
return(DB_ERROR);
}
skip_size_check:
2014-05-06 21:13:16 +02:00
/* This is the earliest location where we can load
the double write buffer. */
if (i == 0) {
buf_dblwr_init_or_load_pages(
files[i], srv_data_file_names[i], true);
}
bool retry = true;
check_first_page:
check_msg = fil_read_first_page(
files[i], one_opened, &flags, &space,
#ifdef UNIV_LOG_ARCHIVE
min_arch_log_no, max_arch_log_no,
#endif /* UNIV_LOG_ARCHIVE */
MDEV-12113: install_db shows corruption for rest encryption with innodb_data_file_path=ibdata1:3M; Problem was that FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION field that for encrypted pages even in system datafiles should contain key_version except very first page (0:0) is after encryption overwritten with flush lsn. Ported WL#7990 Repurpose FIL_PAGE_FLUSH_LSN to 10.1 The field FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION is consulted during InnoDB startup. At startup, InnoDB reads the FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION from the first page of each file in the InnoDB system tablespace. If there are multiple files, the minimum and maximum LSN can differ. These numbers are passed to InnoDB startup. Having the number in other files than the first file of the InnoDB system tablespace is not providing much additional value. It is conflicting with other use of the field, such as on InnoDB R-tree index pages and encryption key_version. This worklog will stop writing FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION to other files than the first file of the InnoDB system tablespace (page number 0:0) when system tablespace is encrypted. If tablespace is not encrypted we continue writing FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION to all first pages of system tablespace to avoid unnecessary warnings on downgrade. open_or_create_data_files(): pass only one flushed_lsn parameter xb_load_tablespaces(): pass only one flushed_lsn parameter. buf_page_create(): Improve comment about where FIL_PAGE_FIL_FLUSH_LSN_OR_KEY_VERSION is set. fil_write_flushed_lsn(): A new function, merged from fil_write_lsn_and_arch_no_to_file() and fil_write_flushed_lsn_to_data_files(). Only write to the first page of the system tablespace (page 0:0) if tablespace is encrypted, or write all first pages of system tablespace and invoke fil_flush_file_spaces(FIL_TYPE_TABLESPACE) afterwards. fil_read_first_page(): read flush_lsn and crypt_data only from first datafile. fil_open_single_table_tablespace(): Remove output of LSN, because it was only valid for the system tablespace and the undo tablespaces, not user tablespaces. fil_validate_single_table_tablespace(): Remove output of LSN. checkpoint_now_set(): Use fil_write_flushed_lsn and output a error if operation fails. Remove lsn variable from fsp_open_info. recv_recovery_from_checkpoint_start(): Remove unnecessary second flush_lsn parameter. log_empty_and_mark_files_at_shutdown(): Use fil_writte_flushed_lsn and output error if it fails. open_or_create_data_files(): Pass only one flushed_lsn variable.
2017-05-30 11:30:43 +03:00
flushed_lsn, &crypt_data);
if (check_msg) {
2014-05-06 21:13:16 +02:00
if (retry) {
fsp_open_info fsp;
const ulint page_no = 0;
retry = false;
fsp.id = 0;
fsp.filepath = srv_data_file_names[i];
fsp.file = files[i];
if (fil_user_tablespace_restore_page(
&fsp, page_no)) {
goto check_first_page;
}
}
ib_logf(IB_LOG_LEVEL_ERROR,
2014-05-06 21:13:16 +02:00
"%s in data file %s",
check_msg, name);
return(DB_ERROR);
}
/* The first file of the system tablespace must
have space ID = TRX_SYS_SPACE. The FSP_SPACE_ID
field in files greater than ibdata1 are unreliable. */
ut_a(one_opened || space == TRX_SYS_SPACE);
/* Check the flags for the first system tablespace
file only. */
if (!one_opened
&& UNIV_PAGE_SIZE
!= fsp_flags_get_page_size(flags)) {
ib_logf(IB_LOG_LEVEL_ERROR,
MDEV-12113: install_db shows corruption for rest encryption with innodb_data_file_path=ibdata1:3M; Problem was that FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION field that for encrypted pages even in system datafiles should contain key_version except very first page (0:0) is after encryption overwritten with flush lsn. Ported WL#7990 Repurpose FIL_PAGE_FLUSH_LSN to 10.1 The field FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION is consulted during InnoDB startup. At startup, InnoDB reads the FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION from the first page of each file in the InnoDB system tablespace. If there are multiple files, the minimum and maximum LSN can differ. These numbers are passed to InnoDB startup. Having the number in other files than the first file of the InnoDB system tablespace is not providing much additional value. It is conflicting with other use of the field, such as on InnoDB R-tree index pages and encryption key_version. This worklog will stop writing FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION to other files than the first file of the InnoDB system tablespace (page number 0:0) when system tablespace is encrypted. If tablespace is not encrypted we continue writing FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION to all first pages of system tablespace to avoid unnecessary warnings on downgrade. open_or_create_data_files(): pass only one flushed_lsn parameter xb_load_tablespaces(): pass only one flushed_lsn parameter. buf_page_create(): Improve comment about where FIL_PAGE_FIL_FLUSH_LSN_OR_KEY_VERSION is set. fil_write_flushed_lsn(): A new function, merged from fil_write_lsn_and_arch_no_to_file() and fil_write_flushed_lsn_to_data_files(). Only write to the first page of the system tablespace (page 0:0) if tablespace is encrypted, or write all first pages of system tablespace and invoke fil_flush_file_spaces(FIL_TYPE_TABLESPACE) afterwards. fil_read_first_page(): read flush_lsn and crypt_data only from first datafile. fil_open_single_table_tablespace(): Remove output of LSN, because it was only valid for the system tablespace and the undo tablespaces, not user tablespaces. fil_validate_single_table_tablespace(): Remove output of LSN. checkpoint_now_set(): Use fil_write_flushed_lsn and output a error if operation fails. Remove lsn variable from fsp_open_info. recv_recovery_from_checkpoint_start(): Remove unnecessary second flush_lsn parameter. log_empty_and_mark_files_at_shutdown(): Use fil_writte_flushed_lsn and output error if it fails. open_or_create_data_files(): Pass only one flushed_lsn variable.
2017-05-30 11:30:43 +03:00
"Data file \"%s\" uses page size " ULINTPF " ,"
"but the start-up parameter "
MDEV-12113: install_db shows corruption for rest encryption with innodb_data_file_path=ibdata1:3M; Problem was that FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION field that for encrypted pages even in system datafiles should contain key_version except very first page (0:0) is after encryption overwritten with flush lsn. Ported WL#7990 Repurpose FIL_PAGE_FLUSH_LSN to 10.1 The field FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION is consulted during InnoDB startup. At startup, InnoDB reads the FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION from the first page of each file in the InnoDB system tablespace. If there are multiple files, the minimum and maximum LSN can differ. These numbers are passed to InnoDB startup. Having the number in other files than the first file of the InnoDB system tablespace is not providing much additional value. It is conflicting with other use of the field, such as on InnoDB R-tree index pages and encryption key_version. This worklog will stop writing FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION to other files than the first file of the InnoDB system tablespace (page number 0:0) when system tablespace is encrypted. If tablespace is not encrypted we continue writing FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION to all first pages of system tablespace to avoid unnecessary warnings on downgrade. open_or_create_data_files(): pass only one flushed_lsn parameter xb_load_tablespaces(): pass only one flushed_lsn parameter. buf_page_create(): Improve comment about where FIL_PAGE_FIL_FLUSH_LSN_OR_KEY_VERSION is set. fil_write_flushed_lsn(): A new function, merged from fil_write_lsn_and_arch_no_to_file() and fil_write_flushed_lsn_to_data_files(). Only write to the first page of the system tablespace (page 0:0) if tablespace is encrypted, or write all first pages of system tablespace and invoke fil_flush_file_spaces(FIL_TYPE_TABLESPACE) afterwards. fil_read_first_page(): read flush_lsn and crypt_data only from first datafile. fil_open_single_table_tablespace(): Remove output of LSN, because it was only valid for the system tablespace and the undo tablespaces, not user tablespaces. fil_validate_single_table_tablespace(): Remove output of LSN. checkpoint_now_set(): Use fil_write_flushed_lsn and output a error if operation fails. Remove lsn variable from fsp_open_info. recv_recovery_from_checkpoint_start(): Remove unnecessary second flush_lsn parameter. log_empty_and_mark_files_at_shutdown(): Use fil_writte_flushed_lsn and output error if it fails. open_or_create_data_files(): Pass only one flushed_lsn variable.
2017-05-30 11:30:43 +03:00
"is --innodb-page-size=" ULINTPF " .",
name,
fsp_flags_get_page_size(flags),
UNIV_PAGE_SIZE);
return(DB_ERROR);
}
one_opened = TRUE;
} else if (!srv_read_only_mode) {
/* We created the data file and now write it full of
zeros */
one_created = TRUE;
if (i > 0) {
ib_logf(IB_LOG_LEVEL_INFO,
"Data file %s did not"
" exist: new to be created",
name);
} else {
ib_logf(IB_LOG_LEVEL_INFO,
"The first specified "
"data file %s did not exist: "
"a new database to be created!",
name);
*create_new_db = TRUE;
}
ib_logf(IB_LOG_LEVEL_INFO,
MDEV-12113: install_db shows corruption for rest encryption with innodb_data_file_path=ibdata1:3M; Problem was that FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION field that for encrypted pages even in system datafiles should contain key_version except very first page (0:0) is after encryption overwritten with flush lsn. Ported WL#7990 Repurpose FIL_PAGE_FLUSH_LSN to 10.1 The field FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION is consulted during InnoDB startup. At startup, InnoDB reads the FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION from the first page of each file in the InnoDB system tablespace. If there are multiple files, the minimum and maximum LSN can differ. These numbers are passed to InnoDB startup. Having the number in other files than the first file of the InnoDB system tablespace is not providing much additional value. It is conflicting with other use of the field, such as on InnoDB R-tree index pages and encryption key_version. This worklog will stop writing FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION to other files than the first file of the InnoDB system tablespace (page number 0:0) when system tablespace is encrypted. If tablespace is not encrypted we continue writing FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION to all first pages of system tablespace to avoid unnecessary warnings on downgrade. open_or_create_data_files(): pass only one flushed_lsn parameter xb_load_tablespaces(): pass only one flushed_lsn parameter. buf_page_create(): Improve comment about where FIL_PAGE_FIL_FLUSH_LSN_OR_KEY_VERSION is set. fil_write_flushed_lsn(): A new function, merged from fil_write_lsn_and_arch_no_to_file() and fil_write_flushed_lsn_to_data_files(). Only write to the first page of the system tablespace (page 0:0) if tablespace is encrypted, or write all first pages of system tablespace and invoke fil_flush_file_spaces(FIL_TYPE_TABLESPACE) afterwards. fil_read_first_page(): read flush_lsn and crypt_data only from first datafile. fil_open_single_table_tablespace(): Remove output of LSN, because it was only valid for the system tablespace and the undo tablespaces, not user tablespaces. fil_validate_single_table_tablespace(): Remove output of LSN. checkpoint_now_set(): Use fil_write_flushed_lsn and output a error if operation fails. Remove lsn variable from fsp_open_info. recv_recovery_from_checkpoint_start(): Remove unnecessary second flush_lsn parameter. log_empty_and_mark_files_at_shutdown(): Use fil_writte_flushed_lsn and output error if it fails. open_or_create_data_files(): Pass only one flushed_lsn variable.
2017-05-30 11:30:43 +03:00
"Setting file %s size to " ULINTPF " MB",
name,
MDEV-12113: install_db shows corruption for rest encryption with innodb_data_file_path=ibdata1:3M; Problem was that FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION field that for encrypted pages even in system datafiles should contain key_version except very first page (0:0) is after encryption overwritten with flush lsn. Ported WL#7990 Repurpose FIL_PAGE_FLUSH_LSN to 10.1 The field FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION is consulted during InnoDB startup. At startup, InnoDB reads the FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION from the first page of each file in the InnoDB system tablespace. If there are multiple files, the minimum and maximum LSN can differ. These numbers are passed to InnoDB startup. Having the number in other files than the first file of the InnoDB system tablespace is not providing much additional value. It is conflicting with other use of the field, such as on InnoDB R-tree index pages and encryption key_version. This worklog will stop writing FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION to other files than the first file of the InnoDB system tablespace (page number 0:0) when system tablespace is encrypted. If tablespace is not encrypted we continue writing FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION to all first pages of system tablespace to avoid unnecessary warnings on downgrade. open_or_create_data_files(): pass only one flushed_lsn parameter xb_load_tablespaces(): pass only one flushed_lsn parameter. buf_page_create(): Improve comment about where FIL_PAGE_FIL_FLUSH_LSN_OR_KEY_VERSION is set. fil_write_flushed_lsn(): A new function, merged from fil_write_lsn_and_arch_no_to_file() and fil_write_flushed_lsn_to_data_files(). Only write to the first page of the system tablespace (page 0:0) if tablespace is encrypted, or write all first pages of system tablespace and invoke fil_flush_file_spaces(FIL_TYPE_TABLESPACE) afterwards. fil_read_first_page(): read flush_lsn and crypt_data only from first datafile. fil_open_single_table_tablespace(): Remove output of LSN, because it was only valid for the system tablespace and the undo tablespaces, not user tablespaces. fil_validate_single_table_tablespace(): Remove output of LSN. checkpoint_now_set(): Use fil_write_flushed_lsn and output a error if operation fails. Remove lsn variable from fsp_open_info. recv_recovery_from_checkpoint_start(): Remove unnecessary second flush_lsn parameter. log_empty_and_mark_files_at_shutdown(): Use fil_writte_flushed_lsn and output error if it fails. open_or_create_data_files(): Pass only one flushed_lsn variable.
2017-05-30 11:30:43 +03:00
(srv_data_file_sizes[i]
>> (20 - UNIV_PAGE_SIZE_SHIFT)));
ret = os_file_set_size(
name, files[i],
(os_offset_t) srv_data_file_sizes[i]
<< UNIV_PAGE_SIZE_SHIFT
/* TODO: enable page_compression on the
system tablespace and add
, FSP_FLAGS_HAS_PAGE_COMPRESSION(flags)*/);
if (!ret) {
ib_logf(IB_LOG_LEVEL_ERROR,
"Error in creating %s: "
"probably out of disk space",
name);
return(DB_ERROR);
}
*sum_of_new_sizes += srv_data_file_sizes[i];
}
ret = os_file_close(files[i]);
ut_a(ret);
if (i == 0) {
MDEV-11623 MariaDB 10.1 fails to start datadir created with MariaDB 10.0/MySQL 5.6 using innodb-page-size!=16K The storage format of FSP_SPACE_FLAGS was accidentally broken already in MariaDB 10.1.0. This fix is bringing the format in line with other MySQL and MariaDB release series. Please refer to the comments that were added to fsp0fsp.h for details. This is an INCOMPATIBLE CHANGE that affects users of page_compression and non-default innodb_page_size. Upgrading to this release will correct the flags in the data files. If you want to downgrade to earlier MariaDB 10.1.x, please refer to the test innodb.101_compatibility how to reset the FSP_SPACE_FLAGS in the files. NOTE: MariaDB 10.1.0 to 10.1.20 can misinterpret uncompressed data files with innodb_page_size=4k or 64k as compressed innodb_page_size=16k files, and then probably fail when trying to access the pages. See the comments in the function fsp_flags_convert_from_101() for detailed analysis. Move PAGE_COMPRESSION to FSP_SPACE_FLAGS bit position 16. In this way, compressed innodb_page_size=16k tablespaces will not be mistaken for uncompressed ones by MariaDB 10.1.0 to 10.1.20. Derive PAGE_COMPRESSION_LEVEL, ATOMIC_WRITES and DATA_DIR from the dict_table_t::flags when the table is available, in fil_space_for_table_exists_in_mem() or fil_open_single_table_tablespace(). During crash recovery, fil_load_single_table_tablespace() will use innodb_compression_level for the PAGE_COMPRESSION_LEVEL. FSP_FLAGS_MEM_MASK: A bitmap of the memory-only fil_space_t::flags that are not to be written to FSP_SPACE_FLAGS. Currently, these will include PAGE_COMPRESSION_LEVEL, ATOMIC_WRITES and DATA_DIR. Introduce the macro FSP_FLAGS_PAGE_SSIZE(). We only support one innodb_page_size for the whole instance. When creating a dummy tablespace for the redo log, use fil_space_t::flags=0. The flags are never written to the redo log files. Remove many FSP_FLAGS_SET_ macros. dict_tf_verify_flags(): Remove. This is basically only duplicating the logic of dict_tf_to_fsp_flags(), used in a debug assertion. fil_space_t::mark: Remove. This flag was not used for anything. fil_space_for_table_exists_in_mem(): Remove the unnecessary parameter mark_space, and add a parameter for table flags. Check that fil_space_t::flags match the table flags, and adjust the (memory-only) flags based on the table flags. fil_node_open_file(): Remove some redundant or unreachable conditions, do not use stderr for output, and avoid unnecessary server aborts. fil_user_tablespace_restore_page(): Convert the flags, so that the correct page_size will be used when restoring a page from the doublewrite buffer. fil_space_get_page_compressed(), fsp_flags_is_page_compressed(): Remove. It suffices to have fil_space_is_page_compressed(). FSP_FLAGS_WIDTH_DATA_DIR, FSP_FLAGS_WIDTH_PAGE_COMPRESSION_LEVEL, FSP_FLAGS_WIDTH_ATOMIC_WRITES: Remove, because these flags do not exist in the FSP_SPACE_FLAGS but only in memory. fsp_flags_try_adjust(): New function, to adjust the FSP_SPACE_FLAGS in page 0. Called by fil_open_single_table_tablespace(), fil_space_for_table_exists_in_mem(), innobase_start_or_create_for_mysql() except if --innodb-read-only is active. fsp_flags_is_valid(ulint): Reimplement from the scratch, with accurate comments. Do not display any details of detected inconsistencies, because the output could be confusing when dealing with MariaDB 10.1.x data files. fsp_flags_convert_from_101(ulint): Convert flags from buggy MariaDB 10.1.x format, or return ULINT_UNDEFINED if the flags cannot be in MariaDB 10.1.x format. fsp_flags_match(): Check the flags when probing files. Implemented based on fsp_flags_is_valid() and fsp_flags_convert_from_101(). dict_check_tablespaces_and_store_max_id(): Do not access the page after committing the mini-transaction. IMPORT TABLESPACE fixes: AbstractCallback::init(): Convert the flags. FetchIndexRootPages::operator(): Check that the tablespace flags match the table flags. Do not attempt to convert tablespace flags to table flags, because the conversion would necessarily be lossy. PageConverter::update_header(): Write back the correct flags. This takes care of the flags in IMPORT TABLESPACE.
2017-01-14 00:13:16 +02:00
flags = FSP_FLAGS_PAGE_SSIZE();
2014-12-22 16:53:17 +02:00
fil_space_create(name, 0, flags, FIL_TABLESPACE,
MDEV-11738: Mariadb uses 100% of several of my 8 cpus doing nothing MDEV-11581: Mariadb starts InnoDB encryption threads when key has not changed or data scrubbing turned off Background: Key rotation is based on background threads (innodb-encryption-threads) periodically going through all tablespaces on fil_system. For each tablespace current used key version is compared to max key age (innodb-encryption-rotate-key-age). This process naturally takes CPU. Similarly, in same time need for scrubbing is investigated. Currently, key rotation is fully supported on Amazon AWS key management plugin only but InnoDB does not have knowledge what key management plugin is used. This patch re-purposes innodb-encryption-rotate-key-age=0 to disable key rotation and background data scrubbing. All new tables are added to special list for key rotation and key rotation is based on sending a event to background encryption threads instead of using periodic checking (i.e. timeout). fil0fil.cc: Added functions fil_space_acquire_low() to acquire a tablespace when it could be dropped concurrently. This function is used from fil_space_acquire() or fil_space_acquire_silent() that will not print any messages if we try to acquire space that does not exist. fil_space_release() to release a acquired tablespace. fil_space_next() to iterate tablespaces in fil_system using fil_space_acquire() and fil_space_release(). Similarly, fil_space_keyrotation_next() to iterate new list fil_system->rotation_list where new tables. are added if key rotation is disabled. Removed unnecessary functions fil_get_first_space_safe() fil_get_next_space_safe() fil_node_open_file(): After page 0 is read read also crypt_info if it is not yet read. btr_scrub_lock_dict_func() buf_page_check_corrupt() buf_page_encrypt_before_write() buf_merge_or_delete_for_page() lock_print_info_all_transactions() row_fts_psort_info_init() row_truncate_table_for_mysql() row_drop_table_for_mysql() Use fil_space_acquire()/release() to access fil_space_t. buf_page_decrypt_after_read(): Use fil_space_get_crypt_data() because at this point we might not yet have read page 0. fil0crypt.cc/fil0fil.h: Lot of changes. Pass fil_space_t* directly to functions needing it and store fil_space_t* to rotation state. Use fil_space_acquire()/release() when iterating tablespaces and removed unnecessary is_closing from fil_crypt_t. Use fil_space_t::is_stopping() to detect when access to tablespace should be stopped. Removed unnecessary fil_space_get_crypt_data(). fil_space_create(): Inform key rotation that there could be something to do if key rotation is disabled and new table with encryption enabled is created. Remove unnecessary functions fil_get_first_space_safe() and fil_get_next_space_safe(). fil_space_acquire() and fil_space_release() are used instead. Moved fil_space_get_crypt_data() and fil_space_set_crypt_data() to fil0crypt.cc. fsp_header_init(): Acquire fil_space_t*, write crypt_data and release space. check_table_options() Renamed FIL_SPACE_ENCRYPTION_* TO FIL_ENCRYPTION_* i_s.cc: Added ROTATING_OR_FLUSHING field to information_schema.innodb_tablespace_encryption to show current status of key rotation.
2017-03-14 12:56:01 +02:00
crypt_data, (*create_new_db) == true);
}
ut_a(fil_validate());
if (!fil_node_create(name, srv_data_file_sizes[i], 0,
srv_data_file_is_raw_partition[i] != 0)) {
return(DB_ERROR);
}
}
return(DB_SUCCESS);
}
/*********************************************************************//**
Create undo tablespace.
@return DB_SUCCESS or error code */
static
dberr_t
srv_undo_tablespace_create(
/*=======================*/
const char* name, /*!< in: tablespace name */
ulint size) /*!< in: tablespace size in pages */
{
2017-05-15 17:17:16 +03:00
pfs_os_file_t fh;
ibool ret;
dberr_t err = DB_SUCCESS;
os_file_create_subdirs_if_needed(name);
fh = os_file_create(
innodb_file_data_key,
name,
srv_read_only_mode ? OS_FILE_OPEN : OS_FILE_CREATE,
OS_FILE_NORMAL, OS_DATA_FILE, &ret, FALSE);
if (srv_read_only_mode && ret) {
ib_logf(IB_LOG_LEVEL_INFO,
"%s opened in read-only mode", name);
} else if (ret == FALSE) {
if (os_file_get_last_error(false) != OS_FILE_ALREADY_EXISTS
#ifdef UNIV_AIX
/* AIX 5.1 after security patch ML7 may have
errno set to 0 here, which causes our function
to return 100; work around that AIX problem */
&& os_file_get_last_error(false) != 100
#endif /* UNIV_AIX */
) {
ib_logf(IB_LOG_LEVEL_ERROR,
"Can't create UNDO tablespace %s", name);
} else {
ib_logf(IB_LOG_LEVEL_ERROR,
"Creating system tablespace with"
" existing undo tablespaces is not"
" supported. Please delete all undo"
" tablespaces before creating new"
" system tablespace.");
}
err = DB_ERROR;
} else {
ut_a(!srv_read_only_mode);
/* We created the data file and now write it full of zeros */
ib_logf(IB_LOG_LEVEL_INFO,
"Data file %s did not exist: new to be created",
name);
ib_logf(IB_LOG_LEVEL_INFO,
"Setting file %s size to %lu MB",
name, size >> (20 - UNIV_PAGE_SIZE_SHIFT));
ret = os_file_set_size(name, fh, size << UNIV_PAGE_SIZE_SHIFT
/* TODO: enable page_compression on the
system tablespace and add
FSP_FLAGS_HAS_PAGE_COMPRESSION(flags)
*/);
if (!ret) {
ib_logf(IB_LOG_LEVEL_INFO,
"Error in creating %s: probably out of "
"disk space", name);
err = DB_ERROR;
}
os_file_close(fh);
}
return(err);
}
/*********************************************************************//**
Open an undo tablespace.
@return DB_SUCCESS or error code */
static
dberr_t
srv_undo_tablespace_open(
/*=====================*/
const char* name, /*!< in: tablespace name */
ulint space) /*!< in: tablespace id */
{
2017-05-15 17:17:16 +03:00
pfs_os_file_t fh;
dberr_t err = DB_ERROR;
ibool ret;
ulint flags;
if (!srv_file_check_mode(name)) {
ib_logf(IB_LOG_LEVEL_ERROR,
"UNDO tablespaces must be %s!",
srv_read_only_mode ? "writable" : "readable");
return(DB_ERROR);
}
fh = os_file_create(
innodb_file_data_key, name,
OS_FILE_OPEN_RETRY
| OS_FILE_ON_ERROR_NO_EXIT
| OS_FILE_ON_ERROR_SILENT,
OS_FILE_NORMAL,
OS_DATA_FILE,
&ret,
FALSE);
/* If the file open was successful then load the tablespace. */
if (ret) {
os_offset_t size;
size = os_file_get_size(fh);
ut_a(size != (os_offset_t) -1);
/* Load the tablespace into InnoDB's internal
data structures. */
const char* check_msg;
fil_space_crypt_t* crypt_data = NULL;
/* Set the compressed page size to 0 (non-compressed) */
flags = FSP_FLAGS_PAGE_SSIZE();
/* Read first page to find out does the crypt_info
exists on undo tablespace. */
check_msg = fil_read_first_page(
fh, FALSE, &flags, &space,
NULL, &crypt_data, false);
ret = os_file_close(fh);
ut_a(ret);
if (check_msg) {
ib_logf(IB_LOG_LEVEL_ERROR,
"%s in data file %s",
check_msg, name);
return (err);
}
/* We set the biggest space id to the undo tablespace
because InnoDB hasn't opened any other tablespace apart
from the system tablespace. */
fil_set_max_space_id_if_bigger(space);
2014-12-22 16:53:17 +02:00
fil_space_create(name, space, flags, FIL_TABLESPACE,
crypt_data,
true /* create */);
ut_a(fil_validate());
os_offset_t n_pages = size / UNIV_PAGE_SIZE;
/* On 64 bit Windows ulint can be 32 bit and os_offset_t
is 64 bit. It is OK to cast the n_pages to ulint because
the unit has been scaled to pages and they are always
32 bit. */
if (fil_node_create(name, (ulint) n_pages, space, FALSE)) {
err = DB_SUCCESS;
}
}
return(err);
}
/********************************************************************
Opens the configured number of undo tablespaces.
@return DB_SUCCESS or error code */
static
dberr_t
srv_undo_tablespaces_init(
/*======================*/
ibool create_new_db, /*!< in: TRUE if new db being
created */
const ulint n_conf_tablespaces, /*!< in: configured undo
tablespaces */
ulint* n_opened) /*!< out: number of UNDO
tablespaces successfully
discovered and opened */
{
ulint i;
dberr_t err = DB_SUCCESS;
ulint prev_space_id = 0;
ulint n_undo_tablespaces;
ulint undo_tablespace_ids[TRX_SYS_N_RSEGS + 1];
*n_opened = 0;
ut_a(n_conf_tablespaces <= TRX_SYS_N_RSEGS);
memset(undo_tablespace_ids, 0x0, sizeof(undo_tablespace_ids));
/* Create the undo spaces only if we are creating a new
instance. We don't allow creating of new undo tablespaces
in an existing instance (yet). This restriction exists because
we check in several places for SYSTEM tablespaces to be less than
the min of user defined tablespace ids. Once we implement saving
the location of the undo tablespaces and their space ids this
restriction will/should be lifted. */
for (i = 0; create_new_db && i < n_conf_tablespaces; ++i) {
char name[OS_FILE_MAX_PATH];
2017-05-15 17:17:16 +03:00
ulint space_id = i + 1;
DBUG_EXECUTE_IF("innodb_undo_upgrade",
space_id = i + 3;);
ut_snprintf(
name, sizeof(name),
"%s%cundo%03lu",
2017-05-15 17:17:16 +03:00
srv_undo_dir, SRV_PATH_SEPARATOR, space_id);
if (i == 0) {
srv_undo_space_id_start = space_id;
prev_space_id = srv_undo_space_id_start - 1;
}
undo_tablespace_ids[i] = space_id;
err = srv_undo_tablespace_create(
name, SRV_UNDO_TABLESPACE_SIZE_IN_PAGES);
if (err != DB_SUCCESS) {
ib_logf(IB_LOG_LEVEL_ERROR,
"Could not create undo tablespace '%s'.",
name);
return(err);
}
}
/* Get the tablespace ids of all the undo segments excluding
the system tablespace (0). If we are creating a new instance then
we build the undo_tablespace_ids ourselves since they don't
already exist. */
if (!create_new_db) {
n_undo_tablespaces = trx_rseg_get_n_undo_tablespaces(
undo_tablespace_ids);
2017-05-15 17:17:16 +03:00
if (n_undo_tablespaces != 0) {
srv_undo_space_id_start = undo_tablespace_ids[0];
prev_space_id = srv_undo_space_id_start - 1;
}
2017-05-15 17:17:16 +03:00
} else {
n_undo_tablespaces = n_conf_tablespaces;
undo_tablespace_ids[n_conf_tablespaces] = ULINT_UNDEFINED;
}
/* Open all the undo tablespaces that are currently in use. If we
fail to open any of these it is a fatal error. The tablespace ids
should be contiguous. It is a fatal error because they are required
for recovery and are referenced by the UNDO logs (a.k.a RBS). */
for (i = 0; i < n_undo_tablespaces; ++i) {
char name[OS_FILE_MAX_PATH];
ut_snprintf(
name, sizeof(name),
"%s%cundo%03lu",
srv_undo_dir, SRV_PATH_SEPARATOR,
undo_tablespace_ids[i]);
/* Should be no gaps in undo tablespace ids. */
ut_a(prev_space_id + 1 == undo_tablespace_ids[i]);
/* The system space id should not be in this array. */
ut_a(undo_tablespace_ids[i] != 0);
ut_a(undo_tablespace_ids[i] != ULINT_UNDEFINED);
err = srv_undo_tablespace_open(name, undo_tablespace_ids[i]);
if (err != DB_SUCCESS) {
ib_logf(IB_LOG_LEVEL_ERROR,
"Unable to open undo tablespace '%s'.", name);
return(err);
}
prev_space_id = undo_tablespace_ids[i];
++*n_opened;
}
/* Open any extra unused undo tablespaces. These must be contiguous.
We stop at the first failure. These are undo tablespaces that are
not in use and therefore not required by recovery. We only check
that there are no gaps. */
for (i = prev_space_id + 1; i < TRX_SYS_N_RSEGS; ++i) {
char name[OS_FILE_MAX_PATH];
ut_snprintf(
name, sizeof(name),
"%s%cundo%03lu", srv_undo_dir, SRV_PATH_SEPARATOR, i);
/* Undo space ids start from 1. */
err = srv_undo_tablespace_open(name, i);
if (err != DB_SUCCESS) {
break;
}
2017-05-15 17:17:16 +03:00
/** Note the first undo tablespace id in case of
no active undo tablespace. */
if (n_undo_tablespaces == 0) {
srv_undo_space_id_start = i;
}
++n_undo_tablespaces;
++*n_opened;
}
2017-05-15 17:17:16 +03:00
/** Explictly specify the srv_undo_space_id_start
as zero when there are no undo tablespaces. */
if (n_undo_tablespaces == 0) {
srv_undo_space_id_start = 0;
}
/* If the user says that there are fewer than what we find we
tolerate that discrepancy but not the inverse. Because there could
be unused undo tablespaces for future use. */
if (n_conf_tablespaces > n_undo_tablespaces) {
ut_print_timestamp(stderr);
fprintf(stderr,
" InnoDB: Expected to open %lu undo "
"tablespaces but was able\n",
n_conf_tablespaces);
ut_print_timestamp(stderr);
fprintf(stderr,
" InnoDB: to find only %lu undo "
"tablespaces.\n", n_undo_tablespaces);
ut_print_timestamp(stderr);
fprintf(stderr,
" InnoDB: Set the "
"innodb_undo_tablespaces parameter to "
"the\n");
ut_print_timestamp(stderr);
fprintf(stderr,
" InnoDB: correct value and retry. Suggested "
"value is %lu\n", n_undo_tablespaces);
return(err != DB_SUCCESS ? err : DB_ERROR);
} else if (n_undo_tablespaces > 0) {
ib_logf(IB_LOG_LEVEL_INFO, "Opened %lu undo tablespaces",
n_undo_tablespaces);
if (n_conf_tablespaces == 0) {
ib_logf(IB_LOG_LEVEL_WARN,
"Using the system tablespace for all UNDO "
"logging because innodb_undo_tablespaces=0");
}
}
if (create_new_db) {
mtr_t mtr;
mtr_start(&mtr);
/* The undo log tablespace */
2017-05-15 17:17:16 +03:00
for (i = 0; i < n_undo_tablespaces; ++i) {
fsp_header_init(
2017-05-15 17:17:16 +03:00
undo_tablespace_ids[i],
SRV_UNDO_TABLESPACE_SIZE_IN_PAGES, &mtr);
}
mtr_commit(&mtr);
}
return(DB_SUCCESS);
}
/********************************************************************
Wait for the purge thread(s) to start up. */
static
void
srv_start_wait_for_purge_to_start()
/*===============================*/
{
/* Wait for the purge coordinator and master thread to startup. */
purge_state_t state = trx_purge_state();
ut_a(state != PURGE_STATE_DISABLED);
while (srv_shutdown_state == SRV_SHUTDOWN_NONE
&& srv_force_recovery < SRV_FORCE_NO_BACKGROUND
&& state == PURGE_STATE_INIT) {
switch (state = trx_purge_state()) {
case PURGE_STATE_RUN:
case PURGE_STATE_STOP:
break;
case PURGE_STATE_INIT:
ib_logf(IB_LOG_LEVEL_INFO,
"Waiting for purge to start");
os_thread_sleep(50000);
break;
case PURGE_STATE_EXIT:
case PURGE_STATE_DISABLED:
ut_error;
}
}
}
/********************************************************************
Starts InnoDB and creates a new database if database files
are not found and the user wants.
@return DB_SUCCESS or error code */
UNIV_INTERN
dberr_t
MDEV-13039 innodb_fast_shutdown=0 may fail to purge all undo log When a slow shutdown is performed soon after spawning some work for background threads that can create or commit transactions, it is possible that new transactions are started or committed after the purge has finished. This is violating the specification of innodb_fast_shutdown=0, namely that the purge must be completed. (None of the history of the recent transactions would be purged.) Also, it is possible that the purge threads would exit in slow shutdown while there exist active transactions, such as recovered incomplete transactions that are being rolled back. Thus, the slow shutdown could fail to purge some undo log that becomes purgeable after the transaction commit or rollback. srv_undo_sources: A flag that indicates if undo log can be generated or the persistent, whether by background threads or by user SQL. Even when this flag is clear, active transactions that already exist in the system may be committed or rolled back. innodb_shutdown(): Renamed from innobase_shutdown_for_mysql(). Do not return an error code; the operation never fails. Clear the srv_undo_sources flag, and also ensure that the background DROP TABLE queue is empty. srv_purge_should_exit(): Do not allow the purge to exit if srv_undo_sources are active or the background DROP TABLE queue is not empty, or in slow shutdown, if any active transactions exist (and are being rolled back). srv_purge_coordinator_thread(): Remove some previous workarounds for this bug. innobase_start_or_create_for_mysql(): Set buf_page_cleaner_is_active and srv_dict_stats_thread_active directly. Set srv_undo_sources before starting the purge subsystem, to prevent immediate shutdown of the purge. Create dict_stats_thread and fts_optimize_thread immediately after setting srv_undo_sources, so that shutdown can use this flag to determine if these subsystems were started. dict_stats_shutdown(): Shut down dict_stats_thread. Backported from 10.2. srv_shutdown_table_bg_threads(): Remove (unused).
2017-06-08 15:43:06 +03:00
innobase_start_or_create_for_mysql()
{
MDEV-12113: install_db shows corruption for rest encryption with innodb_data_file_path=ibdata1:3M; Problem was that FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION field that for encrypted pages even in system datafiles should contain key_version except very first page (0:0) is after encryption overwritten with flush lsn. Ported WL#7990 Repurpose FIL_PAGE_FLUSH_LSN to 10.1 The field FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION is consulted during InnoDB startup. At startup, InnoDB reads the FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION from the first page of each file in the InnoDB system tablespace. If there are multiple files, the minimum and maximum LSN can differ. These numbers are passed to InnoDB startup. Having the number in other files than the first file of the InnoDB system tablespace is not providing much additional value. It is conflicting with other use of the field, such as on InnoDB R-tree index pages and encryption key_version. This worklog will stop writing FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION to other files than the first file of the InnoDB system tablespace (page number 0:0) when system tablespace is encrypted. If tablespace is not encrypted we continue writing FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION to all first pages of system tablespace to avoid unnecessary warnings on downgrade. open_or_create_data_files(): pass only one flushed_lsn parameter xb_load_tablespaces(): pass only one flushed_lsn parameter. buf_page_create(): Improve comment about where FIL_PAGE_FIL_FLUSH_LSN_OR_KEY_VERSION is set. fil_write_flushed_lsn(): A new function, merged from fil_write_lsn_and_arch_no_to_file() and fil_write_flushed_lsn_to_data_files(). Only write to the first page of the system tablespace (page 0:0) if tablespace is encrypted, or write all first pages of system tablespace and invoke fil_flush_file_spaces(FIL_TYPE_TABLESPACE) afterwards. fil_read_first_page(): read flush_lsn and crypt_data only from first datafile. fil_open_single_table_tablespace(): Remove output of LSN, because it was only valid for the system tablespace and the undo tablespaces, not user tablespaces. fil_validate_single_table_tablespace(): Remove output of LSN. checkpoint_now_set(): Use fil_write_flushed_lsn and output a error if operation fails. Remove lsn variable from fsp_open_info. recv_recovery_from_checkpoint_start(): Remove unnecessary second flush_lsn parameter. log_empty_and_mark_files_at_shutdown(): Use fil_writte_flushed_lsn and output error if it fails. open_or_create_data_files(): Pass only one flushed_lsn variable.
2017-05-30 11:30:43 +03:00
bool create_new_db;
lsn_t flushed_lsn;
#ifdef UNIV_LOG_ARCHIVE
ulint min_arch_log_no;
ulint max_arch_log_no;
#endif /* UNIV_LOG_ARCHIVE */
ulint sum_of_new_sizes;
dberr_t err;
unsigned i;
ulint srv_n_log_files_found = srv_n_log_files;
ulint io_limit;
mtr_t mtr;
ib_bh_t* ib_bh;
ulint n_recovered_trx;
char logfilename[10000];
char* logfile0 = NULL;
size_t dirnamelen;
bool sys_datafiles_created = false;
/* Check that os_fast_mutexes work as expected */
os_fast_mutex_init(PFS_NOT_INSTRUMENTED, &srv_os_test_mutex);
ut_a(0 == os_fast_mutex_trylock(&srv_os_test_mutex));
os_fast_mutex_unlock(&srv_os_test_mutex);
os_fast_mutex_lock(&srv_os_test_mutex);
os_fast_mutex_unlock(&srv_os_test_mutex);
os_fast_mutex_free(&srv_os_test_mutex);
/* This should be initialized early */
ut_init_timer();
2017-05-15 17:17:16 +03:00
if (srv_force_recovery == SRV_FORCE_NO_LOG_REDO) {
srv_read_only_mode = 1;
}
2015-10-09 17:21:46 +02:00
high_level_read_only = srv_read_only_mode
|| srv_force_recovery > SRV_FORCE_NO_TRX_UNDO;
2014-02-26 19:23:04 +01:00
if (srv_read_only_mode) {
ib_logf(IB_LOG_LEVEL_INFO, "Started in read only mode");
}
#ifdef HAVE_DARWIN_THREADS
# ifdef F_FULLFSYNC
/* This executable has been compiled on Mac OS X 10.3 or later.
Assume that F_FULLFSYNC is available at run-time. */
srv_have_fullfsync = TRUE;
# else /* F_FULLFSYNC */
/* This executable has been compiled on Mac OS X 10.2
or earlier. Determine if the executable is running
on Mac OS X 10.3 or later. */
struct utsname utsname;
if (uname(&utsname)) {
ut_print_timestamp(stderr);
fputs(" InnoDB: cannot determine Mac OS X version!\n", stderr);
} else {
srv_have_fullfsync = strcmp(utsname.release, "7.") >= 0;
}
if (!srv_have_fullfsync) {
ut_print_timestamp(stderr);
fputs(" InnoDB: On Mac OS X, fsync() may be "
"broken on internal drives,\n", stderr);
ut_print_timestamp(stderr);
fputs(" InnoDB: making transactions unsafe!\n", stderr);
}
# endif /* F_FULLFSYNC */
#endif /* HAVE_DARWIN_THREADS */
2014-05-05 18:20:28 +02:00
ib_logf(IB_LOG_LEVEL_INFO,
"Using %s to ref count buffer pool pages",
#ifdef PAGE_ATOMIC_REF_COUNT
"atomics"
#else
"mutexes"
#endif /* PAGE_ATOMIC_REF_COUNT */
);
compile_time_assert(sizeof(ulint) == sizeof(void*));
#ifdef UNIV_DEBUG
ib_logf(IB_LOG_LEVEL_INFO,
" InnoDB: !!!!!!!! UNIV_DEBUG switched on !!!!!!!!!");
#endif
#ifdef UNIV_IBUF_DEBUG
ib_logf(IB_LOG_LEVEL_INFO,
" InnoDB: !!!!!!!! UNIV_IBUF_DEBUG switched on !!!!!!!!!");
# ifdef UNIV_IBUF_COUNT_DEBUG
ib_logf(IB_LOG_LEVEL_INFO,
" InnoDB: !!!!!!!! UNIV_IBUF_COUNT_DEBUG switched on "
"!!!!!!!!!");
ib_logf(IB_LOG_LEVEL_INFO,
" InnoDB: Crash recovery will fail with UNIV_IBUF_COUNT_DEBUG");
# endif
#endif
#ifdef UNIV_BLOB_DEBUG
ib_logf(IB_LOG_LEVEL_INFO,
"InnoDB: !!!!!!!! UNIV_BLOB_DEBUG switched on !!!!!!!!!\n"
"InnoDB: Server restart may fail with UNIV_BLOB_DEBUG");
#endif /* UNIV_BLOB_DEBUG */
#ifdef UNIV_SYNC_DEBUG
ib_logf(IB_LOG_LEVEL_INFO,
" InnoDB: !!!!!!!! UNIV_SYNC_DEBUG switched on !!!!!!!!!");
#endif
#ifdef UNIV_SEARCH_DEBUG
ib_logf(IB_LOG_LEVEL_INFO,
" InnoDB: !!!!!!!! UNIV_SEARCH_DEBUG switched on !!!!!!!!!");
#endif
#ifdef UNIV_LOG_LSN_DEBUG
ib_logf(IB_LOG_LEVEL_INFO,
" InnoDB: !!!!!!!! UNIV_LOG_LSN_DEBUG switched on !!!!!!!!!");
#endif /* UNIV_LOG_LSN_DEBUG */
#ifdef UNIV_MEM_DEBUG
ib_logf(IB_LOG_LEVEL_INFO,
" InnoDB: !!!!!!!! UNIV_MEM_DEBUG switched on !!!!!!!!!");
#endif
if (srv_use_sys_malloc) {
ib_logf(IB_LOG_LEVEL_INFO,
"The InnoDB memory heap is disabled");
}
#if defined(COMPILER_HINTS_ENABLED)
ib_logf(IB_LOG_LEVEL_INFO,
" InnoDB: Compiler hints enabled.");
#endif /* defined(COMPILER_HINTS_ENABLED) */
ib_logf(IB_LOG_LEVEL_INFO,
"" IB_ATOMICS_STARTUP_MSG "");
2014-09-11 10:13:35 +02:00
ib_logf(IB_LOG_LEVEL_INFO,
"" IB_MEMORY_BARRIER_STARTUP_MSG "");
#ifndef HAVE_MEMORY_BARRIER
#if defined __i386__ || defined __x86_64__ || defined _M_IX86 || defined _M_X64 || defined __WIN__
#else
ib_logf(IB_LOG_LEVEL_WARN,
"MySQL was built without a memory barrier capability on this"
" architecture, which might allow a mutex/rw_lock violation"
" under high thread concurrency. This may cause a hang.");
#endif /* IA32 or AMD64 */
#endif /* HAVE_MEMORY_BARRIER */
ib_logf(IB_LOG_LEVEL_INFO,
"Compressed tables use zlib " ZLIB_VERSION
#ifdef UNIV_ZIP_DEBUG
" with validation"
#endif /* UNIV_ZIP_DEBUG */
);
#ifdef UNIV_ZIP_COPY
ib_logf(IB_LOG_LEVEL_INFO, "and extra copying");
#endif /* UNIV_ZIP_COPY */
/* Since InnoDB does not currently clean up all its internal data
structures in MySQL Embedded Server Library server_end(), we
print an error message if someone tries to start up InnoDB a
second time during the process lifetime. */
if (srv_start_has_been_called) {
ut_print_timestamp(stderr);
fprintf(stderr, " InnoDB: Error: startup called second time "
"during the process\n");
ut_print_timestamp(stderr);
fprintf(stderr, " InnoDB: lifetime. In the MySQL Embedded "
"Server Library you\n");
ut_print_timestamp(stderr);
fprintf(stderr, " InnoDB: cannot call server_init() more "
"than once during the\n");
ut_print_timestamp(stderr);
fprintf(stderr, " InnoDB: process lifetime.\n");
}
srv_start_has_been_called = TRUE;
#ifdef UNIV_DEBUG
log_do_write = TRUE;
#endif /* UNIV_DEBUG */
/* yydebug = TRUE; */
srv_is_being_started = TRUE;
srv_startup_is_before_trx_rollback_phase = TRUE;
#ifdef __WIN__
switch (os_get_os_version()) {
case OS_WIN95:
case OS_WIN31:
case OS_WINNT:
/* On Win 95, 98, ME, Win32 subsystem for Windows 3.1,
and NT use simulated aio. In NT Windows provides async i/o,
but when run in conjunction with InnoDB Hot Backup, it seemed
to corrupt the data files. */
srv_use_native_aio = FALSE;
break;
case OS_WIN2000:
case OS_WINXP:
/* On 2000 and XP, async IO is available. */
srv_use_native_aio = TRUE;
break;
default:
/* Vista and later have both async IO and condition variables */
srv_use_native_aio = TRUE;
srv_use_native_conditions = TRUE;
break;
}
#elif defined(LINUX_NATIVE_AIO)
if (srv_use_native_aio) {
ib_logf(IB_LOG_LEVEL_INFO, "Using Linux native AIO");
}
#else
/* Currently native AIO is supported only on windows and linux
and that also when the support is compiled in. In all other
cases, we ignore the setting of innodb_use_native_aio. */
srv_use_native_aio = FALSE;
#endif /* __WIN__ */
if (srv_file_flush_method_str == NULL) {
/* These are the default options */
srv_unix_file_flush_method = SRV_UNIX_FSYNC;
srv_win_file_flush_method = SRV_WIN_IO_UNBUFFERED;
#ifndef __WIN__
} else if (0 == ut_strcmp(srv_file_flush_method_str, "fsync")) {
srv_unix_file_flush_method = SRV_UNIX_FSYNC;
} else if (0 == ut_strcmp(srv_file_flush_method_str, "O_DSYNC")) {
srv_unix_file_flush_method = SRV_UNIX_O_DSYNC;
} else if (0 == ut_strcmp(srv_file_flush_method_str, "O_DIRECT")) {
srv_unix_file_flush_method = SRV_UNIX_O_DIRECT;
} else if (0 == ut_strcmp(srv_file_flush_method_str, "O_DIRECT_NO_FSYNC")) {
srv_unix_file_flush_method = SRV_UNIX_O_DIRECT_NO_FSYNC;
} else if (0 == ut_strcmp(srv_file_flush_method_str, "littlesync")) {
srv_unix_file_flush_method = SRV_UNIX_LITTLESYNC;
} else if (0 == ut_strcmp(srv_file_flush_method_str, "nosync")) {
srv_unix_file_flush_method = SRV_UNIX_NOSYNC;
#else
} else if (0 == ut_strcmp(srv_file_flush_method_str, "normal")) {
srv_win_file_flush_method = SRV_WIN_IO_NORMAL;
srv_use_native_aio = FALSE;
} else if (0 == ut_strcmp(srv_file_flush_method_str, "unbuffered")) {
srv_win_file_flush_method = SRV_WIN_IO_UNBUFFERED;
srv_use_native_aio = FALSE;
} else if (0 == ut_strcmp(srv_file_flush_method_str,
"async_unbuffered")) {
srv_win_file_flush_method = SRV_WIN_IO_UNBUFFERED;
#endif /* __WIN__ */
} else {
ib_logf(IB_LOG_LEVEL_ERROR,
"Unrecognized value %s for innodb_flush_method",
srv_file_flush_method_str);
return(DB_ERROR);
}
/* Note that the call srv_boot() also changes the values of
some variables to the units used by InnoDB internally */
/* Set the maximum number of threads which can wait for a semaphore
inside InnoDB: this is the 'sync wait array' size, as well as the
maximum number of threads that can wait in the 'srv_conc array' for
their time to enter InnoDB. */
2014-02-26 19:23:04 +01:00
#define BUF_POOL_SIZE_THRESHOLD (1024 * 1024 * 1024)
srv_max_n_threads = 1 /* io_ibuf_thread */
+ 1 /* io_log_thread */
+ 1 /* lock_wait_timeout_thread */
+ 1 /* srv_error_monitor_thread */
+ 1 /* srv_monitor_thread */
+ 1 /* srv_master_thread */
+ 1 /* srv_purge_coordinator_thread */
+ 1 /* buf_dump_thread */
+ 1 /* dict_stats_thread */
+ 1 /* fts_optimize_thread */
+ 1 /* recv_writer_thread */
+ 1 /* buf_flush_page_cleaner_thread */
+ 1 /* trx_rollback_or_clean_all_recovered */
+ 128 /* added as margin, for use of
InnoDB Memcached etc. */
+ max_connections
+ srv_n_read_io_threads
+ srv_n_write_io_threads
+ srv_n_purge_threads
/* FTS Parallel Sort */
+ fts_sort_pll_degree * FTS_NUM_AUX_INDEX
* max_connections;
if (srv_buf_pool_size < BUF_POOL_SIZE_THRESHOLD) {
/* If buffer pool is less than 1 GB,
use only one buffer pool instance */
srv_buf_pool_instances = 1;
}
srv_boot();
if (ut_crc32_sse2_enabled) {
ib_logf(IB_LOG_LEVEL_INFO, "Using SSE crc32 instructions");
} else if (ut_crc32_power8_enabled) {
ib_logf(IB_LOG_LEVEL_INFO, "Using POWER8 crc32 instructions");
} else {
ib_logf(IB_LOG_LEVEL_INFO, "Using generic crc32 instructions");
}
if (!srv_read_only_mode) {
mutex_create(srv_monitor_file_mutex_key,
&srv_monitor_file_mutex, SYNC_NO_ORDER_CHECK);
if (srv_innodb_status) {
srv_monitor_file_name = static_cast<char*>(
mem_alloc(
strlen(fil_path_to_mysql_datadir)
+ 20 + sizeof "/innodb_status."));
sprintf(srv_monitor_file_name, "%s/innodb_status.%lu",
fil_path_to_mysql_datadir,
os_proc_get_number());
srv_monitor_file = fopen(srv_monitor_file_name, "w+");
if (!srv_monitor_file) {
ib_logf(IB_LOG_LEVEL_ERROR,
"Unable to create %s: %s",
srv_monitor_file_name,
strerror(errno));
return(DB_ERROR);
}
} else {
srv_monitor_file_name = NULL;
2016-02-16 12:07:18 +01:00
srv_monitor_file = os_file_create_tmpfile(NULL);
if (!srv_monitor_file) {
return(DB_ERROR);
}
}
mutex_create(srv_dict_tmpfile_mutex_key,
&srv_dict_tmpfile_mutex, SYNC_DICT_OPERATION);
2016-02-16 12:07:18 +01:00
srv_dict_tmpfile = os_file_create_tmpfile(NULL);
if (!srv_dict_tmpfile) {
return(DB_ERROR);
}
mutex_create(srv_misc_tmpfile_mutex_key,
&srv_misc_tmpfile_mutex, SYNC_ANY_LATCH);
2016-02-16 12:07:18 +01:00
srv_misc_tmpfile = os_file_create_tmpfile(NULL);
if (!srv_misc_tmpfile) {
return(DB_ERROR);
}
}
/* If user has set the value of innodb_file_io_threads then
we'll emit a message telling the user that this parameter
is now deprecated. */
if (srv_n_file_io_threads != 4) {
ib_logf(IB_LOG_LEVEL_WARN,
"innodb_file_io_threads is deprecated. Please use "
"innodb_read_io_threads and innodb_write_io_threads "
"instead");
}
/* Now overwrite the value on srv_n_file_io_threads */
srv_n_file_io_threads = srv_n_read_io_threads;
if (!srv_read_only_mode) {
/* Add the log and ibuf IO threads. */
srv_n_file_io_threads += 2;
srv_n_file_io_threads += srv_n_write_io_threads;
} else {
ib_logf(IB_LOG_LEVEL_INFO,
"Disabling background IO write threads.");
srv_n_write_io_threads = 0;
}
ut_a(srv_n_file_io_threads <= SRV_MAX_N_IO_THREADS);
io_limit = 8 * SRV_N_PENDING_IOS_PER_THREAD;
/* On Windows when using native aio the number of aio requests
that a thread can handle at a given time is limited to 32
i.e.: SRV_N_PENDING_IOS_PER_THREAD */
# ifdef __WIN__
if (srv_use_native_aio) {
io_limit = SRV_N_PENDING_IOS_PER_THREAD;
}
# endif /* __WIN__ */
if (!os_aio_init(io_limit,
srv_n_read_io_threads,
srv_n_write_io_threads,
SRV_MAX_N_PENDING_SYNC_IOS)) {
ib_logf(IB_LOG_LEVEL_ERROR,
"Fatal : Cannot initialize AIO sub-system");
return(DB_ERROR);
}
fil_init(srv_file_per_table ? 50000 : 5000, srv_max_n_open_files);
double size;
char unit;
if (srv_buf_pool_size >= 1024 * 1024 * 1024) {
size = ((double) srv_buf_pool_size) / (1024 * 1024 * 1024);
unit = 'G';
} else {
size = ((double) srv_buf_pool_size) / (1024 * 1024);
unit = 'M';
}
/* Print time to initialize the buffer pool */
ib_logf(IB_LOG_LEVEL_INFO,
"Initializing buffer pool, size = %.1f%c", size, unit);
err = buf_pool_init(srv_buf_pool_size, srv_buf_pool_instances);
if (err != DB_SUCCESS) {
ib_logf(IB_LOG_LEVEL_ERROR,
"Cannot allocate memory for the buffer pool");
return(DB_ERROR);
}
ib_logf(IB_LOG_LEVEL_INFO,
"Completed initialization of buffer pool");
#ifdef UNIV_DEBUG
/* We have observed deadlocks with a 5MB buffer pool but
the actual lower limit could very well be a little higher. */
if (srv_buf_pool_size <= 5 * 1024 * 1024) {
ib_logf(IB_LOG_LEVEL_INFO,
"Small buffer pool size (%luM), the flst_validate() "
"debug function can cause a deadlock if the "
"buffer pool fills up.",
srv_buf_pool_size / 1024 / 1024);
}
#endif /* UNIV_DEBUG */
fsp_init();
log_init();
lock_sys_create(srv_lock_table_size);
/* Create i/o-handler threads: */
for (i = 0; i < srv_n_file_io_threads; ++i) {
n[i] = i;
thread_handles[i] = os_thread_create(io_handler_thread, n + i, thread_ids + i);
thread_started[i] = true;
}
#ifdef UNIV_LOG_ARCHIVE
if (0 != ut_strcmp(srv_log_group_home_dir, srv_arch_dir)) {
ut_print_timestamp(stderr);
fprintf(stderr, " InnoDB: Error: you must set the log group home dir in my.cnf\n");
ut_print_timestamp(stderr);
fprintf(stderr, " InnoDB: the same as log arch dir.\n");
return(DB_ERROR);
}
#endif /* UNIV_LOG_ARCHIVE */
if (srv_n_log_files * srv_log_file_size * UNIV_PAGE_SIZE
>= 512ULL * 1024ULL * 1024ULL * 1024ULL) {
/* log_block_convert_lsn_to_no() limits the returned block
number to 1G and given that OS_FILE_LOG_BLOCK_SIZE is 512
bytes, then we have a limit of 512 GB. If that limit is to
be raised, then log_block_convert_lsn_to_no() must be
modified. */
ib_logf(IB_LOG_LEVEL_ERROR,
"Combined size of log files must be < 512 GB");
return(DB_ERROR);
}
if (srv_n_log_files * srv_log_file_size >= ULINT_MAX) {
/* fil_io() takes ulint as an argument and we are passing
(next_offset / UNIV_PAGE_SIZE) to it in log_group_write_buf().
So (next_offset / UNIV_PAGE_SIZE) must be less than ULINT_MAX.
So next_offset must be < ULINT_MAX * UNIV_PAGE_SIZE. This
means that we are limited to ULINT_MAX * UNIV_PAGE_SIZE which
is 64 TB on 32 bit systems. */
fprintf(stderr,
" InnoDB: Error: combined size of log files"
" must be < %lu GB\n",
ULINT_MAX / 1073741824 * UNIV_PAGE_SIZE);
return(DB_ERROR);
}
sum_of_new_sizes = 0;
for (i = 0; i < srv_n_data_files; i++) {
#ifndef __WIN__
if (sizeof(off_t) < 5
&& srv_data_file_sizes[i]
>= (ulint) (1 << (32 - UNIV_PAGE_SIZE_SHIFT))) {
ut_print_timestamp(stderr);
fprintf(stderr,
" InnoDB: Error: file size must be < 4 GB"
" with this MySQL binary\n");
ut_print_timestamp(stderr);
fprintf(stderr,
" InnoDB: and operating system combination,"
" in some OS's < 2 GB\n");
return(DB_ERROR);
}
#endif
sum_of_new_sizes += srv_data_file_sizes[i];
}
MDEV-11556 InnoDB redo log apply fails to adjust data file sizes fil_space_t::recv_size: New member: recovered tablespace size in pages; 0 if no size change was read from the redo log, or if the size change was implemented. fil_space_set_recv_size(): New function for setting space->recv_size. innodb_data_file_size_debug: A debug parameter for setting the system tablespace size in recovery even when the redo log does not contain any size changes. It is hard to write a small test case that would cause the system tablespace to be extended at the critical moment. recv_parse_log_rec(): Note those tablespaces whose size is being changed by the redo log, by invoking fil_space_set_recv_size(). innobase_init(): Correct an error message, and do not require a larger innodb_buffer_pool_size when starting up with a smaller innodb_page_size. innobase_start_or_create_for_mysql(): Allow startup with any initial size of the ibdata1 file if the autoextend attribute is set. Require the minimum size of fixed-size system tablespaces to be 640 pages, not 10 megabytes. Implement innodb_data_file_size_debug. open_or_create_data_files(): Round the system tablespace size down to pages, not to full megabytes, (Our test truncates the system tablespace to more than 800 pages with innodb_page_size=4k. InnoDB should not imagine that it was truncated to 768 pages and then overwrite good pages in the tablespace.) fil_flush_low(): Refactored from fil_flush(). fil_space_extend_must_retry(): Refactored from fil_extend_space_to_desired_size(). fil_mutex_enter_and_prepare_for_io(): Extend the tablespace if fil_space_set_recv_size() was called. The test case has been successfully run with all the innodb_page_size values 4k, 8k, 16k, 32k, 64k.
2016-12-28 12:05:43 +02:00
if (!srv_auto_extend_last_data_file && sum_of_new_sizes < 640) {
ib_logf(IB_LOG_LEVEL_ERROR,
MDEV-11556 InnoDB redo log apply fails to adjust data file sizes fil_space_t::recv_size: New member: recovered tablespace size in pages; 0 if no size change was read from the redo log, or if the size change was implemented. fil_space_set_recv_size(): New function for setting space->recv_size. innodb_data_file_size_debug: A debug parameter for setting the system tablespace size in recovery even when the redo log does not contain any size changes. It is hard to write a small test case that would cause the system tablespace to be extended at the critical moment. recv_parse_log_rec(): Note those tablespaces whose size is being changed by the redo log, by invoking fil_space_set_recv_size(). innobase_init(): Correct an error message, and do not require a larger innodb_buffer_pool_size when starting up with a smaller innodb_page_size. innobase_start_or_create_for_mysql(): Allow startup with any initial size of the ibdata1 file if the autoextend attribute is set. Require the minimum size of fixed-size system tablespaces to be 640 pages, not 10 megabytes. Implement innodb_data_file_size_debug. open_or_create_data_files(): Round the system tablespace size down to pages, not to full megabytes, (Our test truncates the system tablespace to more than 800 pages with innodb_page_size=4k. InnoDB should not imagine that it was truncated to 768 pages and then overwrite good pages in the tablespace.) fil_flush_low(): Refactored from fil_flush(). fil_space_extend_must_retry(): Refactored from fil_extend_space_to_desired_size(). fil_mutex_enter_and_prepare_for_io(): Extend the tablespace if fil_space_set_recv_size() was called. The test case has been successfully run with all the innodb_page_size values 4k, 8k, 16k, 32k, 64k.
2016-12-28 12:05:43 +02:00
"Combined size in innodb_data_file_path"
" must be at least %u MiB",
640 >> (20 - UNIV_PAGE_SIZE_SHIFT));
return(DB_ERROR);
}
2014-05-06 21:13:16 +02:00
recv_sys_create();
recv_sys_init(buf_pool_get_curr_size());
err = open_or_create_data_files(&create_new_db,
#ifdef UNIV_LOG_ARCHIVE
&min_arch_log_no, &max_arch_log_no,
#endif /* UNIV_LOG_ARCHIVE */
MDEV-12113: install_db shows corruption for rest encryption with innodb_data_file_path=ibdata1:3M; Problem was that FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION field that for encrypted pages even in system datafiles should contain key_version except very first page (0:0) is after encryption overwritten with flush lsn. Ported WL#7990 Repurpose FIL_PAGE_FLUSH_LSN to 10.1 The field FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION is consulted during InnoDB startup. At startup, InnoDB reads the FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION from the first page of each file in the InnoDB system tablespace. If there are multiple files, the minimum and maximum LSN can differ. These numbers are passed to InnoDB startup. Having the number in other files than the first file of the InnoDB system tablespace is not providing much additional value. It is conflicting with other use of the field, such as on InnoDB R-tree index pages and encryption key_version. This worklog will stop writing FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION to other files than the first file of the InnoDB system tablespace (page number 0:0) when system tablespace is encrypted. If tablespace is not encrypted we continue writing FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION to all first pages of system tablespace to avoid unnecessary warnings on downgrade. open_or_create_data_files(): pass only one flushed_lsn parameter xb_load_tablespaces(): pass only one flushed_lsn parameter. buf_page_create(): Improve comment about where FIL_PAGE_FIL_FLUSH_LSN_OR_KEY_VERSION is set. fil_write_flushed_lsn(): A new function, merged from fil_write_lsn_and_arch_no_to_file() and fil_write_flushed_lsn_to_data_files(). Only write to the first page of the system tablespace (page 0:0) if tablespace is encrypted, or write all first pages of system tablespace and invoke fil_flush_file_spaces(FIL_TYPE_TABLESPACE) afterwards. fil_read_first_page(): read flush_lsn and crypt_data only from first datafile. fil_open_single_table_tablespace(): Remove output of LSN, because it was only valid for the system tablespace and the undo tablespaces, not user tablespaces. fil_validate_single_table_tablespace(): Remove output of LSN. checkpoint_now_set(): Use fil_write_flushed_lsn and output a error if operation fails. Remove lsn variable from fsp_open_info. recv_recovery_from_checkpoint_start(): Remove unnecessary second flush_lsn parameter. log_empty_and_mark_files_at_shutdown(): Use fil_writte_flushed_lsn and output error if it fails. open_or_create_data_files(): Pass only one flushed_lsn variable.
2017-05-30 11:30:43 +03:00
&flushed_lsn,
&sum_of_new_sizes);
if (err == DB_FAIL) {
ib_logf(IB_LOG_LEVEL_ERROR,
"The system tablespace must be writable!");
return(DB_ERROR);
} else if (err != DB_SUCCESS) {
ib_logf(IB_LOG_LEVEL_ERROR,
"Could not open or create the system tablespace. If "
"you tried to add new data files to the system "
"tablespace, and it failed here, you should now "
"edit innodb_data_file_path in my.cnf back to what "
"it was, and remove the new ibdata files InnoDB "
"created in this failed attempt. InnoDB only wrote "
"those files full of zeros, but did not yet use "
"them in any way. But be careful: do not remove "
"old data files which contain your precious data!");
return(err);
}
#ifdef UNIV_LOG_ARCHIVE
srv_normalize_path_for_win(srv_arch_dir);
srv_arch_dir = srv_add_path_separator_if_needed(srv_arch_dir);
#endif /* UNIV_LOG_ARCHIVE */
dirnamelen = strlen(srv_log_group_home_dir);
ut_a(dirnamelen < (sizeof logfilename) - 10 - sizeof "ib_logfile");
memcpy(logfilename, srv_log_group_home_dir, dirnamelen);
/* Add a path separator if needed. */
if (dirnamelen && logfilename[dirnamelen - 1] != SRV_PATH_SEPARATOR) {
logfilename[dirnamelen++] = SRV_PATH_SEPARATOR;
}
srv_log_file_size_requested = srv_log_file_size;
if (create_new_db) {
bool success = buf_flush_list(ULINT_MAX, LSN_MAX, NULL);
ut_a(success);
MDEV-12113: install_db shows corruption for rest encryption with innodb_data_file_path=ibdata1:3M; Problem was that FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION field that for encrypted pages even in system datafiles should contain key_version except very first page (0:0) is after encryption overwritten with flush lsn. Ported WL#7990 Repurpose FIL_PAGE_FLUSH_LSN to 10.1 The field FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION is consulted during InnoDB startup. At startup, InnoDB reads the FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION from the first page of each file in the InnoDB system tablespace. If there are multiple files, the minimum and maximum LSN can differ. These numbers are passed to InnoDB startup. Having the number in other files than the first file of the InnoDB system tablespace is not providing much additional value. It is conflicting with other use of the field, such as on InnoDB R-tree index pages and encryption key_version. This worklog will stop writing FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION to other files than the first file of the InnoDB system tablespace (page number 0:0) when system tablespace is encrypted. If tablespace is not encrypted we continue writing FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION to all first pages of system tablespace to avoid unnecessary warnings on downgrade. open_or_create_data_files(): pass only one flushed_lsn parameter xb_load_tablespaces(): pass only one flushed_lsn parameter. buf_page_create(): Improve comment about where FIL_PAGE_FIL_FLUSH_LSN_OR_KEY_VERSION is set. fil_write_flushed_lsn(): A new function, merged from fil_write_lsn_and_arch_no_to_file() and fil_write_flushed_lsn_to_data_files(). Only write to the first page of the system tablespace (page 0:0) if tablespace is encrypted, or write all first pages of system tablespace and invoke fil_flush_file_spaces(FIL_TYPE_TABLESPACE) afterwards. fil_read_first_page(): read flush_lsn and crypt_data only from first datafile. fil_open_single_table_tablespace(): Remove output of LSN, because it was only valid for the system tablespace and the undo tablespaces, not user tablespaces. fil_validate_single_table_tablespace(): Remove output of LSN. checkpoint_now_set(): Use fil_write_flushed_lsn and output a error if operation fails. Remove lsn variable from fsp_open_info. recv_recovery_from_checkpoint_start(): Remove unnecessary second flush_lsn parameter. log_empty_and_mark_files_at_shutdown(): Use fil_writte_flushed_lsn and output error if it fails. open_or_create_data_files(): Pass only one flushed_lsn variable.
2017-05-30 11:30:43 +03:00
flushed_lsn = log_get_lsn();
buf_flush_wait_batch_end(NULL, BUF_FLUSH_LIST);
err = create_log_files(create_new_db, logfilename, dirnamelen,
MDEV-12113: install_db shows corruption for rest encryption with innodb_data_file_path=ibdata1:3M; Problem was that FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION field that for encrypted pages even in system datafiles should contain key_version except very first page (0:0) is after encryption overwritten with flush lsn. Ported WL#7990 Repurpose FIL_PAGE_FLUSH_LSN to 10.1 The field FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION is consulted during InnoDB startup. At startup, InnoDB reads the FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION from the first page of each file in the InnoDB system tablespace. If there are multiple files, the minimum and maximum LSN can differ. These numbers are passed to InnoDB startup. Having the number in other files than the first file of the InnoDB system tablespace is not providing much additional value. It is conflicting with other use of the field, such as on InnoDB R-tree index pages and encryption key_version. This worklog will stop writing FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION to other files than the first file of the InnoDB system tablespace (page number 0:0) when system tablespace is encrypted. If tablespace is not encrypted we continue writing FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION to all first pages of system tablespace to avoid unnecessary warnings on downgrade. open_or_create_data_files(): pass only one flushed_lsn parameter xb_load_tablespaces(): pass only one flushed_lsn parameter. buf_page_create(): Improve comment about where FIL_PAGE_FIL_FLUSH_LSN_OR_KEY_VERSION is set. fil_write_flushed_lsn(): A new function, merged from fil_write_lsn_and_arch_no_to_file() and fil_write_flushed_lsn_to_data_files(). Only write to the first page of the system tablespace (page 0:0) if tablespace is encrypted, or write all first pages of system tablespace and invoke fil_flush_file_spaces(FIL_TYPE_TABLESPACE) afterwards. fil_read_first_page(): read flush_lsn and crypt_data only from first datafile. fil_open_single_table_tablespace(): Remove output of LSN, because it was only valid for the system tablespace and the undo tablespaces, not user tablespaces. fil_validate_single_table_tablespace(): Remove output of LSN. checkpoint_now_set(): Use fil_write_flushed_lsn and output a error if operation fails. Remove lsn variable from fsp_open_info. recv_recovery_from_checkpoint_start(): Remove unnecessary second flush_lsn parameter. log_empty_and_mark_files_at_shutdown(): Use fil_writte_flushed_lsn and output error if it fails. open_or_create_data_files(): Pass only one flushed_lsn variable.
2017-05-30 11:30:43 +03:00
flushed_lsn, logfile0);
if (err != DB_SUCCESS) {
return(err);
}
} else {
MDEV-11556 InnoDB redo log apply fails to adjust data file sizes fil_space_t::recv_size: New member: recovered tablespace size in pages; 0 if no size change was read from the redo log, or if the size change was implemented. fil_space_set_recv_size(): New function for setting space->recv_size. innodb_data_file_size_debug: A debug parameter for setting the system tablespace size in recovery even when the redo log does not contain any size changes. It is hard to write a small test case that would cause the system tablespace to be extended at the critical moment. recv_parse_log_rec(): Note those tablespaces whose size is being changed by the redo log, by invoking fil_space_set_recv_size(). innobase_init(): Correct an error message, and do not require a larger innodb_buffer_pool_size when starting up with a smaller innodb_page_size. innobase_start_or_create_for_mysql(): Allow startup with any initial size of the ibdata1 file if the autoextend attribute is set. Require the minimum size of fixed-size system tablespaces to be 640 pages, not 10 megabytes. Implement innodb_data_file_size_debug. open_or_create_data_files(): Round the system tablespace size down to pages, not to full megabytes, (Our test truncates the system tablespace to more than 800 pages with innodb_page_size=4k. InnoDB should not imagine that it was truncated to 768 pages and then overwrite good pages in the tablespace.) fil_flush_low(): Refactored from fil_flush(). fil_space_extend_must_retry(): Refactored from fil_extend_space_to_desired_size(). fil_mutex_enter_and_prepare_for_io(): Extend the tablespace if fil_space_set_recv_size() was called. The test case has been successfully run with all the innodb_page_size values 4k, 8k, 16k, 32k, 64k.
2016-12-28 12:05:43 +02:00
ut_d(fil_space_get(0)->recv_size = srv_sys_space_size_debug);
for (i = 0; i < SRV_N_LOG_FILES_MAX; i++) {
os_offset_t size;
os_file_stat_t stat_info;
sprintf(logfilename + dirnamelen,
"ib_logfile%u", i);
err = os_file_get_status(
logfilename, &stat_info, false);
if (err == DB_NOT_FOUND) {
if (i == 0) {
MDEV-12113: install_db shows corruption for rest encryption with innodb_data_file_path=ibdata1:3M; Problem was that FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION field that for encrypted pages even in system datafiles should contain key_version except very first page (0:0) is after encryption overwritten with flush lsn. Ported WL#7990 Repurpose FIL_PAGE_FLUSH_LSN to 10.1 The field FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION is consulted during InnoDB startup. At startup, InnoDB reads the FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION from the first page of each file in the InnoDB system tablespace. If there are multiple files, the minimum and maximum LSN can differ. These numbers are passed to InnoDB startup. Having the number in other files than the first file of the InnoDB system tablespace is not providing much additional value. It is conflicting with other use of the field, such as on InnoDB R-tree index pages and encryption key_version. This worklog will stop writing FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION to other files than the first file of the InnoDB system tablespace (page number 0:0) when system tablespace is encrypted. If tablespace is not encrypted we continue writing FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION to all first pages of system tablespace to avoid unnecessary warnings on downgrade. open_or_create_data_files(): pass only one flushed_lsn parameter xb_load_tablespaces(): pass only one flushed_lsn parameter. buf_page_create(): Improve comment about where FIL_PAGE_FIL_FLUSH_LSN_OR_KEY_VERSION is set. fil_write_flushed_lsn(): A new function, merged from fil_write_lsn_and_arch_no_to_file() and fil_write_flushed_lsn_to_data_files(). Only write to the first page of the system tablespace (page 0:0) if tablespace is encrypted, or write all first pages of system tablespace and invoke fil_flush_file_spaces(FIL_TYPE_TABLESPACE) afterwards. fil_read_first_page(): read flush_lsn and crypt_data only from first datafile. fil_open_single_table_tablespace(): Remove output of LSN, because it was only valid for the system tablespace and the undo tablespaces, not user tablespaces. fil_validate_single_table_tablespace(): Remove output of LSN. checkpoint_now_set(): Use fil_write_flushed_lsn and output a error if operation fails. Remove lsn variable from fsp_open_info. recv_recovery_from_checkpoint_start(): Remove unnecessary second flush_lsn parameter. log_empty_and_mark_files_at_shutdown(): Use fil_writte_flushed_lsn and output error if it fails. open_or_create_data_files(): Pass only one flushed_lsn variable.
2017-05-30 11:30:43 +03:00
if (flushed_lsn < (lsn_t) 1000) {
ib_logf(IB_LOG_LEVEL_ERROR,
"Cannot create"
" log files because"
" data files are"
" corrupt or the"
" database was not"
" shut down cleanly"
" after creating"
" the data files.");
return(DB_ERROR);
}
err = create_log_files(
create_new_db, logfilename,
MDEV-12113: install_db shows corruption for rest encryption with innodb_data_file_path=ibdata1:3M; Problem was that FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION field that for encrypted pages even in system datafiles should contain key_version except very first page (0:0) is after encryption overwritten with flush lsn. Ported WL#7990 Repurpose FIL_PAGE_FLUSH_LSN to 10.1 The field FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION is consulted during InnoDB startup. At startup, InnoDB reads the FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION from the first page of each file in the InnoDB system tablespace. If there are multiple files, the minimum and maximum LSN can differ. These numbers are passed to InnoDB startup. Having the number in other files than the first file of the InnoDB system tablespace is not providing much additional value. It is conflicting with other use of the field, such as on InnoDB R-tree index pages and encryption key_version. This worklog will stop writing FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION to other files than the first file of the InnoDB system tablespace (page number 0:0) when system tablespace is encrypted. If tablespace is not encrypted we continue writing FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION to all first pages of system tablespace to avoid unnecessary warnings on downgrade. open_or_create_data_files(): pass only one flushed_lsn parameter xb_load_tablespaces(): pass only one flushed_lsn parameter. buf_page_create(): Improve comment about where FIL_PAGE_FIL_FLUSH_LSN_OR_KEY_VERSION is set. fil_write_flushed_lsn(): A new function, merged from fil_write_lsn_and_arch_no_to_file() and fil_write_flushed_lsn_to_data_files(). Only write to the first page of the system tablespace (page 0:0) if tablespace is encrypted, or write all first pages of system tablespace and invoke fil_flush_file_spaces(FIL_TYPE_TABLESPACE) afterwards. fil_read_first_page(): read flush_lsn and crypt_data only from first datafile. fil_open_single_table_tablespace(): Remove output of LSN, because it was only valid for the system tablespace and the undo tablespaces, not user tablespaces. fil_validate_single_table_tablespace(): Remove output of LSN. checkpoint_now_set(): Use fil_write_flushed_lsn and output a error if operation fails. Remove lsn variable from fsp_open_info. recv_recovery_from_checkpoint_start(): Remove unnecessary second flush_lsn parameter. log_empty_and_mark_files_at_shutdown(): Use fil_writte_flushed_lsn and output error if it fails. open_or_create_data_files(): Pass only one flushed_lsn variable.
2017-05-30 11:30:43 +03:00
dirnamelen, flushed_lsn,
logfile0);
if (err == DB_SUCCESS) {
err = create_log_files_rename(
logfilename,
dirnamelen,
MDEV-12113: install_db shows corruption for rest encryption with innodb_data_file_path=ibdata1:3M; Problem was that FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION field that for encrypted pages even in system datafiles should contain key_version except very first page (0:0) is after encryption overwritten with flush lsn. Ported WL#7990 Repurpose FIL_PAGE_FLUSH_LSN to 10.1 The field FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION is consulted during InnoDB startup. At startup, InnoDB reads the FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION from the first page of each file in the InnoDB system tablespace. If there are multiple files, the minimum and maximum LSN can differ. These numbers are passed to InnoDB startup. Having the number in other files than the first file of the InnoDB system tablespace is not providing much additional value. It is conflicting with other use of the field, such as on InnoDB R-tree index pages and encryption key_version. This worklog will stop writing FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION to other files than the first file of the InnoDB system tablespace (page number 0:0) when system tablespace is encrypted. If tablespace is not encrypted we continue writing FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION to all first pages of system tablespace to avoid unnecessary warnings on downgrade. open_or_create_data_files(): pass only one flushed_lsn parameter xb_load_tablespaces(): pass only one flushed_lsn parameter. buf_page_create(): Improve comment about where FIL_PAGE_FIL_FLUSH_LSN_OR_KEY_VERSION is set. fil_write_flushed_lsn(): A new function, merged from fil_write_lsn_and_arch_no_to_file() and fil_write_flushed_lsn_to_data_files(). Only write to the first page of the system tablespace (page 0:0) if tablespace is encrypted, or write all first pages of system tablespace and invoke fil_flush_file_spaces(FIL_TYPE_TABLESPACE) afterwards. fil_read_first_page(): read flush_lsn and crypt_data only from first datafile. fil_open_single_table_tablespace(): Remove output of LSN, because it was only valid for the system tablespace and the undo tablespaces, not user tablespaces. fil_validate_single_table_tablespace(): Remove output of LSN. checkpoint_now_set(): Use fil_write_flushed_lsn and output a error if operation fails. Remove lsn variable from fsp_open_info. recv_recovery_from_checkpoint_start(): Remove unnecessary second flush_lsn parameter. log_empty_and_mark_files_at_shutdown(): Use fil_writte_flushed_lsn and output error if it fails. open_or_create_data_files(): Pass only one flushed_lsn variable.
2017-05-30 11:30:43 +03:00
flushed_lsn,
logfile0);
}
if (err != DB_SUCCESS) {
return(err);
}
/* Suppress the message about
crash recovery. */
MDEV-12113: install_db shows corruption for rest encryption with innodb_data_file_path=ibdata1:3M; Problem was that FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION field that for encrypted pages even in system datafiles should contain key_version except very first page (0:0) is after encryption overwritten with flush lsn. Ported WL#7990 Repurpose FIL_PAGE_FLUSH_LSN to 10.1 The field FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION is consulted during InnoDB startup. At startup, InnoDB reads the FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION from the first page of each file in the InnoDB system tablespace. If there are multiple files, the minimum and maximum LSN can differ. These numbers are passed to InnoDB startup. Having the number in other files than the first file of the InnoDB system tablespace is not providing much additional value. It is conflicting with other use of the field, such as on InnoDB R-tree index pages and encryption key_version. This worklog will stop writing FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION to other files than the first file of the InnoDB system tablespace (page number 0:0) when system tablespace is encrypted. If tablespace is not encrypted we continue writing FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION to all first pages of system tablespace to avoid unnecessary warnings on downgrade. open_or_create_data_files(): pass only one flushed_lsn parameter xb_load_tablespaces(): pass only one flushed_lsn parameter. buf_page_create(): Improve comment about where FIL_PAGE_FIL_FLUSH_LSN_OR_KEY_VERSION is set. fil_write_flushed_lsn(): A new function, merged from fil_write_lsn_and_arch_no_to_file() and fil_write_flushed_lsn_to_data_files(). Only write to the first page of the system tablespace (page 0:0) if tablespace is encrypted, or write all first pages of system tablespace and invoke fil_flush_file_spaces(FIL_TYPE_TABLESPACE) afterwards. fil_read_first_page(): read flush_lsn and crypt_data only from first datafile. fil_open_single_table_tablespace(): Remove output of LSN, because it was only valid for the system tablespace and the undo tablespaces, not user tablespaces. fil_validate_single_table_tablespace(): Remove output of LSN. checkpoint_now_set(): Use fil_write_flushed_lsn and output a error if operation fails. Remove lsn variable from fsp_open_info. recv_recovery_from_checkpoint_start(): Remove unnecessary second flush_lsn parameter. log_empty_and_mark_files_at_shutdown(): Use fil_writte_flushed_lsn and output error if it fails. open_or_create_data_files(): Pass only one flushed_lsn variable.
2017-05-30 11:30:43 +03:00
flushed_lsn = log_get_lsn();
goto files_checked;
} else if (i < 2) {
/* must have at least 2 log files */
ib_logf(IB_LOG_LEVEL_ERROR,
"Only one log file found.");
return(err);
}
/* opened all files */
break;
}
if (!srv_file_check_mode(logfilename)) {
return(DB_ERROR);
}
err = open_log_file(&files[i], logfilename, &size);
if (err != DB_SUCCESS) {
return(err);
}
ut_a(size != (os_offset_t) -1);
if (size & ((1 << UNIV_PAGE_SIZE_SHIFT) - 1)) {
ib_logf(IB_LOG_LEVEL_ERROR,
"Log file %s size "
UINT64PF " is not a multiple of"
" innodb_page_size",
logfilename, size);
return(DB_ERROR);
}
size >>= UNIV_PAGE_SIZE_SHIFT;
if (i == 0) {
srv_log_file_size = size;
} else if (size != srv_log_file_size) {
ib_logf(IB_LOG_LEVEL_ERROR,
"Log file %s is"
2014-06-09 18:16:00 +02:00
" of different size " UINT64PF " bytes"
" than other log"
2014-06-09 18:16:00 +02:00
" files " UINT64PF " bytes!",
logfilename,
size << UNIV_PAGE_SIZE_SHIFT,
(os_offset_t) srv_log_file_size
<< UNIV_PAGE_SIZE_SHIFT);
return(DB_ERROR);
}
}
srv_n_log_files_found = i;
/* Create the in-memory file space objects. */
sprintf(logfilename + dirnamelen, "ib_logfile%u", 0);
fil_space_create(logfilename,
MDEV-11623 MariaDB 10.1 fails to start datadir created with MariaDB 10.0/MySQL 5.6 using innodb-page-size!=16K The storage format of FSP_SPACE_FLAGS was accidentally broken already in MariaDB 10.1.0. This fix is bringing the format in line with other MySQL and MariaDB release series. Please refer to the comments that were added to fsp0fsp.h for details. This is an INCOMPATIBLE CHANGE that affects users of page_compression and non-default innodb_page_size. Upgrading to this release will correct the flags in the data files. If you want to downgrade to earlier MariaDB 10.1.x, please refer to the test innodb.101_compatibility how to reset the FSP_SPACE_FLAGS in the files. NOTE: MariaDB 10.1.0 to 10.1.20 can misinterpret uncompressed data files with innodb_page_size=4k or 64k as compressed innodb_page_size=16k files, and then probably fail when trying to access the pages. See the comments in the function fsp_flags_convert_from_101() for detailed analysis. Move PAGE_COMPRESSION to FSP_SPACE_FLAGS bit position 16. In this way, compressed innodb_page_size=16k tablespaces will not be mistaken for uncompressed ones by MariaDB 10.1.0 to 10.1.20. Derive PAGE_COMPRESSION_LEVEL, ATOMIC_WRITES and DATA_DIR from the dict_table_t::flags when the table is available, in fil_space_for_table_exists_in_mem() or fil_open_single_table_tablespace(). During crash recovery, fil_load_single_table_tablespace() will use innodb_compression_level for the PAGE_COMPRESSION_LEVEL. FSP_FLAGS_MEM_MASK: A bitmap of the memory-only fil_space_t::flags that are not to be written to FSP_SPACE_FLAGS. Currently, these will include PAGE_COMPRESSION_LEVEL, ATOMIC_WRITES and DATA_DIR. Introduce the macro FSP_FLAGS_PAGE_SSIZE(). We only support one innodb_page_size for the whole instance. When creating a dummy tablespace for the redo log, use fil_space_t::flags=0. The flags are never written to the redo log files. Remove many FSP_FLAGS_SET_ macros. dict_tf_verify_flags(): Remove. This is basically only duplicating the logic of dict_tf_to_fsp_flags(), used in a debug assertion. fil_space_t::mark: Remove. This flag was not used for anything. fil_space_for_table_exists_in_mem(): Remove the unnecessary parameter mark_space, and add a parameter for table flags. Check that fil_space_t::flags match the table flags, and adjust the (memory-only) flags based on the table flags. fil_node_open_file(): Remove some redundant or unreachable conditions, do not use stderr for output, and avoid unnecessary server aborts. fil_user_tablespace_restore_page(): Convert the flags, so that the correct page_size will be used when restoring a page from the doublewrite buffer. fil_space_get_page_compressed(), fsp_flags_is_page_compressed(): Remove. It suffices to have fil_space_is_page_compressed(). FSP_FLAGS_WIDTH_DATA_DIR, FSP_FLAGS_WIDTH_PAGE_COMPRESSION_LEVEL, FSP_FLAGS_WIDTH_ATOMIC_WRITES: Remove, because these flags do not exist in the FSP_SPACE_FLAGS but only in memory. fsp_flags_try_adjust(): New function, to adjust the FSP_SPACE_FLAGS in page 0. Called by fil_open_single_table_tablespace(), fil_space_for_table_exists_in_mem(), innobase_start_or_create_for_mysql() except if --innodb-read-only is active. fsp_flags_is_valid(ulint): Reimplement from the scratch, with accurate comments. Do not display any details of detected inconsistencies, because the output could be confusing when dealing with MariaDB 10.1.x data files. fsp_flags_convert_from_101(ulint): Convert flags from buggy MariaDB 10.1.x format, or return ULINT_UNDEFINED if the flags cannot be in MariaDB 10.1.x format. fsp_flags_match(): Check the flags when probing files. Implemented based on fsp_flags_is_valid() and fsp_flags_convert_from_101(). dict_check_tablespaces_and_store_max_id(): Do not access the page after committing the mini-transaction. IMPORT TABLESPACE fixes: AbstractCallback::init(): Convert the flags. FetchIndexRootPages::operator(): Check that the tablespace flags match the table flags. Do not attempt to convert tablespace flags to table flags, because the conversion would necessarily be lossy. PageConverter::update_header(): Write back the correct flags. This takes care of the flags in IMPORT TABLESPACE.
2017-01-14 00:13:16 +02:00
SRV_LOG_SPACE_FIRST_ID, 0, FIL_LOG,
NULL /* no encryption yet */,
true /* create */);
ut_a(fil_validate());
/* srv_log_file_size is measured in pages; if page size is 16KB,
then we have a limit of 64TB on 32 bit systems */
ut_a(srv_log_file_size <= ULINT_MAX);
for (unsigned j = 0; j < i; j++) {
sprintf(logfilename + dirnamelen, "ib_logfile%u", j);
if (!fil_node_create(logfilename,
(ulint) srv_log_file_size,
SRV_LOG_SPACE_FIRST_ID, FALSE)) {
return(DB_ERROR);
}
}
#ifdef UNIV_LOG_ARCHIVE
/* Create the file space object for archived logs. Under
MySQL, no archiving ever done. */
fil_space_create("arch_log_space", SRV_LOG_SPACE_FIRST_ID + 1,
0, FIL_LOG, NULL, true);
#endif /* UNIV_LOG_ARCHIVE */
log_group_init(0, i, srv_log_file_size * UNIV_PAGE_SIZE,
SRV_LOG_SPACE_FIRST_ID,
SRV_LOG_SPACE_FIRST_ID + 1);
}
files_checked:
/* Open all log files and data files in the system
tablespace: we keep them open until database
shutdown */
fil_open_log_and_system_tablespace_files();
err = srv_undo_tablespaces_init(
create_new_db,
srv_undo_tablespaces,
&srv_undo_tablespaces_open);
/* If the force recovery is set very high then we carry on regardless
of all errors. Basically this is fingers crossed mode. */
if (err != DB_SUCCESS
&& srv_force_recovery < SRV_FORCE_NO_UNDO_LOG_SCAN) {
return(err);
}
/* Initialize objects used by dict stats gathering thread, which
can also be used by recovery if it tries to drop some table */
if (!srv_read_only_mode) {
dict_stats_thread_init();
}
trx_sys_file_format_init();
trx_sys_create();
if (create_new_db) {
ut_a(!srv_read_only_mode);
mtr_start(&mtr);
fsp_header_init(0, sum_of_new_sizes, &mtr);
compile_time_assert(TRX_SYS_SPACE == 0);
compile_time_assert(IBUF_SPACE_ID == 0);
ulint ibuf_root = btr_create(
DICT_CLUSTERED | DICT_UNIVERSAL | DICT_IBUF,
0, 0, DICT_IBUF_ID_MIN,
dict_ind_redundant, &mtr);
mtr_commit(&mtr);
if (ibuf_root == FIL_NULL) {
return(srv_init_abort(true, __FILE__, __LINE__,
DB_ERROR));
}
ut_ad(ibuf_root == IBUF_TREE_ROOT_PAGE_NO);
/* To maintain backward compatibility we create only
the first rollback segment before the double write buffer.
All the remaining rollback segments will be created later,
after the double write buffer has been created. */
trx_sys_create_sys_pages();
ib_bh = trx_sys_init_at_db_start();
n_recovered_trx = UT_LIST_GET_LEN(trx_sys->rw_trx_list);
/* The purge system needs to create the purge view and
therefore requires that the trx_sys is inited. */
trx_purge_sys_create(srv_n_purge_threads, ib_bh);
err = dict_create();
if (err != DB_SUCCESS) {
return(err);
}
srv_startup_is_before_trx_rollback_phase = FALSE;
bool success = buf_flush_list(ULINT_MAX, LSN_MAX, NULL);
ut_a(success);
MDEV-12113: install_db shows corruption for rest encryption with innodb_data_file_path=ibdata1:3M; Problem was that FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION field that for encrypted pages even in system datafiles should contain key_version except very first page (0:0) is after encryption overwritten with flush lsn. Ported WL#7990 Repurpose FIL_PAGE_FLUSH_LSN to 10.1 The field FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION is consulted during InnoDB startup. At startup, InnoDB reads the FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION from the first page of each file in the InnoDB system tablespace. If there are multiple files, the minimum and maximum LSN can differ. These numbers are passed to InnoDB startup. Having the number in other files than the first file of the InnoDB system tablespace is not providing much additional value. It is conflicting with other use of the field, such as on InnoDB R-tree index pages and encryption key_version. This worklog will stop writing FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION to other files than the first file of the InnoDB system tablespace (page number 0:0) when system tablespace is encrypted. If tablespace is not encrypted we continue writing FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION to all first pages of system tablespace to avoid unnecessary warnings on downgrade. open_or_create_data_files(): pass only one flushed_lsn parameter xb_load_tablespaces(): pass only one flushed_lsn parameter. buf_page_create(): Improve comment about where FIL_PAGE_FIL_FLUSH_LSN_OR_KEY_VERSION is set. fil_write_flushed_lsn(): A new function, merged from fil_write_lsn_and_arch_no_to_file() and fil_write_flushed_lsn_to_data_files(). Only write to the first page of the system tablespace (page 0:0) if tablespace is encrypted, or write all first pages of system tablespace and invoke fil_flush_file_spaces(FIL_TYPE_TABLESPACE) afterwards. fil_read_first_page(): read flush_lsn and crypt_data only from first datafile. fil_open_single_table_tablespace(): Remove output of LSN, because it was only valid for the system tablespace and the undo tablespaces, not user tablespaces. fil_validate_single_table_tablespace(): Remove output of LSN. checkpoint_now_set(): Use fil_write_flushed_lsn and output a error if operation fails. Remove lsn variable from fsp_open_info. recv_recovery_from_checkpoint_start(): Remove unnecessary second flush_lsn parameter. log_empty_and_mark_files_at_shutdown(): Use fil_writte_flushed_lsn and output error if it fails. open_or_create_data_files(): Pass only one flushed_lsn variable.
2017-05-30 11:30:43 +03:00
flushed_lsn = log_get_lsn();
buf_flush_wait_batch_end(NULL, BUF_FLUSH_LIST);
/* Stamp the LSN to the data files. */
MDEV-12113: install_db shows corruption for rest encryption with innodb_data_file_path=ibdata1:3M; Problem was that FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION field that for encrypted pages even in system datafiles should contain key_version except very first page (0:0) is after encryption overwritten with flush lsn. Ported WL#7990 Repurpose FIL_PAGE_FLUSH_LSN to 10.1 The field FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION is consulted during InnoDB startup. At startup, InnoDB reads the FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION from the first page of each file in the InnoDB system tablespace. If there are multiple files, the minimum and maximum LSN can differ. These numbers are passed to InnoDB startup. Having the number in other files than the first file of the InnoDB system tablespace is not providing much additional value. It is conflicting with other use of the field, such as on InnoDB R-tree index pages and encryption key_version. This worklog will stop writing FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION to other files than the first file of the InnoDB system tablespace (page number 0:0) when system tablespace is encrypted. If tablespace is not encrypted we continue writing FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION to all first pages of system tablespace to avoid unnecessary warnings on downgrade. open_or_create_data_files(): pass only one flushed_lsn parameter xb_load_tablespaces(): pass only one flushed_lsn parameter. buf_page_create(): Improve comment about where FIL_PAGE_FIL_FLUSH_LSN_OR_KEY_VERSION is set. fil_write_flushed_lsn(): A new function, merged from fil_write_lsn_and_arch_no_to_file() and fil_write_flushed_lsn_to_data_files(). Only write to the first page of the system tablespace (page 0:0) if tablespace is encrypted, or write all first pages of system tablespace and invoke fil_flush_file_spaces(FIL_TYPE_TABLESPACE) afterwards. fil_read_first_page(): read flush_lsn and crypt_data only from first datafile. fil_open_single_table_tablespace(): Remove output of LSN, because it was only valid for the system tablespace and the undo tablespaces, not user tablespaces. fil_validate_single_table_tablespace(): Remove output of LSN. checkpoint_now_set(): Use fil_write_flushed_lsn and output a error if operation fails. Remove lsn variable from fsp_open_info. recv_recovery_from_checkpoint_start(): Remove unnecessary second flush_lsn parameter. log_empty_and_mark_files_at_shutdown(): Use fil_writte_flushed_lsn and output error if it fails. open_or_create_data_files(): Pass only one flushed_lsn variable.
2017-05-30 11:30:43 +03:00
err = fil_write_flushed_lsn(flushed_lsn);
MDEV-12113: install_db shows corruption for rest encryption with innodb_data_file_path=ibdata1:3M; Problem was that FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION field that for encrypted pages even in system datafiles should contain key_version except very first page (0:0) is after encryption overwritten with flush lsn. Ported WL#7990 Repurpose FIL_PAGE_FLUSH_LSN to 10.1 The field FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION is consulted during InnoDB startup. At startup, InnoDB reads the FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION from the first page of each file in the InnoDB system tablespace. If there are multiple files, the minimum and maximum LSN can differ. These numbers are passed to InnoDB startup. Having the number in other files than the first file of the InnoDB system tablespace is not providing much additional value. It is conflicting with other use of the field, such as on InnoDB R-tree index pages and encryption key_version. This worklog will stop writing FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION to other files than the first file of the InnoDB system tablespace (page number 0:0) when system tablespace is encrypted. If tablespace is not encrypted we continue writing FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION to all first pages of system tablespace to avoid unnecessary warnings on downgrade. open_or_create_data_files(): pass only one flushed_lsn parameter xb_load_tablespaces(): pass only one flushed_lsn parameter. buf_page_create(): Improve comment about where FIL_PAGE_FIL_FLUSH_LSN_OR_KEY_VERSION is set. fil_write_flushed_lsn(): A new function, merged from fil_write_lsn_and_arch_no_to_file() and fil_write_flushed_lsn_to_data_files(). Only write to the first page of the system tablespace (page 0:0) if tablespace is encrypted, or write all first pages of system tablespace and invoke fil_flush_file_spaces(FIL_TYPE_TABLESPACE) afterwards. fil_read_first_page(): read flush_lsn and crypt_data only from first datafile. fil_open_single_table_tablespace(): Remove output of LSN, because it was only valid for the system tablespace and the undo tablespaces, not user tablespaces. fil_validate_single_table_tablespace(): Remove output of LSN. checkpoint_now_set(): Use fil_write_flushed_lsn and output a error if operation fails. Remove lsn variable from fsp_open_info. recv_recovery_from_checkpoint_start(): Remove unnecessary second flush_lsn parameter. log_empty_and_mark_files_at_shutdown(): Use fil_writte_flushed_lsn and output error if it fails. open_or_create_data_files(): Pass only one flushed_lsn variable.
2017-05-30 11:30:43 +03:00
if (err != DB_SUCCESS) {
return(err);
}
err = create_log_files_rename(logfilename, dirnamelen,
MDEV-12113: install_db shows corruption for rest encryption with innodb_data_file_path=ibdata1:3M; Problem was that FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION field that for encrypted pages even in system datafiles should contain key_version except very first page (0:0) is after encryption overwritten with flush lsn. Ported WL#7990 Repurpose FIL_PAGE_FLUSH_LSN to 10.1 The field FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION is consulted during InnoDB startup. At startup, InnoDB reads the FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION from the first page of each file in the InnoDB system tablespace. If there are multiple files, the minimum and maximum LSN can differ. These numbers are passed to InnoDB startup. Having the number in other files than the first file of the InnoDB system tablespace is not providing much additional value. It is conflicting with other use of the field, such as on InnoDB R-tree index pages and encryption key_version. This worklog will stop writing FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION to other files than the first file of the InnoDB system tablespace (page number 0:0) when system tablespace is encrypted. If tablespace is not encrypted we continue writing FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION to all first pages of system tablespace to avoid unnecessary warnings on downgrade. open_or_create_data_files(): pass only one flushed_lsn parameter xb_load_tablespaces(): pass only one flushed_lsn parameter. buf_page_create(): Improve comment about where FIL_PAGE_FIL_FLUSH_LSN_OR_KEY_VERSION is set. fil_write_flushed_lsn(): A new function, merged from fil_write_lsn_and_arch_no_to_file() and fil_write_flushed_lsn_to_data_files(). Only write to the first page of the system tablespace (page 0:0) if tablespace is encrypted, or write all first pages of system tablespace and invoke fil_flush_file_spaces(FIL_TYPE_TABLESPACE) afterwards. fil_read_first_page(): read flush_lsn and crypt_data only from first datafile. fil_open_single_table_tablespace(): Remove output of LSN, because it was only valid for the system tablespace and the undo tablespaces, not user tablespaces. fil_validate_single_table_tablespace(): Remove output of LSN. checkpoint_now_set(): Use fil_write_flushed_lsn and output a error if operation fails. Remove lsn variable from fsp_open_info. recv_recovery_from_checkpoint_start(): Remove unnecessary second flush_lsn parameter. log_empty_and_mark_files_at_shutdown(): Use fil_writte_flushed_lsn and output error if it fails. open_or_create_data_files(): Pass only one flushed_lsn variable.
2017-05-30 11:30:43 +03:00
flushed_lsn, logfile0);
if (err != DB_SUCCESS) {
return(err);
}
#ifdef UNIV_LOG_ARCHIVE
} else if (srv_archive_recovery) {
ib_logf(IB_LOG_LEVEL_INFO,
" Starting archive recovery from a backup...");
err = recv_recovery_from_archive_start(
min_flushed_lsn, srv_archive_recovery_limit_lsn,
min_arch_log_no);
if (err != DB_SUCCESS) {
return(DB_ERROR);
}
/* Since ibuf init is in dict_boot, and ibuf is needed
in any disk i/o, first call dict_boot */
err = dict_boot();
if (err != DB_SUCCESS) {
return(err);
}
ib_bh = trx_sys_init_at_db_start();
n_recovered_trx = UT_LIST_GET_LEN(trx_sys->rw_trx_list);
/* The purge system needs to create the purge view and
therefore requires that the trx_sys is inited. */
trx_purge_sys_create(srv_n_purge_threads, ib_bh);
srv_startup_is_before_trx_rollback_phase = FALSE;
recv_recovery_from_archive_finish();
#endif /* UNIV_LOG_ARCHIVE */
} else {
/* Check if we support the max format that is stamped
on the system tablespace.
Note: We are NOT allowed to make any modifications to
the TRX_SYS_PAGE_NO page before recovery because this
page also contains the max_trx_id etc. important system
variables that are required for recovery. We need to
ensure that we return the system to a state where normal
recovery is guaranteed to work. We do this by
invalidating the buffer cache, this will force the
reread of the page and restoration to its last known
consistent state, this is REQUIRED for the recovery
process to work. */
err = trx_sys_file_format_max_check(
srv_max_file_format_at_startup);
if (err != DB_SUCCESS) {
return(err);
}
/* Invalidate the buffer pool to ensure that we reread
the page that we read above, during recovery.
Note that this is not as heavy weight as it seems. At
this point there will be only ONE page in the buf_LRU
and there must be no page in the buf_flush list. */
buf_pool_invalidate();
/* We always try to do a recovery, even if the database had
been shut down normally: this is the normal startup path */
err = recv_recovery_from_checkpoint_start(
LOG_CHECKPOINT, LSN_MAX,
MDEV-12113: install_db shows corruption for rest encryption with innodb_data_file_path=ibdata1:3M; Problem was that FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION field that for encrypted pages even in system datafiles should contain key_version except very first page (0:0) is after encryption overwritten with flush lsn. Ported WL#7990 Repurpose FIL_PAGE_FLUSH_LSN to 10.1 The field FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION is consulted during InnoDB startup. At startup, InnoDB reads the FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION from the first page of each file in the InnoDB system tablespace. If there are multiple files, the minimum and maximum LSN can differ. These numbers are passed to InnoDB startup. Having the number in other files than the first file of the InnoDB system tablespace is not providing much additional value. It is conflicting with other use of the field, such as on InnoDB R-tree index pages and encryption key_version. This worklog will stop writing FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION to other files than the first file of the InnoDB system tablespace (page number 0:0) when system tablespace is encrypted. If tablespace is not encrypted we continue writing FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION to all first pages of system tablespace to avoid unnecessary warnings on downgrade. open_or_create_data_files(): pass only one flushed_lsn parameter xb_load_tablespaces(): pass only one flushed_lsn parameter. buf_page_create(): Improve comment about where FIL_PAGE_FIL_FLUSH_LSN_OR_KEY_VERSION is set. fil_write_flushed_lsn(): A new function, merged from fil_write_lsn_and_arch_no_to_file() and fil_write_flushed_lsn_to_data_files(). Only write to the first page of the system tablespace (page 0:0) if tablespace is encrypted, or write all first pages of system tablespace and invoke fil_flush_file_spaces(FIL_TYPE_TABLESPACE) afterwards. fil_read_first_page(): read flush_lsn and crypt_data only from first datafile. fil_open_single_table_tablespace(): Remove output of LSN, because it was only valid for the system tablespace and the undo tablespaces, not user tablespaces. fil_validate_single_table_tablespace(): Remove output of LSN. checkpoint_now_set(): Use fil_write_flushed_lsn and output a error if operation fails. Remove lsn variable from fsp_open_info. recv_recovery_from_checkpoint_start(): Remove unnecessary second flush_lsn parameter. log_empty_and_mark_files_at_shutdown(): Use fil_writte_flushed_lsn and output error if it fails. open_or_create_data_files(): Pass only one flushed_lsn variable.
2017-05-30 11:30:43 +03:00
flushed_lsn);
if (err == DB_SUCCESS) {
/* Initialize the change buffer. */
err = dict_boot();
}
if (err != DB_SUCCESS) {
return(err);
}
/* This must precede recv_apply_hashed_log_recs(true). */
2017-01-26 23:40:11 +02:00
ib_bh = trx_sys_init_at_db_start();
if (srv_force_recovery < SRV_FORCE_NO_LOG_REDO) {
/* Apply the hashed log records to the
respective file pages, for the last batch of
recv_group_scan_log_recs(). */
recv_apply_hashed_log_recs(true);
MDEV-12253: Buffer pool blocks are accessed after they have been freed Problem was that bpage was referenced after it was already freed from LRU. Fixed by adding a new variable encrypted that is passed down to buf_page_check_corrupt() and used in buf_page_get_gen() to stop processing page read. This patch should also address following test failures and bugs: MDEV-12419: IMPORT should not look up tablespace in PageConverter::validate(). This is now removed. MDEV-10099: encryption.innodb_onlinealter_encryption fails sporadically in buildbot MDEV-11420: encryption.innodb_encryption-page-compression failed in buildbot MDEV-11222: encryption.encrypt_and_grep failed in buildbot on P8 Removed dict_table_t::is_encrypted and dict_table_t::ibd_file_missing and replaced these with dict_table_t::file_unreadable. Table ibd file is missing if fil_get_space(space_id) returns NULL and encrypted if not. Removed dict_table_t::is_corrupted field. Ported FilSpace class from 10.2 and using that on buf_page_check_corrupt(), buf_page_decrypt_after_read(), buf_page_encrypt_before_write(), buf_dblwr_process(), buf_read_page(), dict_stats_save_defrag_stats(). Added test cases when enrypted page could be read while doing redo log crash recovery. Also added test case for row compressed blobs. btr_cur_open_at_index_side_func(), btr_cur_open_at_rnd_pos_func(): Avoid referencing block that is NULL. buf_page_get_zip(): Issue error if page read fails. buf_page_get_gen(): Use dberr_t for error detection and do not reference bpage after we hare freed it. buf_mark_space_corrupt(): remove bpage from LRU also when it is encrypted. buf_page_check_corrupt(): @return DB_SUCCESS if page has been read and is not corrupted, DB_PAGE_CORRUPTED if page based on checksum check is corrupted, DB_DECRYPTION_FAILED if page post encryption checksum matches but after decryption normal page checksum does not match. In read case only DB_SUCCESS is possible. buf_page_io_complete(): use dberr_t for error handling. buf_flush_write_block_low(), buf_read_ahead_random(), buf_read_page_async(), buf_read_ahead_linear(), buf_read_ibuf_merge_pages(), buf_read_recv_pages(), fil_aio_wait(): Issue error if page read fails. btr_pcur_move_to_next_page(): Do not reference page if it is NULL. Introduced dict_table_t::is_readable() and dict_index_t::is_readable() that will return true if tablespace exists and pages read from tablespace are not corrupted or page decryption failed. Removed buf_page_t::key_version. After page decryption the key version is not removed from page frame. For unencrypted pages, old key_version is removed at buf_page_encrypt_before_write() dict_stats_update_transient_for_index(), dict_stats_update_transient() Do not continue if table decryption failed or table is corrupted. dict0stats.cc: Introduced a dict_stats_report_error function to avoid code duplication. fil_parse_write_crypt_data(): Check that key read from redo log entry is found from encryption plugin and if it is not, refuse to start. PageConverter::validate(): Removed access to fil_space_t as tablespace is not available during import. Fixed error code on innodb.innodb test. Merged test cased innodb-bad-key-change5 and innodb-bad-key-shutdown to innodb-bad-key-change2. Removed innodb-bad-key-change5 test. Decreased unnecessary complexity on some long lasting tests. Removed fil_inc_pending_ops(), fil_decr_pending_ops(), fil_get_first_space(), fil_get_next_space(), fil_get_first_space_safe(), fil_get_next_space_safe() functions. fil_space_verify_crypt_checksum(): Fixed bug found using ASAN where FIL_PAGE_END_LSN_OLD_CHECKSUM field was incorrectly accessed from row compressed tables. Fixed out of page frame bug for row compressed tables in fil_space_verify_crypt_checksum() found using ASAN. Incorrect function was called for compressed table. Added new tests for discard, rename table and drop (we should allow them even when page decryption fails). Alter table rename is not allowed. Added test for restart with innodb-force-recovery=1 when page read on redo-recovery cant be decrypted. Added test for corrupted table where both page data and FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION is corrupted. Adjusted the test case innodb_bug14147491 so that it does not anymore expect crash. Instead table is just mostly not usable. fil0fil.h: fil_space_acquire_low is not visible function and fil_space_acquire and fil_space_acquire_silent are inline functions. FilSpace class uses fil_space_acquire_low directly. recv_apply_hashed_log_recs() does not return anything.
2017-04-26 15:19:16 +03:00
if (recv_sys->found_corrupt_log) {
return (DB_CORRUPTION);
}
2017-01-26 23:40:11 +02:00
DBUG_PRINT("ib_log", ("apply completed"));
}
if (!srv_read_only_mode) {
2017-01-26 23:40:11 +02:00
const ulint flags = FSP_FLAGS_PAGE_SSIZE();
for (ulint id = 0; id <= srv_undo_tablespaces; id++) {
if (fil_space_get(id)) {
fsp_flags_try_adjust(id, flags);
}
}
if (sum_of_new_sizes > 0) {
/* New data file(s) were added */
mtr_start(&mtr);
fsp_header_inc_size(0, sum_of_new_sizes, &mtr);
mtr_commit(&mtr);
/* Immediately write the log record about
increased tablespace size to disk, so that it
is durable even if mysqld would crash
quickly */
log_buffer_flush_to_disk();
}
}
const ulint tablespace_size_in_header
= fsp_header_get_tablespace_size();
#ifdef UNIV_DEBUG
/* buf_debug_prints = TRUE; */
#endif /* UNIV_DEBUG */
ulint sum_of_data_file_sizes = 0;
for (ulint d = 0; d < srv_n_data_files; d++) {
sum_of_data_file_sizes += srv_data_file_sizes[d];
}
/* Compare the system tablespace file size to what is
stored in FSP_SIZE. In open_or_create_data_files()
we already checked that the file sizes match the
innodb_data_file_path specification. */
if (srv_read_only_mode
|| sum_of_data_file_sizes == tablespace_size_in_header) {
/* Do not complain about the size. */
} else if (!srv_auto_extend_last_data_file
|| sum_of_data_file_sizes
< tablespace_size_in_header) {
ib_logf(IB_LOG_LEVEL_ERROR,
"Tablespace size stored in header is " ULINTPF
" pages, but the sum of data file sizes is "
ULINTPF " pages",
tablespace_size_in_header,
sum_of_data_file_sizes);
if (srv_force_recovery == 0
&& sum_of_data_file_sizes
< tablespace_size_in_header) {
ib_logf(IB_LOG_LEVEL_ERROR,
"Cannot start InnoDB. The tail of"
" the system tablespace is"
" missing. Have you edited"
" innodb_data_file_path in my.cnf"
" in an inappropriate way, removing"
" data files from there?"
" You can set innodb_force_recovery=1"
" in my.cnf to force"
" a startup if you are trying to"
" recover a badly corrupt database.");
return(DB_ERROR);
}
}
n_recovered_trx = UT_LIST_GET_LEN(trx_sys->rw_trx_list);
/* The purge system needs to create the purge view and
therefore requires that the trx_sys is inited. */
trx_purge_sys_create(srv_n_purge_threads, ib_bh);
/* recv_recovery_from_checkpoint_finish needs trx lists which
are initialized in trx_sys_init_at_db_start(). */
recv_recovery_from_checkpoint_finish();
if (srv_force_recovery < SRV_FORCE_NO_IBUF_MERGE) {
/* The following call is necessary for the insert
buffer to work with multiple tablespaces. We must
know the mapping between space id's and .ibd file
names.
In a crash recovery, we check that the info in data
dictionary is consistent with what we already know
about space id's from the call of
fil_load_single_table_tablespaces().
In a normal startup, we create the space objects for
every table in the InnoDB data dictionary that has
an .ibd file.
We also determine the maximum tablespace id used. */
dict_check_t dict_check;
if (recv_needed_recovery) {
dict_check = DICT_CHECK_ALL_LOADED;
} else if (n_recovered_trx) {
dict_check = DICT_CHECK_SOME_LOADED;
} else {
dict_check = DICT_CHECK_NONE_LOADED;
}
/* Create the SYS_TABLESPACES and SYS_DATAFILES system table */
err = dict_create_or_check_sys_tablespace();
if (err != DB_SUCCESS) {
return(err);
}
sys_datafiles_created = true;
/* This function assumes that SYS_DATAFILES exists */
dict_check_tablespaces_and_store_max_id(dict_check);
}
if (!srv_force_recovery
&& !recv_sys->found_corrupt_log
&& (srv_log_file_size_requested != srv_log_file_size
|| srv_n_log_files_found != srv_n_log_files)) {
/* Prepare to replace the redo log files. */
if (srv_read_only_mode) {
ib_logf(IB_LOG_LEVEL_ERROR,
"Cannot resize log files "
"in read-only mode.");
return(DB_READ_ONLY);
}
/* Clean the buffer pool. */
bool success = buf_flush_list(
ULINT_MAX, LSN_MAX, NULL);
ut_a(success);
DBUG_EXECUTE_IF("innodb_log_abort_1",
return(DB_ERROR););
MDEV-12113: install_db shows corruption for rest encryption with innodb_data_file_path=ibdata1:3M; Problem was that FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION field that for encrypted pages even in system datafiles should contain key_version except very first page (0:0) is after encryption overwritten with flush lsn. Ported WL#7990 Repurpose FIL_PAGE_FLUSH_LSN to 10.1 The field FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION is consulted during InnoDB startup. At startup, InnoDB reads the FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION from the first page of each file in the InnoDB system tablespace. If there are multiple files, the minimum and maximum LSN can differ. These numbers are passed to InnoDB startup. Having the number in other files than the first file of the InnoDB system tablespace is not providing much additional value. It is conflicting with other use of the field, such as on InnoDB R-tree index pages and encryption key_version. This worklog will stop writing FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION to other files than the first file of the InnoDB system tablespace (page number 0:0) when system tablespace is encrypted. If tablespace is not encrypted we continue writing FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION to all first pages of system tablespace to avoid unnecessary warnings on downgrade. open_or_create_data_files(): pass only one flushed_lsn parameter xb_load_tablespaces(): pass only one flushed_lsn parameter. buf_page_create(): Improve comment about where FIL_PAGE_FIL_FLUSH_LSN_OR_KEY_VERSION is set. fil_write_flushed_lsn(): A new function, merged from fil_write_lsn_and_arch_no_to_file() and fil_write_flushed_lsn_to_data_files(). Only write to the first page of the system tablespace (page 0:0) if tablespace is encrypted, or write all first pages of system tablespace and invoke fil_flush_file_spaces(FIL_TYPE_TABLESPACE) afterwards. fil_read_first_page(): read flush_lsn and crypt_data only from first datafile. fil_open_single_table_tablespace(): Remove output of LSN, because it was only valid for the system tablespace and the undo tablespaces, not user tablespaces. fil_validate_single_table_tablespace(): Remove output of LSN. checkpoint_now_set(): Use fil_write_flushed_lsn and output a error if operation fails. Remove lsn variable from fsp_open_info. recv_recovery_from_checkpoint_start(): Remove unnecessary second flush_lsn parameter. log_empty_and_mark_files_at_shutdown(): Use fil_writte_flushed_lsn and output error if it fails. open_or_create_data_files(): Pass only one flushed_lsn variable.
2017-05-30 11:30:43 +03:00
flushed_lsn = log_get_lsn();
ib_logf(IB_LOG_LEVEL_WARN,
"Resizing redo log from %u*%u to %u*%u pages"
", LSN=" LSN_PF,
(unsigned) i,
(unsigned) srv_log_file_size,
(unsigned) srv_n_log_files,
(unsigned) srv_log_file_size_requested,
MDEV-12113: install_db shows corruption for rest encryption with innodb_data_file_path=ibdata1:3M; Problem was that FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION field that for encrypted pages even in system datafiles should contain key_version except very first page (0:0) is after encryption overwritten with flush lsn. Ported WL#7990 Repurpose FIL_PAGE_FLUSH_LSN to 10.1 The field FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION is consulted during InnoDB startup. At startup, InnoDB reads the FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION from the first page of each file in the InnoDB system tablespace. If there are multiple files, the minimum and maximum LSN can differ. These numbers are passed to InnoDB startup. Having the number in other files than the first file of the InnoDB system tablespace is not providing much additional value. It is conflicting with other use of the field, such as on InnoDB R-tree index pages and encryption key_version. This worklog will stop writing FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION to other files than the first file of the InnoDB system tablespace (page number 0:0) when system tablespace is encrypted. If tablespace is not encrypted we continue writing FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION to all first pages of system tablespace to avoid unnecessary warnings on downgrade. open_or_create_data_files(): pass only one flushed_lsn parameter xb_load_tablespaces(): pass only one flushed_lsn parameter. buf_page_create(): Improve comment about where FIL_PAGE_FIL_FLUSH_LSN_OR_KEY_VERSION is set. fil_write_flushed_lsn(): A new function, merged from fil_write_lsn_and_arch_no_to_file() and fil_write_flushed_lsn_to_data_files(). Only write to the first page of the system tablespace (page 0:0) if tablespace is encrypted, or write all first pages of system tablespace and invoke fil_flush_file_spaces(FIL_TYPE_TABLESPACE) afterwards. fil_read_first_page(): read flush_lsn and crypt_data only from first datafile. fil_open_single_table_tablespace(): Remove output of LSN, because it was only valid for the system tablespace and the undo tablespaces, not user tablespaces. fil_validate_single_table_tablespace(): Remove output of LSN. checkpoint_now_set(): Use fil_write_flushed_lsn and output a error if operation fails. Remove lsn variable from fsp_open_info. recv_recovery_from_checkpoint_start(): Remove unnecessary second flush_lsn parameter. log_empty_and_mark_files_at_shutdown(): Use fil_writte_flushed_lsn and output error if it fails. open_or_create_data_files(): Pass only one flushed_lsn variable.
2017-05-30 11:30:43 +03:00
flushed_lsn);
buf_flush_wait_batch_end(NULL, BUF_FLUSH_LIST);
/* Flush the old log files. */
log_buffer_flush_to_disk();
/* If innodb_flush_method=O_DSYNC,
we need to explicitly flush the log buffers. */
fil_flush(SRV_LOG_SPACE_FIRST_ID);
MDEV-12113: install_db shows corruption for rest encryption with innodb_data_file_path=ibdata1:3M; Problem was that FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION field that for encrypted pages even in system datafiles should contain key_version except very first page (0:0) is after encryption overwritten with flush lsn. Ported WL#7990 Repurpose FIL_PAGE_FLUSH_LSN to 10.1 The field FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION is consulted during InnoDB startup. At startup, InnoDB reads the FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION from the first page of each file in the InnoDB system tablespace. If there are multiple files, the minimum and maximum LSN can differ. These numbers are passed to InnoDB startup. Having the number in other files than the first file of the InnoDB system tablespace is not providing much additional value. It is conflicting with other use of the field, such as on InnoDB R-tree index pages and encryption key_version. This worklog will stop writing FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION to other files than the first file of the InnoDB system tablespace (page number 0:0) when system tablespace is encrypted. If tablespace is not encrypted we continue writing FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION to all first pages of system tablespace to avoid unnecessary warnings on downgrade. open_or_create_data_files(): pass only one flushed_lsn parameter xb_load_tablespaces(): pass only one flushed_lsn parameter. buf_page_create(): Improve comment about where FIL_PAGE_FIL_FLUSH_LSN_OR_KEY_VERSION is set. fil_write_flushed_lsn(): A new function, merged from fil_write_lsn_and_arch_no_to_file() and fil_write_flushed_lsn_to_data_files(). Only write to the first page of the system tablespace (page 0:0) if tablespace is encrypted, or write all first pages of system tablespace and invoke fil_flush_file_spaces(FIL_TYPE_TABLESPACE) afterwards. fil_read_first_page(): read flush_lsn and crypt_data only from first datafile. fil_open_single_table_tablespace(): Remove output of LSN, because it was only valid for the system tablespace and the undo tablespaces, not user tablespaces. fil_validate_single_table_tablespace(): Remove output of LSN. checkpoint_now_set(): Use fil_write_flushed_lsn and output a error if operation fails. Remove lsn variable from fsp_open_info. recv_recovery_from_checkpoint_start(): Remove unnecessary second flush_lsn parameter. log_empty_and_mark_files_at_shutdown(): Use fil_writte_flushed_lsn and output error if it fails. open_or_create_data_files(): Pass only one flushed_lsn variable.
2017-05-30 11:30:43 +03:00
ut_ad(flushed_lsn == log_get_lsn());
/* Prohibit redo log writes from any other
threads until creating a log checkpoint at the
end of create_log_files(). */
ut_d(recv_no_log_write = TRUE);
ut_ad(!buf_pool_check_no_pending_io());
DBUG_EXECUTE_IF("innodb_log_abort_3",
return(DB_ERROR););
/* Stamp the LSN to the data files. */
MDEV-12113: install_db shows corruption for rest encryption with innodb_data_file_path=ibdata1:3M; Problem was that FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION field that for encrypted pages even in system datafiles should contain key_version except very first page (0:0) is after encryption overwritten with flush lsn. Ported WL#7990 Repurpose FIL_PAGE_FLUSH_LSN to 10.1 The field FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION is consulted during InnoDB startup. At startup, InnoDB reads the FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION from the first page of each file in the InnoDB system tablespace. If there are multiple files, the minimum and maximum LSN can differ. These numbers are passed to InnoDB startup. Having the number in other files than the first file of the InnoDB system tablespace is not providing much additional value. It is conflicting with other use of the field, such as on InnoDB R-tree index pages and encryption key_version. This worklog will stop writing FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION to other files than the first file of the InnoDB system tablespace (page number 0:0) when system tablespace is encrypted. If tablespace is not encrypted we continue writing FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION to all first pages of system tablespace to avoid unnecessary warnings on downgrade. open_or_create_data_files(): pass only one flushed_lsn parameter xb_load_tablespaces(): pass only one flushed_lsn parameter. buf_page_create(): Improve comment about where FIL_PAGE_FIL_FLUSH_LSN_OR_KEY_VERSION is set. fil_write_flushed_lsn(): A new function, merged from fil_write_lsn_and_arch_no_to_file() and fil_write_flushed_lsn_to_data_files(). Only write to the first page of the system tablespace (page 0:0) if tablespace is encrypted, or write all first pages of system tablespace and invoke fil_flush_file_spaces(FIL_TYPE_TABLESPACE) afterwards. fil_read_first_page(): read flush_lsn and crypt_data only from first datafile. fil_open_single_table_tablespace(): Remove output of LSN, because it was only valid for the system tablespace and the undo tablespaces, not user tablespaces. fil_validate_single_table_tablespace(): Remove output of LSN. checkpoint_now_set(): Use fil_write_flushed_lsn and output a error if operation fails. Remove lsn variable from fsp_open_info. recv_recovery_from_checkpoint_start(): Remove unnecessary second flush_lsn parameter. log_empty_and_mark_files_at_shutdown(): Use fil_writte_flushed_lsn and output error if it fails. open_or_create_data_files(): Pass only one flushed_lsn variable.
2017-05-30 11:30:43 +03:00
err = fil_write_flushed_lsn(flushed_lsn);
DBUG_EXECUTE_IF("innodb_log_abort_4", err = DB_ERROR;);
if (err != DB_SUCCESS) {
return(err);
}
/* Close and free the redo log files, so that
we can replace them. */
fil_close_log_files(true);
DBUG_EXECUTE_IF("innodb_log_abort_5",
return(DB_ERROR););
/* Free the old log file space. */
log_group_close_all();
ib_logf(IB_LOG_LEVEL_WARN,
"Starting to delete and rewrite log files.");
srv_log_file_size = srv_log_file_size_requested;
err = create_log_files(create_new_db, logfilename,
MDEV-12113: install_db shows corruption for rest encryption with innodb_data_file_path=ibdata1:3M; Problem was that FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION field that for encrypted pages even in system datafiles should contain key_version except very first page (0:0) is after encryption overwritten with flush lsn. Ported WL#7990 Repurpose FIL_PAGE_FLUSH_LSN to 10.1 The field FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION is consulted during InnoDB startup. At startup, InnoDB reads the FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION from the first page of each file in the InnoDB system tablespace. If there are multiple files, the minimum and maximum LSN can differ. These numbers are passed to InnoDB startup. Having the number in other files than the first file of the InnoDB system tablespace is not providing much additional value. It is conflicting with other use of the field, such as on InnoDB R-tree index pages and encryption key_version. This worklog will stop writing FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION to other files than the first file of the InnoDB system tablespace (page number 0:0) when system tablespace is encrypted. If tablespace is not encrypted we continue writing FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION to all first pages of system tablespace to avoid unnecessary warnings on downgrade. open_or_create_data_files(): pass only one flushed_lsn parameter xb_load_tablespaces(): pass only one flushed_lsn parameter. buf_page_create(): Improve comment about where FIL_PAGE_FIL_FLUSH_LSN_OR_KEY_VERSION is set. fil_write_flushed_lsn(): A new function, merged from fil_write_lsn_and_arch_no_to_file() and fil_write_flushed_lsn_to_data_files(). Only write to the first page of the system tablespace (page 0:0) if tablespace is encrypted, or write all first pages of system tablespace and invoke fil_flush_file_spaces(FIL_TYPE_TABLESPACE) afterwards. fil_read_first_page(): read flush_lsn and crypt_data only from first datafile. fil_open_single_table_tablespace(): Remove output of LSN, because it was only valid for the system tablespace and the undo tablespaces, not user tablespaces. fil_validate_single_table_tablespace(): Remove output of LSN. checkpoint_now_set(): Use fil_write_flushed_lsn and output a error if operation fails. Remove lsn variable from fsp_open_info. recv_recovery_from_checkpoint_start(): Remove unnecessary second flush_lsn parameter. log_empty_and_mark_files_at_shutdown(): Use fil_writte_flushed_lsn and output error if it fails. open_or_create_data_files(): Pass only one flushed_lsn variable.
2017-05-30 11:30:43 +03:00
dirnamelen, flushed_lsn,
logfile0);
MDEV-12113: install_db shows corruption for rest encryption with innodb_data_file_path=ibdata1:3M; Problem was that FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION field that for encrypted pages even in system datafiles should contain key_version except very first page (0:0) is after encryption overwritten with flush lsn. Ported WL#7990 Repurpose FIL_PAGE_FLUSH_LSN to 10.1 The field FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION is consulted during InnoDB startup. At startup, InnoDB reads the FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION from the first page of each file in the InnoDB system tablespace. If there are multiple files, the minimum and maximum LSN can differ. These numbers are passed to InnoDB startup. Having the number in other files than the first file of the InnoDB system tablespace is not providing much additional value. It is conflicting with other use of the field, such as on InnoDB R-tree index pages and encryption key_version. This worklog will stop writing FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION to other files than the first file of the InnoDB system tablespace (page number 0:0) when system tablespace is encrypted. If tablespace is not encrypted we continue writing FIL_PAGE_FLUSH_LSN_OR_KEY_VERSION to all first pages of system tablespace to avoid unnecessary warnings on downgrade. open_or_create_data_files(): pass only one flushed_lsn parameter xb_load_tablespaces(): pass only one flushed_lsn parameter. buf_page_create(): Improve comment about where FIL_PAGE_FIL_FLUSH_LSN_OR_KEY_VERSION is set. fil_write_flushed_lsn(): A new function, merged from fil_write_lsn_and_arch_no_to_file() and fil_write_flushed_lsn_to_data_files(). Only write to the first page of the system tablespace (page 0:0) if tablespace is encrypted, or write all first pages of system tablespace and invoke fil_flush_file_spaces(FIL_TYPE_TABLESPACE) afterwards. fil_read_first_page(): read flush_lsn and crypt_data only from first datafile. fil_open_single_table_tablespace(): Remove output of LSN, because it was only valid for the system tablespace and the undo tablespaces, not user tablespaces. fil_validate_single_table_tablespace(): Remove output of LSN. checkpoint_now_set(): Use fil_write_flushed_lsn and output a error if operation fails. Remove lsn variable from fsp_open_info. recv_recovery_from_checkpoint_start(): Remove unnecessary second flush_lsn parameter. log_empty_and_mark_files_at_shutdown(): Use fil_writte_flushed_lsn and output error if it fails. open_or_create_data_files(): Pass only one flushed_lsn variable.
2017-05-30 11:30:43 +03:00
if (err != DB_SUCCESS) {
return(err);
}
err = create_log_files_rename(logfilename, dirnamelen,
log_get_lsn(), logfile0);
if (err != DB_SUCCESS) {
return(err);
}
}
recv_recovery_rollback_active();
MDEV-13039 innodb_fast_shutdown=0 may fail to purge all undo log When a slow shutdown is performed soon after spawning some work for background threads that can create or commit transactions, it is possible that new transactions are started or committed after the purge has finished. This is violating the specification of innodb_fast_shutdown=0, namely that the purge must be completed. (None of the history of the recent transactions would be purged.) Also, it is possible that the purge threads would exit in slow shutdown while there exist active transactions, such as recovered incomplete transactions that are being rolled back. Thus, the slow shutdown could fail to purge some undo log that becomes purgeable after the transaction commit or rollback. srv_undo_sources: A flag that indicates if undo log can be generated or the persistent, whether by background threads or by user SQL. Even when this flag is clear, active transactions that already exist in the system may be committed or rolled back. innodb_shutdown(): Renamed from innobase_shutdown_for_mysql(). Do not return an error code; the operation never fails. Clear the srv_undo_sources flag, and also ensure that the background DROP TABLE queue is empty. srv_purge_should_exit(): Do not allow the purge to exit if srv_undo_sources are active or the background DROP TABLE queue is not empty, or in slow shutdown, if any active transactions exist (and are being rolled back). srv_purge_coordinator_thread(): Remove some previous workarounds for this bug. innobase_start_or_create_for_mysql(): Set buf_page_cleaner_is_active and srv_dict_stats_thread_active directly. Set srv_undo_sources before starting the purge subsystem, to prevent immediate shutdown of the purge. Create dict_stats_thread and fts_optimize_thread immediately after setting srv_undo_sources, so that shutdown can use this flag to determine if these subsystems were started. dict_stats_shutdown(): Shut down dict_stats_thread. Backported from 10.2. srv_shutdown_table_bg_threads(): Remove (unused).
2017-06-08 15:43:06 +03:00
srv_startup_is_before_trx_rollback_phase = FALSE;
/* It is possible that file_format tag has never
been set. In this case we initialize it to minimum
value. Important to note that we can do it ONLY after
we have finished the recovery process so that the
image of TRX_SYS_PAGE_NO is not stale. */
trx_sys_file_format_tag_init();
}
ut_ad(err == DB_SUCCESS);
ut_a(sum_of_new_sizes != ULINT_UNDEFINED);
#ifdef UNIV_LOG_ARCHIVE
/* Archiving is always off under MySQL */
if (!srv_log_archive_on) {
ut_a(DB_SUCCESS == log_archive_noarchivelog());
} else {
mutex_enter(&(log_sys->mutex));
start_archive = FALSE;
if (log_sys->archiving_state == LOG_ARCH_OFF) {
start_archive = TRUE;
}
mutex_exit(&(log_sys->mutex));
if (start_archive) {
ut_a(DB_SUCCESS == log_archive_archivelog());
}
}
#endif /* UNIV_LOG_ARCHIVE */
/* fprintf(stderr, "Max allowed record size %lu\n",
page_get_free_space_of_empty() / 2); */
if (!buf_dblwr_create()) {
return(srv_init_abort(create_new_db, __FILE__, __LINE__,
DB_ERROR));
}
/* Here the double write buffer has already been created and so
any new rollback segments will be allocated after the double
write buffer. The default segment should already exist.
We create the new segments only if it's a new database or
the database was shutdown cleanly. */
/* Note: When creating the extra rollback segments during an upgrade
we violate the latching order, even if the change buffer is empty.
We make an exception in sync0sync.cc and check srv_is_being_started
for that violation. It cannot create a deadlock because we are still
running in single threaded mode essentially. Only the IO threads
should be running at this stage. */
ut_a(srv_undo_logs > 0);
ut_a(srv_undo_logs <= TRX_SYS_N_RSEGS);
/* The number of rsegs that exist in InnoDB is given by status
variable srv_available_undo_logs. The number of rsegs to use can
be set using the dynamic global variable srv_undo_logs. */
srv_available_undo_logs = trx_sys_create_rsegs(
srv_undo_tablespaces, srv_undo_logs);
if (srv_available_undo_logs == ULINT_UNDEFINED) {
2014-02-26 19:23:04 +01:00
/* Can only happen if server is read only. */
ut_a(srv_read_only_mode);
srv_undo_logs = ULONG_UNDEFINED;
} else if (srv_available_undo_logs < srv_undo_logs) {
/* Should due to out of file space. */
return (srv_init_abort(create_new_db, __FILE__, __LINE__, DB_ERROR));
}
if (!srv_read_only_mode) {
/* Create the thread which watches the timeouts
for lock waits */
thread_handles[2 + SRV_MAX_N_IO_THREADS] = os_thread_create(
lock_wait_timeout_thread,
NULL, thread_ids + 2 + SRV_MAX_N_IO_THREADS);
thread_started[2 + SRV_MAX_N_IO_THREADS] = true;
MDEV-11638 Encryption causes race conditions in InnoDB shutdown InnoDB shutdown failed to properly take fil_crypt_thread() into account. The encryption threads were signalled to shut down together with other non-critical tasks. This could be much too early in case of slow shutdown, which could need minutes to complete the purge. Furthermore, InnoDB failed to wait for the fil_crypt_thread() to actually exit before proceeding to the final steps of shutdown, causing the race conditions. Furthermore, the log_scrub_thread() was shut down way too early. Also it should remain until the SRV_SHUTDOWN_FLUSH_PHASE. fil_crypt_threads_end(): Remove. This would cause the threads to be terminated way too early. srv_buf_dump_thread_active, srv_dict_stats_thread_active, lock_sys->timeout_thread_active, log_scrub_thread_active, srv_monitor_active, srv_error_monitor_active: Remove a race condition between startup and shutdown, by setting these in the startup thread that creates threads, not in each created thread. In this way, once the flag is cleared, it will remain cleared during shutdown. srv_n_fil_crypt_threads_started, fil_crypt_threads_event: Declare in global rather than static scope. log_scrub_event, srv_log_scrub_thread_active, log_scrub_thread(): Declare in static rather than global scope. Let these be created by log_init() and freed by log_shutdown(). rotate_thread_t::should_shutdown(): Do not shut down before the SRV_SHUTDOWN_FLUSH_PHASE. srv_any_background_threads_are_active(): Remove. These checks now exist in logs_empty_and_mark_files_at_shutdown(). logs_empty_and_mark_files_at_shutdown(): Shut down the threads in the proper order. Keep fil_crypt_thread() and log_scrub_thread() alive until SRV_SHUTDOWN_FLUSH_PHASE, and check that they actually terminate.
2017-01-04 18:43:32 +02:00
lock_sys->timeout_thread_active = true;
/* Create the thread which warns of long semaphore waits */
MDEV-11638 Encryption causes race conditions in InnoDB shutdown InnoDB shutdown failed to properly take fil_crypt_thread() into account. The encryption threads were signalled to shut down together with other non-critical tasks. This could be much too early in case of slow shutdown, which could need minutes to complete the purge. Furthermore, InnoDB failed to wait for the fil_crypt_thread() to actually exit before proceeding to the final steps of shutdown, causing the race conditions. Furthermore, the log_scrub_thread() was shut down way too early. Also it should remain until the SRV_SHUTDOWN_FLUSH_PHASE. fil_crypt_threads_end(): Remove. This would cause the threads to be terminated way too early. srv_buf_dump_thread_active, srv_dict_stats_thread_active, lock_sys->timeout_thread_active, log_scrub_thread_active, srv_monitor_active, srv_error_monitor_active: Remove a race condition between startup and shutdown, by setting these in the startup thread that creates threads, not in each created thread. In this way, once the flag is cleared, it will remain cleared during shutdown. srv_n_fil_crypt_threads_started, fil_crypt_threads_event: Declare in global rather than static scope. log_scrub_event, srv_log_scrub_thread_active, log_scrub_thread(): Declare in static rather than global scope. Let these be created by log_init() and freed by log_shutdown(). rotate_thread_t::should_shutdown(): Do not shut down before the SRV_SHUTDOWN_FLUSH_PHASE. srv_any_background_threads_are_active(): Remove. These checks now exist in logs_empty_and_mark_files_at_shutdown(). logs_empty_and_mark_files_at_shutdown(): Shut down the threads in the proper order. Keep fil_crypt_thread() and log_scrub_thread() alive until SRV_SHUTDOWN_FLUSH_PHASE, and check that they actually terminate.
2017-01-04 18:43:32 +02:00
srv_error_monitor_active = true;
thread_handles[3 + SRV_MAX_N_IO_THREADS] = os_thread_create(
srv_error_monitor_thread,
NULL, thread_ids + 3 + SRV_MAX_N_IO_THREADS);
thread_started[3 + SRV_MAX_N_IO_THREADS] = true;
/* Create the thread which prints InnoDB monitor info */
MDEV-11638 Encryption causes race conditions in InnoDB shutdown InnoDB shutdown failed to properly take fil_crypt_thread() into account. The encryption threads were signalled to shut down together with other non-critical tasks. This could be much too early in case of slow shutdown, which could need minutes to complete the purge. Furthermore, InnoDB failed to wait for the fil_crypt_thread() to actually exit before proceeding to the final steps of shutdown, causing the race conditions. Furthermore, the log_scrub_thread() was shut down way too early. Also it should remain until the SRV_SHUTDOWN_FLUSH_PHASE. fil_crypt_threads_end(): Remove. This would cause the threads to be terminated way too early. srv_buf_dump_thread_active, srv_dict_stats_thread_active, lock_sys->timeout_thread_active, log_scrub_thread_active, srv_monitor_active, srv_error_monitor_active: Remove a race condition between startup and shutdown, by setting these in the startup thread that creates threads, not in each created thread. In this way, once the flag is cleared, it will remain cleared during shutdown. srv_n_fil_crypt_threads_started, fil_crypt_threads_event: Declare in global rather than static scope. log_scrub_event, srv_log_scrub_thread_active, log_scrub_thread(): Declare in static rather than global scope. Let these be created by log_init() and freed by log_shutdown(). rotate_thread_t::should_shutdown(): Do not shut down before the SRV_SHUTDOWN_FLUSH_PHASE. srv_any_background_threads_are_active(): Remove. These checks now exist in logs_empty_and_mark_files_at_shutdown(). logs_empty_and_mark_files_at_shutdown(): Shut down the threads in the proper order. Keep fil_crypt_thread() and log_scrub_thread() alive until SRV_SHUTDOWN_FLUSH_PHASE, and check that they actually terminate.
2017-01-04 18:43:32 +02:00
srv_monitor_active = true;
thread_handles[4 + SRV_MAX_N_IO_THREADS] = os_thread_create(
srv_monitor_thread,
NULL, thread_ids + 4 + SRV_MAX_N_IO_THREADS);
thread_started[4 + SRV_MAX_N_IO_THREADS] = true;
}
/* Create the SYS_FOREIGN and SYS_FOREIGN_COLS system tables */
err = dict_create_or_check_foreign_constraint_tables();
if (err != DB_SUCCESS) {
return(err);
}
/* Create the SYS_TABLESPACES and SYS_DATAFILES system tables if we
have not done that already on crash recovery. */
if (sys_datafiles_created == false) {
err = dict_create_or_check_sys_tablespace();
if (err != DB_SUCCESS) {
return(err);
}
}
srv_is_being_started = FALSE;
ut_a(trx_purge_state() == PURGE_STATE_INIT);
/* Create the master thread which does purge and other utility
operations */
if (!srv_read_only_mode) {
thread_handles[1 + SRV_MAX_N_IO_THREADS] = os_thread_create(
srv_master_thread,
NULL, thread_ids + (1 + SRV_MAX_N_IO_THREADS));
thread_started[1 + SRV_MAX_N_IO_THREADS] = true;
MDEV-13039 innodb_fast_shutdown=0 may fail to purge all undo log When a slow shutdown is performed soon after spawning some work for background threads that can create or commit transactions, it is possible that new transactions are started or committed after the purge has finished. This is violating the specification of innodb_fast_shutdown=0, namely that the purge must be completed. (None of the history of the recent transactions would be purged.) Also, it is possible that the purge threads would exit in slow shutdown while there exist active transactions, such as recovered incomplete transactions that are being rolled back. Thus, the slow shutdown could fail to purge some undo log that becomes purgeable after the transaction commit or rollback. srv_undo_sources: A flag that indicates if undo log can be generated or the persistent, whether by background threads or by user SQL. Even when this flag is clear, active transactions that already exist in the system may be committed or rolled back. innodb_shutdown(): Renamed from innobase_shutdown_for_mysql(). Do not return an error code; the operation never fails. Clear the srv_undo_sources flag, and also ensure that the background DROP TABLE queue is empty. srv_purge_should_exit(): Do not allow the purge to exit if srv_undo_sources are active or the background DROP TABLE queue is not empty, or in slow shutdown, if any active transactions exist (and are being rolled back). srv_purge_coordinator_thread(): Remove some previous workarounds for this bug. innobase_start_or_create_for_mysql(): Set buf_page_cleaner_is_active and srv_dict_stats_thread_active directly. Set srv_undo_sources before starting the purge subsystem, to prevent immediate shutdown of the purge. Create dict_stats_thread and fts_optimize_thread immediately after setting srv_undo_sources, so that shutdown can use this flag to determine if these subsystems were started. dict_stats_shutdown(): Shut down dict_stats_thread. Backported from 10.2. srv_shutdown_table_bg_threads(): Remove (unused).
2017-06-08 15:43:06 +03:00
srv_undo_sources = true;
/* Create the dict stats gathering thread */
srv_dict_stats_thread_active = true;
dict_stats_thread_handle = os_thread_create(
dict_stats_thread, NULL, NULL);
dict_stats_thread_started = true;
/* Create the thread that will optimize the FTS sub-system. */
fts_optimize_init();
}
if (!srv_read_only_mode
&& srv_force_recovery < SRV_FORCE_NO_BACKGROUND) {
thread_handles[5 + SRV_MAX_N_IO_THREADS] = os_thread_create(
srv_purge_coordinator_thread,
NULL, thread_ids + 5 + SRV_MAX_N_IO_THREADS);
thread_started[5 + SRV_MAX_N_IO_THREADS] = true;
ut_a(UT_ARR_SIZE(thread_ids)
> 5 + srv_n_purge_threads + SRV_MAX_N_IO_THREADS);
/* We've already created the purge coordinator thread above. */
for (i = 1; i < srv_n_purge_threads; ++i) {
thread_handles[5 + i + SRV_MAX_N_IO_THREADS] = os_thread_create(
srv_worker_thread, NULL,
thread_ids + 5 + i + SRV_MAX_N_IO_THREADS);
thread_started[5 + i + SRV_MAX_N_IO_THREADS] = true;
}
srv_start_wait_for_purge_to_start();
} else {
purge_sys->state = PURGE_STATE_DISABLED;
}
if (!srv_read_only_mode) {
if (srv_use_mtflush) {
/* Start multi-threaded flush threads */
mtflush_ctx = buf_mtflu_handler_init(
srv_mtflush_threads,
srv_buf_pool_instances);
/* Set up the thread ids */
buf_mtflu_set_thread_ids(
srv_mtflush_threads,
mtflush_ctx,
2014-02-28 09:05:36 +02:00
(thread_ids + 6 + 32));
}
MDEV-13039 innodb_fast_shutdown=0 may fail to purge all undo log When a slow shutdown is performed soon after spawning some work for background threads that can create or commit transactions, it is possible that new transactions are started or committed after the purge has finished. This is violating the specification of innodb_fast_shutdown=0, namely that the purge must be completed. (None of the history of the recent transactions would be purged.) Also, it is possible that the purge threads would exit in slow shutdown while there exist active transactions, such as recovered incomplete transactions that are being rolled back. Thus, the slow shutdown could fail to purge some undo log that becomes purgeable after the transaction commit or rollback. srv_undo_sources: A flag that indicates if undo log can be generated or the persistent, whether by background threads or by user SQL. Even when this flag is clear, active transactions that already exist in the system may be committed or rolled back. innodb_shutdown(): Renamed from innobase_shutdown_for_mysql(). Do not return an error code; the operation never fails. Clear the srv_undo_sources flag, and also ensure that the background DROP TABLE queue is empty. srv_purge_should_exit(): Do not allow the purge to exit if srv_undo_sources are active or the background DROP TABLE queue is not empty, or in slow shutdown, if any active transactions exist (and are being rolled back). srv_purge_coordinator_thread(): Remove some previous workarounds for this bug. innobase_start_or_create_for_mysql(): Set buf_page_cleaner_is_active and srv_dict_stats_thread_active directly. Set srv_undo_sources before starting the purge subsystem, to prevent immediate shutdown of the purge. Create dict_stats_thread and fts_optimize_thread immediately after setting srv_undo_sources, so that shutdown can use this flag to determine if these subsystems were started. dict_stats_shutdown(): Shut down dict_stats_thread. Backported from 10.2. srv_shutdown_table_bg_threads(): Remove (unused).
2017-06-08 15:43:06 +03:00
buf_page_cleaner_is_active = true;
buf_flush_page_cleaner_thread_handle = os_thread_create(buf_flush_page_cleaner_thread, NULL, NULL);
buf_flush_page_cleaner_thread_started = true;
}
if (srv_print_verbose_log) {
ib_logf(IB_LOG_LEVEL_INFO,
"%s started; log sequence number " LSN_PF "",
INNODB_VERSION_STR, srv_start_lsn);
}
if (srv_force_recovery > 0) {
ib_logf(IB_LOG_LEVEL_INFO,
"!!! innodb_force_recovery is set to %lu !!!",
(ulong) srv_force_recovery);
}
if (!srv_read_only_mode) {
/*
Create a checkpoint before logging anything new, so that
the current encryption key in use is definitely logged
before any log blocks encrypted with that key.
*/
log_make_checkpoint_at(LSN_MAX, TRUE);
}
if (srv_force_recovery == 0) {
/* In the insert buffer we may have even bigger tablespace
id's, because we may have dropped those tablespaces, but
insert buffer merge has not had time to clean the records from
the ibuf tree. */
ibuf_update_max_tablespace_id();
}
if (!srv_read_only_mode) {
2015-07-14 16:05:29 -04:00
#ifdef WITH_WSREP
/*
Create the dump/load thread only when not running with
--wsrep-recover.
*/
if (!wsrep_recovery) {
#endif /* WITH_WSREP */
/* Create the buffer pool dump/load thread */
2017-06-12 14:26:32 +03:00
srv_buf_dump_thread_active = true;
2015-07-14 16:05:29 -04:00
buf_dump_thread_handle=
os_thread_create(buf_dump_thread, NULL, NULL);
MDEV-11638 Encryption causes race conditions in InnoDB shutdown InnoDB shutdown failed to properly take fil_crypt_thread() into account. The encryption threads were signalled to shut down together with other non-critical tasks. This could be much too early in case of slow shutdown, which could need minutes to complete the purge. Furthermore, InnoDB failed to wait for the fil_crypt_thread() to actually exit before proceeding to the final steps of shutdown, causing the race conditions. Furthermore, the log_scrub_thread() was shut down way too early. Also it should remain until the SRV_SHUTDOWN_FLUSH_PHASE. fil_crypt_threads_end(): Remove. This would cause the threads to be terminated way too early. srv_buf_dump_thread_active, srv_dict_stats_thread_active, lock_sys->timeout_thread_active, log_scrub_thread_active, srv_monitor_active, srv_error_monitor_active: Remove a race condition between startup and shutdown, by setting these in the startup thread that creates threads, not in each created thread. In this way, once the flag is cleared, it will remain cleared during shutdown. srv_n_fil_crypt_threads_started, fil_crypt_threads_event: Declare in global rather than static scope. log_scrub_event, srv_log_scrub_thread_active, log_scrub_thread(): Declare in static rather than global scope. Let these be created by log_init() and freed by log_shutdown(). rotate_thread_t::should_shutdown(): Do not shut down before the SRV_SHUTDOWN_FLUSH_PHASE. srv_any_background_threads_are_active(): Remove. These checks now exist in logs_empty_and_mark_files_at_shutdown(). logs_empty_and_mark_files_at_shutdown(): Shut down the threads in the proper order. Keep fil_crypt_thread() and log_scrub_thread() alive until SRV_SHUTDOWN_FLUSH_PHASE, and check that they actually terminate.
2017-01-04 18:43:32 +02:00
buf_dump_thread_started = true;
2015-07-14 16:05:29 -04:00
#ifdef WITH_WSREP
} else {
ib_logf(IB_LOG_LEVEL_WARN,
"Skipping buffer pool dump/restore during "
"wsrep recovery.");
}
#endif /* WITH_WSREP */
2014-12-22 16:53:17 +02:00
/* Create thread(s) that handles key rotation */
fil_system_enter();
2014-12-22 16:53:17 +02:00
fil_crypt_threads_init();
fil_system_exit();
/* Init data for datafile scrub threads */
btr_scrub_init();
/* Initialize online defragmentation. */
btr_defragment_init();
btr_defragment_thread_active = true;
os_thread_create(btr_defragment_thread, NULL, NULL);
}
srv_was_started = TRUE;
return(DB_SUCCESS);
}
#if 0
/********************************************************************
Sync all FTS cache before shutdown */
static
void
srv_fts_close(void)
/*===============*/
{
dict_table_t* table;
for (table = UT_LIST_GET_FIRST(dict_sys->table_LRU);
table; table = UT_LIST_GET_NEXT(table_LRU, table)) {
fts_t* fts = table->fts;
if (fts != NULL) {
fts_sync_table(table);
}
}
for (table = UT_LIST_GET_FIRST(dict_sys->table_non_LRU);
table; table = UT_LIST_GET_NEXT(table_LRU, table)) {
fts_t* fts = table->fts;
if (fts != NULL) {
fts_sync_table(table);
}
}
}
#endif
MDEV-13039 innodb_fast_shutdown=0 may fail to purge all undo log When a slow shutdown is performed soon after spawning some work for background threads that can create or commit transactions, it is possible that new transactions are started or committed after the purge has finished. This is violating the specification of innodb_fast_shutdown=0, namely that the purge must be completed. (None of the history of the recent transactions would be purged.) Also, it is possible that the purge threads would exit in slow shutdown while there exist active transactions, such as recovered incomplete transactions that are being rolled back. Thus, the slow shutdown could fail to purge some undo log that becomes purgeable after the transaction commit or rollback. srv_undo_sources: A flag that indicates if undo log can be generated or the persistent, whether by background threads or by user SQL. Even when this flag is clear, active transactions that already exist in the system may be committed or rolled back. innodb_shutdown(): Renamed from innobase_shutdown_for_mysql(). Do not return an error code; the operation never fails. Clear the srv_undo_sources flag, and also ensure that the background DROP TABLE queue is empty. srv_purge_should_exit(): Do not allow the purge to exit if srv_undo_sources are active or the background DROP TABLE queue is not empty, or in slow shutdown, if any active transactions exist (and are being rolled back). srv_purge_coordinator_thread(): Remove some previous workarounds for this bug. innobase_start_or_create_for_mysql(): Set buf_page_cleaner_is_active and srv_dict_stats_thread_active directly. Set srv_undo_sources before starting the purge subsystem, to prevent immediate shutdown of the purge. Create dict_stats_thread and fts_optimize_thread immediately after setting srv_undo_sources, so that shutdown can use this flag to determine if these subsystems were started. dict_stats_shutdown(): Shut down dict_stats_thread. Backported from 10.2. srv_shutdown_table_bg_threads(): Remove (unused).
2017-06-08 15:43:06 +03:00
/** Shut down InnoDB. */
UNIV_INTERN
MDEV-13039 innodb_fast_shutdown=0 may fail to purge all undo log When a slow shutdown is performed soon after spawning some work for background threads that can create or commit transactions, it is possible that new transactions are started or committed after the purge has finished. This is violating the specification of innodb_fast_shutdown=0, namely that the purge must be completed. (None of the history of the recent transactions would be purged.) Also, it is possible that the purge threads would exit in slow shutdown while there exist active transactions, such as recovered incomplete transactions that are being rolled back. Thus, the slow shutdown could fail to purge some undo log that becomes purgeable after the transaction commit or rollback. srv_undo_sources: A flag that indicates if undo log can be generated or the persistent, whether by background threads or by user SQL. Even when this flag is clear, active transactions that already exist in the system may be committed or rolled back. innodb_shutdown(): Renamed from innobase_shutdown_for_mysql(). Do not return an error code; the operation never fails. Clear the srv_undo_sources flag, and also ensure that the background DROP TABLE queue is empty. srv_purge_should_exit(): Do not allow the purge to exit if srv_undo_sources are active or the background DROP TABLE queue is not empty, or in slow shutdown, if any active transactions exist (and are being rolled back). srv_purge_coordinator_thread(): Remove some previous workarounds for this bug. innobase_start_or_create_for_mysql(): Set buf_page_cleaner_is_active and srv_dict_stats_thread_active directly. Set srv_undo_sources before starting the purge subsystem, to prevent immediate shutdown of the purge. Create dict_stats_thread and fts_optimize_thread immediately after setting srv_undo_sources, so that shutdown can use this flag to determine if these subsystems were started. dict_stats_shutdown(): Shut down dict_stats_thread. Backported from 10.2. srv_shutdown_table_bg_threads(): Remove (unused).
2017-06-08 15:43:06 +03:00
void
innodb_shutdown()
{
ulint i;
if (!srv_was_started) {
if (srv_is_being_started) {
ib_logf(IB_LOG_LEVEL_WARN,
"Shutting down an improperly started, "
"or created database!");
}
}
MDEV-13039 innodb_fast_shutdown=0 may fail to purge all undo log When a slow shutdown is performed soon after spawning some work for background threads that can create or commit transactions, it is possible that new transactions are started or committed after the purge has finished. This is violating the specification of innodb_fast_shutdown=0, namely that the purge must be completed. (None of the history of the recent transactions would be purged.) Also, it is possible that the purge threads would exit in slow shutdown while there exist active transactions, such as recovered incomplete transactions that are being rolled back. Thus, the slow shutdown could fail to purge some undo log that becomes purgeable after the transaction commit or rollback. srv_undo_sources: A flag that indicates if undo log can be generated or the persistent, whether by background threads or by user SQL. Even when this flag is clear, active transactions that already exist in the system may be committed or rolled back. innodb_shutdown(): Renamed from innobase_shutdown_for_mysql(). Do not return an error code; the operation never fails. Clear the srv_undo_sources flag, and also ensure that the background DROP TABLE queue is empty. srv_purge_should_exit(): Do not allow the purge to exit if srv_undo_sources are active or the background DROP TABLE queue is not empty, or in slow shutdown, if any active transactions exist (and are being rolled back). srv_purge_coordinator_thread(): Remove some previous workarounds for this bug. innobase_start_or_create_for_mysql(): Set buf_page_cleaner_is_active and srv_dict_stats_thread_active directly. Set srv_undo_sources before starting the purge subsystem, to prevent immediate shutdown of the purge. Create dict_stats_thread and fts_optimize_thread immediately after setting srv_undo_sources, so that shutdown can use this flag to determine if these subsystems were started. dict_stats_shutdown(): Shut down dict_stats_thread. Backported from 10.2. srv_shutdown_table_bg_threads(): Remove (unused).
2017-06-08 15:43:06 +03:00
if (srv_undo_sources) {
ut_ad(!srv_read_only_mode);
/* Shutdown the FTS optimize sub system. */
fts_optimize_start_shutdown();
fts_optimize_end();
MDEV-13039 innodb_fast_shutdown=0 may fail to purge all undo log When a slow shutdown is performed soon after spawning some work for background threads that can create or commit transactions, it is possible that new transactions are started or committed after the purge has finished. This is violating the specification of innodb_fast_shutdown=0, namely that the purge must be completed. (None of the history of the recent transactions would be purged.) Also, it is possible that the purge threads would exit in slow shutdown while there exist active transactions, such as recovered incomplete transactions that are being rolled back. Thus, the slow shutdown could fail to purge some undo log that becomes purgeable after the transaction commit or rollback. srv_undo_sources: A flag that indicates if undo log can be generated or the persistent, whether by background threads or by user SQL. Even when this flag is clear, active transactions that already exist in the system may be committed or rolled back. innodb_shutdown(): Renamed from innobase_shutdown_for_mysql(). Do not return an error code; the operation never fails. Clear the srv_undo_sources flag, and also ensure that the background DROP TABLE queue is empty. srv_purge_should_exit(): Do not allow the purge to exit if srv_undo_sources are active or the background DROP TABLE queue is not empty, or in slow shutdown, if any active transactions exist (and are being rolled back). srv_purge_coordinator_thread(): Remove some previous workarounds for this bug. innobase_start_or_create_for_mysql(): Set buf_page_cleaner_is_active and srv_dict_stats_thread_active directly. Set srv_undo_sources before starting the purge subsystem, to prevent immediate shutdown of the purge. Create dict_stats_thread and fts_optimize_thread immediately after setting srv_undo_sources, so that shutdown can use this flag to determine if these subsystems were started. dict_stats_shutdown(): Shut down dict_stats_thread. Backported from 10.2. srv_shutdown_table_bg_threads(): Remove (unused).
2017-06-08 15:43:06 +03:00
dict_stats_shutdown();
while (row_get_background_drop_list_len_low()) {
srv_wake_master_thread();
os_thread_yield();
}
srv_undo_sources = false;
}
/* 1. Flush the buffer pool to disk, write the current lsn to
the tablespace header(s), and copy all log data to archive.
The step 1 is the real InnoDB shutdown. The remaining steps 2 - ...
just free data structures after the shutdown. */
logs_empty_and_mark_files_at_shutdown();
if (srv_conc_get_active_threads() != 0) {
ib_logf(IB_LOG_LEVEL_WARN,
"Query counter shows %ld queries still "
"inside InnoDB at shutdown",
srv_conc_get_active_threads());
}
/* 2. Make all threads created by InnoDB to exit */
srv_shutdown_state = SRV_SHUTDOWN_EXIT_THREADS;
/* All threads end up waiting for certain events. Put those events
to the signaled state. Then the threads will exit themselves after
os_event_wait(). */
for (i = 0; i < 1000; i++) {
/* NOTE: IF YOU CREATE THREADS IN INNODB, YOU MUST EXIT THEM
HERE OR EARLIER */
if (!srv_read_only_mode) {
/* a. Let the lock timeout thread exit */
os_event_set(lock_sys->timeout_event);
/* b. srv error monitor thread exits automatically,
no need to do anything here */
/* c. We wake the master thread so that it exits */
srv_wake_master_thread();
/* d. Wakeup purge threads. */
srv_purge_wakeup();
}
/* e. Exit the i/o threads */
os_aio_wake_all_threads_at_shutdown();
/* f. dict_stats_thread is signaled from
logs_empty_and_mark_files_at_shutdown() and should have
already quit or is quitting right now. */
if (srv_use_mtflush) {
/* g. Exit the multi threaded flush threads */
buf_mtflu_io_thread_exit();
}
os_mutex_enter(os_sync_mutex);
if (os_thread_count == 0) {
/* All the threads have exited or are just exiting;
NOTE that the threads may not have completed their
exit yet. Should we use pthread_join() to make sure
they have exited? If we did, we would have to
remove the pthread_detach() from
os_thread_exit(). Now we just sleep 0.1
seconds and hope that is enough! */
os_mutex_exit(os_sync_mutex);
os_thread_sleep(100000);
break;
}
os_mutex_exit(os_sync_mutex);
os_thread_sleep(100000);
}
if (i == 1000) {
ib_logf(IB_LOG_LEVEL_WARN,
"%lu threads created by InnoDB"
" had not exited at shutdown!",
(ulong) os_thread_count);
}
if (srv_monitor_file) {
fclose(srv_monitor_file);
srv_monitor_file = 0;
if (srv_monitor_file_name) {
unlink(srv_monitor_file_name);
mem_free(srv_monitor_file_name);
}
}
if (srv_dict_tmpfile) {
fclose(srv_dict_tmpfile);
srv_dict_tmpfile = 0;
}
if (srv_misc_tmpfile) {
fclose(srv_misc_tmpfile);
srv_misc_tmpfile = 0;
}
if (!srv_read_only_mode) {
dict_stats_thread_deinit();
2014-12-22 16:53:17 +02:00
fil_crypt_threads_cleanup();
btr_scrub_cleanup();
btr_defragment_shutdown();
}
#ifdef __WIN__
/* MDEV-361: ha_innodb.dll leaks handles on Windows
MDEV-7403: should not pass recv_writer_thread_handle to
CloseHandle().
On Windows we should call CloseHandle() for all
open thread handles. */
if (os_thread_count == 0) {
for (i = 0; i < SRV_MAX_N_IO_THREADS + 6 + 32; ++i) {
if (thread_started[i]) {
CloseHandle(thread_handles[i]);
}
}
if (buf_flush_page_cleaner_thread_started) {
CloseHandle(buf_flush_page_cleaner_thread_handle);
}
if (buf_dump_thread_started) {
CloseHandle(buf_dump_thread_handle);
}
if (dict_stats_thread_started) {
CloseHandle(dict_stats_thread_handle);
}
}
#endif /* __WIN __ */
/* This must be disabled before closing the buffer pool
and closing the data dictionary. */
btr_search_disable();
ibuf_close();
log_shutdown();
trx_sys_file_format_close();
trx_sys_close();
2015-06-16 10:57:05 +02:00
lock_sys_close();
/* We don't create these mutexes in RO mode because we don't create
the temp files that the cover. */
if (!srv_read_only_mode) {
mutex_free(&srv_monitor_file_mutex);
mutex_free(&srv_dict_tmpfile_mutex);
mutex_free(&srv_misc_tmpfile_mutex);
}
dict_close();
btr_search_sys_free();
/* 3. Free all InnoDB's own mutexes and the os_fast_mutexes inside
them */
os_aio_free();
que_close();
row_mysql_close();
srv_mon_free();
fil_close();
sync_close();
srv_free();
/* 4. Free all os_events and os_mutexes */
os_sync_free();
/* 5. Free all allocated memory */
pars_lexer_close();
log_mem_free();
buf_pool_free(srv_buf_pool_instances);
mem_close();
/* ut_free_all_mem() frees all allocated memory not freed yet
in shutdown, and it will also free the ut_list_mutex, so it
should be the last one for all operation */
ut_free_all_mem();
if (os_thread_count != 0
|| os_event_count != 0
|| os_mutex_count != 0
|| os_fast_mutex_count != 0) {
ib_logf(IB_LOG_LEVEL_WARN,
"Some resources were not cleaned up in shutdown: "
"threads %lu, events %lu, os_mutexes %lu, "
"os_fast_mutexes %lu",
(ulong) os_thread_count, (ulong) os_event_count,
(ulong) os_mutex_count, (ulong) os_fast_mutex_count);
}
if (dict_foreign_err_file) {
fclose(dict_foreign_err_file);
}
if (srv_print_verbose_log) {
ib_logf(IB_LOG_LEVEL_INFO,
"Shutdown completed; log sequence number " LSN_PF "",
srv_shutdown_lsn);
}
srv_was_started = FALSE;
srv_start_has_been_called = FALSE;
}
#endif /* !UNIV_HOTBACKUP */
/*****************************************************************//**
Get the meta-data filename from the table name. */
UNIV_INTERN
void
srv_get_meta_data_filename(
/*=======================*/
dict_table_t* table, /*!< in: table */
char* filename, /*!< out: filename */
ulint max_len) /*!< in: filename max length */
{
ulint len;
char* path;
char* suffix;
static const ulint suffix_len = strlen(".cfg");
if (DICT_TF_HAS_DATA_DIR(table->flags)) {
dict_get_and_save_data_dir_path(table, false);
ut_a(table->data_dir_path);
path = os_file_make_remote_pathname(
table->data_dir_path, table->name, "cfg");
} else {
path = fil_make_ibd_name(table->name, false);
}
ut_a(path);
len = ut_strlen(path);
ut_a(max_len >= len);
suffix = path + (len - suffix_len);
if (strncmp(suffix, ".cfg", suffix_len) == 0) {
strcpy(filename, path);
} else {
ut_ad(strncmp(suffix, ".ibd", suffix_len) == 0);
strncpy(filename, path, len - suffix_len);
suffix = filename + (len - suffix_len);
strcpy(suffix, ".cfg");
}
mem_free(path);
srv_normalize_path_for_win(filename);
}