From 92f87f2cf0193309247c5e5b52ca99cfe76c6ed1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Mon, 12 Feb 2024 17:01:35 +0200 Subject: [PATCH] Cleanup: Remove changed_pages_bitmap The innodb_changed_pages plugin only was part of XtraDB, never InnoDB. It would be useful for incremental backups. We will remove the code from mariadb-backup for now, because it cannot serve any useful purpose until the server part has been implemented. --- extra/mariabackup/CMakeLists.txt | 1 - extra/mariabackup/backup_mysql.cc | 41 - extra/mariabackup/backup_mysql.h | 4 - extra/mariabackup/changed_page_bitmap.cc | 1043 ---------------------- extra/mariabackup/changed_page_bitmap.h | 85 -- extra/mariabackup/fil_cur.cc | 7 +- extra/mariabackup/read_filt.cc | 142 +-- extra/mariabackup/read_filt.h | 29 +- extra/mariabackup/xtrabackup.cc | 18 +- extra/mariabackup/xtrabackup.h | 4 +- mysql-test/suite/mariabackup/suite.opt | 2 +- 11 files changed, 16 insertions(+), 1360 deletions(-) delete mode 100644 extra/mariabackup/changed_page_bitmap.cc delete mode 100644 extra/mariabackup/changed_page_bitmap.h diff --git a/extra/mariabackup/CMakeLists.txt b/extra/mariabackup/CMakeLists.txt index 0ebfba54534..6a83ba0238a 100644 --- a/extra/mariabackup/CMakeLists.txt +++ b/extra/mariabackup/CMakeLists.txt @@ -57,7 +57,6 @@ ADD_DEFINITIONS(${SSL_DEFINES}) MYSQL_ADD_EXECUTABLE(mariadb-backup xtrabackup.cc innobackupex.cc - changed_page_bitmap.cc datasink.cc ds_buffer.cc ds_compress.cc diff --git a/extra/mariabackup/backup_mysql.cc b/extra/mariabackup/backup_mysql.cc index effa2dbabab..862f8b46a3a 100644 --- a/extra/mariabackup/backup_mysql.cc +++ b/extra/mariabackup/backup_mysql.cc @@ -73,7 +73,6 @@ mysql_flavor_t server_flavor = FLAVOR_UNKNOWN; unsigned long mysql_server_version = 0; /* server capabilities */ -bool have_changed_page_bitmaps = false; bool have_backup_locks = false; bool have_lock_wait_timeout = false; bool have_galera_enabled = false; @@ -606,34 +605,6 @@ Query the server to find out what backup capabilities it supports. bool detect_mysql_capabilities_for_backup() { - const char *query = "SELECT 'INNODB_CHANGED_PAGES', COUNT(*) FROM " - "INFORMATION_SCHEMA.PLUGINS " - "WHERE PLUGIN_NAME LIKE 'INNODB_CHANGED_PAGES'"; - char *innodb_changed_pages = NULL; - mysql_variable vars[] = { - {"INNODB_CHANGED_PAGES", &innodb_changed_pages}, {NULL, NULL}}; - - if (xtrabackup_incremental) { - - read_mysql_variables(mysql_connection, query, vars, true); - - ut_ad(innodb_changed_pages != NULL); - - have_changed_page_bitmaps = (atoi(innodb_changed_pages) == 1); - - /* INNODB_CHANGED_PAGES are listed in - INFORMATION_SCHEMA.PLUGINS in MariaDB, but - FLUSH NO_WRITE_TO_BINLOG CHANGED_PAGE_BITMAPS - is not supported for versions below 10.1.6 - (see MDEV-7472) */ - if (server_flavor == FLAVOR_MARIADB && - mysql_server_version < 100106) { - have_changed_page_bitmaps = false; - } - - free_mysql_variables(vars); - } - /* do some sanity checks */ if (opt_galera_info && !have_galera_enabled) { msg("--galera-info is specified on the command " @@ -1997,18 +1968,6 @@ select_history() return(true); } -bool -flush_changed_page_bitmaps() -{ - if (xtrabackup_incremental && have_changed_page_bitmaps && - !xtrabackup_incremental_force_scan) { - xb_mysql_query(mysql_connection, - "FLUSH NO_WRITE_TO_BINLOG CHANGED_PAGE_BITMAPS", false); - } - return(true); -} - - /*********************************************************************//** Deallocate memory, disconnect from server, etc. @return true on success. */ diff --git a/extra/mariabackup/backup_mysql.h b/extra/mariabackup/backup_mysql.h index d80f3bb7bc1..5614070173d 100644 --- a/extra/mariabackup/backup_mysql.h +++ b/extra/mariabackup/backup_mysql.h @@ -10,7 +10,6 @@ extern mysql_flavor_t server_flavor; extern unsigned long mysql_server_version; /* server capabilities */ -extern bool have_changed_page_bitmaps; extern bool have_backup_locks; extern bool have_lock_wait_timeout; extern bool have_galera_enabled; @@ -39,9 +38,6 @@ capture_tool_command(int argc, char **argv); bool select_history(); -bool -flush_changed_page_bitmaps(); - void backup_cleanup(); diff --git a/extra/mariabackup/changed_page_bitmap.cc b/extra/mariabackup/changed_page_bitmap.cc deleted file mode 100644 index 793d7378b0f..00000000000 --- a/extra/mariabackup/changed_page_bitmap.cc +++ /dev/null @@ -1,1043 +0,0 @@ -/****************************************************** -XtraBackup: hot backup tool for InnoDB -(c) 2009-2012 Percona Inc. -Originally Created 3/3/2009 Yasufumi Kinoshita -Written by Alexey Kopytov, Aleksandr Kuzminsky, Stewart Smith, Vadim Tkachenko, -Yasufumi Kinoshita, Ignacio Nin and Baron Schwartz. - -This program is free software; you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation; version 2 of the License. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program; if not, write to the Free Software -Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA - -*******************************************************/ - -/* Changed page bitmap implementation */ - -#include "changed_page_bitmap.h" - -#include "common.h" -#include "xtrabackup.h" -#include "srv0srv.h" - -/* TODO: copy-pasted shared definitions from the XtraDB bitmap write code. -Remove these on the first opportunity, i.e. single-binary XtraBackup. */ - -/* log0online.h */ - -/** Single bitmap file information */ -struct log_online_bitmap_file_t { - char name[FN_REFLEN]; /*!< Name with full path */ - pfs_os_file_t file; /*!< Handle to opened file */ - ib_uint64_t size; /*!< Size of the file */ - ib_uint64_t offset; /*!< Offset of the next read, - or count of already-read bytes - */ -}; - -/** A set of bitmap files containing some LSN range */ -struct log_online_bitmap_file_range_t { - size_t count; /*!< Number of files */ - /*!< Dynamically-allocated array of info about individual files */ - struct files_t { - char name[FN_REFLEN];/*!< Name of a file */ - lsn_t start_lsn; /*!< Starting LSN of data in this - file */ - ulong seq_num; /*!< Sequence number of this file */ - } *files; -}; - -/* log0online.c */ - -/** File name stem for bitmap files. */ -static const char* bmp_file_name_stem = "ib_modified_log_"; - -/** The bitmap file block size in bytes. All writes will be multiples of this. - */ -enum { - MODIFIED_PAGE_BLOCK_SIZE = 4096 -}; - -/** Offsets in a file bitmap block */ -enum { - MODIFIED_PAGE_IS_LAST_BLOCK = 0,/* 1 if last block in the current - write, 0 otherwise. */ - MODIFIED_PAGE_START_LSN = 4, /* The starting tracked LSN of this and - other blocks in the same write */ - MODIFIED_PAGE_END_LSN = 12, /* The ending tracked LSN of this and - other blocks in the same write */ - MODIFIED_PAGE_SPACE_ID = 20, /* The space ID of tracked pages in - this block */ - MODIFIED_PAGE_1ST_PAGE_ID = 24, /* The page ID of the first tracked - page in this block */ - MODIFIED_PAGE_BLOCK_UNUSED_1 = 28,/* Unused in order to align the start - of bitmap at 8 byte boundary */ - MODIFIED_PAGE_BLOCK_BITMAP = 32,/* Start of the bitmap itself */ - MODIFIED_PAGE_BLOCK_UNUSED_2 = MODIFIED_PAGE_BLOCK_SIZE - 8, - /* Unused in order to align the end of - bitmap at 8 byte boundary */ - MODIFIED_PAGE_BLOCK_CHECKSUM = MODIFIED_PAGE_BLOCK_SIZE - 4 - /* The checksum of the current block */ -}; - -/** Length of the bitmap data in a block */ -enum { MODIFIED_PAGE_BLOCK_BITMAP_LEN - = MODIFIED_PAGE_BLOCK_UNUSED_2 - MODIFIED_PAGE_BLOCK_BITMAP }; - -/** Length of the bitmap data in a block in page ids */ -enum { MODIFIED_PAGE_BLOCK_ID_COUNT = MODIFIED_PAGE_BLOCK_BITMAP_LEN * 8 }; - -typedef ib_uint64_t bitmap_word_t; - -/****************************************************************//** -Calculate a bitmap block checksum. Algorithm borrowed from -log_block_calc_checksum. -@return checksum */ -UNIV_INLINE -ulint -log_online_calc_checksum( -/*=====================*/ - const byte* block); /*! p2 -*/ -static -int -log_online_compare_bmp_keys( -/*========================*/ - const void* p1, /*! k2_start_page ? 1 : 0; - } - return k1_space < k2_space ? -1 : 1; -} - -/****************************************************************//** -Calculate a bitmap block checksum. Algorithm borrowed from -log_block_calc_checksum. -@return checksum */ -UNIV_INLINE -ulint -log_online_calc_checksum( -/*=====================*/ - const byte* block) /*! 24) { - - sh = 0; - } - } - - return sum; -} - -/****************************************************************//** -Read one bitmap data page and check it for corruption. - -@return TRUE if page read OK, FALSE if I/O error */ -static -ibool -log_online_read_bitmap_page( -/*========================*/ - log_online_bitmap_file_t *bitmap_file, /*!size >= MODIFIED_PAGE_BLOCK_SIZE); - ut_a(bitmap_file->offset - <= bitmap_file->size - MODIFIED_PAGE_BLOCK_SIZE); - ut_a(bitmap_file->offset % MODIFIED_PAGE_BLOCK_SIZE == 0); - success = os_file_read(IORequestRead, - bitmap_file->file, page, bitmap_file->offset, - MODIFIED_PAGE_BLOCK_SIZE) == DB_SUCCESS; - - if (UNIV_UNLIKELY(!success)) { - - /* The following call prints an error message */ - os_file_get_last_error(TRUE); - msg("InnoDB: Warning: failed reading changed page bitmap " - "file \'%s\'", bitmap_file->name); - return FALSE; - } - - bitmap_file->offset += MODIFIED_PAGE_BLOCK_SIZE; - ut_ad(bitmap_file->offset <= bitmap_file->size); - - checksum = mach_read_from_4(page + MODIFIED_PAGE_BLOCK_CHECKSUM); - actual_checksum = log_online_calc_checksum(page); - *checksum_ok = (checksum == actual_checksum); - - return TRUE; -} - -/*********************************************************************//** -Check the name of a given file if it's a changed page bitmap file and -return file sequence and start LSN name components if it is. If is not, -the values of output parameters are undefined. - -@return TRUE if a given file is a changed page bitmap file. */ -static -ibool -log_online_is_bitmap_file( -/*======================*/ - const os_file_stat_t* file_info, /*!name) < OS_FILE_MAX_PATH); - - return ((file_info->type == OS_FILE_TYPE_FILE - || file_info->type == OS_FILE_TYPE_LINK) - && (sscanf(file_info->name, "%[a-z_]%lu_" LSN_PF ".xdb", stem, - bitmap_file_seq_num, bitmap_file_start_lsn) == 3) - && (!strcmp(stem, bmp_file_name_stem))); -} - -/*********************************************************************//** -List the bitmap files in srv_data_home and setup their range that contains the -specified LSN interval. This range, if non-empty, will start with a file that -has the greatest LSN equal to or less than the start LSN and will include all -the files up to the one with the greatest LSN less than the end LSN. Caller -must free bitmap_files->files when done if bitmap_files set to non-NULL and -this function returned TRUE. Field bitmap_files->count might be set to a -larger value than the actual count of the files, and space for the unused array -slots will be allocated but cleared to zeroes. - -@return TRUE if succeeded -*/ -static -ibool -log_online_setup_bitmap_file_range( -/*===============================*/ - log_online_bitmap_file_range_t *bitmap_files, /*!= range_start); - - bitmap_files->count = 0; - bitmap_files->files = NULL; - - /* 1st pass: size the info array */ - - bitmap_dir = os_file_opendir(srv_data_home); - if (UNIV_UNLIKELY(bitmap_dir == IF_WIN(INVALID_HANDLE_VALUE, NULL))) { - msg("InnoDB: Error: failed to open bitmap directory \'%s\'", - srv_data_home); - return FALSE; - } - - while (!os_file_readdir_next_file(srv_data_home, bitmap_dir, - &bitmap_dir_file_info)) { - - ulong file_seq_num; - lsn_t file_start_lsn; - - if (!log_online_is_bitmap_file(&bitmap_dir_file_info, - &file_seq_num, - &file_start_lsn) - || file_start_lsn >= range_end) { - - continue; - } - - if (file_seq_num > last_file_seq_num) { - - last_file_seq_num = file_seq_num; - } - - if (file_start_lsn >= range_start - || file_start_lsn == first_file_start_lsn - || first_file_start_lsn > range_start) { - - /* A file that falls into the range */ - - if (file_start_lsn < first_file_start_lsn) { - - first_file_start_lsn = file_start_lsn; - } - if (file_seq_num < first_file_seq_num) { - - first_file_seq_num = file_seq_num; - } - } else if (file_start_lsn > first_file_start_lsn) { - - /* A file that has LSN closer to the range start - but smaller than it, replacing another such file */ - first_file_start_lsn = file_start_lsn; - first_file_seq_num = file_seq_num; - } - } - - if (UNIV_UNLIKELY(os_file_closedir_failed(bitmap_dir))) { - os_file_get_last_error(TRUE); - msg("InnoDB: Error: cannot close \'%s\'",srv_data_home); - return FALSE; - } - - if (first_file_seq_num == ULONG_MAX && last_file_seq_num == 0) { - - bitmap_files->count = 0; - return TRUE; - } - - bitmap_files->count = last_file_seq_num - first_file_seq_num + 1; - - /* 2nd pass: get the file names in the file_seq_num order */ - - bitmap_dir = os_file_opendir(srv_data_home); - if (UNIV_UNLIKELY(bitmap_dir == IF_WIN(INVALID_HANDLE_VALUE, NULL))) { - msg("InnoDB: Error: failed to open bitmap directory \'%s\'", - srv_data_home); - return FALSE; - } - - bitmap_files->files = - static_cast - (malloc(bitmap_files->count * sizeof(bitmap_files->files[0]))); - memset(bitmap_files->files, 0, - bitmap_files->count * sizeof(bitmap_files->files[0])); - - while (!os_file_readdir_next_file(srv_data_home, bitmap_dir, - &bitmap_dir_file_info)) { - - ulong file_seq_num; - lsn_t file_start_lsn; - size_t array_pos; - - if (!log_online_is_bitmap_file(&bitmap_dir_file_info, - &file_seq_num, - &file_start_lsn) - || file_start_lsn >= range_end - || file_start_lsn < first_file_start_lsn) { - - continue; - } - - array_pos = file_seq_num - first_file_seq_num; - if (UNIV_UNLIKELY(array_pos >= bitmap_files->count)) { - - msg("InnoDB: Error: inconsistent bitmap file " - "directory"); - os_file_closedir(bitmap_dir); - free(bitmap_files->files); - return FALSE; - } - - if (file_seq_num > bitmap_files->files[array_pos].seq_num) { - - bitmap_files->files[array_pos].seq_num = file_seq_num; - strncpy(bitmap_files->files[array_pos].name, - bitmap_dir_file_info.name, FN_REFLEN - 1); - bitmap_files->files[array_pos].name[FN_REFLEN - 1] - = '\0'; - bitmap_files->files[array_pos].start_lsn - = file_start_lsn; - } - } - - if (UNIV_UNLIKELY(os_file_closedir_failed(bitmap_dir))) { - os_file_get_last_error(TRUE); - msg("InnoDB: Error: cannot close \'%s\'", srv_data_home); - free(bitmap_files->files); - return FALSE; - } - -#ifdef UNIV_DEBUG - ut_ad(bitmap_files->files[0].seq_num == first_file_seq_num); - - for (size_t i = 1; i < bitmap_files->count; i++) { - if (!bitmap_files->files[i].seq_num) { - - break; - } - ut_ad(bitmap_files->files[i].seq_num - > bitmap_files->files[i - 1].seq_num); - ut_ad(bitmap_files->files[i].start_lsn - >= bitmap_files->files[i - 1].start_lsn); - } -#endif - - return TRUE; -} - -/****************************************************************//** -Open a bitmap file for reading. - -@return whether opened successfully */ -static -bool -log_online_open_bitmap_file_read_only( -/*==================================*/ - const char* name, /*!name, FN_REFLEN, "%s%s", srv_data_home, name); - bitmap_file->file = os_file_create_simple_no_error_handling( - 0, bitmap_file->name, - OS_FILE_OPEN, OS_FILE_READ_ONLY, true, &success); - if (UNIV_UNLIKELY(!success)) { - - /* Here and below assume that bitmap file names do not - contain apostrophes, thus no need for ut_print_filename(). */ - msg("InnoDB: Warning: error opening the changed page " - "bitmap \'%s\'", bitmap_file->name); - return success; - } - - bitmap_file->size = os_file_get_size(bitmap_file->file); - bitmap_file->offset = 0; - -#ifdef __linux__ - posix_fadvise(bitmap_file->file, 0, 0, POSIX_FADV_SEQUENTIAL); - posix_fadvise(bitmap_file->file, 0, 0, POSIX_FADV_NOREUSE); -#endif - - return success; -} - -/****************************************************************//** -Diagnose one or both of the following situations if we read close to -the end of bitmap file: -1) Warn if the remainder of the file is less than one page. -2) Error if we cannot read any more full pages but the last read page -did not have the last-in-run flag set. - -@return FALSE for the error */ -static -ibool -log_online_diagnose_bitmap_eof( -/*===========================*/ - const log_online_bitmap_file_t* bitmap_file, /*!< in: bitmap file */ - ibool last_page_in_run)/*!< in: "last page in - run" flag value in the - last read page */ -{ - /* Check if we are too close to EOF to read a full page */ - if ((bitmap_file->size < MODIFIED_PAGE_BLOCK_SIZE) - || (bitmap_file->offset - > bitmap_file->size - MODIFIED_PAGE_BLOCK_SIZE)) { - - if (UNIV_UNLIKELY(bitmap_file->offset != bitmap_file->size)) { - - /* If we are not at EOF and we have less than one page - to read, it's junk. This error is not fatal in - itself. */ - - msg("InnoDB: Warning: junk at the end of changed " - "page bitmap file \'%s\'.", bitmap_file->name); - } - - if (UNIV_UNLIKELY(!last_page_in_run)) { - - /* We are at EOF but the last read page did not finish - a run */ - /* It's a "Warning" here because it's not a fatal error - for the whole server */ - msg("InnoDB: Warning: changed page bitmap " - "file \'%s\' does not contain a complete run " - "at the end.", bitmap_file->name); - return FALSE; - } - } - return TRUE; -} - -/* End of copy-pasted definitions */ - -/** Iterator structure over changed page bitmap */ -struct xb_page_bitmap_range_struct { - const xb_page_bitmap *bitmap; /* Bitmap with data */ - ulint space_id; /* Space id for this - iterator */ - ulint bit_i; /* Bit index of the iterator - position in the current page */ - const ib_rbt_node_t *bitmap_node; /* Current bitmap tree node */ - const byte *bitmap_page; /* Current bitmap page */ - ulint current_page_id;/* Current page id */ -}; - -/****************************************************************//** -Print a diagnostic message on missing bitmap data for an LSN range. */ -static -void -xb_msg_missing_lsn_data( -/*====================*/ - lsn_t missing_interval_start, /*!size >= MODIFIED_PAGE_BLOCK_SIZE); - - *page_end_lsn = 0; - - while ((*page_end_lsn <= lsn) - && (bitmap_file->offset - <= bitmap_file->size - MODIFIED_PAGE_BLOCK_SIZE)) { - - next_to_last_page_ok = last_page_ok; - if (!log_online_read_bitmap_page(bitmap_file, page, - &last_page_ok)) { - - return FALSE; - } - - *page_end_lsn = mach_read_from_8(page + MODIFIED_PAGE_END_LSN); - } - - /* We check two pages here because the last read page already contains - the required LSN data. If the next to the last one page is corrupted, - then we have no way of telling if that page contained the required LSN - range data too */ - return last_page_ok && next_to_last_page_ok; -} - -/****************************************************************//** -Read the disk bitmap and build the changed page bitmap tree for the -LSN interval incremental_lsn to checkpoint_lsn_start. - -@return the built bitmap tree or NULL if unable to read the full interval for -any reason. */ -xb_page_bitmap* -xb_page_bitmap_init(void) -/*=====================*/ -{ - log_online_bitmap_file_t bitmap_file; - lsn_t bmp_start_lsn = incremental_lsn; - lsn_t bmp_end_lsn = checkpoint_lsn_start; - byte page[MODIFIED_PAGE_BLOCK_SIZE]; - lsn_t current_page_end_lsn; - xb_page_bitmap *result; - ibool last_page_in_run= FALSE; - log_online_bitmap_file_range_t bitmap_files; - size_t bmp_i; - ibool last_page_ok = TRUE; - - if (UNIV_UNLIKELY(bmp_start_lsn > bmp_end_lsn)) { - - msg("mariabackup: incremental backup LSN " LSN_PF - " is larger than than the last checkpoint LSN " LSN_PF - , bmp_start_lsn, bmp_end_lsn); - return NULL; - } - - if (!log_online_setup_bitmap_file_range(&bitmap_files, bmp_start_lsn, - bmp_end_lsn)) { - - return NULL; - } - - /* Only accept no bitmap files returned if start LSN == end LSN */ - if (bitmap_files.count == 0 && bmp_end_lsn != bmp_start_lsn) { - - return NULL; - } - - result = rbt_create(MODIFIED_PAGE_BLOCK_SIZE, - log_online_compare_bmp_keys); - - if (bmp_start_lsn == bmp_end_lsn) { - - /* Empty range - empty bitmap */ - return result; - } - - bmp_i = 0; - - if (UNIV_UNLIKELY(bitmap_files.files[bmp_i].start_lsn - > bmp_start_lsn)) { - - /* The 1st file does not have the starting LSN data */ - xb_msg_missing_lsn_data(bmp_start_lsn, - bitmap_files.files[bmp_i].start_lsn); - rbt_free(result); - free(bitmap_files.files); - return NULL; - } - - /* Skip any zero-sized files at the start */ - while ((bmp_i < bitmap_files.count - 1) - && (bitmap_files.files[bmp_i].start_lsn - == bitmap_files.files[bmp_i + 1].start_lsn)) { - - bmp_i++; - } - - /* Is the 1st bitmap file missing? */ - if (UNIV_UNLIKELY(bitmap_files.files[bmp_i].name[0] == '\0')) { - - /* TODO: this is not the exact missing range */ - xb_msg_missing_lsn_data(bmp_start_lsn, bmp_end_lsn); - rbt_free(result); - free(bitmap_files.files); - return NULL; - } - - /* Open the 1st bitmap file */ - if (UNIV_UNLIKELY(!log_online_open_bitmap_file_read_only( - bitmap_files.files[bmp_i].name, - &bitmap_file))) { - - rbt_free(result); - free(bitmap_files.files); - return NULL; - } - - /* If the 1st file is truncated, no data. Not merged with the case - below because zero-length file indicates not a corruption but missing - subsequent files instead. */ - if (UNIV_UNLIKELY(bitmap_file.size < MODIFIED_PAGE_BLOCK_SIZE)) { - - xb_msg_missing_lsn_data(bmp_start_lsn, bmp_end_lsn); - rbt_free(result); - free(bitmap_files.files); - os_file_close(bitmap_file.file); - return NULL; - } - - /* Find the start of the required LSN range in the file */ - if (UNIV_UNLIKELY(!xb_find_lsn_in_bitmap_file(&bitmap_file, page, - ¤t_page_end_lsn, - bmp_start_lsn))) { - - msg("mariabackup: Warning: changed page bitmap file " - "\'%s\' corrupted", bitmap_file.name); - rbt_free(result); - free(bitmap_files.files); - os_file_close(bitmap_file.file); - return NULL; - } - - last_page_in_run - = mach_read_from_4(page + MODIFIED_PAGE_IS_LAST_BLOCK); - - if (UNIV_UNLIKELY(!log_online_diagnose_bitmap_eof(&bitmap_file, - last_page_in_run))) { - - rbt_free(result); - free(bitmap_files.files); - os_file_close(bitmap_file.file); - return NULL; - } - - if (UNIV_UNLIKELY(current_page_end_lsn < bmp_start_lsn)) { - - xb_msg_missing_lsn_data(current_page_end_lsn, bmp_start_lsn); - rbt_free(result); - free(bitmap_files.files); - os_file_close(bitmap_file.file); - return NULL; - } - - /* 1st bitmap page found, add it to the tree. */ - rbt_insert(result, page, page); - - /* Read next pages/files until all required data is read */ - while (last_page_ok - && (current_page_end_lsn < bmp_end_lsn - || (current_page_end_lsn == bmp_end_lsn - && !last_page_in_run))) { - - ib_rbt_bound_t tree_search_pos; - - /* If EOF, advance the file skipping over any empty files */ - while (bitmap_file.size < MODIFIED_PAGE_BLOCK_SIZE - || (bitmap_file.offset - > bitmap_file.size - MODIFIED_PAGE_BLOCK_SIZE)) { - - os_file_close(bitmap_file.file); - - if (UNIV_UNLIKELY( - !log_online_diagnose_bitmap_eof( - &bitmap_file, last_page_in_run))) { - - rbt_free(result); - free(bitmap_files.files); - return NULL; - } - - bmp_i++; - - if (UNIV_UNLIKELY(bmp_i == bitmap_files.count - || (bitmap_files.files[bmp_i].seq_num - == 0))) { - - xb_msg_missing_lsn_data(current_page_end_lsn, - bmp_end_lsn); - rbt_free(result); - free(bitmap_files.files); - return NULL; - } - - /* Is the next file missing? */ - if (UNIV_UNLIKELY(bitmap_files.files[bmp_i].name[0] - == '\0')) { - - /* TODO: this is not the exact missing range */ - xb_msg_missing_lsn_data(bitmap_files.files - [bmp_i - 1].start_lsn, - bmp_end_lsn); - rbt_free(result); - free(bitmap_files.files); - return NULL; - } - - if (UNIV_UNLIKELY( - !log_online_open_bitmap_file_read_only( - bitmap_files.files[bmp_i].name, - &bitmap_file))) { - - rbt_free(result); - free(bitmap_files.files); - return NULL; - } - } - - if (UNIV_UNLIKELY( - !log_online_read_bitmap_page(&bitmap_file, page, - &last_page_ok))) { - - rbt_free(result); - free(bitmap_files.files); - os_file_close(bitmap_file.file); - return NULL; - } - - if (UNIV_UNLIKELY(!last_page_ok)) { - - msg("mariabackup: warning: changed page bitmap file " - "\'%s\' corrupted.", bitmap_file.name); - rbt_free(result); - free(bitmap_files.files); - os_file_close(bitmap_file.file); - return NULL; - } - - /* Merge the current page with an existing page or insert a new - page into the tree */ - - if (!rbt_search(result, &tree_search_pos, page)) { - - /* Merge the bitmap pages */ - byte *existing_page - = rbt_value(byte, tree_search_pos.last); - bitmap_word_t *bmp_word_1 = (bitmap_word_t *) - (existing_page + MODIFIED_PAGE_BLOCK_BITMAP); - bitmap_word_t *bmp_end = (bitmap_word_t *) - (existing_page + MODIFIED_PAGE_BLOCK_UNUSED_2); - bitmap_word_t *bmp_word_2 = (bitmap_word_t *) - (page + MODIFIED_PAGE_BLOCK_BITMAP); - while (bmp_word_1 < bmp_end) { - - *bmp_word_1++ |= *bmp_word_2++; - } - xb_a (bmp_word_1 == bmp_end); - } else { - - /* Add a new page */ - rbt_add_node(result, &tree_search_pos, page); - } - - current_page_end_lsn - = mach_read_from_8(page + MODIFIED_PAGE_END_LSN); - last_page_in_run - = mach_read_from_4(page + MODIFIED_PAGE_IS_LAST_BLOCK); - } - - xb_a (current_page_end_lsn >= bmp_end_lsn); - - free(bitmap_files.files); - os_file_close(bitmap_file.file); - - return result; -} - -/****************************************************************//** -Free the bitmap tree. */ -void -xb_page_bitmap_deinit( -/*==================*/ - xb_page_bitmap* bitmap) /*!bitmap_page has been -already found/bumped by rbt_search()/rbt_next(). - -@return FALSE if no more bitmap data for the range space ID */ -static -ibool -xb_page_bitmap_setup_next_page( -/*===========================*/ - xb_page_bitmap_range* bitmap_range) /*!bitmap_node == NULL) { - - bitmap_range->current_page_id = ULINT_UNDEFINED; - return FALSE; - } - - bitmap_range->bitmap_page = rbt_value(byte, bitmap_range->bitmap_node); - - new_space_id = mach_read_from_4(bitmap_range->bitmap_page - + MODIFIED_PAGE_SPACE_ID); - if (new_space_id != bitmap_range->space_id) { - - /* No more data for the current page id. */ - xb_a(new_space_id > bitmap_range->space_id); - bitmap_range->current_page_id = ULINT_UNDEFINED; - return FALSE; - } - - new_1st_page_id = mach_read_from_4(bitmap_range->bitmap_page + - MODIFIED_PAGE_1ST_PAGE_ID); - xb_a (new_1st_page_id >= bitmap_range->current_page_id - || bitmap_range->current_page_id == ULINT_UNDEFINED); - - bitmap_range->current_page_id = new_1st_page_id; - bitmap_range->bit_i = 0; - - return TRUE; -} - -/** Find the node with the smallest key that greater than equal to search key. -@param[in] tree red-black tree -@param[in] key search key -@return node with the smallest greater-than-or-equal key -@retval NULL if none was found */ -static -const ib_rbt_node_t* -rbt_lower_bound(const ib_rbt_t* tree, const void* key) -{ - ut_ad(!tree->cmp_arg); - const ib_rbt_node_t* ge = NULL; - - for (const ib_rbt_node_t *node = tree->root->left; - node != tree->nil; ) { - int result = tree->compare(node->value, key); - - if (result < 0) { - node = node->right; - } else { - ge = node; - if (result == 0) { - break; - } - - node = node->left; - } - } - - return(ge); -} - -/****************************************************************//** -Set up a new bitmap range iterator over a given space id changed -pages in a given bitmap. - -@return bitmap range iterator */ -xb_page_bitmap_range* -xb_page_bitmap_range_init( -/*======================*/ - xb_page_bitmap* bitmap, /*!< in: bitmap to iterate over */ - ulint space_id) /*!< in: space id */ -{ - byte search_page[MODIFIED_PAGE_BLOCK_SIZE]; - xb_page_bitmap_range *result - = static_cast(malloc(sizeof(*result))); - - memset(result, 0, sizeof(*result)); - result->bitmap = bitmap; - result->space_id = space_id; - result->current_page_id = ULINT_UNDEFINED; - - /* Search for the 1st page for the given space id */ - /* This also sets MODIFIED_PAGE_1ST_PAGE_ID to 0, which is what we - want. */ - memset(search_page, 0, MODIFIED_PAGE_BLOCK_SIZE); - mach_write_to_4(search_page + MODIFIED_PAGE_SPACE_ID, space_id); - - result->bitmap_node = rbt_lower_bound(result->bitmap, search_page); - - xb_page_bitmap_setup_next_page(result); - - return result; -} - -/****************************************************************//** -Get the value of the bitmap->range->bit_i bitmap bit - -@return the current bit value */ -static inline -ibool -is_bit_set( -/*=======*/ - const xb_page_bitmap_range* bitmap_range) /*!< in: bitmap - range */ -{ - return ((*(((bitmap_word_t *)(bitmap_range->bitmap_page - + MODIFIED_PAGE_BLOCK_BITMAP)) - + (bitmap_range->bit_i >> 6))) - & (1ULL << (bitmap_range->bit_i & 0x3F))) ? TRUE : FALSE; -} - -/****************************************************************//** -Get the next page id that has its bit set or cleared, i.e. equal to -bit_value. - -@return page id */ -ulint -xb_page_bitmap_range_get_next_bit( -/*==============================*/ - xb_page_bitmap_range* bitmap_range, /*!< in/out: bitmap range */ - ibool bit_value) /*!< in: bit value */ -{ - if (UNIV_UNLIKELY(bitmap_range->current_page_id - == ULINT_UNDEFINED)) { - - return ULINT_UNDEFINED; - } - - do { - while (bitmap_range->bit_i < MODIFIED_PAGE_BLOCK_ID_COUNT) { - - while (is_bit_set(bitmap_range) != bit_value - && (bitmap_range->bit_i - < MODIFIED_PAGE_BLOCK_ID_COUNT)) { - - bitmap_range->current_page_id++; - bitmap_range->bit_i++; - } - - if (bitmap_range->bit_i - < MODIFIED_PAGE_BLOCK_ID_COUNT) { - - ulint result = bitmap_range->current_page_id; - bitmap_range->current_page_id++; - bitmap_range->bit_i++; - return result; - } - } - - bitmap_range->bitmap_node - = rbt_next(bitmap_range->bitmap, - bitmap_range->bitmap_node); - - } while (xb_page_bitmap_setup_next_page(bitmap_range)); - - return ULINT_UNDEFINED; -} - -/****************************************************************//** -Free the bitmap range iterator. */ -void -xb_page_bitmap_range_deinit( -/*========================*/ - xb_page_bitmap_range* bitmap_range) /*! in/out: bitmap range */ -{ - free(bitmap_range); -} diff --git a/extra/mariabackup/changed_page_bitmap.h b/extra/mariabackup/changed_page_bitmap.h deleted file mode 100644 index 1a0e2ec37f0..00000000000 --- a/extra/mariabackup/changed_page_bitmap.h +++ /dev/null @@ -1,85 +0,0 @@ -/****************************************************** -XtraBackup: hot backup tool for InnoDB -(c) 2009-2012 Percona Inc. -Originally Created 3/3/2009 Yasufumi Kinoshita -Written by Alexey Kopytov, Aleksandr Kuzminsky, Stewart Smith, Vadim Tkachenko, -Yasufumi Kinoshita, Ignacio Nin and Baron Schwartz. - -This program is free software; you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation; version 2 of the License. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program; if not, write to the Free Software -Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA - -*******************************************************/ - -/* Changed page bitmap interface */ - -#ifndef XB_CHANGED_PAGE_BITMAP_H -#define XB_CHANGED_PAGE_BITMAP_H - -#include -#include - -/* The changed page bitmap structure */ -typedef ib_rbt_t xb_page_bitmap; - -struct xb_page_bitmap_range_struct; - -/* The bitmap range iterator over one space id */ -typedef struct xb_page_bitmap_range_struct xb_page_bitmap_range; - -/****************************************************************//** -Read the disk bitmap and build the changed page bitmap tree for the -LSN interval incremental_lsn to checkpoint_lsn_start. - -@return the built bitmap tree */ -xb_page_bitmap* -xb_page_bitmap_init(void); -/*=====================*/ - -/****************************************************************//** -Free the bitmap tree. */ -void -xb_page_bitmap_deinit( -/*==================*/ - xb_page_bitmap* bitmap); /*!page_size); cursor->read_filter = read_filter; - cursor->read_filter->init(&cursor->read_filter_ctxt, cursor, - node->space->id); + cursor->read_filter->init(&cursor->read_filter_ctxt, cursor); return(XB_FIL_CUR_SUCCESS); } @@ -503,10 +502,6 @@ xb_fil_cur_close( /*=============*/ xb_fil_cur_t *cursor) /*!< in/out: source file cursor */ { - if (cursor->read_filter) { - cursor->read_filter->deinit(&cursor->read_filter_ctxt); - } - aligned_free(cursor->buf); cursor->buf = NULL; diff --git a/extra/mariabackup/read_filt.cc b/extra/mariabackup/read_filt.cc index 589200552a2..c7c0aa555a4 100644 --- a/extra/mariabackup/read_filt.cc +++ b/extra/mariabackup/read_filt.cc @@ -32,29 +32,13 @@ Perform read filter context initialization that is common to all read filters. */ static void -common_init( -/*========*/ +rf_pass_through_init( xb_read_filt_ctxt_t* ctxt, /*!offset = 0; ctxt->data_file_size = cursor->statinfo.st_size; ctxt->buffer_capacity = cursor->buf_size; - ctxt->page_size = cursor->page_size; -} - -/****************************************************************//** -Initialize the pass-through read filter. */ -static -void -rf_pass_through_init( -/*=================*/ - xb_read_filt_ctxt_t* ctxt, /*!offset; *read_batch_len = ctxt->data_file_size - ctxt->offset; - if (*read_batch_len > (ib_int64_t)ctxt->buffer_capacity) { + if (*read_batch_len > (int64_t)ctxt->buffer_capacity) { *read_batch_len = ctxt->buffer_capacity; } ctxt->offset += *read_batch_len; } -/****************************************************************//** -Deinitialize the pass-through read filter. */ -static -void -rf_pass_through_deinit( -/*===================*/ - xb_read_filt_ctxt_t* ctxt __attribute__((unused))) - /*!bitmap_range = xb_page_bitmap_range_init(changed_page_bitmap, - space_id); - ctxt->filter_batch_end = 0; -} - -/****************************************************************//** -Get the next batch of pages for the bitmap read filter. */ -static -void -rf_bitmap_get_next_batch( -/*=====================*/ - xb_read_filt_ctxt_t* ctxt, /*!page_size; - - start_page_id = (ulint)(ctxt->offset / page_size); - - xb_a (ctxt->offset % page_size == 0); - - if (start_page_id == ctxt->filter_batch_end) { - - /* Used up all the previous bitmap range, get some more */ - ulint next_page_id; - - /* Find the next changed page using the bitmap */ - next_page_id = xb_page_bitmap_range_get_next_bit - (ctxt->bitmap_range, TRUE); - - if (next_page_id == ULINT_UNDEFINED) { - *read_batch_len = 0; - return; - } - - ctxt->offset = next_page_id * page_size; - - /* Find the end of the current changed page block by searching - for the next cleared bitmap bit */ - ctxt->filter_batch_end - = xb_page_bitmap_range_get_next_bit(ctxt->bitmap_range, - FALSE); - xb_a(next_page_id < ctxt->filter_batch_end); - } - - *read_batch_start = ctxt->offset; - if (ctxt->filter_batch_end == ULINT_UNDEFINED) { - /* No more cleared bits in the bitmap, need to copy all the - remaining pages. */ - *read_batch_len = ctxt->data_file_size - ctxt->offset; - } else { - *read_batch_len = ctxt->filter_batch_end * page_size - - ctxt->offset; - } - - /* If the page block is larger than the buffer capacity, limit it to - buffer capacity. The subsequent invocations will continue returning - the current block in buffer-sized pieces until ctxt->filter_batch_end - is reached, trigerring the next bitmap query. */ - if (*read_batch_len > (ib_int64_t)ctxt->buffer_capacity) { - *read_batch_len = ctxt->buffer_capacity; - } - - ctxt->offset += *read_batch_len; - xb_a (ctxt->offset % page_size == 0); - xb_a (*read_batch_start % page_size == 0); - xb_a (*read_batch_len % page_size == 0); -} - -/****************************************************************//** -Deinitialize the changed page bitmap-based read filter. */ -static -void -rf_bitmap_deinit( -/*=============*/ - xb_read_filt_ctxt_t* ctxt) /*!bitmap_range); -} - /* The pass-through read filter */ xb_read_filt_t rf_pass_through = { &rf_pass_through_init, &rf_pass_through_get_next_batch, - &rf_pass_through_deinit -}; - -/* The changed page bitmap-based read filter */ -xb_read_filt_t rf_bitmap = { - &rf_bitmap_init, - &rf_bitmap_get_next_batch, - &rf_bitmap_deinit }; diff --git a/extra/mariabackup/read_filt.h b/extra/mariabackup/read_filt.h index 2cd52d3d581..caf8ac566e0 100644 --- a/extra/mariabackup/read_filt.h +++ b/extra/mariabackup/read_filt.h @@ -25,42 +25,27 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA #ifndef XB_READ_FILT_H #define XB_READ_FILT_H -#include "changed_page_bitmap.h" - -typedef ulint space_id_t; +#include +#include struct xb_fil_cur_t; /* The read filter context */ struct xb_read_filt_ctxt_t { - ib_int64_t offset; /*!< current file offset */ - ib_int64_t data_file_size; /*!< data file size */ + int64_t offset; /*!< current file offset */ + int64_t data_file_size; /*!< data file size */ size_t buffer_capacity;/*!< read buffer capacity */ - space_id_t space_id; /*!< space id */ - /* The following fields used only in bitmap filter */ - /* Move these to union if any other filters are added in future */ - xb_page_bitmap_range *bitmap_range; /*!< changed page bitmap range - iterator for space_id */ - ulint page_size; /*!< page size */ - ulint filter_batch_end;/*!< the ending page id of the - current changed page block in - the bitmap */ - /** TODO: remove this default constructor */ - xb_read_filt_ctxt_t() : page_size(0) {} }; /* The read filter */ struct xb_read_filt_t { void (*init)(xb_read_filt_ctxt_t* ctxt, - const xb_fil_cur_t* cursor, - ulint space_id); + const xb_fil_cur_t* cursor); void (*get_next_batch)(xb_read_filt_ctxt_t* ctxt, - ib_int64_t* read_batch_start, - ib_int64_t* read_batch_len); - void (*deinit)(xb_read_filt_ctxt_t* ctxt); + int64_t* read_batch_start, + int64_t* read_batch_len); }; extern xb_read_filt_t rf_pass_through; -extern xb_read_filt_t rf_bitmap; #endif diff --git a/extra/mariabackup/xtrabackup.cc b/extra/mariabackup/xtrabackup.cc index 5f6ec1b0c0c..fcea942cd3e 100644 --- a/extra/mariabackup/xtrabackup.cc +++ b/extra/mariabackup/xtrabackup.cc @@ -101,7 +101,6 @@ Street, Fifth Floor, Boston, MA 02110-1335 USA #include "ds_buffer.h" #include "ds_tmpfile.h" #include "xbstream.h" -#include "changed_page_bitmap.h" #include "read_filt.h" #include "backup_wsrep.h" #include "innobackupex.h" @@ -155,7 +154,6 @@ char *xtrabackup_incremental; lsn_t incremental_lsn; lsn_t incremental_to_lsn; lsn_t incremental_last_lsn; -xb_page_bitmap *changed_page_bitmap; char *xtrabackup_incremental_basedir; /* for --backup */ char *xtrabackup_extra_lsndir; /* for --backup with --extra-lsndir */ @@ -424,6 +422,7 @@ char orig_argv1[FN_REFLEN]; pthread_mutex_t backup_mutex; pthread_cond_t scanned_lsn_cond; +typedef decltype(fil_space_t::id) space_id_t; typedef std::map space_id_to_name_t; struct ddl_tracker_t { @@ -2855,12 +2854,7 @@ static my_bool xtrabackup_copy_datafile(ds_ctxt *ds_data, goto skip; } - if (!changed_page_bitmap) { - read_filter = &rf_pass_through; - } - else { - read_filter = &rf_bitmap; - } + read_filter = &rf_pass_through; res = xb_fil_cur_open(&cursor, read_filter, node, thread_n, ULLONG_MAX); if (res == XB_FIL_CUR_SKIP) { @@ -4788,11 +4782,6 @@ fail_before_log_copying_thread_start: log_copying_stop = os_event_create(0); os_thread_create(log_copying_thread); - /* FLUSH CHANGED_PAGE_BITMAPS call */ - if (!flush_changed_page_bitmaps()) { - goto fail; - } - ut_a(xtrabackup_parallel > 0); if (xtrabackup_parallel > 1) { @@ -4876,9 +4865,6 @@ fail_before_log_copying_thread_start: goto fail; } - if (changed_page_bitmap) { - xb_page_bitmap_deinit(changed_page_bitmap); - } backup_datasinks.destroy(); msg("Redo log (from LSN " LSN_PF " to " LSN_PF diff --git a/extra/mariabackup/xtrabackup.h b/extra/mariabackup/xtrabackup.h index 260fdcaa818..2cc1d3b7558 100644 --- a/extra/mariabackup/xtrabackup.h +++ b/extra/mariabackup/xtrabackup.h @@ -24,7 +24,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA #include #include "datasink.h" #include "xbstream.h" -#include "changed_page_bitmap.h" +#include "fil0fil.h" #include #define XB_TOOL_NAME "mariadb-backup" @@ -86,8 +86,6 @@ extern uint opt_protocol; /* The last checkpoint LSN at the backup startup time */ extern lsn_t checkpoint_lsn_start; -extern xb_page_bitmap *changed_page_bitmap; - extern char *xtrabackup_incremental; extern my_bool xtrabackup_incremental_force_scan; diff --git a/mysql-test/suite/mariabackup/suite.opt b/mysql-test/suite/mariabackup/suite.opt index 1df4643562e..e6b20c328b2 100644 --- a/mysql-test/suite/mariabackup/suite.opt +++ b/mysql-test/suite/mariabackup/suite.opt @@ -1 +1 @@ ---innodb --loose-changed_page_bitmaps --innodb-sys-tables --innodb-flush-log-at-trx-commit=2 --sequence +--innodb --innodb-sys-tables --innodb-flush-log-at-trx-commit=2 --sequence