2006-04-11 15:45:10 +02:00
|
|
|
/* Copyright (C) 2006 MySQL AB & MySQL Finland AB & TCX DataKonsult AB
|
|
|
|
|
|
|
|
This program is free software; you can redistribute it and/or modify
|
|
|
|
it under the terms of the GNU General Public License as published by
|
2007-03-02 11:20:23 +01:00
|
|
|
the Free Software Foundation; version 2 of the License.
|
2006-04-11 15:45:10 +02:00
|
|
|
|
|
|
|
This program is distributed in the hope that it will be useful,
|
|
|
|
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
|
|
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
|
|
GNU General Public License for more details.
|
|
|
|
|
|
|
|
You should have received a copy of the GNU General Public License
|
|
|
|
along with this program; if not, write to the Free Software
|
|
|
|
Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */
|
|
|
|
|
|
|
|
/* Test av isam-databas: stor test */
|
|
|
|
|
|
|
|
#ifndef USE_MY_FUNC /* We want to be able to dbug this !! */
|
|
|
|
#define USE_MY_FUNC
|
|
|
|
#endif
|
|
|
|
#ifdef DBUG_OFF
|
|
|
|
#undef DBUG_OFF
|
|
|
|
#endif
|
|
|
|
#ifndef SAFEMALLOC
|
|
|
|
#define SAFEMALLOC
|
|
|
|
#endif
|
|
|
|
#include "maria_def.h"
|
WL#3071 Maria checkpoint
Finally this is the real checkpoint code.
It however exhibits unstabilities when a checkpoint runs concurrently
with data-modifying clients (table corruption, transaction log's
assertions) so for now a checkpoint is taken only at startup after
recovery and at shutdown, i.e. not in concurrent situations. Later
we will let it run periodically, as well as flush dirty pages
periodically (almost all needed code is there already, only pagecache
code is written but not committed).
WL#3072 Maria recovery
* replacing UNDO_ROW_PURGE with CLR_END; testing of those CLR_END via
ma_test2 which has INSERTs failing with duplicate keys.
* replaying of REDO_RENAME_TABLE
Now, off to test Recovery in ha_maria :)
BitKeeper/deleted/.del-ma_least_recently_dirtied.c:
Delete: storage/maria/ma_least_recently_dirtied.c
BitKeeper/deleted/.del-ma_least_recently_dirtied.h:
Delete: storage/maria/ma_least_recently_dirtied.h
storage/maria/Makefile.am:
compile Checkpoint module
storage/maria/ha_maria.cc:
When ha_maria starts, do a recovery from last checkpoint.
Take a checkpoint when that recovery has ended and when ha_maria
shuts down cleanly.
storage/maria/ma_blockrec.c:
* even if my_sync() fails we have to my_close() (otherwise we leak
a descriptor)
* UNDO_ROW_PURGE is replaced by a simple CLR_END for UNDO_ROW_INSERT,
as promised in the old comment; it gives us skipping during the
UNDO phase.
storage/maria/ma_check.c:
All REDOs before create_rename_lsn are ignored by Recovery. So
create_rename_lsn must be set only after all data/index has been
flushed and forced to disk. We thus move write_log_record_for_repair()
to after _ma_flush_tables_files_after_repair().
storage/maria/ma_checkpoint.c:
Checkpoint module.
storage/maria/ma_checkpoint.h:
optional argument if caller wants a thread to periodically take
checkpoints and flush dirty pages.
storage/maria/ma_create.c:
* no need to init some vars as the initial bzero(share) takes care of this.
* update to new function's name
* even if we fail in my_sync() we have to my_close()
storage/maria/ma_extra.c:
Checkpoint reads share->last_version under intern_lock, so we make
maria_extra() update it under intern_lock. THR_LOCK_maria still needed
because of _ma_test_if_reopen().
storage/maria/ma_init.c:
destroy checkpoint module when Maria shuts down.
storage/maria/ma_loghandler.c:
* UNDO_ROW_PURGE gone (see ma_blockrec.c)
* we need to remember the LSN of the LOGREC_FILE_ID for a share,
because this LSN is needed into the checkpoint record (Recovery wants
to know the validity domain of an id->name mapping)
* translog_get_horizon_no_lock() needed for Checkpoint
* comment about failing assertion (Sanja knows)
* translog_init_reader_data() thought that translog_read_record_header_scan()
returns 0 in case of error, but 0 just means "0-length header".
* translog_assign_id_to_share() now needs the MARIA_HA because
LOGREC_FILE_ID uses a log-write hook.
* Verify that (de)assignment of share->id happens only under intern_lock,
as Checkpoint reads this id with intern_lock.
* translog_purge() can accept TRANSLOG_ADDRESS, not necessarily
a real LSN.
storage/maria/ma_loghandler.h:
prototype updates
storage/maria/ma_open.c:
no need to initialize "res"
storage/maria/ma_pagecache.c:
When taking a checkpoint, we don't need to know the maximum rec_lsn
of dirty pages; this LSN was intended to be used in the two-checkpoint
rule, but last_checkpoint_lsn is as good.
4 bytes for stored_list_size is enough as PAGECACHE::blocks (number
of blocks which the pagecache can contain) is int.
storage/maria/ma_pagecache.h:
new prototype
storage/maria/ma_recovery.c:
* added replaying of REDO_RENAME_TABLE
* UNDO_ROW_PURGE gone (see ma_blockrec.c), replaced by CLR_END
* Recovery from the last checkpoint record now possible
* In new_table() we skip the table if the id->name mapping is older than
create_rename_lsn (mapping dates from lsn_of_file_id).
* in get_MARIA_HA_from_REDO_record() we skip the record
if the id->name mapping is newer than the record (can happen if processing
a record which is before the checkpoint record).
* parse_checkpoint_record() has to return a LSN, that's what caller expects
storage/maria/ma_rename.c:
new function's name; log end zeroes of tables' names (ease recovery)
storage/maria/ma_test2.c:
* equivalent of ma_test1's --test-undo added (named -u here).
* -t=1 now stops right after creating the table, so that
we can test undoing of INSERTs with duplicate keys (which tests the
CLR_END logged by _ma_write_abort_block_record()).
storage/maria/ma_test_recovery.expected:
Result of testing undoing of INSERTs with duplicate keys; there are
some differences in maria_chk -dvv but they are normal (removing
records does not shrink data/index file, does not put back the
"analyzed, optimized keys"(etc) index state.
storage/maria/ma_test_recovery:
Test undoing of INSERTs with duplicate keys, using ma_test2;
when such INSERT happens, it logs REDO_INSERT, UNDO_INSERT, REDO_DELETE,
CLR_END; we abort after that, and test that CLR_END causes recovery
to jump over UNDO_INSERT.
storage/maria/ma_write.c:
comment
storage/maria/maria_chk.c:
comment
storage/maria/maria_def.h:
* a new bit in MARIA_SHARE::in_checkpoint, used to build a list
of unique shares during Checkpoint.
* MARIA_SHARE::lsn_of_file_id added: the LSN of the last LOGREC_FILE_ID
for this share; needed to know to which LSN domain the mappings
found in the Checkpoint record apply (new mappings should not apply
to old REDOs).
storage/maria/trnman.c:
* small changes to how trnman_collect_transactions() fills its buffer;
it also uses a non-dummy lsn_read_non_atomic() found in ma_checkpoint.h
2007-09-12 11:27:34 +02:00
|
|
|
#include "trnman.h"
|
2006-04-11 15:45:10 +02:00
|
|
|
#include <m_ctype.h>
|
2006-08-11 01:16:51 +02:00
|
|
|
#include <my_bit.h>
|
WL#3071 Maria checkpoint
Ability for flush_pagecache_blocks() to flush only certain pages of
a file, as instructed by an option "filter" pointer-to-function argument;
Checkpoint and background dirty page flushing use that to flush only
pages which have been dirty for long enough and bitmap pages.
Fix for a bug in flush_cached_blocks() (no idea if it could produce
a bug in real life, but theoretically it is).
Testing checkpoint in ma_test_recovery via ma_test1 and ma_test2.
Background checkpoint & dirty pages flush thread is still disabled
by default in ha_maria.
mysql-test/r/maria.result:
result update
storage/maria/ha_maria.cc:
blank after function comment
storage/maria/ma_checkpoint.c:
Using an enum instead of 0/1/2 (applying Sanja's review comments).
The comment about "this is an horizon" can be removed as Sanja
created translog_next_LSN() which parse_checkpoint_record() uses.
Variables in ma_checkpoint_background() cannot be declared in the
for() as their value must not be reset at each iteration!
storage/maria/ma_pagecache.c:
adding to flush_pagecache_blocks() optional arguments 'filter'
(pointer to function) and 'filter_arg'; if filter!=NULL this function
will be called for each block of the file and will reply if this
block and following ones should be flushed or not (3 possible
replies).
Fixing a bug when flush_cached_blocks() skips a pinned page: it has
to unset PCBLOCK_IN_FLUSH set by flush_pagecache_blocks_int().
storage/maria/ma_pagecache.h:
flush_pagecache_blocks() is changed to take "filter" and "filter_arg"
arguments. "filter", if it is not NULL, may return one value
among enum pagecache_flush_filter_result.
storage/maria/ma_recovery.c:
open_count=0 when closing tables at the end of recovery.
storage/maria/ma_test1.c:
Optional checkpoints (-H#) at various stages (stages similar
to --testflag), for testing of checkpoints.
storage/maria/ma_test2.c:
Optional checkpoints (-H#) at various stages (stages similar
to -t), for testing of checkpoints.
storage/maria/ma_test_recovery.expected:
Result update: the results of the additional test run with -H#
(checkpoints) are added here. They are exactly identical to without
checkpoints except that the index's Root (printed by maria_chk)
is more correct when using checkpoints. This is because checkpoint
flushed the state, so it happens to be correct, while no-checkpoint
does not flush the state, and recovery does not recover indexes
so Root is never fixed. When we recover indices, this will go away.
storage/maria/ma_test_recovery:
We duplicate the loop of tests to add an additional run with
checkpoints at various stages, to see if maria_read_log
uses them fine.
2007-10-17 16:55:26 +02:00
|
|
|
#include "ma_checkpoint.h"
|
2007-06-04 13:07:18 +02:00
|
|
|
|
2006-04-11 15:45:10 +02:00
|
|
|
#define STANDARD_LENGTH 37
|
|
|
|
#define MARIA_KEYS 6
|
|
|
|
#define MAX_PARTS 4
|
|
|
|
#if !defined(MSDOS) && !defined(labs)
|
|
|
|
#define labs(a) abs(a)
|
|
|
|
#endif
|
|
|
|
|
|
|
|
static void get_options(int argc, char *argv[]);
|
|
|
|
static uint rnd(uint max_value);
|
2007-07-02 19:45:15 +02:00
|
|
|
static void fix_length(uchar *record,uint length);
|
2007-10-04 19:33:42 +02:00
|
|
|
static void put_blob_in_record(uchar *blob_pos,char **blob_buffer,
|
2007-01-18 20:38:14 +01:00
|
|
|
ulong *length);
|
2006-04-11 15:45:10 +02:00
|
|
|
static void copy_key(struct st_maria_info *info,uint inx,
|
|
|
|
uchar *record,uchar *key);
|
|
|
|
|
|
|
|
static int verbose=0,testflag=0,
|
2007-04-04 22:37:09 +02:00
|
|
|
first_key=0,async_io=0,pagecacheing=0,write_cacheing=0,locking=0,
|
2006-05-05 20:32:02 +02:00
|
|
|
rec_pointer_size=0,pack_fields=1,silent=0,
|
WL#3071 Maria checkpoint
Finally this is the real checkpoint code.
It however exhibits unstabilities when a checkpoint runs concurrently
with data-modifying clients (table corruption, transaction log's
assertions) so for now a checkpoint is taken only at startup after
recovery and at shutdown, i.e. not in concurrent situations. Later
we will let it run periodically, as well as flush dirty pages
periodically (almost all needed code is there already, only pagecache
code is written but not committed).
WL#3072 Maria recovery
* replacing UNDO_ROW_PURGE with CLR_END; testing of those CLR_END via
ma_test2 which has INSERTs failing with duplicate keys.
* replaying of REDO_RENAME_TABLE
Now, off to test Recovery in ha_maria :)
BitKeeper/deleted/.del-ma_least_recently_dirtied.c:
Delete: storage/maria/ma_least_recently_dirtied.c
BitKeeper/deleted/.del-ma_least_recently_dirtied.h:
Delete: storage/maria/ma_least_recently_dirtied.h
storage/maria/Makefile.am:
compile Checkpoint module
storage/maria/ha_maria.cc:
When ha_maria starts, do a recovery from last checkpoint.
Take a checkpoint when that recovery has ended and when ha_maria
shuts down cleanly.
storage/maria/ma_blockrec.c:
* even if my_sync() fails we have to my_close() (otherwise we leak
a descriptor)
* UNDO_ROW_PURGE is replaced by a simple CLR_END for UNDO_ROW_INSERT,
as promised in the old comment; it gives us skipping during the
UNDO phase.
storage/maria/ma_check.c:
All REDOs before create_rename_lsn are ignored by Recovery. So
create_rename_lsn must be set only after all data/index has been
flushed and forced to disk. We thus move write_log_record_for_repair()
to after _ma_flush_tables_files_after_repair().
storage/maria/ma_checkpoint.c:
Checkpoint module.
storage/maria/ma_checkpoint.h:
optional argument if caller wants a thread to periodically take
checkpoints and flush dirty pages.
storage/maria/ma_create.c:
* no need to init some vars as the initial bzero(share) takes care of this.
* update to new function's name
* even if we fail in my_sync() we have to my_close()
storage/maria/ma_extra.c:
Checkpoint reads share->last_version under intern_lock, so we make
maria_extra() update it under intern_lock. THR_LOCK_maria still needed
because of _ma_test_if_reopen().
storage/maria/ma_init.c:
destroy checkpoint module when Maria shuts down.
storage/maria/ma_loghandler.c:
* UNDO_ROW_PURGE gone (see ma_blockrec.c)
* we need to remember the LSN of the LOGREC_FILE_ID for a share,
because this LSN is needed into the checkpoint record (Recovery wants
to know the validity domain of an id->name mapping)
* translog_get_horizon_no_lock() needed for Checkpoint
* comment about failing assertion (Sanja knows)
* translog_init_reader_data() thought that translog_read_record_header_scan()
returns 0 in case of error, but 0 just means "0-length header".
* translog_assign_id_to_share() now needs the MARIA_HA because
LOGREC_FILE_ID uses a log-write hook.
* Verify that (de)assignment of share->id happens only under intern_lock,
as Checkpoint reads this id with intern_lock.
* translog_purge() can accept TRANSLOG_ADDRESS, not necessarily
a real LSN.
storage/maria/ma_loghandler.h:
prototype updates
storage/maria/ma_open.c:
no need to initialize "res"
storage/maria/ma_pagecache.c:
When taking a checkpoint, we don't need to know the maximum rec_lsn
of dirty pages; this LSN was intended to be used in the two-checkpoint
rule, but last_checkpoint_lsn is as good.
4 bytes for stored_list_size is enough as PAGECACHE::blocks (number
of blocks which the pagecache can contain) is int.
storage/maria/ma_pagecache.h:
new prototype
storage/maria/ma_recovery.c:
* added replaying of REDO_RENAME_TABLE
* UNDO_ROW_PURGE gone (see ma_blockrec.c), replaced by CLR_END
* Recovery from the last checkpoint record now possible
* In new_table() we skip the table if the id->name mapping is older than
create_rename_lsn (mapping dates from lsn_of_file_id).
* in get_MARIA_HA_from_REDO_record() we skip the record
if the id->name mapping is newer than the record (can happen if processing
a record which is before the checkpoint record).
* parse_checkpoint_record() has to return a LSN, that's what caller expects
storage/maria/ma_rename.c:
new function's name; log end zeroes of tables' names (ease recovery)
storage/maria/ma_test2.c:
* equivalent of ma_test1's --test-undo added (named -u here).
* -t=1 now stops right after creating the table, so that
we can test undoing of INSERTs with duplicate keys (which tests the
CLR_END logged by _ma_write_abort_block_record()).
storage/maria/ma_test_recovery.expected:
Result of testing undoing of INSERTs with duplicate keys; there are
some differences in maria_chk -dvv but they are normal (removing
records does not shrink data/index file, does not put back the
"analyzed, optimized keys"(etc) index state.
storage/maria/ma_test_recovery:
Test undoing of INSERTs with duplicate keys, using ma_test2;
when such INSERT happens, it logs REDO_INSERT, UNDO_INSERT, REDO_DELETE,
CLR_END; we abort after that, and test that CLR_END causes recovery
to jump over UNDO_INSERT.
storage/maria/ma_write.c:
comment
storage/maria/maria_chk.c:
comment
storage/maria/maria_def.h:
* a new bit in MARIA_SHARE::in_checkpoint, used to build a list
of unique shares during Checkpoint.
* MARIA_SHARE::lsn_of_file_id added: the LSN of the last LOGREC_FILE_ID
for this share; needed to know to which LSN domain the mappings
found in the Checkpoint record apply (new mappings should not apply
to old REDOs).
storage/maria/trnman.c:
* small changes to how trnman_collect_transactions() fills its buffer;
it also uses a non-dummy lsn_read_non_atomic() found in ma_checkpoint.h
2007-09-12 11:27:34 +02:00
|
|
|
opt_quick_mode=0, transactional= 0, skip_update= 0,
|
|
|
|
die_in_middle_of_transaction= 0;
|
2007-01-18 20:38:14 +01:00
|
|
|
static int pack_seg=HA_SPACE_PACK,pack_type=HA_PACK_KEY,remove_count=-1;
|
WL#3071 Maria checkpoint
Ability for flush_pagecache_blocks() to flush only certain pages of
a file, as instructed by an option "filter" pointer-to-function argument;
Checkpoint and background dirty page flushing use that to flush only
pages which have been dirty for long enough and bitmap pages.
Fix for a bug in flush_cached_blocks() (no idea if it could produce
a bug in real life, but theoretically it is).
Testing checkpoint in ma_test_recovery via ma_test1 and ma_test2.
Background checkpoint & dirty pages flush thread is still disabled
by default in ha_maria.
mysql-test/r/maria.result:
result update
storage/maria/ha_maria.cc:
blank after function comment
storage/maria/ma_checkpoint.c:
Using an enum instead of 0/1/2 (applying Sanja's review comments).
The comment about "this is an horizon" can be removed as Sanja
created translog_next_LSN() which parse_checkpoint_record() uses.
Variables in ma_checkpoint_background() cannot be declared in the
for() as their value must not be reset at each iteration!
storage/maria/ma_pagecache.c:
adding to flush_pagecache_blocks() optional arguments 'filter'
(pointer to function) and 'filter_arg'; if filter!=NULL this function
will be called for each block of the file and will reply if this
block and following ones should be flushed or not (3 possible
replies).
Fixing a bug when flush_cached_blocks() skips a pinned page: it has
to unset PCBLOCK_IN_FLUSH set by flush_pagecache_blocks_int().
storage/maria/ma_pagecache.h:
flush_pagecache_blocks() is changed to take "filter" and "filter_arg"
arguments. "filter", if it is not NULL, may return one value
among enum pagecache_flush_filter_result.
storage/maria/ma_recovery.c:
open_count=0 when closing tables at the end of recovery.
storage/maria/ma_test1.c:
Optional checkpoints (-H#) at various stages (stages similar
to --testflag), for testing of checkpoints.
storage/maria/ma_test2.c:
Optional checkpoints (-H#) at various stages (stages similar
to -t), for testing of checkpoints.
storage/maria/ma_test_recovery.expected:
Result update: the results of the additional test run with -H#
(checkpoints) are added here. They are exactly identical to without
checkpoints except that the index's Root (printed by maria_chk)
is more correct when using checkpoints. This is because checkpoint
flushed the state, so it happens to be correct, while no-checkpoint
does not flush the state, and recovery does not recover indexes
so Root is never fixed. When we recover indices, this will go away.
storage/maria/ma_test_recovery:
We duplicate the loop of tests to add an additional run with
checkpoints at various stages, to see if maria_read_log
uses them fine.
2007-10-17 16:55:26 +02:00
|
|
|
static int create_flag= 0, srand_arg= 0, checkpoint= 0;
|
2007-04-04 22:37:09 +02:00
|
|
|
static ulong pagecache_size=IO_SIZE*16;
|
2007-01-18 20:38:14 +01:00
|
|
|
static enum data_file_type record_type= DYNAMIC_RECORD;
|
2006-04-11 15:45:10 +02:00
|
|
|
|
|
|
|
static uint keys=MARIA_KEYS,recant=1000;
|
|
|
|
static uint use_blob=0;
|
|
|
|
static uint16 key1[1001],key3[5000];
|
2007-10-04 19:33:42 +02:00
|
|
|
static uchar record[300],record2[300],key[100],key2[100];
|
|
|
|
static uchar read_record[300],read_record2[300],read_record3[300];
|
2006-04-11 15:45:10 +02:00
|
|
|
static HA_KEYSEG glob_keyseg[MARIA_KEYS][MAX_PARTS];
|
|
|
|
|
|
|
|
/* Test program */
|
|
|
|
|
|
|
|
int main(int argc, char *argv[])
|
|
|
|
{
|
|
|
|
uint i;
|
|
|
|
int j,n1,n2,n3,error,k;
|
|
|
|
uint write_count,update,dupp_keys,opt_delete,start,length,blob_pos,
|
|
|
|
reclength,ant,found_parts;
|
|
|
|
my_off_t lastpos;
|
|
|
|
ha_rows range_records,records;
|
|
|
|
MARIA_HA *file;
|
|
|
|
MARIA_KEYDEF keyinfo[10];
|
|
|
|
MARIA_COLUMNDEF recinfo[10];
|
|
|
|
MARIA_INFO info;
|
|
|
|
const char *filename;
|
|
|
|
char *blob_buffer;
|
|
|
|
MARIA_CREATE_INFO create_info;
|
|
|
|
MY_INIT(argv[0]);
|
|
|
|
|
|
|
|
filename= "test2";
|
|
|
|
get_options(argc,argv);
|
|
|
|
if (! async_io)
|
|
|
|
my_disable_async_io=1;
|
|
|
|
|
WL#3072 - Maria recovery
Unit test for recovery: runs ma_test1 and ma_test2 (both only with
INSERTs and DELETEs; UPDATEs disabled as not handled by recovery)
then moves the tables elswhere; recreates tables from the log, and
compares and fails if there is a difference. Passes now.
Most of maria_read_log.c moved to ma_recovery.c, as it will be re-used
for recovery-from-ha_maria.
Bugfixes of applying of REDO_INSERT, REDO_PURGE_ROW.
Applying of REDO_PURGE_BLOCKS, REDO_DELETE_ALL, REDO_DROP_TABLE,
UNDO_ROW_INSERT (in REDO phase only, i.e. just doing records++),
UNDO_ROW_DELETE, UNDO_ROW_PURGE.
Code cleanups.
Monty: please look for "QQ". Sanja: please look for "Sanja".
Future tasks: recovery of the bitmap (easy), recovery of the state
(make it idempotent), more REDOs (Monty to work on
REDO_UPDATE?), UNDO phase...
Pushing this cset as it looks safe, contains test and bugfixes which
will help Monty implement applying of REDO_UPDATE.
sql/handler.cc:
typo
storage/maria/Makefile.am:
Adding ma_test_recovery (which ma_test_all invokes, and which can
also be run alone). Most of maria_read_log.c moved to ma_recovery.c
storage/maria/ha_maria.cc:
comments
storage/maria/ma_bitmap.c:
fixing comments. 2 -> sizeof(maria_bitmap_marker).
Bitmap-related part of _ma_initialize_datafile() moves in bitmap module.
Now putting the "bm" signature when creating the first bitmap page
(it used to happen only at next open, but that
caused an annoying difference when testing Recovery if the original
run didn't open the table, and it looks more
logical like this: it goes to disk only with its signature correct);
see the "QQ" comment towards the _ma_initialize_data_file() call
in ma_create.c for more).
When reading a bitmap page, verify its signature (happens when normally
using the table or when CHECKing it; not when REPAIRing it).
storage/maria/ma_blockrec.c:
* no need to sync the data file if table is not transactional
* Comments, code cleanup (log-related data moved to log-related code
block, int5store->page_store).
* Store the table's short id into LOGREC_UNDO_ROW_PURGE, like we
do for other records (though this record will soon be replaced
with a CLR).
* If "page" is 1 it means the page which extends from byte
page*block_size+1 to (page+1)*block_size (byte number 1 being
the first byte of the file). The last byte of the file is
data_file_length (same convention).
A new page needs to be created if the last byte of the page is
beyond the last byte of the file, i.e.
(page+1)*block_size+1 > data_file_length, so we correct the test
(bug found when testing log applying for ma_test1 -M -T --skip-update).
* update the page's LSN when removing a row from it during
execution of a REDO_PURGE_ROW record (bug found when testing log
applying for ma_test1 -M -T --skip-update).
* applying of REDO_PURGE_BLOCKs (limited to a one-page range for now).
storage/maria/ma_blockrec.h:
new functions. maria_bitmap_marker does not need to be exported.
storage/maria/ma_close.c:
we can always flush the table's state when closing the last instance
of the table. And it is needed for maria_read_log (as it does
not use maria_lock_database()).
storage/maria/ma_control_file.c:
when in Recovery, some assertions should not be used.
storage/maria/ma_control_file.h:
double-inclusion safe
storage/maria/ma_create.c:
during recovery, don't log records. Comments.
Moving the creation of the first bitmap page to ma_bitmap.c
storage/maria/ma_delete_table.c:
during recovery, don't log records. Log the end-zero of the dropped
table's name, so that recovery can use the string in place without
extending it to fit an end zero.
storage/maria/ma_loghandler.c:
* inwrite_rec_hook also needs access to the MARIA_SHARE, like
prewrite_rec_hook. This will be needed to update
share->records_diff (in the upcoming patch "recovery of the state").
* LOG_DESC::record_ends_group changed to an enum.
* LOG_DESC for LOGREC_REDO_PURGE_BLOCKS and LOGREC_UNDO_ROW_PURGE
corrected
* Sanja please see the @todo LOG BUG
* avoiding DBUG_RETURN(func()) as it gives confusing debug traces.
storage/maria/ma_loghandler.h:
- log write hooks called while the log's lock is held (inwrite_rec_hook)
now need the MARIA_SHARE, like prewrite_rec_hook already had
- instead of a bool saying if this record's type ends groups or not,
we refine: it may not end a group, it may end a group, or it may
be a group in itself. Imagine that we had a physical write failure
to a table before we log the UNDO, we still end up in
external_lock(F_UNLCK) and then we log a COMMIT: we don't want
to consider this COMMIT as ending the group of REDOs (don't want
to execute those REDOs during Recovery), that's why we say "COMMIT
is a group in itself, it aborts any previous group". This also
gives one more sanity check in maria_read_log.
storage/maria/ma_recovery.c:
New Recovery code, replacing the old pseudocode.
Most of maria_read_log moved here.
Call-able from ha_maria, but not enabled yet.
Compared to the previous version of maria_read_log, some bugs have
been fixed, debugging output can go to stdout or a disk file (for now
it's useful for me, later it can be changed), execution of
REDO_DROP_TABLE, REDO_DELETE_ALL, REDO_PURGE_BLOCKS has been added. Duplicate code
has been factored into functions. We abort an unfinished group
of records if we see a record which is a group in itself (like COMMIT).
No need for maria_panic() after a bug (which caused tables to not
be closed) was fixed; if there is yet another bug I prefer to see it.
When opening a table for Recovery, set data_file_length
and key_file_length to their real physical value (these are the
easiest state members to restore :). Warn us if the last page
was truncated (but Recovery handles it).
MARIA_SHARE::state::state::records is now partly recovered (not
idempotent, but works if recreating tables from scracth).
When applying a REDO to a page, stamp it with the UNDO's LSN
(current_group_end_lsn), not with the REDO's LSN; it makes
the table more identical to the original table (easier to compare
the two tables in the end).
Big thing missing: some types of REDOs are not handled,
and the UNDO phase does not exist (missing functions to execute UNDOs
to actually rollback). So for now tests are only inserting/deleting
a few 100 rows, closing the table and seeing if the log is applied ok;
it works. UPDATE not handled.
storage/maria/ma_recovery.h:
new functions: ma_recover() for recovery from inside ha_maria;
_ma_apply_log() for maria_read_log (ma_recover() calls _ma_apply_log()).
Btw, we need to not use the word "recover" for REPAIR/maria_chk anymore.
storage/maria/ma_rename.c:
don't write log records during recovery
storage/maria/ma_test2.c:
- fail if maria_info() or other subtests find some wrong information
- new option -g to skip updates.
- init the translog before creating the table, so that log applying
can work.
- in "#if 0" you'll see some fixed bugs (will be removed).
storage/maria/ma_test_all.sh:
cleanup files. Test log applying.
storage/maria/maria_read_log.c:
most of the logic moves to ma_recovery.c to be shared between
maria_read_log and recovery-from-inside-mysqld.
See ma_recovery.c for additional changes made to the moved code.
storage/maria/ma_test_recovery:
unit test for Recovery. Tests insert and delete,
REDO_UPDATE not yet coded.
Script is called from ma_test_all. Can run standalone.
2007-07-26 11:56:21 +02:00
|
|
|
maria_data_root= ".";
|
|
|
|
/* Maria requires that we always have a page cache */
|
|
|
|
if (maria_init() ||
|
|
|
|
(init_pagecache(maria_pagecache, pagecache_size, 0, 0,
|
|
|
|
maria_block_size) == 0) ||
|
- WL#3072 Maria Recovery:
Recovery of state.records (the count of records which is stored into
the header of the index file). For that, state.is_of_lsn is introduced;
logic is explained in ma_recovery.c (look for "Recovery of the state").
The net gain is that in case of crash, we now recover state.records,
and it is idempotent (ma_test_recovery tests it).
state.checksum is not recovered yet, mail sent for discussion.
- WL#3071 Maria Checkpoint: preparation for it, by protecting
all modifications of the state in memory or on disk with intern_lock
(with the exception of the really-often-modified state.records,
which is now protected with the log's lock, see ma_recovery.c
(look for "Recovery of the state"). Also, if maria_close() sees that
Checkpoint is looking at this table it will not my_free() the share.
- don't compute row's checksum twice in case of UPDATE (correction
to a bugfix I made yesterday).
storage/maria/ha_maria.cc:
protect state write with intern_lock (against Checkpoint)
storage/maria/ma_blockrec.c:
* don't reset trn->rec_lsn in _ma_unpin_all_pages(), because it
should wait until we have corrected the allocation in the bitmap
(as the REDO can serve to correct the allocation during Recovery);
introducing _ma_finalize_row() for that.
* In a changeset yesterday I moved computation of the checksum
into write_block_record(), to fix a bug in UPDATE. Now I notice
that maria_update() already computes the checksum, it's just that
it puts it into info->cur_row while _ma_update_block_record()
uses info->new_row; so, removing the checksum computation from
write_block_record(), putting it back into allocate_and_write_block_record()
(which is called only by INSERT and UNDO_DELETE), and copying
cur_row->checksum into new_row->checksum in _ma_update_block_record().
storage/maria/ma_check.c:
new prototypes, they will take intern_lock when writing the state;
also take intern_lock when changing share->kfile. In both cases
this is to protect against Checkpoint reading/writing the state or reading
kfile at the same time.
Not updating create_rename_lsn directly at end of write_log_record_for_repair()
as it wouldn't have intern_lock.
storage/maria/ma_close.c:
Checkpoint builds a list of shares (under THR_LOCK_maria), then it
handles each such share (under intern_lock) (doing flushing etc);
if maria_close() freed this share between the two, Checkpoint
would see a bad pointer. To avoid this, when building the list Checkpoint
marks each share, so that maria_close() knows it should not free it
and Checkpoint will free it itself.
Extending the zone covered by intern_lock to protect against
Checkpoint reading kfile, writing state.
storage/maria/ma_create.c:
When we update create_rename_lsn, we also update is_of_lsn to
the same value: it is logical, and allows us to test in maria_open()
that the former is not bigger than the latter (the contrary is a sign
of index header corruption, or severe logging bug which hinders
Recovery, table needs a repair).
_ma_update_create_rename_lsn_on_disk() also writes is_of_lsn;
it now operates under intern_lock (protect against Checkpoint),
a shortcut function is available for cases where acquiring
intern_lock is not needed (table's creation or first open).
storage/maria/ma_delete.c:
if table is transactional, "records" is already decremented
when logging UNDO_ROW_DELETE.
storage/maria/ma_delete_all.c:
comments
storage/maria/ma_extra.c:
Protect modifications of the state, in memory and/or on disk,
with intern_lock, against a concurrent Checkpoint.
When state goes to disk, update it's is_of_lsn (by calling
the new _ma_state_info_write()).
In HA_EXTRA_FORCE_REOPEN, don't set share->changed to 0 (undoing
a change I made a few days ago) and ASK_MONTY
storage/maria/ma_locking.c:
no real code change here.
storage/maria/ma_loghandler.c:
Log-write-hooks for updating "state.records" under log's mutex
when writing/updating/deleting a row or deleting all rows.
storage/maria/ma_loghandler_lsn.h:
merge (make LSN_ERROR and LSN_REPAIRED_BY_MARIA_CHK different)
storage/maria/ma_open.c:
When opening a table verify that is_of_lsn >= create_rename_lsn; if
false the header must be corrupted.
_ma_state_info_write() is split in two: _ma_state_info_write_sub()
which is the old _ma_state_info_write(), and _ma_state_info_write()
which additionally takes intern_lock if requested (to protect
against Checkpoint) and updates is_of_lsn.
_ma_open_keyfile() should change kfile.file under intern_lock
to protect Checkpoint from reading a wrong kfile.file.
storage/maria/ma_recovery.c:
Recovery of state.records: when the REDO phase sees UNDO_ROW_INSERT
which has a LSN > state.is_of_lsn it increments state.records.
Same for UNDO_ROW_DELETE and UNDO_ROW_PURGE.
When closing a table during Recovery, we know its state is at least
as new as the current log record we are looking at, so increase
is_of_lsn to the LSN of the current log record.
storage/maria/ma_rename.c:
update for new behaviour of _ma_update_create_rename_lsn_on_disk().
storage/maria/ma_test1.c:
update to new prototype
storage/maria/ma_test2.c:
update to new prototype (actually prototype was changed days ago,
but compiler does not complain about the extra argument??)
storage/maria/ma_test_recovery.expected:
new result file of ma_test_recovery. Improvements: record
count read from index's header is now always correct.
storage/maria/ma_test_recovery:
"rm" fails if file does not exist. Redirect stderr of script.
storage/maria/ma_write.c:
if table is transactional, "records" is already incremented when
logging UNDO_ROW_INSERT. Comments.
storage/maria/maria_chk.c:
update is_of_lsn too
storage/maria/maria_def.h:
- MARIA_STATE_INFO::is_of_lsn which is used by Recovery. It is stored
into the index file's header.
- Checkpoint can now mark a table as "don't free this", and maria_close()
can reply "ok then you will free it".
- new functions
storage/maria/maria_pack.c:
update for new name
2007-09-07 15:02:30 +02:00
|
|
|
ma_control_file_create_or_open() ||
|
WL#3072 - Maria recovery
Unit test for recovery: runs ma_test1 and ma_test2 (both only with
INSERTs and DELETEs; UPDATEs disabled as not handled by recovery)
then moves the tables elswhere; recreates tables from the log, and
compares and fails if there is a difference. Passes now.
Most of maria_read_log.c moved to ma_recovery.c, as it will be re-used
for recovery-from-ha_maria.
Bugfixes of applying of REDO_INSERT, REDO_PURGE_ROW.
Applying of REDO_PURGE_BLOCKS, REDO_DELETE_ALL, REDO_DROP_TABLE,
UNDO_ROW_INSERT (in REDO phase only, i.e. just doing records++),
UNDO_ROW_DELETE, UNDO_ROW_PURGE.
Code cleanups.
Monty: please look for "QQ". Sanja: please look for "Sanja".
Future tasks: recovery of the bitmap (easy), recovery of the state
(make it idempotent), more REDOs (Monty to work on
REDO_UPDATE?), UNDO phase...
Pushing this cset as it looks safe, contains test and bugfixes which
will help Monty implement applying of REDO_UPDATE.
sql/handler.cc:
typo
storage/maria/Makefile.am:
Adding ma_test_recovery (which ma_test_all invokes, and which can
also be run alone). Most of maria_read_log.c moved to ma_recovery.c
storage/maria/ha_maria.cc:
comments
storage/maria/ma_bitmap.c:
fixing comments. 2 -> sizeof(maria_bitmap_marker).
Bitmap-related part of _ma_initialize_datafile() moves in bitmap module.
Now putting the "bm" signature when creating the first bitmap page
(it used to happen only at next open, but that
caused an annoying difference when testing Recovery if the original
run didn't open the table, and it looks more
logical like this: it goes to disk only with its signature correct);
see the "QQ" comment towards the _ma_initialize_data_file() call
in ma_create.c for more).
When reading a bitmap page, verify its signature (happens when normally
using the table or when CHECKing it; not when REPAIRing it).
storage/maria/ma_blockrec.c:
* no need to sync the data file if table is not transactional
* Comments, code cleanup (log-related data moved to log-related code
block, int5store->page_store).
* Store the table's short id into LOGREC_UNDO_ROW_PURGE, like we
do for other records (though this record will soon be replaced
with a CLR).
* If "page" is 1 it means the page which extends from byte
page*block_size+1 to (page+1)*block_size (byte number 1 being
the first byte of the file). The last byte of the file is
data_file_length (same convention).
A new page needs to be created if the last byte of the page is
beyond the last byte of the file, i.e.
(page+1)*block_size+1 > data_file_length, so we correct the test
(bug found when testing log applying for ma_test1 -M -T --skip-update).
* update the page's LSN when removing a row from it during
execution of a REDO_PURGE_ROW record (bug found when testing log
applying for ma_test1 -M -T --skip-update).
* applying of REDO_PURGE_BLOCKs (limited to a one-page range for now).
storage/maria/ma_blockrec.h:
new functions. maria_bitmap_marker does not need to be exported.
storage/maria/ma_close.c:
we can always flush the table's state when closing the last instance
of the table. And it is needed for maria_read_log (as it does
not use maria_lock_database()).
storage/maria/ma_control_file.c:
when in Recovery, some assertions should not be used.
storage/maria/ma_control_file.h:
double-inclusion safe
storage/maria/ma_create.c:
during recovery, don't log records. Comments.
Moving the creation of the first bitmap page to ma_bitmap.c
storage/maria/ma_delete_table.c:
during recovery, don't log records. Log the end-zero of the dropped
table's name, so that recovery can use the string in place without
extending it to fit an end zero.
storage/maria/ma_loghandler.c:
* inwrite_rec_hook also needs access to the MARIA_SHARE, like
prewrite_rec_hook. This will be needed to update
share->records_diff (in the upcoming patch "recovery of the state").
* LOG_DESC::record_ends_group changed to an enum.
* LOG_DESC for LOGREC_REDO_PURGE_BLOCKS and LOGREC_UNDO_ROW_PURGE
corrected
* Sanja please see the @todo LOG BUG
* avoiding DBUG_RETURN(func()) as it gives confusing debug traces.
storage/maria/ma_loghandler.h:
- log write hooks called while the log's lock is held (inwrite_rec_hook)
now need the MARIA_SHARE, like prewrite_rec_hook already had
- instead of a bool saying if this record's type ends groups or not,
we refine: it may not end a group, it may end a group, or it may
be a group in itself. Imagine that we had a physical write failure
to a table before we log the UNDO, we still end up in
external_lock(F_UNLCK) and then we log a COMMIT: we don't want
to consider this COMMIT as ending the group of REDOs (don't want
to execute those REDOs during Recovery), that's why we say "COMMIT
is a group in itself, it aborts any previous group". This also
gives one more sanity check in maria_read_log.
storage/maria/ma_recovery.c:
New Recovery code, replacing the old pseudocode.
Most of maria_read_log moved here.
Call-able from ha_maria, but not enabled yet.
Compared to the previous version of maria_read_log, some bugs have
been fixed, debugging output can go to stdout or a disk file (for now
it's useful for me, later it can be changed), execution of
REDO_DROP_TABLE, REDO_DELETE_ALL, REDO_PURGE_BLOCKS has been added. Duplicate code
has been factored into functions. We abort an unfinished group
of records if we see a record which is a group in itself (like COMMIT).
No need for maria_panic() after a bug (which caused tables to not
be closed) was fixed; if there is yet another bug I prefer to see it.
When opening a table for Recovery, set data_file_length
and key_file_length to their real physical value (these are the
easiest state members to restore :). Warn us if the last page
was truncated (but Recovery handles it).
MARIA_SHARE::state::state::records is now partly recovered (not
idempotent, but works if recreating tables from scracth).
When applying a REDO to a page, stamp it with the UNDO's LSN
(current_group_end_lsn), not with the REDO's LSN; it makes
the table more identical to the original table (easier to compare
the two tables in the end).
Big thing missing: some types of REDOs are not handled,
and the UNDO phase does not exist (missing functions to execute UNDOs
to actually rollback). So for now tests are only inserting/deleting
a few 100 rows, closing the table and seeing if the log is applied ok;
it works. UPDATE not handled.
storage/maria/ma_recovery.h:
new functions: ma_recover() for recovery from inside ha_maria;
_ma_apply_log() for maria_read_log (ma_recover() calls _ma_apply_log()).
Btw, we need to not use the word "recover" for REPAIR/maria_chk anymore.
storage/maria/ma_rename.c:
don't write log records during recovery
storage/maria/ma_test2.c:
- fail if maria_info() or other subtests find some wrong information
- new option -g to skip updates.
- init the translog before creating the table, so that log applying
can work.
- in "#if 0" you'll see some fixed bugs (will be removed).
storage/maria/ma_test_all.sh:
cleanup files. Test log applying.
storage/maria/maria_read_log.c:
most of the logic moves to ma_recovery.c to be shared between
maria_read_log and recovery-from-inside-mysqld.
See ma_recovery.c for additional changes made to the moved code.
storage/maria/ma_test_recovery:
unit test for Recovery. Tests insert and delete,
REDO_UPDATE not yet coded.
Script is called from ma_test_all. Can run standalone.
2007-07-26 11:56:21 +02:00
|
|
|
(init_pagecache(maria_log_pagecache,
|
|
|
|
TRANSLOG_PAGECACHE_SIZE, 0, 0,
|
|
|
|
TRANSLOG_PAGE_SIZE) == 0) ||
|
|
|
|
translog_init(maria_data_root, TRANSLOG_FILE_SIZE,
|
|
|
|
0, 0, maria_log_pagecache,
|
|
|
|
TRANSLOG_DEFAULT_FLAGS) ||
|
WL#3071 Maria checkpoint
Ability for flush_pagecache_blocks() to flush only certain pages of
a file, as instructed by an option "filter" pointer-to-function argument;
Checkpoint and background dirty page flushing use that to flush only
pages which have been dirty for long enough and bitmap pages.
Fix for a bug in flush_cached_blocks() (no idea if it could produce
a bug in real life, but theoretically it is).
Testing checkpoint in ma_test_recovery via ma_test1 and ma_test2.
Background checkpoint & dirty pages flush thread is still disabled
by default in ha_maria.
mysql-test/r/maria.result:
result update
storage/maria/ha_maria.cc:
blank after function comment
storage/maria/ma_checkpoint.c:
Using an enum instead of 0/1/2 (applying Sanja's review comments).
The comment about "this is an horizon" can be removed as Sanja
created translog_next_LSN() which parse_checkpoint_record() uses.
Variables in ma_checkpoint_background() cannot be declared in the
for() as their value must not be reset at each iteration!
storage/maria/ma_pagecache.c:
adding to flush_pagecache_blocks() optional arguments 'filter'
(pointer to function) and 'filter_arg'; if filter!=NULL this function
will be called for each block of the file and will reply if this
block and following ones should be flushed or not (3 possible
replies).
Fixing a bug when flush_cached_blocks() skips a pinned page: it has
to unset PCBLOCK_IN_FLUSH set by flush_pagecache_blocks_int().
storage/maria/ma_pagecache.h:
flush_pagecache_blocks() is changed to take "filter" and "filter_arg"
arguments. "filter", if it is not NULL, may return one value
among enum pagecache_flush_filter_result.
storage/maria/ma_recovery.c:
open_count=0 when closing tables at the end of recovery.
storage/maria/ma_test1.c:
Optional checkpoints (-H#) at various stages (stages similar
to --testflag), for testing of checkpoints.
storage/maria/ma_test2.c:
Optional checkpoints (-H#) at various stages (stages similar
to -t), for testing of checkpoints.
storage/maria/ma_test_recovery.expected:
Result update: the results of the additional test run with -H#
(checkpoints) are added here. They are exactly identical to without
checkpoints except that the index's Root (printed by maria_chk)
is more correct when using checkpoints. This is because checkpoint
flushed the state, so it happens to be correct, while no-checkpoint
does not flush the state, and recovery does not recover indexes
so Root is never fixed. When we recover indices, this will go away.
storage/maria/ma_test_recovery:
We duplicate the loop of tests to add an additional run with
checkpoints at various stages, to see if maria_read_log
uses them fine.
2007-10-17 16:55:26 +02:00
|
|
|
(transactional && (trnman_init(0) || ma_checkpoint_init(FALSE))))
|
WL#3072 - Maria recovery
Unit test for recovery: runs ma_test1 and ma_test2 (both only with
INSERTs and DELETEs; UPDATEs disabled as not handled by recovery)
then moves the tables elswhere; recreates tables from the log, and
compares and fails if there is a difference. Passes now.
Most of maria_read_log.c moved to ma_recovery.c, as it will be re-used
for recovery-from-ha_maria.
Bugfixes of applying of REDO_INSERT, REDO_PURGE_ROW.
Applying of REDO_PURGE_BLOCKS, REDO_DELETE_ALL, REDO_DROP_TABLE,
UNDO_ROW_INSERT (in REDO phase only, i.e. just doing records++),
UNDO_ROW_DELETE, UNDO_ROW_PURGE.
Code cleanups.
Monty: please look for "QQ". Sanja: please look for "Sanja".
Future tasks: recovery of the bitmap (easy), recovery of the state
(make it idempotent), more REDOs (Monty to work on
REDO_UPDATE?), UNDO phase...
Pushing this cset as it looks safe, contains test and bugfixes which
will help Monty implement applying of REDO_UPDATE.
sql/handler.cc:
typo
storage/maria/Makefile.am:
Adding ma_test_recovery (which ma_test_all invokes, and which can
also be run alone). Most of maria_read_log.c moved to ma_recovery.c
storage/maria/ha_maria.cc:
comments
storage/maria/ma_bitmap.c:
fixing comments. 2 -> sizeof(maria_bitmap_marker).
Bitmap-related part of _ma_initialize_datafile() moves in bitmap module.
Now putting the "bm" signature when creating the first bitmap page
(it used to happen only at next open, but that
caused an annoying difference when testing Recovery if the original
run didn't open the table, and it looks more
logical like this: it goes to disk only with its signature correct);
see the "QQ" comment towards the _ma_initialize_data_file() call
in ma_create.c for more).
When reading a bitmap page, verify its signature (happens when normally
using the table or when CHECKing it; not when REPAIRing it).
storage/maria/ma_blockrec.c:
* no need to sync the data file if table is not transactional
* Comments, code cleanup (log-related data moved to log-related code
block, int5store->page_store).
* Store the table's short id into LOGREC_UNDO_ROW_PURGE, like we
do for other records (though this record will soon be replaced
with a CLR).
* If "page" is 1 it means the page which extends from byte
page*block_size+1 to (page+1)*block_size (byte number 1 being
the first byte of the file). The last byte of the file is
data_file_length (same convention).
A new page needs to be created if the last byte of the page is
beyond the last byte of the file, i.e.
(page+1)*block_size+1 > data_file_length, so we correct the test
(bug found when testing log applying for ma_test1 -M -T --skip-update).
* update the page's LSN when removing a row from it during
execution of a REDO_PURGE_ROW record (bug found when testing log
applying for ma_test1 -M -T --skip-update).
* applying of REDO_PURGE_BLOCKs (limited to a one-page range for now).
storage/maria/ma_blockrec.h:
new functions. maria_bitmap_marker does not need to be exported.
storage/maria/ma_close.c:
we can always flush the table's state when closing the last instance
of the table. And it is needed for maria_read_log (as it does
not use maria_lock_database()).
storage/maria/ma_control_file.c:
when in Recovery, some assertions should not be used.
storage/maria/ma_control_file.h:
double-inclusion safe
storage/maria/ma_create.c:
during recovery, don't log records. Comments.
Moving the creation of the first bitmap page to ma_bitmap.c
storage/maria/ma_delete_table.c:
during recovery, don't log records. Log the end-zero of the dropped
table's name, so that recovery can use the string in place without
extending it to fit an end zero.
storage/maria/ma_loghandler.c:
* inwrite_rec_hook also needs access to the MARIA_SHARE, like
prewrite_rec_hook. This will be needed to update
share->records_diff (in the upcoming patch "recovery of the state").
* LOG_DESC::record_ends_group changed to an enum.
* LOG_DESC for LOGREC_REDO_PURGE_BLOCKS and LOGREC_UNDO_ROW_PURGE
corrected
* Sanja please see the @todo LOG BUG
* avoiding DBUG_RETURN(func()) as it gives confusing debug traces.
storage/maria/ma_loghandler.h:
- log write hooks called while the log's lock is held (inwrite_rec_hook)
now need the MARIA_SHARE, like prewrite_rec_hook already had
- instead of a bool saying if this record's type ends groups or not,
we refine: it may not end a group, it may end a group, or it may
be a group in itself. Imagine that we had a physical write failure
to a table before we log the UNDO, we still end up in
external_lock(F_UNLCK) and then we log a COMMIT: we don't want
to consider this COMMIT as ending the group of REDOs (don't want
to execute those REDOs during Recovery), that's why we say "COMMIT
is a group in itself, it aborts any previous group". This also
gives one more sanity check in maria_read_log.
storage/maria/ma_recovery.c:
New Recovery code, replacing the old pseudocode.
Most of maria_read_log moved here.
Call-able from ha_maria, but not enabled yet.
Compared to the previous version of maria_read_log, some bugs have
been fixed, debugging output can go to stdout or a disk file (for now
it's useful for me, later it can be changed), execution of
REDO_DROP_TABLE, REDO_DELETE_ALL, REDO_PURGE_BLOCKS has been added. Duplicate code
has been factored into functions. We abort an unfinished group
of records if we see a record which is a group in itself (like COMMIT).
No need for maria_panic() after a bug (which caused tables to not
be closed) was fixed; if there is yet another bug I prefer to see it.
When opening a table for Recovery, set data_file_length
and key_file_length to their real physical value (these are the
easiest state members to restore :). Warn us if the last page
was truncated (but Recovery handles it).
MARIA_SHARE::state::state::records is now partly recovered (not
idempotent, but works if recreating tables from scracth).
When applying a REDO to a page, stamp it with the UNDO's LSN
(current_group_end_lsn), not with the REDO's LSN; it makes
the table more identical to the original table (easier to compare
the two tables in the end).
Big thing missing: some types of REDOs are not handled,
and the UNDO phase does not exist (missing functions to execute UNDOs
to actually rollback). So for now tests are only inserting/deleting
a few 100 rows, closing the table and seeing if the log is applied ok;
it works. UPDATE not handled.
storage/maria/ma_recovery.h:
new functions: ma_recover() for recovery from inside ha_maria;
_ma_apply_log() for maria_read_log (ma_recover() calls _ma_apply_log()).
Btw, we need to not use the word "recover" for REPAIR/maria_chk anymore.
storage/maria/ma_rename.c:
don't write log records during recovery
storage/maria/ma_test2.c:
- fail if maria_info() or other subtests find some wrong information
- new option -g to skip updates.
- init the translog before creating the table, so that log applying
can work.
- in "#if 0" you'll see some fixed bugs (will be removed).
storage/maria/ma_test_all.sh:
cleanup files. Test log applying.
storage/maria/maria_read_log.c:
most of the logic moves to ma_recovery.c to be shared between
maria_read_log and recovery-from-inside-mysqld.
See ma_recovery.c for additional changes made to the moved code.
storage/maria/ma_test_recovery:
unit test for Recovery. Tests insert and delete,
REDO_UPDATE not yet coded.
Script is called from ma_test_all. Can run standalone.
2007-07-26 11:56:21 +02:00
|
|
|
{
|
|
|
|
fprintf(stderr, "Error in initialization");
|
|
|
|
exit(1);
|
|
|
|
}
|
|
|
|
|
2006-04-11 15:45:10 +02:00
|
|
|
reclength=STANDARD_LENGTH+60+(use_blob ? 8 : 0);
|
|
|
|
blob_pos=STANDARD_LENGTH+60;
|
|
|
|
keyinfo[0].seg= &glob_keyseg[0][0];
|
|
|
|
keyinfo[0].seg[0].start=0;
|
|
|
|
keyinfo[0].seg[0].length=6;
|
|
|
|
keyinfo[0].seg[0].type=HA_KEYTYPE_TEXT;
|
|
|
|
keyinfo[0].seg[0].language= default_charset_info->number;
|
|
|
|
keyinfo[0].seg[0].flag=(uint8) pack_seg;
|
|
|
|
keyinfo[0].seg[0].null_bit=0;
|
|
|
|
keyinfo[0].seg[0].null_pos=0;
|
|
|
|
keyinfo[0].key_alg=HA_KEY_ALG_BTREE;
|
|
|
|
keyinfo[0].keysegs=1;
|
|
|
|
keyinfo[0].flag = pack_type;
|
Completion of merge of mysql-5.1 into mysql-maria.
Manually imported changes done to MyISAM (include/myisam.h,
storage/myisam/*, sql/ha_myisam.*, mysql-test/t/myisam.test,
mysql-test/t/ps_2myisam.test) the last
months into Maria (tedious, should do it more frequently in the
future), including those not done at the previous 5.1->Maria merge
(please in the future don't forget to apply MyISAM changes to Maria
when you merge 5.1 into Maria).
Note: I didn't try to import anything which could be MyISAM-related
in other tests of mysql-test (I didn't want to dig in all csets),
but as QA is working to make most tests re-usable for other engines
(Falcon), it is likely that we'll benefit from this and just have
to set engine=Maria somewhere to run those tests on Maria.
func_group and partition tests fail but they already do in main 5.1
on my machine. No Valgrind error in t/*maria*.test.
Monty: please see the commit comment of maria.result and check.
BitKeeper/deleted/.del-ha_maria.m4:
Delete: config/ac-macros/ha_maria.m4
configure.in:
fix for the new way of enabling engines
include/maria.h:
importing changes done to MyISAM the last months into Maria
include/my_handler.h:
importing changes done to MyISAM the last months into Maria
include/myisam.h:
importing changes done to MyISAM the last months into Maria
mysql-test/r/maria.result:
identical to myisam.result, except the engine name in some places
AND in the line testing key_block_size=1000000000000000000:
Maria gives a key block size of 8192 while MyISAM gives 4096;
is it explainable by the difference between MARIA_KEY_BLOCK_LENGTH
and the same constant in MyISAM? Monty?
mysql-test/r/ps_maria.result:
identical to ps_2myisam.result (except the engine name in some places)
mysql-test/t/maria.test:
instead of engine=maria everywhere, I use @@storage_engine (reduces
the diff with myisam.test).
importing changes done to MyISAM the last months into Maria
mysys/my_handler.c:
importing changes done to MyISAM the last months into Maria
sql/ha_maria.cc:
importing changes done to MyISAM the last months into Maria
sql/ha_maria.h:
importing changes done to MyISAM the last months into Maria
sql/mysqld.cc:
unneeded
storage/maria/Makefile.am:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_check.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_create.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_delete_table.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_dynrec.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_extra.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_ft_boolean_search.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_ft_eval.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_ft_nlq_search.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_ft_parser.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_ft_test1.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_ft_update.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_ftdefs.h:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_key.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_open.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_page.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_rkey.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_rsamepos.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_rt_index.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_rt_mbr.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_search.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_sort.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_test1.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_test2.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_test3.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_update.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_write.c:
importing changes done to MyISAM the last months into Maria
storage/maria/maria_chk.c:
importing changes done to MyISAM the last months into Maria
storage/maria/maria_def.h:
importing changes done to MyISAM the last months into Maria
storage/maria/maria_ftdump.c:
importing changes done to MyISAM the last months into Maria
storage/maria/maria_pack.c:
importing changes done to MyISAM the last months into Maria
2006-08-10 16:36:54 +02:00
|
|
|
keyinfo[0].block_length= 0; /* Default block length */
|
2006-04-11 15:45:10 +02:00
|
|
|
keyinfo[1].seg= &glob_keyseg[1][0];
|
|
|
|
keyinfo[1].seg[0].start=7;
|
|
|
|
keyinfo[1].seg[0].length=6;
|
|
|
|
keyinfo[1].seg[0].type=HA_KEYTYPE_BINARY;
|
|
|
|
keyinfo[1].seg[0].flag=0;
|
|
|
|
keyinfo[1].seg[0].null_bit=0;
|
|
|
|
keyinfo[1].seg[0].null_pos=0;
|
|
|
|
keyinfo[1].seg[1].start=0; /* two part key */
|
|
|
|
keyinfo[1].seg[1].length=6;
|
|
|
|
keyinfo[1].seg[1].type=HA_KEYTYPE_NUM;
|
|
|
|
keyinfo[1].seg[1].flag=HA_REVERSE_SORT;
|
|
|
|
keyinfo[1].seg[1].null_bit=0;
|
|
|
|
keyinfo[1].seg[1].null_pos=0;
|
|
|
|
keyinfo[1].key_alg=HA_KEY_ALG_BTREE;
|
|
|
|
keyinfo[1].keysegs=2;
|
|
|
|
keyinfo[1].flag =0;
|
Completion of merge of mysql-5.1 into mysql-maria.
Manually imported changes done to MyISAM (include/myisam.h,
storage/myisam/*, sql/ha_myisam.*, mysql-test/t/myisam.test,
mysql-test/t/ps_2myisam.test) the last
months into Maria (tedious, should do it more frequently in the
future), including those not done at the previous 5.1->Maria merge
(please in the future don't forget to apply MyISAM changes to Maria
when you merge 5.1 into Maria).
Note: I didn't try to import anything which could be MyISAM-related
in other tests of mysql-test (I didn't want to dig in all csets),
but as QA is working to make most tests re-usable for other engines
(Falcon), it is likely that we'll benefit from this and just have
to set engine=Maria somewhere to run those tests on Maria.
func_group and partition tests fail but they already do in main 5.1
on my machine. No Valgrind error in t/*maria*.test.
Monty: please see the commit comment of maria.result and check.
BitKeeper/deleted/.del-ha_maria.m4:
Delete: config/ac-macros/ha_maria.m4
configure.in:
fix for the new way of enabling engines
include/maria.h:
importing changes done to MyISAM the last months into Maria
include/my_handler.h:
importing changes done to MyISAM the last months into Maria
include/myisam.h:
importing changes done to MyISAM the last months into Maria
mysql-test/r/maria.result:
identical to myisam.result, except the engine name in some places
AND in the line testing key_block_size=1000000000000000000:
Maria gives a key block size of 8192 while MyISAM gives 4096;
is it explainable by the difference between MARIA_KEY_BLOCK_LENGTH
and the same constant in MyISAM? Monty?
mysql-test/r/ps_maria.result:
identical to ps_2myisam.result (except the engine name in some places)
mysql-test/t/maria.test:
instead of engine=maria everywhere, I use @@storage_engine (reduces
the diff with myisam.test).
importing changes done to MyISAM the last months into Maria
mysys/my_handler.c:
importing changes done to MyISAM the last months into Maria
sql/ha_maria.cc:
importing changes done to MyISAM the last months into Maria
sql/ha_maria.h:
importing changes done to MyISAM the last months into Maria
sql/mysqld.cc:
unneeded
storage/maria/Makefile.am:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_check.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_create.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_delete_table.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_dynrec.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_extra.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_ft_boolean_search.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_ft_eval.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_ft_nlq_search.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_ft_parser.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_ft_test1.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_ft_update.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_ftdefs.h:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_key.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_open.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_page.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_rkey.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_rsamepos.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_rt_index.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_rt_mbr.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_search.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_sort.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_test1.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_test2.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_test3.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_update.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_write.c:
importing changes done to MyISAM the last months into Maria
storage/maria/maria_chk.c:
importing changes done to MyISAM the last months into Maria
storage/maria/maria_def.h:
importing changes done to MyISAM the last months into Maria
storage/maria/maria_ftdump.c:
importing changes done to MyISAM the last months into Maria
storage/maria/maria_pack.c:
importing changes done to MyISAM the last months into Maria
2006-08-10 16:36:54 +02:00
|
|
|
keyinfo[1].block_length= MARIA_MIN_KEY_BLOCK_LENGTH; /* Diff blocklength */
|
2006-04-11 15:45:10 +02:00
|
|
|
keyinfo[2].seg= &glob_keyseg[2][0];
|
|
|
|
keyinfo[2].seg[0].start=12;
|
|
|
|
keyinfo[2].seg[0].length=8;
|
|
|
|
keyinfo[2].seg[0].type=HA_KEYTYPE_BINARY;
|
|
|
|
keyinfo[2].seg[0].flag=HA_REVERSE_SORT;
|
|
|
|
keyinfo[2].seg[0].null_bit=0;
|
|
|
|
keyinfo[2].seg[0].null_pos=0;
|
|
|
|
keyinfo[2].key_alg=HA_KEY_ALG_BTREE;
|
|
|
|
keyinfo[2].keysegs=1;
|
|
|
|
keyinfo[2].flag =HA_NOSAME;
|
Completion of merge of mysql-5.1 into mysql-maria.
Manually imported changes done to MyISAM (include/myisam.h,
storage/myisam/*, sql/ha_myisam.*, mysql-test/t/myisam.test,
mysql-test/t/ps_2myisam.test) the last
months into Maria (tedious, should do it more frequently in the
future), including those not done at the previous 5.1->Maria merge
(please in the future don't forget to apply MyISAM changes to Maria
when you merge 5.1 into Maria).
Note: I didn't try to import anything which could be MyISAM-related
in other tests of mysql-test (I didn't want to dig in all csets),
but as QA is working to make most tests re-usable for other engines
(Falcon), it is likely that we'll benefit from this and just have
to set engine=Maria somewhere to run those tests on Maria.
func_group and partition tests fail but they already do in main 5.1
on my machine. No Valgrind error in t/*maria*.test.
Monty: please see the commit comment of maria.result and check.
BitKeeper/deleted/.del-ha_maria.m4:
Delete: config/ac-macros/ha_maria.m4
configure.in:
fix for the new way of enabling engines
include/maria.h:
importing changes done to MyISAM the last months into Maria
include/my_handler.h:
importing changes done to MyISAM the last months into Maria
include/myisam.h:
importing changes done to MyISAM the last months into Maria
mysql-test/r/maria.result:
identical to myisam.result, except the engine name in some places
AND in the line testing key_block_size=1000000000000000000:
Maria gives a key block size of 8192 while MyISAM gives 4096;
is it explainable by the difference between MARIA_KEY_BLOCK_LENGTH
and the same constant in MyISAM? Monty?
mysql-test/r/ps_maria.result:
identical to ps_2myisam.result (except the engine name in some places)
mysql-test/t/maria.test:
instead of engine=maria everywhere, I use @@storage_engine (reduces
the diff with myisam.test).
importing changes done to MyISAM the last months into Maria
mysys/my_handler.c:
importing changes done to MyISAM the last months into Maria
sql/ha_maria.cc:
importing changes done to MyISAM the last months into Maria
sql/ha_maria.h:
importing changes done to MyISAM the last months into Maria
sql/mysqld.cc:
unneeded
storage/maria/Makefile.am:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_check.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_create.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_delete_table.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_dynrec.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_extra.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_ft_boolean_search.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_ft_eval.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_ft_nlq_search.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_ft_parser.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_ft_test1.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_ft_update.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_ftdefs.h:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_key.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_open.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_page.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_rkey.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_rsamepos.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_rt_index.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_rt_mbr.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_search.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_sort.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_test1.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_test2.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_test3.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_update.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_write.c:
importing changes done to MyISAM the last months into Maria
storage/maria/maria_chk.c:
importing changes done to MyISAM the last months into Maria
storage/maria/maria_def.h:
importing changes done to MyISAM the last months into Maria
storage/maria/maria_ftdump.c:
importing changes done to MyISAM the last months into Maria
storage/maria/maria_pack.c:
importing changes done to MyISAM the last months into Maria
2006-08-10 16:36:54 +02:00
|
|
|
keyinfo[2].block_length= 0; /* Default block length */
|
2006-04-11 15:45:10 +02:00
|
|
|
keyinfo[3].seg= &glob_keyseg[3][0];
|
|
|
|
keyinfo[3].seg[0].start=0;
|
|
|
|
keyinfo[3].seg[0].length=reclength-(use_blob ? 8 : 0);
|
|
|
|
keyinfo[3].seg[0].type=HA_KEYTYPE_TEXT;
|
|
|
|
keyinfo[3].seg[0].language=default_charset_info->number;
|
|
|
|
keyinfo[3].seg[0].flag=(uint8) pack_seg;
|
|
|
|
keyinfo[3].seg[0].null_bit=0;
|
|
|
|
keyinfo[3].seg[0].null_pos=0;
|
|
|
|
keyinfo[3].key_alg=HA_KEY_ALG_BTREE;
|
|
|
|
keyinfo[3].keysegs=1;
|
|
|
|
keyinfo[3].flag = pack_type;
|
Completion of merge of mysql-5.1 into mysql-maria.
Manually imported changes done to MyISAM (include/myisam.h,
storage/myisam/*, sql/ha_myisam.*, mysql-test/t/myisam.test,
mysql-test/t/ps_2myisam.test) the last
months into Maria (tedious, should do it more frequently in the
future), including those not done at the previous 5.1->Maria merge
(please in the future don't forget to apply MyISAM changes to Maria
when you merge 5.1 into Maria).
Note: I didn't try to import anything which could be MyISAM-related
in other tests of mysql-test (I didn't want to dig in all csets),
but as QA is working to make most tests re-usable for other engines
(Falcon), it is likely that we'll benefit from this and just have
to set engine=Maria somewhere to run those tests on Maria.
func_group and partition tests fail but they already do in main 5.1
on my machine. No Valgrind error in t/*maria*.test.
Monty: please see the commit comment of maria.result and check.
BitKeeper/deleted/.del-ha_maria.m4:
Delete: config/ac-macros/ha_maria.m4
configure.in:
fix for the new way of enabling engines
include/maria.h:
importing changes done to MyISAM the last months into Maria
include/my_handler.h:
importing changes done to MyISAM the last months into Maria
include/myisam.h:
importing changes done to MyISAM the last months into Maria
mysql-test/r/maria.result:
identical to myisam.result, except the engine name in some places
AND in the line testing key_block_size=1000000000000000000:
Maria gives a key block size of 8192 while MyISAM gives 4096;
is it explainable by the difference between MARIA_KEY_BLOCK_LENGTH
and the same constant in MyISAM? Monty?
mysql-test/r/ps_maria.result:
identical to ps_2myisam.result (except the engine name in some places)
mysql-test/t/maria.test:
instead of engine=maria everywhere, I use @@storage_engine (reduces
the diff with myisam.test).
importing changes done to MyISAM the last months into Maria
mysys/my_handler.c:
importing changes done to MyISAM the last months into Maria
sql/ha_maria.cc:
importing changes done to MyISAM the last months into Maria
sql/ha_maria.h:
importing changes done to MyISAM the last months into Maria
sql/mysqld.cc:
unneeded
storage/maria/Makefile.am:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_check.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_create.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_delete_table.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_dynrec.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_extra.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_ft_boolean_search.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_ft_eval.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_ft_nlq_search.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_ft_parser.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_ft_test1.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_ft_update.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_ftdefs.h:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_key.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_open.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_page.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_rkey.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_rsamepos.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_rt_index.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_rt_mbr.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_search.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_sort.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_test1.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_test2.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_test3.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_update.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_write.c:
importing changes done to MyISAM the last months into Maria
storage/maria/maria_chk.c:
importing changes done to MyISAM the last months into Maria
storage/maria/maria_def.h:
importing changes done to MyISAM the last months into Maria
storage/maria/maria_ftdump.c:
importing changes done to MyISAM the last months into Maria
storage/maria/maria_pack.c:
importing changes done to MyISAM the last months into Maria
2006-08-10 16:36:54 +02:00
|
|
|
keyinfo[3].block_length= 0; /* Default block length */
|
2006-04-11 15:45:10 +02:00
|
|
|
keyinfo[4].seg= &glob_keyseg[4][0];
|
|
|
|
keyinfo[4].seg[0].start=0;
|
|
|
|
keyinfo[4].seg[0].length=5;
|
|
|
|
keyinfo[4].seg[0].type=HA_KEYTYPE_TEXT;
|
|
|
|
keyinfo[4].seg[0].language=default_charset_info->number;
|
|
|
|
keyinfo[4].seg[0].flag=0;
|
|
|
|
keyinfo[4].seg[0].null_bit=0;
|
|
|
|
keyinfo[4].seg[0].null_pos=0;
|
|
|
|
keyinfo[4].key_alg=HA_KEY_ALG_BTREE;
|
|
|
|
keyinfo[4].keysegs=1;
|
|
|
|
keyinfo[4].flag = pack_type;
|
Completion of merge of mysql-5.1 into mysql-maria.
Manually imported changes done to MyISAM (include/myisam.h,
storage/myisam/*, sql/ha_myisam.*, mysql-test/t/myisam.test,
mysql-test/t/ps_2myisam.test) the last
months into Maria (tedious, should do it more frequently in the
future), including those not done at the previous 5.1->Maria merge
(please in the future don't forget to apply MyISAM changes to Maria
when you merge 5.1 into Maria).
Note: I didn't try to import anything which could be MyISAM-related
in other tests of mysql-test (I didn't want to dig in all csets),
but as QA is working to make most tests re-usable for other engines
(Falcon), it is likely that we'll benefit from this and just have
to set engine=Maria somewhere to run those tests on Maria.
func_group and partition tests fail but they already do in main 5.1
on my machine. No Valgrind error in t/*maria*.test.
Monty: please see the commit comment of maria.result and check.
BitKeeper/deleted/.del-ha_maria.m4:
Delete: config/ac-macros/ha_maria.m4
configure.in:
fix for the new way of enabling engines
include/maria.h:
importing changes done to MyISAM the last months into Maria
include/my_handler.h:
importing changes done to MyISAM the last months into Maria
include/myisam.h:
importing changes done to MyISAM the last months into Maria
mysql-test/r/maria.result:
identical to myisam.result, except the engine name in some places
AND in the line testing key_block_size=1000000000000000000:
Maria gives a key block size of 8192 while MyISAM gives 4096;
is it explainable by the difference between MARIA_KEY_BLOCK_LENGTH
and the same constant in MyISAM? Monty?
mysql-test/r/ps_maria.result:
identical to ps_2myisam.result (except the engine name in some places)
mysql-test/t/maria.test:
instead of engine=maria everywhere, I use @@storage_engine (reduces
the diff with myisam.test).
importing changes done to MyISAM the last months into Maria
mysys/my_handler.c:
importing changes done to MyISAM the last months into Maria
sql/ha_maria.cc:
importing changes done to MyISAM the last months into Maria
sql/ha_maria.h:
importing changes done to MyISAM the last months into Maria
sql/mysqld.cc:
unneeded
storage/maria/Makefile.am:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_check.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_create.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_delete_table.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_dynrec.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_extra.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_ft_boolean_search.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_ft_eval.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_ft_nlq_search.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_ft_parser.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_ft_test1.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_ft_update.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_ftdefs.h:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_key.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_open.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_page.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_rkey.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_rsamepos.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_rt_index.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_rt_mbr.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_search.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_sort.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_test1.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_test2.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_test3.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_update.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_write.c:
importing changes done to MyISAM the last months into Maria
storage/maria/maria_chk.c:
importing changes done to MyISAM the last months into Maria
storage/maria/maria_def.h:
importing changes done to MyISAM the last months into Maria
storage/maria/maria_ftdump.c:
importing changes done to MyISAM the last months into Maria
storage/maria/maria_pack.c:
importing changes done to MyISAM the last months into Maria
2006-08-10 16:36:54 +02:00
|
|
|
keyinfo[4].block_length= 0; /* Default block length */
|
2006-04-11 15:45:10 +02:00
|
|
|
keyinfo[5].seg= &glob_keyseg[5][0];
|
|
|
|
keyinfo[5].seg[0].start=0;
|
|
|
|
keyinfo[5].seg[0].length=4;
|
|
|
|
keyinfo[5].seg[0].type=HA_KEYTYPE_TEXT;
|
|
|
|
keyinfo[5].seg[0].language=default_charset_info->number;
|
|
|
|
keyinfo[5].seg[0].flag=pack_seg;
|
|
|
|
keyinfo[5].seg[0].null_bit=0;
|
|
|
|
keyinfo[5].seg[0].null_pos=0;
|
|
|
|
keyinfo[5].key_alg=HA_KEY_ALG_BTREE;
|
|
|
|
keyinfo[5].keysegs=1;
|
|
|
|
keyinfo[5].flag = pack_type;
|
Completion of merge of mysql-5.1 into mysql-maria.
Manually imported changes done to MyISAM (include/myisam.h,
storage/myisam/*, sql/ha_myisam.*, mysql-test/t/myisam.test,
mysql-test/t/ps_2myisam.test) the last
months into Maria (tedious, should do it more frequently in the
future), including those not done at the previous 5.1->Maria merge
(please in the future don't forget to apply MyISAM changes to Maria
when you merge 5.1 into Maria).
Note: I didn't try to import anything which could be MyISAM-related
in other tests of mysql-test (I didn't want to dig in all csets),
but as QA is working to make most tests re-usable for other engines
(Falcon), it is likely that we'll benefit from this and just have
to set engine=Maria somewhere to run those tests on Maria.
func_group and partition tests fail but they already do in main 5.1
on my machine. No Valgrind error in t/*maria*.test.
Monty: please see the commit comment of maria.result and check.
BitKeeper/deleted/.del-ha_maria.m4:
Delete: config/ac-macros/ha_maria.m4
configure.in:
fix for the new way of enabling engines
include/maria.h:
importing changes done to MyISAM the last months into Maria
include/my_handler.h:
importing changes done to MyISAM the last months into Maria
include/myisam.h:
importing changes done to MyISAM the last months into Maria
mysql-test/r/maria.result:
identical to myisam.result, except the engine name in some places
AND in the line testing key_block_size=1000000000000000000:
Maria gives a key block size of 8192 while MyISAM gives 4096;
is it explainable by the difference between MARIA_KEY_BLOCK_LENGTH
and the same constant in MyISAM? Monty?
mysql-test/r/ps_maria.result:
identical to ps_2myisam.result (except the engine name in some places)
mysql-test/t/maria.test:
instead of engine=maria everywhere, I use @@storage_engine (reduces
the diff with myisam.test).
importing changes done to MyISAM the last months into Maria
mysys/my_handler.c:
importing changes done to MyISAM the last months into Maria
sql/ha_maria.cc:
importing changes done to MyISAM the last months into Maria
sql/ha_maria.h:
importing changes done to MyISAM the last months into Maria
sql/mysqld.cc:
unneeded
storage/maria/Makefile.am:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_check.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_create.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_delete_table.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_dynrec.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_extra.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_ft_boolean_search.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_ft_eval.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_ft_nlq_search.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_ft_parser.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_ft_test1.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_ft_update.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_ftdefs.h:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_key.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_open.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_page.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_rkey.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_rsamepos.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_rt_index.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_rt_mbr.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_search.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_sort.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_test1.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_test2.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_test3.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_update.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_write.c:
importing changes done to MyISAM the last months into Maria
storage/maria/maria_chk.c:
importing changes done to MyISAM the last months into Maria
storage/maria/maria_def.h:
importing changes done to MyISAM the last months into Maria
storage/maria/maria_ftdump.c:
importing changes done to MyISAM the last months into Maria
storage/maria/maria_pack.c:
importing changes done to MyISAM the last months into Maria
2006-08-10 16:36:54 +02:00
|
|
|
keyinfo[5].block_length= 0; /* Default block length */
|
2006-04-11 15:45:10 +02:00
|
|
|
|
|
|
|
recinfo[0].type=pack_fields ? FIELD_SKIP_PRESPACE : 0;
|
|
|
|
recinfo[0].length=7;
|
|
|
|
recinfo[0].null_bit=0;
|
|
|
|
recinfo[0].null_pos=0;
|
|
|
|
recinfo[1].type=pack_fields ? FIELD_SKIP_PRESPACE : 0;
|
|
|
|
recinfo[1].length=5;
|
|
|
|
recinfo[1].null_bit=0;
|
|
|
|
recinfo[1].null_pos=0;
|
|
|
|
recinfo[2].type=pack_fields ? FIELD_SKIP_PRESPACE : 0;
|
|
|
|
recinfo[2].length=9;
|
|
|
|
recinfo[2].null_bit=0;
|
|
|
|
recinfo[2].null_pos=0;
|
|
|
|
recinfo[3].type=FIELD_NORMAL;
|
|
|
|
recinfo[3].length=STANDARD_LENGTH-7-5-9-4;
|
|
|
|
recinfo[3].null_bit=0;
|
|
|
|
recinfo[3].null_pos=0;
|
|
|
|
recinfo[4].type=pack_fields ? FIELD_SKIP_ZERO : 0;
|
|
|
|
recinfo[4].length=4;
|
|
|
|
recinfo[4].null_bit=0;
|
|
|
|
recinfo[4].null_pos=0;
|
|
|
|
recinfo[5].type=pack_fields ? FIELD_SKIP_ENDSPACE : 0;
|
|
|
|
recinfo[5].length=60;
|
|
|
|
recinfo[5].null_bit=0;
|
|
|
|
recinfo[5].null_pos=0;
|
|
|
|
if (use_blob)
|
|
|
|
{
|
|
|
|
recinfo[6].type=FIELD_BLOB;
|
2007-04-19 12:18:56 +02:00
|
|
|
recinfo[6].length=4+portable_sizeof_char_ptr;
|
2006-04-11 15:45:10 +02:00
|
|
|
recinfo[6].null_bit=0;
|
|
|
|
recinfo[6].null_pos=0;
|
|
|
|
}
|
|
|
|
|
|
|
|
write_count=update=dupp_keys=opt_delete=0;
|
|
|
|
blob_buffer=0;
|
|
|
|
|
|
|
|
for (i=1000 ; i>0 ; i--) key1[i]=0;
|
|
|
|
for (i=4999 ; i>0 ; i--) key3[i]=0;
|
|
|
|
|
|
|
|
if (!silent)
|
2007-01-18 20:38:14 +01:00
|
|
|
printf("- Creating maria-file\n");
|
2006-04-11 15:45:10 +02:00
|
|
|
file= 0;
|
|
|
|
bzero((char*) &create_info,sizeof(create_info));
|
|
|
|
create_info.max_rows=(ha_rows) (rec_pointer_size ?
|
|
|
|
(1L << (rec_pointer_size*8))/
|
|
|
|
reclength : 0);
|
|
|
|
create_info.reloc_rows=(ha_rows) 100;
|
This patch is a collection of patches from from Sanja, Sergei and Monty.
Added logging and pinning of pages to block format.
Integration of transaction manager, log handler.
Better page cache intergration
Split trnman.h into two files, so that we don't have to include my_atomic.h into C++ programs.
Renaming of structures, more comments, more debugging etc.
Fixed problem with small head block + long varchar.
Added extra argument to delete_record() and update_record() (needed for UNDO logging)
Small changes to interface of pagecache and log handler.
Change initialization of log_record_type_descriptors to not be depending on enum order.
Use array of LEX_STRING's to send data to log handler
Added 'dummy' transaction option to MARIA_INFO so that we can always assume 'trn' exists.
include/lf.h:
Interface fixes
Rename of structures
(Patch from Sergei via Sanja)
include/my_atomic.h:
More comments
include/my_global.h:
Added MY_ERRPTR
include/pagecache.h:
Added undo LSN when unlocking pages
mysql-test/r/maria.result:
Updated results
mysql-test/t/maria.test:
Added autocommit around lock tables
(Patch from Sanja)
mysys/lf_alloc-pin.c:
Post-review fixes, simple optimizations
More comments
Struct slot renames
Check amount of memory on stack
(Patch from Sergei)
mysys/lf_dynarray.c:
More comments
mysys/lf_hash.c:
More comments
After review fixes
(Patch from Sergei)
storage/maria/ha_maria.cc:
Split trnman.h into two files, so that we don't have to include my_atomic.h into the .cc program.
(Temporary fix to avoid bug in gcc)
Move out all deferencing of the transaction structure.
Transaction manager integrated (Patch from Sergei)
storage/maria/ha_maria.h:
Added prototype for start_stmt()
storage/maria/lockman.c:
Function call rename
storage/maria/ma_bitmap.c:
Mark deleted pages free from page cache
storage/maria/ma_blockrec.c:
Offset -> rownr
More debugging
Fixed problem with small head block + long varchar
Added logging of changed pages
Added logging of undo (Including only loggging of changed fields in case of update)
Added pinning/unpinning of all changed pages
More comments
Added free_full_pages() as the same code was used in several places.
fill_rows_parts() renamed as fill_insert_undo_parts()
offset -> rownr
Added some optimization of not transactional tables
_ma_update_block_record() has new parameter, as we need original row to do efficent undo for update
storage/maria/ma_blockrec.h:
Added ROW_EXTENTS_ON_STACK
Changed prototype for update and delete of row
storage/maria/ma_check.c:
Added original row to delete_record() call
storage/maria/ma_control_file.h:
Added ifdefs for C++
storage/maria/ma_delete.c:
Added original row to delete_record() call
(Needed for efficent undo logging)
storage/maria/ma_dynrec.c:
Added extra argument to delete_record() and update_record()
Removed not used variable
storage/maria/ma_init.c:
Initialize log handler
storage/maria/ma_loghandler.c:
Removed not used variable
Change initialization of log_record_type_descriptors to not be depending on enum order
Use array of LEX_STRING's to send data to log handler
storage/maria/ma_loghandler.h:
New defines
Use array of LEX_STRING's to send data to log handler
storage/maria/ma_open.c:
Added 'dummy' transaction option to MARIA_INFO so that we can always assume 'trn' exists.
Store in MARIA_SHARE->page_type if pages will have up to date LSN's
storage/maria/ma_pagecache.c:
Don't decrease number of readers when using pagecache_write()/pagecache_read()
In pagecache_write() decrement request count if page was left pinned
Added pagecache_delete_pages()
Removed some casts
Make trace output consistent with rest of code
Simplify calling of DBUG_ASSERT(0)
Only update LSN if the LSN is bigger than what's already on the page
Added LSN parameter pagecache_unpin_page(), pagecache_unpin(), and pagecache_unlock()
(Part of patch from Sanja)
storage/maria/ma_static.c:
Added 'dummy' transaction option to MARIA_INFO so that we can always assume 'trn' exists.
Added default page cache
storage/maria/ma_statrec.c:
Added extra argument to delete_record() and update_record()
storage/maria/ma_test1.c:
Added option -T for transactions
storage/maria/ma_test2.c:
Added option -T for transactions
storage/maria/ma_test_all.sh:
Test with transactions
storage/maria/ma_update.c:
Changed prototype for update of row
storage/maria/maria_def.h:
Changed prototype for update & delete of row as block records need to access the old row
Store in MARIA_SHARE->page_type if pages will have up to date LSN's
Added MARIA_MAX_TREE_LEVELS to allow us to calculate the number of possible pinned pages we may need.
Removed not used 'empty_bits_buffer'
Added pointer to transaction object
Added array for pinned pages
Added log_row_parts array for logging of field data.
Added MARIA_PINNED_PAGE to store pinned pages
storage/maria/trnman.c:
Added accessor functions to transaction object
Added missing DBUG_RETURN()
More debugging
More comments
Changed // comment of code to #ifdef NOT_USED
Transaction manager integrated.
Post review fixes
Part of patch originally from Sergei
storage/maria/trnman.h:
Split trnman.h into two files, so that we don't have to include my_atomic.h into the .cc program.
(Temporary fix to avoid bug in gcc)
storage/maria/unittest/ma_pagecache_single.c:
Added missing argument
Added SKIP_BIG_TESTS
(Patch from Sanja)
storage/maria/unittest/ma_test_loghandler-t.c:
Test logging with new LEX_STRING parameter
(Patch from Sanja)
storage/maria/unittest/ma_test_loghandler_multigroup-t.c:
Test logging with new LEX_STRING parameter
(Patch from Sanja)
storage/maria/unittest/ma_test_loghandler_multithread-t.c:
Test logging with new LEX_STRING parameter
(Patch from Sanja)
storage/maria/unittest/ma_test_loghandler_pagecache-t.c:
Test logging with new LEX_STRING parameter
(Patch from Sanja)
storage/maria/unittest/trnman-t.c:
Stack overflow detection
(Patch from Sergei)
unittest/unit.pl:
Command-line options --big and --verbose
(Patch from Sergei)
unittest/mytap/tap.c:
Detect --big
(Patch from Sergei)
unittest/mytap/tap.h:
Skip_big_tests and SKIP_BIG_TESTS
(Patch from Sergei)
storage/maria/trnman_public.h:
New BitKeeper file ``storage/maria/trnman_public.h''
2007-05-29 19:13:56 +02:00
|
|
|
create_info.transactional= transactional;
|
2007-01-18 20:38:14 +01:00
|
|
|
if (maria_create(filename, record_type, keys,&keyinfo[first_key],
|
2006-04-11 15:45:10 +02:00
|
|
|
use_blob ? 7 : 6, &recinfo[0],
|
|
|
|
0,(MARIA_UNIQUEDEF*) 0,
|
|
|
|
&create_info,create_flag))
|
|
|
|
goto err;
|
|
|
|
if (!(file=maria_open(filename,2,HA_OPEN_ABORT_IF_LOCKED)))
|
|
|
|
goto err;
|
WL#3071 Maria checkpoint
Finally this is the real checkpoint code.
It however exhibits unstabilities when a checkpoint runs concurrently
with data-modifying clients (table corruption, transaction log's
assertions) so for now a checkpoint is taken only at startup after
recovery and at shutdown, i.e. not in concurrent situations. Later
we will let it run periodically, as well as flush dirty pages
periodically (almost all needed code is there already, only pagecache
code is written but not committed).
WL#3072 Maria recovery
* replacing UNDO_ROW_PURGE with CLR_END; testing of those CLR_END via
ma_test2 which has INSERTs failing with duplicate keys.
* replaying of REDO_RENAME_TABLE
Now, off to test Recovery in ha_maria :)
BitKeeper/deleted/.del-ma_least_recently_dirtied.c:
Delete: storage/maria/ma_least_recently_dirtied.c
BitKeeper/deleted/.del-ma_least_recently_dirtied.h:
Delete: storage/maria/ma_least_recently_dirtied.h
storage/maria/Makefile.am:
compile Checkpoint module
storage/maria/ha_maria.cc:
When ha_maria starts, do a recovery from last checkpoint.
Take a checkpoint when that recovery has ended and when ha_maria
shuts down cleanly.
storage/maria/ma_blockrec.c:
* even if my_sync() fails we have to my_close() (otherwise we leak
a descriptor)
* UNDO_ROW_PURGE is replaced by a simple CLR_END for UNDO_ROW_INSERT,
as promised in the old comment; it gives us skipping during the
UNDO phase.
storage/maria/ma_check.c:
All REDOs before create_rename_lsn are ignored by Recovery. So
create_rename_lsn must be set only after all data/index has been
flushed and forced to disk. We thus move write_log_record_for_repair()
to after _ma_flush_tables_files_after_repair().
storage/maria/ma_checkpoint.c:
Checkpoint module.
storage/maria/ma_checkpoint.h:
optional argument if caller wants a thread to periodically take
checkpoints and flush dirty pages.
storage/maria/ma_create.c:
* no need to init some vars as the initial bzero(share) takes care of this.
* update to new function's name
* even if we fail in my_sync() we have to my_close()
storage/maria/ma_extra.c:
Checkpoint reads share->last_version under intern_lock, so we make
maria_extra() update it under intern_lock. THR_LOCK_maria still needed
because of _ma_test_if_reopen().
storage/maria/ma_init.c:
destroy checkpoint module when Maria shuts down.
storage/maria/ma_loghandler.c:
* UNDO_ROW_PURGE gone (see ma_blockrec.c)
* we need to remember the LSN of the LOGREC_FILE_ID for a share,
because this LSN is needed into the checkpoint record (Recovery wants
to know the validity domain of an id->name mapping)
* translog_get_horizon_no_lock() needed for Checkpoint
* comment about failing assertion (Sanja knows)
* translog_init_reader_data() thought that translog_read_record_header_scan()
returns 0 in case of error, but 0 just means "0-length header".
* translog_assign_id_to_share() now needs the MARIA_HA because
LOGREC_FILE_ID uses a log-write hook.
* Verify that (de)assignment of share->id happens only under intern_lock,
as Checkpoint reads this id with intern_lock.
* translog_purge() can accept TRANSLOG_ADDRESS, not necessarily
a real LSN.
storage/maria/ma_loghandler.h:
prototype updates
storage/maria/ma_open.c:
no need to initialize "res"
storage/maria/ma_pagecache.c:
When taking a checkpoint, we don't need to know the maximum rec_lsn
of dirty pages; this LSN was intended to be used in the two-checkpoint
rule, but last_checkpoint_lsn is as good.
4 bytes for stored_list_size is enough as PAGECACHE::blocks (number
of blocks which the pagecache can contain) is int.
storage/maria/ma_pagecache.h:
new prototype
storage/maria/ma_recovery.c:
* added replaying of REDO_RENAME_TABLE
* UNDO_ROW_PURGE gone (see ma_blockrec.c), replaced by CLR_END
* Recovery from the last checkpoint record now possible
* In new_table() we skip the table if the id->name mapping is older than
create_rename_lsn (mapping dates from lsn_of_file_id).
* in get_MARIA_HA_from_REDO_record() we skip the record
if the id->name mapping is newer than the record (can happen if processing
a record which is before the checkpoint record).
* parse_checkpoint_record() has to return a LSN, that's what caller expects
storage/maria/ma_rename.c:
new function's name; log end zeroes of tables' names (ease recovery)
storage/maria/ma_test2.c:
* equivalent of ma_test1's --test-undo added (named -u here).
* -t=1 now stops right after creating the table, so that
we can test undoing of INSERTs with duplicate keys (which tests the
CLR_END logged by _ma_write_abort_block_record()).
storage/maria/ma_test_recovery.expected:
Result of testing undoing of INSERTs with duplicate keys; there are
some differences in maria_chk -dvv but they are normal (removing
records does not shrink data/index file, does not put back the
"analyzed, optimized keys"(etc) index state.
storage/maria/ma_test_recovery:
Test undoing of INSERTs with duplicate keys, using ma_test2;
when such INSERT happens, it logs REDO_INSERT, UNDO_INSERT, REDO_DELETE,
CLR_END; we abort after that, and test that CLR_END causes recovery
to jump over UNDO_INSERT.
storage/maria/ma_write.c:
comment
storage/maria/maria_chk.c:
comment
storage/maria/maria_def.h:
* a new bit in MARIA_SHARE::in_checkpoint, used to build a list
of unique shares during Checkpoint.
* MARIA_SHARE::lsn_of_file_id added: the LSN of the last LOGREC_FILE_ID
for this share; needed to know to which LSN domain the mappings
found in the Checkpoint record apply (new mappings should not apply
to old REDOs).
storage/maria/trnman.c:
* small changes to how trnman_collect_transactions() fills its buffer;
it also uses a non-dummy lsn_read_non_atomic() found in ma_checkpoint.h
2007-09-12 11:27:34 +02:00
|
|
|
maria_begin(file);
|
|
|
|
if (testflag == 1)
|
|
|
|
goto end;
|
WL#3071 Maria checkpoint
Ability for flush_pagecache_blocks() to flush only certain pages of
a file, as instructed by an option "filter" pointer-to-function argument;
Checkpoint and background dirty page flushing use that to flush only
pages which have been dirty for long enough and bitmap pages.
Fix for a bug in flush_cached_blocks() (no idea if it could produce
a bug in real life, but theoretically it is).
Testing checkpoint in ma_test_recovery via ma_test1 and ma_test2.
Background checkpoint & dirty pages flush thread is still disabled
by default in ha_maria.
mysql-test/r/maria.result:
result update
storage/maria/ha_maria.cc:
blank after function comment
storage/maria/ma_checkpoint.c:
Using an enum instead of 0/1/2 (applying Sanja's review comments).
The comment about "this is an horizon" can be removed as Sanja
created translog_next_LSN() which parse_checkpoint_record() uses.
Variables in ma_checkpoint_background() cannot be declared in the
for() as their value must not be reset at each iteration!
storage/maria/ma_pagecache.c:
adding to flush_pagecache_blocks() optional arguments 'filter'
(pointer to function) and 'filter_arg'; if filter!=NULL this function
will be called for each block of the file and will reply if this
block and following ones should be flushed or not (3 possible
replies).
Fixing a bug when flush_cached_blocks() skips a pinned page: it has
to unset PCBLOCK_IN_FLUSH set by flush_pagecache_blocks_int().
storage/maria/ma_pagecache.h:
flush_pagecache_blocks() is changed to take "filter" and "filter_arg"
arguments. "filter", if it is not NULL, may return one value
among enum pagecache_flush_filter_result.
storage/maria/ma_recovery.c:
open_count=0 when closing tables at the end of recovery.
storage/maria/ma_test1.c:
Optional checkpoints (-H#) at various stages (stages similar
to --testflag), for testing of checkpoints.
storage/maria/ma_test2.c:
Optional checkpoints (-H#) at various stages (stages similar
to -t), for testing of checkpoints.
storage/maria/ma_test_recovery.expected:
Result update: the results of the additional test run with -H#
(checkpoints) are added here. They are exactly identical to without
checkpoints except that the index's Root (printed by maria_chk)
is more correct when using checkpoints. This is because checkpoint
flushed the state, so it happens to be correct, while no-checkpoint
does not flush the state, and recovery does not recover indexes
so Root is never fixed. When we recover indices, this will go away.
storage/maria/ma_test_recovery:
We duplicate the loop of tests to add an additional run with
checkpoints at various stages, to see if maria_read_log
uses them fine.
2007-10-17 16:55:26 +02:00
|
|
|
if (checkpoint == 1 && ma_checkpoint_execute(CHECKPOINT_MEDIUM, FALSE))
|
|
|
|
goto err;
|
2006-04-11 15:45:10 +02:00
|
|
|
if (!silent)
|
|
|
|
printf("- Writing key:s\n");
|
|
|
|
if (locking)
|
|
|
|
maria_lock_database(file,F_WRLCK);
|
|
|
|
if (write_cacheing)
|
|
|
|
maria_extra(file,HA_EXTRA_WRITE_CACHE,0);
|
|
|
|
if (opt_quick_mode)
|
|
|
|
maria_extra(file,HA_EXTRA_QUICK,0);
|
|
|
|
|
|
|
|
for (i=0 ; i < recant ; i++)
|
|
|
|
{
|
2007-01-18 20:38:14 +01:00
|
|
|
ulong blob_length;
|
WL#3072 - Maria recovery
Unit test for recovery: runs ma_test1 and ma_test2 (both only with
INSERTs and DELETEs; UPDATEs disabled as not handled by recovery)
then moves the tables elswhere; recreates tables from the log, and
compares and fails if there is a difference. Passes now.
Most of maria_read_log.c moved to ma_recovery.c, as it will be re-used
for recovery-from-ha_maria.
Bugfixes of applying of REDO_INSERT, REDO_PURGE_ROW.
Applying of REDO_PURGE_BLOCKS, REDO_DELETE_ALL, REDO_DROP_TABLE,
UNDO_ROW_INSERT (in REDO phase only, i.e. just doing records++),
UNDO_ROW_DELETE, UNDO_ROW_PURGE.
Code cleanups.
Monty: please look for "QQ". Sanja: please look for "Sanja".
Future tasks: recovery of the bitmap (easy), recovery of the state
(make it idempotent), more REDOs (Monty to work on
REDO_UPDATE?), UNDO phase...
Pushing this cset as it looks safe, contains test and bugfixes which
will help Monty implement applying of REDO_UPDATE.
sql/handler.cc:
typo
storage/maria/Makefile.am:
Adding ma_test_recovery (which ma_test_all invokes, and which can
also be run alone). Most of maria_read_log.c moved to ma_recovery.c
storage/maria/ha_maria.cc:
comments
storage/maria/ma_bitmap.c:
fixing comments. 2 -> sizeof(maria_bitmap_marker).
Bitmap-related part of _ma_initialize_datafile() moves in bitmap module.
Now putting the "bm" signature when creating the first bitmap page
(it used to happen only at next open, but that
caused an annoying difference when testing Recovery if the original
run didn't open the table, and it looks more
logical like this: it goes to disk only with its signature correct);
see the "QQ" comment towards the _ma_initialize_data_file() call
in ma_create.c for more).
When reading a bitmap page, verify its signature (happens when normally
using the table or when CHECKing it; not when REPAIRing it).
storage/maria/ma_blockrec.c:
* no need to sync the data file if table is not transactional
* Comments, code cleanup (log-related data moved to log-related code
block, int5store->page_store).
* Store the table's short id into LOGREC_UNDO_ROW_PURGE, like we
do for other records (though this record will soon be replaced
with a CLR).
* If "page" is 1 it means the page which extends from byte
page*block_size+1 to (page+1)*block_size (byte number 1 being
the first byte of the file). The last byte of the file is
data_file_length (same convention).
A new page needs to be created if the last byte of the page is
beyond the last byte of the file, i.e.
(page+1)*block_size+1 > data_file_length, so we correct the test
(bug found when testing log applying for ma_test1 -M -T --skip-update).
* update the page's LSN when removing a row from it during
execution of a REDO_PURGE_ROW record (bug found when testing log
applying for ma_test1 -M -T --skip-update).
* applying of REDO_PURGE_BLOCKs (limited to a one-page range for now).
storage/maria/ma_blockrec.h:
new functions. maria_bitmap_marker does not need to be exported.
storage/maria/ma_close.c:
we can always flush the table's state when closing the last instance
of the table. And it is needed for maria_read_log (as it does
not use maria_lock_database()).
storage/maria/ma_control_file.c:
when in Recovery, some assertions should not be used.
storage/maria/ma_control_file.h:
double-inclusion safe
storage/maria/ma_create.c:
during recovery, don't log records. Comments.
Moving the creation of the first bitmap page to ma_bitmap.c
storage/maria/ma_delete_table.c:
during recovery, don't log records. Log the end-zero of the dropped
table's name, so that recovery can use the string in place without
extending it to fit an end zero.
storage/maria/ma_loghandler.c:
* inwrite_rec_hook also needs access to the MARIA_SHARE, like
prewrite_rec_hook. This will be needed to update
share->records_diff (in the upcoming patch "recovery of the state").
* LOG_DESC::record_ends_group changed to an enum.
* LOG_DESC for LOGREC_REDO_PURGE_BLOCKS and LOGREC_UNDO_ROW_PURGE
corrected
* Sanja please see the @todo LOG BUG
* avoiding DBUG_RETURN(func()) as it gives confusing debug traces.
storage/maria/ma_loghandler.h:
- log write hooks called while the log's lock is held (inwrite_rec_hook)
now need the MARIA_SHARE, like prewrite_rec_hook already had
- instead of a bool saying if this record's type ends groups or not,
we refine: it may not end a group, it may end a group, or it may
be a group in itself. Imagine that we had a physical write failure
to a table before we log the UNDO, we still end up in
external_lock(F_UNLCK) and then we log a COMMIT: we don't want
to consider this COMMIT as ending the group of REDOs (don't want
to execute those REDOs during Recovery), that's why we say "COMMIT
is a group in itself, it aborts any previous group". This also
gives one more sanity check in maria_read_log.
storage/maria/ma_recovery.c:
New Recovery code, replacing the old pseudocode.
Most of maria_read_log moved here.
Call-able from ha_maria, but not enabled yet.
Compared to the previous version of maria_read_log, some bugs have
been fixed, debugging output can go to stdout or a disk file (for now
it's useful for me, later it can be changed), execution of
REDO_DROP_TABLE, REDO_DELETE_ALL, REDO_PURGE_BLOCKS has been added. Duplicate code
has been factored into functions. We abort an unfinished group
of records if we see a record which is a group in itself (like COMMIT).
No need for maria_panic() after a bug (which caused tables to not
be closed) was fixed; if there is yet another bug I prefer to see it.
When opening a table for Recovery, set data_file_length
and key_file_length to their real physical value (these are the
easiest state members to restore :). Warn us if the last page
was truncated (but Recovery handles it).
MARIA_SHARE::state::state::records is now partly recovered (not
idempotent, but works if recreating tables from scracth).
When applying a REDO to a page, stamp it with the UNDO's LSN
(current_group_end_lsn), not with the REDO's LSN; it makes
the table more identical to the original table (easier to compare
the two tables in the end).
Big thing missing: some types of REDOs are not handled,
and the UNDO phase does not exist (missing functions to execute UNDOs
to actually rollback). So for now tests are only inserting/deleting
a few 100 rows, closing the table and seeing if the log is applied ok;
it works. UPDATE not handled.
storage/maria/ma_recovery.h:
new functions: ma_recover() for recovery from inside ha_maria;
_ma_apply_log() for maria_read_log (ma_recover() calls _ma_apply_log()).
Btw, we need to not use the word "recover" for REPAIR/maria_chk anymore.
storage/maria/ma_rename.c:
don't write log records during recovery
storage/maria/ma_test2.c:
- fail if maria_info() or other subtests find some wrong information
- new option -g to skip updates.
- init the translog before creating the table, so that log applying
can work.
- in "#if 0" you'll see some fixed bugs (will be removed).
storage/maria/ma_test_all.sh:
cleanup files. Test log applying.
storage/maria/maria_read_log.c:
most of the logic moves to ma_recovery.c to be shared between
maria_read_log and recovery-from-inside-mysqld.
See ma_recovery.c for additional changes made to the moved code.
storage/maria/ma_test_recovery:
unit test for Recovery. Tests insert and delete,
REDO_UPDATE not yet coded.
Script is called from ma_test_all. Can run standalone.
2007-07-26 11:56:21 +02:00
|
|
|
#if 0
|
|
|
|
/*
|
|
|
|
Starting from i==72, there was a difference between runtime and
|
2007-08-29 08:03:10 +02:00
|
|
|
log-applying. This is now fixed, by not using non_header_data_len in
|
WL#3072 - Maria recovery
Unit test for recovery: runs ma_test1 and ma_test2 (both only with
INSERTs and DELETEs; UPDATEs disabled as not handled by recovery)
then moves the tables elswhere; recreates tables from the log, and
compares and fails if there is a difference. Passes now.
Most of maria_read_log.c moved to ma_recovery.c, as it will be re-used
for recovery-from-ha_maria.
Bugfixes of applying of REDO_INSERT, REDO_PURGE_ROW.
Applying of REDO_PURGE_BLOCKS, REDO_DELETE_ALL, REDO_DROP_TABLE,
UNDO_ROW_INSERT (in REDO phase only, i.e. just doing records++),
UNDO_ROW_DELETE, UNDO_ROW_PURGE.
Code cleanups.
Monty: please look for "QQ". Sanja: please look for "Sanja".
Future tasks: recovery of the bitmap (easy), recovery of the state
(make it idempotent), more REDOs (Monty to work on
REDO_UPDATE?), UNDO phase...
Pushing this cset as it looks safe, contains test and bugfixes which
will help Monty implement applying of REDO_UPDATE.
sql/handler.cc:
typo
storage/maria/Makefile.am:
Adding ma_test_recovery (which ma_test_all invokes, and which can
also be run alone). Most of maria_read_log.c moved to ma_recovery.c
storage/maria/ha_maria.cc:
comments
storage/maria/ma_bitmap.c:
fixing comments. 2 -> sizeof(maria_bitmap_marker).
Bitmap-related part of _ma_initialize_datafile() moves in bitmap module.
Now putting the "bm" signature when creating the first bitmap page
(it used to happen only at next open, but that
caused an annoying difference when testing Recovery if the original
run didn't open the table, and it looks more
logical like this: it goes to disk only with its signature correct);
see the "QQ" comment towards the _ma_initialize_data_file() call
in ma_create.c for more).
When reading a bitmap page, verify its signature (happens when normally
using the table or when CHECKing it; not when REPAIRing it).
storage/maria/ma_blockrec.c:
* no need to sync the data file if table is not transactional
* Comments, code cleanup (log-related data moved to log-related code
block, int5store->page_store).
* Store the table's short id into LOGREC_UNDO_ROW_PURGE, like we
do for other records (though this record will soon be replaced
with a CLR).
* If "page" is 1 it means the page which extends from byte
page*block_size+1 to (page+1)*block_size (byte number 1 being
the first byte of the file). The last byte of the file is
data_file_length (same convention).
A new page needs to be created if the last byte of the page is
beyond the last byte of the file, i.e.
(page+1)*block_size+1 > data_file_length, so we correct the test
(bug found when testing log applying for ma_test1 -M -T --skip-update).
* update the page's LSN when removing a row from it during
execution of a REDO_PURGE_ROW record (bug found when testing log
applying for ma_test1 -M -T --skip-update).
* applying of REDO_PURGE_BLOCKs (limited to a one-page range for now).
storage/maria/ma_blockrec.h:
new functions. maria_bitmap_marker does not need to be exported.
storage/maria/ma_close.c:
we can always flush the table's state when closing the last instance
of the table. And it is needed for maria_read_log (as it does
not use maria_lock_database()).
storage/maria/ma_control_file.c:
when in Recovery, some assertions should not be used.
storage/maria/ma_control_file.h:
double-inclusion safe
storage/maria/ma_create.c:
during recovery, don't log records. Comments.
Moving the creation of the first bitmap page to ma_bitmap.c
storage/maria/ma_delete_table.c:
during recovery, don't log records. Log the end-zero of the dropped
table's name, so that recovery can use the string in place without
extending it to fit an end zero.
storage/maria/ma_loghandler.c:
* inwrite_rec_hook also needs access to the MARIA_SHARE, like
prewrite_rec_hook. This will be needed to update
share->records_diff (in the upcoming patch "recovery of the state").
* LOG_DESC::record_ends_group changed to an enum.
* LOG_DESC for LOGREC_REDO_PURGE_BLOCKS and LOGREC_UNDO_ROW_PURGE
corrected
* Sanja please see the @todo LOG BUG
* avoiding DBUG_RETURN(func()) as it gives confusing debug traces.
storage/maria/ma_loghandler.h:
- log write hooks called while the log's lock is held (inwrite_rec_hook)
now need the MARIA_SHARE, like prewrite_rec_hook already had
- instead of a bool saying if this record's type ends groups or not,
we refine: it may not end a group, it may end a group, or it may
be a group in itself. Imagine that we had a physical write failure
to a table before we log the UNDO, we still end up in
external_lock(F_UNLCK) and then we log a COMMIT: we don't want
to consider this COMMIT as ending the group of REDOs (don't want
to execute those REDOs during Recovery), that's why we say "COMMIT
is a group in itself, it aborts any previous group". This also
gives one more sanity check in maria_read_log.
storage/maria/ma_recovery.c:
New Recovery code, replacing the old pseudocode.
Most of maria_read_log moved here.
Call-able from ha_maria, but not enabled yet.
Compared to the previous version of maria_read_log, some bugs have
been fixed, debugging output can go to stdout or a disk file (for now
it's useful for me, later it can be changed), execution of
REDO_DROP_TABLE, REDO_DELETE_ALL, REDO_PURGE_BLOCKS has been added. Duplicate code
has been factored into functions. We abort an unfinished group
of records if we see a record which is a group in itself (like COMMIT).
No need for maria_panic() after a bug (which caused tables to not
be closed) was fixed; if there is yet another bug I prefer to see it.
When opening a table for Recovery, set data_file_length
and key_file_length to their real physical value (these are the
easiest state members to restore :). Warn us if the last page
was truncated (but Recovery handles it).
MARIA_SHARE::state::state::records is now partly recovered (not
idempotent, but works if recreating tables from scracth).
When applying a REDO to a page, stamp it with the UNDO's LSN
(current_group_end_lsn), not with the REDO's LSN; it makes
the table more identical to the original table (easier to compare
the two tables in the end).
Big thing missing: some types of REDOs are not handled,
and the UNDO phase does not exist (missing functions to execute UNDOs
to actually rollback). So for now tests are only inserting/deleting
a few 100 rows, closing the table and seeing if the log is applied ok;
it works. UPDATE not handled.
storage/maria/ma_recovery.h:
new functions: ma_recover() for recovery from inside ha_maria;
_ma_apply_log() for maria_read_log (ma_recover() calls _ma_apply_log()).
Btw, we need to not use the word "recover" for REPAIR/maria_chk anymore.
storage/maria/ma_rename.c:
don't write log records during recovery
storage/maria/ma_test2.c:
- fail if maria_info() or other subtests find some wrong information
- new option -g to skip updates.
- init the translog before creating the table, so that log applying
can work.
- in "#if 0" you'll see some fixed bugs (will be removed).
storage/maria/ma_test_all.sh:
cleanup files. Test log applying.
storage/maria/maria_read_log.c:
most of the logic moves to ma_recovery.c to be shared between
maria_read_log and recovery-from-inside-mysqld.
See ma_recovery.c for additional changes made to the moved code.
storage/maria/ma_test_recovery:
unit test for Recovery. Tests insert and delete,
REDO_UPDATE not yet coded.
Script is called from ma_test_all. Can run standalone.
2007-07-26 11:56:21 +02:00
|
|
|
log-applying.
|
|
|
|
*/
|
|
|
|
if (i == 72) goto end;
|
|
|
|
#endif
|
2006-04-11 15:45:10 +02:00
|
|
|
n1=rnd(1000); n2=rnd(100); n3=rnd(5000);
|
2007-10-04 19:33:42 +02:00
|
|
|
sprintf((char*) record,"%6d:%4d:%8d:Pos: %4d ",n1,n2,n3,write_count);
|
2006-04-11 15:45:10 +02:00
|
|
|
int4store(record+STANDARD_LENGTH-4,(long) i);
|
|
|
|
fix_length(record,(uint) STANDARD_LENGTH+rnd(60));
|
2007-01-18 20:38:14 +01:00
|
|
|
put_blob_in_record(record+blob_pos,&blob_buffer, &blob_length);
|
|
|
|
DBUG_PRINT("test",("record: %d blob_length: %lu", i, blob_length));
|
2006-04-11 15:45:10 +02:00
|
|
|
|
|
|
|
if (maria_write(file,record))
|
|
|
|
{
|
|
|
|
if (my_errno != HA_ERR_FOUND_DUPP_KEY || key3[n3] == 0)
|
|
|
|
{
|
|
|
|
printf("Error: %d in write at record: %d\n",my_errno,i);
|
|
|
|
goto err;
|
|
|
|
}
|
WL#3072 - Maria recovery
Unit test for recovery: runs ma_test1 and ma_test2 (both only with
INSERTs and DELETEs; UPDATEs disabled as not handled by recovery)
then moves the tables elswhere; recreates tables from the log, and
compares and fails if there is a difference. Passes now.
Most of maria_read_log.c moved to ma_recovery.c, as it will be re-used
for recovery-from-ha_maria.
Bugfixes of applying of REDO_INSERT, REDO_PURGE_ROW.
Applying of REDO_PURGE_BLOCKS, REDO_DELETE_ALL, REDO_DROP_TABLE,
UNDO_ROW_INSERT (in REDO phase only, i.e. just doing records++),
UNDO_ROW_DELETE, UNDO_ROW_PURGE.
Code cleanups.
Monty: please look for "QQ". Sanja: please look for "Sanja".
Future tasks: recovery of the bitmap (easy), recovery of the state
(make it idempotent), more REDOs (Monty to work on
REDO_UPDATE?), UNDO phase...
Pushing this cset as it looks safe, contains test and bugfixes which
will help Monty implement applying of REDO_UPDATE.
sql/handler.cc:
typo
storage/maria/Makefile.am:
Adding ma_test_recovery (which ma_test_all invokes, and which can
also be run alone). Most of maria_read_log.c moved to ma_recovery.c
storage/maria/ha_maria.cc:
comments
storage/maria/ma_bitmap.c:
fixing comments. 2 -> sizeof(maria_bitmap_marker).
Bitmap-related part of _ma_initialize_datafile() moves in bitmap module.
Now putting the "bm" signature when creating the first bitmap page
(it used to happen only at next open, but that
caused an annoying difference when testing Recovery if the original
run didn't open the table, and it looks more
logical like this: it goes to disk only with its signature correct);
see the "QQ" comment towards the _ma_initialize_data_file() call
in ma_create.c for more).
When reading a bitmap page, verify its signature (happens when normally
using the table or when CHECKing it; not when REPAIRing it).
storage/maria/ma_blockrec.c:
* no need to sync the data file if table is not transactional
* Comments, code cleanup (log-related data moved to log-related code
block, int5store->page_store).
* Store the table's short id into LOGREC_UNDO_ROW_PURGE, like we
do for other records (though this record will soon be replaced
with a CLR).
* If "page" is 1 it means the page which extends from byte
page*block_size+1 to (page+1)*block_size (byte number 1 being
the first byte of the file). The last byte of the file is
data_file_length (same convention).
A new page needs to be created if the last byte of the page is
beyond the last byte of the file, i.e.
(page+1)*block_size+1 > data_file_length, so we correct the test
(bug found when testing log applying for ma_test1 -M -T --skip-update).
* update the page's LSN when removing a row from it during
execution of a REDO_PURGE_ROW record (bug found when testing log
applying for ma_test1 -M -T --skip-update).
* applying of REDO_PURGE_BLOCKs (limited to a one-page range for now).
storage/maria/ma_blockrec.h:
new functions. maria_bitmap_marker does not need to be exported.
storage/maria/ma_close.c:
we can always flush the table's state when closing the last instance
of the table. And it is needed for maria_read_log (as it does
not use maria_lock_database()).
storage/maria/ma_control_file.c:
when in Recovery, some assertions should not be used.
storage/maria/ma_control_file.h:
double-inclusion safe
storage/maria/ma_create.c:
during recovery, don't log records. Comments.
Moving the creation of the first bitmap page to ma_bitmap.c
storage/maria/ma_delete_table.c:
during recovery, don't log records. Log the end-zero of the dropped
table's name, so that recovery can use the string in place without
extending it to fit an end zero.
storage/maria/ma_loghandler.c:
* inwrite_rec_hook also needs access to the MARIA_SHARE, like
prewrite_rec_hook. This will be needed to update
share->records_diff (in the upcoming patch "recovery of the state").
* LOG_DESC::record_ends_group changed to an enum.
* LOG_DESC for LOGREC_REDO_PURGE_BLOCKS and LOGREC_UNDO_ROW_PURGE
corrected
* Sanja please see the @todo LOG BUG
* avoiding DBUG_RETURN(func()) as it gives confusing debug traces.
storage/maria/ma_loghandler.h:
- log write hooks called while the log's lock is held (inwrite_rec_hook)
now need the MARIA_SHARE, like prewrite_rec_hook already had
- instead of a bool saying if this record's type ends groups or not,
we refine: it may not end a group, it may end a group, or it may
be a group in itself. Imagine that we had a physical write failure
to a table before we log the UNDO, we still end up in
external_lock(F_UNLCK) and then we log a COMMIT: we don't want
to consider this COMMIT as ending the group of REDOs (don't want
to execute those REDOs during Recovery), that's why we say "COMMIT
is a group in itself, it aborts any previous group". This also
gives one more sanity check in maria_read_log.
storage/maria/ma_recovery.c:
New Recovery code, replacing the old pseudocode.
Most of maria_read_log moved here.
Call-able from ha_maria, but not enabled yet.
Compared to the previous version of maria_read_log, some bugs have
been fixed, debugging output can go to stdout or a disk file (for now
it's useful for me, later it can be changed), execution of
REDO_DROP_TABLE, REDO_DELETE_ALL, REDO_PURGE_BLOCKS has been added. Duplicate code
has been factored into functions. We abort an unfinished group
of records if we see a record which is a group in itself (like COMMIT).
No need for maria_panic() after a bug (which caused tables to not
be closed) was fixed; if there is yet another bug I prefer to see it.
When opening a table for Recovery, set data_file_length
and key_file_length to their real physical value (these are the
easiest state members to restore :). Warn us if the last page
was truncated (but Recovery handles it).
MARIA_SHARE::state::state::records is now partly recovered (not
idempotent, but works if recreating tables from scracth).
When applying a REDO to a page, stamp it with the UNDO's LSN
(current_group_end_lsn), not with the REDO's LSN; it makes
the table more identical to the original table (easier to compare
the two tables in the end).
Big thing missing: some types of REDOs are not handled,
and the UNDO phase does not exist (missing functions to execute UNDOs
to actually rollback). So for now tests are only inserting/deleting
a few 100 rows, closing the table and seeing if the log is applied ok;
it works. UPDATE not handled.
storage/maria/ma_recovery.h:
new functions: ma_recover() for recovery from inside ha_maria;
_ma_apply_log() for maria_read_log (ma_recover() calls _ma_apply_log()).
Btw, we need to not use the word "recover" for REPAIR/maria_chk anymore.
storage/maria/ma_rename.c:
don't write log records during recovery
storage/maria/ma_test2.c:
- fail if maria_info() or other subtests find some wrong information
- new option -g to skip updates.
- init the translog before creating the table, so that log applying
can work.
- in "#if 0" you'll see some fixed bugs (will be removed).
storage/maria/ma_test_all.sh:
cleanup files. Test log applying.
storage/maria/maria_read_log.c:
most of the logic moves to ma_recovery.c to be shared between
maria_read_log and recovery-from-inside-mysqld.
See ma_recovery.c for additional changes made to the moved code.
storage/maria/ma_test_recovery:
unit test for Recovery. Tests insert and delete,
REDO_UPDATE not yet coded.
Script is called from ma_test_all. Can run standalone.
2007-07-26 11:56:21 +02:00
|
|
|
if (verbose) printf(" Double key: %d at record# %d\n", n3, i);
|
2006-04-11 15:45:10 +02:00
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
|
|
|
if (key3[n3] == 1 && first_key <3 && first_key+keys >= 3)
|
|
|
|
{
|
|
|
|
printf("Error: Didn't get error when writing second key: '%8d'\n",n3);
|
|
|
|
goto err;
|
|
|
|
}
|
|
|
|
write_count++; key1[n1]++; key3[n3]=1;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Check if we can find key without flushing database */
|
2007-01-18 20:38:14 +01:00
|
|
|
if (i % 10 == 0)
|
2006-04-11 15:45:10 +02:00
|
|
|
{
|
|
|
|
for (j=rnd(1000)+1 ; j>0 && key1[j] == 0 ; j--) ;
|
|
|
|
if (!j)
|
|
|
|
for (j=999 ; j>0 && key1[j] == 0 ; j--) ;
|
2007-10-04 19:33:42 +02:00
|
|
|
sprintf((char*) key,"%6d",j);
|
2007-07-27 16:11:40 +02:00
|
|
|
if (maria_rkey(file,read_record,0,key,HA_WHOLE_KEY,HA_READ_KEY_EXACT))
|
2006-04-11 15:45:10 +02:00
|
|
|
{
|
|
|
|
printf("Test in loop: Can't find key: \"%s\"\n",key);
|
|
|
|
goto err;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
WL#3071 Maria checkpoint
Finally this is the real checkpoint code.
It however exhibits unstabilities when a checkpoint runs concurrently
with data-modifying clients (table corruption, transaction log's
assertions) so for now a checkpoint is taken only at startup after
recovery and at shutdown, i.e. not in concurrent situations. Later
we will let it run periodically, as well as flush dirty pages
periodically (almost all needed code is there already, only pagecache
code is written but not committed).
WL#3072 Maria recovery
* replacing UNDO_ROW_PURGE with CLR_END; testing of those CLR_END via
ma_test2 which has INSERTs failing with duplicate keys.
* replaying of REDO_RENAME_TABLE
Now, off to test Recovery in ha_maria :)
BitKeeper/deleted/.del-ma_least_recently_dirtied.c:
Delete: storage/maria/ma_least_recently_dirtied.c
BitKeeper/deleted/.del-ma_least_recently_dirtied.h:
Delete: storage/maria/ma_least_recently_dirtied.h
storage/maria/Makefile.am:
compile Checkpoint module
storage/maria/ha_maria.cc:
When ha_maria starts, do a recovery from last checkpoint.
Take a checkpoint when that recovery has ended and when ha_maria
shuts down cleanly.
storage/maria/ma_blockrec.c:
* even if my_sync() fails we have to my_close() (otherwise we leak
a descriptor)
* UNDO_ROW_PURGE is replaced by a simple CLR_END for UNDO_ROW_INSERT,
as promised in the old comment; it gives us skipping during the
UNDO phase.
storage/maria/ma_check.c:
All REDOs before create_rename_lsn are ignored by Recovery. So
create_rename_lsn must be set only after all data/index has been
flushed and forced to disk. We thus move write_log_record_for_repair()
to after _ma_flush_tables_files_after_repair().
storage/maria/ma_checkpoint.c:
Checkpoint module.
storage/maria/ma_checkpoint.h:
optional argument if caller wants a thread to periodically take
checkpoints and flush dirty pages.
storage/maria/ma_create.c:
* no need to init some vars as the initial bzero(share) takes care of this.
* update to new function's name
* even if we fail in my_sync() we have to my_close()
storage/maria/ma_extra.c:
Checkpoint reads share->last_version under intern_lock, so we make
maria_extra() update it under intern_lock. THR_LOCK_maria still needed
because of _ma_test_if_reopen().
storage/maria/ma_init.c:
destroy checkpoint module when Maria shuts down.
storage/maria/ma_loghandler.c:
* UNDO_ROW_PURGE gone (see ma_blockrec.c)
* we need to remember the LSN of the LOGREC_FILE_ID for a share,
because this LSN is needed into the checkpoint record (Recovery wants
to know the validity domain of an id->name mapping)
* translog_get_horizon_no_lock() needed for Checkpoint
* comment about failing assertion (Sanja knows)
* translog_init_reader_data() thought that translog_read_record_header_scan()
returns 0 in case of error, but 0 just means "0-length header".
* translog_assign_id_to_share() now needs the MARIA_HA because
LOGREC_FILE_ID uses a log-write hook.
* Verify that (de)assignment of share->id happens only under intern_lock,
as Checkpoint reads this id with intern_lock.
* translog_purge() can accept TRANSLOG_ADDRESS, not necessarily
a real LSN.
storage/maria/ma_loghandler.h:
prototype updates
storage/maria/ma_open.c:
no need to initialize "res"
storage/maria/ma_pagecache.c:
When taking a checkpoint, we don't need to know the maximum rec_lsn
of dirty pages; this LSN was intended to be used in the two-checkpoint
rule, but last_checkpoint_lsn is as good.
4 bytes for stored_list_size is enough as PAGECACHE::blocks (number
of blocks which the pagecache can contain) is int.
storage/maria/ma_pagecache.h:
new prototype
storage/maria/ma_recovery.c:
* added replaying of REDO_RENAME_TABLE
* UNDO_ROW_PURGE gone (see ma_blockrec.c), replaced by CLR_END
* Recovery from the last checkpoint record now possible
* In new_table() we skip the table if the id->name mapping is older than
create_rename_lsn (mapping dates from lsn_of_file_id).
* in get_MARIA_HA_from_REDO_record() we skip the record
if the id->name mapping is newer than the record (can happen if processing
a record which is before the checkpoint record).
* parse_checkpoint_record() has to return a LSN, that's what caller expects
storage/maria/ma_rename.c:
new function's name; log end zeroes of tables' names (ease recovery)
storage/maria/ma_test2.c:
* equivalent of ma_test1's --test-undo added (named -u here).
* -t=1 now stops right after creating the table, so that
we can test undoing of INSERTs with duplicate keys (which tests the
CLR_END logged by _ma_write_abort_block_record()).
storage/maria/ma_test_recovery.expected:
Result of testing undoing of INSERTs with duplicate keys; there are
some differences in maria_chk -dvv but they are normal (removing
records does not shrink data/index file, does not put back the
"analyzed, optimized keys"(etc) index state.
storage/maria/ma_test_recovery:
Test undoing of INSERTs with duplicate keys, using ma_test2;
when such INSERT happens, it logs REDO_INSERT, UNDO_INSERT, REDO_DELETE,
CLR_END; we abort after that, and test that CLR_END causes recovery
to jump over UNDO_INSERT.
storage/maria/ma_write.c:
comment
storage/maria/maria_chk.c:
comment
storage/maria/maria_def.h:
* a new bit in MARIA_SHARE::in_checkpoint, used to build a list
of unique shares during Checkpoint.
* MARIA_SHARE::lsn_of_file_id added: the LSN of the last LOGREC_FILE_ID
for this share; needed to know to which LSN domain the mappings
found in the Checkpoint record apply (new mappings should not apply
to old REDOs).
storage/maria/trnman.c:
* small changes to how trnman_collect_transactions() fills its buffer;
it also uses a non-dummy lsn_read_non_atomic() found in ma_checkpoint.h
2007-09-12 11:27:34 +02:00
|
|
|
if (testflag == 2)
|
2007-01-18 20:38:14 +01:00
|
|
|
goto end;
|
WL#3071 Maria checkpoint
Ability for flush_pagecache_blocks() to flush only certain pages of
a file, as instructed by an option "filter" pointer-to-function argument;
Checkpoint and background dirty page flushing use that to flush only
pages which have been dirty for long enough and bitmap pages.
Fix for a bug in flush_cached_blocks() (no idea if it could produce
a bug in real life, but theoretically it is).
Testing checkpoint in ma_test_recovery via ma_test1 and ma_test2.
Background checkpoint & dirty pages flush thread is still disabled
by default in ha_maria.
mysql-test/r/maria.result:
result update
storage/maria/ha_maria.cc:
blank after function comment
storage/maria/ma_checkpoint.c:
Using an enum instead of 0/1/2 (applying Sanja's review comments).
The comment about "this is an horizon" can be removed as Sanja
created translog_next_LSN() which parse_checkpoint_record() uses.
Variables in ma_checkpoint_background() cannot be declared in the
for() as their value must not be reset at each iteration!
storage/maria/ma_pagecache.c:
adding to flush_pagecache_blocks() optional arguments 'filter'
(pointer to function) and 'filter_arg'; if filter!=NULL this function
will be called for each block of the file and will reply if this
block and following ones should be flushed or not (3 possible
replies).
Fixing a bug when flush_cached_blocks() skips a pinned page: it has
to unset PCBLOCK_IN_FLUSH set by flush_pagecache_blocks_int().
storage/maria/ma_pagecache.h:
flush_pagecache_blocks() is changed to take "filter" and "filter_arg"
arguments. "filter", if it is not NULL, may return one value
among enum pagecache_flush_filter_result.
storage/maria/ma_recovery.c:
open_count=0 when closing tables at the end of recovery.
storage/maria/ma_test1.c:
Optional checkpoints (-H#) at various stages (stages similar
to --testflag), for testing of checkpoints.
storage/maria/ma_test2.c:
Optional checkpoints (-H#) at various stages (stages similar
to -t), for testing of checkpoints.
storage/maria/ma_test_recovery.expected:
Result update: the results of the additional test run with -H#
(checkpoints) are added here. They are exactly identical to without
checkpoints except that the index's Root (printed by maria_chk)
is more correct when using checkpoints. This is because checkpoint
flushed the state, so it happens to be correct, while no-checkpoint
does not flush the state, and recovery does not recover indexes
so Root is never fixed. When we recover indices, this will go away.
storage/maria/ma_test_recovery:
We duplicate the loop of tests to add an additional run with
checkpoints at various stages, to see if maria_read_log
uses them fine.
2007-10-17 16:55:26 +02:00
|
|
|
if (checkpoint == 2 && ma_checkpoint_execute(CHECKPOINT_MEDIUM, FALSE))
|
|
|
|
goto err;
|
2006-04-11 15:45:10 +02:00
|
|
|
|
|
|
|
if (write_cacheing)
|
|
|
|
{
|
|
|
|
if (maria_extra(file,HA_EXTRA_NO_CACHE,0))
|
|
|
|
{
|
|
|
|
puts("got error from maria_extra(HA_EXTRA_NO_CACHE)");
|
WL#3072 - Maria recovery
Unit test for recovery: runs ma_test1 and ma_test2 (both only with
INSERTs and DELETEs; UPDATEs disabled as not handled by recovery)
then moves the tables elswhere; recreates tables from the log, and
compares and fails if there is a difference. Passes now.
Most of maria_read_log.c moved to ma_recovery.c, as it will be re-used
for recovery-from-ha_maria.
Bugfixes of applying of REDO_INSERT, REDO_PURGE_ROW.
Applying of REDO_PURGE_BLOCKS, REDO_DELETE_ALL, REDO_DROP_TABLE,
UNDO_ROW_INSERT (in REDO phase only, i.e. just doing records++),
UNDO_ROW_DELETE, UNDO_ROW_PURGE.
Code cleanups.
Monty: please look for "QQ". Sanja: please look for "Sanja".
Future tasks: recovery of the bitmap (easy), recovery of the state
(make it idempotent), more REDOs (Monty to work on
REDO_UPDATE?), UNDO phase...
Pushing this cset as it looks safe, contains test and bugfixes which
will help Monty implement applying of REDO_UPDATE.
sql/handler.cc:
typo
storage/maria/Makefile.am:
Adding ma_test_recovery (which ma_test_all invokes, and which can
also be run alone). Most of maria_read_log.c moved to ma_recovery.c
storage/maria/ha_maria.cc:
comments
storage/maria/ma_bitmap.c:
fixing comments. 2 -> sizeof(maria_bitmap_marker).
Bitmap-related part of _ma_initialize_datafile() moves in bitmap module.
Now putting the "bm" signature when creating the first bitmap page
(it used to happen only at next open, but that
caused an annoying difference when testing Recovery if the original
run didn't open the table, and it looks more
logical like this: it goes to disk only with its signature correct);
see the "QQ" comment towards the _ma_initialize_data_file() call
in ma_create.c for more).
When reading a bitmap page, verify its signature (happens when normally
using the table or when CHECKing it; not when REPAIRing it).
storage/maria/ma_blockrec.c:
* no need to sync the data file if table is not transactional
* Comments, code cleanup (log-related data moved to log-related code
block, int5store->page_store).
* Store the table's short id into LOGREC_UNDO_ROW_PURGE, like we
do for other records (though this record will soon be replaced
with a CLR).
* If "page" is 1 it means the page which extends from byte
page*block_size+1 to (page+1)*block_size (byte number 1 being
the first byte of the file). The last byte of the file is
data_file_length (same convention).
A new page needs to be created if the last byte of the page is
beyond the last byte of the file, i.e.
(page+1)*block_size+1 > data_file_length, so we correct the test
(bug found when testing log applying for ma_test1 -M -T --skip-update).
* update the page's LSN when removing a row from it during
execution of a REDO_PURGE_ROW record (bug found when testing log
applying for ma_test1 -M -T --skip-update).
* applying of REDO_PURGE_BLOCKs (limited to a one-page range for now).
storage/maria/ma_blockrec.h:
new functions. maria_bitmap_marker does not need to be exported.
storage/maria/ma_close.c:
we can always flush the table's state when closing the last instance
of the table. And it is needed for maria_read_log (as it does
not use maria_lock_database()).
storage/maria/ma_control_file.c:
when in Recovery, some assertions should not be used.
storage/maria/ma_control_file.h:
double-inclusion safe
storage/maria/ma_create.c:
during recovery, don't log records. Comments.
Moving the creation of the first bitmap page to ma_bitmap.c
storage/maria/ma_delete_table.c:
during recovery, don't log records. Log the end-zero of the dropped
table's name, so that recovery can use the string in place without
extending it to fit an end zero.
storage/maria/ma_loghandler.c:
* inwrite_rec_hook also needs access to the MARIA_SHARE, like
prewrite_rec_hook. This will be needed to update
share->records_diff (in the upcoming patch "recovery of the state").
* LOG_DESC::record_ends_group changed to an enum.
* LOG_DESC for LOGREC_REDO_PURGE_BLOCKS and LOGREC_UNDO_ROW_PURGE
corrected
* Sanja please see the @todo LOG BUG
* avoiding DBUG_RETURN(func()) as it gives confusing debug traces.
storage/maria/ma_loghandler.h:
- log write hooks called while the log's lock is held (inwrite_rec_hook)
now need the MARIA_SHARE, like prewrite_rec_hook already had
- instead of a bool saying if this record's type ends groups or not,
we refine: it may not end a group, it may end a group, or it may
be a group in itself. Imagine that we had a physical write failure
to a table before we log the UNDO, we still end up in
external_lock(F_UNLCK) and then we log a COMMIT: we don't want
to consider this COMMIT as ending the group of REDOs (don't want
to execute those REDOs during Recovery), that's why we say "COMMIT
is a group in itself, it aborts any previous group". This also
gives one more sanity check in maria_read_log.
storage/maria/ma_recovery.c:
New Recovery code, replacing the old pseudocode.
Most of maria_read_log moved here.
Call-able from ha_maria, but not enabled yet.
Compared to the previous version of maria_read_log, some bugs have
been fixed, debugging output can go to stdout or a disk file (for now
it's useful for me, later it can be changed), execution of
REDO_DROP_TABLE, REDO_DELETE_ALL, REDO_PURGE_BLOCKS has been added. Duplicate code
has been factored into functions. We abort an unfinished group
of records if we see a record which is a group in itself (like COMMIT).
No need for maria_panic() after a bug (which caused tables to not
be closed) was fixed; if there is yet another bug I prefer to see it.
When opening a table for Recovery, set data_file_length
and key_file_length to their real physical value (these are the
easiest state members to restore :). Warn us if the last page
was truncated (but Recovery handles it).
MARIA_SHARE::state::state::records is now partly recovered (not
idempotent, but works if recreating tables from scracth).
When applying a REDO to a page, stamp it with the UNDO's LSN
(current_group_end_lsn), not with the REDO's LSN; it makes
the table more identical to the original table (easier to compare
the two tables in the end).
Big thing missing: some types of REDOs are not handled,
and the UNDO phase does not exist (missing functions to execute UNDOs
to actually rollback). So for now tests are only inserting/deleting
a few 100 rows, closing the table and seeing if the log is applied ok;
it works. UPDATE not handled.
storage/maria/ma_recovery.h:
new functions: ma_recover() for recovery from inside ha_maria;
_ma_apply_log() for maria_read_log (ma_recover() calls _ma_apply_log()).
Btw, we need to not use the word "recover" for REPAIR/maria_chk anymore.
storage/maria/ma_rename.c:
don't write log records during recovery
storage/maria/ma_test2.c:
- fail if maria_info() or other subtests find some wrong information
- new option -g to skip updates.
- init the translog before creating the table, so that log applying
can work.
- in "#if 0" you'll see some fixed bugs (will be removed).
storage/maria/ma_test_all.sh:
cleanup files. Test log applying.
storage/maria/maria_read_log.c:
most of the logic moves to ma_recovery.c to be shared between
maria_read_log and recovery-from-inside-mysqld.
See ma_recovery.c for additional changes made to the moved code.
storage/maria/ma_test_recovery:
unit test for Recovery. Tests insert and delete,
REDO_UPDATE not yet coded.
Script is called from ma_test_all. Can run standalone.
2007-07-26 11:56:21 +02:00
|
|
|
goto err;
|
2006-04-11 15:45:10 +02:00
|
|
|
}
|
|
|
|
}
|
2007-04-19 17:48:36 +02:00
|
|
|
#ifdef REMOVE_WHEN_WE_HAVE_RESIZE
|
2007-04-04 22:37:09 +02:00
|
|
|
if (pagecacheing)
|
2007-04-19 17:48:36 +02:00
|
|
|
resize_pagecache(maria_pagecache, maria_block_size,
|
2007-04-04 22:37:09 +02:00
|
|
|
pagecache_size * 2, 0, 0);
|
2007-04-19 17:48:36 +02:00
|
|
|
#endif
|
2006-04-11 15:45:10 +02:00
|
|
|
if (!silent)
|
|
|
|
printf("- Delete\n");
|
2007-01-18 20:38:14 +01:00
|
|
|
if (srand_arg)
|
|
|
|
srand(srand_arg);
|
2006-04-11 15:45:10 +02:00
|
|
|
for (i=0 ; i<recant/10 ; i++)
|
|
|
|
{
|
|
|
|
for (j=rnd(1000)+1 ; j>0 && key1[j] == 0 ; j--) ;
|
|
|
|
if (j != 0)
|
|
|
|
{
|
2007-10-04 19:33:42 +02:00
|
|
|
sprintf((char*) key,"%6d",j);
|
2007-07-27 16:11:40 +02:00
|
|
|
if (maria_rkey(file,read_record,0,key,HA_WHOLE_KEY,HA_READ_KEY_EXACT))
|
2006-04-11 15:45:10 +02:00
|
|
|
{
|
|
|
|
printf("can't find key1: \"%s\"\n",key);
|
|
|
|
goto err;
|
|
|
|
}
|
2007-01-18 20:38:14 +01:00
|
|
|
if (bcmp(read_record+keyinfo[0].seg[0].start,
|
|
|
|
key, keyinfo[0].seg[0].length))
|
|
|
|
{
|
|
|
|
printf("Found wrong record when searching for key: \"%s\"\n",key);
|
|
|
|
goto err;
|
|
|
|
}
|
2006-04-11 15:45:10 +02:00
|
|
|
if (opt_delete == (uint) remove_count) /* While testing */
|
|
|
|
goto end;
|
|
|
|
if (maria_delete(file,read_record))
|
|
|
|
{
|
|
|
|
printf("error: %d; can't delete record: \"%s\"\n", my_errno,read_record);
|
|
|
|
goto err;
|
|
|
|
}
|
|
|
|
opt_delete++;
|
2007-10-04 19:33:42 +02:00
|
|
|
key1[atoi((char*) read_record+keyinfo[0].seg[0].start)]--;
|
|
|
|
key3[atoi((char*) read_record+keyinfo[2].seg[0].start)]=0;
|
2006-04-11 15:45:10 +02:00
|
|
|
}
|
|
|
|
else
|
|
|
|
puts("Warning: Skipping delete test because no dupplicate keys");
|
|
|
|
}
|
WL#3071 Maria checkpoint
Finally this is the real checkpoint code.
It however exhibits unstabilities when a checkpoint runs concurrently
with data-modifying clients (table corruption, transaction log's
assertions) so for now a checkpoint is taken only at startup after
recovery and at shutdown, i.e. not in concurrent situations. Later
we will let it run periodically, as well as flush dirty pages
periodically (almost all needed code is there already, only pagecache
code is written but not committed).
WL#3072 Maria recovery
* replacing UNDO_ROW_PURGE with CLR_END; testing of those CLR_END via
ma_test2 which has INSERTs failing with duplicate keys.
* replaying of REDO_RENAME_TABLE
Now, off to test Recovery in ha_maria :)
BitKeeper/deleted/.del-ma_least_recently_dirtied.c:
Delete: storage/maria/ma_least_recently_dirtied.c
BitKeeper/deleted/.del-ma_least_recently_dirtied.h:
Delete: storage/maria/ma_least_recently_dirtied.h
storage/maria/Makefile.am:
compile Checkpoint module
storage/maria/ha_maria.cc:
When ha_maria starts, do a recovery from last checkpoint.
Take a checkpoint when that recovery has ended and when ha_maria
shuts down cleanly.
storage/maria/ma_blockrec.c:
* even if my_sync() fails we have to my_close() (otherwise we leak
a descriptor)
* UNDO_ROW_PURGE is replaced by a simple CLR_END for UNDO_ROW_INSERT,
as promised in the old comment; it gives us skipping during the
UNDO phase.
storage/maria/ma_check.c:
All REDOs before create_rename_lsn are ignored by Recovery. So
create_rename_lsn must be set only after all data/index has been
flushed and forced to disk. We thus move write_log_record_for_repair()
to after _ma_flush_tables_files_after_repair().
storage/maria/ma_checkpoint.c:
Checkpoint module.
storage/maria/ma_checkpoint.h:
optional argument if caller wants a thread to periodically take
checkpoints and flush dirty pages.
storage/maria/ma_create.c:
* no need to init some vars as the initial bzero(share) takes care of this.
* update to new function's name
* even if we fail in my_sync() we have to my_close()
storage/maria/ma_extra.c:
Checkpoint reads share->last_version under intern_lock, so we make
maria_extra() update it under intern_lock. THR_LOCK_maria still needed
because of _ma_test_if_reopen().
storage/maria/ma_init.c:
destroy checkpoint module when Maria shuts down.
storage/maria/ma_loghandler.c:
* UNDO_ROW_PURGE gone (see ma_blockrec.c)
* we need to remember the LSN of the LOGREC_FILE_ID for a share,
because this LSN is needed into the checkpoint record (Recovery wants
to know the validity domain of an id->name mapping)
* translog_get_horizon_no_lock() needed for Checkpoint
* comment about failing assertion (Sanja knows)
* translog_init_reader_data() thought that translog_read_record_header_scan()
returns 0 in case of error, but 0 just means "0-length header".
* translog_assign_id_to_share() now needs the MARIA_HA because
LOGREC_FILE_ID uses a log-write hook.
* Verify that (de)assignment of share->id happens only under intern_lock,
as Checkpoint reads this id with intern_lock.
* translog_purge() can accept TRANSLOG_ADDRESS, not necessarily
a real LSN.
storage/maria/ma_loghandler.h:
prototype updates
storage/maria/ma_open.c:
no need to initialize "res"
storage/maria/ma_pagecache.c:
When taking a checkpoint, we don't need to know the maximum rec_lsn
of dirty pages; this LSN was intended to be used in the two-checkpoint
rule, but last_checkpoint_lsn is as good.
4 bytes for stored_list_size is enough as PAGECACHE::blocks (number
of blocks which the pagecache can contain) is int.
storage/maria/ma_pagecache.h:
new prototype
storage/maria/ma_recovery.c:
* added replaying of REDO_RENAME_TABLE
* UNDO_ROW_PURGE gone (see ma_blockrec.c), replaced by CLR_END
* Recovery from the last checkpoint record now possible
* In new_table() we skip the table if the id->name mapping is older than
create_rename_lsn (mapping dates from lsn_of_file_id).
* in get_MARIA_HA_from_REDO_record() we skip the record
if the id->name mapping is newer than the record (can happen if processing
a record which is before the checkpoint record).
* parse_checkpoint_record() has to return a LSN, that's what caller expects
storage/maria/ma_rename.c:
new function's name; log end zeroes of tables' names (ease recovery)
storage/maria/ma_test2.c:
* equivalent of ma_test1's --test-undo added (named -u here).
* -t=1 now stops right after creating the table, so that
we can test undoing of INSERTs with duplicate keys (which tests the
CLR_END logged by _ma_write_abort_block_record()).
storage/maria/ma_test_recovery.expected:
Result of testing undoing of INSERTs with duplicate keys; there are
some differences in maria_chk -dvv but they are normal (removing
records does not shrink data/index file, does not put back the
"analyzed, optimized keys"(etc) index state.
storage/maria/ma_test_recovery:
Test undoing of INSERTs with duplicate keys, using ma_test2;
when such INSERT happens, it logs REDO_INSERT, UNDO_INSERT, REDO_DELETE,
CLR_END; we abort after that, and test that CLR_END causes recovery
to jump over UNDO_INSERT.
storage/maria/ma_write.c:
comment
storage/maria/maria_chk.c:
comment
storage/maria/maria_def.h:
* a new bit in MARIA_SHARE::in_checkpoint, used to build a list
of unique shares during Checkpoint.
* MARIA_SHARE::lsn_of_file_id added: the LSN of the last LOGREC_FILE_ID
for this share; needed to know to which LSN domain the mappings
found in the Checkpoint record apply (new mappings should not apply
to old REDOs).
storage/maria/trnman.c:
* small changes to how trnman_collect_transactions() fills its buffer;
it also uses a non-dummy lsn_read_non_atomic() found in ma_checkpoint.h
2007-09-12 11:27:34 +02:00
|
|
|
if (testflag == 3)
|
2007-01-18 20:38:14 +01:00
|
|
|
goto end;
|
WL#3071 Maria checkpoint
Ability for flush_pagecache_blocks() to flush only certain pages of
a file, as instructed by an option "filter" pointer-to-function argument;
Checkpoint and background dirty page flushing use that to flush only
pages which have been dirty for long enough and bitmap pages.
Fix for a bug in flush_cached_blocks() (no idea if it could produce
a bug in real life, but theoretically it is).
Testing checkpoint in ma_test_recovery via ma_test1 and ma_test2.
Background checkpoint & dirty pages flush thread is still disabled
by default in ha_maria.
mysql-test/r/maria.result:
result update
storage/maria/ha_maria.cc:
blank after function comment
storage/maria/ma_checkpoint.c:
Using an enum instead of 0/1/2 (applying Sanja's review comments).
The comment about "this is an horizon" can be removed as Sanja
created translog_next_LSN() which parse_checkpoint_record() uses.
Variables in ma_checkpoint_background() cannot be declared in the
for() as their value must not be reset at each iteration!
storage/maria/ma_pagecache.c:
adding to flush_pagecache_blocks() optional arguments 'filter'
(pointer to function) and 'filter_arg'; if filter!=NULL this function
will be called for each block of the file and will reply if this
block and following ones should be flushed or not (3 possible
replies).
Fixing a bug when flush_cached_blocks() skips a pinned page: it has
to unset PCBLOCK_IN_FLUSH set by flush_pagecache_blocks_int().
storage/maria/ma_pagecache.h:
flush_pagecache_blocks() is changed to take "filter" and "filter_arg"
arguments. "filter", if it is not NULL, may return one value
among enum pagecache_flush_filter_result.
storage/maria/ma_recovery.c:
open_count=0 when closing tables at the end of recovery.
storage/maria/ma_test1.c:
Optional checkpoints (-H#) at various stages (stages similar
to --testflag), for testing of checkpoints.
storage/maria/ma_test2.c:
Optional checkpoints (-H#) at various stages (stages similar
to -t), for testing of checkpoints.
storage/maria/ma_test_recovery.expected:
Result update: the results of the additional test run with -H#
(checkpoints) are added here. They are exactly identical to without
checkpoints except that the index's Root (printed by maria_chk)
is more correct when using checkpoints. This is because checkpoint
flushed the state, so it happens to be correct, while no-checkpoint
does not flush the state, and recovery does not recover indexes
so Root is never fixed. When we recover indices, this will go away.
storage/maria/ma_test_recovery:
We duplicate the loop of tests to add an additional run with
checkpoints at various stages, to see if maria_read_log
uses them fine.
2007-10-17 16:55:26 +02:00
|
|
|
if (checkpoint == 3 && ma_checkpoint_execute(CHECKPOINT_MEDIUM, FALSE))
|
|
|
|
goto err;
|
2006-04-11 15:45:10 +02:00
|
|
|
|
|
|
|
if (!silent)
|
|
|
|
printf("- Update\n");
|
2007-01-18 20:38:14 +01:00
|
|
|
if (srand_arg)
|
|
|
|
srand(srand_arg);
|
2006-04-11 15:45:10 +02:00
|
|
|
for (i=0 ; i<recant/10 ; i++)
|
|
|
|
{
|
|
|
|
n1=rnd(1000); n2=rnd(100); n3=rnd(5000);
|
2007-10-04 19:33:42 +02:00
|
|
|
sprintf((char*) record2,"%6d:%4d:%8d:XXX: %4d ",n1,n2,n3,update);
|
2006-04-11 15:45:10 +02:00
|
|
|
int4store(record2+STANDARD_LENGTH-4,(long) i);
|
|
|
|
fix_length(record2,(uint) STANDARD_LENGTH+rnd(60));
|
|
|
|
|
|
|
|
for (j=rnd(1000)+1 ; j>0 && key1[j] == 0 ; j--) ;
|
|
|
|
if (j != 0)
|
|
|
|
{
|
2007-10-04 19:33:42 +02:00
|
|
|
sprintf((char*) key,"%6d",j);
|
2007-07-27 16:11:40 +02:00
|
|
|
if (maria_rkey(file,read_record,0,key,HA_WHOLE_KEY,HA_READ_KEY_EXACT))
|
2006-04-11 15:45:10 +02:00
|
|
|
{
|
2007-10-04 19:33:42 +02:00
|
|
|
printf("can't find key1: \"%s\"\n", (char*) key);
|
2006-04-11 15:45:10 +02:00
|
|
|
goto err;
|
|
|
|
}
|
2007-01-18 20:38:14 +01:00
|
|
|
if (bcmp(read_record+keyinfo[0].seg[0].start,
|
|
|
|
key, keyinfo[0].seg[0].length))
|
|
|
|
{
|
|
|
|
printf("Found wrong record when searching for key: \"%s\"; Found \"%.*s\"\n",
|
|
|
|
key, keyinfo[0].seg[0].length,
|
|
|
|
read_record+keyinfo[0].seg[0].start);
|
|
|
|
goto err;
|
|
|
|
}
|
2006-04-11 15:45:10 +02:00
|
|
|
if (use_blob)
|
|
|
|
{
|
2007-01-18 20:38:14 +01:00
|
|
|
ulong blob_length;
|
2006-04-11 15:45:10 +02:00
|
|
|
if (i & 1)
|
2007-01-18 20:38:14 +01:00
|
|
|
put_blob_in_record(record+blob_pos,&blob_buffer, &blob_length);
|
2006-04-11 15:45:10 +02:00
|
|
|
else
|
|
|
|
bmove(record+blob_pos,read_record+blob_pos,8);
|
|
|
|
}
|
WL#3072 - Maria recovery
Unit test for recovery: runs ma_test1 and ma_test2 (both only with
INSERTs and DELETEs; UPDATEs disabled as not handled by recovery)
then moves the tables elswhere; recreates tables from the log, and
compares and fails if there is a difference. Passes now.
Most of maria_read_log.c moved to ma_recovery.c, as it will be re-used
for recovery-from-ha_maria.
Bugfixes of applying of REDO_INSERT, REDO_PURGE_ROW.
Applying of REDO_PURGE_BLOCKS, REDO_DELETE_ALL, REDO_DROP_TABLE,
UNDO_ROW_INSERT (in REDO phase only, i.e. just doing records++),
UNDO_ROW_DELETE, UNDO_ROW_PURGE.
Code cleanups.
Monty: please look for "QQ". Sanja: please look for "Sanja".
Future tasks: recovery of the bitmap (easy), recovery of the state
(make it idempotent), more REDOs (Monty to work on
REDO_UPDATE?), UNDO phase...
Pushing this cset as it looks safe, contains test and bugfixes which
will help Monty implement applying of REDO_UPDATE.
sql/handler.cc:
typo
storage/maria/Makefile.am:
Adding ma_test_recovery (which ma_test_all invokes, and which can
also be run alone). Most of maria_read_log.c moved to ma_recovery.c
storage/maria/ha_maria.cc:
comments
storage/maria/ma_bitmap.c:
fixing comments. 2 -> sizeof(maria_bitmap_marker).
Bitmap-related part of _ma_initialize_datafile() moves in bitmap module.
Now putting the "bm" signature when creating the first bitmap page
(it used to happen only at next open, but that
caused an annoying difference when testing Recovery if the original
run didn't open the table, and it looks more
logical like this: it goes to disk only with its signature correct);
see the "QQ" comment towards the _ma_initialize_data_file() call
in ma_create.c for more).
When reading a bitmap page, verify its signature (happens when normally
using the table or when CHECKing it; not when REPAIRing it).
storage/maria/ma_blockrec.c:
* no need to sync the data file if table is not transactional
* Comments, code cleanup (log-related data moved to log-related code
block, int5store->page_store).
* Store the table's short id into LOGREC_UNDO_ROW_PURGE, like we
do for other records (though this record will soon be replaced
with a CLR).
* If "page" is 1 it means the page which extends from byte
page*block_size+1 to (page+1)*block_size (byte number 1 being
the first byte of the file). The last byte of the file is
data_file_length (same convention).
A new page needs to be created if the last byte of the page is
beyond the last byte of the file, i.e.
(page+1)*block_size+1 > data_file_length, so we correct the test
(bug found when testing log applying for ma_test1 -M -T --skip-update).
* update the page's LSN when removing a row from it during
execution of a REDO_PURGE_ROW record (bug found when testing log
applying for ma_test1 -M -T --skip-update).
* applying of REDO_PURGE_BLOCKs (limited to a one-page range for now).
storage/maria/ma_blockrec.h:
new functions. maria_bitmap_marker does not need to be exported.
storage/maria/ma_close.c:
we can always flush the table's state when closing the last instance
of the table. And it is needed for maria_read_log (as it does
not use maria_lock_database()).
storage/maria/ma_control_file.c:
when in Recovery, some assertions should not be used.
storage/maria/ma_control_file.h:
double-inclusion safe
storage/maria/ma_create.c:
during recovery, don't log records. Comments.
Moving the creation of the first bitmap page to ma_bitmap.c
storage/maria/ma_delete_table.c:
during recovery, don't log records. Log the end-zero of the dropped
table's name, so that recovery can use the string in place without
extending it to fit an end zero.
storage/maria/ma_loghandler.c:
* inwrite_rec_hook also needs access to the MARIA_SHARE, like
prewrite_rec_hook. This will be needed to update
share->records_diff (in the upcoming patch "recovery of the state").
* LOG_DESC::record_ends_group changed to an enum.
* LOG_DESC for LOGREC_REDO_PURGE_BLOCKS and LOGREC_UNDO_ROW_PURGE
corrected
* Sanja please see the @todo LOG BUG
* avoiding DBUG_RETURN(func()) as it gives confusing debug traces.
storage/maria/ma_loghandler.h:
- log write hooks called while the log's lock is held (inwrite_rec_hook)
now need the MARIA_SHARE, like prewrite_rec_hook already had
- instead of a bool saying if this record's type ends groups or not,
we refine: it may not end a group, it may end a group, or it may
be a group in itself. Imagine that we had a physical write failure
to a table before we log the UNDO, we still end up in
external_lock(F_UNLCK) and then we log a COMMIT: we don't want
to consider this COMMIT as ending the group of REDOs (don't want
to execute those REDOs during Recovery), that's why we say "COMMIT
is a group in itself, it aborts any previous group". This also
gives one more sanity check in maria_read_log.
storage/maria/ma_recovery.c:
New Recovery code, replacing the old pseudocode.
Most of maria_read_log moved here.
Call-able from ha_maria, but not enabled yet.
Compared to the previous version of maria_read_log, some bugs have
been fixed, debugging output can go to stdout or a disk file (for now
it's useful for me, later it can be changed), execution of
REDO_DROP_TABLE, REDO_DELETE_ALL, REDO_PURGE_BLOCKS has been added. Duplicate code
has been factored into functions. We abort an unfinished group
of records if we see a record which is a group in itself (like COMMIT).
No need for maria_panic() after a bug (which caused tables to not
be closed) was fixed; if there is yet another bug I prefer to see it.
When opening a table for Recovery, set data_file_length
and key_file_length to their real physical value (these are the
easiest state members to restore :). Warn us if the last page
was truncated (but Recovery handles it).
MARIA_SHARE::state::state::records is now partly recovered (not
idempotent, but works if recreating tables from scracth).
When applying a REDO to a page, stamp it with the UNDO's LSN
(current_group_end_lsn), not with the REDO's LSN; it makes
the table more identical to the original table (easier to compare
the two tables in the end).
Big thing missing: some types of REDOs are not handled,
and the UNDO phase does not exist (missing functions to execute UNDOs
to actually rollback). So for now tests are only inserting/deleting
a few 100 rows, closing the table and seeing if the log is applied ok;
it works. UPDATE not handled.
storage/maria/ma_recovery.h:
new functions: ma_recover() for recovery from inside ha_maria;
_ma_apply_log() for maria_read_log (ma_recover() calls _ma_apply_log()).
Btw, we need to not use the word "recover" for REPAIR/maria_chk anymore.
storage/maria/ma_rename.c:
don't write log records during recovery
storage/maria/ma_test2.c:
- fail if maria_info() or other subtests find some wrong information
- new option -g to skip updates.
- init the translog before creating the table, so that log applying
can work.
- in "#if 0" you'll see some fixed bugs (will be removed).
storage/maria/ma_test_all.sh:
cleanup files. Test log applying.
storage/maria/maria_read_log.c:
most of the logic moves to ma_recovery.c to be shared between
maria_read_log and recovery-from-inside-mysqld.
See ma_recovery.c for additional changes made to the moved code.
storage/maria/ma_test_recovery:
unit test for Recovery. Tests insert and delete,
REDO_UPDATE not yet coded.
Script is called from ma_test_all. Can run standalone.
2007-07-26 11:56:21 +02:00
|
|
|
if (skip_update)
|
|
|
|
continue;
|
2006-04-11 15:45:10 +02:00
|
|
|
if (maria_update(file,read_record,record2))
|
|
|
|
{
|
|
|
|
if (my_errno != HA_ERR_FOUND_DUPP_KEY || key3[n3] == 0)
|
|
|
|
{
|
|
|
|
printf("error: %d; can't update:\nFrom: \"%s\"\nTo: \"%s\"\n",
|
|
|
|
my_errno,read_record,record2);
|
|
|
|
goto err;
|
|
|
|
}
|
|
|
|
if (verbose)
|
|
|
|
printf("Double key when tried to update:\nFrom: \"%s\"\nTo: \"%s\"\n",record,record2);
|
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
2007-10-04 19:33:42 +02:00
|
|
|
key1[atoi((char*) read_record+keyinfo[0].seg[0].start)]--;
|
|
|
|
key3[atoi((char*) read_record+keyinfo[2].seg[0].start)]=0;
|
2006-04-11 15:45:10 +02:00
|
|
|
key1[n1]++; key3[n3]=1;
|
|
|
|
update++;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
WL#3071 Maria checkpoint
Finally this is the real checkpoint code.
It however exhibits unstabilities when a checkpoint runs concurrently
with data-modifying clients (table corruption, transaction log's
assertions) so for now a checkpoint is taken only at startup after
recovery and at shutdown, i.e. not in concurrent situations. Later
we will let it run periodically, as well as flush dirty pages
periodically (almost all needed code is there already, only pagecache
code is written but not committed).
WL#3072 Maria recovery
* replacing UNDO_ROW_PURGE with CLR_END; testing of those CLR_END via
ma_test2 which has INSERTs failing with duplicate keys.
* replaying of REDO_RENAME_TABLE
Now, off to test Recovery in ha_maria :)
BitKeeper/deleted/.del-ma_least_recently_dirtied.c:
Delete: storage/maria/ma_least_recently_dirtied.c
BitKeeper/deleted/.del-ma_least_recently_dirtied.h:
Delete: storage/maria/ma_least_recently_dirtied.h
storage/maria/Makefile.am:
compile Checkpoint module
storage/maria/ha_maria.cc:
When ha_maria starts, do a recovery from last checkpoint.
Take a checkpoint when that recovery has ended and when ha_maria
shuts down cleanly.
storage/maria/ma_blockrec.c:
* even if my_sync() fails we have to my_close() (otherwise we leak
a descriptor)
* UNDO_ROW_PURGE is replaced by a simple CLR_END for UNDO_ROW_INSERT,
as promised in the old comment; it gives us skipping during the
UNDO phase.
storage/maria/ma_check.c:
All REDOs before create_rename_lsn are ignored by Recovery. So
create_rename_lsn must be set only after all data/index has been
flushed and forced to disk. We thus move write_log_record_for_repair()
to after _ma_flush_tables_files_after_repair().
storage/maria/ma_checkpoint.c:
Checkpoint module.
storage/maria/ma_checkpoint.h:
optional argument if caller wants a thread to periodically take
checkpoints and flush dirty pages.
storage/maria/ma_create.c:
* no need to init some vars as the initial bzero(share) takes care of this.
* update to new function's name
* even if we fail in my_sync() we have to my_close()
storage/maria/ma_extra.c:
Checkpoint reads share->last_version under intern_lock, so we make
maria_extra() update it under intern_lock. THR_LOCK_maria still needed
because of _ma_test_if_reopen().
storage/maria/ma_init.c:
destroy checkpoint module when Maria shuts down.
storage/maria/ma_loghandler.c:
* UNDO_ROW_PURGE gone (see ma_blockrec.c)
* we need to remember the LSN of the LOGREC_FILE_ID for a share,
because this LSN is needed into the checkpoint record (Recovery wants
to know the validity domain of an id->name mapping)
* translog_get_horizon_no_lock() needed for Checkpoint
* comment about failing assertion (Sanja knows)
* translog_init_reader_data() thought that translog_read_record_header_scan()
returns 0 in case of error, but 0 just means "0-length header".
* translog_assign_id_to_share() now needs the MARIA_HA because
LOGREC_FILE_ID uses a log-write hook.
* Verify that (de)assignment of share->id happens only under intern_lock,
as Checkpoint reads this id with intern_lock.
* translog_purge() can accept TRANSLOG_ADDRESS, not necessarily
a real LSN.
storage/maria/ma_loghandler.h:
prototype updates
storage/maria/ma_open.c:
no need to initialize "res"
storage/maria/ma_pagecache.c:
When taking a checkpoint, we don't need to know the maximum rec_lsn
of dirty pages; this LSN was intended to be used in the two-checkpoint
rule, but last_checkpoint_lsn is as good.
4 bytes for stored_list_size is enough as PAGECACHE::blocks (number
of blocks which the pagecache can contain) is int.
storage/maria/ma_pagecache.h:
new prototype
storage/maria/ma_recovery.c:
* added replaying of REDO_RENAME_TABLE
* UNDO_ROW_PURGE gone (see ma_blockrec.c), replaced by CLR_END
* Recovery from the last checkpoint record now possible
* In new_table() we skip the table if the id->name mapping is older than
create_rename_lsn (mapping dates from lsn_of_file_id).
* in get_MARIA_HA_from_REDO_record() we skip the record
if the id->name mapping is newer than the record (can happen if processing
a record which is before the checkpoint record).
* parse_checkpoint_record() has to return a LSN, that's what caller expects
storage/maria/ma_rename.c:
new function's name; log end zeroes of tables' names (ease recovery)
storage/maria/ma_test2.c:
* equivalent of ma_test1's --test-undo added (named -u here).
* -t=1 now stops right after creating the table, so that
we can test undoing of INSERTs with duplicate keys (which tests the
CLR_END logged by _ma_write_abort_block_record()).
storage/maria/ma_test_recovery.expected:
Result of testing undoing of INSERTs with duplicate keys; there are
some differences in maria_chk -dvv but they are normal (removing
records does not shrink data/index file, does not put back the
"analyzed, optimized keys"(etc) index state.
storage/maria/ma_test_recovery:
Test undoing of INSERTs with duplicate keys, using ma_test2;
when such INSERT happens, it logs REDO_INSERT, UNDO_INSERT, REDO_DELETE,
CLR_END; we abort after that, and test that CLR_END causes recovery
to jump over UNDO_INSERT.
storage/maria/ma_write.c:
comment
storage/maria/maria_chk.c:
comment
storage/maria/maria_def.h:
* a new bit in MARIA_SHARE::in_checkpoint, used to build a list
of unique shares during Checkpoint.
* MARIA_SHARE::lsn_of_file_id added: the LSN of the last LOGREC_FILE_ID
for this share; needed to know to which LSN domain the mappings
found in the Checkpoint record apply (new mappings should not apply
to old REDOs).
storage/maria/trnman.c:
* small changes to how trnman_collect_transactions() fills its buffer;
it also uses a non-dummy lsn_read_non_atomic() found in ma_checkpoint.h
2007-09-12 11:27:34 +02:00
|
|
|
if (testflag == 4)
|
2006-04-11 15:45:10 +02:00
|
|
|
goto end;
|
WL#3071 Maria checkpoint
Ability for flush_pagecache_blocks() to flush only certain pages of
a file, as instructed by an option "filter" pointer-to-function argument;
Checkpoint and background dirty page flushing use that to flush only
pages which have been dirty for long enough and bitmap pages.
Fix for a bug in flush_cached_blocks() (no idea if it could produce
a bug in real life, but theoretically it is).
Testing checkpoint in ma_test_recovery via ma_test1 and ma_test2.
Background checkpoint & dirty pages flush thread is still disabled
by default in ha_maria.
mysql-test/r/maria.result:
result update
storage/maria/ha_maria.cc:
blank after function comment
storage/maria/ma_checkpoint.c:
Using an enum instead of 0/1/2 (applying Sanja's review comments).
The comment about "this is an horizon" can be removed as Sanja
created translog_next_LSN() which parse_checkpoint_record() uses.
Variables in ma_checkpoint_background() cannot be declared in the
for() as their value must not be reset at each iteration!
storage/maria/ma_pagecache.c:
adding to flush_pagecache_blocks() optional arguments 'filter'
(pointer to function) and 'filter_arg'; if filter!=NULL this function
will be called for each block of the file and will reply if this
block and following ones should be flushed or not (3 possible
replies).
Fixing a bug when flush_cached_blocks() skips a pinned page: it has
to unset PCBLOCK_IN_FLUSH set by flush_pagecache_blocks_int().
storage/maria/ma_pagecache.h:
flush_pagecache_blocks() is changed to take "filter" and "filter_arg"
arguments. "filter", if it is not NULL, may return one value
among enum pagecache_flush_filter_result.
storage/maria/ma_recovery.c:
open_count=0 when closing tables at the end of recovery.
storage/maria/ma_test1.c:
Optional checkpoints (-H#) at various stages (stages similar
to --testflag), for testing of checkpoints.
storage/maria/ma_test2.c:
Optional checkpoints (-H#) at various stages (stages similar
to -t), for testing of checkpoints.
storage/maria/ma_test_recovery.expected:
Result update: the results of the additional test run with -H#
(checkpoints) are added here. They are exactly identical to without
checkpoints except that the index's Root (printed by maria_chk)
is more correct when using checkpoints. This is because checkpoint
flushed the state, so it happens to be correct, while no-checkpoint
does not flush the state, and recovery does not recover indexes
so Root is never fixed. When we recover indices, this will go away.
storage/maria/ma_test_recovery:
We duplicate the loop of tests to add an additional run with
checkpoints at various stages, to see if maria_read_log
uses them fine.
2007-10-17 16:55:26 +02:00
|
|
|
if (checkpoint == 4 && ma_checkpoint_execute(CHECKPOINT_MEDIUM, FALSE))
|
|
|
|
goto err;
|
2006-04-11 15:45:10 +02:00
|
|
|
|
|
|
|
for (i=999, dupp_keys=j=0 ; i>0 ; i--)
|
|
|
|
{
|
|
|
|
if (key1[i] > dupp_keys)
|
|
|
|
{
|
|
|
|
dupp_keys=key1[i]; j=i;
|
|
|
|
}
|
|
|
|
}
|
2007-10-04 19:33:42 +02:00
|
|
|
sprintf((char*) key,"%6d",j);
|
2006-04-11 15:45:10 +02:00
|
|
|
start=keyinfo[0].seg[0].start;
|
|
|
|
length=keyinfo[0].seg[0].length;
|
|
|
|
if (dupp_keys)
|
|
|
|
{
|
|
|
|
if (!silent)
|
|
|
|
printf("- Same key: first - next -> last - prev -> first\n");
|
|
|
|
DBUG_PRINT("progpos",("first - next -> last - prev -> first"));
|
|
|
|
if (verbose) printf(" Using key: \"%s\" Keys: %d\n",key,dupp_keys);
|
|
|
|
|
2007-07-27 16:11:40 +02:00
|
|
|
if (maria_rkey(file,read_record,0,key,HA_WHOLE_KEY,HA_READ_KEY_EXACT))
|
2006-04-11 15:45:10 +02:00
|
|
|
goto err;
|
|
|
|
if (maria_rsame(file,read_record2,-1))
|
|
|
|
goto err;
|
|
|
|
if (memcmp(read_record,read_record2,reclength) != 0)
|
|
|
|
{
|
|
|
|
printf("maria_rsame didn't find same record\n");
|
WL#3072 - Maria recovery
Unit test for recovery: runs ma_test1 and ma_test2 (both only with
INSERTs and DELETEs; UPDATEs disabled as not handled by recovery)
then moves the tables elswhere; recreates tables from the log, and
compares and fails if there is a difference. Passes now.
Most of maria_read_log.c moved to ma_recovery.c, as it will be re-used
for recovery-from-ha_maria.
Bugfixes of applying of REDO_INSERT, REDO_PURGE_ROW.
Applying of REDO_PURGE_BLOCKS, REDO_DELETE_ALL, REDO_DROP_TABLE,
UNDO_ROW_INSERT (in REDO phase only, i.e. just doing records++),
UNDO_ROW_DELETE, UNDO_ROW_PURGE.
Code cleanups.
Monty: please look for "QQ". Sanja: please look for "Sanja".
Future tasks: recovery of the bitmap (easy), recovery of the state
(make it idempotent), more REDOs (Monty to work on
REDO_UPDATE?), UNDO phase...
Pushing this cset as it looks safe, contains test and bugfixes which
will help Monty implement applying of REDO_UPDATE.
sql/handler.cc:
typo
storage/maria/Makefile.am:
Adding ma_test_recovery (which ma_test_all invokes, and which can
also be run alone). Most of maria_read_log.c moved to ma_recovery.c
storage/maria/ha_maria.cc:
comments
storage/maria/ma_bitmap.c:
fixing comments. 2 -> sizeof(maria_bitmap_marker).
Bitmap-related part of _ma_initialize_datafile() moves in bitmap module.
Now putting the "bm" signature when creating the first bitmap page
(it used to happen only at next open, but that
caused an annoying difference when testing Recovery if the original
run didn't open the table, and it looks more
logical like this: it goes to disk only with its signature correct);
see the "QQ" comment towards the _ma_initialize_data_file() call
in ma_create.c for more).
When reading a bitmap page, verify its signature (happens when normally
using the table or when CHECKing it; not when REPAIRing it).
storage/maria/ma_blockrec.c:
* no need to sync the data file if table is not transactional
* Comments, code cleanup (log-related data moved to log-related code
block, int5store->page_store).
* Store the table's short id into LOGREC_UNDO_ROW_PURGE, like we
do for other records (though this record will soon be replaced
with a CLR).
* If "page" is 1 it means the page which extends from byte
page*block_size+1 to (page+1)*block_size (byte number 1 being
the first byte of the file). The last byte of the file is
data_file_length (same convention).
A new page needs to be created if the last byte of the page is
beyond the last byte of the file, i.e.
(page+1)*block_size+1 > data_file_length, so we correct the test
(bug found when testing log applying for ma_test1 -M -T --skip-update).
* update the page's LSN when removing a row from it during
execution of a REDO_PURGE_ROW record (bug found when testing log
applying for ma_test1 -M -T --skip-update).
* applying of REDO_PURGE_BLOCKs (limited to a one-page range for now).
storage/maria/ma_blockrec.h:
new functions. maria_bitmap_marker does not need to be exported.
storage/maria/ma_close.c:
we can always flush the table's state when closing the last instance
of the table. And it is needed for maria_read_log (as it does
not use maria_lock_database()).
storage/maria/ma_control_file.c:
when in Recovery, some assertions should not be used.
storage/maria/ma_control_file.h:
double-inclusion safe
storage/maria/ma_create.c:
during recovery, don't log records. Comments.
Moving the creation of the first bitmap page to ma_bitmap.c
storage/maria/ma_delete_table.c:
during recovery, don't log records. Log the end-zero of the dropped
table's name, so that recovery can use the string in place without
extending it to fit an end zero.
storage/maria/ma_loghandler.c:
* inwrite_rec_hook also needs access to the MARIA_SHARE, like
prewrite_rec_hook. This will be needed to update
share->records_diff (in the upcoming patch "recovery of the state").
* LOG_DESC::record_ends_group changed to an enum.
* LOG_DESC for LOGREC_REDO_PURGE_BLOCKS and LOGREC_UNDO_ROW_PURGE
corrected
* Sanja please see the @todo LOG BUG
* avoiding DBUG_RETURN(func()) as it gives confusing debug traces.
storage/maria/ma_loghandler.h:
- log write hooks called while the log's lock is held (inwrite_rec_hook)
now need the MARIA_SHARE, like prewrite_rec_hook already had
- instead of a bool saying if this record's type ends groups or not,
we refine: it may not end a group, it may end a group, or it may
be a group in itself. Imagine that we had a physical write failure
to a table before we log the UNDO, we still end up in
external_lock(F_UNLCK) and then we log a COMMIT: we don't want
to consider this COMMIT as ending the group of REDOs (don't want
to execute those REDOs during Recovery), that's why we say "COMMIT
is a group in itself, it aborts any previous group". This also
gives one more sanity check in maria_read_log.
storage/maria/ma_recovery.c:
New Recovery code, replacing the old pseudocode.
Most of maria_read_log moved here.
Call-able from ha_maria, but not enabled yet.
Compared to the previous version of maria_read_log, some bugs have
been fixed, debugging output can go to stdout or a disk file (for now
it's useful for me, later it can be changed), execution of
REDO_DROP_TABLE, REDO_DELETE_ALL, REDO_PURGE_BLOCKS has been added. Duplicate code
has been factored into functions. We abort an unfinished group
of records if we see a record which is a group in itself (like COMMIT).
No need for maria_panic() after a bug (which caused tables to not
be closed) was fixed; if there is yet another bug I prefer to see it.
When opening a table for Recovery, set data_file_length
and key_file_length to their real physical value (these are the
easiest state members to restore :). Warn us if the last page
was truncated (but Recovery handles it).
MARIA_SHARE::state::state::records is now partly recovered (not
idempotent, but works if recreating tables from scracth).
When applying a REDO to a page, stamp it with the UNDO's LSN
(current_group_end_lsn), not with the REDO's LSN; it makes
the table more identical to the original table (easier to compare
the two tables in the end).
Big thing missing: some types of REDOs are not handled,
and the UNDO phase does not exist (missing functions to execute UNDOs
to actually rollback). So for now tests are only inserting/deleting
a few 100 rows, closing the table and seeing if the log is applied ok;
it works. UPDATE not handled.
storage/maria/ma_recovery.h:
new functions: ma_recover() for recovery from inside ha_maria;
_ma_apply_log() for maria_read_log (ma_recover() calls _ma_apply_log()).
Btw, we need to not use the word "recover" for REPAIR/maria_chk anymore.
storage/maria/ma_rename.c:
don't write log records during recovery
storage/maria/ma_test2.c:
- fail if maria_info() or other subtests find some wrong information
- new option -g to skip updates.
- init the translog before creating the table, so that log applying
can work.
- in "#if 0" you'll see some fixed bugs (will be removed).
storage/maria/ma_test_all.sh:
cleanup files. Test log applying.
storage/maria/maria_read_log.c:
most of the logic moves to ma_recovery.c to be shared between
maria_read_log and recovery-from-inside-mysqld.
See ma_recovery.c for additional changes made to the moved code.
storage/maria/ma_test_recovery:
unit test for Recovery. Tests insert and delete,
REDO_UPDATE not yet coded.
Script is called from ma_test_all. Can run standalone.
2007-07-26 11:56:21 +02:00
|
|
|
goto err;
|
2006-04-11 15:45:10 +02:00
|
|
|
}
|
|
|
|
info.recpos=maria_position(file);
|
|
|
|
if (maria_rfirst(file,read_record2,0) ||
|
|
|
|
maria_rsame_with_pos(file,read_record2,0,info.recpos) ||
|
|
|
|
memcmp(read_record,read_record2,reclength) != 0)
|
|
|
|
{
|
|
|
|
printf("maria_rsame_with_pos didn't find same record\n");
|
WL#3072 - Maria recovery
Unit test for recovery: runs ma_test1 and ma_test2 (both only with
INSERTs and DELETEs; UPDATEs disabled as not handled by recovery)
then moves the tables elswhere; recreates tables from the log, and
compares and fails if there is a difference. Passes now.
Most of maria_read_log.c moved to ma_recovery.c, as it will be re-used
for recovery-from-ha_maria.
Bugfixes of applying of REDO_INSERT, REDO_PURGE_ROW.
Applying of REDO_PURGE_BLOCKS, REDO_DELETE_ALL, REDO_DROP_TABLE,
UNDO_ROW_INSERT (in REDO phase only, i.e. just doing records++),
UNDO_ROW_DELETE, UNDO_ROW_PURGE.
Code cleanups.
Monty: please look for "QQ". Sanja: please look for "Sanja".
Future tasks: recovery of the bitmap (easy), recovery of the state
(make it idempotent), more REDOs (Monty to work on
REDO_UPDATE?), UNDO phase...
Pushing this cset as it looks safe, contains test and bugfixes which
will help Monty implement applying of REDO_UPDATE.
sql/handler.cc:
typo
storage/maria/Makefile.am:
Adding ma_test_recovery (which ma_test_all invokes, and which can
also be run alone). Most of maria_read_log.c moved to ma_recovery.c
storage/maria/ha_maria.cc:
comments
storage/maria/ma_bitmap.c:
fixing comments. 2 -> sizeof(maria_bitmap_marker).
Bitmap-related part of _ma_initialize_datafile() moves in bitmap module.
Now putting the "bm" signature when creating the first bitmap page
(it used to happen only at next open, but that
caused an annoying difference when testing Recovery if the original
run didn't open the table, and it looks more
logical like this: it goes to disk only with its signature correct);
see the "QQ" comment towards the _ma_initialize_data_file() call
in ma_create.c for more).
When reading a bitmap page, verify its signature (happens when normally
using the table or when CHECKing it; not when REPAIRing it).
storage/maria/ma_blockrec.c:
* no need to sync the data file if table is not transactional
* Comments, code cleanup (log-related data moved to log-related code
block, int5store->page_store).
* Store the table's short id into LOGREC_UNDO_ROW_PURGE, like we
do for other records (though this record will soon be replaced
with a CLR).
* If "page" is 1 it means the page which extends from byte
page*block_size+1 to (page+1)*block_size (byte number 1 being
the first byte of the file). The last byte of the file is
data_file_length (same convention).
A new page needs to be created if the last byte of the page is
beyond the last byte of the file, i.e.
(page+1)*block_size+1 > data_file_length, so we correct the test
(bug found when testing log applying for ma_test1 -M -T --skip-update).
* update the page's LSN when removing a row from it during
execution of a REDO_PURGE_ROW record (bug found when testing log
applying for ma_test1 -M -T --skip-update).
* applying of REDO_PURGE_BLOCKs (limited to a one-page range for now).
storage/maria/ma_blockrec.h:
new functions. maria_bitmap_marker does not need to be exported.
storage/maria/ma_close.c:
we can always flush the table's state when closing the last instance
of the table. And it is needed for maria_read_log (as it does
not use maria_lock_database()).
storage/maria/ma_control_file.c:
when in Recovery, some assertions should not be used.
storage/maria/ma_control_file.h:
double-inclusion safe
storage/maria/ma_create.c:
during recovery, don't log records. Comments.
Moving the creation of the first bitmap page to ma_bitmap.c
storage/maria/ma_delete_table.c:
during recovery, don't log records. Log the end-zero of the dropped
table's name, so that recovery can use the string in place without
extending it to fit an end zero.
storage/maria/ma_loghandler.c:
* inwrite_rec_hook also needs access to the MARIA_SHARE, like
prewrite_rec_hook. This will be needed to update
share->records_diff (in the upcoming patch "recovery of the state").
* LOG_DESC::record_ends_group changed to an enum.
* LOG_DESC for LOGREC_REDO_PURGE_BLOCKS and LOGREC_UNDO_ROW_PURGE
corrected
* Sanja please see the @todo LOG BUG
* avoiding DBUG_RETURN(func()) as it gives confusing debug traces.
storage/maria/ma_loghandler.h:
- log write hooks called while the log's lock is held (inwrite_rec_hook)
now need the MARIA_SHARE, like prewrite_rec_hook already had
- instead of a bool saying if this record's type ends groups or not,
we refine: it may not end a group, it may end a group, or it may
be a group in itself. Imagine that we had a physical write failure
to a table before we log the UNDO, we still end up in
external_lock(F_UNLCK) and then we log a COMMIT: we don't want
to consider this COMMIT as ending the group of REDOs (don't want
to execute those REDOs during Recovery), that's why we say "COMMIT
is a group in itself, it aborts any previous group". This also
gives one more sanity check in maria_read_log.
storage/maria/ma_recovery.c:
New Recovery code, replacing the old pseudocode.
Most of maria_read_log moved here.
Call-able from ha_maria, but not enabled yet.
Compared to the previous version of maria_read_log, some bugs have
been fixed, debugging output can go to stdout or a disk file (for now
it's useful for me, later it can be changed), execution of
REDO_DROP_TABLE, REDO_DELETE_ALL, REDO_PURGE_BLOCKS has been added. Duplicate code
has been factored into functions. We abort an unfinished group
of records if we see a record which is a group in itself (like COMMIT).
No need for maria_panic() after a bug (which caused tables to not
be closed) was fixed; if there is yet another bug I prefer to see it.
When opening a table for Recovery, set data_file_length
and key_file_length to their real physical value (these are the
easiest state members to restore :). Warn us if the last page
was truncated (but Recovery handles it).
MARIA_SHARE::state::state::records is now partly recovered (not
idempotent, but works if recreating tables from scracth).
When applying a REDO to a page, stamp it with the UNDO's LSN
(current_group_end_lsn), not with the REDO's LSN; it makes
the table more identical to the original table (easier to compare
the two tables in the end).
Big thing missing: some types of REDOs are not handled,
and the UNDO phase does not exist (missing functions to execute UNDOs
to actually rollback). So for now tests are only inserting/deleting
a few 100 rows, closing the table and seeing if the log is applied ok;
it works. UPDATE not handled.
storage/maria/ma_recovery.h:
new functions: ma_recover() for recovery from inside ha_maria;
_ma_apply_log() for maria_read_log (ma_recover() calls _ma_apply_log()).
Btw, we need to not use the word "recover" for REPAIR/maria_chk anymore.
storage/maria/ma_rename.c:
don't write log records during recovery
storage/maria/ma_test2.c:
- fail if maria_info() or other subtests find some wrong information
- new option -g to skip updates.
- init the translog before creating the table, so that log applying
can work.
- in "#if 0" you'll see some fixed bugs (will be removed).
storage/maria/ma_test_all.sh:
cleanup files. Test log applying.
storage/maria/maria_read_log.c:
most of the logic moves to ma_recovery.c to be shared between
maria_read_log and recovery-from-inside-mysqld.
See ma_recovery.c for additional changes made to the moved code.
storage/maria/ma_test_recovery:
unit test for Recovery. Tests insert and delete,
REDO_UPDATE not yet coded.
Script is called from ma_test_all. Can run standalone.
2007-07-26 11:56:21 +02:00
|
|
|
goto err;
|
2006-04-11 15:45:10 +02:00
|
|
|
}
|
|
|
|
{
|
2007-01-18 20:38:14 +01:00
|
|
|
info.recpos= maria_position(file);
|
2006-04-11 15:45:10 +02:00
|
|
|
int skr=maria_rnext(file,read_record2,0);
|
|
|
|
if ((skr && my_errno != HA_ERR_END_OF_FILE) ||
|
|
|
|
maria_rprev(file,read_record2,-1) ||
|
2007-01-18 20:38:14 +01:00
|
|
|
memcmp(read_record,read_record2,reclength) != 0 ||
|
|
|
|
info.recpos != maria_position(file))
|
2006-04-11 15:45:10 +02:00
|
|
|
{
|
|
|
|
printf("maria_rsame_with_pos lost position\n");
|
WL#3072 - Maria recovery
Unit test for recovery: runs ma_test1 and ma_test2 (both only with
INSERTs and DELETEs; UPDATEs disabled as not handled by recovery)
then moves the tables elswhere; recreates tables from the log, and
compares and fails if there is a difference. Passes now.
Most of maria_read_log.c moved to ma_recovery.c, as it will be re-used
for recovery-from-ha_maria.
Bugfixes of applying of REDO_INSERT, REDO_PURGE_ROW.
Applying of REDO_PURGE_BLOCKS, REDO_DELETE_ALL, REDO_DROP_TABLE,
UNDO_ROW_INSERT (in REDO phase only, i.e. just doing records++),
UNDO_ROW_DELETE, UNDO_ROW_PURGE.
Code cleanups.
Monty: please look for "QQ". Sanja: please look for "Sanja".
Future tasks: recovery of the bitmap (easy), recovery of the state
(make it idempotent), more REDOs (Monty to work on
REDO_UPDATE?), UNDO phase...
Pushing this cset as it looks safe, contains test and bugfixes which
will help Monty implement applying of REDO_UPDATE.
sql/handler.cc:
typo
storage/maria/Makefile.am:
Adding ma_test_recovery (which ma_test_all invokes, and which can
also be run alone). Most of maria_read_log.c moved to ma_recovery.c
storage/maria/ha_maria.cc:
comments
storage/maria/ma_bitmap.c:
fixing comments. 2 -> sizeof(maria_bitmap_marker).
Bitmap-related part of _ma_initialize_datafile() moves in bitmap module.
Now putting the "bm" signature when creating the first bitmap page
(it used to happen only at next open, but that
caused an annoying difference when testing Recovery if the original
run didn't open the table, and it looks more
logical like this: it goes to disk only with its signature correct);
see the "QQ" comment towards the _ma_initialize_data_file() call
in ma_create.c for more).
When reading a bitmap page, verify its signature (happens when normally
using the table or when CHECKing it; not when REPAIRing it).
storage/maria/ma_blockrec.c:
* no need to sync the data file if table is not transactional
* Comments, code cleanup (log-related data moved to log-related code
block, int5store->page_store).
* Store the table's short id into LOGREC_UNDO_ROW_PURGE, like we
do for other records (though this record will soon be replaced
with a CLR).
* If "page" is 1 it means the page which extends from byte
page*block_size+1 to (page+1)*block_size (byte number 1 being
the first byte of the file). The last byte of the file is
data_file_length (same convention).
A new page needs to be created if the last byte of the page is
beyond the last byte of the file, i.e.
(page+1)*block_size+1 > data_file_length, so we correct the test
(bug found when testing log applying for ma_test1 -M -T --skip-update).
* update the page's LSN when removing a row from it during
execution of a REDO_PURGE_ROW record (bug found when testing log
applying for ma_test1 -M -T --skip-update).
* applying of REDO_PURGE_BLOCKs (limited to a one-page range for now).
storage/maria/ma_blockrec.h:
new functions. maria_bitmap_marker does not need to be exported.
storage/maria/ma_close.c:
we can always flush the table's state when closing the last instance
of the table. And it is needed for maria_read_log (as it does
not use maria_lock_database()).
storage/maria/ma_control_file.c:
when in Recovery, some assertions should not be used.
storage/maria/ma_control_file.h:
double-inclusion safe
storage/maria/ma_create.c:
during recovery, don't log records. Comments.
Moving the creation of the first bitmap page to ma_bitmap.c
storage/maria/ma_delete_table.c:
during recovery, don't log records. Log the end-zero of the dropped
table's name, so that recovery can use the string in place without
extending it to fit an end zero.
storage/maria/ma_loghandler.c:
* inwrite_rec_hook also needs access to the MARIA_SHARE, like
prewrite_rec_hook. This will be needed to update
share->records_diff (in the upcoming patch "recovery of the state").
* LOG_DESC::record_ends_group changed to an enum.
* LOG_DESC for LOGREC_REDO_PURGE_BLOCKS and LOGREC_UNDO_ROW_PURGE
corrected
* Sanja please see the @todo LOG BUG
* avoiding DBUG_RETURN(func()) as it gives confusing debug traces.
storage/maria/ma_loghandler.h:
- log write hooks called while the log's lock is held (inwrite_rec_hook)
now need the MARIA_SHARE, like prewrite_rec_hook already had
- instead of a bool saying if this record's type ends groups or not,
we refine: it may not end a group, it may end a group, or it may
be a group in itself. Imagine that we had a physical write failure
to a table before we log the UNDO, we still end up in
external_lock(F_UNLCK) and then we log a COMMIT: we don't want
to consider this COMMIT as ending the group of REDOs (don't want
to execute those REDOs during Recovery), that's why we say "COMMIT
is a group in itself, it aborts any previous group". This also
gives one more sanity check in maria_read_log.
storage/maria/ma_recovery.c:
New Recovery code, replacing the old pseudocode.
Most of maria_read_log moved here.
Call-able from ha_maria, but not enabled yet.
Compared to the previous version of maria_read_log, some bugs have
been fixed, debugging output can go to stdout or a disk file (for now
it's useful for me, later it can be changed), execution of
REDO_DROP_TABLE, REDO_DELETE_ALL, REDO_PURGE_BLOCKS has been added. Duplicate code
has been factored into functions. We abort an unfinished group
of records if we see a record which is a group in itself (like COMMIT).
No need for maria_panic() after a bug (which caused tables to not
be closed) was fixed; if there is yet another bug I prefer to see it.
When opening a table for Recovery, set data_file_length
and key_file_length to their real physical value (these are the
easiest state members to restore :). Warn us if the last page
was truncated (but Recovery handles it).
MARIA_SHARE::state::state::records is now partly recovered (not
idempotent, but works if recreating tables from scracth).
When applying a REDO to a page, stamp it with the UNDO's LSN
(current_group_end_lsn), not with the REDO's LSN; it makes
the table more identical to the original table (easier to compare
the two tables in the end).
Big thing missing: some types of REDOs are not handled,
and the UNDO phase does not exist (missing functions to execute UNDOs
to actually rollback). So for now tests are only inserting/deleting
a few 100 rows, closing the table and seeing if the log is applied ok;
it works. UPDATE not handled.
storage/maria/ma_recovery.h:
new functions: ma_recover() for recovery from inside ha_maria;
_ma_apply_log() for maria_read_log (ma_recover() calls _ma_apply_log()).
Btw, we need to not use the word "recover" for REPAIR/maria_chk anymore.
storage/maria/ma_rename.c:
don't write log records during recovery
storage/maria/ma_test2.c:
- fail if maria_info() or other subtests find some wrong information
- new option -g to skip updates.
- init the translog before creating the table, so that log applying
can work.
- in "#if 0" you'll see some fixed bugs (will be removed).
storage/maria/ma_test_all.sh:
cleanup files. Test log applying.
storage/maria/maria_read_log.c:
most of the logic moves to ma_recovery.c to be shared between
maria_read_log and recovery-from-inside-mysqld.
See ma_recovery.c for additional changes made to the moved code.
storage/maria/ma_test_recovery:
unit test for Recovery. Tests insert and delete,
REDO_UPDATE not yet coded.
Script is called from ma_test_all. Can run standalone.
2007-07-26 11:56:21 +02:00
|
|
|
goto err;
|
2006-04-11 15:45:10 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
ant=1;
|
|
|
|
while (maria_rnext(file,read_record2,0) == 0 &&
|
|
|
|
memcmp(read_record2+start,key,length) == 0) ant++;
|
|
|
|
if (ant != dupp_keys)
|
|
|
|
{
|
|
|
|
printf("next: Found: %d keys of %d\n",ant,dupp_keys);
|
WL#3072 - Maria recovery
Unit test for recovery: runs ma_test1 and ma_test2 (both only with
INSERTs and DELETEs; UPDATEs disabled as not handled by recovery)
then moves the tables elswhere; recreates tables from the log, and
compares and fails if there is a difference. Passes now.
Most of maria_read_log.c moved to ma_recovery.c, as it will be re-used
for recovery-from-ha_maria.
Bugfixes of applying of REDO_INSERT, REDO_PURGE_ROW.
Applying of REDO_PURGE_BLOCKS, REDO_DELETE_ALL, REDO_DROP_TABLE,
UNDO_ROW_INSERT (in REDO phase only, i.e. just doing records++),
UNDO_ROW_DELETE, UNDO_ROW_PURGE.
Code cleanups.
Monty: please look for "QQ". Sanja: please look for "Sanja".
Future tasks: recovery of the bitmap (easy), recovery of the state
(make it idempotent), more REDOs (Monty to work on
REDO_UPDATE?), UNDO phase...
Pushing this cset as it looks safe, contains test and bugfixes which
will help Monty implement applying of REDO_UPDATE.
sql/handler.cc:
typo
storage/maria/Makefile.am:
Adding ma_test_recovery (which ma_test_all invokes, and which can
also be run alone). Most of maria_read_log.c moved to ma_recovery.c
storage/maria/ha_maria.cc:
comments
storage/maria/ma_bitmap.c:
fixing comments. 2 -> sizeof(maria_bitmap_marker).
Bitmap-related part of _ma_initialize_datafile() moves in bitmap module.
Now putting the "bm" signature when creating the first bitmap page
(it used to happen only at next open, but that
caused an annoying difference when testing Recovery if the original
run didn't open the table, and it looks more
logical like this: it goes to disk only with its signature correct);
see the "QQ" comment towards the _ma_initialize_data_file() call
in ma_create.c for more).
When reading a bitmap page, verify its signature (happens when normally
using the table or when CHECKing it; not when REPAIRing it).
storage/maria/ma_blockrec.c:
* no need to sync the data file if table is not transactional
* Comments, code cleanup (log-related data moved to log-related code
block, int5store->page_store).
* Store the table's short id into LOGREC_UNDO_ROW_PURGE, like we
do for other records (though this record will soon be replaced
with a CLR).
* If "page" is 1 it means the page which extends from byte
page*block_size+1 to (page+1)*block_size (byte number 1 being
the first byte of the file). The last byte of the file is
data_file_length (same convention).
A new page needs to be created if the last byte of the page is
beyond the last byte of the file, i.e.
(page+1)*block_size+1 > data_file_length, so we correct the test
(bug found when testing log applying for ma_test1 -M -T --skip-update).
* update the page's LSN when removing a row from it during
execution of a REDO_PURGE_ROW record (bug found when testing log
applying for ma_test1 -M -T --skip-update).
* applying of REDO_PURGE_BLOCKs (limited to a one-page range for now).
storage/maria/ma_blockrec.h:
new functions. maria_bitmap_marker does not need to be exported.
storage/maria/ma_close.c:
we can always flush the table's state when closing the last instance
of the table. And it is needed for maria_read_log (as it does
not use maria_lock_database()).
storage/maria/ma_control_file.c:
when in Recovery, some assertions should not be used.
storage/maria/ma_control_file.h:
double-inclusion safe
storage/maria/ma_create.c:
during recovery, don't log records. Comments.
Moving the creation of the first bitmap page to ma_bitmap.c
storage/maria/ma_delete_table.c:
during recovery, don't log records. Log the end-zero of the dropped
table's name, so that recovery can use the string in place without
extending it to fit an end zero.
storage/maria/ma_loghandler.c:
* inwrite_rec_hook also needs access to the MARIA_SHARE, like
prewrite_rec_hook. This will be needed to update
share->records_diff (in the upcoming patch "recovery of the state").
* LOG_DESC::record_ends_group changed to an enum.
* LOG_DESC for LOGREC_REDO_PURGE_BLOCKS and LOGREC_UNDO_ROW_PURGE
corrected
* Sanja please see the @todo LOG BUG
* avoiding DBUG_RETURN(func()) as it gives confusing debug traces.
storage/maria/ma_loghandler.h:
- log write hooks called while the log's lock is held (inwrite_rec_hook)
now need the MARIA_SHARE, like prewrite_rec_hook already had
- instead of a bool saying if this record's type ends groups or not,
we refine: it may not end a group, it may end a group, or it may
be a group in itself. Imagine that we had a physical write failure
to a table before we log the UNDO, we still end up in
external_lock(F_UNLCK) and then we log a COMMIT: we don't want
to consider this COMMIT as ending the group of REDOs (don't want
to execute those REDOs during Recovery), that's why we say "COMMIT
is a group in itself, it aborts any previous group". This also
gives one more sanity check in maria_read_log.
storage/maria/ma_recovery.c:
New Recovery code, replacing the old pseudocode.
Most of maria_read_log moved here.
Call-able from ha_maria, but not enabled yet.
Compared to the previous version of maria_read_log, some bugs have
been fixed, debugging output can go to stdout or a disk file (for now
it's useful for me, later it can be changed), execution of
REDO_DROP_TABLE, REDO_DELETE_ALL, REDO_PURGE_BLOCKS has been added. Duplicate code
has been factored into functions. We abort an unfinished group
of records if we see a record which is a group in itself (like COMMIT).
No need for maria_panic() after a bug (which caused tables to not
be closed) was fixed; if there is yet another bug I prefer to see it.
When opening a table for Recovery, set data_file_length
and key_file_length to their real physical value (these are the
easiest state members to restore :). Warn us if the last page
was truncated (but Recovery handles it).
MARIA_SHARE::state::state::records is now partly recovered (not
idempotent, but works if recreating tables from scracth).
When applying a REDO to a page, stamp it with the UNDO's LSN
(current_group_end_lsn), not with the REDO's LSN; it makes
the table more identical to the original table (easier to compare
the two tables in the end).
Big thing missing: some types of REDOs are not handled,
and the UNDO phase does not exist (missing functions to execute UNDOs
to actually rollback). So for now tests are only inserting/deleting
a few 100 rows, closing the table and seeing if the log is applied ok;
it works. UPDATE not handled.
storage/maria/ma_recovery.h:
new functions: ma_recover() for recovery from inside ha_maria;
_ma_apply_log() for maria_read_log (ma_recover() calls _ma_apply_log()).
Btw, we need to not use the word "recover" for REPAIR/maria_chk anymore.
storage/maria/ma_rename.c:
don't write log records during recovery
storage/maria/ma_test2.c:
- fail if maria_info() or other subtests find some wrong information
- new option -g to skip updates.
- init the translog before creating the table, so that log applying
can work.
- in "#if 0" you'll see some fixed bugs (will be removed).
storage/maria/ma_test_all.sh:
cleanup files. Test log applying.
storage/maria/maria_read_log.c:
most of the logic moves to ma_recovery.c to be shared between
maria_read_log and recovery-from-inside-mysqld.
See ma_recovery.c for additional changes made to the moved code.
storage/maria/ma_test_recovery:
unit test for Recovery. Tests insert and delete,
REDO_UPDATE not yet coded.
Script is called from ma_test_all. Can run standalone.
2007-07-26 11:56:21 +02:00
|
|
|
goto err;
|
2006-04-11 15:45:10 +02:00
|
|
|
}
|
|
|
|
ant=0;
|
|
|
|
while (maria_rprev(file,read_record3,0) == 0 &&
|
|
|
|
bcmp(read_record3+start,key,length) == 0) ant++;
|
|
|
|
if (ant != dupp_keys)
|
|
|
|
{
|
|
|
|
printf("prev: Found: %d records of %d\n",ant,dupp_keys);
|
WL#3072 - Maria recovery
Unit test for recovery: runs ma_test1 and ma_test2 (both only with
INSERTs and DELETEs; UPDATEs disabled as not handled by recovery)
then moves the tables elswhere; recreates tables from the log, and
compares and fails if there is a difference. Passes now.
Most of maria_read_log.c moved to ma_recovery.c, as it will be re-used
for recovery-from-ha_maria.
Bugfixes of applying of REDO_INSERT, REDO_PURGE_ROW.
Applying of REDO_PURGE_BLOCKS, REDO_DELETE_ALL, REDO_DROP_TABLE,
UNDO_ROW_INSERT (in REDO phase only, i.e. just doing records++),
UNDO_ROW_DELETE, UNDO_ROW_PURGE.
Code cleanups.
Monty: please look for "QQ". Sanja: please look for "Sanja".
Future tasks: recovery of the bitmap (easy), recovery of the state
(make it idempotent), more REDOs (Monty to work on
REDO_UPDATE?), UNDO phase...
Pushing this cset as it looks safe, contains test and bugfixes which
will help Monty implement applying of REDO_UPDATE.
sql/handler.cc:
typo
storage/maria/Makefile.am:
Adding ma_test_recovery (which ma_test_all invokes, and which can
also be run alone). Most of maria_read_log.c moved to ma_recovery.c
storage/maria/ha_maria.cc:
comments
storage/maria/ma_bitmap.c:
fixing comments. 2 -> sizeof(maria_bitmap_marker).
Bitmap-related part of _ma_initialize_datafile() moves in bitmap module.
Now putting the "bm" signature when creating the first bitmap page
(it used to happen only at next open, but that
caused an annoying difference when testing Recovery if the original
run didn't open the table, and it looks more
logical like this: it goes to disk only with its signature correct);
see the "QQ" comment towards the _ma_initialize_data_file() call
in ma_create.c for more).
When reading a bitmap page, verify its signature (happens when normally
using the table or when CHECKing it; not when REPAIRing it).
storage/maria/ma_blockrec.c:
* no need to sync the data file if table is not transactional
* Comments, code cleanup (log-related data moved to log-related code
block, int5store->page_store).
* Store the table's short id into LOGREC_UNDO_ROW_PURGE, like we
do for other records (though this record will soon be replaced
with a CLR).
* If "page" is 1 it means the page which extends from byte
page*block_size+1 to (page+1)*block_size (byte number 1 being
the first byte of the file). The last byte of the file is
data_file_length (same convention).
A new page needs to be created if the last byte of the page is
beyond the last byte of the file, i.e.
(page+1)*block_size+1 > data_file_length, so we correct the test
(bug found when testing log applying for ma_test1 -M -T --skip-update).
* update the page's LSN when removing a row from it during
execution of a REDO_PURGE_ROW record (bug found when testing log
applying for ma_test1 -M -T --skip-update).
* applying of REDO_PURGE_BLOCKs (limited to a one-page range for now).
storage/maria/ma_blockrec.h:
new functions. maria_bitmap_marker does not need to be exported.
storage/maria/ma_close.c:
we can always flush the table's state when closing the last instance
of the table. And it is needed for maria_read_log (as it does
not use maria_lock_database()).
storage/maria/ma_control_file.c:
when in Recovery, some assertions should not be used.
storage/maria/ma_control_file.h:
double-inclusion safe
storage/maria/ma_create.c:
during recovery, don't log records. Comments.
Moving the creation of the first bitmap page to ma_bitmap.c
storage/maria/ma_delete_table.c:
during recovery, don't log records. Log the end-zero of the dropped
table's name, so that recovery can use the string in place without
extending it to fit an end zero.
storage/maria/ma_loghandler.c:
* inwrite_rec_hook also needs access to the MARIA_SHARE, like
prewrite_rec_hook. This will be needed to update
share->records_diff (in the upcoming patch "recovery of the state").
* LOG_DESC::record_ends_group changed to an enum.
* LOG_DESC for LOGREC_REDO_PURGE_BLOCKS and LOGREC_UNDO_ROW_PURGE
corrected
* Sanja please see the @todo LOG BUG
* avoiding DBUG_RETURN(func()) as it gives confusing debug traces.
storage/maria/ma_loghandler.h:
- log write hooks called while the log's lock is held (inwrite_rec_hook)
now need the MARIA_SHARE, like prewrite_rec_hook already had
- instead of a bool saying if this record's type ends groups or not,
we refine: it may not end a group, it may end a group, or it may
be a group in itself. Imagine that we had a physical write failure
to a table before we log the UNDO, we still end up in
external_lock(F_UNLCK) and then we log a COMMIT: we don't want
to consider this COMMIT as ending the group of REDOs (don't want
to execute those REDOs during Recovery), that's why we say "COMMIT
is a group in itself, it aborts any previous group". This also
gives one more sanity check in maria_read_log.
storage/maria/ma_recovery.c:
New Recovery code, replacing the old pseudocode.
Most of maria_read_log moved here.
Call-able from ha_maria, but not enabled yet.
Compared to the previous version of maria_read_log, some bugs have
been fixed, debugging output can go to stdout or a disk file (for now
it's useful for me, later it can be changed), execution of
REDO_DROP_TABLE, REDO_DELETE_ALL, REDO_PURGE_BLOCKS has been added. Duplicate code
has been factored into functions. We abort an unfinished group
of records if we see a record which is a group in itself (like COMMIT).
No need for maria_panic() after a bug (which caused tables to not
be closed) was fixed; if there is yet another bug I prefer to see it.
When opening a table for Recovery, set data_file_length
and key_file_length to their real physical value (these are the
easiest state members to restore :). Warn us if the last page
was truncated (but Recovery handles it).
MARIA_SHARE::state::state::records is now partly recovered (not
idempotent, but works if recreating tables from scracth).
When applying a REDO to a page, stamp it with the UNDO's LSN
(current_group_end_lsn), not with the REDO's LSN; it makes
the table more identical to the original table (easier to compare
the two tables in the end).
Big thing missing: some types of REDOs are not handled,
and the UNDO phase does not exist (missing functions to execute UNDOs
to actually rollback). So for now tests are only inserting/deleting
a few 100 rows, closing the table and seeing if the log is applied ok;
it works. UPDATE not handled.
storage/maria/ma_recovery.h:
new functions: ma_recover() for recovery from inside ha_maria;
_ma_apply_log() for maria_read_log (ma_recover() calls _ma_apply_log()).
Btw, we need to not use the word "recover" for REPAIR/maria_chk anymore.
storage/maria/ma_rename.c:
don't write log records during recovery
storage/maria/ma_test2.c:
- fail if maria_info() or other subtests find some wrong information
- new option -g to skip updates.
- init the translog before creating the table, so that log applying
can work.
- in "#if 0" you'll see some fixed bugs (will be removed).
storage/maria/ma_test_all.sh:
cleanup files. Test log applying.
storage/maria/maria_read_log.c:
most of the logic moves to ma_recovery.c to be shared between
maria_read_log and recovery-from-inside-mysqld.
See ma_recovery.c for additional changes made to the moved code.
storage/maria/ma_test_recovery:
unit test for Recovery. Tests insert and delete,
REDO_UPDATE not yet coded.
Script is called from ma_test_all. Can run standalone.
2007-07-26 11:56:21 +02:00
|
|
|
goto err;
|
2006-04-11 15:45:10 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
/* Check of maria_rnext_same */
|
2007-07-27 16:11:40 +02:00
|
|
|
if (maria_rkey(file,read_record,0,key,HA_WHOLE_KEY,HA_READ_KEY_EXACT))
|
2006-04-11 15:45:10 +02:00
|
|
|
goto err;
|
|
|
|
ant=1;
|
|
|
|
while (!maria_rnext_same(file,read_record3) && ant < dupp_keys+10)
|
|
|
|
ant++;
|
|
|
|
if (ant != dupp_keys || my_errno != HA_ERR_END_OF_FILE)
|
|
|
|
{
|
|
|
|
printf("maria_rnext_same: Found: %d records of %d\n",ant,dupp_keys);
|
WL#3072 - Maria recovery
Unit test for recovery: runs ma_test1 and ma_test2 (both only with
INSERTs and DELETEs; UPDATEs disabled as not handled by recovery)
then moves the tables elswhere; recreates tables from the log, and
compares and fails if there is a difference. Passes now.
Most of maria_read_log.c moved to ma_recovery.c, as it will be re-used
for recovery-from-ha_maria.
Bugfixes of applying of REDO_INSERT, REDO_PURGE_ROW.
Applying of REDO_PURGE_BLOCKS, REDO_DELETE_ALL, REDO_DROP_TABLE,
UNDO_ROW_INSERT (in REDO phase only, i.e. just doing records++),
UNDO_ROW_DELETE, UNDO_ROW_PURGE.
Code cleanups.
Monty: please look for "QQ". Sanja: please look for "Sanja".
Future tasks: recovery of the bitmap (easy), recovery of the state
(make it idempotent), more REDOs (Monty to work on
REDO_UPDATE?), UNDO phase...
Pushing this cset as it looks safe, contains test and bugfixes which
will help Monty implement applying of REDO_UPDATE.
sql/handler.cc:
typo
storage/maria/Makefile.am:
Adding ma_test_recovery (which ma_test_all invokes, and which can
also be run alone). Most of maria_read_log.c moved to ma_recovery.c
storage/maria/ha_maria.cc:
comments
storage/maria/ma_bitmap.c:
fixing comments. 2 -> sizeof(maria_bitmap_marker).
Bitmap-related part of _ma_initialize_datafile() moves in bitmap module.
Now putting the "bm" signature when creating the first bitmap page
(it used to happen only at next open, but that
caused an annoying difference when testing Recovery if the original
run didn't open the table, and it looks more
logical like this: it goes to disk only with its signature correct);
see the "QQ" comment towards the _ma_initialize_data_file() call
in ma_create.c for more).
When reading a bitmap page, verify its signature (happens when normally
using the table or when CHECKing it; not when REPAIRing it).
storage/maria/ma_blockrec.c:
* no need to sync the data file if table is not transactional
* Comments, code cleanup (log-related data moved to log-related code
block, int5store->page_store).
* Store the table's short id into LOGREC_UNDO_ROW_PURGE, like we
do for other records (though this record will soon be replaced
with a CLR).
* If "page" is 1 it means the page which extends from byte
page*block_size+1 to (page+1)*block_size (byte number 1 being
the first byte of the file). The last byte of the file is
data_file_length (same convention).
A new page needs to be created if the last byte of the page is
beyond the last byte of the file, i.e.
(page+1)*block_size+1 > data_file_length, so we correct the test
(bug found when testing log applying for ma_test1 -M -T --skip-update).
* update the page's LSN when removing a row from it during
execution of a REDO_PURGE_ROW record (bug found when testing log
applying for ma_test1 -M -T --skip-update).
* applying of REDO_PURGE_BLOCKs (limited to a one-page range for now).
storage/maria/ma_blockrec.h:
new functions. maria_bitmap_marker does not need to be exported.
storage/maria/ma_close.c:
we can always flush the table's state when closing the last instance
of the table. And it is needed for maria_read_log (as it does
not use maria_lock_database()).
storage/maria/ma_control_file.c:
when in Recovery, some assertions should not be used.
storage/maria/ma_control_file.h:
double-inclusion safe
storage/maria/ma_create.c:
during recovery, don't log records. Comments.
Moving the creation of the first bitmap page to ma_bitmap.c
storage/maria/ma_delete_table.c:
during recovery, don't log records. Log the end-zero of the dropped
table's name, so that recovery can use the string in place without
extending it to fit an end zero.
storage/maria/ma_loghandler.c:
* inwrite_rec_hook also needs access to the MARIA_SHARE, like
prewrite_rec_hook. This will be needed to update
share->records_diff (in the upcoming patch "recovery of the state").
* LOG_DESC::record_ends_group changed to an enum.
* LOG_DESC for LOGREC_REDO_PURGE_BLOCKS and LOGREC_UNDO_ROW_PURGE
corrected
* Sanja please see the @todo LOG BUG
* avoiding DBUG_RETURN(func()) as it gives confusing debug traces.
storage/maria/ma_loghandler.h:
- log write hooks called while the log's lock is held (inwrite_rec_hook)
now need the MARIA_SHARE, like prewrite_rec_hook already had
- instead of a bool saying if this record's type ends groups or not,
we refine: it may not end a group, it may end a group, or it may
be a group in itself. Imagine that we had a physical write failure
to a table before we log the UNDO, we still end up in
external_lock(F_UNLCK) and then we log a COMMIT: we don't want
to consider this COMMIT as ending the group of REDOs (don't want
to execute those REDOs during Recovery), that's why we say "COMMIT
is a group in itself, it aborts any previous group". This also
gives one more sanity check in maria_read_log.
storage/maria/ma_recovery.c:
New Recovery code, replacing the old pseudocode.
Most of maria_read_log moved here.
Call-able from ha_maria, but not enabled yet.
Compared to the previous version of maria_read_log, some bugs have
been fixed, debugging output can go to stdout or a disk file (for now
it's useful for me, later it can be changed), execution of
REDO_DROP_TABLE, REDO_DELETE_ALL, REDO_PURGE_BLOCKS has been added. Duplicate code
has been factored into functions. We abort an unfinished group
of records if we see a record which is a group in itself (like COMMIT).
No need for maria_panic() after a bug (which caused tables to not
be closed) was fixed; if there is yet another bug I prefer to see it.
When opening a table for Recovery, set data_file_length
and key_file_length to their real physical value (these are the
easiest state members to restore :). Warn us if the last page
was truncated (but Recovery handles it).
MARIA_SHARE::state::state::records is now partly recovered (not
idempotent, but works if recreating tables from scracth).
When applying a REDO to a page, stamp it with the UNDO's LSN
(current_group_end_lsn), not with the REDO's LSN; it makes
the table more identical to the original table (easier to compare
the two tables in the end).
Big thing missing: some types of REDOs are not handled,
and the UNDO phase does not exist (missing functions to execute UNDOs
to actually rollback). So for now tests are only inserting/deleting
a few 100 rows, closing the table and seeing if the log is applied ok;
it works. UPDATE not handled.
storage/maria/ma_recovery.h:
new functions: ma_recover() for recovery from inside ha_maria;
_ma_apply_log() for maria_read_log (ma_recover() calls _ma_apply_log()).
Btw, we need to not use the word "recover" for REPAIR/maria_chk anymore.
storage/maria/ma_rename.c:
don't write log records during recovery
storage/maria/ma_test2.c:
- fail if maria_info() or other subtests find some wrong information
- new option -g to skip updates.
- init the translog before creating the table, so that log applying
can work.
- in "#if 0" you'll see some fixed bugs (will be removed).
storage/maria/ma_test_all.sh:
cleanup files. Test log applying.
storage/maria/maria_read_log.c:
most of the logic moves to ma_recovery.c to be shared between
maria_read_log and recovery-from-inside-mysqld.
See ma_recovery.c for additional changes made to the moved code.
storage/maria/ma_test_recovery:
unit test for Recovery. Tests insert and delete,
REDO_UPDATE not yet coded.
Script is called from ma_test_all. Can run standalone.
2007-07-26 11:56:21 +02:00
|
|
|
goto err;
|
2006-04-11 15:45:10 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if (!silent)
|
|
|
|
printf("- All keys: first - next -> last - prev -> first\n");
|
|
|
|
DBUG_PRINT("progpos",("All keys: first - next -> last - prev -> first"));
|
|
|
|
ant=1;
|
|
|
|
if (maria_rfirst(file,read_record,0))
|
|
|
|
{
|
|
|
|
printf("Can't find first record\n");
|
WL#3072 - Maria recovery
Unit test for recovery: runs ma_test1 and ma_test2 (both only with
INSERTs and DELETEs; UPDATEs disabled as not handled by recovery)
then moves the tables elswhere; recreates tables from the log, and
compares and fails if there is a difference. Passes now.
Most of maria_read_log.c moved to ma_recovery.c, as it will be re-used
for recovery-from-ha_maria.
Bugfixes of applying of REDO_INSERT, REDO_PURGE_ROW.
Applying of REDO_PURGE_BLOCKS, REDO_DELETE_ALL, REDO_DROP_TABLE,
UNDO_ROW_INSERT (in REDO phase only, i.e. just doing records++),
UNDO_ROW_DELETE, UNDO_ROW_PURGE.
Code cleanups.
Monty: please look for "QQ". Sanja: please look for "Sanja".
Future tasks: recovery of the bitmap (easy), recovery of the state
(make it idempotent), more REDOs (Monty to work on
REDO_UPDATE?), UNDO phase...
Pushing this cset as it looks safe, contains test and bugfixes which
will help Monty implement applying of REDO_UPDATE.
sql/handler.cc:
typo
storage/maria/Makefile.am:
Adding ma_test_recovery (which ma_test_all invokes, and which can
also be run alone). Most of maria_read_log.c moved to ma_recovery.c
storage/maria/ha_maria.cc:
comments
storage/maria/ma_bitmap.c:
fixing comments. 2 -> sizeof(maria_bitmap_marker).
Bitmap-related part of _ma_initialize_datafile() moves in bitmap module.
Now putting the "bm" signature when creating the first bitmap page
(it used to happen only at next open, but that
caused an annoying difference when testing Recovery if the original
run didn't open the table, and it looks more
logical like this: it goes to disk only with its signature correct);
see the "QQ" comment towards the _ma_initialize_data_file() call
in ma_create.c for more).
When reading a bitmap page, verify its signature (happens when normally
using the table or when CHECKing it; not when REPAIRing it).
storage/maria/ma_blockrec.c:
* no need to sync the data file if table is not transactional
* Comments, code cleanup (log-related data moved to log-related code
block, int5store->page_store).
* Store the table's short id into LOGREC_UNDO_ROW_PURGE, like we
do for other records (though this record will soon be replaced
with a CLR).
* If "page" is 1 it means the page which extends from byte
page*block_size+1 to (page+1)*block_size (byte number 1 being
the first byte of the file). The last byte of the file is
data_file_length (same convention).
A new page needs to be created if the last byte of the page is
beyond the last byte of the file, i.e.
(page+1)*block_size+1 > data_file_length, so we correct the test
(bug found when testing log applying for ma_test1 -M -T --skip-update).
* update the page's LSN when removing a row from it during
execution of a REDO_PURGE_ROW record (bug found when testing log
applying for ma_test1 -M -T --skip-update).
* applying of REDO_PURGE_BLOCKs (limited to a one-page range for now).
storage/maria/ma_blockrec.h:
new functions. maria_bitmap_marker does not need to be exported.
storage/maria/ma_close.c:
we can always flush the table's state when closing the last instance
of the table. And it is needed for maria_read_log (as it does
not use maria_lock_database()).
storage/maria/ma_control_file.c:
when in Recovery, some assertions should not be used.
storage/maria/ma_control_file.h:
double-inclusion safe
storage/maria/ma_create.c:
during recovery, don't log records. Comments.
Moving the creation of the first bitmap page to ma_bitmap.c
storage/maria/ma_delete_table.c:
during recovery, don't log records. Log the end-zero of the dropped
table's name, so that recovery can use the string in place without
extending it to fit an end zero.
storage/maria/ma_loghandler.c:
* inwrite_rec_hook also needs access to the MARIA_SHARE, like
prewrite_rec_hook. This will be needed to update
share->records_diff (in the upcoming patch "recovery of the state").
* LOG_DESC::record_ends_group changed to an enum.
* LOG_DESC for LOGREC_REDO_PURGE_BLOCKS and LOGREC_UNDO_ROW_PURGE
corrected
* Sanja please see the @todo LOG BUG
* avoiding DBUG_RETURN(func()) as it gives confusing debug traces.
storage/maria/ma_loghandler.h:
- log write hooks called while the log's lock is held (inwrite_rec_hook)
now need the MARIA_SHARE, like prewrite_rec_hook already had
- instead of a bool saying if this record's type ends groups or not,
we refine: it may not end a group, it may end a group, or it may
be a group in itself. Imagine that we had a physical write failure
to a table before we log the UNDO, we still end up in
external_lock(F_UNLCK) and then we log a COMMIT: we don't want
to consider this COMMIT as ending the group of REDOs (don't want
to execute those REDOs during Recovery), that's why we say "COMMIT
is a group in itself, it aborts any previous group". This also
gives one more sanity check in maria_read_log.
storage/maria/ma_recovery.c:
New Recovery code, replacing the old pseudocode.
Most of maria_read_log moved here.
Call-able from ha_maria, but not enabled yet.
Compared to the previous version of maria_read_log, some bugs have
been fixed, debugging output can go to stdout or a disk file (for now
it's useful for me, later it can be changed), execution of
REDO_DROP_TABLE, REDO_DELETE_ALL, REDO_PURGE_BLOCKS has been added. Duplicate code
has been factored into functions. We abort an unfinished group
of records if we see a record which is a group in itself (like COMMIT).
No need for maria_panic() after a bug (which caused tables to not
be closed) was fixed; if there is yet another bug I prefer to see it.
When opening a table for Recovery, set data_file_length
and key_file_length to their real physical value (these are the
easiest state members to restore :). Warn us if the last page
was truncated (but Recovery handles it).
MARIA_SHARE::state::state::records is now partly recovered (not
idempotent, but works if recreating tables from scracth).
When applying a REDO to a page, stamp it with the UNDO's LSN
(current_group_end_lsn), not with the REDO's LSN; it makes
the table more identical to the original table (easier to compare
the two tables in the end).
Big thing missing: some types of REDOs are not handled,
and the UNDO phase does not exist (missing functions to execute UNDOs
to actually rollback). So for now tests are only inserting/deleting
a few 100 rows, closing the table and seeing if the log is applied ok;
it works. UPDATE not handled.
storage/maria/ma_recovery.h:
new functions: ma_recover() for recovery from inside ha_maria;
_ma_apply_log() for maria_read_log (ma_recover() calls _ma_apply_log()).
Btw, we need to not use the word "recover" for REPAIR/maria_chk anymore.
storage/maria/ma_rename.c:
don't write log records during recovery
storage/maria/ma_test2.c:
- fail if maria_info() or other subtests find some wrong information
- new option -g to skip updates.
- init the translog before creating the table, so that log applying
can work.
- in "#if 0" you'll see some fixed bugs (will be removed).
storage/maria/ma_test_all.sh:
cleanup files. Test log applying.
storage/maria/maria_read_log.c:
most of the logic moves to ma_recovery.c to be shared between
maria_read_log and recovery-from-inside-mysqld.
See ma_recovery.c for additional changes made to the moved code.
storage/maria/ma_test_recovery:
unit test for Recovery. Tests insert and delete,
REDO_UPDATE not yet coded.
Script is called from ma_test_all. Can run standalone.
2007-07-26 11:56:21 +02:00
|
|
|
goto err;
|
2006-04-11 15:45:10 +02:00
|
|
|
}
|
|
|
|
while ((error=maria_rnext(file,read_record3,0)) == 0 && ant < write_count+10)
|
|
|
|
ant++;
|
|
|
|
if (ant != write_count - opt_delete || error != HA_ERR_END_OF_FILE)
|
|
|
|
{
|
|
|
|
printf("next: I found: %d records of %d (error: %d)\n",
|
|
|
|
ant, write_count - opt_delete, error);
|
WL#3072 - Maria recovery
Unit test for recovery: runs ma_test1 and ma_test2 (both only with
INSERTs and DELETEs; UPDATEs disabled as not handled by recovery)
then moves the tables elswhere; recreates tables from the log, and
compares and fails if there is a difference. Passes now.
Most of maria_read_log.c moved to ma_recovery.c, as it will be re-used
for recovery-from-ha_maria.
Bugfixes of applying of REDO_INSERT, REDO_PURGE_ROW.
Applying of REDO_PURGE_BLOCKS, REDO_DELETE_ALL, REDO_DROP_TABLE,
UNDO_ROW_INSERT (in REDO phase only, i.e. just doing records++),
UNDO_ROW_DELETE, UNDO_ROW_PURGE.
Code cleanups.
Monty: please look for "QQ". Sanja: please look for "Sanja".
Future tasks: recovery of the bitmap (easy), recovery of the state
(make it idempotent), more REDOs (Monty to work on
REDO_UPDATE?), UNDO phase...
Pushing this cset as it looks safe, contains test and bugfixes which
will help Monty implement applying of REDO_UPDATE.
sql/handler.cc:
typo
storage/maria/Makefile.am:
Adding ma_test_recovery (which ma_test_all invokes, and which can
also be run alone). Most of maria_read_log.c moved to ma_recovery.c
storage/maria/ha_maria.cc:
comments
storage/maria/ma_bitmap.c:
fixing comments. 2 -> sizeof(maria_bitmap_marker).
Bitmap-related part of _ma_initialize_datafile() moves in bitmap module.
Now putting the "bm" signature when creating the first bitmap page
(it used to happen only at next open, but that
caused an annoying difference when testing Recovery if the original
run didn't open the table, and it looks more
logical like this: it goes to disk only with its signature correct);
see the "QQ" comment towards the _ma_initialize_data_file() call
in ma_create.c for more).
When reading a bitmap page, verify its signature (happens when normally
using the table or when CHECKing it; not when REPAIRing it).
storage/maria/ma_blockrec.c:
* no need to sync the data file if table is not transactional
* Comments, code cleanup (log-related data moved to log-related code
block, int5store->page_store).
* Store the table's short id into LOGREC_UNDO_ROW_PURGE, like we
do for other records (though this record will soon be replaced
with a CLR).
* If "page" is 1 it means the page which extends from byte
page*block_size+1 to (page+1)*block_size (byte number 1 being
the first byte of the file). The last byte of the file is
data_file_length (same convention).
A new page needs to be created if the last byte of the page is
beyond the last byte of the file, i.e.
(page+1)*block_size+1 > data_file_length, so we correct the test
(bug found when testing log applying for ma_test1 -M -T --skip-update).
* update the page's LSN when removing a row from it during
execution of a REDO_PURGE_ROW record (bug found when testing log
applying for ma_test1 -M -T --skip-update).
* applying of REDO_PURGE_BLOCKs (limited to a one-page range for now).
storage/maria/ma_blockrec.h:
new functions. maria_bitmap_marker does not need to be exported.
storage/maria/ma_close.c:
we can always flush the table's state when closing the last instance
of the table. And it is needed for maria_read_log (as it does
not use maria_lock_database()).
storage/maria/ma_control_file.c:
when in Recovery, some assertions should not be used.
storage/maria/ma_control_file.h:
double-inclusion safe
storage/maria/ma_create.c:
during recovery, don't log records. Comments.
Moving the creation of the first bitmap page to ma_bitmap.c
storage/maria/ma_delete_table.c:
during recovery, don't log records. Log the end-zero of the dropped
table's name, so that recovery can use the string in place without
extending it to fit an end zero.
storage/maria/ma_loghandler.c:
* inwrite_rec_hook also needs access to the MARIA_SHARE, like
prewrite_rec_hook. This will be needed to update
share->records_diff (in the upcoming patch "recovery of the state").
* LOG_DESC::record_ends_group changed to an enum.
* LOG_DESC for LOGREC_REDO_PURGE_BLOCKS and LOGREC_UNDO_ROW_PURGE
corrected
* Sanja please see the @todo LOG BUG
* avoiding DBUG_RETURN(func()) as it gives confusing debug traces.
storage/maria/ma_loghandler.h:
- log write hooks called while the log's lock is held (inwrite_rec_hook)
now need the MARIA_SHARE, like prewrite_rec_hook already had
- instead of a bool saying if this record's type ends groups or not,
we refine: it may not end a group, it may end a group, or it may
be a group in itself. Imagine that we had a physical write failure
to a table before we log the UNDO, we still end up in
external_lock(F_UNLCK) and then we log a COMMIT: we don't want
to consider this COMMIT as ending the group of REDOs (don't want
to execute those REDOs during Recovery), that's why we say "COMMIT
is a group in itself, it aborts any previous group". This also
gives one more sanity check in maria_read_log.
storage/maria/ma_recovery.c:
New Recovery code, replacing the old pseudocode.
Most of maria_read_log moved here.
Call-able from ha_maria, but not enabled yet.
Compared to the previous version of maria_read_log, some bugs have
been fixed, debugging output can go to stdout or a disk file (for now
it's useful for me, later it can be changed), execution of
REDO_DROP_TABLE, REDO_DELETE_ALL, REDO_PURGE_BLOCKS has been added. Duplicate code
has been factored into functions. We abort an unfinished group
of records if we see a record which is a group in itself (like COMMIT).
No need for maria_panic() after a bug (which caused tables to not
be closed) was fixed; if there is yet another bug I prefer to see it.
When opening a table for Recovery, set data_file_length
and key_file_length to their real physical value (these are the
easiest state members to restore :). Warn us if the last page
was truncated (but Recovery handles it).
MARIA_SHARE::state::state::records is now partly recovered (not
idempotent, but works if recreating tables from scracth).
When applying a REDO to a page, stamp it with the UNDO's LSN
(current_group_end_lsn), not with the REDO's LSN; it makes
the table more identical to the original table (easier to compare
the two tables in the end).
Big thing missing: some types of REDOs are not handled,
and the UNDO phase does not exist (missing functions to execute UNDOs
to actually rollback). So for now tests are only inserting/deleting
a few 100 rows, closing the table and seeing if the log is applied ok;
it works. UPDATE not handled.
storage/maria/ma_recovery.h:
new functions: ma_recover() for recovery from inside ha_maria;
_ma_apply_log() for maria_read_log (ma_recover() calls _ma_apply_log()).
Btw, we need to not use the word "recover" for REPAIR/maria_chk anymore.
storage/maria/ma_rename.c:
don't write log records during recovery
storage/maria/ma_test2.c:
- fail if maria_info() or other subtests find some wrong information
- new option -g to skip updates.
- init the translog before creating the table, so that log applying
can work.
- in "#if 0" you'll see some fixed bugs (will be removed).
storage/maria/ma_test_all.sh:
cleanup files. Test log applying.
storage/maria/maria_read_log.c:
most of the logic moves to ma_recovery.c to be shared between
maria_read_log and recovery-from-inside-mysqld.
See ma_recovery.c for additional changes made to the moved code.
storage/maria/ma_test_recovery:
unit test for Recovery. Tests insert and delete,
REDO_UPDATE not yet coded.
Script is called from ma_test_all. Can run standalone.
2007-07-26 11:56:21 +02:00
|
|
|
goto err;
|
2006-04-11 15:45:10 +02:00
|
|
|
}
|
|
|
|
if (maria_rlast(file,read_record2,0) ||
|
|
|
|
bcmp(read_record2,read_record3,reclength))
|
|
|
|
{
|
|
|
|
printf("Can't find last record\n");
|
2007-07-02 19:45:15 +02:00
|
|
|
DBUG_DUMP("record2",(uchar*) read_record2,reclength);
|
|
|
|
DBUG_DUMP("record3",(uchar*) read_record3,reclength);
|
WL#3072 - Maria recovery
Unit test for recovery: runs ma_test1 and ma_test2 (both only with
INSERTs and DELETEs; UPDATEs disabled as not handled by recovery)
then moves the tables elswhere; recreates tables from the log, and
compares and fails if there is a difference. Passes now.
Most of maria_read_log.c moved to ma_recovery.c, as it will be re-used
for recovery-from-ha_maria.
Bugfixes of applying of REDO_INSERT, REDO_PURGE_ROW.
Applying of REDO_PURGE_BLOCKS, REDO_DELETE_ALL, REDO_DROP_TABLE,
UNDO_ROW_INSERT (in REDO phase only, i.e. just doing records++),
UNDO_ROW_DELETE, UNDO_ROW_PURGE.
Code cleanups.
Monty: please look for "QQ". Sanja: please look for "Sanja".
Future tasks: recovery of the bitmap (easy), recovery of the state
(make it idempotent), more REDOs (Monty to work on
REDO_UPDATE?), UNDO phase...
Pushing this cset as it looks safe, contains test and bugfixes which
will help Monty implement applying of REDO_UPDATE.
sql/handler.cc:
typo
storage/maria/Makefile.am:
Adding ma_test_recovery (which ma_test_all invokes, and which can
also be run alone). Most of maria_read_log.c moved to ma_recovery.c
storage/maria/ha_maria.cc:
comments
storage/maria/ma_bitmap.c:
fixing comments. 2 -> sizeof(maria_bitmap_marker).
Bitmap-related part of _ma_initialize_datafile() moves in bitmap module.
Now putting the "bm" signature when creating the first bitmap page
(it used to happen only at next open, but that
caused an annoying difference when testing Recovery if the original
run didn't open the table, and it looks more
logical like this: it goes to disk only with its signature correct);
see the "QQ" comment towards the _ma_initialize_data_file() call
in ma_create.c for more).
When reading a bitmap page, verify its signature (happens when normally
using the table or when CHECKing it; not when REPAIRing it).
storage/maria/ma_blockrec.c:
* no need to sync the data file if table is not transactional
* Comments, code cleanup (log-related data moved to log-related code
block, int5store->page_store).
* Store the table's short id into LOGREC_UNDO_ROW_PURGE, like we
do for other records (though this record will soon be replaced
with a CLR).
* If "page" is 1 it means the page which extends from byte
page*block_size+1 to (page+1)*block_size (byte number 1 being
the first byte of the file). The last byte of the file is
data_file_length (same convention).
A new page needs to be created if the last byte of the page is
beyond the last byte of the file, i.e.
(page+1)*block_size+1 > data_file_length, so we correct the test
(bug found when testing log applying for ma_test1 -M -T --skip-update).
* update the page's LSN when removing a row from it during
execution of a REDO_PURGE_ROW record (bug found when testing log
applying for ma_test1 -M -T --skip-update).
* applying of REDO_PURGE_BLOCKs (limited to a one-page range for now).
storage/maria/ma_blockrec.h:
new functions. maria_bitmap_marker does not need to be exported.
storage/maria/ma_close.c:
we can always flush the table's state when closing the last instance
of the table. And it is needed for maria_read_log (as it does
not use maria_lock_database()).
storage/maria/ma_control_file.c:
when in Recovery, some assertions should not be used.
storage/maria/ma_control_file.h:
double-inclusion safe
storage/maria/ma_create.c:
during recovery, don't log records. Comments.
Moving the creation of the first bitmap page to ma_bitmap.c
storage/maria/ma_delete_table.c:
during recovery, don't log records. Log the end-zero of the dropped
table's name, so that recovery can use the string in place without
extending it to fit an end zero.
storage/maria/ma_loghandler.c:
* inwrite_rec_hook also needs access to the MARIA_SHARE, like
prewrite_rec_hook. This will be needed to update
share->records_diff (in the upcoming patch "recovery of the state").
* LOG_DESC::record_ends_group changed to an enum.
* LOG_DESC for LOGREC_REDO_PURGE_BLOCKS and LOGREC_UNDO_ROW_PURGE
corrected
* Sanja please see the @todo LOG BUG
* avoiding DBUG_RETURN(func()) as it gives confusing debug traces.
storage/maria/ma_loghandler.h:
- log write hooks called while the log's lock is held (inwrite_rec_hook)
now need the MARIA_SHARE, like prewrite_rec_hook already had
- instead of a bool saying if this record's type ends groups or not,
we refine: it may not end a group, it may end a group, or it may
be a group in itself. Imagine that we had a physical write failure
to a table before we log the UNDO, we still end up in
external_lock(F_UNLCK) and then we log a COMMIT: we don't want
to consider this COMMIT as ending the group of REDOs (don't want
to execute those REDOs during Recovery), that's why we say "COMMIT
is a group in itself, it aborts any previous group". This also
gives one more sanity check in maria_read_log.
storage/maria/ma_recovery.c:
New Recovery code, replacing the old pseudocode.
Most of maria_read_log moved here.
Call-able from ha_maria, but not enabled yet.
Compared to the previous version of maria_read_log, some bugs have
been fixed, debugging output can go to stdout or a disk file (for now
it's useful for me, later it can be changed), execution of
REDO_DROP_TABLE, REDO_DELETE_ALL, REDO_PURGE_BLOCKS has been added. Duplicate code
has been factored into functions. We abort an unfinished group
of records if we see a record which is a group in itself (like COMMIT).
No need for maria_panic() after a bug (which caused tables to not
be closed) was fixed; if there is yet another bug I prefer to see it.
When opening a table for Recovery, set data_file_length
and key_file_length to their real physical value (these are the
easiest state members to restore :). Warn us if the last page
was truncated (but Recovery handles it).
MARIA_SHARE::state::state::records is now partly recovered (not
idempotent, but works if recreating tables from scracth).
When applying a REDO to a page, stamp it with the UNDO's LSN
(current_group_end_lsn), not with the REDO's LSN; it makes
the table more identical to the original table (easier to compare
the two tables in the end).
Big thing missing: some types of REDOs are not handled,
and the UNDO phase does not exist (missing functions to execute UNDOs
to actually rollback). So for now tests are only inserting/deleting
a few 100 rows, closing the table and seeing if the log is applied ok;
it works. UPDATE not handled.
storage/maria/ma_recovery.h:
new functions: ma_recover() for recovery from inside ha_maria;
_ma_apply_log() for maria_read_log (ma_recover() calls _ma_apply_log()).
Btw, we need to not use the word "recover" for REPAIR/maria_chk anymore.
storage/maria/ma_rename.c:
don't write log records during recovery
storage/maria/ma_test2.c:
- fail if maria_info() or other subtests find some wrong information
- new option -g to skip updates.
- init the translog before creating the table, so that log applying
can work.
- in "#if 0" you'll see some fixed bugs (will be removed).
storage/maria/ma_test_all.sh:
cleanup files. Test log applying.
storage/maria/maria_read_log.c:
most of the logic moves to ma_recovery.c to be shared between
maria_read_log and recovery-from-inside-mysqld.
See ma_recovery.c for additional changes made to the moved code.
storage/maria/ma_test_recovery:
unit test for Recovery. Tests insert and delete,
REDO_UPDATE not yet coded.
Script is called from ma_test_all. Can run standalone.
2007-07-26 11:56:21 +02:00
|
|
|
goto err;
|
2006-04-11 15:45:10 +02:00
|
|
|
}
|
|
|
|
ant=1;
|
|
|
|
while (maria_rprev(file,read_record3,0) == 0 && ant < write_count+10)
|
|
|
|
ant++;
|
|
|
|
if (ant != write_count - opt_delete)
|
|
|
|
{
|
|
|
|
printf("prev: I found: %d records of %d\n",ant,write_count);
|
WL#3072 - Maria recovery
Unit test for recovery: runs ma_test1 and ma_test2 (both only with
INSERTs and DELETEs; UPDATEs disabled as not handled by recovery)
then moves the tables elswhere; recreates tables from the log, and
compares and fails if there is a difference. Passes now.
Most of maria_read_log.c moved to ma_recovery.c, as it will be re-used
for recovery-from-ha_maria.
Bugfixes of applying of REDO_INSERT, REDO_PURGE_ROW.
Applying of REDO_PURGE_BLOCKS, REDO_DELETE_ALL, REDO_DROP_TABLE,
UNDO_ROW_INSERT (in REDO phase only, i.e. just doing records++),
UNDO_ROW_DELETE, UNDO_ROW_PURGE.
Code cleanups.
Monty: please look for "QQ". Sanja: please look for "Sanja".
Future tasks: recovery of the bitmap (easy), recovery of the state
(make it idempotent), more REDOs (Monty to work on
REDO_UPDATE?), UNDO phase...
Pushing this cset as it looks safe, contains test and bugfixes which
will help Monty implement applying of REDO_UPDATE.
sql/handler.cc:
typo
storage/maria/Makefile.am:
Adding ma_test_recovery (which ma_test_all invokes, and which can
also be run alone). Most of maria_read_log.c moved to ma_recovery.c
storage/maria/ha_maria.cc:
comments
storage/maria/ma_bitmap.c:
fixing comments. 2 -> sizeof(maria_bitmap_marker).
Bitmap-related part of _ma_initialize_datafile() moves in bitmap module.
Now putting the "bm" signature when creating the first bitmap page
(it used to happen only at next open, but that
caused an annoying difference when testing Recovery if the original
run didn't open the table, and it looks more
logical like this: it goes to disk only with its signature correct);
see the "QQ" comment towards the _ma_initialize_data_file() call
in ma_create.c for more).
When reading a bitmap page, verify its signature (happens when normally
using the table or when CHECKing it; not when REPAIRing it).
storage/maria/ma_blockrec.c:
* no need to sync the data file if table is not transactional
* Comments, code cleanup (log-related data moved to log-related code
block, int5store->page_store).
* Store the table's short id into LOGREC_UNDO_ROW_PURGE, like we
do for other records (though this record will soon be replaced
with a CLR).
* If "page" is 1 it means the page which extends from byte
page*block_size+1 to (page+1)*block_size (byte number 1 being
the first byte of the file). The last byte of the file is
data_file_length (same convention).
A new page needs to be created if the last byte of the page is
beyond the last byte of the file, i.e.
(page+1)*block_size+1 > data_file_length, so we correct the test
(bug found when testing log applying for ma_test1 -M -T --skip-update).
* update the page's LSN when removing a row from it during
execution of a REDO_PURGE_ROW record (bug found when testing log
applying for ma_test1 -M -T --skip-update).
* applying of REDO_PURGE_BLOCKs (limited to a one-page range for now).
storage/maria/ma_blockrec.h:
new functions. maria_bitmap_marker does not need to be exported.
storage/maria/ma_close.c:
we can always flush the table's state when closing the last instance
of the table. And it is needed for maria_read_log (as it does
not use maria_lock_database()).
storage/maria/ma_control_file.c:
when in Recovery, some assertions should not be used.
storage/maria/ma_control_file.h:
double-inclusion safe
storage/maria/ma_create.c:
during recovery, don't log records. Comments.
Moving the creation of the first bitmap page to ma_bitmap.c
storage/maria/ma_delete_table.c:
during recovery, don't log records. Log the end-zero of the dropped
table's name, so that recovery can use the string in place without
extending it to fit an end zero.
storage/maria/ma_loghandler.c:
* inwrite_rec_hook also needs access to the MARIA_SHARE, like
prewrite_rec_hook. This will be needed to update
share->records_diff (in the upcoming patch "recovery of the state").
* LOG_DESC::record_ends_group changed to an enum.
* LOG_DESC for LOGREC_REDO_PURGE_BLOCKS and LOGREC_UNDO_ROW_PURGE
corrected
* Sanja please see the @todo LOG BUG
* avoiding DBUG_RETURN(func()) as it gives confusing debug traces.
storage/maria/ma_loghandler.h:
- log write hooks called while the log's lock is held (inwrite_rec_hook)
now need the MARIA_SHARE, like prewrite_rec_hook already had
- instead of a bool saying if this record's type ends groups or not,
we refine: it may not end a group, it may end a group, or it may
be a group in itself. Imagine that we had a physical write failure
to a table before we log the UNDO, we still end up in
external_lock(F_UNLCK) and then we log a COMMIT: we don't want
to consider this COMMIT as ending the group of REDOs (don't want
to execute those REDOs during Recovery), that's why we say "COMMIT
is a group in itself, it aborts any previous group". This also
gives one more sanity check in maria_read_log.
storage/maria/ma_recovery.c:
New Recovery code, replacing the old pseudocode.
Most of maria_read_log moved here.
Call-able from ha_maria, but not enabled yet.
Compared to the previous version of maria_read_log, some bugs have
been fixed, debugging output can go to stdout or a disk file (for now
it's useful for me, later it can be changed), execution of
REDO_DROP_TABLE, REDO_DELETE_ALL, REDO_PURGE_BLOCKS has been added. Duplicate code
has been factored into functions. We abort an unfinished group
of records if we see a record which is a group in itself (like COMMIT).
No need for maria_panic() after a bug (which caused tables to not
be closed) was fixed; if there is yet another bug I prefer to see it.
When opening a table for Recovery, set data_file_length
and key_file_length to their real physical value (these are the
easiest state members to restore :). Warn us if the last page
was truncated (but Recovery handles it).
MARIA_SHARE::state::state::records is now partly recovered (not
idempotent, but works if recreating tables from scracth).
When applying a REDO to a page, stamp it with the UNDO's LSN
(current_group_end_lsn), not with the REDO's LSN; it makes
the table more identical to the original table (easier to compare
the two tables in the end).
Big thing missing: some types of REDOs are not handled,
and the UNDO phase does not exist (missing functions to execute UNDOs
to actually rollback). So for now tests are only inserting/deleting
a few 100 rows, closing the table and seeing if the log is applied ok;
it works. UPDATE not handled.
storage/maria/ma_recovery.h:
new functions: ma_recover() for recovery from inside ha_maria;
_ma_apply_log() for maria_read_log (ma_recover() calls _ma_apply_log()).
Btw, we need to not use the word "recover" for REPAIR/maria_chk anymore.
storage/maria/ma_rename.c:
don't write log records during recovery
storage/maria/ma_test2.c:
- fail if maria_info() or other subtests find some wrong information
- new option -g to skip updates.
- init the translog before creating the table, so that log applying
can work.
- in "#if 0" you'll see some fixed bugs (will be removed).
storage/maria/ma_test_all.sh:
cleanup files. Test log applying.
storage/maria/maria_read_log.c:
most of the logic moves to ma_recovery.c to be shared between
maria_read_log and recovery-from-inside-mysqld.
See ma_recovery.c for additional changes made to the moved code.
storage/maria/ma_test_recovery:
unit test for Recovery. Tests insert and delete,
REDO_UPDATE not yet coded.
Script is called from ma_test_all. Can run standalone.
2007-07-26 11:56:21 +02:00
|
|
|
goto err;
|
2006-04-11 15:45:10 +02:00
|
|
|
}
|
|
|
|
if (bcmp(read_record,read_record3,reclength))
|
|
|
|
{
|
|
|
|
printf("Can't find first record\n");
|
WL#3072 - Maria recovery
Unit test for recovery: runs ma_test1 and ma_test2 (both only with
INSERTs and DELETEs; UPDATEs disabled as not handled by recovery)
then moves the tables elswhere; recreates tables from the log, and
compares and fails if there is a difference. Passes now.
Most of maria_read_log.c moved to ma_recovery.c, as it will be re-used
for recovery-from-ha_maria.
Bugfixes of applying of REDO_INSERT, REDO_PURGE_ROW.
Applying of REDO_PURGE_BLOCKS, REDO_DELETE_ALL, REDO_DROP_TABLE,
UNDO_ROW_INSERT (in REDO phase only, i.e. just doing records++),
UNDO_ROW_DELETE, UNDO_ROW_PURGE.
Code cleanups.
Monty: please look for "QQ". Sanja: please look for "Sanja".
Future tasks: recovery of the bitmap (easy), recovery of the state
(make it idempotent), more REDOs (Monty to work on
REDO_UPDATE?), UNDO phase...
Pushing this cset as it looks safe, contains test and bugfixes which
will help Monty implement applying of REDO_UPDATE.
sql/handler.cc:
typo
storage/maria/Makefile.am:
Adding ma_test_recovery (which ma_test_all invokes, and which can
also be run alone). Most of maria_read_log.c moved to ma_recovery.c
storage/maria/ha_maria.cc:
comments
storage/maria/ma_bitmap.c:
fixing comments. 2 -> sizeof(maria_bitmap_marker).
Bitmap-related part of _ma_initialize_datafile() moves in bitmap module.
Now putting the "bm" signature when creating the first bitmap page
(it used to happen only at next open, but that
caused an annoying difference when testing Recovery if the original
run didn't open the table, and it looks more
logical like this: it goes to disk only with its signature correct);
see the "QQ" comment towards the _ma_initialize_data_file() call
in ma_create.c for more).
When reading a bitmap page, verify its signature (happens when normally
using the table or when CHECKing it; not when REPAIRing it).
storage/maria/ma_blockrec.c:
* no need to sync the data file if table is not transactional
* Comments, code cleanup (log-related data moved to log-related code
block, int5store->page_store).
* Store the table's short id into LOGREC_UNDO_ROW_PURGE, like we
do for other records (though this record will soon be replaced
with a CLR).
* If "page" is 1 it means the page which extends from byte
page*block_size+1 to (page+1)*block_size (byte number 1 being
the first byte of the file). The last byte of the file is
data_file_length (same convention).
A new page needs to be created if the last byte of the page is
beyond the last byte of the file, i.e.
(page+1)*block_size+1 > data_file_length, so we correct the test
(bug found when testing log applying for ma_test1 -M -T --skip-update).
* update the page's LSN when removing a row from it during
execution of a REDO_PURGE_ROW record (bug found when testing log
applying for ma_test1 -M -T --skip-update).
* applying of REDO_PURGE_BLOCKs (limited to a one-page range for now).
storage/maria/ma_blockrec.h:
new functions. maria_bitmap_marker does not need to be exported.
storage/maria/ma_close.c:
we can always flush the table's state when closing the last instance
of the table. And it is needed for maria_read_log (as it does
not use maria_lock_database()).
storage/maria/ma_control_file.c:
when in Recovery, some assertions should not be used.
storage/maria/ma_control_file.h:
double-inclusion safe
storage/maria/ma_create.c:
during recovery, don't log records. Comments.
Moving the creation of the first bitmap page to ma_bitmap.c
storage/maria/ma_delete_table.c:
during recovery, don't log records. Log the end-zero of the dropped
table's name, so that recovery can use the string in place without
extending it to fit an end zero.
storage/maria/ma_loghandler.c:
* inwrite_rec_hook also needs access to the MARIA_SHARE, like
prewrite_rec_hook. This will be needed to update
share->records_diff (in the upcoming patch "recovery of the state").
* LOG_DESC::record_ends_group changed to an enum.
* LOG_DESC for LOGREC_REDO_PURGE_BLOCKS and LOGREC_UNDO_ROW_PURGE
corrected
* Sanja please see the @todo LOG BUG
* avoiding DBUG_RETURN(func()) as it gives confusing debug traces.
storage/maria/ma_loghandler.h:
- log write hooks called while the log's lock is held (inwrite_rec_hook)
now need the MARIA_SHARE, like prewrite_rec_hook already had
- instead of a bool saying if this record's type ends groups or not,
we refine: it may not end a group, it may end a group, or it may
be a group in itself. Imagine that we had a physical write failure
to a table before we log the UNDO, we still end up in
external_lock(F_UNLCK) and then we log a COMMIT: we don't want
to consider this COMMIT as ending the group of REDOs (don't want
to execute those REDOs during Recovery), that's why we say "COMMIT
is a group in itself, it aborts any previous group". This also
gives one more sanity check in maria_read_log.
storage/maria/ma_recovery.c:
New Recovery code, replacing the old pseudocode.
Most of maria_read_log moved here.
Call-able from ha_maria, but not enabled yet.
Compared to the previous version of maria_read_log, some bugs have
been fixed, debugging output can go to stdout or a disk file (for now
it's useful for me, later it can be changed), execution of
REDO_DROP_TABLE, REDO_DELETE_ALL, REDO_PURGE_BLOCKS has been added. Duplicate code
has been factored into functions. We abort an unfinished group
of records if we see a record which is a group in itself (like COMMIT).
No need for maria_panic() after a bug (which caused tables to not
be closed) was fixed; if there is yet another bug I prefer to see it.
When opening a table for Recovery, set data_file_length
and key_file_length to their real physical value (these are the
easiest state members to restore :). Warn us if the last page
was truncated (but Recovery handles it).
MARIA_SHARE::state::state::records is now partly recovered (not
idempotent, but works if recreating tables from scracth).
When applying a REDO to a page, stamp it with the UNDO's LSN
(current_group_end_lsn), not with the REDO's LSN; it makes
the table more identical to the original table (easier to compare
the two tables in the end).
Big thing missing: some types of REDOs are not handled,
and the UNDO phase does not exist (missing functions to execute UNDOs
to actually rollback). So for now tests are only inserting/deleting
a few 100 rows, closing the table and seeing if the log is applied ok;
it works. UPDATE not handled.
storage/maria/ma_recovery.h:
new functions: ma_recover() for recovery from inside ha_maria;
_ma_apply_log() for maria_read_log (ma_recover() calls _ma_apply_log()).
Btw, we need to not use the word "recover" for REPAIR/maria_chk anymore.
storage/maria/ma_rename.c:
don't write log records during recovery
storage/maria/ma_test2.c:
- fail if maria_info() or other subtests find some wrong information
- new option -g to skip updates.
- init the translog before creating the table, so that log applying
can work.
- in "#if 0" you'll see some fixed bugs (will be removed).
storage/maria/ma_test_all.sh:
cleanup files. Test log applying.
storage/maria/maria_read_log.c:
most of the logic moves to ma_recovery.c to be shared between
maria_read_log and recovery-from-inside-mysqld.
See ma_recovery.c for additional changes made to the moved code.
storage/maria/ma_test_recovery:
unit test for Recovery. Tests insert and delete,
REDO_UPDATE not yet coded.
Script is called from ma_test_all. Can run standalone.
2007-07-26 11:56:21 +02:00
|
|
|
goto err;
|
2006-04-11 15:45:10 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
if (!silent)
|
|
|
|
printf("- Test if: Read first - next - prev - prev - next == first\n");
|
|
|
|
DBUG_PRINT("progpos",("- Read first - next - prev - prev - next == first"));
|
|
|
|
if (maria_rfirst(file,read_record,0) ||
|
|
|
|
maria_rnext(file,read_record3,0) ||
|
|
|
|
maria_rprev(file,read_record3,0) ||
|
|
|
|
maria_rprev(file,read_record3,0) == 0 ||
|
|
|
|
maria_rnext(file,read_record3,0))
|
|
|
|
goto err;
|
|
|
|
if (bcmp(read_record,read_record3,reclength) != 0)
|
|
|
|
printf("Can't find first record\n");
|
|
|
|
|
|
|
|
if (!silent)
|
|
|
|
printf("- Test if: Read last - prev - next - next - prev == last\n");
|
|
|
|
DBUG_PRINT("progpos",("Read last - prev - next - next - prev == last"));
|
|
|
|
if (maria_rlast(file,read_record2,0) ||
|
|
|
|
maria_rprev(file,read_record3,0) ||
|
|
|
|
maria_rnext(file,read_record3,0) ||
|
|
|
|
maria_rnext(file,read_record3,0) == 0 ||
|
|
|
|
maria_rprev(file,read_record3,0))
|
|
|
|
goto err;
|
|
|
|
if (bcmp(read_record2,read_record3,reclength))
|
|
|
|
printf("Can't find last record\n");
|
2007-07-27 16:11:40 +02:00
|
|
|
#ifdef NOT_ANYMORE
|
2006-04-11 15:45:10 +02:00
|
|
|
if (!silent)
|
|
|
|
puts("- Test read key-part");
|
|
|
|
strmov(key2,key);
|
|
|
|
for(i=strlen(key2) ; i-- > 1 ;)
|
|
|
|
{
|
|
|
|
key2[i]=0;
|
|
|
|
|
|
|
|
/* The following row is just to catch some bugs in the key code */
|
|
|
|
bzero((char*) file->lastkey,file->s->base.max_key_length*2);
|
|
|
|
if (maria_rkey(file,read_record,0,key2,(uint) i,HA_READ_PREFIX))
|
|
|
|
goto err;
|
|
|
|
if (bcmp(read_record+start,key,(uint) i))
|
|
|
|
{
|
|
|
|
puts("Didn't find right record");
|
WL#3072 - Maria recovery
Unit test for recovery: runs ma_test1 and ma_test2 (both only with
INSERTs and DELETEs; UPDATEs disabled as not handled by recovery)
then moves the tables elswhere; recreates tables from the log, and
compares and fails if there is a difference. Passes now.
Most of maria_read_log.c moved to ma_recovery.c, as it will be re-used
for recovery-from-ha_maria.
Bugfixes of applying of REDO_INSERT, REDO_PURGE_ROW.
Applying of REDO_PURGE_BLOCKS, REDO_DELETE_ALL, REDO_DROP_TABLE,
UNDO_ROW_INSERT (in REDO phase only, i.e. just doing records++),
UNDO_ROW_DELETE, UNDO_ROW_PURGE.
Code cleanups.
Monty: please look for "QQ". Sanja: please look for "Sanja".
Future tasks: recovery of the bitmap (easy), recovery of the state
(make it idempotent), more REDOs (Monty to work on
REDO_UPDATE?), UNDO phase...
Pushing this cset as it looks safe, contains test and bugfixes which
will help Monty implement applying of REDO_UPDATE.
sql/handler.cc:
typo
storage/maria/Makefile.am:
Adding ma_test_recovery (which ma_test_all invokes, and which can
also be run alone). Most of maria_read_log.c moved to ma_recovery.c
storage/maria/ha_maria.cc:
comments
storage/maria/ma_bitmap.c:
fixing comments. 2 -> sizeof(maria_bitmap_marker).
Bitmap-related part of _ma_initialize_datafile() moves in bitmap module.
Now putting the "bm" signature when creating the first bitmap page
(it used to happen only at next open, but that
caused an annoying difference when testing Recovery if the original
run didn't open the table, and it looks more
logical like this: it goes to disk only with its signature correct);
see the "QQ" comment towards the _ma_initialize_data_file() call
in ma_create.c for more).
When reading a bitmap page, verify its signature (happens when normally
using the table or when CHECKing it; not when REPAIRing it).
storage/maria/ma_blockrec.c:
* no need to sync the data file if table is not transactional
* Comments, code cleanup (log-related data moved to log-related code
block, int5store->page_store).
* Store the table's short id into LOGREC_UNDO_ROW_PURGE, like we
do for other records (though this record will soon be replaced
with a CLR).
* If "page" is 1 it means the page which extends from byte
page*block_size+1 to (page+1)*block_size (byte number 1 being
the first byte of the file). The last byte of the file is
data_file_length (same convention).
A new page needs to be created if the last byte of the page is
beyond the last byte of the file, i.e.
(page+1)*block_size+1 > data_file_length, so we correct the test
(bug found when testing log applying for ma_test1 -M -T --skip-update).
* update the page's LSN when removing a row from it during
execution of a REDO_PURGE_ROW record (bug found when testing log
applying for ma_test1 -M -T --skip-update).
* applying of REDO_PURGE_BLOCKs (limited to a one-page range for now).
storage/maria/ma_blockrec.h:
new functions. maria_bitmap_marker does not need to be exported.
storage/maria/ma_close.c:
we can always flush the table's state when closing the last instance
of the table. And it is needed for maria_read_log (as it does
not use maria_lock_database()).
storage/maria/ma_control_file.c:
when in Recovery, some assertions should not be used.
storage/maria/ma_control_file.h:
double-inclusion safe
storage/maria/ma_create.c:
during recovery, don't log records. Comments.
Moving the creation of the first bitmap page to ma_bitmap.c
storage/maria/ma_delete_table.c:
during recovery, don't log records. Log the end-zero of the dropped
table's name, so that recovery can use the string in place without
extending it to fit an end zero.
storage/maria/ma_loghandler.c:
* inwrite_rec_hook also needs access to the MARIA_SHARE, like
prewrite_rec_hook. This will be needed to update
share->records_diff (in the upcoming patch "recovery of the state").
* LOG_DESC::record_ends_group changed to an enum.
* LOG_DESC for LOGREC_REDO_PURGE_BLOCKS and LOGREC_UNDO_ROW_PURGE
corrected
* Sanja please see the @todo LOG BUG
* avoiding DBUG_RETURN(func()) as it gives confusing debug traces.
storage/maria/ma_loghandler.h:
- log write hooks called while the log's lock is held (inwrite_rec_hook)
now need the MARIA_SHARE, like prewrite_rec_hook already had
- instead of a bool saying if this record's type ends groups or not,
we refine: it may not end a group, it may end a group, or it may
be a group in itself. Imagine that we had a physical write failure
to a table before we log the UNDO, we still end up in
external_lock(F_UNLCK) and then we log a COMMIT: we don't want
to consider this COMMIT as ending the group of REDOs (don't want
to execute those REDOs during Recovery), that's why we say "COMMIT
is a group in itself, it aborts any previous group". This also
gives one more sanity check in maria_read_log.
storage/maria/ma_recovery.c:
New Recovery code, replacing the old pseudocode.
Most of maria_read_log moved here.
Call-able from ha_maria, but not enabled yet.
Compared to the previous version of maria_read_log, some bugs have
been fixed, debugging output can go to stdout or a disk file (for now
it's useful for me, later it can be changed), execution of
REDO_DROP_TABLE, REDO_DELETE_ALL, REDO_PURGE_BLOCKS has been added. Duplicate code
has been factored into functions. We abort an unfinished group
of records if we see a record which is a group in itself (like COMMIT).
No need for maria_panic() after a bug (which caused tables to not
be closed) was fixed; if there is yet another bug I prefer to see it.
When opening a table for Recovery, set data_file_length
and key_file_length to their real physical value (these are the
easiest state members to restore :). Warn us if the last page
was truncated (but Recovery handles it).
MARIA_SHARE::state::state::records is now partly recovered (not
idempotent, but works if recreating tables from scracth).
When applying a REDO to a page, stamp it with the UNDO's LSN
(current_group_end_lsn), not with the REDO's LSN; it makes
the table more identical to the original table (easier to compare
the two tables in the end).
Big thing missing: some types of REDOs are not handled,
and the UNDO phase does not exist (missing functions to execute UNDOs
to actually rollback). So for now tests are only inserting/deleting
a few 100 rows, closing the table and seeing if the log is applied ok;
it works. UPDATE not handled.
storage/maria/ma_recovery.h:
new functions: ma_recover() for recovery from inside ha_maria;
_ma_apply_log() for maria_read_log (ma_recover() calls _ma_apply_log()).
Btw, we need to not use the word "recover" for REPAIR/maria_chk anymore.
storage/maria/ma_rename.c:
don't write log records during recovery
storage/maria/ma_test2.c:
- fail if maria_info() or other subtests find some wrong information
- new option -g to skip updates.
- init the translog before creating the table, so that log applying
can work.
- in "#if 0" you'll see some fixed bugs (will be removed).
storage/maria/ma_test_all.sh:
cleanup files. Test log applying.
storage/maria/maria_read_log.c:
most of the logic moves to ma_recovery.c to be shared between
maria_read_log and recovery-from-inside-mysqld.
See ma_recovery.c for additional changes made to the moved code.
storage/maria/ma_test_recovery:
unit test for Recovery. Tests insert and delete,
REDO_UPDATE not yet coded.
Script is called from ma_test_all. Can run standalone.
2007-07-26 11:56:21 +02:00
|
|
|
goto err;
|
2006-04-11 15:45:10 +02:00
|
|
|
}
|
|
|
|
}
|
2007-07-27 16:11:40 +02:00
|
|
|
#endif
|
2006-04-11 15:45:10 +02:00
|
|
|
if (dupp_keys > 2)
|
|
|
|
{
|
|
|
|
if (!silent)
|
|
|
|
printf("- Read key (first) - next - delete - next -> last\n");
|
|
|
|
DBUG_PRINT("progpos",("first - next - delete - next -> last"));
|
2007-07-27 16:11:40 +02:00
|
|
|
if (maria_rkey(file,read_record,0,key,HA_WHOLE_KEY,HA_READ_KEY_EXACT))
|
|
|
|
goto err;
|
2006-04-11 15:45:10 +02:00
|
|
|
if (maria_rnext(file,read_record3,0)) goto err;
|
|
|
|
if (maria_delete(file,read_record3)) goto err;
|
|
|
|
opt_delete++;
|
|
|
|
ant=1;
|
|
|
|
while (maria_rnext(file,read_record3,0) == 0 &&
|
|
|
|
bcmp(read_record3+start,key,length) == 0) ant++;
|
|
|
|
if (ant != dupp_keys-1)
|
|
|
|
{
|
|
|
|
printf("next: I can only find: %d keys of %d\n",ant,dupp_keys-1);
|
WL#3072 - Maria recovery
Unit test for recovery: runs ma_test1 and ma_test2 (both only with
INSERTs and DELETEs; UPDATEs disabled as not handled by recovery)
then moves the tables elswhere; recreates tables from the log, and
compares and fails if there is a difference. Passes now.
Most of maria_read_log.c moved to ma_recovery.c, as it will be re-used
for recovery-from-ha_maria.
Bugfixes of applying of REDO_INSERT, REDO_PURGE_ROW.
Applying of REDO_PURGE_BLOCKS, REDO_DELETE_ALL, REDO_DROP_TABLE,
UNDO_ROW_INSERT (in REDO phase only, i.e. just doing records++),
UNDO_ROW_DELETE, UNDO_ROW_PURGE.
Code cleanups.
Monty: please look for "QQ". Sanja: please look for "Sanja".
Future tasks: recovery of the bitmap (easy), recovery of the state
(make it idempotent), more REDOs (Monty to work on
REDO_UPDATE?), UNDO phase...
Pushing this cset as it looks safe, contains test and bugfixes which
will help Monty implement applying of REDO_UPDATE.
sql/handler.cc:
typo
storage/maria/Makefile.am:
Adding ma_test_recovery (which ma_test_all invokes, and which can
also be run alone). Most of maria_read_log.c moved to ma_recovery.c
storage/maria/ha_maria.cc:
comments
storage/maria/ma_bitmap.c:
fixing comments. 2 -> sizeof(maria_bitmap_marker).
Bitmap-related part of _ma_initialize_datafile() moves in bitmap module.
Now putting the "bm" signature when creating the first bitmap page
(it used to happen only at next open, but that
caused an annoying difference when testing Recovery if the original
run didn't open the table, and it looks more
logical like this: it goes to disk only with its signature correct);
see the "QQ" comment towards the _ma_initialize_data_file() call
in ma_create.c for more).
When reading a bitmap page, verify its signature (happens when normally
using the table or when CHECKing it; not when REPAIRing it).
storage/maria/ma_blockrec.c:
* no need to sync the data file if table is not transactional
* Comments, code cleanup (log-related data moved to log-related code
block, int5store->page_store).
* Store the table's short id into LOGREC_UNDO_ROW_PURGE, like we
do for other records (though this record will soon be replaced
with a CLR).
* If "page" is 1 it means the page which extends from byte
page*block_size+1 to (page+1)*block_size (byte number 1 being
the first byte of the file). The last byte of the file is
data_file_length (same convention).
A new page needs to be created if the last byte of the page is
beyond the last byte of the file, i.e.
(page+1)*block_size+1 > data_file_length, so we correct the test
(bug found when testing log applying for ma_test1 -M -T --skip-update).
* update the page's LSN when removing a row from it during
execution of a REDO_PURGE_ROW record (bug found when testing log
applying for ma_test1 -M -T --skip-update).
* applying of REDO_PURGE_BLOCKs (limited to a one-page range for now).
storage/maria/ma_blockrec.h:
new functions. maria_bitmap_marker does not need to be exported.
storage/maria/ma_close.c:
we can always flush the table's state when closing the last instance
of the table. And it is needed for maria_read_log (as it does
not use maria_lock_database()).
storage/maria/ma_control_file.c:
when in Recovery, some assertions should not be used.
storage/maria/ma_control_file.h:
double-inclusion safe
storage/maria/ma_create.c:
during recovery, don't log records. Comments.
Moving the creation of the first bitmap page to ma_bitmap.c
storage/maria/ma_delete_table.c:
during recovery, don't log records. Log the end-zero of the dropped
table's name, so that recovery can use the string in place without
extending it to fit an end zero.
storage/maria/ma_loghandler.c:
* inwrite_rec_hook also needs access to the MARIA_SHARE, like
prewrite_rec_hook. This will be needed to update
share->records_diff (in the upcoming patch "recovery of the state").
* LOG_DESC::record_ends_group changed to an enum.
* LOG_DESC for LOGREC_REDO_PURGE_BLOCKS and LOGREC_UNDO_ROW_PURGE
corrected
* Sanja please see the @todo LOG BUG
* avoiding DBUG_RETURN(func()) as it gives confusing debug traces.
storage/maria/ma_loghandler.h:
- log write hooks called while the log's lock is held (inwrite_rec_hook)
now need the MARIA_SHARE, like prewrite_rec_hook already had
- instead of a bool saying if this record's type ends groups or not,
we refine: it may not end a group, it may end a group, or it may
be a group in itself. Imagine that we had a physical write failure
to a table before we log the UNDO, we still end up in
external_lock(F_UNLCK) and then we log a COMMIT: we don't want
to consider this COMMIT as ending the group of REDOs (don't want
to execute those REDOs during Recovery), that's why we say "COMMIT
is a group in itself, it aborts any previous group". This also
gives one more sanity check in maria_read_log.
storage/maria/ma_recovery.c:
New Recovery code, replacing the old pseudocode.
Most of maria_read_log moved here.
Call-able from ha_maria, but not enabled yet.
Compared to the previous version of maria_read_log, some bugs have
been fixed, debugging output can go to stdout or a disk file (for now
it's useful for me, later it can be changed), execution of
REDO_DROP_TABLE, REDO_DELETE_ALL, REDO_PURGE_BLOCKS has been added. Duplicate code
has been factored into functions. We abort an unfinished group
of records if we see a record which is a group in itself (like COMMIT).
No need for maria_panic() after a bug (which caused tables to not
be closed) was fixed; if there is yet another bug I prefer to see it.
When opening a table for Recovery, set data_file_length
and key_file_length to their real physical value (these are the
easiest state members to restore :). Warn us if the last page
was truncated (but Recovery handles it).
MARIA_SHARE::state::state::records is now partly recovered (not
idempotent, but works if recreating tables from scracth).
When applying a REDO to a page, stamp it with the UNDO's LSN
(current_group_end_lsn), not with the REDO's LSN; it makes
the table more identical to the original table (easier to compare
the two tables in the end).
Big thing missing: some types of REDOs are not handled,
and the UNDO phase does not exist (missing functions to execute UNDOs
to actually rollback). So for now tests are only inserting/deleting
a few 100 rows, closing the table and seeing if the log is applied ok;
it works. UPDATE not handled.
storage/maria/ma_recovery.h:
new functions: ma_recover() for recovery from inside ha_maria;
_ma_apply_log() for maria_read_log (ma_recover() calls _ma_apply_log()).
Btw, we need to not use the word "recover" for REPAIR/maria_chk anymore.
storage/maria/ma_rename.c:
don't write log records during recovery
storage/maria/ma_test2.c:
- fail if maria_info() or other subtests find some wrong information
- new option -g to skip updates.
- init the translog before creating the table, so that log applying
can work.
- in "#if 0" you'll see some fixed bugs (will be removed).
storage/maria/ma_test_all.sh:
cleanup files. Test log applying.
storage/maria/maria_read_log.c:
most of the logic moves to ma_recovery.c to be shared between
maria_read_log and recovery-from-inside-mysqld.
See ma_recovery.c for additional changes made to the moved code.
storage/maria/ma_test_recovery:
unit test for Recovery. Tests insert and delete,
REDO_UPDATE not yet coded.
Script is called from ma_test_all. Can run standalone.
2007-07-26 11:56:21 +02:00
|
|
|
goto err;
|
2006-04-11 15:45:10 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
if (dupp_keys>4)
|
|
|
|
{
|
|
|
|
if (!silent)
|
|
|
|
printf("- Read last of key - prev - delete - prev -> first\n");
|
|
|
|
DBUG_PRINT("progpos",("last - prev - delete - prev -> first"));
|
|
|
|
if (maria_rprev(file,read_record3,0)) goto err;
|
|
|
|
if (maria_rprev(file,read_record3,0)) goto err;
|
|
|
|
if (maria_delete(file,read_record3)) goto err;
|
|
|
|
opt_delete++;
|
|
|
|
ant=1;
|
|
|
|
while (maria_rprev(file,read_record3,0) == 0 &&
|
|
|
|
bcmp(read_record3+start,key,length) == 0) ant++;
|
|
|
|
if (ant != dupp_keys-2)
|
|
|
|
{
|
|
|
|
printf("next: I can only find: %d keys of %d\n",ant,dupp_keys-2);
|
WL#3072 - Maria recovery
Unit test for recovery: runs ma_test1 and ma_test2 (both only with
INSERTs and DELETEs; UPDATEs disabled as not handled by recovery)
then moves the tables elswhere; recreates tables from the log, and
compares and fails if there is a difference. Passes now.
Most of maria_read_log.c moved to ma_recovery.c, as it will be re-used
for recovery-from-ha_maria.
Bugfixes of applying of REDO_INSERT, REDO_PURGE_ROW.
Applying of REDO_PURGE_BLOCKS, REDO_DELETE_ALL, REDO_DROP_TABLE,
UNDO_ROW_INSERT (in REDO phase only, i.e. just doing records++),
UNDO_ROW_DELETE, UNDO_ROW_PURGE.
Code cleanups.
Monty: please look for "QQ". Sanja: please look for "Sanja".
Future tasks: recovery of the bitmap (easy), recovery of the state
(make it idempotent), more REDOs (Monty to work on
REDO_UPDATE?), UNDO phase...
Pushing this cset as it looks safe, contains test and bugfixes which
will help Monty implement applying of REDO_UPDATE.
sql/handler.cc:
typo
storage/maria/Makefile.am:
Adding ma_test_recovery (which ma_test_all invokes, and which can
also be run alone). Most of maria_read_log.c moved to ma_recovery.c
storage/maria/ha_maria.cc:
comments
storage/maria/ma_bitmap.c:
fixing comments. 2 -> sizeof(maria_bitmap_marker).
Bitmap-related part of _ma_initialize_datafile() moves in bitmap module.
Now putting the "bm" signature when creating the first bitmap page
(it used to happen only at next open, but that
caused an annoying difference when testing Recovery if the original
run didn't open the table, and it looks more
logical like this: it goes to disk only with its signature correct);
see the "QQ" comment towards the _ma_initialize_data_file() call
in ma_create.c for more).
When reading a bitmap page, verify its signature (happens when normally
using the table or when CHECKing it; not when REPAIRing it).
storage/maria/ma_blockrec.c:
* no need to sync the data file if table is not transactional
* Comments, code cleanup (log-related data moved to log-related code
block, int5store->page_store).
* Store the table's short id into LOGREC_UNDO_ROW_PURGE, like we
do for other records (though this record will soon be replaced
with a CLR).
* If "page" is 1 it means the page which extends from byte
page*block_size+1 to (page+1)*block_size (byte number 1 being
the first byte of the file). The last byte of the file is
data_file_length (same convention).
A new page needs to be created if the last byte of the page is
beyond the last byte of the file, i.e.
(page+1)*block_size+1 > data_file_length, so we correct the test
(bug found when testing log applying for ma_test1 -M -T --skip-update).
* update the page's LSN when removing a row from it during
execution of a REDO_PURGE_ROW record (bug found when testing log
applying for ma_test1 -M -T --skip-update).
* applying of REDO_PURGE_BLOCKs (limited to a one-page range for now).
storage/maria/ma_blockrec.h:
new functions. maria_bitmap_marker does not need to be exported.
storage/maria/ma_close.c:
we can always flush the table's state when closing the last instance
of the table. And it is needed for maria_read_log (as it does
not use maria_lock_database()).
storage/maria/ma_control_file.c:
when in Recovery, some assertions should not be used.
storage/maria/ma_control_file.h:
double-inclusion safe
storage/maria/ma_create.c:
during recovery, don't log records. Comments.
Moving the creation of the first bitmap page to ma_bitmap.c
storage/maria/ma_delete_table.c:
during recovery, don't log records. Log the end-zero of the dropped
table's name, so that recovery can use the string in place without
extending it to fit an end zero.
storage/maria/ma_loghandler.c:
* inwrite_rec_hook also needs access to the MARIA_SHARE, like
prewrite_rec_hook. This will be needed to update
share->records_diff (in the upcoming patch "recovery of the state").
* LOG_DESC::record_ends_group changed to an enum.
* LOG_DESC for LOGREC_REDO_PURGE_BLOCKS and LOGREC_UNDO_ROW_PURGE
corrected
* Sanja please see the @todo LOG BUG
* avoiding DBUG_RETURN(func()) as it gives confusing debug traces.
storage/maria/ma_loghandler.h:
- log write hooks called while the log's lock is held (inwrite_rec_hook)
now need the MARIA_SHARE, like prewrite_rec_hook already had
- instead of a bool saying if this record's type ends groups or not,
we refine: it may not end a group, it may end a group, or it may
be a group in itself. Imagine that we had a physical write failure
to a table before we log the UNDO, we still end up in
external_lock(F_UNLCK) and then we log a COMMIT: we don't want
to consider this COMMIT as ending the group of REDOs (don't want
to execute those REDOs during Recovery), that's why we say "COMMIT
is a group in itself, it aborts any previous group". This also
gives one more sanity check in maria_read_log.
storage/maria/ma_recovery.c:
New Recovery code, replacing the old pseudocode.
Most of maria_read_log moved here.
Call-able from ha_maria, but not enabled yet.
Compared to the previous version of maria_read_log, some bugs have
been fixed, debugging output can go to stdout or a disk file (for now
it's useful for me, later it can be changed), execution of
REDO_DROP_TABLE, REDO_DELETE_ALL, REDO_PURGE_BLOCKS has been added. Duplicate code
has been factored into functions. We abort an unfinished group
of records if we see a record which is a group in itself (like COMMIT).
No need for maria_panic() after a bug (which caused tables to not
be closed) was fixed; if there is yet another bug I prefer to see it.
When opening a table for Recovery, set data_file_length
and key_file_length to their real physical value (these are the
easiest state members to restore :). Warn us if the last page
was truncated (but Recovery handles it).
MARIA_SHARE::state::state::records is now partly recovered (not
idempotent, but works if recreating tables from scracth).
When applying a REDO to a page, stamp it with the UNDO's LSN
(current_group_end_lsn), not with the REDO's LSN; it makes
the table more identical to the original table (easier to compare
the two tables in the end).
Big thing missing: some types of REDOs are not handled,
and the UNDO phase does not exist (missing functions to execute UNDOs
to actually rollback). So for now tests are only inserting/deleting
a few 100 rows, closing the table and seeing if the log is applied ok;
it works. UPDATE not handled.
storage/maria/ma_recovery.h:
new functions: ma_recover() for recovery from inside ha_maria;
_ma_apply_log() for maria_read_log (ma_recover() calls _ma_apply_log()).
Btw, we need to not use the word "recover" for REPAIR/maria_chk anymore.
storage/maria/ma_rename.c:
don't write log records during recovery
storage/maria/ma_test2.c:
- fail if maria_info() or other subtests find some wrong information
- new option -g to skip updates.
- init the translog before creating the table, so that log applying
can work.
- in "#if 0" you'll see some fixed bugs (will be removed).
storage/maria/ma_test_all.sh:
cleanup files. Test log applying.
storage/maria/maria_read_log.c:
most of the logic moves to ma_recovery.c to be shared between
maria_read_log and recovery-from-inside-mysqld.
See ma_recovery.c for additional changes made to the moved code.
storage/maria/ma_test_recovery:
unit test for Recovery. Tests insert and delete,
REDO_UPDATE not yet coded.
Script is called from ma_test_all. Can run standalone.
2007-07-26 11:56:21 +02:00
|
|
|
goto err;
|
2006-04-11 15:45:10 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
if (dupp_keys > 6)
|
|
|
|
{
|
|
|
|
if (!silent)
|
|
|
|
printf("- Read first - delete - next -> last\n");
|
|
|
|
DBUG_PRINT("progpos",("first - delete - next -> last"));
|
2007-07-27 16:11:40 +02:00
|
|
|
if (maria_rkey(file,read_record3,0,key,HA_WHOLE_KEY,HA_READ_KEY_EXACT))
|
|
|
|
goto err;
|
2006-04-11 15:45:10 +02:00
|
|
|
if (maria_delete(file,read_record3)) goto err;
|
|
|
|
opt_delete++;
|
|
|
|
ant=1;
|
|
|
|
if (maria_rnext(file,read_record,0))
|
|
|
|
goto err; /* Skall finnas poster */
|
|
|
|
while (maria_rnext(file,read_record3,0) == 0 &&
|
|
|
|
bcmp(read_record3+start,key,length) == 0) ant++;
|
|
|
|
if (ant != dupp_keys-3)
|
|
|
|
{
|
|
|
|
printf("next: I can only find: %d keys of %d\n",ant,dupp_keys-3);
|
WL#3072 - Maria recovery
Unit test for recovery: runs ma_test1 and ma_test2 (both only with
INSERTs and DELETEs; UPDATEs disabled as not handled by recovery)
then moves the tables elswhere; recreates tables from the log, and
compares and fails if there is a difference. Passes now.
Most of maria_read_log.c moved to ma_recovery.c, as it will be re-used
for recovery-from-ha_maria.
Bugfixes of applying of REDO_INSERT, REDO_PURGE_ROW.
Applying of REDO_PURGE_BLOCKS, REDO_DELETE_ALL, REDO_DROP_TABLE,
UNDO_ROW_INSERT (in REDO phase only, i.e. just doing records++),
UNDO_ROW_DELETE, UNDO_ROW_PURGE.
Code cleanups.
Monty: please look for "QQ". Sanja: please look for "Sanja".
Future tasks: recovery of the bitmap (easy), recovery of the state
(make it idempotent), more REDOs (Monty to work on
REDO_UPDATE?), UNDO phase...
Pushing this cset as it looks safe, contains test and bugfixes which
will help Monty implement applying of REDO_UPDATE.
sql/handler.cc:
typo
storage/maria/Makefile.am:
Adding ma_test_recovery (which ma_test_all invokes, and which can
also be run alone). Most of maria_read_log.c moved to ma_recovery.c
storage/maria/ha_maria.cc:
comments
storage/maria/ma_bitmap.c:
fixing comments. 2 -> sizeof(maria_bitmap_marker).
Bitmap-related part of _ma_initialize_datafile() moves in bitmap module.
Now putting the "bm" signature when creating the first bitmap page
(it used to happen only at next open, but that
caused an annoying difference when testing Recovery if the original
run didn't open the table, and it looks more
logical like this: it goes to disk only with its signature correct);
see the "QQ" comment towards the _ma_initialize_data_file() call
in ma_create.c for more).
When reading a bitmap page, verify its signature (happens when normally
using the table or when CHECKing it; not when REPAIRing it).
storage/maria/ma_blockrec.c:
* no need to sync the data file if table is not transactional
* Comments, code cleanup (log-related data moved to log-related code
block, int5store->page_store).
* Store the table's short id into LOGREC_UNDO_ROW_PURGE, like we
do for other records (though this record will soon be replaced
with a CLR).
* If "page" is 1 it means the page which extends from byte
page*block_size+1 to (page+1)*block_size (byte number 1 being
the first byte of the file). The last byte of the file is
data_file_length (same convention).
A new page needs to be created if the last byte of the page is
beyond the last byte of the file, i.e.
(page+1)*block_size+1 > data_file_length, so we correct the test
(bug found when testing log applying for ma_test1 -M -T --skip-update).
* update the page's LSN when removing a row from it during
execution of a REDO_PURGE_ROW record (bug found when testing log
applying for ma_test1 -M -T --skip-update).
* applying of REDO_PURGE_BLOCKs (limited to a one-page range for now).
storage/maria/ma_blockrec.h:
new functions. maria_bitmap_marker does not need to be exported.
storage/maria/ma_close.c:
we can always flush the table's state when closing the last instance
of the table. And it is needed for maria_read_log (as it does
not use maria_lock_database()).
storage/maria/ma_control_file.c:
when in Recovery, some assertions should not be used.
storage/maria/ma_control_file.h:
double-inclusion safe
storage/maria/ma_create.c:
during recovery, don't log records. Comments.
Moving the creation of the first bitmap page to ma_bitmap.c
storage/maria/ma_delete_table.c:
during recovery, don't log records. Log the end-zero of the dropped
table's name, so that recovery can use the string in place without
extending it to fit an end zero.
storage/maria/ma_loghandler.c:
* inwrite_rec_hook also needs access to the MARIA_SHARE, like
prewrite_rec_hook. This will be needed to update
share->records_diff (in the upcoming patch "recovery of the state").
* LOG_DESC::record_ends_group changed to an enum.
* LOG_DESC for LOGREC_REDO_PURGE_BLOCKS and LOGREC_UNDO_ROW_PURGE
corrected
* Sanja please see the @todo LOG BUG
* avoiding DBUG_RETURN(func()) as it gives confusing debug traces.
storage/maria/ma_loghandler.h:
- log write hooks called while the log's lock is held (inwrite_rec_hook)
now need the MARIA_SHARE, like prewrite_rec_hook already had
- instead of a bool saying if this record's type ends groups or not,
we refine: it may not end a group, it may end a group, or it may
be a group in itself. Imagine that we had a physical write failure
to a table before we log the UNDO, we still end up in
external_lock(F_UNLCK) and then we log a COMMIT: we don't want
to consider this COMMIT as ending the group of REDOs (don't want
to execute those REDOs during Recovery), that's why we say "COMMIT
is a group in itself, it aborts any previous group". This also
gives one more sanity check in maria_read_log.
storage/maria/ma_recovery.c:
New Recovery code, replacing the old pseudocode.
Most of maria_read_log moved here.
Call-able from ha_maria, but not enabled yet.
Compared to the previous version of maria_read_log, some bugs have
been fixed, debugging output can go to stdout or a disk file (for now
it's useful for me, later it can be changed), execution of
REDO_DROP_TABLE, REDO_DELETE_ALL, REDO_PURGE_BLOCKS has been added. Duplicate code
has been factored into functions. We abort an unfinished group
of records if we see a record which is a group in itself (like COMMIT).
No need for maria_panic() after a bug (which caused tables to not
be closed) was fixed; if there is yet another bug I prefer to see it.
When opening a table for Recovery, set data_file_length
and key_file_length to their real physical value (these are the
easiest state members to restore :). Warn us if the last page
was truncated (but Recovery handles it).
MARIA_SHARE::state::state::records is now partly recovered (not
idempotent, but works if recreating tables from scracth).
When applying a REDO to a page, stamp it with the UNDO's LSN
(current_group_end_lsn), not with the REDO's LSN; it makes
the table more identical to the original table (easier to compare
the two tables in the end).
Big thing missing: some types of REDOs are not handled,
and the UNDO phase does not exist (missing functions to execute UNDOs
to actually rollback). So for now tests are only inserting/deleting
a few 100 rows, closing the table and seeing if the log is applied ok;
it works. UPDATE not handled.
storage/maria/ma_recovery.h:
new functions: ma_recover() for recovery from inside ha_maria;
_ma_apply_log() for maria_read_log (ma_recover() calls _ma_apply_log()).
Btw, we need to not use the word "recover" for REPAIR/maria_chk anymore.
storage/maria/ma_rename.c:
don't write log records during recovery
storage/maria/ma_test2.c:
- fail if maria_info() or other subtests find some wrong information
- new option -g to skip updates.
- init the translog before creating the table, so that log applying
can work.
- in "#if 0" you'll see some fixed bugs (will be removed).
storage/maria/ma_test_all.sh:
cleanup files. Test log applying.
storage/maria/maria_read_log.c:
most of the logic moves to ma_recovery.c to be shared between
maria_read_log and recovery-from-inside-mysqld.
See ma_recovery.c for additional changes made to the moved code.
storage/maria/ma_test_recovery:
unit test for Recovery. Tests insert and delete,
REDO_UPDATE not yet coded.
Script is called from ma_test_all. Can run standalone.
2007-07-26 11:56:21 +02:00
|
|
|
goto err;
|
2006-04-11 15:45:10 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
if (!silent)
|
|
|
|
printf("- Read last - delete - prev -> first\n");
|
|
|
|
DBUG_PRINT("progpos",("last - delete - prev -> first"));
|
|
|
|
if (maria_rprev(file,read_record3,0)) goto err;
|
|
|
|
if (maria_delete(file,read_record3)) goto err;
|
|
|
|
opt_delete++;
|
|
|
|
ant=0;
|
|
|
|
while (maria_rprev(file,read_record3,0) == 0 &&
|
|
|
|
bcmp(read_record3+start,key,length) == 0) ant++;
|
|
|
|
if (ant != dupp_keys-4)
|
|
|
|
{
|
|
|
|
printf("next: I can only find: %d keys of %d\n",ant,dupp_keys-4);
|
WL#3072 - Maria recovery
Unit test for recovery: runs ma_test1 and ma_test2 (both only with
INSERTs and DELETEs; UPDATEs disabled as not handled by recovery)
then moves the tables elswhere; recreates tables from the log, and
compares and fails if there is a difference. Passes now.
Most of maria_read_log.c moved to ma_recovery.c, as it will be re-used
for recovery-from-ha_maria.
Bugfixes of applying of REDO_INSERT, REDO_PURGE_ROW.
Applying of REDO_PURGE_BLOCKS, REDO_DELETE_ALL, REDO_DROP_TABLE,
UNDO_ROW_INSERT (in REDO phase only, i.e. just doing records++),
UNDO_ROW_DELETE, UNDO_ROW_PURGE.
Code cleanups.
Monty: please look for "QQ". Sanja: please look for "Sanja".
Future tasks: recovery of the bitmap (easy), recovery of the state
(make it idempotent), more REDOs (Monty to work on
REDO_UPDATE?), UNDO phase...
Pushing this cset as it looks safe, contains test and bugfixes which
will help Monty implement applying of REDO_UPDATE.
sql/handler.cc:
typo
storage/maria/Makefile.am:
Adding ma_test_recovery (which ma_test_all invokes, and which can
also be run alone). Most of maria_read_log.c moved to ma_recovery.c
storage/maria/ha_maria.cc:
comments
storage/maria/ma_bitmap.c:
fixing comments. 2 -> sizeof(maria_bitmap_marker).
Bitmap-related part of _ma_initialize_datafile() moves in bitmap module.
Now putting the "bm" signature when creating the first bitmap page
(it used to happen only at next open, but that
caused an annoying difference when testing Recovery if the original
run didn't open the table, and it looks more
logical like this: it goes to disk only with its signature correct);
see the "QQ" comment towards the _ma_initialize_data_file() call
in ma_create.c for more).
When reading a bitmap page, verify its signature (happens when normally
using the table or when CHECKing it; not when REPAIRing it).
storage/maria/ma_blockrec.c:
* no need to sync the data file if table is not transactional
* Comments, code cleanup (log-related data moved to log-related code
block, int5store->page_store).
* Store the table's short id into LOGREC_UNDO_ROW_PURGE, like we
do for other records (though this record will soon be replaced
with a CLR).
* If "page" is 1 it means the page which extends from byte
page*block_size+1 to (page+1)*block_size (byte number 1 being
the first byte of the file). The last byte of the file is
data_file_length (same convention).
A new page needs to be created if the last byte of the page is
beyond the last byte of the file, i.e.
(page+1)*block_size+1 > data_file_length, so we correct the test
(bug found when testing log applying for ma_test1 -M -T --skip-update).
* update the page's LSN when removing a row from it during
execution of a REDO_PURGE_ROW record (bug found when testing log
applying for ma_test1 -M -T --skip-update).
* applying of REDO_PURGE_BLOCKs (limited to a one-page range for now).
storage/maria/ma_blockrec.h:
new functions. maria_bitmap_marker does not need to be exported.
storage/maria/ma_close.c:
we can always flush the table's state when closing the last instance
of the table. And it is needed for maria_read_log (as it does
not use maria_lock_database()).
storage/maria/ma_control_file.c:
when in Recovery, some assertions should not be used.
storage/maria/ma_control_file.h:
double-inclusion safe
storage/maria/ma_create.c:
during recovery, don't log records. Comments.
Moving the creation of the first bitmap page to ma_bitmap.c
storage/maria/ma_delete_table.c:
during recovery, don't log records. Log the end-zero of the dropped
table's name, so that recovery can use the string in place without
extending it to fit an end zero.
storage/maria/ma_loghandler.c:
* inwrite_rec_hook also needs access to the MARIA_SHARE, like
prewrite_rec_hook. This will be needed to update
share->records_diff (in the upcoming patch "recovery of the state").
* LOG_DESC::record_ends_group changed to an enum.
* LOG_DESC for LOGREC_REDO_PURGE_BLOCKS and LOGREC_UNDO_ROW_PURGE
corrected
* Sanja please see the @todo LOG BUG
* avoiding DBUG_RETURN(func()) as it gives confusing debug traces.
storage/maria/ma_loghandler.h:
- log write hooks called while the log's lock is held (inwrite_rec_hook)
now need the MARIA_SHARE, like prewrite_rec_hook already had
- instead of a bool saying if this record's type ends groups or not,
we refine: it may not end a group, it may end a group, or it may
be a group in itself. Imagine that we had a physical write failure
to a table before we log the UNDO, we still end up in
external_lock(F_UNLCK) and then we log a COMMIT: we don't want
to consider this COMMIT as ending the group of REDOs (don't want
to execute those REDOs during Recovery), that's why we say "COMMIT
is a group in itself, it aborts any previous group". This also
gives one more sanity check in maria_read_log.
storage/maria/ma_recovery.c:
New Recovery code, replacing the old pseudocode.
Most of maria_read_log moved here.
Call-able from ha_maria, but not enabled yet.
Compared to the previous version of maria_read_log, some bugs have
been fixed, debugging output can go to stdout or a disk file (for now
it's useful for me, later it can be changed), execution of
REDO_DROP_TABLE, REDO_DELETE_ALL, REDO_PURGE_BLOCKS has been added. Duplicate code
has been factored into functions. We abort an unfinished group
of records if we see a record which is a group in itself (like COMMIT).
No need for maria_panic() after a bug (which caused tables to not
be closed) was fixed; if there is yet another bug I prefer to see it.
When opening a table for Recovery, set data_file_length
and key_file_length to their real physical value (these are the
easiest state members to restore :). Warn us if the last page
was truncated (but Recovery handles it).
MARIA_SHARE::state::state::records is now partly recovered (not
idempotent, but works if recreating tables from scracth).
When applying a REDO to a page, stamp it with the UNDO's LSN
(current_group_end_lsn), not with the REDO's LSN; it makes
the table more identical to the original table (easier to compare
the two tables in the end).
Big thing missing: some types of REDOs are not handled,
and the UNDO phase does not exist (missing functions to execute UNDOs
to actually rollback). So for now tests are only inserting/deleting
a few 100 rows, closing the table and seeing if the log is applied ok;
it works. UPDATE not handled.
storage/maria/ma_recovery.h:
new functions: ma_recover() for recovery from inside ha_maria;
_ma_apply_log() for maria_read_log (ma_recover() calls _ma_apply_log()).
Btw, we need to not use the word "recover" for REPAIR/maria_chk anymore.
storage/maria/ma_rename.c:
don't write log records during recovery
storage/maria/ma_test2.c:
- fail if maria_info() or other subtests find some wrong information
- new option -g to skip updates.
- init the translog before creating the table, so that log applying
can work.
- in "#if 0" you'll see some fixed bugs (will be removed).
storage/maria/ma_test_all.sh:
cleanup files. Test log applying.
storage/maria/maria_read_log.c:
most of the logic moves to ma_recovery.c to be shared between
maria_read_log and recovery-from-inside-mysqld.
See ma_recovery.c for additional changes made to the moved code.
storage/maria/ma_test_recovery:
unit test for Recovery. Tests insert and delete,
REDO_UPDATE not yet coded.
Script is called from ma_test_all. Can run standalone.
2007-07-26 11:56:21 +02:00
|
|
|
goto err;
|
2006-04-11 15:45:10 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if (!silent)
|
|
|
|
puts("- Test if: Read rrnd - same");
|
|
|
|
DBUG_PRINT("progpos",("Read rrnd - same"));
|
2007-01-18 20:38:14 +01:00
|
|
|
assert(maria_scan_init(file) == 0);
|
2006-04-11 15:45:10 +02:00
|
|
|
for (i=0 ; i < write_count ; i++)
|
|
|
|
{
|
2007-01-18 20:38:14 +01:00
|
|
|
int tmp;
|
|
|
|
if ((tmp= maria_scan(file,read_record)) &&
|
|
|
|
tmp != HA_ERR_END_OF_FILE &&
|
|
|
|
tmp != HA_ERR_RECORD_DELETED)
|
|
|
|
{
|
|
|
|
printf("Got error %d when scanning table\n", tmp);
|
2006-04-11 15:45:10 +02:00
|
|
|
break;
|
2007-01-18 20:38:14 +01:00
|
|
|
}
|
2006-04-11 15:45:10 +02:00
|
|
|
}
|
2007-01-18 20:38:14 +01:00
|
|
|
maria_scan_end(file);
|
|
|
|
if (i != write_count && i != write_count - opt_delete)
|
|
|
|
{
|
|
|
|
printf("Found wrong number of rows while scanning table\n");
|
2006-04-11 15:45:10 +02:00
|
|
|
goto err;
|
2007-01-18 20:38:14 +01:00
|
|
|
}
|
2006-04-11 15:45:10 +02:00
|
|
|
|
|
|
|
bmove(read_record2,read_record,reclength);
|
|
|
|
for (i=min(2,keys) ; i-- > 0 ;)
|
|
|
|
{
|
|
|
|
if (maria_rsame(file,read_record2,(int) i)) goto err;
|
|
|
|
if (bcmp(read_record,read_record2,reclength) != 0)
|
|
|
|
{
|
2007-01-18 20:38:14 +01:00
|
|
|
printf("maria_rsame didn't find same record\n");
|
WL#3072 - Maria recovery
Unit test for recovery: runs ma_test1 and ma_test2 (both only with
INSERTs and DELETEs; UPDATEs disabled as not handled by recovery)
then moves the tables elswhere; recreates tables from the log, and
compares and fails if there is a difference. Passes now.
Most of maria_read_log.c moved to ma_recovery.c, as it will be re-used
for recovery-from-ha_maria.
Bugfixes of applying of REDO_INSERT, REDO_PURGE_ROW.
Applying of REDO_PURGE_BLOCKS, REDO_DELETE_ALL, REDO_DROP_TABLE,
UNDO_ROW_INSERT (in REDO phase only, i.e. just doing records++),
UNDO_ROW_DELETE, UNDO_ROW_PURGE.
Code cleanups.
Monty: please look for "QQ". Sanja: please look for "Sanja".
Future tasks: recovery of the bitmap (easy), recovery of the state
(make it idempotent), more REDOs (Monty to work on
REDO_UPDATE?), UNDO phase...
Pushing this cset as it looks safe, contains test and bugfixes which
will help Monty implement applying of REDO_UPDATE.
sql/handler.cc:
typo
storage/maria/Makefile.am:
Adding ma_test_recovery (which ma_test_all invokes, and which can
also be run alone). Most of maria_read_log.c moved to ma_recovery.c
storage/maria/ha_maria.cc:
comments
storage/maria/ma_bitmap.c:
fixing comments. 2 -> sizeof(maria_bitmap_marker).
Bitmap-related part of _ma_initialize_datafile() moves in bitmap module.
Now putting the "bm" signature when creating the first bitmap page
(it used to happen only at next open, but that
caused an annoying difference when testing Recovery if the original
run didn't open the table, and it looks more
logical like this: it goes to disk only with its signature correct);
see the "QQ" comment towards the _ma_initialize_data_file() call
in ma_create.c for more).
When reading a bitmap page, verify its signature (happens when normally
using the table or when CHECKing it; not when REPAIRing it).
storage/maria/ma_blockrec.c:
* no need to sync the data file if table is not transactional
* Comments, code cleanup (log-related data moved to log-related code
block, int5store->page_store).
* Store the table's short id into LOGREC_UNDO_ROW_PURGE, like we
do for other records (though this record will soon be replaced
with a CLR).
* If "page" is 1 it means the page which extends from byte
page*block_size+1 to (page+1)*block_size (byte number 1 being
the first byte of the file). The last byte of the file is
data_file_length (same convention).
A new page needs to be created if the last byte of the page is
beyond the last byte of the file, i.e.
(page+1)*block_size+1 > data_file_length, so we correct the test
(bug found when testing log applying for ma_test1 -M -T --skip-update).
* update the page's LSN when removing a row from it during
execution of a REDO_PURGE_ROW record (bug found when testing log
applying for ma_test1 -M -T --skip-update).
* applying of REDO_PURGE_BLOCKs (limited to a one-page range for now).
storage/maria/ma_blockrec.h:
new functions. maria_bitmap_marker does not need to be exported.
storage/maria/ma_close.c:
we can always flush the table's state when closing the last instance
of the table. And it is needed for maria_read_log (as it does
not use maria_lock_database()).
storage/maria/ma_control_file.c:
when in Recovery, some assertions should not be used.
storage/maria/ma_control_file.h:
double-inclusion safe
storage/maria/ma_create.c:
during recovery, don't log records. Comments.
Moving the creation of the first bitmap page to ma_bitmap.c
storage/maria/ma_delete_table.c:
during recovery, don't log records. Log the end-zero of the dropped
table's name, so that recovery can use the string in place without
extending it to fit an end zero.
storage/maria/ma_loghandler.c:
* inwrite_rec_hook also needs access to the MARIA_SHARE, like
prewrite_rec_hook. This will be needed to update
share->records_diff (in the upcoming patch "recovery of the state").
* LOG_DESC::record_ends_group changed to an enum.
* LOG_DESC for LOGREC_REDO_PURGE_BLOCKS and LOGREC_UNDO_ROW_PURGE
corrected
* Sanja please see the @todo LOG BUG
* avoiding DBUG_RETURN(func()) as it gives confusing debug traces.
storage/maria/ma_loghandler.h:
- log write hooks called while the log's lock is held (inwrite_rec_hook)
now need the MARIA_SHARE, like prewrite_rec_hook already had
- instead of a bool saying if this record's type ends groups or not,
we refine: it may not end a group, it may end a group, or it may
be a group in itself. Imagine that we had a physical write failure
to a table before we log the UNDO, we still end up in
external_lock(F_UNLCK) and then we log a COMMIT: we don't want
to consider this COMMIT as ending the group of REDOs (don't want
to execute those REDOs during Recovery), that's why we say "COMMIT
is a group in itself, it aborts any previous group". This also
gives one more sanity check in maria_read_log.
storage/maria/ma_recovery.c:
New Recovery code, replacing the old pseudocode.
Most of maria_read_log moved here.
Call-able from ha_maria, but not enabled yet.
Compared to the previous version of maria_read_log, some bugs have
been fixed, debugging output can go to stdout or a disk file (for now
it's useful for me, later it can be changed), execution of
REDO_DROP_TABLE, REDO_DELETE_ALL, REDO_PURGE_BLOCKS has been added. Duplicate code
has been factored into functions. We abort an unfinished group
of records if we see a record which is a group in itself (like COMMIT).
No need for maria_panic() after a bug (which caused tables to not
be closed) was fixed; if there is yet another bug I prefer to see it.
When opening a table for Recovery, set data_file_length
and key_file_length to their real physical value (these are the
easiest state members to restore :). Warn us if the last page
was truncated (but Recovery handles it).
MARIA_SHARE::state::state::records is now partly recovered (not
idempotent, but works if recreating tables from scracth).
When applying a REDO to a page, stamp it with the UNDO's LSN
(current_group_end_lsn), not with the REDO's LSN; it makes
the table more identical to the original table (easier to compare
the two tables in the end).
Big thing missing: some types of REDOs are not handled,
and the UNDO phase does not exist (missing functions to execute UNDOs
to actually rollback). So for now tests are only inserting/deleting
a few 100 rows, closing the table and seeing if the log is applied ok;
it works. UPDATE not handled.
storage/maria/ma_recovery.h:
new functions: ma_recover() for recovery from inside ha_maria;
_ma_apply_log() for maria_read_log (ma_recover() calls _ma_apply_log()).
Btw, we need to not use the word "recover" for REPAIR/maria_chk anymore.
storage/maria/ma_rename.c:
don't write log records during recovery
storage/maria/ma_test2.c:
- fail if maria_info() or other subtests find some wrong information
- new option -g to skip updates.
- init the translog before creating the table, so that log applying
can work.
- in "#if 0" you'll see some fixed bugs (will be removed).
storage/maria/ma_test_all.sh:
cleanup files. Test log applying.
storage/maria/maria_read_log.c:
most of the logic moves to ma_recovery.c to be shared between
maria_read_log and recovery-from-inside-mysqld.
See ma_recovery.c for additional changes made to the moved code.
storage/maria/ma_test_recovery:
unit test for Recovery. Tests insert and delete,
REDO_UPDATE not yet coded.
Script is called from ma_test_all. Can run standalone.
2007-07-26 11:56:21 +02:00
|
|
|
goto err;
|
2006-04-11 15:45:10 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
if (!silent)
|
|
|
|
puts("- Test maria_records_in_range");
|
|
|
|
maria_status(file,&info,HA_STATUS_VARIABLE);
|
|
|
|
for (i=0 ; i < info.keys ; i++)
|
|
|
|
{
|
|
|
|
key_range min_key, max_key;
|
|
|
|
if (maria_rfirst(file,read_record,(int) i) ||
|
|
|
|
maria_rlast(file,read_record2,(int) i))
|
|
|
|
goto err;
|
|
|
|
copy_key(file,(uint) i,(uchar*) read_record,(uchar*) key);
|
|
|
|
copy_key(file,(uint) i,(uchar*) read_record2,(uchar*) key2);
|
|
|
|
min_key.key= key;
|
2007-07-27 16:11:40 +02:00
|
|
|
min_key.keypart_map= HA_WHOLE_KEY;
|
2006-04-11 15:45:10 +02:00
|
|
|
min_key.flag= HA_READ_KEY_EXACT;
|
|
|
|
max_key.key= key2;
|
2007-07-27 16:11:40 +02:00
|
|
|
max_key.keypart_map= HA_WHOLE_KEY;
|
2006-04-11 15:45:10 +02:00
|
|
|
max_key.flag= HA_READ_AFTER_KEY;
|
|
|
|
|
|
|
|
range_records= maria_records_in_range(file,(int) i, &min_key, &max_key);
|
|
|
|
if (range_records < info.records*8/10 ||
|
|
|
|
range_records > info.records*12/10)
|
|
|
|
{
|
|
|
|
printf("maria_records_range returned %ld; Should be about %ld\n",
|
|
|
|
(long) range_records,(long) info.records);
|
WL#3072 - Maria recovery
Unit test for recovery: runs ma_test1 and ma_test2 (both only with
INSERTs and DELETEs; UPDATEs disabled as not handled by recovery)
then moves the tables elswhere; recreates tables from the log, and
compares and fails if there is a difference. Passes now.
Most of maria_read_log.c moved to ma_recovery.c, as it will be re-used
for recovery-from-ha_maria.
Bugfixes of applying of REDO_INSERT, REDO_PURGE_ROW.
Applying of REDO_PURGE_BLOCKS, REDO_DELETE_ALL, REDO_DROP_TABLE,
UNDO_ROW_INSERT (in REDO phase only, i.e. just doing records++),
UNDO_ROW_DELETE, UNDO_ROW_PURGE.
Code cleanups.
Monty: please look for "QQ". Sanja: please look for "Sanja".
Future tasks: recovery of the bitmap (easy), recovery of the state
(make it idempotent), more REDOs (Monty to work on
REDO_UPDATE?), UNDO phase...
Pushing this cset as it looks safe, contains test and bugfixes which
will help Monty implement applying of REDO_UPDATE.
sql/handler.cc:
typo
storage/maria/Makefile.am:
Adding ma_test_recovery (which ma_test_all invokes, and which can
also be run alone). Most of maria_read_log.c moved to ma_recovery.c
storage/maria/ha_maria.cc:
comments
storage/maria/ma_bitmap.c:
fixing comments. 2 -> sizeof(maria_bitmap_marker).
Bitmap-related part of _ma_initialize_datafile() moves in bitmap module.
Now putting the "bm" signature when creating the first bitmap page
(it used to happen only at next open, but that
caused an annoying difference when testing Recovery if the original
run didn't open the table, and it looks more
logical like this: it goes to disk only with its signature correct);
see the "QQ" comment towards the _ma_initialize_data_file() call
in ma_create.c for more).
When reading a bitmap page, verify its signature (happens when normally
using the table or when CHECKing it; not when REPAIRing it).
storage/maria/ma_blockrec.c:
* no need to sync the data file if table is not transactional
* Comments, code cleanup (log-related data moved to log-related code
block, int5store->page_store).
* Store the table's short id into LOGREC_UNDO_ROW_PURGE, like we
do for other records (though this record will soon be replaced
with a CLR).
* If "page" is 1 it means the page which extends from byte
page*block_size+1 to (page+1)*block_size (byte number 1 being
the first byte of the file). The last byte of the file is
data_file_length (same convention).
A new page needs to be created if the last byte of the page is
beyond the last byte of the file, i.e.
(page+1)*block_size+1 > data_file_length, so we correct the test
(bug found when testing log applying for ma_test1 -M -T --skip-update).
* update the page's LSN when removing a row from it during
execution of a REDO_PURGE_ROW record (bug found when testing log
applying for ma_test1 -M -T --skip-update).
* applying of REDO_PURGE_BLOCKs (limited to a one-page range for now).
storage/maria/ma_blockrec.h:
new functions. maria_bitmap_marker does not need to be exported.
storage/maria/ma_close.c:
we can always flush the table's state when closing the last instance
of the table. And it is needed for maria_read_log (as it does
not use maria_lock_database()).
storage/maria/ma_control_file.c:
when in Recovery, some assertions should not be used.
storage/maria/ma_control_file.h:
double-inclusion safe
storage/maria/ma_create.c:
during recovery, don't log records. Comments.
Moving the creation of the first bitmap page to ma_bitmap.c
storage/maria/ma_delete_table.c:
during recovery, don't log records. Log the end-zero of the dropped
table's name, so that recovery can use the string in place without
extending it to fit an end zero.
storage/maria/ma_loghandler.c:
* inwrite_rec_hook also needs access to the MARIA_SHARE, like
prewrite_rec_hook. This will be needed to update
share->records_diff (in the upcoming patch "recovery of the state").
* LOG_DESC::record_ends_group changed to an enum.
* LOG_DESC for LOGREC_REDO_PURGE_BLOCKS and LOGREC_UNDO_ROW_PURGE
corrected
* Sanja please see the @todo LOG BUG
* avoiding DBUG_RETURN(func()) as it gives confusing debug traces.
storage/maria/ma_loghandler.h:
- log write hooks called while the log's lock is held (inwrite_rec_hook)
now need the MARIA_SHARE, like prewrite_rec_hook already had
- instead of a bool saying if this record's type ends groups or not,
we refine: it may not end a group, it may end a group, or it may
be a group in itself. Imagine that we had a physical write failure
to a table before we log the UNDO, we still end up in
external_lock(F_UNLCK) and then we log a COMMIT: we don't want
to consider this COMMIT as ending the group of REDOs (don't want
to execute those REDOs during Recovery), that's why we say "COMMIT
is a group in itself, it aborts any previous group". This also
gives one more sanity check in maria_read_log.
storage/maria/ma_recovery.c:
New Recovery code, replacing the old pseudocode.
Most of maria_read_log moved here.
Call-able from ha_maria, but not enabled yet.
Compared to the previous version of maria_read_log, some bugs have
been fixed, debugging output can go to stdout or a disk file (for now
it's useful for me, later it can be changed), execution of
REDO_DROP_TABLE, REDO_DELETE_ALL, REDO_PURGE_BLOCKS has been added. Duplicate code
has been factored into functions. We abort an unfinished group
of records if we see a record which is a group in itself (like COMMIT).
No need for maria_panic() after a bug (which caused tables to not
be closed) was fixed; if there is yet another bug I prefer to see it.
When opening a table for Recovery, set data_file_length
and key_file_length to their real physical value (these are the
easiest state members to restore :). Warn us if the last page
was truncated (but Recovery handles it).
MARIA_SHARE::state::state::records is now partly recovered (not
idempotent, but works if recreating tables from scracth).
When applying a REDO to a page, stamp it with the UNDO's LSN
(current_group_end_lsn), not with the REDO's LSN; it makes
the table more identical to the original table (easier to compare
the two tables in the end).
Big thing missing: some types of REDOs are not handled,
and the UNDO phase does not exist (missing functions to execute UNDOs
to actually rollback). So for now tests are only inserting/deleting
a few 100 rows, closing the table and seeing if the log is applied ok;
it works. UPDATE not handled.
storage/maria/ma_recovery.h:
new functions: ma_recover() for recovery from inside ha_maria;
_ma_apply_log() for maria_read_log (ma_recover() calls _ma_apply_log()).
Btw, we need to not use the word "recover" for REPAIR/maria_chk anymore.
storage/maria/ma_rename.c:
don't write log records during recovery
storage/maria/ma_test2.c:
- fail if maria_info() or other subtests find some wrong information
- new option -g to skip updates.
- init the translog before creating the table, so that log applying
can work.
- in "#if 0" you'll see some fixed bugs (will be removed).
storage/maria/ma_test_all.sh:
cleanup files. Test log applying.
storage/maria/maria_read_log.c:
most of the logic moves to ma_recovery.c to be shared between
maria_read_log and recovery-from-inside-mysqld.
See ma_recovery.c for additional changes made to the moved code.
storage/maria/ma_test_recovery:
unit test for Recovery. Tests insert and delete,
REDO_UPDATE not yet coded.
Script is called from ma_test_all. Can run standalone.
2007-07-26 11:56:21 +02:00
|
|
|
goto err;
|
2006-04-11 15:45:10 +02:00
|
|
|
}
|
|
|
|
if (verbose)
|
|
|
|
{
|
|
|
|
printf("maria_records_range returned %ld; Exact is %ld (diff: %4.2g %%)\n",
|
|
|
|
(long) range_records, (long) info.records,
|
|
|
|
labs((long) range_records - (long) info.records)*100.0/
|
|
|
|
info.records);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
for (i=0 ; i < 5 ; i++)
|
|
|
|
{
|
|
|
|
for (j=rnd(1000)+1 ; j>0 && key1[j] == 0 ; j--) ;
|
|
|
|
for (k=rnd(1000)+1 ; k>0 && key1[k] == 0 ; k--) ;
|
|
|
|
if (j != 0 && k != 0)
|
|
|
|
{
|
|
|
|
key_range min_key, max_key;
|
|
|
|
if (j > k)
|
|
|
|
swap_variables(int, j, k);
|
2007-10-04 19:33:42 +02:00
|
|
|
sprintf((char*) key,"%6d",j);
|
|
|
|
sprintf((char*) key2,"%6d",k);
|
2006-04-11 15:45:10 +02:00
|
|
|
|
|
|
|
min_key.key= key;
|
2007-08-06 16:13:42 +02:00
|
|
|
min_key.keypart_map= HA_WHOLE_KEY;
|
2006-04-11 15:45:10 +02:00
|
|
|
min_key.flag= HA_READ_AFTER_KEY;
|
|
|
|
max_key.key= key2;
|
2007-08-06 16:13:42 +02:00
|
|
|
max_key.keypart_map= HA_WHOLE_KEY;
|
2006-04-11 15:45:10 +02:00
|
|
|
max_key.flag= HA_READ_BEFORE_KEY;
|
|
|
|
range_records= maria_records_in_range(file, 0, &min_key, &max_key);
|
|
|
|
records=0;
|
|
|
|
for (j++ ; j < k ; j++)
|
|
|
|
records+=key1[j];
|
|
|
|
if ((long) range_records < (long) records*7/10-2 ||
|
|
|
|
(long) range_records > (long) records*14/10+2)
|
|
|
|
{
|
|
|
|
printf("maria_records_range for key: %d returned %lu; Should be about %lu\n",
|
|
|
|
i, (ulong) range_records, (ulong) records);
|
WL#3072 - Maria recovery
Unit test for recovery: runs ma_test1 and ma_test2 (both only with
INSERTs and DELETEs; UPDATEs disabled as not handled by recovery)
then moves the tables elswhere; recreates tables from the log, and
compares and fails if there is a difference. Passes now.
Most of maria_read_log.c moved to ma_recovery.c, as it will be re-used
for recovery-from-ha_maria.
Bugfixes of applying of REDO_INSERT, REDO_PURGE_ROW.
Applying of REDO_PURGE_BLOCKS, REDO_DELETE_ALL, REDO_DROP_TABLE,
UNDO_ROW_INSERT (in REDO phase only, i.e. just doing records++),
UNDO_ROW_DELETE, UNDO_ROW_PURGE.
Code cleanups.
Monty: please look for "QQ". Sanja: please look for "Sanja".
Future tasks: recovery of the bitmap (easy), recovery of the state
(make it idempotent), more REDOs (Monty to work on
REDO_UPDATE?), UNDO phase...
Pushing this cset as it looks safe, contains test and bugfixes which
will help Monty implement applying of REDO_UPDATE.
sql/handler.cc:
typo
storage/maria/Makefile.am:
Adding ma_test_recovery (which ma_test_all invokes, and which can
also be run alone). Most of maria_read_log.c moved to ma_recovery.c
storage/maria/ha_maria.cc:
comments
storage/maria/ma_bitmap.c:
fixing comments. 2 -> sizeof(maria_bitmap_marker).
Bitmap-related part of _ma_initialize_datafile() moves in bitmap module.
Now putting the "bm" signature when creating the first bitmap page
(it used to happen only at next open, but that
caused an annoying difference when testing Recovery if the original
run didn't open the table, and it looks more
logical like this: it goes to disk only with its signature correct);
see the "QQ" comment towards the _ma_initialize_data_file() call
in ma_create.c for more).
When reading a bitmap page, verify its signature (happens when normally
using the table or when CHECKing it; not when REPAIRing it).
storage/maria/ma_blockrec.c:
* no need to sync the data file if table is not transactional
* Comments, code cleanup (log-related data moved to log-related code
block, int5store->page_store).
* Store the table's short id into LOGREC_UNDO_ROW_PURGE, like we
do for other records (though this record will soon be replaced
with a CLR).
* If "page" is 1 it means the page which extends from byte
page*block_size+1 to (page+1)*block_size (byte number 1 being
the first byte of the file). The last byte of the file is
data_file_length (same convention).
A new page needs to be created if the last byte of the page is
beyond the last byte of the file, i.e.
(page+1)*block_size+1 > data_file_length, so we correct the test
(bug found when testing log applying for ma_test1 -M -T --skip-update).
* update the page's LSN when removing a row from it during
execution of a REDO_PURGE_ROW record (bug found when testing log
applying for ma_test1 -M -T --skip-update).
* applying of REDO_PURGE_BLOCKs (limited to a one-page range for now).
storage/maria/ma_blockrec.h:
new functions. maria_bitmap_marker does not need to be exported.
storage/maria/ma_close.c:
we can always flush the table's state when closing the last instance
of the table. And it is needed for maria_read_log (as it does
not use maria_lock_database()).
storage/maria/ma_control_file.c:
when in Recovery, some assertions should not be used.
storage/maria/ma_control_file.h:
double-inclusion safe
storage/maria/ma_create.c:
during recovery, don't log records. Comments.
Moving the creation of the first bitmap page to ma_bitmap.c
storage/maria/ma_delete_table.c:
during recovery, don't log records. Log the end-zero of the dropped
table's name, so that recovery can use the string in place without
extending it to fit an end zero.
storage/maria/ma_loghandler.c:
* inwrite_rec_hook also needs access to the MARIA_SHARE, like
prewrite_rec_hook. This will be needed to update
share->records_diff (in the upcoming patch "recovery of the state").
* LOG_DESC::record_ends_group changed to an enum.
* LOG_DESC for LOGREC_REDO_PURGE_BLOCKS and LOGREC_UNDO_ROW_PURGE
corrected
* Sanja please see the @todo LOG BUG
* avoiding DBUG_RETURN(func()) as it gives confusing debug traces.
storage/maria/ma_loghandler.h:
- log write hooks called while the log's lock is held (inwrite_rec_hook)
now need the MARIA_SHARE, like prewrite_rec_hook already had
- instead of a bool saying if this record's type ends groups or not,
we refine: it may not end a group, it may end a group, or it may
be a group in itself. Imagine that we had a physical write failure
to a table before we log the UNDO, we still end up in
external_lock(F_UNLCK) and then we log a COMMIT: we don't want
to consider this COMMIT as ending the group of REDOs (don't want
to execute those REDOs during Recovery), that's why we say "COMMIT
is a group in itself, it aborts any previous group". This also
gives one more sanity check in maria_read_log.
storage/maria/ma_recovery.c:
New Recovery code, replacing the old pseudocode.
Most of maria_read_log moved here.
Call-able from ha_maria, but not enabled yet.
Compared to the previous version of maria_read_log, some bugs have
been fixed, debugging output can go to stdout or a disk file (for now
it's useful for me, later it can be changed), execution of
REDO_DROP_TABLE, REDO_DELETE_ALL, REDO_PURGE_BLOCKS has been added. Duplicate code
has been factored into functions. We abort an unfinished group
of records if we see a record which is a group in itself (like COMMIT).
No need for maria_panic() after a bug (which caused tables to not
be closed) was fixed; if there is yet another bug I prefer to see it.
When opening a table for Recovery, set data_file_length
and key_file_length to their real physical value (these are the
easiest state members to restore :). Warn us if the last page
was truncated (but Recovery handles it).
MARIA_SHARE::state::state::records is now partly recovered (not
idempotent, but works if recreating tables from scracth).
When applying a REDO to a page, stamp it with the UNDO's LSN
(current_group_end_lsn), not with the REDO's LSN; it makes
the table more identical to the original table (easier to compare
the two tables in the end).
Big thing missing: some types of REDOs are not handled,
and the UNDO phase does not exist (missing functions to execute UNDOs
to actually rollback). So for now tests are only inserting/deleting
a few 100 rows, closing the table and seeing if the log is applied ok;
it works. UPDATE not handled.
storage/maria/ma_recovery.h:
new functions: ma_recover() for recovery from inside ha_maria;
_ma_apply_log() for maria_read_log (ma_recover() calls _ma_apply_log()).
Btw, we need to not use the word "recover" for REPAIR/maria_chk anymore.
storage/maria/ma_rename.c:
don't write log records during recovery
storage/maria/ma_test2.c:
- fail if maria_info() or other subtests find some wrong information
- new option -g to skip updates.
- init the translog before creating the table, so that log applying
can work.
- in "#if 0" you'll see some fixed bugs (will be removed).
storage/maria/ma_test_all.sh:
cleanup files. Test log applying.
storage/maria/maria_read_log.c:
most of the logic moves to ma_recovery.c to be shared between
maria_read_log and recovery-from-inside-mysqld.
See ma_recovery.c for additional changes made to the moved code.
storage/maria/ma_test_recovery:
unit test for Recovery. Tests insert and delete,
REDO_UPDATE not yet coded.
Script is called from ma_test_all. Can run standalone.
2007-07-26 11:56:21 +02:00
|
|
|
goto err;
|
2006-04-11 15:45:10 +02:00
|
|
|
}
|
|
|
|
if (verbose && records)
|
|
|
|
{
|
|
|
|
printf("maria_records_range returned %lu; Exact is %lu (diff: %4.2g %%)\n",
|
|
|
|
(ulong) range_records, (ulong) records,
|
|
|
|
labs((long) range_records-(long) records)*100.0/records);
|
|
|
|
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if (!silent)
|
|
|
|
printf("- maria_info\n");
|
|
|
|
maria_status(file,&info,HA_STATUS_VARIABLE | HA_STATUS_CONST);
|
|
|
|
if (info.records != write_count-opt_delete || info.deleted > opt_delete + update
|
|
|
|
|| info.keys != keys)
|
|
|
|
{
|
|
|
|
puts("Wrong info from maria_info");
|
|
|
|
printf("Got: records: %lu delete: %lu i_keys: %d\n",
|
|
|
|
(ulong) info.records, (ulong) info.deleted, info.keys);
|
WL#3072 - Maria recovery
Unit test for recovery: runs ma_test1 and ma_test2 (both only with
INSERTs and DELETEs; UPDATEs disabled as not handled by recovery)
then moves the tables elswhere; recreates tables from the log, and
compares and fails if there is a difference. Passes now.
Most of maria_read_log.c moved to ma_recovery.c, as it will be re-used
for recovery-from-ha_maria.
Bugfixes of applying of REDO_INSERT, REDO_PURGE_ROW.
Applying of REDO_PURGE_BLOCKS, REDO_DELETE_ALL, REDO_DROP_TABLE,
UNDO_ROW_INSERT (in REDO phase only, i.e. just doing records++),
UNDO_ROW_DELETE, UNDO_ROW_PURGE.
Code cleanups.
Monty: please look for "QQ". Sanja: please look for "Sanja".
Future tasks: recovery of the bitmap (easy), recovery of the state
(make it idempotent), more REDOs (Monty to work on
REDO_UPDATE?), UNDO phase...
Pushing this cset as it looks safe, contains test and bugfixes which
will help Monty implement applying of REDO_UPDATE.
sql/handler.cc:
typo
storage/maria/Makefile.am:
Adding ma_test_recovery (which ma_test_all invokes, and which can
also be run alone). Most of maria_read_log.c moved to ma_recovery.c
storage/maria/ha_maria.cc:
comments
storage/maria/ma_bitmap.c:
fixing comments. 2 -> sizeof(maria_bitmap_marker).
Bitmap-related part of _ma_initialize_datafile() moves in bitmap module.
Now putting the "bm" signature when creating the first bitmap page
(it used to happen only at next open, but that
caused an annoying difference when testing Recovery if the original
run didn't open the table, and it looks more
logical like this: it goes to disk only with its signature correct);
see the "QQ" comment towards the _ma_initialize_data_file() call
in ma_create.c for more).
When reading a bitmap page, verify its signature (happens when normally
using the table or when CHECKing it; not when REPAIRing it).
storage/maria/ma_blockrec.c:
* no need to sync the data file if table is not transactional
* Comments, code cleanup (log-related data moved to log-related code
block, int5store->page_store).
* Store the table's short id into LOGREC_UNDO_ROW_PURGE, like we
do for other records (though this record will soon be replaced
with a CLR).
* If "page" is 1 it means the page which extends from byte
page*block_size+1 to (page+1)*block_size (byte number 1 being
the first byte of the file). The last byte of the file is
data_file_length (same convention).
A new page needs to be created if the last byte of the page is
beyond the last byte of the file, i.e.
(page+1)*block_size+1 > data_file_length, so we correct the test
(bug found when testing log applying for ma_test1 -M -T --skip-update).
* update the page's LSN when removing a row from it during
execution of a REDO_PURGE_ROW record (bug found when testing log
applying for ma_test1 -M -T --skip-update).
* applying of REDO_PURGE_BLOCKs (limited to a one-page range for now).
storage/maria/ma_blockrec.h:
new functions. maria_bitmap_marker does not need to be exported.
storage/maria/ma_close.c:
we can always flush the table's state when closing the last instance
of the table. And it is needed for maria_read_log (as it does
not use maria_lock_database()).
storage/maria/ma_control_file.c:
when in Recovery, some assertions should not be used.
storage/maria/ma_control_file.h:
double-inclusion safe
storage/maria/ma_create.c:
during recovery, don't log records. Comments.
Moving the creation of the first bitmap page to ma_bitmap.c
storage/maria/ma_delete_table.c:
during recovery, don't log records. Log the end-zero of the dropped
table's name, so that recovery can use the string in place without
extending it to fit an end zero.
storage/maria/ma_loghandler.c:
* inwrite_rec_hook also needs access to the MARIA_SHARE, like
prewrite_rec_hook. This will be needed to update
share->records_diff (in the upcoming patch "recovery of the state").
* LOG_DESC::record_ends_group changed to an enum.
* LOG_DESC for LOGREC_REDO_PURGE_BLOCKS and LOGREC_UNDO_ROW_PURGE
corrected
* Sanja please see the @todo LOG BUG
* avoiding DBUG_RETURN(func()) as it gives confusing debug traces.
storage/maria/ma_loghandler.h:
- log write hooks called while the log's lock is held (inwrite_rec_hook)
now need the MARIA_SHARE, like prewrite_rec_hook already had
- instead of a bool saying if this record's type ends groups or not,
we refine: it may not end a group, it may end a group, or it may
be a group in itself. Imagine that we had a physical write failure
to a table before we log the UNDO, we still end up in
external_lock(F_UNLCK) and then we log a COMMIT: we don't want
to consider this COMMIT as ending the group of REDOs (don't want
to execute those REDOs during Recovery), that's why we say "COMMIT
is a group in itself, it aborts any previous group". This also
gives one more sanity check in maria_read_log.
storage/maria/ma_recovery.c:
New Recovery code, replacing the old pseudocode.
Most of maria_read_log moved here.
Call-able from ha_maria, but not enabled yet.
Compared to the previous version of maria_read_log, some bugs have
been fixed, debugging output can go to stdout or a disk file (for now
it's useful for me, later it can be changed), execution of
REDO_DROP_TABLE, REDO_DELETE_ALL, REDO_PURGE_BLOCKS has been added. Duplicate code
has been factored into functions. We abort an unfinished group
of records if we see a record which is a group in itself (like COMMIT).
No need for maria_panic() after a bug (which caused tables to not
be closed) was fixed; if there is yet another bug I prefer to see it.
When opening a table for Recovery, set data_file_length
and key_file_length to their real physical value (these are the
easiest state members to restore :). Warn us if the last page
was truncated (but Recovery handles it).
MARIA_SHARE::state::state::records is now partly recovered (not
idempotent, but works if recreating tables from scracth).
When applying a REDO to a page, stamp it with the UNDO's LSN
(current_group_end_lsn), not with the REDO's LSN; it makes
the table more identical to the original table (easier to compare
the two tables in the end).
Big thing missing: some types of REDOs are not handled,
and the UNDO phase does not exist (missing functions to execute UNDOs
to actually rollback). So for now tests are only inserting/deleting
a few 100 rows, closing the table and seeing if the log is applied ok;
it works. UPDATE not handled.
storage/maria/ma_recovery.h:
new functions: ma_recover() for recovery from inside ha_maria;
_ma_apply_log() for maria_read_log (ma_recover() calls _ma_apply_log()).
Btw, we need to not use the word "recover" for REPAIR/maria_chk anymore.
storage/maria/ma_rename.c:
don't write log records during recovery
storage/maria/ma_test2.c:
- fail if maria_info() or other subtests find some wrong information
- new option -g to skip updates.
- init the translog before creating the table, so that log applying
can work.
- in "#if 0" you'll see some fixed bugs (will be removed).
storage/maria/ma_test_all.sh:
cleanup files. Test log applying.
storage/maria/maria_read_log.c:
most of the logic moves to ma_recovery.c to be shared between
maria_read_log and recovery-from-inside-mysqld.
See ma_recovery.c for additional changes made to the moved code.
storage/maria/ma_test_recovery:
unit test for Recovery. Tests insert and delete,
REDO_UPDATE not yet coded.
Script is called from ma_test_all. Can run standalone.
2007-07-26 11:56:21 +02:00
|
|
|
goto err;
|
2006-04-11 15:45:10 +02:00
|
|
|
}
|
|
|
|
if (verbose)
|
|
|
|
{
|
|
|
|
char buff[80];
|
|
|
|
get_date(buff,3,info.create_time);
|
|
|
|
printf("info: Created %s\n",buff);
|
|
|
|
get_date(buff,3,info.check_time);
|
|
|
|
printf("info: checked %s\n",buff);
|
|
|
|
get_date(buff,3,info.update_time);
|
|
|
|
printf("info: Modified %s\n",buff);
|
|
|
|
}
|
|
|
|
|
|
|
|
maria_panic(HA_PANIC_WRITE);
|
|
|
|
maria_panic(HA_PANIC_READ);
|
|
|
|
if (maria_is_changed(file))
|
|
|
|
puts("Warning: maria_is_changed reported that datafile was changed");
|
|
|
|
|
|
|
|
if (!silent)
|
|
|
|
printf("- maria_extra(CACHE) + maria_rrnd.... + maria_extra(NO_CACHE)\n");
|
Completion of merge of mysql-5.1 into mysql-maria.
Manually imported changes done to MyISAM (include/myisam.h,
storage/myisam/*, sql/ha_myisam.*, mysql-test/t/myisam.test,
mysql-test/t/ps_2myisam.test) the last
months into Maria (tedious, should do it more frequently in the
future), including those not done at the previous 5.1->Maria merge
(please in the future don't forget to apply MyISAM changes to Maria
when you merge 5.1 into Maria).
Note: I didn't try to import anything which could be MyISAM-related
in other tests of mysql-test (I didn't want to dig in all csets),
but as QA is working to make most tests re-usable for other engines
(Falcon), it is likely that we'll benefit from this and just have
to set engine=Maria somewhere to run those tests on Maria.
func_group and partition tests fail but they already do in main 5.1
on my machine. No Valgrind error in t/*maria*.test.
Monty: please see the commit comment of maria.result and check.
BitKeeper/deleted/.del-ha_maria.m4:
Delete: config/ac-macros/ha_maria.m4
configure.in:
fix for the new way of enabling engines
include/maria.h:
importing changes done to MyISAM the last months into Maria
include/my_handler.h:
importing changes done to MyISAM the last months into Maria
include/myisam.h:
importing changes done to MyISAM the last months into Maria
mysql-test/r/maria.result:
identical to myisam.result, except the engine name in some places
AND in the line testing key_block_size=1000000000000000000:
Maria gives a key block size of 8192 while MyISAM gives 4096;
is it explainable by the difference between MARIA_KEY_BLOCK_LENGTH
and the same constant in MyISAM? Monty?
mysql-test/r/ps_maria.result:
identical to ps_2myisam.result (except the engine name in some places)
mysql-test/t/maria.test:
instead of engine=maria everywhere, I use @@storage_engine (reduces
the diff with myisam.test).
importing changes done to MyISAM the last months into Maria
mysys/my_handler.c:
importing changes done to MyISAM the last months into Maria
sql/ha_maria.cc:
importing changes done to MyISAM the last months into Maria
sql/ha_maria.h:
importing changes done to MyISAM the last months into Maria
sql/mysqld.cc:
unneeded
storage/maria/Makefile.am:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_check.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_create.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_delete_table.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_dynrec.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_extra.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_ft_boolean_search.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_ft_eval.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_ft_nlq_search.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_ft_parser.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_ft_test1.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_ft_update.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_ftdefs.h:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_key.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_open.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_page.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_rkey.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_rsamepos.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_rt_index.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_rt_mbr.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_search.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_sort.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_test1.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_test2.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_test3.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_update.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_write.c:
importing changes done to MyISAM the last months into Maria
storage/maria/maria_chk.c:
importing changes done to MyISAM the last months into Maria
storage/maria/maria_def.h:
importing changes done to MyISAM the last months into Maria
storage/maria/maria_ftdump.c:
importing changes done to MyISAM the last months into Maria
storage/maria/maria_pack.c:
importing changes done to MyISAM the last months into Maria
2006-08-10 16:36:54 +02:00
|
|
|
if (maria_reset(file) || maria_extra(file,HA_EXTRA_CACHE,0))
|
2006-04-11 15:45:10 +02:00
|
|
|
{
|
|
|
|
if (locking || (!use_blob && !pack_fields))
|
|
|
|
{
|
|
|
|
puts("got error from maria_extra(HA_EXTRA_CACHE)");
|
WL#3072 - Maria recovery
Unit test for recovery: runs ma_test1 and ma_test2 (both only with
INSERTs and DELETEs; UPDATEs disabled as not handled by recovery)
then moves the tables elswhere; recreates tables from the log, and
compares and fails if there is a difference. Passes now.
Most of maria_read_log.c moved to ma_recovery.c, as it will be re-used
for recovery-from-ha_maria.
Bugfixes of applying of REDO_INSERT, REDO_PURGE_ROW.
Applying of REDO_PURGE_BLOCKS, REDO_DELETE_ALL, REDO_DROP_TABLE,
UNDO_ROW_INSERT (in REDO phase only, i.e. just doing records++),
UNDO_ROW_DELETE, UNDO_ROW_PURGE.
Code cleanups.
Monty: please look for "QQ". Sanja: please look for "Sanja".
Future tasks: recovery of the bitmap (easy), recovery of the state
(make it idempotent), more REDOs (Monty to work on
REDO_UPDATE?), UNDO phase...
Pushing this cset as it looks safe, contains test and bugfixes which
will help Monty implement applying of REDO_UPDATE.
sql/handler.cc:
typo
storage/maria/Makefile.am:
Adding ma_test_recovery (which ma_test_all invokes, and which can
also be run alone). Most of maria_read_log.c moved to ma_recovery.c
storage/maria/ha_maria.cc:
comments
storage/maria/ma_bitmap.c:
fixing comments. 2 -> sizeof(maria_bitmap_marker).
Bitmap-related part of _ma_initialize_datafile() moves in bitmap module.
Now putting the "bm" signature when creating the first bitmap page
(it used to happen only at next open, but that
caused an annoying difference when testing Recovery if the original
run didn't open the table, and it looks more
logical like this: it goes to disk only with its signature correct);
see the "QQ" comment towards the _ma_initialize_data_file() call
in ma_create.c for more).
When reading a bitmap page, verify its signature (happens when normally
using the table or when CHECKing it; not when REPAIRing it).
storage/maria/ma_blockrec.c:
* no need to sync the data file if table is not transactional
* Comments, code cleanup (log-related data moved to log-related code
block, int5store->page_store).
* Store the table's short id into LOGREC_UNDO_ROW_PURGE, like we
do for other records (though this record will soon be replaced
with a CLR).
* If "page" is 1 it means the page which extends from byte
page*block_size+1 to (page+1)*block_size (byte number 1 being
the first byte of the file). The last byte of the file is
data_file_length (same convention).
A new page needs to be created if the last byte of the page is
beyond the last byte of the file, i.e.
(page+1)*block_size+1 > data_file_length, so we correct the test
(bug found when testing log applying for ma_test1 -M -T --skip-update).
* update the page's LSN when removing a row from it during
execution of a REDO_PURGE_ROW record (bug found when testing log
applying for ma_test1 -M -T --skip-update).
* applying of REDO_PURGE_BLOCKs (limited to a one-page range for now).
storage/maria/ma_blockrec.h:
new functions. maria_bitmap_marker does not need to be exported.
storage/maria/ma_close.c:
we can always flush the table's state when closing the last instance
of the table. And it is needed for maria_read_log (as it does
not use maria_lock_database()).
storage/maria/ma_control_file.c:
when in Recovery, some assertions should not be used.
storage/maria/ma_control_file.h:
double-inclusion safe
storage/maria/ma_create.c:
during recovery, don't log records. Comments.
Moving the creation of the first bitmap page to ma_bitmap.c
storage/maria/ma_delete_table.c:
during recovery, don't log records. Log the end-zero of the dropped
table's name, so that recovery can use the string in place without
extending it to fit an end zero.
storage/maria/ma_loghandler.c:
* inwrite_rec_hook also needs access to the MARIA_SHARE, like
prewrite_rec_hook. This will be needed to update
share->records_diff (in the upcoming patch "recovery of the state").
* LOG_DESC::record_ends_group changed to an enum.
* LOG_DESC for LOGREC_REDO_PURGE_BLOCKS and LOGREC_UNDO_ROW_PURGE
corrected
* Sanja please see the @todo LOG BUG
* avoiding DBUG_RETURN(func()) as it gives confusing debug traces.
storage/maria/ma_loghandler.h:
- log write hooks called while the log's lock is held (inwrite_rec_hook)
now need the MARIA_SHARE, like prewrite_rec_hook already had
- instead of a bool saying if this record's type ends groups or not,
we refine: it may not end a group, it may end a group, or it may
be a group in itself. Imagine that we had a physical write failure
to a table before we log the UNDO, we still end up in
external_lock(F_UNLCK) and then we log a COMMIT: we don't want
to consider this COMMIT as ending the group of REDOs (don't want
to execute those REDOs during Recovery), that's why we say "COMMIT
is a group in itself, it aborts any previous group". This also
gives one more sanity check in maria_read_log.
storage/maria/ma_recovery.c:
New Recovery code, replacing the old pseudocode.
Most of maria_read_log moved here.
Call-able from ha_maria, but not enabled yet.
Compared to the previous version of maria_read_log, some bugs have
been fixed, debugging output can go to stdout or a disk file (for now
it's useful for me, later it can be changed), execution of
REDO_DROP_TABLE, REDO_DELETE_ALL, REDO_PURGE_BLOCKS has been added. Duplicate code
has been factored into functions. We abort an unfinished group
of records if we see a record which is a group in itself (like COMMIT).
No need for maria_panic() after a bug (which caused tables to not
be closed) was fixed; if there is yet another bug I prefer to see it.
When opening a table for Recovery, set data_file_length
and key_file_length to their real physical value (these are the
easiest state members to restore :). Warn us if the last page
was truncated (but Recovery handles it).
MARIA_SHARE::state::state::records is now partly recovered (not
idempotent, but works if recreating tables from scracth).
When applying a REDO to a page, stamp it with the UNDO's LSN
(current_group_end_lsn), not with the REDO's LSN; it makes
the table more identical to the original table (easier to compare
the two tables in the end).
Big thing missing: some types of REDOs are not handled,
and the UNDO phase does not exist (missing functions to execute UNDOs
to actually rollback). So for now tests are only inserting/deleting
a few 100 rows, closing the table and seeing if the log is applied ok;
it works. UPDATE not handled.
storage/maria/ma_recovery.h:
new functions: ma_recover() for recovery from inside ha_maria;
_ma_apply_log() for maria_read_log (ma_recover() calls _ma_apply_log()).
Btw, we need to not use the word "recover" for REPAIR/maria_chk anymore.
storage/maria/ma_rename.c:
don't write log records during recovery
storage/maria/ma_test2.c:
- fail if maria_info() or other subtests find some wrong information
- new option -g to skip updates.
- init the translog before creating the table, so that log applying
can work.
- in "#if 0" you'll see some fixed bugs (will be removed).
storage/maria/ma_test_all.sh:
cleanup files. Test log applying.
storage/maria/maria_read_log.c:
most of the logic moves to ma_recovery.c to be shared between
maria_read_log and recovery-from-inside-mysqld.
See ma_recovery.c for additional changes made to the moved code.
storage/maria/ma_test_recovery:
unit test for Recovery. Tests insert and delete,
REDO_UPDATE not yet coded.
Script is called from ma_test_all. Can run standalone.
2007-07-26 11:56:21 +02:00
|
|
|
goto err;
|
2006-04-11 15:45:10 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
ant=0;
|
2007-01-18 20:38:14 +01:00
|
|
|
assert(maria_scan_init(file) == 0);
|
|
|
|
while ((error= maria_scan(file,record)) != HA_ERR_END_OF_FILE &&
|
2006-04-11 15:45:10 +02:00
|
|
|
ant < write_count + 10)
|
2007-01-18 20:38:14 +01:00
|
|
|
ant+= error ? 0 : 1;
|
|
|
|
maria_scan_end(file);
|
2006-04-11 15:45:10 +02:00
|
|
|
if (ant != write_count-opt_delete)
|
|
|
|
{
|
2007-01-18 20:38:14 +01:00
|
|
|
printf("scan with cache: I can only find: %d records of %d\n",
|
2006-04-11 15:45:10 +02:00
|
|
|
ant,write_count-opt_delete);
|
2007-10-04 19:33:42 +02:00
|
|
|
maria_scan_end(file);
|
WL#3072 - Maria recovery
Unit test for recovery: runs ma_test1 and ma_test2 (both only with
INSERTs and DELETEs; UPDATEs disabled as not handled by recovery)
then moves the tables elswhere; recreates tables from the log, and
compares and fails if there is a difference. Passes now.
Most of maria_read_log.c moved to ma_recovery.c, as it will be re-used
for recovery-from-ha_maria.
Bugfixes of applying of REDO_INSERT, REDO_PURGE_ROW.
Applying of REDO_PURGE_BLOCKS, REDO_DELETE_ALL, REDO_DROP_TABLE,
UNDO_ROW_INSERT (in REDO phase only, i.e. just doing records++),
UNDO_ROW_DELETE, UNDO_ROW_PURGE.
Code cleanups.
Monty: please look for "QQ". Sanja: please look for "Sanja".
Future tasks: recovery of the bitmap (easy), recovery of the state
(make it idempotent), more REDOs (Monty to work on
REDO_UPDATE?), UNDO phase...
Pushing this cset as it looks safe, contains test and bugfixes which
will help Monty implement applying of REDO_UPDATE.
sql/handler.cc:
typo
storage/maria/Makefile.am:
Adding ma_test_recovery (which ma_test_all invokes, and which can
also be run alone). Most of maria_read_log.c moved to ma_recovery.c
storage/maria/ha_maria.cc:
comments
storage/maria/ma_bitmap.c:
fixing comments. 2 -> sizeof(maria_bitmap_marker).
Bitmap-related part of _ma_initialize_datafile() moves in bitmap module.
Now putting the "bm" signature when creating the first bitmap page
(it used to happen only at next open, but that
caused an annoying difference when testing Recovery if the original
run didn't open the table, and it looks more
logical like this: it goes to disk only with its signature correct);
see the "QQ" comment towards the _ma_initialize_data_file() call
in ma_create.c for more).
When reading a bitmap page, verify its signature (happens when normally
using the table or when CHECKing it; not when REPAIRing it).
storage/maria/ma_blockrec.c:
* no need to sync the data file if table is not transactional
* Comments, code cleanup (log-related data moved to log-related code
block, int5store->page_store).
* Store the table's short id into LOGREC_UNDO_ROW_PURGE, like we
do for other records (though this record will soon be replaced
with a CLR).
* If "page" is 1 it means the page which extends from byte
page*block_size+1 to (page+1)*block_size (byte number 1 being
the first byte of the file). The last byte of the file is
data_file_length (same convention).
A new page needs to be created if the last byte of the page is
beyond the last byte of the file, i.e.
(page+1)*block_size+1 > data_file_length, so we correct the test
(bug found when testing log applying for ma_test1 -M -T --skip-update).
* update the page's LSN when removing a row from it during
execution of a REDO_PURGE_ROW record (bug found when testing log
applying for ma_test1 -M -T --skip-update).
* applying of REDO_PURGE_BLOCKs (limited to a one-page range for now).
storage/maria/ma_blockrec.h:
new functions. maria_bitmap_marker does not need to be exported.
storage/maria/ma_close.c:
we can always flush the table's state when closing the last instance
of the table. And it is needed for maria_read_log (as it does
not use maria_lock_database()).
storage/maria/ma_control_file.c:
when in Recovery, some assertions should not be used.
storage/maria/ma_control_file.h:
double-inclusion safe
storage/maria/ma_create.c:
during recovery, don't log records. Comments.
Moving the creation of the first bitmap page to ma_bitmap.c
storage/maria/ma_delete_table.c:
during recovery, don't log records. Log the end-zero of the dropped
table's name, so that recovery can use the string in place without
extending it to fit an end zero.
storage/maria/ma_loghandler.c:
* inwrite_rec_hook also needs access to the MARIA_SHARE, like
prewrite_rec_hook. This will be needed to update
share->records_diff (in the upcoming patch "recovery of the state").
* LOG_DESC::record_ends_group changed to an enum.
* LOG_DESC for LOGREC_REDO_PURGE_BLOCKS and LOGREC_UNDO_ROW_PURGE
corrected
* Sanja please see the @todo LOG BUG
* avoiding DBUG_RETURN(func()) as it gives confusing debug traces.
storage/maria/ma_loghandler.h:
- log write hooks called while the log's lock is held (inwrite_rec_hook)
now need the MARIA_SHARE, like prewrite_rec_hook already had
- instead of a bool saying if this record's type ends groups or not,
we refine: it may not end a group, it may end a group, or it may
be a group in itself. Imagine that we had a physical write failure
to a table before we log the UNDO, we still end up in
external_lock(F_UNLCK) and then we log a COMMIT: we don't want
to consider this COMMIT as ending the group of REDOs (don't want
to execute those REDOs during Recovery), that's why we say "COMMIT
is a group in itself, it aborts any previous group". This also
gives one more sanity check in maria_read_log.
storage/maria/ma_recovery.c:
New Recovery code, replacing the old pseudocode.
Most of maria_read_log moved here.
Call-able from ha_maria, but not enabled yet.
Compared to the previous version of maria_read_log, some bugs have
been fixed, debugging output can go to stdout or a disk file (for now
it's useful for me, later it can be changed), execution of
REDO_DROP_TABLE, REDO_DELETE_ALL, REDO_PURGE_BLOCKS has been added. Duplicate code
has been factored into functions. We abort an unfinished group
of records if we see a record which is a group in itself (like COMMIT).
No need for maria_panic() after a bug (which caused tables to not
be closed) was fixed; if there is yet another bug I prefer to see it.
When opening a table for Recovery, set data_file_length
and key_file_length to their real physical value (these are the
easiest state members to restore :). Warn us if the last page
was truncated (but Recovery handles it).
MARIA_SHARE::state::state::records is now partly recovered (not
idempotent, but works if recreating tables from scracth).
When applying a REDO to a page, stamp it with the UNDO's LSN
(current_group_end_lsn), not with the REDO's LSN; it makes
the table more identical to the original table (easier to compare
the two tables in the end).
Big thing missing: some types of REDOs are not handled,
and the UNDO phase does not exist (missing functions to execute UNDOs
to actually rollback). So for now tests are only inserting/deleting
a few 100 rows, closing the table and seeing if the log is applied ok;
it works. UPDATE not handled.
storage/maria/ma_recovery.h:
new functions: ma_recover() for recovery from inside ha_maria;
_ma_apply_log() for maria_read_log (ma_recover() calls _ma_apply_log()).
Btw, we need to not use the word "recover" for REPAIR/maria_chk anymore.
storage/maria/ma_rename.c:
don't write log records during recovery
storage/maria/ma_test2.c:
- fail if maria_info() or other subtests find some wrong information
- new option -g to skip updates.
- init the translog before creating the table, so that log applying
can work.
- in "#if 0" you'll see some fixed bugs (will be removed).
storage/maria/ma_test_all.sh:
cleanup files. Test log applying.
storage/maria/maria_read_log.c:
most of the logic moves to ma_recovery.c to be shared between
maria_read_log and recovery-from-inside-mysqld.
See ma_recovery.c for additional changes made to the moved code.
storage/maria/ma_test_recovery:
unit test for Recovery. Tests insert and delete,
REDO_UPDATE not yet coded.
Script is called from ma_test_all. Can run standalone.
2007-07-26 11:56:21 +02:00
|
|
|
goto err;
|
2006-04-11 15:45:10 +02:00
|
|
|
}
|
|
|
|
if (maria_extra(file,HA_EXTRA_NO_CACHE,0))
|
|
|
|
{
|
|
|
|
puts("got error from maria_extra(HA_EXTRA_NO_CACHE)");
|
2007-10-04 19:33:42 +02:00
|
|
|
maria_scan_end(file);
|
WL#3072 - Maria recovery
Unit test for recovery: runs ma_test1 and ma_test2 (both only with
INSERTs and DELETEs; UPDATEs disabled as not handled by recovery)
then moves the tables elswhere; recreates tables from the log, and
compares and fails if there is a difference. Passes now.
Most of maria_read_log.c moved to ma_recovery.c, as it will be re-used
for recovery-from-ha_maria.
Bugfixes of applying of REDO_INSERT, REDO_PURGE_ROW.
Applying of REDO_PURGE_BLOCKS, REDO_DELETE_ALL, REDO_DROP_TABLE,
UNDO_ROW_INSERT (in REDO phase only, i.e. just doing records++),
UNDO_ROW_DELETE, UNDO_ROW_PURGE.
Code cleanups.
Monty: please look for "QQ". Sanja: please look for "Sanja".
Future tasks: recovery of the bitmap (easy), recovery of the state
(make it idempotent), more REDOs (Monty to work on
REDO_UPDATE?), UNDO phase...
Pushing this cset as it looks safe, contains test and bugfixes which
will help Monty implement applying of REDO_UPDATE.
sql/handler.cc:
typo
storage/maria/Makefile.am:
Adding ma_test_recovery (which ma_test_all invokes, and which can
also be run alone). Most of maria_read_log.c moved to ma_recovery.c
storage/maria/ha_maria.cc:
comments
storage/maria/ma_bitmap.c:
fixing comments. 2 -> sizeof(maria_bitmap_marker).
Bitmap-related part of _ma_initialize_datafile() moves in bitmap module.
Now putting the "bm" signature when creating the first bitmap page
(it used to happen only at next open, but that
caused an annoying difference when testing Recovery if the original
run didn't open the table, and it looks more
logical like this: it goes to disk only with its signature correct);
see the "QQ" comment towards the _ma_initialize_data_file() call
in ma_create.c for more).
When reading a bitmap page, verify its signature (happens when normally
using the table or when CHECKing it; not when REPAIRing it).
storage/maria/ma_blockrec.c:
* no need to sync the data file if table is not transactional
* Comments, code cleanup (log-related data moved to log-related code
block, int5store->page_store).
* Store the table's short id into LOGREC_UNDO_ROW_PURGE, like we
do for other records (though this record will soon be replaced
with a CLR).
* If "page" is 1 it means the page which extends from byte
page*block_size+1 to (page+1)*block_size (byte number 1 being
the first byte of the file). The last byte of the file is
data_file_length (same convention).
A new page needs to be created if the last byte of the page is
beyond the last byte of the file, i.e.
(page+1)*block_size+1 > data_file_length, so we correct the test
(bug found when testing log applying for ma_test1 -M -T --skip-update).
* update the page's LSN when removing a row from it during
execution of a REDO_PURGE_ROW record (bug found when testing log
applying for ma_test1 -M -T --skip-update).
* applying of REDO_PURGE_BLOCKs (limited to a one-page range for now).
storage/maria/ma_blockrec.h:
new functions. maria_bitmap_marker does not need to be exported.
storage/maria/ma_close.c:
we can always flush the table's state when closing the last instance
of the table. And it is needed for maria_read_log (as it does
not use maria_lock_database()).
storage/maria/ma_control_file.c:
when in Recovery, some assertions should not be used.
storage/maria/ma_control_file.h:
double-inclusion safe
storage/maria/ma_create.c:
during recovery, don't log records. Comments.
Moving the creation of the first bitmap page to ma_bitmap.c
storage/maria/ma_delete_table.c:
during recovery, don't log records. Log the end-zero of the dropped
table's name, so that recovery can use the string in place without
extending it to fit an end zero.
storage/maria/ma_loghandler.c:
* inwrite_rec_hook also needs access to the MARIA_SHARE, like
prewrite_rec_hook. This will be needed to update
share->records_diff (in the upcoming patch "recovery of the state").
* LOG_DESC::record_ends_group changed to an enum.
* LOG_DESC for LOGREC_REDO_PURGE_BLOCKS and LOGREC_UNDO_ROW_PURGE
corrected
* Sanja please see the @todo LOG BUG
* avoiding DBUG_RETURN(func()) as it gives confusing debug traces.
storage/maria/ma_loghandler.h:
- log write hooks called while the log's lock is held (inwrite_rec_hook)
now need the MARIA_SHARE, like prewrite_rec_hook already had
- instead of a bool saying if this record's type ends groups or not,
we refine: it may not end a group, it may end a group, or it may
be a group in itself. Imagine that we had a physical write failure
to a table before we log the UNDO, we still end up in
external_lock(F_UNLCK) and then we log a COMMIT: we don't want
to consider this COMMIT as ending the group of REDOs (don't want
to execute those REDOs during Recovery), that's why we say "COMMIT
is a group in itself, it aborts any previous group". This also
gives one more sanity check in maria_read_log.
storage/maria/ma_recovery.c:
New Recovery code, replacing the old pseudocode.
Most of maria_read_log moved here.
Call-able from ha_maria, but not enabled yet.
Compared to the previous version of maria_read_log, some bugs have
been fixed, debugging output can go to stdout or a disk file (for now
it's useful for me, later it can be changed), execution of
REDO_DROP_TABLE, REDO_DELETE_ALL, REDO_PURGE_BLOCKS has been added. Duplicate code
has been factored into functions. We abort an unfinished group
of records if we see a record which is a group in itself (like COMMIT).
No need for maria_panic() after a bug (which caused tables to not
be closed) was fixed; if there is yet another bug I prefer to see it.
When opening a table for Recovery, set data_file_length
and key_file_length to their real physical value (these are the
easiest state members to restore :). Warn us if the last page
was truncated (but Recovery handles it).
MARIA_SHARE::state::state::records is now partly recovered (not
idempotent, but works if recreating tables from scracth).
When applying a REDO to a page, stamp it with the UNDO's LSN
(current_group_end_lsn), not with the REDO's LSN; it makes
the table more identical to the original table (easier to compare
the two tables in the end).
Big thing missing: some types of REDOs are not handled,
and the UNDO phase does not exist (missing functions to execute UNDOs
to actually rollback). So for now tests are only inserting/deleting
a few 100 rows, closing the table and seeing if the log is applied ok;
it works. UPDATE not handled.
storage/maria/ma_recovery.h:
new functions: ma_recover() for recovery from inside ha_maria;
_ma_apply_log() for maria_read_log (ma_recover() calls _ma_apply_log()).
Btw, we need to not use the word "recover" for REPAIR/maria_chk anymore.
storage/maria/ma_rename.c:
don't write log records during recovery
storage/maria/ma_test2.c:
- fail if maria_info() or other subtests find some wrong information
- new option -g to skip updates.
- init the translog before creating the table, so that log applying
can work.
- in "#if 0" you'll see some fixed bugs (will be removed).
storage/maria/ma_test_all.sh:
cleanup files. Test log applying.
storage/maria/maria_read_log.c:
most of the logic moves to ma_recovery.c to be shared between
maria_read_log and recovery-from-inside-mysqld.
See ma_recovery.c for additional changes made to the moved code.
storage/maria/ma_test_recovery:
unit test for Recovery. Tests insert and delete,
REDO_UPDATE not yet coded.
Script is called from ma_test_all. Can run standalone.
2007-07-26 11:56:21 +02:00
|
|
|
goto err;
|
2006-04-11 15:45:10 +02:00
|
|
|
}
|
2007-10-04 19:33:42 +02:00
|
|
|
maria_scan_end(file);
|
2006-04-11 15:45:10 +02:00
|
|
|
|
|
|
|
ant=0;
|
|
|
|
maria_scan_init(file);
|
|
|
|
while ((error=maria_scan(file,record)) != HA_ERR_END_OF_FILE &&
|
|
|
|
ant < write_count + 10)
|
|
|
|
ant+= error ? 0 : 1;
|
|
|
|
if (ant != write_count-opt_delete)
|
|
|
|
{
|
|
|
|
printf("scan with cache: I can only find: %d records of %d\n",
|
|
|
|
ant,write_count-opt_delete);
|
2007-10-04 19:33:42 +02:00
|
|
|
maria_scan_end(file);
|
WL#3072 - Maria recovery
Unit test for recovery: runs ma_test1 and ma_test2 (both only with
INSERTs and DELETEs; UPDATEs disabled as not handled by recovery)
then moves the tables elswhere; recreates tables from the log, and
compares and fails if there is a difference. Passes now.
Most of maria_read_log.c moved to ma_recovery.c, as it will be re-used
for recovery-from-ha_maria.
Bugfixes of applying of REDO_INSERT, REDO_PURGE_ROW.
Applying of REDO_PURGE_BLOCKS, REDO_DELETE_ALL, REDO_DROP_TABLE,
UNDO_ROW_INSERT (in REDO phase only, i.e. just doing records++),
UNDO_ROW_DELETE, UNDO_ROW_PURGE.
Code cleanups.
Monty: please look for "QQ". Sanja: please look for "Sanja".
Future tasks: recovery of the bitmap (easy), recovery of the state
(make it idempotent), more REDOs (Monty to work on
REDO_UPDATE?), UNDO phase...
Pushing this cset as it looks safe, contains test and bugfixes which
will help Monty implement applying of REDO_UPDATE.
sql/handler.cc:
typo
storage/maria/Makefile.am:
Adding ma_test_recovery (which ma_test_all invokes, and which can
also be run alone). Most of maria_read_log.c moved to ma_recovery.c
storage/maria/ha_maria.cc:
comments
storage/maria/ma_bitmap.c:
fixing comments. 2 -> sizeof(maria_bitmap_marker).
Bitmap-related part of _ma_initialize_datafile() moves in bitmap module.
Now putting the "bm" signature when creating the first bitmap page
(it used to happen only at next open, but that
caused an annoying difference when testing Recovery if the original
run didn't open the table, and it looks more
logical like this: it goes to disk only with its signature correct);
see the "QQ" comment towards the _ma_initialize_data_file() call
in ma_create.c for more).
When reading a bitmap page, verify its signature (happens when normally
using the table or when CHECKing it; not when REPAIRing it).
storage/maria/ma_blockrec.c:
* no need to sync the data file if table is not transactional
* Comments, code cleanup (log-related data moved to log-related code
block, int5store->page_store).
* Store the table's short id into LOGREC_UNDO_ROW_PURGE, like we
do for other records (though this record will soon be replaced
with a CLR).
* If "page" is 1 it means the page which extends from byte
page*block_size+1 to (page+1)*block_size (byte number 1 being
the first byte of the file). The last byte of the file is
data_file_length (same convention).
A new page needs to be created if the last byte of the page is
beyond the last byte of the file, i.e.
(page+1)*block_size+1 > data_file_length, so we correct the test
(bug found when testing log applying for ma_test1 -M -T --skip-update).
* update the page's LSN when removing a row from it during
execution of a REDO_PURGE_ROW record (bug found when testing log
applying for ma_test1 -M -T --skip-update).
* applying of REDO_PURGE_BLOCKs (limited to a one-page range for now).
storage/maria/ma_blockrec.h:
new functions. maria_bitmap_marker does not need to be exported.
storage/maria/ma_close.c:
we can always flush the table's state when closing the last instance
of the table. And it is needed for maria_read_log (as it does
not use maria_lock_database()).
storage/maria/ma_control_file.c:
when in Recovery, some assertions should not be used.
storage/maria/ma_control_file.h:
double-inclusion safe
storage/maria/ma_create.c:
during recovery, don't log records. Comments.
Moving the creation of the first bitmap page to ma_bitmap.c
storage/maria/ma_delete_table.c:
during recovery, don't log records. Log the end-zero of the dropped
table's name, so that recovery can use the string in place without
extending it to fit an end zero.
storage/maria/ma_loghandler.c:
* inwrite_rec_hook also needs access to the MARIA_SHARE, like
prewrite_rec_hook. This will be needed to update
share->records_diff (in the upcoming patch "recovery of the state").
* LOG_DESC::record_ends_group changed to an enum.
* LOG_DESC for LOGREC_REDO_PURGE_BLOCKS and LOGREC_UNDO_ROW_PURGE
corrected
* Sanja please see the @todo LOG BUG
* avoiding DBUG_RETURN(func()) as it gives confusing debug traces.
storage/maria/ma_loghandler.h:
- log write hooks called while the log's lock is held (inwrite_rec_hook)
now need the MARIA_SHARE, like prewrite_rec_hook already had
- instead of a bool saying if this record's type ends groups or not,
we refine: it may not end a group, it may end a group, or it may
be a group in itself. Imagine that we had a physical write failure
to a table before we log the UNDO, we still end up in
external_lock(F_UNLCK) and then we log a COMMIT: we don't want
to consider this COMMIT as ending the group of REDOs (don't want
to execute those REDOs during Recovery), that's why we say "COMMIT
is a group in itself, it aborts any previous group". This also
gives one more sanity check in maria_read_log.
storage/maria/ma_recovery.c:
New Recovery code, replacing the old pseudocode.
Most of maria_read_log moved here.
Call-able from ha_maria, but not enabled yet.
Compared to the previous version of maria_read_log, some bugs have
been fixed, debugging output can go to stdout or a disk file (for now
it's useful for me, later it can be changed), execution of
REDO_DROP_TABLE, REDO_DELETE_ALL, REDO_PURGE_BLOCKS has been added. Duplicate code
has been factored into functions. We abort an unfinished group
of records if we see a record which is a group in itself (like COMMIT).
No need for maria_panic() after a bug (which caused tables to not
be closed) was fixed; if there is yet another bug I prefer to see it.
When opening a table for Recovery, set data_file_length
and key_file_length to their real physical value (these are the
easiest state members to restore :). Warn us if the last page
was truncated (but Recovery handles it).
MARIA_SHARE::state::state::records is now partly recovered (not
idempotent, but works if recreating tables from scracth).
When applying a REDO to a page, stamp it with the UNDO's LSN
(current_group_end_lsn), not with the REDO's LSN; it makes
the table more identical to the original table (easier to compare
the two tables in the end).
Big thing missing: some types of REDOs are not handled,
and the UNDO phase does not exist (missing functions to execute UNDOs
to actually rollback). So for now tests are only inserting/deleting
a few 100 rows, closing the table and seeing if the log is applied ok;
it works. UPDATE not handled.
storage/maria/ma_recovery.h:
new functions: ma_recover() for recovery from inside ha_maria;
_ma_apply_log() for maria_read_log (ma_recover() calls _ma_apply_log()).
Btw, we need to not use the word "recover" for REPAIR/maria_chk anymore.
storage/maria/ma_rename.c:
don't write log records during recovery
storage/maria/ma_test2.c:
- fail if maria_info() or other subtests find some wrong information
- new option -g to skip updates.
- init the translog before creating the table, so that log applying
can work.
- in "#if 0" you'll see some fixed bugs (will be removed).
storage/maria/ma_test_all.sh:
cleanup files. Test log applying.
storage/maria/maria_read_log.c:
most of the logic moves to ma_recovery.c to be shared between
maria_read_log and recovery-from-inside-mysqld.
See ma_recovery.c for additional changes made to the moved code.
storage/maria/ma_test_recovery:
unit test for Recovery. Tests insert and delete,
REDO_UPDATE not yet coded.
Script is called from ma_test_all. Can run standalone.
2007-07-26 11:56:21 +02:00
|
|
|
goto err;
|
2006-04-11 15:45:10 +02:00
|
|
|
}
|
2007-10-04 19:33:42 +02:00
|
|
|
maria_scan_end(file);
|
2006-04-11 15:45:10 +02:00
|
|
|
|
WL#3071 Maria checkpoint
Finally this is the real checkpoint code.
It however exhibits unstabilities when a checkpoint runs concurrently
with data-modifying clients (table corruption, transaction log's
assertions) so for now a checkpoint is taken only at startup after
recovery and at shutdown, i.e. not in concurrent situations. Later
we will let it run periodically, as well as flush dirty pages
periodically (almost all needed code is there already, only pagecache
code is written but not committed).
WL#3072 Maria recovery
* replacing UNDO_ROW_PURGE with CLR_END; testing of those CLR_END via
ma_test2 which has INSERTs failing with duplicate keys.
* replaying of REDO_RENAME_TABLE
Now, off to test Recovery in ha_maria :)
BitKeeper/deleted/.del-ma_least_recently_dirtied.c:
Delete: storage/maria/ma_least_recently_dirtied.c
BitKeeper/deleted/.del-ma_least_recently_dirtied.h:
Delete: storage/maria/ma_least_recently_dirtied.h
storage/maria/Makefile.am:
compile Checkpoint module
storage/maria/ha_maria.cc:
When ha_maria starts, do a recovery from last checkpoint.
Take a checkpoint when that recovery has ended and when ha_maria
shuts down cleanly.
storage/maria/ma_blockrec.c:
* even if my_sync() fails we have to my_close() (otherwise we leak
a descriptor)
* UNDO_ROW_PURGE is replaced by a simple CLR_END for UNDO_ROW_INSERT,
as promised in the old comment; it gives us skipping during the
UNDO phase.
storage/maria/ma_check.c:
All REDOs before create_rename_lsn are ignored by Recovery. So
create_rename_lsn must be set only after all data/index has been
flushed and forced to disk. We thus move write_log_record_for_repair()
to after _ma_flush_tables_files_after_repair().
storage/maria/ma_checkpoint.c:
Checkpoint module.
storage/maria/ma_checkpoint.h:
optional argument if caller wants a thread to periodically take
checkpoints and flush dirty pages.
storage/maria/ma_create.c:
* no need to init some vars as the initial bzero(share) takes care of this.
* update to new function's name
* even if we fail in my_sync() we have to my_close()
storage/maria/ma_extra.c:
Checkpoint reads share->last_version under intern_lock, so we make
maria_extra() update it under intern_lock. THR_LOCK_maria still needed
because of _ma_test_if_reopen().
storage/maria/ma_init.c:
destroy checkpoint module when Maria shuts down.
storage/maria/ma_loghandler.c:
* UNDO_ROW_PURGE gone (see ma_blockrec.c)
* we need to remember the LSN of the LOGREC_FILE_ID for a share,
because this LSN is needed into the checkpoint record (Recovery wants
to know the validity domain of an id->name mapping)
* translog_get_horizon_no_lock() needed for Checkpoint
* comment about failing assertion (Sanja knows)
* translog_init_reader_data() thought that translog_read_record_header_scan()
returns 0 in case of error, but 0 just means "0-length header".
* translog_assign_id_to_share() now needs the MARIA_HA because
LOGREC_FILE_ID uses a log-write hook.
* Verify that (de)assignment of share->id happens only under intern_lock,
as Checkpoint reads this id with intern_lock.
* translog_purge() can accept TRANSLOG_ADDRESS, not necessarily
a real LSN.
storage/maria/ma_loghandler.h:
prototype updates
storage/maria/ma_open.c:
no need to initialize "res"
storage/maria/ma_pagecache.c:
When taking a checkpoint, we don't need to know the maximum rec_lsn
of dirty pages; this LSN was intended to be used in the two-checkpoint
rule, but last_checkpoint_lsn is as good.
4 bytes for stored_list_size is enough as PAGECACHE::blocks (number
of blocks which the pagecache can contain) is int.
storage/maria/ma_pagecache.h:
new prototype
storage/maria/ma_recovery.c:
* added replaying of REDO_RENAME_TABLE
* UNDO_ROW_PURGE gone (see ma_blockrec.c), replaced by CLR_END
* Recovery from the last checkpoint record now possible
* In new_table() we skip the table if the id->name mapping is older than
create_rename_lsn (mapping dates from lsn_of_file_id).
* in get_MARIA_HA_from_REDO_record() we skip the record
if the id->name mapping is newer than the record (can happen if processing
a record which is before the checkpoint record).
* parse_checkpoint_record() has to return a LSN, that's what caller expects
storage/maria/ma_rename.c:
new function's name; log end zeroes of tables' names (ease recovery)
storage/maria/ma_test2.c:
* equivalent of ma_test1's --test-undo added (named -u here).
* -t=1 now stops right after creating the table, so that
we can test undoing of INSERTs with duplicate keys (which tests the
CLR_END logged by _ma_write_abort_block_record()).
storage/maria/ma_test_recovery.expected:
Result of testing undoing of INSERTs with duplicate keys; there are
some differences in maria_chk -dvv but they are normal (removing
records does not shrink data/index file, does not put back the
"analyzed, optimized keys"(etc) index state.
storage/maria/ma_test_recovery:
Test undoing of INSERTs with duplicate keys, using ma_test2;
when such INSERT happens, it logs REDO_INSERT, UNDO_INSERT, REDO_DELETE,
CLR_END; we abort after that, and test that CLR_END causes recovery
to jump over UNDO_INSERT.
storage/maria/ma_write.c:
comment
storage/maria/maria_chk.c:
comment
storage/maria/maria_def.h:
* a new bit in MARIA_SHARE::in_checkpoint, used to build a list
of unique shares during Checkpoint.
* MARIA_SHARE::lsn_of_file_id added: the LSN of the last LOGREC_FILE_ID
for this share; needed to know to which LSN domain the mappings
found in the Checkpoint record apply (new mappings should not apply
to old REDOs).
storage/maria/trnman.c:
* small changes to how trnman_collect_transactions() fills its buffer;
it also uses a non-dummy lsn_read_non_atomic() found in ma_checkpoint.h
2007-09-12 11:27:34 +02:00
|
|
|
if (testflag == 5)
|
2007-01-18 20:38:14 +01:00
|
|
|
goto end;
|
WL#3071 Maria checkpoint
Ability for flush_pagecache_blocks() to flush only certain pages of
a file, as instructed by an option "filter" pointer-to-function argument;
Checkpoint and background dirty page flushing use that to flush only
pages which have been dirty for long enough and bitmap pages.
Fix for a bug in flush_cached_blocks() (no idea if it could produce
a bug in real life, but theoretically it is).
Testing checkpoint in ma_test_recovery via ma_test1 and ma_test2.
Background checkpoint & dirty pages flush thread is still disabled
by default in ha_maria.
mysql-test/r/maria.result:
result update
storage/maria/ha_maria.cc:
blank after function comment
storage/maria/ma_checkpoint.c:
Using an enum instead of 0/1/2 (applying Sanja's review comments).
The comment about "this is an horizon" can be removed as Sanja
created translog_next_LSN() which parse_checkpoint_record() uses.
Variables in ma_checkpoint_background() cannot be declared in the
for() as their value must not be reset at each iteration!
storage/maria/ma_pagecache.c:
adding to flush_pagecache_blocks() optional arguments 'filter'
(pointer to function) and 'filter_arg'; if filter!=NULL this function
will be called for each block of the file and will reply if this
block and following ones should be flushed or not (3 possible
replies).
Fixing a bug when flush_cached_blocks() skips a pinned page: it has
to unset PCBLOCK_IN_FLUSH set by flush_pagecache_blocks_int().
storage/maria/ma_pagecache.h:
flush_pagecache_blocks() is changed to take "filter" and "filter_arg"
arguments. "filter", if it is not NULL, may return one value
among enum pagecache_flush_filter_result.
storage/maria/ma_recovery.c:
open_count=0 when closing tables at the end of recovery.
storage/maria/ma_test1.c:
Optional checkpoints (-H#) at various stages (stages similar
to --testflag), for testing of checkpoints.
storage/maria/ma_test2.c:
Optional checkpoints (-H#) at various stages (stages similar
to -t), for testing of checkpoints.
storage/maria/ma_test_recovery.expected:
Result update: the results of the additional test run with -H#
(checkpoints) are added here. They are exactly identical to without
checkpoints except that the index's Root (printed by maria_chk)
is more correct when using checkpoints. This is because checkpoint
flushed the state, so it happens to be correct, while no-checkpoint
does not flush the state, and recovery does not recover indexes
so Root is never fixed. When we recover indices, this will go away.
storage/maria/ma_test_recovery:
We duplicate the loop of tests to add an additional run with
checkpoints at various stages, to see if maria_read_log
uses them fine.
2007-10-17 16:55:26 +02:00
|
|
|
if (checkpoint == 5 && ma_checkpoint_execute(CHECKPOINT_MEDIUM, FALSE))
|
|
|
|
goto err;
|
2006-04-11 15:45:10 +02:00
|
|
|
|
|
|
|
if (!silent)
|
|
|
|
printf("- Removing keys\n");
|
|
|
|
DBUG_PRINT("progpos",("Removing keys"));
|
|
|
|
lastpos = HA_OFFSET_ERROR;
|
|
|
|
/* DBUG_POP(); */
|
Completion of merge of mysql-5.1 into mysql-maria.
Manually imported changes done to MyISAM (include/myisam.h,
storage/myisam/*, sql/ha_myisam.*, mysql-test/t/myisam.test,
mysql-test/t/ps_2myisam.test) the last
months into Maria (tedious, should do it more frequently in the
future), including those not done at the previous 5.1->Maria merge
(please in the future don't forget to apply MyISAM changes to Maria
when you merge 5.1 into Maria).
Note: I didn't try to import anything which could be MyISAM-related
in other tests of mysql-test (I didn't want to dig in all csets),
but as QA is working to make most tests re-usable for other engines
(Falcon), it is likely that we'll benefit from this and just have
to set engine=Maria somewhere to run those tests on Maria.
func_group and partition tests fail but they already do in main 5.1
on my machine. No Valgrind error in t/*maria*.test.
Monty: please see the commit comment of maria.result and check.
BitKeeper/deleted/.del-ha_maria.m4:
Delete: config/ac-macros/ha_maria.m4
configure.in:
fix for the new way of enabling engines
include/maria.h:
importing changes done to MyISAM the last months into Maria
include/my_handler.h:
importing changes done to MyISAM the last months into Maria
include/myisam.h:
importing changes done to MyISAM the last months into Maria
mysql-test/r/maria.result:
identical to myisam.result, except the engine name in some places
AND in the line testing key_block_size=1000000000000000000:
Maria gives a key block size of 8192 while MyISAM gives 4096;
is it explainable by the difference between MARIA_KEY_BLOCK_LENGTH
and the same constant in MyISAM? Monty?
mysql-test/r/ps_maria.result:
identical to ps_2myisam.result (except the engine name in some places)
mysql-test/t/maria.test:
instead of engine=maria everywhere, I use @@storage_engine (reduces
the diff with myisam.test).
importing changes done to MyISAM the last months into Maria
mysys/my_handler.c:
importing changes done to MyISAM the last months into Maria
sql/ha_maria.cc:
importing changes done to MyISAM the last months into Maria
sql/ha_maria.h:
importing changes done to MyISAM the last months into Maria
sql/mysqld.cc:
unneeded
storage/maria/Makefile.am:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_check.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_create.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_delete_table.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_dynrec.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_extra.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_ft_boolean_search.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_ft_eval.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_ft_nlq_search.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_ft_parser.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_ft_test1.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_ft_update.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_ftdefs.h:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_key.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_open.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_page.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_rkey.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_rsamepos.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_rt_index.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_rt_mbr.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_search.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_sort.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_test1.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_test2.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_test3.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_update.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_write.c:
importing changes done to MyISAM the last months into Maria
storage/maria/maria_chk.c:
importing changes done to MyISAM the last months into Maria
storage/maria/maria_def.h:
importing changes done to MyISAM the last months into Maria
storage/maria/maria_ftdump.c:
importing changes done to MyISAM the last months into Maria
storage/maria/maria_pack.c:
importing changes done to MyISAM the last months into Maria
2006-08-10 16:36:54 +02:00
|
|
|
maria_reset(file);
|
2006-04-11 15:45:10 +02:00
|
|
|
found_parts=0;
|
2007-01-18 20:38:14 +01:00
|
|
|
maria_scan_init(file);
|
|
|
|
while ((error= maria_scan(file,read_record)) != HA_ERR_END_OF_FILE)
|
2006-04-11 15:45:10 +02:00
|
|
|
{
|
|
|
|
info.recpos=maria_position(file);
|
|
|
|
if (lastpos >= info.recpos && lastpos != HA_OFFSET_ERROR)
|
|
|
|
{
|
|
|
|
printf("maria_rrnd didn't advance filepointer; old: %ld, new: %ld\n",
|
|
|
|
(long) lastpos, (long) info.recpos);
|
|
|
|
goto err;
|
|
|
|
}
|
|
|
|
lastpos=info.recpos;
|
|
|
|
if (error == 0)
|
|
|
|
{
|
|
|
|
if (opt_delete == (uint) remove_count) /* While testing */
|
|
|
|
goto end;
|
2007-01-18 20:38:14 +01:00
|
|
|
if (rnd(2) == 1 && maria_rsame(file,read_record,-1))
|
2006-04-11 15:45:10 +02:00
|
|
|
{
|
|
|
|
printf("can't find record %lx\n",(long) info.recpos);
|
|
|
|
goto err;
|
|
|
|
}
|
|
|
|
if (use_blob)
|
|
|
|
{
|
|
|
|
ulong blob_length,pos;
|
|
|
|
uchar *ptr;
|
2007-04-05 13:38:05 +02:00
|
|
|
memcpy_fixed(&ptr, read_record+blob_pos+4, sizeof(ptr));
|
2006-04-11 15:45:10 +02:00
|
|
|
longget(blob_length,read_record+blob_pos);
|
|
|
|
for (pos=0 ; pos < blob_length ; pos++)
|
|
|
|
{
|
|
|
|
if (ptr[pos] != (uchar) (blob_length+pos))
|
|
|
|
{
|
2007-01-18 20:38:14 +01:00
|
|
|
printf("Found blob with wrong info at %ld\n",(long) lastpos);
|
|
|
|
maria_scan_end(file);
|
|
|
|
my_errno= 0;
|
|
|
|
goto err;
|
2006-04-11 15:45:10 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if (maria_delete(file,read_record))
|
|
|
|
{
|
|
|
|
printf("can't delete record: %6.6s, delete_count: %d\n",
|
|
|
|
read_record, opt_delete);
|
2007-01-18 20:38:14 +01:00
|
|
|
maria_scan_end(file);
|
2006-04-11 15:45:10 +02:00
|
|
|
goto err;
|
|
|
|
}
|
|
|
|
opt_delete++;
|
WL#3072 - Maria recovery
Unit test for recovery: runs ma_test1 and ma_test2 (both only with
INSERTs and DELETEs; UPDATEs disabled as not handled by recovery)
then moves the tables elswhere; recreates tables from the log, and
compares and fails if there is a difference. Passes now.
Most of maria_read_log.c moved to ma_recovery.c, as it will be re-used
for recovery-from-ha_maria.
Bugfixes of applying of REDO_INSERT, REDO_PURGE_ROW.
Applying of REDO_PURGE_BLOCKS, REDO_DELETE_ALL, REDO_DROP_TABLE,
UNDO_ROW_INSERT (in REDO phase only, i.e. just doing records++),
UNDO_ROW_DELETE, UNDO_ROW_PURGE.
Code cleanups.
Monty: please look for "QQ". Sanja: please look for "Sanja".
Future tasks: recovery of the bitmap (easy), recovery of the state
(make it idempotent), more REDOs (Monty to work on
REDO_UPDATE?), UNDO phase...
Pushing this cset as it looks safe, contains test and bugfixes which
will help Monty implement applying of REDO_UPDATE.
sql/handler.cc:
typo
storage/maria/Makefile.am:
Adding ma_test_recovery (which ma_test_all invokes, and which can
also be run alone). Most of maria_read_log.c moved to ma_recovery.c
storage/maria/ha_maria.cc:
comments
storage/maria/ma_bitmap.c:
fixing comments. 2 -> sizeof(maria_bitmap_marker).
Bitmap-related part of _ma_initialize_datafile() moves in bitmap module.
Now putting the "bm" signature when creating the first bitmap page
(it used to happen only at next open, but that
caused an annoying difference when testing Recovery if the original
run didn't open the table, and it looks more
logical like this: it goes to disk only with its signature correct);
see the "QQ" comment towards the _ma_initialize_data_file() call
in ma_create.c for more).
When reading a bitmap page, verify its signature (happens when normally
using the table or when CHECKing it; not when REPAIRing it).
storage/maria/ma_blockrec.c:
* no need to sync the data file if table is not transactional
* Comments, code cleanup (log-related data moved to log-related code
block, int5store->page_store).
* Store the table's short id into LOGREC_UNDO_ROW_PURGE, like we
do for other records (though this record will soon be replaced
with a CLR).
* If "page" is 1 it means the page which extends from byte
page*block_size+1 to (page+1)*block_size (byte number 1 being
the first byte of the file). The last byte of the file is
data_file_length (same convention).
A new page needs to be created if the last byte of the page is
beyond the last byte of the file, i.e.
(page+1)*block_size+1 > data_file_length, so we correct the test
(bug found when testing log applying for ma_test1 -M -T --skip-update).
* update the page's LSN when removing a row from it during
execution of a REDO_PURGE_ROW record (bug found when testing log
applying for ma_test1 -M -T --skip-update).
* applying of REDO_PURGE_BLOCKs (limited to a one-page range for now).
storage/maria/ma_blockrec.h:
new functions. maria_bitmap_marker does not need to be exported.
storage/maria/ma_close.c:
we can always flush the table's state when closing the last instance
of the table. And it is needed for maria_read_log (as it does
not use maria_lock_database()).
storage/maria/ma_control_file.c:
when in Recovery, some assertions should not be used.
storage/maria/ma_control_file.h:
double-inclusion safe
storage/maria/ma_create.c:
during recovery, don't log records. Comments.
Moving the creation of the first bitmap page to ma_bitmap.c
storage/maria/ma_delete_table.c:
during recovery, don't log records. Log the end-zero of the dropped
table's name, so that recovery can use the string in place without
extending it to fit an end zero.
storage/maria/ma_loghandler.c:
* inwrite_rec_hook also needs access to the MARIA_SHARE, like
prewrite_rec_hook. This will be needed to update
share->records_diff (in the upcoming patch "recovery of the state").
* LOG_DESC::record_ends_group changed to an enum.
* LOG_DESC for LOGREC_REDO_PURGE_BLOCKS and LOGREC_UNDO_ROW_PURGE
corrected
* Sanja please see the @todo LOG BUG
* avoiding DBUG_RETURN(func()) as it gives confusing debug traces.
storage/maria/ma_loghandler.h:
- log write hooks called while the log's lock is held (inwrite_rec_hook)
now need the MARIA_SHARE, like prewrite_rec_hook already had
- instead of a bool saying if this record's type ends groups or not,
we refine: it may not end a group, it may end a group, or it may
be a group in itself. Imagine that we had a physical write failure
to a table before we log the UNDO, we still end up in
external_lock(F_UNLCK) and then we log a COMMIT: we don't want
to consider this COMMIT as ending the group of REDOs (don't want
to execute those REDOs during Recovery), that's why we say "COMMIT
is a group in itself, it aborts any previous group". This also
gives one more sanity check in maria_read_log.
storage/maria/ma_recovery.c:
New Recovery code, replacing the old pseudocode.
Most of maria_read_log moved here.
Call-able from ha_maria, but not enabled yet.
Compared to the previous version of maria_read_log, some bugs have
been fixed, debugging output can go to stdout or a disk file (for now
it's useful for me, later it can be changed), execution of
REDO_DROP_TABLE, REDO_DELETE_ALL, REDO_PURGE_BLOCKS has been added. Duplicate code
has been factored into functions. We abort an unfinished group
of records if we see a record which is a group in itself (like COMMIT).
No need for maria_panic() after a bug (which caused tables to not
be closed) was fixed; if there is yet another bug I prefer to see it.
When opening a table for Recovery, set data_file_length
and key_file_length to their real physical value (these are the
easiest state members to restore :). Warn us if the last page
was truncated (but Recovery handles it).
MARIA_SHARE::state::state::records is now partly recovered (not
idempotent, but works if recreating tables from scracth).
When applying a REDO to a page, stamp it with the UNDO's LSN
(current_group_end_lsn), not with the REDO's LSN; it makes
the table more identical to the original table (easier to compare
the two tables in the end).
Big thing missing: some types of REDOs are not handled,
and the UNDO phase does not exist (missing functions to execute UNDOs
to actually rollback). So for now tests are only inserting/deleting
a few 100 rows, closing the table and seeing if the log is applied ok;
it works. UPDATE not handled.
storage/maria/ma_recovery.h:
new functions: ma_recover() for recovery from inside ha_maria;
_ma_apply_log() for maria_read_log (ma_recover() calls _ma_apply_log()).
Btw, we need to not use the word "recover" for REPAIR/maria_chk anymore.
storage/maria/ma_rename.c:
don't write log records during recovery
storage/maria/ma_test2.c:
- fail if maria_info() or other subtests find some wrong information
- new option -g to skip updates.
- init the translog before creating the table, so that log applying
can work.
- in "#if 0" you'll see some fixed bugs (will be removed).
storage/maria/ma_test_all.sh:
cleanup files. Test log applying.
storage/maria/maria_read_log.c:
most of the logic moves to ma_recovery.c to be shared between
maria_read_log and recovery-from-inside-mysqld.
See ma_recovery.c for additional changes made to the moved code.
storage/maria/ma_test_recovery:
unit test for Recovery. Tests insert and delete,
REDO_UPDATE not yet coded.
Script is called from ma_test_all. Can run standalone.
2007-07-26 11:56:21 +02:00
|
|
|
#if 0
|
|
|
|
/
|
|
|
|
/*
|
|
|
|
179 is ok, 180 causes a difference between runtime and log-applying.
|
|
|
|
This is now fixed (we zero the last directory entry during
|
|
|
|
log-applying, just to eliminate this irrelevant difference).
|
|
|
|
*/
|
|
|
|
if (opt_delete==180) goto end;
|
|
|
|
#endif
|
2006-04-11 15:45:10 +02:00
|
|
|
}
|
|
|
|
else
|
|
|
|
found_parts++;
|
|
|
|
}
|
|
|
|
if (my_errno != HA_ERR_END_OF_FILE && my_errno != HA_ERR_RECORD_DELETED)
|
|
|
|
printf("error: %d from maria_rrnd\n",my_errno);
|
|
|
|
if (write_count != opt_delete)
|
|
|
|
{
|
|
|
|
printf("Deleted only %d of %d records (%d parts)\n",opt_delete,write_count,
|
|
|
|
found_parts);
|
2007-10-04 19:33:42 +02:00
|
|
|
maria_scan_end(file);
|
2006-04-11 15:45:10 +02:00
|
|
|
goto err;
|
|
|
|
}
|
|
|
|
end:
|
2007-10-04 19:33:42 +02:00
|
|
|
maria_scan_end(file);
|
WL#3071 Maria checkpoint
Finally this is the real checkpoint code.
It however exhibits unstabilities when a checkpoint runs concurrently
with data-modifying clients (table corruption, transaction log's
assertions) so for now a checkpoint is taken only at startup after
recovery and at shutdown, i.e. not in concurrent situations. Later
we will let it run periodically, as well as flush dirty pages
periodically (almost all needed code is there already, only pagecache
code is written but not committed).
WL#3072 Maria recovery
* replacing UNDO_ROW_PURGE with CLR_END; testing of those CLR_END via
ma_test2 which has INSERTs failing with duplicate keys.
* replaying of REDO_RENAME_TABLE
Now, off to test Recovery in ha_maria :)
BitKeeper/deleted/.del-ma_least_recently_dirtied.c:
Delete: storage/maria/ma_least_recently_dirtied.c
BitKeeper/deleted/.del-ma_least_recently_dirtied.h:
Delete: storage/maria/ma_least_recently_dirtied.h
storage/maria/Makefile.am:
compile Checkpoint module
storage/maria/ha_maria.cc:
When ha_maria starts, do a recovery from last checkpoint.
Take a checkpoint when that recovery has ended and when ha_maria
shuts down cleanly.
storage/maria/ma_blockrec.c:
* even if my_sync() fails we have to my_close() (otherwise we leak
a descriptor)
* UNDO_ROW_PURGE is replaced by a simple CLR_END for UNDO_ROW_INSERT,
as promised in the old comment; it gives us skipping during the
UNDO phase.
storage/maria/ma_check.c:
All REDOs before create_rename_lsn are ignored by Recovery. So
create_rename_lsn must be set only after all data/index has been
flushed and forced to disk. We thus move write_log_record_for_repair()
to after _ma_flush_tables_files_after_repair().
storage/maria/ma_checkpoint.c:
Checkpoint module.
storage/maria/ma_checkpoint.h:
optional argument if caller wants a thread to periodically take
checkpoints and flush dirty pages.
storage/maria/ma_create.c:
* no need to init some vars as the initial bzero(share) takes care of this.
* update to new function's name
* even if we fail in my_sync() we have to my_close()
storage/maria/ma_extra.c:
Checkpoint reads share->last_version under intern_lock, so we make
maria_extra() update it under intern_lock. THR_LOCK_maria still needed
because of _ma_test_if_reopen().
storage/maria/ma_init.c:
destroy checkpoint module when Maria shuts down.
storage/maria/ma_loghandler.c:
* UNDO_ROW_PURGE gone (see ma_blockrec.c)
* we need to remember the LSN of the LOGREC_FILE_ID for a share,
because this LSN is needed into the checkpoint record (Recovery wants
to know the validity domain of an id->name mapping)
* translog_get_horizon_no_lock() needed for Checkpoint
* comment about failing assertion (Sanja knows)
* translog_init_reader_data() thought that translog_read_record_header_scan()
returns 0 in case of error, but 0 just means "0-length header".
* translog_assign_id_to_share() now needs the MARIA_HA because
LOGREC_FILE_ID uses a log-write hook.
* Verify that (de)assignment of share->id happens only under intern_lock,
as Checkpoint reads this id with intern_lock.
* translog_purge() can accept TRANSLOG_ADDRESS, not necessarily
a real LSN.
storage/maria/ma_loghandler.h:
prototype updates
storage/maria/ma_open.c:
no need to initialize "res"
storage/maria/ma_pagecache.c:
When taking a checkpoint, we don't need to know the maximum rec_lsn
of dirty pages; this LSN was intended to be used in the two-checkpoint
rule, but last_checkpoint_lsn is as good.
4 bytes for stored_list_size is enough as PAGECACHE::blocks (number
of blocks which the pagecache can contain) is int.
storage/maria/ma_pagecache.h:
new prototype
storage/maria/ma_recovery.c:
* added replaying of REDO_RENAME_TABLE
* UNDO_ROW_PURGE gone (see ma_blockrec.c), replaced by CLR_END
* Recovery from the last checkpoint record now possible
* In new_table() we skip the table if the id->name mapping is older than
create_rename_lsn (mapping dates from lsn_of_file_id).
* in get_MARIA_HA_from_REDO_record() we skip the record
if the id->name mapping is newer than the record (can happen if processing
a record which is before the checkpoint record).
* parse_checkpoint_record() has to return a LSN, that's what caller expects
storage/maria/ma_rename.c:
new function's name; log end zeroes of tables' names (ease recovery)
storage/maria/ma_test2.c:
* equivalent of ma_test1's --test-undo added (named -u here).
* -t=1 now stops right after creating the table, so that
we can test undoing of INSERTs with duplicate keys (which tests the
CLR_END logged by _ma_write_abort_block_record()).
storage/maria/ma_test_recovery.expected:
Result of testing undoing of INSERTs with duplicate keys; there are
some differences in maria_chk -dvv but they are normal (removing
records does not shrink data/index file, does not put back the
"analyzed, optimized keys"(etc) index state.
storage/maria/ma_test_recovery:
Test undoing of INSERTs with duplicate keys, using ma_test2;
when such INSERT happens, it logs REDO_INSERT, UNDO_INSERT, REDO_DELETE,
CLR_END; we abort after that, and test that CLR_END causes recovery
to jump over UNDO_INSERT.
storage/maria/ma_write.c:
comment
storage/maria/maria_chk.c:
comment
storage/maria/maria_def.h:
* a new bit in MARIA_SHARE::in_checkpoint, used to build a list
of unique shares during Checkpoint.
* MARIA_SHARE::lsn_of_file_id added: the LSN of the last LOGREC_FILE_ID
for this share; needed to know to which LSN domain the mappings
found in the Checkpoint record apply (new mappings should not apply
to old REDOs).
storage/maria/trnman.c:
* small changes to how trnman_collect_transactions() fills its buffer;
it also uses a non-dummy lsn_read_non_atomic() found in ma_checkpoint.h
2007-09-12 11:27:34 +02:00
|
|
|
if (die_in_middle_of_transaction)
|
|
|
|
{
|
|
|
|
/* As commit record is not done, UNDO entries needs to be rolled back */
|
|
|
|
switch (die_in_middle_of_transaction) {
|
|
|
|
case 1:
|
|
|
|
/*
|
|
|
|
Flush changed pages go to disk. That will also flush log. Recovery
|
|
|
|
will skip REDOs and apply UNDOs.
|
|
|
|
*/
|
|
|
|
_ma_flush_table_files(file, MARIA_FLUSH_DATA, FLUSH_RELEASE,
|
|
|
|
FLUSH_RELEASE);
|
|
|
|
break;
|
|
|
|
case 2:
|
|
|
|
/*
|
|
|
|
Just flush log. Pages are likely to not be on disk. Recovery will
|
|
|
|
then execute REDOs and UNDOs.
|
|
|
|
*/
|
|
|
|
if (translog_flush(file->trn->undo_lsn))
|
|
|
|
goto err;
|
|
|
|
break;
|
|
|
|
case 3:
|
|
|
|
/*
|
|
|
|
Flush nothing. Pages and log are likely to not be on disk. Recovery
|
|
|
|
will then do nothing.
|
|
|
|
*/
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
printf("Dying on request without maria_commit()/maria_close()\n");
|
|
|
|
exit(0);
|
|
|
|
}
|
2007-08-29 08:03:10 +02:00
|
|
|
if (maria_commit(file))
|
|
|
|
goto err;
|
2006-04-11 15:45:10 +02:00
|
|
|
if (maria_close(file))
|
2007-08-31 09:19:54 +02:00
|
|
|
{
|
|
|
|
file= 0;
|
2006-04-11 15:45:10 +02:00
|
|
|
goto err;
|
2007-08-31 09:19:54 +02:00
|
|
|
}
|
|
|
|
file= 0;
|
2006-04-11 15:45:10 +02:00
|
|
|
maria_panic(HA_PANIC_CLOSE); /* Should close log */
|
|
|
|
if (!silent)
|
|
|
|
{
|
|
|
|
printf("\nFollowing test have been made:\n");
|
|
|
|
printf("Write records: %d\nUpdate records: %d\nSame-key-read: %d\nDelete records: %d\n", write_count,update,dupp_keys,opt_delete);
|
|
|
|
if (rec_pointer_size)
|
|
|
|
printf("Record pointer size: %d\n",rec_pointer_size);
|
|
|
|
printf("maria_block_size: %lu\n", maria_block_size);
|
2007-04-19 17:48:36 +02:00
|
|
|
if (write_cacheing)
|
|
|
|
puts("Key cache resized");
|
2006-04-11 15:45:10 +02:00
|
|
|
if (write_cacheing)
|
|
|
|
puts("Write cacheing used");
|
|
|
|
if (write_cacheing)
|
|
|
|
puts("quick mode");
|
|
|
|
if (async_io && locking)
|
|
|
|
puts("Asyncron io with locking used");
|
|
|
|
else if (locking)
|
|
|
|
puts("Locking used");
|
|
|
|
if (use_blob)
|
|
|
|
puts("blobs used");
|
|
|
|
printf("key cache status: \n\
|
|
|
|
blocks used:%10lu\n\
|
|
|
|
not flushed:%10lu\n\
|
|
|
|
w_requests: %10lu\n\
|
|
|
|
writes: %10lu\n\
|
|
|
|
r_requests: %10lu\n\
|
|
|
|
reads: %10lu\n",
|
2007-04-04 22:37:09 +02:00
|
|
|
maria_pagecache->blocks_used,
|
|
|
|
maria_pagecache->global_blocks_changed,
|
|
|
|
(ulong) maria_pagecache->global_cache_w_requests,
|
|
|
|
(ulong) maria_pagecache->global_cache_write,
|
|
|
|
(ulong) maria_pagecache->global_cache_r_requests,
|
|
|
|
(ulong) maria_pagecache->global_cache_read);
|
2006-04-11 15:45:10 +02:00
|
|
|
}
|
2007-10-04 19:33:42 +02:00
|
|
|
maria_end();
|
2007-01-18 20:38:14 +01:00
|
|
|
my_free(blob_buffer, MYF(MY_ALLOW_ZERO_PTR));
|
2006-04-11 15:45:10 +02:00
|
|
|
my_end(silent ? MY_CHECK_ERROR : MY_CHECK_ERROR | MY_GIVE_INFO);
|
|
|
|
return(0);
|
|
|
|
err:
|
|
|
|
printf("got error: %d when using MARIA-database\n",my_errno);
|
|
|
|
if (file)
|
2007-08-31 09:19:54 +02:00
|
|
|
{
|
|
|
|
if (maria_commit(file))
|
|
|
|
goto err;
|
2006-04-11 15:45:10 +02:00
|
|
|
VOID(maria_close(file));
|
2007-08-31 09:19:54 +02:00
|
|
|
}
|
2006-04-11 15:45:10 +02:00
|
|
|
maria_end();
|
|
|
|
return(1);
|
|
|
|
} /* main */
|
|
|
|
|
|
|
|
|
2007-01-18 20:38:14 +01:00
|
|
|
/* Read options */
|
2006-04-11 15:45:10 +02:00
|
|
|
|
|
|
|
static void get_options(int argc, char **argv)
|
|
|
|
{
|
|
|
|
char *pos,*progname;
|
|
|
|
|
|
|
|
progname= argv[0];
|
|
|
|
|
|
|
|
while (--argc >0 && *(pos = *(++argv)) == '-' ) {
|
|
|
|
switch(*++pos) {
|
|
|
|
case 'B':
|
|
|
|
pack_type= HA_BINARY_PACK_KEY;
|
|
|
|
break;
|
|
|
|
case 'b':
|
2007-01-18 20:38:14 +01:00
|
|
|
use_blob= 1;
|
|
|
|
if (*++pos)
|
|
|
|
use_blob= atol(pos);
|
2006-04-11 15:45:10 +02:00
|
|
|
break;
|
|
|
|
case 'K': /* Use key cacheing */
|
2007-04-04 22:37:09 +02:00
|
|
|
pagecacheing=1;
|
2006-04-11 15:45:10 +02:00
|
|
|
if (*++pos)
|
2007-04-04 22:37:09 +02:00
|
|
|
pagecache_size=atol(pos);
|
2006-04-11 15:45:10 +02:00
|
|
|
break;
|
|
|
|
case 'W': /* Use write cacheing */
|
|
|
|
write_cacheing=1;
|
|
|
|
if (*++pos)
|
|
|
|
my_default_record_cache_size=atoi(pos);
|
|
|
|
break;
|
|
|
|
case 'd':
|
|
|
|
remove_count= atoi(++pos);
|
|
|
|
break;
|
|
|
|
case 'i':
|
|
|
|
if (*++pos)
|
2007-01-18 20:38:14 +01:00
|
|
|
srand(srand_arg= atoi(pos));
|
2006-04-11 15:45:10 +02:00
|
|
|
break;
|
|
|
|
case 'L':
|
|
|
|
locking=1;
|
|
|
|
break;
|
|
|
|
case 'A': /* use asyncron io */
|
|
|
|
async_io=1;
|
|
|
|
if (*++pos)
|
|
|
|
my_default_record_cache_size=atoi(pos);
|
|
|
|
break;
|
|
|
|
case 'v': /* verbose */
|
|
|
|
verbose=1;
|
|
|
|
break;
|
|
|
|
case 'm': /* records */
|
WL#3071 Maria checkpoint
Finally this is the real checkpoint code.
It however exhibits unstabilities when a checkpoint runs concurrently
with data-modifying clients (table corruption, transaction log's
assertions) so for now a checkpoint is taken only at startup after
recovery and at shutdown, i.e. not in concurrent situations. Later
we will let it run periodically, as well as flush dirty pages
periodically (almost all needed code is there already, only pagecache
code is written but not committed).
WL#3072 Maria recovery
* replacing UNDO_ROW_PURGE with CLR_END; testing of those CLR_END via
ma_test2 which has INSERTs failing with duplicate keys.
* replaying of REDO_RENAME_TABLE
Now, off to test Recovery in ha_maria :)
BitKeeper/deleted/.del-ma_least_recently_dirtied.c:
Delete: storage/maria/ma_least_recently_dirtied.c
BitKeeper/deleted/.del-ma_least_recently_dirtied.h:
Delete: storage/maria/ma_least_recently_dirtied.h
storage/maria/Makefile.am:
compile Checkpoint module
storage/maria/ha_maria.cc:
When ha_maria starts, do a recovery from last checkpoint.
Take a checkpoint when that recovery has ended and when ha_maria
shuts down cleanly.
storage/maria/ma_blockrec.c:
* even if my_sync() fails we have to my_close() (otherwise we leak
a descriptor)
* UNDO_ROW_PURGE is replaced by a simple CLR_END for UNDO_ROW_INSERT,
as promised in the old comment; it gives us skipping during the
UNDO phase.
storage/maria/ma_check.c:
All REDOs before create_rename_lsn are ignored by Recovery. So
create_rename_lsn must be set only after all data/index has been
flushed and forced to disk. We thus move write_log_record_for_repair()
to after _ma_flush_tables_files_after_repair().
storage/maria/ma_checkpoint.c:
Checkpoint module.
storage/maria/ma_checkpoint.h:
optional argument if caller wants a thread to periodically take
checkpoints and flush dirty pages.
storage/maria/ma_create.c:
* no need to init some vars as the initial bzero(share) takes care of this.
* update to new function's name
* even if we fail in my_sync() we have to my_close()
storage/maria/ma_extra.c:
Checkpoint reads share->last_version under intern_lock, so we make
maria_extra() update it under intern_lock. THR_LOCK_maria still needed
because of _ma_test_if_reopen().
storage/maria/ma_init.c:
destroy checkpoint module when Maria shuts down.
storage/maria/ma_loghandler.c:
* UNDO_ROW_PURGE gone (see ma_blockrec.c)
* we need to remember the LSN of the LOGREC_FILE_ID for a share,
because this LSN is needed into the checkpoint record (Recovery wants
to know the validity domain of an id->name mapping)
* translog_get_horizon_no_lock() needed for Checkpoint
* comment about failing assertion (Sanja knows)
* translog_init_reader_data() thought that translog_read_record_header_scan()
returns 0 in case of error, but 0 just means "0-length header".
* translog_assign_id_to_share() now needs the MARIA_HA because
LOGREC_FILE_ID uses a log-write hook.
* Verify that (de)assignment of share->id happens only under intern_lock,
as Checkpoint reads this id with intern_lock.
* translog_purge() can accept TRANSLOG_ADDRESS, not necessarily
a real LSN.
storage/maria/ma_loghandler.h:
prototype updates
storage/maria/ma_open.c:
no need to initialize "res"
storage/maria/ma_pagecache.c:
When taking a checkpoint, we don't need to know the maximum rec_lsn
of dirty pages; this LSN was intended to be used in the two-checkpoint
rule, but last_checkpoint_lsn is as good.
4 bytes for stored_list_size is enough as PAGECACHE::blocks (number
of blocks which the pagecache can contain) is int.
storage/maria/ma_pagecache.h:
new prototype
storage/maria/ma_recovery.c:
* added replaying of REDO_RENAME_TABLE
* UNDO_ROW_PURGE gone (see ma_blockrec.c), replaced by CLR_END
* Recovery from the last checkpoint record now possible
* In new_table() we skip the table if the id->name mapping is older than
create_rename_lsn (mapping dates from lsn_of_file_id).
* in get_MARIA_HA_from_REDO_record() we skip the record
if the id->name mapping is newer than the record (can happen if processing
a record which is before the checkpoint record).
* parse_checkpoint_record() has to return a LSN, that's what caller expects
storage/maria/ma_rename.c:
new function's name; log end zeroes of tables' names (ease recovery)
storage/maria/ma_test2.c:
* equivalent of ma_test1's --test-undo added (named -u here).
* -t=1 now stops right after creating the table, so that
we can test undoing of INSERTs with duplicate keys (which tests the
CLR_END logged by _ma_write_abort_block_record()).
storage/maria/ma_test_recovery.expected:
Result of testing undoing of INSERTs with duplicate keys; there are
some differences in maria_chk -dvv but they are normal (removing
records does not shrink data/index file, does not put back the
"analyzed, optimized keys"(etc) index state.
storage/maria/ma_test_recovery:
Test undoing of INSERTs with duplicate keys, using ma_test2;
when such INSERT happens, it logs REDO_INSERT, UNDO_INSERT, REDO_DELETE,
CLR_END; we abort after that, and test that CLR_END causes recovery
to jump over UNDO_INSERT.
storage/maria/ma_write.c:
comment
storage/maria/maria_chk.c:
comment
storage/maria/maria_def.h:
* a new bit in MARIA_SHARE::in_checkpoint, used to build a list
of unique shares during Checkpoint.
* MARIA_SHARE::lsn_of_file_id added: the LSN of the last LOGREC_FILE_ID
for this share; needed to know to which LSN domain the mappings
found in the Checkpoint record apply (new mappings should not apply
to old REDOs).
storage/maria/trnman.c:
* small changes to how trnman_collect_transactions() fills its buffer;
it also uses a non-dummy lsn_read_non_atomic() found in ma_checkpoint.h
2007-09-12 11:27:34 +02:00
|
|
|
if ((recant=atoi(++pos)) < 10 && testflag > 2)
|
2006-04-11 15:45:10 +02:00
|
|
|
{
|
WL#3071 Maria checkpoint
Finally this is the real checkpoint code.
It however exhibits unstabilities when a checkpoint runs concurrently
with data-modifying clients (table corruption, transaction log's
assertions) so for now a checkpoint is taken only at startup after
recovery and at shutdown, i.e. not in concurrent situations. Later
we will let it run periodically, as well as flush dirty pages
periodically (almost all needed code is there already, only pagecache
code is written but not committed).
WL#3072 Maria recovery
* replacing UNDO_ROW_PURGE with CLR_END; testing of those CLR_END via
ma_test2 which has INSERTs failing with duplicate keys.
* replaying of REDO_RENAME_TABLE
Now, off to test Recovery in ha_maria :)
BitKeeper/deleted/.del-ma_least_recently_dirtied.c:
Delete: storage/maria/ma_least_recently_dirtied.c
BitKeeper/deleted/.del-ma_least_recently_dirtied.h:
Delete: storage/maria/ma_least_recently_dirtied.h
storage/maria/Makefile.am:
compile Checkpoint module
storage/maria/ha_maria.cc:
When ha_maria starts, do a recovery from last checkpoint.
Take a checkpoint when that recovery has ended and when ha_maria
shuts down cleanly.
storage/maria/ma_blockrec.c:
* even if my_sync() fails we have to my_close() (otherwise we leak
a descriptor)
* UNDO_ROW_PURGE is replaced by a simple CLR_END for UNDO_ROW_INSERT,
as promised in the old comment; it gives us skipping during the
UNDO phase.
storage/maria/ma_check.c:
All REDOs before create_rename_lsn are ignored by Recovery. So
create_rename_lsn must be set only after all data/index has been
flushed and forced to disk. We thus move write_log_record_for_repair()
to after _ma_flush_tables_files_after_repair().
storage/maria/ma_checkpoint.c:
Checkpoint module.
storage/maria/ma_checkpoint.h:
optional argument if caller wants a thread to periodically take
checkpoints and flush dirty pages.
storage/maria/ma_create.c:
* no need to init some vars as the initial bzero(share) takes care of this.
* update to new function's name
* even if we fail in my_sync() we have to my_close()
storage/maria/ma_extra.c:
Checkpoint reads share->last_version under intern_lock, so we make
maria_extra() update it under intern_lock. THR_LOCK_maria still needed
because of _ma_test_if_reopen().
storage/maria/ma_init.c:
destroy checkpoint module when Maria shuts down.
storage/maria/ma_loghandler.c:
* UNDO_ROW_PURGE gone (see ma_blockrec.c)
* we need to remember the LSN of the LOGREC_FILE_ID for a share,
because this LSN is needed into the checkpoint record (Recovery wants
to know the validity domain of an id->name mapping)
* translog_get_horizon_no_lock() needed for Checkpoint
* comment about failing assertion (Sanja knows)
* translog_init_reader_data() thought that translog_read_record_header_scan()
returns 0 in case of error, but 0 just means "0-length header".
* translog_assign_id_to_share() now needs the MARIA_HA because
LOGREC_FILE_ID uses a log-write hook.
* Verify that (de)assignment of share->id happens only under intern_lock,
as Checkpoint reads this id with intern_lock.
* translog_purge() can accept TRANSLOG_ADDRESS, not necessarily
a real LSN.
storage/maria/ma_loghandler.h:
prototype updates
storage/maria/ma_open.c:
no need to initialize "res"
storage/maria/ma_pagecache.c:
When taking a checkpoint, we don't need to know the maximum rec_lsn
of dirty pages; this LSN was intended to be used in the two-checkpoint
rule, but last_checkpoint_lsn is as good.
4 bytes for stored_list_size is enough as PAGECACHE::blocks (number
of blocks which the pagecache can contain) is int.
storage/maria/ma_pagecache.h:
new prototype
storage/maria/ma_recovery.c:
* added replaying of REDO_RENAME_TABLE
* UNDO_ROW_PURGE gone (see ma_blockrec.c), replaced by CLR_END
* Recovery from the last checkpoint record now possible
* In new_table() we skip the table if the id->name mapping is older than
create_rename_lsn (mapping dates from lsn_of_file_id).
* in get_MARIA_HA_from_REDO_record() we skip the record
if the id->name mapping is newer than the record (can happen if processing
a record which is before the checkpoint record).
* parse_checkpoint_record() has to return a LSN, that's what caller expects
storage/maria/ma_rename.c:
new function's name; log end zeroes of tables' names (ease recovery)
storage/maria/ma_test2.c:
* equivalent of ma_test1's --test-undo added (named -u here).
* -t=1 now stops right after creating the table, so that
we can test undoing of INSERTs with duplicate keys (which tests the
CLR_END logged by _ma_write_abort_block_record()).
storage/maria/ma_test_recovery.expected:
Result of testing undoing of INSERTs with duplicate keys; there are
some differences in maria_chk -dvv but they are normal (removing
records does not shrink data/index file, does not put back the
"analyzed, optimized keys"(etc) index state.
storage/maria/ma_test_recovery:
Test undoing of INSERTs with duplicate keys, using ma_test2;
when such INSERT happens, it logs REDO_INSERT, UNDO_INSERT, REDO_DELETE,
CLR_END; we abort after that, and test that CLR_END causes recovery
to jump over UNDO_INSERT.
storage/maria/ma_write.c:
comment
storage/maria/maria_chk.c:
comment
storage/maria/maria_def.h:
* a new bit in MARIA_SHARE::in_checkpoint, used to build a list
of unique shares during Checkpoint.
* MARIA_SHARE::lsn_of_file_id added: the LSN of the last LOGREC_FILE_ID
for this share; needed to know to which LSN domain the mappings
found in the Checkpoint record apply (new mappings should not apply
to old REDOs).
storage/maria/trnman.c:
* small changes to how trnman_collect_transactions() fills its buffer;
it also uses a non-dummy lsn_read_non_atomic() found in ma_checkpoint.h
2007-09-12 11:27:34 +02:00
|
|
|
fprintf(stderr,"record count must be >= 10 (if testflag > 2)\n");
|
2006-04-11 15:45:10 +02:00
|
|
|
exit(1);
|
|
|
|
}
|
|
|
|
break;
|
|
|
|
case 'e': /* maria_block_length */
|
2007-04-19 17:48:36 +02:00
|
|
|
case 'E':
|
Completion of merge of mysql-5.1 into mysql-maria.
Manually imported changes done to MyISAM (include/myisam.h,
storage/myisam/*, sql/ha_myisam.*, mysql-test/t/myisam.test,
mysql-test/t/ps_2myisam.test) the last
months into Maria (tedious, should do it more frequently in the
future), including those not done at the previous 5.1->Maria merge
(please in the future don't forget to apply MyISAM changes to Maria
when you merge 5.1 into Maria).
Note: I didn't try to import anything which could be MyISAM-related
in other tests of mysql-test (I didn't want to dig in all csets),
but as QA is working to make most tests re-usable for other engines
(Falcon), it is likely that we'll benefit from this and just have
to set engine=Maria somewhere to run those tests on Maria.
func_group and partition tests fail but they already do in main 5.1
on my machine. No Valgrind error in t/*maria*.test.
Monty: please see the commit comment of maria.result and check.
BitKeeper/deleted/.del-ha_maria.m4:
Delete: config/ac-macros/ha_maria.m4
configure.in:
fix for the new way of enabling engines
include/maria.h:
importing changes done to MyISAM the last months into Maria
include/my_handler.h:
importing changes done to MyISAM the last months into Maria
include/myisam.h:
importing changes done to MyISAM the last months into Maria
mysql-test/r/maria.result:
identical to myisam.result, except the engine name in some places
AND in the line testing key_block_size=1000000000000000000:
Maria gives a key block size of 8192 while MyISAM gives 4096;
is it explainable by the difference between MARIA_KEY_BLOCK_LENGTH
and the same constant in MyISAM? Monty?
mysql-test/r/ps_maria.result:
identical to ps_2myisam.result (except the engine name in some places)
mysql-test/t/maria.test:
instead of engine=maria everywhere, I use @@storage_engine (reduces
the diff with myisam.test).
importing changes done to MyISAM the last months into Maria
mysys/my_handler.c:
importing changes done to MyISAM the last months into Maria
sql/ha_maria.cc:
importing changes done to MyISAM the last months into Maria
sql/ha_maria.h:
importing changes done to MyISAM the last months into Maria
sql/mysqld.cc:
unneeded
storage/maria/Makefile.am:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_check.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_create.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_delete_table.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_dynrec.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_extra.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_ft_boolean_search.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_ft_eval.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_ft_nlq_search.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_ft_parser.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_ft_test1.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_ft_update.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_ftdefs.h:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_key.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_open.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_page.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_rkey.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_rsamepos.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_rt_index.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_rt_mbr.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_search.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_sort.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_test1.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_test2.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_test3.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_update.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_write.c:
importing changes done to MyISAM the last months into Maria
storage/maria/maria_chk.c:
importing changes done to MyISAM the last months into Maria
storage/maria/maria_def.h:
importing changes done to MyISAM the last months into Maria
storage/maria/maria_ftdump.c:
importing changes done to MyISAM the last months into Maria
storage/maria/maria_pack.c:
importing changes done to MyISAM the last months into Maria
2006-08-10 16:36:54 +02:00
|
|
|
if ((maria_block_size= atoi(++pos)) < MARIA_MIN_KEY_BLOCK_LENGTH ||
|
2006-04-11 15:45:10 +02:00
|
|
|
maria_block_size > MARIA_MAX_KEY_BLOCK_LENGTH)
|
|
|
|
{
|
|
|
|
fprintf(stderr,"Wrong maria_block_length\n");
|
|
|
|
exit(1);
|
|
|
|
}
|
Completion of merge of mysql-5.1 into mysql-maria.
Manually imported changes done to MyISAM (include/myisam.h,
storage/myisam/*, sql/ha_myisam.*, mysql-test/t/myisam.test,
mysql-test/t/ps_2myisam.test) the last
months into Maria (tedious, should do it more frequently in the
future), including those not done at the previous 5.1->Maria merge
(please in the future don't forget to apply MyISAM changes to Maria
when you merge 5.1 into Maria).
Note: I didn't try to import anything which could be MyISAM-related
in other tests of mysql-test (I didn't want to dig in all csets),
but as QA is working to make most tests re-usable for other engines
(Falcon), it is likely that we'll benefit from this and just have
to set engine=Maria somewhere to run those tests on Maria.
func_group and partition tests fail but they already do in main 5.1
on my machine. No Valgrind error in t/*maria*.test.
Monty: please see the commit comment of maria.result and check.
BitKeeper/deleted/.del-ha_maria.m4:
Delete: config/ac-macros/ha_maria.m4
configure.in:
fix for the new way of enabling engines
include/maria.h:
importing changes done to MyISAM the last months into Maria
include/my_handler.h:
importing changes done to MyISAM the last months into Maria
include/myisam.h:
importing changes done to MyISAM the last months into Maria
mysql-test/r/maria.result:
identical to myisam.result, except the engine name in some places
AND in the line testing key_block_size=1000000000000000000:
Maria gives a key block size of 8192 while MyISAM gives 4096;
is it explainable by the difference between MARIA_KEY_BLOCK_LENGTH
and the same constant in MyISAM? Monty?
mysql-test/r/ps_maria.result:
identical to ps_2myisam.result (except the engine name in some places)
mysql-test/t/maria.test:
instead of engine=maria everywhere, I use @@storage_engine (reduces
the diff with myisam.test).
importing changes done to MyISAM the last months into Maria
mysys/my_handler.c:
importing changes done to MyISAM the last months into Maria
sql/ha_maria.cc:
importing changes done to MyISAM the last months into Maria
sql/ha_maria.h:
importing changes done to MyISAM the last months into Maria
sql/mysqld.cc:
unneeded
storage/maria/Makefile.am:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_check.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_create.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_delete_table.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_dynrec.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_extra.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_ft_boolean_search.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_ft_eval.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_ft_nlq_search.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_ft_parser.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_ft_test1.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_ft_update.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_ftdefs.h:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_key.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_open.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_page.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_rkey.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_rsamepos.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_rt_index.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_rt_mbr.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_search.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_sort.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_test1.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_test2.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_test3.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_update.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_write.c:
importing changes done to MyISAM the last months into Maria
storage/maria/maria_chk.c:
importing changes done to MyISAM the last months into Maria
storage/maria/maria_def.h:
importing changes done to MyISAM the last months into Maria
storage/maria/maria_ftdump.c:
importing changes done to MyISAM the last months into Maria
storage/maria/maria_pack.c:
importing changes done to MyISAM the last months into Maria
2006-08-10 16:36:54 +02:00
|
|
|
maria_block_size= my_round_up_to_next_power(maria_block_size);
|
2006-04-11 15:45:10 +02:00
|
|
|
break;
|
|
|
|
case 'f':
|
|
|
|
if ((first_key=atoi(++pos)) < 0 || first_key >= MARIA_KEYS)
|
|
|
|
first_key=0;
|
|
|
|
break;
|
WL#3071 Maria checkpoint
Ability for flush_pagecache_blocks() to flush only certain pages of
a file, as instructed by an option "filter" pointer-to-function argument;
Checkpoint and background dirty page flushing use that to flush only
pages which have been dirty for long enough and bitmap pages.
Fix for a bug in flush_cached_blocks() (no idea if it could produce
a bug in real life, but theoretically it is).
Testing checkpoint in ma_test_recovery via ma_test1 and ma_test2.
Background checkpoint & dirty pages flush thread is still disabled
by default in ha_maria.
mysql-test/r/maria.result:
result update
storage/maria/ha_maria.cc:
blank after function comment
storage/maria/ma_checkpoint.c:
Using an enum instead of 0/1/2 (applying Sanja's review comments).
The comment about "this is an horizon" can be removed as Sanja
created translog_next_LSN() which parse_checkpoint_record() uses.
Variables in ma_checkpoint_background() cannot be declared in the
for() as their value must not be reset at each iteration!
storage/maria/ma_pagecache.c:
adding to flush_pagecache_blocks() optional arguments 'filter'
(pointer to function) and 'filter_arg'; if filter!=NULL this function
will be called for each block of the file and will reply if this
block and following ones should be flushed or not (3 possible
replies).
Fixing a bug when flush_cached_blocks() skips a pinned page: it has
to unset PCBLOCK_IN_FLUSH set by flush_pagecache_blocks_int().
storage/maria/ma_pagecache.h:
flush_pagecache_blocks() is changed to take "filter" and "filter_arg"
arguments. "filter", if it is not NULL, may return one value
among enum pagecache_flush_filter_result.
storage/maria/ma_recovery.c:
open_count=0 when closing tables at the end of recovery.
storage/maria/ma_test1.c:
Optional checkpoints (-H#) at various stages (stages similar
to --testflag), for testing of checkpoints.
storage/maria/ma_test2.c:
Optional checkpoints (-H#) at various stages (stages similar
to -t), for testing of checkpoints.
storage/maria/ma_test_recovery.expected:
Result update: the results of the additional test run with -H#
(checkpoints) are added here. They are exactly identical to without
checkpoints except that the index's Root (printed by maria_chk)
is more correct when using checkpoints. This is because checkpoint
flushed the state, so it happens to be correct, while no-checkpoint
does not flush the state, and recovery does not recover indexes
so Root is never fixed. When we recover indices, this will go away.
storage/maria/ma_test_recovery:
We duplicate the loop of tests to add an additional run with
checkpoints at various stages, to see if maria_read_log
uses them fine.
2007-10-17 16:55:26 +02:00
|
|
|
case 'H':
|
|
|
|
checkpoint= atoi(++pos);
|
|
|
|
break;
|
2006-04-11 15:45:10 +02:00
|
|
|
case 'k':
|
|
|
|
if ((keys=(uint) atoi(++pos)) < 1 ||
|
|
|
|
keys > (uint) (MARIA_KEYS-first_key))
|
|
|
|
keys=MARIA_KEYS-first_key;
|
|
|
|
break;
|
2007-01-18 20:38:14 +01:00
|
|
|
case 'M':
|
|
|
|
record_type= BLOCK_RECORD;
|
|
|
|
break;
|
2006-04-11 15:45:10 +02:00
|
|
|
case 'P':
|
|
|
|
pack_type=0; /* Don't use DIFF_LENGTH */
|
|
|
|
pack_seg=0;
|
|
|
|
break;
|
|
|
|
case 'R': /* Length of record pointer */
|
|
|
|
rec_pointer_size=atoi(++pos);
|
|
|
|
if (rec_pointer_size > 7)
|
|
|
|
rec_pointer_size=0;
|
|
|
|
break;
|
|
|
|
case 'S':
|
|
|
|
pack_fields=0; /* Static-length-records */
|
2007-01-18 20:38:14 +01:00
|
|
|
record_type= STATIC_RECORD;
|
2006-04-11 15:45:10 +02:00
|
|
|
break;
|
|
|
|
case 's':
|
|
|
|
silent=1;
|
|
|
|
break;
|
|
|
|
case 't':
|
|
|
|
testflag=atoi(++pos); /* testmod */
|
|
|
|
break;
|
This patch is a collection of patches from from Sanja, Sergei and Monty.
Added logging and pinning of pages to block format.
Integration of transaction manager, log handler.
Better page cache intergration
Split trnman.h into two files, so that we don't have to include my_atomic.h into C++ programs.
Renaming of structures, more comments, more debugging etc.
Fixed problem with small head block + long varchar.
Added extra argument to delete_record() and update_record() (needed for UNDO logging)
Small changes to interface of pagecache and log handler.
Change initialization of log_record_type_descriptors to not be depending on enum order.
Use array of LEX_STRING's to send data to log handler
Added 'dummy' transaction option to MARIA_INFO so that we can always assume 'trn' exists.
include/lf.h:
Interface fixes
Rename of structures
(Patch from Sergei via Sanja)
include/my_atomic.h:
More comments
include/my_global.h:
Added MY_ERRPTR
include/pagecache.h:
Added undo LSN when unlocking pages
mysql-test/r/maria.result:
Updated results
mysql-test/t/maria.test:
Added autocommit around lock tables
(Patch from Sanja)
mysys/lf_alloc-pin.c:
Post-review fixes, simple optimizations
More comments
Struct slot renames
Check amount of memory on stack
(Patch from Sergei)
mysys/lf_dynarray.c:
More comments
mysys/lf_hash.c:
More comments
After review fixes
(Patch from Sergei)
storage/maria/ha_maria.cc:
Split trnman.h into two files, so that we don't have to include my_atomic.h into the .cc program.
(Temporary fix to avoid bug in gcc)
Move out all deferencing of the transaction structure.
Transaction manager integrated (Patch from Sergei)
storage/maria/ha_maria.h:
Added prototype for start_stmt()
storage/maria/lockman.c:
Function call rename
storage/maria/ma_bitmap.c:
Mark deleted pages free from page cache
storage/maria/ma_blockrec.c:
Offset -> rownr
More debugging
Fixed problem with small head block + long varchar
Added logging of changed pages
Added logging of undo (Including only loggging of changed fields in case of update)
Added pinning/unpinning of all changed pages
More comments
Added free_full_pages() as the same code was used in several places.
fill_rows_parts() renamed as fill_insert_undo_parts()
offset -> rownr
Added some optimization of not transactional tables
_ma_update_block_record() has new parameter, as we need original row to do efficent undo for update
storage/maria/ma_blockrec.h:
Added ROW_EXTENTS_ON_STACK
Changed prototype for update and delete of row
storage/maria/ma_check.c:
Added original row to delete_record() call
storage/maria/ma_control_file.h:
Added ifdefs for C++
storage/maria/ma_delete.c:
Added original row to delete_record() call
(Needed for efficent undo logging)
storage/maria/ma_dynrec.c:
Added extra argument to delete_record() and update_record()
Removed not used variable
storage/maria/ma_init.c:
Initialize log handler
storage/maria/ma_loghandler.c:
Removed not used variable
Change initialization of log_record_type_descriptors to not be depending on enum order
Use array of LEX_STRING's to send data to log handler
storage/maria/ma_loghandler.h:
New defines
Use array of LEX_STRING's to send data to log handler
storage/maria/ma_open.c:
Added 'dummy' transaction option to MARIA_INFO so that we can always assume 'trn' exists.
Store in MARIA_SHARE->page_type if pages will have up to date LSN's
storage/maria/ma_pagecache.c:
Don't decrease number of readers when using pagecache_write()/pagecache_read()
In pagecache_write() decrement request count if page was left pinned
Added pagecache_delete_pages()
Removed some casts
Make trace output consistent with rest of code
Simplify calling of DBUG_ASSERT(0)
Only update LSN if the LSN is bigger than what's already on the page
Added LSN parameter pagecache_unpin_page(), pagecache_unpin(), and pagecache_unlock()
(Part of patch from Sanja)
storage/maria/ma_static.c:
Added 'dummy' transaction option to MARIA_INFO so that we can always assume 'trn' exists.
Added default page cache
storage/maria/ma_statrec.c:
Added extra argument to delete_record() and update_record()
storage/maria/ma_test1.c:
Added option -T for transactions
storage/maria/ma_test2.c:
Added option -T for transactions
storage/maria/ma_test_all.sh:
Test with transactions
storage/maria/ma_update.c:
Changed prototype for update of row
storage/maria/maria_def.h:
Changed prototype for update & delete of row as block records need to access the old row
Store in MARIA_SHARE->page_type if pages will have up to date LSN's
Added MARIA_MAX_TREE_LEVELS to allow us to calculate the number of possible pinned pages we may need.
Removed not used 'empty_bits_buffer'
Added pointer to transaction object
Added array for pinned pages
Added log_row_parts array for logging of field data.
Added MARIA_PINNED_PAGE to store pinned pages
storage/maria/trnman.c:
Added accessor functions to transaction object
Added missing DBUG_RETURN()
More debugging
More comments
Changed // comment of code to #ifdef NOT_USED
Transaction manager integrated.
Post review fixes
Part of patch originally from Sergei
storage/maria/trnman.h:
Split trnman.h into two files, so that we don't have to include my_atomic.h into the .cc program.
(Temporary fix to avoid bug in gcc)
storage/maria/unittest/ma_pagecache_single.c:
Added missing argument
Added SKIP_BIG_TESTS
(Patch from Sanja)
storage/maria/unittest/ma_test_loghandler-t.c:
Test logging with new LEX_STRING parameter
(Patch from Sanja)
storage/maria/unittest/ma_test_loghandler_multigroup-t.c:
Test logging with new LEX_STRING parameter
(Patch from Sanja)
storage/maria/unittest/ma_test_loghandler_multithread-t.c:
Test logging with new LEX_STRING parameter
(Patch from Sanja)
storage/maria/unittest/ma_test_loghandler_pagecache-t.c:
Test logging with new LEX_STRING parameter
(Patch from Sanja)
storage/maria/unittest/trnman-t.c:
Stack overflow detection
(Patch from Sergei)
unittest/unit.pl:
Command-line options --big and --verbose
(Patch from Sergei)
unittest/mytap/tap.c:
Detect --big
(Patch from Sergei)
unittest/mytap/tap.h:
Skip_big_tests and SKIP_BIG_TESTS
(Patch from Sergei)
storage/maria/trnman_public.h:
New BitKeeper file ``storage/maria/trnman_public.h''
2007-05-29 19:13:56 +02:00
|
|
|
case 'T':
|
|
|
|
transactional= 1;
|
|
|
|
break;
|
WL#3071 Maria checkpoint
Finally this is the real checkpoint code.
It however exhibits unstabilities when a checkpoint runs concurrently
with data-modifying clients (table corruption, transaction log's
assertions) so for now a checkpoint is taken only at startup after
recovery and at shutdown, i.e. not in concurrent situations. Later
we will let it run periodically, as well as flush dirty pages
periodically (almost all needed code is there already, only pagecache
code is written but not committed).
WL#3072 Maria recovery
* replacing UNDO_ROW_PURGE with CLR_END; testing of those CLR_END via
ma_test2 which has INSERTs failing with duplicate keys.
* replaying of REDO_RENAME_TABLE
Now, off to test Recovery in ha_maria :)
BitKeeper/deleted/.del-ma_least_recently_dirtied.c:
Delete: storage/maria/ma_least_recently_dirtied.c
BitKeeper/deleted/.del-ma_least_recently_dirtied.h:
Delete: storage/maria/ma_least_recently_dirtied.h
storage/maria/Makefile.am:
compile Checkpoint module
storage/maria/ha_maria.cc:
When ha_maria starts, do a recovery from last checkpoint.
Take a checkpoint when that recovery has ended and when ha_maria
shuts down cleanly.
storage/maria/ma_blockrec.c:
* even if my_sync() fails we have to my_close() (otherwise we leak
a descriptor)
* UNDO_ROW_PURGE is replaced by a simple CLR_END for UNDO_ROW_INSERT,
as promised in the old comment; it gives us skipping during the
UNDO phase.
storage/maria/ma_check.c:
All REDOs before create_rename_lsn are ignored by Recovery. So
create_rename_lsn must be set only after all data/index has been
flushed and forced to disk. We thus move write_log_record_for_repair()
to after _ma_flush_tables_files_after_repair().
storage/maria/ma_checkpoint.c:
Checkpoint module.
storage/maria/ma_checkpoint.h:
optional argument if caller wants a thread to periodically take
checkpoints and flush dirty pages.
storage/maria/ma_create.c:
* no need to init some vars as the initial bzero(share) takes care of this.
* update to new function's name
* even if we fail in my_sync() we have to my_close()
storage/maria/ma_extra.c:
Checkpoint reads share->last_version under intern_lock, so we make
maria_extra() update it under intern_lock. THR_LOCK_maria still needed
because of _ma_test_if_reopen().
storage/maria/ma_init.c:
destroy checkpoint module when Maria shuts down.
storage/maria/ma_loghandler.c:
* UNDO_ROW_PURGE gone (see ma_blockrec.c)
* we need to remember the LSN of the LOGREC_FILE_ID for a share,
because this LSN is needed into the checkpoint record (Recovery wants
to know the validity domain of an id->name mapping)
* translog_get_horizon_no_lock() needed for Checkpoint
* comment about failing assertion (Sanja knows)
* translog_init_reader_data() thought that translog_read_record_header_scan()
returns 0 in case of error, but 0 just means "0-length header".
* translog_assign_id_to_share() now needs the MARIA_HA because
LOGREC_FILE_ID uses a log-write hook.
* Verify that (de)assignment of share->id happens only under intern_lock,
as Checkpoint reads this id with intern_lock.
* translog_purge() can accept TRANSLOG_ADDRESS, not necessarily
a real LSN.
storage/maria/ma_loghandler.h:
prototype updates
storage/maria/ma_open.c:
no need to initialize "res"
storage/maria/ma_pagecache.c:
When taking a checkpoint, we don't need to know the maximum rec_lsn
of dirty pages; this LSN was intended to be used in the two-checkpoint
rule, but last_checkpoint_lsn is as good.
4 bytes for stored_list_size is enough as PAGECACHE::blocks (number
of blocks which the pagecache can contain) is int.
storage/maria/ma_pagecache.h:
new prototype
storage/maria/ma_recovery.c:
* added replaying of REDO_RENAME_TABLE
* UNDO_ROW_PURGE gone (see ma_blockrec.c), replaced by CLR_END
* Recovery from the last checkpoint record now possible
* In new_table() we skip the table if the id->name mapping is older than
create_rename_lsn (mapping dates from lsn_of_file_id).
* in get_MARIA_HA_from_REDO_record() we skip the record
if the id->name mapping is newer than the record (can happen if processing
a record which is before the checkpoint record).
* parse_checkpoint_record() has to return a LSN, that's what caller expects
storage/maria/ma_rename.c:
new function's name; log end zeroes of tables' names (ease recovery)
storage/maria/ma_test2.c:
* equivalent of ma_test1's --test-undo added (named -u here).
* -t=1 now stops right after creating the table, so that
we can test undoing of INSERTs with duplicate keys (which tests the
CLR_END logged by _ma_write_abort_block_record()).
storage/maria/ma_test_recovery.expected:
Result of testing undoing of INSERTs with duplicate keys; there are
some differences in maria_chk -dvv but they are normal (removing
records does not shrink data/index file, does not put back the
"analyzed, optimized keys"(etc) index state.
storage/maria/ma_test_recovery:
Test undoing of INSERTs with duplicate keys, using ma_test2;
when such INSERT happens, it logs REDO_INSERT, UNDO_INSERT, REDO_DELETE,
CLR_END; we abort after that, and test that CLR_END causes recovery
to jump over UNDO_INSERT.
storage/maria/ma_write.c:
comment
storage/maria/maria_chk.c:
comment
storage/maria/maria_def.h:
* a new bit in MARIA_SHARE::in_checkpoint, used to build a list
of unique shares during Checkpoint.
* MARIA_SHARE::lsn_of_file_id added: the LSN of the last LOGREC_FILE_ID
for this share; needed to know to which LSN domain the mappings
found in the Checkpoint record apply (new mappings should not apply
to old REDOs).
storage/maria/trnman.c:
* small changes to how trnman_collect_transactions() fills its buffer;
it also uses a non-dummy lsn_read_non_atomic() found in ma_checkpoint.h
2007-09-12 11:27:34 +02:00
|
|
|
case 'u':
|
|
|
|
die_in_middle_of_transaction= atoi(++pos);
|
|
|
|
break;
|
2006-04-11 15:45:10 +02:00
|
|
|
case 'q':
|
|
|
|
opt_quick_mode=1;
|
|
|
|
break;
|
|
|
|
case 'c':
|
|
|
|
create_flag|= HA_CREATE_CHECKSUM;
|
|
|
|
break;
|
|
|
|
case 'D':
|
|
|
|
create_flag|=HA_CREATE_DELAY_KEY_WRITE;
|
|
|
|
break;
|
WL#3072 - Maria recovery
Unit test for recovery: runs ma_test1 and ma_test2 (both only with
INSERTs and DELETEs; UPDATEs disabled as not handled by recovery)
then moves the tables elswhere; recreates tables from the log, and
compares and fails if there is a difference. Passes now.
Most of maria_read_log.c moved to ma_recovery.c, as it will be re-used
for recovery-from-ha_maria.
Bugfixes of applying of REDO_INSERT, REDO_PURGE_ROW.
Applying of REDO_PURGE_BLOCKS, REDO_DELETE_ALL, REDO_DROP_TABLE,
UNDO_ROW_INSERT (in REDO phase only, i.e. just doing records++),
UNDO_ROW_DELETE, UNDO_ROW_PURGE.
Code cleanups.
Monty: please look for "QQ". Sanja: please look for "Sanja".
Future tasks: recovery of the bitmap (easy), recovery of the state
(make it idempotent), more REDOs (Monty to work on
REDO_UPDATE?), UNDO phase...
Pushing this cset as it looks safe, contains test and bugfixes which
will help Monty implement applying of REDO_UPDATE.
sql/handler.cc:
typo
storage/maria/Makefile.am:
Adding ma_test_recovery (which ma_test_all invokes, and which can
also be run alone). Most of maria_read_log.c moved to ma_recovery.c
storage/maria/ha_maria.cc:
comments
storage/maria/ma_bitmap.c:
fixing comments. 2 -> sizeof(maria_bitmap_marker).
Bitmap-related part of _ma_initialize_datafile() moves in bitmap module.
Now putting the "bm" signature when creating the first bitmap page
(it used to happen only at next open, but that
caused an annoying difference when testing Recovery if the original
run didn't open the table, and it looks more
logical like this: it goes to disk only with its signature correct);
see the "QQ" comment towards the _ma_initialize_data_file() call
in ma_create.c for more).
When reading a bitmap page, verify its signature (happens when normally
using the table or when CHECKing it; not when REPAIRing it).
storage/maria/ma_blockrec.c:
* no need to sync the data file if table is not transactional
* Comments, code cleanup (log-related data moved to log-related code
block, int5store->page_store).
* Store the table's short id into LOGREC_UNDO_ROW_PURGE, like we
do for other records (though this record will soon be replaced
with a CLR).
* If "page" is 1 it means the page which extends from byte
page*block_size+1 to (page+1)*block_size (byte number 1 being
the first byte of the file). The last byte of the file is
data_file_length (same convention).
A new page needs to be created if the last byte of the page is
beyond the last byte of the file, i.e.
(page+1)*block_size+1 > data_file_length, so we correct the test
(bug found when testing log applying for ma_test1 -M -T --skip-update).
* update the page's LSN when removing a row from it during
execution of a REDO_PURGE_ROW record (bug found when testing log
applying for ma_test1 -M -T --skip-update).
* applying of REDO_PURGE_BLOCKs (limited to a one-page range for now).
storage/maria/ma_blockrec.h:
new functions. maria_bitmap_marker does not need to be exported.
storage/maria/ma_close.c:
we can always flush the table's state when closing the last instance
of the table. And it is needed for maria_read_log (as it does
not use maria_lock_database()).
storage/maria/ma_control_file.c:
when in Recovery, some assertions should not be used.
storage/maria/ma_control_file.h:
double-inclusion safe
storage/maria/ma_create.c:
during recovery, don't log records. Comments.
Moving the creation of the first bitmap page to ma_bitmap.c
storage/maria/ma_delete_table.c:
during recovery, don't log records. Log the end-zero of the dropped
table's name, so that recovery can use the string in place without
extending it to fit an end zero.
storage/maria/ma_loghandler.c:
* inwrite_rec_hook also needs access to the MARIA_SHARE, like
prewrite_rec_hook. This will be needed to update
share->records_diff (in the upcoming patch "recovery of the state").
* LOG_DESC::record_ends_group changed to an enum.
* LOG_DESC for LOGREC_REDO_PURGE_BLOCKS and LOGREC_UNDO_ROW_PURGE
corrected
* Sanja please see the @todo LOG BUG
* avoiding DBUG_RETURN(func()) as it gives confusing debug traces.
storage/maria/ma_loghandler.h:
- log write hooks called while the log's lock is held (inwrite_rec_hook)
now need the MARIA_SHARE, like prewrite_rec_hook already had
- instead of a bool saying if this record's type ends groups or not,
we refine: it may not end a group, it may end a group, or it may
be a group in itself. Imagine that we had a physical write failure
to a table before we log the UNDO, we still end up in
external_lock(F_UNLCK) and then we log a COMMIT: we don't want
to consider this COMMIT as ending the group of REDOs (don't want
to execute those REDOs during Recovery), that's why we say "COMMIT
is a group in itself, it aborts any previous group". This also
gives one more sanity check in maria_read_log.
storage/maria/ma_recovery.c:
New Recovery code, replacing the old pseudocode.
Most of maria_read_log moved here.
Call-able from ha_maria, but not enabled yet.
Compared to the previous version of maria_read_log, some bugs have
been fixed, debugging output can go to stdout or a disk file (for now
it's useful for me, later it can be changed), execution of
REDO_DROP_TABLE, REDO_DELETE_ALL, REDO_PURGE_BLOCKS has been added. Duplicate code
has been factored into functions. We abort an unfinished group
of records if we see a record which is a group in itself (like COMMIT).
No need for maria_panic() after a bug (which caused tables to not
be closed) was fixed; if there is yet another bug I prefer to see it.
When opening a table for Recovery, set data_file_length
and key_file_length to their real physical value (these are the
easiest state members to restore :). Warn us if the last page
was truncated (but Recovery handles it).
MARIA_SHARE::state::state::records is now partly recovered (not
idempotent, but works if recreating tables from scracth).
When applying a REDO to a page, stamp it with the UNDO's LSN
(current_group_end_lsn), not with the REDO's LSN; it makes
the table more identical to the original table (easier to compare
the two tables in the end).
Big thing missing: some types of REDOs are not handled,
and the UNDO phase does not exist (missing functions to execute UNDOs
to actually rollback). So for now tests are only inserting/deleting
a few 100 rows, closing the table and seeing if the log is applied ok;
it works. UPDATE not handled.
storage/maria/ma_recovery.h:
new functions: ma_recover() for recovery from inside ha_maria;
_ma_apply_log() for maria_read_log (ma_recover() calls _ma_apply_log()).
Btw, we need to not use the word "recover" for REPAIR/maria_chk anymore.
storage/maria/ma_rename.c:
don't write log records during recovery
storage/maria/ma_test2.c:
- fail if maria_info() or other subtests find some wrong information
- new option -g to skip updates.
- init the translog before creating the table, so that log applying
can work.
- in "#if 0" you'll see some fixed bugs (will be removed).
storage/maria/ma_test_all.sh:
cleanup files. Test log applying.
storage/maria/maria_read_log.c:
most of the logic moves to ma_recovery.c to be shared between
maria_read_log and recovery-from-inside-mysqld.
See ma_recovery.c for additional changes made to the moved code.
storage/maria/ma_test_recovery:
unit test for Recovery. Tests insert and delete,
REDO_UPDATE not yet coded.
Script is called from ma_test_all. Can run standalone.
2007-07-26 11:56:21 +02:00
|
|
|
case 'g':
|
|
|
|
skip_update= TRUE;
|
|
|
|
break;
|
2006-04-11 15:45:10 +02:00
|
|
|
case '?':
|
|
|
|
case 'I':
|
|
|
|
case 'V':
|
2007-01-18 20:38:14 +01:00
|
|
|
printf("%s Ver 1.0 for %s at %s\n",progname,SYSTEM_TYPE,MACHINE_TYPE);
|
2006-04-11 15:45:10 +02:00
|
|
|
puts("By Monty, for your professional use\n");
|
This patch is a collection of patches from from Sanja, Sergei and Monty.
Added logging and pinning of pages to block format.
Integration of transaction manager, log handler.
Better page cache intergration
Split trnman.h into two files, so that we don't have to include my_atomic.h into C++ programs.
Renaming of structures, more comments, more debugging etc.
Fixed problem with small head block + long varchar.
Added extra argument to delete_record() and update_record() (needed for UNDO logging)
Small changes to interface of pagecache and log handler.
Change initialization of log_record_type_descriptors to not be depending on enum order.
Use array of LEX_STRING's to send data to log handler
Added 'dummy' transaction option to MARIA_INFO so that we can always assume 'trn' exists.
include/lf.h:
Interface fixes
Rename of structures
(Patch from Sergei via Sanja)
include/my_atomic.h:
More comments
include/my_global.h:
Added MY_ERRPTR
include/pagecache.h:
Added undo LSN when unlocking pages
mysql-test/r/maria.result:
Updated results
mysql-test/t/maria.test:
Added autocommit around lock tables
(Patch from Sanja)
mysys/lf_alloc-pin.c:
Post-review fixes, simple optimizations
More comments
Struct slot renames
Check amount of memory on stack
(Patch from Sergei)
mysys/lf_dynarray.c:
More comments
mysys/lf_hash.c:
More comments
After review fixes
(Patch from Sergei)
storage/maria/ha_maria.cc:
Split trnman.h into two files, so that we don't have to include my_atomic.h into the .cc program.
(Temporary fix to avoid bug in gcc)
Move out all deferencing of the transaction structure.
Transaction manager integrated (Patch from Sergei)
storage/maria/ha_maria.h:
Added prototype for start_stmt()
storage/maria/lockman.c:
Function call rename
storage/maria/ma_bitmap.c:
Mark deleted pages free from page cache
storage/maria/ma_blockrec.c:
Offset -> rownr
More debugging
Fixed problem with small head block + long varchar
Added logging of changed pages
Added logging of undo (Including only loggging of changed fields in case of update)
Added pinning/unpinning of all changed pages
More comments
Added free_full_pages() as the same code was used in several places.
fill_rows_parts() renamed as fill_insert_undo_parts()
offset -> rownr
Added some optimization of not transactional tables
_ma_update_block_record() has new parameter, as we need original row to do efficent undo for update
storage/maria/ma_blockrec.h:
Added ROW_EXTENTS_ON_STACK
Changed prototype for update and delete of row
storage/maria/ma_check.c:
Added original row to delete_record() call
storage/maria/ma_control_file.h:
Added ifdefs for C++
storage/maria/ma_delete.c:
Added original row to delete_record() call
(Needed for efficent undo logging)
storage/maria/ma_dynrec.c:
Added extra argument to delete_record() and update_record()
Removed not used variable
storage/maria/ma_init.c:
Initialize log handler
storage/maria/ma_loghandler.c:
Removed not used variable
Change initialization of log_record_type_descriptors to not be depending on enum order
Use array of LEX_STRING's to send data to log handler
storage/maria/ma_loghandler.h:
New defines
Use array of LEX_STRING's to send data to log handler
storage/maria/ma_open.c:
Added 'dummy' transaction option to MARIA_INFO so that we can always assume 'trn' exists.
Store in MARIA_SHARE->page_type if pages will have up to date LSN's
storage/maria/ma_pagecache.c:
Don't decrease number of readers when using pagecache_write()/pagecache_read()
In pagecache_write() decrement request count if page was left pinned
Added pagecache_delete_pages()
Removed some casts
Make trace output consistent with rest of code
Simplify calling of DBUG_ASSERT(0)
Only update LSN if the LSN is bigger than what's already on the page
Added LSN parameter pagecache_unpin_page(), pagecache_unpin(), and pagecache_unlock()
(Part of patch from Sanja)
storage/maria/ma_static.c:
Added 'dummy' transaction option to MARIA_INFO so that we can always assume 'trn' exists.
Added default page cache
storage/maria/ma_statrec.c:
Added extra argument to delete_record() and update_record()
storage/maria/ma_test1.c:
Added option -T for transactions
storage/maria/ma_test2.c:
Added option -T for transactions
storage/maria/ma_test_all.sh:
Test with transactions
storage/maria/ma_update.c:
Changed prototype for update of row
storage/maria/maria_def.h:
Changed prototype for update & delete of row as block records need to access the old row
Store in MARIA_SHARE->page_type if pages will have up to date LSN's
Added MARIA_MAX_TREE_LEVELS to allow us to calculate the number of possible pinned pages we may need.
Removed not used 'empty_bits_buffer'
Added pointer to transaction object
Added array for pinned pages
Added log_row_parts array for logging of field data.
Added MARIA_PINNED_PAGE to store pinned pages
storage/maria/trnman.c:
Added accessor functions to transaction object
Added missing DBUG_RETURN()
More debugging
More comments
Changed // comment of code to #ifdef NOT_USED
Transaction manager integrated.
Post review fixes
Part of patch originally from Sergei
storage/maria/trnman.h:
Split trnman.h into two files, so that we don't have to include my_atomic.h into the .cc program.
(Temporary fix to avoid bug in gcc)
storage/maria/unittest/ma_pagecache_single.c:
Added missing argument
Added SKIP_BIG_TESTS
(Patch from Sanja)
storage/maria/unittest/ma_test_loghandler-t.c:
Test logging with new LEX_STRING parameter
(Patch from Sanja)
storage/maria/unittest/ma_test_loghandler_multigroup-t.c:
Test logging with new LEX_STRING parameter
(Patch from Sanja)
storage/maria/unittest/ma_test_loghandler_multithread-t.c:
Test logging with new LEX_STRING parameter
(Patch from Sanja)
storage/maria/unittest/ma_test_loghandler_pagecache-t.c:
Test logging with new LEX_STRING parameter
(Patch from Sanja)
storage/maria/unittest/trnman-t.c:
Stack overflow detection
(Patch from Sergei)
unittest/unit.pl:
Command-line options --big and --verbose
(Patch from Sergei)
unittest/mytap/tap.c:
Detect --big
(Patch from Sergei)
unittest/mytap/tap.h:
Skip_big_tests and SKIP_BIG_TESTS
(Patch from Sergei)
storage/maria/trnman_public.h:
New BitKeeper file ``storage/maria/trnman_public.h''
2007-05-29 19:13:56 +02:00
|
|
|
printf("Usage: %s [-?AbBcDIKLPRqSsTVWltv] [-k#] [-f#] [-m#] [-e#] [-E#] [-t#]\n",
|
2006-04-11 15:45:10 +02:00
|
|
|
progname);
|
|
|
|
exit(0);
|
|
|
|
case '#':
|
|
|
|
DBUG_PUSH (++pos);
|
|
|
|
break;
|
|
|
|
default:
|
|
|
|
printf("Illegal option: '%c'\n",*pos);
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return;
|
|
|
|
} /* get options */
|
|
|
|
|
|
|
|
/* Get a random value 0 <= x <= n */
|
|
|
|
|
|
|
|
static uint rnd(uint max_value)
|
|
|
|
{
|
|
|
|
return (uint) ((rand() & 32767)/32767.0*max_value);
|
|
|
|
} /* rnd */
|
|
|
|
|
|
|
|
|
|
|
|
/* Create a variable length record */
|
|
|
|
|
2007-07-02 19:45:15 +02:00
|
|
|
static void fix_length(uchar *rec, uint length)
|
2006-04-11 15:45:10 +02:00
|
|
|
{
|
|
|
|
bmove(rec+STANDARD_LENGTH,
|
|
|
|
"0123456789012345678901234567890123456789012345678901234567890",
|
|
|
|
length-STANDARD_LENGTH);
|
2007-10-04 19:33:42 +02:00
|
|
|
strfill((char*) rec+length,STANDARD_LENGTH+60-length,' ');
|
2006-04-11 15:45:10 +02:00
|
|
|
} /* fix_length */
|
|
|
|
|
|
|
|
|
|
|
|
/* Put maybe a blob in record */
|
|
|
|
|
2007-10-04 19:33:42 +02:00
|
|
|
static void put_blob_in_record(uchar *blob_pos, char **blob_buffer,
|
2007-01-18 20:38:14 +01:00
|
|
|
ulong *blob_length)
|
2006-04-11 15:45:10 +02:00
|
|
|
{
|
|
|
|
ulong i,length;
|
|
|
|
if (use_blob)
|
|
|
|
{
|
|
|
|
if (rnd(10) == 0)
|
|
|
|
{
|
|
|
|
if (! *blob_buffer &&
|
|
|
|
!(*blob_buffer=my_malloc((uint) use_blob,MYF(MY_WME))))
|
|
|
|
{
|
|
|
|
use_blob=0;
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
length=rnd(use_blob);
|
|
|
|
for (i=0 ; i < length ; i++)
|
|
|
|
(*blob_buffer)[i]=(char) (length+i);
|
|
|
|
int4store(blob_pos,length);
|
|
|
|
memcpy_fixed(blob_pos+4,(char*) blob_buffer,sizeof(char*));
|
2007-01-18 20:38:14 +01:00
|
|
|
*blob_length= length;
|
2006-04-11 15:45:10 +02:00
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
|
|
|
int4store(blob_pos,0);
|
2007-01-18 20:38:14 +01:00
|
|
|
*blob_length= 0;
|
2006-04-11 15:45:10 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
static void copy_key(MARIA_HA *info,uint inx,uchar *rec,uchar *key_buff)
|
|
|
|
{
|
|
|
|
HA_KEYSEG *keyseg;
|
|
|
|
|
|
|
|
for (keyseg=info->s->keyinfo[inx].seg ; keyseg->type ; keyseg++)
|
|
|
|
{
|
|
|
|
memcpy(key_buff,rec+keyseg->start,(size_t) keyseg->length);
|
|
|
|
key_buff+=keyseg->length;
|
|
|
|
}
|
|
|
|
return;
|
|
|
|
}
|