2006-04-11 16:45:10 +03:00
|
|
|
/* Copyright (C) 2006 MySQL AB & MySQL Finland AB & TCX DataKonsult AB
|
|
|
|
|
|
|
|
This program is free software; you can redistribute it and/or modify
|
|
|
|
it under the terms of the GNU General Public License as published by
|
2007-03-02 11:20:23 +01:00
|
|
|
the Free Software Foundation; version 2 of the License.
|
2006-04-11 16:45:10 +03:00
|
|
|
|
|
|
|
This program is distributed in the hope that it will be useful,
|
|
|
|
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
|
|
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
|
|
GNU General Public License for more details.
|
|
|
|
|
|
|
|
You should have received a copy of the GNU General Public License
|
|
|
|
along with this program; if not, write to the Free Software
|
|
|
|
Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */
|
|
|
|
|
|
|
|
/* open a isam-database */
|
|
|
|
|
|
|
|
#include "ma_fulltext.h"
|
|
|
|
#include "ma_sp_defs.h"
|
|
|
|
#include "ma_rt_index.h"
|
2007-01-18 21:38:14 +02:00
|
|
|
#include "ma_blockrec.h"
|
This patch is a collection of patches from from Sanja, Sergei and Monty.
Added logging and pinning of pages to block format.
Integration of transaction manager, log handler.
Better page cache intergration
Split trnman.h into two files, so that we don't have to include my_atomic.h into C++ programs.
Renaming of structures, more comments, more debugging etc.
Fixed problem with small head block + long varchar.
Added extra argument to delete_record() and update_record() (needed for UNDO logging)
Small changes to interface of pagecache and log handler.
Change initialization of log_record_type_descriptors to not be depending on enum order.
Use array of LEX_STRING's to send data to log handler
Added 'dummy' transaction option to MARIA_INFO so that we can always assume 'trn' exists.
include/lf.h:
Interface fixes
Rename of structures
(Patch from Sergei via Sanja)
include/my_atomic.h:
More comments
include/my_global.h:
Added MY_ERRPTR
include/pagecache.h:
Added undo LSN when unlocking pages
mysql-test/r/maria.result:
Updated results
mysql-test/t/maria.test:
Added autocommit around lock tables
(Patch from Sanja)
mysys/lf_alloc-pin.c:
Post-review fixes, simple optimizations
More comments
Struct slot renames
Check amount of memory on stack
(Patch from Sergei)
mysys/lf_dynarray.c:
More comments
mysys/lf_hash.c:
More comments
After review fixes
(Patch from Sergei)
storage/maria/ha_maria.cc:
Split trnman.h into two files, so that we don't have to include my_atomic.h into the .cc program.
(Temporary fix to avoid bug in gcc)
Move out all deferencing of the transaction structure.
Transaction manager integrated (Patch from Sergei)
storage/maria/ha_maria.h:
Added prototype for start_stmt()
storage/maria/lockman.c:
Function call rename
storage/maria/ma_bitmap.c:
Mark deleted pages free from page cache
storage/maria/ma_blockrec.c:
Offset -> rownr
More debugging
Fixed problem with small head block + long varchar
Added logging of changed pages
Added logging of undo (Including only loggging of changed fields in case of update)
Added pinning/unpinning of all changed pages
More comments
Added free_full_pages() as the same code was used in several places.
fill_rows_parts() renamed as fill_insert_undo_parts()
offset -> rownr
Added some optimization of not transactional tables
_ma_update_block_record() has new parameter, as we need original row to do efficent undo for update
storage/maria/ma_blockrec.h:
Added ROW_EXTENTS_ON_STACK
Changed prototype for update and delete of row
storage/maria/ma_check.c:
Added original row to delete_record() call
storage/maria/ma_control_file.h:
Added ifdefs for C++
storage/maria/ma_delete.c:
Added original row to delete_record() call
(Needed for efficent undo logging)
storage/maria/ma_dynrec.c:
Added extra argument to delete_record() and update_record()
Removed not used variable
storage/maria/ma_init.c:
Initialize log handler
storage/maria/ma_loghandler.c:
Removed not used variable
Change initialization of log_record_type_descriptors to not be depending on enum order
Use array of LEX_STRING's to send data to log handler
storage/maria/ma_loghandler.h:
New defines
Use array of LEX_STRING's to send data to log handler
storage/maria/ma_open.c:
Added 'dummy' transaction option to MARIA_INFO so that we can always assume 'trn' exists.
Store in MARIA_SHARE->page_type if pages will have up to date LSN's
storage/maria/ma_pagecache.c:
Don't decrease number of readers when using pagecache_write()/pagecache_read()
In pagecache_write() decrement request count if page was left pinned
Added pagecache_delete_pages()
Removed some casts
Make trace output consistent with rest of code
Simplify calling of DBUG_ASSERT(0)
Only update LSN if the LSN is bigger than what's already on the page
Added LSN parameter pagecache_unpin_page(), pagecache_unpin(), and pagecache_unlock()
(Part of patch from Sanja)
storage/maria/ma_static.c:
Added 'dummy' transaction option to MARIA_INFO so that we can always assume 'trn' exists.
Added default page cache
storage/maria/ma_statrec.c:
Added extra argument to delete_record() and update_record()
storage/maria/ma_test1.c:
Added option -T for transactions
storage/maria/ma_test2.c:
Added option -T for transactions
storage/maria/ma_test_all.sh:
Test with transactions
storage/maria/ma_update.c:
Changed prototype for update of row
storage/maria/maria_def.h:
Changed prototype for update & delete of row as block records need to access the old row
Store in MARIA_SHARE->page_type if pages will have up to date LSN's
Added MARIA_MAX_TREE_LEVELS to allow us to calculate the number of possible pinned pages we may need.
Removed not used 'empty_bits_buffer'
Added pointer to transaction object
Added array for pinned pages
Added log_row_parts array for logging of field data.
Added MARIA_PINNED_PAGE to store pinned pages
storage/maria/trnman.c:
Added accessor functions to transaction object
Added missing DBUG_RETURN()
More debugging
More comments
Changed // comment of code to #ifdef NOT_USED
Transaction manager integrated.
Post review fixes
Part of patch originally from Sergei
storage/maria/trnman.h:
Split trnman.h into two files, so that we don't have to include my_atomic.h into the .cc program.
(Temporary fix to avoid bug in gcc)
storage/maria/unittest/ma_pagecache_single.c:
Added missing argument
Added SKIP_BIG_TESTS
(Patch from Sanja)
storage/maria/unittest/ma_test_loghandler-t.c:
Test logging with new LEX_STRING parameter
(Patch from Sanja)
storage/maria/unittest/ma_test_loghandler_multigroup-t.c:
Test logging with new LEX_STRING parameter
(Patch from Sanja)
storage/maria/unittest/ma_test_loghandler_multithread-t.c:
Test logging with new LEX_STRING parameter
(Patch from Sanja)
storage/maria/unittest/ma_test_loghandler_pagecache-t.c:
Test logging with new LEX_STRING parameter
(Patch from Sanja)
storage/maria/unittest/trnman-t.c:
Stack overflow detection
(Patch from Sergei)
unittest/unit.pl:
Command-line options --big and --verbose
(Patch from Sergei)
unittest/mytap/tap.c:
Detect --big
(Patch from Sergei)
unittest/mytap/tap.h:
Skip_big_tests and SKIP_BIG_TESTS
(Patch from Sergei)
storage/maria/trnman_public.h:
New BitKeeper file ``storage/maria/trnman_public.h''
2007-05-29 20:13:56 +03:00
|
|
|
#include "trnman.h"
|
2006-04-11 16:45:10 +03:00
|
|
|
#include <m_ctype.h>
|
|
|
|
|
|
|
|
#if defined(MSDOS) || defined(__WIN__)
|
|
|
|
#ifdef __WIN__
|
|
|
|
#include <fcntl.h>
|
|
|
|
#else
|
|
|
|
#include <process.h> /* Prototype for getpid */
|
|
|
|
#endif
|
|
|
|
#endif
|
|
|
|
#ifdef VMS
|
|
|
|
#include "static.c"
|
|
|
|
#endif
|
|
|
|
|
|
|
|
static void setup_key_functions(MARIA_KEYDEF *keyinfo);
|
2007-01-18 21:38:14 +02:00
|
|
|
static my_bool maria_scan_init_dummy(MARIA_HA *info);
|
|
|
|
static void maria_scan_end_dummy(MARIA_HA *info);
|
|
|
|
static my_bool maria_once_init_dummy(MARIA_SHARE *, File);
|
|
|
|
static my_bool maria_once_end_dummy(MARIA_SHARE *);
|
2007-07-02 20:45:15 +03:00
|
|
|
static uchar *_ma_base_info_read(uchar *ptr, MARIA_BASE_INFO *base);
|
- speed optimization:
minimize writes to transactional Maria tables: don't write
data pages, state, and open_count at the end of each statement.
Data pages will be written by a background thread periodically.
State will be written by Checkpoint periodically.
open_count serves to detect when a table is potentially damaged
due to an unclean mysqld stop, but thanks to recovery an unclean
mysqld stop will be corrected and so open_count becomes useless.
As state is written less often, it is often obsolete on disk,
we thus should avoid to read it from disk.
- by removing the data page writes above, it is necessary to put
it back at the start of some statements like check, repair and
delete_all. It was already necessary in fact (see ma_delete_all.c).
- disabling CACHE INDEX on Maria tables for now (fixes crash
of test 'key_cache' when run with --default-storage-engine=maria).
- correcting some fishy code in maria_extra.c (we possibly could lose
index pages when doing a DROP TABLE under Windows, in theory).
storage/maria/ha_maria.cc:
disable CACHE INDEX in Maria for now (there is a single cache for now),
it crashes and it's not a priority
storage/maria/ma_bitmap.c:
debug message
storage/maria/ma_check.c:
The statement before maria_repair() may not flush state,
so it needs to be done by maria_repair() (indeed this function
uses maria_open(HA_OPEN_COPY) so reads state from disk,
so needs to find it up-to-date on disk).
For safety (but normally this is not needed) we remove index blocks
out of the cache before repairing.
_ma_flush_blocks() becomes _ma_flush_table_files_after_repair():
it now additionally flushes the data file and state and syncs files.
As a side effect, the assertion "no WRITE_CACHE_USED" from
_ma_flush_table_files() fired so we move all end_io_cache() done
at the end of repair to before the calls to _ma_flush_table_files_after_repair().
storage/maria/ma_close.c:
when closing a transactional table, we fsync it. But we need to
do this only after writing its state.
We need to write the state at close time only for transactional
tables (the other tables do that at last unlock).
Putting back the O_RDONLY||crashed condition which I had
removed earlier.
Unmap the file before syncing it (does not matter now as Maria
does not use mmap)
storage/maria/ma_delete_all.c:
need to flush data pages before chsize-ing it. Was needed even when
we flushed data pages at the end of each statement, because we didn't
anyway do it if under LOCK TABLES: the change here thus fixes this bug:
create table t(a int) engine=maria;lock tables t write;
insert into t values(1);delete from t;unlock tables;check table t;
"Size of datafile is: 16384 Should be: 8192"
(an obsolete page went to disk after the chsize(), at unlock time).
storage/maria/ma_extra.c:
When doing share->last_version=0, we make the MARIA_SHARE-in-memory
invisible to future openers, so need to have an up-to-date state
on disk for them. The same way, future openers will reopen the data
and index file, so they will not find our cached blocks, so we
need to flush them to disk.
In HA_EXTRA_FORCE_REOPEN, this probably happens naturally as all
tables normally get closed, we however add a safety flush.
In HA_EXTRA_PREPARE_FOR_RENAME, we need to do the flushing. On
Windows we additionally need to close files.
In HA_EXTRA_PREPARE_FOR_DROP, we don't need to flush anything but
remove dirty cached blocks from memory. On Windows we need to close
files.
Closing files forces us to sync them before (requirement for transactional
tables).
For mutex reasons (don't lock intern_lock twice), we move
maria_lock_database() and _ma_decrement_open_count() first in the list
of operations.
Flush also data file in HA_EXTRA_FLUSH.
storage/maria/ma_locking.c:
For transactional tables:
- don't write data pages / state at unlock time;
as a consequence, "share->changed=0" cannot be done.
- don't write state in _ma_writeinfo()
- don't maintain open_count on disk (Recovery corrects the table in case of crash
anyway, and we gain speed by not writing open_count to disk),
For non-transactional tables, flush the state at unlock only
if the table was changed (optimization).
Code which read the state from disk is relevant only with
external locking, we disable it (if want to re-enable it, it shouldn't
for transactional tables as state on disk may be obsolete (such tables
does not flush state at unlock anymore).
The comment "We have to flush the write cache" is now wrong because
maria_lock_database(F_UNLCK) now happens before thr_unlock(), and
we are not using external locking.
storage/maria/ma_open.c:
_ma_state_info_read() is only used in ma_open.c, making it static
storage/maria/ma_recovery.c:
set MARIA_SHARE::changed to TRUE when we are going to apply a
REDO/UNDO, so that the state gets flushed at close.
storage/maria/ma_test_recovery.expected:
Changes introduced by this patch:
- good: the "open" (table open, not properly closed) is gone,
it was pointless for a recovered table
- bad: stemming from different moments of writing the index's state
probably (_ma_writeinfo() used to write the state after every row
write in ma_test* programs, doesn't anymore as the table is
transactional): some differences in indexes (not relevant as we don't
yet have recovery for them); some differences in count of records
(changed from a wrong value to another wrong value) (not relevant
as we don't recover this count correctly yet anyway, though
a patch will be pushed soon).
storage/maria/ma_test_recovery:
for repeatable output, no names of varying directories.
storage/maria/maria_chk.c:
function renamed
storage/maria/maria_def.h:
Function became local to ma_open.c. Function renamed.
2007-09-06 16:53:26 +02:00
|
|
|
static uchar *_ma_state_info_read(uchar *ptr, MARIA_STATE_INFO *state);
|
2007-01-18 21:38:14 +02:00
|
|
|
|
2006-04-11 16:45:10 +03:00
|
|
|
#define get_next_element(to,pos,size) { memcpy((char*) to,pos,(size_t) size); \
|
|
|
|
pos+=size;}
|
|
|
|
|
|
|
|
|
|
|
|
#define disk_pos_assert(pos, end_pos) \
|
|
|
|
if (pos > end_pos) \
|
|
|
|
{ \
|
|
|
|
my_errno=HA_ERR_CRASHED; \
|
|
|
|
goto err; \
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
/******************************************************************************
|
|
|
|
** Return the shared struct if the table is already open.
|
|
|
|
** In MySQL the server will handle version issues.
|
|
|
|
******************************************************************************/
|
|
|
|
|
2007-09-27 14:18:28 +03:00
|
|
|
MARIA_HA *_ma_test_if_reopen(const char *filename)
|
2006-04-11 16:45:10 +03:00
|
|
|
{
|
|
|
|
LIST *pos;
|
|
|
|
|
|
|
|
for (pos=maria_open_list ; pos ; pos=pos->next)
|
|
|
|
{
|
|
|
|
MARIA_HA *info=(MARIA_HA*) pos->data;
|
|
|
|
MARIA_SHARE *share=info->s;
|
|
|
|
if (!strcmp(share->unique_file_name,filename) && share->last_version)
|
|
|
|
return info;
|
|
|
|
}
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
|
2007-06-07 01:01:43 +03:00
|
|
|
/*
|
|
|
|
Open a new instance of an already opened Maria table
|
|
|
|
|
|
|
|
SYNOPSIS
|
|
|
|
maria_clone_internal()
|
|
|
|
share Share of already open table
|
|
|
|
mode Mode of table (O_RDONLY | O_RDWR)
|
|
|
|
data_file Filedescriptor of data file to use < 0 if one should open
|
|
|
|
open it.
|
|
|
|
|
|
|
|
RETURN
|
|
|
|
# Maria handler
|
|
|
|
0 Error
|
|
|
|
*/
|
|
|
|
|
|
|
|
|
|
|
|
static MARIA_HA *maria_clone_internal(MARIA_SHARE *share, int mode,
|
|
|
|
File data_file)
|
|
|
|
{
|
|
|
|
int save_errno;
|
|
|
|
uint errpos;
|
|
|
|
MARIA_HA info,*m_info;
|
2007-09-09 19:15:10 +03:00
|
|
|
my_bitmap_map *changed_fields_bitmap;
|
2007-06-07 01:01:43 +03:00
|
|
|
DBUG_ENTER("maria_clone_internal");
|
|
|
|
|
|
|
|
errpos= 0;
|
2007-07-02 20:45:15 +03:00
|
|
|
bzero((uchar*) &info,sizeof(info));
|
2007-06-07 01:01:43 +03:00
|
|
|
|
|
|
|
if (mode == O_RDWR && share->mode == O_RDONLY)
|
|
|
|
{
|
|
|
|
my_errno=EACCES; /* Can't open in write mode */
|
|
|
|
goto err;
|
|
|
|
}
|
|
|
|
if (data_file >= 0)
|
|
|
|
info.dfile.file= data_file;
|
|
|
|
else if (_ma_open_datafile(&info, share, -1))
|
|
|
|
goto err;
|
|
|
|
errpos= 5;
|
|
|
|
|
|
|
|
/* alloc and set up private structure parts */
|
|
|
|
if (!my_multi_malloc(MY_WME,
|
|
|
|
&m_info,sizeof(MARIA_HA),
|
|
|
|
&info.blobs,sizeof(MARIA_BLOB)*share->base.blobs,
|
|
|
|
&info.buff,(share->base.max_key_block_length*2+
|
|
|
|
share->base.max_key_length),
|
|
|
|
&info.lastkey,share->base.max_key_length*3+1,
|
|
|
|
&info.first_mbr_key, share->base.max_key_length,
|
|
|
|
&info.maria_rtree_recursion_state,
|
|
|
|
share->have_rtree ? 1024 : 0,
|
2007-09-09 19:15:10 +03:00
|
|
|
&changed_fields_bitmap,
|
|
|
|
bitmap_buffer_size(share->base.fields),
|
2007-06-07 01:01:43 +03:00
|
|
|
NullS))
|
|
|
|
goto err;
|
|
|
|
errpos= 6;
|
|
|
|
|
|
|
|
memcpy(info.blobs,share->blobs,sizeof(MARIA_BLOB)*share->base.blobs);
|
|
|
|
info.lastkey2=info.lastkey+share->base.max_key_length;
|
|
|
|
|
|
|
|
info.s=share;
|
|
|
|
info.cur_row.lastpos= HA_OFFSET_ERROR;
|
|
|
|
info.update= (short) (HA_STATE_NEXT_FOUND+HA_STATE_PREV_FOUND);
|
|
|
|
info.opt_flag=READ_CHECK_USED;
|
|
|
|
info.this_unique= (ulong) info.dfile.file; /* Uniq number in process */
|
|
|
|
if (share->data_file_type == COMPRESSED_RECORD)
|
|
|
|
info.this_unique= share->state.unique;
|
|
|
|
info.this_loop=0; /* Update counter */
|
|
|
|
info.last_unique= share->state.unique;
|
|
|
|
info.last_loop= share->state.update_count;
|
|
|
|
info.lock_type=F_UNLCK;
|
|
|
|
info.quick_mode=0;
|
|
|
|
info.bulk_insert=0;
|
|
|
|
info.ft1_to_ft2=0;
|
|
|
|
info.errkey= -1;
|
|
|
|
info.page_changed=1;
|
|
|
|
info.keyread_buff= info.buff + share->base.max_key_block_length;
|
2007-09-09 19:15:10 +03:00
|
|
|
bitmap_init(&info.changed_fields, changed_fields_bitmap,
|
|
|
|
share->base.fields, 0);
|
2007-06-07 01:01:43 +03:00
|
|
|
if ((*share->init)(&info))
|
|
|
|
goto err;
|
|
|
|
|
|
|
|
pthread_mutex_lock(&share->intern_lock);
|
|
|
|
info.read_record= share->read_record;
|
|
|
|
share->reopen++;
|
|
|
|
share->write_flag=MYF(MY_NABP | MY_WAIT_IF_FULL);
|
|
|
|
if (share->options & HA_OPTION_READ_ONLY_DATA)
|
|
|
|
{
|
|
|
|
info.lock_type=F_RDLCK;
|
|
|
|
share->r_locks++;
|
|
|
|
share->tot_locks++;
|
|
|
|
}
|
|
|
|
if (share->options & HA_OPTION_TMP_TABLE)
|
|
|
|
{
|
|
|
|
share->temporary= share->delay_key_write= 1;
|
|
|
|
|
|
|
|
share->write_flag=MYF(MY_NABP);
|
|
|
|
share->w_locks++; /* We don't have to update status */
|
|
|
|
share->tot_locks++;
|
|
|
|
info.lock_type=F_WRLCK;
|
|
|
|
}
|
|
|
|
if ((share->options & HA_OPTION_DELAY_KEY_WRITE) &&
|
|
|
|
maria_delay_key_write)
|
|
|
|
share->delay_key_write=1;
|
|
|
|
|
|
|
|
info.state= &share->state.state; /* Change global values by default */
|
WL#3072 - Maria recovery.
* Recovery of the table's live checksum (CREATE TABLE ... CHECKSUM=1)
is achieved in this patch. The table's live checksum
(info->s->state.state.checksum) is updated in inwrite_rec_hook's
under the log mutex when writing UNDO_ROW_INSERT|UPDATE|DELETE
and REDO_DELETE_ALL. The checksum variation caused by the operation
is stored in these UNDOs, so that the REDO phase, when it sees such
UNDOs, can update the live checksum if it is older (state.is_of_lsn is
lower) than the record. It is also used, as a nice add-on with no
cost, to do less row checksum computation during the UNDO phase
(as we have it in the record already).
Doing this work, it became pressing to move in-write hooks
(write_hook_for_redo() et al) to ma_blockrec.c.
The 'parts' argument of inwrite_rec_hook is unpredictable (it comes
mangled at this stage, for example by LSN compression) so it is
replaced by a 'void* hook_arg', which is used to pass down information,
currently only to write_hook_for_clr_end() (previous undo_lsn and
type of undone record).
* If from ha_maria, we print to stderr how many seconds (with one
fractional digit) the REDO phase took, same for UNDO phase and for
final table close. Just to give an indication for debugging and maybe
also for Support.
storage/maria/ha_maria.cc:
question for Monty
storage/maria/ma_blockrec.c:
* log in-write hooks (write_hook_for_redo() etc) move from
ma_loghandler.c to here; this is natural: the hooks are coupled
to their callers (functions in ma_blockrec.c).
* translog_write_record() now has a new argument "hook_arg";
using it to pass down to write_hook_for_clr_end() the transaction's
previous_undo_lsn and the type of the being undone record, and also
to pass down to all UNDOs the live checksum variation caused by the
operation.
* If table has live checksum, store in UNDO_ROW_INSERT|UPDATE|DELETE
and in CLR_END the checksum variation ("delta") caused by the
operation. For example if a DELETE caused the table's live checksum
to change from 123 to 456, we store in the UNDO_ROW_DELETE, in 4 bytes,
the value 333 (456-123).
* Instead of hard-coded "1" as length of the place where we store
the undone record's type in CLR_END, use a symbol CLR_TYPE_STORE_SIZE;
use macros clr_type_store and clr_type_korr.
* write_block_record() has a new parameter 'old_record_checksum'
which is the pre-computed checksum of old_record; that value is used
to update the table's live checksum when writing UNDO_ROW_UPDATE|CLR_END.
* In allocate_write_block_record(), if we are executing UNDO_ROW_DELETE
the row's checksum is already computed.
* _ma_update_block_record2() now expect the new row's checksum into
cur_row.checksum (was already true) and the old row's checksum into
new_row.checksum (that's new). Its two callers, maria_update() and
_ma_apply_undo_row_update(), honour this.
* When executing an UNDO_ROW_INSERT|UPDATE|DELETE in UNDO phase, pick
up the checksum delta from the log record. It is then used to update
the table's live checksum when writing CLR_END, and saves us a
computation of record.
storage/maria/ma_blockrec.h:
in-write hooks move from ma_loghandler.c
storage/maria/ma_check.c:
more straightforward size of buffer
storage/maria/ma_checkpoint.c:
<= is enough
storage/maria/ma_commit.c:
new prototype of translog_write_record()
storage/maria/ma_create.c:
new prototype of translog_write_record()
storage/maria/ma_delete.c:
The row's checksum must be computed before calling(*delete_record)(),
not after, because it must be known inside _ma_delete_block_record()
(to update the table's live checksum when writing UNDO_ROW_DELETE).
If deleting from a transactional table, live checksum was already updated
when writing UNDO_ROW_DELETE.
storage/maria/ma_delete_all.c:
@todo is now done (in ma_loghandler.c)
storage/maria/ma_delete_table.c:
new prototype of translog_write_record()
storage/maria/ma_loghandler.c:
* in-write hooks move to ma_blockrec.c.
* translog_write_record() gets a new argument 'hook_arg' which is
passed down to pre|inwrite_rec_hook. It is more useful that 'parts'
for those hooks, because when those hooks are called, 'parts' has
possibly been mangled (like with LSN compression) and is so
unpredictable.
* fix for compiler warning (unused buffer_start when compiling without
debug support)
* Because checksum delta is stored into UNDO_ROW_INSERT|UPDATE|DELETE
and CLR_END, but only if the table has live checksum, these records
are not PSEUDOFIXEDLENGTH anymore, they are now VARIABLE_LENGTH (their
length is X if no live checksum and X+4 otherwise).
* add an inwrite_rec_hook for UNDO_ROW_UPDATE, which updates the
table's live checksum. Update it also in hooks of UNDO_ROW_INSERT|
DELETE and REDO_DELETE_ALL and CLR_END.
* Bugfix: when reading a record in translog_read_record(), it happened
that "length" became negative, because the function assumed that
the record extended beyond the page's end, whereas it may be shorter.
storage/maria/ma_loghandler.h:
* Instead of hard-coded "1" and "4", use symbols and macros
to store/retrieve the type of record which the CLR_END corresponds
to, and the checksum variation caused by the operation which logs the
record
* translog_write_record() gets a new argument 'hook_arg' which is
passed down to pre|inwrite_rec_hook. It is more useful that 'parts'
for those hooks, because when those hooks are called, 'parts' has
possibly been mangled (like with LSN compression) and is so
unpredictable.
storage/maria/ma_open.c:
fix for "empty body in if() statement" (when compiling without safemutex)
storage/maria/ma_pagecache.c:
<= is enough
storage/maria/ma_recovery.c:
* print the time that each recovery phase (REDO/UNDO/flush) took;
this is enabled only when recovering from ha_maria. Is it printed
n seconds with a fractional part of one digit (like 123.4 seconds).
* In the REDO phase, update the table's live checksum by using
the checksum delta stored in UNDO_ROW_INSERT|DELETE|UPDATE and CLR_END.
Update it too when seeing REDO_DELETE_ALL.
* In the UNDO phase, when executing UNDO_ROW_INSERT, if the table does
not have live checksum then reading the record's header (as done by
the master loop of run_undo_phase()) is enough; otherwise we
do a translog_read_record() to have the checksum delta ready
for _ma_apply_undo_row_insert().
* When at the end of the REDO phase we notice that there is an unfinished
group of REDOs, don't assert in debug binaries, as I verified that it
can happen in real life (with kill -9)
* removing ' in #error as it confuses gcc3
storage/maria/ma_rename.c:
new prototype of translog_write_record()
storage/maria/ma_test_recovery.expected:
Change in output of ma_test_recovery: now all live checksums of
original tables equal those of tables recreated by the REDO phase
and those of tables fixed by the UNDO phase. I.e. recovery of
the live checksum looks like working (which was after all the only
goal of this changeset).
I checked by hand that it's not just all live checksums which are
now 0 and that's why they match. They are the old values like
3757530372. maria.test has hard-coded checksum values in its result
file so checks this too.
storage/maria/ma_update.c:
* It's useless to put up HA_STATE_CHANGED in 'key_changed',
as we put up HA_STATE_CHANGED in info->update anyway.
* We need to compute the old and new rows' checksum before calling
(*update_record)(), as checksum delta must be known when logging
UNDO_ROW_UPDATE which is done by _ma_update_block_record(). Note that
some functions change the 'newrec' record (at least _ma_check_unique()
does) so we cannot move the checksum computation too early in the
function.
storage/maria/ma_write.c:
If inserting into a transactional table, live's checksum was
already updated when writing UNDO_ROW_INSERT. The multiplication
is a trick to save an if().
storage/maria/unittest/ma_test_loghandler-t.c:
new prototype of translog_write_record()
storage/maria/unittest/ma_test_loghandler_first_lsn-t.c:
new prototype of translog_write_record()
storage/maria/unittest/ma_test_loghandler_max_lsn-t.c:
new prototype of translog_write_record()
storage/maria/unittest/ma_test_loghandler_multigroup-t.c:
new prototype of translog_write_record()
storage/maria/unittest/ma_test_loghandler_multithread-t.c:
new prototype of translog_write_record()
storage/maria/unittest/ma_test_loghandler_noflush-t.c:
new prototype of translog_write_record()
storage/maria/unittest/ma_test_loghandler_pagecache-t.c:
new prototype of translog_write_record()
storage/maria/unittest/ma_test_loghandler_purge-t.c:
new prototype of translog_write_record()
storage/myisam/sort.c:
fix for compiler warnings in pushbuild (write_merge_key* functions
didn't have their declaration match MARIA_HA::write_key).
2007-10-02 18:02:09 +02:00
|
|
|
if (!share->base.born_transactional) /* For transactional ones ... */
|
WL#3072 Maria recovery
* create page cache before initializing engine and not after, because
Maria's recovery needs a page cache
* make the creation of a bitmap page more crash-resistent
* bugfix (see ma_blockrec.c)
* back to old way: create an 8k bitmap page when creating table
* preparations for the UNDO phase: recreate TRNs
* preparations for Checkpoint: list of dirty pages, testing
of rec_lsn to know if page should be skipped during Recovery
(unused in this patch as no Checkpoint module pushed yet)
* maria_chk tags repaired table with a special LSN
* reworking all around in ma_recovery.c (less duplication)
mysys/my_realloc.c:
noted an issue in my_realloc()
sql/mysqld.cc:
page cache needs to be created before engines are initialized,
because Maria's initialization may do a recovery which needs
the page cache.
storage/maria/ha_maria.cc:
update to new prototype
storage/maria/ma_bitmap.c:
when creating the first bitmap page we used chsize to 8192 bytes then
pwrite (overwrite) the last 2 bytes (8191-8192). If crash between
the two operations, this leaves a bitmap page full without its end
marker. A later recovery may try to read this page and find it
exists and misses a marker and conclude it's corrupted and fail.
Changing the chsize to only 8190 bytes: recovery will then find
the page is too short and recreate it entirely.
storage/maria/ma_blockrec.c:
Fix for a bug: when executing a REDO, if the data page is created,
data_file_length was increased before _ma_bitmap_set():
_ma_bitmap_set() called _ma_read_bitmap_page() which, due to the
increased data_file_length, expected to find a bitmap page on disk
with a correct end marker; if the bitmap page didn't exist already
in fact, this failed. Fixed by increasing data_file_length only after
_ma_read_bitmap_page() has created the new bitmap page correctly.
This bug could happen every time a REDO is about creating a new
bitmap page.
storage/maria/ma_check.c:
empty data file has a bitmap page
storage/maria/ma_control_file.c:
useless parameter to ma_control_file_create_or_open(), just
test if this is recovery.
storage/maria/ma_control_file.h:
new prototype
storage/maria/ma_create.c:
Back to how it was before: maria_create() creates an 8k bitmap page.
Thus (bugfix) data_file_length needs to reflect this instead of being 0.
storage/maria/ma_loghandler.c:
as ma_test1 and ma_test2 now use real transactions and not
dummy_transaction_object, REDO for INSERT/UPDATE/DELETE are always
about real transactions, can assert this.
A function for Recovery to assign a short id to a table.
storage/maria/ma_loghandler.h:
new function
storage/maria/ma_loghandler_lsn.h:
maria_chk tags repaired tables with this LSN
storage/maria/ma_open.c:
* enforce that DMLs on transactional tables use real transactions
and not dummy_transaction_object.
* test if table was repaired with maria_chk (which has to been
seen as an import of an external table into the server), test
validity of create_rename_lsn (header corruption detection)
* comments.
storage/maria/ma_recovery.c:
* preparations for the UNDO phase: recreate TRNs
* preparations for Checkpoint: list of dirty pages, testing
of rec_lsn to know if page should be skipped during Recovery
(unused in this patch as no Checkpoint module pushed yet)
* reworking all around (less duplication)
storage/maria/ma_recovery.h:
a parameter to say if the UNDO phase should be skipped
storage/maria/maria_chk.c:
tag repaired tables with a special LSN
storage/maria/maria_read_log.c:
* update to new prototype
* no UNDO phase in maria_read_log for now
storage/maria/trnman.c:
* a function for Recovery to create a transaction (TRN), needed
in the UNDO phase
* a function for Recovery to grab an existing transaction, needed
in the UNDO phase (rollback all existing transactions)
storage/maria/trnman_public.h:
new functions
2007-08-29 16:43:01 +02:00
|
|
|
info.trn= &dummy_transaction_object; /* ... force crash if no trn given */
|
2007-06-07 01:01:43 +03:00
|
|
|
pthread_mutex_unlock(&share->intern_lock);
|
|
|
|
|
|
|
|
/* Allocate buffer for one record */
|
|
|
|
/* prerequisites: info->rec_buffer == 0 && info->rec_buff_size == 0 */
|
|
|
|
if (_ma_alloc_buffer(&info.rec_buff, &info.rec_buff_size,
|
|
|
|
share->base.default_rec_buff_size))
|
|
|
|
goto err;
|
|
|
|
|
|
|
|
bzero(info.rec_buff, share->base.default_rec_buff_size);
|
|
|
|
|
|
|
|
*m_info=info;
|
|
|
|
#ifdef THREAD
|
|
|
|
thr_lock_data_init(&share->lock,&m_info->lock,(void*) m_info);
|
|
|
|
#endif
|
|
|
|
m_info->open_list.data=(void*) m_info;
|
|
|
|
maria_open_list=list_add(maria_open_list,&m_info->open_list);
|
|
|
|
|
|
|
|
DBUG_RETURN(m_info);
|
|
|
|
|
|
|
|
err:
|
|
|
|
save_errno=my_errno ? my_errno : HA_ERR_END_OF_FILE;
|
|
|
|
if ((save_errno == HA_ERR_CRASHED) ||
|
|
|
|
(save_errno == HA_ERR_CRASHED_ON_USAGE) ||
|
|
|
|
(save_errno == HA_ERR_CRASHED_ON_REPAIR))
|
|
|
|
_ma_report_error(save_errno, share->open_file_name);
|
|
|
|
switch (errpos) {
|
|
|
|
case 6:
|
|
|
|
(*share->end)(&info);
|
2007-07-02 20:45:15 +03:00
|
|
|
my_free((uchar*) m_info,MYF(0));
|
2007-06-07 01:01:43 +03:00
|
|
|
/* fall through */
|
|
|
|
case 5:
|
|
|
|
if (data_file < 0)
|
|
|
|
VOID(my_close(info.dfile.file, MYF(0)));
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
my_errno=save_errno;
|
|
|
|
DBUG_RETURN (NULL);
|
|
|
|
} /* maria_clone_internal */
|
|
|
|
|
|
|
|
|
|
|
|
/* Make a clone of a maria table */
|
|
|
|
|
|
|
|
MARIA_HA *maria_clone(MARIA_SHARE *share, int mode)
|
|
|
|
{
|
|
|
|
MARIA_HA *new_info;
|
|
|
|
pthread_mutex_lock(&THR_LOCK_maria);
|
|
|
|
new_info= maria_clone_internal(share, mode,
|
|
|
|
share->data_file_type == BLOCK_RECORD ?
|
|
|
|
share->bitmap.file.file : -1);
|
|
|
|
pthread_mutex_unlock(&THR_LOCK_maria);
|
|
|
|
return new_info;
|
|
|
|
}
|
|
|
|
|
|
|
|
|
2006-04-11 16:45:10 +03:00
|
|
|
/******************************************************************************
|
2007-06-07 01:01:43 +03:00
|
|
|
open a MARIA table
|
|
|
|
|
2006-04-11 16:45:10 +03:00
|
|
|
See my_base.h for the handle_locking argument
|
|
|
|
if handle_locking and HA_OPEN_ABORT_IF_CRASHED then abort if the table
|
|
|
|
is marked crashed or if we are not using locking and the table doesn't
|
|
|
|
have an open count of 0.
|
|
|
|
******************************************************************************/
|
|
|
|
|
|
|
|
MARIA_HA *maria_open(const char *name, int mode, uint open_flags)
|
|
|
|
{
|
2007-06-07 01:01:43 +03:00
|
|
|
int kfile,open_mode,save_errno;
|
2007-01-18 21:38:14 +02:00
|
|
|
uint i,j,len,errpos,head_length,base_pos,info_length,keys,
|
2006-04-11 16:45:10 +03:00
|
|
|
key_parts,unique_key_parts,fulltext_keys,uniques;
|
|
|
|
char name_buff[FN_REFLEN], org_name[FN_REFLEN], index_name[FN_REFLEN],
|
|
|
|
data_name[FN_REFLEN];
|
2007-10-04 20:33:42 +03:00
|
|
|
uchar *disk_cache, *disk_pos, *end_pos;
|
2006-04-11 16:45:10 +03:00
|
|
|
MARIA_HA info,*m_info,*old_info;
|
|
|
|
MARIA_SHARE share_buff,*share;
|
|
|
|
ulong rec_per_key_part[HA_MAX_POSSIBLE_KEY*HA_MAX_KEY_SEG];
|
2007-01-18 21:38:14 +02:00
|
|
|
my_off_t key_root[HA_MAX_POSSIBLE_KEY];
|
2006-04-11 16:45:10 +03:00
|
|
|
ulonglong max_key_file_length, max_data_file_length;
|
2007-06-07 01:01:43 +03:00
|
|
|
File data_file= -1;
|
2006-04-11 16:45:10 +03:00
|
|
|
DBUG_ENTER("maria_open");
|
|
|
|
|
|
|
|
LINT_INIT(m_info);
|
|
|
|
kfile= -1;
|
2007-01-18 21:38:14 +02:00
|
|
|
errpos= 0;
|
2006-04-11 16:45:10 +03:00
|
|
|
head_length=sizeof(share_buff.state.header);
|
2007-07-02 20:45:15 +03:00
|
|
|
bzero((uchar*) &info,sizeof(info));
|
2006-04-11 16:45:10 +03:00
|
|
|
|
|
|
|
my_realpath(name_buff, fn_format(org_name,name,"",MARIA_NAME_IEXT,
|
Completion of merge of mysql-5.1 into mysql-maria.
Manually imported changes done to MyISAM (include/myisam.h,
storage/myisam/*, sql/ha_myisam.*, mysql-test/t/myisam.test,
mysql-test/t/ps_2myisam.test) the last
months into Maria (tedious, should do it more frequently in the
future), including those not done at the previous 5.1->Maria merge
(please in the future don't forget to apply MyISAM changes to Maria
when you merge 5.1 into Maria).
Note: I didn't try to import anything which could be MyISAM-related
in other tests of mysql-test (I didn't want to dig in all csets),
but as QA is working to make most tests re-usable for other engines
(Falcon), it is likely that we'll benefit from this and just have
to set engine=Maria somewhere to run those tests on Maria.
func_group and partition tests fail but they already do in main 5.1
on my machine. No Valgrind error in t/*maria*.test.
Monty: please see the commit comment of maria.result and check.
BitKeeper/deleted/.del-ha_maria.m4:
Delete: config/ac-macros/ha_maria.m4
configure.in:
fix for the new way of enabling engines
include/maria.h:
importing changes done to MyISAM the last months into Maria
include/my_handler.h:
importing changes done to MyISAM the last months into Maria
include/myisam.h:
importing changes done to MyISAM the last months into Maria
mysql-test/r/maria.result:
identical to myisam.result, except the engine name in some places
AND in the line testing key_block_size=1000000000000000000:
Maria gives a key block size of 8192 while MyISAM gives 4096;
is it explainable by the difference between MARIA_KEY_BLOCK_LENGTH
and the same constant in MyISAM? Monty?
mysql-test/r/ps_maria.result:
identical to ps_2myisam.result (except the engine name in some places)
mysql-test/t/maria.test:
instead of engine=maria everywhere, I use @@storage_engine (reduces
the diff with myisam.test).
importing changes done to MyISAM the last months into Maria
mysys/my_handler.c:
importing changes done to MyISAM the last months into Maria
sql/ha_maria.cc:
importing changes done to MyISAM the last months into Maria
sql/ha_maria.h:
importing changes done to MyISAM the last months into Maria
sql/mysqld.cc:
unneeded
storage/maria/Makefile.am:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_check.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_create.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_delete_table.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_dynrec.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_extra.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_ft_boolean_search.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_ft_eval.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_ft_nlq_search.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_ft_parser.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_ft_test1.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_ft_update.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_ftdefs.h:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_key.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_open.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_page.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_rkey.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_rsamepos.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_rt_index.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_rt_mbr.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_search.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_sort.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_test1.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_test2.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_test3.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_update.c:
importing changes done to MyISAM the last months into Maria
storage/maria/ma_write.c:
importing changes done to MyISAM the last months into Maria
storage/maria/maria_chk.c:
importing changes done to MyISAM the last months into Maria
storage/maria/maria_def.h:
importing changes done to MyISAM the last months into Maria
storage/maria/maria_ftdump.c:
importing changes done to MyISAM the last months into Maria
storage/maria/maria_pack.c:
importing changes done to MyISAM the last months into Maria
2006-08-10 16:36:54 +02:00
|
|
|
MY_UNPACK_FILENAME),MYF(0));
|
2006-04-11 16:45:10 +03:00
|
|
|
pthread_mutex_lock(&THR_LOCK_maria);
|
2007-07-01 16:20:57 +03:00
|
|
|
old_info= 0;
|
|
|
|
if ((open_flags & HA_OPEN_COPY) ||
|
|
|
|
!(old_info=_ma_test_if_reopen(name_buff)))
|
2006-04-11 16:45:10 +03:00
|
|
|
{
|
|
|
|
share= &share_buff;
|
2007-07-02 20:45:15 +03:00
|
|
|
bzero((uchar*) &share_buff,sizeof(share_buff));
|
2006-04-11 16:45:10 +03:00
|
|
|
share_buff.state.rec_per_key_part=rec_per_key_part;
|
|
|
|
share_buff.state.key_root=key_root;
|
2007-10-04 20:33:42 +03:00
|
|
|
share_buff.pagecache= multi_pagecache_search((uchar*) name_buff,
|
|
|
|
strlen(name_buff),
|
2007-04-04 23:37:09 +03:00
|
|
|
maria_pagecache);
|
2006-04-11 16:45:10 +03:00
|
|
|
|
|
|
|
DBUG_EXECUTE_IF("maria_pretend_crashed_table_on_open",
|
|
|
|
if (strstr(name, "/t1"))
|
|
|
|
{
|
|
|
|
my_errno= HA_ERR_CRASHED;
|
|
|
|
goto err;
|
|
|
|
});
|
|
|
|
if ((kfile=my_open(name_buff,(open_mode=O_RDWR) | O_SHARE,MYF(0))) < 0)
|
|
|
|
{
|
|
|
|
if ((errno != EROFS && errno != EACCES) ||
|
|
|
|
mode != O_RDONLY ||
|
|
|
|
(kfile=my_open(name_buff,(open_mode=O_RDONLY) | O_SHARE,MYF(0))) < 0)
|
|
|
|
goto err;
|
|
|
|
}
|
|
|
|
share->mode=open_mode;
|
2007-01-18 21:38:14 +02:00
|
|
|
errpos= 1;
|
2007-10-04 20:33:42 +03:00
|
|
|
if (my_read(kfile,share->state.header.file_version, head_length,
|
2006-04-11 16:45:10 +03:00
|
|
|
MYF(MY_NABP)))
|
|
|
|
{
|
|
|
|
my_errno= HA_ERR_NOT_A_TABLE;
|
|
|
|
goto err;
|
|
|
|
}
|
2007-07-02 20:45:15 +03:00
|
|
|
if (memcmp((uchar*) share->state.header.file_version,
|
|
|
|
(uchar*) maria_file_magic, 4))
|
2006-04-11 16:45:10 +03:00
|
|
|
{
|
|
|
|
DBUG_PRINT("error",("Wrong header in %s",name_buff));
|
|
|
|
DBUG_DUMP("error_dump",(char*) share->state.header.file_version,
|
|
|
|
head_length);
|
|
|
|
my_errno=HA_ERR_NOT_A_TABLE;
|
|
|
|
goto err;
|
|
|
|
}
|
|
|
|
share->options= mi_uint2korr(share->state.header.options);
|
|
|
|
if (share->options &
|
|
|
|
~(HA_OPTION_PACK_RECORD | HA_OPTION_PACK_KEYS |
|
|
|
|
HA_OPTION_COMPRESS_RECORD | HA_OPTION_READ_ONLY_DATA |
|
|
|
|
HA_OPTION_TEMP_COMPRESS_RECORD | HA_OPTION_CHECKSUM |
|
|
|
|
HA_OPTION_TMP_TABLE | HA_OPTION_DELAY_KEY_WRITE |
|
2007-09-11 01:58:15 +03:00
|
|
|
HA_OPTION_RELIES_ON_SQL_LAYER | HA_OPTION_NULL_FIELDS))
|
2006-04-11 16:45:10 +03:00
|
|
|
{
|
|
|
|
DBUG_PRINT("error",("wrong options: 0x%lx", share->options));
|
|
|
|
my_errno=HA_ERR_OLD_FILE;
|
|
|
|
goto err;
|
|
|
|
}
|
|
|
|
if ((share->options & HA_OPTION_RELIES_ON_SQL_LAYER) &&
|
|
|
|
! (open_flags & HA_OPEN_FROM_SQL_LAYER))
|
|
|
|
{
|
|
|
|
DBUG_PRINT("error", ("table cannot be openned from non-sql layer"));
|
|
|
|
my_errno= HA_ERR_UNSUPPORTED;
|
|
|
|
goto err;
|
|
|
|
}
|
|
|
|
/* Don't call realpath() if the name can't be a link */
|
|
|
|
if (!strcmp(name_buff, org_name) ||
|
|
|
|
my_readlink(index_name, org_name, MYF(0)) == -1)
|
|
|
|
(void) strmov(index_name, org_name);
|
|
|
|
*strrchr(org_name, '.')= '\0';
|
|
|
|
(void) fn_format(data_name,org_name,"",MARIA_NAME_DEXT,
|
|
|
|
MY_APPEND_EXT|MY_UNPACK_FILENAME|MY_RESOLVE_SYMLINKS);
|
|
|
|
|
|
|
|
info_length=mi_uint2korr(share->state.header.header_length);
|
2007-01-18 21:38:14 +02:00
|
|
|
base_pos= mi_uint2korr(share->state.header.base_pos);
|
2007-10-04 20:33:42 +03:00
|
|
|
if (!(disk_cache= (uchar*) my_alloca(info_length+128)))
|
2006-04-11 16:45:10 +03:00
|
|
|
{
|
|
|
|
my_errno=ENOMEM;
|
|
|
|
goto err;
|
|
|
|
}
|
|
|
|
end_pos=disk_cache+info_length;
|
2007-01-18 21:38:14 +02:00
|
|
|
errpos= 2;
|
2006-04-11 16:45:10 +03:00
|
|
|
|
|
|
|
VOID(my_seek(kfile,0L,MY_SEEK_SET,MYF(0)));
|
2007-01-18 21:38:14 +02:00
|
|
|
errpos= 3;
|
2006-04-11 16:45:10 +03:00
|
|
|
if (my_read(kfile,disk_cache,info_length,MYF(MY_NABP)))
|
|
|
|
{
|
|
|
|
my_errno=HA_ERR_CRASHED;
|
|
|
|
goto err;
|
|
|
|
}
|
|
|
|
len=mi_uint2korr(share->state.header.state_info_length);
|
|
|
|
keys= (uint) share->state.header.keys;
|
|
|
|
uniques= (uint) share->state.header.uniques;
|
|
|
|
fulltext_keys= (uint) share->state.header.fulltext_keys;
|
|
|
|
key_parts= mi_uint2korr(share->state.header.key_parts);
|
|
|
|
unique_key_parts= mi_uint2korr(share->state.header.unique_key_parts);
|
|
|
|
if (len != MARIA_STATE_INFO_SIZE)
|
|
|
|
{
|
|
|
|
DBUG_PRINT("warning",
|
|
|
|
("saved_state_info_length: %d state_info_length: %d",
|
|
|
|
len,MARIA_STATE_INFO_SIZE));
|
|
|
|
}
|
|
|
|
share->state_diff_length=len-MARIA_STATE_INFO_SIZE;
|
|
|
|
|
2007-01-18 21:38:14 +02:00
|
|
|
_ma_state_info_read(disk_cache, &share->state);
|
2006-04-11 16:45:10 +03:00
|
|
|
len= mi_uint2korr(share->state.header.base_info_length);
|
|
|
|
if (len != MARIA_BASE_INFO_SIZE)
|
|
|
|
{
|
|
|
|
DBUG_PRINT("warning",("saved_base_info_length: %d base_info_length: %d",
|
|
|
|
len,MARIA_BASE_INFO_SIZE));
|
|
|
|
}
|
2007-01-18 21:38:14 +02:00
|
|
|
disk_pos= _ma_base_info_read(disk_cache + base_pos, &share->base);
|
2006-04-11 16:45:10 +03:00
|
|
|
share->state.state_length=base_pos;
|
|
|
|
|
|
|
|
if (!(open_flags & HA_OPEN_FOR_REPAIR) &&
|
|
|
|
((share->state.changed & STATE_CRASHED) ||
|
|
|
|
((open_flags & HA_OPEN_ABORT_IF_CRASHED) &&
|
|
|
|
(my_disable_locking && share->state.open_count))))
|
|
|
|
{
|
2006-12-19 19:15:53 +01:00
|
|
|
DBUG_PRINT("error",("Table is marked as crashed. open_flags: %u "
|
|
|
|
"changed: %u open_count: %u !locking: %d",
|
|
|
|
open_flags, share->state.changed,
|
|
|
|
share->state.open_count, my_disable_locking));
|
2006-04-11 16:45:10 +03:00
|
|
|
my_errno=((share->state.changed & STATE_CRASHED_ON_REPAIR) ?
|
|
|
|
HA_ERR_CRASHED_ON_REPAIR : HA_ERR_CRASHED_ON_USAGE);
|
|
|
|
goto err;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* sanity check */
|
|
|
|
if (share->base.keystart > 65535 || share->base.rec_reflength > 8)
|
|
|
|
{
|
|
|
|
my_errno=HA_ERR_CRASHED;
|
|
|
|
goto err;
|
|
|
|
}
|
|
|
|
|
|
|
|
key_parts+=fulltext_keys*FT_SEGS;
|
2007-04-19 13:18:56 +03:00
|
|
|
if (share->base.max_key_length > maria_max_key_length() ||
|
2007-07-27 12:06:39 +02:00
|
|
|
keys > MARIA_MAX_KEY || key_parts > MARIA_MAX_KEY * HA_MAX_KEY_SEG)
|
2006-04-11 16:45:10 +03:00
|
|
|
{
|
|
|
|
DBUG_PRINT("error",("Wrong key info: Max_key_length: %d keys: %d key_parts: %d", share->base.max_key_length, keys, key_parts));
|
|
|
|
my_errno=HA_ERR_UNSUPPORTED;
|
|
|
|
goto err;
|
|
|
|
}
|
2007-04-19 18:48:36 +03:00
|
|
|
/*
|
|
|
|
If page cache is not initialized, then assume we will create it
|
|
|
|
after the table is opened!
|
|
|
|
*/
|
|
|
|
if (share->base.block_size != maria_block_size &&
|
|
|
|
share_buff.pagecache->inited != 0)
|
2007-04-19 13:18:56 +03:00
|
|
|
{
|
|
|
|
DBUG_PRINT("error", ("Wrong block size %u; Expected %u",
|
|
|
|
(uint) share->base.block_size,
|
|
|
|
(uint) maria_block_size));
|
|
|
|
my_errno=HA_ERR_UNSUPPORTED;
|
|
|
|
goto err;
|
|
|
|
}
|
2006-04-11 16:45:10 +03:00
|
|
|
|
|
|
|
/* Correct max_file_length based on length of sizeof(off_t) */
|
|
|
|
max_data_file_length=
|
|
|
|
(share->options & (HA_OPTION_PACK_RECORD | HA_OPTION_COMPRESS_RECORD)) ?
|
|
|
|
(((ulonglong) 1 << (share->base.rec_reflength*8))-1) :
|
|
|
|
(_ma_safe_mul(share->base.pack_reclength,
|
|
|
|
(ulonglong) 1 << (share->base.rec_reflength*8))-1);
|
2007-01-18 21:38:14 +02:00
|
|
|
|
2006-04-11 16:45:10 +03:00
|
|
|
max_key_file_length=
|
|
|
|
_ma_safe_mul(MARIA_MIN_KEY_BLOCK_LENGTH,
|
|
|
|
((ulonglong) 1 << (share->base.key_reflength*8))-1);
|
|
|
|
#if SIZEOF_OFF_T == 4
|
|
|
|
set_if_smaller(max_data_file_length, INT_MAX32);
|
|
|
|
set_if_smaller(max_key_file_length, INT_MAX32);
|
|
|
|
#endif
|
|
|
|
share->base.max_data_file_length=(my_off_t) max_data_file_length;
|
|
|
|
share->base.max_key_file_length=(my_off_t) max_key_file_length;
|
|
|
|
|
|
|
|
if (share->options & HA_OPTION_COMPRESS_RECORD)
|
|
|
|
share->base.max_key_length+=2; /* For safety */
|
|
|
|
|
|
|
|
if (!my_multi_malloc(MY_WME,
|
|
|
|
&share,sizeof(*share),
|
|
|
|
&share->state.rec_per_key_part,sizeof(long)*key_parts,
|
|
|
|
&share->keyinfo,keys*sizeof(MARIA_KEYDEF),
|
|
|
|
&share->uniqueinfo,uniques*sizeof(MARIA_UNIQUEDEF),
|
|
|
|
&share->keyparts,
|
|
|
|
(key_parts+unique_key_parts+keys+uniques) *
|
|
|
|
sizeof(HA_KEYSEG),
|
2007-04-19 13:18:56 +03:00
|
|
|
&share->columndef,
|
2006-04-11 16:45:10 +03:00
|
|
|
(share->base.fields+1)*sizeof(MARIA_COLUMNDEF),
|
|
|
|
&share->blobs,sizeof(MARIA_BLOB)*share->base.blobs,
|
|
|
|
&share->unique_file_name,strlen(name_buff)+1,
|
|
|
|
&share->index_file_name,strlen(index_name)+1,
|
|
|
|
&share->data_file_name,strlen(data_name)+1,
|
2007-06-07 01:01:43 +03:00
|
|
|
&share->open_file_name,strlen(name)+1,
|
2006-04-11 16:45:10 +03:00
|
|
|
&share->state.key_root,keys*sizeof(my_off_t),
|
|
|
|
#ifdef THREAD
|
|
|
|
&share->key_root_lock,sizeof(rw_lock_t)*keys,
|
|
|
|
#endif
|
|
|
|
&share->mmap_lock,sizeof(rw_lock_t),
|
|
|
|
NullS))
|
|
|
|
goto err;
|
2007-01-18 21:38:14 +02:00
|
|
|
errpos= 4;
|
|
|
|
|
2006-04-11 16:45:10 +03:00
|
|
|
*share=share_buff;
|
|
|
|
memcpy((char*) share->state.rec_per_key_part,
|
|
|
|
(char*) rec_per_key_part, sizeof(long)*key_parts);
|
|
|
|
memcpy((char*) share->state.key_root,
|
|
|
|
(char*) key_root, sizeof(my_off_t)*keys);
|
|
|
|
strmov(share->unique_file_name, name_buff);
|
|
|
|
share->unique_name_length= strlen(name_buff);
|
|
|
|
strmov(share->index_file_name, index_name);
|
|
|
|
strmov(share->data_file_name, data_name);
|
2007-06-07 01:01:43 +03:00
|
|
|
strmov(share->open_file_name, name);
|
2006-04-11 16:45:10 +03:00
|
|
|
|
2007-04-19 18:48:36 +03:00
|
|
|
share->block_size= share->base.block_size;
|
2006-04-11 16:45:10 +03:00
|
|
|
{
|
|
|
|
HA_KEYSEG *pos=share->keyparts;
|
|
|
|
for (i=0 ; i < keys ; i++)
|
|
|
|
{
|
|
|
|
share->keyinfo[i].share= share;
|
|
|
|
disk_pos=_ma_keydef_read(disk_pos, &share->keyinfo[i]);
|
|
|
|
disk_pos_assert(disk_pos + share->keyinfo[i].keysegs * HA_KEYSEG_SIZE,
|
|
|
|
end_pos);
|
|
|
|
if (share->keyinfo[i].key_alg == HA_KEY_ALG_RTREE)
|
2007-06-07 01:01:43 +03:00
|
|
|
share->have_rtree= 1;
|
2006-04-11 16:45:10 +03:00
|
|
|
share->keyinfo[i].seg=pos;
|
|
|
|
for (j=0 ; j < share->keyinfo[i].keysegs; j++,pos++)
|
|
|
|
{
|
|
|
|
disk_pos=_ma_keyseg_read(disk_pos, pos);
|
|
|
|
if (pos->type == HA_KEYTYPE_TEXT ||
|
|
|
|
pos->type == HA_KEYTYPE_VARTEXT1 ||
|
|
|
|
pos->type == HA_KEYTYPE_VARTEXT2)
|
|
|
|
{
|
|
|
|
if (!pos->language)
|
|
|
|
pos->charset=default_charset_info;
|
|
|
|
else if (!(pos->charset= get_charset(pos->language, MYF(MY_WME))))
|
|
|
|
{
|
|
|
|
my_errno=HA_ERR_UNKNOWN_CHARSET;
|
|
|
|
goto err;
|
|
|
|
}
|
|
|
|
}
|
2006-12-20 18:58:35 +01:00
|
|
|
else if (pos->type == HA_KEYTYPE_BINARY)
|
|
|
|
pos->charset= &my_charset_bin;
|
2006-04-11 16:45:10 +03:00
|
|
|
}
|
|
|
|
if (share->keyinfo[i].flag & HA_SPATIAL)
|
|
|
|
{
|
|
|
|
#ifdef HAVE_SPATIAL
|
|
|
|
uint sp_segs=SPDIMS*2;
|
|
|
|
share->keyinfo[i].seg=pos-sp_segs;
|
|
|
|
share->keyinfo[i].keysegs--;
|
|
|
|
#else
|
|
|
|
my_errno=HA_ERR_UNSUPPORTED;
|
|
|
|
goto err;
|
|
|
|
#endif
|
|
|
|
}
|
|
|
|
else if (share->keyinfo[i].flag & HA_FULLTEXT)
|
|
|
|
{
|
|
|
|
if (!fulltext_keys)
|
|
|
|
{ /* 4.0 compatibility code, to be removed in 5.0 */
|
|
|
|
share->keyinfo[i].seg=pos-FT_SEGS;
|
|
|
|
share->keyinfo[i].keysegs-=FT_SEGS;
|
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
2007-03-01 18:23:58 +01:00
|
|
|
uint k;
|
2006-04-11 16:45:10 +03:00
|
|
|
share->keyinfo[i].seg=pos;
|
2007-03-01 18:23:58 +01:00
|
|
|
for (k=0; k < FT_SEGS; k++)
|
2006-04-11 16:45:10 +03:00
|
|
|
{
|
2007-03-01 18:23:58 +01:00
|
|
|
*pos= ft_keysegs[k];
|
2006-04-11 16:45:10 +03:00
|
|
|
pos[0].language= pos[-1].language;
|
|
|
|
if (!(pos[0].charset= pos[-1].charset))
|
|
|
|
{
|
|
|
|
my_errno=HA_ERR_CRASHED;
|
|
|
|
goto err;
|
|
|
|
}
|
|
|
|
pos++;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if (!share->ft2_keyinfo.seg)
|
|
|
|
{
|
|
|
|
memcpy(& share->ft2_keyinfo, & share->keyinfo[i], sizeof(MARIA_KEYDEF));
|
|
|
|
share->ft2_keyinfo.keysegs=1;
|
|
|
|
share->ft2_keyinfo.flag=0;
|
|
|
|
share->ft2_keyinfo.keylength=
|
|
|
|
share->ft2_keyinfo.minlength=
|
|
|
|
share->ft2_keyinfo.maxlength=HA_FT_WLEN+share->base.rec_reflength;
|
|
|
|
share->ft2_keyinfo.seg=pos-1;
|
|
|
|
share->ft2_keyinfo.end=pos;
|
|
|
|
setup_key_functions(& share->ft2_keyinfo);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
setup_key_functions(share->keyinfo+i);
|
|
|
|
share->keyinfo[i].end=pos;
|
|
|
|
pos->type=HA_KEYTYPE_END; /* End */
|
|
|
|
pos->length=share->base.rec_reflength;
|
|
|
|
pos->null_bit=0;
|
|
|
|
pos->flag=0; /* For purify */
|
|
|
|
pos++;
|
|
|
|
}
|
|
|
|
for (i=0 ; i < uniques ; i++)
|
|
|
|
{
|
|
|
|
disk_pos=_ma_uniquedef_read(disk_pos, &share->uniqueinfo[i]);
|
|
|
|
disk_pos_assert(disk_pos + share->uniqueinfo[i].keysegs *
|
|
|
|
HA_KEYSEG_SIZE, end_pos);
|
|
|
|
share->uniqueinfo[i].seg=pos;
|
|
|
|
for (j=0 ; j < share->uniqueinfo[i].keysegs; j++,pos++)
|
|
|
|
{
|
|
|
|
disk_pos=_ma_keyseg_read(disk_pos, pos);
|
|
|
|
if (pos->type == HA_KEYTYPE_TEXT ||
|
|
|
|
pos->type == HA_KEYTYPE_VARTEXT1 ||
|
|
|
|
pos->type == HA_KEYTYPE_VARTEXT2)
|
|
|
|
{
|
|
|
|
if (!pos->language)
|
|
|
|
pos->charset=default_charset_info;
|
|
|
|
else if (!(pos->charset= get_charset(pos->language, MYF(MY_WME))))
|
|
|
|
{
|
|
|
|
my_errno=HA_ERR_UNKNOWN_CHARSET;
|
|
|
|
goto err;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
share->uniqueinfo[i].end=pos;
|
|
|
|
pos->type=HA_KEYTYPE_END; /* End */
|
|
|
|
pos->null_bit=0;
|
|
|
|
pos->flag=0;
|
|
|
|
pos++;
|
|
|
|
}
|
|
|
|
share->ftparsers= 0;
|
|
|
|
}
|
2007-01-18 21:38:14 +02:00
|
|
|
share->data_file_type= share->state.header.data_file_type;
|
|
|
|
share->base_length= (BASE_ROW_HEADER_SIZE +
|
|
|
|
share->base.is_nulls_extended +
|
|
|
|
share->base.null_bytes +
|
|
|
|
share->base.pack_bytes +
|
|
|
|
test(share->options & HA_OPTION_CHECKSUM));
|
2007-07-01 16:20:57 +03:00
|
|
|
if (open_flags & HA_OPEN_COPY)
|
2007-06-27 12:58:08 +02:00
|
|
|
{
|
Maria:
* Don't modify share->base.born_transactional; now it is a value carved
in stone at creation time. share->now_transactional is what can be
modified: it starts at born_transactional, can become false during
ALTER TABLE (when we want no logging), and restored later.
* Not resetting create_rename_lsn to 0 during delete_all or repair.
* when we temporarily disable transactionality, we also change
the page type to PAGECACHE_PLAIN_PAGE: it bypasses some work in the
page cache (optimization), and avoids assertions related to LSNs.
* Disable INSERT DELAYED for transactional tables, because
durability could not be guaranteed (insertion may even not happen)
mysys/mf_keycache.c:
comment
storage/maria/ha_maria.cc:
* a transactional table cannot do INSERT DELAYED
* ha_maria::save_transactional not needed anymore, as now instead
we don't modify MARIA_SHARE::MARIA_BASE_INFO::born_transactional
(born_transactional plays the role of save_transactional), and modify
MARIA_SHARE::now_transactional.
* REPAIR_TABLE log record is now logged by maria_repair()
* comment why we rely on born_transactional to know if we should
skipping a transaction.
* putting together two if()s which test for F_UNLCK
storage/maria/ha_maria.h:
ha_maria::save_transactional not needed anymore (moved to the C layer)
storage/maria/ma_blockrec.c:
* For the block record's code (writing/updating/deleting records),
all that counts is now_transactional, not born_transactional.
* As we now set the page type to PAGECACHE_PLAIN_PAGE for tables
which have now_transactional==FALSE, pagecache will not expect
a meaningful LSN for them in pagecache_unlock_by_link(), so
we can pass it LSN_IMPOSSIBLE.
storage/maria/ma_check.c:
* writing LOGREC_REPAIR_TABLE moves from ha_maria::repair()
to maria_repair(), sounds cleaner (less functions to export).
* when opening a table during REPAIR, don't use the realpath-ed name,
as this may fail if the table has symlinked files (maria_open()
would try to find the data and index file in the directory
of unique_file_name, it would fail if data and index files are in
different dirs); use the unresolved name, open_file_name, which is
the argument which was passed to the maria_open() which created 'info'.
storage/maria/ma_close.c:
assert that when a statement is done with a table, it cleans up
storage/maria/ma_create.c:
new name
storage/maria/ma_delete_all.c:
* using now_transactional
* no reason to reset create_rename_lsn during delete_all (a bug);
also no reason to do it during repair: it was put there because
a positive create_rename_lsn caused a call to check_and_set_lsn()
which asserted in DBUG_ASSERT(block->type == PAGECACHE_LSN_PAGE);
first solution was to use LSN_IMPOSSIBLE in _ma_unpin_all_pages() if
not transactional; but then in the case of ALTER TABLE, with
transactionality temporarily disabled, it asserted in
DBUG_ASSERT(LSN_VALID(lsn)) in pagecache_fwrite() (PAGECACHE_LSN_PAGE
page with zero LSN - bad). The additional solution is to use
PAGECACHE_PLAIN_PAGE when we disable transactionality temporarily: this
avoids checks on the LSN, and also bypasses (optimization) the "flush
log up to LSN" call when the pagecache flushes our page (in other
words, no WAL needed).
storage/maria/ma_delete_table.c:
use now_transactional
storage/maria/ma_locking.c:
assert that when a statement is done with a table, it cleans up.
storage/maria/ma_loghandler.c:
* now_transactional should be used to test if we want a log record.
* Assertions to make sure dummy_transaction_object is not spoilt
by its many users.
storage/maria/ma_open.c:
base.transactional -> base.born_transactional
storage/maria/ma_pagecache.c:
missing name for page's type. Comment for future.
storage/maria/ma_rename.c:
use now_transactional
storage/maria/maria_chk.c:
use born_transactional
storage/maria/maria_def.h:
MARIA_BASE_INFO::transactional renamed to born_transactional.
MARIA_SHARE::now_transactional introduced.
_ma_repair_write_log_record() is made local to ma_check.c.
Macros to temporarily disable, and re-enable, transactionality for a
table.
storage/maria/maria_read_log.c:
assertions and using the new macros. Adding a forgotten resetting
when we finally close all tables.
2007-07-03 15:20:41 +02:00
|
|
|
/*
|
|
|
|
this instance will be a temporary one used just to create a data
|
|
|
|
file for REPAIR. Don't do logging. This base information will not go
|
|
|
|
to disk.
|
|
|
|
*/
|
|
|
|
share->base.born_transactional= FALSE;
|
|
|
|
}
|
|
|
|
if (share->base.born_transactional)
|
|
|
|
{
|
|
|
|
share->page_type= PAGECACHE_LSN_PAGE;
|
2007-08-31 10:19:54 +03:00
|
|
|
#ifdef ENABLE_WHEN_WE_HAVE_TRANS_ROW_ID /* QQ */
|
2007-01-18 21:38:14 +02:00
|
|
|
share->base_length+= TRANS_ROW_EXTRA_HEADER_SIZE;
|
2007-08-31 10:19:54 +03:00
|
|
|
#endif
|
WL#3072 Maria recovery
* create page cache before initializing engine and not after, because
Maria's recovery needs a page cache
* make the creation of a bitmap page more crash-resistent
* bugfix (see ma_blockrec.c)
* back to old way: create an 8k bitmap page when creating table
* preparations for the UNDO phase: recreate TRNs
* preparations for Checkpoint: list of dirty pages, testing
of rec_lsn to know if page should be skipped during Recovery
(unused in this patch as no Checkpoint module pushed yet)
* maria_chk tags repaired table with a special LSN
* reworking all around in ma_recovery.c (less duplication)
mysys/my_realloc.c:
noted an issue in my_realloc()
sql/mysqld.cc:
page cache needs to be created before engines are initialized,
because Maria's initialization may do a recovery which needs
the page cache.
storage/maria/ha_maria.cc:
update to new prototype
storage/maria/ma_bitmap.c:
when creating the first bitmap page we used chsize to 8192 bytes then
pwrite (overwrite) the last 2 bytes (8191-8192). If crash between
the two operations, this leaves a bitmap page full without its end
marker. A later recovery may try to read this page and find it
exists and misses a marker and conclude it's corrupted and fail.
Changing the chsize to only 8190 bytes: recovery will then find
the page is too short and recreate it entirely.
storage/maria/ma_blockrec.c:
Fix for a bug: when executing a REDO, if the data page is created,
data_file_length was increased before _ma_bitmap_set():
_ma_bitmap_set() called _ma_read_bitmap_page() which, due to the
increased data_file_length, expected to find a bitmap page on disk
with a correct end marker; if the bitmap page didn't exist already
in fact, this failed. Fixed by increasing data_file_length only after
_ma_read_bitmap_page() has created the new bitmap page correctly.
This bug could happen every time a REDO is about creating a new
bitmap page.
storage/maria/ma_check.c:
empty data file has a bitmap page
storage/maria/ma_control_file.c:
useless parameter to ma_control_file_create_or_open(), just
test if this is recovery.
storage/maria/ma_control_file.h:
new prototype
storage/maria/ma_create.c:
Back to how it was before: maria_create() creates an 8k bitmap page.
Thus (bugfix) data_file_length needs to reflect this instead of being 0.
storage/maria/ma_loghandler.c:
as ma_test1 and ma_test2 now use real transactions and not
dummy_transaction_object, REDO for INSERT/UPDATE/DELETE are always
about real transactions, can assert this.
A function for Recovery to assign a short id to a table.
storage/maria/ma_loghandler.h:
new function
storage/maria/ma_loghandler_lsn.h:
maria_chk tags repaired tables with this LSN
storage/maria/ma_open.c:
* enforce that DMLs on transactional tables use real transactions
and not dummy_transaction_object.
* test if table was repaired with maria_chk (which has to been
seen as an import of an external table into the server), test
validity of create_rename_lsn (header corruption detection)
* comments.
storage/maria/ma_recovery.c:
* preparations for the UNDO phase: recreate TRNs
* preparations for Checkpoint: list of dirty pages, testing
of rec_lsn to know if page should be skipped during Recovery
(unused in this patch as no Checkpoint module pushed yet)
* reworking all around (less duplication)
storage/maria/ma_recovery.h:
a parameter to say if the UNDO phase should be skipped
storage/maria/maria_chk.c:
tag repaired tables with a special LSN
storage/maria/maria_read_log.c:
* update to new prototype
* no UNDO phase in maria_read_log for now
storage/maria/trnman.c:
* a function for Recovery to create a transaction (TRN), needed
in the UNDO phase
* a function for Recovery to grab an existing transaction, needed
in the UNDO phase (rollback all existing transactions)
storage/maria/trnman_public.h:
new functions
2007-08-29 16:43:01 +02:00
|
|
|
if (share->state.create_rename_lsn == LSN_REPAIRED_BY_MARIA_CHK)
|
2007-06-27 12:58:08 +02:00
|
|
|
{
|
|
|
|
/*
|
WL#3072 Maria recovery
* create page cache before initializing engine and not after, because
Maria's recovery needs a page cache
* make the creation of a bitmap page more crash-resistent
* bugfix (see ma_blockrec.c)
* back to old way: create an 8k bitmap page when creating table
* preparations for the UNDO phase: recreate TRNs
* preparations for Checkpoint: list of dirty pages, testing
of rec_lsn to know if page should be skipped during Recovery
(unused in this patch as no Checkpoint module pushed yet)
* maria_chk tags repaired table with a special LSN
* reworking all around in ma_recovery.c (less duplication)
mysys/my_realloc.c:
noted an issue in my_realloc()
sql/mysqld.cc:
page cache needs to be created before engines are initialized,
because Maria's initialization may do a recovery which needs
the page cache.
storage/maria/ha_maria.cc:
update to new prototype
storage/maria/ma_bitmap.c:
when creating the first bitmap page we used chsize to 8192 bytes then
pwrite (overwrite) the last 2 bytes (8191-8192). If crash between
the two operations, this leaves a bitmap page full without its end
marker. A later recovery may try to read this page and find it
exists and misses a marker and conclude it's corrupted and fail.
Changing the chsize to only 8190 bytes: recovery will then find
the page is too short and recreate it entirely.
storage/maria/ma_blockrec.c:
Fix for a bug: when executing a REDO, if the data page is created,
data_file_length was increased before _ma_bitmap_set():
_ma_bitmap_set() called _ma_read_bitmap_page() which, due to the
increased data_file_length, expected to find a bitmap page on disk
with a correct end marker; if the bitmap page didn't exist already
in fact, this failed. Fixed by increasing data_file_length only after
_ma_read_bitmap_page() has created the new bitmap page correctly.
This bug could happen every time a REDO is about creating a new
bitmap page.
storage/maria/ma_check.c:
empty data file has a bitmap page
storage/maria/ma_control_file.c:
useless parameter to ma_control_file_create_or_open(), just
test if this is recovery.
storage/maria/ma_control_file.h:
new prototype
storage/maria/ma_create.c:
Back to how it was before: maria_create() creates an 8k bitmap page.
Thus (bugfix) data_file_length needs to reflect this instead of being 0.
storage/maria/ma_loghandler.c:
as ma_test1 and ma_test2 now use real transactions and not
dummy_transaction_object, REDO for INSERT/UPDATE/DELETE are always
about real transactions, can assert this.
A function for Recovery to assign a short id to a table.
storage/maria/ma_loghandler.h:
new function
storage/maria/ma_loghandler_lsn.h:
maria_chk tags repaired tables with this LSN
storage/maria/ma_open.c:
* enforce that DMLs on transactional tables use real transactions
and not dummy_transaction_object.
* test if table was repaired with maria_chk (which has to been
seen as an import of an external table into the server), test
validity of create_rename_lsn (header corruption detection)
* comments.
storage/maria/ma_recovery.c:
* preparations for the UNDO phase: recreate TRNs
* preparations for Checkpoint: list of dirty pages, testing
of rec_lsn to know if page should be skipped during Recovery
(unused in this patch as no Checkpoint module pushed yet)
* reworking all around (less duplication)
storage/maria/ma_recovery.h:
a parameter to say if the UNDO phase should be skipped
storage/maria/maria_chk.c:
tag repaired tables with a special LSN
storage/maria/maria_read_log.c:
* update to new prototype
* no UNDO phase in maria_read_log for now
storage/maria/trnman.c:
* a function for Recovery to create a transaction (TRN), needed
in the UNDO phase
* a function for Recovery to grab an existing transaction, needed
in the UNDO phase (rollback all existing transactions)
storage/maria/trnman_public.h:
new functions
2007-08-29 16:43:01 +02:00
|
|
|
Was repaired with maria_chk, maybe later maria_pack-ed. Some sort of
|
|
|
|
import into the server. It starts its existence (from the point of
|
|
|
|
view of the server, including server's recovery) now.
|
2007-06-27 12:58:08 +02:00
|
|
|
*/
|
WL#3072 Maria recovery
* create page cache before initializing engine and not after, because
Maria's recovery needs a page cache
* make the creation of a bitmap page more crash-resistent
* bugfix (see ma_blockrec.c)
* back to old way: create an 8k bitmap page when creating table
* preparations for the UNDO phase: recreate TRNs
* preparations for Checkpoint: list of dirty pages, testing
of rec_lsn to know if page should be skipped during Recovery
(unused in this patch as no Checkpoint module pushed yet)
* maria_chk tags repaired table with a special LSN
* reworking all around in ma_recovery.c (less duplication)
mysys/my_realloc.c:
noted an issue in my_realloc()
sql/mysqld.cc:
page cache needs to be created before engines are initialized,
because Maria's initialization may do a recovery which needs
the page cache.
storage/maria/ha_maria.cc:
update to new prototype
storage/maria/ma_bitmap.c:
when creating the first bitmap page we used chsize to 8192 bytes then
pwrite (overwrite) the last 2 bytes (8191-8192). If crash between
the two operations, this leaves a bitmap page full without its end
marker. A later recovery may try to read this page and find it
exists and misses a marker and conclude it's corrupted and fail.
Changing the chsize to only 8190 bytes: recovery will then find
the page is too short and recreate it entirely.
storage/maria/ma_blockrec.c:
Fix for a bug: when executing a REDO, if the data page is created,
data_file_length was increased before _ma_bitmap_set():
_ma_bitmap_set() called _ma_read_bitmap_page() which, due to the
increased data_file_length, expected to find a bitmap page on disk
with a correct end marker; if the bitmap page didn't exist already
in fact, this failed. Fixed by increasing data_file_length only after
_ma_read_bitmap_page() has created the new bitmap page correctly.
This bug could happen every time a REDO is about creating a new
bitmap page.
storage/maria/ma_check.c:
empty data file has a bitmap page
storage/maria/ma_control_file.c:
useless parameter to ma_control_file_create_or_open(), just
test if this is recovery.
storage/maria/ma_control_file.h:
new prototype
storage/maria/ma_create.c:
Back to how it was before: maria_create() creates an 8k bitmap page.
Thus (bugfix) data_file_length needs to reflect this instead of being 0.
storage/maria/ma_loghandler.c:
as ma_test1 and ma_test2 now use real transactions and not
dummy_transaction_object, REDO for INSERT/UPDATE/DELETE are always
about real transactions, can assert this.
A function for Recovery to assign a short id to a table.
storage/maria/ma_loghandler.h:
new function
storage/maria/ma_loghandler_lsn.h:
maria_chk tags repaired tables with this LSN
storage/maria/ma_open.c:
* enforce that DMLs on transactional tables use real transactions
and not dummy_transaction_object.
* test if table was repaired with maria_chk (which has to been
seen as an import of an external table into the server), test
validity of create_rename_lsn (header corruption detection)
* comments.
storage/maria/ma_recovery.c:
* preparations for the UNDO phase: recreate TRNs
* preparations for Checkpoint: list of dirty pages, testing
of rec_lsn to know if page should be skipped during Recovery
(unused in this patch as no Checkpoint module pushed yet)
* reworking all around (less duplication)
storage/maria/ma_recovery.h:
a parameter to say if the UNDO phase should be skipped
storage/maria/maria_chk.c:
tag repaired tables with a special LSN
storage/maria/maria_read_log.c:
* update to new prototype
* no UNDO phase in maria_read_log for now
storage/maria/trnman.c:
* a function for Recovery to create a transaction (TRN), needed
in the UNDO phase
* a function for Recovery to grab an existing transaction, needed
in the UNDO phase (rollback all existing transactions)
storage/maria/trnman_public.h:
new functions
2007-08-29 16:43:01 +02:00
|
|
|
if ((open_flags & HA_OPEN_FROM_SQL_LAYER) || maria_in_recovery)
|
WL#3071 Maria checkpoint
Finally this is the real checkpoint code.
It however exhibits unstabilities when a checkpoint runs concurrently
with data-modifying clients (table corruption, transaction log's
assertions) so for now a checkpoint is taken only at startup after
recovery and at shutdown, i.e. not in concurrent situations. Later
we will let it run periodically, as well as flush dirty pages
periodically (almost all needed code is there already, only pagecache
code is written but not committed).
WL#3072 Maria recovery
* replacing UNDO_ROW_PURGE with CLR_END; testing of those CLR_END via
ma_test2 which has INSERTs failing with duplicate keys.
* replaying of REDO_RENAME_TABLE
Now, off to test Recovery in ha_maria :)
BitKeeper/deleted/.del-ma_least_recently_dirtied.c:
Delete: storage/maria/ma_least_recently_dirtied.c
BitKeeper/deleted/.del-ma_least_recently_dirtied.h:
Delete: storage/maria/ma_least_recently_dirtied.h
storage/maria/Makefile.am:
compile Checkpoint module
storage/maria/ha_maria.cc:
When ha_maria starts, do a recovery from last checkpoint.
Take a checkpoint when that recovery has ended and when ha_maria
shuts down cleanly.
storage/maria/ma_blockrec.c:
* even if my_sync() fails we have to my_close() (otherwise we leak
a descriptor)
* UNDO_ROW_PURGE is replaced by a simple CLR_END for UNDO_ROW_INSERT,
as promised in the old comment; it gives us skipping during the
UNDO phase.
storage/maria/ma_check.c:
All REDOs before create_rename_lsn are ignored by Recovery. So
create_rename_lsn must be set only after all data/index has been
flushed and forced to disk. We thus move write_log_record_for_repair()
to after _ma_flush_tables_files_after_repair().
storage/maria/ma_checkpoint.c:
Checkpoint module.
storage/maria/ma_checkpoint.h:
optional argument if caller wants a thread to periodically take
checkpoints and flush dirty pages.
storage/maria/ma_create.c:
* no need to init some vars as the initial bzero(share) takes care of this.
* update to new function's name
* even if we fail in my_sync() we have to my_close()
storage/maria/ma_extra.c:
Checkpoint reads share->last_version under intern_lock, so we make
maria_extra() update it under intern_lock. THR_LOCK_maria still needed
because of _ma_test_if_reopen().
storage/maria/ma_init.c:
destroy checkpoint module when Maria shuts down.
storage/maria/ma_loghandler.c:
* UNDO_ROW_PURGE gone (see ma_blockrec.c)
* we need to remember the LSN of the LOGREC_FILE_ID for a share,
because this LSN is needed into the checkpoint record (Recovery wants
to know the validity domain of an id->name mapping)
* translog_get_horizon_no_lock() needed for Checkpoint
* comment about failing assertion (Sanja knows)
* translog_init_reader_data() thought that translog_read_record_header_scan()
returns 0 in case of error, but 0 just means "0-length header".
* translog_assign_id_to_share() now needs the MARIA_HA because
LOGREC_FILE_ID uses a log-write hook.
* Verify that (de)assignment of share->id happens only under intern_lock,
as Checkpoint reads this id with intern_lock.
* translog_purge() can accept TRANSLOG_ADDRESS, not necessarily
a real LSN.
storage/maria/ma_loghandler.h:
prototype updates
storage/maria/ma_open.c:
no need to initialize "res"
storage/maria/ma_pagecache.c:
When taking a checkpoint, we don't need to know the maximum rec_lsn
of dirty pages; this LSN was intended to be used in the two-checkpoint
rule, but last_checkpoint_lsn is as good.
4 bytes for stored_list_size is enough as PAGECACHE::blocks (number
of blocks which the pagecache can contain) is int.
storage/maria/ma_pagecache.h:
new prototype
storage/maria/ma_recovery.c:
* added replaying of REDO_RENAME_TABLE
* UNDO_ROW_PURGE gone (see ma_blockrec.c), replaced by CLR_END
* Recovery from the last checkpoint record now possible
* In new_table() we skip the table if the id->name mapping is older than
create_rename_lsn (mapping dates from lsn_of_file_id).
* in get_MARIA_HA_from_REDO_record() we skip the record
if the id->name mapping is newer than the record (can happen if processing
a record which is before the checkpoint record).
* parse_checkpoint_record() has to return a LSN, that's what caller expects
storage/maria/ma_rename.c:
new function's name; log end zeroes of tables' names (ease recovery)
storage/maria/ma_test2.c:
* equivalent of ma_test1's --test-undo added (named -u here).
* -t=1 now stops right after creating the table, so that
we can test undoing of INSERTs with duplicate keys (which tests the
CLR_END logged by _ma_write_abort_block_record()).
storage/maria/ma_test_recovery.expected:
Result of testing undoing of INSERTs with duplicate keys; there are
some differences in maria_chk -dvv but they are normal (removing
records does not shrink data/index file, does not put back the
"analyzed, optimized keys"(etc) index state.
storage/maria/ma_test_recovery:
Test undoing of INSERTs with duplicate keys, using ma_test2;
when such INSERT happens, it logs REDO_INSERT, UNDO_INSERT, REDO_DELETE,
CLR_END; we abort after that, and test that CLR_END causes recovery
to jump over UNDO_INSERT.
storage/maria/ma_write.c:
comment
storage/maria/maria_chk.c:
comment
storage/maria/maria_def.h:
* a new bit in MARIA_SHARE::in_checkpoint, used to build a list
of unique shares during Checkpoint.
* MARIA_SHARE::lsn_of_file_id added: the LSN of the last LOGREC_FILE_ID
for this share; needed to know to which LSN domain the mappings
found in the Checkpoint record apply (new mappings should not apply
to old REDOs).
storage/maria/trnman.c:
* small changes to how trnman_collect_transactions() fills its buffer;
it also uses a non-dummy lsn_read_non_atomic() found in ma_checkpoint.h
2007-09-12 11:27:34 +02:00
|
|
|
_ma_update_create_rename_lsn_sub(share, translog_get_horizon(),
|
|
|
|
TRUE);
|
WL#3072 Maria recovery
* create page cache before initializing engine and not after, because
Maria's recovery needs a page cache
* make the creation of a bitmap page more crash-resistent
* bugfix (see ma_blockrec.c)
* back to old way: create an 8k bitmap page when creating table
* preparations for the UNDO phase: recreate TRNs
* preparations for Checkpoint: list of dirty pages, testing
of rec_lsn to know if page should be skipped during Recovery
(unused in this patch as no Checkpoint module pushed yet)
* maria_chk tags repaired table with a special LSN
* reworking all around in ma_recovery.c (less duplication)
mysys/my_realloc.c:
noted an issue in my_realloc()
sql/mysqld.cc:
page cache needs to be created before engines are initialized,
because Maria's initialization may do a recovery which needs
the page cache.
storage/maria/ha_maria.cc:
update to new prototype
storage/maria/ma_bitmap.c:
when creating the first bitmap page we used chsize to 8192 bytes then
pwrite (overwrite) the last 2 bytes (8191-8192). If crash between
the two operations, this leaves a bitmap page full without its end
marker. A later recovery may try to read this page and find it
exists and misses a marker and conclude it's corrupted and fail.
Changing the chsize to only 8190 bytes: recovery will then find
the page is too short and recreate it entirely.
storage/maria/ma_blockrec.c:
Fix for a bug: when executing a REDO, if the data page is created,
data_file_length was increased before _ma_bitmap_set():
_ma_bitmap_set() called _ma_read_bitmap_page() which, due to the
increased data_file_length, expected to find a bitmap page on disk
with a correct end marker; if the bitmap page didn't exist already
in fact, this failed. Fixed by increasing data_file_length only after
_ma_read_bitmap_page() has created the new bitmap page correctly.
This bug could happen every time a REDO is about creating a new
bitmap page.
storage/maria/ma_check.c:
empty data file has a bitmap page
storage/maria/ma_control_file.c:
useless parameter to ma_control_file_create_or_open(), just
test if this is recovery.
storage/maria/ma_control_file.h:
new prototype
storage/maria/ma_create.c:
Back to how it was before: maria_create() creates an 8k bitmap page.
Thus (bugfix) data_file_length needs to reflect this instead of being 0.
storage/maria/ma_loghandler.c:
as ma_test1 and ma_test2 now use real transactions and not
dummy_transaction_object, REDO for INSERT/UPDATE/DELETE are always
about real transactions, can assert this.
A function for Recovery to assign a short id to a table.
storage/maria/ma_loghandler.h:
new function
storage/maria/ma_loghandler_lsn.h:
maria_chk tags repaired tables with this LSN
storage/maria/ma_open.c:
* enforce that DMLs on transactional tables use real transactions
and not dummy_transaction_object.
* test if table was repaired with maria_chk (which has to been
seen as an import of an external table into the server), test
validity of create_rename_lsn (header corruption detection)
* comments.
storage/maria/ma_recovery.c:
* preparations for the UNDO phase: recreate TRNs
* preparations for Checkpoint: list of dirty pages, testing
of rec_lsn to know if page should be skipped during Recovery
(unused in this patch as no Checkpoint module pushed yet)
* reworking all around (less duplication)
storage/maria/ma_recovery.h:
a parameter to say if the UNDO phase should be skipped
storage/maria/maria_chk.c:
tag repaired tables with a special LSN
storage/maria/maria_read_log.c:
* update to new prototype
* no UNDO phase in maria_read_log for now
storage/maria/trnman.c:
* a function for Recovery to create a transaction (TRN), needed
in the UNDO phase
* a function for Recovery to grab an existing transaction, needed
in the UNDO phase (rollback all existing transactions)
storage/maria/trnman_public.h:
new functions
2007-08-29 16:43:01 +02:00
|
|
|
}
|
- WL#3072 Maria Recovery:
Recovery of state.records (the count of records which is stored into
the header of the index file). For that, state.is_of_lsn is introduced;
logic is explained in ma_recovery.c (look for "Recovery of the state").
The net gain is that in case of crash, we now recover state.records,
and it is idempotent (ma_test_recovery tests it).
state.checksum is not recovered yet, mail sent for discussion.
- WL#3071 Maria Checkpoint: preparation for it, by protecting
all modifications of the state in memory or on disk with intern_lock
(with the exception of the really-often-modified state.records,
which is now protected with the log's lock, see ma_recovery.c
(look for "Recovery of the state"). Also, if maria_close() sees that
Checkpoint is looking at this table it will not my_free() the share.
- don't compute row's checksum twice in case of UPDATE (correction
to a bugfix I made yesterday).
storage/maria/ha_maria.cc:
protect state write with intern_lock (against Checkpoint)
storage/maria/ma_blockrec.c:
* don't reset trn->rec_lsn in _ma_unpin_all_pages(), because it
should wait until we have corrected the allocation in the bitmap
(as the REDO can serve to correct the allocation during Recovery);
introducing _ma_finalize_row() for that.
* In a changeset yesterday I moved computation of the checksum
into write_block_record(), to fix a bug in UPDATE. Now I notice
that maria_update() already computes the checksum, it's just that
it puts it into info->cur_row while _ma_update_block_record()
uses info->new_row; so, removing the checksum computation from
write_block_record(), putting it back into allocate_and_write_block_record()
(which is called only by INSERT and UNDO_DELETE), and copying
cur_row->checksum into new_row->checksum in _ma_update_block_record().
storage/maria/ma_check.c:
new prototypes, they will take intern_lock when writing the state;
also take intern_lock when changing share->kfile. In both cases
this is to protect against Checkpoint reading/writing the state or reading
kfile at the same time.
Not updating create_rename_lsn directly at end of write_log_record_for_repair()
as it wouldn't have intern_lock.
storage/maria/ma_close.c:
Checkpoint builds a list of shares (under THR_LOCK_maria), then it
handles each such share (under intern_lock) (doing flushing etc);
if maria_close() freed this share between the two, Checkpoint
would see a bad pointer. To avoid this, when building the list Checkpoint
marks each share, so that maria_close() knows it should not free it
and Checkpoint will free it itself.
Extending the zone covered by intern_lock to protect against
Checkpoint reading kfile, writing state.
storage/maria/ma_create.c:
When we update create_rename_lsn, we also update is_of_lsn to
the same value: it is logical, and allows us to test in maria_open()
that the former is not bigger than the latter (the contrary is a sign
of index header corruption, or severe logging bug which hinders
Recovery, table needs a repair).
_ma_update_create_rename_lsn_on_disk() also writes is_of_lsn;
it now operates under intern_lock (protect against Checkpoint),
a shortcut function is available for cases where acquiring
intern_lock is not needed (table's creation or first open).
storage/maria/ma_delete.c:
if table is transactional, "records" is already decremented
when logging UNDO_ROW_DELETE.
storage/maria/ma_delete_all.c:
comments
storage/maria/ma_extra.c:
Protect modifications of the state, in memory and/or on disk,
with intern_lock, against a concurrent Checkpoint.
When state goes to disk, update it's is_of_lsn (by calling
the new _ma_state_info_write()).
In HA_EXTRA_FORCE_REOPEN, don't set share->changed to 0 (undoing
a change I made a few days ago) and ASK_MONTY
storage/maria/ma_locking.c:
no real code change here.
storage/maria/ma_loghandler.c:
Log-write-hooks for updating "state.records" under log's mutex
when writing/updating/deleting a row or deleting all rows.
storage/maria/ma_loghandler_lsn.h:
merge (make LSN_ERROR and LSN_REPAIRED_BY_MARIA_CHK different)
storage/maria/ma_open.c:
When opening a table verify that is_of_lsn >= create_rename_lsn; if
false the header must be corrupted.
_ma_state_info_write() is split in two: _ma_state_info_write_sub()
which is the old _ma_state_info_write(), and _ma_state_info_write()
which additionally takes intern_lock if requested (to protect
against Checkpoint) and updates is_of_lsn.
_ma_open_keyfile() should change kfile.file under intern_lock
to protect Checkpoint from reading a wrong kfile.file.
storage/maria/ma_recovery.c:
Recovery of state.records: when the REDO phase sees UNDO_ROW_INSERT
which has a LSN > state.is_of_lsn it increments state.records.
Same for UNDO_ROW_DELETE and UNDO_ROW_PURGE.
When closing a table during Recovery, we know its state is at least
as new as the current log record we are looking at, so increase
is_of_lsn to the LSN of the current log record.
storage/maria/ma_rename.c:
update for new behaviour of _ma_update_create_rename_lsn_on_disk().
storage/maria/ma_test1.c:
update to new prototype
storage/maria/ma_test2.c:
update to new prototype (actually prototype was changed days ago,
but compiler does not complain about the extra argument??)
storage/maria/ma_test_recovery.expected:
new result file of ma_test_recovery. Improvements: record
count read from index's header is now always correct.
storage/maria/ma_test_recovery:
"rm" fails if file does not exist. Redirect stderr of script.
storage/maria/ma_write.c:
if table is transactional, "records" is already incremented when
logging UNDO_ROW_INSERT. Comments.
storage/maria/maria_chk.c:
update is_of_lsn too
storage/maria/maria_def.h:
- MARIA_STATE_INFO::is_of_lsn which is used by Recovery. It is stored
into the index file's header.
- Checkpoint can now mark a table as "don't free this", and maria_close()
can reply "ok then you will free it".
- new functions
storage/maria/maria_pack.c:
update for new name
2007-09-07 15:02:30 +02:00
|
|
|
else if ((!LSN_VALID(share->state.create_rename_lsn) ||
|
WL#3071 Maria checkpoint
Finally this is the real checkpoint code.
It however exhibits unstabilities when a checkpoint runs concurrently
with data-modifying clients (table corruption, transaction log's
assertions) so for now a checkpoint is taken only at startup after
recovery and at shutdown, i.e. not in concurrent situations. Later
we will let it run periodically, as well as flush dirty pages
periodically (almost all needed code is there already, only pagecache
code is written but not committed).
WL#3072 Maria recovery
* replacing UNDO_ROW_PURGE with CLR_END; testing of those CLR_END via
ma_test2 which has INSERTs failing with duplicate keys.
* replaying of REDO_RENAME_TABLE
Now, off to test Recovery in ha_maria :)
BitKeeper/deleted/.del-ma_least_recently_dirtied.c:
Delete: storage/maria/ma_least_recently_dirtied.c
BitKeeper/deleted/.del-ma_least_recently_dirtied.h:
Delete: storage/maria/ma_least_recently_dirtied.h
storage/maria/Makefile.am:
compile Checkpoint module
storage/maria/ha_maria.cc:
When ha_maria starts, do a recovery from last checkpoint.
Take a checkpoint when that recovery has ended and when ha_maria
shuts down cleanly.
storage/maria/ma_blockrec.c:
* even if my_sync() fails we have to my_close() (otherwise we leak
a descriptor)
* UNDO_ROW_PURGE is replaced by a simple CLR_END for UNDO_ROW_INSERT,
as promised in the old comment; it gives us skipping during the
UNDO phase.
storage/maria/ma_check.c:
All REDOs before create_rename_lsn are ignored by Recovery. So
create_rename_lsn must be set only after all data/index has been
flushed and forced to disk. We thus move write_log_record_for_repair()
to after _ma_flush_tables_files_after_repair().
storage/maria/ma_checkpoint.c:
Checkpoint module.
storage/maria/ma_checkpoint.h:
optional argument if caller wants a thread to periodically take
checkpoints and flush dirty pages.
storage/maria/ma_create.c:
* no need to init some vars as the initial bzero(share) takes care of this.
* update to new function's name
* even if we fail in my_sync() we have to my_close()
storage/maria/ma_extra.c:
Checkpoint reads share->last_version under intern_lock, so we make
maria_extra() update it under intern_lock. THR_LOCK_maria still needed
because of _ma_test_if_reopen().
storage/maria/ma_init.c:
destroy checkpoint module when Maria shuts down.
storage/maria/ma_loghandler.c:
* UNDO_ROW_PURGE gone (see ma_blockrec.c)
* we need to remember the LSN of the LOGREC_FILE_ID for a share,
because this LSN is needed into the checkpoint record (Recovery wants
to know the validity domain of an id->name mapping)
* translog_get_horizon_no_lock() needed for Checkpoint
* comment about failing assertion (Sanja knows)
* translog_init_reader_data() thought that translog_read_record_header_scan()
returns 0 in case of error, but 0 just means "0-length header".
* translog_assign_id_to_share() now needs the MARIA_HA because
LOGREC_FILE_ID uses a log-write hook.
* Verify that (de)assignment of share->id happens only under intern_lock,
as Checkpoint reads this id with intern_lock.
* translog_purge() can accept TRANSLOG_ADDRESS, not necessarily
a real LSN.
storage/maria/ma_loghandler.h:
prototype updates
storage/maria/ma_open.c:
no need to initialize "res"
storage/maria/ma_pagecache.c:
When taking a checkpoint, we don't need to know the maximum rec_lsn
of dirty pages; this LSN was intended to be used in the two-checkpoint
rule, but last_checkpoint_lsn is as good.
4 bytes for stored_list_size is enough as PAGECACHE::blocks (number
of blocks which the pagecache can contain) is int.
storage/maria/ma_pagecache.h:
new prototype
storage/maria/ma_recovery.c:
* added replaying of REDO_RENAME_TABLE
* UNDO_ROW_PURGE gone (see ma_blockrec.c), replaced by CLR_END
* Recovery from the last checkpoint record now possible
* In new_table() we skip the table if the id->name mapping is older than
create_rename_lsn (mapping dates from lsn_of_file_id).
* in get_MARIA_HA_from_REDO_record() we skip the record
if the id->name mapping is newer than the record (can happen if processing
a record which is before the checkpoint record).
* parse_checkpoint_record() has to return a LSN, that's what caller expects
storage/maria/ma_rename.c:
new function's name; log end zeroes of tables' names (ease recovery)
storage/maria/ma_test2.c:
* equivalent of ma_test1's --test-undo added (named -u here).
* -t=1 now stops right after creating the table, so that
we can test undoing of INSERTs with duplicate keys (which tests the
CLR_END logged by _ma_write_abort_block_record()).
storage/maria/ma_test_recovery.expected:
Result of testing undoing of INSERTs with duplicate keys; there are
some differences in maria_chk -dvv but they are normal (removing
records does not shrink data/index file, does not put back the
"analyzed, optimized keys"(etc) index state.
storage/maria/ma_test_recovery:
Test undoing of INSERTs with duplicate keys, using ma_test2;
when such INSERT happens, it logs REDO_INSERT, UNDO_INSERT, REDO_DELETE,
CLR_END; we abort after that, and test that CLR_END causes recovery
to jump over UNDO_INSERT.
storage/maria/ma_write.c:
comment
storage/maria/maria_chk.c:
comment
storage/maria/maria_def.h:
* a new bit in MARIA_SHARE::in_checkpoint, used to build a list
of unique shares during Checkpoint.
* MARIA_SHARE::lsn_of_file_id added: the LSN of the last LOGREC_FILE_ID
for this share; needed to know to which LSN domain the mappings
found in the Checkpoint record apply (new mappings should not apply
to old REDOs).
storage/maria/trnman.c:
* small changes to how trnman_collect_transactions() fills its buffer;
it also uses a non-dummy lsn_read_non_atomic() found in ma_checkpoint.h
2007-09-12 11:27:34 +02:00
|
|
|
!LSN_VALID(share->state.is_of_horizon) ||
|
- WL#3072 Maria Recovery:
Recovery of state.records (the count of records which is stored into
the header of the index file). For that, state.is_of_lsn is introduced;
logic is explained in ma_recovery.c (look for "Recovery of the state").
The net gain is that in case of crash, we now recover state.records,
and it is idempotent (ma_test_recovery tests it).
state.checksum is not recovered yet, mail sent for discussion.
- WL#3071 Maria Checkpoint: preparation for it, by protecting
all modifications of the state in memory or on disk with intern_lock
(with the exception of the really-often-modified state.records,
which is now protected with the log's lock, see ma_recovery.c
(look for "Recovery of the state"). Also, if maria_close() sees that
Checkpoint is looking at this table it will not my_free() the share.
- don't compute row's checksum twice in case of UPDATE (correction
to a bugfix I made yesterday).
storage/maria/ha_maria.cc:
protect state write with intern_lock (against Checkpoint)
storage/maria/ma_blockrec.c:
* don't reset trn->rec_lsn in _ma_unpin_all_pages(), because it
should wait until we have corrected the allocation in the bitmap
(as the REDO can serve to correct the allocation during Recovery);
introducing _ma_finalize_row() for that.
* In a changeset yesterday I moved computation of the checksum
into write_block_record(), to fix a bug in UPDATE. Now I notice
that maria_update() already computes the checksum, it's just that
it puts it into info->cur_row while _ma_update_block_record()
uses info->new_row; so, removing the checksum computation from
write_block_record(), putting it back into allocate_and_write_block_record()
(which is called only by INSERT and UNDO_DELETE), and copying
cur_row->checksum into new_row->checksum in _ma_update_block_record().
storage/maria/ma_check.c:
new prototypes, they will take intern_lock when writing the state;
also take intern_lock when changing share->kfile. In both cases
this is to protect against Checkpoint reading/writing the state or reading
kfile at the same time.
Not updating create_rename_lsn directly at end of write_log_record_for_repair()
as it wouldn't have intern_lock.
storage/maria/ma_close.c:
Checkpoint builds a list of shares (under THR_LOCK_maria), then it
handles each such share (under intern_lock) (doing flushing etc);
if maria_close() freed this share between the two, Checkpoint
would see a bad pointer. To avoid this, when building the list Checkpoint
marks each share, so that maria_close() knows it should not free it
and Checkpoint will free it itself.
Extending the zone covered by intern_lock to protect against
Checkpoint reading kfile, writing state.
storage/maria/ma_create.c:
When we update create_rename_lsn, we also update is_of_lsn to
the same value: it is logical, and allows us to test in maria_open()
that the former is not bigger than the latter (the contrary is a sign
of index header corruption, or severe logging bug which hinders
Recovery, table needs a repair).
_ma_update_create_rename_lsn_on_disk() also writes is_of_lsn;
it now operates under intern_lock (protect against Checkpoint),
a shortcut function is available for cases where acquiring
intern_lock is not needed (table's creation or first open).
storage/maria/ma_delete.c:
if table is transactional, "records" is already decremented
when logging UNDO_ROW_DELETE.
storage/maria/ma_delete_all.c:
comments
storage/maria/ma_extra.c:
Protect modifications of the state, in memory and/or on disk,
with intern_lock, against a concurrent Checkpoint.
When state goes to disk, update it's is_of_lsn (by calling
the new _ma_state_info_write()).
In HA_EXTRA_FORCE_REOPEN, don't set share->changed to 0 (undoing
a change I made a few days ago) and ASK_MONTY
storage/maria/ma_locking.c:
no real code change here.
storage/maria/ma_loghandler.c:
Log-write-hooks for updating "state.records" under log's mutex
when writing/updating/deleting a row or deleting all rows.
storage/maria/ma_loghandler_lsn.h:
merge (make LSN_ERROR and LSN_REPAIRED_BY_MARIA_CHK different)
storage/maria/ma_open.c:
When opening a table verify that is_of_lsn >= create_rename_lsn; if
false the header must be corrupted.
_ma_state_info_write() is split in two: _ma_state_info_write_sub()
which is the old _ma_state_info_write(), and _ma_state_info_write()
which additionally takes intern_lock if requested (to protect
against Checkpoint) and updates is_of_lsn.
_ma_open_keyfile() should change kfile.file under intern_lock
to protect Checkpoint from reading a wrong kfile.file.
storage/maria/ma_recovery.c:
Recovery of state.records: when the REDO phase sees UNDO_ROW_INSERT
which has a LSN > state.is_of_lsn it increments state.records.
Same for UNDO_ROW_DELETE and UNDO_ROW_PURGE.
When closing a table during Recovery, we know its state is at least
as new as the current log record we are looking at, so increase
is_of_lsn to the LSN of the current log record.
storage/maria/ma_rename.c:
update for new behaviour of _ma_update_create_rename_lsn_on_disk().
storage/maria/ma_test1.c:
update to new prototype
storage/maria/ma_test2.c:
update to new prototype (actually prototype was changed days ago,
but compiler does not complain about the extra argument??)
storage/maria/ma_test_recovery.expected:
new result file of ma_test_recovery. Improvements: record
count read from index's header is now always correct.
storage/maria/ma_test_recovery:
"rm" fails if file does not exist. Redirect stderr of script.
storage/maria/ma_write.c:
if table is transactional, "records" is already incremented when
logging UNDO_ROW_INSERT. Comments.
storage/maria/maria_chk.c:
update is_of_lsn too
storage/maria/maria_def.h:
- MARIA_STATE_INFO::is_of_lsn which is used by Recovery. It is stored
into the index file's header.
- Checkpoint can now mark a table as "don't free this", and maria_close()
can reply "ok then you will free it".
- new functions
storage/maria/maria_pack.c:
update for new name
2007-09-07 15:02:30 +02:00
|
|
|
(cmp_translog_addr(share->state.create_rename_lsn,
|
WL#3071 Maria checkpoint
Finally this is the real checkpoint code.
It however exhibits unstabilities when a checkpoint runs concurrently
with data-modifying clients (table corruption, transaction log's
assertions) so for now a checkpoint is taken only at startup after
recovery and at shutdown, i.e. not in concurrent situations. Later
we will let it run periodically, as well as flush dirty pages
periodically (almost all needed code is there already, only pagecache
code is written but not committed).
WL#3072 Maria recovery
* replacing UNDO_ROW_PURGE with CLR_END; testing of those CLR_END via
ma_test2 which has INSERTs failing with duplicate keys.
* replaying of REDO_RENAME_TABLE
Now, off to test Recovery in ha_maria :)
BitKeeper/deleted/.del-ma_least_recently_dirtied.c:
Delete: storage/maria/ma_least_recently_dirtied.c
BitKeeper/deleted/.del-ma_least_recently_dirtied.h:
Delete: storage/maria/ma_least_recently_dirtied.h
storage/maria/Makefile.am:
compile Checkpoint module
storage/maria/ha_maria.cc:
When ha_maria starts, do a recovery from last checkpoint.
Take a checkpoint when that recovery has ended and when ha_maria
shuts down cleanly.
storage/maria/ma_blockrec.c:
* even if my_sync() fails we have to my_close() (otherwise we leak
a descriptor)
* UNDO_ROW_PURGE is replaced by a simple CLR_END for UNDO_ROW_INSERT,
as promised in the old comment; it gives us skipping during the
UNDO phase.
storage/maria/ma_check.c:
All REDOs before create_rename_lsn are ignored by Recovery. So
create_rename_lsn must be set only after all data/index has been
flushed and forced to disk. We thus move write_log_record_for_repair()
to after _ma_flush_tables_files_after_repair().
storage/maria/ma_checkpoint.c:
Checkpoint module.
storage/maria/ma_checkpoint.h:
optional argument if caller wants a thread to periodically take
checkpoints and flush dirty pages.
storage/maria/ma_create.c:
* no need to init some vars as the initial bzero(share) takes care of this.
* update to new function's name
* even if we fail in my_sync() we have to my_close()
storage/maria/ma_extra.c:
Checkpoint reads share->last_version under intern_lock, so we make
maria_extra() update it under intern_lock. THR_LOCK_maria still needed
because of _ma_test_if_reopen().
storage/maria/ma_init.c:
destroy checkpoint module when Maria shuts down.
storage/maria/ma_loghandler.c:
* UNDO_ROW_PURGE gone (see ma_blockrec.c)
* we need to remember the LSN of the LOGREC_FILE_ID for a share,
because this LSN is needed into the checkpoint record (Recovery wants
to know the validity domain of an id->name mapping)
* translog_get_horizon_no_lock() needed for Checkpoint
* comment about failing assertion (Sanja knows)
* translog_init_reader_data() thought that translog_read_record_header_scan()
returns 0 in case of error, but 0 just means "0-length header".
* translog_assign_id_to_share() now needs the MARIA_HA because
LOGREC_FILE_ID uses a log-write hook.
* Verify that (de)assignment of share->id happens only under intern_lock,
as Checkpoint reads this id with intern_lock.
* translog_purge() can accept TRANSLOG_ADDRESS, not necessarily
a real LSN.
storage/maria/ma_loghandler.h:
prototype updates
storage/maria/ma_open.c:
no need to initialize "res"
storage/maria/ma_pagecache.c:
When taking a checkpoint, we don't need to know the maximum rec_lsn
of dirty pages; this LSN was intended to be used in the two-checkpoint
rule, but last_checkpoint_lsn is as good.
4 bytes for stored_list_size is enough as PAGECACHE::blocks (number
of blocks which the pagecache can contain) is int.
storage/maria/ma_pagecache.h:
new prototype
storage/maria/ma_recovery.c:
* added replaying of REDO_RENAME_TABLE
* UNDO_ROW_PURGE gone (see ma_blockrec.c), replaced by CLR_END
* Recovery from the last checkpoint record now possible
* In new_table() we skip the table if the id->name mapping is older than
create_rename_lsn (mapping dates from lsn_of_file_id).
* in get_MARIA_HA_from_REDO_record() we skip the record
if the id->name mapping is newer than the record (can happen if processing
a record which is before the checkpoint record).
* parse_checkpoint_record() has to return a LSN, that's what caller expects
storage/maria/ma_rename.c:
new function's name; log end zeroes of tables' names (ease recovery)
storage/maria/ma_test2.c:
* equivalent of ma_test1's --test-undo added (named -u here).
* -t=1 now stops right after creating the table, so that
we can test undoing of INSERTs with duplicate keys (which tests the
CLR_END logged by _ma_write_abort_block_record()).
storage/maria/ma_test_recovery.expected:
Result of testing undoing of INSERTs with duplicate keys; there are
some differences in maria_chk -dvv but they are normal (removing
records does not shrink data/index file, does not put back the
"analyzed, optimized keys"(etc) index state.
storage/maria/ma_test_recovery:
Test undoing of INSERTs with duplicate keys, using ma_test2;
when such INSERT happens, it logs REDO_INSERT, UNDO_INSERT, REDO_DELETE,
CLR_END; we abort after that, and test that CLR_END causes recovery
to jump over UNDO_INSERT.
storage/maria/ma_write.c:
comment
storage/maria/maria_chk.c:
comment
storage/maria/maria_def.h:
* a new bit in MARIA_SHARE::in_checkpoint, used to build a list
of unique shares during Checkpoint.
* MARIA_SHARE::lsn_of_file_id added: the LSN of the last LOGREC_FILE_ID
for this share; needed to know to which LSN domain the mappings
found in the Checkpoint record apply (new mappings should not apply
to old REDOs).
storage/maria/trnman.c:
* small changes to how trnman_collect_transactions() fills its buffer;
it also uses a non-dummy lsn_read_non_atomic() found in ma_checkpoint.h
2007-09-12 11:27:34 +02:00
|
|
|
share->state.is_of_horizon) > 0)) &&
|
WL#3072 Maria recovery
* create page cache before initializing engine and not after, because
Maria's recovery needs a page cache
* make the creation of a bitmap page more crash-resistent
* bugfix (see ma_blockrec.c)
* back to old way: create an 8k bitmap page when creating table
* preparations for the UNDO phase: recreate TRNs
* preparations for Checkpoint: list of dirty pages, testing
of rec_lsn to know if page should be skipped during Recovery
(unused in this patch as no Checkpoint module pushed yet)
* maria_chk tags repaired table with a special LSN
* reworking all around in ma_recovery.c (less duplication)
mysys/my_realloc.c:
noted an issue in my_realloc()
sql/mysqld.cc:
page cache needs to be created before engines are initialized,
because Maria's initialization may do a recovery which needs
the page cache.
storage/maria/ha_maria.cc:
update to new prototype
storage/maria/ma_bitmap.c:
when creating the first bitmap page we used chsize to 8192 bytes then
pwrite (overwrite) the last 2 bytes (8191-8192). If crash between
the two operations, this leaves a bitmap page full without its end
marker. A later recovery may try to read this page and find it
exists and misses a marker and conclude it's corrupted and fail.
Changing the chsize to only 8190 bytes: recovery will then find
the page is too short and recreate it entirely.
storage/maria/ma_blockrec.c:
Fix for a bug: when executing a REDO, if the data page is created,
data_file_length was increased before _ma_bitmap_set():
_ma_bitmap_set() called _ma_read_bitmap_page() which, due to the
increased data_file_length, expected to find a bitmap page on disk
with a correct end marker; if the bitmap page didn't exist already
in fact, this failed. Fixed by increasing data_file_length only after
_ma_read_bitmap_page() has created the new bitmap page correctly.
This bug could happen every time a REDO is about creating a new
bitmap page.
storage/maria/ma_check.c:
empty data file has a bitmap page
storage/maria/ma_control_file.c:
useless parameter to ma_control_file_create_or_open(), just
test if this is recovery.
storage/maria/ma_control_file.h:
new prototype
storage/maria/ma_create.c:
Back to how it was before: maria_create() creates an 8k bitmap page.
Thus (bugfix) data_file_length needs to reflect this instead of being 0.
storage/maria/ma_loghandler.c:
as ma_test1 and ma_test2 now use real transactions and not
dummy_transaction_object, REDO for INSERT/UPDATE/DELETE are always
about real transactions, can assert this.
A function for Recovery to assign a short id to a table.
storage/maria/ma_loghandler.h:
new function
storage/maria/ma_loghandler_lsn.h:
maria_chk tags repaired tables with this LSN
storage/maria/ma_open.c:
* enforce that DMLs on transactional tables use real transactions
and not dummy_transaction_object.
* test if table was repaired with maria_chk (which has to been
seen as an import of an external table into the server), test
validity of create_rename_lsn (header corruption detection)
* comments.
storage/maria/ma_recovery.c:
* preparations for the UNDO phase: recreate TRNs
* preparations for Checkpoint: list of dirty pages, testing
of rec_lsn to know if page should be skipped during Recovery
(unused in this patch as no Checkpoint module pushed yet)
* reworking all around (less duplication)
storage/maria/ma_recovery.h:
a parameter to say if the UNDO phase should be skipped
storage/maria/maria_chk.c:
tag repaired tables with a special LSN
storage/maria/maria_read_log.c:
* update to new prototype
* no UNDO phase in maria_read_log for now
storage/maria/trnman.c:
* a function for Recovery to create a transaction (TRN), needed
in the UNDO phase
* a function for Recovery to grab an existing transaction, needed
in the UNDO phase (rollback all existing transactions)
storage/maria/trnman_public.h:
new functions
2007-08-29 16:43:01 +02:00
|
|
|
!(open_flags & HA_OPEN_FOR_REPAIR))
|
|
|
|
{
|
|
|
|
/*
|
|
|
|
If in Recovery, it will not work. If LSN is invalid and not
|
|
|
|
LSN_REPAIRED_BY_MARIA_CHK, header must be corrupted.
|
|
|
|
In both cases, must repair.
|
|
|
|
*/
|
|
|
|
my_errno=((share->state.changed & STATE_CRASHED_ON_REPAIR) ?
|
|
|
|
HA_ERR_CRASHED_ON_REPAIR : HA_ERR_CRASHED_ON_USAGE);
|
|
|
|
goto err;
|
2007-06-27 12:58:08 +02:00
|
|
|
}
|
|
|
|
}
|
Maria:
* Don't modify share->base.born_transactional; now it is a value carved
in stone at creation time. share->now_transactional is what can be
modified: it starts at born_transactional, can become false during
ALTER TABLE (when we want no logging), and restored later.
* Not resetting create_rename_lsn to 0 during delete_all or repair.
* when we temporarily disable transactionality, we also change
the page type to PAGECACHE_PLAIN_PAGE: it bypasses some work in the
page cache (optimization), and avoids assertions related to LSNs.
* Disable INSERT DELAYED for transactional tables, because
durability could not be guaranteed (insertion may even not happen)
mysys/mf_keycache.c:
comment
storage/maria/ha_maria.cc:
* a transactional table cannot do INSERT DELAYED
* ha_maria::save_transactional not needed anymore, as now instead
we don't modify MARIA_SHARE::MARIA_BASE_INFO::born_transactional
(born_transactional plays the role of save_transactional), and modify
MARIA_SHARE::now_transactional.
* REPAIR_TABLE log record is now logged by maria_repair()
* comment why we rely on born_transactional to know if we should
skipping a transaction.
* putting together two if()s which test for F_UNLCK
storage/maria/ha_maria.h:
ha_maria::save_transactional not needed anymore (moved to the C layer)
storage/maria/ma_blockrec.c:
* For the block record's code (writing/updating/deleting records),
all that counts is now_transactional, not born_transactional.
* As we now set the page type to PAGECACHE_PLAIN_PAGE for tables
which have now_transactional==FALSE, pagecache will not expect
a meaningful LSN for them in pagecache_unlock_by_link(), so
we can pass it LSN_IMPOSSIBLE.
storage/maria/ma_check.c:
* writing LOGREC_REPAIR_TABLE moves from ha_maria::repair()
to maria_repair(), sounds cleaner (less functions to export).
* when opening a table during REPAIR, don't use the realpath-ed name,
as this may fail if the table has symlinked files (maria_open()
would try to find the data and index file in the directory
of unique_file_name, it would fail if data and index files are in
different dirs); use the unresolved name, open_file_name, which is
the argument which was passed to the maria_open() which created 'info'.
storage/maria/ma_close.c:
assert that when a statement is done with a table, it cleans up
storage/maria/ma_create.c:
new name
storage/maria/ma_delete_all.c:
* using now_transactional
* no reason to reset create_rename_lsn during delete_all (a bug);
also no reason to do it during repair: it was put there because
a positive create_rename_lsn caused a call to check_and_set_lsn()
which asserted in DBUG_ASSERT(block->type == PAGECACHE_LSN_PAGE);
first solution was to use LSN_IMPOSSIBLE in _ma_unpin_all_pages() if
not transactional; but then in the case of ALTER TABLE, with
transactionality temporarily disabled, it asserted in
DBUG_ASSERT(LSN_VALID(lsn)) in pagecache_fwrite() (PAGECACHE_LSN_PAGE
page with zero LSN - bad). The additional solution is to use
PAGECACHE_PLAIN_PAGE when we disable transactionality temporarily: this
avoids checks on the LSN, and also bypasses (optimization) the "flush
log up to LSN" call when the pagecache flushes our page (in other
words, no WAL needed).
storage/maria/ma_delete_table.c:
use now_transactional
storage/maria/ma_locking.c:
assert that when a statement is done with a table, it cleans up.
storage/maria/ma_loghandler.c:
* now_transactional should be used to test if we want a log record.
* Assertions to make sure dummy_transaction_object is not spoilt
by its many users.
storage/maria/ma_open.c:
base.transactional -> base.born_transactional
storage/maria/ma_pagecache.c:
missing name for page's type. Comment for future.
storage/maria/ma_rename.c:
use now_transactional
storage/maria/maria_chk.c:
use born_transactional
storage/maria/maria_def.h:
MARIA_BASE_INFO::transactional renamed to born_transactional.
MARIA_SHARE::now_transactional introduced.
_ma_repair_write_log_record() is made local to ma_check.c.
Macros to temporarily disable, and re-enable, transactionality for a
table.
storage/maria/maria_read_log.c:
assertions and using the new macros. Adding a forgotten resetting
when we finally close all tables.
2007-07-03 15:20:41 +02:00
|
|
|
else
|
|
|
|
share->page_type= PAGECACHE_PLAIN_PAGE;
|
|
|
|
share->now_transactional= share->base.born_transactional;
|
|
|
|
|
2007-01-18 21:38:14 +02:00
|
|
|
share->base.default_rec_buff_size= max(share->base.pack_reclength,
|
2007-04-19 13:18:56 +03:00
|
|
|
share->base.max_key_length);
|
2007-01-18 21:38:14 +02:00
|
|
|
if (share->data_file_type == DYNAMIC_RECORD)
|
|
|
|
{
|
|
|
|
share->base.extra_rec_buff_size=
|
|
|
|
(ALIGN_SIZE(MARIA_MAX_DYN_BLOCK_HEADER) + MARIA_SPLIT_LENGTH +
|
|
|
|
MARIA_REC_BUFF_OFFSET);
|
|
|
|
share->base.default_rec_buff_size+= share->base.extra_rec_buff_size;
|
|
|
|
}
|
|
|
|
disk_pos_assert(disk_pos + share->base.fields *MARIA_COLUMNDEF_SIZE,
|
|
|
|
end_pos);
|
|
|
|
for (i= j= 0 ; i < share->base.fields ; i++)
|
2006-04-11 16:45:10 +03:00
|
|
|
{
|
2007-04-19 13:18:56 +03:00
|
|
|
disk_pos=_ma_columndef_read(disk_pos,&share->columndef[i]);
|
|
|
|
share->columndef[i].pack_type=0;
|
|
|
|
share->columndef[i].huff_tree=0;
|
|
|
|
if (share->columndef[i].type == (int) FIELD_BLOB)
|
2006-04-11 16:45:10 +03:00
|
|
|
{
|
|
|
|
share->blobs[j].pack_length=
|
2007-04-19 13:18:56 +03:00
|
|
|
share->columndef[i].length-portable_sizeof_char_ptr;;
|
|
|
|
share->blobs[j].offset= share->columndef[i].offset;
|
2006-04-11 16:45:10 +03:00
|
|
|
j++;
|
|
|
|
}
|
|
|
|
}
|
2007-04-19 13:18:56 +03:00
|
|
|
share->columndef[i].type=(int) FIELD_LAST; /* End marker */
|
2007-06-07 01:01:43 +03:00
|
|
|
|
|
|
|
if ((share->data_file_type == BLOCK_RECORD ||
|
|
|
|
share->data_file_type == COMPRESSED_RECORD))
|
2007-03-01 18:23:58 +01:00
|
|
|
{
|
2007-06-07 01:01:43 +03:00
|
|
|
if (_ma_open_datafile(&info, share, -1))
|
|
|
|
goto err;
|
|
|
|
data_file= info.dfile.file;
|
2007-03-01 18:23:58 +01:00
|
|
|
}
|
2007-01-18 21:38:14 +02:00
|
|
|
errpos= 5;
|
2006-04-11 16:45:10 +03:00
|
|
|
|
2007-04-04 23:37:09 +03:00
|
|
|
share->kfile.file= kfile;
|
2006-04-11 16:45:10 +03:00
|
|
|
share->this_process=(ulong) getpid();
|
|
|
|
share->last_process= share->state.process;
|
|
|
|
share->base.key_parts=key_parts;
|
|
|
|
share->base.all_key_parts=key_parts+unique_key_parts;
|
|
|
|
if (!(share->last_version=share->state.version))
|
|
|
|
share->last_version=1; /* Safety */
|
|
|
|
share->rec_reflength=share->base.rec_reflength; /* May be changed */
|
|
|
|
share->base.margin_key_file_length=(share->base.max_key_file_length -
|
|
|
|
(keys ? MARIA_INDEX_BLOCK_MARGIN *
|
2007-01-18 21:38:14 +02:00
|
|
|
share->block_size * keys : 0));
|
|
|
|
share->block_size= share->base.block_size;
|
2007-10-04 20:33:42 +03:00
|
|
|
my_afree(disk_cache);
|
2006-04-11 16:45:10 +03:00
|
|
|
_ma_setup_functions(share);
|
2007-04-04 23:37:09 +03:00
|
|
|
if ((*share->once_init)(share, info.dfile.file))
|
2007-01-18 21:38:14 +02:00
|
|
|
goto err;
|
2006-09-07 17:07:17 +02:00
|
|
|
share->is_log_table= FALSE;
|
2007-06-07 01:01:43 +03:00
|
|
|
if (open_flags & HA_OPEN_TMP_TABLE)
|
|
|
|
share->options|= HA_OPTION_TMP_TABLE;
|
|
|
|
if (open_flags & HA_OPEN_DELAY_KEY_WRITE)
|
|
|
|
share->options|= HA_OPTION_DELAY_KEY_WRITE;
|
|
|
|
if (mode == O_RDONLY)
|
|
|
|
share->options|= HA_OPTION_READ_ONLY_DATA;
|
|
|
|
|
2006-04-11 16:45:10 +03:00
|
|
|
#ifdef THREAD
|
|
|
|
thr_lock_init(&share->lock);
|
|
|
|
VOID(pthread_mutex_init(&share->intern_lock,MY_MUTEX_INIT_FAST));
|
|
|
|
for (i=0; i<keys; i++)
|
|
|
|
VOID(my_rwlock_init(&share->key_root_lock[i], NULL));
|
|
|
|
VOID(my_rwlock_init(&share->mmap_lock, NULL));
|
|
|
|
if (!thr_lock_inited)
|
|
|
|
{
|
|
|
|
/* Probably a single threaded program; Don't use concurrent inserts */
|
|
|
|
maria_concurrent_insert=0;
|
|
|
|
}
|
|
|
|
else if (maria_concurrent_insert)
|
|
|
|
{
|
|
|
|
share->concurrent_insert=
|
|
|
|
((share->options & (HA_OPTION_READ_ONLY_DATA | HA_OPTION_TMP_TABLE |
|
|
|
|
HA_OPTION_COMPRESS_RECORD |
|
|
|
|
HA_OPTION_TEMP_COMPRESS_RECORD)) ||
|
|
|
|
(open_flags & HA_OPEN_TMP_TABLE) ||
|
2007-01-18 21:38:14 +02:00
|
|
|
share->data_file_type == BLOCK_RECORD ||
|
2007-06-07 01:01:43 +03:00
|
|
|
share->have_rtree) ? 0 : 1;
|
2006-04-11 16:45:10 +03:00
|
|
|
if (share->concurrent_insert)
|
|
|
|
{
|
|
|
|
share->lock.get_status=_ma_get_status;
|
|
|
|
share->lock.copy_status=_ma_copy_status;
|
WL#3072 Maria recovery
* create page cache before initializing engine and not after, because
Maria's recovery needs a page cache
* make the creation of a bitmap page more crash-resistent
* bugfix (see ma_blockrec.c)
* back to old way: create an 8k bitmap page when creating table
* preparations for the UNDO phase: recreate TRNs
* preparations for Checkpoint: list of dirty pages, testing
of rec_lsn to know if page should be skipped during Recovery
(unused in this patch as no Checkpoint module pushed yet)
* maria_chk tags repaired table with a special LSN
* reworking all around in ma_recovery.c (less duplication)
mysys/my_realloc.c:
noted an issue in my_realloc()
sql/mysqld.cc:
page cache needs to be created before engines are initialized,
because Maria's initialization may do a recovery which needs
the page cache.
storage/maria/ha_maria.cc:
update to new prototype
storage/maria/ma_bitmap.c:
when creating the first bitmap page we used chsize to 8192 bytes then
pwrite (overwrite) the last 2 bytes (8191-8192). If crash between
the two operations, this leaves a bitmap page full without its end
marker. A later recovery may try to read this page and find it
exists and misses a marker and conclude it's corrupted and fail.
Changing the chsize to only 8190 bytes: recovery will then find
the page is too short and recreate it entirely.
storage/maria/ma_blockrec.c:
Fix for a bug: when executing a REDO, if the data page is created,
data_file_length was increased before _ma_bitmap_set():
_ma_bitmap_set() called _ma_read_bitmap_page() which, due to the
increased data_file_length, expected to find a bitmap page on disk
with a correct end marker; if the bitmap page didn't exist already
in fact, this failed. Fixed by increasing data_file_length only after
_ma_read_bitmap_page() has created the new bitmap page correctly.
This bug could happen every time a REDO is about creating a new
bitmap page.
storage/maria/ma_check.c:
empty data file has a bitmap page
storage/maria/ma_control_file.c:
useless parameter to ma_control_file_create_or_open(), just
test if this is recovery.
storage/maria/ma_control_file.h:
new prototype
storage/maria/ma_create.c:
Back to how it was before: maria_create() creates an 8k bitmap page.
Thus (bugfix) data_file_length needs to reflect this instead of being 0.
storage/maria/ma_loghandler.c:
as ma_test1 and ma_test2 now use real transactions and not
dummy_transaction_object, REDO for INSERT/UPDATE/DELETE are always
about real transactions, can assert this.
A function for Recovery to assign a short id to a table.
storage/maria/ma_loghandler.h:
new function
storage/maria/ma_loghandler_lsn.h:
maria_chk tags repaired tables with this LSN
storage/maria/ma_open.c:
* enforce that DMLs on transactional tables use real transactions
and not dummy_transaction_object.
* test if table was repaired with maria_chk (which has to been
seen as an import of an external table into the server), test
validity of create_rename_lsn (header corruption detection)
* comments.
storage/maria/ma_recovery.c:
* preparations for the UNDO phase: recreate TRNs
* preparations for Checkpoint: list of dirty pages, testing
of rec_lsn to know if page should be skipped during Recovery
(unused in this patch as no Checkpoint module pushed yet)
* reworking all around (less duplication)
storage/maria/ma_recovery.h:
a parameter to say if the UNDO phase should be skipped
storage/maria/maria_chk.c:
tag repaired tables with a special LSN
storage/maria/maria_read_log.c:
* update to new prototype
* no UNDO phase in maria_read_log for now
storage/maria/trnman.c:
* a function for Recovery to create a transaction (TRN), needed
in the UNDO phase
* a function for Recovery to grab an existing transaction, needed
in the UNDO phase (rollback all existing transactions)
storage/maria/trnman_public.h:
new functions
2007-08-29 16:43:01 +02:00
|
|
|
/**
|
|
|
|
@todo RECOVERY
|
|
|
|
INSERT DELAYED and concurrent inserts are currently disabled for
|
|
|
|
transactional tables; when enabled again, we should re-evaluate
|
|
|
|
what problems the call to _ma_update_status() by
|
|
|
|
thr_reschedule_write_lock() can do (it may hurt Checkpoint as it
|
|
|
|
would be without intern_lock, and it modifies the state).
|
|
|
|
*/
|
2006-04-11 16:45:10 +03:00
|
|
|
share->lock.update_status=_ma_update_status;
|
2007-03-01 18:23:58 +01:00
|
|
|
share->lock.restore_status=_ma_restore_status;
|
2006-04-11 16:45:10 +03:00
|
|
|
share->lock.check_status=_ma_check_status;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
#endif
|
2007-07-27 12:06:39 +02:00
|
|
|
/*
|
|
|
|
Memory mapping can only be requested after initializing intern_lock.
|
|
|
|
*/
|
|
|
|
if (open_flags & HA_OPEN_MMAP)
|
|
|
|
{
|
|
|
|
info.s= share;
|
|
|
|
maria_extra(&info, HA_EXTRA_MMAP, 0);
|
|
|
|
}
|
2006-04-11 16:45:10 +03:00
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
|
|
|
share= old_info->s;
|
2007-01-18 21:38:14 +02:00
|
|
|
if (share->data_file_type == BLOCK_RECORD)
|
2007-06-07 01:01:43 +03:00
|
|
|
data_file= share->bitmap.file.file; /* Only opened once */
|
2006-04-11 16:45:10 +03:00
|
|
|
}
|
|
|
|
|
2007-06-07 01:01:43 +03:00
|
|
|
if (!(m_info= maria_clone_internal(share, mode, data_file)))
|
2006-04-11 16:45:10 +03:00
|
|
|
goto err;
|
|
|
|
pthread_mutex_unlock(&THR_LOCK_maria);
|
|
|
|
DBUG_RETURN(m_info);
|
|
|
|
|
|
|
|
err:
|
|
|
|
save_errno=my_errno ? my_errno : HA_ERR_END_OF_FILE;
|
|
|
|
if ((save_errno == HA_ERR_CRASHED) ||
|
|
|
|
(save_errno == HA_ERR_CRASHED_ON_USAGE) ||
|
|
|
|
(save_errno == HA_ERR_CRASHED_ON_REPAIR))
|
|
|
|
_ma_report_error(save_errno, name);
|
|
|
|
switch (errpos) {
|
|
|
|
case 5:
|
2007-06-07 01:01:43 +03:00
|
|
|
if (data_file >= 0)
|
|
|
|
VOID(my_close(data_file, MYF(0)));
|
2006-04-11 16:45:10 +03:00
|
|
|
if (old_info)
|
|
|
|
break; /* Don't remove open table */
|
2007-01-18 21:38:14 +02:00
|
|
|
(*share->once_end)(share);
|
2006-04-11 16:45:10 +03:00
|
|
|
/* fall through */
|
|
|
|
case 4:
|
2007-07-02 20:45:15 +03:00
|
|
|
my_free((uchar*) share,MYF(0));
|
2006-04-11 16:45:10 +03:00
|
|
|
/* fall through */
|
|
|
|
case 3:
|
|
|
|
/* fall through */
|
|
|
|
case 2:
|
2007-07-02 20:45:15 +03:00
|
|
|
my_afree((uchar*) disk_cache);
|
2006-04-11 16:45:10 +03:00
|
|
|
/* fall through */
|
|
|
|
case 1:
|
|
|
|
VOID(my_close(kfile,MYF(0)));
|
|
|
|
/* fall through */
|
|
|
|
case 0:
|
|
|
|
default:
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
pthread_mutex_unlock(&THR_LOCK_maria);
|
2007-06-07 01:01:43 +03:00
|
|
|
my_errno= save_errno;
|
2006-04-11 16:45:10 +03:00
|
|
|
DBUG_RETURN (NULL);
|
|
|
|
} /* maria_open */
|
|
|
|
|
|
|
|
|
2007-01-18 21:38:14 +02:00
|
|
|
/*
|
|
|
|
Reallocate a buffer, if the current buffer is not large enough
|
|
|
|
*/
|
2006-04-11 16:45:10 +03:00
|
|
|
|
2007-07-02 20:45:15 +03:00
|
|
|
my_bool _ma_alloc_buffer(uchar **old_addr, size_t *old_size,
|
|
|
|
size_t new_size)
|
2007-01-18 21:38:14 +02:00
|
|
|
{
|
|
|
|
if (*old_size < new_size)
|
2006-04-11 16:45:10 +03:00
|
|
|
{
|
2007-07-02 20:45:15 +03:00
|
|
|
uchar *addr;
|
|
|
|
if (!(addr= (uchar*) my_realloc((uchar*) *old_addr, new_size,
|
2007-01-18 21:38:14 +02:00
|
|
|
MYF(MY_ALLOW_ZERO_PTR))))
|
|
|
|
return 1;
|
|
|
|
*old_addr= addr;
|
|
|
|
*old_size= new_size;
|
2006-04-11 16:45:10 +03:00
|
|
|
}
|
2007-01-18 21:38:14 +02:00
|
|
|
return 0;
|
2006-04-11 16:45:10 +03:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
ulonglong _ma_safe_mul(ulonglong a, ulonglong b)
|
|
|
|
{
|
|
|
|
ulonglong max_val= ~ (ulonglong) 0; /* my_off_t is unsigned */
|
|
|
|
|
|
|
|
if (!a || max_val / a < b)
|
|
|
|
return max_val;
|
|
|
|
return a*b;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Set up functions in structs */
|
|
|
|
|
|
|
|
void _ma_setup_functions(register MARIA_SHARE *share)
|
|
|
|
{
|
2007-01-18 21:38:14 +02:00
|
|
|
share->once_init= maria_once_init_dummy;
|
|
|
|
share->once_end= maria_once_end_dummy;
|
|
|
|
share->init= maria_scan_init_dummy;
|
|
|
|
share->end= maria_scan_end_dummy;
|
2007-04-19 13:18:56 +03:00
|
|
|
share->scan_init= maria_scan_init_dummy;/* Compat. dummy function */
|
|
|
|
share->scan_end= maria_scan_end_dummy;/* Compat. dummy function */
|
2007-01-18 21:38:14 +02:00
|
|
|
share->write_record_init= _ma_write_init_default;
|
|
|
|
share->write_record_abort= _ma_write_abort_default;
|
|
|
|
|
|
|
|
switch (share->data_file_type) {
|
|
|
|
case COMPRESSED_RECORD:
|
2006-04-11 16:45:10 +03:00
|
|
|
share->read_record= _ma_read_pack_record;
|
2007-01-18 21:38:14 +02:00
|
|
|
share->scan= _ma_read_rnd_pack_record;
|
|
|
|
share->once_init= _ma_once_init_pack_row;
|
|
|
|
share->once_end= _ma_once_end_pack_row;
|
2007-04-19 13:18:56 +03:00
|
|
|
/*
|
|
|
|
Calculate checksum according to data in the original, not compressed,
|
|
|
|
row.
|
|
|
|
*/
|
2007-09-11 01:58:15 +03:00
|
|
|
if (share->state.header.org_data_file_type == STATIC_RECORD &&
|
|
|
|
! (share->options & HA_OPTION_NULL_FIELDS))
|
2006-04-11 16:45:10 +03:00
|
|
|
share->calc_checksum= _ma_static_checksum;
|
2007-01-18 21:38:14 +02:00
|
|
|
else
|
|
|
|
share->calc_checksum= _ma_checksum;
|
|
|
|
share->calc_write_checksum= share->calc_checksum;
|
|
|
|
break;
|
|
|
|
case DYNAMIC_RECORD:
|
2006-04-11 16:45:10 +03:00
|
|
|
share->read_record= _ma_read_dynamic_record;
|
2007-01-18 21:38:14 +02:00
|
|
|
share->scan= _ma_read_rnd_dynamic_record;
|
2006-04-11 16:45:10 +03:00
|
|
|
share->delete_record= _ma_delete_dynamic_record;
|
|
|
|
share->compare_record= _ma_cmp_dynamic_record;
|
|
|
|
share->compare_unique= _ma_cmp_dynamic_unique;
|
2007-01-18 21:38:14 +02:00
|
|
|
share->calc_checksum= share->calc_write_checksum= _ma_checksum;
|
2006-04-11 16:45:10 +03:00
|
|
|
/* add bits used to pack data to pack_reclength for faster allocation */
|
2007-01-18 21:38:14 +02:00
|
|
|
share->base.pack_reclength+= share->base.pack_bytes;
|
2006-04-11 16:45:10 +03:00
|
|
|
if (share->base.blobs)
|
|
|
|
{
|
|
|
|
share->update_record= _ma_update_blob_record;
|
|
|
|
share->write_record= _ma_write_blob_record;
|
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
|
|
|
share->write_record= _ma_write_dynamic_record;
|
|
|
|
share->update_record= _ma_update_dynamic_record;
|
|
|
|
}
|
2007-01-18 21:38:14 +02:00
|
|
|
break;
|
|
|
|
case STATIC_RECORD:
|
2006-04-11 16:45:10 +03:00
|
|
|
share->read_record= _ma_read_static_record;
|
2007-01-18 21:38:14 +02:00
|
|
|
share->scan= _ma_read_rnd_static_record;
|
2006-04-11 16:45:10 +03:00
|
|
|
share->delete_record= _ma_delete_static_record;
|
|
|
|
share->compare_record= _ma_cmp_static_record;
|
|
|
|
share->update_record= _ma_update_static_record;
|
|
|
|
share->write_record= _ma_write_static_record;
|
|
|
|
share->compare_unique= _ma_cmp_static_unique;
|
2007-09-11 01:58:15 +03:00
|
|
|
if (share->state.header.org_data_file_type == STATIC_RECORD &&
|
|
|
|
! (share->options & HA_OPTION_NULL_FIELDS))
|
|
|
|
share->calc_checksum= _ma_static_checksum;
|
|
|
|
else
|
|
|
|
share->calc_checksum= _ma_checksum;
|
2007-01-18 21:38:14 +02:00
|
|
|
break;
|
|
|
|
case BLOCK_RECORD:
|
2007-04-19 13:18:56 +03:00
|
|
|
share->once_init= _ma_once_init_block_record;
|
|
|
|
share->once_end= _ma_once_end_block_record;
|
|
|
|
share->init= _ma_init_block_record;
|
|
|
|
share->end= _ma_end_block_record;
|
2007-01-18 21:38:14 +02:00
|
|
|
share->write_record_init= _ma_write_init_block_record;
|
|
|
|
share->write_record_abort= _ma_write_abort_block_record;
|
|
|
|
share->scan_init= _ma_scan_init_block_record;
|
|
|
|
share->scan_end= _ma_scan_end_block_record;
|
|
|
|
share->read_record= _ma_read_block_record;
|
|
|
|
share->scan= _ma_scan_block_record;
|
|
|
|
share->delete_record= _ma_delete_block_record;
|
|
|
|
share->compare_record= _ma_compare_block_record;
|
|
|
|
share->update_record= _ma_update_block_record;
|
|
|
|
share->write_record= _ma_write_block_record;
|
|
|
|
share->compare_unique= _ma_cmp_block_unique;
|
|
|
|
share->calc_checksum= _ma_checksum;
|
2007-04-19 13:18:56 +03:00
|
|
|
/*
|
|
|
|
write_block_record() will calculate the checksum; Tell maria_write()
|
|
|
|
that it doesn't have to do this.
|
|
|
|
*/
|
2007-01-18 21:38:14 +02:00
|
|
|
share->calc_write_checksum= 0;
|
|
|
|
break;
|
2006-04-11 16:45:10 +03:00
|
|
|
}
|
|
|
|
share->file_read= _ma_nommap_pread;
|
|
|
|
share->file_write= _ma_nommap_pwrite;
|
2007-07-01 16:20:57 +03:00
|
|
|
share->calc_check_checksum= share->calc_checksum;
|
|
|
|
|
2007-01-18 21:38:14 +02:00
|
|
|
if (!(share->options & HA_OPTION_CHECKSUM) &&
|
|
|
|
share->data_file_type != COMPRESSED_RECORD)
|
|
|
|
share->calc_checksum= share->calc_write_checksum= 0;
|
2006-04-11 16:45:10 +03:00
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
static void setup_key_functions(register MARIA_KEYDEF *keyinfo)
|
|
|
|
{
|
|
|
|
if (keyinfo->key_alg == HA_KEY_ALG_RTREE)
|
|
|
|
{
|
|
|
|
#ifdef HAVE_RTREE_KEYS
|
|
|
|
keyinfo->ck_insert = maria_rtree_insert;
|
|
|
|
keyinfo->ck_delete = maria_rtree_delete;
|
|
|
|
#else
|
|
|
|
DBUG_ASSERT(0); /* maria_open should check it never happens */
|
|
|
|
#endif
|
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
|
|
|
keyinfo->ck_insert = _ma_ck_write;
|
|
|
|
keyinfo->ck_delete = _ma_ck_delete;
|
|
|
|
}
|
|
|
|
if (keyinfo->flag & HA_BINARY_PACK_KEY)
|
|
|
|
{ /* Simple prefix compression */
|
|
|
|
keyinfo->bin_search= _ma_seq_search;
|
|
|
|
keyinfo->get_key= _ma_get_binary_pack_key;
|
|
|
|
keyinfo->pack_key= _ma_calc_bin_pack_key_length;
|
|
|
|
keyinfo->store_key= _ma_store_bin_pack_key;
|
|
|
|
}
|
|
|
|
else if (keyinfo->flag & HA_VAR_LENGTH_KEY)
|
|
|
|
{
|
|
|
|
keyinfo->get_key= _ma_get_pack_key;
|
|
|
|
if (keyinfo->seg[0].flag & HA_PACK_KEY)
|
|
|
|
{ /* Prefix compression */
|
|
|
|
if (!keyinfo->seg->charset || use_strnxfrm(keyinfo->seg->charset) ||
|
|
|
|
(keyinfo->seg->flag & HA_NULL_PART))
|
|
|
|
keyinfo->bin_search= _ma_seq_search;
|
|
|
|
else
|
|
|
|
keyinfo->bin_search= _ma_prefix_search;
|
|
|
|
keyinfo->pack_key= _ma_calc_var_pack_key_length;
|
|
|
|
keyinfo->store_key= _ma_store_var_pack_key;
|
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
|
|
|
keyinfo->bin_search= _ma_seq_search;
|
|
|
|
keyinfo->pack_key= _ma_calc_var_key_length; /* Variable length key */
|
|
|
|
keyinfo->store_key= _ma_store_static_key;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
|
|
|
keyinfo->bin_search= _ma_bin_search;
|
|
|
|
keyinfo->get_key= _ma_get_static_key;
|
|
|
|
keyinfo->pack_key= _ma_calc_static_key_length;
|
|
|
|
keyinfo->store_key= _ma_store_static_key;
|
|
|
|
}
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
|
- WL#3239 "log CREATE TABLE in Maria"
- WL#3240 "log DROP TABLE in Maria"
- similarly, log RENAME TABLE, REPAIR/OPTIMIZE TABLE, and
DELETE no_WHERE_clause (== the DELETE which just truncates the files)
- create_rename_lsn added to MARIA_SHARE's state
- all these operations (except DROP TABLE) also update the table's
create_rename_lsn, which is needed for the correctness of
Recovery (see function comment of _ma_repair_write_log_record()
in ma_check.c)
- write a COMMIT record when transaction commits.
- don't log REDOs/UNDOs if this is an internal temporary table
like inside ALTER TABLE (I expect this to be a big win). There was
already no logging for user-created "CREATE TEMPORARY" tables.
- don't fsync files/directories if the table is not transactional
- in translog_write_record(), autogenerate a 2-byte-id for the table
and log the "id->name" pair (LOGREC_FILE_ID); log
LOGREC_LONG_TRANSACTION_ID; automatically store
the table's 2-byte-id in any log record.
- preparations for Checkpoint: translog_get_horizon(); pausing Checkpoint
when some dirty pages are unknown; capturing trn->rec_lsn,
trn->first_undo_lsn for Checkpoint and log's low-water-mark computing.
- assertions, comments.
storage/maria/Makefile.am:
more files to build
storage/maria/ha_maria.cc:
- logging a REPAIR log record if REPAIR/OPTIMIZE was successful.
- ha_maria::data_file_type does not have to be set in every info()
call, just do it once in open().
- if caller said that transactionality can be disabled (like if
caller is ALTER TABLE) i.e. thd->transaction.on==FALSE, then we
temporarily disable transactionality of the table in external_lock();
that will ensure that no REDOs/UNDOs are logged for this possibly
massive write operation (they are not needed, as if any write fails,
the table will be dropped). We re-enable in external_lock(F_UNLCK),
which in ALTER TABLE happens before the tmp table replaces the original
one (which is good, as thus the final table will have a REDO RENAME
and a correct create_rename_lsn).
- when we commit we also have to write a log record, so
trnman_commit_trn() calls become ma_commit() calls
- at end of engine's initialization, we are potentially entering a
multi-threaded dangerous world (clients are going to be accepted)
and so some assertions of mutex-owning become enforceable, for that
we set maria_multi_threaded=TRUE (see ma_control_file.c)
storage/maria/ha_maria.h:
new member ha_maria::save_transactional (see also ha_maria.cc)
storage/maria/ma_blockrec.c:
- fixing comments according to discussion with Monty
- if a table is transactional but temporarily non-transactional
(like in ALTER TABLE), we need to give a sensible LSN to the pages
(and, if we give 0, pagecache asserts).
- translog_write_record() now takes care of storing the share's
2-byte-id in the log record
storage/maria/ma_blockrec.h:
fixing comment according to discussion with Monty
storage/maria/ma_check.c:
When REPAIR/OPTIMIZE modify the data/index file, if this is a
transactional table, they must sync it; if they remove files or rename
files, they must sync the directory, so that everything is durable.
This is just applying to REPAIR/OPTIMIZE the logic already implemented
in CREATE/DROP/RENAME a few months ago.
Adding a function to write a LOGREC_REPAIR_TABLE at end of
REPAIR/OPTIMIZE (called only by ha_maria, not by maria_chk), and
to update the table's create_rename_lsn.
storage/maria/ma_close.c:
fix for a future bug
storage/maria/ma_control_file.c:
ensuring that if Maria is running in multi-threaded mode, anybody
wanting to write to the control file and update
last_checkpoint_lsn/last_logno owns the log's lock.
storage/maria/ma_control_file.h:
see ma_control_file.c
storage/maria/ma_create.c:
when creating a table:
- sync it and its directory only if this is a transactional table
and there is a log (no point in syncing in maria_chk)
- decouple the two uses of linkname/linkname_ptr (for index file and
for data file) into more variables, as we need to know all links
until the moment we write the LOGREC_CREATE_TABLE.
- set share.data_file_type early so that _ma_initialize_data_file()
knows it (Monty's bugfix so that a table always has at least a bitmap
page when it is created; so data-file is not 0 bytes anymore).
- log a LOGREC_CREATE_TABLE; it contains the bytes which we have
just written to the index file's header. Update table's
create_rename_lsn.
- syncing of kfile had been bugified in a previous merge, correcting
- syncing of dfile is now needed as it's not empty anymore
- in _ma_initialize_data_file(), use share's block_size and not the
global one. This is a gratuitous change, both variables are equal,
just that I find it more future-proof to use share-bound variable
rather than global one.
storage/maria/ma_delete_all.c:
log a LOGREC_DELETE_ALL record when doing ma_delete_all_rows();
update create_rename_lsn then.
storage/maria/ma_delete_table.c:
- logging LOGREC_DROP_TABLE; knowing if this is needed, requires
knowing if the table is transactional, which requires opening the
table.
- we need to sync directories only if the table is transactional
storage/maria/ma_extra.c:
questions
storage/maria/ma_init.c:
when maria_end() is called, engine is not multithreaded
storage/maria/ma_loghandler.c:
- translog_inited has to be visible to ma_create() (see how it is used
in ma_create())
- checkpoint record will be a single record, not three
- no REDO for TRUNCATE (TRUNCATE calls ma_create() internally so will
log a REDO_CREATE)
- adding REDO for DELETE no_WHERE_clause (fast DELETE of all rows by
truncating the files), REPAIR.
- MY_WAIT_IF_FULL to wait&retry if a log write hits a full disk
- in translog_write_record(), if MARIA_SHARE does not yet have a
2-byte-id, generate one for it and log LOGREC_FILE_ID; automatically
store this short id into log records.
- in translog_write_record(), if transaction has not logged its
long trid, log LOGREC_LONG_TRANSACTION_ID.
- For Checkpoint, we need to know the current end-of-log: adding
translog_get_horizon().
- For Control File, adding an assertion that the thread owns the
log's lock (control file is protected by this lock)
storage/maria/ma_loghandler.h:
Changes in log records (see ma_loghandler.c).
new prototypes, new functions.
storage/maria/ma_loghandler_lsn.h:
adding a type LSN_WITH_FLAGS especially for TRN::first_undo_lsn,
where the most significant byte is used for flags.
storage/maria/ma_open.c:
storing the create_rename_lsn in the index file's header (in the
state, precisely) and retrieving it from there.
storage/maria/ma_pagecache.c:
- my set_if_bigger was wrong, correcting it
- if the first_in_switch list is not empty, it means that
changed_blocks misses some dirty pages, so Checkpoint cannot run and
needs to wait. A variable missing_blocks_in_changed_list is added to
tell that (should it be named missing_blocks_in_changed_blocks?)
- pagecache_collect_changed_blocks_with_lsn() now also tells the
minimum rec_lsn (needed for low-water mark computation).
storage/maria/ma_pagecache.h:
see ma_pagecache.c
storage/maria/ma_panic.c:
comment
storage/maria/ma_range.c:
comment
storage/maria/ma_rename.c:
- logging LOGREC_RENAME_TABLE; knowing if this is needed, requires
knowing if the table is transactional, which requires opening the
table.
- update create_rename_lsn
- we need to sync directories only if the table is transactional
storage/maria/ma_static.c:
comment
storage/maria/ma_test_all.sh:
- tip for Valgrind-ing ma_test_all
- do "export maria_path=somepath" before calling ma_test_all,
if you want to run ma_test_all out of storage/maria (useful
to have parallel runs, like one normal and one Valgrind, they
must not use the same tables so need to run in different directories)
storage/maria/maria_def.h:
- state now contains, in memory and on disk, the create_rename_lsn
- share now contains a 2-byte-id
storage/maria/trnman.c:
preparations for Checkpoint: capture trn->rec_lsn, trn->first_undo_lsn;
minimum first_undo_lsn needed to know log's low-water-mark
storage/maria/trnman.h:
using most significant byte of first_undo_lsn to hold miscellaneous
flags, for now TRANSACTION_LOGGED_LONG_ID.
dummy_transaction_object is already declared in ma_static.c.
storage/maria/trnman_public.h:
dummy_transaction_object was declared in all files including
trnman_public.h, while in fact it's a single object.
new prototype
storage/maria/unittest/ma_test_loghandler-t.c:
update for new prototype
storage/maria/unittest/ma_test_loghandler_multigroup-t.c:
update for new prototype
storage/maria/unittest/ma_test_loghandler_multithread-t.c:
update for new prototype
storage/maria/unittest/ma_test_loghandler_pagecache-t.c:
update for new prototype
storage/maria/ma_commit.c:
function which wraps:
- writing a LOGREC_COMMIT record (==commit on disk)
- calling trnman_commit_trn() (=commit in memory)
storage/maria/ma_commit.h:
new header file
.tree-is-private:
this file is now needed to keep our tree private (don't push it
to public trees). When 5.1 is merged into mysql-maria, we can abandon
our maria-specific post-commit trigger; .tree_is_private will take
care of keeping commit mails private. Don't push this file to public
trees.
2007-06-22 14:49:37 +02:00
|
|
|
/**
|
|
|
|
@brief Function to save and store the header in the index file (.MYI)
|
|
|
|
|
- WL#3072 Maria Recovery:
Recovery of state.records (the count of records which is stored into
the header of the index file). For that, state.is_of_lsn is introduced;
logic is explained in ma_recovery.c (look for "Recovery of the state").
The net gain is that in case of crash, we now recover state.records,
and it is idempotent (ma_test_recovery tests it).
state.checksum is not recovered yet, mail sent for discussion.
- WL#3071 Maria Checkpoint: preparation for it, by protecting
all modifications of the state in memory or on disk with intern_lock
(with the exception of the really-often-modified state.records,
which is now protected with the log's lock, see ma_recovery.c
(look for "Recovery of the state"). Also, if maria_close() sees that
Checkpoint is looking at this table it will not my_free() the share.
- don't compute row's checksum twice in case of UPDATE (correction
to a bugfix I made yesterday).
storage/maria/ha_maria.cc:
protect state write with intern_lock (against Checkpoint)
storage/maria/ma_blockrec.c:
* don't reset trn->rec_lsn in _ma_unpin_all_pages(), because it
should wait until we have corrected the allocation in the bitmap
(as the REDO can serve to correct the allocation during Recovery);
introducing _ma_finalize_row() for that.
* In a changeset yesterday I moved computation of the checksum
into write_block_record(), to fix a bug in UPDATE. Now I notice
that maria_update() already computes the checksum, it's just that
it puts it into info->cur_row while _ma_update_block_record()
uses info->new_row; so, removing the checksum computation from
write_block_record(), putting it back into allocate_and_write_block_record()
(which is called only by INSERT and UNDO_DELETE), and copying
cur_row->checksum into new_row->checksum in _ma_update_block_record().
storage/maria/ma_check.c:
new prototypes, they will take intern_lock when writing the state;
also take intern_lock when changing share->kfile. In both cases
this is to protect against Checkpoint reading/writing the state or reading
kfile at the same time.
Not updating create_rename_lsn directly at end of write_log_record_for_repair()
as it wouldn't have intern_lock.
storage/maria/ma_close.c:
Checkpoint builds a list of shares (under THR_LOCK_maria), then it
handles each such share (under intern_lock) (doing flushing etc);
if maria_close() freed this share between the two, Checkpoint
would see a bad pointer. To avoid this, when building the list Checkpoint
marks each share, so that maria_close() knows it should not free it
and Checkpoint will free it itself.
Extending the zone covered by intern_lock to protect against
Checkpoint reading kfile, writing state.
storage/maria/ma_create.c:
When we update create_rename_lsn, we also update is_of_lsn to
the same value: it is logical, and allows us to test in maria_open()
that the former is not bigger than the latter (the contrary is a sign
of index header corruption, or severe logging bug which hinders
Recovery, table needs a repair).
_ma_update_create_rename_lsn_on_disk() also writes is_of_lsn;
it now operates under intern_lock (protect against Checkpoint),
a shortcut function is available for cases where acquiring
intern_lock is not needed (table's creation or first open).
storage/maria/ma_delete.c:
if table is transactional, "records" is already decremented
when logging UNDO_ROW_DELETE.
storage/maria/ma_delete_all.c:
comments
storage/maria/ma_extra.c:
Protect modifications of the state, in memory and/or on disk,
with intern_lock, against a concurrent Checkpoint.
When state goes to disk, update it's is_of_lsn (by calling
the new _ma_state_info_write()).
In HA_EXTRA_FORCE_REOPEN, don't set share->changed to 0 (undoing
a change I made a few days ago) and ASK_MONTY
storage/maria/ma_locking.c:
no real code change here.
storage/maria/ma_loghandler.c:
Log-write-hooks for updating "state.records" under log's mutex
when writing/updating/deleting a row or deleting all rows.
storage/maria/ma_loghandler_lsn.h:
merge (make LSN_ERROR and LSN_REPAIRED_BY_MARIA_CHK different)
storage/maria/ma_open.c:
When opening a table verify that is_of_lsn >= create_rename_lsn; if
false the header must be corrupted.
_ma_state_info_write() is split in two: _ma_state_info_write_sub()
which is the old _ma_state_info_write(), and _ma_state_info_write()
which additionally takes intern_lock if requested (to protect
against Checkpoint) and updates is_of_lsn.
_ma_open_keyfile() should change kfile.file under intern_lock
to protect Checkpoint from reading a wrong kfile.file.
storage/maria/ma_recovery.c:
Recovery of state.records: when the REDO phase sees UNDO_ROW_INSERT
which has a LSN > state.is_of_lsn it increments state.records.
Same for UNDO_ROW_DELETE and UNDO_ROW_PURGE.
When closing a table during Recovery, we know its state is at least
as new as the current log record we are looking at, so increase
is_of_lsn to the LSN of the current log record.
storage/maria/ma_rename.c:
update for new behaviour of _ma_update_create_rename_lsn_on_disk().
storage/maria/ma_test1.c:
update to new prototype
storage/maria/ma_test2.c:
update to new prototype (actually prototype was changed days ago,
but compiler does not complain about the extra argument??)
storage/maria/ma_test_recovery.expected:
new result file of ma_test_recovery. Improvements: record
count read from index's header is now always correct.
storage/maria/ma_test_recovery:
"rm" fails if file does not exist. Redirect stderr of script.
storage/maria/ma_write.c:
if table is transactional, "records" is already incremented when
logging UNDO_ROW_INSERT. Comments.
storage/maria/maria_chk.c:
update is_of_lsn too
storage/maria/maria_def.h:
- MARIA_STATE_INFO::is_of_lsn which is used by Recovery. It is stored
into the index file's header.
- Checkpoint can now mark a table as "don't free this", and maria_close()
can reply "ok then you will free it".
- new functions
storage/maria/maria_pack.c:
update for new name
2007-09-07 15:02:30 +02:00
|
|
|
Operates under MARIA_SHARE::intern_lock if requested.
|
WL#3071 Maria checkpoint
Finally this is the real checkpoint code.
It however exhibits unstabilities when a checkpoint runs concurrently
with data-modifying clients (table corruption, transaction log's
assertions) so for now a checkpoint is taken only at startup after
recovery and at shutdown, i.e. not in concurrent situations. Later
we will let it run periodically, as well as flush dirty pages
periodically (almost all needed code is there already, only pagecache
code is written but not committed).
WL#3072 Maria recovery
* replacing UNDO_ROW_PURGE with CLR_END; testing of those CLR_END via
ma_test2 which has INSERTs failing with duplicate keys.
* replaying of REDO_RENAME_TABLE
Now, off to test Recovery in ha_maria :)
BitKeeper/deleted/.del-ma_least_recently_dirtied.c:
Delete: storage/maria/ma_least_recently_dirtied.c
BitKeeper/deleted/.del-ma_least_recently_dirtied.h:
Delete: storage/maria/ma_least_recently_dirtied.h
storage/maria/Makefile.am:
compile Checkpoint module
storage/maria/ha_maria.cc:
When ha_maria starts, do a recovery from last checkpoint.
Take a checkpoint when that recovery has ended and when ha_maria
shuts down cleanly.
storage/maria/ma_blockrec.c:
* even if my_sync() fails we have to my_close() (otherwise we leak
a descriptor)
* UNDO_ROW_PURGE is replaced by a simple CLR_END for UNDO_ROW_INSERT,
as promised in the old comment; it gives us skipping during the
UNDO phase.
storage/maria/ma_check.c:
All REDOs before create_rename_lsn are ignored by Recovery. So
create_rename_lsn must be set only after all data/index has been
flushed and forced to disk. We thus move write_log_record_for_repair()
to after _ma_flush_tables_files_after_repair().
storage/maria/ma_checkpoint.c:
Checkpoint module.
storage/maria/ma_checkpoint.h:
optional argument if caller wants a thread to periodically take
checkpoints and flush dirty pages.
storage/maria/ma_create.c:
* no need to init some vars as the initial bzero(share) takes care of this.
* update to new function's name
* even if we fail in my_sync() we have to my_close()
storage/maria/ma_extra.c:
Checkpoint reads share->last_version under intern_lock, so we make
maria_extra() update it under intern_lock. THR_LOCK_maria still needed
because of _ma_test_if_reopen().
storage/maria/ma_init.c:
destroy checkpoint module when Maria shuts down.
storage/maria/ma_loghandler.c:
* UNDO_ROW_PURGE gone (see ma_blockrec.c)
* we need to remember the LSN of the LOGREC_FILE_ID for a share,
because this LSN is needed into the checkpoint record (Recovery wants
to know the validity domain of an id->name mapping)
* translog_get_horizon_no_lock() needed for Checkpoint
* comment about failing assertion (Sanja knows)
* translog_init_reader_data() thought that translog_read_record_header_scan()
returns 0 in case of error, but 0 just means "0-length header".
* translog_assign_id_to_share() now needs the MARIA_HA because
LOGREC_FILE_ID uses a log-write hook.
* Verify that (de)assignment of share->id happens only under intern_lock,
as Checkpoint reads this id with intern_lock.
* translog_purge() can accept TRANSLOG_ADDRESS, not necessarily
a real LSN.
storage/maria/ma_loghandler.h:
prototype updates
storage/maria/ma_open.c:
no need to initialize "res"
storage/maria/ma_pagecache.c:
When taking a checkpoint, we don't need to know the maximum rec_lsn
of dirty pages; this LSN was intended to be used in the two-checkpoint
rule, but last_checkpoint_lsn is as good.
4 bytes for stored_list_size is enough as PAGECACHE::blocks (number
of blocks which the pagecache can contain) is int.
storage/maria/ma_pagecache.h:
new prototype
storage/maria/ma_recovery.c:
* added replaying of REDO_RENAME_TABLE
* UNDO_ROW_PURGE gone (see ma_blockrec.c), replaced by CLR_END
* Recovery from the last checkpoint record now possible
* In new_table() we skip the table if the id->name mapping is older than
create_rename_lsn (mapping dates from lsn_of_file_id).
* in get_MARIA_HA_from_REDO_record() we skip the record
if the id->name mapping is newer than the record (can happen if processing
a record which is before the checkpoint record).
* parse_checkpoint_record() has to return a LSN, that's what caller expects
storage/maria/ma_rename.c:
new function's name; log end zeroes of tables' names (ease recovery)
storage/maria/ma_test2.c:
* equivalent of ma_test1's --test-undo added (named -u here).
* -t=1 now stops right after creating the table, so that
we can test undoing of INSERTs with duplicate keys (which tests the
CLR_END logged by _ma_write_abort_block_record()).
storage/maria/ma_test_recovery.expected:
Result of testing undoing of INSERTs with duplicate keys; there are
some differences in maria_chk -dvv but they are normal (removing
records does not shrink data/index file, does not put back the
"analyzed, optimized keys"(etc) index state.
storage/maria/ma_test_recovery:
Test undoing of INSERTs with duplicate keys, using ma_test2;
when such INSERT happens, it logs REDO_INSERT, UNDO_INSERT, REDO_DELETE,
CLR_END; we abort after that, and test that CLR_END causes recovery
to jump over UNDO_INSERT.
storage/maria/ma_write.c:
comment
storage/maria/maria_chk.c:
comment
storage/maria/maria_def.h:
* a new bit in MARIA_SHARE::in_checkpoint, used to build a list
of unique shares during Checkpoint.
* MARIA_SHARE::lsn_of_file_id added: the LSN of the last LOGREC_FILE_ID
for this share; needed to know to which LSN domain the mappings
found in the Checkpoint record apply (new mappings should not apply
to old REDOs).
storage/maria/trnman.c:
* small changes to how trnman_collect_transactions() fills its buffer;
it also uses a non-dummy lsn_read_non_atomic() found in ma_checkpoint.h
2007-09-12 11:27:34 +02:00
|
|
|
Sets MARIA_SHARE::MARIA_STATE_INFO::is_of_horizon if transactional table.
|
- WL#3072 Maria Recovery:
Recovery of state.records (the count of records which is stored into
the header of the index file). For that, state.is_of_lsn is introduced;
logic is explained in ma_recovery.c (look for "Recovery of the state").
The net gain is that in case of crash, we now recover state.records,
and it is idempotent (ma_test_recovery tests it).
state.checksum is not recovered yet, mail sent for discussion.
- WL#3071 Maria Checkpoint: preparation for it, by protecting
all modifications of the state in memory or on disk with intern_lock
(with the exception of the really-often-modified state.records,
which is now protected with the log's lock, see ma_recovery.c
(look for "Recovery of the state"). Also, if maria_close() sees that
Checkpoint is looking at this table it will not my_free() the share.
- don't compute row's checksum twice in case of UPDATE (correction
to a bugfix I made yesterday).
storage/maria/ha_maria.cc:
protect state write with intern_lock (against Checkpoint)
storage/maria/ma_blockrec.c:
* don't reset trn->rec_lsn in _ma_unpin_all_pages(), because it
should wait until we have corrected the allocation in the bitmap
(as the REDO can serve to correct the allocation during Recovery);
introducing _ma_finalize_row() for that.
* In a changeset yesterday I moved computation of the checksum
into write_block_record(), to fix a bug in UPDATE. Now I notice
that maria_update() already computes the checksum, it's just that
it puts it into info->cur_row while _ma_update_block_record()
uses info->new_row; so, removing the checksum computation from
write_block_record(), putting it back into allocate_and_write_block_record()
(which is called only by INSERT and UNDO_DELETE), and copying
cur_row->checksum into new_row->checksum in _ma_update_block_record().
storage/maria/ma_check.c:
new prototypes, they will take intern_lock when writing the state;
also take intern_lock when changing share->kfile. In both cases
this is to protect against Checkpoint reading/writing the state or reading
kfile at the same time.
Not updating create_rename_lsn directly at end of write_log_record_for_repair()
as it wouldn't have intern_lock.
storage/maria/ma_close.c:
Checkpoint builds a list of shares (under THR_LOCK_maria), then it
handles each such share (under intern_lock) (doing flushing etc);
if maria_close() freed this share between the two, Checkpoint
would see a bad pointer. To avoid this, when building the list Checkpoint
marks each share, so that maria_close() knows it should not free it
and Checkpoint will free it itself.
Extending the zone covered by intern_lock to protect against
Checkpoint reading kfile, writing state.
storage/maria/ma_create.c:
When we update create_rename_lsn, we also update is_of_lsn to
the same value: it is logical, and allows us to test in maria_open()
that the former is not bigger than the latter (the contrary is a sign
of index header corruption, or severe logging bug which hinders
Recovery, table needs a repair).
_ma_update_create_rename_lsn_on_disk() also writes is_of_lsn;
it now operates under intern_lock (protect against Checkpoint),
a shortcut function is available for cases where acquiring
intern_lock is not needed (table's creation or first open).
storage/maria/ma_delete.c:
if table is transactional, "records" is already decremented
when logging UNDO_ROW_DELETE.
storage/maria/ma_delete_all.c:
comments
storage/maria/ma_extra.c:
Protect modifications of the state, in memory and/or on disk,
with intern_lock, against a concurrent Checkpoint.
When state goes to disk, update it's is_of_lsn (by calling
the new _ma_state_info_write()).
In HA_EXTRA_FORCE_REOPEN, don't set share->changed to 0 (undoing
a change I made a few days ago) and ASK_MONTY
storage/maria/ma_locking.c:
no real code change here.
storage/maria/ma_loghandler.c:
Log-write-hooks for updating "state.records" under log's mutex
when writing/updating/deleting a row or deleting all rows.
storage/maria/ma_loghandler_lsn.h:
merge (make LSN_ERROR and LSN_REPAIRED_BY_MARIA_CHK different)
storage/maria/ma_open.c:
When opening a table verify that is_of_lsn >= create_rename_lsn; if
false the header must be corrupted.
_ma_state_info_write() is split in two: _ma_state_info_write_sub()
which is the old _ma_state_info_write(), and _ma_state_info_write()
which additionally takes intern_lock if requested (to protect
against Checkpoint) and updates is_of_lsn.
_ma_open_keyfile() should change kfile.file under intern_lock
to protect Checkpoint from reading a wrong kfile.file.
storage/maria/ma_recovery.c:
Recovery of state.records: when the REDO phase sees UNDO_ROW_INSERT
which has a LSN > state.is_of_lsn it increments state.records.
Same for UNDO_ROW_DELETE and UNDO_ROW_PURGE.
When closing a table during Recovery, we know its state is at least
as new as the current log record we are looking at, so increase
is_of_lsn to the LSN of the current log record.
storage/maria/ma_rename.c:
update for new behaviour of _ma_update_create_rename_lsn_on_disk().
storage/maria/ma_test1.c:
update to new prototype
storage/maria/ma_test2.c:
update to new prototype (actually prototype was changed days ago,
but compiler does not complain about the extra argument??)
storage/maria/ma_test_recovery.expected:
new result file of ma_test_recovery. Improvements: record
count read from index's header is now always correct.
storage/maria/ma_test_recovery:
"rm" fails if file does not exist. Redirect stderr of script.
storage/maria/ma_write.c:
if table is transactional, "records" is already incremented when
logging UNDO_ROW_INSERT. Comments.
storage/maria/maria_chk.c:
update is_of_lsn too
storage/maria/maria_def.h:
- MARIA_STATE_INFO::is_of_lsn which is used by Recovery. It is stored
into the index file's header.
- Checkpoint can now mark a table as "don't free this", and maria_close()
can reply "ok then you will free it".
- new functions
storage/maria/maria_pack.c:
update for new name
2007-09-07 15:02:30 +02:00
|
|
|
Then calls _ma_state_info_write_sub().
|
|
|
|
|
|
|
|
@param share table
|
|
|
|
@param pWrite bitmap: if 1 is set my_pwrite() is used otherwise
|
|
|
|
my_write(); if 2 is set, info about keys is written
|
|
|
|
(should only be needed after ALTER TABLE
|
|
|
|
ENABLE/DISABLE KEYS, and REPAIR/OPTIMIZE); if 4 is
|
|
|
|
set, MARIA_SHARE::intern_lock is taken.
|
|
|
|
|
|
|
|
@return Operation status
|
|
|
|
@retval 0 OK
|
|
|
|
@retval 1 Error
|
|
|
|
*/
|
|
|
|
|
|
|
|
uint _ma_state_info_write(MARIA_SHARE *share, uint pWrite)
|
|
|
|
{
|
WL#3071 Maria checkpoint
Finally this is the real checkpoint code.
It however exhibits unstabilities when a checkpoint runs concurrently
with data-modifying clients (table corruption, transaction log's
assertions) so for now a checkpoint is taken only at startup after
recovery and at shutdown, i.e. not in concurrent situations. Later
we will let it run periodically, as well as flush dirty pages
periodically (almost all needed code is there already, only pagecache
code is written but not committed).
WL#3072 Maria recovery
* replacing UNDO_ROW_PURGE with CLR_END; testing of those CLR_END via
ma_test2 which has INSERTs failing with duplicate keys.
* replaying of REDO_RENAME_TABLE
Now, off to test Recovery in ha_maria :)
BitKeeper/deleted/.del-ma_least_recently_dirtied.c:
Delete: storage/maria/ma_least_recently_dirtied.c
BitKeeper/deleted/.del-ma_least_recently_dirtied.h:
Delete: storage/maria/ma_least_recently_dirtied.h
storage/maria/Makefile.am:
compile Checkpoint module
storage/maria/ha_maria.cc:
When ha_maria starts, do a recovery from last checkpoint.
Take a checkpoint when that recovery has ended and when ha_maria
shuts down cleanly.
storage/maria/ma_blockrec.c:
* even if my_sync() fails we have to my_close() (otherwise we leak
a descriptor)
* UNDO_ROW_PURGE is replaced by a simple CLR_END for UNDO_ROW_INSERT,
as promised in the old comment; it gives us skipping during the
UNDO phase.
storage/maria/ma_check.c:
All REDOs before create_rename_lsn are ignored by Recovery. So
create_rename_lsn must be set only after all data/index has been
flushed and forced to disk. We thus move write_log_record_for_repair()
to after _ma_flush_tables_files_after_repair().
storage/maria/ma_checkpoint.c:
Checkpoint module.
storage/maria/ma_checkpoint.h:
optional argument if caller wants a thread to periodically take
checkpoints and flush dirty pages.
storage/maria/ma_create.c:
* no need to init some vars as the initial bzero(share) takes care of this.
* update to new function's name
* even if we fail in my_sync() we have to my_close()
storage/maria/ma_extra.c:
Checkpoint reads share->last_version under intern_lock, so we make
maria_extra() update it under intern_lock. THR_LOCK_maria still needed
because of _ma_test_if_reopen().
storage/maria/ma_init.c:
destroy checkpoint module when Maria shuts down.
storage/maria/ma_loghandler.c:
* UNDO_ROW_PURGE gone (see ma_blockrec.c)
* we need to remember the LSN of the LOGREC_FILE_ID for a share,
because this LSN is needed into the checkpoint record (Recovery wants
to know the validity domain of an id->name mapping)
* translog_get_horizon_no_lock() needed for Checkpoint
* comment about failing assertion (Sanja knows)
* translog_init_reader_data() thought that translog_read_record_header_scan()
returns 0 in case of error, but 0 just means "0-length header".
* translog_assign_id_to_share() now needs the MARIA_HA because
LOGREC_FILE_ID uses a log-write hook.
* Verify that (de)assignment of share->id happens only under intern_lock,
as Checkpoint reads this id with intern_lock.
* translog_purge() can accept TRANSLOG_ADDRESS, not necessarily
a real LSN.
storage/maria/ma_loghandler.h:
prototype updates
storage/maria/ma_open.c:
no need to initialize "res"
storage/maria/ma_pagecache.c:
When taking a checkpoint, we don't need to know the maximum rec_lsn
of dirty pages; this LSN was intended to be used in the two-checkpoint
rule, but last_checkpoint_lsn is as good.
4 bytes for stored_list_size is enough as PAGECACHE::blocks (number
of blocks which the pagecache can contain) is int.
storage/maria/ma_pagecache.h:
new prototype
storage/maria/ma_recovery.c:
* added replaying of REDO_RENAME_TABLE
* UNDO_ROW_PURGE gone (see ma_blockrec.c), replaced by CLR_END
* Recovery from the last checkpoint record now possible
* In new_table() we skip the table if the id->name mapping is older than
create_rename_lsn (mapping dates from lsn_of_file_id).
* in get_MARIA_HA_from_REDO_record() we skip the record
if the id->name mapping is newer than the record (can happen if processing
a record which is before the checkpoint record).
* parse_checkpoint_record() has to return a LSN, that's what caller expects
storage/maria/ma_rename.c:
new function's name; log end zeroes of tables' names (ease recovery)
storage/maria/ma_test2.c:
* equivalent of ma_test1's --test-undo added (named -u here).
* -t=1 now stops right after creating the table, so that
we can test undoing of INSERTs with duplicate keys (which tests the
CLR_END logged by _ma_write_abort_block_record()).
storage/maria/ma_test_recovery.expected:
Result of testing undoing of INSERTs with duplicate keys; there are
some differences in maria_chk -dvv but they are normal (removing
records does not shrink data/index file, does not put back the
"analyzed, optimized keys"(etc) index state.
storage/maria/ma_test_recovery:
Test undoing of INSERTs with duplicate keys, using ma_test2;
when such INSERT happens, it logs REDO_INSERT, UNDO_INSERT, REDO_DELETE,
CLR_END; we abort after that, and test that CLR_END causes recovery
to jump over UNDO_INSERT.
storage/maria/ma_write.c:
comment
storage/maria/maria_chk.c:
comment
storage/maria/maria_def.h:
* a new bit in MARIA_SHARE::in_checkpoint, used to build a list
of unique shares during Checkpoint.
* MARIA_SHARE::lsn_of_file_id added: the LSN of the last LOGREC_FILE_ID
for this share; needed to know to which LSN domain the mappings
found in the Checkpoint record apply (new mappings should not apply
to old REDOs).
storage/maria/trnman.c:
* small changes to how trnman_collect_transactions() fills its buffer;
it also uses a non-dummy lsn_read_non_atomic() found in ma_checkpoint.h
2007-09-12 11:27:34 +02:00
|
|
|
uint res;
|
- WL#3072 Maria Recovery:
Recovery of state.records (the count of records which is stored into
the header of the index file). For that, state.is_of_lsn is introduced;
logic is explained in ma_recovery.c (look for "Recovery of the state").
The net gain is that in case of crash, we now recover state.records,
and it is idempotent (ma_test_recovery tests it).
state.checksum is not recovered yet, mail sent for discussion.
- WL#3071 Maria Checkpoint: preparation for it, by protecting
all modifications of the state in memory or on disk with intern_lock
(with the exception of the really-often-modified state.records,
which is now protected with the log's lock, see ma_recovery.c
(look for "Recovery of the state"). Also, if maria_close() sees that
Checkpoint is looking at this table it will not my_free() the share.
- don't compute row's checksum twice in case of UPDATE (correction
to a bugfix I made yesterday).
storage/maria/ha_maria.cc:
protect state write with intern_lock (against Checkpoint)
storage/maria/ma_blockrec.c:
* don't reset trn->rec_lsn in _ma_unpin_all_pages(), because it
should wait until we have corrected the allocation in the bitmap
(as the REDO can serve to correct the allocation during Recovery);
introducing _ma_finalize_row() for that.
* In a changeset yesterday I moved computation of the checksum
into write_block_record(), to fix a bug in UPDATE. Now I notice
that maria_update() already computes the checksum, it's just that
it puts it into info->cur_row while _ma_update_block_record()
uses info->new_row; so, removing the checksum computation from
write_block_record(), putting it back into allocate_and_write_block_record()
(which is called only by INSERT and UNDO_DELETE), and copying
cur_row->checksum into new_row->checksum in _ma_update_block_record().
storage/maria/ma_check.c:
new prototypes, they will take intern_lock when writing the state;
also take intern_lock when changing share->kfile. In both cases
this is to protect against Checkpoint reading/writing the state or reading
kfile at the same time.
Not updating create_rename_lsn directly at end of write_log_record_for_repair()
as it wouldn't have intern_lock.
storage/maria/ma_close.c:
Checkpoint builds a list of shares (under THR_LOCK_maria), then it
handles each such share (under intern_lock) (doing flushing etc);
if maria_close() freed this share between the two, Checkpoint
would see a bad pointer. To avoid this, when building the list Checkpoint
marks each share, so that maria_close() knows it should not free it
and Checkpoint will free it itself.
Extending the zone covered by intern_lock to protect against
Checkpoint reading kfile, writing state.
storage/maria/ma_create.c:
When we update create_rename_lsn, we also update is_of_lsn to
the same value: it is logical, and allows us to test in maria_open()
that the former is not bigger than the latter (the contrary is a sign
of index header corruption, or severe logging bug which hinders
Recovery, table needs a repair).
_ma_update_create_rename_lsn_on_disk() also writes is_of_lsn;
it now operates under intern_lock (protect against Checkpoint),
a shortcut function is available for cases where acquiring
intern_lock is not needed (table's creation or first open).
storage/maria/ma_delete.c:
if table is transactional, "records" is already decremented
when logging UNDO_ROW_DELETE.
storage/maria/ma_delete_all.c:
comments
storage/maria/ma_extra.c:
Protect modifications of the state, in memory and/or on disk,
with intern_lock, against a concurrent Checkpoint.
When state goes to disk, update it's is_of_lsn (by calling
the new _ma_state_info_write()).
In HA_EXTRA_FORCE_REOPEN, don't set share->changed to 0 (undoing
a change I made a few days ago) and ASK_MONTY
storage/maria/ma_locking.c:
no real code change here.
storage/maria/ma_loghandler.c:
Log-write-hooks for updating "state.records" under log's mutex
when writing/updating/deleting a row or deleting all rows.
storage/maria/ma_loghandler_lsn.h:
merge (make LSN_ERROR and LSN_REPAIRED_BY_MARIA_CHK different)
storage/maria/ma_open.c:
When opening a table verify that is_of_lsn >= create_rename_lsn; if
false the header must be corrupted.
_ma_state_info_write() is split in two: _ma_state_info_write_sub()
which is the old _ma_state_info_write(), and _ma_state_info_write()
which additionally takes intern_lock if requested (to protect
against Checkpoint) and updates is_of_lsn.
_ma_open_keyfile() should change kfile.file under intern_lock
to protect Checkpoint from reading a wrong kfile.file.
storage/maria/ma_recovery.c:
Recovery of state.records: when the REDO phase sees UNDO_ROW_INSERT
which has a LSN > state.is_of_lsn it increments state.records.
Same for UNDO_ROW_DELETE and UNDO_ROW_PURGE.
When closing a table during Recovery, we know its state is at least
as new as the current log record we are looking at, so increase
is_of_lsn to the LSN of the current log record.
storage/maria/ma_rename.c:
update for new behaviour of _ma_update_create_rename_lsn_on_disk().
storage/maria/ma_test1.c:
update to new prototype
storage/maria/ma_test2.c:
update to new prototype (actually prototype was changed days ago,
but compiler does not complain about the extra argument??)
storage/maria/ma_test_recovery.expected:
new result file of ma_test_recovery. Improvements: record
count read from index's header is now always correct.
storage/maria/ma_test_recovery:
"rm" fails if file does not exist. Redirect stderr of script.
storage/maria/ma_write.c:
if table is transactional, "records" is already incremented when
logging UNDO_ROW_INSERT. Comments.
storage/maria/maria_chk.c:
update is_of_lsn too
storage/maria/maria_def.h:
- MARIA_STATE_INFO::is_of_lsn which is used by Recovery. It is stored
into the index file's header.
- Checkpoint can now mark a table as "don't free this", and maria_close()
can reply "ok then you will free it".
- new functions
storage/maria/maria_pack.c:
update for new name
2007-09-07 15:02:30 +02:00
|
|
|
if (pWrite & 4)
|
|
|
|
pthread_mutex_lock(&share->intern_lock);
|
|
|
|
else if (maria_multi_threaded)
|
2007-09-27 14:18:28 +03:00
|
|
|
{
|
- WL#3072 Maria Recovery:
Recovery of state.records (the count of records which is stored into
the header of the index file). For that, state.is_of_lsn is introduced;
logic is explained in ma_recovery.c (look for "Recovery of the state").
The net gain is that in case of crash, we now recover state.records,
and it is idempotent (ma_test_recovery tests it).
state.checksum is not recovered yet, mail sent for discussion.
- WL#3071 Maria Checkpoint: preparation for it, by protecting
all modifications of the state in memory or on disk with intern_lock
(with the exception of the really-often-modified state.records,
which is now protected with the log's lock, see ma_recovery.c
(look for "Recovery of the state"). Also, if maria_close() sees that
Checkpoint is looking at this table it will not my_free() the share.
- don't compute row's checksum twice in case of UPDATE (correction
to a bugfix I made yesterday).
storage/maria/ha_maria.cc:
protect state write with intern_lock (against Checkpoint)
storage/maria/ma_blockrec.c:
* don't reset trn->rec_lsn in _ma_unpin_all_pages(), because it
should wait until we have corrected the allocation in the bitmap
(as the REDO can serve to correct the allocation during Recovery);
introducing _ma_finalize_row() for that.
* In a changeset yesterday I moved computation of the checksum
into write_block_record(), to fix a bug in UPDATE. Now I notice
that maria_update() already computes the checksum, it's just that
it puts it into info->cur_row while _ma_update_block_record()
uses info->new_row; so, removing the checksum computation from
write_block_record(), putting it back into allocate_and_write_block_record()
(which is called only by INSERT and UNDO_DELETE), and copying
cur_row->checksum into new_row->checksum in _ma_update_block_record().
storage/maria/ma_check.c:
new prototypes, they will take intern_lock when writing the state;
also take intern_lock when changing share->kfile. In both cases
this is to protect against Checkpoint reading/writing the state or reading
kfile at the same time.
Not updating create_rename_lsn directly at end of write_log_record_for_repair()
as it wouldn't have intern_lock.
storage/maria/ma_close.c:
Checkpoint builds a list of shares (under THR_LOCK_maria), then it
handles each such share (under intern_lock) (doing flushing etc);
if maria_close() freed this share between the two, Checkpoint
would see a bad pointer. To avoid this, when building the list Checkpoint
marks each share, so that maria_close() knows it should not free it
and Checkpoint will free it itself.
Extending the zone covered by intern_lock to protect against
Checkpoint reading kfile, writing state.
storage/maria/ma_create.c:
When we update create_rename_lsn, we also update is_of_lsn to
the same value: it is logical, and allows us to test in maria_open()
that the former is not bigger than the latter (the contrary is a sign
of index header corruption, or severe logging bug which hinders
Recovery, table needs a repair).
_ma_update_create_rename_lsn_on_disk() also writes is_of_lsn;
it now operates under intern_lock (protect against Checkpoint),
a shortcut function is available for cases where acquiring
intern_lock is not needed (table's creation or first open).
storage/maria/ma_delete.c:
if table is transactional, "records" is already decremented
when logging UNDO_ROW_DELETE.
storage/maria/ma_delete_all.c:
comments
storage/maria/ma_extra.c:
Protect modifications of the state, in memory and/or on disk,
with intern_lock, against a concurrent Checkpoint.
When state goes to disk, update it's is_of_lsn (by calling
the new _ma_state_info_write()).
In HA_EXTRA_FORCE_REOPEN, don't set share->changed to 0 (undoing
a change I made a few days ago) and ASK_MONTY
storage/maria/ma_locking.c:
no real code change here.
storage/maria/ma_loghandler.c:
Log-write-hooks for updating "state.records" under log's mutex
when writing/updating/deleting a row or deleting all rows.
storage/maria/ma_loghandler_lsn.h:
merge (make LSN_ERROR and LSN_REPAIRED_BY_MARIA_CHK different)
storage/maria/ma_open.c:
When opening a table verify that is_of_lsn >= create_rename_lsn; if
false the header must be corrupted.
_ma_state_info_write() is split in two: _ma_state_info_write_sub()
which is the old _ma_state_info_write(), and _ma_state_info_write()
which additionally takes intern_lock if requested (to protect
against Checkpoint) and updates is_of_lsn.
_ma_open_keyfile() should change kfile.file under intern_lock
to protect Checkpoint from reading a wrong kfile.file.
storage/maria/ma_recovery.c:
Recovery of state.records: when the REDO phase sees UNDO_ROW_INSERT
which has a LSN > state.is_of_lsn it increments state.records.
Same for UNDO_ROW_DELETE and UNDO_ROW_PURGE.
When closing a table during Recovery, we know its state is at least
as new as the current log record we are looking at, so increase
is_of_lsn to the LSN of the current log record.
storage/maria/ma_rename.c:
update for new behaviour of _ma_update_create_rename_lsn_on_disk().
storage/maria/ma_test1.c:
update to new prototype
storage/maria/ma_test2.c:
update to new prototype (actually prototype was changed days ago,
but compiler does not complain about the extra argument??)
storage/maria/ma_test_recovery.expected:
new result file of ma_test_recovery. Improvements: record
count read from index's header is now always correct.
storage/maria/ma_test_recovery:
"rm" fails if file does not exist. Redirect stderr of script.
storage/maria/ma_write.c:
if table is transactional, "records" is already incremented when
logging UNDO_ROW_INSERT. Comments.
storage/maria/maria_chk.c:
update is_of_lsn too
storage/maria/maria_def.h:
- MARIA_STATE_INFO::is_of_lsn which is used by Recovery. It is stored
into the index file's header.
- Checkpoint can now mark a table as "don't free this", and maria_close()
can reply "ok then you will free it".
- new functions
storage/maria/maria_pack.c:
update for new name
2007-09-07 15:02:30 +02:00
|
|
|
safe_mutex_assert_owner(&share->intern_lock);
|
2007-09-27 14:18:28 +03:00
|
|
|
}
|
- WL#3072 Maria Recovery:
Recovery of state.records (the count of records which is stored into
the header of the index file). For that, state.is_of_lsn is introduced;
logic is explained in ma_recovery.c (look for "Recovery of the state").
The net gain is that in case of crash, we now recover state.records,
and it is idempotent (ma_test_recovery tests it).
state.checksum is not recovered yet, mail sent for discussion.
- WL#3071 Maria Checkpoint: preparation for it, by protecting
all modifications of the state in memory or on disk with intern_lock
(with the exception of the really-often-modified state.records,
which is now protected with the log's lock, see ma_recovery.c
(look for "Recovery of the state"). Also, if maria_close() sees that
Checkpoint is looking at this table it will not my_free() the share.
- don't compute row's checksum twice in case of UPDATE (correction
to a bugfix I made yesterday).
storage/maria/ha_maria.cc:
protect state write with intern_lock (against Checkpoint)
storage/maria/ma_blockrec.c:
* don't reset trn->rec_lsn in _ma_unpin_all_pages(), because it
should wait until we have corrected the allocation in the bitmap
(as the REDO can serve to correct the allocation during Recovery);
introducing _ma_finalize_row() for that.
* In a changeset yesterday I moved computation of the checksum
into write_block_record(), to fix a bug in UPDATE. Now I notice
that maria_update() already computes the checksum, it's just that
it puts it into info->cur_row while _ma_update_block_record()
uses info->new_row; so, removing the checksum computation from
write_block_record(), putting it back into allocate_and_write_block_record()
(which is called only by INSERT and UNDO_DELETE), and copying
cur_row->checksum into new_row->checksum in _ma_update_block_record().
storage/maria/ma_check.c:
new prototypes, they will take intern_lock when writing the state;
also take intern_lock when changing share->kfile. In both cases
this is to protect against Checkpoint reading/writing the state or reading
kfile at the same time.
Not updating create_rename_lsn directly at end of write_log_record_for_repair()
as it wouldn't have intern_lock.
storage/maria/ma_close.c:
Checkpoint builds a list of shares (under THR_LOCK_maria), then it
handles each such share (under intern_lock) (doing flushing etc);
if maria_close() freed this share between the two, Checkpoint
would see a bad pointer. To avoid this, when building the list Checkpoint
marks each share, so that maria_close() knows it should not free it
and Checkpoint will free it itself.
Extending the zone covered by intern_lock to protect against
Checkpoint reading kfile, writing state.
storage/maria/ma_create.c:
When we update create_rename_lsn, we also update is_of_lsn to
the same value: it is logical, and allows us to test in maria_open()
that the former is not bigger than the latter (the contrary is a sign
of index header corruption, or severe logging bug which hinders
Recovery, table needs a repair).
_ma_update_create_rename_lsn_on_disk() also writes is_of_lsn;
it now operates under intern_lock (protect against Checkpoint),
a shortcut function is available for cases where acquiring
intern_lock is not needed (table's creation or first open).
storage/maria/ma_delete.c:
if table is transactional, "records" is already decremented
when logging UNDO_ROW_DELETE.
storage/maria/ma_delete_all.c:
comments
storage/maria/ma_extra.c:
Protect modifications of the state, in memory and/or on disk,
with intern_lock, against a concurrent Checkpoint.
When state goes to disk, update it's is_of_lsn (by calling
the new _ma_state_info_write()).
In HA_EXTRA_FORCE_REOPEN, don't set share->changed to 0 (undoing
a change I made a few days ago) and ASK_MONTY
storage/maria/ma_locking.c:
no real code change here.
storage/maria/ma_loghandler.c:
Log-write-hooks for updating "state.records" under log's mutex
when writing/updating/deleting a row or deleting all rows.
storage/maria/ma_loghandler_lsn.h:
merge (make LSN_ERROR and LSN_REPAIRED_BY_MARIA_CHK different)
storage/maria/ma_open.c:
When opening a table verify that is_of_lsn >= create_rename_lsn; if
false the header must be corrupted.
_ma_state_info_write() is split in two: _ma_state_info_write_sub()
which is the old _ma_state_info_write(), and _ma_state_info_write()
which additionally takes intern_lock if requested (to protect
against Checkpoint) and updates is_of_lsn.
_ma_open_keyfile() should change kfile.file under intern_lock
to protect Checkpoint from reading a wrong kfile.file.
storage/maria/ma_recovery.c:
Recovery of state.records: when the REDO phase sees UNDO_ROW_INSERT
which has a LSN > state.is_of_lsn it increments state.records.
Same for UNDO_ROW_DELETE and UNDO_ROW_PURGE.
When closing a table during Recovery, we know its state is at least
as new as the current log record we are looking at, so increase
is_of_lsn to the LSN of the current log record.
storage/maria/ma_rename.c:
update for new behaviour of _ma_update_create_rename_lsn_on_disk().
storage/maria/ma_test1.c:
update to new prototype
storage/maria/ma_test2.c:
update to new prototype (actually prototype was changed days ago,
but compiler does not complain about the extra argument??)
storage/maria/ma_test_recovery.expected:
new result file of ma_test_recovery. Improvements: record
count read from index's header is now always correct.
storage/maria/ma_test_recovery:
"rm" fails if file does not exist. Redirect stderr of script.
storage/maria/ma_write.c:
if table is transactional, "records" is already incremented when
logging UNDO_ROW_INSERT. Comments.
storage/maria/maria_chk.c:
update is_of_lsn too
storage/maria/maria_def.h:
- MARIA_STATE_INFO::is_of_lsn which is used by Recovery. It is stored
into the index file's header.
- Checkpoint can now mark a table as "don't free this", and maria_close()
can reply "ok then you will free it".
- new functions
storage/maria/maria_pack.c:
update for new name
2007-09-07 15:02:30 +02:00
|
|
|
if (share->base.born_transactional && translog_inited &&
|
|
|
|
!maria_in_recovery)
|
|
|
|
{
|
|
|
|
/*
|
WL#3071 Maria checkpoint
Finally this is the real checkpoint code.
It however exhibits unstabilities when a checkpoint runs concurrently
with data-modifying clients (table corruption, transaction log's
assertions) so for now a checkpoint is taken only at startup after
recovery and at shutdown, i.e. not in concurrent situations. Later
we will let it run periodically, as well as flush dirty pages
periodically (almost all needed code is there already, only pagecache
code is written but not committed).
WL#3072 Maria recovery
* replacing UNDO_ROW_PURGE with CLR_END; testing of those CLR_END via
ma_test2 which has INSERTs failing with duplicate keys.
* replaying of REDO_RENAME_TABLE
Now, off to test Recovery in ha_maria :)
BitKeeper/deleted/.del-ma_least_recently_dirtied.c:
Delete: storage/maria/ma_least_recently_dirtied.c
BitKeeper/deleted/.del-ma_least_recently_dirtied.h:
Delete: storage/maria/ma_least_recently_dirtied.h
storage/maria/Makefile.am:
compile Checkpoint module
storage/maria/ha_maria.cc:
When ha_maria starts, do a recovery from last checkpoint.
Take a checkpoint when that recovery has ended and when ha_maria
shuts down cleanly.
storage/maria/ma_blockrec.c:
* even if my_sync() fails we have to my_close() (otherwise we leak
a descriptor)
* UNDO_ROW_PURGE is replaced by a simple CLR_END for UNDO_ROW_INSERT,
as promised in the old comment; it gives us skipping during the
UNDO phase.
storage/maria/ma_check.c:
All REDOs before create_rename_lsn are ignored by Recovery. So
create_rename_lsn must be set only after all data/index has been
flushed and forced to disk. We thus move write_log_record_for_repair()
to after _ma_flush_tables_files_after_repair().
storage/maria/ma_checkpoint.c:
Checkpoint module.
storage/maria/ma_checkpoint.h:
optional argument if caller wants a thread to periodically take
checkpoints and flush dirty pages.
storage/maria/ma_create.c:
* no need to init some vars as the initial bzero(share) takes care of this.
* update to new function's name
* even if we fail in my_sync() we have to my_close()
storage/maria/ma_extra.c:
Checkpoint reads share->last_version under intern_lock, so we make
maria_extra() update it under intern_lock. THR_LOCK_maria still needed
because of _ma_test_if_reopen().
storage/maria/ma_init.c:
destroy checkpoint module when Maria shuts down.
storage/maria/ma_loghandler.c:
* UNDO_ROW_PURGE gone (see ma_blockrec.c)
* we need to remember the LSN of the LOGREC_FILE_ID for a share,
because this LSN is needed into the checkpoint record (Recovery wants
to know the validity domain of an id->name mapping)
* translog_get_horizon_no_lock() needed for Checkpoint
* comment about failing assertion (Sanja knows)
* translog_init_reader_data() thought that translog_read_record_header_scan()
returns 0 in case of error, but 0 just means "0-length header".
* translog_assign_id_to_share() now needs the MARIA_HA because
LOGREC_FILE_ID uses a log-write hook.
* Verify that (de)assignment of share->id happens only under intern_lock,
as Checkpoint reads this id with intern_lock.
* translog_purge() can accept TRANSLOG_ADDRESS, not necessarily
a real LSN.
storage/maria/ma_loghandler.h:
prototype updates
storage/maria/ma_open.c:
no need to initialize "res"
storage/maria/ma_pagecache.c:
When taking a checkpoint, we don't need to know the maximum rec_lsn
of dirty pages; this LSN was intended to be used in the two-checkpoint
rule, but last_checkpoint_lsn is as good.
4 bytes for stored_list_size is enough as PAGECACHE::blocks (number
of blocks which the pagecache can contain) is int.
storage/maria/ma_pagecache.h:
new prototype
storage/maria/ma_recovery.c:
* added replaying of REDO_RENAME_TABLE
* UNDO_ROW_PURGE gone (see ma_blockrec.c), replaced by CLR_END
* Recovery from the last checkpoint record now possible
* In new_table() we skip the table if the id->name mapping is older than
create_rename_lsn (mapping dates from lsn_of_file_id).
* in get_MARIA_HA_from_REDO_record() we skip the record
if the id->name mapping is newer than the record (can happen if processing
a record which is before the checkpoint record).
* parse_checkpoint_record() has to return a LSN, that's what caller expects
storage/maria/ma_rename.c:
new function's name; log end zeroes of tables' names (ease recovery)
storage/maria/ma_test2.c:
* equivalent of ma_test1's --test-undo added (named -u here).
* -t=1 now stops right after creating the table, so that
we can test undoing of INSERTs with duplicate keys (which tests the
CLR_END logged by _ma_write_abort_block_record()).
storage/maria/ma_test_recovery.expected:
Result of testing undoing of INSERTs with duplicate keys; there are
some differences in maria_chk -dvv but they are normal (removing
records does not shrink data/index file, does not put back the
"analyzed, optimized keys"(etc) index state.
storage/maria/ma_test_recovery:
Test undoing of INSERTs with duplicate keys, using ma_test2;
when such INSERT happens, it logs REDO_INSERT, UNDO_INSERT, REDO_DELETE,
CLR_END; we abort after that, and test that CLR_END causes recovery
to jump over UNDO_INSERT.
storage/maria/ma_write.c:
comment
storage/maria/maria_chk.c:
comment
storage/maria/maria_def.h:
* a new bit in MARIA_SHARE::in_checkpoint, used to build a list
of unique shares during Checkpoint.
* MARIA_SHARE::lsn_of_file_id added: the LSN of the last LOGREC_FILE_ID
for this share; needed to know to which LSN domain the mappings
found in the Checkpoint record apply (new mappings should not apply
to old REDOs).
storage/maria/trnman.c:
* small changes to how trnman_collect_transactions() fills its buffer;
it also uses a non-dummy lsn_read_non_atomic() found in ma_checkpoint.h
2007-09-12 11:27:34 +02:00
|
|
|
In a recovery, we want to set is_of_horizon to the LSN of the last
|
- WL#3072 Maria Recovery:
Recovery of state.records (the count of records which is stored into
the header of the index file). For that, state.is_of_lsn is introduced;
logic is explained in ma_recovery.c (look for "Recovery of the state").
The net gain is that in case of crash, we now recover state.records,
and it is idempotent (ma_test_recovery tests it).
state.checksum is not recovered yet, mail sent for discussion.
- WL#3071 Maria Checkpoint: preparation for it, by protecting
all modifications of the state in memory or on disk with intern_lock
(with the exception of the really-often-modified state.records,
which is now protected with the log's lock, see ma_recovery.c
(look for "Recovery of the state"). Also, if maria_close() sees that
Checkpoint is looking at this table it will not my_free() the share.
- don't compute row's checksum twice in case of UPDATE (correction
to a bugfix I made yesterday).
storage/maria/ha_maria.cc:
protect state write with intern_lock (against Checkpoint)
storage/maria/ma_blockrec.c:
* don't reset trn->rec_lsn in _ma_unpin_all_pages(), because it
should wait until we have corrected the allocation in the bitmap
(as the REDO can serve to correct the allocation during Recovery);
introducing _ma_finalize_row() for that.
* In a changeset yesterday I moved computation of the checksum
into write_block_record(), to fix a bug in UPDATE. Now I notice
that maria_update() already computes the checksum, it's just that
it puts it into info->cur_row while _ma_update_block_record()
uses info->new_row; so, removing the checksum computation from
write_block_record(), putting it back into allocate_and_write_block_record()
(which is called only by INSERT and UNDO_DELETE), and copying
cur_row->checksum into new_row->checksum in _ma_update_block_record().
storage/maria/ma_check.c:
new prototypes, they will take intern_lock when writing the state;
also take intern_lock when changing share->kfile. In both cases
this is to protect against Checkpoint reading/writing the state or reading
kfile at the same time.
Not updating create_rename_lsn directly at end of write_log_record_for_repair()
as it wouldn't have intern_lock.
storage/maria/ma_close.c:
Checkpoint builds a list of shares (under THR_LOCK_maria), then it
handles each such share (under intern_lock) (doing flushing etc);
if maria_close() freed this share between the two, Checkpoint
would see a bad pointer. To avoid this, when building the list Checkpoint
marks each share, so that maria_close() knows it should not free it
and Checkpoint will free it itself.
Extending the zone covered by intern_lock to protect against
Checkpoint reading kfile, writing state.
storage/maria/ma_create.c:
When we update create_rename_lsn, we also update is_of_lsn to
the same value: it is logical, and allows us to test in maria_open()
that the former is not bigger than the latter (the contrary is a sign
of index header corruption, or severe logging bug which hinders
Recovery, table needs a repair).
_ma_update_create_rename_lsn_on_disk() also writes is_of_lsn;
it now operates under intern_lock (protect against Checkpoint),
a shortcut function is available for cases where acquiring
intern_lock is not needed (table's creation or first open).
storage/maria/ma_delete.c:
if table is transactional, "records" is already decremented
when logging UNDO_ROW_DELETE.
storage/maria/ma_delete_all.c:
comments
storage/maria/ma_extra.c:
Protect modifications of the state, in memory and/or on disk,
with intern_lock, against a concurrent Checkpoint.
When state goes to disk, update it's is_of_lsn (by calling
the new _ma_state_info_write()).
In HA_EXTRA_FORCE_REOPEN, don't set share->changed to 0 (undoing
a change I made a few days ago) and ASK_MONTY
storage/maria/ma_locking.c:
no real code change here.
storage/maria/ma_loghandler.c:
Log-write-hooks for updating "state.records" under log's mutex
when writing/updating/deleting a row or deleting all rows.
storage/maria/ma_loghandler_lsn.h:
merge (make LSN_ERROR and LSN_REPAIRED_BY_MARIA_CHK different)
storage/maria/ma_open.c:
When opening a table verify that is_of_lsn >= create_rename_lsn; if
false the header must be corrupted.
_ma_state_info_write() is split in two: _ma_state_info_write_sub()
which is the old _ma_state_info_write(), and _ma_state_info_write()
which additionally takes intern_lock if requested (to protect
against Checkpoint) and updates is_of_lsn.
_ma_open_keyfile() should change kfile.file under intern_lock
to protect Checkpoint from reading a wrong kfile.file.
storage/maria/ma_recovery.c:
Recovery of state.records: when the REDO phase sees UNDO_ROW_INSERT
which has a LSN > state.is_of_lsn it increments state.records.
Same for UNDO_ROW_DELETE and UNDO_ROW_PURGE.
When closing a table during Recovery, we know its state is at least
as new as the current log record we are looking at, so increase
is_of_lsn to the LSN of the current log record.
storage/maria/ma_rename.c:
update for new behaviour of _ma_update_create_rename_lsn_on_disk().
storage/maria/ma_test1.c:
update to new prototype
storage/maria/ma_test2.c:
update to new prototype (actually prototype was changed days ago,
but compiler does not complain about the extra argument??)
storage/maria/ma_test_recovery.expected:
new result file of ma_test_recovery. Improvements: record
count read from index's header is now always correct.
storage/maria/ma_test_recovery:
"rm" fails if file does not exist. Redirect stderr of script.
storage/maria/ma_write.c:
if table is transactional, "records" is already incremented when
logging UNDO_ROW_INSERT. Comments.
storage/maria/maria_chk.c:
update is_of_lsn too
storage/maria/maria_def.h:
- MARIA_STATE_INFO::is_of_lsn which is used by Recovery. It is stored
into the index file's header.
- Checkpoint can now mark a table as "don't free this", and maria_close()
can reply "ok then you will free it".
- new functions
storage/maria/maria_pack.c:
update for new name
2007-09-07 15:02:30 +02:00
|
|
|
record executed by Recovery, not the current EOF of the log (which
|
|
|
|
is too new). Recovery does it by itself.
|
|
|
|
*/
|
WL#3071 Maria checkpoint
Finally this is the real checkpoint code.
It however exhibits unstabilities when a checkpoint runs concurrently
with data-modifying clients (table corruption, transaction log's
assertions) so for now a checkpoint is taken only at startup after
recovery and at shutdown, i.e. not in concurrent situations. Later
we will let it run periodically, as well as flush dirty pages
periodically (almost all needed code is there already, only pagecache
code is written but not committed).
WL#3072 Maria recovery
* replacing UNDO_ROW_PURGE with CLR_END; testing of those CLR_END via
ma_test2 which has INSERTs failing with duplicate keys.
* replaying of REDO_RENAME_TABLE
Now, off to test Recovery in ha_maria :)
BitKeeper/deleted/.del-ma_least_recently_dirtied.c:
Delete: storage/maria/ma_least_recently_dirtied.c
BitKeeper/deleted/.del-ma_least_recently_dirtied.h:
Delete: storage/maria/ma_least_recently_dirtied.h
storage/maria/Makefile.am:
compile Checkpoint module
storage/maria/ha_maria.cc:
When ha_maria starts, do a recovery from last checkpoint.
Take a checkpoint when that recovery has ended and when ha_maria
shuts down cleanly.
storage/maria/ma_blockrec.c:
* even if my_sync() fails we have to my_close() (otherwise we leak
a descriptor)
* UNDO_ROW_PURGE is replaced by a simple CLR_END for UNDO_ROW_INSERT,
as promised in the old comment; it gives us skipping during the
UNDO phase.
storage/maria/ma_check.c:
All REDOs before create_rename_lsn are ignored by Recovery. So
create_rename_lsn must be set only after all data/index has been
flushed and forced to disk. We thus move write_log_record_for_repair()
to after _ma_flush_tables_files_after_repair().
storage/maria/ma_checkpoint.c:
Checkpoint module.
storage/maria/ma_checkpoint.h:
optional argument if caller wants a thread to periodically take
checkpoints and flush dirty pages.
storage/maria/ma_create.c:
* no need to init some vars as the initial bzero(share) takes care of this.
* update to new function's name
* even if we fail in my_sync() we have to my_close()
storage/maria/ma_extra.c:
Checkpoint reads share->last_version under intern_lock, so we make
maria_extra() update it under intern_lock. THR_LOCK_maria still needed
because of _ma_test_if_reopen().
storage/maria/ma_init.c:
destroy checkpoint module when Maria shuts down.
storage/maria/ma_loghandler.c:
* UNDO_ROW_PURGE gone (see ma_blockrec.c)
* we need to remember the LSN of the LOGREC_FILE_ID for a share,
because this LSN is needed into the checkpoint record (Recovery wants
to know the validity domain of an id->name mapping)
* translog_get_horizon_no_lock() needed for Checkpoint
* comment about failing assertion (Sanja knows)
* translog_init_reader_data() thought that translog_read_record_header_scan()
returns 0 in case of error, but 0 just means "0-length header".
* translog_assign_id_to_share() now needs the MARIA_HA because
LOGREC_FILE_ID uses a log-write hook.
* Verify that (de)assignment of share->id happens only under intern_lock,
as Checkpoint reads this id with intern_lock.
* translog_purge() can accept TRANSLOG_ADDRESS, not necessarily
a real LSN.
storage/maria/ma_loghandler.h:
prototype updates
storage/maria/ma_open.c:
no need to initialize "res"
storage/maria/ma_pagecache.c:
When taking a checkpoint, we don't need to know the maximum rec_lsn
of dirty pages; this LSN was intended to be used in the two-checkpoint
rule, but last_checkpoint_lsn is as good.
4 bytes for stored_list_size is enough as PAGECACHE::blocks (number
of blocks which the pagecache can contain) is int.
storage/maria/ma_pagecache.h:
new prototype
storage/maria/ma_recovery.c:
* added replaying of REDO_RENAME_TABLE
* UNDO_ROW_PURGE gone (see ma_blockrec.c), replaced by CLR_END
* Recovery from the last checkpoint record now possible
* In new_table() we skip the table if the id->name mapping is older than
create_rename_lsn (mapping dates from lsn_of_file_id).
* in get_MARIA_HA_from_REDO_record() we skip the record
if the id->name mapping is newer than the record (can happen if processing
a record which is before the checkpoint record).
* parse_checkpoint_record() has to return a LSN, that's what caller expects
storage/maria/ma_rename.c:
new function's name; log end zeroes of tables' names (ease recovery)
storage/maria/ma_test2.c:
* equivalent of ma_test1's --test-undo added (named -u here).
* -t=1 now stops right after creating the table, so that
we can test undoing of INSERTs with duplicate keys (which tests the
CLR_END logged by _ma_write_abort_block_record()).
storage/maria/ma_test_recovery.expected:
Result of testing undoing of INSERTs with duplicate keys; there are
some differences in maria_chk -dvv but they are normal (removing
records does not shrink data/index file, does not put back the
"analyzed, optimized keys"(etc) index state.
storage/maria/ma_test_recovery:
Test undoing of INSERTs with duplicate keys, using ma_test2;
when such INSERT happens, it logs REDO_INSERT, UNDO_INSERT, REDO_DELETE,
CLR_END; we abort after that, and test that CLR_END causes recovery
to jump over UNDO_INSERT.
storage/maria/ma_write.c:
comment
storage/maria/maria_chk.c:
comment
storage/maria/maria_def.h:
* a new bit in MARIA_SHARE::in_checkpoint, used to build a list
of unique shares during Checkpoint.
* MARIA_SHARE::lsn_of_file_id added: the LSN of the last LOGREC_FILE_ID
for this share; needed to know to which LSN domain the mappings
found in the Checkpoint record apply (new mappings should not apply
to old REDOs).
storage/maria/trnman.c:
* small changes to how trnman_collect_transactions() fills its buffer;
it also uses a non-dummy lsn_read_non_atomic() found in ma_checkpoint.h
2007-09-12 11:27:34 +02:00
|
|
|
share->state.is_of_horizon= translog_get_horizon();
|
- WL#3072 Maria Recovery:
Recovery of state.records (the count of records which is stored into
the header of the index file). For that, state.is_of_lsn is introduced;
logic is explained in ma_recovery.c (look for "Recovery of the state").
The net gain is that in case of crash, we now recover state.records,
and it is idempotent (ma_test_recovery tests it).
state.checksum is not recovered yet, mail sent for discussion.
- WL#3071 Maria Checkpoint: preparation for it, by protecting
all modifications of the state in memory or on disk with intern_lock
(with the exception of the really-often-modified state.records,
which is now protected with the log's lock, see ma_recovery.c
(look for "Recovery of the state"). Also, if maria_close() sees that
Checkpoint is looking at this table it will not my_free() the share.
- don't compute row's checksum twice in case of UPDATE (correction
to a bugfix I made yesterday).
storage/maria/ha_maria.cc:
protect state write with intern_lock (against Checkpoint)
storage/maria/ma_blockrec.c:
* don't reset trn->rec_lsn in _ma_unpin_all_pages(), because it
should wait until we have corrected the allocation in the bitmap
(as the REDO can serve to correct the allocation during Recovery);
introducing _ma_finalize_row() for that.
* In a changeset yesterday I moved computation of the checksum
into write_block_record(), to fix a bug in UPDATE. Now I notice
that maria_update() already computes the checksum, it's just that
it puts it into info->cur_row while _ma_update_block_record()
uses info->new_row; so, removing the checksum computation from
write_block_record(), putting it back into allocate_and_write_block_record()
(which is called only by INSERT and UNDO_DELETE), and copying
cur_row->checksum into new_row->checksum in _ma_update_block_record().
storage/maria/ma_check.c:
new prototypes, they will take intern_lock when writing the state;
also take intern_lock when changing share->kfile. In both cases
this is to protect against Checkpoint reading/writing the state or reading
kfile at the same time.
Not updating create_rename_lsn directly at end of write_log_record_for_repair()
as it wouldn't have intern_lock.
storage/maria/ma_close.c:
Checkpoint builds a list of shares (under THR_LOCK_maria), then it
handles each such share (under intern_lock) (doing flushing etc);
if maria_close() freed this share between the two, Checkpoint
would see a bad pointer. To avoid this, when building the list Checkpoint
marks each share, so that maria_close() knows it should not free it
and Checkpoint will free it itself.
Extending the zone covered by intern_lock to protect against
Checkpoint reading kfile, writing state.
storage/maria/ma_create.c:
When we update create_rename_lsn, we also update is_of_lsn to
the same value: it is logical, and allows us to test in maria_open()
that the former is not bigger than the latter (the contrary is a sign
of index header corruption, or severe logging bug which hinders
Recovery, table needs a repair).
_ma_update_create_rename_lsn_on_disk() also writes is_of_lsn;
it now operates under intern_lock (protect against Checkpoint),
a shortcut function is available for cases where acquiring
intern_lock is not needed (table's creation or first open).
storage/maria/ma_delete.c:
if table is transactional, "records" is already decremented
when logging UNDO_ROW_DELETE.
storage/maria/ma_delete_all.c:
comments
storage/maria/ma_extra.c:
Protect modifications of the state, in memory and/or on disk,
with intern_lock, against a concurrent Checkpoint.
When state goes to disk, update it's is_of_lsn (by calling
the new _ma_state_info_write()).
In HA_EXTRA_FORCE_REOPEN, don't set share->changed to 0 (undoing
a change I made a few days ago) and ASK_MONTY
storage/maria/ma_locking.c:
no real code change here.
storage/maria/ma_loghandler.c:
Log-write-hooks for updating "state.records" under log's mutex
when writing/updating/deleting a row or deleting all rows.
storage/maria/ma_loghandler_lsn.h:
merge (make LSN_ERROR and LSN_REPAIRED_BY_MARIA_CHK different)
storage/maria/ma_open.c:
When opening a table verify that is_of_lsn >= create_rename_lsn; if
false the header must be corrupted.
_ma_state_info_write() is split in two: _ma_state_info_write_sub()
which is the old _ma_state_info_write(), and _ma_state_info_write()
which additionally takes intern_lock if requested (to protect
against Checkpoint) and updates is_of_lsn.
_ma_open_keyfile() should change kfile.file under intern_lock
to protect Checkpoint from reading a wrong kfile.file.
storage/maria/ma_recovery.c:
Recovery of state.records: when the REDO phase sees UNDO_ROW_INSERT
which has a LSN > state.is_of_lsn it increments state.records.
Same for UNDO_ROW_DELETE and UNDO_ROW_PURGE.
When closing a table during Recovery, we know its state is at least
as new as the current log record we are looking at, so increase
is_of_lsn to the LSN of the current log record.
storage/maria/ma_rename.c:
update for new behaviour of _ma_update_create_rename_lsn_on_disk().
storage/maria/ma_test1.c:
update to new prototype
storage/maria/ma_test2.c:
update to new prototype (actually prototype was changed days ago,
but compiler does not complain about the extra argument??)
storage/maria/ma_test_recovery.expected:
new result file of ma_test_recovery. Improvements: record
count read from index's header is now always correct.
storage/maria/ma_test_recovery:
"rm" fails if file does not exist. Redirect stderr of script.
storage/maria/ma_write.c:
if table is transactional, "records" is already incremented when
logging UNDO_ROW_INSERT. Comments.
storage/maria/maria_chk.c:
update is_of_lsn too
storage/maria/maria_def.h:
- MARIA_STATE_INFO::is_of_lsn which is used by Recovery. It is stored
into the index file's header.
- Checkpoint can now mark a table as "don't free this", and maria_close()
can reply "ok then you will free it".
- new functions
storage/maria/maria_pack.c:
update for new name
2007-09-07 15:02:30 +02:00
|
|
|
}
|
|
|
|
res= _ma_state_info_write_sub(share->kfile.file, &share->state, pWrite);
|
|
|
|
if (pWrite & 4)
|
|
|
|
pthread_mutex_unlock(&share->intern_lock);
|
|
|
|
return res;
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
@brief Function to save and store the header in the index file (.MYI).
|
|
|
|
|
|
|
|
Shortcut to use instead of _ma_state_info_write() when appropriate.
|
|
|
|
|
- WL#3239 "log CREATE TABLE in Maria"
- WL#3240 "log DROP TABLE in Maria"
- similarly, log RENAME TABLE, REPAIR/OPTIMIZE TABLE, and
DELETE no_WHERE_clause (== the DELETE which just truncates the files)
- create_rename_lsn added to MARIA_SHARE's state
- all these operations (except DROP TABLE) also update the table's
create_rename_lsn, which is needed for the correctness of
Recovery (see function comment of _ma_repair_write_log_record()
in ma_check.c)
- write a COMMIT record when transaction commits.
- don't log REDOs/UNDOs if this is an internal temporary table
like inside ALTER TABLE (I expect this to be a big win). There was
already no logging for user-created "CREATE TEMPORARY" tables.
- don't fsync files/directories if the table is not transactional
- in translog_write_record(), autogenerate a 2-byte-id for the table
and log the "id->name" pair (LOGREC_FILE_ID); log
LOGREC_LONG_TRANSACTION_ID; automatically store
the table's 2-byte-id in any log record.
- preparations for Checkpoint: translog_get_horizon(); pausing Checkpoint
when some dirty pages are unknown; capturing trn->rec_lsn,
trn->first_undo_lsn for Checkpoint and log's low-water-mark computing.
- assertions, comments.
storage/maria/Makefile.am:
more files to build
storage/maria/ha_maria.cc:
- logging a REPAIR log record if REPAIR/OPTIMIZE was successful.
- ha_maria::data_file_type does not have to be set in every info()
call, just do it once in open().
- if caller said that transactionality can be disabled (like if
caller is ALTER TABLE) i.e. thd->transaction.on==FALSE, then we
temporarily disable transactionality of the table in external_lock();
that will ensure that no REDOs/UNDOs are logged for this possibly
massive write operation (they are not needed, as if any write fails,
the table will be dropped). We re-enable in external_lock(F_UNLCK),
which in ALTER TABLE happens before the tmp table replaces the original
one (which is good, as thus the final table will have a REDO RENAME
and a correct create_rename_lsn).
- when we commit we also have to write a log record, so
trnman_commit_trn() calls become ma_commit() calls
- at end of engine's initialization, we are potentially entering a
multi-threaded dangerous world (clients are going to be accepted)
and so some assertions of mutex-owning become enforceable, for that
we set maria_multi_threaded=TRUE (see ma_control_file.c)
storage/maria/ha_maria.h:
new member ha_maria::save_transactional (see also ha_maria.cc)
storage/maria/ma_blockrec.c:
- fixing comments according to discussion with Monty
- if a table is transactional but temporarily non-transactional
(like in ALTER TABLE), we need to give a sensible LSN to the pages
(and, if we give 0, pagecache asserts).
- translog_write_record() now takes care of storing the share's
2-byte-id in the log record
storage/maria/ma_blockrec.h:
fixing comment according to discussion with Monty
storage/maria/ma_check.c:
When REPAIR/OPTIMIZE modify the data/index file, if this is a
transactional table, they must sync it; if they remove files or rename
files, they must sync the directory, so that everything is durable.
This is just applying to REPAIR/OPTIMIZE the logic already implemented
in CREATE/DROP/RENAME a few months ago.
Adding a function to write a LOGREC_REPAIR_TABLE at end of
REPAIR/OPTIMIZE (called only by ha_maria, not by maria_chk), and
to update the table's create_rename_lsn.
storage/maria/ma_close.c:
fix for a future bug
storage/maria/ma_control_file.c:
ensuring that if Maria is running in multi-threaded mode, anybody
wanting to write to the control file and update
last_checkpoint_lsn/last_logno owns the log's lock.
storage/maria/ma_control_file.h:
see ma_control_file.c
storage/maria/ma_create.c:
when creating a table:
- sync it and its directory only if this is a transactional table
and there is a log (no point in syncing in maria_chk)
- decouple the two uses of linkname/linkname_ptr (for index file and
for data file) into more variables, as we need to know all links
until the moment we write the LOGREC_CREATE_TABLE.
- set share.data_file_type early so that _ma_initialize_data_file()
knows it (Monty's bugfix so that a table always has at least a bitmap
page when it is created; so data-file is not 0 bytes anymore).
- log a LOGREC_CREATE_TABLE; it contains the bytes which we have
just written to the index file's header. Update table's
create_rename_lsn.
- syncing of kfile had been bugified in a previous merge, correcting
- syncing of dfile is now needed as it's not empty anymore
- in _ma_initialize_data_file(), use share's block_size and not the
global one. This is a gratuitous change, both variables are equal,
just that I find it more future-proof to use share-bound variable
rather than global one.
storage/maria/ma_delete_all.c:
log a LOGREC_DELETE_ALL record when doing ma_delete_all_rows();
update create_rename_lsn then.
storage/maria/ma_delete_table.c:
- logging LOGREC_DROP_TABLE; knowing if this is needed, requires
knowing if the table is transactional, which requires opening the
table.
- we need to sync directories only if the table is transactional
storage/maria/ma_extra.c:
questions
storage/maria/ma_init.c:
when maria_end() is called, engine is not multithreaded
storage/maria/ma_loghandler.c:
- translog_inited has to be visible to ma_create() (see how it is used
in ma_create())
- checkpoint record will be a single record, not three
- no REDO for TRUNCATE (TRUNCATE calls ma_create() internally so will
log a REDO_CREATE)
- adding REDO for DELETE no_WHERE_clause (fast DELETE of all rows by
truncating the files), REPAIR.
- MY_WAIT_IF_FULL to wait&retry if a log write hits a full disk
- in translog_write_record(), if MARIA_SHARE does not yet have a
2-byte-id, generate one for it and log LOGREC_FILE_ID; automatically
store this short id into log records.
- in translog_write_record(), if transaction has not logged its
long trid, log LOGREC_LONG_TRANSACTION_ID.
- For Checkpoint, we need to know the current end-of-log: adding
translog_get_horizon().
- For Control File, adding an assertion that the thread owns the
log's lock (control file is protected by this lock)
storage/maria/ma_loghandler.h:
Changes in log records (see ma_loghandler.c).
new prototypes, new functions.
storage/maria/ma_loghandler_lsn.h:
adding a type LSN_WITH_FLAGS especially for TRN::first_undo_lsn,
where the most significant byte is used for flags.
storage/maria/ma_open.c:
storing the create_rename_lsn in the index file's header (in the
state, precisely) and retrieving it from there.
storage/maria/ma_pagecache.c:
- my set_if_bigger was wrong, correcting it
- if the first_in_switch list is not empty, it means that
changed_blocks misses some dirty pages, so Checkpoint cannot run and
needs to wait. A variable missing_blocks_in_changed_list is added to
tell that (should it be named missing_blocks_in_changed_blocks?)
- pagecache_collect_changed_blocks_with_lsn() now also tells the
minimum rec_lsn (needed for low-water mark computation).
storage/maria/ma_pagecache.h:
see ma_pagecache.c
storage/maria/ma_panic.c:
comment
storage/maria/ma_range.c:
comment
storage/maria/ma_rename.c:
- logging LOGREC_RENAME_TABLE; knowing if this is needed, requires
knowing if the table is transactional, which requires opening the
table.
- update create_rename_lsn
- we need to sync directories only if the table is transactional
storage/maria/ma_static.c:
comment
storage/maria/ma_test_all.sh:
- tip for Valgrind-ing ma_test_all
- do "export maria_path=somepath" before calling ma_test_all,
if you want to run ma_test_all out of storage/maria (useful
to have parallel runs, like one normal and one Valgrind, they
must not use the same tables so need to run in different directories)
storage/maria/maria_def.h:
- state now contains, in memory and on disk, the create_rename_lsn
- share now contains a 2-byte-id
storage/maria/trnman.c:
preparations for Checkpoint: capture trn->rec_lsn, trn->first_undo_lsn;
minimum first_undo_lsn needed to know log's low-water-mark
storage/maria/trnman.h:
using most significant byte of first_undo_lsn to hold miscellaneous
flags, for now TRANSACTION_LOGGED_LONG_ID.
dummy_transaction_object is already declared in ma_static.c.
storage/maria/trnman_public.h:
dummy_transaction_object was declared in all files including
trnman_public.h, while in fact it's a single object.
new prototype
storage/maria/unittest/ma_test_loghandler-t.c:
update for new prototype
storage/maria/unittest/ma_test_loghandler_multigroup-t.c:
update for new prototype
storage/maria/unittest/ma_test_loghandler_multithread-t.c:
update for new prototype
storage/maria/unittest/ma_test_loghandler_pagecache-t.c:
update for new prototype
storage/maria/ma_commit.c:
function which wraps:
- writing a LOGREC_COMMIT record (==commit on disk)
- calling trnman_commit_trn() (=commit in memory)
storage/maria/ma_commit.h:
new header file
.tree-is-private:
this file is now needed to keep our tree private (don't push it
to public trees). When 5.1 is merged into mysql-maria, we can abandon
our maria-specific post-commit trigger; .tree_is_private will take
care of keeping commit mails private. Don't push this file to public
trees.
2007-06-22 14:49:37 +02:00
|
|
|
@param file descriptor of the index file to write
|
|
|
|
@param state state information to write to the file
|
- WL#3072 Maria Recovery:
Recovery of state.records (the count of records which is stored into
the header of the index file). For that, state.is_of_lsn is introduced;
logic is explained in ma_recovery.c (look for "Recovery of the state").
The net gain is that in case of crash, we now recover state.records,
and it is idempotent (ma_test_recovery tests it).
state.checksum is not recovered yet, mail sent for discussion.
- WL#3071 Maria Checkpoint: preparation for it, by protecting
all modifications of the state in memory or on disk with intern_lock
(with the exception of the really-often-modified state.records,
which is now protected with the log's lock, see ma_recovery.c
(look for "Recovery of the state"). Also, if maria_close() sees that
Checkpoint is looking at this table it will not my_free() the share.
- don't compute row's checksum twice in case of UPDATE (correction
to a bugfix I made yesterday).
storage/maria/ha_maria.cc:
protect state write with intern_lock (against Checkpoint)
storage/maria/ma_blockrec.c:
* don't reset trn->rec_lsn in _ma_unpin_all_pages(), because it
should wait until we have corrected the allocation in the bitmap
(as the REDO can serve to correct the allocation during Recovery);
introducing _ma_finalize_row() for that.
* In a changeset yesterday I moved computation of the checksum
into write_block_record(), to fix a bug in UPDATE. Now I notice
that maria_update() already computes the checksum, it's just that
it puts it into info->cur_row while _ma_update_block_record()
uses info->new_row; so, removing the checksum computation from
write_block_record(), putting it back into allocate_and_write_block_record()
(which is called only by INSERT and UNDO_DELETE), and copying
cur_row->checksum into new_row->checksum in _ma_update_block_record().
storage/maria/ma_check.c:
new prototypes, they will take intern_lock when writing the state;
also take intern_lock when changing share->kfile. In both cases
this is to protect against Checkpoint reading/writing the state or reading
kfile at the same time.
Not updating create_rename_lsn directly at end of write_log_record_for_repair()
as it wouldn't have intern_lock.
storage/maria/ma_close.c:
Checkpoint builds a list of shares (under THR_LOCK_maria), then it
handles each such share (under intern_lock) (doing flushing etc);
if maria_close() freed this share between the two, Checkpoint
would see a bad pointer. To avoid this, when building the list Checkpoint
marks each share, so that maria_close() knows it should not free it
and Checkpoint will free it itself.
Extending the zone covered by intern_lock to protect against
Checkpoint reading kfile, writing state.
storage/maria/ma_create.c:
When we update create_rename_lsn, we also update is_of_lsn to
the same value: it is logical, and allows us to test in maria_open()
that the former is not bigger than the latter (the contrary is a sign
of index header corruption, or severe logging bug which hinders
Recovery, table needs a repair).
_ma_update_create_rename_lsn_on_disk() also writes is_of_lsn;
it now operates under intern_lock (protect against Checkpoint),
a shortcut function is available for cases where acquiring
intern_lock is not needed (table's creation or first open).
storage/maria/ma_delete.c:
if table is transactional, "records" is already decremented
when logging UNDO_ROW_DELETE.
storage/maria/ma_delete_all.c:
comments
storage/maria/ma_extra.c:
Protect modifications of the state, in memory and/or on disk,
with intern_lock, against a concurrent Checkpoint.
When state goes to disk, update it's is_of_lsn (by calling
the new _ma_state_info_write()).
In HA_EXTRA_FORCE_REOPEN, don't set share->changed to 0 (undoing
a change I made a few days ago) and ASK_MONTY
storage/maria/ma_locking.c:
no real code change here.
storage/maria/ma_loghandler.c:
Log-write-hooks for updating "state.records" under log's mutex
when writing/updating/deleting a row or deleting all rows.
storage/maria/ma_loghandler_lsn.h:
merge (make LSN_ERROR and LSN_REPAIRED_BY_MARIA_CHK different)
storage/maria/ma_open.c:
When opening a table verify that is_of_lsn >= create_rename_lsn; if
false the header must be corrupted.
_ma_state_info_write() is split in two: _ma_state_info_write_sub()
which is the old _ma_state_info_write(), and _ma_state_info_write()
which additionally takes intern_lock if requested (to protect
against Checkpoint) and updates is_of_lsn.
_ma_open_keyfile() should change kfile.file under intern_lock
to protect Checkpoint from reading a wrong kfile.file.
storage/maria/ma_recovery.c:
Recovery of state.records: when the REDO phase sees UNDO_ROW_INSERT
which has a LSN > state.is_of_lsn it increments state.records.
Same for UNDO_ROW_DELETE and UNDO_ROW_PURGE.
When closing a table during Recovery, we know its state is at least
as new as the current log record we are looking at, so increase
is_of_lsn to the LSN of the current log record.
storage/maria/ma_rename.c:
update for new behaviour of _ma_update_create_rename_lsn_on_disk().
storage/maria/ma_test1.c:
update to new prototype
storage/maria/ma_test2.c:
update to new prototype (actually prototype was changed days ago,
but compiler does not complain about the extra argument??)
storage/maria/ma_test_recovery.expected:
new result file of ma_test_recovery. Improvements: record
count read from index's header is now always correct.
storage/maria/ma_test_recovery:
"rm" fails if file does not exist. Redirect stderr of script.
storage/maria/ma_write.c:
if table is transactional, "records" is already incremented when
logging UNDO_ROW_INSERT. Comments.
storage/maria/maria_chk.c:
update is_of_lsn too
storage/maria/maria_def.h:
- MARIA_STATE_INFO::is_of_lsn which is used by Recovery. It is stored
into the index file's header.
- Checkpoint can now mark a table as "don't free this", and maria_close()
can reply "ok then you will free it".
- new functions
storage/maria/maria_pack.c:
update for new name
2007-09-07 15:02:30 +02:00
|
|
|
@param pWrite bitmap: if 1 is set my_pwrite() is used otherwise
|
|
|
|
my_write(); if 2 is set, info about keys is written
|
|
|
|
(should only be needed after ALTER TABLE
|
|
|
|
ENABLE/DISABLE KEYS, and REPAIR/OPTIMIZE).
|
- WL#3239 "log CREATE TABLE in Maria"
- WL#3240 "log DROP TABLE in Maria"
- similarly, log RENAME TABLE, REPAIR/OPTIMIZE TABLE, and
DELETE no_WHERE_clause (== the DELETE which just truncates the files)
- create_rename_lsn added to MARIA_SHARE's state
- all these operations (except DROP TABLE) also update the table's
create_rename_lsn, which is needed for the correctness of
Recovery (see function comment of _ma_repair_write_log_record()
in ma_check.c)
- write a COMMIT record when transaction commits.
- don't log REDOs/UNDOs if this is an internal temporary table
like inside ALTER TABLE (I expect this to be a big win). There was
already no logging for user-created "CREATE TEMPORARY" tables.
- don't fsync files/directories if the table is not transactional
- in translog_write_record(), autogenerate a 2-byte-id for the table
and log the "id->name" pair (LOGREC_FILE_ID); log
LOGREC_LONG_TRANSACTION_ID; automatically store
the table's 2-byte-id in any log record.
- preparations for Checkpoint: translog_get_horizon(); pausing Checkpoint
when some dirty pages are unknown; capturing trn->rec_lsn,
trn->first_undo_lsn for Checkpoint and log's low-water-mark computing.
- assertions, comments.
storage/maria/Makefile.am:
more files to build
storage/maria/ha_maria.cc:
- logging a REPAIR log record if REPAIR/OPTIMIZE was successful.
- ha_maria::data_file_type does not have to be set in every info()
call, just do it once in open().
- if caller said that transactionality can be disabled (like if
caller is ALTER TABLE) i.e. thd->transaction.on==FALSE, then we
temporarily disable transactionality of the table in external_lock();
that will ensure that no REDOs/UNDOs are logged for this possibly
massive write operation (they are not needed, as if any write fails,
the table will be dropped). We re-enable in external_lock(F_UNLCK),
which in ALTER TABLE happens before the tmp table replaces the original
one (which is good, as thus the final table will have a REDO RENAME
and a correct create_rename_lsn).
- when we commit we also have to write a log record, so
trnman_commit_trn() calls become ma_commit() calls
- at end of engine's initialization, we are potentially entering a
multi-threaded dangerous world (clients are going to be accepted)
and so some assertions of mutex-owning become enforceable, for that
we set maria_multi_threaded=TRUE (see ma_control_file.c)
storage/maria/ha_maria.h:
new member ha_maria::save_transactional (see also ha_maria.cc)
storage/maria/ma_blockrec.c:
- fixing comments according to discussion with Monty
- if a table is transactional but temporarily non-transactional
(like in ALTER TABLE), we need to give a sensible LSN to the pages
(and, if we give 0, pagecache asserts).
- translog_write_record() now takes care of storing the share's
2-byte-id in the log record
storage/maria/ma_blockrec.h:
fixing comment according to discussion with Monty
storage/maria/ma_check.c:
When REPAIR/OPTIMIZE modify the data/index file, if this is a
transactional table, they must sync it; if they remove files or rename
files, they must sync the directory, so that everything is durable.
This is just applying to REPAIR/OPTIMIZE the logic already implemented
in CREATE/DROP/RENAME a few months ago.
Adding a function to write a LOGREC_REPAIR_TABLE at end of
REPAIR/OPTIMIZE (called only by ha_maria, not by maria_chk), and
to update the table's create_rename_lsn.
storage/maria/ma_close.c:
fix for a future bug
storage/maria/ma_control_file.c:
ensuring that if Maria is running in multi-threaded mode, anybody
wanting to write to the control file and update
last_checkpoint_lsn/last_logno owns the log's lock.
storage/maria/ma_control_file.h:
see ma_control_file.c
storage/maria/ma_create.c:
when creating a table:
- sync it and its directory only if this is a transactional table
and there is a log (no point in syncing in maria_chk)
- decouple the two uses of linkname/linkname_ptr (for index file and
for data file) into more variables, as we need to know all links
until the moment we write the LOGREC_CREATE_TABLE.
- set share.data_file_type early so that _ma_initialize_data_file()
knows it (Monty's bugfix so that a table always has at least a bitmap
page when it is created; so data-file is not 0 bytes anymore).
- log a LOGREC_CREATE_TABLE; it contains the bytes which we have
just written to the index file's header. Update table's
create_rename_lsn.
- syncing of kfile had been bugified in a previous merge, correcting
- syncing of dfile is now needed as it's not empty anymore
- in _ma_initialize_data_file(), use share's block_size and not the
global one. This is a gratuitous change, both variables are equal,
just that I find it more future-proof to use share-bound variable
rather than global one.
storage/maria/ma_delete_all.c:
log a LOGREC_DELETE_ALL record when doing ma_delete_all_rows();
update create_rename_lsn then.
storage/maria/ma_delete_table.c:
- logging LOGREC_DROP_TABLE; knowing if this is needed, requires
knowing if the table is transactional, which requires opening the
table.
- we need to sync directories only if the table is transactional
storage/maria/ma_extra.c:
questions
storage/maria/ma_init.c:
when maria_end() is called, engine is not multithreaded
storage/maria/ma_loghandler.c:
- translog_inited has to be visible to ma_create() (see how it is used
in ma_create())
- checkpoint record will be a single record, not three
- no REDO for TRUNCATE (TRUNCATE calls ma_create() internally so will
log a REDO_CREATE)
- adding REDO for DELETE no_WHERE_clause (fast DELETE of all rows by
truncating the files), REPAIR.
- MY_WAIT_IF_FULL to wait&retry if a log write hits a full disk
- in translog_write_record(), if MARIA_SHARE does not yet have a
2-byte-id, generate one for it and log LOGREC_FILE_ID; automatically
store this short id into log records.
- in translog_write_record(), if transaction has not logged its
long trid, log LOGREC_LONG_TRANSACTION_ID.
- For Checkpoint, we need to know the current end-of-log: adding
translog_get_horizon().
- For Control File, adding an assertion that the thread owns the
log's lock (control file is protected by this lock)
storage/maria/ma_loghandler.h:
Changes in log records (see ma_loghandler.c).
new prototypes, new functions.
storage/maria/ma_loghandler_lsn.h:
adding a type LSN_WITH_FLAGS especially for TRN::first_undo_lsn,
where the most significant byte is used for flags.
storage/maria/ma_open.c:
storing the create_rename_lsn in the index file's header (in the
state, precisely) and retrieving it from there.
storage/maria/ma_pagecache.c:
- my set_if_bigger was wrong, correcting it
- if the first_in_switch list is not empty, it means that
changed_blocks misses some dirty pages, so Checkpoint cannot run and
needs to wait. A variable missing_blocks_in_changed_list is added to
tell that (should it be named missing_blocks_in_changed_blocks?)
- pagecache_collect_changed_blocks_with_lsn() now also tells the
minimum rec_lsn (needed for low-water mark computation).
storage/maria/ma_pagecache.h:
see ma_pagecache.c
storage/maria/ma_panic.c:
comment
storage/maria/ma_range.c:
comment
storage/maria/ma_rename.c:
- logging LOGREC_RENAME_TABLE; knowing if this is needed, requires
knowing if the table is transactional, which requires opening the
table.
- update create_rename_lsn
- we need to sync directories only if the table is transactional
storage/maria/ma_static.c:
comment
storage/maria/ma_test_all.sh:
- tip for Valgrind-ing ma_test_all
- do "export maria_path=somepath" before calling ma_test_all,
if you want to run ma_test_all out of storage/maria (useful
to have parallel runs, like one normal and one Valgrind, they
must not use the same tables so need to run in different directories)
storage/maria/maria_def.h:
- state now contains, in memory and on disk, the create_rename_lsn
- share now contains a 2-byte-id
storage/maria/trnman.c:
preparations for Checkpoint: capture trn->rec_lsn, trn->first_undo_lsn;
minimum first_undo_lsn needed to know log's low-water-mark
storage/maria/trnman.h:
using most significant byte of first_undo_lsn to hold miscellaneous
flags, for now TRANSACTION_LOGGED_LONG_ID.
dummy_transaction_object is already declared in ma_static.c.
storage/maria/trnman_public.h:
dummy_transaction_object was declared in all files including
trnman_public.h, while in fact it's a single object.
new prototype
storage/maria/unittest/ma_test_loghandler-t.c:
update for new prototype
storage/maria/unittest/ma_test_loghandler_multigroup-t.c:
update for new prototype
storage/maria/unittest/ma_test_loghandler_multithread-t.c:
update for new prototype
storage/maria/unittest/ma_test_loghandler_pagecache-t.c:
update for new prototype
storage/maria/ma_commit.c:
function which wraps:
- writing a LOGREC_COMMIT record (==commit on disk)
- calling trnman_commit_trn() (=commit in memory)
storage/maria/ma_commit.h:
new header file
.tree-is-private:
this file is now needed to keep our tree private (don't push it
to public trees). When 5.1 is merged into mysql-maria, we can abandon
our maria-specific post-commit trigger; .tree_is_private will take
care of keeping commit mails private. Don't push this file to public
trees.
2007-06-22 14:49:37 +02:00
|
|
|
|
|
|
|
@return Operation status
|
|
|
|
@retval 0 OK
|
|
|
|
@retval 1 Error
|
2006-04-11 16:45:10 +03:00
|
|
|
*/
|
|
|
|
|
- WL#3072 Maria Recovery:
Recovery of state.records (the count of records which is stored into
the header of the index file). For that, state.is_of_lsn is introduced;
logic is explained in ma_recovery.c (look for "Recovery of the state").
The net gain is that in case of crash, we now recover state.records,
and it is idempotent (ma_test_recovery tests it).
state.checksum is not recovered yet, mail sent for discussion.
- WL#3071 Maria Checkpoint: preparation for it, by protecting
all modifications of the state in memory or on disk with intern_lock
(with the exception of the really-often-modified state.records,
which is now protected with the log's lock, see ma_recovery.c
(look for "Recovery of the state"). Also, if maria_close() sees that
Checkpoint is looking at this table it will not my_free() the share.
- don't compute row's checksum twice in case of UPDATE (correction
to a bugfix I made yesterday).
storage/maria/ha_maria.cc:
protect state write with intern_lock (against Checkpoint)
storage/maria/ma_blockrec.c:
* don't reset trn->rec_lsn in _ma_unpin_all_pages(), because it
should wait until we have corrected the allocation in the bitmap
(as the REDO can serve to correct the allocation during Recovery);
introducing _ma_finalize_row() for that.
* In a changeset yesterday I moved computation of the checksum
into write_block_record(), to fix a bug in UPDATE. Now I notice
that maria_update() already computes the checksum, it's just that
it puts it into info->cur_row while _ma_update_block_record()
uses info->new_row; so, removing the checksum computation from
write_block_record(), putting it back into allocate_and_write_block_record()
(which is called only by INSERT and UNDO_DELETE), and copying
cur_row->checksum into new_row->checksum in _ma_update_block_record().
storage/maria/ma_check.c:
new prototypes, they will take intern_lock when writing the state;
also take intern_lock when changing share->kfile. In both cases
this is to protect against Checkpoint reading/writing the state or reading
kfile at the same time.
Not updating create_rename_lsn directly at end of write_log_record_for_repair()
as it wouldn't have intern_lock.
storage/maria/ma_close.c:
Checkpoint builds a list of shares (under THR_LOCK_maria), then it
handles each such share (under intern_lock) (doing flushing etc);
if maria_close() freed this share between the two, Checkpoint
would see a bad pointer. To avoid this, when building the list Checkpoint
marks each share, so that maria_close() knows it should not free it
and Checkpoint will free it itself.
Extending the zone covered by intern_lock to protect against
Checkpoint reading kfile, writing state.
storage/maria/ma_create.c:
When we update create_rename_lsn, we also update is_of_lsn to
the same value: it is logical, and allows us to test in maria_open()
that the former is not bigger than the latter (the contrary is a sign
of index header corruption, or severe logging bug which hinders
Recovery, table needs a repair).
_ma_update_create_rename_lsn_on_disk() also writes is_of_lsn;
it now operates under intern_lock (protect against Checkpoint),
a shortcut function is available for cases where acquiring
intern_lock is not needed (table's creation or first open).
storage/maria/ma_delete.c:
if table is transactional, "records" is already decremented
when logging UNDO_ROW_DELETE.
storage/maria/ma_delete_all.c:
comments
storage/maria/ma_extra.c:
Protect modifications of the state, in memory and/or on disk,
with intern_lock, against a concurrent Checkpoint.
When state goes to disk, update it's is_of_lsn (by calling
the new _ma_state_info_write()).
In HA_EXTRA_FORCE_REOPEN, don't set share->changed to 0 (undoing
a change I made a few days ago) and ASK_MONTY
storage/maria/ma_locking.c:
no real code change here.
storage/maria/ma_loghandler.c:
Log-write-hooks for updating "state.records" under log's mutex
when writing/updating/deleting a row or deleting all rows.
storage/maria/ma_loghandler_lsn.h:
merge (make LSN_ERROR and LSN_REPAIRED_BY_MARIA_CHK different)
storage/maria/ma_open.c:
When opening a table verify that is_of_lsn >= create_rename_lsn; if
false the header must be corrupted.
_ma_state_info_write() is split in two: _ma_state_info_write_sub()
which is the old _ma_state_info_write(), and _ma_state_info_write()
which additionally takes intern_lock if requested (to protect
against Checkpoint) and updates is_of_lsn.
_ma_open_keyfile() should change kfile.file under intern_lock
to protect Checkpoint from reading a wrong kfile.file.
storage/maria/ma_recovery.c:
Recovery of state.records: when the REDO phase sees UNDO_ROW_INSERT
which has a LSN > state.is_of_lsn it increments state.records.
Same for UNDO_ROW_DELETE and UNDO_ROW_PURGE.
When closing a table during Recovery, we know its state is at least
as new as the current log record we are looking at, so increase
is_of_lsn to the LSN of the current log record.
storage/maria/ma_rename.c:
update for new behaviour of _ma_update_create_rename_lsn_on_disk().
storage/maria/ma_test1.c:
update to new prototype
storage/maria/ma_test2.c:
update to new prototype (actually prototype was changed days ago,
but compiler does not complain about the extra argument??)
storage/maria/ma_test_recovery.expected:
new result file of ma_test_recovery. Improvements: record
count read from index's header is now always correct.
storage/maria/ma_test_recovery:
"rm" fails if file does not exist. Redirect stderr of script.
storage/maria/ma_write.c:
if table is transactional, "records" is already incremented when
logging UNDO_ROW_INSERT. Comments.
storage/maria/maria_chk.c:
update is_of_lsn too
storage/maria/maria_def.h:
- MARIA_STATE_INFO::is_of_lsn which is used by Recovery. It is stored
into the index file's header.
- Checkpoint can now mark a table as "don't free this", and maria_close()
can reply "ok then you will free it".
- new functions
storage/maria/maria_pack.c:
update for new name
2007-09-07 15:02:30 +02:00
|
|
|
uint _ma_state_info_write_sub(File file, MARIA_STATE_INFO *state, uint pWrite)
|
2006-04-11 16:45:10 +03:00
|
|
|
{
|
- WL#3239 "log CREATE TABLE in Maria"
- WL#3240 "log DROP TABLE in Maria"
- similarly, log RENAME TABLE, REPAIR/OPTIMIZE TABLE, and
DELETE no_WHERE_clause (== the DELETE which just truncates the files)
- create_rename_lsn added to MARIA_SHARE's state
- all these operations (except DROP TABLE) also update the table's
create_rename_lsn, which is needed for the correctness of
Recovery (see function comment of _ma_repair_write_log_record()
in ma_check.c)
- write a COMMIT record when transaction commits.
- don't log REDOs/UNDOs if this is an internal temporary table
like inside ALTER TABLE (I expect this to be a big win). There was
already no logging for user-created "CREATE TEMPORARY" tables.
- don't fsync files/directories if the table is not transactional
- in translog_write_record(), autogenerate a 2-byte-id for the table
and log the "id->name" pair (LOGREC_FILE_ID); log
LOGREC_LONG_TRANSACTION_ID; automatically store
the table's 2-byte-id in any log record.
- preparations for Checkpoint: translog_get_horizon(); pausing Checkpoint
when some dirty pages are unknown; capturing trn->rec_lsn,
trn->first_undo_lsn for Checkpoint and log's low-water-mark computing.
- assertions, comments.
storage/maria/Makefile.am:
more files to build
storage/maria/ha_maria.cc:
- logging a REPAIR log record if REPAIR/OPTIMIZE was successful.
- ha_maria::data_file_type does not have to be set in every info()
call, just do it once in open().
- if caller said that transactionality can be disabled (like if
caller is ALTER TABLE) i.e. thd->transaction.on==FALSE, then we
temporarily disable transactionality of the table in external_lock();
that will ensure that no REDOs/UNDOs are logged for this possibly
massive write operation (they are not needed, as if any write fails,
the table will be dropped). We re-enable in external_lock(F_UNLCK),
which in ALTER TABLE happens before the tmp table replaces the original
one (which is good, as thus the final table will have a REDO RENAME
and a correct create_rename_lsn).
- when we commit we also have to write a log record, so
trnman_commit_trn() calls become ma_commit() calls
- at end of engine's initialization, we are potentially entering a
multi-threaded dangerous world (clients are going to be accepted)
and so some assertions of mutex-owning become enforceable, for that
we set maria_multi_threaded=TRUE (see ma_control_file.c)
storage/maria/ha_maria.h:
new member ha_maria::save_transactional (see also ha_maria.cc)
storage/maria/ma_blockrec.c:
- fixing comments according to discussion with Monty
- if a table is transactional but temporarily non-transactional
(like in ALTER TABLE), we need to give a sensible LSN to the pages
(and, if we give 0, pagecache asserts).
- translog_write_record() now takes care of storing the share's
2-byte-id in the log record
storage/maria/ma_blockrec.h:
fixing comment according to discussion with Monty
storage/maria/ma_check.c:
When REPAIR/OPTIMIZE modify the data/index file, if this is a
transactional table, they must sync it; if they remove files or rename
files, they must sync the directory, so that everything is durable.
This is just applying to REPAIR/OPTIMIZE the logic already implemented
in CREATE/DROP/RENAME a few months ago.
Adding a function to write a LOGREC_REPAIR_TABLE at end of
REPAIR/OPTIMIZE (called only by ha_maria, not by maria_chk), and
to update the table's create_rename_lsn.
storage/maria/ma_close.c:
fix for a future bug
storage/maria/ma_control_file.c:
ensuring that if Maria is running in multi-threaded mode, anybody
wanting to write to the control file and update
last_checkpoint_lsn/last_logno owns the log's lock.
storage/maria/ma_control_file.h:
see ma_control_file.c
storage/maria/ma_create.c:
when creating a table:
- sync it and its directory only if this is a transactional table
and there is a log (no point in syncing in maria_chk)
- decouple the two uses of linkname/linkname_ptr (for index file and
for data file) into more variables, as we need to know all links
until the moment we write the LOGREC_CREATE_TABLE.
- set share.data_file_type early so that _ma_initialize_data_file()
knows it (Monty's bugfix so that a table always has at least a bitmap
page when it is created; so data-file is not 0 bytes anymore).
- log a LOGREC_CREATE_TABLE; it contains the bytes which we have
just written to the index file's header. Update table's
create_rename_lsn.
- syncing of kfile had been bugified in a previous merge, correcting
- syncing of dfile is now needed as it's not empty anymore
- in _ma_initialize_data_file(), use share's block_size and not the
global one. This is a gratuitous change, both variables are equal,
just that I find it more future-proof to use share-bound variable
rather than global one.
storage/maria/ma_delete_all.c:
log a LOGREC_DELETE_ALL record when doing ma_delete_all_rows();
update create_rename_lsn then.
storage/maria/ma_delete_table.c:
- logging LOGREC_DROP_TABLE; knowing if this is needed, requires
knowing if the table is transactional, which requires opening the
table.
- we need to sync directories only if the table is transactional
storage/maria/ma_extra.c:
questions
storage/maria/ma_init.c:
when maria_end() is called, engine is not multithreaded
storage/maria/ma_loghandler.c:
- translog_inited has to be visible to ma_create() (see how it is used
in ma_create())
- checkpoint record will be a single record, not three
- no REDO for TRUNCATE (TRUNCATE calls ma_create() internally so will
log a REDO_CREATE)
- adding REDO for DELETE no_WHERE_clause (fast DELETE of all rows by
truncating the files), REPAIR.
- MY_WAIT_IF_FULL to wait&retry if a log write hits a full disk
- in translog_write_record(), if MARIA_SHARE does not yet have a
2-byte-id, generate one for it and log LOGREC_FILE_ID; automatically
store this short id into log records.
- in translog_write_record(), if transaction has not logged its
long trid, log LOGREC_LONG_TRANSACTION_ID.
- For Checkpoint, we need to know the current end-of-log: adding
translog_get_horizon().
- For Control File, adding an assertion that the thread owns the
log's lock (control file is protected by this lock)
storage/maria/ma_loghandler.h:
Changes in log records (see ma_loghandler.c).
new prototypes, new functions.
storage/maria/ma_loghandler_lsn.h:
adding a type LSN_WITH_FLAGS especially for TRN::first_undo_lsn,
where the most significant byte is used for flags.
storage/maria/ma_open.c:
storing the create_rename_lsn in the index file's header (in the
state, precisely) and retrieving it from there.
storage/maria/ma_pagecache.c:
- my set_if_bigger was wrong, correcting it
- if the first_in_switch list is not empty, it means that
changed_blocks misses some dirty pages, so Checkpoint cannot run and
needs to wait. A variable missing_blocks_in_changed_list is added to
tell that (should it be named missing_blocks_in_changed_blocks?)
- pagecache_collect_changed_blocks_with_lsn() now also tells the
minimum rec_lsn (needed for low-water mark computation).
storage/maria/ma_pagecache.h:
see ma_pagecache.c
storage/maria/ma_panic.c:
comment
storage/maria/ma_range.c:
comment
storage/maria/ma_rename.c:
- logging LOGREC_RENAME_TABLE; knowing if this is needed, requires
knowing if the table is transactional, which requires opening the
table.
- update create_rename_lsn
- we need to sync directories only if the table is transactional
storage/maria/ma_static.c:
comment
storage/maria/ma_test_all.sh:
- tip for Valgrind-ing ma_test_all
- do "export maria_path=somepath" before calling ma_test_all,
if you want to run ma_test_all out of storage/maria (useful
to have parallel runs, like one normal and one Valgrind, they
must not use the same tables so need to run in different directories)
storage/maria/maria_def.h:
- state now contains, in memory and on disk, the create_rename_lsn
- share now contains a 2-byte-id
storage/maria/trnman.c:
preparations for Checkpoint: capture trn->rec_lsn, trn->first_undo_lsn;
minimum first_undo_lsn needed to know log's low-water-mark
storage/maria/trnman.h:
using most significant byte of first_undo_lsn to hold miscellaneous
flags, for now TRANSACTION_LOGGED_LONG_ID.
dummy_transaction_object is already declared in ma_static.c.
storage/maria/trnman_public.h:
dummy_transaction_object was declared in all files including
trnman_public.h, while in fact it's a single object.
new prototype
storage/maria/unittest/ma_test_loghandler-t.c:
update for new prototype
storage/maria/unittest/ma_test_loghandler_multigroup-t.c:
update for new prototype
storage/maria/unittest/ma_test_loghandler_multithread-t.c:
update for new prototype
storage/maria/unittest/ma_test_loghandler_pagecache-t.c:
update for new prototype
storage/maria/ma_commit.c:
function which wraps:
- writing a LOGREC_COMMIT record (==commit on disk)
- calling trnman_commit_trn() (=commit in memory)
storage/maria/ma_commit.h:
new header file
.tree-is-private:
this file is now needed to keep our tree private (don't push it
to public trees). When 5.1 is merged into mysql-maria, we can abandon
our maria-specific post-commit trigger; .tree_is_private will take
care of keeping commit mails private. Don't push this file to public
trees.
2007-06-22 14:49:37 +02:00
|
|
|
/** @todo RECOVERY write it only at checkpoint time */
|
2006-04-11 16:45:10 +03:00
|
|
|
uchar buff[MARIA_STATE_INFO_SIZE + MARIA_STATE_EXTRA_SIZE];
|
|
|
|
uchar *ptr=buff;
|
2007-01-18 21:38:14 +02:00
|
|
|
uint i, keys= (uint) state->header.keys;
|
WL#3072 Maria recovery
* create page cache before initializing engine and not after, because
Maria's recovery needs a page cache
* make the creation of a bitmap page more crash-resistent
* bugfix (see ma_blockrec.c)
* back to old way: create an 8k bitmap page when creating table
* preparations for the UNDO phase: recreate TRNs
* preparations for Checkpoint: list of dirty pages, testing
of rec_lsn to know if page should be skipped during Recovery
(unused in this patch as no Checkpoint module pushed yet)
* maria_chk tags repaired table with a special LSN
* reworking all around in ma_recovery.c (less duplication)
mysys/my_realloc.c:
noted an issue in my_realloc()
sql/mysqld.cc:
page cache needs to be created before engines are initialized,
because Maria's initialization may do a recovery which needs
the page cache.
storage/maria/ha_maria.cc:
update to new prototype
storage/maria/ma_bitmap.c:
when creating the first bitmap page we used chsize to 8192 bytes then
pwrite (overwrite) the last 2 bytes (8191-8192). If crash between
the two operations, this leaves a bitmap page full without its end
marker. A later recovery may try to read this page and find it
exists and misses a marker and conclude it's corrupted and fail.
Changing the chsize to only 8190 bytes: recovery will then find
the page is too short and recreate it entirely.
storage/maria/ma_blockrec.c:
Fix for a bug: when executing a REDO, if the data page is created,
data_file_length was increased before _ma_bitmap_set():
_ma_bitmap_set() called _ma_read_bitmap_page() which, due to the
increased data_file_length, expected to find a bitmap page on disk
with a correct end marker; if the bitmap page didn't exist already
in fact, this failed. Fixed by increasing data_file_length only after
_ma_read_bitmap_page() has created the new bitmap page correctly.
This bug could happen every time a REDO is about creating a new
bitmap page.
storage/maria/ma_check.c:
empty data file has a bitmap page
storage/maria/ma_control_file.c:
useless parameter to ma_control_file_create_or_open(), just
test if this is recovery.
storage/maria/ma_control_file.h:
new prototype
storage/maria/ma_create.c:
Back to how it was before: maria_create() creates an 8k bitmap page.
Thus (bugfix) data_file_length needs to reflect this instead of being 0.
storage/maria/ma_loghandler.c:
as ma_test1 and ma_test2 now use real transactions and not
dummy_transaction_object, REDO for INSERT/UPDATE/DELETE are always
about real transactions, can assert this.
A function for Recovery to assign a short id to a table.
storage/maria/ma_loghandler.h:
new function
storage/maria/ma_loghandler_lsn.h:
maria_chk tags repaired tables with this LSN
storage/maria/ma_open.c:
* enforce that DMLs on transactional tables use real transactions
and not dummy_transaction_object.
* test if table was repaired with maria_chk (which has to been
seen as an import of an external table into the server), test
validity of create_rename_lsn (header corruption detection)
* comments.
storage/maria/ma_recovery.c:
* preparations for the UNDO phase: recreate TRNs
* preparations for Checkpoint: list of dirty pages, testing
of rec_lsn to know if page should be skipped during Recovery
(unused in this patch as no Checkpoint module pushed yet)
* reworking all around (less duplication)
storage/maria/ma_recovery.h:
a parameter to say if the UNDO phase should be skipped
storage/maria/maria_chk.c:
tag repaired tables with a special LSN
storage/maria/maria_read_log.c:
* update to new prototype
* no UNDO phase in maria_read_log for now
storage/maria/trnman.c:
* a function for Recovery to create a transaction (TRN), needed
in the UNDO phase
* a function for Recovery to grab an existing transaction, needed
in the UNDO phase (rollback all existing transactions)
storage/maria/trnman_public.h:
new functions
2007-08-29 16:43:01 +02:00
|
|
|
size_t res;
|
2006-04-11 16:45:10 +03:00
|
|
|
DBUG_ENTER("_ma_state_info_write");
|
|
|
|
|
|
|
|
memcpy_fixed(ptr,&state->header,sizeof(state->header));
|
|
|
|
ptr+=sizeof(state->header);
|
|
|
|
|
|
|
|
/* open_count must be first because of _ma_mark_file_changed ! */
|
2007-01-18 21:38:14 +02:00
|
|
|
mi_int2store(ptr,state->open_count); ptr+= 2;
|
- WL#3239 "log CREATE TABLE in Maria"
- WL#3240 "log DROP TABLE in Maria"
- similarly, log RENAME TABLE, REPAIR/OPTIMIZE TABLE, and
DELETE no_WHERE_clause (== the DELETE which just truncates the files)
- create_rename_lsn added to MARIA_SHARE's state
- all these operations (except DROP TABLE) also update the table's
create_rename_lsn, which is needed for the correctness of
Recovery (see function comment of _ma_repair_write_log_record()
in ma_check.c)
- write a COMMIT record when transaction commits.
- don't log REDOs/UNDOs if this is an internal temporary table
like inside ALTER TABLE (I expect this to be a big win). There was
already no logging for user-created "CREATE TEMPORARY" tables.
- don't fsync files/directories if the table is not transactional
- in translog_write_record(), autogenerate a 2-byte-id for the table
and log the "id->name" pair (LOGREC_FILE_ID); log
LOGREC_LONG_TRANSACTION_ID; automatically store
the table's 2-byte-id in any log record.
- preparations for Checkpoint: translog_get_horizon(); pausing Checkpoint
when some dirty pages are unknown; capturing trn->rec_lsn,
trn->first_undo_lsn for Checkpoint and log's low-water-mark computing.
- assertions, comments.
storage/maria/Makefile.am:
more files to build
storage/maria/ha_maria.cc:
- logging a REPAIR log record if REPAIR/OPTIMIZE was successful.
- ha_maria::data_file_type does not have to be set in every info()
call, just do it once in open().
- if caller said that transactionality can be disabled (like if
caller is ALTER TABLE) i.e. thd->transaction.on==FALSE, then we
temporarily disable transactionality of the table in external_lock();
that will ensure that no REDOs/UNDOs are logged for this possibly
massive write operation (they are not needed, as if any write fails,
the table will be dropped). We re-enable in external_lock(F_UNLCK),
which in ALTER TABLE happens before the tmp table replaces the original
one (which is good, as thus the final table will have a REDO RENAME
and a correct create_rename_lsn).
- when we commit we also have to write a log record, so
trnman_commit_trn() calls become ma_commit() calls
- at end of engine's initialization, we are potentially entering a
multi-threaded dangerous world (clients are going to be accepted)
and so some assertions of mutex-owning become enforceable, for that
we set maria_multi_threaded=TRUE (see ma_control_file.c)
storage/maria/ha_maria.h:
new member ha_maria::save_transactional (see also ha_maria.cc)
storage/maria/ma_blockrec.c:
- fixing comments according to discussion with Monty
- if a table is transactional but temporarily non-transactional
(like in ALTER TABLE), we need to give a sensible LSN to the pages
(and, if we give 0, pagecache asserts).
- translog_write_record() now takes care of storing the share's
2-byte-id in the log record
storage/maria/ma_blockrec.h:
fixing comment according to discussion with Monty
storage/maria/ma_check.c:
When REPAIR/OPTIMIZE modify the data/index file, if this is a
transactional table, they must sync it; if they remove files or rename
files, they must sync the directory, so that everything is durable.
This is just applying to REPAIR/OPTIMIZE the logic already implemented
in CREATE/DROP/RENAME a few months ago.
Adding a function to write a LOGREC_REPAIR_TABLE at end of
REPAIR/OPTIMIZE (called only by ha_maria, not by maria_chk), and
to update the table's create_rename_lsn.
storage/maria/ma_close.c:
fix for a future bug
storage/maria/ma_control_file.c:
ensuring that if Maria is running in multi-threaded mode, anybody
wanting to write to the control file and update
last_checkpoint_lsn/last_logno owns the log's lock.
storage/maria/ma_control_file.h:
see ma_control_file.c
storage/maria/ma_create.c:
when creating a table:
- sync it and its directory only if this is a transactional table
and there is a log (no point in syncing in maria_chk)
- decouple the two uses of linkname/linkname_ptr (for index file and
for data file) into more variables, as we need to know all links
until the moment we write the LOGREC_CREATE_TABLE.
- set share.data_file_type early so that _ma_initialize_data_file()
knows it (Monty's bugfix so that a table always has at least a bitmap
page when it is created; so data-file is not 0 bytes anymore).
- log a LOGREC_CREATE_TABLE; it contains the bytes which we have
just written to the index file's header. Update table's
create_rename_lsn.
- syncing of kfile had been bugified in a previous merge, correcting
- syncing of dfile is now needed as it's not empty anymore
- in _ma_initialize_data_file(), use share's block_size and not the
global one. This is a gratuitous change, both variables are equal,
just that I find it more future-proof to use share-bound variable
rather than global one.
storage/maria/ma_delete_all.c:
log a LOGREC_DELETE_ALL record when doing ma_delete_all_rows();
update create_rename_lsn then.
storage/maria/ma_delete_table.c:
- logging LOGREC_DROP_TABLE; knowing if this is needed, requires
knowing if the table is transactional, which requires opening the
table.
- we need to sync directories only if the table is transactional
storage/maria/ma_extra.c:
questions
storage/maria/ma_init.c:
when maria_end() is called, engine is not multithreaded
storage/maria/ma_loghandler.c:
- translog_inited has to be visible to ma_create() (see how it is used
in ma_create())
- checkpoint record will be a single record, not three
- no REDO for TRUNCATE (TRUNCATE calls ma_create() internally so will
log a REDO_CREATE)
- adding REDO for DELETE no_WHERE_clause (fast DELETE of all rows by
truncating the files), REPAIR.
- MY_WAIT_IF_FULL to wait&retry if a log write hits a full disk
- in translog_write_record(), if MARIA_SHARE does not yet have a
2-byte-id, generate one for it and log LOGREC_FILE_ID; automatically
store this short id into log records.
- in translog_write_record(), if transaction has not logged its
long trid, log LOGREC_LONG_TRANSACTION_ID.
- For Checkpoint, we need to know the current end-of-log: adding
translog_get_horizon().
- For Control File, adding an assertion that the thread owns the
log's lock (control file is protected by this lock)
storage/maria/ma_loghandler.h:
Changes in log records (see ma_loghandler.c).
new prototypes, new functions.
storage/maria/ma_loghandler_lsn.h:
adding a type LSN_WITH_FLAGS especially for TRN::first_undo_lsn,
where the most significant byte is used for flags.
storage/maria/ma_open.c:
storing the create_rename_lsn in the index file's header (in the
state, precisely) and retrieving it from there.
storage/maria/ma_pagecache.c:
- my set_if_bigger was wrong, correcting it
- if the first_in_switch list is not empty, it means that
changed_blocks misses some dirty pages, so Checkpoint cannot run and
needs to wait. A variable missing_blocks_in_changed_list is added to
tell that (should it be named missing_blocks_in_changed_blocks?)
- pagecache_collect_changed_blocks_with_lsn() now also tells the
minimum rec_lsn (needed for low-water mark computation).
storage/maria/ma_pagecache.h:
see ma_pagecache.c
storage/maria/ma_panic.c:
comment
storage/maria/ma_range.c:
comment
storage/maria/ma_rename.c:
- logging LOGREC_RENAME_TABLE; knowing if this is needed, requires
knowing if the table is transactional, which requires opening the
table.
- update create_rename_lsn
- we need to sync directories only if the table is transactional
storage/maria/ma_static.c:
comment
storage/maria/ma_test_all.sh:
- tip for Valgrind-ing ma_test_all
- do "export maria_path=somepath" before calling ma_test_all,
if you want to run ma_test_all out of storage/maria (useful
to have parallel runs, like one normal and one Valgrind, they
must not use the same tables so need to run in different directories)
storage/maria/maria_def.h:
- state now contains, in memory and on disk, the create_rename_lsn
- share now contains a 2-byte-id
storage/maria/trnman.c:
preparations for Checkpoint: capture trn->rec_lsn, trn->first_undo_lsn;
minimum first_undo_lsn needed to know log's low-water-mark
storage/maria/trnman.h:
using most significant byte of first_undo_lsn to hold miscellaneous
flags, for now TRANSACTION_LOGGED_LONG_ID.
dummy_transaction_object is already declared in ma_static.c.
storage/maria/trnman_public.h:
dummy_transaction_object was declared in all files including
trnman_public.h, while in fact it's a single object.
new prototype
storage/maria/unittest/ma_test_loghandler-t.c:
update for new prototype
storage/maria/unittest/ma_test_loghandler_multigroup-t.c:
update for new prototype
storage/maria/unittest/ma_test_loghandler_multithread-t.c:
update for new prototype
storage/maria/unittest/ma_test_loghandler_pagecache-t.c:
update for new prototype
storage/maria/ma_commit.c:
function which wraps:
- writing a LOGREC_COMMIT record (==commit on disk)
- calling trnman_commit_trn() (=commit in memory)
storage/maria/ma_commit.h:
new header file
.tree-is-private:
this file is now needed to keep our tree private (don't push it
to public trees). When 5.1 is merged into mysql-maria, we can abandon
our maria-specific post-commit trigger; .tree_is_private will take
care of keeping commit mails private. Don't push this file to public
trees.
2007-06-22 14:49:37 +02:00
|
|
|
/*
|
WL#3071 Maria checkpoint
Finally this is the real checkpoint code.
It however exhibits unstabilities when a checkpoint runs concurrently
with data-modifying clients (table corruption, transaction log's
assertions) so for now a checkpoint is taken only at startup after
recovery and at shutdown, i.e. not in concurrent situations. Later
we will let it run periodically, as well as flush dirty pages
periodically (almost all needed code is there already, only pagecache
code is written but not committed).
WL#3072 Maria recovery
* replacing UNDO_ROW_PURGE with CLR_END; testing of those CLR_END via
ma_test2 which has INSERTs failing with duplicate keys.
* replaying of REDO_RENAME_TABLE
Now, off to test Recovery in ha_maria :)
BitKeeper/deleted/.del-ma_least_recently_dirtied.c:
Delete: storage/maria/ma_least_recently_dirtied.c
BitKeeper/deleted/.del-ma_least_recently_dirtied.h:
Delete: storage/maria/ma_least_recently_dirtied.h
storage/maria/Makefile.am:
compile Checkpoint module
storage/maria/ha_maria.cc:
When ha_maria starts, do a recovery from last checkpoint.
Take a checkpoint when that recovery has ended and when ha_maria
shuts down cleanly.
storage/maria/ma_blockrec.c:
* even if my_sync() fails we have to my_close() (otherwise we leak
a descriptor)
* UNDO_ROW_PURGE is replaced by a simple CLR_END for UNDO_ROW_INSERT,
as promised in the old comment; it gives us skipping during the
UNDO phase.
storage/maria/ma_check.c:
All REDOs before create_rename_lsn are ignored by Recovery. So
create_rename_lsn must be set only after all data/index has been
flushed and forced to disk. We thus move write_log_record_for_repair()
to after _ma_flush_tables_files_after_repair().
storage/maria/ma_checkpoint.c:
Checkpoint module.
storage/maria/ma_checkpoint.h:
optional argument if caller wants a thread to periodically take
checkpoints and flush dirty pages.
storage/maria/ma_create.c:
* no need to init some vars as the initial bzero(share) takes care of this.
* update to new function's name
* even if we fail in my_sync() we have to my_close()
storage/maria/ma_extra.c:
Checkpoint reads share->last_version under intern_lock, so we make
maria_extra() update it under intern_lock. THR_LOCK_maria still needed
because of _ma_test_if_reopen().
storage/maria/ma_init.c:
destroy checkpoint module when Maria shuts down.
storage/maria/ma_loghandler.c:
* UNDO_ROW_PURGE gone (see ma_blockrec.c)
* we need to remember the LSN of the LOGREC_FILE_ID for a share,
because this LSN is needed into the checkpoint record (Recovery wants
to know the validity domain of an id->name mapping)
* translog_get_horizon_no_lock() needed for Checkpoint
* comment about failing assertion (Sanja knows)
* translog_init_reader_data() thought that translog_read_record_header_scan()
returns 0 in case of error, but 0 just means "0-length header".
* translog_assign_id_to_share() now needs the MARIA_HA because
LOGREC_FILE_ID uses a log-write hook.
* Verify that (de)assignment of share->id happens only under intern_lock,
as Checkpoint reads this id with intern_lock.
* translog_purge() can accept TRANSLOG_ADDRESS, not necessarily
a real LSN.
storage/maria/ma_loghandler.h:
prototype updates
storage/maria/ma_open.c:
no need to initialize "res"
storage/maria/ma_pagecache.c:
When taking a checkpoint, we don't need to know the maximum rec_lsn
of dirty pages; this LSN was intended to be used in the two-checkpoint
rule, but last_checkpoint_lsn is as good.
4 bytes for stored_list_size is enough as PAGECACHE::blocks (number
of blocks which the pagecache can contain) is int.
storage/maria/ma_pagecache.h:
new prototype
storage/maria/ma_recovery.c:
* added replaying of REDO_RENAME_TABLE
* UNDO_ROW_PURGE gone (see ma_blockrec.c), replaced by CLR_END
* Recovery from the last checkpoint record now possible
* In new_table() we skip the table if the id->name mapping is older than
create_rename_lsn (mapping dates from lsn_of_file_id).
* in get_MARIA_HA_from_REDO_record() we skip the record
if the id->name mapping is newer than the record (can happen if processing
a record which is before the checkpoint record).
* parse_checkpoint_record() has to return a LSN, that's what caller expects
storage/maria/ma_rename.c:
new function's name; log end zeroes of tables' names (ease recovery)
storage/maria/ma_test2.c:
* equivalent of ma_test1's --test-undo added (named -u here).
* -t=1 now stops right after creating the table, so that
we can test undoing of INSERTs with duplicate keys (which tests the
CLR_END logged by _ma_write_abort_block_record()).
storage/maria/ma_test_recovery.expected:
Result of testing undoing of INSERTs with duplicate keys; there are
some differences in maria_chk -dvv but they are normal (removing
records does not shrink data/index file, does not put back the
"analyzed, optimized keys"(etc) index state.
storage/maria/ma_test_recovery:
Test undoing of INSERTs with duplicate keys, using ma_test2;
when such INSERT happens, it logs REDO_INSERT, UNDO_INSERT, REDO_DELETE,
CLR_END; we abort after that, and test that CLR_END causes recovery
to jump over UNDO_INSERT.
storage/maria/ma_write.c:
comment
storage/maria/maria_chk.c:
comment
storage/maria/maria_def.h:
* a new bit in MARIA_SHARE::in_checkpoint, used to build a list
of unique shares during Checkpoint.
* MARIA_SHARE::lsn_of_file_id added: the LSN of the last LOGREC_FILE_ID
for this share; needed to know to which LSN domain the mappings
found in the Checkpoint record apply (new mappings should not apply
to old REDOs).
storage/maria/trnman.c:
* small changes to how trnman_collect_transactions() fills its buffer;
it also uses a non-dummy lsn_read_non_atomic() found in ma_checkpoint.h
2007-09-12 11:27:34 +02:00
|
|
|
if you change the offset of create_rename_lsn/is_of_horizon inside the
|
|
|
|
index file's header, fix ma_create + ma_rename + ma_delete_all +
|
|
|
|
backward-compatibility.
|
- WL#3239 "log CREATE TABLE in Maria"
- WL#3240 "log DROP TABLE in Maria"
- similarly, log RENAME TABLE, REPAIR/OPTIMIZE TABLE, and
DELETE no_WHERE_clause (== the DELETE which just truncates the files)
- create_rename_lsn added to MARIA_SHARE's state
- all these operations (except DROP TABLE) also update the table's
create_rename_lsn, which is needed for the correctness of
Recovery (see function comment of _ma_repair_write_log_record()
in ma_check.c)
- write a COMMIT record when transaction commits.
- don't log REDOs/UNDOs if this is an internal temporary table
like inside ALTER TABLE (I expect this to be a big win). There was
already no logging for user-created "CREATE TEMPORARY" tables.
- don't fsync files/directories if the table is not transactional
- in translog_write_record(), autogenerate a 2-byte-id for the table
and log the "id->name" pair (LOGREC_FILE_ID); log
LOGREC_LONG_TRANSACTION_ID; automatically store
the table's 2-byte-id in any log record.
- preparations for Checkpoint: translog_get_horizon(); pausing Checkpoint
when some dirty pages are unknown; capturing trn->rec_lsn,
trn->first_undo_lsn for Checkpoint and log's low-water-mark computing.
- assertions, comments.
storage/maria/Makefile.am:
more files to build
storage/maria/ha_maria.cc:
- logging a REPAIR log record if REPAIR/OPTIMIZE was successful.
- ha_maria::data_file_type does not have to be set in every info()
call, just do it once in open().
- if caller said that transactionality can be disabled (like if
caller is ALTER TABLE) i.e. thd->transaction.on==FALSE, then we
temporarily disable transactionality of the table in external_lock();
that will ensure that no REDOs/UNDOs are logged for this possibly
massive write operation (they are not needed, as if any write fails,
the table will be dropped). We re-enable in external_lock(F_UNLCK),
which in ALTER TABLE happens before the tmp table replaces the original
one (which is good, as thus the final table will have a REDO RENAME
and a correct create_rename_lsn).
- when we commit we also have to write a log record, so
trnman_commit_trn() calls become ma_commit() calls
- at end of engine's initialization, we are potentially entering a
multi-threaded dangerous world (clients are going to be accepted)
and so some assertions of mutex-owning become enforceable, for that
we set maria_multi_threaded=TRUE (see ma_control_file.c)
storage/maria/ha_maria.h:
new member ha_maria::save_transactional (see also ha_maria.cc)
storage/maria/ma_blockrec.c:
- fixing comments according to discussion with Monty
- if a table is transactional but temporarily non-transactional
(like in ALTER TABLE), we need to give a sensible LSN to the pages
(and, if we give 0, pagecache asserts).
- translog_write_record() now takes care of storing the share's
2-byte-id in the log record
storage/maria/ma_blockrec.h:
fixing comment according to discussion with Monty
storage/maria/ma_check.c:
When REPAIR/OPTIMIZE modify the data/index file, if this is a
transactional table, they must sync it; if they remove files or rename
files, they must sync the directory, so that everything is durable.
This is just applying to REPAIR/OPTIMIZE the logic already implemented
in CREATE/DROP/RENAME a few months ago.
Adding a function to write a LOGREC_REPAIR_TABLE at end of
REPAIR/OPTIMIZE (called only by ha_maria, not by maria_chk), and
to update the table's create_rename_lsn.
storage/maria/ma_close.c:
fix for a future bug
storage/maria/ma_control_file.c:
ensuring that if Maria is running in multi-threaded mode, anybody
wanting to write to the control file and update
last_checkpoint_lsn/last_logno owns the log's lock.
storage/maria/ma_control_file.h:
see ma_control_file.c
storage/maria/ma_create.c:
when creating a table:
- sync it and its directory only if this is a transactional table
and there is a log (no point in syncing in maria_chk)
- decouple the two uses of linkname/linkname_ptr (for index file and
for data file) into more variables, as we need to know all links
until the moment we write the LOGREC_CREATE_TABLE.
- set share.data_file_type early so that _ma_initialize_data_file()
knows it (Monty's bugfix so that a table always has at least a bitmap
page when it is created; so data-file is not 0 bytes anymore).
- log a LOGREC_CREATE_TABLE; it contains the bytes which we have
just written to the index file's header. Update table's
create_rename_lsn.
- syncing of kfile had been bugified in a previous merge, correcting
- syncing of dfile is now needed as it's not empty anymore
- in _ma_initialize_data_file(), use share's block_size and not the
global one. This is a gratuitous change, both variables are equal,
just that I find it more future-proof to use share-bound variable
rather than global one.
storage/maria/ma_delete_all.c:
log a LOGREC_DELETE_ALL record when doing ma_delete_all_rows();
update create_rename_lsn then.
storage/maria/ma_delete_table.c:
- logging LOGREC_DROP_TABLE; knowing if this is needed, requires
knowing if the table is transactional, which requires opening the
table.
- we need to sync directories only if the table is transactional
storage/maria/ma_extra.c:
questions
storage/maria/ma_init.c:
when maria_end() is called, engine is not multithreaded
storage/maria/ma_loghandler.c:
- translog_inited has to be visible to ma_create() (see how it is used
in ma_create())
- checkpoint record will be a single record, not three
- no REDO for TRUNCATE (TRUNCATE calls ma_create() internally so will
log a REDO_CREATE)
- adding REDO for DELETE no_WHERE_clause (fast DELETE of all rows by
truncating the files), REPAIR.
- MY_WAIT_IF_FULL to wait&retry if a log write hits a full disk
- in translog_write_record(), if MARIA_SHARE does not yet have a
2-byte-id, generate one for it and log LOGREC_FILE_ID; automatically
store this short id into log records.
- in translog_write_record(), if transaction has not logged its
long trid, log LOGREC_LONG_TRANSACTION_ID.
- For Checkpoint, we need to know the current end-of-log: adding
translog_get_horizon().
- For Control File, adding an assertion that the thread owns the
log's lock (control file is protected by this lock)
storage/maria/ma_loghandler.h:
Changes in log records (see ma_loghandler.c).
new prototypes, new functions.
storage/maria/ma_loghandler_lsn.h:
adding a type LSN_WITH_FLAGS especially for TRN::first_undo_lsn,
where the most significant byte is used for flags.
storage/maria/ma_open.c:
storing the create_rename_lsn in the index file's header (in the
state, precisely) and retrieving it from there.
storage/maria/ma_pagecache.c:
- my set_if_bigger was wrong, correcting it
- if the first_in_switch list is not empty, it means that
changed_blocks misses some dirty pages, so Checkpoint cannot run and
needs to wait. A variable missing_blocks_in_changed_list is added to
tell that (should it be named missing_blocks_in_changed_blocks?)
- pagecache_collect_changed_blocks_with_lsn() now also tells the
minimum rec_lsn (needed for low-water mark computation).
storage/maria/ma_pagecache.h:
see ma_pagecache.c
storage/maria/ma_panic.c:
comment
storage/maria/ma_range.c:
comment
storage/maria/ma_rename.c:
- logging LOGREC_RENAME_TABLE; knowing if this is needed, requires
knowing if the table is transactional, which requires opening the
table.
- update create_rename_lsn
- we need to sync directories only if the table is transactional
storage/maria/ma_static.c:
comment
storage/maria/ma_test_all.sh:
- tip for Valgrind-ing ma_test_all
- do "export maria_path=somepath" before calling ma_test_all,
if you want to run ma_test_all out of storage/maria (useful
to have parallel runs, like one normal and one Valgrind, they
must not use the same tables so need to run in different directories)
storage/maria/maria_def.h:
- state now contains, in memory and on disk, the create_rename_lsn
- share now contains a 2-byte-id
storage/maria/trnman.c:
preparations for Checkpoint: capture trn->rec_lsn, trn->first_undo_lsn;
minimum first_undo_lsn needed to know log's low-water-mark
storage/maria/trnman.h:
using most significant byte of first_undo_lsn to hold miscellaneous
flags, for now TRANSACTION_LOGGED_LONG_ID.
dummy_transaction_object is already declared in ma_static.c.
storage/maria/trnman_public.h:
dummy_transaction_object was declared in all files including
trnman_public.h, while in fact it's a single object.
new prototype
storage/maria/unittest/ma_test_loghandler-t.c:
update for new prototype
storage/maria/unittest/ma_test_loghandler_multigroup-t.c:
update for new prototype
storage/maria/unittest/ma_test_loghandler_multithread-t.c:
update for new prototype
storage/maria/unittest/ma_test_loghandler_pagecache-t.c:
update for new prototype
storage/maria/ma_commit.c:
function which wraps:
- writing a LOGREC_COMMIT record (==commit on disk)
- calling trnman_commit_trn() (=commit in memory)
storage/maria/ma_commit.h:
new header file
.tree-is-private:
this file is now needed to keep our tree private (don't push it
to public trees). When 5.1 is merged into mysql-maria, we can abandon
our maria-specific post-commit trigger; .tree_is_private will take
care of keeping commit mails private. Don't push this file to public
trees.
2007-06-22 14:49:37 +02:00
|
|
|
*/
|
|
|
|
lsn_store(ptr, state->create_rename_lsn); ptr+= LSN_STORE_SIZE;
|
WL#3071 Maria checkpoint
Finally this is the real checkpoint code.
It however exhibits unstabilities when a checkpoint runs concurrently
with data-modifying clients (table corruption, transaction log's
assertions) so for now a checkpoint is taken only at startup after
recovery and at shutdown, i.e. not in concurrent situations. Later
we will let it run periodically, as well as flush dirty pages
periodically (almost all needed code is there already, only pagecache
code is written but not committed).
WL#3072 Maria recovery
* replacing UNDO_ROW_PURGE with CLR_END; testing of those CLR_END via
ma_test2 which has INSERTs failing with duplicate keys.
* replaying of REDO_RENAME_TABLE
Now, off to test Recovery in ha_maria :)
BitKeeper/deleted/.del-ma_least_recently_dirtied.c:
Delete: storage/maria/ma_least_recently_dirtied.c
BitKeeper/deleted/.del-ma_least_recently_dirtied.h:
Delete: storage/maria/ma_least_recently_dirtied.h
storage/maria/Makefile.am:
compile Checkpoint module
storage/maria/ha_maria.cc:
When ha_maria starts, do a recovery from last checkpoint.
Take a checkpoint when that recovery has ended and when ha_maria
shuts down cleanly.
storage/maria/ma_blockrec.c:
* even if my_sync() fails we have to my_close() (otherwise we leak
a descriptor)
* UNDO_ROW_PURGE is replaced by a simple CLR_END for UNDO_ROW_INSERT,
as promised in the old comment; it gives us skipping during the
UNDO phase.
storage/maria/ma_check.c:
All REDOs before create_rename_lsn are ignored by Recovery. So
create_rename_lsn must be set only after all data/index has been
flushed and forced to disk. We thus move write_log_record_for_repair()
to after _ma_flush_tables_files_after_repair().
storage/maria/ma_checkpoint.c:
Checkpoint module.
storage/maria/ma_checkpoint.h:
optional argument if caller wants a thread to periodically take
checkpoints and flush dirty pages.
storage/maria/ma_create.c:
* no need to init some vars as the initial bzero(share) takes care of this.
* update to new function's name
* even if we fail in my_sync() we have to my_close()
storage/maria/ma_extra.c:
Checkpoint reads share->last_version under intern_lock, so we make
maria_extra() update it under intern_lock. THR_LOCK_maria still needed
because of _ma_test_if_reopen().
storage/maria/ma_init.c:
destroy checkpoint module when Maria shuts down.
storage/maria/ma_loghandler.c:
* UNDO_ROW_PURGE gone (see ma_blockrec.c)
* we need to remember the LSN of the LOGREC_FILE_ID for a share,
because this LSN is needed into the checkpoint record (Recovery wants
to know the validity domain of an id->name mapping)
* translog_get_horizon_no_lock() needed for Checkpoint
* comment about failing assertion (Sanja knows)
* translog_init_reader_data() thought that translog_read_record_header_scan()
returns 0 in case of error, but 0 just means "0-length header".
* translog_assign_id_to_share() now needs the MARIA_HA because
LOGREC_FILE_ID uses a log-write hook.
* Verify that (de)assignment of share->id happens only under intern_lock,
as Checkpoint reads this id with intern_lock.
* translog_purge() can accept TRANSLOG_ADDRESS, not necessarily
a real LSN.
storage/maria/ma_loghandler.h:
prototype updates
storage/maria/ma_open.c:
no need to initialize "res"
storage/maria/ma_pagecache.c:
When taking a checkpoint, we don't need to know the maximum rec_lsn
of dirty pages; this LSN was intended to be used in the two-checkpoint
rule, but last_checkpoint_lsn is as good.
4 bytes for stored_list_size is enough as PAGECACHE::blocks (number
of blocks which the pagecache can contain) is int.
storage/maria/ma_pagecache.h:
new prototype
storage/maria/ma_recovery.c:
* added replaying of REDO_RENAME_TABLE
* UNDO_ROW_PURGE gone (see ma_blockrec.c), replaced by CLR_END
* Recovery from the last checkpoint record now possible
* In new_table() we skip the table if the id->name mapping is older than
create_rename_lsn (mapping dates from lsn_of_file_id).
* in get_MARIA_HA_from_REDO_record() we skip the record
if the id->name mapping is newer than the record (can happen if processing
a record which is before the checkpoint record).
* parse_checkpoint_record() has to return a LSN, that's what caller expects
storage/maria/ma_rename.c:
new function's name; log end zeroes of tables' names (ease recovery)
storage/maria/ma_test2.c:
* equivalent of ma_test1's --test-undo added (named -u here).
* -t=1 now stops right after creating the table, so that
we can test undoing of INSERTs with duplicate keys (which tests the
CLR_END logged by _ma_write_abort_block_record()).
storage/maria/ma_test_recovery.expected:
Result of testing undoing of INSERTs with duplicate keys; there are
some differences in maria_chk -dvv but they are normal (removing
records does not shrink data/index file, does not put back the
"analyzed, optimized keys"(etc) index state.
storage/maria/ma_test_recovery:
Test undoing of INSERTs with duplicate keys, using ma_test2;
when such INSERT happens, it logs REDO_INSERT, UNDO_INSERT, REDO_DELETE,
CLR_END; we abort after that, and test that CLR_END causes recovery
to jump over UNDO_INSERT.
storage/maria/ma_write.c:
comment
storage/maria/maria_chk.c:
comment
storage/maria/maria_def.h:
* a new bit in MARIA_SHARE::in_checkpoint, used to build a list
of unique shares during Checkpoint.
* MARIA_SHARE::lsn_of_file_id added: the LSN of the last LOGREC_FILE_ID
for this share; needed to know to which LSN domain the mappings
found in the Checkpoint record apply (new mappings should not apply
to old REDOs).
storage/maria/trnman.c:
* small changes to how trnman_collect_transactions() fills its buffer;
it also uses a non-dummy lsn_read_non_atomic() found in ma_checkpoint.h
2007-09-12 11:27:34 +02:00
|
|
|
lsn_store(ptr, state->is_of_horizon); ptr+= LSN_STORE_SIZE;
|
2007-01-18 21:38:14 +02:00
|
|
|
*ptr++= (uchar)state->changed;
|
|
|
|
*ptr++= state->sortkey;
|
|
|
|
mi_rowstore(ptr,state->state.records); ptr+= 8;
|
|
|
|
mi_rowstore(ptr,state->state.del); ptr+= 8;
|
|
|
|
mi_rowstore(ptr,state->split); ptr+= 8;
|
|
|
|
mi_sizestore(ptr,state->dellink); ptr+= 8;
|
|
|
|
mi_sizestore(ptr,state->first_bitmap_with_space); ptr+= 8;
|
|
|
|
mi_sizestore(ptr,state->state.key_file_length); ptr+= 8;
|
|
|
|
mi_sizestore(ptr,state->state.data_file_length); ptr+= 8;
|
|
|
|
mi_sizestore(ptr,state->state.empty); ptr+= 8;
|
|
|
|
mi_sizestore(ptr,state->state.key_empty); ptr+= 8;
|
|
|
|
mi_int8store(ptr,state->auto_increment); ptr+= 8;
|
|
|
|
mi_int8store(ptr,(ulonglong) state->state.checksum); ptr+= 8;
|
|
|
|
mi_int4store(ptr,state->process); ptr+= 4;
|
|
|
|
mi_int4store(ptr,state->unique); ptr+= 4;
|
|
|
|
mi_int4store(ptr,state->status); ptr+= 4;
|
|
|
|
mi_int4store(ptr,state->update_count); ptr+= 4;
|
|
|
|
|
|
|
|
ptr+= state->state_diff_length;
|
2006-04-11 16:45:10 +03:00
|
|
|
|
|
|
|
for (i=0; i < keys; i++)
|
|
|
|
{
|
2007-01-18 21:38:14 +02:00
|
|
|
mi_sizestore(ptr,state->key_root[i]); ptr+= 8;
|
2006-04-11 16:45:10 +03:00
|
|
|
}
|
- WL#3072 Maria Recovery:
Recovery of state.records (the count of records which is stored into
the header of the index file). For that, state.is_of_lsn is introduced;
logic is explained in ma_recovery.c (look for "Recovery of the state").
The net gain is that in case of crash, we now recover state.records,
and it is idempotent (ma_test_recovery tests it).
state.checksum is not recovered yet, mail sent for discussion.
- WL#3071 Maria Checkpoint: preparation for it, by protecting
all modifications of the state in memory or on disk with intern_lock
(with the exception of the really-often-modified state.records,
which is now protected with the log's lock, see ma_recovery.c
(look for "Recovery of the state"). Also, if maria_close() sees that
Checkpoint is looking at this table it will not my_free() the share.
- don't compute row's checksum twice in case of UPDATE (correction
to a bugfix I made yesterday).
storage/maria/ha_maria.cc:
protect state write with intern_lock (against Checkpoint)
storage/maria/ma_blockrec.c:
* don't reset trn->rec_lsn in _ma_unpin_all_pages(), because it
should wait until we have corrected the allocation in the bitmap
(as the REDO can serve to correct the allocation during Recovery);
introducing _ma_finalize_row() for that.
* In a changeset yesterday I moved computation of the checksum
into write_block_record(), to fix a bug in UPDATE. Now I notice
that maria_update() already computes the checksum, it's just that
it puts it into info->cur_row while _ma_update_block_record()
uses info->new_row; so, removing the checksum computation from
write_block_record(), putting it back into allocate_and_write_block_record()
(which is called only by INSERT and UNDO_DELETE), and copying
cur_row->checksum into new_row->checksum in _ma_update_block_record().
storage/maria/ma_check.c:
new prototypes, they will take intern_lock when writing the state;
also take intern_lock when changing share->kfile. In both cases
this is to protect against Checkpoint reading/writing the state or reading
kfile at the same time.
Not updating create_rename_lsn directly at end of write_log_record_for_repair()
as it wouldn't have intern_lock.
storage/maria/ma_close.c:
Checkpoint builds a list of shares (under THR_LOCK_maria), then it
handles each such share (under intern_lock) (doing flushing etc);
if maria_close() freed this share between the two, Checkpoint
would see a bad pointer. To avoid this, when building the list Checkpoint
marks each share, so that maria_close() knows it should not free it
and Checkpoint will free it itself.
Extending the zone covered by intern_lock to protect against
Checkpoint reading kfile, writing state.
storage/maria/ma_create.c:
When we update create_rename_lsn, we also update is_of_lsn to
the same value: it is logical, and allows us to test in maria_open()
that the former is not bigger than the latter (the contrary is a sign
of index header corruption, or severe logging bug which hinders
Recovery, table needs a repair).
_ma_update_create_rename_lsn_on_disk() also writes is_of_lsn;
it now operates under intern_lock (protect against Checkpoint),
a shortcut function is available for cases where acquiring
intern_lock is not needed (table's creation or first open).
storage/maria/ma_delete.c:
if table is transactional, "records" is already decremented
when logging UNDO_ROW_DELETE.
storage/maria/ma_delete_all.c:
comments
storage/maria/ma_extra.c:
Protect modifications of the state, in memory and/or on disk,
with intern_lock, against a concurrent Checkpoint.
When state goes to disk, update it's is_of_lsn (by calling
the new _ma_state_info_write()).
In HA_EXTRA_FORCE_REOPEN, don't set share->changed to 0 (undoing
a change I made a few days ago) and ASK_MONTY
storage/maria/ma_locking.c:
no real code change here.
storage/maria/ma_loghandler.c:
Log-write-hooks for updating "state.records" under log's mutex
when writing/updating/deleting a row or deleting all rows.
storage/maria/ma_loghandler_lsn.h:
merge (make LSN_ERROR and LSN_REPAIRED_BY_MARIA_CHK different)
storage/maria/ma_open.c:
When opening a table verify that is_of_lsn >= create_rename_lsn; if
false the header must be corrupted.
_ma_state_info_write() is split in two: _ma_state_info_write_sub()
which is the old _ma_state_info_write(), and _ma_state_info_write()
which additionally takes intern_lock if requested (to protect
against Checkpoint) and updates is_of_lsn.
_ma_open_keyfile() should change kfile.file under intern_lock
to protect Checkpoint from reading a wrong kfile.file.
storage/maria/ma_recovery.c:
Recovery of state.records: when the REDO phase sees UNDO_ROW_INSERT
which has a LSN > state.is_of_lsn it increments state.records.
Same for UNDO_ROW_DELETE and UNDO_ROW_PURGE.
When closing a table during Recovery, we know its state is at least
as new as the current log record we are looking at, so increase
is_of_lsn to the LSN of the current log record.
storage/maria/ma_rename.c:
update for new behaviour of _ma_update_create_rename_lsn_on_disk().
storage/maria/ma_test1.c:
update to new prototype
storage/maria/ma_test2.c:
update to new prototype (actually prototype was changed days ago,
but compiler does not complain about the extra argument??)
storage/maria/ma_test_recovery.expected:
new result file of ma_test_recovery. Improvements: record
count read from index's header is now always correct.
storage/maria/ma_test_recovery:
"rm" fails if file does not exist. Redirect stderr of script.
storage/maria/ma_write.c:
if table is transactional, "records" is already incremented when
logging UNDO_ROW_INSERT. Comments.
storage/maria/maria_chk.c:
update is_of_lsn too
storage/maria/maria_def.h:
- MARIA_STATE_INFO::is_of_lsn which is used by Recovery. It is stored
into the index file's header.
- Checkpoint can now mark a table as "don't free this", and maria_close()
can reply "ok then you will free it".
- new functions
storage/maria/maria_pack.c:
update for new name
2007-09-07 15:02:30 +02:00
|
|
|
/** @todo RECOVERY BUG key_del is a problem for recovery */
|
2007-01-18 21:38:14 +02:00
|
|
|
mi_sizestore(ptr,state->key_del); ptr+= 8;
|
|
|
|
if (pWrite & 2) /* From maria_chk */
|
2006-04-11 16:45:10 +03:00
|
|
|
{
|
|
|
|
uint key_parts= mi_uint2korr(state->header.key_parts);
|
2007-01-18 21:38:14 +02:00
|
|
|
mi_int4store(ptr,state->sec_index_changed); ptr+= 4;
|
|
|
|
mi_int4store(ptr,state->sec_index_used); ptr+= 4;
|
|
|
|
mi_int4store(ptr,state->version); ptr+= 4;
|
|
|
|
mi_int8store(ptr,state->key_map); ptr+= 8;
|
|
|
|
mi_int8store(ptr,(ulonglong) state->create_time); ptr+= 8;
|
|
|
|
mi_int8store(ptr,(ulonglong) state->recover_time); ptr+= 8;
|
|
|
|
mi_int8store(ptr,(ulonglong) state->check_time); ptr+= 8;
|
|
|
|
mi_sizestore(ptr,state->rec_per_key_rows); ptr+= 8;
|
2006-04-11 16:45:10 +03:00
|
|
|
for (i=0 ; i < key_parts ; i++)
|
|
|
|
{
|
2007-01-18 21:38:14 +02:00
|
|
|
mi_int4store(ptr,state->rec_per_key_part[i]); ptr+=4;
|
2006-04-11 16:45:10 +03:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
WL#3072 Maria recovery
* create page cache before initializing engine and not after, because
Maria's recovery needs a page cache
* make the creation of a bitmap page more crash-resistent
* bugfix (see ma_blockrec.c)
* back to old way: create an 8k bitmap page when creating table
* preparations for the UNDO phase: recreate TRNs
* preparations for Checkpoint: list of dirty pages, testing
of rec_lsn to know if page should be skipped during Recovery
(unused in this patch as no Checkpoint module pushed yet)
* maria_chk tags repaired table with a special LSN
* reworking all around in ma_recovery.c (less duplication)
mysys/my_realloc.c:
noted an issue in my_realloc()
sql/mysqld.cc:
page cache needs to be created before engines are initialized,
because Maria's initialization may do a recovery which needs
the page cache.
storage/maria/ha_maria.cc:
update to new prototype
storage/maria/ma_bitmap.c:
when creating the first bitmap page we used chsize to 8192 bytes then
pwrite (overwrite) the last 2 bytes (8191-8192). If crash between
the two operations, this leaves a bitmap page full without its end
marker. A later recovery may try to read this page and find it
exists and misses a marker and conclude it's corrupted and fail.
Changing the chsize to only 8190 bytes: recovery will then find
the page is too short and recreate it entirely.
storage/maria/ma_blockrec.c:
Fix for a bug: when executing a REDO, if the data page is created,
data_file_length was increased before _ma_bitmap_set():
_ma_bitmap_set() called _ma_read_bitmap_page() which, due to the
increased data_file_length, expected to find a bitmap page on disk
with a correct end marker; if the bitmap page didn't exist already
in fact, this failed. Fixed by increasing data_file_length only after
_ma_read_bitmap_page() has created the new bitmap page correctly.
This bug could happen every time a REDO is about creating a new
bitmap page.
storage/maria/ma_check.c:
empty data file has a bitmap page
storage/maria/ma_control_file.c:
useless parameter to ma_control_file_create_or_open(), just
test if this is recovery.
storage/maria/ma_control_file.h:
new prototype
storage/maria/ma_create.c:
Back to how it was before: maria_create() creates an 8k bitmap page.
Thus (bugfix) data_file_length needs to reflect this instead of being 0.
storage/maria/ma_loghandler.c:
as ma_test1 and ma_test2 now use real transactions and not
dummy_transaction_object, REDO for INSERT/UPDATE/DELETE are always
about real transactions, can assert this.
A function for Recovery to assign a short id to a table.
storage/maria/ma_loghandler.h:
new function
storage/maria/ma_loghandler_lsn.h:
maria_chk tags repaired tables with this LSN
storage/maria/ma_open.c:
* enforce that DMLs on transactional tables use real transactions
and not dummy_transaction_object.
* test if table was repaired with maria_chk (which has to been
seen as an import of an external table into the server), test
validity of create_rename_lsn (header corruption detection)
* comments.
storage/maria/ma_recovery.c:
* preparations for the UNDO phase: recreate TRNs
* preparations for Checkpoint: list of dirty pages, testing
of rec_lsn to know if page should be skipped during Recovery
(unused in this patch as no Checkpoint module pushed yet)
* reworking all around (less duplication)
storage/maria/ma_recovery.h:
a parameter to say if the UNDO phase should be skipped
storage/maria/maria_chk.c:
tag repaired tables with a special LSN
storage/maria/maria_read_log.c:
* update to new prototype
* no UNDO phase in maria_read_log for now
storage/maria/trnman.c:
* a function for Recovery to create a transaction (TRN), needed
in the UNDO phase
* a function for Recovery to grab an existing transaction, needed
in the UNDO phase (rollback all existing transactions)
storage/maria/trnman_public.h:
new functions
2007-08-29 16:43:01 +02:00
|
|
|
res= (pWrite & 1) ?
|
|
|
|
my_pwrite(file, buff, (size_t) (ptr-buff), 0L,
|
|
|
|
MYF(MY_NABP | MY_THREADSAFE)) :
|
|
|
|
my_write(file, buff, (size_t) (ptr-buff),
|
|
|
|
MYF(MY_NABP));
|
|
|
|
DBUG_RETURN(res != 0);
|
2006-04-11 16:45:10 +03:00
|
|
|
}
|
|
|
|
|
|
|
|
|
- speed optimization:
minimize writes to transactional Maria tables: don't write
data pages, state, and open_count at the end of each statement.
Data pages will be written by a background thread periodically.
State will be written by Checkpoint periodically.
open_count serves to detect when a table is potentially damaged
due to an unclean mysqld stop, but thanks to recovery an unclean
mysqld stop will be corrected and so open_count becomes useless.
As state is written less often, it is often obsolete on disk,
we thus should avoid to read it from disk.
- by removing the data page writes above, it is necessary to put
it back at the start of some statements like check, repair and
delete_all. It was already necessary in fact (see ma_delete_all.c).
- disabling CACHE INDEX on Maria tables for now (fixes crash
of test 'key_cache' when run with --default-storage-engine=maria).
- correcting some fishy code in maria_extra.c (we possibly could lose
index pages when doing a DROP TABLE under Windows, in theory).
storage/maria/ha_maria.cc:
disable CACHE INDEX in Maria for now (there is a single cache for now),
it crashes and it's not a priority
storage/maria/ma_bitmap.c:
debug message
storage/maria/ma_check.c:
The statement before maria_repair() may not flush state,
so it needs to be done by maria_repair() (indeed this function
uses maria_open(HA_OPEN_COPY) so reads state from disk,
so needs to find it up-to-date on disk).
For safety (but normally this is not needed) we remove index blocks
out of the cache before repairing.
_ma_flush_blocks() becomes _ma_flush_table_files_after_repair():
it now additionally flushes the data file and state and syncs files.
As a side effect, the assertion "no WRITE_CACHE_USED" from
_ma_flush_table_files() fired so we move all end_io_cache() done
at the end of repair to before the calls to _ma_flush_table_files_after_repair().
storage/maria/ma_close.c:
when closing a transactional table, we fsync it. But we need to
do this only after writing its state.
We need to write the state at close time only for transactional
tables (the other tables do that at last unlock).
Putting back the O_RDONLY||crashed condition which I had
removed earlier.
Unmap the file before syncing it (does not matter now as Maria
does not use mmap)
storage/maria/ma_delete_all.c:
need to flush data pages before chsize-ing it. Was needed even when
we flushed data pages at the end of each statement, because we didn't
anyway do it if under LOCK TABLES: the change here thus fixes this bug:
create table t(a int) engine=maria;lock tables t write;
insert into t values(1);delete from t;unlock tables;check table t;
"Size of datafile is: 16384 Should be: 8192"
(an obsolete page went to disk after the chsize(), at unlock time).
storage/maria/ma_extra.c:
When doing share->last_version=0, we make the MARIA_SHARE-in-memory
invisible to future openers, so need to have an up-to-date state
on disk for them. The same way, future openers will reopen the data
and index file, so they will not find our cached blocks, so we
need to flush them to disk.
In HA_EXTRA_FORCE_REOPEN, this probably happens naturally as all
tables normally get closed, we however add a safety flush.
In HA_EXTRA_PREPARE_FOR_RENAME, we need to do the flushing. On
Windows we additionally need to close files.
In HA_EXTRA_PREPARE_FOR_DROP, we don't need to flush anything but
remove dirty cached blocks from memory. On Windows we need to close
files.
Closing files forces us to sync them before (requirement for transactional
tables).
For mutex reasons (don't lock intern_lock twice), we move
maria_lock_database() and _ma_decrement_open_count() first in the list
of operations.
Flush also data file in HA_EXTRA_FLUSH.
storage/maria/ma_locking.c:
For transactional tables:
- don't write data pages / state at unlock time;
as a consequence, "share->changed=0" cannot be done.
- don't write state in _ma_writeinfo()
- don't maintain open_count on disk (Recovery corrects the table in case of crash
anyway, and we gain speed by not writing open_count to disk),
For non-transactional tables, flush the state at unlock only
if the table was changed (optimization).
Code which read the state from disk is relevant only with
external locking, we disable it (if want to re-enable it, it shouldn't
for transactional tables as state on disk may be obsolete (such tables
does not flush state at unlock anymore).
The comment "We have to flush the write cache" is now wrong because
maria_lock_database(F_UNLCK) now happens before thr_unlock(), and
we are not using external locking.
storage/maria/ma_open.c:
_ma_state_info_read() is only used in ma_open.c, making it static
storage/maria/ma_recovery.c:
set MARIA_SHARE::changed to TRUE when we are going to apply a
REDO/UNDO, so that the state gets flushed at close.
storage/maria/ma_test_recovery.expected:
Changes introduced by this patch:
- good: the "open" (table open, not properly closed) is gone,
it was pointless for a recovered table
- bad: stemming from different moments of writing the index's state
probably (_ma_writeinfo() used to write the state after every row
write in ma_test* programs, doesn't anymore as the table is
transactional): some differences in indexes (not relevant as we don't
yet have recovery for them); some differences in count of records
(changed from a wrong value to another wrong value) (not relevant
as we don't recover this count correctly yet anyway, though
a patch will be pushed soon).
storage/maria/ma_test_recovery:
for repeatable output, no names of varying directories.
storage/maria/maria_chk.c:
function renamed
storage/maria/maria_def.h:
Function became local to ma_open.c. Function renamed.
2007-09-06 16:53:26 +02:00
|
|
|
static uchar *_ma_state_info_read(uchar *ptr, MARIA_STATE_INFO *state)
|
2006-04-11 16:45:10 +03:00
|
|
|
{
|
2007-01-18 21:38:14 +02:00
|
|
|
uint i,keys,key_parts;
|
2006-04-11 16:45:10 +03:00
|
|
|
memcpy_fixed(&state->header,ptr, sizeof(state->header));
|
2007-01-18 21:38:14 +02:00
|
|
|
ptr+= sizeof(state->header);
|
|
|
|
keys= (uint) state->header.keys;
|
|
|
|
key_parts= mi_uint2korr(state->header.key_parts);
|
|
|
|
|
|
|
|
state->open_count = mi_uint2korr(ptr); ptr+= 2;
|
- WL#3239 "log CREATE TABLE in Maria"
- WL#3240 "log DROP TABLE in Maria"
- similarly, log RENAME TABLE, REPAIR/OPTIMIZE TABLE, and
DELETE no_WHERE_clause (== the DELETE which just truncates the files)
- create_rename_lsn added to MARIA_SHARE's state
- all these operations (except DROP TABLE) also update the table's
create_rename_lsn, which is needed for the correctness of
Recovery (see function comment of _ma_repair_write_log_record()
in ma_check.c)
- write a COMMIT record when transaction commits.
- don't log REDOs/UNDOs if this is an internal temporary table
like inside ALTER TABLE (I expect this to be a big win). There was
already no logging for user-created "CREATE TEMPORARY" tables.
- don't fsync files/directories if the table is not transactional
- in translog_write_record(), autogenerate a 2-byte-id for the table
and log the "id->name" pair (LOGREC_FILE_ID); log
LOGREC_LONG_TRANSACTION_ID; automatically store
the table's 2-byte-id in any log record.
- preparations for Checkpoint: translog_get_horizon(); pausing Checkpoint
when some dirty pages are unknown; capturing trn->rec_lsn,
trn->first_undo_lsn for Checkpoint and log's low-water-mark computing.
- assertions, comments.
storage/maria/Makefile.am:
more files to build
storage/maria/ha_maria.cc:
- logging a REPAIR log record if REPAIR/OPTIMIZE was successful.
- ha_maria::data_file_type does not have to be set in every info()
call, just do it once in open().
- if caller said that transactionality can be disabled (like if
caller is ALTER TABLE) i.e. thd->transaction.on==FALSE, then we
temporarily disable transactionality of the table in external_lock();
that will ensure that no REDOs/UNDOs are logged for this possibly
massive write operation (they are not needed, as if any write fails,
the table will be dropped). We re-enable in external_lock(F_UNLCK),
which in ALTER TABLE happens before the tmp table replaces the original
one (which is good, as thus the final table will have a REDO RENAME
and a correct create_rename_lsn).
- when we commit we also have to write a log record, so
trnman_commit_trn() calls become ma_commit() calls
- at end of engine's initialization, we are potentially entering a
multi-threaded dangerous world (clients are going to be accepted)
and so some assertions of mutex-owning become enforceable, for that
we set maria_multi_threaded=TRUE (see ma_control_file.c)
storage/maria/ha_maria.h:
new member ha_maria::save_transactional (see also ha_maria.cc)
storage/maria/ma_blockrec.c:
- fixing comments according to discussion with Monty
- if a table is transactional but temporarily non-transactional
(like in ALTER TABLE), we need to give a sensible LSN to the pages
(and, if we give 0, pagecache asserts).
- translog_write_record() now takes care of storing the share's
2-byte-id in the log record
storage/maria/ma_blockrec.h:
fixing comment according to discussion with Monty
storage/maria/ma_check.c:
When REPAIR/OPTIMIZE modify the data/index file, if this is a
transactional table, they must sync it; if they remove files or rename
files, they must sync the directory, so that everything is durable.
This is just applying to REPAIR/OPTIMIZE the logic already implemented
in CREATE/DROP/RENAME a few months ago.
Adding a function to write a LOGREC_REPAIR_TABLE at end of
REPAIR/OPTIMIZE (called only by ha_maria, not by maria_chk), and
to update the table's create_rename_lsn.
storage/maria/ma_close.c:
fix for a future bug
storage/maria/ma_control_file.c:
ensuring that if Maria is running in multi-threaded mode, anybody
wanting to write to the control file and update
last_checkpoint_lsn/last_logno owns the log's lock.
storage/maria/ma_control_file.h:
see ma_control_file.c
storage/maria/ma_create.c:
when creating a table:
- sync it and its directory only if this is a transactional table
and there is a log (no point in syncing in maria_chk)
- decouple the two uses of linkname/linkname_ptr (for index file and
for data file) into more variables, as we need to know all links
until the moment we write the LOGREC_CREATE_TABLE.
- set share.data_file_type early so that _ma_initialize_data_file()
knows it (Monty's bugfix so that a table always has at least a bitmap
page when it is created; so data-file is not 0 bytes anymore).
- log a LOGREC_CREATE_TABLE; it contains the bytes which we have
just written to the index file's header. Update table's
create_rename_lsn.
- syncing of kfile had been bugified in a previous merge, correcting
- syncing of dfile is now needed as it's not empty anymore
- in _ma_initialize_data_file(), use share's block_size and not the
global one. This is a gratuitous change, both variables are equal,
just that I find it more future-proof to use share-bound variable
rather than global one.
storage/maria/ma_delete_all.c:
log a LOGREC_DELETE_ALL record when doing ma_delete_all_rows();
update create_rename_lsn then.
storage/maria/ma_delete_table.c:
- logging LOGREC_DROP_TABLE; knowing if this is needed, requires
knowing if the table is transactional, which requires opening the
table.
- we need to sync directories only if the table is transactional
storage/maria/ma_extra.c:
questions
storage/maria/ma_init.c:
when maria_end() is called, engine is not multithreaded
storage/maria/ma_loghandler.c:
- translog_inited has to be visible to ma_create() (see how it is used
in ma_create())
- checkpoint record will be a single record, not three
- no REDO for TRUNCATE (TRUNCATE calls ma_create() internally so will
log a REDO_CREATE)
- adding REDO for DELETE no_WHERE_clause (fast DELETE of all rows by
truncating the files), REPAIR.
- MY_WAIT_IF_FULL to wait&retry if a log write hits a full disk
- in translog_write_record(), if MARIA_SHARE does not yet have a
2-byte-id, generate one for it and log LOGREC_FILE_ID; automatically
store this short id into log records.
- in translog_write_record(), if transaction has not logged its
long trid, log LOGREC_LONG_TRANSACTION_ID.
- For Checkpoint, we need to know the current end-of-log: adding
translog_get_horizon().
- For Control File, adding an assertion that the thread owns the
log's lock (control file is protected by this lock)
storage/maria/ma_loghandler.h:
Changes in log records (see ma_loghandler.c).
new prototypes, new functions.
storage/maria/ma_loghandler_lsn.h:
adding a type LSN_WITH_FLAGS especially for TRN::first_undo_lsn,
where the most significant byte is used for flags.
storage/maria/ma_open.c:
storing the create_rename_lsn in the index file's header (in the
state, precisely) and retrieving it from there.
storage/maria/ma_pagecache.c:
- my set_if_bigger was wrong, correcting it
- if the first_in_switch list is not empty, it means that
changed_blocks misses some dirty pages, so Checkpoint cannot run and
needs to wait. A variable missing_blocks_in_changed_list is added to
tell that (should it be named missing_blocks_in_changed_blocks?)
- pagecache_collect_changed_blocks_with_lsn() now also tells the
minimum rec_lsn (needed for low-water mark computation).
storage/maria/ma_pagecache.h:
see ma_pagecache.c
storage/maria/ma_panic.c:
comment
storage/maria/ma_range.c:
comment
storage/maria/ma_rename.c:
- logging LOGREC_RENAME_TABLE; knowing if this is needed, requires
knowing if the table is transactional, which requires opening the
table.
- update create_rename_lsn
- we need to sync directories only if the table is transactional
storage/maria/ma_static.c:
comment
storage/maria/ma_test_all.sh:
- tip for Valgrind-ing ma_test_all
- do "export maria_path=somepath" before calling ma_test_all,
if you want to run ma_test_all out of storage/maria (useful
to have parallel runs, like one normal and one Valgrind, they
must not use the same tables so need to run in different directories)
storage/maria/maria_def.h:
- state now contains, in memory and on disk, the create_rename_lsn
- share now contains a 2-byte-id
storage/maria/trnman.c:
preparations for Checkpoint: capture trn->rec_lsn, trn->first_undo_lsn;
minimum first_undo_lsn needed to know log's low-water-mark
storage/maria/trnman.h:
using most significant byte of first_undo_lsn to hold miscellaneous
flags, for now TRANSACTION_LOGGED_LONG_ID.
dummy_transaction_object is already declared in ma_static.c.
storage/maria/trnman_public.h:
dummy_transaction_object was declared in all files including
trnman_public.h, while in fact it's a single object.
new prototype
storage/maria/unittest/ma_test_loghandler-t.c:
update for new prototype
storage/maria/unittest/ma_test_loghandler_multigroup-t.c:
update for new prototype
storage/maria/unittest/ma_test_loghandler_multithread-t.c:
update for new prototype
storage/maria/unittest/ma_test_loghandler_pagecache-t.c:
update for new prototype
storage/maria/ma_commit.c:
function which wraps:
- writing a LOGREC_COMMIT record (==commit on disk)
- calling trnman_commit_trn() (=commit in memory)
storage/maria/ma_commit.h:
new header file
.tree-is-private:
this file is now needed to keep our tree private (don't push it
to public trees). When 5.1 is merged into mysql-maria, we can abandon
our maria-specific post-commit trigger; .tree_is_private will take
care of keeping commit mails private. Don't push this file to public
trees.
2007-06-22 14:49:37 +02:00
|
|
|
state->create_rename_lsn= lsn_korr(ptr); ptr+= LSN_STORE_SIZE;
|
WL#3071 Maria checkpoint
Finally this is the real checkpoint code.
It however exhibits unstabilities when a checkpoint runs concurrently
with data-modifying clients (table corruption, transaction log's
assertions) so for now a checkpoint is taken only at startup after
recovery and at shutdown, i.e. not in concurrent situations. Later
we will let it run periodically, as well as flush dirty pages
periodically (almost all needed code is there already, only pagecache
code is written but not committed).
WL#3072 Maria recovery
* replacing UNDO_ROW_PURGE with CLR_END; testing of those CLR_END via
ma_test2 which has INSERTs failing with duplicate keys.
* replaying of REDO_RENAME_TABLE
Now, off to test Recovery in ha_maria :)
BitKeeper/deleted/.del-ma_least_recently_dirtied.c:
Delete: storage/maria/ma_least_recently_dirtied.c
BitKeeper/deleted/.del-ma_least_recently_dirtied.h:
Delete: storage/maria/ma_least_recently_dirtied.h
storage/maria/Makefile.am:
compile Checkpoint module
storage/maria/ha_maria.cc:
When ha_maria starts, do a recovery from last checkpoint.
Take a checkpoint when that recovery has ended and when ha_maria
shuts down cleanly.
storage/maria/ma_blockrec.c:
* even if my_sync() fails we have to my_close() (otherwise we leak
a descriptor)
* UNDO_ROW_PURGE is replaced by a simple CLR_END for UNDO_ROW_INSERT,
as promised in the old comment; it gives us skipping during the
UNDO phase.
storage/maria/ma_check.c:
All REDOs before create_rename_lsn are ignored by Recovery. So
create_rename_lsn must be set only after all data/index has been
flushed and forced to disk. We thus move write_log_record_for_repair()
to after _ma_flush_tables_files_after_repair().
storage/maria/ma_checkpoint.c:
Checkpoint module.
storage/maria/ma_checkpoint.h:
optional argument if caller wants a thread to periodically take
checkpoints and flush dirty pages.
storage/maria/ma_create.c:
* no need to init some vars as the initial bzero(share) takes care of this.
* update to new function's name
* even if we fail in my_sync() we have to my_close()
storage/maria/ma_extra.c:
Checkpoint reads share->last_version under intern_lock, so we make
maria_extra() update it under intern_lock. THR_LOCK_maria still needed
because of _ma_test_if_reopen().
storage/maria/ma_init.c:
destroy checkpoint module when Maria shuts down.
storage/maria/ma_loghandler.c:
* UNDO_ROW_PURGE gone (see ma_blockrec.c)
* we need to remember the LSN of the LOGREC_FILE_ID for a share,
because this LSN is needed into the checkpoint record (Recovery wants
to know the validity domain of an id->name mapping)
* translog_get_horizon_no_lock() needed for Checkpoint
* comment about failing assertion (Sanja knows)
* translog_init_reader_data() thought that translog_read_record_header_scan()
returns 0 in case of error, but 0 just means "0-length header".
* translog_assign_id_to_share() now needs the MARIA_HA because
LOGREC_FILE_ID uses a log-write hook.
* Verify that (de)assignment of share->id happens only under intern_lock,
as Checkpoint reads this id with intern_lock.
* translog_purge() can accept TRANSLOG_ADDRESS, not necessarily
a real LSN.
storage/maria/ma_loghandler.h:
prototype updates
storage/maria/ma_open.c:
no need to initialize "res"
storage/maria/ma_pagecache.c:
When taking a checkpoint, we don't need to know the maximum rec_lsn
of dirty pages; this LSN was intended to be used in the two-checkpoint
rule, but last_checkpoint_lsn is as good.
4 bytes for stored_list_size is enough as PAGECACHE::blocks (number
of blocks which the pagecache can contain) is int.
storage/maria/ma_pagecache.h:
new prototype
storage/maria/ma_recovery.c:
* added replaying of REDO_RENAME_TABLE
* UNDO_ROW_PURGE gone (see ma_blockrec.c), replaced by CLR_END
* Recovery from the last checkpoint record now possible
* In new_table() we skip the table if the id->name mapping is older than
create_rename_lsn (mapping dates from lsn_of_file_id).
* in get_MARIA_HA_from_REDO_record() we skip the record
if the id->name mapping is newer than the record (can happen if processing
a record which is before the checkpoint record).
* parse_checkpoint_record() has to return a LSN, that's what caller expects
storage/maria/ma_rename.c:
new function's name; log end zeroes of tables' names (ease recovery)
storage/maria/ma_test2.c:
* equivalent of ma_test1's --test-undo added (named -u here).
* -t=1 now stops right after creating the table, so that
we can test undoing of INSERTs with duplicate keys (which tests the
CLR_END logged by _ma_write_abort_block_record()).
storage/maria/ma_test_recovery.expected:
Result of testing undoing of INSERTs with duplicate keys; there are
some differences in maria_chk -dvv but they are normal (removing
records does not shrink data/index file, does not put back the
"analyzed, optimized keys"(etc) index state.
storage/maria/ma_test_recovery:
Test undoing of INSERTs with duplicate keys, using ma_test2;
when such INSERT happens, it logs REDO_INSERT, UNDO_INSERT, REDO_DELETE,
CLR_END; we abort after that, and test that CLR_END causes recovery
to jump over UNDO_INSERT.
storage/maria/ma_write.c:
comment
storage/maria/maria_chk.c:
comment
storage/maria/maria_def.h:
* a new bit in MARIA_SHARE::in_checkpoint, used to build a list
of unique shares during Checkpoint.
* MARIA_SHARE::lsn_of_file_id added: the LSN of the last LOGREC_FILE_ID
for this share; needed to know to which LSN domain the mappings
found in the Checkpoint record apply (new mappings should not apply
to old REDOs).
storage/maria/trnman.c:
* small changes to how trnman_collect_transactions() fills its buffer;
it also uses a non-dummy lsn_read_non_atomic() found in ma_checkpoint.h
2007-09-12 11:27:34 +02:00
|
|
|
state->is_of_horizon= lsn_korr(ptr); ptr+= LSN_STORE_SIZE;
|
2007-01-18 21:38:14 +02:00
|
|
|
state->changed= (my_bool) *ptr++;
|
|
|
|
state->sortkey= (uint) *ptr++;
|
|
|
|
state->state.records= mi_rowkorr(ptr); ptr+= 8;
|
|
|
|
state->state.del = mi_rowkorr(ptr); ptr+= 8;
|
|
|
|
state->split = mi_rowkorr(ptr); ptr+= 8;
|
|
|
|
state->dellink= mi_sizekorr(ptr); ptr+= 8;
|
|
|
|
state->first_bitmap_with_space= mi_sizekorr(ptr); ptr+= 8;
|
|
|
|
state->state.key_file_length = mi_sizekorr(ptr); ptr+= 8;
|
|
|
|
state->state.data_file_length= mi_sizekorr(ptr); ptr+= 8;
|
|
|
|
state->state.empty = mi_sizekorr(ptr); ptr+= 8;
|
|
|
|
state->state.key_empty= mi_sizekorr(ptr); ptr+= 8;
|
|
|
|
state->auto_increment=mi_uint8korr(ptr); ptr+= 8;
|
|
|
|
state->state.checksum=(ha_checksum) mi_uint8korr(ptr);ptr+= 8;
|
|
|
|
state->process= mi_uint4korr(ptr); ptr+= 4;
|
|
|
|
state->unique = mi_uint4korr(ptr); ptr+= 4;
|
|
|
|
state->status = mi_uint4korr(ptr); ptr+= 4;
|
|
|
|
state->update_count=mi_uint4korr(ptr); ptr+= 4;
|
2006-04-11 16:45:10 +03:00
|
|
|
|
|
|
|
ptr+= state->state_diff_length;
|
|
|
|
|
|
|
|
for (i=0; i < keys; i++)
|
|
|
|
{
|
2007-01-18 21:38:14 +02:00
|
|
|
state->key_root[i]= mi_sizekorr(ptr); ptr+= 8;
|
2006-04-11 16:45:10 +03:00
|
|
|
}
|
2007-01-18 21:38:14 +02:00
|
|
|
state->key_del= mi_sizekorr(ptr); ptr+= 8;
|
|
|
|
state->sec_index_changed = mi_uint4korr(ptr); ptr+= 4;
|
|
|
|
state->sec_index_used = mi_uint4korr(ptr); ptr+= 4;
|
|
|
|
state->version = mi_uint4korr(ptr); ptr+= 4;
|
|
|
|
state->key_map = mi_uint8korr(ptr); ptr+= 8;
|
|
|
|
state->create_time = (time_t) mi_sizekorr(ptr); ptr+= 8;
|
|
|
|
state->recover_time =(time_t) mi_sizekorr(ptr); ptr+= 8;
|
|
|
|
state->check_time = (time_t) mi_sizekorr(ptr); ptr+= 8;
|
|
|
|
state->rec_per_key_rows=mi_sizekorr(ptr); ptr+= 8;
|
2006-04-11 16:45:10 +03:00
|
|
|
for (i=0 ; i < key_parts ; i++)
|
|
|
|
{
|
2007-01-18 21:38:14 +02:00
|
|
|
state->rec_per_key_part[i]= mi_uint4korr(ptr); ptr+=4;
|
2006-04-11 16:45:10 +03:00
|
|
|
}
|
|
|
|
return ptr;
|
|
|
|
}
|
|
|
|
|
|
|
|
|
WL#3072 Maria recovery
* create page cache before initializing engine and not after, because
Maria's recovery needs a page cache
* make the creation of a bitmap page more crash-resistent
* bugfix (see ma_blockrec.c)
* back to old way: create an 8k bitmap page when creating table
* preparations for the UNDO phase: recreate TRNs
* preparations for Checkpoint: list of dirty pages, testing
of rec_lsn to know if page should be skipped during Recovery
(unused in this patch as no Checkpoint module pushed yet)
* maria_chk tags repaired table with a special LSN
* reworking all around in ma_recovery.c (less duplication)
mysys/my_realloc.c:
noted an issue in my_realloc()
sql/mysqld.cc:
page cache needs to be created before engines are initialized,
because Maria's initialization may do a recovery which needs
the page cache.
storage/maria/ha_maria.cc:
update to new prototype
storage/maria/ma_bitmap.c:
when creating the first bitmap page we used chsize to 8192 bytes then
pwrite (overwrite) the last 2 bytes (8191-8192). If crash between
the two operations, this leaves a bitmap page full without its end
marker. A later recovery may try to read this page and find it
exists and misses a marker and conclude it's corrupted and fail.
Changing the chsize to only 8190 bytes: recovery will then find
the page is too short and recreate it entirely.
storage/maria/ma_blockrec.c:
Fix for a bug: when executing a REDO, if the data page is created,
data_file_length was increased before _ma_bitmap_set():
_ma_bitmap_set() called _ma_read_bitmap_page() which, due to the
increased data_file_length, expected to find a bitmap page on disk
with a correct end marker; if the bitmap page didn't exist already
in fact, this failed. Fixed by increasing data_file_length only after
_ma_read_bitmap_page() has created the new bitmap page correctly.
This bug could happen every time a REDO is about creating a new
bitmap page.
storage/maria/ma_check.c:
empty data file has a bitmap page
storage/maria/ma_control_file.c:
useless parameter to ma_control_file_create_or_open(), just
test if this is recovery.
storage/maria/ma_control_file.h:
new prototype
storage/maria/ma_create.c:
Back to how it was before: maria_create() creates an 8k bitmap page.
Thus (bugfix) data_file_length needs to reflect this instead of being 0.
storage/maria/ma_loghandler.c:
as ma_test1 and ma_test2 now use real transactions and not
dummy_transaction_object, REDO for INSERT/UPDATE/DELETE are always
about real transactions, can assert this.
A function for Recovery to assign a short id to a table.
storage/maria/ma_loghandler.h:
new function
storage/maria/ma_loghandler_lsn.h:
maria_chk tags repaired tables with this LSN
storage/maria/ma_open.c:
* enforce that DMLs on transactional tables use real transactions
and not dummy_transaction_object.
* test if table was repaired with maria_chk (which has to been
seen as an import of an external table into the server), test
validity of create_rename_lsn (header corruption detection)
* comments.
storage/maria/ma_recovery.c:
* preparations for the UNDO phase: recreate TRNs
* preparations for Checkpoint: list of dirty pages, testing
of rec_lsn to know if page should be skipped during Recovery
(unused in this patch as no Checkpoint module pushed yet)
* reworking all around (less duplication)
storage/maria/ma_recovery.h:
a parameter to say if the UNDO phase should be skipped
storage/maria/maria_chk.c:
tag repaired tables with a special LSN
storage/maria/maria_read_log.c:
* update to new prototype
* no UNDO phase in maria_read_log for now
storage/maria/trnman.c:
* a function for Recovery to create a transaction (TRN), needed
in the UNDO phase
* a function for Recovery to grab an existing transaction, needed
in the UNDO phase (rollback all existing transactions)
storage/maria/trnman_public.h:
new functions
2007-08-29 16:43:01 +02:00
|
|
|
/**
|
|
|
|
@brief Fills the state by reading its copy on disk.
|
|
|
|
|
- speed optimization:
minimize writes to transactional Maria tables: don't write
data pages, state, and open_count at the end of each statement.
Data pages will be written by a background thread periodically.
State will be written by Checkpoint periodically.
open_count serves to detect when a table is potentially damaged
due to an unclean mysqld stop, but thanks to recovery an unclean
mysqld stop will be corrected and so open_count becomes useless.
As state is written less often, it is often obsolete on disk,
we thus should avoid to read it from disk.
- by removing the data page writes above, it is necessary to put
it back at the start of some statements like check, repair and
delete_all. It was already necessary in fact (see ma_delete_all.c).
- disabling CACHE INDEX on Maria tables for now (fixes crash
of test 'key_cache' when run with --default-storage-engine=maria).
- correcting some fishy code in maria_extra.c (we possibly could lose
index pages when doing a DROP TABLE under Windows, in theory).
storage/maria/ha_maria.cc:
disable CACHE INDEX in Maria for now (there is a single cache for now),
it crashes and it's not a priority
storage/maria/ma_bitmap.c:
debug message
storage/maria/ma_check.c:
The statement before maria_repair() may not flush state,
so it needs to be done by maria_repair() (indeed this function
uses maria_open(HA_OPEN_COPY) so reads state from disk,
so needs to find it up-to-date on disk).
For safety (but normally this is not needed) we remove index blocks
out of the cache before repairing.
_ma_flush_blocks() becomes _ma_flush_table_files_after_repair():
it now additionally flushes the data file and state and syncs files.
As a side effect, the assertion "no WRITE_CACHE_USED" from
_ma_flush_table_files() fired so we move all end_io_cache() done
at the end of repair to before the calls to _ma_flush_table_files_after_repair().
storage/maria/ma_close.c:
when closing a transactional table, we fsync it. But we need to
do this only after writing its state.
We need to write the state at close time only for transactional
tables (the other tables do that at last unlock).
Putting back the O_RDONLY||crashed condition which I had
removed earlier.
Unmap the file before syncing it (does not matter now as Maria
does not use mmap)
storage/maria/ma_delete_all.c:
need to flush data pages before chsize-ing it. Was needed even when
we flushed data pages at the end of each statement, because we didn't
anyway do it if under LOCK TABLES: the change here thus fixes this bug:
create table t(a int) engine=maria;lock tables t write;
insert into t values(1);delete from t;unlock tables;check table t;
"Size of datafile is: 16384 Should be: 8192"
(an obsolete page went to disk after the chsize(), at unlock time).
storage/maria/ma_extra.c:
When doing share->last_version=0, we make the MARIA_SHARE-in-memory
invisible to future openers, so need to have an up-to-date state
on disk for them. The same way, future openers will reopen the data
and index file, so they will not find our cached blocks, so we
need to flush them to disk.
In HA_EXTRA_FORCE_REOPEN, this probably happens naturally as all
tables normally get closed, we however add a safety flush.
In HA_EXTRA_PREPARE_FOR_RENAME, we need to do the flushing. On
Windows we additionally need to close files.
In HA_EXTRA_PREPARE_FOR_DROP, we don't need to flush anything but
remove dirty cached blocks from memory. On Windows we need to close
files.
Closing files forces us to sync them before (requirement for transactional
tables).
For mutex reasons (don't lock intern_lock twice), we move
maria_lock_database() and _ma_decrement_open_count() first in the list
of operations.
Flush also data file in HA_EXTRA_FLUSH.
storage/maria/ma_locking.c:
For transactional tables:
- don't write data pages / state at unlock time;
as a consequence, "share->changed=0" cannot be done.
- don't write state in _ma_writeinfo()
- don't maintain open_count on disk (Recovery corrects the table in case of crash
anyway, and we gain speed by not writing open_count to disk),
For non-transactional tables, flush the state at unlock only
if the table was changed (optimization).
Code which read the state from disk is relevant only with
external locking, we disable it (if want to re-enable it, it shouldn't
for transactional tables as state on disk may be obsolete (such tables
does not flush state at unlock anymore).
The comment "We have to flush the write cache" is now wrong because
maria_lock_database(F_UNLCK) now happens before thr_unlock(), and
we are not using external locking.
storage/maria/ma_open.c:
_ma_state_info_read() is only used in ma_open.c, making it static
storage/maria/ma_recovery.c:
set MARIA_SHARE::changed to TRUE when we are going to apply a
REDO/UNDO, so that the state gets flushed at close.
storage/maria/ma_test_recovery.expected:
Changes introduced by this patch:
- good: the "open" (table open, not properly closed) is gone,
it was pointless for a recovered table
- bad: stemming from different moments of writing the index's state
probably (_ma_writeinfo() used to write the state after every row
write in ma_test* programs, doesn't anymore as the table is
transactional): some differences in indexes (not relevant as we don't
yet have recovery for them); some differences in count of records
(changed from a wrong value to another wrong value) (not relevant
as we don't recover this count correctly yet anyway, though
a patch will be pushed soon).
storage/maria/ma_test_recovery:
for repeatable output, no names of varying directories.
storage/maria/maria_chk.c:
function renamed
storage/maria/maria_def.h:
Function became local to ma_open.c. Function renamed.
2007-09-06 16:53:26 +02:00
|
|
|
Should not be called for transactional tables, as their state on disk is
|
|
|
|
rarely current and so is often misleading for a reader.
|
|
|
|
Does nothing in single user mode.
|
WL#3072 Maria recovery
* create page cache before initializing engine and not after, because
Maria's recovery needs a page cache
* make the creation of a bitmap page more crash-resistent
* bugfix (see ma_blockrec.c)
* back to old way: create an 8k bitmap page when creating table
* preparations for the UNDO phase: recreate TRNs
* preparations for Checkpoint: list of dirty pages, testing
of rec_lsn to know if page should be skipped during Recovery
(unused in this patch as no Checkpoint module pushed yet)
* maria_chk tags repaired table with a special LSN
* reworking all around in ma_recovery.c (less duplication)
mysys/my_realloc.c:
noted an issue in my_realloc()
sql/mysqld.cc:
page cache needs to be created before engines are initialized,
because Maria's initialization may do a recovery which needs
the page cache.
storage/maria/ha_maria.cc:
update to new prototype
storage/maria/ma_bitmap.c:
when creating the first bitmap page we used chsize to 8192 bytes then
pwrite (overwrite) the last 2 bytes (8191-8192). If crash between
the two operations, this leaves a bitmap page full without its end
marker. A later recovery may try to read this page and find it
exists and misses a marker and conclude it's corrupted and fail.
Changing the chsize to only 8190 bytes: recovery will then find
the page is too short and recreate it entirely.
storage/maria/ma_blockrec.c:
Fix for a bug: when executing a REDO, if the data page is created,
data_file_length was increased before _ma_bitmap_set():
_ma_bitmap_set() called _ma_read_bitmap_page() which, due to the
increased data_file_length, expected to find a bitmap page on disk
with a correct end marker; if the bitmap page didn't exist already
in fact, this failed. Fixed by increasing data_file_length only after
_ma_read_bitmap_page() has created the new bitmap page correctly.
This bug could happen every time a REDO is about creating a new
bitmap page.
storage/maria/ma_check.c:
empty data file has a bitmap page
storage/maria/ma_control_file.c:
useless parameter to ma_control_file_create_or_open(), just
test if this is recovery.
storage/maria/ma_control_file.h:
new prototype
storage/maria/ma_create.c:
Back to how it was before: maria_create() creates an 8k bitmap page.
Thus (bugfix) data_file_length needs to reflect this instead of being 0.
storage/maria/ma_loghandler.c:
as ma_test1 and ma_test2 now use real transactions and not
dummy_transaction_object, REDO for INSERT/UPDATE/DELETE are always
about real transactions, can assert this.
A function for Recovery to assign a short id to a table.
storage/maria/ma_loghandler.h:
new function
storage/maria/ma_loghandler_lsn.h:
maria_chk tags repaired tables with this LSN
storage/maria/ma_open.c:
* enforce that DMLs on transactional tables use real transactions
and not dummy_transaction_object.
* test if table was repaired with maria_chk (which has to been
seen as an import of an external table into the server), test
validity of create_rename_lsn (header corruption detection)
* comments.
storage/maria/ma_recovery.c:
* preparations for the UNDO phase: recreate TRNs
* preparations for Checkpoint: list of dirty pages, testing
of rec_lsn to know if page should be skipped during Recovery
(unused in this patch as no Checkpoint module pushed yet)
* reworking all around (less duplication)
storage/maria/ma_recovery.h:
a parameter to say if the UNDO phase should be skipped
storage/maria/maria_chk.c:
tag repaired tables with a special LSN
storage/maria/maria_read_log.c:
* update to new prototype
* no UNDO phase in maria_read_log for now
storage/maria/trnman.c:
* a function for Recovery to create a transaction (TRN), needed
in the UNDO phase
* a function for Recovery to grab an existing transaction, needed
in the UNDO phase (rollback all existing transactions)
storage/maria/trnman_public.h:
new functions
2007-08-29 16:43:01 +02:00
|
|
|
|
|
|
|
@param file file to read from
|
|
|
|
@param state state which will be filled
|
|
|
|
@param pRead if true, use my_pread(), otherwise my_read()
|
|
|
|
*/
|
|
|
|
|
2007-09-03 12:05:17 +03:00
|
|
|
uint _ma_state_info_read_dsk(File file, MARIA_STATE_INFO *state)
|
2006-04-11 16:45:10 +03:00
|
|
|
{
|
2007-10-04 20:33:42 +03:00
|
|
|
uchar buff[MARIA_STATE_INFO_SIZE + MARIA_STATE_EXTRA_SIZE];
|
2006-04-11 16:45:10 +03:00
|
|
|
|
- speed optimization:
minimize writes to transactional Maria tables: don't write
data pages, state, and open_count at the end of each statement.
Data pages will be written by a background thread periodically.
State will be written by Checkpoint periodically.
open_count serves to detect when a table is potentially damaged
due to an unclean mysqld stop, but thanks to recovery an unclean
mysqld stop will be corrected and so open_count becomes useless.
As state is written less often, it is often obsolete on disk,
we thus should avoid to read it from disk.
- by removing the data page writes above, it is necessary to put
it back at the start of some statements like check, repair and
delete_all. It was already necessary in fact (see ma_delete_all.c).
- disabling CACHE INDEX on Maria tables for now (fixes crash
of test 'key_cache' when run with --default-storage-engine=maria).
- correcting some fishy code in maria_extra.c (we possibly could lose
index pages when doing a DROP TABLE under Windows, in theory).
storage/maria/ha_maria.cc:
disable CACHE INDEX in Maria for now (there is a single cache for now),
it crashes and it's not a priority
storage/maria/ma_bitmap.c:
debug message
storage/maria/ma_check.c:
The statement before maria_repair() may not flush state,
so it needs to be done by maria_repair() (indeed this function
uses maria_open(HA_OPEN_COPY) so reads state from disk,
so needs to find it up-to-date on disk).
For safety (but normally this is not needed) we remove index blocks
out of the cache before repairing.
_ma_flush_blocks() becomes _ma_flush_table_files_after_repair():
it now additionally flushes the data file and state and syncs files.
As a side effect, the assertion "no WRITE_CACHE_USED" from
_ma_flush_table_files() fired so we move all end_io_cache() done
at the end of repair to before the calls to _ma_flush_table_files_after_repair().
storage/maria/ma_close.c:
when closing a transactional table, we fsync it. But we need to
do this only after writing its state.
We need to write the state at close time only for transactional
tables (the other tables do that at last unlock).
Putting back the O_RDONLY||crashed condition which I had
removed earlier.
Unmap the file before syncing it (does not matter now as Maria
does not use mmap)
storage/maria/ma_delete_all.c:
need to flush data pages before chsize-ing it. Was needed even when
we flushed data pages at the end of each statement, because we didn't
anyway do it if under LOCK TABLES: the change here thus fixes this bug:
create table t(a int) engine=maria;lock tables t write;
insert into t values(1);delete from t;unlock tables;check table t;
"Size of datafile is: 16384 Should be: 8192"
(an obsolete page went to disk after the chsize(), at unlock time).
storage/maria/ma_extra.c:
When doing share->last_version=0, we make the MARIA_SHARE-in-memory
invisible to future openers, so need to have an up-to-date state
on disk for them. The same way, future openers will reopen the data
and index file, so they will not find our cached blocks, so we
need to flush them to disk.
In HA_EXTRA_FORCE_REOPEN, this probably happens naturally as all
tables normally get closed, we however add a safety flush.
In HA_EXTRA_PREPARE_FOR_RENAME, we need to do the flushing. On
Windows we additionally need to close files.
In HA_EXTRA_PREPARE_FOR_DROP, we don't need to flush anything but
remove dirty cached blocks from memory. On Windows we need to close
files.
Closing files forces us to sync them before (requirement for transactional
tables).
For mutex reasons (don't lock intern_lock twice), we move
maria_lock_database() and _ma_decrement_open_count() first in the list
of operations.
Flush also data file in HA_EXTRA_FLUSH.
storage/maria/ma_locking.c:
For transactional tables:
- don't write data pages / state at unlock time;
as a consequence, "share->changed=0" cannot be done.
- don't write state in _ma_writeinfo()
- don't maintain open_count on disk (Recovery corrects the table in case of crash
anyway, and we gain speed by not writing open_count to disk),
For non-transactional tables, flush the state at unlock only
if the table was changed (optimization).
Code which read the state from disk is relevant only with
external locking, we disable it (if want to re-enable it, it shouldn't
for transactional tables as state on disk may be obsolete (such tables
does not flush state at unlock anymore).
The comment "We have to flush the write cache" is now wrong because
maria_lock_database(F_UNLCK) now happens before thr_unlock(), and
we are not using external locking.
storage/maria/ma_open.c:
_ma_state_info_read() is only used in ma_open.c, making it static
storage/maria/ma_recovery.c:
set MARIA_SHARE::changed to TRUE when we are going to apply a
REDO/UNDO, so that the state gets flushed at close.
storage/maria/ma_test_recovery.expected:
Changes introduced by this patch:
- good: the "open" (table open, not properly closed) is gone,
it was pointless for a recovered table
- bad: stemming from different moments of writing the index's state
probably (_ma_writeinfo() used to write the state after every row
write in ma_test* programs, doesn't anymore as the table is
transactional): some differences in indexes (not relevant as we don't
yet have recovery for them); some differences in count of records
(changed from a wrong value to another wrong value) (not relevant
as we don't recover this count correctly yet anyway, though
a patch will be pushed soon).
storage/maria/ma_test_recovery:
for repeatable output, no names of varying directories.
storage/maria/maria_chk.c:
function renamed
storage/maria/maria_def.h:
Function became local to ma_open.c. Function renamed.
2007-09-06 16:53:26 +02:00
|
|
|
/* trick to detect transactional tables */
|
|
|
|
DBUG_ASSERT(state->create_rename_lsn == LSN_IMPOSSIBLE);
|
2006-04-11 16:45:10 +03:00
|
|
|
if (!maria_single_user)
|
|
|
|
{
|
2007-09-03 12:05:17 +03:00
|
|
|
if (my_pread(file, buff, state->state_length, 0L, MYF(MY_NABP)))
|
2007-07-27 12:06:39 +02:00
|
|
|
return 1;
|
2007-01-18 21:38:14 +02:00
|
|
|
_ma_state_info_read(buff, state);
|
2006-04-11 16:45:10 +03:00
|
|
|
}
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
/****************************************************************************
|
|
|
|
** store and read of MARIA_BASE_INFO
|
|
|
|
****************************************************************************/
|
|
|
|
|
|
|
|
uint _ma_base_info_write(File file, MARIA_BASE_INFO *base)
|
|
|
|
{
|
|
|
|
uchar buff[MARIA_BASE_INFO_SIZE], *ptr=buff;
|
|
|
|
|
2007-01-18 21:38:14 +02:00
|
|
|
mi_sizestore(ptr,base->keystart); ptr+= 8;
|
|
|
|
mi_sizestore(ptr,base->max_data_file_length); ptr+= 8;
|
|
|
|
mi_sizestore(ptr,base->max_key_file_length); ptr+= 8;
|
|
|
|
mi_rowstore(ptr,base->records); ptr+= 8;
|
|
|
|
mi_rowstore(ptr,base->reloc); ptr+= 8;
|
|
|
|
mi_int4store(ptr,base->mean_row_length); ptr+= 4;
|
|
|
|
mi_int4store(ptr,base->reclength); ptr+= 4;
|
|
|
|
mi_int4store(ptr,base->pack_reclength); ptr+= 4;
|
|
|
|
mi_int4store(ptr,base->min_pack_length); ptr+= 4;
|
|
|
|
mi_int4store(ptr,base->max_pack_length); ptr+= 4;
|
|
|
|
mi_int4store(ptr,base->min_block_length); ptr+= 4;
|
|
|
|
mi_int2store(ptr,base->fields); ptr+= 2;
|
|
|
|
mi_int2store(ptr,base->fixed_not_null_fields); ptr+= 2;
|
|
|
|
mi_int2store(ptr,base->fixed_not_null_fields_length); ptr+= 2;
|
|
|
|
mi_int2store(ptr,base->max_field_lengths); ptr+= 2;
|
|
|
|
mi_int2store(ptr,base->pack_fields); ptr+= 2;
|
|
|
|
mi_int2store(ptr,0); ptr+= 2;
|
|
|
|
mi_int2store(ptr,base->null_bytes); ptr+= 2;
|
|
|
|
mi_int2store(ptr,base->original_null_bytes); ptr+= 2;
|
|
|
|
mi_int2store(ptr,base->field_offsets); ptr+= 2;
|
|
|
|
mi_int2store(ptr,base->min_row_length); ptr+= 2;
|
|
|
|
mi_int2store(ptr,base->block_size); ptr+= 2;
|
|
|
|
*ptr++= base->rec_reflength;
|
|
|
|
*ptr++= base->key_reflength;
|
|
|
|
*ptr++= base->keys;
|
|
|
|
*ptr++= base->auto_key;
|
Maria:
* Don't modify share->base.born_transactional; now it is a value carved
in stone at creation time. share->now_transactional is what can be
modified: it starts at born_transactional, can become false during
ALTER TABLE (when we want no logging), and restored later.
* Not resetting create_rename_lsn to 0 during delete_all or repair.
* when we temporarily disable transactionality, we also change
the page type to PAGECACHE_PLAIN_PAGE: it bypasses some work in the
page cache (optimization), and avoids assertions related to LSNs.
* Disable INSERT DELAYED for transactional tables, because
durability could not be guaranteed (insertion may even not happen)
mysys/mf_keycache.c:
comment
storage/maria/ha_maria.cc:
* a transactional table cannot do INSERT DELAYED
* ha_maria::save_transactional not needed anymore, as now instead
we don't modify MARIA_SHARE::MARIA_BASE_INFO::born_transactional
(born_transactional plays the role of save_transactional), and modify
MARIA_SHARE::now_transactional.
* REPAIR_TABLE log record is now logged by maria_repair()
* comment why we rely on born_transactional to know if we should
skipping a transaction.
* putting together two if()s which test for F_UNLCK
storage/maria/ha_maria.h:
ha_maria::save_transactional not needed anymore (moved to the C layer)
storage/maria/ma_blockrec.c:
* For the block record's code (writing/updating/deleting records),
all that counts is now_transactional, not born_transactional.
* As we now set the page type to PAGECACHE_PLAIN_PAGE for tables
which have now_transactional==FALSE, pagecache will not expect
a meaningful LSN for them in pagecache_unlock_by_link(), so
we can pass it LSN_IMPOSSIBLE.
storage/maria/ma_check.c:
* writing LOGREC_REPAIR_TABLE moves from ha_maria::repair()
to maria_repair(), sounds cleaner (less functions to export).
* when opening a table during REPAIR, don't use the realpath-ed name,
as this may fail if the table has symlinked files (maria_open()
would try to find the data and index file in the directory
of unique_file_name, it would fail if data and index files are in
different dirs); use the unresolved name, open_file_name, which is
the argument which was passed to the maria_open() which created 'info'.
storage/maria/ma_close.c:
assert that when a statement is done with a table, it cleans up
storage/maria/ma_create.c:
new name
storage/maria/ma_delete_all.c:
* using now_transactional
* no reason to reset create_rename_lsn during delete_all (a bug);
also no reason to do it during repair: it was put there because
a positive create_rename_lsn caused a call to check_and_set_lsn()
which asserted in DBUG_ASSERT(block->type == PAGECACHE_LSN_PAGE);
first solution was to use LSN_IMPOSSIBLE in _ma_unpin_all_pages() if
not transactional; but then in the case of ALTER TABLE, with
transactionality temporarily disabled, it asserted in
DBUG_ASSERT(LSN_VALID(lsn)) in pagecache_fwrite() (PAGECACHE_LSN_PAGE
page with zero LSN - bad). The additional solution is to use
PAGECACHE_PLAIN_PAGE when we disable transactionality temporarily: this
avoids checks on the LSN, and also bypasses (optimization) the "flush
log up to LSN" call when the pagecache flushes our page (in other
words, no WAL needed).
storage/maria/ma_delete_table.c:
use now_transactional
storage/maria/ma_locking.c:
assert that when a statement is done with a table, it cleans up.
storage/maria/ma_loghandler.c:
* now_transactional should be used to test if we want a log record.
* Assertions to make sure dummy_transaction_object is not spoilt
by its many users.
storage/maria/ma_open.c:
base.transactional -> base.born_transactional
storage/maria/ma_pagecache.c:
missing name for page's type. Comment for future.
storage/maria/ma_rename.c:
use now_transactional
storage/maria/maria_chk.c:
use born_transactional
storage/maria/maria_def.h:
MARIA_BASE_INFO::transactional renamed to born_transactional.
MARIA_SHARE::now_transactional introduced.
_ma_repair_write_log_record() is made local to ma_check.c.
Macros to temporarily disable, and re-enable, transactionality for a
table.
storage/maria/maria_read_log.c:
assertions and using the new macros. Adding a forgotten resetting
when we finally close all tables.
2007-07-03 15:20:41 +02:00
|
|
|
*ptr++= base->born_transactional;
|
2007-01-18 21:38:14 +02:00
|
|
|
*ptr++= 0; /* Reserved */
|
|
|
|
mi_int2store(ptr,base->pack_bytes); ptr+= 2;
|
|
|
|
mi_int2store(ptr,base->blobs); ptr+= 2;
|
|
|
|
mi_int2store(ptr,base->max_key_block_length); ptr+= 2;
|
|
|
|
mi_int2store(ptr,base->max_key_length); ptr+= 2;
|
|
|
|
mi_int2store(ptr,base->extra_alloc_bytes); ptr+= 2;
|
2006-04-11 16:45:10 +03:00
|
|
|
*ptr++= base->extra_alloc_procent;
|
2007-01-18 21:38:14 +02:00
|
|
|
bzero(ptr,16); ptr+= 16; /* extra */
|
|
|
|
DBUG_ASSERT((ptr - buff) == MARIA_BASE_INFO_SIZE);
|
2007-07-27 12:06:39 +02:00
|
|
|
return my_write(file, buff, (size_t) (ptr-buff), MYF(MY_NABP)) != 0;
|
2006-04-11 16:45:10 +03:00
|
|
|
}
|
|
|
|
|
|
|
|
|
2007-07-02 20:45:15 +03:00
|
|
|
static uchar *_ma_base_info_read(uchar *ptr, MARIA_BASE_INFO *base)
|
2006-04-11 16:45:10 +03:00
|
|
|
{
|
2007-01-18 21:38:14 +02:00
|
|
|
base->keystart= mi_sizekorr(ptr); ptr+= 8;
|
|
|
|
base->max_data_file_length= mi_sizekorr(ptr); ptr+= 8;
|
|
|
|
base->max_key_file_length= mi_sizekorr(ptr); ptr+= 8;
|
|
|
|
base->records= (ha_rows) mi_sizekorr(ptr); ptr+= 8;
|
|
|
|
base->reloc= (ha_rows) mi_sizekorr(ptr); ptr+= 8;
|
|
|
|
base->mean_row_length= mi_uint4korr(ptr); ptr+= 4;
|
|
|
|
base->reclength= mi_uint4korr(ptr); ptr+= 4;
|
|
|
|
base->pack_reclength= mi_uint4korr(ptr); ptr+= 4;
|
|
|
|
base->min_pack_length= mi_uint4korr(ptr); ptr+= 4;
|
|
|
|
base->max_pack_length= mi_uint4korr(ptr); ptr+= 4;
|
|
|
|
base->min_block_length= mi_uint4korr(ptr); ptr+= 4;
|
|
|
|
base->fields= mi_uint2korr(ptr); ptr+= 2;
|
|
|
|
base->fixed_not_null_fields= mi_uint2korr(ptr); ptr+= 2;
|
|
|
|
base->fixed_not_null_fields_length= mi_uint2korr(ptr);ptr+= 2;
|
|
|
|
base->max_field_lengths= mi_uint2korr(ptr); ptr+= 2;
|
|
|
|
base->pack_fields= mi_uint2korr(ptr); ptr+= 2;
|
|
|
|
ptr+= 2;
|
|
|
|
base->null_bytes= mi_uint2korr(ptr); ptr+= 2;
|
|
|
|
base->original_null_bytes= mi_uint2korr(ptr); ptr+= 2;
|
|
|
|
base->field_offsets= mi_uint2korr(ptr); ptr+= 2;
|
|
|
|
base->min_row_length= mi_uint2korr(ptr); ptr+= 2;
|
|
|
|
base->block_size= mi_uint2korr(ptr); ptr+= 2;
|
|
|
|
|
|
|
|
base->rec_reflength= *ptr++;
|
|
|
|
base->key_reflength= *ptr++;
|
|
|
|
base->keys= *ptr++;
|
|
|
|
base->auto_key= *ptr++;
|
Maria:
* Don't modify share->base.born_transactional; now it is a value carved
in stone at creation time. share->now_transactional is what can be
modified: it starts at born_transactional, can become false during
ALTER TABLE (when we want no logging), and restored later.
* Not resetting create_rename_lsn to 0 during delete_all or repair.
* when we temporarily disable transactionality, we also change
the page type to PAGECACHE_PLAIN_PAGE: it bypasses some work in the
page cache (optimization), and avoids assertions related to LSNs.
* Disable INSERT DELAYED for transactional tables, because
durability could not be guaranteed (insertion may even not happen)
mysys/mf_keycache.c:
comment
storage/maria/ha_maria.cc:
* a transactional table cannot do INSERT DELAYED
* ha_maria::save_transactional not needed anymore, as now instead
we don't modify MARIA_SHARE::MARIA_BASE_INFO::born_transactional
(born_transactional plays the role of save_transactional), and modify
MARIA_SHARE::now_transactional.
* REPAIR_TABLE log record is now logged by maria_repair()
* comment why we rely on born_transactional to know if we should
skipping a transaction.
* putting together two if()s which test for F_UNLCK
storage/maria/ha_maria.h:
ha_maria::save_transactional not needed anymore (moved to the C layer)
storage/maria/ma_blockrec.c:
* For the block record's code (writing/updating/deleting records),
all that counts is now_transactional, not born_transactional.
* As we now set the page type to PAGECACHE_PLAIN_PAGE for tables
which have now_transactional==FALSE, pagecache will not expect
a meaningful LSN for them in pagecache_unlock_by_link(), so
we can pass it LSN_IMPOSSIBLE.
storage/maria/ma_check.c:
* writing LOGREC_REPAIR_TABLE moves from ha_maria::repair()
to maria_repair(), sounds cleaner (less functions to export).
* when opening a table during REPAIR, don't use the realpath-ed name,
as this may fail if the table has symlinked files (maria_open()
would try to find the data and index file in the directory
of unique_file_name, it would fail if data and index files are in
different dirs); use the unresolved name, open_file_name, which is
the argument which was passed to the maria_open() which created 'info'.
storage/maria/ma_close.c:
assert that when a statement is done with a table, it cleans up
storage/maria/ma_create.c:
new name
storage/maria/ma_delete_all.c:
* using now_transactional
* no reason to reset create_rename_lsn during delete_all (a bug);
also no reason to do it during repair: it was put there because
a positive create_rename_lsn caused a call to check_and_set_lsn()
which asserted in DBUG_ASSERT(block->type == PAGECACHE_LSN_PAGE);
first solution was to use LSN_IMPOSSIBLE in _ma_unpin_all_pages() if
not transactional; but then in the case of ALTER TABLE, with
transactionality temporarily disabled, it asserted in
DBUG_ASSERT(LSN_VALID(lsn)) in pagecache_fwrite() (PAGECACHE_LSN_PAGE
page with zero LSN - bad). The additional solution is to use
PAGECACHE_PLAIN_PAGE when we disable transactionality temporarily: this
avoids checks on the LSN, and also bypasses (optimization) the "flush
log up to LSN" call when the pagecache flushes our page (in other
words, no WAL needed).
storage/maria/ma_delete_table.c:
use now_transactional
storage/maria/ma_locking.c:
assert that when a statement is done with a table, it cleans up.
storage/maria/ma_loghandler.c:
* now_transactional should be used to test if we want a log record.
* Assertions to make sure dummy_transaction_object is not spoilt
by its many users.
storage/maria/ma_open.c:
base.transactional -> base.born_transactional
storage/maria/ma_pagecache.c:
missing name for page's type. Comment for future.
storage/maria/ma_rename.c:
use now_transactional
storage/maria/maria_chk.c:
use born_transactional
storage/maria/maria_def.h:
MARIA_BASE_INFO::transactional renamed to born_transactional.
MARIA_SHARE::now_transactional introduced.
_ma_repair_write_log_record() is made local to ma_check.c.
Macros to temporarily disable, and re-enable, transactionality for a
table.
storage/maria/maria_read_log.c:
assertions and using the new macros. Adding a forgotten resetting
when we finally close all tables.
2007-07-03 15:20:41 +02:00
|
|
|
base->born_transactional= *ptr++;
|
2007-01-18 21:38:14 +02:00
|
|
|
ptr++;
|
|
|
|
base->pack_bytes= mi_uint2korr(ptr); ptr+= 2;
|
|
|
|
base->blobs= mi_uint2korr(ptr); ptr+= 2;
|
|
|
|
base->max_key_block_length= mi_uint2korr(ptr); ptr+= 2;
|
|
|
|
base->max_key_length= mi_uint2korr(ptr); ptr+= 2;
|
|
|
|
base->extra_alloc_bytes= mi_uint2korr(ptr); ptr+= 2;
|
|
|
|
base->extra_alloc_procent= *ptr++;
|
|
|
|
ptr+= 16;
|
2006-04-11 16:45:10 +03:00
|
|
|
return ptr;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*--------------------------------------------------------------------------
|
|
|
|
maria_keydef
|
|
|
|
---------------------------------------------------------------------------*/
|
|
|
|
|
|
|
|
uint _ma_keydef_write(File file, MARIA_KEYDEF *keydef)
|
|
|
|
{
|
|
|
|
uchar buff[MARIA_KEYDEF_SIZE];
|
|
|
|
uchar *ptr=buff;
|
|
|
|
|
2007-01-18 21:38:14 +02:00
|
|
|
*ptr++= (uchar) keydef->keysegs;
|
|
|
|
*ptr++= keydef->key_alg; /* Rtree or Btree */
|
|
|
|
mi_int2store(ptr,keydef->flag); ptr+= 2;
|
|
|
|
mi_int2store(ptr,keydef->block_length); ptr+= 2;
|
|
|
|
mi_int2store(ptr,keydef->keylength); ptr+= 2;
|
|
|
|
mi_int2store(ptr,keydef->minlength); ptr+= 2;
|
|
|
|
mi_int2store(ptr,keydef->maxlength); ptr+= 2;
|
2007-07-27 12:06:39 +02:00
|
|
|
return my_write(file, buff, (size_t) (ptr-buff), MYF(MY_NABP)) != 0;
|
2006-04-11 16:45:10 +03:00
|
|
|
}
|
|
|
|
|
2007-10-04 20:33:42 +03:00
|
|
|
uchar *_ma_keydef_read(uchar *ptr, MARIA_KEYDEF *keydef)
|
2006-04-11 16:45:10 +03:00
|
|
|
{
|
|
|
|
keydef->keysegs = (uint) *ptr++;
|
|
|
|
keydef->key_alg = *ptr++; /* Rtree or Btree */
|
|
|
|
|
2007-01-18 21:38:14 +02:00
|
|
|
keydef->flag = mi_uint2korr(ptr); ptr+= 2;
|
|
|
|
keydef->block_length = mi_uint2korr(ptr); ptr+= 2;
|
|
|
|
keydef->keylength = mi_uint2korr(ptr); ptr+= 2;
|
|
|
|
keydef->minlength = mi_uint2korr(ptr); ptr+= 2;
|
|
|
|
keydef->maxlength = mi_uint2korr(ptr); ptr+= 2;
|
2006-04-11 16:45:10 +03:00
|
|
|
keydef->underflow_block_length=keydef->block_length/3;
|
|
|
|
keydef->version = 0; /* Not saved */
|
|
|
|
keydef->parser = &ft_default_parser;
|
|
|
|
keydef->ftparser_nr = 0;
|
|
|
|
return ptr;
|
|
|
|
}
|
|
|
|
|
|
|
|
/***************************************************************************
|
|
|
|
** maria_keyseg
|
|
|
|
***************************************************************************/
|
|
|
|
|
|
|
|
int _ma_keyseg_write(File file, const HA_KEYSEG *keyseg)
|
|
|
|
{
|
|
|
|
uchar buff[HA_KEYSEG_SIZE];
|
|
|
|
uchar *ptr=buff;
|
|
|
|
ulong pos;
|
|
|
|
|
|
|
|
*ptr++= keyseg->type;
|
|
|
|
*ptr++= keyseg->language;
|
|
|
|
*ptr++= keyseg->null_bit;
|
|
|
|
*ptr++= keyseg->bit_start;
|
|
|
|
*ptr++= keyseg->bit_end;
|
|
|
|
*ptr++= keyseg->bit_length;
|
2007-01-18 21:38:14 +02:00
|
|
|
mi_int2store(ptr,keyseg->flag); ptr+= 2;
|
|
|
|
mi_int2store(ptr,keyseg->length); ptr+= 2;
|
|
|
|
mi_int4store(ptr,keyseg->start); ptr+= 4;
|
2006-04-11 16:45:10 +03:00
|
|
|
pos= keyseg->null_bit ? keyseg->null_pos : keyseg->bit_pos;
|
|
|
|
mi_int4store(ptr, pos);
|
|
|
|
ptr+=4;
|
|
|
|
|
2007-07-27 12:06:39 +02:00
|
|
|
return my_write(file, buff, (size_t) (ptr-buff), MYF(MY_NABP)) != 0;
|
2006-04-11 16:45:10 +03:00
|
|
|
}
|
|
|
|
|
|
|
|
|
2007-10-04 20:33:42 +03:00
|
|
|
uchar *_ma_keyseg_read(uchar *ptr, HA_KEYSEG *keyseg)
|
2006-04-11 16:45:10 +03:00
|
|
|
{
|
|
|
|
keyseg->type = *ptr++;
|
|
|
|
keyseg->language = *ptr++;
|
|
|
|
keyseg->null_bit = *ptr++;
|
|
|
|
keyseg->bit_start = *ptr++;
|
|
|
|
keyseg->bit_end = *ptr++;
|
|
|
|
keyseg->bit_length = *ptr++;
|
2007-01-18 21:38:14 +02:00
|
|
|
keyseg->flag = mi_uint2korr(ptr); ptr+= 2;
|
|
|
|
keyseg->length = mi_uint2korr(ptr); ptr+= 2;
|
|
|
|
keyseg->start = mi_uint4korr(ptr); ptr+= 4;
|
|
|
|
keyseg->null_pos = mi_uint4korr(ptr); ptr+= 4;
|
2006-04-11 16:45:10 +03:00
|
|
|
keyseg->charset=0; /* Will be filled in later */
|
|
|
|
if (keyseg->null_bit)
|
|
|
|
keyseg->bit_pos= (uint16)(keyseg->null_pos + (keyseg->null_bit == 7));
|
|
|
|
else
|
|
|
|
{
|
|
|
|
keyseg->bit_pos= (uint16)keyseg->null_pos;
|
|
|
|
keyseg->null_pos= 0;
|
|
|
|
}
|
|
|
|
return ptr;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*--------------------------------------------------------------------------
|
|
|
|
maria_uniquedef
|
|
|
|
---------------------------------------------------------------------------*/
|
|
|
|
|
|
|
|
uint _ma_uniquedef_write(File file, MARIA_UNIQUEDEF *def)
|
|
|
|
{
|
|
|
|
uchar buff[MARIA_UNIQUEDEF_SIZE];
|
|
|
|
uchar *ptr=buff;
|
|
|
|
|
|
|
|
mi_int2store(ptr,def->keysegs); ptr+=2;
|
|
|
|
*ptr++= (uchar) def->key;
|
|
|
|
*ptr++ = (uchar) def->null_are_equal;
|
|
|
|
|
2007-07-27 12:06:39 +02:00
|
|
|
return my_write(file, buff, (size_t) (ptr-buff), MYF(MY_NABP)) != 0;
|
2006-04-11 16:45:10 +03:00
|
|
|
}
|
|
|
|
|
2007-10-04 20:33:42 +03:00
|
|
|
uchar *_ma_uniquedef_read(uchar *ptr, MARIA_UNIQUEDEF *def)
|
2006-04-11 16:45:10 +03:00
|
|
|
{
|
|
|
|
def->keysegs = mi_uint2korr(ptr);
|
|
|
|
def->key = ptr[2];
|
|
|
|
def->null_are_equal=ptr[3];
|
2007-07-02 20:45:15 +03:00
|
|
|
return ptr+4; /* 1 extra uchar */
|
2006-04-11 16:45:10 +03:00
|
|
|
}
|
|
|
|
|
|
|
|
/***************************************************************************
|
|
|
|
** MARIA_COLUMNDEF
|
|
|
|
***************************************************************************/
|
|
|
|
|
2007-04-19 13:18:56 +03:00
|
|
|
uint _ma_columndef_write(File file, MARIA_COLUMNDEF *columndef)
|
2006-04-11 16:45:10 +03:00
|
|
|
{
|
|
|
|
uchar buff[MARIA_COLUMNDEF_SIZE];
|
|
|
|
uchar *ptr=buff;
|
|
|
|
|
2007-04-19 13:18:56 +03:00
|
|
|
mi_int6store(ptr,columndef->offset); ptr+= 6;
|
|
|
|
mi_int2store(ptr,columndef->type); ptr+= 2;
|
|
|
|
mi_int2store(ptr,columndef->length); ptr+= 2;
|
|
|
|
mi_int2store(ptr,columndef->fill_length); ptr+= 2;
|
|
|
|
mi_int2store(ptr,columndef->null_pos); ptr+= 2;
|
|
|
|
mi_int2store(ptr,columndef->empty_pos); ptr+= 2;
|
|
|
|
(*ptr++)= columndef->null_bit;
|
|
|
|
(*ptr++)= columndef->empty_bit;
|
2007-07-27 12:06:39 +02:00
|
|
|
return my_write(file, buff, (size_t) (ptr-buff), MYF(MY_NABP)) != 0;
|
2006-04-11 16:45:10 +03:00
|
|
|
}
|
|
|
|
|
2007-10-04 20:33:42 +03:00
|
|
|
uchar *_ma_columndef_read(uchar *ptr, MARIA_COLUMNDEF *columndef)
|
2006-04-11 16:45:10 +03:00
|
|
|
{
|
2007-04-19 13:18:56 +03:00
|
|
|
columndef->offset= mi_uint6korr(ptr); ptr+= 6;
|
|
|
|
columndef->type= mi_sint2korr(ptr); ptr+= 2;
|
|
|
|
columndef->length= mi_uint2korr(ptr); ptr+= 2;
|
|
|
|
columndef->fill_length= mi_uint2korr(ptr); ptr+= 2;
|
|
|
|
columndef->null_pos= mi_uint2korr(ptr); ptr+= 2;
|
|
|
|
columndef->empty_pos= mi_uint2korr(ptr); ptr+= 2;
|
|
|
|
columndef->null_bit= (uint8) *ptr++;
|
|
|
|
columndef->empty_bit= (uint8) *ptr++;
|
2007-01-18 21:38:14 +02:00
|
|
|
return ptr;
|
2006-04-11 16:45:10 +03:00
|
|
|
}
|
|
|
|
|
|
|
|
/**************************************************************************
|
2007-01-18 21:38:14 +02:00
|
|
|
Open data file
|
|
|
|
We can't use dup() here as the data file descriptors need to have different
|
|
|
|
active seek-positions.
|
2006-04-11 16:45:10 +03:00
|
|
|
|
2007-01-18 21:38:14 +02:00
|
|
|
The argument file_to_dup is here for the future if there would on some OS
|
|
|
|
exist a dup()-like call that would give us two different file descriptors.
|
2006-04-11 16:45:10 +03:00
|
|
|
*************************************************************************/
|
|
|
|
|
2007-01-18 21:38:14 +02:00
|
|
|
int _ma_open_datafile(MARIA_HA *info, MARIA_SHARE *share,
|
|
|
|
File file_to_dup __attribute__((unused)))
|
2006-04-11 16:45:10 +03:00
|
|
|
{
|
2007-08-09 10:51:15 +02:00
|
|
|
info->dfile.file= share->bitmap.file.file=
|
|
|
|
my_open(share->data_file_name, share->mode | O_SHARE,
|
|
|
|
MYF(MY_WME));
|
2007-04-04 23:37:09 +03:00
|
|
|
return info->dfile.file >= 0 ? 0 : 1;
|
2006-04-11 16:45:10 +03:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
int _ma_open_keyfile(MARIA_SHARE *share)
|
|
|
|
{
|
- WL#3072 Maria Recovery:
Recovery of state.records (the count of records which is stored into
the header of the index file). For that, state.is_of_lsn is introduced;
logic is explained in ma_recovery.c (look for "Recovery of the state").
The net gain is that in case of crash, we now recover state.records,
and it is idempotent (ma_test_recovery tests it).
state.checksum is not recovered yet, mail sent for discussion.
- WL#3071 Maria Checkpoint: preparation for it, by protecting
all modifications of the state in memory or on disk with intern_lock
(with the exception of the really-often-modified state.records,
which is now protected with the log's lock, see ma_recovery.c
(look for "Recovery of the state"). Also, if maria_close() sees that
Checkpoint is looking at this table it will not my_free() the share.
- don't compute row's checksum twice in case of UPDATE (correction
to a bugfix I made yesterday).
storage/maria/ha_maria.cc:
protect state write with intern_lock (against Checkpoint)
storage/maria/ma_blockrec.c:
* don't reset trn->rec_lsn in _ma_unpin_all_pages(), because it
should wait until we have corrected the allocation in the bitmap
(as the REDO can serve to correct the allocation during Recovery);
introducing _ma_finalize_row() for that.
* In a changeset yesterday I moved computation of the checksum
into write_block_record(), to fix a bug in UPDATE. Now I notice
that maria_update() already computes the checksum, it's just that
it puts it into info->cur_row while _ma_update_block_record()
uses info->new_row; so, removing the checksum computation from
write_block_record(), putting it back into allocate_and_write_block_record()
(which is called only by INSERT and UNDO_DELETE), and copying
cur_row->checksum into new_row->checksum in _ma_update_block_record().
storage/maria/ma_check.c:
new prototypes, they will take intern_lock when writing the state;
also take intern_lock when changing share->kfile. In both cases
this is to protect against Checkpoint reading/writing the state or reading
kfile at the same time.
Not updating create_rename_lsn directly at end of write_log_record_for_repair()
as it wouldn't have intern_lock.
storage/maria/ma_close.c:
Checkpoint builds a list of shares (under THR_LOCK_maria), then it
handles each such share (under intern_lock) (doing flushing etc);
if maria_close() freed this share between the two, Checkpoint
would see a bad pointer. To avoid this, when building the list Checkpoint
marks each share, so that maria_close() knows it should not free it
and Checkpoint will free it itself.
Extending the zone covered by intern_lock to protect against
Checkpoint reading kfile, writing state.
storage/maria/ma_create.c:
When we update create_rename_lsn, we also update is_of_lsn to
the same value: it is logical, and allows us to test in maria_open()
that the former is not bigger than the latter (the contrary is a sign
of index header corruption, or severe logging bug which hinders
Recovery, table needs a repair).
_ma_update_create_rename_lsn_on_disk() also writes is_of_lsn;
it now operates under intern_lock (protect against Checkpoint),
a shortcut function is available for cases where acquiring
intern_lock is not needed (table's creation or first open).
storage/maria/ma_delete.c:
if table is transactional, "records" is already decremented
when logging UNDO_ROW_DELETE.
storage/maria/ma_delete_all.c:
comments
storage/maria/ma_extra.c:
Protect modifications of the state, in memory and/or on disk,
with intern_lock, against a concurrent Checkpoint.
When state goes to disk, update it's is_of_lsn (by calling
the new _ma_state_info_write()).
In HA_EXTRA_FORCE_REOPEN, don't set share->changed to 0 (undoing
a change I made a few days ago) and ASK_MONTY
storage/maria/ma_locking.c:
no real code change here.
storage/maria/ma_loghandler.c:
Log-write-hooks for updating "state.records" under log's mutex
when writing/updating/deleting a row or deleting all rows.
storage/maria/ma_loghandler_lsn.h:
merge (make LSN_ERROR and LSN_REPAIRED_BY_MARIA_CHK different)
storage/maria/ma_open.c:
When opening a table verify that is_of_lsn >= create_rename_lsn; if
false the header must be corrupted.
_ma_state_info_write() is split in two: _ma_state_info_write_sub()
which is the old _ma_state_info_write(), and _ma_state_info_write()
which additionally takes intern_lock if requested (to protect
against Checkpoint) and updates is_of_lsn.
_ma_open_keyfile() should change kfile.file under intern_lock
to protect Checkpoint from reading a wrong kfile.file.
storage/maria/ma_recovery.c:
Recovery of state.records: when the REDO phase sees UNDO_ROW_INSERT
which has a LSN > state.is_of_lsn it increments state.records.
Same for UNDO_ROW_DELETE and UNDO_ROW_PURGE.
When closing a table during Recovery, we know its state is at least
as new as the current log record we are looking at, so increase
is_of_lsn to the LSN of the current log record.
storage/maria/ma_rename.c:
update for new behaviour of _ma_update_create_rename_lsn_on_disk().
storage/maria/ma_test1.c:
update to new prototype
storage/maria/ma_test2.c:
update to new prototype (actually prototype was changed days ago,
but compiler does not complain about the extra argument??)
storage/maria/ma_test_recovery.expected:
new result file of ma_test_recovery. Improvements: record
count read from index's header is now always correct.
storage/maria/ma_test_recovery:
"rm" fails if file does not exist. Redirect stderr of script.
storage/maria/ma_write.c:
if table is transactional, "records" is already incremented when
logging UNDO_ROW_INSERT. Comments.
storage/maria/maria_chk.c:
update is_of_lsn too
storage/maria/maria_def.h:
- MARIA_STATE_INFO::is_of_lsn which is used by Recovery. It is stored
into the index file's header.
- Checkpoint can now mark a table as "don't free this", and maria_close()
can reply "ok then you will free it".
- new functions
storage/maria/maria_pack.c:
update for new name
2007-09-07 15:02:30 +02:00
|
|
|
/*
|
|
|
|
Modifications to share->kfile should be under intern_lock to protect
|
|
|
|
against a concurrent checkpoint.
|
|
|
|
*/
|
|
|
|
pthread_mutex_lock(&share->intern_lock);
|
|
|
|
share->kfile.file= my_open(share->unique_file_name,
|
|
|
|
share->mode | O_SHARE,
|
|
|
|
MYF(MY_WME));
|
|
|
|
pthread_mutex_unlock(&share->intern_lock);
|
|
|
|
return (share->kfile.file < 0);
|
2006-04-11 16:45:10 +03:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
Disable all indexes.
|
|
|
|
|
|
|
|
SYNOPSIS
|
|
|
|
maria_disable_indexes()
|
|
|
|
info A pointer to the MARIA storage engine MARIA_HA struct.
|
|
|
|
|
|
|
|
DESCRIPTION
|
|
|
|
Disable all indexes.
|
|
|
|
|
|
|
|
RETURN
|
|
|
|
0 ok
|
|
|
|
*/
|
|
|
|
|
|
|
|
int maria_disable_indexes(MARIA_HA *info)
|
|
|
|
{
|
|
|
|
MARIA_SHARE *share= info->s;
|
|
|
|
|
|
|
|
maria_clear_all_keys_active(share->state.key_map);
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
Enable all indexes
|
|
|
|
|
|
|
|
SYNOPSIS
|
|
|
|
maria_enable_indexes()
|
|
|
|
info A pointer to the MARIA storage engine MARIA_HA struct.
|
|
|
|
|
|
|
|
DESCRIPTION
|
|
|
|
Enable all indexes. The indexes might have been disabled
|
|
|
|
by maria_disable_index() before.
|
|
|
|
The function works only if both data and indexes are empty,
|
|
|
|
otherwise a repair is required.
|
|
|
|
To be sure, call handler::delete_all_rows() before.
|
|
|
|
|
|
|
|
RETURN
|
|
|
|
0 ok
|
|
|
|
HA_ERR_CRASHED data or index is non-empty.
|
|
|
|
*/
|
|
|
|
|
|
|
|
int maria_enable_indexes(MARIA_HA *info)
|
|
|
|
{
|
|
|
|
int error= 0;
|
|
|
|
MARIA_SHARE *share= info->s;
|
|
|
|
|
|
|
|
if (share->state.state.data_file_length ||
|
|
|
|
(share->state.state.key_file_length != share->base.keystart))
|
|
|
|
{
|
|
|
|
maria_print_error(info->s, HA_ERR_CRASHED);
|
|
|
|
error= HA_ERR_CRASHED;
|
|
|
|
}
|
|
|
|
else
|
|
|
|
maria_set_all_keys_active(share->state.key_map, share->base.keys);
|
|
|
|
return error;
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
Test if indexes are disabled.
|
|
|
|
|
|
|
|
SYNOPSIS
|
|
|
|
maria_indexes_are_disabled()
|
|
|
|
info A pointer to the MARIA storage engine MARIA_HA struct.
|
|
|
|
|
|
|
|
DESCRIPTION
|
|
|
|
Test if indexes are disabled.
|
|
|
|
|
|
|
|
RETURN
|
|
|
|
0 indexes are not disabled
|
|
|
|
1 all indexes are disabled
|
2006-12-20 18:58:35 +01:00
|
|
|
2 non-unique indexes are disabled
|
2006-04-11 16:45:10 +03:00
|
|
|
*/
|
|
|
|
|
|
|
|
int maria_indexes_are_disabled(MARIA_HA *info)
|
|
|
|
{
|
|
|
|
MARIA_SHARE *share= info->s;
|
|
|
|
|
2006-12-20 18:58:35 +01:00
|
|
|
/*
|
|
|
|
No keys or all are enabled. keys is the number of keys. Left shifted
|
|
|
|
gives us only one bit set. When decreased by one, gives us all all bits
|
|
|
|
up to this one set and it gets unset.
|
|
|
|
*/
|
|
|
|
if (!share->base.keys ||
|
|
|
|
(maria_is_all_keys_active(share->state.key_map, share->base.keys)))
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
/* All are disabled */
|
|
|
|
if (maria_is_any_key_active(share->state.key_map))
|
|
|
|
return 1;
|
|
|
|
|
|
|
|
/*
|
|
|
|
We have keys. Some enabled, some disabled.
|
|
|
|
Don't check for any non-unique disabled but return directly 2
|
|
|
|
*/
|
|
|
|
return 2;
|
2006-04-11 16:45:10 +03:00
|
|
|
}
|
2007-01-18 21:38:14 +02:00
|
|
|
|
|
|
|
|
|
|
|
static my_bool maria_scan_init_dummy(MARIA_HA *info __attribute__((unused)))
|
|
|
|
{
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void maria_scan_end_dummy(MARIA_HA *info __attribute__((unused)))
|
|
|
|
{
|
|
|
|
}
|
|
|
|
|
|
|
|
static my_bool maria_once_init_dummy(MARIA_SHARE *share
|
|
|
|
__attribute__((unused)),
|
|
|
|
File dfile __attribute__((unused)))
|
|
|
|
{
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static my_bool maria_once_end_dummy(MARIA_SHARE *share __attribute__((unused)))
|
|
|
|
{
|
|
|
|
return 0;
|
|
|
|
}
|