mariadb/sql/log_cache.h
Monty b9f5793176 MDEV-9101 Limit size of created disk temporary files and tables
Two new variables added:
- max_tmp_space_usage : Limits the the temporary space allowance per user
- max_total_tmp_space_usage: Limits the temporary space allowance for
  all users.

New status variables: tmp_space_used & max_tmp_space_used
New field in information_schema.process_list: TMP_SPACE_USED

The temporary space is counted for:
- All SQL level temporary files. This includes files for filesort,
  transaction temporary space, analyze, binlog_stmt_cache etc.
  It does not include engine internal temporary files used for repair,
  alter table, index pre sorting etc.
- All internal on disk temporary tables created as part of resolving a
  SELECT, multi-source update etc.

Special cases:
- When doing a commit, the last flush of the binlog_stmt_cache
  will not cause an error even if the temporary space limit is exceeded.
  This is to avoid giving errors on commit. This means that a user
  can temporary go over the limit with up to binlog_stmt_cache_size.

Noteworthy issue:
- One has to be careful when using small values for max_tmp_space_limit
  together with binary logging and with non transactional tables.
  If a the binary log entry for the query is bigger than
  binlog_stmt_cache_size and one hits the limit of max_tmp_space_limit
  when flushing the entry to disk, the query will abort and the
  binary log will not contain the last changes to the table.
  This will also stop the slave!
  This is also true for all Aria tables as Aria cannot do rollback
  (except in case of crashes)!
  One way to avoid it is to use @@binlog_format=statement for
  queries that updates a lot of rows.

Implementation:
- All writes to temporary files or internal temporary tables, that
  increases the file size, are routed through temp_file_size_cb_func()
  which updates and checks the temp space usage.
- Most of the temporary file monitoring is done inside IO_CACHE.
  Temporary file monitoring is done inside the Aria engine.
- MY_TRACK and MY_TRACK_WITH_LIMIT are new flags for ini_io_cache().
  MY_TRACK means that we track the file usage. TRACK_WITH_LIMIT means
  that we track the file usage and we give an error if the limit is
  breached. This is used to not give an error on commit when
  binlog_stmp_cache is flushed.
- global_tmp_space_used contains the total tmp space used so far.
  This is needed quickly check against max_total_tmp_space_usage.
- Temporary space errors are using EE_LOCAL_TMP_SPACE_FULL and
  handler errors are using HA_ERR_LOCAL_TMP_SPACE_FULL.
  This is needed until we move general errors to it's own error space
  so that they cannot conflict with system error numbers.
- Return value of my_chsize() and mysql_file_chsize() has changed
  so that -1 is returned in the case my_chsize() could not decrease
  the file size (very unlikely and will not happen on modern systems).
  All calls to _chsize() are updated to check for > 0 as the error
  condition.
- At the destruction of THD we check that THD::tmp_file_space == 0
- At server end we check that global_tmp_space_used == 0
- As a precaution against errors in the tmp_space_used code, one can set
  max_tmp_space_usage and max_total_tmp_space_usage to 0 to disable
  the tmp space quota errors.
- truncate_io_cache() function added.
- Aria tables using static or dynamic row length are registered in 8K
  increments to avoid some calls to update_tmp_file_size().

Other things:
- Ensure that all handler errors are registered.  Before, some engine
  errors could be printed as "Unknown error".
- Fixed bug in filesort() that causes a assert if there was an error
  when writing to the temporay file.
- Fixed that compute_window_func() now takes into account write errors.
- In case of parallel replication, rpl_group_info::cleanup_context()
  could call trans_rollback() with thd->error set, which would cause
  an assert. Fixed by resetting the error before calling trans_rollback().
- Fixed bug in subselect3.inc which caused following test to use
  heap tables with low value for max_heap_table_size
- Fixed bug in sql_expression_cache where it did not overflow
  heap table to Aria table.
- Added Max_tmp_disk_space_used to slow query log.
- Fixed some bugs in log_slow_innodb.test
2024-05-27 12:39:04 +02:00

277 lines
7.7 KiB
C++

/*
Copyright (c) 2023, MariaDB plc
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA
*/
#include "log_event.h"
static constexpr my_off_t MY_OFF_T_UNDEF= ~0ULL;
/** Truncate cache log files bigger than this */
static constexpr my_off_t CACHE_FILE_TRUNC_SIZE = 65536;
/*
Helper classes to store non-transactional and transactional data
before copying it to the binary log.
*/
class binlog_cache_data
{
public:
binlog_cache_data(bool precompute_checksums):
before_stmt_pos(MY_OFF_T_UNDEF), m_pending(0), status(0),
incident(FALSE), precompute_checksums(precompute_checksums),
saved_max_binlog_cache_size(0), ptr_binlog_cache_use(0),
ptr_binlog_cache_disk_use(0)
{
/*
Read the current checksum setting. We will use this setting to decide
whether to pre-compute checksums in the cache. Then when writing the cache
to the actual binlog, another check will be made and checksums recomputed
in the unlikely case that the setting changed meanwhile.
*/
checksum_opt= !precompute_checksums ? BINLOG_CHECKSUM_ALG_OFF :
(enum_binlog_checksum_alg)binlog_checksum_options;
}
~binlog_cache_data()
{
DBUG_ASSERT(empty());
close_cached_file(&cache_log);
}
/*
Return 1 if there is no relevant entries in the cache
This is:
- Cache is empty
- There are row or critical (DDL?) events in the cache
The status test is needed to avoid writing entries with only
a table map entry, which would crash in do_apply_event() on the slave
as it assumes that there is always a row entry after a table map.
*/
bool empty() const
{
return (pending() == NULL &&
(my_b_write_tell(&cache_log) == 0 ||
((status & (LOGGED_ROW_EVENT | LOGGED_CRITICAL)) == 0)));
}
Rows_log_event *pending() const
{
return m_pending;
}
void set_pending(Rows_log_event *const pending_arg)
{
m_pending= pending_arg;
}
void set_incident(void)
{
incident= TRUE;
}
bool has_incident(void) const
{
return(incident);
}
void reset()
{
bool cache_was_empty= empty();
bool truncate_file= (cache_log.file != -1 &&
my_b_write_tell(&cache_log) >
MY_MIN(CACHE_FILE_TRUNC_SIZE, binlog_stmt_cache_size));
truncate(0,1); // Forget what's in cache
checksum_opt= !precompute_checksums ? BINLOG_CHECKSUM_ALG_OFF :
(enum_binlog_checksum_alg)binlog_checksum_options;
if (!cache_was_empty)
compute_statistics();
if (truncate_file)
truncate_io_cache(&cache_log);
status= 0;
incident= FALSE;
before_stmt_pos= MY_OFF_T_UNDEF;
DBUG_ASSERT(empty());
}
my_off_t get_byte_position() const
{
return my_b_tell(&cache_log);
}
my_off_t get_prev_position() const
{
return(before_stmt_pos);
}
void set_prev_position(my_off_t pos)
{
before_stmt_pos= pos;
}
void restore_prev_position()
{
truncate(before_stmt_pos);
}
void restore_savepoint(my_off_t pos)
{
truncate(pos);
if (pos < before_stmt_pos)
before_stmt_pos= MY_OFF_T_UNDEF;
}
void set_binlog_cache_info(my_off_t param_max_binlog_cache_size,
ulong *param_ptr_binlog_cache_use,
ulong *param_ptr_binlog_cache_disk_use)
{
/*
The assertions guarantee that the set_binlog_cache_info is
called just once and information passed as parameters are
never zero.
This is done while calling the constructor binlog_cache_mngr.
We cannot set information in the constructor binlog_cache_data
because the space for binlog_cache_mngr is allocated through
a placement new.
In the future, we can refactor this and change it to avoid
the set_binlog_info.
*/
DBUG_ASSERT(saved_max_binlog_cache_size == 0);
DBUG_ASSERT(param_max_binlog_cache_size != 0);
DBUG_ASSERT(ptr_binlog_cache_use == 0);
DBUG_ASSERT(param_ptr_binlog_cache_use != 0);
DBUG_ASSERT(ptr_binlog_cache_disk_use == 0);
DBUG_ASSERT(param_ptr_binlog_cache_disk_use != 0);
saved_max_binlog_cache_size= param_max_binlog_cache_size;
ptr_binlog_cache_use= param_ptr_binlog_cache_use;
ptr_binlog_cache_disk_use= param_ptr_binlog_cache_disk_use;
cache_log.end_of_file= saved_max_binlog_cache_size;
}
void add_status(enum_logged_status status_arg)
{
status|= status_arg;
}
/*
Cache to store data before copying it to the binary log.
*/
IO_CACHE cache_log;
protected:
/*
Binlog position before the start of the current statement.
*/
my_off_t before_stmt_pos;
private:
/*
Pending binrows event. This event is the event where the rows are currently
written.
*/
Rows_log_event *m_pending;
/*
Bit flags for what has been writing to cache. Used to
discard logs without any data changes.
see enum_logged_status;
*/
uint32 status;
public:
/*
The algorithm (if any) used to pre-compute checksums in the cache.
Initialized from binlog_checksum_options when the cache is reset.
*/
enum_binlog_checksum_alg checksum_opt;
private:
/*
This indicates that some events did not get into the cache and most likely
it is corrupted.
*/
bool incident;
/* Whether the caller requested precomputing checksums. */
bool precompute_checksums;
/**
This function computes binlog cache and disk usage.
*/
void compute_statistics()
{
statistic_increment(*ptr_binlog_cache_use, &LOCK_status);
if (cache_log.disk_writes != 0)
{
#ifdef REAL_STATISTICS
statistic_add(*ptr_binlog_cache_disk_use,
cache_log.disk_writes, &LOCK_status);
#else
statistic_increment(*ptr_binlog_cache_disk_use, &LOCK_status);
#endif
cache_log.disk_writes= 0;
}
}
/*
Stores the values of maximum size of the cache allowed when this cache
is configured. This corresponds to either
. max_binlog_cache_size or max_binlog_stmt_cache_size.
*/
my_off_t saved_max_binlog_cache_size;
/*
Stores a pointer to the status variable that keeps track of the in-memory
cache usage. This corresponds to either
. binlog_cache_use or binlog_stmt_cache_use.
*/
ulong *ptr_binlog_cache_use;
/*
Stores a pointer to the status variable that keeps track of the disk
cache usage. This corresponds to either
. binlog_cache_disk_use or binlog_stmt_cache_disk_use.
*/
ulong *ptr_binlog_cache_disk_use;
/*
It truncates the cache to a certain position. This includes deleting the
pending event.
*/
void truncate(my_off_t pos, bool reset_cache=0)
{
DBUG_PRINT("info", ("truncating to position %lu", (ulong) pos));
cache_log.error=0;
if (pending())
{
delete pending();
set_pending(0);
}
my_bool res __attribute__((unused))=
reinit_io_cache(&cache_log, WRITE_CACHE, pos, 0, reset_cache);
DBUG_ASSERT(res == 0);
cache_log.end_of_file= saved_max_binlog_cache_size;
}
binlog_cache_data& operator=(const binlog_cache_data& info);
binlog_cache_data(const binlog_cache_data& info);
};