mirror of
https://github.com/MariaDB/server.git
synced 2025-01-16 12:02:42 +01:00
Added support for LZO compression method.
Removed: innodb_use_lz4 configuration parameter Added: innodb_compression_algorithm configuration parameter 0 = no compression, 1 = ZLIB, 2 = LZ4, 3 = LZO Fixed issue with incorrect trim calculations
This commit is contained in:
Jan Lindström
parent
88765c3b4d
commit
13c73c31c3
16 changed files with 297 additions and 137 deletions
|
|
@ -14,22 +14,22 @@
|
|||
|
||||
MACRO (MYSQL_CHECK_LZ4)
|
||||
|
||||
CHECK_INCLUDE_FILES(lz4.h HAVE_LZ4_H)
|
||||
CHECK_LIBRARY_EXISTS(liblz4.a LZ4_compress_limitedOutput "" HAVE_LZ4_LIB)
|
||||
|
||||
IF(HAVE_LZ4_LIB AND HAVE_LZ4_H)
|
||||
ADD_DEFINITIONS(-DHAVE_LZ4=1)
|
||||
LINK_LIBRARIES(liblz4.a)
|
||||
ENDIF()
|
||||
ENDMACRO()
|
||||
|
||||
MACRO (MYSQL_CHECK_SHARED_LZ4)
|
||||
|
||||
CHECK_INCLUDE_FILES(lz4.h HAVE_LZ4_H)
|
||||
CHECK_LIBRARY_EXISTS(lz4 LZ4_compress_limitedOutput "" HAVE_LZ4_SHARED_LIB)
|
||||
|
||||
IF (HAVE_LZ4_SHARED_LIB AND HAVE_LZ4_H)
|
||||
ADD_DEFINITIONS(-DHAVE_LZ4=1)
|
||||
LINK_LIBRARIES(lz4)
|
||||
LINK_LIBRARIES(lz4)
|
||||
ENDIF()
|
||||
ENDMACRO()
|
||||
|
||||
MACRO (MYSQL_CHECK_LZ4_STATIC)
|
||||
|
||||
CHECK_INCLUDE_FILES(lz4.h HAVE_LZ4_H)
|
||||
CHECK_LIBRARY_EXISTS(liblz4.a LZ4_compress_limitedOutput "" HAVE_LZ4_LIB)
|
||||
|
||||
IF(HAVE_LZ4_LIB AND HAVE_LZ4_H)
|
||||
ADD_DEFINITIONS(-DHAVE_LZ4=1)
|
||||
LINK_LIBRARIES(liblz4.a)
|
||||
ENDIF()
|
||||
ENDMACRO()
|
||||
35
cmake/lzo.cmake
Normal file
35
cmake/lzo.cmake
Normal file
|
|
@ -0,0 +1,35 @@
|
|||
# Copyright (C) 2014, SkySQL Ab. All Rights Reserved.
|
||||
#
|
||||
# This program is free software; you can redistribute it and/or modify it under
|
||||
# the terms of the GNU General Public License as published by the Free Software
|
||||
# Foundation; version 2 of the License.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful, but WITHOUT
|
||||
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
|
||||
# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License along with
|
||||
# this program; if not, write to the Free Software Foundation, Inc.,
|
||||
# 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
|
||||
|
||||
MACRO (MYSQL_CHECK_LZO_STATIC)
|
||||
|
||||
CHECK_INCLUDE_FILES(lzo/lzo1x.h HAVE_LZO_H)
|
||||
CHECK_LIBRARY_EXISTS(liblzo2.a lzo1x_1_compress "" HAVE_LZO_LIB)
|
||||
|
||||
IF(HAVE_LZO_LIB AND HAVE_LZO_H)
|
||||
ADD_DEFINITIONS(-DHAVE_LZO=1)
|
||||
LINK_LIBRARIES(liblzo2.a)
|
||||
ENDIF()
|
||||
ENDMACRO()
|
||||
|
||||
MACRO (MYSQL_CHECK_LZO)
|
||||
|
||||
CHECK_INCLUDE_FILES(lzo/lzo1x.h HAVE_LZO_H)
|
||||
CHECK_LIBRARY_EXISTS(lzo2 lzo1x_1_compress "" HAVE_LZO_LIB)
|
||||
|
||||
IF(HAVE_LZO_LIB AND HAVE_LZO_H)
|
||||
ADD_DEFINITIONS(-DHAVE_LZO=1)
|
||||
LINK_LIBRARIES(lzo2)
|
||||
ENDIF()
|
||||
ENDMACRO()
|
||||
|
|
@ -19,8 +19,10 @@ INCLUDE(CheckFunctionExists)
|
|||
INCLUDE(CheckCSourceCompiles)
|
||||
INCLUDE(CheckCSourceRuns)
|
||||
INCLUDE(lz4)
|
||||
INCLUDE(lzo)
|
||||
|
||||
MYSQL_CHECK_SHARED_LZ4()
|
||||
MYSQL_CHECK_LZ4()
|
||||
MYSQL_CHECK_LZO()
|
||||
|
||||
# OS tests
|
||||
IF(UNIX)
|
||||
|
|
|
|||
|
|
@ -66,6 +66,10 @@ static ulint srv_data_read, srv_data_written;
|
|||
#ifdef HAVE_LZ4
|
||||
#include "lz4.h"
|
||||
#endif
|
||||
#ifdef HAVE_LZO
|
||||
#include "lzo/lzo1x.h"
|
||||
#endif
|
||||
|
||||
|
||||
/****************************************************************//**
|
||||
For page compressed pages compress the page before actual write
|
||||
|
|
@ -81,7 +85,9 @@ fil_compress_page(
|
|||
byte* out_buf, /*!< out: compressed buffer */
|
||||
ulint len, /*!< in: length of input buffer.*/
|
||||
ulint compression_level, /* in: compression level */
|
||||
ulint* out_len) /*!< out: actual length of compressed page */
|
||||
ulint* out_len, /*!< out: actual length of compressed
|
||||
page */
|
||||
byte* lzo_mem) /*!< in: temporal memory used by LZO */
|
||||
{
|
||||
int err = Z_OK;
|
||||
int level = 0;
|
||||
|
|
@ -114,9 +120,11 @@ fil_compress_page(
|
|||
|
||||
write_size = UNIV_PAGE_SIZE - header_len;
|
||||
|
||||
switch(innodb_compression_algorithm) {
|
||||
#ifdef HAVE_LZ4
|
||||
if (srv_use_lz4) {
|
||||
err = LZ4_compress_limitedOutput((const char *)buf, (char *)out_buf+header_len, len, write_size);
|
||||
case PAGE_LZ4_ALGORITHM:
|
||||
err = LZ4_compress_limitedOutput((const char *)buf,
|
||||
(char *)out_buf+header_len, len, write_size);
|
||||
write_size = err;
|
||||
|
||||
if (err == 0) {
|
||||
|
|
@ -130,8 +138,25 @@ fil_compress_page(
|
|||
*out_len = len;
|
||||
return (buf);
|
||||
}
|
||||
} else {
|
||||
break;
|
||||
#endif /* HAVE_LZ4 */
|
||||
#ifdef HAVE_LZO
|
||||
case PAGE_LZO_ALGORITHM:
|
||||
err = lzo1x_1_15_compress(
|
||||
buf, len, out_buf+header_len, &write_size, lzo_mem);
|
||||
|
||||
if (err != LZO_E_OK || write_size > len) {
|
||||
fprintf(stderr,
|
||||
"InnoDB: Warning: Compression failed for space %lu name %s len %lu err %d write_size %lu",
|
||||
space_id, fil_space_name(space), len, err, write_size);
|
||||
srv_stats.pages_page_compression_error.inc();
|
||||
*out_len = len;
|
||||
return (buf);
|
||||
}
|
||||
|
||||
break;
|
||||
#endif /* HAVE_LZO */
|
||||
case PAGE_ZLIB_ALGORITHM:
|
||||
err = compress2(out_buf+header_len, (ulong*)&write_size, buf, len, level);
|
||||
|
||||
if (err != Z_OK) {
|
||||
|
|
@ -145,9 +170,12 @@ fil_compress_page(
|
|||
*out_len = len;
|
||||
return (buf);
|
||||
}
|
||||
#ifdef HAVE_LZ4
|
||||
break;
|
||||
|
||||
default:
|
||||
ut_error;
|
||||
break;
|
||||
}
|
||||
#endif /* HAVE_LZ4 */
|
||||
|
||||
/* Set up the page header */
|
||||
memcpy(out_buf, buf, FIL_PAGE_DATA);
|
||||
|
|
@ -156,18 +184,7 @@ fil_compress_page(
|
|||
/* Set up the correct page type */
|
||||
mach_write_to_2(out_buf+FIL_PAGE_TYPE, FIL_PAGE_PAGE_COMPRESSED);
|
||||
/* Set up the flush lsn to be compression algorithm */
|
||||
|
||||
#ifdef HAVE_LZ4
|
||||
if (srv_use_lz4) {
|
||||
mach_write_to_8(out_buf+FIL_PAGE_FILE_FLUSH_LSN, FIL_PAGE_COMPRESSION_LZ4);
|
||||
} else {
|
||||
#endif /* HAVE_LZ4 */
|
||||
mach_write_to_8(out_buf+FIL_PAGE_FILE_FLUSH_LSN, FIL_PAGE_COMPRESSION_ZLIB);
|
||||
|
||||
#ifdef HAVE_LZ4
|
||||
}
|
||||
#endif /* HAVE_LZ4 */
|
||||
|
||||
mach_write_to_8(out_buf+FIL_PAGE_FILE_FLUSH_LSN, innodb_compression_algorithm);
|
||||
/* Set up the actual payload lenght */
|
||||
mach_write_to_2(out_buf+FIL_PAGE_DATA, write_size);
|
||||
|
||||
|
|
@ -176,17 +193,7 @@ fil_compress_page(
|
|||
ut_ad(fil_page_is_compressed(out_buf));
|
||||
ut_ad(mach_read_from_4(out_buf+FIL_PAGE_SPACE_OR_CHKSUM) == BUF_NO_CHECKSUM_MAGIC);
|
||||
ut_ad(mach_read_from_2(out_buf+FIL_PAGE_DATA) == write_size);
|
||||
|
||||
#ifdef HAVE_LZ4
|
||||
if (srv_use_lz4) {
|
||||
ut_ad(mach_read_from_8(out_buf+FIL_PAGE_FILE_FLUSH_LSN) == FIL_PAGE_COMPRESSION_LZ4);
|
||||
} else {
|
||||
#endif /* HAVE_LZ4 */
|
||||
ut_ad(mach_read_from_8(out_buf+FIL_PAGE_FILE_FLUSH_LSN) == FIL_PAGE_COMPRESSION_ZLIB);
|
||||
|
||||
#ifdef HAVE_LZ4
|
||||
}
|
||||
#endif /* HAVE_LZ4 */
|
||||
ut_ad(mach_read_from_8(out_buf+FIL_PAGE_FILE_FLUSH_LSN) == innodb_compression_algorithm);
|
||||
#endif /* UNIV_DEBUG */
|
||||
|
||||
write_size+=header_len;
|
||||
|
|
@ -207,11 +214,6 @@ fil_compress_page(
|
|||
|
||||
|
||||
srv_stats.page_compression_saved.add((len - write_size));
|
||||
if ((len - write_size) > 0) {
|
||||
srv_stats.page_compression_trim_sect512.add(((len - write_size) / SECT_SIZE));
|
||||
srv_stats.page_compression_trim_sect4096.add(((len - write_size) / (SECT_SIZE*8)));
|
||||
}
|
||||
|
||||
srv_stats.pages_page_compressed.inc();
|
||||
*out_len = write_size;
|
||||
|
||||
|
|
@ -236,6 +238,7 @@ fil_decompress_page(
|
|||
ulint actual_size = 0;
|
||||
ulint compression_alg = 0;
|
||||
byte *in_buf;
|
||||
ulint olen=0;
|
||||
|
||||
ut_ad(buf);
|
||||
ut_ad(len);
|
||||
|
|
@ -287,16 +290,16 @@ fil_decompress_page(
|
|||
*write_size = actual_size;
|
||||
}
|
||||
|
||||
if (compression_alg == FIL_PAGE_COMPRESSION_ZLIB) {
|
||||
|
||||
#ifdef UNIV_PAGECOMPRESS_DEBUG
|
||||
fprintf(stderr,
|
||||
"InnoDB: Note: Preparing for decompress for len %lu\n",
|
||||
actual_size);
|
||||
fprintf(stderr,
|
||||
"InnoDB: Note: Preparing for decompress for len %lu\n",
|
||||
actual_size);
|
||||
#endif /* UNIV_PAGECOMPRESS_DEBUG */
|
||||
|
||||
err= uncompress(in_buf, &len, buf+FIL_PAGE_DATA+FIL_PAGE_COMPRESSED_SIZE, (unsigned long)actual_size);
|
||||
|
||||
switch(compression_alg) {
|
||||
case PAGE_ZLIB_ALGORITHM:
|
||||
err= uncompress(in_buf, &len, buf+FIL_PAGE_DATA+FIL_PAGE_COMPRESSED_SIZE, (unsigned long)actual_size);
|
||||
|
||||
/* If uncompress fails it means that page is corrupted */
|
||||
if (err != Z_OK) {
|
||||
|
|
@ -311,14 +314,10 @@ fil_decompress_page(
|
|||
|
||||
ut_error;
|
||||
}
|
||||
break;
|
||||
|
||||
#ifdef UNIV_PAGECOMPRESS_DEBUG
|
||||
fprintf(stderr,
|
||||
"InnoDB: Note: Decompression succeeded for len %lu \n",
|
||||
len);
|
||||
#endif /* UNIV_PAGECOMPRESS_DEBUG */
|
||||
#ifdef HAVE_LZ4
|
||||
} else if (compression_alg == FIL_PAGE_COMPRESSION_LZ4) {
|
||||
case PAGE_LZ4_ALGORITHM:
|
||||
err = LZ4_decompress_fast((const char *)buf+FIL_PAGE_DATA+FIL_PAGE_COMPRESSED_SIZE, (char *)in_buf, UNIV_PAGE_SIZE);
|
||||
|
||||
if (err != (int)actual_size) {
|
||||
|
|
@ -331,8 +330,26 @@ fil_decompress_page(
|
|||
|
||||
ut_error;
|
||||
}
|
||||
break;
|
||||
#endif /* HAVE_LZ4 */
|
||||
} else {
|
||||
#ifdef HAVE_LZO
|
||||
case PAGE_LZO_ALGORITHM:
|
||||
err = lzo1x_decompress((const unsigned char *)buf+FIL_PAGE_DATA+FIL_PAGE_COMPRESSED_SIZE,
|
||||
actual_size,(unsigned char *)in_buf, &olen, NULL);
|
||||
|
||||
if (err != LZO_E_OK || (olen == 0 || olen > UNIV_PAGE_SIZE)) {
|
||||
fprintf(stderr,
|
||||
"InnoDB: Corruption: Page is marked as compressed\n"
|
||||
"InnoDB: but decompression read only %d bytes.\n"
|
||||
"InnoDB: size %lu len %lu\n",
|
||||
olen, actual_size, len);
|
||||
fflush(stderr);
|
||||
|
||||
ut_error;
|
||||
}
|
||||
break;
|
||||
#endif
|
||||
default:
|
||||
fprintf(stderr,
|
||||
"InnoDB: Corruption: Page is marked as compressed\n"
|
||||
"InnoDB: but compression algorithm %s\n"
|
||||
|
|
@ -341,8 +358,15 @@ fil_decompress_page(
|
|||
|
||||
fflush(stderr);
|
||||
ut_error;
|
||||
break;
|
||||
}
|
||||
|
||||
#ifdef UNIV_PAGECOMPRESS_DEBUG
|
||||
fprintf(stderr,
|
||||
"InnoDB: Note: Decompression succeeded for len %lu \n",
|
||||
len);
|
||||
#endif /* UNIV_PAGECOMPRESS_DEBUG */
|
||||
|
||||
srv_stats.pages_page_decompressed.inc();
|
||||
|
||||
/* Copy the uncompressed page to the buffer pool, not
|
||||
|
|
|
|||
|
|
@ -101,6 +101,7 @@ this program; if not, write to the Free Software Foundation, Inc.,
|
|||
#endif /* UNIV_DEBUG */
|
||||
#include "fts0priv.h"
|
||||
#include "page0zip.h"
|
||||
#include "fil0pagecompress.h"
|
||||
|
||||
#define thd_get_trx_isolation(X) ((enum_tx_isolation)thd_tx_isolation(X))
|
||||
|
||||
|
|
@ -16793,12 +16794,20 @@ static MYSQL_SYSVAR_BOOL(use_trim, srv_use_trim,
|
|||
"Use trim. Default FALSE.",
|
||||
NULL, NULL, FALSE);
|
||||
|
||||
#ifdef HAVE_LZ4
|
||||
static MYSQL_SYSVAR_BOOL(use_lz4, srv_use_lz4,
|
||||
PLUGIN_VAR_OPCMDARG ,
|
||||
"Use LZ4 for page compression",
|
||||
NULL, NULL, FALSE);
|
||||
#endif /* HAVE_LZ4 */
|
||||
static MYSQL_SYSVAR_LONG(compression_algorithm, innodb_compression_algorithm,
|
||||
PLUGIN_VAR_OPCMDARG,
|
||||
"Compression algorithm used on page compression. 1 for zlib, 2 for lz3, 3 for lzo",
|
||||
NULL, NULL,
|
||||
PAGE_ZLIB_ALGORITHM,
|
||||
0,
|
||||
#if defined(HAVE_LZO) && defined(HAVE_LZ4)
|
||||
PAGE_ALGORITHM_LAST,
|
||||
#elif defined(HAVE_LZ4) && !defined(HAVE_LZO)
|
||||
PAGE_ALGORITHM_LZ4,
|
||||
#else
|
||||
PAGE_ALGORITHM_ZLIB,
|
||||
#endif
|
||||
0);
|
||||
|
||||
static MYSQL_SYSVAR_LONG(mtflush_threads, srv_mtflush_threads,
|
||||
PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY,
|
||||
|
|
@ -16967,9 +16976,7 @@ static struct st_mysql_sys_var* innobase_system_variables[]= {
|
|||
MYSQL_SYSVAR(trx_purge_view_update_only_debug),
|
||||
#endif /* UNIV_DEBUG */
|
||||
MYSQL_SYSVAR(use_trim),
|
||||
#ifdef HAVE_LZ4
|
||||
MYSQL_SYSVAR(use_lz4),
|
||||
#endif
|
||||
MYSQL_SYSVAR(compression_algorithm),
|
||||
MYSQL_SYSVAR(mtflush_threads),
|
||||
MYSQL_SYSVAR(use_mtflush),
|
||||
NULL
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
/*****************************************************************************
|
||||
|
||||
Copyright (C) 2013 SkySQL Ab. All Rights Reserved.
|
||||
Copyright (C) 2013, 2014 SkySQL Ab. All Rights Reserved.
|
||||
|
||||
This program is free software; you can redistribute it and/or modify it under
|
||||
the terms of the GNU General Public License as published by the Free Software
|
||||
|
|
@ -22,6 +22,12 @@ this program; if not, write to the Free Software Foundation, Inc.,
|
|||
#include "fsp0fsp.h"
|
||||
#include "fsp0pagecompress.h"
|
||||
|
||||
#define PAGE_UNCOMPRESSED 0
|
||||
#define PAGE_ZLIB_ALGORITHM 1
|
||||
#define PAGE_LZ4_ALGORITHM 2
|
||||
#define PAGE_LZO_ALGORITHM 3
|
||||
#define PAGE_ALGORITHM_LAST PAGE_LZO_ALGORITHM
|
||||
|
||||
/******************************************************************//**
|
||||
@file include/fil0pagecompress.h
|
||||
Helper functions for extracting/storing page compression and
|
||||
|
|
@ -85,7 +91,9 @@ fil_compress_page(
|
|||
byte* out_buf, /*!< out: compressed buffer */
|
||||
ulint len, /*!< in: length of input buffer.*/
|
||||
ulint compression_level, /*!< in: compression level */
|
||||
ulint* out_len); /*!< out: actual length of compressed page */
|
||||
ulint* out_len, /*!< out: actual length of compressed
|
||||
page */
|
||||
byte* lzo_mem); /*!< in: temporal memory used by LZO */
|
||||
|
||||
/****************************************************************//**
|
||||
For page compressed pages decompress the page after actual read
|
||||
|
|
|
|||
|
|
@ -249,8 +249,8 @@ extern my_bool srv_use_posix_fallocate;
|
|||
/* Use atomic writes i.e disable doublewrite buffer */
|
||||
extern my_bool srv_use_atomic_writes;
|
||||
|
||||
/* If this flag IS TRUE, then we use lz4 to compress/decompress pages */
|
||||
extern my_bool srv_use_lz4;
|
||||
/* Compression algorithm*/
|
||||
extern long innodb_compression_algorithm;
|
||||
|
||||
/* Number of flush threads */
|
||||
#define MTFLUSH_MAX_WORKER 64
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@
|
|||
|
||||
Copyright (c) 1995, 2013, Oracle and/or its affiliates. All Rights Reserved.
|
||||
Copyright (c) 2009, Percona Inc.
|
||||
Copyright (c) 2013, SkySQL Ab. All Rights Reserved.
|
||||
Copyright (c) 2013, 2014, SkySQL Ab. All Rights Reserved.
|
||||
|
||||
Portions of this file contain modifications contributed and copyrighted
|
||||
by Percona Inc.. Those modifications are
|
||||
|
|
@ -74,6 +74,10 @@ Created 10/21/1995 Heikki Tuuri
|
|||
# endif
|
||||
#endif
|
||||
|
||||
#ifdef HAVE_LZO
|
||||
#include "lzo/lzo1x.h"
|
||||
#endif
|
||||
|
||||
/** Insert buffer segment id */
|
||||
static const ulint IO_IBUF_SEGMENT = 0;
|
||||
|
||||
|
|
@ -221,6 +225,12 @@ struct os_aio_slot_t{
|
|||
int n_bytes; /* bytes written/read. */
|
||||
int ret; /* AIO return code */
|
||||
#endif /* WIN_ASYNC_IO */
|
||||
#ifdef HAVE_LZO
|
||||
byte lzo_mem[LZO1X_1_15_MEM_COMPRESS];
|
||||
#else
|
||||
byte lzo_mem; /* Temporal memory used by LZO */
|
||||
#endif
|
||||
|
||||
};
|
||||
|
||||
/** The asynchronous i/o array structure */
|
||||
|
|
@ -4480,7 +4490,15 @@ found:
|
|||
|
||||
ut_ad(slot->page_buf);
|
||||
|
||||
tmp = fil_compress_page(fil_node_get_space_id(slot->message1), (byte *)buf, slot->page_buf, len, page_compression_level, &real_len);
|
||||
/* Call page compression */
|
||||
tmp = fil_compress_page(fil_node_get_space_id(slot->message1),
|
||||
(byte *)buf,
|
||||
slot->page_buf,
|
||||
len,
|
||||
page_compression_level,
|
||||
&real_len,
|
||||
slot->lzo_mem
|
||||
);
|
||||
|
||||
/* If compression succeeded, set up the length and buffer */
|
||||
if (tmp != buf) {
|
||||
|
|
|
|||
|
|
@ -71,6 +71,7 @@ Created 10/8/1995 Heikki Tuuri
|
|||
|
||||
#include "mysql/plugin.h"
|
||||
#include "mysql/service_thd_wait.h"
|
||||
#include "fil0pagecompress.h"
|
||||
|
||||
/* The following is the maximum allowed duration of a lock wait. */
|
||||
UNIV_INTERN ulint srv_fatal_semaphore_wait_threshold = 600;
|
||||
|
|
@ -154,7 +155,7 @@ UNIV_INTERN my_bool srv_use_posix_fallocate = FALSE;
|
|||
/* If this flag is TRUE, then we disable doublewrite buffer */
|
||||
UNIV_INTERN my_bool srv_use_atomic_writes = FALSE;
|
||||
/* If this flag IS TRUE, then we use lz4 to compress/decompress pages */
|
||||
UNIV_INTERN my_bool srv_use_lz4 = FALSE;
|
||||
UNIV_INTERN long innodb_compression_algorithm = PAGE_ZLIB_ALGORITHM;
|
||||
/* Number of threads used for multi-threaded flush */
|
||||
UNIV_INTERN long srv_mtflush_threads = MTFLUSH_DEFAULT_WORKER;
|
||||
/* If this flag is TRUE, then we will use multi threaded flush. */
|
||||
|
|
|
|||
|
|
@ -19,8 +19,10 @@ INCLUDE(CheckFunctionExists)
|
|||
INCLUDE(CheckCSourceCompiles)
|
||||
INCLUDE(CheckCSourceRuns)
|
||||
INCLUDE(lz4)
|
||||
INCLUDE(lzo)
|
||||
|
||||
MYSQL_CHECK_LZ4()
|
||||
MYSQL_CHECK_LZ4_STATIC()
|
||||
MYSQL_CHECK_LZO_STATIC()
|
||||
|
||||
# OS tests
|
||||
IF(UNIX)
|
||||
|
|
|
|||
|
|
@ -66,6 +66,10 @@ static ulint srv_data_read, srv_data_written;
|
|||
#ifdef HAVE_LZ4
|
||||
#include "lz4.h"
|
||||
#endif
|
||||
#ifdef HAVE_LZO
|
||||
#include "lzo/lzo1x.h"
|
||||
#endif
|
||||
|
||||
|
||||
/****************************************************************//**
|
||||
For page compressed pages compress the page before actual write
|
||||
|
|
@ -81,7 +85,9 @@ fil_compress_page(
|
|||
byte* out_buf, /*!< out: compressed buffer */
|
||||
ulint len, /*!< in: length of input buffer.*/
|
||||
ulint compression_level, /* in: compression level */
|
||||
ulint* out_len) /*!< out: actual length of compressed page */
|
||||
ulint* out_len, /*!< out: actual length of compressed
|
||||
page */
|
||||
byte* lzo_mem) /*!< in: temporal memory used by LZO */
|
||||
{
|
||||
int err = Z_OK;
|
||||
int level = 0;
|
||||
|
|
@ -114,9 +120,11 @@ fil_compress_page(
|
|||
|
||||
write_size = UNIV_PAGE_SIZE - header_len;
|
||||
|
||||
switch(innodb_compression_algorithm) {
|
||||
#ifdef HAVE_LZ4
|
||||
if (srv_use_lz4) {
|
||||
err = LZ4_compress_limitedOutput((const char *)buf, (char *)out_buf+header_len, len, write_size);
|
||||
case PAGE_LZ4_ALGORITHM:
|
||||
err = LZ4_compress_limitedOutput((const char *)buf,
|
||||
(char *)out_buf+header_len, len, write_size);
|
||||
write_size = err;
|
||||
|
||||
if (err == 0) {
|
||||
|
|
@ -130,9 +138,26 @@ fil_compress_page(
|
|||
*out_len = len;
|
||||
return (buf);
|
||||
}
|
||||
} else {
|
||||
break;
|
||||
#endif /* HAVE_LZ4 */
|
||||
err = compress2(out_buf+header_len, (ulong *)&write_size, buf, len, level);
|
||||
#ifdef HAVE_LZO
|
||||
case PAGE_LZO_ALGORITHM:
|
||||
err = lzo1x_1_15_compress(
|
||||
buf, len, out_buf+header_len, &write_size, lzo_mem);
|
||||
|
||||
if (err != LZO_E_OK || write_size > len) {
|
||||
fprintf(stderr,
|
||||
"InnoDB: Warning: Compression failed for space %lu name %s len %lu err %d write_size %lu",
|
||||
space_id, fil_space_name(space), len, err, write_size);
|
||||
srv_stats.pages_page_compression_error.inc();
|
||||
*out_len = len;
|
||||
return (buf);
|
||||
}
|
||||
|
||||
break;
|
||||
#endif /* HAVE_LZO */
|
||||
case PAGE_ZLIB_ALGORITHM:
|
||||
err = compress2(out_buf+header_len, (ulong*)&write_size, buf, len, level);
|
||||
|
||||
if (err != Z_OK) {
|
||||
/* If error we leave the actual page as it was */
|
||||
|
|
@ -145,9 +170,12 @@ fil_compress_page(
|
|||
*out_len = len;
|
||||
return (buf);
|
||||
}
|
||||
#ifdef HAVE_LZ4
|
||||
break;
|
||||
|
||||
default:
|
||||
ut_error;
|
||||
break;
|
||||
}
|
||||
#endif /* HAVE_LZ4 */
|
||||
|
||||
/* Set up the page header */
|
||||
memcpy(out_buf, buf, FIL_PAGE_DATA);
|
||||
|
|
@ -156,15 +184,7 @@ fil_compress_page(
|
|||
/* Set up the correct page type */
|
||||
mach_write_to_2(out_buf+FIL_PAGE_TYPE, FIL_PAGE_PAGE_COMPRESSED);
|
||||
/* Set up the flush lsn to be compression algorithm */
|
||||
#ifdef HAVE_LZ4
|
||||
if (srv_use_lz4) {
|
||||
mach_write_to_8(out_buf+FIL_PAGE_FILE_FLUSH_LSN, FIL_PAGE_COMPRESSION_LZ4);
|
||||
} else {
|
||||
#endif /* HAVE_LZ4 */
|
||||
mach_write_to_8(out_buf+FIL_PAGE_FILE_FLUSH_LSN, FIL_PAGE_COMPRESSION_ZLIB);
|
||||
#ifdef HAVE_LZ4
|
||||
}
|
||||
#endif /* HAVE_LZ4 */
|
||||
mach_write_to_8(out_buf+FIL_PAGE_FILE_FLUSH_LSN, innodb_compression_algorithm);
|
||||
/* Set up the actual payload lenght */
|
||||
mach_write_to_2(out_buf+FIL_PAGE_DATA, write_size);
|
||||
|
||||
|
|
@ -173,16 +193,7 @@ fil_compress_page(
|
|||
ut_ad(fil_page_is_compressed(out_buf));
|
||||
ut_ad(mach_read_from_4(out_buf+FIL_PAGE_SPACE_OR_CHKSUM) == BUF_NO_CHECKSUM_MAGIC);
|
||||
ut_ad(mach_read_from_2(out_buf+FIL_PAGE_DATA) == write_size);
|
||||
|
||||
#ifdef HAVE_LZ4
|
||||
if (srv_use_lz4) {
|
||||
ut_ad(mach_read_from_8(out_buf+FIL_PAGE_FILE_FLUSH_LSN) == FIL_PAGE_COMPRESSION_LZ4);
|
||||
} else {
|
||||
#endif /* HAVE_LZ4 */
|
||||
ut_ad(mach_read_from_8(out_buf+FIL_PAGE_FILE_FLUSH_LSN) == FIL_PAGE_COMPRESSION_ZLIB);
|
||||
#ifdef HAVE_LZ4
|
||||
}
|
||||
#endif /* HAVE_LZ4 */
|
||||
ut_ad(mach_read_from_8(out_buf+FIL_PAGE_FILE_FLUSH_LSN) == innodb_compression_algorithm);
|
||||
#endif /* UNIV_DEBUG */
|
||||
|
||||
write_size+=header_len;
|
||||
|
|
@ -203,11 +214,6 @@ fil_compress_page(
|
|||
|
||||
|
||||
srv_stats.page_compression_saved.add((len - write_size));
|
||||
if ((len - write_size) > 0) {
|
||||
srv_stats.page_compression_trim_sect512.add(((len - write_size) / SECT_SIZE));
|
||||
srv_stats.page_compression_trim_sect4096.add(((len - write_size) / (SECT_SIZE*8)));
|
||||
}
|
||||
|
||||
srv_stats.pages_page_compressed.inc();
|
||||
*out_len = write_size;
|
||||
|
||||
|
|
@ -232,6 +238,7 @@ fil_decompress_page(
|
|||
ulint actual_size = 0;
|
||||
ulint compression_alg = 0;
|
||||
byte *in_buf;
|
||||
ulint olen=0;
|
||||
|
||||
ut_ad(buf);
|
||||
ut_ad(len);
|
||||
|
|
@ -258,7 +265,7 @@ fil_decompress_page(
|
|||
if (page_buf == NULL) {
|
||||
#ifdef UNIV_PAGECOMPRESS_DEBUG
|
||||
fprintf(stderr,
|
||||
"InnoDB: FIL: Note: Compression buffer not given, allocating...\n");
|
||||
"InnoDB: Note: FIL: Compression buffer not given, allocating...\n");
|
||||
#endif /* UNIV_PAGECOMPRESS_DEBUG */
|
||||
in_buf = static_cast<byte *>(ut_malloc(UNIV_PAGE_SIZE));
|
||||
} else {
|
||||
|
|
@ -283,14 +290,15 @@ fil_decompress_page(
|
|||
*write_size = actual_size;
|
||||
}
|
||||
|
||||
if (compression_alg == FIL_PAGE_COMPRESSION_ZLIB) {
|
||||
|
||||
#ifdef UNIV_PAGECOMPRESS_DEBUG
|
||||
fprintf(stderr,
|
||||
"InnoDB: Note: Preparing for decompress for len %lu\n",
|
||||
actual_size);
|
||||
fprintf(stderr,
|
||||
"InnoDB: Note: Preparing for decompress for len %lu\n",
|
||||
actual_size);
|
||||
#endif /* UNIV_PAGECOMPRESS_DEBUG */
|
||||
|
||||
|
||||
switch(compression_alg) {
|
||||
case PAGE_ZLIB_ALGORITHM:
|
||||
err= uncompress(in_buf, &len, buf+FIL_PAGE_DATA+FIL_PAGE_COMPRESSED_SIZE, (unsigned long)actual_size);
|
||||
|
||||
/* If uncompress fails it means that page is corrupted */
|
||||
|
|
@ -306,14 +314,10 @@ fil_decompress_page(
|
|||
|
||||
ut_error;
|
||||
}
|
||||
break;
|
||||
|
||||
#ifdef UNIV_PAGECOMPRESS_DEBUG
|
||||
fprintf(stderr,
|
||||
"InnoDB: Note: Decompression succeeded for len %lu \n",
|
||||
len);
|
||||
#endif /* UNIV_PAGECOMPRESS_DEBUG */
|
||||
#ifdef HAVE_LZ4
|
||||
} else if (compression_alg == FIL_PAGE_COMPRESSION_LZ4) {
|
||||
case PAGE_LZ4_ALGORITHM:
|
||||
err = LZ4_decompress_fast((const char *)buf+FIL_PAGE_DATA+FIL_PAGE_COMPRESSED_SIZE, (char *)in_buf, UNIV_PAGE_SIZE);
|
||||
|
||||
if (err != (int)actual_size) {
|
||||
|
|
@ -326,8 +330,26 @@ fil_decompress_page(
|
|||
|
||||
ut_error;
|
||||
}
|
||||
break;
|
||||
#endif /* HAVE_LZ4 */
|
||||
} else {
|
||||
#ifdef HAVE_LZO
|
||||
case PAGE_LZO_ALGORITHM:
|
||||
err = lzo1x_decompress((const unsigned char *)buf+FIL_PAGE_DATA+FIL_PAGE_COMPRESSED_SIZE,
|
||||
actual_size,(unsigned char *)in_buf, &olen, NULL);
|
||||
|
||||
if (err != LZO_E_OK || (olen == 0 || olen > UNIV_PAGE_SIZE)) {
|
||||
fprintf(stderr,
|
||||
"InnoDB: Corruption: Page is marked as compressed\n"
|
||||
"InnoDB: but decompression read only %d bytes.\n"
|
||||
"InnoDB: size %lu len %lu\n",
|
||||
olen, actual_size, len);
|
||||
fflush(stderr);
|
||||
|
||||
ut_error;
|
||||
}
|
||||
break;
|
||||
#endif
|
||||
default:
|
||||
fprintf(stderr,
|
||||
"InnoDB: Corruption: Page is marked as compressed\n"
|
||||
"InnoDB: but compression algorithm %s\n"
|
||||
|
|
@ -336,8 +358,15 @@ fil_decompress_page(
|
|||
|
||||
fflush(stderr);
|
||||
ut_error;
|
||||
break;
|
||||
}
|
||||
|
||||
#ifdef UNIV_PAGECOMPRESS_DEBUG
|
||||
fprintf(stderr,
|
||||
"InnoDB: Note: Decompression succeeded for len %lu \n",
|
||||
len);
|
||||
#endif /* UNIV_PAGECOMPRESS_DEBUG */
|
||||
|
||||
srv_stats.pages_page_decompressed.inc();
|
||||
|
||||
/* Copy the uncompressed page to the buffer pool, not
|
||||
|
|
|
|||
|
|
@ -103,6 +103,7 @@ this program; if not, write to the Free Software Foundation, Inc.,
|
|||
#endif /* UNIV_DEBUG */
|
||||
#include "fts0priv.h"
|
||||
#include "page0zip.h"
|
||||
#include "fil0pagecompress.h"
|
||||
|
||||
#define thd_get_trx_isolation(X) ((enum_tx_isolation)thd_tx_isolation(X))
|
||||
|
||||
|
|
@ -17947,12 +17948,20 @@ static MYSQL_SYSVAR_BOOL(use_trim, srv_use_trim,
|
|||
"Use trim. Default FALSE.",
|
||||
NULL, NULL, FALSE);
|
||||
|
||||
#ifdef HAVE_LZ4
|
||||
static MYSQL_SYSVAR_BOOL(use_lz4, srv_use_lz4,
|
||||
PLUGIN_VAR_OPCMDARG ,
|
||||
"Use LZ4 for page compression",
|
||||
NULL, NULL, FALSE);
|
||||
#endif /* HAVE_LZ4 */
|
||||
static MYSQL_SYSVAR_LONG(compression_algorithm, innodb_compression_algorithm,
|
||||
PLUGIN_VAR_OPCMDARG,
|
||||
"Compression algorithm used on page compression. 1 for zlib, 2 for lz3, 3 for lzo",
|
||||
NULL, NULL,
|
||||
PAGE_ZLIB_ALGORITHM,
|
||||
0,
|
||||
#if defined(HAVE_LZO) && defined(HAVE_LZ4)
|
||||
PAGE_ALGORITHM_LAST,
|
||||
#elif defined(HAVE_LZ4) && !defined(HAVE_LZO)
|
||||
PAGE_ALGORITHM_LZ4,
|
||||
#else
|
||||
PAGE_ALGORITHM_ZLIB,
|
||||
#endif
|
||||
0);
|
||||
|
||||
static MYSQL_SYSVAR_LONG(mtflush_threads, srv_mtflush_threads,
|
||||
PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY,
|
||||
|
|
@ -18159,9 +18168,7 @@ static struct st_mysql_sys_var* innobase_system_variables[]= {
|
|||
MYSQL_SYSVAR(locking_fake_changes),
|
||||
MYSQL_SYSVAR(use_stacktrace),
|
||||
MYSQL_SYSVAR(use_trim),
|
||||
#ifdef HAVE_LZ4
|
||||
MYSQL_SYSVAR(use_lz4),
|
||||
#endif
|
||||
MYSQL_SYSVAR(compression_algorithm),
|
||||
MYSQL_SYSVAR(mtflush_threads),
|
||||
MYSQL_SYSVAR(use_mtflush),
|
||||
NULL
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
/*****************************************************************************
|
||||
|
||||
Copyright (C) 2013 SkySQL Ab. All Rights Reserved.
|
||||
Copyright (C) 2013, 2014 SkySQL Ab. All Rights Reserved.
|
||||
|
||||
This program is free software; you can redistribute it and/or modify it under
|
||||
the terms of the GNU General Public License as published by the Free Software
|
||||
|
|
@ -22,6 +22,12 @@ this program; if not, write to the Free Software Foundation, Inc.,
|
|||
#include "fsp0fsp.h"
|
||||
#include "fsp0pagecompress.h"
|
||||
|
||||
#define PAGE_UNCOMPRESSED 0
|
||||
#define PAGE_ZLIB_ALGORITHM 1
|
||||
#define PAGE_LZ4_ALGORITHM 2
|
||||
#define PAGE_LZO_ALGORITHM 3
|
||||
#define PAGE_ALGORITHM_LAST PAGE_LZO_ALGORITHM
|
||||
|
||||
/******************************************************************//**
|
||||
@file include/fil0pagecompress.h
|
||||
Helper functions for extracting/storing page compression and
|
||||
|
|
@ -85,7 +91,9 @@ fil_compress_page(
|
|||
byte* out_buf, /*!< out: compressed buffer */
|
||||
ulint len, /*!< in: length of input buffer.*/
|
||||
ulint compression_level, /*!< in: compression level */
|
||||
ulint* out_len); /*!< out: actual length of compressed page */
|
||||
ulint* out_len, /*!< out: actual length of compressed
|
||||
page */
|
||||
byte* lzo_mem); /*!< in: temporal memory used by LZO */
|
||||
|
||||
/****************************************************************//**
|
||||
For page compressed pages decompress the page after actual read
|
||||
|
|
|
|||
|
|
@ -269,8 +269,8 @@ extern my_bool srv_use_posix_fallocate;
|
|||
/* Use atomic writes i.e disable doublewrite buffer */
|
||||
extern my_bool srv_use_atomic_writes;
|
||||
|
||||
/* If this flag IS TRUE, then we use lz4 to compress/decompress pages */
|
||||
extern my_bool srv_use_lz4;
|
||||
/* Compression algorithm*/
|
||||
extern long innodb_compression_algorithm;
|
||||
|
||||
/* Number of flush threads */
|
||||
#define MTFLUSH_MAX_WORKER 64
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@
|
|||
|
||||
Copyright (c) 1995, 2013, Oracle and/or its affiliates. All Rights Reserved.
|
||||
Copyright (c) 2009, Percona Inc.
|
||||
Copyright (c) 2013, SkySQL Ab. All Rights Reserved.
|
||||
Copyright (c) 2013, 2014, SkySQL Ab. All Rights Reserved.
|
||||
|
||||
Portions of this file contain modifications contributed and copyrighted
|
||||
by Percona Inc.. Those modifications are
|
||||
|
|
@ -80,6 +80,10 @@ Created 10/21/1995 Heikki Tuuri
|
|||
# endif
|
||||
#endif
|
||||
|
||||
#ifdef HAVE_LZO
|
||||
#include "lzo/lzo1x.h"
|
||||
#endif
|
||||
|
||||
/** Insert buffer segment id */
|
||||
static const ulint IO_IBUF_SEGMENT = 0;
|
||||
|
||||
|
|
@ -230,6 +234,12 @@ struct os_aio_slot_t{
|
|||
int n_bytes; /* bytes written/read. */
|
||||
int ret; /* AIO return code */
|
||||
#endif /* WIN_ASYNC_IO */
|
||||
#ifdef HAVE_LZO
|
||||
byte lzo_mem[LZO1X_1_15_MEM_COMPRESS];
|
||||
#else
|
||||
byte lzo_mem; /* Temporal memory used by LZO */
|
||||
#endif
|
||||
|
||||
};
|
||||
|
||||
/** The asynchronous i/o array structure */
|
||||
|
|
@ -4596,7 +4606,15 @@ found:
|
|||
|
||||
ut_ad(slot->page_buf);
|
||||
|
||||
tmp = fil_compress_page(fil_node_get_space_id(slot->message1), (byte *)buf, slot->page_buf, len, page_compression_level, &real_len);
|
||||
/* Call page compression */
|
||||
tmp = fil_compress_page(fil_node_get_space_id(slot->message1),
|
||||
(byte *)buf,
|
||||
slot->page_buf,
|
||||
len,
|
||||
page_compression_level,
|
||||
&real_len,
|
||||
slot->lzo_mem
|
||||
);
|
||||
|
||||
/* If compression succeeded, set up the length and buffer */
|
||||
if (tmp != buf) {
|
||||
|
|
|
|||
|
|
@ -73,6 +73,7 @@ Created 10/8/1995 Heikki Tuuri
|
|||
|
||||
#include "mysql/plugin.h"
|
||||
#include "mysql/service_thd_wait.h"
|
||||
#include "fil0pagecompress.h"
|
||||
|
||||
/* prototypes of new functions added to ha_innodb.cc for kill_idle_transaction */
|
||||
ibool innobase_thd_is_idle(const void* thd);
|
||||
|
|
@ -172,7 +173,7 @@ UNIV_INTERN my_bool srv_use_posix_fallocate = FALSE;
|
|||
/* If this flag is TRUE, then we disable doublewrite buffer */
|
||||
UNIV_INTERN my_bool srv_use_atomic_writes = FALSE;
|
||||
/* If this flag IS TRUE, then we use lz4 to compress/decompress pages */
|
||||
UNIV_INTERN my_bool srv_use_lz4 = FALSE;
|
||||
UNIV_INTERN long innodb_compression_algorithm = PAGE_ZLIB_ALGORITHM;
|
||||
/* Number of threads used for multi-threaded flush */
|
||||
UNIV_INTERN long srv_mtflush_threads = MTFLUSH_DEFAULT_WORKER;
|
||||
/* If this flag is TRUE, then we will use multi threaded flush. */
|
||||
|
|
|
|||
Loading…
Reference in a new issue