mirror/mariadb
1
0
Fork 0
mirror of https://github.com/MariaDB/server.git synced 2025-01-21 22:34:18 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions

closes #5916 merge to main

Browse source 
git-svn-id: file:///svn/toku/tokudb@52402 c7de825b-a66e-492c-adef-691d508d4ae1
This commit is contained in:
Leif Walsh 2013-04-17 00:01:27 -04:00 • committed by Yoni Fogel
parent b1bc557927
commit 2ebaaac149
7 changed files with 50 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
buildheader/make_tdb.cc
View file

@ -580,6 +580,7 @@ int main (int argc, char *const argv[] __attribute__((__unused__))) {
printf(" TOKU_ZLIB_METHOD = 8,\n"); // RFC 1950 says use 8 for zlib. It reserves 15 to allow more bytes.
printf(" TOKU_QUICKLZ_METHOD = 9,\n"); // We use 9 for QUICKLZ (the QLZ compression level is stored int he high-order nibble). I couldn't find any standard for any other numbers, so I just use 9. -Bradley
printf(" TOKU_LZMA_METHOD = 10,\n"); // We use 10 for LZMA. (Note the compression level is stored in the high-order nibble).
printf(" TOKU_ZLIB_WITHOUT_CHECKSUM_METHOD = 11,\n"); // We wrap a zlib without checksumming compression technique in our own checksummed metadata.
printf(" TOKU_DEFAULT_COMPRESSION_METHOD = 1,\n"); // default is actually quicklz
printf(" TOKU_FAST_COMPRESSION_METHOD = 2,\n"); // friendlier names
printf(" TOKU_SMALL_COMPRESSION_METHOD = 3,\n");

46
ft/compress.cc
View file

@ -40,6 +40,8 @@ size_t toku_compress_bound (enum toku_compression_method a, size_t size)
return size+400 + 1; // quicklz manual says 400 bytes is enough. We need one more byte for the rfc1950-style header byte. bits 0-3 are 9, bits 4-7 are the QLZ_COMPRESSION_LEVEL.
case TOKU_ZLIB_METHOD:
return compressBound (size);
case TOKU_ZLIB_WITHOUT_CHECKSUM_METHOD:
return 2+deflateBound(nullptr, size); // We need one extra for the rfc1950-style header byte, and one extra to store windowBits (a bit over cautious about future upgrades maybe).
default:
break;
}
@ -47,14 +49,15 @@ size_t toku_compress_bound (enum toku_compression_method a, size_t size)
assert(0); return 0;
}
static const int zlib_compression_level = 5;
void toku_compress (enum toku_compression_method a,
// the following types and naming conventions come from zlib.h
Bytef *dest, uLongf *destLen,
const Bytef *source, uLong sourceLen)
// See compress.h for the specification of this function.
{
static const int zlib_compression_level = 5;
static const int zlib_without_checksum_windowbits = -15;
a = normalize_compression_method(a);
assert(sourceLen < (1LL << 32));
switch (a) {
@ -107,6 +110,27 @@ void toku_compress (enum toku_compression_method a,
return;
}
case TOKU_ZLIB_WITHOUT_CHECKSUM_METHOD: {
z_stream strm;
strm.zalloc = Z_NULL;
strm.zfree = Z_NULL;
strm.opaque = Z_NULL;
strm.next_in = const_cast<Bytef *>(source);
strm.avail_in = sourceLen;
int r = deflateInit2(&strm, zlib_compression_level, Z_DEFLATED,
zlib_without_checksum_windowbits, 8, Z_DEFAULT_STRATEGY);
lazy_assert(r == Z_OK);
strm.next_out = dest + 2;
strm.avail_out = *destLen - 2;
r = deflate(&strm, Z_FINISH);
lazy_assert(r == Z_STREAM_END);
r = deflateEnd(&strm);
lazy_assert(r == Z_OK);
*destLen = strm.total_out + 2;
dest[0] = TOKU_ZLIB_WITHOUT_CHECKSUM_METHOD + (zlib_compression_level << 4);
dest[1] = zlib_without_checksum_windowbits;
return;
}
default:
break;
}
@ -159,6 +183,24 @@ void toku_decompress (Bytef *dest, uLongf destLen,
}
return;
}
case TOKU_ZLIB_WITHOUT_CHECKSUM_METHOD: {
z_stream strm;
strm.next_in = const_cast<Bytef *>(source + 2);
strm.avail_in = sourceLen - 2;
strm.zalloc = Z_NULL;
strm.zfree = Z_NULL;
strm.opaque = Z_NULL;
char windowBits = source[1];
int r = inflateInit2(&strm, windowBits);
lazy_assert(r == Z_OK);
strm.next_out = dest;
strm.avail_out = destLen;
r = inflate(&strm, Z_FINISH);
lazy_assert(r == Z_STREAM_END);
r = inflateEnd(&strm);
lazy_assert(r == Z_OK);
return;
}
}
// default fall through to error.
assert(0);

1
ft/tests/compress-test.cc
View file

@ -22,6 +22,7 @@ static void test_compress_buf_method (unsigned char *buf, int i, enum toku_compr
static void test_compress_buf (unsigned char *buf, int i) {
test_compress_buf_method(buf, i, TOKU_ZLIB_METHOD);
test_compress_buf_method(buf, i, TOKU_ZLIB_WITHOUT_CHECKSUM_METHOD);
test_compress_buf_method(buf, i, TOKU_QUICKLZ_METHOD);
test_compress_buf_method(buf, i, TOKU_LZMA_METHOD);
}

1
ft/tests/subblock-test-checksum.cc
View file

@ -147,6 +147,7 @@ test_main (int argc, const char *argv[]) {
for (int size = total_size - e; size <= total_size + e; size++) {
run_test(size, n_cores, pool, TOKU_NO_COMPRESSION);
run_test(size, n_cores, pool, TOKU_ZLIB_METHOD);
run_test(size, n_cores, pool, TOKU_ZLIB_WITHOUT_CHECKSUM_METHOD);
run_test(size, n_cores, pool, TOKU_QUICKLZ_METHOD);
run_test(size, n_cores, pool, TOKU_LZMA_METHOD);
}

1
ft/tests/subblock-test-compression.cc
View file

@ -98,6 +98,7 @@ test_main (int argc, const char *argv[]) {
for (int size = total_size - e; size <= total_size + e; size++) {
run_test(size, n_cores, TOKU_NO_COMPRESSION);
run_test(size, n_cores, TOKU_ZLIB_METHOD);
run_test(size, n_cores, TOKU_ZLIB_WITHOUT_CHECKSUM_METHOD);
run_test(size, n_cores, TOKU_QUICKLZ_METHOD);
run_test(size, n_cores, TOKU_LZMA_METHOD);
}

1
src/tests/test_compression_methods.cc
View file

@ -115,6 +115,7 @@ test_main(int argc, char *const argv[])
parse_args(argc, argv);
run_test(TOKU_NO_COMPRESSION);
run_test(TOKU_ZLIB_METHOD);
run_test(TOKU_ZLIB_WITHOUT_CHECKSUM_METHOD);
run_test(TOKU_QUICKLZ_METHOD);
run_test(TOKU_LZMA_METHOD);
return 0;

2
src/tests/threaded_stress_test_helpers.h
View file

@ -2442,7 +2442,7 @@ static inline void parse_stress_test_args (int argc, char *const argv[], struct
if (strcmp(compression_method_s, "quicklz") == 0) {
args->compression_method = TOKU_QUICKLZ_METHOD;
} else if (strcmp(compression_method_s, "zlib") == 0) {
args->compression_method = TOKU_ZLIB_METHOD;
args->compression_method = TOKU_ZLIB_WITHOUT_CHECKSUM_METHOD;
} else if (strcmp(compression_method_s, "lzma") == 0) {
args->compression_method = TOKU_LZMA_METHOD;
} else if (strcmp(compression_method_s, "none") == 0) {