From 2ebaaac149c898d8b0216022b859f95db214d355 Mon Sep 17 00:00:00 2001 From: Leif Walsh Date: Wed, 17 Apr 2013 00:01:27 -0400 Subject: [PATCH] closes #5916 merge to main git-svn-id: file:///svn/toku/tokudb@52402 c7de825b-a66e-492c-adef-691d508d4ae1 --- buildheader/make_tdb.cc | 1 + ft/compress.cc | 46 ++++++++++++++++++++++-- ft/tests/compress-test.cc | 1 + ft/tests/subblock-test-checksum.cc | 1 + ft/tests/subblock-test-compression.cc | 1 + src/tests/test_compression_methods.cc | 1 + src/tests/threaded_stress_test_helpers.h | 2 +- 7 files changed, 50 insertions(+), 3 deletions(-) diff --git a/buildheader/make_tdb.cc b/buildheader/make_tdb.cc index cac3188c224..946e2af2aa9 100644 --- a/buildheader/make_tdb.cc +++ b/buildheader/make_tdb.cc @@ -580,6 +580,7 @@ int main (int argc, char *const argv[] __attribute__((__unused__))) { printf(" TOKU_ZLIB_METHOD = 8,\n"); // RFC 1950 says use 8 for zlib. It reserves 15 to allow more bytes. printf(" TOKU_QUICKLZ_METHOD = 9,\n"); // We use 9 for QUICKLZ (the QLZ compression level is stored int he high-order nibble). I couldn't find any standard for any other numbers, so I just use 9. -Bradley printf(" TOKU_LZMA_METHOD = 10,\n"); // We use 10 for LZMA. (Note the compression level is stored in the high-order nibble). + printf(" TOKU_ZLIB_WITHOUT_CHECKSUM_METHOD = 11,\n"); // We wrap a zlib without checksumming compression technique in our own checksummed metadata. printf(" TOKU_DEFAULT_COMPRESSION_METHOD = 1,\n"); // default is actually quicklz printf(" TOKU_FAST_COMPRESSION_METHOD = 2,\n"); // friendlier names printf(" TOKU_SMALL_COMPRESSION_METHOD = 3,\n"); diff --git a/ft/compress.cc b/ft/compress.cc index 2bffac6d303..862902026f1 100644 --- a/ft/compress.cc +++ b/ft/compress.cc @@ -40,6 +40,8 @@ size_t toku_compress_bound (enum toku_compression_method a, size_t size) return size+400 + 1; // quicklz manual says 400 bytes is enough. We need one more byte for the rfc1950-style header byte. bits 0-3 are 9, bits 4-7 are the QLZ_COMPRESSION_LEVEL. case TOKU_ZLIB_METHOD: return compressBound (size); + case TOKU_ZLIB_WITHOUT_CHECKSUM_METHOD: + return 2+deflateBound(nullptr, size); // We need one extra for the rfc1950-style header byte, and one extra to store windowBits (a bit over cautious about future upgrades maybe). default: break; } @@ -47,14 +49,15 @@ size_t toku_compress_bound (enum toku_compression_method a, size_t size) assert(0); return 0; } -static const int zlib_compression_level = 5; - void toku_compress (enum toku_compression_method a, // the following types and naming conventions come from zlib.h Bytef *dest, uLongf *destLen, const Bytef *source, uLong sourceLen) // See compress.h for the specification of this function. { + static const int zlib_compression_level = 5; + static const int zlib_without_checksum_windowbits = -15; + a = normalize_compression_method(a); assert(sourceLen < (1LL << 32)); switch (a) { @@ -107,6 +110,27 @@ void toku_compress (enum toku_compression_method a, return; } + case TOKU_ZLIB_WITHOUT_CHECKSUM_METHOD: { + z_stream strm; + strm.zalloc = Z_NULL; + strm.zfree = Z_NULL; + strm.opaque = Z_NULL; + strm.next_in = const_cast(source); + strm.avail_in = sourceLen; + int r = deflateInit2(&strm, zlib_compression_level, Z_DEFLATED, + zlib_without_checksum_windowbits, 8, Z_DEFAULT_STRATEGY); + lazy_assert(r == Z_OK); + strm.next_out = dest + 2; + strm.avail_out = *destLen - 2; + r = deflate(&strm, Z_FINISH); + lazy_assert(r == Z_STREAM_END); + r = deflateEnd(&strm); + lazy_assert(r == Z_OK); + *destLen = strm.total_out + 2; + dest[0] = TOKU_ZLIB_WITHOUT_CHECKSUM_METHOD + (zlib_compression_level << 4); + dest[1] = zlib_without_checksum_windowbits; + return; + } default: break; } @@ -159,6 +183,24 @@ void toku_decompress (Bytef *dest, uLongf destLen, } return; } + case TOKU_ZLIB_WITHOUT_CHECKSUM_METHOD: { + z_stream strm; + strm.next_in = const_cast(source + 2); + strm.avail_in = sourceLen - 2; + strm.zalloc = Z_NULL; + strm.zfree = Z_NULL; + strm.opaque = Z_NULL; + char windowBits = source[1]; + int r = inflateInit2(&strm, windowBits); + lazy_assert(r == Z_OK); + strm.next_out = dest; + strm.avail_out = destLen; + r = inflate(&strm, Z_FINISH); + lazy_assert(r == Z_STREAM_END); + r = inflateEnd(&strm); + lazy_assert(r == Z_OK); + return; + } } // default fall through to error. assert(0); diff --git a/ft/tests/compress-test.cc b/ft/tests/compress-test.cc index e3d7098104f..4d8ae0743f6 100644 --- a/ft/tests/compress-test.cc +++ b/ft/tests/compress-test.cc @@ -22,6 +22,7 @@ static void test_compress_buf_method (unsigned char *buf, int i, enum toku_compr static void test_compress_buf (unsigned char *buf, int i) { test_compress_buf_method(buf, i, TOKU_ZLIB_METHOD); + test_compress_buf_method(buf, i, TOKU_ZLIB_WITHOUT_CHECKSUM_METHOD); test_compress_buf_method(buf, i, TOKU_QUICKLZ_METHOD); test_compress_buf_method(buf, i, TOKU_LZMA_METHOD); } diff --git a/ft/tests/subblock-test-checksum.cc b/ft/tests/subblock-test-checksum.cc index 2e99ac3c086..566c6542374 100644 --- a/ft/tests/subblock-test-checksum.cc +++ b/ft/tests/subblock-test-checksum.cc @@ -147,6 +147,7 @@ test_main (int argc, const char *argv[]) { for (int size = total_size - e; size <= total_size + e; size++) { run_test(size, n_cores, pool, TOKU_NO_COMPRESSION); run_test(size, n_cores, pool, TOKU_ZLIB_METHOD); + run_test(size, n_cores, pool, TOKU_ZLIB_WITHOUT_CHECKSUM_METHOD); run_test(size, n_cores, pool, TOKU_QUICKLZ_METHOD); run_test(size, n_cores, pool, TOKU_LZMA_METHOD); } diff --git a/ft/tests/subblock-test-compression.cc b/ft/tests/subblock-test-compression.cc index 84ef40a874c..3c12e721949 100644 --- a/ft/tests/subblock-test-compression.cc +++ b/ft/tests/subblock-test-compression.cc @@ -98,6 +98,7 @@ test_main (int argc, const char *argv[]) { for (int size = total_size - e; size <= total_size + e; size++) { run_test(size, n_cores, TOKU_NO_COMPRESSION); run_test(size, n_cores, TOKU_ZLIB_METHOD); + run_test(size, n_cores, TOKU_ZLIB_WITHOUT_CHECKSUM_METHOD); run_test(size, n_cores, TOKU_QUICKLZ_METHOD); run_test(size, n_cores, TOKU_LZMA_METHOD); } diff --git a/src/tests/test_compression_methods.cc b/src/tests/test_compression_methods.cc index b542a635f73..2592604296c 100644 --- a/src/tests/test_compression_methods.cc +++ b/src/tests/test_compression_methods.cc @@ -115,6 +115,7 @@ test_main(int argc, char *const argv[]) parse_args(argc, argv); run_test(TOKU_NO_COMPRESSION); run_test(TOKU_ZLIB_METHOD); + run_test(TOKU_ZLIB_WITHOUT_CHECKSUM_METHOD); run_test(TOKU_QUICKLZ_METHOD); run_test(TOKU_LZMA_METHOD); return 0; diff --git a/src/tests/threaded_stress_test_helpers.h b/src/tests/threaded_stress_test_helpers.h index 3c04282328d..dc5e6d450de 100644 --- a/src/tests/threaded_stress_test_helpers.h +++ b/src/tests/threaded_stress_test_helpers.h @@ -2442,7 +2442,7 @@ static inline void parse_stress_test_args (int argc, char *const argv[], struct if (strcmp(compression_method_s, "quicklz") == 0) { args->compression_method = TOKU_QUICKLZ_METHOD; } else if (strcmp(compression_method_s, "zlib") == 0) { - args->compression_method = TOKU_ZLIB_METHOD; + args->compression_method = TOKU_ZLIB_WITHOUT_CHECKSUM_METHOD; } else if (strcmp(compression_method_s, "lzma") == 0) { args->compression_method = TOKU_LZMA_METHOD; } else if (strcmp(compression_method_s, "none") == 0) {