From 1902b5c99c125ab80b695777f846743a1d4def82 Mon Sep 17 00:00:00 2001 From: Leif Walsh Date: Tue, 29 Apr 2014 13:24:31 -0400 Subject: [PATCH] added MEMORY_MAX_REQUESTED_SIZE and MEMORY_LAST_FAILED_SIZE #230 fixes #230 --- portability/memory.cc | 38 +++++++++++++++++++++++++++++++++++--- src/ydb.cc | 16 ++++++++++------ toku_include/memory.h | 18 ++++++++++-------- 3 files changed, 55 insertions(+), 17 deletions(-) diff --git a/portability/memory.cc b/portability/memory.cc index 885a94c508b..e2a08d60a99 100644 --- a/portability/memory.cc +++ b/portability/memory.cc @@ -221,6 +221,9 @@ toku_memory_footprint(void * p, size_t touched) { void * toku_malloc(size_t size) { + if (size > status.max_requested_size) { + status.max_requested_size = size; + } void *p = t_malloc ? t_malloc(size) : os_malloc(size); if (p) { TOKU_ANNOTATE_NEW_MEMORY(p, size); // see #4671 and https://bugs.kde.org/show_bug.cgi?id=297147 @@ -233,11 +236,15 @@ toku_malloc(size_t size) { } } else { toku_sync_add_and_fetch(&status.malloc_fail, 1); + status.last_failed_size = size; } return p; } void *toku_malloc_aligned(size_t alignment, size_t size) { + if (size > status.max_requested_size) { + status.max_requested_size = size; + } void *p = t_malloc_aligned ? t_malloc_aligned(alignment, size) : os_malloc_aligned(alignment, size); if (p) { TOKU_ANNOTATE_NEW_MEMORY(p, size); // see #4671 and https://bugs.kde.org/show_bug.cgi?id=297147 @@ -250,6 +257,7 @@ void *toku_malloc_aligned(size_t alignment, size_t size) { } } else { toku_sync_add_and_fetch(&status.malloc_fail, 1); + status.last_failed_size = size; } return p; } @@ -264,6 +272,9 @@ toku_calloc(size_t nmemb, size_t size) { void * toku_realloc(void *p, size_t size) { + if (size > status.max_requested_size) { + status.max_requested_size = size; + } size_t used_orig = p ? my_malloc_usable_size(p) : 0; void *q = t_realloc ? t_realloc(p, size) : os_realloc(p, size); if (q) { @@ -277,11 +288,15 @@ toku_realloc(void *p, size_t size) { } } else { toku_sync_add_and_fetch(&status.realloc_fail, 1); + status.last_failed_size = size; } return q; } void *toku_realloc_aligned(size_t alignment, void *p, size_t size) { + if (size > status.max_requested_size) { + status.max_requested_size = size; + } size_t used_orig = p ? my_malloc_usable_size(p) : 0; void *q = t_realloc_aligned ? t_realloc_aligned(alignment, p, size) : os_realloc_aligned(alignment, p, size); if (q) { @@ -295,6 +310,7 @@ void *toku_realloc_aligned(size_t alignment, void *p, size_t size) { } } else { toku_sync_add_and_fetch(&status.realloc_fail, 1); + status.last_failed_size = size; } return q; } @@ -329,9 +345,14 @@ toku_free(void *p) { void * toku_xmalloc(size_t size) { + if (size > status.max_requested_size) { + status.max_requested_size = size; + } void *p = t_xmalloc ? t_xmalloc(size) : os_malloc(size); - if (p == NULL) // avoid function call in common case + if (p == NULL) { // avoid function call in common case + status.last_failed_size = size; resource_assert(p); + } TOKU_ANNOTATE_NEW_MEMORY(p, size); // see #4671 and https://bugs.kde.org/show_bug.cgi?id=297147 if (toku_memory_do_stats) { size_t used = my_malloc_usable_size(p); @@ -348,8 +369,14 @@ void* toku_xmalloc_aligned(size_t alignment, size_t size) // Fail with a resource_assert if the allocation fails (don't return an error code). // Requires: alignment is a power of two. { + if (size > status.max_requested_size) { + status.max_requested_size = size; + } void *p = t_xmalloc_aligned ? t_xmalloc_aligned(alignment, size) : os_malloc_aligned(alignment,size); - resource_assert(p); + if (p == NULL) { + status.last_failed_size = size; + resource_assert(p); + } if (toku_memory_do_stats) { size_t used = my_malloc_usable_size(p); toku_sync_add_and_fetch(&status.malloc_count, 1); @@ -370,10 +397,15 @@ toku_xcalloc(size_t nmemb, size_t size) { void * toku_xrealloc(void *v, size_t size) { + if (size > status.max_requested_size) { + status.max_requested_size = size; + } size_t used_orig = v ? my_malloc_usable_size(v) : 0; void *p = t_xrealloc ? t_xrealloc(v, size) : os_realloc(v, size); - if (p == 0) // avoid function call in common case + if (p == 0) { // avoid function call in common case + status.last_failed_size = size; resource_assert(p); + } if (toku_memory_do_stats) { size_t used = my_malloc_usable_size(p); toku_sync_add_and_fetch(&status.realloc_count, 1); diff --git a/src/ydb.cc b/src/ydb.cc index 41dc98a3094..3ebb157b86d 100644 --- a/src/ydb.cc +++ b/src/ydb.cc @@ -1832,13 +1832,15 @@ fs_get_status(DB_ENV * env, fs_redzone_state * redzone_state) { // Local status struct used to get information from memory.c typedef enum { MEMORY_MALLOC_COUNT = 0, - MEMORY_FREE_COUNT, + MEMORY_FREE_COUNT, MEMORY_REALLOC_COUNT, - MEMORY_MALLOC_FAIL, - MEMORY_REALLOC_FAIL, - MEMORY_REQUESTED, - MEMORY_USED, - MEMORY_FREED, + MEMORY_MALLOC_FAIL, + MEMORY_REALLOC_FAIL, + MEMORY_REQUESTED, + MEMORY_USED, + MEMORY_FREED, + MEMORY_MAX_REQUESTED_SIZE, + MEMORY_LAST_FAILED_SIZE, MEMORY_MAX_IN_USE, MEMORY_MALLOCATOR_VERSION, MEMORY_MMAP_THRESHOLD, @@ -1866,6 +1868,8 @@ memory_status_init(void) { STATUS_INIT(MEMORY_REQUESTED, nullptr, UINT64, "number of bytes requested", TOKU_ENGINE_STATUS); STATUS_INIT(MEMORY_USED, nullptr, UINT64, "number of bytes used (requested + overhead)", TOKU_ENGINE_STATUS); STATUS_INIT(MEMORY_FREED, nullptr, UINT64, "number of bytes freed", TOKU_ENGINE_STATUS); + STATUS_INIT(MEMORY_MAX_REQUESTED_SIZE, nullptr, UINT64, "largest attempted allocation size", TOKU_ENGINE_STATUS); + STATUS_INIT(MEMORY_LAST_FAILED_SIZE, nullptr, UINT64, "size of the last failed allocation attempt", TOKU_ENGINE_STATUS); STATUS_INIT(MEMORY_MAX_IN_USE, MEM_ESTIMATED_MAXIMUM_MEMORY_FOOTPRINT, UINT64, "estimated maximum memory footprint", TOKU_ENGINE_STATUS|TOKU_GLOBAL_STATUS); STATUS_INIT(MEMORY_MALLOCATOR_VERSION, nullptr, CHARSTR, "mallocator version", TOKU_ENGINE_STATUS); STATUS_INIT(MEMORY_MMAP_THRESHOLD, nullptr, UINT64, "mmap threshold", TOKU_ENGINE_STATUS); diff --git a/toku_include/memory.h b/toku_include/memory.h index 215ea2e209d..e4608d6108f 100644 --- a/toku_include/memory.h +++ b/toku_include/memory.h @@ -207,14 +207,16 @@ void toku_set_func_realloc_only(realloc_fun_t f); void toku_set_func_free(free_fun_t f); typedef struct memory_status { - uint64_t malloc_count; // number of malloc operations - uint64_t free_count; // number of free operations - uint64_t realloc_count; // number of realloc operations - uint64_t malloc_fail; // number of malloc operations that failed - uint64_t realloc_fail; // number of realloc operations that failed - uint64_t requested; // number of bytes requested - uint64_t used; // number of bytes used (requested + overhead), obtained from malloc_usable_size() - uint64_t freed; // number of bytes freed; + uint64_t malloc_count; // number of malloc operations + uint64_t free_count; // number of free operations + uint64_t realloc_count; // number of realloc operations + uint64_t malloc_fail; // number of malloc operations that failed + uint64_t realloc_fail; // number of realloc operations that failed + uint64_t requested; // number of bytes requested + uint64_t used; // number of bytes used (requested + overhead), obtained from malloc_usable_size() + uint64_t freed; // number of bytes freed; + uint64_t max_requested_size; // largest attempted allocation size + uint64_t last_failed_size; // size of the last failed allocation attempt volatile uint64_t max_in_use; // maximum memory footprint (used - freed), approximate (not worth threadsafety overhead for exact) const char *mallocator_version; uint64_t mmap_threshold;