From fef9b7499c7c4008b71c4ff8f20889047a9af680 Mon Sep 17 00:00:00 2001 From: unknown Date: Sun, 22 Jan 2006 18:23:48 +0100 Subject: [PATCH] wl1497 - ndb - dynamic mem in ndbd Impl. NdbdSuperPool (subclass of SuperPool) that uses Ndbd_mem_manager Impl. micro benchmark storage/ndb/src/kernel/vm/Makefile.am: Add NdbdSuperPool , which impl. SuperPool storage/ndb/src/kernel/vm/SuperPool.cpp: remove virtuallity storage/ndb/src/kernel/vm/SuperPool.hpp: Remove unneccesary virtuality storage/ndb/src/kernel/vm/ndbd_malloc_impl.cpp: Add util methods used by NdbdSuperPool storage/ndb/src/kernel/vm/ndbd_malloc_impl.hpp: Add util methods used by NdbdSuperPool storage/ndb/src/kernel/vm/NdbdSuperPool.cpp: New BitKeeper file ``storage/ndb/src/kernel/vm/NdbdSuperPool.cpp'' storage/ndb/src/kernel/vm/NdbdSuperPool.hpp: New BitKeeper file ``storage/ndb/src/kernel/vm/NdbdSuperPool.hpp'' storage/ndb/src/kernel/vm/bench_pool.cpp: New BitKeeper file ``storage/ndb/src/kernel/vm/bench_pool.cpp'' --- storage/ndb/src/kernel/vm/Makefile.am | 13 +- storage/ndb/src/kernel/vm/NdbdSuperPool.cpp | 228 ++++++++++++++++ storage/ndb/src/kernel/vm/NdbdSuperPool.hpp | 55 ++++ storage/ndb/src/kernel/vm/SuperPool.cpp | 4 +- storage/ndb/src/kernel/vm/SuperPool.hpp | 3 +- storage/ndb/src/kernel/vm/bench_pool.cpp | 249 ++++++++++++++++++ .../ndb/src/kernel/vm/ndbd_malloc_impl.cpp | 21 +- .../ndb/src/kernel/vm/ndbd_malloc_impl.hpp | 9 +- 8 files changed, 574 insertions(+), 8 deletions(-) create mode 100644 storage/ndb/src/kernel/vm/NdbdSuperPool.cpp create mode 100644 storage/ndb/src/kernel/vm/NdbdSuperPool.hpp create mode 100644 storage/ndb/src/kernel/vm/bench_pool.cpp diff --git a/storage/ndb/src/kernel/vm/Makefile.am b/storage/ndb/src/kernel/vm/Makefile.am index f2bef8ba1f7..3e89508e851 100644 --- a/storage/ndb/src/kernel/vm/Makefile.am +++ b/storage/ndb/src/kernel/vm/Makefile.am @@ -20,7 +20,8 @@ libkernel_a_SOURCES = \ Mutex.cpp SafeCounter.cpp \ Rope.cpp \ SuperPool.cpp \ - ndbd_malloc.cpp ndbd_malloc_impl.cpp + ndbd_malloc.cpp ndbd_malloc_impl.cpp \ + NdbdSuperPool.cpp INCLUDES_LOC = -I$(top_srcdir)/storage/ndb/src/mgmapi @@ -44,7 +45,7 @@ libkernel.dsp: Makefile \ @$(top_srcdir)/storage/ndb/config/win-sources $@ $(libkernel_a_SOURCES) @$(top_srcdir)/storage/ndb/config/win-libraries $@ LIB $(LDADD) -EXTRA_PROGRAMS = ndbd_malloc_impl_test +EXTRA_PROGRAMS = ndbd_malloc_impl_test bench_pool ndbd_malloc_impl_test_CXXFLAGS = -DUNIT_TEST ndbd_malloc_impl_test_SOURCES = ndbd_malloc_impl.cpp ndbd_malloc_impl_test_LDFLAGS = @ndb_bin_am_ldflags@ \ @@ -52,3 +53,11 @@ ndbd_malloc_impl_test_LDFLAGS = @ndb_bin_am_ldflags@ \ $(top_builddir)/mysys/libmysys.a \ $(top_builddir)/dbug/libdbug.a \ $(top_builddir)/strings/libmystrings.a + +bench_pool_SOURCES = bench_pool.cpp ndbd_malloc.cpp \ + SuperPool.cpp NdbdSuperPool.cpp ndbd_malloc_impl.cpp +bench_pool_LDFLAGS = @ndb_bin_am_ldflags@ \ + $(top_builddir)/storage/ndb/src/libndbclient.la \ + $(top_builddir)/mysys/libmysys.a \ + $(top_builddir)/dbug/libdbug.a \ + $(top_builddir)/strings/libmystrings.a diff --git a/storage/ndb/src/kernel/vm/NdbdSuperPool.cpp b/storage/ndb/src/kernel/vm/NdbdSuperPool.cpp new file mode 100644 index 00000000000..29174ab2e43 --- /dev/null +++ b/storage/ndb/src/kernel/vm/NdbdSuperPool.cpp @@ -0,0 +1,228 @@ +/* Copyright (C) 2003 MySQL AB + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ + +#include +#include "SuperPool.hpp" +#include "ndbd_malloc_impl.hpp" +#include "NdbdSuperPool.hpp" + +#define PSI (1 << (BMW_2LOG + 2)) + +struct AllocArea +{ + AllocArea(AllocArea* next); + + Uint16 m_currPage; // 2 + Uint16 m_numPages; // number of pages // 2 + SuperPool::PtrI m_firstPageI; // 4 + void* m_memory; // page-aligned pages // 4/8 + struct AllocArea* m_nextArea; // 4/8 + // tot 16/24 +}; + +AllocArea::AllocArea(AllocArea* next) +{ + m_nextArea = next; + m_firstPageI = RNIL; + m_currPage = m_numPages = 0; + m_memory = 0; +} + +NdbdSuperPool::NdbdSuperPool(class Ndbd_mem_manager & mm, + Uint32 pageSize, Uint32 pageBits) : + SuperPool(pageSize, pageBits), + m_mm(mm), + m_currArea(0), m_firstArea(0) +{ + m_memRoot = m_mm.get_memroot(); + + m_shift = Ndbd_mem_manager::log2((1 << (BMW_2LOG + 2)) / pageSize) - 1; + m_add = (1 << m_shift) - 1; +} + +NdbdSuperPool::~NdbdSuperPool() +{ + Uint32 cnt = PSI / sizeof(AllocArea); + AllocArea* ap = m_firstArea; + while(ap != 0) + { + AllocArea * first = ap; + for(Uint32 i = 0; im_numPages) + { + m_mm.release(ap->m_memory, ap->m_numPages >> m_shift); + } + ap = ap->m_nextArea; + } + m_mm.release((void*)first, 1); + } +} + +bool +NdbdSuperPool::init_1() +{ + Uint32 pageCount = (1 << m_pageBits); + if (m_pageEnt == 0) { + // allocate page entry array + Uint32 bytes = pageCount * sizeof(PageEnt); + m_pageEnt = static_cast(malloc(bytes)); + if (m_pageEnt == 0) + return false; + for (Uint32 i = 0; i < pageCount; i++) + new (&m_pageEnt[i]) PageEnt(); + } + if (m_pageType == 0) { + // allocate type check array + Uint32 bytes = pageCount; + m_pageType = static_cast(malloc(bytes)); + if (m_pageType == 0) + return false; + memset(m_pageType, 0, bytes); + } + + return true; +} + +static +void +initAllocAreaPage(AllocArea * p1) +{ + AllocArea * ap = p1; + Uint32 cnt = PSI / sizeof(AllocArea); + for(Uint32 i = 0; im_nextArea = 0; +} + +bool +NdbdSuperPool::init_2() +{ + m_memRoot = m_mm.get_memroot(); + + Uint32 cnt = 1; + AllocArea* p1 = (AllocArea*)m_mm.alloc(&cnt, 1); + if (p1 == 0) + return false; + + initAllocAreaPage(p1); + m_currArea = p1; + m_firstArea = p1; + return true; +} + +SuperPool::PtrI +NdbdSuperPool::getNewPage() +{ + AllocArea* ap = m_currArea; + Uint32 curr = ap->m_currPage; + Uint32 cnt = ap->m_numPages; + if (curr == cnt) + { + // area is used up + if (! (ap = allocMem())) + { + abort(); + return RNIL; + } + curr = ap->m_currPage; + cnt = ap->m_numPages; + } + + assert(curr < cnt); + PtrI pageI = ap->m_firstPageI; + Uint32 recBits = m_recBits; + Int32 ip = ((Int32)pageI >> recBits) + curr; + pageI = ip << recBits; + ap->m_currPage = curr + 1; + return pageI; +} + +Uint32 +NdbdSuperPool::allocAreaMemory(AllocArea* ap, Uint32 tryPages) +{ + Uint32 cnt = (tryPages + m_add) >> m_shift; + void* p1 = m_mm.alloc(&cnt, 1); + if (p1 == 0) + { + abort(); + return 0; + } + Uint32 pageI = getPageI(p1); + ap->m_firstPageI = pageI; + ap->m_currPage = 0; + ap->m_memory = p1; + ap->m_numPages = cnt << m_shift; + return cnt; +} + +AllocArea* +NdbdSuperPool::allocArea() +{ + AllocArea * curr = m_currArea; + AllocArea * next = curr->m_nextArea; + if (next == 0) + { + Uint32 cnt = 1; + AllocArea* p1 = (AllocArea*)m_mm.alloc(&cnt, 1); + if (p1 == 0) + return 0; + + initAllocAreaPage(p1); + + m_currArea->m_nextArea = p1; + return m_currArea = p1; + } + else + { + m_currArea = m_currArea->m_nextArea; + return m_currArea; + } +} + +AllocArea* +NdbdSuperPool::allocMem() +{ + // compute number of additional pages needed + if (m_totPages >= m_maxPages) + { + abort(); + return 0; + } + Uint32 needPages = (m_totPages == 0 ? m_initPages : m_incrPages); + + // add new area + AllocArea* ap = allocArea(); + if (ap == 0) + { + abort(); + return 0; + } + + Uint32 numPages; + if (!(numPages = allocAreaMemory(ap, needPages))) + { + abort(); + return 0; + } + + // update counts + m_totPages += numPages; + return ap; +} diff --git a/storage/ndb/src/kernel/vm/NdbdSuperPool.hpp b/storage/ndb/src/kernel/vm/NdbdSuperPool.hpp new file mode 100644 index 00000000000..fc552357a4f --- /dev/null +++ b/storage/ndb/src/kernel/vm/NdbdSuperPool.hpp @@ -0,0 +1,55 @@ +/* Copyright (C) 2003 MySQL AB + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ + +#ifndef NDBD_SUPER_POOL_HPP +#define NDBD_SUPER_POOL_HPP + +#include "SuperPool.hpp" + +struct AllocArea; + +class NdbdSuperPool : public SuperPool +{ +public: + NdbdSuperPool(class Ndbd_mem_manager&, Uint32 pageSize, Uint32 pageBits); + + // Destructor. + virtual ~NdbdSuperPool(); + + // Get new page from current area. + virtual PtrI getNewPage(); + + // Call first...on all superpools (uses malloc) + bool init_1(); + + // Call second...uses mm + bool init_2(); + + virtual bool allocMemory() { return allocMem() != 0; } +private: + Uint32 allocAreaMemory(AllocArea*, Uint32 pages); + AllocArea* allocArea(); + AllocArea* allocMem(); + + // List of malloc areas. + Uint32 m_shift, m_add; + class Ndbd_mem_manager & m_mm; + + AllocArea* m_currArea; + AllocArea* m_firstArea; +}; + +#endif diff --git a/storage/ndb/src/kernel/vm/SuperPool.cpp b/storage/ndb/src/kernel/vm/SuperPool.cpp index c61bfcc3541..00b6fbc780e 100644 --- a/storage/ndb/src/kernel/vm/SuperPool.cpp +++ b/storage/ndb/src/kernel/vm/SuperPool.cpp @@ -644,7 +644,7 @@ HeapPool::getNewPage() if (ap->m_currPage == ap->m_numPages) { // area is used up if (ap->m_nextArea == 0) { - if (! allocMemory()) + if (! allocMemoryImpl()) return RNIL; } ap = m_currArea = ap->m_nextArea; @@ -711,7 +711,7 @@ HeapPool::allocArea(Area* ap, Uint32 tryPages) } bool -HeapPool::allocMemory() +HeapPool::allocMemoryImpl() { if (! allocInit()) return false; diff --git a/storage/ndb/src/kernel/vm/SuperPool.hpp b/storage/ndb/src/kernel/vm/SuperPool.hpp index 187383f5f71..8f649a2b32a 100644 --- a/storage/ndb/src/kernel/vm/SuperPool.hpp +++ b/storage/ndb/src/kernel/vm/SuperPool.hpp @@ -580,7 +580,8 @@ public: bool allocArea(Area* ap, Uint32 tryPages); // Allocate memory. - virtual bool allocMemory(); + virtual bool allocMemory() { return allocMemoryImpl();} + bool allocMemoryImpl(); // List of malloc areas. Area m_areaHead; diff --git a/storage/ndb/src/kernel/vm/bench_pool.cpp b/storage/ndb/src/kernel/vm/bench_pool.cpp new file mode 100644 index 00000000000..1042059b0c5 --- /dev/null +++ b/storage/ndb/src/kernel/vm/bench_pool.cpp @@ -0,0 +1,249 @@ +/* Copyright (C) 2003 MySQL AB + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ + + +#include "NdbdSuperPool.hpp" +#include "ArrayPool.hpp" +#include +#include "ndbd_malloc_impl.hpp" + +template +inline +void +init(ArrayPool & pool, Uint32 cnt) +{ + pool.setSize(cnt + 1); +} + +template +inline +void +init(RecordPool & pool, Uint32 cnt) +{ +} + + +template +inline +void +test_pool(R& pool, Uint32 cnt, Uint32 loops) +{ + init(pool, cnt); + Ptr ptr; + Uint32 *arr = (Uint32*)alloca(cnt * sizeof(Uint32)); + { + printf(" ; seize "); fflush(stdout); + Uint64 sum = 0; + for(Uint32 i = 0; i struct Rec { char data[sz-4]; Uint32 nextPool; }; +typedef Rec<32> Rec32; +typedef Rec<36> Rec36; +typedef Rec<256> Rec256; +typedef Rec<260> Rec260; + +Ndbd_mem_manager mem; + +template +inline +void test_rp(Uint32 cnt, Uint32 loop, Uint32 pgsz) +{ + printf("RP ; %d ; ws ; %d ; page ; %d", + sizeof(T), (sizeof(T)*cnt) >> 10, pgsz >> 10); + NdbdSuperPool sp(mem, pgsz, 19); + GroupPool gp(sp); + sp.init_1(); + sp.init_2(); + + sp.setInitPages(4); + sp.setIncrPages(4); + sp.setMaxPages(~0); + sp.allocMemory(); + + RecordPool pool(gp); + test_pool >(pool, cnt, loop); +} + +template +inline +void test_ap(Uint32 cnt, Uint32 loop) +{ + printf("AP ; %d ; ws ; %d ; page ; n/a", sizeof(T), (cnt * sizeof(T))>>10); + ArrayPool pool; + test_pool >(pool, cnt, loop); +} + +int +main(int argc, char **argv) +{ + mem.init(10000); + + Uint32 cnt = 100; + Uint32 loop = 300000; + + while(cnt <= 1000000) + { + test_rp(cnt, loop, 8192); + test_rp(cnt, loop, 32768); + test_ap(cnt, loop); + + test_rp(cnt, loop, 8192); + test_rp(cnt, loop, 32768); + test_ap(cnt, loop); + + test_rp(cnt, loop, 8192); + test_rp(cnt, loop, 32768); + test_ap(cnt, loop); + + test_rp(cnt, loop, 8192); + test_rp(cnt, loop, 32768); + test_ap(cnt, loop); + + cnt *= 100; + loop /= 100; + } +} + +void +ErrorReporter::handleAssert(const char * msg, const char * file, + int line, int) +{ + ndbout << "ErrorReporter::handleAssert activated - " + << " line= " << line << endl; + abort(); +} diff --git a/storage/ndb/src/kernel/vm/ndbd_malloc_impl.cpp b/storage/ndb/src/kernel/vm/ndbd_malloc_impl.cpp index 5443fdf1300..e78a1cc1194 100644 --- a/storage/ndb/src/kernel/vm/ndbd_malloc_impl.cpp +++ b/storage/ndb/src/kernel/vm/ndbd_malloc_impl.cpp @@ -84,7 +84,7 @@ Ndbd_mem_manager::init(Uint32 pages) release(start+1, end - 1 - start); } - return 0; + return true; } void @@ -200,6 +200,25 @@ Ndbd_mem_manager::alloc(Uint32* ret, Uint32 *pages, Uint32 min) * pages = 0; } +void* +Ndbd_mem_manager::alloc(Uint32 *pages, Uint32 min) +{ + Uint32 ret; + alloc(&ret, pages, min); + if (pages) + { + return m_base_page + ret; + } + return 0; +} + +void +Ndbd_mem_manager::release(void* ptr, Uint32 cnt) +{ + Uint32 page = ((Alloc_page*)ptr) - m_base_page; + release(page, cnt); +} + void Ndbd_mem_manager::insert_free_list(Uint32 start, Uint32 size) { diff --git a/storage/ndb/src/kernel/vm/ndbd_malloc_impl.hpp b/storage/ndb/src/kernel/vm/ndbd_malloc_impl.hpp index e497f62b92a..15b5c9c390e 100644 --- a/storage/ndb/src/kernel/vm/ndbd_malloc_impl.hpp +++ b/storage/ndb/src/kernel/vm/ndbd_malloc_impl.hpp @@ -64,8 +64,12 @@ public: bool grow(Uint32 pages = 0); void dump() const ; -private: - + + void* get_memroot() const { return (void*)m_base_page;} + + void* alloc(Uint32 * pages, Uint32 min_requested); + void release(void* ptr, Uint32 cnt); + /** * Compute 2log of size * @note size = 0 -> 0 @@ -73,6 +77,7 @@ private: */ static Uint32 log2(Uint32 size); +private: /** * Return pointer to free page data on page */