From 4d5557f70444bcb04b5f719cb7c1731ceacbad14 Mon Sep 17 00:00:00 2001 From: unknown Date: Sat, 17 Jul 2004 19:31:16 +0200 Subject: [PATCH 1/3] Added new testcase for Bug #4479 testBasic -n MassiveTransaction Inserts as many records as defined in one transaction using loadTable --- ndb/test/include/HugoTransactions.hpp | 3 +- ndb/test/ndbapi/testBasic.cpp | 18 ++++++++-- ndb/test/src/HugoTransactions.cpp | 47 +++++++++++++++++---------- 3 files changed, 48 insertions(+), 20 deletions(-) diff --git a/ndb/test/include/HugoTransactions.hpp b/ndb/test/include/HugoTransactions.hpp index 5ff1fef16bc..3d373652cbc 100644 --- a/ndb/test/include/HugoTransactions.hpp +++ b/ndb/test/include/HugoTransactions.hpp @@ -34,7 +34,8 @@ public: int records, int batch = 512, bool allowConstraintViolation = true, - int doSleep = 0); + int doSleep = 0, + bool oneTrans = false); int scanReadRecords(Ndb*, int records, int abort = 0, diff --git a/ndb/test/ndbapi/testBasic.cpp b/ndb/test/ndbapi/testBasic.cpp index 64dfe492c2c..af25a36dde2 100644 --- a/ndb/test/ndbapi/testBasic.cpp +++ b/ndb/test/ndbapi/testBasic.cpp @@ -29,9 +29,18 @@ * delete should be visible to same transaction * */ +int runLoadTable2(NDBT_Context* ctx, NDBT_Step* step) +{ + int records = ctx->getNumRecords(); + HugoTransactions hugoTrans(*ctx->getTab()); + if (hugoTrans.loadTable(GETNDB(step), records, 512, false, 0, true) != 0){ + return NDBT_FAILED; + } + return NDBT_OK; +} -int runLoadTable(NDBT_Context* ctx, NDBT_Step* step){ - +int runLoadTable(NDBT_Context* ctx, NDBT_Step* step) +{ int records = ctx->getNumRecords(); HugoTransactions hugoTrans(*ctx->getTab()); if (hugoTrans.loadTable(GETNDB(step), records) != 0){ @@ -1255,6 +1264,11 @@ TESTCASE("MassiveRollback2", INITIALIZER(runMassiveRollback2); FINALIZER(runClearTable2); } +TESTCASE("MassiveTransaction", + "Test very large insert transaction"){ + INITIALIZER(runLoadTable2); + FINALIZER(runClearTable2); +} NDBT_TESTSUITE_END(testBasic); int main(int argc, const char** argv){ diff --git a/ndb/test/src/HugoTransactions.cpp b/ndb/test/src/HugoTransactions.cpp index 7f12484ddc8..994ad3284bb 100644 --- a/ndb/test/src/HugoTransactions.cpp +++ b/ndb/test/src/HugoTransactions.cpp @@ -819,12 +819,14 @@ HugoTransactions::loadTable(Ndb* pNdb, int records, int batch, bool allowConstraintViolation, - int doSleep){ + int doSleep, + bool oneTrans){ int check; int retryAttempt = 0; int retryMax = 5; NdbConnection *pTrans; NdbOperation *pOp; + bool first_batch = true; const int org = batch; const int cols = tab.getNoOfColumns(); @@ -833,7 +835,7 @@ HugoTransactions::loadTable(Ndb* pNdb, batch = (batch * 256); // -> 512 -> 65536k per commit batch = batch/bytes; // batch = batch == 0 ? 1 : batch; - + if(batch != org){ g_info << "batch = " << org << " rowsize = " << bytes << " -> rows/commit = " << batch << endl; @@ -841,7 +843,7 @@ HugoTransactions::loadTable(Ndb* pNdb, g_info << "|- Inserting records..." << endl; for (int c=0 ; c= retryMax){ g_info << "Record " << c << " could not be inserted, has retried " << retryAttempt << " times " << endl; @@ -852,19 +854,22 @@ HugoTransactions::loadTable(Ndb* pNdb, if (doSleep > 0) NdbSleep_MilliSleep(doSleep); - pTrans = pNdb->startTransaction(); + if (first_batch || !oneTrans) { + first_batch = false; + pTrans = pNdb->startTransaction(); + + if (pTrans == NULL) { + const NdbError err = pNdb->getNdbError(); - if (pTrans == NULL) { - const NdbError err = pNdb->getNdbError(); - - if (err.status == NdbError::TemporaryError){ - ERR(err); - NdbSleep_MilliSleep(50); - retryAttempt++; - continue; + if (err.status == NdbError::TemporaryError){ + ERR(err); + NdbSleep_MilliSleep(50); + retryAttempt++; + continue; + } + ERR(err); + return NDBT_FAILED; } - ERR(err); - return NDBT_FAILED; } for(int b = 0; b < batch && c+bexecute( Commit ); + if (!oneTrans || (c + batch) >= records) { + closeTrans = true; + check = pTrans->execute( Commit ); + } else { + closeTrans = false; + check = pTrans->execute( NoCommit ); + } if(check == -1 ) { const NdbError err = pTrans->getNdbError(); pNdb->closeTransaction(pTrans); @@ -937,8 +948,10 @@ HugoTransactions::loadTable(Ndb* pNdb, break; } } - else{ - pNdb->closeTransaction(pTrans); + else{ + if (closeTrans) { + pNdb->closeTransaction(pTrans); + } } // Step to next record From abfdb34830c19e45ee6b31c8a9c0534aaecf540f Mon Sep 17 00:00:00 2001 From: unknown Date: Tue, 20 Jul 2004 00:23:49 +0200 Subject: [PATCH 2/3] Bug #4479 Ensures that the node doesn't crash by overflowing the UNDO log buffer at local checkpoints. Inserts a real-time break after 512 operations and when low on UNDO log buffer. --- ndb/src/kernel/blocks/dbacc/Dbacc.hpp | 2 + ndb/src/kernel/blocks/dbacc/DbaccMain.cpp | 106 ++++++++++++++++------ 2 files changed, 78 insertions(+), 30 deletions(-) diff --git a/ndb/src/kernel/blocks/dbacc/Dbacc.hpp b/ndb/src/kernel/blocks/dbacc/Dbacc.hpp index 6ba2d083e58..5185e91caac 100644 --- a/ndb/src/kernel/blocks/dbacc/Dbacc.hpp +++ b/ndb/src/kernel/blocks/dbacc/Dbacc.hpp @@ -218,6 +218,7 @@ ndbout << "Ptr: " << ptr.p->word32 << " \tIndex: " << tmp_string << " \tValue: " #define ZREL_FRAG 6 #define ZREL_DIR 7 #define ZREPORT_MEMORY_USAGE 8 +#define ZLCP_OP_WRITE_RT_BREAK 9 /* ------------------------------------------------------------------------- */ /* ERROR CODES */ @@ -1190,6 +1191,7 @@ private: void zpagesize_error(const char* where); void reportMemoryUsage(Signal* signal, int gth); + void lcp_write_op_to_undolog(Signal* signal); // Initialisation diff --git a/ndb/src/kernel/blocks/dbacc/DbaccMain.cpp b/ndb/src/kernel/blocks/dbacc/DbaccMain.cpp index 933ee2cf8e1..ccc1acdd273 100644 --- a/ndb/src/kernel/blocks/dbacc/DbaccMain.cpp +++ b/ndb/src/kernel/blocks/dbacc/DbaccMain.cpp @@ -46,13 +46,17 @@ Dbacc::remainingUndoPages(){ ndbrequire(HeadPage>=TailPage); Uint32 UsedPages = HeadPage - TailPage; - Uint32 Remaining = cundopagesize - UsedPages; + Int32 Remaining = cundopagesize - UsedPages; // There can not be more than cundopagesize remaining - ndbrequire(Remaining<=cundopagesize); - + if (Remaining <= 0){ + // No more undolog, crash node + progError(__LINE__, + ERR_NO_MORE_UNDOLOG, + "There are more than 1Mbyte undolog writes outstanding"); + } return Remaining; -}//Dbacc::remainingUndoPages() +} void Dbacc::updateLastUndoPageIdWritten(Signal* signal, Uint32 aNewValue){ @@ -193,6 +197,17 @@ void Dbacc::execCONTINUEB(Signal* signal) return; } + case ZLCP_OP_WRITE_RT_BREAK: + { + operationRecPtr.i= signal->theData[1]; + fragrecptr.i= signal->theData[2]; + lcpConnectptr.i= signal->theData[3]; + ptrCheckGuard(operationRecPtr, coprecsize, operationrec); + ptrCheckGuard(fragrecptr, cfragmentsize, fragmentrec); + ptrCheckGuard(lcpConnectptr, clcpConnectsize, lcpConnectrec); + lcp_write_op_to_undolog(signal); + return; + } default: ndbrequire(false); break; @@ -7697,32 +7712,70 @@ void Dbacc::execACC_LCPREQ(Signal* signal) fragrecptr.p->lcpMaxOverDirIndex = fragrecptr.p->lastOverIndex; fragrecptr.p->createLcp = ZTRUE; operationRecPtr.i = fragrecptr.p->lockOwnersList; - while (operationRecPtr.i != RNIL) { - jam(); - ptrCheckGuard(operationRecPtr, coprecsize, operationrec); + lcp_write_op_to_undolog(signal); +} - if ((operationRecPtr.p->operation == ZINSERT) || - (operationRecPtr.p->elementIsDisappeared == ZTRUE)){ +void +Dbacc::lcp_write_op_to_undolog(Signal* signal) +{ + bool delay_continueb= false; + Uint32 i, j; + for (i= 0; i < 16; i++) { + jam(); + if (remainingUndoPages() <= ZMIN_UNDO_PAGES_AT_COMMIT) { + jam(); + delay_continueb= true; + break; + } + for (j= 0; j < 32; j++) { + if (operationRecPtr.i == RNIL) { + jam(); + break; + } + jam(); + ptrCheckGuard(operationRecPtr, coprecsize, operationrec); + + if ((operationRecPtr.p->operation == ZINSERT) || + (operationRecPtr.p->elementIsDisappeared == ZTRUE)){ /******************************************************************* * Only log inserts and elements that are marked as dissapeared. * All other operations update the element header and that is handled * when pages are written to disk ********************************************************************/ - undopageptr.i = (cundoposition>>ZUNDOPAGEINDEXBITS) & (cundopagesize-1); - ptrAss(undopageptr, undopage); - theadundoindex = cundoposition & ZUNDOPAGEINDEX_MASK; - tundoindex = theadundoindex + ZUNDOHEADSIZE; + undopageptr.i = (cundoposition>>ZUNDOPAGEINDEXBITS) & (cundopagesize-1); + ptrAss(undopageptr, undopage); + theadundoindex = cundoposition & ZUNDOPAGEINDEX_MASK; + tundoindex = theadundoindex + ZUNDOHEADSIZE; - writeUndoOpInfo(signal);/* THE INFORMATION ABOUT ELEMENT HEADER, STORED*/ - /* IN OP REC, IS WRITTEN AT UNDO PAGES */ - cundoElemIndex = 0;/* DEFAULT VALUE USED BY WRITE_UNDO_HEADER SUBROTINE */ - writeUndoHeader(signal, RNIL, UndoHeader::ZOP_INFO); /* WRITE THE HEAD OF THE UNDO ELEMENT */ - checkUndoPages(signal); /* SEND UNDO PAGE TO DISK WHEN A GROUP OF */ + writeUndoOpInfo(signal);/* THE INFORMATION ABOUT ELEMENT HEADER, STORED*/ + /* IN OP REC, IS WRITTEN AT UNDO PAGES */ + cundoElemIndex = 0;/* DEFAULT VALUE USED BY WRITE_UNDO_HEADER SUBROTINE */ + writeUndoHeader(signal, RNIL, UndoHeader::ZOP_INFO); /* WRITE THE HEAD OF THE UNDO ELEMENT */ + checkUndoPages(signal); /* SEND UNDO PAGE TO DISK WHEN A GROUP OF */ /* UNDO PAGES,CURRENTLY 8, IS FILLED */ - }//if - - operationRecPtr.i = operationRecPtr.p->nextLockOwnerOp; - }//while + } + operationRecPtr.i = operationRecPtr.p->nextLockOwnerOp; + } + if (operationRecPtr.i == RNIL) { + jam(); + break; + } + } + if (operationRecPtr.i != RNIL) { + jam(); + signal->theData[0]= ZLCP_OP_WRITE_RT_BREAK; + signal->theData[1]= operationRecPtr.i; + signal->theData[2]= fragrecptr.i; + signal->theData[3]= lcpConnectptr.i; + if (delay_continueb) { + jam(); + sendSignalWithDelay(cownBlockref, GSN_CONTINUEB, signal, 10, 4); + } else { + jam(); + sendSignal(cownBlockref, GSN_CONTINUEB, signal, 4, JBB); + } + return; + } signal->theData[0] = fragrecptr.p->lcpLqhPtr; sendSignal(lcpConnectptr.p->lcpUserblockref, GSN_ACC_LCPSTARTED, @@ -7735,8 +7788,7 @@ void Dbacc::execACC_LCPREQ(Signal* signal) signal->theData[0] = lcpConnectptr.i; signal->theData[1] = fragrecptr.i; sendSignal(cownBlockref, GSN_ACC_SAVE_PAGES, signal, 2, JBB); - return; -}//Dbacc::execACC_LCPREQ() +} /* ******************--------------------------------------------------------------- */ /* ACC_SAVE_PAGES A GROUP OF PAGES IS ALLOCATED. THE PAGES AND OVERFLOW */ @@ -8595,12 +8647,6 @@ void Dbacc::checkUndoPages(Signal* signal) * RECORDS IN */ Uint16 nextUndoPageId = tundoPageId + 1; - if (nextUndoPageId > (clastUndoPageIdWritten + cundopagesize)){ - // No more undolog, crash node - progError(__LINE__, - ERR_NO_MORE_UNDOLOG, - "There are more than 1Mbyte undolog writes outstanding"); - } updateUndoPositionPage(signal, nextUndoPageId << ZUNDOPAGEINDEXBITS); if ((tundoPageId & (ZWRITE_UNDOPAGESIZE - 1)) == (ZWRITE_UNDOPAGESIZE - 1)) { From d392ff21307626e0b7605ebd3105eb45d2208863 Mon Sep 17 00:00:00 2001 From: unknown Date: Fri, 30 Jul 2004 12:46:27 +0200 Subject: [PATCH 3/3] Removed an error check from debug mode that gets executed n**2 times in closeTransaction where n is number of signals sent in transaction. n can easily become 250.000 in a large transaction. --- ndb/src/ndbapi/Ndblist.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ndb/src/ndbapi/Ndblist.cpp b/ndb/src/ndbapi/Ndblist.cpp index e557fdc0a5f..a27b911eb07 100644 --- a/ndb/src/ndbapi/Ndblist.cpp +++ b/ndb/src/ndbapi/Ndblist.cpp @@ -592,13 +592,14 @@ Ndb::releaseSignal(NdbApiSignal* aSignal) #if defined VM_TRACE // Check that signal is not null assert(aSignal != NULL); - +#if 0 // Check that signal is not already in list NdbApiSignal* tmp = theSignalIdleList; while (tmp != NULL){ assert(tmp != aSignal); tmp = tmp->next(); } +#endif #endif creleaseSignals++; aSignal->next(theSignalIdleList);