From 3a69c854c55025acc687e5852d7be5e38a38934e Mon Sep 17 00:00:00 2001 From: Olivier Bertrand Date: Sat, 16 Aug 2014 16:46:35 +0200 Subject: [PATCH] - Modifies the way indexed UPDATE/DELETE are sorted in order to execute them sorted by file position. Firstly a new value is stored in indexes to know if they are sorted, preventing to do the sorting when it is not needed. Secondly, almost all in now done in connect instead of being done by the different file access method classes. This pepares the future use of temporary files for all table types and also fix the bug that was occuring when partially using a multi-column index because of false MRR like call of position followed by unsorted rnd_pos no more using indexing. modified: storage/connect/connect.cc storage/connect/filamap.cpp storage/connect/filamap.h storage/connect/filamdbf.cpp storage/connect/filamdbf.h storage/connect/filamfix.cpp storage/connect/filamfix.h storage/connect/filamtxt.cpp storage/connect/filamtxt.h storage/connect/filamvct.cpp storage/connect/filamvct.h storage/connect/tabdos.cpp storage/connect/tabdos.h storage/connect/tabfix.h storage/connect/tabfmt.cpp storage/connect/tabfmt.h storage/connect/xindex.cpp storage/connect/xindex.h storage/connect/xtable.h --- storage/connect/connect.cc | 71 +++++-- storage/connect/filamap.cpp | 85 ++------ storage/connect/filamap.h | 5 +- storage/connect/filamdbf.cpp | 36 +++- storage/connect/filamdbf.h | 5 +- storage/connect/filamfix.cpp | 155 ++------------ storage/connect/filamfix.h | 4 +- storage/connect/filamtxt.cpp | 364 ++++++++++++++++----------------- storage/connect/filamtxt.h | 17 +- storage/connect/filamvct.cpp | 278 +------------------------ storage/connect/filamvct.h | 4 - storage/connect/tabdos.cpp | 30 ++- storage/connect/tabdos.h | 3 + storage/connect/tabfix.h | 2 + storage/connect/tabfmt.cpp | 18 +- storage/connect/tabfmt.h | 381 ++++++++++++++++++----------------- storage/connect/xindex.cpp | 40 +++- storage/connect/xindex.h | 1 + storage/connect/xtable.h | 5 + 19 files changed, 591 insertions(+), 913 deletions(-) diff --git a/storage/connect/connect.cc b/storage/connect/connect.cc index e495a0f62b9..a458c0e0c5a 100644 --- a/storage/connect/connect.cc +++ b/storage/connect/connect.cc @@ -479,7 +479,7 @@ RCODE CntReadNext(PGLOBAL g, PTDB tdbp) /***********************************************************************/ RCODE CntWriteRow(PGLOBAL g, PTDB tdbp) { - RCODE rc; + RCODE rc; PCOL colp; PTDBASE tp= (PTDBASE)tdbp; @@ -503,11 +503,14 @@ RCODE CntWriteRow(PGLOBAL g, PTDB tdbp) if (!colp->GetColUse(U_VIRTUAL)) colp->WriteColumn(g); -// if (tdbp->GetMode() == MODE_INSERT) -// tbxp->SetModified(true); - - // Return result code from write operation - rc= (RCODE)tdbp->WriteDB(g); +//if (tp->GetMode() == MODE_UPDATE && tp->IsUsingTemp(g) && +// tp->GetKindex() && !tp->GetKindex()->IsSorted()) + if (tp->IsIndexed()) + // Index values must be sorted before updating + rc= (RCODE)((PTDBDOS)tp)->GetTxfp()->StoreValues(g, true); + else + // Return result code from write operation + rc= (RCODE)tdbp->WriteDB(g); err: g->jump_level--; @@ -517,7 +520,7 @@ RCODE CntWriteRow(PGLOBAL g, PTDB tdbp) /***********************************************************************/ /* UpdateRow: Update a row into a table. */ /***********************************************************************/ -RCODE CntUpdateRow(PGLOBAL g, PTDB tdbp) +RCODE CntUpdateRow(PGLOBAL g, PTDB tdbp) { if (!tdbp || tdbp->GetMode() != MODE_UPDATE) return RC_FX; @@ -531,19 +534,28 @@ RCODE CntUpdateRow(PGLOBAL g, PTDB tdbp) /***********************************************************************/ RCODE CntDeleteRow(PGLOBAL g, PTDB tdbp, bool all) { - RCODE rc; + RCODE rc; + PTDBASE tp= (PTDBASE)tdbp; if (!tdbp || tdbp->GetMode() != MODE_DELETE) return RC_FX; else if (tdbp->IsReadOnly()) return RC_NF; - if (((PTDBASE)tdbp)->GetDef()->Indexable() && all) - ((PTDBDOS)tdbp)->Cardinal= 0; + if (all) { + if (((PTDBASE)tdbp)->GetDef()->Indexable()) + ((PTDBDOS)tdbp)->Cardinal= 0; + + // Note: if all, this call will be done when closing the table + rc= (RCODE)tdbp->DeleteDB(g, RC_FX); +//} else if (tp->GetKindex() && !tp->GetKindex()->IsSorted() && +// tp->Txfp->GetAmType() != TYPE_AM_DBF) { + } else if(tp->IsIndexed()) { + // Index values must be sorted before updating + rc= (RCODE)((PTDBDOS)tp)->GetTxfp()->StoreValues(g, false); + } else // Return result code from delete operation + rc= (RCODE)tdbp->DeleteDB(g, RC_OK); - // Return result code from delete operation - // Note: if all, this call will be done when closing the table - rc= (RCODE)tdbp->DeleteDB(g, (all) ? RC_FX : RC_OK); return rc; } // end of CntDeleteRow @@ -553,7 +565,7 @@ RCODE CntDeleteRow(PGLOBAL g, PTDB tdbp, bool all) int CntCloseTable(PGLOBAL g, PTDB tdbp, bool nox, bool abort) { int rc= RC_OK; - TDBDOX *tbxp= NULL; + TDBASE *tbxp= (PTDBASE)tdbp; if (!tdbp) return rc; // Nothing to do @@ -568,8 +580,30 @@ int CntCloseTable(PGLOBAL g, PTDB tdbp, bool nox, bool abort) printf("CntCloseTable: tdbp=%p mode=%d nox=%d abort=%d\n", tdbp, tdbp->GetMode(), nox, abort); - if (tdbp->GetMode() == MODE_DELETE && tdbp->GetUse() == USE_OPEN) - rc= tdbp->DeleteDB(g, RC_EF); // Specific A.M. delete routine +#if 0 + if (tbxp->GetMode() == MODE_UPDATE && + tbxp->GetKindex() && !tbxp->GetKindex()->IsSorted()) { + rc= tbxp->Txfp->UpdateSortedRows(g); + } else + if (tdbp->GetMode() == MODE_DELETE && tdbp->GetUse() == USE_OPEN) { + if (tbxp->GetKindex() && !tbxp->GetKindex()->IsSorted()) + rc= tbxp->Txfp->DeleteSortedRows(g); + + if (!rc) + rc= tdbp->DeleteDB(g, RC_EF); // Specific A.M. delete routine + + } // endif Mode +#endif // 0 + + if (tdbp->GetMode() == MODE_DELETE && tdbp->GetUse() == USE_OPEN) { + if (tbxp->IsIndexed()) + rc= ((PTDBDOS)tdbp)->GetTxfp()->DeleteSortedRows(g); + + if (!rc) + rc= tdbp->DeleteDB(g, RC_EF); // Specific A.M. delete routine + + } else if (tbxp->GetMode() == MODE_UPDATE && tbxp->IsIndexed()) + rc= ((PTDBDOX)tdbp)->Txfp->UpdateSortedRows(g); // Prepare error return if (g->jump_level == MAX_JUMP) { @@ -606,9 +640,8 @@ int CntCloseTable(PGLOBAL g, PTDB tdbp, bool nox, bool abort) // Make all the eventual indexes tbxp= (TDBDOX*)tdbp; tbxp->ResetKindex(g, NULL); - tbxp->To_Key_Col= NULL; - rc= tbxp->ResetTableOpt(g, true, - ((PTDBASE)tdbp)->GetDef()->Indexable() == 1); + tbxp->SetKey_Col(NULL); + rc= tbxp->ResetTableOpt(g, true, tbxp->GetDef()->Indexable() == 1); err: if (trace > 1) diff --git a/storage/connect/filamap.cpp b/storage/connect/filamap.cpp index b5958c1a854..3523c688788 100644 --- a/storage/connect/filamap.cpp +++ b/storage/connect/filamap.cpp @@ -48,11 +48,6 @@ extern "C" int trace; -/***********************************************************************/ -/* Routine called externally by MAPFAM MakeDeletedFile function. */ -/***********************************************************************/ -PARRAY MakeValueArray(PGLOBAL g, PPARM pp); - /* --------------------------- Class MAPFAM -------------------------- */ /***********************************************************************/ @@ -290,6 +285,16 @@ bool MAPFAM::RecordPos(PGLOBAL g) return false; } // end of RecordPos +/***********************************************************************/ +/* Initialize Fpos and Mempos for indexed DELETE. */ +/***********************************************************************/ +int MAPFAM::InitDelete(PGLOBAL g, int fpos, int spos) + { + Fpos = Memory + fpos; + Mempos = Memory + spos; + return RC_OK; + } // end of InitDelete + /***********************************************************************/ /* Skip one record in file. */ /***********************************************************************/ @@ -409,13 +414,6 @@ int MAPFAM::DeleteRecords(PGLOBAL g, int irc) /* not required here, just setting of future Spos and Tpos. */ /*******************************************************************/ Tpos = Spos = Fpos; - Indxd = Tdbp->GetKindex() != NULL; - } // endif Tpos - - if (Indxd) { - // Moving will be done later, must be done in sequential order - (void)AddListValue(g, TYPE_PCHAR, Fpos, &To_Pos); - (void)AddListValue(g, TYPE_PCHAR, Mempos, &To_Sos); } else if ((n = Fpos - Spos) > 0) { /*****************************************************************/ /* Non consecutive line to delete. Move intermediate lines. */ @@ -437,10 +435,6 @@ int MAPFAM::DeleteRecords(PGLOBAL g, int irc) } else if (To_Fb) { // Can be NULL for deleted files /*******************************************************************/ /* Last call after EOF has been reached. */ - /*******************************************************************/ - Abort = (Indxd && MakeDeletedFile(g)); - - /*******************************************************************/ /* We must firstly Unmap the view and use the saved file handle */ /* to put an EOF at the end of the copied part of the file. */ /*******************************************************************/ @@ -495,55 +489,6 @@ int MAPFAM::DeleteRecords(PGLOBAL g, int irc) return RC_OK; // All is correct } // end of DeleteRecords -/***********************************************************************/ -/* MakeDeletedFile. When deleting using indexing, the issue is that */ -/* record are not necessarily deleted in sequential order. Moving */ -/* intermediate lines cannot be done while deleting them. */ -/* What we do here is to reorder the deleted records and move the */ -/* intermediate files from the ordered deleted record positions. */ -/***********************************************************************/ -bool MAPFAM::MakeDeletedFile(PGLOBAL g) - { - int *ix, i, n; - - /*********************************************************************/ - /* Make and order the arrays from the saved values. */ - /*********************************************************************/ - if (!(Posar = MakeValueArray(g, To_Pos))) { - strcpy(g->Message, "Position array is null"); - goto err; - } else if (!(Sosar = MakeValueArray(g, To_Sos))) { - strcpy(g->Message, "Start position array is null"); - goto err; - } else if (!(ix = (int*)Posar->GetSortIndex(g))) { - strcpy(g->Message, "Error getting array sort index"); - goto err; - } // endif's - - for (i = 0; i < Posar->GetNval(); i++) { - Fpos = Posar->GetStringValue(ix[i]); - - if (!i) { - Tpos = Fpos; - } else if ((n = Fpos - Spos) >= 0) { - // Move all not deleted lines preceding this one - memmove(Tpos, Spos, n); - Tpos += n; - } // endif n - - // New start position - Spos = Sosar->GetStringValue(ix[i]); - } // endfor i - - return false; - -err: - if (trace) - htrc("%s\n", g->Message); - - return true; - } // end of MakeDeletedFile - /***********************************************************************/ /* Table file close routine for MAP access method. */ /***********************************************************************/ @@ -734,6 +679,16 @@ bool MPXFAM::SetPos(PGLOBAL g, int pos) return false; } // end of SetPos +/***********************************************************************/ +/* Initialize CurBlk, CurNum, Mempos and Fpos for indexed DELETE. */ +/***********************************************************************/ +int MPXFAM::InitDelete(PGLOBAL g, int fpos, int spos) + { + Fpos = Memory + Headlen + fpos * Lrecl; + Mempos = Fpos + Lrecl; + return RC_OK; + } // end of InitDelete + /***********************************************************************/ /* ReadBuffer: Read one line for a mapped Fix file. */ /***********************************************************************/ diff --git a/storage/connect/filamap.h b/storage/connect/filamap.h index 7d3203d7ff2..1d85fa36155 100644 --- a/storage/connect/filamap.h +++ b/storage/connect/filamap.h @@ -47,7 +47,7 @@ class DllExport MAPFAM : public TXTFAM { virtual void Rewind(void); protected: - bool MakeDeletedFile(PGLOBAL g); + virtual int InitDelete(PGLOBAL g, int fpos, int spos); // Members char *Memory; // Pointer on file mapping view. @@ -104,11 +104,14 @@ class DllExport MPXFAM : public MBKFAM { virtual int MaxBlkSize(PGLOBAL g, int s) {return TXTFAM::MaxBlkSize(g, s);} virtual bool SetPos(PGLOBAL g, int recpos); + virtual int GetNextPos(void) {return (int)Fpos + Nrec;} virtual bool DeferReading(void) {return false;} virtual int ReadBuffer(PGLOBAL g); virtual int WriteBuffer(PGLOBAL g); protected: + virtual int InitDelete(PGLOBAL g, int fpos, int spos); + // No additional members }; // end of class MPXFAM diff --git a/storage/connect/filamdbf.cpp b/storage/connect/filamdbf.cpp index 5d01ee06df0..50a97739002 100644 --- a/storage/connect/filamdbf.cpp +++ b/storage/connect/filamdbf.cpp @@ -750,6 +750,36 @@ bool DBFFAM::CopyHeader(PGLOBAL g) return rc; } // end of CopyHeader +#if 0 // Not useful when UseTemp is false. +/***********************************************************************/ +/* Mark the line to delete with '*' (soft delete). */ +/* NOTE: this is not ready for UseTemp. */ +/***********************************************************************/ +int DBFFAM::InitDelete(PGLOBAL g, int fpos, int spos) + { + int rc = RC_FX; + size_t lrecl = (size_t)Lrecl; + + if (Nrec != 1) + strcpy(g->Message, "Cannot delete in block mode"); + else if (fseek(Stream, Headlen + fpos * Lrecl, SEEK_SET)) + sprintf(g->Message, MSG(FSETPOS_ERROR), 0); + else if (fread(To_Buf, 1, lrecl, Stream) != lrecl) + sprintf(g->Message, MSG(READ_ERROR), To_File, strerror(errno)); + else + *To_Buf = '*'; + + if (fseek(Stream, Headlen + fpos * Lrecl, SEEK_SET)) + sprintf(g->Message, MSG(FSETPOS_ERROR), 0); + else if (fwrite(To_Buf, 1, lrecl, Stream) != lrecl) + sprintf(g->Message, MSG(FWRITE_ERROR), strerror(errno)); + else + rc = RC_NF; // Ok, Nothing else to do + + return rc; + } // end of InitDelete +#endif // 0 + /***********************************************************************/ /* Data Base delete line routine for DBF access methods. */ /* Deleted lines are just flagged in the first buffer character. */ @@ -760,16 +790,12 @@ int DBFFAM::DeleteRecords(PGLOBAL g, int irc) // T_Stream is the temporary stream or the table file stream itself if (!T_Stream) if (UseTemp) { - if ((Indxd = Tdbp->GetKindex() != NULL)) { - strcpy(g->Message, "DBF indexed udate using temp file NIY"); - return RC_FX; - } else if (OpenTempFile(g)) + if (OpenTempFile(g)) return RC_FX; if (CopyHeader(g)) // For DBF tables return RC_FX; -// Indxd = Tdbp->GetKindex() != NULL; } else T_Stream = Stream; diff --git a/storage/connect/filamdbf.h b/storage/connect/filamdbf.h index 0345c0338e8..9be8bfd980d 100644 --- a/storage/connect/filamdbf.h +++ b/storage/connect/filamdbf.h @@ -40,8 +40,8 @@ class DllExport DBFBASE { // Members int Records; /* records in the file */ bool Accept; /* true if bad lines are accepted */ - int Nerr; /* Number of bad records */ - int Maxerr; /* Maximum number of bad records */ + int Nerr; /* Number of bad records */ + int Maxerr; /* Maximum number of bad records */ int ReadMode; /* 1: ALL 2: DEL 0: NOT DEL */ }; // end of class DBFBASE @@ -72,6 +72,7 @@ class DllExport DBFFAM : public FIXFAM, public DBFBASE { protected: virtual bool CopyHeader(PGLOBAL g); +//virtual int InitDelete(PGLOBAL g, int fpos, int spos); // Members }; // end of class DBFFAM diff --git a/storage/connect/filamfix.cpp b/storage/connect/filamfix.cpp index 78b6ee58b24..1c9451bb982 100644 --- a/storage/connect/filamfix.cpp +++ b/storage/connect/filamfix.cpp @@ -55,11 +55,6 @@ extern "C" int trace; extern int num_read, num_there, num_eq[2]; // Statistics -/***********************************************************************/ -/* Routine called externally by BGXFAM MakeDeletedFile function. */ -/***********************************************************************/ -PARRAY MakeValueArray(PGLOBAL g, PPARM pp); - /* --------------------------- Class FIXFAM -------------------------- */ /***********************************************************************/ @@ -105,6 +100,16 @@ bool FIXFAM::SetPos(PGLOBAL g, int pos) return false; } // end of SetPos +/***********************************************************************/ +/* Initialize CurBlk and CurNum for indexed DELETE. */ +/***********************************************************************/ +int FIXFAM::InitDelete(PGLOBAL g, int fpos, int spos) + { + CurBlk = fpos / Nrec; + CurNum = fpos % Nrec; + return RC_OK; + } // end of InitDelete + /***********************************************************************/ /* Allocate the block buffer for the table. */ /***********************************************************************/ @@ -340,15 +345,11 @@ int FIXFAM::WriteBuffer(PGLOBAL g) // T_Stream is the temporary stream or the table file stream itself if (!T_Stream) { if (UseTemp) { - if ((Indxd = Tdbp->GetKindex() != NULL)) { - strcpy(g->Message, "FIX indexed udate using temp file NIY"); - return RC_FX; - } else if (OpenTempFile(g)) + if (OpenTempFile(g)) return RC_FX; else if (CopyHeader(g)) // For DBF tables return RC_FX; -// Indxd = Tdbp->GetKindex() != NULL; } else T_Stream = Stream; @@ -413,17 +414,12 @@ int FIXFAM::DeleteRecords(PGLOBAL g, int irc) Spos = Tpos = Fpos; } // endif UseTemp - Indxd = Tdbp->GetKindex() != NULL; } // endif Tpos == Spos /*********************************************************************/ /* Move any intermediate lines. */ /*********************************************************************/ - if (Indxd) { - // Moving will be done later, must be done in sequential order - (void)AddListValue(g, TYPE_INT, &Fpos, &To_Pos); - moved = false; - } else if (MoveIntermediateLines(g, &moved)) + if (MoveIntermediateLines(g, &moved)) return RC_FX; if (irc == RC_OK) { @@ -456,9 +452,6 @@ int FIXFAM::DeleteRecords(PGLOBAL g, int irc) return RC_FX; } else { - if (Indxd && (Abort = MakeDeletedFile(g))) - return RC_FX; - /*****************************************************************/ /* Because the chsize functionality is only accessible with a */ /* system call we must close the file and reopen it with the */ @@ -559,59 +552,6 @@ bool FIXFAM::MoveIntermediateLines(PGLOBAL g, bool *b) return false; } // end of MoveIntermediate Lines -/***********************************************************************/ -/* MakeDeletedFile. When deleting using indexing, the issue is that */ -/* record are not necessarily deleted in sequential order. Moving */ -/* intermediate lines cannot be done while deleing them because */ -/* this can cause extra wrong records to be included in the new file. */ -/* What we do here is to reorder the deleted record and make the new */ -/* deleted file from the ordered deleted records. */ -/***********************************************************************/ -bool FIXFAM::MakeDeletedFile(PGLOBAL g) - { - char *crlf = "\n", *mode = UseTemp ? "rb" : "r+b"; - int *ix, i; - bool moved; - - /*********************************************************************/ - /* Open the temporary file, Spos is at the beginning of file. */ - /*********************************************************************/ - if (!(Posar = MakeValueArray(g, To_Pos))) { - strcpy(g->Message, "Position array is null"); - goto err; - } else if (!(ix = (int*)Posar->GetSortIndex(g))) { - strcpy(g->Message, "Error getting array sort index"); - goto err; - } // endif's - - Spos = 0; - - for (i = 0; i < Posar->GetNval(); i++) { - Fpos = Posar->GetIntValue(ix[i]); - - if (i || UseTemp) { - // Copy all not updated lines preceding this one - if (MoveIntermediateLines(g, &moved)) - goto err; - - } else - Tpos = Fpos; - - // New start position - Spos = Fpos + 1; - } // endfor i - - if (!PlugCloseFile(g, To_Fbt) && !PlugCloseFile(g, To_Fb)) - return false; - -err: - if (trace) - htrc("%s\n", g->Message); - - PlugCloseFile(g, To_Fbt); - return true; - } // end of MakeDeletedFile - /***********************************************************************/ /* Table file close routine for FIX access method. */ /***********************************************************************/ @@ -640,7 +580,6 @@ void FIXFAM::CloseTableFile(PGLOBAL g, bool abort) // Copy any remaining lines bool b; - // Note: Indxd is not implemented yet Fpos = Tdbp->Cardinality(g); Abort = MoveIntermediateLines(g, &b) != RC_OK; } // endif Abort @@ -1233,10 +1172,7 @@ int BGXFAM::WriteBuffer(PGLOBAL g) if (Tfile == INVALID_HANDLE_VALUE) { if (UseTemp /*&& Tdbp->GetMode() == MODE_UPDATE*/) { - if ((Indxd = Tdbp->GetKindex() != NULL)) { - strcpy(g->Message, "FIX indexed udate using temp file NIY"); - return RC_FX; - } else if (OpenTempFile(g)) + if (OpenTempFile(g)) return RC_FX; } else @@ -1303,19 +1239,15 @@ int BGXFAM::DeleteRecords(PGLOBAL g, int irc) Spos = Tpos = Fpos; } // endif UseTemp - Indxd = Tdbp->GetKindex() != NULL; } // endif Tpos == Spos /*********************************************************************/ /* Move any intermediate lines. */ /*********************************************************************/ - if (Indxd) - // Moving will be done later, must be done in sequential order - (void)AddListValue(g, TYPE_INT, &Fpos, &To_Pos); - else if (MoveIntermediateLines(g, &moved)) + if (MoveIntermediateLines(g, &moved)) return RC_FX; - if (irc == RC_OK && !Indxd) { + if (irc == RC_OK) { if (trace) assert(Spos == Fpos); @@ -1343,9 +1275,6 @@ int BGXFAM::DeleteRecords(PGLOBAL g, int irc) return RC_FX; } else { - if (Indxd && (Abort = MakeDeletedFile(g))) - return RC_FX; - /*****************************************************************/ /* Remove extra records. */ /*****************************************************************/ @@ -1469,59 +1398,6 @@ bool BGXFAM::MoveIntermediateLines(PGLOBAL g, bool *b) return false; } // end of MoveIntermediateLines -/***********************************************************************/ -/* MakeDeletedFile. When deleting using indexing, the issue is that */ -/* record are not necessarily deleted in sequential order. Moving */ -/* intermediate lines cannot be done while deleing them because */ -/* this can cause extra wrong records to be included in the new file. */ -/* What we do here is to reorder the deleted record and make the new */ -/* deleted file from the ordered deleted records. */ -/***********************************************************************/ -bool BGXFAM::MakeDeletedFile(PGLOBAL g) - { - char *crlf = "\n", *mode = UseTemp ? "rb" : "r+b"; - int *ix, i; - bool moved; - - /*********************************************************************/ - /* Open the temporary file, Spos is at the beginning of file. */ - /*********************************************************************/ - if (!(Posar = MakeValueArray(g, To_Pos))) { - strcpy(g->Message, "Position array is null"); - goto err; - } else if (!(ix = (int*)Posar->GetSortIndex(g))) { - strcpy(g->Message, "Error getting array sort index"); - goto err; - } // endif's - - Spos = 0; - - for (i = 0; i < Posar->GetNval(); i++) { - Fpos = Posar->GetIntValue(ix[i]); - - if (i || UseTemp) { - // Copy all not updated lines preceding this one - if (MoveIntermediateLines(g, &moved)) - goto err; - - } else - Tpos = Fpos; - - // New start position - Spos = Fpos + 1; - } // endfor i - - if (!PlugCloseFile(g, To_Fbt)) - return false; - -err: - if (trace) - htrc("%s\n", g->Message); - - PlugCloseFile(g, To_Fbt); - return true; - } // end of MakeDeletedFile - /***********************************************************************/ /* Data Base close routine for BIGFIX access method. */ /***********************************************************************/ @@ -1549,7 +1425,6 @@ void BGXFAM::CloseTableFile(PGLOBAL g, bool abort) // Copy any remaining lines bool b; - // Indxd is not implemented yet Fpos = Tdbp->Cardinality(g); Abort = MoveIntermediateLines(g, &b) != RC_OK; } // endif Abort diff --git a/storage/connect/filamfix.h b/storage/connect/filamfix.h index a29bfbbeb48..694e653898a 100644 --- a/storage/connect/filamfix.h +++ b/storage/connect/filamfix.h @@ -34,6 +34,7 @@ class DllExport FIXFAM : public BLKFAM { virtual int MaxBlkSize(PGLOBAL g, int s) {return TXTFAM::MaxBlkSize(g, s);} virtual bool SetPos(PGLOBAL g, int recpos); + virtual int GetNextPos(void) {return Fpos + 1;} virtual bool AllocateBuffer(PGLOBAL g); virtual void ResetBuffer(PGLOBAL g); virtual int ReadBuffer(PGLOBAL g); @@ -44,7 +45,7 @@ class DllExport FIXFAM : public BLKFAM { protected: virtual bool CopyHeader(PGLOBAL g) {return false;} virtual bool MoveIntermediateLines(PGLOBAL g, bool *b); - virtual bool MakeDeletedFile(PGLOBAL g); + virtual int InitDelete(PGLOBAL g, int fpos, int spos); // No additional members }; // end of class FIXFAM @@ -77,7 +78,6 @@ class BGXFAM : public FIXFAM { protected: virtual bool OpenTempFile(PGLOBAL g); virtual bool MoveIntermediateLines(PGLOBAL g, bool *b = NULL); - virtual bool MakeDeletedFile(PGLOBAL g); int BigRead(PGLOBAL g, HANDLE h, void *inbuf, int req); bool BigWrite(PGLOBAL g, HANDLE h, void *inbuf, int req); bool BigSeek(PGLOBAL g, HANDLE h, BIGINT pos diff --git a/storage/connect/filamtxt.cpp b/storage/connect/filamtxt.cpp index 7c2b93afbb1..3e4c5adc319 100644 --- a/storage/connect/filamtxt.cpp +++ b/storage/connect/filamtxt.cpp @@ -59,7 +59,7 @@ extern int num_read, num_there, num_eq[2]; // Statistics extern "C" int trace; /***********************************************************************/ -/* Routine called externally by DOSFAM MakeUpdatedFile function. */ +/* Routine called externally by TXTFAM SortedRows functions. */ /***********************************************************************/ PARRAY MakeValueArray(PGLOBAL g, PPARM pp); @@ -102,10 +102,10 @@ TXTFAM::TXTFAM(PDOSDEF tdp) Rbuf = 0; Modif = 0; Blksize = 0; + Fpos = Spos = Tpos = 0; Padded = false; Eof = tdp->Eof; Ending = tdp->Ending; - Indxd = false; Abort = false; CrLf = (char*)(Ending == 1 ? "\n" : "\r\n"); } // end of TXTFAM standard constructor @@ -144,10 +144,12 @@ TXTFAM::TXTFAM(PTXF txfp) Rbuf = txfp->Rbuf; Modif = txfp->Modif; Blksize = txfp->Blksize; + Fpos = txfp->Fpos; + Spos = txfp->Spos; + Tpos = txfp->Tpos; Padded = txfp->Padded; Eof = txfp->Eof; Ending = txfp->Ending; - Indxd = txfp->Indxd; Abort = txfp->Abort; CrLf = txfp->CrLf; } // end of TXTFAM copy constructor @@ -302,6 +304,134 @@ bool TXTFAM::AddListValue(PGLOBAL g, int type, void *val, PPARM *top) return false; } // end of AddListValue +/***********************************************************************/ +/* Store needed values for indexed UPDATE or DELETE. */ +/***********************************************************************/ +int TXTFAM::StoreValues(PGLOBAL g, bool upd) +{ + int pos = GetPos(); + bool rc = AddListValue(g, TYPE_INT, &pos, &To_Pos); + + if (!rc) { + pos = GetNextPos(); + rc = AddListValue(g, TYPE_INT, &pos, &To_Sos); + } // endif rc + + if (upd && !rc) { + if (Tdbp->PrepareWriting(g)) + return RC_FX; + + rc = AddListValue(g, TYPE_STRING, Tdbp->GetLine(), &To_Upd); + } // endif upd + + return rc ? RC_FX : RC_OK; +} // end of StoreValues + +/***********************************************************************/ +/* UpdateSortedRows. When updating using indexing, the issue is that */ +/* record are not necessarily updated in sequential order. */ +/* Moving intermediate lines cannot be done while making them because */ +/* this can cause extra wrong records to be included in the new file. */ +/* What we do here is to reorder the updated records and do all the */ +/* updates ordered by record position. */ +/***********************************************************************/ +int TXTFAM::UpdateSortedRows(PGLOBAL g) + { + int *ix, i, rc = RC_OK; + + /*********************************************************************/ + /* Get the stored update values and sort them. */ + /*********************************************************************/ + if (!(Posar = MakeValueArray(g, To_Pos))) { + strcpy(g->Message, "Position array is null"); + goto err; + } else if (!(Sosar = MakeValueArray(g, To_Sos))) { + strcpy(g->Message, "Start position array is null"); + goto err; + } else if (!(Updar = MakeValueArray(g, To_Upd))) { + strcpy(g->Message, "Updated line array is null"); + goto err; + } else if (!(ix = (int*)Posar->GetSortIndex(g))) { + strcpy(g->Message, "Error getting array sort index"); + goto err; + } // endif's + + Rewind(); + + for (i = 0; i < Posar->GetNval(); i++) { + SetPos(g, Sosar->GetIntValue(ix[i])); + Fpos = Posar->GetIntValue(ix[i]); + strcpy(Tdbp->To_Line, Updar->GetStringValue(ix[i])); + + // Now write the updated line. + if ((rc = WriteBuffer(g))) + goto err; + + } // endfor i + +err: + if (trace && rc) + htrc("%s\n", g->Message); + + return rc; + } // end of UpdateSortedRows + +/***********************************************************************/ +/* DeleteSortedRows. When deleting using indexing, the issue is that */ +/* record are not necessarily deleted in sequential order. Moving */ +/* intermediate lines cannot be done while deleing them because */ +/* this can cause extra wrong records to be included in the new file. */ +/* What we do here is to reorder the deleted record and delete from */ +/* the file from the ordered deleted records. */ +/***********************************************************************/ +int TXTFAM::DeleteSortedRows(PGLOBAL g) + { + int *ix, i, irc, rc = RC_OK; + + /*********************************************************************/ + /* Get the stored delete values and sort them. */ + /*********************************************************************/ + if (!(Posar = MakeValueArray(g, To_Pos))) { + strcpy(g->Message, "Position array is null"); + goto err; + } else if (!(Sosar = MakeValueArray(g, To_Sos))) { + strcpy(g->Message, "Start position array is null"); + goto err; + } else if (!(ix = (int*)Posar->GetSortIndex(g))) { + strcpy(g->Message, "Error getting array sort index"); + goto err; + } // endif's + + Tpos = Spos = 0; + + for (i = 0; i < Posar->GetNval(); i++) { + if ((irc = InitDelete(g, Posar->GetIntValue(ix[i]), + Sosar->GetIntValue(ix[i])) == RC_FX)) + goto err; + + // Now delete the sorted rows + if ((rc = DeleteRecords(g, irc))) + goto err; + + } // endfor i + +err: + if (trace && rc) + htrc("%s\n", g->Message); + + return rc; + } // end of DeleteSortedRows + +/***********************************************************************/ +/* The purpose of this function is to deal with access methods that */ +/* are not coherent regarding the use of SetPos and GetPos. */ +/***********************************************************************/ +int TXTFAM::InitDelete(PGLOBAL g, int fpos, int spos) + { + strcpy(g->Message, "InitDelete should not be used by this table type"); + return RC_FX; + } // end of InitDelete + /* --------------------------- Class DOSFAM -------------------------- */ /***********************************************************************/ @@ -312,7 +442,6 @@ DOSFAM::DOSFAM(PDOSDEF tdp) : TXTFAM(tdp) To_Fbt = NULL; Stream = NULL; T_Stream = NULL; - Fpos = Spos = Tpos = 0; UseTemp = false; Bin = false; } // end of DOSFAM standard constructor @@ -322,9 +451,6 @@ DOSFAM::DOSFAM(PDOSFAM tdfp) : TXTFAM(tdfp) To_Fbt = tdfp->To_Fbt; Stream = tdfp->Stream; T_Stream = tdfp->T_Stream; - Fpos = tdfp->Fpos; - Spos = tdfp->Spos; - Tpos = tdfp->Tpos; UseTemp = tdfp->UseTemp; Bin = tdfp->Bin; } // end of DOSFAM copy constructor @@ -551,6 +677,21 @@ bool DOSFAM::RecordPos(PGLOBAL g) return false; } // end of RecordPos +/***********************************************************************/ +/* Initialize Fpos and the current position for indexed DELETE. */ +/***********************************************************************/ +int DOSFAM::InitDelete(PGLOBAL g, int fpos, int spos) + { + Fpos = fpos; + + if (fseek(Stream, spos, SEEK_SET)) { + sprintf(g->Message, MSG(FSETPOS_ERROR), Fpos); + return RC_FX; + } // endif + + return RC_OK; + } // end of InitDelete + /***********************************************************************/ /* Skip one record in file. */ /***********************************************************************/ @@ -713,7 +854,6 @@ int DOSFAM::WriteBuffer(PGLOBAL g) if (OpenTempFile(g)) return RC_FX; - Indxd = Tdbp->To_Kindex != NULL; } else T_Stream = Stream; @@ -735,20 +875,13 @@ int DOSFAM::WriteBuffer(PGLOBAL g) if (UseTemp) { /*****************************************************************/ /* We are using a temporary file. */ + /* Before writing the updated record, we must eventually copy */ + /* all the intermediate records that have not been updated. */ /*****************************************************************/ - if (Indxd) { - // Copying will be done later, must be done in sequential order - (void)AddListValue(g, TYPE_INT, &Fpos, &To_Pos); - (void)AddListValue(g, TYPE_INT, &curpos, &To_Sos); - } else { - // Before writing the updated record, we must eventually copy - // all the intermediate records that have not been updated. - if (MoveIntermediateLines(g, &moved)) - return RC_FX; - - Spos = curpos; // New start position - } // endif Indxd + if (MoveIntermediateLines(g, &moved)) + return RC_FX; + Spos = curpos; // New start position } else // Update is directly written back into the file, // with this (fast) method, record size cannot change. @@ -762,28 +895,24 @@ int DOSFAM::WriteBuffer(PGLOBAL g) /*********************************************************************/ /* Prepare the write the updated line. */ /*********************************************************************/ - if (!Indxd) { - strcat(strcpy(To_Buf, Tdbp->To_Line), (Bin) ? CrLf : "\n"); + strcat(strcpy(To_Buf, Tdbp->To_Line), (Bin) ? CrLf : "\n"); - /*******************************************************************/ - /* Now start the writing process. */ - /*******************************************************************/ - if ((fputs(To_Buf, T_Stream)) == EOF) { - sprintf(g->Message, MSG(FPUTS_ERROR), strerror(errno)); + /*********************************************************************/ + /* Now start the writing process. */ + /*********************************************************************/ + if ((fputs(To_Buf, T_Stream)) == EOF) { + sprintf(g->Message, MSG(FPUTS_ERROR), strerror(errno)); + return RC_FX; + } // endif EOF + + if (Tdbp->Mode == MODE_UPDATE && moved) + if (fseek(Stream, curpos, SEEK_SET)) { + sprintf(g->Message, MSG(FSEEK_ERROR), strerror(errno)); return RC_FX; - } // endif EOF + } // endif - if (Tdbp->Mode == MODE_UPDATE && moved) - if (fseek(Stream, curpos, SEEK_SET)) { - sprintf(g->Message, MSG(FSEEK_ERROR), strerror(errno)); - return RC_FX; - } // endif - - if (trace) - htrc("write done\n"); - - } else // Add this updated line to the updated line list - (void)AddListValue(g, TYPE_STRING, Tdbp->To_Line, &To_Upd); + if (trace) + htrc("write done\n"); return RC_OK; } // end of WriteBuffer @@ -840,18 +969,12 @@ int DOSFAM::DeleteRecords(PGLOBAL g, int irc) Spos = Tpos = Fpos; } // endif UseTemp - Indxd = Tdbp->To_Kindex != NULL; } // endif Tpos == Spos /*********************************************************************/ /* Move any intermediate lines. */ /*********************************************************************/ - if (Indxd) { - // Moving will be done later, must be done in sequential order - (void)AddListValue(g, TYPE_INT, &Fpos, &To_Pos); - (void)AddListValue(g, TYPE_INT, &curpos, &To_Sos); - moved = false; - } else if (MoveIntermediateLines(g, &moved)) + if (MoveIntermediateLines(g, &moved)) return RC_FX; if (irc == RC_OK) { @@ -874,9 +997,6 @@ int DOSFAM::DeleteRecords(PGLOBAL g, int irc) /* Last call after EOF has been reached. */ /* The UseTemp case is treated in CloseTableFile. */ /*******************************************************************/ - if (Indxd) - Abort = MakeDeletedFile(g); - if (!UseTemp & !Abort) { /*****************************************************************/ /* Because the chsize functionality is only accessible with a */ @@ -1000,136 +1120,6 @@ bool DOSFAM::MoveIntermediateLines(PGLOBAL g, bool *b) return false; } // end of MoveIntermediate Lines -/***********************************************************************/ -/* MakeUpdatedFile. When updating using indexing, the issue is that */ -/* record are not necessarily updated in sequential order. */ -/* Moving intermediate lines cannot be done while making them because */ -/* this can cause extra wrong records to be included in the new file. */ -/* What we do here is to reorder the updated record and make the new */ -/* updated file from the ordered updated records. */ -/***********************************************************************/ -bool DOSFAM::MakeUpdatedFile(PGLOBAL g) - { - char *crlf = "\n", *mode = UseTemp ? "rb" : "r+b"; - int *ix, i; - bool moved, b = false; - - /*********************************************************************/ - /* Open the temporary file, Spos is at the beginning of file. */ - /*********************************************************************/ - if (!(Stream = PlugReopenFile(g, To_Fb, mode))) { - goto err; - } else if (!(Posar = MakeValueArray(g, To_Pos))) { - strcpy(g->Message, "Position array is null"); - goto err; - } else if (!(Sosar = MakeValueArray(g, To_Sos))) { - strcpy(g->Message, "Start position array is null"); - goto err; - } else if (!(Updar = MakeValueArray(g, To_Upd))) { - strcpy(g->Message, "Updated line array is null"); - goto err; - } else if (!(ix = (int*)Posar->GetSortIndex(g))) { - strcpy(g->Message, "Error getting array sort index"); - goto err; - } // endif's - - Spos = 0; - - for (i = 0; i < Posar->GetNval(); i++) { - Fpos = Posar->GetIntValue(ix[i]); - - if (i || UseTemp) { - // Copy all not updated lines preceding this one - if (MoveIntermediateLines(g, &moved)) - goto err; - - } else - Tpos = Fpos; - - // Now write the updated line. - strcat(strcpy(To_Buf, Updar->GetStringValue(ix[i])), CrLf); - - if ((fputs(To_Buf, T_Stream)) == EOF) { - sprintf(g->Message, MSG(FPUTS_ERROR), strerror(errno)); - goto err; - } // endif EOF - - // New start position - Spos = Sosar->GetIntValue(ix[i]); - } // endfor i - - // Copy eventually remaining lines - fseek(Stream, 0, SEEK_END); - Fpos = ftell(Stream); - b = MoveIntermediateLines(g, &moved) != RC_OK; - - if (!PlugCloseFile(g, To_Fbt) && !PlugCloseFile(g, To_Fb) && !b) - return false; - -err: - if (trace) - htrc("%s\n", g->Message); - - PlugCloseFile(g, To_Fbt); - return true; - } // end of MakeUpdatedFile - -/***********************************************************************/ -/* MakeDeletedFile. When deleting using indexing, the issue is that */ -/* record are not necessarily deleted in sequential order. Moving */ -/* intermediate lines cannot be done while deleing them because */ -/* this can cause extra wrong records to be included in the new file. */ -/* What we do here is to reorder the deleted record and make the new */ -/* deleted file from the ordered deleted records. */ -/***********************************************************************/ -bool DOSFAM::MakeDeletedFile(PGLOBAL g) - { - char *crlf = "\n", *mode = UseTemp ? "rb" : "r+b"; - int *ix, i; - bool moved; - - /*********************************************************************/ - /* Open the temporary file, Spos is at the beginning of file. */ - /*********************************************************************/ - if (!(Posar = MakeValueArray(g, To_Pos))) { - strcpy(g->Message, "Position array is null"); - goto err; - } else if (!(Sosar = MakeValueArray(g, To_Sos))) { - strcpy(g->Message, "Start position array is null"); - goto err; - } else if (!(ix = (int*)Posar->GetSortIndex(g))) { - strcpy(g->Message, "Error getting array sort index"); - goto err; - } // endif's - - Spos = 0; - - for (i = 0; i < Posar->GetNval(); i++) { - Fpos = Posar->GetIntValue(ix[i]); - - if (i || UseTemp) { - // Copy all not updated lines preceding this one - if (MoveIntermediateLines(g, &moved)) - goto err; - - } else - Tpos = Fpos; - - // New start position - Spos = Sosar->GetIntValue(ix[i]); - } // endfor i - - if (!PlugCloseFile(g, To_Fbt) && !PlugCloseFile(g, To_Fb)) - return false; - -err: - if (trace) - htrc("%s\n", g->Message); - - PlugCloseFile(g, To_Fbt); - return true; - } // end of MakeDeletedFile - /***********************************************************************/ /* Delete the old file and rename the new temp file. */ /* If aborting just delete the new temp file. */ @@ -1148,22 +1138,10 @@ int DOSFAM::RenameTempFile(PGLOBAL g) // This loop is necessary because, in case of join, // To_File can have been open several times. for (PFBLOCK fb = PlgGetUser(g)->Openlist; fb; fb = fb->Next) - if (fb == To_Fb || (fb == To_Fbt && !Indxd)) + if (fb == To_Fb || (fb == To_Fbt)) rc = PlugCloseFile(g, fb); if (!Abort) { - // If indexed the temp file must be made - if (Indxd) { - Abort = (Tdbp->Mode == MODE_UPDATE) ? MakeUpdatedFile(g) - : MakeDeletedFile(g); - - if (Abort) { - remove(tempname); - return RC_FX; - } // endif Abort - - } // endif Indxd - PlugSetPath(filename, To_File, Tdbp->GetPath()); strcat(PlugRemoveType(filetemp, filename), ".ttt"); remove(filetemp); // May still be there from previous error @@ -1199,7 +1177,7 @@ void DOSFAM::CloseTableFile(PGLOBAL g, bool abort) Abort = abort; if (UseTemp && T_Stream) { - if (Tdbp->Mode == MODE_UPDATE && !Indxd && !Abort) { + if (Tdbp->Mode == MODE_UPDATE && !Abort) { // Copy eventually remaining lines bool b; diff --git a/storage/connect/filamtxt.h b/storage/connect/filamtxt.h index 83c93ecc6f2..b89d58965f9 100644 --- a/storage/connect/filamtxt.h +++ b/storage/connect/filamtxt.h @@ -69,10 +69,13 @@ class DllExport TXTFAM : public BLOCK { virtual int DeleteRecords(PGLOBAL g, int irc) = 0; virtual void CloseTableFile(PGLOBAL g, bool abort) = 0; virtual void Rewind(void) = 0; + virtual int InitDelete(PGLOBAL g, int fpos, int spos); + bool AddListValue(PGLOBAL g, int type, void *val, PPARM *top); + int StoreValues(PGLOBAL g, bool upd); + int UpdateSortedRows(PGLOBAL g); + int DeleteSortedRows(PGLOBAL g); protected: - bool AddListValue(PGLOBAL g, int type, void *val, PPARM *top); - // Members PTDBDOS Tdbp; // To table class PSZ To_File; // Points to table file name @@ -107,9 +110,11 @@ class DllExport TXTFAM : public BLOCK { int Modif; // Number of modified lines in block int Blksize; // Size of padded blocks int Ending; // Length of line end + int Fpos; // Position of last read record + int Spos; // Start position for update/delete move + int Tpos; // Target Position for delete move bool Padded; // true if fixed size blocks are padded bool Eof; // true if an EOF (0xA) character exists - bool Indxd; // True for indexed UPDATE/DELETE bool Abort; // To abort on error char *CrLf; // End of line character(s) }; // end of class TXTFAM @@ -154,16 +159,12 @@ class DllExport DOSFAM : public TXTFAM { virtual bool OpenTempFile(PGLOBAL g); virtual bool MoveIntermediateLines(PGLOBAL g, bool *b); virtual int RenameTempFile(PGLOBAL g); - virtual bool MakeUpdatedFile(PGLOBAL g); - virtual bool MakeDeletedFile(PGLOBAL g); + virtual int InitDelete(PGLOBAL g, int fpos, int spos); // Members FILE *Stream; // Points to Dos file structure FILE *T_Stream; // Points to temporary file structure PFBLOCK To_Fbt; // Pointer to temp file block - int Fpos; // Position of last read record - int Tpos; // Target Position for delete move - int Spos; // Start position for update/delete move bool UseTemp; // True to use a temporary file in Upd/Del bool Bin; // True to force binary mode }; // end of class DOSFAM diff --git a/storage/connect/filamvct.cpp b/storage/connect/filamvct.cpp index 34f55cf831b..b93adbd13dd 100755 --- a/storage/connect/filamvct.cpp +++ b/storage/connect/filamvct.cpp @@ -93,11 +93,6 @@ typedef struct _vecheader { PVBLK AllocValBlock(PGLOBAL, void *, int, int, int, int, bool check = true, bool blank = true, bool un = false); -/***********************************************************************/ -/* Routine called externally by VCTFAM MakeUpdatedFile function. */ -/***********************************************************************/ -PARRAY MakeValueArray(PGLOBAL g, PPARM pp); - /* -------------------------- Class VCTFAM --------------------------- */ /***********************************************************************/ @@ -670,10 +665,7 @@ int VCTFAM::WriteBuffer(PGLOBAL g) // Mode Update is done in ReadDB, we just initialize it here if (!T_Stream) { if (UseTemp) { - if ((Indxd = Tdbp->GetKindex() != NULL)) { - strcpy(g->Message, "VCT indexed udate using temp file NIY"); - return RC_FX; - } else if (OpenTempFile(g)) + if (OpenTempFile(g)) return RC_FX; // Most of the time, not all table columns are updated. @@ -792,17 +784,12 @@ int VCTFAM::DeleteRecords(PGLOBAL g, int irc) Spos = Tpos = Fpos; } // endif UseTemp - Indxd = Tdbp->GetKindex() != NULL; } // endif Tpos == Spos /*********************************************************************/ /* Move any intermediate lines. */ /*********************************************************************/ - if (Indxd) { - // Moving will be done later, must be done in sequential order - (void)AddListValue(g, TYPE_INT, &Fpos, &To_Pos); - Spos = Fpos; - } else if (MoveIntermediateLines(g, &eof)) + if (MoveIntermediateLines(g, &eof)) return RC_FX; if (irc == RC_OK) { @@ -822,11 +809,6 @@ int VCTFAM::DeleteRecords(PGLOBAL g, int irc) /* Last call after EOF has been reached. */ /* Update the Block and Last values. */ /*******************************************************************/ - if (Indxd && (Abort = MakeDeletedFile(g))) - return RC_FX; - else - Indxd = false; // Not to be redone by RenameTempFile - Block = (Tpos > 0) ? (Tpos + Nrec - 1) / Nrec : 0; Last = (Tpos + Nrec - 1) % Nrec + 1; @@ -1026,63 +1008,6 @@ bool VCTFAM::MoveIntermediateLines(PGLOBAL g, bool *b) return false; } // end of MoveIntermediateLines -/***********************************************************************/ -/* MakeDeletedFile. When deleting using indexing, the issue is that */ -/* record are not necessarily deleted in sequential order. Moving */ -/* intermediate lines cannot be done while deleing them because */ -/* this can cause extra wrong records to be included in the new file. */ -/* What we do here is to reorder the deleted record and make the new */ -/* deleted file from the ordered deleted records. */ -/***********************************************************************/ -bool VCTFAM::MakeDeletedFile(PGLOBAL g) - { -//char *crlf = "\n", *mode = UseTemp ? "rb" : "r+b"; - int *ix, i, n; - bool eof = false; - - /*********************************************************************/ - /* Open the temporary file, Spos is at the beginning of file. */ - /*********************************************************************/ - if (!(Posar = MakeValueArray(g, To_Pos))) { - strcpy(g->Message, "Position array is null"); - goto err; - } else if (!(ix = (int*)Posar->GetSortIndex(g))) { - strcpy(g->Message, "Error getting array sort index"); - goto err; - } // endif's - - n = Posar->GetNval(); - Spos = 0; - - for (i = 0; i < n; i++) { - if (i == n - 1 && !MaxBlk && UseTemp) - eof = true; - - Fpos = Posar->GetIntValue(ix[i]); - - if (i || UseTemp) { - // Copy all not updated lines preceding this one - if (MoveIntermediateLines(g, &eof)) - goto err; - - } else - Tpos = Fpos; - - // New start position - Spos = Fpos + 1; - } // endfor i - - if (!PlugCloseFile(g, To_Fbt)) - return false; - -err: - if (trace) - htrc("%s\n", g->Message); - - PlugCloseFile(g, To_Fbt); - return true; - } // end of MakeDeletedFile - /***********************************************************************/ /* Clean deleted space in a VCT or Vec table file. */ /***********************************************************************/ @@ -1701,13 +1626,7 @@ int VCMFAM::DeleteRecords(PGLOBAL g, int irc) /* not required here, just setting of future Spos and Tpos. */ /*******************************************************************/ Tpos = Spos = Fpos; - Indxd = Tdbp->GetKindex() != NULL; - } // endif Tpos - - if (Indxd) - // Moving will be done later, must be done in sequential order - (void)AddListValue(g, TYPE_INT, &Fpos, &To_Pos); - else + } else (void)MoveIntermediateLines(g); if (irc == RC_OK) { @@ -1716,7 +1635,7 @@ int VCMFAM::DeleteRecords(PGLOBAL g, int irc) if (trace) htrc("after: Tpos=%p Spos=%p\n", Tpos, Spos); - } else if (!(Abort = (Indxd && MakeDeletedFile(g)))) { + } else { /*******************************************************************/ /* Last call after EOF has been reached. */ /*******************************************************************/ @@ -1786,8 +1705,7 @@ int VCMFAM::DeleteRecords(PGLOBAL g, int irc) // Reset Last and Block values in the catalog PlugCloseFile(g, To_Fb); // in case of Header ResetTableSize(g, Block, Last); - } else - return RC_FX; + } // endif irc return RC_OK; // All is correct } // end of DeleteRecords @@ -1841,49 +1759,6 @@ bool VCMFAM::MoveIntermediateLines(PGLOBAL g, bool *b) return false; } // end of MoveIntermediate Lines -/***********************************************************************/ -/* MakeDeletedFile. When deleting using indexing, the issue is that */ -/* record are not necessarily deleted in sequential order. Moving */ -/* intermediate lines cannot be done while deleting them. */ -/* What we do here is to reorder the deleted records and move the */ -/* intermediate files from the ordered deleted record positions. */ -/***********************************************************************/ -bool VCMFAM::MakeDeletedFile(PGLOBAL g) - { - int *ix, i; - - /*********************************************************************/ - /* Make and order the arrays from the saved values. */ - /*********************************************************************/ - if (!(Posar = MakeValueArray(g, To_Pos))) { - strcpy(g->Message, "Position array is null"); - goto err; - } else if (!(ix = (int*)Posar->GetSortIndex(g))) { - strcpy(g->Message, "Error getting array sort index"); - goto err; - } // endif's - - for (i = 0; i < Posar->GetNval(); i++) { - Fpos = Posar->GetIntValue(ix[i]); - - if (!i) { - Tpos = Fpos; - } else - (void)MoveIntermediateLines(g); - - // New start position - Spos = Fpos + 1; - } // endfor i - - return false; - -err: - if (trace) - htrc("%s\n", g->Message); - - return true; - } // end of MakeDeletedFile - /***********************************************************************/ /* Data Base close routine for VMP access method. */ /***********************************************************************/ @@ -2309,10 +2184,7 @@ int VECFAM::WriteBuffer(PGLOBAL g) } else // Mode Update // Writing updates being done in ReadDB we do initialization only. if (InitUpdate) { - if ((Indxd = Tdbp->GetKindex() != NULL)) { - strcpy(g->Message, "VEC indexed udate using temp file NIY"); - return RC_FX; - } else if (OpenTempFile(g)) + if (OpenTempFile(g)) return RC_FX; InitUpdate = false; // Done @@ -2360,17 +2232,12 @@ int VECFAM::DeleteRecords(PGLOBAL g, int irc) /*****************************************************************/ Spos = Tpos = Fpos; - Indxd = Tdbp->GetKindex() != NULL; } // endif Tpos == Spos /*********************************************************************/ /* Move any intermediate lines. */ /*********************************************************************/ - if (Indxd) { - // Moving will be done later, must be done in sequential order - (void)AddListValue(g, TYPE_INT, &Fpos, &To_Pos); - Spos = Fpos; - } else if (MoveIntermediateLines(g)) + if (MoveIntermediateLines(g)) return RC_FX; if (irc == RC_OK) { @@ -2386,11 +2253,6 @@ int VECFAM::DeleteRecords(PGLOBAL g, int irc) /*******************************************************************/ /* Last call after EOF has been reached. */ /*******************************************************************/ - if (Indxd && (Abort = MakeDeletedFile(g))) - return RC_FX; -// else -// Indxd = false; // Not to be redone by RenameTempFile - if (!UseTemp) { /*****************************************************************/ /* Because the chsize functionality is only accessible with a */ @@ -2560,56 +2422,6 @@ bool VECFAM::MoveIntermediateLines(PGLOBAL g, bool *bn) return false; } // end of MoveIntermediate Lines -/***********************************************************************/ -/* MakeDeletedFile. When deleting using indexing, the issue is that */ -/* record are not necessarily deleted in sequential order. Moving */ -/* intermediate lines cannot be done while deleing them because */ -/* this can cause extra wrong records to be included in the new file. */ -/* What we do here is to reorder the deleted record and make the new */ -/* deleted file from the ordered deleted records. */ -/***********************************************************************/ -bool VECFAM::MakeDeletedFile(PGLOBAL g) - { - int *ix, i, n; - - /*********************************************************************/ - /* Open the temporary file, Spos is at the beginning of file. */ - /*********************************************************************/ - if (!(Posar = MakeValueArray(g, To_Pos))) { - strcpy(g->Message, "Position array is null"); - goto err; - } else if (!(ix = (int*)Posar->GetSortIndex(g))) { - strcpy(g->Message, "Error getting array sort index"); - goto err; - } // endif's - - n = Posar->GetNval(); - Spos = 0; - - for (i = 0; i < n; i++) { - Fpos = Posar->GetIntValue(ix[i]); - - if (i || UseTemp) { - // Copy all not updated lines preceding this one - if (MoveIntermediateLines(g)) - goto err; - - } else - Tpos = Fpos; - - // New start position - Spos = Fpos + 1; - } // endfor i - - return false; - -err: - if (trace) - htrc("%s\n", g->Message); - - return true; - } // end of MakeDeletedFile - /***********************************************************************/ /* Delete the old files and rename the new temporary files. */ /***********************************************************************/ @@ -3109,13 +2921,7 @@ int VMPFAM::DeleteRecords(PGLOBAL g, int irc) /* not required here, just setting of future Spos and Tpos. */ /*******************************************************************/ Tpos = Fpos; // Spos is set below - Indxd = Tdbp->GetKindex() != NULL; - } // endif Tpos - - if (Indxd) - // Moving will be done later, must be done in sequential order - (void)AddListValue(g, TYPE_INT, &Fpos, &To_Pos); - else if ((n = Fpos - Spos) > 0) { + } else if ((n = Fpos - Spos) > 0) { /*******************************************************************/ /* Non consecutive line to delete. Move intermediate lines. */ /*******************************************************************/ @@ -3137,7 +2943,7 @@ int VMPFAM::DeleteRecords(PGLOBAL g, int irc) if (trace) htrc("after: Tpos=%p Spos=%p\n", Tpos, Spos); - } else if (!(Abort = (Indxd && MakeDeletedFile(g)))) { + } else { /*******************************************************************/ /* Last call after EOF has been reached. */ /* We must firstly Unmap the view and use the saved file handle */ @@ -3198,55 +3004,6 @@ int VMPFAM::DeleteRecords(PGLOBAL g, int irc) return RC_OK; // All is correct } // end of DeleteRecords -/***********************************************************************/ -/* MakeDeletedFile. When deleting using indexing, the issue is that */ -/* record are not necessarily deleted in sequential order. Moving */ -/* intermediate lines cannot be done while deleting them. */ -/* What we do here is to reorder the deleted records and move the */ -/* intermediate files from the ordered deleted record positions. */ -/***********************************************************************/ -bool VMPFAM::MakeDeletedFile(PGLOBAL g) - { - int *ix, i, j, m, n; - - /*********************************************************************/ - /* Make and order the arrays from the saved values. */ - /*********************************************************************/ - if (!(Posar = MakeValueArray(g, To_Pos))) { - strcpy(g->Message, "Position array is null"); - goto err; - } else if (!(ix = (int*)Posar->GetSortIndex(g))) { - strcpy(g->Message, "Error getting array sort index"); - goto err; - } // endif's - - for (i = 0; i < Posar->GetNval(); i++) { - Fpos = Posar->GetIntValue(ix[i]); - - if (!i) { - Tpos = Fpos; - } else if ((n = Fpos - Spos) > 0) { - for (j = 0; j < Ncol; j++) { - m = Clens[j]; - memmove(Memcol[j] + Tpos * m, Memcol[j] + Spos * m, m * n); - } // endif j - - Tpos += n; - } // endif n - - // New start position - Spos = Fpos + 1; - } // endfor i - - return false; - -err: - if (trace) - htrc("%s\n", g->Message); - - return true; - } // end of MakeDeletedFile - /***********************************************************************/ /* Data Base close routine for VMP access method. */ /***********************************************************************/ @@ -3969,10 +3726,7 @@ int BGVFAM::WriteBuffer(PGLOBAL g) // Mode Update is done in ReadDB, we just initialize it here if (Tfile == INVALID_HANDLE_VALUE) { if (UseTemp) { - if ((Indxd = Tdbp->GetKindex() != NULL)) { - strcpy(g->Message, "VEC indexed udate using temp file NIY"); - return RC_FX; - } else if (OpenTempFile(g)) + if (OpenTempFile(g)) return RC_FX; // Most of the time, not all table columns are updated. @@ -4099,17 +3853,12 @@ int BGVFAM::DeleteRecords(PGLOBAL g, int irc) Spos = Tpos = Fpos; } // endif UseTemp - Indxd = Tdbp->GetKindex() != NULL; } // endif Tpos == Spos /*********************************************************************/ /* Move any intermediate lines. */ /*********************************************************************/ - if (Indxd) { - // Moving will be done later, must be done in sequential order - (void)AddListValue(g, TYPE_INT, &Fpos, &To_Pos); - Spos = Fpos; - } else if (MoveIntermediateLines(g, &eof)) + if (MoveIntermediateLines(g, &eof)) return RC_FX; if (irc == RC_OK) { @@ -4125,11 +3874,6 @@ int BGVFAM::DeleteRecords(PGLOBAL g, int irc) /*******************************************************************/ /* Last call after EOF has been reached. */ /*******************************************************************/ - if (Indxd && (Abort = MakeDeletedFile(g))) - return RC_FX; - else - Indxd = false; // Not to be redone by RenameTempFile - Block = (Tpos > 0) ? (Tpos + Nrec - 1) / Nrec : 0; Last = (Tpos + Nrec - 1) % Nrec + 1; diff --git a/storage/connect/filamvct.h b/storage/connect/filamvct.h index 8acb62b14dc..be66232acfb 100644 --- a/storage/connect/filamvct.h +++ b/storage/connect/filamvct.h @@ -66,7 +66,6 @@ class DllExport VCTFAM : public FIXFAM { virtual bool MoveLines(PGLOBAL g) {return false;} virtual bool MoveIntermediateLines(PGLOBAL g, bool *b = NULL); virtual bool CleanUnusedSpace(PGLOBAL g); - virtual bool MakeDeletedFile(PGLOBAL g); virtual int GetBlockInfo(PGLOBAL g); virtual bool SetBlockInfo(PGLOBAL g); bool ResetTableSize(PGLOBAL g, int block, int last); @@ -116,7 +115,6 @@ class DllExport VCMFAM : public VCTFAM { protected: // Specific functions virtual bool MoveIntermediateLines(PGLOBAL g, bool *b = NULL); - virtual bool MakeDeletedFile(PGLOBAL g); virtual bool ReadBlock(PGLOBAL g, PVCTCOL colp); virtual bool WriteBlock(PGLOBAL g, PVCTCOL colp); @@ -162,7 +160,6 @@ class DllExport VECFAM : public VCTFAM { virtual bool MoveLines(PGLOBAL g); virtual bool MoveIntermediateLines(PGLOBAL g, bool *b = NULL); virtual int RenameTempFile(PGLOBAL g); - virtual bool MakeDeletedFile(PGLOBAL g); bool OpenColumnFile(PGLOBAL g, char *opmode, int i); // Members @@ -199,7 +196,6 @@ class DllExport VMPFAM : public VCMFAM { virtual void CloseTableFile(PGLOBAL g, bool abort); protected: - virtual bool MakeDeletedFile(PGLOBAL g); bool MapColumnFile(PGLOBAL g, MODE mode, int i); // Members diff --git a/storage/connect/tabdos.cpp b/storage/connect/tabdos.cpp index 4a04b0c2db4..861a9da106d 100644 --- a/storage/connect/tabdos.cpp +++ b/storage/connect/tabdos.cpp @@ -432,6 +432,7 @@ TDBDOS::TDBDOS(PDOSDEF tdp, PTXF txfp) : TDBASE(tdp) //Xeval = 0; Beval = 0; Abort = false; + Indxd = false; } // end of TDBDOS standard constructor TDBDOS::TDBDOS(PGLOBAL g, PTDBDOS tdbp) : TDBASE(tdbp) @@ -446,6 +447,8 @@ TDBDOS::TDBDOS(PGLOBAL g, PTDBDOS tdbp) : TDBASE(tdbp) SavFil = tdbp->SavFil; //Xeval = tdbp->Xeval; Beval = tdbp->Beval; + Abort = tdbp->Abort; + Indxd = tdbp->Indxd; } // end of TDBDOS copy constructor // Method @@ -1807,7 +1810,11 @@ bool TDBDOS::InitialyzeIndex(PGLOBAL g, PIXDEF xdp) To_BlkFil = NULL; } // endif AmType - To_Kindex= kxp; + if (!(To_Kindex= kxp)->IsSorted() && + ((Mode == MODE_UPDATE && IsUsingTemp(g)) || + (Mode == MODE_DELETE && Txfp->GetAmType() != TYPE_AM_DBF))) + Indxd = true; + } // endif brc } else @@ -2153,13 +2160,10 @@ int TDBDOS::ReadDB(PGLOBAL g) } // end of ReadDB /***********************************************************************/ -/* WriteDB: Data Base write routine for DOS access method. */ +/* PrepareWriting: Prepare the line to write. */ /***********************************************************************/ -int TDBDOS::WriteDB(PGLOBAL g) +bool TDBDOS::PrepareWriting(PGLOBAL g) { - if (trace > 1) - htrc("DOS WriteDB: R%d Mode=%d \n", Tdb_No, Mode); - if (!Ftype && (Mode == MODE_INSERT || Txfp->GetUseTemp())) { char *p; @@ -2174,6 +2178,20 @@ int TDBDOS::WriteDB(PGLOBAL g) *(++p) = '\0'; } // endif Mode + return false; + } // end of WriteDB + +/***********************************************************************/ +/* WriteDB: Data Base write routine for DOS access method. */ +/***********************************************************************/ +int TDBDOS::WriteDB(PGLOBAL g) + { + if (trace > 1) + htrc("DOS WriteDB: R%d Mode=%d \n", Tdb_No, Mode); + + // Make the line to write + (void)PrepareWriting(g); + if (trace > 1) htrc("Write: line is='%s'\n", To_Line); diff --git a/storage/connect/tabdos.h b/storage/connect/tabdos.h index 7f2c4f5e7ee..e49f3188795 100644 --- a/storage/connect/tabdos.h +++ b/storage/connect/tabdos.h @@ -136,6 +136,7 @@ class DllExport TDBDOS : public TDBASE { virtual PTDB CopyOne(PTABS t); virtual void ResetDB(void) {Txfp->Reset();} virtual bool IsUsingTemp(PGLOBAL g); + virtual bool IsIndexed(void) {return Indxd;} virtual void ResetSize(void) {MaxSize = Cardinal = -1;} virtual int ResetTableOpt(PGLOBAL g, bool dop, bool dox); virtual int MakeBlockValues(PGLOBAL g); @@ -176,6 +177,7 @@ class DllExport TDBDOS : public TDBASE { bool GetDistinctColumnValues(PGLOBAL g, int nrec); protected: + virtual bool PrepareWriting(PGLOBAL g); PBF CheckBlockFilari(PGLOBAL g, PXOB *arg, int op, bool *cnv); // Members @@ -185,6 +187,7 @@ class DllExport TDBDOS : public TDBASE { PFIL SavFil; // Saved hidden filter char *To_Line; // Points to current processed line bool Abort; // TRUE when aborting UPDATE/DELETE + bool Indxd; // TRUE for indexed UPDATE/DELETE int Lrecl; // Logical Record Length int AvgLen; // Logical Record Average Length //int Xeval; // BlockTest return value diff --git a/storage/connect/tabfix.h b/storage/connect/tabfix.h index 5feb3589928..00439fea0e8 100644 --- a/storage/connect/tabfix.h +++ b/storage/connect/tabfix.h @@ -51,6 +51,8 @@ class DllExport TDBFIX : public TDBDOS { virtual int WriteDB(PGLOBAL g); protected: + virtual bool PrepareWriting(PGLOBAL g) {return false;} + // Members are inherited from TDBDOS }; // end of class TDBFIX diff --git a/storage/connect/tabfmt.cpp b/storage/connect/tabfmt.cpp index c015b6adad3..b2d60d87f3b 100644 --- a/storage/connect/tabfmt.cpp +++ b/storage/connect/tabfmt.cpp @@ -899,9 +899,9 @@ int TDBCSV::ReadBuffer(PGLOBAL g) } // end of ReadBuffer /***********************************************************************/ -/* Data Base write routine CSV file access method. */ +/* Prepare the line to write. */ /***********************************************************************/ -int TDBCSV::WriteDB(PGLOBAL g) +bool TDBCSV::PrepareWriting(PGLOBAL g) { char sep[2], qot[2]; int i, nlen, oldlen = strlen(To_Line); @@ -912,7 +912,7 @@ int TDBCSV::WriteDB(PGLOBAL g) // Before writing the line we must check its length if ((nlen = CheckWrite(g)) < 0) - return RC_FX; + return true; // Before writing the line we must make it sep[0] = Sep; @@ -975,6 +975,18 @@ int TDBCSV::WriteDB(PGLOBAL g) if (trace > 1) htrc("Write: line is=%s", To_Line); + return false; + } // end of PrepareWriting + +/***********************************************************************/ +/* Data Base write routine CSV file access method. */ +/***********************************************************************/ +int TDBCSV::WriteDB(PGLOBAL g) + { + // Before writing the line we must check and prepare it + if (PrepareWriting(g)) + return RC_FX; + /*********************************************************************/ /* Now start the writing process. */ /*********************************************************************/ diff --git a/storage/connect/tabfmt.h b/storage/connect/tabfmt.h index aa14b4481f0..939c1677f93 100644 --- a/storage/connect/tabfmt.h +++ b/storage/connect/tabfmt.h @@ -1,188 +1,193 @@ -/*************** TabFmt H Declares Source Code File (.H) ***************/ -/* Name: TABFMT.H Version 2.4 */ -/* */ -/* (C) Copyright to the author Olivier BERTRAND 2001-2014 */ -/* */ -/* This file contains the CSV and FMT classes declares. */ -/***********************************************************************/ -#include "xtable.h" // Base class declares -#include "tabdos.h" - -typedef class TDBFMT *PTDBFMT; - -/***********************************************************************/ -/* Functions used externally. */ -/***********************************************************************/ -PQRYRES CSVColumns(PGLOBAL g, const char *fn, char sep, char q, - int hdr, int mxr, bool info); - -/***********************************************************************/ -/* CSV table. */ -/***********************************************************************/ -class DllExport CSVDEF : public DOSDEF { /* Logical table description */ - friend class TDBCSV; - friend class TDBCCL; - public: - // Constructor - CSVDEF(void); - - // Implementation - virtual const char *GetType(void) {return "CSV";} - char GetSep(void) {return Sep;} - char GetQot(void) {return Qot;} - - // Methods - virtual bool DefineAM(PGLOBAL g, LPCSTR am, int poff); - virtual PTDB GetTable(PGLOBAL g, MODE mode); - - protected: - // Members - bool Fmtd; /* true for formatted files */ -//bool Accept; /* true if wrong lines are accepted */ - bool Header; /* true if first line contains headers */ -//int Maxerr; /* Maximum number of bad records */ - int Quoted; /* Quoting level for quoted fields */ - char Sep; /* Separator for standard CSV files */ - char Qot; /* Character for quoted strings */ - }; // end of CSVDEF - -/***********************************************************************/ -/* This is the DOS/UNIX Access Method class declaration for files */ -/* that are CSV files with columns separated by the Sep character. */ -/***********************************************************************/ -class TDBCSV : public TDBDOS { - friend class CSVCOL; - public: - // Constructor - TDBCSV(PCSVDEF tdp, PTXF txfp); - TDBCSV(PGLOBAL g, PTDBCSV tdbp); - - // Implementation - virtual AMT GetAmType(void) {return TYPE_AM_CSV;} - virtual PTDB Duplicate(PGLOBAL g) - {return (PTDB)new(g) TDBCSV(g, this);} - - // Methods - virtual PTDB CopyOne(PTABS t); -//virtual bool IsUsingTemp(PGLOBAL g); - virtual int GetBadLines(void) {return (int)Nerr;} - - // Database routines - virtual PCOL MakeCol(PGLOBAL g, PCOLDEF cdp, PCOL cprec, int n); - virtual bool OpenDB(PGLOBAL g); - virtual int WriteDB(PGLOBAL g); - virtual int CheckWrite(PGLOBAL g); - virtual int ReadBuffer(PGLOBAL g); // Physical file read - - // Specific routines - virtual int EstimatedLength(PGLOBAL g); - virtual bool SkipHeader(PGLOBAL g); - virtual bool CheckErr(void); - - protected: - // Members - PSZ *Field; // Field to write to current line - int *Offset; // Column offsets for current record - int *Fldlen; // Column field length for current record - bool *Fldtyp; // true for numeric fields - int Fields; // Number of fields to handle - int Nerr; // Number of bad records - int Maxerr; // Maximum number of bad records - int Quoted; // Quoting level for quoted fields - bool Accept; // true if bad lines are accepted - bool Header; // true if first line contains column headers - char Sep; // Separator - char Qot; // Quoting character - }; // end of class TDBCSV - -/***********************************************************************/ -/* Class CSVCOL: CSV access method column descriptor. */ -/* This A.M. is used for Comma Separated V(?) files. */ -/***********************************************************************/ -class CSVCOL : public DOSCOL { - friend class TDBCSV; - friend class TDBFMT; - public: - // Constructors - CSVCOL(PGLOBAL g, PCOLDEF cdp, PTDB tdbp, PCOL cprec, int i); - CSVCOL(CSVCOL *colp, PTDB tdbp); // Constructor used in copy process - - // Implementation - virtual int GetAmType() {return TYPE_AM_CSV;} - - // Methods - virtual bool VarSize(void); - virtual void ReadColumn(PGLOBAL g); - virtual void WriteColumn(PGLOBAL g); - - protected: - // Default constructor not to be used - CSVCOL(void) {} - - // Members - int Fldnum; // Field ordinal number (0 based) - }; // end of class CSVCOL - -/***********************************************************************/ -/* This is the DOS/UNIX Access Method class declaration for files */ -/* whose record format is described by a Format keyword. */ -/***********************************************************************/ -class TDBFMT : public TDBCSV { - friend class CSVCOL; -//friend class FMTCOL; - public: - // Standard constructor - TDBFMT(PCSVDEF tdp, PTXF txfp) : TDBCSV(tdp, txfp) - {FldFormat = NULL; To_Fld = NULL; FmtTest = NULL; Linenum = 0;} - - // Copy constructor - TDBFMT(PGLOBAL g, PTDBFMT tdbp); - - // Implementation - virtual AMT GetAmType(void) {return TYPE_AM_FMT;} - virtual PTDB Duplicate(PGLOBAL g) - {return (PTDB)new(g) TDBFMT(g, this);} - - // Methods - virtual PTDB CopyOne(PTABS t); - - // Database routines - virtual PCOL MakeCol(PGLOBAL g, PCOLDEF cdp, PCOL cprec, int n); -//virtual int GetMaxSize(PGLOBAL g); - virtual bool OpenDB(PGLOBAL g); - virtual int WriteDB(PGLOBAL g); -//virtual int CheckWrite(PGLOBAL g); - virtual int ReadBuffer(PGLOBAL g); // Physical file read - - // Specific routines - virtual int EstimatedLength(PGLOBAL g); - - protected: - // Members - PSZ *FldFormat; // Field read format - void *To_Fld; // To field test buffer - int *FmtTest; // Test on ending by %n or %m - int Linenum; // Last read line - }; // end of class TDBFMT - -/***********************************************************************/ -/* This is the class declaration for the CSV catalog table. */ -/***********************************************************************/ -class TDBCCL : public TDBCAT { - public: - // Constructor - TDBCCL(PCSVDEF tdp); - - protected: - // Specific routines - virtual PQRYRES GetResult(PGLOBAL g); - - // Members - char *Fn; // The CSV file (path) name - bool Hdr; // true if first line contains headers - int Mxr; // Maximum number of bad records - int Qtd; // Quoting level for quoted fields - char Sep; // Separator for standard CSV files - }; // end of class TDBCCL - -/* ------------------------- End of TabFmt.H ------------------------- */ +/*************** TabFmt H Declares Source Code File (.H) ***************/ +/* Name: TABFMT.H Version 2.4 */ +/* */ +/* (C) Copyright to the author Olivier BERTRAND 2001-2014 */ +/* */ +/* This file contains the CSV and FMT classes declares. */ +/***********************************************************************/ +#include "xtable.h" // Base class declares +#include "tabdos.h" + +typedef class TDBFMT *PTDBFMT; + +/***********************************************************************/ +/* Functions used externally. */ +/***********************************************************************/ +PQRYRES CSVColumns(PGLOBAL g, const char *fn, char sep, char q, + int hdr, int mxr, bool info); + +/***********************************************************************/ +/* CSV table. */ +/***********************************************************************/ +class DllExport CSVDEF : public DOSDEF { /* Logical table description */ + friend class TDBCSV; + friend class TDBCCL; + public: + // Constructor + CSVDEF(void); + + // Implementation + virtual const char *GetType(void) {return "CSV";} + char GetSep(void) {return Sep;} + char GetQot(void) {return Qot;} + + // Methods + virtual bool DefineAM(PGLOBAL g, LPCSTR am, int poff); + virtual PTDB GetTable(PGLOBAL g, MODE mode); + + protected: + // Members + bool Fmtd; /* true for formatted files */ +//bool Accept; /* true if wrong lines are accepted */ + bool Header; /* true if first line contains headers */ +//int Maxerr; /* Maximum number of bad records */ + int Quoted; /* Quoting level for quoted fields */ + char Sep; /* Separator for standard CSV files */ + char Qot; /* Character for quoted strings */ + }; // end of CSVDEF + +/***********************************************************************/ +/* This is the DOS/UNIX Access Method class declaration for files */ +/* that are CSV files with columns separated by the Sep character. */ +/***********************************************************************/ +class TDBCSV : public TDBDOS { + friend class CSVCOL; + public: + // Constructor + TDBCSV(PCSVDEF tdp, PTXF txfp); + TDBCSV(PGLOBAL g, PTDBCSV tdbp); + + // Implementation + virtual AMT GetAmType(void) {return TYPE_AM_CSV;} + virtual PTDB Duplicate(PGLOBAL g) + {return (PTDB)new(g) TDBCSV(g, this);} + + // Methods + virtual PTDB CopyOne(PTABS t); +//virtual bool IsUsingTemp(PGLOBAL g); + virtual int GetBadLines(void) {return (int)Nerr;} + + // Database routines + virtual PCOL MakeCol(PGLOBAL g, PCOLDEF cdp, PCOL cprec, int n); + virtual bool OpenDB(PGLOBAL g); + virtual int WriteDB(PGLOBAL g); + virtual int CheckWrite(PGLOBAL g); + virtual int ReadBuffer(PGLOBAL g); // Physical file read + + // Specific routines + virtual int EstimatedLength(PGLOBAL g); + virtual bool SkipHeader(PGLOBAL g); + virtual bool CheckErr(void); + + protected: + virtual bool PrepareWriting(PGLOBAL g); + + // Members + PSZ *Field; // Field to write to current line + int *Offset; // Column offsets for current record + int *Fldlen; // Column field length for current record + bool *Fldtyp; // true for numeric fields + int Fields; // Number of fields to handle + int Nerr; // Number of bad records + int Maxerr; // Maximum number of bad records + int Quoted; // Quoting level for quoted fields + bool Accept; // true if bad lines are accepted + bool Header; // true if first line contains column headers + char Sep; // Separator + char Qot; // Quoting character + }; // end of class TDBCSV + +/***********************************************************************/ +/* Class CSVCOL: CSV access method column descriptor. */ +/* This A.M. is used for Comma Separated V(?) files. */ +/***********************************************************************/ +class CSVCOL : public DOSCOL { + friend class TDBCSV; + friend class TDBFMT; + public: + // Constructors + CSVCOL(PGLOBAL g, PCOLDEF cdp, PTDB tdbp, PCOL cprec, int i); + CSVCOL(CSVCOL *colp, PTDB tdbp); // Constructor used in copy process + + // Implementation + virtual int GetAmType() {return TYPE_AM_CSV;} + + // Methods + virtual bool VarSize(void); + virtual void ReadColumn(PGLOBAL g); + virtual void WriteColumn(PGLOBAL g); + + protected: + // Default constructor not to be used + CSVCOL(void) {} + + // Members + int Fldnum; // Field ordinal number (0 based) + }; // end of class CSVCOL + +/***********************************************************************/ +/* This is the DOS/UNIX Access Method class declaration for files */ +/* whose record format is described by a Format keyword. */ +/***********************************************************************/ +class TDBFMT : public TDBCSV { + friend class CSVCOL; +//friend class FMTCOL; + public: + // Standard constructor + TDBFMT(PCSVDEF tdp, PTXF txfp) : TDBCSV(tdp, txfp) + {FldFormat = NULL; To_Fld = NULL; FmtTest = NULL; Linenum = 0;} + + // Copy constructor + TDBFMT(PGLOBAL g, PTDBFMT tdbp); + + // Implementation + virtual AMT GetAmType(void) {return TYPE_AM_FMT;} + virtual PTDB Duplicate(PGLOBAL g) + {return (PTDB)new(g) TDBFMT(g, this);} + + // Methods + virtual PTDB CopyOne(PTABS t); + + // Database routines + virtual PCOL MakeCol(PGLOBAL g, PCOLDEF cdp, PCOL cprec, int n); +//virtual int GetMaxSize(PGLOBAL g); + virtual bool OpenDB(PGLOBAL g); + virtual int WriteDB(PGLOBAL g); +//virtual int CheckWrite(PGLOBAL g); + virtual int ReadBuffer(PGLOBAL g); // Physical file read + + // Specific routines + virtual int EstimatedLength(PGLOBAL g); + + protected: + virtual bool PrepareWriting(PGLOBAL g) + {strcpy(g->Message, "FMT is read only"); return true;} + + // Members + PSZ *FldFormat; // Field read format + void *To_Fld; // To field test buffer + int *FmtTest; // Test on ending by %n or %m + int Linenum; // Last read line + }; // end of class TDBFMT + +/***********************************************************************/ +/* This is the class declaration for the CSV catalog table. */ +/***********************************************************************/ +class TDBCCL : public TDBCAT { + public: + // Constructor + TDBCCL(PCSVDEF tdp); + + protected: + // Specific routines + virtual PQRYRES GetResult(PGLOBAL g); + + // Members + char *Fn; // The CSV file (path) name + bool Hdr; // true if first line contains headers + int Mxr; // Maximum number of bad records + int Qtd; // Quoting level for quoted fields + char Sep; // Separator for standard CSV files + }; // end of class TDBCCL + +/* ------------------------- End of TabFmt.H ------------------------- */ diff --git a/storage/connect/xindex.cpp b/storage/connect/xindex.cpp index 12781b2ac05..7b339be9ed2 100755 --- a/storage/connect/xindex.cpp +++ b/storage/connect/xindex.cpp @@ -50,7 +50,7 @@ /***********************************************************************/ /* Macro or external routine definition */ /***********************************************************************/ -#define NZ 7 +#define NZ 8 #define NW 5 #define MAX_INDX 10 #ifndef INVALID_SET_FILE_POINTER @@ -869,17 +869,18 @@ bool XINDEX::SaveIndex(PGLOBAL g, PIXDEF sxp) /*********************************************************************/ /* Write the index values on the index file. */ /*********************************************************************/ - n[0] = ID; // To check validity + n[0] = ID + MAX_INDX; // To check validity n[1] = Nk; // The number of indexed columns n[2] = nof; // The offset array size or 0 n[3] = Num_K; // The index size n[4] = Incr; // Increment of record positions n[5] = Nblk; n[6] = Sblk; + n[7] = Srtd ? 1 : 0; // Values are sorted in the file if (trace) { htrc("Saving index %s\n", Xdp->GetName()); - htrc("ID=%d Nk=%d nof=%d Num_K=%d Incr=%d Nblk=%d Sblk=%d\n", - ID, Nk, nof, Num_K, Incr, Nblk, Sblk); + htrc("ID=%d Nk=%d nof=%d Num_K=%d Incr=%d Nblk=%d Sblk=%d Srtd=%d\n", + ID, Nk, nof, Num_K, Incr, Nblk, Sblk, Srtd); } // endif trace size = X->Write(g, n, NZ, sizeof(int), rc); @@ -1019,12 +1020,22 @@ bool XINDEX::Init(PGLOBAL g) goto err; // No saved values // Now start the reading process. - if (X->Read(g, nv, NZ, sizeof(int))) + if (X->Read(g, nv, NZ - 1, sizeof(int))) goto err; + if (nv[0] >= MAX_INDX) { + // New index format + if (X->Read(g, nv + 7, 1, sizeof(int))) + goto err; + + Srtd = nv[7] != 0; + nv[0] -= MAX_INDX; + } else + Srtd = false; + if (trace) - htrc("nv=%d %d %d %d %d %d %d\n", - nv[0], nv[1], nv[2], nv[3], nv[4], nv[5], nv[6]); + htrc("nv=%d %d %d %d %d %d %d (%d)\n", + nv[0], nv[1], nv[2], nv[3], nv[4], nv[5], nv[6], Srtd); // The test on ID was suppressed because MariaDB can change an index ID // when other indexes are added or deleted @@ -1271,11 +1282,20 @@ bool XINDEX::MapInit(PGLOBAL g) // Now start the mapping process. nv = (int*)mbase; - mbase += NZ * sizeof(int); + + if (nv[0] >= MAX_INDX) { + // New index format + Srtd = nv[7] != 0; + nv[0] -= MAX_INDX; + mbase += NZ * sizeof(int); + } else { + Srtd = false; + mbase += (NZ - 1) * sizeof(int); + } // endif nv if (trace) - htrc("nv=%d %d %d %d %d %d %d\n", - nv[0], nv[1], nv[2], nv[3], nv[4], nv[5], nv[6]); + htrc("nv=%d %d %d %d %d %d %d %d\n", + nv[0], nv[1], nv[2], nv[3], nv[4], nv[5], nv[6], Srtd); // The test on ID was suppressed because MariaDB can change an index ID // when other indexes are added or deleted diff --git a/storage/connect/xindex.h b/storage/connect/xindex.h index 4988a12326a..c702baeec83 100644 --- a/storage/connect/xindex.h +++ b/storage/connect/xindex.h @@ -192,6 +192,7 @@ class DllExport XXBASE : public CSORT, public BLOCK { void SetNth(int n) {Nth = n;} int *GetPof(void) {return Pof;} int *GetPex(void) {return Pex;} + bool IsSorted(void) {return Srtd;} void FreeIndex(void) {PlgDBfree(Index);} // Methods diff --git a/storage/connect/xtable.h b/storage/connect/xtable.h index 2d95acdb6d4..628ab96135d 100644 --- a/storage/connect/xtable.h +++ b/storage/connect/xtable.h @@ -147,6 +147,7 @@ class DllExport TDBASE : public TDB { inline PKXBASE GetKindex(void) {return To_Kindex;} inline PCOL GetSetCols(void) {return To_SetCols;} inline void SetSetCols(PCOL colp) {To_SetCols = colp;} + inline void SetKey_Col(PCOL *cpp) {To_Key_Col = cpp;} inline void SetXdp(PIXDEF xdp) {To_Xdp = xdp;} inline void SetKindex(PKXBASE kxp) {To_Kindex = kxp;} @@ -156,6 +157,7 @@ class DllExport TDBASE : public TDB { // Methods virtual bool IsUsingTemp(PGLOBAL g) {return false;} + virtual bool IsIndexed(void) {return false;} virtual PCATLG GetCat(void); virtual PSZ GetPath(void); virtual void PrintAM(FILE *f, char *m); @@ -190,6 +192,9 @@ class DllExport TDBASE : public TDB { {assert(false); return true;} protected: + virtual bool PrepareWriting(PGLOBAL g) {strcpy(g->Message, + "This function should not be called for this table"); return true;} + // Members PTABDEF To_Def; // Points to catalog description block PXOB *To_Link; // Points to column of previous relations