diff options
Diffstat (limited to '')
| -rw-r--r-- | storage/connect/tabdos.cpp | 2926 |
1 files changed, 2926 insertions, 0 deletions
diff --git a/storage/connect/tabdos.cpp b/storage/connect/tabdos.cpp new file mode 100644 index 00000000..0fdc182f --- /dev/null +++ b/storage/connect/tabdos.cpp @@ -0,0 +1,2926 @@ +/************* TabDos C++ Program Source Code File (.CPP) **************/ +/* PROGRAM NAME: TABDOS */ +/* ------------- */ +/* Version 4.9.5 */ +/* */ +/* COPYRIGHT: */ +/* ---------- */ +/* (C) Copyright to the author Olivier BERTRAND 1998-2020 */ +/* */ +/* WHAT THIS PROGRAM DOES: */ +/* ----------------------- */ +/* This program are the DOS tables classes. */ +/* */ +/***********************************************************************/ + +/***********************************************************************/ +/* Include relevant sections of the System header files. */ +/***********************************************************************/ +#include "my_global.h" +#if defined(_WIN32) +#include <io.h> +#include <sys\timeb.h> // For testing only +#include <fcntl.h> +#include <errno.h> +#if defined(__BORLANDC__) +#define __MFC_COMPAT__ // To define min/max as macro +#endif // __BORLANDC__ +//#include <windows.h> +#else // !_WIN32 +#if defined(UNIX) +#include <errno.h> +#include <unistd.h> +#else // !UNIX +#include <io.h> +#endif // !UNIX +#include <fcntl.h> +#endif // !_WIN32 + +/***********************************************************************/ +/* Include application header files: */ +/* global.h is header containing all global declarations. */ +/* plgdbsem.h is header containing the DB application declarations. */ +/* filamtxt.h is header containing the file AM classes declarations. */ +/***********************************************************************/ +#include "global.h" +#include "osutil.h" +#include "plgdbsem.h" +//#include "catalog.h" +#include "mycat.h" +#include "xindex.h" +#include "filamap.h" +#include "filamfix.h" +#include "filamdbf.h" +#if defined(GZ_SUPPORT) +#include "filamgz.h" +#endif // GZ_SUPPORT +#if defined(ZIP_SUPPORT) +#include "filamzip.h" +#endif // ZIP_SUPPORT +#include "tabdos.h" +#include "tabfix.h" +#include "tabmul.h" +#include "array.h" +#include "blkfil.h" +#include "m_string.h" + +/***********************************************************************/ +/* DB static variables. */ +/***********************************************************************/ +int num_read, num_there, num_eq[2]; // Statistics + +/***********************************************************************/ +/* Size of optimize file header. */ +/***********************************************************************/ +#define NZ 4 + +/***********************************************************************/ +/* External function. */ +/***********************************************************************/ +bool ExactInfo(void); +USETEMP UseTemp(void); + +/***********************************************************************/ +/* Min and Max blocks contains zero ended fields (blank = false). */ +/* No conversion of block values (check = true). */ +/***********************************************************************/ +PVBLK AllocValBlock(PGLOBAL, void *, int, int, int len= 0, int prec= 0, + bool check= true, bool blank= false, bool un= false); + +/* --------------------------- Class DOSDEF -------------------------- */ + +/***********************************************************************/ +/* Constructor. */ +/***********************************************************************/ +DOSDEF::DOSDEF(void) + { + Pseudo = 3; + Fn = NULL; + Ofn = NULL; + Entry = NULL; + To_Indx = NULL; + Pwd = NULL; + Recfm = RECFM_VAR; + Mapped = false; + Zipped = false; + Mulentries = false; + Append = false; + Padded = false; + Huge = false; + Accept = false; + Eof = false; + To_Pos = NULL; + Optimized = 0; + AllocBlks = 0; + Compressed = 0; + Lrecl = 0; + AvgLen = 0; + Block = 0; + Last = 0; + Blksize = 0; + Maxerr = 0; + ReadMode = 0; + Ending = 0; + Teds = 0; + } // end of DOSDEF constructor + +/***********************************************************************/ +/* DefineAM: define specific AM block values from XDB file. */ +/***********************************************************************/ +bool DOSDEF::DefineAM(PGLOBAL g, LPCSTR am, int) + { + char buf[8]; + bool map = (am && (*am == 'M' || *am == 'm')); + LPCSTR dfm = (am && (*am == 'F' || *am == 'f')) ? "F" + : (am && (*am == 'B' || *am == 'b')) ? "B" + : (am && (*am == 'X' || *am == 'x')) ? "X" + : (am && !stricmp(am, "DBF")) ? "D" : "V"; + + if ((Zipped = GetBoolCatInfo("Zipped", false))) { + Entry = GetStringCatInfo(g, "Entry", NULL); + Mulentries = (Entry && *Entry) ? strchr(Entry, '*') || strchr(Entry, '?') + : false; + Mulentries = GetBoolCatInfo("Mulentries", Mulentries); + Append = GetBoolCatInfo("Append", false); + Pwd = GetStringCatInfo(g, "Password", NULL); + } // endif Zipped + + Desc = Fn = GetStringCatInfo(g, "Filename", NULL); + Ofn = GetStringCatInfo(g, "Optname", Fn); + GetCharCatInfo("Recfm", (PSZ)dfm, buf, sizeof(buf)); + Recfm = (toupper(*buf) == 'F') ? RECFM_FIX : + (toupper(*buf) == 'B') ? RECFM_BIN : + (toupper(*buf) == 'X') ? RECFM_NAF : // MGO + (toupper(*buf) == 'D') ? RECFM_DBF : RECFM_VAR; + Lrecl = GetIntCatInfo("Lrecl", 0); + + if (Recfm != RECFM_DBF) + Compressed = GetIntCatInfo("Compressed", 0); + + Mapped = GetBoolCatInfo("Mapped", map); +//Block = GetIntCatInfo("Blocks", 0); +//Last = GetIntCatInfo("Last", 0); + Ending = GetIntCatInfo("Ending", CRLF); + + if (Ending <= 0) { + Ending = (Recfm == RECFM_BIN || Recfm == RECFM_VCT) ? 0 : CRLF; + SetIntCatInfo("Ending", Ending); + } // endif ending + + if (Recfm == RECFM_FIX || Recfm == RECFM_BIN) { + Huge = GetBoolCatInfo("Huge", Cat->GetDefHuge()); + Padded = GetBoolCatInfo("Padded", false); + Blksize = GetIntCatInfo("Blksize", 0); + Eof = (GetIntCatInfo("EOF", 0) != 0); + Teds = toupper(*GetStringCatInfo(g, "Endian", "")); + } else if (Recfm == RECFM_DBF) { + Maxerr = GetIntCatInfo("Maxerr", 0); + Accept = GetBoolCatInfo("Accept", false); + ReadMode = GetIntCatInfo("Readmode", 0); + } else // (Recfm == RECFM_VAR) + AvgLen = GetIntCatInfo("Avglen", 0); + + // Ignore wrong Index definitions for catalog commands + SetIndexInfo(); + return false; + } // end of DefineAM + +/***********************************************************************/ +/* Get the full path/name of the optization file. */ +/***********************************************************************/ +bool DOSDEF::GetOptFileName(PGLOBAL g, char *filename) + { + PCSZ ftype; + + switch (Recfm) { + case RECFM_VAR: ftype = ".dop"; break; + case RECFM_FIX: ftype = ".fop"; break; + case RECFM_BIN: ftype = ".bop"; break; + case RECFM_VCT: ftype = ".vop"; break; + case RECFM_CSV: ftype = ".cop"; break; + case RECFM_DBF: ftype = ".dbp"; break; + default: + snprintf(g->Message, sizeof(g->Message), MSG(INVALID_FTYPE), Recfm); + return true; + } // endswitch Ftype + + PlugSetPath(filename, Ofn, GetPath()); + strcat(PlugRemoveType(filename, filename), ftype); + return false; + } // end of GetOptFileName + +/***********************************************************************/ +/* After an optimize error occurred, remove all set optimize values. */ +/***********************************************************************/ +void DOSDEF::RemoveOptValues(PGLOBAL g) + { + char filename[_MAX_PATH]; + PCOLDEF cdp; + + // Delete settings of optimized columns + for (cdp = To_Cols; cdp; cdp = cdp->GetNext()) + if (cdp->GetOpt()) { + cdp->SetMin(NULL); + cdp->SetMax(NULL); + cdp->SetNdv(0); + cdp->SetNbm(0); + cdp->SetDval(NULL); + cdp->SetBmap(NULL); + } // endif Opt + + // Delete block position setting for not fixed tables + To_Pos = NULL; + AllocBlks = 0; + + // Delete any eventually ill formed non matching optimization file + if (!GetOptFileName(g, filename)) +#if defined(_WIN32) + DeleteFile(filename); +#else // UNIX + remove(filename); +#endif // _WIN32 + + Optimized = 0; + } // end of RemoveOptValues + +/***********************************************************************/ +/* DeleteIndexFile: Delete DOS/UNIX index file(s) using platform API. */ +/***********************************************************************/ +bool DOSDEF::DeleteIndexFile(PGLOBAL g, PIXDEF pxdf) + { + PCSZ ftype; + char filename[_MAX_PATH]; + bool sep, rc = false; + + if (!To_Indx) + return false; // No index + + // If true indexes are in separate files + sep = GetBoolCatInfo("SepIndex", false); + + if (!sep && pxdf) { + safe_strcpy(g->Message, sizeof(g->Message), MSG(NO_RECOV_SPACE)); + return true; + } // endif sep + + switch (Recfm) { + case RECFM_VAR: ftype = ".dnx"; break; + case RECFM_FIX: ftype = ".fnx"; break; + case RECFM_BIN: ftype = ".bnx"; break; + case RECFM_VCT: ftype = ".vnx"; break; + case RECFM_CSV: ftype = ".cnx"; break; + case RECFM_DBF: ftype = ".dbx"; break; + default: + snprintf(g->Message, sizeof(g->Message), MSG(BAD_RECFM_VAL), Recfm); + return true; + } // endswitch Ftype + + /*********************************************************************/ + /* Check for existence of an index file. */ + /*********************************************************************/ + if (sep) { + // Indexes are save in separate files +#if defined(_WIN32) + char drive[_MAX_DRIVE]; +#else + char *drive = NULL; +#endif + char direc[_MAX_DIR]; + char fname[_MAX_FNAME]; + bool all = !pxdf; + + if (all) + pxdf = To_Indx; + + for (; pxdf; pxdf = pxdf->GetNext()) { + _splitpath(Ofn, drive, direc, fname, NULL); + safe_strcat(fname, sizeof(fname), "_"); + safe_strcat(fname, sizeof(fname), pxdf->GetName()); + _makepath(filename, drive, direc, fname, ftype); + PlugSetPath(filename, filename, GetPath()); +#if defined(_WIN32) + if (!DeleteFile(filename)) + rc |= (GetLastError() != ERROR_FILE_NOT_FOUND); +#else // UNIX + if (remove(filename)) + rc |= (errno != ENOENT); +#endif // UNIX + + if (!all) + break; + + } // endfor pxdf + + } else { // !sep + // Drop all indexes, delete the common file + PlugSetPath(filename, Ofn, GetPath()); + safe_strcat(PlugRemoveType(filename, filename), sizeof(filename), ftype); +#if defined(_WIN32) + if (!DeleteFile(filename)) + rc = (GetLastError() != ERROR_FILE_NOT_FOUND); +#else // UNIX + if (remove(filename)) + rc = (errno != ENOENT); +#endif // UNIX + } // endif sep + + if (rc) + snprintf(g->Message, sizeof(g->Message), MSG(DEL_FILE_ERR), filename); + + return rc; // Return true if error + } // end of DeleteIndexFile + +/***********************************************************************/ +/* InvalidateIndex: mark all indexes as invalid. */ +/***********************************************************************/ +bool DOSDEF::InvalidateIndex(PGLOBAL) + { + if (To_Indx) + for (PIXDEF xp = To_Indx; xp; xp = xp->Next) + xp->Invalid = true; + + return false; + } // end of InvalidateIndex + +/***********************************************************************/ +/* GetTable: makes a new Table Description Block. */ +/***********************************************************************/ +PTDB DOSDEF::GetTable(PGLOBAL g, MODE mode) + { + // Mapping not used for insert + USETEMP tmp = UseTemp(); + bool map = Mapped && mode != MODE_INSERT && + !(tmp != TMP_NO && Recfm == RECFM_VAR + && mode == MODE_UPDATE) && + !(tmp == TMP_FORCE && + (mode == MODE_UPDATE || mode == MODE_DELETE)); + PTXF txfp = NULL; + PTDBASE tdbp; + + /*********************************************************************/ + /* Allocate table and file processing class of the proper type. */ + /* Column blocks will be allocated only when needed. */ + /*********************************************************************/ + if (Recfm == RECFM_DBF) { + if (Catfunc == FNC_NO) { + if (Zipped) { + if (mode == MODE_READ || mode == MODE_ANY || mode == MODE_ALTER) { + txfp = new(g) UZDFAM(this); + } else { + safe_strcpy(g->Message, sizeof(g->Message), "Zipped DBF tables are read only"); + return NULL; + } // endif's mode + + } else if (map) + txfp = new(g) DBMFAM(this); + else + txfp = new(g) DBFFAM(this); + + tdbp = new(g) TDBFIX(this, txfp); + } else + tdbp = new(g) TDBDCL(this); // Catfunc should be 'C' + + } else if (Zipped) { +#if defined(ZIP_SUPPORT) + if (Recfm == RECFM_VAR) { + if (mode == MODE_READ || mode == MODE_ANY || mode == MODE_ALTER) { + txfp = new(g) UNZFAM(this); + } else if (mode == MODE_INSERT) { + txfp = new(g) ZIPFAM(this); + } else { + safe_strcpy(g->Message, sizeof(g->Message), "UPDATE/DELETE not supported for ZIP"); + return NULL; + } // endif's mode + + tdbp = new(g) TDBDOS(this, txfp); + } else { + if (mode == MODE_READ || mode == MODE_ANY || mode == MODE_ALTER) { + txfp = new(g) UZXFAM(this); + } else if (mode == MODE_INSERT) { + txfp = new(g) ZPXFAM(this); + } else { + safe_strcpy(g->Message, sizeof(g->Message), "UPDATE/DELETE not supported for ZIP"); + return NULL; + } // endif's mode + + tdbp = new(g)TDBFIX(this, txfp); + } // endif Recfm + +#else // !ZIP_SUPPORT + snprintf(g->Message, sizeof(g->Message), MSG(NO_FEAT_SUPPORT), "ZIP"); + return NULL; +#endif // !ZIP_SUPPORT + } else if (Recfm != RECFM_VAR && Compressed < 2) { + if (Huge) + txfp = new(g) BGXFAM(this); + else if (map) + txfp = new(g) MPXFAM(this); + else if (Compressed) { +#if defined(GZ_SUPPORT) + txfp = new(g) GZXFAM(this); +#else // !GZ_SUPPORT + snprintf(g->Message, sizeof(g->Message), MSG(NO_FEAT_SUPPORT), "GZ"); + return NULL; +#endif // !GZ_SUPPORT + } else + txfp = new(g) FIXFAM(this); + + tdbp = new(g) TDBFIX(this, txfp); + } else { + if (Compressed) { +#if defined(GZ_SUPPORT) + if (Compressed == 1) + txfp = new(g) GZFAM(this); + else + txfp = new(g) ZLBFAM(this); + +#else // !GZ_SUPPORT + snprintf(g->Message, sizeof(g->Message), MSG(NO_FEAT_SUPPORT), "GZ"); + return NULL; +#endif // !GZ_SUPPORT + } else if (map) + txfp = new(g) MAPFAM(this); + else + txfp = new(g) DOSFAM(this); + + // Txfp must be set even for not multiple tables because + // it is needed when calling Cardinality in GetBlockValues. + tdbp = new(g) TDBDOS(this, txfp); + } // endif Recfm + + if (Multiple) + tdbp = new(g) TDBMUL(tdbp); + else + /*******************************************************************/ + /* For block tables, get eventually saved optimization values. */ + /*******************************************************************/ + if (tdbp->GetBlockValues(g)) { + PushWarning(g, tdbp); +// return NULL; // causes a crash when deleting index + } else if (Recfm == RECFM_VAR || Compressed > 1) { + if (IsOptimized()) { + if (map) { + txfp = new(g) MBKFAM(this); + } else if (Compressed) { +#if defined(GZ_SUPPORT) + if (Compressed == 1) + txfp = new(g) ZBKFAM(this); + else { + txfp->SetBlkPos(To_Pos); + ((PZLBFAM)txfp)->SetOptimized(To_Pos != NULL); + } // endelse +#else + snprintf(g->Message, sizeof(g->Message), MSG(NO_FEAT_SUPPORT), "GZ"); + return NULL; +#endif + } else + txfp = new(g) BLKFAM(this); + + ((PTDBDOS)tdbp)->SetTxfp(txfp); + } // endif Optimized + + } // endif Recfm + + return tdbp; + } // end of GetTable + +/* ------------------------ Class TDBDOS ----------------------------- */ + +/***********************************************************************/ +/* Implementation of the TDBDOS class. This is the common class that */ +/* contain all that is common between the TDBDOS and TDBMAP classes. */ +/***********************************************************************/ +TDBDOS::TDBDOS(PDOSDEF tdp, PTXF txfp) : TDBASE(tdp) + { + if ((Txfp = txfp)) + Txfp->SetTdbp(this); + + Lrecl = tdp->Lrecl; + AvgLen = tdp->AvgLen; + Ftype = tdp->Recfm; + To_Line = NULL; +//To_BlkIdx = NULL; + To_BlkFil = NULL; + SavFil = NULL; +//Xeval = 0; + Beval = 0; + Abort = false; + Indxd = false; + } // end of TDBDOS standard constructor + +TDBDOS::TDBDOS(PGLOBAL g, PTDBDOS tdbp) : TDBASE(tdbp) + { + Txfp = (g) ? tdbp->Txfp->Duplicate(g) : tdbp->Txfp; + Lrecl = tdbp->Lrecl; + AvgLen = tdbp->AvgLen; + Ftype = tdbp->Ftype; + To_Line = tdbp->To_Line; +//To_BlkIdx = tdbp->To_BlkIdx; + To_BlkFil = tdbp->To_BlkFil; + SavFil = tdbp->SavFil; +//Xeval = tdbp->Xeval; + Beval = tdbp->Beval; + Abort = tdbp->Abort; + Indxd = tdbp->Indxd; + } // end of TDBDOS copy constructor + +// Method +PTDB TDBDOS::Clone(PTABS t) + { + PTDB tp; + PDOSCOL cp1, cp2; + PGLOBAL g = t->G; + + tp = new(g) TDBDOS(g, this); + + for (cp1 = (PDOSCOL)Columns; cp1; cp1 = (PDOSCOL)cp1->GetNext()) { + cp2 = new(g) DOSCOL(cp1, tp); // Make a copy + NewPointer(t, cp1, cp2); + } // endfor cp1 + + return tp; + } // end of Clone + +/***********************************************************************/ +/* Allocate DOS column description block. */ +/***********************************************************************/ +PCOL TDBDOS::MakeCol(PGLOBAL g, PCOLDEF cdp, PCOL cprec, int n) + { + return new(g) DOSCOL(g, cdp, this, cprec, n); + } // end of MakeCol + +/***********************************************************************/ +/* Print debug information. */ +/***********************************************************************/ +void TDBDOS::PrintAM(FILE *f, char *m) + { + fprintf(f, "%s AM(%d): mode=%d\n", m, GetAmType(), Mode); + + if (Txfp->To_File) + fprintf(f, "%s File: %s\n", m, Txfp->To_File); + + } // end of PrintAM + +/***********************************************************************/ +/* Remake the indexes after the table was modified. */ +/***********************************************************************/ +int TDBDOS::ResetTableOpt(PGLOBAL g, bool dop, bool dox) + { + int prc = RC_OK, rc = RC_OK; + + if (!GetFileLength(g)) { + // Void table, delete all opt and index files + PDOSDEF defp = (PDOSDEF)To_Def; + + defp->RemoveOptValues(g); + return (defp->DeleteIndexFile(g, NULL)) ? RC_INFO : RC_OK; + } // endif GetFileLength + + MaxSize = -1; // Size must be recalculated + Cardinal = -1; // as well as Cardinality + + To_Filter = NULL; // Disable filtering +//To_BlkIdx = NULL; // and index filtering + To_BlkFil = NULL; // and block filtering + + // After the table was modified the indexes + // are invalid and we should mark them as such... + (void)((PDOSDEF)To_Def)->InvalidateIndex(g); + + if (dop) { + Columns = NULL; // Not used anymore + + if (Txfp->Blocked) { + // MakeBlockValues must be executed in non blocked mode + // except for ZLIB access method. + if (Txfp->GetAmType() == TYPE_AM_MAP) { + Txfp = new(g) MAPFAM((PDOSDEF)To_Def); +#if defined(GZ_SUPPORT) + } else if (Txfp->GetAmType() == TYPE_AM_GZ) { + Txfp = new(g) GZFAM((PDOSDEF)To_Def); + } else if (Txfp->GetAmType() == TYPE_AM_ZLIB) { + Txfp->Reset(); + ((PZLBFAM)Txfp)->SetOptimized(false); +#endif // GZ_SUPPORT + } else if (Txfp->GetAmType() == TYPE_AM_BLK) + Txfp = new(g) DOSFAM((PDOSDEF)To_Def); + + Txfp->SetTdbp(this); + } else + Txfp->Reset(); + + Use = USE_READY; // So the table can be reopened + Mode = MODE_ANY; // Just to be clean + rc = MakeBlockValues(g); // Redo optimization + } // endif dop + + if (dox && (rc == RC_OK || rc == RC_INFO)) { + // Remake eventual indexes +// if (Mode != MODE_UPDATE) + To_SetCols = NULL; // Positions are changed + + Columns = NULL; // Not used anymore + Txfp->Reset(); // New start + Use = USE_READY; // So the table can be reopened + Mode = MODE_READ; // New mode + prc = rc; + + if (PlgGetUser(g)->Check & CHK_OPT) + // We must remake all indexes. + rc = MakeIndex(g, NULL, false); + + rc = (rc == RC_INFO) ? prc : rc; + } // endif dox + + return rc; + } // end of ResetTableOpt + +/***********************************************************************/ +/* Calculate the block sizes so block I/O can be used and also the */ +/* Min/Max values for clustered/sorted table columns. */ +/***********************************************************************/ +int TDBDOS::MakeBlockValues(PGLOBAL g) + { + int i, lg, nrec, rc, n = 0; + int curnum, curblk, block, savndv, savnbm; + void *savmin, *savmax; + bool blocked, xdb2 = false; +//POOLHEADER save; + PCOLDEF cdp; + PDOSDEF defp = (PDOSDEF)To_Def; + PDOSCOL colp = NULL; + PDBUSER dup = PlgGetUser(g); +//void *memp = cat->GetDescp(); + (void) defp->GetCat(); // XXX Should be removed? + + + if ((nrec = defp->GetElemt()) < 2) { + if (!To_Def->Partitioned()) { + // This may be wrong to do in some cases + safe_strcpy(g->Message, sizeof(g->Message), MSG(TABLE_NOT_OPT)); + return RC_INFO; // Not to be optimized + } else + return RC_OK; + + } else if (GetMaxSize(g) == 0 || !(dup->Check & CHK_OPT)) { + // Suppress the opt file firstly if the table is void, + // secondly when it was modified with OPTIMIZATION unchecked + // because it is no more valid. + defp->RemoveOptValues(g); // Erase opt file + return RC_OK; // void table + } else if (MaxSize < 0) + return RC_FX; + + defp->SetOptimized(0); + + // Estimate the number of needed blocks + if ((block = (int)((MaxSize + (int)nrec - 1) / (int)nrec)) < 2) { + // This may be wrong to do in some cases + defp->RemoveOptValues(g); + safe_strcpy(g->Message, sizeof(g->Message), MSG(TABLE_NOT_OPT)); + return RC_INFO; // Not to be optimized + } // endif block + + // We have to use local variables because Txfp->CurBlk is set + // to Rows+1 by unblocked variable length table access methods. + curblk = -1; + curnum = nrec - 1; +//last = 0; + Txfp->Block = block; // This is useful mainly for + Txfp->CurBlk = curblk; // blocked tables (ZLBFAM), for + Txfp->CurNum = curnum; // others it is just to be clean. + + /*********************************************************************/ + /* Allocate the array of block starting positions. */ + /*********************************************************************/ +//if (memp) +// save = *(PPOOLHEADER)memp; + + Txfp->BlkPos = (int*)PlugSubAlloc(g, NULL, (block + 1) * sizeof(int)); + + /*********************************************************************/ + /* Allocate the blocks for clustered columns. */ + /*********************************************************************/ + blocked = Txfp->Blocked; // Save + Txfp->Blocked = true; // So column block can be allocated + + for (cdp = defp->GetCols(), i = 1; cdp; cdp = cdp->GetNext(), i++) + if (cdp->GetOpt()) { + lg = cdp->GetClen(); + + if (cdp->GetFreq() && cdp->GetFreq() <= dup->Maxbmp) { + cdp->SetXdb2(true); + savndv = cdp->GetNdv(); + cdp->SetNdv(0); // Reset Dval number of values + xdb2 = true; + savmax = cdp->GetDval(); + cdp->SetDval(PlugSubAlloc(g, NULL, cdp->GetFreq() * lg)); + savnbm = cdp->GetNbm(); + cdp->SetNbm(0); // Prevent Bmap allocation +// savmin = cdp->GetBmap(); +// cdp->SetBmap(PlugSubAlloc(g, NULL, block * sizeof(int))); + + if (trace(1)) + htrc("Dval(%p) Bmap(%p) col(%d) %s Block=%d lg=%d\n", + cdp->GetDval(), cdp->GetBmap(), i, cdp->GetName(), block, lg); + + // colp will be initialized with proper Dval VALBLK + colp = (PDOSCOL)MakeCol(g, cdp, colp, i); + colp->InitValue(g); // Allocate column value buffer + cdp->SetNbm(savnbm); +// cdp->SetBmap(savmin); // Can be reused if the new size + cdp->SetDval(savmax); // is not greater than this one. + cdp->SetNdv(savndv); + } else { + cdp->SetXdb2(false); // Maxbmp may have been reset + savmin = cdp->GetMin(); + savmax = cdp->GetMax(); + cdp->SetMin(PlugSubAlloc(g, NULL, block * lg)); + cdp->SetMax(PlugSubAlloc(g, NULL, block * lg)); + + // Valgrind complains if there are uninitialised bytes + // after the null character ending + if (IsTypeChar(cdp->GetType())) { + memset(cdp->GetMin(), 0, block * lg); + memset(cdp->GetMax(), 0, block * lg); + } // endif Type + + if (trace(1)) + htrc("min(%p) max(%p) col(%d) %s Block=%d lg=%d\n", + cdp->GetMin(), cdp->GetMax(), i, cdp->GetName(), block, lg); + + // colp will be initialized with proper opt VALBLK's + colp = (PDOSCOL)MakeCol(g, cdp, colp, i); + colp->InitValue(g); // Allocate column value buffer + cdp->SetMin(savmin); // Can be reused if the number + cdp->SetMax(savmax); // of blocks does not change. + } // endif Freq + + } // endif Clustered + + // No optimised columns. Still useful for blocked variable tables. + if (!colp && defp->Recfm != RECFM_VAR) { + safe_strcpy(g->Message, sizeof(g->Message), "No optimised columns"); + return RC_INFO; + } // endif colp + + Txfp->Blocked = blocked; + + /*********************************************************************/ + /* Now do calculate the optimization values. */ + /*********************************************************************/ + Mode = MODE_READ; + + if (OpenDB(g)) + return RC_FX; + + if (xdb2) { + /*********************************************************************/ + /* Retrieve the distinct values of XDB2 columns. */ + /*********************************************************************/ + if (GetDistinctColumnValues(g, nrec)) + return RC_FX; + + OpenDB(g); // Rewind the table file + } // endif xdb2 + +#if defined(PROG_INFO) + /*********************************************************************/ + /* Initialize progress information */ + /*********************************************************************/ + char *p = (char *)PlugSubAlloc(g, NULL, 24 + strlen(Name)); + + snprintf(p, 24 + strlen(Name), "%s%s", MSG(OPTIMIZING), Name); + dup->Step = p; + dup->ProgMax = GetProgMax(g); + dup->ProgCur = 0; +#endif // SOCKET_MODE || THREAD + + /*********************************************************************/ + /* Make block starting pos and min/max values of cluster columns. */ + /*********************************************************************/ + while ((rc = ReadDB(g)) == RC_OK) { + if (blocked) { + // A blocked FAM class handles CurNum and CurBlk (ZLBFAM) + if (!Txfp->CurNum) + Txfp->BlkPos[Txfp->CurBlk] = Txfp->GetPos(); + + } else { + if (++curnum >= nrec) { + if (++curblk >= block) { + safe_strcpy(g->Message, sizeof(g->Message), MSG(BAD_BLK_ESTIM)); + goto err; + } else + curnum = 0; + + // Get block starting position + Txfp->BlkPos[curblk] = Txfp->GetPos(); + } // endif CurNum + +// last = curnum + 1; // curnum is zero based + Txfp->CurBlk = curblk; // Used in COLDOS::SetMinMax + Txfp->CurNum = curnum; // Used in COLDOS::SetMinMax + } // endif blocked + + /*******************************************************************/ + /* Now calculate the min and max values for the cluster columns. */ + /*******************************************************************/ + for (colp = (PDOSCOL)Columns; colp; colp = (PDOSCOL)colp->GetNext()) + if (colp->Clustered == 2) { + if (colp->SetBitMap(g)) + goto err; + + } else + if (colp->SetMinMax(g)) + goto err; // Currently: column is not sorted + +#if defined(PROG_INFO) + if (!dup->Step) { + safe_strcpy(g->Message, sizeof(g->Message), MSG(OPT_CANCELLED)); + goto err; + } else + dup->ProgCur = GetProgCur(); +#endif // PROG_INFO + + n++; // Used to calculate block and last + } // endwhile + + if (rc == RC_EF) { + Txfp->Nrec = nrec; + +#if 0 // No good because Curblk and CurNum after EOF are different + // depending on whether the file is mapped or not mapped. + if (blocked) { +// Txfp->Block = Txfp->CurBlk + 1; + Txfp->Last = (Txfp->CurNum) ? Txfp->CurNum : nrec; +// Txfp->Last = (Txfp->CurNum) ? Txfp->CurNum + 1 : nrec; + Txfp->Block = Txfp->CurBlk + (Txfp->Last == nrec ? 0 : 1); + } else { + Txfp->Block = curblk + 1; + Txfp->Last = last; + } // endif blocked +#endif // 0 + + // New values of Block and Last + Txfp->Block = (n + nrec - 1) / nrec; + Txfp->Last = (n % nrec) ? (n % nrec) : nrec; + + // This is needed to be able to calculate the last block size + Txfp->BlkPos[Txfp->Block] = Txfp->GetNextPos(); + } else + goto err; + + /*********************************************************************/ + /* Save the optimization values for this table. */ + /*********************************************************************/ + if (!SaveBlockValues(g)) { + defp->Block = Txfp->Block; + defp->Last = Txfp->Last; + CloseDB(g); + defp->SetIntCatInfo("Blocks", Txfp->Block); + defp->SetIntCatInfo("Last", Txfp->Last); + return RC_OK; + } // endif SaveBlockValues + + err: + // Restore Desc memory suballocation +//if (memp) +// *(PPOOLHEADER)memp = save; + + defp->RemoveOptValues(g); + CloseDB(g); + return RC_FX; + } // end of MakeBlockValues + +/***********************************************************************/ +/* Save the block and Min/Max values for this table. */ +/* The problem here is to avoid name duplication, because more than */ +/* one data file can have the same name (but different types) and/or */ +/* the same data file can be used with different block sizes. This is */ +/* why we use Ofn that defaults to the file name but can be set to a */ +/* different name if necessary. */ +/***********************************************************************/ +bool TDBDOS::SaveBlockValues(PGLOBAL g) + { + char filename[_MAX_PATH]; + int lg, n[NZ + 2]; + size_t nbk, ndv, nbm, block = Txfp->Block; + bool rc = false; + FILE *opfile; + PDOSCOL colp; + PDOSDEF defp = (PDOSDEF)To_Def; + + if (defp->GetOptFileName(g, filename)) + return true; + + if (!(opfile = fopen(filename, "wb"))) { + snprintf(g->Message, sizeof(g->Message), MSG(OPEN_MODE_ERROR), + "wb", (int)errno, filename); + safe_strcat(g->Message, sizeof(g->Message), ": "); + safe_strcat(g->Message, sizeof(g->Message), strerror(errno)); + + if (trace(1)) + htrc("%s\n", g->Message); + + return true; + } // endif opfile + + memset(n, 0, sizeof(n)); // To avoid valgrind warning + + if (Ftype == RECFM_VAR || defp->Compressed == 2) { + /*******************************************************************/ + /* Write block starting positions into the opt file. */ + /*******************************************************************/ + block++; + lg = sizeof(int); + n[0] = Txfp->Last; n[1] = lg; n[2] = Txfp->Nrec; n[3] = Txfp->Block; + + if (fwrite(n, sizeof(int), NZ, opfile) != NZ) { + snprintf(g->Message, sizeof(g->Message), MSG(OPT_HEAD_WR_ERR), strerror(errno)); + rc = true; + } // endif size + + if (fwrite(Txfp->BlkPos, lg, block, opfile) != block) { + snprintf(g->Message, sizeof(g->Message), MSG(OPTBLK_WR_ERR), strerror(errno)); + rc = true; + } // endif size + + block--; // = Txfp->Block; + } // endif Ftype + + /*********************************************************************/ + /* Write the Min/Max values into the opt file. */ + /*********************************************************************/ + for (colp = (PDOSCOL)Columns; colp; colp = (PDOSCOL)colp->Next) { + lg = colp->Value->GetClen(); + + // Now start the writing process + if (colp->Clustered == 2) { + // New XDB2 block optimization. Will be recognized when reading + // because the column index is negated. + ndv = colp->Ndv; nbm = colp->Nbm; + nbk = nbm * block; + n[0] = -colp->Index; n[1] = lg; n[2] = Txfp->Nrec; n[3] = block; + n[4] = ndv; n[5] = nbm; + + if (fwrite(n, sizeof(int), NZ + 2, opfile) != NZ + 2) { + snprintf(g->Message, sizeof(g->Message), MSG(OPT_HEAD_WR_ERR), strerror(errno)); + rc = true; + } // endif size + + if (fwrite(colp->Dval->GetValPointer(), lg, ndv, opfile) != ndv) { + snprintf(g->Message, sizeof(g->Message), MSG(OPT_DVAL_WR_ERR), strerror(errno)); + rc = true; + } // endif size + + if (fwrite(colp->Bmap->GetValPointer(), sizeof(int), nbk, opfile) != nbk) { + snprintf(g->Message, sizeof(g->Message), MSG(OPT_BMAP_WR_ERR), strerror(errno)); + rc = true; + } // endif size + + } else { + n[0] = colp->Index; n[1] = lg; n[2] = Txfp->Nrec; n[3] = block; + + if (fwrite(n, sizeof(int), NZ, opfile) != NZ) { + snprintf(g->Message, sizeof(g->Message), MSG(OPT_HEAD_WR_ERR), strerror(errno)); + rc = true; + } // endif size + + if (fwrite(colp->Min->GetValPointer(), lg, block, opfile) != block) { + snprintf(g->Message, sizeof(g->Message), MSG(OPT_MIN_WR_ERR), strerror(errno)); + rc = true; + } // endif size + + if (fwrite(colp->Max->GetValPointer(), lg, block, opfile) != block) { + snprintf(g->Message, sizeof(g->Message), MSG(OPT_MAX_WR_ERR), strerror(errno)); + rc = true; + } // endif size + + } // endif Clustered + + } // endfor colp + + fclose(opfile); + return rc; + } // end of SaveBlockValues + +/***********************************************************************/ +/* Read the Min/Max values for this table. */ +/* The problem here is to avoid name duplication, because more than */ +/* one data file can have the same name (but different types) and/or */ +/* the same data file can be used with different block sizes. This is */ +/* why we use Ofn that defaults to the file name but can be set to a */ +/* different name if necessary. */ +/***********************************************************************/ +bool TDBDOS::GetBlockValues(PGLOBAL g) + { + char filename[_MAX_PATH]; + int i, lg, n[NZ]; + int nrec, block = 0, last = 0; + int len; + bool newblk = false; + size_t ndv, nbm, nbk, blk; + FILE *opfile; + PCOLDEF cdp; + PDOSDEF defp = (PDOSDEF)To_Def; + PDBUSER dup = PlgGetUser(g); + + (void) defp->GetCat(); // XXX Should be removed? + + +#if 0 + if (Mode == MODE_INSERT && Txfp->GetAmType() == TYPE_AM_DOS) + return false; +#endif // _WIN32 + + if (defp->Optimized || !(dup->Check & CHK_OPT)) + return false; // Already done or to be redone + + if (Ftype == RECFM_VAR || defp->Compressed == 2) { + /*******************************************************************/ + /* Variable length file that can be read by block. */ + /*******************************************************************/ + nrec = (defp->GetElemt()) ? defp->GetElemt() : 1; + + if (nrec > 1) { + // The table can be declared optimized if it is void. + // This is useful to handle Insert in optimized mode. + char filename[_MAX_PATH]; + int h; + int flen = -1; + + PlugSetPath(filename, defp->Fn, GetPath()); + h = open(filename, O_RDONLY); + flen = (h == -1 && errno == ENOENT) ? 0 : _filelength(h); + + if (h != -1) + close(h); + + if (!flen) { + defp->SetOptimized(1); + return false; + } // endif flen + + } else + return false; // Not optimisable + + cdp = defp->GetCols(); + i = 1; + } else { + /*******************************************************************/ + /* Fixed length file. Opt file exists only for clustered columns. */ + /*******************************************************************/ + // Check for existence of clustered columns + for (cdp = defp->GetCols(), i = 1; cdp; cdp = cdp->GetNext(), i++) + if (cdp->GetOpt()) + break; + + if (!cdp) + return false; // No optimization needed + + if ((len = Cardinality(g)) < 0) + return true; // Table error + else if (!len) + return false; // File does not exist yet + + block = Txfp->Block; // Was set in Cardinality + nrec = Txfp->Nrec; + } // endif Ftype + + if (defp->GetOptFileName(g, filename)) + return true; + + if (!(opfile = fopen(filename, "rb"))) + return false; // No saved values + + if (Ftype == RECFM_VAR || defp->Compressed == 2) { + /*******************************************************************/ + /* Read block starting positions from the opt file. */ + /*******************************************************************/ + lg = sizeof(int); + + if (fread(n, sizeof(int), NZ, opfile) != NZ) { + snprintf(g->Message, sizeof(g->Message), MSG(OPT_HEAD_RD_ERR), strerror(errno)); + goto err; + } // endif size + + if (n[1] != lg || n[2] != nrec) { + snprintf(g->Message, sizeof(g->Message), MSG(OPT_NOT_MATCH), filename); + goto err; + } // endif + + last = n[0]; + block = n[3]; + blk = block + 1; + + defp->To_Pos = (int*)PlugSubAlloc(g, NULL, blk * lg); + + if (fread(defp->To_Pos, lg, blk, opfile) != blk) { + snprintf(g->Message, sizeof(g->Message), MSG(OPTBLK_RD_ERR), strerror(errno)); + goto err; + } // endif size + + } // endif Ftype + + /*********************************************************************/ + /* Read the Min/Max values from the opt file. */ + /*********************************************************************/ + for (; cdp; cdp = cdp->GetNext(), i++) + if (cdp->GetOpt()) { + lg = cdp->GetClen(); + blk = block; + + // Now start the reading process. + if (fread(n, sizeof(int), NZ, opfile) != NZ) { + snprintf(g->Message, sizeof(g->Message), MSG(OPT_HEAD_RD_ERR), strerror(errno)); + goto err; + } // endif size + + if (n[0] == -i) { + // Read the XDB2 opt values from the opt file + if (n[1] != lg || n[2] != nrec || n[3] != block) { + snprintf(g->Message, sizeof(g->Message), MSG(OPT_NOT_MATCH), filename); + goto err; + } // endif + + if (fread(n, sizeof(int), 2, opfile) != 2) { + snprintf(g->Message, sizeof(g->Message), MSG(OPT_HEAD_RD_ERR), strerror(errno)); + goto err; + } // endif fread + + ndv = n[0]; nbm = n[1]; nbk = nbm * blk; + + if (cdp->GetNdv() < (int)ndv || !cdp->GetDval()) + cdp->SetDval(PlugSubAlloc(g, NULL, ndv * lg)); + + cdp->SetNdv((int)ndv); + + if (fread(cdp->GetDval(), lg, ndv, opfile) != ndv) { + snprintf(g->Message, sizeof(g->Message), MSG(OPT_DVAL_RD_ERR), strerror(errno)); + goto err; + } // endif size + + if (newblk || cdp->GetNbm() < (int)nbm || !cdp->GetBmap()) + cdp->SetBmap(PlugSubAlloc(g, NULL, nbk * sizeof(int))); + + cdp->SetNbm((int)nbm); + + if (fread(cdp->GetBmap(), sizeof(int), nbk, opfile) != nbk) { + snprintf(g->Message, sizeof(g->Message), MSG(OPT_BMAP_RD_ERR), strerror(errno)); + goto err; + } // endif size + + cdp->SetXdb2(true); + } else { + // Read the Min/Max values from the opt file + if (n[0] != i || n[1] != lg || n[2] != nrec || n[3] != block) { + snprintf(g->Message, sizeof(g->Message), MSG(OPT_NOT_MATCH), filename); + goto err; + } // endif + + if (newblk || !cdp->GetMin()) + cdp->SetMin(PlugSubAlloc(g, NULL, blk * lg)); + + if (fread(cdp->GetMin(), lg, blk, opfile) != blk) { + snprintf(g->Message, sizeof(g->Message), MSG(OPT_MIN_RD_ERR), strerror(errno)); + goto err; + } // endif size + + if (newblk || !cdp->GetMax()) + cdp->SetMax(PlugSubAlloc(g, NULL, blk * lg)); + + if (fread(cdp->GetMax(), lg, blk, opfile) != blk) { + snprintf(g->Message, sizeof(g->Message), MSG(OPT_MAX_RD_ERR), strerror(errno)); + goto err; + } // endif size + + cdp->SetXdb2(false); + } // endif n[0] (XDB2) + + } // endif Clustered + + defp->SetBlock(block); + defp->Last = last; // For Cardinality + defp->SetAllocBlks(block); + defp->SetOptimized(1); + fclose(opfile); + MaxSize = -1; // Can be refined later + return false; + + err: + defp->RemoveOptValues(g); + fclose(opfile); + + // Ignore error if not in mode CHK_OPT + return (PlgGetUser(g)->Check & CHK_OPT) != 0; + } // end of GetBlockValues + +/***********************************************************************/ +/* This fonction is used while making XDB2 block optimization. */ +/* It constructs for each elligible columns, the sorted list of the */ +/* distinct values existing in the column. This function uses an */ +/* algorithm that permit to get several sets of distinct values by */ +/* reading the table only once, which cannot be done using a standard */ +/* SQL query. */ +/***********************************************************************/ +bool TDBDOS::GetDistinctColumnValues(PGLOBAL g, int nrec) + { + char *p; + int rc, blk, n = 0; + PDOSCOL colp; + PDBUSER dup = PlgGetUser(g); + + /*********************************************************************/ + /* Initialize progress information */ + /*********************************************************************/ + p = (char *)PlugSubAlloc(g, NULL, 48 + strlen(Name)); + snprintf(p, 48 + strlen(Name), "%s%s", MSG(GET_DIST_VALS), Name); + dup->Step = p; + dup->ProgMax = GetProgMax(g); + dup->ProgCur = 0; + + while ((rc = ReadDB(g)) == RC_OK) { + for (colp = (PDOSCOL)Columns; colp; colp = (PDOSCOL)colp->Next) + if (colp->Clustered == 2) + if (colp->AddDistinctValue(g)) + return true; // Too many distinct values + +#if defined(SOCKET_MODE) + if (SendProgress(dup)) { + safe_strcpy(g->Message, sizeof(g->Message), MSG(OPT_CANCELLED)); + return true; + } else +#elif defined(THREAD) + if (!dup->Step) { + safe_strcpy(g->Message, sizeof(g->Message), MSG(OPT_CANCELLED)); + return true; + } else +#endif // THREAD + dup->ProgCur = GetProgCur(); + + n++; + } // endwhile + + if (rc != RC_EF) + return true; + + // Reset the number of table blocks +//nrec = ((PDOSDEF)To_Def)->GetElemt(); (or default value) + blk = (n + nrec - 1) / nrec; + Txfp->Block = blk; // Useful mainly for ZLBFAM ??? + + // Set Nbm, Bmap for XDB2 columns + for (colp = (PDOSCOL)Columns; colp; colp = (PDOSCOL)colp->Next) + if (colp->Clustered == 2) { +// colp->Cdp->SetNdv(colp->Ndv); + colp->Nbm = (colp->Ndv + MAXBMP - 1) / MAXBMP; + colp->Bmap = AllocValBlock(g, NULL, TYPE_INT, colp->Nbm * blk); + } // endif Clustered + + return false; + } // end of GetDistinctColumnValues + +/***********************************************************************/ +/* Analyze the filter and construct the Block Evaluation Filter. */ +/* This is possible when a filter contains predicates implying a */ +/* column marked as "clustered" or "sorted" matched to a constant */ +/* argument. It is then possible by comparison against the smallest */ +/* and largest column values in each block to determine whether the */ +/* filter condition will be always true or always false for the block.*/ +/***********************************************************************/ +PBF TDBDOS::InitBlockFilter(PGLOBAL g, PFIL filp) + { + bool blk = Txfp->Blocked; + + if (To_BlkFil) + return To_BlkFil; // Already done + else if (!filp) + return NULL; + else if (blk) { + if (Txfp->GetAmType() == TYPE_AM_DBF) + /*****************************************************************/ + /* If RowID is used in this query, block optimization cannot be */ + /* used because currently the file must be read sequentially. */ + /*****************************************************************/ + for (PCOL cp = Columns; cp; cp = cp->GetNext()) + if (cp->GetAmType() == TYPE_AM_ROWID && !((RIDBLK*)cp)->GetRnm()) + return NULL; + + } // endif blk + + int i, op = filp->GetOpc(), opm = filp->GetOpm(); + bool cnv[2]; + PCOL colp; + PXOB arg[2] = {NULL,NULL}; + PBF *fp = NULL, bfp = NULL; + + switch (op) { + case OP_EQ: + case OP_NE: + case OP_GT: + case OP_GE: + case OP_LT: + case OP_LE: + if (! opm) { + for (i = 0; i < 2; i++) { + arg[i] = filp->Arg(i); + cnv[i] = filp->Conv(i); + } // endfor i + + bfp = CheckBlockFilari(g, arg, op, cnv); + break; + } // endif !opm + + // if opm, pass thru + // fall through + case OP_IN: + if (filp->GetArgType(0) == TYPE_COLBLK && + filp->GetArgType(1) == TYPE_ARRAY) { + arg[0] = filp->Arg(0); + arg[1] = filp->Arg(1); + colp = (PCOL)arg[0]; + + if (colp->GetTo_Tdb() == this) { + // Block evaluation is possible for... + if (colp->GetAmType() == TYPE_AM_ROWID) { + // Special column ROWID and constant array, but + // currently we don't know how to retrieve a RowID + // from a DBF table that is not sequentially read. +// if (Txfp->GetAmType() != TYPE_AM_DBF || +// ((RIDBLK*)arg[0])->GetRnm()) + bfp = new(g) BLKSPCIN(g, this, op, opm, arg, Txfp->Nrec); + + } else if (blk && Txfp->Nrec > 1 && colp->IsClustered()) + { + // Clustered column and constant array + if (colp->GetClustered() == 2) + bfp = new(g) BLKFILIN2(g, this, op, opm, arg); + else + bfp = new(g) BLKFILIN(g, this, op, opm, arg); + } + } // endif this + +#if 0 + } else if (filp->GetArgType(0) == TYPE_SCALF && + filp->GetArgType(1) == TYPE_ARRAY) { + arg[0] = filp->Arg(0); + arg[1] = filp->Arg(1); + + if (((PSCALF)arg[0])->GetOp() == OP_ROW && + arg[1]->GetResultType() == TYPE_LIST) { + PARRAY par = (PARRAY)arg[1]; + LSTVAL *vlp = (LSTVAL*)par->GetValue(); + + ((SFROW*)arg[0])->GetParms(n); + + if (n != vlp->GetN()) + return NULL; + else + n = par->GetNval(); + + arg[1] = new(g) CONSTANT(vlp); + fp = (PBF*)PlugSubAlloc(g, NULL, n * sizeof(PBF)); + cnv[0] = cnv[1] = false; + + if (op == OP_IN) + op = OP_EQ; + + for (i = 0; i < n; i++) { + par->GetNthValue(vlp, i); + + if (!(fp[i] = CheckBlockFilari(g, arg, op, cnv))) + return NULL; + + } // endfor i + + bfp = new(g) BLKFILLOG(this, (opm == 2 ? OP_AND : OP_OR), fp, n); + } // endif ROW +#endif // 0 + + } // endif Type + + break; + case OP_AND: + case OP_OR: + fp = (PBF*)PlugSubAlloc(g, NULL, 2 * sizeof(PBF)); + fp[0] = InitBlockFilter(g, (PFIL)(filp->Arg(0))); + fp[1] = InitBlockFilter(g, (PFIL)(filp->Arg(1))); + + if (fp[0] || fp[1]) + bfp = new(g) BLKFILLOG(this, op, fp, 2); + + break; + case OP_NOT: + fp = (PBF*)PlugSubAlloc(g, NULL, sizeof(PBF)); + + if ((*fp = InitBlockFilter(g, (PFIL)(filp->Arg(0))))) + bfp = new(g) BLKFILLOG(this, op, fp, 1); + + break; + case OP_LIKE: + default: + break; + } // endswitch op + + return bfp; + } // end of InitBlockFilter + +/***********************************************************************/ +/* Analyze the passed arguments and construct the Block Filter. */ +/***********************************************************************/ +PBF TDBDOS::CheckBlockFilari(PGLOBAL g, PXOB *arg, int op, bool *cnv) + { +//int i, n1, n2, ctype = TYPE_ERROR, n = 0, type[2] = {0,0}; +//bool conv = false, xdb2 = false, ok = false, b[2]; +//PXOB *xarg1, *xarg2 = NULL, xp[2]; + int i, n = 0, type[2] = {0,0}; + bool conv = false, xdb2 = false; + PXOB xp[2]; + PCOL colp; + PBF bfp = NULL; + + for (i = 0; i < 2; i++) { + switch (arg[i]->GetType()) { + case TYPE_CONST: + type[i] = 1; + // ctype = arg[i]->GetResultType(); + break; + case TYPE_COLBLK: + conv = cnv[i]; + colp = (PCOL)arg[i]; + + if (colp->GetTo_Tdb() == this) { + if (colp->GetAmType() == TYPE_AM_ROWID) { + // Currently we don't know how to retrieve a RowID + // from a DBF table that is not sequentially read. +// if (Txfp->GetAmType() != TYPE_AM_DBF || +// ((RIDBLK*)arg[i])->GetRnm()) + type[i] = 5; + + } else if (Txfp->Blocked && Txfp->Nrec > 1 && + colp->IsClustered()) { + type[i] = 2; + xdb2 = colp->GetClustered() == 2; + } // endif Clustered + + } else if (colp->GetColUse(U_CORREL)) { + // This is a column pointing to the outer query of a + // correlated subquery, it has a constant value during + // each execution of the subquery. + type[i] = 1; +// ctype = arg[i]->GetResultType(); + } // endif this + + break; +// case TYPE_SCALF: +// if (((PSCALF)arg[i])->GetOp() == OP_ROW) { +// sfr[i] = (SFROW*)arg[i]; +// type[i] = 7; +// } // endif Op + +// break; + default: + break; + } // endswitch ArgType + + if (!type[i]) + break; + + n += type[i]; + } // endfor i + + if (n == 3 || n == 6) { + if (conv) { + // The constant has not the good type and will not match + // the block min/max values. Warn and abort. + snprintf(g->Message, sizeof(g->Message), "Block opt: %s", MSG(VALTYPE_NOMATCH)); + PushWarning(g, this); + return NULL; + } // endif Conv + + if (type[0] == 1) { + // Make it always as Column-op-Value + *xp = arg[0]; + arg[0] = arg[1]; + arg[1] = *xp; + + switch (op) { + case OP_GT: op = OP_LT; break; + case OP_GE: op = OP_LE; break; + case OP_LT: op = OP_GT; break; + case OP_LE: op = OP_GE; break; + } // endswitch op + + } // endif + +#if defined(_DEBUG) +// assert(arg[0]->GetResultType() == ctype); +#endif + + if (n == 3) { + if (xdb2) { + if (((PDOSCOL)arg[0])->GetNbm() == 1) + bfp = new(g) BLKFILAR2(g, this, op, arg); + else // Multiple bitmap made of several ULONG's + bfp = new(g) BLKFILMR2(g, this, op, arg); + } else + bfp = new(g) BLKFILARI(g, this, op, arg); + + } else // n = 6 + bfp = new(g) BLKSPCARI(this, op, arg, Txfp->Nrec); + +#if 0 + } else if (n == 8 || n == 14) { + if (n == 8 && ctype != TYPE_LIST) { + // Should never happen + safe_strcpy(g->Message, sizeof(g->Message), "Block opt: bad constant"); + throw 99; + } // endif Conv + + if (type[0] == 1) { + // Make it always as Column-op-Value + sfr[0] = sfr[1]; + arg[1] = arg[0]; + + switch (op) { + case OP_GT: op = OP_LT; break; + case OP_GE: op = OP_LE; break; + case OP_LT: op = OP_GT; break; + case OP_LE: op = OP_GE; break; + } // endswitch op + + } // endif + + xarg1 = sfr[0]->GetParms(n1); + + if (n == 8) { + vlp = (LSTVAL*)arg[1]->GetValue(); + n2 = vlp->GetN(); + xp[1] = new(g) CONSTANT((PVAL)NULL); + } else + xarg2 = sfr[1]->GetParms(n2); + + if (n1 != n2) + return NULL; // Should we flag an error ? + + fp = (PBF*)PlugSubAlloc(g, NULL, n1 * sizeof(PBF)); + + for (i = 0; i < n1; i++) { + xp[0] = xarg1[i]; + + if (n == 8) + ((CONSTANT*)xp[1])->SetValue(vlp->GetSubVal(i)); + else + xp[1] = xarg2[i]; + + b[0] = b[1] = (xp[0]->GetResultType() != xp[1]->GetResultType()); + ok |= ((fp[i] = CheckBlockFilari(g, xp, op, b)) != NULL); + } // endfor i + + if (ok) + bfp = new(g) BLKFILLOG(this, OP_AND, fp, n1); +#endif // 0 + + } // endif n + + return bfp; + } // end of CheckBlockFilari + +/***********************************************************************/ +/* ResetBlkFil: reset the block filter and restore filtering, or make */ +/* the block filter if To_Filter was not set when opening the table. */ +/***********************************************************************/ +void TDBDOS::ResetBlockFilter(PGLOBAL g) + { + if (!To_BlkFil) { + if (To_Filter) + if ((To_BlkFil = InitBlockFilter(g, To_Filter))) { + htrc("BlkFil=%p\n", To_BlkFil); + MaxSize = -1; // To be recalculated + } // endif To_BlkFil + + return; + } // endif To_BlkFil + + To_BlkFil->Reset(g); + + if (SavFil && !To_Filter) { + // Restore filter if it was disabled by optimization + To_Filter = SavFil; + SavFil = NULL; + } // endif + + Beval = 0; + } // end of ResetBlockFilter + +/***********************************************************************/ +/* Block optimization: evaluate the block index filter against */ +/* the min and max values of this block and return: */ +/* RC_OK: if some records in the block can meet filter criteria. */ +/* RC_NF: if no record in the block can meet filter criteria. */ +/* RC_EF: if no record in the remaining file can meet filter criteria.*/ +/* In addition, temporarily supress filtering if all the records in */ +/* the block meet filter criteria. */ +/***********************************************************************/ +int TDBDOS::TestBlock(PGLOBAL g) + { + int rc = RC_OK; + + if (To_BlkFil && Beval != 2) { + // Check for block filtering evaluation + if (Beval == 1) { + // Filter was removed for last block, restore it + To_Filter = SavFil; + SavFil = NULL; + } // endif Beval + + // Check for valid records in new block + switch (Beval = To_BlkFil->BlockEval(g)) { + case -2: // No more valid values in file + rc = RC_EF; + break; + case -1: // No valid values in block + rc = RC_NF; + break; + case 1: // All block values are valid + case 2: // All subsequent file values are Ok + // Before suppressing the filter for the block(s) it is + // necessary to reset the filtered columns to NOT_READ + // so their new values are retrieved by the SELECT list. + if (To_Filter) // Can be NULL when externally called (XDB) + To_Filter->Reset(); + + SavFil = To_Filter; + To_Filter = NULL; // So remove filter + } // endswitch Beval + + if (trace(1)) + htrc("BF Eval Beval=%d\n", Beval); + + } // endif To_BlkFil + + return rc; + } // end of TestBlock + +/***********************************************************************/ +/* Check whether we have to create/update permanent indexes. */ +/***********************************************************************/ +int TDBDOS::MakeIndex(PGLOBAL g, PIXDEF pxdf, bool add) + { + int k, n, rc = RC_OK; + bool fixed, doit, sep; + PCOL *keycols, colp; + PIXDEF xdp, sxp = NULL; + PKPDEF kdp; + PDOSDEF dfp; +//PCOLDEF cdp; + PXINDEX x; + PXLOAD pxp; + + Mode = MODE_READ; + Use = USE_READY; + dfp = (PDOSDEF)To_Def; + + if (!Cardinality(g)) { + // Void table erase eventual index file(s) + (void)dfp->DeleteIndexFile(g, NULL); + return RC_OK; + } else + fixed = Ftype != RECFM_VAR; + + // Are we are called from CreateTable or CreateIndex? + if (pxdf) { + if (!add && dfp->GetIndx()) { + safe_strcpy(g->Message, sizeof(g->Message), MSG(INDX_EXIST_YET)); + return RC_FX; + } // endif To_Indx + + if (add && dfp->GetIndx()) { + for (sxp = dfp->GetIndx(); sxp; sxp = sxp->GetNext()) + if (!stricmp(sxp->GetName(), pxdf->GetName())) { + snprintf(g->Message, sizeof(g->Message), MSG(INDEX_YET_ON), pxdf->GetName(), Name); + return RC_FX; + } else if (!sxp->GetNext()) + break; + + sxp->SetNext(pxdf); +// first = false; + } else + dfp->SetIndx(pxdf); + +// pxdf->SetDef(dfp); + } else if (!(pxdf = dfp->GetIndx())) + return RC_INFO; // No index to make + + try { + // Allocate all columns that will be used by indexes. + // This must be done before opening the table so specific + // column initialization can be done (in particular by TDBVCT) + for (n = 0, xdp = pxdf; xdp; xdp = xdp->GetNext()) + for (kdp = xdp->GetToKeyParts(); kdp; kdp = kdp->GetNext()) { + if (!(colp = ColDB(g, kdp->GetName(), 0))) { + snprintf(g->Message, sizeof(g->Message), MSG(INDX_COL_NOTIN), kdp->GetName(), Name); + goto err; + } else if (colp->GetResultType() == TYPE_DECIM) { + snprintf(g->Message, sizeof(g->Message), "Decimal columns are not indexable yet"); + goto err; + } // endif Type + + colp->InitValue(g); + n = MY_MAX(n, xdp->GetNparts()); + } // endfor kdp + + keycols = (PCOL*)PlugSubAlloc(g, NULL, n * sizeof(PCOL)); + sep = dfp->GetBoolCatInfo("SepIndex", false); + + /*********************************************************************/ + /* Construct and save the defined indexes. */ + /*********************************************************************/ + for (xdp = pxdf; xdp; xdp = xdp->GetNext()) + if (!OpenDB(g)) { + if (xdp->IsAuto() && fixed) + // Auto increment key and fixed file: use an XXROW index + continue; // XXROW index doesn't need to be made + + // On Update, redo only indexes that are modified + doit = !To_SetCols; + n = 0; + + if (sxp) + xdp->SetID(sxp->GetID() + 1); + + for (kdp = xdp->GetToKeyParts(); kdp; kdp = kdp->GetNext()) { + // Check whether this column was updated + for (colp = To_SetCols; !doit && colp; colp = colp->GetNext()) + if (!stricmp(kdp->GetName(), colp->GetName())) + doit = true; + + keycols[n++] = ColDB(g, kdp->GetName(), 0); + } // endfor kdp + + // If no indexed columns were updated, don't remake the index + // if indexes are in separate files. + if (!doit && sep) + continue; + + k = xdp->GetNparts(); + + // Make the index and save it + if (dfp->Huge) + pxp = new(g) XHUGE; + else + pxp = new(g) XFILE; + + if (k == 1) // Simple index + x = new(g) XINDXS(this, xdp, pxp, keycols); + else // Multi-Column index + x = new(g) XINDEX(this, xdp, pxp, keycols); + + if (!x->Make(g, sxp)) { + // Retreive define values from the index + xdp->SetMaxSame(x->GetMaxSame()); + // xdp->SetSize(x->GetSize()); + + // store KXYCOL Mxs in KPARTDEF Mxsame + xdp->SetMxsame(x); + +#if defined(TRACE) + printf("Make done...\n"); +#endif // TRACE + + // if (x->GetSize() > 0) + sxp = xdp; + + xdp->SetInvalid(false); + } else + goto err; + + } else + return RC_INFO; // Error or Physical table does not exist + + } catch (int n) { + if (trace(1)) + htrc("Exception %d: %s\n", n, g->Message); + rc = RC_FX; + } catch (const char *msg) { + safe_strcpy(g->Message, sizeof(g->Message), msg); + rc = RC_FX; + } // end catch + + if (Use == USE_OPEN) + CloseDB(g); + + return rc; + +err: + if (sxp) + sxp->SetNext(NULL); + else + dfp->SetIndx(NULL); + + return RC_FX; + } // end of MakeIndex + +/***********************************************************************/ +/* Make a dynamic index. */ +/***********************************************************************/ +bool TDBDOS::InitialyzeIndex(PGLOBAL g, volatile PIXDEF xdp, bool sorted) +{ + int k; + volatile bool dynamic; + bool brc; + PCOL colp; + PCOLDEF cdp; + PVAL valp; + PXLOAD pxp; + volatile PKXBASE kxp; + PKPDEF kdp; + + if (!xdp && !(xdp = To_Xdp)) { + safe_strcpy(g->Message, sizeof(g->Message), "NULL dynamic index"); + return true; + } else + dynamic = To_Filter && xdp->IsUnique() && xdp->IsDynamic(); +// dynamic = To_Filter && xdp->IsDynamic(); NIY + + // Allocate the key columns definition block + Knum = xdp->GetNparts(); + To_Key_Col = (PCOL*)PlugSubAlloc(g, NULL, Knum * sizeof(PCOL)); + + // Get the key column description list + for (k = 0, kdp = xdp->GetToKeyParts(); kdp; kdp = kdp->GetNext()) + if (!(colp = ColDB(g, kdp->GetName(), 0)) || colp->InitValue(g)) { + snprintf(g->Message, sizeof(g->Message), "Wrong column %s", kdp->GetName()); + return true; + } else + To_Key_Col[k++] = colp; + +#if defined(_DEBUG) + if (k != Knum) { + snprintf(g->Message, sizeof(g->Message), "Key part number mismatch for %s", + xdp->GetName()); + return 0; + } // endif k +#endif // _DEBUG + + // Allocate the pseudo constants that will contain the key values + To_Link = (PXOB*)PlugSubAlloc(g, NULL, Knum * sizeof(PXOB)); + + for (k = 0, kdp = xdp->GetToKeyParts(); kdp; k++, kdp = kdp->GetNext()) { + if ((cdp = Key(k)->GetCdp())) + valp = AllocateValue(g, cdp->GetType(), cdp->GetLength()); + else { // Special column ? + colp = Key(k); + valp = AllocateValue(g, colp->GetResultType(), colp->GetLength()); + } // endif cdp + + To_Link[k]= new(g) CONSTANT(valp); + } // endfor k + + // Make the index on xdp + if (!xdp->IsAuto()) { + if (!dynamic) { + if (((PDOSDEF)To_Def)->Huge) + pxp = new(g) XHUGE; + else + pxp = new(g) XFILE; + + } else + pxp = NULL; + + if (Knum == 1) // Single index + kxp = new(g) XINDXS(this, xdp, pxp, To_Key_Col, To_Link); + else // Multi-Column index + kxp = new(g) XINDEX(this, xdp, pxp, To_Key_Col, To_Link); + + } else // Column contains same values as ROWID + kxp = new(g) XXROW(this); + + try { + if (dynamic) { + ResetBlockFilter(g); + kxp->SetDynamic(dynamic); + brc = kxp->Make(g, xdp); + } else + brc = kxp->Init(g); + + if (!brc) { + if (Txfp->GetAmType() == TYPE_AM_BLK) { + // Cannot use indexing in DOS block mode + Txfp = new(g) DOSFAM((PBLKFAM)Txfp, (PDOSDEF)To_Def); + Txfp->AllocateBuffer(g); + To_BlkFil = NULL; + } // endif AmType + + To_Kindex= kxp; + + if (!(sorted && To_Kindex->IsSorted()) && + ((Mode == MODE_UPDATE && IsUsingTemp(g)) || + (Mode == MODE_DELETE && Txfp->GetAmType() != TYPE_AM_DBF))) + Indxd = true; + + } // endif brc + + } catch (int n) { + if (trace(1)) + htrc("Exception %d: %s\n", n, g->Message); + brc = true; + } catch (const char *msg) { + safe_strcpy(g->Message, sizeof(g->Message), msg); + brc = true; + } // end catch + + return brc; +} // end of InitialyzeIndex + +/***********************************************************************/ +/* DOS GetProgMax: get the max value for progress information. */ +/***********************************************************************/ +int TDBDOS::GetProgMax(PGLOBAL g) + { + return (To_Kindex) ? GetMaxSize(g) : GetFileLength(g); + } // end of GetProgMax + +/***********************************************************************/ +/* DOS GetProgCur: get the current value for progress information. */ +/***********************************************************************/ +int TDBDOS::GetProgCur(void) + { + return (To_Kindex) ? To_Kindex->GetCur_K() + 1 : GetRecpos(); + } // end of GetProgCur + +/***********************************************************************/ +/* RowNumber: return the ordinal number of the current row. */ +/***********************************************************************/ +int TDBDOS::RowNumber(PGLOBAL g, bool) + { + if (To_Kindex) { + /*******************************************************************/ + /* Don't know how to retrieve RowID from file address. */ + /*******************************************************************/ + snprintf(g->Message, sizeof(g->Message), MSG(NO_ROWID_FOR_AM), + GetAmName(g, Txfp->GetAmType())); + return 0; + } else + return Txfp->GetRowID(); + + } // end of RowNumber + +/***********************************************************************/ +/* DOS Cardinality: returns table cardinality in number of rows. */ +/* This function can be called with a null argument to test the */ +/* availability of Cardinality implementation (1 yes, 0 no). */ +/***********************************************************************/ +int TDBDOS::Cardinality(PGLOBAL g) + { + int n = Txfp->Cardinality(NULL); + + if (!g) + return (Mode == MODE_ANY) ? 1 : n; + + if (Cardinal < 0) { + if (!Txfp->Blocked && n == 0) { + // Info command, we try to return exact row number + PDOSDEF dfp = (PDOSDEF)To_Def; + PIXDEF xdp = dfp->To_Indx; + + if (xdp && xdp->IsValid()) { + // Cardinality can be retreived from one index + PXLOAD pxp; + + if (dfp->Huge) + pxp = new(g) XHUGE; + else + pxp = new(g) XFILE; + + PXINDEX kxp = new(g) XINDEX(this, xdp, pxp, NULL, NULL); + + if (!(kxp->GetAllSizes(g, Cardinal))) + return Cardinal; + + } // endif Mode + + if (Mode == MODE_ANY && ExactInfo()) { + // Using index impossible or failed, do it the hard way + Mode = MODE_READ; + To_Line = (char*)PlugSubAlloc(g, NULL, (size_t)Lrecl + 1); + + if (Txfp->OpenTableFile(g)) + return (Cardinal = Txfp->Cardinality(g)); + + for (Cardinal = 0; n != RC_EF;) + if (!(n = Txfp->ReadBuffer(g))) + Cardinal++; + + Txfp->CloseTableFile(g, false); + Mode = MODE_ANY; + } else { + // Return the best estimate + int len = GetFileLength(g); + + if (len >= 0) { + int rec; + + if (trace(1)) + htrc("Estimating lines len=%d ending=%d/n", + len, ((PDOSDEF)To_Def)->Ending); + + /*************************************************************/ + /* Estimate the number of lines in the table (if not known) */ + /* by dividing the file length by the average record length. */ + /*************************************************************/ + rec = ((PDOSDEF)To_Def)->Ending; + + if (AvgLen <= 0) // No given average estimate + rec += EstimatedLength(); + else // An estimate was given for the average record length + rec += AvgLen; + + Cardinal = (len + rec - 1) / rec; + + if (trace(1)) + htrc("avglen=%d MaxSize%d\n", rec, Cardinal); + + } // endif len + + } // endif Mode + + } else + Cardinal = Txfp->Cardinality(g); + + } // endif Cardinal + + return Cardinal; + } // end of Cardinality + +/***********************************************************************/ +/* DOS GetMaxSize: returns file size estimate in number of lines. */ +/* This function covers variable record length files. */ +/***********************************************************************/ +int TDBDOS::GetMaxSize(PGLOBAL g) + { + if (MaxSize >= 0) + return MaxSize; + + if (!Cardinality(NULL)) { + int len = GetFileLength(g); + + if (len >= 0) { + int rec; + + if (trace(1)) + htrc("Estimating lines len=%d ending=%d/n", + len, ((PDOSDEF)To_Def)->Ending); + + /*****************************************************************/ + /* Estimate the number of lines in the table (if not known) by */ + /* dividing the file length by minimum record length. */ + /*****************************************************************/ + rec = EstimatedLength() + ((PDOSDEF)To_Def)->Ending; + MaxSize = (len + rec - 1) / rec; + + if (trace(1)) + htrc("avglen=%d MaxSize%d\n", rec, MaxSize); + + } // endif len + + } else + MaxSize = Cardinality(g); + + return MaxSize; + } // end of GetMaxSize + +/***********************************************************************/ +/* DOS EstimatedLength. Returns an estimated minimum line length. */ +/***********************************************************************/ +int TDBDOS::EstimatedLength(void) + { + int dep = 0; + PCOLDEF cdp = To_Def->GetCols(); + + if (!cdp->GetNext()) { + // One column table, we are going to return a ridiculous + // result if we set dep to 1 + dep = 1 + cdp->GetLong() / 20; // Why 20 ????? + } else for (; cdp; cdp = cdp->GetNext()) + if (!(cdp->Flags & (U_VIRTUAL|U_SPECIAL))) + dep = MY_MAX(dep, cdp->GetOffset()); + + return (int)dep; + } // end of Estimated Length + +/***********************************************************************/ +/* DOS tables favor the use temporary files for Update. */ +/***********************************************************************/ +bool TDBDOS::IsUsingTemp(PGLOBAL) + { + USETEMP utp = UseTemp(); + + return (utp == TMP_YES || utp == TMP_FORCE || + (utp == TMP_AUTO && Mode == MODE_UPDATE)); + } // end of IsUsingTemp + +/***********************************************************************/ +/* DOS Access Method opening routine. */ +/* New method now that this routine is called recursively (last table */ +/* first in reverse order): index blocks are immediately linked to */ +/* join block of next table if it exists or else are discarted. */ +/***********************************************************************/ +bool TDBDOS::OpenDB(PGLOBAL g) + { + if (trace(1)) + htrc("DOS OpenDB: tdbp=%p tdb=R%d use=%d mode=%d\n", + this, Tdb_No, Use, Mode); + + if (Use == USE_OPEN) { + /*******************************************************************/ + /* Table already open, just replace it at its beginning. */ + /*******************************************************************/ + if (!To_Kindex) { + Txfp->Rewind(); // see comment in Work.log + + if (SkipHeader(g)) + return true; + + } else + /*****************************************************************/ + /* Table is to be accessed through a sorted index table. */ + /*****************************************************************/ + To_Kindex->Reset(); + + ResetBlockFilter(g); + return false; + } // endif use + + if (Mode == MODE_DELETE && !Next && Txfp->GetAmType() != TYPE_AM_DOS +#if defined(BSON_SUPPORT) + && Txfp->GetAmType() != TYPE_AM_BIN +#endif // BSON_SUPPORT + && Txfp->GetAmType() != TYPE_AM_MGO) { + // Delete all lines. Not handled in MAP or block mode + Txfp = new(g) DOSFAM((PDOSDEF)To_Def); + Txfp->SetTdbp(this); + } else if (Txfp->Blocked && (Mode == MODE_DELETE || + (Mode == MODE_UPDATE && UseTemp() != TMP_NO))) { + /*******************************************************************/ + /* Delete is not currently handled in block mode neither Update */ + /* when using a temporary file. */ + /*******************************************************************/ + if (Txfp->GetAmType() == TYPE_AM_MAP && Mode == MODE_DELETE) + Txfp = new(g) MAPFAM((PDOSDEF)To_Def); +#if defined(GZ_SUPPORT) + else if (Txfp->GetAmType() == TYPE_AM_GZ) + Txfp = new(g) GZFAM((PDOSDEF)To_Def); +#endif // GZ_SUPPORT + else // if (Txfp->GetAmType() != TYPE_AM_DOS) ??? + Txfp = new(g) DOSFAM((PDOSDEF)To_Def); + + Txfp->SetTdbp(this); + } // endif Mode + + /*********************************************************************/ + /* Open according to logical input/output mode required. */ + /* Use conventionnal input/output functions. */ + /* Treat files as binary in Delete mode (for line moving) */ + /*********************************************************************/ + if (Txfp->OpenTableFile(g)) + return true; + + Use = USE_OPEN; // Do it now in case we are recursively called + + /*********************************************************************/ + /* Allocate the block filter tree if evaluation is possible. */ + /*********************************************************************/ + To_BlkFil = InitBlockFilter(g, To_Filter); + + /*********************************************************************/ + /* Lrecl does not include line ending */ + /*********************************************************************/ + size_t linelen = Lrecl + ((PDOSDEF)To_Def)->Ending + 1; + + To_Line = (char*)PlugSubAlloc(g, NULL, linelen); + + if (Mode == MODE_INSERT) { + // Spaces between fields must be filled with blanks + memset(To_Line, ' ', Lrecl); + To_Line[Lrecl] = '\0'; + } else + memset(To_Line, 0, linelen); + + if (trace(1)) + htrc("OpenDos: R%hd mode=%d To_Line=%p\n", Tdb_No, Mode, To_Line); + + if (SkipHeader(g)) // When called from CSV/FMT files + return true; + + /*********************************************************************/ + /* Reset statistics values. */ + /*********************************************************************/ + num_read = num_there = num_eq[0] = num_eq[1] = 0; + return false; + } // end of OpenDB + +/***********************************************************************/ +/* ReadDB: Data Base read routine for DOS access method. */ +/***********************************************************************/ +int TDBDOS::ReadDB(PGLOBAL g) + { + if (trace(2)) + htrc("DOS ReadDB: R%d Mode=%d key=%p link=%p Kindex=%p To_Line=%p\n", + GetTdb_No(), Mode, To_Key_Col, To_Link, To_Kindex, To_Line); + + if (To_Kindex) { + /*******************************************************************/ + /* Reading is by an index table. */ + /*******************************************************************/ + int recpos = To_Kindex->Fetch(g); + + switch (recpos) { + case -1: // End of file reached + return RC_EF; + case -2: // No match for join + return RC_NF; + case -3: // Same record as non null last one + num_there++; + return RC_OK; + default: + /***************************************************************/ + /* Set the file position according to record to read. */ + /***************************************************************/ + if (SetRecpos(g, recpos)) + return RC_FX; + + if (trace(2)) + htrc("File position is now %d\n", GetRecpos()); + + if (Mode == MODE_READ) + /*************************************************************/ + /* Defer physical reading until one column setting needs it */ + /* as it can be a big saving on joins where no other column */ + /* than the keys are used, so reading is unnecessary. */ + /*************************************************************/ + if (Txfp->DeferReading()) + return RC_OK; + + } // endswitch recpos + + } // endif To_Kindex + + if (trace(2)) + htrc(" ReadDB: this=%p To_Line=%p\n", this, To_Line); + + /*********************************************************************/ + /* Now start the reading process. */ + /*********************************************************************/ + return ReadBuffer(g); + } // end of ReadDB + +/***********************************************************************/ +/* PrepareWriting: Prepare the line to write. */ +/***********************************************************************/ +bool TDBDOS::PrepareWriting(PGLOBAL) + { + if (Ftype == RECFM_VAR && (Mode == MODE_INSERT || Txfp->GetUseTemp())) { + char *p; + + /*******************************************************************/ + /* Suppress trailing blanks. */ + /* Also suppress eventual null from last line. */ + /*******************************************************************/ + for (p = To_Line + Lrecl -1; p >= To_Line; p--) + if (*p && *p != ' ') + break; + + *(++p) = '\0'; + } // endif Mode + + return false; + } // end of PrepareWriting + +/***********************************************************************/ +/* WriteDB: Data Base write routine for DOS access method. */ +/***********************************************************************/ +int TDBDOS::WriteDB(PGLOBAL g) + { + if (trace(2)) + htrc("DOS WriteDB: R%d Mode=%d \n", Tdb_No, Mode); + + // Make the line to write + if (PrepareWriting(g)) + return RC_FX; + + if (trace(2)) + htrc("Write: line is='%s'\n", To_Line); + + // Now start the writing process + return Txfp->WriteBuffer(g); + } // end of WriteDB + +/***********************************************************************/ +/* Data Base delete line routine for DOS (and FIX) access method. */ +/* RC_FX means delete all. Nothing to do here (was done at open). */ +/***********************************************************************/ +int TDBDOS::DeleteDB(PGLOBAL g, int irc) + { + return (irc == RC_FX) ? RC_OK : Txfp->DeleteRecords(g, irc); + } // end of DeleteDB + +/***********************************************************************/ +/* Data Base close routine for DOS access method. */ +/***********************************************************************/ +void TDBDOS::CloseDB(PGLOBAL g) + { + if (To_Kindex) { + To_Kindex->Close(); + To_Kindex = NULL; + } // endif + + Txfp->CloseTableFile(g, Abort); + RestoreNrec(); + } // end of CloseDB + +// ------------------------ DOSCOL functions ---------------------------- + +/***********************************************************************/ +/* DOSCOL public constructor (also called by MAPCOL). */ +/***********************************************************************/ +DOSCOL::DOSCOL(PGLOBAL g, PCOLDEF cdp, PTDB tp, PCOL cp, int i, PCSZ am) + : COLBLK(cdp, tp, i) + { + char *p; + int prec = Format.Prec; + PTXF txfp = ((PTDBDOS)tp)->Txfp; + + assert(cdp); + + if (cp) { + Next = cp->GetNext(); + cp->SetNext(this); + } else { + Next = tp->GetColumns(); + tp->SetColumns(this); + } // endif cprec + + // Set additional Dos access method information for column. + Deplac = cdp->GetOffset(); + Long = cdp->GetLong(); + To_Val = NULL; + Clustered = cdp->GetOpt(); + Sorted = (cdp->GetOpt() == 2) ? 1 : 0; + Ndv = 0; // Currently used only for XDB2 + Nbm = 0; // Currently used only for XDB2 + Min = NULL; + Max = NULL; + Bmap = NULL; + Dval = NULL; + Buf = NULL; + + if (txfp && txfp->Blocked && Opt && (cdp->GetMin() || cdp->GetDval())) { + int nblk = txfp->GetBlock(); + + Clustered = (cdp->GetXdb2()) ? 2 : 1; + Sorted = (cdp->GetOpt() > 1) ? 1 : 0; // Currently ascending only + + if (Clustered == 1) { + Min = AllocValBlock(g, cdp->GetMin(), Buf_Type, nblk, Long, prec); + Max = AllocValBlock(g, cdp->GetMax(), Buf_Type, nblk, Long, prec); + } else { // Clustered == 2 + // Ndv is the number of distinct values in Dval. Ndv and Nbm + // may be 0 when optimizing because Ndval is not filled yet, + // but the size of the passed Dval memory block is Ok. + Ndv = cdp->GetNdv(); + Dval = AllocValBlock(g, cdp->GetDval(), Buf_Type, Ndv, Long, prec); + + // Bmap cannot be allocated when optimizing, we must know Nbm first + if ((Nbm = cdp->GetNbm())) + Bmap = AllocValBlock(g, cdp->GetBmap(), TYPE_INT, Nbm * nblk); + + } // endif Clustered + + } // endif Opt + + OldVal = NULL; // Currently used only in MinMax + Dsp = 0; + Ldz = false; + Nod = false; + Dcm = -1; + p = cdp->GetFmt(); + Buf = NULL; + + if (p && IsTypeNum(Buf_Type)) { + // Formatted numeric value + for (; p && *p && isalpha(*p); p++) + switch (toupper(*p)) { + case 'Z': // Have leading zeros + Ldz = true; + break; + case 'N': // Have no decimal point + Nod = true; + break; + case 'D': // Decimal separator + Dsp = *(++p); + break; + } // endswitch p + + // Set number of decimal digits + Dcm = (*p) ? atoi(p) : GetScale(); + } // endif fmt + + if (trace(1)) + htrc(" making new %sCOL C%d %s at %p\n", am, Index, Name, this); + + } // end of DOSCOL constructor + +/***********************************************************************/ +/* DOSCOL constructor used for copying columns. */ +/* tdbp is the pointer to the new table descriptor. */ +/***********************************************************************/ +DOSCOL::DOSCOL(DOSCOL *col1, PTDB tdbp) : COLBLK(col1, tdbp) + { + Deplac = col1->Deplac; + Long = col1->Long; + To_Val = col1->To_Val; + Ldz = col1->Ldz; + Dsp = col1->Dsp; + Nod = col1->Nod; + Dcm = col1->Dcm; + OldVal = col1->OldVal; + Buf = col1->Buf; + Clustered = col1->Clustered; + Sorted = col1->Sorted; + Min = col1->Min; + Max = col1->Max; + Bmap = col1->Bmap; + Dval = col1->Dval; + Ndv = col1->Ndv; + Nbm = col1->Nbm; + } // end of DOSCOL copy constructor + +/***********************************************************************/ +/* VarSize: This function tells UpdateDB whether or not the block */ +/* optimization file must be redone if this column is updated, even */ +/* it is not sorted or clustered. This applies to the last column of */ +/* a variable length table that is blocked, because if it is updated */ +/* using a temporary file, the block size may be modified. */ +/***********************************************************************/ +bool DOSCOL::VarSize(void) + { + PTDBDOS tdbp = (PTDBDOS)To_Tdb; + PTXF txfp = tdbp->Txfp; + + if (Cdp && !Cdp->GetNext() // Must be the last column + && tdbp->Ftype == RECFM_VAR // of a DOS variable length + && txfp->Blocked // blocked table + && txfp->GetUseTemp()) // using a temporary file. + return true; + else + return false; + + } // end VarSize + +/***********************************************************************/ +/* SetBuffer: prepare a column block for write operation. */ +/***********************************************************************/ +bool DOSCOL::SetBuffer(PGLOBAL g, PVAL value, bool ok, bool check) + { + if (!(To_Val = value)) { + snprintf(g->Message, sizeof(g->Message), MSG(VALUE_ERROR), Name); + return true; + } else if (Buf_Type == value->GetType()) { + // Values are of the (good) column type + if (Buf_Type == TYPE_DATE) { + // If any of the date values is formatted + // output format must be set for the receiving table + if (GetDomain() || ((DTVAL *)value)->IsFormatted()) + goto newval; // This will make a new value; + + } else if (Buf_Type == TYPE_DOUBLE) + // Float values must be written with the correct (column) precision + // Note: maybe this should be forced by ShowValue instead of this ? + value->SetPrec(GetScale()); + + Value = value; // Directly access the external value + } else { + // Values are not of the (good) column type + if (check) { + snprintf(g->Message, sizeof(g->Message), MSG(TYPE_VALUE_ERR), Name, + GetTypeName(Buf_Type), GetTypeName(value->GetType())); + return true; + } // endif check + + newval: + if (InitValue(g)) // Allocate the matching value block + return true; + + } // endif's Value, Buf_Type + + // Allocate the buffer used in WriteColumn for numeric columns + if (!Buf && IsTypeNum(Buf_Type)) + Buf = (char*)PlugSubAlloc(g, NULL, MY_MAX(64, Long + 1)); + else // Text columns do not need additional buffer + Buf = (char*)Value->GetTo_Val(); + + // Because Colblk's have been made from a copy of the original TDB in + // case of Update, we must reset them to point to the original one. + if (To_Tdb->GetOrig()) + To_Tdb = (PTDB)To_Tdb->GetOrig(); + + // Set the Column + Status = (ok) ? BUF_EMPTY : BUF_NO; + return false; + } // end of SetBuffer + +/***********************************************************************/ +/* ReadColumn: what this routine does is to access the last line */ +/* read from the corresponding table, extract from it the field */ +/* corresponding to this column and convert it to buffer type. */ +/***********************************************************************/ +void DOSCOL::ReadColumn(PGLOBAL g) + { + char *p = NULL; + int i, rc; + int field; + bool err = false; + double dval; + PTDBDOS tdbp = (PTDBDOS)To_Tdb; + + if (trace(2)) + htrc( + "DOS ReadColumn: col %s R%d coluse=%.4X status=%.4X buf_type=%d\n", + Name, tdbp->GetTdb_No(), ColUse, Status, Buf_Type); + + /*********************************************************************/ + /* If physical reading of the line was deferred, do it now. */ + /*********************************************************************/ + if (!tdbp->IsRead()) + if ((rc = tdbp->ReadBuffer(g)) != RC_OK) { + if (rc == RC_EF) + snprintf(g->Message, sizeof(g->Message), MSG(INV_DEF_READ), rc); + + throw 11; + } // endif + + p = tdbp->To_Line + Deplac; + field = Long; + + /*********************************************************************/ + /* For a variable length file, check if the field exists. */ + /*********************************************************************/ + if ((tdbp->Ftype == RECFM_VAR || tdbp->Ftype == RECFM_CSV) + && strlen(tdbp->To_Line) < (unsigned)Deplac) + field = 0; + else if (Dsp) + for(i = 0; i < field; i++) + if (p[i] == Dsp) + p[i] = '.'; + + switch (tdbp->Ftype) { + case RECFM_VAR: + case RECFM_FIX: // Fixed length text file + case RECFM_CSV: // Variable length CSV or FMT file + case RECFM_DBF: // Fixed length DBase file + if (Nod) switch (Buf_Type) { + case TYPE_INT: + case TYPE_SHORT: + case TYPE_TINY: + case TYPE_BIGINT: + err = Value->SetValue_char(p, field - Dcm); + break; + case TYPE_DOUBLE: + if (!(err = Value->SetValue_char(p, field))) { + dval = Value->GetFloatValue(); + + for (i = 0; i < Dcm; i++) + dval /= 10.0; + + Value->SetValue(dval); + } // endif err + + break; + default: + err = Value->SetValue_char(p, field); + + if (!err && Buf_Type == TYPE_DECIM) { + char* s = Value->GetCharValue(); + + if (!(err = ((i = strlen(s)) >= Value->GetClen()))) { + for (int d = Dcm + 1; d; i--, d--) + s[i + 1] = s[i]; + + s[i + 1] = '.'; + } // endif err + + } // endif DECIM + + break; + } // endswitch Buf_Type + + else + err = Value->SetValue_char(p, field); + + break; + default: + snprintf(g->Message, sizeof(g->Message), MSG(BAD_RECFM), tdbp->Ftype); + throw 34; + } // endswitch Ftype + + if (err) { + snprintf(g->Message, sizeof(g->Message), "Out of range value for column %s at row %d", + Name, tdbp->RowNumber(g)); + PushWarning(g, tdbp); + } // endif err + + // Set null when applicable + if (Nullable) + Value->SetNull(Value->IsZero()); + + } // end of ReadColumn + +/***********************************************************************/ +/* WriteColumn: what this routine does is to access the last line */ +/* read from the corresponding table, and rewrite the field */ +/* corresponding to this column from the column buffer and type. */ +/***********************************************************************/ +void DOSCOL::WriteColumn(PGLOBAL g) + { + char *p, fmt[32]; + int i, k, n, len, field; + PTDBDOS tdbp = (PTDBDOS)To_Tdb; + + if (trace(2)) + htrc("DOS WriteColumn: col %s R%d coluse=%.4X status=%.4X\n", + Name, tdbp->GetTdb_No(), ColUse, Status); + + p = tdbp->To_Line + Deplac; + + if (trace(2)) + htrc("Lrecl=%d deplac=%d int=%d\n", tdbp->Lrecl, Deplac, Long); + + field = Long; + + if (tdbp->Ftype == RECFM_VAR && tdbp->Mode == MODE_UPDATE) { + len = (signed)strlen(tdbp->To_Line); + + if (tdbp->IsUsingTemp(g)) + // Because of eventual missing field(s) the buffer must be reset + memset(tdbp->To_Line + len, ' ', tdbp->Lrecl - len); + else + // The size actually available must be recalculated + field = MY_MIN(len - Deplac, Long); + + } // endif Ftype + + if (trace(2)) + htrc("Long=%d field=%d coltype=%d colval=%p\n", + Long, field, Buf_Type, Value); + + /*********************************************************************/ + /* Get the string representation of Value according to column type. */ + /*********************************************************************/ + if (Value != To_Val) + Value->SetValue_pval(To_Val, false); // Convert the updated value + + /*********************************************************************/ + /* This test is only useful for compressed(2) tables. */ + /*********************************************************************/ + if (tdbp->Ftype != RECFM_BIN) { + if (Ldz || Nod || Dcm >= 0) { + switch (Buf_Type) { + case TYPE_SHORT: + safe_strcpy(fmt, sizeof(fmt), (Ldz) ? "%0*hd" : "%*.hd"); + i = 0; + + if (Nod) + for (; i < Dcm; i++) + safe_strcat(fmt, sizeof(fmt), "0"); + + len = sprintf(Buf, fmt, field - i, Value->GetShortValue()); + break; + case TYPE_INT: + safe_strcpy(fmt, sizeof(fmt), (Ldz) ? "%0*d" : "%*.d"); + i = 0; + + if (Nod) + for (; i < Dcm; i++) + safe_strcat(fmt,sizeof(fmt), "0"); + + len = sprintf(Buf, fmt, field - i, Value->GetIntValue()); + break; + case TYPE_TINY: + safe_strcpy(fmt, sizeof(fmt), (Ldz) ? "%0*d" : "%*.d"); + i = 0; + + if (Nod) + for (; i < Dcm; i++) + safe_strcat(fmt, sizeof(fmt), "0"); + + len = sprintf(Buf, fmt, field - i, Value->GetTinyValue()); + break; + case TYPE_DOUBLE: + case TYPE_DECIM: + safe_strcpy(fmt, sizeof(fmt), (Ldz) ? "%0*.*lf" : "%*.*lf"); + len = field + ((Nod && Dcm) ? 1 : 0); + snprintf(Buf, len + 1, fmt, len, Dcm, Value->GetFloatValue()); + len = strlen(Buf); + + if (Nod && Dcm) + for (i = k = 0; i < len; i++, k++) + if (Buf[i] != ' ') { + if (Buf[i] == '.') + k++; + + Buf[i] = Buf[k]; + } // endif Buf(i) + + len = strlen(Buf); + break; + default: + snprintf(g->Message, sizeof(g->Message), "Invalid field format for column %s", Name); + throw 31; + } // endswitch BufType + + n = strlen(Buf); + } else // Standard CONNECT format + n = Value->ShowValue(Buf, field); + + if (trace(1)) + htrc("new length(%p)=%d\n", Buf, n); + + if ((len = n) > field) { + char *p = Value->GetCharString(Buf); + + snprintf(g->Message, sizeof(g->Message), MSG(VALUE_TOO_LONG), p, Name, field); + throw 31; + } else if (Dsp) + for (i = 0; i < len; i++) + if (Buf[i] == '.') + Buf[i] = Dsp; + + if (trace(2)) + htrc("buffer=%s\n", Buf); + + /*******************************************************************/ + /* Updating must be done only when not in checking pass. */ + /*******************************************************************/ + if (Status) { + memset(p, ' ', field); + memcpy(p, Buf, len); + + if (trace(2)) + htrc(" col write: '%.*s'\n", len, p); + + } // endif Status + + } else // BIN compressed table + /*******************************************************************/ + /* Check if updating is Ok, meaning col value is not too long. */ + /* Updating to be done only during the second pass (Status=true) */ + /*******************************************************************/ + if (Value->GetBinValue(p, Long, Status)) { + snprintf(g->Message, sizeof(g->Message), MSG(BIN_F_TOO_LONG), + Name, Value->GetSize(), Long); + throw 31; + } // endif + + } // end of WriteColumn + +/***********************************************************************/ +/* SetMinMax: Calculate minimum and maximum values for one block. */ +/* Note: TYPE_STRING is stored and processed with zero ended strings */ +/* to be matching the way the FILTER Eval function processes them. */ +/***********************************************************************/ +bool DOSCOL::SetMinMax(PGLOBAL g) + { + PTDBDOS tp = (PTDBDOS)To_Tdb; + + ReadColumn(g); // Extract column value from current line + + if (CheckSorted(g)) + return true; + + if (!tp->Txfp->CurNum) { + Min->SetValue(Value, tp->Txfp->CurBlk); + Max->SetValue(Value, tp->Txfp->CurBlk); + } else { + Min->SetMin(Value, tp->Txfp->CurBlk); + Max->SetMax(Value, tp->Txfp->CurBlk); + } // endif CurNum + + return false; + } // end of SetMinMax + +/***********************************************************************/ +/* SetBitMap: Calculate the bit map of existing values in one block. */ +/* Note: TYPE_STRING is processed with zero ended strings */ +/* to be matching the way the FILTER Eval function processes them. */ +/***********************************************************************/ +bool DOSCOL::SetBitMap(PGLOBAL g) + { + int i, m, n; + uint *bmp; + PTDBDOS tp = (PTDBDOS)To_Tdb; + PDBUSER dup = PlgGetUser(g); + + n = tp->Txfp->CurNum; + bmp = (uint*)Bmap->GetValPtr(Nbm * tp->Txfp->CurBlk); + + // Extract column value from current line + ReadColumn(g); + + if (CheckSorted(g)) + return true; + + if (!n) // New block + for (m = 0; m < Nbm; m++) + bmp[m] = 0; // Reset the new bit map + + if ((i = Dval->Find(Value)) < 0) { + char buf[32]; + + snprintf(g->Message, sizeof(g->Message), MSG(DVAL_NOTIN_LIST), + Value->GetCharString(buf), Name); + return true; + } else if (i >= dup->Maxbmp) { + snprintf(g->Message, sizeof(g->Message), MSG(OPT_LOGIC_ERR), i); + return true; + } else { + m = i / MAXBMP; +#if defined(_DEBUG) + assert (m < Nbm); +#endif // _DEBUG + bmp[m] |= (1 << (i % MAXBMP)); + } // endif's i + + return false; + } // end of SetBitMap + +/***********************************************************************/ +/* Checks whether a column declared as sorted is sorted indeed. */ +/***********************************************************************/ +bool DOSCOL::CheckSorted(PGLOBAL g) + { + if (Sorted) + { + if (OldVal) { + // Verify whether this column is sorted all right + if (OldVal->CompareValue(Value) > 0) { + // Column is no more in ascending order + snprintf(g->Message, sizeof(g->Message), MSG(COL_NOT_SORTED), Name, To_Tdb->GetName()); + Sorted = false; + return true; + } else + OldVal->SetValue_pval(Value); + + } else + OldVal = AllocateValue(g, Value); + } + return false; + } // end of CheckSorted + +/***********************************************************************/ +/* AddDistinctValue: Check whether this value already exist in the */ +/* list and if not add it to the distinct values list. */ +/***********************************************************************/ +bool DOSCOL::AddDistinctValue(PGLOBAL g) + { + bool found = false; + int i, m, n; + + ReadColumn(g); // Extract column value from current line + + // Perhaps a better algorithm can be used when Ndv gets bigger + // Here we cannot use Find because we must get the index of where + // to insert a new value if it is not found in the array. + for (n = 0; n < Ndv; n++) { + m = Dval->CompVal(Value, n); + + if (m > 0) + continue; + else if (!m) + found = true; // Already there + + break; + } // endfor n + + if (!found) { + // Check whether we have room for an additional value + if (Ndv == Freq) { + // Too many values because of wrong Freq setting + snprintf(g->Message, sizeof(g->Message), MSG(BAD_FREQ_SET), Name); + return true; + } // endif Ndv + + // New value, add it to the list before the nth value + Dval->SetNval(Ndv + 1); + + for (i = Ndv; i > n; i--) + Dval->Move(i - 1, i); + + Dval->SetValue(Value, n); + Ndv++; + } // endif found + + return false; + } // end of AddDistinctValue + +/* ------------------------------------------------------------------- */ |