/************* TabDos C++ Program Source Code File (.CPP) **************/ /* PROGRAM NAME: TABDOS */ /* ------------- */ /* Version 4.9.5 */ /* */ /* COPYRIGHT: */ /* ---------- */ /* (C) Copyright to the author Olivier BERTRAND 1998-2020 */ /* */ /* WHAT THIS PROGRAM DOES: */ /* ----------------------- */ /* This program are the DOS tables classes. */ /* */ /***********************************************************************/ /***********************************************************************/ /* Include relevant sections of the System header files. */ /***********************************************************************/ #include "my_global.h" #if defined(_WIN32) #include <io.h> #include <sys\timeb.h> // For testing only #include <fcntl.h> #include <errno.h> #if defined(__BORLANDC__) #define __MFC_COMPAT__ // To define min/max as macro #endif // __BORLANDC__ //#include <windows.h> #else // !_WIN32 #if defined(UNIX) #include <errno.h> #include <unistd.h> #else // !UNIX #include <io.h> #endif // !UNIX #include <fcntl.h> #endif // !_WIN32 /***********************************************************************/ /* Include application header files: */ /* global.h is header containing all global declarations. */ /* plgdbsem.h is header containing the DB application declarations. */ /* filamtxt.h is header containing the file AM classes declarations. */ /***********************************************************************/ #include "global.h" #include "osutil.h" #include "plgdbsem.h" //#include "catalog.h" #include "mycat.h" #include "xindex.h" #include "filamap.h" #include "filamfix.h" #include "filamdbf.h" #if defined(GZ_SUPPORT) #include "filamgz.h" #endif // GZ_SUPPORT #if defined(ZIP_SUPPORT) #include "filamzip.h" #endif // ZIP_SUPPORT #include "tabdos.h" #include "tabfix.h" #include "tabmul.h" #include "array.h" #include "blkfil.h" #include "m_string.h" /***********************************************************************/ /* DB static variables. */ /***********************************************************************/ int num_read, num_there, num_eq[2]; // Statistics /***********************************************************************/ /* Size of optimize file header. */ /***********************************************************************/ #define NZ 4 /***********************************************************************/ /* External function. */ /***********************************************************************/ bool ExactInfo(void); USETEMP UseTemp(void); /***********************************************************************/ /* Min and Max blocks contains zero ended fields (blank = false). */ /* No conversion of block values (check = true). */ /***********************************************************************/ PVBLK AllocValBlock(PGLOBAL, void *, int, int, int len= 0, int prec= 0, bool check= true, bool blank= false, bool un= false); /* --------------------------- Class DOSDEF -------------------------- */ /***********************************************************************/ /* Constructor. */ /***********************************************************************/ DOSDEF::DOSDEF(void) { Pseudo = 3; Fn = NULL; Ofn = NULL; Entry = NULL; To_Indx = NULL; Pwd = NULL; Recfm = RECFM_VAR; Mapped = false; Zipped = false; Mulentries = false; Append = false; Padded = false; Huge = false; Accept = false; Eof = false; To_Pos = NULL; Optimized = 0; AllocBlks = 0; Compressed = 0; Lrecl = 0; AvgLen = 0; Block = 0; Last = 0; Blksize = 0; Maxerr = 0; ReadMode = 0; Ending = 0; Teds = 0; } // end of DOSDEF constructor /***********************************************************************/ /* DefineAM: define specific AM block values from XDB file. */ /***********************************************************************/ bool DOSDEF::DefineAM(PGLOBAL g, LPCSTR am, int) { char buf[8]; bool map = (am && (*am == 'M' || *am == 'm')); LPCSTR dfm = (am && (*am == 'F' || *am == 'f')) ? "F" : (am && (*am == 'B' || *am == 'b')) ? "B" : (am && (*am == 'X' || *am == 'x')) ? "X" : (am && !stricmp(am, "DBF")) ? "D" : "V"; if ((Zipped = GetBoolCatInfo("Zipped", false))) { Entry = GetStringCatInfo(g, "Entry", NULL); Mulentries = (Entry && *Entry) ? strchr(Entry, '*') || strchr(Entry, '?') : false; Mulentries = GetBoolCatInfo("Mulentries", Mulentries); Append = GetBoolCatInfo("Append", false); Pwd = GetStringCatInfo(g, "Password", NULL); } // endif Zipped Desc = Fn = GetStringCatInfo(g, "Filename", NULL); Ofn = GetStringCatInfo(g, "Optname", Fn); GetCharCatInfo("Recfm", (PSZ)dfm, buf, sizeof(buf)); Recfm = (toupper(*buf) == 'F') ? RECFM_FIX : (toupper(*buf) == 'B') ? RECFM_BIN : (toupper(*buf) == 'X') ? RECFM_NAF : // MGO (toupper(*buf) == 'D') ? RECFM_DBF : RECFM_VAR; Lrecl = GetIntCatInfo("Lrecl", 0); if (Recfm != RECFM_DBF) Compressed = GetIntCatInfo("Compressed", 0); Mapped = GetBoolCatInfo("Mapped", map); //Block = GetIntCatInfo("Blocks", 0); //Last = GetIntCatInfo("Last", 0); Ending = GetIntCatInfo("Ending", CRLF); if (Ending <= 0) { Ending = (Recfm == RECFM_BIN || Recfm == RECFM_VCT) ? 0 : CRLF; SetIntCatInfo("Ending", Ending); } // endif ending if (Recfm == RECFM_FIX || Recfm == RECFM_BIN) { Huge = GetBoolCatInfo("Huge", Cat->GetDefHuge()); Padded = GetBoolCatInfo("Padded", false); Blksize = GetIntCatInfo("Blksize", 0); Eof = (GetIntCatInfo("EOF", 0) != 0); Teds = toupper(*GetStringCatInfo(g, "Endian", "")); } else if (Recfm == RECFM_DBF) { Maxerr = GetIntCatInfo("Maxerr", 0); Accept = GetBoolCatInfo("Accept", false); ReadMode = GetIntCatInfo("Readmode", 0); } else // (Recfm == RECFM_VAR) AvgLen = GetIntCatInfo("Avglen", 0); // Ignore wrong Index definitions for catalog commands SetIndexInfo(); return false; } // end of DefineAM /***********************************************************************/ /* Get the full path/name of the optization file. */ /***********************************************************************/ bool DOSDEF::GetOptFileName(PGLOBAL g, char *filename) { PCSZ ftype; switch (Recfm) { case RECFM_VAR: ftype = ".dop"; break; case RECFM_FIX: ftype = ".fop"; break; case RECFM_BIN: ftype = ".bop"; break; case RECFM_VCT: ftype = ".vop"; break; case RECFM_CSV: ftype = ".cop"; break; case RECFM_DBF: ftype = ".dbp"; break; default: snprintf(g->Message, sizeof(g->Message), MSG(INVALID_FTYPE), Recfm); return true; } // endswitch Ftype PlugSetPath(filename, Ofn, GetPath()); strcat(PlugRemoveType(filename, filename), ftype); return false; } // end of GetOptFileName /***********************************************************************/ /* After an optimize error occurred, remove all set optimize values. */ /***********************************************************************/ void DOSDEF::RemoveOptValues(PGLOBAL g) { char filename[_MAX_PATH]; PCOLDEF cdp; // Delete settings of optimized columns for (cdp = To_Cols; cdp; cdp = cdp->GetNext()) if (cdp->GetOpt()) { cdp->SetMin(NULL); cdp->SetMax(NULL); cdp->SetNdv(0); cdp->SetNbm(0); cdp->SetDval(NULL); cdp->SetBmap(NULL); } // endif Opt // Delete block position setting for not fixed tables To_Pos = NULL; AllocBlks = 0; // Delete any eventually ill formed non matching optimization file if (!GetOptFileName(g, filename)) #if defined(_WIN32) DeleteFile(filename); #else // UNIX remove(filename); #endif // _WIN32 Optimized = 0; } // end of RemoveOptValues /***********************************************************************/ /* DeleteIndexFile: Delete DOS/UNIX index file(s) using platform API. */ /***********************************************************************/ bool DOSDEF::DeleteIndexFile(PGLOBAL g, PIXDEF pxdf) { PCSZ ftype; char filename[_MAX_PATH]; bool sep, rc = false; if (!To_Indx) return false; // No index // If true indexes are in separate files sep = GetBoolCatInfo("SepIndex", false); if (!sep && pxdf) { safe_strcpy(g->Message, sizeof(g->Message), MSG(NO_RECOV_SPACE)); return true; } // endif sep switch (Recfm) { case RECFM_VAR: ftype = ".dnx"; break; case RECFM_FIX: ftype = ".fnx"; break; case RECFM_BIN: ftype = ".bnx"; break; case RECFM_VCT: ftype = ".vnx"; break; case RECFM_CSV: ftype = ".cnx"; break; case RECFM_DBF: ftype = ".dbx"; break; default: snprintf(g->Message, sizeof(g->Message), MSG(BAD_RECFM_VAL), Recfm); return true; } // endswitch Ftype /*********************************************************************/ /* Check for existence of an index file. */ /*********************************************************************/ if (sep) { // Indexes are save in separate files #if defined(_WIN32) char drive[_MAX_DRIVE]; #else char *drive = NULL; #endif char direc[_MAX_DIR]; char fname[_MAX_FNAME]; bool all = !pxdf; if (all) pxdf = To_Indx; for (; pxdf; pxdf = pxdf->GetNext()) { _splitpath(Ofn, drive, direc, fname, NULL); safe_strcat(fname, sizeof(fname), "_"); safe_strcat(fname, sizeof(fname), pxdf->GetName()); _makepath(filename, drive, direc, fname, ftype); PlugSetPath(filename, filename, GetPath()); #if defined(_WIN32) if (!DeleteFile(filename)) rc |= (GetLastError() != ERROR_FILE_NOT_FOUND); #else // UNIX if (remove(filename)) rc |= (errno != ENOENT); #endif // UNIX if (!all) break; } // endfor pxdf } else { // !sep // Drop all indexes, delete the common file PlugSetPath(filename, Ofn, GetPath()); safe_strcat(PlugRemoveType(filename, filename), sizeof(filename), ftype); #if defined(_WIN32) if (!DeleteFile(filename)) rc = (GetLastError() != ERROR_FILE_NOT_FOUND); #else // UNIX if (remove(filename)) rc = (errno != ENOENT); #endif // UNIX } // endif sep if (rc) snprintf(g->Message, sizeof(g->Message), MSG(DEL_FILE_ERR), filename); return rc; // Return true if error } // end of DeleteIndexFile /***********************************************************************/ /* InvalidateIndex: mark all indexes as invalid. */ /***********************************************************************/ bool DOSDEF::InvalidateIndex(PGLOBAL) { if (To_Indx) for (PIXDEF xp = To_Indx; xp; xp = xp->Next) xp->Invalid = true; return false; } // end of InvalidateIndex /***********************************************************************/ /* GetTable: makes a new Table Description Block. */ /***********************************************************************/ PTDB DOSDEF::GetTable(PGLOBAL g, MODE mode) { // Mapping not used for insert USETEMP tmp = UseTemp(); bool map = Mapped && mode != MODE_INSERT && !(tmp != TMP_NO && Recfm == RECFM_VAR && mode == MODE_UPDATE) && !(tmp == TMP_FORCE && (mode == MODE_UPDATE || mode == MODE_DELETE)); PTXF txfp = NULL; PTDBASE tdbp; /*********************************************************************/ /* Allocate table and file processing class of the proper type. */ /* Column blocks will be allocated only when needed. */ /*********************************************************************/ if (Recfm == RECFM_DBF) { if (Catfunc == FNC_NO) { if (Zipped) { if (mode == MODE_READ || mode == MODE_ANY || mode == MODE_ALTER) { txfp = new(g) UZDFAM(this); } else { safe_strcpy(g->Message, sizeof(g->Message), "Zipped DBF tables are read only"); return NULL; } // endif's mode } else if (map) txfp = new(g) DBMFAM(this); else txfp = new(g) DBFFAM(this); tdbp = new(g) TDBFIX(this, txfp); } else tdbp = new(g) TDBDCL(this); // Catfunc should be 'C' } else if (Zipped) { #if defined(ZIP_SUPPORT) if (Recfm == RECFM_VAR) { if (mode == MODE_READ || mode == MODE_ANY || mode == MODE_ALTER) { txfp = new(g) UNZFAM(this); } else if (mode == MODE_INSERT) { txfp = new(g) ZIPFAM(this); } else { safe_strcpy(g->Message, sizeof(g->Message), "UPDATE/DELETE not supported for ZIP"); return NULL; } // endif's mode tdbp = new(g) TDBDOS(this, txfp); } else { if (mode == MODE_READ || mode == MODE_ANY || mode == MODE_ALTER) { txfp = new(g) UZXFAM(this); } else if (mode == MODE_INSERT) { txfp = new(g) ZPXFAM(this); } else { safe_strcpy(g->Message, sizeof(g->Message), "UPDATE/DELETE not supported for ZIP"); return NULL; } // endif's mode tdbp = new(g)TDBFIX(this, txfp); } // endif Recfm #else // !ZIP_SUPPORT snprintf(g->Message, sizeof(g->Message), MSG(NO_FEAT_SUPPORT), "ZIP"); return NULL; #endif // !ZIP_SUPPORT } else if (Recfm != RECFM_VAR && Compressed < 2) { if (Huge) txfp = new(g) BGXFAM(this); else if (map) txfp = new(g) MPXFAM(this); else if (Compressed) { #if defined(GZ_SUPPORT) txfp = new(g) GZXFAM(this); #else // !GZ_SUPPORT snprintf(g->Message, sizeof(g->Message), MSG(NO_FEAT_SUPPORT), "GZ"); return NULL; #endif // !GZ_SUPPORT } else txfp = new(g) FIXFAM(this); tdbp = new(g) TDBFIX(this, txfp); } else { if (Compressed) { #if defined(GZ_SUPPORT) if (Compressed == 1) txfp = new(g) GZFAM(this); else txfp = new(g) ZLBFAM(this); #else // !GZ_SUPPORT snprintf(g->Message, sizeof(g->Message), MSG(NO_FEAT_SUPPORT), "GZ"); return NULL; #endif // !GZ_SUPPORT } else if (map) txfp = new(g) MAPFAM(this); else txfp = new(g) DOSFAM(this); // Txfp must be set even for not multiple tables because // it is needed when calling Cardinality in GetBlockValues. tdbp = new(g) TDBDOS(this, txfp); } // endif Recfm if (Multiple) tdbp = new(g) TDBMUL(tdbp); else /*******************************************************************/ /* For block tables, get eventually saved optimization values. */ /*******************************************************************/ if (tdbp->GetBlockValues(g)) { PushWarning(g, tdbp); // return NULL; // causes a crash when deleting index } else if (Recfm == RECFM_VAR || Compressed > 1) { if (IsOptimized()) { if (map) { txfp = new(g) MBKFAM(this); } else if (Compressed) { #if defined(GZ_SUPPORT) if (Compressed == 1) txfp = new(g) ZBKFAM(this); else { txfp->SetBlkPos(To_Pos); ((PZLBFAM)txfp)->SetOptimized(To_Pos != NULL); } // endelse #else snprintf(g->Message, sizeof(g->Message), MSG(NO_FEAT_SUPPORT), "GZ"); return NULL; #endif } else txfp = new(g) BLKFAM(this); ((PTDBDOS)tdbp)->SetTxfp(txfp); } // endif Optimized } // endif Recfm return tdbp; } // end of GetTable /* ------------------------ Class TDBDOS ----------------------------- */ /***********************************************************************/ /* Implementation of the TDBDOS class. This is the common class that */ /* contain all that is common between the TDBDOS and TDBMAP classes. */ /***********************************************************************/ TDBDOS::TDBDOS(PDOSDEF tdp, PTXF txfp) : TDBASE(tdp) { if ((Txfp = txfp)) Txfp->SetTdbp(this); Lrecl = tdp->Lrecl; AvgLen = tdp->AvgLen; Ftype = tdp->Recfm; To_Line = NULL; //To_BlkIdx = NULL; To_BlkFil = NULL; SavFil = NULL; //Xeval = 0; Beval = 0; Abort = false; Indxd = false; } // end of TDBDOS standard constructor TDBDOS::TDBDOS(PGLOBAL g, PTDBDOS tdbp) : TDBASE(tdbp) { Txfp = (g) ? tdbp->Txfp->Duplicate(g) : tdbp->Txfp; Lrecl = tdbp->Lrecl; AvgLen = tdbp->AvgLen; Ftype = tdbp->Ftype; To_Line = tdbp->To_Line; //To_BlkIdx = tdbp->To_BlkIdx; To_BlkFil = tdbp->To_BlkFil; SavFil = tdbp->SavFil; //Xeval = tdbp->Xeval; Beval = tdbp->Beval; Abort = tdbp->Abort; Indxd = tdbp->Indxd; } // end of TDBDOS copy constructor // Method PTDB TDBDOS::Clone(PTABS t) { PTDB tp; PDOSCOL cp1, cp2; PGLOBAL g = t->G; tp = new(g) TDBDOS(g, this); for (cp1 = (PDOSCOL)Columns; cp1; cp1 = (PDOSCOL)cp1->GetNext()) { cp2 = new(g) DOSCOL(cp1, tp); // Make a copy NewPointer(t, cp1, cp2); } // endfor cp1 return tp; } // end of Clone /***********************************************************************/ /* Allocate DOS column description block. */ /***********************************************************************/ PCOL TDBDOS::MakeCol(PGLOBAL g, PCOLDEF cdp, PCOL cprec, int n) { return new(g) DOSCOL(g, cdp, this, cprec, n); } // end of MakeCol /***********************************************************************/ /* Print debug information. */ /***********************************************************************/ void TDBDOS::PrintAM(FILE *f, char *m) { fprintf(f, "%s AM(%d): mode=%d\n", m, GetAmType(), Mode); if (Txfp->To_File) fprintf(f, "%s File: %s\n", m, Txfp->To_File); } // end of PrintAM /***********************************************************************/ /* Remake the indexes after the table was modified. */ /***********************************************************************/ int TDBDOS::ResetTableOpt(PGLOBAL g, bool dop, bool dox) { int prc = RC_OK, rc = RC_OK; if (!GetFileLength(g)) { // Void table, delete all opt and index files PDOSDEF defp = (PDOSDEF)To_Def; defp->RemoveOptValues(g); return (defp->DeleteIndexFile(g, NULL)) ? RC_INFO : RC_OK; } // endif GetFileLength MaxSize = -1; // Size must be recalculated Cardinal = -1; // as well as Cardinality To_Filter = NULL; // Disable filtering //To_BlkIdx = NULL; // and index filtering To_BlkFil = NULL; // and block filtering // After the table was modified the indexes // are invalid and we should mark them as such... (void)((PDOSDEF)To_Def)->InvalidateIndex(g); if (dop) { Columns = NULL; // Not used anymore if (Txfp->Blocked) { // MakeBlockValues must be executed in non blocked mode // except for ZLIB access method. if (Txfp->GetAmType() == TYPE_AM_MAP) { Txfp = new(g) MAPFAM((PDOSDEF)To_Def); #if defined(GZ_SUPPORT) } else if (Txfp->GetAmType() == TYPE_AM_GZ) { Txfp = new(g) GZFAM((PDOSDEF)To_Def); } else if (Txfp->GetAmType() == TYPE_AM_ZLIB) { Txfp->Reset(); ((PZLBFAM)Txfp)->SetOptimized(false); #endif // GZ_SUPPORT } else if (Txfp->GetAmType() == TYPE_AM_BLK) Txfp = new(g) DOSFAM((PDOSDEF)To_Def); Txfp->SetTdbp(this); } else Txfp->Reset(); Use = USE_READY; // So the table can be reopened Mode = MODE_ANY; // Just to be clean rc = MakeBlockValues(g); // Redo optimization } // endif dop if (dox && (rc == RC_OK || rc == RC_INFO)) { // Remake eventual indexes // if (Mode != MODE_UPDATE) To_SetCols = NULL; // Positions are changed Columns = NULL; // Not used anymore Txfp->Reset(); // New start Use = USE_READY; // So the table can be reopened Mode = MODE_READ; // New mode prc = rc; if (PlgGetUser(g)->Check & CHK_OPT) // We must remake all indexes. rc = MakeIndex(g, NULL, false); rc = (rc == RC_INFO) ? prc : rc; } // endif dox return rc; } // end of ResetTableOpt /***********************************************************************/ /* Calculate the block sizes so block I/O can be used and also the */ /* Min/Max values for clustered/sorted table columns. */ /***********************************************************************/ int TDBDOS::MakeBlockValues(PGLOBAL g) { int i, lg, nrec, rc, n = 0; int curnum, curblk, block, savndv, savnbm; void *savmin, *savmax; bool blocked, xdb2 = false; //POOLHEADER save; PCOLDEF cdp; PDOSDEF defp = (PDOSDEF)To_Def; PDOSCOL colp = NULL; PDBUSER dup = PlgGetUser(g); //void *memp = cat->GetDescp(); (void) defp->GetCat(); // XXX Should be removed? if ((nrec = defp->GetElemt()) < 2) { if (!To_Def->Partitioned()) { // This may be wrong to do in some cases safe_strcpy(g->Message, sizeof(g->Message), MSG(TABLE_NOT_OPT)); return RC_INFO; // Not to be optimized } else return RC_OK; } else if (GetMaxSize(g) == 0 || !(dup->Check & CHK_OPT)) { // Suppress the opt file firstly if the table is void, // secondly when it was modified with OPTIMIZATION unchecked // because it is no more valid. defp->RemoveOptValues(g); // Erase opt file return RC_OK; // void table } else if (MaxSize < 0) return RC_FX; defp->SetOptimized(0); // Estimate the number of needed blocks if ((block = (int)((MaxSize + (int)nrec - 1) / (int)nrec)) < 2) { // This may be wrong to do in some cases defp->RemoveOptValues(g); safe_strcpy(g->Message, sizeof(g->Message), MSG(TABLE_NOT_OPT)); return RC_INFO; // Not to be optimized } // endif block // We have to use local variables because Txfp->CurBlk is set // to Rows+1 by unblocked variable length table access methods. curblk = -1; curnum = nrec - 1; //last = 0; Txfp->Block = block; // This is useful mainly for Txfp->CurBlk = curblk; // blocked tables (ZLBFAM), for Txfp->CurNum = curnum; // others it is just to be clean. /*********************************************************************/ /* Allocate the array of block starting positions. */ /*********************************************************************/ //if (memp) // save = *(PPOOLHEADER)memp; Txfp->BlkPos = (int*)PlugSubAlloc(g, NULL, (block + 1) * sizeof(int)); /*********************************************************************/ /* Allocate the blocks for clustered columns. */ /*********************************************************************/ blocked = Txfp->Blocked; // Save Txfp->Blocked = true; // So column block can be allocated for (cdp = defp->GetCols(), i = 1; cdp; cdp = cdp->GetNext(), i++) if (cdp->GetOpt()) { lg = cdp->GetClen(); if (cdp->GetFreq() && cdp->GetFreq() <= dup->Maxbmp) { cdp->SetXdb2(true); savndv = cdp->GetNdv(); cdp->SetNdv(0); // Reset Dval number of values xdb2 = true; savmax = cdp->GetDval(); cdp->SetDval(PlugSubAlloc(g, NULL, cdp->GetFreq() * lg)); savnbm = cdp->GetNbm(); cdp->SetNbm(0); // Prevent Bmap allocation // savmin = cdp->GetBmap(); // cdp->SetBmap(PlugSubAlloc(g, NULL, block * sizeof(int))); if (trace(1)) htrc("Dval(%p) Bmap(%p) col(%d) %s Block=%d lg=%d\n", cdp->GetDval(), cdp->GetBmap(), i, cdp->GetName(), block, lg); // colp will be initialized with proper Dval VALBLK colp = (PDOSCOL)MakeCol(g, cdp, colp, i); colp->InitValue(g); // Allocate column value buffer cdp->SetNbm(savnbm); // cdp->SetBmap(savmin); // Can be reused if the new size cdp->SetDval(savmax); // is not greater than this one. cdp->SetNdv(savndv); } else { cdp->SetXdb2(false); // Maxbmp may have been reset savmin = cdp->GetMin(); savmax = cdp->GetMax(); cdp->SetMin(PlugSubAlloc(g, NULL, block * lg)); cdp->SetMax(PlugSubAlloc(g, NULL, block * lg)); // Valgrind complains if there are uninitialised bytes // after the null character ending if (IsTypeChar(cdp->GetType())) { memset(cdp->GetMin(), 0, block * lg); memset(cdp->GetMax(), 0, block * lg); } // endif Type if (trace(1)) htrc("min(%p) max(%p) col(%d) %s Block=%d lg=%d\n", cdp->GetMin(), cdp->GetMax(), i, cdp->GetName(), block, lg); // colp will be initialized with proper opt VALBLK's colp = (PDOSCOL)MakeCol(g, cdp, colp, i); colp->InitValue(g); // Allocate column value buffer cdp->SetMin(savmin); // Can be reused if the number cdp->SetMax(savmax); // of blocks does not change. } // endif Freq } // endif Clustered // No optimised columns. Still useful for blocked variable tables. if (!colp && defp->Recfm != RECFM_VAR) { safe_strcpy(g->Message, sizeof(g->Message), "No optimised columns"); return RC_INFO; } // endif colp Txfp->Blocked = blocked; /*********************************************************************/ /* Now do calculate the optimization values. */ /*********************************************************************/ Mode = MODE_READ; if (OpenDB(g)) return RC_FX; if (xdb2) { /*********************************************************************/ /* Retrieve the distinct values of XDB2 columns. */ /*********************************************************************/ if (GetDistinctColumnValues(g, nrec)) return RC_FX; OpenDB(g); // Rewind the table file } // endif xdb2 #if defined(PROG_INFO) /*********************************************************************/ /* Initialize progress information */ /*********************************************************************/ char *p = (char *)PlugSubAlloc(g, NULL, 24 + strlen(Name)); snprintf(p, 24 + strlen(Name), "%s%s", MSG(OPTIMIZING), Name); dup->Step = p; dup->ProgMax = GetProgMax(g); dup->ProgCur = 0; #endif // SOCKET_MODE || THREAD /*********************************************************************/ /* Make block starting pos and min/max values of cluster columns. */ /*********************************************************************/ while ((rc = ReadDB(g)) == RC_OK) { if (blocked) { // A blocked FAM class handles CurNum and CurBlk (ZLBFAM) if (!Txfp->CurNum) Txfp->BlkPos[Txfp->CurBlk] = Txfp->GetPos(); } else { if (++curnum >= nrec) { if (++curblk >= block) { safe_strcpy(g->Message, sizeof(g->Message), MSG(BAD_BLK_ESTIM)); goto err; } else curnum = 0; // Get block starting position Txfp->BlkPos[curblk] = Txfp->GetPos(); } // endif CurNum // last = curnum + 1; // curnum is zero based Txfp->CurBlk = curblk; // Used in COLDOS::SetMinMax Txfp->CurNum = curnum; // Used in COLDOS::SetMinMax } // endif blocked /*******************************************************************/ /* Now calculate the min and max values for the cluster columns. */ /*******************************************************************/ for (colp = (PDOSCOL)Columns; colp; colp = (PDOSCOL)colp->GetNext()) if (colp->Clustered == 2) { if (colp->SetBitMap(g)) goto err; } else if (colp->SetMinMax(g)) goto err; // Currently: column is not sorted #if defined(PROG_INFO) if (!dup->Step) { safe_strcpy(g->Message, sizeof(g->Message), MSG(OPT_CANCELLED)); goto err; } else dup->ProgCur = GetProgCur(); #endif // PROG_INFO n++; // Used to calculate block and last } // endwhile if (rc == RC_EF) { Txfp->Nrec = nrec; #if 0 // No good because Curblk and CurNum after EOF are different // depending on whether the file is mapped or not mapped. if (blocked) { // Txfp->Block = Txfp->CurBlk + 1; Txfp->Last = (Txfp->CurNum) ? Txfp->CurNum : nrec; // Txfp->Last = (Txfp->CurNum) ? Txfp->CurNum + 1 : nrec; Txfp->Block = Txfp->CurBlk + (Txfp->Last == nrec ? 0 : 1); } else { Txfp->Block = curblk + 1; Txfp->Last = last; } // endif blocked #endif // 0 // New values of Block and Last Txfp->Block = (n + nrec - 1) / nrec; Txfp->Last = (n % nrec) ? (n % nrec) : nrec; // This is needed to be able to calculate the last block size Txfp->BlkPos[Txfp->Block] = Txfp->GetNextPos(); } else goto err; /*********************************************************************/ /* Save the optimization values for this table. */ /*********************************************************************/ if (!SaveBlockValues(g)) { defp->Block = Txfp->Block; defp->Last = Txfp->Last; CloseDB(g); defp->SetIntCatInfo("Blocks", Txfp->Block); defp->SetIntCatInfo("Last", Txfp->Last); return RC_OK; } // endif SaveBlockValues err: // Restore Desc memory suballocation //if (memp) // *(PPOOLHEADER)memp = save; defp->RemoveOptValues(g); CloseDB(g); return RC_FX; } // end of MakeBlockValues /***********************************************************************/ /* Save the block and Min/Max values for this table. */ /* The problem here is to avoid name duplication, because more than */ /* one data file can have the same name (but different types) and/or */ /* the same data file can be used with different block sizes. This is */ /* why we use Ofn that defaults to the file name but can be set to a */ /* different name if necessary. */ /***********************************************************************/ bool TDBDOS::SaveBlockValues(PGLOBAL g) { char filename[_MAX_PATH]; int lg, n[NZ + 2]; size_t nbk, ndv, nbm, block = Txfp->Block; bool rc = false; FILE *opfile; PDOSCOL colp; PDOSDEF defp = (PDOSDEF)To_Def; if (defp->GetOptFileName(g, filename)) return true; if (!(opfile = fopen(filename, "wb"))) { snprintf(g->Message, sizeof(g->Message), MSG(OPEN_MODE_ERROR), "wb", (int)errno, filename); safe_strcat(g->Message, sizeof(g->Message), ": "); safe_strcat(g->Message, sizeof(g->Message), strerror(errno)); if (trace(1)) htrc("%s\n", g->Message); return true; } // endif opfile memset(n, 0, sizeof(n)); // To avoid valgrind warning if (Ftype == RECFM_VAR || defp->Compressed == 2) { /*******************************************************************/ /* Write block starting positions into the opt file. */ /*******************************************************************/ block++; lg = sizeof(int); n[0] = Txfp->Last; n[1] = lg; n[2] = Txfp->Nrec; n[3] = Txfp->Block; if (fwrite(n, sizeof(int), NZ, opfile) != NZ) { snprintf(g->Message, sizeof(g->Message), MSG(OPT_HEAD_WR_ERR), strerror(errno)); rc = true; } // endif size if (fwrite(Txfp->BlkPos, lg, block, opfile) != block) { snprintf(g->Message, sizeof(g->Message), MSG(OPTBLK_WR_ERR), strerror(errno)); rc = true; } // endif size block--; // = Txfp->Block; } // endif Ftype /*********************************************************************/ /* Write the Min/Max values into the opt file. */ /*********************************************************************/ for (colp = (PDOSCOL)Columns; colp; colp = (PDOSCOL)colp->Next) { lg = colp->Value->GetClen(); // Now start the writing process if (colp->Clustered == 2) { // New XDB2 block optimization. Will be recognized when reading // because the column index is negated. ndv = colp->Ndv; nbm = colp->Nbm; nbk = nbm * block; n[0] = -colp->Index; n[1] = lg; n[2] = Txfp->Nrec; n[3] = block; n[4] = ndv; n[5] = nbm; if (fwrite(n, sizeof(int), NZ + 2, opfile) != NZ + 2) { snprintf(g->Message, sizeof(g->Message), MSG(OPT_HEAD_WR_ERR), strerror(errno)); rc = true; } // endif size if (fwrite(colp->Dval->GetValPointer(), lg, ndv, opfile) != ndv) { snprintf(g->Message, sizeof(g->Message), MSG(OPT_DVAL_WR_ERR), strerror(errno)); rc = true; } // endif size if (fwrite(colp->Bmap->GetValPointer(), sizeof(int), nbk, opfile) != nbk) { snprintf(g->Message, sizeof(g->Message), MSG(OPT_BMAP_WR_ERR), strerror(errno)); rc = true; } // endif size } else { n[0] = colp->Index; n[1] = lg; n[2] = Txfp->Nrec; n[3] = block; if (fwrite(n, sizeof(int), NZ, opfile) != NZ) { snprintf(g->Message, sizeof(g->Message), MSG(OPT_HEAD_WR_ERR), strerror(errno)); rc = true; } // endif size if (fwrite(colp->Min->GetValPointer(), lg, block, opfile) != block) { snprintf(g->Message, sizeof(g->Message), MSG(OPT_MIN_WR_ERR), strerror(errno)); rc = true; } // endif size if (fwrite(colp->Max->GetValPointer(), lg, block, opfile) != block) { snprintf(g->Message, sizeof(g->Message), MSG(OPT_MAX_WR_ERR), strerror(errno)); rc = true; } // endif size } // endif Clustered } // endfor colp fclose(opfile); return rc; } // end of SaveBlockValues /***********************************************************************/ /* Read the Min/Max values for this table. */ /* The problem here is to avoid name duplication, because more than */ /* one data file can have the same name (but different types) and/or */ /* the same data file can be used with different block sizes. This is */ /* why we use Ofn that defaults to the file name but can be set to a */ /* different name if necessary. */ /***********************************************************************/ bool TDBDOS::GetBlockValues(PGLOBAL g) { char filename[_MAX_PATH]; int i, lg, n[NZ]; int nrec, block = 0, last = 0; int len; bool newblk = false; size_t ndv, nbm, nbk, blk; FILE *opfile; PCOLDEF cdp; PDOSDEF defp = (PDOSDEF)To_Def; PDBUSER dup = PlgGetUser(g); (void) defp->GetCat(); // XXX Should be removed? #if 0 if (Mode == MODE_INSERT && Txfp->GetAmType() == TYPE_AM_DOS) return false; #endif // _WIN32 if (defp->Optimized || !(dup->Check & CHK_OPT)) return false; // Already done or to be redone if (Ftype == RECFM_VAR || defp->Compressed == 2) { /*******************************************************************/ /* Variable length file that can be read by block. */ /*******************************************************************/ nrec = (defp->GetElemt()) ? defp->GetElemt() : 1; if (nrec > 1) { // The table can be declared optimized if it is void. // This is useful to handle Insert in optimized mode. char filename[_MAX_PATH]; int h; int flen = -1; PlugSetPath(filename, defp->Fn, GetPath()); h = open(filename, O_RDONLY); flen = (h == -1 && errno == ENOENT) ? 0 : _filelength(h); if (h != -1) close(h); if (!flen) { defp->SetOptimized(1); return false; } // endif flen } else return false; // Not optimisable cdp = defp->GetCols(); i = 1; } else { /*******************************************************************/ /* Fixed length file. Opt file exists only for clustered columns. */ /*******************************************************************/ // Check for existence of clustered columns for (cdp = defp->GetCols(), i = 1; cdp; cdp = cdp->GetNext(), i++) if (cdp->GetOpt()) break; if (!cdp) return false; // No optimization needed if ((len = Cardinality(g)) < 0) return true; // Table error else if (!len) return false; // File does not exist yet block = Txfp->Block; // Was set in Cardinality nrec = Txfp->Nrec; } // endif Ftype if (defp->GetOptFileName(g, filename)) return true; if (!(opfile = fopen(filename, "rb"))) return false; // No saved values if (Ftype == RECFM_VAR || defp->Compressed == 2) { /*******************************************************************/ /* Read block starting positions from the opt file. */ /*******************************************************************/ lg = sizeof(int); if (fread(n, sizeof(int), NZ, opfile) != NZ) { snprintf(g->Message, sizeof(g->Message), MSG(OPT_HEAD_RD_ERR), strerror(errno)); goto err; } // endif size if (n[1] != lg || n[2] != nrec) { snprintf(g->Message, sizeof(g->Message), MSG(OPT_NOT_MATCH), filename); goto err; } // endif last = n[0]; block = n[3]; blk = block + 1; defp->To_Pos = (int*)PlugSubAlloc(g, NULL, blk * lg); if (fread(defp->To_Pos, lg, blk, opfile) != blk) { snprintf(g->Message, sizeof(g->Message), MSG(OPTBLK_RD_ERR), strerror(errno)); goto err; } // endif size } // endif Ftype /*********************************************************************/ /* Read the Min/Max values from the opt file. */ /*********************************************************************/ for (; cdp; cdp = cdp->GetNext(), i++) if (cdp->GetOpt()) { lg = cdp->GetClen(); blk = block; // Now start the reading process. if (fread(n, sizeof(int), NZ, opfile) != NZ) { snprintf(g->Message, sizeof(g->Message), MSG(OPT_HEAD_RD_ERR), strerror(errno)); goto err; } // endif size if (n[0] == -i) { // Read the XDB2 opt values from the opt file if (n[1] != lg || n[2] != nrec || n[3] != block) { snprintf(g->Message, sizeof(g->Message), MSG(OPT_NOT_MATCH), filename); goto err; } // endif if (fread(n, sizeof(int), 2, opfile) != 2) { snprintf(g->Message, sizeof(g->Message), MSG(OPT_HEAD_RD_ERR), strerror(errno)); goto err; } // endif fread ndv = n[0]; nbm = n[1]; nbk = nbm * blk; if (cdp->GetNdv() < (int)ndv || !cdp->GetDval()) cdp->SetDval(PlugSubAlloc(g, NULL, ndv * lg)); cdp->SetNdv((int)ndv); if (fread(cdp->GetDval(), lg, ndv, opfile) != ndv) { snprintf(g->Message, sizeof(g->Message), MSG(OPT_DVAL_RD_ERR), strerror(errno)); goto err; } // endif size if (newblk || cdp->GetNbm() < (int)nbm || !cdp->GetBmap()) cdp->SetBmap(PlugSubAlloc(g, NULL, nbk * sizeof(int))); cdp->SetNbm((int)nbm); if (fread(cdp->GetBmap(), sizeof(int), nbk, opfile) != nbk) { snprintf(g->Message, sizeof(g->Message), MSG(OPT_BMAP_RD_ERR), strerror(errno)); goto err; } // endif size cdp->SetXdb2(true); } else { // Read the Min/Max values from the opt file if (n[0] != i || n[1] != lg || n[2] != nrec || n[3] != block) { snprintf(g->Message, sizeof(g->Message), MSG(OPT_NOT_MATCH), filename); goto err; } // endif if (newblk || !cdp->GetMin()) cdp->SetMin(PlugSubAlloc(g, NULL, blk * lg)); if (fread(cdp->GetMin(), lg, blk, opfile) != blk) { snprintf(g->Message, sizeof(g->Message), MSG(OPT_MIN_RD_ERR), strerror(errno)); goto err; } // endif size if (newblk || !cdp->GetMax()) cdp->SetMax(PlugSubAlloc(g, NULL, blk * lg)); if (fread(cdp->GetMax(), lg, blk, opfile) != blk) { snprintf(g->Message, sizeof(g->Message), MSG(OPT_MAX_RD_ERR), strerror(errno)); goto err; } // endif size cdp->SetXdb2(false); } // endif n[0] (XDB2) } // endif Clustered defp->SetBlock(block); defp->Last = last; // For Cardinality defp->SetAllocBlks(block); defp->SetOptimized(1); fclose(opfile); MaxSize = -1; // Can be refined later return false; err: defp->RemoveOptValues(g); fclose(opfile); // Ignore error if not in mode CHK_OPT return (PlgGetUser(g)->Check & CHK_OPT) != 0; } // end of GetBlockValues /***********************************************************************/ /* This fonction is used while making XDB2 block optimization. */ /* It constructs for each elligible columns, the sorted list of the */ /* distinct values existing in the column. This function uses an */ /* algorithm that permit to get several sets of distinct values by */ /* reading the table only once, which cannot be done using a standard */ /* SQL query. */ /***********************************************************************/ bool TDBDOS::GetDistinctColumnValues(PGLOBAL g, int nrec) { char *p; int rc, blk, n = 0; PDOSCOL colp; PDBUSER dup = PlgGetUser(g); /*********************************************************************/ /* Initialize progress information */ /*********************************************************************/ p = (char *)PlugSubAlloc(g, NULL, 48 + strlen(Name)); snprintf(p, 48 + strlen(Name), "%s%s", MSG(GET_DIST_VALS), Name); dup->Step = p; dup->ProgMax = GetProgMax(g); dup->ProgCur = 0; while ((rc = ReadDB(g)) == RC_OK) { for (colp = (PDOSCOL)Columns; colp; colp = (PDOSCOL)colp->Next) if (colp->Clustered == 2) if (colp->AddDistinctValue(g)) return true; // Too many distinct values #if defined(SOCKET_MODE) if (SendProgress(dup)) { safe_strcpy(g->Message, sizeof(g->Message), MSG(OPT_CANCELLED)); return true; } else #elif defined(THREAD) if (!dup->Step) { safe_strcpy(g->Message, sizeof(g->Message), MSG(OPT_CANCELLED)); return true; } else #endif // THREAD dup->ProgCur = GetProgCur(); n++; } // endwhile if (rc != RC_EF) return true; // Reset the number of table blocks //nrec = ((PDOSDEF)To_Def)->GetElemt(); (or default value) blk = (n + nrec - 1) / nrec; Txfp->Block = blk; // Useful mainly for ZLBFAM ??? // Set Nbm, Bmap for XDB2 columns for (colp = (PDOSCOL)Columns; colp; colp = (PDOSCOL)colp->Next) if (colp->Clustered == 2) { // colp->Cdp->SetNdv(colp->Ndv); colp->Nbm = (colp->Ndv + MAXBMP - 1) / MAXBMP; colp->Bmap = AllocValBlock(g, NULL, TYPE_INT, colp->Nbm * blk); } // endif Clustered return false; } // end of GetDistinctColumnValues /***********************************************************************/ /* Analyze the filter and construct the Block Evaluation Filter. */ /* This is possible when a filter contains predicates implying a */ /* column marked as "clustered" or "sorted" matched to a constant */ /* argument. It is then possible by comparison against the smallest */ /* and largest column values in each block to determine whether the */ /* filter condition will be always true or always false for the block.*/ /***********************************************************************/ PBF TDBDOS::InitBlockFilter(PGLOBAL g, PFIL filp) { bool blk = Txfp->Blocked; if (To_BlkFil) return To_BlkFil; // Already done else if (!filp) return NULL; else if (blk) { if (Txfp->GetAmType() == TYPE_AM_DBF) /*****************************************************************/ /* If RowID is used in this query, block optimization cannot be */ /* used because currently the file must be read sequentially. */ /*****************************************************************/ for (PCOL cp = Columns; cp; cp = cp->GetNext()) if (cp->GetAmType() == TYPE_AM_ROWID && !((RIDBLK*)cp)->GetRnm()) return NULL; } // endif blk int i, op = filp->GetOpc(), opm = filp->GetOpm(); bool cnv[2]; PCOL colp; PXOB arg[2] = {NULL,NULL}; PBF *fp = NULL, bfp = NULL; switch (op) { case OP_EQ: case OP_NE: case OP_GT: case OP_GE: case OP_LT: case OP_LE: if (! opm) { for (i = 0; i < 2; i++) { arg[i] = filp->Arg(i); cnv[i] = filp->Conv(i); } // endfor i bfp = CheckBlockFilari(g, arg, op, cnv); break; } // endif !opm // if opm, pass thru // fall through case OP_IN: if (filp->GetArgType(0) == TYPE_COLBLK && filp->GetArgType(1) == TYPE_ARRAY) { arg[0] = filp->Arg(0); arg[1] = filp->Arg(1); colp = (PCOL)arg[0]; if (colp->GetTo_Tdb() == this) { // Block evaluation is possible for... if (colp->GetAmType() == TYPE_AM_ROWID) { // Special column ROWID and constant array, but // currently we don't know how to retrieve a RowID // from a DBF table that is not sequentially read. // if (Txfp->GetAmType() != TYPE_AM_DBF || // ((RIDBLK*)arg[0])->GetRnm()) bfp = new(g) BLKSPCIN(g, this, op, opm, arg, Txfp->Nrec); } else if (blk && Txfp->Nrec > 1 && colp->IsClustered()) { // Clustered column and constant array if (colp->GetClustered() == 2) bfp = new(g) BLKFILIN2(g, this, op, opm, arg); else bfp = new(g) BLKFILIN(g, this, op, opm, arg); } } // endif this #if 0 } else if (filp->GetArgType(0) == TYPE_SCALF && filp->GetArgType(1) == TYPE_ARRAY) { arg[0] = filp->Arg(0); arg[1] = filp->Arg(1); if (((PSCALF)arg[0])->GetOp() == OP_ROW && arg[1]->GetResultType() == TYPE_LIST) { PARRAY par = (PARRAY)arg[1]; LSTVAL *vlp = (LSTVAL*)par->GetValue(); ((SFROW*)arg[0])->GetParms(n); if (n != vlp->GetN()) return NULL; else n = par->GetNval(); arg[1] = new(g) CONSTANT(vlp); fp = (PBF*)PlugSubAlloc(g, NULL, n * sizeof(PBF)); cnv[0] = cnv[1] = false; if (op == OP_IN) op = OP_EQ; for (i = 0; i < n; i++) { par->GetNthValue(vlp, i); if (!(fp[i] = CheckBlockFilari(g, arg, op, cnv))) return NULL; } // endfor i bfp = new(g) BLKFILLOG(this, (opm == 2 ? OP_AND : OP_OR), fp, n); } // endif ROW #endif // 0 } // endif Type break; case OP_AND: case OP_OR: fp = (PBF*)PlugSubAlloc(g, NULL, 2 * sizeof(PBF)); fp[0] = InitBlockFilter(g, (PFIL)(filp->Arg(0))); fp[1] = InitBlockFilter(g, (PFIL)(filp->Arg(1))); if (fp[0] || fp[1]) bfp = new(g) BLKFILLOG(this, op, fp, 2); break; case OP_NOT: fp = (PBF*)PlugSubAlloc(g, NULL, sizeof(PBF)); if ((*fp = InitBlockFilter(g, (PFIL)(filp->Arg(0))))) bfp = new(g) BLKFILLOG(this, op, fp, 1); break; case OP_LIKE: default: break; } // endswitch op return bfp; } // end of InitBlockFilter /***********************************************************************/ /* Analyze the passed arguments and construct the Block Filter. */ /***********************************************************************/ PBF TDBDOS::CheckBlockFilari(PGLOBAL g, PXOB *arg, int op, bool *cnv) { //int i, n1, n2, ctype = TYPE_ERROR, n = 0, type[2] = {0,0}; //bool conv = false, xdb2 = false, ok = false, b[2]; //PXOB *xarg1, *xarg2 = NULL, xp[2]; int i, n = 0, type[2] = {0,0}; bool conv = false, xdb2 = false; PXOB xp[2]; PCOL colp; PBF bfp = NULL; for (i = 0; i < 2; i++) { switch (arg[i]->GetType()) { case TYPE_CONST: type[i] = 1; // ctype = arg[i]->GetResultType(); break; case TYPE_COLBLK: conv = cnv[i]; colp = (PCOL)arg[i]; if (colp->GetTo_Tdb() == this) { if (colp->GetAmType() == TYPE_AM_ROWID) { // Currently we don't know how to retrieve a RowID // from a DBF table that is not sequentially read. // if (Txfp->GetAmType() != TYPE_AM_DBF || // ((RIDBLK*)arg[i])->GetRnm()) type[i] = 5; } else if (Txfp->Blocked && Txfp->Nrec > 1 && colp->IsClustered()) { type[i] = 2; xdb2 = colp->GetClustered() == 2; } // endif Clustered } else if (colp->GetColUse(U_CORREL)) { // This is a column pointing to the outer query of a // correlated subquery, it has a constant value during // each execution of the subquery. type[i] = 1; // ctype = arg[i]->GetResultType(); } // endif this break; // case TYPE_SCALF: // if (((PSCALF)arg[i])->GetOp() == OP_ROW) { // sfr[i] = (SFROW*)arg[i]; // type[i] = 7; // } // endif Op // break; default: break; } // endswitch ArgType if (!type[i]) break; n += type[i]; } // endfor i if (n == 3 || n == 6) { if (conv) { // The constant has not the good type and will not match // the block min/max values. Warn and abort. snprintf(g->Message, sizeof(g->Message), "Block opt: %s", MSG(VALTYPE_NOMATCH)); PushWarning(g, this); return NULL; } // endif Conv if (type[0] == 1) { // Make it always as Column-op-Value *xp = arg[0]; arg[0] = arg[1]; arg[1] = *xp; switch (op) { case OP_GT: op = OP_LT; break; case OP_GE: op = OP_LE; break; case OP_LT: op = OP_GT; break; case OP_LE: op = OP_GE; break; } // endswitch op } // endif #if defined(_DEBUG) // assert(arg[0]->GetResultType() == ctype); #endif if (n == 3) { if (xdb2) { if (((PDOSCOL)arg[0])->GetNbm() == 1) bfp = new(g) BLKFILAR2(g, this, op, arg); else // Multiple bitmap made of several ULONG's bfp = new(g) BLKFILMR2(g, this, op, arg); } else bfp = new(g) BLKFILARI(g, this, op, arg); } else // n = 6 bfp = new(g) BLKSPCARI(this, op, arg, Txfp->Nrec); #if 0 } else if (n == 8 || n == 14) { if (n == 8 && ctype != TYPE_LIST) { // Should never happen safe_strcpy(g->Message, sizeof(g->Message), "Block opt: bad constant"); throw 99; } // endif Conv if (type[0] == 1) { // Make it always as Column-op-Value sfr[0] = sfr[1]; arg[1] = arg[0]; switch (op) { case OP_GT: op = OP_LT; break; case OP_GE: op = OP_LE; break; case OP_LT: op = OP_GT; break; case OP_LE: op = OP_GE; break; } // endswitch op } // endif xarg1 = sfr[0]->GetParms(n1); if (n == 8) { vlp = (LSTVAL*)arg[1]->GetValue(); n2 = vlp->GetN(); xp[1] = new(g) CONSTANT((PVAL)NULL); } else xarg2 = sfr[1]->GetParms(n2); if (n1 != n2) return NULL; // Should we flag an error ? fp = (PBF*)PlugSubAlloc(g, NULL, n1 * sizeof(PBF)); for (i = 0; i < n1; i++) { xp[0] = xarg1[i]; if (n == 8) ((CONSTANT*)xp[1])->SetValue(vlp->GetSubVal(i)); else xp[1] = xarg2[i]; b[0] = b[1] = (xp[0]->GetResultType() != xp[1]->GetResultType()); ok |= ((fp[i] = CheckBlockFilari(g, xp, op, b)) != NULL); } // endfor i if (ok) bfp = new(g) BLKFILLOG(this, OP_AND, fp, n1); #endif // 0 } // endif n return bfp; } // end of CheckBlockFilari /***********************************************************************/ /* ResetBlkFil: reset the block filter and restore filtering, or make */ /* the block filter if To_Filter was not set when opening the table. */ /***********************************************************************/ void TDBDOS::ResetBlockFilter(PGLOBAL g) { if (!To_BlkFil) { if (To_Filter) if ((To_BlkFil = InitBlockFilter(g, To_Filter))) { htrc("BlkFil=%p\n", To_BlkFil); MaxSize = -1; // To be recalculated } // endif To_BlkFil return; } // endif To_BlkFil To_BlkFil->Reset(g); if (SavFil && !To_Filter) { // Restore filter if it was disabled by optimization To_Filter = SavFil; SavFil = NULL; } // endif Beval = 0; } // end of ResetBlockFilter /***********************************************************************/ /* Block optimization: evaluate the block index filter against */ /* the min and max values of this block and return: */ /* RC_OK: if some records in the block can meet filter criteria. */ /* RC_NF: if no record in the block can meet filter criteria. */ /* RC_EF: if no record in the remaining file can meet filter criteria.*/ /* In addition, temporarily supress filtering if all the records in */ /* the block meet filter criteria. */ /***********************************************************************/ int TDBDOS::TestBlock(PGLOBAL g) { int rc = RC_OK; if (To_BlkFil && Beval != 2) { // Check for block filtering evaluation if (Beval == 1) { // Filter was removed for last block, restore it To_Filter = SavFil; SavFil = NULL; } // endif Beval // Check for valid records in new block switch (Beval = To_BlkFil->BlockEval(g)) { case -2: // No more valid values in file rc = RC_EF; break; case -1: // No valid values in block rc = RC_NF; break; case 1: // All block values are valid case 2: // All subsequent file values are Ok // Before suppressing the filter for the block(s) it is // necessary to reset the filtered columns to NOT_READ // so their new values are retrieved by the SELECT list. if (To_Filter) // Can be NULL when externally called (XDB) To_Filter->Reset(); SavFil = To_Filter; To_Filter = NULL; // So remove filter } // endswitch Beval if (trace(1)) htrc("BF Eval Beval=%d\n", Beval); } // endif To_BlkFil return rc; } // end of TestBlock /***********************************************************************/ /* Check whether we have to create/update permanent indexes. */ /***********************************************************************/ int TDBDOS::MakeIndex(PGLOBAL g, PIXDEF pxdf, bool add) { int k, n, rc = RC_OK; bool fixed, doit, sep; PCOL *keycols, colp; PIXDEF xdp, sxp = NULL; PKPDEF kdp; PDOSDEF dfp; //PCOLDEF cdp; PXINDEX x; PXLOAD pxp; Mode = MODE_READ; Use = USE_READY; dfp = (PDOSDEF)To_Def; if (!Cardinality(g)) { // Void table erase eventual index file(s) (void)dfp->DeleteIndexFile(g, NULL); return RC_OK; } else fixed = Ftype != RECFM_VAR; // Are we are called from CreateTable or CreateIndex? if (pxdf) { if (!add && dfp->GetIndx()) { safe_strcpy(g->Message, sizeof(g->Message), MSG(INDX_EXIST_YET)); return RC_FX; } // endif To_Indx if (add && dfp->GetIndx()) { for (sxp = dfp->GetIndx(); sxp; sxp = sxp->GetNext()) if (!stricmp(sxp->GetName(), pxdf->GetName())) { snprintf(g->Message, sizeof(g->Message), MSG(INDEX_YET_ON), pxdf->GetName(), Name); return RC_FX; } else if (!sxp->GetNext()) break; sxp->SetNext(pxdf); // first = false; } else dfp->SetIndx(pxdf); // pxdf->SetDef(dfp); } else if (!(pxdf = dfp->GetIndx())) return RC_INFO; // No index to make try { // Allocate all columns that will be used by indexes. // This must be done before opening the table so specific // column initialization can be done (in particular by TDBVCT) for (n = 0, xdp = pxdf; xdp; xdp = xdp->GetNext()) for (kdp = xdp->GetToKeyParts(); kdp; kdp = kdp->GetNext()) { if (!(colp = ColDB(g, kdp->GetName(), 0))) { snprintf(g->Message, sizeof(g->Message), MSG(INDX_COL_NOTIN), kdp->GetName(), Name); goto err; } else if (colp->GetResultType() == TYPE_DECIM) { snprintf(g->Message, sizeof(g->Message), "Decimal columns are not indexable yet"); goto err; } // endif Type colp->InitValue(g); n = MY_MAX(n, xdp->GetNparts()); } // endfor kdp keycols = (PCOL*)PlugSubAlloc(g, NULL, n * sizeof(PCOL)); sep = dfp->GetBoolCatInfo("SepIndex", false); /*********************************************************************/ /* Construct and save the defined indexes. */ /*********************************************************************/ for (xdp = pxdf; xdp; xdp = xdp->GetNext()) if (!OpenDB(g)) { if (xdp->IsAuto() && fixed) // Auto increment key and fixed file: use an XXROW index continue; // XXROW index doesn't need to be made // On Update, redo only indexes that are modified doit = !To_SetCols; n = 0; if (sxp) xdp->SetID(sxp->GetID() + 1); for (kdp = xdp->GetToKeyParts(); kdp; kdp = kdp->GetNext()) { // Check whether this column was updated for (colp = To_SetCols; !doit && colp; colp = colp->GetNext()) if (!stricmp(kdp->GetName(), colp->GetName())) doit = true; keycols[n++] = ColDB(g, kdp->GetName(), 0); } // endfor kdp // If no indexed columns were updated, don't remake the index // if indexes are in separate files. if (!doit && sep) continue; k = xdp->GetNparts(); // Make the index and save it if (dfp->Huge) pxp = new(g) XHUGE; else pxp = new(g) XFILE; if (k == 1) // Simple index x = new(g) XINDXS(this, xdp, pxp, keycols); else // Multi-Column index x = new(g) XINDEX(this, xdp, pxp, keycols); if (!x->Make(g, sxp)) { // Retreive define values from the index xdp->SetMaxSame(x->GetMaxSame()); // xdp->SetSize(x->GetSize()); // store KXYCOL Mxs in KPARTDEF Mxsame xdp->SetMxsame(x); #if defined(TRACE) printf("Make done...\n"); #endif // TRACE // if (x->GetSize() > 0) sxp = xdp; xdp->SetInvalid(false); } else goto err; } else return RC_INFO; // Error or Physical table does not exist } catch (int n) { if (trace(1)) htrc("Exception %d: %s\n", n, g->Message); rc = RC_FX; } catch (const char *msg) { safe_strcpy(g->Message, sizeof(g->Message), msg); rc = RC_FX; } // end catch if (Use == USE_OPEN) CloseDB(g); return rc; err: if (sxp) sxp->SetNext(NULL); else dfp->SetIndx(NULL); return RC_FX; } // end of MakeIndex /***********************************************************************/ /* Make a dynamic index. */ /***********************************************************************/ bool TDBDOS::InitialyzeIndex(PGLOBAL g, volatile PIXDEF xdp, bool sorted) { int k; volatile bool dynamic; bool brc; PCOL colp; PCOLDEF cdp; PVAL valp; PXLOAD pxp; volatile PKXBASE kxp; PKPDEF kdp; if (!xdp && !(xdp = To_Xdp)) { safe_strcpy(g->Message, sizeof(g->Message), "NULL dynamic index"); return true; } else dynamic = To_Filter && xdp->IsUnique() && xdp->IsDynamic(); // dynamic = To_Filter && xdp->IsDynamic(); NIY // Allocate the key columns definition block Knum = xdp->GetNparts(); To_Key_Col = (PCOL*)PlugSubAlloc(g, NULL, Knum * sizeof(PCOL)); // Get the key column description list for (k = 0, kdp = xdp->GetToKeyParts(); kdp; kdp = kdp->GetNext()) if (!(colp = ColDB(g, kdp->GetName(), 0)) || colp->InitValue(g)) { snprintf(g->Message, sizeof(g->Message), "Wrong column %s", kdp->GetName()); return true; } else To_Key_Col[k++] = colp; #if defined(_DEBUG) if (k != Knum) { snprintf(g->Message, sizeof(g->Message), "Key part number mismatch for %s", xdp->GetName()); return 0; } // endif k #endif // _DEBUG // Allocate the pseudo constants that will contain the key values To_Link = (PXOB*)PlugSubAlloc(g, NULL, Knum * sizeof(PXOB)); for (k = 0, kdp = xdp->GetToKeyParts(); kdp; k++, kdp = kdp->GetNext()) { if ((cdp = Key(k)->GetCdp())) valp = AllocateValue(g, cdp->GetType(), cdp->GetLength()); else { // Special column ? colp = Key(k); valp = AllocateValue(g, colp->GetResultType(), colp->GetLength()); } // endif cdp To_Link[k]= new(g) CONSTANT(valp); } // endfor k // Make the index on xdp if (!xdp->IsAuto()) { if (!dynamic) { if (((PDOSDEF)To_Def)->Huge) pxp = new(g) XHUGE; else pxp = new(g) XFILE; } else pxp = NULL; if (Knum == 1) // Single index kxp = new(g) XINDXS(this, xdp, pxp, To_Key_Col, To_Link); else // Multi-Column index kxp = new(g) XINDEX(this, xdp, pxp, To_Key_Col, To_Link); } else // Column contains same values as ROWID kxp = new(g) XXROW(this); try { if (dynamic) { ResetBlockFilter(g); kxp->SetDynamic(dynamic); brc = kxp->Make(g, xdp); } else brc = kxp->Init(g); if (!brc) { if (Txfp->GetAmType() == TYPE_AM_BLK) { // Cannot use indexing in DOS block mode Txfp = new(g) DOSFAM((PBLKFAM)Txfp, (PDOSDEF)To_Def); Txfp->AllocateBuffer(g); To_BlkFil = NULL; } // endif AmType To_Kindex= kxp; if (!(sorted && To_Kindex->IsSorted()) && ((Mode == MODE_UPDATE && IsUsingTemp(g)) || (Mode == MODE_DELETE && Txfp->GetAmType() != TYPE_AM_DBF))) Indxd = true; } // endif brc } catch (int n) { if (trace(1)) htrc("Exception %d: %s\n", n, g->Message); brc = true; } catch (const char *msg) { safe_strcpy(g->Message, sizeof(g->Message), msg); brc = true; } // end catch return brc; } // end of InitialyzeIndex /***********************************************************************/ /* DOS GetProgMax: get the max value for progress information. */ /***********************************************************************/ int TDBDOS::GetProgMax(PGLOBAL g) { return (To_Kindex) ? GetMaxSize(g) : GetFileLength(g); } // end of GetProgMax /***********************************************************************/ /* DOS GetProgCur: get the current value for progress information. */ /***********************************************************************/ int TDBDOS::GetProgCur(void) { return (To_Kindex) ? To_Kindex->GetCur_K() + 1 : GetRecpos(); } // end of GetProgCur /***********************************************************************/ /* RowNumber: return the ordinal number of the current row. */ /***********************************************************************/ int TDBDOS::RowNumber(PGLOBAL g, bool) { if (To_Kindex) { /*******************************************************************/ /* Don't know how to retrieve RowID from file address. */ /*******************************************************************/ snprintf(g->Message, sizeof(g->Message), MSG(NO_ROWID_FOR_AM), GetAmName(g, Txfp->GetAmType())); return 0; } else return Txfp->GetRowID(); } // end of RowNumber /***********************************************************************/ /* DOS Cardinality: returns table cardinality in number of rows. */ /* This function can be called with a null argument to test the */ /* availability of Cardinality implementation (1 yes, 0 no). */ /***********************************************************************/ int TDBDOS::Cardinality(PGLOBAL g) { int n = Txfp->Cardinality(NULL); if (!g) return (Mode == MODE_ANY) ? 1 : n; if (Cardinal < 0) { if (!Txfp->Blocked && n == 0) { // Info command, we try to return exact row number PDOSDEF dfp = (PDOSDEF)To_Def; PIXDEF xdp = dfp->To_Indx; if (xdp && xdp->IsValid()) { // Cardinality can be retreived from one index PXLOAD pxp; if (dfp->Huge) pxp = new(g) XHUGE; else pxp = new(g) XFILE; PXINDEX kxp = new(g) XINDEX(this, xdp, pxp, NULL, NULL); if (!(kxp->GetAllSizes(g, Cardinal))) return Cardinal; } // endif Mode if (Mode == MODE_ANY && ExactInfo()) { // Using index impossible or failed, do it the hard way Mode = MODE_READ; To_Line = (char*)PlugSubAlloc(g, NULL, (size_t)Lrecl + 1); if (Txfp->OpenTableFile(g)) return (Cardinal = Txfp->Cardinality(g)); for (Cardinal = 0; n != RC_EF;) if (!(n = Txfp->ReadBuffer(g))) Cardinal++; Txfp->CloseTableFile(g, false); Mode = MODE_ANY; } else { // Return the best estimate int len = GetFileLength(g); if (len >= 0) { int rec; if (trace(1)) htrc("Estimating lines len=%d ending=%d/n", len, ((PDOSDEF)To_Def)->Ending); /*************************************************************/ /* Estimate the number of lines in the table (if not known) */ /* by dividing the file length by the average record length. */ /*************************************************************/ rec = ((PDOSDEF)To_Def)->Ending; if (AvgLen <= 0) // No given average estimate rec += EstimatedLength(); else // An estimate was given for the average record length rec += AvgLen; Cardinal = (len + rec - 1) / rec; if (trace(1)) htrc("avglen=%d MaxSize%d\n", rec, Cardinal); } // endif len } // endif Mode } else Cardinal = Txfp->Cardinality(g); } // endif Cardinal return Cardinal; } // end of Cardinality /***********************************************************************/ /* DOS GetMaxSize: returns file size estimate in number of lines. */ /* This function covers variable record length files. */ /***********************************************************************/ int TDBDOS::GetMaxSize(PGLOBAL g) { if (MaxSize >= 0) return MaxSize; if (!Cardinality(NULL)) { int len = GetFileLength(g); if (len >= 0) { int rec; if (trace(1)) htrc("Estimating lines len=%d ending=%d/n", len, ((PDOSDEF)To_Def)->Ending); /*****************************************************************/ /* Estimate the number of lines in the table (if not known) by */ /* dividing the file length by minimum record length. */ /*****************************************************************/ rec = EstimatedLength() + ((PDOSDEF)To_Def)->Ending; MaxSize = (len + rec - 1) / rec; if (trace(1)) htrc("avglen=%d MaxSize%d\n", rec, MaxSize); } // endif len } else MaxSize = Cardinality(g); return MaxSize; } // end of GetMaxSize /***********************************************************************/ /* DOS EstimatedLength. Returns an estimated minimum line length. */ /***********************************************************************/ int TDBDOS::EstimatedLength(void) { int dep = 0; PCOLDEF cdp = To_Def->GetCols(); if (!cdp->GetNext()) { // One column table, we are going to return a ridiculous // result if we set dep to 1 dep = 1 + cdp->GetLong() / 20; // Why 20 ????? } else for (; cdp; cdp = cdp->GetNext()) if (!(cdp->Flags & (U_VIRTUAL|U_SPECIAL))) dep = MY_MAX(dep, cdp->GetOffset()); return (int)dep; } // end of Estimated Length /***********************************************************************/ /* DOS tables favor the use temporary files for Update. */ /***********************************************************************/ bool TDBDOS::IsUsingTemp(PGLOBAL) { USETEMP utp = UseTemp(); return (utp == TMP_YES || utp == TMP_FORCE || (utp == TMP_AUTO && Mode == MODE_UPDATE)); } // end of IsUsingTemp /***********************************************************************/ /* DOS Access Method opening routine. */ /* New method now that this routine is called recursively (last table */ /* first in reverse order): index blocks are immediately linked to */ /* join block of next table if it exists or else are discarted. */ /***********************************************************************/ bool TDBDOS::OpenDB(PGLOBAL g) { if (trace(1)) htrc("DOS OpenDB: tdbp=%p tdb=R%d use=%d mode=%d\n", this, Tdb_No, Use, Mode); if (Use == USE_OPEN) { /*******************************************************************/ /* Table already open, just replace it at its beginning. */ /*******************************************************************/ if (!To_Kindex) { Txfp->Rewind(); // see comment in Work.log if (SkipHeader(g)) return true; } else /*****************************************************************/ /* Table is to be accessed through a sorted index table. */ /*****************************************************************/ To_Kindex->Reset(); ResetBlockFilter(g); return false; } // endif use if (Mode == MODE_DELETE && !Next && Txfp->GetAmType() != TYPE_AM_DOS #if defined(BSON_SUPPORT) && Txfp->GetAmType() != TYPE_AM_BIN #endif // BSON_SUPPORT && Txfp->GetAmType() != TYPE_AM_MGO) { // Delete all lines. Not handled in MAP or block mode Txfp = new(g) DOSFAM((PDOSDEF)To_Def); Txfp->SetTdbp(this); } else if (Txfp->Blocked && (Mode == MODE_DELETE || (Mode == MODE_UPDATE && UseTemp() != TMP_NO))) { /*******************************************************************/ /* Delete is not currently handled in block mode neither Update */ /* when using a temporary file. */ /*******************************************************************/ if (Txfp->GetAmType() == TYPE_AM_MAP && Mode == MODE_DELETE) Txfp = new(g) MAPFAM((PDOSDEF)To_Def); #if defined(GZ_SUPPORT) else if (Txfp->GetAmType() == TYPE_AM_GZ) Txfp = new(g) GZFAM((PDOSDEF)To_Def); #endif // GZ_SUPPORT else // if (Txfp->GetAmType() != TYPE_AM_DOS) ??? Txfp = new(g) DOSFAM((PDOSDEF)To_Def); Txfp->SetTdbp(this); } // endif Mode /*********************************************************************/ /* Open according to logical input/output mode required. */ /* Use conventionnal input/output functions. */ /* Treat files as binary in Delete mode (for line moving) */ /*********************************************************************/ if (Txfp->OpenTableFile(g)) return true; Use = USE_OPEN; // Do it now in case we are recursively called /*********************************************************************/ /* Allocate the block filter tree if evaluation is possible. */ /*********************************************************************/ To_BlkFil = InitBlockFilter(g, To_Filter); /*********************************************************************/ /* Lrecl does not include line ending */ /*********************************************************************/ size_t linelen = Lrecl + ((PDOSDEF)To_Def)->Ending + 1; To_Line = (char*)PlugSubAlloc(g, NULL, linelen); if (Mode == MODE_INSERT) { // Spaces between fields must be filled with blanks memset(To_Line, ' ', Lrecl); To_Line[Lrecl] = '\0'; } else memset(To_Line, 0, linelen); if (trace(1)) htrc("OpenDos: R%hd mode=%d To_Line=%p\n", Tdb_No, Mode, To_Line); if (SkipHeader(g)) // When called from CSV/FMT files return true; /*********************************************************************/ /* Reset statistics values. */ /*********************************************************************/ num_read = num_there = num_eq[0] = num_eq[1] = 0; return false; } // end of OpenDB /***********************************************************************/ /* ReadDB: Data Base read routine for DOS access method. */ /***********************************************************************/ int TDBDOS::ReadDB(PGLOBAL g) { if (trace(2)) htrc("DOS ReadDB: R%d Mode=%d key=%p link=%p Kindex=%p To_Line=%p\n", GetTdb_No(), Mode, To_Key_Col, To_Link, To_Kindex, To_Line); if (To_Kindex) { /*******************************************************************/ /* Reading is by an index table. */ /*******************************************************************/ int recpos = To_Kindex->Fetch(g); switch (recpos) { case -1: // End of file reached return RC_EF; case -2: // No match for join return RC_NF; case -3: // Same record as non null last one num_there++; return RC_OK; default: /***************************************************************/ /* Set the file position according to record to read. */ /***************************************************************/ if (SetRecpos(g, recpos)) return RC_FX; if (trace(2)) htrc("File position is now %d\n", GetRecpos()); if (Mode == MODE_READ) /*************************************************************/ /* Defer physical reading until one column setting needs it */ /* as it can be a big saving on joins where no other column */ /* than the keys are used, so reading is unnecessary. */ /*************************************************************/ if (Txfp->DeferReading()) return RC_OK; } // endswitch recpos } // endif To_Kindex if (trace(2)) htrc(" ReadDB: this=%p To_Line=%p\n", this, To_Line); /*********************************************************************/ /* Now start the reading process. */ /*********************************************************************/ return ReadBuffer(g); } // end of ReadDB /***********************************************************************/ /* PrepareWriting: Prepare the line to write. */ /***********************************************************************/ bool TDBDOS::PrepareWriting(PGLOBAL) { if (Ftype == RECFM_VAR && (Mode == MODE_INSERT || Txfp->GetUseTemp())) { char *p; /*******************************************************************/ /* Suppress trailing blanks. */ /* Also suppress eventual null from last line. */ /*******************************************************************/ for (p = To_Line + Lrecl -1; p >= To_Line; p--) if (*p && *p != ' ') break; *(++p) = '\0'; } // endif Mode return false; } // end of PrepareWriting /***********************************************************************/ /* WriteDB: Data Base write routine for DOS access method. */ /***********************************************************************/ int TDBDOS::WriteDB(PGLOBAL g) { if (trace(2)) htrc("DOS WriteDB: R%d Mode=%d \n", Tdb_No, Mode); // Make the line to write if (PrepareWriting(g)) return RC_FX; if (trace(2)) htrc("Write: line is='%s'\n", To_Line); // Now start the writing process return Txfp->WriteBuffer(g); } // end of WriteDB /***********************************************************************/ /* Data Base delete line routine for DOS (and FIX) access method. */ /* RC_FX means delete all. Nothing to do here (was done at open). */ /***********************************************************************/ int TDBDOS::DeleteDB(PGLOBAL g, int irc) { return (irc == RC_FX) ? RC_OK : Txfp->DeleteRecords(g, irc); } // end of DeleteDB /***********************************************************************/ /* Data Base close routine for DOS access method. */ /***********************************************************************/ void TDBDOS::CloseDB(PGLOBAL g) { if (To_Kindex) { To_Kindex->Close(); To_Kindex = NULL; } // endif Txfp->CloseTableFile(g, Abort); RestoreNrec(); } // end of CloseDB // ------------------------ DOSCOL functions ---------------------------- /***********************************************************************/ /* DOSCOL public constructor (also called by MAPCOL). */ /***********************************************************************/ DOSCOL::DOSCOL(PGLOBAL g, PCOLDEF cdp, PTDB tp, PCOL cp, int i, PCSZ am) : COLBLK(cdp, tp, i) { char *p; int prec = Format.Prec; PTXF txfp = ((PTDBDOS)tp)->Txfp; assert(cdp); if (cp) { Next = cp->GetNext(); cp->SetNext(this); } else { Next = tp->GetColumns(); tp->SetColumns(this); } // endif cprec // Set additional Dos access method information for column. Deplac = cdp->GetOffset(); Long = cdp->GetLong(); To_Val = NULL; Clustered = cdp->GetOpt(); Sorted = (cdp->GetOpt() == 2) ? 1 : 0; Ndv = 0; // Currently used only for XDB2 Nbm = 0; // Currently used only for XDB2 Min = NULL; Max = NULL; Bmap = NULL; Dval = NULL; Buf = NULL; if (txfp && txfp->Blocked && Opt && (cdp->GetMin() || cdp->GetDval())) { int nblk = txfp->GetBlock(); Clustered = (cdp->GetXdb2()) ? 2 : 1; Sorted = (cdp->GetOpt() > 1) ? 1 : 0; // Currently ascending only if (Clustered == 1) { Min = AllocValBlock(g, cdp->GetMin(), Buf_Type, nblk, Long, prec); Max = AllocValBlock(g, cdp->GetMax(), Buf_Type, nblk, Long, prec); } else { // Clustered == 2 // Ndv is the number of distinct values in Dval. Ndv and Nbm // may be 0 when optimizing because Ndval is not filled yet, // but the size of the passed Dval memory block is Ok. Ndv = cdp->GetNdv(); Dval = AllocValBlock(g, cdp->GetDval(), Buf_Type, Ndv, Long, prec); // Bmap cannot be allocated when optimizing, we must know Nbm first if ((Nbm = cdp->GetNbm())) Bmap = AllocValBlock(g, cdp->GetBmap(), TYPE_INT, Nbm * nblk); } // endif Clustered } // endif Opt OldVal = NULL; // Currently used only in MinMax Dsp = 0; Ldz = false; Nod = false; Dcm = -1; p = cdp->GetFmt(); Buf = NULL; if (p && IsTypeNum(Buf_Type)) { // Formatted numeric value for (; p && *p && isalpha(*p); p++) switch (toupper(*p)) { case 'Z': // Have leading zeros Ldz = true; break; case 'N': // Have no decimal point Nod = true; break; case 'D': // Decimal separator Dsp = *(++p); break; } // endswitch p // Set number of decimal digits Dcm = (*p) ? atoi(p) : GetScale(); } // endif fmt if (trace(1)) htrc(" making new %sCOL C%d %s at %p\n", am, Index, Name, this); } // end of DOSCOL constructor /***********************************************************************/ /* DOSCOL constructor used for copying columns. */ /* tdbp is the pointer to the new table descriptor. */ /***********************************************************************/ DOSCOL::DOSCOL(DOSCOL *col1, PTDB tdbp) : COLBLK(col1, tdbp) { Deplac = col1->Deplac; Long = col1->Long; To_Val = col1->To_Val; Ldz = col1->Ldz; Dsp = col1->Dsp; Nod = col1->Nod; Dcm = col1->Dcm; OldVal = col1->OldVal; Buf = col1->Buf; Clustered = col1->Clustered; Sorted = col1->Sorted; Min = col1->Min; Max = col1->Max; Bmap = col1->Bmap; Dval = col1->Dval; Ndv = col1->Ndv; Nbm = col1->Nbm; } // end of DOSCOL copy constructor /***********************************************************************/ /* VarSize: This function tells UpdateDB whether or not the block */ /* optimization file must be redone if this column is updated, even */ /* it is not sorted or clustered. This applies to the last column of */ /* a variable length table that is blocked, because if it is updated */ /* using a temporary file, the block size may be modified. */ /***********************************************************************/ bool DOSCOL::VarSize(void) { PTDBDOS tdbp = (PTDBDOS)To_Tdb; PTXF txfp = tdbp->Txfp; if (Cdp && !Cdp->GetNext() // Must be the last column && tdbp->Ftype == RECFM_VAR // of a DOS variable length && txfp->Blocked // blocked table && txfp->GetUseTemp()) // using a temporary file. return true; else return false; } // end VarSize /***********************************************************************/ /* SetBuffer: prepare a column block for write operation. */ /***********************************************************************/ bool DOSCOL::SetBuffer(PGLOBAL g, PVAL value, bool ok, bool check) { if (!(To_Val = value)) { snprintf(g->Message, sizeof(g->Message), MSG(VALUE_ERROR), Name); return true; } else if (Buf_Type == value->GetType()) { // Values are of the (good) column type if (Buf_Type == TYPE_DATE) { // If any of the date values is formatted // output format must be set for the receiving table if (GetDomain() || ((DTVAL *)value)->IsFormatted()) goto newval; // This will make a new value; } else if (Buf_Type == TYPE_DOUBLE) // Float values must be written with the correct (column) precision // Note: maybe this should be forced by ShowValue instead of this ? value->SetPrec(GetScale()); Value = value; // Directly access the external value } else { // Values are not of the (good) column type if (check) { snprintf(g->Message, sizeof(g->Message), MSG(TYPE_VALUE_ERR), Name, GetTypeName(Buf_Type), GetTypeName(value->GetType())); return true; } // endif check newval: if (InitValue(g)) // Allocate the matching value block return true; } // endif's Value, Buf_Type // Allocate the buffer used in WriteColumn for numeric columns if (!Buf && IsTypeNum(Buf_Type)) Buf = (char*)PlugSubAlloc(g, NULL, MY_MAX(64, Long + 1)); else // Text columns do not need additional buffer Buf = (char*)Value->GetTo_Val(); // Because Colblk's have been made from a copy of the original TDB in // case of Update, we must reset them to point to the original one. if (To_Tdb->GetOrig()) To_Tdb = (PTDB)To_Tdb->GetOrig(); // Set the Column Status = (ok) ? BUF_EMPTY : BUF_NO; return false; } // end of SetBuffer /***********************************************************************/ /* ReadColumn: what this routine does is to access the last line */ /* read from the corresponding table, extract from it the field */ /* corresponding to this column and convert it to buffer type. */ /***********************************************************************/ void DOSCOL::ReadColumn(PGLOBAL g) { char *p = NULL; int i, rc; int field; bool err = false; double dval; PTDBDOS tdbp = (PTDBDOS)To_Tdb; if (trace(2)) htrc( "DOS ReadColumn: col %s R%d coluse=%.4X status=%.4X buf_type=%d\n", Name, tdbp->GetTdb_No(), ColUse, Status, Buf_Type); /*********************************************************************/ /* If physical reading of the line was deferred, do it now. */ /*********************************************************************/ if (!tdbp->IsRead()) if ((rc = tdbp->ReadBuffer(g)) != RC_OK) { if (rc == RC_EF) snprintf(g->Message, sizeof(g->Message), MSG(INV_DEF_READ), rc); throw 11; } // endif p = tdbp->To_Line + Deplac; field = Long; /*********************************************************************/ /* For a variable length file, check if the field exists. */ /*********************************************************************/ if ((tdbp->Ftype == RECFM_VAR || tdbp->Ftype == RECFM_CSV) && strlen(tdbp->To_Line) < (unsigned)Deplac) field = 0; else if (Dsp) for(i = 0; i < field; i++) if (p[i] == Dsp) p[i] = '.'; switch (tdbp->Ftype) { case RECFM_VAR: case RECFM_FIX: // Fixed length text file case RECFM_CSV: // Variable length CSV or FMT file case RECFM_DBF: // Fixed length DBase file if (Nod) switch (Buf_Type) { case TYPE_INT: case TYPE_SHORT: case TYPE_TINY: case TYPE_BIGINT: err = Value->SetValue_char(p, field - Dcm); break; case TYPE_DOUBLE: if (!(err = Value->SetValue_char(p, field))) { dval = Value->GetFloatValue(); for (i = 0; i < Dcm; i++) dval /= 10.0; Value->SetValue(dval); } // endif err break; default: err = Value->SetValue_char(p, field); if (!err && Buf_Type == TYPE_DECIM) { char* s = Value->GetCharValue(); if (!(err = ((i = strlen(s)) >= Value->GetClen()))) { for (int d = Dcm + 1; d; i--, d--) s[i + 1] = s[i]; s[i + 1] = '.'; } // endif err } // endif DECIM break; } // endswitch Buf_Type else err = Value->SetValue_char(p, field); break; default: snprintf(g->Message, sizeof(g->Message), MSG(BAD_RECFM), tdbp->Ftype); throw 34; } // endswitch Ftype if (err) { snprintf(g->Message, sizeof(g->Message), "Out of range value for column %s at row %d", Name, tdbp->RowNumber(g)); PushWarning(g, tdbp); } // endif err // Set null when applicable if (Nullable) Value->SetNull(Value->IsZero()); } // end of ReadColumn /***********************************************************************/ /* WriteColumn: what this routine does is to access the last line */ /* read from the corresponding table, and rewrite the field */ /* corresponding to this column from the column buffer and type. */ /***********************************************************************/ void DOSCOL::WriteColumn(PGLOBAL g) { char *p, fmt[32]; int i, k, n, len, field; PTDBDOS tdbp = (PTDBDOS)To_Tdb; if (trace(2)) htrc("DOS WriteColumn: col %s R%d coluse=%.4X status=%.4X\n", Name, tdbp->GetTdb_No(), ColUse, Status); p = tdbp->To_Line + Deplac; if (trace(2)) htrc("Lrecl=%d deplac=%d int=%d\n", tdbp->Lrecl, Deplac, Long); field = Long; if (tdbp->Ftype == RECFM_VAR && tdbp->Mode == MODE_UPDATE) { len = (signed)strlen(tdbp->To_Line); if (tdbp->IsUsingTemp(g)) // Because of eventual missing field(s) the buffer must be reset memset(tdbp->To_Line + len, ' ', tdbp->Lrecl - len); else // The size actually available must be recalculated field = MY_MIN(len - Deplac, Long); } // endif Ftype if (trace(2)) htrc("Long=%d field=%d coltype=%d colval=%p\n", Long, field, Buf_Type, Value); /*********************************************************************/ /* Get the string representation of Value according to column type. */ /*********************************************************************/ if (Value != To_Val) Value->SetValue_pval(To_Val, false); // Convert the updated value /*********************************************************************/ /* This test is only useful for compressed(2) tables. */ /*********************************************************************/ if (tdbp->Ftype != RECFM_BIN) { if (Ldz || Nod || Dcm >= 0) { switch (Buf_Type) { case TYPE_SHORT: safe_strcpy(fmt, sizeof(fmt), (Ldz) ? "%0*hd" : "%*.hd"); i = 0; if (Nod) for (; i < Dcm; i++) safe_strcat(fmt, sizeof(fmt), "0"); len = sprintf(Buf, fmt, field - i, Value->GetShortValue()); break; case TYPE_INT: safe_strcpy(fmt, sizeof(fmt), (Ldz) ? "%0*d" : "%*.d"); i = 0; if (Nod) for (; i < Dcm; i++) safe_strcat(fmt,sizeof(fmt), "0"); len = sprintf(Buf, fmt, field - i, Value->GetIntValue()); break; case TYPE_TINY: safe_strcpy(fmt, sizeof(fmt), (Ldz) ? "%0*d" : "%*.d"); i = 0; if (Nod) for (; i < Dcm; i++) safe_strcat(fmt, sizeof(fmt), "0"); len = sprintf(Buf, fmt, field - i, Value->GetTinyValue()); break; case TYPE_DOUBLE: case TYPE_DECIM: safe_strcpy(fmt, sizeof(fmt), (Ldz) ? "%0*.*lf" : "%*.*lf"); len = field + ((Nod && Dcm) ? 1 : 0); snprintf(Buf, len + 1, fmt, len, Dcm, Value->GetFloatValue()); len = strlen(Buf); if (Nod && Dcm) for (i = k = 0; i < len; i++, k++) if (Buf[i] != ' ') { if (Buf[i] == '.') k++; Buf[i] = Buf[k]; } // endif Buf(i) len = strlen(Buf); break; default: snprintf(g->Message, sizeof(g->Message), "Invalid field format for column %s", Name); throw 31; } // endswitch BufType n = strlen(Buf); } else // Standard CONNECT format n = Value->ShowValue(Buf, field); if (trace(1)) htrc("new length(%p)=%d\n", Buf, n); if ((len = n) > field) { char *p = Value->GetCharString(Buf); snprintf(g->Message, sizeof(g->Message), MSG(VALUE_TOO_LONG), p, Name, field); throw 31; } else if (Dsp) for (i = 0; i < len; i++) if (Buf[i] == '.') Buf[i] = Dsp; if (trace(2)) htrc("buffer=%s\n", Buf); /*******************************************************************/ /* Updating must be done only when not in checking pass. */ /*******************************************************************/ if (Status) { memset(p, ' ', field); memcpy(p, Buf, len); if (trace(2)) htrc(" col write: '%.*s'\n", len, p); } // endif Status } else // BIN compressed table /*******************************************************************/ /* Check if updating is Ok, meaning col value is not too long. */ /* Updating to be done only during the second pass (Status=true) */ /*******************************************************************/ if (Value->GetBinValue(p, Long, Status)) { snprintf(g->Message, sizeof(g->Message), MSG(BIN_F_TOO_LONG), Name, Value->GetSize(), Long); throw 31; } // endif } // end of WriteColumn /***********************************************************************/ /* SetMinMax: Calculate minimum and maximum values for one block. */ /* Note: TYPE_STRING is stored and processed with zero ended strings */ /* to be matching the way the FILTER Eval function processes them. */ /***********************************************************************/ bool DOSCOL::SetMinMax(PGLOBAL g) { PTDBDOS tp = (PTDBDOS)To_Tdb; ReadColumn(g); // Extract column value from current line if (CheckSorted(g)) return true; if (!tp->Txfp->CurNum) { Min->SetValue(Value, tp->Txfp->CurBlk); Max->SetValue(Value, tp->Txfp->CurBlk); } else { Min->SetMin(Value, tp->Txfp->CurBlk); Max->SetMax(Value, tp->Txfp->CurBlk); } // endif CurNum return false; } // end of SetMinMax /***********************************************************************/ /* SetBitMap: Calculate the bit map of existing values in one block. */ /* Note: TYPE_STRING is processed with zero ended strings */ /* to be matching the way the FILTER Eval function processes them. */ /***********************************************************************/ bool DOSCOL::SetBitMap(PGLOBAL g) { int i, m, n; uint *bmp; PTDBDOS tp = (PTDBDOS)To_Tdb; PDBUSER dup = PlgGetUser(g); n = tp->Txfp->CurNum; bmp = (uint*)Bmap->GetValPtr(Nbm * tp->Txfp->CurBlk); // Extract column value from current line ReadColumn(g); if (CheckSorted(g)) return true; if (!n) // New block for (m = 0; m < Nbm; m++) bmp[m] = 0; // Reset the new bit map if ((i = Dval->Find(Value)) < 0) { char buf[32]; snprintf(g->Message, sizeof(g->Message), MSG(DVAL_NOTIN_LIST), Value->GetCharString(buf), Name); return true; } else if (i >= dup->Maxbmp) { snprintf(g->Message, sizeof(g->Message), MSG(OPT_LOGIC_ERR), i); return true; } else { m = i / MAXBMP; #if defined(_DEBUG) assert (m < Nbm); #endif // _DEBUG bmp[m] |= (1 << (i % MAXBMP)); } // endif's i return false; } // end of SetBitMap /***********************************************************************/ /* Checks whether a column declared as sorted is sorted indeed. */ /***********************************************************************/ bool DOSCOL::CheckSorted(PGLOBAL g) { if (Sorted) { if (OldVal) { // Verify whether this column is sorted all right if (OldVal->CompareValue(Value) > 0) { // Column is no more in ascending order snprintf(g->Message, sizeof(g->Message), MSG(COL_NOT_SORTED), Name, To_Tdb->GetName()); Sorted = false; return true; } else OldVal->SetValue_pval(Value); } else OldVal = AllocateValue(g, Value); } return false; } // end of CheckSorted /***********************************************************************/ /* AddDistinctValue: Check whether this value already exist in the */ /* list and if not add it to the distinct values list. */ /***********************************************************************/ bool DOSCOL::AddDistinctValue(PGLOBAL g) { bool found = false; int i, m, n; ReadColumn(g); // Extract column value from current line // Perhaps a better algorithm can be used when Ndv gets bigger // Here we cannot use Find because we must get the index of where // to insert a new value if it is not found in the array. for (n = 0; n < Ndv; n++) { m = Dval->CompVal(Value, n); if (m > 0) continue; else if (!m) found = true; // Already there break; } // endfor n if (!found) { // Check whether we have room for an additional value if (Ndv == Freq) { // Too many values because of wrong Freq setting snprintf(g->Message, sizeof(g->Message), MSG(BAD_FREQ_SET), Name); return true; } // endif Ndv // New value, add it to the list before the nth value Dval->SetNval(Ndv + 1); for (i = Ndv; i > n; i--) Dval->Move(i - 1, i); Dval->SetValue(Value, n); Ndv++; } // endif found return false; } // end of AddDistinctValue /* ------------------------------------------------------------------- */