diff options
| author | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-04-13 13:39:13 +0000 |
|---|---|---|
| committer | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-04-13 13:39:13 +0000 |
| commit | 86fbb58c3ac0865482819c10a3e81f2eea001c36 (patch) | |
| tree | 28c9e526ea739c6f9b89e36115e1e2698bddf981 /storage | |
| parent | Releasing progress-linux version 1:10.11.6-2~progress7.99u1. (diff) | |
| download | mariadb-86fbb58c3ac0865482819c10a3e81f2eea001c36.tar.xz mariadb-86fbb58c3ac0865482819c10a3e81f2eea001c36.zip | |
Merging upstream version 1:10.11.7.
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to '')
227 files changed, 4809 insertions, 3813 deletions
diff --git a/storage/columnstore/columnstore/VERSION b/storage/columnstore/columnstore/VERSION index 17d053eb..35b4ad7f 100644 --- a/storage/columnstore/columnstore/VERSION +++ b/storage/columnstore/columnstore/VERSION @@ -1,4 +1,4 @@ COLUMNSTORE_VERSION_MAJOR=6 COLUMNSTORE_VERSION_MINOR=4 -COLUMNSTORE_VERSION_PATCH=7 -COLUMNSTORE_VERSION_RELEASE=2 +COLUMNSTORE_VERSION_PATCH=8 +COLUMNSTORE_VERSION_RELEASE=1 diff --git a/storage/columnstore/columnstore/dbcon/mysql/ha_mcs.h b/storage/columnstore/columnstore/dbcon/mysql/ha_mcs.h index 33c2c666..04ba27d3 100644 --- a/storage/columnstore/columnstore/dbcon/mysql/ha_mcs.h +++ b/storage/columnstore/columnstore/dbcon/mysql/ha_mcs.h @@ -118,6 +118,22 @@ class ha_mcs : public handler return HA_MAX_REC_LENGTH; } +#ifdef MARIADB_NEW_COST_MODEL + /** @brief + Called in test_quick_select to determine if indexes should be used. + */ + virtual IO_AND_CPU_COST scan_time() override + { + IO_AND_CPU_COST cost; + cost.io= 0.0; + /* + For now, assume all cost is CPU cost. + The numbers are also very inadequate for the new cost model. + */ + cost.cpu= (double)(stats.records + stats.deleted) / 20.0 + 10; + return cost; + } +#else /** @brief Called in test_quick_select to determine if indexes should be used. */ @@ -125,6 +141,7 @@ class ha_mcs : public handler { return (double)(stats.records + stats.deleted) / 20.0 + 10; } +#endif /** @brief Analyze table command. diff --git a/storage/columnstore/columnstore/dbcon/mysql/ha_mcs_impl_if.h b/storage/columnstore/columnstore/dbcon/mysql/ha_mcs_impl_if.h index 40066976..037b2598 100644 --- a/storage/columnstore/columnstore/dbcon/mysql/ha_mcs_impl_if.h +++ b/storage/columnstore/columnstore/dbcon/mysql/ha_mcs_impl_if.h @@ -30,6 +30,7 @@ #include <boost/shared_ptr.hpp> #include <stack> #include <vector> +#include <bitset> #include "idb_mysql.h" #include "ha_mcs_sysvars.h" diff --git a/storage/columnstore/columnstore/storage-manager/src/MetadataFile.cpp b/storage/columnstore/columnstore/storage-manager/src/MetadataFile.cpp index e45c9517..fb947364 100644 --- a/storage/columnstore/columnstore/storage-manager/src/MetadataFile.cpp +++ b/storage/columnstore/columnstore/storage-manager/src/MetadataFile.cpp @@ -28,6 +28,7 @@ #include <boost/uuid/uuid_io.hpp> #include <boost/uuid/random_generator.hpp> #include <unistd.h> +#include <set> #define max(x, y) (x > y ? x : y) #define min(x, y) (x < y ? x : y) diff --git a/storage/columnstore/columnstore/storage-manager/src/S3Storage.h b/storage/columnstore/columnstore/storage-manager/src/S3Storage.h index 338de2f6..a7ff7a44 100644 --- a/storage/columnstore/columnstore/storage-manager/src/S3Storage.h +++ b/storage/columnstore/columnstore/storage-manager/src/S3Storage.h @@ -18,6 +18,7 @@ #ifndef S3STORAGE_H_ #define S3STORAGE_H_ +#include <deque> #include <string> #include <map> #include "CloudStorage.h" diff --git a/storage/columnstore/columnstore/tools/passwd/secrets.cpp b/storage/columnstore/columnstore/tools/passwd/secrets.cpp index aac32ffa..b6096254 100644 --- a/storage/columnstore/columnstore/tools/passwd/secrets.cpp +++ b/storage/columnstore/columnstore/tools/passwd/secrets.cpp @@ -13,6 +13,7 @@ #include "secrets.h" #include <cctype> +#include <array> #include <fstream> #include <pwd.h> #include <sys/stat.h> @@ -35,12 +36,11 @@ using std::string; - #ifdef OPENSSL_VERSION_PREREQ -#if OPENSSL_VERSION_PREREQ(3,0) - #define EVP_CIPHER_key_length EVP_CIPHER_get_key_length - #define EVP_CIPHER_iv_length EVP_CIPHER_get_iv_length - #define EVP_CIPHER_blocksize EVP_CIPHER_get_blocksize +#if OPENSSL_VERSION_PREREQ(3, 0) +#define EVP_CIPHER_key_length EVP_CIPHER_get_key_length +#define EVP_CIPHER_iv_length EVP_CIPHER_get_iv_length +#define EVP_CIPHER_blocksize EVP_CIPHER_get_blocksize #endif #endif @@ -96,10 +96,6 @@ void CSPasswdLogging::log(int priority, const char* format, ...) namespace { using HexLookupTable = std::array<uint8_t, 256>; -HexLookupTable init_hex_lookup_table() noexcept; - -// Hex char -> byte val lookup table. -const HexLookupTable hex_lookup_table = init_hex_lookup_table(); /* used in the bin2hex function */ const char hex_upper[] = "0123456789ABCDEF"; @@ -134,6 +130,9 @@ HexLookupTable init_hex_lookup_table() noexcept return rval; } +// Hex char -> byte val lookup table. +const HexLookupTable hex_lookup_table = init_hex_lookup_table(); + bool hex2bin(const char* in, unsigned int in_len, uint8_t* out) { // Input length must be multiple of two. diff --git a/storage/columnstore/columnstore/utils/cloudio/SocketPool.h b/storage/columnstore/columnstore/utils/cloudio/SocketPool.h index 7e1bc940..310171b8 100644 --- a/storage/columnstore/columnstore/utils/cloudio/SocketPool.h +++ b/storage/columnstore/columnstore/utils/cloudio/SocketPool.h @@ -18,6 +18,7 @@ #ifndef _SOCKETPOOL_H_ #define _SOCKETPOOL_H_ +#include <deque> #include <boost/utility.hpp> #include <boost/thread/mutex.hpp> #include <boost/thread/condition_variable.hpp> diff --git a/storage/connect/tabvct.cpp b/storage/connect/tabvct.cpp index 9cf5f41d..f5710688 100644 --- a/storage/connect/tabvct.cpp +++ b/storage/connect/tabvct.cpp @@ -71,11 +71,6 @@ #include "tabvct.h" #include "valblk.h" -#if defined(UNIX) -//add dummy strerror (NGC) -char *strerror(int num); -#endif // UNIX - /***********************************************************************/ /* External function. */ /***********************************************************************/ diff --git a/storage/connect/zip.c b/storage/connect/zip.c index f6a10601..3d3d4cad 100644 --- a/storage/connect/zip.c +++ b/storage/connect/zip.c @@ -14,8 +14,8 @@ Oct-2009 - Mathias Svensson - Added Zip64 Support when creating new file archives Oct-2009 - Mathias Svensson - Did some code cleanup and refactoring to get better overview of some functions. Oct-2009 - Mathias Svensson - Added zipRemoveExtraInfoBlock to strip extra field data from its ZIP64 data - It is used when recreting zip archive with RAW when deleting items from a zip. - ZIP64 data is automaticly added to items that needs it, and existing ZIP64 data need to be removed. + It is used when recreating zip archive with RAW when deleting items from a zip. + ZIP64 data is automatically added to items that needs it, and existing ZIP64 data need to be removed. Oct-2009 - Mathias Svensson - Added support for BZIP2 as compression mode (bzip2 lib is required) Jan-2010 - back to unzip and minizip 1.0 name scheme, with compatibility layer @@ -25,15 +25,13 @@ #include <stdio.h> #include <stdlib.h> #include <string.h> +#include <stdint.h> #include <time.h> #include "zlib.h" #include "zip.h" -#include "my_attribute.h" #ifdef STDC # include <stddef.h> -# include <string.h> -# include <stdlib.h> #endif #ifdef NO_ERRNO_H extern int errno; @@ -48,7 +46,7 @@ /* compile with -Dlocal if your debugger can't find static symbols */ #ifndef VERSIONMADEBY -# define VERSIONMADEBY (0x0) /* platform depedent */ +# define VERSIONMADEBY (0x0) /* platform dependent */ #endif #ifndef Z_BUFSIZE @@ -62,9 +60,6 @@ #ifndef ALLOC # define ALLOC(size) (malloc(size)) #endif -#ifndef TRYFREE -# define TRYFREE(p) {if (p) free(p);} -#endif /* #define SIZECENTRALDIRITEM (0x2e) @@ -117,7 +112,7 @@ typedef struct linkedlist_datablock_internal_s struct linkedlist_datablock_internal_s* next_datablock; uLong avail_in_this_block; uLong filled_in_this_block; - uLong unused; /* for future use and alignement */ + uLong unused; /* for future use and alignment */ unsigned char data[SIZEDATA_INDATABLOCK]; } linkedlist_datablock_internal; @@ -139,40 +134,40 @@ typedef struct uInt pos_in_buffered_data; /* last written byte in buffered_data */ ZPOS64_T pos_local_header; /* offset of the local header of the file - currenty writing */ + currently writing */ char* central_header; /* central header data for the current file */ uLong size_centralExtra; uLong size_centralheader; /* size of the central header for cur file */ uLong size_centralExtraFree; /* Extra bytes allocated to the centralheader but that are not used */ uLong flag; /* flag of the file currently writing */ - int method; /* compression method of file currenty wr.*/ + int method; /* compression method of file currently wr.*/ int raw; /* 1 for directly writing raw data */ Byte buffered_data[Z_BUFSIZE];/* buffer contain compressed data to be writ*/ uLong dosDate; uLong crc32; int encrypt; - int zip64; /* Add ZIP64 extened information in the extra field */ + int zip64; /* Add ZIP64 extended information in the extra field */ ZPOS64_T pos_zip64extrainfo; ZPOS64_T totalCompressedData; ZPOS64_T totalUncompressedData; #ifndef NOCRYPT unsigned long keys[3]; /* keys defining the pseudo-random sequence */ const z_crc_t* pcrc_32_tab; - int crypt_header_size; + unsigned crypt_header_size; #endif } curfile64_info; typedef struct { zlib_filefunc64_32_def z_filefunc; - voidpf filestream; /* io structore of the zipfile */ + voidpf filestream; /* io structure of the zipfile */ linkedlist_data central_dir;/* datablock with central dir in construction*/ int in_opened_file_inzip; /* 1 if a file in the zip is currently writ.*/ - curfile64_info ci; /* info on the file curretly writing */ + curfile64_info ci; /* info on the file currently writing */ ZPOS64_T begin_pos; /* position of the beginning of the zipfile */ - ZPOS64_T add_position_when_writting_offset; + ZPOS64_T add_position_when_writing_offset; ZPOS64_T number_entry; #ifndef NO_ADDFILEINEXISTINGZIP @@ -187,8 +182,7 @@ typedef struct #include "crypt.h" #endif -local linkedlist_datablock_internal* allocate_new_datablock() -{ +local linkedlist_datablock_internal* allocate_new_datablock(void) { linkedlist_datablock_internal* ldi; ldi = (linkedlist_datablock_internal*) ALLOC(sizeof(linkedlist_datablock_internal)); @@ -201,30 +195,26 @@ local linkedlist_datablock_internal* allocate_new_datablock() return ldi; } -local void free_datablock(linkedlist_datablock_internal* ldi) -{ +local void free_datablock(linkedlist_datablock_internal* ldi) { while (ldi!=NULL) { linkedlist_datablock_internal* ldinext = ldi->next_datablock; - TRYFREE(ldi); + free(ldi); ldi = ldinext; } } -local void init_linkedlist(linkedlist_data* ll) -{ +local void init_linkedlist(linkedlist_data* ll) { ll->first_block = ll->last_block = NULL; } -local void free_linkedlist(linkedlist_data* ll) -{ +local void free_linkedlist(linkedlist_data* ll) { free_datablock(ll->first_block); ll->first_block = ll->last_block = NULL; } -local int add_data_in_datablock(linkedlist_data* ll, const void* buf, uLong len) -{ +local int add_data_in_datablock(linkedlist_data* ll, const void* buf, uLong len) { linkedlist_datablock_internal* ldi; const unsigned char* from_copy; @@ -239,7 +229,7 @@ local int add_data_in_datablock(linkedlist_data* ll, const void* buf, uLong len) } ldi = ll->last_block; - from_copy = (unsigned char*)buf; + from_copy = (const unsigned char*)buf; while (len>0) { @@ -284,9 +274,7 @@ local int add_data_in_datablock(linkedlist_data* ll, const void* buf, uLong len) nbByte == 1, 2 ,4 or 8 (byte, short or long, ZPOS64_T) */ -local int zip64local_putValue OF((const zlib_filefunc64_32_def* pzlib_filefunc_def, voidpf filestream, ZPOS64_T x, int nbByte)); -local int zip64local_putValue (const zlib_filefunc64_32_def* pzlib_filefunc_def, voidpf filestream, ZPOS64_T x, int nbByte) -{ +local int zip64local_putValue(const zlib_filefunc64_32_def* pzlib_filefunc_def, voidpf filestream, ZPOS64_T x, int nbByte) { unsigned char buf[8]; int n; for (n = 0; n < nbByte; n++) @@ -302,15 +290,13 @@ local int zip64local_putValue (const zlib_filefunc64_32_def* pzlib_filefunc_def, } } - if (ZWRITE64(*pzlib_filefunc_def,filestream,buf,nbByte)!=(uLong)nbByte) + if (ZWRITE64(*pzlib_filefunc_def,filestream,buf,(uLong)nbByte)!=(uLong)nbByte) return ZIP_ERRNO; else return ZIP_OK; } -local void zip64local_putValue_inmemory OF((void* dest, ZPOS64_T x, int nbByte)); -local void zip64local_putValue_inmemory (void* dest, ZPOS64_T x, int nbByte) -{ +local void zip64local_putValue_inmemory (void* dest, ZPOS64_T x, int nbByte) { unsigned char* buf=(unsigned char*)dest; int n; for (n = 0; n < nbByte; n++) { @@ -330,25 +316,21 @@ local void zip64local_putValue_inmemory (void* dest, ZPOS64_T x, int nbByte) /****************************************************************************/ -local uLong zip64local_TmzDateToDosDate(const tm_zip* ptm) -{ +local uLong zip64local_TmzDateToDosDate(const tm_zip* ptm) { uLong year = (uLong)ptm->tm_year; if (year>=1980) year-=1980; else if (year>=80) year-=80; return - (uLong) (((ptm->tm_mday) + (32 * (ptm->tm_mon+1)) + (512 * year)) << 16) | - ((ptm->tm_sec/2) + (32* ptm->tm_min) + (2048 * (uLong)ptm->tm_hour)); + (uLong) (((uLong)(ptm->tm_mday) + (32 * (uLong)(ptm->tm_mon+1)) + (512 * year)) << 16) | + (((uLong)ptm->tm_sec/2) + (32 * (uLong)ptm->tm_min) + (2048 * (uLong)ptm->tm_hour)); } /****************************************************************************/ -local int zip64local_getByte OF((const zlib_filefunc64_32_def* pzlib_filefunc_def, voidpf filestream, int *pi)); - -local int zip64local_getByte(const zlib_filefunc64_32_def* pzlib_filefunc_def,voidpf filestream,int* pi) -{ +local int zip64local_getByte(const zlib_filefunc64_32_def* pzlib_filefunc_def, voidpf filestream, int* pi) { unsigned char c; int err = (int)ZREAD64(*pzlib_filefunc_def,filestream,&c,1); if (err==1) @@ -369,10 +351,7 @@ local int zip64local_getByte(const zlib_filefunc64_32_def* pzlib_filefunc_def,vo /* =========================================================================== Reads a long in LSB order from the given gz_stream. Sets */ -local int zip64local_getShort OF((const zlib_filefunc64_32_def* pzlib_filefunc_def, voidpf filestream, uLong *pX)); - -local int zip64local_getShort (const zlib_filefunc64_32_def* pzlib_filefunc_def, voidpf filestream, uLong* pX) -{ +local int zip64local_getShort(const zlib_filefunc64_32_def* pzlib_filefunc_def, voidpf filestream, uLong* pX) { uLong x ; int i = 0; int err; @@ -391,10 +370,7 @@ local int zip64local_getShort (const zlib_filefunc64_32_def* pzlib_filefunc_def, return err; } -local int zip64local_getLong OF((const zlib_filefunc64_32_def* pzlib_filefunc_def, voidpf filestream, uLong *pX)); - -local int zip64local_getLong (const zlib_filefunc64_32_def* pzlib_filefunc_def, voidpf filestream, uLong* pX) -{ +local int zip64local_getLong(const zlib_filefunc64_32_def* pzlib_filefunc_def, voidpf filestream, uLong* pX) { uLong x ; int i = 0; int err; @@ -421,11 +397,8 @@ local int zip64local_getLong (const zlib_filefunc64_32_def* pzlib_filefunc_def, return err; } -local int zip64local_getLong64 OF((const zlib_filefunc64_32_def* pzlib_filefunc_def, voidpf filestream, ZPOS64_T *pX)); - -local int zip64local_getLong64 (const zlib_filefunc64_32_def* pzlib_filefunc_def, voidpf filestream, ZPOS64_T *pX) -{ +local int zip64local_getLong64(const zlib_filefunc64_32_def* pzlib_filefunc_def, voidpf filestream, ZPOS64_T *pX) { ZPOS64_T x; int i = 0; int err; @@ -476,10 +449,7 @@ local int zip64local_getLong64 (const zlib_filefunc64_32_def* pzlib_filefunc_def Locate the Central directory of a zipfile (at the end, just before the global comment) */ -local ZPOS64_T zip64local_SearchCentralDir OF((const zlib_filefunc64_32_def* pzlib_filefunc_def, voidpf filestream)); - -local ZPOS64_T zip64local_SearchCentralDir(const zlib_filefunc64_32_def* pzlib_filefunc_def, voidpf filestream) -{ +local ZPOS64_T zip64local_SearchCentralDir(const zlib_filefunc64_32_def* pzlib_filefunc_def, voidpf filestream) { unsigned char* buf; ZPOS64_T uSizeFile; ZPOS64_T uBackRead; @@ -519,18 +489,18 @@ local ZPOS64_T zip64local_SearchCentralDir(const zlib_filefunc64_32_def* pzlib_f if (ZREAD64(*pzlib_filefunc_def,filestream,buf,uReadSize)!=uReadSize) break; - for (i=(int)uReadSize-3; (i--)>0;) { + for (i=(int)uReadSize-3; (i--)>0;) if (((*(buf+i))==0x50) && ((*(buf+i+1))==0x4b) && ((*(buf+i+2))==0x05) && ((*(buf+i+3))==0x06)) { - uPosFound = uReadPos+i; + uPosFound = uReadPos+(unsigned)i; break; } - } + if (uPosFound!=0) break; } - TRYFREE(buf); + free(buf); return uPosFound; } @@ -538,10 +508,7 @@ local ZPOS64_T zip64local_SearchCentralDir(const zlib_filefunc64_32_def* pzlib_f Locate the End of Zip64 Central directory locator and from there find the CD of a zipfile (at the end, just before the global comment) */ -local ZPOS64_T zip64local_SearchCentralDir64 OF((const zlib_filefunc64_32_def* pzlib_filefunc_def, voidpf filestream)); - -local ZPOS64_T zip64local_SearchCentralDir64(const zlib_filefunc64_32_def* pzlib_filefunc_def, voidpf filestream) -{ +local ZPOS64_T zip64local_SearchCentralDir64(const zlib_filefunc64_32_def* pzlib_filefunc_def, voidpf filestream) { unsigned char* buf; ZPOS64_T uSizeFile; ZPOS64_T uBackRead; @@ -587,7 +554,7 @@ local ZPOS64_T zip64local_SearchCentralDir64(const zlib_filefunc64_32_def* pzlib // Signature "0x07064b50" Zip64 end of central directory locater if (((*(buf+i))==0x50) && ((*(buf+i+1))==0x4b) && ((*(buf+i+2))==0x06) && ((*(buf+i+3))==0x07)) { - uPosFound = uReadPos+i; + uPosFound = uReadPos+(unsigned)i; break; } } @@ -596,7 +563,7 @@ local ZPOS64_T zip64local_SearchCentralDir64(const zlib_filefunc64_32_def* pzlib break; } - TRYFREE(buf); + free(buf); if (uPosFound == 0) return 0; @@ -638,8 +605,7 @@ local ZPOS64_T zip64local_SearchCentralDir64(const zlib_filefunc64_32_def* pzlib return relativeOffset; } -static int LoadCentralDirectoryRecord(zip64_internal* pziinit) -{ +local int LoadCentralDirectoryRecord(zip64_internal* pziinit) { int err=ZIP_OK; ZPOS64_T byte_before_the_zipfile;/* byte before the zipfile, (>0 for sfx)*/ @@ -649,9 +615,9 @@ static int LoadCentralDirectoryRecord(zip64_internal* pziinit) uLong uL; uLong number_disk; /* number of the current dist, used for - spaning ZIP, unsupported, always 0*/ + spanning ZIP, unsupported, always 0*/ uLong number_disk_with_CD; /* number the the disk with central dir, used - for spaning ZIP, unsupported, always 0*/ + for spanning ZIP, unsupported, always 0*/ ZPOS64_T number_entry; ZPOS64_T number_entry_CD; /* total number of entries in the central dir @@ -808,7 +774,7 @@ static int LoadCentralDirectoryRecord(zip64_internal* pziinit) } byte_before_the_zipfile = central_pos - (offset_central_dir+size_central_dir); - pziinit->add_position_when_writting_offset = byte_before_the_zipfile; + pziinit->add_position_when_writing_offset = byte_before_the_zipfile; { ZPOS64_T size_central_dir_to_read = size_central_dir; @@ -831,7 +797,7 @@ static int LoadCentralDirectoryRecord(zip64_internal* pziinit) size_central_dir_to_read-=read_this; } - TRYFREE(buf_read); + free(buf_read); } pziinit->begin_pos = byte_before_the_zipfile; pziinit->number_entry = number_entry_CD; @@ -847,8 +813,7 @@ static int LoadCentralDirectoryRecord(zip64_internal* pziinit) /************************************************************/ -static zipFile zipOpen3 (const void *pathname, int append, zipcharpc* globalcomment, zlib_filefunc64_32_def* pzlib_filefunc64_32_def) -{ +extern zipFile ZEXPORT zipOpen3(const void *pathname, int append, zipcharpc* globalcomment, zlib_filefunc64_32_def* pzlib_filefunc64_32_def) { zip64_internal ziinit; zip64_internal* zi; int err=ZIP_OK; @@ -876,7 +841,7 @@ static zipFile zipOpen3 (const void *pathname, int append, zipcharpc* globalcomm ziinit.in_opened_file_inzip = 0; ziinit.ci.stream_initialised = 0; ziinit.number_entry = 0; - ziinit.add_position_when_writting_offset = 0; + ziinit.add_position_when_writing_offset = 0; init_linkedlist(&(ziinit.central_dir)); @@ -906,9 +871,9 @@ static zipFile zipOpen3 (const void *pathname, int append, zipcharpc* globalcomm if (err != ZIP_OK) { # ifndef NO_ADDFILEINEXISTINGZIP - TRYFREE(ziinit.globalcomment); + free(ziinit.globalcomment); # endif /* !NO_ADDFILEINEXISTINGZIP*/ - TRYFREE(zi); + free(zi); return NULL; } else @@ -918,8 +883,7 @@ static zipFile zipOpen3 (const void *pathname, int append, zipcharpc* globalcomm } } -extern zipFile ZEXPORT zipOpen2 (const char *pathname, int append, zipcharpc* globalcomment, zlib_filefunc_def* pzlib_filefunc32_def) -{ +extern zipFile ZEXPORT zipOpen2(const char *pathname, int append, zipcharpc* globalcomment, zlib_filefunc_def* pzlib_filefunc32_def) { if (pzlib_filefunc32_def != NULL) { zlib_filefunc64_32_def zlib_filefunc64_32_def_fill; @@ -930,8 +894,7 @@ extern zipFile ZEXPORT zipOpen2 (const char *pathname, int append, zipcharpc* gl return zipOpen3(pathname, append, globalcomment, NULL); } -extern zipFile ZEXPORT zipOpen2_64 (const void *pathname, int append, zipcharpc* globalcomment, zlib_filefunc64_def* pzlib_filefunc_def) -{ +extern zipFile ZEXPORT zipOpen2_64(const void *pathname, int append, zipcharpc* globalcomment, zlib_filefunc64_def* pzlib_filefunc_def) { if (pzlib_filefunc_def != NULL) { zlib_filefunc64_32_def zlib_filefunc64_32_def_fill; @@ -946,18 +909,15 @@ extern zipFile ZEXPORT zipOpen2_64 (const void *pathname, int append, zipcharpc* -extern zipFile ZEXPORT zipOpen (const char* pathname, int append) -{ +extern zipFile ZEXPORT zipOpen(const char* pathname, int append) { return zipOpen3((const void*)pathname,append,NULL,NULL); } -extern zipFile ZEXPORT zipOpen64 (const void* pathname, int append) -{ +extern zipFile ZEXPORT zipOpen64(const void* pathname, int append) { return zipOpen3(pathname,append,NULL,NULL); } -static int Write_LocalFileHeader(zip64_internal* zi, const char* filename, uInt size_extrafield_local, const void* extrafield_local) -{ +local int Write_LocalFileHeader(zip64_internal* zi, const char* filename, uInt size_extrafield_local, const void* extrafield_local) { /* write the local header */ int err; uInt size_filename = (uInt)strlen(filename); @@ -1035,8 +995,8 @@ static int Write_LocalFileHeader(zip64_internal* zi, const char* filename, uInt // Remember position of Zip64 extended info for the local file header. (needed when we update size after done with file) zi->ci.pos_zip64extrainfo = ZTELL64(zi->z_filefunc,zi->filestream); - err = zip64local_putValue(&zi->z_filefunc, zi->filestream, (short)HeaderID,2); - err = zip64local_putValue(&zi->z_filefunc, zi->filestream, (short)DataSize,2); + err = zip64local_putValue(&zi->z_filefunc, zi->filestream, (ZPOS64_T)HeaderID,2); + err = zip64local_putValue(&zi->z_filefunc, zi->filestream, (ZPOS64_T)DataSize,2); err = zip64local_putValue(&zi->z_filefunc, zi->filestream, (ZPOS64_T)UncompressedSize,8); err = zip64local_putValue(&zi->z_filefunc, zi->filestream, (ZPOS64_T)CompressedSize,8); @@ -1053,24 +1013,24 @@ static int Write_LocalFileHeader(zip64_internal* zi, const char* filename, uInt It is not done here because then we need to realloc a new buffer since parameters are 'const' and I want to minimize unnecessary allocations. */ -extern int ZEXPORT zipOpenNewFileInZip4_64 (zipFile file, const char* filename, const zip_fileinfo* zipfi, - const void* extrafield_local, uInt size_extrafield_local, - const void* extrafield_global, uInt size_extrafield_global, - const char* comment, int method, int level, int raw, - int windowBits,int memLevel, int strategy, - const char* password, uLong crcForCrypting __attribute__((unused)), - uLong versionMadeBy, uLong flagBase, int zip64) -{ +extern int ZEXPORT zipOpenNewFileInZip4_64(zipFile file, const char* filename, const zip_fileinfo* zipfi, + const void* extrafield_local, uInt size_extrafield_local, + const void* extrafield_global, uInt size_extrafield_global, + const char* comment, int method, int level, int raw, + int windowBits,int memLevel, int strategy, + const char* password, uLong crcForCrypting, + uLong versionMadeBy, uLong flagBase, int zip64) { zip64_internal* zi; uInt size_filename; uInt size_comment; uInt i; int err = ZIP_OK; -#ifdef NOCRYPT +# ifdef NOCRYPT + (crcForCrypting); if (password != NULL) return ZIP_PARAMERROR; -#endif +# endif if (file == NULL) return ZIP_PARAMERROR; @@ -1164,7 +1124,7 @@ extern int ZEXPORT zipOpenNewFileInZip4_64 (zipFile file, const char* filename, if(zi->ci.pos_local_header >= 0xffffffff) zip64local_putValue_inmemory(zi->ci.central_header+42,(uLong)0xffffffff,4); else - zip64local_putValue_inmemory(zi->ci.central_header+42,(uLong)zi->ci.pos_local_header - zi->add_position_when_writting_offset,4); + zip64local_putValue_inmemory(zi->ci.central_header+42,(uLong)zi->ci.pos_local_header - zi->add_position_when_writing_offset,4); for (i=0;i<size_filename;i++) *(zi->ci.central_header+SIZECENTRALHEADER+i) = *(filename+i); @@ -1262,35 +1222,33 @@ extern int ZEXPORT zipOpenNewFileInZip4_64 (zipFile file, const char* filename, return err; } -extern int ZEXPORT zipOpenNewFileInZip4 (zipFile file, const char* filename, const zip_fileinfo* zipfi, - const void* extrafield_local, uInt size_extrafield_local, - const void* extrafield_global, uInt size_extrafield_global, - const char* comment, int method, int level, int raw, - int windowBits,int memLevel, int strategy, - const char* password, uLong crcForCrypting, - uLong versionMadeBy, uLong flagBase) -{ - return zipOpenNewFileInZip4_64 (file, filename, zipfi, - extrafield_local, size_extrafield_local, - extrafield_global, size_extrafield_global, - comment, method, level, raw, - windowBits, memLevel, strategy, - password, crcForCrypting, versionMadeBy, flagBase, 0); +extern int ZEXPORT zipOpenNewFileInZip4(zipFile file, const char* filename, const zip_fileinfo* zipfi, + const void* extrafield_local, uInt size_extrafield_local, + const void* extrafield_global, uInt size_extrafield_global, + const char* comment, int method, int level, int raw, + int windowBits,int memLevel, int strategy, + const char* password, uLong crcForCrypting, + uLong versionMadeBy, uLong flagBase) { + return zipOpenNewFileInZip4_64(file, filename, zipfi, + extrafield_local, size_extrafield_local, + extrafield_global, size_extrafield_global, + comment, method, level, raw, + windowBits, memLevel, strategy, + password, crcForCrypting, versionMadeBy, flagBase, 0); } -extern int ZEXPORT zipOpenNewFileInZip3 (zipFile file, const char* filename, const zip_fileinfo* zipfi, - const void* extrafield_local, uInt size_extrafield_local, - const void* extrafield_global, uInt size_extrafield_global, - const char* comment, int method, int level, int raw, - int windowBits,int memLevel, int strategy, - const char* password, uLong crcForCrypting) -{ - return zipOpenNewFileInZip4_64 (file, filename, zipfi, - extrafield_local, size_extrafield_local, - extrafield_global, size_extrafield_global, - comment, method, level, raw, - windowBits, memLevel, strategy, - password, crcForCrypting, VERSIONMADEBY, 0, 0); +extern int ZEXPORT zipOpenNewFileInZip3(zipFile file, const char* filename, const zip_fileinfo* zipfi, + const void* extrafield_local, uInt size_extrafield_local, + const void* extrafield_global, uInt size_extrafield_global, + const char* comment, int method, int level, int raw, + int windowBits,int memLevel, int strategy, + const char* password, uLong crcForCrypting) { + return zipOpenNewFileInZip4_64(file, filename, zipfi, + extrafield_local, size_extrafield_local, + extrafield_global, size_extrafield_global, + comment, method, level, raw, + windowBits, memLevel, strategy, + password, crcForCrypting, VERSIONMADEBY, 0, 0); } extern int ZEXPORT zipOpenNewFileInZip3_64(zipFile file, const char* filename, const zip_fileinfo* zipfi, @@ -1298,70 +1256,64 @@ extern int ZEXPORT zipOpenNewFileInZip3_64(zipFile file, const char* filename, c const void* extrafield_global, uInt size_extrafield_global, const char* comment, int method, int level, int raw, int windowBits,int memLevel, int strategy, - const char* password, uLong crcForCrypting, int zip64) -{ - return zipOpenNewFileInZip4_64 (file, filename, zipfi, - extrafield_local, size_extrafield_local, - extrafield_global, size_extrafield_global, - comment, method, level, raw, - windowBits, memLevel, strategy, - password, crcForCrypting, VERSIONMADEBY, 0, zip64); + const char* password, uLong crcForCrypting, int zip64) { + return zipOpenNewFileInZip4_64(file, filename, zipfi, + extrafield_local, size_extrafield_local, + extrafield_global, size_extrafield_global, + comment, method, level, raw, + windowBits, memLevel, strategy, + password, crcForCrypting, VERSIONMADEBY, 0, zip64); } extern int ZEXPORT zipOpenNewFileInZip2(zipFile file, const char* filename, const zip_fileinfo* zipfi, const void* extrafield_local, uInt size_extrafield_local, const void* extrafield_global, uInt size_extrafield_global, - const char* comment, int method, int level, int raw) -{ - return zipOpenNewFileInZip4_64 (file, filename, zipfi, - extrafield_local, size_extrafield_local, - extrafield_global, size_extrafield_global, - comment, method, level, raw, - -MAX_WBITS, DEF_MEM_LEVEL, Z_DEFAULT_STRATEGY, - NULL, 0, VERSIONMADEBY, 0, 0); + const char* comment, int method, int level, int raw) { + return zipOpenNewFileInZip4_64(file, filename, zipfi, + extrafield_local, size_extrafield_local, + extrafield_global, size_extrafield_global, + comment, method, level, raw, + -MAX_WBITS, DEF_MEM_LEVEL, Z_DEFAULT_STRATEGY, + NULL, 0, VERSIONMADEBY, 0, 0); } extern int ZEXPORT zipOpenNewFileInZip2_64(zipFile file, const char* filename, const zip_fileinfo* zipfi, - const void* extrafield_local, uInt size_extrafield_local, - const void* extrafield_global, uInt size_extrafield_global, - const char* comment, int method, int level, int raw, int zip64) -{ - return zipOpenNewFileInZip4_64 (file, filename, zipfi, - extrafield_local, size_extrafield_local, - extrafield_global, size_extrafield_global, - comment, method, level, raw, - -MAX_WBITS, DEF_MEM_LEVEL, Z_DEFAULT_STRATEGY, - NULL, 0, VERSIONMADEBY, 0, zip64); + const void* extrafield_local, uInt size_extrafield_local, + const void* extrafield_global, uInt size_extrafield_global, + const char* comment, int method, int level, int raw, int zip64) { + return zipOpenNewFileInZip4_64(file, filename, zipfi, + extrafield_local, size_extrafield_local, + extrafield_global, size_extrafield_global, + comment, method, level, raw, + -MAX_WBITS, DEF_MEM_LEVEL, Z_DEFAULT_STRATEGY, + NULL, 0, VERSIONMADEBY, 0, zip64); } -extern int ZEXPORT zipOpenNewFileInZip64 (zipFile file, const char* filename, const zip_fileinfo* zipfi, - const void* extrafield_local, uInt size_extrafield_local, - const void*extrafield_global, uInt size_extrafield_global, - const char* comment, int method, int level, int zip64) -{ - return zipOpenNewFileInZip4_64 (file, filename, zipfi, - extrafield_local, size_extrafield_local, - extrafield_global, size_extrafield_global, - comment, method, level, 0, - -MAX_WBITS, DEF_MEM_LEVEL, Z_DEFAULT_STRATEGY, - NULL, 0, VERSIONMADEBY, 0, zip64); +extern int ZEXPORT zipOpenNewFileInZip64(zipFile file, const char* filename, const zip_fileinfo* zipfi, + const void* extrafield_local, uInt size_extrafield_local, + const void*extrafield_global, uInt size_extrafield_global, + const char* comment, int method, int level, int zip64) { + return zipOpenNewFileInZip4_64(file, filename, zipfi, + extrafield_local, size_extrafield_local, + extrafield_global, size_extrafield_global, + comment, method, level, 0, + -MAX_WBITS, DEF_MEM_LEVEL, Z_DEFAULT_STRATEGY, + NULL, 0, VERSIONMADEBY, 0, zip64); } -extern int ZEXPORT zipOpenNewFileInZip (zipFile file, const char* filename, const zip_fileinfo* zipfi, - const void* extrafield_local, uInt size_extrafield_local, - const void*extrafield_global, uInt size_extrafield_global, - const char* comment, int method, int level) -{ - return zipOpenNewFileInZip4_64 (file, filename, zipfi, - extrafield_local, size_extrafield_local, - extrafield_global, size_extrafield_global, - comment, method, level, 0, - -MAX_WBITS, DEF_MEM_LEVEL, Z_DEFAULT_STRATEGY, - NULL, 0, VERSIONMADEBY, 0, 0); +extern int ZEXPORT zipOpenNewFileInZip(zipFile file, const char* filename, const zip_fileinfo* zipfi, + const void* extrafield_local, uInt size_extrafield_local, + const void*extrafield_global, uInt size_extrafield_global, + const char* comment, int method, int level) { + return zipOpenNewFileInZip4_64(file, filename, zipfi, + extrafield_local, size_extrafield_local, + extrafield_global, size_extrafield_global, + comment, method, level, 0, + -MAX_WBITS, DEF_MEM_LEVEL, Z_DEFAULT_STRATEGY, + NULL, 0, VERSIONMADEBY, 0, 0); } -local int zip64FlushWriteBuffer(zip64_internal* zi) -{ +local int zip64FlushWriteBuffer(zip64_internal* zi) { int err=ZIP_OK; if (zi->ci.encrypt != 0) @@ -1399,8 +1351,7 @@ local int zip64FlushWriteBuffer(zip64_internal* zi) return err; } -extern int ZEXPORT zipWriteInFileInZip (zipFile file,const void* buf,unsigned int len) -{ +extern int ZEXPORT zipWriteInFileInZip(zipFile file, const void* buf, unsigned int len) { zip64_internal* zi; int err=ZIP_OK; @@ -1450,7 +1401,7 @@ extern int ZEXPORT zipWriteInFileInZip (zipFile file,const void* buf,unsigned in else #endif { - zi->ci.stream.next_in = (Bytef*)buf; + zi->ci.stream.next_in = (Bytef*)(uintptr_t)buf; zi->ci.stream.avail_in = len; while ((err==ZIP_OK) && (zi->ci.stream.avail_in>0)) @@ -1501,17 +1452,15 @@ extern int ZEXPORT zipWriteInFileInZip (zipFile file,const void* buf,unsigned in return err; } -extern int ZEXPORT zipCloseFileInZipRaw (zipFile file, uLong uncompressed_size, uLong crc32) -{ +extern int ZEXPORT zipCloseFileInZipRaw(zipFile file, uLong uncompressed_size, uLong crc32) { return zipCloseFileInZipRaw64 (file, uncompressed_size, crc32); } -extern int ZEXPORT zipCloseFileInZipRaw64 (zipFile file, ZPOS64_T uncompressed_size, uLong crc32) -{ +extern int ZEXPORT zipCloseFileInZipRaw64(zipFile file, ZPOS64_T uncompressed_size, uLong crc32) { zip64_internal* zi; ZPOS64_T compressed_size; uLong invalidValue = 0xffffffff; - short datasize = 0; + unsigned datasize = 0; int err=ZIP_OK; if (file == NULL) @@ -1742,15 +1691,13 @@ extern int ZEXPORT zipCloseFileInZipRaw64 (zipFile file, ZPOS64_T uncompressed_s return err; } -extern int ZEXPORT zipCloseFileInZip (zipFile file) -{ +extern int ZEXPORT zipCloseFileInZip(zipFile file) { return zipCloseFileInZipRaw (file,0,0); } -static int Write_Zip64EndOfCentralDirectoryLocator(zip64_internal* zi, ZPOS64_T zip64eocd_pos_inzip) -{ +local int Write_Zip64EndOfCentralDirectoryLocator(zip64_internal* zi, ZPOS64_T zip64eocd_pos_inzip) { int err = ZIP_OK; - ZPOS64_T pos = zip64eocd_pos_inzip - zi->add_position_when_writting_offset; + ZPOS64_T pos = zip64eocd_pos_inzip - zi->add_position_when_writing_offset; err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)ZIP64ENDLOCHEADERMAGIC,4); @@ -1769,8 +1716,7 @@ static int Write_Zip64EndOfCentralDirectoryLocator(zip64_internal* zi, ZPOS64_T return err; } -static int Write_Zip64EndOfCentralDirectoryRecord(zip64_internal* zi, uLong size_centraldir, ZPOS64_T centraldir_pos_inzip) -{ +local int Write_Zip64EndOfCentralDirectoryRecord(zip64_internal* zi, uLong size_centraldir, ZPOS64_T centraldir_pos_inzip) { int err = ZIP_OK; uLong Zip64DataSize = 44; @@ -1803,13 +1749,13 @@ static int Write_Zip64EndOfCentralDirectoryRecord(zip64_internal* zi, uLong size if (err==ZIP_OK) /* offset of start of central directory with respect to the starting disk number */ { - ZPOS64_T pos = centraldir_pos_inzip - zi->add_position_when_writting_offset; + ZPOS64_T pos = centraldir_pos_inzip - zi->add_position_when_writing_offset; err = zip64local_putValue(&zi->z_filefunc,zi->filestream, (ZPOS64_T)pos,8); } return err; } -static int Write_EndOfCentralDirectoryRecord(zip64_internal* zi, uLong size_centraldir, ZPOS64_T centraldir_pos_inzip) -{ + +local int Write_EndOfCentralDirectoryRecord(zip64_internal* zi, uLong size_centraldir, ZPOS64_T centraldir_pos_inzip) { int err = ZIP_OK; /*signature*/ @@ -1844,20 +1790,19 @@ static int Write_EndOfCentralDirectoryRecord(zip64_internal* zi, uLong size_cent if (err==ZIP_OK) /* offset of start of central directory with respect to the starting disk number */ { - ZPOS64_T pos = centraldir_pos_inzip - zi->add_position_when_writting_offset; + ZPOS64_T pos = centraldir_pos_inzip - zi->add_position_when_writing_offset; if(pos >= 0xffffffff) { err = zip64local_putValue(&zi->z_filefunc,zi->filestream, (uLong)0xffffffff,4); } else - err = zip64local_putValue(&zi->z_filefunc,zi->filestream, (uLong)(centraldir_pos_inzip - zi->add_position_when_writting_offset),4); + err = zip64local_putValue(&zi->z_filefunc,zi->filestream, (uLong)(centraldir_pos_inzip - zi->add_position_when_writing_offset),4); } return err; } -static int Write_GlobalComment(zip64_internal* zi, const char* global_comment) -{ +local int Write_GlobalComment(zip64_internal* zi, const char* global_comment) { int err = ZIP_OK; uInt size_global_comment = 0; @@ -1874,8 +1819,7 @@ static int Write_GlobalComment(zip64_internal* zi, const char* global_comment) return err; } -extern int ZEXPORT zipClose (zipFile file, const char* global_comment) -{ +extern int ZEXPORT zipClose(zipFile file, const char* global_comment) { zip64_internal* zi; int err = 0; uLong size_centraldir = 0; @@ -1916,7 +1860,7 @@ extern int ZEXPORT zipClose (zipFile file, const char* global_comment) } free_linkedlist(&(zi->central_dir)); - pos = centraldir_pos_inzip - zi->add_position_when_writting_offset; + pos = centraldir_pos_inzip - zi->add_position_when_writing_offset; if(pos >= 0xffffffff || zi->number_entry > 0xFFFF) { ZPOS64_T Zip64EOCDpos = ZTELL64(zi->z_filefunc,zi->filestream); @@ -1936,15 +1880,14 @@ extern int ZEXPORT zipClose (zipFile file, const char* global_comment) err = ZIP_ERRNO; #ifndef NO_ADDFILEINEXISTINGZIP - TRYFREE(zi->globalcomment); + free(zi->globalcomment); #endif - TRYFREE(zi); + free(zi); return err; } -extern int ZEXPORT zipRemoveExtraInfoBlock (char* pData, int* dataLen, short sHeader) -{ +extern int ZEXPORT zipRemoveExtraInfoBlock(char* pData, int* dataLen, short sHeader) { char* p = pData; int size = 0; char* pNewHeader; @@ -1954,10 +1897,10 @@ extern int ZEXPORT zipRemoveExtraInfoBlock (char* pData, int* dataLen, short sHe int retVal = ZIP_OK; - if(pData == NULL || *dataLen < 4) + if(pData == NULL || dataLen == NULL || *dataLen < 4) return ZIP_PARAMERROR; - pNewHeader = (char*)ALLOC(*dataLen); + pNewHeader = (char*)ALLOC((unsigned)*dataLen); pTmp = pNewHeader; while(p < (pData + *dataLen)) @@ -1996,7 +1939,7 @@ extern int ZEXPORT zipRemoveExtraInfoBlock (char* pData, int* dataLen, short sHe else retVal = ZIP_ERRNO; - TRYFREE(pNewHeader); + free(pNewHeader); return retVal; } diff --git a/storage/connect/zip.h b/storage/connect/zip.h index 8aaebb62..5fc08413 100644 --- a/storage/connect/zip.h +++ b/storage/connect/zip.h @@ -88,12 +88,12 @@ typedef voidp zipFile; /* tm_zip contain date/time info */ typedef struct tm_zip_s { - uInt tm_sec; /* seconds after the minute - [0,59] */ - uInt tm_min; /* minutes after the hour - [0,59] */ - uInt tm_hour; /* hours since midnight - [0,23] */ - uInt tm_mday; /* day of the month - [1,31] */ - uInt tm_mon; /* months since January - [0,11] */ - uInt tm_year; /* years - [1980..2044] */ + int tm_sec; /* seconds after the minute - [0,59] */ + int tm_min; /* minutes after the hour - [0,59] */ + int tm_hour; /* hours since midnight - [0,23] */ + int tm_mday; /* day of the month - [1,31] */ + int tm_mon; /* months since January - [0,11] */ + int tm_year; /* years - [1980..2044] */ } tm_zip; typedef struct @@ -113,8 +113,8 @@ typedef const char* zipcharpc; #define APPEND_STATUS_CREATEAFTER (1) #define APPEND_STATUS_ADDINZIP (2) -extern zipFile ZEXPORT zipOpen OF((const char *pathname, int append)); -extern zipFile ZEXPORT zipOpen64 OF((const void *pathname, int append)); +extern zipFile ZEXPORT zipOpen(const char *pathname, int append); +extern zipFile ZEXPORT zipOpen64(const void *pathname, int append); /* Create a zipfile. pathname contain on Windows XP a filename like "c:\\zlib\\zlib113.zip" or on @@ -131,41 +131,46 @@ extern zipFile ZEXPORT zipOpen64 OF((const void *pathname, int append)); /* Note : there is no delete function into a zipfile. If you want delete file into a zipfile, you must open a zipfile, and create another - Of couse, you can use RAW reading and writing to copy the file you did not want delte + Of course, you can use RAW reading and writing to copy the file you did not want delete */ -extern zipFile ZEXPORT zipOpen2 OF((const char *pathname, - int append, - zipcharpc* globalcomment, - zlib_filefunc_def* pzlib_filefunc_def)); +extern zipFile ZEXPORT zipOpen2(const char *pathname, + int append, + zipcharpc* globalcomment, + zlib_filefunc_def* pzlib_filefunc_def); -extern zipFile ZEXPORT zipOpen2_64 OF((const void *pathname, +extern zipFile ZEXPORT zipOpen2_64(const void *pathname, int append, zipcharpc* globalcomment, - zlib_filefunc64_def* pzlib_filefunc_def)); - -extern int ZEXPORT zipOpenNewFileInZip OF((zipFile file, - const char* filename, - const zip_fileinfo* zipfi, - const void* extrafield_local, - uInt size_extrafield_local, - const void* extrafield_global, - uInt size_extrafield_global, - const char* comment, - int method, - int level)); - -extern int ZEXPORT zipOpenNewFileInZip64 OF((zipFile file, - const char* filename, - const zip_fileinfo* zipfi, - const void* extrafield_local, - uInt size_extrafield_local, - const void* extrafield_global, - uInt size_extrafield_global, - const char* comment, - int method, - int level, - int zip64)); + zlib_filefunc64_def* pzlib_filefunc_def); + +extern zipFile ZEXPORT zipOpen3(const void *pathname, + int append, + zipcharpc* globalcomment, + zlib_filefunc64_32_def* pzlib_filefunc64_32_def); + +extern int ZEXPORT zipOpenNewFileInZip(zipFile file, + const char* filename, + const zip_fileinfo* zipfi, + const void* extrafield_local, + uInt size_extrafield_local, + const void* extrafield_global, + uInt size_extrafield_global, + const char* comment, + int method, + int level); + +extern int ZEXPORT zipOpenNewFileInZip64(zipFile file, + const char* filename, + const zip_fileinfo* zipfi, + const void* extrafield_local, + uInt size_extrafield_local, + const void* extrafield_global, + uInt size_extrafield_global, + const char* comment, + int method, + int level, + int zip64); /* Open a file in the ZIP for writing. @@ -184,70 +189,69 @@ extern int ZEXPORT zipOpenNewFileInZip64 OF((zipFile file, */ -extern int ZEXPORT zipOpenNewFileInZip2 OF((zipFile file, - const char* filename, - const zip_fileinfo* zipfi, - const void* extrafield_local, - uInt size_extrafield_local, - const void* extrafield_global, - uInt size_extrafield_global, - const char* comment, - int method, - int level, - int raw)); - - -extern int ZEXPORT zipOpenNewFileInZip2_64 OF((zipFile file, - const char* filename, - const zip_fileinfo* zipfi, - const void* extrafield_local, - uInt size_extrafield_local, - const void* extrafield_global, - uInt size_extrafield_global, - const char* comment, - int method, - int level, - int raw, - int zip64)); +extern int ZEXPORT zipOpenNewFileInZip2(zipFile file, + const char* filename, + const zip_fileinfo* zipfi, + const void* extrafield_local, + uInt size_extrafield_local, + const void* extrafield_global, + uInt size_extrafield_global, + const char* comment, + int method, + int level, + int raw); + + +extern int ZEXPORT zipOpenNewFileInZip2_64(zipFile file, + const char* filename, + const zip_fileinfo* zipfi, + const void* extrafield_local, + uInt size_extrafield_local, + const void* extrafield_global, + uInt size_extrafield_global, + const char* comment, + int method, + int level, + int raw, + int zip64); /* Same than zipOpenNewFileInZip, except if raw=1, we write raw file */ -extern int ZEXPORT zipOpenNewFileInZip3 OF((zipFile file, - const char* filename, - const zip_fileinfo* zipfi, - const void* extrafield_local, - uInt size_extrafield_local, - const void* extrafield_global, - uInt size_extrafield_global, - const char* comment, - int method, - int level, - int raw, - int windowBits, - int memLevel, - int strategy, - const char* password, - uLong crcForCrypting)); - -extern int ZEXPORT zipOpenNewFileInZip3_64 OF((zipFile file, - const char* filename, - const zip_fileinfo* zipfi, - const void* extrafield_local, - uInt size_extrafield_local, - const void* extrafield_global, - uInt size_extrafield_global, - const char* comment, - int method, - int level, - int raw, - int windowBits, - int memLevel, - int strategy, - const char* password, - uLong crcForCrypting, - int zip64 - )); +extern int ZEXPORT zipOpenNewFileInZip3(zipFile file, + const char* filename, + const zip_fileinfo* zipfi, + const void* extrafield_local, + uInt size_extrafield_local, + const void* extrafield_global, + uInt size_extrafield_global, + const char* comment, + int method, + int level, + int raw, + int windowBits, + int memLevel, + int strategy, + const char* password, + uLong crcForCrypting); + +extern int ZEXPORT zipOpenNewFileInZip3_64(zipFile file, + const char* filename, + const zip_fileinfo* zipfi, + const void* extrafield_local, + uInt size_extrafield_local, + const void* extrafield_global, + uInt size_extrafield_global, + const char* comment, + int method, + int level, + int raw, + int windowBits, + int memLevel, + int strategy, + const char* password, + uLong crcForCrypting, + int zip64); /* Same than zipOpenNewFileInZip2, except @@ -256,47 +260,45 @@ extern int ZEXPORT zipOpenNewFileInZip3_64 OF((zipFile file, crcForCrypting : crc of file to compress (needed for crypting) */ -extern int ZEXPORT zipOpenNewFileInZip4 OF((zipFile file, - const char* filename, - const zip_fileinfo* zipfi, - const void* extrafield_local, - uInt size_extrafield_local, - const void* extrafield_global, - uInt size_extrafield_global, - const char* comment, - int method, - int level, - int raw, - int windowBits, - int memLevel, - int strategy, - const char* password, - uLong crcForCrypting, - uLong versionMadeBy, - uLong flagBase - )); - - -extern int ZEXPORT zipOpenNewFileInZip4_64 OF((zipFile file, - const char* filename, - const zip_fileinfo* zipfi, - const void* extrafield_local, - uInt size_extrafield_local, - const void* extrafield_global, - uInt size_extrafield_global, - const char* comment, - int method, - int level, - int raw, - int windowBits, - int memLevel, - int strategy, - const char* password, - uLong crcForCrypting, - uLong versionMadeBy, - uLong flagBase, - int zip64 - )); +extern int ZEXPORT zipOpenNewFileInZip4(zipFile file, + const char* filename, + const zip_fileinfo* zipfi, + const void* extrafield_local, + uInt size_extrafield_local, + const void* extrafield_global, + uInt size_extrafield_global, + const char* comment, + int method, + int level, + int raw, + int windowBits, + int memLevel, + int strategy, + const char* password, + uLong crcForCrypting, + uLong versionMadeBy, + uLong flagBase); + + +extern int ZEXPORT zipOpenNewFileInZip4_64(zipFile file, + const char* filename, + const zip_fileinfo* zipfi, + const void* extrafield_local, + uInt size_extrafield_local, + const void* extrafield_global, + uInt size_extrafield_global, + const char* comment, + int method, + int level, + int raw, + int windowBits, + int memLevel, + int strategy, + const char* password, + uLong crcForCrypting, + uLong versionMadeBy, + uLong flagBase, + int zip64); /* Same than zipOpenNewFileInZip4, except versionMadeBy : value for Version made by field @@ -304,25 +306,25 @@ extern int ZEXPORT zipOpenNewFileInZip4_64 OF((zipFile file, */ -extern int ZEXPORT zipWriteInFileInZip OF((zipFile file, - const void* buf, - unsigned len)); +extern int ZEXPORT zipWriteInFileInZip(zipFile file, + const void* buf, + unsigned len); /* Write data in the zipfile */ -extern int ZEXPORT zipCloseFileInZip OF((zipFile file)); +extern int ZEXPORT zipCloseFileInZip(zipFile file); /* Close the current file in the zipfile */ -extern int ZEXPORT zipCloseFileInZipRaw OF((zipFile file, - uLong uncompressed_size, - uLong crc32)); +extern int ZEXPORT zipCloseFileInZipRaw(zipFile file, + uLong uncompressed_size, + uLong crc32); -extern int ZEXPORT zipCloseFileInZipRaw64 OF((zipFile file, - ZPOS64_T uncompressed_size, - uLong crc32)); +extern int ZEXPORT zipCloseFileInZipRaw64(zipFile file, + ZPOS64_T uncompressed_size, + uLong crc32); /* Close the current file in the zipfile, for file opened with @@ -330,14 +332,14 @@ extern int ZEXPORT zipCloseFileInZipRaw64 OF((zipFile file, uncompressed_size and crc32 are value for the uncompressed size */ -extern int ZEXPORT zipClose OF((zipFile file, - const char* global_comment)); +extern int ZEXPORT zipClose(zipFile file, + const char* global_comment); /* Close the zipfile */ -extern int ZEXPORT zipRemoveExtraInfoBlock OF((char* pData, int* dataLen, short sHeader)); +extern int ZEXPORT zipRemoveExtraInfoBlock(char* pData, int* dataLen, short sHeader); /* zipRemoveExtraInfoBlock - Added by Mathias Svensson diff --git a/storage/federatedx/federatedx_io_mysql.cc b/storage/federatedx/federatedx_io_mysql.cc index fc32146b..cc234f1e 100644 --- a/storage/federatedx/federatedx_io_mysql.cc +++ b/storage/federatedx/federatedx_io_mysql.cc @@ -451,11 +451,14 @@ int federatedx_io_mysql::actual_query(const char *buffer, size_t length) get_port(), get_socket(), 0)) DBUG_RETURN(ER_CONNECT_TO_FOREIGN_DATA_SOURCE); + + if ((error= mysql_real_query(&mysql, STRING_WITH_LEN("set time_zone='+00:00'")))) + DBUG_RETURN(error); + mysql.reconnect= 1; } - if (!(error= mysql_real_query(&mysql, STRING_WITH_LEN("set time_zone='+00:00'")))) - error= mysql_real_query(&mysql, buffer, (ulong)length); + error= mysql_real_query(&mysql, buffer, (ulong)length); DBUG_RETURN(error); } diff --git a/storage/innobase/btr/btr0btr.cc b/storage/innobase/btr/btr0btr.cc index 08be1991..705ff035 100644 --- a/storage/innobase/btr/btr0btr.cc +++ b/storage/innobase/btr/btr0btr.cc @@ -216,10 +216,11 @@ ATTRIBUTE_COLD void btr_decryption_failed(const dict_index_t &index) @param[in] merge whether change buffer merge should be attempted @param[in,out] mtr mini-transaction @param[out] err error code +@param[out] first set if this is a first-time access to the page @return block */ buf_block_t *btr_block_get(const dict_index_t &index, uint32_t page, rw_lock_type_t mode, bool merge, - mtr_t *mtr, dberr_t *err) + mtr_t *mtr, dberr_t *err, bool *first) { ut_ad(mode != RW_NO_LATCH); dberr_t local_err; @@ -242,6 +243,8 @@ buf_block_t *btr_block_get(const dict_index_t &index, *err= DB_PAGE_CORRUPTED; block= nullptr; } + else if (!buf_page_make_young_if_needed(&block->page) && first) + *first= true; } else if (*err == DB_DECRYPTION_FAILED) btr_decryption_failed(index); @@ -302,6 +305,8 @@ btr_root_block_get( *err= DB_CORRUPTION; block= nullptr; } + else + buf_page_make_young_if_needed(&block->page); } else if (*err == DB_DECRYPTION_FAILED) btr_decryption_failed(*index); @@ -553,8 +558,11 @@ btr_page_alloc_for_ibuf( root->page.frame)), 0, RW_X_LATCH, nullptr, BUF_GET, mtr, err); if (new_block) + { + buf_page_make_young_if_needed(&new_block->page); *err= flst_remove(root, PAGE_HEADER + PAGE_BTR_IBUF_FREE_LIST, new_block, PAGE_HEADER + PAGE_BTR_IBUF_FREE_LIST_NODE, mtr); + } ut_d(if (*err == DB_SUCCESS) flst_validate(root, PAGE_HEADER + PAGE_BTR_IBUF_FREE_LIST, mtr)); return new_block; @@ -873,7 +881,8 @@ static rec_offs *btr_page_get_parent(rec_offs *offsets, mem_heap_t *heap, /************************************************************//** Returns the upper level node pointer to a page. It is assumed that mtr holds an x-latch on the tree. -@return rec_get_offsets() of the node pointer record */ +@return rec_get_offsets() of the node pointer record +@retval nullptr on corruption */ static rec_offs* btr_page_get_father_block( @@ -1351,6 +1360,7 @@ btr_write_autoinc(dict_index_t* index, ib_uint64_t autoinc, bool reset) if (buf_block_t *root= buf_page_get(page_id_t(space->id, index->page), space->zip_size(), RW_SX_LATCH, &mtr)) { + buf_page_make_young_if_needed(&root->page); mtr.set_named_space(space); page_set_autoinc(root, autoinc, &mtr, reset); } @@ -2542,6 +2552,11 @@ btr_attach_half_pages( offsets = btr_page_get_father_block(nullptr, heap, mtr, &cursor); + if (UNIV_UNLIKELY(!offsets)) { + mem_heap_free(heap); + return DB_CORRUPTION; + } + /* Replace the address of the old child node (= page) with the address of the new lower half */ @@ -3478,6 +3493,14 @@ btr_lift_page_up( offsets = btr_page_get_father_block(offsets, heap, mtr, &cursor); } + + if (UNIV_UNLIKELY(!offsets)) { +parent_corrupted: + mem_heap_free(heap); + *err = DB_CORRUPTION; + return nullptr; + } + father_block = btr_cur_get_block(&cursor); father_page_zip = buf_block_get_page_zip(father_block); @@ -3502,6 +3525,10 @@ btr_lift_page_up( &cursor); } + if (UNIV_UNLIKELY(!offsets)) { + goto parent_corrupted; + } + blocks[n_blocks++] = b = btr_cur_get_block(&cursor); } @@ -3717,6 +3744,11 @@ btr_compress( NULL, heap, mtr, &father_cursor); } + if (UNIV_UNLIKELY(!offsets)) { + err = DB_CORRUPTION; + goto func_exit; + } + if (adjust) { nth_rec = page_rec_get_n_recs_before(btr_cur_get_rec(cursor)); if (UNIV_UNLIKELY(!nth_rec || nth_rec == ULINT_UNDEFINED)) { diff --git a/storage/innobase/btr/btr0bulk.cc b/storage/innobase/btr/btr0bulk.cc index 013cd131..5bf68c58 100644 --- a/storage/innobase/btr/btr0bulk.cc +++ b/storage/innobase/btr/btr0bulk.cc @@ -52,6 +52,7 @@ PageBulk::init() if (m_page_no == FIL_NULL) { mtr_t alloc_mtr; + dberr_t err= DB_SUCCESS; /* We commit redo log for allocation by a separate mtr, because we don't guarantee pages are committed following @@ -60,28 +61,15 @@ PageBulk::init() alloc_mtr.start(); m_index->set_modified(alloc_mtr); - uint32_t n_reserved; - dberr_t err = fsp_reserve_free_extents( - &n_reserved, m_index->table->space, 1, FSP_NORMAL, - &alloc_mtr); - if (UNIV_UNLIKELY(err != DB_SUCCESS)) { -oom: - alloc_mtr.commit(); - m_mtr.commit(); - return err; - } - /* Allocate a new page. */ new_block = btr_page_alloc(m_index, 0, FSP_UP, m_level, &alloc_mtr, &m_mtr, &err); + alloc_mtr.commit(); if (!new_block) { - goto oom; + m_mtr.commit(); + return err; } - m_index->table->space->release_free_extents(n_reserved); - - alloc_mtr.commit(); - new_page = buf_block_get_frame(new_block); m_page_no = new_block->page.id().page_no(); @@ -969,10 +957,10 @@ BtrBulk::pageCommit( /** Log free check */ inline void BtrBulk::logFreeCheck() { - if (log_sys.check_flush_or_checkpoint()) { + if (log_sys.check_for_checkpoint()) { release(); - log_check_margins(); + log_free_check(); latch(); } diff --git a/storage/innobase/btr/btr0cur.cc b/storage/innobase/btr/btr0cur.cc index e736f338..46afb73b 100644 --- a/storage/innobase/btr/btr0cur.cc +++ b/storage/innobase/btr/btr0cur.cc @@ -1156,6 +1156,19 @@ dberr_t btr_cur_t::search_leaf(const dtuple_t *tuple, page_cur_mode_t mode, mtr_s_lock_index(index(), mtr); } + dberr_t err; + + if (!index()->table->space) + { + corrupted: + ut_ad("corrupted" == 0); // FIXME: remove this + err= DB_CORRUPTION; + func_exit: + if (UNIV_LIKELY_NULL(heap)) + mem_heap_free(heap); + return err; + } + const ulint zip_size= index()->table->space->zip_size(); /* Start with the root page. */ @@ -1169,7 +1182,6 @@ dberr_t btr_cur_t::search_leaf(const dtuple_t *tuple, page_cur_mode_t mode, low_bytes= 0; ulint buf_mode= BUF_GET; search_loop: - dberr_t err; auto block_savepoint= mtr->get_savepoint(); buf_block_t *block= buf_page_get_gen(page_id, zip_size, rw_latch, guess, buf_mode, mtr, @@ -1181,10 +1193,7 @@ dberr_t btr_cur_t::search_leaf(const dtuple_t *tuple, page_cur_mode_t mode, btr_decryption_failed(*index()); /* fall through */ default: - func_exit: - if (UNIV_LIKELY_NULL(heap)) - mem_heap_free(heap); - return err; + goto func_exit; case DB_SUCCESS: /* This must be a search to perform an insert, delete mark, or delete; try using the change buffer */ @@ -1251,16 +1260,11 @@ dberr_t btr_cur_t::search_leaf(const dtuple_t *tuple, page_cur_mode_t mode, btr_page_get_index_id(block->page.frame) != index()->id || fil_page_get_type(block->page.frame) == FIL_PAGE_RTREE || !fil_page_index_page_check(block->page.frame)) - { - corrupted: - ut_ad("corrupted" == 0); // FIXME: remove this - err= DB_CORRUPTION; - goto func_exit; - } + goto corrupted; page_cur.block= block; ut_ad(block == mtr->at_savepoint(block_savepoint)); - ut_ad(rw_latch != RW_NO_LATCH); + const bool not_first_access{buf_page_make_young_if_needed(&block->page)}; #ifdef UNIV_ZIP_DEBUG if (const page_zip_des_t *page_zip= buf_block_get_page_zip(block)) ut_a(page_zip_validate(page_zip, block->page.frame, index())); @@ -1539,6 +1543,9 @@ release_tree: case BTR_SEARCH_PREV: /* btr_pcur_move_to_prev() */ ut_ad(rw_latch == RW_S_LATCH || rw_latch == RW_X_LATCH); + if (!not_first_access) + buf_read_ahead_linear(page_id, zip_size, false); + if (page_has_prev(block->page.frame) && page_rec_is_first(page_cur.rec, block->page.frame)) { @@ -1578,6 +1585,8 @@ release_tree: buf_mode= btr_op == BTR_DELETE_OP ? BUF_GET_IF_IN_POOL_OR_WATCH : BUF_GET_IF_IN_POOL; + else if (!not_first_access) + buf_read_ahead_linear(page_id, zip_size, false); break; case BTR_MODIFY_TREE: ut_ad(rw_latch == RW_X_LATCH); @@ -1611,6 +1620,14 @@ ATTRIBUTE_COLD void mtr_t::index_lock_upgrade() slot.type= MTR_MEMO_X_LOCK; } +/** Mark a non-leaf page "least recently used", but avoid invoking +buf_page_t::set_accessed(), because we do not want linear read-ahead */ +static void btr_cur_nonleaf_make_young(buf_page_t *bpage) +{ + if (UNIV_UNLIKELY(buf_page_peek_if_too_old(bpage))) + buf_page_make_young(bpage); +} + ATTRIBUTE_COLD dberr_t btr_cur_t::pessimistic_search_leaf(const dtuple_t *tuple, page_cur_mode_t mode, mtr_t *mtr) @@ -1713,6 +1730,8 @@ dberr_t btr_cur_t::pessimistic_search_leaf(const dtuple_t *tuple, if (height != btr_page_get_level(block->page.frame)) goto corrupted; + btr_cur_nonleaf_make_young(&block->page); + #ifdef UNIV_ZIP_DEBUG const page_zip_des_t *page_zip= buf_block_get_page_zip(block); ut_a(!page_zip || page_zip_validate(page_zip, block->page.frame, index())); @@ -1799,6 +1818,8 @@ search_loop: btr_decryption_failed(*index); goto func_exit; } + else + btr_cur_nonleaf_make_young(&block->page); #ifdef UNIV_ZIP_DEBUG if (const page_zip_des_t *page_zip= buf_block_get_page_zip(block)) @@ -1934,18 +1955,15 @@ index_locked: ut_ad(n_blocks < BTR_MAX_LEVELS); ut_ad(savepoint + n_blocks == mtr->get_savepoint()); + bool first_access= false; buf_block_t* block= btr_block_get(*index, page, height ? upper_rw_latch : root_leaf_rw_latch, - !height, mtr, &err); + !height, mtr, &err, &first_access); ut_ad(!block == (err != DB_SUCCESS)); if (!block) - { - if (err == DB_DECRYPTION_FAILED) - btr_decryption_failed(*index); break; - } if (first) page_cur_set_before_first(block, &page_cur); @@ -2029,10 +2047,16 @@ index_locked: offsets= rec_get_offsets(page_cur.rec, index, offsets, 0, ULINT_UNDEFINED, &heap); + page= btr_node_ptr_get_child_page_no(page_cur.rec, offsets); ut_ad(latch_mode != BTR_MODIFY_TREE || upper_rw_latch == RW_X_LATCH); - if (latch_mode != BTR_MODIFY_TREE); + if (latch_mode != BTR_MODIFY_TREE) + { + if (!height && first && first_access) + buf_read_ahead_linear(page_id_t(block->page.id().space(), page), + block->page.zip_size(), false); + } else if (btr_cur_need_opposite_intention(block->page, index->is_clust(), lock_intention, node_ptr_max_size, compress_limit, @@ -2070,7 +2094,6 @@ index_locked: } /* Go to the child node */ - page= btr_node_ptr_get_child_page_no(page_cur.rec, offsets); n_blocks++; } @@ -3837,22 +3860,14 @@ btr_cur_pess_upd_restore_supremum( const page_id_t block_id{block->page.id()}; const page_id_t prev_id(block_id.space(), prev_page_no); - dberr_t err; buf_block_t* prev_block - = buf_page_get_gen(prev_id, 0, RW_NO_LATCH, nullptr, - BUF_PEEK_IF_IN_POOL, mtr, &err); - /* Since we already held an x-latch on prev_block, it must - be available and not be corrupted unless the buffer pool got - corrupted somehow. */ + = mtr->get_already_latched(prev_id, MTR_MEMO_PAGE_X_FIX); if (UNIV_UNLIKELY(!prev_block)) { - return err; + return DB_CORRUPTION; } ut_ad(!memcmp_aligned<4>(prev_block->page.frame + FIL_PAGE_NEXT, block->page.frame + FIL_PAGE_OFFSET, 4)); - /* We must already have an x-latch on prev_block! */ - ut_ad(mtr->memo_contains_flagged(prev_block, MTR_MEMO_PAGE_X_FIX)); - lock_rec_reset_and_inherit_gap_locks(*prev_block, block_id, PAGE_HEAP_NO_SUPREMUM, page_rec_get_heap_no(rec)); @@ -6660,6 +6675,10 @@ btr_copy_blob_prefix( mtr.commit(); return copied_len; } + if (!buf_page_make_young_if_needed(&block->page)) { + buf_read_ahead_linear(id, 0, false); + } + page = buf_block_get_frame(block); blob_header = page + offset; diff --git a/storage/innobase/btr/btr0pcur.cc b/storage/innobase/btr/btr0pcur.cc index 54dd15ac..2131fb94 100644 --- a/storage/innobase/btr/btr0pcur.cc +++ b/storage/innobase/btr/btr0pcur.cc @@ -25,9 +25,10 @@ Created 2/23/1996 Heikki Tuuri *******************************************************/ #include "btr0pcur.h" -#include "ut0byte.h" +#include "buf0rea.h" #include "rem0cmp.h" #include "trx0trx.h" +#include "ibuf0ibuf.h" /**************************************************************//** Resets a persistent cursor object, freeing ::old_rec_buf if it is @@ -261,13 +262,15 @@ static bool btr_pcur_optimistic_latch_leaves(buf_block_t *block, buf_page_get_gen(page_id_t(id.space(), left_page_no), zip_size, mode, nullptr, BUF_GET_POSSIBLY_FREED, mtr); - if (left_block && - btr_page_get_next(left_block->page.frame) != id.page_no()) + if (!left_block); + else if (btr_page_get_next(left_block->page.frame) != id.page_no()) { release_left_block: mtr->release_last_page(); return false; } + else + buf_page_make_young_if_needed(&left_block->page); } if (buf_page_optimistic_get(mode, block, pcur->modify_clock, mtr)) @@ -539,10 +542,11 @@ btr_pcur_move_to_next_page( } dberr_t err; + bool first_access = false; buf_block_t* next_block = btr_block_get( *cursor->index(), next_page_no, rw_lock_type_t(cursor->latch_mode & (RW_X_LATCH | RW_S_LATCH)), - page_is_leaf(page), mtr, &err); + page_is_leaf(page), mtr, &err, &first_access); if (UNIV_UNLIKELY(!next_block)) { return err; @@ -561,6 +565,11 @@ btr_pcur_move_to_next_page( const auto s = mtr->get_savepoint(); mtr->rollback_to_savepoint(s - 2, s - 1); + if (first_access) { + buf_read_ahead_linear(next_block->page.id(), + next_block->zip_size(), + ibuf_inside(mtr)); + } return DB_SUCCESS; } diff --git a/storage/innobase/btr/btr0sea.cc b/storage/innobase/btr/btr0sea.cc index 8435047c..1c5928c4 100644 --- a/storage/innobase/btr/btr0sea.cc +++ b/storage/innobase/btr/btr0sea.cc @@ -1143,7 +1143,6 @@ block_and_ahi_release_and_fail: } block->page.fix(); - block->page.set_accessed(); buf_page_make_young_if_needed(&block->page); static_assert(ulint{MTR_MEMO_PAGE_S_FIX} == ulint{BTR_SEARCH_LEAF}, ""); diff --git a/storage/innobase/buf/buf0buf.cc b/storage/innobase/buf/buf0buf.cc index 8ef18ee0..23b5b776 100644 --- a/storage/innobase/buf/buf0buf.cc +++ b/storage/innobase/buf/buf0buf.cc @@ -404,7 +404,7 @@ static bool buf_page_decrypt_after_read(buf_page_t *bpage, if (id.space() == SRV_TMP_SPACE_ID && innodb_encrypt_temporary_tables) { - slot = buf_pool.io_buf_reserve(); + slot = buf_pool.io_buf_reserve(false); slot->allocate(); bool ok = buf_tmp_page_decrypt(slot->crypt_buf, dst_frame); slot->release(); @@ -426,7 +426,7 @@ decompress: return false; } - slot = buf_pool.io_buf_reserve(); + slot = buf_pool.io_buf_reserve(false); slot->allocate(); decompress_with_slot: @@ -449,7 +449,7 @@ decrypt_failed: return false; } - slot = buf_pool.io_buf_reserve(); + slot = buf_pool.io_buf_reserve(false); slot->allocate(); /* decrypt using crypt_buf to dst_frame */ @@ -742,6 +742,205 @@ bool buf_page_is_corrupted(bool check_lsn, const byte *read_buf, #ifndef UNIV_INNOCHECKSUM +#ifdef __linux__ +#include <poll.h> +#include <sys/eventfd.h> +#include <fstream> + +/** Memory Pressure + +based off https://www.kernel.org/doc/html/latest/accounting/psi.html#pressure-interface +and https://www.kernel.org/doc/html/latest/admin-guide/cgroup-v2.html#memory */ +class mem_pressure +{ + /* triggers + eventfd */ + struct pollfd m_fds[3]; + nfds_t m_num_fds; + int m_event_fd= -1; + Atomic_relaxed<bool> m_abort= false; + + std::thread m_thd; + /* mem pressure garbage collection restricted to interval */ + static constexpr ulonglong max_interval_us= 60*1000000; + +public: + mem_pressure() : m_num_fds(0) {} + + bool setup() + { + static_assert(array_elements(m_fds) == (array_elements(m_triggers) + 1), + "insufficient fds"); + std::string memcgroup{"/sys/fs/cgroup"}; + std::string cgroup; + { + std::ifstream selfcgroup("/proc/self/cgroup"); + std::getline(selfcgroup, cgroup, '\n'); + } + + cgroup.erase(0, 3); // Remove "0::" + memcgroup+= cgroup + "/memory.pressure"; + + m_num_fds= 0; + for (auto trig= std::begin(m_triggers); trig!= std::end(m_triggers); ++trig) + { + if ((m_fds[m_num_fds].fd= + open(memcgroup.c_str(), O_RDWR | O_NONBLOCK | O_CLOEXEC)) < 0) + { + /* User can't do anything about it, no point giving warning */ + shutdown(); + return false; + } + my_register_filename(m_fds[m_num_fds].fd, memcgroup.c_str(), FILE_BY_OPEN, 0, MYF(0)); + ssize_t slen= strlen(*trig); + if (write(m_fds[m_num_fds].fd, *trig, slen) < slen) + { + /* we may fail this one, but continue to the next */ + my_close(m_fds[m_num_fds].fd, MYF(MY_WME)); + continue; + } + m_fds[m_num_fds].events= POLLPRI; + m_num_fds++; + } + if (m_num_fds < 1) + return false; + + if ((m_event_fd= eventfd(0, EFD_CLOEXEC|EFD_NONBLOCK)) == -1) + { + /* User can't do anything about it, no point giving warning */ + shutdown(); + return false; + } + my_register_filename(m_event_fd, "mem_pressure_eventfd", FILE_BY_DUP, 0, MYF(0)); + m_fds[m_num_fds].fd= m_event_fd; + m_fds[m_num_fds].events= POLLIN; + m_num_fds++; + m_thd= std::thread(pressure_routine, this); + sql_print_information("InnoDB: Initialized memory pressure event listener"); + return true; + } + + void shutdown() + { + /* m_event_fd is in this list */ + while (m_num_fds) + { + m_num_fds--; + my_close(m_fds[m_num_fds].fd, MYF(MY_WME)); + m_fds[m_num_fds].fd= -1; + } + } + + static void pressure_routine(mem_pressure *m); + +#ifdef UNIV_DEBUG + void trigger_collection() + { + uint64_t u= 1; + if (m_event_fd >=0 && write(m_event_fd, &u, sizeof(uint64_t)) != sizeof(uint64_t)) + sql_print_information("InnoDB: (Debug) Failed to trigger memory pressure"); + else /* assumed failed to meet intialization criteria, so trigger directy */ + buf_pool.garbage_collect(); + } +#endif + + void quit() + { + uint64_t u= 1; + m_abort= true; +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-result" + /* return result ignored, cannot do anything with it */ + write(m_event_fd, &u, sizeof(uint64_t)); +#pragma GCC diagnostic pop + } + + void join() + { + if (m_thd.joinable()) + { + quit(); + m_thd.join(); + } + } + + static const char* const m_triggers[2]; +}; + + +/* + ref: https://docs.kernel.org/accounting/psi.html + maximum window size (second number) 10 seconds. + window size in multiples of 2 second interval required (for Unprivileged) + Time is in usec. +*/ +const char* const mem_pressure::m_triggers[]= + {"some 5000000 10000000", /* 5s out of 10s */ + "full 10000 2000000"}; /* 10ms out of 2s */ + +static mem_pressure mem_pressure_obj; + +void mem_pressure::pressure_routine(mem_pressure *m) +{ + DBUG_ASSERT(m == &mem_pressure_obj); + if (my_thread_init()) + { + m->shutdown(); + return; + } + + ulonglong last= microsecond_interval_timer() - max_interval_us; + while (!m->m_abort) + { + if (poll(&m->m_fds[0], m->m_num_fds, -1) < 0) + { + if (errno == EINTR) + continue; + else + break; + } + if (!m->m_abort) + break; + + for (pollfd &p : st_::span<pollfd>(m->m_fds, m->m_num_fds)) + { + if (p.revents & POLLPRI) + { + ulonglong now= microsecond_interval_timer(); + if ((now - last) > max_interval_us) + { + last= now; + buf_pool.garbage_collect(); + } + } + +#ifdef UNIV_DEBUG + if (p.revents & POLLIN) + { + uint64_t u; + /* we haven't aborted, so this must be a debug trigger */ + if (read(p.fd, &u, sizeof(u)) >=0) + buf_pool.garbage_collect(); + } +#endif + } + } + m->shutdown(); + + my_thread_end(); +} + +/** Initialize mem pressure. */ +ATTRIBUTE_COLD void buf_mem_pressure_detect_init() +{ + mem_pressure_obj.setup(); +} + +ATTRIBUTE_COLD void buf_mem_pressure_shutdown() +{ + mem_pressure_obj.join(); +} +#endif /* __linux__ */ + #if defined(DBUG_OFF) && defined(HAVE_MADVISE) && defined(MADV_DODUMP) /** Enable buffers to be dumped to core files @@ -1099,6 +1298,11 @@ bool buf_pool_t::create() chunk_t::map_ref= chunk_t::map_reg; buf_LRU_old_ratio_update(100 * 3 / 8, false); btr_search_sys_create(); + +#ifdef __linux__ + if (srv_operation == SRV_OPERATION_NORMAL) + buf_mem_pressure_detect_init(); +#endif ut_ad(is_initialised()); return false; } @@ -1300,14 +1504,17 @@ void buf_pool_t::io_buf_t::close() n_slots= 0; } -buf_tmp_buffer_t *buf_pool_t::io_buf_t::reserve() +buf_tmp_buffer_t *buf_pool_t::io_buf_t::reserve(bool wait_for_reads) { for (;;) { for (buf_tmp_buffer_t *s= slots, *e= slots + n_slots; s != e; s++) if (s->acquire()) return s; + buf_dblwr.flush_buffered_writes(); os_aio_wait_until_no_pending_writes(true); + if (!wait_for_reads) + continue; for (buf_tmp_buffer_t *s= slots, *e= slots + n_slots; s != e; s++) if (s->acquire()) return s; @@ -1536,6 +1743,7 @@ struct find_interesting_trx inline void buf_pool_t::resize() { ut_ad(this == &buf_pool); + ut_ad(srv_shutdown_state < SRV_SHUTDOWN_CLEANUP); bool warning = false; @@ -1878,6 +2086,100 @@ calc_buf_pool_size: return; } +#ifdef __linux__ +inline void buf_pool_t::garbage_collect() +{ + mysql_mutex_lock(&mutex); + size_t freed= 0; + +#ifdef BTR_CUR_HASH_ADAPT + /* buf_LRU_free_page() will temporarily release and reacquire + buf_pool.mutex for invoking btr_search_drop_page_hash_index(). Thus, + we must protect ourselves with the hazard pointer. */ +rescan: +#else + lru_hp.set(nullptr); +#endif + for (buf_page_t *bpage= UT_LIST_GET_LAST(LRU), *prev; bpage; bpage= prev) + { + prev= UT_LIST_GET_PREV(LRU, bpage); +#ifdef BTR_CUR_HASH_ADAPT + lru_hp.set(prev); +#endif + auto state= bpage->state(); + ut_ad(state >= buf_page_t::FREED); + ut_ad(bpage->in_LRU_list); + + /* We try to free any pages that can be freed without writing out + anything. */ + switch (bpage->oldest_modification()) { + case 0: + try_to_evict: + if (buf_LRU_free_page(bpage, true)) + { + evicted: + freed++; +#ifdef BTR_CUR_HASH_ADAPT + bpage= prev; + prev= lru_hp.get(); + if (!prev && bpage) + goto rescan; +#endif + } + continue; + case 1: + break; + default: + if (state >= buf_page_t::UNFIXED) + continue; + } + + if (state < buf_page_t::READ_FIX && bpage->lock.u_lock_try(true)) + { + ut_ad(!bpage->is_io_fixed()); + lsn_t oldest_modification= bpage->oldest_modification(); + switch (oldest_modification) { + case 1: + mysql_mutex_lock(&flush_list_mutex); + oldest_modification= bpage->oldest_modification(); + if (oldest_modification) + { + ut_ad(oldest_modification == 1); + delete_from_flush_list(bpage); + } + mysql_mutex_unlock(&flush_list_mutex); + /* fall through */ + case 0: + bpage->lock.u_unlock(true); + goto try_to_evict; + default: + if (bpage->state() < buf_page_t::UNFIXED && + oldest_modification <= log_sys.get_flushed_lsn()) + { + release_freed_page(bpage); + goto evicted; + } + else + bpage->lock.u_unlock(true); + } + } + } + +#if defined MADV_FREE + /* FIXME: Issue fewer calls for larger contiguous blocks of + memory. For now, we assume that this is acceptable, because this + code should be executed rarely. */ + for (buf_page_t *bpage= UT_LIST_GET_FIRST(free); bpage; + bpage= UT_LIST_GET_NEXT(list, bpage)) + madvise(bpage->frame, srv_page_size, MADV_FREE); +#endif + mysql_mutex_unlock(&mutex); + sql_print_information("InnoDB: Memory pressure event freed %zu pages", + freed); + return; +} +#endif /* __linux__ */ + /** Thread pool task invoked by innodb_buffer_pool_size changes. */ static void buf_resize_callback(void *) { @@ -1906,12 +2208,23 @@ static tpool::waitable_task buf_resize_task(buf_resize_callback, void buf_resize_start() { - srv_thread_pool->submit_task(&buf_resize_task); +#if !defined(DBUG_OFF) && defined(__linux__) + DBUG_EXECUTE_IF("trigger_garbage_collection", + { + mem_pressure_obj.trigger_collection(); + } + ); +#endif + + srv_thread_pool->submit_task(&buf_resize_task); } void buf_resize_shutdown() { - buf_resize_task.wait(); +#ifdef __linux__ + buf_mem_pressure_shutdown(); +#endif + buf_resize_task.wait(); } @@ -2220,14 +2533,21 @@ lookup: if (discard_attempted || !bpage->frame) { - /* Even when we are holding a hash_lock, it should be - acceptable to wait for a page S-latch here, because - buf_page_t::read_complete() will not wait for buf_pool.mutex, - and because S-latch would not conflict with a U-latch - that would be protecting buf_page_t::write_complete(). */ - bpage->lock.s_lock(); + const bool got_s_latch= bpage->lock.s_lock_try(); hash_lock.unlock_shared(); - break; + if (UNIV_LIKELY(got_s_latch)) + break; + /* We may fail to acquire bpage->lock because + buf_page_t::read_complete() may be invoking + buf_pool_t::corrupted_evict() on this block, which it would + hold an exclusive latch on. + + Let us aqcuire and release buf_pool.mutex to ensure that any + buf_pool_t::corrupted_evict() will proceed before we reacquire + the hash_lock that it could be waiting for. */ + mysql_mutex_lock(&buf_pool.mutex); + mysql_mutex_unlock(&buf_pool.mutex); + goto lookup; } hash_lock.unlock_shared(); @@ -2246,7 +2566,6 @@ lookup: ut_ad(s < buf_page_t::READ_FIX || s >= buf_page_t::WRITE_FIX); } - bpage->set_accessed(); buf_page_make_young_if_needed(bpage); #ifdef UNIV_DEBUG @@ -2873,18 +3192,6 @@ get_latch_valid: ut_ad(page_id_t(page_get_space_id(block->page.frame), page_get_page_no(block->page.frame)) == page_id); - - if (mode == BUF_GET_POSSIBLY_FREED - || mode == BUF_PEEK_IF_IN_POOL) { - return block; - } - - const bool not_first_access{block->page.set_accessed()}; - buf_page_make_young_if_needed(&block->page); - if (!not_first_access) { - buf_read_ahead_linear(page_id, block->zip_size(), - ibuf_inside(mtr)); - } } return block; @@ -3057,7 +3364,6 @@ bool buf_page_optimistic_get(ulint rw_latch, buf_block_t *block, block->page.fix(); ut_ad(!block->page.is_read_fixed()); - block->page.set_accessed(); buf_page_make_young_if_needed(&block->page); mtr->memo_push(block, mtr_memo_type_t(rw_latch)); } diff --git a/storage/innobase/buf/buf0dblwr.cc b/storage/innobase/buf/buf0dblwr.cc index e9aea355..e2702adc 100644 --- a/storage/innobase/buf/buf0dblwr.cc +++ b/storage/innobase/buf/buf0dblwr.cc @@ -336,11 +336,14 @@ func_exit: os_file_flush(file); } else - for (ulint i= 0; i < size * 2; i++, page += srv_page_size) - if (mach_read_from_8(my_assume_aligned<8>(page + FIL_PAGE_LSN))) - /* Each valid page header must contain a nonzero FIL_PAGE_LSN field. */ + { + alignas(8) char checkpoint[8]; + mach_write_to_8(checkpoint, log_sys.next_checkpoint_lsn); + for (auto i= size * 2; i--; page += srv_page_size) + if (memcmp_aligned<8>(page + FIL_PAGE_LSN, checkpoint, 8) >= 0) + /* Valid pages are not older than the log checkpoint. */ recv_sys.dblwr.add(page); - + } err= DB_SUCCESS; goto func_exit; } diff --git a/storage/innobase/buf/buf0dump.cc b/storage/innobase/buf/buf0dump.cc index 957632db..cc51f8c6 100644 --- a/storage/innobase/buf/buf0dump.cc +++ b/storage/innobase/buf/buf0dump.cc @@ -33,7 +33,7 @@ Created April 08, 2011 Vasil Dimov #include "buf0rea.h" #include "buf0dump.h" -#include "dict0dict.h" +#include "dict0load.h" #include "os0file.h" #include "srv0srv.h" #include "srv0start.h" @@ -180,7 +180,7 @@ static void buf_dump_generate_path(char *path, size_t path_size) char buf[FN_REFLEN]; mysql_mutex_lock(&LOCK_global_system_variables); - snprintf(buf, sizeof buf, "%s/%s", get_buf_dump_dir(), + snprintf(buf, sizeof buf, "%s" FN_ROOTDIR "%s", get_buf_dump_dir(), srv_buf_dump_filename); mysql_mutex_unlock(&LOCK_global_system_variables); @@ -214,7 +214,7 @@ static void buf_dump_generate_path(char *path, size_t path_size) format = "%s%s"; break; default: - format = "%s/%s"; + format = "%s" FN_ROOTDIR "%s"; } snprintf(path, path_size, format, @@ -562,6 +562,22 @@ buf_load() if (!SHUTTING_DOWN()) { std::sort(dump, dump + dump_n); + std::set<uint32_t> missing; + for (const page_id_t id : st_::span<const page_id_t> + (dump, dump_n)) { + missing.emplace(id.space()); + } + for (std::set<uint32_t>::iterator i = missing.begin(); + i != missing.end(); ) { + auto j = i++; + if (fil_space_t* space = fil_space_t::get(*j)) { + space->release(); + missing.erase(j); + } + } + if (!missing.empty()) { + dict_check_tablespaces_and_store_max_id(&missing); + } } /* Avoid calling the expensive fil_space_t::get() for each diff --git a/storage/innobase/buf/buf0flu.cc b/storage/innobase/buf/buf0flu.cc index b6357989..d4628985 100644 --- a/storage/innobase/buf/buf0flu.cc +++ b/storage/innobase/buf/buf0flu.cc @@ -655,7 +655,7 @@ static byte *buf_page_encrypt(fil_space_t* space, buf_page_t* bpage, byte* s, ut_ad(!bpage->zip_size() || !page_compressed); /* Find free slot from temporary memory array */ - *slot= buf_pool.io_buf_reserve(); + *slot= buf_pool.io_buf_reserve(true); ut_a(*slot); (*slot)->allocate(); @@ -754,16 +754,20 @@ bool buf_page_t::flush(bool evict, fil_space_t *space) ut_ad(space->referenced()); const auto s= state(); - ut_a(s >= FREED); + + const lsn_t lsn= + mach_read_from_8(my_assume_aligned<8> + (FIL_PAGE_LSN + (zip.data ? zip.data : frame))); + ut_ad(lsn + ? lsn >= oldest_modification() || oldest_modification() == 2 + : space->purpose != FIL_TYPE_TABLESPACE); if (s < UNFIXED) { + ut_a(s >= FREED); if (UNIV_LIKELY(space->purpose == FIL_TYPE_TABLESPACE)) { - const lsn_t lsn= - mach_read_from_8(my_assume_aligned<8> - (FIL_PAGE_LSN + (zip.data ? zip.data : frame))); - ut_ad(lsn >= oldest_modification()); + freed: if (lsn > log_sys.get_flushed_lsn()) { mysql_mutex_unlock(&buf_pool.mutex); @@ -775,6 +779,12 @@ bool buf_page_t::flush(bool evict, fil_space_t *space) return false; } + if (UNIV_UNLIKELY(lsn < space->get_create_lsn())) + { + ut_ad(space->purpose == FIL_TYPE_TABLESPACE); + goto freed; + } + ut_d(const auto f=) zip.fix.fetch_add(WRITE_FIX - UNFIXED); ut_ad(f >= UNFIXED); ut_ad(f < READ_FIX); @@ -869,15 +879,9 @@ bool buf_page_t::flush(bool evict, fil_space_t *space) if ((s & LRU_MASK) == REINIT || !space->use_doublewrite()) { - if (UNIV_LIKELY(space->purpose == FIL_TYPE_TABLESPACE)) - { - const lsn_t lsn= - mach_read_from_8(my_assume_aligned<8>(FIL_PAGE_LSN + - (write_frame ? write_frame - : frame))); - ut_ad(lsn >= oldest_modification()); + if (UNIV_LIKELY(space->purpose == FIL_TYPE_TABLESPACE) && + lsn > log_sys.get_flushed_lsn()) log_write_up_to(lsn, true); - } space->io(IORequest{type, this, slot}, physical_offset(), size, write_frame, this); } @@ -1057,11 +1061,25 @@ static ulint buf_flush_try_neighbors(fil_space_t *space, bool contiguous, bool evict, ulint n_flushed, ulint n_to_flush) { - mysql_mutex_unlock(&buf_pool.mutex); - ut_ad(space->id == page_id.space()); ut_ad(bpage->id() == page_id); + { + const lsn_t lsn= + mach_read_from_8(my_assume_aligned<8> + (FIL_PAGE_LSN + + (bpage->zip.data ? bpage->zip.data : bpage->frame))); + ut_ad(lsn >= bpage->oldest_modification()); + if (UNIV_UNLIKELY(lsn < space->get_create_lsn())) + { + ut_a(!bpage->flush(evict, space)); + mysql_mutex_unlock(&buf_pool.mutex); + return 0; + } + } + + mysql_mutex_unlock(&buf_pool.mutex); + ulint count= 0; page_id_t id= page_id; page_id_t high= buf_flush_check_neighbors(*space, id, contiguous, evict); @@ -1741,6 +1759,28 @@ ulint buf_flush_LRU(ulint max_n, bool evict) buf_pool.try_LRU_scan= true; pthread_cond_broadcast(&buf_pool.done_free); } + else if (!pages && !buf_pool.try_LRU_scan && + !buf_pool.LRU_warned.test_and_set(std::memory_order_acquire)) + { + /* For example, with the minimum innodb_buffer_pool_size=5M and + the default innodb_page_size=16k there are only a little over 316 + pages in the buffer pool. The buffer pool can easily be exhausted + by a workload of some dozen concurrent connections. The system could + reach a deadlock like the following: + + (1) Many threads are waiting in buf_LRU_get_free_block() + for buf_pool.done_free. + (2) Some threads are waiting for a page latch which is held by + another thread that is waiting in buf_LRU_get_free_block(). + (3) This thread is the only one that could make progress, but + we fail to do so because all the pages that we scanned are + buffer-fixed or latched by some thread. */ + sql_print_warning("InnoDB: Could not free any blocks in the buffer pool!" + " %zu blocks are in use and %zu free." + " Consider increasing innodb_buffer_pool_size.", + UT_LIST_GET_LEN(buf_pool.LRU), + UT_LIST_GET_LEN(buf_pool.free)); + } return pages; } @@ -2124,6 +2164,8 @@ ATTRIBUTE_COLD void buf_flush_ahead(lsn_t lsn, bool furious) limit= lsn; buf_pool.page_cleaner_set_idle(false); pthread_cond_signal(&buf_pool.do_flush_list); + if (furious) + log_sys.set_check_for_checkpoint(); } mysql_mutex_unlock(&buf_pool.flush_list_mutex); } @@ -2371,11 +2413,19 @@ func_exit: goto func_exit; } +TPOOL_SUPPRESS_TSAN +bool buf_pool_t::need_LRU_eviction() const +{ + /* try_LRU_scan==false means that buf_LRU_get_free_block() is waiting + for buf_flush_page_cleaner() to evict some blocks */ + return UNIV_UNLIKELY(!try_LRU_scan || + (UT_LIST_GET_LEN(LRU) > BUF_LRU_MIN_LEN && + UT_LIST_GET_LEN(free) < srv_LRU_scan_depth / 2)); +} + #if defined __aarch64__&&defined __GNUC__&&__GNUC__==4&&!defined __clang__ -/* Avoid GCC 4.8.5 internal compiler error "could not split insn". -We would only need this for buf_flush_page_cleaner(), -but GCC 4.8.5 does not support pop_options. */ -# pragma GCC optimize ("O0") +/* Avoid GCC 4.8.5 internal compiler error "could not split insn". */ +__attribute__((optimize(0))) #endif /** page_cleaner thread tasked with flushing dirty pages from the buffer pools. As of now we'll have only one coordinator. */ @@ -2409,21 +2459,24 @@ static void buf_flush_page_cleaner() } mysql_mutex_lock(&buf_pool.flush_list_mutex); - if (buf_pool.ran_out()) - goto no_wait; - else if (srv_shutdown_state > SRV_SHUTDOWN_INITIATED) - break; + if (!buf_pool.need_LRU_eviction()) + { + if (srv_shutdown_state > SRV_SHUTDOWN_INITIATED) + break; - if (buf_pool.page_cleaner_idle() && - (!UT_LIST_GET_LEN(buf_pool.flush_list) || - srv_max_dirty_pages_pct_lwm == 0.0)) - /* We are idle; wait for buf_pool.page_cleaner_wakeup() */ - my_cond_wait(&buf_pool.do_flush_list, - &buf_pool.flush_list_mutex.m_mutex); - else - my_cond_timedwait(&buf_pool.do_flush_list, - &buf_pool.flush_list_mutex.m_mutex, &abstime); - no_wait: + if (buf_pool.page_cleaner_idle() && + (!UT_LIST_GET_LEN(buf_pool.flush_list) || + srv_max_dirty_pages_pct_lwm == 0.0)) + { + buf_pool.LRU_warned.clear(std::memory_order_release); + /* We are idle; wait for buf_pool.page_cleaner_wakeup() */ + my_cond_wait(&buf_pool.do_flush_list, + &buf_pool.flush_list_mutex.m_mutex); + } + else + my_cond_timedwait(&buf_pool.do_flush_list, + &buf_pool.flush_list_mutex.m_mutex, &abstime); + } set_timespec(abstime, 1); lsn_limit= buf_flush_sync_lsn; @@ -2445,9 +2498,9 @@ static void buf_flush_page_cleaner() do { - DBUG_EXECUTE_IF("ib_log_checkpoint_avoid", continue;); - DBUG_EXECUTE_IF("ib_log_checkpoint_avoid_hard", continue;); - + IF_DBUG(if (_db_keyword_(nullptr, "ib_log_checkpoint_avoid", 1) || + _db_keyword_(nullptr, "ib_log_checkpoint_avoid_hard", 1)) + continue,); if (!recv_recovery_is_on() && !srv_startup_is_before_trx_rollback_phase && srv_operation <= SRV_OPERATION_EXPORT_RESTORED) @@ -2455,7 +2508,7 @@ static void buf_flush_page_cleaner() } while (false); - if (!buf_pool.ran_out()) + if (!buf_pool.need_LRU_eviction()) continue; mysql_mutex_lock(&buf_pool.flush_list_mutex); oldest_lsn= buf_pool.get_oldest_modification(0); @@ -2484,7 +2537,7 @@ static void buf_flush_page_cleaner() if (oldest_lsn >= soft_lsn_limit) buf_flush_async_lsn= soft_lsn_limit= 0; } - else if (buf_pool.ran_out()) + else if (buf_pool.need_LRU_eviction()) { buf_pool.page_cleaner_set_idle(false); buf_pool.n_flush_inc(); @@ -2549,10 +2602,11 @@ static void buf_flush_page_cleaner() else { maybe_unemployed: - const bool below{dirty_pct < pct_lwm}; - pct_lwm= 0.0; - if (below) + if (dirty_pct < pct_lwm) + { + pct_lwm= 0.0; goto possibly_unemployed; + } } } else if (dirty_pct < srv_max_buf_pool_modified_pct) @@ -2598,9 +2652,13 @@ static void buf_flush_page_cleaner() MONITOR_FLUSH_ADAPTIVE_PAGES, n_flushed); } - else if (buf_flush_async_lsn <= oldest_lsn) + else if (buf_flush_async_lsn <= oldest_lsn && + !buf_pool.need_LRU_eviction()) goto check_oldest_and_set_idle; + else + mysql_mutex_lock(&buf_pool.mutex); + n= srv_max_io_capacity; n= n >= n_flushed ? n - n_flushed : 0; goto LRU_flush; } diff --git a/storage/innobase/buf/buf0lru.cc b/storage/innobase/buf/buf0lru.cc index 65ee8fa3..2a8d6ff2 100644 --- a/storage/innobase/buf/buf0lru.cc +++ b/storage/innobase/buf/buf0lru.cc @@ -60,10 +60,6 @@ static constexpr ulint BUF_LRU_OLD_TOLERANCE = 20; frames in the buffer pool, we set this to TRUE */ static bool buf_lru_switched_on_innodb_mon = false; -/** True if diagnostic message about difficult to find free blocks -in the buffer bool has already printed. */ -static bool buf_lru_free_blocks_error_printed; - /******************************************************************//** These statistics are not 'of' LRU but 'for' LRU. We keep count of I/O and page_zip_decompress() operations. Based on the statistics, @@ -408,6 +404,7 @@ got_mutex: buf_LRU_check_size_of_non_data_objects(); buf_block_t* block; + IF_DBUG(static bool buf_lru_free_blocks_error_printed,); DBUG_EXECUTE_IF("ib_lru_force_no_free_page", if (!buf_lru_free_blocks_error_printed) { n_iterations = 21; @@ -417,9 +414,25 @@ retry: /* If there is a block in the free list, take it */ if ((block = buf_LRU_get_free_only()) != nullptr) { got_block: + const ulint LRU_size = UT_LIST_GET_LEN(buf_pool.LRU); + const ulint available = UT_LIST_GET_LEN(buf_pool.free); + const ulint scan_depth = srv_LRU_scan_depth / 2; + ut_ad(LRU_size <= BUF_LRU_MIN_LEN || available >= scan_depth + || buf_pool.need_LRU_eviction()); + if (!have_mutex) { mysql_mutex_unlock(&buf_pool.mutex); } + + if (UNIV_UNLIKELY(available < scan_depth) + && LRU_size > BUF_LRU_MIN_LEN) { + mysql_mutex_lock(&buf_pool.flush_list_mutex); + if (!buf_pool.page_cleaner_active()) { + buf_pool.page_cleaner_wakeup(true); + } + mysql_mutex_unlock(&buf_pool.flush_list_mutex); + } + block->page.zip.clear(); return block; } @@ -445,10 +458,11 @@ got_block: if ((block = buf_LRU_get_free_only()) != nullptr) { goto got_block; } + const bool wake = buf_pool.need_LRU_eviction(); mysql_mutex_unlock(&buf_pool.mutex); mysql_mutex_lock(&buf_pool.flush_list_mutex); const auto n_flush = buf_pool.n_flush(); - if (!buf_pool.try_LRU_scan) { + if (wake && !buf_pool.page_cleaner_active()) { buf_pool.page_cleaner_wakeup(true); } mysql_mutex_unlock(&buf_pool.flush_list_mutex); @@ -467,9 +481,10 @@ not_found: MONITOR_INC( MONITOR_LRU_GET_FREE_WAITS ); } - if (n_iterations == 21 && !buf_lru_free_blocks_error_printed - && srv_buf_pool_old_size == srv_buf_pool_size) { - buf_lru_free_blocks_error_printed = true; + if (n_iterations == 21 + && srv_buf_pool_old_size == srv_buf_pool_size + && buf_pool.LRU_warned.test_and_set(std::memory_order_acquire)) { + IF_DBUG(buf_lru_free_blocks_error_printed = true,); mysql_mutex_unlock(&buf_pool.mutex); ib::warn() << "Difficult to find free blocks in the buffer pool" " (" << n_iterations << " search iterations)! " @@ -787,6 +802,14 @@ void buf_page_make_young(buf_page_t *bpage) mysql_mutex_unlock(&buf_pool.mutex); } +bool buf_page_make_young_if_needed(buf_page_t *bpage) +{ + const bool not_first{bpage->set_accessed()}; + if (UNIV_UNLIKELY(buf_page_peek_if_too_old(bpage))) + buf_page_make_young(bpage); + return not_first; +} + /** Try to free a block. If bpage is a descriptor of a compressed-only ROW_FORMAT=COMPRESSED page, the buf_page_t object will be freed as well. The caller must hold buf_pool.mutex. diff --git a/storage/innobase/buf/buf0rea.cc b/storage/innobase/buf/buf0rea.cc index c4f07738..9041c6a2 100644 --- a/storage/innobase/buf/buf0rea.cc +++ b/storage/innobase/buf/buf0rea.cc @@ -597,6 +597,12 @@ failed: uint32_t prev= mach_read_from_4(my_assume_aligned<4>(f + FIL_PAGE_PREV)); uint32_t next= mach_read_from_4(my_assume_aligned<4>(f + FIL_PAGE_NEXT)); hash_lock.unlock_shared(); + /* The underlying file page of this buffer pool page could actually + be marked as freed, or a read of the page into the buffer pool might + be in progress. We may read uninitialized data here. + Suppress warnings of comparing uninitialized values. */ + MEM_MAKE_DEFINED(&prev, sizeof prev); + MEM_MAKE_DEFINED(&next, sizeof next); if (prev == FIL_NULL || next == FIL_NULL) goto fail; page_id_t id= page_id; diff --git a/storage/innobase/dict/dict0boot.cc b/storage/innobase/dict/dict0boot.cc index 5516bce9..cb60d813 100644 --- a/storage/innobase/dict/dict0boot.cc +++ b/storage/innobase/dict/dict0boot.cc @@ -42,7 +42,10 @@ static constexpr page_id_t hdr_page_id{DICT_HDR_SPACE, DICT_HDR_PAGE_NO}; static buf_block_t *dict_hdr_get(mtr_t *mtr) { /* We assume that the DICT_HDR page is always readable and available. */ - return buf_page_get_gen(hdr_page_id, 0, RW_X_LATCH, nullptr, BUF_GET, mtr); + buf_block_t *b= + buf_page_get_gen(hdr_page_id, 0, RW_X_LATCH, nullptr, BUF_GET, mtr); + buf_page_make_young_if_needed(&b->page); + return b; } /**********************************************************************//** diff --git a/storage/innobase/dict/dict0crea.cc b/storage/innobase/dict/dict0crea.cc index cce5f2f2..dd858287 100644 --- a/storage/innobase/dict/dict0crea.cc +++ b/storage/innobase/dict/dict0crea.cc @@ -353,9 +353,6 @@ dict_build_table_def_step( /* Always set this bit for all new created tables */ DICT_TF2_FLAG_SET(table, DICT_TF2_FTS_AUX_HEX_NAME); - DBUG_EXECUTE_IF("innodb_test_wrong_fts_aux_table_name", - DICT_TF2_FLAG_UNSET(table, - DICT_TF2_FTS_AUX_HEX_NAME);); if (DICT_TF2_FLAG_IS_SET(table, DICT_TF2_USE_FILE_PER_TABLE)) { /* This table will need a new tablespace. */ diff --git a/storage/innobase/dict/dict0dict.cc b/storage/innobase/dict/dict0dict.cc index 5bc7ab6e..5d3cab17 100644 --- a/storage/innobase/dict/dict0dict.cc +++ b/storage/innobase/dict/dict0dict.cc @@ -2809,8 +2809,7 @@ dict_foreign_find_index( for (dict_index_t* index = dict_table_get_first_index(table); index; index = dict_table_get_next_index(index)) { - if (types_idx != index - && !index->to_be_dropped + if (!index->to_be_dropped && !dict_index_is_online_ddl(index) && dict_foreign_qualify_index( table, col_names, columns, n_cols, @@ -3530,6 +3529,7 @@ dict_foreign_parse_drop_constraints( const char* ptr1; const char* id; CHARSET_INFO* cs; + bool if_exists = false; ut_a(trx->mysql_thd); @@ -3583,6 +3583,7 @@ loop: ptr1 = dict_accept(cs, ptr1, "EXISTS", &success); if (success) { ptr = ptr1; + if_exists = true; } } @@ -3593,14 +3594,14 @@ loop: goto syntax_error; } - ut_a(*n < 1000); - (*constraints_to_drop)[*n] = id; - (*n)++; - if (std::find_if(table->foreign_set.begin(), - table->foreign_set.end(), - dict_foreign_matches_id(id)) - == table->foreign_set.end()) { + table->foreign_set.end(), + dict_foreign_matches_id(id)) + == table->foreign_set.end()) { + + if (if_exists) { + goto loop; + } if (!srv_read_only_mode) { FILE* ef = dict_foreign_err_file; @@ -3622,6 +3623,9 @@ loop: return(DB_CANNOT_DROP_CONSTRAINT); } + ut_a(*n < 1000); + (*constraints_to_drop)[*n] = id; + (*n)++; goto loop; syntax_error: diff --git a/storage/innobase/dict/dict0load.cc b/storage/innobase/dict/dict0load.cc index f769839d..e7735586 100644 --- a/storage/innobase/dict/dict0load.cc +++ b/storage/innobase/dict/dict0load.cc @@ -33,8 +33,8 @@ Created 4/24/1996 Heikki Tuuri #include "dict0boot.h" #include "dict0crea.h" #include "dict0dict.h" -#include "dict0mem.h" #include "dict0stats.h" +#include "ibuf0ibuf.h" #include "fsp0file.h" #include "fts0priv.h" #include "mach0data.h" @@ -865,18 +865,30 @@ err_exit: return READ_OK; } -/** Check each tablespace found in the data dictionary. -Then look at each table defined in SYS_TABLES that has a space_id > 0 -to find all the file-per-table tablespaces. +/** @return SELECT MAX(space) FROM sys_tables */ +static uint32_t dict_find_max_space_id(btr_pcur_t *pcur, mtr_t *mtr) +{ + uint32_t max_space_id= 0; -In a crash recovery we already have some tablespace objects created from -processing the REDO log. We will compare the -space_id information in the data dictionary to what we find in the -tablespace file. In addition, more validation will be done if recovery -was needed and force_recovery is not set. + for (const rec_t *rec= dict_startscan_system(pcur, mtr, dict_sys.sys_tables); + rec; rec= dict_getnext_system_low(pcur, mtr)) + if (!dict_sys_tables_rec_check(rec)) + { + ulint len; + const byte *field= + rec_get_nth_field_old(rec, DICT_FLD__SYS_TABLES__SPACE, &len); + ut_ad(len == 4); + max_space_id= std::max(max_space_id, mach_read_from_4(field)); + } + + return max_space_id; +} -We also scan the biggest space id, and store it to fil_system. */ -void dict_check_tablespaces_and_store_max_id() +/** Check MAX(SPACE) FROM SYS_TABLES and store it in fil_system. +Open each data file if an encryption plugin has been loaded. + +@param spaces set of tablespace files to open */ +void dict_check_tablespaces_and_store_max_id(const std::set<uint32_t> *spaces) { uint32_t max_space_id = 0; btr_pcur_t pcur; @@ -888,6 +900,12 @@ void dict_check_tablespaces_and_store_max_id() dict_sys.lock(SRW_LOCK_CALL); + if (!spaces && ibuf.empty + && !encryption_key_id_exists(FIL_DEFAULT_ENCRYPTION_KEY)) { + max_space_id = dict_find_max_space_id(&pcur, &mtr); + goto done; + } + for (const rec_t *rec = dict_startscan_system(&pcur, &mtr, dict_sys.sys_tables); rec; rec = dict_getnext_system_low(&pcur, &mtr)) { @@ -919,14 +937,6 @@ void dict_check_tablespaces_and_store_max_id() continue; } - if (flags2 & DICT_TF2_DISCARDED) { - sql_print_information("InnoDB: Ignoring tablespace" - " for %.*s because " - "the DISCARD flag is set", - static_cast<int>(len), field); - continue; - } - /* For tables or partitions using .ibd files, the flag DICT_TF2_USE_FILE_PER_TABLE was not set in MIX_LEN before MySQL 5.6.5. The flag should not have been @@ -939,6 +949,19 @@ void dict_check_tablespaces_and_store_max_id() continue; } + if (spaces && spaces->find(uint32_t(space_id)) + == spaces->end()) { + continue; + } + + if (flags2 & DICT_TF2_DISCARDED) { + sql_print_information("InnoDB: Ignoring tablespace" + " for %.*s because " + "the DISCARD flag is set", + static_cast<int>(len), field); + continue; + } + const span<const char> name{field, len}; char* filepath = fil_make_filepath(nullptr, name, @@ -971,6 +994,7 @@ void dict_check_tablespaces_and_store_max_id() ut_free(filepath); } +done: mtr.commit(); fil_set_max_space_id_if_bigger(max_space_id); @@ -2246,22 +2270,10 @@ dict_load_tablespace( /* The tablespace may already be open. */ table->space = fil_space_for_table_exists_in_mem(table->space_id, table->flags); - if (table->space) { + if (table->space || table->file_unreadable) { return; } - if (ignore_err >= DICT_ERR_IGNORE_TABLESPACE) { - table->file_unreadable = true; - return; - } - - if (!(ignore_err & DICT_ERR_IGNORE_RECOVER_LOCK)) { - ib::error() << "Failed to find tablespace for table " - << table->name << " in the cache. Attempting" - " to load the tablespace with space id " - << table->space_id; - } - /* Use the remote filepath if needed. This parameter is optional in the call to fil_ibd_open(). If not supplied, it will be built from the table->name. */ @@ -2284,6 +2296,12 @@ dict_load_tablespace( if (!table->space) { /* We failed to find a sensible tablespace file */ table->file_unreadable = true; + + if (!(ignore_err & DICT_ERR_IGNORE_RECOVER_LOCK)) { + sql_print_error("InnoDB: Failed to load tablespace " + ULINTPF " for table %s", + table->space_id, table->name); + } } ut_free(filepath); diff --git a/storage/innobase/dict/dict0stats.cc b/storage/innobase/dict/dict0stats.cc index 40969335..f11187b9 100644 --- a/storage/innobase/dict/dict0stats.cc +++ b/storage/innobase/dict/dict0stats.cc @@ -752,16 +752,9 @@ dict_stats_empty_index( } } -/*********************************************************************//** -Write all zeros (or 1 where it makes sense) into a table and its indexes' -statistics members. The resulting stats correspond to an empty table. */ -static -void -dict_stats_empty_table( -/*===================*/ - dict_table_t* table, /*!< in/out: table */ +void dict_stats_empty_table( + dict_table_t* table, bool empty_defrag_stats) - /*!< in: whether to empty defrag stats */ { /* Initialize table/index level stats is now protected by table level lock_mutex.*/ diff --git a/storage/innobase/dict/dict0stats_bg.cc b/storage/innobase/dict/dict0stats_bg.cc index a66aac22..b0c34dc6 100644 --- a/storage/innobase/dict/dict0stats_bg.cc +++ b/storage/innobase/dict/dict0stats_bg.cc @@ -69,6 +69,8 @@ static recalc_pool_t recalc_pool; /** Whether the global data structures have been initialized */ static bool stats_initialised; +static THD *dict_stats_thd; + /*****************************************************************//** Free the resources occupied by the recalc pool, called once during thread de-initialization. */ @@ -90,6 +92,9 @@ static void dict_stats_recalc_pool_deinit() defrag_pool_t defrag_empty_pool; recalc_pool.swap(recalc_empty_pool); defrag_pool.swap(defrag_empty_pool); + + if (dict_stats_thd) + destroy_background_thd(dict_stats_thd); } /*****************************************************************//** @@ -361,52 +366,50 @@ done: { ut_ad(i->state == recalc::IN_PROGRESS); recalc_pool.erase(i); - const bool reschedule= !update_now && recalc_pool.empty(); if (err == DB_SUCCESS_LOCKED_REC) recalc_pool.emplace_back(recalc{table_id, recalc::IDLE}); mysql_mutex_unlock(&recalc_pool_mutex); - if (reschedule) - dict_stats_schedule(MIN_RECALC_INTERVAL * 1000); } return update_now; } -static tpool::timer* dict_stats_timer; -static std::mutex dict_stats_mutex; +/** Check if the recalc pool is empty. */ +static bool is_recalc_pool_empty() +{ + mysql_mutex_lock(&recalc_pool_mutex); + bool empty= recalc_pool.empty(); + mysql_mutex_unlock(&recalc_pool_mutex); + return empty; +} +static tpool::timer* dict_stats_timer; static void dict_stats_func(void*) { - THD *thd= innobase_create_background_thd("InnoDB statistics"); - set_current_thd(thd); - while (dict_stats_process_entry_from_recalc_pool(thd)) {} - dict_defrag_process_entries_from_defrag_pool(thd); + if (!dict_stats_thd) + dict_stats_thd= innobase_create_background_thd("InnoDB statistics"); + set_current_thd(dict_stats_thd); + + while (dict_stats_process_entry_from_recalc_pool(dict_stats_thd)) {} + dict_defrag_process_entries_from_defrag_pool(dict_stats_thd); + + innobase_reset_background_thd(dict_stats_thd); set_current_thd(nullptr); - destroy_background_thd(thd); + if (!is_recalc_pool_empty()) + dict_stats_schedule(MIN_RECALC_INTERVAL * 1000); } void dict_stats_start() { - std::lock_guard<std::mutex> lk(dict_stats_mutex); - if (!dict_stats_timer) - dict_stats_timer= srv_thread_pool->create_timer(dict_stats_func); + DBUG_ASSERT(!dict_stats_timer); + dict_stats_timer= srv_thread_pool->create_timer(dict_stats_func); } static void dict_stats_schedule(int ms) { - std::unique_lock<std::mutex> lk(dict_stats_mutex, std::defer_lock); - /* - Use try_lock() to avoid deadlock in dict_stats_shutdown(), which - uses dict_stats_mutex too. If there is simultaneous timer reschedule, - the first one will win, which is fine. - */ - if (!lk.try_lock()) - { - return; - } - if (dict_stats_timer) + if(dict_stats_timer) dict_stats_timer->set_time(ms,0); } @@ -418,7 +421,6 @@ void dict_stats_schedule_now() /** Shut down the dict_stats_thread. */ void dict_stats_shutdown() { - std::lock_guard<std::mutex> lk(dict_stats_mutex); delete dict_stats_timer; dict_stats_timer= 0; } diff --git a/storage/innobase/fil/fil0fil.cc b/storage/innobase/fil/fil0fil.cc index 8a88f4e2..bd0ace7c 100644 --- a/storage/innobase/fil/fil0fil.cc +++ b/storage/innobase/fil/fil0fil.cc @@ -341,8 +341,9 @@ static bool fil_node_open_file_low(fil_node_t *node) ut_ad(!node->is_open()); ut_ad(node->space->is_closing()); mysql_mutex_assert_owner(&fil_system.mutex); - ulint type; static_assert(((UNIV_ZIP_SIZE_MIN >> 1) << 3) == 4096, "compatibility"); +#if defined _WIN32 || defined HAVE_FCNTL_DIRECT + ulint type; switch (FSP_FLAGS_GET_ZIP_SSIZE(node->space->flags)) { case 1: case 2: @@ -351,6 +352,9 @@ static bool fil_node_open_file_low(fil_node_t *node) default: type= OS_DATA_FILE; } +#else + constexpr auto type= OS_DATA_FILE; +#endif for (;;) { @@ -560,7 +564,7 @@ fil_space_extend_must_retry( ut_ad(UT_LIST_GET_LAST(space->chain) == node); ut_ad(size >= FIL_IBD_FILE_INITIAL_SIZE); ut_ad(node->space == space); - ut_ad(space->referenced() || space->is_being_truncated); + ut_ad(space->referenced()); *success = space->size >= size; @@ -649,8 +653,7 @@ fil_space_extend_must_retry( default: ut_ad(space->purpose == FIL_TYPE_TABLESPACE || space->purpose == FIL_TYPE_IMPORT); - if (space->purpose == FIL_TYPE_TABLESPACE - && !space->is_being_truncated) { + if (space->purpose == FIL_TYPE_TABLESPACE) { goto do_flush; } break; @@ -735,12 +738,10 @@ bool fil_space_extend(fil_space_t *space, uint32_t size) bool success= false; const bool acquired= space->acquire(); mysql_mutex_lock(&fil_system.mutex); - if (acquired || space->is_being_truncated) - { + if (acquired) while (fil_space_extend_must_retry(space, UT_LIST_GET_LAST(space->chain), size, &success)) mysql_mutex_lock(&fil_system.mutex); - } mysql_mutex_unlock(&fil_system.mutex); if (acquired) space->release(); @@ -1903,9 +1904,10 @@ fil_ibd_create( mtr.flag_wr_unlock(); log_write_up_to(lsn, true); - ulint type; static_assert(((UNIV_ZIP_SIZE_MIN >> 1) << 3) == 4096, "compatibility"); +#if defined _WIN32 || defined HAVE_FCNTL_DIRECT + ulint type; switch (FSP_FLAGS_GET_ZIP_SSIZE(flags)) { case 1: case 2: @@ -1914,6 +1916,9 @@ fil_ibd_create( default: type = OS_DATA_FILE; } +#else + constexpr auto type = OS_DATA_FILE; +#endif file = os_file_create( innodb_data_file_key, path, @@ -2184,8 +2189,6 @@ func_exit: goto corrupted; } - os_file_get_last_error(operation_not_for_export, - !operation_not_for_export); if (!operation_not_for_export) { goto corrupted; } @@ -2448,21 +2451,15 @@ fil_ibd_load(uint32_t space_id, const char *filename, fil_space_t *&space) mysql_mutex_unlock(&fil_system.mutex); if (space) { - /* Compare the filename we are trying to open with the - filename from the first node of the tablespace we opened - previously. Fail if it is different. */ - fil_node_t* node = UT_LIST_GET_FIRST(space->chain); - if (0 != strcmp(innobase_basename(filename), - innobase_basename(node->name))) { - ib::info() - << "Ignoring data file '" << filename - << "' with space ID " << space->id - << ". Another data file called " << node->name - << " exists with the same space ID."; - space = NULL; - return(FIL_LOAD_ID_CHANGED); - } - return(FIL_LOAD_OK); + sql_print_information("InnoDB: Ignoring data file '%s'" + " with space ID " ULINTPF + ". Another data file called %s" + " exists" + " with the same space ID.", + filename, space->id, + UT_LIST_GET_FIRST(space->chain)->name); + space = NULL; + return FIL_LOAD_ID_CHANGED; } if (srv_operation == SRV_OPERATION_RESTORE) { @@ -3027,11 +3024,9 @@ fil_space_validate_for_mtr_commit( ut_ad(!is_predefined_tablespace(space->id)); /* We are serving mtr_commit(). While there is an active - mini-transaction, we should have !space->stop_new_ops. This is + mini-transaction, we should have !space->is_stopping(). This is guaranteed by meta-data locks or transactional locks. */ - ut_ad(!space->is_stopping() - || space->is_being_truncated /* fil_truncate_prepare() */ - || space->referenced()); + ut_ad(!space->is_stopping() || space->referenced()); } #endif /* UNIV_DEBUG */ @@ -3080,7 +3075,7 @@ ATTRIBUTE_NOINLINE ATTRIBUTE_COLD void mtr_t::name_write() and write out FILE_MODIFY if needed, and write FILE_CHECKPOINT. @param lsn checkpoint LSN @return current LSN */ -lsn_t fil_names_clear(lsn_t lsn) +ATTRIBUTE_COLD lsn_t fil_names_clear(lsn_t lsn) { mtr_t mtr; diff --git a/storage/innobase/fil/fil0pagecompress.cc b/storage/innobase/fil/fil0pagecompress.cc index 16aea2a7..eaf4e04a 100644 --- a/storage/innobase/fil/fil0pagecompress.cc +++ b/storage/innobase/fil/fil0pagecompress.cc @@ -49,11 +49,6 @@ Updated 14/02/2015 #include "buf0lru.h" #include "ibuf0ibuf.h" #include "zlib.h" -#ifdef __linux__ -#include <linux/fs.h> -#include <sys/ioctl.h> -#include <fcntl.h> -#endif #include "row0mysql.h" #include "lz4.h" #include "lzo/lzo1x.h" diff --git a/storage/innobase/fsp/fsp0file.cc b/storage/innobase/fsp/fsp0file.cc index cafff419..1c20efcd 100644 --- a/storage/innobase/fsp/fsp0file.cc +++ b/storage/innobase/fsp/fsp0file.cc @@ -435,12 +435,22 @@ Datafile::validate_for_recovery() return(err); } + if (!m_space_id) { + m_space_id = recv_sys.dblwr.find_first_page( + m_filepath, m_handle); + if (m_space_id) { + m_defer= false; + goto free_first_page; + } else return err; + } + if (!m_defer) { err = find_space_id(); if (err != DB_SUCCESS || m_space_id == 0) { - ib::error() << "Datafile '" << m_filepath - << "' is corrupted. Cannot determine " - "the space ID from the first 64 pages."; + sql_print_error( + "InnoDB: Datafile '%s' is corrupted." + " Cannot determine the space ID from" + " the first 64 pages.", m_filepath); return(err); } } @@ -453,7 +463,7 @@ Datafile::validate_for_recovery() m_space_id, m_filepath, m_handle)) { return m_defer ? err : DB_CORRUPTION; } - +free_first_page: /* Free the previously read first page and then re-validate. */ free_first_page(); m_defer = false; @@ -492,11 +502,11 @@ err_exit: return DB_SUCCESS; } - ib::info() << error_txt << " in datafile: " << m_filepath - << ", Space ID:" << m_space_id << ", Flags: " - << m_flags; + sql_print_error("InnoDB: %s in datafile: %s, Space ID: " + UINT32PF ", " "Flags: " UINT32PF, + error_txt, m_filepath, m_space_id, m_flags); m_is_valid = false; - return(DB_CORRUPTION); + return DB_CORRUPTION; } /* Check if the whole page is blank. */ diff --git a/storage/innobase/fsp/fsp0fsp.cc b/storage/innobase/fsp/fsp0fsp.cc index 6c5c354e..87672a82 100644 --- a/storage/innobase/fsp/fsp0fsp.cc +++ b/storage/innobase/fsp/fsp0fsp.cc @@ -42,8 +42,6 @@ Created 11/29/1995 Heikki Tuuri #include "fsp0types.h" #include "log.h" -typedef uint32_t page_no_t; - /** Returns the first extent descriptor for a segment. We think of the extent lists of the segment catenated in the order FSEG_FULL -> FSEG_NOT_FULL -> FSEG_FREE. @@ -332,7 +330,7 @@ xdes_t* xdes_get_descriptor_with_space_hdr( buf_block_t* header, const fil_space_t* space, - page_no_t offset, + uint32_t offset, mtr_t* mtr, dberr_t* err = nullptr, buf_block_t** desc_block = nullptr, @@ -396,7 +394,7 @@ try to add new extents to the space free list @param[out] err error code @param[out] xdes extent descriptor page @return the extent descriptor */ -static xdes_t *xdes_get_descriptor(const fil_space_t *space, page_no_t offset, +static xdes_t *xdes_get_descriptor(const fil_space_t *space, uint32_t offset, mtr_t *mtr, dberr_t *err= nullptr, buf_block_t **xdes= nullptr) { @@ -842,8 +840,7 @@ fsp_fill_free_list( if (i) { buf_block_t *f= buf_LRU_get_free_block(false); - buf_block_t *block= buf_page_create(space, static_cast<uint32_t>(i), - zip_size, mtr, f); + buf_block_t *block= buf_page_create(space, i, zip_size, mtr, f); if (UNIV_UNLIKELY(block != f)) buf_pool.free_block(f); fsp_init_file_page(space, block, mtr); @@ -855,9 +852,7 @@ fsp_fill_free_list( { buf_block_t *f= buf_LRU_get_free_block(false); buf_block_t *block= - buf_page_create(space, - static_cast<uint32_t>(i + FSP_IBUF_BITMAP_OFFSET), - zip_size, mtr, f); + buf_page_create(space, i + FSP_IBUF_BITMAP_OFFSET, zip_size, mtr, f); if (UNIV_UNLIKELY(block != f)) buf_pool.free_block(f); fsp_init_file_page(space, block, mtr); @@ -1028,40 +1023,13 @@ fsp_alloc_from_free_frag(buf_block_t *header, buf_block_t *xdes, xdes_t *descr, @param[in] offset page number of the allocated page @param[in,out] mtr mini-transaction @return block, initialized */ -static -buf_block_t* -fsp_page_create(fil_space_t *space, page_no_t offset, mtr_t *mtr) +static buf_block_t* fsp_page_create(fil_space_t *space, uint32_t offset, + mtr_t *mtr) { - buf_block_t *block, *free_block; - - if (UNIV_UNLIKELY(space->is_being_truncated)) - { - const page_id_t page_id{space->id, offset}; - buf_pool_t::hash_chain &chain= buf_pool.page_hash.cell_get(page_id.fold()); - mysql_mutex_lock(&buf_pool.mutex); - block= reinterpret_cast<buf_block_t*> - (buf_pool.page_hash.get(page_id, chain)); - if (block && block->page.oldest_modification() <= 1) - block= nullptr; - mysql_mutex_unlock(&buf_pool.mutex); - - if (block) - { - ut_ad(block->page.buf_fix_count() >= 1); - ut_ad(block->page.lock.x_lock_count() == 1); - ut_ad(mtr->have_x_latch(*block)); - free_block= block; - goto got_free_block; - } - } - - free_block= buf_LRU_get_free_block(false); -got_free_block: - block= buf_page_create(space, static_cast<uint32_t>(offset), - space->zip_size(), mtr, free_block); + buf_block_t *free_block= buf_LRU_get_free_block(false), + *block= buf_page_create(space, offset, space->zip_size(), mtr, free_block); if (UNIV_UNLIKELY(block != free_block)) buf_pool.free_block(free_block); - fsp_init_file_page(space, block, mtr); return block; } @@ -1179,7 +1147,7 @@ MY_ATTRIBUTE((nonnull, warn_unused_result)) @param[in] offset page number in the extent @param[in,out] mtr mini-transaction @return error code */ -static dberr_t fsp_free_extent(fil_space_t* space, page_no_t offset, +static dberr_t fsp_free_extent(fil_space_t* space, uint32_t offset, mtr_t* mtr) { ut_ad(space->is_owner()); @@ -1216,7 +1184,7 @@ The page is marked as free and clean. @param[in] offset page number @param[in,out] mtr mini-transaction @return error code */ -static dberr_t fsp_free_page(fil_space_t *space, page_no_t offset, mtr_t *mtr) +static dberr_t fsp_free_page(fil_space_t *space, uint32_t offset, mtr_t *mtr) { xdes_t* descr; ulint frag_n_used; @@ -1756,7 +1724,6 @@ page_alloc: ut_d(const auto x = block->page.lock.x_lock_count()); ut_ad(x || block->page.lock.not_recursive()); - ut_ad(x == 1 || space->is_being_truncated); ut_ad(x <= 2); ut_ad(!fil_page_get_type(block->page.frame)); mtr->write<1>(*block, FIL_PAGE_TYPE + 1 + block->page.frame, @@ -2493,7 +2460,7 @@ fseg_free_page_low( fseg_inode_t* seg_inode, buf_block_t* iblock, fil_space_t* space, - page_no_t offset, + uint32_t offset, mtr_t* mtr #ifdef BTR_CUR_HASH_ADAPT ,bool ahi=false @@ -2859,7 +2826,7 @@ fseg_free_step( return true; } - page_no_t page_no = fseg_get_nth_frag_page_no(inode, n); + uint32_t page_no = fseg_get_nth_frag_page_no(inode, n); if (fseg_free_page_low(inode, iblock, space, page_no, mtr #ifdef BTR_CUR_HASH_ADAPT diff --git a/storage/innobase/fsp/fsp0sysspace.cc b/storage/innobase/fsp/fsp0sysspace.cc index e4a43e48..4ac9da50 100644 --- a/storage/innobase/fsp/fsp0sysspace.cc +++ b/storage/innobase/fsp/fsp0sysspace.cc @@ -33,6 +33,7 @@ Refactored 2013-7-26 by Kevin Lewis #include "os0file.h" #include "row0mysql.h" #include "buf0dblwr.h" +#include "log.h" /** The server header file is included to access opt_initialize global variable. If server passes the option for create/open DB to SE, we should remove such @@ -568,7 +569,7 @@ inline dberr_t SysTablespace::read_lsn_and_check_flags() } err = it->read_first_page( - m_ignore_read_only ? false : srv_read_only_mode); + m_ignore_read_only && srv_read_only_mode); if (err != DB_SUCCESS) { return(err); @@ -582,47 +583,62 @@ inline dberr_t SysTablespace::read_lsn_and_check_flags() /* Check the contents of the first page of the first datafile. */ - for (int retry = 0; retry < 2; ++retry) { + err = it->validate_first_page(); - err = it->validate_first_page(); - - if (err != DB_SUCCESS - && (retry == 1 - || recv_sys.dblwr.restore_first_page( + if (err != DB_SUCCESS) { + if (recv_sys.dblwr.restore_first_page( it->m_space_id, it->m_filepath, - it->handle()))) { - + it->handle())) { it->close(); - return(err); } + err = it->read_first_page( + m_ignore_read_only && srv_read_only_mode); } /* Make sure the tablespace space ID matches the space ID on the first page of the first datafile. */ - if (space_id() != it->m_space_id) { - - ib::error() - << "The data file '" << it->filepath() - << "' has the wrong space ID. It should be " - << space_id() << ", but " << it->m_space_id - << " was found"; - + if (err != DB_SUCCESS || space_id() != it->m_space_id) { + sql_print_error("InnoDB: The data file '%s'" + " has the wrong space ID." + " It should be " UINT32PF ", but " UINT32PF + " was found", it->filepath(), + space_id(), it->m_space_id); it->close(); - - return(err); + return err; } - if (srv_operation == SRV_OPERATION_NORMAL) { + if (srv_force_recovery != 6 + && srv_operation == SRV_OPERATION_NORMAL + && !log_sys.next_checkpoint_lsn + && log_sys.format == log_t::FORMAT_3_23) { + + log_sys.latch.wr_lock(SRW_LOCK_CALL); /* Prepare for possible upgrade from 0-sized ib_logfile0. */ - ut_ad(!log_sys.next_checkpoint_lsn); log_sys.next_checkpoint_lsn = mach_read_from_8( it->m_first_page + 26/*FIL_PAGE_FILE_FLUSH_LSN*/); + if (log_sys.next_checkpoint_lsn < 8204) { + /* Before MDEV-14425, InnoDB had a minimum LSN + of 8192+12=8204. Likewise, mariadb-backup + --prepare would create an empty ib_logfile0 + after applying the log. We will allow an + upgrade from such an empty log. */ + sql_print_error("InnoDB: ib_logfile0 is " + "empty, and LSN is unknown."); + err = DB_CORRUPTION; + } else { + log_sys.last_checkpoint_lsn = + recv_sys.lsn = recv_sys.file_checkpoint = + log_sys.next_checkpoint_lsn; + log_sys.set_recovered_lsn(log_sys.next_checkpoint_lsn); + log_sys.next_checkpoint_no = 0; + } + + log_sys.latch.wr_unlock(); } it->close(); - - return(DB_SUCCESS); + return err; } /** Check if a file can be opened in the correct mode. diff --git a/storage/innobase/gis/gis0sea.cc b/storage/innobase/gis/gis0sea.cc index 8ca8681b..0df9a7de 100644 --- a/storage/innobase/gis/gis0sea.cc +++ b/storage/innobase/gis/gis0sea.cc @@ -304,6 +304,8 @@ rtr_pcur_getnext_from_path( break; } + buf_page_make_young_if_needed(&block->page); + page = buf_block_get_frame(block); page_ssn = page_get_ssn_id(page); @@ -683,6 +685,8 @@ dberr_t rtr_search_to_nth_level(ulint level, const dtuple_t *tuple, return err; } + buf_page_make_young_if_needed(&block->page); + const page_t *page= buf_block_get_frame(block); #ifdef UNIV_ZIP_DEBUG if (rw_latch != RW_NO_LATCH) { @@ -1703,6 +1707,8 @@ corrupted: goto func_exit; } + buf_page_make_young_if_needed(&page_cursor->block->page); + /* Get the page SSN */ page = buf_block_get_frame(page_cursor->block); page_ssn = page_get_ssn_id(page); diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index 21bf10a1..407834f2 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -1575,7 +1575,8 @@ static void innodb_drop_database(handlerton*, char *path) ibuf_delete_for_discarded_space(id); /* Any changes must be persisted before we return. */ - log_write_up_to(mtr.commit_lsn(), true); + if (mtr.commit_lsn()) + log_write_up_to(mtr.commit_lsn(), true); } my_free(namebuf); @@ -2080,7 +2081,7 @@ all_fail: ut_d(purge_sys.resume_FTS()); } -static void innodb_ddl_recovery_done(handlerton*) +static int innodb_ddl_recovery_done(handlerton*) { ut_ad(!ddl_recovery_done); ut_d(ddl_recovery_done= true); @@ -2091,6 +2092,7 @@ static void innodb_ddl_recovery_done(handlerton*) drop_garbage_tables_after_restore(); srv_init_purge_tasks(); } + return 0; } /********************************************************************//** @@ -4001,7 +4003,7 @@ static int innodb_init_params() data_mysql_default_charset_coll = (ulint) default_charset_info->number; -#ifndef _WIN32 +#ifdef HAVE_FCNTL_DIRECT if (srv_use_atomic_writes && my_may_have_atomic_write) { /* Force O_DIRECT on Unixes (on Windows writes are always @@ -4026,11 +4028,6 @@ static int innodb_init_params() } #endif - if (srv_read_only_mode) { - ib::info() << "Started in read only mode"; - srv_use_doublewrite_buf = FALSE; - } - #if !defined LINUX_NATIVE_AIO && !defined HAVE_URING && !defined _WIN32 /* Currently native AIO is supported only on windows and linux and that also when the support is compiled in. In all other @@ -4046,9 +4043,7 @@ static int innodb_init_params() } #endif -#ifndef _WIN32 - ut_ad(srv_file_flush_method <= SRV_O_DIRECT_NO_FSYNC); -#else +#ifdef _WIN32 switch (srv_file_flush_method) { case SRV_ALL_O_DIRECT_FSYNC + 1 /* "async_unbuffered"="unbuffered" */: srv_file_flush_method = SRV_ALL_O_DIRECT_FSYNC; @@ -4059,6 +4054,8 @@ static int innodb_init_params() default: ut_ad(srv_file_flush_method <= SRV_ALL_O_DIRECT_FSYNC); } +#else + ut_ad(srv_file_flush_method <= SRV_O_DIRECT_NO_FSYNC); #endif innodb_buffer_pool_size_init(); @@ -7817,20 +7814,6 @@ ha_innobase::write_row( #endif if ((error_result = update_auto_increment())) { - /* We don't want to mask autoinc overflow errors. */ - - /* Handle the case where the AUTOINC sub-system - failed during initialization. */ - if (m_prebuilt->autoinc_error == DB_UNSUPPORTED) { - error_result = ER_AUTOINC_READ_FAILED; - /* Set the error message to report too. */ - my_error(ER_AUTOINC_READ_FAILED, MYF(0)); - goto func_exit; - } else if (m_prebuilt->autoinc_error != DB_SUCCESS) { - error = m_prebuilt->autoinc_error; - goto report_error; - } - /* MySQL errors are passed straight back. */ goto func_exit; } @@ -7968,7 +7951,6 @@ set_max_autoinc: } } -report_error: /* Cleanup and exit. */ if (error == DB_TABLESPACE_DELETED) { ib_senderrf( @@ -11809,8 +11791,6 @@ index_bad: /* Set the flags2 when create table or alter tables */ m_flags2 |= DICT_TF2_FTS_AUX_HEX_NAME; - DBUG_EXECUTE_IF("innodb_test_wrong_fts_aux_table_name", - m_flags2 &= ~DICT_TF2_FTS_AUX_HEX_NAME;); DBUG_RETURN(true); } @@ -14710,12 +14690,7 @@ ha_innobase::info_low( DBUG_ASSERT(ib_table->get_ref_count() > 0); if (!ib_table->is_readable()) { - ib_table->stats_mutex_lock(); - ib_table->stat_initialized = true; - ib_table->stat_n_rows = 0; - ib_table->stat_clustered_index_size = 0; - ib_table->stat_sum_of_other_index_sizes = 0; - ib_table->stats_mutex_unlock(); + dict_stats_empty_table(ib_table, true); } if (flag & HA_STATUS_TIME) { @@ -15674,15 +15649,17 @@ ha_innobase::extra( { /* Warning: since it is not sure that MariaDB calls external_lock() before calling this function, m_prebuilt->trx can be obsolete! */ - trx_t* trx = check_trx_exists(ha_thd()); + trx_t* trx; switch (operation) { case HA_EXTRA_FLUSH: + (void)check_trx_exists(ha_thd()); if (m_prebuilt->blob_heap) { row_mysql_prebuilt_free_blob_heap(m_prebuilt); } break; case HA_EXTRA_RESET_STATE: + trx = check_trx_exists(ha_thd()); reset_template(); trx->duplicates = 0; stmt_boundary: @@ -15691,18 +15668,23 @@ ha_innobase::extra( trx->bulk_insert = false; break; case HA_EXTRA_NO_KEYREAD: + (void)check_trx_exists(ha_thd()); m_prebuilt->read_just_key = 0; break; case HA_EXTRA_KEYREAD: + (void)check_trx_exists(ha_thd()); m_prebuilt->read_just_key = 1; break; case HA_EXTRA_KEYREAD_PRESERVE_FIELDS: + (void)check_trx_exists(ha_thd()); m_prebuilt->keep_other_fields_on_keyread = 1; break; case HA_EXTRA_INSERT_WITH_UPDATE: + trx = check_trx_exists(ha_thd()); trx->duplicates |= TRX_DUP_IGNORE; goto stmt_boundary; case HA_EXTRA_NO_IGNORE_DUP_KEY: + trx = check_trx_exists(ha_thd()); trx->duplicates &= ~TRX_DUP_IGNORE; if (trx->is_bulk_insert()) { /* Allow a subsequent INSERT into an empty table @@ -15714,9 +15696,11 @@ ha_innobase::extra( } goto stmt_boundary; case HA_EXTRA_WRITE_CAN_REPLACE: + trx = check_trx_exists(ha_thd()); trx->duplicates |= TRX_DUP_REPLACE; goto stmt_boundary; case HA_EXTRA_WRITE_CANNOT_REPLACE: + trx = check_trx_exists(ha_thd()); trx->duplicates &= ~TRX_DUP_REPLACE; if (trx->is_bulk_insert()) { /* Allow a subsequent INSERT into an empty table @@ -15725,6 +15709,7 @@ ha_innobase::extra( } goto stmt_boundary; case HA_EXTRA_BEGIN_ALTER_COPY: + trx = check_trx_exists(ha_thd()); m_prebuilt->table->skip_alter_undo = 1; if (m_prebuilt->table->is_temporary() || !m_prebuilt->table->versioned_by_id()) { @@ -15737,8 +15722,10 @@ ha_innobase::extra( .first->second.set_versioned(0); break; case HA_EXTRA_END_ALTER_COPY: + trx = check_trx_exists(ha_thd()); m_prebuilt->table->skip_alter_undo = 0; - if (!m_prebuilt->table->is_temporary()) { + if (!m_prebuilt->table->is_temporary() + && !high_level_read_only) { log_buffer_flush_to_disk(); } break; @@ -18270,11 +18257,18 @@ static void buf_flush_list_now_set(THD*, st_mysql_sys_var*, void*, const void* save) { - if (*(my_bool*) save) { - mysql_mutex_unlock(&LOCK_global_system_variables); - buf_flush_sync(); - mysql_mutex_lock(&LOCK_global_system_variables); - } + if (!*(my_bool*) save) + return; + const uint s= srv_fil_make_page_dirty_debug; + mysql_mutex_unlock(&LOCK_global_system_variables); + if (s) + buf_flush_sync(); + else + { + while (buf_flush_list_space(fil_system.sys_space, nullptr)); + os_aio_wait_until_no_pending_writes(true); + } + mysql_mutex_lock(&LOCK_global_system_variables); } /** Override current MERGE_THRESHOLD setting for all indexes at dictionary @@ -19368,8 +19362,10 @@ static MYSQL_SYSVAR_ULONGLONG(max_undo_log_size, srv_max_undo_log_size, 10 << 20, 10 << 20, 1ULL << (32 + UNIV_PAGE_SIZE_SHIFT_MAX), 0); +static ulong innodb_purge_rseg_truncate_frequency; + static MYSQL_SYSVAR_ULONG(purge_rseg_truncate_frequency, - srv_purge_rseg_truncate_frequency, + innodb_purge_rseg_truncate_frequency, PLUGIN_VAR_OPCMDARG | PLUGIN_VAR_DEPRECATED, "Deprecated parameter with no effect", NULL, NULL, 128, 1, 128, 0); @@ -20606,6 +20602,10 @@ Compare_keys ha_innobase::compare_key_parts( if (old_part.length >= new_part.length) return Compare_keys::NotEqual; + if (old_part.length == old_field.key_length() && + new_part.length != new_field.length) + return Compare_keys::NotEqual; + return Compare_keys::EqualButKeyPartLength; } diff --git a/storage/innobase/handler/handler0alter.cc b/storage/innobase/handler/handler0alter.cc index 40370ac5..1401136f 100644 --- a/storage/innobase/handler/handler0alter.cc +++ b/storage/innobase/handler/handler0alter.cc @@ -2317,12 +2317,16 @@ innodb_instant_alter_column_allowed_reason: } } + bool need_rebuild = false; + switch (ha_alter_info->handler_flags & ~INNOBASE_INPLACE_IGNORE) { case ALTER_OPTIONS: - if (alter_options_need_rebuild(ha_alter_info, table)) { + if ((srv_file_per_table && !m_prebuilt->table->space_id) + || alter_options_need_rebuild(ha_alter_info, table)) { reason_rebuild = my_get_err_msg( ER_ALTER_OPERATION_TABLE_OPTIONS_NEED_REBUILD); ha_alter_info->unsupported_reason = reason_rebuild; + need_rebuild= true; break; } /* fall through */ @@ -2434,7 +2438,7 @@ innodb_instant_alter_column_allowed_reason: /* We should be able to do the operation in-place. See if we can do it online (LOCK=NONE) or without rebuild. */ - bool online = true, need_rebuild = false; + bool online = true; const uint fulltext_indexes = innobase_fulltext_exist(altered_table); /* Fix the key parts. */ @@ -4338,7 +4342,8 @@ static void unlock_and_close_files(const std::vector<pfs_os_file_t> &deleted, row_mysql_unlock_data_dictionary(trx); for (pfs_os_file_t d : deleted) os_file_close(d); - log_write_up_to(trx->commit_lsn, true); + if (trx->commit_lsn) + log_write_up_to(trx->commit_lsn, true); } /** Commit a DDL transaction and unlink any deleted files. */ @@ -4681,11 +4686,13 @@ innobase_build_col_map( col_map[old_i - num_old_v] = i; if (!old_table->versioned() || !altered_table->versioned()) { - } else if (old_i == old_table->vers_start) { - new_table->vers_start = (i + num_v) + } else if (old_i - num_old_v == old_table->vers_start) { + ut_ad(field->vers_sys_start()); + new_table->vers_start = i & dict_index_t::MAX_N_FIELDS; - } else if (old_i == old_table->vers_end) { - new_table->vers_end = (i + num_v) + } else if (old_i - num_old_v == old_table->vers_end) { + ut_ad(field->vers_sys_end()); + new_table->vers_end = i & dict_index_t::MAX_N_FIELDS; } goto found_col; @@ -6217,24 +6224,20 @@ empty_table: /* Convert the table to the instant ALTER TABLE format. */ mtr.commit(); mtr.start(); - index->set_modified(mtr); - if (buf_block_t* root = btr_root_block_get(index, RW_SX_LATCH, &mtr, + if (buf_block_t* root = btr_root_block_get(index, RW_S_LATCH, &mtr, &err)) { if (fil_page_get_type(root->page.frame) != FIL_PAGE_INDEX) { DBUG_ASSERT("wrong page type" == 0); err = DB_CORRUPTION; goto func_exit; } - - btr_set_instant(root, *index, &mtr); - mtr.commit(); - mtr.start(); - index->set_modified(mtr); - err = row_ins_clust_index_entry_low( - BTR_NO_LOCKING_FLAG, BTR_MODIFY_TREE, index, - index->n_uniq, entry, 0, thr); } + mtr.commit(); + mtr.start(); + err = row_ins_clust_index_entry_low( + BTR_NO_LOCKING_FLAG, BTR_MODIFY_TREE, index, + index->n_uniq, entry, 0, thr); goto func_exit; } @@ -7775,6 +7778,7 @@ bool check_col_is_in_fk_indexes( for (const auto &a : add_fk) { + if (!a->foreign_index) continue; for (ulint i= 0; i < a->n_fields; i++) { if (a->foreign_index->fields[i].col == col) @@ -11666,7 +11670,6 @@ foreign_fail: } unlock_and_close_files(deleted, trx); - log_write_up_to(trx->commit_lsn, true); DBUG_EXECUTE_IF("innodb_alter_commit_crash_after_commit", DBUG_SUICIDE();); trx->free(); @@ -11723,7 +11726,6 @@ foreign_fail: } unlock_and_close_files(deleted, trx); - log_write_up_to(trx->commit_lsn, true); DBUG_EXECUTE_IF("innodb_alter_commit_crash_after_commit", DBUG_SUICIDE();); trx->free(); diff --git a/storage/innobase/handler/i_s.cc b/storage/innobase/handler/i_s.cc index b00308d7..711144e3 100644 --- a/storage/innobase/handler/i_s.cc +++ b/storage/innobase/handler/i_s.cc @@ -4539,6 +4539,15 @@ i_s_dict_fill_sys_tables( DBUG_RETURN(0); } +/** Handle the error for information schema query +@param err error value +@param thd thread +@return 0 if query is interrupted or error */ +static int i_s_sys_error_handling(int err, THD *thd) +{ + return thd_kill_level(thd) ? 0 : err; +} + /** Convert one SYS_TABLES record to dict_table_t. @param pcur persistent cursor position on SYS_TABLES record @param mtr mini-transaction (nullptr=use the dict_sys cache) @@ -4587,6 +4596,7 @@ i_s_sys_tables_fill_table( { btr_pcur_t pcur; mtr_t mtr; + int err = 0; DBUG_ENTER("i_s_sys_tables_fill_table"); RETURN_IF_INNODB_NOT_STARTED(tables->schema_table_name.str); @@ -4616,8 +4626,15 @@ i_s_sys_tables_fill_table( dict_sys.unlock(); if (!err_msg) { - i_s_dict_fill_sys_tables(thd, table_rec, - tables->table); + err = i_s_dict_fill_sys_tables( + thd, table_rec, tables->table); + if (err) { + err = i_s_sys_error_handling(err, thd); + if (table_rec) { + dict_mem_table_free(table_rec); + } + goto func_exit; + } } else { push_warning_printf(thd, Sql_condition::WARN_LEVEL_WARN, ER_CANT_FIND_SYSTEM_REC, "%s", @@ -4635,8 +4652,10 @@ i_s_sys_tables_fill_table( mtr.commit(); dict_sys.unlock(); +func_exit: + ut_free(pcur.old_rec_buf); - DBUG_RETURN(0); + DBUG_RETURN(err); } /*******************************************************************//** @@ -4807,6 +4826,7 @@ i_s_sys_tables_fill_table_stats( btr_pcur_t pcur; const rec_t* rec; mtr_t mtr; + int err = 0; DBUG_ENTER("i_s_sys_tables_fill_table_stats"); RETURN_IF_INNODB_NOT_STARTED(tables->schema_table_name.str); @@ -4832,8 +4852,12 @@ i_s_sys_tables_fill_table_stats( &table_rec); if (UNIV_LIKELY(!err_msg)) { - i_s_dict_fill_sys_tablestats(thd, table_rec, + err = i_s_dict_fill_sys_tablestats(thd, table_rec, tables->table); + if (err) { + err = i_s_sys_error_handling(err, thd); + goto func_exit; + } } else { ut_ad(!table_rec); dict_sys.unlock(); @@ -4851,8 +4875,9 @@ i_s_sys_tables_fill_table_stats( mtr.commit(); dict_sys.unlock(); - - DBUG_RETURN(0); +func_exit: + ut_free(pcur.old_rec_buf); + DBUG_RETURN(err); } /*******************************************************************//** @@ -5024,6 +5049,7 @@ i_s_sys_indexes_fill_table( const rec_t* rec; mem_heap_t* heap; mtr_t mtr; + int err = 0; DBUG_ENTER("i_s_sys_indexes_fill_table"); RETURN_IF_INNODB_NOT_STARTED(tables->schema_table_name.str); @@ -5059,11 +5085,13 @@ i_s_sys_indexes_fill_table( dict_sys.unlock(); if (!err_msg) { - if (int err = i_s_dict_fill_sys_indexes( - thd, table_id, space_id, &index_rec, - tables->table)) { - mem_heap_free(heap); - DBUG_RETURN(err); + err = i_s_dict_fill_sys_indexes( + thd, table_id, space_id, + &index_rec, + tables->table); + if (err) { + err = i_s_sys_error_handling(err, thd); + goto func_exit; } } else { push_warning_printf(thd, Sql_condition::WARN_LEVEL_WARN, @@ -5081,9 +5109,11 @@ i_s_sys_indexes_fill_table( mtr.commit(); dict_sys.unlock(); +func_exit: mem_heap_free(heap); + ut_free(pcur.old_rec_buf); - DBUG_RETURN(0); + DBUG_RETURN(err); } /*******************************************************************//** Bind the dynamic table INFORMATION_SCHEMA.innodb_sys_indexes @@ -5232,6 +5262,7 @@ i_s_sys_columns_fill_table( const char* col_name; mem_heap_t* heap; mtr_t mtr; + int err = 0; DBUG_ENTER("i_s_sys_columns_fill_table"); RETURN_IF_INNODB_NOT_STARTED(tables->schema_table_name.str); @@ -5263,9 +5294,14 @@ i_s_sys_columns_fill_table( dict_sys.unlock(); if (!err_msg) { - i_s_dict_fill_sys_columns(thd, table_id, col_name, - &column_rec, nth_v_col, - tables->table); + err = i_s_dict_fill_sys_columns( + thd, table_id, col_name, + &column_rec, nth_v_col, + tables->table); + if (err) { + err = i_s_sys_error_handling(err, thd); + goto func_exit; + } } else { push_warning_printf(thd, Sql_condition::WARN_LEVEL_WARN, ER_CANT_FIND_SYSTEM_REC, "%s", @@ -5282,9 +5318,11 @@ i_s_sys_columns_fill_table( mtr.commit(); dict_sys.unlock(); +func_exit: mem_heap_free(heap); + ut_free(pcur.old_rec_buf); - DBUG_RETURN(0); + DBUG_RETURN(err); } /*******************************************************************//** @@ -5416,6 +5454,7 @@ i_s_sys_virtual_fill_table( ulint pos; ulint base_pos; mtr_t mtr; + int err = 0; DBUG_ENTER("i_s_sys_virtual_fill_table"); RETURN_IF_INNODB_NOT_STARTED(tables->schema_table_name.str); @@ -5444,8 +5483,13 @@ i_s_sys_virtual_fill_table( dict_sys.unlock(); if (!err_msg) { - i_s_dict_fill_sys_virtual(thd, table_id, pos, base_pos, - tables->table); + err = i_s_dict_fill_sys_virtual( + thd, table_id, pos, base_pos, + tables->table); + if (err) { + err = i_s_sys_error_handling(err, thd); + goto func_exit; + } } else { push_warning_printf(thd, Sql_condition::WARN_LEVEL_WARN, ER_CANT_FIND_SYSTEM_REC, "%s", @@ -5462,6 +5506,9 @@ i_s_sys_virtual_fill_table( dict_sys.unlock(); DBUG_RETURN(0); +func_exit: + ut_free(pcur.old_rec_buf); + DBUG_RETURN(err); } /** Bind the dynamic table INFORMATION_SCHEMA.innodb_sys_virtual @@ -5589,6 +5636,7 @@ i_s_sys_fields_fill_table( mem_heap_t* heap; index_id_t last_id; mtr_t mtr; + int err = 0; DBUG_ENTER("i_s_sys_fields_fill_table"); RETURN_IF_INNODB_NOT_STARTED(tables->schema_table_name.str); @@ -5624,8 +5672,13 @@ i_s_sys_fields_fill_table( dict_sys.unlock(); if (!err_msg) { - i_s_dict_fill_sys_fields(thd, index_id, &field_rec, - pos, tables->table); + err = i_s_dict_fill_sys_fields( + thd, index_id, &field_rec, + pos, tables->table); + if (err) { + err = i_s_sys_error_handling(err, thd); + goto func_exit; + } last_id = index_id; } else { push_warning_printf(thd, Sql_condition::WARN_LEVEL_WARN, @@ -5643,9 +5696,11 @@ i_s_sys_fields_fill_table( mtr.commit(); dict_sys.unlock(); +func_exit: mem_heap_free(heap); + ut_free(pcur.old_rec_buf); - DBUG_RETURN(0); + DBUG_RETURN(err); } /*******************************************************************//** Bind the dynamic table INFORMATION_SCHEMA.innodb_sys_fields @@ -5782,6 +5837,7 @@ i_s_sys_foreign_fill_table( const rec_t* rec; mem_heap_t* heap; mtr_t mtr; + int err = 0; DBUG_ENTER("i_s_sys_foreign_fill_table"); RETURN_IF_INNODB_NOT_STARTED(tables->schema_table_name.str); @@ -5809,8 +5865,12 @@ i_s_sys_foreign_fill_table( dict_sys.unlock(); if (!err_msg) { - i_s_dict_fill_sys_foreign(thd, &foreign_rec, - tables->table); + err = i_s_dict_fill_sys_foreign( + thd, &foreign_rec, tables->table); + if (err) { + err = i_s_sys_error_handling(err, thd); + goto func_exit; + } } else { push_warning_printf(thd, Sql_condition::WARN_LEVEL_WARN, ER_CANT_FIND_SYSTEM_REC, "%s", @@ -5827,9 +5887,11 @@ i_s_sys_foreign_fill_table( mtr.commit(); dict_sys.unlock(); +func_exit: mem_heap_free(heap); + ut_free(pcur.old_rec_buf); - DBUG_RETURN(0); + DBUG_RETURN(err); } /*******************************************************************//** @@ -5963,6 +6025,7 @@ i_s_sys_foreign_cols_fill_table( const rec_t* rec; mem_heap_t* heap; mtr_t mtr; + int err = 0; DBUG_ENTER("i_s_sys_foreign_cols_fill_table"); RETURN_IF_INNODB_NOT_STARTED(tables->schema_table_name.str); @@ -5994,9 +6057,13 @@ i_s_sys_foreign_cols_fill_table( dict_sys.unlock(); if (!err_msg) { - i_s_dict_fill_sys_foreign_cols( - thd, name, for_col_name, ref_col_name, pos, - tables->table); + err = i_s_dict_fill_sys_foreign_cols( + thd, name, for_col_name, + ref_col_name, pos, tables->table); + if (err) { + err = i_s_sys_error_handling(err, thd); + goto func_exit; + } } else { push_warning_printf(thd, Sql_condition::WARN_LEVEL_WARN, ER_CANT_FIND_SYSTEM_REC, "%s", @@ -6013,9 +6080,11 @@ i_s_sys_foreign_cols_fill_table( mtr.commit(); dict_sys.unlock(); +func_exit: mem_heap_free(heap); + ut_free(pcur.old_rec_buf); - DBUG_RETURN(0); + DBUG_RETURN(err); } /*******************************************************************//** Bind the dynamic table INFORMATION_SCHEMA.innodb_sys_foreign_cols @@ -6218,6 +6287,8 @@ static int i_s_sys_tablespaces_fill_table(THD *thd, TABLE_LIST *tables, Item*) mysql_mutex_unlock(&fil_system.mutex); if (err == DB_SUCCESS) err= i_s_sys_tablespaces_fill(thd, *fil_system.temp_space, tables->table); + else + err = i_s_sys_error_handling(err, thd); DBUG_RETURN(err); } diff --git a/storage/innobase/ibuf/ibuf0ibuf.cc b/storage/innobase/ibuf/ibuf0ibuf.cc index b9e94a67..4ec07b81 100644 --- a/storage/innobase/ibuf/ibuf0ibuf.cc +++ b/storage/innobase/ibuf/ibuf0ibuf.cc @@ -309,8 +309,13 @@ ibuf_header_page_get( buf_block_t* block = buf_page_get( page_id_t(IBUF_SPACE_ID, FSP_IBUF_HEADER_PAGE_NO), 0, RW_X_LATCH, mtr); + if (UNIV_UNLIKELY(!block)) { + return nullptr; + } + + buf_page_make_young_if_needed(&block->page); - return block ? block->page.frame : nullptr; + return block->page.frame; } /** Acquire the change buffer root page. @@ -326,7 +331,12 @@ static buf_block_t *ibuf_tree_root_get(mtr_t *mtr, dberr_t *err= nullptr) buf_block_t *block= buf_page_get_gen(page_id_t{IBUF_SPACE_ID, FSP_IBUF_TREE_ROOT_PAGE_NO}, 0, RW_SX_LATCH, nullptr, BUF_GET, mtr, err); - ut_ad(!block || ibuf.empty == page_is_empty(block->page.frame)); + if (block) + { + ut_ad(ibuf.empty == page_is_empty(block->page.frame)); + buf_page_make_young_if_needed(&block->page); + } + return block; } @@ -408,7 +418,8 @@ err_exit: + header_page->page.frame, &ibuf.seg_size, &mtr); do { - DBUG_EXECUTE_IF("intermittent_read_failure", continue;); + IF_DBUG(if (_db_keyword_(nullptr, "intermittent_read_failure", + 1)) continue,); ut_ad(ibuf.seg_size >= 2); } while (0); diff --git a/storage/innobase/include/btr0btr.h b/storage/innobase/include/btr0btr.h index 5a0401fa..b42c543c 100644 --- a/storage/innobase/include/btr0btr.h +++ b/storage/innobase/include/btr0btr.h @@ -89,10 +89,12 @@ ATTRIBUTE_COLD void btr_decryption_failed(const dict_index_t &index); @param[in] merge whether change buffer merge should be attempted @param[in,out] mtr mini-transaction @param[out] err error code +@param[out] first set if this is a first-time access to the page @return block */ buf_block_t *btr_block_get(const dict_index_t &index, uint32_t page, rw_lock_type_t mode, bool merge, - mtr_t *mtr, dberr_t *err= nullptr); + mtr_t *mtr, dberr_t *err= nullptr, + bool *first= nullptr); /**************************************************************//** Gets the index id field of a page. diff --git a/storage/innobase/include/buf0buf.h b/storage/innobase/include/buf0buf.h index 332b2039..cd7cc294 100644 --- a/storage/innobase/include/buf0buf.h +++ b/storage/innobase/include/buf0buf.h @@ -262,8 +262,6 @@ buf_block_t* buf_page_create_deferred(uint32_t space_id, ulint zip_size, mtr_t *mtr, buf_block_t *free_block); -/** Move a block to the start of the LRU list. */ -void buf_page_make_young(buf_page_t *bpage); /** Mark the page status as FREED for the given tablespace and page number. @param[in,out] space tablespace @param[in] page page number @@ -285,15 +283,6 @@ there is danger of dropping from the buffer pool. @return true if bpage should be made younger */ inline bool buf_page_peek_if_too_old(const buf_page_t *bpage); -/** Move a page to the start of the buffer pool LRU list if it is too old. -@param[in,out] bpage buffer pool page */ -inline void buf_page_make_young_if_needed(buf_page_t *bpage) -{ - if (UNIV_UNLIKELY(buf_page_peek_if_too_old(bpage))) { - buf_page_make_young(bpage); - } -} - /********************************************************************//** Increments the modify clock of a frame by 1. The caller must (1) own the buf_pool.mutex and block bufferfix count has to be zero, (2) or own an x-lock @@ -656,12 +645,9 @@ public: access_time= 0; } - void set_os_unused() + void set_os_unused() const { MEM_NOACCESS(frame, srv_page_size); -#ifdef MADV_FREE - madvise(frame, srv_page_size, MADV_FREE); -#endif } void set_os_used() const @@ -1301,6 +1287,11 @@ public: /** Resize from srv_buf_pool_old_size to srv_buf_pool_size. */ inline void resize(); +#ifdef __linux__ + /** Collect garbage (release pages from the LRU list) */ + inline void garbage_collect(); +#endif + /** @return whether resize() is in progress */ bool resize_in_progress() const { @@ -1507,10 +1498,8 @@ public: n_chunks_new / 4 * chunks->size; } - /** @return whether the buffer pool has run out */ - TPOOL_SUPPRESS_TSAN - bool ran_out() const - { return UNIV_UNLIKELY(!try_LRU_scan || !UT_LIST_GET_LEN(free)); } + /** @return whether the buffer pool is running low */ + bool need_LRU_eviction() const; /** @return whether the buffer pool is shrinking */ inline bool is_shrinking() const @@ -1836,6 +1825,9 @@ public: Set whenever the free list grows, along with a broadcast of done_free. Protected by buf_pool.mutex. */ Atomic_relaxed<bool> try_LRU_scan; + /** Whether we have warned to be running out of buffer pool */ + std::atomic_flag LRU_warned; + /* @} */ /** @name LRU replacement algorithm fields */ @@ -1898,7 +1890,8 @@ public: a delete-buffering operation is pending. Protected by mutex. */ buf_page_t watch[innodb_purge_threads_MAX + 1]; /** Reserve a buffer. */ - buf_tmp_buffer_t *io_buf_reserve() { return io_buf.reserve(); } + buf_tmp_buffer_t *io_buf_reserve(bool wait_for_reads) + { return io_buf.reserve(wait_for_reads); } /** Remove a block from flush_list. @param bpage buffer pool page */ @@ -1933,7 +1926,7 @@ private: void close(); /** Reserve a buffer */ - buf_tmp_buffer_t *reserve(); + buf_tmp_buffer_t *reserve(bool wait_for_reads); } io_buf; /** whether resize() is in the critical path */ diff --git a/storage/innobase/include/buf0dblwr.h b/storage/innobase/include/buf0dblwr.h index 9932b0e5..6e7662d9 100644 --- a/storage/innobase/include/buf0dblwr.h +++ b/storage/innobase/include/buf0dblwr.h @@ -105,7 +105,8 @@ public: If we are upgrading from a version before MySQL 4.1, then this function performs the necessary update operations to support innodb_file_per_table. If we are in a crash recovery, this function - loads the pages from double write buffer into memory. + loads the pages from double write buffer which are not older than + the checkpoint into memory. @param file File handle @param path Path name of file @return DB_SUCCESS or error code */ diff --git a/storage/innobase/include/buf0lru.h b/storage/innobase/include/buf0lru.h index aec08e77..28410276 100644 --- a/storage/innobase/include/buf0lru.h +++ b/storage/innobase/include/buf0lru.h @@ -108,6 +108,16 @@ buf_LRU_add_block( blocks in the LRU list, else put to the start; if the LRU list is very short, added to the start regardless of this parameter */ + +/** Move a block to the start of the buf_pool.LRU list. +@param bpage buffer pool page */ +void buf_page_make_young(buf_page_t *bpage); +/** Flag a page accessed in buf_pool and move it to the start of buf_pool.LRU +if it is too old. +@param bpage buffer pool page +@return whether this is not the first access */ +bool buf_page_make_young_if_needed(buf_page_t *bpage); + /******************************************************************//** Adds a block to the LRU list of decompressed zip pages. */ void diff --git a/storage/innobase/include/dict0load.h b/storage/innobase/include/dict0load.h index f7d33d5b..3143aafd 100644 --- a/storage/innobase/include/dict0load.h +++ b/storage/innobase/include/dict0load.h @@ -35,22 +35,16 @@ Created 4/24/1996 Heikki Tuuri #include "btr0types.h" #include <deque> +#include <set> /** A stack of table names related through foreign key constraints */ typedef std::deque<const char*, ut_allocator<const char*> > dict_names_t; -/** Check each tablespace found in the data dictionary. -Then look at each table defined in SYS_TABLES that has a space_id > 0 -to find all the file-per-table tablespaces. +/** Check MAX(SPACE) FROM SYS_TABLES and store it in fil_system. +Open each data file if an encryption plugin has been loaded. -In a crash recovery we already have some tablespace objects created from -processing the REDO log. We will compare the -space_id information in the data dictionary to what we find in the -tablespace file. In addition, more validation will be done if recovery -was needed and force_recovery is not set. - -We also scan the biggest space id, and store it to fil_system. */ -void dict_check_tablespaces_and_store_max_id(); +@param spaces set of tablespace files to open */ +void dict_check_tablespaces_and_store_max_id(const std::set<uint32_t> *spaces); /** Make sure the data_file_name is saved in dict_table_t if needed. @param[in,out] table Table object */ diff --git a/storage/innobase/include/dict0stats.h b/storage/innobase/include/dict0stats.h index 0dc1b984..3b006daf 100644 --- a/storage/innobase/include/dict0stats.h +++ b/storage/innobase/include/dict0stats.h @@ -235,4 +235,13 @@ dict_stats_report_error(dict_table_t* table, bool defragment = false) void test_dict_stats_all(); #endif /* UNIV_ENABLE_UNIT_TEST_DICT_STATS */ +/** Write all zeros (or 1 where it makes sense) into a table +and its indexes'statistics members. The resulting stats +correspond to an empty table. +@param table table stats to be emptied +@param empty_defrag_stats empty the defrag stats */ +void +dict_stats_empty_table( + dict_table_t* table, + bool empty_defrag_stats); #endif /* dict0stats_h */ diff --git a/storage/innobase/include/fil0fil.h b/storage/innobase/include/fil0fil.h index 6f58e3c1..cdc32515 100644 --- a/storage/innobase/include/fil0fil.h +++ b/storage/innobase/include/fil0fil.h @@ -359,8 +359,6 @@ struct fil_space_t final lsn_t max_lsn; /** tablespace identifier */ uint32_t id; - /** whether undo tablespace truncation is in progress */ - bool is_being_truncated; fil_type_t purpose;/*!< purpose */ UT_LIST_BASE_NODE_T(fil_node_t) chain; /*!< base node for the file chain */ @@ -440,6 +438,8 @@ private: /** LSN of freeing last page; protected by freed_range_mutex */ lsn_t last_freed_lsn; + /** LSN of undo tablespace creation or 0; protected by latch */ + lsn_t create_lsn; public: /** @return whether doublewrite buffering is needed */ inline bool use_doublewrite() const; @@ -447,6 +447,12 @@ public: /** @return whether a page has been freed */ inline bool is_freed(uint32_t page); + /** Set create_lsn. */ + inline void set_create_lsn(lsn_t lsn); + + /** @return the latest tablespace rebuild LSN, or 0 */ + lsn_t get_create_lsn() const { return create_lsn; } + /** Apply freed_ranges to the file. @param writable whether the file is writable @return number of pages written or hole-punched */ @@ -524,9 +530,6 @@ public: /** Note that operations on the tablespace must stop. */ inline void set_stopping(); - /** Note that operations on the tablespace can resume after truncation */ - inline void clear_stopping(); - /** Drop the tablespace and wait for any pending operations to cease @param id tablespace identifier @param detached_handle pointer to file to be closed later, or nullptr @@ -1555,14 +1558,6 @@ inline void fil_space_t::set_stopping() #endif } -inline void fil_space_t::clear_stopping() -{ - mysql_mutex_assert_owner(&fil_system.mutex); - static_assert(STOPPING_WRITES == 1U << 30, "compatibility"); - ut_d(auto n=) n_pending.fetch_sub(STOPPING_WRITES, std::memory_order_relaxed); - ut_ad((n & STOPPING) == STOPPING_WRITES); -} - /** Flush pending writes from the file system cache to the file. */ template<bool have_reference> inline void fil_space_t::flush() { @@ -1802,7 +1797,7 @@ bool fil_comp_algo_loaded(ulint comp_algo); and write out FILE_MODIFY if needed, and write FILE_CHECKPOINT. @param lsn checkpoint LSN @return current LSN */ -lsn_t fil_names_clear(lsn_t lsn); +ATTRIBUTE_COLD lsn_t fil_names_clear(lsn_t lsn); #ifdef UNIV_ENABLE_UNIT_TEST_MAKE_FILEPATH void test_make_filepath(); diff --git a/storage/innobase/include/fts0priv.inl b/storage/innobase/include/fts0priv.inl index da14cfcb..3cb09c92 100644 --- a/storage/innobase/include/fts0priv.inl +++ b/storage/innobase/include/fts0priv.inl @@ -34,29 +34,6 @@ fts_write_object_id( ib_id_t id, /* in: a table/index id */ char* str) /* in: buffer to write the id to */ { - -#ifdef _WIN32 - - DBUG_EXECUTE_IF("innodb_test_wrong_non_windows_fts_aux_table_name", - return(sprintf(str, UINT64PFx, id));); - - /* Use this to construct old(5.6.14 and 5.7.3) windows - ambiguous aux table names */ - DBUG_EXECUTE_IF("innodb_test_wrong_fts_aux_table_name", - return(sprintf(str, "%016llu", (ulonglong) id));); - -#else /* _WIN32 */ - - /* Use this to construct old(5.6.14 and 5.7.3) windows - ambiguous aux table names */ - DBUG_EXECUTE_IF("innodb_test_wrong_windows_fts_aux_table_name", - return(sprintf(str, "%016llu", (ulonglong) id));); - - DBUG_EXECUTE_IF("innodb_test_wrong_fts_aux_table_name", - return(sprintf(str, "%016llx", (ulonglong) id));); - -#endif /* _WIN32 */ - return(sprintf(str, "%016llx", (ulonglong) id)); } diff --git a/storage/innobase/include/log0log.h b/storage/innobase/include/log0log.h index f873eabf..54851ca0 100644 --- a/storage/innobase/include/log0log.h +++ b/storage/innobase/include/log0log.h @@ -79,13 +79,6 @@ ATTRIBUTE_COLD void log_make_checkpoint(); /** Make a checkpoint at the latest lsn on shutdown. */ ATTRIBUTE_COLD void logs_empty_and_mark_files_at_shutdown(); -/** -Checks that there is enough free space in the log to start a new query step. -Flushes the log buffer or makes a new checkpoint if necessary. NOTE: this -function may only be called if the calling thread owns no synchronization -objects! */ -ATTRIBUTE_COLD void log_check_margins(); - /******************************************************//** Prints info of the log. */ void @@ -179,24 +172,33 @@ private: std::atomic<lsn_t> flushed_to_disk_lsn; /** log sequence number when log resizing was initiated, or 0 */ std::atomic<lsn_t> resize_lsn; - /** set when there may be need to flush the log buffer, or - preflush buffer pool pages, or initiate a log checkpoint. + /** set when there may be need to initiate a log checkpoint. This must hold if lsn - last_checkpoint_lsn > max_checkpoint_age. */ - std::atomic<bool> check_flush_or_checkpoint_; - + std::atomic<bool> need_checkpoint; #if defined(__aarch64__) -/* On ARM, we do more spinning */ -typedef srw_spin_lock log_rwlock_t; -#define LSN_LOCK_ATTR MY_MUTEX_INIT_FAST + /* On ARM, we do more spinning */ + typedef srw_spin_lock log_rwlock; + typedef pthread_mutex_wrapper<true> log_lsn_lock; #else -typedef srw_lock log_rwlock_t; -#define LSN_LOCK_ATTR nullptr + typedef srw_lock log_rwlock; + typedef srw_mutex log_lsn_lock; #endif public: - /** rw-lock protecting buf */ - alignas(CPU_LEVEL1_DCACHE_LINESIZE) log_rwlock_t latch; + /** rw-lock protecting writes to buf; normal mtr_t::commit() + outside any log checkpoint is covered by a shared latch */ + alignas(CPU_LEVEL1_DCACHE_LINESIZE) log_rwlock latch; +private: + /** mutex protecting buf_free et al, together with latch */ + log_lsn_lock lsn_lock; +public: + /** first free offset within buf use; protected by lsn_lock */ + Atomic_relaxed<size_t> buf_free; + /** number of write requests (to buf); protected by lsn_lock */ + size_t write_to_buf; + /** number of append_prepare_wait(); protected by lsn_lock */ + size_t waits; private: /** Last written LSN */ lsn_t write_lsn; @@ -227,20 +229,12 @@ private: /** Buffer for writing to resize_log; @see flush_buf */ byte *resize_flush_buf; - /** spin lock protecting lsn, buf_free in append_prepare() */ - alignas(CPU_LEVEL1_DCACHE_LINESIZE) pthread_mutex_t lsn_lock; - void init_lsn_lock() { pthread_mutex_init(&lsn_lock, LSN_LOCK_ATTR); } - void lock_lsn() { pthread_mutex_lock(&lsn_lock); } - void unlock_lsn() { pthread_mutex_unlock(&lsn_lock); } - void destroy_lsn_lock() { pthread_mutex_destroy(&lsn_lock); } + void init_lsn_lock() {lsn_lock.init(); } + void lock_lsn() { lsn_lock.wr_lock(); } + void unlock_lsn() {lsn_lock.wr_unlock(); } + void destroy_lsn_lock() { lsn_lock.destroy(); } public: - /** first free offset within buf use; protected by lsn_lock */ - Atomic_relaxed<size_t> buf_free; - /** number of write requests (to buf); protected by exclusive lsn_lock */ - ulint write_to_buf; - /** number of waits in append_prepare(); protected by lsn_lock */ - ulint waits; /** recommended maximum size of buf, after which the buffer is flushed */ size_t max_buf_free; @@ -308,6 +302,9 @@ public: bool is_opened() const noexcept { return log.is_opened(); } + /** @return target write LSN to react on buf_free >= max_buf_free */ + inline lsn_t get_write_target() const; + /** @return LSN at which log resizing was started and is still in progress @retval 0 if no log resizing is in progress */ lsn_t resize_in_progress() const noexcept @@ -419,13 +416,14 @@ public: inline void persist(lsn_t lsn) noexcept; #endif - bool check_flush_or_checkpoint() const + bool check_for_checkpoint() const + { + return UNIV_UNLIKELY(need_checkpoint.load(std::memory_order_relaxed)); + } + void set_check_for_checkpoint(bool need= true) { - return UNIV_UNLIKELY - (check_flush_or_checkpoint_.load(std::memory_order_relaxed)); + need_checkpoint.store(need, std::memory_order_relaxed); } - void set_check_flush_or_checkpoint(bool flag= true) - { check_flush_or_checkpoint_.store(flag, std::memory_order_relaxed); } /** Make previous write_buf() durable and update flushed_to_disk_lsn. */ bool flush(lsn_t lsn) noexcept; @@ -446,8 +444,9 @@ public: private: /** Wait in append_prepare() for buffer to become available + @param lsn log sequence number to write up to @param ex whether log_sys.latch is exclusively locked */ - ATTRIBUTE_COLD static void append_prepare_wait(bool ex) noexcept; + ATTRIBUTE_COLD void append_prepare_wait(lsn_t lsn, bool ex) noexcept; public: /** Reserve space in the log buffer for appending data. @tparam pmem log_sys.is_pmem() diff --git a/storage/innobase/include/log0recv.h b/storage/innobase/include/log0recv.h index 6d75e15a..a73b7279 100644 --- a/storage/innobase/include/log0recv.h +++ b/storage/innobase/include/log0recv.h @@ -44,6 +44,11 @@ ATTRIBUTE_COLD MY_ATTRIBUTE((nonnull, warn_unused_result)) @return whether the page was recovered correctly */ bool recv_recover_page(fil_space_t* space, buf_page_t* bpage); +/** Read the latest checkpoint information from log file +and store it in log_sys.next_checkpoint and recv_sys.file_checkpoint +@return error code or DB_SUCCESS */ +dberr_t recv_recovery_read_checkpoint(); + /** Start recovering from a redo log checkpoint. of first system tablespace page @return error code or DB_SUCCESS */ @@ -114,7 +119,19 @@ struct recv_dblwr_t @param name tablespace filepath @param file tablespace file handle @return whether the operation failed */ - bool restore_first_page(uint32_t space_id, const char *name, os_file_t file); + bool restore_first_page(uint32_t space_id, const char *name, + pfs_os_file_t file); + + /** Restore the first page of the given tablespace from + doublewrite buffer. + 1) Find the page which has page_no as 0 + 2) Read first 3 pages from tablespace file + 3) Compare the space_ids from the pages with page0 which + was retrieved from doublewrite buffer + @param name tablespace filepath + @param file tablespace file handle + @return space_id or 0 in case of error */ + uint32_t find_first_page(const char *name, pfs_os_file_t file); typedef std::deque<byte*, ut_allocator<byte*> > list; diff --git a/storage/innobase/include/mtr0mtr.h b/storage/innobase/include/mtr0mtr.h index 841cfab1..c916edc9 100644 --- a/storage/innobase/include/mtr0mtr.h +++ b/storage/innobase/include/mtr0mtr.h @@ -89,8 +89,9 @@ struct mtr_t { { auto s= m_memo.size(); rollback_to_savepoint(s - 1, s); } /** Commit a mini-transaction that is shrinking a tablespace. - @param space tablespace that is being shrunk */ - ATTRIBUTE_COLD void commit_shrink(fil_space_t &space); + @param space tablespace that is being shrunk + @param size new size in pages */ + ATTRIBUTE_COLD void commit_shrink(fil_space_t &space, uint32_t size); /** Commit a mini-transaction that is deleting or renaming a file. @param space tablespace that is being renamed or deleted @@ -105,7 +106,7 @@ struct mtr_t { This is to be used at log_checkpoint(). @param checkpoint_lsn the log sequence number of a checkpoint, or 0 @return current LSN */ - lsn_t commit_files(lsn_t checkpoint_lsn= 0); + ATTRIBUTE_COLD lsn_t commit_files(lsn_t checkpoint_lsn= 0); /** @return mini-transaction savepoint (current size of m_memo) */ ulint get_savepoint() const diff --git a/storage/innobase/include/os0file.h b/storage/innobase/include/os0file.h index c9db6a1f..c8374515 100644 --- a/storage/innobase/include/os0file.h +++ b/storage/innobase/include/os0file.h @@ -142,9 +142,11 @@ static const ulint OS_FILE_NORMAL = 62; /* @} */ /** Types for file create @{ */ -static const ulint OS_DATA_FILE = 100; -static const ulint OS_LOG_FILE = 101; -static const ulint OS_DATA_FILE_NO_O_DIRECT = 103; +static constexpr ulint OS_DATA_FILE = 100; +static constexpr ulint OS_LOG_FILE = 101; +#if defined _WIN32 || defined HAVE_FCNTL_DIRECT +static constexpr ulint OS_DATA_FILE_NO_O_DIRECT = 103; +#endif /* @} */ /** Error codes from os_file_get_last_error @{ */ @@ -373,7 +375,7 @@ os_file_create_simple_no_error_handling_func( bool* success) MY_ATTRIBUTE((warn_unused_result)); -#ifdef _WIN32 +#ifndef HAVE_FCNTL_DIRECT #define os_file_set_nocache(fd, file_name, operation_name) do{}while(0) #else /** Tries to disable OS caching on an opened file descriptor. diff --git a/storage/innobase/include/srv0srv.h b/storage/innobase/include/srv0srv.h index db846795..457d9ab5 100644 --- a/storage/innobase/include/srv0srv.h +++ b/storage/innobase/include/srv0srv.h @@ -210,14 +210,11 @@ extern unsigned long long srv_max_undo_log_size; extern uint srv_n_fil_crypt_threads; extern uint srv_n_fil_crypt_threads_started; -/** Rate at which UNDO records should be purged. */ -extern ulong srv_purge_rseg_truncate_frequency; - /** Enable or Disable Truncate of UNDO tablespace. */ extern my_bool srv_undo_log_truncate; /** Default size of UNDO tablespace (10MiB for innodb_page_size=16k) */ -constexpr ulint SRV_UNDO_TABLESPACE_SIZE_IN_PAGES= (10U << 20) / +constexpr uint32_t SRV_UNDO_TABLESPACE_SIZE_IN_PAGES= (10U << 20) / UNIV_PAGE_SIZE_DEF; extern char* srv_log_group_home_dir; diff --git a/storage/innobase/include/srw_lock.h b/storage/innobase/include/srw_lock.h index 1dca0cc1..01067322 100644 --- a/storage/innobase/include/srw_lock.h +++ b/storage/innobase/include/srw_lock.h @@ -34,7 +34,6 @@ this program; if not, write to the Free Software Foundation, Inc., # define SUX_LOCK_GENERIC /* Use dummy implementation for debugging purposes */ #endif -#ifdef SUX_LOCK_GENERIC /** An exclusive-only variant of srw_lock */ template<bool spinloop> class pthread_mutex_wrapper final @@ -70,7 +69,6 @@ template<> inline void pthread_mutex_wrapper<true>::wr_lock() { if (!wr_lock_try()) wr_wait(); } # endif -#endif /** Futex-based mutex */ template<bool spinloop> @@ -541,7 +539,7 @@ public: /** @return whether any lock may be held by any thread */ bool is_locked_or_waiting() const noexcept { return lock.is_locked_or_waiting(); } - /** @return whether an exclusive lock may be held by any thread */ + /** @return whether a shared or exclusive lock may be held by any thread */ bool is_locked() const noexcept { return lock.is_locked(); } /** @return whether an exclusive lock may be held by any thread */ bool is_write_locked() const noexcept { return lock.is_write_locked(); } diff --git a/storage/innobase/include/trx0purge.h b/storage/innobase/include/trx0purge.h index 3ddd2e98..0f4f8afa 100644 --- a/storage/innobase/include/trx0purge.h +++ b/storage/innobase/include/trx0purge.h @@ -140,6 +140,15 @@ private: bool m_initialized{false}; /** whether purge is enabled; protected by latch and std::atomic */ std::atomic<bool> m_enabled{false}; + /** The primary candidate for iterator::free_history() is + rseg=trx_sys.rseg_array[skipped_rseg]. This field may be changed + after invoking rseg.set_skip_allocation() and rseg.clear_skip_allocation() + and while holding the exclusive rseg.latch. + + This may only be 0 if innodb_undo_tablespaces=0, because rollback segment + 0 always resides in the system tablespace and would never be used when + dedicated undo tablespaces are in use. */ + Atomic_relaxed<uint8_t> skipped_rseg; public: /** whether purge is active (may hold table handles) */ std::atomic<bool> m_active{false}; @@ -197,6 +206,11 @@ public: return undo_no <= other.undo_no; } + /** Remove unnecessary history data from a rollback segment. + @param rseg rollback segment + @return error code */ + inline dberr_t free_history_rseg(trx_rseg_t &rseg) const; + /** Free the undo pages up to this. */ dberr_t free_history() const; @@ -240,14 +254,15 @@ public: by the pq_mutex */ mysql_mutex_t pq_mutex; /*!< Mutex protecting purge_queue */ - /** Undo tablespace file truncation (only accessed by the - srv_purge_coordinator_thread) */ - struct { - /** The undo tablespace that is currently being truncated */ - fil_space_t* current; - /** The undo tablespace that was last truncated */ - fil_space_t* last; - } truncate; + /** innodb_undo_log_truncate=ON state; + only modified by purge_coordinator_callback() */ + struct { + /** The undo tablespace that is currently being truncated */ + Atomic_relaxed<fil_space_t*> current; + /** The number of the undo tablespace that was last truncated, + relative from srv_undo_space_id_start */ + uint32_t last; + } truncate_undo_space; /** Create the instance */ void create(); @@ -357,6 +372,26 @@ public: typically via purge_sys_t::view_guard. */ return view.sees(id); } + +private: + /** Enable the use of a rollback segment and advance skipped_rseg, + after iterator::free_history_rseg() had invoked + rseg.set_skip_allocation(). */ + inline void rseg_enable(trx_rseg_t &rseg); + + /** Try to start truncating a tablespace. + @param id undo tablespace identifier + @param size the maximum desired undo tablespace size, in pages + @return undo tablespace whose truncation was started + @retval nullptr if truncation is not currently possible */ + inline fil_space_t *undo_truncate_try(uint32_t id, uint32_t size); +public: + /** Check if innodb_undo_log_truncate=ON needs to be handled. + This is only to be called by purge_coordinator_callback(). + @return undo tablespace chosen by innodb_undo_log_truncate=ON + @retval nullptr if truncation is not currently possible */ + fil_space_t *truncating_tablespace(); + /** A wrapper around trx_sys_t::clone_oldest_view(). */ template<bool also_end_view= false> void clone_oldest_view() diff --git a/storage/innobase/include/trx0rseg.h b/storage/innobase/include/trx0rseg.h index 43e0c290..7fa43047 100644 --- a/storage/innobase/include/trx0rseg.h +++ b/storage/innobase/include/trx0rseg.h @@ -73,14 +73,15 @@ private: /** Reference counter to track is_persistent() transactions, with SKIP flag. */ std::atomic<uint32_t> ref; - +public: /** Whether undo tablespace truncation is pending */ static constexpr uint32_t SKIP= 1; /** Transaction reference count multiplier */ static constexpr uint32_t REF= 2; + /** @return the reference count and flags */ uint32_t ref_load() const { return ref.load(std::memory_order_relaxed); } - +private: /** Set the SKIP bit */ void ref_set_skip() { diff --git a/storage/innobase/include/trx0sys.h b/storage/innobase/include/trx0sys.h index 5dd0169f..3fa41fdf 100644 --- a/storage/innobase/include/trx0sys.h +++ b/storage/innobase/include/trx0sys.h @@ -902,8 +902,8 @@ public: uint64_t recovered_binlog_offset; /** Latest recovered binlog file name */ char recovered_binlog_filename[TRX_SYS_MYSQL_LOG_NAME_LEN]; - /** FIL_PAGE_LSN of the page with the latest recovered binlog metadata */ - lsn_t recovered_binlog_lsn; + /** Set when latest position is from pre-version 10.3.5 TRX_SYS. */ + bool recovered_binlog_is_legacy_pos; /** @@ -1191,6 +1191,11 @@ public: return count; } + /** Disable further allocation of transactions in a rollback segment + that are subject to innodb_undo_log_truncate=ON + @param space undo tablespace that will be truncated */ + inline void undo_truncate_start(fil_space_t &space); + /** Set the undo log empty value */ void set_undo_non_empty(bool val) { diff --git a/storage/innobase/include/trx0trx.h b/storage/innobase/include/trx0trx.h index 3cfbe331..0a3e0d62 100644 --- a/storage/innobase/include/trx0trx.h +++ b/storage/innobase/include/trx0trx.h @@ -1108,6 +1108,7 @@ public: { ut_ad(state == TRX_STATE_NOT_STARTED); ut_ad(!id); + ut_ad(!*detailed_error); ut_ad(!mutex_is_owner()); ut_ad(!has_logged()); ut_ad(!is_referenced()); diff --git a/storage/innobase/log/log0log.cc b/storage/innobase/log/log0log.cc index 91999c81..9f39b303 100644 --- a/storage/innobase/log/log0log.cc +++ b/storage/innobase/log/log0log.cc @@ -100,6 +100,7 @@ bool log_t::create() /* LSN 0 and 1 are reserved; @see buf_page_t::oldest_modification_ */ lsn.store(FIRST_LSN, std::memory_order_relaxed); flushed_to_disk_lsn.store(FIRST_LSN, std::memory_order_relaxed); + need_checkpoint.store(true, std::memory_order_relaxed); write_lsn= FIRST_LSN; #ifndef HAVE_PMEM @@ -124,18 +125,17 @@ bool log_t::create() TRASH_ALLOC(flush_buf, buf_size); checkpoint_buf= static_cast<byte*>(aligned_malloc(4096, 4096)); memset_aligned<4096>(checkpoint_buf, 0, 4096); + max_buf_free= buf_size / LOG_BUF_FLUSH_RATIO - LOG_BUF_FLUSH_MARGIN; #else ut_ad(!checkpoint_buf); ut_ad(!buf); ut_ad(!flush_buf); + max_buf_free= 1; #endif latch.SRW_LOCK_INIT(log_latch_key); init_lsn_lock(); - max_buf_free= buf_size / LOG_BUF_FLUSH_RATIO - LOG_BUF_FLUSH_MARGIN; - set_check_flush_or_checkpoint(); - last_checkpoint_lsn= FIRST_LSN; log_capacity= 0; max_modified_age_async= 0; @@ -236,6 +236,7 @@ void log_t::attach_low(log_file_t file, os_offset_t size) log.close(); mprotect(ptr, size_t(size), PROT_READ); buf= static_cast<byte*>(ptr); + max_buf_free= size; # if defined __linux__ || defined _WIN32 set_block_size(CPU_LEVEL1_DCACHE_LINESIZE); # endif @@ -264,6 +265,7 @@ void log_t::attach_low(log_file_t file, os_offset_t size) TRASH_ALLOC(buf, buf_size); TRASH_ALLOC(flush_buf, buf_size); + max_buf_free= buf_size / LOG_BUF_FLUSH_RATIO - LOG_BUF_FLUSH_MARGIN; #endif #if defined __linux__ || defined _WIN32 @@ -813,8 +815,8 @@ template<bool release_latch> inline lsn_t log_t::write_buf() noexcept #ifndef SUX_LOCK_GENERIC ut_ad(latch.is_write_locked()); #endif - ut_ad(!srv_read_only_mode); ut_ad(!is_pmem()); + ut_ad(!srv_read_only_mode); const lsn_t lsn{get_lsn(std::memory_order_relaxed)}; @@ -849,7 +851,7 @@ template<bool release_latch> inline lsn_t log_t::write_buf() noexcept ... /* TODO: Update the LSN and adjust other code. */ #else /* The rest of the block will be written as garbage. - (We want to avoid memset() while holding mutex.) + (We want to avoid memset() while holding exclusive log_sys.latch) This block will be overwritten later, once records beyond the current LSN are generated. */ # ifdef HAVE_valgrind @@ -886,6 +888,7 @@ template<bool release_latch> inline lsn_t log_t::write_buf() noexcept write_lsn= lsn; } + set_check_for_checkpoint(false); return lsn; } @@ -927,8 +930,9 @@ wait and check if an already running write is covering the request. void log_write_up_to(lsn_t lsn, bool durable, const completion_callback *callback) { - ut_ad(!srv_read_only_mode); + ut_ad(!srv_read_only_mode || (log_sys.buf_free < log_sys.max_buf_free)); ut_ad(lsn != LSN_MAX); + ut_ad(lsn != 0); if (UNIV_UNLIKELY(recv_no_ibuf_operations)) { @@ -985,7 +989,6 @@ repeat: @param durable whether to wait for a durable write to complete */ void log_buffer_flush_to_disk(bool durable) { - ut_ad(!srv_read_only_mode); log_write_up_to(log_sys.get_lsn(std::memory_order_acquire), durable); } @@ -1017,16 +1020,6 @@ ATTRIBUTE_COLD void log_write_and_flush() #endif } -/******************************************************************** - -Tries to establish a big enough margin of free space in the log buffer, such -that a new log entry can be catenated without an immediate need for a flush. */ -ATTRIBUTE_COLD static void log_flush_margin() -{ - if (log_sys.buf_free > log_sys.max_buf_free) - log_buffer_flush_to_disk(false); -} - /****************************************************************//** Tries to establish a big enough margin of free space in the log, such that a new log entry can be catenated without an immediate need for a @@ -1034,12 +1027,12 @@ checkpoint. NOTE: this function may only be called if the calling thread owns no synchronization objects! */ ATTRIBUTE_COLD static void log_checkpoint_margin() { - while (log_sys.check_flush_or_checkpoint()) + while (log_sys.check_for_checkpoint()) { log_sys.latch.rd_lock(SRW_LOCK_CALL); ut_ad(!recv_no_log_write); - if (!log_sys.check_flush_or_checkpoint()) + if (!log_sys.check_for_checkpoint()) { func_exit: log_sys.latch.rd_unlock(); @@ -1055,7 +1048,7 @@ func_exit: #ifndef DBUG_OFF skip_checkpoint: #endif - log_sys.set_check_flush_or_checkpoint(false); + log_sys.set_check_for_checkpoint(false); goto func_exit; } @@ -1069,30 +1062,17 @@ func_exit: } } -/** -Checks that there is enough free space in the log to start a new query step. -Flushes the log buffer or makes a new checkpoint if necessary. NOTE: this -function may only be called if the calling thread owns no synchronization -objects! */ -ATTRIBUTE_COLD void log_check_margins() -{ - do - { - log_flush_margin(); - log_checkpoint_margin(); - ut_ad(!recv_no_log_write); - } - while (log_sys.check_flush_or_checkpoint()); -} - /** Wait for a log checkpoint if needed. NOTE that this function may only be called while not holding any synchronization objects except dict_sys.latch. */ void log_free_check() { ut_ad(!lock_sys.is_writer()); - if (log_sys.check_flush_or_checkpoint()) - log_check_margins(); + if (log_sys.check_for_checkpoint()) + { + ut_ad(!recv_no_log_write); + log_checkpoint_margin(); + } } extern void buf_resize_shutdown(); diff --git a/storage/innobase/log/log0recv.cc b/storage/innobase/log/log0recv.cc index 3c3fe41e..e72f842f 100644 --- a/storage/innobase/log/log0recv.cc +++ b/storage/innobase/log/log0recv.cc @@ -833,7 +833,22 @@ processed: filename= tbl_name + 1; } } - space->add(filename, OS_FILE_CLOSED, size, false, false); + pfs_os_file_t handle= OS_FILE_CLOSED; + if (srv_operation == SRV_OPERATION_RESTORE) + { + /* During mariadb-backup --backup, a table could be renamed, + created and dropped, and we may be missing the file at this + point of --prepare. Try to create the file if it does not exist + already. If the file exists, we'll pass handle=OS_FILE_CLOSED + and the file will be opened normally in fil_space_t::acquire() + inside recv_sys_t::recover_deferred(). */ + bool success; + handle= os_file_create(innodb_data_file_key, filename, + OS_FILE_CREATE | OS_FILE_ON_ERROR_NO_EXIT | + OS_FILE_ON_ERROR_SILENT, + OS_FILE_AIO, OS_DATA_FILE, false, &success); + } + space->add(filename, handle, size, false, false); space->recv_size= it->second.size; space->size_in_header= size; return space; @@ -1238,7 +1253,8 @@ static void fil_name_process(const char *name, ulint len, uint32_t space_id, file_name_t& f = p.first->second; - if (auto d = deferred_spaces.find(space_id)) { + auto d = deferred_spaces.find(space_id); + if (d) { if (deleted) { d->deleted = true; goto got_deleted; @@ -1311,7 +1327,16 @@ same_space: FILE_* record. */ ut_ad(space == NULL); - if (srv_force_recovery) { + if (srv_operation == SRV_OPERATION_RESTORE && d + && ftype == FILE_RENAME) { +rename: + d->file_name = fname.name; + f.name = fname.name; + break; + } + + if (srv_force_recovery + || srv_operation == SRV_OPERATION_RESTORE) { /* Without innodb_force_recovery, missing tablespaces will only be reported in @@ -1330,7 +1355,11 @@ same_space: break; case FIL_LOAD_DEFER: - /** Skip the deferred spaces + if (d && ftype == FILE_RENAME + && srv_operation == SRV_OPERATION_RESTORE) { + goto rename; + } + /* Skip the deferred spaces when lsn is already processed */ if (!if_exists) { deferred_spaces.add( @@ -1735,20 +1764,6 @@ dberr_t recv_sys_t::find_checkpoint() { if (wrong_size) return DB_CORRUPTION; - if (log_sys.next_checkpoint_lsn < 8204) - { - /* Before MDEV-14425, InnoDB had a minimum LSN of 8192+12=8204. - Likewise, mariadb-backup --prepare would create an empty - ib_logfile0 after applying the log. We will allow an upgrade - from such an empty log. - - If a user replaces the redo log with an empty file and the - FIL_PAGE_FILE_FLUSH_LSN field was zero in the system - tablespace (see SysTablespace::read_lsn_and_check_flags()) we - must refuse to start up. */ - sql_print_error("InnoDB: ib_logfile0 is empty, and LSN is unknown."); - return DB_CORRUPTION; - } lsn= log_sys.next_checkpoint_lsn; log_sys.format= log_t::FORMAT_3_23; goto upgrade; @@ -2409,7 +2424,7 @@ struct recv_ring : public recv_buf { const size_t s(*this - start); ut_ad(s + len <= srv_page_size); - if (!log_sys.is_encrypted()) + if (!len || !log_sys.is_encrypted()) { if (start.ptr + s == ptr && ptr + len <= end()) return ptr; @@ -3205,7 +3220,7 @@ static buf_block_t *recv_recover_page(buf_block_t *block, mtr_t &mtr, skipped_after_init = false; ut_ad(end_lsn == page_lsn); if (end_lsn != page_lsn) { - sql_print_warning( + sql_print_information( "InnoDB: The last skipped log record" " LSN " LSN_PF " is not equal to page LSN " LSN_PF, @@ -4012,7 +4027,6 @@ static bool recv_scan_log(bool last_phase) const size_t block_size_1{log_sys.get_block_size() - 1}; mysql_mutex_lock(&recv_sys.mutex); - ut_d(recv_sys.after_apply= last_phase); if (!last_phase) recv_sys.clear(); else @@ -4221,6 +4235,7 @@ static bool recv_scan_log(bool last_phase) recv_sys.lsn= rewound_lsn; } func_exit: + ut_d(recv_sys.after_apply= last_phase); mysql_mutex_unlock(&recv_sys.mutex); DBUG_RETURN(!store); } @@ -4507,12 +4522,36 @@ done: return err; } +dberr_t recv_recovery_read_checkpoint() +{ + ut_ad(srv_operation <= SRV_OPERATION_EXPORT_RESTORED || + srv_operation == SRV_OPERATION_RESTORE || + srv_operation == SRV_OPERATION_RESTORE_EXPORT); + ut_d(mysql_mutex_lock(&buf_pool.mutex)); + ut_ad(UT_LIST_GET_LEN(buf_pool.LRU) == 0); + ut_ad(UT_LIST_GET_LEN(buf_pool.unzip_LRU) == 0); + ut_d(mysql_mutex_unlock(&buf_pool.mutex)); + + if (srv_force_recovery >= SRV_FORCE_NO_LOG_REDO) + { + sql_print_information("InnoDB: innodb_force_recovery=6" + " skips redo log apply"); + return DB_SUCCESS; + } + + log_sys.latch.wr_lock(SRW_LOCK_CALL); + dberr_t err= recv_sys.find_checkpoint(); + log_sys.latch.wr_unlock(); + return err; +} + /** Start recovering from a redo log checkpoint. of first system tablespace page @return error code or DB_SUCCESS */ dberr_t recv_recovery_from_checkpoint_start() { - bool rescan = false; + bool rescan = false; + dberr_t err = DB_SUCCESS; ut_ad(srv_operation <= SRV_OPERATION_EXPORT_RESTORED || srv_operation == SRV_OPERATION_RESTORE @@ -4525,20 +4564,12 @@ dberr_t recv_recovery_from_checkpoint_start() if (srv_force_recovery >= SRV_FORCE_NO_LOG_REDO) { sql_print_information("InnoDB: innodb_force_recovery=6" " skips redo log apply"); - return(DB_SUCCESS); + return err; } recv_sys.recovery_on = true; log_sys.latch.wr_lock(SRW_LOCK_CALL); - - dberr_t err = recv_sys.find_checkpoint(); - if (err != DB_SUCCESS) { -early_exit: - log_sys.latch.wr_unlock(); - return err; - } - log_sys.set_capacity(); /* Start reading the log from the checkpoint lsn. The variable @@ -4548,7 +4579,9 @@ early_exit: ut_ad(recv_sys.pages.empty()); if (log_sys.format == log_t::FORMAT_3_23) { - goto early_exit; +early_exit: + log_sys.latch.wr_unlock(); + return err; } if (log_sys.is_latest()) { @@ -4843,7 +4876,7 @@ byte *recv_dblwr_t::find_page(const page_id_t page_id, } bool recv_dblwr_t::restore_first_page(uint32_t space_id, const char *name, - os_file_t file) + pfs_os_file_t file) { const page_id_t page_id(space_id, 0); const byte* page= find_page(page_id); @@ -4851,10 +4884,10 @@ bool recv_dblwr_t::restore_first_page(uint32_t space_id, const char *name, { /* If the first page of the given user tablespace is not there in the doublewrite buffer, then the recovery is going to fail - now. Hence this is treated as error. */ - ib::error() - << "Corrupted page " << page_id << " of datafile '" - << name <<"' could not be found in the doublewrite buffer."; + now. Report error only when doublewrite buffer is not empty */ + if (pages.size()) + ib::error() << "Corrupted page " << page_id << " of datafile '" + << name << "' could not be found in the doublewrite buffer."; return true; } @@ -4868,3 +4901,58 @@ bool recv_dblwr_t::restore_first_page(uint32_t space_id, const char *name, IORequestWrite, name, file, page, 0, physical_size) != DB_SUCCESS; } + +uint32_t recv_dblwr_t::find_first_page(const char *name, pfs_os_file_t file) +{ + os_offset_t file_size= os_file_get_size(file); + if (file_size != (os_offset_t) -1) + { + for (const page_t *page : pages) + { + uint32_t space_id= page_get_space_id(page); + byte *read_page= nullptr; + if (page_get_page_no(page) > 0 || space_id == 0) + { +next_page: + aligned_free(read_page); + continue; + } + uint32_t flags= mach_read_from_4( + FSP_HEADER_OFFSET + FSP_SPACE_FLAGS + page); + page_id_t page_id(space_id, 0); + size_t page_size= fil_space_t::physical_size(flags); + if (file_size < 4 * page_size) + goto next_page; + read_page= + static_cast<byte*>(aligned_malloc(3 * page_size, page_size)); + /* Read 3 pages from the file and match the space id + with the space id which is stored in + doublewrite buffer page. */ + if (os_file_read(IORequestRead, file, read_page, page_size, + 3 * page_size, nullptr) != DB_SUCCESS) + goto next_page; + for (ulint j= 0; j <= 2; j++) + { + byte *cur_page= read_page + j * page_size; + if (buf_is_zeroes(span<const byte>(cur_page, page_size))) + { + space_id= 0; + goto early_exit; + } + if (mach_read_from_4(cur_page + FIL_PAGE_OFFSET) != j + 1 || + memcmp(cur_page + FIL_PAGE_SPACE_ID, + page + FIL_PAGE_SPACE_ID, 4) || + buf_page_is_corrupted(false, cur_page, flags)) + goto next_page; + } + if (!restore_first_page(space_id, name, file)) + { +early_exit: + aligned_free(read_page); + return space_id; + } + break; + } + } + return 0; +} diff --git a/storage/innobase/mtr/mtr0mtr.cc b/storage/innobase/mtr/mtr0mtr.cc index 1834a164..01641f74 100644 --- a/storage/innobase/mtr/mtr0mtr.cc +++ b/storage/innobase/mtr/mtr0mtr.cc @@ -308,6 +308,22 @@ void mtr_t::release() m_memo.clear(); } +inline lsn_t log_t::get_write_target() const +{ +#ifndef SUX_LOCK_GENERIC + ut_ad(latch.is_locked()); +#endif + if (UNIV_LIKELY(buf_free < max_buf_free)) + return 0; + ut_ad(!is_pmem()); + /* The LSN corresponding to the end of buf is + write_lsn - (first_lsn & 4095) + buf_free, + but we use simpler arithmetics to return a smaller write target in + order to minimize waiting in log_write_up_to(). */ + ut_ad(max_buf_free >= 4096 * 4); + return write_lsn + max_buf_free / 2; +} + /** Commit a mini-transaction. */ void mtr_t::commit() { @@ -331,6 +347,7 @@ void mtr_t::commit() std::pair<lsn_t,page_flush_ahead> lsns{do_write()}; process_freed_pages(); size_t modified= 0; + const lsn_t write_lsn= log_sys.get_write_target(); if (m_made_dirty) { @@ -408,7 +425,8 @@ void mtr_t::commit() break; default: buf_page_t *bpage= static_cast<buf_page_t*>(slot.object); - const auto s= bpage->unfix(); + ut_d(const auto s=) + bpage->unfix(); if (slot.type & MTR_MEMO_MODIFY) { ut_ad(slot.type == MTR_MEMO_PAGE_X_MODIFY || @@ -420,13 +438,10 @@ void mtr_t::commit() ut_ad(s < buf_page_t::READ_FIX); ut_ad(mach_read_from_8(bpage->frame + FIL_PAGE_LSN) <= m_commit_lsn); - if (s >= buf_page_t::UNFIXED) - { - mach_write_to_8(bpage->frame + FIL_PAGE_LSN, m_commit_lsn); - if (UNIV_LIKELY_NULL(bpage->zip.data)) - memcpy_aligned<8>(FIL_PAGE_LSN + bpage->zip.data, - FIL_PAGE_LSN + bpage->frame, 8); - } + mach_write_to_8(bpage->frame + FIL_PAGE_LSN, m_commit_lsn); + if (UNIV_LIKELY_NULL(bpage->zip.data)) + memcpy_aligned<8>(FIL_PAGE_LSN + bpage->zip.data, + FIL_PAGE_LSN + bpage->frame, 8); modified++; } switch (auto latch= slot.type & ~MTR_MEMO_MODIFY) { @@ -451,6 +466,9 @@ void mtr_t::commit() if (UNIV_UNLIKELY(lsns.second != PAGE_FLUSH_NO)) buf_flush_ahead(m_commit_lsn, lsns.second == PAGE_FLUSH_SYNC); + + if (UNIV_UNLIKELY(write_lsn != 0)) + log_write_up_to(write_lsn, false); } else { @@ -492,9 +510,20 @@ void mtr_t::rollback_to_savepoint(ulint begin, ulint end) m_memo.erase(m_memo.begin() + begin, m_memo.begin() + end); } +/** Set create_lsn. */ +inline void fil_space_t::set_create_lsn(lsn_t lsn) +{ +#ifndef SUX_LOCK_GENERIC + /* Concurrent log_checkpoint_low() must be impossible. */ + ut_ad(latch.is_write_locked()); +#endif + create_lsn= lsn; +} + /** Commit a mini-transaction that is shrinking a tablespace. -@param space tablespace that is being shrunk */ -void mtr_t::commit_shrink(fil_space_t &space) +@param space tablespace that is being shrunk +@param size new size in pages */ +void mtr_t::commit_shrink(fil_space_t &space, uint32_t size) { ut_ad(is_active()); ut_ad(!is_inside_ibuf()); @@ -514,6 +543,15 @@ void mtr_t::commit_shrink(fil_space_t &space) const lsn_t start_lsn= do_write().first; ut_d(m_log.erase()); + fil_node_t *file= UT_LIST_GET_LAST(space.chain); + mysql_mutex_lock(&fil_system.mutex); + ut_ad(file->is_open()); + space.size= file->size= size; + space.set_create_lsn(m_commit_lsn); + mysql_mutex_unlock(&fil_system.mutex); + + space.clear_freed_ranges(); + /* Durably write the reduced FSP_SIZE before truncating the data file. */ log_write_and_flush(); #ifndef SUX_LOCK_GENERIC @@ -521,11 +559,11 @@ void mtr_t::commit_shrink(fil_space_t &space) #endif os_file_truncate(space.chain.start->name, space.chain.start->handle, - os_offset_t{space.size} << srv_page_size_shift, true); + os_offset_t{size} << srv_page_size_shift, true); space.clear_freed_ranges(); - const page_id_t high{space.id, space.size}; + const page_id_t high{space.id, size}; size_t modified= 0; auto it= m_memo.rbegin(); mysql_mutex_lock(&buf_pool.flush_list_mutex); @@ -586,13 +624,6 @@ void mtr_t::commit_shrink(fil_space_t &space) log_sys.latch.wr_unlock(); m_latch_ex= false; - mysql_mutex_lock(&fil_system.mutex); - ut_ad(space.is_being_truncated); - ut_ad(space.is_stopping_writes()); - space.clear_stopping(); - space.is_being_truncated= false; - mysql_mutex_unlock(&fil_system.mutex); - release(); release_resources(); } @@ -680,7 +711,7 @@ The caller must hold exclusive log_sys.latch. This is to be used at log_checkpoint(). @param checkpoint_lsn the log sequence number of a checkpoint, or 0 @return current LSN */ -lsn_t mtr_t::commit_files(lsn_t checkpoint_lsn) +ATTRIBUTE_COLD lsn_t mtr_t::commit_files(lsn_t checkpoint_lsn) { #ifndef SUX_LOCK_GENERIC ut_ad(log_sys.latch.is_write_locked()); @@ -840,26 +871,26 @@ ATTRIBUTE_COLD static void log_overwrite_warning(lsn_t lsn) } /** Wait in append_prepare() for buffer to become available +@param lsn log sequence number to write up to @param ex whether log_sys.latch is exclusively locked */ -ATTRIBUTE_COLD void log_t::append_prepare_wait(bool ex) noexcept +ATTRIBUTE_COLD void log_t::append_prepare_wait(lsn_t lsn, bool ex) noexcept { - log_sys.waits++; - log_sys.unlock_lsn(); + waits++; + unlock_lsn(); if (ex) - log_sys.latch.wr_unlock(); + latch.wr_unlock(); else - log_sys.latch.rd_unlock(); + latch.rd_unlock(); - DEBUG_SYNC_C("log_buf_size_exceeded"); - log_buffer_flush_to_disk(log_sys.is_pmem()); + log_write_up_to(lsn, is_pmem()); if (ex) - log_sys.latch.wr_lock(SRW_LOCK_CALL); + latch.wr_lock(SRW_LOCK_CALL); else - log_sys.latch.rd_lock(SRW_LOCK_CALL); + latch.rd_lock(SRW_LOCK_CALL); - log_sys.lock_lsn(); + lock_lsn(); } /** Reserve space in the log buffer for appending data. @@ -878,34 +909,30 @@ std::pair<lsn_t,byte*> log_t::append_prepare(size_t size, bool ex) noexcept # endif #endif ut_ad(pmem == is_pmem()); - const lsn_t checkpoint_margin{last_checkpoint_lsn + log_capacity - size}; - const size_t avail{(pmem ? size_t(capacity()) : buf_size) - size}; lock_lsn(); write_to_buf++; - for (ut_d(int count= 50); - UNIV_UNLIKELY((pmem - ? size_t(get_lsn() - - get_flushed_lsn(std::memory_order_relaxed)) - : size_t{buf_free}) > avail); ) + const lsn_t l{lsn.load(std::memory_order_relaxed)}, end_lsn{l + size}; + size_t b{buf_free}; + + if (UNIV_UNLIKELY(pmem + ? (end_lsn - + get_flushed_lsn(std::memory_order_relaxed)) > capacity() + : b + size >= buf_size)) { - append_prepare_wait(ex); - ut_ad(count--); + append_prepare_wait(l, ex); + b= buf_free; } - const lsn_t l{lsn.load(std::memory_order_relaxed)}; - lsn.store(l + size, std::memory_order_relaxed); - const size_t b{buf_free}; - size_t new_buf_free{b}; - new_buf_free+= size; + lsn.store(end_lsn, std::memory_order_relaxed); + size_t new_buf_free= b + size; if (pmem && new_buf_free >= file_size) new_buf_free-= size_t(capacity()); buf_free= new_buf_free; unlock_lsn(); - if (UNIV_UNLIKELY(l > checkpoint_margin) || - (!pmem && b >= max_buf_free)) - set_check_flush_or_checkpoint(); + if (UNIV_UNLIKELY(end_lsn >= last_checkpoint_lsn + log_capacity)) + set_check_for_checkpoint(); return {l, &buf[b]}; } @@ -930,7 +957,7 @@ static mtr_t::page_flush_ahead log_close(lsn_t lsn) noexcept else if (UNIV_LIKELY(checkpoint_age <= log_sys.max_checkpoint_age)) return mtr_t::PAGE_FLUSH_ASYNC; - log_sys.set_check_flush_or_checkpoint(); + log_sys.set_check_for_checkpoint(); return mtr_t::PAGE_FLUSH_SYNC; } @@ -989,10 +1016,9 @@ std::pair<lsn_t,mtr_t::page_flush_ahead> mtr_t::do_write() #ifndef DBUG_OFF do { - if (m_log_mode != MTR_LOG_ALL) + if (m_log_mode != MTR_LOG_ALL || + _db_keyword_(nullptr, "skip_page_checksum", 1)) continue; - DBUG_EXECUTE_IF("skip_page_checksum", continue;); - for (const mtr_memo_slot_t& slot : m_memo) if (slot.type & MTR_MEMO_MODIFY) { @@ -1150,9 +1176,6 @@ inline void log_t::resize_write(lsn_t lsn, const byte *end, size_t len, } } -/** Write the mini-transaction log to the redo log buffer. -@param len number of bytes to write -@return {start_lsn,flush_ahead} */ std::pair<lsn_t,mtr_t::page_flush_ahead> mtr_t::finish_write(size_t len) { diff --git a/storage/innobase/os/os0file.cc b/storage/innobase/os/os0file.cc index 5e674806..31bec346 100644 --- a/storage/innobase/os/os0file.cc +++ b/storage/innobase/os/os0file.cc @@ -975,7 +975,7 @@ os_file_create_simple_func( *success = false; int create_flag; - const char* mode_str = NULL; + const char* mode_str __attribute__((unused)); ut_a(!(create_mode & OS_FILE_ON_ERROR_SILENT)); ut_a(!(create_mode & OS_FILE_ON_ERROR_NO_EXIT)); @@ -1051,6 +1051,7 @@ os_file_create_simple_func( } while (retry); +#ifdef HAVE_FCNTL_DIRECT /* This function is always called for data files, we should disable OS caching (O_DIRECT) here as we do in os_file_create_func(), so we open the same file in the same mode, see man page of open(2). */ @@ -1065,6 +1066,7 @@ os_file_create_simple_func( break; } } +#endif #ifndef _WIN32 if (!read_only @@ -1150,7 +1152,7 @@ os_file_create_func( ); int create_flag; - const char* mode_str = NULL; + const char* mode_str __attribute__((unused)); on_error_no_exit = create_mode & OS_FILE_ON_ERROR_NO_EXIT ? true : false; @@ -1192,10 +1194,13 @@ os_file_create_func( return(OS_FILE_CLOSED); } +#ifdef HAVE_FCNTL_DIRECT ut_a(type == OS_LOG_FILE || type == OS_DATA_FILE || type == OS_DATA_FILE_NO_O_DIRECT); - +#else + ut_a(type == OS_LOG_FILE || type == OS_DATA_FILE); +#endif ut_a(purpose == OS_FILE_AIO || purpose == OS_FILE_NORMAL); /* We let O_DSYNC only affect log files */ @@ -1241,7 +1246,7 @@ os_file_create_func( return file; } -#if (defined __sun__ && defined DIRECTIO_ON) || defined O_DIRECT +#ifdef HAVE_FCNTL_DIRECT if (type == OS_DATA_FILE) { switch (srv_file_flush_method) { case SRV_O_DSYNC: @@ -2175,10 +2180,8 @@ os_file_create_func( if (srv_file_flush_method == SRV_O_DSYNC) attributes|= FILE_FLAG_WRITE_THROUGH; } - else if (type == OS_DATA_FILE) - { - switch (srv_file_flush_method) - { + else if (type == OS_DATA_FILE) { + switch (srv_file_flush_method) { case SRV_FSYNC: case SRV_LITTLESYNC: case SRV_NOSYNC: @@ -3042,30 +3045,15 @@ os_file_handle_error_cond_exit( return(false); } -#ifndef _WIN32 +#ifdef HAVE_FCNTL_DIRECT /** Tries to disable OS caching on an opened file descriptor. @param[in] fd file descriptor to alter @param[in] file_name file name, used in the diagnostic message @param[in] name "open" or "create"; used in the diagnostic message */ void -os_file_set_nocache( - int fd MY_ATTRIBUTE((unused)), - const char* file_name MY_ATTRIBUTE((unused)), - const char* operation_name MY_ATTRIBUTE((unused))) +os_file_set_nocache(int fd, const char *file_name, const char *operation_name) { - /* some versions of Solaris may not have DIRECTIO_ON */ -#if defined(__sun__) && defined(DIRECTIO_ON) - if (directio(fd, DIRECTIO_ON) == -1) { - int errno_save = errno; - - ib::error() - << "Failed to set DIRECTIO_ON on file " - << file_name << "; " << operation_name << ": " - << strerror(errno_save) << "," - " continuing anyway."; - } -#elif defined(O_DIRECT) if (fcntl(fd, F_SETFL, O_DIRECT) == -1) { int errno_save = errno; static bool warning_message_printed = false; @@ -3084,10 +3072,8 @@ os_file_set_nocache( << ", continuing anyway."; } } -#endif /* defined(__sun__) && defined(DIRECTIO_ON) */ } - -#endif /* _WIN32 */ +#endif /* HAVE_FCNTL_DIRECT */ /** Check if the file system supports sparse files. @param fh file handle @@ -3177,8 +3163,18 @@ fallback: return true; } current_size &= ~4095ULL; +# ifdef __linux__ + if (!fallocate(file, 0, current_size, + size - current_size)) { + err = 0; + break; + } + + err = errno; +# else err = posix_fallocate(file, current_size, size - current_size); +# endif } } while (err == EINTR && srv_shutdown_state <= SRV_SHUTDOWN_INITIATED); @@ -3457,7 +3453,7 @@ static void write_io_callback(void *c) if (UNIV_UNLIKELY(cb->m_err != 0)) ib::info () << "IO Error: " << cb->m_err - << "during write of " + << " during write of " << cb->m_len << " bytes, for file " << request.node->name << "(" << cb->m_fh << "), returned " << cb->m_ret_len; @@ -4194,7 +4190,6 @@ bool fil_node_t::read_page0() != DB_SUCCESS) { sql_print_error("InnoDB: Unable to read first page of file %s", name); -corrupted: aligned_free(page); return false; } @@ -4211,25 +4206,35 @@ corrupted: if (!fil_space_t::is_valid_flags(flags, space->id)) { uint32_t cflags= fsp_flags_convert_from_101(flags); - if (cflags == UINT32_MAX) + if (cflags != UINT32_MAX) { -invalid: - ib::error() << "Expected tablespace flags " - << ib::hex(space->flags) - << " but found " << ib::hex(flags) - << " in the file " << name; - goto corrupted; + uint32_t cf= cflags & ~FSP_FLAGS_MEM_MASK; + uint32_t sf= space->flags & ~FSP_FLAGS_MEM_MASK; + + if (fil_space_t::is_flags_equal(cf, sf) || + fil_space_t::is_flags_equal(sf, cf)) + { + flags= cflags; + goto flags_ok; + } } - uint32_t cf= cflags & ~FSP_FLAGS_MEM_MASK; - uint32_t sf= space->flags & ~FSP_FLAGS_MEM_MASK; + aligned_free(page); + goto invalid; + } - if (!fil_space_t::is_flags_equal(cf, sf) && - !fil_space_t::is_flags_equal(sf, cf)) - goto invalid; - flags= cflags; + if (!fil_space_t::is_flags_equal((flags & ~FSP_FLAGS_MEM_MASK), + (space->flags & ~FSP_FLAGS_MEM_MASK)) && + !fil_space_t::is_flags_equal((space->flags & ~FSP_FLAGS_MEM_MASK), + (flags & ~FSP_FLAGS_MEM_MASK))) + { +invalid: + sql_print_error("InnoDB: Expected tablespace flags 0x%zx but found 0x%zx" + " in the file %s", space->flags, flags, name); + return false; } + flags_ok: ut_ad(!(flags & FSP_FLAGS_MEM_MASK)); /* Try to read crypt_data from page 0 if it is not yet read. */ diff --git a/storage/innobase/pars/pars0pars.cc b/storage/innobase/pars/pars0pars.cc index 61614007..51bcc954 100644 --- a/storage/innobase/pars/pars0pars.cc +++ b/storage/innobase/pars/pars0pars.cc @@ -1778,9 +1778,6 @@ pars_create_table( ulint flags = 0; ulint flags2 = DICT_TF2_FTS_AUX_HEX_NAME; - DBUG_EXECUTE_IF("innodb_test_wrong_fts_aux_table_name", - flags2 &= ~DICT_TF2_FTS_AUX_HEX_NAME;); - n_cols = que_node_list_get_len(column_defs); table = dict_table_t::create( diff --git a/storage/innobase/row/row0ftsort.cc b/storage/innobase/row/row0ftsort.cc index 17a2f034..9d85e2b1 100644 --- a/storage/innobase/row/row0ftsort.cc +++ b/storage/innobase/row/row0ftsort.cc @@ -1630,9 +1630,6 @@ row_fts_merge_insert( /* We should set the flags2 with aux_table_name here, in order to get the correct aux table names. */ index->table->flags2 |= DICT_TF2_FTS_AUX_HEX_NAME; - DBUG_EXECUTE_IF("innodb_test_wrong_fts_aux_table_name", - index->table->flags2 &= ~DICT_TF2_FTS_AUX_HEX_NAME - & ((1U << DICT_TF2_BITS) - 1);); fts_table.type = FTS_INDEX_TABLE; fts_table.index_id = index->id; fts_table.table_id = table->id; diff --git a/storage/innobase/row/row0import.cc b/storage/innobase/row/row0import.cc index d2609fdb..2516e24e 100644 --- a/storage/innobase/row/row0import.cc +++ b/storage/innobase/row/row0import.cc @@ -117,7 +117,6 @@ struct row_import { row_import() UNIV_NOTHROW : m_table(NULL), - m_version(0), m_hostname(NULL), m_table_name(NULL), m_autoinc(0), @@ -196,8 +195,6 @@ struct row_import { dict_table_t* m_table; /*!< Table instance */ - ulint m_version; /*!< Version of config file */ - byte* m_hostname; /*!< Hostname where the tablespace was exported */ byte* m_table_name; /*!< Exporting instance table @@ -2992,17 +2989,13 @@ row_import_read_meta_data( return(DB_IO_ERROR); } - cfg.m_version = mach_read_from_4(row); - /* Check the version number. */ - switch (cfg.m_version) { + switch (mach_read_from_4(row)) { case IB_EXPORT_CFG_VERSION_V1: - return(row_import_read_v1(file, thd, &cfg)); default: - ib_errf(thd, IB_LOG_LEVEL_ERROR, ER_IO_READ_ERROR, - "Unsupported meta-data version number (" ULINTPF "), " - "file ignored", cfg.m_version); + ib_senderrf(thd, IB_LOG_LEVEL_ERROR, ER_NOT_SUPPORTED_YET, + "meta-data version"); } return(DB_ERROR); diff --git a/storage/innobase/row/row0ins.cc b/storage/innobase/row/row0ins.cc index bdee0ed1..9c3c5d22 100644 --- a/storage/innobase/row/row0ins.cc +++ b/storage/innobase/row/row0ins.cc @@ -2638,14 +2638,17 @@ row_ins_clust_index_entry_low( ut_ad(!dict_index_is_online_ddl(index)); ut_ad(!index->table->persistent_autoinc); ut_ad(!index->is_instant()); + ut_ad(!entry->info_bits); mtr.set_log_mode(MTR_LOG_NO_REDO); } else { index->set_modified(mtr); - if (UNIV_UNLIKELY(entry->is_metadata())) { + if (UNIV_UNLIKELY(entry->info_bits != 0)) { + ut_ad(entry->is_metadata()); ut_ad(index->is_instant()); ut_ad(!dict_index_is_online_ddl(index)); ut_ad(mode == BTR_MODIFY_TREE); + ut_ad(flags == BTR_NO_LOCKING_FLAG); } else { if (mode == BTR_MODIFY_LEAF && dict_index_is_online_ddl(index)) { @@ -2787,11 +2790,6 @@ avoid_bulk: skip_bulk_insert: if (UNIV_UNLIKELY(entry->info_bits != 0)) { - ut_ad(entry->is_metadata()); - ut_ad(flags == BTR_NO_LOCKING_FLAG); - ut_ad(index->is_instant()); - ut_ad(!dict_index_is_online_ddl(index)); - const rec_t* rec = btr_pcur_get_rec(&pcur); if (rec_get_info_bits(rec, page_rec_is_comp(rec)) @@ -2895,9 +2893,20 @@ do_insert: } } + if (err == DB_SUCCESS && entry->info_bits) { + if (buf_block_t* root + = btr_root_block_get(index, RW_X_LATCH, &mtr, + &err)) { + btr_set_instant(root, *index, &mtr); + } else { + ut_ad("cannot find root page" == 0); + } + } + mtr.commit(); if (big_rec) { + ut_ad(err == DB_SUCCESS); /* Online table rebuild could read (and ignore) the incomplete record at this point. If online rebuild is in progress, the diff --git a/storage/innobase/row/row0merge.cc b/storage/innobase/row/row0merge.cc index 5df93fe6..188d8ba5 100644 --- a/storage/innobase/row/row0merge.cc +++ b/storage/innobase/row/row0merge.cc @@ -120,7 +120,7 @@ public: ut_ad(mtr_started == scan_mtr->is_active()); DBUG_EXECUTE_IF("row_merge_instrument_log_check_flush", - log_sys.set_check_flush_or_checkpoint();); + log_sys.set_check_for_checkpoint();); for (idx_tuple_vec::iterator it = m_dtuple_vec.begin(); it != m_dtuple_vec.end(); @@ -128,7 +128,7 @@ public: dtuple = *it; ut_ad(dtuple); - if (log_sys.check_flush_or_checkpoint()) { + if (log_sys.check_for_checkpoint()) { if (mtr_started) { if (!btr_pcur_move_to_prev_on_page(pcur)) { error = DB_CORRUPTION; @@ -2235,6 +2235,8 @@ end_of_index: goto err_exit; } + buf_page_make_young_if_needed(&block->page); + page_cur_set_before_first(block, cur); if (!page_cur_move_to_next(cur) || page_cur_is_after_last(cur)) { @@ -3545,17 +3547,6 @@ row_merge_sort( of file marker). Thus, it must be at least one block. */ ut_ad(file->offset > 0); - /* These thd_progress* calls will crash on sol10-64 when innodb_plugin - is used. MDEV-9356: innodb.innodb_bug53290 fails (crashes) on - sol10-64 in buildbot. - */ -#ifndef __sun__ - /* Progress report only for "normal" indexes. */ - if (dup && !(dup->index->type & DICT_FTS)) { - thd_progress_init(trx->mysql_thd, 1); - } -#endif /* __sun__ */ - if (global_system_variables.log_warnings > 2) { sql_print_information("InnoDB: Online DDL : merge-sorting" " has estimated " ULINTPF " runs", @@ -3564,15 +3555,6 @@ row_merge_sort( /* Merge the runs until we have one big run */ do { - /* Report progress of merge sort to MySQL for - show processlist progress field */ - /* Progress report only for "normal" indexes. */ -#ifndef __sun__ - if (dup && !(dup->index->type & DICT_FTS)) { - thd_progress_report(trx->mysql_thd, file->offset - num_runs, file->offset); - } -#endif /* __sun__ */ - error = row_merge(trx, dup, file, block, tmpfd, &num_runs, run_offset, stage, crypt_block, space); @@ -3596,13 +3578,6 @@ row_merge_sort( ut_free(run_offset); - /* Progress report only for "normal" indexes. */ -#ifndef __sun__ - if (dup && !(dup->index->type & DICT_FTS)) { - thd_progress_end(trx->mysql_thd); - } -#endif /* __sun__ */ - DBUG_RETURN(error); } @@ -4436,13 +4411,14 @@ row_merge_file_create( merge_file->fd = row_merge_file_create_low(path); merge_file->offset = 0; merge_file->n_rec = 0; - +#ifdef HAVE_FCNTL_DIRECT if (merge_file->fd != OS_FILE_CLOSED) { if (srv_disable_sort_file_cache) { os_file_set_nocache(merge_file->fd, "row0merge.cc", "sort"); } } +#endif return(merge_file->fd); } diff --git a/storage/innobase/row/row0purge.cc b/storage/innobase/row/row0purge.cc index 4756cc37..d83ab861 100644 --- a/storage/innobase/row/row0purge.cc +++ b/storage/innobase/row/row0purge.cc @@ -822,7 +822,6 @@ skip_secondaries: buf_page_get(page_id_t(rseg.space->id, page_no), 0, RW_X_LATCH, &mtr)) { - block->page.set_accessed(); buf_page_make_young_if_needed(&block->page); byte* data_field = block->page.frame diff --git a/storage/innobase/row/row0sel.cc b/storage/innobase/row/row0sel.cc index 6c76dd91..33f4d81f 100644 --- a/storage/innobase/row/row0sel.cc +++ b/storage/innobase/row/row0sel.cc @@ -1222,6 +1222,7 @@ re_scan: if (!cur_block) { goto func_end; } + buf_page_make_young_if_needed(&cur_block->page); } else { mtr->start(); goto func_end; diff --git a/storage/innobase/row/row0undo.cc b/storage/innobase/row/row0undo.cc index 8a1041c8..f14673c1 100644 --- a/storage/innobase/row/row0undo.cc +++ b/storage/innobase/row/row0undo.cc @@ -318,6 +318,8 @@ static buf_block_t* row_undo_rec_get(undo_node_t* node) return nullptr; } + buf_page_make_young_if_needed(&undo_page->page); + uint16_t offset = undo->top_offset; buf_block_t* prev_page = undo_page; diff --git a/storage/innobase/row/row0upd.cc b/storage/innobase/row/row0upd.cc index bec53841..a39574d2 100644 --- a/storage/innobase/row/row0upd.cc +++ b/storage/innobase/row/row0upd.cc @@ -2158,6 +2158,25 @@ row_upd_clust_rec_by_insert_inherit_func( return(inherit); } +/** Mark 'disowned' BLOBs as 'owned' and 'inherited' again, +after resuming from a lock wait. +@param entry clustered index entry */ +static ATTRIBUTE_COLD void row_upd_reown_inherited_fields(dtuple_t *entry) +{ + for (ulint i= 0; i < entry->n_fields; i++) + { + const dfield_t *dfield= dtuple_get_nth_field(entry, i); + if (dfield_is_ext(dfield)) + { + byte *blob_len= static_cast<byte*>(dfield->data) + + dfield->len - (BTR_EXTERN_FIELD_REF_SIZE - BTR_EXTERN_LEN); + ut_ad(*blob_len & BTR_EXTERN_OWNER_FLAG); + *blob_len= byte((*blob_len & ~BTR_EXTERN_OWNER_FLAG) | + BTR_EXTERN_INHERITED_FLAG); + } + } +} + /***********************************************************//** Marks the clustered index record deleted and inserts the updated version of the record to the index. This function should be used when the ordering @@ -2236,12 +2255,16 @@ row_upd_clust_rec_by_insert( /* If the clustered index record is already delete marked, then we are here after a DB_LOCK_WAIT. Skip delete marking clustered index and disowning - its blobs. */ + its blobs. Mark the BLOBs in the index entry + (which we copied from the already "disowned" rec) + as "owned", like it was on the previous call of + row_upd_clust_rec_by_insert(). */ ut_ad(row_get_rec_trx_id(rec, index, offsets) == trx->id); ut_ad(!trx_undo_roll_ptr_is_insert( row_get_rec_roll_ptr(rec, index, offsets))); + row_upd_reown_inherited_fields(entry); goto check_fk; } diff --git a/storage/innobase/srv/srv0srv.cc b/storage/innobase/srv/srv0srv.cc index bf9755fb..7c0c4b92 100644 --- a/storage/innobase/srv/srv0srv.cc +++ b/storage/innobase/srv/srv0srv.cc @@ -106,9 +106,6 @@ segment). It is quite possible that some of the tablespaces doesn't host any of the rollback-segment based on configuration used. */ uint32_t srv_undo_tablespaces_active; -/** Rate at which UNDO records should be purged. */ -ulong srv_purge_rseg_truncate_frequency; - /** Enable or Disable Truncate of UNDO tablespace. Note: If enabled then UNDO tablespace will be selected for truncate. While Server waits for undo-tablespace to truncate if user disables @@ -901,6 +898,9 @@ srv_export_innodb_status(void) export_vars.innodb_data_written = srv_stats.data_written + (dblwr << srv_page_size_shift); + export_vars.innodb_buffer_pool_read_requests + = buf_pool.stat.n_page_gets; + export_vars.innodb_buffer_pool_bytes_data = buf_pool.stat.LRU_bytes + (UT_LIST_GET_LEN(buf_pool.unzip_LRU) @@ -1503,7 +1503,8 @@ inline void purge_coordinator_state::do_purge() ulint n_pages_handled= trx_purge(n_threads, history_size); if (!trx_sys.history_exists()) goto no_history; - if (purge_sys.truncate.current || srv_shutdown_state != SRV_SHUTDOWN_NONE) + if (purge_sys.truncating_tablespace() || + srv_shutdown_state != SRV_SHUTDOWN_NONE) { purge_truncation_task.wait(); trx_purge_truncate_history(); diff --git a/storage/innobase/srv/srv0start.cc b/storage/innobase/srv/srv0start.cc index ef5bcb67..738e0a7e 100644 --- a/storage/innobase/srv/srv0start.cc +++ b/storage/innobase/srv/srv0start.cc @@ -468,7 +468,7 @@ ATTRIBUTE_COLD static dberr_t srv_undo_tablespaces_reinit() rseg->init(nullptr, FIL_NULL); } - if (trx_sys.recovered_binlog_lsn + if (*trx_sys.recovered_binlog_filename #ifdef WITH_WSREP || !trx_sys.recovered_wsrep_xid.is_null() #endif /* WITH_WSREP */ @@ -476,7 +476,7 @@ ATTRIBUTE_COLD static dberr_t srv_undo_tablespaces_reinit() { /* Update binlog offset, binlog file name & wsrep xid in system tablespace rollback segment */ - if (trx_sys.recovered_binlog_lsn) + if (*trx_sys.recovered_binlog_filename) { ut_d(const size_t len = strlen(trx_sys.recovered_binlog_filename) + 1); ut_ad(len > 1); @@ -1122,10 +1122,14 @@ dberr_t srv_start(bool create_new_db) if (srv_force_recovery) { ib::info() << "!!! innodb_force_recovery is set to " << srv_force_recovery << " !!!"; + if (srv_force_recovery == SRV_FORCE_NO_LOG_REDO) { + srv_read_only_mode = true; + } } - if (srv_force_recovery == SRV_FORCE_NO_LOG_REDO) { - srv_read_only_mode = true; + if (srv_read_only_mode) { + sql_print_information("InnoDB: Started in read only mode"); + srv_use_doublewrite_buf = false; } high_level_read_only = srv_read_only_mode @@ -1302,6 +1306,10 @@ dberr_t srv_start(bool create_new_db) ut_ad(buf_page_cleaner_is_active); } + if (innodb_encrypt_temporary_tables && !log_crypt_init()) { + return srv_init_abort(DB_ERROR); + } + /* Check if undo tablespaces and redo log files exist before creating a new system tablespace */ if (create_new_db) { @@ -1310,6 +1318,11 @@ dberr_t srv_start(bool create_new_db) return(srv_init_abort(DB_ERROR)); } recv_sys.debug_free(); + } else { + err = recv_recovery_read_checkpoint(); + if (err != DB_SUCCESS) { + return srv_init_abort(err); + } } /* Open or create the data files. */ @@ -1334,12 +1347,9 @@ dberr_t srv_start(bool create_new_db) " old data files which contain your precious data!"; /* fall through */ default: - /* Other errors might come from Datafile::validate_first_page() */ - return(srv_init_abort(err)); - } - - if (innodb_encrypt_temporary_tables && !log_crypt_init()) { - return srv_init_abort(DB_ERROR); + /* Other errors might be flagged by + Datafile::validate_first_page() */ + return srv_init_abort(err); } if (create_new_db) { @@ -1355,10 +1365,10 @@ dberr_t srv_start(bool create_new_db) return srv_init_abort(err); } - srv_undo_space_id_start= 1; + srv_undo_space_id_start = 1; } - /* Open log file and data files in the systemtablespace: we keep + /* Open data files in the system tablespace: we keep them open until database shutdown */ ut_d(fil_system.sys_space->recv_size = srv_sys_space_size_debug); @@ -1771,21 +1781,13 @@ dberr_t srv_start(bool create_new_db) } if (srv_force_recovery < SRV_FORCE_NO_UNDO_LOG_SCAN) { - /* The following call is necessary for the insert + /* The following call is necessary for the change buffer to work with multiple tablespaces. We must know the mapping between space id's and .ibd file names. - In a crash recovery, we check that the info in data - dictionary is consistent with what we already know - about space id's from the calls to fil_ibd_load(). - - In a normal startup, we create the space objects for - every table in the InnoDB data dictionary that has - an .ibd file. - We also determine the maximum tablespace id used. */ - dict_check_tablespaces_and_store_max_id(); + dict_check_tablespaces_and_store_max_id(nullptr); } if (srv_force_recovery < SRV_FORCE_NO_TRX_UNDO @@ -1933,7 +1935,7 @@ void innodb_preshutdown() better prevent any further changes from being buffered. */ innodb_change_buffering= 0; - if (trx_sys.is_initialised()) + if (srv_force_recovery < SRV_FORCE_NO_TRX_UNDO && srv_was_started) while (trx_sys.any_active_transactions()) std::this_thread::sleep_for(std::chrono::milliseconds(1)); } diff --git a/storage/innobase/sync/srw_lock.cc b/storage/innobase/sync/srw_lock.cc index e41451d8..5afb79f2 100644 --- a/storage/innobase/sync/srw_lock.cc +++ b/storage/innobase/sync/srw_lock.cc @@ -143,8 +143,7 @@ static inline void srw_pause(unsigned delay) HMT_medium(); } -#ifdef SUX_LOCK_GENERIC -# ifndef PTHREAD_ADAPTIVE_MUTEX_INITIALIZER_NP +#ifndef PTHREAD_ADAPTIVE_MUTEX_INITIALIZER_NP template<> void pthread_mutex_wrapper<true>::wr_wait() { const unsigned delay= srw_pause_delay(); @@ -158,8 +157,9 @@ template<> void pthread_mutex_wrapper<true>::wr_wait() pthread_mutex_lock(&lock); } -# endif +#endif +#ifdef SUX_LOCK_GENERIC template void ssux_lock_impl<false>::init(); template void ssux_lock_impl<true>::init(); template void ssux_lock_impl<false>::destroy(); diff --git a/storage/innobase/trx/trx0purge.cc b/storage/innobase/trx/trx0purge.cc index 1f31ceda..cff16d9c 100644 --- a/storage/innobase/trx/trx0purge.cc +++ b/storage/innobase/trx/trx0purge.cc @@ -41,6 +41,7 @@ Created 3/26/1996 Heikki Tuuri #include "dict0load.h" #include <mysql/service_thd_mdl.h> #include <mysql/service_wsrep.h> +#include "log.h" /** Maximum allowable purge history length. <=0 means 'infinite'. */ ulong srv_max_purge_lag = 0; @@ -168,10 +169,15 @@ void purge_sys_t::create() ut_ad(this == &purge_sys); ut_ad(!m_initialized); ut_ad(!enabled()); + ut_ad(!m_active); + /* If innodb_undo_tablespaces>0, the rollback segment 0 + (which always resides in the system tablespace) will + never be used; @see trx_assign_rseg_low() */ + skipped_rseg= srv_undo_tablespaces > 0; m_paused= 0; query= purge_graph_build(); next_stored= false; - rseg= NULL; + rseg= nullptr; page_no= 0; offset= 0; hdr_page_no= 0; @@ -179,8 +185,8 @@ void purge_sys_t::create() latch.SRW_LOCK_INIT(trx_purge_latch_key); end_latch.init(); mysql_mutex_init(purge_sys_pq_mutex_key, &pq_mutex, nullptr); - truncate.current= NULL; - truncate.last= NULL; + truncate_undo_space.current= nullptr; + truncate_undo_space.last= 0; m_initialized= true; } @@ -350,14 +356,21 @@ trx_purge_add_undo_to_history(const trx_t* trx, trx_undo_t*& undo, mtr_t* mtr) } /** Free an undo log segment. -@param block rollback segment header page +@param rseg_hdr rollback segment header page +@param block undo segment header page @param mtr mini-transaction */ -static void trx_purge_free_segment(buf_block_t *block, mtr_t &mtr) +static void trx_purge_free_segment(buf_block_t *rseg_hdr, buf_block_t *block, + mtr_t &mtr) { + ut_ad(mtr.memo_contains_flagged(rseg_hdr, MTR_MEMO_PAGE_X_FIX)); + ut_ad(mtr.memo_contains_flagged(block, MTR_MEMO_PAGE_X_FIX)); + while (!fseg_free_step_not_header(TRX_UNDO_SEG_HDR + TRX_UNDO_FSEG_HEADER + block->page.frame, &mtr)) { + rseg_hdr->fix(); block->fix(); + ut_d(const page_id_t rseg_hdr_id{rseg_hdr->page.id()}); ut_d(const page_id_t id{block->page.id()}); mtr.commit(); /* NOTE: If the server is killed after the log that was produced @@ -368,26 +381,62 @@ static void trx_purge_free_segment(buf_block_t *block, mtr_t &mtr) This does not matter when using multiple innodb_undo_tablespaces; innodb_undo_log_truncate=ON will be able to reclaim the space. */ mtr.start(); + rseg_hdr->page.lock.x_lock(); + ut_ad(rseg_hdr->page.id() == rseg_hdr_id); block->page.lock.x_lock(); ut_ad(block->page.id() == id); - mtr.memo_push(block, MTR_MEMO_PAGE_X_MODIFY); + mtr.memo_push(rseg_hdr, MTR_MEMO_PAGE_X_FIX); + mtr.memo_push(block, MTR_MEMO_PAGE_X_FIX); } while (!fseg_free_step(TRX_UNDO_SEG_HDR + TRX_UNDO_FSEG_HEADER + block->page.frame, &mtr)); } +void purge_sys_t::rseg_enable(trx_rseg_t &rseg) +{ + ut_ad(this == &purge_sys); +#ifndef SUX_LOCK_GENERIC + ut_ad(rseg.latch.is_write_locked()); +#endif + uint8_t skipped= skipped_rseg; + ut_ad(skipped < TRX_SYS_N_RSEGS); + if (&rseg == &trx_sys.rseg_array[skipped]) + { + /* If this rollback segment is subject to innodb_undo_log_truncate=ON, + we must not clear the flag. But we will advance purge_sys.skipped_rseg + to be able to choose another candidate for this soft truncation, and + to prevent the following scenario: + + (1) purge_sys_t::iterator::free_history_rseg() had invoked + rseg.set_skip_allocation() + (2) undo log truncation had completed on this rollback segment + (3) SET GLOBAL innodb_undo_log_truncate=OFF + (4) purge_sys_t::iterator::free_history_rseg() would not be able to + invoke rseg.set_skip_allocation() on any other rollback segment + before this rseg has grown enough */ + if (truncate_undo_space.current != rseg.space) + rseg.clear_skip_allocation(); + skipped++; + /* If innodb_undo_tablespaces>0, the rollback segment 0 + (which always resides in the system tablespace) will + never be used; @see trx_assign_rseg_low() */ + if (!(skipped&= (TRX_SYS_N_RSEGS - 1)) && srv_undo_tablespaces) + skipped++; + skipped_rseg= skipped; + } +} + /** Remove unnecessary history data from a rollback segment. @param rseg rollback segment @param limit truncate anything before this -@param all whether everything can be truncated @return error code */ -static dberr_t -trx_purge_truncate_rseg_history(trx_rseg_t &rseg, - const purge_sys_t::iterator &limit, bool all) +inline dberr_t purge_sys_t::iterator::free_history_rseg(trx_rseg_t &rseg) const { fil_addr_t hdr_addr; mtr_t mtr; + bool freed= false; + uint32_t rseg_ref= 0; mtr.start(); @@ -397,6 +446,8 @@ trx_purge_truncate_rseg_history(trx_rseg_t &rseg, { func_exit: mtr.commit(); + if (freed && (rseg.SKIP & rseg_ref)) + purge_sys.rseg_enable(rseg); return err; } @@ -418,16 +469,40 @@ loop: const trx_id_t undo_trx_no= mach_read_from_8(b->page.frame + hdr_addr.boffset + TRX_UNDO_TRX_NO); - if (undo_trx_no >= limit.trx_no) + if (undo_trx_no >= trx_no) { - if (undo_trx_no == limit.trx_no) - err = trx_undo_truncate_start(&rseg, hdr_addr.page, - hdr_addr.boffset, limit.undo_no); + if (undo_trx_no == trx_no) + err= trx_undo_truncate_start(&rseg, hdr_addr.page, + hdr_addr.boffset, undo_no); goto func_exit; } - - if (!all) - goto func_exit; + else + { + rseg_ref= rseg.ref_load(); + if (rseg_ref >= rseg.REF || !purge_sys.sees(rseg.needs_purge)) + { + /* We cannot clear this entire rseg because trx_assign_rseg_low() + has already chosen it for a future trx_undo_assign(), or + because some recently started transaction needs purging. + + If this invocation could not reduce rseg.history_size at all + (!freed), we will try to ensure progress and prevent our + starvation by disabling one rollback segment for future + trx_assign_rseg_low() invocations until a future invocation has + made progress and invoked purge_sys_t::rseg_enable(rseg) on that + rollback segment. */ + + if (!(rseg.SKIP & rseg_ref) && !freed && + ut_d(!trx_rseg_n_slots_debug &&) + &rseg == &trx_sys.rseg_array[purge_sys.skipped_rseg]) + /* If rseg.space == purge_sys.truncate_undo_space.current + the following will be a no-op. A possible conflict + with innodb_undo_log_truncate=ON will be handled in + purge_sys_t::rseg_enable(). */ + rseg.set_skip_allocation(); + goto func_exit; + } + } fil_addr_t prev_hdr_addr= flst_get_prev_addr(b->page.frame + hdr_addr.boffset + @@ -459,7 +534,7 @@ loop: free_segment: ut_ad(rseg.curr_size >= seg_size); rseg.curr_size-= seg_size; - trx_purge_free_segment(b, mtr); + trx_purge_free_segment(rseg_hdr, b, mtr); break; case TRX_UNDO_CACHED: /* rseg.undo_cached must point to this page */ @@ -490,10 +565,11 @@ loop: mtr.commit(); ut_ad(rseg.history_size > 0); rseg.history_size--; + freed= true; mtr.start(); rseg_hdr->page.lock.x_lock(); ut_ad(rseg_hdr->page.id() == rseg.page_id()); - mtr.memo_push(rseg_hdr, MTR_MEMO_PAGE_X_MODIFY); + mtr.memo_push(rseg_hdr, MTR_MEMO_PAGE_X_FIX); goto loop; } @@ -544,9 +620,7 @@ dberr_t purge_sys_t::iterator::free_history() const ut_ad(rseg.is_persistent()); log_free_check(); rseg.latch.wr_lock(SRW_LOCK_CALL); - dberr_t err= - trx_purge_truncate_rseg_history(rseg, *this, !rseg.is_referenced() && - purge_sys.sees(rseg.needs_purge)); + dberr_t err= free_history_rseg(rseg); rseg.latch.wr_unlock(); if (err) return err; @@ -554,6 +628,62 @@ dberr_t purge_sys_t::iterator::free_history() const return DB_SUCCESS; } +inline void trx_sys_t::undo_truncate_start(fil_space_t &space) +{ + ut_ad(this == &trx_sys); + /* Undo tablespace always are a single file. */ + ut_a(UT_LIST_GET_LEN(space.chain) == 1); + fil_node_t *file= UT_LIST_GET_FIRST(space.chain); + /* The undo tablespace files are never closed. */ + ut_ad(file->is_open()); + sql_print_information("InnoDB: Starting to truncate %s", file->name); + + for (auto &rseg : rseg_array) + if (rseg.space == &space) + { + /* Prevent a race with purge_sys_t::iterator::free_history_rseg() */ + rseg.latch.rd_lock(SRW_LOCK_CALL); + /* Once set, this rseg will not be allocated to subsequent + transactions, but we will wait for existing active + transactions to finish. */ + rseg.set_skip_allocation(); + rseg.latch.rd_unlock(); + } +} + +inline fil_space_t *purge_sys_t::undo_truncate_try(uint32_t id, uint32_t size) +{ + ut_ad(srv_is_undo_tablespace(id)); + fil_space_t *space= fil_space_get(id); + if (space && space->get_size() > size) + { + truncate_undo_space.current= space; + trx_sys.undo_truncate_start(*space); + return space; + } + return nullptr; +} + +fil_space_t *purge_sys_t::truncating_tablespace() +{ + ut_ad(this == &purge_sys); + + fil_space_t *space= truncate_undo_space.current; + if (space || srv_undo_tablespaces_active < 2 || !srv_undo_log_truncate) + return space; + + const uint32_t size= uint32_t(srv_max_undo_log_size >> srv_page_size_shift); + for (uint32_t i= truncate_undo_space.last, j= i;; ) + { + if (fil_space_t *s= undo_truncate_try(srv_undo_space_id_start + i, size)) + return s; + ++i; + i%= srv_undo_tablespaces_active; + if (i == j) + return nullptr; + } +} + #if defined __GNUC__ && __GNUC__ == 4 && !defined __clang__ # if defined __arm__ || defined __aarch64__ /* Work around an internal compiler error in GCC 4.8.5 */ @@ -579,55 +709,14 @@ TRANSACTIONAL_TARGET void trx_purge_truncate_history() head.undo_no= 0; } - if (head.free_history() != DB_SUCCESS || srv_undo_tablespaces_active < 2) + if (head.free_history() != DB_SUCCESS) return; - while (srv_undo_log_truncate) + while (fil_space_t *space= purge_sys.truncating_tablespace()) { - if (!purge_sys.truncate.current) - { - const ulint threshold= - ulint(srv_max_undo_log_size >> srv_page_size_shift); - for (uint32_t i= purge_sys.truncate.last - ? purge_sys.truncate.last->id - srv_undo_space_id_start : 0, - j= i;; ) - { - const uint32_t space_id= srv_undo_space_id_start + i; - ut_ad(srv_is_undo_tablespace(space_id)); - fil_space_t *space= fil_space_get(space_id); - ut_a(UT_LIST_GET_LEN(space->chain) == 1); - - if (space && space->get_size() > threshold) - { - purge_sys.truncate.current= space; - break; - } - - ++i; - i %= srv_undo_tablespaces_active; - if (i == j) - return; - } - } - - fil_space_t &space= *purge_sys.truncate.current; - /* Undo tablespace always are a single file. */ - fil_node_t *file= UT_LIST_GET_FIRST(space.chain); - /* The undo tablespace files are never closed. */ - ut_ad(file->is_open()); - - DBUG_LOG("undo", "marking for truncate: " << file->name); - - for (auto &rseg : trx_sys.rseg_array) - if (rseg.space == &space) - /* Once set, this rseg will not be allocated to subsequent - transactions, but we will wait for existing active - transactions to finish. */ - rseg.set_skip_allocation(); - for (auto &rseg : trx_sys.rseg_array) { - if (rseg.space != &space) + if (rseg.space != space) continue; rseg.latch.rd_lock(SRW_LOCK_CALL); @@ -660,15 +749,9 @@ not_free: rseg.latch.rd_unlock(); } - ib::info() << "Truncating " << file->name; - trx_purge_cleanse_purge_queue(space); - - log_free_check(); - - mtr_t mtr; - mtr.start(); - mtr.x_lock_space(&space); - const auto space_id= space.id; + const char *file_name= UT_LIST_GET_FIRST(space->chain)->name; + sql_print_information("InnoDB: Truncating %s", file_name); + trx_purge_cleanse_purge_queue(*space); /* Lock all modified pages of the tablespace. @@ -678,104 +761,41 @@ not_free: mini-transaction commit and the server was killed, then discarding the to-be-trimmed pages without flushing would break crash recovery. */ - rescan: - mysql_mutex_lock(&buf_pool.flush_list_mutex); - for (buf_page_t *bpage= UT_LIST_GET_LAST(buf_pool.flush_list); bpage; ) - { - ut_ad(bpage->oldest_modification()); - ut_ad(bpage->in_file()); - - buf_page_t *prev= UT_LIST_GET_PREV(list, bpage); - - if (bpage->oldest_modification() > 2 && bpage->id().space() == space_id) - { - ut_ad(bpage->frame); - bpage->fix(); - { - /* Try to acquire an exclusive latch while the cache line is - fresh after fix(). */ - const bool got_lock{bpage->lock.x_lock_try()}; - buf_pool.flush_hp.set(prev); - mysql_mutex_unlock(&buf_pool.flush_list_mutex); - if (!got_lock) - bpage->lock.x_lock(); - } - -#ifdef BTR_CUR_HASH_ADAPT - /* There is no AHI on undo tablespaces. */ - ut_ad(!reinterpret_cast<buf_block_t*>(bpage)->index); -#endif - ut_ad(!bpage->is_io_fixed()); - ut_ad(bpage->id().space() == space_id); - - if (bpage->oldest_modification() > 2) - { - mtr.memo_push(reinterpret_cast<buf_block_t*>(bpage), - MTR_MEMO_PAGE_X_FIX); - mysql_mutex_lock(&buf_pool.flush_list_mutex); - ut_ad(bpage->oldest_modification() > 2); - bpage->reset_oldest_modification(); - } - else - { - bpage->unfix(); - bpage->lock.x_unlock(); - mysql_mutex_lock(&buf_pool.flush_list_mutex); - } - - if (prev != buf_pool.flush_hp.get()) - { - mysql_mutex_unlock(&buf_pool.flush_list_mutex); - goto rescan; - } - } - bpage= prev; - } - - mysql_mutex_unlock(&buf_pool.flush_list_mutex); - - /* Re-initialize tablespace, in a single mini-transaction. */ - const ulint size= SRV_UNDO_TABLESPACE_SIZE_IN_PAGES; + if (UNIV_UNLIKELY(srv_shutdown_state != SRV_SHUTDOWN_NONE) && + srv_fast_shutdown) + return; /* Adjust the tablespace metadata. */ mysql_mutex_lock(&fil_system.mutex); - space.set_stopping(); - space.is_being_truncated= true; - if (space.crypt_data) + if (space->crypt_data) { - space.reacquire(); + space->reacquire(); mysql_mutex_unlock(&fil_system.mutex); - fil_space_crypt_close_tablespace(&space); - space.release(); + fil_space_crypt_close_tablespace(space); + space->release(); } else mysql_mutex_unlock(&fil_system.mutex); - for (auto i= 6000; space.referenced(); - std::this_thread::sleep_for(std::chrono::milliseconds(10))) - { - if (!--i) - { - mtr.commit(); - ib::error() << "Failed to freeze UNDO tablespace " << file->name; - return; - } - } + /* Re-initialize tablespace, in a single mini-transaction. */ + const uint32_t size= SRV_UNDO_TABLESPACE_SIZE_IN_PAGES; + + log_free_check(); + mtr_t mtr; + mtr.start(); + mtr.x_lock_space(space); /* Associate the undo tablespace with mtr. During mtr::commit_shrink(), InnoDB can use the undo tablespace object to clear all freed ranges */ - mtr.set_named_space(&space); - mtr.trim_pages(page_id_t(space.id, size)); - ut_a(fsp_header_init(&space, size, &mtr) == DB_SUCCESS); - mysql_mutex_lock(&fil_system.mutex); - space.size= file->size= size; - mysql_mutex_unlock(&fil_system.mutex); + mtr.set_named_space(space); + mtr.trim_pages(page_id_t(space->id, size)); + ut_a(fsp_header_init(space, size, &mtr) == DB_SUCCESS); for (auto &rseg : trx_sys.rseg_array) { - if (rseg.space != &space) + if (rseg.space != space) continue; ut_ad(!rseg.is_referenced()); @@ -784,7 +804,7 @@ not_free: possibly before this server had been started up. */ dberr_t err; - buf_block_t *rblock= trx_rseg_header_create(&space, + buf_block_t *rblock= trx_rseg_header_create(space, &rseg - trx_sys.rseg_array, trx_sys.get_max_trx_id(), &mtr, &err); @@ -797,7 +817,7 @@ not_free: rseg.reinit(rblock->page.id().page_no()); } - mtr.commit_shrink(space); + mtr.commit_shrink(*space, size); /* No mutex; this is only updated by the purge coordinator. */ export_vars.innodb_undo_truncations++; @@ -814,14 +834,15 @@ not_free: purge_sys.next_stored= false; } - DBUG_EXECUTE_IF("ib_undo_trunc", ib::info() << "ib_undo_trunc"; + DBUG_EXECUTE_IF("ib_undo_trunc", + sql_print_information("InnoDB: ib_undo_trunc"); log_buffer_flush_to_disk(); DBUG_SUICIDE();); - ib::info() << "Truncated " << file->name; - purge_sys.truncate.last= purge_sys.truncate.current; - ut_ad(&space == purge_sys.truncate.current); - purge_sys.truncate.current= nullptr; + sql_print_information("InnoDB: Truncated %s", file_name); + ut_ad(space == purge_sys.truncate_undo_space.current); + purge_sys.truncate_undo_space.current= nullptr; + purge_sys.truncate_undo_space.last= space->id - srv_undo_space_id_start; } } @@ -853,7 +874,9 @@ void purge_sys_t::rseg_get_next_history_log() { fil_addr_t prev_log_addr; +#ifndef SUX_LOCK_GENERIC ut_ad(rseg->latch.is_write_locked()); +#endif ut_a(rseg->last_page_no != FIL_NULL); tail.trx_no= rseg->last_trx_no() + 1; @@ -969,7 +992,9 @@ inline trx_purge_rec_t purge_sys_t::get_next_rec(roll_ptr_t roll_ptr) { ut_ad(next_stored); ut_ad(tail.trx_no < low_limit_no()); +#ifndef SUX_LOCK_GENERIC ut_ad(rseg->latch.is_write_locked()); +#endif if (!offset) { diff --git a/storage/innobase/trx/trx0rec.cc b/storage/innobase/trx/trx0rec.cc index b381c9de..2923dc64 100644 --- a/storage/innobase/trx/trx0rec.cc +++ b/storage/innobase/trx/trx0rec.cc @@ -2069,9 +2069,10 @@ trx_undo_get_undo_rec_low( mtr.start(); trx_undo_rec_t *undo_rec= nullptr; - if (const buf_block_t* undo_page= + if (buf_block_t* undo_page= buf_page_get(page_id_t(rseg->space->id, page_no), 0, RW_S_LATCH, &mtr)) { + buf_page_make_young_if_needed(&undo_page->page); undo_rec= undo_page->page.frame + offset; const size_t end= mach_read_from_2(undo_rec); if (UNIV_UNLIKELY(end <= offset || diff --git a/storage/innobase/trx/trx0rseg.cc b/storage/innobase/trx/trx0rseg.cc index 8d1a381c..87a2ac7b 100644 --- a/storage/innobase/trx/trx0rseg.cc +++ b/storage/innobase/trx/trx0rseg.cc @@ -296,8 +296,13 @@ buf_block_t *trx_rseg_t::get(mtr_t *mtr, dberr_t *err) const if (err) *err= DB_TABLESPACE_NOT_FOUND; return nullptr; } - return buf_page_get_gen(page_id(), 0, RW_X_LATCH, nullptr, - BUF_GET, mtr, err); + + buf_block_t *block= buf_page_get_gen(page_id(), 0, RW_X_LATCH, nullptr, + BUF_GET, mtr, err); + if (UNIV_LIKELY(block != nullptr)) + buf_page_make_young_if_needed(&block->page); + + return block; } /** Upgrade a rollback segment header page to MariaDB 10.3 format. @@ -462,20 +467,32 @@ static dberr_t trx_rseg_mem_restore(trx_rseg_t *rseg, mtr_t *mtr) TRX_RSEG + TRX_RSEG_BINLOG_NAME + rseg_hdr->page.frame; if (*binlog_name) { - lsn_t lsn= mach_read_from_8(my_assume_aligned<8> - (FIL_PAGE_LSN + rseg_hdr->page.frame)); static_assert(TRX_RSEG_BINLOG_NAME_LEN == sizeof trx_sys.recovered_binlog_filename, "compatibility"); - if (lsn > trx_sys.recovered_binlog_lsn) - { - trx_sys.recovered_binlog_lsn= lsn; - trx_sys.recovered_binlog_offset= + + /* Always prefer a position from rollback segment over + a legacy position from before version 10.3.5. */ + int cmp= *trx_sys.recovered_binlog_filename && + !trx_sys.recovered_binlog_is_legacy_pos + ? strncmp(reinterpret_cast<const char*>(binlog_name), + trx_sys.recovered_binlog_filename, + TRX_RSEG_BINLOG_NAME_LEN) + : 1; + + if (cmp >= 0) { + uint64_t binlog_offset = mach_read_from_8(TRX_RSEG + TRX_RSEG_BINLOG_OFFSET + rseg_hdr->page.frame); - memcpy(trx_sys.recovered_binlog_filename, binlog_name, - TRX_RSEG_BINLOG_NAME_LEN); + if (cmp) + { + memcpy(trx_sys.recovered_binlog_filename, binlog_name, + TRX_RSEG_BINLOG_NAME_LEN); + trx_sys.recovered_binlog_offset= binlog_offset; + } + else if (binlog_offset > trx_sys.recovered_binlog_offset) + trx_sys.recovered_binlog_offset= binlog_offset; + trx_sys.recovered_binlog_is_legacy_pos= false; } - #ifdef WITH_WSREP trx_rseg_read_wsrep_checkpoint(rseg_hdr, trx_sys.recovered_wsrep_xid); #endif @@ -548,6 +565,7 @@ static void trx_rseg_init_binlog_info(const page_t* page) trx_sys.recovered_binlog_offset = mach_read_from_8( TRX_SYS_MYSQL_LOG_INFO + TRX_SYS_MYSQL_LOG_OFFSET + TRX_SYS + page); + trx_sys.recovered_binlog_is_legacy_pos= true; } #ifdef WITH_WSREP @@ -562,6 +580,7 @@ dberr_t trx_rseg_array_init() *trx_sys.recovered_binlog_filename = '\0'; trx_sys.recovered_binlog_offset = 0; + trx_sys.recovered_binlog_is_legacy_pos= false; #ifdef WITH_WSREP trx_sys.recovered_wsrep_xid.null(); XID wsrep_sys_xid; diff --git a/storage/innobase/trx/trx0trx.cc b/storage/innobase/trx/trx0trx.cc index e5e2ef9e..942b8bd4 100644 --- a/storage/innobase/trx/trx0trx.cc +++ b/storage/innobase/trx/trx0trx.cc @@ -582,6 +582,7 @@ static dberr_t trx_resurrect_table_locks(trx_t *trx, const trx_undo_t &undo) undo.top_page_no), 0, RW_S_LATCH, nullptr, BUF_GET, &mtr, &err)) { + buf_page_make_young_if_needed(&block->page); buf_block_t *undo_block= block; const trx_undo_rec_t *undo_rec= block->page.frame + undo.top_offset; @@ -980,7 +981,13 @@ void trx_t::commit_empty(mtr_t *mtr) trx_undo_t *&undo= rsegs.m_redo.undo; ut_ad(undo->state == TRX_UNDO_ACTIVE || undo->state == TRX_UNDO_PREPARED); - ut_ad(undo->size == 1); + + if (UNIV_UNLIKELY(undo->size != 1)) + { + sql_print_error("InnoDB: Undo log for transaction " TRX_ID_FMT + " is corrupted (" UINT32PF "!=1)", id, undo->size); + ut_ad("corrupted undo log" == 0); + } if (buf_block_t *u= buf_page_get(page_id_t(rseg->space->id, undo->hdr_page_no), 0, @@ -1504,6 +1511,7 @@ void trx_t::commit_cleanup() mutex.wr_lock(); state= TRX_STATE_NOT_STARTED; + *detailed_error= '\0'; mod_tables.clear(); check_foreigns= true; diff --git a/storage/innobase/trx/trx0undo.cc b/storage/innobase/trx/trx0undo.cc index 203edd9f..ccc68dfe 100644 --- a/storage/innobase/trx/trx0undo.cc +++ b/storage/innobase/trx/trx0undo.cc @@ -25,8 +25,8 @@ Created 3/26/1996 Heikki Tuuri *******************************************************/ #include "trx0undo.h" +#include "buf0rea.h" #include "fsp0fsp.h" -#include "mach0data.h" #include "mtr0log.h" #include "srv0mon.h" #include "srv0srv.h" @@ -178,8 +178,12 @@ trx_undo_get_prev_rec_from_prev_page(buf_block_t *&block, uint16_t rec, block= buf_page_get(page_id_t(block->page.id().space(), prev_page_no), 0, shared ? RW_S_LATCH : RW_X_LATCH, mtr); + if (UNIV_UNLIKELY(!block)) + return nullptr; - return block ? trx_undo_page_get_last_rec(block, page_no, offset) : nullptr; + if (!buf_page_make_young_if_needed(&block->page)) + buf_read_ahead_linear(block->page.id(), 0, false); + return trx_undo_page_get_last_rec(block, page_no, offset); } /** Get the previous undo log record. @@ -268,12 +272,16 @@ trx_undo_get_first_rec(const fil_space_t &space, uint32_t page_no, uint16_t offset, ulint mode, const buf_block_t*& block, mtr_t *mtr, dberr_t *err) { - block= buf_page_get_gen(page_id_t{space.id, page_no}, 0, mode, - nullptr, BUF_GET, mtr, err); + buf_block_t *b= buf_page_get_gen(page_id_t{space.id, page_no}, 0, mode, + nullptr, BUF_GET, mtr, err); + block= b; if (!block) return nullptr; - if (trx_undo_rec_t *rec= trx_undo_page_get_first_rec(block, page_no, offset)) + if (!buf_page_make_young_if_needed(&b->page)) + buf_read_ahead_linear(b->page.id(), 0, false); + + if (trx_undo_rec_t *rec= trx_undo_page_get_first_rec(b, page_no, offset)) return rec; return trx_undo_get_next_rec_from_next_page(block, page_no, offset, mode, @@ -663,6 +671,8 @@ buf_block_t *trx_undo_add_page(trx_undo_t *undo, mtr_t *mtr, dberr_t *err) 0, RW_X_LATCH, nullptr, BUF_GET, mtr, err); if (!header_block) goto func_exit; + buf_page_make_young_if_needed(&header_block->page); + *err= fsp_reserve_free_extents(&n_reserved, rseg->space, 1, FSP_UNDO, mtr); if (UNIV_UNLIKELY(*err != DB_SUCCESS)) @@ -732,6 +742,8 @@ trx_undo_free_page( return FIL_NULL; } + buf_page_make_young_if_needed(&header_block->page); + *err = flst_remove(header_block, TRX_UNDO_SEG_HDR + TRX_UNDO_PAGE_LIST, undo_block, TRX_UNDO_PAGE_HDR + TRX_UNDO_PAGE_NODE, mtr); @@ -740,6 +752,14 @@ trx_undo_free_page( return FIL_NULL; } + const fil_addr_t last_addr = flst_get_last( + TRX_UNDO_SEG_HDR + TRX_UNDO_PAGE_LIST + + header_block->page.frame); + if (UNIV_UNLIKELY(last_addr.page == page_no)) { + *err = DB_CORRUPTION; + return FIL_NULL; + } + *err = fseg_free_page(TRX_UNDO_SEG_HDR + TRX_UNDO_FSEG_HEADER + header_block->page.frame, rseg->space, page_no, mtr); @@ -748,9 +768,6 @@ trx_undo_free_page( } buf_page_free(rseg->space, page_no, mtr); - const fil_addr_t last_addr = flst_get_last( - TRX_UNDO_SEG_HDR + TRX_UNDO_PAGE_LIST - + header_block->page.frame); rseg->curr_size--; if (!in_history) { @@ -794,6 +811,9 @@ static dberr_t trx_undo_truncate_end(trx_undo_t &undo, undo_no_t limit, { ut_ad(is_temp == !undo.rseg->is_persistent()); + if (UNIV_UNLIKELY(undo.last_page_no == FIL_NULL)) + return DB_CORRUPTION; + for (mtr_t mtr;;) { mtr.start(); @@ -887,15 +907,13 @@ trx_undo_truncate_start( trx_undo_rec_t* last_rec; mtr_t mtr; + ut_ad(rseg->is_persistent()); + if (!limit) { return DB_SUCCESS; } loop: - mtr_start(&mtr); - - if (!rseg->is_persistent()) { - mtr.set_log_mode(MTR_LOG_NO_REDO); - } + mtr.start(); dberr_t err; const buf_block_t* undo_page; @@ -1263,6 +1281,8 @@ trx_undo_reuse_cached(trx_t* trx, trx_rseg_t* rseg, trx_undo_t** pundo, return NULL; } + buf_page_make_young_if_needed(&block->page); + UT_LIST_REMOVE(rseg->undo_cached, undo); *pundo = undo; @@ -1297,19 +1317,24 @@ trx_undo_assign(trx_t* trx, dberr_t* err, mtr_t* mtr) ut_ad(mtr->get_log_mode() == MTR_LOG_ALL); trx_undo_t* undo = trx->rsegs.m_redo.undo; + buf_block_t* block; if (undo) { - return buf_page_get_gen( + block = buf_page_get_gen( page_id_t(undo->rseg->space->id, undo->last_page_no), 0, RW_X_LATCH, undo->guess_block, BUF_GET, mtr, err); + if (UNIV_LIKELY(block != nullptr)) { + buf_page_make_young_if_needed(&block->page); + } + return block; } *err = DB_SUCCESS; trx_rseg_t* rseg = trx->rsegs.m_redo.rseg; rseg->latch.wr_lock(SRW_LOCK_CALL); - buf_block_t* block = trx_undo_reuse_cached( + block = trx_undo_reuse_cached( trx, rseg, &trx->rsegs.m_redo.undo, mtr, err); if (!block) { @@ -1350,12 +1375,17 @@ trx_undo_assign_low(trx_t *trx, trx_rseg_t *rseg, trx_undo_t **undo, : &trx->rsegs.m_redo.undo)); ut_ad(mtr->get_log_mode() == (is_temp ? MTR_LOG_NO_REDO : MTR_LOG_ALL)); + buf_block_t* block; if (*undo) { - return buf_page_get_gen( + block = buf_page_get_gen( page_id_t(rseg->space->id, (*undo)->last_page_no), 0, RW_X_LATCH, (*undo)->guess_block, BUF_GET, mtr, err); + if (UNIV_LIKELY(block != nullptr)) { + buf_page_make_young_if_needed(&block->page); + } + return block; } DBUG_EXECUTE_IF( @@ -1365,7 +1395,6 @@ trx_undo_assign_low(trx_t *trx, trx_rseg_t *rseg, trx_undo_t **undo, *err = DB_SUCCESS; rseg->latch.wr_lock(SRW_LOCK_CALL); - buf_block_t* block; if (is_temp) { ut_ad(!UT_LIST_GET_LEN(rseg->undo_cached)); } else { diff --git a/storage/maria/aria_s3_copy.cc b/storage/maria/aria_s3_copy.cc index 77c41ba4..b8a0f5b7 100644 --- a/storage/maria/aria_s3_copy.cc +++ b/storage/maria/aria_s3_copy.cc @@ -87,7 +87,9 @@ static struct my_option my_long_options[] = &opt_block_size, &opt_block_size, 0, GET_ULONG, REQUIRED_ARG, 4*1024*1024, 64*1024, 16*1024*1024, MALLOC_OVERHEAD, 1024, 0 }, {"s3_protocol_version", 'L', - "Protocol used to communication with S3. One of \"Auto\", \"Amazon\" or \"Original\".", + "Protocol used to communication with S3. One of \"Auto\", \"Legacy\", " + "\"Original\", \"Amazon\", \"Path\" or \"Domain\". " + "Note: \"Legacy\", \"Original\" and \"Amazon\" are deprecated.", &opt_protocol_version, &opt_protocol_version, &s3_protocol_typelib, GET_ENUM, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, {"force", 'f', "Force copy even if target exists", @@ -195,7 +197,7 @@ static void get_options(int *argc, char ***argv) my_exit(-1); } if (opt_s3_debug) - ms3_debug(); + ms3_debug(1); } /* get_options */ @@ -218,9 +220,20 @@ int main(int argc, char** argv) ms3_set_option(global_s3_client, MS3_OPT_BUFFER_CHUNK_SIZE, &block_size); - if (opt_protocol_version) + if (opt_protocol_version > 2) { - uint8_t protocol_version= (uint8_t) opt_protocol_version; + uint8_t protocol_version; + switch (opt_protocol_version) + { + case 3: /* Legacy means v1 */ + case 4: /* Path means v1 */ + protocol_version= 1; + break; + case 5: /* Domain means v2 */ + protocol_version= 2; + break; + } + ms3_set_option(global_s3_client, MS3_OPT_FORCE_PROTOCOL_VERSION, &protocol_version); } diff --git a/storage/maria/ha_s3.cc b/storage/maria/ha_s3.cc index 8c105522..0abb3f07 100644 --- a/storage/maria/ha_s3.cc +++ b/storage/maria/ha_s3.cc @@ -121,6 +121,29 @@ static void update_secret_key(MYSQL_THD thd, } } +static void update_s3_debug(MYSQL_THD thd, + struct st_mysql_sys_var *var + __attribute__((unused)), + void *var_ptr __attribute__((unused)), + const void *save) +{ + char new_state= *(char *) save; + if (s3_debug != new_state) + { + s3_debug= new_state; + if (s3_hton) // If library is initalized + { + ms3_debug(new_state); + if (!new_state) + { + /* Ensure that all logging is written to log */ + fflush(stderr); + } + } + } +} + + /* Define system variables for S3 */ static MYSQL_SYSVAR_ULONG(block_size, s3_block_size, @@ -129,9 +152,9 @@ static MYSQL_SYSVAR_ULONG(block_size, s3_block_size, 4*1024*1024, 65536, 16*1024*1024, 8192); static MYSQL_SYSVAR_BOOL(debug, s3_debug, - PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, + PLUGIN_VAR_RQCMDARG, "Generates trace file from libmarias3 on stderr for debugging", - 0, 0, 0); + 0, update_s3_debug, 0); static MYSQL_SYSVAR_BOOL(slave_ignore_updates, s3_slave_ignore_updates, PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, @@ -147,7 +170,10 @@ static MYSQL_SYSVAR_BOOL(replicate_alter_as_create_select, static MYSQL_SYSVAR_ENUM(protocol_version, s3_protocol_version, PLUGIN_VAR_RQCMDARG, "Protocol used to communication with S3. One of " - "\"Auto\", \"Amazon\" or \"Original\".", + "\"Auto\", \"Legacy\", \"Original\", \"Amazon\", " + "\"Path\" or \"Domain\". " + "Note: \"Legacy\", \"Original\" and \"Amazon\" are " + "deprecated.", NULL, NULL, 0, &s3_protocol_typelib); static MYSQL_SYSVAR_ULONG(pagecache_age_threshold, @@ -1048,7 +1074,7 @@ static int ha_s3_init(void *p) s3_pagecache.big_block_free= s3_free; s3_init_library(); if (s3_debug) - ms3_debug(); + ms3_debug(1); struct s3_func s3f_real = { diff --git a/storage/maria/libmarias3/README.rst b/storage/maria/libmarias3/README.rst index 2d9a7f49..b318e4ae 100644 --- a/storage/maria/libmarias3/README.rst +++ b/storage/maria/libmarias3/README.rst @@ -45,6 +45,17 @@ You will need the following OS environment variables set to run the tests: | S3NOVERIFY | Set to ``1`` if the host should not use SSL verification | +------------+----------------------------------------------------------+ +If you have minion installed, you should be able to use same settings as used by +MariaDB mtr s3 tests: + +export S3KEY=minio +export S3SECRET=minioadmin +export S3REGION= +export S3BUCKET=storage-engine +export S3HOST=127.0.0.1 +export S3PORT=9000 +export S3USEHTTP=1 + The test suite is automatically built along with the library and can be executed with ``make check`` or ``make distcheck``. Before pushing, please ALWAYS ensure that ``make check`` and ``make distcheck`` works! diff --git a/storage/maria/libmarias3/docs/api/functions.rst b/storage/maria/libmarias3/docs/api/functions.rst index b30fac92..9be9718e 100644 --- a/storage/maria/libmarias3/docs/api/functions.rst +++ b/storage/maria/libmarias3/docs/api/functions.rst @@ -147,7 +147,7 @@ Example res= ms3_list(ms3, s3bucket, NULL, &list); if (res) { - printf("Error occured: %d\n", res); + printf("Error occurred: %d\n", res); return; } list_it= list; @@ -220,7 +220,7 @@ Example res= ms3_put(ms3, s3bucket, "test/ms3.txt", (const uint8_t*)test_string, strlen(test_string)); if (res) { - printf("Error occured: %d\n", res); + printf("Error occurred: %d\n", res); return; } ms3_deinit(ms3); @@ -290,7 +290,7 @@ Example res= ms3_get(ms3, s3bucket, "test/ms3.txt", &data, &length); if (res) { - printf("Error occured: %d\n", res); + printf("Error occurred: %d\n", res); return; } printf("File contents: %s\n", data); @@ -348,7 +348,7 @@ Example res = ms3_delete(ms3, s3bucket, "test/ms3.txt"); if (res) { - printf("Error occured: %d\n", res); + printf("Error occurred: %d\n", res); return; } ms3_deinit(ms3); @@ -384,7 +384,7 @@ Example res= ms3_status(ms3, s3bucket, "test/ms3.txt", &status); if (res) { - printf("Error occured: %d\n", res); + printf("Error occurred: %d\n", res); return; } printf("File length: %ld\n", status.length); diff --git a/storage/maria/libmarias3/docs/api/types.rst b/storage/maria/libmarias3/docs/api/types.rst index eba57466..4ef620b9 100644 --- a/storage/maria/libmarias3/docs/api/types.rst +++ b/storage/maria/libmarias3/docs/api/types.rst @@ -49,6 +49,8 @@ Constants * ``MS3_OPT_BUFFER_CHUNK_SIZE`` - Set the chunk size in bytes for the receive buffer. Default is 1MB. If you are receiving a large file a realloc will have to happen every time the buffer is full. For performance reasons you may want to increase the size of this buffer to reduce the reallocs and associated memory copies. The ``value`` parameter of :c:func:`ms3_set_option` should be a pointer to a :c:type:`size_t` greater than 1. * ``MS3_OPT_FORCE_LIST_VERSION`` - An internal option for the regression suite only. The ``value`` parameter of :c:func:`ms3_set_option` should be a pointer to a :c:type:`uint8_t` of value ``1`` or ``2`` * ``MS3_OPT_FORCE_PROTOCOL_VERSION`` - Set to 1 to force talking to the S3 server using version 1 of the List Bucket API, this is for S3 compatible servers. Set to 2 to force talking to the S3 server version 2 of the List Bucket API. This is for use when the autodetect bsaed on providing a base_domain does the wrong thing. The ``value`` parameter of :c:func:`ms3_set_option` should be a pointer to a :c:type:`uint8_t` of value ``1`` or ``2`` + * ``MS3_OPT_READ_CB`` - Custom read callback for :c:func:`ms3_get`. The ``value`` parameter of :c:func:`ms3_set_option` should be a :c:type:`ms3_read_callback` function. + * ``MS3_OPT_USER_DATA`` - User data for the custom read callback. The ``value`` parameter of :c:func:`ms3_set_option` is the pointer that will be passed as the ``userdata`` argument of the callback. Built-In Types ============== diff --git a/storage/maria/libmarias3/libmarias3/marias3.h b/storage/maria/libmarias3/libmarias3/marias3.h index 80b5e77c..80b1ef78 100644 --- a/storage/maria/libmarias3/libmarias3/marias3.h +++ b/storage/maria/libmarias3/libmarias3/marias3.h @@ -55,6 +55,12 @@ typedef void *(*ms3_realloc_callback)(void *ptr, size_t size); typedef char *(*ms3_strdup_callback)(const char *str); typedef void *(*ms3_calloc_callback)(size_t nmemb, size_t size); +/** The callback function for MS3_OPT_READ_CB. The function and the user data + * set with MS3_OPT_USER_DATA are passed to Curl. For more information, refer + * to CURLOPT_WRITE_FUNCTION. */ +typedef size_t (*ms3_read_callback)(void *buffer, size_t size, + size_t nitems, void *userdata); + enum ms3_error_code_t { MS3_ERR_NONE, @@ -70,6 +76,7 @@ enum ms3_error_code_t MS3_ERR_SERVER, MS3_ERR_TOO_BIG, MS3_ERR_AUTH_ROLE, + MS3_ERR_ENDPOINT, MS3_ERR_MAX // Always the last error }; @@ -82,6 +89,8 @@ enum ms3_set_option_t MS3_OPT_BUFFER_CHUNK_SIZE, MS3_OPT_FORCE_LIST_VERSION, MS3_OPT_FORCE_PROTOCOL_VERSION, + MS3_OPT_READ_CB, + MS3_OPT_USER_DATA, MS3_OPT_PORT_NUMBER }; @@ -124,7 +133,7 @@ MS3_API const char *ms3_error(uint8_t errcode); MS3_API -void ms3_debug(void); +void ms3_debug(int debug_state); MS3_API uint8_t ms3_list(ms3_st *ms3, const char *bucket, const char *prefix, diff --git a/storage/maria/libmarias3/src/assume_role.c b/storage/maria/libmarias3/src/assume_role.c index 255b1eca..4135504c 100644 --- a/storage/maria/libmarias3/src/assume_role.c +++ b/storage/maria/libmarias3/src/assume_role.c @@ -373,7 +373,7 @@ build_assume_role_request_headers(CURL *curl, struct curl_slist **head, time_t now; struct tm tmp_tm; char headerbuf[3072]; - char secrethead[45]; + char secrethead[MAX_S3_SECRET_LENGTH + S3_SECRET_EXTRA_LENGTH]; char date[9]; char sha256hash[65]; char post_hash[65]; @@ -445,7 +445,7 @@ build_assume_role_request_headers(CURL *curl, struct curl_slist **head, // User signing key hash // Date hashed using AWS4:secret_key - snprintf(secrethead, sizeof(secrethead), "AWS4%.*s", 40, secret); + snprintf(secrethead, sizeof(secrethead), "AWS4%.*s", MAX_S3_SECRET_LENGTH, secret); strftime(headerbuf, sizeof(headerbuf), "%Y%m%d", &tmp_tm); hmac_sha256((uint8_t *)secrethead, strlen(secrethead), (uint8_t *)headerbuf, strlen(headerbuf), hmac_hash); diff --git a/storage/maria/libmarias3/src/error.c b/storage/maria/libmarias3/src/error.c index edf95d05..2723c846 100644 --- a/storage/maria/libmarias3/src/error.c +++ b/storage/maria/libmarias3/src/error.c @@ -35,5 +35,7 @@ const char *errmsgs[] = "Authentication error", "File not found", "S3 server error", - "Data too big. Maximum data size is 4GB" + "Data too big. Maximum data size is 4GB", + "Error in role", + "Endpoint permanently moved" }; diff --git a/storage/maria/libmarias3/src/include.am b/storage/maria/libmarias3/src/include.am index b8c07a6f..f786df3b 100644 --- a/storage/maria/libmarias3/src/include.am +++ b/storage/maria/libmarias3/src/include.am @@ -18,7 +18,7 @@ lib_LTLIBRARIES+= src/libmarias3.la src_libmarias3_la_SOURCES= src_libmarias3_la_LIBADD= src_libmarias3_la_LDFLAGS= -src_libmarias3_la_CFLAGS= -DBUILDING_MS3 +src_libmarias3_la_CFLAGS= -DBUILDING_MS3 -fPIC src_libmarias3_la_SOURCES+= src/marias3.c src_libmarias3_la_SOURCES+= src/request.c diff --git a/storage/maria/libmarias3/src/marias3.c b/storage/maria/libmarias3/src/marias3.c index 74d7233a..b146da55 100644 --- a/storage/maria/libmarias3/src/marias3.c +++ b/storage/maria/libmarias3/src/marias3.c @@ -52,7 +52,7 @@ static void locking_function(int mode, int n, const char *file, int line) pthread_mutex_unlock(&(mutex_buf[n])); } -static int curl_needs_openssl_locking() +static int curl_needs_openssl_locking(void) { curl_version_info_data *data = curl_version_info(CURLVERSION_NOW); @@ -214,6 +214,8 @@ ms3_st *ms3_init(const char *s3key, const char *s3secret, ms3->list_container.start = NULL; ms3->list_container.pool_list = NULL; ms3->list_container.pool_free = 0; + ms3->read_cb= 0; + ms3->user_data= 0; ms3->iam_role = NULL; ms3->role_key = NULL; @@ -354,14 +356,16 @@ const char *ms3_server_error(ms3_st *ms3) return ms3->last_error; } -void ms3_debug(void) +void ms3_debug(int debug_state) { bool state = ms3debug_get(); - ms3debug_set(!state); - - if (state) + if (state != (bool) debug_state) { - ms3debug("enabling debug"); + ms3debug_set((bool) debug_state); + if (debug_state) + { + ms3debug("enabling debug"); + } } } @@ -449,15 +453,23 @@ uint8_t ms3_get(ms3_st *ms3, const char *bucket, const char *key, buf.data = NULL; buf.length = 0; - if (!ms3 || !bucket || !key || key[0] == '\0' || !data || !length) + if (!ms3 || !bucket || !key || key[0] == '\0') + { + return MS3_ERR_PARAMETER; + } + else if (!ms3->read_cb && (!data || !length)) { return MS3_ERR_PARAMETER; } res = execute_request(ms3, MS3_CMD_GET, bucket, key, NULL, NULL, NULL, NULL, 0, NULL, &buf); - *data = buf.data; - *length = buf.length; + if (!ms3->read_cb) + { + *data = buf.data; + *length = buf.length; + } + return res; } @@ -617,7 +629,7 @@ uint8_t ms3_set_option(ms3_st *ms3, ms3_set_option_t option, void *value) return MS3_ERR_PARAMETER; } - ms3->list_version = protocol_version; + ms3->protocol_version = protocol_version; break; } @@ -634,6 +646,24 @@ uint8_t ms3_set_option(ms3_st *ms3, ms3_set_option_t option, void *value) ms3->port = port_number; break; } + + case MS3_OPT_READ_CB: + { + if (!value) + { + return MS3_ERR_PARAMETER; + } + + ms3->read_cb = value; + break; + } + + case MS3_OPT_USER_DATA: + { + ms3->user_data = value; + break; + } + default: return MS3_ERR_PARAMETER; } diff --git a/storage/maria/libmarias3/src/request.c b/storage/maria/libmarias3/src/request.c index 26165474..b2924f9d 100644 --- a/storage/maria/libmarias3/src/request.c +++ b/storage/maria/libmarias3/src/request.c @@ -413,7 +413,7 @@ static uint8_t build_request_headers(CURL *curl, struct curl_slist **head, time_t now; struct tm tmp_tm; char headerbuf[3072]; - char secrethead[45]; + char secrethead[MAX_S3_SECRET_LENGTH + S3_SECRET_EXTRA_LENGTH]; char date[9]; char sha256hash[65]; char post_hash[65]; @@ -520,7 +520,7 @@ static uint8_t build_request_headers(CURL *curl, struct curl_slist **head, // User signing key hash // Date hashed using AWS4:secret_key - snprintf(secrethead, sizeof(secrethead), "AWS4%.*s", 40, secret); + snprintf(secrethead, sizeof(secrethead), "AWS4%.*s", MAX_S3_SECRET_LENGTH, secret); strftime(headerbuf, sizeof(headerbuf), "%Y%m%d", &tmp_tm); hmac_sha256((uint8_t *)secrethead, strlen(secrethead), (uint8_t *)headerbuf, strlen(headerbuf), hmac_hash); @@ -829,9 +829,19 @@ uint8_t execute_request(ms3_st *ms3, command_t cmd, const char *bucket, if (ms3->port) curl_easy_setopt(curl, CURLOPT_PORT, (long)ms3->port); + if (ms3->read_cb && cmd == MS3_CMD_GET) + { + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, ms3->read_cb); + curl_easy_setopt(curl, CURLOPT_WRITEDATA, ms3->user_data); + } + else + { + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, body_callback); + curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *)&mem); + } + + curl_easy_setopt(curl, CURLOPT_BUFFERSIZE, ms3->buffer_chunk_size); curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, header_callback); - curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, body_callback); - curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *)&mem); curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1); curl_res = curl_easy_perform(curl); @@ -848,6 +858,18 @@ uint8_t execute_request(ms3_st *ms3, command_t cmd, const char *bucket, curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &response_code); ms3debug("Response code: %ld", response_code); + if (response_code == 301) + { + char *message = parse_error_message((char *)mem.data, mem.length); + + if (message) + { + ms3debug("Response message: %s", message); + } + + set_error_nocopy(ms3, message); + res = MS3_ERR_ENDPOINT; + } if (response_code == 404) { char *message = parse_error_message((char *)mem.data, mem.length); diff --git a/storage/maria/libmarias3/src/request.h b/storage/maria/libmarias3/src/request.h index 9ce8bb5c..29ea79b3 100644 --- a/storage/maria/libmarias3/src/request.h +++ b/storage/maria/libmarias3/src/request.h @@ -26,6 +26,8 @@ // Maxmum S3 file size is 1024 bytes so for protection we make the maximum // URI length this #define MAX_URI_LENGTH 1024 +#define MAX_S3_SECRET_LENGTH 128 +#define S3_SECRET_EXTRA_LENGTH 5 #define READ_BUFFER_DEFAULT_SIZE 1024*1024 diff --git a/storage/maria/libmarias3/src/response.c b/storage/maria/libmarias3/src/response.c index 4e976aba..da5d6b3c 100644 --- a/storage/maria/libmarias3/src/response.c +++ b/storage/maria/libmarias3/src/response.c @@ -395,7 +395,7 @@ uint8_t parse_role_list_response(const char *data, size_t length, char *role_nam } } while ((member = xml_node_child(roles, ++roles_it))); - if (!strcmp(response_role_name, role_name)) + if (response_role_name && !strcmp(response_role_name, role_name)) { ms3debug("Role Found ARN = %s",response_role_arn); sprintf(arn, "%s", response_role_arn); diff --git a/storage/maria/libmarias3/src/structs.h b/storage/maria/libmarias3/src/structs.h index 34cbd817..30a7fff0 100644 --- a/storage/maria/libmarias3/src/structs.h +++ b/storage/maria/libmarias3/src/structs.h @@ -64,6 +64,8 @@ struct ms3_st bool first_run; char *path_buffer; char *query_buffer; + void *read_cb; + void *user_data; struct ms3_list_container_st list_container; }; diff --git a/storage/maria/libmarias3/src/xml.c b/storage/maria/libmarias3/src/xml.c index 2c48a4ea..25bd125f 100644 --- a/storage/maria/libmarias3/src/xml.c +++ b/storage/maria/libmarias3/src/xml.c @@ -800,7 +800,7 @@ node_creation:; return node; - /* A failure occured, so free all allocalted resources + /* A failure occurred, so free all allocalted resources */ exit_failure: if (tag_open) { diff --git a/storage/maria/libmarias3/tests/error.c b/storage/maria/libmarias3/tests/error.c index 9be45d44..1b38a81c 100644 --- a/storage/maria/libmarias3/tests/error.c +++ b/storage/maria/libmarias3/tests/error.c @@ -35,7 +35,7 @@ int main(int argc, char *argv[]) (void) argv; // Enable here so cppcheck shows coverage - ms3_debug(); + ms3_debug(1); ASSERT_NOT_NULL(ms3); errmsg = ms3_error(255); ASSERT_STREQ(errmsg, "No such error code"); diff --git a/storage/maria/libmarias3/tests/include.am b/storage/maria/libmarias3/tests/include.am index 1cb25a1c..fc4b267b 100644 --- a/storage/maria/libmarias3/tests/include.am +++ b/storage/maria/libmarias3/tests/include.am @@ -67,3 +67,7 @@ t_list_LDADD= src/libmarias3.la check_PROGRAMS+= t/list noinst_PROGRAMS+= t/list +t_read_cb_SOURCES= tests/read_cb.c +t_read_cb_LDADD= src/libmarias3.la +check_PROGRAMS+= t/read_cb +noinst_PROGRAMS+= t/read_cb diff --git a/storage/maria/libmarias3/tests/read_cb.c b/storage/maria/libmarias3/tests/read_cb.c new file mode 100644 index 00000000..57d50d59 --- /dev/null +++ b/storage/maria/libmarias3/tests/read_cb.c @@ -0,0 +1,129 @@ +/* vim:expandtab:shiftwidth=2:tabstop=2:smarttab: + * Copyright 2023 MariaDB Corporation Ab. All rights reserved. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + * MA 02110-1301 USA + */ + +#include <yatl/lite.h> +#include <libmarias3/marias3.h> + +#define NUM_BYTES 64 * 1024 + +/* Tests basic GET with a custom read callback */ +int total_reads = 0; + +static size_t read_cb(void *buf, size_t size, size_t nitems, void *userdata) +{ + char** dat = (char**)userdata; + char* ptr = *dat; + memcpy(ptr, buf, size * nitems); + ptr += size * nitems; + *dat = ptr; + total_reads++; + return nitems * size; +} + +int main(int argc, char *argv[]) +{ + int res; + int initial_reads; + uint8_t *data; + size_t length; + ms3_st *ms3; + size_t buffer_size; + char *test_string = malloc(NUM_BYTES); + char *dest = malloc(NUM_BYTES); + char* userdata = dest; + char *s3key = getenv("S3KEY"); + char *s3secret = getenv("S3SECRET"); + char *s3region = getenv("S3REGION"); + char *s3bucket = getenv("S3BUCKET"); + char *s3host = getenv("S3HOST"); + char *s3noverify = getenv("S3NOVERIFY"); + char *s3usehttp = getenv("S3USEHTTP"); + char *s3port = getenv("S3PORT"); + + SKIP_IF_(!s3key, "Environment variable S3KEY missing"); + SKIP_IF_(!s3secret, "Environment variable S3SECRET missing"); + SKIP_IF_(!s3region, "Environment variable S3REGION missing"); + SKIP_IF_(!s3bucket, "Environment variable S3BUCKET missing"); + + (void) argc; + (void) argv; + + memset(test_string, 'a', NUM_BYTES); + memset(dest, 'b', NUM_BYTES); + + ms3_library_init(); + ms3 = ms3_init(s3key, s3secret, s3region, s3host); + + if (s3noverify && !strcmp(s3noverify, "1")) + { + ms3_set_option(ms3, MS3_OPT_DISABLE_SSL_VERIFY, NULL); + } + + if (s3usehttp && !strcmp(s3usehttp, "1")) + { + ms3_set_option(ms3, MS3_OPT_USE_HTTP, NULL); + } + + if (s3port) + { + int port = atol(s3port); + ms3_set_option(ms3, MS3_OPT_PORT_NUMBER, &port); + } + + ASSERT_NOT_NULL(ms3); + + res = ms3_put(ms3, s3bucket, "test/read_cb.dat", + (const uint8_t *)test_string, + NUM_BYTES); + ASSERT_EQ_(res, 0, "Result: %u", res); + res = ms3_set_option(ms3, MS3_OPT_READ_CB, read_cb); + ASSERT_EQ_(res, 0, "Result: %u", res); + res = ms3_set_option(ms3, MS3_OPT_USER_DATA, &userdata); + ASSERT_EQ_(res, 0, "Result: %u", res); + length = 0; + data = 0; + res = ms3_get(ms3, s3bucket, "test/read_cb.dat", &data, &length); + ASSERT_EQ_(res, 0, "Result: %u", res); + ASSERT_EQ(data, 0); + ASSERT_EQ(length, 0); + ASSERT_EQ(memcmp(test_string, dest, NUM_BYTES), 0); + + /** Test that the callbacks work with a smaller chunk size */ + memset(dest, 'c', NUM_BYTES); + userdata = dest; + buffer_size = 1024; + res = ms3_set_option(ms3, MS3_OPT_BUFFER_CHUNK_SIZE, &buffer_size); + ASSERT_EQ_(res, 0, "Result: %u", res); + initial_reads = total_reads; + res = ms3_get(ms3, s3bucket, "test/read_cb.dat", &data, &length); + ASSERT_EQ_(res, 0, "Result: %u", res); + ASSERT_EQ(memcmp(test_string, dest, NUM_BYTES), 0); + ASSERT_TRUE_((total_reads - initial_reads) > initial_reads * 2, + "Expected more than %d reads but got only %d", + initial_reads * 2, total_reads - initial_reads); + + res = ms3_delete(ms3, s3bucket, "test/read_cb.dat"); + ASSERT_EQ_(res, 0, "Result: %u", res); + free(test_string); + free(dest); + ms3_free(data); + ms3_deinit(ms3); + ms3_library_deinit(); + return 0; +} diff --git a/storage/maria/s3_func.c b/storage/maria/s3_func.c index 3d18ba88..d85dc8a0 100644 --- a/storage/maria/s3_func.c +++ b/storage/maria/s3_func.c @@ -39,7 +39,7 @@ static int s3_read_file_from_disk(const char *filename, uchar **to, /* Used by ha_s3.cc and tools to define different protocol options */ -static const char *protocol_types[]= {"Auto", "Original", "Amazon", NullS}; +static const char *protocol_types[]= {"Auto", "Original", "Amazon", "Legacy", "Path", "Domain", NullS}; TYPELIB s3_protocol_typelib= {array_elements(protocol_types)-1,"", protocol_types, NULL}; @@ -154,9 +154,23 @@ ms3_st *s3_open_connection(S3_INFO *s3) errno, ms3_error(errno)); my_errno= HA_ERR_NO_SUCH_TABLE; } - if (s3->protocol_version) + if (s3->protocol_version > 2) + { + uint8_t protocol_version; + switch (s3->protocol_version) + { + case 3: /* Legacy means v1 */ + case 4: /* Path means v1 */ + protocol_version= 1; + break; + case 5: /* Domain means v2 */ + protocol_version= 2; + break; + } + ms3_set_option(s3_client, MS3_OPT_FORCE_PROTOCOL_VERSION, - &s3->protocol_version); + &protocol_version); + } if (s3->port) ms3_set_option(s3_client, MS3_OPT_PORT_NUMBER, &s3->port); diff --git a/storage/maria/unittest/ma_pagecache_consist.c b/storage/maria/unittest/ma_pagecache_consist.c index ff4a2bcb..938e5b10 100644 --- a/storage/maria/unittest/ma_pagecache_consist.c +++ b/storage/maria/unittest/ma_pagecache_consist.c @@ -403,10 +403,6 @@ int main(int argc __attribute__((unused)), exit(1); } -#ifdef HAVE_THR_SETCONCURRENCY - thr_setconcurrency(2); -#endif - if ((pagen= init_pagecache(&pagecache, PCACHE_SIZE, 0, 0, TEST_PAGE_SIZE, 0, 0)) == 0) { diff --git a/storage/maria/unittest/ma_pagecache_rwconsist.c b/storage/maria/unittest/ma_pagecache_rwconsist.c index 24c30245..e059ac45 100644 --- a/storage/maria/unittest/ma_pagecache_rwconsist.c +++ b/storage/maria/unittest/ma_pagecache_rwconsist.c @@ -272,10 +272,6 @@ int main(int argc __attribute__((unused)), exit(1); } -#ifdef HAVE_THR_SETCONCURRENCY - thr_setconcurrency(2); -#endif - if ((pagen= init_pagecache(&pagecache, PCACHE_SIZE, 0, 0, TEST_PAGE_SIZE, 0, 0)) == 0) { diff --git a/storage/maria/unittest/ma_pagecache_rwconsist2.c b/storage/maria/unittest/ma_pagecache_rwconsist2.c index c92bec3c..28d3bb8d 100644 --- a/storage/maria/unittest/ma_pagecache_rwconsist2.c +++ b/storage/maria/unittest/ma_pagecache_rwconsist2.c @@ -268,10 +268,6 @@ int main(int argc __attribute__((unused)), exit(1); } -#ifdef HAVE_THR_SETCONCURRENCY - thr_setconcurrency(2); -#endif - if ((pagen= init_pagecache(&pagecache, PCACHE_SIZE, 0, 0, TEST_PAGE_SIZE, 0, 0)) == 0) { diff --git a/storage/maria/unittest/ma_pagecache_single.c b/storage/maria/unittest/ma_pagecache_single.c index de2ecaec..9009e59c 100644 --- a/storage/maria/unittest/ma_pagecache_single.c +++ b/storage/maria/unittest/ma_pagecache_single.c @@ -795,10 +795,6 @@ int main(int argc __attribute__((unused)), exit(1); } -#ifdef HAVE_THR_SETCONCURRENCY - thr_setconcurrency(2); -#endif - if ((pagen= init_pagecache(&pagecache, PCACHE_SIZE, 0, 0, TEST_PAGE_SIZE, 0, MYF(MY_WME))) == 0) { diff --git a/storage/maria/unittest/ma_test_loghandler_multithread-t.c b/storage/maria/unittest/ma_test_loghandler_multithread-t.c index ec097ede..be6046ab 100644 --- a/storage/maria/unittest/ma_test_loghandler_multithread-t.c +++ b/storage/maria/unittest/ma_test_loghandler_multithread-t.c @@ -331,10 +331,6 @@ int main(int argc __attribute__((unused)), exit(1); } -#ifdef HAVE_THR_SETCONCURRENCY - thr_setconcurrency(2); -#endif - if (ma_control_file_open(TRUE, TRUE, TRUE)) { fprintf(stderr, "Can't init control file (%d)\n", errno); diff --git a/storage/myisam/ha_myisam.cc b/storage/myisam/ha_myisam.cc index 34809f41..14311c87 100644 --- a/storage/myisam/ha_myisam.cc +++ b/storage/myisam/ha_myisam.cc @@ -710,6 +710,16 @@ my_bool mi_killed_in_mariadb(MI_INFO *info) return (((TABLE*) (info->external_ref))->in_use->killed != 0); } +static void init_compute_vcols(void *table) +{ + /* + To evaluate vcols we must have current_thd set. + This will set current_thd in all threads to the same THD, but it's + safe, because vcols are always evaluated under info->s->intern_lock. + */ + set_current_thd(static_cast<TABLE *>(table)->in_use); +} + static int compute_vcols(MI_INFO *info, uchar *record, int keynum) { /* This mutex is needed for parallel repair */ @@ -1011,6 +1021,7 @@ void ha_myisam::setup_vcols_for_repair(HA_CHECK *param) } DBUG_ASSERT(file->s->base.reclength < file->s->vreclength || !table->s->stored_fields); + param->init_fix_record= init_compute_vcols; param->fix_record= compute_vcols; table->use_all_columns(); } diff --git a/storage/myisam/sort.c b/storage/myisam/sort.c index 375c1840..0fb475c4 100644 --- a/storage/myisam/sort.c +++ b/storage/myisam/sort.c @@ -529,6 +529,11 @@ pthread_handler_t thr_find_all_keys(void *arg) { MI_SORT_PARAM *sort_param= (MI_SORT_PARAM*) arg; my_bool error= FALSE; + + MI_SORT_INFO *si= sort_param->sort_info; + if (si->param->init_fix_record) + si->param->init_fix_record(si->info->external_ref); + /* If my_thread_init fails */ if (my_thread_init() || thr_find_all_keys_exec(sort_param)) error= TRUE; diff --git a/storage/myisammrg/ha_myisammrg.h b/storage/myisammrg/ha_myisammrg.h index 6da327ec..9964add9 100644 --- a/storage/myisammrg/ha_myisammrg.h +++ b/storage/myisammrg/ha_myisammrg.h @@ -34,7 +34,7 @@ class Mrg_child_def: public Sql_alloc { /* Remembered MERGE child def version. See top comment in ha_myisammrg.cc */ enum_table_ref_type m_child_table_ref_type; - ulong m_child_def_version; + ulonglong m_child_def_version; public: LEX_STRING db; LEX_STRING name; @@ -44,12 +44,12 @@ public: { return m_child_table_ref_type; } - inline ulong get_child_def_version() + inline ulonglong get_child_def_version() { return m_child_def_version; } inline void set_child_def_version(enum_table_ref_type child_table_ref_type, - ulong version) + ulonglong version) { m_child_table_ref_type= child_table_ref_type; m_child_def_version= version; diff --git a/storage/perfschema/CMakeLists.txt b/storage/perfschema/CMakeLists.txt index b4f5e96b..e703e43f 100644 --- a/storage/perfschema/CMakeLists.txt +++ b/storage/perfschema/CMakeLists.txt @@ -24,7 +24,7 @@ INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/include ${CMAKE_SOURCE_DIR}/sql ${CMAKE_BINARY_DIR}/sql ${CMAKE_CURRENT_BINARY_DIR} - ${PCRE_INCLUDES} + ${PCRE_INCLUDE_DIRS} ${SSL_INCLUDE_DIRS}) ADD_DEFINITIONS(-DMYSQL_SERVER) diff --git a/storage/perfschema/pfs_variable.cc b/storage/perfschema/pfs_variable.cc index 239c55b6..a33fe2ed 100644 --- a/storage/perfschema/pfs_variable.cc +++ b/storage/perfschema/pfs_variable.cc @@ -66,7 +66,7 @@ static inline SHOW_SCOPE show_scope_from_type(enum enum_mysql_show_type type) case SHOW_SIMPLE_FUNC: case SHOW_UNDEF: default: - return SHOW_SCOPE_UNDEF; + return SHOW_SCOPE_ALL; } return SHOW_SCOPE_UNDEF; } @@ -254,7 +254,8 @@ int PFS_system_variable_cache::do_materialize_all(THD *unsafe_thd) } /* Release lock taken in get_THD(). */ - mysql_mutex_unlock(&m_safe_thd->LOCK_thd_data); + if (m_safe_thd != current_thd) + mysql_mutex_unlock(&m_safe_thd->LOCK_thd_kill); m_materialized= true; ret= 0; @@ -354,7 +355,8 @@ int PFS_system_variable_cache::do_materialize_session(PFS_thread *pfs_thread) } /* Release lock taken in get_THD(). */ - mysql_mutex_unlock(&m_safe_thd->LOCK_thd_data); + if (m_safe_thd != current_thd) + mysql_mutex_unlock(&m_safe_thd->LOCK_thd_kill); m_materialized= true; ret= 0; @@ -407,7 +409,8 @@ int PFS_system_variable_cache::do_materialize_session(PFS_thread *pfs_thread, ui } /* Release lock taken in get_THD(). */ - mysql_mutex_unlock(&m_safe_thd->LOCK_thd_data); + if (m_safe_thd != current_thd) + mysql_mutex_unlock(&m_safe_thd->LOCK_thd_kill); m_materialized= true; ret= 0; @@ -458,7 +461,8 @@ int PFS_system_variable_cache::do_materialize_session(THD *unsafe_thd) } /* Release lock taken in get_THD(). */ - mysql_mutex_unlock(&m_safe_thd->LOCK_thd_data); + if (m_safe_thd != current_thd) + mysql_mutex_unlock(&m_safe_thd->LOCK_thd_kill); m_materialized= true; ret= 0; @@ -716,6 +720,7 @@ bool PFS_status_variable_cache::can_aggregate(enum_mysql_show_type variable_type case SHOW_CHAR_PTR: case SHOW_ARRAY: case SHOW_FUNC: + case SHOW_SIMPLE_FUNC: case SHOW_INT: case SHOW_LONG: case SHOW_LONGLONG: @@ -761,7 +766,7 @@ bool PFS_status_variable_cache::filter_show_var(const SHOW_VAR *show_var, bool s /** Build an array of SHOW_VARs from the global status array. Expand nested subarrays, filter unwanted variables. - NOTE: Must be done inside of LOCK_status to guard against plugin load/unload. + NOTE: Must be done under LOCK_all_status_vars */ bool PFS_status_variable_cache::init_show_var_array(enum_var_type scope, bool strict) { @@ -880,14 +885,12 @@ char * PFS_status_variable_cache::make_show_var_name(const char* prefix, const c */ bool PFS_status_variable_cache::do_initialize_session(void) { - /* Acquire LOCK_status to guard against plugin load/unload. */ - //if (m_current_thd->fill_status_recursion_level++ == 0) - mysql_mutex_lock(&LOCK_status); + /* Acquire LOCK_all_status_vars to guard against plugin load/unload. */ + mysql_rwlock_rdlock(&LOCK_all_status_vars); bool ret= init_show_var_array(OPT_SESSION, true); - //if (m_current_thd->fill_status_recursion_level-- == 1) - mysql_mutex_unlock(&LOCK_status); + mysql_rwlock_unlock(&LOCK_all_status_vars); return ret; } @@ -916,13 +919,12 @@ int PFS_status_variable_cache::do_materialize_global(void) m_materialized= false; DEBUG_SYNC(m_current_thd, "before_materialize_global_status_array"); - /* Acquire LOCK_status to guard against plugin load/unload. */ - //if (m_current_thd->fill_status_recursion_level++ == 0) - mysql_mutex_lock(&LOCK_status); + /* Acquire LOCK_all_status_vars to guard against plugin load/unload. */ + mysql_rwlock_rdlock(&LOCK_all_status_vars); /* - Build array of SHOW_VARs from global status array. Do this within - LOCK_status to ensure that the array remains unchanged during + Build array of SHOW_VARs from global status array. Do this under + LOCK_all_status_vars to ensure that the array remains unchanged during materialization. */ if (!m_external_init) @@ -945,8 +947,7 @@ int PFS_status_variable_cache::do_materialize_global(void) */ manifest(m_current_thd, m_show_var_array.front(), &status_totals, "", false, true); - //if (m_current_thd->fill_status_recursion_level-- == 1) - mysql_mutex_unlock(&LOCK_status); + mysql_rwlock_unlock(&LOCK_all_status_vars); m_materialized= true; DEBUG_SYNC(m_current_thd, "after_materialize_global_status_array"); @@ -966,13 +967,11 @@ int PFS_status_variable_cache::do_materialize_all(THD* unsafe_thd) m_materialized= false; m_cache.clear(); - /* Avoid recursive acquisition of LOCK_status. */ - //if (m_current_thd->fill_status_recursion_level++ == 0) - mysql_mutex_lock(&LOCK_status); + mysql_rwlock_rdlock(&LOCK_all_status_vars); /* - Build array of SHOW_VARs from global status array. Do this within - LOCK_status to ensure that the array remains unchanged while this + Build array of SHOW_VARs from global status array. Do this under + LOCK_all_status_vars to ensure that the array remains unchanged while this thread is materialized. */ if (!m_external_init) @@ -989,14 +988,14 @@ int PFS_status_variable_cache::do_materialize_all(THD* unsafe_thd) manifest(m_safe_thd, m_show_var_array.front(), status_vars, "", false, false); /* Release lock taken in get_THD(). */ - mysql_mutex_unlock(&m_safe_thd->LOCK_thd_data); + if (m_safe_thd != current_thd) + mysql_mutex_unlock(&m_safe_thd->LOCK_thd_kill); m_materialized= true; ret= 0; } - //if (m_current_thd->fill_status_recursion_level-- == 1) - mysql_mutex_unlock(&LOCK_status); + mysql_rwlock_unlock(&LOCK_all_status_vars); return ret; } @@ -1012,13 +1011,11 @@ int PFS_status_variable_cache::do_materialize_session(THD* unsafe_thd) m_materialized= false; m_cache.clear(); - /* Avoid recursive acquisition of LOCK_status. */ - //if (m_current_thd->fill_status_recursion_level++ == 0) - mysql_mutex_lock(&LOCK_status); + mysql_rwlock_rdlock(&LOCK_all_status_vars); /* - Build array of SHOW_VARs from global status array. Do this within - LOCK_status to ensure that the array remains unchanged while this + Build array of SHOW_VARs from global status array. Do this under + LOCK_all_status_vars to ensure that the array remains unchanged while this thread is materialized. */ if (!m_external_init) @@ -1035,14 +1032,14 @@ int PFS_status_variable_cache::do_materialize_session(THD* unsafe_thd) manifest(m_safe_thd, m_show_var_array.front(), status_vars, "", false, true); /* Release lock taken in get_THD(). */ - mysql_mutex_unlock(&m_safe_thd->LOCK_thd_data); + if (m_safe_thd != current_thd) + mysql_mutex_unlock(&m_safe_thd->LOCK_thd_kill); m_materialized= true; ret= 0; } - //if (m_current_thd->fill_status_recursion_level-- == 1) - mysql_mutex_unlock(&LOCK_status); + mysql_rwlock_unlock(&LOCK_all_status_vars); return ret; } @@ -1059,9 +1056,8 @@ int PFS_status_variable_cache::do_materialize_session(PFS_thread *pfs_thread) m_materialized= false; m_cache.clear(); - /* Acquire LOCK_status to guard against plugin load/unload. */ - //if (m_current_thd->fill_status_recursion_level++ == 0) - mysql_mutex_lock(&LOCK_status); + /* Acquire LOCK_all_status_vars to guard against plugin load/unload. */ + mysql_rwlock_rdlock(&LOCK_all_status_vars); /* The SHOW_VAR array must be initialized externally. */ assert(m_initialized); @@ -1077,14 +1073,14 @@ int PFS_status_variable_cache::do_materialize_session(PFS_thread *pfs_thread) manifest(m_safe_thd, m_show_var_array.front(), status_vars, "", false, true); /* Release lock taken in get_THD(). */ - mysql_mutex_unlock(&m_safe_thd->LOCK_thd_data); + if (m_safe_thd != current_thd) + mysql_mutex_unlock(&m_safe_thd->LOCK_thd_kill); m_materialized= true; ret= 0; } - //if (m_current_thd->fill_status_recursion_level-- == 1) - mysql_mutex_unlock(&LOCK_status); + mysql_rwlock_unlock(&LOCK_all_status_vars); return ret; } @@ -1103,9 +1099,8 @@ int PFS_status_variable_cache::do_materialize_client(PFS_client *pfs_client) m_materialized= false; m_cache.clear(); - /* Acquire LOCK_status to guard against plugin load/unload. */ - //if (m_current_thd->fill_status_recursion_level++ == 0) - mysql_mutex_lock(&LOCK_status); + /* Acquire LOCK_all_status_vars to guard against plugin load/unload. */ + mysql_rwlock_rdlock(&LOCK_all_status_vars); /* The SHOW_VAR array must be initialized externally. */ assert(m_initialized); @@ -1122,8 +1117,7 @@ int PFS_status_variable_cache::do_materialize_client(PFS_client *pfs_client) */ manifest(m_current_thd, m_show_var_array.front(), &status_totals, "", false, true); - //if (m_current_thd->fill_status_recursion_level-- == 1) - mysql_mutex_unlock(&LOCK_status); + mysql_rwlock_unlock(&LOCK_all_status_vars); m_materialized= true; return 0; @@ -1152,16 +1146,19 @@ void PFS_status_variable_cache::manifest(THD *thd, const SHOW_VAR *show_var_arra reevaluate the new SHOW_TYPE and value. Handle nested case where SHOW_FUNC resolves to another SHOW_FUNC. */ - if (show_var_ptr->type == SHOW_FUNC) + if (show_var_ptr->type == SHOW_FUNC || show_var_ptr->type == SHOW_SIMPLE_FUNC) { show_var_tmp= *show_var_ptr; /* Execute the function reference in show_var_tmp->value, which returns show_var_tmp with a new type and new value. */ - for (const SHOW_VAR *var= show_var_ptr; var->type == SHOW_FUNC; var= &show_var_tmp) + for (const SHOW_VAR *var= show_var_ptr; + var->type == SHOW_FUNC || var->type == SHOW_SIMPLE_FUNC; + var= &show_var_tmp) { - ((mysql_show_var_func)(var->value))(thd, &show_var_tmp, value_buf.data, NULL, m_query_scope); + ((mysql_show_var_func)(var->value))(thd, &show_var_tmp, value_buf.data, + &thd->status_var, m_query_scope); } show_var_ptr= &show_var_tmp; } @@ -1213,7 +1210,7 @@ Status_variable::Status_variable(const SHOW_VAR *show_var, STATUS_VAR *status_va /** Resolve status value, convert to string. show_var->value is an offset into status_vars. - NOTE: Assumes LOCK_status is held. + NOTE: Assumes LOCK_all_status_vars is held. */ void Status_variable::init(const SHOW_VAR *show_var, STATUS_VAR *status_vars, enum_var_type query_scope) { @@ -1279,7 +1276,7 @@ void sum_account_status(PFS_client *pfs_account, STATUS_VAR *status_totals) /** Reset aggregated status counter stats for account, user and host. - NOTE: Assumes LOCK_status is held. + NOTE: Assumes LOCK_all_status_vars is held. */ void reset_pfs_status_stats() { diff --git a/storage/perfschema/pfs_variable.h b/storage/perfschema/pfs_variable.h index d3ad4c7f..7dc24826 100644 --- a/storage/perfschema/pfs_variable.h +++ b/storage/perfschema/pfs_variable.h @@ -211,8 +211,12 @@ public: if (thd != m_unsafe_thd) return false; - /* Hold this lock to keep THD during materialization. */ - mysql_mutex_lock(&thd->LOCK_thd_data); + /* + Hold this lock to keep THD during materialization. + But don't lock current_thd (to be able to use set_killed() later + */ + if (thd != current_thd) + mysql_mutex_lock(&thd->LOCK_thd_kill); return true; } void set_unsafe_thd(THD *unsafe_thd) { m_unsafe_thd= unsafe_thd; } diff --git a/storage/perfschema/pfs_visitor.cc b/storage/perfschema/pfs_visitor.cc index 92a5c99e..7e3027ac 100644 --- a/storage/perfschema/pfs_visitor.cc +++ b/storage/perfschema/pfs_visitor.cc @@ -1356,8 +1356,7 @@ PFS_connection_status_visitor::~PFS_connection_status_visitor() = default; /** Aggregate from global status. */ void PFS_connection_status_visitor::visit_global() { - /* NOTE: Requires lock on LOCK_status. */ - mysql_mutex_assert_owner(&LOCK_status); + /* NOTE: Requires lock on LOCK_all_status_vars. */ add_to_status(m_status_vars, &global_status_var); } diff --git a/storage/perfschema/table_helper.h b/storage/perfschema/table_helper.h index ddea4c08..87572ef2 100644 --- a/storage/perfschema/table_helper.h +++ b/storage/perfschema/table_helper.h @@ -659,7 +659,7 @@ public: private: void make_row(const CHARSET_INFO *cs, const char* str, size_t length); - char m_str[1024]; + char m_str[2048]; uint m_length; const CHARSET_INFO *m_charset; }; diff --git a/storage/perfschema/unittest/CMakeLists.txt b/storage/perfschema/unittest/CMakeLists.txt index 2a22990f..600795c7 100644 --- a/storage/perfschema/unittest/CMakeLists.txt +++ b/storage/perfschema/unittest/CMakeLists.txt @@ -22,7 +22,7 @@ INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/include ${CMAKE_SOURCE_DIR}/include/mysql - ${PCRE_INCLUDES} + ${PCRE_INCLUDE_DIRS} ${CMAKE_SOURCE_DIR}/sql ${SSL_INCLUDE_DIRS} ${CMAKE_SOURCE_DIR}/unittest/mytap diff --git a/storage/perfschema/unittest/stub_pfs_global.h b/storage/perfschema/unittest/stub_pfs_global.h index 4b792f9b..e6876dcc 100644 --- a/storage/perfschema/unittest/stub_pfs_global.h +++ b/storage/perfschema/unittest/stub_pfs_global.h @@ -58,7 +58,7 @@ void *pfs_malloc(PFS_builtin_memory_class *klass, size_t size, myf) void pfs_free(PFS_builtin_memory_class *, size_t, void *ptr) { if (ptr != NULL) - free(ptr); + aligned_free(ptr); } void *pfs_malloc_array(PFS_builtin_memory_class *klass, size_t n, size_t size, myf flags) diff --git a/storage/sphinx/ha_sphinx.cc b/storage/sphinx/ha_sphinx.cc index 3c5b23f5..103ce50d 100644 --- a/storage/sphinx/ha_sphinx.cc +++ b/storage/sphinx/ha_sphinx.cc @@ -3541,7 +3541,8 @@ CSphSEStats * sphinx_get_stats ( THD * thd, SHOW_VAR * out ) return 0; } -int sphinx_showfunc_total ( THD * thd, SHOW_VAR * out, char * ) +static int sphinx_showfunc_total ( THD * thd, SHOW_VAR * out, void *, + system_status_var *, enum_var_type ) { CSphSEStats * pStats = sphinx_get_stats ( thd, out ); if ( pStats ) @@ -3552,7 +3553,8 @@ int sphinx_showfunc_total ( THD * thd, SHOW_VAR * out, char * ) return 0; } -int sphinx_showfunc_total_found ( THD * thd, SHOW_VAR * out, char * ) +static int sphinx_showfunc_total_found ( THD * thd, SHOW_VAR * out, void *, + system_status_var *, enum_var_type ) { CSphSEStats * pStats = sphinx_get_stats ( thd, out ); if ( pStats ) @@ -3563,7 +3565,8 @@ int sphinx_showfunc_total_found ( THD * thd, SHOW_VAR * out, char * ) return 0; } -int sphinx_showfunc_time ( THD * thd, SHOW_VAR * out, char * ) +static int sphinx_showfunc_time ( THD * thd, SHOW_VAR * out, void *, + system_status_var *, enum_var_type ) { CSphSEStats * pStats = sphinx_get_stats ( thd, out ); if ( pStats ) @@ -3574,7 +3577,8 @@ int sphinx_showfunc_time ( THD * thd, SHOW_VAR * out, char * ) return 0; } -int sphinx_showfunc_word_count ( THD * thd, SHOW_VAR * out, char * ) +static int sphinx_showfunc_word_count ( THD * thd, SHOW_VAR * out, void *, + system_status_var *, enum_var_type ) { CSphSEStats * pStats = sphinx_get_stats ( thd, out ); if ( pStats ) @@ -3585,9 +3589,11 @@ int sphinx_showfunc_word_count ( THD * thd, SHOW_VAR * out, char * ) return 0; } -int sphinx_showfunc_words ( THD * thd, SHOW_VAR * out, char * sBuffer ) +static int sphinx_showfunc_words ( THD * thd, SHOW_VAR * out, void * buf, + system_status_var *, enum_var_type ) { #if MYSQL_VERSION_ID>50100 + char *sBuffer = static_cast<char*>(buf); if ( sphinx_hton_ptr ) { CSphTLS * pTls = (CSphTLS *) thd_get_ha_data ( thd, sphinx_hton_ptr ); @@ -3642,7 +3648,8 @@ int sphinx_showfunc_words ( THD * thd, SHOW_VAR * out, char * sBuffer ) return 0; } -int sphinx_showfunc_error ( THD * thd, SHOW_VAR * out, char * ) +static int sphinx_showfunc_error ( THD * thd, SHOW_VAR * out, void *, + system_status_var *, enum_var_type ) { CSphSEStats * pStats = sphinx_get_stats ( thd, out ); out->type = SHOW_CHAR; diff --git a/storage/sphinx/ha_sphinx.h b/storage/sphinx/ha_sphinx.h index f03e9d8c..5a1541f1 100644 --- a/storage/sphinx/ha_sphinx.h +++ b/storage/sphinx/ha_sphinx.h @@ -164,12 +164,6 @@ private: bool sphinx_show_status ( THD * thd ); #endif -int sphinx_showfunc_total_found ( THD *, SHOW_VAR *, char * ); -int sphinx_showfunc_total ( THD *, SHOW_VAR *, char * ); -int sphinx_showfunc_time ( THD *, SHOW_VAR *, char * ); -int sphinx_showfunc_word_count ( THD *, SHOW_VAR *, char * ); -int sphinx_showfunc_words ( THD *, SHOW_VAR *, char * ); - // // $Id: ha_sphinx.h 4818 2014-09-24 08:53:38Z tomat $ // diff --git a/storage/spider/ha_spider.cc b/storage/spider/ha_spider.cc index 312f94a8..5b85b3a2 100644 --- a/storage/spider/ha_spider.cc +++ b/storage/spider/ha_spider.cc @@ -58,7 +58,7 @@ ha_spider::ha_spider( { DBUG_ENTER("ha_spider::ha_spider"); DBUG_PRINT("info",("spider this=%p", this)); - spider_alloc_calc_mem_init(mem_calc, 139); + spider_alloc_calc_mem_init(mem_calc, SPD_MID_HA_SPIDER_HA_SPIDER_1); spider_alloc_calc_mem(spider_current_trx, mem_calc, sizeof(*this)); share = NULL; conns = NULL; @@ -118,7 +118,7 @@ ha_spider::ha_spider( { DBUG_ENTER("ha_spider::ha_spider"); DBUG_PRINT("info",("spider this=%p", this)); - spider_alloc_calc_mem_init(mem_calc, 0); + spider_alloc_calc_mem_init(mem_calc, SPD_MID_HA_SPIDER_HA_SPIDER_2); spider_alloc_calc_mem(spider_current_trx, mem_calc, sizeof(*this)); share = NULL; conns = NULL; @@ -358,10 +358,10 @@ int ha_spider::open( } for (roop_count = 0; roop_count < (int) share->link_count; roop_count++) { - result_list.sqls[roop_count].init_calc_mem(80); - result_list.insert_sqls[roop_count].init_calc_mem(81); - result_list.update_sqls[roop_count].init_calc_mem(82); - result_list.tmp_sqls[roop_count].init_calc_mem(83); + result_list.sqls[roop_count].init_calc_mem(SPD_MID_HA_SPIDER_OPEN_3); + result_list.insert_sqls[roop_count].init_calc_mem(SPD_MID_HA_SPIDER_OPEN_4); + result_list.update_sqls[roop_count].init_calc_mem(SPD_MID_HA_SPIDER_OPEN_5); + result_list.tmp_sqls[roop_count].init_calc_mem(SPD_MID_HA_SPIDER_OPEN_6); uint all_link_idx = conn_link_idx[roop_count]; uint dbton_id = share->sql_dbton_ids[all_link_idx]; if (share->dbton_share[dbton_id]->need_change_db_table_name()) @@ -392,7 +392,7 @@ int ha_spider::open( } for (roop_count = 0; roop_count < (int) table_share->fields; roop_count++) { - blob_buff[roop_count].init_calc_mem(84); + blob_buff[roop_count].init_calc_mem(SPD_MID_HA_SPIDER_OPEN_7); blob_buff[roop_count].set_charset(table->field[roop_count]->charset()); } } @@ -860,6 +860,14 @@ int ha_spider::external_lock( wide_handler->trx= trx; /* End of wide_handler setup */ + if (lock_type == F_UNLCK) + { + if (!trx->locked_connections) + { + DBUG_RETURN(0); /* No remote table actually locked by Spider */ + } + } + if (store_error_num) { DBUG_RETURN(store_error_num); @@ -888,10 +896,7 @@ int ha_spider::external_lock( if (lock_type == F_UNLCK) { - if (sql_command != SQLCOM_UNLOCK_TABLES) - { - DBUG_RETURN(0); /* Unlock remote tables only by UNLOCK TABLES. */ - } + wide_handler->sql_command = SQLCOM_UNLOCK_TABLES; if (!trx->locked_connections) { DBUG_RETURN(0); /* No remote table actually locked by Spider */ @@ -3592,7 +3597,7 @@ int ha_spider::multi_range_read_next_first( spider_free(spider_current_trx, multi_range_keys, MYF(0)); } if (!(multi_range_keys = (range_id_t *) - spider_malloc(spider_current_trx, 1, sizeof(range_id_t) * + spider_malloc(spider_current_trx, SPD_MID_HA_SPIDER_MULTI_RANGE_READ_NEXT_FIRST_1, sizeof(range_id_t) * (multi_range_num < result_list.multi_split_read ? multi_range_num : result_list.multi_split_read), MYF(MY_WME))) ) @@ -3605,7 +3610,7 @@ int ha_spider::multi_range_read_next_first( DBUG_RETURN(HA_ERR_OUT_OF_MEM); } for (roop_count = 0; roop_count < 2; roop_count++) - mrr_key_buff[roop_count].init_calc_mem(235); + mrr_key_buff[roop_count].init_calc_mem(SPD_MID_HA_SPIDER_MULTI_RANGE_READ_NEXT_FIRST_3); } error_num = 0; if ((range_res = mrr_funcs.next(mrr_iter, &mrr_cur_range))) @@ -6068,7 +6073,7 @@ FT_INFO *ha_spider::ft_init_ext( if (!ft_current) { if (!(ft_current = (st_spider_ft_info *) - spider_malloc(spider_current_trx, 2, sizeof(st_spider_ft_info), + spider_malloc(spider_current_trx, SPD_MID_HA_SPIDER_FT_INIT_EXT_1, sizeof(st_spider_ft_info), MYF(MY_WME | MY_ZEROFILL)))) { my_error(HA_ERR_OUT_OF_MEM, MYF(0)); @@ -6234,7 +6239,7 @@ int ha_spider::ft_read_internal( DBUG_RETURN(check_error_mode_eof(error_num)); } } else { - uint dbton_id = share->use_sql_dbton_ids[roop_count]; + uint dbton_id = share->sql_dbton_ids[roop_count]; spider_db_handler *dbton_hdl = dbton_handler[dbton_id]; SPIDER_CONN *conn = conns[roop_count]; pthread_mutex_assert_not_owner(&conn->mta_conn_mutex); @@ -8484,7 +8489,7 @@ int ha_spider::create( if (form->s->keys > 0) { if (!(tmp_share.static_key_cardinality = (longlong *) - spider_bulk_malloc(spider_current_trx, 246, MYF(MY_WME), + spider_bulk_malloc(spider_current_trx, SPD_MID_HA_SPIDER_CREATE_1, MYF(MY_WME), &tmp_share.static_key_cardinality, (uint) (sizeof(*tmp_share.static_key_cardinality) * form->s->keys), NullS)) @@ -8499,7 +8504,7 @@ int ha_spider::create( } } for (roop_count = 0; roop_count < form->s->keys; roop_count++) - tmp_share.key_hint[roop_count].init_calc_mem(85); + tmp_share.key_hint[roop_count].init_calc_mem(SPD_MID_HA_SPIDER_CREATE_2); DBUG_PRINT("info",("spider tmp_share.key_hint=%p", tmp_share.key_hint)); if ((error_num = spider_parse_connect_info(&tmp_share, form->s, form->part_info, @@ -8521,7 +8526,10 @@ int ha_spider::create( if ( thd->lex->create_info.or_replace() && (error_num = spider_delete_tables( - table_tables, tmp_share.table_name, &dummy)) + table_tables, tmp_share.table_name, &dummy)) && + /* In this context, no key found in mysql.spider_tables means + the Spider table does not exist */ + error_num != HA_ERR_KEY_NOT_FOUND ) { goto error; } @@ -8948,12 +8956,6 @@ int ha_spider::delete_table( ) need_lock = TRUE; - if ((error_num = spider_sys_delete_table_sts( - current_thd, name, name_len, need_lock))) - goto error; - if ((error_num = spider_sys_delete_table_crd( - current_thd, name, name_len, need_lock))) - goto error; if ( !(table_tables = spider_open_sys_table( current_thd, SPIDER_SYS_TABLES_TABLE_NAME_STR, @@ -8966,6 +8968,10 @@ int ha_spider::delete_table( (error_num = spider_delete_tables( table_tables, name, &old_link_count)) ) { + /* In this context, no key found in mysql.spider_tables means + the Spider table does not exist */ + if (error_num == HA_ERR_KEY_NOT_FOUND) + error_num= HA_ERR_NO_SUCH_TABLE; goto error; } spider_close_sys_table(current_thd, table_tables, @@ -9157,7 +9163,7 @@ const COND *ha_spider::cond_push( { SPIDER_CONDITION *tmp_cond; if (!(tmp_cond = (SPIDER_CONDITION *) - spider_malloc(spider_current_trx, 3, sizeof(*tmp_cond), MYF(MY_WME))) + spider_malloc(spider_current_trx, SPD_MID_HA_SPIDER_COND_PUSH_1, sizeof(*tmp_cond), MYF(MY_WME))) ) DBUG_RETURN(cond); tmp_cond->cond = (COND *) cond; @@ -9786,7 +9792,7 @@ int ha_spider::drop_tmp_tables() ) { if (spider_bit_is_set(result_list.tmp_table_created, roop_count)) { - uint dbton_id = share->use_sql_dbton_ids[roop_count]; + uint dbton_id = share->sql_dbton_ids[roop_count]; spider_db_handler *dbton_hdl = dbton_handler[dbton_id]; SPIDER_CONN *conn = conns[roop_count]; pthread_mutex_assert_not_owner(&conn->mta_conn_mutex); diff --git a/storage/spider/mysql-test/spider/bugfix/disabled.def b/storage/spider/mysql-test/spider/bugfix/disabled.def index 3fd0a2d4..530b2825 100644 --- a/storage/spider/mysql-test/spider/bugfix/disabled.def +++ b/storage/spider/mysql-test/spider/bugfix/disabled.def @@ -1,3 +1,4 @@ wait_timeout : MDEV-26045 -mdev_29904 : MDEV-31101 mdev_27239: MDEV-32046 +mdev_27575 : MDEV-32997 +mdev_28739_simple : MDEV-33343 diff --git a/storage/spider/mysql-test/spider/bugfix/include/spider_table_sts_deinit.inc b/storage/spider/mysql-test/spider/bugfix/include/spider_table_sts_deinit.inc deleted file mode 100644 index 01645e85..00000000 --- a/storage/spider/mysql-test/spider/bugfix/include/spider_table_sts_deinit.inc +++ /dev/null @@ -1,12 +0,0 @@ ---connection master_1 -alter table mysql.spider_table_sts add column checksum bigint unsigned default null after update_time; -DROP DATABASE IF EXISTS auto_test_local; - ---let $MASTER_1_COMMENT_2_1= $MASTER_1_COMMENT_2_1_BACKUP ---disable_warnings ---disable_query_log ---disable_result_log ---source ../t/test_deinit.inc ---enable_result_log ---enable_query_log ---enable_warnings diff --git a/storage/spider/mysql-test/spider/bugfix/include/spider_table_sts_init.inc b/storage/spider/mysql-test/spider/bugfix/include/spider_table_sts_init.inc deleted file mode 100644 index 1e438812..00000000 --- a/storage/spider/mysql-test/spider/bugfix/include/spider_table_sts_init.inc +++ /dev/null @@ -1,13 +0,0 @@ ---disable_warnings ---disable_query_log ---disable_result_log ---source ../t/test_init.inc ---enable_result_log ---enable_query_log ---enable_warnings ---let $MASTER_1_COMMENT_2_1_BACKUP= $MASTER_1_COMMENT_2_1 -let $MASTER_1_COMMENT_2_1= - COMMENT='table "tbl_a", host "127.0.0.1", port "$MASTER_1_MYPORT", user "root"'; ---connection master_1 -alter table mysql.spider_table_sts drop column checksum; -insert into mysql.spider_table_sts values ('auto_test_local', 'tbl_a', 0, 0, 0, 0, 0, '2019-01-01 00:00:00', '2019-01-01 00:00:00', '2019-01-01 00:00:00'); diff --git a/storage/spider/mysql-test/spider/bugfix/r/gbh_outer_fields_in_join.result b/storage/spider/mysql-test/spider/bugfix/r/gbh_outer_fields_in_join.result new file mode 100644 index 00000000..75c2f2c2 --- /dev/null +++ b/storage/spider/mysql-test/spider/bugfix/r/gbh_outer_fields_in_join.result @@ -0,0 +1,50 @@ +# +# MDEV-26247 Spider: Valid LEFT JOIN results in ERROR 1064 +# +for master_1 +for child2 +child2_1 +child2_2 +child2_3 +for child3 +connection child2_1; +CREATE DATABASE auto_test_remote; +USE auto_test_remote; +CREATE TABLE t1 (a int) ENGINE=InnoDB DEFAULT CHARSET=utf8; +CREATE TABLE t2 (a int) ENGINE=InnoDB DEFAULT CHARSET=utf8; +CREATE TABLE t3 (a int) ENGINE=InnoDB DEFAULT CHARSET=utf8; +INSERT INTO t1 VALUES (1); +INSERT INTO t2 VALUES (1), (2); +INSERT INTO t3 VALUES (1), (2), (3); +connection master_1; +CREATE DATABASE auto_test_local; +USE auto_test_local; +CREATE TABLE t1 (a int) ENGINE=Spider DEFAULT CHARSET=utf8 COMMENT='srv "s_2_1", table "t1"'; +CREATE TABLE t2 (a int) ENGINE=Spider DEFAULT CHARSET=utf8 COMMENT='srv "s_2_1", table "t2"'; +CREATE TABLE t3 (a int) ENGINE=Spider DEFAULT CHARSET=utf8 COMMENT='srv "s_2_1", table "t3"'; +select * from t3 left join t1 on t3.a = t1.a left join t2 on t3.a = t2.a; +a a a +1 1 1 +2 NULL 2 +3 NULL NULL +select * from t1 left join t2 on t1.a = t2.a right join t3 on t3.a = t1.a; +a a a +1 1 1 +NULL NULL 2 +NULL NULL 3 +select * from t3 left join (t1 left join t2 on t1.a = t2.a) on t3.a = t1.a; +a a a +1 1 1 +2 NULL NULL +3 NULL NULL +drop table t1, t2, t3; +connection master_1; +DROP DATABASE IF EXISTS auto_test_local; +connection child2_1; +DROP DATABASE IF EXISTS auto_test_remote; +for master_1 +for child2 +child2_1 +child2_2 +child2_3 +for child3 diff --git a/storage/spider/mysql-test/spider/bugfix/r/mdev_19866.result b/storage/spider/mysql-test/spider/bugfix/r/mdev_19866.result index dbf0f54c..4386b4cb 100644 --- a/storage/spider/mysql-test/spider/bugfix/r/mdev_19866.result +++ b/storage/spider/mysql-test/spider/bugfix/r/mdev_19866.result @@ -72,6 +72,7 @@ SELECT argument FROM mysql.general_log WHERE command_type != 'Execute' AND argum argument select `pkey`,`val` from `auto_test_remote`.`tbl_a` select `pkey`,`val` from `auto_test_remote`.`tbl_a` where `pkey` = 1 +select 1 from (select 1) t0 select `pkey`,`val` from `auto_test_remote`.`tbl_a` select `pkey`,`val` from `auto_test_remote`.`tbl_a` SELECT argument FROM mysql.general_log WHERE command_type != 'Execute' AND argument LIKE '%select %' @@ -85,6 +86,7 @@ argument select `pkey`,`val` from `auto_test_remote2`.`tbl_a` select `pkey`,`val` from `auto_test_remote2`.`tbl_a` select `pkey`,`val` from `auto_test_remote2`.`tbl_a` where `pkey` = 2 +select 1 from (select 1) t0 select `pkey`,`val` from `auto_test_remote2`.`tbl_a` SELECT argument FROM mysql.general_log WHERE command_type != 'Execute' AND argument LIKE '%select %' SELECT pkey, val FROM tbl_a ORDER BY pkey; diff --git a/storage/spider/mysql-test/spider/bugfix/r/mdev_26151.result b/storage/spider/mysql-test/spider/bugfix/r/mdev_26151.result index b0a430e0..326b84a0 100644 --- a/storage/spider/mysql-test/spider/bugfix/r/mdev_26151.result +++ b/storage/spider/mysql-test/spider/bugfix/r/mdev_26151.result @@ -6,6 +6,9 @@ for child2 for child3 set @old_spider_bgs_mode= @@spider_bgs_mode; set session spider_bgs_mode=1; +set spider_same_server_link=1; +set @old_spider_same_server_link=@@global.spider_same_server_link; +set global spider_same_server_link=1; CREATE SERVER $srv FOREIGN DATA WRAPPER MYSQL OPTIONS (SOCKET "$MASTER_1_MYSOCK", DATABASE 'test',user 'root'); create table td (a int, PRIMARY KEY (a)); create table ts (a int, PRIMARY KEY (a)) ENGINE=Spider COMMENT='WRAPPER "mysql", srv "srv_mdev_26151",TABLE "td", casual_read "3"'; @@ -26,6 +29,7 @@ min(a) drop table td, ts; drop server srv_mdev_26151; set session spider_bgs_mode=@old_spider_bgs_mode; +set global spider_same_server_link=@old_spider_same_server_link; for master_1 for child2 for child3 diff --git a/storage/spider/mysql-test/spider/bugfix/r/mdev_26247.result b/storage/spider/mysql-test/spider/bugfix/r/mdev_26247.result index 723a81a9..6eddf9a7 100644 --- a/storage/spider/mysql-test/spider/bugfix/r/mdev_26247.result +++ b/storage/spider/mysql-test/spider/bugfix/r/mdev_26247.result @@ -28,8 +28,6 @@ INSERT INTO t1 VALUES (1), (2), (3); INSERT INTO t2 VALUES (1, 11), (2, 22), (3, 33); INSERT INTO t3 VALUES (1, 111), (2, 222), (3, 333); connection master_1; -set @old_spider_disable_group_by_handler=@@spider_disable_group_by_handler; -set spider_disable_group_by_handler=1; CREATE DATABASE auto_test_local; USE auto_test_local; CREATE TABLE t1 ( @@ -82,7 +80,6 @@ a22 22 connection master_1; DROP DATABASE IF EXISTS auto_test_local; -set spider_disable_group_by_handler=@old_spider_disable_group_by_handler; connection child2_1; DROP DATABASE IF EXISTS auto_test_remote; for master_1 @@ -91,6 +88,3 @@ child2_1 child2_2 child2_3 for child3 -# -# end of test mdev_26247 -# diff --git a/storage/spider/mysql-test/spider/bugfix/r/mdev_26541.result b/storage/spider/mysql-test/spider/bugfix/r/mdev_26541.result index 72921d2e..35a9d916 100644 --- a/storage/spider/mysql-test/spider/bugfix/r/mdev_26541.result +++ b/storage/spider/mysql-test/spider/bugfix/r/mdev_26541.result @@ -2,18 +2,3 @@ # MDEV-26541 Undefined symbol: _ZTI12ha_partition when attempting to use ha_spider.so in UBSAN builds # INSTALL SONAME 'ha_spider.so'; -DROP FUNCTION spider_flush_table_mon_cache; -DROP FUNCTION spider_copy_tables; -DROP FUNCTION spider_ping_table; -DROP FUNCTION spider_bg_direct_sql; -DROP FUNCTION spider_direct_sql; -UNINSTALL SONAME IF EXISTS "ha_spider"; -DROP TABLE IF EXISTS mysql.spider_xa; -DROP TABLE IF EXISTS mysql.spider_xa_member; -DROP TABLE IF EXISTS mysql.spider_xa_failed_log; -DROP TABLE IF EXISTS mysql.spider_tables; -DROP TABLE IF EXISTS mysql.spider_link_mon_servers; -DROP TABLE IF EXISTS mysql.spider_link_failed_log; -DROP TABLE IF EXISTS mysql.spider_table_position_for_recovery; -DROP TABLE IF EXISTS mysql.spider_table_sts; -DROP TABLE IF EXISTS mysql.spider_table_crd; diff --git a/storage/spider/mysql-test/spider/bugfix/r/mdev_27575.result b/storage/spider/mysql-test/spider/bugfix/r/mdev_27575.result new file mode 100644 index 00000000..3b137eaa --- /dev/null +++ b/storage/spider/mysql-test/spider/bugfix/r/mdev_27575.result @@ -0,0 +1,17 @@ +# +# MDEV-27575 Spider: UBSAN member access within null pointer of type 'struct st_plugin_int and SIGSEGV in intern_plugin_lock on SHUTDOWN when setting Spider as default storage engine (temporary or global) +# +for master_1 +for child2 +for child3 +SET GLOBAL default_tmp_storage_engine=spider; +ERROR HY000: Table storage engine 'SPIDER' does not support the create option 'TEMPORARY' +# restart +SET GLOBAL default_storage_engine=Spider; +# restart +for master_1 +for child2 +for child3 +# +# end of test mdev_27575 +# diff --git a/storage/spider/mysql-test/spider/bugfix/r/mdev_28683.result b/storage/spider/mysql-test/spider/bugfix/r/mdev_28683.result new file mode 100644 index 00000000..358c794e --- /dev/null +++ b/storage/spider/mysql-test/spider/bugfix/r/mdev_28683.result @@ -0,0 +1,22 @@ +# +# MDEV-28683 Spider: SIGSEGV in spider_db_direct_delete, SIGSEGV in spider_db_connect, ASAN: heap-use-after-free in spider_db_direct_delete +# +for master_1 +for child2 +for child3 +CREATE TABLE t (c INT) ENGINE=Spider; +SELECT * FROM t; +ERROR HY000: Unable to connect to foreign data source: localhost +INSERT INTO t (SELECT 1 FROM t); +ERROR HY000: Unable to connect to foreign data source: localhost +LOCK TABLES t WRITE CONCURRENT; +DELETE FROM t; +ERROR HY000: Unable to connect to foreign data source: localhost +UNLOCK TABLES; +DROP TABLE t; +for master_1 +for child2 +for child3 +# +# end of test mdev_28683 +# diff --git a/storage/spider/mysql-test/spider/bugfix/r/mdev_28739.result b/storage/spider/mysql-test/spider/bugfix/r/mdev_28739.result new file mode 100644 index 00000000..ee497bb3 --- /dev/null +++ b/storage/spider/mysql-test/spider/bugfix/r/mdev_28739.result @@ -0,0 +1,34 @@ +# +# MDEV-28739 Trying to lock uninitialized mutex or hang upon shutdown after using Spider with query_cache +# +for master_1 +for child2 +child2_1 +child2_2 +child2_3 +for child3 +connection child2_1; +CREATE DATABASE auto_test_remote; +USE auto_test_remote; +CREATE TABLE tbl_a (id INT); +connection master_1; +CREATE DATABASE auto_test_local; +USE auto_test_local; +set global query_cache_type= on; +set spider_same_server_link = on; +CREATE TABLE tbl_a ( +id INT +) ENGINE=Spider DEFAULT CHARSET=utf8 COMMENT='table "tbl_a", srv "s_2_1"'; +SELECT * FROM tbl_a; +id +# restart +connection master_1; +DROP DATABASE IF EXISTS auto_test_local; +connection child2_1; +DROP DATABASE IF EXISTS auto_test_remote; +for master_1 +for child2 +child2_1 +child2_2 +child2_3 +for child3 diff --git a/storage/spider/mysql-test/spider/bugfix/r/mdev_28739_simple.result b/storage/spider/mysql-test/spider/bugfix/r/mdev_28739_simple.result new file mode 100644 index 00000000..1c337c3d --- /dev/null +++ b/storage/spider/mysql-test/spider/bugfix/r/mdev_28739_simple.result @@ -0,0 +1,21 @@ +# +# MDEV-28739 Trying to lock uninitialized mutex or hang upon shutdown after using Spider with query_cache +# +for master_1 +for child2 +for child3 +set global query_cache_type= on; +set spider_same_server_link=1; +CREATE SERVER srv FOREIGN DATA WRAPPER mysql +OPTIONS (SOCKET "$MASTER_1_MYSOCK", DATABASE 'test',user 'root'); +create table t2 (c int); +create table t1 (c int) ENGINE=Spider +COMMENT='WRAPPER "mysql", srv "srv",TABLE "t2"'; +SELECT * FROM t1; +c +# restart +drop table t1, t2; +drop server srv; +for master_1 +for child2 +for child3 diff --git a/storage/spider/mysql-test/spider/bugfix/r/mdev_28856.result b/storage/spider/mysql-test/spider/bugfix/r/mdev_28856.result index fae3cc6b..7e4fd3cd 100644 --- a/storage/spider/mysql-test/spider/bugfix/r/mdev_28856.result +++ b/storage/spider/mysql-test/spider/bugfix/r/mdev_28856.result @@ -4,6 +4,7 @@ for master_1 for child2 for child3 +set spider_same_server_link=1; CREATE SERVER srv FOREIGN DATA WRAPPER mysql OPTIONS (SOCKET "$MASTER_1_MYSOCK", DATABASE 'test',user 'root'); # testing monitoring_* diff --git a/storage/spider/mysql-test/spider/bugfix/r/mdev_28998.result b/storage/spider/mysql-test/spider/bugfix/r/mdev_28998.result index d664b090..e92fb199 100644 --- a/storage/spider/mysql-test/spider/bugfix/r/mdev_28998.result +++ b/storage/spider/mysql-test/spider/bugfix/r/mdev_28998.result @@ -4,8 +4,7 @@ for master_1 for child2 for child3 -set @old_spider_disable_group_by_handler=@@spider_disable_group_by_handler; -set spider_disable_group_by_handler=1; +set spider_same_server_link=1; CREATE SERVER s FOREIGN DATA WRAPPER MYSQL OPTIONS (SOCKET "$MASTER_1_MYSOCK", DATABASE 'test',user 'root'); CREATE TABLE t1 (a INT); INSERT INTO t1 VALUES (1),(2); @@ -17,7 +16,6 @@ a 2 DROP TABLE t2, t1_SPIDER, t1; drop server s; -set spider_disable_group_by_handler=@old_spider_disable_group_by_handler; for master_1 for child2 for child3 diff --git a/storage/spider/mysql-test/spider/bugfix/r/mdev_29002.result b/storage/spider/mysql-test/spider/bugfix/r/mdev_29002.result new file mode 100644 index 00000000..894f51c5 --- /dev/null +++ b/storage/spider/mysql-test/spider/bugfix/r/mdev_29002.result @@ -0,0 +1,34 @@ +for master_1 +for child2 +for child3 +SET spider_same_server_link= on; +CREATE SERVER s FOREIGN DATA WRAPPER mysql +OPTIONS (SOCKET "$MASTER_1_MYSOCK", DATABASE 'test',user 'root'); +CREATE TABLE t (a INT); +CREATE TABLE t1_spider (a INT) ENGINE=SPIDER COMMENT = "wrapper 'mysql', srv 's', table 't'"; +CREATE TABLE t2_spider (a INT) ENGINE=SPIDER COMMENT = "wrapper 'mysql', srv 's', table 't'"; +SELECT * FROM t1_spider, t2_spider; +a a +SELECT table_name, index_name, cardinality FROM INFORMATION_SCHEMA.STATISTICS WHERE table_name IN ('t1_spider','t2_spider'); +table_name index_name cardinality +RENAME TABLE t1_spider TO t3_spider; +SELECT * FROM t3_spider; +a +DROP TABLE t3_spider, t2_spider, t; +drop server s; +CREATE TABLE t1 (c INT) ENGINE=Spider COMMENT='WRAPPER "mysql",HOST "srv",TABLE "t"'; +CREATE TABLE t2 (c INT) ENGINE=Spider COMMENT='WRAPPER "mysql",HOST "srv",TABLE "t"'; +CREATE TABLE t3 (c INT) ENGINE=Spider COMMENT='WRAPPER "mysql",HOST "srv",TABLE "t"'; +SHOW TABLE STATUS; +Name Engine Version Row_format Rows Avg_row_length Data_length Max_data_length Index_length Data_free Auto_increment Create_time Update_time Check_time Collation Checksum Create_options Comment Max_index_length Temporary +t1 SPIDER 10 NULL 0 0 0 0 0 NULL NULL NULL NULL latin1_swedish_ci NULL Unable to connect to foreign data source: srv 0 +t2 SPIDER 10 NULL 0 0 0 0 0 NULL NULL NULL NULL latin1_swedish_ci NULL Unable to connect to foreign data source: srv 0 +t3 SPIDER 10 NULL 0 0 0 0 0 NULL NULL NULL NULL latin1_swedish_ci NULL Unable to connect to foreign data source: srv 0 +Warnings: +Warning 1429 Unable to connect to foreign data source: srv +Warning 1429 Unable to connect to foreign data source: srv +Warning 1429 Unable to connect to foreign data source: srv +drop table t1, t2, t3; +for master_1 +for child2 +for child3 diff --git a/storage/spider/mysql-test/spider/bugfix/r/mdev_29163.result b/storage/spider/mysql-test/spider/bugfix/r/mdev_29163.result index 96d8a729..f58ab605 100644 --- a/storage/spider/mysql-test/spider/bugfix/r/mdev_29163.result +++ b/storage/spider/mysql-test/spider/bugfix/r/mdev_29163.result @@ -4,8 +4,7 @@ for master_1 for child2 for child3 -set @old_spider_disable_group_by_handler=@@spider_disable_group_by_handler; -set spider_disable_group_by_handler=1; +set spider_same_server_link=1; CREATE SERVER s FOREIGN DATA WRAPPER MYSQL OPTIONS (SOCKET "$MASTER_1_MYSOCK", DATABASE 'test',user 'root'); CREATE TABLE t1 (a INT); CREATE TABLE t2 (b INT); @@ -17,7 +16,6 @@ SELECT t1_spider.* FROM t1_spider LEFT JOIN t2_spider LEFT JOIN t3_spider ON b = a DROP TABLE t1_spider, t2_spider, t3_spider, t1, t2, t3; drop server s; -set spider_disable_group_by_handler=@old_spider_disable_group_by_handler; for master_1 for child2 for child3 diff --git a/storage/spider/mysql-test/spider/bugfix/r/mdev_29421.result b/storage/spider/mysql-test/spider/bugfix/r/mdev_29421.result new file mode 100644 index 00000000..bcbf050d --- /dev/null +++ b/storage/spider/mysql-test/spider/bugfix/r/mdev_29421.result @@ -0,0 +1,19 @@ +for master_1 +for child2 +for child3 +set @old_table_open_cache=@@global.table_open_cache; +set global table_open_cache=10; +CREATE SERVER srv FOREIGN DATA WRAPPER mysql +OPTIONS (SOCKET "$MASTER_1_MYSOCK", DATABASE 'test',user 'root'); +CREATE TABLE t (c INT) ENGINE=InnoDB; +CREATE TABLE t_s (c INT) ENGINE=Spider COMMENT='WRAPPER "mysql", srv "srv", TABLE "t"'; +CREATE TABLE t1 (a INT) ENGINE=Spider; +SELECT * FROM t1; +ERROR HY000: Unable to connect to foreign data source: localhost +SELECT * FROM information_schema.tables; +DROP TABLE t, t_s, t1; +drop server srv; +set global table_open_cache=@old_table_open_cache; +for master_1 +for child2 +for child3 diff --git a/storage/spider/mysql-test/spider/bugfix/r/mdev_29456.result b/storage/spider/mysql-test/spider/bugfix/r/mdev_29456.result new file mode 100644 index 00000000..365c3d63 --- /dev/null +++ b/storage/spider/mysql-test/spider/bugfix/r/mdev_29456.result @@ -0,0 +1,42 @@ +# +# MDEV-29456 Spider hangs in 'Waiting for table metadata lock' state on CREATE TABLE after LOCK TABLES +# +for master_1 +for child2 +for child3 +set spider_same_server_link=1; +CREATE SERVER srv FOREIGN DATA WRAPPER mysql +OPTIONS (SOCKET "$MASTER_1_MYSOCK", DATABASE 'test',user 'root'); +create table t1 (c int); +create table t2 (c int) ENGINE=Spider +COMMENT='WRAPPER "mysql", srv "srv",TABLE "t1"'; +CREATE TABLE t3 (c INT KEY) ENGINE=Spider; +LOCK TABLE t2 WRITE; +LOCK TABLE t3 WRITE; +ERROR HY000: Unable to connect to foreign data source: localhost +UNLOCK TABLES; +drop table t1, t2, t3; +CREATE TABLE t (c INT) ENGINE=InnoDB; +CREATE TABLE t1 (c INT) ENGINE=Spider; +CREATE TABLE t2 (c INT) ENGINE=Spider COMMENT='WRAPPER "mysql",srv "srv",TABLE "t"'; +LOCK TABLES t2 WRITE; +LOCK TABLES t1 READ; +ERROR HY000: Unable to connect to foreign data source: localhost +CREATE TABLE t (c INT) ENGINE=Spider; +ERROR 42S01: Table 't' already exists +drop table t, t1, t2; +CREATE TABLE t (c INT) ENGINE=InnoDB; +CREATE TABLE t2 (c INT) ENGINE=Spider COMMENT='WRAPPER "mysql",srv "srv",TABLE "t"'; +CREATE TABLE t3 (c INT) ENGINE=InnoDB; +LOCK TABLES t2 WRITE; +LOCK TABLES mysql.proc WRITE,mysql.user WRITE; +ERROR HY000: You can't combine write-locking of system tables with other tables or lock types +INSERT INTO t3 SELECT * FROM t; +drop table t, t2, t3; +drop server srv; +for master_1 +for child2 +for child3 +# +# end of test mdev_29456 +# diff --git a/storage/spider/mysql-test/spider/bugfix/r/mdev_29502.result b/storage/spider/mysql-test/spider/bugfix/r/mdev_29502.result index 54c5e9a6..7749898e 100644 --- a/storage/spider/mysql-test/spider/bugfix/r/mdev_29502.result +++ b/storage/spider/mysql-test/spider/bugfix/r/mdev_29502.result @@ -4,6 +4,7 @@ for master_1 for child2 for child3 +set spider_same_server_link=1; CREATE SERVER $srv FOREIGN DATA WRAPPER MYSQL OPTIONS (SOCKET "$MASTER_1_MYSOCK", DATABASE 'test',user 'root'); CREATE TABLE t (a INT); INSERT INTO t VALUES (23),(48); diff --git a/storage/spider/mysql-test/spider/bugfix/r/mdev_29667.result b/storage/spider/mysql-test/spider/bugfix/r/mdev_29667.result new file mode 100644 index 00000000..f2e90760 --- /dev/null +++ b/storage/spider/mysql-test/spider/bugfix/r/mdev_29667.result @@ -0,0 +1,40 @@ +# +# MDEV-29667 Server hangs on DROP DATABASE after failing LOCK TABLES on Spider table +# +for master_1 +for child2 +child2_1 +child2_2 +child2_3 +for child3 +connection child2_1; +CREATE DATABASE auto_test_remote; +USE auto_test_remote; +CREATE TABLE tbl_a ( +a INT +) ENGINE=InnoDB DEFAULT CHARSET=utf8; +connection master_1; +CREATE DATABASE auto_test_local; +USE auto_test_local; +CREATE TABLE tbl_a ( +a INT +) ENGINE=Spider DEFAULT CHARSET=utf8 COMMENT='table "tbl_a", srv "s_2_1"'; +CREATE TABLE tbl_b ( +a INT +) ENGINE=Spider DEFAULT CHARSET=utf8; +LOCK TABLES tbl_a WRITE; +LOCK TABLES tbl_b READ, tbl_a READ; +ERROR HY000: Unable to connect to foreign data source: localhost +connection master_1; +DROP DATABASE IF EXISTS auto_test_local; +connection child2_1; +DROP DATABASE IF EXISTS auto_test_remote; +for master_1 +for child2 +child2_1 +child2_2 +child2_3 +for child3 +# +# end of test mdev_29667 +# diff --git a/storage/spider/mysql-test/spider/bugfix/r/mdev_29963.result b/storage/spider/mysql-test/spider/bugfix/r/mdev_29963.result new file mode 100644 index 00000000..60451596 --- /dev/null +++ b/storage/spider/mysql-test/spider/bugfix/r/mdev_29963.result @@ -0,0 +1,44 @@ +# +# MDEV-29963 SIGSEGV in spider_db_mbase::append_lock_tables on LOCK TABLES +# +for master_1 +for child2 +for child3 +set spider_same_server_link=1; +CREATE SERVER srv FOREIGN DATA WRAPPER mysql +OPTIONS (SOCKET "$MASTER_1_MYSOCK", DATABASE 'test',user 'root'); +CREATE TABLE t (a INT) ENGINE=Spider; +CREATE TABLE t2 (b INT) ENGINE=Spider COMMENT='WRAPPER "mysql",srv "srv",TABLE "t"'; +LOCK TABLES t AS a READ,t2 AS b LOW_PRIORITY WRITE,t2 AS c WRITE; +ERROR HY000: Unable to connect to foreign data source: localhost +DROP TABLE t2; +CREATE TABLE t2 (c INT) ENGINE=Spider COMMENT='WRAPPER "mysql",srv "srv",TABLE "t"'; +LOCK TABLES t2 WRITE; +ERROR HY000: Unable to connect to foreign data source: localhost +DROP TABLE t2,t; +CREATE TABLE t (a INT); +CREATE TABLE t1 (a INT) ENGINE=Spider; +CREATE TABLE t2 (b INT) ENGINE=Spider COMMENT='WRAPPER "mysql",srv "srv",TABLE "t"'; +LOCK TABLES t1 READ, t2 WRITE; +ERROR HY000: Unable to connect to foreign data source: localhost +DROP TABLE t2; +CREATE TABLE t2 (c INT) ENGINE=Spider COMMENT='WRAPPER "mysql",srv "srv",TABLE "t"'; +LOCK TABLES t2 WRITE; +UNLOCK TABLES; +DROP TABLE t, t1, t2; +CREATE TABLE t1 (c INT) ENGINE=Spider; +CREATE TABLE t2 (c INT) ENGINE=Spider COMMENT="WRAPPER 'mysql',srv 'srv',TABLE 't1'"; +LOCK TABLES t1 WRITE,t2 WRITE; +ERROR HY000: Unable to connect to foreign data source: localhost +TRUNCATE t2; +ERROR HY000: Unable to connect to foreign data source: localhost +LOCK TABLES t2 AS o WRITE; +ERROR HY000: Unable to connect to foreign data source: localhost +drop table t1, t2; +drop server srv; +for master_1 +for child2 +for child3 +# +# end of test mdev_29963 +# diff --git a/storage/spider/mysql-test/spider/bugfix/r/mdev_30014.result b/storage/spider/mysql-test/spider/bugfix/r/mdev_30014.result new file mode 100644 index 00000000..e971f323 --- /dev/null +++ b/storage/spider/mysql-test/spider/bugfix/r/mdev_30014.result @@ -0,0 +1,37 @@ +# +# MDEV-30014 heap-use-after-free in ha_spider::lock_tables(), highly sporadic SIGSEGV in intern_close_table +# +for master_1 +for child2 +for child3 +set spider_same_server_link=1; +CREATE SERVER srv FOREIGN DATA WRAPPER mysql +OPTIONS (SOCKET "$MASTER_1_MYSOCK", DATABASE 'test',user 'root'); +create table t1 (c int); +create table t2 (c int) ENGINE=Spider +COMMENT='WRAPPER "mysql", srv "srv",TABLE "t1"'; +CREATE TABLE t3 (c INT KEY) ENGINE=Spider; +LOCK TABLE t2 WRITE,t3 WRITE; +ERROR HY000: Unable to connect to foreign data source: localhost +CREATE TABLE t4 (c INT) ENGINE=Spider; +FLUSH NO_WRITE_TO_BINLOG TABLES t4 WITH READ LOCK; +Warnings: +Error 1429 Unable to connect to foreign data source: localhost +Error 1429 Unable to connect to foreign data source: localhost +UNLOCK TABLES; +drop table t1, t2, t3, t4; +create table t1 (c int); +create table t2 (c int) ENGINE=Spider +COMMENT='WRAPPER "mysql", srv "srv",TABLE "t1"'; +CREATE TABLE t3 (c INT KEY) ENGINE=Spider; +LOCK TABLE t2 WRITE, t3 WRITE; +ERROR HY000: Unable to connect to foreign data source: localhost +UNLOCK TABLES; +drop table t1, t2, t3; +drop server srv; +for master_1 +for child2 +for child3 +# +# end of test mdev_30014 +# diff --git a/storage/spider/mysql-test/spider/bugfix/r/mdev_30170.result b/storage/spider/mysql-test/spider/bugfix/r/mdev_30170.result new file mode 100644 index 00000000..2183447b --- /dev/null +++ b/storage/spider/mysql-test/spider/bugfix/r/mdev_30170.result @@ -0,0 +1,7 @@ +install soname 'ha_spider'; +DROP TABLE non_existing_table; +ERROR 42S02: Unknown table 'test.non_existing_table' +create or replace table non_existing_table (c int) engine=Spider; +drop table non_existing_table; +Warnings: +Warning 1620 Plugin is busy and will be uninstalled on shutdown diff --git a/storage/spider/mysql-test/spider/bugfix/r/mdev_30392.result b/storage/spider/mysql-test/spider/bugfix/r/mdev_30392.result index 13297e10..cefa5248 100644 --- a/storage/spider/mysql-test/spider/bugfix/r/mdev_30392.result +++ b/storage/spider/mysql-test/spider/bugfix/r/mdev_30392.result @@ -4,8 +4,7 @@ for master_1 for child2 for child3 -set @old_spider_disable_group_by_handler=@@spider_disable_group_by_handler; -set spider_disable_group_by_handler=1; +set spider_same_server_link=1; CREATE SERVER srv FOREIGN DATA WRAPPER MYSQL OPTIONS (SOCKET "$MASTER_1_MYSOCK", DATABASE 'test',user 'root'); CREATE TABLE t1 (a INT); INSERT INTO t1 VALUES (1),(2); @@ -16,7 +15,6 @@ a 2 DROP TABLE t1, t2; DROP SERVER srv; -set spider_disable_group_by_handler=@old_spider_disable_group_by_handler; for master_1 for child2 for child3 diff --git a/storage/spider/mysql-test/spider/bugfix/r/mdev_31338.result b/storage/spider/mysql-test/spider/bugfix/r/mdev_31338.result index 62b06336..f156cf38 100644 --- a/storage/spider/mysql-test/spider/bugfix/r/mdev_31338.result +++ b/storage/spider/mysql-test/spider/bugfix/r/mdev_31338.result @@ -4,6 +4,7 @@ for master_1 for child2 for child3 +set spider_same_server_link=1; CREATE SERVER srv FOREIGN DATA WRAPPER MYSQL OPTIONS (SOCKET "$MASTER_1_MYSOCK", DATABASE 'test',user 'root'); CREATE TABLE t (c BLOB) ENGINE=InnoDB; CREATE TABLE ts (c BLOB) ENGINE=Spider COMMENT='WRAPPER "mysql",srv "srv",TABLE "t"'; diff --git a/storage/spider/mysql-test/spider/bugfix/r/mdev_31524.result b/storage/spider/mysql-test/spider/bugfix/r/mdev_31524.result index c1aa7a1f..26877967 100644 --- a/storage/spider/mysql-test/spider/bugfix/r/mdev_31524.result +++ b/storage/spider/mysql-test/spider/bugfix/r/mdev_31524.result @@ -5,6 +5,7 @@ for master_1 for child2 for child3 SET @old_spider_read_only_mode = @@session.spider_read_only_mode; +set spider_same_server_link=1; CREATE SERVER $srv FOREIGN DATA WRAPPER MYSQL OPTIONS (SOCKET "$MASTER_1_MYSOCK", DATABASE 'test',user 'root'); set session spider_read_only_mode = default; create table t2 (c int); diff --git a/storage/spider/mysql-test/spider/bugfix/r/mdev_31645.result b/storage/spider/mysql-test/spider/bugfix/r/mdev_31645.result index 28cb546e..5197abd3 100644 --- a/storage/spider/mysql-test/spider/bugfix/r/mdev_31645.result +++ b/storage/spider/mysql-test/spider/bugfix/r/mdev_31645.result @@ -4,22 +4,17 @@ for master_1 for child2 for child3 -set @old_spider_disable_group_by_handler=@@spider_disable_group_by_handler; -set spider_disable_group_by_handler=1; +set spider_same_server_link=1; CREATE SERVER srv FOREIGN DATA WRAPPER MYSQL OPTIONS (SOCKET "$MASTER_1_MYSOCK", DATABASE 'test',user 'root'); CREATE TABLE t1 ( a bigint(20) NOT NULL, b bigint(20) DEFAULT 0, PRIMARY KEY (a)); CREATE TABLE t2 ( a bigint(20) NOT NULL, b bigint(20) DEFAULT 0, PRIMARY KEY (a)) ENGINE=SPIDER COMMENT='srv "srv", WRAPPER "mysql", TABLE "t1"'; -SET SESSION optimizer_switch='semijoin=off'; -SELECT * FROM t2 -WHERE A BETWEEN 0 AND 10 AND B IN(SELECT B FROM t2 WHERE A BETWEEN 11 AND 20); +SELECT * FROM t2 WHERE b IN (SELECT b FROM t2 WHERE a > 10); a b -SET SESSION optimizer_switch='semijoin=on'; SELECT * FROM t2 WHERE A BETWEEN 0 AND 10 AND B IN(SELECT B FROM t2 WHERE A BETWEEN 11 AND 20); a b drop table t1, t2; drop server srv; -set spider_disable_group_by_handler=@old_spider_disable_group_by_handler; for master_1 for child2 for child3 diff --git a/storage/spider/mysql-test/spider/bugfix/r/mdev_31996.result b/storage/spider/mysql-test/spider/bugfix/r/mdev_31996.result index 04d7e884..cbc91432 100644 --- a/storage/spider/mysql-test/spider/bugfix/r/mdev_31996.result +++ b/storage/spider/mysql-test/spider/bugfix/r/mdev_31996.result @@ -1,6 +1,7 @@ for master_1 for child2 for child3 +set spider_same_server_link=1; CREATE SERVER srv FOREIGN DATA WRAPPER mysql OPTIONS (SOCKET "$MASTER_1_MYSOCK", DATABASE 'test',user 'root'); set session spider_delete_all_rows_type=0; diff --git a/storage/spider/mysql-test/spider/bugfix/r/mdev_32753.result b/storage/spider/mysql-test/spider/bugfix/r/mdev_32753.result new file mode 100644 index 00000000..4260d80f --- /dev/null +++ b/storage/spider/mysql-test/spider/bugfix/r/mdev_32753.result @@ -0,0 +1,10 @@ +# +# MDEV-32753 Spider engine does not load in ORACLE mode +# +select * from mysql.plugin; +name dl +create table t (c int) Engine=SPIDER; +drop table t; +# +# end of test mdev_32753 +# diff --git a/storage/spider/mysql-test/spider/bugfix/r/mdev_32753_after_start.result b/storage/spider/mysql-test/spider/bugfix/r/mdev_32753_after_start.result new file mode 100644 index 00000000..4e046d16 --- /dev/null +++ b/storage/spider/mysql-test/spider/bugfix/r/mdev_32753_after_start.result @@ -0,0 +1,14 @@ +# +# MDEV-32753 Spider engine does not load in ORACLE mode +# +install soname 'ha_spider'; +select * from mysql.plugin; +name dl +SPIDER ha_spider.so +SPIDER_ALLOC_MEM ha_spider.so +SPIDER_WRAPPER_PROTOCOLS ha_spider.so +create table t (c int) Engine=SPIDER; +drop table t; +# +# end of test mdev_32753_after_start +# diff --git a/storage/spider/mysql-test/spider/bugfix/r/mdev_32753_after_start_session.result b/storage/spider/mysql-test/spider/bugfix/r/mdev_32753_after_start_session.result new file mode 100644 index 00000000..b9d02563 --- /dev/null +++ b/storage/spider/mysql-test/spider/bugfix/r/mdev_32753_after_start_session.result @@ -0,0 +1,17 @@ +# +# MDEV-32753 Spider engine does not load in ORACLE mode +# +set @old_sql_mode=@@sql_mode; +SET @@sql_mode = CONCAT(@@sql_mode, ',ORACLE'); +install soname 'ha_spider'; +select * from mysql.plugin; +name dl +SPIDER ha_spider.so +SPIDER_ALLOC_MEM ha_spider.so +SPIDER_WRAPPER_PROTOCOLS ha_spider.so +create table t (c int) Engine=SPIDER; +drop table t; +set sql_mode=@old_sql_mode; +# +# end of test mdev_32753_after_start +# diff --git a/storage/spider/mysql-test/spider/bugfix/r/mdev_32986.result b/storage/spider/mysql-test/spider/bugfix/r/mdev_32986.result new file mode 100644 index 00000000..c3bdef98 --- /dev/null +++ b/storage/spider/mysql-test/spider/bugfix/r/mdev_32986.result @@ -0,0 +1,24 @@ +# +# MDEV-32907 +# +for master_1 +for child2 +for child3 +set spider_same_server_link=1; +CREATE SERVER srv FOREIGN DATA WRAPPER mysql +OPTIONS (SOCKET "$MASTER_1_MYSOCK", DATABASE 'test',user 'root'); +create table t2 (c varchar(16)); +create table t1 (c varchar(16)) ENGINE=Spider +COMMENT='WRAPPER "mysql", srv "srv",TABLE "t2"'; +insert into t1 values ('TestSpiderRegex'); +select c from t1 where c regexp '(Test|Spider|Regex)'; +c +TestSpiderRegex +drop table t1, t2; +drop server srv; +for master_1 +for child2 +for child3 +# +# end of test mdev_32907 +# diff --git a/storage/spider/mysql-test/spider/bugfix/r/mdev_33008.result b/storage/spider/mysql-test/spider/bugfix/r/mdev_33008.result new file mode 100644 index 00000000..3bcb4bb0 --- /dev/null +++ b/storage/spider/mysql-test/spider/bugfix/r/mdev_33008.result @@ -0,0 +1,25 @@ +for master_1 +for child2 +for child3 +set spider_same_server_link=on; +CREATE SERVER srv FOREIGN DATA WRAPPER mysql +OPTIONS (SOCKET "$MASTER_1_MYSOCK", DATABASE 'test',user 'root'); +create table t2 ( +`id` int(11) NOT NULL AUTO_INCREMENT, +`code` varchar(10) DEFAULT NULL, +PRIMARY KEY (`id`) +); +create table t1 ENGINE=Spider +COMMENT='WRAPPER "mysql", srv "srv",TABLE "t2"'; +show create table t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `code` varchar(10) DEFAULT NULL, + PRIMARY KEY (`id`) +) ENGINE=SPIDER DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci COMMENT='WRAPPER "mysql", srv "srv",TABLE "t2"' +drop table t1, t2; +drop server srv; +for master_1 +for child2 +for child3 diff --git a/storage/spider/mysql-test/spider/bugfix/r/mdev_33191.result b/storage/spider/mysql-test/spider/bugfix/r/mdev_33191.result new file mode 100644 index 00000000..75ca6c2f --- /dev/null +++ b/storage/spider/mysql-test/spider/bugfix/r/mdev_33191.result @@ -0,0 +1,14 @@ +INSTALL SONAME 'ha_spider'; +set spider_same_server_link=on; +CREATE TABLE t2(c INT); +CREATE TABLE t1(c INT) ENGINE=Spider COMMENT='socket "$SOCKET", user "root", table "t2 t3"'; +Warnings: +Warning 138 The high availability feature of Spider has been deprecated and will be removed in a future release +ALTER TABLE t1 ENGINE=Spider; +Warnings: +Warning 138 The high availability feature of Spider has been deprecated and will be removed in a future release +TRUNCATE TABLE t1; +ERROR 42S02: Table 'test.t3' doesn't exist +drop table t1, t2; +Warnings: +Warning 1620 Plugin is busy and will be uninstalled on shutdown diff --git a/storage/spider/mysql-test/spider/bugfix/r/perfschema.result b/storage/spider/mysql-test/spider/bugfix/r/perfschema.result new file mode 100644 index 00000000..9ce2e38f --- /dev/null +++ b/storage/spider/mysql-test/spider/bugfix/r/perfschema.result @@ -0,0 +1,11 @@ +# +# MDEV-33031 Assertion failure upon reading from performance schema with binlog enabled +# +connect foo,localhost,root; +select variable_name, variable_value from performance_schema.status_by_thread +where variable_name like '%spider_direct_aggregate%'; +variable_name variable_value +Spider_direct_aggregate 0 +Spider_direct_aggregate 0 +disconnect foo; +connection default; diff --git a/storage/spider/mysql-test/spider/bugfix/r/self_reference_multi.result b/storage/spider/mysql-test/spider/bugfix/r/self_reference_multi.result index 196e48d5..0118f575 100644 --- a/storage/spider/mysql-test/spider/bugfix/r/self_reference_multi.result +++ b/storage/spider/mysql-test/spider/bugfix/r/self_reference_multi.result @@ -4,6 +4,9 @@ for child3 MDEV-6268 SPIDER table with no COMMENT clause causes queries to wait forever +set spider_same_server_link=1; +set @old_spider_same_server_link=@@global.spider_same_server_link; +set global spider_same_server_link=1; CREATE SERVER $srv FOREIGN DATA WRAPPER MYSQL OPTIONS (SOCKET "$MASTER_1_MYSOCK", DATABASE 'test',user 'root'); create table t2 (c int); create table t1 (c int) ENGINE=Spider COMMENT='WRAPPER "mysql", srv "srv_self_reference_multi",TABLE "t2"'; @@ -17,6 +20,7 @@ select * from t2; ERROR HY000: An infinite loop is detected when opening table test.t0 drop table t0, t1, t2; drop server srv_self_reference_multi; +set global spider_same_server_link=@old_spider_same_server_link; for master_1 for child2 for child3 diff --git a/storage/spider/mysql-test/spider/bugfix/r/signal_ddl_fail.result b/storage/spider/mysql-test/spider/bugfix/r/signal_ddl_fail.result new file mode 100644 index 00000000..c86e600b --- /dev/null +++ b/storage/spider/mysql-test/spider/bugfix/r/signal_ddl_fail.result @@ -0,0 +1,8 @@ +# +# MDEV-32559 Move alter table statements in spider init queries to be executed in the signal_ddl_recovery_done callback +# +select * from mysql.plugin; +name dl +# +# end of test signal_ddl_fail +# diff --git a/storage/spider/mysql-test/spider/bugfix/r/spider_join_with_non_spider.result b/storage/spider/mysql-test/spider/bugfix/r/spider_join_with_non_spider.result new file mode 100644 index 00000000..420ca657 --- /dev/null +++ b/storage/spider/mysql-test/spider/bugfix/r/spider_join_with_non_spider.result @@ -0,0 +1,24 @@ +# +# Test joining a spider table with a non-spider table +# +for master_1 +for child2 +for child3 +set spider_same_server_link=1; +CREATE SERVER srv FOREIGN DATA WRAPPER MYSQL OPTIONS (SOCKET "$MASTER_1_MYSOCK", DATABASE 'test',user 'root'); +create table t1 (c int); +create table t2 (d int); +insert into t2 values (1), (2); +create table t3 (c int) ENGINE=Spider COMMENT='WRAPPER "mysql", srv "srv",TABLE "t1"'; +insert into t3 values (2), (3); +select c from t3 join t2 on c = d; +c +2 +drop table t1, t2, t3; +drop server srv; +for master_1 +for child2 +for child3 +# +# end of test spider_join_with_non_spider +# diff --git a/storage/spider/mysql-test/spider/bugfix/r/spider_table_sts.result b/storage/spider/mysql-test/spider/bugfix/r/spider_table_sts.result deleted file mode 100644 index f915cc95..00000000 --- a/storage/spider/mysql-test/spider/bugfix/r/spider_table_sts.result +++ /dev/null @@ -1,38 +0,0 @@ -for master_1 -for child2 -for child3 -connection master_1; -alter table mysql.spider_table_sts drop column checksum; -insert into mysql.spider_table_sts values ('auto_test_local', 'tbl_a', 0, 0, 0, 0, 0, '2019-01-01 00:00:00', '2019-01-01 00:00:00', '2019-01-01 00:00:00'); - -this test is for MDEV-19842 - -drop and create databases -connection master_1; -CREATE DATABASE auto_test_local; -USE auto_test_local; - -create table -connection master_1; -CREATE TABLE tbl_a ( -pkey int NOT NULL, -PRIMARY KEY (pkey) -) MASTER_1_ENGINE MASTER_1_CHARSET MASTER_1_COMMENT_2_1 - -select test 1 -connection master_1; -SELECT pkey FROM tbl_a; -ERROR HY000: System table spider_table_sts is different version - -deinit -connection master_1; -DROP DATABASE IF EXISTS auto_test_local; -ERROR HY000: System table spider_table_sts is different version -connection master_1; -alter table mysql.spider_table_sts add column checksum bigint unsigned default null after update_time; -DROP DATABASE IF EXISTS auto_test_local; -for master_1 -for child2 -for child3 - -end of test diff --git a/storage/spider/mysql-test/spider/bugfix/r/subquery.result b/storage/spider/mysql-test/spider/bugfix/r/subquery.result new file mode 100644 index 00000000..280f5715 --- /dev/null +++ b/storage/spider/mysql-test/spider/bugfix/r/subquery.result @@ -0,0 +1,25 @@ +# +# Test spider select with subqueries +# +for master_1 +for child2 +for child3 +set spider_same_server_link=1; +CREATE SERVER srv FOREIGN DATA WRAPPER MYSQL OPTIONS (SOCKET "$MASTER_1_MYSOCK", DATABASE 'test',user 'root'); +create table t1 (c1 int); +create table t2 (c2 int); +insert into t1 values (1), (2); +insert into t2 values (0), (1), (2); +create table t1s (c1 int) ENGINE=Spider COMMENT='WRAPPER "mysql", srv "srv",TABLE "t1"'; +create table t2s (c2 int) ENGINE=Spider COMMENT='WRAPPER "mysql", srv "srv",TABLE "t2"'; +select c1 from t1s, (select c2 from t2s where c2 > 0) t where c1 + 1 = c2; +c1 +1 +drop table t1, t2, t1s, t2s; +drop server srv; +for master_1 +for child2 +for child3 +# +# end of test subquery +# diff --git a/storage/spider/mysql-test/spider/bugfix/t/spider_table_sts.cnf b/storage/spider/mysql-test/spider/bugfix/t/gbh_outer_fields_in_join.cnf index b0853e32..05dfd8a0 100644 --- a/storage/spider/mysql-test/spider/bugfix/t/spider_table_sts.cnf +++ b/storage/spider/mysql-test/spider/bugfix/t/gbh_outer_fields_in_join.cnf @@ -1,2 +1,3 @@ !include include/default_mysqld.cnf !include ../my_1_1.cnf +!include ../my_2_1.cnf diff --git a/storage/spider/mysql-test/spider/bugfix/t/gbh_outer_fields_in_join.test b/storage/spider/mysql-test/spider/bugfix/t/gbh_outer_fields_in_join.test new file mode 100644 index 00000000..f60abc30 --- /dev/null +++ b/storage/spider/mysql-test/spider/bugfix/t/gbh_outer_fields_in_join.test @@ -0,0 +1,46 @@ +--echo # +--echo # MDEV-26247 Spider: Valid LEFT JOIN results in ERROR 1064 +--echo # + +--disable_query_log +--disable_result_log +--source ../../t/test_init.inc +--enable_result_log +--enable_query_log + +--connection child2_1 +CREATE DATABASE auto_test_remote; +USE auto_test_remote; + +eval CREATE TABLE t1 (a int) $CHILD2_1_ENGINE $CHILD2_1_CHARSET; +eval CREATE TABLE t2 (a int) $CHILD2_1_ENGINE $CHILD2_1_CHARSET; +eval CREATE TABLE t3 (a int) $CHILD2_1_ENGINE $CHILD2_1_CHARSET; + +INSERT INTO t1 VALUES (1); +INSERT INTO t2 VALUES (1), (2); +INSERT INTO t3 VALUES (1), (2), (3); + +--connection master_1 +CREATE DATABASE auto_test_local; +USE auto_test_local; + +eval CREATE TABLE t1 (a int) $MASTER_1_ENGINE $MASTER_1_CHARSET COMMENT='srv "s_2_1", table "t1"'; +eval CREATE TABLE t2 (a int) $MASTER_1_ENGINE $MASTER_1_CHARSET COMMENT='srv "s_2_1", table "t2"'; +eval CREATE TABLE t3 (a int) $MASTER_1_ENGINE $MASTER_1_CHARSET COMMENT='srv "s_2_1", table "t3"'; + +select * from t3 left join t1 on t3.a = t1.a left join t2 on t3.a = t2.a; +select * from t1 left join t2 on t1.a = t2.a right join t3 on t3.a = t1.a; +select * from t3 left join (t1 left join t2 on t1.a = t2.a) on t3.a = t1.a; + +drop table t1, t2, t3; + +--connection master_1 +DROP DATABASE IF EXISTS auto_test_local; +--connection child2_1 +DROP DATABASE IF EXISTS auto_test_remote; + +--disable_query_log +--disable_result_log +--source ../t/test_deinit.inc +--enable_query_log +--enable_result_log diff --git a/storage/spider/mysql-test/spider/bugfix/t/mdev_26151.test b/storage/spider/mysql-test/spider/bugfix/t/mdev_26151.test index f9e157d3..dcf1438f 100644 --- a/storage/spider/mysql-test/spider/bugfix/t/mdev_26151.test +++ b/storage/spider/mysql-test/spider/bugfix/t/mdev_26151.test @@ -14,6 +14,10 @@ --let $srv=srv_mdev_26151 set @old_spider_bgs_mode= @@spider_bgs_mode; set session spider_bgs_mode=1; +set spider_same_server_link=1; +set @old_spider_same_server_link=@@global.spider_same_server_link; +set global spider_same_server_link=1; + evalp CREATE SERVER $srv FOREIGN DATA WRAPPER MYSQL OPTIONS (SOCKET "$MASTER_1_MYSOCK", DATABASE 'test',user 'root'); # casual_read != 0 && casual_read != 1 @@ -42,6 +46,7 @@ drop table td, ts; eval drop server $srv; set session spider_bgs_mode=@old_spider_bgs_mode; +set global spider_same_server_link=@old_spider_same_server_link; --disable_query_log --disable_result_log diff --git a/storage/spider/mysql-test/spider/bugfix/t/mdev_26247.test b/storage/spider/mysql-test/spider/bugfix/t/mdev_26247.test index 8d27c6e0..80328e05 100644 --- a/storage/spider/mysql-test/spider/bugfix/t/mdev_26247.test +++ b/storage/spider/mysql-test/spider/bugfix/t/mdev_26247.test @@ -7,6 +7,7 @@ --source ../../t/test_init.inc --enable_result_log --enable_query_log + --connection child2_1 CREATE DATABASE auto_test_remote; USE auto_test_remote; @@ -33,10 +34,6 @@ INSERT INTO t2 VALUES (1, 11), (2, 22), (3, 33); INSERT INTO t3 VALUES (1, 111), (2, 222), (3, 333); --connection master_1 -#FIXME: this is a workaround. -set @old_spider_disable_group_by_handler=@@spider_disable_group_by_handler; -set spider_disable_group_by_handler=1; - CREATE DATABASE auto_test_local; USE auto_test_local; @@ -71,7 +68,6 @@ SELECT a22 FROM t1 RIGHT JOIN t2 ON a21 = a11 WHERE a11 IN (1,2); --connection master_1 DROP DATABASE IF EXISTS auto_test_local; -set spider_disable_group_by_handler=@old_spider_disable_group_by_handler; --connection child2_1 DROP DATABASE IF EXISTS auto_test_remote; @@ -80,6 +76,3 @@ DROP DATABASE IF EXISTS auto_test_remote; --source ../t/test_deinit.inc --enable_query_log --enable_result_log ---echo # ---echo # end of test mdev_26247 ---echo # diff --git a/storage/spider/mysql-test/spider/bugfix/t/mdev_26541.test b/storage/spider/mysql-test/spider/bugfix/t/mdev_26541.test index bf6cb255..add5f621 100644 --- a/storage/spider/mysql-test/spider/bugfix/t/mdev_26541.test +++ b/storage/spider/mysql-test/spider/bugfix/t/mdev_26541.test @@ -2,10 +2,7 @@ --echo # MDEV-26541 Undefined symbol: _ZTI12ha_partition when attempting to use ha_spider.so in UBSAN builds --echo # -if (`select not(count(*)) from information_schema.system_variables where variable_name='have_sanitizer' and global_value like "%UBSAN%"`) -{ ---skip test needs to be run with UBSAN -} +# this test should be checked with ubsan # init spider @@ -20,4 +17,5 @@ while (!$PLUGIN_EXIST) `SELECT COUNT(*) FROM mysql.func WHERE name = '$PLUGIN_NAME'`; } +--disable_query_log --source ../../include/clean_up_spider.inc diff --git a/storage/spider/mysql-test/spider/bugfix/t/mdev_27575.test b/storage/spider/mysql-test/spider/bugfix/t/mdev_27575.test new file mode 100644 index 00000000..79a08489 --- /dev/null +++ b/storage/spider/mysql-test/spider/bugfix/t/mdev_27575.test @@ -0,0 +1,23 @@ +--echo # +--echo # MDEV-27575 Spider: UBSAN member access within null pointer of type 'struct st_plugin_int and SIGSEGV in intern_plugin_lock on SHUTDOWN when setting Spider as default storage engine (temporary or global) +--echo # +--disable_query_log +--disable_result_log +--source ../../t/test_init.inc +--enable_result_log +--enable_query_log + +SET GLOBAL default_tmp_storage_engine=spider; +--source include/restart_mysqld.inc + +SET GLOBAL default_storage_engine=Spider; +--source include/restart_mysqld.inc + +--disable_query_log +--disable_result_log +--source ../../t/test_deinit.inc +--enable_result_log +--enable_query_log +--echo # +--echo # end of test mdev_27575 +--echo # diff --git a/storage/spider/mysql-test/spider/bugfix/t/mdev_28683.test b/storage/spider/mysql-test/spider/bugfix/t/mdev_28683.test new file mode 100644 index 00000000..0508a999 --- /dev/null +++ b/storage/spider/mysql-test/spider/bugfix/t/mdev_28683.test @@ -0,0 +1,29 @@ +--echo # +--echo # MDEV-28683 Spider: SIGSEGV in spider_db_direct_delete, SIGSEGV in spider_db_connect, ASAN: heap-use-after-free in spider_db_direct_delete +--echo # +--disable_query_log +--disable_result_log +--source ../../t/test_init.inc +--enable_result_log +--enable_query_log + +CREATE TABLE t (c INT) ENGINE=Spider; +--error ER_CONNECT_TO_FOREIGN_DATA_SOURCE +SELECT * FROM t; +--error ER_CONNECT_TO_FOREIGN_DATA_SOURCE +INSERT INTO t (SELECT 1 FROM t); +LOCK TABLES t WRITE CONCURRENT; +--error ER_CONNECT_TO_FOREIGN_DATA_SOURCE +DELETE FROM t; + +UNLOCK TABLES; +DROP TABLE t; + +--disable_query_log +--disable_result_log +--source ../../t/test_deinit.inc +--enable_result_log +--enable_query_log +--echo # +--echo # end of test mdev_28683 +--echo # diff --git a/storage/spider/mysql-test/spider/bugfix/t/mdev_28739.cnf b/storage/spider/mysql-test/spider/bugfix/t/mdev_28739.cnf new file mode 100644 index 00000000..05dfd8a0 --- /dev/null +++ b/storage/spider/mysql-test/spider/bugfix/t/mdev_28739.cnf @@ -0,0 +1,3 @@ +!include include/default_mysqld.cnf +!include ../my_1_1.cnf +!include ../my_2_1.cnf diff --git a/storage/spider/mysql-test/spider/bugfix/t/mdev_28739.test b/storage/spider/mysql-test/spider/bugfix/t/mdev_28739.test new file mode 100644 index 00000000..d4ac8e42 --- /dev/null +++ b/storage/spider/mysql-test/spider/bugfix/t/mdev_28739.test @@ -0,0 +1,41 @@ +--echo # +--echo # MDEV-28739 Trying to lock uninitialized mutex or hang upon shutdown after using Spider with query_cache +--echo # + +--disable_query_log +--disable_result_log +--source ../../t/test_init.inc +--enable_result_log +--enable_query_log + +--connection child2_1 +CREATE DATABASE auto_test_remote; +USE auto_test_remote; + +CREATE TABLE tbl_a (id INT); + +--connection master_1 +CREATE DATABASE auto_test_local; +USE auto_test_local; + +set global query_cache_type= on; +set spider_same_server_link = on; + +eval CREATE TABLE tbl_a ( + id INT +) $MASTER_1_ENGINE $MASTER_1_CHARSET COMMENT='table "tbl_a", srv "s_2_1"'; + +SELECT * FROM tbl_a; + +--source include/restart_mysqld.inc + +--connection master_1 +DROP DATABASE IF EXISTS auto_test_local; +--connection child2_1 +DROP DATABASE IF EXISTS auto_test_remote; + +--disable_query_log +--disable_result_log +--source ../t/test_deinit.inc +--enable_query_log +--enable_result_log diff --git a/storage/spider/mysql-test/spider/bugfix/t/mdev_28739_simple.test b/storage/spider/mysql-test/spider/bugfix/t/mdev_28739_simple.test new file mode 100644 index 00000000..feff85df --- /dev/null +++ b/storage/spider/mysql-test/spider/bugfix/t/mdev_28739_simple.test @@ -0,0 +1,31 @@ +--echo # +--echo # MDEV-28739 Trying to lock uninitialized mutex or hang upon shutdown after using Spider with query_cache +--echo # + +--disable_query_log +--disable_result_log +--source ../../t/test_init.inc +--enable_result_log +--enable_query_log + +# set @@global.debug_dbug="+d,xid_thd_trace,enter,exit,info,error:o,/tmp/trace2.out:i:F:L"; +#set @@global.debug_dbug="d:t:i:o,mysqld.trace"; + +set global query_cache_type= on; +set spider_same_server_link=1; +evalp CREATE SERVER srv FOREIGN DATA WRAPPER mysql +OPTIONS (SOCKET "$MASTER_1_MYSOCK", DATABASE 'test',user 'root'); +create table t2 (c int); +create table t1 (c int) ENGINE=Spider +COMMENT='WRAPPER "mysql", srv "srv",TABLE "t2"'; +SELECT * FROM t1; +#shutdown; +--source include/restart_mysqld.inc +drop table t1, t2; +drop server srv; + +--disable_query_log +--disable_result_log +--source ../t/test_deinit.inc +--enable_query_log +--enable_result_log diff --git a/storage/spider/mysql-test/spider/bugfix/t/mdev_28856.test b/storage/spider/mysql-test/spider/bugfix/t/mdev_28856.test index 4f23168e..a1642f7a 100644 --- a/storage/spider/mysql-test/spider/bugfix/t/mdev_28856.test +++ b/storage/spider/mysql-test/spider/bugfix/t/mdev_28856.test @@ -9,6 +9,7 @@ # This test covers some table params under consideration for inclusion # in the engine-defined options to be implemented in MDEV-28856. +set spider_same_server_link=1; evalp CREATE SERVER srv FOREIGN DATA WRAPPER mysql OPTIONS (SOCKET "$MASTER_1_MYSOCK", DATABASE 'test',user 'root'); diff --git a/storage/spider/mysql-test/spider/bugfix/t/mdev_28998.test b/storage/spider/mysql-test/spider/bugfix/t/mdev_28998.test index d67011ca..51d4c5c9 100644 --- a/storage/spider/mysql-test/spider/bugfix/t/mdev_28998.test +++ b/storage/spider/mysql-test/spider/bugfix/t/mdev_28998.test @@ -2,19 +2,14 @@ --echo # MDEV-28998 ASAN errors in spider_fields::free_conn_holder or spider_create_group_by_handler --echo # -if (`select not(count(*)) from information_schema.system_variables where variable_name='have_sanitizer' and global_value like "%ASAN%"`) -{ ---skip test needs to be run with ASAN -} +# this test should be checked with ubsan --disable_query_log --disable_result_log --source ../../t/test_init.inc --enable_result_log --enable_query_log -# FIXME: this is a workaround. -set @old_spider_disable_group_by_handler=@@spider_disable_group_by_handler; -set spider_disable_group_by_handler=1; +set spider_same_server_link=1; evalp CREATE SERVER s FOREIGN DATA WRAPPER MYSQL OPTIONS (SOCKET "$MASTER_1_MYSOCK", DATABASE 'test',user 'root'); CREATE TABLE t1 (a INT); @@ -27,7 +22,6 @@ SELECT * FROM t2; # Cleanup DROP TABLE t2, t1_SPIDER, t1; drop server s; -set spider_disable_group_by_handler=@old_spider_disable_group_by_handler; --disable_query_log --disable_result_log diff --git a/storage/spider/mysql-test/spider/bugfix/t/mdev_29002.test b/storage/spider/mysql-test/spider/bugfix/t/mdev_29002.test new file mode 100644 index 00000000..51620a5a --- /dev/null +++ b/storage/spider/mysql-test/spider/bugfix/t/mdev_29002.test @@ -0,0 +1,32 @@ +--disable_query_log +--disable_result_log +--source ../../t/test_init.inc +--enable_result_log +--enable_query_log +SET spider_same_server_link= on; +evalp CREATE SERVER s FOREIGN DATA WRAPPER mysql +OPTIONS (SOCKET "$MASTER_1_MYSOCK", DATABASE 'test',user 'root'); + +CREATE TABLE t (a INT); +CREATE TABLE t1_spider (a INT) ENGINE=SPIDER COMMENT = "wrapper 'mysql', srv 's', table 't'"; +CREATE TABLE t2_spider (a INT) ENGINE=SPIDER COMMENT = "wrapper 'mysql', srv 's', table 't'"; +SELECT * FROM t1_spider, t2_spider; +SELECT table_name, index_name, cardinality FROM INFORMATION_SCHEMA.STATISTICS WHERE table_name IN ('t1_spider','t2_spider'); +RENAME TABLE t1_spider TO t3_spider; +SELECT * FROM t3_spider; + +DROP TABLE t3_spider, t2_spider, t; +drop server s; + +# case by roel +CREATE TABLE t1 (c INT) ENGINE=Spider COMMENT='WRAPPER "mysql",HOST "srv",TABLE "t"'; +CREATE TABLE t2 (c INT) ENGINE=Spider COMMENT='WRAPPER "mysql",HOST "srv",TABLE "t"'; +CREATE TABLE t3 (c INT) ENGINE=Spider COMMENT='WRAPPER "mysql",HOST "srv",TABLE "t"'; +SHOW TABLE STATUS; +drop table t1, t2, t3; + +--disable_query_log +--disable_result_log +--source ../../t/test_deinit.inc +--enable_result_log +--enable_query_log diff --git a/storage/spider/mysql-test/spider/bugfix/t/mdev_29163.test b/storage/spider/mysql-test/spider/bugfix/t/mdev_29163.test index 0bfcc46f..2e56583d 100644 --- a/storage/spider/mysql-test/spider/bugfix/t/mdev_29163.test +++ b/storage/spider/mysql-test/spider/bugfix/t/mdev_29163.test @@ -6,9 +6,7 @@ --source ../../t/test_init.inc --enable_result_log --enable_query_log -#FIXME: this is a workaround. -set @old_spider_disable_group_by_handler=@@spider_disable_group_by_handler; -set spider_disable_group_by_handler=1; +set spider_same_server_link=1; evalp CREATE SERVER s FOREIGN DATA WRAPPER MYSQL OPTIONS (SOCKET "$MASTER_1_MYSOCK", DATABASE 'test',user 'root'); CREATE TABLE t1 (a INT); @@ -24,8 +22,6 @@ SELECT t1_spider.* FROM t1_spider LEFT JOIN t2_spider LEFT JOIN t3_spider ON b = # Cleanup DROP TABLE t1_spider, t2_spider, t3_spider, t1, t2, t3; drop server s; -set spider_disable_group_by_handler=@old_spider_disable_group_by_handler; - --disable_query_log --disable_result_log --source ../../t/test_deinit.inc diff --git a/storage/spider/mysql-test/spider/bugfix/t/mdev_29421.test b/storage/spider/mysql-test/spider/bugfix/t/mdev_29421.test new file mode 100644 index 00000000..47ad42d3 --- /dev/null +++ b/storage/spider/mysql-test/spider/bugfix/t/mdev_29421.test @@ -0,0 +1,26 @@ +--disable_query_log +--disable_result_log +--source ../../t/test_init.inc +--enable_result_log +--enable_query_log +set @old_table_open_cache=@@global.table_open_cache; +set global table_open_cache=10; +evalp CREATE SERVER srv FOREIGN DATA WRAPPER mysql +OPTIONS (SOCKET "$MASTER_1_MYSOCK", DATABASE 'test',user 'root'); +CREATE TABLE t (c INT) ENGINE=InnoDB; +CREATE TABLE t_s (c INT) ENGINE=Spider COMMENT='WRAPPER "mysql", srv "srv", TABLE "t"'; +CREATE TABLE t1 (a INT) ENGINE=Spider; +--error 1429 +SELECT * FROM t1; +--disable_result_log +SELECT * FROM information_schema.tables; +--enable_result_log +DROP TABLE t, t_s, t1; +drop server srv; +set global table_open_cache=@old_table_open_cache; + +--disable_query_log +--disable_result_log +--source ../../t/test_deinit.inc +--enable_result_log +--enable_query_log diff --git a/storage/spider/mysql-test/spider/bugfix/t/mdev_29456.test b/storage/spider/mysql-test/spider/bugfix/t/mdev_29456.test new file mode 100644 index 00000000..89d53227 --- /dev/null +++ b/storage/spider/mysql-test/spider/bugfix/t/mdev_29456.test @@ -0,0 +1,53 @@ +--echo # +--echo # MDEV-29456 Spider hangs in 'Waiting for table metadata lock' state on CREATE TABLE after LOCK TABLES +--echo # +--source include/have_innodb.inc +--disable_query_log +--disable_result_log +--source ../../t/test_init.inc +--enable_result_log +--enable_query_log + +set spider_same_server_link=1; +evalp CREATE SERVER srv FOREIGN DATA WRAPPER mysql +OPTIONS (SOCKET "$MASTER_1_MYSOCK", DATABASE 'test',user 'root'); + +create table t1 (c int); +create table t2 (c int) ENGINE=Spider +COMMENT='WRAPPER "mysql", srv "srv",TABLE "t1"'; +CREATE TABLE t3 (c INT KEY) ENGINE=Spider; +LOCK TABLE t2 WRITE; +--error ER_CONNECT_TO_FOREIGN_DATA_SOURCE +LOCK TABLE t3 WRITE; +UNLOCK TABLES; +drop table t1, t2, t3; + +CREATE TABLE t (c INT) ENGINE=InnoDB; +CREATE TABLE t1 (c INT) ENGINE=Spider; +CREATE TABLE t2 (c INT) ENGINE=Spider COMMENT='WRAPPER "mysql",srv "srv",TABLE "t"'; +LOCK TABLES t2 WRITE; +--error ER_CONNECT_TO_FOREIGN_DATA_SOURCE +LOCK TABLES t1 READ; +--error ER_TABLE_EXISTS_ERROR +CREATE TABLE t (c INT) ENGINE=Spider; +drop table t, t1, t2; + +# MDEV-30049 +CREATE TABLE t (c INT) ENGINE=InnoDB; +CREATE TABLE t2 (c INT) ENGINE=Spider COMMENT='WRAPPER "mysql",srv "srv",TABLE "t"'; +CREATE TABLE t3 (c INT) ENGINE=InnoDB; +LOCK TABLES t2 WRITE; +--error 1428 +LOCK TABLES mysql.proc WRITE,mysql.user WRITE; # ERROR 1428 (HY000): You can't combine write-locking of system tables with other tables or lock types +INSERT INTO t3 SELECT * FROM t; +drop table t, t2, t3; + +drop server srv; +--disable_query_log +--disable_result_log +--source ../../t/test_deinit.inc +--enable_result_log +--enable_query_log +--echo # +--echo # end of test mdev_29456 +--echo # diff --git a/storage/spider/mysql-test/spider/bugfix/t/mdev_29502.test b/storage/spider/mysql-test/spider/bugfix/t/mdev_29502.test index 88591f80..bacf5073 100644 --- a/storage/spider/mysql-test/spider/bugfix/t/mdev_29502.test +++ b/storage/spider/mysql-test/spider/bugfix/t/mdev_29502.test @@ -8,6 +8,7 @@ --enable_result_log --enable_query_log +set spider_same_server_link=1; --let $srv=srv_mdev_29502 evalp CREATE SERVER $srv FOREIGN DATA WRAPPER MYSQL OPTIONS (SOCKET "$MASTER_1_MYSOCK", DATABASE 'test',user 'root'); diff --git a/storage/spider/mysql-test/spider/bugfix/t/mdev_29667.cnf b/storage/spider/mysql-test/spider/bugfix/t/mdev_29667.cnf new file mode 100644 index 00000000..05dfd8a0 --- /dev/null +++ b/storage/spider/mysql-test/spider/bugfix/t/mdev_29667.cnf @@ -0,0 +1,3 @@ +!include include/default_mysqld.cnf +!include ../my_1_1.cnf +!include ../my_2_1.cnf diff --git a/storage/spider/mysql-test/spider/bugfix/t/mdev_29667.test b/storage/spider/mysql-test/spider/bugfix/t/mdev_29667.test new file mode 100644 index 00000000..c96eb60e --- /dev/null +++ b/storage/spider/mysql-test/spider/bugfix/t/mdev_29667.test @@ -0,0 +1,44 @@ +--echo # +--echo # MDEV-29667 Server hangs on DROP DATABASE after failing LOCK TABLES on Spider table +--echo # +--disable_query_log +--disable_result_log +--source ../../t/test_init.inc +--enable_result_log +--enable_query_log + +--connection child2_1 +CREATE DATABASE auto_test_remote; +USE auto_test_remote; +eval CREATE TABLE tbl_a ( + a INT +) $CHILD2_1_ENGINE $CHILD2_1_CHARSET; + +--connection master_1 +CREATE DATABASE auto_test_local; +USE auto_test_local; +eval CREATE TABLE tbl_a ( + a INT +) $MASTER_1_ENGINE $MASTER_1_CHARSET COMMENT='table "tbl_a", srv "s_2_1"'; +eval CREATE TABLE tbl_b ( + a INT +) $MASTER_1_ENGINE $MASTER_1_CHARSET; + +LOCK TABLES tbl_a WRITE; +--error 1429 +LOCK TABLES tbl_b READ, tbl_a READ; + +--connection master_1 +DROP DATABASE IF EXISTS auto_test_local; + +--connection child2_1 +DROP DATABASE IF EXISTS auto_test_remote; + +--disable_query_log +--disable_result_log +--source ../t/test_deinit.inc +--enable_query_log +--enable_result_log +--echo # +--echo # end of test mdev_29667 +--echo # diff --git a/storage/spider/mysql-test/spider/bugfix/t/mdev_29963.test b/storage/spider/mysql-test/spider/bugfix/t/mdev_29963.test new file mode 100644 index 00000000..93b38c79 --- /dev/null +++ b/storage/spider/mysql-test/spider/bugfix/t/mdev_29963.test @@ -0,0 +1,56 @@ +--echo # +--echo # MDEV-29963 SIGSEGV in spider_db_mbase::append_lock_tables on LOCK TABLES +--echo # +--disable_query_log +--disable_result_log +--source ../../t/test_init.inc +--enable_result_log +--enable_query_log + +set spider_same_server_link=1; +evalp CREATE SERVER srv FOREIGN DATA WRAPPER mysql +OPTIONS (SOCKET "$MASTER_1_MYSOCK", DATABASE 'test',user 'root'); + +CREATE TABLE t (a INT) ENGINE=Spider; +CREATE TABLE t2 (b INT) ENGINE=Spider COMMENT='WRAPPER "mysql",srv "srv",TABLE "t"'; +--error ER_CONNECT_TO_FOREIGN_DATA_SOURCE +LOCK TABLES t AS a READ,t2 AS b LOW_PRIORITY WRITE,t2 AS c WRITE; +DROP TABLE t2; +CREATE TABLE t2 (c INT) ENGINE=Spider COMMENT='WRAPPER "mysql",srv "srv",TABLE "t"'; +--error ER_CONNECT_TO_FOREIGN_DATA_SOURCE +LOCK TABLES t2 WRITE; +DROP TABLE t2,t; + +# A less complex scenario +CREATE TABLE t (a INT); +CREATE TABLE t1 (a INT) ENGINE=Spider; +CREATE TABLE t2 (b INT) ENGINE=Spider COMMENT='WRAPPER "mysql",srv "srv",TABLE "t"'; +--error ER_CONNECT_TO_FOREIGN_DATA_SOURCE +LOCK TABLES t1 READ, t2 WRITE; +DROP TABLE t2; +CREATE TABLE t2 (c INT) ENGINE=Spider COMMENT='WRAPPER "mysql",srv "srv",TABLE "t"'; +LOCK TABLES t2 WRITE; +UNLOCK TABLES; +DROP TABLE t, t1, t2; + +# MDEV-31357 +CREATE TABLE t1 (c INT) ENGINE=Spider; +CREATE TABLE t2 (c INT) ENGINE=Spider COMMENT="WRAPPER 'mysql',srv 'srv',TABLE 't1'"; +--error ER_CONNECT_TO_FOREIGN_DATA_SOURCE +LOCK TABLES t1 WRITE,t2 WRITE; +--error ER_CONNECT_TO_FOREIGN_DATA_SOURCE +TRUNCATE t2; +--error ER_CONNECT_TO_FOREIGN_DATA_SOURCE +LOCK TABLES t2 AS o WRITE; +drop table t1, t2; + +drop server srv; + +--disable_query_log +--disable_result_log +--source ../../t/test_deinit.inc +--enable_result_log +--enable_query_log +--echo # +--echo # end of test mdev_29963 +--echo # diff --git a/storage/spider/mysql-test/spider/bugfix/t/mdev_30014.test b/storage/spider/mysql-test/spider/bugfix/t/mdev_30014.test new file mode 100644 index 00000000..9c59adc8 --- /dev/null +++ b/storage/spider/mysql-test/spider/bugfix/t/mdev_30014.test @@ -0,0 +1,46 @@ +--echo # +--echo # MDEV-30014 heap-use-after-free in ha_spider::lock_tables(), highly sporadic SIGSEGV in intern_close_table +--echo # +--disable_query_log +--disable_result_log +--source ../../t/test_init.inc +--enable_result_log +--enable_query_log + +set spider_same_server_link=1; +evalp CREATE SERVER srv FOREIGN DATA WRAPPER mysql +OPTIONS (SOCKET "$MASTER_1_MYSOCK", DATABASE 'test',user 'root'); + +create table t1 (c int); +create table t2 (c int) ENGINE=Spider +COMMENT='WRAPPER "mysql", srv "srv",TABLE "t1"'; +CREATE TABLE t3 (c INT KEY) ENGINE=Spider; +--error ER_CONNECT_TO_FOREIGN_DATA_SOURCE +LOCK TABLE t2 WRITE,t3 WRITE; +CREATE TABLE t4 (c INT) ENGINE=Spider; +FLUSH NO_WRITE_TO_BINLOG TABLES t4 WITH READ LOCK; +UNLOCK TABLES; + +drop table t1, t2, t3, t4; + +# This is a test case in MDEV-29456 but it is more like the above +# case. +create table t1 (c int); +create table t2 (c int) ENGINE=Spider +COMMENT='WRAPPER "mysql", srv "srv",TABLE "t1"'; +CREATE TABLE t3 (c INT KEY) ENGINE=Spider; +--error ER_CONNECT_TO_FOREIGN_DATA_SOURCE +LOCK TABLE t2 WRITE, t3 WRITE; +UNLOCK TABLES; +drop table t1, t2, t3; + +drop server srv; + +--disable_query_log +--disable_result_log +--source ../../t/test_deinit.inc +--enable_result_log +--enable_query_log +--echo # +--echo # end of test mdev_30014 +--echo # diff --git a/storage/spider/mysql-test/spider/bugfix/t/mdev_30170.test b/storage/spider/mysql-test/spider/bugfix/t/mdev_30170.test new file mode 100644 index 00000000..69026843 --- /dev/null +++ b/storage/spider/mysql-test/spider/bugfix/t/mdev_30170.test @@ -0,0 +1,8 @@ +install soname 'ha_spider'; +--error ER_BAD_TABLE_ERROR +DROP TABLE non_existing_table; +# Test that create or replace a non existing spider table work +create or replace table non_existing_table (c int) engine=Spider; +drop table non_existing_table; +--disable_query_log +--source ../../include/clean_up_spider.inc diff --git a/storage/spider/mysql-test/spider/bugfix/t/mdev_30392.test b/storage/spider/mysql-test/spider/bugfix/t/mdev_30392.test index 6d14f155..36e06f3f 100644 --- a/storage/spider/mysql-test/spider/bugfix/t/mdev_30392.test +++ b/storage/spider/mysql-test/spider/bugfix/t/mdev_30392.test @@ -6,9 +6,7 @@ --source ../../t/test_init.inc --enable_result_log --enable_query_log -set @old_spider_disable_group_by_handler=@@spider_disable_group_by_handler; -set spider_disable_group_by_handler=1; - +set spider_same_server_link=1; evalp CREATE SERVER srv FOREIGN DATA WRAPPER MYSQL OPTIONS (SOCKET "$MASTER_1_MYSOCK", DATABASE 'test',user 'root'); CREATE TABLE t1 (a INT); @@ -21,7 +19,6 @@ SELECT a FROM t2 WHERE a IN ( SELECT a FROM t2 ); DROP TABLE t1, t2; DROP SERVER srv; -set spider_disable_group_by_handler=@old_spider_disable_group_by_handler; --disable_query_log --disable_result_log --source ../../t/test_deinit.inc diff --git a/storage/spider/mysql-test/spider/bugfix/t/mdev_30981.test b/storage/spider/mysql-test/spider/bugfix/t/mdev_30981.test index cc24ce82..ca3f000c 100644 --- a/storage/spider/mysql-test/spider/bugfix/t/mdev_30981.test +++ b/storage/spider/mysql-test/spider/bugfix/t/mdev_30981.test @@ -2,21 +2,18 @@ --echo # MDEV-30981 Spider UBSAN: null pointer passed as argument 2, which is declared to never be null in spider_create_trx_alter_table on ALTER --echo # -if (`select not(count(*)) from information_schema.system_variables where variable_name='have_sanitizer' and global_value like "%UBSAN%"`) -{ ---skip test needs to be run with UBSAN -} - +# this test should be checked with ubsan + --disable_query_log --disable_result_log --source ../../t/test_init.inc --enable_result_log --enable_query_log - + CREATE TABLE t (c INT) ENGINE=Spider PARTITION BY LIST (c) (PARTITION p VALUES IN (1,2)); ALTER TABLE t ENGINE=InnoDB; drop table t; - + --disable_query_log --disable_result_log --source ../t/test_deinit.inc diff --git a/storage/spider/mysql-test/spider/bugfix/t/mdev_31338.test b/storage/spider/mysql-test/spider/bugfix/t/mdev_31338.test index e628c3b9..a3698c97 100644 --- a/storage/spider/mysql-test/spider/bugfix/t/mdev_31338.test +++ b/storage/spider/mysql-test/spider/bugfix/t/mdev_31338.test @@ -9,6 +9,7 @@ --enable_result_log --enable_query_log +set spider_same_server_link=1; evalp CREATE SERVER srv FOREIGN DATA WRAPPER MYSQL OPTIONS (SOCKET "$MASTER_1_MYSOCK", DATABASE 'test',user 'root'); CREATE TABLE t (c BLOB) ENGINE=InnoDB; CREATE TABLE ts (c BLOB) ENGINE=Spider COMMENT='WRAPPER "mysql",srv "srv",TABLE "t"'; diff --git a/storage/spider/mysql-test/spider/bugfix/t/mdev_31524.test b/storage/spider/mysql-test/spider/bugfix/t/mdev_31524.test index 64cbf415..a5942fad 100644 --- a/storage/spider/mysql-test/spider/bugfix/t/mdev_31524.test +++ b/storage/spider/mysql-test/spider/bugfix/t/mdev_31524.test @@ -10,6 +10,7 @@ --let $srv=srv_mdev_31524 SET @old_spider_read_only_mode = @@session.spider_read_only_mode; +set spider_same_server_link=1; evalp CREATE SERVER $srv FOREIGN DATA WRAPPER MYSQL OPTIONS (SOCKET "$MASTER_1_MYSOCK", DATABASE 'test',user 'root'); # when the user does not set var nor the table option, the default diff --git a/storage/spider/mysql-test/spider/bugfix/t/mdev_31645.test b/storage/spider/mysql-test/spider/bugfix/t/mdev_31645.test index 48bfa740..4dfe3b57 100644 --- a/storage/spider/mysql-test/spider/bugfix/t/mdev_31645.test +++ b/storage/spider/mysql-test/spider/bugfix/t/mdev_31645.test @@ -6,25 +6,17 @@ --source ../../t/test_init.inc --enable_result_log --enable_query_log -set @old_spider_disable_group_by_handler=@@spider_disable_group_by_handler; -set spider_disable_group_by_handler=1; - +set spider_same_server_link=1; evalp CREATE SERVER srv FOREIGN DATA WRAPPER MYSQL OPTIONS (SOCKET "$MASTER_1_MYSOCK", DATABASE 'test',user 'root'); CREATE TABLE t1 ( a bigint(20) NOT NULL, b bigint(20) DEFAULT 0, PRIMARY KEY (a)); CREATE TABLE t2 ( a bigint(20) NOT NULL, b bigint(20) DEFAULT 0, PRIMARY KEY (a)) ENGINE=SPIDER COMMENT='srv "srv", WRAPPER "mysql", TABLE "t1"'; -SET SESSION optimizer_switch='semijoin=off'; -SELECT * FROM t2 -WHERE A BETWEEN 0 AND 10 AND B IN(SELECT B FROM t2 WHERE A BETWEEN 11 AND 20); - -SET SESSION optimizer_switch='semijoin=on'; - +SELECT * FROM t2 WHERE b IN (SELECT b FROM t2 WHERE a > 10); SELECT * FROM t2 WHERE A BETWEEN 0 AND 10 AND B IN(SELECT B FROM t2 WHERE A BETWEEN 11 AND 20); drop table t1, t2; drop server srv; -set spider_disable_group_by_handler=@old_spider_disable_group_by_handler; --disable_query_log --disable_result_log --source ../../t/test_deinit.inc diff --git a/storage/spider/mysql-test/spider/bugfix/t/mdev_31996.test b/storage/spider/mysql-test/spider/bugfix/t/mdev_31996.test index 3e823790..93b004a0 100644 --- a/storage/spider/mysql-test/spider/bugfix/t/mdev_31996.test +++ b/storage/spider/mysql-test/spider/bugfix/t/mdev_31996.test @@ -4,6 +4,7 @@ --enable_result_log --enable_query_log +set spider_same_server_link=1; evalp CREATE SERVER srv FOREIGN DATA WRAPPER mysql OPTIONS (SOCKET "$MASTER_1_MYSOCK", DATABASE 'test',user 'root'); diff --git a/storage/spider/mysql-test/spider/bugfix/t/mdev_32753.opt b/storage/spider/mysql-test/spider/bugfix/t/mdev_32753.opt new file mode 100644 index 00000000..c3151b0e --- /dev/null +++ b/storage/spider/mysql-test/spider/bugfix/t/mdev_32753.opt @@ -0,0 +1,2 @@ +--sql-mode=oracle +--plugin-load-add=ha_spider diff --git a/storage/spider/mysql-test/spider/bugfix/t/mdev_32753.test b/storage/spider/mysql-test/spider/bugfix/t/mdev_32753.test new file mode 100644 index 00000000..2be7289f --- /dev/null +++ b/storage/spider/mysql-test/spider/bugfix/t/mdev_32753.test @@ -0,0 +1,12 @@ +--echo # +--echo # MDEV-32753 Spider engine does not load in ORACLE mode +--echo # + +# This test tests spider init during server startup under global +# ORACLE mode +select * from mysql.plugin; +create table t (c int) Engine=SPIDER; +drop table t; +--echo # +--echo # end of test mdev_32753 +--echo # diff --git a/storage/spider/mysql-test/spider/bugfix/t/mdev_32753_after_start.opt b/storage/spider/mysql-test/spider/bugfix/t/mdev_32753_after_start.opt new file mode 100644 index 00000000..a918abb9 --- /dev/null +++ b/storage/spider/mysql-test/spider/bugfix/t/mdev_32753_after_start.opt @@ -0,0 +1 @@ +--sql-mode=oracle diff --git a/storage/spider/mysql-test/spider/bugfix/t/mdev_32753_after_start.test b/storage/spider/mysql-test/spider/bugfix/t/mdev_32753_after_start.test new file mode 100644 index 00000000..de2ab789 --- /dev/null +++ b/storage/spider/mysql-test/spider/bugfix/t/mdev_32753_after_start.test @@ -0,0 +1,20 @@ +--echo # +--echo # MDEV-32753 Spider engine does not load in ORACLE mode +--echo # + +# This test tests spider init after startup under global ORACLE mode +install soname 'ha_spider'; +--replace_regex /\.dll/.so/ +select * from mysql.plugin; +create table t (c int) Engine=SPIDER; +drop table t; + +--disable_query_log +--disable_result_log +--source ../../include/clean_up_spider.inc +--enable_result_log +--enable_query_log + +--echo # +--echo # end of test mdev_32753_after_start +--echo # diff --git a/storage/spider/mysql-test/spider/bugfix/t/mdev_32753_after_start_session.test b/storage/spider/mysql-test/spider/bugfix/t/mdev_32753_after_start_session.test new file mode 100644 index 00000000..bf7bdb4f --- /dev/null +++ b/storage/spider/mysql-test/spider/bugfix/t/mdev_32753_after_start_session.test @@ -0,0 +1,22 @@ +--echo # +--echo # MDEV-32753 Spider engine does not load in ORACLE mode +--echo # + +# This test tests spider init after startup under session ORACLE mode +set @old_sql_mode=@@sql_mode; +SET @@sql_mode = CONCAT(@@sql_mode, ',ORACLE'); +install soname 'ha_spider'; +select * from mysql.plugin; +create table t (c int) Engine=SPIDER; +drop table t; +set sql_mode=@old_sql_mode; + +--disable_query_log +--disable_result_log +--source ../../include/clean_up_spider.inc +--enable_result_log +--enable_query_log + +--echo # +--echo # end of test mdev_32753_after_start +--echo # diff --git a/storage/spider/mysql-test/spider/bugfix/t/mdev_32986.test b/storage/spider/mysql-test/spider/bugfix/t/mdev_32986.test new file mode 100644 index 00000000..14438745 --- /dev/null +++ b/storage/spider/mysql-test/spider/bugfix/t/mdev_32986.test @@ -0,0 +1,29 @@ +--echo # +--echo # MDEV-32907 +--echo # +--disable_query_log +--disable_result_log +--source ../../t/test_init.inc +--enable_result_log +--enable_query_log +set spider_same_server_link=1; +evalp CREATE SERVER srv FOREIGN DATA WRAPPER mysql +OPTIONS (SOCKET "$MASTER_1_MYSOCK", DATABASE 'test',user 'root'); + +create table t2 (c varchar(16)); +create table t1 (c varchar(16)) ENGINE=Spider +COMMENT='WRAPPER "mysql", srv "srv",TABLE "t2"'; + +insert into t1 values ('TestSpiderRegex'); +select c from t1 where c regexp '(Test|Spider|Regex)'; +drop table t1, t2; +drop server srv; + +--disable_query_log +--disable_result_log +--source ../../t/test_deinit.inc +--enable_result_log +--enable_query_log +--echo # +--echo # end of test mdev_32907 +--echo # diff --git a/storage/spider/mysql-test/spider/bugfix/t/mdev_33008.test b/storage/spider/mysql-test/spider/bugfix/t/mdev_33008.test new file mode 100644 index 00000000..48d9a4f0 --- /dev/null +++ b/storage/spider/mysql-test/spider/bugfix/t/mdev_33008.test @@ -0,0 +1,24 @@ +--disable_query_log +--disable_result_log +--source ../../t/test_init.inc +--enable_result_log +--enable_query_log +set spider_same_server_link=on; +evalp CREATE SERVER srv FOREIGN DATA WRAPPER mysql +OPTIONS (SOCKET "$MASTER_1_MYSOCK", DATABASE 'test',user 'root'); +create table t2 ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `code` varchar(10) DEFAULT NULL, + PRIMARY KEY (`id`) +); +create table t1 ENGINE=Spider +COMMENT='WRAPPER "mysql", srv "srv",TABLE "t2"'; +show create table t1; +drop table t1, t2; + +drop server srv; +--disable_query_log +--disable_result_log +--source ../../t/test_deinit.inc +--enable_result_log +--enable_query_log diff --git a/storage/spider/mysql-test/spider/bugfix/t/mdev_33191.test b/storage/spider/mysql-test/spider/bugfix/t/mdev_33191.test new file mode 100644 index 00000000..90709127 --- /dev/null +++ b/storage/spider/mysql-test/spider/bugfix/t/mdev_33191.test @@ -0,0 +1,11 @@ +INSTALL SONAME 'ha_spider'; +set spider_same_server_link=on; +CREATE TABLE t2(c INT); +--let $SOCKET=`SELECT @@global.socket` +evalp CREATE TABLE t1(c INT) ENGINE=Spider COMMENT='socket "$SOCKET", user "root", table "t2 t3"'; +ALTER TABLE t1 ENGINE=Spider; +--error ER_NO_SUCH_TABLE +TRUNCATE TABLE t1; +drop table t1, t2; +--disable_query_log +--source ../../include/clean_up_spider.inc diff --git a/storage/spider/mysql-test/spider/bugfix/t/perfschema.opt b/storage/spider/mysql-test/spider/bugfix/t/perfschema.opt new file mode 100644 index 00000000..d2ed32dd --- /dev/null +++ b/storage/spider/mysql-test/spider/bugfix/t/perfschema.opt @@ -0,0 +1 @@ +--loose-performance-schema diff --git a/storage/spider/mysql-test/spider/bugfix/t/perfschema.test b/storage/spider/mysql-test/spider/bugfix/t/perfschema.test new file mode 100644 index 00000000..9346d2b5 --- /dev/null +++ b/storage/spider/mysql-test/spider/bugfix/t/perfschema.test @@ -0,0 +1,16 @@ +source include/have_perfschema.inc; +disable_query_log; +source ../../include/init_spider.inc; +enable_query_log; + +--echo # +--echo # MDEV-33031 Assertion failure upon reading from performance schema with binlog enabled +--echo # +connect foo,localhost,root; +select variable_name, variable_value from performance_schema.status_by_thread +where variable_name like '%spider_direct_aggregate%'; +disconnect foo; +connection default; + +disable_query_log; +source ../../include/deinit_spider.inc; diff --git a/storage/spider/mysql-test/spider/bugfix/t/self_reference_multi.test b/storage/spider/mysql-test/spider/bugfix/t/self_reference_multi.test index 2ac927d9..4263560b 100644 --- a/storage/spider/mysql-test/spider/bugfix/t/self_reference_multi.test +++ b/storage/spider/mysql-test/spider/bugfix/t/self_reference_multi.test @@ -8,6 +8,9 @@ --echo MDEV-6268 SPIDER table with no COMMENT clause causes queries to wait forever --echo +set spider_same_server_link=1; +set @old_spider_same_server_link=@@global.spider_same_server_link; +set global spider_same_server_link=1; --let $srv=srv_self_reference_multi evalp CREATE SERVER $srv FOREIGN DATA WRAPPER MYSQL OPTIONS (SOCKET "$MASTER_1_MYSOCK", DATABASE 'test',user 'root'); create table t2 (c int); @@ -22,6 +25,7 @@ select * from t1; select * from t2; drop table t0, t1, t2; eval drop server $srv; +set global spider_same_server_link=@old_spider_same_server_link; --disable_query_log --disable_result_log diff --git a/storage/spider/mysql-test/spider/bugfix/t/signal_ddl_fail.opt b/storage/spider/mysql-test/spider/bugfix/t/signal_ddl_fail.opt new file mode 100644 index 00000000..d883df7b --- /dev/null +++ b/storage/spider/mysql-test/spider/bugfix/t/signal_ddl_fail.opt @@ -0,0 +1,2 @@ +--plugin-load-add=ha_spider +--debug-dbug=d,fail_spider_ddl_recovery_done diff --git a/storage/spider/mysql-test/spider/bugfix/t/signal_ddl_fail.test b/storage/spider/mysql-test/spider/bugfix/t/signal_ddl_fail.test new file mode 100644 index 00000000..f13eae3a --- /dev/null +++ b/storage/spider/mysql-test/spider/bugfix/t/signal_ddl_fail.test @@ -0,0 +1,10 @@ +--source include/have_debug.inc +--echo # +--echo # MDEV-32559 Move alter table statements in spider init queries to be executed in the signal_ddl_recovery_done callback +--echo # +# This test tests that failure in ddl_recovery callback causes the +# plugin to be deinitialized. +select * from mysql.plugin; +--echo # +--echo # end of test signal_ddl_fail +--echo # diff --git a/storage/spider/mysql-test/spider/bugfix/t/spider_join_with_non_spider.test b/storage/spider/mysql-test/spider/bugfix/t/spider_join_with_non_spider.test new file mode 100644 index 00000000..294b469a --- /dev/null +++ b/storage/spider/mysql-test/spider/bugfix/t/spider_join_with_non_spider.test @@ -0,0 +1,29 @@ +--echo # +--echo # Test joining a spider table with a non-spider table +--echo # + +--disable_query_log +--disable_result_log +--source ../../t/test_init.inc +--enable_result_log +--enable_query_log + +set spider_same_server_link=1; +evalp CREATE SERVER srv FOREIGN DATA WRAPPER MYSQL OPTIONS (SOCKET "$MASTER_1_MYSOCK", DATABASE 'test',user 'root'); +create table t1 (c int); +create table t2 (d int); +insert into t2 values (1), (2); +create table t3 (c int) ENGINE=Spider COMMENT='WRAPPER "mysql", srv "srv",TABLE "t1"'; +insert into t3 values (2), (3); +select c from t3 join t2 on c = d; +drop table t1, t2, t3; +drop server srv; + +--disable_query_log +--disable_result_log +--source ../../t/test_deinit.inc +--enable_result_log +--enable_query_log +--echo # +--echo # end of test spider_join_with_non_spider +--echo # diff --git a/storage/spider/mysql-test/spider/bugfix/t/spider_table_sts.test b/storage/spider/mysql-test/spider/bugfix/t/spider_table_sts.test deleted file mode 100644 index 72130175..00000000 --- a/storage/spider/mysql-test/spider/bugfix/t/spider_table_sts.test +++ /dev/null @@ -1,46 +0,0 @@ ---source ../include/spider_table_sts_init.inc ---echo ---echo this test is for MDEV-19842 ---echo ---echo drop and create databases - ---connection master_1 ---disable_warnings -CREATE DATABASE auto_test_local; -USE auto_test_local; ---enable_warnings - ---echo ---echo create table - ---connection master_1 ---disable_query_log -echo CREATE TABLE tbl_a ( - pkey int NOT NULL, - PRIMARY KEY (pkey) -) MASTER_1_ENGINE MASTER_1_CHARSET MASTER_1_COMMENT_2_1; -eval CREATE TABLE tbl_a ( - pkey int NOT NULL, - PRIMARY KEY (pkey) -) $MASTER_1_ENGINE $MASTER_1_CHARSET $MASTER_1_COMMENT_2_1; ---enable_query_log - ---echo ---echo select test 1 - ---connection master_1 ---error 12609 -SELECT pkey FROM tbl_a; - ---echo ---echo deinit ---disable_warnings - ---connection master_1 ---error 12609 -DROP DATABASE IF EXISTS auto_test_local; - ---enable_warnings ---source ../include/spider_table_sts_deinit.inc ---echo ---echo end of test diff --git a/storage/spider/mysql-test/spider/bugfix/t/subquery.test b/storage/spider/mysql-test/spider/bugfix/t/subquery.test new file mode 100644 index 00000000..70238a52 --- /dev/null +++ b/storage/spider/mysql-test/spider/bugfix/t/subquery.test @@ -0,0 +1,31 @@ +--echo # +--echo # Test spider select with subqueries +--echo # +--disable_query_log +--disable_result_log +--source ../../t/test_init.inc +--enable_result_log +--enable_query_log +set spider_same_server_link=1; +evalp CREATE SERVER srv FOREIGN DATA WRAPPER MYSQL OPTIONS (SOCKET "$MASTER_1_MYSOCK", DATABASE 'test',user 'root'); +create table t1 (c1 int); +create table t2 (c2 int); + +insert into t1 values (1), (2); +insert into t2 values (0), (1), (2); + +create table t1s (c1 int) ENGINE=Spider COMMENT='WRAPPER "mysql", srv "srv",TABLE "t1"'; +create table t2s (c2 int) ENGINE=Spider COMMENT='WRAPPER "mysql", srv "srv",TABLE "t2"'; + +select c1 from t1s, (select c2 from t2s where c2 > 0) t where c1 + 1 = c2; + +drop table t1, t2, t1s, t2s; +drop server srv; +--disable_query_log +--disable_result_log +--source ../../t/test_deinit.inc +--enable_result_log +--enable_query_log +--echo # +--echo # end of test subquery +--echo # diff --git a/storage/spider/mysql-test/spider/include/clean_up_spider.inc b/storage/spider/mysql-test/spider/include/clean_up_spider.inc index 1f0659dc..1c977bfb 100644 --- a/storage/spider/mysql-test/spider/include/clean_up_spider.inc +++ b/storage/spider/mysql-test/spider/include/clean_up_spider.inc @@ -3,7 +3,8 @@ DROP FUNCTION spider_copy_tables; DROP FUNCTION spider_ping_table; DROP FUNCTION spider_bg_direct_sql; DROP FUNCTION spider_direct_sql; -UNINSTALL SONAME IF EXISTS "ha_spider"; +--replace_regex /\.dll/.so/ +UNINSTALL SONAME IF EXISTS 'ha_spider'; DROP TABLE IF EXISTS mysql.spider_xa; DROP TABLE IF EXISTS mysql.spider_xa_member; DROP TABLE IF EXISTS mysql.spider_xa_failed_log; diff --git a/storage/spider/mysql-test/spider/include/init_spider.inc b/storage/spider/mysql-test/spider/include/init_spider.inc index ccf9e305..6014b1d7 100644 --- a/storage/spider/mysql-test/spider/include/init_spider.inc +++ b/storage/spider/mysql-test/spider/include/init_spider.inc @@ -158,27 +158,6 @@ let $PLUGIN_VERSION= `SELECT SUBSTRING_INDEX(plugin_version, '.', 1) FROM information_schema.plugins WHERE plugin_name = 'SPIDER'`; -if (`SELECT IF($PLUGIN_VERSION = 3, 1, 0)`) -{ - let $HAS_REWRITE= - `SELECT IF (STRCMP('$SERVER_NAME', 'MariaDB') = 0, - IF ($SERVER_MAJOR_VERSION = 10, - IF ($SERVER_MINOR_VERSION < 4, 0, 1), - IF ($SERVER_MAJOR_VERSION < 10, 0, 1)), - 0)`; - let $HAS_REWRITE= 0; - if ($HAS_REWRITE) - { - let $PLUGIN_NAME= spider_flush_rewrite_cache; - let $PLUGIN_EXIST= - `SELECT COUNT(*) FROM mysql.func WHERE name = '$PLUGIN_NAME'`; - while (!$PLUGIN_EXIST) - { - let $PLUGIN_EXIST= - `SELECT COUNT(*) FROM mysql.func WHERE name = '$PLUGIN_NAME'`; - } - } -} let $PLUGIN_NAME= spider_flush_table_mon_cache; let $PLUGIN_EXIST= `SELECT COUNT(*) FROM mysql.func WHERE name = '$PLUGIN_NAME'`; diff --git a/storage/spider/mysql-test/spider/r/direct_join.result b/storage/spider/mysql-test/spider/r/direct_join.result index 398985ac..96f18a54 100644 --- a/storage/spider/mysql-test/spider/r/direct_join.result +++ b/storage/spider/mysql-test/spider/r/direct_join.result @@ -167,7 +167,7 @@ connection child2_1; SELECT argument FROM mysql.general_log WHERE command_type != 'Execute' AND argument LIKE '%select %'; argument select `id`,`hr_status`,`region_code`,`region` from `auto_test_remote`.`tbl_person` where `id` = '24FC3F0A5119432BAE13DD65AABAA39C' and `region` = 510411 -select count(0) `count(0)` from `auto_test_remote`.`tbl_ncd_cm_person` t0 where ((t0.`person_id` = '24FC3F0A5119432BAE13DD65AABAA39C') and (t0.`diseaseKind_id` = '52A0328740914BCE86ED10A4D2521816')) +select count(0) `count(0)` from (select 1) t0 join `auto_test_remote`.`tbl_ncd_cm_person` t1 where ((t1.`person_id` = '24FC3F0A5119432BAE13DD65AABAA39C') and (t1.`diseaseKind_id` = '52A0328740914BCE86ED10A4D2521816')) SELECT argument FROM mysql.general_log WHERE command_type != 'Execute' AND argument LIKE '%select %' SELECT * FROM tbl_person; id hr_status region_code region diff --git a/storage/spider/mysql-test/spider/r/direct_left_right_join_nullable.result b/storage/spider/mysql-test/spider/r/direct_left_right_join_nullable.result index a6bd3a7c..96b00c61 100644 --- a/storage/spider/mysql-test/spider/r/direct_left_right_join_nullable.result +++ b/storage/spider/mysql-test/spider/r/direct_left_right_join_nullable.result @@ -87,7 +87,7 @@ NULL NULL NULL 3 connection child2_1; SELECT argument FROM mysql.general_log WHERE command_type != 'Execute' AND argument LIKE '%select %'; argument -select t0.`a` `a`,t2.`b` `b`,t2.`c` `c`,t3.`a` `a` from `auto_test_remote`.`ta_r_no_idx` t3 left join (`auto_test_remote`.`ta_r_auto_inc` t2 join `auto_test_remote`.`ta_r_3` t1 join `auto_test_remote`.`ta_r` t0) on ((t2.`b` = t3.`b`) and (t2.`c` = t1.`c`) and (t0.`a` = t1.`a`) and (t1.`a` is not null)) where 1 order by t3.`a` desc +select t0.`a` `a`,t2.`b` `b`,t2.`c` `c`,t3.`a` `a` from `auto_test_remote`.`ta_r_no_idx` t3 left join (`auto_test_remote`.`ta_r` t0 join `auto_test_remote`.`ta_r_3` t1 join `auto_test_remote`.`ta_r_auto_inc` t2) on ((t2.`b` = t3.`b`) and (t2.`c` = t1.`c`) and (t0.`a` = t1.`a`) and (t1.`a` is not null)) where 1 order by t3.`a` desc SELECT argument FROM mysql.general_log WHERE command_type != 'Execute' AND argument LIKE '%select %' SELECT a, b, date_format(c, '%Y-%m-%d %H:%i:%s') FROM ta_r ORDER BY a; a b date_format(c, '%Y-%m-%d %H:%i:%s') diff --git a/storage/spider/mysql-test/spider/r/direct_left_right_left_join_nullable.result b/storage/spider/mysql-test/spider/r/direct_left_right_left_join_nullable.result index 8fab6d24..7ba1084a 100644 --- a/storage/spider/mysql-test/spider/r/direct_left_right_left_join_nullable.result +++ b/storage/spider/mysql-test/spider/r/direct_left_right_left_join_nullable.result @@ -86,7 +86,7 @@ NULL c 2000-01-03 00:00:00 3 connection child2_1; SELECT argument FROM mysql.general_log WHERE command_type != 'Execute' AND argument LIKE '%select %'; argument -select t0.`a` `a`,t2.`b` `b`,t2.`c` `c`,t3.`a` `a` from `auto_test_remote`.`ta_r_auto_inc` t2 left join (`auto_test_remote`.`ta_r_3` t1 join `auto_test_remote`.`ta_r` t0) on ((t1.`c` = t2.`c`) and (t0.`a` = t1.`a`) and (t1.`a` is not null)) left join `auto_test_remote`.`ta_r_no_idx` t3 on (t3.`b` = t2.`b`) where 1 order by t3.`a` desc +select t0.`a` `a`,t2.`b` `b`,t2.`c` `c`,t3.`a` `a` from `auto_test_remote`.`ta_r_auto_inc` t2 left join (`auto_test_remote`.`ta_r` t0 join `auto_test_remote`.`ta_r_3` t1) on ((t1.`c` = t2.`c`) and (t0.`a` = t1.`a`) and (t1.`a` is not null)) left join `auto_test_remote`.`ta_r_no_idx` t3 on (t3.`b` = t2.`b`) where 1 order by t3.`a` desc SELECT argument FROM mysql.general_log WHERE command_type != 'Execute' AND argument LIKE '%select %' SELECT a, b, date_format(c, '%Y-%m-%d %H:%i:%s') FROM ta_r ORDER BY a; a b date_format(c, '%Y-%m-%d %H:%i:%s') diff --git a/storage/spider/mysql-test/spider/r/direct_right_join.result b/storage/spider/mysql-test/spider/r/direct_right_join.result index d7780b94..ee0940a4 100644 --- a/storage/spider/mysql-test/spider/r/direct_right_join.result +++ b/storage/spider/mysql-test/spider/r/direct_right_join.result @@ -79,7 +79,7 @@ a b c connection child2_1; SELECT argument FROM mysql.general_log WHERE command_type != 'Execute' AND argument LIKE '%select %'; argument -select t0.`b` `b`,t0.`a` `a`,t2.`b` `b`,t2.`c` `c` from `auto_test_remote`.`ta_r_int` t2 left join (`auto_test_remote`.`ta_r` t0 join `auto_test_remote`.`ta_r_3` t1) on ((t0.`a` = t2.`a`) and (t1.`a` = t2.`a`)) where 1 order by t0.`b` desc +select t0.`b` `b`,t0.`a` `a`,t2.`b` `b`,t2.`c` `c` from `auto_test_remote`.`ta_r_int` t2 left join (`auto_test_remote`.`ta_r_3` t1 join `auto_test_remote`.`ta_r` t0) on ((t0.`a` = t2.`a`) and (t1.`a` = t2.`a`)) where 1 order by t0.`b` desc SELECT argument FROM mysql.general_log WHERE command_type != 'Execute' AND argument LIKE '%select %' SELECT a, b, date_format(c, '%Y-%m-%d %H:%i:%s') FROM ta_r ORDER BY a; a b date_format(c, '%Y-%m-%d %H:%i:%s') diff --git a/storage/spider/mysql-test/spider/r/direct_right_join_nullable.result b/storage/spider/mysql-test/spider/r/direct_right_join_nullable.result index 5101ea50..fefb67fa 100644 --- a/storage/spider/mysql-test/spider/r/direct_right_join_nullable.result +++ b/storage/spider/mysql-test/spider/r/direct_right_join_nullable.result @@ -87,7 +87,7 @@ NULL c 2000-01-03 00:00:00 3 connection child2_1; SELECT argument FROM mysql.general_log WHERE command_type != 'Execute' AND argument LIKE '%select %'; argument -select t0.`a` `a`,t2.`b` `b`,t2.`c` `c`,t3.`a` `a` from `auto_test_remote`.`ta_r_no_idx` t3 left join `auto_test_remote`.`ta_r_auto_inc` t2 on (t2.`b` = t3.`b`) left join `auto_test_remote`.`ta_r_3` t1 on (t1.`c` = t2.`c`) left join `auto_test_remote`.`ta_r` t0 on ((t0.`a` = t1.`a`) and (t1.`a` is not null)) where 1 order by t3.`a` desc +select t0.`a` `a`,t2.`b` `b`,t2.`c` `c`,t3.`a` `a` from `auto_test_remote`.`ta_r_no_idx` t3 left join (`auto_test_remote`.`ta_r_auto_inc` t2 left join (`auto_test_remote`.`ta_r_3` t1 left join `auto_test_remote`.`ta_r` t0 on ((t0.`a` = t1.`a`) and (t1.`a` is not null))) on (t1.`c` = t2.`c`)) on (t2.`b` = t3.`b`) where 1 order by t3.`a` desc SELECT argument FROM mysql.general_log WHERE command_type != 'Execute' AND argument LIKE '%select %' SELECT a, b, date_format(c, '%Y-%m-%d %H:%i:%s') FROM ta_r ORDER BY a; a b date_format(c, '%Y-%m-%d %H:%i:%s') diff --git a/storage/spider/mysql-test/spider/r/direct_right_left_join_nullable.result b/storage/spider/mysql-test/spider/r/direct_right_left_join_nullable.result index c90fe3ab..64c3b57c 100644 --- a/storage/spider/mysql-test/spider/r/direct_right_left_join_nullable.result +++ b/storage/spider/mysql-test/spider/r/direct_right_left_join_nullable.result @@ -86,7 +86,7 @@ NULL c 2000-01-03 00:00:00 3 connection child2_1; SELECT argument FROM mysql.general_log WHERE command_type != 'Execute' AND argument LIKE '%select %'; argument -select t0.`a` `a`,t2.`b` `b`,t2.`c` `c`,t3.`a` `a` from `auto_test_remote`.`ta_r_auto_inc` t2 left join `auto_test_remote`.`ta_r_3` t1 on (t1.`c` = t2.`c`) left join `auto_test_remote`.`ta_r` t0 on ((t0.`a` = t1.`a`) and (t1.`a` is not null)) left join `auto_test_remote`.`ta_r_no_idx` t3 on (t3.`b` = t2.`b`) where 1 order by t3.`a` desc +select t0.`a` `a`,t2.`b` `b`,t2.`c` `c`,t3.`a` `a` from `auto_test_remote`.`ta_r_auto_inc` t2 left join (`auto_test_remote`.`ta_r_3` t1 left join `auto_test_remote`.`ta_r` t0 on ((t0.`a` = t1.`a`) and (t1.`a` is not null))) on (t1.`c` = t2.`c`) left join `auto_test_remote`.`ta_r_no_idx` t3 on (t3.`b` = t2.`b`) where 1 order by t3.`a` desc SELECT argument FROM mysql.general_log WHERE command_type != 'Execute' AND argument LIKE '%select %' SELECT a, b, date_format(c, '%Y-%m-%d %H:%i:%s') FROM ta_r ORDER BY a; a b date_format(c, '%Y-%m-%d %H:%i:%s') diff --git a/storage/spider/mysql-test/spider/r/direct_right_left_right_join_nullable.result b/storage/spider/mysql-test/spider/r/direct_right_left_right_join_nullable.result index f6c808be..68d11934 100644 --- a/storage/spider/mysql-test/spider/r/direct_right_left_right_join_nullable.result +++ b/storage/spider/mysql-test/spider/r/direct_right_left_right_join_nullable.result @@ -87,7 +87,7 @@ NULL c 2000-01-03 00:00:00 3 connection child2_1; SELECT argument FROM mysql.general_log WHERE command_type != 'Execute' AND argument LIKE '%select %'; argument -select t0.`a` `a`,t2.`b` `b`,t2.`c` `c`,t3.`a` `a` from `auto_test_remote`.`ta_r_no_idx` t3 left join (`auto_test_remote`.`ta_r_auto_inc` t2 join `auto_test_remote`.`ta_r_3` t1 left join `auto_test_remote`.`ta_r` t0 on ((t0.`a` = t1.`a`) and (t1.`a` is not null))) on ((t2.`b` = t3.`b`) and (t2.`c` = t1.`c`)) where 1 order by t3.`a` desc +select t0.`a` `a`,t2.`b` `b`,t2.`c` `c`,t3.`a` `a` from `auto_test_remote`.`ta_r_no_idx` t3 left join (`auto_test_remote`.`ta_r_3` t1 left join `auto_test_remote`.`ta_r` t0 on ((t0.`a` = t1.`a`) and (t1.`a` is not null)) join `auto_test_remote`.`ta_r_auto_inc` t2) on ((t2.`b` = t3.`b`) and (t2.`c` = t1.`c`)) where 1 order by t3.`a` desc SELECT argument FROM mysql.general_log WHERE command_type != 'Execute' AND argument LIKE '%select %' SELECT a, b, date_format(c, '%Y-%m-%d %H:%i:%s') FROM ta_r ORDER BY a; a b date_format(c, '%Y-%m-%d %H:%i:%s') diff --git a/storage/spider/mysql-test/spider/r/partition_join_pushdown_for_single_partition.result b/storage/spider/mysql-test/spider/r/partition_join_pushdown_for_single_partition.result index 3eed8df2..850546fa 100644 --- a/storage/spider/mysql-test/spider/r/partition_join_pushdown_for_single_partition.result +++ b/storage/spider/mysql-test/spider/r/partition_join_pushdown_for_single_partition.result @@ -86,8 +86,10 @@ SELECT argument FROM mysql.general_log WHERE command_type != 'Execute' AND argum argument select `value` from `auto_test_remote2`.`tbl_a` where `value` = 5 select `value2` from `auto_test_remote2`.`tbl_b` where `value2` = 5 +select sum('5') `sum(a.value)`,count('5') `count(b.value2)` from (select 1) t0 join (select 1) t1 select `value` from `auto_test_remote2`.`tbl_a` where `value` = 5 select `value2` from `auto_test_remote2`.`tbl_b` where `value2` = 5 +select sum('5') `sum(a.value)`,count('5') `count(b.value2)` from (select 1) t0 join (select 1) t1 SELECT argument FROM mysql.general_log WHERE command_type != 'Execute' AND argument LIKE '%select %' SELECT value FROM tbl_a ORDER BY value; value diff --git a/storage/spider/mysql-test/spider/r/variable_deprecation.result b/storage/spider/mysql-test/spider/r/variable_deprecation.result index cea36ff2..be964ce4 100644 --- a/storage/spider/mysql-test/spider/r/variable_deprecation.result +++ b/storage/spider/mysql-test/spider/r/variable_deprecation.result @@ -75,8 +75,6 @@ Warning 1287 '@@spider_load_crd_at_startup' is deprecated and will be removed in SHOW VARIABLES LIKE "spider_load_crd_at_startup"; Variable_name Value spider_load_crd_at_startup 0 -DROP TABLE tbl_a; -DROP TABLE tbl_b; # MDEV-28008 Deprecate spider_crd_mode and spider_sts_mode SET spider_crd_mode = 1; Warnings: diff --git a/storage/spider/mysql-test/spider/regression/e1121/r/direct_join_by_pkey_key.result b/storage/spider/mysql-test/spider/regression/e1121/r/direct_join_by_pkey_key.result index a5553304..72645f4f 100644 --- a/storage/spider/mysql-test/spider/regression/e1121/r/direct_join_by_pkey_key.result +++ b/storage/spider/mysql-test/spider/regression/e1121/r/direct_join_by_pkey_key.result @@ -46,7 +46,7 @@ connection child2_1; SELECT argument FROM mysql.general_log WHERE command_type != 'Execute' AND argument LIKE '%`tbl_a`%' ; SELECT argument FROM mysql.general_log WHERE command_type != 'Execute' AND argument LIKE '%`tbl_b`%'; argument -select t0.`val` `val`,t0.`akey` `akey` from `auto_test_remote`.`tbl_a` t0 where (t0.`akey` = '4') +select t0.`val` `val`,t0.`akey` `akey` from `auto_test_remote`.`tbl_a` t0 join (select 1) t1 where (t0.`akey` = '4') SELECT argument FROM mysql.general_log WHERE command_type != 'Execute' AND argument LIKE '%`tbl_a`%' ; SELECT argument FROM mysql.general_log WHERE command_type != 'Execute' AND argument LIKE '%`tbl_b`%' argument diff --git a/storage/spider/mysql-test/spider/t/variable_deprecation.test b/storage/spider/mysql-test/spider/t/variable_deprecation.test index 3ed8dcf5..6159b25b 100644 --- a/storage/spider/mysql-test/spider/t/variable_deprecation.test +++ b/storage/spider/mysql-test/spider/t/variable_deprecation.test @@ -47,9 +47,6 @@ SHOW VARIABLES LIKE "spider_load_sts_at_startup"; SET GLOBAL spider_load_crd_at_startup = 0; SHOW VARIABLES LIKE "spider_load_crd_at_startup"; -DROP TABLE tbl_a; -DROP TABLE tbl_b; - --echo # MDEV-28008 Deprecate spider_crd_mode and spider_sts_mode SET spider_crd_mode = 1; SHOW VARIABLES LIKE "spider_crd_mode"; diff --git a/storage/spider/spd_conn.cc b/storage/spider/spd_conn.cc index 443912b5..92775896 100644 --- a/storage/spider/spd_conn.cc +++ b/storage/spider/spd_conn.cc @@ -411,7 +411,7 @@ SPIDER_CONN *spider_create_conn( tables_on_different_db_are_joinable = TRUE; } if (!(conn = (SPIDER_CONN *) - spider_bulk_malloc(spider_current_trx, 18, MYF(MY_WME | MY_ZEROFILL), + spider_bulk_malloc(spider_current_trx, SPD_MID_CREATE_CONN_1, MYF(MY_WME | MY_ZEROFILL), &conn, (uint) (sizeof(*conn)), &tmp_name, (uint) (share->conn_keys_lengths[link_idx] + 1), &tmp_host, (uint) (share->tgt_hosts_lengths[link_idx] + 1), @@ -450,7 +450,7 @@ SPIDER_CONN *spider_create_conn( goto error_alloc_conn; } - conn->default_database.init_calc_mem(75); + conn->default_database.init_calc_mem(SPD_MID_CREATE_CONN_2); conn->conn_key_length = share->conn_keys_lengths[link_idx]; conn->conn_key = tmp_name; memcpy(conn->conn_key, share->conn_keys[link_idx], @@ -1204,13 +1204,7 @@ int spider_conn_queue_and_merge_loop_check( lcptr->flag = SPIDER_LOP_CHK_MERAGED; lcptr->next = NULL; if (!conn->loop_check_meraged_first) - { conn->loop_check_meraged_first = lcptr; - conn->loop_check_meraged_last = lcptr; - } else { - conn->loop_check_meraged_last->next = lcptr; - conn->loop_check_meraged_last = lcptr; - } } DBUG_RETURN(0); @@ -1303,7 +1297,7 @@ int spider_conn_queue_loop_check( loop_check_buf[lex_str.length] = '\0'; DBUG_PRINT("info", ("spider param name=%s", lex_str.str)); loop_check = get_variable(&thd->user_vars, &lex_str, FALSE); - if (!loop_check || loop_check->type != STRING_RESULT) + if (!loop_check || loop_check->type_handler()->result_type() != STRING_RESULT) { DBUG_PRINT("info", ("spider client is not Spider")); lex_str.str = ""; @@ -1865,7 +1859,7 @@ int spider_create_conn_thread( error_num = HA_ERR_OUT_OF_MEM; goto error_job_stack_init; } - spider_alloc_calc_mem_init(conn->bg_job_stack, 163); + spider_alloc_calc_mem_init(conn->bg_job_stack, SPD_MID_CREATE_CONN_THREAD_1); spider_alloc_calc_mem(spider_current_trx, conn->bg_job_stack, conn->bg_job_stack.max_element * @@ -3476,7 +3470,7 @@ int spider_create_mon_threads( char *buf = (char *) my_alloca(share->table_name_length + SPIDER_SQL_INT_LEN + 1); spider_string conv_name_str(buf, share->table_name_length + SPIDER_SQL_INT_LEN + 1, system_charset_info); - conv_name_str.init_calc_mem(105); + conv_name_str.init_calc_mem(SPD_MID_CREATE_MON_THREADS_1); conv_name_str.length(0); conv_name_str.q_append(share->table_name, share->table_name_length); for (roop_count = 0; roop_count < (int) share->all_link_count; @@ -3509,7 +3503,7 @@ int spider_create_mon_threads( } } if (!(share->bg_mon_thds = (THD **) - spider_bulk_malloc(spider_current_trx, 23, MYF(MY_WME | MY_ZEROFILL), + spider_bulk_malloc(spider_current_trx, SPD_MID_CREATE_MON_THREADS_2, MYF(MY_WME | MY_ZEROFILL), &share->bg_mon_thds, (uint) (sizeof(THD *) * share->all_link_count), &share->bg_mon_threads, diff --git a/storage/spider/spd_copy_tables.cc b/storage/spider/spd_copy_tables.cc index ab1f8d35..70ca17f8 100644 --- a/storage/spider/spd_copy_tables.cc +++ b/storage/spider/spd_copy_tables.cc @@ -274,7 +274,7 @@ int spider_udf_get_copy_tgt_tables( } do { if (!(table_conn = (SPIDER_COPY_TABLE_CONN *) - spider_bulk_malloc(spider_current_trx, 25, MYF(MY_WME | MY_ZEROFILL), + spider_bulk_malloc(spider_current_trx, SPD_MID_UDF_GET_COPY_TGT_TABLES_1, MYF(MY_WME | MY_ZEROFILL), &table_conn, (uint) (sizeof(SPIDER_COPY_TABLE_CONN)), &tmp_share, (uint) (sizeof(SPIDER_SHARE)), &tmp_connect_info, @@ -590,7 +590,7 @@ int spider_udf_copy_tables_create_table_list( } if (!(copy_tables->link_idxs[0] = (int *) - spider_bulk_malloc(spider_current_trx, 26, MYF(MY_WME | MY_ZEROFILL), + spider_bulk_malloc(spider_current_trx, SPD_MID_UDF_COPY_TABLES_CREATE_TABLE_LIST_1, MYF(MY_WME | MY_ZEROFILL), ©_tables->link_idxs[0], (uint) (sizeof(int) * copy_tables->link_idx_count[0]), ©_tables->link_idxs[1], @@ -772,7 +772,7 @@ long long spider_copy_tables_body( } if (!(copy_tables = (SPIDER_COPY_TABLES *) - spider_bulk_malloc(spider_current_trx, 27, MYF(MY_WME | MY_ZEROFILL), + spider_bulk_malloc(spider_current_trx, SPD_MID_COPY_TABLES_BODY_1, MYF(MY_WME | MY_ZEROFILL), ©_tables, (uint) (sizeof(SPIDER_COPY_TABLES)), NullS)) ) { @@ -984,7 +984,7 @@ long long spider_copy_tables_body( { tmp_spider = &spider[roop_count]; if (!(tmp_spider->dbton_handler = (spider_db_handler **) - spider_bulk_alloc_mem(spider_current_trx, 205, + spider_bulk_alloc_mem(spider_current_trx, SPD_MID_COPY_TABLES_BODY_2, __func__, __FILE__, __LINE__, MYF(MY_WME | MY_ZEROFILL), &tmp_spider->dbton_handler, sizeof(spider_db_handler *) * SPIDER_DBTON_SIZE, @@ -1005,7 +1005,7 @@ long long spider_copy_tables_body( } */ tmp_spider->conns = &table_conn->conn; - tmp_sql[roop_count].init_calc_mem(122); + tmp_sql[roop_count].init_calc_mem(SPD_MID_COPY_TABLES_BODY_3); tmp_sql[roop_count].set_charset(copy_tables->access_charset); tmp_spider->result_list.sqls = &tmp_sql[roop_count]; tmp_spider->need_mons = &table_conn->need_mon; @@ -1030,7 +1030,7 @@ long long spider_copy_tables_body( { tmp_spider = &spider[roop_count]; if (!(tmp_spider->dbton_handler = (spider_db_handler **) - spider_bulk_alloc_mem(spider_current_trx, 206, + spider_bulk_alloc_mem(spider_current_trx, SPD_MID_COPY_TABLES_BODY_4, __func__, __FILE__, __LINE__, MYF(MY_WME | MY_ZEROFILL), &tmp_spider->dbton_handler, sizeof(spider_db_handler *) * SPIDER_DBTON_SIZE, @@ -1051,7 +1051,7 @@ long long spider_copy_tables_body( } */ tmp_spider->conns = &table_conn->conn; - tmp_sql[roop_count].init_calc_mem(201); + tmp_sql[roop_count].init_calc_mem(SPD_MID_COPY_TABLES_BODY_5); tmp_sql[roop_count].set_charset(copy_tables->access_charset); tmp_spider->result_list.sqls = &tmp_sql[roop_count]; tmp_spider->need_mons = &table_conn->need_mon; diff --git a/storage/spider/spd_db_conn.cc b/storage/spider/spd_db_conn.cc index a541a5db..16d5b836 100644 --- a/storage/spider/spd_db_conn.cc +++ b/storage/spider/spd_db_conn.cc @@ -250,7 +250,7 @@ int spider_db_conn_queue_action( spider_string sql_str(sql_buf, sizeof(sql_buf), system_charset_info); DBUG_ENTER("spider_db_conn_queue_action"); DBUG_PRINT("info", ("spider conn=%p", conn)); - sql_str.init_calc_mem(106); + sql_str.init_calc_mem(SPD_MID_DB_CONN_QUEUE_ACTION_1); sql_str.length(0); if (conn->queued_connect) { @@ -651,7 +651,7 @@ int spider_db_query( DBUG_RETURN(error_num); #ifndef DBUG_OFF spider_string tmp_query_str(sizeof(char) * (length + 1)); - tmp_query_str.init_calc_mem(107); + tmp_query_str.init_calc_mem(SPD_MID_DB_QUERY_1); char *tmp_query = (char *) tmp_query_str.c_ptr_safe(); memcpy(tmp_query, query, length); tmp_query[length] = '\0'; @@ -2174,7 +2174,7 @@ int spider_db_fetch_for_item_sum_func( { if (!spider->direct_aggregate_item_first) { - if (!spider_bulk_malloc(spider_current_trx, 240, MYF(MY_WME), + if (!spider_bulk_malloc(spider_current_trx, SPD_MID_DB_FETCH_FOR_ITEM_SUM_FUNC_1, MYF(MY_WME), &spider->direct_aggregate_item_first, (uint) (sizeof(SPIDER_ITEM_HLD)), NullS) @@ -2191,7 +2191,7 @@ int spider_db_fetch_for_item_sum_func( } else { if (!spider->direct_aggregate_item_current->next) { - if (!spider_bulk_malloc(spider_current_trx, 241, MYF(MY_WME), + if (!spider_bulk_malloc(spider_current_trx, SPD_MID_DB_FETCH_FOR_ITEM_SUM_FUNC_2, MYF(MY_WME), &spider->direct_aggregate_item_current->next, (uint) (sizeof(SPIDER_ITEM_HLD)), NullS) ) { @@ -2235,7 +2235,7 @@ int spider_db_fetch_for_item_sum_func( } else { char buf[MAX_FIELD_WIDTH]; spider_string tmp_str(buf, MAX_FIELD_WIDTH, share->access_charset); - tmp_str.init_calc_mem(242); + tmp_str.init_calc_mem(SPD_MID_DB_FETCH_FOR_ITEM_SUM_FUNC_3); tmp_str.length(0); if ((error_num = row->append_to_str(&tmp_str))) DBUG_RETURN(error_num); @@ -3089,7 +3089,7 @@ int spider_db_store_result( if (!result_list->first) { if (!(result_list->first = (SPIDER_RESULT *) - spider_malloc(spider_current_trx, 4, sizeof(*result_list->first), + spider_malloc(spider_current_trx, SPD_MID_DB_STORE_RESULT_1, sizeof(*result_list->first), MYF(MY_WME | MY_ZEROFILL))) ) { if (!conn->mta_conn_mutex_unlock_later) @@ -3119,7 +3119,7 @@ int spider_db_store_result( if (result_list->bgs_current == result_list->last) { if (!(result_list->last = (SPIDER_RESULT *) - spider_malloc(spider_current_trx, 5, sizeof(*result_list->last), + spider_malloc(spider_current_trx, SPD_MID_DB_STORE_RESULT_2, sizeof(*result_list->last), MYF(MY_WME | MY_ZEROFILL))) ) { if (!conn->mta_conn_mutex_unlock_later) @@ -3162,7 +3162,7 @@ int spider_db_store_result( if (result_list->current == result_list->last) { if (!(result_list->last = (SPIDER_RESULT *) - spider_malloc(spider_current_trx, 6, sizeof(*result_list->last), + spider_malloc(spider_current_trx, SPD_MID_DB_STORE_RESULT_3, sizeof(*result_list->last), MYF(MY_WME | MY_ZEROFILL))) ) { if (!conn->mta_conn_mutex_unlock_later) @@ -3373,7 +3373,7 @@ int spider_db_store_result( } current->field_count = field_count; if (!(position = (SPIDER_POSITION *) - spider_bulk_malloc(spider_current_trx, 7, MYF(MY_WME | MY_ZEROFILL), + spider_bulk_malloc(spider_current_trx, SPD_MID_DB_STORE_RESULT_4, MYF(MY_WME | MY_ZEROFILL), &position, (uint) (sizeof(SPIDER_POSITION) * page_size), &tmp_row, (uint) (sizeof(SPIDER_DB_ROW) * field_count), NullS)) @@ -3436,7 +3436,7 @@ int spider_db_store_result( THD *thd = current_thd; char buf[MAX_FIELD_WIDTH]; spider_string tmp_str(buf, MAX_FIELD_WIDTH, &my_charset_bin); - tmp_str.init_calc_mem(120); + tmp_str.init_calc_mem(SPD_MID_DB_STORE_RESULT_5); DBUG_PRINT("info",("spider store result to temporary table")); DBUG_ASSERT(!current->result_tmp_tbl); @@ -3797,7 +3797,7 @@ int spider_db_store_result_for_reuse_cursor( THD *thd = current_thd; char buf[MAX_FIELD_WIDTH]; spider_string tmp_str(buf, MAX_FIELD_WIDTH, &my_charset_bin); - tmp_str.init_calc_mem(120); + tmp_str.init_calc_mem(SPD_MID_DB_STORE_RESULT_FOR_REUSE_CURSOR_1); DBUG_PRINT("info",("spider store result to temporary table")); DBUG_ASSERT(!current->result_tmp_tbl); @@ -6537,7 +6537,7 @@ int spider_db_delete_all_rows( spider->conn_link_idx, roop_count, share->link_count, SPIDER_LINK_STATUS_RECOVERY) ) { - uint dbton_id = share->use_sql_dbton_ids[roop_count]; + uint dbton_id = share->sql_dbton_ids[roop_count]; spider_db_handler *dbton_hdl = spider->dbton_handler[dbton_id]; conn = spider->conns[roop_count]; pthread_mutex_assert_not_owner(&conn->mta_conn_mutex); @@ -7468,19 +7468,20 @@ int spider_db_open_item_ident( } else { if (str) { - SPIDER_FIELD_CHAIN *field_chain = fields->get_next_field_chain(); - SPIDER_FIELD_HOLDER *field_holder = field_chain->field_holder; - spider = field_holder->spider; + SPIDER_TABLE_HOLDER *table= fields->find_table(field); + /* If table or table->spider is NULL the GBH creation + would have been skipped the first pass (see below). */ + spider = table->spider; share = spider->share; if ((error_num = share->dbton_share[dbton_id]-> append_column_name_with_alias(str, field->field_index, - field_holder->alias->ptr(), field_holder->alias->length()))) + table->alias->ptr(), table->alias->length()))) DBUG_RETURN(error_num); - } else { - if ((error_num = fields->add_field(field))) - { - DBUG_RETURN(error_num); - } + } else + { + SPIDER_TABLE_HOLDER *table= fields->find_table(field); + if (!table || !table->spider) + DBUG_RETURN(ER_SPIDER_COND_SKIP_NUM); } } } @@ -7583,21 +7584,20 @@ int spider_db_open_item_field( } else { if (str) { - SPIDER_FIELD_CHAIN *field_chain = fields->get_next_field_chain(); - SPIDER_FIELD_HOLDER *field_holder = field_chain->field_holder; - spider = field_holder->spider; + SPIDER_TABLE_HOLDER *table= fields->find_table(field); + /* If table or table->spider is NULL the GBH creation + would have been skipped the first pass (see below). */ + spider = table->spider; share = spider->share; - field = spider->field_exchange(field); - DBUG_ASSERT(field); if ((error_num = share->dbton_share[dbton_id]-> append_column_name_with_alias(str, field->field_index, - field_holder->alias->ptr(), field_holder->alias->length()))) + table->alias->ptr(), table->alias->length()))) DBUG_RETURN(error_num); - } else { - if ((error_num = fields->add_field(field))) - { - DBUG_RETURN(error_num); - } + } else + { + SPIDER_TABLE_HOLDER *table= fields->find_table(field); + if (!table || !table->spider) + DBUG_RETURN(ER_SPIDER_COND_SKIP_NUM); } DBUG_RETURN(0); } @@ -7740,7 +7740,7 @@ int spider_db_open_item_string( char tmp_buf[MAX_FIELD_WIDTH]; spider_string tmp_str(tmp_buf, MAX_FIELD_WIDTH, str->charset()); String *tmp_str2; - tmp_str.init_calc_mem(126); + tmp_str.init_calc_mem(SPD_MID_DB_OPEN_ITEM_STRING_1); if (!(tmp_str2 = item->val_str(tmp_str.get_str()))) { @@ -7859,7 +7859,7 @@ int spider_db_open_item_int( char tmp_buf[MAX_FIELD_WIDTH]; spider_string tmp_str(tmp_buf, MAX_FIELD_WIDTH, str->charset()); String *tmp_str2; - tmp_str.init_calc_mem(127); + tmp_str.init_calc_mem(SPD_MID_DB_OPEN_ITEM_INT_1); if (!(tmp_str2 = item->val_str(tmp_str.get_str()))) { @@ -8909,8 +8909,8 @@ int spider_db_udf_ping_table( system_charset_info); spider_string where_str(where_buf, sizeof(where_buf), system_charset_info); - sql_str.init_calc_mem(128); - where_str.init_calc_mem(129); + sql_str.init_calc_mem(SPD_MID_DB_UDF_PING_TABLE_1); + where_str.init_calc_mem(SPD_MID_DB_UDF_PING_TABLE_2); sql_str.length(0); where_str.length(0); if ( @@ -9020,8 +9020,8 @@ int spider_db_udf_ping_table_append_mon_next( spider_string where_clause_str(where_clause ? where_clause : "", where_clause_length + 1, str->charset()); DBUG_ENTER("spider_db_udf_ping_table_append_mon_next"); - child_table_name_str.init_calc_mem(130); - where_clause_str.init_calc_mem(131); + child_table_name_str.init_calc_mem(SPD_MID_DB_UDF_PING_TABLE_APPEND_MON_NEXT_1); + where_clause_str.init_calc_mem(SPD_MID_DB_UDF_PING_TABLE_APPEND_MON_NEXT_2); child_table_name_str.length(child_table_name_length); where_clause_str.length(where_clause_length); limit_str_length = my_sprintf(limit_str, (limit_str, "%lld", limit)); @@ -9174,7 +9174,7 @@ int spider_db_udf_ping_table_mon_next( } spider_string sql_str(sql_buf, sizeof(sql_buf), thd->variables.character_set_client); - sql_str.init_calc_mem(132); + sql_str.init_calc_mem(SPD_MID_DB_UDF_PING_TABLE_MON_NEXT_1); sql_str.length(0); trx.thd = thd; spider.share = share; @@ -9328,7 +9328,7 @@ int spider_db_udf_copy_tables( KEY *key_info = &table->key_info[table->s->primary_key]; DBUG_ENTER("spider_db_udf_copy_tables"); if (!(last_row_pos = (ulong *) - spider_bulk_malloc(spider_current_trx, 30, MYF(MY_WME), + spider_bulk_malloc(spider_current_trx, SPD_MID_DB_UDF_COPY_TABLES_1, MYF(MY_WME), &last_row_pos, (uint) (sizeof(ulong) * table->s->fields), &last_lengths, (uint) (sizeof(ulong) * table->s->fields), NullS)) diff --git a/storage/spider/spd_db_include.h b/storage/spider/spd_db_include.h index ef0fb8f2..c2cbe5f2 100644 --- a/storage/spider/spd_db_include.h +++ b/storage/spider/spd_db_include.h @@ -542,34 +542,26 @@ typedef struct spider_conn_holder spider_conn_holder *next; } SPIDER_CONN_HOLDER; +/* Record information of a local (spider) table, for use of the spider +group by handler. */ typedef struct spider_table_holder { TABLE *table; ha_spider *spider; + /* alias of the table, in the form of tk, where k is the index of + the table from `query->from' indexed by next_local. */ spider_string *alias; } SPIDER_TABLE_HOLDER; -typedef struct spider_field_holder -{ - Field *field; - ha_spider *spider; - spider_string *alias; - spider_field_holder *next; -} SPIDER_FIELD_HOLDER; - -typedef struct spider_field_chain -{ - spider_field_holder *field_holder; - spider_field_chain *next; -} SPIDER_FIELD_CHAIN; - +/* For use of the spider group by handler. */ class spider_fields { uint dbton_count; uint current_dbton_num; uint dbton_ids[SPIDER_DBTON_SIZE]; + /* Number of tables in `query->from'. */ uint table_count; - uint current_table_num; + /* All tables in `query->from', in the same order by next_local. */ SPIDER_TABLE_HOLDER *table_holder; SPIDER_LINK_IDX_CHAIN *first_link_idx_chain; SPIDER_LINK_IDX_CHAIN *last_link_idx_chain; @@ -578,13 +570,6 @@ class spider_fields SPIDER_CONN_HOLDER *first_conn_holder; SPIDER_CONN_HOLDER *last_conn_holder; SPIDER_CONN_HOLDER *current_conn_holder; - SPIDER_FIELD_HOLDER *first_field_holder; - SPIDER_FIELD_HOLDER *last_field_holder; - SPIDER_FIELD_HOLDER *current_field_holder; - SPIDER_FIELD_CHAIN *first_field_chain; - SPIDER_FIELD_CHAIN *last_field_chain; - SPIDER_FIELD_CHAIN *current_field_chain; - Field **first_field_ptr; Field **current_field_ptr; public: spider_fields(); @@ -627,8 +612,6 @@ public: long access_balance ); SPIDER_CONN_HOLDER *create_conn_holder(); - void set_pos_to_first_conn_holder(); - SPIDER_CONN_HOLDER *get_next_conn_holder(); bool has_conn_holder(); void clear_conn_holder_from_conn(); bool check_conn_same_conn( @@ -642,24 +625,14 @@ public: void free_conn_holder( SPIDER_CONN_HOLDER *conn_holder_arg ); - SPIDER_TABLE_HOLDER *add_table( - ha_spider *spider_arg - ); - bool all_query_fields_are_query_table_members(); - int create_table_holder( + SPIDER_TABLE_HOLDER *find_table(Field *field); + void set_table_holder( + SPIDER_TABLE_HOLDER *table_holder_arg, uint table_count_arg ); - void set_pos_to_first_table_holder(); - SPIDER_TABLE_HOLDER *get_next_table_holder(); + SPIDER_TABLE_HOLDER *get_first_table_holder(); SPIDER_TABLE_HOLDER *get_table_holder(TABLE *table); uint get_table_count(); - int add_field(Field *field_arg); - SPIDER_FIELD_HOLDER *create_field_holder(); - void set_pos_to_first_field_holder(); - SPIDER_FIELD_HOLDER *get_next_field_holder(); - SPIDER_FIELD_CHAIN *create_field_chain(); - void set_pos_to_first_field_chain(); - SPIDER_FIELD_CHAIN *get_next_field_chain(); void set_field_ptr(Field **field_arg); Field **get_next_field_ptr(); int ping_table_mon_from_table( @@ -797,11 +770,6 @@ public: TABLE_LIST *table_list, uint table_count ) = 0; - virtual int reappend_tables( - spider_fields *fields, - SPIDER_LINK_IDX_CHAIN *link_idx_chain, - spider_string *str - ) = 0; virtual int append_where( spider_string *str ) = 0; @@ -1530,10 +1498,6 @@ public: spider_fields *fields, ulong sql_type ) = 0; - virtual int reappend_tables_part( - spider_fields *fields, - ulong sql_type - ) = 0; virtual int append_where_part( ulong sql_type ) = 0; @@ -1674,6 +1638,12 @@ static const LEX_CSTRING maturity_name[] = { STRING_WITH_LEN("Stable") } }; +/* + Type of singletons based on the type of the remote database. + + All such singletons are stored in the array `spider_dbton', see + `spider_db_init()'. +*/ typedef struct st_spider_dbton { uint dbton_id; diff --git a/storage/spider/spd_db_mysql.cc b/storage/spider/spd_db_mysql.cc index e0dffe57..c2f0c6b8 100644 --- a/storage/spider/spd_db_mysql.cc +++ b/storage/spider/spd_db_mysql.cc @@ -451,7 +451,7 @@ int spider_db_mbase_row::append_escaped_to_str( DBUG_ENTER("spider_db_mbase_row::append_escaped_to_str"); DBUG_PRINT("info",("spider this=%p", this)); spider_string tmp_str(*row, *lengths + 1, str->charset()); - tmp_str.init_calc_mem(133); + tmp_str.init_calc_mem(SPD_MID_DB_MBASE_ROW_APPEND_ESCAPED_TO_STR_1); tmp_str.length(*lengths); if (str->reserve(*lengths * 2 + 2)) DBUG_RETURN(HA_ERR_OUT_OF_MEM); @@ -544,7 +544,7 @@ SPIDER_DB_ROW *spider_db_mbase_row::clone() row_size = record_size + field_count; } if (!spider_bulk_malloc( - spider_current_trx, 29, MYF(MY_WME), + spider_current_trx, SPD_MID_DB_MBASE_ROW_CLONE_1, MYF(MY_WME), &clone_row->row, (uint) (sizeof(char*) * (field_count + 1)), &tmp_char, (uint) (row_size), &clone_row->lengths, (uint) (sizeof(ulong) * field_count), @@ -774,9 +774,9 @@ SPIDER_DB_ROW *spider_db_mbase_result::fetch_row_from_tmp_table( uint field_count; DBUG_ENTER("spider_db_mbase_result::fetch_row_from_tmp_table"); DBUG_PRINT("info",("spider this=%p", this)); - tmp_str1.init_calc_mem(117); - tmp_str2.init_calc_mem(118); - tmp_str3.init_calc_mem(170); + tmp_str1.init_calc_mem(SPD_MID_DB_MBASE_RESULT_FETCH_ROW_FROM_TMP_TABLE_1); + tmp_str2.init_calc_mem(SPD_MID_DB_MBASE_RESULT_FETCH_ROW_FROM_TMP_TABLE_2); + tmp_str3.init_calc_mem(SPD_MID_DB_MBASE_RESULT_FETCH_ROW_FROM_TMP_TABLE_3); tmp_table->field[0]->val_str(tmp_str1.get_str()); tmp_table->field[1]->val_str(tmp_str2.get_str()); tmp_table->field[2]->val_str(tmp_str3.get_str()); @@ -1529,10 +1529,13 @@ int spider_db_mbase_result::fetch_index_for_discover_table_structure( } DBUG_RETURN(0); } - if (num_fields() != 13) + if (num_fields() < 13) { - DBUG_PRINT("info",("spider num_fields != 13")); - my_printf_error(ER_SPIDER_UNKNOWN_NUM, ER_SPIDER_UNKNOWN_STR, MYF(0)); + DBUG_PRINT("info",("spider num_fields < 13")); + my_printf_error(ER_SPIDER_CANT_NUM, ER_SPIDER_CANT_STR1, MYF(0), + "fetch index for table structure discovery because of " + "wrong number of columns in SHOW INDEX FROM output: ", + num_fields()); DBUG_RETURN(ER_SPIDER_UNKNOWN_NUM); } bool first = TRUE; @@ -1826,7 +1829,7 @@ int spider_db_mbase::init() ) { DBUG_RETURN(HA_ERR_OUT_OF_MEM); } - spider_alloc_calc_mem_init(lock_table_hash, 140); + spider_alloc_calc_mem_init(lock_table_hash, SPD_MID_DB_MBASE_INIT_1); spider_alloc_calc_mem(spider_current_trx, lock_table_hash, lock_table_hash.array.max_element * @@ -1929,7 +1932,7 @@ int spider_db_mbase::connect( if (!spider_param_same_server_link(thd)) { - if (!strcmp(tgt_host, my_localhost)) + if (!strcmp(tgt_host, my_localhost) || !tgt_host || !tgt_host[0]) { if (!strcmp(tgt_socket, *spd_mysqld_unix_port)) { @@ -1939,7 +1942,7 @@ int spider_db_mbase::connect( DBUG_RETURN(ER_SPIDER_SAME_SERVER_LINK_NUM); } } else if (!strcmp(tgt_host, "127.0.0.1") || - !strcmp(tgt_host, glob_hostname)) + !strcmp(tgt_host, glob_hostname) || !tgt_host || !tgt_host[0]) { if (tgt_port == (long) *spd_mysqld_port) { @@ -2066,7 +2069,7 @@ int spider_db_mbase::exec_query( const char *tgt_str = conn->tgt_host; uint32 tgt_len = conn->tgt_host_length; spider_string tmp_query_str; - tmp_query_str.init_calc_mem(230); + tmp_query_str.init_calc_mem(SPD_MID_DB_MBASE_EXEC_QUERY_1); if (tmp_query_str.reserve( length + conn->tgt_wrapper_length + tgt_len + (SPIDER_SQL_SPACE_LEN * 2))) @@ -2096,7 +2099,7 @@ int spider_db_mbase::exec_query( struct tm lt; struct tm *l_time = localtime_r(&cur_time, <); spider_string tmp_query_str; - tmp_query_str.init_calc_mem(243); + tmp_query_str.init_calc_mem(SPD_MID_DB_MBASE_EXEC_QUERY_2); uint query_length = thd->query_length(); if ((log_result_error_with_sql & 2) && query_length) { @@ -2640,7 +2643,7 @@ int spider_db_mbase::xa_end( spider_string sql_str(sql_buf, sizeof(sql_buf), &my_charset_bin); DBUG_ENTER("spider_db_mbase::xa_end"); DBUG_PRINT("info",("spider this=%p", this)); - sql_str.init_calc_mem(108); + sql_str.init_calc_mem(SPD_MID_DB_MBASE_XA_END_1); sql_str.length(0); sql_str.q_append(SPIDER_SQL_XA_END_STR, SPIDER_SQL_XA_END_LEN); @@ -2683,7 +2686,7 @@ int spider_db_mbase::xa_prepare( spider_string sql_str(sql_buf, sizeof(sql_buf), &my_charset_bin); DBUG_ENTER("spider_db_mbase::xa_prepare"); DBUG_PRINT("info",("spider this=%p", this)); - sql_str.init_calc_mem(109); + sql_str.init_calc_mem(SPD_MID_DB_MBASE_XA_PREPARE_1); sql_str.length(0); sql_str.q_append(SPIDER_SQL_XA_PREPARE_STR, SPIDER_SQL_XA_PREPARE_LEN); @@ -2726,7 +2729,7 @@ int spider_db_mbase::xa_commit( spider_string sql_str(sql_buf, sizeof(sql_buf), &my_charset_bin); DBUG_ENTER("spider_db_mbase::xa_commit"); DBUG_PRINT("info",("spider this=%p", this)); - sql_str.init_calc_mem(110); + sql_str.init_calc_mem(SPD_MID_DB_MBASE_XA_COMMIT_1); sql_str.length(0); sql_str.q_append(SPIDER_SQL_XA_COMMIT_STR, SPIDER_SQL_XA_COMMIT_LEN); @@ -2769,7 +2772,7 @@ int spider_db_mbase::xa_rollback( spider_string sql_str(sql_buf, sizeof(sql_buf), &my_charset_bin); DBUG_ENTER("spider_db_mbase::xa_rollback"); DBUG_PRINT("info",("spider this=%p", this)); - sql_str.init_calc_mem(111); + sql_str.init_calc_mem(SPD_MID_DB_MBASE_XA_ROLLBACK_1); sql_str.length(0); sql_str.q_append(SPIDER_SQL_XA_ROLLBACK_STR, SPIDER_SQL_XA_ROLLBACK_LEN); @@ -3106,7 +3109,7 @@ int spider_db_mbase::set_wait_timeout( spider_string sql_str(sql_buf, sizeof(sql_buf), &my_charset_bin); DBUG_ENTER("spider_db_mbase::set_wait_timeout"); DBUG_PRINT("info",("spider this=%p", this)); - sql_str.init_calc_mem(264); + sql_str.init_calc_mem(SPD_MID_DB_MBASE_SET_WAIT_TIMEOUT_1); sql_str.length(0); timeout_str_length = my_sprintf(timeout_str, (timeout_str, "%d", wait_timeout)); @@ -3160,7 +3163,7 @@ int spider_db_mbase::set_sql_mode( spider_string sql_str(sql_buf, sizeof(sql_buf), &my_charset_bin); DBUG_ENTER("spider_db_mbase::set_sql_mode"); DBUG_PRINT("info",("spider this=%p", this)); - sql_str.init_calc_mem(265); + sql_str.init_calc_mem(SPD_MID_DB_MBASE_SET_SQL_MODE_1); sql_str.length(0); if (sql_str.reserve(SPIDER_SQL_SQL_MODE_LEN)) DBUG_RETURN(HA_ERR_OUT_OF_MEM); @@ -3225,7 +3228,7 @@ int spider_db_mbase::set_time_zone( spider_string sql_str(sql_buf, sizeof(sql_buf), &my_charset_bin); DBUG_ENTER("spider_db_mbase::set_time_zone"); DBUG_PRINT("info",("spider this=%p", this)); - sql_str.init_calc_mem(214); + sql_str.init_calc_mem(SPD_MID_DB_MBASE_SET_TIME_ZONE_1); sql_str.length(0); if (sql_str.reserve(SPIDER_SQL_TIME_ZONE_LEN + tz_str->length() + SPIDER_SQL_VALUE_QUOTE_LEN)) @@ -3278,7 +3281,7 @@ int spider_db_mbase::set_loop_check( spider_string sql_str(sql_buf, sizeof(sql_buf), &my_charset_bin); DBUG_ENTER("spider_db_mbase::set_loop_check"); DBUG_PRINT("info",("spider this=%p", this)); - sql_str.init_calc_mem(270); + sql_str.init_calc_mem(SPD_MID_DB_MBASE_SET_LOOP_CHECK_1); while ((lcptr = (SPIDER_CONN_LOOP_CHECK *) my_hash_element( &conn->loop_check_queue, 0))) { @@ -3960,7 +3963,7 @@ int spider_db_mariadb_util::append_column_value(ha_spider *spider, THD *thd = field->table->in_use; Time_zone *saved_time_zone = thd->variables.time_zone; DBUG_ENTER("spider_db_mariadb_util::append_column_value"); - tmp_str.init_calc_mem(113); + tmp_str.init_calc_mem(SPD_MID_DB_MARIADB_UTIL_APPEND_COLUMN_VALUE_1); thd->variables.time_zone = UTC; @@ -4036,7 +4039,7 @@ int spider_db_mariadb_util::append_column_value(ha_spider *spider, DBUG_PRINT("info", ("spider append_escaped")); char buf2[MAX_FIELD_WIDTH]; spider_string tmp_str2(buf2, MAX_FIELD_WIDTH, field->charset()); - tmp_str2.init_calc_mem(114); + tmp_str2.init_calc_mem(SPD_MID_DB_MARIADB_UTIL_APPEND_COLUMN_VALUE_2); tmp_str2.length(0); if ( tmp_str2.append(ptr->ptr(), ptr->length(), field->charset()) || @@ -4104,7 +4107,7 @@ int spider_db_mysql_util::append_column_value( THD *thd = field->table->in_use; Time_zone *saved_time_zone = thd->variables.time_zone; DBUG_ENTER("spider_db_mysql_util::append_column_value"); - tmp_str.init_calc_mem(266); + tmp_str.init_calc_mem(SPD_MID_DB_MYSQL_UTIL_APPEND_COLUMN_VALUE_1); thd->variables.time_zone = UTC; @@ -4255,7 +4258,7 @@ int spider_db_mysql_util::append_column_value( DBUG_PRINT("info", ("spider append_escaped")); char buf2[MAX_FIELD_WIDTH]; spider_string tmp_str2(buf2, MAX_FIELD_WIDTH, field->charset()); - tmp_str2.init_calc_mem(267); + tmp_str2.init_calc_mem(SPD_MID_DB_MYSQL_UTIL_APPEND_COLUMN_VALUE_2); tmp_str2.length(0); if ( tmp_str2.append(ptr->ptr(), ptr->length(), field->charset()) || @@ -5828,12 +5831,17 @@ int spider_db_mbase_util::print_item_func( item_count -= 2; break; } - } else if (func_name_length == 6 && - !strncasecmp("istrue", func_name, func_name_length) - ) { - last_str = SPIDER_SQL_IS_TRUE_STR; - last_str_length = SPIDER_SQL_IS_TRUE_LEN; - break; + } else if (func_name_length == 6) + { + if (!strncasecmp("istrue", func_name, func_name_length)) + { + last_str= SPIDER_SQL_IS_TRUE_STR; + last_str_length= SPIDER_SQL_IS_TRUE_LEN; + break; + } + else if (!strncasecmp("regexp", func_name, func_name_length)) + /* Keep the infix expression */ + break; } else if (func_name_length == 7) { if (!strncasecmp("isfalse", func_name, func_name_length)) @@ -6093,7 +6101,7 @@ int spider_db_mbase_util::print_item_func( char *tmp_ptr, *tmp_ptr2; DBUG_ASSERT(tmp_str.length() == 0); tmp_str.set_charset(str->charset()); - tmp_str.init_calc_mem(123); + tmp_str.init_calc_mem(SPD_MID_DB_MBASE_UTIL_PRINT_ITEM_FUNC_1); tmp_str.reserve(MAX_FIELD_WIDTH); str->length(str->length() - SPIDER_SQL_OPEN_PAREN_LEN); if (!merge_func) @@ -6218,7 +6226,7 @@ int spider_db_mbase_util::print_item_func( char *tmp_ptr, *tmp_ptr2; DBUG_ASSERT(tmp_str.length() == 0); tmp_str.set_charset(str->charset()); - tmp_str.init_calc_mem(124); + tmp_str.init_calc_mem(SPD_MID_DB_MBASE_UTIL_PRINT_ITEM_FUNC_2); tmp_str.reserve(MAX_FIELD_WIDTH); str->length(str->length() - SPIDER_SQL_OPEN_PAREN_LEN); if (!merge_func) @@ -6355,7 +6363,7 @@ int spider_db_mbase_util::print_item_func( char *tmp_ptr, *tmp_ptr2; DBUG_ASSERT(tmp_str.length() == 0); tmp_str.set_charset(str->charset()); - tmp_str.init_calc_mem(125); + tmp_str.init_calc_mem(SPD_MID_DB_MBASE_UTIL_PRINT_ITEM_FUNC_3); tmp_str.reserve(MAX_FIELD_WIDTH); str->length(str->length() - SPIDER_SQL_OPEN_PAREN_LEN); if (!merge_func) @@ -6803,440 +6811,250 @@ int spider_db_mbase_util::append_escaped_util( DBUG_RETURN(0); } -int spider_db_mbase_util::append_table( - ha_spider *spider, - spider_fields *fields, - spider_string *str, - TABLE_LIST *table_list, - TABLE_LIST **used_table_list, - uint *current_pos, - TABLE_LIST **cond_table_list_ptr, - bool top_down, - bool first -) { - int error_num; - bool use_cond_table_list = FALSE; - spider_mbase_share *db_share; - spider_mbase_handler *dbton_hdl; - SPIDER_TABLE_HOLDER *table_holder; - TABLE_LIST *cond_table_list = *cond_table_list_ptr; - ha_spider *spd; - DBUG_ENTER("spider_db_mbase_util::append_table"); - DBUG_PRINT("info",("spider table_list=%p", table_list)); - DBUG_PRINT("info",("spider table_list->outer_join=%u", - table_list->outer_join)); - DBUG_PRINT("info",("spider table_list->on_expr=%p", - table_list->on_expr)); - DBUG_PRINT("info",("spider table_list->join_using_fields=%p", - table_list->join_using_fields)); - DBUG_PRINT("info",("spider table_list->table=%p", - table_list->table)); - if (!top_down && table_list->embedding) - { - if ((error_num = append_embedding_tables(spider, fields, str, - table_list->embedding, used_table_list, current_pos, - cond_table_list_ptr))) +/* + Walk a TABLE_LIST, or format it to a string and append it. + + If str is NULL, walk the nested join (if any) to determine whether + to create a group by handler. Otherwise, format the TABLE_LIST to a + string and append it to str. + + Adapted from TABLE_LIST::print(). +*/ +int spider_db_mbase_util::append_table_list(spider_fields *fields, + spider_string *str, + TABLE_LIST *table, + table_map *upper_usable_tables, + table_map eliminated_tables) +{ + DBUG_ENTER("spider_db_mbase_util::append_table_list"); + /* Eliminated tables were removed from append_join(). */ + DBUG_ASSERT(!is_eliminated_table(eliminated_tables, table)); + if (!str) /* First pass (GBH creation) */ + { + DBUG_ASSERT(upper_usable_tables); + if (table->nested_join) + DBUG_RETURN(append_join(fields, str, + &table->nested_join->join_list, + upper_usable_tables, eliminated_tables)); + /* jtbm is a kind of semi join, and TABLE_LIST::print() adds an + extra " <materialize> " annotation. */ + if (table->jtbm_subselect) + DBUG_RETURN(ER_SPIDER_COND_SKIP_NUM); + /* TODO: These conditions are printed in a different way in + TABLE_LIST::print(), but they do not seem to occur very often. + Let's not worry about them now. */ + if (table->view_name.str || table->derived || table->table_function) + DBUG_RETURN(ER_SPIDER_COND_SKIP_NUM); + *upper_usable_tables |= table->table->map; + DBUG_RETURN(0); + } + /* Second pass (query execution) */ + DBUG_ASSERT(!upper_usable_tables); + if (table->nested_join) + { + if (str->append("(")) + DBUG_RETURN(HA_ERR_OUT_OF_MEM); + if (int error_num= append_join( + fields, str, &table->nested_join->join_list, + upper_usable_tables, eliminated_tables)) DBUG_RETURN(error_num); - } else if (!table_list->table) + if (str->append(")")) + DBUG_RETURN(HA_ERR_OUT_OF_MEM); + DBUG_RETURN(0); + } + /* These have been ruled out during the creation of the group by + handler, see above. */ + DBUG_ASSERT(!table->jtbm_subselect); + DBUG_ASSERT(!table->view_name.str); + DBUG_ASSERT(!table->derived); + DBUG_ASSERT(!table->table_function); + /* We have a "normal" table. Print it and append to str. */ + SPIDER_TABLE_HOLDER *table_holder = fields->get_table_holder(table->table); + ha_spider *spd = table_holder->spider; + spider_mbase_share *db_share = + (spider_mbase_share *) spd->share->dbton_share[dbton_id]; + spider_mbase_handler *dbton_hdl = + (spider_mbase_handler *) spd->dbton_handler[dbton_id]; + if (table->table->const_table) { - if ((error_num = append_tables_top_down(spider, fields, str, table_list, - used_table_list, current_pos, cond_table_list_ptr))) + if (str->append(STRING_WITH_LEN("(select 1)"))) + DBUG_RETURN(HA_ERR_OUT_OF_MEM); + } + else + if (int error_num= db_share->append_table_name( + str, spd->conn_link_idx[dbton_hdl->first_link_idx])) DBUG_RETURN(error_num); - } else { - if ( - table_list->outer_join || - table_list->on_expr || - table_list->join_using_fields - ) { - DBUG_PRINT("info",("spider use table_list")); - if (table_list->outer_join & JOIN_TYPE_LEFT) - { - if (str) - { - if (str->reserve(SPIDER_SQL_LEFT_JOIN_LEN)) - { - DBUG_RETURN(HA_ERR_OUT_OF_MEM); - } - str->q_append(SPIDER_SQL_LEFT_JOIN_STR, SPIDER_SQL_LEFT_JOIN_LEN); - } - } else { - if (str) - { - if (str->reserve(SPIDER_SQL_JOIN_LEN)) - { - DBUG_RETURN(HA_ERR_OUT_OF_MEM); - } - str->q_append(SPIDER_SQL_JOIN_STR, SPIDER_SQL_JOIN_LEN); - } - } - } else if ( - cond_table_list && - ( - cond_table_list->outer_join || - cond_table_list->on_expr || - cond_table_list->join_using_fields - ) - ) { - DBUG_PRINT("info",("spider use cond_table_list")); - if (cond_table_list->outer_join & (JOIN_TYPE_LEFT | JOIN_TYPE_RIGHT)) - { - if (str) - { - if (str->reserve(SPIDER_SQL_LEFT_JOIN_LEN)) - { - DBUG_RETURN(HA_ERR_OUT_OF_MEM); - } - str->q_append(SPIDER_SQL_LEFT_JOIN_STR, SPIDER_SQL_LEFT_JOIN_LEN); - } - } else { - if (str) - { - if (str->reserve(SPIDER_SQL_JOIN_LEN)) - { - DBUG_RETURN(HA_ERR_OUT_OF_MEM); - } - str->q_append(SPIDER_SQL_JOIN_STR, SPIDER_SQL_JOIN_LEN); - } - } - use_cond_table_list = TRUE; - } else if (*current_pos > 0 && !first) - { - DBUG_PRINT("info",("spider no condition")); - if (str) - { - if (str->reserve(SPIDER_SQL_JOIN_LEN)) - { - DBUG_RETURN(HA_ERR_OUT_OF_MEM); - } - str->q_append(SPIDER_SQL_JOIN_STR, SPIDER_SQL_JOIN_LEN); - } - } - - if (str) - { - table_holder = fields->get_table_holder(table_list->table); - spd = table_holder->spider; - db_share = (spider_mbase_share *) - spd->share->dbton_share[dbton_id]; - dbton_hdl = (spider_mbase_handler *) - spd->dbton_handler[dbton_id]; + if (str->append(" ") || + str->append(table_holder->alias->ptr(), + /* Don't append the trailing dot */ + table_holder->alias->length() - 1)) + DBUG_RETURN(HA_ERR_OUT_OF_MEM); + DBUG_RETURN(0); +} - dbton_hdl->table_name_pos = str->length(); +/* + Walk an array of TABLE_LIST's, or format it to a string and append it. - if (str->reserve( - db_share->db_nm_max_length + - SPIDER_SQL_DOT_LEN + /* SPIDER_SQL_NAME_QUOTE_LEN */ 4 + - db_share->table_nm_max_length + SPIDER_SQL_SPACE_LEN + - table_holder->alias->length() - SPIDER_SQL_DOT_LEN - )) { - DBUG_RETURN(HA_ERR_OUT_OF_MEM); - } + If str is NULL, walk each TABLE_LIST to determine whether to create + a group by handler. Otherwise, format the TABLE_LISTs to a string + and append it to str. - if ((error_num = db_share->append_table_name_with_adjusting(str, - spd->conn_link_idx[dbton_hdl->first_link_idx]))) - { - DBUG_RETURN(error_num); - } - str->q_append(SPIDER_SQL_SPACE_STR, SPIDER_SQL_SPACE_LEN); - str->q_append(table_holder->alias->ptr(), - table_holder->alias->length() - SPIDER_SQL_DOT_LEN); - } - used_table_list[(*current_pos)++] = table_list; + Adapted from print_table_array(). +*/ +int spider_db_mbase_util::append_table_array(spider_fields *fields, + spider_string *str, + TABLE_LIST **table, + TABLE_LIST **end, + table_map *upper_usable_tables, + table_map eliminated_tables) +{ + DBUG_ENTER("spider_db_mbase_util::append_table_array"); + if (str) + { + DBUG_ASSERT(!upper_usable_tables); + if (int error_num= append_table_list(fields, str, *table, NULL, + eliminated_tables)) + DBUG_RETURN(error_num); - if (str) + for (TABLE_LIST **tbl= table + 1; tbl < end; tbl++) { - List<String> *join_using_fields = table_list->join_using_fields; - if (!join_using_fields && cond_table_list) - { - join_using_fields = cond_table_list->join_using_fields; - } + TABLE_LIST *curr= *tbl; - if (join_using_fields) + /* JOIN_TYPE_OUTER is just a marker unrelated to real join */ + if (curr->outer_join & (JOIN_TYPE_LEFT|JOIN_TYPE_RIGHT)) { - if (str->reserve(SPIDER_SQL_USING_LEN + SPIDER_SQL_OPEN_PAREN_LEN)) - { + /* MySQL converts right to left joins */ + if (str->append(STRING_WITH_LEN(" left join "))) DBUG_RETURN(HA_ERR_OUT_OF_MEM); - } - str->q_append(SPIDER_SQL_USING_STR, SPIDER_SQL_USING_LEN); - str->q_append(SPIDER_SQL_OPEN_PAREN_STR, - SPIDER_SQL_OPEN_PAREN_LEN); - List_iterator_fast<String> it2(*join_using_fields); - String *ptr; - while ((ptr = it2++)) - { - if (str->reserve(ptr->length() + SPIDER_SQL_COMMA_LEN)) - { - DBUG_RETURN(HA_ERR_OUT_OF_MEM); - } - str->q_append(ptr->ptr(), ptr->length()); - str->q_append(SPIDER_SQL_COMMA_STR, SPIDER_SQL_COMMA_LEN); - } - str->length(str->length() - SPIDER_SQL_COMMA_LEN); - if (str->reserve(SPIDER_SQL_CLOSE_PAREN_LEN)) - { - DBUG_RETURN(HA_ERR_OUT_OF_MEM); - } - str->q_append(SPIDER_SQL_CLOSE_PAREN_STR, - SPIDER_SQL_CLOSE_PAREN_LEN); } - } - - Item *on_expr = table_list->on_expr; - if (!on_expr && cond_table_list) - { - on_expr = cond_table_list->on_expr; - } - - if (on_expr) - { - if (str) + else if (curr->straight) { - if (str->reserve(SPIDER_SQL_ON_LEN)) - { + if (str->append(STRING_WITH_LEN(" straight_join "))) DBUG_RETURN(HA_ERR_OUT_OF_MEM); - } - str->q_append(SPIDER_SQL_ON_STR, SPIDER_SQL_ON_LEN); } - if ((error_num = spider_db_print_item_type(on_expr, NULL, - spider, str, NULL, 0, dbton_id, TRUE, fields))) - { - DBUG_RETURN(error_num); - } - } + /* semi join should already have been ruled out during the + creation of the group by handler. */ + else if (curr->sj_inner_tables) + DBUG_ASSERT(0); + else + if (str->append(STRING_WITH_LEN(" join "))) + DBUG_RETURN(HA_ERR_OUT_OF_MEM); - if (use_cond_table_list) - { - (*cond_table_list_ptr) = NULL; - DBUG_PRINT("info",("spider cond_table_list=%p", (*cond_table_list_ptr))); - } - } - DBUG_RETURN(0); -} + if (int error_num= append_table_list(fields, str, curr, NULL, + eliminated_tables)) + DBUG_RETURN(error_num); -int spider_db_mbase_util::append_tables_top_down( - ha_spider *spider, - spider_fields *fields, - spider_string *str, - TABLE_LIST *table_list, - TABLE_LIST **used_table_list, - uint *current_pos, - TABLE_LIST **cond_table_list_ptr -) { - int error_num; - uint outer_join_backup; - TABLE_LIST *cur_table_list, *prev_table_list = NULL, *cond_table_list = NULL; - bool first = TRUE; - DBUG_ENTER("spider_db_mbase_util::append_tables_top_down"); - DBUG_PRINT("info",("spider this=%p", this)); - if ( - table_list->outer_join || - table_list->on_expr || - table_list->join_using_fields - ) { - DBUG_ASSERT(!(*cond_table_list_ptr)); - (*cond_table_list_ptr) = table_list; - DBUG_PRINT("info",("spider cond_table_list=%p", table_list)); - } - List_iterator_fast<TABLE_LIST> it1(table_list->nested_join->join_list); - cur_table_list = it1++; - if (cur_table_list->outer_join & JOIN_TYPE_RIGHT) - { - first = FALSE; - prev_table_list = cur_table_list; - cur_table_list = it1++; - } else if (*cond_table_list_ptr) - { - first = TRUE; - cond_table_list = (*cond_table_list_ptr); - (*cond_table_list_ptr) = NULL; - if (cond_table_list->outer_join & JOIN_TYPE_LEFT) - { - if (str) + if (curr->on_expr) { - if (str->reserve(SPIDER_SQL_LEFT_JOIN_LEN + SPIDER_SQL_OPEN_PAREN_LEN)) - { + if (str->append(STRING_WITH_LEN(" on "))) DBUG_RETURN(HA_ERR_OUT_OF_MEM); - } - str->q_append(SPIDER_SQL_LEFT_JOIN_STR, SPIDER_SQL_LEFT_JOIN_LEN); - str->q_append(SPIDER_SQL_OPEN_PAREN_STR, SPIDER_SQL_OPEN_PAREN_LEN); - } - } else { - if (str) - { - if (str->reserve(SPIDER_SQL_JOIN_LEN + SPIDER_SQL_OPEN_PAREN_LEN)) - { - DBUG_RETURN(HA_ERR_OUT_OF_MEM); - } - str->q_append(SPIDER_SQL_JOIN_STR, SPIDER_SQL_JOIN_LEN); - str->q_append(SPIDER_SQL_OPEN_PAREN_STR, SPIDER_SQL_OPEN_PAREN_LEN); + if (int error_num= + spider_db_print_item_type(curr->on_expr, NULL, + fields->get_first_table_holder()->spider, + str, NULL, 0, dbton_id, TRUE, fields)) + DBUG_RETURN(error_num); } } } - - do { - if (cur_table_list->outer_join & JOIN_TYPE_RIGHT) + else /* str == NULL */ + { + table_map usable_tables= 0; + if (int error_num= append_table_list(fields, str, *table, + &usable_tables, eliminated_tables)) + DBUG_RETURN(error_num); + for (TABLE_LIST **tbl= table + 1; tbl < end; tbl++) { - prev_table_list = cur_table_list; - } else { - if ((error_num = append_table(spider, fields, str, cur_table_list, - used_table_list, current_pos, cond_table_list_ptr, TRUE, first))) + TABLE_LIST *curr= *tbl; + /* semi join is an "internal" join and is unsupported. */ + if (curr->sj_inner_tables) + DBUG_RETURN(ER_SPIDER_COND_SKIP_NUM); + if (int error_num= append_table_list(fields, str, curr, + &usable_tables, eliminated_tables)) DBUG_RETURN(error_num); - first = FALSE; - if (prev_table_list) + if (curr->on_expr) { - outer_join_backup = prev_table_list->outer_join; - prev_table_list->outer_join = JOIN_TYPE_LEFT; - if ((error_num = append_table(spider, fields, str, prev_table_list, - used_table_list, current_pos, cond_table_list_ptr, TRUE, FALSE))) - { - prev_table_list->outer_join = outer_join_backup; + /* The join refers to fields outside of the current context, + and cannot be handled by a group by handler. */ + if ((curr->on_expr->used_tables() & usable_tables) != + curr->on_expr->used_tables()) + DBUG_RETURN(ER_SPIDER_COND_SKIP_NUM); + if (int error_num= + spider_db_print_item_type(curr->on_expr, NULL, + fields->get_first_table_holder()->spider, + str, NULL, 0, dbton_id, TRUE, fields)) DBUG_RETURN(error_num); - } - prev_table_list->outer_join = outer_join_backup; - prev_table_list = NULL; - } - } - } while ((cur_table_list = it1++)); - - if (cond_table_list) - { - if (str) - { - if (str->reserve(SPIDER_SQL_CLOSE_PAREN_LEN)) - { - DBUG_RETURN(HA_ERR_OUT_OF_MEM); - } - str->q_append(SPIDER_SQL_CLOSE_PAREN_STR, - SPIDER_SQL_CLOSE_PAREN_LEN); - - List<String> *join_using_fields = cond_table_list->join_using_fields; - if (join_using_fields) - { - if (str->reserve(SPIDER_SQL_USING_LEN + SPIDER_SQL_OPEN_PAREN_LEN)) - { - DBUG_RETURN(HA_ERR_OUT_OF_MEM); - } - str->q_append(SPIDER_SQL_USING_STR, SPIDER_SQL_USING_LEN); - str->q_append(SPIDER_SQL_OPEN_PAREN_STR, - SPIDER_SQL_OPEN_PAREN_LEN); - List_iterator_fast<String> it2(*join_using_fields); - String *ptr; - while ((ptr = it2++)) - { - if (str->reserve(ptr->length() + SPIDER_SQL_COMMA_LEN)) - { - DBUG_RETURN(HA_ERR_OUT_OF_MEM); - } - str->q_append(ptr->ptr(), ptr->length()); - str->q_append(SPIDER_SQL_COMMA_STR, SPIDER_SQL_COMMA_LEN); - } - str->length(str->length() - SPIDER_SQL_COMMA_LEN); - if (str->reserve(SPIDER_SQL_CLOSE_PAREN_LEN)) - { - DBUG_RETURN(HA_ERR_OUT_OF_MEM); - } - str->q_append(SPIDER_SQL_CLOSE_PAREN_STR, - SPIDER_SQL_CLOSE_PAREN_LEN); - } - } - - Item *on_expr = cond_table_list->on_expr; - if (on_expr) - { - if (str) - { - if (str->reserve(SPIDER_SQL_ON_LEN)) - { - DBUG_RETURN(HA_ERR_OUT_OF_MEM); - } - str->q_append(SPIDER_SQL_ON_STR, SPIDER_SQL_ON_LEN); - } - if ((error_num = spider_db_print_item_type(on_expr, NULL, - spider, str, NULL, 0, dbton_id, TRUE, fields))) - { - DBUG_RETURN(error_num); } } + /* Update usable tables in the outer context. */ + if (upper_usable_tables) + *upper_usable_tables |= usable_tables; } DBUG_RETURN(0); } -int spider_db_mbase_util::append_tables_top_down_check( - TABLE_LIST *table_list, - TABLE_LIST **used_table_list, - uint *current_pos -) { - int error_num; - TABLE_LIST *cur_table_list; - DBUG_ENTER("spider_db_mbase_util::append_tables_top_down_check"); - DBUG_PRINT("info",("spider this=%p", this)); - List_iterator_fast<TABLE_LIST> it1(table_list->nested_join->join_list); - while ((cur_table_list = it1++)) +/* + Walk a join, or format it to a string and append the string. + + Skip all eliminated tables. + + If str is NULL, walk the tables to determine whether to create a + group by handler. Otherwise, format the join to a string and append + it to str. + + Adapted from print_join(). +*/ +int spider_db_mbase_util::append_join(spider_fields *fields, + spider_string *str, + List<TABLE_LIST> *tables, + table_map *upper_usable_tables, + table_map eliminated_tables) +{ + /* List is reversed => we should reverse it before using */ + List_iterator_fast<TABLE_LIST> ti(*tables); + TABLE_LIST **table; + THD *thd= fields->get_first_table_holder()->spider->wide_handler->trx->thd; + DBUG_ENTER("spider_db_mbase_util::append_join"); + + size_t tables_to_print= 0; + + for (TABLE_LIST *t= ti++; t ; t= ti++) { - if (!cur_table_list->table) - { - if ((error_num = append_tables_top_down_check( - cur_table_list, used_table_list, current_pos))) - DBUG_RETURN(error_num); - } else { - used_table_list[(*current_pos)++] = cur_table_list; - } + /* optimized_away implies const_table */ + DBUG_ASSERT(!t->optimized_away || t->table->const_table); + if (!is_eliminated_table(eliminated_tables, t)) + tables_to_print++; } - DBUG_RETURN(0); -} + if (tables_to_print == 0) + { + if (str && str->append(STRING_WITH_LEN("dual"))) + DBUG_RETURN(HA_ERR_OUT_OF_MEM); + DBUG_RETURN(0); + } + ti.rewind(); -int spider_db_mbase_util::append_embedding_tables( - ha_spider *spider, - spider_fields *fields, - spider_string *str, - TABLE_LIST *table_list, - TABLE_LIST **used_table_list, - uint *current_pos, - TABLE_LIST **cond_table_list_ptr -) { - int error_num; - TABLE_LIST *embedding = table_list->embedding; - DBUG_ENTER("spider_db_mbase_util::append_embedding_tables"); - DBUG_PRINT("info",("spider this=%p", this)); - if (embedding) - { - DBUG_PRINT("info",("spider embedding=%p", embedding)); - DBUG_PRINT("info",("spider embedding->outer_join=%u", - embedding->outer_join)); - DBUG_PRINT("info",("spider embedding->on_expr=%p", - embedding->on_expr)); - DBUG_PRINT("info",("spider embedding->join_using_fields=%p", - embedding->join_using_fields)); - DBUG_PRINT("info",("spider embedding->table=%p", - embedding->table)); - if ((error_num = append_embedding_tables(spider, fields, str, embedding, - used_table_list, current_pos, cond_table_list_ptr))) - DBUG_RETURN(error_num); - } else { - DBUG_PRINT("info",("spider table_list=%p", table_list)); - DBUG_PRINT("info",("spider table_list->outer_join=%u", - table_list->outer_join)); - DBUG_PRINT("info",("spider table_list->on_expr=%p", - table_list->on_expr)); - DBUG_PRINT("info",("spider table_list->join_using_fields=%p", - table_list->join_using_fields)); - DBUG_PRINT("info",("spider table_list->table=%p", - table_list->table)); - if (table_list->outer_join & JOIN_TYPE_RIGHT) - { - if ((error_num = append_tables_top_down_check(table_list, - used_table_list, current_pos))) - DBUG_RETURN(error_num); - DBUG_ASSERT(!(*cond_table_list_ptr)); - (*cond_table_list_ptr) = table_list; - DBUG_PRINT("info",("spider cond_table_list=%p", table_list)); - } else { - if ((error_num = append_tables_top_down(spider, fields, str, table_list, - used_table_list, current_pos, cond_table_list_ptr))) - DBUG_RETURN(error_num); - } + if (!(table= static_cast<TABLE_LIST **>(thd->alloc(sizeof(TABLE_LIST*) * + tables_to_print)))) + DBUG_RETURN(HA_ERR_OUT_OF_MEM); + + TABLE_LIST *tmp, **t= table + (tables_to_print - 1); + while ((tmp= ti++)) + if (!is_eliminated_table(eliminated_tables, tmp)) + *t--= tmp; + + DBUG_ASSERT(tables->elements >= 1); + if ((*table)->sj_inner_tables) + { + /* Semi join is not supported. */ + if (!str) + DBUG_RETURN(ER_SPIDER_COND_SKIP_NUM); + /* Semi join should have been skipped in the first pass. */ + else + DBUG_ASSERT(0); } - DBUG_RETURN(0); + int error_num= append_table_array( + fields, str, table, table + tables_to_print, upper_usable_tables, + eliminated_tables); + DBUG_RETURN(error_num); } int spider_db_mbase_util::append_from_and_tables( @@ -7245,116 +7063,17 @@ int spider_db_mbase_util::append_from_and_tables( spider_string *str, TABLE_LIST *table_list, uint table_count -) { - int error_num; - uint current_pos = 0, roop_count, backup_pos, outer_join_backup; - TABLE *table; - TABLE_LIST **used_table_list, *prev_table_list = NULL, - *cond_table_list = NULL; +) +{ DBUG_ENTER("spider_db_mbase_util::append_from_and_tables"); - DBUG_PRINT("info",("spider this=%p", this)); - used_table_list = (TABLE_LIST **) - my_alloca(sizeof(TABLE_LIST *) * table_count); - if (!used_table_list) + if (str && str->append(" from ")) DBUG_RETURN(HA_ERR_OUT_OF_MEM); - - if (str) - { - if (str->reserve(SPIDER_SQL_FROM_LEN)) - { - my_afree(used_table_list); - DBUG_RETURN(HA_ERR_OUT_OF_MEM); - } - str->q_append(SPIDER_SQL_FROM_STR, SPIDER_SQL_FROM_LEN); - } - - do { - table = table_list->table; - if (table->const_table) - continue; - - for (roop_count = 0; roop_count < current_pos; ++roop_count) - { - if (used_table_list[roop_count] == table_list) - break; - } - if (roop_count < current_pos) - continue; - - if (prev_table_list) - current_pos = backup_pos; - else - backup_pos = current_pos; - if ((error_num = append_table(spider, fields, str, table_list, used_table_list, - ¤t_pos, &cond_table_list, FALSE, FALSE))) - { - my_afree(used_table_list); - DBUG_RETURN(error_num); - } - if (prev_table_list) - { - outer_join_backup = prev_table_list->outer_join; - prev_table_list->outer_join = JOIN_TYPE_LEFT; - if ((error_num = append_table(spider, fields, str, prev_table_list, - used_table_list, ¤t_pos, &cond_table_list, FALSE, FALSE))) - { - prev_table_list->outer_join = outer_join_backup; - my_afree(used_table_list); - DBUG_RETURN(error_num); - } - prev_table_list->outer_join = outer_join_backup; - prev_table_list = NULL; - } - if (cond_table_list && (cond_table_list->outer_join & JOIN_TYPE_RIGHT)) - { - prev_table_list = cond_table_list; - cond_table_list = NULL; - DBUG_PRINT("info",("spider cond_table_list=%p", cond_table_list)); - } - } while ((table_list = table_list->next_local)); - my_afree(used_table_list); - DBUG_RETURN(0); -} - -int spider_db_mbase_util::reappend_tables( - spider_fields *fields, - SPIDER_LINK_IDX_CHAIN *link_idx_chain, - spider_string *str -) { - int error_num; - uint32 length; - ha_spider *spider; - spider_mbase_share *db_share; - spider_mbase_handler *dbton_hdl; - SPIDER_TABLE_HOLDER *table_holder; - SPIDER_LINK_IDX_HOLDER *link_idx_holder; - DBUG_ENTER("spider_db_mbase_util::reappend_tables"); - DBUG_PRINT("info",("spider this=%p", this)); - length = str->length(); - fields->set_pos_to_first_table_on_link_idx_chain(link_idx_chain); - fields->set_pos_to_first_table_holder(); - while ((table_holder = fields->get_next_table_holder())) - { - link_idx_holder = - fields->get_next_table_on_link_idx_chain(link_idx_chain); - spider = table_holder->spider; - db_share = (spider_mbase_share *) - spider->share->dbton_share[dbton_id]; - if (!db_share->same_db_table_name) - { - dbton_hdl = (spider_mbase_handler *) spider->dbton_handler[dbton_id]; - str->length(dbton_hdl->table_name_pos); - if ((error_num = db_share->append_table_name_with_adjusting(str, - spider->conn_link_idx[link_idx_holder->link_idx]))) - { - DBUG_RETURN(error_num); - } - } - } - str->length(length); - DBUG_RETURN(0); + const table_map eliminated_tables= table_list->select_lex->join ? + table_list->select_lex->join->eliminated_tables : 0; + int error_num = append_join(fields, str, table_list->select_lex->join_list, + NULL, eliminated_tables); + DBUG_RETURN(error_num); } - int spider_db_mbase_util::append_where( spider_string *str ) { @@ -7455,7 +7174,7 @@ spider_mbase_share::spider_mbase_share( { DBUG_ENTER("spider_mbase_share::spider_mbase_share"); DBUG_PRINT("info",("spider this=%p", this)); - spider_alloc_calc_mem_init(mem_calc, 71); + spider_alloc_calc_mem_init(mem_calc, SPD_MID_MBASE_SHARE_SPIDER_MBASE_SHARE_1); spider_alloc_calc_mem(spider_current_trx, mem_calc, sizeof(*this)); DBUG_VOID_RETURN; } @@ -7529,7 +7248,7 @@ int spider_mbase_share::init() DBUG_ENTER("spider_mbase_share::init"); DBUG_PRINT("info",("spider this=%p", this)); if (!(key_select_pos = (int *) - spider_bulk_alloc_mem(spider_current_trx, 112, + spider_bulk_alloc_mem(spider_current_trx, SPD_MID_MBASE_SHARE_INIT_1, __func__, __FILE__, __LINE__, MYF(MY_WME | MY_ZEROFILL), &key_select_pos, sizeof(int) * keys, @@ -7547,7 +7266,7 @@ int spider_mbase_share::init() } for (roop_count = 0; roop_count < keys; roop_count++) { - key_hint[roop_count].init_calc_mem(189); + key_hint[roop_count].init_calc_mem(SPD_MID_MBASE_SHARE_INIT_2); key_hint[roop_count].set_charset(spider_share->access_charset); } DBUG_PRINT("info",("spider key_hint=%p", key_hint)); @@ -7571,13 +7290,13 @@ int spider_mbase_share::init() DBUG_RETURN(HA_ERR_OUT_OF_MEM); } - table_select->init_calc_mem(96); + table_select->init_calc_mem(SPD_MID_MBASE_SHARE_INIT_3); if (table_share && (error_num = append_table_select())) DBUG_RETURN(error_num); for (roop_count = 0; roop_count < keys; roop_count++) { - key_select[roop_count].init_calc_mem(97); + key_select[roop_count].init_calc_mem(SPD_MID_MBASE_SHARE_INIT_4); if ((error_num = append_key_select(roop_count))) DBUG_RETURN(error_num); } @@ -7728,9 +7447,9 @@ int spider_mbase_share::create_table_names_str() for (roop_count = 0; roop_count < (int) spider_share->all_link_count; roop_count++) { - table_names_str[roop_count].init_calc_mem(86); - db_names_str[roop_count].init_calc_mem(87); - db_table_str[roop_count].init_calc_mem(88); + table_names_str[roop_count].init_calc_mem(SPD_MID_MBASE_SHARE_CREATE_TABLE_NAMES_STR_1); + db_names_str[roop_count].init_calc_mem(SPD_MID_MBASE_SHARE_CREATE_TABLE_NAMES_STR_2); + db_table_str[roop_count].init_calc_mem(SPD_MID_MBASE_SHARE_CREATE_TABLE_NAMES_STR_3); if (spider_share->sql_dbton_ids[roop_count] != dbton_id) continue; if (first_all_link_idx == -1) @@ -7869,7 +7588,7 @@ int spider_mbase_share::create_column_name_str() for (field = table_share->field, str = column_name_str; *field; field++, str++) { - str->init_calc_mem(89); + str->init_calc_mem(SPD_MID_MBASE_SHARE_CREATE_COLUMN_NAME_STR_1); str->set_charset(spider_share->access_charset); if ((error_num = spider_db_append_name_with_quote_str(str, (*field)->field_name, dbton_id))) @@ -7937,8 +7656,8 @@ int spider_mbase_share::append_show_table_status() for (roop_count = 0; roop_count < (int) spider_share->all_link_count; roop_count++) { - show_table_status[0 + (2 * roop_count)].init_calc_mem(90); - show_table_status[1 + (2 * roop_count)].init_calc_mem(91); + show_table_status[0 + (2 * roop_count)].init_calc_mem(SPD_MID_MBASE_SHARE_APPEND_SHOW_TABLE_STATUS_1); + show_table_status[1 + (2 * roop_count)].init_calc_mem(SPD_MID_MBASE_SHARE_APPEND_SHOW_TABLE_STATUS_2); if (spider_share->sql_dbton_ids[roop_count] != dbton_id) continue; @@ -8018,7 +7737,7 @@ int spider_mbase_share::append_show_records() for (roop_count = 0; roop_count < (int) spider_share->all_link_count; roop_count++) { - show_records[roop_count].init_calc_mem(92); + show_records[roop_count].init_calc_mem(SPD_MID_MBASE_SHARE_APPEND_SHOW_RECORDS_1); if (spider_share->sql_dbton_ids[roop_count] != dbton_id) continue; @@ -8068,8 +7787,8 @@ int spider_mbase_share::append_show_index() for (roop_count = 0; roop_count < (int) spider_share->all_link_count; roop_count++) { - show_index[0 + (2 * roop_count)].init_calc_mem(93); - show_index[1 + (2 * roop_count)].init_calc_mem(94); + show_index[0 + (2 * roop_count)].init_calc_mem(SPD_MID_MBASE_SHARE_APPEND_SHOW_INDEX_1); + show_index[1 + (2 * roop_count)].init_calc_mem(SPD_MID_MBASE_SHARE_APPEND_SHOW_INDEX_2); if (spider_share->sql_dbton_ids[roop_count] != dbton_id) continue; @@ -8202,7 +7921,7 @@ int spider_mbase_share::discover_table_structure( uint strlen = str->length(); DBUG_ENTER("spider_mbase_share::discover_table_structure"); DBUG_PRINT("info",("spider this=%p", this)); - sql_str.init_calc_mem(228); + sql_str.init_calc_mem(SPD_MID_MBASE_SHARE_DISCOVER_TABLE_STRUCTURE_1); for (roop_count = 0; roop_count < (int) spider_share->all_link_count; roop_count++) { @@ -8536,7 +8255,7 @@ spider_mbase_handler::spider_mbase_handler( { DBUG_ENTER("spider_mbase_handler::spider_mbase_handler"); DBUG_PRINT("info",("spider this=%p", this)); - spider_alloc_calc_mem_init(mem_calc, 183); + spider_alloc_calc_mem_init(mem_calc, SPD_MID_MBASE_HANDLER_SPIDER_MBASE_HANDLER_1); spider_alloc_calc_mem(spider_current_trx, mem_calc, sizeof(*this)); DBUG_VOID_RETURN; } @@ -8609,14 +8328,14 @@ int spider_mbase_handler::init() TABLE *table = spider->get_table(); DBUG_ENTER("spider_mbase_handler::init"); DBUG_PRINT("info",("spider this=%p", this)); - sql.init_calc_mem(59); - sql_part.init_calc_mem(60); - sql_part2.init_calc_mem(61); - ha_sql.init_calc_mem(62); - insert_sql.init_calc_mem(64); - update_sql.init_calc_mem(65); - tmp_sql.init_calc_mem(66); - dup_update_sql.init_calc_mem(166); + sql.init_calc_mem(SPD_MID_MBASE_HANDLER_INIT_1); + sql_part.init_calc_mem(SPD_MID_MBASE_HANDLER_INIT_2); + sql_part2.init_calc_mem(SPD_MID_MBASE_HANDLER_INIT_3); + ha_sql.init_calc_mem(SPD_MID_MBASE_HANDLER_INIT_4); + insert_sql.init_calc_mem(SPD_MID_MBASE_HANDLER_INIT_5); + update_sql.init_calc_mem(SPD_MID_MBASE_HANDLER_INIT_6); + tmp_sql.init_calc_mem(SPD_MID_MBASE_HANDLER_INIT_7); + dup_update_sql.init_calc_mem(SPD_MID_MBASE_HANDLER_INIT_8); if ( (sql.real_alloc(init_sql_alloc_size)) || (insert_sql.real_alloc(init_sql_alloc_size)) || @@ -8636,7 +8355,7 @@ int spider_mbase_handler::init() upd_tmp_tbl_prm.init(); upd_tmp_tbl_prm.field_count = 1; if (!(link_for_hash = (SPIDER_LINK_FOR_HASH *) - spider_bulk_alloc_mem(spider_current_trx, 141, + spider_bulk_alloc_mem(spider_current_trx, SPD_MID_MBASE_HANDLER_INIT_9, __func__, __FILE__, __LINE__, MYF(MY_WME | MY_ZEROFILL), &link_for_hash, sizeof(SPIDER_LINK_FOR_HASH) * share->link_count, @@ -8740,7 +8459,7 @@ int spider_mbase_handler::append_key_column_types( spider_string tmp_str(tmp_buf, sizeof(tmp_buf), system_charset_info); DBUG_ENTER("spider_mbase_handler::append_key_column_types"); DBUG_PRINT("info",("spider this=%p", this)); - tmp_str.init_calc_mem(115); + tmp_str.init_calc_mem(SPD_MID_MBASE_HANDLER_APPEND_KEY_COLUMN_TYPES_1); start_key_part_map = start_key->keypart_map & full_key_part_map; DBUG_PRINT("info", ("spider spider_user_defined_key_parts=%u", @@ -8870,7 +8589,7 @@ int spider_mbase_handler::append_tmp_table_and_sql_for_bka( const char *table_names[2], *table_aliases[2], *table_dot_aliases[2]; uint table_name_lengths[2], table_alias_lengths[2], table_dot_alias_lengths[2]; - tgt_table_name_str.init_calc_mem(99); + tgt_table_name_str.init_calc_mem(SPD_MID_MBASE_HANDLER_APPEND_TMP_TABLE_AND_SQL_FOR_BKA_1); tgt_table_name_str.length(0); create_tmp_bka_table_name(tmp_table_name, &tmp_table_name_length, first_link_idx); @@ -9121,7 +8840,7 @@ int spider_mbase_handler::append_union_table_and_sql_for_bka( const char *table_names[2], *table_aliases[2], *table_dot_aliases[2]; uint table_name_lengths[2], table_alias_lengths[2], table_dot_alias_lengths[2]; - tgt_table_name_str.init_calc_mem(233); + tgt_table_name_str.init_calc_mem(SPD_MID_MBASE_HANDLER_APPEND_UNION_TABLE_AND_SQL_FOR_BKA_1); tgt_table_name_str.length(0); if ((error_num = append_table_name_with_adjusting(&tgt_table_name_str, first_link_idx, SPIDER_SQL_TYPE_SELECT_SQL))) @@ -11107,7 +10826,7 @@ int spider_mbase_handler::append_match_against( char buf[MAX_FIELD_WIDTH]; spider_string tmp_str(buf, MAX_FIELD_WIDTH, share->access_charset); - tmp_str.init_calc_mem(116); + tmp_str.init_calc_mem(SPD_MID_MBASE_HANDLER_APPEND_MATCH_AGAINST_1); tmp_str.length(0); if ( tmp_str.append(ft_init_key->ptr(), ft_init_key->length(), @@ -13223,14 +12942,10 @@ int spider_mbase_handler::set_sql_for_exec( int link_idx, SPIDER_LINK_IDX_CHAIN *link_idx_chain ) { - int error_num; DBUG_ENTER("spider_mbase_handler::set_sql_for_exec"); DBUG_PRINT("info",("spider this=%p", this)); if (sql_type & SPIDER_SQL_TYPE_SELECT_SQL) { - if ((error_num = spider_db_mbase_utility->reappend_tables( - spider->fields, link_idx_chain, &sql))) - DBUG_RETURN(error_num); exec_sql = &sql; } DBUG_RETURN(0); @@ -13264,7 +12979,7 @@ int spider_mbase_handler::set_sql_for_exec( mysql_share->db_names_str[link_idx].charset()); const char *table_names[2], *table_aliases[2]; uint table_name_lengths[2], table_alias_lengths[2]; - tgt_table_name_str.init_calc_mem(104); + tgt_table_name_str.init_calc_mem(SPD_MID_MBASE_HANDLER_SET_SQL_FOR_EXEC_1); tgt_table_name_str.length(0); if (result_list->tmp_table_join && spider->bka_mode != 2) { @@ -15213,7 +14928,7 @@ int spider_mbase_handler::init_union_table_name_pos() DBUG_PRINT("info",("spider this=%p", this)); if (!union_table_name_pos_first) { - if (!spider_bulk_malloc(spider_current_trx, 236, MYF(MY_WME), + if (!spider_bulk_malloc(spider_current_trx, SPD_MID_MBASE_HANDLER_INIT_UNION_TABLE_NAME_POS_1, MYF(MY_WME), &union_table_name_pos_first, (uint) (sizeof(SPIDER_INT_HLD)), NullS) ) { @@ -15234,7 +14949,7 @@ int spider_mbase_handler::set_union_table_name_pos() { if (!union_table_name_pos_current->next) { - if (!spider_bulk_malloc(spider_current_trx, 237, MYF(MY_WME), + if (!spider_bulk_malloc(spider_current_trx, SPD_MID_MBASE_HANDLER_SET_UNION_TABLE_NAME_POS_1, MYF(MY_WME), &union_table_name_pos_current->next, (uint) (sizeof(SPIDER_INT_HLD)), NullS) ) { @@ -15296,8 +15011,7 @@ int spider_mbase_handler::append_from_and_tables_part( default: DBUG_RETURN(0); } - fields->set_pos_to_first_table_holder(); - table_holder = fields->get_next_table_holder(); + table_holder = fields->get_first_table_holder(); table_list = table_holder->table->pos_in_table_list; error_num = spider_db_mbase_utility->append_from_and_tables( table_holder->spider, fields, str, @@ -15305,27 +15019,6 @@ int spider_mbase_handler::append_from_and_tables_part( DBUG_RETURN(error_num); } -int spider_mbase_handler::reappend_tables_part( - spider_fields *fields, - ulong sql_type -) { - int error_num; - spider_string *str; - DBUG_ENTER("spider_mbase_handler::reappend_tables_part"); - DBUG_PRINT("info",("spider this=%p", this)); - switch (sql_type) - { - case SPIDER_SQL_TYPE_SELECT_SQL: - str = &sql; - break; - default: - DBUG_RETURN(0); - } - error_num = spider_db_mbase_utility->reappend_tables(fields, - link_idx_chain, str); - DBUG_RETURN(error_num); -} - int spider_mbase_handler::append_where_part( ulong sql_type ) { @@ -15678,7 +15371,7 @@ int spider_mbase_copy_table::init() { DBUG_ENTER("spider_mbase_copy_table::init"); DBUG_PRINT("info",("spider this=%p", this)); - sql.init_calc_mem(78); + sql.init_calc_mem(SPD_MID_MBASE_COPY_TABLE_INIT_1); DBUG_RETURN(0); } diff --git a/storage/spider/spd_db_mysql.h b/storage/spider/spd_db_mysql.h index ec72a97f..9ccc0e33 100644 --- a/storage/spider/spd_db_mysql.h +++ b/storage/spider/spd_db_mysql.h @@ -158,40 +158,22 @@ public: spider_string *to, String *from ) override; - int append_table( - ha_spider *spider, - spider_fields *fields, - spider_string *str, - TABLE_LIST *table_list, - TABLE_LIST **used_table_list, - uint *current_pos, - TABLE_LIST **cond_table_list_ptr, - bool top_down, - bool first - ); - int append_tables_top_down( - ha_spider *spider, - spider_fields *fields, - spider_string *str, - TABLE_LIST *table_list, - TABLE_LIST **used_table_list, - uint *current_pos, - TABLE_LIST **cond_table_list_ptr - ); int append_tables_top_down_check( TABLE_LIST *table_list, TABLE_LIST **used_table_list, uint *current_pos ); - int append_embedding_tables( - ha_spider *spider, - spider_fields *fields, - spider_string *str, - TABLE_LIST *table_list, - TABLE_LIST **used_table_list, - uint *current_pos, - TABLE_LIST **cond_table_list_ptr - ); + int append_table_list(spider_fields *fields, + spider_string *str, TABLE_LIST *table, + table_map *upper_usable_tables, + table_map eliminated_tables); + int append_table_array(spider_fields *fields, + spider_string *str, TABLE_LIST **table, + TABLE_LIST **end, table_map *upper_usable_tables, + table_map eliminated_tables); + int append_join(spider_fields *fields, spider_string *str, + List<TABLE_LIST> *tables, table_map *upper_usable_tables, + table_map eliminated_tables); int append_from_and_tables( ha_spider *spider, spider_fields *fields, @@ -199,11 +181,6 @@ public: TABLE_LIST *table_list, uint table_count ) override; - int reappend_tables( - spider_fields *fields, - SPIDER_LINK_IDX_CHAIN *link_idx_chain, - spider_string *str - ) override; int append_where( spider_string *str ) override; @@ -634,8 +611,11 @@ public: spider_string *show_table_status; spider_string *show_records; spider_string *show_index; + /* The remote table names */ spider_string *table_names_str; + /* The remote db names */ spider_string *db_names_str; + /* fixme: this field looks useless */ spider_string *db_table_str; my_hash_value_type *db_table_str_hash_value; uint table_nm_max_length; @@ -1485,10 +1465,6 @@ public: spider_fields *fields, ulong sql_type ); - int reappend_tables_part( - spider_fields *fields, - ulong sql_type - ); int append_where_part( ulong sql_type ); diff --git a/storage/spider/spd_direct_sql.cc b/storage/spider/spd_direct_sql.cc index dbf7cc8f..d345bb04 100644 --- a/storage/spider/spd_direct_sql.cc +++ b/storage/spider/spd_direct_sql.cc @@ -109,7 +109,7 @@ int spider_udf_direct_sql_create_table_list( break; } if (!(direct_sql->db_names = (char**) - spider_bulk_malloc(spider_current_trx, 31, MYF(MY_WME | MY_ZEROFILL), + spider_bulk_malloc(spider_current_trx, SPD_MID_UDF_DIRECT_SQL_CREATE_TABLE_LIST_2, MYF(MY_WME | MY_ZEROFILL), &direct_sql->db_names, (uint) (sizeof(char*) * table_count), &direct_sql->table_names, (uint) (sizeof(char*) * table_count), &direct_sql->tables, (uint) (sizeof(TABLE*) * table_count), @@ -238,7 +238,7 @@ int spider_udf_direct_sql_create_conn_key( + direct_sql->tgt_filedsn_length + 1 + direct_sql->tgt_driver_length; if (!(direct_sql->conn_key = (char *) - spider_malloc(spider_current_trx, 9, direct_sql->conn_key_length + 1, + spider_malloc(spider_current_trx, SPD_MID_UDF_DIRECT_SQL_CREATE_CONN_KEY_1, direct_sql->conn_key_length + 1, MYF(MY_WME | MY_ZEROFILL))) ) DBUG_RETURN(HA_ERR_OUT_OF_MEM); @@ -389,7 +389,7 @@ SPIDER_CONN *spider_udf_direct_sql_create_conn( spider_dbton[direct_sql->dbton_id].db_util-> tables_on_different_db_are_joinable(); if (!(conn = (SPIDER_CONN *) - spider_bulk_malloc(spider_current_trx, 32, MYF(MY_WME | MY_ZEROFILL), + spider_bulk_malloc(spider_current_trx, SPD_MID_UDF_DIRECT_SQL_CREATE_CONN_1, MYF(MY_WME | MY_ZEROFILL), &conn, (uint) (sizeof(*conn)), &tmp_name, (uint) (direct_sql->conn_key_length + 1), &tmp_host, (uint) (direct_sql->tgt_host_length + 1), @@ -420,7 +420,7 @@ SPIDER_CONN *spider_udf_direct_sql_create_conn( *error_num = HA_ERR_OUT_OF_MEM; goto error_alloc_conn; } - conn->default_database.init_calc_mem(138); + conn->default_database.init_calc_mem(SPD_MID_UDF_DIRECT_SQL_CREATE_CONN_2); conn->conn_key_length = direct_sql->conn_key_length; conn->conn_key = tmp_name; @@ -1439,7 +1439,7 @@ long long spider_direct_sql_body( DBUG_ENTER("spider_direct_sql_body"); SPIDER_BACKUP_DASTATUS; if (!(direct_sql = (SPIDER_DIRECT_SQL *) - spider_bulk_malloc(spider_current_trx, 34, MYF(MY_WME | MY_ZEROFILL), + spider_bulk_malloc(spider_current_trx, SPD_MID_DIRECT_SQL_BODY_1, MYF(MY_WME | MY_ZEROFILL), &direct_sql, (uint) (sizeof(SPIDER_DIRECT_SQL)), &sql, (uint) (sizeof(char) * args->lengths[0]), NullS)) @@ -1674,7 +1674,7 @@ my_bool spider_direct_sql_init_body( if (bg) { if (!(bg_direct_sql = (SPIDER_BG_DIRECT_SQL *) - spider_malloc(spider_current_trx, 10, sizeof(SPIDER_BG_DIRECT_SQL), + spider_malloc(spider_current_trx, SPD_MID_DIRECT_SQL_INIT_BODY_1, sizeof(SPIDER_BG_DIRECT_SQL), MYF(MY_WME | MY_ZEROFILL))) ) { strcpy(message, "spider_bg_direct_sql() out of memory"); diff --git a/storage/spider/spd_group_by_handler.cc b/storage/spider/spd_group_by_handler.cc index 3cd299c6..6236a7b9 100644 --- a/storage/spider/spd_group_by_handler.cc +++ b/storage/spider/spd_group_by_handler.cc @@ -22,6 +22,7 @@ #include "probes_mysql.h" #include "sql_class.h" #include "sql_partition.h" +#include "sql_select.h" #include "ha_partition.h" #include "sql_common.h" #include <errmsg.h> @@ -42,11 +43,9 @@ extern SPIDER_DBTON spider_dbton[SPIDER_DBTON_SIZE]; spider_fields::spider_fields() : dbton_count(0), current_dbton_num(0), - table_count(0), current_table_num(0), table_holder(NULL), + table_count(0), table_holder(NULL), first_link_idx_chain(NULL), last_link_idx_chain(NULL), current_link_idx_chain(NULL), - first_conn_holder(NULL), last_conn_holder(NULL), current_conn_holder(NULL), - first_field_holder(NULL), last_field_holder(NULL), current_field_holder(NULL), - first_field_chain(NULL), last_field_chain(NULL), current_field_chain(NULL) + first_conn_holder(NULL), last_conn_holder(NULL), current_conn_holder(NULL) { DBUG_ENTER("spider_fields::spider_fields"); DBUG_PRINT("info",("spider this=%p", this)); @@ -65,24 +64,6 @@ spider_fields::~spider_fields() spider_free(spider_current_trx, current_link_idx_chain, MYF(0)); } } - if (first_field_chain) - { - while ((current_field_chain = first_field_chain)) - { - first_field_chain = current_field_chain->next; - spider_free(spider_current_trx, current_field_chain, MYF(0)); - } - } - if (first_field_holder) - { - while ((current_field_holder = first_field_holder)) - { - first_field_holder = current_field_holder->next; - spider_free(spider_current_trx, current_field_holder, MYF(0)); - } - } - if (table_holder) - spider_free(spider_current_trx, table_holder, MYF(0)); if (first_conn_holder) { while ((current_conn_holder = first_conn_holder)) @@ -364,7 +345,7 @@ SPIDER_LINK_IDX_CHAIN *spider_fields::create_link_idx_chain( DBUG_ENTER("spider_fields::create_link_idx_chain"); DBUG_PRINT("info",("spider this=%p", this)); DBUG_RETURN((SPIDER_LINK_IDX_CHAIN *) - spider_malloc(spider_current_trx, 254, sizeof(SPIDER_LINK_IDX_CHAIN), + spider_malloc(spider_current_trx, SPD_MID_FIELDS_CREATE_LINK_IDX_CHAIN_1, sizeof(SPIDER_LINK_IDX_CHAIN), MYF(MY_WME | MY_ZEROFILL))); } @@ -471,7 +452,6 @@ int spider_fields::get_ok_link_idx( void spider_fields::set_first_link_idx( ) { - SPIDER_TABLE_HOLDER *table_holder; SPIDER_LINK_IDX_HOLDER *link_idx_holder; SPIDER_LINK_IDX_CHAIN *link_idx_chain; uint dbton_id; @@ -493,11 +473,10 @@ void spider_fields::set_first_link_idx( DBUG_ASSERT(link_idx_chain); set_pos_to_first_table_on_link_idx_chain(link_idx_chain); - set_pos_to_first_table_holder(); - while ((table_holder = get_next_table_holder())) + for (uint i= 0; i < table_count; i++) { link_idx_holder = get_next_table_on_link_idx_chain(link_idx_chain); - spider = table_holder->spider; + spider = table_holder[i].spider; dbton_hdl = spider->dbton_handler[dbton_id]; dbton_hdl->first_link_idx = link_idx_holder->link_idx; } @@ -554,7 +533,7 @@ SPIDER_LINK_IDX_HOLDER *spider_fields::create_link_idx_holder( DBUG_ENTER("spider_fields::create_link_idx_holder"); DBUG_PRINT("info",("spider this=%p", this)); DBUG_RETURN((SPIDER_LINK_IDX_HOLDER *) - spider_malloc(spider_current_trx, 253, sizeof(SPIDER_LINK_IDX_HOLDER), + spider_malloc(spider_current_trx, SPD_MID_FIELDS_CREATE_LINK_IDX_HOLDER_1, sizeof(SPIDER_LINK_IDX_HOLDER), MYF(MY_WME | MY_ZEROFILL))); } @@ -632,7 +611,7 @@ SPIDER_CONN_HOLDER *spider_fields::create_conn_holder( DBUG_ENTER("spider_fields::create_conn_holder"); DBUG_PRINT("info",("spider this=%p", this)); return_conn_holder = (SPIDER_CONN_HOLDER *) - spider_bulk_malloc(spider_current_trx, 252, MYF(MY_WME | MY_ZEROFILL), + spider_bulk_malloc(spider_current_trx, SPD_MID_FIELDS_CREATE_CONN_HOLDER_1, MYF(MY_WME | MY_ZEROFILL), &return_conn_holder, (uint) (sizeof(SPIDER_CONN_HOLDER)), &table_link_idx_holder, (uint) (table_count * sizeof(SPIDER_TABLE_LINK_IDX_HOLDER)), @@ -646,24 +625,6 @@ SPIDER_CONN_HOLDER *spider_fields::create_conn_holder( DBUG_RETURN(return_conn_holder); } -void spider_fields::set_pos_to_first_conn_holder( -) { - DBUG_ENTER("spider_fields::set_pos_to_first_conn_holder"); - DBUG_PRINT("info",("spider this=%p", this)); - current_conn_holder = first_conn_holder; - DBUG_VOID_RETURN; -} - -SPIDER_CONN_HOLDER *spider_fields::get_next_conn_holder( -) { - SPIDER_CONN_HOLDER *return_conn_holder = current_conn_holder; - DBUG_ENTER("spider_fields::get_next_conn_holder"); - DBUG_PRINT("info",("spider this=%p", this)); - if (current_conn_holder) - current_conn_holder = current_conn_holder->next; - DBUG_RETURN(return_conn_holder); -} - bool spider_fields::has_conn_holder( ) { DBUG_ENTER("spider_fields::has_conn_holder"); @@ -862,19 +823,17 @@ void spider_fields::free_conn_holder( DBUG_VOID_RETURN; } -SPIDER_TABLE_HOLDER *spider_fields::add_table( - ha_spider *spider_arg +/* Add the table associated with an ha_spider to a table_holder. +Return the table_holder. */ +static SPIDER_TABLE_HOLDER *spider_add_table_holder( + ha_spider *spider_arg, + SPIDER_TABLE_HOLDER *table_holder ) { spider_string *str; uint length; char tmp_buf[SPIDER_SQL_INT_LEN + 2]; SPIDER_TABLE_HOLDER *return_table_holder; - SPIDER_FIELD_HOLDER *field_holder; - TABLE *table = spider_arg->get_table(); - Field *field; DBUG_ENTER("spider_fields::add_table"); - DBUG_PRINT("info",("spider this=%p", this)); - DBUG_PRINT("info",("spider table_count=%u", table_count)); DBUG_PRINT("info",("spider idx_for_direct_join=%u", spider_arg->idx_for_direct_join)); length = my_sprintf(tmp_buf, (tmp_buf, "t%u", @@ -893,90 +852,49 @@ SPIDER_TABLE_HOLDER *spider_fields::add_table( return_table_holder->spider = spider_arg; return_table_holder->alias = str; - set_pos_to_first_field_holder(); - while ((field_holder = get_next_field_holder())) - { - if (!field_holder->spider) - { - field = field_holder->field; - if ( - field->field_index < table->s->fields && - field == table->field[field->field_index] - ) { - field_holder->spider = spider_arg; - field_holder->alias = str; - } - } - } DBUG_RETURN(return_table_holder); } -/** - Verify that all fields in the query are members of tables that are in the - query. - - @return TRUE All fields in the query are members of tables - that are in the query. - FALSE At least one field in the query is not a - member of a table that is in the query. -*/ - -bool spider_fields::all_query_fields_are_query_table_members() +/* Return the table that field belongs to, or NULL if none exists. */ +SPIDER_TABLE_HOLDER *spider_fields::find_table(Field *field) { - SPIDER_FIELD_HOLDER *field_holder; - DBUG_ENTER("spider_fields::all_query_fields_are_query_table_members"); - DBUG_PRINT("info",("spider this=%p", this)); - - set_pos_to_first_field_holder(); - while ((field_holder = get_next_field_holder())) - { - if (!field_holder->spider) - { - DBUG_PRINT("info", ("spider field is not a member of a query table")); - DBUG_RETURN(FALSE); - } - } + for (uint i = 0; i < table_count; i++) + if (field->table == table_holder[i].table) + return &table_holder[i]; + return NULL; +} - DBUG_RETURN(TRUE); +void spider_fields::set_table_holder(SPIDER_TABLE_HOLDER *table_holder_arg, + uint table_count_arg) +{ + table_holder= table_holder_arg; + table_count= table_count_arg; } -int spider_fields::create_table_holder( +/* Allocate space for table_count_arg table holders. */ +static SPIDER_TABLE_HOLDER *spider_create_table_holder( uint table_count_arg ) { - DBUG_ENTER("spider_fields::create_table_holder"); - DBUG_PRINT("info",("spider this=%p", this)); - DBUG_ASSERT(!table_holder); + SPIDER_TABLE_HOLDER* table_holder; + DBUG_ENTER("spider_create_table_holder"); + if (table_count_arg == 0) + DBUG_RETURN(0); table_holder = (SPIDER_TABLE_HOLDER *) - spider_malloc(spider_current_trx, 249, + spider_malloc(spider_current_trx, SPD_MID_CREATE_TABLE_HOLDER_1, table_count_arg * sizeof(SPIDER_TABLE_HOLDER), MYF(MY_WME | MY_ZEROFILL)); - if (!table_holder) - DBUG_RETURN(HA_ERR_OUT_OF_MEM); - table_count = table_count_arg; - current_table_num = 0; - DBUG_RETURN(0); + DBUG_RETURN(table_holder); } -void spider_fields::set_pos_to_first_table_holder( -) { - DBUG_ENTER("spider_fields::set_pos_to_first_table_holder"); - DBUG_PRINT("info",("spider this=%p", this)); - current_table_num = 0; - DBUG_VOID_RETURN; -} - -SPIDER_TABLE_HOLDER *spider_fields::get_next_table_holder( -) { - SPIDER_TABLE_HOLDER *return_table_holder; - DBUG_ENTER("spider_fields::get_next_table_holder"); - DBUG_PRINT("info",("spider this=%p", this)); - if (current_table_num >= table_count) - DBUG_RETURN(NULL); - return_table_holder = &table_holder[current_table_num]; - ++current_table_num; - DBUG_RETURN(return_table_holder); +/* Return pointer to the first table holder. */ +SPIDER_TABLE_HOLDER *spider_fields::get_first_table_holder() +{ + DBUG_ENTER("spider_fields::get_first_spider"); + DBUG_RETURN(table_holder); } +/* Return the first table holder associated with a given table, or +NULL if not found. */ SPIDER_TABLE_HOLDER *spider_fields::get_table_holder(TABLE *table) { uint table_num; @@ -996,117 +914,12 @@ uint spider_fields::get_table_count() DBUG_RETURN(table_count); } -int spider_fields::add_field( - Field *field_arg -) { - SPIDER_FIELD_HOLDER *field_holder; - SPIDER_FIELD_CHAIN *field_chain; - DBUG_ENTER("spider_fields::add_field"); - DBUG_PRINT("info",("spider this=%p", this)); - DBUG_PRINT("info",("spider field=%p", field_arg)); - if (!first_field_holder) - { - field_holder = create_field_holder(); - DBUG_PRINT("info",("spider field_holder=%p", field_holder)); - if (!field_holder) - DBUG_RETURN(HA_ERR_OUT_OF_MEM); - field_holder->field = field_arg; - first_field_holder = field_holder; - last_field_holder = field_holder; - } else { - field_holder = first_field_holder; - do { - if (field_holder->field == field_arg) - break; - } while ((field_holder = field_holder->next)); - if (!field_holder) - { - field_holder = create_field_holder(); - DBUG_PRINT("info",("spider field_holder=%p", field_holder)); - if (!field_holder) - DBUG_RETURN(HA_ERR_OUT_OF_MEM); - field_holder->field = field_arg; - last_field_holder->next = field_holder; - last_field_holder = field_holder; - } - } - field_chain = create_field_chain(); - DBUG_PRINT("info",("spider field_chain=%p", field_chain)); - if (!field_chain) - DBUG_RETURN(HA_ERR_OUT_OF_MEM); - field_chain->field_holder = field_holder; - if (!first_field_chain) - { - first_field_chain = field_chain; - last_field_chain = field_chain; - } else { - last_field_chain->next = field_chain; - last_field_chain = field_chain; - } - DBUG_RETURN(0); -} - -SPIDER_FIELD_HOLDER *spider_fields::create_field_holder( -) { - DBUG_ENTER("spider_fields::create_field_holder"); - DBUG_PRINT("info",("spider this=%p", this)); - DBUG_RETURN((SPIDER_FIELD_HOLDER *) - spider_malloc(spider_current_trx, 250, sizeof(SPIDER_FIELD_HOLDER), - MYF(MY_WME | MY_ZEROFILL))); -} - -void spider_fields::set_pos_to_first_field_holder( -) { - DBUG_ENTER("spider_fields::set_pos_to_first_field_holder"); - DBUG_PRINT("info",("spider this=%p", this)); - current_field_holder = first_field_holder; - DBUG_VOID_RETURN; -} - -SPIDER_FIELD_HOLDER *spider_fields::get_next_field_holder( -) { - SPIDER_FIELD_HOLDER *return_field_holder = current_field_holder; - DBUG_ENTER("spider_fields::get_next_field_holder"); - DBUG_PRINT("info",("spider this=%p", this)); - if (current_field_holder) - current_field_holder = current_field_holder->next; - DBUG_RETURN(return_field_holder); -} - -SPIDER_FIELD_CHAIN *spider_fields::create_field_chain( -) { - DBUG_ENTER("spider_fields::create_field_chain"); - DBUG_PRINT("info",("spider this=%p", this)); - DBUG_RETURN((SPIDER_FIELD_CHAIN *) - spider_malloc(spider_current_trx, 251, sizeof(SPIDER_FIELD_CHAIN), - MYF(MY_WME | MY_ZEROFILL))); -} - -void spider_fields::set_pos_to_first_field_chain( -) { - DBUG_ENTER("spider_fields::set_pos_to_first_field_chain"); - DBUG_PRINT("info",("spider this=%p", this)); - current_field_chain = first_field_chain; - DBUG_VOID_RETURN; -} - -SPIDER_FIELD_CHAIN *spider_fields::get_next_field_chain( -) { - SPIDER_FIELD_CHAIN *return_field_chain = current_field_chain; - DBUG_ENTER("spider_fields::get_next_field_chain"); - DBUG_PRINT("info",("spider this=%p", this)); - if (current_field_chain) - current_field_chain = current_field_chain->next; - DBUG_RETURN(return_field_chain); -} - void spider_fields::set_field_ptr( Field **field_arg ) { DBUG_ENTER("spider_fields::set_field_ptr"); DBUG_PRINT("info",("spider this=%p", this)); DBUG_PRINT("info",("spider field_ptr=%p", field_arg)); - first_field_ptr = field_arg; current_field_ptr = field_arg; DBUG_VOID_RETURN; } @@ -1129,15 +942,13 @@ int spider_fields::ping_table_mon_from_table( ha_spider *tmp_spider; SPIDER_SHARE *tmp_share; int tmp_link_idx; - SPIDER_TABLE_HOLDER *table_holder; SPIDER_LINK_IDX_HOLDER *link_idx_holder; DBUG_ENTER("spider_fields::ping_table_mon_from_table"); set_pos_to_first_table_on_link_idx_chain(link_idx_chain); - set_pos_to_first_table_holder(); - while ((table_holder = get_next_table_holder())) + for (uint i= 0; i < table_count; i++) { link_idx_holder = get_next_table_on_link_idx_chain(link_idx_chain); - tmp_spider = table_holder->spider; + tmp_spider = table_holder[i].spider; tmp_link_idx = link_idx_holder->link_idx; tmp_share = tmp_spider->share; if (tmp_share->monitoring_kind[tmp_link_idx]) @@ -1173,9 +984,7 @@ spider_group_by_handler::spider_group_by_handler( query(*query_arg), fields(fields_arg) { DBUG_ENTER("spider_group_by_handler::spider_group_by_handler"); - fields->set_pos_to_first_table_holder(); - SPIDER_TABLE_HOLDER *table_holder = fields->get_next_table_holder(); - spider = table_holder->spider; + spider = fields->get_first_table_holder()->spider; trx = spider->wide_handler->trx; DBUG_VOID_RETURN; } @@ -1183,42 +992,20 @@ spider_group_by_handler::spider_group_by_handler( spider_group_by_handler::~spider_group_by_handler() { DBUG_ENTER("spider_group_by_handler::~spider_group_by_handler"); + spider_free(spider_current_trx, fields->get_first_table_holder(), MYF(0)); delete fields; DBUG_VOID_RETURN; } -int spider_group_by_handler::init_scan() +static int spider_prepare_init_scan( + const Query& query, spider_fields *fields, ha_spider *spider, + SPIDER_TRX *trx, longlong& offset_limit, THD *thd) { - int error_num, link_idx; - uint dbton_id; - spider_db_handler *dbton_hdl; + SPIDER_RESULT_LIST *result_list = &spider->result_list; st_select_lex *select_lex; - longlong select_limit; - longlong direct_order_limit; + longlong select_limit, direct_order_limit; SPIDER_SHARE *share = spider->share; - SPIDER_CONN *conn; - SPIDER_RESULT_LIST *result_list = &spider->result_list; - SPIDER_LINK_IDX_CHAIN *link_idx_chain; - SPIDER_LINK_IDX_HOLDER *link_idx_holder; - DBUG_ENTER("spider_group_by_handler::init_scan"); - store_error = 0; -#ifndef DBUG_OFF - Field **field; - for ( - field = table->field; - *field; - field++ - ) { - DBUG_PRINT("info",("spider field_name=%s", - SPIDER_field_name_str(*field))); - } -#endif - - if (trx->thd->killed) - { - my_error(ER_QUERY_INTERRUPTED, MYF(0)); - DBUG_RETURN(ER_QUERY_INTERRUPTED); - } + DBUG_ENTER("spider_prepare_init_scan"); spider->use_fields = TRUE; spider->fields = fields; @@ -1264,7 +1051,7 @@ int spider_group_by_handler::init_scan() } result_list->semi_split_read_base = 0; result_list->set_split_read = TRUE; - if ((error_num = spider_set_conn_bg_param(spider))) + if (int error_num = spider_set_conn_bg_param(spider)) DBUG_RETURN(error_num); DBUG_PRINT("info",("spider result_list.finish_flg = FALSE")); result_list->finish_flg = FALSE; @@ -1284,86 +1071,87 @@ int spider_group_by_handler::init_scan() } else { offset_limit = 0; } + DBUG_RETURN(0); +} + +static int spider_make_query(const Query& query, spider_fields* fields, ha_spider *spider, TABLE *table) +{ + uint dbton_id; + spider_db_handler* dbton_hdl; + SPIDER_RESULT_LIST *result_list = &spider->result_list; + int error_num; + DBUG_ENTER("spider_make_query"); - /* making a query */ fields->set_pos_to_first_dbton_id(); while ((dbton_id = fields->get_next_dbton_id()) < SPIDER_DBTON_SIZE) { dbton_hdl = spider->dbton_handler[dbton_id]; result_list->direct_distinct = query.distinct; - fields->set_pos_to_first_field_chain(); if ((error_num = dbton_hdl->reset_sql(SPIDER_SQL_TYPE_SELECT_SQL))) - { DBUG_RETURN(error_num); - } if ((error_num = dbton_hdl->append_select_part(SPIDER_SQL_TYPE_SELECT_SQL))) - { DBUG_RETURN(error_num); - } fields->set_field_ptr(table->field); if ((error_num = dbton_hdl->append_list_item_select_part( - query.select, NULL, 0, TRUE, fields, SPIDER_SQL_TYPE_SELECT_SQL))) - { + query.select, NULL, 0, TRUE, fields, SPIDER_SQL_TYPE_SELECT_SQL))) DBUG_RETURN(error_num); - } if ((error_num = dbton_hdl->append_from_and_tables_part( - fields, SPIDER_SQL_TYPE_SELECT_SQL))) - { + fields, SPIDER_SQL_TYPE_SELECT_SQL))) DBUG_RETURN(error_num); - } if (query.where) { if ((error_num = - dbton_hdl->append_where_part(SPIDER_SQL_TYPE_SELECT_SQL))) - { + dbton_hdl->append_where_part(SPIDER_SQL_TYPE_SELECT_SQL))) DBUG_RETURN(error_num); - } if ((error_num = dbton_hdl->append_item_type_part( - query.where, NULL, 0, TRUE, fields, SPIDER_SQL_TYPE_SELECT_SQL))) - { + query.where, NULL, 0, TRUE, fields, SPIDER_SQL_TYPE_SELECT_SQL))) DBUG_RETURN(error_num); - } } if (query.group_by) { if ((error_num = dbton_hdl->append_group_by_part( - query.group_by, NULL, 0, TRUE, fields, SPIDER_SQL_TYPE_SELECT_SQL))) - { + query.group_by, NULL, 0, TRUE, fields, SPIDER_SQL_TYPE_SELECT_SQL))) DBUG_RETURN(error_num); - } } if (query.having) { if ((error_num = - dbton_hdl->append_having_part(SPIDER_SQL_TYPE_SELECT_SQL))) - { + dbton_hdl->append_having_part(SPIDER_SQL_TYPE_SELECT_SQL))) DBUG_RETURN(error_num); - } if ((error_num = dbton_hdl->append_item_type_part( - query.having, NULL, 0, TRUE, fields, SPIDER_SQL_TYPE_SELECT_SQL))) - { + query.having, NULL, 0, TRUE, fields, SPIDER_SQL_TYPE_SELECT_SQL))) DBUG_RETURN(error_num); - } } if (query.order_by) { if ((error_num = dbton_hdl->append_order_by_part( - query.order_by, NULL, 0, TRUE, fields, SPIDER_SQL_TYPE_SELECT_SQL))) - { + query.order_by, NULL, 0, TRUE, fields, + SPIDER_SQL_TYPE_SELECT_SQL))) DBUG_RETURN(error_num); - } } - if ((error_num = dbton_hdl->append_limit_part(result_list->internal_offset, - result_list->limit_num, SPIDER_SQL_TYPE_SELECT_SQL))) - { + if ((error_num = dbton_hdl->append_limit_part( + result_list->internal_offset, result_list->limit_num, + SPIDER_SQL_TYPE_SELECT_SQL))) DBUG_RETURN(error_num); - } if ((error_num = dbton_hdl->append_select_lock_part( - SPIDER_SQL_TYPE_SELECT_SQL))) - { + SPIDER_SQL_TYPE_SELECT_SQL))) DBUG_RETURN(error_num); - } } + DBUG_RETURN(0); +} + +static int spider_send_query( + spider_fields *fields, ha_spider *spider, SPIDER_TRX *trx, TABLE *table, + int& store_error) +{ + int error_num, link_idx; + spider_db_handler *dbton_hdl; + SPIDER_RESULT_LIST *result_list = &spider->result_list; + SPIDER_SHARE *share = spider->share; + SPIDER_CONN *conn; + SPIDER_LINK_IDX_CHAIN *link_idx_chain; + SPIDER_LINK_IDX_HOLDER *link_idx_holder; + DBUG_ENTER("spider_send_query"); fields->set_pos_to_first_link_idx_chain(); while ((link_idx_chain = fields->get_next_link_idx_chain())) @@ -1382,12 +1170,8 @@ int spider_group_by_handler::init_scan() dbton_hdl->first_link_idx, TRUE, FALSE, !fields->is_first_link_ok_chain(link_idx_chain)))) { - if ( - error_num != HA_ERR_END_OF_FILE && - spider->need_mons[link_idx] - ) { + if (error_num != HA_ERR_END_OF_FILE && spider->need_mons[link_idx]) error_num = fields->ping_table_mon_from_table(link_idx_chain); - } if ((error_num = spider->check_error_mode_eof(error_num)) == HA_ERR_END_OF_FILE) { store_error = HA_ERR_END_OF_FILE; @@ -1395,14 +1179,12 @@ int spider_group_by_handler::init_scan() } DBUG_RETURN(error_num); } - } else { + } else + { pthread_mutex_assert_not_owner(&conn->mta_conn_mutex); - if ((error_num = - dbton_hdl->set_sql_for_exec(SPIDER_SQL_TYPE_SELECT_SQL, link_idx, - link_idx_chain))) - { + if ((error_num = dbton_hdl->set_sql_for_exec( + SPIDER_SQL_TYPE_SELECT_SQL, link_idx, link_idx_chain))) DBUG_RETURN(error_num); - } pthread_mutex_lock(&conn->mta_conn_mutex); SPIDER_SET_FILE_POS(&conn->mta_conn_mutex_file_pos); conn->need_mon = &spider->need_mons[link_idx]; @@ -1412,6 +1194,7 @@ int spider_group_by_handler::init_scan() conn->mta_conn_mutex_unlock_later = TRUE; if ((error_num = spider_db_set_names(spider, conn, link_idx))) + if ((error_num = spider_db_set_names(spider, conn, link_idx))) { DBUG_ASSERT(conn->mta_conn_mutex_lock_already); DBUG_ASSERT(conn->mta_conn_mutex_unlock_later); @@ -1419,37 +1202,32 @@ int spider_group_by_handler::init_scan() conn->mta_conn_mutex_unlock_later = FALSE; SPIDER_CLEAR_FILE_POS(&conn->mta_conn_mutex_file_pos); pthread_mutex_unlock(&conn->mta_conn_mutex); - if ( - spider->need_mons[link_idx] - ) { + if (spider->need_mons[link_idx]) error_num = fields->ping_table_mon_from_table(link_idx_chain); - } - if ((error_num = spider->check_error_mode_eof(error_num)) == HA_ERR_END_OF_FILE) + if ((error_num = spider->check_error_mode_eof(error_num)) == + HA_ERR_END_OF_FILE) { store_error = HA_ERR_END_OF_FILE; error_num = 0; } DBUG_RETURN(error_num); } - spider_conn_set_timeout_from_share(conn, link_idx, - trx->thd, share); + spider_conn_set_timeout_from_share(conn, link_idx, trx->thd, share); if (dbton_hdl->execute_sql( SPIDER_SQL_TYPE_SELECT_SQL, conn, spider->result_list.quick_mode, - &spider->need_mons[link_idx]) - ) { + &spider->need_mons[link_idx])) + { DBUG_ASSERT(conn->mta_conn_mutex_lock_already); DBUG_ASSERT(conn->mta_conn_mutex_unlock_later); conn->mta_conn_mutex_lock_already = FALSE; conn->mta_conn_mutex_unlock_later = FALSE; error_num = spider_db_errorno(conn); - if ( - spider->need_mons[link_idx] - ) { + if (spider->need_mons[link_idx]) error_num = fields->ping_table_mon_from_table(link_idx_chain); - } - if ((error_num = spider->check_error_mode_eof(error_num)) == HA_ERR_END_OF_FILE) + if ((error_num = spider->check_error_mode_eof(error_num)) == + HA_ERR_END_OF_FILE) { store_error = HA_ERR_END_OF_FILE; error_num = 0; @@ -1465,13 +1243,10 @@ int spider_group_by_handler::init_scan() { if ((error_num = spider_db_store_result(spider, link_idx, table))) { - if ( - error_num != HA_ERR_END_OF_FILE && - spider->need_mons[link_idx] - ) { + if (error_num != HA_ERR_END_OF_FILE && spider->need_mons[link_idx]) error_num = fields->ping_table_mon_from_table(link_idx_chain); - } - if ((error_num = spider->check_error_mode_eof(error_num)) == HA_ERR_END_OF_FILE) + if ((error_num = spider->check_error_mode_eof(error_num)) == + HA_ERR_END_OF_FILE) { store_error = HA_ERR_END_OF_FILE; error_num = 0; @@ -1480,13 +1255,45 @@ int spider_group_by_handler::init_scan() } spider->result_link_idx = link_idx; spider->result_link_idx_chain = link_idx_chain; - } else { + } else + { spider_db_discard_result(spider, link_idx, conn); SPIDER_CLEAR_FILE_POS(&conn->mta_conn_mutex_file_pos); pthread_mutex_unlock(&conn->mta_conn_mutex); } } } + DBUG_RETURN(0); +} + +/* + Prepare and send query to data nodes and store the query results. +*/ +int spider_group_by_handler::init_scan() +{ + int error_num; + DBUG_ENTER("spider_group_by_handler::init_scan"); + store_error = 0; +#ifndef DBUG_OFF + for (Field **field = table->field; *field; field++) + DBUG_PRINT("info",("spider field_name=%s", SPIDER_field_name_str(*field))); +#endif + + if (trx->thd->killed) + { + my_error(ER_QUERY_INTERRUPTED, MYF(0)); + DBUG_RETURN(ER_QUERY_INTERRUPTED); + } + + if ((error_num = spider_prepare_init_scan( + query, fields, spider, trx, offset_limit, thd))) + DBUG_RETURN(error_num); + + if ((error_num = spider_make_query(query, fields, spider, table))) + DBUG_RETURN(error_num); + + if ((error_num = spider_send_query(fields, spider, trx, table, store_error))) + DBUG_RETURN(error_num); first = TRUE; DBUG_RETURN(0); @@ -1595,7 +1402,8 @@ group_by_handler *spider_create_group_by_handler( bool keep_going; bool find_dbton = FALSE; spider_fields *fields = NULL, *fields_arg = NULL; - uint table_idx, dbton_id; + SPIDER_TABLE_HOLDER *table_holder; + uint table_idx, dbton_id, table_count= 0; long tgt_link_status; DBUG_ENTER("spider_create_group_by_handler"); @@ -1617,8 +1425,7 @@ group_by_handler *spider_create_group_by_handler( from = query->from; do { DBUG_PRINT("info",("spider from=%p", from)); - if (from->table->const_table) - continue; + ++table_count; if (from->table->part_info) { DBUG_PRINT("info",("spider partition handler")); @@ -1633,17 +1440,11 @@ group_by_handler *spider_create_group_by_handler( } } while ((from = from->next_local)); + if (!(table_holder= spider_create_table_holder(table_count))) + DBUG_RETURN(NULL); + table_idx = 0; from = query->from; - while (from && from->table->const_table) - { - from = from->next_local; - } - if (!from) - { - /* all tables are const_table */ - DBUG_RETURN(NULL); - } if (from->table->part_info) { partition_info *part_info = from->table->part_info; @@ -1657,6 +1458,11 @@ group_by_handler *spider_create_group_by_handler( share = spider->share; spider->idx_for_direct_join = table_idx; ++table_idx; + if (!spider_add_table_holder(spider, table_holder)) + { + DBUG_PRINT("info",("spider can not add a table")); + goto skip_free_table_holder; + } memset(dbton_bitmap, 0, spider_bitmap_size(SPIDER_DBTON_SIZE)); for (roop_count = 0; roop_count < (int) share->use_dbton_count; ++roop_count) { @@ -1670,8 +1476,6 @@ group_by_handler *spider_create_group_by_handler( } while ((from = from->next_local)) { - if (from->table->const_table) - continue; if (from->table->part_info) { partition_info *part_info = from->table->part_info; @@ -1685,6 +1489,11 @@ group_by_handler *spider_create_group_by_handler( share = spider->share; spider->idx_for_direct_join = table_idx; ++table_idx; + if (!spider_add_table_holder(spider, table_holder)) + { + DBUG_PRINT("info",("spider can not add a table")); + goto skip_free_table_holder; + } memset(dbton_bitmap_tmp, 0, spider_bitmap_size(SPIDER_DBTON_SIZE)); for (roop_count = 0; roop_count < (int) share->use_dbton_count; ++roop_count) { @@ -1705,8 +1514,6 @@ group_by_handler *spider_create_group_by_handler( from = query->from; do { - if (from->table->const_table) - continue; if (from->table->part_info) { partition_info *part_info = from->table->part_info; @@ -1734,10 +1541,9 @@ group_by_handler *spider_create_group_by_handler( { fields_arg = new spider_fields(); if (!fields_arg) - { - DBUG_RETURN(NULL); - } + goto skip_free_table_holder; } + fields_arg->set_table_holder(table_holder, table_count); keep_going = TRUE; it.init(*query->select); while ((item = it++)) @@ -1844,21 +1650,9 @@ group_by_handler *spider_create_group_by_handler( } } if (!find_dbton) - { - DBUG_RETURN(NULL); - } - - if (fields->create_table_holder(table_idx)) - { - delete fields; - DBUG_RETURN(NULL); - } + goto skip_free_table_holder; from = query->from; - while (from->table->const_table) - { - from = from->next_local; - } if (from->table->part_info) { partition_info *part_info = from->table->part_info; @@ -1879,17 +1673,10 @@ group_by_handler *spider_create_group_by_handler( } DBUG_PRINT("info",("spider s->db=%s", from->table->s->db.str)); DBUG_PRINT("info",("spider s->table_name=%s", from->table->s->table_name.str)); - if (!fields->add_table(spider)) - { - DBUG_PRINT("info",("spider can not add a table")); - delete fields; - DBUG_RETURN(NULL); - } if (spider->dml_init()) { DBUG_PRINT("info",("spider can not init for dml")); - delete fields; - DBUG_RETURN(NULL); + goto skip_free_fields; } for ( roop_count = spider_conn_link_idx_next(share->link_statuses, @@ -1909,8 +1696,7 @@ group_by_handler *spider_create_group_by_handler( DBUG_PRINT("info",("spider direct_join does not support with lock tables yet")); if (lock_mode) { - delete fields; - DBUG_RETURN(NULL); + goto skip_free_fields; } continue; } @@ -1918,26 +1704,21 @@ group_by_handler *spider_create_group_by_handler( share->access_balances[spider->conn_link_idx[roop_count]])) { DBUG_PRINT("info",("spider can not create conn_holder")); - delete fields; - DBUG_RETURN(NULL); + goto skip_free_fields; } if (fields->add_link_idx(conn->conn_holder_for_direct_join, spider, roop_count)) { DBUG_PRINT("info",("spider can not create link_idx_holder")); - delete fields; - DBUG_RETURN(NULL); + goto skip_free_fields; } } if (!fields->has_conn_holder()) { - delete fields; - DBUG_RETURN(NULL); + goto skip_free_fields; } while ((from = from->next_local)) { - if (from->table->const_table) - continue; fields->clear_conn_holder_from_conn(); if (from->table->part_info) @@ -1951,19 +1732,12 @@ group_by_handler *spider_create_group_by_handler( spider = (ha_spider *) from->table->file; } share = spider->share; - if (!fields->add_table(spider)) - { - DBUG_PRINT("info",("spider can not add a table")); - delete fields; - DBUG_RETURN(NULL); - } DBUG_PRINT("info",("spider s->db=%s", from->table->s->db.str)); DBUG_PRINT("info",("spider s->table_name=%s", from->table->s->table_name.str)); if (spider->dml_init()) { DBUG_PRINT("info",("spider can not init for dml")); - delete fields; - DBUG_RETURN(NULL); + goto skip_free_fields; } for ( roop_count = spider_conn_link_idx_next(share->link_statuses, @@ -1982,17 +1756,13 @@ group_by_handler *spider_create_group_by_handler( DBUG_PRINT("info",("spider connection %p can not be used for this query with locking", conn)); if (lock_mode) - { - delete fields; - DBUG_RETURN(NULL); - } + goto skip_free_fields; continue; } if (fields->add_link_idx(conn->conn_holder_for_direct_join, spider, roop_count)) { DBUG_PRINT("info",("spider can not create link_idx_holder")); - delete fields; - DBUG_RETURN(NULL); + goto skip_free_fields; } } @@ -2001,30 +1771,20 @@ group_by_handler *spider_create_group_by_handler( if (lock_mode) { DBUG_PRINT("info",("spider some connections can not be used for this query with locking")); - delete fields; - DBUG_RETURN(NULL); + goto skip_free_fields; } } if (!fields->has_conn_holder()) { - delete fields; - DBUG_RETURN(NULL); + goto skip_free_fields; } } - if (!fields->all_query_fields_are_query_table_members()) - { - DBUG_PRINT("info", ("spider found a query field that is not a query table member")); - delete fields; - DBUG_RETURN(NULL); - } - fields->check_support_dbton(dbton_bitmap); if (!fields->has_conn_holder()) { DBUG_PRINT("info",("spider all chosen connections can't match dbton_id")); - delete fields; - DBUG_RETURN(NULL); + goto skip_free_fields; } /* choose a connection */ @@ -2036,16 +1796,14 @@ group_by_handler *spider_create_group_by_handler( if (fields->make_link_idx_chain(tgt_link_status)) { DBUG_PRINT("info",("spider can not create link_idx_chain")); - delete fields; - DBUG_RETURN(NULL); + goto skip_free_fields; } /* choose link_id */ if (fields->check_link_ok_chain()) { DBUG_PRINT("info",("spider do not have link ok status")); - delete fields; - DBUG_RETURN(NULL); + goto skip_free_fields; } fields->set_first_link_idx(); @@ -2053,8 +1811,7 @@ group_by_handler *spider_create_group_by_handler( if (!(group_by_handler = new spider_group_by_handler(thd, query, fields))) { DBUG_PRINT("info",("spider can't create group_by_handler")); - delete fields; - DBUG_RETURN(NULL); + goto skip_free_fields; } query->distinct = FALSE; query->where = NULL; @@ -2062,4 +1819,10 @@ group_by_handler *spider_create_group_by_handler( query->having = NULL; query->order_by = NULL; DBUG_RETURN(group_by_handler); + +skip_free_fields: + delete fields; +skip_free_table_holder: + spider_free(spider_current_trx, table_holder, MYF(0)); + DBUG_RETURN(NULL); } diff --git a/storage/spider/spd_include.h b/storage/spider/spd_include.h index 4763cd89..0f61984e 100644 --- a/storage/spider/spd_include.h +++ b/storage/spider/spd_include.h @@ -168,6 +168,279 @@ typedef start_new_trans *SPIDER_Open_tables_backup; #define SPIDER_MEM_CALC_LIST_NUM 314 #define SPIDER_CONN_META_BUF_LEN 64 +/* + IDs for spider mem alloc functions, including + - spider_alloc_calc_mem_init() + - spider_string::init_calc_mem() + - spider_malloc() + - spider_bulk_alloc_mem() + - spider_bulk_malloc() + In the format of + SPD_MID_<CALLSITE_FUNC_NAME_SANS_SPIDER_PREFIX>_<NO> +*/ +enum spider_malloc_id { + SPD_MID_CHECK_HS_PK_UPDATE_1, + SPD_MID_COPY_TABLES_BODY_1, + SPD_MID_COPY_TABLES_BODY_2, + SPD_MID_COPY_TABLES_BODY_3, + SPD_MID_COPY_TABLES_BODY_4, + SPD_MID_COPY_TABLES_BODY_5, + SPD_MID_CREATE_CONN_1, + SPD_MID_CREATE_CONN_2, + SPD_MID_CREATE_CONN_3, + SPD_MID_CREATE_CONN_4, + SPD_MID_CREATE_CONN_5, + SPD_MID_CREATE_CONN_6, + SPD_MID_CREATE_CONN_KEYS_1, + SPD_MID_CREATE_CONN_THREAD_1, + SPD_MID_CREATE_LONGLONG_LIST_1, + SPD_MID_CREATE_LONG_LIST_1, + SPD_MID_CREATE_MON_THREADS_1, + SPD_MID_CREATE_MON_THREADS_2, + SPD_MID_CREATE_SHARE_1, + SPD_MID_CREATE_SHARE_2, + SPD_MID_CREATE_SPIDER_OBJECT_FOR_SHARE_1, + SPD_MID_CREATE_SPIDER_OBJECT_FOR_SHARE_2, + SPD_MID_CREATE_STRING_1, + SPD_MID_CREATE_STRING_LIST_1, + SPD_MID_CREATE_TABLE_HOLDER_1, + SPD_MID_CREATE_TABLE_NAME_STRING_1, + SPD_MID_CREATE_TRX_ALTER_TABLE_1, + SPD_MID_CREATE_TRX_HA_1, + SPD_MID_DB_CONN_QUEUE_ACTION_1, + SPD_MID_DB_FETCH_FOR_ITEM_SUM_FUNC_1, + SPD_MID_DB_FETCH_FOR_ITEM_SUM_FUNC_2, + SPD_MID_DB_FETCH_FOR_ITEM_SUM_FUNC_3, + SPD_MID_DB_HANDLERSOCKET_APPEND_REQUEST_KEY_1, + SPD_MID_DB_HANDLERSOCKET_EXEC_QUERY_1, + SPD_MID_DB_HANDLERSOCKET_INIT_1, + SPD_MID_DB_HANDLERSOCKET_RESULT_FETCH_ROW_FROM_TMP_TABLE_1, + SPD_MID_DB_HANDLERSOCKET_RESULT_FETCH_ROW_FROM_TMP_TABLE_2, + SPD_MID_DB_HANDLERSOCKET_ROW_APPEND_ESCAPED_TO_STR_1, + SPD_MID_DB_HANDLERSOCKET_ROW_CLONE_1, + SPD_MID_DB_HANDLERSOCKET_ROW_STORE_TO_FIELD_1, + SPD_MID_DB_HANDLERSOCKET_UTIL_APPEND_COLUMN_VALUE_1, + SPD_MID_DB_HANDLERSOCKET_UTIL_OPEN_ITEM_FUNC_1, + SPD_MID_DB_HANDLERSOCKET_UTIL_OPEN_ITEM_FUNC_2, + SPD_MID_DB_HANDLERSOCKET_UTIL_OPEN_ITEM_FUNC_3, + SPD_MID_DB_HS_STRING_REF_BUFFER_INIT_1, + SPD_MID_DB_HS_STR_BUFFER_ADD_1, + SPD_MID_DB_HS_STR_BUFFER_ADD_2, + SPD_MID_DB_HS_STR_BUFFER_INIT_1, + SPD_MID_DB_INIT_1, + SPD_MID_DB_INIT_10, + SPD_MID_DB_INIT_11, + SPD_MID_DB_INIT_12, + SPD_MID_DB_INIT_2, + SPD_MID_DB_INIT_3, + SPD_MID_DB_INIT_4, + SPD_MID_DB_INIT_5, + SPD_MID_DB_INIT_6, + SPD_MID_DB_INIT_7, + SPD_MID_DB_INIT_8, + SPD_MID_DB_INIT_9, + SPD_MID_DB_MARIADB_UTIL_APPEND_COLUMN_VALUE_1, + SPD_MID_DB_MARIADB_UTIL_APPEND_COLUMN_VALUE_2, + SPD_MID_DB_MBASE_EXEC_QUERY_1, + SPD_MID_DB_MBASE_EXEC_QUERY_2, + SPD_MID_DB_MBASE_INIT_1, + SPD_MID_DB_MBASE_INIT_2, + SPD_MID_DB_MBASE_RESULT_FETCH_ROW_FROM_TMP_TABLE_1, + SPD_MID_DB_MBASE_RESULT_FETCH_ROW_FROM_TMP_TABLE_2, + SPD_MID_DB_MBASE_RESULT_FETCH_ROW_FROM_TMP_TABLE_3, + SPD_MID_DB_MBASE_ROW_APPEND_ESCAPED_TO_STR_1, + SPD_MID_DB_MBASE_ROW_CLONE_1, + SPD_MID_DB_MBASE_SET_LOOP_CHECK_1, + SPD_MID_DB_MBASE_SET_SQL_MODE_1, + SPD_MID_DB_MBASE_SET_TIME_ZONE_1, + SPD_MID_DB_MBASE_SET_WAIT_TIMEOUT_1, + SPD_MID_DB_MBASE_UTIL_PRINT_ITEM_FUNC_1, + SPD_MID_DB_MBASE_UTIL_PRINT_ITEM_FUNC_2, + SPD_MID_DB_MBASE_UTIL_PRINT_ITEM_FUNC_3, + SPD_MID_DB_MBASE_XA_COMMIT_1, + SPD_MID_DB_MBASE_XA_END_1, + SPD_MID_DB_MBASE_XA_PREPARE_1, + SPD_MID_DB_MBASE_XA_ROLLBACK_1, + SPD_MID_DB_MYSQL_UTIL_APPEND_COLUMN_VALUE_1, + SPD_MID_DB_MYSQL_UTIL_APPEND_COLUMN_VALUE_2, + SPD_MID_DB_OPEN_ITEM_INT_1, + SPD_MID_DB_OPEN_ITEM_STRING_1, + SPD_MID_DB_ORACLE_EXEC_QUERY_1, + SPD_MID_DB_ORACLE_GET_ERROR_1, + SPD_MID_DB_ORACLE_INIT_1, + SPD_MID_DB_ORACLE_INIT_2, + SPD_MID_DB_ORACLE_RESULT_FETCH_ROW_FROM_TMP_TABLE_1, + SPD_MID_DB_ORACLE_RESULT_FETCH_ROW_FROM_TMP_TABLE_2, + SPD_MID_DB_ORACLE_RESULT_FETCH_ROW_FROM_TMP_TABLE_3, + SPD_MID_DB_ORACLE_ROW_APPEND_ESCAPED_TO_STR_1, + SPD_MID_DB_ORACLE_ROW_INIT_1, + SPD_MID_DB_ORACLE_ROW_INIT_2, + SPD_MID_DB_ORACLE_UTIL_APPEND_COLUMN_VALUE_1, + SPD_MID_DB_ORACLE_UTIL_APPEND_COLUMN_VALUE_2, + SPD_MID_DB_ORACLE_UTIL_OPEN_ITEM_FUNC_1, + SPD_MID_DB_ORACLE_UTIL_OPEN_ITEM_FUNC_2, + SPD_MID_DB_ORACLE_UTIL_OPEN_ITEM_FUNC_3, + SPD_MID_DB_QUERY_1, + SPD_MID_DB_STORE_RESULT_1, + SPD_MID_DB_STORE_RESULT_2, + SPD_MID_DB_STORE_RESULT_3, + SPD_MID_DB_STORE_RESULT_4, + SPD_MID_DB_STORE_RESULT_5, + SPD_MID_DB_STORE_RESULT_FOR_REUSE_CURSOR_1, + SPD_MID_DB_UDF_COPY_TABLES_1, + SPD_MID_DB_UDF_PING_TABLE_1, + SPD_MID_DB_UDF_PING_TABLE_2, + SPD_MID_DB_UDF_PING_TABLE_APPEND_MON_NEXT_1, + SPD_MID_DB_UDF_PING_TABLE_APPEND_MON_NEXT_2, + SPD_MID_DB_UDF_PING_TABLE_MON_NEXT_1, + SPD_MID_DIRECT_SQL_BODY_1, + SPD_MID_DIRECT_SQL_INIT_BODY_1, + SPD_MID_DISCOVER_TABLE_STRUCTURE_1, + SPD_MID_FIELDS_CREATE_CONN_HOLDER_1, + SPD_MID_FIELDS_CREATE_LINK_IDX_CHAIN_1, + SPD_MID_FIELDS_CREATE_LINK_IDX_HOLDER_1, + SPD_MID_GET_INIT_ERROR_TABLE_1, + SPD_MID_GET_LGTM_TBLHND_SHARE_1, + SPD_MID_GET_PING_TABLE_MON_1, + SPD_MID_GET_PING_TABLE_TGT_1, + SPD_MID_GET_PT_SHARE_1, + SPD_MID_GET_PT_SHARE_2, + SPD_MID_GET_SHARE_1, + SPD_MID_GET_SHARE_2, + SPD_MID_GET_TRX_1, + SPD_MID_GET_TRX_10, + SPD_MID_GET_TRX_2, + SPD_MID_GET_TRX_3, + SPD_MID_GET_TRX_4, + SPD_MID_GET_TRX_5, + SPD_MID_GET_TRX_6, + SPD_MID_GET_TRX_7, + SPD_MID_GET_TRX_8, + SPD_MID_GET_TRX_9, + SPD_MID_HANDLERSOCKET_HANDLER_INIT_1, + SPD_MID_HANDLERSOCKET_HANDLER_INIT_2, + SPD_MID_HANDLERSOCKET_HANDLER_SPIDER_HANDLERSOCKET_HANDLER_1, + SPD_MID_HANDLERSOCKET_SHARE_CREATE_COLUMN_NAME_STR_1, + SPD_MID_HANDLERSOCKET_SHARE_CREATE_TABLE_NAMES_STR_1, + SPD_MID_HANDLERSOCKET_SHARE_CREATE_TABLE_NAMES_STR_2, + SPD_MID_HANDLERSOCKET_SHARE_CREATE_TABLE_NAMES_STR_3, + SPD_MID_HANDLERSOCKET_SHARE_INIT_1, + SPD_MID_HANDLERSOCKET_SHARE_SPIDER_HANDLERSOCKET_SHARE_1, + SPD_MID_HA_SPIDER_COND_PUSH_1, + SPD_MID_HA_SPIDER_CREATE_1, + SPD_MID_HA_SPIDER_CREATE_2, + SPD_MID_HA_SPIDER_CREATE_BULK_ACCESS_LINK_1, + SPD_MID_HA_SPIDER_FT_INIT_EXT_1, + SPD_MID_HA_SPIDER_HA_SPIDER_1, + SPD_MID_HA_SPIDER_HA_SPIDER_2, + SPD_MID_HA_SPIDER_INFO_PUSH_1, + SPD_MID_HA_SPIDER_MULTI_RANGE_READ_NEXT_FIRST_1, + SPD_MID_HA_SPIDER_MULTI_RANGE_READ_NEXT_FIRST_2, + SPD_MID_HA_SPIDER_MULTI_RANGE_READ_NEXT_FIRST_3, + SPD_MID_HA_SPIDER_OPEN_1, + SPD_MID_HA_SPIDER_OPEN_2, + SPD_MID_HA_SPIDER_OPEN_3, + SPD_MID_HA_SPIDER_OPEN_4, + SPD_MID_HA_SPIDER_OPEN_5, + SPD_MID_HA_SPIDER_OPEN_6, + SPD_MID_HA_SPIDER_OPEN_7, + SPD_MID_INCREASE_LONGLONG_LIST_1, + SPD_MID_INCREASE_LONG_LIST_1, + SPD_MID_INCREASE_NULL_STRING_LIST_1, + SPD_MID_INCREASE_STRING_LIST_1, + SPD_MID_MBASE_COPY_TABLE_INIT_1, + SPD_MID_MBASE_HANDLER_APPEND_KEY_COLUMN_TYPES_1, + SPD_MID_MBASE_HANDLER_APPEND_MATCH_AGAINST_1, + SPD_MID_MBASE_HANDLER_APPEND_TMP_TABLE_AND_SQL_FOR_BKA_1, + SPD_MID_MBASE_HANDLER_APPEND_UNION_TABLE_AND_SQL_FOR_BKA_1, + SPD_MID_MBASE_HANDLER_INIT_1, + SPD_MID_MBASE_HANDLER_INIT_2, + SPD_MID_MBASE_HANDLER_INIT_3, + SPD_MID_MBASE_HANDLER_INIT_4, + SPD_MID_MBASE_HANDLER_INIT_5, + SPD_MID_MBASE_HANDLER_INIT_6, + SPD_MID_MBASE_HANDLER_INIT_7, + SPD_MID_MBASE_HANDLER_INIT_8, + SPD_MID_MBASE_HANDLER_INIT_9, + SPD_MID_MBASE_HANDLER_INIT_UNION_TABLE_NAME_POS_1, + SPD_MID_MBASE_HANDLER_SET_SQL_FOR_EXEC_1, + SPD_MID_MBASE_HANDLER_SET_UNION_TABLE_NAME_POS_1, + SPD_MID_MBASE_HANDLER_SPIDER_MBASE_HANDLER_1, + SPD_MID_MBASE_SHARE_APPEND_SHOW_INDEX_1, + SPD_MID_MBASE_SHARE_APPEND_SHOW_INDEX_2, + SPD_MID_MBASE_SHARE_APPEND_SHOW_RECORDS_1, + SPD_MID_MBASE_SHARE_APPEND_SHOW_TABLE_STATUS_1, + SPD_MID_MBASE_SHARE_APPEND_SHOW_TABLE_STATUS_2, + SPD_MID_MBASE_SHARE_CREATE_COLUMN_NAME_STR_1, + SPD_MID_MBASE_SHARE_CREATE_TABLE_NAMES_STR_1, + SPD_MID_MBASE_SHARE_CREATE_TABLE_NAMES_STR_2, + SPD_MID_MBASE_SHARE_CREATE_TABLE_NAMES_STR_3, + SPD_MID_MBASE_SHARE_DISCOVER_TABLE_STRUCTURE_1, + SPD_MID_MBASE_SHARE_INIT_1, + SPD_MID_MBASE_SHARE_INIT_2, + SPD_MID_MBASE_SHARE_INIT_3, + SPD_MID_MBASE_SHARE_INIT_4, + SPD_MID_MBASE_SHARE_SPIDER_MBASE_SHARE_1, + SPD_MID_OPEN_ALL_TABLES_1, + SPD_MID_OPEN_SYS_TABLE_1, + SPD_MID_ORACLE_COPY_TABLE_COPY_ROWS_1, + SPD_MID_ORACLE_COPY_TABLE_COPY_ROWS_2, + SPD_MID_ORACLE_COPY_TABLE_COPY_ROWS_3, + SPD_MID_ORACLE_COPY_TABLE_COPY_ROWS_4, + SPD_MID_ORACLE_COPY_TABLE_INIT_1, + SPD_MID_ORACLE_COPY_TABLE_INIT_2, + SPD_MID_ORACLE_HANDLER_APPEND_KEY_COLUMN_TYPES_1, + SPD_MID_ORACLE_HANDLER_APPEND_MATCH_AGAINST_1, + SPD_MID_ORACLE_HANDLER_APPEND_TMP_TABLE_AND_SQL_FOR_BKA_1, + SPD_MID_ORACLE_HANDLER_APPEND_UNION_TABLE_AND_SQL_FOR_BKA_1, + SPD_MID_ORACLE_HANDLER_INIT_1, + SPD_MID_ORACLE_HANDLER_INIT_2, + SPD_MID_ORACLE_HANDLER_INIT_3, + SPD_MID_ORACLE_HANDLER_INIT_4, + SPD_MID_ORACLE_HANDLER_INIT_5, + SPD_MID_ORACLE_HANDLER_INIT_6, + SPD_MID_ORACLE_HANDLER_INIT_7, + SPD_MID_ORACLE_HANDLER_INIT_8, + SPD_MID_ORACLE_HANDLER_INIT_9, + SPD_MID_ORACLE_HANDLER_INIT_UNION_TABLE_NAME_POS_1, + SPD_MID_ORACLE_HANDLER_SET_SQL_FOR_EXEC_1, + SPD_MID_ORACLE_HANDLER_SET_UNION_TABLE_NAME_POS_1, + SPD_MID_ORACLE_HANDLER_SPIDER_ORACLE_HANDLER_1, + SPD_MID_ORACLE_SHARE_APPEND_SHOW_AUTOINC_1, + SPD_MID_ORACLE_SHARE_APPEND_SHOW_INDEX_1, + SPD_MID_ORACLE_SHARE_APPEND_SHOW_INDEX_2, + SPD_MID_ORACLE_SHARE_APPEND_SHOW_LAST_INSERT_ID_1, + SPD_MID_ORACLE_SHARE_APPEND_SHOW_LAST_INSERT_ID_2, + SPD_MID_ORACLE_SHARE_APPEND_SHOW_RECORDS_1, + SPD_MID_ORACLE_SHARE_APPEND_SHOW_TABLE_STATUS_1, + SPD_MID_ORACLE_SHARE_APPEND_SHOW_TABLE_STATUS_2, + SPD_MID_ORACLE_SHARE_CREATE_COLUMN_NAME_STR_1, + SPD_MID_ORACLE_SHARE_CREATE_TABLE_NAMES_STR_1, + SPD_MID_ORACLE_SHARE_CREATE_TABLE_NAMES_STR_2, + SPD_MID_ORACLE_SHARE_CREATE_TABLE_NAMES_STR_3, + SPD_MID_ORACLE_SHARE_INIT_1, + SPD_MID_ORACLE_SHARE_INIT_2, + SPD_MID_ORACLE_SHARE_INIT_3, + SPD_MID_ORACLE_SHARE_INIT_4, + SPD_MID_ORACLE_SHARE_SPIDER_ORACLE_SHARE_1, + SPD_MID_PARSE_CONNECT_INFO_1, + SPD_MID_PING_TABLE_BODY_1, + SPD_MID_PING_TABLE_BODY_2, + SPD_MID_PING_TABLE_INIT_BODY_1, + SPD_MID_PING_TABLE_MON_FROM_TABLE_1, + SPD_MID_RELEASE_PING_TABLE_MON_LIST_1, + SPD_MID_TRX_ANOTHER_LOCK_TABLES_1, + SPD_MID_UDF_COPY_TABLES_CREATE_TABLE_LIST_1, + SPD_MID_UDF_DIRECT_SQL_CREATE_CONN_1, + SPD_MID_UDF_DIRECT_SQL_CREATE_CONN_2, + SPD_MID_UDF_DIRECT_SQL_CREATE_CONN_3, + SPD_MID_UDF_DIRECT_SQL_CREATE_CONN_4, + SPD_MID_UDF_DIRECT_SQL_CREATE_CONN_KEY_1, + SPD_MID_UDF_DIRECT_SQL_CREATE_TABLE_LIST_1, + SPD_MID_UDF_DIRECT_SQL_CREATE_TABLE_LIST_2, + SPD_MID_UDF_GET_COPY_TGT_TABLES_1 +}; + #define SPIDER_BACKUP_DASTATUS \ bool da_status; if (thd) da_status = thd->is_error(); else da_status = FALSE; #define SPIDER_RESTORE_DASTATUS \ @@ -518,7 +791,6 @@ typedef struct st_spider_conn SPIDER_CONN_LOOP_CHECK *loop_check_ignored_first; SPIDER_CONN_LOOP_CHECK *loop_check_ignored_last; SPIDER_CONN_LOOP_CHECK *loop_check_meraged_first; - SPIDER_CONN_LOOP_CHECK *loop_check_meraged_last; } SPIDER_CONN; typedef struct st_spider_lgtm_tblhnd_share @@ -934,6 +1206,7 @@ typedef struct st_spider_share uint *tgt_pk_names_lengths; uint *tgt_sequence_names_lengths; uint *conn_keys_lengths; + /* The index in `spider_dbton' of each data node link. */ uint *sql_dbton_ids; uint server_names_charlen; @@ -1003,10 +1276,16 @@ typedef struct st_spider_share uchar dbton_bitmap[spider_bitmap_size(SPIDER_DBTON_SIZE)]; spider_db_share *dbton_share[SPIDER_DBTON_SIZE]; uint use_dbton_count; + /* Actual size is `use_dbton_count'. Values are the indices of item + in `spider_dbton'. */ uint use_dbton_ids[SPIDER_DBTON_SIZE]; + /* Inverse map of `use_dbton_ids'. */ uint dbton_id_to_seq[SPIDER_DBTON_SIZE]; uint use_sql_dbton_count; + /* Actual size is `use_sql_dbton_count'. Values are the indices of + item in `spider_dbton'. */ uint use_sql_dbton_ids[SPIDER_DBTON_SIZE]; + /* Inverse map of `use_sql_dbton_ids'. */ uint sql_dbton_id_to_seq[SPIDER_DBTON_SIZE]; SPIDER_ALTER_TABLE alter_table; diff --git a/storage/spider/spd_init_query.h b/storage/spider/spd_init_query.h index fdf6b22f..35a250c7 100644 --- a/storage/spider/spd_init_query.h +++ b/storage/spider/spd_init_query.h @@ -21,6 +21,9 @@ static LEX_STRING spider_init_queries[] = { {C_STRING_WITH_LEN( + "SET @@SQL_MODE = REPLACE(@@SQL_MODE, 'ORACLE', '');" + )}, + {C_STRING_WITH_LEN( "create table if not exists mysql.spider_xa(" " format_id int not null default 0," " gtrid_length int not null default 0," @@ -694,6 +697,46 @@ static LEX_STRING spider_init_queries[] = { " algorithm=copy, lock=shared;" )}, {C_STRING_WITH_LEN( + "alter table mysql.spider_link_mon_servers" + " add column if not exists filedsn text default null after dsn," + " algorithm=copy, lock=shared;" + )}, + {C_STRING_WITH_LEN( + "alter table mysql.spider_tables" + " add column if not exists filedsn text default null after dsn," + " algorithm=copy, lock=shared;" + )}, + {C_STRING_WITH_LEN( + "alter table mysql.spider_xa_failed_log" + " add column if not exists filedsn text default null after dsn," + " algorithm=copy, lock=shared;" + )}, + {C_STRING_WITH_LEN( + "alter table mysql.spider_xa_member" + " add column if not exists filedsn text default null after dsn," + " algorithm=copy, lock=shared;" + )}, + {C_STRING_WITH_LEN( + "alter table mysql.spider_link_mon_servers" + " add column if not exists driver char(64) default null after filedsn," + " algorithm=copy, lock=shared;" + )}, + {C_STRING_WITH_LEN( + "alter table mysql.spider_tables" + " add column if not exists driver char(64) default null after filedsn," + " algorithm=copy, lock=shared;" + )}, + {C_STRING_WITH_LEN( + "alter table mysql.spider_xa_failed_log" + " add column if not exists driver char(64) default null after filedsn," + " algorithm=copy, lock=shared;" + )}, + {C_STRING_WITH_LEN( + "alter table mysql.spider_xa_member" + " add column if not exists driver char(64) default null after filedsn," + " algorithm=copy, lock=shared;" + )}, + {C_STRING_WITH_LEN( "set @win_plugin := IF(@@version_compile_os like 'Win%', 1, 0);" )}, /* Install UDFs. If udf is not initialised, then install by diff --git a/storage/spider/spd_param.cc b/storage/spider/spd_param.cc index 2328cce7..d86e8532 100644 --- a/storage/spider/spd_param.cc +++ b/storage/spider/spd_param.cc @@ -109,59 +109,61 @@ extern volatile ulonglong spider_mon_table_cache_version_req; MYSQL_SYSVAR_NAME(param_name).def_val; \ } -static int spider_direct_update(THD *thd, SHOW_VAR *var, char *buff) +extern handlerton *spider_hton_ptr; +static void spider_trx_status_var(THD *thd, SHOW_VAR *var, void *buff, + ulonglong SPIDER_TRX::*counter) { - int error_num = 0; - SPIDER_TRX *trx; DBUG_ENTER("spider_direct_update"); var->type = SHOW_LONGLONG; - if ((trx = spider_get_trx(thd, TRUE, &error_num))) - var->value = (char *) &trx->direct_update_count; - DBUG_RETURN(error_num); + var->value= buff; + if (thd != current_thd) + mysql_mutex_lock(&thd->LOCK_thd_data); + SPIDER_TRX *trx = (SPIDER_TRX*)thd_get_ha_data(thd, spider_hton_ptr); + *(ulonglong*)buff= trx ? trx->*counter : 0; + if (thd != current_thd) + mysql_mutex_unlock(&thd->LOCK_thd_data); + DBUG_VOID_RETURN; } -static int spider_direct_delete(THD *thd, SHOW_VAR *var, char *buff) + +static int spider_direct_update(THD *thd, SHOW_VAR *var, void *buff, + system_status_var *, enum_var_type) +{ + DBUG_ENTER("spider_direct_update"); + spider_trx_status_var(thd, var, buff, &SPIDER_TRX::direct_update_count); + DBUG_RETURN(0); +} + +static int spider_direct_delete(THD *thd, SHOW_VAR *var, void *buff, + system_status_var *, enum_var_type) { - int error_num = 0; - SPIDER_TRX *trx; DBUG_ENTER("spider_direct_delete"); - var->type = SHOW_LONGLONG; - if ((trx = spider_get_trx(thd, TRUE, &error_num))) - var->value = (char *) &trx->direct_delete_count; - DBUG_RETURN(error_num); + spider_trx_status_var(thd, var, buff, &SPIDER_TRX::direct_delete_count); + DBUG_RETURN(0); } -static int spider_direct_order_limit(THD *thd, SHOW_VAR *var, char *buff) +static int spider_direct_order_limit(THD *thd, SHOW_VAR *var, void *buff, + system_status_var *, enum_var_type) { - int error_num = 0; - SPIDER_TRX *trx; DBUG_ENTER("spider_direct_order_limit"); - var->type = SHOW_LONGLONG; - if ((trx = spider_get_trx(thd, TRUE, &error_num))) - var->value = (char *) &trx->direct_order_limit_count; - DBUG_RETURN(error_num); + spider_trx_status_var(thd, var, buff, &SPIDER_TRX::direct_order_limit_count); + DBUG_RETURN(0); } -static int spider_direct_aggregate(THD *thd, SHOW_VAR *var, char *buff) +static int spider_direct_aggregate(THD *thd, SHOW_VAR *var, void *buff, + system_status_var *, enum_var_type) { - int error_num = 0; - SPIDER_TRX *trx; DBUG_ENTER("spider_direct_aggregate"); - var->type = SHOW_LONGLONG; - if ((trx = spider_get_trx(thd, TRUE, &error_num))) - var->value = (char *) &trx->direct_aggregate_count; - DBUG_RETURN(error_num); + spider_trx_status_var(thd, var, buff, &SPIDER_TRX::direct_aggregate_count); + DBUG_RETURN(0); } -static int spider_parallel_search(THD *thd, SHOW_VAR *var, char *buff) +static int spider_parallel_search(THD *thd, SHOW_VAR *var, void *buff, + system_status_var *, enum_var_type) { - int error_num = 0; - SPIDER_TRX *trx; DBUG_ENTER("spider_parallel_search"); - var->type = SHOW_LONGLONG; - if ((trx = spider_get_trx(thd, TRUE, &error_num))) - var->value = (char *) &trx->parallel_search_count; - DBUG_RETURN(error_num); + spider_trx_status_var(thd, var, buff, &SPIDER_TRX::parallel_search_count); + DBUG_RETURN(0); } struct st_mysql_show_var spider_status_variables[] = @@ -2217,7 +2219,7 @@ static MYSQL_SYSVAR_UINT( "Static thread count of table sts", NULL, NULL, - 10, + 1, 1, 4294967295U, 0 @@ -2236,7 +2238,7 @@ static MYSQL_SYSVAR_UINT( "Static thread count of table crd", NULL, NULL, - 10, + 1, 1, 4294967295U, 0 diff --git a/storage/spider/spd_ping_table.cc b/storage/spider/spd_ping_table.cc index b331a9fe..722b4cca 100644 --- a/storage/spider/spd_ping_table.cc +++ b/storage/spider/spd_ping_table.cc @@ -219,7 +219,7 @@ int spider_release_ping_table_mon_list( } spider_string conv_name_str(buf, conv_name_length + link_idx_str_length + 1, system_charset_info); - conv_name_str.init_calc_mem(134); + conv_name_str.init_calc_mem(SPD_MID_RELEASE_PING_TABLE_MON_LIST_1); conv_name_str.length(0); conv_name_str.q_append(conv_name, conv_name_length); conv_name_str.q_append(link_idx_str, link_idx_str_length); @@ -325,7 +325,7 @@ create_table_mon: do { if (!(table_mon = (SPIDER_TABLE_MON *) - spider_bulk_malloc(spider_current_trx, 35, MYF(MY_WME | MY_ZEROFILL), + spider_bulk_malloc(spider_current_trx, SPD_MID_GET_PING_TABLE_MON_1, MYF(MY_WME | MY_ZEROFILL), &table_mon, (uint) (sizeof(SPIDER_TABLE_MON)), &tmp_share, (uint) (sizeof(SPIDER_SHARE)), &tmp_connect_info, @@ -446,7 +446,7 @@ SPIDER_TABLE_MON_LIST *spider_get_ping_table_tgt( SPD_INIT_ALLOC_ROOT(&mem_root, 4096, 0, MYF(MY_WME)); if (!(table_mon_list = (SPIDER_TABLE_MON_LIST *) - spider_bulk_malloc(spider_current_trx, 36, MYF(MY_WME | MY_ZEROFILL), + spider_bulk_malloc(spider_current_trx, SPD_MID_GET_PING_TABLE_TGT_1, MYF(MY_WME | MY_ZEROFILL), &table_mon_list, (uint) (sizeof(SPIDER_TABLE_MON_LIST)), &tmp_share, (uint) (sizeof(SPIDER_SHARE)), &tmp_connect_info, @@ -1024,8 +1024,8 @@ long long spider_ping_table_body( int static_link_id_length = 0; bool get_lock = FALSE, status_changed_to_ng = FALSE; DBUG_ENTER("spider_ping_table_body"); - conv_name.init_calc_mem(135); - tmp_str.init_calc_mem(247); + conv_name.init_calc_mem(SPD_MID_PING_TABLE_BODY_1); + tmp_str.init_calc_mem(SPD_MID_PING_TABLE_BODY_2); conv_name.length(0); server_id = global_system_variables.server_id; if ( @@ -1463,7 +1463,7 @@ my_bool spider_ping_table_init_body( } if (!(mon_table_result = (SPIDER_MON_TABLE_RESULT *) - spider_malloc(spider_current_trx, 11, sizeof(SPIDER_MON_TABLE_RESULT), + spider_malloc(spider_current_trx, SPD_MID_PING_TABLE_INIT_BODY_1, sizeof(SPIDER_MON_TABLE_RESULT), MYF(MY_WME | MY_ZEROFILL))) ) { strcpy(message, "spider_ping_table() out of memory"); @@ -1606,7 +1606,7 @@ int spider_ping_table_mon_from_table( buf[conv_name_length + link_idx_str_length] = '\0'; spider_string conv_name_str(buf, conv_name_length + link_idx_str_length + 1, system_charset_info); - conv_name_str.init_calc_mem(136); + conv_name_str.init_calc_mem(SPD_MID_PING_TABLE_MON_FROM_TABLE_1); conv_name_str.length(0); conv_name_str.q_append(conv_name, conv_name_length); conv_name_str.q_append(link_idx_str, link_idx_str_length + 1); diff --git a/storage/spider/spd_sys_table.cc b/storage/spider/spd_sys_table.cc index a0cf104d..912e02a7 100644 --- a/storage/spider/spd_sys_table.cc +++ b/storage/spider/spd_sys_table.cc @@ -325,38 +325,6 @@ TABLE *spider_open_sys_table( } break; } - if (!memcmp(table_name, SPIDER_SYS_TABLE_STS_TABLE_NAME_STR, - SPIDER_SYS_TABLE_STS_TABLE_NAME_LEN)) - { - DBUG_PRINT("info",("spider checking for SYS_TABLE_STS")); - if (table->s->fields != SPIDER_SYS_TABLE_STS_COL_CNT) - { - spider_close_sys_table(thd, table, open_tables_backup, need_lock); - table = NULL; - my_printf_error(ER_SPIDER_SYS_TABLE_VERSION_NUM, - ER_SPIDER_SYS_TABLE_VERSION_STR, MYF(0), - SPIDER_SYS_TABLE_STS_TABLE_NAME_STR); - *error_num = ER_SPIDER_SYS_TABLE_VERSION_NUM; - goto error_col_num_chk; - } - break; - } - if (!memcmp(table_name, SPIDER_SYS_TABLE_CRD_TABLE_NAME_STR, - SPIDER_SYS_TABLE_CRD_TABLE_NAME_LEN)) - { - DBUG_PRINT("info",("spider checking for SYS_TABLE_CRD")); - if (table->s->fields != SPIDER_SYS_TABLE_CRD_COL_CNT) - { - spider_close_sys_table(thd, table, open_tables_backup, need_lock); - table = NULL; - my_printf_error(ER_SPIDER_SYS_TABLE_VERSION_NUM, - ER_SPIDER_SYS_TABLE_VERSION_STR, MYF(0), - SPIDER_SYS_TABLE_CRD_TABLE_NAME_STR); - *error_num = ER_SPIDER_SYS_TABLE_VERSION_NUM; - goto error_col_num_chk; - } - break; - } DBUG_ASSERT(0); break; case 20: @@ -378,24 +346,6 @@ TABLE *spider_open_sys_table( } DBUG_ASSERT(0); break; - case 21: - if (!memcmp(table_name, SPIDER_SYS_RW_TBLS_TABLE_NAME_STR, - SPIDER_SYS_RW_TBLS_TABLE_NAME_LEN)) - { - DBUG_PRINT("info",("spider checking for SYS_RW_TBLS")); - if (table->s->fields != SPIDER_SYS_RW_TBLS_COL_CNT) - { - spider_close_sys_table(thd, table, open_tables_backup, need_lock); - table = NULL; - my_printf_error(ER_SPIDER_SYS_TABLE_VERSION_NUM, - ER_SPIDER_SYS_TABLE_VERSION_STR, MYF(0), - SPIDER_SYS_RW_TBLS_TABLE_NAME_STR); - *error_num = ER_SPIDER_SYS_TABLE_VERSION_NUM; - goto error_col_num_chk; - } - } - DBUG_ASSERT(0); - break; case 22: if (!memcmp(table_name, SPIDER_SYS_LINK_FAILED_TABLE_NAME_STR, SPIDER_SYS_LINK_FAILED_TABLE_NAME_LEN)) @@ -432,60 +382,6 @@ TABLE *spider_open_sys_table( } break; } - if (!memcmp(table_name, SPIDER_SYS_RWN_TBLS_TABLE_NAME_STR, - SPIDER_SYS_RWN_TBLS_TABLE_NAME_LEN)) - { - DBUG_PRINT("info",("spider checking for SYS_RWN_TBLS")); - if (table->s->fields != SPIDER_SYS_RWN_TBLS_COL_CNT) - { - spider_close_sys_table(thd, table, open_tables_backup, need_lock); - table = NULL; - my_printf_error(ER_SPIDER_SYS_TABLE_VERSION_NUM, - ER_SPIDER_SYS_TABLE_VERSION_STR, MYF(0), - SPIDER_SYS_RWN_TBLS_TABLE_NAME_STR); - *error_num = ER_SPIDER_SYS_TABLE_VERSION_NUM; - goto error_col_num_chk; - } - break; - } - DBUG_ASSERT(0); - break; - case 27: - if (!memcmp(table_name, SPIDER_SYS_RW_TBL_TBLS_TABLE_NAME_STR, - SPIDER_SYS_RW_TBL_TBLS_TABLE_NAME_LEN)) - { - DBUG_PRINT("info",("spider checking for SYS_RW_TBL_TBLS")); - if (table->s->fields != SPIDER_SYS_RW_TBL_TBLS_COL_CNT) - { - spider_close_sys_table(thd, table, open_tables_backup, need_lock); - table = NULL; - my_printf_error(ER_SPIDER_SYS_TABLE_VERSION_NUM, - ER_SPIDER_SYS_TABLE_VERSION_STR, MYF(0), - SPIDER_SYS_RW_TBL_TBLS_TABLE_NAME_STR); - *error_num = ER_SPIDER_SYS_TABLE_VERSION_NUM; - goto error_col_num_chk; - } - break; - } - DBUG_ASSERT(0); - break; - case 31: - if (!memcmp(table_name, SPIDER_SYS_RW_TBL_PTTS_TABLE_NAME_STR, - SPIDER_SYS_RW_TBL_PTTS_TABLE_NAME_LEN)) - { - DBUG_PRINT("info",("spider checking for SYS_RW_TBL_PTTS")); - if (table->s->fields != SPIDER_SYS_RW_TBL_PTTS_COL_CNT) - { - spider_close_sys_table(thd, table, open_tables_backup, need_lock); - table = NULL; - my_printf_error(ER_SPIDER_SYS_TABLE_VERSION_NUM, - ER_SPIDER_SYS_TABLE_VERSION_STR, MYF(0), - SPIDER_SYS_RW_TBL_PTTS_TABLE_NAME_STR); - *error_num = ER_SPIDER_SYS_TABLE_VERSION_NUM; - goto error_col_num_chk; - } - break; - } DBUG_ASSERT(0); break; case 34: @@ -505,22 +401,6 @@ TABLE *spider_open_sys_table( } break; } - if (!memcmp(table_name, SPIDER_SYS_RW_TBL_SPTTS_TABLE_NAME_STR, - SPIDER_SYS_RW_TBL_SPTTS_TABLE_NAME_LEN)) - { - DBUG_PRINT("info",("spider checking for SYS_RW_TBL_SPTTS")); - if (table->s->fields != SPIDER_SYS_RW_TBL_SPTTS_COL_CNT) - { - spider_close_sys_table(thd, table, open_tables_backup, need_lock); - table = NULL; - my_printf_error(ER_SPIDER_SYS_TABLE_VERSION_NUM, - ER_SPIDER_SYS_TABLE_VERSION_STR, MYF(0), - SPIDER_SYS_RW_TBL_SPTTS_TABLE_NAME_STR); - *error_num = ER_SPIDER_SYS_TABLE_VERSION_NUM; - goto error_col_num_chk; - } - break; - } DBUG_ASSERT(0); break; default: @@ -1455,52 +1335,6 @@ void spider_store_binlog_pos_gtid( DBUG_VOID_RETURN; } -void spider_store_table_sts_info( - TABLE *table, - ha_statistics *stat -) { - MYSQL_TIME mysql_time; - DBUG_ENTER("spider_store_table_sts_info"); - table->field[SPIDER_TABLE_STS_DATA_FILE_LENGTH_POS]->store( - (longlong) stat->data_file_length, TRUE); - table->field[SPIDER_TABLE_STS_MAX_DATA_FILE_LENGTH_POS]->store( - (longlong) stat->max_data_file_length, TRUE); - table->field[SPIDER_TABLE_STS_INDEX_FILE_LENGTH_POS]->store( - (longlong) stat->index_file_length, TRUE); - table->field[SPIDER_TABLE_STS_RECORDS_POS]->store( - (longlong) stat->records, TRUE); - table->field[SPIDER_TABLE_STS_MEAN_REC_LENGTH_POS]->store( - (longlong) stat->mean_rec_length, TRUE); - spd_tz_system->gmt_sec_to_TIME(&mysql_time, (my_time_t) stat->check_time); - table->field[SPIDER_TABLE_STS_CHECK_TIME_POS]->store_time(&mysql_time); - spd_tz_system->gmt_sec_to_TIME(&mysql_time, (my_time_t) stat->create_time); - table->field[SPIDER_TABLE_STS_CREATE_TIME_POS]->store_time(&mysql_time); - spd_tz_system->gmt_sec_to_TIME(&mysql_time, (my_time_t) stat->update_time); - table->field[SPIDER_TABLE_STS_UPDATE_TIME_POS]->store_time(&mysql_time); - if (stat->checksum_null) - { - table->field[SPIDER_TABLE_STS_CHECKSUM_POS]->set_null(); - table->field[SPIDER_TABLE_STS_CHECKSUM_POS]->reset(); - } else { - table->field[SPIDER_TABLE_STS_CHECKSUM_POS]->set_notnull(); - table->field[SPIDER_TABLE_STS_CHECKSUM_POS]->store( - (longlong) stat->checksum, TRUE); - } - DBUG_VOID_RETURN; -} - -void spider_store_table_crd_info( - TABLE *table, - uint *seq, - longlong *cardinality -) { - DBUG_ENTER("spider_store_table_crd_info"); - table->field[SPIDER_TABLE_CRD_KEY_SEQ_POS]->store((longlong) *seq, TRUE); - table->field[SPIDER_TABLE_CRD_CARDINALITY_POS]->store( - (longlong) *cardinality, FALSE); - DBUG_VOID_RETURN; -} - int spider_insert_xa( TABLE *table, XID *xid, @@ -1607,83 +1441,6 @@ int spider_insert_sys_table( DBUG_RETURN(error_num); } -int spider_insert_or_update_table_sts( - TABLE *table, - const char *name, - uint name_length, - ha_statistics *stat -) { - int error_num; - char table_key[MAX_KEY_LENGTH]; - DBUG_ENTER("spider_insert_or_update_table_sts"); - table->use_all_columns(); - spider_store_tables_name(table, name, name_length); - spider_store_table_sts_info( - table, - stat - ); - - if ((error_num = spider_check_sys_table_for_update_all_columns(table, table_key))) - { - if (error_num != HA_ERR_KEY_NOT_FOUND && error_num != HA_ERR_END_OF_FILE) - { - table->file->print_error(error_num, MYF(0)); - DBUG_RETURN(error_num); - } - if ((error_num = spider_write_sys_table_row(table))) - { - DBUG_RETURN(error_num); - } - } else { - if ((error_num = spider_update_sys_table_row(table, FALSE))) - { - table->file->print_error(error_num, MYF(0)); - DBUG_RETURN(error_num); - } - } - - DBUG_RETURN(0); -} - -int spider_insert_or_update_table_crd( - TABLE *table, - const char *name, - uint name_length, - longlong *cardinality, - uint number_of_keys -) { - int error_num; - uint roop_count; - char table_key[MAX_KEY_LENGTH]; - DBUG_ENTER("spider_insert_or_update_table_crd"); - table->use_all_columns(); - spider_store_tables_name(table, name, name_length); - - for (roop_count = 0; roop_count < number_of_keys; ++roop_count) - { - spider_store_table_crd_info(table, &roop_count, &cardinality[roop_count]); - if ((error_num = spider_check_sys_table_for_update_all_columns(table, table_key))) - { - if (error_num != HA_ERR_KEY_NOT_FOUND && error_num != HA_ERR_END_OF_FILE) - { - table->file->print_error(error_num, MYF(0)); - DBUG_RETURN(error_num); - } - if ((error_num = spider_write_sys_table_row(table))) - { - DBUG_RETURN(error_num); - } - } else { - if ((error_num = spider_update_sys_table_row(table, FALSE))) - { - table->file->print_error(error_num, MYF(0)); - DBUG_RETURN(error_num); - } - } - } - DBUG_RETURN(0); -} - int spider_log_tables_link_failed( TABLE *table, char *name, @@ -2009,6 +1766,16 @@ int spider_delete_xa_member( DBUG_RETURN(0); } +/** + Delete a Spider table from mysql.spider_tables. + + @param table The table mysql.spider_tables + @param name The name of the Spider table to delete + @param old_link_count The number of links in the deleted table + + @retval 0 Success + @retval nonzero Failure +*/ int spider_delete_tables( TABLE *table, const char *name, @@ -2024,10 +1791,20 @@ int spider_delete_tables( { spider_store_tables_link_idx(table, roop_count); if ((error_num = spider_check_sys_table(table, table_key))) - break; + { + /* There's a problem with finding the first record for the + spider table, likely because it does not exist. Fail */ + if (roop_count == 0) + DBUG_RETURN(error_num); + /* At least one row has been deleted for the Spider table. + Success */ + else + break; + } else { if ((error_num = spider_delete_sys_table_row(table))) { + /* There's a problem deleting the row. Fail */ DBUG_RETURN(error_num); } } @@ -2728,64 +2505,6 @@ int spider_get_sys_tables_static_link_id( DBUG_RETURN(error_num); } -void spider_get_sys_table_sts_info( - TABLE *table, - ha_statistics *stat -) { - MYSQL_TIME mysql_time; - uint not_used_uint; - long not_used_long; - DBUG_ENTER("spider_get_sys_table_sts_info"); - stat->data_file_length = (ulonglong) table-> - field[SPIDER_TABLE_STS_DATA_FILE_LENGTH_POS]->val_int(); - stat->max_data_file_length = (ulonglong) table-> - field[SPIDER_TABLE_STS_MAX_DATA_FILE_LENGTH_POS]->val_int(); - stat->index_file_length = (ulonglong) table-> - field[SPIDER_TABLE_STS_INDEX_FILE_LENGTH_POS]->val_int(); - stat->records = (ha_rows) table-> - field[SPIDER_TABLE_STS_RECORDS_POS]->val_int(); - stat->mean_rec_length = (ulong) table-> - field[SPIDER_TABLE_STS_MEAN_REC_LENGTH_POS]->val_int(); - table->field[SPIDER_TABLE_STS_CHECK_TIME_POS]->get_date(&mysql_time, - SPIDER_date_mode_t(0)); - stat->check_time = (time_t) my_system_gmt_sec(&mysql_time, - ¬_used_long, ¬_used_uint); - table->field[SPIDER_TABLE_STS_CREATE_TIME_POS]->get_date(&mysql_time, - SPIDER_date_mode_t(0)); - stat->create_time = (time_t) my_system_gmt_sec(&mysql_time, - ¬_used_long, ¬_used_uint); - table->field[SPIDER_TABLE_STS_UPDATE_TIME_POS]->get_date(&mysql_time, - SPIDER_date_mode_t(0)); - stat->update_time = (time_t) my_system_gmt_sec(&mysql_time, - ¬_used_long, ¬_used_uint); - if (table->field[SPIDER_TABLE_STS_CHECKSUM_POS]->is_null()) - { - stat->checksum_null = TRUE; - stat->checksum = 0; - } else { - stat->checksum_null = FALSE; - stat->checksum = (ha_checksum) table-> - field[SPIDER_TABLE_STS_CHECKSUM_POS]->val_int(); - } - DBUG_VOID_RETURN; -} - -void spider_get_sys_table_crd_info( - TABLE *table, - longlong *cardinality, - uint number_of_keys -) { - uint seq; - DBUG_ENTER("spider_get_sys_table_crd_info"); - seq = (uint) table->field[SPIDER_TABLE_CRD_KEY_SEQ_POS]->val_int(); - if (seq < number_of_keys) - { - cardinality[seq] = (longlong) table-> - field[SPIDER_TABLE_CRD_CARDINALITY_POS]->val_int(); - } - DBUG_VOID_RETURN; -} - int spider_sys_update_tables_link_status( THD *thd, char *name, @@ -3226,260 +2945,6 @@ int spider_get_link_statuses( DBUG_RETURN(0); } -int spider_sys_insert_or_update_table_sts( - THD *thd, - const char *name, - uint name_length, - ha_statistics *stat, - bool need_lock -) { - int error_num; - TABLE *table_sts = NULL; - SPIDER_Open_tables_backup open_tables_backup; - DBUG_ENTER("spider_sys_insert_or_update_table_sts"); - if ( - !(table_sts = spider_open_sys_table( - thd, SPIDER_SYS_TABLE_STS_TABLE_NAME_STR, - SPIDER_SYS_TABLE_STS_TABLE_NAME_LEN, TRUE, - &open_tables_backup, need_lock, &error_num)) - ) { - goto error; - } - if ((error_num = spider_insert_or_update_table_sts( - table_sts, - name, - name_length, - stat - ))) - goto error; - spider_close_sys_table(thd, table_sts, &open_tables_backup, need_lock); - table_sts = NULL; - DBUG_RETURN(0); - -error: - if (table_sts) - spider_close_sys_table(thd, table_sts, &open_tables_backup, need_lock); - DBUG_RETURN(error_num); -} - -int spider_sys_insert_or_update_table_crd( - THD *thd, - const char *name, - uint name_length, - longlong *cardinality, - uint number_of_keys, - bool need_lock -) { - int error_num; - TABLE *table_crd = NULL; - SPIDER_Open_tables_backup open_tables_backup; - DBUG_ENTER("spider_sys_insert_or_update_table_crd"); - if ( - !(table_crd = spider_open_sys_table( - thd, SPIDER_SYS_TABLE_CRD_TABLE_NAME_STR, - SPIDER_SYS_TABLE_CRD_TABLE_NAME_LEN, TRUE, - &open_tables_backup, need_lock, &error_num)) - ) { - goto error; - } - if ((error_num = spider_insert_or_update_table_crd( - table_crd, - name, - name_length, - cardinality, - number_of_keys - ))) - goto error; - spider_close_sys_table(thd, table_crd, &open_tables_backup, need_lock); - table_crd = NULL; - DBUG_RETURN(0); - -error: - if (table_crd) - spider_close_sys_table(thd, table_crd, &open_tables_backup, need_lock); - DBUG_RETURN(error_num); -} - -int spider_sys_delete_table_sts( - THD *thd, - const char *name, - uint name_length, - bool need_lock -) { - int error_num; - TABLE *table_sts = NULL; - SPIDER_Open_tables_backup open_tables_backup; - DBUG_ENTER("spider_sys_delete_table_sts"); - if ( - !(table_sts = spider_open_sys_table( - thd, SPIDER_SYS_TABLE_STS_TABLE_NAME_STR, - SPIDER_SYS_TABLE_STS_TABLE_NAME_LEN, TRUE, - &open_tables_backup, need_lock, &error_num)) - ) { - goto error; - } - if ((error_num = spider_delete_table_sts( - table_sts, - name, - name_length - ))) - goto error; - spider_close_sys_table(thd, table_sts, &open_tables_backup, need_lock); - table_sts = NULL; - DBUG_RETURN(0); - -error: - if (table_sts) - spider_close_sys_table(thd, table_sts, &open_tables_backup, need_lock); - DBUG_RETURN(error_num); -} - -int spider_sys_delete_table_crd( - THD *thd, - const char *name, - uint name_length, - bool need_lock -) { - int error_num; - TABLE *table_crd = NULL; - SPIDER_Open_tables_backup open_tables_backup; - DBUG_ENTER("spider_sys_delete_table_crd"); - if ( - !(table_crd = spider_open_sys_table( - thd, SPIDER_SYS_TABLE_CRD_TABLE_NAME_STR, - SPIDER_SYS_TABLE_CRD_TABLE_NAME_LEN, TRUE, - &open_tables_backup, need_lock, &error_num)) - ) { - goto error; - } - if ((error_num = spider_delete_table_crd( - table_crd, - name, - name_length - ))) - goto error; - spider_close_sys_table(thd, table_crd, &open_tables_backup, need_lock); - table_crd = NULL; - DBUG_RETURN(0); - -error: - if (table_crd) - spider_close_sys_table(thd, table_crd, &open_tables_backup, need_lock); - DBUG_RETURN(error_num); -} - -int spider_sys_get_table_sts( - THD *thd, - const char *name, - uint name_length, - ha_statistics *stat, - bool need_lock -) { - int error_num; - char table_key[MAX_KEY_LENGTH]; - TABLE *table_sts = NULL; - SPIDER_Open_tables_backup open_tables_backup; - DBUG_ENTER("spider_sys_get_table_sts"); - if ( - !(table_sts = spider_open_sys_table( - thd, SPIDER_SYS_TABLE_STS_TABLE_NAME_STR, - SPIDER_SYS_TABLE_STS_TABLE_NAME_LEN, TRUE, - &open_tables_backup, need_lock, &error_num)) - ) { - goto error; - } - - table_sts->use_all_columns(); - spider_store_tables_name(table_sts, name, name_length); - if ((error_num = spider_check_sys_table(table_sts, table_key))) - { - if (error_num != HA_ERR_KEY_NOT_FOUND && error_num != HA_ERR_END_OF_FILE) - { - table_sts->file->print_error(error_num, MYF(0)); - } - goto error; - } else { - spider_get_sys_table_sts_info( - table_sts, - stat - ); - } - - spider_close_sys_table(thd, table_sts, &open_tables_backup, need_lock); - table_sts = NULL; - DBUG_RETURN(0); - -error: - if (table_sts) - spider_close_sys_table(thd, table_sts, &open_tables_backup, need_lock); - DBUG_RETURN(error_num); -} - -int spider_sys_get_table_crd( - THD *thd, - const char *name, - uint name_length, - longlong *cardinality, - uint number_of_keys, - bool need_lock -) { - int error_num; - char table_key[MAX_KEY_LENGTH]; - bool index_inited = FALSE; - TABLE *table_crd = NULL; - SPIDER_Open_tables_backup open_tables_backup; - DBUG_ENTER("spider_sys_get_table_crd"); - - if ( - !(table_crd = spider_open_sys_table( - thd, SPIDER_SYS_TABLE_CRD_TABLE_NAME_STR, - SPIDER_SYS_TABLE_CRD_TABLE_NAME_LEN, TRUE, - &open_tables_backup, need_lock, &error_num)) - ) { - goto error; - } - - table_crd->use_all_columns(); - spider_store_tables_name(table_crd, name, name_length); - if ((error_num = spider_get_sys_table_by_idx(table_crd, table_key, 0, - SPIDER_SYS_TABLE_CRD_PK_COL_CNT - 1))) - { - if (error_num != HA_ERR_KEY_NOT_FOUND && error_num != HA_ERR_END_OF_FILE) - { - table_crd->file->print_error(error_num, MYF(0)); - } - goto error; - } else { - index_inited = TRUE; - do { - spider_get_sys_table_crd_info( - table_crd, - cardinality, - number_of_keys - ); - error_num = spider_sys_index_next_same(table_crd, table_key); - } while (error_num == 0); - } - index_inited = FALSE; - if ((error_num = spider_sys_index_end(table_crd))) - { - table_crd->file->print_error(error_num, MYF(0)); - goto error; - } - - spider_close_sys_table(thd, table_crd, &open_tables_backup, need_lock); - table_crd = NULL; - DBUG_RETURN(0); - -error: - if (index_inited) - spider_sys_index_end(table_crd); - - if (table_crd) - spider_close_sys_table(thd, table_crd, &open_tables_backup, need_lock); - DBUG_RETURN(error_num); -} - int spider_sys_replace( TABLE *table, bool *modified_non_trans_table diff --git a/storage/spider/spd_sys_table.h b/storage/spider/spd_sys_table.h index 36f72375..f1983323 100644 --- a/storage/spider/spd_sys_table.h +++ b/storage/spider/spd_sys_table.h @@ -28,20 +28,6 @@ #define SPIDER_SYS_XA_FAILED_TABLE_NAME_LEN 20 #define SPIDER_SYS_POS_FOR_RECOVERY_TABLE_NAME_STR "spider_table_position_for_recovery" #define SPIDER_SYS_POS_FOR_RECOVERY_TABLE_NAME_LEN 34 -#define SPIDER_SYS_TABLE_STS_TABLE_NAME_STR "spider_table_sts" -#define SPIDER_SYS_TABLE_STS_TABLE_NAME_LEN 16 -#define SPIDER_SYS_TABLE_CRD_TABLE_NAME_STR "spider_table_crd" -#define SPIDER_SYS_TABLE_CRD_TABLE_NAME_LEN 16 -#define SPIDER_SYS_RW_TBLS_TABLE_NAME_STR "spider_rewrite_tables" -#define SPIDER_SYS_RW_TBLS_TABLE_NAME_LEN 21 -#define SPIDER_SYS_RW_TBL_TBLS_TABLE_NAME_STR "spider_rewrite_table_tables" -#define SPIDER_SYS_RW_TBL_TBLS_TABLE_NAME_LEN 27 -#define SPIDER_SYS_RW_TBL_PTTS_TABLE_NAME_STR "spider_rewrite_table_partitions" -#define SPIDER_SYS_RW_TBL_PTTS_TABLE_NAME_LEN 31 -#define SPIDER_SYS_RW_TBL_SPTTS_TABLE_NAME_STR "spider_rewrite_table_subpartitions" -#define SPIDER_SYS_RW_TBL_SPTTS_TABLE_NAME_LEN 34 -#define SPIDER_SYS_RWN_TBLS_TABLE_NAME_STR "spider_rewritten_tables" -#define SPIDER_SYS_RWN_TBLS_TABLE_NAME_LEN 23 #define SPIDER_SYS_XA_PREPARED_STR "PREPARED" #define SPIDER_SYS_XA_NOT_YET_STR "NOT YET" @@ -65,11 +51,6 @@ #define SPIDER_SYS_TABLE_STS_PK_COL_CNT 2 #define SPIDER_SYS_TABLE_CRD_COL_CNT 4 #define SPIDER_SYS_TABLE_CRD_PK_COL_CNT 3 -#define SPIDER_SYS_RW_TBLS_COL_CNT 3 -#define SPIDER_SYS_RW_TBL_TBLS_COL_CNT 8 -#define SPIDER_SYS_RW_TBL_PTTS_COL_CNT 7 -#define SPIDER_SYS_RW_TBL_SPTTS_COL_CNT 8 -#define SPIDER_SYS_RWN_TBLS_COL_CNT 4 #define SPIDER_SYS_LINK_MON_TABLE_DB_NAME_SIZE 64 #define SPIDER_SYS_LINK_MON_TABLE_TABLE_NAME_SIZE 64 @@ -313,21 +294,6 @@ int spider_insert_sys_table( TABLE *table ); -int spider_insert_or_update_table_sts( - TABLE *table, - const char *name, - uint name_length, - ha_statistics *stat -); - -int spider_insert_or_update_table_crd( - TABLE *table, - const char *name, - uint name_length, - longlong *cardinality, - uint number_of_keys -); - int spider_log_tables_link_failed( TABLE *table, char *name, @@ -471,17 +437,6 @@ int spider_get_sys_tables_static_link_id( MEM_ROOT *mem_root ); -void spider_get_sys_table_sts_info( - TABLE *table, - ha_statistics *stat -); - -void spider_get_sys_table_crd_info( - TABLE *table, - longlong *cardinality, - uint number_of_keys -); - int spider_sys_update_tables_link_status( THD *thd, char *name, @@ -533,54 +488,6 @@ int spider_get_link_statuses( MEM_ROOT *mem_root ); -int spider_sys_insert_or_update_table_sts( - THD *thd, - const char *name, - uint name_length, - ha_statistics *stat, - bool need_lock -); - -int spider_sys_insert_or_update_table_crd( - THD *thd, - const char *name, - uint name_length, - longlong *cardinality, - uint number_of_keys, - bool need_lock -); - -int spider_sys_delete_table_sts( - THD *thd, - const char *name, - uint name_length, - bool need_lock -); - -int spider_sys_delete_table_crd( - THD *thd, - const char *name, - uint name_length, - bool need_lock -); - -int spider_sys_get_table_sts( - THD *thd, - const char *name, - uint name_length, - ha_statistics *stat, - bool need_lock -); - -int spider_sys_get_table_crd( - THD *thd, - const char *name, - uint name_length, - longlong *cardinality, - uint number_of_keys, - bool need_lock -); - int spider_sys_replace( TABLE *table, bool *modified_non_trans_table diff --git a/storage/spider/spd_table.cc b/storage/spider/spd_table.cc index bf101858..208b804f 100644 --- a/storage/spider/spd_table.cc +++ b/storage/spider/spd_table.cc @@ -1032,7 +1032,7 @@ int spider_create_string_list( } if (!(*string_list = (char**) - spider_bulk_malloc(spider_current_trx, 37, MYF(MY_WME | MY_ZEROFILL), + spider_bulk_malloc(spider_current_trx, SPD_MID_CREATE_STRING_LIST_1, MYF(MY_WME | MY_ZEROFILL), string_list, (uint) (sizeof(char*) * (*list_length)), string_length_list, (uint) (sizeof(int) * (*list_length)), NullS)) @@ -1226,7 +1226,7 @@ int spider_create_long_list( } if (!(*long_list = (long*) - spider_bulk_malloc(spider_current_trx, 38, MYF(MY_WME | MY_ZEROFILL), + spider_bulk_malloc(spider_current_trx, SPD_MID_CREATE_LONG_LIST_1, MYF(MY_WME | MY_ZEROFILL), long_list, (uint) (sizeof(long) * (*list_length)), NullS)) ) { @@ -1305,7 +1305,7 @@ int spider_create_longlong_list( } if (!(*longlong_list = (longlong *) - spider_bulk_malloc(spider_current_trx, 39, MYF(MY_WME | MY_ZEROFILL), + spider_bulk_malloc(spider_current_trx, SPD_MID_CREATE_LONGLONG_LIST_1, MYF(MY_WME | MY_ZEROFILL), longlong_list, (uint) (sizeof(longlong) * (*list_length)), NullS)) ) { @@ -1373,7 +1373,7 @@ int spider_increase_string_list( } if (!(tmp_str_list = (char**) - spider_bulk_malloc(spider_current_trx, 40, MYF(MY_WME | MY_ZEROFILL), + spider_bulk_malloc(spider_current_trx, SPD_MID_INCREASE_STRING_LIST_1, MYF(MY_WME | MY_ZEROFILL), &tmp_str_list, (uint) (sizeof(char*) * link_count), &tmp_length_list, (uint) (sizeof(uint) * link_count), NullS)) @@ -1436,7 +1436,7 @@ int spider_increase_null_string_list( DBUG_RETURN(0); if (!(tmp_str_list = (char**) - spider_bulk_malloc(spider_current_trx, 247, MYF(MY_WME | MY_ZEROFILL), + spider_bulk_malloc(spider_current_trx, SPD_MID_INCREASE_NULL_STRING_LIST_1, MYF(MY_WME | MY_ZEROFILL), &tmp_str_list, (uint) (sizeof(char*) * link_count), &tmp_length_list, (uint) (sizeof(uint) * link_count), NullS)) @@ -1494,7 +1494,7 @@ int spider_increase_long_list( tmp_long = -1; if (!(tmp_long_list = (long*) - spider_bulk_malloc(spider_current_trx, 41, MYF(MY_WME | MY_ZEROFILL), + spider_bulk_malloc(spider_current_trx, SPD_MID_INCREASE_LONG_LIST_1, MYF(MY_WME | MY_ZEROFILL), &tmp_long_list, (uint) (sizeof(long) * link_count), NullS)) ) { @@ -1539,7 +1539,7 @@ int spider_increase_longlong_list( tmp_longlong = -1; if (!(tmp_longlong_list = (longlong*) - spider_bulk_malloc(spider_current_trx, 42, MYF(MY_WME | MY_ZEROFILL), + spider_bulk_malloc(spider_current_trx, SPD_MID_INCREASE_LONGLONG_LIST_1, MYF(MY_WME | MY_ZEROFILL), &tmp_longlong_list, (uint) (sizeof(longlong) * link_count), NullS)) ) { @@ -2880,7 +2880,7 @@ int spider_parse_connect_info( share_alter = &share->alter_table; share_alter->all_link_count = share->all_link_count; if (!(share_alter->tmp_server_names = (char **) - spider_bulk_malloc(spider_current_trx, 43, MYF(MY_WME | MY_ZEROFILL), + spider_bulk_malloc(spider_current_trx, SPD_MID_PARSE_CONNECT_INFO_1, MYF(MY_WME | MY_ZEROFILL), &share_alter->tmp_server_names, (uint) (sizeof(char *) * share->all_link_count), &share_alter->tmp_tgt_table_names, @@ -4025,7 +4025,7 @@ int spider_create_conn_keys( share->conn_keys_charlen += conn_keys_lengths[roop_count] + 2; } if (!(share->conn_keys = (char **) - spider_bulk_alloc_mem(spider_current_trx, 45, + spider_bulk_alloc_mem(spider_current_trx, SPD_MID_CREATE_CONN_KEYS_1, __func__, __FILE__, __LINE__, MYF(MY_WME | MY_ZEROFILL), &share->conn_keys, sizeof(char *) * share->all_link_count, &share->conn_keys_lengths, length_base, @@ -4218,7 +4218,7 @@ SPIDER_SHARE *spider_create_share( length = (uint) strlen(table_name); bitmap_size = spider_bitmap_size(table_share->fields); if (!(share = (SPIDER_SHARE *) - spider_bulk_malloc(spider_current_trx, 46, MYF(MY_WME | MY_ZEROFILL), + spider_bulk_malloc(spider_current_trx, SPD_MID_CREATE_SHARE_1, MYF(MY_WME | MY_ZEROFILL), &share, (uint) (sizeof(*share)), &tmp_name, (uint) (length + 1), &tmp_static_key_cardinality, @@ -4263,7 +4263,7 @@ SPIDER_SHARE *spider_create_share( goto error_init_hint_string; } for (roop_count = 0; roop_count < (int) table_share->keys; roop_count++) - share->key_hint[roop_count].init_calc_mem(95); + share->key_hint[roop_count].init_calc_mem(SPD_MID_CREATE_SHARE_2); DBUG_PRINT("info",("spider share->key_hint=%p", share->key_hint)); if ((*error_num = spider_parse_connect_info(share, table_share, @@ -4450,7 +4450,7 @@ SPIDER_SHARE *spider_get_share( ((char *) lex_str.str)[lex_str.length] = '\0'; DBUG_PRINT("info",("spider loop check param name=%s", lex_str.str)); loop_check = get_variable(&thd->user_vars, &lex_str, FALSE); - if (loop_check && loop_check->type == STRING_RESULT) + if (loop_check && loop_check->type_handler()->result_type() == STRING_RESULT) { lex_str.length = top_share->path.length + spider_unique_id.length + 1; lex_str.str = loop_check_buf + buf_sz - top_share->path.length - @@ -4685,7 +4685,7 @@ SPIDER_SHARE *spider_get_share( } if (!(spider->conn_keys = (char **) - spider_bulk_alloc_mem(spider_current_trx, 47, + spider_bulk_alloc_mem(spider_current_trx, SPD_MID_GET_SHARE_1, __func__, __FILE__, __LINE__, MYF(MY_WME | MY_ZEROFILL), &spider->conn_keys, sizeof(char *) * share->link_count, &tmp_name, sizeof(char) * share->conn_keys_charlen, @@ -5140,7 +5140,7 @@ SPIDER_SHARE *spider_get_share( } if (!(spider->conn_keys = (char **) - spider_bulk_alloc_mem(spider_current_trx, 49, + spider_bulk_alloc_mem(spider_current_trx, SPD_MID_GET_SHARE_2, __func__, __FILE__, __LINE__, MYF(MY_WME | MY_ZEROFILL), &spider->conn_keys, sizeof(char *) * share->link_count, &tmp_name, sizeof(char) * share->conn_keys_charlen, @@ -5443,8 +5443,6 @@ int spider_free_share( ) { DBUG_ENTER("spider_free_share"); pthread_mutex_lock(&spider_tbl_mutex); - bool do_delete_thd = false; - THD *thd = current_thd; if (!--share->use_count) { spider_free_sts_thread(share); @@ -5460,49 +5458,6 @@ int spider_free_share( spider_table_remove_share_from_crd_thread(share); spider_free_spider_object_for_share(&share->crd_spider); } - if ( - share->sts_init && - share->table_share->tmp_table == NO_TMP_TABLE && - spider_param_store_last_sts(share->store_last_sts) - ) { - if (!thd) - { - /* Create a thread for Spider system table update */ - thd = spider_create_thd(); - if (!thd) - DBUG_RETURN(HA_ERR_OUT_OF_MEM); - do_delete_thd = TRUE; - } - spider_sys_insert_or_update_table_sts( - thd, - share->lgtm_tblhnd_share->table_name, - share->lgtm_tblhnd_share->table_name_length, - &share->stat, - FALSE - ); - } - if ( - share->crd_init && - share->table_share->tmp_table == NO_TMP_TABLE && - spider_param_store_last_crd(share->store_last_crd) - ) { - if (!thd) - { - /* Create a thread for Spider system table update */ - thd = spider_create_thd(); - if (!thd) - DBUG_RETURN(HA_ERR_OUT_OF_MEM); - do_delete_thd = TRUE; - } - spider_sys_insert_or_update_table_crd( - thd, - share->lgtm_tblhnd_share->table_name, - share->lgtm_tblhnd_share->table_name_length, - share->cardinality, - share->table_share->fields, - FALSE - ); - } spider_free_share_alloc(share); my_hash_delete(&spider_open_tables, (uchar*) share); pthread_mutex_destroy(&share->crd_mutex); @@ -5511,8 +5466,6 @@ int spider_free_share( free_root(&share->mem_root, MYF(0)); spider_free(spider_current_trx, share, MYF(0)); } - if (do_delete_thd) - spider_destroy_thd(thd); pthread_mutex_unlock(&spider_tbl_mutex); DBUG_RETURN(0); } @@ -5568,7 +5521,7 @@ SPIDER_LGTM_TBLHND_SHARE *spider_get_lgtm_tblhnd_share( { DBUG_PRINT("info",("spider create new lgtm tblhnd share")); if (!(lgtm_tblhnd_share = (SPIDER_LGTM_TBLHND_SHARE *) - spider_bulk_malloc(spider_current_trx, 244, MYF(MY_WME | MY_ZEROFILL), + spider_bulk_malloc(spider_current_trx, SPD_MID_GET_LGTM_TBLHND_SHARE_1, MYF(MY_WME | MY_ZEROFILL), &lgtm_tblhnd_share, (uint) (sizeof(*lgtm_tblhnd_share)), &tmp_name, (uint) (table_name_length + 1), NullS)) @@ -5654,7 +5607,7 @@ SPIDER_WIDE_SHARE *spider_get_wide_share( { DBUG_PRINT("info",("spider create new wide share")); if (!(wide_share = (SPIDER_WIDE_SHARE *) - spider_bulk_malloc(spider_current_trx, 51, MYF(MY_WME | MY_ZEROFILL), + spider_bulk_malloc(spider_current_trx, SPD_MID_GET_PT_SHARE_1, MYF(MY_WME | MY_ZEROFILL), &wide_share, sizeof(SPIDER_WIDE_SHARE), &tmp_name, (uint) (table_share->path.length + 1), &tmp_cardinality, @@ -5946,7 +5899,7 @@ int spider_open_all_tables( spider->wide_handler->lock_type = TL_READ_NO_INSERT; if (!(share = (SPIDER_SHARE *) - spider_bulk_malloc(spider_current_trx, 52, MYF(MY_WME | MY_ZEROFILL), + spider_bulk_malloc(spider_current_trx, SPD_MID_OPEN_ALL_TABLES_1, MYF(MY_WME | MY_ZEROFILL), &share, (uint) (sizeof(*share)), &connect_info, (uint) (sizeof(char *) * SPIDER_TMP_SHARE_CHAR_PTR_COUNT), @@ -6162,26 +6115,13 @@ int spider_db_done( void *p ) { int roop_count; - bool do_delete_thd; - THD *thd = current_thd, *tmp_thd; + THD *tmp_thd; SPIDER_CONN *conn; SPIDER_INIT_ERROR_TABLE *spider_init_error_table; SPIDER_TABLE_MON_LIST *table_mon_list; SPIDER_LGTM_TBLHND_SHARE *lgtm_tblhnd_share; DBUG_ENTER("spider_db_done"); - /* Begin Spider plugin deinit */ - if (thd) - do_delete_thd = FALSE; - else - { - /* Create a thread for Spider plugin deinit */ - thd = spider_create_thd(); - if (!thd) - DBUG_RETURN(HA_ERR_OUT_OF_MEM); - do_delete_thd = TRUE; - } - for (roop_count = SPIDER_DBTON_SIZE - 1; roop_count >= 0; roop_count--) { if (spider_dbton[roop_count].deinit) @@ -6331,13 +6271,6 @@ int spider_db_done( )); } - /* End Spider plugin deinit */ - if (do_delete_thd) - spider_destroy_thd(thd); - -/* -DBUG_ASSERT(0); -*/ DBUG_RETURN(0); } @@ -6393,6 +6326,17 @@ bool spider_init_system_tables() DBUG_RETURN(FALSE); } + +/* + Spider is typically loaded before ddl_recovery, but DDL statements + cannot be executed before ddl_recovery, so we delay system table creation. +*/ +static int spider_after_ddl_recovery(handlerton *) +{ + DBUG_EXECUTE_IF("fail_spider_ddl_recovery_done", return 1;); + return spider_init_system_tables(); +} + int spider_db_init( void *p ) { @@ -6412,17 +6356,8 @@ int spider_db_init( #ifdef HTON_CAN_READ_CONNECT_STRING_IN_PARTITION spider_hton->flags |= HTON_CAN_READ_CONNECT_STRING_IN_PARTITION; #endif - /* spider_hton->db_type = DB_TYPE_SPIDER; */ - /* - spider_hton->savepoint_offset; - spider_hton->savepoint_set = spider_savepoint_set; - spider_hton->savepoint_rollback = spider_savepoint_rollback; - spider_hton->savepoint_release = spider_savepoint_release; - spider_hton->create_cursor_read_view = spider_create_cursor_read_view; - spider_hton->set_cursor_read_view = spider_set_cursor_read_view; - spider_hton->close_cursor_read_view = spider_close_cursor_read_view; - */ spider_hton->panic = spider_panic; + spider_hton->signal_ddl_recovery_done= spider_after_ddl_recovery; spider_hton->close_connection = spider_close_connection; spider_hton->start_consistent_snapshot = spider_start_consistent_snapshot; spider_hton->flush_logs = spider_flush_logs; @@ -6478,10 +6413,6 @@ int spider_db_init( if (pthread_attr_init(&spider_pt_attr)) goto error_pt_attr_init; -/* - if (pthread_attr_setdetachstate(&spider_pt_attr, PTHREAD_CREATE_DETACHED)) - goto error_pt_attr_setstate; -*/ if (mysql_mutex_init(spd_key_mutex_tbl, &spider_tbl_mutex, MY_MUTEX_INIT_FAST)) @@ -6535,7 +6466,7 @@ int spider_db_init( (my_hash_get_key) spider_tbl_get_key, 0, 0)) goto error_open_tables_hash_init; - spider_alloc_calc_mem_init(spider_open_tables, 143); + spider_alloc_calc_mem_init(spider_open_tables, SPD_MID_DB_INIT_1); spider_alloc_calc_mem(NULL, spider_open_tables, spider_open_tables.array.max_element * @@ -6544,7 +6475,7 @@ int spider_db_init( (my_hash_get_key) spider_tbl_get_key, 0, 0)) goto error_init_error_tables_hash_init; - spider_alloc_calc_mem_init(spider_init_error_tables, 144); + spider_alloc_calc_mem_init(spider_init_error_tables, SPD_MID_DB_INIT_2); spider_alloc_calc_mem(NULL, spider_init_error_tables, spider_init_error_tables.array.max_element * @@ -6555,7 +6486,7 @@ int spider_db_init( ) goto error_open_wide_share_hash_init; - spider_alloc_calc_mem_init(spider_open_wide_share, 145); + spider_alloc_calc_mem_init(spider_open_wide_share, SPD_MID_DB_INIT_3); spider_alloc_calc_mem(NULL, spider_open_wide_share, spider_open_wide_share.array.max_element * @@ -6565,7 +6496,7 @@ int spider_db_init( (my_hash_get_key) spider_lgtm_tblhnd_share_hash_get_key, 0, 0)) goto error_lgtm_tblhnd_share_hash_init; - spider_alloc_calc_mem_init(spider_lgtm_tblhnd_share_hash, 245); + spider_alloc_calc_mem_init(spider_lgtm_tblhnd_share_hash, SPD_MID_DB_INIT_4); spider_alloc_calc_mem(NULL, spider_lgtm_tblhnd_share_hash, spider_lgtm_tblhnd_share_hash.array.max_element * @@ -6579,7 +6510,7 @@ int spider_db_init( spider_free_ipport_conn, 0)) goto error_ipport_conn__hash_init; - spider_alloc_calc_mem_init(spider_open_connections, 146); + spider_alloc_calc_mem_init(spider_open_connections, SPD_MID_DB_INIT_5); spider_alloc_calc_mem(NULL, spider_open_connections, spider_open_connections.array.max_element * @@ -6588,7 +6519,7 @@ int spider_db_init( (my_hash_get_key) spider_allocated_thds_get_key, 0, 0)) goto error_allocated_thds_hash_init; - spider_alloc_calc_mem_init(spider_allocated_thds, 149); + spider_alloc_calc_mem_init(spider_allocated_thds, SPD_MID_DB_INIT_8); spider_alloc_calc_mem(NULL, spider_allocated_thds, spider_allocated_thds.array.max_element * @@ -6598,14 +6529,14 @@ int spider_db_init( NULL, 64, 64, MYF(MY_WME))) goto error_mon_table_cache_array_init; - spider_alloc_calc_mem_init(spider_mon_table_cache, 165); + spider_alloc_calc_mem_init(spider_mon_table_cache, SPD_MID_DB_INIT_9); spider_alloc_calc_mem(NULL, spider_mon_table_cache, spider_mon_table_cache.max_element * spider_mon_table_cache.size_of_element); if (!(spider_udf_table_mon_mutexes = (pthread_mutex_t *) - spider_bulk_malloc(NULL, 53, MYF(MY_WME | MY_ZEROFILL), + spider_bulk_malloc(NULL, SPD_MID_DB_INIT_10, MYF(MY_WME | MY_ZEROFILL), &spider_udf_table_mon_mutexes, (uint) (sizeof(pthread_mutex_t) * spider_udf_table_mon_mutex_count), &spider_udf_table_mon_conds, (uint) (sizeof(pthread_cond_t) * @@ -6641,7 +6572,7 @@ int spider_db_init( (my_hash_get_key) spider_udf_tbl_mon_list_key, 0, 0)) goto error_init_udf_table_mon_list_hash; - spider_alloc_calc_mem_init(spider_udf_table_mon_list_hash, 150); + spider_alloc_calc_mem_init(spider_udf_table_mon_list_hash, SPD_MID_DB_INIT_11); spider_alloc_calc_mem(NULL, spider_udf_table_mon_list_hash, spider_udf_table_mon_list_hash[roop_count].array.max_element * @@ -6654,7 +6585,7 @@ int spider_db_init( } if (!(spider_table_sts_threads = (SPIDER_THREAD *) - spider_bulk_malloc(NULL, 256, MYF(MY_WME | MY_ZEROFILL), + spider_bulk_malloc(NULL, SPD_MID_DB_INIT_12, MYF(MY_WME | MY_ZEROFILL), &spider_table_sts_threads, (uint) (sizeof(SPIDER_THREAD) * spider_param_table_sts_thread_count()), &spider_table_crd_threads, (uint) (sizeof(SPIDER_THREAD) * @@ -6826,7 +6757,7 @@ char *spider_create_string( ) { char *res; DBUG_ENTER("spider_create_string"); - if (!(res = (char*) spider_malloc(spider_current_trx, 13, length + 1, + if (!(res = (char*) spider_malloc(spider_current_trx, SPD_MID_CREATE_STRING_1, length + 1, MYF(MY_WME)))) DBUG_RETURN(NULL); memcpy(res, str, length); @@ -6848,7 +6779,7 @@ char *spider_create_table_name_string( if (sub_name) length += sizeof("#SP#") - 1 + strlen(sub_name); } - if (!(res = (char*) spider_malloc(spider_current_trx, 14, length + 1, + if (!(res = (char*) spider_malloc(spider_current_trx, SPD_MID_CREATE_TABLE_NAME_STRING_1, length + 1, MYF(MY_WME)))) DBUG_RETURN(NULL); tmp = strmov(res, table_name); @@ -6968,7 +6899,6 @@ int spider_get_sts( ) { int get_type; int error_num = 0; - bool need_to_get = TRUE; DBUG_ENTER("spider_get_sts"); if ( @@ -7004,34 +6934,10 @@ int spider_get_sts( /* copy */ get_type = 0; } - if ( - !share->sts_init && - share->table_share->tmp_table == NO_TMP_TABLE && - spider_param_load_sts_at_startup(share->load_sts_at_startup) && - (!share->init || share->init_error) - ) { - error_num = spider_sys_get_table_sts( - current_thd, - share->lgtm_tblhnd_share->table_name, - share->lgtm_tblhnd_share->table_name_length, - &share->stat, - FALSE - ); - if ( - !error_num || - (error_num != HA_ERR_KEY_NOT_FOUND && error_num != HA_ERR_END_OF_FILE) - ) - need_to_get = FALSE; - } - - if (need_to_get) - { - if (get_type == 0) - spider_copy_sts_to_share(share, share->wide_share); - else { - error_num = spider_db_show_table_status(spider, link_idx, sts_mode, flag); - } - } + if (get_type == 0) + spider_copy_sts_to_share(share, share->wide_share); + else + error_num = spider_db_show_table_status(spider, link_idx, sts_mode, flag); if (get_type >= 2) pthread_mutex_unlock(&share->wide_share->sts_mutex); if (error_num) @@ -7103,7 +7009,6 @@ int spider_get_crd( ) { int get_type; int error_num = 0; - bool need_to_get = TRUE; DBUG_ENTER("spider_get_crd"); if ( @@ -7139,35 +7044,6 @@ int spider_get_crd( /* copy */ get_type = 0; } - if ( - !share->crd_init && - share->table_share->tmp_table == NO_TMP_TABLE && - spider_param_load_sts_at_startup(share->load_crd_at_startup) - ) { - error_num = spider_sys_get_table_crd( - current_thd, - share->lgtm_tblhnd_share->table_name, - share->lgtm_tblhnd_share->table_name_length, - share->cardinality, - table->s->fields, - FALSE - ); - if ( - !error_num || - (error_num != HA_ERR_KEY_NOT_FOUND && error_num != HA_ERR_END_OF_FILE) - ) - need_to_get = FALSE; - } - - if (need_to_get) - { - if (get_type == 0) - spider_copy_crd_to_share(share, share->wide_share, - table->s->fields); - else { - error_num = spider_db_show_index(spider, link_idx, table, crd_mode); - } - } if (get_type >= 2) pthread_mutex_unlock(&share->wide_share->crd_mutex); if (error_num) @@ -7284,7 +7160,7 @@ SPIDER_INIT_ERROR_TABLE *spider_get_init_error_table( DBUG_RETURN(NULL); } if (!(spider_init_error_table = (SPIDER_INIT_ERROR_TABLE *) - spider_bulk_malloc(spider_current_trx, 54, MYF(MY_WME | MY_ZEROFILL), + spider_bulk_malloc(spider_current_trx, SPD_MID_GET_INIT_ERROR_TABLE_1, MYF(MY_WME | MY_ZEROFILL), &spider_init_error_table, (uint) (sizeof(*spider_init_error_table)), &tmp_name, (uint) (share->table_name_length + 1), NullS)) @@ -8345,7 +8221,7 @@ int spider_discover_table_structure( char buf[MAX_FIELD_WIDTH]; spider_string str(buf, sizeof(buf), system_charset_info); DBUG_ENTER("spider_discover_table_structure"); - str.init_calc_mem(229); + str.init_calc_mem(SPD_MID_DISCOVER_TABLE_STRUCTURE_1); str.length(0); if (str.reserve( SPIDER_SQL_CREATE_TABLE_LEN + share->db.length + @@ -8676,7 +8552,7 @@ int spider_create_spider_object_for_share( } DBUG_PRINT("info",("spider spider=%p", (*spider))); if (!(need_mons = (int *) - spider_bulk_malloc(spider_current_trx, 255, MYF(MY_WME | MY_ZEROFILL), + spider_bulk_malloc(spider_current_trx, SPD_MID_CREATE_SPIDER_OBJECT_FOR_SHARE_2, MYF(MY_WME | MY_ZEROFILL), &need_mons, (uint) (sizeof(int) * share->link_count), &conns, (uint) (sizeof(SPIDER_CONN *) * share->link_count), &conn_link_idx, (uint) (sizeof(uint) * share->link_count), diff --git a/storage/spider/spd_trx.cc b/storage/spider/spd_trx.cc index c689e40d..ebf9a777 100644 --- a/storage/spider/spd_trx.cc +++ b/storage/spider/spd_trx.cc @@ -166,7 +166,7 @@ int spider_trx_another_lock_tables( spider_string sql_str(sql_buf, sizeof(sql_buf), system_charset_info); DBUG_ENTER("spider_trx_another_lock_tables"); SPIDER_BACKUP_DASTATUS; - sql_str.init_calc_mem(188); + sql_str.init_calc_mem(SPD_MID_TRX_ANOTHER_LOCK_TABLES_1); sql_str.length(0); memset((void*)&tmp_spider, 0, sizeof(ha_spider)); memset((void*)&tmp_share, 0, sizeof(SPIDER_SHARE)); @@ -482,7 +482,7 @@ int spider_create_trx_alter_table( share_alter = &share->alter_table; if (!(alter_table = (SPIDER_ALTER_TABLE *) - spider_bulk_malloc(spider_current_trx, 55, MYF(MY_WME | MY_ZEROFILL), + spider_bulk_malloc(spider_current_trx, SPD_MID_CREATE_TRX_ALTER_TABLE_1, MYF(MY_WME | MY_ZEROFILL), &alter_table, (uint) (sizeof(*alter_table)), &tmp_name, (uint) (sizeof(char) * (share->table_name_length + 1)), @@ -1142,7 +1142,7 @@ SPIDER_TRX *spider_get_trx( ) { DBUG_PRINT("info",("spider create new trx")); if (!(trx = (SPIDER_TRX *) - spider_bulk_malloc(NULL, 56, MYF(MY_WME | MY_ZEROFILL), + spider_bulk_malloc(NULL, SPD_MID_GET_TRX_1, MYF(MY_WME | MY_ZEROFILL), &trx, (uint) (sizeof(*trx)), &tmp_share, (uint) (sizeof(SPIDER_SHARE)), &tmp_wide_handler, (uint) sizeof(SPIDER_WIDE_HANDLER), @@ -1171,7 +1171,7 @@ SPIDER_TRX *spider_get_trx( spider_conn_get_key, 0, 0) ) goto error_init_hash; - spider_alloc_calc_mem_init(trx->trx_conn_hash, 151); + spider_alloc_calc_mem_init(trx->trx_conn_hash, SPD_MID_GET_TRX_2); spider_alloc_calc_mem( thd ? ((SPIDER_TRX *) thd_get_ha_data(thd, spider_hton_ptr)) : NULL, trx->trx_conn_hash, @@ -1184,7 +1184,7 @@ SPIDER_TRX *spider_get_trx( spider_conn_get_key, 0, 0) ) goto error_init_another_hash; - spider_alloc_calc_mem_init(trx->trx_another_conn_hash, 152); + spider_alloc_calc_mem_init(trx->trx_another_conn_hash, SPD_MID_GET_TRX_3); spider_alloc_calc_mem( thd ? ((SPIDER_TRX *) thd_get_ha_data(thd, spider_hton_ptr)) : NULL, trx->trx_another_conn_hash, @@ -1197,7 +1197,7 @@ SPIDER_TRX *spider_get_trx( spider_alter_tbl_get_key, 0, 0) ) goto error_init_alter_hash; - spider_alloc_calc_mem_init(trx->trx_alter_table_hash, 157); + spider_alloc_calc_mem_init(trx->trx_alter_table_hash, SPD_MID_GET_TRX_8); spider_alloc_calc_mem( thd ? ((SPIDER_TRX *) thd_get_ha_data(thd, spider_hton_ptr)) : NULL, trx->trx_alter_table_hash, @@ -1210,7 +1210,7 @@ SPIDER_TRX *spider_get_trx( spider_trx_ha_get_key, 0, 0) ) goto error_init_trx_ha_hash; - spider_alloc_calc_mem_init(trx->trx_ha_hash, 158); + spider_alloc_calc_mem_init(trx->trx_ha_hash, SPD_MID_GET_TRX_9); spider_alloc_calc_mem( thd ? ((SPIDER_TRX *) thd_get_ha_data(thd, spider_hton_ptr)) : NULL, trx->trx_ha_hash, @@ -1262,7 +1262,7 @@ SPIDER_TRX *spider_get_trx( for (roop_count2 = 0; roop_count2 < (int) trx->tmp_share->link_count; ++roop_count2) { - trx->tmp_spider->result_list.sqls[roop_count2].init_calc_mem(121); + trx->tmp_spider->result_list.sqls[roop_count2].init_calc_mem(SPD_MID_GET_TRX_10); trx->tmp_spider->result_list.sqls[roop_count2].set_charset( trx->tmp_share->access_charset); } @@ -3164,6 +3164,14 @@ int spider_rollback( DBUG_RETURN(0); /* transaction is not started */ + /* In case the rollback happens due to failure of LOCK TABLE, we + need to clear the list of tables to lock. */ + for (uint i= 0; i < trx->trx_conn_hash.records; i++) + { + conn= (SPIDER_CONN *) my_hash_element(&trx->trx_conn_hash, i); + conn->db_conn->reset_lock_table_hash(); + } + if (all || (!thd_test_options(thd, OPTION_NOT_AUTOCOMMIT | OPTION_BEGIN))) { if (trx->trx_start) @@ -3695,7 +3703,7 @@ int spider_create_trx_ha( if (need_create) { if (!(trx_ha = (SPIDER_TRX_HA *) - spider_bulk_malloc(spider_current_trx, 58, MYF(MY_WME), + spider_bulk_malloc(spider_current_trx, SPD_MID_CREATE_TRX_HA_1, MYF(MY_WME), &trx_ha, (uint) (sizeof(SPIDER_TRX_HA)), &tmp_name, (uint) (sizeof(char *) * (share->table_name_length + 1)), &conn_link_idx, (uint) (sizeof(uint) * share->link_count), |